From deb763b70bc9c93a1e41977d3e64dbe0f2c40b01 Mon Sep 17 00:00:00 2001
From: root <songlf>
Date: Fri, 6 Feb 2026 14:07:36 +0800
Subject: [PATCH] clone code from github

---
 CITATION.bib                                  |    7 +
 CODE_OF_CONDUCT.md                            |    7 +
 LICENSE                                       |   20 +
 MANIFEST.in                                   |   18 +
 codecov.yml                                   |   13 +
 cupy/__init__.py                              |  920 +++
 cupy/_binary/__init__.py                      |    2 +
 cupy/_binary/elementwise.py                   |   22 +
 cupy/_binary/packing.py                       |  104 +
 cupy/_core/__init__.pxd                       |    0
 cupy/_core/__init__.py                        |   79 +
 cupy/_core/_accelerator.pxd                   |   10 +
 cupy/_core/_accelerator.pyx                   |   59 +
 cupy/_core/_carray.pxd                        |   55 +
 cupy/_core/_carray.pyx                        |   57 +
 cupy/_core/_codeblock.py                      |   38 +
 cupy/_core/_cub_reduction.pxd                 |   12 +
 cupy/_core/_cub_reduction.pyx                 |  712 +++
 cupy/_core/_dtype.pxd                         |   10 +
 cupy/_core/_dtype.pyx                         |  126 +
 cupy/_core/_fusion_interface.py               |  272 +
 cupy/_core/_fusion_kernel.pyx                 |  364 ++
 cupy/_core/_fusion_op.py                      |  316 +
 cupy/_core/_fusion_optimization.py            |   90 +
 cupy/_core/_fusion_thread_local.pyx           |   46 +
 cupy/_core/_fusion_trace.pyx                  |  616 ++
 cupy/_core/_fusion_variable.pxd               |    5 +
 cupy/_core/_fusion_variable.pyx               |  340 ++
 cupy/_core/_gufuncs.py                        |  729 +++
 cupy/_core/_kernel.pxd                        |  170 +
 cupy/_core/_kernel.pyx                        | 1622 ++++++
 cupy/_core/_memory_range.pxd                  |    7 +
 cupy/_core/_memory_range.pyx                  |   40 +
 cupy/_core/_optimize_config.pxd               |   22 +
 cupy/_core/_optimize_config.pyx               |   81 +
 cupy/_core/_reduction.pxd                     |   77 +
 cupy/_core/_reduction.pyx                     |  906 +++
 cupy/_core/_routines_binary.pxd               |    6 +
 cupy/_core/_routines_binary.pyx               |   96 +
 cupy/_core/_routines_indexing.pxd             |   15 +
 cupy/_core/_routines_indexing.pyx             | 1161 ++++
 cupy/_core/_routines_linalg.pxd               |   27 +
 cupy/_core/_routines_linalg.pyx               | 1067 ++++
 cupy/_core/_routines_logic.pxd                |   11 +
 cupy/_core/_routines_logic.pyx                |  141 +
 cupy/_core/_routines_manipulation.pxd         |   40 +
 cupy/_core/_routines_manipulation.pyx         |  885 +++
 cupy/_core/_routines_math.pxd                 |   39 +
 cupy/_core/_routines_math.pyx                 | 1145 ++++
 cupy/_core/_routines_sorting.pxd              |    7 +
 cupy/_core/_routines_sorting.pyx              |  534 ++
 cupy/_core/_routines_statistics.pxd           |   30 +
 cupy/_core/_routines_statistics.pyx           |  763 +++
 cupy/_core/_scalar.pxd                        |   37 +
 cupy/_core/_scalar.pyx                        |  386 ++
 cupy/_core/_ufuncs.py                         |    9 +
 cupy/_core/core.pxd                           |  115 +
 cupy/_core/core.pyx                           | 2813 +++++++++
 cupy/_core/dlpack.pxd                         |   12 +
 cupy/_core/dlpack.pyx                         |  412 ++
 cupy/_core/flags.pyx                          |   34 +
 cupy/_core/fusion.pyx                         | 1004 ++++
 cupy/_core/halffloat.h                        |  125 +
 cupy/_core/include/cupy/README.md             |   22 +
 cupy/_core/include/cupy/_cuda/README.md       |    9 +
 .../include/cupy/_cuda/cuda-10.2/cuda_fp16.h  | 3052 ++++++++++
 .../cupy/_cuda/cuda-10.2/cuda_fp16.hpp        | 2071 +++++++
 .../include/cupy/_cuda/cuda-11.0/cuda_fp16.h  | 3612 ++++++++++++
 .../cupy/_cuda/cuda-11.0/cuda_fp16.hpp        | 2285 ++++++++
 .../include/cupy/_cuda/cuda-11.1/cuda_fp16.h  | 3631 ++++++++++++
 .../cupy/_cuda/cuda-11.1/cuda_fp16.hpp        | 2453 ++++++++
 .../include/cupy/_cuda/cuda-11/cuda_fp16.h    | 3794 ++++++++++++
 .../include/cupy/_cuda/cuda-11/cuda_fp16.hpp  | 2614 +++++++++
 .../include/cupy/_cuda/cuda-12/cuda_fp16.h    | 4023 +++++++++++++
 .../include/cupy/_cuda/cuda-12/cuda_fp16.hpp  | 2738 +++++++++
 cupy/_core/include/cupy/atomics.cuh           |  114 +
 cupy/_core/include/cupy/carray.cuh            |  849 +++
 cupy/_core/include/cupy/complex.cuh           |  100 +
 cupy/_core/include/cupy/complex/README.md     |    3 +
 cupy/_core/include/cupy/complex/arithmetic.h  |  314 +
 cupy/_core/include/cupy/complex/catrig.h      |  730 +++
 cupy/_core/include/cupy/complex/catrigf.h     |  444 ++
 cupy/_core/include/cupy/complex/ccosh.h       |  205 +
 cupy/_core/include/cupy/complex/ccoshf.h      |  135 +
 cupy/_core/include/cupy/complex/cexp.h        |  173 +
 cupy/_core/include/cupy/complex/cexpf.h       |  153 +
 cupy/_core/include/cupy/complex/clog.h        |  203 +
 cupy/_core/include/cupy/complex/clogf.h       |  192 +
 cupy/_core/include/cupy/complex/complex.h     |  674 +++
 cupy/_core/include/cupy/complex/complex_inl.h |  164 +
 cupy/_core/include/cupy/complex/cpow.h        |   44 +
 cupy/_core/include/cupy/complex/cproj.h       |   64 +
 cupy/_core/include/cupy/complex/csinh.h       |  192 +
 cupy/_core/include/cupy/complex/csinhf.h      |  133 +
 cupy/_core/include/cupy/complex/csqrt.h       |  144 +
 cupy/_core/include/cupy/complex/csqrtf.h      |  141 +
 cupy/_core/include/cupy/complex/ctanh.h       |  191 +
 cupy/_core/include/cupy/complex/ctanhf.h      |  116 +
 .../_core/include/cupy/complex/math_private.h |  192 +
 cupy/_core/include/cupy/cuComplex_bridge.h    |   34 +
 cupy/_core/include/cupy/cub/.gitattributes    |    1 +
 cupy/_core/include/cupy/cub/LICENSE.TXT       |    1 +
 cupy/_core/include/cupy/cub/cub               |    1 +
 cupy/_core/include/cupy/cuda_workaround.h     |   13 +
 cupy/_core/include/cupy/dlpack/README.md      |    4 +
 cupy/_core/include/cupy/dlpack/dlpack.h       |  232 +
 cupy/_core/include/cupy/hip_workaround.cuh    |   20 +
 cupy/_core/include/cupy/jitify/.clang-format  |  149 +
 cupy/_core/include/cupy/jitify/.gitignore     |   10 +
 cupy/_core/include/cupy/jitify/Doxyfile       | 2427 ++++++++
 cupy/_core/include/cupy/jitify/LICENSE        |   29 +
 cupy/_core/include/cupy/jitify/Makefile       |  105 +
 cupy/_core/include/cupy/jitify/README.md      |  116 +
 .../example_headers/class_arg_kernel.cuh      |   69 +
 .../example_headers/constant_header.cuh       |   43 +
 .../jitify/example_headers/my_header1.cuh     |   34 +
 .../jitify/example_headers/my_header2.cuh     |   34 +
 .../jitify/example_headers/my_header3.cuh     |   34 +
 cupy/_core/include/cupy/jitify/jitify.hpp     | 4454 ++++++++++++++
 .../include/cupy/jitify/jitify_example.cpp    |  359 ++
 cupy/_core/include/cupy/jitify/jitify_test.cu | 1081 ++++
 cupy/_core/include/cupy/jitify/stringify.cpp  |   86 +
 cupy/_core/include/cupy/math_constants.h      |   20 +
 cupy/_core/include/cupy/pair.cuh              |  279 +
 cupy/_core/include/cupy/swap.cuh              |   37 +
 cupy/_core/include/cupy/tuple.cuh             |  568 ++
 cupy/_core/include/cupy/tuple/pair.h          |  228 +
 cupy/_core/include/cupy/tuple/tuple.h         |  960 +++
 cupy/_core/include/cupy/tuple/type_traits.h   |   70 +
 cupy/_core/include/cupy/type_dispatcher.cuh   |   63 +
 cupy/_core/internal.pxd                       |   66 +
 cupy/_core/internal.pyx                       |  536 ++
 cupy/_core/new_fusion.pyx                     |  176 +
 cupy/_core/raw.pxd                            |   30 +
 cupy/_core/raw.pyx                            |  545 ++
 cupy/_core/syncdetect.py                      |   68 +
 cupy/_creation/__init__.py                    |    2 +
 cupy/_creation/basic.py                       |  314 +
 cupy/_creation/from_data.py                   |  226 +
 cupy/_creation/matrix.py                      |  178 +
 cupy/_creation/ranges.py                      |  442 ++
 cupy/_environment.py                          |  529 ++
 cupy/_functional/__init__.py                  |    0
 cupy/_functional/piecewise.py                 |   61 +
 cupy/_functional/vectorize.py                 |  104 +
 cupy/_indexing/__init__.py                    |    2 +
 cupy/_indexing/generate.py                    |  588 ++
 cupy/_indexing/indexing.py                    |  223 +
 cupy/_indexing/insert.py                      |  260 +
 cupy/_indexing/iterate.py                     |  155 +
 cupy/_io/__init__.py                          |    2 +
 cupy/_io/formatting.py                        |   75 +
 cupy/_io/npz.py                               |  146 +
 cupy/_io/text.py                              |   13 +
 cupy/_logic/__init__.py                       |    2 +
 cupy/_logic/comparison.py                     |  174 +
 cupy/_logic/content.py                        |  131 +
 cupy/_logic/ops.py                            |   43 +
 cupy/_logic/truth.py                          |  316 +
 cupy/_logic/type_testing.py                   |  183 +
 cupy/_manipulation/__init__.py                |    2 +
 cupy/_manipulation/add_remove.py              |  213 +
 cupy/_manipulation/basic.py                   |  114 +
 cupy/_manipulation/dims.py                    |  167 +
 cupy/_manipulation/join.py                    |  135 +
 cupy/_manipulation/kind.py                    |  122 +
 cupy/_manipulation/rearrange.py               |  200 +
 cupy/_manipulation/shape.py                   |   89 +
 cupy/_manipulation/split.py                   |   91 +
 cupy/_manipulation/tiling.py                  |   70 +
 cupy/_manipulation/transpose.py               |   82 +
 cupy/_math/__init__.py                        |    2 +
 cupy/_math/arithmetic.py                      |  163 +
 cupy/_math/explog.py                          |  103 +
 cupy/_math/floating.py                        |   62 +
 cupy/_math/hyperbolic.py                      |   55 +
 cupy/_math/misc.py                            |  569 ++
 cupy/_math/rational.py                        |   62 +
 cupy/_math/rounding.py                        |   77 +
 cupy/_math/special.py                         |   26 +
 cupy/_math/sumprod.py                         |  626 ++
 cupy/_math/trigonometric.py                   |  163 +
 cupy/_math/ufunc.py                           |   17 +
 cupy/_math/window.py                          |  197 +
 cupy/_misc/__init__.py                        |    0
 cupy/_misc/byte_bounds.py                     |   23 +
 cupy/_misc/memory_ranges.py                   |   44 +
 cupy/_misc/who.py                             |  106 +
 cupy/_padding/__init__.py                     |    2 +
 cupy/_padding/pad.py                          |  752 +++
 cupy/_sorting/__init__.py                     |    2 +
 cupy/_sorting/count.py                        |   31 +
 cupy/_sorting/search.py                       |  466 ++
 cupy/_sorting/sort.py                         |  217 +
 cupy/_statistics/__init__.py                  |    2 +
 cupy/_statistics/correlation.py               |  212 +
 cupy/_statistics/histogram.py                 |  565 ++
 cupy/_statistics/meanvar.py                   |  288 +
 cupy/_statistics/order.py                     |  401 ++
 cupy/_util.pyi                                |    0
 cupy/_util.pyx                                |  218 +
 cupy/_version.py                              |    1 +
 cupy/array_api/__init__.py                    |  388 ++
 cupy/array_api/_array_object.py               | 1138 ++++
 cupy/array_api/_constants.py                  |    6 +
 cupy/array_api/_creation_functions.py         |  448 ++
 cupy/array_api/_data_type_functions.py        |  150 +
 cupy/array_api/_dtypes.py                     |  143 +
 cupy/array_api/_elementwise_functions.py      |  729 +++
 cupy/array_api/_indexing_functions.py         |   17 +
 cupy/array_api/_manipulation_functions.py     |   98 +
 cupy/array_api/_searching_functions.py        |   47 +
 cupy/array_api/_set_functions.py              |  106 +
 cupy/array_api/_sorting_functions.py          |   53 +
 cupy/array_api/_statistical_functions.py      |  115 +
 cupy/array_api/_typing.py                     |   75 +
 cupy/array_api/_utility_functions.py          |   37 +
 cupy/array_api/linalg.py                      |  445 ++
 cupy/cublas.py                                | 1012 ++++
 cupy/cuda/__init__.pxd                        |    0
 cupy/cuda/__init__.py                         |  184 +
 cupy/cuda/common.pxd                          |   22 +
 cupy/cuda/common.pyx                          |   60 +
 cupy/cuda/compiler.py                         |  943 +++
 cupy/cuda/cub.pxd                             |   20 +
 cupy/cuda/cub.pyx                             |  585 ++
 cupy/cuda/cudnn.py                            |   17 +
 cupy/cuda/cufft.pxd                           |   99 +
 cupy/cuda/cufft.pyx                           | 1205 ++++
 cupy/cuda/cupy_cub.cu                         | 1030 ++++
 cupy/cuda/cupy_cub.h                          |   97 +
 cupy/cuda/cupy_cufft.h                        |  324 ++
 cupy/cuda/cupy_cufftXt.cu                     |   68 +
 cupy/cuda/cupy_cufftXt.h                      |   10 +
 cupy/cuda/cupy_jitify.h                       |   41 +
 cupy/cuda/cupy_thrust.cu                      |  526 ++
 cupy/cuda/cupy_thrust.h                       |   26 +
 cupy/cuda/cutensor.py                         |   14 +
 cupy/cuda/device.pxd                          |   23 +
 cupy/cuda/device.pyx                          |  368 ++
 cupy/cuda/function.pxd                        |   39 +
 cupy/cuda/function.pyx                        |  302 +
 cupy/cuda/graph.pxd                           |   15 +
 cupy/cuda/graph.pyx                           |   80 +
 cupy/cuda/jitify.pyx                          |  108 +
 cupy/cuda/memory.pxd                          |   95 +
 cupy/cuda/memory.pyx                          | 1982 +++++++
 cupy/cuda/memory_hook.pxd                     |    2 +
 cupy/cuda/memory_hook.pyx                     |  178 +
 cupy/cuda/memory_hooks/__init__.py            |    6 +
 cupy/cuda/memory_hooks/debug_print.py         |   78 +
 cupy/cuda/memory_hooks/line_profile.py        |  171 +
 cupy/cuda/nccl.py                             |   14 +
 cupy/cuda/nvtx.py                             |    1 +
 cupy/cuda/pinned_memory.pxd                   |   38 +
 cupy/cuda/pinned_memory.pyx                   |  352 ++
 cupy/cuda/profiler.py                         |    1 +
 cupy/cuda/runtime.py                          |    1 +
 cupy/cuda/stream.pxd                          |    5 +
 cupy/cuda/stream.pyx                          |  524 ++
 cupy/cuda/texture.pxd                         |   47 +
 cupy/cuda/texture.pyx                         |  561 ++
 cupy/cuda/thrust.pyx                          |  134 +
 cupy/cudnn.py                                 |    8 +
 cupy/cusolver.py                              |    8 +
 cupy/cusparse.py                              |    8 +
 cupy/cutensor.py                              |    8 +
 cupy/fft/__init__.pxd                         |    0
 cupy/fft/__init__.py                          |   19 +
 cupy/fft/_cache.pyx                           |  682 +++
 cupy/fft/_callback.pyx                        |  605 ++
 cupy/fft/_fft.py                              | 1103 ++++
 cupy/fft/config.py                            |   61 +
 cupy/lib/__init__.py                          |    1 +
 cupy/lib/_polynomial.pyx                      |  257 +
 cupy/lib/_routines_poly.py                    |  396 ++
 cupy/lib/_shape_base.py                       |   63 +
 cupy/lib/stride_tricks.py                     |   39 +
 cupy/linalg/__init__.py                       |   43 +
 cupy/linalg/_decomposition.py                 |  585 ++
 cupy/linalg/_eigenvalue.py                    |  190 +
 cupy/linalg/_einsum.py                        |  696 +++
 cupy/linalg/_einsum_cutn.py                   |  168 +
 cupy/linalg/_einsum_opt.py                    |  412 ++
 cupy/linalg/_norms.py                         |  304 +
 cupy/linalg/_product.py                       |  427 ++
 cupy/linalg/_solve.py                         |  407 ++
 cupy/linalg/_util.py                          |  195 +
 cupy/polynomial/__init__.py                   |    6 +
 cupy/polynomial/polynomial.py                 |   46 +
 cupy/polynomial/polyutils.py                  |  110 +
 cupy/prof/__init__.py                         |    2 +
 cupy/prof/_time_range.py                      |   78 +
 cupy/random/LICENSE                           |  109 +
 cupy/random/__init__.py                       |  101 +
 cupy/random/_bit_generator.pyx                |  184 +
 cupy/random/_distributions.py                 |  952 +++
 cupy/random/_generator.py                     | 1309 +++++
 cupy/random/_generator_api.pyx                | 1054 ++++
 cupy/random/_kernels.py                       | 1087 ++++
 cupy/random/_permutations.py                  |   31 +
 cupy/random/_sample.py                        |  239 +
 cupy/random/cupy_distributions.cu             |  908 +++
 cupy/random/cupy_distributions.cuh            |  103 +
 cupy/sparse/__init__.py                       |   22 +
 cupy/sparse/linalg/__init__.py                |   17 +
 cupy/testing/__init__.py                      |   50 +
 cupy/testing/_array.py                        |  157 +
 cupy/testing/_attr.py                         |   43 +
 cupy/testing/_bundle.py                       |   47 +
 cupy/testing/_condition.py                    |  124 +
 cupy/testing/_helper.py                       |  316 +
 cupy/testing/_hypothesis.py                   |  137 +
 cupy/testing/_loops.py                        | 1247 ++++
 cupy/testing/_parameterized.py                |  115 +
 cupy/testing/_pytest_impl.py                  |   70 +
 cupy/testing/_random.py                       |  142 +
 cupy/typing/__init__.py                       |    4 +
 cupy/typing/_generic_alias.py                 |   43 +
 cupy_backends/__init__.pxd                    |    0
 cupy_backends/__init__.py                     |    0
 cupy_backends/cuda/__init__.pxd               |    0
 cupy_backends/cuda/__init__.py                |    0
 cupy_backends/cuda/_softlink.pxd              |    7 +
 cupy_backends/cuda/_softlink.pyx              |   51 +
 cupy_backends/cuda/api/__init__.pxd           |    0
 cupy_backends/cuda/api/__init__.py            |    0
 cupy_backends/cuda/api/_driver_enum.pxd       |   78 +
 cupy_backends/cuda/api/_driver_enum.pyx       |    0
 cupy_backends/cuda/api/_driver_extern.pxi     |   66 +
 cupy_backends/cuda/api/_driver_typedef.pxi    |   30 +
 cupy_backends/cuda/api/_runtime_enum.pxd      |  342 ++
 cupy_backends/cuda/api/_runtime_enum.pyx      |    0
 cupy_backends/cuda/api/_runtime_extern.pxi    |  163 +
 cupy_backends/cuda/api/_runtime_typedef.pxi   |  449 ++
 cupy_backends/cuda/api/driver.pxd             |  101 +
 cupy_backends/cuda/api/driver.pyx             |  331 ++
 cupy_backends/cuda/api/runtime.pxd            |  321 +
 cupy_backends/cuda/api/runtime.pyx            | 1097 ++++
 cupy_backends/cuda/cupy_cublas.h              |   24 +
 cupy_backends/cuda/cupy_cuda.h                |    7 +
 cupy_backends/cuda/cupy_cuda_profiler_api.h   |   18 +
 cupy_backends/cuda/cupy_cuda_runtime.h        |   89 +
 cupy_backends/cuda/cupy_cudnn.h               |  425 ++
 cupy_backends/cuda/cupy_cusolver.h            |  245 +
 cupy_backends/cuda/cupy_cusparse.h            |  910 +++
 cupy_backends/cuda/cupy_cutensor.h            |   19 +
 cupy_backends/cuda/cupy_nccl.h                |   12 +
 cupy_backends/cuda/cupy_nvrtc.h               |   22 +
 cupy_backends/cuda/libs/__init__.pxd          |    0
 cupy_backends/cuda/libs/__init__.py           |    0
 cupy_backends/cuda/libs/cublas.pxd            |  332 ++
 cupy_backends/cuda/libs/cublas.pyx            | 1514 +++++
 cupy_backends/cuda/libs/cudnn.pxd             |  778 +++
 cupy_backends/cuda/libs/cudnn.pyx             | 2541 ++++++++
 cupy_backends/cuda/libs/curand.pxd            |   32 +
 cupy_backends/cuda/libs/curand.pyx            |  233 +
 cupy_backends/cuda/libs/cusolver.pxd          |  727 +++
 cupy_backends/cuda/libs/cusolver.pyx          | 3656 ++++++++++++
 cupy_backends/cuda/libs/cusparse.pxd          |  220 +
 cupy_backends/cuda/libs/cusparse.pyx          | 5173 +++++++++++++++++
 cupy_backends/cuda/libs/cusparselt.pxd        |   39 +
 cupy_backends/cuda/libs/cusparselt.pyx        |  505 ++
 cupy_backends/cuda/libs/cutensor.pxd          |  288 +
 cupy_backends/cuda/libs/cutensor.pyx          | 1000 ++++
 cupy_backends/cuda/libs/nccl.pxd              |   25 +
 cupy_backends/cuda/libs/nccl.pyx              |  494 ++
 cupy_backends/cuda/libs/nvrtc.pxd             |   39 +
 cupy_backends/cuda/libs/nvrtc.pyx             |  278 +
 cupy_backends/cuda/libs/nvtx.pyx              |  232 +
 cupy_backends/cuda/libs/profiler.pxd          |   11 +
 cupy_backends/cuda/libs/profiler.pyx          |   70 +
 cupy_backends/cuda/stream.pxd                 |    6 +
 cupy_backends/cuda/stream.pyx                 |   92 +
 cupy_backends/cupy_backend.h                  |   17 +
 cupy_backends/cupy_backend_runtime.h          |   17 +
 cupy_backends/cupy_blas.h                     |   17 +
 cupy_backends/cupy_complex.h                  |   17 +
 cupy_backends/cupy_cudnn.h                    |  441 ++
 cupy_backends/cupy_cusparselt.h               |   18 +
 cupy_backends/cupy_cutensor.h                 |   20 +
 cupy_backends/cupy_lapack.h                   |  230 +
 cupy_backends/cupy_nccl.h                     |  169 +
 cupy_backends/cupy_profiler.h                 |   17 +
 cupy_backends/cupy_rand.h                     |   17 +
 cupy_backends/cupy_rtc.h                      |   18 +
 cupy_backends/cupy_sparse.h                   |   21 +
 cupy_backends/cupy_tx.h                       |   37 +
 cupy_backends/hip/cupy_cuComplex.h            |   20 +
 cupy_backends/hip/cupy_hip.h                  |  138 +
 cupy_backends/hip/cupy_hip_common.h           |  156 +
 cupy_backends/hip/cupy_hip_runtime.h          |  521 ++
 cupy_backends/hip/cupy_hipblas.h              |  998 ++++
 cupy_backends/hip/cupy_hiprand.h              |  104 +
 cupy_backends/hip/cupy_hiprtc.h               |   74 +
 cupy_backends/hip/cupy_hipsparse.h            | 3881 +++++++++++++
 cupy_backends/hip/cupy_profiler.h             |   24 +
 cupy_backends/hip/cupy_rccl.h                 |    7 +
 cupy_backends/hip/cupy_rocsolver.h            | 2167 +++++++
 cupy_backends/hip/cupy_roctx.h                |   31 +
 cupy_backends/stub/cupy_cuComplex.h           |   22 +
 cupy_backends/stub/cupy_cublas.h              |  430 ++
 cupy_backends/stub/cupy_cuda.h                |  121 +
 cupy_backends/stub/cupy_cuda_common.h         |  235 +
 cupy_backends/stub/cupy_cuda_runtime.h        |  427 ++
 cupy_backends/stub/cupy_cudnn.h               |  644 ++
 cupy_backends/stub/cupy_curand.h              |   94 +
 cupy_backends/stub/cupy_cusolver.h            | 1000 ++++
 cupy_backends/stub/cupy_cusparse.h            | 1253 ++++
 cupy_backends/stub/cupy_cusparselt.h          |  137 +
 cupy_backends/stub/cupy_cutensor.h            |  105 +
 cupy_backends/stub/cupy_nccl.h                |  119 +
 cupy_backends/stub/cupy_nvrtc.h               |   69 +
 cupy_backends/stub/cupy_nvtx.h                |   65 +
 cupy_backends/stub/cupy_profiler.h            |   26 +
 cupyx/__init__.py                             |   25 +
 cupyx/_gufunc.py                              |    5 +
 cupyx/_pinned_array.py                        |  141 +
 cupyx/_rsqrt.py                               |    7 +
 cupyx/_runtime.py                             |  320 +
 cupyx/_scatter.py                             |  131 +
 cupyx/_texture.py                             |  197 +
 cupyx/_ufunc_config.py                        |  123 +
 cupyx/cudnn.pyx                               | 2613 +++++++++
 cupyx/cusolver.pyx                            |  984 ++++
 cupyx/cusparse.py                             | 2092 +++++++
 cupyx/cutensor.pyx                            |  919 +++
 cupyx/distributed/__init__.py                 |    2 +
 cupyx/distributed/_array.py                   |  181 +
 cupyx/distributed/_comm.py                    |   67 +
 cupyx/distributed/_init.py                    |   89 +
 cupyx/distributed/_klv_utils.py               |   58 +
 cupyx/distributed/_nccl_comm.py               |  840 +++
 cupyx/distributed/_store.py                   |  153 +
 cupyx/distributed/_store_actions.py           |  177 +
 cupyx/fallback_mode/__init__.py               |   10 +
 cupyx/fallback_mode/fallback.py               |  596 ++
 cupyx/fallback_mode/notification.py           |   75 +
 cupyx/jit/__init__.py                         |   36 +
 cupyx/jit/_builtin_funcs.py                   |  497 ++
 cupyx/jit/_compile.py                         | 1009 ++++
 cupyx/jit/_cuda_typerules.py                  |  162 +
 cupyx/jit/_cuda_types.py                      |  334 ++
 cupyx/jit/_interface.py                       |  190 +
 cupyx/jit/_internal_types.py                  |  130 +
 cupyx/jit/cg.py                               |  453 ++
 cupyx/jit/cub.py                              |  115 +
 cupyx/jit/thrust.py                           |  952 +++
 cupyx/lapack.py                               |  343 ++
 cupyx/linalg/__init__.py                      |    3 +
 cupyx/linalg/_solve.py                        |   26 +
 cupyx/linalg/sparse/__init__.py               |    3 +
 cupyx/linalg/sparse/_solve.py                 |   62 +
 cupyx/optimizing/__init__.py                  |    1 +
 cupyx/optimizing/_optimize.py                 |  109 +
 cupyx/profiler/__init__.py                    |   31 +
 cupyx/profiler/_time.py                       |  227 +
 cupyx/profiler/_time_range.py                 |   89 +
 cupyx/scipy/__init__.py                       |   35 +
 cupyx/scipy/_lib/__init__.py                  |    0
 cupyx/scipy/_lib/_util.py                     |   73 +
 cupyx/scipy/fft/__init__.py                   |   16 +
 cupyx/scipy/fft/_fft.py                       |  677 +++
 cupyx/scipy/fft/_fftlog.py                    |  225 +
 cupyx/scipy/fft/_helper.py                    |   51 +
 cupyx/scipy/fft/_realtransforms.py            |  924 +++
 cupyx/scipy/fftpack/__init__.py               |    9 +
 cupyx/scipy/fftpack/_fft.py                   |  493 ++
 cupyx/scipy/interpolate/__init__.py           |   21 +
 cupyx/scipy/interpolate/_bspline.py           |  943 +++
 cupyx/scipy/interpolate/_bspline2.py          |  564 ++
 cupyx/scipy/interpolate/_cubic.py             |  414 ++
 cupyx/scipy/interpolate/_interpolate.py       | 1763 ++++++
 cupyx/scipy/interpolate/_polyint.py           |  527 ++
 cupyx/scipy/interpolate/_rbfinterp.py         |  790 +++
 cupyx/scipy/interpolate/_rgi.py               |  622 ++
 cupyx/scipy/linalg/__init__.py                |   16 +
 cupyx/scipy/linalg/_decomp_lu.py              |  354 ++
 cupyx/scipy/linalg/_solve_triangular.py       |  101 +
 cupyx/scipy/linalg/_special_matrices.py       |  570 ++
 cupyx/scipy/linalg/_uarray.py                 |   75 +
 cupyx/scipy/ndimage/__init__.py               |   71 +
 cupyx/scipy/ndimage/_filters.py               | 1255 ++++
 cupyx/scipy/ndimage/_filters_core.py          |  309 +
 cupyx/scipy/ndimage/_filters_generic.py       |  268 +
 cupyx/scipy/ndimage/_fourier.py               |  253 +
 cupyx/scipy/ndimage/_interp_kernels.py        |  598 ++
 cupyx/scipy/ndimage/_interpolation.py         |  779 +++
 cupyx/scipy/ndimage/_measurements.py          | 1239 ++++
 cupyx/scipy/ndimage/_morphology.py            | 1017 ++++
 cupyx/scipy/ndimage/_spline_kernel_weights.py |   73 +
 cupyx/scipy/ndimage/_spline_prefilter_core.py |  261 +
 cupyx/scipy/ndimage/_util.py                  |  162 +
 cupyx/scipy/signal/__init__.py                |   13 +
 cupyx/scipy/signal/_bsplines.py               |   39 +
 cupyx/scipy/signal/_signaltools.py            |  577 ++
 cupyx/scipy/signal/_signaltools_core.py       |  302 +
 cupyx/scipy/sparse/__init__.py                |   44 +
 cupyx/scipy/sparse/_base.py                   |  582 ++
 cupyx/scipy/sparse/_compressed.py             |  861 +++
 cupyx/scipy/sparse/_construct.py              |  582 ++
 cupyx/scipy/sparse/_coo.py                    |  553 ++
 cupyx/scipy/sparse/_csc.py                    |  400 ++
 cupyx/scipy/sparse/_csr.py                    | 1211 ++++
 cupyx/scipy/sparse/_data.py                   |  398 ++
 cupyx/scipy/sparse/_dia.py                    |  219 +
 cupyx/scipy/sparse/_extract.py                |   81 +
 cupyx/scipy/sparse/_index.py                  |  702 +++
 cupyx/scipy/sparse/_sputils.py                |  169 +
 cupyx/scipy/sparse/_util.py                   |   26 +
 cupyx/scipy/sparse/csgraph/__init__.py        |    4 +
 cupyx/scipy/sparse/csgraph/_traversal.py      |  119 +
 cupyx/scipy/sparse/linalg/__init__.py         |   22 +
 cupyx/scipy/sparse/linalg/_eigen.py           |  422 ++
 cupyx/scipy/sparse/linalg/_interface.py       |  578 ++
 cupyx/scipy/sparse/linalg/_iterative.py       |  408 ++
 cupyx/scipy/sparse/linalg/_lobpcg.py          |  674 +++
 cupyx/scipy/sparse/linalg/_norm.py            |  111 +
 cupyx/scipy/sparse/linalg/_solve.py           | 1028 ++++
 cupyx/scipy/spatial/__init__.py               |    1 +
 cupyx/scipy/spatial/distance.py               |  579 ++
 cupyx/scipy/special/__init__.py               |  109 +
 cupyx/scipy/special/_basic.py                 |  287 +
 cupyx/scipy/special/_bessel.py                |  462 ++
 cupyx/scipy/special/_beta.py                  | 1091 ++++
 cupyx/scipy/special/_complexstuff.py          |   54 +
 cupyx/scipy/special/_convex_analysis.py       |  120 +
 cupyx/scipy/special/_digamma.py               |  191 +
 cupyx/scipy/special/_erf.py                   |   59 +
 cupyx/scipy/special/_exp1.py                  |   62 +
 cupyx/scipy/special/_expi.py                  |   60 +
 cupyx/scipy/special/_expn.py                  |  359 ++
 cupyx/scipy/special/_gamma.py                 |  220 +
 cupyx/scipy/special/_gammainc.py              | 1144 ++++
 cupyx/scipy/special/_gammaln.py               |   65 +
 cupyx/scipy/special/_gammasgn.py              |   51 +
 cupyx/scipy/special/_loggamma.py              |  229 +
 cupyx/scipy/special/_logsoftmax.py            |   53 +
 cupyx/scipy/special/_logsumexp.py             |   75 +
 cupyx/scipy/special/_lpmv.py                  |  417 ++
 cupyx/scipy/special/_poch.py                  |  119 +
 cupyx/scipy/special/_polygamma.py             |   22 +
 cupyx/scipy/special/_softmax.py               |   28 +
 cupyx/scipy/special/_sph_harm.py              |   84 +
 cupyx/scipy/special/_statistics.py            |  230 +
 cupyx/scipy/special/_stats_distributions.py   | 1125 ++++
 cupyx/scipy/special/_trig.py                  |   97 +
 cupyx/scipy/special/_xlogy.py                 |   68 +
 cupyx/scipy/special/_zeta.py                  |  129 +
 cupyx/scipy/stats/__init__.py                 |   11 +
 cupyx/scipy/stats/_distributions.py           |   59 +
 cupyx/scipy/stats/_morestats.py               |   47 +
 cupyx/scipy/stats/_stats.py                   |   77 +
 cupyx/scipy/stats/_stats_py.py                |  138 +
 cupyx/time.py                                 |   90 +
 cupyx/tools/__init__.py                       |    0
 cupyx/tools/_hipsparse_stub_mapper.py         |  402 ++
 cupyx/tools/install_library.py                |  303 +
 docker/python3/Dockerfile                     |   12 +
 docker/rocm/Dockerfile                        |   22 +
 docker/rocm/README.md                         |    5 +
 docs/LICENSE_THIRD_PARTY                      |   70 +
 docs/Makefile                                 |  197 +
 docs/image/cupy_logo_1000px.png               |  Bin 0 -> 56999 bytes
 docs/make.bat                                 |  263 +
 docs/requirements.txt                         |    8 +
 docs/source/.gitignore                        |    1 +
 docs/source/_comparison_generator.py          |  297 +
 docs/source/_static/favicon.ico               |  Bin 0 -> 203401 bytes
 docs/source/_templates/autosummary/class.rst  |   53 +
 docs/source/conf.py                           |  518 ++
 docs/source/contribution.rst                  |  420 ++
 docs/source/index.rst                         |   26 +
 docs/source/install.rst                       |  476 ++
 docs/source/license.rst                       |  107 +
 docs/source/overview.rst                      |   68 +
 docs/source/reference/_deprecated.rst         |   42 +
 docs/source/reference/_private.rst            |   31 +
 docs/source/reference/array_api.rst           |   26 +
 docs/source/reference/array_api_array.rst     |   18 +
 docs/source/reference/array_api_functions.rst |    8 +
 docs/source/reference/binary.rst              |   38 +
 docs/source/reference/comparison.rst          |    9 +
 docs/source/reference/creation.rst            |   70 +
 docs/source/reference/cuda.rst                |  209 +
 docs/source/reference/distributed.rst         |   15 +
 docs/source/reference/dtype.rst               |   61 +
 docs/source/reference/environment.rst         |  187 +
 docs/source/reference/ext.rst                 |   51 +
 docs/source/reference/fft.rst                 |   97 +
 docs/source/reference/fftpack.rst             |    3 +
 docs/source/reference/functional.rst          |   18 +
 docs/source/reference/index.rst               |   33 +
 docs/source/reference/indexing.rst            |   65 +
 docs/source/reference/io.rst                  |   51 +
 docs/source/reference/kernel.rst              |   68 +
 docs/source/reference/linalg.rst              |   71 +
 docs/source/reference/logic.rst               |   73 +
 docs/source/reference/manipulation.rst        |  134 +
 docs/source/reference/math.rst                |  187 +
 docs/source/reference/misc.rst                |   32 +
 docs/source/reference/ndarray.rst             |   56 +
 docs/source/reference/ndimage.rst             |    3 +
 docs/source/reference/pad.rst                 |   10 +
 docs/source/reference/polynomials.rst         |   75 +
 docs/source/reference/random.rst              |  127 +
 docs/source/reference/routines.rst            |   31 +
 docs/source/reference/scipy.rst               |   27 +
 docs/source/reference/scipy_fft.rst           |   84 +
 docs/source/reference/scipy_fftpack.rst       |   38 +
 docs/source/reference/scipy_interpolate.rst   |   47 +
 docs/source/reference/scipy_linalg.rst        |   50 +
 docs/source/reference/scipy_ndimage.rst       |  139 +
 docs/source/reference/scipy_signal.rst        |   33 +
 docs/source/reference/scipy_sparse.rst        |  102 +
 .../source/reference/scipy_sparse_csgraph.rst |   22 +
 docs/source/reference/scipy_sparse_linalg.rst |   83 +
 docs/source/reference/scipy_spatial.rst       |   22 +
 .../reference/scipy_spatial_distance.rst      |   51 +
 docs/source/reference/scipy_special.rst       |  164 +
 docs/source/reference/scipy_stats.rst         |   28 +
 docs/source/reference/set.rst                 |   27 +
 docs/source/reference/signal.rst              |    3 +
 docs/source/reference/sorting.rst             |   47 +
 docs/source/reference/sparse.rst              |    3 +
 docs/source/reference/special.rst             |    4 +
 docs/source/reference/statistics.rst          |   61 +
 docs/source/reference/stats.rst               |    3 +
 docs/source/reference/testing.rst             |   87 +
 docs/source/reference/ufunc.rst               |  186 +
 docs/source/reference/window.rst              |   18 +
 docs/source/spelling_wordlist.txt             |  173 +
 docs/source/upgrade.rst                       |  709 +++
 docs/source/user_guide/basic.rst              |  222 +
 docs/source/user_guide/compatibility.rst      |  150 +
 docs/source/user_guide/cuda_api.rst           |   89 +
 docs/source/user_guide/difference.rst         |  194 +
 docs/source/user_guide/fft.rst                |  315 +
 docs/source/user_guide/index.rst              |   17 +
 docs/source/user_guide/interoperability.rst   |  462 ++
 docs/source/user_guide/kernel.rst             |  581 ++
 docs/source/user_guide/memory.rst             |  200 +
 docs/source/user_guide/performance.rst        |  128 +
 examples/cg/cg.py                             |   84 +
 examples/cusparselt/matmul.py                 |   99 +
 examples/custom_struct/README.md              |   11 +
 examples/custom_struct/builtin_vectors.py     |   48 +
 examples/custom_struct/complex_struct.py      |  133 +
 examples/custom_struct/packed_matrix.py       |   91 +
 examples/cutensor/contraction.py              |   45 +
 examples/cutensor/elementwise_binary.py       |   46 +
 examples/cutensor/elementwise_trinary.py      |   56 +
 examples/cutensor/reduction.py                |   43 +
 examples/finance/black_scholes.py             |  144 +
 examples/finance/monte_carlo.py               |  161 +
 examples/finance/monte_carlo_multigpu.py      |   93 +
 examples/gemm/README.md                       |   37 +
 examples/gemm/sgemm.cu                        |  200 +
 examples/gemm/sgemm.py                        |   89 +
 examples/gemm/utils.py                        |   22 +
 examples/gmm/README.md                        |   19 +
 examples/gmm/gmm.py                           |  177 +
 .../mpi4py_multiple_devices.py                |   53 +
 examples/jit/elmentwise_op.py                 |   21 +
 examples/jit/reduction_atomic.py              |   33 +
 examples/jit/reduction_simple.py              |   33 +
 examples/kmeans/README.md                     |   20 +
 examples/kmeans/kmeans.py                     |  152 +
 examples/peer/peer_matrix.py                  |   17 +
 examples/stream/cublas.py                     |   19 +
 examples/stream/cudnn.py                      |   21 +
 examples/stream/cufft.py                      |   18 +
 examples/stream/cupy_event.py                 |   32 +
 examples/stream/cupy_kernel.py                |   18 +
 examples/stream/cupy_memcpy.py                |   50 +
 examples/stream/curand.py                     |   12 +
 examples/stream/cusolver.py                   |   20 +
 examples/stream/cusparse.py                   |   30 +
 examples/stream/map_reduce.py                 |   47 +
 examples/stream/thrust.py                     |   18 +
 install/cupy_builder/__init__.py              |   24 +
 install/cupy_builder/_command.py              |  209 +
 install/cupy_builder/_compiler.py             |  292 +
 install/cupy_builder/_context.py              |   93 +
 install/cupy_builder/_environment.py          |   37 +
 install/cupy_builder/_features.py             |  490 ++
 install/cupy_builder/_preflight.py            |   39 +
 install/cupy_builder/cupy_setup_build.py      |  451 ++
 install/cupy_builder/install_build.py         |  748 +++
 install/cupy_builder/install_utils.py         |   22 +
 install/mypy.ini                              |   23 +
 install/universal_pkg/RELEASE.md              |    5 +
 install/universal_pkg/__init__.py             |    0
 install/universal_pkg/setup.py                |  285 +
 setup.cfg                                     |   61 +
 setup.py                                      |  149 +
 tests/conftest.py                             |   74 +
 tests/cupy_tests/__init__.py                  |    0
 tests/cupy_tests/array_api_tests/__init__.py  |   26 +
 .../array_api_tests/test_array_object.py      |  366 ++
 .../test_creation_functions.py                |  155 +
 .../test_data_type_functions.py               |   19 +
 .../test_elementwise_functions.py             |  111 +
 .../test_indexing_functions.py                |   22 +
 .../array_api_tests/test_set_functions.py     |   21 +
 .../array_api_tests/test_sorting_functions.py |   23 +
 .../array_api_tests/test_validation.py        |   27 +
 tests/cupy_tests/binary_tests/__init__.py     |    0
 .../binary_tests/test_elementwise.py          |   37 +
 tests/cupy_tests/binary_tests/test_packing.py |   78 +
 tests/cupy_tests/core_tests/__init__.py       |    0
 .../core_tests/fusion_tests/__init__.py       |    0
 .../core_tests/fusion_tests/fusion_utils.py   |  144 +
 .../core_tests/fusion_tests/test_array.py     |  309 +
 .../core_tests/fusion_tests/test_example.py   |   46 +
 .../core_tests/fusion_tests/test_indexing.py  |  117 +
 .../fusion_tests/test_kernel_cache.py         |  176 +
 .../core_tests/fusion_tests/test_misc.py      |  422 ++
 .../fusion_tests/test_optimization.py         |  199 +
 .../core_tests/fusion_tests/test_reduction.py |  242 +
 .../core_tests/fusion_tests/test_routines.py  |  412 ++
 .../core_tests/fusion_tests/test_ufunc.py     |  316 +
 .../core_tests/test_array_function.py         |   58 +
 tests/cupy_tests/core_tests/test_carray.py    |   85 +
 tests/cupy_tests/core_tests/test_core.py      |  120 +
 .../core_tests/test_cub_reduction.py          |  148 +
 tests/cupy_tests/core_tests/test_dlpack.py    |  150 +
 .../cupy_tests/core_tests/test_elementwise.py |  145 +
 tests/cupy_tests/core_tests/test_flags.py     |   67 +
 tests/cupy_tests/core_tests/test_function.py  |  218 +
 tests/cupy_tests/core_tests/test_gufuncs.py   |  282 +
 tests/cupy_tests/core_tests/test_include.py   |   86 +
 tests/cupy_tests/core_tests/test_internal.py  |  245 +
 tests/cupy_tests/core_tests/test_iter.py      |   42 +
 tests/cupy_tests/core_tests/test_ndarray.py   |  648 +++
 .../core_tests/test_ndarray_adv_indexing.py   |  709 +++
 .../core_tests/test_ndarray_complex_ops.py    |  166 +
 .../core_tests/test_ndarray_contiguity.py     |   16 +
 .../core_tests/test_ndarray_conversion.py     |   59 +
 .../core_tests/test_ndarray_copy_and_view.py  |  460 ++
 .../test_ndarray_cuda_array_interface.py      |  314 +
 .../core_tests/test_ndarray_elementwise_op.py |  775 +++
 .../cupy_tests/core_tests/test_ndarray_get.py |  142 +
 .../core_tests/test_ndarray_indexing.py       |  232 +
 .../core_tests/test_ndarray_math.py           |  114 +
 .../core_tests/test_ndarray_owndata.py        |   20 +
 .../core_tests/test_ndarray_reduction.py      |  498 ++
 .../core_tests/test_ndarray_scatter.py        |  283 +
 .../core_tests/test_ndarray_ufunc.py          |  270 +
 .../core_tests/test_ndarray_unary_op.py       |  145 +
 tests/cupy_tests/core_tests/test_raw.py       | 1367 +++++
 tests/cupy_tests/core_tests/test_reduction.py |  233 +
 tests/cupy_tests/core_tests/test_scan.py      |   55 +
 .../cupy_tests/core_tests/test_syncdetect.py  |   33 +
 .../core_tests/test_ufunc_methods.py          |  227 +
 .../cupy_tests/core_tests/test_userkernel.py  |  359 ++
 tests/cupy_tests/creation_tests/__init__.py   |    0
 tests/cupy_tests/creation_tests/test_basic.py |  496 ++
 .../creation_tests/test_from_data.py          |  779 +++
 .../cupy_tests/creation_tests/test_matrix.py  |  220 +
 .../cupy_tests/creation_tests/test_ranges.py  |  401 ++
 tests/cupy_tests/cuda_tests/__init__.py       |    0
 .../cuda_tests/memory_hooks_tests/__init__.py |    0
 .../memory_hooks_tests/test_debug_print.py    |   49 +
 .../memory_hooks_tests/test_line_profile.py   |   50 +
 tests/cupy_tests/cuda_tests/test_compiler.py  |  136 +
 tests/cupy_tests/cuda_tests/test_cublas.py    |   13 +
 tests/cupy_tests/cuda_tests/test_cudnn.py     |   17 +
 tests/cupy_tests/cuda_tests/test_cufft.py     |  232 +
 tests/cupy_tests/cuda_tests/test_curand.py    |   47 +
 tests/cupy_tests/cuda_tests/test_cusolver.py  |   13 +
 tests/cupy_tests/cuda_tests/test_cusparse.py  |   27 +
 tests/cupy_tests/cuda_tests/test_cutensor.py  |   14 +
 tests/cupy_tests/cuda_tests/test_device.py    |  238 +
 tests/cupy_tests/cuda_tests/test_driver.py    |   64 +
 tests/cupy_tests/cuda_tests/test_graph.py     |  345 ++
 tests/cupy_tests/cuda_tests/test_memory.py    | 1341 +++++
 .../cupy_tests/cuda_tests/test_memory_hook.py |   85 +
 tests/cupy_tests/cuda_tests/test_nccl.py      |  101 +
 tests/cupy_tests/cuda_tests/test_nvrtc.py     |   13 +
 tests/cupy_tests/cuda_tests/test_nvtx.py      |   21 +
 .../cuda_tests/test_pinned_memory.py          |  129 +
 tests/cupy_tests/cuda_tests/test_profile.py   |   29 +
 tests/cupy_tests/cuda_tests/test_runtime.py   |   57 +
 tests/cupy_tests/cuda_tests/test_stream.py    |  286 +
 tests/cupy_tests/cuda_tests/test_texture.py   |  425 ++
 tests/cupy_tests/fft_tests/__init__.py        |    0
 tests/cupy_tests/fft_tests/test_cache.py      |  493 ++
 tests/cupy_tests/fft_tests/test_callback.py   |  708 +++
 tests/cupy_tests/fft_tests/test_fft.py        | 1362 +++++
 tests/cupy_tests/functional_tests/__init__.py |    0
 .../functional_tests/test_piecewise.py        |  119 +
 .../functional_tests/test_vectorize.py        |  668 +++
 tests/cupy_tests/indexing_tests/__init__.py   |    0
 .../indexing_tests/test_generate.py           |  410 ++
 .../indexing_tests/test_indexing.py           |  383 ++
 .../cupy_tests/indexing_tests/test_insert.py  |  305 +
 .../cupy_tests/indexing_tests/test_iterate.py |  138 +
 tests/cupy_tests/io_tests/__init__.py         |    0
 tests/cupy_tests/io_tests/test_base_n.py      |   48 +
 tests/cupy_tests/io_tests/test_formatting.py  |   41 +
 tests/cupy_tests/io_tests/test_npz.py         |  100 +
 tests/cupy_tests/io_tests/test_text.py        |   25 +
 tests/cupy_tests/lib_tests/__init__.py        |    0
 tests/cupy_tests/lib_tests/test_polynomial.py |  898 +++
 tests/cupy_tests/lib_tests/test_shape_base.py |  108 +
 .../lib_tests/test_strided_tricks.py          |   46 +
 tests/cupy_tests/linalg_tests/__init__.py     |    0
 .../linalg_tests/test_decomposition.py        |  368 ++
 .../linalg_tests/test_eigenvalue.py           |  181 +
 tests/cupy_tests/linalg_tests/test_einsum.py  |  563 ++
 tests/cupy_tests/linalg_tests/test_norms.py   |  208 +
 tests/cupy_tests/linalg_tests/test_product.py |  478 ++
 tests/cupy_tests/linalg_tests/test_solve.py   |  334 ++
 tests/cupy_tests/logic_tests/__init__.py      |    0
 .../cupy_tests/logic_tests/test_comparison.py |  291 +
 tests/cupy_tests/logic_tests/test_content.py  |   46 +
 tests/cupy_tests/logic_tests/test_ops.py      |   31 +
 tests/cupy_tests/logic_tests/test_truth.py    |  305 +
 .../cupy_tests/logic_tests/test_type_test.py  |  106 +
 .../cupy_tests/manipulation_tests/__init__.py |    0
 .../manipulation_tests/test_add_remove.py     |  211 +
 .../manipulation_tests/test_basic.py          |  259 +
 .../manipulation_tests/test_dims.py           |  344 ++
 .../manipulation_tests/test_join.py           |  452 ++
 .../manipulation_tests/test_kind.py           |  127 +
 .../manipulation_tests/test_rearrange.py      |  272 +
 .../manipulation_tests/test_shape.py          |  243 +
 .../manipulation_tests/test_split.py          |   96 +
 .../manipulation_tests/test_tiling.py         |  131 +
 .../manipulation_tests/test_transpose.py      |  170 +
 tests/cupy_tests/math_tests/__init__.py       |    0
 .../cupy_tests/math_tests/test_arithmetic.py  |  533 ++
 tests/cupy_tests/math_tests/test_explog.py    |   70 +
 tests/cupy_tests/math_tests/test_floating.py  |   67 +
 .../cupy_tests/math_tests/test_hyperbolic.py  |   39 +
 tests/cupy_tests/math_tests/test_matmul.py    |  400 ++
 tests/cupy_tests/math_tests/test_misc.py      |  537 ++
 tests/cupy_tests/math_tests/test_rational.py  |   37 +
 tests/cupy_tests/math_tests/test_rounding.py  |  165 +
 tests/cupy_tests/math_tests/test_special.py   |   24 +
 tests/cupy_tests/math_tests/test_sumprod.py   | 1150 ++++
 .../math_tests/test_trigonometric.py          |  113 +
 tests/cupy_tests/math_tests/test_window.py    |   38 +
 tests/cupy_tests/misc_tests/__init__.py       |    0
 .../cupy_tests/misc_tests/test_byte_bounds.py |   76 +
 .../misc_tests/test_memory_ranges.py          |  152 +
 tests/cupy_tests/misc_tests/test_who.py       |   65 +
 tests/cupy_tests/padding_tests/__init__.py    |    0
 tests/cupy_tests/padding_tests/test_pad.py    |  310 +
 .../polynomial_tests/test_polynomial.py       |  134 +
 .../polynomial_tests/test_polyutils.py        |  172 +
 tests/cupy_tests/prof_tests/__init__.py       |    0
 tests/cupy_tests/prof_tests/test_range.py     |   98 +
 tests/cupy_tests/random_tests/__init__.py     |    0
 .../random_tests/common_distributions.py      |  521 ++
 .../random_tests/test_bit_generator.py        |   75 +
 .../random_tests/test_distributions.py        |  852 +++
 .../cupy_tests/random_tests/test_generator.py | 1290 ++++
 .../random_tests/test_generator_api.py        |  370 ++
 tests/cupy_tests/random_tests/test_init.py    |    6 +
 .../random_tests/test_permutations.py         |  182 +
 tests/cupy_tests/random_tests/test_random.py  |   19 +
 tests/cupy_tests/random_tests/test_sample.py  |  300 +
 tests/cupy_tests/sorting_tests/__init__.py    |    0
 tests/cupy_tests/sorting_tests/test_count.py  |   60 +
 tests/cupy_tests/sorting_tests/test_search.py |  832 +++
 tests/cupy_tests/sorting_tests/test_sort.py   |  783 +++
 tests/cupy_tests/statistics_tests/__init__.py |    0
 .../statistics_tests/test_correlation.py      |  184 +
 .../statistics_tests/test_histogram.py        |  566 ++
 .../statistics_tests/test_meanvar.py          |  514 ++
 .../cupy_tests/statistics_tests/test_order.py |  430 ++
 tests/cupy_tests/test_cublas.py               |  629 ++
 tests/cupy_tests/test_init.py                 |  153 +
 tests/cupy_tests/test_ndim.py                 |   66 +
 tests/cupy_tests/test_numpy_interop.py        |  150 +
 tests/cupy_tests/test_type_routines.py        |   91 +
 tests/cupy_tests/test_typing.py               |    8 +
 tests/cupy_tests/testing_tests/__init__.py    |    0
 tests/cupy_tests/testing_tests/test_array.py  |  116 +
 .../testing_tests/test_condition.py           |  190 +
 tests/cupy_tests/testing_tests/test_helper.py |  185 +
 tests/cupy_tests/testing_tests/test_loops.py  |  510 ++
 .../testing_tests/test_parameterized.py       |  232 +
 tests/cupyx_tests/__init__.py                 |    0
 .../distributed_tests/comm_runner.py          |  607 ++
 .../distributed_tests/test_array.py           |  106 +
 .../distributed_tests/test_comm.py            |  189 +
 .../distributed_tests/test_store.py           |   43 +
 .../fallback_mode_tests/__init__.py           |    0
 .../fallback_mode_tests/test_fallback.py      |  657 +++
 .../fallback_mode_tests/test_notifications.py |  172 +
 tests/cupyx_tests/jit_tests/__init__.py       |    0
 .../jit_tests/test_cooperative_groups.py      |  185 +
 tests/cupyx_tests/jit_tests/test_cub.py       |  128 +
 .../jit_tests/test_device_function.py         |  103 +
 tests/cupyx_tests/jit_tests/test_raw.py       |  770 +++
 tests/cupyx_tests/jit_tests/test_thrust.py    | 1125 ++++
 tests/cupyx_tests/linalg_tests/__init__.py    |    0
 .../linalg_tests/sparse_tests/__init__.py     |    8 +
 .../linalg_tests/sparse_tests/test_solve.py   |   67 +
 tests/cupyx_tests/linalg_tests/test_solve.py  |   84 +
 tests/cupyx_tests/profiler_tests/__init__.py  |    0
 .../profiler_tests/test_benchmark.py          |  132 +
 .../profiler_tests/test_profile.py            |   29 +
 .../profiler_tests/test_time_range.py         |   98 +
 tests/cupyx_tests/scipy_tests/__init__.py     |    0
 .../scipy_tests/fft_tests/__init__.py         |    0
 .../scipy_tests/fft_tests/test_fft.py         | 1786 ++++++
 .../scipy_tests/fft_tests/test_fftlog.py      |   75 +
 .../scipy_tests/fft_tests/test_helper.py      |   21 +
 .../fft_tests/test_realtransforms.py          |  173 +
 .../scipy_tests/fftpack_tests/__init__.py     |    0
 .../scipy_tests/fftpack_tests/test_fftpack.py |  689 +++
 .../scipy_tests/interpolate_tests/__init__.py |    0
 .../interpolate_tests/test_bspline.py         |  489 ++
 .../interpolate_tests/test_bspline2.py        |  458 ++
 .../interpolate_tests/test_polyint.py         |  617 ++
 .../interpolate_tests/test_ppoly.py           | 1315 +++++
 .../interpolate_tests/test_rbfinterp.py       |  576 ++
 .../scipy_tests/interpolate_tests/test_rgi.py |  741 +++
 .../scipy_tests/linalg_tests/__init__.py      |    0
 .../linalg_tests/test_decomp_lu.py            |  155 +
 .../linalg_tests/test_solve_triangular.py     |   95 +
 .../linalg_tests/test_special_matrices.py     |  107 +
 .../scipy_tests/linalg_tests/test_uarray.py   |    9 +
 .../scipy_tests/ndimage_tests/__init__.py     |    0
 .../scipy_tests/ndimage_tests/test_filters.py |  824 +++
 .../scipy_tests/ndimage_tests/test_fourier.py |  421 ++
 .../ndimage_tests/test_interpolation.py       | 1029 ++++
 .../ndimage_tests/test_measurements.py        |  452 ++
 .../ndimage_tests/test_morphology.py          |  610 ++
 .../scipy_tests/signal_tests/test_bsplines.py |   33 +
 .../signal_tests/test_signaltools.py          |  362 ++
 .../scipy_tests/sparse_tests/__init__.py      |    0
 .../csgraph_tests/test_traversal.py           |   63 +
 .../scipy_tests/sparse_tests/test_base.py     |   84 +
 .../sparse_tests/test_construct.py            |  476 ++
 .../scipy_tests/sparse_tests/test_coo.py      | 1191 ++++
 .../scipy_tests/sparse_tests/test_csc.py      | 1560 +++++
 .../scipy_tests/sparse_tests/test_csr.py      | 2170 +++++++
 .../scipy_tests/sparse_tests/test_dia.py      |  370 ++
 .../scipy_tests/sparse_tests/test_extract.py  |   75 +
 .../scipy_tests/sparse_tests/test_index.py    |  579 ++
 .../scipy_tests/sparse_tests/test_linalg.py   | 1697 ++++++
 .../scipy_tests/spatial_tests/__init__.py     |    0
 .../spatial_tests/test_distance.py            |  232 +
 .../scipy_tests/special_tests/__init__.py     |    0
 .../scipy_tests/special_tests/test_basic.py   |  218 +
 .../scipy_tests/special_tests/test_bessel.py  |  135 +
 .../scipy_tests/special_tests/test_beta.py    |  144 +
 .../special_tests/test_convex_analysis.py     |   59 +
 .../scipy_tests/special_tests/test_digamma.py |   59 +
 .../scipy_tests/special_tests/test_erf.py     |  148 +
 .../scipy_tests/special_tests/test_exp1.py    |   40 +
 .../scipy_tests/special_tests/test_expi.py    |   40 +
 .../scipy_tests/special_tests/test_expn.py    |   52 +
 .../scipy_tests/special_tests/test_gamma.py   |   63 +
 .../special_tests/test_gammainc.py            |  169 +
 .../scipy_tests/special_tests/test_gammaln.py |  119 +
 .../special_tests/test_log_softmax.py         |   66 +
 .../special_tests/test_logsumexp.py           |  112 +
 .../special_tests/test_polygamma.py           |   58 +
 .../scipy_tests/special_tests/test_softmax.py |   82 +
 .../special_tests/test_sph_harm.py            |   28 +
 .../special_tests/test_statistics.py          |  302 +
 .../special_tests/test_ufunc_dispatch.py      |   54 +
 .../scipy_tests/special_tests/test_zeta.py    |   48 +
 .../stats_tests/test_distributions.py         |  138 +
 .../scipy_tests/stats_tests/test_morestats.py |  145 +
 .../scipy_tests/stats_tests/test_stats.py     |  244 +
 .../scipy_tests/test_get_array_module.py      |   28 +
 tests/cupyx_tests/test_cudnn.py               |  457 ++
 tests/cupyx_tests/test_cupyx.py               |   30 +
 tests/cupyx_tests/test_cusolver.py            |  316 +
 tests/cupyx_tests/test_cusparse.py            | 1118 ++++
 tests/cupyx_tests/test_cutensor.py            |  370 ++
 tests/cupyx_tests/test_lapack.py              |  167 +
 tests/cupyx_tests/test_optimize.py            |  212 +
 tests/cupyx_tests/test_pinned_array.py        |  422 ++
 tests/cupyx_tests/test_rsqrt.py               |   18 +
 tests/cupyx_tests/test_runtime.py             |   46 +
 tests/cupyx_tests/test_time.py                |  131 +
 tests/cupyx_tests/tools_tests/__init__.py     |    0
 .../tools_tests/test_install_library.py       |   78 +
 tests/example_tests/__init__.py               |    0
 tests/example_tests/example_test.py           |   18 +
 tests/example_tests/test_custom_struct.py     |   34 +
 tests/example_tests/test_finance.py           |   62 +
 tests/example_tests/test_gemm.py              |    9 +
 tests/example_tests/test_gmm.py               |   37 +
 tests/example_tests/test_kmeans.py            |   43 +
 tests/install_tests/__init__.py               |   20 +
 tests/install_tests/test_build.py             |   39 +
 .../test_cupy_builder/__init__.py             |    0
 .../test_cupy_builder/test_command.py         |   17 +
 .../test_cupy_builder/test_context.py         |   86 +
 .../test_cupy_builder/test_features.py        |   30 +
 .../test_universal_pkg/__init__.py            |    0
 .../test_universal_pkg/test_setup.py          |   54 +
 tests/install_tests/test_utils.py             |   18 +
 third_party/cub/.cproject                     | 1223 ++++
 third_party/cub/.project                      |   27 +
 third_party/cub/.settings/.gitignore          |    1 +
 .../org.eclipse.cdt.codan.core.prefs          |   72 +
 .../cub/.settings/org.eclipse.cdt.core.prefs  |  177 +
 .../cub/.settings/org.eclipse.cdt.ui.prefs    |    3 +
 .../.settings/org.eclipse.core.runtime.prefs  |    4 +
 third_party/cub/CHANGE_LOG.TXT                |  403 ++
 third_party/cub/LICENSE.TXT                   |   24 +
 third_party/cub/README.md                     |  128 +
 third_party/cub/common.mk                     |  233 +
 third_party/cub/cub/agent/agent_histogram.cuh |  787 +++
 .../cub/agent/agent_radix_sort_downsweep.cuh  |  789 +++
 .../cub/agent/agent_radix_sort_upsweep.cuh    |  526 ++
 third_party/cub/cub/agent/agent_reduce.cuh    |  385 ++
 .../cub/cub/agent/agent_reduce_by_key.cuh     |  547 ++
 third_party/cub/cub/agent/agent_rle.cuh       |  837 +++
 third_party/cub/cub/agent/agent_scan.cuh      |  471 ++
 .../cub/cub/agent/agent_segment_fixup.cuh     |  375 ++
 third_party/cub/cub/agent/agent_select_if.cuh |  703 +++
 third_party/cub/cub/agent/agent_spmv_orig.cuh |  670 +++
 .../cub/agent/single_pass_scan_operators.cuh  |  815 +++
 .../cub/block/block_adjacent_difference.cuh   |  596 ++
 .../cub/cub/block/block_discontinuity.cuh     | 1148 ++++
 third_party/cub/cub/block/block_exchange.cuh  | 1248 ++++
 third_party/cub/cub/block/block_histogram.cuh |  415 ++
 third_party/cub/cub/block/block_load.cuh      | 1241 ++++
 .../cub/cub/block/block_radix_rank.cuh        |  696 +++
 .../cub/cub/block/block_radix_sort.cuh        |  863 +++
 .../cub/cub/block/block_raking_layout.cuh     |  152 +
 third_party/cub/cub/block/block_reduce.cuh    |  607 ++
 third_party/cub/cub/block/block_scan.cuh      | 2126 +++++++
 third_party/cub/cub/block/block_shuffle.cuh   |  305 +
 third_party/cub/cub/block/block_store.cuh     | 1000 ++++
 .../block_histogram_atomic.cuh                |   82 +
 .../specializations/block_histogram_sort.cuh  |  226 +
 .../specializations/block_reduce_raking.cuh   |  226 +
 .../block_reduce_raking_commutative_only.cuh  |  199 +
 .../block_reduce_warp_reductions.cuh          |  218 +
 .../specializations/block_scan_raking.cuh     |  666 +++
 .../specializations/block_scan_warp_scans.cuh |  392 ++
 .../block_scan_warp_scans2.cuh                |  436 ++
 .../block_scan_warp_scans3.cuh                |  418 ++
 third_party/cub/cub/cub.cuh                   |   95 +
 .../cub/cub/device/device_histogram.cuh       |  866 +++
 .../cub/cub/device/device_partition.cuh       |  273 +
 .../cub/cub/device/device_radix_sort.cuh      |  797 +++
 third_party/cub/cub/device/device_reduce.cuh  |  734 +++
 .../cub/device/device_run_length_encode.cuh   |  278 +
 third_party/cub/cub/device/device_scan.cuh    |  443 ++
 .../device/device_segmented_radix_sort.cuh    |  876 +++
 .../cub/device/device_segmented_reduce.cuh    |  619 ++
 third_party/cub/cub/device/device_select.cuh  |  369 ++
 third_party/cub/cub/device/device_spmv.cuh    |  174 +
 .../device/dispatch/dispatch_histogram.cuh    | 1096 ++++
 .../device/dispatch/dispatch_radix_sort.cuh   | 1619 ++++++
 .../cub/device/dispatch/dispatch_reduce.cuh   |  882 +++
 .../dispatch/dispatch_reduce_by_key.cuh       |  554 ++
 .../cub/cub/device/dispatch/dispatch_rle.cuh  |  538 ++
 .../cub/cub/device/dispatch/dispatch_scan.cuh |  563 ++
 .../device/dispatch/dispatch_select_if.cuh    |  542 ++
 .../device/dispatch/dispatch_spmv_orig.cuh    |  834 +++
 third_party/cub/cub/grid/grid_barrier.cuh     |  211 +
 third_party/cub/cub/grid/grid_even_share.cuh  |  222 +
 third_party/cub/cub/grid/grid_mapping.cuh     |  113 +
 third_party/cub/cub/grid/grid_queue.cuh       |  220 +
 third_party/cub/cub/host/mutex.cuh            |  171 +
 .../cub/iterator/arg_index_input_iterator.cuh |  259 +
 .../cache_modified_input_iterator.cuh         |  240 +
 .../cache_modified_output_iterator.cuh        |  254 +
 .../cub/iterator/constant_input_iterator.cuh  |  235 +
 .../cub/iterator/counting_input_iterator.cuh  |  228 +
 .../cub/iterator/discard_output_iterator.cuh  |  220 +
 .../cub/iterator/tex_obj_input_iterator.cuh   |  310 +
 .../cub/iterator/tex_ref_input_iterator.cuh   |  374 ++
 .../cub/iterator/transform_input_iterator.cuh |  252 +
 third_party/cub/cub/thread/thread_load.cuh    |  438 ++
 .../cub/cub/thread/thread_operators.cuh       |  317 +
 third_party/cub/cub/thread/thread_reduce.cuh  |  152 +
 third_party/cub/cub/thread/thread_scan.cuh    |  268 +
 third_party/cub/cub/thread/thread_search.cuh  |  154 +
 third_party/cub/cub/thread/thread_store.cuh   |  422 ++
 third_party/cub/cub/util_allocator.cuh        |  708 +++
 third_party/cub/cub/util_arch.cuh             |  151 +
 third_party/cub/cub/util_debug.cuh            |  145 +
 third_party/cub/cub/util_device.cuh           |  347 ++
 third_party/cub/cub/util_macro.cuh            |  103 +
 third_party/cub/cub/util_namespace.cuh        |   46 +
 third_party/cub/cub/util_ptx.cuh              |  758 +++
 third_party/cub/cub/util_type.cuh             | 1167 ++++
 .../warp/specializations/warp_reduce_shfl.cuh |  541 ++
 .../warp/specializations/warp_reduce_smem.cuh |  372 ++
 .../warp/specializations/warp_scan_shfl.cuh   |  632 ++
 .../warp/specializations/warp_scan_smem.cuh   |  397 ++
 third_party/cub/cub/warp/warp_reduce.cuh      |  612 ++
 third_party/cub/cub/warp/warp_scan.cuh        |  936 +++
 .../cub/eclipse code style profile.xml        |  155 +
 third_party/cub/examples/block/.gitignore     |    7 +
 third_party/cub/examples/block/Makefile       |  128 +
 .../block/example_block_radix_sort.cu         |  323 +
 .../examples/block/example_block_reduce.cu    |  290 +
 .../cub/examples/block/example_block_scan.cu  |  334 ++
 .../cub/examples/block/reduce_by_key.cu       |   57 +
 third_party/cub/examples/device/.gitignore    |    8 +
 third_party/cub/examples/device/Makefile      |  197 +
 .../example_device_partition_flagged.cu       |  233 +
 .../device/example_device_partition_if.cu     |  244 +
 .../device/example_device_radix_sort.cu       |  226 +
 .../examples/device/example_device_reduce.cu  |  180 +
 .../examples/device/example_device_scan.cu    |  186 +
 .../device/example_device_select_flagged.cu   |  233 +
 .../device/example_device_select_if.cu        |  242 +
 .../device/example_device_select_unique.cu    |  221 +
 ...ample_device_sort_find_non_trivial_runs.cu |  384 ++
 third_party/cub/experimental/.gitignore       |    1 +
 third_party/cub/experimental/Makefile         |  125 +
 .../experimental/defunct/example_coo_spmv.cu  | 1070 ++++
 .../defunct/test_device_seg_reduce.cu         | 2142 +++++++
 .../experimental/histogram/histogram_cub.h    |  109 +
 .../histogram/histogram_gmem_atomics.h        |  185 +
 .../histogram/histogram_smem_atomics.h        |  195 +
 .../cub/experimental/histogram_compare.cu     |  635 ++
 third_party/cub/experimental/sparse_matrix.h  | 1244 ++++
 third_party/cub/experimental/spmv_compare.cu  |  917 +++
 third_party/cub/experimental/spmv_script.sh   |   30 +
 third_party/cub/test/.gitignore               |    3 +
 third_party/cub/test/Makefile                 |  468 ++
 third_party/cub/test/half.h                   |  298 +
 third_party/cub/test/link_a.cu                |   11 +
 third_party/cub/test/link_b.cu                |   11 +
 third_party/cub/test/link_main.cpp            |   10 +
 third_party/cub/test/mersenne.h               |  160 +
 third_party/cub/test/test_allocator.cu        |  459 ++
 third_party/cub/test/test_block_histogram.cu  |  310 +
 third_party/cub/test/test_block_load_store.cu |  549 ++
 third_party/cub/test/test_block_radix_sort.cu |  717 +++
 third_party/cub/test/test_block_reduce.cu     |  822 +++
 third_party/cub/test/test_block_scan.cu       |  929 +++
 third_party/cub/test/test_device_histogram.cu | 1669 ++++++
 .../cub/test/test_device_radix_sort.cu        | 1298 +++++
 third_party/cub/test/test_device_reduce.cu    | 1359 +++++
 .../cub/test/test_device_reduce_by_key.cu     |  853 +++
 .../cub/test/test_device_run_length_encode.cu |  890 +++
 third_party/cub/test/test_device_scan.cu      | 1015 ++++
 third_party/cub/test/test_device_select_if.cu | 1039 ++++
 .../cub/test/test_device_select_unique.cu     |  651 +++
 third_party/cub/test/test_grid_barrier.cu     |  152 +
 third_party/cub/test/test_iterator.cu         |  805 +++
 third_party/cub/test/test_util.h              | 1628 ++++++
 third_party/cub/test/test_warp_reduce.cu      |  840 +++
 third_party/cub/test/test_warp_scan.cu        |  661 +++
 third_party/cub/tune/.gitignore               |    1 +
 third_party/cub/tune/Makefile                 |  192 +
 third_party/cub/tune/tune_device_reduce.cu    |  763 +++
 1156 files changed, 357999 insertions(+)
 create mode 100644 CITATION.bib
 create mode 100644 CODE_OF_CONDUCT.md
 create mode 100644 LICENSE
 create mode 100644 MANIFEST.in
 create mode 100644 codecov.yml
 create mode 100644 cupy/__init__.py
 create mode 100644 cupy/_binary/__init__.py
 create mode 100644 cupy/_binary/elementwise.py
 create mode 100644 cupy/_binary/packing.py
 create mode 100644 cupy/_core/__init__.pxd
 create mode 100644 cupy/_core/__init__.py
 create mode 100644 cupy/_core/_accelerator.pxd
 create mode 100644 cupy/_core/_accelerator.pyx
 create mode 100644 cupy/_core/_carray.pxd
 create mode 100644 cupy/_core/_carray.pyx
 create mode 100644 cupy/_core/_codeblock.py
 create mode 100644 cupy/_core/_cub_reduction.pxd
 create mode 100644 cupy/_core/_cub_reduction.pyx
 create mode 100644 cupy/_core/_dtype.pxd
 create mode 100644 cupy/_core/_dtype.pyx
 create mode 100644 cupy/_core/_fusion_interface.py
 create mode 100644 cupy/_core/_fusion_kernel.pyx
 create mode 100644 cupy/_core/_fusion_op.py
 create mode 100644 cupy/_core/_fusion_optimization.py
 create mode 100644 cupy/_core/_fusion_thread_local.pyx
 create mode 100644 cupy/_core/_fusion_trace.pyx
 create mode 100644 cupy/_core/_fusion_variable.pxd
 create mode 100644 cupy/_core/_fusion_variable.pyx
 create mode 100644 cupy/_core/_gufuncs.py
 create mode 100644 cupy/_core/_kernel.pxd
 create mode 100644 cupy/_core/_kernel.pyx
 create mode 100644 cupy/_core/_memory_range.pxd
 create mode 100644 cupy/_core/_memory_range.pyx
 create mode 100644 cupy/_core/_optimize_config.pxd
 create mode 100644 cupy/_core/_optimize_config.pyx
 create mode 100644 cupy/_core/_reduction.pxd
 create mode 100644 cupy/_core/_reduction.pyx
 create mode 100644 cupy/_core/_routines_binary.pxd
 create mode 100644 cupy/_core/_routines_binary.pyx
 create mode 100644 cupy/_core/_routines_indexing.pxd
 create mode 100644 cupy/_core/_routines_indexing.pyx
 create mode 100644 cupy/_core/_routines_linalg.pxd
 create mode 100644 cupy/_core/_routines_linalg.pyx
 create mode 100644 cupy/_core/_routines_logic.pxd
 create mode 100644 cupy/_core/_routines_logic.pyx
 create mode 100644 cupy/_core/_routines_manipulation.pxd
 create mode 100644 cupy/_core/_routines_manipulation.pyx
 create mode 100644 cupy/_core/_routines_math.pxd
 create mode 100644 cupy/_core/_routines_math.pyx
 create mode 100644 cupy/_core/_routines_sorting.pxd
 create mode 100644 cupy/_core/_routines_sorting.pyx
 create mode 100644 cupy/_core/_routines_statistics.pxd
 create mode 100644 cupy/_core/_routines_statistics.pyx
 create mode 100644 cupy/_core/_scalar.pxd
 create mode 100644 cupy/_core/_scalar.pyx
 create mode 100644 cupy/_core/_ufuncs.py
 create mode 100644 cupy/_core/core.pxd
 create mode 100644 cupy/_core/core.pyx
 create mode 100644 cupy/_core/dlpack.pxd
 create mode 100644 cupy/_core/dlpack.pyx
 create mode 100644 cupy/_core/flags.pyx
 create mode 100644 cupy/_core/fusion.pyx
 create mode 100644 cupy/_core/halffloat.h
 create mode 100644 cupy/_core/include/cupy/README.md
 create mode 100644 cupy/_core/include/cupy/_cuda/README.md
 create mode 100755 cupy/_core/include/cupy/_cuda/cuda-10.2/cuda_fp16.h
 create mode 100755 cupy/_core/include/cupy/_cuda/cuda-10.2/cuda_fp16.hpp
 create mode 100755 cupy/_core/include/cupy/_cuda/cuda-11.0/cuda_fp16.h
 create mode 100755 cupy/_core/include/cupy/_cuda/cuda-11.0/cuda_fp16.hpp
 create mode 100755 cupy/_core/include/cupy/_cuda/cuda-11.1/cuda_fp16.h
 create mode 100755 cupy/_core/include/cupy/_cuda/cuda-11.1/cuda_fp16.hpp
 create mode 100644 cupy/_core/include/cupy/_cuda/cuda-11/cuda_fp16.h
 create mode 100644 cupy/_core/include/cupy/_cuda/cuda-11/cuda_fp16.hpp
 create mode 100644 cupy/_core/include/cupy/_cuda/cuda-12/cuda_fp16.h
 create mode 100644 cupy/_core/include/cupy/_cuda/cuda-12/cuda_fp16.hpp
 create mode 100644 cupy/_core/include/cupy/atomics.cuh
 create mode 100644 cupy/_core/include/cupy/carray.cuh
 create mode 100644 cupy/_core/include/cupy/complex.cuh
 create mode 100644 cupy/_core/include/cupy/complex/README.md
 create mode 100644 cupy/_core/include/cupy/complex/arithmetic.h
 create mode 100644 cupy/_core/include/cupy/complex/catrig.h
 create mode 100644 cupy/_core/include/cupy/complex/catrigf.h
 create mode 100644 cupy/_core/include/cupy/complex/ccosh.h
 create mode 100644 cupy/_core/include/cupy/complex/ccoshf.h
 create mode 100644 cupy/_core/include/cupy/complex/cexp.h
 create mode 100644 cupy/_core/include/cupy/complex/cexpf.h
 create mode 100644 cupy/_core/include/cupy/complex/clog.h
 create mode 100644 cupy/_core/include/cupy/complex/clogf.h
 create mode 100644 cupy/_core/include/cupy/complex/complex.h
 create mode 100644 cupy/_core/include/cupy/complex/complex_inl.h
 create mode 100644 cupy/_core/include/cupy/complex/cpow.h
 create mode 100644 cupy/_core/include/cupy/complex/cproj.h
 create mode 100644 cupy/_core/include/cupy/complex/csinh.h
 create mode 100644 cupy/_core/include/cupy/complex/csinhf.h
 create mode 100644 cupy/_core/include/cupy/complex/csqrt.h
 create mode 100644 cupy/_core/include/cupy/complex/csqrtf.h
 create mode 100644 cupy/_core/include/cupy/complex/ctanh.h
 create mode 100644 cupy/_core/include/cupy/complex/ctanhf.h
 create mode 100644 cupy/_core/include/cupy/complex/math_private.h
 create mode 100644 cupy/_core/include/cupy/cuComplex_bridge.h
 create mode 100644 cupy/_core/include/cupy/cub/.gitattributes
 create mode 120000 cupy/_core/include/cupy/cub/LICENSE.TXT
 create mode 120000 cupy/_core/include/cupy/cub/cub
 create mode 100644 cupy/_core/include/cupy/cuda_workaround.h
 create mode 100644 cupy/_core/include/cupy/dlpack/README.md
 create mode 100644 cupy/_core/include/cupy/dlpack/dlpack.h
 create mode 100644 cupy/_core/include/cupy/hip_workaround.cuh
 create mode 100644 cupy/_core/include/cupy/jitify/.clang-format
 create mode 100644 cupy/_core/include/cupy/jitify/.gitignore
 create mode 100644 cupy/_core/include/cupy/jitify/Doxyfile
 create mode 100644 cupy/_core/include/cupy/jitify/LICENSE
 create mode 100644 cupy/_core/include/cupy/jitify/Makefile
 create mode 100644 cupy/_core/include/cupy/jitify/README.md
 create mode 100644 cupy/_core/include/cupy/jitify/example_headers/class_arg_kernel.cuh
 create mode 100644 cupy/_core/include/cupy/jitify/example_headers/constant_header.cuh
 create mode 100644 cupy/_core/include/cupy/jitify/example_headers/my_header1.cuh
 create mode 100644 cupy/_core/include/cupy/jitify/example_headers/my_header2.cuh
 create mode 100644 cupy/_core/include/cupy/jitify/example_headers/my_header3.cuh
 create mode 100644 cupy/_core/include/cupy/jitify/jitify.hpp
 create mode 100644 cupy/_core/include/cupy/jitify/jitify_example.cpp
 create mode 100644 cupy/_core/include/cupy/jitify/jitify_test.cu
 create mode 100644 cupy/_core/include/cupy/jitify/stringify.cpp
 create mode 100644 cupy/_core/include/cupy/math_constants.h
 create mode 100644 cupy/_core/include/cupy/pair.cuh
 create mode 100644 cupy/_core/include/cupy/swap.cuh
 create mode 100644 cupy/_core/include/cupy/tuple.cuh
 create mode 100644 cupy/_core/include/cupy/tuple/pair.h
 create mode 100644 cupy/_core/include/cupy/tuple/tuple.h
 create mode 100644 cupy/_core/include/cupy/tuple/type_traits.h
 create mode 100644 cupy/_core/include/cupy/type_dispatcher.cuh
 create mode 100644 cupy/_core/internal.pxd
 create mode 100644 cupy/_core/internal.pyx
 create mode 100644 cupy/_core/new_fusion.pyx
 create mode 100644 cupy/_core/raw.pxd
 create mode 100644 cupy/_core/raw.pyx
 create mode 100644 cupy/_core/syncdetect.py
 create mode 100644 cupy/_creation/__init__.py
 create mode 100644 cupy/_creation/basic.py
 create mode 100644 cupy/_creation/from_data.py
 create mode 100644 cupy/_creation/matrix.py
 create mode 100644 cupy/_creation/ranges.py
 create mode 100644 cupy/_environment.py
 create mode 100644 cupy/_functional/__init__.py
 create mode 100644 cupy/_functional/piecewise.py
 create mode 100644 cupy/_functional/vectorize.py
 create mode 100644 cupy/_indexing/__init__.py
 create mode 100644 cupy/_indexing/generate.py
 create mode 100644 cupy/_indexing/indexing.py
 create mode 100644 cupy/_indexing/insert.py
 create mode 100644 cupy/_indexing/iterate.py
 create mode 100644 cupy/_io/__init__.py
 create mode 100644 cupy/_io/formatting.py
 create mode 100644 cupy/_io/npz.py
 create mode 100644 cupy/_io/text.py
 create mode 100644 cupy/_logic/__init__.py
 create mode 100644 cupy/_logic/comparison.py
 create mode 100644 cupy/_logic/content.py
 create mode 100644 cupy/_logic/ops.py
 create mode 100644 cupy/_logic/truth.py
 create mode 100644 cupy/_logic/type_testing.py
 create mode 100644 cupy/_manipulation/__init__.py
 create mode 100644 cupy/_manipulation/add_remove.py
 create mode 100644 cupy/_manipulation/basic.py
 create mode 100644 cupy/_manipulation/dims.py
 create mode 100644 cupy/_manipulation/join.py
 create mode 100644 cupy/_manipulation/kind.py
 create mode 100644 cupy/_manipulation/rearrange.py
 create mode 100644 cupy/_manipulation/shape.py
 create mode 100644 cupy/_manipulation/split.py
 create mode 100644 cupy/_manipulation/tiling.py
 create mode 100644 cupy/_manipulation/transpose.py
 create mode 100644 cupy/_math/__init__.py
 create mode 100644 cupy/_math/arithmetic.py
 create mode 100644 cupy/_math/explog.py
 create mode 100644 cupy/_math/floating.py
 create mode 100644 cupy/_math/hyperbolic.py
 create mode 100644 cupy/_math/misc.py
 create mode 100644 cupy/_math/rational.py
 create mode 100644 cupy/_math/rounding.py
 create mode 100644 cupy/_math/special.py
 create mode 100644 cupy/_math/sumprod.py
 create mode 100644 cupy/_math/trigonometric.py
 create mode 100644 cupy/_math/ufunc.py
 create mode 100644 cupy/_math/window.py
 create mode 100644 cupy/_misc/__init__.py
 create mode 100644 cupy/_misc/byte_bounds.py
 create mode 100644 cupy/_misc/memory_ranges.py
 create mode 100644 cupy/_misc/who.py
 create mode 100644 cupy/_padding/__init__.py
 create mode 100644 cupy/_padding/pad.py
 create mode 100644 cupy/_sorting/__init__.py
 create mode 100644 cupy/_sorting/count.py
 create mode 100644 cupy/_sorting/search.py
 create mode 100644 cupy/_sorting/sort.py
 create mode 100644 cupy/_statistics/__init__.py
 create mode 100644 cupy/_statistics/correlation.py
 create mode 100644 cupy/_statistics/histogram.py
 create mode 100644 cupy/_statistics/meanvar.py
 create mode 100644 cupy/_statistics/order.py
 create mode 100644 cupy/_util.pyi
 create mode 100644 cupy/_util.pyx
 create mode 100644 cupy/_version.py
 create mode 100644 cupy/array_api/__init__.py
 create mode 100644 cupy/array_api/_array_object.py
 create mode 100644 cupy/array_api/_constants.py
 create mode 100644 cupy/array_api/_creation_functions.py
 create mode 100644 cupy/array_api/_data_type_functions.py
 create mode 100644 cupy/array_api/_dtypes.py
 create mode 100644 cupy/array_api/_elementwise_functions.py
 create mode 100644 cupy/array_api/_indexing_functions.py
 create mode 100644 cupy/array_api/_manipulation_functions.py
 create mode 100644 cupy/array_api/_searching_functions.py
 create mode 100644 cupy/array_api/_set_functions.py
 create mode 100644 cupy/array_api/_sorting_functions.py
 create mode 100644 cupy/array_api/_statistical_functions.py
 create mode 100644 cupy/array_api/_typing.py
 create mode 100644 cupy/array_api/_utility_functions.py
 create mode 100644 cupy/array_api/linalg.py
 create mode 100644 cupy/cublas.py
 create mode 100644 cupy/cuda/__init__.pxd
 create mode 100644 cupy/cuda/__init__.py
 create mode 100644 cupy/cuda/common.pxd
 create mode 100644 cupy/cuda/common.pyx
 create mode 100644 cupy/cuda/compiler.py
 create mode 100644 cupy/cuda/cub.pxd
 create mode 100644 cupy/cuda/cub.pyx
 create mode 100644 cupy/cuda/cudnn.py
 create mode 100644 cupy/cuda/cufft.pxd
 create mode 100644 cupy/cuda/cufft.pyx
 create mode 100644 cupy/cuda/cupy_cub.cu
 create mode 100644 cupy/cuda/cupy_cub.h
 create mode 100644 cupy/cuda/cupy_cufft.h
 create mode 100644 cupy/cuda/cupy_cufftXt.cu
 create mode 100644 cupy/cuda/cupy_cufftXt.h
 create mode 100644 cupy/cuda/cupy_jitify.h
 create mode 100644 cupy/cuda/cupy_thrust.cu
 create mode 100644 cupy/cuda/cupy_thrust.h
 create mode 100644 cupy/cuda/cutensor.py
 create mode 100644 cupy/cuda/device.pxd
 create mode 100644 cupy/cuda/device.pyx
 create mode 100644 cupy/cuda/function.pxd
 create mode 100644 cupy/cuda/function.pyx
 create mode 100644 cupy/cuda/graph.pxd
 create mode 100644 cupy/cuda/graph.pyx
 create mode 100644 cupy/cuda/jitify.pyx
 create mode 100644 cupy/cuda/memory.pxd
 create mode 100644 cupy/cuda/memory.pyx
 create mode 100644 cupy/cuda/memory_hook.pxd
 create mode 100644 cupy/cuda/memory_hook.pyx
 create mode 100644 cupy/cuda/memory_hooks/__init__.py
 create mode 100644 cupy/cuda/memory_hooks/debug_print.py
 create mode 100644 cupy/cuda/memory_hooks/line_profile.py
 create mode 100644 cupy/cuda/nccl.py
 create mode 100644 cupy/cuda/nvtx.py
 create mode 100644 cupy/cuda/pinned_memory.pxd
 create mode 100644 cupy/cuda/pinned_memory.pyx
 create mode 100644 cupy/cuda/profiler.py
 create mode 100644 cupy/cuda/runtime.py
 create mode 100644 cupy/cuda/stream.pxd
 create mode 100644 cupy/cuda/stream.pyx
 create mode 100644 cupy/cuda/texture.pxd
 create mode 100644 cupy/cuda/texture.pyx
 create mode 100644 cupy/cuda/thrust.pyx
 create mode 100644 cupy/cudnn.py
 create mode 100644 cupy/cusolver.py
 create mode 100644 cupy/cusparse.py
 create mode 100644 cupy/cutensor.py
 create mode 100644 cupy/fft/__init__.pxd
 create mode 100644 cupy/fft/__init__.py
 create mode 100644 cupy/fft/_cache.pyx
 create mode 100644 cupy/fft/_callback.pyx
 create mode 100644 cupy/fft/_fft.py
 create mode 100644 cupy/fft/config.py
 create mode 100644 cupy/lib/__init__.py
 create mode 100644 cupy/lib/_polynomial.pyx
 create mode 100644 cupy/lib/_routines_poly.py
 create mode 100644 cupy/lib/_shape_base.py
 create mode 100644 cupy/lib/stride_tricks.py
 create mode 100644 cupy/linalg/__init__.py
 create mode 100644 cupy/linalg/_decomposition.py
 create mode 100644 cupy/linalg/_eigenvalue.py
 create mode 100644 cupy/linalg/_einsum.py
 create mode 100644 cupy/linalg/_einsum_cutn.py
 create mode 100644 cupy/linalg/_einsum_opt.py
 create mode 100644 cupy/linalg/_norms.py
 create mode 100644 cupy/linalg/_product.py
 create mode 100644 cupy/linalg/_solve.py
 create mode 100644 cupy/linalg/_util.py
 create mode 100644 cupy/polynomial/__init__.py
 create mode 100644 cupy/polynomial/polynomial.py
 create mode 100644 cupy/polynomial/polyutils.py
 create mode 100644 cupy/prof/__init__.py
 create mode 100644 cupy/prof/_time_range.py
 create mode 100644 cupy/random/LICENSE
 create mode 100644 cupy/random/__init__.py
 create mode 100644 cupy/random/_bit_generator.pyx
 create mode 100644 cupy/random/_distributions.py
 create mode 100644 cupy/random/_generator.py
 create mode 100644 cupy/random/_generator_api.pyx
 create mode 100644 cupy/random/_kernels.py
 create mode 100644 cupy/random/_permutations.py
 create mode 100644 cupy/random/_sample.py
 create mode 100644 cupy/random/cupy_distributions.cu
 create mode 100644 cupy/random/cupy_distributions.cuh
 create mode 100644 cupy/sparse/__init__.py
 create mode 100644 cupy/sparse/linalg/__init__.py
 create mode 100644 cupy/testing/__init__.py
 create mode 100644 cupy/testing/_array.py
 create mode 100644 cupy/testing/_attr.py
 create mode 100644 cupy/testing/_bundle.py
 create mode 100644 cupy/testing/_condition.py
 create mode 100644 cupy/testing/_helper.py
 create mode 100644 cupy/testing/_hypothesis.py
 create mode 100644 cupy/testing/_loops.py
 create mode 100644 cupy/testing/_parameterized.py
 create mode 100644 cupy/testing/_pytest_impl.py
 create mode 100644 cupy/testing/_random.py
 create mode 100644 cupy/typing/__init__.py
 create mode 100644 cupy/typing/_generic_alias.py
 create mode 100644 cupy_backends/__init__.pxd
 create mode 100755 cupy_backends/__init__.py
 create mode 100644 cupy_backends/cuda/__init__.pxd
 create mode 100644 cupy_backends/cuda/__init__.py
 create mode 100644 cupy_backends/cuda/_softlink.pxd
 create mode 100644 cupy_backends/cuda/_softlink.pyx
 create mode 100644 cupy_backends/cuda/api/__init__.pxd
 create mode 100644 cupy_backends/cuda/api/__init__.py
 create mode 100644 cupy_backends/cuda/api/_driver_enum.pxd
 create mode 100644 cupy_backends/cuda/api/_driver_enum.pyx
 create mode 100644 cupy_backends/cuda/api/_driver_extern.pxi
 create mode 100644 cupy_backends/cuda/api/_driver_typedef.pxi
 create mode 100644 cupy_backends/cuda/api/_runtime_enum.pxd
 create mode 100644 cupy_backends/cuda/api/_runtime_enum.pyx
 create mode 100644 cupy_backends/cuda/api/_runtime_extern.pxi
 create mode 100644 cupy_backends/cuda/api/_runtime_typedef.pxi
 create mode 100644 cupy_backends/cuda/api/driver.pxd
 create mode 100644 cupy_backends/cuda/api/driver.pyx
 create mode 100644 cupy_backends/cuda/api/runtime.pxd
 create mode 100644 cupy_backends/cuda/api/runtime.pyx
 create mode 100644 cupy_backends/cuda/cupy_cublas.h
 create mode 100644 cupy_backends/cuda/cupy_cuda.h
 create mode 100644 cupy_backends/cuda/cupy_cuda_profiler_api.h
 create mode 100644 cupy_backends/cuda/cupy_cuda_runtime.h
 create mode 100644 cupy_backends/cuda/cupy_cudnn.h
 create mode 100644 cupy_backends/cuda/cupy_cusolver.h
 create mode 100644 cupy_backends/cuda/cupy_cusparse.h
 create mode 100644 cupy_backends/cuda/cupy_cutensor.h
 create mode 100644 cupy_backends/cuda/cupy_nccl.h
 create mode 100644 cupy_backends/cuda/cupy_nvrtc.h
 create mode 100644 cupy_backends/cuda/libs/__init__.pxd
 create mode 100644 cupy_backends/cuda/libs/__init__.py
 create mode 100644 cupy_backends/cuda/libs/cublas.pxd
 create mode 100644 cupy_backends/cuda/libs/cublas.pyx
 create mode 100644 cupy_backends/cuda/libs/cudnn.pxd
 create mode 100644 cupy_backends/cuda/libs/cudnn.pyx
 create mode 100644 cupy_backends/cuda/libs/curand.pxd
 create mode 100644 cupy_backends/cuda/libs/curand.pyx
 create mode 100644 cupy_backends/cuda/libs/cusolver.pxd
 create mode 100644 cupy_backends/cuda/libs/cusolver.pyx
 create mode 100644 cupy_backends/cuda/libs/cusparse.pxd
 create mode 100644 cupy_backends/cuda/libs/cusparse.pyx
 create mode 100644 cupy_backends/cuda/libs/cusparselt.pxd
 create mode 100644 cupy_backends/cuda/libs/cusparselt.pyx
 create mode 100644 cupy_backends/cuda/libs/cutensor.pxd
 create mode 100644 cupy_backends/cuda/libs/cutensor.pyx
 create mode 100644 cupy_backends/cuda/libs/nccl.pxd
 create mode 100644 cupy_backends/cuda/libs/nccl.pyx
 create mode 100644 cupy_backends/cuda/libs/nvrtc.pxd
 create mode 100644 cupy_backends/cuda/libs/nvrtc.pyx
 create mode 100644 cupy_backends/cuda/libs/nvtx.pyx
 create mode 100644 cupy_backends/cuda/libs/profiler.pxd
 create mode 100644 cupy_backends/cuda/libs/profiler.pyx
 create mode 100755 cupy_backends/cuda/stream.pxd
 create mode 100755 cupy_backends/cuda/stream.pyx
 create mode 100644 cupy_backends/cupy_backend.h
 create mode 100644 cupy_backends/cupy_backend_runtime.h
 create mode 100644 cupy_backends/cupy_blas.h
 create mode 100644 cupy_backends/cupy_complex.h
 create mode 100644 cupy_backends/cupy_cudnn.h
 create mode 100644 cupy_backends/cupy_cusparselt.h
 create mode 100644 cupy_backends/cupy_cutensor.h
 create mode 100644 cupy_backends/cupy_lapack.h
 create mode 100644 cupy_backends/cupy_nccl.h
 create mode 100644 cupy_backends/cupy_profiler.h
 create mode 100644 cupy_backends/cupy_rand.h
 create mode 100644 cupy_backends/cupy_rtc.h
 create mode 100644 cupy_backends/cupy_sparse.h
 create mode 100644 cupy_backends/cupy_tx.h
 create mode 100644 cupy_backends/hip/cupy_cuComplex.h
 create mode 100644 cupy_backends/hip/cupy_hip.h
 create mode 100644 cupy_backends/hip/cupy_hip_common.h
 create mode 100644 cupy_backends/hip/cupy_hip_runtime.h
 create mode 100644 cupy_backends/hip/cupy_hipblas.h
 create mode 100644 cupy_backends/hip/cupy_hiprand.h
 create mode 100644 cupy_backends/hip/cupy_hiprtc.h
 create mode 100644 cupy_backends/hip/cupy_hipsparse.h
 create mode 100644 cupy_backends/hip/cupy_profiler.h
 create mode 100644 cupy_backends/hip/cupy_rccl.h
 create mode 100644 cupy_backends/hip/cupy_rocsolver.h
 create mode 100644 cupy_backends/hip/cupy_roctx.h
 create mode 100644 cupy_backends/stub/cupy_cuComplex.h
 create mode 100644 cupy_backends/stub/cupy_cublas.h
 create mode 100644 cupy_backends/stub/cupy_cuda.h
 create mode 100644 cupy_backends/stub/cupy_cuda_common.h
 create mode 100644 cupy_backends/stub/cupy_cuda_runtime.h
 create mode 100644 cupy_backends/stub/cupy_cudnn.h
 create mode 100644 cupy_backends/stub/cupy_curand.h
 create mode 100644 cupy_backends/stub/cupy_cusolver.h
 create mode 100644 cupy_backends/stub/cupy_cusparse.h
 create mode 100644 cupy_backends/stub/cupy_cusparselt.h
 create mode 100644 cupy_backends/stub/cupy_cutensor.h
 create mode 100644 cupy_backends/stub/cupy_nccl.h
 create mode 100644 cupy_backends/stub/cupy_nvrtc.h
 create mode 100644 cupy_backends/stub/cupy_nvtx.h
 create mode 100644 cupy_backends/stub/cupy_profiler.h
 create mode 100644 cupyx/__init__.py
 create mode 100644 cupyx/_gufunc.py
 create mode 100644 cupyx/_pinned_array.py
 create mode 100644 cupyx/_rsqrt.py
 create mode 100644 cupyx/_runtime.py
 create mode 100644 cupyx/_scatter.py
 create mode 100644 cupyx/_texture.py
 create mode 100644 cupyx/_ufunc_config.py
 create mode 100644 cupyx/cudnn.pyx
 create mode 100644 cupyx/cusolver.pyx
 create mode 100644 cupyx/cusparse.py
 create mode 100644 cupyx/cutensor.pyx
 create mode 100644 cupyx/distributed/__init__.py
 create mode 100644 cupyx/distributed/_array.py
 create mode 100644 cupyx/distributed/_comm.py
 create mode 100644 cupyx/distributed/_init.py
 create mode 100644 cupyx/distributed/_klv_utils.py
 create mode 100644 cupyx/distributed/_nccl_comm.py
 create mode 100644 cupyx/distributed/_store.py
 create mode 100644 cupyx/distributed/_store_actions.py
 create mode 100644 cupyx/fallback_mode/__init__.py
 create mode 100644 cupyx/fallback_mode/fallback.py
 create mode 100644 cupyx/fallback_mode/notification.py
 create mode 100644 cupyx/jit/__init__.py
 create mode 100644 cupyx/jit/_builtin_funcs.py
 create mode 100644 cupyx/jit/_compile.py
 create mode 100644 cupyx/jit/_cuda_typerules.py
 create mode 100644 cupyx/jit/_cuda_types.py
 create mode 100644 cupyx/jit/_interface.py
 create mode 100644 cupyx/jit/_internal_types.py
 create mode 100644 cupyx/jit/cg.py
 create mode 100644 cupyx/jit/cub.py
 create mode 100644 cupyx/jit/thrust.py
 create mode 100644 cupyx/lapack.py
 create mode 100644 cupyx/linalg/__init__.py
 create mode 100644 cupyx/linalg/_solve.py
 create mode 100644 cupyx/linalg/sparse/__init__.py
 create mode 100644 cupyx/linalg/sparse/_solve.py
 create mode 100644 cupyx/optimizing/__init__.py
 create mode 100644 cupyx/optimizing/_optimize.py
 create mode 100644 cupyx/profiler/__init__.py
 create mode 100644 cupyx/profiler/_time.py
 create mode 100644 cupyx/profiler/_time_range.py
 create mode 100644 cupyx/scipy/__init__.py
 create mode 100644 cupyx/scipy/_lib/__init__.py
 create mode 100644 cupyx/scipy/_lib/_util.py
 create mode 100644 cupyx/scipy/fft/__init__.py
 create mode 100644 cupyx/scipy/fft/_fft.py
 create mode 100644 cupyx/scipy/fft/_fftlog.py
 create mode 100644 cupyx/scipy/fft/_helper.py
 create mode 100644 cupyx/scipy/fft/_realtransforms.py
 create mode 100644 cupyx/scipy/fftpack/__init__.py
 create mode 100644 cupyx/scipy/fftpack/_fft.py
 create mode 100644 cupyx/scipy/interpolate/__init__.py
 create mode 100644 cupyx/scipy/interpolate/_bspline.py
 create mode 100644 cupyx/scipy/interpolate/_bspline2.py
 create mode 100644 cupyx/scipy/interpolate/_cubic.py
 create mode 100644 cupyx/scipy/interpolate/_interpolate.py
 create mode 100644 cupyx/scipy/interpolate/_polyint.py
 create mode 100644 cupyx/scipy/interpolate/_rbfinterp.py
 create mode 100644 cupyx/scipy/interpolate/_rgi.py
 create mode 100644 cupyx/scipy/linalg/__init__.py
 create mode 100644 cupyx/scipy/linalg/_decomp_lu.py
 create mode 100644 cupyx/scipy/linalg/_solve_triangular.py
 create mode 100644 cupyx/scipy/linalg/_special_matrices.py
 create mode 100644 cupyx/scipy/linalg/_uarray.py
 create mode 100644 cupyx/scipy/ndimage/__init__.py
 create mode 100644 cupyx/scipy/ndimage/_filters.py
 create mode 100644 cupyx/scipy/ndimage/_filters_core.py
 create mode 100644 cupyx/scipy/ndimage/_filters_generic.py
 create mode 100644 cupyx/scipy/ndimage/_fourier.py
 create mode 100644 cupyx/scipy/ndimage/_interp_kernels.py
 create mode 100644 cupyx/scipy/ndimage/_interpolation.py
 create mode 100644 cupyx/scipy/ndimage/_measurements.py
 create mode 100644 cupyx/scipy/ndimage/_morphology.py
 create mode 100644 cupyx/scipy/ndimage/_spline_kernel_weights.py
 create mode 100644 cupyx/scipy/ndimage/_spline_prefilter_core.py
 create mode 100644 cupyx/scipy/ndimage/_util.py
 create mode 100644 cupyx/scipy/signal/__init__.py
 create mode 100644 cupyx/scipy/signal/_bsplines.py
 create mode 100644 cupyx/scipy/signal/_signaltools.py
 create mode 100644 cupyx/scipy/signal/_signaltools_core.py
 create mode 100644 cupyx/scipy/sparse/__init__.py
 create mode 100644 cupyx/scipy/sparse/_base.py
 create mode 100644 cupyx/scipy/sparse/_compressed.py
 create mode 100644 cupyx/scipy/sparse/_construct.py
 create mode 100644 cupyx/scipy/sparse/_coo.py
 create mode 100644 cupyx/scipy/sparse/_csc.py
 create mode 100644 cupyx/scipy/sparse/_csr.py
 create mode 100644 cupyx/scipy/sparse/_data.py
 create mode 100644 cupyx/scipy/sparse/_dia.py
 create mode 100644 cupyx/scipy/sparse/_extract.py
 create mode 100644 cupyx/scipy/sparse/_index.py
 create mode 100644 cupyx/scipy/sparse/_sputils.py
 create mode 100644 cupyx/scipy/sparse/_util.py
 create mode 100644 cupyx/scipy/sparse/csgraph/__init__.py
 create mode 100644 cupyx/scipy/sparse/csgraph/_traversal.py
 create mode 100644 cupyx/scipy/sparse/linalg/__init__.py
 create mode 100644 cupyx/scipy/sparse/linalg/_eigen.py
 create mode 100644 cupyx/scipy/sparse/linalg/_interface.py
 create mode 100644 cupyx/scipy/sparse/linalg/_iterative.py
 create mode 100644 cupyx/scipy/sparse/linalg/_lobpcg.py
 create mode 100644 cupyx/scipy/sparse/linalg/_norm.py
 create mode 100644 cupyx/scipy/sparse/linalg/_solve.py
 create mode 100644 cupyx/scipy/spatial/__init__.py
 create mode 100644 cupyx/scipy/spatial/distance.py
 create mode 100644 cupyx/scipy/special/__init__.py
 create mode 100644 cupyx/scipy/special/_basic.py
 create mode 100644 cupyx/scipy/special/_bessel.py
 create mode 100644 cupyx/scipy/special/_beta.py
 create mode 100644 cupyx/scipy/special/_complexstuff.py
 create mode 100644 cupyx/scipy/special/_convex_analysis.py
 create mode 100644 cupyx/scipy/special/_digamma.py
 create mode 100644 cupyx/scipy/special/_erf.py
 create mode 100644 cupyx/scipy/special/_exp1.py
 create mode 100644 cupyx/scipy/special/_expi.py
 create mode 100644 cupyx/scipy/special/_expn.py
 create mode 100644 cupyx/scipy/special/_gamma.py
 create mode 100644 cupyx/scipy/special/_gammainc.py
 create mode 100644 cupyx/scipy/special/_gammaln.py
 create mode 100644 cupyx/scipy/special/_gammasgn.py
 create mode 100644 cupyx/scipy/special/_loggamma.py
 create mode 100644 cupyx/scipy/special/_logsoftmax.py
 create mode 100644 cupyx/scipy/special/_logsumexp.py
 create mode 100644 cupyx/scipy/special/_lpmv.py
 create mode 100644 cupyx/scipy/special/_poch.py
 create mode 100644 cupyx/scipy/special/_polygamma.py
 create mode 100644 cupyx/scipy/special/_softmax.py
 create mode 100644 cupyx/scipy/special/_sph_harm.py
 create mode 100644 cupyx/scipy/special/_statistics.py
 create mode 100644 cupyx/scipy/special/_stats_distributions.py
 create mode 100644 cupyx/scipy/special/_trig.py
 create mode 100644 cupyx/scipy/special/_xlogy.py
 create mode 100644 cupyx/scipy/special/_zeta.py
 create mode 100644 cupyx/scipy/stats/__init__.py
 create mode 100644 cupyx/scipy/stats/_distributions.py
 create mode 100644 cupyx/scipy/stats/_morestats.py
 create mode 100644 cupyx/scipy/stats/_stats.py
 create mode 100644 cupyx/scipy/stats/_stats_py.py
 create mode 100644 cupyx/time.py
 create mode 100644 cupyx/tools/__init__.py
 create mode 100644 cupyx/tools/_hipsparse_stub_mapper.py
 create mode 100755 cupyx/tools/install_library.py
 create mode 100644 docker/python3/Dockerfile
 create mode 100644 docker/rocm/Dockerfile
 create mode 100644 docker/rocm/README.md
 create mode 100644 docs/LICENSE_THIRD_PARTY
 create mode 100644 docs/Makefile
 create mode 100644 docs/image/cupy_logo_1000px.png
 create mode 100644 docs/make.bat
 create mode 100644 docs/requirements.txt
 create mode 100644 docs/source/.gitignore
 create mode 100644 docs/source/_comparison_generator.py
 create mode 100644 docs/source/_static/favicon.ico
 create mode 100644 docs/source/_templates/autosummary/class.rst
 create mode 100644 docs/source/conf.py
 create mode 100644 docs/source/contribution.rst
 create mode 100644 docs/source/index.rst
 create mode 100644 docs/source/install.rst
 create mode 100644 docs/source/license.rst
 create mode 100644 docs/source/overview.rst
 create mode 100644 docs/source/reference/_deprecated.rst
 create mode 100644 docs/source/reference/_private.rst
 create mode 100644 docs/source/reference/array_api.rst
 create mode 100644 docs/source/reference/array_api_array.rst
 create mode 100644 docs/source/reference/array_api_functions.rst
 create mode 100644 docs/source/reference/binary.rst
 create mode 100644 docs/source/reference/comparison.rst
 create mode 100644 docs/source/reference/creation.rst
 create mode 100644 docs/source/reference/cuda.rst
 create mode 100644 docs/source/reference/distributed.rst
 create mode 100644 docs/source/reference/dtype.rst
 create mode 100644 docs/source/reference/environment.rst
 create mode 100644 docs/source/reference/ext.rst
 create mode 100644 docs/source/reference/fft.rst
 create mode 100644 docs/source/reference/fftpack.rst
 create mode 100644 docs/source/reference/functional.rst
 create mode 100644 docs/source/reference/index.rst
 create mode 100644 docs/source/reference/indexing.rst
 create mode 100644 docs/source/reference/io.rst
 create mode 100644 docs/source/reference/kernel.rst
 create mode 100644 docs/source/reference/linalg.rst
 create mode 100644 docs/source/reference/logic.rst
 create mode 100644 docs/source/reference/manipulation.rst
 create mode 100644 docs/source/reference/math.rst
 create mode 100644 docs/source/reference/misc.rst
 create mode 100644 docs/source/reference/ndarray.rst
 create mode 100644 docs/source/reference/ndimage.rst
 create mode 100644 docs/source/reference/pad.rst
 create mode 100644 docs/source/reference/polynomials.rst
 create mode 100644 docs/source/reference/random.rst
 create mode 100644 docs/source/reference/routines.rst
 create mode 100644 docs/source/reference/scipy.rst
 create mode 100644 docs/source/reference/scipy_fft.rst
 create mode 100644 docs/source/reference/scipy_fftpack.rst
 create mode 100644 docs/source/reference/scipy_interpolate.rst
 create mode 100644 docs/source/reference/scipy_linalg.rst
 create mode 100644 docs/source/reference/scipy_ndimage.rst
 create mode 100644 docs/source/reference/scipy_signal.rst
 create mode 100644 docs/source/reference/scipy_sparse.rst
 create mode 100644 docs/source/reference/scipy_sparse_csgraph.rst
 create mode 100644 docs/source/reference/scipy_sparse_linalg.rst
 create mode 100644 docs/source/reference/scipy_spatial.rst
 create mode 100644 docs/source/reference/scipy_spatial_distance.rst
 create mode 100644 docs/source/reference/scipy_special.rst
 create mode 100644 docs/source/reference/scipy_stats.rst
 create mode 100644 docs/source/reference/set.rst
 create mode 100644 docs/source/reference/signal.rst
 create mode 100644 docs/source/reference/sorting.rst
 create mode 100644 docs/source/reference/sparse.rst
 create mode 100644 docs/source/reference/special.rst
 create mode 100644 docs/source/reference/statistics.rst
 create mode 100644 docs/source/reference/stats.rst
 create mode 100644 docs/source/reference/testing.rst
 create mode 100644 docs/source/reference/ufunc.rst
 create mode 100644 docs/source/reference/window.rst
 create mode 100644 docs/source/spelling_wordlist.txt
 create mode 100644 docs/source/upgrade.rst
 create mode 100644 docs/source/user_guide/basic.rst
 create mode 100644 docs/source/user_guide/compatibility.rst
 create mode 100644 docs/source/user_guide/cuda_api.rst
 create mode 100644 docs/source/user_guide/difference.rst
 create mode 100644 docs/source/user_guide/fft.rst
 create mode 100644 docs/source/user_guide/index.rst
 create mode 100644 docs/source/user_guide/interoperability.rst
 create mode 100644 docs/source/user_guide/kernel.rst
 create mode 100644 docs/source/user_guide/memory.rst
 create mode 100644 docs/source/user_guide/performance.rst
 create mode 100644 examples/cg/cg.py
 create mode 100644 examples/cusparselt/matmul.py
 create mode 100644 examples/custom_struct/README.md
 create mode 100644 examples/custom_struct/builtin_vectors.py
 create mode 100644 examples/custom_struct/complex_struct.py
 create mode 100644 examples/custom_struct/packed_matrix.py
 create mode 100644 examples/cutensor/contraction.py
 create mode 100644 examples/cutensor/elementwise_binary.py
 create mode 100644 examples/cutensor/elementwise_trinary.py
 create mode 100644 examples/cutensor/reduction.py
 create mode 100644 examples/finance/black_scholes.py
 create mode 100644 examples/finance/monte_carlo.py
 create mode 100644 examples/finance/monte_carlo_multigpu.py
 create mode 100644 examples/gemm/README.md
 create mode 100644 examples/gemm/sgemm.cu
 create mode 100644 examples/gemm/sgemm.py
 create mode 100644 examples/gemm/utils.py
 create mode 100644 examples/gmm/README.md
 create mode 100644 examples/gmm/gmm.py
 create mode 100644 examples/interoperability/mpi4py_multiple_devices.py
 create mode 100644 examples/jit/elmentwise_op.py
 create mode 100644 examples/jit/reduction_atomic.py
 create mode 100644 examples/jit/reduction_simple.py
 create mode 100644 examples/kmeans/README.md
 create mode 100644 examples/kmeans/kmeans.py
 create mode 100644 examples/peer/peer_matrix.py
 create mode 100644 examples/stream/cublas.py
 create mode 100644 examples/stream/cudnn.py
 create mode 100644 examples/stream/cufft.py
 create mode 100644 examples/stream/cupy_event.py
 create mode 100644 examples/stream/cupy_kernel.py
 create mode 100644 examples/stream/cupy_memcpy.py
 create mode 100644 examples/stream/curand.py
 create mode 100644 examples/stream/cusolver.py
 create mode 100644 examples/stream/cusparse.py
 create mode 100644 examples/stream/map_reduce.py
 create mode 100644 examples/stream/thrust.py
 create mode 100644 install/cupy_builder/__init__.py
 create mode 100644 install/cupy_builder/_command.py
 create mode 100644 install/cupy_builder/_compiler.py
 create mode 100644 install/cupy_builder/_context.py
 create mode 100644 install/cupy_builder/_environment.py
 create mode 100644 install/cupy_builder/_features.py
 create mode 100644 install/cupy_builder/_preflight.py
 create mode 100644 install/cupy_builder/cupy_setup_build.py
 create mode 100644 install/cupy_builder/install_build.py
 create mode 100644 install/cupy_builder/install_utils.py
 create mode 100644 install/mypy.ini
 create mode 100644 install/universal_pkg/RELEASE.md
 create mode 100644 install/universal_pkg/__init__.py
 create mode 100644 install/universal_pkg/setup.py
 create mode 100644 setup.cfg
 create mode 100644 setup.py
 create mode 100644 tests/conftest.py
 create mode 100644 tests/cupy_tests/__init__.py
 create mode 100644 tests/cupy_tests/array_api_tests/__init__.py
 create mode 100644 tests/cupy_tests/array_api_tests/test_array_object.py
 create mode 100644 tests/cupy_tests/array_api_tests/test_creation_functions.py
 create mode 100644 tests/cupy_tests/array_api_tests/test_data_type_functions.py
 create mode 100644 tests/cupy_tests/array_api_tests/test_elementwise_functions.py
 create mode 100644 tests/cupy_tests/array_api_tests/test_indexing_functions.py
 create mode 100644 tests/cupy_tests/array_api_tests/test_set_functions.py
 create mode 100644 tests/cupy_tests/array_api_tests/test_sorting_functions.py
 create mode 100644 tests/cupy_tests/array_api_tests/test_validation.py
 create mode 100644 tests/cupy_tests/binary_tests/__init__.py
 create mode 100644 tests/cupy_tests/binary_tests/test_elementwise.py
 create mode 100644 tests/cupy_tests/binary_tests/test_packing.py
 create mode 100644 tests/cupy_tests/core_tests/__init__.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/__init__.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/fusion_utils.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_array.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_example.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_indexing.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_kernel_cache.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_misc.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_optimization.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_reduction.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_routines.py
 create mode 100644 tests/cupy_tests/core_tests/fusion_tests/test_ufunc.py
 create mode 100644 tests/cupy_tests/core_tests/test_array_function.py
 create mode 100644 tests/cupy_tests/core_tests/test_carray.py
 create mode 100644 tests/cupy_tests/core_tests/test_core.py
 create mode 100644 tests/cupy_tests/core_tests/test_cub_reduction.py
 create mode 100644 tests/cupy_tests/core_tests/test_dlpack.py
 create mode 100644 tests/cupy_tests/core_tests/test_elementwise.py
 create mode 100644 tests/cupy_tests/core_tests/test_flags.py
 create mode 100644 tests/cupy_tests/core_tests/test_function.py
 create mode 100644 tests/cupy_tests/core_tests/test_gufuncs.py
 create mode 100644 tests/cupy_tests/core_tests/test_include.py
 create mode 100644 tests/cupy_tests/core_tests/test_internal.py
 create mode 100644 tests/cupy_tests/core_tests/test_iter.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_adv_indexing.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_complex_ops.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_contiguity.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_conversion.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_copy_and_view.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_cuda_array_interface.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_elementwise_op.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_get.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_indexing.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_math.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_owndata.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_reduction.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_scatter.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_ufunc.py
 create mode 100644 tests/cupy_tests/core_tests/test_ndarray_unary_op.py
 create mode 100644 tests/cupy_tests/core_tests/test_raw.py
 create mode 100644 tests/cupy_tests/core_tests/test_reduction.py
 create mode 100644 tests/cupy_tests/core_tests/test_scan.py
 create mode 100644 tests/cupy_tests/core_tests/test_syncdetect.py
 create mode 100644 tests/cupy_tests/core_tests/test_ufunc_methods.py
 create mode 100644 tests/cupy_tests/core_tests/test_userkernel.py
 create mode 100644 tests/cupy_tests/creation_tests/__init__.py
 create mode 100644 tests/cupy_tests/creation_tests/test_basic.py
 create mode 100644 tests/cupy_tests/creation_tests/test_from_data.py
 create mode 100644 tests/cupy_tests/creation_tests/test_matrix.py
 create mode 100644 tests/cupy_tests/creation_tests/test_ranges.py
 create mode 100644 tests/cupy_tests/cuda_tests/__init__.py
 create mode 100644 tests/cupy_tests/cuda_tests/memory_hooks_tests/__init__.py
 create mode 100644 tests/cupy_tests/cuda_tests/memory_hooks_tests/test_debug_print.py
 create mode 100644 tests/cupy_tests/cuda_tests/memory_hooks_tests/test_line_profile.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_compiler.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_cublas.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_cudnn.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_cufft.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_curand.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_cusolver.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_cusparse.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_cutensor.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_device.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_driver.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_graph.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_memory.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_memory_hook.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_nccl.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_nvrtc.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_nvtx.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_pinned_memory.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_profile.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_runtime.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_stream.py
 create mode 100644 tests/cupy_tests/cuda_tests/test_texture.py
 create mode 100644 tests/cupy_tests/fft_tests/__init__.py
 create mode 100644 tests/cupy_tests/fft_tests/test_cache.py
 create mode 100644 tests/cupy_tests/fft_tests/test_callback.py
 create mode 100644 tests/cupy_tests/fft_tests/test_fft.py
 create mode 100644 tests/cupy_tests/functional_tests/__init__.py
 create mode 100644 tests/cupy_tests/functional_tests/test_piecewise.py
 create mode 100644 tests/cupy_tests/functional_tests/test_vectorize.py
 create mode 100644 tests/cupy_tests/indexing_tests/__init__.py
 create mode 100644 tests/cupy_tests/indexing_tests/test_generate.py
 create mode 100644 tests/cupy_tests/indexing_tests/test_indexing.py
 create mode 100644 tests/cupy_tests/indexing_tests/test_insert.py
 create mode 100644 tests/cupy_tests/indexing_tests/test_iterate.py
 create mode 100644 tests/cupy_tests/io_tests/__init__.py
 create mode 100644 tests/cupy_tests/io_tests/test_base_n.py
 create mode 100644 tests/cupy_tests/io_tests/test_formatting.py
 create mode 100644 tests/cupy_tests/io_tests/test_npz.py
 create mode 100644 tests/cupy_tests/io_tests/test_text.py
 create mode 100644 tests/cupy_tests/lib_tests/__init__.py
 create mode 100644 tests/cupy_tests/lib_tests/test_polynomial.py
 create mode 100644 tests/cupy_tests/lib_tests/test_shape_base.py
 create mode 100644 tests/cupy_tests/lib_tests/test_strided_tricks.py
 create mode 100644 tests/cupy_tests/linalg_tests/__init__.py
 create mode 100644 tests/cupy_tests/linalg_tests/test_decomposition.py
 create mode 100644 tests/cupy_tests/linalg_tests/test_eigenvalue.py
 create mode 100644 tests/cupy_tests/linalg_tests/test_einsum.py
 create mode 100644 tests/cupy_tests/linalg_tests/test_norms.py
 create mode 100644 tests/cupy_tests/linalg_tests/test_product.py
 create mode 100644 tests/cupy_tests/linalg_tests/test_solve.py
 create mode 100644 tests/cupy_tests/logic_tests/__init__.py
 create mode 100644 tests/cupy_tests/logic_tests/test_comparison.py
 create mode 100644 tests/cupy_tests/logic_tests/test_content.py
 create mode 100644 tests/cupy_tests/logic_tests/test_ops.py
 create mode 100644 tests/cupy_tests/logic_tests/test_truth.py
 create mode 100644 tests/cupy_tests/logic_tests/test_type_test.py
 create mode 100644 tests/cupy_tests/manipulation_tests/__init__.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_add_remove.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_basic.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_dims.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_join.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_kind.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_rearrange.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_shape.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_split.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_tiling.py
 create mode 100644 tests/cupy_tests/manipulation_tests/test_transpose.py
 create mode 100644 tests/cupy_tests/math_tests/__init__.py
 create mode 100644 tests/cupy_tests/math_tests/test_arithmetic.py
 create mode 100644 tests/cupy_tests/math_tests/test_explog.py
 create mode 100644 tests/cupy_tests/math_tests/test_floating.py
 create mode 100644 tests/cupy_tests/math_tests/test_hyperbolic.py
 create mode 100644 tests/cupy_tests/math_tests/test_matmul.py
 create mode 100644 tests/cupy_tests/math_tests/test_misc.py
 create mode 100644 tests/cupy_tests/math_tests/test_rational.py
 create mode 100644 tests/cupy_tests/math_tests/test_rounding.py
 create mode 100644 tests/cupy_tests/math_tests/test_special.py
 create mode 100644 tests/cupy_tests/math_tests/test_sumprod.py
 create mode 100644 tests/cupy_tests/math_tests/test_trigonometric.py
 create mode 100644 tests/cupy_tests/math_tests/test_window.py
 create mode 100644 tests/cupy_tests/misc_tests/__init__.py
 create mode 100644 tests/cupy_tests/misc_tests/test_byte_bounds.py
 create mode 100644 tests/cupy_tests/misc_tests/test_memory_ranges.py
 create mode 100644 tests/cupy_tests/misc_tests/test_who.py
 create mode 100644 tests/cupy_tests/padding_tests/__init__.py
 create mode 100644 tests/cupy_tests/padding_tests/test_pad.py
 create mode 100644 tests/cupy_tests/polynomial_tests/test_polynomial.py
 create mode 100644 tests/cupy_tests/polynomial_tests/test_polyutils.py
 create mode 100644 tests/cupy_tests/prof_tests/__init__.py
 create mode 100644 tests/cupy_tests/prof_tests/test_range.py
 create mode 100644 tests/cupy_tests/random_tests/__init__.py
 create mode 100644 tests/cupy_tests/random_tests/common_distributions.py
 create mode 100644 tests/cupy_tests/random_tests/test_bit_generator.py
 create mode 100644 tests/cupy_tests/random_tests/test_distributions.py
 create mode 100644 tests/cupy_tests/random_tests/test_generator.py
 create mode 100644 tests/cupy_tests/random_tests/test_generator_api.py
 create mode 100644 tests/cupy_tests/random_tests/test_init.py
 create mode 100644 tests/cupy_tests/random_tests/test_permutations.py
 create mode 100644 tests/cupy_tests/random_tests/test_random.py
 create mode 100644 tests/cupy_tests/random_tests/test_sample.py
 create mode 100644 tests/cupy_tests/sorting_tests/__init__.py
 create mode 100644 tests/cupy_tests/sorting_tests/test_count.py
 create mode 100644 tests/cupy_tests/sorting_tests/test_search.py
 create mode 100644 tests/cupy_tests/sorting_tests/test_sort.py
 create mode 100644 tests/cupy_tests/statistics_tests/__init__.py
 create mode 100644 tests/cupy_tests/statistics_tests/test_correlation.py
 create mode 100644 tests/cupy_tests/statistics_tests/test_histogram.py
 create mode 100644 tests/cupy_tests/statistics_tests/test_meanvar.py
 create mode 100644 tests/cupy_tests/statistics_tests/test_order.py
 create mode 100644 tests/cupy_tests/test_cublas.py
 create mode 100644 tests/cupy_tests/test_init.py
 create mode 100644 tests/cupy_tests/test_ndim.py
 create mode 100644 tests/cupy_tests/test_numpy_interop.py
 create mode 100644 tests/cupy_tests/test_type_routines.py
 create mode 100644 tests/cupy_tests/test_typing.py
 create mode 100644 tests/cupy_tests/testing_tests/__init__.py
 create mode 100644 tests/cupy_tests/testing_tests/test_array.py
 create mode 100644 tests/cupy_tests/testing_tests/test_condition.py
 create mode 100644 tests/cupy_tests/testing_tests/test_helper.py
 create mode 100644 tests/cupy_tests/testing_tests/test_loops.py
 create mode 100644 tests/cupy_tests/testing_tests/test_parameterized.py
 create mode 100644 tests/cupyx_tests/__init__.py
 create mode 100644 tests/cupyx_tests/distributed_tests/comm_runner.py
 create mode 100644 tests/cupyx_tests/distributed_tests/test_array.py
 create mode 100644 tests/cupyx_tests/distributed_tests/test_comm.py
 create mode 100644 tests/cupyx_tests/distributed_tests/test_store.py
 create mode 100644 tests/cupyx_tests/fallback_mode_tests/__init__.py
 create mode 100644 tests/cupyx_tests/fallback_mode_tests/test_fallback.py
 create mode 100644 tests/cupyx_tests/fallback_mode_tests/test_notifications.py
 create mode 100644 tests/cupyx_tests/jit_tests/__init__.py
 create mode 100644 tests/cupyx_tests/jit_tests/test_cooperative_groups.py
 create mode 100644 tests/cupyx_tests/jit_tests/test_cub.py
 create mode 100644 tests/cupyx_tests/jit_tests/test_device_function.py
 create mode 100644 tests/cupyx_tests/jit_tests/test_raw.py
 create mode 100644 tests/cupyx_tests/jit_tests/test_thrust.py
 create mode 100644 tests/cupyx_tests/linalg_tests/__init__.py
 create mode 100644 tests/cupyx_tests/linalg_tests/sparse_tests/__init__.py
 create mode 100644 tests/cupyx_tests/linalg_tests/sparse_tests/test_solve.py
 create mode 100644 tests/cupyx_tests/linalg_tests/test_solve.py
 create mode 100644 tests/cupyx_tests/profiler_tests/__init__.py
 create mode 100644 tests/cupyx_tests/profiler_tests/test_benchmark.py
 create mode 100644 tests/cupyx_tests/profiler_tests/test_profile.py
 create mode 100644 tests/cupyx_tests/profiler_tests/test_time_range.py
 create mode 100644 tests/cupyx_tests/scipy_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/fft_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/fft_tests/test_fft.py
 create mode 100644 tests/cupyx_tests/scipy_tests/fft_tests/test_fftlog.py
 create mode 100644 tests/cupyx_tests/scipy_tests/fft_tests/test_helper.py
 create mode 100644 tests/cupyx_tests/scipy_tests/fft_tests/test_realtransforms.py
 create mode 100644 tests/cupyx_tests/scipy_tests/fftpack_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/fftpack_tests/test_fftpack.py
 create mode 100644 tests/cupyx_tests/scipy_tests/interpolate_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/interpolate_tests/test_bspline.py
 create mode 100644 tests/cupyx_tests/scipy_tests/interpolate_tests/test_bspline2.py
 create mode 100644 tests/cupyx_tests/scipy_tests/interpolate_tests/test_polyint.py
 create mode 100644 tests/cupyx_tests/scipy_tests/interpolate_tests/test_ppoly.py
 create mode 100644 tests/cupyx_tests/scipy_tests/interpolate_tests/test_rbfinterp.py
 create mode 100644 tests/cupyx_tests/scipy_tests/interpolate_tests/test_rgi.py
 create mode 100644 tests/cupyx_tests/scipy_tests/linalg_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/linalg_tests/test_decomp_lu.py
 create mode 100644 tests/cupyx_tests/scipy_tests/linalg_tests/test_solve_triangular.py
 create mode 100644 tests/cupyx_tests/scipy_tests/linalg_tests/test_special_matrices.py
 create mode 100644 tests/cupyx_tests/scipy_tests/linalg_tests/test_uarray.py
 create mode 100644 tests/cupyx_tests/scipy_tests/ndimage_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/ndimage_tests/test_filters.py
 create mode 100644 tests/cupyx_tests/scipy_tests/ndimage_tests/test_fourier.py
 create mode 100644 tests/cupyx_tests/scipy_tests/ndimage_tests/test_interpolation.py
 create mode 100644 tests/cupyx_tests/scipy_tests/ndimage_tests/test_measurements.py
 create mode 100644 tests/cupyx_tests/scipy_tests/ndimage_tests/test_morphology.py
 create mode 100644 tests/cupyx_tests/scipy_tests/signal_tests/test_bsplines.py
 create mode 100644 tests/cupyx_tests/scipy_tests/signal_tests/test_signaltools.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/csgraph_tests/test_traversal.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_base.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_construct.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_coo.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_csc.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_csr.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_dia.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_extract.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_index.py
 create mode 100644 tests/cupyx_tests/scipy_tests/sparse_tests/test_linalg.py
 create mode 100644 tests/cupyx_tests/scipy_tests/spatial_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/spatial_tests/test_distance.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/__init__.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_basic.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_bessel.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_beta.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_convex_analysis.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_digamma.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_erf.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_exp1.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_expi.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_expn.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_gamma.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_gammainc.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_gammaln.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_log_softmax.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_logsumexp.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_polygamma.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_softmax.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_sph_harm.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_statistics.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_ufunc_dispatch.py
 create mode 100644 tests/cupyx_tests/scipy_tests/special_tests/test_zeta.py
 create mode 100644 tests/cupyx_tests/scipy_tests/stats_tests/test_distributions.py
 create mode 100644 tests/cupyx_tests/scipy_tests/stats_tests/test_morestats.py
 create mode 100644 tests/cupyx_tests/scipy_tests/stats_tests/test_stats.py
 create mode 100644 tests/cupyx_tests/scipy_tests/test_get_array_module.py
 create mode 100644 tests/cupyx_tests/test_cudnn.py
 create mode 100644 tests/cupyx_tests/test_cupyx.py
 create mode 100644 tests/cupyx_tests/test_cusolver.py
 create mode 100644 tests/cupyx_tests/test_cusparse.py
 create mode 100644 tests/cupyx_tests/test_cutensor.py
 create mode 100644 tests/cupyx_tests/test_lapack.py
 create mode 100644 tests/cupyx_tests/test_optimize.py
 create mode 100644 tests/cupyx_tests/test_pinned_array.py
 create mode 100644 tests/cupyx_tests/test_rsqrt.py
 create mode 100644 tests/cupyx_tests/test_runtime.py
 create mode 100644 tests/cupyx_tests/test_time.py
 create mode 100644 tests/cupyx_tests/tools_tests/__init__.py
 create mode 100644 tests/cupyx_tests/tools_tests/test_install_library.py
 create mode 100644 tests/example_tests/__init__.py
 create mode 100644 tests/example_tests/example_test.py
 create mode 100644 tests/example_tests/test_custom_struct.py
 create mode 100644 tests/example_tests/test_finance.py
 create mode 100644 tests/example_tests/test_gemm.py
 create mode 100644 tests/example_tests/test_gmm.py
 create mode 100644 tests/example_tests/test_kmeans.py
 create mode 100644 tests/install_tests/__init__.py
 create mode 100644 tests/install_tests/test_build.py
 create mode 100644 tests/install_tests/test_cupy_builder/__init__.py
 create mode 100644 tests/install_tests/test_cupy_builder/test_command.py
 create mode 100644 tests/install_tests/test_cupy_builder/test_context.py
 create mode 100644 tests/install_tests/test_cupy_builder/test_features.py
 create mode 100644 tests/install_tests/test_universal_pkg/__init__.py
 create mode 100644 tests/install_tests/test_universal_pkg/test_setup.py
 create mode 100644 tests/install_tests/test_utils.py
 create mode 100644 third_party/cub/.cproject
 create mode 100644 third_party/cub/.project
 create mode 100644 third_party/cub/.settings/.gitignore
 create mode 100644 third_party/cub/.settings/org.eclipse.cdt.codan.core.prefs
 create mode 100644 third_party/cub/.settings/org.eclipse.cdt.core.prefs
 create mode 100644 third_party/cub/.settings/org.eclipse.cdt.ui.prefs
 create mode 100644 third_party/cub/.settings/org.eclipse.core.runtime.prefs
 create mode 100644 third_party/cub/CHANGE_LOG.TXT
 create mode 100644 third_party/cub/LICENSE.TXT
 create mode 100644 third_party/cub/README.md
 create mode 100644 third_party/cub/common.mk
 create mode 100644 third_party/cub/cub/agent/agent_histogram.cuh
 create mode 100644 third_party/cub/cub/agent/agent_radix_sort_downsweep.cuh
 create mode 100644 third_party/cub/cub/agent/agent_radix_sort_upsweep.cuh
 create mode 100644 third_party/cub/cub/agent/agent_reduce.cuh
 create mode 100644 third_party/cub/cub/agent/agent_reduce_by_key.cuh
 create mode 100644 third_party/cub/cub/agent/agent_rle.cuh
 create mode 100644 third_party/cub/cub/agent/agent_scan.cuh
 create mode 100644 third_party/cub/cub/agent/agent_segment_fixup.cuh
 create mode 100644 third_party/cub/cub/agent/agent_select_if.cuh
 create mode 100644 third_party/cub/cub/agent/agent_spmv_orig.cuh
 create mode 100644 third_party/cub/cub/agent/single_pass_scan_operators.cuh
 create mode 100644 third_party/cub/cub/block/block_adjacent_difference.cuh
 create mode 100644 third_party/cub/cub/block/block_discontinuity.cuh
 create mode 100644 third_party/cub/cub/block/block_exchange.cuh
 create mode 100644 third_party/cub/cub/block/block_histogram.cuh
 create mode 100644 third_party/cub/cub/block/block_load.cuh
 create mode 100644 third_party/cub/cub/block/block_radix_rank.cuh
 create mode 100644 third_party/cub/cub/block/block_radix_sort.cuh
 create mode 100644 third_party/cub/cub/block/block_raking_layout.cuh
 create mode 100644 third_party/cub/cub/block/block_reduce.cuh
 create mode 100644 third_party/cub/cub/block/block_scan.cuh
 create mode 100644 third_party/cub/cub/block/block_shuffle.cuh
 create mode 100644 third_party/cub/cub/block/block_store.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_histogram_atomic.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_histogram_sort.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_reduce_raking.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_reduce_raking_commutative_only.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_reduce_warp_reductions.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_scan_raking.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_scan_warp_scans.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_scan_warp_scans2.cuh
 create mode 100644 third_party/cub/cub/block/specializations/block_scan_warp_scans3.cuh
 create mode 100644 third_party/cub/cub/cub.cuh
 create mode 100644 third_party/cub/cub/device/device_histogram.cuh
 create mode 100644 third_party/cub/cub/device/device_partition.cuh
 create mode 100644 third_party/cub/cub/device/device_radix_sort.cuh
 create mode 100644 third_party/cub/cub/device/device_reduce.cuh
 create mode 100644 third_party/cub/cub/device/device_run_length_encode.cuh
 create mode 100644 third_party/cub/cub/device/device_scan.cuh
 create mode 100644 third_party/cub/cub/device/device_segmented_radix_sort.cuh
 create mode 100644 third_party/cub/cub/device/device_segmented_reduce.cuh
 create mode 100644 third_party/cub/cub/device/device_select.cuh
 create mode 100644 third_party/cub/cub/device/device_spmv.cuh
 create mode 100644 third_party/cub/cub/device/dispatch/dispatch_histogram.cuh
 create mode 100644 third_party/cub/cub/device/dispatch/dispatch_radix_sort.cuh
 create mode 100644 third_party/cub/cub/device/dispatch/dispatch_reduce.cuh
 create mode 100644 third_party/cub/cub/device/dispatch/dispatch_reduce_by_key.cuh
 create mode 100644 third_party/cub/cub/device/dispatch/dispatch_rle.cuh
 create mode 100644 third_party/cub/cub/device/dispatch/dispatch_scan.cuh
 create mode 100644 third_party/cub/cub/device/dispatch/dispatch_select_if.cuh
 create mode 100644 third_party/cub/cub/device/dispatch/dispatch_spmv_orig.cuh
 create mode 100644 third_party/cub/cub/grid/grid_barrier.cuh
 create mode 100644 third_party/cub/cub/grid/grid_even_share.cuh
 create mode 100644 third_party/cub/cub/grid/grid_mapping.cuh
 create mode 100644 third_party/cub/cub/grid/grid_queue.cuh
 create mode 100644 third_party/cub/cub/host/mutex.cuh
 create mode 100644 third_party/cub/cub/iterator/arg_index_input_iterator.cuh
 create mode 100644 third_party/cub/cub/iterator/cache_modified_input_iterator.cuh
 create mode 100644 third_party/cub/cub/iterator/cache_modified_output_iterator.cuh
 create mode 100644 third_party/cub/cub/iterator/constant_input_iterator.cuh
 create mode 100644 third_party/cub/cub/iterator/counting_input_iterator.cuh
 create mode 100644 third_party/cub/cub/iterator/discard_output_iterator.cuh
 create mode 100644 third_party/cub/cub/iterator/tex_obj_input_iterator.cuh
 create mode 100644 third_party/cub/cub/iterator/tex_ref_input_iterator.cuh
 create mode 100644 third_party/cub/cub/iterator/transform_input_iterator.cuh
 create mode 100644 third_party/cub/cub/thread/thread_load.cuh
 create mode 100644 third_party/cub/cub/thread/thread_operators.cuh
 create mode 100644 third_party/cub/cub/thread/thread_reduce.cuh
 create mode 100644 third_party/cub/cub/thread/thread_scan.cuh
 create mode 100644 third_party/cub/cub/thread/thread_search.cuh
 create mode 100644 third_party/cub/cub/thread/thread_store.cuh
 create mode 100644 third_party/cub/cub/util_allocator.cuh
 create mode 100644 third_party/cub/cub/util_arch.cuh
 create mode 100644 third_party/cub/cub/util_debug.cuh
 create mode 100644 third_party/cub/cub/util_device.cuh
 create mode 100644 third_party/cub/cub/util_macro.cuh
 create mode 100644 third_party/cub/cub/util_namespace.cuh
 create mode 100644 third_party/cub/cub/util_ptx.cuh
 create mode 100644 third_party/cub/cub/util_type.cuh
 create mode 100644 third_party/cub/cub/warp/specializations/warp_reduce_shfl.cuh
 create mode 100644 third_party/cub/cub/warp/specializations/warp_reduce_smem.cuh
 create mode 100644 third_party/cub/cub/warp/specializations/warp_scan_shfl.cuh
 create mode 100644 third_party/cub/cub/warp/specializations/warp_scan_smem.cuh
 create mode 100644 third_party/cub/cub/warp/warp_reduce.cuh
 create mode 100644 third_party/cub/cub/warp/warp_scan.cuh
 create mode 100644 third_party/cub/eclipse code style profile.xml
 create mode 100644 third_party/cub/examples/block/.gitignore
 create mode 100644 third_party/cub/examples/block/Makefile
 create mode 100644 third_party/cub/examples/block/example_block_radix_sort.cu
 create mode 100644 third_party/cub/examples/block/example_block_reduce.cu
 create mode 100644 third_party/cub/examples/block/example_block_scan.cu
 create mode 100644 third_party/cub/examples/block/reduce_by_key.cu
 create mode 100644 third_party/cub/examples/device/.gitignore
 create mode 100644 third_party/cub/examples/device/Makefile
 create mode 100644 third_party/cub/examples/device/example_device_partition_flagged.cu
 create mode 100644 third_party/cub/examples/device/example_device_partition_if.cu
 create mode 100644 third_party/cub/examples/device/example_device_radix_sort.cu
 create mode 100644 third_party/cub/examples/device/example_device_reduce.cu
 create mode 100644 third_party/cub/examples/device/example_device_scan.cu
 create mode 100644 third_party/cub/examples/device/example_device_select_flagged.cu
 create mode 100644 third_party/cub/examples/device/example_device_select_if.cu
 create mode 100644 third_party/cub/examples/device/example_device_select_unique.cu
 create mode 100644 third_party/cub/examples/device/example_device_sort_find_non_trivial_runs.cu
 create mode 100644 third_party/cub/experimental/.gitignore
 create mode 100644 third_party/cub/experimental/Makefile
 create mode 100644 third_party/cub/experimental/defunct/example_coo_spmv.cu
 create mode 100644 third_party/cub/experimental/defunct/test_device_seg_reduce.cu
 create mode 100644 third_party/cub/experimental/histogram/histogram_cub.h
 create mode 100644 third_party/cub/experimental/histogram/histogram_gmem_atomics.h
 create mode 100644 third_party/cub/experimental/histogram/histogram_smem_atomics.h
 create mode 100644 third_party/cub/experimental/histogram_compare.cu
 create mode 100644 third_party/cub/experimental/sparse_matrix.h
 create mode 100644 third_party/cub/experimental/spmv_compare.cu
 create mode 100755 third_party/cub/experimental/spmv_script.sh
 create mode 100644 third_party/cub/test/.gitignore
 create mode 100644 third_party/cub/test/Makefile
 create mode 100644 third_party/cub/test/half.h
 create mode 100644 third_party/cub/test/link_a.cu
 create mode 100644 third_party/cub/test/link_b.cu
 create mode 100644 third_party/cub/test/link_main.cpp
 create mode 100644 third_party/cub/test/mersenne.h
 create mode 100644 third_party/cub/test/test_allocator.cu
 create mode 100644 third_party/cub/test/test_block_histogram.cu
 create mode 100644 third_party/cub/test/test_block_load_store.cu
 create mode 100644 third_party/cub/test/test_block_radix_sort.cu
 create mode 100644 third_party/cub/test/test_block_reduce.cu
 create mode 100644 third_party/cub/test/test_block_scan.cu
 create mode 100644 third_party/cub/test/test_device_histogram.cu
 create mode 100644 third_party/cub/test/test_device_radix_sort.cu
 create mode 100644 third_party/cub/test/test_device_reduce.cu
 create mode 100644 third_party/cub/test/test_device_reduce_by_key.cu
 create mode 100644 third_party/cub/test/test_device_run_length_encode.cu
 create mode 100644 third_party/cub/test/test_device_scan.cu
 create mode 100644 third_party/cub/test/test_device_select_if.cu
 create mode 100644 third_party/cub/test/test_device_select_unique.cu
 create mode 100644 third_party/cub/test/test_grid_barrier.cu
 create mode 100644 third_party/cub/test/test_iterator.cu
 create mode 100644 third_party/cub/test/test_util.h
 create mode 100644 third_party/cub/test/test_warp_reduce.cu
 create mode 100644 third_party/cub/test/test_warp_scan.cu
 create mode 100644 third_party/cub/tune/.gitignore
 create mode 100644 third_party/cub/tune/Makefile
 create mode 100644 third_party/cub/tune/tune_device_reduce.cu

diff --git a/CITATION.bib b/CITATION.bib
new file mode 100644
index 0000000..2b4ee94
--- /dev/null
+++ b/CITATION.bib
@@ -0,0 +1,7 @@
+@inproceedings{cupy_learningsys2017,
+  author       = "Okuta, Ryosuke and Unno, Yuya and Nishino, Daisuke and Hido, Shohei and Loomis, Crissman",
+  title        = "CuPy: A NumPy-Compatible Library for NVIDIA GPU Calculations",
+  booktitle    = "Proceedings of Workshop on Machine Learning Systems (LearningSys) in The Thirty-first Annual Conference on Neural Information Processing Systems (NIPS)",
+  year         = "2017",
+  url          = "http://learningsys.org/nips17/assets/papers/paper_16.pdf"
+}
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000..3ad1a10
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,7 @@
+# CuPy Code of Conduct
+
+CuPy follows the [NumFOCUS Code of Conduct][homepage] available at https://numfocus.org/code-of-conduct.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at `dlfw@preferred.jp`.
+
+[homepage]: https://numfocus.org/
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..de1113b
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,20 @@
+Copyright (c) 2015 Preferred Infrastructure, Inc.
+Copyright (c) 2015 Preferred Networks, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..254ff70
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,18 @@
+# Contents of sdist. See also `setup.py`.
+recursive-include cupy *.h *.hpp
+recursive-include cupy *.pyx *.pxd *.pxi
+recursive-include cupy_backends *.h *.hpp
+recursive-include cupy_backends *.pyx *.pxd *.pxi
+
+# Fail-safe to avoid including Cythoinzed sources in sdist.
+recursive-exclude cupy *.cpp
+recursive-exclude cupy_backends *.cpp
+
+# Installers
+recursive-include install *.py
+recursive-include tests *.py
+
+# Licenses
+include LICENSE
+include docs/LICENSE_THIRD_PARTY
+include docs/source/license.rst
diff --git a/codecov.yml b/codecov.yml
new file mode 100644
index 0000000..e98b0be
--- /dev/null
+++ b/codecov.yml
@@ -0,0 +1,13 @@
+comment: false
+github_checks:
+  annotations: false
+
+coverage:
+  status:
+    # Disable coverage measurement for overall codebase.
+    project: off
+    # Enable coverage measurement for diff introduced in the pull-request,
+    # but do not mark "X" on commit status for now.
+    patch:
+      default:
+        target: '0%'
diff --git a/cupy/__init__.py b/cupy/__init__.py
new file mode 100644
index 0000000..2852d86
--- /dev/null
+++ b/cupy/__init__.py
@@ -0,0 +1,920 @@
+import functools as _functools
+import sys as _sys
+
+import numpy as _numpy
+
+from cupy import _environment
+from cupy import _version
+
+
+_environment._detect_duplicate_installation()  # NOQA
+_environment._setup_win32_dll_directory()  # NOQA
+_environment._preload_library('cutensor')  # NOQA
+_environment._preload_library('nccl')  # NOQA
+
+
+try:
+    from cupy import _core  # NOQA
+except ImportError as exc:
+    raise ImportError(f'''
+================================================================
+{_environment._diagnose_import_error()}
+
+Original error:
+  {type(exc).__name__}: {exc}
+================================================================
+''') from exc
+
+
+from cupy import cuda  # NOQA
+# Do not make `cupy.cupyx` available because it is confusing.
+import cupyx as _cupyx  # NOQA
+
+
+def is_available():
+    return cuda.is_available()
+
+
+__version__ = _version.__version__
+
+
+from cupy import fft  # NOQA
+from cupy import linalg  # NOQA
+from cupy import polynomial  # NOQA
+from cupy import random  # NOQA
+# `cupy.sparse` is deprecated in v8
+from cupy import sparse  # NOQA
+from cupy import testing  # NOQA  # NOQA
+
+
+# import class and function
+from cupy._core import ndarray  # NOQA
+from cupy._core import ufunc  # NOQA
+
+
+# =============================================================================
+# Constants (borrowed from NumPy)
+# =============================================================================
+from numpy import e  # NOQA
+from numpy import euler_gamma  # NOQA
+from numpy import inf  # NOQA
+from numpy import nan  # NOQA
+from numpy import newaxis  # == None  # NOQA
+from numpy import pi  # NOQA
+
+# APIs to be removed in NumPy 2.0.
+# Remove these when bumping the baseline API to NumPy 2.0.
+# https://github.com/cupy/cupy/pull/7800
+PINF = Inf = Infinity = infty = inf  # NOQA
+NINF = -inf  # NOQA
+NAN = NaN = nan  # NOQA
+PZERO = 0.0  # NOQA
+NZERO = -0.0  # NOQA
+
+# =============================================================================
+# Data types (borrowed from NumPy)
+#
+# The order of these declarations are borrowed from the NumPy document:
+# https://numpy.org/doc/stable/reference/arrays.scalars.html
+# =============================================================================
+
+# -----------------------------------------------------------------------------
+# Generic types
+# -----------------------------------------------------------------------------
+from numpy import complexfloating  # NOQA
+from numpy import floating  # NOQA
+from numpy import generic  # NOQA
+from numpy import inexact  # NOQA
+from numpy import integer  # NOQA
+from numpy import number  # NOQA
+from numpy import signedinteger  # NOQA
+from numpy import unsignedinteger  # NOQA
+
+# Not supported by CuPy:
+# from numpy import flexible
+# from numpy import character
+
+# -----------------------------------------------------------------------------
+# Booleans
+# -----------------------------------------------------------------------------
+from numpy import bool_  # NOQA
+
+# -----------------------------------------------------------------------------
+# Integers
+# -----------------------------------------------------------------------------
+from numpy import byte  # NOQA
+from numpy import short  # NOQA
+from numpy import intc  # NOQA
+from numpy import int_  # NOQA
+from numpy import longlong  # NOQA
+from numpy import intp  # NOQA
+from numpy import int8  # NOQA
+from numpy import int16  # NOQA
+from numpy import int32  # NOQA
+from numpy import int64  # NOQA
+
+# -----------------------------------------------------------------------------
+# Unsigned integers
+# -----------------------------------------------------------------------------
+from numpy import ubyte  # NOQA
+from numpy import ushort  # NOQA
+from numpy import uintc  # NOQA
+from numpy import uint  # NOQA
+from numpy import ulonglong  # NOQA
+from numpy import uintp  # NOQA
+from numpy import uint8  # NOQA
+from numpy import uint16  # NOQA
+from numpy import uint32  # NOQA
+from numpy import uint64  # NOQA
+
+# -----------------------------------------------------------------------------
+# Floating-point numbers
+# -----------------------------------------------------------------------------
+from numpy import half  # NOQA
+from numpy import single  # NOQA
+from numpy import double  # NOQA
+from numpy import float_  # NOQA
+from numpy import longfloat  # NOQA
+from numpy import float16  # NOQA
+from numpy import float32  # NOQA
+from numpy import float64  # NOQA
+
+# Not supported by CuPy:
+# from numpy import float96
+# from numpy import float128
+
+# -----------------------------------------------------------------------------
+# Complex floating-point numbers
+# -----------------------------------------------------------------------------
+from numpy import csingle  # NOQA
+from numpy import singlecomplex  # NOQA
+from numpy import cdouble  # NOQA
+from numpy import cfloat  # NOQA
+from numpy import complex_  # NOQA
+from numpy import complex64  # NOQA
+from numpy import complex128  # NOQA
+
+# Not supported by CuPy:
+# from numpy import complex192
+# from numpy import complex256
+# from numpy import clongfloat
+
+# -----------------------------------------------------------------------------
+# Any Python object
+# -----------------------------------------------------------------------------
+
+# Not supported by CuPy:
+# from numpy import object_
+# from numpy import bytes_
+# from numpy import unicode_
+# from numpy import void
+
+# -----------------------------------------------------------------------------
+# Built-in Python types
+# -----------------------------------------------------------------------------
+
+# =============================================================================
+# Routines
+#
+# The order of these declarations are borrowed from the NumPy document:
+# https://numpy.org/doc/stable/reference/routines.html
+# =============================================================================
+
+# -----------------------------------------------------------------------------
+# Array creation routines
+# -----------------------------------------------------------------------------
+from cupy._creation.basic import empty  # NOQA
+from cupy._creation.basic import empty_like  # NOQA
+from cupy._creation.basic import eye  # NOQA
+from cupy._creation.basic import full  # NOQA
+from cupy._creation.basic import full_like  # NOQA
+from cupy._creation.basic import identity  # NOQA
+from cupy._creation.basic import ones  # NOQA
+from cupy._creation.basic import ones_like  # NOQA
+from cupy._creation.basic import zeros  # NOQA
+from cupy._creation.basic import zeros_like  # NOQA
+
+from cupy._creation.from_data import copy  # NOQA
+from cupy._creation.from_data import array  # NOQA
+from cupy._creation.from_data import asanyarray  # NOQA
+from cupy._creation.from_data import asarray  # NOQA
+from cupy._creation.from_data import ascontiguousarray  # NOQA
+from cupy._creation.from_data import fromfile  # NOQA
+from cupy._creation.from_data import fromfunction  # NOQA
+from cupy._creation.from_data import fromiter  # NOQA
+from cupy._creation.from_data import frombuffer  # NOQA
+from cupy._creation.from_data import fromstring  # NOQA
+from cupy._creation.from_data import loadtxt  # NOQA
+from cupy._creation.from_data import genfromtxt  # NOQA
+
+from cupy._creation.ranges import arange  # NOQA
+from cupy._creation.ranges import linspace  # NOQA
+from cupy._creation.ranges import logspace  # NOQA
+from cupy._creation.ranges import meshgrid  # NOQA
+from cupy._creation.ranges import mgrid  # NOQA
+from cupy._creation.ranges import ogrid  # NOQA
+
+from cupy._creation.matrix import diag  # NOQA
+from cupy._creation.matrix import diagflat  # NOQA
+from cupy._creation.matrix import tri  # NOQA
+from cupy._creation.matrix import tril  # NOQA
+from cupy._creation.matrix import triu  # NOQA
+from cupy._creation.matrix import vander  # NOQA
+
+# -----------------------------------------------------------------------------
+# Functional routines
+# -----------------------------------------------------------------------------
+from cupy._functional.piecewise import piecewise  # NOQA
+from cupy._functional.vectorize import vectorize  # NOQA
+from cupy.lib._shape_base import apply_along_axis  # NOQA
+
+# -----------------------------------------------------------------------------
+# Array manipulation routines
+# -----------------------------------------------------------------------------
+from cupy._manipulation.basic import copyto  # NOQA
+
+from cupy._manipulation.shape import shape  # NOQA
+from cupy._manipulation.shape import ravel  # NOQA
+from cupy._manipulation.shape import reshape  # NOQA
+
+from cupy._manipulation.transpose import moveaxis  # NOQA
+from cupy._manipulation.transpose import rollaxis  # NOQA
+from cupy._manipulation.transpose import swapaxes  # NOQA
+from cupy._manipulation.transpose import transpose  # NOQA
+
+from cupy._manipulation.dims import atleast_1d  # NOQA
+from cupy._manipulation.dims import atleast_2d  # NOQA
+from cupy._manipulation.dims import atleast_3d  # NOQA
+from cupy._manipulation.dims import broadcast  # NOQA
+from cupy._manipulation.dims import broadcast_arrays  # NOQA
+from cupy._manipulation.dims import broadcast_to  # NOQA
+from cupy._manipulation.dims import expand_dims  # NOQA
+from cupy._manipulation.dims import squeeze  # NOQA
+
+from cupy._manipulation.join import column_stack  # NOQA
+from cupy._manipulation.join import concatenate  # NOQA
+from cupy._manipulation.join import dstack  # NOQA
+from cupy._manipulation.join import hstack  # NOQA
+from cupy._manipulation.join import stack  # NOQA
+from cupy._manipulation.join import vstack  # NOQA
+from cupy._manipulation.join import vstack as row_stack  # NOQA
+
+from cupy._manipulation.kind import asarray_chkfinite  # NOQA
+from cupy._manipulation.kind import asfarray  # NOQA
+from cupy._manipulation.kind import asfortranarray  # NOQA
+from cupy._manipulation.kind import require  # NOQA
+
+from cupy._manipulation.split import array_split  # NOQA
+from cupy._manipulation.split import dsplit  # NOQA
+from cupy._manipulation.split import hsplit  # NOQA
+from cupy._manipulation.split import split  # NOQA
+from cupy._manipulation.split import vsplit  # NOQA
+
+from cupy._manipulation.tiling import repeat  # NOQA
+from cupy._manipulation.tiling import tile  # NOQA
+
+from cupy._manipulation.add_remove import append  # NOQA
+from cupy._manipulation.add_remove import resize  # NOQA
+from cupy._manipulation.add_remove import unique  # NOQA
+from cupy._manipulation.add_remove import trim_zeros  # NOQA
+
+from cupy._manipulation.rearrange import flip  # NOQA
+from cupy._manipulation.rearrange import fliplr  # NOQA
+from cupy._manipulation.rearrange import flipud  # NOQA
+from cupy._manipulation.rearrange import roll  # NOQA
+from cupy._manipulation.rearrange import rot90  # NOQA
+
+# Borrowed from NumPy
+if hasattr(_numpy, 'broadcast_shapes'):  # NumPy 1.20
+    from numpy import broadcast_shapes  # NOQA
+
+# -----------------------------------------------------------------------------
+# Binary operations
+# -----------------------------------------------------------------------------
+from cupy._binary.elementwise import bitwise_and  # NOQA
+from cupy._binary.elementwise import bitwise_or  # NOQA
+from cupy._binary.elementwise import bitwise_xor  # NOQA
+from cupy._binary.elementwise import bitwise_not  # NOQA
+from cupy._binary.elementwise import invert  # NOQA
+from cupy._binary.elementwise import left_shift  # NOQA
+from cupy._binary.elementwise import right_shift  # NOQA
+
+from cupy._binary.packing import packbits  # NOQA
+from cupy._binary.packing import unpackbits  # NOQA
+
+
+def binary_repr(num, width=None):
+    """Return the binary representation of the input number as a string.
+
+    .. seealso:: :func:`numpy.binary_repr`
+    """
+    return _numpy.binary_repr(num, width)
+
+
+# -----------------------------------------------------------------------------
+# Data type routines (mostly borrowed from NumPy)
+# -----------------------------------------------------------------------------
+def can_cast(from_, to, casting='safe'):
+    """Returns True if cast between data types can occur according to the
+    casting rule. If from is a scalar or array scalar, also returns True if the
+    scalar value can be cast without overflow or truncation to an integer.
+
+    .. seealso:: :func:`numpy.can_cast`
+    """
+    from_ = from_.dtype if isinstance(from_, ndarray) else from_
+    return _numpy.can_cast(from_, to, casting=casting)
+
+
+def common_type(*arrays):
+    """Return a scalar type which is common to the input arrays.
+
+    .. seealso:: :func:`numpy.common_type`
+    """
+    if len(arrays) == 0:
+        return _numpy.float16
+
+    default_float_dtype = _numpy.dtype('float64')
+    dtypes = []
+    for a in arrays:
+        if a.dtype.kind == 'b':
+            raise TypeError('can\'t get common type for non-numeric array')
+        elif a.dtype.kind in 'iu':
+            dtypes.append(default_float_dtype)
+        else:
+            dtypes.append(a.dtype)
+
+    return _functools.reduce(_numpy.promote_types, dtypes).type
+
+
+def result_type(*arrays_and_dtypes):
+    """Returns the type that results from applying the NumPy type promotion
+    rules to the arguments.
+
+    .. seealso:: :func:`numpy.result_type`
+    """
+    dtypes = [a.dtype if isinstance(a, ndarray)
+              else a for a in arrays_and_dtypes]
+    return _numpy.result_type(*dtypes)
+
+
+from cupy._core.core import min_scalar_type  # NOQA
+
+from numpy import obj2sctype  # NOQA
+from numpy import promote_types  # NOQA
+
+from numpy import dtype  # NOQA
+from numpy import format_parser  # NOQA
+
+from numpy import finfo  # NOQA
+from numpy import iinfo  # NOQA
+
+from numpy import find_common_type  # NOQA
+from numpy import issctype  # NOQA
+from numpy import issubclass_  # NOQA
+from numpy import issubdtype  # NOQA
+from numpy import issubsctype  # NOQA
+
+from numpy import mintypecode  # NOQA
+from numpy import sctype2char  # NOQA
+from numpy import typename  # NOQA
+
+# -----------------------------------------------------------------------------
+# Optionally Scipy-accelerated routines
+# -----------------------------------------------------------------------------
+# TODO(beam2d): Implement it
+
+# -----------------------------------------------------------------------------
+# Discrete Fourier Transform
+# -----------------------------------------------------------------------------
+# TODO(beam2d): Implement it
+
+# -----------------------------------------------------------------------------
+# Indexing routines
+# -----------------------------------------------------------------------------
+from cupy._indexing.generate import c_  # NOQA
+from cupy._indexing.generate import indices  # NOQA
+from cupy._indexing.generate import ix_  # NOQA
+from cupy._indexing.generate import mask_indices  # NOQA
+from cupy._indexing.generate import tril_indices  # NOQA
+from cupy._indexing.generate import tril_indices_from  # NOQA
+from cupy._indexing.generate import triu_indices  # NOQA
+from cupy._indexing.generate import triu_indices_from  # NOQA
+from cupy._indexing.generate import r_  # NOQA
+from cupy._indexing.generate import ravel_multi_index  # NOQA
+from cupy._indexing.generate import unravel_index  # NOQA
+
+from cupy._indexing.indexing import choose  # NOQA
+from cupy._indexing.indexing import compress  # NOQA
+from cupy._indexing.indexing import diagonal  # NOQA
+from cupy._indexing.indexing import extract  # NOQA
+from cupy._indexing.indexing import select  # NOQA
+from cupy._indexing.indexing import take  # NOQA
+from cupy._indexing.indexing import take_along_axis  # NOQA
+
+from cupy._indexing.insert import place  # NOQA
+from cupy._indexing.insert import put  # NOQA
+from cupy._indexing.insert import putmask  # NOQA
+from cupy._indexing.insert import fill_diagonal  # NOQA
+from cupy._indexing.insert import diag_indices  # NOQA
+from cupy._indexing.insert import diag_indices_from  # NOQA
+
+from cupy._indexing.iterate import flatiter  # NOQA
+
+# Borrowed from NumPy
+from numpy import get_array_wrap  # NOQA
+from numpy import index_exp  # NOQA
+from numpy import ndindex  # NOQA
+from numpy import s_  # NOQA
+
+# -----------------------------------------------------------------------------
+# Input and output
+# -----------------------------------------------------------------------------
+from cupy._io.npz import load  # NOQA
+from cupy._io.npz import save  # NOQA
+from cupy._io.npz import savez  # NOQA
+from cupy._io.npz import savez_compressed  # NOQA
+
+from cupy._io.formatting import array_repr  # NOQA
+from cupy._io.formatting import array_str  # NOQA
+from cupy._io.formatting import array2string  # NOQA
+from cupy._io.formatting import format_float_positional  # NOQA
+from cupy._io.formatting import format_float_scientific  # NOQA
+
+from cupy._io.text import savetxt  # NOQA
+
+
+def base_repr(number, base=2, padding=0):  # NOQA (needed to avoid redefinition of `number`)
+    """Return a string representation of a number in the given base system.
+
+    .. seealso:: :func:`numpy.base_repr`
+    """
+    return _numpy.base_repr(number, base, padding)
+
+
+# Borrowed from NumPy
+from numpy import DataSource  # NOQA
+from numpy import get_printoptions  # NOQA
+from numpy import set_printoptions  # NOQA
+from numpy import printoptions  # NOQA
+from numpy import set_string_function  # NOQA
+
+
+# -----------------------------------------------------------------------------
+# Linear algebra
+# -----------------------------------------------------------------------------
+from cupy.linalg._einsum import einsum  # NOQA
+
+from cupy.linalg._product import cross  # NOQA
+from cupy.linalg._product import dot  # NOQA
+from cupy.linalg._product import inner  # NOQA
+from cupy.linalg._product import kron  # NOQA
+from cupy.linalg._product import matmul  # NOQA
+from cupy.linalg._product import outer  # NOQA
+from cupy.linalg._product import tensordot  # NOQA
+from cupy.linalg._product import vdot  # NOQA
+
+from cupy.linalg._norms import trace  # NOQA
+
+# -----------------------------------------------------------------------------
+# Logic functions
+# -----------------------------------------------------------------------------
+from cupy._logic.comparison import allclose  # NOQA
+from cupy._logic.comparison import array_equal  # NOQA
+from cupy._logic.comparison import array_equiv  # NOQA
+from cupy._logic.comparison import isclose  # NOQA
+
+from cupy._logic.content import isfinite  # NOQA
+from cupy._logic.content import isinf  # NOQA
+from cupy._logic.content import isnan  # NOQA
+from cupy._logic.content import isneginf  # NOQA
+from cupy._logic.content import isposinf  # NOQA
+
+from cupy._logic.truth import in1d  # NOQA
+from cupy._logic.truth import isin  # NOQA
+
+from cupy._logic.type_testing import iscomplex  # NOQA
+from cupy._logic.type_testing import iscomplexobj  # NOQA
+from cupy._logic.type_testing import isfortran  # NOQA
+from cupy._logic.type_testing import isreal  # NOQA
+from cupy._logic.type_testing import isrealobj  # NOQA
+
+from cupy._logic.truth import in1d  # NOQA
+from cupy._logic.truth import intersect1d  # NOQA
+from cupy._logic.truth import isin  # NOQA
+from cupy._logic.truth import setdiff1d  # NOQA
+from cupy._logic.truth import setxor1d  # NOQA
+from cupy._logic.truth import union1d  # NOQA
+
+
+def isscalar(element):
+    """Returns True if the type of num is a scalar type.
+
+    .. seealso:: :func:`numpy.isscalar`
+    """
+    return _numpy.isscalar(element)
+
+
+from cupy._logic.ops import logical_and  # NOQA
+from cupy._logic.ops import logical_not  # NOQA
+from cupy._logic.ops import logical_or  # NOQA
+from cupy._logic.ops import logical_xor  # NOQA
+
+from cupy._logic.comparison import equal  # NOQA
+from cupy._logic.comparison import greater  # NOQA
+from cupy._logic.comparison import greater_equal  # NOQA
+from cupy._logic.comparison import less  # NOQA
+from cupy._logic.comparison import less_equal  # NOQA
+from cupy._logic.comparison import not_equal  # NOQA
+
+from cupy._logic.truth import all  # NOQA
+from cupy._logic.truth import all as alltrue  # NOQA
+from cupy._logic.truth import any  # NOQA
+from cupy._logic.truth import any as sometrue  # NOQA
+
+# ------------------------------------------------------------------------------
+# Polynomial functions
+# ------------------------------------------------------------------------------
+from cupy.lib._polynomial import poly1d  # NOQA
+from cupy.lib._routines_poly import poly  # NOQA
+from cupy.lib._routines_poly import polyadd  # NOQA
+from cupy.lib._routines_poly import polysub  # NOQA
+from cupy.lib._routines_poly import polymul  # NOQA
+from cupy.lib._routines_poly import polyfit  # NOQA
+from cupy.lib._routines_poly import polyval  # NOQA
+from cupy.lib._routines_poly import roots  # NOQA
+
+# Borrowed from NumPy
+from numpy import RankWarning  # NOQA
+
+# -----------------------------------------------------------------------------
+# Mathematical functions
+# -----------------------------------------------------------------------------
+from cupy._math.trigonometric import arccos  # NOQA
+from cupy._math.trigonometric import arcsin  # NOQA
+from cupy._math.trigonometric import arctan  # NOQA
+from cupy._math.trigonometric import arctan2  # NOQA
+from cupy._math.trigonometric import cos  # NOQA
+from cupy._math.trigonometric import deg2rad  # NOQA
+from cupy._math.trigonometric import degrees  # NOQA
+from cupy._math.trigonometric import hypot  # NOQA
+from cupy._math.trigonometric import rad2deg  # NOQA
+from cupy._math.trigonometric import radians  # NOQA
+from cupy._math.trigonometric import sin  # NOQA
+from cupy._math.trigonometric import tan  # NOQA
+from cupy._math.trigonometric import unwrap  # NOQA
+
+from cupy._math.hyperbolic import arccosh  # NOQA
+from cupy._math.hyperbolic import arcsinh  # NOQA
+from cupy._math.hyperbolic import arctanh  # NOQA
+from cupy._math.hyperbolic import cosh  # NOQA
+from cupy._math.hyperbolic import sinh  # NOQA
+from cupy._math.hyperbolic import tanh  # NOQA
+
+from cupy._math.rounding import around  # NOQA
+from cupy._math.rounding import ceil  # NOQA
+from cupy._math.rounding import fix  # NOQA
+from cupy._math.rounding import floor  # NOQA
+from cupy._math.rounding import rint  # NOQA
+from cupy._math.rounding import round_  # NOQA
+from cupy._math.rounding import round_ as round  # NOQA
+from cupy._math.rounding import trunc  # NOQA
+
+from cupy._math.sumprod import prod  # NOQA
+from cupy._math.sumprod import prod as product  # NOQA
+from cupy._math.sumprod import sum  # NOQA
+from cupy._math.sumprod import cumprod  # NOQA
+from cupy._math.sumprod import cumprod as cumproduct  # NOQA
+from cupy._math.sumprod import cumsum  # NOQA
+from cupy._math.sumprod import ediff1d  # NOQA
+from cupy._math.sumprod import nancumprod  # NOQA
+from cupy._math.sumprod import nancumsum  # NOQA
+from cupy._math.sumprod import nansum  # NOQA
+from cupy._math.sumprod import nanprod  # NOQA
+from cupy._math.sumprod import diff  # NOQA
+from cupy._math.sumprod import gradient  # NOQA
+from cupy._math.sumprod import trapz  # NOQA
+from cupy._math.window import bartlett  # NOQA
+from cupy._math.window import blackman  # NOQA
+from cupy._math.window import hamming  # NOQA
+from cupy._math.window import hanning  # NOQA
+from cupy._math.window import kaiser  # NOQA
+
+from cupy._math.explog import exp  # NOQA
+from cupy._math.explog import exp2  # NOQA
+from cupy._math.explog import expm1  # NOQA
+from cupy._math.explog import log  # NOQA
+from cupy._math.explog import log10  # NOQA
+from cupy._math.explog import log1p  # NOQA
+from cupy._math.explog import log2  # NOQA
+from cupy._math.explog import logaddexp  # NOQA
+from cupy._math.explog import logaddexp2  # NOQA
+
+from cupy._math.special import i0  # NOQA
+from cupy._math.special import sinc  # NOQA
+
+from cupy._math.floating import copysign  # NOQA
+from cupy._math.floating import frexp  # NOQA
+from cupy._math.floating import ldexp  # NOQA
+from cupy._math.floating import nextafter  # NOQA
+from cupy._math.floating import signbit  # NOQA
+
+from cupy._math.rational import gcd  # NOQA
+from cupy._math.rational import lcm  # NOQA
+
+from cupy._math.arithmetic import add  # NOQA
+from cupy._math.arithmetic import divide  # NOQA
+from cupy._math.arithmetic import divmod  # NOQA
+from cupy._math.arithmetic import floor_divide  # NOQA
+from cupy._math.arithmetic import float_power  # NOQA
+from cupy._math.arithmetic import fmod  # NOQA
+from cupy._math.arithmetic import modf  # NOQA
+from cupy._math.arithmetic import multiply  # NOQA
+from cupy._math.arithmetic import negative  # NOQA
+from cupy._math.arithmetic import positive  # NOQA
+from cupy._math.arithmetic import power  # NOQA
+from cupy._math.arithmetic import reciprocal  # NOQA
+from cupy._math.arithmetic import remainder  # NOQA
+from cupy._math.arithmetic import remainder as mod  # NOQA
+from cupy._math.arithmetic import subtract  # NOQA
+from cupy._math.arithmetic import true_divide  # NOQA
+
+from cupy._math.arithmetic import angle  # NOQA
+from cupy._math.arithmetic import conjugate as conj  # NOQA
+from cupy._math.arithmetic import conjugate  # NOQA
+from cupy._math.arithmetic import imag  # NOQA
+from cupy._math.arithmetic import real  # NOQA
+
+from cupy._math.misc import absolute as abs  # NOQA
+from cupy._math.misc import absolute  # NOQA
+from cupy._math.misc import cbrt  # NOQA
+from cupy._math.misc import clip  # NOQA
+from cupy._math.misc import fabs  # NOQA
+from cupy._math.misc import fmax  # NOQA
+from cupy._math.misc import fmin  # NOQA
+from cupy._math.misc import interp  # NOQA
+from cupy._math.misc import maximum  # NOQA
+from cupy._math.misc import minimum  # NOQA
+from cupy._math.misc import nan_to_num  # NOQA
+from cupy._math.misc import real_if_close  # NOQA
+from cupy._math.misc import sign  # NOQA
+from cupy._math.misc import heaviside  # NOQA
+from cupy._math.misc import sqrt  # NOQA
+from cupy._math.misc import square  # NOQA
+from cupy._math.misc import convolve  # NOQA
+
+# -----------------------------------------------------------------------------
+# Miscellaneous routines
+# -----------------------------------------------------------------------------
+from cupy._misc.byte_bounds import byte_bounds  # NOQA
+from cupy._misc.memory_ranges import may_share_memory  # NOQA
+from cupy._misc.memory_ranges import shares_memory  # NOQA
+from cupy._misc.who import who  # NOQA
+
+# Borrowed from NumPy
+from numpy import disp  # NOQA
+from numpy import iterable  # NOQA
+from numpy import safe_eval  # NOQA
+from numpy import AxisError  # NOQA
+
+
+# -----------------------------------------------------------------------------
+# Padding
+# -----------------------------------------------------------------------------
+from cupy._padding.pad import pad  # NOQA
+
+
+# -----------------------------------------------------------------------------
+# Sorting, searching, and counting
+# -----------------------------------------------------------------------------
+from cupy._sorting.count import count_nonzero  # NOQA
+
+from cupy._sorting.search import argmax  # NOQA
+from cupy._sorting.search import argmin  # NOQA
+from cupy._sorting.search import argwhere  # NOQA
+from cupy._sorting.search import flatnonzero  # NOQA
+from cupy._sorting.search import nanargmax  # NOQA
+from cupy._sorting.search import nanargmin  # NOQA
+from cupy._sorting.search import nonzero  # NOQA
+from cupy._sorting.search import searchsorted  # NOQA
+from cupy._sorting.search import where  # NOQA
+
+from cupy._sorting.sort import argpartition  # NOQA
+from cupy._sorting.sort import argsort  # NOQA
+from cupy._sorting.sort import lexsort  # NOQA
+from cupy._sorting.sort import msort  # NOQA
+from cupy._sorting.sort import sort_complex  # NOQA
+from cupy._sorting.sort import partition  # NOQA
+from cupy._sorting.sort import sort  # NOQA
+
+# -----------------------------------------------------------------------------
+# Statistics
+# -----------------------------------------------------------------------------
+from cupy._statistics.correlation import corrcoef  # NOQA
+from cupy._statistics.correlation import cov  # NOQA
+from cupy._statistics.correlation import correlate  # NOQA
+
+from cupy._statistics.order import amax  # NOQA
+from cupy._statistics.order import amax as max  # NOQA
+from cupy._statistics.order import amin  # NOQA
+from cupy._statistics.order import amin as min  # NOQA
+from cupy._statistics.order import nanmax  # NOQA
+from cupy._statistics.order import nanmin  # NOQA
+from cupy._statistics.order import percentile  # NOQA
+from cupy._statistics.order import ptp  # NOQA
+from cupy._statistics.order import quantile  # NOQA
+
+from cupy._statistics.meanvar import median  # NOQA
+from cupy._statistics.meanvar import average  # NOQA
+from cupy._statistics.meanvar import mean  # NOQA
+from cupy._statistics.meanvar import std  # NOQA
+from cupy._statistics.meanvar import var  # NOQA
+from cupy._statistics.meanvar import nanmedian  # NOQA
+from cupy._statistics.meanvar import nanmean  # NOQA
+from cupy._statistics.meanvar import nanstd  # NOQA
+from cupy._statistics.meanvar import nanvar  # NOQA
+
+from cupy._statistics.histogram import bincount  # NOQA
+from cupy._statistics.histogram import digitize  # NOQA
+from cupy._statistics.histogram import histogram  # NOQA
+from cupy._statistics.histogram import histogram2d  # NOQA
+from cupy._statistics.histogram import histogramdd  # NOQA
+
+# -----------------------------------------------------------------------------
+# Classes without their own docs
+# -----------------------------------------------------------------------------
+from numpy import ComplexWarning  # NOQA
+from numpy import ModuleDeprecationWarning  # NOQA
+from numpy import TooHardError  # NOQA
+from numpy import VisibleDeprecationWarning  # NOQA
+
+
+# -----------------------------------------------------------------------------
+# Undocumented functions
+# -----------------------------------------------------------------------------
+from cupy._core import size  # NOQA
+
+
+def ndim(a):
+    """Returns the number of dimensions of an array.
+
+    Args:
+        a (array-like): If it is not already an `cupy.ndarray`, a conversion
+            via :func:`numpy.asarray` is attempted.
+
+    Returns:
+        (int): The number of dimensions in `a`.
+
+    """
+    try:
+        return a.ndim
+    except AttributeError:
+        return _numpy.ndim(a)
+
+
+# -----------------------------------------------------------------------------
+# CuPy specific functions
+# -----------------------------------------------------------------------------
+
+from cupy._util import clear_memo  # NOQA
+from cupy._util import memoize  # NOQA
+
+from cupy._core import ElementwiseKernel  # NOQA
+from cupy._core import RawKernel  # NOQA
+from cupy._core import RawModule  # NOQA
+from cupy._core._reduction import ReductionKernel  # NOQA
+
+# -----------------------------------------------------------------------------
+# DLPack
+# -----------------------------------------------------------------------------
+
+from cupy._core import fromDlpack  # NOQA
+from cupy._core import from_dlpack  # NOQA
+
+
+def asnumpy(a, stream=None, order='C', out=None):
+    """Returns an array on the host memory from an arbitrary source array.
+
+    Args:
+        a: Arbitrary object that can be converted to :class:`numpy.ndarray`.
+        stream (cupy.cuda.Stream): CUDA stream object. If it is specified, then
+            the device-to-host copy runs asynchronously. Otherwise, the copy is
+            synchronous. Note that if ``a`` is not a :class:`cupy.ndarray`
+            object, then this argument has no effect.
+        order ({'C', 'F', 'A'}): The desired memory layout of the host
+            array. When ``order`` is 'A', it uses 'F' if ``a`` is
+            fortran-contiguous and 'C' otherwise.
+        out (numpy.ndarray): The output array to be written to. It must have
+            compatible shape and dtype with those of ``a``'s.
+
+    Returns:
+        numpy.ndarray: Converted array on the host memory.
+
+    """
+    if isinstance(a, ndarray):
+        return a.get(stream=stream, order=order, out=out)
+    elif hasattr(a, "__cuda_array_interface__"):
+        return array(a).get(stream=stream, order=order, out=out)
+    else:
+        temp = _numpy.asarray(a, order=order)
+        if out is not None:
+            out[...] = temp
+        else:
+            out = temp
+        return out
+
+
+_cupy = _sys.modules[__name__]
+
+
+def get_array_module(*args):
+    """Returns the array module for arguments.
+
+    This function is used to implement CPU/GPU generic code. If at least one of
+    the arguments is a :class:`cupy.ndarray` object, the :mod:`cupy` module is
+    returned.
+
+    Args:
+        args: Values to determine whether NumPy or CuPy should be used.
+
+    Returns:
+        module: :mod:`cupy` or :mod:`numpy` is returned based on the types of
+        the arguments.
+
+    .. admonition:: Example
+
+       A NumPy/CuPy generic function can be written as follows
+
+       >>> def softplus(x):
+       ...     xp = cupy.get_array_module(x)
+       ...     return xp.maximum(0, x) + xp.log1p(xp.exp(-abs(x)))
+
+    """
+    for arg in args:
+        if isinstance(arg, (ndarray, _cupyx.scipy.sparse.spmatrix,
+                            _core.fusion._FusionVarArray,
+                            _core.new_fusion._ArrayProxy)):
+            return _cupy
+    return _numpy
+
+
+fuse = _core.fusion.fuse
+
+disable_experimental_feature_warning = False
+
+
+# set default allocator
+_default_memory_pool = cuda.MemoryPool()
+_default_pinned_memory_pool = cuda.PinnedMemoryPool()
+
+cuda.set_allocator(_default_memory_pool.malloc)
+cuda.set_pinned_memory_allocator(_default_pinned_memory_pool.malloc)
+
+
+def get_default_memory_pool():
+    """Returns CuPy default memory pool for GPU memory.
+
+    Returns:
+        cupy.cuda.MemoryPool: The memory pool object.
+
+    .. note::
+       If you want to disable memory pool, please use the following code.
+
+       >>> cupy.cuda.set_allocator(None)
+
+    """
+    return _default_memory_pool
+
+
+def get_default_pinned_memory_pool():
+    """Returns CuPy default memory pool for pinned memory.
+
+    Returns:
+        cupy.cuda.PinnedMemoryPool: The memory pool object.
+
+    .. note::
+       If you want to disable memory pool, please use the following code.
+
+       >>> cupy.cuda.set_pinned_memory_allocator(None)
+
+    """
+    return _default_pinned_memory_pool
+
+
+def show_config(*, _full=False):
+    """Prints the current runtime configuration to standard output."""
+    _sys.stdout.write(str(_cupyx.get_runtime_info(full=_full)))
+    _sys.stdout.flush()
+
+
+_deprecated_apis = [
+    'int0',
+    'uint0',
+    'bool8',
+]
+
+
+def __getattr__(name):
+    if name in _deprecated_apis:
+        return getattr(_numpy, name)
+
+    raise AttributeError(f"module 'cupy' has no attribute {name!r}")
diff --git a/cupy/_binary/__init__.py b/cupy/_binary/__init__.py
new file mode 100644
index 0000000..3feab0f
--- /dev/null
+++ b/cupy/_binary/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.bitwise.html
diff --git a/cupy/_binary/elementwise.py b/cupy/_binary/elementwise.py
new file mode 100644
index 0000000..11e4fa1
--- /dev/null
+++ b/cupy/_binary/elementwise.py
@@ -0,0 +1,22 @@
+from cupy import _core
+
+
+bitwise_and = _core.bitwise_and
+
+
+bitwise_or = _core.bitwise_or
+
+
+bitwise_xor = _core.bitwise_xor
+
+
+bitwise_not = _core.invert
+
+
+invert = _core.invert
+
+
+left_shift = _core.left_shift
+
+
+right_shift = _core.right_shift
diff --git a/cupy/_binary/packing.py b/cupy/_binary/packing.py
new file mode 100644
index 0000000..8d6c5a0
--- /dev/null
+++ b/cupy/_binary/packing.py
@@ -0,0 +1,104 @@
+import cupy
+from cupy import _core
+
+
+_packbits_kernel = {
+    'big': _core.ElementwiseKernel(
+        'raw T a, raw int32 a_size', 'uint8 packed',
+        '''for (int j = 0; j < 8; ++j) {
+                    int k = i * 8 + j;
+                    int bit = k < a_size && a[k] != 0;
+                    packed |= bit << (7 - j);
+                }''',
+        'cupy_packbits_big'
+    ),
+    'little': _core.ElementwiseKernel(
+        'raw T a, raw int32 a_size', 'uint8 packed',
+        '''for (int j = 0; j < 8; ++j) {
+                    int k = i * 8 + j;
+                    int bit = k < a_size && a[k] != 0;
+                    packed |= bit << j;
+                }''',
+        'cupy_packbits_little'
+    )
+}
+
+
+def packbits(a, axis=None, bitorder='big'):
+    """Packs the elements of a binary-valued array into bits in a uint8 array.
+
+    This function currently does not support ``axis`` option.
+
+    Args:
+        a (cupy.ndarray): Input array.
+        axis (int, optional): Not supported yet.
+        bitorder (str, optional): bit order to use when packing the array,
+            allowed values are `'little'` and `'big'`. Defaults to `'big'`.
+
+    Returns:
+        cupy.ndarray: The packed array.
+
+    .. note::
+        When the input array is empty, this function returns a copy of it,
+        i.e., the type of the output array is not necessarily always uint8.
+        This exactly follows the NumPy's behaviour (as of version 1.11),
+        alghough this is inconsistent to the documentation.
+
+    .. seealso:: :func:`numpy.packbits`
+    """
+    if a.dtype.kind not in 'biu':
+        raise TypeError(
+            'Expected an input array of integer or boolean data type')
+
+    if axis is not None:
+        raise NotImplementedError('axis option is not supported yet')
+
+    if bitorder not in ('big', 'little'):
+        raise ValueError("bitorder must be either 'big' or 'little'")
+
+    a = a.ravel()
+    packed_size = (a.size + 7) // 8
+    packed = cupy.zeros((packed_size,), dtype=cupy.uint8)
+    return _packbits_kernel[bitorder](a, a.size, packed)
+
+
+_unpackbits_kernel = {
+    'big': _core.ElementwiseKernel(
+        'raw uint8 a', 'T unpacked',
+        'unpacked = (a[i / 8] >> (7 - i % 8)) & 1;',
+        'cupy_unpackbits_big'
+    ),
+    'little': _core.ElementwiseKernel(
+        'raw uint8 a', 'T unpacked',
+        'unpacked = (a[i / 8] >> (i % 8)) & 1;',
+        'cupy_unpackbits_little'
+    )
+}
+
+
+def unpackbits(a, axis=None, bitorder='big'):
+    """Unpacks elements of a uint8 array into a binary-valued output array.
+
+    This function currently does not support ``axis`` option.
+
+    Args:
+        a (cupy.ndarray): Input array.
+        bitorder (str, optional): bit order to use when unpacking the array,
+            allowed values are `'little'` and `'big'`. Defaults to `'big'`.
+
+    Returns:
+        cupy.ndarray: The unpacked array.
+
+    .. seealso:: :func:`numpy.unpackbits`
+    """
+    if a.dtype != cupy.uint8:
+        raise TypeError('Expected an input array of unsigned byte data type')
+
+    if axis is not None:
+        raise NotImplementedError('axis option is not supported yet')
+
+    if bitorder not in ('big', 'little'):
+        raise ValueError("bitorder must be either 'big' or 'little'")
+
+    unpacked = cupy.ndarray((a.size * 8), dtype=cupy.uint8)
+    return _unpackbits_kernel[bitorder](a, unpacked)
diff --git a/cupy/_core/__init__.pxd b/cupy/_core/__init__.pxd
new file mode 100644
index 0000000..e69de29
diff --git a/cupy/_core/__init__.py b/cupy/_core/__init__.py
new file mode 100644
index 0000000..5531b27
--- /dev/null
+++ b/cupy/_core/__init__.py
@@ -0,0 +1,79 @@
+# mypy: ignore-errors
+
+from cupy._core import core  # NOQA
+from cupy._core import fusion  # NOQA
+from cupy._core import internal  # NOQA
+
+
+# internal APIs for testing and developement
+from cupy._core._accelerator import set_elementwise_accelerators  # NOQA
+from cupy._core._accelerator import set_reduction_accelerators  # NOQA
+from cupy._core._accelerator import set_routine_accelerators  # NOQA
+from cupy._core._accelerator import get_elementwise_accelerators  # NOQA
+from cupy._core._accelerator import get_reduction_accelerators  # NOQA
+from cupy._core._accelerator import get_routine_accelerators  # NOQA
+
+
+# import class and function
+from cupy._core._kernel import create_ufunc  # NOQA
+from cupy._core._kernel import ElementwiseKernel  # NOQA
+from cupy._core._kernel import ufunc  # NOQA
+from cupy._core._kernel import _get_warpsize  # NOQA
+from cupy._core._reduction import create_reduction_func  # NOQA
+from cupy._core._reduction import ReductionKernel  # NOQA
+from cupy._core._routines_binary import bitwise_and  # NOQA
+from cupy._core._routines_binary import bitwise_or  # NOQA
+from cupy._core._routines_binary import bitwise_xor  # NOQA
+from cupy._core._routines_binary import invert  # NOQA
+from cupy._core._routines_binary import left_shift  # NOQA
+from cupy._core._routines_binary import right_shift  # NOQA
+from cupy._core._routines_linalg import _mat_ptrs  # NOQA
+from cupy._core._routines_linalg import dot  # NOQA
+from cupy._core._routines_linalg import get_compute_type  # NOQA
+from cupy._core._routines_linalg import matmul  # NOQA
+from cupy._core._routines_linalg import set_compute_type  # NOQA
+from cupy._core._routines_linalg import tensordot_core  # NOQA
+from cupy._core._routines_logic import create_comparison  # NOQA
+from cupy._core._routines_logic import equal  # NOQA
+from cupy._core._routines_logic import greater  # NOQA
+from cupy._core._routines_logic import greater_equal  # NOQA
+from cupy._core._routines_logic import less  # NOQA
+from cupy._core._routines_logic import less_equal  # NOQA
+from cupy._core._routines_logic import not_equal  # NOQA
+from cupy._core._routines_manipulation import array_split  # NOQA
+from cupy._core._routines_manipulation import broadcast  # NOQA
+from cupy._core._routines_manipulation import broadcast_to  # NOQA
+from cupy._core._routines_manipulation import concatenate_method  # NOQA
+from cupy._core._routines_manipulation import moveaxis  # NOQA
+from cupy._core._routines_manipulation import rollaxis  # NOQA
+from cupy._core._routines_manipulation import size  # NOQA'
+from cupy._core._routines_math import absolute  # NOQA
+from cupy._core._routines_math import add  # NOQA
+from cupy._core._routines_math import angle, angle_deg  # NOQA
+from cupy._core._routines_math import conjugate  # NOQA
+from cupy._core._routines_math import divide  # NOQA
+from cupy._core._routines_math import floor_divide  # NOQA
+from cupy._core._routines_math import multiply  # NOQA
+from cupy._core._routines_math import negative  # NOQA
+from cupy._core._routines_math import positive  # NOQA
+from cupy._core._routines_math import power  # NOQA
+from cupy._core._routines_math import remainder  # NOQA
+from cupy._core._routines_math import sqrt  # NOQA
+from cupy._core._routines_math import subtract  # NOQA
+from cupy._core._routines_math import true_divide  # NOQA
+from cupy._core._routines_statistics import nanmax  # NOQA
+from cupy._core._routines_statistics import nanmin  # NOQA
+from cupy._core.core import _internal_ascontiguousarray  # NOQA
+from cupy._core.core import _internal_asfortranarray  # NOQA
+from cupy._core.core import array  # NOQA
+from cupy._core.core import ascontiguousarray  # NOQA
+from cupy._core.core import asfortranarray  # NOQA
+from cupy._core.core import divmod  # NOQA
+from cupy._core.core import elementwise_copy  # NOQA
+from cupy._core.core import ndarray  # NOQA
+from cupy._core.dlpack import fromDlpack  # NOQA
+from cupy._core.dlpack import from_dlpack  # NOQA
+from cupy._core.internal import complete_slice  # NOQA
+from cupy._core.internal import get_size  # NOQA
+from cupy._core.raw import RawKernel  # NOQA
+from cupy._core.raw import RawModule  # NOQA
diff --git a/cupy/_core/_accelerator.pxd b/cupy/_core/_accelerator.pxd
new file mode 100644
index 0000000..773a5ec
--- /dev/null
+++ b/cupy/_core/_accelerator.pxd
@@ -0,0 +1,10 @@
+cdef list _elementwise_accelerators
+
+cdef list _reduction_accelerators
+
+cdef list _routine_accelerators
+
+cpdef enum accelerator_type:
+    ACCELERATOR_CUB = 1
+    ACCELERATOR_CUTENSOR = 2
+    ACCELERATOR_CUTENSORNET = 3
diff --git a/cupy/_core/_accelerator.pyx b/cupy/_core/_accelerator.pyx
new file mode 100644
index 0000000..6ce2991
--- /dev/null
+++ b/cupy/_core/_accelerator.pyx
@@ -0,0 +1,59 @@
+import os
+
+from cupy_backends.cuda.api cimport runtime
+
+
+cdef list _elementwise_accelerators = []
+cdef list _reduction_accelerators = []
+cdef list _routine_accelerators = []
+
+
+cdef int _get_accelerator(accelerator) except -1:
+    if isinstance(accelerator, int):
+        return accelerator
+    if accelerator == 'cub':
+        return ACCELERATOR_CUB
+    if accelerator == 'cutensor':
+        return ACCELERATOR_CUTENSOR
+    if accelerator == 'cutensornet':
+        return ACCELERATOR_CUTENSORNET
+    raise ValueError('Unknown accelerator: {}'.format(accelerator))
+
+
+def set_elementwise_accelerators(accelerators):
+    global _elementwise_accelerators
+    _elementwise_accelerators = [_get_accelerator(b) for b in accelerators]
+
+
+def set_reduction_accelerators(accelerators):
+    global _reduction_accelerators
+    _reduction_accelerators = [_get_accelerator(b) for b in accelerators]
+
+
+def set_routine_accelerators(accelerators):
+    global _routine_accelerators
+    _routine_accelerators = [_get_accelerator(b) for b in accelerators]
+
+
+def get_elementwise_accelerators():
+    return _elementwise_accelerators
+
+
+def get_reduction_accelerators():
+    return _reduction_accelerators
+
+
+def get_routine_accelerators():
+    return _routine_accelerators
+
+
+cdef _set_default_accelerators():
+    cdef str b, accelerator_names = os.getenv(
+        'CUPY_ACCELERATORS', '' if runtime._is_hip_environment else 'cub')
+    cdef list accelerators = [b for b in accelerator_names.split(',') if b]
+    set_elementwise_accelerators(accelerators)
+    set_reduction_accelerators(accelerators)
+    set_routine_accelerators(accelerators)
+
+
+_set_default_accelerators()
diff --git a/cupy/_core/_carray.pxd b/cupy/_core/_carray.pxd
new file mode 100644
index 0000000..c443917
--- /dev/null
+++ b/cupy/_core/_carray.pxd
@@ -0,0 +1,55 @@
+cimport cython  # NOQA
+from libcpp cimport vector
+
+from cupy.cuda cimport function
+
+
+ctypedef vector.vector[Py_ssize_t] shape_t
+ctypedef vector.vector[Py_ssize_t] strides_t
+
+# this matches NPY_MAXDIMS
+# Note: we make it an enum to work around cython/cython#4369
+cdef enum: MAX_NDIM = 32
+
+
+cdef struct _CArray:
+    void* data
+    Py_ssize_t size
+    Py_ssize_t shape_and_strides[MAX_NDIM * 2]
+
+
+@cython.final
+cdef class CArray(function.CPointer):
+
+    cdef:
+        _CArray val
+
+    cdef void init(
+        self, void* data_ptr, Py_ssize_t data_size,
+        const shape_t& shape, const strides_t& strides) except*
+
+
+cdef struct _CIndexer:
+    Py_ssize_t size
+    Py_ssize_t shape_and_index[MAX_NDIM * 2]
+
+
+cdef class CIndexer(function.CPointer):
+    cdef:
+        _CIndexer val
+
+    cdef void init(self, Py_ssize_t size, const shape_t &shape) except*
+
+
+cdef class Indexer:
+    cdef:
+        readonly Py_ssize_t size
+        readonly shape_t shape
+        readonly bint _index_32_bits
+
+    cdef void init(self, const shape_t& shape)
+
+    cdef function.CPointer get_pointer(self)
+
+
+cdef Indexer _indexer_init(const shape_t& shape)
diff --git a/cupy/_core/_carray.pyx b/cupy/_core/_carray.pyx
new file mode 100644
index 0000000..e9b7841
--- /dev/null
+++ b/cupy/_core/_carray.pyx
@@ -0,0 +1,57 @@
+from cupy.cuda cimport function
+from cupy._core cimport internal
+
+
+cdef class CArray(function.CPointer):
+
+    cdef void init(
+            self, void* data_ptr, Py_ssize_t data_size,
+            const shape_t& shape, const strides_t& strides) except*:
+        cdef size_t ndim = shape.size()
+        assert ndim == strides.size()
+        assert ndim <= MAX_NDIM
+        cdef Py_ssize_t* shape_and_strides = (
+            self.val.shape_and_strides)
+        cdef size_t i
+
+        self.val.data = data_ptr
+        self.val.size = data_size
+        for i in range(ndim):
+            shape_and_strides[i] = shape[i]
+            shape_and_strides[i + ndim] = strides[i]
+        self.ptr = <void*>&self.val
+
+
+cdef class CIndexer(function.CPointer):
+
+    cdef void init(self, Py_ssize_t size, const shape_t &shape) except*:
+        cdef size_t ndim = shape.size()
+        assert ndim <= MAX_NDIM
+        self.val.size = size
+        cdef Py_ssize_t i
+        for i in range(<Py_ssize_t>shape.size()):
+            self.val.shape_and_index[i] = shape[i]
+        self.ptr = <void*>&self.val
+
+
+cdef class Indexer:
+
+    cdef void init(self, const shape_t& shape):
+        self.shape = shape
+        self.size = internal.prod(shape)
+        self._index_32_bits = self.size <= (1 << 31)
+
+    @property
+    def ndim(self):
+        return self.shape.size()
+
+    cdef function.CPointer get_pointer(self):
+        cdef CIndexer indexer = CIndexer.__new__(CIndexer)
+        indexer.init(self.size, self.shape)
+        return indexer
+
+
+cdef inline Indexer _indexer_init(const shape_t& shape):
+    cdef Indexer indexer = Indexer.__new__(Indexer)
+    indexer.init(shape)
+    return indexer
diff --git a/cupy/_core/_codeblock.py b/cupy/_core/_codeblock.py
new file mode 100644
index 0000000..b5ac4bc
--- /dev/null
+++ b/cupy/_core/_codeblock.py
@@ -0,0 +1,38 @@
+from typing import Any, List
+
+_CodeType = Any  # TODO(asi1024): Correct type annotation
+
+
+class CodeBlock:
+    """Code fragment for the readable format.
+    """
+
+    def __init__(self, head: str, codes: _CodeType) -> None:
+        self._head = '' if head == '' else head + ' '
+        self._codes = codes
+
+    def _to_str_list(self, indent_width: int = 0) -> List[str]:
+        codes: List[str] = []
+        codes.append(' ' * indent_width + self._head + '{')
+        for code in self._codes:
+            next_indent_width = indent_width + 2
+            if isinstance(code, str):
+                codes.append(' ' * next_indent_width + code)
+            elif isinstance(code, CodeBlock):
+                codes += code._to_str_list(indent_width=next_indent_width)
+            else:
+                assert False
+        codes.append(' ' * indent_width + '}')
+        return codes
+
+    def __str__(self) -> str:
+        """Emit CUDA program like the following format.
+
+        <<head>> {
+          <<begin codes>>
+          ...;
+          <<end codes>>
+        }
+        """
+
+        return '\n'.join(self._to_str_list())
diff --git a/cupy/_core/_cub_reduction.pxd b/cupy/_core/_cub_reduction.pxd
new file mode 100644
index 0000000..2eae617
--- /dev/null
+++ b/cupy/_core/_cub_reduction.pxd
@@ -0,0 +1,12 @@
+from cupy._core._carray cimport shape_t
+from cupy._core._kernel cimport _TypeMap
+from cupy._core.core cimport _ndarray_base
+
+
+cdef bint _try_to_call_cub_reduction(
+    self, list in_args, list out_args, const shape_t& a_shape,
+    stream, optimize_context, tuple key,
+    map_expr, reduce_expr, post_map_expr,
+    reduce_type, _TypeMap type_map,
+    tuple reduce_axis, tuple out_axis, const shape_t& out_shape,
+    _ndarray_base ret) except *
diff --git a/cupy/_core/_cub_reduction.pyx b/cupy/_core/_cub_reduction.pyx
new file mode 100644
index 0000000..b5504ed
--- /dev/null
+++ b/cupy/_core/_cub_reduction.pyx
@@ -0,0 +1,712 @@
+from cupy._core._carray cimport shape_t
+from cupy._core cimport _kernel
+from cupy._core cimport _optimize_config
+from cupy._core cimport _reduction
+from cupy._core cimport _scalar
+from cupy._core.core cimport compile_with_cache
+from cupy._core.core cimport _ndarray_base
+from cupy._core.core cimport _internal_ascontiguousarray
+from cupy._core cimport internal
+from cupy.cuda cimport cub
+from cupy.cuda cimport function
+from cupy.cuda cimport memory
+from cupy_backends.cuda.api cimport runtime
+
+import math
+import string
+import sys
+from cupy import _environment
+from cupy._core._kernel import _get_param_info
+from cupy.cuda import driver
+from cupy import _util
+
+
+cdef function.Function _create_cub_reduction_function(
+        name, block_size, items_per_thread,
+        reduce_type, params, arginfos, identity,
+        pre_map_expr, reduce_expr, post_map_expr,
+        _kernel._TypeMap type_map, preamble, options):
+    # A (incomplete) list of internal variables:
+    # _J            : the index of an element in the array
+    # ROCm5.3 and above requires c++14
+    if runtime._is_hip_environment:
+        options += ('--std=c++14',)
+    else:
+        # static_assert needs at least C++11 in NVRTC
+        options += ('--std=c++11',)
+
+    cdef str backend
+    if runtime._is_hip_environment:
+        # In ROCm, we need to set the include path. This does not work for
+        # hiprtc as of ROCm 3.5.0, so we must use hipcc.
+        options += ('-I' + _rocm_path + '/include', '-O2')
+        backend = 'nvcc'  # this is confusing...
+    elif sys.platform.startswith('win32'):
+        # See #4771. NVRTC on Windows seems to have problems in handling empty
+        # macros, so any usage like this:
+        #     #ifndef CUB_NS_PREFIX
+        #     #define CUB_NS_PREFIX
+        #     #endif
+        # will drive NVRTC nuts (error: this declaration has no storage class
+        # or type specifier). However, we cannot find a minimum reproducer to
+        # confirm this is the root cause, so we work around by using nvcc.
+        backend = 'nvcc'
+    else:
+        # use jitify + nvrtc
+        # TODO(leofang): how about simply specifying jitify=True when calling
+        # compile_with_cache()?
+        options += ('-DCUPY_USE_JITIFY',)
+        backend = 'nvrtc'
+
+    # TODO(leofang): try splitting the for-loop into full tiles and partial
+    # tiles to utilize LoadDirectBlockedVectorized? See, for example,
+    # https://github.com/NVlabs/cub/blob/c3cceac115c072fb63df1836ff46d8c60d9eb304/cub/agent/agent_reduce.cuh#L311-L346
+
+    cdef str module_code = _get_cub_header_include()
+    module_code += '''
+${type_preamble}
+${preamble}
+
+typedef ${reduce_type} _type_reduce;
+
+static_assert(sizeof(_type_reduce) <= 32,
+    "The intermediate reduction type is assumed to be at most 32 bytes.");
+
+// Compile-time constants for CUB template specializations
+#define ITEMS_PER_THREAD  ${items_per_thread}
+#define BLOCK_SIZE        ${block_size}
+
+// for hipCUB: use the hipcub namespace
+#ifdef __HIP_DEVICE_COMPILE__
+#define cub hipcub
+#endif
+
+#if defined FIRST_PASS
+    typedef type_in0_raw  type_mid_in;
+    typedef _type_reduce  type_mid_out;
+    #define POST_MAP(a)   out0 = a;
+#elif defined SECOND_PASS
+    typedef _type_reduce  type_mid_in;
+    typedef type_out0_raw type_mid_out;
+    #define POST_MAP(a)   (${post_map_expr})
+#else  // one-pass reduction
+    typedef type_in0_raw  type_mid_in;
+    typedef type_out0_raw type_mid_out;
+    #define POST_MAP(a)   (${post_map_expr})
+#endif
+
+struct _reduction_op {
+    __device__ __forceinline__ _type_reduce operator()(
+        const _type_reduce &a, const _type_reduce &b) const {
+        return ${reduce_expr};
+    }
+};
+
+extern "C"
+__global__ void ${name}(${params}) {
+  unsigned int _tid = threadIdx.x;
+'''
+
+    if pre_map_expr == 'in0':
+        module_code += '''
+  // Specialize BlockLoad type for faster (?) loading
+  typedef cub::BlockLoad<_type_reduce, BLOCK_SIZE,
+                         ITEMS_PER_THREAD, cub::BLOCK_LOAD_DIRECT> BlockLoadT;
+
+  // Shared memory for loading
+  __shared__ typename BlockLoadT::TempStorage temp_storage_load;
+'''
+
+    module_code += '''
+  // Specialize BlockReduce type for our thread block
+  typedef cub::BlockReduce<_type_reduce, BLOCK_SIZE> BlockReduceT;
+
+  // Shared memory for reduction
+  __shared__ typename BlockReduceT::TempStorage temp_storage;
+
+  // Declare reduction operation
+  _reduction_op op;
+
+  // input & output raw pointers
+  const type_mid_in* _in0 = static_cast<const type_mid_in*>(_raw_in0);
+  type_mid_out* _out0 = static_cast<type_mid_out*>(_raw_out0);
+
+  // Per-thread tile data
+  _type_reduce _sdata[ITEMS_PER_THREAD];
+  #pragma unroll
+  for (int j = 0; j < ITEMS_PER_THREAD; j++) {
+      _sdata[j] = _type_reduce(${identity});
+  }
+
+  // each block handles the reduction of 1 segment
+  size_t segment_idx = blockIdx.x * _segment_size;
+  const type_mid_in* segment_head = _in0 + segment_idx;
+  size_t i = 0;  // tile head within the segment
+  int tile_size = (BLOCK_SIZE * ITEMS_PER_THREAD < _segment_size ?
+                   BLOCK_SIZE * ITEMS_PER_THREAD :
+                   _segment_size);
+  sizeT _seg_size = _segment_size;
+
+  #if defined FIRST_PASS
+  // for two-pass reduction only: "last segment" is special
+  if (_array_size > 0) {
+      if (_array_size - segment_idx <= _segment_size) {
+          _seg_size = _array_size - segment_idx;
+      }
+      #ifdef __HIP_DEVICE_COMPILE__
+      // We don't understand HIP...
+      __syncthreads();  // Propagate the new value back to memory
+      #endif
+  }
+  #endif
+
+  // loop over tiles within 1 segment
+  _type_reduce aggregate = _type_reduce(${identity});
+  for (i = 0; i < _seg_size; i += BLOCK_SIZE * ITEMS_PER_THREAD) {
+      // for the last tile
+      if (_seg_size - i <= tile_size) { tile_size = _seg_size - i; }
+'''
+
+    if pre_map_expr == 'in0':
+        module_code += '''
+      // load a tile
+      BlockLoadT(temp_storage_load).Load(segment_head + i, _sdata, tile_size,
+                                         _type_reduce(${identity}));
+'''
+    else:  # pre_map_expr could be something like "in0 != type_in0_raw(0)"
+        module_code += '''
+      // load a tile
+      #pragma unroll
+      for (int j = 0; j < ITEMS_PER_THREAD; j++) {
+          // index of the element in a tile
+          int e_idx = _tid * ITEMS_PER_THREAD + j;
+
+          // some pre_map_expr uses _J internally...
+          #if defined FIRST_PASS
+          int _J = (segment_idx + i + e_idx);
+          #else  // only one pass
+          int _J = (segment_idx + i + e_idx) % _seg_size;
+          #endif
+
+          if (e_idx < tile_size) {
+              const type_mid_in in0 = *(segment_head + i + e_idx);
+              _sdata[j] = static_cast<_type_reduce>(${pre_map_expr});
+          } else {
+              _sdata[j] = _type_reduce(${identity});
+          }
+      }
+'''
+
+    module_code += '''
+      // Compute block reduction
+      // Note that the output is only meaningful for thread 0
+      aggregate = op(aggregate, BlockReduceT(temp_storage).Reduce(_sdata, op));
+
+      __syncthreads();  // for reusing temp_storage
+  }
+
+  if (_tid == 0) {
+      type_mid_out& out0 = *(_out0 + blockIdx.x);
+      POST_MAP(aggregate);
+  }
+}
+'''
+
+    module_code = string.Template(module_code).substitute(
+        name=name,
+        block_size=block_size,
+        items_per_thread=items_per_thread,
+        reduce_type=reduce_type,
+        params=_get_cub_kernel_params(params, arginfos),
+        identity=identity,
+        reduce_expr=reduce_expr,
+        pre_map_expr=pre_map_expr,
+        post_map_expr=post_map_expr,
+        type_preamble=type_map.get_typedef_code(),
+        preamble=preamble)
+
+    # To specify the backend, we have to explicitly spell out the default
+    # values for arch, cachd, and prepend_cupy_headers to bypass cdef/cpdef
+    # limitation...
+    module = compile_with_cache(
+        module_code, options, arch=None, cachd_dir=None,
+        prepend_cupy_headers=True, backend=backend)
+    return module.get_function(name)
+
+
+@_util.memoize(for_each_device=True)
+def _SimpleCubReductionKernel_get_cached_function(
+        map_expr, reduce_expr, post_map_expr, reduce_type,
+        params, arginfos, _kernel._TypeMap type_map,
+        name, block_size, identity, preamble,
+        options, cub_params):
+    items_per_thread = cub_params[0]
+    name = name.replace('cupy_', 'cupy_cub_')
+    name = name.replace('cupyx_', 'cupyx_cub_')
+    return _create_cub_reduction_function(
+        name, block_size, items_per_thread,
+        reduce_type, params, arginfos, identity,
+        map_expr, reduce_expr, post_map_expr,
+        type_map, preamble, options)
+
+
+cdef str _cub_path = _environment.get_cub_path()
+cdef str _nvcc_path = _environment.get_nvcc_path()
+cdef str _rocm_path = _environment.get_rocm_path()
+cdef str _hipcc_path = _environment.get_hipcc_path()
+cdef str _cub_header = None
+
+
+cdef str _get_cub_header_include():
+    global _cub_header
+    if _cub_header is not None:
+        return _cub_header
+
+    assert _cub_path is not None
+    if _cub_path == '<bundle>':
+        _cub_header = '''
+#include <cupy/cuda_workaround.h>
+#include <cupy/cub/cub/block/block_reduce.cuh>
+#include <cupy/cub/cub/block/block_load.cuh>
+'''
+    elif _cub_path == '<CUDA>':
+        _cub_header = '''
+#include <cub/block/block_reduce.cuh>
+#include <cub/block/block_load.cuh>
+'''
+    elif _cub_path == '<ROCm>':
+        # As of ROCm 3.5.0, the block headers cannot be included by themselves
+        # (many macros left undefined), so we must use the master header.
+        _cub_header = '''
+#include <hipcub/hipcub.hpp>
+'''
+    return _cub_header
+
+
+# make it cpdef'd for unit tests
+cpdef inline tuple _can_use_cub_block_reduction(
+        list in_args, list out_args, tuple reduce_axis, tuple out_axis):
+    '''
+    If CUB BlockReduce can be used, this function returns a tuple of the needed
+    parameters, otherwise returns None.
+    '''
+    cdef tuple axis_permutes_cub
+    cdef _ndarray_base in_arr
+    cdef Py_ssize_t contiguous_size = 1
+    cdef str order
+
+    # detect whether CUB headers exists somewhere:
+    if _cub_path is None:
+        import warnings
+        warnings.warn('CUB headers are not found.', RuntimeWarning)
+        return None
+
+    # we currently support reductions with 1 input and 1 output
+    if len(in_args) != 1 or len(out_args) != 1:
+        return None
+
+    in_arr = in_args[0]
+
+    # the axes might not be sorted when we arrive here...
+    reduce_axis = tuple(sorted(reduce_axis))
+    out_axis = tuple(sorted(out_axis))
+
+    # check reduction axes, if not contiguous then fall back to old kernel
+    if in_arr._f_contiguous:
+        order = 'F'
+        if not cub._cub_device_segmented_reduce_axis_compatible(
+                reduce_axis, in_arr.ndim, order):
+            return None
+        axis_permutes_cub = reduce_axis + out_axis
+    elif in_arr._c_contiguous:
+        order = 'C'
+        if not cub._cub_device_segmented_reduce_axis_compatible(
+                reduce_axis, in_arr.ndim, order):
+            return None
+        axis_permutes_cub = out_axis + reduce_axis
+    else:
+        return None
+    if axis_permutes_cub != tuple(range(in_arr.ndim)):
+        return None
+
+    # full-reduction of N-D array: need to invoke the kernel twice
+    cdef bint full_reduction = True if len(out_axis) == 0 else False
+
+    # check if the number of elements is too large
+    # (ref: cupy/cupy#3309 for CUB limit)
+    for i in reduce_axis:
+        contiguous_size *= in_arr.shape[i]
+    if contiguous_size > 0x7fffffffffffffff or contiguous_size == 0:
+        return None
+    if full_reduction:
+        # assume a GPU has at most 64 GB of physical memory
+        if contiguous_size > 0x1000000000:
+            return None
+    else:
+        # the number of blocks to be launched exceeds INT_MAX:
+        if in_arr.size // contiguous_size > 0x7fffffff:
+            return None
+
+    # rare event (mainly for conda-forge users): nvcc is not found!
+    if not runtime._is_hip_environment:
+        if _nvcc_path is None:
+            return None
+    else:
+        if _hipcc_path is None:
+            return None
+
+    return (axis_permutes_cub, contiguous_size, full_reduction)
+
+
+# similar to cupy._core._kernel._get_kernel_params()
+cdef str _get_cub_kernel_params(tuple params, tuple arginfos):
+    cdef _kernel.ParameterInfo p
+    cdef _kernel._ArgInfo arginfo
+    cdef lst = []
+    cdef str c_type, c_name
+    cdef int i
+    assert len(params) == len(arginfos)
+
+    for i, (p, arginfo) in enumerate(zip(params, arginfos)):
+        c_name = arginfo.get_c_var_name(p)
+        if i < len(params) - 2:
+            c_type = 'const void*' if p.is_const else 'void*'
+        else:
+            # for segment size and array size
+            c_type = arginfo.get_param_c_type(p)
+        lst.append('{} {}'.format(c_type, c_name))
+    return ', '.join(lst)
+
+
+cdef Py_ssize_t _cub_default_block_size = (
+    256 if runtime._is_hip_environment else 512)
+
+
+cdef (Py_ssize_t, Py_ssize_t) _get_cub_block_specs(  # NOQA
+        Py_ssize_t contiguous_size):
+    # This is recommended in the CUB internal and should be an
+    # even number
+    items_per_thread = 4
+
+    # Calculate the reduction block dimensions.
+    # Ideally, we want each block to handle one segment, so:
+    # 1. block size < segment size: the block loops over the segment
+    # 2. block size >= segment size: the segment fits in the block
+    block_size = (contiguous_size + items_per_thread - 1) // items_per_thread
+    block_size = internal.clp2(block_size)
+    warp_size = 32 if not runtime._is_hip_environment else 64
+    if block_size < warp_size:
+        block_size = warp_size
+    elif block_size > _cub_default_block_size:
+        block_size = _cub_default_block_size
+
+    return items_per_thread, block_size
+
+
+cdef _scalar.CScalar _cub_convert_to_c_scalar(
+        Py_ssize_t segment_size, Py_ssize_t value):
+    if segment_size > 0x7fffffff:
+        return _scalar.scalar_to_c_scalar(value)
+    else:
+        return _scalar.CScalar.from_int32(value)
+
+
+cdef inline void _cub_two_pass_launch(
+        str name, Py_ssize_t block_size, Py_ssize_t segment_size,
+        Py_ssize_t items_per_thread, str reduce_type, tuple params,
+        list in_args, list out_args,
+        str identity, str pre_map_expr, str reduce_expr, str post_map_expr,
+        _kernel._TypeMap type_map, str preamble,
+        tuple options, stream) except*:
+    '''
+    Notes:
+    1. Two-pass reduction: the first pass distributes an even share over
+       a number of blocks (with block_size threads), and the second pass
+       does reduction over 1 block of threads
+    '''
+
+    cdef list out_args_2nd_pass = [out_args[0]]
+    cdef Py_ssize_t contiguous_size, out_block_num
+    cdef function.Function func
+    cdef memory.MemoryPointer memptr
+    cdef str post_map_expr1, post_map_expr2, f
+    cdef list inout_args
+    cdef tuple cub_params
+    cdef size_t gridx, blockx
+    cdef _ndarray_base in_arr
+
+    # fair share
+    contiguous_size = min(segment_size, block_size * items_per_thread)
+    out_block_num = (segment_size + contiguous_size - 1) // contiguous_size
+    assert out_block_num <= 0x7fffffff
+
+    # Because we can't know sizeof(reduce_type) in advance, here we
+    # conservatively assume it's 32 bytes and allocate a work area
+    memptr = memory.alloc(out_block_num * 32)
+    out_args[0] = memptr
+
+    # ************************ 1st pass ************************
+    name += '_pass1'
+    inout_args = [in_args[0], out_args[0],
+                  _cub_convert_to_c_scalar(segment_size, contiguous_size),
+                  _cub_convert_to_c_scalar(segment_size, segment_size)]
+    cub_params = (items_per_thread,)
+
+    if 'mean' in name:
+        post_map_expr1 = post_map_expr.replace('_in_ind.size()', '1.0')
+        post_map_expr1 = post_map_expr1.replace('_out_ind.size()', '1.0')
+    elif any((f in name for f in ('argmax', 'argmin'))):
+        # Workaround: in NumPy the indices are always generated based on
+        # a C-order array (since PyArray_ContiguousFromAny was called).
+        # We have to do a conversion here (?) since we do not retain the
+        # info on strides.
+        # TODO(leofang): improve this workaround
+        in_arr = in_args[0]
+        if in_arr.ndim > 1 and in_arr._f_contiguous:
+            in_arr = _internal_ascontiguousarray(in_arr)
+            inout_args[0] = in_args[0] = in_arr
+        post_map_expr1 = post_map_expr
+    else:
+        post_map_expr1 = post_map_expr
+
+    # Retrieve the kernel function
+    func = _SimpleCubReductionKernel_get_cached_function(
+        pre_map_expr, reduce_expr, post_map_expr1, reduce_type,
+        params,
+        _kernel._get_arginfos(inout_args),
+        type_map,
+        name, block_size, identity, preamble,
+        ('-DFIRST_PASS=1',), cub_params)
+
+    # Kernel arguments passed to the __global__ function.
+    gridx = <size_t>(out_block_num * block_size)
+    blockx = <size_t>block_size
+
+    # Launch the kernel
+    func.linear_launch(gridx, inout_args, 0, blockx, stream)
+
+    # ************************ 2nd pass ************************
+    name = name[:-1] + '2'
+    contiguous_size = out_block_num
+    out_block_num = 1
+    in_args = out_args
+    out_args = out_args_2nd_pass
+    inout_args = [in_args[0], out_args[0],
+                  _cub_convert_to_c_scalar(segment_size, contiguous_size),
+                  _cub_convert_to_c_scalar(segment_size, segment_size)]
+
+    # For mean()
+    if 'mean' in name:
+        post_map_expr2 = post_map_expr.replace('_in_ind.size()',
+                                               '_array_size')
+        post_map_expr2 = post_map_expr2.replace('_out_ind.size()', '1.0')
+    else:
+        post_map_expr2 = post_map_expr
+
+    # Retrieve the kernel function
+    func = _SimpleCubReductionKernel_get_cached_function(
+        'in0', reduce_expr, post_map_expr2, reduce_type,
+        params,
+        _kernel._get_arginfos(inout_args),
+        type_map,
+        name, block_size, identity, preamble,
+        ('-DSECOND_PASS=1',), cub_params)
+
+    # Kernel arguments passed to the __global__ function.
+    gridx = <size_t>(out_block_num * block_size)
+    blockx = <size_t>block_size
+
+    # Launch the kernel
+    func.linear_launch(gridx, inout_args, 0, blockx, stream)
+
+
+cdef inline void _launch_cub(
+        self, out_block_num, block_size, block_stride,
+        in_args, out_args, in_shape, out_shape, type_map,
+        map_expr, reduce_expr, post_map_expr, reduce_type,
+        stream, params, cub_params) except *:
+    cdef bint full_reduction
+    cdef Py_ssize_t contiguous_size, items_per_thread
+    cdef function.Function func
+
+    # Kernel arguments passed to the __global__ function.
+    items_per_thread = cub_params[0]
+    contiguous_size = cub_params[1]
+    full_reduction = cub_params[2]
+
+    if full_reduction:
+        _cub_two_pass_launch(
+            self.name, block_size, contiguous_size, items_per_thread,
+            reduce_type, params, in_args, out_args, self.identity,
+            map_expr, reduce_expr, post_map_expr,
+            type_map, self.preamble, (), stream)
+        return
+    else:
+        inout_args = (
+            in_args + out_args +
+            [_cub_convert_to_c_scalar(
+                contiguous_size, contiguous_size),
+             _cub_convert_to_c_scalar(
+                 contiguous_size, 0)])
+        arginfos = _kernel._get_arginfos(inout_args)
+        func = _SimpleCubReductionKernel_get_cached_function(
+            map_expr, reduce_expr, post_map_expr, reduce_type,
+            params, arginfos, type_map,
+            self.name, block_size, self.identity, self.preamble,
+            (), cub_params)
+
+        func.linear_launch(
+            out_block_num * block_size, inout_args, 0, block_size, stream)
+
+
+def _get_cub_optimized_params(
+        self, optimize_config, in_args, out_args, in_shape, out_shape,
+        type_map, map_expr, reduce_expr, post_map_expr, reduce_type,
+        stream, full_reduction, out_block_num, contiguous_size, params):
+    in_args = [_reduction._optimizer_copy_arg(a) for a in in_args]
+    out_args = [_reduction._optimizer_copy_arg(a) for a in out_args]
+
+    items_per_thread, block_size = (
+        _get_cub_block_specs(contiguous_size))
+    default_block_size_log = math.floor(math.log2(block_size))
+    default_items_per_thread = items_per_thread
+
+    def target_func(block_size, items_per_thread):
+        block_stride = block_size * items_per_thread
+        cub_params = (
+            items_per_thread, contiguous_size, full_reduction)
+        _launch_cub(
+            self,
+            out_block_num, block_size, block_stride, in_args, out_args,
+            in_shape, out_shape, type_map, map_expr, reduce_expr,
+            post_map_expr, reduce_type, stream, params, cub_params)
+
+    def suggest_func(trial):
+        block_size_log = trial.suggest_int('block_size_log', 5, 10)
+        block_size = 2 ** block_size_log
+        items_per_thread = trial.suggest_int(
+            'items_per_thread', 2, 32, step=2)
+
+        trial.set_user_attr('block_size', block_size)
+        return block_size, items_per_thread
+
+    # CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES is a possible error
+    optimize_impl = optimize_config.optimize_impl
+    best = optimize_impl(
+        optimize_config, target_func, suggest_func,
+        default_best={
+            'block_size_log': default_block_size_log,
+            'items_per_thread': default_items_per_thread,
+        }, ignore_error=(driver.CUDADriverError,))
+
+    return best.params['items_per_thread'], best.user_attrs['block_size']
+
+
+cdef bint _try_to_call_cub_reduction(
+        self, list in_args, list out_args, const shape_t& a_shape,
+        stream, optimize_context, tuple key,
+        map_expr, reduce_expr, post_map_expr,
+        reduce_type, _kernel._TypeMap type_map,
+        tuple reduce_axis, tuple out_axis, const shape_t& out_shape,
+        _ndarray_base ret) except *:
+    """Try to use cub.
+
+    Updates `ret` and returns a boolean value whether cub is used.
+
+    Note: input_expr and output_expr are not used in CUB kernels.
+    """
+    cdef tuple axis_permutes
+    cdef tuple params, opt_params
+    cdef shape_t in_shape
+    cdef Py_ssize_t i
+    cdef Py_ssize_t contiguous_size = -1
+    cdef Py_ssize_t block_size, block_stride, out_block_num = 0
+
+    # decide to use CUB or not
+    can_use_cub = _can_use_cub_block_reduction(
+        in_args, out_args, reduce_axis, out_axis)
+
+    if can_use_cub is None:
+        return False
+
+    axis_permutes, contiguous_size, full_reduction = can_use_cub
+
+    in_shape = _reduction._set_permuted_args(
+        in_args, axis_permutes, a_shape, self.in_params)
+
+    if in_args[0]._f_contiguous:
+        ret._set_contiguous_strides(ret.dtype.itemsize, False)
+        out_args[0] = ret
+
+    if not full_reduction:  # just need one pass
+        out_block_num = 1  # = number of segments
+        for i in out_axis:
+            out_block_num *= in_shape[i]
+
+        if 'mean' in self.name:
+            post_map_expr = post_map_expr.replace(
+                '_in_ind.size()', '_segment_size')
+            post_map_expr = post_map_expr.replace(
+                '_out_ind.size()', '1.0')
+
+    if contiguous_size > 0x7fffffff:  # INT_MAX
+        size_type = 'uint64'
+    else:
+        size_type = 'int32'
+    type_map = _kernel._TypeMap(type_map._pairs + (('sizeT', size_type),))
+    params = (self._params[0:2]
+              + _get_param_info(size_type + ' _segment_size', True)
+              + _get_param_info(size_type + ' _array_size', True))
+
+    # HACK for ReductionKernel:
+    # 1. input/output arguments might not be named as in0/out0
+    # 2. pre-/post- maps might not contain in0/out0
+    # 3. type_map does not contain the expected names (type_in0_raw and
+    #    type_out0_raw)
+    cdef str old_in0 = params[0].name, old_out0 = params[1].name
+    if old_in0 != 'in0' or old_out0 != 'out0':
+        # avoid overwriting self's attributes
+        params = (_get_param_info('T in0', True)
+                  + _get_param_info('T out0', False)
+                  + params[2:])
+        map_expr = map_expr.replace(old_in0, 'in0')
+        post_map_expr = post_map_expr.replace(old_out0, 'out0')
+        type_map = _kernel._TypeMap(type_map._pairs + (
+            ('type_in0_raw', in_args[0].dtype.type),
+            ('type_out0_raw', out_args[0].dtype.type),
+        ))
+
+    # Calculate the reduction block dimensions.
+    optimize_context = _optimize_config.get_current_context()
+    if optimize_context is None:
+        # Calculate manually
+        items_per_thread, block_size = _get_cub_block_specs(contiguous_size)
+    else:
+        # Optimize dynamically
+        key = ('cub_reduction',) + key
+        opt_params = optimize_context.get_params(key)
+        if opt_params is None:
+            opt_params = _get_cub_optimized_params(
+                self,
+                optimize_context.config, in_args, out_args,
+                in_shape, out_shape, type_map, map_expr, reduce_expr,
+                post_map_expr, reduce_type, stream,
+                full_reduction, out_block_num, contiguous_size, params)
+            optimize_context.set_params(key, opt_params)
+        items_per_thread, block_size = opt_params
+
+    block_stride = block_size * items_per_thread
+    cub_params = (items_per_thread, contiguous_size, full_reduction)
+
+    _launch_cub(
+        self,
+        out_block_num,
+        block_size,
+        block_stride,
+        in_args, out_args,
+        in_shape, out_shape,
+        type_map,
+        map_expr, reduce_expr, post_map_expr, reduce_type,
+        stream, params, cub_params)
+
+    return True
diff --git a/cupy/_core/_dtype.pxd b/cupy/_core/_dtype.pxd
new file mode 100644
index 0000000..6dfc4af
--- /dev/null
+++ b/cupy/_core/_dtype.pxd
@@ -0,0 +1,10 @@
+cpdef get_dtype(t)
+cpdef tuple get_dtype_with_itemsize(t)
+cpdef int to_cuda_dtype(dtype, bint is_half_allowed=*) except -1
+
+cpdef void _raise_if_invalid_cast(
+    from_dt,
+    to_dt,
+    str casting,
+    argname=*
+) except *
diff --git a/cupy/_core/_dtype.pyx b/cupy/_core/_dtype.pyx
new file mode 100644
index 0000000..e5bc725
--- /dev/null
+++ b/cupy/_core/_dtype.pyx
@@ -0,0 +1,126 @@
+cimport cython  # NOQA
+import numpy
+import warnings
+
+from cupy_backends.cuda.api cimport runtime
+
+
+all_type_chars = '?bhilqBHILQefdFD'
+# for c in '?bhilqBHILQefdFD':
+#    print('#', c, '...', np.dtype(c).name)
+# ? ... bool
+# b ... int8
+# h ... int16
+# i ... int32
+# l ... int64  (int32 in windows)
+# q ... int64
+# B ... uint8
+# H ... uint16
+# I ... uint32
+# L ... uint64  (uint32 in windows)
+# Q ... uint64
+# e ... float16
+# f ... float32
+# d ... float64
+# F ... complex64
+# D ... complex128
+
+cdef dict _dtype_dict = {}
+cdef _dtype = numpy.dtype
+
+
+cdef _init_dtype_dict():
+    for i in (int, float, bool, complex, None):
+        dtype = _dtype(i)
+        _dtype_dict[i] = (dtype, dtype.itemsize)
+    for i in all_type_chars:
+        dtype = _dtype(i)
+        item = (dtype, dtype.itemsize)
+        _dtype_dict[i] = item
+        _dtype_dict[dtype.type] = item
+    for i in {str(_dtype(i)) for i in all_type_chars}:
+        dtype = _dtype(i)
+        _dtype_dict[i] = (dtype, dtype.itemsize)
+
+
+_init_dtype_dict()
+
+
+@cython.profile(False)
+cpdef get_dtype(t):
+    ret = _dtype_dict.get(t, None)
+    if ret is None:
+        return _dtype(t)
+    return ret[0]
+
+
+@cython.profile(False)
+cpdef tuple get_dtype_with_itemsize(t):
+    ret = _dtype_dict.get(t, None)
+    if ret is None:
+        t = _dtype(t)
+        return t, t.itemsize
+    return ret
+
+
+cpdef int to_cuda_dtype(dtype, bint is_half_allowed=False) except -1:
+    cdef str dtype_char
+    try:
+        dtype_char = dtype.char
+    except AttributeError:
+        dtype_char = dtype
+
+    if dtype_char == 'e' and is_half_allowed:
+        return runtime.CUDA_R_16F
+    elif dtype_char == 'f':
+        return runtime.CUDA_R_32F
+    elif dtype_char == 'd':
+        return runtime.CUDA_R_64F
+    elif dtype_char == 'F':
+        return runtime.CUDA_C_32F
+    elif dtype_char == 'D':
+        return runtime.CUDA_C_64F
+    elif dtype_char == 'E' and is_half_allowed:
+        # complex32, not supported in NumPy
+        return runtime.CUDA_C_16F
+    else:
+        raise TypeError('dtype is not supported: {}'.format(dtype))
+
+
+cdef _numpy_can_cast = numpy.can_cast
+
+
+cpdef void _raise_if_invalid_cast(
+    from_dt, to_dt, str casting, argname="array data"
+) except *:
+    """Raise an error if a cast is not valid.  Also checks whether the cast
+    goes from complex to real and warns if it does.
+
+    The error raised can be customized by giving `obj`.  May pass a (lambda)
+    function to avoid string construction on success.
+    This function exists mainly to build a similar error everywhere.
+
+    """
+    if from_dt is to_dt:
+        return
+
+    to_dt = get_dtype(to_dt)  # may still be a type not a dtype instance
+
+    if casting == "same_kind" and from_dt.kind == to_dt.kind:
+        # same-kind is the most common casting used and for NumPy dtypes.
+        return
+    if _numpy_can_cast(from_dt, to_dt, casting):
+        if casting == "unsafe" and from_dt.kind == "c" and to_dt.kind in "iuf":
+            # Complex warning, we are dropping the imagine part:
+            warnings.warn(
+                'Casting complex values to real discards the imaginary part',
+                numpy.ComplexWarning)
+
+        return
+
+    # Casting is not possible, raise the error
+    if not isinstance(argname, str):
+        argname = argname()
+    raise TypeError(
+        f'Cannot cast {argname} from {from_dt!r} to {to_dt!r} '
+        f'according to the rule \'{casting}\'')
diff --git a/cupy/_core/_fusion_interface.py b/cupy/_core/_fusion_interface.py
new file mode 100644
index 0000000..44f9a4b
--- /dev/null
+++ b/cupy/_core/_fusion_interface.py
@@ -0,0 +1,272 @@
+import numpy
+
+from cupy._core._dtype import get_dtype
+import cupy
+from cupy._core import _fusion_thread_local
+from cupy._core import core
+from cupy._core._scalar import get_typename
+
+
+_thread_local = _fusion_thread_local.thread_local
+
+
+_dtype_to_astype_dict = None
+
+
+def _set_dtype_to_astype_dict():
+    """Set a dict with dtypes and astype ufuncs to `_dtype_to_astype_dict`.
+
+    Creates a ufunc for type cast operations, and set a dict with keys
+    as the dtype of the output array and values as astype ufuncs.
+    This function is called at most once.
+    """
+    global _dtype_to_astype_dict
+    _dtype_to_astype_dict = {}
+
+    dtype_list = [numpy.dtype(type_char) for type_char in '?bhilqBHILQefdFD']
+
+    for t in dtype_list:
+        name = 'astype_{}'.format(t)
+        rules = tuple(['{}->{}'.format(s.char, t.char) for s in dtype_list])
+        command = 'out0 = static_cast< {} >(in0)'.format(get_typename(t))
+        _dtype_to_astype_dict[t] = core.create_ufunc(name, rules, command)
+
+
+class _VariableProxy:
+    """Abstracted array/scalar object passed to the target function.
+    """
+
+    def __init__(self, content):
+        assert isinstance(content, cupy._core._fusion_variable._TraceVariable)
+        self.content = content
+
+    def __neg__(self):
+        return cupy.negative(self)
+
+    def __add__(self, other):
+        return cupy.add(self, other)
+
+    def __radd__(self, other):
+        return cupy.add(other, self)
+
+    def __sub__(self, other):
+        return cupy.subtract(self, other)
+
+    def __rsub__(self, other):
+        return cupy.subtract(other, self)
+
+    def __mul__(self, other):
+        return cupy.multiply(self, other)
+
+    def __rmul__(self, other):
+        return cupy.multiply(other, self)
+
+    def __div__(self, other):
+        return cupy.divide(self, other)
+
+    def __rdiv__(self, other):
+        return cupy.divide(other, self)
+
+    def __truediv__(self, other):
+        return cupy.true_divide(self, other)
+
+    def __rtruediv__(self, other):
+        return cupy.true_divide(other, self)
+
+    def __floordiv__(self, other):
+        return cupy.floor_divide(self, other)
+
+    def __rfloordiv__(self, other):
+        return cupy.floor_divide(other, self)
+
+    def __mod__(self, other):
+        return cupy.remainder(self, other)
+
+    def __rmod__(self, other):
+        return cupy.remainder(other, self)
+
+    def __pow__(self, other):
+        return cupy.power(self, other)
+
+    def __lshift__(self, other):
+        return cupy.left_shift(self, other)
+
+    def __rlshift__(self, other):
+        return cupy.left_shift(other, self)
+
+    def __rshift__(self, other):
+        return cupy.right_shift(self, other)
+
+    def __rrshift__(self, other):
+        return cupy.right_shift(other, self)
+
+    def __invert__(self):
+        return cupy.invert(self)
+
+    def __and__(self, other):
+        return cupy.bitwise_and(self, other)
+
+    def __rand__(self, other):
+        return cupy.bitwise_and(other, self)
+
+    def __or__(self, other):
+        return cupy.bitwise_or(self, other)
+
+    def __ror__(self, other):
+        return cupy.bitwise_or(other, self)
+
+    def __xor__(self, other):
+        return cupy.bitwise_xor(self, other)
+
+    def __rxor__(self, other):
+        return cupy.bitwise_xor(other, self)
+
+    def __lt__(self, other):
+        return cupy.less(self, other)
+
+    def __le__(self, other):
+        return cupy.less_equal(self, other)
+
+    def __eq__(self, other):
+        return cupy.equal(self, other)
+
+    def __ne__(self, other):
+        return cupy.not_equal(self, other)
+
+    def __ge__(self, other):
+        return cupy.greater_equal(self, other)
+
+    def __gt__(self, other):
+        return cupy.greater(self, other)
+
+    def copy(self):
+        return cupy.copy(self)
+
+    def astype(self, dtype, order=None, casting=None, subok=None, copy=True):
+        dtype = get_dtype(dtype)
+        if order is not None:
+            raise TypeError('order is not supported yet')
+        if casting is not None:
+            raise TypeError('casting is not supported yet')
+        if subok is not None:
+            raise TypeError('subok is not supported yet')
+        if not copy and self.dtype == dtype:
+            return self
+        if _dtype_to_astype_dict is None:
+            _set_dtype_to_astype_dict()
+        return _dtype_to_astype_dict[dtype](self)
+
+    def sum(self, axis=None, dtype=None, out=None, keepdims=False):
+        return cupy.sum(
+            self, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+    def prod(self, axis=None, dtype=None, out=None, keepdims=False):
+        return cupy.prod(
+            self, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+    def max(self, axis=None, out=None, keepdims=False):
+        return cupy.max(self, axis=axis, out=out, keepdims=keepdims)
+
+    def min(self, axis=None, out=None, keepdims=False):
+        return cupy.min(self, axis=axis, out=out, keepdims=keepdims)
+
+    def all(self, axis=None, out=None, keepdims=False):
+        return cupy.all(self, axis=axis, out=out, keepdims=keepdims)
+
+    def any(self, axis=None, out=None, keepdims=False):
+        return cupy.any(self, axis=axis, out=out, keepdims=keepdims)
+
+    @property
+    def dtype(self):
+        return self.content.dtype
+
+    @property
+    def ndim(self):
+        return self.content.ndim
+
+    @property
+    def shape(self):
+        raise NotImplementedError('`shape` is not supported, currently.')
+
+
+class _ScalarProxy(_VariableProxy):
+    """An abstracted scalar object passed to the target function.
+
+    Attributes:
+        dtype(dtype): The dtype of the array.
+        imag(_ArrayProxy): The imaginary part of the array (Not implemented)
+        real(_ArrayProxy): The real part of the array (Not implemented)
+        ndim(int): The number of dimensions of the array.
+    """
+
+    def __repr__(self):
+        return '_ScalarProxy({}, dtype={})'.format(
+            self._emit_param_name(), self.dtype)
+
+
+class _ArrayProxy(_VariableProxy):
+    """An abstracted array object passed to the target function.
+
+    Attributes:
+        dtype(dtype): The dtype of the array.
+        imag(_ArrayProxy): The imaginary part of the array (Not implemented)
+        real(_ArrayProxy): The real part of the array (Not implemented)
+        ndim(int): The number of dimensions of the array.
+    """
+
+    def __repr__(self):
+        return '_ArrayProxy([...], dtype=\'{}\', ndim={})'.format(
+            self.dtype.char, self.ndim)
+
+    def _inplace_op(self, ufunc, other):
+        return ufunc(self, other, self)
+
+    def __iadd__(self, other):
+        return self._inplace_op(cupy.add, other)
+
+    def __isub__(self, other):
+        return self._inplace_op(cupy.subtract, other)
+
+    def __imul__(self, other):
+        return self._inplace_op(cupy.multiply, other)
+
+    def __idiv__(self, other):
+        return self._inplace_op(cupy.divide, other)
+
+    def __itruediv__(self, other):
+        return self._inplace_op(cupy.true_divide, other)
+
+    def __ifloordiv__(self, other):
+        return self._inplace_op(cupy.floor_divide, other)
+
+    def __imod__(self, other):
+        return self._inplace_op(cupy.remainder, other)
+
+    def __ipow__(self, other):
+        return self._inplace_op(cupy.power, other)
+
+    def __ilshift__(self, other):
+        return self._inplace_op(cupy.left_shift, other)
+
+    def __irshift__(self, other):
+        return self._inplace_op(cupy.right_shift, other)
+
+    def __iand__(self, other):
+        return self._inplace_op(cupy.bitwise_and, other)
+
+    def __ior__(self, other):
+        return self._inplace_op(cupy.bitwise_or, other)
+
+    def __ixor__(self, other):
+        return self._inplace_op(cupy.bitwise_xor, other)
+
+    def __getitem__(self, index):
+        return _fusion_thread_local.call_indexing(self, index)
+
+    def __setitem__(self, slices, value):
+        if slices is Ellipsis or (
+                isinstance(slices, slice) and slices == slice(None)):
+            _fusion_thread_local.call_ufunc(
+                core.elementwise_copy, value, out=self)
+        else:
+            raise ValueError('The fusion supports `[...]` or `[:]`.')
diff --git a/cupy/_core/_fusion_kernel.pyx b/cupy/_core/_fusion_kernel.pyx
new file mode 100644
index 0000000..ed8f837
--- /dev/null
+++ b/cupy/_core/_fusion_kernel.pyx
@@ -0,0 +1,364 @@
+import itertools
+import string
+
+from libcpp cimport vector
+
+from cupy._core cimport _carray
+from cupy._core.core cimport _ndarray_init
+from cupy._core.core cimport compile_with_cache
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport internal
+from cupy._core cimport _routines_manipulation as _manipulation
+from cupy_backends.cuda.api cimport driver
+from cupy_backends.cuda.api cimport runtime
+
+import cupy as _cupy
+from cupy._core import _dtype
+from cupy import _util
+from cupy._core import _codeblock
+from cupy._core import _fusion_op
+from cupy._core._fusion_variable import _TraceVariable
+from cupy._core._fusion_variable import _TraceScalar
+from cupy._core._fusion_variable import _TraceArray
+
+
+cdef Py_ssize_t _default_block_size = (
+    256 if runtime._is_hip_environment else 512)
+
+
+@_util.memoize(for_each_device=True)
+def _cuda_compile(preamble, name, cuda_params, cuda_body, use_grid_sync):
+    template = (
+        '${preamble}\n\n'
+        'extern "C" __global__ void ${name}(${cuda_params}) ${cuda_body}\n'
+    )
+
+    if use_grid_sync:
+        template = '#include <cooperative_groups.h>\n\n' + template
+
+    code = string.Template(template).substitute(
+        preamble=preamble,
+        name=name,
+        cuda_params=cuda_params,
+        cuda_body=cuda_body)
+
+    # (For contributers) We can view the whole generated CUDA code
+    # by uncommenting the following line.
+    # print(code)
+
+    module = compile_with_cache(
+        code, (), None, None, True, 'nvrtc', False, use_grid_sync)
+    return module.get_function(name)
+
+
+cdef class FusedKernel:
+    cdef:
+        readonly object shape_constraints
+
+        readonly str _name
+        readonly list _params
+        readonly int _return_size
+        readonly str _submodule_code
+        readonly str _cuda_body
+        readonly dict _cuda_params_memo
+        readonly list _block_strides
+        readonly bint _use_grid_sync
+
+        readonly list _reduction_in_array
+        readonly list _reduction_out_array
+        readonly vector.vector[bint] _is_base
+        readonly list _dtypes
+        readonly vector.vector[Py_ssize_t] _input_index
+        readonly vector.vector[Py_ssize_t] _view_of
+        readonly vector.vector[Py_ssize_t] _out_params
+
+    def __init__(self, name, trace_result):
+        op_list = trace_result.op_list
+        params = trace_result.params
+        return_size = trace_result.return_size
+        self.shape_constraints = trace_result.shape_constraints
+
+        self._name = name
+        self._params = sorted(params, key=lambda x: x.serial_number)
+        self._cuda_params_memo = {}
+
+        # Generate the device functions.
+        submodule_code = '\n\n'.join(set(itertools.chain.from_iterable([
+            op.emit_preamble_codes() for op in op_list]))) + '\n\n'
+        submodule_code += '\n\n'.join(itertools.chain.from_iterable([
+            op.emit_submodule_codes() for op in op_list]))
+
+        # Generate the function body of a __global__ function.
+        codes = []
+
+        self._use_grid_sync = len(op_list) > 1
+
+        if self._use_grid_sync:
+            codes.append('namespace _cg = cooperative_groups;')
+            codes.append('_cg::grid_group _grid = _cg::this_grid();')
+
+        for i, op in enumerate(op_list):
+            if i > 0:
+                codes.append('_cg::sync(_grid);')
+            codes.append(op.emit_code())
+
+        self._submodule_code = submodule_code
+        self._cuda_body = str(_codeblock.CodeBlock('', codes))
+
+        # Check the format of the return value.
+        if return_size == 'none':
+            self._return_size = -1
+            self._out_params.resize(0)
+        elif return_size == 'single':
+            self._return_size = -2
+            self._out_params.resize(1)
+        else:
+            assert isinstance(return_size, int)
+            assert return_size >= 0
+            self._return_size = return_size
+            self._out_params.resize(return_size)
+
+        for p in self._params:
+            assert isinstance(p, _TraceVariable)
+
+        # Analyse the relationship between variables.
+
+        array_dict = {}
+        self._reduction_in_array = []
+        self._reduction_out_array = []
+        self._dtypes = []
+
+        for i, p in enumerate(self._params):
+            view_of = -1
+            input_index = -1
+            if p.input_index is not None:
+                input_index = p.input_index
+            if isinstance(p, _TraceArray):
+                if p._view_of is not None:
+                    view_of = array_dict[p._view_of.key()]
+                if p.is_output:
+                    self._out_params[p.output_index] = i
+            array_dict[p.key()] = i
+            self._is_base.push_back(p.is_base)
+            self._dtypes.append(_dtype.get_dtype(p.dtype))
+            self._input_index.push_back(input_index)
+            self._view_of.push_back(view_of)
+
+        self._block_strides = []
+
+        for op in op_list:
+            if isinstance(op, _fusion_op._ReductionTraceOp):
+                self._reduction_in_array.append(
+                    array_dict[op.in_params.item().key()])
+                self._reduction_out_array.append(
+                    array_dict[op.out_params.item().key()])
+                self._block_strides.append(
+                    'int {}'.format(op.block_stride_name))
+
+    def get_shapes_of_kernel_params(self, tuple args):
+        """Returns the shapes of parameters passed to kern.linear_launch.
+        """
+        cdef list kernel_param_shapes = []
+        cdef int axis
+        cdef list shape
+
+        for param in self._params:
+            shape = []
+            if isinstance(param, _TraceArray):
+                ashape = param.ashape
+                for axis in range(len(ashape)):
+                    dim = ashape[axis]
+                    if not isinstance(dim, int):
+                        dim = args[dim.input_index].shape[dim.axis]
+                    shape.append(dim)
+            kernel_param_shapes.append(tuple(shape))
+        return kernel_param_shapes
+
+    cdef list _get_ndarray_list(self, tuple args, list shapes):
+        """Get the list of ndarray corresponding to ``self._params``.
+        """
+        cdef list ndarray_list = []
+        cdef list params = self._params
+        cdef int i
+        for i in range(len(params)):
+            param = params[i]
+            shape = shapes[i]
+            if self._input_index[i] >= 0:
+                array = args[<Py_ssize_t>self._input_index[i]]
+            elif isinstance(param, _TraceScalar):
+                array = None
+            elif self._is_base[i]:
+                array = _ndarray_init(
+                    _cupy.ndarray, shape, self._dtypes[i], None)
+            else:
+                view_of = ndarray_list[<Py_ssize_t>self._view_of[i]]
+                if param.is_broadcast:
+                    array = _manipulation.broadcast_to(view_of, shape)
+                elif param.slice_key is not None:
+                    array = view_of[param.slice_key]
+                elif param.rotate_axis is not None:
+                    axis_permutes = list(param.rotate_axis)
+                    for i in range(param.ndim):
+                        if i not in param.rotate_axis:
+                            axis_permutes.append(i)
+                    axis_permutes = tuple(axis_permutes)
+                    array = _manipulation._transpose(view_of, axis_permutes)
+                else:
+                    assert False
+            # For debug
+            # if isinstance(array, ndarray) and param.rotate_axis is None:
+            #     assert array.shape == shape, (array.shape, shape)
+            ndarray_list.append(array)
+
+        return ndarray_list
+
+    cdef object _get_return_value(self, list ndarray_list):
+        """Get the return value of ``self.execute``.
+        """
+        cdef int i
+
+        if self._return_size == -1:
+            return None
+
+        if self._return_size == -2:
+            return ndarray_list[<Py_ssize_t>self._out_params[0]]
+
+        return tuple([
+            ndarray_list[<Py_ssize_t>self._out_params[i]]
+            for i in range(self._return_size)
+        ])
+
+    cdef tuple _get_kernel_size(self, list ndarray_list):
+        """Calculate the numnber of contiguous blocks in non-reduction axes
+        of input arrays, and set them to ``self._contiguous_size``.
+        """
+        cdef _ndarray_base in_array, out_array
+        cdef Py_ssize_t block_size, block_stride, contiguous_size
+
+        cdef list block_strides = []
+
+        if len(self._reduction_in_array) == 0:
+            return [], 256, 0
+
+        block_size = _default_block_size
+        for i in range(len(self._reduction_in_array)):
+            in_array = ndarray_list[self._reduction_in_array[i]]
+            out_array = ndarray_list[self._reduction_out_array[i]]
+
+            # TODO(asi1024): Fix block strides for performance.
+            contiguous_size = 1
+            itemsize = in_array.dtype.itemsize
+            for i in range(out_array.ndim):
+                if in_array.strides[-i-1] != contiguous_size * itemsize:
+                    break
+                contiguous_size *= in_array.shape[-i-1]
+            contiguous_size = min(contiguous_size, 32)
+
+            reduce_block_size = max(1, in_array.size // max(1, out_array.size))
+            block_stride = max(
+                contiguous_size, block_size // reduce_block_size)
+            block_stride = internal.clp2(block_stride // 2 + 1)  # floor
+            block_strides.append(block_stride)
+
+        shared_mem = block_size * 32  # max bytesize of reduce_ctype.
+        return block_strides, block_size, shared_mem
+
+    cdef tuple _reduce_dims(self, list ndarray_list):
+        """Reduce number of dimensions of ndarrays and returns the cache key.
+        """
+        cdef list params = self._params
+        cdef list ndims = []
+        cdef _ndarray_base array
+        cdef int i
+
+        for i in range(len(params)):
+            param = params[i]
+            if param.ndim <= 1:
+                continue
+            array = ndarray_list[i]
+            array = array.reduced_view()
+            ndarray_list[i] = array
+            ndims.append(array.ndim)
+
+        return tuple(ndims)
+
+    cdef list _get_inout_args(self, tuple args, list ndarray_list):
+        """Get the arguments passed to ``kern.linear_launch``.
+        """
+        cdef list params = []
+        cdef list indexers = []
+        cdef _carray.Indexer indexer
+
+        for i in range(len(self._params)):
+            array = ndarray_list[i]
+            if isinstance(array, _ndarray_base):
+                indexer = _carray.Indexer.__new__(_carray.Indexer)
+                indexer.init(array._shape)
+                indexers.append(indexer)
+                params.append(array)
+            elif self._input_index[i] >= 0:
+                obj = args[<Py_ssize_t>self._input_index[i]]
+                params.append(obj)
+
+        return params + indexers
+
+    cdef str _get_cuda_params(self, tuple key, list ndarray_list):
+        """Get a string of parameters of CUDA main function code.
+        """
+        cdef int i
+
+        if key in self._cuda_params_memo:
+            return self._cuda_params_memo[key]
+
+        cuda_params = []
+        indexers = []
+
+        for i in range(len(self._params)):
+            a = self._params[i]
+            if isinstance(a, _TraceArray):
+                array = ndarray_list[i]
+                ndim = array.ndim
+                c_contiguous = 'true' if array._c_contiguous else 'false'
+                index_32_bits = 'true' if array._index_32_bits else 'false'
+                cuda_params.append(a.format(
+                    'CArray<${type}, ${ndim}, ${cont}, ${ind32}> ${var}',
+                    ndim=ndim, cont=c_contiguous, ind32=index_32_bits))
+                indexers.append(
+                    a.format('CIndexer<${ndim}> ${indexer}', ndim=ndim))
+            elif isinstance(a, _TraceScalar):
+                if a.const_value is None:
+                    cuda_params.append(a.format('${type} ${var}'))
+            else:
+                raise TypeError('Unknown type {}.'.format(type(a)))
+
+        ret = cuda_params + indexers + self._block_strides
+        ret = ', '.join(ret)
+        self._cuda_params_memo[key] = ret
+        return ret
+
+    def execute(self, tuple args, list shapes):
+        ndarray_list = self._get_ndarray_list(args, shapes)
+        ret = self._get_return_value(ndarray_list)
+        reduce_key = self._reduce_dims(ndarray_list)
+        inout_args = self._get_inout_args(args, ndarray_list)
+        cuda_params = self._get_cuda_params(reduce_key, ndarray_list)
+        kern = _cuda_compile(
+            self._submodule_code, self._name, cuda_params, self._cuda_body,
+            self._use_grid_sync)
+
+        block_strides, block_size, shared_mem = (
+            self._get_kernel_size(ndarray_list))
+
+        # TODO(asi1024): Optimize kernel size parameter.
+        if not runtime._is_hip_environment:
+            kern_size = driver.occupancyMaxActiveBlocksPerMultiprocessor(
+                kern.ptr, block_size, shared_mem) * block_size
+        else:
+            # In HIP sometimes the occupancy calc seems to be broken
+            kern_size = block_size * 512
+
+        kargs = inout_args + block_strides
+        kern.linear_launch(
+            kern_size, kargs, shared_mem, block_size,
+            enable_cooperative_groups=self._use_grid_sync)
+        return ret
diff --git a/cupy/_core/_fusion_op.py b/cupy/_core/_fusion_op.py
new file mode 100644
index 0000000..e5a1bad
--- /dev/null
+++ b/cupy/_core/_fusion_op.py
@@ -0,0 +1,316 @@
+import string
+
+import numpy
+
+from cupy._core import _codeblock
+from cupy._core._fusion_variable import _TraceVariable
+from cupy._core._fusion_variable import _TraceArray
+from cupy._core._fusion_variable import _VariableSet
+from cupy._core import _fusion_thread_local
+from cupy._core import _kernel
+from cupy._core import _reduction
+from cupy._core._scalar import get_typename
+
+
+class _UfuncRoutine:
+    """A device function for single elementwise operations.
+    """
+
+    def __init__(
+            self, name, ufunc, routine_code, in_params, out_params,
+            compute_dtypes):
+        assert isinstance(name, str)
+        assert isinstance(ufunc, _kernel.ufunc)
+        assert isinstance(routine_code, str)
+        assert isinstance(compute_dtypes, tuple)
+        assert all(isinstance(t, numpy.dtype) for t in compute_dtypes)
+        assert isinstance(in_params, list)
+        assert all(isinstance(p, _TraceVariable) for p in in_params)
+        assert isinstance(out_params, list)
+        assert all(isinstance(p, _TraceArray) for p in out_params)
+
+        self.name = name
+        self.in_params = in_params
+        self.out_params = out_params
+        self.preamble = ufunc._preamble
+        self.routine_code = routine_code
+        self.compute_dtypes = compute_dtypes
+
+    def emit_code(self):
+        """Returns a CUDA device function code.
+
+        Returns a string like:
+        ```
+        __device__ void cupy_add_0(int &in0_, float &in1_, double &out0_) {
+            typedef double in0_type;
+            typedef double in1_type;
+            typedef double out0_type;
+            double in0 = (double) in0_;
+            double in1 = (double) in1_;
+            double out0 = (double) out0_;
+            out0 = in0 + in1;
+            out0_ = out0;
+        }
+        ```
+        """
+        nin = len(self.in_params)
+        dtypes = self.compute_dtypes
+        assert len(self.in_params) == len(self.compute_dtypes[:nin])
+        in_params = [
+            (get_typename(p.dtype), get_typename(t), 'in{}'.format(i))
+            for i, (p, t) in enumerate(zip(self.in_params, dtypes[:nin]))
+        ]
+        out_params = [
+            (get_typename(p.dtype), get_typename(t), 'out{}'.format(i))
+            for i, (p, t) in enumerate(zip(self.out_params, dtypes[nin:]))
+        ]
+        params = in_params + out_params
+
+        params_code = ', '.join(['{} &{}_'.format(t, s) for t, _, s in params])
+        typedef = ['typedef {} {}_type;'.format(t, s) for _, t, s in params]
+        read = ['{} {} = ({}) {}_;'.format(t, s, t, s) for _, t, s in params]
+        write = ['{}_ = {};'.format(s, s) for _, _, s in out_params]
+
+        return _codeblock.CodeBlock(
+            '__device__ void {}({})'.format(self.name, params_code),
+            typedef + read + [self.routine_code + ';'] + write)
+
+    def emit_call_code(self):
+        params = self.in_params + self.out_params
+        return '{op_name}({params});'.format(
+            op_name=self.name,
+            params=', '.join([var.lvar_name for var in params]))
+
+
+class _ElementwiseTraceOp:
+    """Ufunc or elementwise kernel with types.
+    """
+
+    def __init__(self, ufunc_routines, in_params, out_params, ashape):
+        # The `in_params` and `out_params` should be already broadcasted to
+        # `ashape`, but they don't guarantee to be exactly same as
+        # `param.ashape`.
+
+        _fusion_thread_local.check_not_runtime()
+        assert isinstance(ufunc_routines, list)
+        assert all(isinstance(r, _UfuncRoutine) for r in ufunc_routines)
+        assert isinstance(ashape, tuple)
+
+        self.ops = ufunc_routines
+        self.in_params = _VariableSet(*in_params)
+        self.out_params = _VariableSet(*out_params)
+        self.ashape = ashape
+
+    @property
+    def params(self):
+        """Returns the set of all variable the loop uses.
+        """
+        res = _VariableSet()
+        for op in self.ops:
+            res += _VariableSet(*op.in_params)
+            res += _VariableSet(*op.out_params)
+        return res
+
+    @staticmethod
+    def _emit_declaration(params, in_params):
+        """Returns a tuple of size 2.
+
+        1. CUDA code: declaring local variables.
+            2. The set of arrays which require indexer.
+        """
+        _fusion_thread_local.check_not_runtime()
+
+        indexed_arrays = _VariableSet()
+        code = []
+        for var in params:
+            if var in in_params:
+                if isinstance(var, _TraceArray):
+                    indexed_arrays.add(var)
+                    f = '${type} ${lvar} = ${var}[${indexer}.get()];'
+                else:
+                    f = '${type} ${lvar} = ${var};'
+            else:
+                f = '${type} ${lvar};'
+            code.append(var.format(f))
+
+        return code, indexed_arrays
+
+    @staticmethod
+    def _emit_after_operation(out_params):
+        """Returns a tuple of size 2.
+        1. CUDA code: writing the results of operations back to global memory.
+        2. The set of arrays which require indexer.
+        """
+
+        _fusion_thread_local.check_not_runtime()
+
+        indexed_arrays = _VariableSet()
+        codes = []
+        for var in out_params:
+            if isinstance(var, _TraceArray):
+                indexed_arrays.add(var)
+                f = '${var}[${indexer}.get()] = ${lvar};'
+            else:
+                f = '${var} = ${lvar};'
+            codes.append(var.format(f))
+
+        return codes, indexed_arrays
+
+    @staticmethod
+    def _emit_set_index(indexed_params, tid):
+        """Returns a CUDA code: setting a raw index to indexers.
+        """
+        _fusion_thread_local.check_not_runtime()
+        assert isinstance(indexed_params, _VariableSet)
+
+        return [
+            p.format('${indexer}.set(${tid});', tid=tid)
+            for p in indexed_params
+        ]
+
+    def emit_code(self):
+        _fusion_thread_local.check_not_runtime()
+
+        declaration, s1 = self._emit_declaration(self.params, self.in_params)
+        operation = [op.emit_call_code() for op in self.ops]
+        after_operation, s2 = self._emit_after_operation(self.out_params)
+        index_name = 'i'
+        indexed_array = s1 + s2
+        indexer_name = next(iter(indexed_array)).indexer_name
+        indexer_setup = self._emit_set_index(indexed_array, index_name)
+
+        return _codeblock.CodeBlock(
+            'CUPY_FOR({}, {}.size())'.format(index_name, indexer_name),
+            indexer_setup + declaration + operation + after_operation)
+
+    def emit_preamble_codes(self):
+        return [subm.preamble for subm in self.ops if subm.preamble != '']
+
+    def emit_submodule_codes(self):
+        return [str(subm.emit_code()) for subm in self.ops]
+
+
+class _ReductionTraceOp:
+    def __init__(self, name, reduce_func, expr, in_param, out_param, axis):
+        """Reduction operation.
+        """
+        _fusion_thread_local.check_not_runtime()
+        assert isinstance(name, str)
+        assert isinstance(reduce_func, _reduction._SimpleReductionKernel)
+        assert isinstance(in_param, _TraceArray)
+        assert isinstance(out_param, _TraceArray)
+        assert isinstance(axis, tuple)
+        assert all(0 <= x < in_param.ndim for x in axis)
+
+        self.name = name
+        self.preamble = reduce_func.preamble
+        self.in_params = _VariableSet(in_param)
+        self.out_params = _VariableSet(out_param)
+        self.block_stride_name = 'block_stride_' + name
+        self.axis = axis
+
+        if reduce_func.identity is None:
+            self.identity = ''
+        else:
+            self.identity = str(reduce_func.identity)
+
+        _, self.expr, self.postmap_cast_code, self.reduce_ctype = expr
+        if self.reduce_ctype is None:
+            out_param, = self.out_params
+            self.reduce_ctype = get_typename(out_param.dtype)
+
+        self.premap_op = None
+        self.postmap_op = None
+
+    @property
+    def params(self):
+        return self.in_params + self.out_params
+
+    def emit_code(self):
+        _fusion_thread_local.check_not_runtime()
+        assert len(self.in_params) == 1
+        assert len(self.out_params) == 1
+        in_param = list(self.in_params)[0]
+        out_param = list(self.out_params)[0]
+        params = ', '.join([
+            in_param.var_name,
+            out_param.var_name,
+            in_param.indexer_name,
+            out_param.indexer_name,
+        ])
+        return '{}({}, {});'.format(
+            self.name, params, self.block_stride_name)
+
+    def emit_preamble_codes(self):
+        preamble = self.preamble
+        return [preamble] if preamble != '' else []
+
+    def emit_submodule_codes(self):
+        """Returns a CUDA device function code.
+
+        The emitted code assumes that ``block_stride`` and `blockDim.x` is a
+        power of 2.
+        """
+
+        in_param, = self.in_params
+        out_param, = self.out_params
+        op_name = '{}_op'.format(self.name)
+        postmap_name = '{}_postmap'.format(self.name)
+
+        template = string.Template('''
+#define ${op_name}(a, b) (${reduce_expr})
+#define ${postmap_name}(a, out0) (${postmap_cast})
+
+template <typename InType, typename OutType, typename InIndexerType, typename OutIndexerType>
+__device__ void ${name}(
+        InType in_arr, OutType out_arr,
+        InIndexerType in_ind, OutIndexerType out_ind, int block_stride) {
+    typedef ${in_type} type_in0_raw;
+    typedef ${out_type} type_out0_raw;
+    typedef ${reduce_ctype} _type_reduce;
+    extern __shared__ char _sdata_raw[];
+    _type_reduce *sdata = reinterpret_cast<_type_reduce*>(_sdata_raw);
+    unsigned int tid = threadIdx.x;
+    int _J = tid >> __popc(block_stride - 1);
+    ptrdiff_t _j = (ptrdiff_t)_J * out_ind.size();
+    int J_stride = blockDim.x >> __popc(block_stride - 1);
+    ptrdiff_t j_stride = (ptrdiff_t)J_stride * out_ind.size();
+
+    for (ptrdiff_t _i = (ptrdiff_t)blockIdx.x * block_stride; _i < out_ind.size(); _i += (ptrdiff_t)gridDim.x * block_stride) {
+        _type_reduce s = _type_reduce(${identity});
+        ptrdiff_t i = _i + (tid & (block_stride - 1));
+        for (ptrdiff_t j = i + _j; j < in_ind.size(); j += j_stride) {
+            in_ind.set(j);
+            s = ${op_name}(s, static_cast<_type_reduce>(in_arr[in_ind.get()]));
+        }
+        sdata[tid] = s;
+        __syncthreads();
+        for (unsigned int block = blockDim.x / 2; block >= block_stride; block >>= 1) {
+            if (tid < block) {
+                sdata[tid] = ${op_name}(sdata[tid], sdata[tid + block]);
+            }
+            __syncthreads();
+        }
+        if (tid < block_stride) {
+            s = sdata[tid];
+        }
+        if (tid < block_stride && i < out_ind.size()) {
+            out_ind.set(i);
+            ${postmap_name}(s, out_arr[out_ind.get()]);
+        }
+        __syncthreads();
+    }
+}''')  # NOQA
+        code = template.substitute(
+            name=self.name,
+            op_name=op_name,
+            postmap_name=postmap_name,
+            in_type=get_typename(in_param.dtype),
+            out_type=get_typename(out_param.dtype),
+            reduce_ctype=self.reduce_ctype,
+            reduce_expr=self.expr,
+            identity=self.identity,
+            postmap_cast=self.postmap_cast_code
+        )
+
+        return [code]
diff --git a/cupy/_core/_fusion_optimization.py b/cupy/_core/_fusion_optimization.py
new file mode 100644
index 0000000..d7f2485
--- /dev/null
+++ b/cupy/_core/_fusion_optimization.py
@@ -0,0 +1,90 @@
+from cupy._core import _fusion_variable
+from cupy._core import _fusion_op
+
+
+def _reduce_memory_access(ops):
+    required_memories = set()
+
+    for op in ops:
+        for p in op.in_params + op.out_params:
+            if p.memory.is_inout:
+                required_memories.add(p.memory)
+
+    for op in ops[::-1]:
+        in_memories = set([p.memory for p in op.in_params])
+
+        new_out_params = []
+        for p in op.out_params:
+            if p.memory in required_memories:
+                new_out_params.append(p)
+        op.out_params = _fusion_variable._VariableSet(*new_out_params)
+
+        # TODO(asi1024): The following improvement can be applicable only
+        # when the memory space is used at most once.
+        # `required_memories -= out_memories`
+        required_memories |= in_memories
+
+    return [op for op in ops if len(op.out_params) > 0]
+
+
+def _normalize_ashapes(ops, variables, shape_constraints):
+    def normalize(shape):
+        return tuple([shape_constraints.evaluate(d) for d in shape])
+
+    for var in variables:
+        var.ashape = normalize(var.ashape)
+
+    for op in ops:
+        if isinstance(op, _fusion_op._ElementwiseTraceOp):
+            op.ashape = normalize(op.ashape)
+
+
+def _fuse_two_ops(op1, op2):
+    """Returns a fused Op if the two ops can be fused, and ``None`` otherwise.
+    """
+    # TODO(asi1024): Supoort reduction postmap.
+    if not isinstance(op1, _fusion_op._ElementwiseTraceOp):
+        return None
+
+    # TODO(asi1024): Supoort reduction premap.
+    if not isinstance(op2, _fusion_op._ElementwiseTraceOp):
+        return None
+
+    if op1.ashape != op2.ashape:
+        return None
+
+    new_in_params = op1.in_params + (op2.in_params - op1.out_params)
+    new_out_params = op1.out_params + op2.out_params
+    for in_param in new_in_params:
+        for out_param in new_out_params:
+            # Checks if two arrays may share the same memory space.
+            if in_param.memory == out_param.memory and in_param != out_param:
+                return None
+
+    op1.ops.extend(op2.ops)
+    op1.in_params = new_in_params
+    op1.out_params = new_out_params
+    return op1
+
+
+def _fuse_consecutive_ops(ops, shape_constraints):
+    res = []
+    for op in ops:
+        if len(res) == 0:
+            res.append(op)
+        else:
+            prev_op = res.pop(-1)
+            new_op = _fuse_two_ops(prev_op, op)
+            if new_op is None:
+                res.extend([prev_op, op])
+            else:
+                res.append(new_op)
+    return res
+
+
+def optimize(ops, variables, shape_constraints):
+    _normalize_ashapes(ops, variables, shape_constraints)
+    ops = _reduce_memory_access(ops)
+    ops = _fuse_consecutive_ops(ops, shape_constraints)
+    ops = _reduce_memory_access(ops)
+    return ops
diff --git a/cupy/_core/_fusion_thread_local.pyx b/cupy/_core/_fusion_thread_local.pyx
new file mode 100644
index 0000000..d3b6c0d
--- /dev/null
+++ b/cupy/_core/_fusion_thread_local.pyx
@@ -0,0 +1,46 @@
+import threading
+
+
+thread_local = threading.local()
+
+
+cpdef inline bint is_old_fusing() except? -1:
+    try:
+        return thread_local.is_old_fusing
+    except AttributeError:
+        thread_local.is_old_fusing = False
+    return False
+
+
+cpdef inline bint is_new_fusing() except? -1:
+    try:
+        return thread_local.is_new_fusing
+    except AttributeError:
+        thread_local.is_new_fusing = False
+    return False
+
+
+cpdef inline bint is_fusing() except? -1:
+    return is_old_fusing() or is_new_fusing()
+
+
+def check_not_runtime():
+    assert is_new_fusing()
+
+
+def call_ufunc(fusion_op, *args, **kwargs):
+    if is_new_fusing():
+        return thread_local.history.call_ufunc(fusion_op, *args, **kwargs)
+    import cupy
+    return cupy._core.fusion._call_ufunc(fusion_op, *args, **kwargs)
+
+
+def call_reduction(fusion_op, *args, **kwargs):
+    if is_new_fusing():
+        return thread_local.history.call_reduction(fusion_op, *args, **kwargs)
+    import cupy
+    return cupy._core.fusion._call_reduction(fusion_op, *args, **kwargs)
+
+
+def call_indexing(fusion_op, *args, **kwargs):
+    return thread_local.history.call_indexing(fusion_op, *args, **kwargs)
diff --git a/cupy/_core/_fusion_trace.pyx b/cupy/_core/_fusion_trace.pyx
new file mode 100644
index 0000000..ff0775b
--- /dev/null
+++ b/cupy/_core/_fusion_trace.pyx
@@ -0,0 +1,616 @@
+import numpy
+
+from cupy._core import _kernel
+from cupy._core import _reduction
+from cupy._core import core
+from cupy._core._fusion_interface import _VariableProxy
+from cupy._core._fusion_interface import _ArrayProxy
+from cupy._core import _fusion_thread_local
+from cupy._core import _fusion_variable
+from cupy._core._fusion_variable import _AbstractDim
+from cupy._core._fusion_variable import _TraceScalar
+from cupy._core._fusion_variable import _TraceArray
+from cupy._core._fusion_variable import _VariableSet
+from cupy._core import _fusion_op
+from cupy._core import _fusion_optimization
+
+from cupy._core cimport internal
+from cupy._core._dtype cimport _raise_if_invalid_cast
+
+
+_thread_local = _fusion_thread_local.thread_local
+_accepted_types = (int, float, bool, complex, numpy.generic)
+
+
+cdef class _ShapeConstraints:
+    """A data structure that manages the conditions between the shapes.
+    """
+
+    cdef:
+        # A list of tuple of _AbstractDim and _AbstractDim which represents
+        # the equality between dimensions.
+        readonly list eq_constraints
+        # A list of tuple of _AbstractDim and int which is an associative list
+        readonly list const_constraints
+
+    def __init__(self):
+        self.eq_constraints = []
+        self.const_constraints = []
+
+    def add_eq_constraint(self, x, y):
+        """Add a constraint: x == y.
+        """
+        _fusion_thread_local.check_not_runtime()
+        assert isinstance(x, (_AbstractDim, int))
+        assert isinstance(y, (_AbstractDim, int))
+        x = self.evaluate(x)
+        y = self.evaluate(y)
+        if x == y:
+            return
+        if isinstance(x, _AbstractDim) and isinstance(y, _AbstractDim):
+            self.eq_constraints.append((x, y))
+        elif isinstance(x, _AbstractDim) and not isinstance(y, _AbstractDim):
+            self.add_const_constraint(x, y)
+        elif not isinstance(x, _AbstractDim) and isinstance(y, _AbstractDim):
+            self.add_const_constraint(y, x)
+        else:
+            assert False
+
+    def add_const_constraint(self, x, value):
+        """Add a constraint: x == value.
+        """
+        _fusion_thread_local.check_not_runtime()
+        assert isinstance(x, (_AbstractDim, int))
+        assert isinstance(value, int)
+        x = self.evaluate(x)
+        if isinstance(x, _AbstractDim):
+            self.const_constraints.append((x, value))
+        else:
+            assert x == value
+
+    def evaluate(self, x):
+        """Substitute repeatedly from the equalities.
+        """
+        _fusion_thread_local.check_not_runtime()
+        assert isinstance(x, (_AbstractDim, int))
+        for src, dest in self.eq_constraints + self.const_constraints:
+            if isinstance(x, int):
+                return x
+            if x == src:
+                x = dest
+        return x
+
+    # Used in runtime.
+    def satisfy(self, dict dim_map):
+        """Check if the given dicionary satisfies the constraints.
+
+        Args:
+            dim_map (dict):
+                A dictionary with keys of _AbstractDim type and
+                values of int type.
+        """
+        for a, b in self.eq_constraints:
+            if dim_map[a] != dim_map[b]:
+                return False
+        for a, b in self.const_constraints:
+            if dim_map[a] != b:
+                return False
+        return True
+
+
+def _guess_routine(func, args, dtype):
+    assert isinstance(func, (_kernel.ufunc, _reduction._SimpleReductionKernel))
+
+    # Feeds dummy arguments with appropriate dtypes passed to `guess_routine`.
+    dummy_args = []
+    for x in args:
+        if isinstance(x, _TraceScalar):
+            obj = x.dtype.type(0)
+        else:
+            assert isinstance(x, _TraceArray)
+            obj = core.ndarray((0,), x.dtype)
+        dummy_args.append(obj)
+
+    op = func._ops.guess_routine(
+        func.name, func._routine_cache, dummy_args, dtype, None)
+    return op.get_in_dtypes(), op.get_out_dtypes(), op.routine
+
+
+def _base(array):
+    """Returns the base array object of given array.
+    """
+    assert isinstance(array, core.ndarray)
+    return array if array.base is None else array.base
+
+
+class _VariableCoordinator:
+    """Variable constuct manager.
+
+    This class calls ``_TraceArray`` or ``_TraceScalar`` internally
+    with unique serial numbers and returns the variable object. In
+    ``TraceImpl`` class, a method of ``history.vc``, which is of
+    ``_VariableConduductor`` class, should be called instead of
+    ```_TraceArray.__init__`` or ``_TraceScalar.__init__``.
+    """
+
+    def __init__(self):
+        self._memory_number = 0
+        self._serial_number = 0
+        self._variables_dict = {}
+
+    def _normalize_variable(self, var):
+        """If the input variable is already generated previously, returns it.
+        """
+        key = var.key()
+        if key not in self._variables_dict:
+            self._variables_dict[key] = var
+        return self._variables_dict[key]
+
+    def _generate_new_variable(self, var_module, dtype, **kwargs):
+        serial_number = self._serial_number
+        memory = _fusion_variable._MemorySpace(
+            self._memory_number, serial_number)
+        self._serial_number += 1
+        self._memory_number += 1
+
+        ret = var_module(memory, serial_number, dtype, **kwargs)
+        memory.is_input = ret.is_input
+        return self._normalize_variable(ret)
+
+    def generate_new_array(self, dtype, rshape, ashape, input_index=None):
+        """Generate new _TraceArray object with a new memory space.
+        """
+        ret = self._generate_new_variable(
+            _TraceArray,
+            dtype, rshape=rshape, ashape=ashape, input_index=input_index)
+        ret.memory.base_ashape = ret.ashape
+        return ret
+
+    def generate_new_scalar(self, dtype, **kwargs):
+        """Generate new _TraceScalar object with a new memory space.
+        """
+        return self._generate_new_variable(_TraceScalar, dtype, **kwargs)
+
+    def make_view(self, var, **kwargs):
+        assert isinstance(var, _TraceArray)
+        serial_number = self._serial_number
+        self._serial_number += 1
+        ret = var.make_view(serial_number, **kwargs)
+        return self._normalize_variable(ret)
+
+    def broadcast_to(self, var, ashape, rshape):
+        """Make a view of the input array with the given shape.
+        """
+        return self.make_view(
+            var, ashape=ashape, rshape=rshape, broadcasted_from=var)
+
+    def rotate_with_axis(self, var, axis):
+        """Make a view of an array by rotating ``var`` with given axis.
+        """
+        assert isinstance(var, _TraceArray)
+        return self.make_view(var, rotated_from=var, axis=axis)
+
+    def indexing(self, var, indices):
+        """Make a view of an array. by indexing ``var`` with given tuple.
+        """
+        skip = var.ndim - sum([isinstance(x, (int, slice)) for x in indices])
+        it = 0
+        ashape = []
+        rshape = []
+
+        if skip < 0:
+            raise IndexError('Too many indices for array.')
+
+        for index in indices:
+            if isinstance(index, int):
+                it += 1
+            elif isinstance(index, slice):
+                if not (index.start is None
+                        and index.stop is None
+                        and index.step in (1, -1, None)):
+                    raise NotImplementedError(
+                        'Only full range ``x[::]`` or reverse ``x[::-1]`` is '
+                        'supported for basic slicing in CuPy fusion.')
+                ashape.append(var.ashape[it])
+                rshape.append(var.rshape[it])
+                it += 1
+            elif index is None:
+                ashape.append(1)
+                rshape.append(1)
+            elif index is Ellipsis:
+                ashape.extend(var.ashape[it:it + skip])
+                rshape.extend(var.rshape[it:it + skip])
+                it += skip
+
+        ashape.extend(var.ashape[it:var.ndim])
+        rshape.extend(var.rshape[it:var.ndim])
+
+        return self.make_view(
+            var, indexed_from=var, index_key=indices,
+            ashape=tuple(ashape), rshape=tuple(rshape))
+
+    @property
+    def all_variables(self):
+        """Returns the list of all variables this class emitted.
+        """
+        return list(self._variables_dict.values())
+
+
+class TraceImpl:
+    """Emit a fused kernel from the given target function.
+    """
+
+    def __init__(self):
+        self.vc = _VariableCoordinator()
+        self.shape_constraints = _ShapeConstraints()
+        self.op_list = []
+
+    @staticmethod
+    def _make_interface(x):
+        """Returns an _array or a _scalar object which packs the given value.
+        """
+        if x is None:
+            return None
+        assert isinstance(x, _fusion_variable._TraceVariable)
+        return x.as_interface()
+
+    def _unwrap_interface(self, x, *, allow_none=False):
+        """Returns ``_TraceVariable`` object from the input.
+        """
+        if allow_none and x is None:
+            return None
+        if isinstance(x, _VariableProxy):
+            return x.content
+        if isinstance(x, _accepted_types):
+            dtype = numpy.dtype(type(x))
+            return self.vc.generate_new_scalar(dtype, const_value=x)
+        if isinstance(x, (numpy.ndarray, core.ndarray)):
+            raise TypeError('Concrete ndarray is not supported in fusion.')
+        raise TypeError('{} type is not supported'.format(type(x)))
+
+    def call_ufunc(self, ufunc, *args, **kwargs):
+        """Register an elementwise operation with the given parameters.
+
+        Args:
+            ufunc(_kernel.ufunc): The ufunc to operate.
+            args(tuple): The arguments.
+            kwargs(dict): The keyword arguments.
+        """
+
+        assert isinstance(ufunc, _kernel.ufunc)
+
+        # Parse Inputs.
+        nin = ufunc.nin
+        nout = ufunc.nout
+        dtype = kwargs.pop('dtype', None)
+
+        if 'out' in kwargs and len(args) > nin:
+            raise ValueError(
+                'cannot specify \'out\' as both a positional and '
+                'keyword argument')
+
+        in_params = [self._unwrap_interface(x) for x in args[:nin]]
+        out_params = [
+            self._unwrap_interface(x, allow_none=True)
+            for x in args[nin:] + (kwargs.pop('out', None),)
+            if x is not None
+        ]
+        params = in_params + out_params
+
+        if len(kwargs) > 0:
+            raise TypeError('Wrong arguments {}'.format(kwargs))
+        if len(in_params) != nin or len(out_params) > nout:
+            raise ValueError('Invalid number of arguments')
+        if not all([isinstance(v, _TraceArray) for v in out_params]):
+            raise TypeError('Return arrays must be of ArrayType')
+
+        # Check for inplace operation.
+        for i, out_param1 in enumerate(out_params):
+            for out_param2 in out_params[:i]:
+                if out_param1.memory == out_param2.memory:
+                    # NumPy does not raise this error.
+                    raise ValueError('Outputs of ufunc must not share memory')
+
+        # Copy the input array data before the operation when the input array
+        # shares the same memory area with an output array.
+        for i, in_param in enumerate(in_params):
+            should_copy = any([
+                in_param.memory == out_param.memory and in_param != out_param
+                for out_param in out_params
+            ])
+            if should_copy:
+                in_params[i] = self._unwrap_interface(
+                    self.call_ufunc(
+                        core.elementwise_copy,
+                        self._make_interface(in_param)))
+
+        # Broadcast shapes
+        out_rshape = internal._broadcast_shapes([p.rshape for p in params])
+        out_ashape = [None for _ in range(len(out_rshape))]
+
+        for p in params:
+            for axis in range(-p.ndim, 0):
+                if p.rshape[axis] == out_rshape[axis]:
+                    out_ashape[axis] = p.ashape[axis]
+
+        assert all([dim is not None for dim in out_ashape])
+        out_ashape = tuple(out_ashape)
+
+        # Broadcast input params and make their views.
+        for i, p in enumerate(in_params):
+            for axis in range(-p.ndim, 0):
+                if p.rshape[axis] == out_rshape[axis]:
+                    self.shape_constraints.add_eq_constraint(
+                        p.ashape[axis], out_ashape[axis])
+                elif p.rshape[axis] == 1:
+                    self.shape_constraints.add_const_constraint(
+                        p.ashape[axis], 1)
+                else:
+                    assert False
+            if isinstance(p, _TraceArray) and p.rshape != out_rshape:
+                # Broadcst input if needed.
+                in_params[i] = self.vc.broadcast_to(p, out_ashape, out_rshape)
+
+        # Get operation code from dtypes.
+        in_dtypes, out_dtypes, expr = _guess_routine(
+            ufunc, in_params, dtype)
+
+        # Make output arrays.
+        ret = []
+        for i in range(nout):
+            if i >= len(out_params):
+                # Omitted output.
+                out_pvar = self.vc.generate_new_array(
+                    out_dtypes[i], out_rshape, out_ashape)
+                out_params.append(out_pvar)
+            elif isinstance(out_params, _TraceScalar):
+                raise TypeError('return arrays must be of ArrayType')
+            elif out_params[i].rshape != out_rshape:
+                raise ValueError(
+                    'non-broadcastable output operand with shape {} '
+                    'doesn\'t match the broadcast shape {}'.format(
+                        out_params[i].rshape, out_rshape))
+
+            _raise_if_invalid_cast(
+                out_dtypes[i], out_params[i].dtype, 'same_kind',
+                'output operand')
+
+            out_pvar = out_params[i]
+            ret.append(out_pvar)
+
+        # Register Op.
+        name = ufunc.name + '_' + str(len(self.op_list))
+        ufunc_routine = _fusion_op._UfuncRoutine(
+            name, ufunc, expr, in_params, out_params, in_dtypes + out_dtypes)
+        op = _fusion_op._ElementwiseTraceOp(
+            [ufunc_routine], in_params, out_params, out_ashape)
+        self.op_list.append(op)
+
+        # Returns.
+        assert len(ret) > 0
+        if len(ret) == 1:
+            return self._make_interface(ret[0])
+        else:
+            return tuple([self._make_interface(x) for x in ret])
+
+    def call_reduction(
+            self, reduce_func, a, axis=None, dtype=None, out=None,
+            keepdims=False):
+        """Register a reduction operation with the given parameters.
+
+        Args:
+            reduce_func(_reduction._SimpleReductionKernel):
+                The reduction function to operate.
+            a(array_like): The input array.
+            axis(int, tuple of int or None): The axis.
+            dtype(numpy.dtype or None): The dtype
+            out(_array or None): The output array.
+        """
+
+        assert isinstance(reduce_func, _reduction._SimpleReductionKernel)
+
+        # Parse inputs.
+        in_param = self._unwrap_interface(a)
+
+        if not isinstance(in_param, _TraceArray):
+            raise NotImplementedError(
+                'Reduction for scalar arguments is not supported.')
+
+        axes = internal._normalize_axis_indices(axis, in_param.ndim)
+
+        if dtype is not None:
+            dtype = numpy.dtype(dtype)
+
+        if keepdims:
+            raise NotImplementedError('keepdims is not supported.')
+
+        # Determine the shape of out_param.
+        out_ashape = tuple([
+            d for axis, d in enumerate(in_param.ashape) if axis not in axes])
+        out_rshape = tuple([
+            d for axis, d in enumerate(in_param.rshape) if axis not in axes])
+
+        # Rotate axes.
+        # This condition is only for performance improvement,
+        if not all([i == axis for i, axis in enumerate(axes)]):
+            in_param = self.vc.rotate_with_axis(in_param, axes)
+
+        # Get operation code from dtypes.
+        _, (out_dtype,), expr = _guess_routine(reduce_func, [in_param], dtype)
+
+        # Make an output array.
+        if out is None:
+            # Omitted output.
+            out_param = self.vc.generate_new_array(
+                out_dtype, out_rshape, out_ashape)
+        else:
+            out_param = self._unwrap_interface(out)
+            if out_param.rshape != out_rshape:
+                raise ValueError(
+                    'Shape of specified output variable is not consistent '
+                    'with reduced shape.')
+
+        # Register Op.
+        name = 'reduce{}'.format(len(self.op_list))
+        op = _fusion_op._ReductionTraceOp(
+            name, reduce_func, expr, in_param, out_param, axes)
+        self.op_list.append(op)
+
+        # Returns.
+        return self._make_interface(out_param)
+
+    def call_indexing(self, in_param, indices):
+        """Call indexing routines.
+        """
+        in_param = self._unwrap_interface(in_param)
+
+        if not isinstance(indices, tuple):
+            indices = (indices,)
+
+        for x in indices:
+            if isinstance(indices, (list, _TraceArray)):
+                # Advanced indexing
+                raise NotImplementedError(
+                    'Advanced indexing is not supported, currently.')
+
+            if not (isinstance(x, (int, slice)) or x is None or x is Ellipsis):
+                raise IndexError(
+                    'Indices must be integers, slices, ellipsis, None or '
+                    'integer or boolean arrays.')
+
+        # Basic indexing
+        out_param = self.vc.indexing(in_param, indices)
+        return self._make_interface(out_param)
+
+    def trace(self, func, args):
+        """Call ``self.func`` with _TraceVariable arguments.
+
+        Returns:
+            out_params(list of _TraceVariable): The list of outputs.
+            return_size(int or str): If ``return_size`` is of int type,
+                it indicates the size of tuple of outputs.
+                If `none`, the output is ``None`` and ``out_params`` is empty.
+                If `single`, the output is single array and ``out_params``
+                is a singleton list.
+
+        During the function call, ``call_ufunc``, ``call_reduction`` and
+        ``call_indexing`` are called internally.
+        """
+
+        # Register input variables.
+        in_params = []
+        array_dict = {}
+        memory_dict = {}
+        for input_index, arg in enumerate(args):
+            if arg is None:
+                var = None
+            elif isinstance(arg, core.ndarray):
+                arg_id = id(arg)
+                base_id = id(_base(arg))
+                if arg_id in array_dict:
+                    # The array is already given as an input.
+                    var = in_params[array_dict[arg_id]]
+                    assert isinstance(var, _TraceArray)
+                elif base_id in memory_dict:
+                    # The is an array which shares the same memory.
+                    base = in_params[memory_dict[base_id]]
+                    assert isinstance(base, _TraceArray)
+                    var = self.vc.make_view(base, input_index=input_index)
+                else:
+                    # Otherwise.
+                    var = self.vc.generate_new_array(
+                        arg.dtype, arg.shape, None, input_index=input_index)
+                array_dict[arg_id] = input_index
+                memory_dict[base_id] = input_index
+            else:
+                # Scalar input.
+                dtype = numpy.dtype(type(arg))
+                var = self.vc.generate_new_scalar(
+                    dtype, input_index=input_index)
+            in_params.append(var)
+
+        # Call the target function.
+        inputs = [self._make_interface(x) for x in in_params]
+        output = func(*inputs)
+
+        # Register output variables.
+        if output is None:
+            return_size = 'none'
+            out_params = []
+        elif isinstance(output, _ArrayProxy):
+            return_size = 'single'
+            out_params = [self._unwrap_interface(output, allow_none=True)]
+        elif isinstance(output, tuple):
+            if all(isinstance(x, _ArrayProxy) for x in output):
+                return_size = len(output)
+                out_params = [
+                    self._unwrap_interface(x, allow_none=True) for x in output]
+            else:
+                raise ValueError(
+                    'The all elements of return value of fused function '
+                    'must be of _ArrayProxy type.'
+                )
+        else:
+            raise ValueError(
+                'The return value of fused functions must be `None`, '
+                'ndarray or a tuple of ndarays.'
+            )
+
+        for output_index, out_param in enumerate(out_params):
+            assert isinstance(out_param, _TraceArray)
+            out_param.output_index = output_index
+            out_param.memory.is_output = True
+
+        return out_params, return_size
+
+
+def _get_ancestors_of_trace_variable(var):
+    if var is None:
+        return _VariableSet()
+    res = _VariableSet(var)
+    if isinstance(var, _TraceArray):
+        res += _get_ancestors_of_trace_variable(var._view_of)
+    return res
+
+
+class _TraceResult:
+
+    def __init__(self, op_list, params, return_size, shape_constraints):
+        self.op_list = op_list
+        self.params = params
+        self.return_size = return_size
+        self.shape_constraints = shape_constraints
+
+
+def trace(func, args):
+    history = TraceImpl()
+
+    try:
+        _thread_local.history = history
+
+        # Call `func(args)` and update `op_list`.
+        out_params, return_size = history.trace(func, args)
+    finally:
+        _thread_local.history = None
+
+    op_list = history.op_list
+    shape_constraints = history.shape_constraints
+    all_variables = history.vc.all_variables
+
+    op_list = _fusion_optimization.optimize(
+        op_list, all_variables, shape_constraints)
+
+    # Make info passed to FusedKernel.
+    kernel_params = _VariableSet()
+    for p in out_params:
+        kernel_params += _get_ancestors_of_trace_variable(p)
+    for op in op_list:
+        for p in op.in_params + op.out_params:
+            kernel_params += _get_ancestors_of_trace_variable(p)
+    kernel_params = list(kernel_params)
+
+    # used in mock tests.
+    history.kernel_params = kernel_params
+    history.op_list = op_list
+
+    return _TraceResult(op_list, kernel_params, return_size, shape_constraints)
diff --git a/cupy/_core/_fusion_variable.pxd b/cupy/_core/_fusion_variable.pxd
new file mode 100644
index 0000000..7515ec7
--- /dev/null
+++ b/cupy/_core/_fusion_variable.pxd
@@ -0,0 +1,5 @@
+cdef class _AbstractDim:
+
+    cdef:
+        readonly int input_index
+        readonly int axis
diff --git a/cupy/_core/_fusion_variable.pyx b/cupy/_core/_fusion_variable.pyx
new file mode 100644
index 0000000..d76911a
--- /dev/null
+++ b/cupy/_core/_fusion_variable.pyx
@@ -0,0 +1,340 @@
+import string
+
+import numpy
+
+from cupy._core import _fusion_interface
+
+from cupy._core._scalar cimport get_typename
+
+
+cdef class _AbstractDim:
+    """An abstracted data structure for a length of dimensions.
+
+    Attributes:
+        input_index (int):
+            The position of the element in the arguments passed to the
+            fused function
+        axis (int):
+            The index of dimensions
+    """
+
+    def __init__(self, int input_index, int axis):
+        self.input_index = input_index
+        self.axis = axis
+
+    def __hash__(self):
+        return hash((self.input_index, self.axis))
+
+    def __eq__(self, object other):
+        if isinstance(other, int):
+            return False
+        return (
+            self.input_index == other.input_index
+            and self.axis == other.axis
+        )
+
+
+class _MemorySpace:
+    """A memory space object.
+
+    Attributes:
+        id(int): The serial number of memory space.
+        base_serial_number(int): The serial number of the base variable
+            which have this memory space.
+        is_input(bool): If this is set to ``True``, the memory space is
+            already allocated as an input array. If this is set to ``False``,
+            the memory space should be allocated before launching the kernel.
+        is_output(bool): If this is set to ``True``, the memory space is
+            used in the return values.
+    """
+    def __init__(self, memory_id, base_serial_number):
+        assert isinstance(memory_id, int)
+        assert isinstance(base_serial_number, int)
+
+        self.id = memory_id
+        self.base_serial_number = base_serial_number
+
+        # Initially, these attributes are set to be `False`, but might be
+        # updated from outside.
+        self.is_input = False
+        self.is_output = False
+
+    @property
+    def is_inout(self):
+        """Returns ``True`` if the memory space is used for inputs or outputs.
+
+        If ``True``, the memory space should not be deallocated just after
+        the kernel launch. If ``False``, the memory space is used only for
+        temporary value in the fused kernel."""
+        return self.is_input or self.is_output
+
+
+class _TraceVariable:
+    """Variable object to trace operations in the target function to be fused.
+
+    Attributes:
+        index(_MemorySpace): The memory space the variable uses.
+        serial_number(int): The serial number of the variable object.
+        dtype(dtype): The dtype of the variable.
+        rshape(tuple of int): The real shape of the variable.
+        ashape(tuple of _AbstractDim): An abstracted shape of the variable.
+        input_index(int or None): If not `None`, this variable is used as
+            the `input_index`-th input parameter.
+        output_index(int or None): If not `None`, this variable is used as
+            the `output_index`-th output parameter.
+    """
+    def __init__(
+            self, memory_space, serial_number, dtype, rshape, ashape,
+            input_index, output_index):
+        assert isinstance(memory_space, _MemorySpace)
+        assert isinstance(serial_number, int)
+        assert isinstance(dtype, numpy.dtype)
+        assert input_index is None or isinstance(input_index, int)
+        assert output_index is None or isinstance(output_index, int)
+        assert isinstance(rshape, tuple)
+        assert isinstance(ashape, tuple)
+        assert len(rshape) == len(ashape)
+        for rdim, adim in zip(rshape, ashape):
+            assert isinstance(rdim, int)
+            assert isinstance(adim, (int, _AbstractDim))
+
+        self.memory = memory_space
+        self.serial_number = serial_number
+        self.dtype = dtype
+        self.rshape = rshape
+        self.ashape = ashape
+        self.input_index = input_index
+        self.output_index = output_index
+
+    @property
+    def ndim(self):
+        return len(self.ashape)
+
+    @property
+    def is_base(self):
+        return self.serial_number == self.memory.base_serial_number
+
+    @property
+    def is_input(self):
+        return self.input_index is not None
+
+    @property
+    def is_output(self):
+        return self.output_index is not None
+
+    @property
+    def var_name(self):
+        # The name of varialbe stored in global memory space.
+        raise NotImplementedError
+
+    @property
+    def lvar_name(self):
+        # The name of varialbe stored in registers in each thread.
+        raise NotImplementedError
+
+    @property
+    def indexer_name(self):
+        """The name of CUDA CIndxer variable for the variable.
+        """
+        # TODO(asi1024): Unify indexer with other variables which have the
+        # same shape, for performance improvements.
+        return 'ind{}_{}'.format(self.memory.id, self.serial_number)
+
+    def format(self, form, **kwargs):
+        """Returns a string following the format taken as an input.
+        """
+        kwargs = dict([
+            (k, get_typename(v) if isinstance(v, numpy.dtype) else v)
+            for k, v in kwargs.items()]
+        )
+        return string.Template(form).substitute(
+            type=get_typename(self.dtype),
+            var=self.var_name,
+            lvar=self.lvar_name,
+            indexer=self.indexer_name,
+            **kwargs
+        )
+
+    def __hash__(self):
+        assert False, (
+            '__hash__ is not defined. Use _VariableSet instead of '
+            'set/dict because they do not guarantee the order of contents.')
+
+
+class _TraceScalar(_TraceVariable):
+    """An abstracted scalar object.
+
+    Attributes:
+        const_value(scalar object or None): A compile-time constant value.
+            Actually, it is `None` iff self.is_input is `True`.
+    """
+
+    # TODO(asi1024): Remove index argument.
+    def __init__(
+            self, index, serial_number, dtype, input_index=None, *,
+            const_value=None,):
+        super().__init__(
+            index, serial_number, dtype, (), (), input_index, None)
+
+        self.const_value = const_value
+
+    @property
+    def var_name(self):
+        if self.const_value is None:
+            return 'a{}'.format(self.memory.id)
+        if self.dtype == '?':
+            return str(self.const_value).lower()
+        if self.dtype.kind == 'c':
+            return '{}({}, {})'.format(
+                get_typename(self.dtype),
+                self.const_value.real,
+                self.const_value.imag)
+        return str(self.const_value)
+
+    @property
+    def lvar_name(self):
+        return 'v{}'.format(self.memory.id)
+
+    def as_interface(self):
+        return _fusion_interface._ScalarProxy(self)
+
+    def key(self):
+        return (self.memory.id,)
+
+
+class _TraceArray(_TraceVariable):
+    """An abstracted array object.
+
+    Attributes:
+        broadcasted_from(_TraceArray optional): TODO
+        rotated_from(_TraceArray optional): TODO
+        axis(int optional): The axis to rotate.
+        indexed_from(_TraceArray optional): TODO
+        index_key(slice): TODO
+    """
+
+    def __init__(
+            self, index, serial_number, dtype, input_index=None,
+            output_index=None, *, rshape, ashape, **kwargs):
+
+        if ashape is None:
+            assert input_index is not None
+            ndim = len(rshape)
+            ashape = tuple([
+                _AbstractDim(input_index, axis) for axis in range(ndim)])
+
+        super().__init__(
+            index, serial_number, dtype, rshape, ashape,
+            input_index, output_index)
+
+        self._view_of = None
+        self.is_broadcast = False
+        self.rotate_axis = None
+        self.slice_key = None
+
+        if 'broadcasted_from' in kwargs:
+            self._view_of = kwargs.pop('broadcasted_from')
+            self.is_broadcast = True
+        elif 'rotated_from' in kwargs:
+            self._view_of = kwargs.pop('rotated_from')
+            self.rotate_axis = kwargs.pop('axis')
+        elif 'indexed_from' in kwargs:
+            self._view_of = kwargs.pop('indexed_from')
+            self.slice_key = kwargs.pop('index_key')
+
+        assert len(kwargs) == 0, kwargs
+
+    @property
+    def var_name(self):
+        return 'a{}_{}'.format(self.memory.id, self.serial_number)
+
+    @property
+    def lvar_name(self):
+        return 'v{}_{}'.format(self.memory.id, self.serial_number)
+
+    def as_interface(self):
+        return _fusion_interface._ArrayProxy(self)
+
+    def make_view(self, serial_number, **kwargs):
+        rshape = kwargs.pop('rshape', self.rshape)
+        ashape = kwargs.pop('ashape', self.ashape)
+        return _TraceArray(
+            self.memory, serial_number, self.dtype,
+            rshape=rshape, ashape=ashape, **kwargs)
+
+    def key(self):
+        """Two variables can be identified if they have the same key.
+        """
+        if isinstance(self.slice_key, tuple):
+            slice_key = []
+            for s in self.slice_key:
+                if isinstance(s, slice):
+                    if not (s.start is None
+                            and s.stop is None
+                            and s.step in (None, 1, -1)):
+                        raise NotImplementedError(
+                            'Basic slice supports only x[::] and x[::-1].')
+                    slice_key.append((s.start, s.stop, s.step))
+                else:
+                    slice_key.append(s)
+            slice_key = tuple(slice_key)
+        else:
+            slice_key = self.slice_key
+
+        return (
+            self.memory.id, self.ashape, self.input_index,
+            getattr(self._view_of, 'serial_number', None),
+            self.is_broadcast, self.rotate_axis, slice_key,
+        )
+
+
+class _VariableSet:
+    """A stable set of variables
+    """
+
+    def __init__(self, *args):
+        self.contents = []
+        for x in args:
+            assert isinstance(x, _TraceVariable)
+            if x not in self.contents:
+                self.contents.append(x)
+
+    def __len__(self):
+        return len(self.contents)
+
+    def item(self):
+        assert len(self.contents) == 1
+        return self.contents[0]
+
+    def add(self, x):
+        if x not in self.contents:
+            self.contents.append(x)
+
+    def __iadd__(self, other):
+        assert isinstance(other, _VariableSet)
+        for x in other.contents:
+            self.add(x)
+        return self
+
+    def __add__(self, other):
+        res = _VariableSet(*self.contents)
+        res += other
+        return res
+
+    def __contains__(self, elem):
+        return elem in self.contents
+
+    def __iter__(self):
+        return iter(self.contents)
+
+    def __isub__(self, other):
+        assert isinstance(other, _VariableSet)
+        for x in other.contents:
+            if x in self.contents:
+                self.contents.remove(x)
+        return self
+
+    def __sub__(self, other):
+        res = _VariableSet(*self.contents)
+        res -= other
+        return res
diff --git a/cupy/_core/_gufuncs.py b/cupy/_core/_gufuncs.py
new file mode 100644
index 0000000..f1adcc6
--- /dev/null
+++ b/cupy/_core/_gufuncs.py
@@ -0,0 +1,729 @@
+import re
+
+import numpy
+
+import cupy
+import cupy._core._routines_manipulation as _manipulation
+from cupy._core._dtype import get_dtype, _raise_if_invalid_cast
+from cupy._core import internal
+
+
+# Signature parsing code and dimension accessing has been borrowed
+# from dask
+# https://github.com/dask/dask/blob/61b578f5a3ad88cbc6a8b9a73ce08c551bd969fa/dask/array/gufunc.py#L12-L55
+_DIMENSION_NAME = r'\w+\?*'
+_CORE_DIMENSION_LIST = '(?:{0:}(?:,{0:})*,?)?'.format(_DIMENSION_NAME)
+_ARGUMENT = r'\({}\)'.format(_CORE_DIMENSION_LIST)
+_INPUT_ARGUMENTS = '(?:{0:}(?:,{0:})*,?)?'.format(_ARGUMENT)
+_OUTPUT_ARGUMENTS = '{0:}(?:,{0:})*'.format(
+    _ARGUMENT
+)  # Use `'{0:}(?:,{0:})*,?'` if gufunc-
+# signature should be allowed for length 1 tuple returns
+_SIGNATURE = '^{0:}->{1:}$'.format(_INPUT_ARGUMENTS, _OUTPUT_ARGUMENTS)
+
+
+def _parse_gufunc_signature(signature):
+    # The code has been modifyed from dask to support optional dimensions
+    if not isinstance(signature, str):
+        raise TypeError('Signature is not a string')
+
+    if signature == '' or signature is None:
+        raise ValueError('Signature cannot be empty')
+
+    signature = signature.replace(' ', '')
+    if not re.match(_SIGNATURE, signature):
+        raise ValueError('Not a valid gufunc signature: {}'.format(signature))
+    in_txt, out_txt = signature.split('->')
+    ins = [tuple(x.split(',')) if x != '' else ()
+           for x in in_txt[1:-1].split('),(')]
+    outs = [tuple(y.split(',')) if y != '' else ()
+            for y in out_txt[1:-1].split('),(')]
+    # TODO(ecastill) multiple output support
+    if len(outs) > 1:
+        raise ValueError('Currently more than 1 output is not supported')
+    return ins, outs
+
+
+def _validate_normalize_axes(
+    axes, axis, keepdims, input_coredimss, output_coredimss
+):
+    # This code credit goes to Dask
+    # https://github.com/dask/dask/blob/61b578f5a3ad88cbc6a8b9a73ce08c551bd969fa/dask/array/gufunc.py#L58-L172
+    nin = len(input_coredimss)
+    nout = (
+        1 if not isinstance(output_coredimss, list) else len(output_coredimss)
+    )
+
+    if axes is not None and axis is not None:
+        raise ValueError(
+            'Only one of `axis` or `axes` keyword arguments should be given')
+    if axes and not isinstance(axes, list):
+        raise ValueError('`axes` has to be of type list')
+
+    # output_coredimss = output_coredimss if nout > 1 else [output_coredimss]
+    filtered_core_dims = list(filter(len, input_coredimss))
+    nr_outputs_with_coredims = len(
+        [True for x in output_coredimss if len(x) > 0])
+
+    if keepdims:
+        if nr_outputs_with_coredims > 0:
+            raise ValueError('`keepdims` can only be used for scalar outputs')
+        output_coredimss = len(output_coredimss) * [filtered_core_dims[0]]
+
+    core_dims = input_coredimss + output_coredimss
+    if axis is not None:
+        if not isinstance(axis, int):
+            raise ValueError('`axis` argument has to be an integer value')
+        if filtered_core_dims:
+            cd0 = filtered_core_dims[0]
+            if len(cd0) != 1:
+                raise ValueError(
+                    '`axis` can be used only, if one core dimension is present'
+                )
+            for cd in filtered_core_dims:
+                if cd0 != cd:
+                    raise ValueError(
+                        'To use `axis`, all core dimensions have to be equal'
+                    )
+
+    # Expand defaults or axis
+    if axes is None:
+        if axis is not None:
+            axes = [(axis,) if cd else tuple() for cd in core_dims]
+        else:
+            axes = [tuple(range(-len(icd), 0)) for icd in core_dims]
+
+    axes = [(a,) if isinstance(a, int) else a for a in axes]
+
+    if (
+        (nr_outputs_with_coredims == 0)
+        and (nin != len(axes))
+        and (nin + nout != len(axes))
+    ) or ((nr_outputs_with_coredims > 0) and (nin + nout != len(axes))):
+        raise ValueError(
+            'The number of `axes` entries is not equal the number'
+            ' of input and output arguments')
+
+    # Treat outputs
+    output_axes = axes[nin:]
+    output_axes = (
+        output_axes
+        if output_axes
+        else [tuple(range(-len(ocd), 0)) for ocd in output_coredimss]
+    )
+    input_axes = axes[:nin]
+
+    # Assert we have as many axes as output core dimensions
+    for idx, (iax, icd) in enumerate(zip(input_axes, input_coredimss)):
+        if len(iax) != len(icd):
+            raise ValueError(
+                f'The number of `axes` entries for argument #{idx}'
+                ' is not equal the number of respective input core'
+                ' dimensions in signature')
+    if not keepdims:
+        for idx, (oax, ocd) in enumerate(zip(output_axes, output_coredimss)):
+            if len(oax) != len(ocd):
+                raise ValueError(
+                    f'The number of `axes` entries for argument #{idx}'
+                    ' is not equal the number of respective output core'
+                    ' dimensions in signature')
+    else:
+        if input_coredimss:
+            icd0 = input_coredimss[0]
+            for icd in input_coredimss:
+                if icd0 != icd:
+                    raise ValueError(
+                        'To use `keepdims`, all core dimensions'
+                        ' have to be equal')
+            iax0 = input_axes[0]
+            output_axes = [iax0 for _ in output_coredimss]
+
+    return input_axes, output_axes
+
+
+class _OpsRegister:
+    '''
+    Holds the ops for each dtypes signature like ('ff->f', func1)
+    and allows to do look ups for these
+    '''
+    class _Op:
+        def __init__(self, in_types, out_types, func):
+            self.func = func
+            self.in_types = tuple(numpy.dtype(i) for i in in_types)
+            self.out_types = tuple(numpy.dtype(o) for o in out_types)
+            self.sig_str = (''.join(
+                in_t.char for in_t in self.in_types) + '->' + ''.join(
+                    out_t.char for out_t in self.out_types))
+
+    def __init__(self, signatures, default_func, nin, nout, name):
+        self._default_func = default_func
+        self._nin = nin
+        self._nout = nout
+        self._ops = self._process_signatures(signatures)
+        self._name = name
+
+    def _sig_str_to_tuple(self, sig):
+        sig = sig.replace(' ', '')
+        toks = sig.split('->')
+        if len(toks) != 2:
+            raise ValueError(f'signature {sig} for dtypes is invalid')
+        else:
+            ins, outs = toks
+        return ins, outs
+
+    def _process_signatures(self, signatures):
+        ops = []
+        for sig in signatures:
+            if isinstance(sig, tuple):
+                sig, op = sig
+            else:
+                op = self._default_func
+            ins, outs = self._sig_str_to_tuple(sig)
+            # Check the number of inputs and outputs matches the gufunc sig
+            if len(ins) != self._nin:
+                raise ValueError(
+                    f'signature {sig} for dtypes is invalid number of inputs '
+                    'is not consistent with general signature')
+            if len(outs) != self._nout:
+                raise ValueError(
+                    f'signature {sig} for dtypes is invalid number of inputs '
+                    'is not consistent with general signature')
+
+            ops.append(_OpsRegister._Op(ins, outs, op))
+        return ops
+
+    def _determine_from_args(self, args, casting):
+        n = len(args)
+        in_types = tuple(arg.dtype for arg in args)
+        for op in self._ops:
+            op_types = op.in_types
+            for i in range(n):
+                it = in_types[i]
+                ot = op_types[i]
+                if not numpy.can_cast(it, ot, casting=casting):
+                    break
+            else:
+                return op
+        return None
+
+    def _determine_from_dtype(self, dtype):
+        for op in self._ops:
+            op_types = op.out_types
+            for t in op_types:
+                if t != dtype:
+                    break
+            else:
+                return op
+        return None
+
+    def _determine_from_signature(self, signature):
+        # Lets convert the signature as it can be a tuple of tuples
+        # or a string
+        if isinstance(signature, tuple):
+            # create a string to do a look-up on the ops
+            if len(signature) == 1:
+                raise TypeError(
+                    'The use of a length 1 tuple for the ufunc `signature` is'
+                    ' not allowed. Use `dtype` or  fill the tuple with'
+                    ' `None`s.')
+            nin = self._nin
+            nout = self._nout
+            if len(signature) != (nin + nout):
+                raise TypeError(
+                    'A type-tuple must be specified of length 1 or 3 for ufunc'
+                    f' {self._name}')
+            signature = ''.join(
+                numpy.dtype(t).char for t in signature[:nin]) + '->' + ''.join(
+                    numpy.dtype(t).char for t in signature[nin:nin+nout])
+
+        if isinstance(signature, str):
+            is_out = len(signature) == 1
+            for op in self._ops:
+                if is_out:
+                    for t in op.out_types:
+                        if t.char != signature:
+                            break
+                    else:
+                        return op
+                else:
+                    if op.sig_str == signature:
+                        return op
+        raise TypeError('No loop matching the specified signature and'
+                        f' casting was found for ufunc {self._name}')
+
+    def determine_dtype(self, args, dtype, casting, signature):
+        ret_dtype = None
+        func = self._default_func
+        if signature is not None:
+            # TODO(ecastill) use an externally provided signature to
+            # find the typecasting rules
+            op = self._determine_from_signature(signature)
+        elif dtype is not None:
+            if type(dtype) == tuple:
+                # TODO(ecastill) support dtype tuples
+                raise RuntimeError('dtype with tuple is not yet supported')
+            op = self._determine_from_dtype(dtype)
+        else:
+            op = self._determine_from_args(args, casting)
+
+        if op is None:
+            # Should we allow op to be none?
+            if dtype is None:
+                dtype = args[0].dtype
+                for arg in args:
+                    ret_dtype = numpy.promote_types(dtype, arg.dtype)
+            else:
+                ret_dtype = get_dtype(dtype)
+        else:
+            # Convert args to the op specified in_types
+            n_args = []
+            def argname(): return f'ufunc {self._name} input {i}'
+            for i, (arg, in_type) in enumerate(zip(args, op.in_types)):
+                _raise_if_invalid_cast(arg.dtype, in_type, casting, argname)
+
+                n_args.append(arg.astype(in_type, copy=False))
+            args = n_args
+            ret_dtype = op.out_types[0]
+            func = op.func
+
+        return args, ret_dtype, func
+
+
+class _GUFunc:
+    '''
+    Creates a Generalized Universal Function by wrapping a user
+    provided function with the signature.
+
+    ``signature`` determines if the function consumes or produces core
+    dimensions. The remaining dimensions in given input arrays (``*args``)
+    are considered loop dimensions and are required to broadcast
+    naturally against each other.
+
+    Args:
+        func (callable):
+            Function to call like ``func(*args, **kwargs)`` on input arrays
+            (``*args``) that returns an array or tuple of arrays. If
+            multiple arguments with non-matching dimensions are supplied,
+            this function is expected to vectorize (broadcast) over axes of
+            positional arguments in the style of NumPy universal functions.
+        signature (string):
+            Specifies what core dimensions are consumed and produced by
+            ``func``.  According to the specification of numpy.gufunc
+            signature.
+        supports_batched (bool, optional):
+            If the wrapped function supports to pass the complete input
+            array with the loop and the core dimensions.
+            Defaults to `False`. Dimensions will be iterated in the
+            `GUFunc` processing code.
+        supports_out (bool, optional):
+            If the wrapped function supports out as one of its kwargs.
+            Defaults to `False`.
+        signatures (list of tuple of str):
+            Contains strings in the form of 'ii->i' with i being the char of a
+            dtype. Each element of the list is a tuple with the string
+            and a alternative function to `func` to be executed when the inputs
+            of the function can be casted as described by this function.
+        name (str, optional):
+            Name for the GUFunc object. If not specified, ``func``'s name
+            is used.
+        doc (str, optional):
+            Docstring for the GUFunc object. If not specified, ``func.__doc__``
+            is used.
+    '''
+
+    def __init__(self, func, signature, **kwargs):
+        # We would like to create gufuncs from cupy regular ufuncs
+        # so we can avoid most of the __call__ stuff
+        self._func = func
+        self._signature = signature
+        self.__name__ = kwargs.pop('name', func.__name__)
+        self.__doc__ = kwargs.pop('doc', func.__doc__)
+
+        # The following are attributes to avoid applying certain steps
+        # when wrapping cupy functions that do some of the gufunc
+        # stuff internally due to CUDA libraries requirements
+        self._supports_batched = kwargs.pop('supports_batched', False)
+        self._supports_out = kwargs.pop('supports_out', False)
+        signatures = kwargs.pop('signatures', [])
+
+        if kwargs:
+            raise TypeError(
+                'got unexpected keyword arguments: '
+                + ', '.join([repr(k) for k in kwargs])
+            )
+
+        # Preprocess the signature here
+        input_coredimss, output_coredimss = _parse_gufunc_signature(
+            self._signature)
+        self._input_coredimss = input_coredimss
+        self._output_coredimss = output_coredimss
+        # This is pre-calculated to later check the minimum number of
+        # dimensions required per input
+        self._min_dims = [0] * len(input_coredimss)
+        for i, inp in enumerate(input_coredimss):
+            for d in inp:
+                if d[-1] != '?':
+                    self._min_dims[i] += 1
+
+        # Determine nout: nout = None for functions of one
+        # direct return; nout = int for return tuples
+        self._nout = (
+            0
+            if not isinstance(output_coredimss, list)
+            else len(output_coredimss)
+        )
+        self._nin = (
+            0
+            if not isinstance(input_coredimss, list)
+            else len(input_coredimss)
+        )
+        # Determines the function that will be run depending on the datatypes
+        # Pass a list of signatures that are either the types in format
+        # ii->o or a tuple with the string and a function other than func to be
+        # executed for those types
+        # For some reason _nout is a tuple and now we get it with 0s
+        self._ops_register = _OpsRegister(
+            signatures, self._func, self._nin, self._nout, self.__name__)
+
+    def _apply_func_to_inputs(self, func, dim, sizes, dims, args, outs):
+        # Apply function
+        # The resulting array is loop_output_dims+the specified dims
+        # Some functions have batching logic inside due to higly
+        # optimized CUDA libraries so we just call them
+        if self._supports_batched or dim == len(dims):
+            # Check if the function supports out, order and other args
+            if self._supports_out and outs is not None:
+                outs = outs[0] if len(outs) == 1 else outs
+                func(*args, out=outs)
+            else:
+                fouts = func(*args)
+                # TODO(ecastill) improve this check
+                if isinstance(fouts, cupy.ndarray):
+                    fouts = (fouts,)
+                for o, fo in zip(outs, fouts):
+                    cupy._core.elementwise_copy(fo, o)
+        else:
+            dim_size = sizes[dims[dim]][0]
+            for i in range(dim_size):
+                n_args = [a[i] for a in args]
+                if outs is not None:
+                    n_outs = [o[i] for o in outs]
+                    self._apply_func_to_inputs(
+                        func, dim + 1, sizes, dims, n_args, n_outs)
+
+    def _transpose_element(self, arg, iax, shape):
+        iax = tuple(a if a < 0 else a - len(shape) for a in iax)
+        tidc = (
+            tuple(i for i in range(
+                -len(shape) + 0, 0) if i not in iax) + iax
+        )
+        return arg.transpose(tidc)
+
+    def _get_args_transposed(self, args, input_axes, outs, output_axes):
+        # This code credit goes to Dask
+        # https://github.com/dask/dask/blob/61b578f5a3ad88cbc6a8b9a73ce08c551bd969fa/dask/array/gufunc.py#L349-L377
+        # modifications have been done to support arguments broadcast
+        # out argument, and optional core dims.
+        transposed_args = []
+        # This is used when reshaping the outputs so that we can delete
+        # dims that were not specified in the input
+        missing_dims = set()
+        for i, (arg, iax, input_coredims, md) in enumerate(zip(
+                args, input_axes, self._input_coredimss, self._min_dims)):
+            shape = arg.shape
+            nds = len(shape)
+            # For the inputs that has missing dimensions we need to reshape
+            if nds < md:
+                raise ValueError(f'Input operand {i} does not have enough'
+                                 f' dimensions (has {nds}, gufunc core with'
+                                 f' signature {self._signature} requires {md}')
+            optionals = len(input_coredims) - nds
+            if optionals > 0:
+                # Look for optional dimensions
+                # We only allow the first or the last dimensions to be optional
+                if input_coredims[0][-1] == '?':
+                    shape = (1,) * optionals + shape
+                    missing_dims.update(set(input_coredims[:optionals]))
+                else:
+                    shape = shape + (1,) * optionals
+                    missing_dims.update(
+                        set(input_coredims[min(0, len(shape)-1):]))
+                arg = arg.reshape(shape)
+            transposed_args.append(self._transpose_element(arg, iax, shape))
+        args = transposed_args
+
+        if outs is not None:
+            transposed_outs = []
+            # outs should be transposed to the intermediate form before
+            # copying all results
+            for out, iox, coredims in zip(
+                    outs, output_axes, self._output_coredimss):
+                transposed_outs.append(self._transpose_element(
+                    out, iox, out.shape))
+            # check that outs has been correctly transposed
+            # if the function returns a scalar, outs will be ignored
+            if len(transposed_outs) == len(outs):
+                outs = transposed_outs
+
+        # we cant directly broadcast arrays together since their core dims
+        # might differ. Only the loop dimensions are broadcastable
+        shape = internal._broadcast_shapes(
+            [a.shape[:-len(self._input_coredimss)] for a in args])
+        args = [_manipulation.broadcast_to(
+            a, shape + a.shape[-len(self._input_coredimss):]) for a in args]
+
+        # Assess input args for loop dims
+        input_shapes = [a.shape for a in args]
+        num_loopdims = [
+            len(s) - len(cd) for s, cd in zip(
+                input_shapes, self._input_coredimss)
+        ]
+        max_loopdims = max(num_loopdims) if num_loopdims else None
+        core_input_shapes = [
+            dict(zip(icd, s[n:]))
+            for s, n, icd in zip(
+                input_shapes, num_loopdims, self._input_coredimss)
+        ]
+        core_shapes = {}
+        for d in core_input_shapes:
+            core_shapes.update(d)
+
+        loop_input_dimss = [
+            tuple(
+                '__loopdim%d__' % d for d in range(
+                    max_loopdims - n, max_loopdims)
+            )
+            for n in num_loopdims
+        ]
+        input_dimss = [li + c for li, c in zip(
+            loop_input_dimss, self._input_coredimss)]
+
+        loop_output_dims = max(loop_input_dimss, key=len, default=())
+
+        # Assess input args for same size and chunk sizes
+        # Collect sizes and chunksizes of all dims in all arrays
+        dimsizess = {}
+        for dims, shape in zip(input_dimss, input_shapes):
+            for dim, size in zip(dims, shape):
+                dimsizes = dimsizess.get(dim, [])
+                dimsizes.append(size)
+                dimsizess[dim] = dimsizes
+
+        # Assert correct partitioning, for case:
+        for dim, sizes in dimsizess.items():
+            if set(sizes).union({1}) != {1, max(sizes)}:
+                raise ValueError(
+                    f'Dimension {dim} with different lengths in arrays'
+                )
+
+        return args, dimsizess, loop_output_dims, outs, missing_dims
+
+    def _determine_order(self, args, order):
+        if order.upper() in ('C', 'K'):
+            # Order is determined to be C to allocate the out array
+            # but we will change the strides of the out array
+            # to be K later in __call__
+            return 'C'
+        elif order.upper() == 'A':
+            # order is F if all arrays are strictly F
+            order = ('F' if all([a.flags.f_contiguous
+                                 and not a.flags.c_contiguous
+                                 for a in args]) else 'C')
+            return order
+
+        elif order.upper() == 'F':
+            return 'F'
+        else:
+            raise RuntimeError(f'Unknown order {order}')
+
+    def __call__(self, *args, **kwargs):
+        '''
+        Apply a generalized ufunc.
+
+        Args:
+            args: Input arguments. Each of them can be a :class:`cupy.ndarray`
+                object or a scalar. The output arguments can be omitted or be
+                specified by the ``out`` argument.
+            axes (List of tuples of int, optional):
+                A list of tuples with indices of axes a generalized ufunc
+                should operate on.
+                For instance, for a signature of ``'(i,j),(j,k)->(i,k)'``
+                appropriate for matrix multiplication, the base elements are
+                two-dimensional matrices and these are taken to be stored in
+                the two last axes of each argument.  The corresponding
+                axes keyword would be ``[(-2, -1), (-2, -1), (-2, -1)]``.
+                For simplicity, for generalized ufuncs that operate on
+                1-dimensional arrays (vectors), a single integer is accepted
+                instead of a single-element tuple, and for generalized ufuncs
+                for which all outputs are scalars, the output tuples
+                can be omitted.
+            axis (int, optional):
+                A single axis over which a generalized ufunc should operate.
+                This is a short-cut for ufuncs that operate over a single,
+                shared core dimension, equivalent to passing in axes with
+                entries of (axis,) for each single-core-dimension argument
+                and ``()`` for all others.
+                For instance, for a signature ``'(i),(i)->()'``, it is
+                equivalent to passing in ``axes=[(axis,), (axis,), ()]``.
+            keepdims (bool, optional):
+                If this is set to True, axes which are reduced over will be
+                left in the result as a dimension with size one, so that the
+                result will broadcast correctly against the inputs. This
+                option can only be used for generalized ufuncs that operate
+                on inputs that all have the same number of core dimensions
+                and with outputs that have no core dimensions , i.e., with
+                signatures like ``'(i),(i)->()'`` or ``'(m,m)->()'``.
+                If used, the location of the dimensions in the output can
+                be controlled with axes and axis.
+            casting (str, optional):
+                Provides a policy for what kind of casting is permitted.
+                Defaults to ``'same_kind'``
+            dtype (dtype, optional):
+                Overrides the dtype of the calculation and output arrays.
+                Similar to signature.
+            signature (str or tuple of dtype, optional):
+                Either a data-type, a tuple of data-types, or a special
+                signature string indicating the input and output types of a
+                ufunc. This argument allows you to provide a specific
+                signature for the function to be used if registered in the
+                ``signatures`` kwarg of the ``__init__`` method.
+                If the loop specified does not exist for the ufunc, then
+                a TypeError is raised. Normally, a suitable loop is found
+                automatically by comparing the input types with what is
+                available and searching for a loop with data-types to
+                which all inputs can be cast safely. This keyword argument
+                lets you bypass that search and choose a particular loop.
+            order (str, optional):
+                Specifies the memory layout of the output array. Defaults to
+                ``'K'``.``'C'`` means the output should be C-contiguous,
+                ``'F'`` means F-contiguous, ``'A'`` means F-contiguous
+                if the inputs are F-contiguous and not also not C-contiguous,
+                C-contiguous otherwise, and ``'K'`` means to match the element
+                ordering of the inputs as closely as possible.
+            out (cupy.ndarray): Output array. It outputs to new arrays
+                default.
+
+        Returns:
+            Output array or a tuple of output arrays.
+        '''
+
+        #  This argument cannot be used for generalized ufuncs
+        #  as those take non-scalar input.
+        # where = kwargs.pop('where', None)
+
+        outs = kwargs.pop('out', None)
+        axes = kwargs.pop('axes', None)
+        axis = kwargs.pop('axis', None)
+        order = kwargs.pop('order', 'K')
+        dtype = kwargs.pop('dtype', None)
+        keepdims = kwargs.pop('keepdims', False)
+        signature = kwargs.pop('signature', None)
+        casting = kwargs.pop('casting', 'same_kind')
+        if len(kwargs) > 0:
+            raise RuntimeError(
+                'Unknown kwargs {}'.format(' '.join(kwargs.keys())))
+
+        ret_dtype = None
+        func = self._func
+
+        # this will cast the inputs appropiately
+        args, ret_dtype, func = self._ops_register.determine_dtype(
+            args, dtype, casting, signature)
+
+        if not type(self._signature) == str:
+            raise TypeError('`signature` has to be of type string')
+
+        if outs is not None and type(outs) != tuple:
+            if isinstance(outs, cupy.ndarray):
+                outs = (outs,)
+            else:
+                raise TypeError('`outs` must be a tuple or `cupy.ndarray`')
+
+        filter_order = self._determine_order(args, order)
+
+        input_coredimss = self._input_coredimss
+        output_coredimss = self._output_coredimss
+        if outs is not None and type(outs) != tuple:
+            raise TypeError('`outs` must be a tuple')
+        # Axes
+        input_axes, output_axes = _validate_normalize_axes(
+            axes, axis, keepdims, input_coredimss, output_coredimss
+        )
+
+        if len(input_coredimss) != len(args):
+            ValueError(
+                'According to `signature`, `func` requires %d arguments,'
+                ' but %s given' % (len(input_coredimss), len(args)))
+
+        args, dimsizess, loop_output_dims, outs, m_dims = self._get_args_transposed(  # NOQA
+            args, input_axes, outs, output_axes)
+
+        # The output shape varies depending on optional dims or not
+        # TODO(ecastill) this only works for one out argument
+        out_shape = [dimsizess[od][0] for od in loop_output_dims]
+        if self._nout > 0:
+            out_shape += [dimsizess[od][0] for od in output_coredimss[0]]
+        out_shape = tuple(out_shape)
+
+        if outs is None:
+            outs = cupy.empty(out_shape, dtype=ret_dtype, order=filter_order)
+            if order == 'K':
+                strides = internal._get_strides_for_order_K(
+                    outs, ret_dtype, out_shape)
+                outs._set_shape_and_strides(out_shape, strides, True, True)
+            outs = (outs,)
+        else:
+            if outs[0].shape != out_shape:
+                raise ValueError(f'Invalid shape for out {outs[0].shape}'
+                                 f' needs {out_shape}')
+
+            _raise_if_invalid_cast(
+                ret_dtype, outs[0].dtype, casting, "out dtype")
+
+        self._apply_func_to_inputs(
+            func, 0, dimsizess, loop_output_dims, args, outs)
+
+        # This code credit goes to Dask
+        # https://github.com/dask/dask/blob/61b578f5a3ad88cbc6a8b9a73ce08c551bd969fa/dask/array/gufunc.py#L462-L503
+        # Treat direct output
+
+        if self._nout == 0:
+            output_coredimss = [output_coredimss]
+
+        # Split output
+        # tmp might be a tuple of outs
+        # we changed the way we apply the function compared to dask
+        # we have added support for optional dims
+        leaf_arrs = []
+        for tmp in outs:
+            for i, (ocd, oax) in enumerate(zip(output_coredimss, output_axes)):
+                leaf_arr = tmp
+
+                # Axes:
+                if keepdims:
+                    slices = (len(leaf_arr.shape) * (slice(None),)
+                              + len(oax) * (numpy.newaxis,))
+                    leaf_arr = leaf_arr[slices]
+
+                tidcs = [None] * len(leaf_arr.shape)
+                for i, oa in zip(range(-len(oax), 0), oax):
+                    tidcs[oa] = i
+                j = 0
+                for i in range(len(tidcs)):
+                    if tidcs[i] is None:
+                        tidcs[i] = j
+                        j += 1
+                leaf_arr = leaf_arr.transpose(tidcs)
+                # Delete the dims that were optionals after the input expansion
+                if len(m_dims) > 0:
+                    shape = leaf_arr.shape
+                    # This line deletes the dimensions that were not present
+                    # in the input
+                    core_shape = shape[-len(ocd):]
+                    core_shape = tuple([
+                        d for d, n in zip(core_shape, ocd) if n not in m_dims])
+                    shape = shape[:-len(ocd)] + core_shape
+                    leaf_arr = leaf_arr.reshape(shape)
+                # leaf_arrs.append(leaf_arr.astype(leaf_arr.dtype, order=order))  # NOQA
+                leaf_arrs.append(leaf_arr)
+        return tuple(leaf_arrs) if self._nout > 1 else leaf_arrs[0]
diff --git a/cupy/_core/_kernel.pxd b/cupy/_core/_kernel.pxd
new file mode 100644
index 0000000..e4b139e
--- /dev/null
+++ b/cupy/_core/_kernel.pxd
@@ -0,0 +1,170 @@
+from libcpp cimport vector
+
+from cupy._core cimport _carray
+from cupy._core cimport _scalar
+from cupy._core._carray cimport shape_t
+from cupy._core.core cimport _ndarray_base
+from cupy.cuda cimport memory
+from cupy.cuda cimport texture
+
+
+cdef class ParameterInfo:
+    cdef:
+        readonly str name
+        readonly object dtype
+        readonly str ctype
+        readonly bint raw
+        readonly bint is_const
+
+
+cdef enum _ArgKind:
+    ARG_KIND_NDARRAY = 1
+    ARG_KIND_INDEXER
+    ARG_KIND_SCALAR
+    ARG_KIND_POINTER
+    ARG_KIND_TEXTURE
+
+
+cdef class _ArgInfo:
+    # Holds metadata of an argument.
+    # This class is immutable and used as a part of hash keys.
+
+    cdef:
+        readonly _ArgKind arg_kind
+        readonly type type
+        readonly object dtype
+        readonly int ndim
+        readonly bint c_contiguous
+        readonly bint index_32_bits
+
+    cdef _ArgInfo _init(
+        self,
+        _ArgKind arg_kind,
+        type typ,
+        object dtype,
+        int ndim,
+        bint c_contiguous,
+        bint index_32_bits)
+
+    @staticmethod
+    cdef _ArgInfo from_arg(object arg)
+
+    @staticmethod
+    cdef _ArgInfo from_ndarray(_ndarray_base arg)
+
+    @staticmethod
+    cdef _ArgInfo from_scalar(_scalar.CScalar arg)
+
+    @staticmethod
+    cdef _ArgInfo from_indexer(_carray.Indexer arg)
+
+    @staticmethod
+    cdef _ArgInfo from_memptr(memory.MemoryPointer arg)
+
+    @staticmethod
+    cdef _ArgInfo from_texture(texture.TextureObject arg)
+
+    cdef _ArgInfo as_ndarray_with_ndim(self, int ndim)
+
+    cdef bint is_ndarray(self)
+
+    cdef bint is_scalar(self)
+
+    cdef str get_c_type(self)
+
+    cdef str get_param_c_type(self, ParameterInfo p)
+
+    cdef str get_c_var_name(self, ParameterInfo p)
+
+
+cdef class _TypeMap:
+    # Typedef mapping between C types.
+    # This class is immutable.
+
+    cdef:
+        tuple _pairs
+
+    cdef str get_typedef_code(self)
+
+
+cdef class _Op:
+    """Simple data structure that represents a kernel routine with single \
+concrete dtype mapping.
+    """
+
+    cdef:
+        readonly tuple in_types
+        readonly tuple out_types
+        readonly int nin
+        readonly int nout
+        readonly object routine
+        # If the type combination specified by in_types and out_types is
+        # disallowed, error_func must be set instead of routine.
+        # It's called by check_valid() method.
+        readonly object error_func
+
+    @staticmethod
+    cdef _Op _from_type_and_routine_or_error_func(
+        str typ, object routine, object error_func)
+
+    # Creates an op instance parsing a dtype mapping.
+    @staticmethod
+    cdef _Op from_type_and_routine(str typ, routine)
+
+    cpdef tuple get_in_dtypes(self)
+
+    cpdef tuple get_out_dtypes(self)
+
+    # Creates an op instance parsing a dtype mapping with given error function.
+    @staticmethod
+    cdef _Op from_type_and_error_func(str typ, error_func)
+
+    # Raises an error if error_func is given.
+    cdef check_valid(self)
+
+
+cdef class _Ops:
+    """A kernel routine representation with various dtype mappings.
+    """
+
+    cdef:
+        readonly tuple ops
+        readonly int nin
+        readonly int nout
+
+    @staticmethod
+    cdef _Ops from_tuples(object ops, routine)
+
+    # Queries a single op from input arguments.
+    cpdef _Op guess_routine(
+        self, str name, dict cache, list in_args, dtype, _Ops out_ops)
+
+    cpdef _Op _guess_routine_from_in_types(
+        self, tuple in_types, object can_cast=*)
+
+    cpdef _Op _guess_routine_from_dtype(self, object dtype)
+
+
+cpdef create_ufunc(name, ops, routine=*, preamble=*, doc=*,
+                   default_casting=*, loop_prep=*, out_ops=*,
+                   cutensor_op=*, scatter_op=*)
+
+cdef tuple _get_arginfos(list args)
+
+cdef str _get_kernel_params(tuple params, tuple arginfos)
+
+cdef list _broadcast(list args, tuple params, bint use_size, shape_t& shape)
+
+cdef list _get_out_args_from_optionals(
+    subtype, list out_args, tuple out_types, const shape_t& out_shape, casting,
+    obj)
+
+cdef list _get_out_args_with_params(
+    list out_args, tuple out_types,
+    const shape_t& out_shape, tuple out_params, bint is_size_specified)
+
+cdef _check_peer_access(_ndarray_base arr, int device_id)
+
+cdef list _preprocess_args(int dev_id, args, bint use_c_scalar)
+
+cdef shape_t _reduce_dims(list args, tuple params, const shape_t& shape)
diff --git a/cupy/_core/_kernel.pyx b/cupy/_core/_kernel.pyx
new file mode 100644
index 0000000..1e719f6
--- /dev/null
+++ b/cupy/_core/_kernel.pyx
@@ -0,0 +1,1622 @@
+import string
+import warnings
+
+import numpy
+
+import cupy
+from cupy.cuda import compiler
+from cupy import _util
+
+cimport cython  # NOQA
+
+from libcpp cimport vector
+
+from cupy.cuda cimport device
+from cupy.cuda cimport function
+from cupy.cuda cimport memory
+from cupy.cuda cimport texture
+from cupy._core cimport _accelerator
+from cupy._core cimport _carray
+from cupy._core cimport _scalar
+from cupy._core._dtype cimport get_dtype, _raise_if_invalid_cast
+from cupy._core._memory_range cimport may_share_bounds
+from cupy._core._scalar import get_typename as _get_typename
+from cupy._core cimport core
+from cupy._core.core cimport _convert_object_with_cuda_array_interface
+from cupy._core.core cimport _ndarray_init
+from cupy._core.core cimport compile_with_cache
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport internal
+from cupy_backends.cuda.api cimport runtime
+
+try:
+    import cupy_backends.cuda.libs.cutensor as cuda_cutensor
+except ImportError:
+    cuda_cutensor = None
+
+from cupy._core import _fusion_thread_local
+
+
+cdef inline bint _contains_zero(const shape_t& v) except? -1:
+    for i in range(v.size()):
+        if v[i] == 0:
+            return True
+    return False
+
+
+@_util.memoize(for_each_device=True)
+def _get_warpsize():
+    device_id = runtime.getDevice()
+    return runtime.getDeviceProperties(device_id)['warpSize']
+
+
+cdef str _get_simple_elementwise_kernel_code(
+        tuple params, tuple arginfos, str operation, str name,
+        _TypeMap type_map, str preamble, str loop_prep='', str after_loop=''):
+    # No loop unrolling due to avoid 64-bit division
+    module_code = string.Template('''
+    ${typedef_preamble}
+    ${preamble}
+    extern "C" __global__ void ${name}(${params}) {
+      ${loop_prep};
+      #pragma unroll 1
+      CUPY_FOR(i, _ind.size()) {
+        _ind.set(i);
+        ${operation};
+      }
+      ${after_loop};
+    }
+    ''').substitute(
+        typedef_preamble=type_map.get_typedef_code(),
+        params=_get_kernel_params(params, arginfos),
+        operation=operation,
+        name=name,
+        preamble=preamble,
+        loop_prep=loop_prep,
+        after_loop=after_loop)
+    return module_code
+
+
+cdef function.Function _get_simple_elementwise_kernel_from_code(
+        str name, str code, tuple options=()):
+    module = compile_with_cache(code, options)
+    return module.get_function(name)
+
+
+cdef function.Function _get_simple_elementwise_kernel(
+        tuple params, tuple arginfos, str operation, str name,
+        _TypeMap type_map, str preamble, str loop_prep='', str after_loop='',
+        tuple options=()):
+    code = _get_simple_elementwise_kernel_code(
+        params, arginfos, operation, name, type_map, preamble, loop_prep,
+        after_loop
+    )
+    return _get_simple_elementwise_kernel_from_code(name, code, options)
+
+
+cdef inline int _get_kind_score(int kind):
+    if b'b' == kind:
+        return 0
+    if b'u' == kind or b'i' == kind:
+        return 1
+    if b'f' == kind or b'c' == kind:
+        return 2
+    return -1
+
+
+@cython.profile(False)
+cdef inline _check_peer_access(_ndarray_base arr, int device_id):
+    if arr.data.device_id == device_id:
+        return
+
+    msg = (
+        f'The device where the array resides ({arr.data.device_id}) is '
+        f'different from the current device ({device_id}).'
+    )
+
+    cdef bint peer_access = device._enable_peer_access(
+        device_id, arr.data.device_id)
+    if not peer_access:
+        raise ValueError(
+            f'{msg} Peer access is unavailable between these devices.')
+    warnings.warn(
+        f'{msg} Peer access has been activated automatically.',
+        _util.PerformanceWarning)
+
+
+cdef inline _preprocess_arg(int dev_id, arg, bint use_c_scalar):
+    if isinstance(arg, _ndarray_base):
+        s = arg
+        _check_peer_access(<_ndarray_base>s, dev_id)
+    elif isinstance(arg, texture.TextureObject):
+        s = arg
+    elif hasattr(arg, '__cuda_array_interface__'):
+        s = _convert_object_with_cuda_array_interface(arg)
+        _check_peer_access(<_ndarray_base>s, dev_id)
+    elif hasattr(arg, '__cupy_get_ndarray__'):
+        s = arg.__cupy_get_ndarray__()
+        _check_peer_access(<_ndarray_base>s, dev_id)
+    else:  # scalars or invalid args
+        if use_c_scalar:
+            s = _scalar.scalar_to_c_scalar(arg)
+        else:
+            s = _scalar.scalar_to_numpy_scalar(arg)
+        if s is None:
+            raise TypeError('Unsupported type %s' % type(arg))
+    return s
+
+
+cdef list _preprocess_args(int dev_id, args, bint use_c_scalar):
+    """Preprocesses arguments for kernel invocation
+
+    - Checks device compatibility for ndarrays
+    - Converts Python/NumPy scalars:
+      - If use_c_scalar is True, into CScalars.
+      - If use_c_scalar is False, into NumPy scalars.
+    """
+    cdef list ret = []
+    for arg in args:
+        ret.append(_preprocess_arg(dev_id, arg, use_c_scalar))
+    return ret
+
+
+cdef list _preprocess_optional_args(int dev_id, args, bint use_c_scalar):
+    """Preprocesses arguments for kernel invocation
+
+    - Checks device compatibility for ndarrays
+    - Converts Python/NumPy scalars:
+      - If use_c_scalar is True, into CScalars.
+      - If use_c_scalar is False, into NumPy scalars.
+    """
+    cdef list ret = []
+    for arg in args:
+        if arg is None:
+            ret.append(None)
+        else:
+            ret.append(_preprocess_arg(dev_id, arg, use_c_scalar))
+    return ret
+
+
+cdef class _ArgInfo:
+    # Holds metadata of an argument.
+    # This class is immutable and used as a part of hash keys.
+
+    def __init__(self, *args):
+        arg_kind, typ, dtype, ndim, c_contiguous, index_32_bits = args
+        self._init(arg_kind, typ, dtype, ndim, c_contiguous, index_32_bits)
+
+    cdef _ArgInfo _init(
+            self,
+            _ArgKind arg_kind,
+            type typ,
+            object dtype,
+            int ndim,
+            bint c_contiguous,
+            bint index_32_bits):
+        self.arg_kind = arg_kind
+        self.type = typ
+        self.dtype = dtype
+        self.ndim = ndim
+        self.c_contiguous = c_contiguous
+        self.index_32_bits = index_32_bits
+
+    @staticmethod
+    cdef _ArgInfo from_arg(object arg):
+        typ = type(arg)
+        if issubclass(typ, _ndarray_base):
+            return _ArgInfo.from_ndarray(arg)
+        if typ is _scalar.CScalar:
+            return _ArgInfo.from_scalar(arg)
+        if typ is _carray.Indexer:
+            return _ArgInfo.from_indexer(arg)
+        if typ is memory.MemoryPointer:
+            return _ArgInfo.from_memptr(arg)
+        if typ is texture.TextureObject:
+            return _ArgInfo.from_texture(arg)
+        assert False, typ
+
+    @staticmethod
+    cdef _ArgInfo from_ndarray(_ndarray_base arg):
+        cdef _ArgInfo ret = _ArgInfo.__new__(_ArgInfo)
+        ret._init(
+            ARG_KIND_NDARRAY,
+            type(arg),
+            arg.dtype.type,
+            arg._shape.size(),
+            arg._c_contiguous,
+            arg._index_32_bits)
+        return ret
+
+    @staticmethod
+    cdef _ArgInfo from_scalar(_scalar.CScalar arg):
+        cdef _ArgInfo ret = _ArgInfo.__new__(_ArgInfo)
+        dtype = arg.get_numpy_type()
+        ret._init(ARG_KIND_SCALAR, _scalar.CScalar, dtype, 0, True, True)
+        return ret
+
+    @staticmethod
+    cdef _ArgInfo from_indexer(_carray.Indexer arg):
+        cdef _ArgInfo ret = _ArgInfo.__new__(_ArgInfo)
+        ret._init(
+            ARG_KIND_INDEXER, _carray.Indexer, None, arg.ndim, True,
+            arg._index_32_bits)
+        return ret
+
+    @staticmethod
+    cdef _ArgInfo from_memptr(memory.MemoryPointer arg):
+        cdef _ArgInfo ret = _ArgInfo.__new__(_ArgInfo)
+        ret._init(
+            ARG_KIND_POINTER, memory.MemoryPointer, None, 0, True, True)
+        return ret
+
+    @staticmethod
+    cdef _ArgInfo from_texture(texture.TextureObject arg):
+        cdef _ArgInfo ret = _ArgInfo.__new__(_ArgInfo)
+        ret._init(
+            ARG_KIND_TEXTURE, texture.TextureObject, None, 0, True, True)
+        return ret
+
+    def __hash__(self):
+        return hash((self.arg_kind, self.type, self.dtype, self.ndim,
+                     self.c_contiguous, self.index_32_bits))
+
+    def __eq__(self, other):
+        cdef _ArgInfo oth
+        if not isinstance(other, _ArgInfo):
+            return False
+        oth = other
+        return (
+            self.arg_kind == oth.arg_kind
+            and self.type is oth.type
+            and self.dtype == oth.dtype
+            and self.ndim == oth.ndim
+            and self.c_contiguous == oth.c_contiguous
+            and self.index_32_bits == oth.index_32_bits)
+
+    def __repr__(self):
+        return '<_ArgInfo({})>'.format(
+            ' '.join([
+                'arg_kind={!r}'.format(self.arg_kind),
+                'type={!r}'.format(self.type),
+                'dtype={!r}'.format(self.dtype),
+                'ndim={!r}'.format(self.ndim),
+                'c_contiguous={!r}'.format(self.c_contiguous),
+                'index_32_bits={!r}'.format(self.index_32_bits),
+            ]))
+
+    cdef _ArgInfo as_ndarray_with_ndim(self, int ndim):
+        # Returns an ndarray _ArgInfo with altered ndim.
+        # If ndim is the same, self is returned untouched.
+        assert self.arg_kind == ARG_KIND_NDARRAY
+        if self.ndim == ndim:
+            return self
+        return _ArgInfo(
+            ARG_KIND_NDARRAY, self.dtype, self.dtype, ndim, False, False)
+
+    cdef bint is_ndarray(self):
+        return self.arg_kind == ARG_KIND_NDARRAY
+
+    cdef bint is_scalar(self):
+        return self.arg_kind == ARG_KIND_SCALAR
+
+    cdef str get_c_type(self):
+        # Returns the C type representation.
+        if self.arg_kind == ARG_KIND_NDARRAY:
+            return 'CArray<%s, %d, %d, %d>' % (
+                _get_typename(self.dtype), self.ndim,
+                self.c_contiguous, self.index_32_bits)
+        if self.arg_kind == ARG_KIND_SCALAR:
+            return _get_typename(self.dtype)
+        if self.arg_kind == ARG_KIND_INDEXER:
+            return 'CIndexer<%d, %d>' % (self.ndim, self.index_32_bits)
+        if self.arg_kind == ARG_KIND_TEXTURE:
+            return 'cudaTextureObject_t'
+        assert False
+
+    cdef str get_param_c_type(self, ParameterInfo p):
+        # Returns the C type representation in the global function's
+        # parameter list.
+        cdef str ctyp = self.get_c_type()
+        if p.is_const:
+            return 'const ' + ctyp
+        return ctyp
+
+    cdef str get_c_var_name(self, ParameterInfo p):
+        if self.arg_kind in (ARG_KIND_NDARRAY, ARG_KIND_POINTER) and not p.raw:
+            return '_raw_' + p.name
+        return p.name
+
+
+cdef tuple _get_arginfos(list args):
+    return tuple([_ArgInfo.from_arg(a) for a in args])
+
+
+cdef str _get_kernel_params(tuple params, tuple arginfos):
+    cdef ParameterInfo p
+    cdef _ArgInfo arginfo
+    assert len(params) == len(arginfos)
+    lst = []
+    for i in range(len(params)):
+        p = params[i]
+        arginfo = arginfos[i]
+        lst.append('{} {}'.format(
+            arginfo.get_param_c_type(p),
+            arginfo.get_c_var_name(p)))
+    return ', '.join(lst)
+
+
+cdef shape_t _reduce_dims(list args, tuple params, const shape_t& shape):
+    """ Remove contiguous stride to optimize CUDA kernel."""
+    cdef _ndarray_base arr
+
+    if shape.size() <= 1 or len(args) == 0:
+        return shape
+
+    if len(args) == 1:  # fast path for reduction
+        a = args[0]
+        if (<ParameterInfo>params[0]).raw or not isinstance(a, _ndarray_base):
+            return shape
+        arr = a
+        arr = arr.reduced_view()
+        if arr is a:
+            return shape
+        else:
+            args[0] = arr
+            return arr._shape
+    return _reduced_view_core(args, params, shape)
+
+
+cdef shape_t _reduced_view_core(list args, tuple params, const shape_t& shape):
+    cdef int i, ax, last_ax, ndim
+    cdef Py_ssize_t total_size
+    cdef shape_t vecshape, newshape, newstrides
+    cdef vector.vector[int] array_indexes, axes
+    cdef vector.vector[int] strides_indexes
+    cdef ParameterInfo p
+    cdef _ndarray_base arr
+
+    ndim = shape.size()
+    array_indexes.reserve(len(args))
+    strides_indexes.reserve(len(args))
+    for i in range(len(args)):
+        p = params[i]
+        if p.raw:
+            continue
+        a = args[i]
+        if isinstance(a, _ndarray_base):
+            array_indexes.push_back(i)
+            arr = a
+            if not arr._c_contiguous:
+                if ndim == 2:  # short cut
+                    return shape
+                strides_indexes.push_back(i)
+
+    if array_indexes.size() == 0:
+        return shape
+
+    if strides_indexes.size() == 0:
+        # The input arrays are all c_contiguous
+        i = array_indexes[0]
+        arr = args[i]
+        total_size = arr.size
+        newshape.assign(<Py_ssize_t>1, total_size)
+        newstrides.resize(1)
+        for i in array_indexes:
+            arr = args[i]
+            newstrides[0] = arr.dtype.itemsize
+            # TODO(niboshi): Confirm update_x_contiguity flags
+            args[i] = arr._view(
+                type(arr), newshape, newstrides, False, True, arr)
+        return newshape
+
+    axes.reserve(ndim)
+    vecshape.reserve(ndim)
+    for ax in range(ndim):
+        vecshape.push_back(shape[ax])
+    last_ax = -1
+    for ax in range(ndim):
+        if vecshape[ax] == 1:
+            continue
+        if last_ax < 0:
+            last_ax = ax
+            continue
+        for i in strides_indexes:
+            arr = args[i]
+            if arr._strides[ax] * vecshape[ax] != arr._strides[last_ax]:
+                axes.push_back(last_ax)
+                break
+        else:
+            vecshape[ax] *= vecshape[last_ax]
+        last_ax = ax
+    if last_ax >= 0:
+        axes.push_back(last_ax)
+    if <int>axes.size() == ndim:
+        return shape
+
+    newshape.reserve(axes.size())
+    newstrides.reserve(axes.size())
+    for ax in axes:
+        newshape.push_back(vecshape[ax])
+    for i in array_indexes:
+        arr = args[i]
+        newstrides.clear()
+        for ax in axes:
+            newstrides.push_back(arr._strides[ax])
+        # TODO(niboshi): Confirm update_x_contiguity flags
+        args[i] = arr._view(type(arr), newshape, newstrides, False, True, arr)
+    return newshape
+
+
+cdef class ParameterInfo:
+
+    def __init__(self, str param, bint is_const):
+        self.name = None
+        self.dtype = None
+        self.ctype = None
+        self.raw = False
+        self.is_const = is_const
+        s = tuple([i for i in param.split() if len(i) != 0])
+        if len(s) < 2:
+            raise Exception('Syntax error: %s' % param)
+
+        t, self.name = s[-2:]
+        if t == 'CIndexer':
+            pass
+        elif len(t) == 1:
+            self.ctype = t
+        else:
+            dtype = get_dtype(t)
+            self.dtype = dtype.type
+            if dtype.name != t:
+                raise ValueError('Wrong type %s' % t)
+            self.ctype = _get_typename(self.dtype)
+
+        for i in s[:-2]:
+            if i == 'raw':
+                self.raw = True
+            elif i == '_non_const':
+                self.is_const = False
+            else:
+                raise Exception('Unknown keyword "%s"' % i)
+
+    def __hash__(self):
+        return hash((
+            self.name, self.dtype, self.ctype, self.raw, self.is_const))
+
+    def __eq__(self, other):
+        cdef ParameterInfo oth
+        if not isinstance(other, ParameterInfo):
+            return False
+        oth = other
+        return (
+            self.name == oth.name
+            and self.dtype == oth.dtype
+            and self.ctype == oth.ctype
+            and self.raw == oth.raw
+            and self.is_const == oth.is_const)
+
+    def __repr__(self):
+        return '<ParameterInfo({})>'.format(
+            ' '.join([
+                'name={!r}'.format(self.name),
+                'dtype={!r}'.format(self.dtype),
+                'ctype={!r}'.format(self.ctype),
+                'raw={!r}'.format(self.raw),
+                'is_const={!r}'.format(self.is_const),
+            ]))
+
+
+@_util.memoize()
+def _get_param_info(str s, is_const):
+    if len(s) == 0:
+        return ()
+    return tuple([ParameterInfo(i, is_const) for i in s.strip().split(',')])
+
+
+@_util.memoize()
+def _decide_params_type(in_params, out_params, in_args_dtype, out_args_dtype):
+    return _decide_params_type_core(in_params, out_params, in_args_dtype,
+                                    out_args_dtype)
+
+
+cdef class _TypeMap:
+
+    def __init__(self, pairs):
+        self._pairs = pairs
+
+    def __hash__(self):
+        return hash(self._pairs)
+
+    def __eq__(self, other):
+        if not isinstance(other, _TypeMap):
+            return False
+        return self._pairs == (<_TypeMap>other)._pairs
+
+    def __str__(self):
+        return '<_TypeMap {}>'.format(self._pairs)
+
+    cdef str get_typedef_code(self):
+        # Returns a code fragment of typedef statements used as preamble.
+        return ''.join([
+            'typedef %s %s;\n' % (_get_typename(ctype2), ctype1)
+            for ctype1, ctype2 in self._pairs])
+
+
+cdef tuple _decide_params_type_core(
+        tuple in_params, tuple out_params, tuple in_args_dtype,
+        tuple out_args_dtype):
+    type_dict = {}
+    if out_args_dtype:
+        assert len(out_params) == len(out_args_dtype)
+        for p, a in zip(out_params, out_args_dtype):
+            if a is None:
+                raise TypeError('Output arguments must be cupy.ndarray')
+            if p.dtype is not None:
+                if get_dtype(a) != get_dtype(p.dtype):
+                    raise TypeError(
+                        'Type is mismatched. %s %s %s' % (p.name, a, p.dtype))
+            elif p.ctype in type_dict:
+                t = type_dict[p.ctype]
+                if get_dtype(t) != get_dtype(a):
+                    raise TypeError(
+                        'Type is mismatched. %s %s %s %s' % (
+                            p.name, a, t, p.ctype))
+            else:
+                type_dict[p.ctype] = a
+
+    assert len(in_params) == len(in_args_dtype)
+    unknown_ctype = []  # TODO(leofang): remove this as it's unused?
+    for p, a in zip(in_params, in_args_dtype):
+        if a is None:
+            if p.dtype is None:
+                unknown_ctype.append(p.ctype)
+        else:
+            if p.dtype is not None:
+                if numpy.dtype(a) != numpy.dtype(p.dtype):
+                    raise TypeError(
+                        'Type is mismatched. %s %s %s' % (p.name, a, p.dtype))
+            elif p.ctype in type_dict:
+                t = type_dict[p.ctype]
+                if numpy.dtype(t) != numpy.dtype(a):
+                    raise TypeError(
+                        'Type is mismatched. %s %s %s %s' % (
+                            p.name, a, t, p.ctype))
+            else:
+                type_dict[p.ctype] = a
+
+    in_types = tuple([type_dict[p.ctype] if p.dtype is None else p.dtype
+                      for p in in_params])
+    out_types = tuple([type_dict[p.ctype] if p.dtype is None else p.dtype
+                       for p in out_params])
+    type_map = _TypeMap(tuple(sorted(type_dict.items())))
+    return in_types, out_types, type_map
+
+
+cdef list _broadcast(list args, tuple params, bint use_size, shape_t& shape):
+    # `shape` is an output argument
+    cdef Py_ssize_t i
+    cdef ParameterInfo p
+    cdef bint any_nonraw_array = False
+
+    # Collect non-raw arrays
+    value = []
+    for i, a in enumerate(args):
+        p = params[i]
+        if not p.raw and isinstance(a, _ndarray_base):
+            # Non-raw array
+            any_nonraw_array = True
+            value.append(a)
+        else:
+            value.append(None)
+
+    if use_size:
+        if any_nonraw_array:
+            raise ValueError('Specified \'size\' can be used only '
+                             'if all of the ndarray are \'raw\'.')
+    else:
+        if not any_nonraw_array:
+            raise ValueError('Loop size is undecided.')
+
+    # Perform broadcast.
+    # Note that arrays in `value` are replaced with broadcasted ones.
+    internal._broadcast_core(value, shape)
+
+    # Restore raw arrays and scalars from the original list.
+    for i, a in enumerate(value):
+        if a is None:
+            value[i] = args[i]
+    return value
+
+
+cdef _numpy_can_cast = numpy.can_cast
+
+
+cdef list _get_out_args_from_optionals(
+    subtype, list out_args, tuple out_types, const shape_t& out_shape, casting,
+    obj
+):
+    cdef _ndarray_base arr
+
+    while len(out_args) < len(out_types):
+        out_args.append(None)
+
+    for i, a in enumerate(out_args):
+        if a is None:
+            out_args[i] = _ndarray_init(
+                subtype, out_shape, out_types[i], obj)
+            continue
+
+        if not isinstance(a, _ndarray_base):
+            raise TypeError(
+                'Output arguments type must be cupy.ndarray')
+        arr = a
+        if not internal.vector_equal(arr._shape, out_shape):
+            raise ValueError('Out shape is mismatched')
+        out_type = get_dtype(out_types[i])
+
+        _raise_if_invalid_cast(out_type, arr.dtype, casting, "output operand")
+    return out_args
+
+
+cdef _copy_in_args_if_needed(list in_args, list out_args):
+    # `in_args` is an input and output argument
+    cdef _ndarray_base inp, out
+    for i in range(len(in_args)):
+        a = in_args[i]
+        if isinstance(a, _ndarray_base):
+            inp = a
+            for out in out_args:
+                if inp is not out and may_share_bounds(inp, out):
+                    in_args[i] = inp.copy()
+                    break
+
+
+cdef list _get_out_args_with_params(
+        list out_args, tuple out_types, const shape_t& out_shape,
+        tuple out_params, bint is_size_specified):
+    cdef ParameterInfo p
+    cdef _ndarray_base arr
+    if not out_args:
+        for p in out_params:
+            if p.raw and not is_size_specified:
+                raise ValueError('Output array size is Undecided')
+        return [_ndarray_init(
+            cupy.ndarray, out_shape, t, None) for t in out_types]
+
+    for i, p in enumerate(out_params):
+        a = out_args[i]
+        if not isinstance(a, _ndarray_base):
+            raise TypeError(
+                'Output arguments type must be cupy.ndarray')
+        arr = a
+        if not p.raw and not internal.vector_equal(arr._shape, out_shape):
+            raise ValueError('Out shape is mismatched')
+    return out_args
+
+
+@_util.memoize()
+def _get_elementwise_kernel_code(
+        tuple arginfos, _TypeMap type_map,
+        tuple params, str operation, str name,
+        str preamble, str loop_prep='', str after_loop='', tuple options=()):
+    cdef _ArgInfo arginfo
+
+    op = []
+    for p, arginfo in zip(params, arginfos):
+        if arginfo.is_ndarray() and not p.raw:
+            if p.is_const:
+                fmt = 'const {t} &{n} = _raw_{n}[_ind.get()];'
+            else:
+                fmt = '{t} &{n} = _raw_{n}[_ind.get()];'
+            op.append(fmt.format(t=p.ctype, n=p.name))
+    op.append(operation)
+    operation = '\n'.join(op)
+    return _get_simple_elementwise_kernel_code(
+        params, arginfos, operation, name, type_map,
+        preamble, loop_prep, after_loop)
+
+
+@_util.memoize(for_each_device=True)
+def _get_elementwise_kernel(
+        tuple arginfos, _TypeMap type_map,
+        tuple params, str operation, str name,
+        str preamble, str loop_prep='', str after_loop='', tuple options=()):
+    cdef str code = _get_elementwise_kernel_code(
+        arginfos, type_map, params, operation, name, preamble, loop_prep,
+        after_loop
+    )
+    return _get_simple_elementwise_kernel_from_code(name, code, options)
+
+
+cdef class ElementwiseKernel:
+
+    """User-defined elementwise kernel.
+
+    This class can be used to define an elementwise kernel with or without
+    broadcasting.
+
+    The kernel is compiled at an invocation of the
+    :meth:`~ElementwiseKernel.__call__` method,
+    which is cached for each device.
+    The compiled binary is also cached into a file under the
+    ``$HOME/.cupy/kernel_cache/`` directory with a hashed file name. The cached
+    binary is reused by other processes.
+
+    Args:
+        in_params (str): Input argument list.
+        out_params (str): Output argument list.
+        operation (str): The body in the loop written in CUDA-C/C++.
+        name (str): Name of the kernel function. It should be set for
+            readability of the performance profiling.
+        reduce_dims (bool): If ``False``, the shapes of array arguments are
+            kept within the kernel invocation. The shapes are reduced
+            (i.e., the arrays are reshaped without copy to the minimum
+            dimension) by default. It may make the kernel fast by reducing the
+            index calculations.
+        options (tuple): Compile options passed to NVRTC. For details, see
+            https://docs.nvidia.com/cuda/nvrtc/index.html#group__options.
+        preamble (str): Fragment of the CUDA-C/C++ code that is inserted at the
+            top of the cu file.
+        no_return (bool): If ``True``, __call__ returns ``None``.
+        return_tuple (bool): If ``True``, __call__ always returns tuple of
+            array even if single value is returned.
+        loop_prep (str): Fragment of the CUDA-C/C++ code that is inserted at
+            the top of the kernel function definition and above the ``for``
+            loop.
+        after_loop (str): Fragment of the CUDA-C/C++ code that is inserted at
+            the bottom of the kernel function definition.
+
+    """
+
+    cdef:
+        readonly tuple in_params
+        readonly tuple out_params
+        readonly Py_ssize_t nin
+        readonly Py_ssize_t nout
+        readonly Py_ssize_t nargs
+        readonly tuple params
+        readonly object operation
+        readonly str name
+        readonly str __name__
+        readonly bint reduce_dims
+        readonly object preamble
+        readonly bint no_return
+        readonly bint return_tuple
+        readonly dict kwargs
+        readonly dict _params_type_memo
+        readonly dict _elementwise_kernel_memo
+        readonly dict _cached_codes
+
+    def __init__(self, in_params, out_params, operation,
+                 name='kernel', reduce_dims=True, preamble='',
+                 no_return=False, return_tuple=False, **kwargs):
+        if not compiler.is_valid_kernel_name(name):
+            raise ValueError(
+                'Invalid kernel name: "%s"' % name)
+
+        self.in_params = _get_param_info(in_params, True)
+        self.out_params = _get_param_info(out_params, False)
+        self.nin = len(self.in_params)
+        self.nout = len(self.out_params)
+        self.nargs = self.nin + self.nout
+        param_rest = _get_param_info('CIndexer _ind', False)
+        self.params = self.in_params + self.out_params + param_rest
+        self.operation = operation
+        self.name = name
+        self.reduce_dims = reduce_dims
+        self.preamble = preamble
+        self.no_return = no_return
+        self.return_tuple = return_tuple
+        self.kwargs = kwargs
+        self._params_type_memo = {}
+        self._cached_codes = {}
+        names = [p.name for p in self.in_params + self.out_params]
+        if 'i' in names:
+            raise ValueError('Can not use \'i\' as a parameter name')
+        self._elementwise_kernel_memo = {}
+        # This is for profiling mechanisms to auto infer a name
+        self.__name__ = name
+
+    def __call__(self, *args, **kwargs):
+        """Compiles and invokes the elementwise kernel.
+
+        The compilation runs only if the kernel is not cached. Note that the
+        kernels with different argument dtypes or dimensions are not
+        compatible. It means that single ElementwiseKernel object may be
+        compiled into multiple kernel binaries.
+
+        Args:
+            args: Arguments of the kernel.
+            size (int): Range size of the indices.  By default, the range size
+                is automatically determined from the result of broadcasting.
+                This parameter must be specified if and only if all ndarrays
+                are `raw` and the range size cannot be determined
+                automatically.
+            block_size (int): Number of threads per block. By default, the
+                value is set to 128.
+
+        Returns:
+            If ``no_return`` has not set, arrays are returned according to the
+            ``out_params`` argument of the ``__init__`` method.
+            If ``no_return`` has set, ``None`` is returned.
+
+        """
+        cdef function.Function kern
+        cdef Py_ssize_t size, i
+        cdef list in_args, out_args
+        cdef tuple in_types, out_types
+        cdef shape_t shape
+
+        size = kwargs.pop('size', -1)
+        stream = kwargs.pop('stream', None)
+        block_size = kwargs.pop('block_size', 128)
+        if len(kwargs):
+            raise TypeError('Wrong arguments %s' % kwargs)
+        if block_size <= 0:
+            raise ValueError('block_size must be greater than zero')
+        n_args = len(args)
+        if n_args != self.nin and n_args != self.nargs:
+            raise TypeError(
+                'Wrong number of arguments for {!r}. '
+                'It must be either {} or {} (with outputs), '
+                'but given {}.'.format(
+                    self.name, self.nin, self.nargs, n_args))
+        for arg in args:
+            if hasattr(arg, '__cupy_override_elementwise_kernel__'):
+                return arg.__cupy_override_elementwise_kernel__(
+                    self, *args, **kwargs)
+        dev_id = device.get_device_id()
+        arg_list = _preprocess_args(dev_id, args, True)
+
+        out_args = arg_list[self.nin:]
+        # _broadcast updates shape
+        in_args = _broadcast(
+            arg_list, self.params, size != -1, shape)[:self.nin]
+
+        in_ndarray_types = []
+        for a in in_args:
+            if isinstance(a, _ndarray_base):
+                t = a.dtype.type
+            elif isinstance(a, texture.TextureObject):
+                t = 'cudaTextureObject_t'
+            else:
+                t = None
+            in_ndarray_types.append(t)
+        in_ndarray_types = tuple(in_ndarray_types)
+        out_ndarray_types = tuple([a.dtype.type for a in out_args])
+
+        in_types, out_types, type_map = self._decide_params_type(
+            in_ndarray_types, out_ndarray_types)
+
+        is_size_specified = False
+        if size != -1:
+            shape.assign(1, size)
+            is_size_specified = True
+
+        out_args = _get_out_args_with_params(
+            out_args, out_types, shape, self.out_params, is_size_specified)
+        if self.no_return:
+            ret = None
+        elif not self.return_tuple and self.nout == 1:
+            ret = out_args[0]
+        else:
+            ret = tuple(out_args)
+
+        if _contains_zero(shape):
+            return ret
+
+        for i, x in enumerate(in_args):
+            if type(x) is _scalar.CScalar:
+                (<_scalar.CScalar>x).apply_dtype(in_types[i])
+
+        inout_args = in_args + out_args
+
+        if self.reduce_dims:
+            shape = _reduce_dims(inout_args, self.params, shape)
+        indexer = _carray._indexer_init(shape)
+        inout_args.append(indexer)
+
+        arginfos = _get_arginfos(inout_args)
+        kern = self._get_elementwise_kernel(dev_id, arginfos, type_map)
+        kern.linear_launch(indexer.size, inout_args, shared_mem=0,
+                           block_max_size=block_size, stream=stream)
+        return ret
+
+    cpdef tuple _decide_params_type(
+            self, tuple in_args_dtype, tuple out_args_dtype):
+        key = (in_args_dtype, out_args_dtype)
+        ret = self._params_type_memo.get(key, None)
+        if ret is not None:
+            return ret
+        ret = _decide_params_type_core(
+            self.in_params, self.out_params, in_args_dtype, out_args_dtype)
+        self._params_type_memo[key] = ret
+        return ret
+
+    cpdef function.Function _get_elementwise_kernel(
+            self, int dev_id, tuple arginfos, _TypeMap type_map):
+        key = (
+            dev_id,
+            arginfos,
+            type_map)
+        kern = self._elementwise_kernel_memo.get(key, None)
+        if kern is not None:
+            return kern
+        kern = _get_elementwise_kernel(
+            arginfos, type_map, self.params, self.operation,
+            self.name, self.preamble, **self.kwargs)
+
+        # Store the compiled kernel in the cache.
+        # Potentially overwrite a duplicate cache entry because
+        # _get_elementwise_kernel() may include IO wait.
+        in_types = []
+        for x in arginfos:
+            if x.type is cupy.ndarray:
+                in_types.append(cupy.dtype(x.dtype).char)
+        in_types = tuple(in_types)
+        if in_types not in self._cached_codes:
+            code = _get_elementwise_kernel_code(
+                arginfos, type_map, self.params, self.operation,
+                self.name, self.preamble, **self.kwargs)
+            self._cached_codes[in_types] = code
+        self._elementwise_kernel_memo[key] = kern
+        return kern
+
+    @property
+    def cached_codes(self):
+        """Returns a dict that has input types as keys and codes values.
+
+        This proprety method is for debugging purpose.
+        The return value is not guaranteed to keep backward compatibility.
+        """
+        if len(self._cached_codes) == 0:
+            warnings.warn(
+                'No codes are cached because compilation is deferred until '
+                'the first function call.')
+        return dict([(k, v) for k, v in self._cached_codes.items()])
+
+    @property
+    def cached_code(self):
+        """Returns `next(iter(self.cached_codes.values()))`.
+
+        This proprety method is for debugging purpose.
+        The return value is not guaranteed to keep backward compatibility.
+        """
+        codes = self._cached_codes
+        if len(codes) > 1:
+            warnings.warn(
+                'The input types of the kernel could not be inferred. '
+                'Please use `.cached_codes` instead.')
+        return next(iter(codes.values()))
+
+
+cdef str fix_cast_expr(src_type, dst_type, str expr):
+    src_kind = get_dtype(src_type).kind
+    dst_kind = get_dtype(dst_type).kind
+    if src_kind == dst_kind:
+        return expr
+    if src_kind == 'b':
+        # HIP has an issue with bool conversions detailed below
+        if runtime._is_hip_environment:
+            return f'_hip_bool_cast({expr})'
+        else:
+            return f'({expr}) ? 1 : 0'
+    if src_kind == 'c':
+        if dst_kind == 'b':
+            return f'({expr}) != {_scalar.get_typename(src_type)}()'
+        else:  # dst_kind in 'iuf' (int, uint, float)
+            return f'({expr}).real()'
+    return expr
+
+
+cdef function.Function _get_ufunc_kernel(
+        tuple in_types, tuple out_types, routine, tuple arginfos,
+        bint has_where, params,
+        name, preamble, loop_prep):
+    cdef _ArgInfo arginfo
+    cdef str str_type, str_var
+
+    offset_where = len(in_types)
+    offset_out = offset_where
+    if has_where:
+        offset_out += 1
+
+    types = []
+    op = []
+    if has_where:
+        arginfo = arginfos[offset_where]
+        if arginfo.is_ndarray():
+            op.append('if(!_raw__where[_ind.get()]) continue;')
+        else:
+            op.append('if(!_where) continue;')
+    for i, x in enumerate(in_types):
+        str_var = 'in%d' % i
+        str_type = str_var + '_type'
+        types.append((str_type, x))
+        arginfo = arginfos[i]
+        if arginfo.is_ndarray():
+            op.append('const {} {}({});'.format(
+                str_type,
+                str_var,
+                fix_cast_expr(arginfo.dtype, x, f'_raw_{str_var}[_ind.get()]')
+            ))
+
+    out_op = []
+    for i, x in enumerate(out_types):
+        str_var = 'out%d' % i
+        str_type = str_var + '_type'
+        types.append((str_type, x))
+        arginfo = arginfos[i + offset_out]
+        op.append(f'{str_type} {str_var};')
+        out_op.append('{} = {};'.format(
+            f'_raw_{str_var}[_ind.get()]',
+            fix_cast_expr(x, arginfo.dtype, str_var)
+        ))
+
+    type_map = _TypeMap(tuple(types))
+
+    op.append(routine)
+    op.append(';')
+    op.extend(out_op)
+    operation = '\n'.join(op)
+    # HIP/ROCm 4.3 has an issue with ifs and ternary operators
+    #
+    # int bool(int x) {
+    #     if (x != 0) return 1;
+    #     return 0;
+    # }
+    #
+    # bool(5) == 1;  //false
+    # bool(5) == 5;  //true
+    #
+    # also it simplifies  (a ? 1 : 0)  directly to a, and yields
+    # an incorrect value
+    if runtime._is_hip_environment:
+        preamble += """
+        __device__ int _hip_bool_cast(long long int x) {
+            volatile int a = 1;
+            if (x == 0) a = 0;
+            return a;
+        }
+        """
+    return _get_simple_elementwise_kernel(
+        params, arginfos, operation, name, type_map, preamble,
+        loop_prep=loop_prep)
+
+
+cdef inline bint _check_should_use_min_scalar(list in_args) except? -1:
+    cdef int kind, max_array_kind, max_scalar_kind
+    cdef bint all_scalars
+    all_scalars = True
+    max_array_kind = -1
+    max_scalar_kind = -1
+    for i in in_args:
+        kind = _get_kind_score(ord(i.dtype.kind))
+        if isinstance(i, _ndarray_base):
+            all_scalars = False
+            max_array_kind = max(max_array_kind, kind)
+        else:
+            max_scalar_kind = max(max_scalar_kind, kind)
+    return (max_scalar_kind != -1 and
+            not all_scalars and
+            max_array_kind >= max_scalar_kind)
+
+
+cdef dict _mst_unsigned_to_signed = {
+    i: (numpy.iinfo(j).max, (i, j))
+    for i, j in [(numpy.dtype(i).type, numpy.dtype(i.lower()).type)
+                 for i in "BHILQ"]}
+cdef _numpy_min_scalar_type = numpy.min_scalar_type
+
+cdef _min_scalar_type(x):
+    # A non-negative integer may have two locally minimum scalar
+    # types: signed/unsigned integer.
+    # Return both for can_cast, while numpy.min_scalar_type only returns
+    # the unsigned type.
+    t = _numpy_min_scalar_type(x)
+    dt = t.type
+    if t.kind == 'u':
+        m, dt2 = <tuple>_mst_unsigned_to_signed[dt]
+        if x <= m:
+            return dt2
+    return dt
+
+
+cdef class ufunc:
+
+    """Universal function.
+
+    Attributes:
+        ~ufunc.name (str): The name of the universal function.
+        ~ufunc.nin (int): Number of input arguments.
+        ~ufunc.nout (int): Number of output arguments.
+        ~ufunc.nargs (int): Number of all arguments.
+
+    """
+
+    cdef:
+        readonly Py_ssize_t nin
+        readonly Py_ssize_t nout
+        readonly Py_ssize_t nargs
+        readonly object name
+        readonly _Ops _ops  # normal routines
+        # routines based on explicitly given output dtype
+        readonly _Ops _out_ops
+        readonly object _preamble
+        readonly object _loop_prep
+        readonly object _default_casting
+        readonly object _cutensor_op
+        readonly int _cutensor_alpha
+        readonly int _cutensor_gamma
+        readonly str _scatter_op
+        readonly tuple _params
+        readonly tuple _params_with_where
+        readonly dict _routine_cache
+        readonly dict _kernel_memo
+        readonly object __doc__
+        readonly object __name__
+        readonly object __module__
+
+    def __init__(
+            self, name, nin, nout, _Ops ops, preamble='', loop_prep='', doc='',
+            default_casting=None, *, _Ops out_ops=None, cutensor_op=None,
+            scatter_op=None):
+        self.name = name
+        self.__name__ = name
+        self.nin = nin
+        self.nout = nout
+        self.nargs = nin + nout
+        self._ops = ops
+        self._out_ops = out_ops
+        self._preamble = preamble
+        self._loop_prep = loop_prep
+        self.__doc__ = doc
+        if default_casting is None:
+            self._default_casting = 'same_kind'
+        else:
+            self._default_casting = default_casting
+        if cutensor_op is not None and cuda_cutensor is not None:
+            self._cutensor_op, self._cutensor_alpha, self._cutensor_gamma = (
+                getattr(cuda_cutensor, cutensor_op[0]),
+                cutensor_op[1], cutensor_op[2])
+        self._scatter_op = scatter_op
+
+        _in_params = tuple(
+            ParameterInfo('T in%d' % i, True)
+            for i in range(nin))
+        _out_params = tuple(
+            ParameterInfo('T out%d' % i, False)
+            for i in range(nout))
+        _other_params = (
+            ParameterInfo('CIndexer _ind', False),)
+        self._params = _in_params + _out_params + _other_params
+        self._params_with_where = (
+            _in_params + (ParameterInfo('T _where', False),)
+            + _out_params + _other_params)
+        self._routine_cache = {}
+        self._kernel_memo = {}
+
+    def __repr__(self):
+        return '<ufunc \'%s\'>' % self.name
+
+    @property
+    def types(self):
+        """A list of type signatures.
+
+        Each type signature is represented by type character codes of inputs
+        and outputs separated by '->'.
+
+        """
+        types = []
+        for op in self._ops.ops:
+            in_str = ''.join([<str>get_dtype(t).char for t in op.in_types])
+            out_str = ''.join([<str>get_dtype(t).char for t in op.out_types])
+            types.append('%s->%s' % (in_str, out_str))
+        return types
+
+    def __call__(self, *args, **kwargs):
+        """Applies the universal function to arguments elementwise.
+
+        Args:
+            args: Input arguments. Each of them can be a :class:`cupy.ndarray`
+                object or a scalar. The output arguments can be omitted or be
+                specified by the ``out`` argument.
+            out (cupy.ndarray): Output array. It outputs to new arrays
+                default.
+            dtype: Data type specifier.
+
+        Returns:
+            Output array or a tuple of output arrays.
+
+        """
+        for arg in args:
+            if hasattr(arg, '__cupy_override_elementwise_kernel__'):
+                return arg.__cupy_override_elementwise_kernel__(
+                    self, *args, **kwargs)
+
+        if _fusion_thread_local.is_fusing():
+            return _fusion_thread_local.call_ufunc(self, *args, **kwargs)
+
+        cdef function.Function kern
+        cdef list broad_values
+        cdef shape_t shape
+
+        out = kwargs.pop('out', None)
+        where = kwargs.pop('_where', None)
+        cdef bint has_where = where is not None
+        dtype = kwargs.pop('dtype', None)
+        # Note default behavior of casting is 'same_kind' on numpy>=1.10
+        casting = kwargs.pop('casting', self._default_casting)
+        if dtype is not None:
+            dtype = get_dtype(dtype).type
+        if kwargs:
+            raise TypeError('Wrong arguments %s' % kwargs)
+
+        n_args = len(args)
+        if not (self.nin <= n_args <= self.nargs):
+            # TODO(kataoka): Fix error message for nout >= 2 (e.g. divmod)
+            raise TypeError(
+                'Wrong number of arguments for {!r}. '
+                'It must be either {} or {} (with outputs), '
+                'but given {}.'.format(
+                    self.name, self.nin, self.nargs, n_args))
+
+        # parse inputs (positional) and outputs (positional or keyword)
+        in_args = args[:self.nin]
+        out_args = args[self.nin:]
+        if out is not None:
+            if out_args:
+                raise ValueError('Cannot specify \'out\' as both '
+                                 'a positional and keyword argument')
+            if isinstance(out, tuple):
+                if len(out) != self.nout:
+                    raise ValueError(
+                        "The 'out' tuple must have exactly one entry per "
+                        "ufunc output")
+                out_args = out
+            else:
+                if 1 != self.nout:
+                    raise ValueError("'out' must be a tuple of arrays")
+                out_args = out,
+
+        dev_id = device.get_device_id()
+        in_args = _preprocess_args(dev_id, in_args, False)
+        out_args = _preprocess_optional_args(dev_id, out_args, False)
+        given_out_args = [o for o in out_args if o is not None]
+
+        # TODO(kataoka): Typecheck `in_args` w.r.t. `casting` (before
+        # broadcast).
+        if has_where:
+            where_args = _preprocess_args(dev_id, (where,), False)
+            x = where_args[0]
+            if isinstance(x, _ndarray_base):
+                # NumPy seems using casting=safe here
+                if x.dtype != bool:
+                    raise TypeError(
+                        f'Cannot cast array data from {x.dtype!r} to '
+                        f'{get_dtype(bool)!r} according to the rule \'safe\'')
+            else:
+                # NumPy does not seem raising TypeError.
+                # CuPy does not have to support `where=object()` etc. and
+                # `_preprocess_args` rejects it anyway.
+                where_args[0] = _scalar.CScalar.from_numpy_scalar_with_dtype(
+                    x, numpy.bool_)
+        else:
+            where_args = []
+
+        # _copy_in_args_if_needed updates in_args
+        _copy_in_args_if_needed(in_args, given_out_args)
+        _copy_in_args_if_needed(where_args, given_out_args)
+        broad_values = in_args + where_args + given_out_args
+        # _broadcast updates shape
+        internal._broadcast_core(broad_values, shape)
+
+        if (self._cutensor_op is not None
+                and _accelerator.ACCELERATOR_CUTENSOR in
+                _accelerator._elementwise_accelerators):
+            if (self.nin == 2 and self.nout == 1 and
+                    isinstance(in_args[0], _ndarray_base) and
+                    isinstance(in_args[1], _ndarray_base)):
+                import cupyx.cutensor
+                ret = cupyx.cutensor._try_elementwise_binary_routine(
+                    in_args[0], in_args[1], dtype,
+                    out_args[0] if len(out_args) == 1 else None,
+                    self._cutensor_op,
+                    self._cutensor_alpha,
+                    self._cutensor_gamma,
+                )
+                if ret is not None:
+                    return ret
+
+        op = self._ops.guess_routine(
+            self.name, self._routine_cache, in_args, dtype, self._out_ops)
+
+        # Determine a template object from which we initialize the output when
+        # inputs have subclass instances
+        def issubclass1(cls, classinfo):
+            return issubclass(cls, classinfo) and cls is not classinfo
+        subtype = cupy.ndarray
+        template = None
+        for in_arg in in_args:
+            in_arg_type = type(in_arg)
+            if issubclass1(in_arg_type, cupy.ndarray):
+                subtype = in_arg_type
+                template = in_arg
+                break
+
+        out_args = _get_out_args_from_optionals(
+            subtype, out_args, op.out_types, shape, casting, template)
+        if self.nout == 1:
+            ret = out_args[0]
+        else:
+            ret = tuple(out_args)
+
+        if _contains_zero(shape):
+            return ret
+
+        inout_args = []
+        for i, t in enumerate(op.in_types):
+            x = broad_values[i]
+            inout_args.append(
+                x if isinstance(x, _ndarray_base) else
+                _scalar.CScalar.from_numpy_scalar_with_dtype(x, t))
+        if has_where:
+            x = broad_values[self.nin]
+            inout_args.append(x)
+        inout_args.extend(out_args)
+        shape = _reduce_dims(inout_args, self._params, shape)
+        indexer = _carray._indexer_init(shape)
+        inout_args.append(indexer)
+        arginfos = _get_arginfos(inout_args)
+
+        kern = self._get_ufunc_kernel(dev_id, op, arginfos, has_where)
+
+        kern.linear_launch(indexer.size, inout_args)
+        return ret
+
+    cdef str _get_name_with_type(self, tuple arginfos, bint has_where):
+        cdef str name = self.name
+        if has_where:
+            name += '_where'
+        cdef _ArgInfo arginfo
+        inout_type_words = []
+        for arginfo in arginfos:
+            dtype = str(numpy.dtype(arginfo.dtype))
+            if arginfo.is_ndarray():
+                inout_type_words.append(dtype)
+            elif arginfo.is_scalar():
+                inout_type_words.append(dtype.rstrip('0123456789'))
+        return '{}__{}'.format(name, '_'.join(inout_type_words))
+
+    cdef function.Function _get_ufunc_kernel(
+            self, int dev_id, _Op op, tuple arginfos, bint has_where):
+        cdef function.Function kern
+        key = (dev_id, op, arginfos, has_where)
+        kern = self._kernel_memo.get(key, None)
+        if kern is None:
+            name = self._get_name_with_type(arginfos, has_where)
+            params = self._params_with_where if has_where else self._params
+            kern = _get_ufunc_kernel(
+                op.in_types, op.out_types, op.routine, arginfos, has_where,
+                params, name, self._preamble, self._loop_prep)
+            self._kernel_memo[key] = kern
+        return kern
+
+    def outer(self, A, B, **kwargs):
+        """Apply the ufunc operation to all pairs of elements in A and B.
+
+        .. seealso::
+           :meth:`numpy.ufunc.outer`
+
+        """
+        A = core.array(A)
+        B = core.array(B)
+        ndim_a = A.ndim
+        ndim_b = B.ndim
+        A = A.reshape(A.shape + (1,) * ndim_b)
+        B = B.reshape((1,) * ndim_a + B.shape)
+        return self(A, B, **kwargs)
+
+    def at(self, a, indices, b=None):
+        """Apply in place operation on the operand ``a`` for elements
+        specified by ``indices``.
+
+        .. seealso::
+           :meth:`numpy.ufunc.at`
+        """
+        if self._scatter_op is not None:
+            a._scatter_op(indices, b, self._scatter_op)
+        else:
+            raise NotImplementedError(f'`{self.name}.at` is not supported yet')
+
+    def reduce(self, array, axis=0, dtype=None, out=None, keepdims=False):
+        """Reduce ``array`` applying ufunc.
+
+        .. seealso::
+           :meth:`numpy.ufunc.reduce`
+        """
+        if self.name == 'cupy_add':
+            return array.sum(axis, dtype, out, keepdims)
+        if self.name == 'cupy_multiply':
+            return array.prod(axis, dtype, out, keepdims)
+        raise NotImplementedError(f'`{self.name}.reduce` is not supported yet')
+
+    def accumulate(self, array, axis=0, dtype=None, out=None):
+        """Accumulate ``array`` applying ufunc.
+
+        .. seealso::
+           :meth:`numpy.ufunc.accumulate`
+        """
+        if self.name == 'cupy_add':
+            return array.cumsum(axis, dtype, out)
+        if self.name == 'cupy_multiply':
+            return array.cumprod(axis, dtype, out)
+        raise NotImplementedError(
+            f'`{self.name}.accumulate` is not supported yet')
+
+    def reduceat(self, array, indices, axis=0, dtype=None, out=None):
+        """Reduce ``array`` applying ufunc with indices.
+
+        .. seealso::
+           :meth:`numpy.ufunc.reduceat`
+        """
+        if self.name == 'cupy_add':
+            return array._add_reduceat(indices, axis, dtype, out)
+        raise NotImplementedError(
+            f'`{self.name}.reduceat` is not supported yet')
+
+
+cdef class _Op:
+
+    def __init__(
+            self, tuple in_types, tuple out_types, object routine,
+            object error_func):
+        if error_func is None:
+            assert routine is not None
+        else:
+            assert callable(error_func)
+        self.in_types = in_types
+        self.out_types = out_types
+        self.nin = len(in_types)
+        self.nout = len(out_types)
+        self.routine = routine
+        self.error_func = error_func
+
+    @staticmethod
+    cdef _Op _from_type_and_routine_or_error_func(
+            str typ, object routine, object error_func):
+        # TODO(niboshi): Write type mapping specification.
+        types = typ.split('->')
+        if len(types) == 1:
+            in_types = out_types = tuple(types)
+        else:
+            in_types, out_types = map(tuple, types)
+        in_types = tuple([get_dtype(t).type for t in in_types])
+        out_types = tuple([get_dtype(t).type for t in out_types])
+        return _Op(in_types, out_types, routine, error_func)
+
+    @staticmethod
+    cdef _Op from_type_and_routine(str typ, routine):
+        return _Op._from_type_and_routine_or_error_func(typ, routine, None)
+
+    @staticmethod
+    cdef _Op from_type_and_error_func(str typ, error_func):
+        return _Op._from_type_and_routine_or_error_func(typ, None, error_func)
+
+    cdef check_valid(self):
+        if self.error_func is not None:
+            self.error_func()
+
+    cpdef tuple get_in_dtypes(self):
+        return tuple([get_dtype(t) for t in self.in_types])
+
+    cpdef tuple get_out_dtypes(self):
+        return tuple([get_dtype(t) for t in self.out_types])
+
+
+cdef class _Ops:
+
+    def __init__(self, tuple ops):
+        assert len(ops) > 0
+        nin = ops[0].nin
+        nout = ops[0].nout
+        assert all(op.nin == nin for op in ops)
+        assert all(op.nout == nout for op in ops)
+        self.ops = ops
+        self.nin = nin
+        self.nout = nout
+
+    @staticmethod
+    cdef _Ops from_tuples(object ops, routine):
+        ops_ = []
+        for t in ops:
+            if isinstance(t, tuple):
+                typ, rt = t
+                if isinstance(rt, tuple):
+                    rt = tuple([r1 or r2 for r1, r2 in zip(rt, routine)])
+                elif not isinstance(rt, str):
+                    assert callable(rt)
+                    ops_.append(_Op.from_type_and_error_func(typ, rt))
+                    continue
+            else:
+                assert isinstance(t, str)
+                typ, rt = t, routine
+            ops_.append(_Op.from_type_and_routine(typ, rt))
+        return _Ops(tuple(ops_))
+
+    cpdef _Op guess_routine(
+            self, str name, dict cache, list in_args, dtype, _Ops out_ops):
+        cdef _Ops ops_
+        if dtype is None:
+            use_raw_value = _check_should_use_min_scalar(in_args)
+            if use_raw_value:
+                in_types = tuple([
+                    a.dtype.type if isinstance(a, _ndarray_base)
+                    else _min_scalar_type(a)
+                    for a in in_args])
+            else:
+                in_types = tuple([a.dtype.type for a in in_args])
+            op = cache.get(in_types, ())
+            if op is ():
+                op = self._guess_routine_from_in_types(in_types)
+                cache[in_types] = op
+        else:
+            op = cache.get(dtype, ())
+            if op is ():
+                ops_ = out_ops or self
+                op = ops_._guess_routine_from_dtype(dtype)
+                cache[dtype] = op
+
+        if op is not None:
+            # raise TypeError if the type combination is disallowed
+            (<_Op>op).check_valid()
+
+            return op
+
+        if dtype is None:
+            dtype = tuple([a.dtype.type for a in in_args])
+        raise TypeError('Wrong type (%s) of arguments for %s' %
+                        (dtype, name))
+
+    cpdef _Op _guess_routine_from_in_types(
+            self, tuple in_types, object can_cast=_numpy_can_cast):
+        cdef _Op op
+        cdef tuple op_types
+        cdef Py_ssize_t n = len(in_types)
+        cdef Py_ssize_t i
+        for op in self.ops:
+            op_types = op.in_types
+            for i in range(n):
+                it = in_types[i]
+                ot = op_types[i]
+                if isinstance(it, tuple):
+                    if not can_cast(it[0], ot) and not can_cast(it[1], ot):
+                        break
+                elif not can_cast(it, ot):
+                    break
+            else:
+                return op
+        return None
+
+    cpdef _Op _guess_routine_from_dtype(self, object dtype):
+        cdef _Op op
+        cdef tuple op_types
+        for op in self.ops:
+            op_types = op.out_types
+            for t in op_types:
+                if t != dtype:
+                    break
+            else:
+                return op
+        return None
+
+
+cpdef create_ufunc(name, ops, routine=None, preamble='', doc='',
+                   default_casting=None, loop_prep='', out_ops=None,
+                   cutensor_op=None, scatter_op=None):
+    ops_ = _Ops.from_tuples(ops, routine)
+    _out_ops = None if out_ops is None else _Ops.from_tuples(out_ops, routine)
+    return ufunc(
+        name, ops_.nin, ops_.nout, ops_, preamble,
+        loop_prep, doc, default_casting=default_casting, out_ops=_out_ops,
+        cutensor_op=cutensor_op, scatter_op=scatter_op)
diff --git a/cupy/_core/_memory_range.pxd b/cupy/_core/_memory_range.pxd
new file mode 100644
index 0000000..d359b0e
--- /dev/null
+++ b/cupy/_core/_memory_range.pxd
@@ -0,0 +1,7 @@
+from cupy._core.core cimport _ndarray_base
+
+from libcpp.pair cimport pair
+
+
+cpdef pair[Py_ssize_t, Py_ssize_t] get_bound(_ndarray_base array)
+cpdef bint may_share_bounds(_ndarray_base a, _ndarray_base b)
diff --git a/cupy/_core/_memory_range.pyx b/cupy/_core/_memory_range.pyx
new file mode 100644
index 0000000..db1a8b1
--- /dev/null
+++ b/cupy/_core/_memory_range.pyx
@@ -0,0 +1,40 @@
+from cupy._core.core cimport _ndarray_base
+from cupy.cuda cimport memory
+
+from libcpp.pair cimport pair
+
+
+cpdef pair[Py_ssize_t, Py_ssize_t] get_bound(_ndarray_base array):
+    cdef Py_ssize_t left = array.data.ptr
+    cdef Py_ssize_t right = left
+    cdef Py_ssize_t tmp
+    cdef pair[Py_ssize_t, Py_ssize_t] ret
+    cdef size_t i
+
+    for i in range(array._shape.size()):
+        # shape[i] != 0 is assumed
+        tmp = (array._shape[i] - 1) * array._strides[i]
+        if tmp > 0:
+            right += tmp
+        else:
+            left += tmp
+
+    ret.first = left
+    ret.second = right + <Py_ssize_t>array.dtype.itemsize
+    return ret
+
+
+cpdef bint may_share_bounds(_ndarray_base a, _ndarray_base b):
+    cdef memory.MemoryPointer a_data = a.data
+    cdef memory.MemoryPointer b_data = b.data
+    cdef pair[Py_ssize_t, Py_ssize_t] a_range, b_range
+
+    if (a_data.device_id != b_data.device_id
+            or a_data.mem.ptr != b_data.mem.ptr
+            or a.size == 0 or b.size == 0):
+        return False
+
+    a_range = get_bound(a)
+    b_range = get_bound(b)
+
+    return a_range.first < b_range.second and b_range.first < a_range.second
diff --git a/cupy/_core/_optimize_config.pxd b/cupy/_core/_optimize_config.pxd
new file mode 100644
index 0000000..070451f
--- /dev/null
+++ b/cupy/_core/_optimize_config.pxd
@@ -0,0 +1,22 @@
+cdef object _thread_local
+cdef dict _contexts
+
+
+cdef class _OptimizationConfig:
+
+    cdef readonly object optimize_impl
+    cdef readonly int max_trials
+    cdef readonly float timeout
+    cdef readonly float expected_total_time_per_trial
+    cdef readonly float max_total_time_per_trial
+
+
+cdef class _OptimizationContext:
+
+    cdef readonly str key
+    cdef readonly _OptimizationConfig config
+    cdef readonly dict _params_map
+    cdef readonly bint _dirty
+
+
+cpdef _OptimizationContext get_current_context()
diff --git a/cupy/_core/_optimize_config.pyx b/cupy/_core/_optimize_config.pyx
new file mode 100644
index 0000000..4b38b37
--- /dev/null
+++ b/cupy/_core/_optimize_config.pyx
@@ -0,0 +1,81 @@
+import pickle
+import threading
+
+
+cdef _thread_local = threading.local()
+cdef _contexts = {}
+
+
+cdef class _OptimizationConfig:
+
+    def __init__(
+            self, optimize_impl, *,
+            int max_trials=100,
+            float timeout=1,
+            float expected_total_time_per_trial=100 * 1e-6,
+            float max_total_time_per_trial=0.1):
+        self.optimize_impl = optimize_impl
+        self.max_trials = max_trials
+        self.timeout = timeout
+        self.expected_total_time_per_trial = expected_total_time_per_trial
+        self.max_total_time_per_trial = max_total_time_per_trial
+
+
+cdef class _OptimizationContext:
+
+    def __init__(self, str key, _OptimizationConfig config):
+        self.key = key
+        self.config = config
+        self._params_map = {}
+        self._dirty = False
+
+    def get_params(self, key):
+        return self._params_map.get(key)
+
+    def set_params(self, key, params):
+        self._params_map[key] = params
+        self._dirty = True
+
+    def save(self, filepath):
+        with open(filepath, mode='wb') as f:
+            pickle.dump((self.key, self._params_map), f)
+        self._dirty = False
+
+    def load(self, filepath):
+        with open(filepath, mode='rb') as f:
+            key, params_map = pickle.load(f)
+        if key != self.key:
+            raise ValueError(
+                'Optimization key mismatch {} != {}'.format(key, self.key))
+        self._params_map = params_map
+        self._dirty = False
+
+    def _is_dirty(self):
+        return self._dirty
+
+
+cpdef _OptimizationContext get_current_context():
+    try:
+        return _thread_local.current_context
+    except AttributeError:
+        return None
+
+
+def set_current_context(_OptimizationContext context):
+    _thread_local.current_context = context
+
+
+def get_new_context(
+        str key, object optimize_impl, dict config_dict):
+    c = _contexts.get(key)
+    if c is None:
+        config = _OptimizationConfig(optimize_impl, **config_dict)
+        c = _OptimizationContext(key, config)
+        _contexts[key] = c
+    return c
+
+
+def _clear_all_contexts_cache():
+    global _contexts
+    assert get_current_context() is None
+    _contexts = {}
diff --git a/cupy/_core/_reduction.pxd b/cupy/_core/_reduction.pxd
new file mode 100644
index 0000000..035ca13
--- /dev/null
+++ b/cupy/_core/_reduction.pxd
@@ -0,0 +1,77 @@
+from cupy._core._carray cimport shape_t
+from cupy._core cimport _kernel
+from cupy._core.core cimport _ndarray_base
+from cupy.cuda cimport function
+
+
+cdef Py_ssize_t _block_size
+
+
+cpdef tuple _get_axis(object axis, Py_ssize_t ndim)
+
+cpdef shape_t _get_out_shape(
+    const shape_t& shape, tuple reduce_axis, tuple out_axis, bint keepdims)
+
+
+cdef class _AbstractReductionKernel:
+
+    cdef:
+        readonly str name
+        public str identity
+        readonly tuple in_params
+        readonly tuple out_params
+        readonly tuple _params
+        readonly str __name__
+        readonly dict _cached_codes
+
+    cpdef _ndarray_base _call(
+        self,
+        list in_args, list out_args,
+        const shape_t& a_shape, axis, dtype,
+        bint keepdims, bint reduce_dims, int device_id,
+        stream, bint try_use_cub=*, bint sort_reduce_axis=*)
+
+    cdef void _launch(
+        self, out_block_num, block_size, block_stride,
+        in_args, out_args, in_shape, out_shape, types,
+        map_expr, reduce_expr, post_map_expr, reduce_type,
+        stream, params)
+
+    cdef tuple _get_expressions_and_types(
+        self, list in_args, list out_args, dtype)
+
+    cdef list _get_out_args(
+        self, list out_args, tuple out_types, const shape_t& out_shape)
+
+    cdef function.Function _get_function(
+        self,
+        tuple params, tuple arginfos, _kernel._TypeMap types,
+        str map_expr, str reduce_expr, str post_map_expr, str reduce_type,
+        Py_ssize_t block_size)
+
+
+cdef class ReductionKernel(_AbstractReductionKernel):
+
+    cdef:
+        readonly int nin
+        readonly int nout
+        readonly int nargs
+        readonly tuple params
+        readonly str reduce_expr
+        readonly str map_expr
+        readonly str post_map_expr
+        readonly object options
+        readonly bint reduce_dims
+        readonly object reduce_type
+        readonly str preamble
+
+
+cdef shape_t _set_permuted_args(
+    list args, tuple axis_permutes, const shape_t& shape, tuple params)
+
+cdef tuple _get_shape_and_strides(list in_args, list out_args)
+
+cdef _optimizer_copy_arg(a)
+
+cpdef create_reduction_func(
+    name, ops, routine=*, identity=*, preamble=*, sort_reduce_axis=*)
diff --git a/cupy/_core/_reduction.pyx b/cupy/_core/_reduction.pyx
new file mode 100644
index 0000000..0046ce8
--- /dev/null
+++ b/cupy/_core/_reduction.pyx
@@ -0,0 +1,906 @@
+from cpython cimport sequence
+
+from cupy._core cimport _carray
+from cupy._core cimport _accelerator
+from cupy._core._carray cimport shape_t
+from cupy._core cimport _cub_reduction
+from cupy._core._dtype cimport get_dtype
+from cupy._core cimport _kernel
+from cupy._core._kernel cimport _broadcast
+from cupy._core._kernel cimport _check_peer_access
+from cupy._core._kernel cimport _get_arginfos
+from cupy._core._kernel cimport _get_out_args_from_optionals
+from cupy._core._kernel cimport _get_out_args_with_params
+from cupy._core._kernel cimport _preprocess_args
+from cupy._core._kernel cimport _reduce_dims
+from cupy._core._kernel cimport ParameterInfo, _ArgInfo
+from cupy._core cimport _optimize_config
+from cupy._core cimport _routines_manipulation as _manipulation
+from cupy._core cimport _scalar
+from cupy._core._scalar import get_typename as _get_typename
+from cupy._core.core cimport _convert_object_with_cuda_array_interface
+from cupy._core.core cimport _create_ndarray_from_shape_strides
+from cupy._core.core cimport compile_with_cache
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport internal
+from cupy.cuda cimport device
+from cupy.cuda cimport function
+from cupy_backends.cuda.api cimport runtime
+
+import math
+import string
+import warnings
+
+import numpy
+
+import cupy
+from cupy._core._kernel import _get_param_info
+from cupy._core._kernel import _decide_params_type
+from cupy._core._ufuncs import elementwise_copy
+from cupy.cuda import compiler
+from cupy import _util
+
+
+cpdef str _create_reduction_function_code(
+        name, block_size, reduce_type, params, arginfos, identity,
+        pre_map_expr, reduce_expr, post_map_expr,
+        _kernel._TypeMap type_map, input_expr, output_expr, preamble, options):
+    # A (incomplete) list of internal variables:
+    # _J            : the index of an element in the array
+    # _block_size   : the number of threads in a block; should be power of 2
+    # _block_stride : the number of elements being processed by a block; should
+    #                 be power of 2 and <= _block_size
+
+    module_code = string.Template('''
+${type_preamble}
+${preamble}
+#define REDUCE(a, b) (${reduce_expr})
+#define POST_MAP(a) (${post_map_expr})
+#define _REDUCE(_offset) if (_tid < _offset) { \
+  _type_reduce _a = _sdata[_tid], _b = _sdata[(_tid + _offset)]; \
+  _sdata[_tid] = REDUCE(_a, _b); \
+}
+
+typedef ${reduce_type} _type_reduce;
+extern "C" __global__ void ${name}(${params}) {
+  __shared__ char _sdata_raw[${block_size} * sizeof(_type_reduce)];
+  _type_reduce *_sdata = reinterpret_cast<_type_reduce*>(_sdata_raw);
+  unsigned int _tid = threadIdx.x;
+
+  int _J_offset = _tid >> __popc(_block_stride - 1);  // _tid / _block_stride
+  ptrdiff_t _j_offset = (ptrdiff_t)_J_offset * _out_ind.size();
+  int _J_stride = ${block_size} >> __popc(_block_stride - 1);
+  ptrdiff_t _j_stride = (ptrdiff_t)_J_stride * _out_ind.size();
+
+  for (ptrdiff_t _i_base = (ptrdiff_t)blockIdx.x * _block_stride;
+       _i_base < _out_ind.size();
+       _i_base += (ptrdiff_t)gridDim.x * _block_stride) {
+    _type_reduce _s = _type_reduce(${identity});
+    ptrdiff_t _i =
+        _i_base + (_tid & (_block_stride - 1));  // _tid % _block_stride
+    int _J = _J_offset;
+    for (ptrdiff_t _j = _i + _j_offset; _j < _in_ind.size();
+         _j += _j_stride, _J += _J_stride) {
+      _in_ind.set(_j);
+      ${input_expr}
+      _type_reduce _a = static_cast<_type_reduce>(${pre_map_expr});
+      _s = REDUCE(_s, _a);
+    }
+    _sdata[_tid] = _s;
+    __syncthreads();
+    for (unsigned int _block = ${block_size} / 2;
+         _block >= _block_stride; _block >>= 1) {
+      if (_tid < _block) {
+        _REDUCE(_block);
+      }
+      __syncthreads();
+    }
+    if (_tid < _block_stride) {
+      _s = _sdata[_tid];
+    }
+    if (_tid < _block_stride && _i < _out_ind.size()) {
+      _out_ind.set(static_cast<ptrdiff_t>(_i));
+      ${output_expr}
+      POST_MAP(_s);
+    }
+  }
+}''').substitute(
+        name=name,
+        block_size=block_size,
+        reduce_type=reduce_type,
+        params=_kernel._get_kernel_params(params, arginfos),
+        identity=identity,
+        reduce_expr=reduce_expr,
+        pre_map_expr=pre_map_expr,
+        post_map_expr=post_map_expr,
+        type_preamble=type_map.get_typedef_code(),
+        input_expr=input_expr,
+        output_expr=output_expr,
+        preamble=preamble)
+    return module_code
+
+
+cpdef function.Function _create_reduction_function_from_code(
+        name, code, options):
+    module = compile_with_cache(code, options)
+    return module.get_function(name)
+
+
+cpdef function.Function _create_reduction_function(
+        name, block_size, reduce_type, params, arginfos, identity,
+        pre_map_expr, reduce_expr, post_map_expr,
+        _kernel._TypeMap type_map, input_expr, output_expr, preamble, options):
+    code = _create_reduction_function_code(
+        name, block_size, reduce_type, params, arginfos, identity,
+        pre_map_expr, reduce_expr, post_map_expr, type_map, input_expr,
+        output_expr, preamble, options
+    )
+    return _create_reduction_function_from_code(name, code, options)
+
+
+cpdef tuple _get_axis(object axis, Py_ssize_t ndim):
+    cdef Py_ssize_t dim
+    if axis is None:
+        return (tuple(range(ndim)), ())
+    elif sequence.PySequence_Check(axis):
+        axis = tuple(axis)
+    else:
+        axis = axis,
+
+    reduce_axis = tuple(sorted(
+        [internal._normalize_axis_index(dim, ndim) for dim in axis]))
+    out_axis = tuple([dim for dim in range(ndim) if dim not in reduce_axis])
+    if len(reduce_axis) + len(out_axis) != ndim:
+        raise ValueError("duplicate value in 'axis'")
+    return reduce_axis, out_axis
+
+
+cpdef shape_t _get_out_shape(
+        const shape_t& shape, tuple reduce_axis, tuple out_axis,
+        bint keepdims):
+    cdef shape_t out_shape
+    if keepdims:
+        out_shape = shape
+        for i in reduce_axis:
+            out_shape[i] = 1
+    else:
+        out_shape.reserve(len(out_axis))
+        for i in out_axis:
+            out_shape.push_back(shape[i])
+    return out_shape
+
+
+cdef shape_t _set_permuted_args(
+        list args, tuple axis_permutes, const shape_t& shape, tuple params):
+    # This function updates `args`
+    cdef ParameterInfo p
+    cdef Py_ssize_t i, s
+    cdef bint need_permutation = False
+    cdef shape_t out_shape
+    for i, s in enumerate(axis_permutes):
+        if i != s:
+            need_permutation = True
+            break
+    if need_permutation:
+        for p in params:
+            if p.raw:
+                raise NotImplementedError('Illegal conditions')
+        for i, a in enumerate(args):
+            if isinstance(a, _ndarray_base):
+                args[i] = _manipulation._transpose(a, axis_permutes)
+        out_shape.reserve(len(axis_permutes))
+        for i in axis_permutes:
+            out_shape.push_back(shape[i])
+        return out_shape
+    else:
+        return shape
+
+
+cdef Py_ssize_t _get_contiguous_size(
+        list args, tuple params, list out_shape, Py_ssize_t ndim) except -1:
+    '''
+    get contiguous size in the *output* axis (not *reduce* axis!)
+    '''
+    cdef int i, j
+    cdef ParameterInfo p
+    cdef Py_ssize_t contiguous_size, tmp_contiguous_size, itemsize
+    out_ndim = len(out_shape)
+    contiguous_size = 1
+    for i, a in enumerate(args):
+        if not isinstance(a, _ndarray_base):
+            continue
+        p = params[i]
+        if p.raw:
+            continue
+        tmp_contiguous_size = 1
+        itemsize = a.dtype.itemsize
+        for j in range(out_ndim):
+            if a._strides[ndim-j-1] != tmp_contiguous_size * itemsize:
+                break
+            tmp_contiguous_size *= out_shape[out_ndim-j-1]
+        contiguous_size = max(contiguous_size, tmp_contiguous_size)
+    return contiguous_size
+
+
+cdef Py_ssize_t _default_block_size = (
+    256 if runtime._is_hip_environment else 512)
+cdef Py_ssize_t _min_block_size_log = 5
+cdef Py_ssize_t _max_block_size_log = (
+    8 if runtime._is_hip_environment else 9)
+
+
+cpdef (Py_ssize_t, Py_ssize_t, Py_ssize_t) _get_block_specs(  # NOQA
+        Py_ssize_t in_size, Py_ssize_t out_size,
+        Py_ssize_t contiguous_size,
+        Py_ssize_t block_size) except*:
+    cdef Py_ssize_t reduce_block_size, block_stride, out_block_num
+    if block_size == -1:
+        block_size = _default_block_size
+
+    reduce_block_size = max(1, in_size // out_size)
+    contiguous_size = min(contiguous_size, 32)
+    block_stride = max(contiguous_size, block_size // reduce_block_size)
+    block_stride = internal.clp2(block_stride // 2 + 1)  # floor
+    out_block_num = (out_size + block_stride - 1) // block_stride
+
+    return block_size, block_stride, out_block_num
+
+
+cdef tuple _sort_axis(tuple axis, tuple strides):
+    # Sorts axis in the decreasing order of absolute values of strides.
+    return tuple(sorted(axis, key=lambda i: -abs(strides[i])))
+
+
+cdef tuple _get_shape_and_strides(list in_args, list out_args):
+    cdef list shape_and_strides = []
+    for x in in_args + out_args:
+        if isinstance(x, _ndarray_base):
+            shape_and_strides.append(x.shape)
+            shape_and_strides.append(x.strides)
+        else:
+            shape_and_strides.append(None)
+            shape_and_strides.append(None)
+    return tuple(shape_and_strides)
+
+
+cdef _optimizer_copy_arg(a):
+    if isinstance(a, _ndarray_base):
+        x = _create_ndarray_from_shape_strides(
+            cupy.ndarray, a._shape, a._strides, a.dtype, None)
+        assert a.data.device_id == x.data.device_id
+        elementwise_copy(a, x)
+        return x
+    return a
+
+
+cdef class _AbstractReductionKernel:
+
+    def __init__(
+            self, str name, str identity, str in_params, str out_params):
+        assert name is not None
+        assert identity is not None
+        assert in_params is not None
+        assert out_params is not None
+
+        in_params_ = _get_param_info(in_params, True)
+        out_params_ = _get_param_info(out_params, False)
+        params = (
+            in_params_
+            + out_params_
+            + _get_param_info('CIndexer _in_ind, CIndexer _out_ind', False)
+            + _get_param_info('int32 _block_stride', True))
+
+        self.name = name
+        self.identity = identity
+        self.in_params = in_params_
+        self.out_params = out_params_
+        self._params = params
+        # This is for profiling mechanisms to auto infer a name
+        self.__name__ = name
+        self._cached_codes = {}
+
+    cpdef _ndarray_base _call(
+            self,
+            list in_args, list out_args,
+            const shape_t& a_shape, axis, dtype,
+            bint keepdims, bint reduce_dims, int device_id,
+            stream, bint try_use_cub=False, bint sort_reduce_axis=True):
+        cdef tuple reduce_axis, out_axis, axis_permutes
+        cdef tuple params, opt_params
+        cdef tuple shape_and_strides
+        cdef Py_ssize_t contiguous_size = -1
+        cdef Py_ssize_t block_size, block_stride, out_block_num = 0
+        cdef shape_t in_shape, out_shape
+        cdef _ndarray_base ret
+        cdef bint cub_success
+
+        if dtype is not None:
+            dtype = get_dtype(dtype).type
+
+        (
+            map_expr, reduce_expr, post_map_expr,
+            in_types, out_types, reduce_type,
+            type_map,
+        ) = self._get_expressions_and_types(in_args, out_args, dtype)
+
+        reduce_axis, out_axis = _get_axis(axis, a_shape.size())
+
+        # When there is only one input array, sort the axes in such a way that
+        # contiguous (C or F) axes can be squashed in _reduce_dims() later.
+        # TODO(niboshi): Support (out_axis) > 1
+        if (len(in_args) == 1
+                and len(out_axis) <= 1
+                and not in_args[0]._c_contiguous):
+            strides = in_args[0].strides
+            if sort_reduce_axis:
+                reduce_axis = _sort_axis(reduce_axis, strides)
+            out_axis = _sort_axis(out_axis, strides)
+
+        out_shape = _get_out_shape(a_shape, reduce_axis, out_axis, keepdims)
+        out_args = self._get_out_args(out_args, out_types, out_shape)
+        ret = out_args[0]
+        if ret.size == 0:
+            return ret
+
+        if self.identity == '' and internal.is_in(a_shape, 0):
+            raise ValueError(('zero-size array to reduction operation'
+                              ' %s which has no identity') % self.name)
+
+        in_args = [x if isinstance(x, _ndarray_base) else
+                   _scalar.CScalar.from_numpy_scalar_with_dtype(x, t)
+                   for x, t in zip(in_args, in_types)]
+
+        optimize_context = _optimize_config.get_current_context()
+        key = ()
+        if optimize_context is not None:
+            # Calculate a key unique to the reduction setting.
+            shape_and_strides = _get_shape_and_strides(in_args, out_args)
+            key = (self.name, shape_and_strides,
+                   in_types, out_types, reduce_type, device_id)
+
+        # Try to use CUB
+        for accelerator in _accelerator._reduction_accelerators:
+            if try_use_cub and accelerator == _accelerator.ACCELERATOR_CUB:
+                cub_success = _cub_reduction._try_to_call_cub_reduction(
+                    self, in_args, out_args, a_shape, stream, optimize_context,
+                    key, map_expr, reduce_expr, post_map_expr, reduce_type,
+                    type_map, reduce_axis, out_axis, out_shape, ret)
+                if cub_success:
+                    return ret
+
+        axis_permutes = reduce_axis + out_axis
+        in_shape = _set_permuted_args(
+            in_args, axis_permutes, a_shape, self.in_params)
+
+        if reduce_dims:
+            in_shape = _reduce_dims(in_args, self.in_params, in_shape)
+            out_shape = _reduce_dims(out_args, self.out_params, out_shape)
+
+        params = self._params
+
+        # Calculate the reduction block dimensions.
+        if optimize_context is None:
+            # Calculate manually
+            contiguous_size = _get_contiguous_size(
+                in_args, self.in_params, out_shape, in_shape.size())
+            block_size, block_stride, out_block_num = _get_block_specs(
+                internal.prod(in_shape),
+                internal.prod(out_shape),
+                contiguous_size, -1)
+        else:
+            # Optimize dynamically
+            key = ('simple_reduction',) + key
+            opt_params = optimize_context.get_params(key)
+            if opt_params is None:
+                opt_params = self._get_optimized_params(
+                    optimize_context.config, in_args, out_args,
+                    in_shape, out_shape, type_map, map_expr, reduce_expr,
+                    post_map_expr, reduce_type, stream)
+                optimize_context.set_params(key, opt_params)
+            block_size, block_stride, out_block_num = opt_params
+
+        # Launch the kernel
+        self._launch(
+            out_block_num,
+            block_size,
+            block_stride,
+            in_args, out_args,
+            in_shape, out_shape,
+            type_map,
+            map_expr, reduce_expr, post_map_expr, reduce_type,
+            stream, params)
+
+        return ret
+
+    def _get_optimized_params(
+            self, optimize_config, in_args, out_args, in_shape, out_shape,
+            type_map, map_expr, reduce_expr, post_map_expr, reduce_type,
+            stream):
+        out_size = internal.prod(out_shape)
+        in_args = [_optimizer_copy_arg(a) for a in in_args]
+        out_args = [_optimizer_copy_arg(a) for a in out_args]
+
+        contiguous_size = _get_contiguous_size(
+            in_args, self.in_params, out_shape, len(in_shape))
+        block_size, block_stride, default_out_block_num = _get_block_specs(
+            internal.prod(in_shape),
+            internal.prod(out_shape),
+            contiguous_size, -1)
+        default_block_size_log = math.floor(math.log2(block_size))
+        default_block_stride_log = math.floor(math.log2(block_stride))
+
+        def target_func(block_size, block_stride, out_block_num):
+            self._launch(
+                out_block_num, block_size, block_stride, in_args, out_args,
+                in_shape, out_shape, type_map, map_expr, reduce_expr,
+                post_map_expr, reduce_type, stream, self._params)
+
+        def suggest_func(trial):
+            block_size_log = trial.suggest_int(
+                'block_size_log', _min_block_size_log, _max_block_size_log)
+            block_size = 2 ** block_size_log
+            block_stride_log = trial.suggest_int(
+                'block_stride_log', 0, block_size_log)
+            block_stride = 2 ** block_stride_log
+            max_out_block_num = (out_size + block_stride - 1) // block_stride
+            out_block_num = trial.suggest_int(
+                'out_block_num', 1, max_out_block_num)
+
+            trial.set_user_attr('block_size', block_size)
+            trial.set_user_attr('block_stride', block_stride)
+            return block_size, block_stride, out_block_num
+
+        optimize_impl = optimize_config.optimize_impl
+        best = optimize_impl(
+            optimize_config, target_func, suggest_func,
+            default_best={
+                'block_size_log': default_block_size_log,
+                'block_stride_log': default_block_stride_log,
+                'out_block_num': default_out_block_num,
+            }
+        )
+        return (
+            best.user_attrs['block_size'],
+            best.user_attrs['block_stride'],
+            best.params['out_block_num'])
+
+    cdef inline void _launch(
+            self, out_block_num, block_size, block_stride,
+            in_args, out_args, in_shape, out_shape, type_map,
+            map_expr, reduce_expr, post_map_expr, reduce_type,
+            stream, params):
+        cdef function.Function func
+
+        inout_args = (
+            in_args
+            + out_args
+            + [
+                _carray._indexer_init(in_shape),
+                _carray._indexer_init(out_shape),
+                # block_stride is passed as the last argument.
+                _scalar.CScalar.from_int32(block_stride),
+            ])
+
+        # Retrieve the kernel function
+        func = self._get_function(
+            params,
+            _get_arginfos(inout_args),
+            type_map,
+            map_expr, reduce_expr, post_map_expr, reduce_type,
+            block_size)
+
+        # Launch the kernel
+        func.linear_launch(
+            out_block_num * block_size, inout_args, 0, block_size, stream)
+
+    cdef tuple _get_expressions_and_types(
+            self, list in_args, list out_args, dtype):
+        raise NotImplementedError()
+
+    cdef list _get_out_args(
+            self, list out_args, tuple out_types, const shape_t& out_shape):
+        raise NotImplementedError()
+
+    cdef function.Function _get_function(
+            self,
+            tuple params, tuple arginfos, _kernel._TypeMap type_map,
+            str map_expr, str reduce_expr, str post_map_expr, str reduce_type,
+            Py_ssize_t block_size):
+        raise NotImplementedError()
+
+    @property
+    def cached_codes(self):
+        """Returns a dict that has input types as keys and codes values.
+
+        This proprety method is for debugging purpose.
+        The return value is not guaranteed to keep backward compatibility.
+        """
+        if len(self._cached_codes) == 0:
+            warnings.warn(
+                'No codes are cached because compilation is deferred until '
+                'the first function call or CUB is enabled.')
+        return dict([(k, v) for k, v in self._cached_codes.items()])
+
+    @property
+    def cached_code(self):
+        """Returns `next(iter(self.cached_codes.values()))`.
+
+        This proprety method is for debugging purpose.
+        The return value is not guaranteed to keep backward compatibility.
+        """
+        codes = self._cached_codes
+        if len(codes) > 1:
+            warnings.warn(
+                'The input types of the kernel could not be inferred. '
+                'Please use `.cached_codes` instead.')
+        return next(iter(codes.values()))
+
+
+# -----------------------------------------------------------------------------
+# create_reduction_func
+# -----------------------------------------------------------------------------
+
+cpdef _SimpleReductionKernel create_reduction_func(
+        name, ops, routine=None, identity=None, preamble='',
+        sort_reduce_axis=True):
+    ops = _kernel._Ops.from_tuples(ops, routine)
+    return _SimpleReductionKernel(
+        name, ops, identity, preamble, sort_reduce_axis)
+
+
+cdef class _SimpleReductionKernel(_AbstractReductionKernel):
+
+    cdef:
+        readonly _kernel._Ops _ops
+        readonly str preamble
+        readonly int nin
+        readonly int nout
+        readonly str _input_expr
+        readonly str _output_expr
+        readonly dict _routine_cache
+        readonly bint _sort_reduce_axis
+
+    def __init__(
+            self, name, _kernel._Ops ops, identity, preamble,
+            sort_reduce_axis=True):
+        super().__init__(
+            name,
+            '' if identity is None else str(identity),
+            'T in0',
+            'T out0',
+        )
+        self._ops = ops
+        self.preamble = preamble
+        self.nin = 1
+        self.nout = 1
+        self._input_expr = 'const type_in0_raw in0 = _raw_in0[_in_ind.get()];'
+        self._output_expr = 'type_out0_raw &out0 = _raw_out0[_out_ind.get()];'
+        self._routine_cache = {}
+        self._sort_reduce_axis = sort_reduce_axis
+
+    def __call__(self, object a, axis=None, dtype=None, _ndarray_base out=None,
+                 bint keepdims=False):
+
+        cdef _ndarray_base arr
+
+        if isinstance(a, _ndarray_base):
+            arr = a
+        elif hasattr(a, '__cuda_array_interface__'):
+            arr = _convert_object_with_cuda_array_interface(a)
+        elif hasattr(a, '__cupy_get_ndarray__'):
+            arr = a.__cupy_get_ndarray__()
+        else:
+            raise TypeError(
+                'Argument \'a\' has incorrect type (expected %s, got %s)' %
+                (cupy.ndarray, type(a)))
+        in_args = [arr]
+
+        dev_id = device.get_device_id()
+        _check_peer_access(arr, dev_id)
+
+        if out is None:
+            out_args = []
+        else:
+            _check_peer_access(out, dev_id)
+            out_args = [out]
+
+        reduce_dims = True
+        return self._call(
+            in_args, out_args,
+            arr._shape, axis, dtype, keepdims, reduce_dims, dev_id,
+            None, True, self._sort_reduce_axis)
+
+    cdef tuple _get_expressions_and_types(
+            self, list in_args, list out_args, dtype):
+        cdef _kernel._Op op
+
+        op = self._ops.guess_routine(
+            self.name, self._routine_cache, in_args, dtype, self._ops)
+        map_expr, reduce_expr, post_map_expr, reduce_type = op.routine
+
+        if reduce_type is None:
+            reduce_type = _get_typename(op.out_types[0])
+
+        if out_args:
+            out_type = out_args[0].dtype.type
+        else:
+            out_type = op.out_types[0]
+
+        # We guessed a routine that requires a C2R casting for the input
+        if (in_args[0].dtype.kind == 'c'
+                and numpy.dtype(op.in_types[0]).kind == 'f'):
+            warnings.warn(
+                'Casting complex values to real discards the imaginary part',
+                numpy.ComplexWarning)
+            in_args[0] = in_args[0].real
+
+        type_map = _kernel._TypeMap((
+            ('type_in0_raw', in_args[0].dtype.type),
+            ('type_out0_raw', out_type),
+        ))
+
+        return (
+            map_expr, reduce_expr, post_map_expr,
+            op.in_types, op.out_types, reduce_type,
+            type_map)
+
+    cdef list _get_out_args(
+            self, list out_args, tuple out_types, const shape_t& out_shape):
+        return _get_out_args_from_optionals(
+            cupy.ndarray, out_args, out_types, out_shape, 'unsafe', None)
+
+    cdef function.Function _get_function(
+            self,
+            tuple params, tuple arginfos, _kernel._TypeMap type_map,
+            str map_expr, str reduce_expr, str post_map_expr, str reduce_type,
+            Py_ssize_t block_size):
+
+        in_types = []
+        for x in arginfos:
+            if x.type is cupy.ndarray:
+                in_types.append(cupy.dtype(x.dtype).char)
+        in_types = tuple(in_types)
+        if in_types not in self._cached_codes:
+            code = _SimpleReductionKernel_get_cached_function_code(
+                map_expr, reduce_expr, post_map_expr, reduce_type,
+                params, arginfos, type_map,
+                self.name, block_size, self.identity,
+                self._input_expr, self._output_expr, self.preamble, ())
+            self._cached_codes[in_types] = code
+
+        return _SimpleReductionKernel_get_cached_function(
+            map_expr, reduce_expr, post_map_expr, reduce_type,
+            params, arginfos, type_map,
+            self.name, block_size, self.identity,
+            self._input_expr, self._output_expr, self.preamble, ())
+
+
+@_util.memoize()
+def _SimpleReductionKernel_get_cached_function_code(
+        map_expr, reduce_expr, post_map_expr, reduce_type,
+        params, arginfos, _kernel._TypeMap type_map,
+        name, block_size, identity, input_expr, output_expr, preamble,
+        options):
+    return _create_reduction_function_code(
+        name, block_size, reduce_type, params, arginfos, identity,
+        map_expr, reduce_expr, post_map_expr,
+        type_map, input_expr, output_expr, preamble, options)
+
+
+@_util.memoize(for_each_device=True)
+def _SimpleReductionKernel_get_cached_function(
+        map_expr, reduce_expr, post_map_expr, reduce_type,
+        params, arginfos, _kernel._TypeMap type_map,
+        name, block_size, identity, input_expr, output_expr, preamble,
+        options):
+    return _create_reduction_function(
+        name, block_size, reduce_type, params, arginfos, identity,
+        map_expr, reduce_expr, post_map_expr,
+        type_map, input_expr, output_expr, preamble, options)
+
+
+# -----------------------------------------------------------------------------
+# ReductionKernel
+# -----------------------------------------------------------------------------
+
+
+cdef class ReductionKernel(_AbstractReductionKernel):
+
+    """User-defined reduction kernel.
+
+    This class can be used to define a reduction kernel with or without
+    broadcasting.
+
+    The kernel is compiled at an invocation of the
+    :meth:`~ReductionKernel.__call__` method, which is cached for each device.
+    The compiled binary is also cached into a file under the
+    ``$HOME/.cupy/kernel_cache/`` directory with a hashed file name. The cached
+    binary is reused by other processes.
+
+    Args:
+        in_params (str): Input argument list.
+        out_params (str): Output argument list.
+        map_expr (str): Mapping expression for input values.
+        reduce_expr (str): Reduction expression.
+        post_map_expr (str): Mapping expression for reduced values.
+        identity (str): Identity value for starting the reduction.
+        name (str): Name of the kernel function. It should be set for
+            readability of the performance profiling.
+        reduce_type (str): Type of values to be used for reduction. This type
+            is used to store the special variables ``a``.
+        reduce_dims (bool): If ``True``, input arrays are reshaped without copy
+            to smaller dimensions for efficiency.
+        preamble (str): Fragment of the CUDA-C/C++ code that is inserted at the
+            top of the cu file.
+        options (tuple of str): Additional compilation options.
+
+    """
+
+    def __init__(self, str in_params, str out_params,
+                 map_expr, reduce_expr, post_map_expr,
+                 identity, name='reduce_kernel', reduce_type=None,
+                 reduce_dims=True, preamble='', options=()):
+        if not compiler.is_valid_kernel_name(name):
+            raise ValueError(
+                'Invalid kernel name: "%s"' % name)
+
+        super().__init__(
+            name,
+            '' if identity is None else str(identity),
+            in_params,
+            out_params,
+        )
+        self.nin = len(self.in_params)
+        self.nout = len(self.out_params)
+        self.nargs = self.nin + self.nout
+        self.reduce_expr = reduce_expr
+        self.map_expr = map_expr
+        self.post_map_expr = post_map_expr
+        self.options = options
+        self.reduce_dims = reduce_dims
+        if reduce_type is None:
+            self.reduce_type = self.out_params[0].ctype
+        else:
+            self.reduce_type = reduce_type
+        self.preamble = preamble
+
+    def __call__(self, *args, **kwargs):
+        """Compiles and invokes the reduction kernel.
+
+        The compilation runs only if the kernel is not cached. Note that the
+        kernels with different argument dtypes, ndims, or axis are not
+        compatible. It means that single ReductionKernel object may be compiled
+        into multiple kernel binaries.
+
+        Args:
+            args: Arguments of the kernel.
+            out (cupy.ndarray): The output array. This can only be specified if
+                ``args`` does not contain the output array.
+            axis (int or tuple of ints): Axis or axes along which the
+                reduction is performed.
+            keepdims (bool): If ``True``, the specified axes are remained as
+                axes of length one.
+            stream (cupy.cuda.Stream, optional): The CUDA stream to launch the
+                kernel on. If not given, the current stream will be used.
+
+        Returns:
+            Arrays are returned according to the ``out_params`` argument of the
+            ``__init__`` method.
+
+        """
+        cdef shape_t broad_shape
+
+        out = kwargs.pop('out', None)
+        axis = kwargs.pop('axis', None)
+        keepdims = kwargs.pop('keepdims', False)
+        stream = kwargs.pop('stream', None)
+        if kwargs:
+            raise TypeError('Wrong arguments %s' % kwargs)
+
+        n_args = len(args)
+        if n_args != self.nin and n_args != self.nargs:
+            raise TypeError('Wrong number of arguments for %s' % self.name)
+
+        out_args = list(args[self.nin:])
+        if out is not None:
+            if self.nout != 1:
+                raise NotImplementedError('')
+            if len(out_args) != 0:
+                raise ValueError("cannot specify 'out' as both "
+                                 "a positional and keyword argument")
+            out_args = [out]
+
+        dev_id = device.get_device_id()
+        in_args = _preprocess_args(dev_id, args[:self.nin], False)
+        out_args = _preprocess_args(dev_id, out_args, False)
+        in_args = _broadcast(in_args, self.in_params, False, broad_shape)
+
+        return self._call(
+            in_args, out_args,
+            broad_shape, axis, None,
+            keepdims, self.reduce_dims, dev_id, stream, True, True)
+
+    cdef tuple _get_expressions_and_types(
+            self, list in_args, list out_args, dtype):
+
+        in_ndarray_types = tuple(
+            [a.dtype.type if isinstance(a, _ndarray_base) else None
+             for a in in_args])
+        out_ndarray_types = tuple(
+            [a.dtype.type if isinstance(a, _ndarray_base) else None
+             for a in out_args])
+        in_types, out_types, type_map = _decide_params_type(
+            self.in_params, self.out_params,
+            in_ndarray_types, out_ndarray_types)
+        return (
+            self.map_expr, self.reduce_expr, self.post_map_expr,
+            in_types, out_types, self.reduce_type,
+            type_map)
+
+    cdef list _get_out_args(
+            self, list out_args, tuple out_types, const shape_t& out_shape):
+        return _get_out_args_with_params(
+            out_args, out_types, out_shape, self.out_params, False)
+
+    cdef function.Function _get_function(
+            self,
+            tuple params, tuple arginfos, _kernel._TypeMap type_map,
+            str map_expr, str reduce_expr, str post_map_expr, str reduce_type,
+            Py_ssize_t block_size):
+
+        in_types = []
+        for x in arginfos:
+            if x.type is cupy.ndarray:
+                in_types.append(cupy.dtype(x.dtype).char)
+        in_types = tuple(in_types)
+        if in_types not in self._cached_codes:
+            code =_ReductionKernel_get_cached_function_code(
+                self.nin, self.nout, params, arginfos, type_map,
+                self.name, block_size, reduce_type, self.identity,
+                map_expr, reduce_expr, post_map_expr,
+                self.preamble, self.options)
+            self._cached_codes[in_types] = code
+        return _ReductionKernel_get_cached_function(
+            self.nin, self.nout, params, arginfos, type_map,
+            self.name, block_size, reduce_type, self.identity,
+            map_expr, reduce_expr, post_map_expr,
+            self.preamble, self.options)
+
+
+@_util.memoize()
+def _ReductionKernel_get_cached_function_code(
+        nin, nout, params, arginfos, _kernel._TypeMap type_map,
+        name, block_size, reduce_type, identity, map_expr, reduce_expr,
+        post_map_expr, preamble, options):
+    cdef ParameterInfo p
+    cdef _ArgInfo arginfo
+    in_arrays = [
+        p for p, arginfo in zip(params[:nin], arginfos[:nin])
+        if not p.raw and arginfo.is_ndarray()]
+    out_arrays = [
+        p for p, arginfo in zip(params[nin:nin+nout], arginfos[nin:nin+nout])
+        if not p.raw and arginfo.is_ndarray()]
+    input_expr = '\n'.join(
+        [(('const {0} {1}' if p.is_const else '{0}& {1}') +
+          ' = _raw_{1}[_in_ind.get()];').format(p.ctype, p.name)
+         for p in in_arrays])
+    output_expr = '\n'.join(
+        ['{0} &{1} = _raw_{1}[_out_ind.get()];'.format(p.ctype, p.name)
+         for p in out_arrays if not p.is_const])
+
+    return _create_reduction_function_code(
+        name, block_size, reduce_type, params, arginfos, identity,
+        map_expr, reduce_expr, post_map_expr,
+        type_map, input_expr, output_expr, preamble, options)
+
+
+@_util.memoize(for_each_device=True)
+def _ReductionKernel_get_cached_function(
+        nin, nout, params, arginfos, _kernel._TypeMap type_map,
+        name, block_size, reduce_type, identity, map_expr, reduce_expr,
+        post_map_expr, preamble, options):
+    code = _ReductionKernel_get_cached_function_code(
+        nin, nout, params, arginfos, type_map,
+        name, block_size, reduce_type, identity, map_expr, reduce_expr,
+        post_map_expr, preamble, options)
+    return _create_reduction_function_from_code(name, code, options)
diff --git a/cupy/_core/_routines_binary.pxd b/cupy/_core/_routines_binary.pxd
new file mode 100644
index 0000000..140f2a3
--- /dev/null
+++ b/cupy/_core/_routines_binary.pxd
@@ -0,0 +1,6 @@
+cdef object _bitwise_and
+cdef object _bitwise_or
+cdef object _bitwise_xor
+cdef object _invert
+cdef object _left_shift
+cdef object _right_shift
diff --git a/cupy/_core/_routines_binary.pyx b/cupy/_core/_routines_binary.pyx
new file mode 100644
index 0000000..e069630
--- /dev/null
+++ b/cupy/_core/_routines_binary.pyx
@@ -0,0 +1,96 @@
+from cupy._core._kernel import create_ufunc
+
+
+cdef _create_bit_op(name, op, no_bool, doc='', scatter_op=None):
+    types = () if no_bool else ('??->?',)
+    return create_ufunc(
+        'cupy_' + name,
+        types + ('bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+                 'LL->L', 'qq->q', 'QQ->Q'),
+        'out0 = in0 %s in1' % op,
+        doc=doc, scatter_op=scatter_op)
+
+
+cdef _bitwise_and = _create_bit_op(
+    'bitwise_and', '&', False,
+    '''Computes the bitwise AND of two arrays elementwise.
+
+    Only integer and boolean arrays are handled.
+
+    .. seealso:: :data:`numpy.bitwise_and`
+
+    ''',
+    scatter_op='and')
+
+
+cdef _bitwise_or = _create_bit_op(
+    'bitwise_or', '|', False,
+    '''Computes the bitwise OR of two arrays elementwise.
+
+    Only integer and boolean arrays are handled.
+
+    .. seealso:: :data:`numpy.bitwise_or`
+
+    ''',
+    scatter_op='or')
+
+
+cdef _bitwise_xor = _create_bit_op(
+    'bitwise_xor', '^', False,
+    '''Computes the bitwise XOR of two arrays elementwise.
+
+    Only integer and boolean arrays are handled.
+
+    .. seealso:: :data:`numpy.bitwise_xor`
+
+    ''',
+    scatter_op='xor')
+
+
+cdef _invert = create_ufunc(
+    'cupy_invert',
+    (('?->?', 'out0 = !in0'), 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I',
+     'l->l', 'L->L', 'q->q', 'Q->Q'),
+    'out0 = ~in0',
+    doc='''Computes the bitwise NOT of an array elementwise.
+
+    Only integer and boolean arrays are handled.
+
+    .. note::
+        :func:`cupy.bitwise_not` is an alias for :func:`cupy.invert`.
+
+    .. seealso:: :data:`numpy.invert`
+
+    ''')
+
+
+cdef _left_shift = _create_bit_op(
+    'left_shift', '<<', True,
+    '''Shifts the bits of each integer element to the left.
+
+    Only integer arrays are handled.
+
+    .. seealso:: :data:`numpy.left_shift`
+
+    ''')
+
+
+cdef _right_shift = _create_bit_op(
+    'right_shift', '>>', True,
+    '''Shifts the bits of each integer element to the right.
+
+    Only integer arrays are handled
+
+    .. seealso:: :data:`numpy.right_shift`
+
+    ''')
+
+
+# Variables to expose to Python
+# (cythonized data cannot be exposed to Python, even with cpdef.)
+bitwise_and = _bitwise_and
+bitwise_or = _bitwise_or
+bitwise_xor = _bitwise_xor
+invert = _invert
+left_shift = _left_shift
+right_shift = _right_shift
diff --git a/cupy/_core/_routines_indexing.pxd b/cupy/_core/_routines_indexing.pxd
new file mode 100644
index 0000000..1fb06cd
--- /dev/null
+++ b/cupy/_core/_routines_indexing.pxd
@@ -0,0 +1,15 @@
+from cupy._core.core cimport _ndarray_base
+
+
+cpdef _ndarray_base _ndarray_argwhere(_ndarray_base self)
+cdef _ndarray_base _ndarray_getitem(_ndarray_base self, slices)
+cdef _ndarray_setitem(_ndarray_base self, slices, value)
+cdef tuple _ndarray_nonzero(_ndarray_base self)
+cdef _scatter_op(_ndarray_base a, slices, value, op)
+cdef _ndarray_base _ndarray_take(_ndarray_base self, indices, axis, out)
+cdef _ndarray_base _ndarray_put(_ndarray_base self, indices, values, mode)
+cdef _ndarray_base _ndarray_choose(_ndarray_base self, choices, out, mode)
+cdef _ndarray_base _ndarray_compress(_ndarray_base self, condition, axis, out)
+cdef _ndarray_base _ndarray_diagonal(_ndarray_base self, offset, axis1, axis2)
+cdef _ndarray_base _add_reduceat(
+    _ndarray_base array, indices, axis, dtype, out)
diff --git a/cupy/_core/_routines_indexing.pyx b/cupy/_core/_routines_indexing.pyx
new file mode 100644
index 0000000..e399ef9
--- /dev/null
+++ b/cupy/_core/_routines_indexing.pyx
@@ -0,0 +1,1161 @@
+# distutils: language = c++
+import warnings
+import string
+
+import numpy
+
+import cupy
+import cupy._core.core as core
+from cupy._core._kernel import ElementwiseKernel, _get_warpsize
+from cupy._core._ufuncs import elementwise_copy
+
+from libcpp cimport vector
+
+from cupy._core._carray cimport shape_t
+from cupy._core._carray cimport strides_t
+from cupy._core cimport core
+from cupy._core cimport _routines_math as _math
+from cupy._core cimport _routines_manipulation as _manipulation
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport internal
+
+
+# _ndarray_base members
+
+
+cdef _ndarray_base _ndarray_getitem(_ndarray_base self, slices):
+    cdef Py_ssize_t axis
+    cdef list slice_list
+    cdef _ndarray_base a
+
+    slice_list = _prepare_slice_list(slices)
+    a, adv = _view_getitem(self, slice_list)
+    if adv is None:
+        return a
+
+    axis = adv
+    if len(slice_list) == 1:
+        s = slice_list[0]
+        if s.dtype.kind == 'b':
+            return _getitem_mask_single(a, s, axis)
+        else:
+            return a.take(s, axis)
+
+    return _getitem_multiple(a, axis, slice_list)
+
+
+cdef _ndarray_setitem(_ndarray_base self, slices, value):
+    if isinstance(value, _ndarray_base):
+        value = _squeeze_leading_unit_dims(value)
+    _scatter_op(self, slices, value, 'update')
+
+
+cdef tuple _ndarray_nonzero(_ndarray_base self):
+    cdef int ndim
+    cdef _ndarray_base dst = _ndarray_argwhere(self)
+    ndim = self.ndim
+    if ndim >= 1:
+        return tuple([dst[:, i] for i in range(ndim)])
+    else:
+        warnings.warn(
+            'calling nonzero on 0d arrays is deprecated',
+            DeprecationWarning)
+        return cupy.zeros(dst.shape[0], numpy.int64),
+
+
+# TODO(kataoka): Rename the function because `_ndarray_base` does not have
+# `argwhere` method
+cpdef _ndarray_base _ndarray_argwhere(_ndarray_base self):
+    cdef Py_ssize_t count_nonzero
+    cdef int ndim
+    cdef _ndarray_base nonzero
+    numpy_int64 = numpy.int64
+    if self.size == 0:
+        count_nonzero = 0
+    else:
+        if self.dtype == numpy.bool_:
+            nonzero = self.ravel()
+        else:
+            nonzero = cupy._core.not_equal(self, 0)
+            nonzero = nonzero.ravel()
+
+        # Get number of True in the mask to determine the shape of the array
+        # after masking.
+        if nonzero.size <= 2 ** 31 - 1:
+            scan_dtype = numpy.int32
+        else:
+            scan_dtype = numpy_int64
+
+        chunk_size = 512
+
+        # TODO(anaruse): Use Optuna to automatically tune the threshold
+        # that determines whether "incomplete scan" is enabled or not.
+        # Basically, "incomplete scan" is fast when the array size is large,
+        # but for small arrays, it is better to use the normal method.
+        incomplete_scan = nonzero.size > chunk_size
+
+        scan_index = _math.scan(
+            nonzero, op=_math.scan_op.SCAN_SUM, dtype=scan_dtype, out=None,
+            incomplete=incomplete_scan, chunk_size=chunk_size)
+        count_nonzero = int(scan_index[-1])  # synchronize!
+
+    ndim = self._shape.size()
+    dst = core.ndarray((count_nonzero, ndim), dtype=numpy_int64)
+    if dst.size == 0:
+        return dst
+
+    nonzero.shape = self.shape
+    if incomplete_scan:
+        warp_size = _get_warpsize()
+        size = scan_index.size * chunk_size
+        _nonzero_kernel_incomplete_scan(chunk_size, warp_size)(
+            nonzero, scan_index, dst,
+            size=size, block_size=chunk_size)
+    else:
+        scan_index.shape = self.shape
+        _nonzero_kernel(nonzero, scan_index, dst)
+
+    return dst
+
+
+cdef _ndarray_base _ndarray_take(_ndarray_base self, indices, axis, out):
+    cdef Py_ssize_t ndim = self._shape.size()
+    if axis is None:
+        return _take(self, indices, 0, ndim, out)
+    elif ndim == 0:
+        # check axis after atleast_1d
+        internal._normalize_axis_index(axis, 1)
+        return _take(self, indices, 0, 0, out)
+    else:
+        axis = internal._normalize_axis_index(axis, ndim)
+        return _take(self, indices, axis, axis + 1, out)
+
+
+cdef _ndarray_base _ndarray_put(_ndarray_base self, indices, values, mode):
+    if mode not in ('raise', 'wrap', 'clip'):
+        raise ValueError('clipmode not understood')
+
+    n = self.size
+    if not isinstance(indices, _ndarray_base):
+        indices = core.array(indices)
+    indices = indices.ravel()
+
+    if not isinstance(values, _ndarray_base):
+        values = core.array(values, dtype=self.dtype)
+    if values.size == 0:
+        return
+
+    if mode == 'raise':
+        err = cupy.zeros((), dtype=numpy.bool_)
+        _put_raise_kernel(indices, values, values.size, n, self, err)
+        if err:
+            raise IndexError('invalid entry in indices array')
+    elif mode == 'wrap':
+        _put_wrap_kernel(indices, values, values.size, n, self)
+    elif mode == 'clip':
+        _put_clip_kernel(indices, values, values.size, n, self)
+
+
+cdef _ndarray_base _ndarray_choose(_ndarray_base self, choices, out, mode):
+    a = self
+    n = choices.shape[0]
+
+    # broadcast `a` and `choices[i]` for all i
+    if a.ndim < choices.ndim - 1:
+        for i in range(choices.ndim - 1 - a.ndim):
+            a = a[None, ...]
+    elif a.ndim > choices.ndim - 1:
+        for i in range(a.ndim + 1 - choices.ndim):
+            choices = choices[:, None, ...]
+    ba, bcs = _manipulation.broadcast(a, choices).values
+
+    if out is None:
+        out = core.ndarray(ba.shape[1:], choices.dtype)
+
+    n_channel = numpy.prod(bcs[0].shape)
+    if mode == 'raise':
+        if not ((a < n).all() and (0 <= a).all()):
+            raise ValueError('invalid entry in choice array')
+        _choose_kernel(ba[0], bcs, n_channel, out)
+    elif mode == 'wrap':
+        ba = ba[0] % n
+        _choose_kernel(ba, bcs, n_channel, out)
+    elif mode == 'clip':
+        _choose_clip_kernel(ba[0], bcs, n_channel, n, out)
+    else:
+        raise ValueError('clipmode not understood')
+
+    return out
+
+
+cdef _ndarray_base _ndarray_compress(_ndarray_base self, condition, axis, out):
+    a = self
+
+    if numpy.isscalar(condition):
+        raise ValueError('condition must be a 1-d array')
+
+    if not isinstance(condition, _ndarray_base):
+        condition = core.array(condition, dtype=int)
+        if condition.ndim != 1:
+            raise ValueError('condition must be a 1-d array')
+
+    # do not test condition.shape
+    res = _ndarray_nonzero(condition)  # synchronize
+
+    # the `take` method/function also make the input atleast_1d
+    return _ndarray_take(a, res[0], axis, out)
+
+
+cdef _ndarray_base _ndarray_diagonal(_ndarray_base self, offset, axis1, axis2):
+    return _diagonal(self, offset, axis1, axis2)
+
+
+# private/internal
+
+
+cdef _ndarray_base _squeeze_leading_unit_dims(_ndarray_base src):
+    # remove leading 1s from the shape greedily.
+    # TODO(kataoka): compute requested ndim and do not remove too much for
+    # printing correct shape in error message.
+    cdef Py_ssize_t i
+    for i in range(src.ndim):
+        if src._shape[i] != 1:
+            break
+    else:
+        i = src.ndim
+
+    if i == 0:
+        return src
+
+    src = src.view()
+    # del src._shape[:i]
+    # del src._strides[:i]
+    src._shape.erase(src._shape.begin(), src._shape.begin()+i)
+    src._strides.erase(src._strides.begin(), src._strides.begin()+i)
+    return src
+
+
+cpdef list _prepare_slice_list(slices):
+    cdef Py_ssize_t i
+    cdef list slice_list
+    cdef bint fix_empty_dtype
+
+    if isinstance(slices, tuple):
+        slice_list = list(slices)
+    else:
+        slice_list = [slices]
+
+    # Convert list/NumPy/CUDA-Array-Interface arrays to cupy.ndarray.
+    # - Scalar int in indices returns a view.
+    # - Other array-like (including ()-shaped array) in indices forces to
+    #   return a new array.
+    for i, s in enumerate(slice_list):
+        if s is None or s is Ellipsis or isinstance(s, (slice, _ndarray_base)):
+            continue
+
+        fix_empty_dtype = False
+        if isinstance(s, (list, tuple)):
+            # This condition looks inaccurate, but so is NumPy.
+            # a[1, [np.empty(0, float)]] is allowed, while
+            # a[1, np.empty((1, 0), float)] raises IndexError.
+            fix_empty_dtype = True
+        elif numpy.isscalar(s):
+            if not isinstance(s, (bool, numpy.bool_)):
+                # keep scalar int
+                continue
+
+        if cupy.min_scalar_type(s).char == 'O':
+            raise IndexError(
+                'arrays used as indices must be of integer (or boolean) type')
+        try:
+            s = core.array(s, dtype=None, copy=False)
+        except ValueError:
+            # "Unsupported dtype"
+            raise IndexError(
+                'only integers, slices (`:`), ellipsis (`...`),'
+                'numpy.newaxis (`None`) and integer or '
+                'boolean arrays are valid indices')
+        if fix_empty_dtype and s.size == 0:
+            # An empty list means empty indices, not empty mask.
+            # Fix default dtype (float64).
+            s = s.astype(numpy.int32)
+        slice_list[i] = s
+
+    return slice_list
+
+
+cdef tuple _view_getitem(_ndarray_base a, list slice_list):
+    # Process scalar/slice/ellipsis indices
+    # Returns a 2-tuple
+    # - [0] (ndarray): view of a
+    # - [1] (int or None): start axis for remaining indices
+    # slice_list will be overwritten.
+    #     input should contain:
+    #         None, Ellipsis, slice (start:stop:step), scalar int, or
+    #         cupy.ndarray
+    #     output will contain:
+    #         cupy.ndarray
+    cdef shape_t shape
+    cdef strides_t strides
+    cdef _ndarray_base v
+    cdef Py_ssize_t ndim_a, axis_a, ndim_v, axis_v, ndim_ellipsis
+    cdef Py_ssize_t i, k, offset
+    cdef Py_ssize_t s_start, s_stop, s_step, dim, ind
+    cdef slice ss
+    cdef list index_list, axes
+    cdef vector.vector[bint] array_like_flags
+    cdef vector.vector[Py_ssize_t] array_ndims
+    cdef bint has_ellipsis, flag
+    cdef char kind
+
+    axis_a = 0
+    has_ellipsis = False
+    for s in slice_list:
+        if s is None:
+            continue
+        elif s is Ellipsis:
+            if has_ellipsis:
+                raise IndexError(
+                    "an index can only have a single ellipsis ('...')")
+            has_ellipsis = True
+        elif isinstance(s, _ndarray_base):
+            kind = ord(s.dtype.kind)
+            if kind == b'b':
+                k = s.ndim
+            elif kind == b'i' or kind == b'u':
+                k = 1
+            else:
+                raise IndexError(
+                    'arrays used as indices must be of integer or boolean '
+                    'type. (actual: {})'.format(s.dtype.type))
+            array_ndims.push_back(k)
+            axis_a += k
+        else:
+            # isinstance(s, slice) or numpy.isscalar(s)
+            axis_a += 1
+    if not has_ellipsis:
+        slice_list.append(Ellipsis)
+
+    ndim_a = a._shape.size()
+    if axis_a > ndim_a:
+        raise IndexError(
+            'too many indices for array: '
+            f'array is {ndim_a}-dimensional, but {axis_a} were indexed')
+    ndim_ellipsis = ndim_a - axis_a
+
+    # Create new shape and stride
+    i = 0
+    axis_a = 0
+    axis_v = 0
+    offset = 0
+    # index_list: remaining indices to be processed.
+    # Each elem is a 3-tuple (array, axis_start, axis_count)
+    index_list = []
+    for s in slice_list:
+        if s is None:
+            shape.push_back(1)
+            strides.push_back(0)
+            axis_v += 1
+            array_like_flags.push_back(False)
+        elif isinstance(s, _ndarray_base):
+            k = array_ndims[i]
+            index_list.append((s, axis_v, k))
+            i += 1
+            kind = ord(s.dtype.kind)
+            if kind == b'b':
+                _check_mask_shape(a, s, axis_a)
+            for _ in range(k):
+                shape.push_back(a._shape[axis_a])
+                strides.push_back(a._strides[axis_a])
+                axis_a += 1
+            axis_v += k
+            array_like_flags.push_back(True)
+        elif s is Ellipsis:
+            for _ in range(ndim_ellipsis):
+                shape.push_back(a._shape[axis_a])
+                strides.push_back(a._strides[axis_a])
+                axis_a += 1
+            axis_v += ndim_ellipsis
+            array_like_flags.push_back(False)
+        elif isinstance(s, slice):
+            ss = internal.complete_slice(s, a._shape[axis_a])
+            s_start = ss.start
+            s_stop = ss.stop
+            s_step = ss.step
+            if s_step > 0:
+                dim = (s_stop - s_start - 1) // s_step + 1
+            else:
+                dim = (s_stop - s_start + 1) // s_step + 1
+
+            if dim == 0:
+                strides.push_back(a._strides[axis_a])
+            else:
+                strides.push_back(a._strides[axis_a] * s_step)
+
+            if s_start > 0:
+                offset += a._strides[axis_a] * s_start
+            shape.push_back(dim)
+            axis_a += 1
+            axis_v += 1
+            array_like_flags.push_back(False)
+        else:
+            # numpy.isscalar(s)
+            ind = int(s)
+            if ind < 0:
+                ind += a._shape[axis_a]
+            if not (0 <= ind < a._shape[axis_a]):
+                msg = ('Index %s is out of bounds for axis %s with '
+                       'size %s' % (s, axis_a, a._shape[axis_a]))
+                raise IndexError(msg)
+            offset += ind * a._strides[axis_a]
+            axis_a += 1
+            # array-like but not array
+            array_like_flags.push_back(True)
+
+    ndim_v = axis_v
+    v = a.view()
+    if a.size != 0:
+        v.data = a.data + offset
+    v._set_shape_and_strides(shape, strides, True, True)
+
+    if array_ndims.empty():
+        # no advanced indexing. no mask.
+        del slice_list[:]
+        return v, None
+
+    slice_list[:] = [s for s, _, _ in index_list]
+
+    # non-consecutive array-like indices => batch dims go first in output
+    # consecutive array-like indices => start batch dims there
+    k = 0
+    for i, flag in enumerate(array_like_flags):
+        if k == 0:
+            if flag:
+                k = 1
+        elif k == 1:
+            if not flag:
+                k = 2
+        else:  # k == 2
+            if flag:
+                break
+    else:
+        return v, index_list[0][1]
+
+    # compute transpose arg
+    axes = []
+    for _, axis_v, k in index_list:
+        for _ in range(k):
+            axes.append(axis_v)
+            axis_v += 1
+    axes.extend([dim for dim in range(ndim_v) if dim not in axes])
+    v = _manipulation._transpose(v, axes)
+    return v, 0
+
+
+@cupy._util.memoize(for_each_device=True)
+def _nonzero_kernel_incomplete_scan(block_size, warp_size=32):
+    in_params = 'raw T a, raw S b'
+    out_params = 'raw O dst'
+    loop_prep = string.Template("""
+        __shared__ S smem[${warp_size}];
+        const int n_warp = ${block_size} / ${warp_size};
+        const int warp_id = threadIdx.x / ${warp_size};
+        const int lane_id = threadIdx.x % ${warp_size};
+    """).substitute(block_size=block_size, warp_size=warp_size)
+    loop_body = string.Template("""
+        S x = 0;
+        if (i < a.size()) x = a[i];
+        for (int j = 1; j < ${warp_size}; j *= 2) {
+            S tmp = __shfl_up_sync(0xffffffff, x, j, ${warp_size});
+            if (lane_id - j >= 0) x += tmp;
+        }
+        if (lane_id == ${warp_size} - 1) smem[warp_id] = x;
+        __syncthreads();
+        if (warp_id == 0) {
+            S y = 0;
+            if (lane_id < n_warp) y = smem[lane_id];
+            for (int j = 1; j < n_warp; j *= 2) {
+                S tmp = __shfl_up_sync(0xffffffff, y, j, ${warp_size});
+                if (lane_id - j >= 0) y += tmp;
+            }
+            int block_id = i / ${block_size};
+            S base = 0;
+            if (block_id > 0) base = b[block_id - 1];
+            if (lane_id == ${warp_size} - 1) y = 0;
+            smem[(lane_id + 1) % ${warp_size}] = y + base;
+        }
+        __syncthreads();
+        x += smem[warp_id];
+        S x0 = __shfl_up_sync(0xffffffff, x, 1, ${warp_size});
+        if (lane_id == 0) {
+            x0 = smem[warp_id];
+        }
+        if (x0 < x && i < a.size()) {
+            O j = i;
+            for (int d = a.ndim - 1; d >= 0; d--) {
+                ptrdiff_t ind[] = {x0, d};
+                O j_next = j / a.shape()[d];
+                dst[ind] = j - j_next * a.shape()[d];
+                j = j_next;
+            }
+        }
+    """).substitute(block_size=block_size, warp_size=warp_size)
+    return cupy.ElementwiseKernel(in_params, out_params, loop_body,
+                                  'cupy_nonzero_kernel_incomplete_scan',
+                                  loop_prep=loop_prep)
+
+
+_nonzero_kernel = ElementwiseKernel(
+    'T src, S index', 'raw U dst',
+    '''
+    if (src != 0){
+        for(int j = 0; j < _ind.ndim; j++){
+            ptrdiff_t ind[] = {index - 1, j};
+            dst[ind] = _ind.get()[j];
+        }
+    }''',
+    'cupy_nonzero_kernel',
+    reduce_dims=False)
+
+
+_take_kernel_core = '''
+ptrdiff_t out_i = indices % index_range;
+if (out_i < 0) out_i += index_range;
+if (ldim != 1) out_i += (i / (cdim * rdim)) * index_range;
+if (rdim != 1) out_i = out_i * rdim + i % rdim;
+out = a[out_i];
+'''
+
+
+_take_kernel = ElementwiseKernel(
+    'raw T a, S indices, uint32 ldim, uint32 cdim, uint32 rdim, '
+    'int64 index_range',
+    'T out', _take_kernel_core, 'cupy_take')
+
+
+_take_kernel_scalar = ElementwiseKernel(
+    'raw T a, int64 indices, uint32 ldim, uint32 cdim, uint32 rdim, '
+    'int64 index_range',
+    'T out', _take_kernel_core, 'cupy_take_scalar')
+
+
+_choose_kernel = ElementwiseKernel(
+    'S a, raw T choices, int32 n_channel',
+    'T y',
+    'y = choices[i + n_channel * a]',
+    'cupy_choose')
+
+
+_choose_clip_kernel = ElementwiseKernel(
+    'S a, raw T choices, int32 n_channel, int32 n',
+    'T y',
+    '''
+      S x = a;
+      if (a < 0) {
+        x = 0;
+      } else if (a >= n) {
+        x = n - 1;
+      }
+      y = choices[i + n_channel * x];
+    ''',
+    'cupy_choose_clip')
+
+
+cdef _put_raise_kernel = ElementwiseKernel(
+    'S ind, raw T vals, int64 n_vals, int64 n',
+    'raw U data, raw bool err',
+    '''
+      ptrdiff_t ind_ = ind;
+      if (!(-n <= ind_ && ind_ < n)) {
+        err[0] = 1;
+      } else {
+        if (ind_ < 0) ind_ += n;
+        data[ind_] = (U)(vals[i % n_vals]);
+      }
+    ''',
+    'cupy_put_raise')
+
+
+cdef _put_wrap_kernel = ElementwiseKernel(
+    'S ind, raw T vals, int64 n_vals, int64 n',
+    'raw U data',
+    '''
+      ptrdiff_t ind_ = ind;
+      ind_ %= n;
+      if (ind_ < 0) ind_ += n;
+      data[ind_] = (U)(vals[i % n_vals]);
+    ''',
+    'cupy_put_wrap')
+
+
+cdef _put_clip_kernel = ElementwiseKernel(
+    'S ind, raw T vals, int64 n_vals, int64 n',
+    'raw U data',
+    '''
+      ptrdiff_t ind_ = ind;
+      if (ind_ < 0) {
+        ind_ = 0;
+      } else if (ind_ >= n) {
+        ind_ = n - 1;
+      }
+      data[ind_] = (U)(vals[i % n_vals]);
+    ''',
+    'cupy_put_clip')
+
+
+cdef _create_scatter_kernel(name, code):
+    return ElementwiseKernel(
+        'T v, S indices, int32 cdim, int32 rdim, int32 adim',
+        'raw T a',
+        string.Template('''
+            S wrap_indices = indices % adim;
+            if (wrap_indices < 0) wrap_indices += adim;
+            ptrdiff_t li = i / (rdim * cdim);
+            ptrdiff_t ri = i % rdim;
+            T &out0 = a[(li * adim + wrap_indices) * rdim + ri];
+            T &in0 = out0;
+            const T &in1 = v;
+            ${code};
+        ''').substitute(code=code),
+        name,
+    )
+
+
+cdef _scatter_update_kernel = _create_scatter_kernel(
+    'cupy_scatter_update', 'out0 = in1')
+
+cdef _scatter_add_kernel = _create_scatter_kernel(
+    'cupy_scatter_add', 'atomicAdd(&out0, in1)')
+
+cdef _scatter_sub_kernel = _create_scatter_kernel(
+    'cupy_scatter_sub', 'atomicSub(&out0, in1)')
+
+cdef _scatter_max_kernel = _create_scatter_kernel(
+    'cupy_scatter_max', 'atomicMax(&out0, in1)')
+
+cdef _scatter_min_kernel = _create_scatter_kernel(
+    'cupy_scatter_min', 'atomicMin(&out0, in1)')
+
+cdef _scatter_and_kernel = _create_scatter_kernel(
+    'cupy_scatter_and', 'atomicAnd(&out0, in1)')
+
+cdef _scatter_or_kernel = _create_scatter_kernel(
+    'cupy_scatter_or', 'atomicOr(&out0, in1)')
+
+cdef _scatter_xor_kernel = _create_scatter_kernel(
+    'cupy_scatter_xor', 'atomicXor(&out0, in1)')
+
+
+cdef _create_scatter_mask_kernel(name, code):
+    return ElementwiseKernel(
+        'raw T v, bool mask, S mask_scanned',
+        'T a',
+        string.Template('''
+            T &out0 = a;
+            T &in0 = a;
+            const T &in1 = v[mask_scanned - 1];
+            if (mask) ${code};
+        ''').substitute(code=code),
+        name,
+    )
+
+
+cdef _scatter_update_mask_kernel = _create_scatter_mask_kernel(
+    'cupy_scatter_update_mask', 'out0 = in1')
+
+cdef _scatter_add_mask_kernel = _create_scatter_mask_kernel(
+    'cupy_scatter_add_mask', 'out0 = in0 + in1')
+
+cdef _scatter_sub_mask_kernel = _create_scatter_mask_kernel(
+    'cupy_scatter_add_mask', 'out0 = in0 - in1')
+
+cdef _scatter_max_mask_kernel = _create_scatter_mask_kernel(
+    'cupy_scatter_max_mask', 'out0 = max(in0, in1)')
+
+cdef _scatter_min_mask_kernel = _create_scatter_mask_kernel(
+    'cupy_scatter_min_mask', 'out0 = min(in0, in1)')
+
+cdef _scatter_and_mask_kernel = _create_scatter_mask_kernel(
+    'cupy_scatter_and_mask', 'out0 = (in0 & in1)')
+
+cdef _scatter_or_mask_kernel = _create_scatter_mask_kernel(
+    'cupy_scatter_or_mask', 'out0 = (in0 | in1)')
+
+cdef _scatter_xor_mask_kernel = _create_scatter_mask_kernel(
+    'cupy_scatter_xor_mask', 'out0 = (in0 ^ in1)')
+
+
+_getitem_mask_kernel = ElementwiseKernel(
+    'T a, bool mask, S mask_scanned',
+    'raw T out',
+    'if (mask) out[mask_scanned - 1] = a',
+    'cupy_getitem_mask')
+
+
+cdef _check_mask_shape(_ndarray_base a, _ndarray_base mask, Py_ssize_t axis):
+    cdef Py_ssize_t i, a_sh, m_sh
+    for i, m_sh in enumerate(mask._shape):
+        a_sh = a._shape[axis + i]
+        if m_sh not in (0, a_sh):
+            raise IndexError(
+                'boolean index did not match indexed array along dimension '
+                f'{axis + i}; dimension is {a_sh} '
+                f'but corresponding boolean dimension is {m_sh}'
+            )
+
+
+cpdef _prepare_mask_indexing_single(
+        _ndarray_base a, _ndarray_base mask, Py_ssize_t axis):
+    cdef _ndarray_base mask_scanned, mask_br
+    cdef int n_true
+    cdef tuple lshape, rshape, a_shape
+    cdef Py_ssize_t a_ndim, mask_ndim
+
+    a_ndim = a._shape.size()
+    mask_ndim = mask._shape.size()
+    a_shape = a.shape
+    lshape = a_shape[:axis]
+    rshape = a_shape[axis + mask._shape.size():]
+
+    if mask.size == 0:
+        masked_shape = lshape + (0,) + rshape
+        mask_br = _manipulation._reshape(mask, masked_shape)
+        return mask_br, mask_br, masked_shape
+
+    # Get number of True in the mask to determine the shape of the array
+    # after masking.
+    if mask.size <= 2 ** 31 - 1:
+        mask_type = numpy.int32
+    else:
+        mask_type = numpy.int64
+    op = _math.scan_op.SCAN_SUM
+
+    # starts with 1
+    mask_scanned = _math.scan(mask.ravel(), op=op, dtype=mask_type)
+    n_true = int(mask_scanned[-1])
+    masked_shape = lshape + (n_true,) + rshape
+    # When mask covers the entire array, broadcasting is not necessary.
+    if mask_ndim == a_ndim and axis == 0:
+        return (
+            mask,
+            _manipulation._reshape(mask_scanned, mask._shape),
+            masked_shape)
+    mask_scanned = None
+
+    # The scan of the broadcasted array is used to index on kernel.
+    mask = _manipulation._reshape(
+        mask,
+        axis * (1,) + mask.shape + (a_ndim - axis - mask_ndim) * (1,))
+    if <Py_ssize_t>mask._shape.size() > a_ndim:
+        raise IndexError('too many indices for array')
+
+    mask = _manipulation.broadcast_to(mask, a_shape)
+    if mask.size <= 2 ** 31 - 1:
+        mask_type = numpy.int32
+    else:
+        mask_type = numpy.int64
+    mask_scanned = _manipulation._reshape(
+        _math.scan(mask.ravel(), op=_math.scan_op.SCAN_SUM, dtype=mask_type),
+        mask._shape)
+    return mask, mask_scanned, masked_shape
+
+
+cpdef _ndarray_base _getitem_mask_single(
+        _ndarray_base a, _ndarray_base mask, int axis):
+    cdef _ndarray_base mask_scanned
+    cdef tuple masked_shape
+
+    mask, mask_scanned, masked_shape = _prepare_mask_indexing_single(
+        a, mask, axis)
+    out = core.ndarray(masked_shape, dtype=a.dtype)
+    if out.size == 0:
+        return out
+    return _getitem_mask_kernel(a, mask, mask_scanned, out)
+
+
+cdef _ndarray_base _take(
+        _ndarray_base a, indices, int start, int stop, _ndarray_base out=None):
+    # Take along (flattened) axes from start to stop.
+    # When start + 1 == stop this function behaves similarly to np.take
+    cdef tuple out_shape, indices_shape
+    cdef int i, ndim = a._shape.size()
+    cdef Py_ssize_t ldim, cdim, rdim, index_range
+
+    assert start <= stop
+
+    if numpy.isscalar(indices):
+        indices_shape = ()
+        cdim = 1
+    else:
+        if not isinstance(indices, _ndarray_base):
+            indices = core.array(indices, dtype=int)
+        indices_shape = indices.shape
+        cdim = indices.size
+
+    ldim = rdim = 1
+    if start == 0 and stop == ndim:
+        out_shape = indices_shape
+        index_range = a.size
+    else:
+        a_shape = a.shape
+        out_shape = a_shape[:start] + indices_shape + a_shape[stop:]
+        if len(indices_shape) != 0:
+            indices = _manipulation._reshape(
+                indices,
+                (1,) * start + indices_shape + (1,) * (ndim - stop))
+        for i in range(start):
+            ldim *= a._shape[i]
+        for i in range(stop, ndim):
+            rdim *= a._shape[i]
+        index_range = 1
+        for i in range(start, stop):
+            index_range *= a._shape[i]
+
+    if out is None:
+        out = core.ndarray(out_shape, dtype=a.dtype)
+    else:
+        if out.dtype != a.dtype:
+            raise TypeError('Output dtype mismatch')
+        if out.shape != out_shape:
+            raise ValueError('Output shape mismatch')
+    if a.size == 0 and out.size != 0:
+        raise IndexError('cannot do a non-empty take from an empty axes.')
+
+    if isinstance(indices, _ndarray_base):
+        return _take_kernel(
+            a.reduced_view(), indices, ldim, cdim, rdim, index_range, out)
+    else:
+        return _take_kernel_scalar(
+            a.reduced_view(), indices, ldim, cdim, rdim, index_range, out)
+
+
+cdef _scatter_op_single(
+        _ndarray_base a, _ndarray_base indices, value, Py_ssize_t start,
+        Py_ssize_t stop, op=''):
+    # When op == 'update', this function behaves similarly to
+    # a code below using NumPy under the condition that a = a._reshape(shape)
+    # does not invoke copy.
+    #
+    # shape = a[:start] +\
+    #     (numpy.prod(a[start:stop]),) + a[stop:]
+    # a = a._reshape(shape)
+    # slices = (slice(None),) * start + indices +\
+    #     (slice(None),) * (a.ndim - stop)
+    # a[slices] = value
+    cdef Py_ssize_t adim, cdim, rdim
+    cdef tuple a_shape, indices_shape, lshape, rshape, v_shape
+    cdef _ndarray_base v
+
+    if not isinstance(value, _ndarray_base):
+        v = core.array(value, dtype=a.dtype)
+    else:
+        v = value.astype(a.dtype, copy=False)
+
+    a_shape = a.shape
+
+    lshape = a_shape[:start]
+    rshape = a_shape[stop:]
+    adim = internal.prod_sequence(a_shape[start:stop])
+
+    indices_shape = indices.shape
+    v_shape = lshape + indices_shape + rshape
+    v = _manipulation.broadcast_to(v, v_shape)
+
+    cdim = indices.size
+    rdim = internal.prod_sequence(rshape)
+    indices = _manipulation._reshape(
+        indices,
+        (1,) * len(lshape) + indices_shape + (1,) * len(rshape))
+    indices = _manipulation.broadcast_to(indices, v_shape)
+
+    if op == 'update':
+        _scatter_update_kernel(
+            v, indices, cdim, rdim, adim, a.reduced_view())
+    elif op == 'add':
+        # There is constraints on types because atomicAdd() in CUDA 7.5
+        # only supports int32, uint32, uint64, and float32.
+        if not issubclass(v.dtype.type,
+                          (numpy.int32, numpy.float16, numpy.float32,
+                           numpy.float64, numpy.uint32, numpy.uint64,
+                           numpy.intc, numpy.uintc, numpy.ulonglong)):
+            raise TypeError(
+                'cupy.add.at only supports int32, float16, float32, float64, '
+                'uint32, uint64, as data type')
+        _scatter_add_kernel(
+            v, indices, cdim, rdim, adim, a.reduced_view())
+    elif op == 'sub':
+        if not issubclass(v.dtype.type,
+                          (numpy.int32, numpy.uint32,
+                           numpy.intc, numpy.uintc)):
+            raise TypeError(
+                'cupy.subtract.at only supports int32, uint32, as data type')
+        _scatter_sub_kernel(
+            v, indices, cdim, rdim, adim, a.reduced_view())
+    elif op == 'max':
+        if not issubclass(v.dtype.type,
+                          (numpy.int32, numpy.float32, numpy.float64,
+                           numpy.uint32, numpy.uint64,
+                           numpy.intc, numpy.uintc, numpy.ulonglong)):
+            raise TypeError(
+                'cupy.maximum.at only supports int32, float32, float64, '
+                'uint32, uint64 as data type')
+        _scatter_max_kernel(
+            v, indices, cdim, rdim, adim, a.reduced_view())
+    elif op == 'min':
+        if not issubclass(v.dtype.type,
+                          (numpy.int32, numpy.float32, numpy.float64,
+                           numpy.uint32, numpy.uint64,
+                           numpy.intc, numpy.uintc, numpy.ulonglong)):
+            raise TypeError(
+                'cupy.minimum.at only supports int32, float32, float64, '
+                'uint32, uint64 as data type')
+        _scatter_min_kernel(
+            v, indices, cdim, rdim, adim, a.reduced_view())
+    elif op == 'and':
+        if not issubclass(v.dtype.type,
+                          (numpy.int32, numpy.int64,
+                           numpy.uint32, numpy.uint64,
+                           numpy.intc, numpy.uintc,
+                           numpy.longlong, numpy.ulonglong)):
+            raise TypeError(
+                'cupy.bitwise_and.at only supports int32, int64, '
+                'uint32, uint64 as data type')
+        _scatter_and_kernel(
+            v, indices, cdim, rdim, adim, a.reduced_view())
+    elif op == 'or':
+        if not issubclass(v.dtype.type,
+                          (numpy.int32, numpy.int64,
+                           numpy.uint32, numpy.uint64,
+                           numpy.intc, numpy.uintc,
+                           numpy.longlong, numpy.ulonglong)):
+            raise TypeError(
+                'cupy.bitwise_or.at only supports int32, int64, '
+                'uint32, uint64 as data type')
+        _scatter_or_kernel(
+            v, indices, cdim, rdim, adim, a.reduced_view())
+    elif op == 'xor':
+        if not issubclass(v.dtype.type,
+                          (numpy.int32, numpy.int64,
+                           numpy.uint32, numpy.uint64,
+                           numpy.intc, numpy.uintc,
+                           numpy.longlong, numpy.ulonglong)):
+            raise TypeError(
+                'cupy.bitwise_xor.at only supports int32, int64, '
+                'uint32, uint64 as data type')
+        _scatter_xor_kernel(
+            v, indices, cdim, rdim, adim, a.reduced_view())
+    else:
+        raise ValueError('provided op is not supported')
+
+
+cdef _scatter_op_mask_single(
+        _ndarray_base a, _ndarray_base mask, v, Py_ssize_t axis, op):
+    cdef _ndarray_base mask_scanned, src
+    cdef tuple masked_shape
+
+    mask, mask_scanned, masked_shape = _prepare_mask_indexing_single(
+        a, mask, axis)
+    if internal.prod(masked_shape) == 0:
+        return
+
+    if not isinstance(v, _ndarray_base):
+        src = core.array(v, dtype=a.dtype)
+    else:
+        src = v
+        # Cython's static resolution does not work because of omitted arguments
+        src = (<object>src).astype(a.dtype, copy=False)
+    # broadcast src to shape determined by the mask
+    src = _manipulation.broadcast_to(src, masked_shape)
+
+    if op == 'update':
+        _scatter_update_mask_kernel(src, mask, mask_scanned, a)
+    elif op == 'add':
+        _scatter_add_mask_kernel(src, mask, mask_scanned, a)
+    elif op == 'sub':
+        _scatter_sub_mask_kernel(src, mask, mask_scanned, a)
+    elif op == 'max':
+        _scatter_max_mask_kernel(src, mask, mask_scanned, a)
+    elif op == 'min':
+        _scatter_min_mask_kernel(src, mask, mask_scanned, a)
+    elif op == 'and':
+        _scatter_and_mask_kernel(src, mask, mask_scanned, a)
+    elif op == 'or':
+        _scatter_or_mask_kernel(src, mask, mask_scanned, a)
+    elif op == 'xor':
+        _scatter_xor_mask_kernel(src, mask, mask_scanned, a)
+    else:
+        raise ValueError('provided op is not supported')
+
+
+cdef _scatter_op(_ndarray_base a, slices, value, op):
+    cdef Py_ssize_t start, stop, axis
+    cdef _ndarray_base x, y, reduced_idx
+    cdef list slice_list
+
+    slice_list = _prepare_slice_list(slices)
+    a, adv = _view_getitem(a, slice_list)
+    if adv is not None:
+        axis = adv
+        if len(slice_list) == 1:
+            s = slice_list[0]
+            if s.dtype.kind == 'b':
+                _scatter_op_mask_single(a, s, value, axis, op)
+            else:
+                _scatter_op_single(a, s, value, axis, axis + 1, op)
+        else:
+            # scatter_op with multiple integer arrays
+            reduced_idx, start, stop = _prepare_multiple_array_indexing(
+                a, axis, slice_list)
+            _scatter_op_single(a, reduced_idx, value, start, stop, op)
+        return
+
+    y = a
+
+    if op == 'update':
+        if not isinstance(value, _ndarray_base):
+            y.fill(value)
+            return
+        x = value
+        if (internal.vector_equal(y._shape, x._shape) and
+                internal.vector_equal(y._strides, x._strides)):
+            if y.data.ptr == x.data.ptr:
+                return  # Skip since x and y are the same array
+            elif y._c_contiguous and x.dtype == y.dtype:
+                y.data.copy_from_device_async(x.data, x.nbytes)
+                return
+        elementwise_copy(x, y)
+        return
+    if op == 'add':
+        _math._add(y, value, y)
+        return
+    if op == 'sub':
+        _math._subtract(y, value, y)
+        return
+    if op == 'max':
+        cupy.maximum(y, value, y)
+        return
+    if op == 'min':
+        cupy.minimum(y, value, y)
+        return
+    if op == 'and':
+        cupy.bitwise_and(y, value, y)
+        return
+    if op == 'or':
+        cupy.bitwise_or(y, value, y)
+        return
+    if op == 'xor':
+        cupy.bitwise_xor(y, value, y)
+        return
+    raise ValueError('this op is not supported')
+
+
+cdef _ndarray_base _diagonal(
+        _ndarray_base a, Py_ssize_t offset=0, Py_ssize_t axis1=0,
+        Py_ssize_t axis2=1):
+    cdef Py_ssize_t ndim = a.ndim
+    if not (-ndim <= axis1 < ndim and -ndim <= axis2 < ndim):
+        raise numpy.AxisError(
+            'axis1(={0}) and axis2(={1}) must be within range '
+            '(ndim={2})'.format(axis1, axis2, ndim))
+
+    axis1 %= ndim
+    axis2 %= ndim
+    if axis1 < axis2:
+        min_axis, max_axis = axis1, axis2
+    else:
+        min_axis, max_axis = axis2, axis1
+
+    tr = list(range(ndim))
+    del tr[max_axis]
+    del tr[min_axis]
+    if offset >= 0:
+        a = _manipulation._transpose(a, tr + [axis1, axis2])
+    else:
+        a = _manipulation._transpose(a, tr + [axis2, axis1])
+        offset = -offset
+
+    diag_size = max(0, min(a.shape[-2], a.shape[-1] - offset))
+    ret_shape = a.shape[:-2] + (diag_size,)
+    if diag_size == 0:
+        return core.ndarray(ret_shape, dtype=a.dtype)
+
+    a = a[..., :diag_size, offset:offset + diag_size]
+
+    ret = a.view()
+    # TODO(niboshi): Confirm update_x_contiguity flags
+    ret._set_shape_and_strides(
+        a.shape[:-2] + (diag_size,),
+        a.strides[:-2] + (a.strides[-1] + a.strides[-2],),
+        True, True)
+    return ret
+
+
+_prepare_array_indexing = ElementwiseKernel(
+    'T s, S len, S stride',
+    'S out',
+    'S in0 = s, in1 = len;'
+    'out += stride * (in0 - _floor_divide(in0, in1) * in1)',
+    'cupy_prepare_array_indexing')
+
+
+cdef tuple _prepare_multiple_array_indexing(
+    _ndarray_base a, Py_ssize_t start, list slices
+):
+    # slices consist of ndarray
+    cdef list indices = [], shapes = []  # int ndarrays
+    cdef Py_ssize_t i, stop, stride
+    cdef _ndarray_base reduced_idx, s
+
+    for s in slices:
+        if s.dtype.kind == 'b':
+            s = _ndarray_argwhere(s).T
+            indices.extend(s)
+            shapes.append(s.shape[1:])
+        else:
+            indices.append(s)
+            shapes.append(s.shape)
+
+    stop = start + len(indices)
+
+    # br = _manipulation.broadcast(*indices)
+    # indices = list(br.values)
+
+    reduced_idx = core.ndarray(
+        internal._broadcast_shapes(shapes), dtype=numpy.int64)
+    reduced_idx.fill(0)
+    stride = 1
+    i = stop
+    for s in reversed(indices):
+        i -= 1
+        a_shape_i = a._shape[i]
+        # wrap all out-of-bound indices
+        if a_shape_i != 0:
+            _prepare_array_indexing(s, a_shape_i, stride, reduced_idx)
+        stride *= a_shape_i
+
+    return reduced_idx, start, stop
+
+
+cdef _ndarray_base _getitem_multiple(
+        _ndarray_base a, Py_ssize_t start, list slices):
+    reduced_idx, start, stop = _prepare_multiple_array_indexing(
+        a, start, slices)
+    return _take(a, reduced_idx, start, stop)
+
+
+cdef _ndarray_base _add_reduceat(
+        _ndarray_base array, indices, axis, dtype, out):
+    from cupy._sorting import search
+    axis = internal._normalize_axis_index(axis, array.ndim)
+    indices = cupy.append(indices, array.shape[axis])
+    shape = [1 if i == axis else dim for i, dim in enumerate(array.shape)]
+    acc = array.cumsum(axis, dtype)
+    acc = cupy.append(cupy.zeros(shape, acc.dtype), acc, axis)
+    mask = indices[:-1] >= indices[1:]
+    mask = mask.reshape(-1, *([1] * (array.ndim - axis - 1)))
+    return search._where_ufunc(
+        mask,
+        array.take(indices[:-1], axis),
+        acc.take(indices[1:], axis) - acc.take(indices[:-1], axis),
+        out
+    )
diff --git a/cupy/_core/_routines_linalg.pxd b/cupy/_core/_routines_linalg.pxd
new file mode 100644
index 0000000..43dcdd1
--- /dev/null
+++ b/cupy/_core/_routines_linalg.pxd
@@ -0,0 +1,27 @@
+from cupy._core._carray cimport shape_t
+from cupy._core.core cimport _ndarray_base
+
+
+cpdef compute_type_to_str(compute_type)
+
+cpdef get_compute_type(dtype)
+
+cpdef _ndarray_base dot(_ndarray_base a, _ndarray_base b, _ndarray_base out=*)
+
+cpdef _ndarray_base tensordot_core(
+    _ndarray_base a, _ndarray_base b, _ndarray_base out, Py_ssize_t n,
+    Py_ssize_t m, Py_ssize_t k, const shape_t& ret_shape)
+
+cpdef _ndarray_base matmul(
+    _ndarray_base a, _ndarray_base b, _ndarray_base out=*)
+
+
+cpdef enum:
+    COMPUTE_TYPE_TBD = 0
+    COMPUTE_TYPE_DEFAULT = 1   # default
+    COMPUTE_TYPE_PEDANTIC = 2  # disable algorithmic optimizations
+    COMPUTE_TYPE_FP16 = 3      # allow converting inputs to FP16
+    COMPUTE_TYPE_FP32 = 4      # allow converting inputs to FP32
+    COMPUTE_TYPE_FP64 = 5      # allow converting inputs to FP64
+    COMPUTE_TYPE_BF16 = 6      # allow converting inputs to BF16
+    COMPUTE_TYPE_TF32 = 7      # allow converting inputs to TF32
diff --git a/cupy/_core/_routines_linalg.pyx b/cupy/_core/_routines_linalg.pyx
new file mode 100644
index 0000000..0458e98
--- /dev/null
+++ b/cupy/_core/_routines_linalg.pyx
@@ -0,0 +1,1067 @@
+import math
+import os
+import warnings
+
+import cython
+import numpy
+
+import cupy
+from cupy._core._kernel import ElementwiseKernel
+from cupy._core._reduction import ReductionKernel
+from cupy._core._ufuncs import elementwise_copy
+import cupy._core.core as core
+
+
+from libc.stdint cimport intptr_t
+
+from cupy._core cimport _accelerator
+from cupy._core._carray cimport shape_t
+from cupy._core._dtype cimport to_cuda_dtype
+from cupy._core._scalar cimport get_typename
+from cupy._core.core cimport _internal_ascontiguousarray
+from cupy._core.core cimport _ndarray_init
+from cupy._core.core cimport ascontiguousarray
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport _memory_range
+from cupy._core cimport _routines_manipulation as _manipulation
+from cupy._core cimport _routines_math as _math
+from cupy.cuda cimport device
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda.libs cimport cublas
+
+
+cdef extern from '../../cupy_backends/cupy_complex.h':
+    ctypedef struct cuComplex 'cuComplex':
+        float x, y
+
+    ctypedef struct cuDoubleComplex 'cuDoubleComplex':
+        double x, y
+
+
+cdef int _cuda_runtime_version = -1
+
+
+cdef list compute_types = [COMPUTE_TYPE_TBD,  # float16
+                           COMPUTE_TYPE_TBD,  # float32
+                           COMPUTE_TYPE_TBD]  # float64
+cdef dict compute_type_str = {
+    0: 'COMPUTE_TYPE_TBD',
+    1: 'COMPUTE_TYPE_DEFAULT',
+    2: 'COMPUTE_TYPE_PEDANTIC',
+    3: 'COMPUTE_TYPE_FP16',
+    4: 'COMPUTE_TYPE_FP32',
+    5: 'COMPUTE_TYPE_FP64',
+    6: 'COMPUTE_TYPE_BF16',
+    7: 'COMPUTE_TYPE_TF32',
+}
+
+
+cpdef int to_compute_type_index(dtype) except -1:
+    cdef str dtype_char = numpy.dtype(dtype).char
+    if dtype_char == 'e':
+        return 0
+    elif dtype_char in 'fF':
+        return 1
+    elif dtype_char in 'dD':
+        return 2
+    else:
+        raise TypeError('dtype is not supported: {}'.format(dtype))
+
+
+cpdef set_compute_type(dtype, compute_type):
+    global compute_types
+    if compute_type in (COMPUTE_TYPE_TBD, COMPUTE_TYPE_DEFAULT,
+                        COMPUTE_TYPE_PEDANTIC, COMPUTE_TYPE_FP16,
+                        COMPUTE_TYPE_FP32, COMPUTE_TYPE_FP64):
+        compute_types[to_compute_type_index(dtype)] = compute_type
+    elif compute_type in (COMPUTE_TYPE_BF16, COMPUTE_TYPE_TF32):
+        if int(device.get_compute_capability()) >= 80:
+            compute_types[to_compute_type_index(dtype)] = compute_type
+        else:
+            warnings.warn('COMPUTE_TYPE_BF16 and COMPUTE_TYPE_TF32 are only '
+                          'available on GPUs with compute capability 8.0 or '
+                          'higher. COMPUTE_TYPE_DEFAULT will be used instead.')
+            compute_types[to_compute_type_index(dtype)] = COMPUTE_TYPE_DEFAULT
+    else:
+        raise ValueError('Unknown compute type: {}'.format(compute_type))
+
+
+cpdef compute_type_to_str(compute_type):
+    if compute_type in compute_type_str:
+        return compute_type_str[compute_type]
+    else:
+        return compute_type
+
+
+def _tensordot_core_int_kernel_impl(config, dtype, code, name):
+    # This code is based in the GEMM implementation from MAGMA
+    # (http://icl.cs.utk.edu/magma/)
+    code = '''
+#define fetch(arr, col, m, n, bound) arr[min(n*col + m, bound)]
+
+template<typename T>
+__device__ void _tensordot_core_int_kernel_impl(
+        int M, int N, int K,
+        const T* A,
+        const T* B,
+        T * C)
+{
+    int idx = threadIdx.x;
+    int idy = threadIdx.y;
+
+    int idt = DIM_X * idy + idx;
+
+    int idxA = idt % DIM_XA;
+    int idyA = idt / DIM_XA;
+
+    int idxB = idt % DIM_XB;
+    int idyB = idt / DIM_XB;
+
+    int blx = blockIdx.x;
+    int bly = blockIdx.y;
+
+    __shared__ T sA[BLK_K][BLK_M + 1];
+    __shared__ T sB[BLK_N][BLK_K + 1];
+
+    // registers for the innermost loop
+    T rC[THR_N][THR_M];
+    T rA[THR_M];
+    T rB[THR_N];
+
+    T ra[BLK_K / DIM_YA][BLK_M / DIM_XA];
+    T rb[BLK_N / DIM_YB][BLK_K / DIM_XB];
+
+    const T* offs_dA = A + blx * BLK_M       + idyA * M + idxA;
+    int boundA = (M * (K - 1) + M) - (blx * BLK_M + idyA * M + idxA) - 1;
+    const T* offs_dB = B + bly * BLK_N * K + idyB * K + idxB;
+    int boundB = (K * (N - 1) + K) - (bly * BLK_N * K + idyB * K + idxB) - 1;
+
+    int m, n, k, kk;
+
+    #pragma unroll
+    for (n = 0; n < THR_N; n++) {
+        #pragma unroll
+        for (m = 0 ; m < THR_M; m++) {
+            rC[n][m] = 0;
+        }
+    }
+
+    // blockwise transpose to transpose load
+    #pragma unroll
+    for (n = 0; n < BLK_K; n += DIM_YA) {
+        #pragma unroll
+        for (m = 0; m < BLK_M; m += DIM_XA) {
+            sA[n + idyA][m + idxA] = fetch(offs_dA, M, m, n, boundA);
+        }
+    }
+    // blockwise transpose to transpose load
+    #pragma unroll
+    for (n = 0; n < BLK_N; n += DIM_YB) {
+        #pragma unroll
+        for (m = 0; m < BLK_K; m += DIM_XB) {
+            sB[n + idyB][m + idxB] = fetch(offs_dB, K, m, n, boundB);
+        }
+    }
+    __syncthreads();
+
+    for (kk = 0; kk < K - BLK_K; kk += BLK_K)
+    {
+        offs_dA += BLK_K * M;
+        boundA -= BLK_K * M;
+        offs_dB += BLK_K;
+        boundB -= BLK_K;
+
+        #pragma unroll
+        for (n = 0; n < BLK_K / DIM_YA; n++) {
+            #pragma unroll
+            for (m = 0; m < BLK_M / DIM_XA; m++) {
+                ra[n][m] = fetch(offs_dA, M, m * DIM_XA, n * DIM_YA, boundA);
+            }
+        }
+
+        #pragma unroll
+        for (n = 0; n < BLK_N / DIM_YB; n++) {
+            #pragma unroll
+            for (m = 0; m < BLK_K / DIM_XB; m++) {
+                rb[n][m] = fetch(offs_dB, K, m * DIM_XB, n * DIM_YB, boundB);
+            }
+        }
+
+        // multiply
+        #pragma unroll
+        for (k = 0; k < BLK_K; k++)
+        {
+            #pragma unroll
+            for (m = 0; m < THR_M; m++) {
+                rA[m] = sA[k][m * DIM_X + idx];
+            }
+
+            #pragma unroll
+            for (n = 0; n < THR_N; n++) {
+                rB[n] = sB[n * DIM_Y + idy][k];
+            }
+
+            // HIP is strange...
+            #ifdef __HIP_DEVICE_COMPILE__
+            __syncthreads();
+            #endif
+
+            #pragma unroll
+            for (n = 0; n < THR_N; n++) {
+                #pragma unroll
+                for (m = 0; m < THR_M; m++) {
+                    rC[n][m] += rA[m] * rB[n];
+                }
+            }
+        }
+        __syncthreads();
+
+        // store A regs->smem
+        #pragma unroll
+        for (n = 0; n < BLK_K / DIM_YA; n++)
+        {
+            #pragma unroll
+            for (m = 0; m < BLK_M / DIM_XA; m++)
+            {
+                sA[n * DIM_YA + idyA][m * DIM_XA + idxA] = ra[n][m];
+            }
+        }
+
+        #pragma unroll
+        for (n = 0; n < BLK_N / DIM_YB; n++)
+        {
+            #pragma unroll
+            for (m = 0; m < BLK_K / DIM_XB; m++)
+            {
+                sB[n * DIM_YB + idyB][m * DIM_XB + idxB] = rb[n][m];
+            }
+        }
+        __syncthreads();
+    }
+
+    // Multiply last full (BLK_K) or partial block of columns of A and
+    // rows of B.
+    // It's okay that m,n exceed matrix bounds as all work is in registers
+    // or shared memory, and out-of-bounds rC[n][m] will not be saved later.
+
+    kk = K - kk;
+    #pragma unroll
+    for (k = 0; k < kk; k++)
+    {
+        #pragma unroll
+        for (m = 0; m < THR_M; m++) {
+            rA[m] = sA[k][m * DIM_X + idx];
+        }
+
+        #pragma unroll
+        for (n = 0; n < THR_N; n++) {
+            rB[n] = sB[n * DIM_Y + idy][k];
+        }
+
+        // HIP is strange...
+        #ifdef __HIP_DEVICE_COMPILE__
+        __syncthreads();
+        #endif
+
+        #pragma unroll
+        for (n = 0; n < THR_N; n++) {
+            #pragma unroll
+            for (m = 0; m < THR_M; m++) {
+                rC[n][m] += rA[m] * rB[n];
+            }
+        }
+    }
+
+    #pragma unroll
+    for (n = 0; n < THR_N; n++) {
+        int coord_dCn = bly * BLK_N + n * DIM_Y + idy;
+        #pragma unroll
+        for (m = 0; m < THR_M; m++) {
+            int coord_dCm = blx * BLK_M + m * DIM_X + idx;
+            if (coord_dCm < M && coord_dCn < N) {
+                C[coord_dCn * M + coord_dCm] = rC[n][m];
+            }
+        }
+    }
+}
+''' + code
+    for k, v in config:
+        code = '#define ' + k + ' ' + str(v) + '\n' + code
+    name_expressions = [f'{name}<bool>',
+                        f'{name}<signed char>',
+                        f'{name}<unsigned char>',
+                        f'{name}<short>',
+                        f'{name}<unsigned short>',
+                        f'{name}<int>',
+                        f'{name}<unsigned int>',
+                        f'{name}<long>',
+                        f'{name}<unsigned long>',
+                        f'{name}<long long>',
+                        f'{name}<unsigned long long>']
+    mod = cupy.RawModule(code=code, options=('--std=c++11',),
+                         name_expressions=name_expressions)
+    ker = mod.get_function(name + '<' + get_typename(dtype) + '>')
+    return ker
+
+
+@cupy._util.memoize(for_each_device=True)
+def _tensordot_core_int_kernel(config, dtype):
+    code = '''
+template<typename T>
+__global__ void _tensordot_core_int_kernel(
+        int M, int N, int K,
+        const T* A,
+        const T* B,
+        T * C)
+{
+    _tensordot_core_int_kernel_impl(M, N, K, A, B, C);
+}
+'''
+    name = '_tensordot_core_int_kernel'
+    return _tensordot_core_int_kernel_impl(config, dtype, code, name)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _tensordot_core_int_batched_kernel(config, dtype):
+    code = '''
+template<typename T>
+__global__ void _tensordot_core_int_batched_kernel(
+        int M, int N, int K,
+        const T* A[], const T* B[],
+        T* C[])
+{
+    int batchid = blockIdx.z;
+    _tensordot_core_int_kernel_impl(
+        M, N, K, A[batchid], B[batchid], C[batchid]
+    );
+}
+'''
+    name = '_tensordot_core_int_batched_kernel'
+    return _tensordot_core_int_kernel_impl(config, dtype, code, name)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _tensordot_core_int_strided_batched_kernel(config, dtype):
+    code = '''
+template<typename T>
+__global__ void _tensordot_core_int_strided_batched_kernel(
+        int M, int N, int K,
+        const T* A, long long strideA,
+        const T* B, long long strideB,
+        T * C, long long strideC)
+{
+    int batchid = blockIdx.z;
+    _tensordot_core_int_kernel_impl(
+        M, N, K,
+        &A[batchid * strideA],
+        &B[batchid * strideB],
+        &C[batchid * strideC]
+    );
+}
+'''
+    name = '_tensordot_core_int_strided_batched_kernel'
+    return _tensordot_core_int_kernel_impl(config, dtype, code, name)
+
+
+cdef tuple _integral_tensordot_core_config():
+    # TODO(leofang): autotune the tuning parameters here? See the discussion
+    # in this thread: https://groups.google.com/a/icl.utk.edu/g/magma-user/c/igc66uduTfI  # NOQA
+    dim_x=16
+    dim_y=16
+    blk_m=128
+    blk_n=128
+    blk_k=2
+    dim_xa=128
+    dim_ya=2
+    dim_xb=2
+    dim_yb=128
+    config = (('DIM_X', dim_x), ('DIM_Y', dim_y),
+              ('BLK_M', blk_m), ('BLK_N', blk_n), ('BLK_K', blk_k),
+              ('DIM_XA', dim_xa), ('DIM_YA', dim_ya),
+              ('DIM_XB', dim_xb), ('DIM_YB', dim_yb),
+              ('THR_M', blk_m // dim_x), ('THR_N', blk_n // dim_y))
+    return config, dim_x, dim_y, blk_m, blk_n
+
+
+cdef _ndarray_base _integral_tensordot_core(
+        _ndarray_base a, _ndarray_base b, _ndarray_base out, Py_ssize_t m,
+        Py_ssize_t n, Py_ssize_t k, str dtype, const shape_t& ret_shape):
+
+    config, dim_x, dim_y, blk_m, blk_n = _integral_tensordot_core_config()
+    kern = _tensordot_core_int_kernel(config, dtype)
+    args = (m, n, k, a, b, out)
+    grid = (int(math.ceil(m / blk_m)), int(math.ceil(n / blk_n)), 1)
+    block = (dim_x, dim_y, 1)
+    kern(grid, block, args=args)
+    return out
+
+
+cdef _ndarray_base _integral_tensordot_core_batched(
+        _ndarray_base a, _ndarray_base b, _ndarray_base out, Py_ssize_t m,
+        Py_ssize_t n, Py_ssize_t k, str dtype, Py_ssize_t batch_count):
+
+    config, dim_x, dim_y, blk_m, blk_n = _integral_tensordot_core_config()
+    kern = _tensordot_core_int_batched_kernel(config, dtype)
+    block = (dim_x, dim_y, 1)
+    matPtrA = _mat_ptrs(a)
+    matPtrB = _mat_ptrs(b)
+    matPtrOut = _mat_ptrs(out)
+    max_batch_count = 65000
+    for i in range(0, batch_count, max_batch_count):
+        ibatch = min(max_batch_count, batch_count - i)
+        args = (
+            m, n, k, matPtrA[i:i + ibatch], matPtrB[i:i + ibatch],
+            matPtrOut[i:i + ibatch])
+        grid = (int(math.ceil(m / blk_m)), int(math.ceil(n / blk_n)), ibatch)
+        kern(grid, block, args=args)
+    return out
+
+
+cdef _ndarray_base _integral_tensordot_core_strided_batched(
+        _ndarray_base a, _ndarray_base b, _ndarray_base out, Py_ssize_t m,
+        Py_ssize_t n, Py_ssize_t k, str dtype, Py_ssize_t batch_count):
+
+    config, dim_x, dim_y, blk_m, blk_n = _integral_tensordot_core_config()
+    kern = _tensordot_core_int_strided_batched_kernel(config, dtype)
+    block = (dim_x, dim_y, 1)
+    a = a.reshape((-1,) + a.shape[-2:])
+    b = b.reshape((-1,) + b.shape[-2:])
+    out = out.reshape((-1,) + out.shape[-2:])
+    strideA = _get_stride_for_strided_batched_gemm(a)
+    strideB = _get_stride_for_strided_batched_gemm(b)
+    strideOut = _get_stride_for_strided_batched_gemm(out)
+    max_batch_count = 65000
+    for i in range(0, batch_count, max_batch_count):
+        ibatch = min(max_batch_count, batch_count - i)
+        args = (
+            m, n, k, a[i:i + ibatch], strideA, b[i:i + ibatch], strideB,
+            out[i:i + ibatch], strideOut)
+        grid = (int(math.ceil(m / blk_m)), int(math.ceil(n / blk_n)), ibatch)
+        kern(grid, block, args=args)
+    return out
+
+
+cdef _tensordot_core_mul_sum = ReductionKernel(
+    'S x, T y', 'U out',
+    'static_cast<U>(x) * static_cast<U>(y)',
+    'a + b', 'out = a', '0', '_tensordot_core_mul_sum')
+
+
+cpdef get_compute_type(dtype):
+    global compute_types
+    cdef int index = to_compute_type_index(dtype)
+    if compute_types[index] == COMPUTE_TYPE_TBD:
+        compute_type = COMPUTE_TYPE_DEFAULT
+        dtype_char = numpy.dtype(dtype).char
+        if dtype_char in 'fF' and int(os.getenv('CUPY_TF32', '0')) > 0:
+            compute_type = COMPUTE_TYPE_TF32
+        set_compute_type(dtype, compute_type)
+    return compute_types[index]
+
+
+@cython.profile(False)
+cpdef inline tuple _mat_to_cublas_contiguous(
+        _ndarray_base a, Py_ssize_t trans):
+    assert a.ndim == 2
+    if a._f_contiguous:
+        # builtin max function is not used for Cython 0.23
+        lda = a._strides[1] // a.itemsize
+        if lda < a._shape[0]:
+            lda = a._shape[0]
+        return a, trans, lda
+    if not a._c_contiguous:
+        a = a.copy()
+    return a, 1 - trans, a._strides[0] // a.itemsize
+
+
+cpdef _ndarray_base dot(
+        _ndarray_base a, _ndarray_base b, _ndarray_base out=None):
+    cdef Py_ssize_t a_ndim, b_ndim, a_axis, b_axis, n, m, k
+    cdef bint input_a_is_vec, input_b_is_vec
+    cdef shape_t ret_shape, shape
+
+    a_ndim = a._shape.size()
+    b_ndim = b._shape.size()
+
+    if out is not None:
+        if numpy.result_type(a.dtype, b.dtype) != out.dtype:
+            raise ValueError('Not supported dtype combination.')
+        if not out._c_contiguous:
+            raise ValueError('Output array must be C-contiguous')
+
+    if a_ndim == 0 or b_ndim == 0:
+        return _math._multiply(a, b, out=out)
+
+    input_a_is_vec = a_ndim == 1
+    input_b_is_vec = b_ndim == 1
+    if input_a_is_vec:
+        shape.clear()
+        shape.push_back(1)
+        shape.push_back(a.size)
+        a = _manipulation._reshape(a, shape)
+        a_ndim = 2
+    if input_b_is_vec:
+        shape.clear()
+        shape.push_back(b.size)
+        shape.push_back(1)
+        b = _manipulation._reshape(b, shape)
+        b_ndim = 2
+
+    a_axis = a_ndim - 1
+    b_axis = b_ndim - 2
+
+    if a._shape[a_axis] != b._shape[b_axis]:
+        raise ValueError('Axis dimension mismatch')
+
+    if a_axis:
+        a = _manipulation.rollaxis(a, a_axis, 0)
+    if b_axis:
+        b = _manipulation.rollaxis(b, b_axis, 0)
+
+    k = a._shape[0]
+    if k != 0:
+        m = b.size // k
+        n = a.size // k
+    else:
+        # When k==0, the function must return a matrix filled with zero
+        # like NumPy.
+        m = 0
+        n = 0
+
+    if not input_a_is_vec:
+        ret_shape.insert(ret_shape.end(), a._shape.begin() + 1, a._shape.end())
+    if not input_b_is_vec:
+        ret_shape.insert(ret_shape.end(), b._shape.begin() + 1, b._shape.end())
+    if out is not None:
+        # TODO(kataoka): Make the condition strict
+        if k != 0 and out.size != n * m:
+            raise ValueError('Output array has an invalid size')
+
+    return tensordot_core(a, b, out, n, m, k, ret_shape)
+
+
+cpdef _ndarray_base tensordot_core(
+        _ndarray_base a, _ndarray_base b, _ndarray_base out, Py_ssize_t n,
+        Py_ssize_t m, Py_ssize_t k, const shape_t& ret_shape):
+    # out, if specified, must be C-contiguous and have correct shape.
+    cdef shape_t shape
+    cdef Py_ssize_t transa, transb, lda, ldb
+    cdef intptr_t handle
+    cdef _ndarray_base copy_to_out = None
+    cdef str dtype = a.dtype.char
+    cdef int compute_capability = int(device.get_compute_capability())
+    if dtype != b.dtype.char:
+        dtype = numpy.promote_types(dtype, b.dtype).char
+    if not a.size or not b.size:
+        if out is None:
+            out = _ndarray_init(cupy.ndarray, ret_shape, dtype, None)
+        out.fill(0)
+        return out
+
+    if out is not None:
+        assert out.flags.c_contiguous and out.dtype == dtype
+    cdef int ace
+    if m == 1 and n == 1:
+        if out is None:
+            out = _ndarray_init(cupy.ndarray, ret_shape, dtype, None)
+        c = _manipulation._reshape(out, ())
+        for ace in _accelerator._routine_accelerators:
+            # fast path using CUB or cuTENSOR
+            if ace in (_accelerator.ACCELERATOR_CUB,
+                       _accelerator.ACCELERATOR_CUTENSOR):
+                (a.ravel() * b.ravel()).sum(out=c)
+                break
+        else:
+            _tensordot_core_mul_sum(a.ravel(), b.ravel(), out=c)
+        return out
+
+    a = a.astype(dtype, order='K', casting=None, subok=None, copy=False)
+    b = b.astype(dtype, order='K', casting=None, subok=None, copy=False)
+    # It copies the operands if needed
+    if a._shape.size() != 2 or a._shape[0] != k or a._shape[1] != n:
+        shape.clear()
+        shape.push_back(k)
+        shape.push_back(n)
+        a = _manipulation._reshape(a, shape)
+    if b._shape.size() != 2 or b._shape[0] != k or b._shape[1] != m:
+        shape.clear()
+        shape.push_back(k)
+        shape.push_back(m)
+        b = _manipulation._reshape(b, shape)
+
+    # Be careful that cuBLAS uses the FORTRAN-order matrix representation.
+    # Matrix-Matrix product A^T * B
+    # c is C-contiguous while cuBLAS assumes F-contiguous inputs, so we
+    # compute C^T = B^T * A here.
+    a, transa, lda = _mat_to_cublas_contiguous(a, 0)
+    b, transb, ldb = _mat_to_cublas_contiguous(b, 1)
+
+    if out is None:
+        out = c = _ndarray_init(cupy.ndarray, ret_shape, dtype, None)
+    elif (
+        _memory_range.may_share_bounds(out, a)
+        or _memory_range.may_share_bounds(out, b)
+    ):
+        copy_to_out = c = _ndarray_init(cupy.ndarray, ret_shape, dtype, None)
+    else:
+        c = out
+
+    if c._shape.size() != 2 or c._shape[0] != n or c._shape[1] != m:
+        c = c.view()
+        c.shape = (n, m)
+
+    if dtype not in 'efdFD':
+        if transa:
+            a = a.T
+            a = _internal_ascontiguousarray(a)
+        if transb:
+            b = _internal_ascontiguousarray(b)
+        _integral_tensordot_core(b, a, c, m, n, k, dtype, ret_shape)
+        if copy_to_out is not None:
+            elementwise_copy(copy_to_out, out)
+        return out
+
+    global _cuda_runtime_version
+    if _cuda_runtime_version < 0:
+        _cuda_runtime_version = runtime.runtimeGetVersion()
+
+    if (
+        not runtime._is_hip_environment and
+        _cuda_runtime_version >= 11000 and
+        compute_capability >= 50
+    ):
+        tensordot_core_v11(transb, transa, m, n, k, b, ldb, a, lda, c, m)
+        if copy_to_out is not None:
+            elementwise_copy(copy_to_out, out)
+        return out
+
+    handle = device.get_cublas_handle()
+    if dtype == 'e':
+        coef_dtype = 'f'
+    else:
+        coef_dtype = dtype
+    one = numpy.array(1.0, dtype=coef_dtype)
+    zero = numpy.array(0.0, dtype=coef_dtype)
+    if runtime._is_hip_environment and dtype == 'e':
+        # On HIP, SgemmEx does not work for half precision
+        dtype = 'f'
+        a = a.astype(dtype, order='K', casting=None, subok=None, copy=True)
+        b = b.astype(dtype, order='K', casting=None, subok=None, copy=True)
+        c = _ndarray_init(cupy.ndarray, ret_shape, dtype, None)
+        copy_to_out = c
+        warnings.warn('On ROCm/HIP, there is no specialized API to handle '
+                      'half precision floating numbers, so the computation '
+                      'will be done by casting to single precision')
+    if dtype == 'e':
+        use_tensor_core = (not runtime._is_hip_environment and
+                           _cuda_runtime_version >= 9000 and
+                           compute_capability >= 70)
+        if use_tensor_core:
+            cublas.setMathMode(handle, cublas.CUBLAS_TENSOR_OP_MATH)
+            cublas.gemmEx(
+                handle, <int>transb, <int> transa, <int>m, <int>n, <int>k,
+                one.ctypes.data, b.data.ptr, runtime.CUDA_R_16F, <int>ldb,
+                a.data.ptr, runtime.CUDA_R_16F, <int>lda, zero.ctypes.data,
+                c.data.ptr, runtime.CUDA_R_16F, <int>m, runtime.CUDA_R_32F,
+                cublas.CUBLAS_GEMM_DEFAULT_TENSOR_OP)
+            cublas.setMathMode(handle, cublas.CUBLAS_DEFAULT_MATH)
+        else:
+            cublas.sgemmEx(
+                handle, <int>transb, <int> transa, <int>m, <int>n, <int>k,
+                one.ctypes.data, b.data.ptr, runtime.CUDA_R_16F, <int>ldb,
+                a.data.ptr, runtime.CUDA_R_16F, <int>lda, zero.ctypes.data,
+                c.data.ptr, runtime.CUDA_R_16F, <int>m)
+    elif dtype == 'f':
+        cublas.sgemmEx(
+            handle, <int>transb, <int> transa, <int>m, <int>n, <int>k,
+            one.ctypes.data, b.data.ptr, runtime.CUDA_R_32F, <int>ldb,
+            a.data.ptr, runtime.CUDA_R_32F, <int>lda, zero.ctypes.data,
+            c.data.ptr, runtime.CUDA_R_32F, <int>m)
+    elif dtype == 'd':
+        cublas.dgemm(
+            handle, <int>transb, <int>transa, <int>m, <int>n, <int>k,
+            one.ctypes.data, b.data.ptr, <int>ldb, a.data.ptr, <int>lda,
+            zero.ctypes.data, c.data.ptr, <int>m)
+    elif dtype == 'F':
+        cublas.cgemm(
+            handle, <int>transb, <int>transa, <int>m, <int>n, <int>k,
+            one.ctypes.data, b.data.ptr, <int>ldb, a.data.ptr, <int>lda,
+            zero.ctypes.data, c.data.ptr, <int>m)
+    elif dtype == 'D':
+        cublas.zgemm(
+            handle, <int>transb, <int>transa, <int>m, <int>n, <int>k,
+            one.ctypes.data, b.data.ptr, <int>ldb, a.data.ptr, <int>lda,
+            zero.ctypes.data, c.data.ptr, <int>m)
+    else:
+        raise ValueError('Invalid dtype: %s' % str(dtype))
+    if copy_to_out is not None:
+        elementwise_copy(copy_to_out, out)
+    return out
+
+
+cpdef _ndarray_base tensordot_core_v11(
+        Py_ssize_t transa, Py_ssize_t transb, Py_ssize_t m, Py_ssize_t n,
+        Py_ssize_t k, _ndarray_base a, Py_ssize_t lda, _ndarray_base b,
+        Py_ssize_t ldb, _ndarray_base c, Py_ssize_t ldc):
+    cdef float one_f, zero_f
+    cdef double one_d, zero_d
+    cdef cuComplex one_F, zero_F
+    cdef cuDoubleComplex one_D, zero_D
+    cdef size_t one_ptr, zero_ptr
+
+    cdef int compute_capability = int(device.get_compute_capability())
+    cdef int compute_type = get_compute_type(c.dtype)
+    cdef int cublas_compute_type = -1
+    if c.dtype.char in 'efF':
+        if compute_type == COMPUTE_TYPE_PEDANTIC:
+            cublas_compute_type = cublas.CUBLAS_COMPUTE_32F_PEDANTIC
+        elif compute_type == COMPUTE_TYPE_TF32 and c.dtype.char in 'fF':
+            cublas_compute_type = cublas.CUBLAS_COMPUTE_32F_FAST_TF32
+        else:
+            cublas_compute_type = cublas.CUBLAS_COMPUTE_32F
+    elif c.dtype.char in 'dD':
+        if compute_type == COMPUTE_TYPE_PEDANTIC:
+            cublas_compute_type = cublas.CUBLAS_COMPUTE_64F_PEDANTIC
+        else:
+            cublas_compute_type = cublas.CUBLAS_COMPUTE_64F
+    else:
+        raise ValueError('Invalid dtype: {}'.format(c.dtype))
+
+    cdef int algo = cublas.CUBLAS_GEMM_DEFAULT
+    if ((compute_capability >= 80) or
+            (compute_capability >= 70 and c.dtype == 'e')):
+        algo = cublas.CUBLAS_GEMM_DEFAULT_TENSOR_OP
+
+    if cublas_compute_type in (cublas.CUBLAS_COMPUTE_32F,
+                               cublas.CUBLAS_COMPUTE_32F_PEDANTIC,
+                               cublas.CUBLAS_COMPUTE_32F_FAST_TF32):
+        if c.dtype.char in 'efd':
+            one_f = 1
+            zero_f = 0
+            one_ptr = <size_t>&one_f
+            zero_ptr = <size_t>&zero_f
+        else:
+            one_F = cuComplex(1, 0)
+            zero_F = cuComplex(0, 0)
+            one_ptr = <size_t>&one_F
+            zero_ptr = <size_t>&zero_F
+    elif cublas_compute_type in (cublas.CUBLAS_COMPUTE_64F,
+                                 cublas.CUBLAS_COMPUTE_64F_PEDANTIC):
+        if c.dtype.char in 'efd':
+            one_d = 1
+            zero_d = 0
+            one_ptr = <size_t>&one_d
+            zero_ptr = <size_t>&zero_d
+        else:
+            one_D = cuDoubleComplex(1, 0)
+            zero_D = cuDoubleComplex(0, 0)
+            one_ptr = <size_t>&one_D
+            zero_ptr = <size_t>&zero_D
+    else:
+        raise ValueError('Invalid cublas compute type: {}'
+                         .format(cublas_compute_type))
+
+    cdef int a_cuda_dtype = to_cuda_dtype(a.dtype, is_half_allowed=True)
+    cdef int b_cuda_dtype = to_cuda_dtype(b.dtype, is_half_allowed=True)
+    cdef int c_cuda_dtype = to_cuda_dtype(c.dtype, is_half_allowed=True)
+    cdef intptr_t handle = device.get_cublas_handle()
+    cublas.gemmEx(
+        handle, <int>transa, <int>transb, <int>m, <int>n, <int>k, one_ptr,
+        a.data.ptr, a_cuda_dtype, <int>lda, b.data.ptr, b_cuda_dtype, <int>ldb,
+        zero_ptr, c.data.ptr, c_cuda_dtype, <int>ldc, cublas_compute_type,
+        algo)
+
+
+cdef Py_ssize_t _get_stride_for_strided_batched_gemm(
+        _ndarray_base a) except? 0:
+    cdef int ndim = a._shape.size()
+    assert ndim > 2
+    assert a._c_contiguous
+    return a._shape[ndim - 2] * a._shape[ndim - 1]
+
+
+cdef _mat_ptrs_kernel = ElementwiseKernel(
+    'T base, T stride', 'T out',
+    'out = base + _ind.get()[_ind.ndim - 1] * stride', 'cupy_mat_ptrs',
+    reduce_dims=False)
+
+
+cpdef _ndarray_base _mat_ptrs(_ndarray_base a):
+    """Creates an array of pointers to matrices
+    Args:
+        a: A batch of matrices on GPU.
+           shape: (A, B, C) -> A ptrs to mat of size (B, C)
+           shape: (A_1, ..., A_N, B, C) -> A_1*...*A_N ptrs to mat of
+                  size (B, C)
+    Returns:
+        GPU array of pointers to matrices.
+    """
+    cdef int ndim = a._shape.size()
+    assert ndim > 2
+    cdef _ndarray_base idx
+    idx = _mat_ptrs_kernel(
+        a.data.ptr, a._strides[0],
+        core.ndarray((a._shape[0],), dtype=numpy.uintp))
+
+    for i in range(1, ndim - 2):
+        idx = _mat_ptrs_kernel(
+            idx[:, None], a._strides[i],
+            core.ndarray((idx.size, a._shape[i]), dtype=numpy.uintp))
+        idx = idx.ravel()
+    return idx
+
+
+cpdef _ndarray_base matmul(
+        _ndarray_base a, _ndarray_base b, _ndarray_base out=None):
+    """Matrix product of two arrays.
+
+    Returns the matrix product of two arrays and is the implementation of
+    the `@` operator introduced in Python 3.5 following PEP465.
+
+    The main difference against cupy.dot are the handling of arrays with more
+    than 2 dimensions. For more information see :func:`numpy.matmul`.
+
+    Args:
+        a (cupy.ndarray): The left argument.
+        b (cupy.ndarray): The right argument.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: Output array.
+
+    .. seealso:: :func:`numpy.matmul`
+
+    """
+
+    cdef Py_ssize_t i, n, m, ka, kb, a_sh, b_sh, c_sh, ldc
+    cdef Py_ssize_t batchCount, a_part_outshape, b_part_outshape
+    cdef int orig_a_ndim, orig_b_ndim, a_ndim, b_ndim, ndim
+    cdef _ndarray_base ap, bp, cp, c_view
+    cdef bint use_broadcast
+
+    orig_a_ndim = a._shape.size()
+    orig_b_ndim = b._shape.size()
+    if orig_a_ndim == 0 or orig_b_ndim == 0:
+        raise ValueError('Scalar operands are not allowed, use \'*\' instead')
+
+    ndim = max(orig_a_ndim, orig_b_ndim)
+    if ndim <= 2:
+        if out is None:
+            return dot(a, b, out)
+        ret_dtype = numpy.promote_types(a.dtype, b.dtype)
+        if out._c_contiguous and ret_dtype == out.dtype:
+            return dot(a, b, out)
+        c = _ndarray_init(cupy.ndarray, out._shape, dtype=ret_dtype, obj=None)
+        dot(a, b, c)
+        elementwise_copy(c, out)
+        return out
+
+    orig_a = a
+    orig_b = b
+    a_part_outshape = b_part_outshape = 0
+    if orig_a_ndim == 1:
+        a = _manipulation._reshape(a, (1, a.size))
+    else:
+        a = a.view()
+        a_part_outshape = a._shape[orig_a_ndim - 2]
+    if orig_b_ndim == 1:
+        b = _manipulation._reshape(b, (b.size, 1))
+        ldc = 1
+    else:
+        b = b.view()
+        b_part_outshape = ldc = b._shape[orig_b_ndim - 1]
+
+    # expand dims
+    a_ndim = a._shape.size()
+    b_ndim = b._shape.size()
+    if a_ndim < ndim:
+        # TODO(niboshi): Confirm update_x_contiguity flags
+        a._set_shape_and_strides(
+            (1,) * (ndim - a_ndim) + a.shape,
+            (0,) * (ndim - a_ndim) + a.strides,
+            True, True)
+    if b_ndim < ndim:
+        # TODO(niboshi): Confirm update_x_contiguity flags
+        b._set_shape_and_strides(
+            (1,) * (ndim - b_ndim) + b.shape,
+            (0,) * (ndim - b_ndim) + b.strides,
+            True, True)
+
+    ret_dtype = numpy.promote_types(a.dtype, b.dtype)
+    dtype = ret_dtype
+    if dtype.char == 'e':
+        dtype = numpy.dtype('f')
+
+    a = ascontiguousarray(a, dtype)
+    b = ascontiguousarray(b, dtype)
+
+    # broadcast
+    batchCount = 1  # batchCount = numpy.prod(out_shape[:-2])
+    out_shape = []
+    use_broadcast = False
+    for i in range(0, ndim - 2):
+        a_sh = a._shape[i]
+        b_sh = b._shape[i]
+        if a_sh != b_sh and a_sh != 1 and b_sh != 1:
+            raise ValueError(
+                'operands could not be broadcast together with '
+                'remapped shapes')
+
+        if a_sh == 0 or b_sh == 0:
+            c_sh = 0
+        else:
+            c_sh = max(a_sh, b_sh)
+        batchCount *= c_sh
+        out_shape.append(c_sh)
+        if a_sh == 1 and c_sh > 1:
+            a._strides[i] = 0
+            a._shape[i] = c_sh
+            a._c_contiguous = a._f_contiguous = False
+            use_broadcast = True
+
+        if b_sh == 1 and c_sh > 1:
+            b._strides[i] = 0
+            b._shape[i] = c_sh
+            b._c_contiguous = b._f_contiguous = False
+            use_broadcast = True
+
+    if orig_a_ndim != 1:
+        out_shape.append(a_part_outshape)
+    if orig_b_ndim != 1:
+        out_shape.append(b_part_outshape)
+
+    # (A B)^T = B^T A^T
+    a, b = b, a
+
+    ka = a._shape[ndim - 2]
+    lda = n = a._shape[ndim - 1]
+    m = b._shape[ndim - 2]
+    ldb = kb = b._shape[ndim - 1]
+
+    if ka != kb:
+        raise ValueError(
+            'shapes ({}) and ({}) not aligned'.format(
+                ','.join([str(_) for _ in orig_a.shape]),
+                ','.join([str(_) for _ in orig_b.shape])))
+
+    if out is not None and out.shape != tuple(out_shape):
+        raise ValueError('Output array has an invalid size')
+
+    if a.size == 0 or b.size == 0:
+        if out is None:
+            return cupy.zeros(out_shape, ret_dtype)
+        else:
+            out.fill(0)
+            return out
+
+    if (
+        out is not None and out.dtype == dtype and out.flags.c_contiguous
+        and not _memory_range.may_share_bounds(out, a)
+        and not _memory_range.may_share_bounds(out, b)
+    ):
+        c = out
+    else:
+        c = core.ndarray(out_shape, dtype=dtype)
+        if out is None:
+            if dtype == ret_dtype:
+                out = c
+            else:
+                out = core.ndarray(out_shape, dtype=ret_dtype)
+
+    if orig_a_ndim == 1 or orig_b_ndim == 1:
+        c_view = c.view()
+        if orig_b_ndim == 1:
+            c_view._shape.push_back(1)
+            c_view._strides.push_back(0)
+        if orig_a_ndim == 1:
+            c_view._shape.insert(c_view._shape.end() - 1, 1)
+            c_view._strides.insert(c_view._strides.end() - 1, 0)
+        assert c_view._c_contiguous
+        c_view._update_f_contiguity()
+    else:
+        c_view = c
+
+    if dtype.char not in 'efdFD':
+        if not use_broadcast:
+            _integral_tensordot_core_strided_batched(
+                a, b, c_view, n, m, ka, dtype.char, batchCount)
+        else:
+            _integral_tensordot_core_batched(
+                a, b, c_view, n, m, ka, dtype.char, batchCount)
+        if out is not c:
+            elementwise_copy(c, out)
+        return out
+
+    global _cuda_runtime_version
+    if _cuda_runtime_version < 0:
+        _cuda_runtime_version = runtime.runtimeGetVersion()
+
+    cdef intptr_t handle = device.get_cublas_handle()
+    cdef int cuda_dtype = to_cuda_dtype(dtype)
+    cdef int algo = cublas.CUBLAS_GEMM_DEFAULT
+
+    one = numpy.array(1, dtype=dtype)
+    zero = numpy.array(0, dtype=dtype)
+    if not use_broadcast:
+        strideA = _get_stride_for_strided_batched_gemm(a)
+        strideB = _get_stride_for_strided_batched_gemm(b)
+        strideC = _get_stride_for_strided_batched_gemm(c_view)
+        if dtype.char in 'fFdD':
+            cublas.gemmStridedBatchedEx(
+                handle,
+                0,  # transa
+                0,  # transb
+                n, m, ka, one.ctypes.data,
+                a.data.ptr, cuda_dtype, lda, strideA,
+                b.data.ptr, cuda_dtype, ldb, strideB,
+                zero.ctypes.data,
+                c_view.data.ptr, cuda_dtype, ldc, strideC,
+                batchCount, cuda_dtype, algo)
+        else:
+            raise TypeError(dtype, a.dtype, b.dtype)
+    else:
+        ap = _mat_ptrs(a)
+        bp = _mat_ptrs(b)
+        cp = _mat_ptrs(c_view)
+        if dtype == numpy.float32:
+            cublas.sgemmBatched(
+                handle,
+                0,  # transa
+                0,  # transb
+                n, m, ka, one.ctypes.data,
+                ap.data.ptr, lda,
+                bp.data.ptr, ldb,
+                zero.ctypes.data, cp.data.ptr, ldc, batchCount)
+        elif dtype == numpy.float64:
+            cublas.dgemmBatched(
+                handle,
+                0,  # transa
+                0,  # transb
+                n, m, ka, one.ctypes.data,
+                ap.data.ptr, lda,
+                bp.data.ptr, ldb,
+                zero.ctypes.data, cp.data.ptr, ldc, batchCount)
+        elif dtype == numpy.complex64:
+            cublas.cgemmBatched(
+                handle,
+                0,  # transa
+                0,  # transb
+                n, m, ka, one.ctypes.data,
+                ap.data.ptr, lda,
+                bp.data.ptr, ldb,
+                zero.ctypes.data, cp.data.ptr, ldc, batchCount)
+        elif dtype == numpy.complex128:
+            cublas.zgemmBatched(
+                handle,
+                0,  # transa
+                0,  # transb
+                n, m, ka, one.ctypes.data,
+                ap.data.ptr, lda,
+                bp.data.ptr, ldb,
+                zero.ctypes.data, cp.data.ptr, ldc, batchCount)
+        else:
+            raise TypeError(dtype, a.dtype, b.dtype)
+
+    if out is not c:
+        elementwise_copy(c, out)
+    return out
diff --git a/cupy/_core/_routines_logic.pxd b/cupy/_core/_routines_logic.pxd
new file mode 100644
index 0000000..f5c49dc
--- /dev/null
+++ b/cupy/_core/_routines_logic.pxd
@@ -0,0 +1,11 @@
+from cupy._core.core cimport _ndarray_base
+
+
+cdef _ndarray_base _ndarray_all(_ndarray_base self, axis, out, keepdims)
+cdef _ndarray_base _ndarray_any(_ndarray_base self, axis, out, keepdims)
+cdef _ndarray_base _ndarray_greater(_ndarray_base self, other)
+cdef _ndarray_base _ndarray_greater_equal(_ndarray_base self, other)
+cdef _ndarray_base _ndarray_less(_ndarray_base self, other)
+cdef _ndarray_base _ndarray_less_equal(_ndarray_base self, other)
+cdef _ndarray_base _ndarray_equal(_ndarray_base self, other)
+cdef _ndarray_base _ndarray_not_equal(_ndarray_base self, other)
diff --git a/cupy/_core/_routines_logic.pyx b/cupy/_core/_routines_logic.pyx
new file mode 100644
index 0000000..5f8e51c
--- /dev/null
+++ b/cupy/_core/_routines_logic.pyx
@@ -0,0 +1,141 @@
+from cupy._core._kernel import create_ufunc
+from cupy._core._reduction import create_reduction_func
+
+from cupy._core.core cimport _ndarray_base
+
+
+cdef _ndarray_base _ndarray_all(_ndarray_base self, axis, out, keepdims):
+    return _all(self, axis=axis, out=out, keepdims=keepdims)
+
+
+cdef _ndarray_base _ndarray_any(_ndarray_base self, axis, out, keepdims):
+    return _any(self, axis=axis, out=out, keepdims=keepdims)
+
+
+cdef _ndarray_base _ndarray_greater(_ndarray_base self, other):
+    return _greater(self, other)
+
+
+cdef _ndarray_base _ndarray_greater_equal(_ndarray_base self, other):
+    return _greater_equal(self, other)
+
+
+cdef _ndarray_base _ndarray_less(_ndarray_base self, other):
+    return _less(self, other)
+
+
+cdef _ndarray_base _ndarray_less_equal(_ndarray_base self, other):
+    return _less_equal(self, other)
+
+
+cdef _ndarray_base _ndarray_equal(_ndarray_base self, other):
+    return _equal(self, other)
+
+
+cdef _ndarray_base _ndarray_not_equal(_ndarray_base self, other):
+    return _not_equal(self, other)
+
+
+cdef _all = create_reduction_func(
+    'cupy_all',
+    ('?->?', 'B->?', 'h->?', 'H->?', 'i->?', 'I->?', 'l->?', 'L->?',
+     'q->?', 'Q->?', 'e->?', 'f->?', 'd->?', 'F->?', 'D->?'),
+    ('in0 != type_in0_raw(0)', 'a & b', 'out0 = a', 'bool'),
+    'true', '')
+
+
+cdef _any = create_reduction_func(
+    'cupy_any',
+    ('?->?', 'B->?', 'h->?', 'H->?', 'i->?', 'I->?', 'l->?', 'L->?',
+     'q->?', 'Q->?', 'e->?', 'f->?', 'd->?', 'F->?', 'D->?'),
+    ('in0 != type_in0_raw(0)', 'a | b', 'out0 = a', 'bool'),
+    'false', '')
+
+
+cpdef create_comparison(name, op, doc='', no_complex_dtype=True):
+
+    if no_complex_dtype:
+        ops = ('??->?', 'bb->?', 'BB->?', 'hh->?', 'HH->?', 'ii->?', 'II->?',
+               'll->?', 'LL->?', 'qq->?', 'QQ->?', 'ee->?', 'ff->?', 'dd->?')
+    else:
+        ops = ('??->?', 'bb->?', 'BB->?', 'hh->?', 'HH->?', 'ii->?', 'II->?',
+               'll->?', 'LL->?', 'qq->?', 'QQ->?', 'ee->?', 'ff->?', 'dd->?',
+               'FF->?', 'DD->?')
+
+    return create_ufunc(
+        'cupy_' + name,
+        ops,
+        'out0 = in0 %s in1' % op,
+        doc=doc)
+
+
+cdef _greater = create_comparison(
+    'greater', '>',
+    '''Tests elementwise if ``x1 > x2``.
+
+    .. seealso:: :data:`numpy.greater`
+
+    ''',
+    no_complex_dtype=False)
+
+
+cdef _greater_equal = create_comparison(
+    'greater_equal', '>=',
+    '''Tests elementwise if ``x1 >= x2``.
+
+    .. seealso:: :data:`numpy.greater_equal`
+
+    ''',
+    no_complex_dtype=False)
+
+
+cdef _less = create_comparison(
+    'less', '<',
+    '''Tests elementwise if ``x1 < x2``.
+
+    .. seealso:: :data:`numpy.less`
+
+    ''',
+    no_complex_dtype=False)
+
+
+cdef _less_equal = create_comparison(
+    'less_equal', '<=',
+    '''Tests elementwise if ``x1 <= x2``.
+
+    .. seealso:: :data:`numpy.less_equal`
+
+    ''',
+    no_complex_dtype=False)
+
+
+cdef _equal = create_comparison(
+    'equal', '==',
+    '''Tests elementwise if ``x1 == x2``.
+
+    .. seealso:: :data:`numpy.equal`
+
+    ''',
+    no_complex_dtype=False)
+
+
+cdef _not_equal = create_comparison(
+    'not_equal', '!=',
+    '''Tests elementwise if ``x1 != x2``.
+
+    .. seealso:: :data:`numpy.equal`
+
+    ''',
+    no_complex_dtype=False)
+
+
+# Variables to expose to Python
+# (cythonized data cannot be exposed to Python, even with cpdef.)
+all = _all
+any = _any
+greater = _greater
+greater_equal = _greater_equal
+less = _less
+less_equal = _less_equal
+equal = _equal
+not_equal = _not_equal
diff --git a/cupy/_core/_routines_manipulation.pxd b/cupy/_core/_routines_manipulation.pxd
new file mode 100644
index 0000000..ae46965
--- /dev/null
+++ b/cupy/_core/_routines_manipulation.pxd
@@ -0,0 +1,40 @@
+from libcpp cimport vector
+
+from cupy._core._carray cimport shape_t
+from cupy._core._carray cimport strides_t
+from cupy._core.core cimport _ndarray_base
+
+
+cdef class broadcast:
+    cdef:
+        readonly tuple values
+        readonly tuple shape
+        readonly Py_ssize_t size
+        readonly Py_ssize_t nd
+
+
+cdef _ndarray_shape_setter(_ndarray_base self, newshape)
+cdef _ndarray_base _ndarray_reshape(_ndarray_base self, tuple shape, order)
+cdef _ndarray_base _ndarray_transpose(_ndarray_base self, tuple axes)
+cdef _ndarray_base _ndarray_swapaxes(
+    _ndarray_base self, Py_ssize_t axis1, Py_ssize_t axis2)
+cdef _ndarray_base _ndarray_flatten(_ndarray_base self, order)
+cdef _ndarray_base _ndarray_ravel(_ndarray_base self, order)
+cdef _ndarray_base _ndarray_squeeze(_ndarray_base self, axis)
+cdef _ndarray_base _ndarray_repeat(_ndarray_base self, repeats, axis)
+
+cpdef _ndarray_base _expand_dims(_ndarray_base a, tuple axis)
+cpdef _ndarray_base moveaxis(_ndarray_base a, source, destination)
+cpdef _ndarray_base _move_single_axis(
+    _ndarray_base a, Py_ssize_t source, Py_ssize_t destination)
+cpdef _ndarray_base rollaxis(
+    _ndarray_base a, Py_ssize_t axis, Py_ssize_t start=*)
+cpdef _ndarray_base broadcast_to(_ndarray_base array, shape)
+cpdef _ndarray_base _reshape(_ndarray_base self, const shape_t &shape_spec)
+cpdef _ndarray_base _T(_ndarray_base self)
+cpdef _ndarray_base _transpose(
+    _ndarray_base self, const vector.vector[Py_ssize_t] &axes)
+cpdef _ndarray_base _concatenate(
+    list arrays, Py_ssize_t axis, tuple shape, _ndarray_base out, str casting)
+cpdef _ndarray_base concatenate_method(
+    tup, int axis, _ndarray_base out=*, dtype=*, casting=*)
diff --git a/cupy/_core/_routines_manipulation.pyx b/cupy/_core/_routines_manipulation.pyx
new file mode 100644
index 0000000..50cb8a3
--- /dev/null
+++ b/cupy/_core/_routines_manipulation.pyx
@@ -0,0 +1,885 @@
+# distutils: language = c++
+import functools
+
+import numpy
+
+from cupy._core._kernel import ElementwiseKernel
+from cupy._core._ufuncs import elementwise_copy
+import cupy._core.core as core
+
+cimport cpython  # NOQA
+cimport cython  # NOQA
+from libcpp cimport vector
+
+from cupy._core._dtype cimport get_dtype, _raise_if_invalid_cast
+from cupy._core cimport core
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport internal
+from cupy._core._kernel cimport _check_peer_access, _preprocess_args
+
+from cupy.cuda import device
+
+
+@cython.final
+cdef class broadcast:
+    """Object that performs broadcasting.
+
+    CuPy actually uses this class to support broadcasting in various
+    operations. Note that this class does not provide an iterator.
+
+    Args:
+        arrays (tuple of arrays): Arrays to be broadcasted.
+
+    Attributes:
+        ~broadcast.shape (tuple of ints): The broadcasted shape.
+        nd (int): Number of dimensions of the broadcasted shape.
+        ~broadcast.size (int): Total size of the broadcasted shape.
+        values (list of arrays): The broadcasted arrays.
+
+    .. seealso:: :class:`numpy.broadcast`
+
+    """
+
+    def __init__(self, *arrays):
+        cdef shape_t shape
+        cdef list val = list(arrays)
+        internal._broadcast_core(val, shape)
+        self.values = tuple(val)
+        self.shape = tuple(shape)
+        self.nd = <Py_ssize_t>shape.size()
+        self.size = internal.prod(shape)
+
+
+# _ndarray_base members
+
+
+cdef _ndarray_shape_setter(_ndarray_base self, newshape):
+    cdef shape_t shape, strides
+    if not cpython.PySequence_Check(newshape):
+        newshape = (newshape,)
+    shape = internal.infer_unknown_dimension(newshape, self.size)
+    _get_strides_for_nocopy_reshape(self, shape, strides)
+    if strides.size() != shape.size():
+        raise AttributeError(
+            'Incompatible shape for in-place modification. Use `.reshape()` '
+            'to make a copy with the desired shape.')
+    self._set_shape_and_strides(shape, strides, False, True)
+
+
+cdef _ndarray_base _ndarray_reshape(_ndarray_base self, tuple shape, order):
+    cdef int order_char = internal._normalize_order(order, False)
+
+    if len(shape) == 1 and cpython.PySequence_Check(shape[0]):
+        shape = tuple(shape[0])
+
+    if order_char == b'A':
+        if self._f_contiguous and not self._c_contiguous:
+            order_char = b'F'
+        else:
+            order_char = b'C'
+    if order_char == b'C':
+        return _reshape(self, shape)
+    else:
+        # TODO(grlee77): Support order within _reshape instead
+
+        # The Fortran-ordered case is equivalent to:
+        #     1.) reverse the axes via transpose
+        #     2.) C-ordered reshape using reversed shape
+        #     3.) reverse the axes via transpose
+        return _T(_reshape(_T(self), shape[::-1]))
+
+
+cdef _ndarray_base _ndarray_transpose(_ndarray_base self, tuple axes):
+    if len(axes) == 0:
+        return _T(self)
+    if len(axes) == 1:
+        a = axes[0]
+        if a is None:
+            return _T(self)
+        elif cpython.PySequence_Check(a):
+            axes = tuple(a)
+    return _transpose(self, axes)
+
+
+cdef _ndarray_base _ndarray_swapaxes(
+        _ndarray_base self, Py_ssize_t axis1, Py_ssize_t axis2):
+    cdef Py_ssize_t ndim = self.ndim
+    cdef vector.vector[Py_ssize_t] axes
+    if axis1 < -ndim or axis1 >= ndim or axis2 < -ndim or axis2 >= ndim:
+        raise ValueError('Axis out of range')
+    axis1 %= ndim
+    axis2 %= ndim
+    for i in range(ndim):
+        axes.push_back(i)
+    axes[axis1], axes[axis2] = axes[axis2], axes[axis1]
+    return _transpose(self, axes)
+
+
+cdef _ndarray_base _ndarray_flatten(_ndarray_base self, order):
+    cdef int order_char
+    cdef vector.vector[Py_ssize_t] axes
+
+    order_char = internal._normalize_order(order, True)
+    if order_char == b'A':
+        if self._f_contiguous and not self._c_contiguous:
+            order_char = b'F'
+        else:
+            order_char = b'C'
+    if order_char == b'C':
+        return _ndarray_flatten_order_c(self)
+    elif order_char == b'F':
+        return _ndarray_flatten_order_c(_T(self))
+    elif order_char == b'K':
+        axes = _npyiter_k_order_axes(self.strides)
+        return _ndarray_flatten_order_c(_transpose(self, axes))
+
+
+cdef _ndarray_base _ndarray_flatten_order_c(_ndarray_base self):
+    newarray = self.copy(order='C')
+    newarray._shape.assign(<Py_ssize_t>1, self.size)
+    newarray._strides.assign(<Py_ssize_t>1,
+                             <Py_ssize_t>self.itemsize)
+    newarray._c_contiguous = True
+    newarray._f_contiguous = True
+    return newarray
+
+
+cdef vector.vector[Py_ssize_t] _npyiter_k_order_axes(strides_t& strides):
+    # output transpose axes such that
+    # x.flatten(order="K") == x.transpose(axes).flatten(order="C")
+    # by reproducing `npyiter_find_best_axis_ordering`
+    # in numpy/core/src/multiarray/nditer_constr.c
+
+    # Note that `flatten` and `ravel` should use this function for order="K",
+    # while `copy(order="K")` should use `internal._get_strides_for_order_K`.
+    cdef vector.vector[Py_ssize_t] axes
+    cdef Py_ssize_t stride0, stride1
+    cdef int ndim, i0, i1, ipos, k
+    ndim = strides.size()
+    for i0 in reversed(range(ndim)):
+        stride0 = abs(strides[i0])
+        if stride0 == 0:  # ambiguous
+            axes.insert(axes.begin(), i0)
+            continue
+        ipos = 0
+        for k, i1 in enumerate(axes):
+            stride1 = abs(strides[i1])
+            if stride1 == 0:  # ambiguous
+                continue
+            elif stride1 <= stride0:  # shouldswap = false
+                break
+            else:  # shouldswap = true
+                ipos = k + 1
+        axes.insert(axes.begin() + ipos, i0)
+    return axes
+
+
+cdef _ndarray_base _ndarray_ravel(_ndarray_base self, order):
+    cdef int order_char
+    cdef shape_t shape
+    cdef vector.vector[Py_ssize_t] axes
+    shape.push_back(self.size)
+
+    order_char = internal._normalize_order(order, True)
+    if order_char == b'A':
+        if self._f_contiguous and not self._c_contiguous:
+            order_char = b'F'
+        else:
+            order_char = b'C'
+    if order_char == b'C':
+        return _reshape(self, shape)
+    elif order_char == b'F':
+        return _reshape(_T(self), shape)
+    elif order_char == b'K':
+        axes = _npyiter_k_order_axes(self.strides)
+        return _reshape(_transpose(self, axes), shape)
+
+
+cdef _ndarray_base _ndarray_squeeze(_ndarray_base self, axis):
+    cdef vector.vector[char] axis_flags
+    cdef shape_t newshape
+    cdef strides_t newstrides
+    cdef Py_ssize_t ndim, naxes, _axis
+
+    ndim = self._shape.size()
+    axis_flags = vector.vector[char](ndim, 0)
+
+    # Convert axis to boolean flag.
+    if axis is None:
+        for idim in range(ndim):
+            if self._shape[idim] == 1:
+                axis_flags[idim] = 1
+    elif isinstance(axis, tuple):
+        naxes = <Py_ssize_t>len(axis)
+        for i in range(naxes):
+            _axis = internal._normalize_axis_index(<Py_ssize_t>axis[i], ndim)
+            if axis_flags[_axis] == 1:
+                raise ValueError('duplicate value in \'axis\'')
+            axis_flags[_axis] = 1
+    else:
+        _axis = <Py_ssize_t>axis
+        if ndim == 0 and (_axis == 0 or _axis == -1):
+            # Special case letting axis={-1,0} slip through for scalars,
+            # for backwards compatibility reasons.
+            pass
+        else:
+            _axis = internal._normalize_axis_index(_axis, ndim)
+            axis_flags[_axis] = 1
+
+    # Verify that the axes requested are all of size one
+    any_ones = 0
+    for idim in range(ndim):
+        if axis_flags[idim] != 0:
+            if self._shape[idim] == 1:
+                any_ones = 1
+            else:
+                raise ValueError('cannot select an axis to squeeze out '
+                                 'which has size not equal to one')
+
+    # If there were no axes to squeeze out, return the same array
+    if any_ones == 0:
+        return self
+
+    for i in range(ndim):
+        if axis_flags[i] == 0:
+            newshape.push_back(self._shape[i])
+            newstrides.push_back(self._strides[i])
+
+    v = self.view()
+    # TODO(niboshi): Confirm update_x_contiguity flags
+    v._set_shape_and_strides(newshape, newstrides, False, True)
+    return v
+
+
+cdef _ndarray_base _ndarray_repeat(_ndarray_base self, repeats, axis):
+    return _repeat(self, repeats, axis)
+
+
+# exposed
+
+
+cpdef _ndarray_base _expand_dims(_ndarray_base a, tuple axis):
+    cdef vector.vector[Py_ssize_t] normalized_axis
+    cdef out_ndim = a.ndim + len(axis)
+    cdef shape_t a_shape = a.shape, out_shape
+    _normalize_axis_tuple(axis, out_ndim, normalized_axis)
+    out_shape.assign(out_ndim, 0)
+    cdef Py_ssize_t i, j
+    for i in normalized_axis:
+        out_shape[i] = 1
+    j = 0
+    for i in range(out_ndim):
+        if out_shape[i] == 1:
+            continue
+        out_shape[i] = a_shape[j]
+        j += 1
+    return _reshape(a, out_shape)
+
+
+cpdef _ndarray_base moveaxis(_ndarray_base a, source, destination):
+    cdef shape_t src, dest
+    cdef Py_ssize_t ndim = a.ndim
+    _normalize_axis_tuple(source, ndim, src)
+    _normalize_axis_tuple(destination, ndim, dest)
+
+    if src.size() != dest.size():
+        raise ValueError('`source` and `destination` arguments must have '
+                         'the same number of elements')
+
+    cdef vector.vector[Py_ssize_t] order
+    cdef Py_ssize_t i
+    for i in range(ndim):
+        if not _has_element(src, i):
+            order.push_back(i)
+
+    cdef Py_ssize_t d, s
+    for d, s in sorted(zip(dest, src)):
+        order.insert(order.begin() + d, s)
+
+    return _transpose(a, order)
+
+
+cpdef _ndarray_base _move_single_axis(
+        _ndarray_base a, Py_ssize_t source, Py_ssize_t destination):
+    """Like moveaxis, but supporting only integer source and destination."""
+    cdef Py_ssize_t ndim = a.ndim
+    source = internal._normalize_axis_index(source, ndim)
+    destination = internal._normalize_axis_index(destination, ndim)
+
+    if source == destination:
+        return a
+
+    cdef vector.vector[Py_ssize_t] order
+    cdef Py_ssize_t i
+    for i in range(ndim):
+        if i != source:
+            order.push_back(i)
+
+    order.insert(order.begin() + destination, source)
+    return _transpose(a, order)
+
+
+cpdef _ndarray_base rollaxis(
+        _ndarray_base a, Py_ssize_t axis, Py_ssize_t start=0):
+    cdef Py_ssize_t i, ndim = a.ndim
+    cdef vector.vector[Py_ssize_t] axes
+    if axis < 0:
+        axis += ndim
+    if start < 0:
+        start += ndim
+    if not (0 <= axis < ndim and 0 <= start <= ndim):
+        raise ValueError('Axis out of range')
+    if axis < start:
+        start -= 1
+    if axis == start:
+        return a
+    if ndim == 2:
+        return _transpose(a, axes)
+
+    for i in range(ndim):
+        axes.push_back(i)
+    axes.erase(axes.begin() + axis)
+    axes.insert(axes.begin() + start, axis)
+    return _transpose(a, axes)
+
+
+cpdef _ndarray_base _reshape(_ndarray_base self, const shape_t &shape_spec):
+    cdef shape_t shape
+    cdef strides_t strides
+    cdef _ndarray_base newarray
+    shape = internal.infer_unknown_dimension(shape_spec, self.size)
+    if internal.vector_equal(shape, self._shape):
+        return self.view()
+
+    _get_strides_for_nocopy_reshape(self, shape, strides)
+    if strides.size() == shape.size():
+        return self._view(type(self), shape, strides, False, True, self)
+    newarray = self.copy()
+    _get_strides_for_nocopy_reshape(newarray, shape, strides)
+
+    # TODO(niboshi): Confirm update_x_contiguity flags
+    newarray._set_shape_and_strides(shape, strides, False, True)
+    return newarray
+
+
+cpdef _ndarray_base _T(_ndarray_base self):
+    ret = self.view()
+    ret._shape.assign(self._shape.rbegin(), self._shape.rend())
+    ret._strides.assign(self._strides.rbegin(), self._strides.rend())
+    ret._c_contiguous = self._f_contiguous
+    ret._f_contiguous = self._c_contiguous
+    return ret
+
+
+cpdef _ndarray_base _transpose(
+        _ndarray_base self, const vector.vector[Py_ssize_t] &axes):
+    cdef vector.vector[Py_ssize_t] a_axes
+    cdef vector.vector[char] axis_flags
+    cdef Py_ssize_t i, ndim, axis, axes_size
+    cdef bint is_normal = True, is_trans = True
+
+    axes_size = axes.size()
+    if axes_size == 0:
+        return _T(self)
+
+    ndim = self._shape.size()
+    if axes_size != ndim:
+        raise ValueError("axes don't match array")
+
+    axis_flags.resize(ndim, 0)
+    for i in range(axes_size):
+        axis = axes[i]
+        if axis < -ndim or axis >= ndim:
+            raise numpy.AxisError(axis, ndim)
+        axis %= ndim
+        a_axes.push_back(axis)
+        if axis_flags[axis]:
+            raise ValueError('repeated axis in transpose')
+        axis_flags[axis] = 1
+        is_normal &= i == axis
+        is_trans &= ndim - 1 - i == axis
+
+    if is_normal:
+        return self.view()
+    if is_trans:
+        return _T(self)
+
+    ret = self.view()
+    ret._shape.clear()
+    ret._strides.clear()
+    for axis in a_axes:
+        ret._shape.push_back(self._shape[axis])
+        ret._strides.push_back(self._strides[axis])
+    ret._update_contiguity()
+    return ret
+
+
+cpdef array_split(_ndarray_base ary, indices_or_sections, Py_ssize_t axis):
+    cdef Py_ssize_t i, ndim, size, each_size, index, prev, stride
+    cdef Py_ssize_t num_large
+    cdef shape_t shape
+
+    ndim = ary.ndim
+    if -ndim > axis or ndim <= axis:
+        raise IndexError('Axis exceeds ndim')
+    if axis < 0:
+        axis += ndim
+    size = ary._shape[axis]
+
+    if numpy.isscalar(indices_or_sections):
+        each_size = (size - 1) // indices_or_sections
+        num_large = (size - 1) % indices_or_sections + 1
+        indices = [i * each_size + min(i, num_large)
+                   for i in range(1, indices_or_sections)]
+    else:
+        indices = [i if i >= 0 else size + i for i in indices_or_sections]
+
+    if len(indices) == 0:
+        return [ary]
+
+    # Make a copy of shape for each view
+    shape = ary._shape
+
+    prev = 0
+    ret = []
+    stride = ary._strides[axis]
+    if ary.size == 0:
+        stride = 0
+    for index in indices:
+        index = min(index, size)
+        shape[axis] = max(index - prev, 0)
+        v = ary.view()
+        v.data = ary.data + prev * stride
+        # TODO(niboshi): Confirm update_x_contiguity flags
+        v._set_shape_and_strides(shape, ary._strides, True, True)
+        ret.append(v)
+
+        prev = index
+
+    shape[axis] = size - prev
+    v = ary.view()
+    v.data = ary.data + prev * stride
+    # TODO(niboshi): Confirm update_x_contiguity flags
+    v._set_shape_and_strides(shape, ary._strides, True, True)
+    ret.append(v)
+
+    return ret
+
+
+cpdef _ndarray_base broadcast_to(_ndarray_base array, shape):
+    """Broadcast an array to a given shape.
+
+    .. seealso::
+        :func:`cupy.broadcast_to` for full documentation,
+        :meth:`numpy.broadcast_to`
+
+    """
+    shape = tuple(shape) if numpy.iterable(shape) else (shape,)
+    cdef int i, j, ndim = array._shape.size(), length = len(shape)
+    cdef Py_ssize_t sh, a_sh
+    if ndim > length:
+        raise ValueError(
+            'input operand has more dimensions than allowed by the axis '
+            'remapping')
+    cdef shape_t _shape = shape
+    cdef strides_t strides
+    strides.assign(length, 0)
+    for i in range(ndim):
+        j = i + length - ndim
+        sh = _shape[j]
+        a_sh = array._shape[i]
+        if sh == a_sh:
+            strides[j] = array._strides[i]
+        elif a_sh != 1:
+            raise ValueError(
+                'operands could not be broadcast together with shape {} and '
+                'requested shape {}'.format(array.shape, shape))
+
+    view = array.view()
+    # TODO(niboshi): Confirm update_x_contiguity flags
+    view._set_shape_and_strides(_shape, strides, True, True)
+    return view
+
+
+cpdef _ndarray_base _repeat(_ndarray_base a, repeats, axis=None):
+    """Repeat arrays along an axis.
+
+    Args:
+        a (cupy.ndarray): Array to transform.
+        repeats (int, list or tuple): The number of repeats.
+        axis (int): The axis to repeat.
+
+    Returns:
+        cupy.ndarray: Transformed array with repeats.
+
+    .. seealso:: :func:`numpy.repeat`
+
+    """
+    cdef _ndarray_base ret
+
+    if isinstance(repeats, _ndarray_base):
+        raise ValueError(
+            'cupy.ndaray cannot be specified as `repeats` argument.')
+
+    # Scalar and size 1 'repeat' arrays broadcast to any shape, for all
+    # other inputs the dimension must match exactly.
+    cdef bint broadcast = False
+    # numpy.issubdtype(1, numpy.integer) fails with old numpy like 1.13.3.
+    if (isinstance(repeats, int) or
+            (hasattr(repeats, 'dtype') and
+             numpy.issubdtype(repeats, numpy.integer))):
+        if repeats < 0:
+            raise ValueError(
+                '\'repeats\' should not be negative: {}'.format(repeats))
+        broadcast = True
+        repeats = [repeats]
+    elif cpython.PySequence_Check(repeats):
+        for rep in repeats:
+            if rep < 0:
+                raise ValueError(
+                    'all elements of \'repeats\' should not be negative: {}'
+                    .format(repeats))
+        if len(repeats) == 1:
+            broadcast = True
+    else:
+        raise ValueError(
+            '\'repeats\' should be int or sequence: {}'.format(repeats))
+
+    if axis is None:
+        if broadcast:
+            a = _reshape(a, (-1, 1))
+            ret = core.ndarray((a.size, repeats[0]), dtype=a.dtype)
+            if ret.size:
+                elementwise_copy(a, ret)
+            return ret.ravel()
+        else:
+            a = a.ravel()
+            axis = 0
+    else:
+        axis = internal._normalize_axis_index(axis, a.ndim)
+
+    if broadcast:
+        repeats = repeats * a._shape[axis]
+    elif a.shape[axis] != len(repeats):
+        raise ValueError(
+            '\'repeats\' and \'axis\' of \'a\' should be same length: {} != {}'
+            .format(a.shape[axis], len(repeats)))
+
+    ret_shape = list(a.shape)
+    ret_shape[axis] = sum(repeats)
+    ret = core.ndarray(ret_shape, dtype=a.dtype)
+    a_index = [slice(None)] * len(ret_shape)
+    ret_index = list(a_index)
+    offset = 0
+    for i in range(a._shape[axis]):
+        if repeats[i] == 0:
+            continue
+        a_index[axis] = slice(i, i + 1)
+        ret_index[axis] = slice(offset, offset + repeats[i])
+        # convert to tuple because cupy has a indexing bug
+        ret[tuple(ret_index)] = a[tuple(a_index)]
+        offset += repeats[i]
+    return ret
+
+
+cpdef _ndarray_base concatenate_method(
+        tup, int axis, _ndarray_base out=None, dtype=None,
+        casting='same_kind'):
+    cdef int ndim0
+    cdef int i
+    cdef _ndarray_base a, a0
+
+    if dtype is not None:
+        dtype = get_dtype(dtype)
+
+    dev_id = device.get_device_id()
+    arrays = _preprocess_args(dev_id, tup, False)
+
+    # Check if the input is not an empty sequence
+    if len(arrays) == 0:
+        raise ValueError('Cannot concatenate from empty tuple')
+
+    # Check types of the input arrays
+    for o in arrays:
+        if not isinstance(o, _ndarray_base):
+            raise TypeError('Only cupy arrays can be concatenated')
+
+    # Check ndim > 0 for the input arrays
+    for o in arrays:
+        a = o
+        if a._shape.size() == 0:
+            raise TypeError('zero-dimensional arrays cannot be concatenated')
+
+    # Check ndim consistency of the input arrays
+    a0 = arrays[0]
+    ndim0 = a0._shape.size()
+    for o in arrays[1:]:
+        a = o
+        if a._shape.size() != ndim0:
+            raise ValueError(
+                'All arrays to concatenate must have the same ndim')
+
+    # Check shape consistency of the input arrays, and compute the output shape
+    shape0 = a0._shape
+    axis = internal._normalize_axis_index(axis, ndim0)
+    for o in arrays[1:]:
+        a = o
+        for i in range(ndim0):
+            if i != axis and shape0[i] != a._shape[i]:
+                raise ValueError(
+                    'All arrays must have same shape except the axis to '
+                    'concatenate')
+        shape0[axis] += a._shape[axis]
+
+    # Compute the output dtype
+    if out is None:
+        if dtype is None:
+            dtype = a0.dtype
+            have_same_types = True
+            for o in arrays[1:]:
+                have_same_types = have_same_types and (o.dtype == dtype)
+            if not have_same_types:
+                dtype = functools.reduce(
+                    numpy.promote_types, set([a.dtype for a in arrays]))
+    else:
+        if dtype is not None:
+            raise TypeError('concatenate() only takes `out` or `dtype` as an '
+                            'argument, but both were provided.')
+        dtype = out.dtype
+
+    # Check casting rule
+    for o in arrays:
+        _raise_if_invalid_cast(o.dtype, dtype, casting)
+
+    # Prpare the output array
+    shape_t = tuple(shape0)
+    if out is None:
+        out = core.ndarray(shape_t, dtype=dtype)
+    else:
+        if len(out.shape) != len(shape_t):
+            raise ValueError('Output array has wrong dimensionality')
+        if out.shape != shape_t:
+            raise ValueError('Output array is the wrong shape')
+
+    return _concatenate(arrays, axis, shape_t, out, casting)
+
+
+cpdef _ndarray_base _concatenate(
+        list arrays, Py_ssize_t axis, tuple shape, _ndarray_base out,
+        str casting):
+    cdef _ndarray_base a, b
+    cdef Py_ssize_t i, aw, itemsize, axis_size
+    cdef bint all_same_type, same_shape_and_contiguous
+    # If arrays are large, Issuing each copy method is efficient.
+    cdef Py_ssize_t threshold_size = 2 * 1024 * 1024
+
+    dtype = out.dtype
+
+    if len(arrays) > 8:
+        all_same_type = True
+        same_shape_and_contiguous = True
+        axis_size = shape[axis] // len(arrays)
+        total_bytes = 0
+        itemsize = dtype.itemsize
+        for a in arrays:
+            if a.dtype != dtype:
+                all_same_type = False
+                break
+            if same_shape_and_contiguous:
+                same_shape_and_contiguous = (
+                    a._c_contiguous and a._shape[axis] == axis_size)
+            total_bytes += a.size * itemsize
+
+        if all_same_type and total_bytes < threshold_size * len(arrays):
+            return _concatenate_single_kernel(
+                arrays, axis, shape, dtype, same_shape_and_contiguous, out)
+
+    i = 0
+    slice_list = [slice(None)] * len(shape)
+    for a in arrays:
+        aw = a._shape[axis]
+        slice_list[axis] = slice(i, i + aw)
+        b = out[tuple(slice_list)]
+        elementwise_copy(a, b, casting=casting)
+        i += aw
+    return out
+
+
+cpdef Py_ssize_t size(_ndarray_base a, axis=None) except? -1:
+    """Returns the number of elements along a given axis.
+
+    Args:
+        a (ndarray): Input data.
+        axis (int or None): Axis along which the elements are counted.
+            When it is ``None``, it returns the total number of elements.
+
+    Returns:
+        int: Number of elements along the given axis.
+
+    """
+    cdef int index, ndim
+    if axis is None:
+        return a.size
+    else:
+        index = axis
+        ndim = a._shape.size()
+        if index < 0:
+            index += ndim
+        if not 0 <= index < ndim:
+            raise IndexError('index out of range')
+        return a._shape[index]
+
+
+# private
+
+
+cdef bint _has_element(const shape_t &source, Py_ssize_t n):
+    for i in range(source.size()):
+        if source[i] == n:
+            return True
+    return False
+
+
+cdef _get_strides_for_nocopy_reshape(
+        _ndarray_base a, const shape_t &newshape, strides_t &newstrides):
+    cdef Py_ssize_t size, itemsize, ndim, dim, last_stride
+    size = a.size
+    newstrides.clear()
+
+    itemsize = a.itemsize
+    if size == 1:
+        newstrides.assign(<Py_ssize_t>newshape.size(), itemsize)
+        return
+    if size == 0:
+        internal.get_contiguous_strides_inplace(
+            newshape, newstrides, itemsize, True, False)
+        return
+
+    cdef shape_t shape
+    cdef strides_t strides
+    internal.get_reduced_dims(a._shape, a._strides, itemsize, shape, strides)
+
+    ndim = shape.size()
+    dim = 0
+    last_stride = shape[0] * strides[0]
+    for i in range(newshape.size()):
+        size = newshape[i]
+        if size <= 1:
+            newstrides.push_back(last_stride)
+            continue
+        if dim >= ndim or shape[dim] % size != 0:
+            newstrides.clear()
+            break
+        shape[dim] //= size
+        last_stride = shape[dim] * strides[dim]
+        newstrides.push_back(last_stride)
+        if shape[dim] == 1:
+            dim += 1
+
+
+cdef _normalize_axis_tuple(axis, Py_ssize_t ndim, shape_t &ret):
+    """Normalizes an axis argument into a tuple of non-negative integer axes.
+
+    Arguments `argname` and `allow_duplicate` are not supported.
+
+    """
+    if numpy.isscalar(axis):
+        axis = (axis,)
+
+    for ax in axis:
+        ax = internal._normalize_axis_index(ax, ndim)
+        if _has_element(ret, ax):
+            # the message in `numpy.core.numeric.normalize_axis_tuple`
+            raise ValueError('repeated axis')
+        ret.push_back(ax)
+
+
+cdef _ndarray_base _concatenate_single_kernel(
+        list arrays, Py_ssize_t axis, tuple shape, dtype,
+        bint same_shape_and_contiguous, _ndarray_base out):
+    cdef _ndarray_base a, x
+    cdef Py_ssize_t base, cum, ndim
+    cdef int i, j
+    cdef Py_ssize_t[:] ptrs
+    cdef Py_ssize_t[:] cum_sizes
+    cdef Py_ssize_t[:, :] x_strides
+    cdef int device_id = device.get_device_id()
+
+    assert out is not None
+
+    ptrs = numpy.ndarray(len(arrays), numpy.int64)
+    for i, a in enumerate(arrays):
+        _check_peer_access(a, device_id)
+        ptrs[i] = a.data.ptr
+    x = core.array(ptrs)
+
+    if same_shape_and_contiguous:
+        base = internal.prod_sequence(shape[axis:]) // len(arrays)
+        _concatenate_kernel_same_size(x, base, out)
+        return out
+
+    ndim = len(shape)
+    x_strides = numpy.ndarray((len(arrays), ndim), numpy.int64)
+    cum_sizes = numpy.ndarray(len(arrays), numpy.int64)
+    cum = 0
+    for i, a in enumerate(arrays):
+        for j in range(ndim):
+            x_strides[i, j] = <int>a._strides[j]
+        cum_sizes[i] = cum
+        cum += <int>a._shape[axis]
+
+    _concatenate_kernel(
+        x, axis, core.array(cum_sizes), core.array(x_strides), out)
+    return out
+
+
+cdef _concatenate_kernel_same_size = ElementwiseKernel(
+    'raw P x, int64 base',
+    'T y',
+    '''
+    ptrdiff_t middle = i / base;
+    ptrdiff_t top = middle / x.size();
+    ptrdiff_t array_ind = middle - top * x.size();
+    ptrdiff_t offset = i + (top - middle) * base;
+    y = reinterpret_cast<T*>(x[array_ind])[offset];
+    ''',
+    'cupy_concatenate_same_size'
+)
+
+
+cdef _concatenate_kernel = ElementwiseKernel(
+    '''raw P x, int32 axis, raw int64 cum_sizes, raw int64 x_strides''',
+    'T y',
+    '''
+    ptrdiff_t axis_ind = _ind.get()[axis];
+    ptrdiff_t left = 0;
+    ptrdiff_t right = cum_sizes.size();
+
+    while (left < right - 1) {
+      ptrdiff_t m = (left + right) / 2;
+      if (axis_ind < cum_sizes[m]) {
+        right = m;
+      } else {
+        left = m;
+      }
+    }
+
+    ptrdiff_t array_ind = left;
+    axis_ind -= cum_sizes[left];
+    char* ptr = reinterpret_cast<char*>(x[array_ind]);
+    for (int j = _ind.ndim - 1; j >= 0; --j) {
+      ptrdiff_t ind[] = {array_ind, j};
+      ptrdiff_t offset;
+      if (j == axis) {
+        offset = axis_ind;
+      } else {
+        offset = _ind.get()[j];
+      }
+      ptr += x_strides[ind] * offset;
+    }
+
+    y = *reinterpret_cast<T*>(ptr);
+    ''',
+    'cupy_concatenate',
+    reduce_dims=False
+)
diff --git a/cupy/_core/_routines_math.pxd b/cupy/_core/_routines_math.pxd
new file mode 100644
index 0000000..864d178
--- /dev/null
+++ b/cupy/_core/_routines_math.pxd
@@ -0,0 +1,39 @@
+from cupy._core.core cimport _ndarray_base
+
+
+cdef _ndarray_base _ndarray_conj(_ndarray_base self)
+cdef _ndarray_base _ndarray_real_getter(_ndarray_base self)
+cdef _ndarray_base _ndarray_real_setter(_ndarray_base self, value)
+cdef _ndarray_base _ndarray_imag_getter(_ndarray_base self)
+cdef _ndarray_base _ndarray_imag_setter(_ndarray_base self, value)
+cdef _ndarray_base _ndarray_prod(
+    _ndarray_base self, axis, dtype, out, keepdims)
+cdef _ndarray_base _ndarray_sum(_ndarray_base self, axis, dtype, out, keepdims)
+cdef _ndarray_base _ndarray_cumsum(_ndarray_base self, axis, dtype, out)
+cdef _ndarray_base _ndarray_cumprod(_ndarray_base self, axis, dtype, out)
+cdef _ndarray_base _ndarray_clip(_ndarray_base self, a_min, a_max, out)
+
+cpdef _ndarray_base _nansum(_ndarray_base a, axis, dtype, out, keepdims)
+cpdef _ndarray_base _nanprod(_ndarray_base a, axis, dtype, out, keepdims)
+
+cpdef enum scan_op:
+    SCAN_SUM = 0
+    SCAN_PROD = 1
+
+cdef _ndarray_base scan(_ndarray_base a, op, dtype=*, _ndarray_base out=*,
+                        incomplete=*, chunk_size=*)
+cdef object _sum_auto_dtype
+cdef object _add
+cdef object _conj
+cdef object _angle
+cdef object _positive
+cdef object _negative
+cdef object _multiply
+cdef object _divide
+cdef object _power
+cdef object _subtract
+cdef object _true_divide
+cdef object _floor_divide
+cdef object _remainder
+cdef object _absolute
+cdef object _sqrt
diff --git a/cupy/_core/_routines_math.pyx b/cupy/_core/_routines_math.pyx
new file mode 100644
index 0000000..440c8cb
--- /dev/null
+++ b/cupy/_core/_routines_math.pyx
@@ -0,0 +1,1145 @@
+import string
+
+import numpy
+
+import cupy
+from cupy._core._reduction import create_reduction_func
+from cupy._core._kernel import create_ufunc, _get_warpsize
+from cupy._core._scalar import get_typename
+from cupy._core._ufuncs import elementwise_copy
+import cupy._core.core as core
+from cupy._core cimport internal
+from cupy import _util
+
+from cupy_backends.cuda.api cimport runtime
+from cupy._core cimport _accelerator
+from cupy._core._dtype cimport get_dtype
+from cupy._core.core cimport _ndarray_init
+from cupy._core.core cimport compile_with_cache
+from cupy._core.core cimport _ndarray_base
+from cupy.cuda cimport memory
+
+from cupy.cuda import cub
+
+try:
+    import cupy_backends.cuda.libs.cutensor as cuda_cutensor
+except ImportError:
+    cuda_cutensor = None
+
+
+# _ndarray_base members
+
+
+cdef _ndarray_base _ndarray_conj(_ndarray_base self):
+    if self.dtype.kind == 'c':
+        return _conjugate(self)
+    else:
+        return self
+
+
+cdef _ndarray_base _ndarray_real_getter(_ndarray_base self):
+    if self.dtype.kind == 'c':
+        dtype = get_dtype(self.dtype.char.lower())
+        view = core.ndarray.__new__(
+            type(self), shape=self._shape, dtype=dtype, _obj=self,
+            memptr=self.data, strides=self._strides)
+        (<_ndarray_base>view).base = (
+            self.base if self.base is not None else self)
+        return view
+    return self
+
+
+cdef _ndarray_base _ndarray_real_setter(_ndarray_base self, value):
+    elementwise_copy(value, _ndarray_real_getter(self))
+
+
+cdef _ndarray_base _ndarray_imag_getter(_ndarray_base self):
+    cdef memory.MemoryPointer memptr
+    if self.dtype.kind == 'c':
+        dtype = get_dtype(self.dtype.char.lower())
+        memptr = self.data
+        # Make the memory pointer point to the first imaginary element.
+        # Note that even if the array doesn't have a valid memory (e.g. 0-size
+        # array), the resulting array should be a view of the original array,
+        # aligning with NumPy behavior.
+        if memptr.ptr != 0:
+            memptr = memptr + self.dtype.itemsize // 2
+        view = core.ndarray.__new__(
+            type(self), shape=self._shape, dtype=dtype, memptr=memptr,
+            strides=self._strides)
+        (<_ndarray_base>view).base = (
+            self.base if self.base is not None else self)
+        return view
+    new_array = core.ndarray.__new__(type(self), self.shape, dtype=self.dtype)
+    new_array.fill(0)
+    return new_array
+
+
+cdef _ndarray_base _ndarray_imag_setter(_ndarray_base self, value):
+    if self.dtype.kind == 'c':
+        elementwise_copy(value, _ndarray_imag_getter(self))
+    else:
+        raise TypeError('cupy.ndarray does not have imaginary part to set')
+
+
+cdef _ndarray_base _ndarray_prod(
+        _ndarray_base self, axis, dtype, out, keepdims):
+    for accelerator in _accelerator._routine_accelerators:
+        result = None
+        if accelerator == _accelerator.ACCELERATOR_CUB:
+            # result will be None if the reduction is not compatible with CUB
+            result = cub.cub_reduction(
+                self, cub.CUPY_CUB_PROD, axis, dtype, out, keepdims)
+        if (accelerator == _accelerator.ACCELERATOR_CUTENSOR and
+                cuda_cutensor is not None):
+            from cupyx import cutensor
+            result = cutensor._try_reduction_routine(
+                self, axis, dtype, out, keepdims, cuda_cutensor.OP_MUL, 1, 0)
+        if result is not None:
+            return result
+    if dtype is None:
+        return _prod_auto_dtype(self, axis, dtype, out, keepdims)
+    else:
+        return _prod_keep_dtype(self, axis, dtype, out, keepdims)
+
+
+cdef _ndarray_base _ndarray_sum(
+        _ndarray_base self, axis, dtype, out, keepdims):
+    for accelerator in _accelerator._routine_accelerators:
+        result = None
+        if accelerator == _accelerator.ACCELERATOR_CUB:
+            # result will be None if the reduction is not compatible with CUB
+            result = cub.cub_reduction(
+                self, cub.CUPY_CUB_SUM, axis, dtype, out, keepdims)
+        if (accelerator == _accelerator.ACCELERATOR_CUTENSOR and
+                cuda_cutensor is not None):
+            from cupyx import cutensor
+            result = cutensor._try_reduction_routine(
+                self, axis, dtype, out, keepdims, cuda_cutensor.OP_ADD, 1, 0)
+        if result is not None:
+            return result
+
+    if dtype is None:
+        return _sum_auto_dtype(self, axis, dtype, out, keepdims)
+    else:
+        return _sum_keep_dtype(self, axis, dtype, out, keepdims)
+
+
+cdef _ndarray_base _ndarray_cumsum(_ndarray_base self, axis, dtype, out):
+    return cupy.cumsum(self, axis, dtype, out)
+
+
+cdef _ndarray_base _ndarray_cumprod(_ndarray_base self, axis, dtype, out):
+    return cupy.cumprod(self, axis, dtype, out)
+
+
+cdef _ndarray_base _ndarray_clip(_ndarray_base self, a_min, a_max, out):
+    if a_min is None and a_max is None:
+        raise ValueError('array_clip: must set either max or min')
+    kind = self.dtype.kind
+    if a_min is None:
+        if kind == 'f':
+            a_min = self.dtype.type('-inf')
+        elif kind in 'iu':
+            a_min = numpy.iinfo(self.dtype.type).min
+    if a_max is None:
+        if kind == 'f':
+            a_max = self.dtype.type('inf')
+        elif kind in 'iu':
+            a_max = numpy.iinfo(self.dtype.type).max
+    return _clip(self, a_min, a_max, out=out)
+
+
+# private/internal
+
+_op_char = {scan_op.SCAN_SUM: '+', scan_op.SCAN_PROD: '*'}
+_identity = {scan_op.SCAN_SUM: 0, scan_op.SCAN_PROD: 1}
+
+
+@cupy._util.memoize(for_each_device=True)
+def _cupy_bsum_shfl(op, chunk_size, warp_size=32):
+    """Returns a kernel that computes the sum/prod of each thread-block.
+
+    Args:
+        op (int): Operation type. SCAN_SUM or SCAN_PROD.
+        chunk_size (int): Number of array elements processed by a single
+            thread-block.
+        warp_size (int); Warp size.
+
+    Returns:
+        cupy.ElementwiseKernel
+
+    Example:
+        a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        _cupy_bsum(op=SCAN_SUM, chunk_size=4)(a, b, ...)
+        b == [10, 26, 19]
+
+    Note:
+        This uses warp shuffle functions to exchange data in a warp.
+        See the link below for details about warp shuffle functions.
+        https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#warp-shuffle-functions
+    """
+    block_size = chunk_size // 2  # each thread handles two elements
+    in_params = 'raw T a'
+    out_params = 'raw O b'
+    loop_prep = string.Template("""
+        __shared__ O smem[${block_size} / ${warp_size}];
+        const int n_warp = ${block_size} / ${warp_size};
+        const int warp_id = threadIdx.x / ${warp_size};
+        const int lane_id = threadIdx.x % ${warp_size};
+    """).substitute(block_size=block_size, warp_size=warp_size)
+    loop_body = string.Template("""
+        O x = ${identity};
+        if (2*i < a.size()) x = a[2*i];
+        if (2*i + 1 < a.size()) x ${op}= a[2*i + 1];
+        for (int j = 1; j < ${warp_size}; j *= 2) {
+            x ${op}= __shfl_xor_sync(0xffffffff, x, j, ${warp_size});
+        }
+        if (lane_id == 0) smem[warp_id] = x;
+        __syncthreads();
+        if (warp_id == 0) {
+            x = ${identity};
+            if (lane_id < n_warp) x = smem[lane_id];
+            for (int j = 1; j < n_warp; j *= 2) {
+                x ${op}= __shfl_xor_sync(0xffffffff, x, j, ${warp_size});
+            }
+            int block_id = i / ${block_size};
+            if (lane_id == 0) b[block_id] = x;
+        }
+    """).substitute(block_size=block_size, warp_size=warp_size,
+                    op=_op_char[op], identity=_identity[op])
+    return cupy.ElementwiseKernel(in_params, out_params, loop_body,
+                                  'cupy_bsum_shfl', loop_prep=loop_prep)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _cupy_bsum_smem(op, chunk_size, warp_size=32):
+    """Returns a kernel that computes the sum/prod of each thread-block.
+
+    Args:
+        op (int): Operation type. SCAN_SUM or SCAN_PROD.
+        chunk_size (int): Number of array elements processed by a single
+            thread-block.
+        warp_size (int); Warp size.
+
+    Returns:
+        cupy.ElementwiseKernel
+
+    Example:
+        a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        _cupy_bsum(op=SCAN_SUM, chunk_size=4)(a, b, ...)
+        b == [10, 26, 19]
+
+    Note:
+        This uses shared memory to exchange data in a warp.
+    """
+    block_size = chunk_size // 2  # each thread handles two elements
+    in_params = 'raw T a'
+    out_params = 'raw O b'
+    loop_prep = string.Template("""
+        __shared__ O smem1[${block_size}];
+        __shared__ O smem2[${warp_size}];
+        const int n_warp = ${block_size} / ${warp_size};
+        const int warp_id = threadIdx.x / ${warp_size};
+        const int lane_id = threadIdx.x % ${warp_size};
+    """).substitute(block_size=block_size, warp_size=warp_size)
+    loop_body = string.Template("""
+        O x = ${identity};
+        if (2*i < a.size()) x = a[2*i];
+        if (2*i + 1 < a.size()) x ${op}= a[2*i + 1];
+        for (int j = 1; j < ${warp_size}; j *= 2) {
+            smem1[threadIdx.x] = x;          __syncwarp();
+            x ${op}= smem1[threadIdx.x ^ j]; __syncwarp();
+        }
+        if (lane_id == 0) smem2[warp_id] = x;
+        __syncthreads();
+        if (warp_id == 0) {
+            x = ${identity};
+            if (lane_id < n_warp) x = smem2[lane_id];
+            for (int j = 1; j < n_warp; j *= 2) {
+                smem2[lane_id] = x;          __syncwarp();
+                x ${op}= smem2[lane_id ^ j]; __syncwarp();
+            }
+            int block_id = i / ${block_size};
+            if (lane_id == 0) b[block_id] = x;
+        }
+    """).substitute(block_size=block_size, warp_size=warp_size,
+                    op=_op_char[op], identity=_identity[op])
+    return cupy.ElementwiseKernel(in_params, out_params, loop_body,
+                                  'cupy_bsum_smem', loop_prep=loop_prep)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _cupy_scan_naive(op, chunk_size, warp_size=32):
+    """Returns a kernel to compute an inclusive scan.
+
+    It first performs an inclusive scan in each thread-block and then add the
+    scan results for the sum/prod of the chunks.
+
+    Args:
+        op (int): Operation type. SCAN_SUM or SCAN_PROD.
+        chunk_size (int): Number of array elements processed by a single
+            thread-block.
+        warp_size (int); Warp size.
+
+    Returns:
+        cupy.ElementwiseKernel
+
+    Example:
+        b = [10, 36, 55]
+        a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        _cupy_scan(op=SCAN_SUM, chunk_size=4)(b, a, out, ...)
+        out == [1, 3, 6, 10, 15, 21, 28, 36, 45, 55]
+
+    Note:
+        This uses a kind of method called "Naive Parallel Scan" for inclusive
+        scan in each thread-block. See below for details about it.
+        https://developer.nvidia.com/gpugems/gpugems3/part-vi-gpu-computing/chapter-39-parallel-prefix-sum-scan-cuda
+    """
+    in_params = 'raw O b'
+    out_params = 'raw T a, raw O out'
+    loop_prep = string.Template("""
+        __shared__ O smem1[${block_size}];
+        __shared__ O smem2[${warp_size}];
+        const int n_warp = ${block_size} / ${warp_size};
+        const int warp_id = threadIdx.x / ${warp_size};
+        const int lane_id = threadIdx.x % ${warp_size};
+    """).substitute(block_size=chunk_size, warp_size=warp_size)
+    loop_body = string.Template("""
+        O x = ${identity};
+        if (i < a.size()) x = a[i];
+        for (int j = 1; j < ${warp_size}; j *= 2) {
+            smem1[threadIdx.x] = x;  __syncwarp();
+            if (lane_id - j >= 0) x ${op}= smem1[threadIdx.x - j];
+            __syncwarp();
+        }
+        if (lane_id == ${warp_size} - 1) smem2[warp_id] = x;
+        __syncthreads();
+        if (warp_id == 0) {
+            O y = ${identity};
+            if (lane_id < n_warp) y = smem2[lane_id];
+            for (int j = 1; j < n_warp; j *= 2) {
+                smem2[lane_id] = y;  __syncwarp();
+                if (lane_id - j >= 0) y ${op}= smem2[lane_id - j];
+                __syncwarp();
+            }
+            smem2[lane_id] = y;
+        }
+        __syncthreads();
+        if (warp_id > 0) x ${op}= smem2[warp_id - 1];
+        int block_id = i / ${block_size};
+        if (block_id > 0) x ${op}= b[block_id - 1];
+        if (i < a.size()) out[i] = x;
+    """).substitute(block_size=chunk_size, warp_size=warp_size,
+                    op=_op_char[op], identity=_identity[op])
+    return cupy.ElementwiseKernel(in_params, out_params, loop_body,
+                                  'cupy_scan_naive', loop_prep=loop_prep)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _cupy_scan_btree(op, chunk_size, warp_size=32):
+    """Returns a kernel to compute an inclusive scan.
+
+    It first performs an inclusive scan in each thread-block and then add the
+    scan results for the sum/prod of the chunks.
+
+    Args:
+        op (int): Operation type. SCAN_SUM or SCAN_PROD.
+        chunk_size (int): Number of array elements processed by a single
+            thread-block.
+        warp_size (int); Warp size.
+
+    Returns:
+        cupy.ElementwiseKernel
+
+    Example:
+        b = [10, 36, 55]
+        a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        _cupy_scan(op=SCAN_SUM, chunk_size=4)(b, a, out, ...)
+        out == [1, 3, 6, 10, 15, 21, 28, 36, 45, 55]
+
+    Note:
+        This uses a kind of method called "Work-Efficient Parallel Scan" for
+        inclusive scan in each thread-block. See below link for details about
+        "Work-Efficient Parallel Scan".
+        https://developer.nvidia.com/gpugems/gpugems3/part-vi-gpu-computing/chapter-39-parallel-prefix-sum-scan-cuda
+    """
+    in_params = 'raw O b'
+    out_params = 'raw T a, raw O out'
+    loop_prep = string.Template("""
+        __shared__ O smem0[${block_size} + 1];
+        O *smem1 = smem0 + 1;
+        __shared__ O smem2[${warp_size}];
+        const int n_warp = ${block_size} / ${warp_size};
+        const int warp_id = threadIdx.x / ${warp_size};
+        const int lane_id = threadIdx.x % ${warp_size};
+        if (threadIdx.x == 0) smem0[0] = ${identity};
+    """).substitute(block_size=chunk_size, warp_size=warp_size,
+                    identity=_identity[op])
+    loop_body = string.Template("""
+        O x = ${identity};
+        if (i < a.size()) x = a[i];
+        for (int j = 1; j < ${warp_size}; j *= 2) {
+            smem1[threadIdx.x] = x;  __syncwarp();
+            if (lane_id % (2*j) == (2*j)-1) {
+                x ${op}= smem1[threadIdx.x - j];
+            }
+            __syncwarp();
+        }
+        smem1[threadIdx.x] = x;
+        __syncthreads();
+        if (warp_id == 0) {
+            O y = ${identity};
+            if (lane_id < n_warp) {
+                y = smem0[${warp_size} * (lane_id + 1)];
+            }
+            for (int j = 1; j < n_warp; j *= 2) {
+                smem2[lane_id] = y;  __syncwarp();
+                if (lane_id % (2*j) == (2*j)-1) {
+                    y ${op}= smem2[lane_id - j];
+                }
+                __syncwarp();
+            }
+            for (int j = n_warp / 4; j > 0; j /= 2) {
+                smem2[lane_id] = y; __syncwarp();
+                if ((lane_id % (2*j) == j-1) && (lane_id >= 2*j)) {
+                    y ${op}= smem2[lane_id - j];
+                }
+                __syncwarp();
+            }
+            if (lane_id < n_warp) {
+                smem0[${warp_size} * (lane_id + 1)] = y;
+            }
+        }
+        __syncthreads();
+        x = smem0[threadIdx.x];
+        for (int j = ${warp_size} / 2; j > 0; j /= 2) {
+            if (lane_id % (2*j) == j) {
+                x ${op}= smem0[threadIdx.x - j];
+            }
+            __syncwarp();
+            smem0[threadIdx.x] = x;  __syncwarp();
+        }
+        __syncthreads();
+        x = smem1[threadIdx.x];
+        int block_id = i / ${block_size};
+        if (block_id > 0) x ${op}= b[block_id - 1];
+        if (i < a.size()) out[i] = x;
+    """).substitute(block_size=chunk_size, warp_size=warp_size,
+                    op=_op_char[op], identity=_identity[op])
+    return cupy.ElementwiseKernel(in_params, out_params, loop_body,
+                                  'cupy_scan_btree', loop_prep=loop_prep)
+
+
+cdef _ndarray_base scan(
+        _ndarray_base a, op, dtype=None, _ndarray_base out=None,
+        incomplete=False, chunk_size=512):
+    """Return the prefix sum(scan) of the elements.
+
+    Args:
+        a (cupy.ndarray): input array.
+        out (cupy.ndarray): Alternative output array in which to place
+            the result. The same size and same type as the input array(a).
+
+    Returns:
+        cupy.ndarray: A new array holding the result is returned.
+
+    """
+    if a._shape.size() != 1:
+        raise TypeError('Input array should be 1D array.')
+
+    if out is None:
+        if dtype is None:
+            dtype = a.dtype
+        if not incomplete:
+            out = _ndarray_init(cupy.ndarray, a._shape, dtype, None)
+    else:
+        if a.size != out.size:
+            raise ValueError('Provided out is the wrong size')
+        dtype = out.dtype
+    dtype = numpy.dtype(dtype)
+
+    warp_size = _get_warpsize()
+    if runtime._is_hip_environment:
+        if dtype.char in 'iIfdlq':
+            # On HIP, __shfl* supports int, unsigned int, float, double,
+            # long, and long long. The documentation is too outdated and
+            # unreliable; refer to the header at
+            # $ROCM_HOME/include/hip/hcc_detail/device_functions.h
+            bsum_kernel = _cupy_bsum_shfl(op, chunk_size, warp_size)
+        else:
+            bsum_kernel = _cupy_bsum_smem(op, chunk_size, warp_size)
+    else:
+        if dtype.char in 'iIlLqQfd':
+            bsum_kernel = _cupy_bsum_shfl(op, chunk_size, warp_size)
+        else:
+            bsum_kernel = _cupy_bsum_smem(op, chunk_size, warp_size)
+    if dtype.char in 'fdFD':
+        scan_kernel = _cupy_scan_btree(op, chunk_size, warp_size)
+    else:
+        scan_kernel = _cupy_scan_naive(op, chunk_size, warp_size)
+    b_size = (a.size + chunk_size - 1) // chunk_size
+    b = cupy.empty((b_size,), dtype=dtype)
+    size = b.size * chunk_size
+
+    if a.size > chunk_size:
+        bsum_kernel(a, b, size=size // 2, block_size=chunk_size // 2)
+        scan(b, op, dtype=dtype, out=b)
+        if incomplete:
+            return b
+        scan_kernel(b, a, out, size=size, block_size=chunk_size)
+    else:
+        scan_kernel(b, a, out, size=size, block_size=chunk_size)
+
+    return out
+
+
+@_util.memoize(for_each_device=True)
+def _inclusive_batch_scan_kernel(
+        dtype, block_size, op, src_c_cont, out_c_cont):
+    """return Prefix Sum(Scan) cuda kernel
+    for a 2d array over axis 1
+    used for scanning over different axes
+
+    e.g
+    if blocksize > len(src[0])
+    src [[1, 2, 3, 4],
+         [5, 6, 7, 8]]
+    dst [[1, 3, 6, 10],
+         [5, 11, 18, 26]]
+
+    if blocksize < len(src[0])
+    block_size: 2
+    # TODO show partialness
+    src [[1, 2, 3, 4],
+         [5, 6, 7, 8]]
+    dst [[1, 3, 3, 7],
+         [5, 11, 7, 15]]
+
+    Args:
+        dtype: src, dst array type
+        block_size: block_size
+
+    Returns:
+         cupy.cuda.Function: cuda function
+    """
+    op_char = {scan_op.SCAN_SUM: '+', scan_op.SCAN_PROD: '*'}
+    identity = {scan_op.SCAN_SUM: 0, scan_op.SCAN_PROD: 1}
+    name = 'cupy_inclusive_batch_scan_kernel'
+    dtype = get_typename(dtype)
+    source = string.Template("""
+    extern "C" __global__ void ${name}(
+        const CArray<${dtype}, 2, ${src_c_cont}> src,
+        CArray<${dtype}, 2, ${out_c_cont}> dst, int batch_size){
+        long long n = src.size();
+
+        extern __shared__ ${dtype} temp[];
+
+        unsigned int thid = threadIdx.x;
+        unsigned int block = blockIdx.x * blockDim.x;
+
+        unsigned int pad_batch_size = batch_size;
+        bool must_copy = true;
+
+        if (batch_size & (batch_size -1)) {
+            pad_batch_size = 1 << (32 - __clz(batch_size));
+            must_copy = (thid & (pad_batch_size-1)) < batch_size;
+        }
+        if (pad_batch_size > ${block_size}) {
+            int blocks_per_batch = (batch_size - 1) / ${block_size} + 1;
+            pad_batch_size = ${block_size} * blocks_per_batch;
+
+            // Must copy enables for all blocks but the last one in the batch
+            bool last_block = (blockIdx.x + 1) % blocks_per_batch == 0;
+            int remaining_batch = batch_size % ${block_size};
+            if (remaining_batch == 0) {
+                remaining_batch = ${block_size};
+            }
+            must_copy = !last_block || (thid < (remaining_batch));
+        }
+
+        int pad_per_batch = pad_batch_size-batch_size;
+        int n_batches_block = ${block_size} / pad_batch_size;
+
+        unsigned int idx0 = thid + block;
+
+        int batch_id = idx0 / pad_batch_size;
+        idx0 = idx0 - pad_per_batch * batch_id;
+
+        int row = idx0 / batch_size;
+        int col = idx0 % batch_size;
+        const ptrdiff_t idx0_idx[] = {row, col};
+
+        if(idx0 < n){
+            temp[thid] = (must_copy) ? src[idx0_idx] : (${dtype}) ${identity};
+            __syncthreads();
+            if (!n_batches_block) {
+                n_batches_block = 1;
+                pad_batch_size = ${block_size};
+            }
+            for (int j = 0; j < n_batches_block; j++) {
+                int offset = j * pad_batch_size;
+                for (int i = 1; i <= pad_batch_size; i <<= 1) {
+                    int index = ((threadIdx.x + 1) * 2 * i - 1);
+                    int index_block = offset + index;
+                    if (index < (pad_batch_size)){
+                        temp[index_block] ${op}= temp[index_block - i];
+                    }
+                    __syncthreads();
+                }
+                for(int i = pad_batch_size >> 1; i > 0; i >>= 1){
+                    int index = ((threadIdx.x + 1) * 2 * i - 1);
+                    int index_block = offset + index;
+                    if((index + i) < (pad_batch_size)){
+                        temp[index_block + i] ${op}= temp[index_block];
+                    }
+                    __syncthreads();
+                }
+            }
+            if(must_copy){
+                dst[idx0_idx] = temp[thid];
+            }
+        }
+    }
+    """).substitute(name=name, dtype=dtype, block_size=block_size,
+                    op=op_char[op], identity=identity[op],
+                    src_c_cont=src_c_cont, out_c_cont=out_c_cont)
+    module = compile_with_cache(source)
+    return module.get_function(name)
+
+
+@_util.memoize(for_each_device=True)
+def _add_scan_batch_blocked_sum_kernel(dtype, op, block_size, c_cont):
+    name = 'cupy_add_scan_blocked_sum_kernel'
+    dtype = get_typename(dtype)
+    ops = {scan_op.SCAN_SUM: '+', scan_op.SCAN_PROD: '*'}
+    source = string.Template("""
+    extern "C" __global__ void ${name}(CArray<${dtype}, 2, ${c_cont}> src_dst,
+        int batch_size){
+        long long n = src_dst.size();
+
+        unsigned int thid = threadIdx.x;
+        unsigned int block = blockIdx.x * ${block_size};
+
+        unsigned int idx0 = thid + block;
+
+        // Respect padding
+        unsigned int row = idx0 / batch_size;
+        unsigned int col = idx0 % batch_size;
+        int my_block = ${block_size} * (col / ${block_size});
+        const ptrdiff_t dst_idx[] = {row, col};
+        const ptrdiff_t src_idx[] = {row, my_block - 1};
+
+        // Avoid for the first block of every row
+        // This can be tweaked with kernel launch settings
+        bool first = col < ${block_size};
+        bool is_block = (col % (${block_size})) == ${block_size} - 1;
+        if(idx0 < n && !first && !is_block){
+            src_dst[dst_idx] ${op}= src_dst[src_idx];
+        }
+    }
+    """).substitute(name=name, dtype=dtype, op=ops[op], block_size=block_size,
+                    c_cont=c_cont)
+    module = compile_with_cache(source)
+    return module.get_function(name)
+
+
+cdef _ndarray_base _batch_scan_op(
+        _ndarray_base a, scan_op op, _ndarray_base out):
+    batch_size = a.shape[1]
+    # TODO(ecastill) replace this with "_reduction._block_size" once it is
+    # properly exposed
+    block_size = 512
+    # Since we need to pad each batch we spawn more threads as some
+    # of them will be idle
+    # Calc the total number of blocks
+    padded_bs = 1 << ((batch_size - 1).bit_length())
+    if padded_bs > block_size:
+        blocks_per_batch = (batch_size - 1) // block_size + 1
+        padded_bs = block_size * blocks_per_batch
+    padded_size = a.size // batch_size * padded_bs
+
+    cdef int src_cont = int(a.flags.c_contiguous)
+    cdef int out_cont = int(out.flags.c_contiguous)
+    kern_scan = _inclusive_batch_scan_kernel(a.dtype, block_size, op,
+                                             src_cont, out_cont)
+    kern_scan(grid=((padded_size - 1) // (block_size) + 1,),
+              block=(block_size,),
+              args=(a, out, batch_size),
+              shared_mem=a.itemsize * block_size)
+    if batch_size > block_size:
+        blocked_sum = out[:, block_size-1::block_size]
+        _batch_scan_op(blocked_sum, op, blocked_sum)
+        kern_add = _add_scan_batch_blocked_sum_kernel(
+            out.dtype, op, block_size, out_cont)
+        kern_add(
+            grid=((out.size - 1) // (block_size) + 1,),
+            block=(block_size,),
+            args=(out, batch_size))
+    return out
+
+
+cdef _proc_as_batch(_ndarray_base x, int axis, scan_op op):
+    if x.shape[axis] == 0:
+        return cupy.empty_like(x)
+    t = cupy.rollaxis(x, axis, x.ndim)
+    s = t.shape
+    r = t.reshape(-1, x.shape[axis])
+    _batch_scan_op(r, op, r)
+    return cupy.rollaxis(r.reshape(s), x.ndim-1, axis)
+
+
+cpdef scan_core(
+        _ndarray_base a, axis, scan_op op, dtype=None, _ndarray_base out=None):
+    if out is None:
+        if dtype is None:
+            kind = a.dtype.kind
+            if kind == 'b':
+                dtype = numpy.dtype('l')
+            elif kind == 'i' and a.dtype.itemsize < numpy.dtype('l').itemsize:
+                dtype = numpy.dtype('l')
+            elif kind == 'u' and a.dtype.itemsize < numpy.dtype('L').itemsize:
+                dtype = numpy.dtype('L')
+            else:
+                dtype = a.dtype
+        result = None
+    else:
+        if (out.flags.c_contiguous or out.flags.f_contiguous):
+            result = out
+            elementwise_copy(a, result)
+        else:
+            result = a.astype(out.dtype, order='C')
+
+    if axis is None:
+        for accelerator in _accelerator._routine_accelerators:
+            if accelerator == _accelerator.ACCELERATOR_CUB:
+                if result is None:
+                    result = a.astype(dtype, order='C').ravel()
+                # result will be None if the scan is not compatible with CUB
+                if op == scan_op.SCAN_SUM:
+                    cub_op = cub.CUPY_CUB_CUMSUM
+                else:
+                    cub_op = cub.CUPY_CUB_CUMPROD
+                res = cub.cub_scan(result, cub_op)
+                if res is not None:
+                    break
+        else:
+            if result is None:
+                result = scan(a.ravel(), op, dtype=dtype)
+            else:
+                scan(result, op, dtype=dtype, out=result)
+    else:
+        if result is None:
+            result = a.astype(dtype, order='C')
+        axis = internal._normalize_axis_index(axis, a.ndim)
+        result = _proc_as_batch(result, axis, op)
+    # This is for when the original out param was not contiguous
+    if out is not None and out.data != result.data:
+        elementwise_copy(result.reshape(out.shape), out)
+    else:
+        out = result
+    return out
+
+
+# Only for test
+def _scan_for_test(a, out=None):
+    return scan(a, scan_op.SCAN_SUM, dtype=None, out=out)
+
+
+cpdef _ndarray_base _nansum(_ndarray_base a, axis, dtype, out, keepdims):
+    if cupy.iscomplexobj(a):
+        return _nansum_complex_dtype(a, axis, dtype, out, keepdims)
+    elif dtype is None:
+        return _nansum_auto_dtype(a, axis, dtype, out, keepdims)
+    else:
+        return _nansum_keep_dtype(a, axis, dtype, out, keepdims)
+
+
+cpdef _ndarray_base _nanprod(_ndarray_base a, axis, dtype, out, keepdims):
+    if cupy.iscomplexobj(a):
+        return _nanprod_complex_dtype(a, axis, dtype, out, keepdims)
+    elif dtype is None:
+        return _nanprod_auto_dtype(a, axis, dtype, out, keepdims)
+    else:
+        return _nanprod_keep_dtype(a, axis, dtype, out, keepdims)
+
+
+_sum_auto_dtype = create_reduction_func(
+    'cupy_sum',
+    ('?->l', 'b->l', 'B->L', 'h->l', 'H->L', 'i->l', 'I->L', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('in0', 'a + b', 'out0 = type_out0_raw(a)', None), 0)
+
+
+_sum_keep_dtype = create_reduction_func(
+    'cupy_sum_with_dtype',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('in0', 'a + b', 'out0 = type_out0_raw(a)', None), 0)
+
+
+_nansum_auto_dtype = create_reduction_func(
+    'cupy_nansum',
+    ('?->l', 'b->l', 'B->L', 'h->l', 'H->L', 'i->l', 'I->L', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('(in0 == in0) ? in0 : type_in0_raw(0)',
+     'a + b', 'out0 = type_out0_raw(a)', None), 0)
+
+
+_nansum_keep_dtype = create_reduction_func(
+    'cupy_nansum_with_dtype',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('(in0 == in0) ? in0 : type_in0_raw(0)',
+     'a + b', 'out0 = type_out0_raw(a)', None), 0)
+
+
+_nansum_complex_dtype = create_reduction_func(
+    'cupy_nansum_complex_dtype',
+    ('F->F', 'D->D'),
+    ('''
+    type_in0_raw((in0.real() == in0.real()) ? in0.real() : 0,
+                 (in0.imag() == in0.imag()) ? in0.imag() : 0)
+    ''',
+     'a + b', 'out0 = type_out0_raw(a)', None), 0)
+
+
+_prod_auto_dtype = create_reduction_func(
+    'cupy_prod',
+    ('?->l', 'b->l', 'B->L', 'h->l', 'H->L', 'i->l', 'I->L', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('in0', 'a * b', 'out0 = type_out0_raw(a)', None), 1)
+
+
+_prod_keep_dtype = create_reduction_func(
+    'cupy_prod_with_dtype',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('in0', 'a * b', 'out0 = type_out0_raw(a)', None), 1)
+
+
+_nanprod_auto_dtype = create_reduction_func(
+    'cupy_nanprod',
+    ('?->l', 'b->l', 'B->L', 'h->l', 'H->L', 'i->l', 'I->L', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('(in0 == in0) ? in0 : type_in0_raw(1)',
+     'a * b', 'out0 = type_out0_raw(a)', None), 1)
+
+
+_nanprod_keep_dtype = create_reduction_func(
+    'cupy_nanprod_with_dtype',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('(in0 == in0) ? in0 : type_in0_raw(1)',
+     'a * b', 'out0 = type_out0_raw(a)', None), 1)
+
+
+_nanprod_complex_dtype = create_reduction_func(
+    'cupy_nanprod_complex_dtype',
+    ('F->F', 'D->D'),
+    ('''
+    type_in0_raw((in0.real() == in0.real()) ? in0.real() : 1,
+                 (in0.imag() == in0.imag()) ? in0.imag() : 1)
+    ''',
+     'a * b', 'out0 = type_out0_raw(a)', None), 1)
+
+cdef create_arithmetic(
+        name, op, boolop, doc, cutensor_op=None, scatter_op=None):
+    # boolop is either
+    #  - str (the operator for bool-bool inputs) or
+    #  - callable (a function to raise an error for bool-bool inputs).
+    if isinstance(boolop, str):
+        boolop = 'out0 = in0 %s in1' % boolop
+
+    return create_ufunc(
+        'cupy_' + name,
+        (('??->?', boolop),
+         'bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+         'LL->L', 'qq->q', 'QQ->Q', 'ee->e', 'ff->f', 'dd->d', 'FF->F',
+         'DD->D'),
+        'out0 = in0 %s in1' % op,
+        doc=doc,
+        cutensor_op=cutensor_op,
+        scatter_op=scatter_op)
+
+
+_add = create_arithmetic(
+    'add', '+', '|',
+    '''Adds two arrays elementwise.
+
+    .. seealso:: :data:`numpy.add`
+
+    ''',
+    cutensor_op=('OP_ADD', 1, 1), scatter_op='add')
+
+
+_conjugate = create_ufunc(
+    'cupy_conjugate',
+    ('b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L', 'q->q',
+     'Q->Q', 'e->e', 'f->f', 'd->d',
+     ('F->F', 'out0 = conj(in0)'),
+     ('D->D', 'out0 = conj(in0)')),
+    'out0 = in0',
+    doc='''Returns the complex conjugate, element-wise.
+
+    .. seealso:: :data:`numpy.conjugate`
+
+    ''')
+
+
+_angle = create_ufunc(
+    'cupy_angle',
+    ('?->d', 'e->e', 'f->f', 'd->d',
+     ('F->f', 'out0 = arg(in0)'),
+     ('D->d', 'out0 = arg(in0)')),
+    'out0 = in0 >= 0 ? 0 : M_PI',
+    doc='''Returns the angle of the complex argument.
+
+    .. seealso:: :func:`numpy.angle`
+
+    ''')
+
+
+_angle_deg = create_ufunc(
+    'cupy_angle_deg',
+    ('?->d', 'e->e', 'f->f', 'd->d',
+     ('F->f', 'out0 = arg(in0) * (180.0 / M_PI)'),
+     ('D->d', 'out0 = arg(in0) * (180.0 / M_PI)')),
+    'out0 = in0 >= 0 ? 0 : 180.0',
+    doc='''Returns the angle of the complex argument.
+
+    .. seealso:: :func:`numpy.angle`
+
+    ''')
+
+
+def _positive_boolean_error():
+    raise TypeError(
+        'The cupy boolean positive, the `+` operator, is not supported.')
+
+
+_positive = create_ufunc(
+    'cupy_positive',
+    (('?->?', _positive_boolean_error),
+     'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q', 'e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    'out0 = +in0',
+    doc='''Takes numerical positive elementwise.
+
+    .. seealso:: :data:`numpy.positive`
+
+    ''')
+
+
+def _negative_boolean_error():
+    raise TypeError(
+        'The cupy boolean negative, the `-` operator, is not supported, '
+        'use the `~` operator or the logical_not function instead.')
+
+
+_negative = create_ufunc(
+    'cupy_negative',
+    (('?->?', _negative_boolean_error),
+     'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q', 'e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    'out0 = -in0',
+    doc='''Takes numerical negative elementwise.
+
+    .. seealso:: :data:`numpy.negative`
+
+    ''')
+
+
+_multiply = create_arithmetic(
+    'multiply', '*', '&',
+    '''Multiplies two arrays elementwise.
+
+    .. seealso:: :data:`numpy.multiply`
+
+    ''',
+    cutensor_op=('OP_MUL', 1, 1))
+
+
+# `integral_power` should return somewhat appropriate values for negative
+# integral powers (for which NumPy would raise errors). Hence the branches in
+# the beginning. This behavior is not officially documented and could change.
+cdef _power_preamble = '''
+template <typename T>
+inline __device__ T integral_power(T in0, T in1) {
+    if (in1 < 0) {
+        if (in0 == -1) {return (in1 & 1) ? -1 : 1;}
+        else {return (in0 == 1) ? 1 : 0;}
+    }
+    T out0 = 1;
+    while (in1 > 0) {
+        if (in1 & 1) {
+            out0 *= in0;
+        }
+        in0 *= in0;
+        in1 >>= 1;
+    }
+    return out0;
+}
+
+template <typename T>
+inline __device__ T complex_power(T in0, T in1) {
+    return in1 == T(0) ? T(1): pow(in0, in1);
+}
+'''
+
+_power = create_ufunc(
+    'cupy_power',
+    ('??->b', 'bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+     'LL->L', 'qq->q', 'QQ->Q',
+     ('ee->e', 'out0 = powf(in0, in1)'),
+     ('ff->f', 'out0 = powf(in0, in1)'),
+     ('dd->d', 'out0 = pow(in0, in1)'),
+     ('FF->F', 'out0 = complex_power(in0, in1)'),
+     ('DD->D', 'out0 = complex_power(in0, in1)')),
+    'out0 = integral_power(in0, in1)',
+    preamble=_power_preamble,
+    doc='''Computes ``x1 ** x2`` elementwise.
+
+    .. seealso:: :data:`numpy.power`
+
+    ''')
+
+
+def _subtract_boolean_error():
+    raise TypeError(
+        'cupy boolean subtract, the `-` operator, is deprecated, use the '
+        'bitwise_xor, the `^` operator, or the logical_xor function instead.')
+
+
+_subtract = create_arithmetic(
+    'subtract', '-', _subtract_boolean_error,
+    '''Subtracts arguments elementwise.
+
+    .. seealso:: :data:`numpy.subtract`
+
+    ''',
+    cutensor_op=('OP_ADD', 1, -1), scatter_op='sub')
+
+
+_true_divide = create_ufunc(
+    'cupy_true_divide',
+    ('bb->d', 'BB->d', 'hh->d', 'HH->d', 'ii->d', 'II->d', 'll->d', 'LL->d',
+     'qq->d', 'QQ->d', 'ee->e', 'ff->f', 'dd->d', 'FF->F', 'DD->D'),
+    'out0 = (out0_type)in0 / (out0_type)in1',
+    doc='''Elementwise true division (i.e. division as floating values).
+
+    .. seealso:: :data:`numpy.true_divide`
+
+    ''',
+    out_ops=('ee->e', 'ff->f', 'dd->d', 'FF->F', 'DD->D'),
+)
+
+
+_divide = _true_divide
+
+
+_floor_divide = create_ufunc(
+    'cupy_floor_divide',
+    ('bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l', 'LL->L',
+     'qq->q', 'QQ->Q', 'ee->e', 'ff->f', 'dd->d'),
+    'out0 = _floor_divide(in0, in1)',
+    doc='''Elementwise floor division (i.e. integer quotient).
+
+    .. seealso:: :data:`numpy.floor_divide`
+
+    ''')
+
+
+_remainder = create_ufunc(
+    'cupy_remainder',
+    ('bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l', 'LL->L',
+     'qq->q', 'QQ->Q',
+     ('ee->e', 'out0 = in0 - _floor_divide(in0, in1) * in1'),
+     ('ff->f', 'out0 = in0 - _floor_divide(in0, in1) * in1'),
+     ('dd->d', 'out0 = in0 - _floor_divide(in0, in1) * in1')),
+    'out0 = (in0 - _floor_divide(in0, in1) * in1) * (in1 != 0)',
+    doc='''Computes the remainder of Python division elementwise.
+
+    .. seealso:: :data:`numpy.remainder`
+
+    ''')
+
+
+_absolute = create_ufunc(
+    'cupy_absolute',
+    (('?->?', 'out0 = in0'),
+     'b->b', ('B->B', 'out0 = in0'), 'h->h', ('H->H', 'out0 = in0'),
+     'i->i', ('I->I', 'out0 = in0'), 'l->l', ('L->L', 'out0 = in0'),
+     'q->q', ('Q->Q', 'out0 = in0'),
+     ('e->e', 'out0 = fabsf(in0)'),
+     ('f->f', 'out0 = fabsf(in0)'),
+     ('d->d', 'out0 = fabs(in0)'),
+     ('F->f', 'out0 = abs(in0)'),
+     ('D->d', 'out0 = abs(in0)')),
+    'out0 = in0 > 0 ? in0 : -in0',
+    doc='''Elementwise absolute value function.
+
+    .. seealso:: :data:`numpy.absolute`
+
+    ''')
+
+
+_sqrt = create_ufunc(
+    'cupy_sqrt',
+    ('e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    'out0 = sqrt(in0)',
+    doc='''Elementwise square root function.
+
+    .. seealso:: :data:`numpy.sqrt`
+
+    ''')
+
+
+_clip = create_ufunc(
+    'cupy_clip',
+    ('???->?', 'bbb->b', 'BBB->B', 'hhh->h', 'HHH->H', 'iii->i', 'III->I',
+     'lll->l', 'LLL->L', 'qqq->q', 'QQQ->Q', 'eee->e', 'fff->f', 'ddd->d'),
+    'out0 = in1 > in2 ? in2 : (in0 < in1 ? in1 : (in0 > in2 ? in2 : in0))')
+
+
+# Variables to expose to Python
+# (cythonized data cannot be exposed to Python, even with cpdef.)
+
+
+add = _add
+conjugate = _conjugate
+angle = _angle
+angle_deg = _angle_deg
+positive = _positive
+negative = _negative
+multiply = _multiply
+divide = _divide
+power = _power
+subtract = _subtract
+true_divide = _true_divide
+floor_divide = _floor_divide
+remainder = _remainder
+absolute = _absolute
+sqrt = _sqrt
+
+sum_auto_dtype = _sum_auto_dtype  # used from cupy/math/sumprod.py
+nansum_auto_dtype = _nansum_auto_dtype  # used from cupy/math/sumprod.py
+prod_auto_dtype = _prod_auto_dtype  # used from cupy/math/sumprod.py
+nanprod_auto_dtype = _nanprod_auto_dtype  # used from cupy/math/sumprod.py
+clip = _clip  # used from cupy/math/misc.py
diff --git a/cupy/_core/_routines_sorting.pxd b/cupy/_core/_routines_sorting.pxd
new file mode 100644
index 0000000..a118987
--- /dev/null
+++ b/cupy/_core/_routines_sorting.pxd
@@ -0,0 +1,7 @@
+from cupy._core.core cimport _ndarray_base
+
+
+cdef _ndarray_sort(_ndarray_base self, int axis)
+cdef _ndarray_base _ndarray_argsort(_ndarray_base self, axis)
+cdef _ndarray_partition(_ndarray_base self, kth, int axis)
+cdef _ndarray_base _ndarray_argpartition(self, kth, axis)
diff --git a/cupy/_core/_routines_sorting.pyx b/cupy/_core/_routines_sorting.pyx
new file mode 100644
index 0000000..3d33942
--- /dev/null
+++ b/cupy/_core/_routines_sorting.pyx
@@ -0,0 +1,534 @@
+import string
+
+import numpy
+
+import cupy
+from cupy._core._scalar import get_typename as _get_typename
+from cupy._core._ufuncs import elementwise_copy
+import cupy._core.core as core
+from cupy import _util
+from cupy.cuda import thrust
+
+from cupy._core cimport _routines_manipulation as _manipulation
+from cupy._core.core cimport compile_with_cache
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport internal
+
+
+cdef _ndarray_sort(_ndarray_base self, int axis):
+    cdef int ndim = self._shape.size()
+    cdef _ndarray_base data
+
+    if not cupy.cuda.thrust.available:
+        raise RuntimeError('Thrust is needed to use cupy.sort. Please '
+                           'install CUDA Toolkit with Thrust then '
+                           'reinstall CuPy after uninstalling it.')
+
+    if ndim == 0:
+        raise numpy.AxisError('Sorting arrays with the rank of zero is not '
+                              'supported')  # as numpy.sort() raises
+
+    # TODO(takagi): Support sorting views
+    if not self._c_contiguous:
+        raise NotImplementedError('Sorting non-contiguous array is not '
+                                  'supported.')
+
+    axis = internal._normalize_axis_index(axis, ndim)
+
+    if axis == ndim - 1:
+        data = self
+    else:
+        data = _manipulation.rollaxis(self, axis, ndim).copy()
+
+    if ndim == 1:
+        thrust.sort(self.dtype, data.data.ptr, 0, self.shape)
+    else:
+        max_size = max(min(1 << 22, data.size) // data.shape[-1], 1)
+        keys_array = core.ndarray(
+            (max_size * data.shape[-1],), dtype=numpy.intp)
+        stop = data.size // data.shape[-1]
+        for offset in range(0, stop, max_size):
+            width = min(max_size, stop - offset)
+            thrust.sort(
+                self.dtype,
+                data.data.ptr + offset * data.shape[-1] * data.itemsize,
+                keys_array.data.ptr,
+                (width, data.shape[-1]),
+            )
+
+    if axis == ndim - 1:
+        pass
+    else:
+        data = _manipulation.rollaxis(data, -1, axis)
+        elementwise_copy(data, self)
+
+
+cdef _ndarray_base _ndarray_argsort(_ndarray_base self, axis):
+    cdef int _axis, ndim
+    cdef _ndarray_base data
+
+    if not cupy.cuda.thrust.available:
+        raise RuntimeError('Thrust is needed to use cupy.argsort. Please '
+                           'install CUDA Toolkit with Thrust then '
+                           'reinstall CuPy after uninstalling it.')
+
+    self = cupy.atleast_1d(self)
+    ndim = self._shape.size()
+
+    if axis is None:
+        data = self.ravel()
+        _axis = -1
+    else:
+        data = self
+        _axis = axis
+
+    _axis = internal._normalize_axis_index(_axis, ndim)
+
+    if _axis == ndim - 1:
+        data = data.copy()
+    else:
+        data = _manipulation.rollaxis(data, _axis, ndim).copy()
+    shape = data.shape
+
+    idx_array = core.ndarray(shape, dtype=numpy.intp)
+
+    if ndim == 1:
+        thrust.argsort(self.dtype, idx_array.data.ptr, data.data.ptr, 0,
+                       shape)
+    else:
+        keys_array = core.ndarray(shape, dtype=numpy.intp)
+        thrust.argsort(self.dtype, idx_array.data.ptr, data.data.ptr,
+                       keys_array.data.ptr, shape)
+
+    if _axis == ndim - 1:
+        return idx_array
+    else:
+        return _manipulation.rollaxis(idx_array, -1, _axis)
+
+
+cdef _ndarray_partition(_ndarray_base self, kth, int axis):
+    """Partitions an array.
+
+    Args:
+        kth (int or sequence of ints): Element index to partition by. If
+            supplied with a sequence of k-th it will partition all elements
+            indexed by k-th of them into their sorted position at once.
+
+        axis (int): Axis along which to sort. Default is -1, which means
+            sort along the last axis.
+
+    .. seealso::
+        :func:`cupy.partition` for full documentation,
+        :meth:`numpy.ndarray.partition`
+
+    """
+
+    cdef int ndim = self._shape.size()
+    cdef Py_ssize_t k, max_k, length, s, sz, t
+    cdef _ndarray_base data
+
+    if ndim == 0:
+        raise numpy.AxisError('Sorting arrays with the rank of zero is not '
+                              'supported')
+
+    if not self._c_contiguous:
+        raise NotImplementedError('Sorting non-contiguous array is not '
+                                  'supported.')
+
+    axis = internal._normalize_axis_index(axis, ndim)
+
+    if axis == ndim - 1:
+        data = self
+    else:
+        data = _manipulation.rollaxis(self, axis, ndim).copy()
+
+    length = self._shape[axis]
+    if isinstance(kth, int):
+        kth = kth,
+    max_k = 0
+    for k in kth:
+        if k < 0:
+            k += length
+        if not (0 <= k < length):
+            raise ValueError('kth(={}) out of bounds {}'.format(k, length))
+        if max_k < k:
+            max_k = k
+
+    # For simplicity, max_k is round up to the power of 2. If max_k is
+    # already the power of 2, it is round up to the next power of 2 because
+    # we need to collect the first max(kth)+1 elements.
+    max_k = max(32, 1 << max_k.bit_length())
+
+    # The parameter t is the length of the list that stores elements to be
+    # selected for each thread. We divide the array into sz subarrays.
+    # These parameters are determined from the measurement on TITAN X.
+    t = 4
+    sz = 512
+    while sz > 0 and length // sz < max_k + 32 * t:
+        sz //= 2
+    sz *= self.size // length
+
+    # If the array size is small or k is large, we simply sort the array.
+    if length < 32 or sz <= 32 or max_k >= 1024:
+        # kth is ignored.
+        data.sort(axis=-1)
+    else:
+        shape = data.shape
+        data = data.ravel()
+
+        # For each subarray, we collect first k elements to the head.
+        kern, merge_kern = _partition_kernel(self.dtype)
+        block_size = 32
+        grid_size = sz
+        kern(grid=(grid_size,), block=(block_size,), args=(
+            data, max_k, self.size, t, sz))
+
+        # Merge heads of subarrays.
+        s = 1
+        while s < sz // (self.size // length):
+            block_size = 32
+            grid_size = sz // s // 2
+            merge_kern(grid=(grid_size,), block=(block_size,), args=(
+                data, max_k, self.size, sz, s))
+            s *= 2
+
+        data = data.reshape(shape)
+
+    if axis != ndim - 1:
+        data = _manipulation.rollaxis(data, -1, axis)
+        elementwise_copy(data, self)
+
+
+cdef _ndarray_base _ndarray_argpartition(self, kth, axis):
+    """Returns the indices that would partially sort an array.
+
+    Args:
+        kth (int or sequence of ints): Element index to partition by. If
+            supplied with a sequence of k-th it will partition all elements
+            indexed by k-th of them into their sorted position at once.
+        axis (int or None): Axis along which to sort. Default is -1, which
+            means sort along the last axis. If None is supplied, the array
+            is flattened before sorting.
+
+    Returns:
+        cupy.ndarray: Array of the same type and shape as ``a``.
+
+    .. seealso::
+        :func:`cupy.argpartition` for full documentation,
+        :meth:`numpy.ndarray.argpartition`
+
+    """
+    cdef int _axis, ndim
+    cdef Py_ssize_t k, max_k, length, s, sz, t
+    cdef _ndarray_base data
+    if axis is None:
+        data = self.ravel()
+        _axis = -1
+    else:
+        data = self
+        _axis = axis
+
+    ndim = data._shape.size()
+    _axis = internal._normalize_axis_index(_axis, ndim)
+
+    if _axis != ndim - 1:
+        data = _manipulation.rollaxis(self, _axis, ndim).copy()
+
+    length = data._shape[ndim - 1]
+
+    if length == 0:
+        return cupy.empty((0,), dtype=cupy.int64)
+
+    if isinstance(kth, int):
+        kth = kth,
+    max_k = 0
+    for k in kth:
+        if k < 0:
+            k += length
+        if not (0 <= k < length):
+            raise ValueError('kth(={}) out of bounds {}'.format(k, length))
+        if max_k < k:
+            max_k = k
+
+    # For simplicity, max_k is round up to the power of 2. If max_k is
+    # already the power of 2, it is round up to the next power of 2 because
+    # we need to collect the first max(kth)+1 elements.
+    max_k = max(32, 1 << max_k.bit_length())
+
+    # The parameter t is the length of the list that stores elements to be
+    # selected for each thread. We divide the array into sz subarrays.
+    # These parameters are determined from the measurement on TITAN X.
+    t = 4
+    sz = 512
+    while sz > 0 and length // sz < max_k + 32 * t:
+        sz //= 2
+    sz *= self.size // length
+    shape = data.shape
+
+    # If the array size is small or k is large, we simply sort the array.
+    if length < 32 or sz < 1 or max_k >= 1024:
+        # kth is ignored.
+        indices = data.argsort(axis=-1)
+    else:
+        data = data.ravel()
+        indices = cupy.arange(0, data.shape[0], dtype=cupy.int64)
+
+        # For each subarray, we collect first k elements to the head.
+        kern, merge_kern = _argpartition_kernel(self.dtype)
+        block_size = 32
+        grid_size = sz
+        kern(grid=(grid_size,), block=(block_size,), args=(
+            data, indices, max_k, self.size, t, sz))
+
+        # Merge heads of subarrays.
+        s = 1
+        while s < sz // (self.size // length):
+            block_size = 32
+            grid_size = sz // s // 2
+            merge_kern(grid=(grid_size,), block=(block_size,), args=(
+                data, indices, max_k, self.size, sz, s))
+            s *= 2
+
+    # Rearrange indices w.r.t the original axis
+    axis_indices = cupy.unravel_index(indices, shape)
+    indices = axis_indices[-1]
+    indices = indices.reshape(shape)
+
+    if _axis != ndim - 1:
+        indices = _manipulation.rollaxis(indices, -1, _axis)
+
+    return indices
+
+
+@_util.memoize(for_each_device=True)
+def _partition_kernel(dtype):
+    name = 'partition_kernel'
+    merge_kernel = 'partition_merge_kernel'
+    dtype = _get_typename(dtype)
+    source = string.Template('''
+    template<typename T>
+    __device__ void bitonic_sort_step(CArray<T, 1, true> a,
+            ptrdiff_t x, ptrdiff_t y, int i, ptrdiff_t s, ptrdiff_t w) {
+        for (ptrdiff_t j = i; j < (y - x) / 2; j += 32) {
+            ptrdiff_t n = j + (j & -w);
+            T v = a[n + x], u = a[n + w + x];
+            if (n & s ? v < u : v > u) {
+                a[n + x] = u;
+                a[n + w + x] = v;
+            }
+        }
+    }
+
+    // Sort a[x:y].
+    template<typename T>
+    __device__ void bitonic_sort(
+            CArray<T, 1, true> a, ptrdiff_t x, ptrdiff_t y, int i) {
+        for (ptrdiff_t s = 2; s <= y - x; s *= 2) {
+            for (ptrdiff_t w = s / 2; w >= 1; w /= 2) {
+                bitonic_sort_step< T >(a, x, y, i, s, w);
+            }
+        }
+    }
+
+    // Merge first k elements and the next 32 times t elements.
+    template<typename T>
+    __device__ void merge(
+            CArray<T, 1, true> a,
+            int k, int i, ptrdiff_t x, ptrdiff_t z, int u) {
+        for (int s = i; s < u; s += 32) {
+            if (a[x + k - s - 1] > a[z + s]) {
+                T tmp = a[x + k - s - 1];
+                a[x + k - s - 1] = a[z + s];
+                a[z + s] = tmp;
+            }
+        }
+
+        // After merge step, the first k elements are already bitonic.
+        // Therefore, we do not need to fully sort.
+        for (int w = k / 2; w >= 1; w /= 2) {
+            bitonic_sort_step< T >(a, x, k + x, i, k, w);
+        }
+    }
+
+    extern "C" {
+    // In this function, 32 threads handle one subarray. This number equals to
+    // the warp size. The first k elements are always sorted and the next 32
+    // times t elements stored values that have possibilities to be selected.
+    __global__ void ${name}(
+            CArray<${dtype}, 1, true> a,
+            int k, ptrdiff_t n, int t, ptrdiff_t sz) {
+
+        // This thread handles a[z:m].
+        ptrdiff_t i = static_cast<ptrdiff_t>(blockIdx.x) * blockDim.x
+            + threadIdx.x;
+        ptrdiff_t z = i / 32 * n / sz;
+        ptrdiff_t m = (i / 32 + 1) * n / sz;
+        int id = i % 32;
+        int x = 0;
+
+        bitonic_sort< ${dtype} >(a, z, k + z, id);
+        ptrdiff_t j;
+        for (j = k + id + z; j < m - (m - z) % 32; j += 32) {
+            if (a[j] < a[k - 1 + z]) {
+                ${dtype} tmp = a[k + 32 * x + id + z];
+                a[k + 32 * x + id + z] = a[j];
+                a[j] = tmp;
+                ++x;
+            }
+
+            // If at least one thread in the warp has found t values that
+            // can be selected, we update the first k elements.
+    #if __CUDACC_VER_MAJOR__ >= 9
+            if (__any_sync(0xffffffff, x >= t)) {
+    #else
+            if (__any(x >= t)) {
+    #endif
+                bitonic_sort< ${dtype} >(a, k + z, 32 * t + k + z, id);
+                merge< ${dtype} >(a, k, id, z, k + z, min(k, 32 * t));
+                x = 0;
+            }
+        }
+        if (j < m && a[j] < a[k - 1 + z]) {
+            ${dtype} tmp = a[k + 32 * x + id + z];
+            a[k + 32 * x + id + z] = a[j];
+            a[j] = tmp;
+        }
+
+        // Finally, we merge the first k elements and the remainders to be
+        // stored.
+        bitonic_sort< ${dtype} >(a, k + z, 32 * t + k + z, id);
+        merge< ${dtype} >(a, k, id, z, k + z, min(k, 32 * t));
+    }
+
+    __global__ void ${merge_kernel}(
+            CArray<${dtype}, 1, true> a, int k, ptrdiff_t n, int sz, int s) {
+        ptrdiff_t i = static_cast<ptrdiff_t>(blockIdx.x) * blockDim.x
+            + threadIdx.x;
+        ptrdiff_t z = i / 32 * 2 * s * n / sz;
+        ptrdiff_t m = (i / 32 * 2 + 1) * s * n / sz;
+        int id = i % 32;
+        merge< ${dtype} >(a, k, id, z, m, k);
+    }
+    }
+    ''').substitute(name=name, merge_kernel=merge_kernel, dtype=dtype)
+    module = compile_with_cache(source)
+    return module.get_function(name), module.get_function(merge_kernel)
+
+
+@_util.memoize(for_each_device=True)
+def _argpartition_kernel(dtype):
+    name = 'argpartition_kernel'
+    merge_kernel = 'argpartition_merge_kernel'
+    dtype = _get_typename(dtype)
+    source = string.Template('''
+    template<typename T>
+    __device__ void bitonic_sort_step(
+            CArray<T, 1, true> a, CArray<long long, 1, true> b,
+            ptrdiff_t x, ptrdiff_t y, int i, ptrdiff_t s, ptrdiff_t w) {
+        for (ptrdiff_t j = i; j < (y - x) / 2; j += 32) {
+            ptrdiff_t n = j + (j & -w);
+            T v = a[b[n + x]], u = a[b[n + w + x]];
+            if (n & s ? v < u : v > u) {
+                long long temp = b[n + x];
+                b[n + x] = b[n + w + x];
+                b[n + w + x] = temp;
+            }
+        }
+    }
+
+    // Sort a[x:y].
+    template<typename T>
+    __device__ void bitonic_sort(
+            CArray<T, 1, true> a, CArray<long long, 1, true> b,
+            ptrdiff_t x, ptrdiff_t y, int i) {
+        for (ptrdiff_t s = 2; s <= y - x; s *= 2) {
+            for (ptrdiff_t w = s / 2; w >= 1; w /= 2) {
+                bitonic_sort_step< T >(a, b, x, y, i, s, w);
+            }
+        }
+    }
+
+    // Merge first k elements and the next 32 times t elements.
+    template<typename T>
+    __device__ void merge(
+            CArray<T, 1, true> a, CArray<long long, 1, true> b,
+            int k, int i, ptrdiff_t x, ptrdiff_t z, int u) {
+        for (int s = i; s < u; s += 32) {
+            if (a[b[x + k - s - 1]] > a[b[z + s]]) {
+                long long tmp = b[x + k - s - 1];
+                b[x + k - s - 1] = b[z + s];
+                b[z + s] = tmp;
+            }
+        }
+
+        // After merge step, the first k elements are already bitonic.
+        // Therefore, we do not need to fully sort.
+        for (int w = k / 2; w >= 1; w /= 2) {
+            bitonic_sort_step< T >(a, b, x, k + x, i, k, w);
+        }
+    }
+
+    extern "C" {
+    // In this function, 32 threads handle one subarray. This number equals to
+    // the warp size. The first k elements are always sorted and the next 32
+    // times t elements stored values that have possibilities to be selected.
+    __global__ void ${name}(
+            CArray<${dtype}, 1, true> a, CArray<long long, 1, true> b,
+            int k, ptrdiff_t n, int t, ptrdiff_t sz) {
+
+        // This thread handles a[z:m].
+        ptrdiff_t i = static_cast<ptrdiff_t>(blockIdx.x) * blockDim.x
+            + threadIdx.x;
+        ptrdiff_t z = i / 32 * n / sz;
+        ptrdiff_t m = (i / 32 + 1) * n / sz;
+        int id = i % 32;
+        int x = 0;
+
+        bitonic_sort< ${dtype} >(a, b, z, k + z, id);
+        ptrdiff_t j;
+        for (j = k + id + z; j < m - (m - z) % 32; j += 32) {
+            if (a[b[j]] < a[b[k - 1 + z]]) {
+                long long tmp = b[k + 32 * x + id + z];
+                b[k + 32 * x + id + z] = b[j];
+                b[j] = tmp;
+                ++x;
+            }
+
+            // If at least one thread in the warp has found t values that
+            // can be selected, we update the first k elements.
+    #if __CUDACC_VER_MAJOR__ >= 9
+            if (__any_sync(0xffffffff, x >= t)) {
+    #else
+            if (__any(x >= t)) {
+    #endif
+                bitonic_sort< ${dtype} >(a, b, k + z, 32 * t + k + z, id);
+                merge< ${dtype} >(a, b, k, id, z, k + z, min(k, 32 * t));
+                x = 0;
+            }
+        }
+        if (j < m && a[b[j]] < a[b[k - 1 + z]]) {
+            long long tmp = b[k + 32 * x + id + z];
+            b[k + 32 * x + id + z] = b[j];
+            b[j] = tmp;
+        }
+
+        // Finally, we merge the first k elements and the remainders to be
+        // stored.
+        bitonic_sort< ${dtype} >(a, b, k + z, 32 * t + k + z, id);
+        merge< ${dtype} >(a, b, k, id, z, k + z, min(k, 32 * t));
+    }
+
+    __global__ void ${merge_kernel}(
+            CArray<${dtype}, 1, true> a,  CArray<long long, 1, true> b,
+            int k, ptrdiff_t n, int sz, int s) {
+        ptrdiff_t i = static_cast<ptrdiff_t>(blockIdx.x) * blockDim.x
+            + threadIdx.x;
+        ptrdiff_t z = i / 32 * 2 * s * n / sz;
+        ptrdiff_t m = (i / 32 * 2 + 1) * s * n / sz;
+        int id = i % 32;
+        merge< ${dtype} >(a, b, k, id, z, m, k);
+    }
+    }
+    ''').substitute(name=name, merge_kernel=merge_kernel, dtype=dtype)
+    module = compile_with_cache(source)
+    return module.get_function(name), module.get_function(merge_kernel)
diff --git a/cupy/_core/_routines_statistics.pxd b/cupy/_core/_routines_statistics.pxd
new file mode 100644
index 0000000..1496961
--- /dev/null
+++ b/cupy/_core/_routines_statistics.pxd
@@ -0,0 +1,30 @@
+from cupy._core.core cimport _ndarray_base
+
+
+# TODO(niboshi): Move {nan,}arg{min,max} to sorting
+
+
+cdef _ndarray_base _ndarray_max(_ndarray_base self, axis, out, dtype, keepdims)
+cdef _ndarray_base _ndarray_min(_ndarray_base self, axis, out, dtype, keepdims)
+cdef _ndarray_base _ndarray_ptp(_ndarray_base self, axis, out, keepdims)
+cdef _ndarray_base _ndarray_argmax(
+    _ndarray_base self, axis, out, dtype, keepdims)
+cdef _ndarray_base _ndarray_argmin(
+    _ndarray_base self, axis, out, dtype, keepdims)
+cdef _ndarray_base _ndarray_mean(
+    _ndarray_base self, axis, dtype, out, keepdims)
+cdef _ndarray_base _ndarray_var(
+    _ndarray_base self, axis, dtype, out, ddof, keepdims)
+cdef _ndarray_base _ndarray_std(
+    _ndarray_base self, axis, dtype, out, ddof, keepdims)
+
+cpdef _ndarray_base _median(
+    _ndarray_base a, axis, out, overwrite_input, keepdims)
+
+cpdef _ndarray_base _nanmean(_ndarray_base a, axis, dtype, out, keepdims)
+cpdef _ndarray_base _nanvar(_ndarray_base a, axis, dtype, out, ddof, keepdims)
+cpdef _ndarray_base _nanstd(_ndarray_base a, axis, dtype, out, ddof, keepdims)
+
+
+cpdef _ndarray_base _nanargmin(_ndarray_base a, axis, out, dtype, keepdims)
+cpdef _ndarray_base _nanargmax(_ndarray_base a, axis, out, dtype, keepdims)
diff --git a/cupy/_core/_routines_statistics.pyx b/cupy/_core/_routines_statistics.pyx
new file mode 100644
index 0000000..f0f74f9
--- /dev/null
+++ b/cupy/_core/_routines_statistics.pyx
@@ -0,0 +1,763 @@
+from cpython cimport sequence
+
+import numpy
+from numpy import nan
+
+import cupy
+from cupy._core import _reduction
+from cupy._core._reduction import create_reduction_func
+from cupy._core._reduction import ReductionKernel
+from cupy._core._kernel import ElementwiseKernel
+from cupy._core._ufuncs import elementwise_copy
+
+from cupy._core cimport _accelerator
+from cupy._core cimport _routines_math as _math
+from cupy._core.core cimport _ndarray_base
+
+from cupy.cuda import cub
+
+try:
+    import cupy_backends.cuda.libs.cutensor as cuda_cutensor
+except ImportError:
+    cuda_cutensor = None
+
+
+cdef _ndarray_base _ndarray_max(
+        _ndarray_base self, axis, out, dtype, keepdims):
+    for accelerator in _accelerator._routine_accelerators:
+        result = None
+        if accelerator == _accelerator.ACCELERATOR_CUB:
+            # result will be None if the reduction is not compatible with CUB
+            result = cub.cub_reduction(
+                self, cub.CUPY_CUB_MAX, axis, dtype, out, keepdims)
+        if (accelerator == _accelerator.ACCELERATOR_CUTENSOR and
+                cuda_cutensor is not None):
+            from cupyx import cutensor
+            if self.dtype.kind == 'c' or dtype in ('F', 'D'):
+                # Complex dtype is not supported
+                continue
+            result = cutensor._try_reduction_routine(
+                self, axis, dtype, out, keepdims, cuda_cutensor.OP_MAX, 1, 0)
+        if result is not None:
+            return result
+    return _amax(self, axis=axis, out=out, dtype=dtype, keepdims=keepdims)
+
+
+cdef _ndarray_base _ndarray_min(
+        _ndarray_base self, axis, out, dtype, keepdims):
+    for accelerator in _accelerator._routine_accelerators:
+        result = None
+        if accelerator == _accelerator.ACCELERATOR_CUB:
+            # result will be None if the reduction is not compatible with CUB
+            result = cub.cub_reduction(
+                self, cub.CUPY_CUB_MIN, axis, out, dtype, keepdims)
+        if (accelerator == _accelerator.ACCELERATOR_CUTENSOR and
+                cuda_cutensor is not None):
+            from cupyx import cutensor
+            if self.dtype.kind == 'c' or dtype in ('F', 'D'):
+                # Complex dtype is not supported
+                continue
+            result = cutensor._try_reduction_routine(
+                self, axis, dtype, out, keepdims, cuda_cutensor.OP_MIN, 1, 0)
+        if result is not None:
+            return result
+    return _amin(self, axis=axis, out=out, dtype=dtype, keepdims=keepdims)
+
+
+cdef _ndarray_base _ndarray_ptp(_ndarray_base self, axis, out, keepdims):
+    for accelerator in _accelerator._routine_accelerators:
+        if accelerator == _accelerator.ACCELERATOR_CUB:
+            # result will be None if the reduction is not compatible with CUB
+            result = cub.cub_reduction(
+                self, cub.CUPY_CUB_MAX, axis, out, None, keepdims)
+            if result is not None:
+                result -= cub.cub_reduction(
+                    self, cub.CUPY_CUB_MIN, axis, None, None, keepdims)
+                return result
+        if (accelerator == _accelerator.ACCELERATOR_CUTENSOR and
+                cuda_cutensor is not None):
+            from cupyx import cutensor
+            if self.dtype.kind == 'c':
+                # Complex dtype is not supported
+                continue
+            maxv = cutensor._try_reduction_routine(
+                self, axis, None, out, keepdims, cuda_cutensor.OP_MAX, 1, 0)
+            if maxv is None:
+                continue
+            return cutensor._try_reduction_routine(
+                self, axis, None, maxv, keepdims, cuda_cutensor.OP_MIN, -1, 1)
+
+    result = _amax(self, axis=axis, out=out, keepdims=keepdims)
+    result -= _amin(self, axis=axis, out=None, keepdims=keepdims)
+    return result
+
+
+# TODO(leofang): this signature is incompatible with NumPy!
+cdef _ndarray_base _ndarray_argmax(
+        _ndarray_base self, axis, out, dtype, keepdims):
+    for accelerator in _accelerator._routine_accelerators:
+        if accelerator == _accelerator.ACCELERATOR_CUB:
+            # result will be None if the reduction is not compatible with CUB
+            if self._f_contiguous and self.dtype == numpy.bool_:
+                # temporary workaround casting the inputs to int8
+                # CUB argmax seems to return different values to
+                # NumPy for F-order bool array inputs
+                self = self.astype(numpy.int8)
+            result = cub.cub_reduction(
+                self, cub.CUPY_CUB_ARGMAX, axis, dtype, out, keepdims)
+            if result is not None:
+                return result
+    return _argmax(self, axis=axis, out=out, dtype=dtype, keepdims=keepdims)
+
+
+# TODO(leofang): this signature is incompatible with NumPy!
+cdef _ndarray_base _ndarray_argmin(
+        _ndarray_base self, axis, out, dtype, keepdims):
+    for accelerator in _accelerator._routine_accelerators:
+        if accelerator == _accelerator.ACCELERATOR_CUB:
+            # result will be None if the reduction is not compatible with CUB
+            result = cub.cub_reduction(
+                self, cub.CUPY_CUB_ARGMIN, axis, dtype, out, keepdims)
+            if result is not None:
+                return result
+    return _argmin(self, axis=axis, out=out, dtype=dtype, keepdims=keepdims)
+
+
+cdef _ndarray_base _ndarray_mean(
+        _ndarray_base self, axis, dtype, out, keepdims):
+    cdef Py_ssize_t n
+
+    dtype_sum = dtype_out = dtype
+    if dtype is None:
+        if self.dtype.kind in 'iub':
+            dtype_out = numpy.float64
+            dtype_sum = numpy.float64
+        elif self.dtype.char == 'e':
+            dtype_sum = numpy.float32
+            dtype_out = numpy.float16
+    elif numpy.dtype(dtype).kind in 'iub':
+        # output will be the requested type, but compute the mean using float
+        dtype_out = dtype
+        dtype_sum = numpy.float64
+
+    for accelerator in _accelerator._routine_accelerators:
+        if accelerator == _accelerator.ACCELERATOR_CUB and self.size != 0:
+            result = cub.cub_reduction(
+                self, cub.CUPY_CUB_SUM, axis, dtype_sum, out, keepdims)
+            if result is not None:
+                n = self.size // result.size
+                cupy.true_divide(result, n, out=result, casting='unsafe')
+                break
+        if (accelerator == _accelerator.ACCELERATOR_CUTENSOR and
+                cuda_cutensor is not None):
+            from cupyx import cutensor
+            reduce_axis, _ = _reduction._get_axis(axis, self._shape.size())
+            n = 1
+            for i in reduce_axis:
+                n *= self._shape[i]
+            n = max(n, 1)
+            result = cutensor._try_reduction_routine(
+                self, axis, dtype_sum, out, keepdims,
+                cuda_cutensor.OP_ADD, 1.0 / n, 0)
+            if result is not None:
+                break
+    else:
+        result = _mean(
+            self, axis=axis, dtype=dtype_sum, out=out, keepdims=keepdims)
+
+    if dtype_out is not None and out is None:
+        result = result.astype(dtype_out)
+    return result
+
+
+cdef _ndarray_base _ndarray_var(
+        _ndarray_base self, axis, dtype, out, ddof, keepdims):
+    return _var(
+        self, axis=axis, dtype=dtype, out=out, ddof=ddof, keepdims=keepdims)
+
+
+cdef _ndarray_base _ndarray_std(
+        _ndarray_base self, axis, dtype, out, ddof, keepdims):
+    return _std(
+        self, axis=axis, dtype=dtype, out=out, ddof=ddof, keepdims=keepdims)
+
+
+cdef _min_max_preamble = '''
+template <typename T>
+struct min_max_st{
+    T value;
+    int index;
+    __device__ min_max_st() : index(-1) { }
+    __device__ min_max_st(T v) : value(v), index(0) { }
+    __device__ min_max_st(T v, int i) : value(v), index(i) { }
+};
+
+template <typename T>
+__device__ min_max_st<T> my_min(
+        const min_max_st<T>& a, const min_max_st<T>& b) {
+    if (a.index == -1) return b;
+    if (b.index == -1) return a;
+    return min_max_st<T>(min(a.value, b.value));
+}
+template <typename T>
+__device__ min_max_st<T> my_min_float(
+        const min_max_st<T>& a, const min_max_st<T>& b) {
+    if (a.index == -1) return b;
+    if (b.index == -1) return a;
+    if (isnan(a.value)) return a;
+    if (isnan(b.value)) return b;
+    return min_max_st<T>(min(a.value, b.value));
+}
+
+template <typename T>
+__device__ min_max_st<T> my_max(
+        const min_max_st<T>& a, const min_max_st<T>& b) {
+    if (a.index == -1) return b;
+    if (b.index == -1) return a;
+    return min_max_st<T>(max(a.value, b.value));
+}
+template <typename T>
+__device__ min_max_st<T> my_max_float(
+        const min_max_st<T>& a, const min_max_st<T>& b) {
+    if (a.index == -1) return b;
+    if (b.index == -1) return a;
+    if (isnan(a.value)) return a;
+    if (isnan(b.value)) return b;
+    return min_max_st<T>(max(a.value, b.value));
+}
+
+template <typename T>
+__device__ min_max_st<T> my_argmin(
+        const min_max_st<T>& a, const min_max_st<T>& b) {
+    if (a.index == -1) return b;
+    if (b.index == -1) return a;
+    if (a.value == b.value)
+        return min_max_st<T>(a.value, min(a.index, b.index));
+    return (a.value <= b.value) ? a : b;
+}
+template <typename T>
+__device__ min_max_st<T> my_argmin_float(
+        const min_max_st<T>& a, const min_max_st<T>& b) {
+    if (a.index == -1) return b;
+    if (b.index == -1) return a;
+    if (a.value == b.value)
+        return min_max_st<T>(a.value, min(a.index, b.index));
+    if (isnan(a.value)) return a;
+    if (isnan(b.value)) return b;
+    return (a.value <= b.value) ? a : b;
+}
+
+template <typename T>
+__device__ min_max_st<T> my_argmax(
+        const min_max_st<T>& a, const min_max_st<T>& b) {
+    if (a.index == -1) return b;
+    if (b.index == -1) return a;
+    if (a.value == b.value)
+        return min_max_st<T>(a.value, min(a.index, b.index));
+    return (a.value >= b.value) ? a : b;
+}
+template <typename T>
+__device__ min_max_st<T> my_argmax_float(
+        const min_max_st<T>& a, const min_max_st<T>& b) {
+    if (a.index == -1) return b;
+    if (b.index == -1) return a;
+    if (a.value == b.value)
+        return min_max_st<T>(a.value, min(a.index, b.index));
+    if (isnan(a.value)) return a;
+    if (isnan(b.value)) return b;
+    return (a.value >= b.value) ? a : b;
+}
+
+'''
+
+
+cdef _amin = create_reduction_func(
+    'cupy_min',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, 'my_min_float(a, b)', None, None)),
+     ('f->f', (None, 'my_min_float(a, b)', None, None)),
+     ('d->d', (None, 'my_min_float(a, b)', None, None)),
+     ('F->F', (None, 'my_min_float(a, b)', None, None)),
+     ('D->D', (None, 'my_min_float(a, b)', None, None))),
+    ('min_max_st<type_in0_raw>(in0)', 'my_min(a, b)', 'out0 = a.value',
+     'min_max_st<type_in0_raw>'),
+    None, _min_max_preamble)
+
+
+cdef _amax = create_reduction_func(
+    'cupy_max',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, 'my_max_float(a, b)', None, None)),
+     ('f->f', (None, 'my_max_float(a, b)', None, None)),
+     ('d->d', (None, 'my_max_float(a, b)', None, None)),
+     ('F->F', (None, 'my_max_float(a, b)', None, None)),
+     ('D->D', (None, 'my_max_float(a, b)', None, None)),
+     ),
+    ('min_max_st<type_in0_raw>(in0)', 'my_max(a, b)', 'out0 = a.value',
+     'min_max_st<type_in0_raw>'),
+    None, _min_max_preamble)
+
+
+nanmin = create_reduction_func(
+    'cupy_nanmin',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q', 'e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    ('min_max_st<type_in0_raw>(in0)', 'my_min(a, b)', 'out0 = a.value',
+     'min_max_st<type_in0_raw>'),
+    None, _min_max_preamble)
+
+
+nanmax = create_reduction_func(
+    'cupy_nanmax',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q', 'e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    ('min_max_st<type_in0_raw>(in0)', 'my_max(a, b)', 'out0 = a.value',
+     'min_max_st<type_in0_raw>'),
+    None, _min_max_preamble)
+
+
+cdef _argmin = create_reduction_func(
+    'cupy_argmin',
+    tuple(['{}->{}'.format(d, r) for r in 'qlihb' for d in '?BhHiIlLqQ'])
+    + (
+        ('e->q', (None, 'my_argmin_float(a, b)', None, None)),
+        ('f->q', (None, 'my_argmin_float(a, b)', None, None)),
+        ('d->q', (None, 'my_argmin_float(a, b)', None, None)),
+        ('F->q', (None, 'my_argmin_float(a, b)', None, None)),
+        ('D->q', (None, 'my_argmin_float(a, b)', None, None))),
+    ('min_max_st<type_in0_raw>(in0, _J)', 'my_argmin(a, b)', 'out0 = a.index',
+     'min_max_st<type_in0_raw>'),
+    None, _min_max_preamble, sort_reduce_axis=False)
+
+
+cdef _argmax = create_reduction_func(
+    'cupy_argmax',
+    tuple(['{}->{}'.format(d, r) for r in 'qlihb' for d in '?BhHiIlLqQ'])
+    + (
+        ('e->q', (None, 'my_argmax_float(a, b)', None, None)),
+        ('f->q', (None, 'my_argmax_float(a, b)', None, None)),
+        ('d->q', (None, 'my_argmax_float(a, b)', None, None)),
+        ('F->q', (None, 'my_argmax_float(a, b)', None, None)),
+        ('D->q', (None, 'my_argmax_float(a, b)', None, None))),
+    ('min_max_st<type_in0_raw>(in0, _J)', 'my_argmax(a, b)', 'out0 = a.index',
+     'min_max_st<type_in0_raw>'),
+    None, _min_max_preamble, sort_reduce_axis=False)
+
+
+cpdef _ndarray_base _nanargmax(_ndarray_base a, axis, out, dtype, keepdims):
+    return _nanargmax_func(
+        a, axis=axis, out=out, dtype=dtype, keepdims=keepdims)
+
+
+cpdef _ndarray_base _nanargmin(_ndarray_base a, axis, out, dtype, keepdims):
+    return _nanargmin_func(
+        a, axis=axis, out=out, dtype=dtype, keepdims=keepdims)
+
+
+cdef _nanargmin_func = create_reduction_func(
+    'cupy_nanargmin',
+    ('?->q', 'B->q', 'h->q', 'H->q', 'i->q', 'I->q', 'l->q', 'L->q',
+     'q->q', 'Q->q',
+     ('e->q', (None, 'my_argmin_float(a, b)', None, None)),
+     ('f->q', (None, 'my_argmin_float(a, b)', None, None)),
+     ('d->q', (None, 'my_argmin_float(a, b)', None, None)),
+     ('F->q', (None, 'my_argmin_float(a, b)', None, None)),
+     ('D->q', (None, 'my_argmin_float(a, b)', None, None))),
+    ('min_max_st<type_in0_raw>(in0, isnan(in0) ? -1 : _J)',
+     'my_argmin(a, b)', 'out0 = a.index', 'min_max_st<type_in0_raw>'),
+    None, _min_max_preamble, sort_reduce_axis=False)
+
+
+cdef _nanargmax_func = create_reduction_func(
+    'cupy_nanargmax',
+    ('?->q', 'B->q', 'h->q', 'H->q', 'i->q', 'I->q', 'l->q', 'L->q',
+     'q->q', 'Q->q',
+     ('e->q', (None, 'my_argmax_float(a, b)', None, None)),
+     ('f->q', (None, 'my_argmax_float(a, b)', None, None)),
+     ('d->q', (None, 'my_argmax_float(a, b)', None, None)),
+     ('F->q', (None, 'my_argmax_float(a, b)', None, None)),
+     ('D->q', (None, 'my_argmax_float(a, b)', None, None))),
+    ('min_max_st<type_in0_raw>(in0, isnan(in0) ? -1 : _J)',
+     'my_argmax(a, b)', 'out0 = a.index', 'min_max_st<type_in0_raw>'),
+    None, _min_max_preamble, sort_reduce_axis=False)
+
+
+cdef _exists_nan = ReductionKernel(
+    'T x', 'bool y', 'isnan(x)', 'a || b', 'y = a', 'false', '_exists_nan')
+
+
+cpdef _ndarray_base _median(
+        _ndarray_base a, axis, out, overwrite_input, keepdims):
+
+    keep_ndim = a.ndim
+
+    out_shape = None
+    if sequence.PySequence_Check(axis):
+        # cupy.sort and cupy.partition only support integer axis, so move
+        # all reduced dimensions to the end and reshape them into a single
+        # reduction axis.
+        reduce_axis, out_axis = _reduction._get_axis(axis, keep_ndim)
+        out_shape = _reduction._get_out_shape(a.shape, reduce_axis, out_axis,
+                                              keepdims)
+        a = a.transpose(out_axis + reduce_axis)
+        sort_shape = tuple([a.shape[n] for n in range(len(out_axis))]) + (-1,)
+        a = a.reshape(sort_shape)
+        if not a.flags.c_contiguous:
+            a = cupy.ascontiguousarray(a)
+        axis = -1
+
+    if axis is None:
+        sz = a.size
+    else:
+        if axis < -keep_ndim or axis >= keep_ndim:
+            raise numpy.AxisError('Axis overrun')
+        sz = a.shape[axis]
+    if sz % 2 == 0:
+        szh = sz // 2
+        kth = [szh - 1, szh]
+    else:
+        kth = [(sz - 1) // 2]
+
+    if overwrite_input:
+        part = a
+    else:
+        part = a.copy()
+
+    if axis is None:
+        part = part.ravel()
+        part.partition(kth)
+    else:
+        part.partition(kth, axis=axis)
+
+    if part.shape == ():
+        return part
+    if axis is None:
+        axis = 0
+
+    indexer = [slice(None)] * part.ndim
+
+    if keepdims and out_shape is None:
+        _indexer = [None] * (keep_ndim - part.ndim)
+        indexer.extend(_indexer)
+
+    index = part.shape[axis] // 2
+    if part.shape[axis] % 2 == 1:
+        indexer[axis] = slice(index, index+1)
+    else:
+        indexer[axis] = slice(index-1, index+1)
+    indexer = tuple(indexer)
+
+    out = _mean(
+        part[indexer], axis=axis, dtype=None, out=out, keepdims=keepdims)
+    if part.dtype.kind in 'fc':
+        isnan = _exists_nan(part, axis=axis, keepdims=keepdims)
+        out = cupy.where(isnan, numpy.nan, out)
+    if out_shape is not None:
+        out = out.reshape(out_shape)
+    return out
+
+
+cpdef _ndarray_base _nanmedian(
+        _ndarray_base a, axis, out, overwrite_input, keepdims):
+
+    if axis is None:
+        axis = tuple(range(a.ndim))
+    if not sequence.PySequence_Check(axis):
+        axis = (axis,)
+
+    reduce_axis = []
+    reduce_shape = []
+    out_axis = []
+    out_shape = []
+    for i in range(a.ndim):
+        if axis is None or i in axis or i - a.ndim in axis:
+            reduce_axis.append(i)
+            reduce_shape.append(a.shape[i])
+        else:
+            out_axis.append(i)
+            out_shape.append(a.shape[i])
+
+    a_data_ptr = a.data.ptr
+    a = a.transpose(out_axis + reduce_axis)
+    a = a.reshape(out_shape + [-1, ])
+    a = cupy.ascontiguousarray(a)
+
+    n_reduce = numpy.prod(reduce_shape)
+    n_reduce_each = cupy.full(out_shape, n_reduce, dtype='int32')
+    if a_data_ptr == a.data.ptr and overwrite_input is False:
+        a = a.copy()
+    _replace_nan_kernel(n_reduce, numpy.finfo(a.dtype).max, a, n_reduce_each)
+    a = cupy.sort(a, axis=-1)
+
+    b = cupy.full(out_shape, cupy.nan, dtype=a.dtype)
+    _pickup_median_kernel(n_reduce, n_reduce_each, a, b)
+
+    if keepdims:
+        b = b.reshape(out_shape + [1, ] * len(reduce_axis))
+        axes = [-1, ] * b.ndim
+        for i, j in enumerate(out_axis + reduce_axis):
+            axes[j] = i
+        b = b.transpose(axes)
+
+    if out is None:
+        out = b
+    else:
+        elementwise_copy(b, out)
+    return out
+
+
+cdef _replace_nan_kernel = ElementwiseKernel(
+    'I n_reduce, T val', 'T a, raw I n_reduce_each',
+    '''
+    if (a != a) {
+        a = val;
+        atomicAdd(&(n_reduce_each[i / n_reduce]), -1);
+    }
+    ''',
+    'cupy_replace_nan'
+)
+
+cdef _pickup_median_kernel = ElementwiseKernel(
+    'I n_reduce, I n_reduce_each, raw T a', 'T b',
+    '''
+    if (n_reduce_each > 0) {
+        int l = (n_reduce_each - 1) / 2;
+        int h = (n_reduce_each    ) / 2;
+        if (l == h) {
+            b = a[l + n_reduce * i];
+        } else {
+            b = (a[l + n_reduce * i] + a[h + n_reduce * i])
+                / static_cast<T>(2.0);
+        }
+    }
+    ''',
+    'cupy_pickup_median'
+)
+
+
+cdef _ndarray_base _mean(
+        _ndarray_base a, axis=None, dtype=None, out=None, keepdims=False):
+    if a.size == 0:
+        # Return nan; see also https://github.com/numpy/numpy/issues/13582
+        return _mean_core_empty(a, axis, dtype, out, keepdims)
+    return _mean_core(a, axis, dtype, out, keepdims)
+
+cdef _ndarray_base _var(
+        _ndarray_base a, axis=None, dtype=None, out=None, ddof=0,
+        keepdims=False):
+
+    if axis is None:
+        axis = tuple(range(a.ndim))
+    if not isinstance(axis, tuple):
+        axis = (axis,)
+
+    dtype_mean = a.dtype
+    dtype_out = numpy.dtype(dtype)
+    if dtype is None:
+        if a.dtype.kind in 'biu':
+            dtype_mean = 'float64'
+            dtype_out = 'float64'
+        else:
+            dtype_mean = a.dtype
+            dtype_out = a.dtype
+            if a.dtype.kind == 'c':
+                dtype_out = numpy.dtype(a.dtype.char.lower())
+
+    shape = a.shape
+    cdef Py_ssize_t items = 1
+    for ax in axis:
+        items *= shape[ax]
+
+    # Make alpha NaN when array is empty, mimics NumPy behavior, resulting in
+    # NaN. See https://github.com/numpy/numpy/issues/13582 for an explanation
+    # on why NaN is the result.
+    div = max(items - ddof, 0)
+    alpha = 1. / div if div != 0 else nan
+
+    arrmean = a.mean(axis=axis, dtype=dtype_mean, out=None, keepdims=True)
+
+    if out is None:
+        if dtype_out == 'float16':
+            var_core = _var_core_float16
+        elif dtype_out == 'float32':
+            var_core = _var_core_float32
+        else:
+            var_core = _var_core_float64
+        return var_core(a, arrmean, alpha, axis=axis, keepdims=keepdims)
+
+    out = _var_core_out(a, arrmean, alpha, out, axis=axis, keepdims=keepdims)
+    return out.astype(dtype_out, copy=False)
+
+
+cdef _ndarray_base _std(
+        _ndarray_base a, axis=None, dtype=None, out=None, ddof=0,
+        keepdims=False):
+    ret = _var(
+        a, axis=axis, dtype=dtype, out=None, ddof=ddof, keepdims=keepdims)
+    return _math._sqrt(ret, dtype=dtype, out=out)
+
+
+cdef _norm_preamble = '''
+template <typename T> __device__ T my_norm(T x) { return x * x; }
+__device__ float my_norm(const complex<float>& x) { return norm(x); }
+__device__ double my_norm(const complex<double>& x) { return norm(x); }
+'''
+
+
+cdef _var_core_float16 = ReductionKernel(
+    'S x, T mean, float32 alpha', 'float16 out',
+    'my_norm(x - mean)',
+    'a + b', 'out = alpha * a', '0', 'cupy_var_core_float16',
+    preamble=_norm_preamble)
+
+
+cdef _var_core_float32 = ReductionKernel(
+    'S x, T mean, float32 alpha', 'float32 out',
+    'my_norm(x - mean)',
+    'a + b', 'out = alpha * a', '0', 'cupy_var_core_float32',
+    preamble=_norm_preamble)
+
+
+cdef _var_core_float64 = ReductionKernel(
+    'S x, T mean, float64 alpha', 'float64 out',
+    'my_norm(x - mean)',
+    'a + b', 'out = alpha * a', '0', 'cupy_var_core_float64',
+    preamble=_norm_preamble)
+
+
+cdef _var_core_out = ReductionKernel(
+    'S x, T mean, U alpha', 'U out',
+    'my_norm(x - mean)',
+    'a + b', 'out = alpha * a', '0', 'cupy_var_core_out',
+    preamble=_norm_preamble)
+
+
+# TODO(okuta) needs cast
+cdef _mean_core = create_reduction_func(
+    'cupy_mean',
+    ('?->d', 'B->d', 'h->d', 'H->d', 'i->d', 'I->d', 'l->d', 'L->d',
+     'q->d', 'Q->d',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('in0', 'a + b',
+     'out0 = a / _type_reduce(_in_ind.size() / _out_ind.size())', None))
+
+cdef _mean_core_empty = create_reduction_func(
+    'cupy_mean_empty',
+    ('?->d', 'B->d', 'h->d', 'H->d', 'i->d', 'I->d', 'l->d', 'L->d',
+     'q->d', 'Q->d',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d', 'F->F', 'D->D'),
+    ('in0', 'a + b',
+     'out0 = a / _type_reduce(_in_ind.size() / _out_ind.size())', None), 0)
+
+cdef _nanmean_preamble = '''
+template <typename T>
+struct nanmean_st{
+    typedef long long ll;
+    T value;
+    ll count;
+    __device__ nanmean_st() : value(0), count(0) { }
+    __device__ nanmean_st(T v) :
+        value(isnan(v) ? T(0) : v), count(isnan(v) ? 0 : 1) { }
+    __device__ nanmean_st(T v, ll c) : value(v), count(c) { }
+};
+
+template <typename T>
+__device__ nanmean_st<T> my_nanmean(
+        const nanmean_st<T>& a, const nanmean_st<T>& b) {
+    return nanmean_st<T>(a.value + b.value, a.count + b.count);
+}
+'''
+
+
+cdef _nanmean_func = create_reduction_func(
+    'cupy_nanmean',
+    ('e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    ('in0', 'my_nanmean(a, b)',
+     'out0 = a.value / type_out0_raw(a.count)', 'nanmean_st<type_out0_raw>'),
+    None, _nanmean_preamble)
+
+
+_count_non_nan = create_reduction_func(
+    'cupy_count_non_nan',
+    ('e->q', 'f->q', 'd->q', 'F->q', 'D->q'),
+    ('isnan(in0) ? 0 : 1', 'a + b', 'out0 = a', None), 0)
+
+
+cpdef _ndarray_base _nanmean(_ndarray_base a, axis, dtype, out, keepdims):
+    return _nanmean_func(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+cpdef _ndarray_base _nanstd(_ndarray_base a, axis, dtype, out, ddof, keepdims):
+    var = _nanvar(a, axis, dtype, None, ddof, keepdims)
+    return _math._sqrt(var, dtype=dtype, out=out)
+
+
+cpdef _ndarray_base _nanvar(_ndarray_base a, axis, dtype, out, ddof, keepdims):
+
+    _count = _count_non_nan(a, axis=axis, keepdims=True)
+    arrsum = _math._nansum(a, axis=axis, dtype=dtype, out=None, keepdims=True)
+
+    if out is None:
+        if a.dtype == cupy.complex64 or dtype == cupy.complex64:
+            nanvar_core = _nanvar_core_complex64
+        elif a.dtype == cupy.complex128 or dtype == cupy.complex128:
+            nanvar_core = _nanvar_core_complex128
+        else:
+            nanvar_core = _nanvar_core
+        out = nanvar_core(
+            a, arrsum, _count, ddof, axis=axis, keepdims=keepdims)
+    else:
+        _nanvar_core_out(
+            a, arrsum, _count, ddof, out, axis=axis, keepdims=keepdims)
+    return out
+
+
+cdef _nanvar_preamble = '''
+template <typename S, typename T>
+__device__ T nanvar_impl(S x, T mean, long long alpha) {
+    return (isnan(x) ? T(0) : T((x - mean) * (x - mean))) / alpha;
+}
+
+template <typename S, typename T>
+__device__ T nanvar_impl(complex<S> x, complex<T> mean, long long alpha) {
+    return (isnan(x) ? T(0) : T(norm(x - mean))) / alpha;
+}
+'''
+
+
+cdef _nanvar_core = ReductionKernel(
+    'S x, T sum, int64 _count, int64 ddof', 'S out',
+    'nanvar_impl(x, sum / _count, max(_count - ddof, 0LL))',
+    'a + b', 'out = a', '0', '_nanvar_core', preamble=_nanvar_preamble)
+
+
+cdef _nanvar_core_complex64 = ReductionKernel(
+    'complex64 x, complex64 sum, int64 _count, int64 ddof', 'float32 out',
+    'nanvar_impl(x, sum/static_cast<float>(_count), max(_count-ddof, 0LL))',
+    'a + b', 'out = a', '0', '_nanvar_core_complex64',
+    preamble=_nanvar_preamble)
+
+
+cdef _nanvar_core_complex128 = ReductionKernel(
+    'complex128 x, complex128 sum, int64 _count, int64 ddof', 'float64 out',
+    'nanvar_impl(x, sum/static_cast<double>(_count), max(_count-ddof, 0LL))',
+    'a + b', 'out = a', '0', '_nanvar_core_complex128',
+    preamble=_nanvar_preamble)
+
+
+cdef _nanvar_core_out = ReductionKernel(
+    'S x, T sum, int64 _count, int64 ddof', 'U out',
+    'nanvar_impl(x, sum / static_cast<T>(_count), max(_count - ddof, 0LL))',
+    'a + b', 'out = a', '0', '_nanvar_core', preamble=_nanvar_preamble)
+
+
+# Variables to expose to Python
+# (cythonized data cannot be exposed to Python, even with cpdef.)
+
+
+amax = _amax
+amin = _amin
diff --git a/cupy/_core/_scalar.pxd b/cupy/_core/_scalar.pxd
new file mode 100644
index 0000000..8cd84c5
--- /dev/null
+++ b/cupy/_core/_scalar.pxd
@@ -0,0 +1,37 @@
+cimport cython  # NOQA
+
+from libc.stdint cimport int8_t
+from libc.stdint cimport int32_t
+
+from cupy.cuda.function cimport CPointer
+
+
+@cython.final
+cdef class CScalar(CPointer):
+
+    cdef:
+        char kind
+        int8_t size
+
+    @staticmethod
+    cdef CScalar from_int32(int32_t value)
+
+    @staticmethod
+    cdef CScalar from_numpy_scalar_with_dtype(object x, object dtype)
+
+    @staticmethod
+    cdef CScalar _from_python_scalar(object x)
+
+    @staticmethod
+    cdef CScalar _from_numpy_scalar(object x)
+
+    cpdef apply_dtype(self, dtype)
+    cpdef get_numpy_type(self)
+
+
+cpdef str get_typename(dtype)
+
+cdef set scalar_type_set
+cdef CScalar scalar_to_c_scalar(object x)
+cdef object scalar_to_numpy_scalar(object x)
+cpdef str _get_cuda_scalar_repr(obj, dtype)
diff --git a/cupy/_core/_scalar.pyx b/cupy/_core/_scalar.pyx
new file mode 100644
index 0000000..f63aef4
--- /dev/null
+++ b/cupy/_core/_scalar.pyx
@@ -0,0 +1,386 @@
+from cpython cimport mem
+from libc.stdint cimport int8_t
+from libc.stdint cimport int16_t
+from libc.stdint cimport int32_t
+from libc.stdint cimport int64_t
+from libc.stdint cimport uint8_t
+from libc.stdint cimport uint16_t
+from libc.stdint cimport uint32_t
+from libc.stdint cimport uint64_t
+
+import numpy
+
+from cupy._core cimport _dtype
+from cupy._core import _dtype as _dtype_module
+from cupy._core cimport internal
+
+
+cdef union Scalar:
+    bint bool_
+    int8_t int8_
+    int16_t int16_
+    int32_t int32_
+    int64_t int64_
+    uint8_t uint8_
+    uint16_t uint16_
+    uint32_t uint32_
+    uint64_t uint64_
+    float float32_
+    double float64_
+
+
+cdef dict _typenames_base = {
+    numpy.dtype('float64'): 'double',
+    numpy.dtype('float32'): 'float',
+    numpy.dtype('float16'): 'float16',
+    numpy.dtype('complex128'): 'complex<double>',
+    numpy.dtype('complex64'): 'complex<float>',
+    numpy.dtype('int64'): 'long long',
+    numpy.dtype('int32'): 'int',
+    numpy.dtype('int16'): 'short',
+    numpy.dtype('int8'): 'signed char',
+    numpy.dtype('uint64'): 'unsigned long long',
+    numpy.dtype('uint32'): 'unsigned int',
+    numpy.dtype('uint16'): 'unsigned short',
+    numpy.dtype('uint8'): 'unsigned char',
+    numpy.dtype('bool'): 'bool',
+}
+
+
+cdef object _numpy_bool_ = numpy.bool_
+cdef object _numpy_int8 = numpy.int8
+cdef object _numpy_int16 = numpy.int16
+cdef object _numpy_int32 = numpy.int32
+cdef object _numpy_int64 = numpy.int64
+cdef object _numpy_uint8 = numpy.uint8
+cdef object _numpy_uint16 = numpy.uint16
+cdef object _numpy_uint32 = numpy.uint32
+cdef object _numpy_uint64 = numpy.uint64
+cdef object _numpy_float16 = numpy.float16
+cdef object _numpy_float32 = numpy.float32
+cdef object _numpy_float64 = numpy.float64
+cdef object _numpy_complex64 = numpy.complex64
+cdef object _numpy_complex128 = numpy.complex128
+cdef object _numpy_float_ = numpy.float_
+cdef object _numpy_complex_ = numpy.complex_
+
+
+cpdef str get_typename(dtype):
+    if dtype is None:
+        raise ValueError('dtype is None')
+    if dtype not in _typenames:
+        dtype = _dtype.get_dtype(dtype).type
+    return _typenames[dtype]
+
+
+cdef dict _typenames = {}
+cdef dict _dtype_kind_size_dict = {}
+
+
+cdef _setup_type_dict():
+    cdef char k
+    for i in _dtype_module.all_type_chars:
+        d = numpy.dtype(i)
+        t = d.type
+        _typenames[t] = _typenames_base[d]
+        k = ord(d.kind)
+        _dtype_kind_size_dict[t] = (k, d.itemsize)
+    # CUDA types
+    for t in ('cudaTextureObject_t',):
+        _typenames[t] = t
+
+
+_setup_type_dict()
+
+
+cdef set _python_scalar_type_set = {int, float, bool, complex}
+cdef set _numpy_scalar_type_set = set(_typenames.keys())
+cdef set scalar_type_set = _python_scalar_type_set | _numpy_scalar_type_set
+
+
+_int_iinfo = numpy.iinfo(int)
+cdef _int_min = _int_iinfo.min
+cdef _int_max = _int_iinfo.max
+cdef _int_type = _int_iinfo.dtype.type
+cdef bint _use_int32 = _int_type != _numpy_int64
+del _int_iinfo
+
+
+cpdef _python_scalar_to_numpy_scalar(x):
+    # Note that isinstance(x, int) matches with bool.
+    typ = type(x)
+    if typ is bool:
+        numpy_type = _numpy_bool_
+    elif typ is float:
+        numpy_type = _numpy_float_
+    elif typ is complex:
+        numpy_type = _numpy_complex_
+    else:
+        if 0x8000000000000000 <= x:
+            numpy_type = _numpy_uint64
+        elif _use_int32 and (x < _int_min or _int_max < x):
+            numpy_type = _numpy_int64
+        else:
+            # Generally `_int_type` is `numpy.int64`.
+            # On Windows, it is `numpy.int32`.
+            numpy_type = _int_type
+    return numpy_type(x)
+
+
+cdef class CScalar(CPointer):
+
+    ndim = 0
+
+    def __cinit__(self):
+        self.ptr = mem.PyMem_Malloc(
+            max(sizeof(Scalar), sizeof(double complex)))
+        self.kind = 0
+        self.size = -1
+
+    def __dealloc__(self):
+        mem.PyMem_Free(self.ptr)
+        self.ptr = <void*>0
+
+    @staticmethod
+    cdef CScalar from_int32(int32_t value):
+        cdef CScalar s = CScalar.__new__(CScalar)
+        (<int32_t *>s.ptr)[0] = value
+        s.kind = b'i'
+        s.size = 4
+        return s
+
+    @staticmethod
+    cdef CScalar from_numpy_scalar_with_dtype(object x, object dtype):
+        cdef CScalar ret = CScalar._from_numpy_scalar(x)
+        ret.apply_dtype(dtype)
+        return ret
+
+    @staticmethod
+    cdef CScalar _from_python_scalar(object x):
+        cdef CScalar ret = CScalar.__new__(CScalar)
+        cdef Scalar* s = <Scalar*>ret.ptr
+        typ = type(x)
+        if typ is bool:
+            s.bool_ = x
+            ret.kind = b'b'
+            ret.size = 1
+        elif typ is float:
+            s.float64_ = x
+            ret.kind = b'f'
+            ret.size = 8
+        elif typ is complex:
+            (<double complex*>ret.ptr)[0] = x
+            ret.kind = b'c'
+            ret.size = 16
+        else:
+            if 0x8000000000000000 <= x:
+                s.uint64_ = x
+                ret.kind = b'u'
+            else:
+                s.int64_ = x
+                ret.kind = b'i'
+            ret.size = 8
+        return ret
+
+    @staticmethod
+    cdef CScalar _from_numpy_scalar(object x):
+        cdef CScalar ret = CScalar.__new__(CScalar)
+        cdef Scalar* s = <Scalar*>ret.ptr
+        ret.kind = ord(x.dtype.kind)
+        if ret.kind == b'i':
+            s.int64_ = x
+            ret.size = 8
+        elif ret.kind == b'u':
+            s.uint64_ = x
+            ret.size = 8
+        elif ret.kind == b'f':
+            s.float64_ = x
+            ret.size = 8
+        elif ret.kind == b'b':
+            s.bool_ = x
+            ret.size = 1
+        elif ret.kind == b'c':
+            (<double complex*>ret.ptr)[0] = x
+            ret.size = 16
+        else:
+            assert False
+        return ret
+
+    cpdef apply_dtype(self, dtype):
+        cdef Scalar* s = <Scalar*>self.ptr
+        if self.kind == b'b':
+            val = s.bool_
+            assert self.size == 1
+        elif self.kind == b'c':
+            assert self.size == 16
+            val = (<double complex*>self.ptr)[0]
+        else:
+            assert self.size == 8
+            if self.kind == b'i':
+                val = s.int64_
+            elif self.kind == b'u':
+                val = s.uint64_
+            elif self.kind == b'f':
+                val = s.float64_
+            else:
+                assert False
+        cdef char kind
+        cdef int size
+        kind, size = <tuple>_dtype_kind_size_dict[dtype]
+        cdef int64_t val_i
+        cdef uint64_t val_u
+        if kind == b'b':
+            s.bool_ = val
+            assert size == 1
+        elif kind == b'i':
+            if self.kind == b'u':
+                # avoid overflow exception
+                val_i = s.uint64_
+            else:
+                val_i = val
+            if size == 1:
+                s.int8_ = val_i
+            elif size == 2:
+                s.int16_ = val_i
+            elif size == 4:
+                s.int32_ = val_i
+            elif size == 8:
+                s.int64_ = val_i
+            else:
+                assert False
+        elif kind == b'u':
+            if self.kind == b'i':
+                # avoid overflow exception
+                val_u = s.int64_
+            else:
+                val_u = val
+            if size == 1:
+                s.uint8_ = val_u
+            elif size == 2:
+                s.uint16_ = val_u
+            elif size == 4:
+                s.uint32_ = val_u
+            elif size == 8:
+                s.uint64_ = val_u
+            else:
+                assert False
+        elif kind == b'f':
+            if size == 2:
+                s.uint16_ = internal.to_float16(<float>val)
+            elif size == 4:
+                s.float32_ = val
+            elif size == 8:
+                s.float64_ = val
+            else:
+                assert False
+        elif kind == b'c':
+            if size == 8:
+                (<float complex*>self.ptr)[0] = val
+            elif size == 16:
+                (<double complex*>self.ptr)[0] = val
+            else:
+                assert False
+        else:
+            assert False
+        self.kind = kind
+        self.size = size
+
+    cpdef get_numpy_type(self):
+        if self.kind == b'b':
+            return _numpy_bool_
+        elif self.kind == b'i':
+            if self.size == 1:
+                return _numpy_int8
+            elif self.size == 2:
+                return _numpy_int16
+            elif self.size == 4:
+                return _numpy_int32
+            elif self.size == 8:
+                return _numpy_int64
+        elif self.kind == b'u':
+            if self.size == 1:
+                return _numpy_uint8
+            elif self.size == 2:
+                return _numpy_uint16
+            elif self.size == 4:
+                return _numpy_uint32
+            elif self.size == 8:
+                return _numpy_uint64
+        elif self.kind == b'f':
+            if self.size == 2:
+                return _numpy_float16
+            elif self.size == 4:
+                return _numpy_float32
+            elif self.size == 8:
+                return _numpy_float64
+        elif self.kind == b'c':
+            if self.size == 8:
+                return _numpy_complex64
+            elif self.size == 16:
+                return _numpy_complex128
+        assert False
+
+
+cdef CScalar scalar_to_c_scalar(object x):
+    # Converts a Python or NumPy scalar to a CScalar.
+    # Returns None if the argument is not a scalar.
+    typ = type(x)
+    if typ in _python_scalar_type_set:
+        return CScalar._from_python_scalar(x)
+    elif typ in _numpy_scalar_type_set:
+        return CScalar._from_numpy_scalar(x)
+    return None
+
+
+cdef object scalar_to_numpy_scalar(object x):
+    # Converts a Python or NumPy scalar to a NumPy scalar.
+    # Returns None if the argument is not a scalar.
+    typ = type(x)
+    if typ in _python_scalar_type_set:
+        return _python_scalar_to_numpy_scalar(x)
+    elif typ in _numpy_scalar_type_set:
+        return x
+    return None
+
+
+cpdef str _get_cuda_scalar_repr(obj, dtype):
+    if dtype.kind == 'b':
+        return str(bool(obj)).lower()
+    elif dtype.kind == 'i':
+        if dtype.itemsize < 8:
+            return str(int(obj))
+        else:
+            return str(int(obj)) + 'll'
+    elif dtype.kind == 'u':
+        if dtype.itemsize < 8:
+            return str(int(obj)) + 'u'
+        else:
+            return str(int(obj)) + 'ull'
+    elif dtype.kind == 'f':
+        if dtype.itemsize < 8:
+            if numpy.isnan(obj):
+                return 'CUDART_NAN_F'
+            elif numpy.isinf(obj):
+                if obj > 0:
+                    return 'CUDART_INF_F'
+                else:
+                    return '-CUDART_INF_F'
+            else:
+                return str(float(obj)) + 'f'
+        else:
+            if numpy.isnan(obj):
+                return 'CUDART_NAN'
+            elif numpy.isinf(obj):
+                if obj > 0:
+                    return 'CUDART_INF'
+                else:
+                    return '-CUDART_INF'
+            else:
+                return str(float(obj))
+    elif dtype.kind == 'c':
+        if dtype.itemsize == 8:
+            return f'thrust::complex<float>({obj.real}, {obj.imag})'
+        elif dtype.itemsize == 16:
+            return f'thrust::complex<double>({obj.real}, {obj.imag})'
+
+    raise TypeError(f'Unsupported dtype: {dtype}')
diff --git a/cupy/_core/_ufuncs.py b/cupy/_core/_ufuncs.py
new file mode 100644
index 0000000..37dfe0c
--- /dev/null
+++ b/cupy/_core/_ufuncs.py
@@ -0,0 +1,9 @@
+from cupy._core._kernel import create_ufunc
+
+
+elementwise_copy = create_ufunc(
+    'cupy_copy',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q', 'e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    'out0 = in0',
+    default_casting='unsafe')
diff --git a/cupy/_core/core.pxd b/cupy/_core/core.pxd
new file mode 100644
index 0000000..d55cada
--- /dev/null
+++ b/cupy/_core/core.pxd
@@ -0,0 +1,115 @@
+from libcpp cimport vector
+from cupy.cuda cimport memory
+
+from cupy.cuda.function cimport CPointer
+from cupy.cuda.function cimport Module
+from cupy._core._carray cimport shape_t
+from cupy._core._carray cimport strides_t
+
+
+cdef class _ndarray_base:
+    cdef:
+        object __weakref__
+        readonly Py_ssize_t size
+        public shape_t _shape
+        public strides_t _strides
+        readonly bint _c_contiguous
+        readonly bint _f_contiguous
+        # To do fast indexing in the CArray class
+        readonly bint _index_32_bits
+        readonly object dtype
+        readonly memory.MemoryPointer data
+        # TODO(niboshi): Return arbitrary owner object as `base` if the
+        # underlying memory is UnownedMemory.
+        readonly _ndarray_base base
+
+    cdef _init_fast(self, const shape_t& shape, dtype, bint c_order)
+    cpdef item(self)
+    cpdef tolist(self)
+    cpdef bytes tobytes(self, order=*)
+    cpdef tofile(self, fid, sep=*, format=*)
+    cpdef dump(self, file)
+    cpdef bytes dumps(self)
+    cpdef _ndarray_base astype(
+        self, dtype, order=*, casting=*, subok=*, copy=*)
+    cpdef _ndarray_base copy(self, order=*)
+    cpdef _ndarray_base view(self, dtype=*, array_class=*)
+    cpdef fill(self, value)
+    cpdef _ndarray_base swapaxes(self, Py_ssize_t axis1, Py_ssize_t axis2)
+    cpdef _ndarray_base flatten(self, order=*)
+    cpdef _ndarray_base ravel(self, order=*)
+    cpdef _ndarray_base squeeze(self, axis=*)
+    cpdef _ndarray_base take(self, indices, axis=*, out=*)
+    cpdef put(self, indices, values, mode=*)
+    cpdef repeat(self, repeats, axis=*)
+    cpdef choose(self, choices, out=*, mode=*)
+    cpdef sort(self, int axis=*)
+    cpdef _ndarray_base argsort(self, axis=*)
+    cpdef partition(self, kth, int axis=*)
+    cpdef _ndarray_base argpartition(self, kth, axis=*)
+    cpdef tuple nonzero(self)
+    cpdef _ndarray_base compress(self, condition, axis=*, out=*)
+    cpdef _ndarray_base diagonal(self, offset=*, axis1=*, axis2=*)
+    cpdef _ndarray_base max(self, axis=*, out=*, keepdims=*)
+    cpdef _ndarray_base argmax(self, axis=*, out=*, dtype=*, keepdims=*)
+    cpdef _ndarray_base min(self, axis=*, out=*, keepdims=*)
+    cpdef _ndarray_base argmin(self, axis=*, out=*, dtype=*, keepdims=*)
+    cpdef _ndarray_base ptp(self, axis=*, out=*, keepdims=*)
+    cpdef _ndarray_base clip(self, min=*, max=*, out=*)
+    cpdef _ndarray_base round(self, decimals=*, out=*)
+
+    cpdef _ndarray_base trace(self, offset=*, axis1=*, axis2=*, dtype=*, out=*)
+    cpdef _ndarray_base sum(self, axis=*, dtype=*, out=*, keepdims=*)
+    cpdef _ndarray_base cumsum(self, axis=*, dtype=*, out=*)
+    cpdef _ndarray_base mean(self, axis=*, dtype=*, out=*, keepdims=*)
+    cpdef _ndarray_base var(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
+    cpdef _ndarray_base std(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
+    cpdef _ndarray_base prod(self, axis=*, dtype=*, out=*, keepdims=*)
+    cpdef _ndarray_base cumprod(self, axis=*, dtype=*, out=*)
+    cpdef _ndarray_base _add_reduceat(self, indices, axis, dtype, out)
+    cpdef _ndarray_base all(self, axis=*, out=*, keepdims=*)
+    cpdef _ndarray_base any(self, axis=*, out=*, keepdims=*)
+    cpdef _ndarray_base conj(self)
+    cpdef _ndarray_base conjugate(self)
+    cpdef get(self, stream=*, order=*, out=*)
+    cpdef set(self, arr, stream=*)
+    cpdef _ndarray_base reduced_view(self, dtype=*)
+    cpdef _update_c_contiguity(self)
+    cpdef _update_f_contiguity(self)
+    cpdef _update_contiguity(self)
+    cpdef _set_shape_and_strides(self, const shape_t& shape,
+                                 const strides_t& strides,
+                                 bint update_c_contiguity,
+                                 bint update_f_contiguity)
+    cdef _ndarray_base _view(self, subtype, const shape_t& shape,
+                             const strides_t& strides,
+                             bint update_c_contiguity,
+                             bint update_f_contiguity, obj)
+    cpdef _set_contiguous_strides(
+        self, Py_ssize_t itemsize, bint is_c_contiguous)
+    cdef CPointer get_pointer(self)
+    cpdef object toDlpack(self)
+
+
+cpdef _ndarray_base _internal_ascontiguousarray(_ndarray_base a)
+cpdef _ndarray_base _internal_asfortranarray(_ndarray_base a)
+cpdef _ndarray_base ascontiguousarray(_ndarray_base a, dtype=*)
+cpdef _ndarray_base asfortranarray(_ndarray_base a, dtype=*)
+
+cpdef Module compile_with_cache(str source, tuple options=*, arch=*,
+                                cachd_dir=*, prepend_cupy_headers=*,
+                                backend=*, translate_cucomplex=*,
+                                enable_cooperative_groups=*,
+                                name_expressions=*, log_stream=*,
+                                bint jitify=*)
+
+
+# TODO(niboshi): Move to _routines_creation.pyx
+cpdef _ndarray_base array(
+    obj, dtype=*, bint copy=*, order=*, bint subok=*, Py_ssize_t ndmin=*)
+cpdef _ndarray_base _convert_object_with_cuda_array_interface(a)
+
+cdef _ndarray_base _ndarray_init(subtype, const shape_t& shape, dtype, obj)
+
+cdef _ndarray_base _create_ndarray_from_shape_strides(
+    subtype, const shape_t& shape, const strides_t& strides, dtype, obj)
diff --git a/cupy/_core/core.pyx b/cupy/_core/core.pyx
new file mode 100644
index 0000000..1cad745
--- /dev/null
+++ b/cupy/_core/core.pyx
@@ -0,0 +1,2813 @@
+# distutils: language = c++
+
+import contextlib
+import functools
+import os
+import pickle
+import re
+import warnings
+
+import numpy
+
+import cupy
+from cupy._core._kernel import create_ufunc
+from cupy._core._kernel import ElementwiseKernel
+from cupy._core._ufuncs import elementwise_copy
+from cupy._core import flags
+from cupy._core import syncdetect
+from cupy import cuda
+from cupy.cuda import memory as memory_module
+from cupy.cuda import stream as stream_mod
+
+
+from cupy_backends.cuda.api.runtime import CUDARuntimeError
+from cupy import _util
+
+cimport cython  # NOQA
+from libc.stdint cimport int64_t, intptr_t
+
+from cupy._core cimport _carray
+from cupy._core cimport _dtype
+from cupy._core._dtype cimport get_dtype
+from cupy._core._kernel cimport create_ufunc
+from cupy._core cimport _routines_binary as _binary
+from cupy._core cimport _routines_indexing as _indexing
+from cupy._core cimport _routines_linalg as _linalg
+from cupy._core cimport _routines_logic as _logic
+from cupy._core cimport _routines_manipulation as _manipulation
+from cupy._core cimport _routines_math as _math
+from cupy._core cimport _routines_sorting as _sorting
+from cupy._core cimport _routines_statistics as _statistics
+from cupy._core cimport _scalar
+from cupy._core cimport dlpack
+from cupy._core cimport internal
+from cupy.cuda cimport device
+from cupy.cuda cimport function
+from cupy.cuda cimport pinned_memory
+from cupy.cuda cimport memory
+from cupy.cuda cimport stream as stream_module
+from cupy_backends.cuda cimport stream as _stream_module
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda.libs cimport cublas
+
+
+# If rop of cupy.ndarray is called, cupy's op is the last chance.
+# If op of cupy.ndarray is called and the `other` is cupy.ndarray, too,
+# it is safe to call cupy's op.
+# Otherwise, use this function `_should_use_rop` to choose
+# * [True] return NotImplemented to defer rhs, or
+# * [False] call NumPy's ufunc to try all `__array_ufunc__`.
+# Note that extension types (`cdef class`) in Cython 0.x shares
+# implementations of op and rop. (i.e. `__radd__(self, other)` is
+# `__add__(other, self)`.)
+#
+# It follows NEP 13 except that cupy also implements the fallback to
+# `__array_priority__`, which seems fair and necessary because of the
+# following facts:
+# * `numpy` : `scipy.sparse` = `cupy` : `cupyx.scipy.sparse`;
+# * NumPy ignores `__array_priority__` attributes of arguments if NumPy finds
+#   `__array_function__` of `cupy.ndarray`;
+# * SciPy sparse classes don't implement `__array_function__` and they even
+#   don't set `__array_function__ = None` to opt-out the feature; and
+# * `__array_priority__` of SciPy sparse classes is respected because
+#   `numpy.ndarray.__array_function__` does not disable `__array_priority__`.
+@cython.profile(False)
+cdef inline _should_use_rop(x, y):
+    try:
+        y_ufunc = y.__array_ufunc__
+    except AttributeError:
+        # NEP 13's recommendation is `return False`.
+        xp = getattr(x, '__array_priority__', 0)
+        yp = getattr(y, '__array_priority__', 0)
+        return xp < yp
+    return y_ufunc is None
+
+
+cdef tuple _HANDLED_TYPES
+
+cdef object _null_context = contextlib.nullcontext()
+
+
+class ndarray(_ndarray_base):
+    """
+    __init__(self, shape, dtype=float, memptr=None, strides=None, order='C')
+
+    Multi-dimensional array on a CUDA device.
+
+    This class implements a subset of methods of :class:`numpy.ndarray`.
+    The difference is that this class allocates the array content on the
+    current GPU device.
+
+    Args:
+        shape (tuple of ints): Length of axes.
+        dtype: Data type. It must be an argument of :class:`numpy.dtype`.
+        memptr (cupy.cuda.MemoryPointer): Pointer to the array content head.
+        strides (tuple of ints or None): Strides of data in memory.
+        order ({'C', 'F'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Attributes:
+        base (None or cupy.ndarray): Base array from which this array is
+            created as a view.
+        data (cupy.cuda.MemoryPointer): Pointer to the array content head.
+        ~ndarray.dtype(numpy.dtype): Dtype object of element type.
+
+            .. seealso::
+               `Data type objects (dtype) \
+               <https://numpy.org/doc/stable/reference/arrays.dtypes.html>`_
+        ~ndarray.size (int): Number of elements this array holds.
+
+            This is equivalent to product over the shape tuple.
+
+            .. seealso:: :attr:`numpy.ndarray.size`
+
+    """
+
+    __module__ = 'cupy'
+
+    def __new__(cls, *args, _obj=None, _no_init=False, **kwargs):
+        x = super().__new__(cls, *args, **kwargs)
+        if _no_init:
+            return x
+        x._init(*args, **kwargs)
+        if cls is not ndarray:
+            x.__array_finalize__(_obj)
+        return x
+
+    def __init__(self, *args, **kwargs):
+        # Prevent from calling the super class `_ndarray_base.__init__()` as
+        # it is used to check accidental direct instantiation of underlaying
+        # `_ndarray_base` extention.
+        pass
+
+    def __array_finalize__(self, obj):
+        pass
+
+    # We provide the Python-level wrapper of `view` method to follow NumPy's
+    # API signature, as it seems that Cython's `cpdef`d methods does not take
+    # an argument named `type`. Cython also does not take starargs
+    # (`*args` and `**kwargs`) for `cpdef`d methods so we can not interpret the
+    # arguments `dtype` and `type` from them.
+    def view(self, dtype=None, type=None):
+        """Returns a view of the array.
+
+        Args:
+            dtype: If this is different from the data type of the array, the
+                returned view reinterpret the memory sequence as an array of
+                this type.
+
+        Returns:
+            cupy.ndarray: A view of the array. A reference to the original
+            array is stored at the :attr:`~ndarray.base` attribute.
+
+        .. seealso:: :meth:`numpy.ndarray.view`
+
+        """
+        return super(ndarray, self).view(dtype=dtype, array_class=type)
+
+
+cdef class _ndarray_base:
+
+    def __init__(self, *args, **kwargs):
+        # Raise an error if underlaying `_ndarray_base` extension type is
+        # directly instantiated. We must instantiate `ndarray` class instead
+        # for our ndarray subclassing mechanism.
+        raise RuntimeError('Must not be directly instantiated')
+
+    def _init(self, shape, dtype=float, memptr=None, strides=None,
+              order='C'):
+        cdef Py_ssize_t x, itemsize
+        cdef tuple s = internal.get_size(shape)
+        del shape
+
+        cdef int order_char = (
+            b'C' if order is None else internal._normalize_order(order))
+
+        # `strides` is prioritized over `order`, but invalid `order` should be
+        # checked even if `strides` is given.
+        if order_char != b'C' and order_char != b'F':
+            raise ValueError('order not understood. order=%s' % order)
+
+        # Check for erroneous shape
+        if len(s) > _carray.MAX_NDIM:
+            msg = 'maximum supported dimension for an ndarray is '
+            msg += f'{_carray.MAX_NDIM}, found {len(s)}'
+            raise ValueError(msg)
+        self._shape.reserve(len(s))
+        for x in s:
+            if x < 0:
+                raise ValueError('Negative dimensions are not allowed')
+            self._shape.push_back(x)
+        del s
+
+        # dtype
+        self.dtype, itemsize = _dtype.get_dtype_with_itemsize(dtype)
+
+        # Store shape and strides
+        if strides is not None:
+            if memptr is None:
+                raise ValueError('memptr is required if strides is given.')
+            self._set_shape_and_strides(self._shape, strides, True, True)
+        elif order_char == b'C':
+            self._set_contiguous_strides(itemsize, True)
+        elif order_char == b'F':
+            self._set_contiguous_strides(itemsize, False)
+        else:
+            assert False
+
+        # data
+        if memptr is None:
+            self.data = memory.alloc(self.size * itemsize)
+            self._index_32_bits = (self.size * itemsize) <= (1 << 31)
+        else:
+            self.data = memptr
+            bound = cupy._core._memory_range.get_bound(self)
+            self._index_32_bits = bound[1] - bound[0] <= (1 << 31)
+
+    cdef _init_fast(self, const shape_t& shape, dtype, bint c_order):
+        """ For internal ndarray creation. """
+        cdef Py_ssize_t itemsize
+        if shape.size() > _carray.MAX_NDIM:
+            msg = 'maximum supported dimension for an ndarray is '
+            msg += f'{_carray.MAX_NDIM}, found {shape.size()}'
+            raise ValueError(msg)
+        self._shape = shape
+        self.dtype, itemsize = _dtype.get_dtype_with_itemsize(dtype)
+        self._set_contiguous_strides(itemsize, c_order)
+        self.data = memory.alloc(self.size * itemsize)
+        self._index_32_bits = (self.size * itemsize) <= (1 << 31)
+
+    @property
+    def __cuda_array_interface__(self):
+        if runtime._is_hip_environment:
+            raise AttributeError(
+                'HIP/ROCm does not support cuda array interface')
+        cdef dict desc = {
+            'shape': self.shape,
+            'typestr': self.dtype.str,
+            'descr': self.dtype.descr,
+        }
+        cdef int ver = _util.CUDA_ARRAY_INTERFACE_EXPORT_VERSION
+        cdef intptr_t stream_ptr
+
+        if ver == 3:
+            stream_ptr = stream_module.get_current_stream_ptr()
+            # CAI v3 says setting the stream field to 0 is disallowed
+            if stream_ptr == 0:
+                stream_ptr = _stream_module.get_default_stream_ptr()
+            desc['stream'] = stream_ptr
+        elif ver == 2:
+            # Old behavior (prior to CAI v3): stream sync is explicitly handled
+            # by users. To restore this behavior, we do not export any stream
+            # if CUPY_CUDA_ARRAY_INTERFACE_EXPORT_VERSION is set to 2 (so that
+            # other participating libraries lacking a finer control over sync
+            # behavior can avoid syncing).
+            pass
+        else:
+            raise ValueError('CUPY_CUDA_ARRAY_INTERFACE_EXPORT_VERSION can '
+                             'only be set to 3 (default) or 2')
+        desc['version'] = ver
+        if self._c_contiguous:
+            desc['strides'] = None
+        else:
+            desc['strides'] = self.strides
+        if self.size > 0:
+            desc['data'] = (self.data.ptr, False)
+        else:
+            desc['data'] = (0, False)
+
+        return desc
+
+    def __dlpack__(self, stream=None):
+        # Note: the stream argument is supplied by the consumer, not by CuPy
+        curr_stream = stream_module.get_current_stream()
+        curr_stream_ptr = curr_stream.ptr
+
+        # stream must be an int for CUDA/ROCm
+        if not runtime._is_hip_environment:  # CUDA
+            if stream is None:
+                stream = runtime.streamLegacy
+            elif not isinstance(stream, int) or stream < -1:
+                # DLPack does not accept 0 as a valid stream, but there is a
+                # bug in PyTorch that exports the default stream as 0, which
+                # renders the protocol unusable, we will accept a 0 value
+                # meanwhile.
+                raise ValueError(
+                    f'On CUDA, the valid stream for the DLPack protocol is -1,'
+                    f' 1, 2, or any larger value, but {stream} was provided')
+            if stream == 0:
+                warnings.warn(
+                    'Stream 0 is passed from a library that you are'
+                    ' converting to; CuPy assumes 0 as a legacy default '
+                    'stream. Please report this problem to the library as this'
+                    ' violates the DLPack protocol.')
+                stream = runtime.streamLegacy
+            if curr_stream_ptr == 0:
+                curr_stream_ptr = runtime.streamLegacy
+        else:  # ROCm/HIP
+            if stream is None:
+                stream = 0
+            elif (not isinstance(stream, int) or stream < -1
+                    or stream in (1, 2)):
+                raise ValueError(
+                    f'On ROCm/HIP, the valid stream for the DLPack protocol is'
+                    f' -1, 0, or any value > 2, but {stream} was provided')
+
+        # if -1, no stream order should be established; otherwise, the consumer
+        # stream should wait for the work on CuPy's current stream to finish
+        if stream >= 0 and stream != curr_stream_ptr:
+            next_stream = stream_mod.ExternalStream(stream)
+            event = curr_stream.record()
+            next_stream.wait_event(event)
+
+        return dlpack.toDlpack(self)
+
+    def __dlpack_device__(self):
+        if not runtime._is_hip_environment:
+            attrs = runtime.pointerGetAttributes(self.data.ptr)
+            is_managed = (
+                attrs.type == runtime.memoryTypeManaged
+                and _util.DLPACK_EXPORT_VERSION >= (0, 6))
+            if is_managed:
+                device_type = dlpack.managed_CUDA
+            else:
+                device_type = dlpack.device_CUDA
+        else:
+            device_type = dlpack.device_ROCM
+        return (device_type, self.device.id)
+
+    # The definition order of attributes and methods are borrowed from the
+    # order of documentation at the following NumPy document.
+    # https://numpy.org/doc/stable/reference/arrays.ndarray.html
+
+    # -------------------------------------------------------------------------
+    # Memory layout
+    # -------------------------------------------------------------------------
+    @property
+    def flags(self):
+        """Object containing memory-layout information.
+
+        It only contains ``c_contiguous``, ``f_contiguous``, and ``owndata``
+        attributes. All of these are read-only. Accessing by indexes is also
+        supported.
+
+        .. seealso:: :attr:`numpy.ndarray.flags`
+
+        """
+        return flags.Flags(self._c_contiguous, self._f_contiguous,
+                           self.base is None)
+
+    property shape:
+        """Lengths of axes.
+
+        Setter of this property involves reshaping without copy. If the array
+        cannot be reshaped without copy, it raises an exception.
+
+        .. seealso: :attr:`numpy.ndarray.shape`
+
+        """
+
+        def __get__(self):
+            return tuple(self._shape)
+
+        def __set__(self, newshape):
+            _manipulation._ndarray_shape_setter(self, newshape)
+
+    @property
+    def strides(self):
+        """Strides of axes in bytes.
+
+        .. seealso:: :attr:`numpy.ndarray.strides`
+
+        """
+        return tuple(self._strides)
+
+    @property
+    def ndim(self):
+        """Number of dimensions.
+
+        ``a.ndim`` is equivalent to ``len(a.shape)``.
+
+        .. seealso:: :attr:`numpy.ndarray.ndim`
+
+        """
+        return self._shape.size()
+
+    @property
+    def itemsize(self):
+        """Size of each element in bytes.
+
+        .. seealso:: :attr:`numpy.ndarray.itemsize`
+
+        """
+        return self.dtype.itemsize
+
+    @property
+    def nbytes(self):
+        """Total size of all elements in bytes.
+
+        It does not count skips between elements.
+
+        .. seealso:: :attr:`numpy.ndarray.nbytes`
+
+        """
+        return self.size * self.dtype.itemsize
+
+    # -------------------------------------------------------------------------
+    # Other attributes
+    # -------------------------------------------------------------------------
+    @property
+    def T(self):
+        """Shape-reversed view of the array.
+
+        If ndim < 2, then this is just a reference to the array itself.
+
+        """
+        if self.ndim < 2:
+            return self
+        else:
+            return _manipulation._T(self)
+
+    @property
+    def flat(self):
+        return cupy.flatiter(self)
+
+    __array_priority__ = 100
+
+    # -------------------------------------------------------------------------
+    # Array interface
+    # -------------------------------------------------------------------------
+    # TODO(beam2d): Implement __array_interface__
+
+    # -------------------------------------------------------------------------
+    # foreign function interface
+    # -------------------------------------------------------------------------
+    @property
+    def cstruct(self):
+        """C representation of the array.
+
+        This property is used for sending an array to CUDA kernels. The type of
+        returned C structure is different for different dtypes and ndims. The
+        definition of C type is written in ``cupy/carray.cuh``.
+
+        """
+        return _CArray_from_ndarray(self)
+
+    # -------------------------------------------------------------------------
+    # Array conversion
+    # -------------------------------------------------------------------------
+    cpdef item(self):
+        """Converts the array with one element to a Python scalar
+
+        Returns:
+            int or float or complex: The element of the array.
+
+        .. seealso:: :meth:`numpy.ndarray.item`
+
+        """
+        if self.size != 1:
+            raise ValueError(
+                'can only convert an array of size 1 to a Python scalar')
+        return self.get().item()
+
+    cpdef tolist(self):
+        """Converts the array to a (possibly nested) Python list.
+
+        Returns:
+            list: The possibly nested Python list of array elements.
+
+        .. seealso:: :meth:`numpy.ndarray.tolist`
+
+        """
+        return self.get().tolist()
+
+    # TODO(okuta): Implement itemset
+    # TODO(okuta): Implement tostring
+
+    cpdef bytes tobytes(self, order='C'):
+        """Turns the array into a Python bytes object."""
+        return self.get().tobytes(order)
+
+    cpdef tofile(self, fid, sep='', format='%s'):
+        """Writes the array to a file.
+
+        .. seealso:: :meth:`numpy.ndarray.tofile`
+
+        """
+        self.get().tofile(fid, sep, format)
+
+    cpdef dump(self, file):
+        """Dumps a pickle of the array to a file.
+
+        Dumped file can be read back to :class:`cupy.ndarray` by
+        :func:`cupy.load`.
+
+        """
+        pickle.dump(self, file, -1)
+
+    cpdef bytes dumps(self):
+        """Dumps a pickle of the array to a string."""
+        return pickle.dumps(self, -1)
+
+    cpdef _ndarray_base astype(
+            self, dtype, order='K', casting=None, subok=None, copy=True):
+        """Casts the array to given data type.
+
+        Args:
+            dtype: Type specifier.
+            order ({'C', 'F', 'A', 'K'}): Row-major (C-style) or column-major
+                (Fortran-style) order.
+                When ``order`` is 'A', it uses 'F' if ``a`` is column-major and
+                uses 'C' otherwise.
+                And when ``order`` is 'K', it keeps strides as closely as
+                possible.
+            copy (bool): If it is False and no cast happens, then this method
+                returns the array itself. Otherwise, a copy is returned.
+
+        Returns:
+            If ``copy`` is False and no cast is required, then the array itself
+            is returned. Otherwise, it returns a (possibly casted) copy of the
+            array.
+
+        .. note::
+           This method currently does not support ``casting``, and ``subok``
+           arguments.
+
+        .. seealso:: :meth:`numpy.ndarray.astype`
+
+        """
+        cdef strides_t strides
+
+        # TODO(beam2d): Support casting and subok option
+        if casting is not None:
+            raise TypeError('casting is not supported yet')
+        if subok is not None:
+            raise TypeError('subok is not supported yet')
+
+        if order is None:
+            order = 'K'
+        cdef int order_char = internal._normalize_order(order)
+
+        dtype = get_dtype(dtype)
+        if dtype == self.dtype:
+            if not copy and (
+                    order_char == b'K' or
+                    order_char == b'A' and (self._c_contiguous or
+                                            self._f_contiguous) or
+                    order_char == b'C' and self._c_contiguous or
+                    order_char == b'F' and self._f_contiguous):
+                return self
+
+        order_char = internal._update_order_char(
+            self._c_contiguous, self._f_contiguous, order_char)
+
+        if order_char == b'K':
+            strides = internal._get_strides_for_order_K(self, dtype)
+            newarray = _ndarray_init(ndarray, self._shape, dtype, None)
+            # TODO(niboshi): Confirm update_x_contiguity flags
+            newarray._set_shape_and_strides(self._shape, strides, True, True)
+        else:
+            newarray = ndarray(self.shape, dtype=dtype, order=chr(order_char))
+
+        if self.size == 0:
+            # skip copy
+            if self.dtype.kind == 'c' and newarray.dtype.kind not in 'bc':
+                warnings.warn(
+                    'Casting complex values to real discards the imaginary '
+                    'part',
+                    numpy.ComplexWarning)
+        else:
+            elementwise_copy(self, newarray)
+        return newarray
+
+    # TODO(okuta): Implement byteswap
+
+    cpdef _ndarray_base copy(self, order='C'):
+        """Returns a copy of the array.
+
+        This method makes a copy of a given array in the current device.
+        Even when a given array is located in another device, you can copy it
+        to the current device.
+
+        Args:
+            order ({'C', 'F', 'A', 'K'}): Row-major (C-style) or column-major
+                (Fortran-style) order.
+                When ``order`` is 'A', it uses 'F' if ``a`` is column-major and
+                uses 'C' otherwise.
+                And when `order` is 'K', it keeps strides as closely as
+                possible.
+
+        .. seealso::
+           :func:`cupy.copy` for full documentation,
+           :meth:`numpy.ndarray.copy`
+
+        """
+        cdef _ndarray_base x
+        if self.size == 0:
+            return self.astype(self.dtype, order=order)
+
+        dev_id = device.get_device_id()
+        if self.data.device_id == dev_id:
+            return self.astype(self.dtype, order=order)
+
+        # It need to make a contiguous copy for copying from another device
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(self.device.id)
+            x = self.astype(self.dtype, order=order, copy=False)
+        finally:
+            runtime.setDevice(prev_device)
+        newarray = _ndarray_init(ndarray, x._shape, x.dtype, None)
+        if not x._c_contiguous and not x._f_contiguous:
+            raise NotImplementedError(
+                'CuPy cannot copy non-contiguous array between devices.')
+        # TODO(niboshi): Confirm update_x_contiguity flags
+        newarray._strides = x._strides
+        newarray._c_contiguous = x._c_contiguous
+        newarray._f_contiguous = x._f_contiguous
+
+        copy_context = _null_context
+        if runtime._is_hip_environment:
+            # HIP requires changing the active device to the one where
+            # src data is before the copy. From the docs:
+            # it is recommended to set the current device to the device
+            # where the src data is physically located.
+            copy_context = self.device
+        with copy_context:
+            newarray.data.copy_from_device_async(x.data, x.nbytes)
+        return newarray
+
+    cpdef _ndarray_base view(self, dtype=None, array_class=None):
+        cdef Py_ssize_t ndim, axis, tmp_size
+        cdef int self_is, v_is
+
+        if dtype is not None:
+            if type(dtype) is type and issubclass(dtype, ndarray):
+                if array_class is not None:
+                    raise ValueError('Cannot specify output type twice.')
+                array_class = dtype
+                dtype = None
+
+        if (
+            array_class is not None and (
+                type(array_class) is not type or
+                not issubclass(array_class, ndarray)
+            )
+        ):
+            raise ValueError('Type must be a sub-type of ndarray type')
+
+        if array_class is None:
+            array_class = type(self)
+
+        v = self._view(
+            array_class, self._shape, self._strides, False, False, self)
+        if dtype is None:
+            return v
+
+        v.dtype, v_is = _dtype.get_dtype_with_itemsize(dtype)
+        self_is = self.dtype.itemsize
+        if v_is == self_is:
+            return v
+
+        ndim = self._shape.size()
+        if ndim == 0:
+            raise ValueError(
+                'Changing the dtype of a 0d array is only supported if '
+                'the itemsize is unchanged')
+        axis = ndim - 1
+        if (
+            self._shape[axis] != 1
+            and self.size != 0
+            and self._strides[axis] != self.dtype.itemsize
+        ):
+            raise ValueError(
+                'To change to a dtype of a different size, the last axis '
+                'must be contiguous')
+
+        # Normalize `_strides[axis]` whenever itemsize changes
+        v._strides[axis] = v_is
+
+        tmp_size = v._shape[axis] * self_is
+        if tmp_size % v_is != 0:
+            raise ValueError(
+                'When changing to a larger dtype, its size must be a '
+                'divisor of the total size in bytes of the last axis '
+                'of the array.')
+            # itemsize of dtype in CuPy is one of 1, 2, 4, 8, 16.
+            # Thus, CuPy does not raise the following:
+            # raise ValueError(
+            #     'When changing to a smaller dtype, its size must be a '
+            #     'divisor of the size of original dtype')
+        v._shape[axis] = tmp_size // v_is
+        v.size = v.size * self_is // v_is  # divisible because shape[axis] is.
+
+        if axis != ndim - 1:
+            v._update_c_contiguity()
+        if axis != 0:
+            v._update_f_contiguity()
+        return v
+
+    # TODO(okuta): Implement getfield
+    # TODO(okuta): Implement setflags
+
+    cpdef fill(self, value):
+        """Fills the array with a scalar value.
+
+        Args:
+            value: A scalar value to fill the array content.
+
+        .. seealso:: :meth:`numpy.ndarray.fill`
+
+        """
+        if isinstance(value, cupy.ndarray):
+            if value.shape != ():
+                raise ValueError(
+                    'non-scalar cupy.ndarray cannot be used for fill')
+            value = value.astype(self.dtype, copy=False)
+            fill_kernel(value, self)
+            return
+
+        if isinstance(value, numpy.ndarray):
+            if value.shape != ():
+                raise ValueError(
+                    'non-scalar numpy.ndarray cannot be used for fill')
+            value = value.astype(self.dtype, copy=False).item()
+
+        if value == 0 and self._c_contiguous:
+            self.data.memset_async(0, self.nbytes)
+        else:
+            fill_kernel(value, self)
+
+    # -------------------------------------------------------------------------
+    # Shape manipulation
+    # -------------------------------------------------------------------------
+    def reshape(self, *shape, order='C'):
+        """Returns an array of a different shape and the same content.
+
+        .. seealso::
+           :func:`cupy.reshape` for full documentation,
+           :meth:`numpy.ndarray.reshape`
+
+        """
+        return _manipulation._ndarray_reshape(self, shape, order)
+
+    # TODO(okuta): Implement resize
+
+    def transpose(self, *axes):
+        """Returns a view of the array with axes permuted.
+
+        .. seealso::
+           :func:`cupy.transpose` for full documentation,
+           :meth:`numpy.ndarray.reshape`
+
+        """
+        return _manipulation._ndarray_transpose(self, axes)
+
+    cpdef _ndarray_base swapaxes(self, Py_ssize_t axis1, Py_ssize_t axis2):
+        """Returns a view of the array with two axes swapped.
+
+        .. seealso::
+           :func:`cupy.swapaxes` for full documentation,
+           :meth:`numpy.ndarray.swapaxes`
+
+        """
+        return _manipulation._ndarray_swapaxes(self, axis1, axis2)
+
+    cpdef _ndarray_base flatten(self, order='C'):
+        """Returns a copy of the array flatten into one dimension.
+
+        Args:
+            order ({'C', 'F', 'A', 'K'}):
+                'C' means to flatten in row-major (C-style) order.
+                'F' means to flatten in column-major (Fortran-
+                style) order. 'A' means to flatten in column-major
+                order if `self` is Fortran *contiguous* in memory,
+                row-major order otherwise. 'K' means to flatten
+                `self` in the order the elements occur in memory.
+                The default is 'C'.
+
+        Returns:
+            cupy.ndarray: A copy of the array with one dimension.
+
+        .. seealso:: :meth:`numpy.ndarray.flatten`
+
+        """
+        return _manipulation._ndarray_flatten(self, order)
+
+    cpdef _ndarray_base ravel(self, order='C'):
+        """Returns an array flattened into one dimension.
+
+        .. seealso::
+           :func:`cupy.ravel` for full documentation,
+           :meth:`numpy.ndarray.ravel`
+
+        """
+        return _internal_ascontiguousarray(
+            _manipulation._ndarray_ravel(self, order))
+
+    cpdef _ndarray_base squeeze(self, axis=None):
+        """Returns a view with size-one axes removed.
+
+        .. seealso::
+           :func:`cupy.squeeze` for full documentation,
+           :meth:`numpy.ndarray.squeeze`
+
+        """
+        return _manipulation._ndarray_squeeze(self, axis)
+
+    # -------------------------------------------------------------------------
+    # Item selection and manipulation
+    # -------------------------------------------------------------------------
+    cpdef _ndarray_base take(self, indices, axis=None, out=None):
+        """Returns an array of elements at given indices along the axis.
+
+        .. seealso::
+           :func:`cupy.take` for full documentation,
+           :meth:`numpy.ndarray.take`
+
+        """
+        return _indexing._ndarray_take(self, indices, axis, out)
+
+    cpdef put(self, indices, values, mode='wrap'):
+        """Replaces specified elements of an array with given values.
+
+        .. seealso::
+           :func:`cupy.put` for full documentation,
+           :meth:`numpy.ndarray.put`
+        """
+        return _indexing._ndarray_put(self, indices, values, mode)
+
+    cpdef repeat(self, repeats, axis=None):
+        """Returns an array with repeated arrays along an axis.
+
+        .. seealso::
+            :func:`cupy.repeat` for full documentation,
+            :meth:`numpy.ndarray.repeat`
+
+        """
+        return _manipulation._ndarray_repeat(self, repeats, axis)
+
+    cpdef choose(self, choices, out=None, mode='raise'):
+        # TODO(niboshi): Write docstring
+        return _indexing._ndarray_choose(self, choices, out, mode)
+
+    cpdef sort(self, int axis=-1):
+        """Sort an array, in-place with a stable sorting algorithm.
+
+        Args:
+            axis (int): Axis along which to sort. Default is -1, which means
+                sort along the last axis.
+
+        .. note::
+           For its implementation reason, ``ndarray.sort`` currently supports
+           only arrays with their own data, and does not support ``kind`` and
+           ``order`` parameters that ``numpy.ndarray.sort`` does support.
+
+        .. seealso::
+            :func:`cupy.sort` for full documentation,
+            :meth:`numpy.ndarray.sort`
+
+        """
+        # TODO(takagi): Support kind argument.
+        _sorting._ndarray_sort(self, axis)
+
+    cpdef _ndarray_base argsort(self, axis=-1):
+        """Returns the indices that would sort an array with stable sorting
+
+        Args:
+            axis (int or None): Axis along which to sort. Default is -1, which
+                means sort along the last axis. If None is supplied, the array
+                is flattened before sorting.
+
+        Returns:
+            cupy.ndarray: Array of indices that sort the array.
+
+        .. seealso::
+            :func:`cupy.argsort` for full documentation,
+            :meth:`numpy.ndarray.argsort`
+
+        """
+        # TODO(takagi): Support kind argument.
+        return _sorting._ndarray_argsort(self, axis)
+
+    cpdef partition(self, kth, int axis=-1):
+        """Partitions an array.
+
+        Args:
+            kth (int or sequence of ints): Element index to partition by. If
+                supplied with a sequence of k-th it will partition all elements
+                indexed by k-th of them into their sorted position at once.
+
+            axis (int): Axis along which to sort. Default is -1, which means
+                sort along the last axis.
+
+        .. seealso::
+            :func:`cupy.partition` for full documentation,
+            :meth:`numpy.ndarray.partition`
+
+        """
+        _sorting._ndarray_partition(self, kth, axis)
+
+    cpdef _ndarray_base argpartition(self, kth, axis=-1):
+        """Returns the indices that would partially sort an array.
+
+        Args:
+            kth (int or sequence of ints): Element index to partition by. If
+                supplied with a sequence of k-th it will partition all elements
+                indexed by k-th of them into their sorted position at once.
+            axis (int or None): Axis along which to sort. Default is -1, which
+                means sort along the last axis. If None is supplied, the array
+                is flattened before sorting.
+
+        Returns:
+            cupy.ndarray: Array of the same type and shape as ``a``.
+
+        .. seealso::
+            :func:`cupy.argpartition` for full documentation,
+            :meth:`numpy.ndarray.argpartition`
+
+        """
+        return _sorting._ndarray_argpartition(self, kth, axis)
+
+    def searchsorted(self, v, side='left', sorter=None):
+        """Finds indices where elements of v should be inserted to maintain order.
+
+        For full documentation, see :func:`cupy.searchsorted`
+
+        Returns:
+
+        .. seealso:: :func:`numpy.searchsorted`
+
+        """  # NOQA
+        return cupy.searchsorted(self, v, side, sorter)
+
+    cpdef tuple nonzero(self):
+        """Return the indices of the elements that are non-zero.
+
+        Returned Array is containing the indices of the non-zero elements
+        in that dimension.
+
+        Returns:
+            tuple of arrays: Indices of elements that are non-zero.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            :func:`numpy.nonzero`
+
+        """
+        return _indexing._ndarray_nonzero(self)
+
+    cpdef _ndarray_base compress(self, condition, axis=None, out=None):
+        """Returns selected slices of this array along given axis.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+           :func:`cupy.compress` for full documentation,
+           :meth:`numpy.ndarray.compress`
+
+        """
+        return _indexing._ndarray_compress(self, condition, axis, out)
+
+    cpdef _ndarray_base diagonal(self, offset=0, axis1=0, axis2=1):
+        """Returns a view of the specified diagonals.
+
+        .. seealso::
+           :func:`cupy.diagonal` for full documentation,
+           :meth:`numpy.ndarray.diagonal`
+
+        """
+        return _indexing._ndarray_diagonal(self, offset, axis1, axis2)
+
+    # -------------------------------------------------------------------------
+    # Calculation
+    # -------------------------------------------------------------------------
+    cpdef _ndarray_base max(self, axis=None, out=None, keepdims=False):
+        """Returns the maximum along a given axis.
+
+        .. seealso::
+           :func:`cupy.amax` for full documentation,
+           :meth:`numpy.ndarray.max`
+
+        """
+        return _statistics._ndarray_max(self, axis, out, None, keepdims)
+
+    cpdef _ndarray_base argmax(
+            self, axis=None, out=None, dtype=None, keepdims=False):
+        """Returns the indices of the maximum along a given axis.
+
+        .. note::
+           ``dtype`` and ``keepdim`` arguments are specific to CuPy. They are
+           not in NumPy.
+
+        .. note::
+           ``axis`` argument accepts a tuple of ints, but this is specific to
+           CuPy. NumPy does not support it.
+
+        .. seealso::
+           :func:`cupy.argmax` for full documentation,
+           :meth:`numpy.ndarray.argmax`
+
+        """
+        return _statistics._ndarray_argmax(self, axis, out, dtype, keepdims)
+
+    cpdef _ndarray_base min(self, axis=None, out=None, keepdims=False):
+        """Returns the minimum along a given axis.
+
+        .. seealso::
+           :func:`cupy.amin` for full documentation,
+           :meth:`numpy.ndarray.min`
+
+        """
+        return _statistics._ndarray_min(self, axis, out, None, keepdims)
+
+    cpdef _ndarray_base argmin(
+            self, axis=None, out=None, dtype=None, keepdims=False):
+        """Returns the indices of the minimum along a given axis.
+
+        .. note::
+           ``dtype`` and ``keepdim`` arguments are specific to CuPy. They are
+           not in NumPy.
+
+        .. note::
+           ``axis`` argument accepts a tuple of ints, but this is specific to
+           CuPy. NumPy does not support it.
+
+        .. seealso::
+           :func:`cupy.argmin` for full documentation,
+           :meth:`numpy.ndarray.argmin`
+
+        """
+        return _statistics._ndarray_argmin(self, axis, out, dtype, keepdims)
+
+    cpdef _ndarray_base ptp(self, axis=None, out=None, keepdims=False):
+        """Returns (maximum - minimum) along a given axis.
+
+        .. seealso::
+           :func:`cupy.ptp` for full documentation,
+           :meth:`numpy.ndarray.ptp`
+
+        """
+        return _statistics._ndarray_ptp(self, axis, out, keepdims)
+
+    cpdef _ndarray_base clip(self, min=None, max=None, out=None):
+        """Returns an array with values limited to [min, max].
+
+        .. seealso::
+           :func:`cupy.clip` for full documentation,
+           :meth:`numpy.ndarray.clip`
+
+        """
+        return _math._ndarray_clip(self, min, max, out)
+
+    cpdef _ndarray_base round(self, decimals=0, out=None):
+        """Returns an array with values rounded to the given number of decimals.
+
+        .. seealso::
+           :func:`cupy.around` for full documentation,
+           :meth:`numpy.ndarray.round`
+
+        """  # NOQA
+        return _round_ufunc(self, decimals, out=out)
+
+    cpdef _ndarray_base trace(
+            self, offset=0, axis1=0, axis2=1, dtype=None, out=None):
+        """Returns the sum along diagonals of the array.
+
+        .. seealso::
+           :func:`cupy.trace` for full documentation,
+           :meth:`numpy.ndarray.trace`
+
+        """
+        d = self.diagonal(offset, axis1, axis2)
+        return d.sum(-1, dtype, out, False)
+
+    cpdef _ndarray_base sum(
+            self, axis=None, dtype=None, out=None, keepdims=False):
+        """Returns the sum along a given axis.
+
+        .. seealso::
+           :func:`cupy.sum` for full documentation,
+           :meth:`numpy.ndarray.sum`
+
+        """
+        return _math._ndarray_sum(self, axis, dtype, out, keepdims)
+
+    cpdef _ndarray_base cumsum(self, axis=None, dtype=None, out=None):
+        """Returns the cumulative sum of an array along a given axis.
+
+        .. seealso::
+           :func:`cupy.cumsum` for full documentation,
+           :meth:`numpy.ndarray.cumsum`
+
+        """
+        return _math._ndarray_cumsum(self, axis, dtype, out)
+
+    cpdef _ndarray_base mean(
+            self, axis=None, dtype=None, out=None, keepdims=False):
+        """Returns the mean along a given axis.
+
+        .. seealso::
+           :func:`cupy.mean` for full documentation,
+           :meth:`numpy.ndarray.mean`
+
+        """
+        return _statistics._ndarray_mean(self, axis, dtype, out, keepdims)
+
+    cpdef _ndarray_base var(
+            self, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
+        """Returns the variance along a given axis.
+
+        .. seealso::
+           :func:`cupy.var` for full documentation,
+           :meth:`numpy.ndarray.var`
+
+        """
+        return _statistics._ndarray_var(
+            self, axis, dtype, out, ddof, keepdims)
+
+    cpdef _ndarray_base std(
+            self, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
+        """Returns the standard deviation along a given axis.
+
+        .. seealso::
+           :func:`cupy.std` for full documentation,
+           :meth:`numpy.ndarray.std`
+
+        """
+        return _statistics._ndarray_std(self, axis, dtype, out, ddof, keepdims)
+
+    cpdef _ndarray_base prod(
+            self, axis=None, dtype=None, out=None, keepdims=None):
+        """Returns the product along a given axis.
+
+        .. seealso::
+           :func:`cupy.prod` for full documentation,
+           :meth:`numpy.ndarray.prod`
+
+        """
+        return _math._ndarray_prod(self, axis, dtype, out, keepdims)
+
+    cpdef _ndarray_base cumprod(self, axis=None, dtype=None, out=None):
+        """Returns the cumulative product of an array along a given axis.
+
+        .. seealso::
+           :func:`cupy.cumprod` for full documentation,
+           :meth:`numpy.ndarray.cumprod`
+
+        """
+        return _math._ndarray_cumprod(self, axis, dtype, out)
+
+    cpdef _ndarray_base _add_reduceat(self, indices, axis, dtype, out):
+        return _indexing._add_reduceat(self, indices, axis, dtype, out)
+
+    cpdef _ndarray_base all(self, axis=None, out=None, keepdims=False):
+        # TODO(niboshi): Write docstring
+        return _logic._ndarray_all(self, axis, out, keepdims)
+
+    cpdef _ndarray_base any(self, axis=None, out=None, keepdims=False):
+        # TODO(niboshi): Write docstring
+        return _logic._ndarray_any(self, axis, out, keepdims)
+
+    # -------------------------------------------------------------------------
+    # Arithmetic and comparison operations
+    # -------------------------------------------------------------------------
+    # Comparison operators:
+
+    def __richcmp__(object self, object other, int op):
+        if isinstance(other, ndarray):
+            if op == 0:
+                return _logic._ndarray_less(self, other)
+            if op == 1:
+                return _logic._ndarray_less_equal(self, other)
+            if op == 2:
+                return _logic._ndarray_equal(self, other)
+            if op == 3:
+                return _logic._ndarray_not_equal(self, other)
+            if op == 4:
+                return _logic._ndarray_greater(self, other)
+            if op == 5:
+                return _logic._ndarray_greater_equal(self, other)
+        elif not _should_use_rop(self, other):
+            if isinstance(other, numpy.ndarray) and other.ndim == 0:
+                other = other.item()  # Workaround for numpy<1.13
+            if op == 0:
+                return numpy.less(self, other)
+            if op == 1:
+                return numpy.less_equal(self, other)
+            if op == 2:
+                # cupy.ndarray does not support dtype=object, but
+                # allow comparison with None, Ellipsis, and etc.
+                if type(other).__eq__ is object.__eq__:
+                    # Implies `other` is neither (Python/NumPy) scalar nor
+                    # ndarray. With object's default __eq__, it never
+                    # equals to an element of cupy.ndarray.
+                    return cupy.zeros(self._shape, dtype=cupy.bool_)
+                return numpy.equal(self, other)
+            if op == 3:
+                if (
+                    type(other).__eq__ is object.__eq__
+                    and type(other).__ne__ is object.__ne__
+                ):
+                    # Similar to eq, but ne falls back to `not __eq__`.
+                    return cupy.ones(self._shape, dtype=cupy.bool_)
+                return numpy.not_equal(self, other)
+            if op == 4:
+                return numpy.greater(self, other)
+            if op == 5:
+                return numpy.greater_equal(self, other)
+        return NotImplemented
+
+    # Truth value of an array (bool):
+
+    def __nonzero__(self):
+        if self.size == 0:
+            msg = ('The truth value of an empty array is ambiguous. Returning '
+                   'False, but in future this will result in an error. Use '
+                   '`array.size > 0` to check that an array is not empty.')
+            warnings.warn(msg, DeprecationWarning)
+            return False
+        elif self.size == 1:
+            return bool(self.get())
+        else:
+            msg = ('The truth value of an array with more than one element is '
+                   'ambiguous. Use a.any() or a.all()')
+            raise ValueError(msg)
+
+    # Unary operations:
+
+    def __neg__(self):
+        return _math._negative(self)
+
+    def __pos__(self):
+        if self.dtype == numpy.bool_:
+            msg = ("Applying '+' to a non-numerical array is ill-defined. "
+                   'Returning a copy, but in the future this will error.')
+            warnings.warn(msg, DeprecationWarning)
+            return self.copy()
+        return _math._positive(self)
+
+    def __abs__(self):
+        return _math._absolute(self)
+
+    def __invert__(self):
+        return _binary._invert(self)
+
+    # Arithmetic:
+
+    def __add__(x, y):
+        if isinstance(y, ndarray):
+            return _math._add(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.add(x, y)
+
+    def __sub__(x, y):
+        if isinstance(y, ndarray):
+            return _math._subtract(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.subtract(x, y)
+
+    def __mul__(x, y):
+        if isinstance(y, ndarray):
+            return _math._multiply(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.multiply(x, y)
+
+    def __matmul__(x, y):
+        if isinstance(y, ndarray):
+            return _linalg.matmul(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.matmul(x, y)
+
+    def __div__(x, y):
+        if isinstance(y, ndarray):
+            return _math._divide(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.divide(x, y)
+
+    def __truediv__(x, y):
+        if isinstance(y, ndarray):
+            return _math._true_divide(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.true_divide(x, y)
+
+    def __floordiv__(x, y):
+        if isinstance(y, ndarray):
+            return _math._floor_divide(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.floor_divide(x, y)
+
+    def __mod__(x, y):
+        if isinstance(y, ndarray):
+            return _math._remainder(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.remainder(x, y)
+
+    def __divmod__(x, y):
+        if isinstance(y, ndarray):
+            return divmod(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.divmod(x, y)
+
+    def __pow__(x, y, modulo):
+        # Note that we ignore the modulo argument as well as NumPy.
+        if isinstance(y, ndarray):
+            return _math._power(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.power(x, y)
+
+    def __lshift__(x, y):
+        if isinstance(y, ndarray):
+            return _binary._left_shift(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.left_shift(x, y)
+
+    def __rshift__(x, y):
+        if isinstance(y, ndarray):
+            return _binary._right_shift(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.right_shift(x, y)
+
+    def __and__(x, y):
+        if isinstance(y, ndarray):
+            return _binary._bitwise_and(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.bitwise_and(x, y)
+
+    def __or__(x, y):
+        if isinstance(y, ndarray):
+            return _binary._bitwise_or(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.bitwise_or(x, y)
+
+    def __xor__(x, y):
+        if isinstance(y, ndarray):
+            return _binary._bitwise_xor(x, y)
+        elif _should_use_rop(x, y):
+            return NotImplemented
+        else:
+            return numpy.bitwise_xor(x, y)
+
+    # Arithmetic, in-place:
+
+    def __iadd__(self, other):
+        return _math._add(self, other, self)
+
+    def __isub__(self, other):
+        return _math._subtract(self, other, self)
+
+    def __imul__(self, other):
+        return _math._multiply(self, other, self)
+
+    def __idiv__(self, other):
+        return _math._divide(self, other, self)
+
+    def __itruediv__(self, other):
+        return _math._true_divide(self, other, self)
+
+    def __ifloordiv__(self, other):
+        return _math._floor_divide(self, other, self)
+
+    def __imod__(self, other):
+        return _math._remainder(self, other, self)
+
+    def __ipow__(self, other):
+        return _math._power(self, other, self)
+
+    def __ilshift__(self, other):
+        return _binary._left_shift(self, other, self)
+
+    def __irshift__(self, other):
+        return _binary._right_shift(self, other, self)
+
+    def __iand__(self, other):
+        return _binary._bitwise_and(self, other, self)
+
+    def __ior__(self, other):
+        return _binary._bitwise_or(self, other, self)
+
+    def __ixor__(self, other):
+        return _binary._bitwise_xor(self, other, self)
+
+    cpdef _ndarray_base conj(self):
+        return _math._ndarray_conj(self)
+
+    cpdef _ndarray_base conjugate(self):
+        return _math._ndarray_conj(self)
+
+    @property
+    def real(self):
+        return _math._ndarray_real_getter(self)
+
+    @real.setter
+    def real(self, value):
+        _math._ndarray_real_setter(self, value)
+
+    @property
+    def imag(self):
+        return _math._ndarray_imag_getter(self)
+
+    @imag.setter
+    def imag(self, value):
+        _math._ndarray_imag_setter(self, value)
+
+    # -------------------------------------------------------------------------
+    # Special methods
+    # -------------------------------------------------------------------------
+    # For standard library functions:
+
+    def __copy__(self):
+        return self.copy()
+
+    def __deepcopy__(self, memo):
+        # It need to make a contiguous copy for copying from another device
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(self.device.id)
+            return self.copy()
+        finally:
+            runtime.setDevice(prev_device)
+
+    def __reduce__(self):
+        return array, (self.get(),)
+
+    # Basic customization:
+
+    # _ndarray_base does not define __new__
+
+    def __array__(self, dtype=None):
+        # TODO(imanishi): Support an environment variable or a global
+        # configure flag that allows implicit conversions to NumPy array.
+        # (See https://github.com/cupy/cupy/issues/589 for the detail.)
+        raise TypeError(
+            'Implicit conversion to a NumPy array is not allowed. '
+            'Please use `.get()` to construct a NumPy array explicitly.')
+
+    @classmethod
+    def __class_getitem__(cls, tuple item):
+        from cupy.typing._generic_alias import GenericAlias
+        item1, item2 = item
+        return GenericAlias(cupy.ndarray, (item1, item2))
+
+    # TODO(okuta): Implement __array_wrap__
+
+    # Container customization:
+
+    def __iter__(self):
+        if self._shape.size() == 0:
+            raise TypeError('iteration over a 0-d array')
+        return (self[i] for i in range(self._shape[0]))
+
+    def __len__(self):
+        if self._shape.size() == 0:
+            raise TypeError('len() of unsized object')
+        return self._shape[0]
+
+    def __getitem__(self, slices):
+        """x.__getitem__(y) <==> x[y]
+
+        Supports both basic and advanced indexing.
+
+        .. note::
+
+            Currently, it does not support ``slices`` that consists of more
+            than one boolean arrays
+
+        .. note::
+
+           CuPy handles out-of-bounds indices differently from NumPy.
+           NumPy handles them by raising an error, but CuPy wraps around them.
+
+        Example:
+
+            >>> a = cupy.arange(3)
+            >>> a[[1, 3]]
+            array([1, 0])
+
+        """
+        return _indexing._ndarray_getitem(self, slices)
+
+    def __setitem__(self, slices, value):
+        """x.__setitem__(slices, y) <==> x[slices] = y
+
+        Supports both basic and advanced indexing.
+
+        .. note::
+
+            Currently, it does not support ``slices`` that consists of more
+            than one boolean arrays
+
+        .. note::
+
+            CuPy handles out-of-bounds indices differently from NumPy when
+            using integer array indexing.
+            NumPy handles them by raising an error, but CuPy wraps around them.
+
+            >>> import cupy
+            >>> x = cupy.arange(3)
+            >>> x[[1, 3]] = 10
+            >>> x
+            array([10, 10,  2])
+
+        .. note::
+
+            The behavior differs from NumPy when integer arrays in ``slices``
+            reference the same location multiple times.
+            In that case, the value that is actually stored is undefined.
+
+            >>> import cupy
+            >>> a = cupy.zeros((2,))
+            >>> i = cupy.arange(10000) % 2
+            >>> v = cupy.arange(10000).astype(cupy.float_)
+            >>> a[i] = v
+            >>> a  # doctest: +SKIP
+            array([9150., 9151.])
+
+            On the other hand, NumPy stores the value corresponding to the
+            last index among the indices referencing duplicate locations.
+
+            >>> import numpy
+            >>> a_cpu = numpy.zeros((2,))
+            >>> i_cpu = numpy.arange(10000) % 2
+            >>> v_cpu = numpy.arange(10000).astype(numpy.float_)
+            >>> a_cpu[i_cpu] = v_cpu
+            >>> a_cpu
+            array([9998., 9999.])
+
+        """
+        if _util.ENABLE_SLICE_COPY and (
+                type(slices) is slice
+                and slices == slice(None, None, None)
+                and isinstance(value, numpy.ndarray)
+        ):
+            if (self.dtype == value.dtype
+                    and self.shape == value.shape
+                    and (self._f_contiguous or self._c_contiguous)):
+                order = 'F' if self._f_contiguous else 'C'
+                tmp = value.ravel(order)
+                ptr = tmp.ctypes.data
+                stream_ptr = stream_module.get_current_stream_ptr()
+                if stream_ptr == 0:
+                    self.data.copy_from_host(ptr, self.nbytes)
+                else:
+                    self.data.copy_from_host_async(ptr, self.nbytes)
+            else:
+                raise ValueError(
+                    'copying a numpy.ndarray to a cupy.ndarray by empty slice '
+                    'assignment must ensure arrays have same shape and dtype')
+        else:
+            _indexing._ndarray_setitem(self, slices, value)
+
+    def scatter_add(self, slices, value):
+        """Adds given values to specified elements of an array.
+
+        .. seealso::
+            :func:`cupyx.scatter_add` for full documentation.
+
+        """
+        warnings.warn(
+            '`ndarray.scatter_add` is deprecated. '
+            'Please use `cupy.add.at` instead.',
+            DeprecationWarning)
+        self._scatter_op(slices, value, 'add')
+
+    def scatter_max(self, slices, value):
+        """Stores a maximum value of elements specified by indices to an array.
+
+        .. seealso::
+            :func:`cupyx.scatter_max` for full documentation.
+
+        """
+        warnings.warn(
+            '`ndarray.scatter_max` is deprecated '
+            'Please use `cupy.maximum.at` instead.',
+            DeprecationWarning)
+        self._scatter_op(slices, value, 'max')
+
+    def scatter_min(self, slices, value):
+        """Stores a minimum value of elements specified by indices to an array.
+
+        .. seealso::
+            :func:`cupyx.scatter_min` for full documentation.
+
+        """
+        warnings.warn(
+            '`ndarray.scatter_min` is deprecated '
+            'Please use `cupy.minimum.at` instead.',
+            DeprecationWarning)
+        self._scatter_op(slices, value, 'min')
+
+    def _scatter_op(self, slices, value, op):
+        _indexing._scatter_op(self, slices, value, op)
+
+    # TODO(okuta): Implement __getslice__
+    # TODO(okuta): Implement __setslice__
+    # TODO(okuta): Implement __contains__
+
+    # numpy/ufunc compat
+    def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
+
+        """Apply unary or binary ufunc to this array
+
+        If binary, only allow if second argument is another cupy ndarray or
+        a number, i.e., raise ValueError instead of silently converting a
+        numpy array.
+        """
+        import cupy  # top-level ufuncs
+        import cupyx.scipy.special  # special ufuncs
+        inout = inputs
+        if 'out' in kwargs:
+            # need to unfold tuple argument in kwargs
+            # TODO(ecastill) GUFuncs support more than one output
+            out = kwargs['out']
+            if len(out) != 1:
+                raise ValueError('The \'out\' parameter must have exactly one '
+                                 'array value')
+            inout += out
+            kwargs['out'] = out[0]
+
+        if method in (
+                '__call__', 'outer', 'at', 'reduce', 'accumulate', 'reduceat'
+        ):
+            name = ufunc.__name__
+            try:
+                func = getattr(cupy, name, None) or getattr(
+                    cupyx.scipy.special, name
+                )
+                if method != '__call__':
+                    func = getattr(func, method)
+            except AttributeError:
+                return NotImplemented
+            for x in inout:
+                # numpy.ndarray is handled and then TypeError is raised due to
+                # implicit host-to-device conversion.
+                # Except for numpy.ndarray, types should be supported by
+                # `_kernel._preprocess_args`.
+                check = (hasattr(x, '__cuda_array_interface__')
+                         or hasattr(x, '__cupy_get_ndarray__'))
+                if runtime._is_hip_environment and isinstance(x, ndarray):
+                    check = True
+                if (not check
+                        and not type(x) in _scalar.scalar_type_set
+                        and not isinstance(x, numpy.ndarray)):
+                    return NotImplemented
+            if name in [
+                    'greater', 'greater_equal', 'less', 'less_equal',
+                    'equal', 'not_equal']:
+                # workaround for numpy/numpy#12142
+                inputs = tuple([
+                    x.item()
+                    if isinstance(x, numpy.ndarray) and x.ndim == 0
+                    else x
+                    for x in inputs
+                ])
+            return func(*inputs, **kwargs)
+        else:
+            return NotImplemented
+
+    def __array_function__(self, func, types, args, kwargs):
+        try:
+            module = functools.reduce(
+                getattr, func.__module__.split('.')[1:], cupy)
+            cupy_func = getattr(module, func.__name__)
+        except AttributeError:
+            return NotImplemented
+        if cupy_func is func:
+            # avoid NumPy func
+            return NotImplemented
+        for t in types:
+            for handled_type in _HANDLED_TYPES:
+                if issubclass(t, handled_type):
+                    break
+            else:
+                return NotImplemented
+        return cupy_func(*args, **kwargs)
+
+    # Conversion:
+
+    def __int__(self):
+        return int(self.get())
+
+    def __float__(self):
+        return float(self.get())
+
+    def __complex__(self):
+        return complex(self.get())
+
+    def __oct__(self):
+        return oct(self.get())
+
+    def __hex__(self):
+        return hex(self.get())
+
+    def __bytes__(self):
+        return bytes(self.get())
+
+    # String representations:
+
+    def __repr__(self):
+        return repr(self.get())
+
+    def __str__(self):
+        return str(self.get())
+
+    def __format__(self, format_spec):
+        return format(self.get(), format_spec)
+
+    # -------------------------------------------------------------------------
+    # Methods outside of the ndarray main documentation
+    # -------------------------------------------------------------------------
+    def dot(self, _ndarray_base b, _ndarray_base out=None):
+        """Returns the dot product with given array.
+
+        .. seealso::
+           :func:`cupy.dot` for full documentation,
+           :meth:`numpy.ndarray.dot`
+
+        """
+        return _linalg.dot(self, b, out)
+
+    # -------------------------------------------------------------------------
+    # Cupy specific attributes and methods
+    # -------------------------------------------------------------------------
+    @property
+    def device(self):
+        """CUDA device on which this array resides."""
+        return self.data.device
+
+    cpdef get(self, stream=None, order='C', out=None):
+        """Returns a copy of the array on host memory.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+                The default uses CUDA stream object of the current context.
+            order ({'C', 'F', 'A'}): The desired memory layout of the host
+                array. When ``order`` is 'A', it uses 'F' if the array is
+                fortran-contiguous and 'C' otherwise. The ``order`` will be
+                ignored if ``out`` is specified.
+            out (numpy.ndarray): Output array. In order to enable asynchronous
+                copy, the underlying memory should be a pinned memory.
+
+        Returns:
+            numpy.ndarray: Copy of the array on host memory.
+
+        """
+        if out is not None:
+            if not isinstance(out, numpy.ndarray):
+                raise TypeError('Only numpy.ndarray can be obtained from'
+                                'cupy.ndarray')
+            if self.dtype != out.dtype:
+                raise TypeError(
+                    '{} array cannot be obtained from {} array'.format(
+                        out.dtype, self.dtype))
+            if self.shape != out.shape:
+                raise ValueError(
+                    'Shape mismatch. Expected shape: {}, '
+                    'actual shape: {}'.format(self.shape, out.shape))
+            if not (out.flags.c_contiguous and self._c_contiguous or
+                    out.flags.f_contiguous and self._f_contiguous):
+                prev_device = runtime.getDevice()
+                try:
+                    runtime.setDevice(self.device.id)
+                    if out.flags.c_contiguous:
+                        a_gpu = _internal_ascontiguousarray(self)
+                    elif out.flags.f_contiguous:
+                        a_gpu = _internal_asfortranarray(self)
+                    else:
+                        raise RuntimeError(
+                            '`out` cannot be specified when copying to '
+                            'non-contiguous ndarray')
+                finally:
+                    runtime.setDevice(prev_device)
+            else:
+                a_gpu = self
+            a_cpu = out
+        else:
+            if self.size == 0:
+                return numpy.ndarray(self._shape, dtype=self.dtype)
+
+            order = order.upper()
+            if order == 'A':
+                if self._f_contiguous:
+                    order = 'F'
+                else:
+                    order = 'C'
+            if not (order == 'C' and self._c_contiguous or
+                    order == 'F' and self._f_contiguous):
+                prev_device = runtime.getDevice()
+                try:
+                    runtime.setDevice(self.device.id)
+                    if order == 'C':
+                        a_gpu = _internal_ascontiguousarray(self)
+                    elif order == 'F':
+                        a_gpu = _internal_asfortranarray(self)
+                    else:
+                        raise ValueError('unsupported order: {}'.format(order))
+                finally:
+                    runtime.setDevice(prev_device)
+            else:
+                a_gpu = self
+            a_cpu = numpy.empty(self._shape, dtype=self.dtype, order=order)
+
+        syncdetect._declare_synchronize()
+        ptr = a_cpu.ctypes.data
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(self.device.id)
+            if stream is not None:
+                a_gpu.data.copy_to_host_async(ptr, a_gpu.nbytes, stream)
+            else:
+                stream_ptr = stream_module.get_current_stream_ptr()
+                if stream_ptr == 0:
+                    a_gpu.data.copy_to_host(ptr, a_gpu.nbytes)
+                else:
+                    a_gpu.data.copy_to_host_async(ptr, a_gpu.nbytes)
+        finally:
+            runtime.setDevice(prev_device)
+        return a_cpu
+
+    cpdef set(self, arr, stream=None):
+        """Copies an array on the host memory to :class:`cupy.ndarray`.
+
+        Args:
+            arr (numpy.ndarray): The source array on the host memory.
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+                The default uses CUDA stream object of the current context.
+
+        """
+        if not isinstance(arr, numpy.ndarray):
+            raise TypeError('Only numpy.ndarray can be set to cupy.ndarray')
+        if self.dtype != arr.dtype:
+            raise TypeError('{} array cannot be set to {} array'.format(
+                arr.dtype, self.dtype))
+        if self.shape != arr.shape:
+            raise ValueError(
+                'Shape mismatch. Old shape: {}, new shape: {}'.format(
+                    self.shape, arr.shape))
+        if self._c_contiguous:
+            arr = numpy.ascontiguousarray(arr)
+        elif self._f_contiguous:
+            arr = numpy.asfortranarray(arr)
+        else:
+            raise RuntimeError('Cannot set to non-contiguous array')
+
+        ptr = arr.ctypes.data
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(self.device.id)
+            if stream is not None:
+                self.data.copy_from_host_async(ptr, self.nbytes, stream)
+            else:
+                stream_ptr = stream_module.get_current_stream_ptr()
+                if stream_ptr == 0:
+                    self.data.copy_from_host(ptr, self.nbytes)
+                else:
+                    self.data.copy_from_host_async(ptr, self.nbytes)
+        finally:
+            runtime.setDevice(prev_device)
+
+    cpdef _ndarray_base reduced_view(self, dtype=None):
+        """Returns a view of the array with minimum number of dimensions.
+
+        Args:
+            dtype: (Deprecated) Data type specifier.
+                If it is given, then the memory
+                sequence is reinterpreted as the new type.
+
+        Returns:
+            cupy.ndarray: A view of the array with reduced dimensions.
+
+        """
+        cdef shape_t shape
+        cdef strides_t strides
+        cdef Py_ssize_t ndim
+        cdef _ndarray_base view
+        if dtype is not None:
+            warnings.warn(
+                'calling reduced_view with dtype is deprecated',
+                DeprecationWarning)
+            return self.reduced_view().view(dtype)
+
+        ndim = self._shape.size()
+        if ndim <= 1:
+            return self
+        if self._c_contiguous:
+            view = self.view()
+            view._shape.assign(1, self.size)
+            view._strides.assign(1, self.dtype.itemsize)
+            view._update_f_contiguity()
+            return view
+
+        internal.get_reduced_dims(
+            self._shape, self._strides, self.dtype.itemsize, shape, strides)
+        if ndim == <Py_ssize_t>shape.size():
+            return self
+
+        # TODO(niboshi): Confirm update_x_contiguity flags
+        return self._view(type(self), shape, strides, False, True, self)
+
+    cpdef _update_c_contiguity(self):
+        if self.size == 0:
+            self._c_contiguous = True
+            return
+        self._c_contiguous = internal.get_c_contiguity(
+            self._shape, self._strides, self.dtype.itemsize)
+
+    cpdef _update_f_contiguity(self):
+        cdef Py_ssize_t i, count
+        cdef shape_t rev_shape
+        cdef strides_t rev_strides
+        if self.size == 0:
+            self._f_contiguous = True
+            return
+        if self._c_contiguous:
+            count = 0
+            for i in self._shape:
+                if i == 1:
+                    count += 1
+            self._f_contiguous = (<Py_ssize_t>self._shape.size()) - count <= 1
+            return
+        rev_shape.assign(self._shape.rbegin(), self._shape.rend())
+        rev_strides.assign(self._strides.rbegin(), self._strides.rend())
+        self._f_contiguous = internal.get_c_contiguity(
+            rev_shape, rev_strides, self.dtype.itemsize)
+
+    cpdef _update_contiguity(self):
+        self._update_c_contiguity()
+        self._update_f_contiguity()
+
+    cpdef _set_shape_and_strides(self, const shape_t& shape,
+                                 const strides_t& strides,
+                                 bint update_c_contiguity,
+                                 bint update_f_contiguity):
+        if shape.size() != strides.size():
+            raise ValueError('len(shape) != len(strides)')
+        if shape.size() > _carray.MAX_NDIM:
+            msg = 'maximum supported dimension for an ndarray is '
+            msg += f'{_carray.MAX_NDIM}, found {shape.size()}'
+            raise ValueError(msg)
+        self._shape = shape
+        self._strides = strides
+        self.size = internal.prod(shape)
+        if update_c_contiguity:
+            self._update_c_contiguity()
+        if update_f_contiguity:
+            self._update_f_contiguity()
+
+    cdef _ndarray_base _view(self, subtype, const shape_t& shape,
+                             const strides_t& strides,
+                             bint update_c_contiguity,
+                             bint update_f_contiguity, obj):
+        cdef _ndarray_base v
+        # Use `_no_init=True` to skip recomputation of contiguity. Now
+        # calling `__array_finalize__` is responsibility of this method.`
+        v = ndarray.__new__(subtype, _obj=obj, _no_init=True)
+        v.data = self.data
+        v.base = self.base if self.base is not None else self
+        v.dtype = self.dtype
+        v._c_contiguous = self._c_contiguous
+        v._f_contiguous = self._f_contiguous
+        v._index_32_bits = self._index_32_bits
+        v._set_shape_and_strides(
+            shape, strides, update_c_contiguity, update_f_contiguity)
+        if subtype is not ndarray:
+            v.__array_finalize__(self)
+        return v
+
+    cpdef _set_contiguous_strides(
+            self, Py_ssize_t itemsize, bint is_c_contiguous):
+        self.size = internal.get_contiguous_strides_inplace(
+            self._shape, self._strides, itemsize, is_c_contiguous, True)
+        if is_c_contiguous:
+            self._c_contiguous = True
+            self._update_f_contiguity()
+        else:
+            self._f_contiguous = True
+            self._update_c_contiguity()
+
+    cdef function.CPointer get_pointer(self):
+        return _CArray_from_ndarray(self)
+
+    cpdef object toDlpack(self):
+        """Zero-copy conversion to a DLPack tensor.
+
+        DLPack is a open in memory tensor structure proposed in this
+        repository: `dmlc/dlpack <https://github.com/dmlc/dlpack>`_.
+
+        This function returns a :class:`PyCapsule` object which contains a
+        pointer to a DLPack tensor converted from the own ndarray. This
+        function does not copy the own data to the output DLpack tensor
+        but it shares the pointer which is pointing to the same memory region
+        for the data.
+
+        Returns:
+            dltensor (:class:`PyCapsule`): Output DLPack tensor which is
+            encapsulated in a :class:`PyCapsule` object.
+
+        .. seealso::
+
+            :meth:`~cupy.fromDlpack` is a method for zero-copy conversion from
+            a DLPack tensor (which is encapsulated in a :class:`PyCapsule`
+            object) to a :class:`ndarray`
+
+        .. warning::
+
+            As of the DLPack v0.3 specification, it is (implicitly) assumed
+            that the user is responsible to ensure the Producer and the
+            Consumer are operating on the same stream. This requirement might
+            be relaxed/changed in a future DLPack version.
+
+        .. admonition:: Example
+
+            >>> import cupy
+            >>> array1 = cupy.array([0, 1, 2], dtype=cupy.float32)
+            >>> dltensor = array1.toDlpack()
+            >>> array2 = cupy.fromDlpack(dltensor)
+            >>> cupy.testing.assert_array_equal(array1, array2)
+
+        """
+        return dlpack.toDlpack(self)
+
+
+cdef inline _carray.CArray _CArray_from_ndarray(_ndarray_base arr):
+    # Creates CArray from ndarray.
+    # Note that this function cannot be defined in _carray.pxd because that
+    # would cause cyclic cimport dependencies.
+    cdef _carray.CArray carr = _carray.CArray.__new__(_carray.CArray)
+    carr.init(<void*>arr.data.ptr, arr.size, arr._shape, arr._strides)
+    return carr
+
+
+_HANDLED_TYPES = (ndarray, numpy.ndarray)
+
+
+# =============================================================================
+# compile_with_cache
+# =============================================================================
+# TODO(niboshi): Move it out of core.pyx
+
+cdef bint _is_hip = runtime._is_hip_environment
+cdef int _cuda_runtime_version = -1
+cdef str _cuda_path = ''  # '' for uninitialized, None for non-existing
+
+cdef list cupy_header_list = [
+    'cupy/complex.cuh',
+    'cupy/carray.cuh',
+    'cupy/atomics.cuh',
+    'cupy/math_constants.h',
+]
+if _is_hip:
+    cupy_header_list.append('cupy/hip_workaround.cuh')
+
+# expose to Python for unit testing
+_cupy_header_list = cupy_header_list
+
+cdef str _cupy_header = ''.join(
+    ['#include <%s>\n' % i for i in cupy_header_list])
+
+# This is indirect include header list.
+# These header files are subject to a hash key.
+cdef list _cupy_extra_header_list = [
+    'cupy/complex/complex.h',
+    'cupy/complex/math_private.h',
+    'cupy/complex/complex_inl.h',
+    'cupy/complex/arithmetic.h',
+    'cupy/complex/cproj.h',
+    'cupy/complex/cexp.h',
+    'cupy/complex/cexpf.h',
+    'cupy/complex/clog.h',
+    'cupy/complex/clogf.h',
+    'cupy/complex/cpow.h',
+    'cupy/complex/ccosh.h',
+    'cupy/complex/ccoshf.h',
+    'cupy/complex/csinh.h',
+    'cupy/complex/csinhf.h',
+    'cupy/complex/ctanh.h',
+    'cupy/complex/ctanhf.h',
+    'cupy/complex/csqrt.h',
+    'cupy/complex/csqrtf.h',
+    'cupy/complex/catrig.h',
+    'cupy/complex/catrigf.h',
+    'cupy/swap.cuh',
+    'cupy/tuple/type_traits.h',
+    'cupy/tuple/tuple.h',
+    'cupy/tuple.cuh',
+]
+
+cdef str _header_path_cache = None
+cdef str _header_source = None
+cdef dict _header_source_map = {}
+
+
+cpdef str _get_header_dir_path():
+    global _header_path_cache
+    if _header_path_cache is None:
+        # Cython cannot use __file__ in global scope
+        _header_path_cache = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), 'include'))
+    return _header_path_cache
+
+
+cpdef str _get_header_source():
+    global _header_source
+    global _header_source_map
+    cdef str header_path, base_path, file_path, header
+    cdef list source
+
+    if _header_source is None or not _header_source_map:
+        source = []
+        base_path = _get_header_dir_path()
+        for file_path in _cupy_extra_header_list + cupy_header_list:
+            header_path = os.path.join(base_path, file_path)
+            with open(header_path) as header_file:
+                header = header_file.read()
+            source.append(header)
+            _header_source_map[file_path.encode()] = header.encode()
+        _header_source = '\n'.join(source)
+    return _header_source
+
+
+cpdef dict _get_header_source_map():
+    global _header_source_map
+    if not _header_source_map:
+        _get_header_source()
+    return _header_source_map
+
+
+# added at the module level for precompiling the regex
+_cucomplex_include_tokens = ['', '#', 'include', '<', r'cuComplex\.h', '>']
+_cucomplex_include_pattern = re.compile(r'\s*'.join(_cucomplex_include_tokens))
+
+
+cdef inline str _translate_cucomplex_to_thrust(str source):
+    lines = []
+    for line in source.splitlines(keepends=True):
+        if _cucomplex_include_pattern.match(line):
+            lines += '#include <cupy/cuComplex_bridge.h>  '\
+                     '// translate_cucomplex\n'
+        else:
+            lines += line
+    return ''.join(lines)
+
+
+cpdef function.Module compile_with_cache(
+        str source, tuple options=(), arch=None, cachd_dir=None,
+        prepend_cupy_headers=True, backend='nvrtc', translate_cucomplex=False,
+        enable_cooperative_groups=False, name_expressions=None,
+        log_stream=None, bint jitify=False):
+    if translate_cucomplex:
+        source = _translate_cucomplex_to_thrust(source)
+        cupy_header_list.append('cupy/cuComplex_bridge.h')
+        prepend_cupy_headers = True
+
+    if prepend_cupy_headers:
+        source = _cupy_header + source
+    extra_source = _get_header_source()
+    options += ('-I%s' % _get_header_dir_path(),)
+
+    # The variable _cuda_runtime_version is declared in cupy/_core/core.pyx,
+    # but it might not have been set appropriately before coming here.
+    global _cuda_runtime_version
+    if _cuda_runtime_version < 0:
+        _cuda_runtime_version = runtime.runtimeGetVersion()
+
+    global _cuda_path
+    if _cuda_path == '':
+        if not _is_hip:
+            _cuda_path = cuda.get_cuda_path()
+        else:
+            _cuda_path = cuda.get_rocm_path()
+
+    if not _is_hip:
+        if 10020 <= _cuda_runtime_version < 10030:
+            bundled_include = 'cuda-10.2'
+        elif 11000 <= _cuda_runtime_version < 11010:
+            bundled_include = 'cuda-11.0'
+        elif 11010 <= _cuda_runtime_version < 11020:
+            bundled_include = 'cuda-11.1'
+        elif 11020 <= _cuda_runtime_version < 12000:
+            # CUDA Enhanced Compatibility
+            bundled_include = 'cuda-11'
+        elif 12000 <= _cuda_runtime_version < 13000:
+            # CUDA Enhanced Compatibility
+            bundled_include = 'cuda-12'
+        else:
+            # CUDA versions not yet supported.
+            bundled_include = None
+
+        if bundled_include is None and _cuda_path is None:
+            raise RuntimeError(
+                'Failed to auto-detect CUDA root directory. '
+                'Please specify `CUDA_PATH` environment variable if you '
+                'are using CUDA versions not yet supported by CuPy.')
+
+        if bundled_include is not None:
+            options += ('-I' + os.path.join(
+                _get_header_dir_path(), 'cupy', '_cuda', bundled_include),)
+    elif _is_hip:
+        if _cuda_path is None:
+            raise RuntimeError(
+                'Failed to auto-detect ROCm root directory. '
+                'Please specify `ROCM_HOME` environment variable.')
+
+    if _cuda_path is not None:
+        options += ('-I' + os.path.join(_cuda_path, 'include'),)
+
+    return cuda.compiler._compile_module_with_cache(
+        source, options, arch, cachd_dir, extra_source, backend,
+        enable_cooperative_groups=enable_cooperative_groups,
+        name_expressions=name_expressions, log_stream=log_stream,
+        jitify=jitify)
+
+
+# =============================================================================
+# Routines
+# =============================================================================
+
+cdef str _id = 'out0 = in0'
+
+cdef fill_kernel = ElementwiseKernel('T x', 'T y', 'y = x', 'cupy_fill')
+
+cdef str _divmod_float = '''
+    out0_type a = _floor_divide(in0, in1);
+    out0 = a;
+    out1 = in0 - a * in1'''
+
+
+divmod = create_ufunc(
+    'cupy_divmod',
+    ('bb->bb', 'BB->BB', 'hh->hh', 'HH->HH', 'ii->ii', 'II->II', 'll->ll',
+     'LL->LL', 'qq->qq', 'QQ->QQ',
+     ('ee->ee', _divmod_float),
+     ('ff->ff', _divmod_float),
+     ('dd->dd', _divmod_float)),
+    '''
+    if (in1 == 0) {
+        out0 = 0;
+        out1 = 0;
+    } else {
+        out0_type a = _floor_divide(in0, in1);
+        out0 = a;
+        out1 = in0 - a * in1;
+    }''')
+
+
+cdef _round_preamble = '''
+#ifdef __HIP_DEVICE_COMPILE__
+#define round_float llrintf
+#else
+#define round_float __float2ll_rn
+#endif
+
+template<typename T> __device__ T pow10(long long n){
+  T x = 1, a = 10;
+  while (n) {
+    if (n & 1) x *= a;
+    a *= a;
+    n >>= 1;
+  }
+  return x;
+};
+'''
+
+
+cdef _round_float = '''
+if (in1 == 0) {
+    out0 = rint(in0);
+} else {
+    double x;
+    x = pow10<double>(abs(in1));  // TODO(okuta): Move before loop
+    out0 = in1 < 0 ? rint(in0 / x) * x : rint(in0 * x) / x;
+}'''
+
+cdef _round_complex = '''
+if (in1 == 0) {
+    out0 = in0_type(rint(in0.real()), rint(in0.imag()));
+} else {
+    double x = pow10<double>(abs(in1));  // TODO(okuta): Move before loop
+    if (in1 < 0) {
+        out0 = in0_type(rint(in0.real() / x) * x,
+                        rint(in0.imag() / x) * x);
+    } else {
+        out0 = in0_type(rint(in0.real() * x) / x,
+                        rint(in0.imag() * x) / x);
+    }
+}'''
+
+
+# There is a known incompatibility with NumPy (as of 1.16.4) such as
+# `numpy.around(2**63, -1) == cupy.around(2**63, -1)` gives `False`.
+#
+# NumPy seems to round integral values via double.  As double has
+# only 53 bit precision, last few bits of (u)int64 value may be lost.
+# As a consequence, `numpy.around(2**63, -1)` does NOT round up the
+# last digit (9223372036854775808 instead of ...810).
+#
+# The following code fixes the problem, so `cupy.around(2**63, -1)`
+# gives `...810`, which (may correct but) is incompatible with NumPy.
+_round_ufunc = create_ufunc(
+    'cupy_round',
+    ('?q->e',
+     'bq->b', 'Bq->B', 'hq->h', 'Hq->H', 'iq->i', 'Iq->I', 'lq->l', 'Lq->L',
+     'qq->q', 'Qq->Q',
+     ('eq->e', _round_float),
+     ('fq->f', _round_float),
+     ('dq->d', _round_float),
+     ('Fq->F', _round_complex),
+     ('Dq->D', _round_complex)),
+    '''
+    if (in1 >= 0) {
+        out0 = in0;
+    } else {
+        // TODO(okuta): Move before loop
+        long long x = pow10<long long>(-in1 - 1);
+
+        // TODO(okuta): Check Numpy
+        // `cupy.around(-123456789, -4)` works as follows:
+        // (1) scale by `x` above: -123456.789
+        // (2) split at the last 2 digits: -123400 + (-5.6789 * 10)
+        // (3) round the latter by `rint()`: -123400 + (-6.0 * 10)
+        // (4) unscale by `x` above: -123460000
+        long long q = in0 / x / 100;
+        int r = in0 - q*x*100;
+        out0 = (q*100 + round_float(r/(x*10.0f))*10) * x;
+    }''', preamble=_round_preamble)
+
+
+# -----------------------------------------------------------------------------
+# Array creation routines
+# -----------------------------------------------------------------------------
+
+cpdef _ndarray_base array(obj, dtype=None, bint copy=True, order='K',
+                          bint subok=False, Py_ssize_t ndmin=0):
+    # TODO(beam2d): Support subok options
+    if subok:
+        raise NotImplementedError
+    if order is None:
+        order = 'K'
+
+    if isinstance(obj, ndarray):
+        return _array_from_cupy_ndarray(obj, dtype, copy, order, ndmin)
+
+    if hasattr(obj, '__cuda_array_interface__'):
+        return _array_from_cuda_array_interface(
+            obj, dtype, copy, order, subok, ndmin)
+    if hasattr(obj, '__cupy_get_ndarray__'):
+        return _array_from_cupy_ndarray(
+            obj.__cupy_get_ndarray__(), dtype, copy, order, ndmin)
+
+    concat_shape, concat_type, concat_dtype = (
+        _array_info_from_nested_sequence(obj))
+    if concat_shape is not None:
+        return _array_from_nested_sequence(
+            obj, dtype, order, ndmin, concat_shape, concat_type, concat_dtype)
+
+    return _array_default(obj, dtype, order, ndmin)
+
+
+cdef _ndarray_base _array_from_cupy_ndarray(
+        obj, dtype, bint copy, order, Py_ssize_t ndmin):
+    cdef Py_ssize_t ndim
+    cdef _ndarray_base a, src
+
+    src = obj
+
+    if dtype is None:
+        dtype = src.dtype
+
+    if src.data.device_id == device.get_device_id():
+        a = src.astype(dtype, order=order, copy=copy)
+    else:
+        a = src.copy(order=order).astype(dtype, copy=False)
+
+    ndim = a._shape.size()
+    if ndmin > ndim:
+        if a is obj:
+            # When `copy` is False, `a` is same as `obj`.
+            a = a.view()
+        a.shape = (1,) * (ndmin - ndim) + a.shape
+
+    return a
+
+
+cdef _ndarray_base _array_from_cuda_array_interface(
+        obj, dtype, bint copy, order, bint subok, Py_ssize_t ndmin):
+    return array(
+        _convert_object_with_cuda_array_interface(obj),
+        dtype, copy, order, subok, ndmin)
+
+
+cdef _ndarray_base _array_from_nested_sequence(
+        obj, dtype, order, Py_ssize_t ndmin, concat_shape, concat_type,
+        concat_dtype):
+    cdef Py_ssize_t ndim
+
+    # resulting array is C order unless 'F' is explicitly specified
+    # (i.e., it ignores order of element arrays in the sequence)
+    order = (
+        'F'
+        if order is not None and len(order) >= 1 and order[0] in 'Ff'
+        else 'C')
+
+    ndim = len(concat_shape)
+    if ndmin > ndim:
+        concat_shape = (1,) * (ndmin - ndim) + concat_shape
+
+    if dtype is None:
+        dtype = concat_dtype.newbyteorder('<')
+
+    if concat_type is numpy.ndarray:
+        return _array_from_nested_numpy_sequence(
+            obj, concat_dtype, dtype, concat_shape, order, ndmin)
+    elif concat_type is ndarray:  # TODO(takagi) Consider subclases
+        return _array_from_nested_cupy_sequence(
+            obj, dtype, concat_shape, order)
+    else:
+        assert False
+
+
+cdef _ndarray_base _array_from_nested_numpy_sequence(
+        arrays, src_dtype, dst_dtype, const shape_t& shape, order,
+        Py_ssize_t ndmin):
+    a_dtype = get_dtype(dst_dtype)  # convert to numpy.dtype
+    if a_dtype.char not in '?bhilqBHILQefdFD':
+        raise ValueError('Unsupported dtype %s' % a_dtype)
+    cdef _ndarray_base a  # allocate it after pinned memory is secured
+    cdef size_t itemcount = internal.prod(shape)
+    cdef size_t nbytes = itemcount * a_dtype.itemsize
+
+    stream = stream_module.get_current_stream()
+    # Note: even if arrays are already backed by pinned memory, we still need
+    # to allocate an extra buffer and copy from it to avoid potential data
+    # race, see the discussion here:
+    # https://github.com/cupy/cupy/pull/5155#discussion_r621808782
+    cdef pinned_memory.PinnedMemoryPointer mem = (
+        _alloc_async_transfer_buffer(nbytes))
+    if mem is not None:
+        # write concatenated arrays to the pinned memory directly
+        src_cpu = (
+            numpy.frombuffer(mem, a_dtype, itemcount)
+            .reshape(shape, order=order))
+        _concatenate_numpy_array(
+            [numpy.expand_dims(e, 0) for e in arrays],
+            0,
+            get_dtype(src_dtype),
+            a_dtype,
+            src_cpu)
+        a = ndarray(shape, dtype=a_dtype, order=order)
+        a.data.copy_from_host_async(mem.ptr, nbytes)
+        pinned_memory._add_to_watch_list(stream.record(), mem)
+    else:
+        # fallback to numpy array and send it to GPU
+        # Note: a_cpu.ndim is always >= 1
+        a_cpu = numpy.array(arrays, dtype=a_dtype, copy=False, order=order,
+                            ndmin=ndmin)
+        a = ndarray(shape, dtype=a_dtype, order=order)
+        a.data.copy_from_host(a_cpu.ctypes.data, nbytes)
+
+    return a
+
+
+cdef _ndarray_base _array_from_nested_cupy_sequence(obj, dtype, shape, order):
+    lst = _flatten_list(obj)
+
+    # convert each scalar (0-dim) ndarray to 1-dim
+    lst = [cupy.expand_dims(x, 0) if x.ndim == 0 else x for x in lst]
+
+    a = _manipulation.concatenate_method(lst, 0)
+    a = a.reshape(shape)
+    a = a.astype(dtype, order=order, copy=False)
+    return a
+
+
+cdef _ndarray_base _array_default(obj, dtype, order, Py_ssize_t ndmin):
+    if order is not None and len(order) >= 1 and order[0] in 'KAka':
+        if isinstance(obj, numpy.ndarray) and obj.flags.fnc:
+            order = 'F'
+        else:
+            order = 'C'
+    a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
+                        ndmin=ndmin)
+    if a_cpu.dtype.char not in '?bhilqBHILQefdFD':
+        raise ValueError('Unsupported dtype %s' % a_cpu.dtype)
+    a_cpu = a_cpu.astype(a_cpu.dtype.newbyteorder('<'), copy=False)
+    a_dtype = a_cpu.dtype
+    cdef shape_t a_shape = a_cpu.shape
+    cdef _ndarray_base a = ndarray(a_shape, dtype=a_dtype, order=order)
+    if a_cpu.ndim == 0:
+        a.fill(a_cpu)
+        return a
+    cdef Py_ssize_t nbytes = a.nbytes
+
+    stream = stream_module.get_current_stream()
+    # Note: even if obj is already backed by pinned memory, we still need to
+    # allocate an extra buffer and copy from it to avoid potential data race,
+    # see the discussion here:
+    # https://github.com/cupy/cupy/pull/5155#discussion_r621808782
+    cdef pinned_memory.PinnedMemoryPointer mem = (
+        _alloc_async_transfer_buffer(nbytes))
+    if mem is not None:
+        src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
+        src_cpu[:] = a_cpu.ravel(order)
+        a.data.copy_from_host_async(mem.ptr, nbytes)
+        pinned_memory._add_to_watch_list(stream.record(), mem)
+    else:
+        a.data.copy_from_host(a_cpu.ctypes.data, nbytes)
+
+    return a
+
+
+cdef tuple _array_info_from_nested_sequence(obj):
+    # Returns a tuple containing information if we can simply concatenate the
+    # input to make a CuPy array (i.e., a (nested) sequence that only contains
+    # NumPy/CuPy arrays with the same shape and dtype). `(None, None, None)`
+    # means we do not concatenate the input.
+    # 1. A concatenated shape
+    # 2. The type of the arrays to concatenate (numpy.ndarray or cupy.ndarray)
+    # 3. The dtype of the arrays to concatenate
+    if isinstance(obj, (list, tuple)):
+        return _compute_concat_info_impl(obj)
+    else:
+        return None, None, None
+
+
+cdef tuple _compute_concat_info_impl(obj):
+    cdef Py_ssize_t dim
+
+    if isinstance(obj, (numpy.ndarray, ndarray)):
+        return obj.shape, type(obj), obj.dtype
+
+    if hasattr(obj, '__cupy_get_ndarray__'):
+        return obj.shape, ndarray, obj.dtype
+
+    if isinstance(obj, (list, tuple)):
+        dim = len(obj)
+        if dim == 0:
+            return None, None, None
+
+        concat_shape, concat_type, concat_dtype = (
+            _compute_concat_info_impl(obj[0]))
+        if concat_shape is None:
+            return None, None, None
+
+        for elem in obj[1:]:
+            concat_shape1, concat_type1, concat_dtype1 = (
+                _compute_concat_info_impl(elem))
+            if concat_shape1 is None:
+                return None, None, None
+
+            if concat_shape != concat_shape1:
+                return None, None, None
+            if concat_type is not concat_type1:
+                return None, None, None
+            if concat_dtype != concat_dtype1:
+                concat_dtype = numpy.promote_types(concat_dtype, concat_dtype1)
+
+        return (dim,) + concat_shape, concat_type, concat_dtype
+
+    return None, None, None
+
+
+cdef list _flatten_list(object obj):
+    ret = []
+    if isinstance(obj, (list, tuple)):
+        for elem in obj:
+            ret += _flatten_list(elem)
+        return ret
+    return [obj]
+
+
+cdef bint _numpy_concatenate_has_out_argument = (
+    numpy.lib.NumpyVersion(numpy.__version__) >= '1.14.0')
+
+
+cdef inline _concatenate_numpy_array(arrays, axis, src_dtype, dst_dtype, out):
+    # type(*_dtype) must be numpy.dtype
+
+    if (_numpy_concatenate_has_out_argument
+            and src_dtype.kind == dst_dtype.kind):
+        # concatenate only accepts same_kind casting
+        numpy.concatenate(arrays, axis, out)
+    else:
+        out[:] = numpy.concatenate(arrays, axis)
+
+
+cdef inline _alloc_async_transfer_buffer(Py_ssize_t nbytes):
+    try:
+        return pinned_memory.alloc_pinned_memory(nbytes)
+    except CUDARuntimeError as e:
+        if e.status != runtime.errorMemoryAllocation:
+            raise
+        warnings.warn(
+            'Using synchronous transfer as pinned memory ({} bytes) '
+            'could not be allocated. '
+            'This generally occurs because of insufficient host memory. '
+            'The original error was: {}'.format(nbytes, e),
+            _util.PerformanceWarning)
+
+    return None
+
+
+cpdef _ndarray_base _internal_ascontiguousarray(_ndarray_base a):
+    if a._c_contiguous:
+        return a
+    newarray = _ndarray_init(ndarray, a._shape, a.dtype, None)
+    elementwise_copy(a, newarray)
+    return newarray
+
+
+cpdef _ndarray_base _internal_asfortranarray(_ndarray_base a):
+    cdef _ndarray_base newarray
+    cdef int m, n
+    cdef intptr_t handle
+
+    if a._f_contiguous:
+        return a
+
+    newarray = ndarray(a.shape, a.dtype, order='F')
+    if (a._c_contiguous and a._shape.size() == 2 and
+            (a.dtype == numpy.float32 or a.dtype == numpy.float64)):
+        m, n = a.shape
+        handle = device.get_cublas_handle()
+        one = numpy.array(1, dtype=a.dtype)
+        zero = numpy.array(0, dtype=a.dtype)
+        if a.dtype == numpy.float32:
+            cublas.sgeam(
+                handle,
+                1,  # transpose a
+                1,  # transpose newarray
+                m, n, one.ctypes.data, a.data.ptr, n,
+                zero.ctypes.data, a.data.ptr, n, newarray.data.ptr, m)
+        elif a.dtype == numpy.float64:
+            cublas.dgeam(
+                handle,
+                1,  # transpose a
+                1,  # transpose newarray
+                m, n, one.ctypes.data, a.data.ptr, n,
+                zero.ctypes.data, a.data.ptr, n, newarray.data.ptr, m)
+    else:
+        elementwise_copy(a, newarray)
+    return newarray
+
+
+cpdef _ndarray_base ascontiguousarray(_ndarray_base a, dtype=None):
+    cdef bint same_dtype = False
+    zero_dim = a._shape.size() == 0
+    if dtype is None:
+        same_dtype = True
+        dtype = a.dtype
+    else:
+        dtype = get_dtype(dtype)
+        same_dtype = dtype == a.dtype
+
+    if same_dtype and a._c_contiguous:
+        if zero_dim:
+            return _manipulation._ndarray_ravel(a, 'C')
+        return a
+
+    shape = (1,) if zero_dim else a.shape
+    newarray = ndarray(shape, dtype)
+    elementwise_copy(a, newarray)
+    return newarray
+
+
+cpdef _ndarray_base asfortranarray(_ndarray_base a, dtype=None):
+    cdef _ndarray_base newarray
+    cdef bint same_dtype = False
+    zero_dim = a._shape.size() == 0
+
+    if dtype is None:
+        dtype = a.dtype
+        same_dtype = True
+    else:
+        dtype = get_dtype(dtype)
+        same_dtype = dtype == a.dtype
+
+    if same_dtype and a._f_contiguous:
+        if zero_dim:
+            return _manipulation._ndarray_ravel(a, 'F')
+        return a
+
+    if same_dtype and not zero_dim:
+        return _internal_asfortranarray(a)
+
+    newarray = ndarray((1,) if zero_dim else a.shape, dtype, order='F')
+    elementwise_copy(a, newarray)
+    return newarray
+
+
+cpdef _ndarray_base _convert_object_with_cuda_array_interface(a):
+    if runtime._is_hip_environment:
+        raise RuntimeError(
+            'HIP/ROCm does not support cuda array interface')
+
+    cdef Py_ssize_t sh, st
+    cdef dict desc = a.__cuda_array_interface__
+    cdef tuple shape = desc['shape']
+    cdef int dev_id = -1
+    cdef size_t nbytes
+
+    ptr = desc['data'][0]
+    dtype = numpy.dtype(desc['typestr'])
+    if dtype.byteorder == '>':
+        raise ValueError('CuPy does not support the big-endian byte-order')
+    mask = desc.get('mask')
+    if mask is not None:
+        raise ValueError('CuPy currently does not support masked arrays.')
+    strides = desc.get('strides')
+    if strides is not None:
+        nbytes = 0
+        for sh, st in zip(shape, strides):
+            nbytes = max(nbytes, abs(sh * st))
+    else:
+        nbytes = internal.prod_sequence(shape) * dtype.itemsize
+    # the v2 protocol sets ptr=0 for 0-size arrays, so we can't look up
+    # the pointer attributes and must use the current device
+    if nbytes == 0:
+        dev_id = device.get_device_id()
+    mem = memory_module.UnownedMemory(ptr, nbytes, a, dev_id)
+    memptr = memory.MemoryPointer(mem, 0)
+    # the v3 protocol requires an immediate synchronization, unless
+    # 1. the stream is not set (ex: from v0 ~ v2) or is None
+    # 2. users explicitly overwrite this requirement
+    stream_ptr = desc.get('stream')
+    if stream_ptr is not None:
+        if _util.CUDA_ARRAY_INTERFACE_SYNC:
+            runtime.streamSynchronize(stream_ptr)
+    return ndarray(shape, dtype, memptr, strides)
+
+
+cdef _ndarray_base _ndarray_init(subtype, const shape_t& shape, dtype, obj):
+    # Use `_no_init=True` for fast init. Now calling `__array_finalize__` is
+    # responsibility of this function.
+    cdef _ndarray_base ret = ndarray.__new__(subtype, _obj=obj, _no_init=True)
+    ret._init_fast(shape, dtype, True)
+    if subtype is not ndarray:
+        ret.__array_finalize__(obj)
+    return ret
+
+
+cdef _ndarray_base _create_ndarray_from_shape_strides(
+        subtype, const shape_t& shape, const strides_t& strides, dtype, obj):
+    cdef int ndim = shape.size()
+    cdef int64_t begin = 0, end = dtype.itemsize
+    cdef memory.MemoryPointer ptr
+    for i in range(ndim):
+        if strides[i] > 0:
+            end += strides[i] * (shape[i] - 1)
+        elif strides[i] < 0:
+            begin += strides[i] * (shape[i] - 1)
+    ptr = memory.alloc(end - begin) + begin
+    return ndarray.__new__(
+        subtype, shape, dtype, _obj=obj, memptr=ptr, strides=strides)
+
+
+cpdef min_scalar_type(a):
+    """
+    For scalar ``a``, returns the data type with the smallest size
+    and smallest scalar kind which can hold its value.  For non-scalar
+    array ``a``, returns the vector's dtype unmodified.
+
+    .. seealso:: :func:`numpy.min_scalar_type`
+    """
+    if isinstance(a, ndarray):
+        return a.dtype
+    _, concat_type, concat_dtype = _array_info_from_nested_sequence(a)
+    if concat_type is not None:
+        return concat_dtype
+    return numpy.min_scalar_type(a)
diff --git a/cupy/_core/dlpack.pxd b/cupy/_core/dlpack.pxd
new file mode 100644
index 0000000..d09cdc3
--- /dev/null
+++ b/cupy/_core/dlpack.pxd
@@ -0,0 +1,12 @@
+from cupy._core.core cimport _ndarray_base
+
+
+cdef extern from './include/cupy/dlpack/dlpack.h' nogil:
+    int device_CUDA 'kDLCUDA'
+    int managed_CUDA 'kDLCUDAManaged'
+    int device_ROCM 'kDLROCM'
+
+
+cpdef object toDlpack(_ndarray_base array) except +
+cpdef _ndarray_base fromDlpack(object dltensor) except +
+cpdef from_dlpack(array)
diff --git a/cupy/_core/dlpack.pyx b/cupy/_core/dlpack.pyx
new file mode 100644
index 0000000..7e4e3c6
--- /dev/null
+++ b/cupy/_core/dlpack.pyx
@@ -0,0 +1,412 @@
+cimport cpython  # NOQA
+
+from libc cimport stdlib
+from libc.stdint cimport uint8_t
+from libc.stdint cimport uint16_t
+from libc.stdint cimport int32_t
+from libc.stdint cimport int64_t
+from libc.stdint cimport uint64_t
+from libc.stdint cimport intptr_t
+from libcpp.vector cimport vector
+
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda cimport stream as stream_module
+from cupy._core.core cimport _ndarray_base
+from cupy.cuda cimport memory
+
+import warnings
+
+import cupy
+import cupy._core.core as core
+
+
+cdef extern from './include/cupy/dlpack/dlpack.h' nogil:
+    cdef int DLPACK_VERSION
+
+    cdef enum DLDeviceType:
+        kDLCPU
+        kDLCUDA
+        kDLCUDAHost
+        kDLOpenCL
+        kDLVulkan
+        kDLMetal
+        kDLVPI
+        kDLROCM
+        kDLROCMHost
+        kDLExtDev
+        kDLCUDAManaged
+        kDLOneAPI
+        kDLWebGPU
+        kDLHexagon
+
+    ctypedef struct DLDevice:
+        DLDeviceType device_type
+        int32_t device_id
+
+    cdef enum DLDataTypeCode:
+        kDLInt
+        kDLUInt
+        kDLFloat
+        kDLBfloat
+        kDLComplex
+        kDLBool
+
+    ctypedef struct DLDataType:
+        uint8_t code
+        uint8_t bits
+        uint16_t lanes
+
+    ctypedef struct DLTensor:
+        void* data
+        DLDevice device
+        int32_t ndim
+        DLDataType dtype
+        int64_t* shape
+        int64_t* strides
+        uint64_t byte_offset
+
+    ctypedef struct DLManagedTensor:
+        DLTensor dl_tensor
+        void* manager_ctx
+        void (*deleter)(DLManagedTensor*)  # noqa: E211
+
+
+def get_build_version():
+    return str(DLPACK_VERSION)
+
+
+cdef void pycapsule_deleter(object dltensor):
+    cdef DLManagedTensor* dlm_tensor
+    # Do not invoke the deleter on a used capsule
+    if cpython.PyCapsule_IsValid(dltensor, 'dltensor'):
+        dlm_tensor = <DLManagedTensor*>cpython.PyCapsule_GetPointer(
+            dltensor, 'dltensor')
+        dlm_tensor.deleter(dlm_tensor)
+
+
+cdef void deleter(DLManagedTensor* tensor) with gil:
+    if tensor.manager_ctx is NULL:
+        return
+    stdlib.free(tensor.dl_tensor.shape)
+    cpython.Py_DECREF(<_ndarray_base>tensor.manager_ctx)
+    tensor.manager_ctx = NULL
+    stdlib.free(tensor)
+
+
+# The name of this function is following the framework integration guide of
+# TensorComprehensions.
+cpdef object toDlpack(_ndarray_base array) except +:
+    cdef DLManagedTensor* dlm_tensor = \
+        <DLManagedTensor*>stdlib.malloc(sizeof(DLManagedTensor))
+
+    cdef size_t ndim = array._shape.size()
+    cdef DLTensor* dl_tensor = &dlm_tensor.dl_tensor
+    cdef intptr_t data_ptr = array.data.ptr
+    dl_tensor.data = <void*>data_ptr
+    dl_tensor.ndim = ndim
+
+    cdef int64_t* shape_strides = \
+        <int64_t*>stdlib.malloc(ndim * sizeof(int64_t) * 2)
+    for n in range(ndim):
+        shape_strides[n] = array._shape[n]
+    dl_tensor.shape = shape_strides
+    for n in range(ndim):
+        shape_strides[n + ndim] = array._strides[n] // array.dtype.itemsize
+
+    dl_tensor.strides = shape_strides + ndim
+    dl_tensor.byte_offset = 0
+
+    cdef DLDevice* device = &dl_tensor.device
+    cdef bint is_managed
+    cdef int dev_id = array.data.device_id
+    if not runtime._is_hip_environment:
+        attrs = runtime.pointerGetAttributes(data_ptr)
+        is_managed = (attrs.type == runtime.memoryTypeManaged)
+        if is_managed:
+            device.device_type = kDLCUDAManaged
+            dev_id = 0  # make it accessible on CPU too
+        else:
+            device.device_type = kDLCUDA
+    else:
+        device.device_type = kDLROCM
+    device.device_id = dev_id
+
+    cdef DLDataType* dtype = &dl_tensor.dtype
+    if array.dtype.kind == 'u':
+        dtype.code = <uint8_t>kDLUInt
+    elif array.dtype.kind == 'i':
+        dtype.code = <uint8_t>kDLInt
+    elif array.dtype.kind == 'f':
+        dtype.code = <uint8_t>kDLFloat
+    elif array.dtype.kind == 'c':
+        dtype.code = <uint8_t>kDLComplex
+    elif array.dtype.kind == 'b':
+        dtype.code = <uint8_t>kDLBool
+    else:
+        raise ValueError('Unknown dtype')
+    dtype.lanes = <uint16_t>1
+    dtype.bits = <uint8_t>(array.dtype.itemsize * 8)
+
+    dlm_tensor.manager_ctx = <void*>array
+    cpython.Py_INCREF(array)
+    dlm_tensor.deleter = deleter
+
+    return cpython.PyCapsule_New(dlm_tensor, 'dltensor', pycapsule_deleter)
+
+
+# TODO(leofang): Support kDLCUDAPinned and kDLROCMPinned
+cdef class DLPackMemory(memory.BaseMemory):
+
+    """Memory object for a dlpack tensor.
+
+    This does not allocate any memory.
+
+    """
+
+    cdef DLManagedTensor* dlm_tensor
+    cdef object dltensor
+
+    def __init__(self, object dltensor):
+        cdef DLManagedTensor* dlm_tensor
+
+        # sanity checks
+        if not cpython.PyCapsule_IsValid(dltensor, 'dltensor'):
+            raise ValueError('A DLPack tensor object cannot be consumed '
+                             'multiple times')
+        dlm_tensor = <DLManagedTensor*>cpython.PyCapsule_GetPointer(
+            dltensor, 'dltensor')
+        if runtime._is_hip_environment:
+            if dlm_tensor.dl_tensor.device.device_type != kDLROCM:
+                raise RuntimeError('CuPy is built against ROCm/HIP, different '
+                                   'from the backend that backs the incoming '
+                                   'DLPack tensor')
+        else:
+            if dlm_tensor.dl_tensor.device.device_type not in (
+                    kDLCUDA, kDLCUDAManaged):
+                raise RuntimeError('CuPy is built against CUDA, different '
+                                   'from the backend that backs the incoming '
+                                   'DLPack tensor')
+
+        self.dltensor = dltensor
+        self.dlm_tensor = dlm_tensor
+        self.ptr = <intptr_t>dlm_tensor.dl_tensor.data
+        if dlm_tensor.dl_tensor.device.device_type == kDLCUDAManaged:
+            # look up the actual physical device as the id from
+            # dl_tensor could be 0
+            attrs = runtime.pointerGetAttributes(self.ptr)
+            self.device_id = attrs.device
+        else:
+            self.device_id = dlm_tensor.dl_tensor.device.device_id
+
+        cdef int n = 0, s = 0
+        cdef int ndim = dlm_tensor.dl_tensor.ndim
+        cdef int64_t* shape = dlm_tensor.dl_tensor.shape
+        for s in shape[:ndim]:
+            n += s
+        self.size = dlm_tensor.dl_tensor.dtype.bits * n // 8
+
+    def __dealloc__(self):
+        cdef DLManagedTensor* dlm_tensor = self.dlm_tensor
+        # dlm_tensor could be uninitialized if an error is raised in __init__
+        if dlm_tensor != NULL:
+            dlm_tensor.deleter(dlm_tensor)
+
+
+# The name of this function is following the framework integration guide of
+# TensorComprehensions.
+cpdef _ndarray_base fromDlpack(object dltensor) except +:
+    """Zero-copy conversion from a DLPack tensor to a :class:`~cupy.ndarray`.
+
+    DLPack is a open in memory tensor structure proposed in this repository:
+    `dmlc/dlpack <https://github.com/dmlc/dlpack>`_.
+
+    This function takes a :class:`PyCapsule` object which contains a pointer to
+    a DLPack tensor as input, and returns a :class:`~cupy.ndarray`. This
+    function does not copy the data in the DLPack tensor but both
+    DLPack tensor and :class:`~cupy.ndarray` have pointers which are pointing
+    to the same memory region for the data.
+
+    Args:
+        dltensor (:class:`PyCapsule`): Input DLPack tensor which is
+            encapsulated in a :class:`PyCapsule` object.
+
+    Returns:
+        array (:class:`~cupy.ndarray`): A CuPy ndarray.
+
+    .. warning::
+
+        This function is deprecated in favor of :func:`~cupy.from_dlpack` and
+        will be removed in a future version of CuPy.
+
+    .. warning::
+
+        As of the DLPack v0.5 specification, it is implicitly assumed that
+        the user is responsible to ensure the Producer and the Consumer are
+        operating on the same stream.
+
+    .. seealso::
+
+        :meth:`cupy.ndarray.toDlpack` is a method for zero-copy conversion
+        from a :class:`~cupy.ndarray` to a DLPack tensor (which is encapsulated
+        in a :class:`PyCapsule` object).
+
+    .. admonition:: Example
+
+        >>> import cupy
+        >>> array1 = cupy.array([0, 1, 2], dtype=cupy.float32)
+        >>> dltensor = array1.toDlpack()
+        >>> array2 = cupy.fromDlpack(dltensor)
+        >>> cupy.testing.assert_array_equal(array1, array2)
+
+    """
+    warnings.warn('This function is deprecated in favor of cupy.from_dlpack',
+                  DeprecationWarning)
+    return _dlpack_to_cupy_array(dltensor)
+
+
+cdef inline _ndarray_base _dlpack_to_cupy_array(dltensor) except +:
+    cdef DLPackMemory mem = DLPackMemory(dltensor)
+    cdef DLDataType dtype = mem.dlm_tensor.dl_tensor.dtype
+    cdef int bits = dtype.bits
+    if dtype.lanes != 1:
+        raise ValueError(f'vector dtypes (lanes={dtype.lanes}) is '
+                         'not supported')
+    if dtype.code == kDLUInt:
+        if bits == 8:
+            cp_dtype = cupy.uint8
+        elif bits == 16:
+            cp_dtype = cupy.uint16
+        elif bits == 32:
+            cp_dtype = cupy.uint32
+        elif bits == 64:
+            cp_dtype = cupy.uint64
+        else:
+            raise TypeError('uint{} is not supported.'.format(bits))
+    elif dtype.code == kDLInt:
+        if bits == 8:
+            cp_dtype = cupy.int8
+        elif bits == 16:
+            cp_dtype = cupy.int16
+        elif bits == 32:
+            cp_dtype = cupy.int32
+        elif bits == 64:
+            cp_dtype = cupy.int64
+        else:
+            raise TypeError('int{} is not supported.'.format(bits))
+    elif dtype.code == kDLFloat:
+        if bits == 16:
+            cp_dtype = cupy.float16
+        elif bits == 32:
+            cp_dtype = cupy.float32
+        elif bits == 64:
+            cp_dtype = cupy.float64
+        else:
+            raise TypeError('float{} is not supported.'.format(bits))
+    elif dtype.code == kDLComplex:
+        # TODO(leofang): support complex32
+        if bits == 64:
+            cp_dtype = cupy.complex64
+        elif bits == 128:
+            cp_dtype = cupy.complex128
+        else:
+            raise TypeError('complex{} is not supported.'.format(bits))
+    elif dtype.code == kDLBool:
+        if bits == 8:
+            cp_dtype = cupy.bool_
+        else:
+            raise TypeError(f'{bits}-bit bool is not supported')
+    elif dtype.code == kDLBfloat:
+        raise NotImplementedError('CuPy does not support bfloat16 yet')
+    else:
+        raise TypeError('Unsupported dtype. dtype code: {}'.format(dtype.code))
+
+    mem_ptr = memory.MemoryPointer(mem, mem.dlm_tensor.dl_tensor.byte_offset)
+    cdef int64_t ndim = mem.dlm_tensor.dl_tensor.ndim
+
+    cdef int64_t* shape = mem.dlm_tensor.dl_tensor.shape
+    cdef vector[Py_ssize_t] shape_vec
+    shape_vec.assign(shape, shape + ndim)
+
+    if mem.dlm_tensor.dl_tensor.strides is NULL:
+        # Make sure this capsule will never be used again.
+        cpython.PyCapsule_SetName(mem.dltensor, 'used_dltensor')
+        return core.ndarray(shape_vec, cp_dtype, mem_ptr, strides=None)
+    cdef int64_t* strides = mem.dlm_tensor.dl_tensor.strides
+    cdef vector[Py_ssize_t] strides_vec
+    for i in range(ndim):
+        strides_vec.push_back(strides[i] * (bits // 8))
+
+    # Make sure this capsule will never be used again.
+    cpython.PyCapsule_SetName(mem.dltensor, 'used_dltensor')
+    return core.ndarray(shape_vec, cp_dtype, mem_ptr, strides=strides_vec)
+
+
+cpdef from_dlpack(array):
+    """Zero-copy conversion between array objects compliant with the DLPack
+    data exchange protocol.
+
+    Args:
+        array (object): an array object that implements two methods:
+            ``__dlpack__()`` and ``__dlpack_device__()``.
+
+    Returns:
+        cupy.ndarray: a CuPy array that can be safely accessed on CuPy's
+        current stream.
+
+    .. note::
+        This function is different from CuPy's legacy :func:`~cupy.fromDlpack`
+        function. This function takes any object implementing the DLPack data
+        exchange protocol, as well as a raw :class:`PyCapsule` object that
+        contains the DLPack tensor as input (for backward compatibility),
+        whereas :func:`~cupy.fromDlpack` only accepts :class:`PyCapsule`
+        objects. If the input object is not compliant with the protocol, users
+        are responsible to ensure data safety.
+
+    .. seealso::
+        :func:`numpy.from_dlpack`,
+        `Python Specification for DLPack`_,
+        `Data interchange mechanisms`_
+
+    .. _Python Specification for DLPack:
+        https://dmlc.github.io/dlpack/latest/python_spec.html
+    .. _Data interchange mechanisms:
+        https://data-apis.org/array-api/latest/design_topics/data_interchange.html
+    """
+    if not hasattr(array, '__dlpack_device__'):
+        # backward compatibility: accept passing in a pycapsule
+        dltensor = array
+        return _dlpack_to_cupy_array(dltensor)
+    else:
+        dev_type, dev_id = array.__dlpack_device__()
+
+    # CuPy is the consumer, so we provide our current stream to the producer
+    if dev_type == <int>kDLCUDA or dev_type == <int>kDLCUDAManaged:
+        prev_device = cupy.cuda.runtime.getDevice()
+        try:
+            cupy.cuda.runtime.setDevice(dev_id)
+            assert not runtime._is_hip_environment
+            stream = stream_module.get_current_stream_ptr()
+            if stream == 0:
+                stream = stream_module.get_default_stream_ptr()
+            dltensor = array.__dlpack__(stream=stream)
+        finally:
+            cupy.cuda.runtime.setDevice(prev_device)
+    elif dev_type == <int>kDLROCM:
+        prev_device = cupy.cuda.runtime.getDevice()
+        try:
+            cupy.cuda.runtime.setDevice(dev_id)
+            assert runtime._is_hip_environment
+            stream = stream_module.get_current_stream_ptr()
+            dltensor = array.__dlpack__(stream=stream)
+        finally:
+            cupy.cuda.runtime.setDevice(prev_device)
+    elif dev_type == <int>kDLCPU:
+        raise TypeError(
+            'CPU arrays cannot be directly imported to CuPy. '
+            'Use `cupy.array(numpy.from_dlpack(input))` instead.')
+    else:
+        # TODO(leofang): support kDLCUDAPinned etc
+        dltensor = None
+        raise TypeError(f'Unsupported array type: {dev_type}')
+
+    return _dlpack_to_cupy_array(dltensor)
diff --git a/cupy/_core/flags.pyx b/cupy/_core/flags.pyx
new file mode 100644
index 0000000..8fc9702
--- /dev/null
+++ b/cupy/_core/flags.pyx
@@ -0,0 +1,34 @@
+# distutils: language = c++
+
+
+class Flags(object):
+
+    def __init__(self, c_contiguous, f_contiguous, owndata):
+        self.c_contiguous = c_contiguous
+        self.f_contiguous = f_contiguous
+        self.owndata = owndata
+
+    @property
+    def fnc(self):
+        return self.f_contiguous and not self.c_contiguous
+
+    @property
+    def forc(self):
+        return self.f_contiguous or self.c_contiguous
+
+    def __getitem__(self, name):
+        if name == 'C_CONTIGUOUS':
+            return self.c_contiguous
+        elif name == 'F_CONTIGUOUS':
+            return self.f_contiguous
+        elif name == 'OWNDATA':
+            return self.owndata
+        else:
+            raise KeyError('%s is not defined for cupy.ndarray.flags' % name)
+
+    def __repr__(self):
+        t = '  %s : %s'
+        ret = []
+        for name in 'C_CONTIGUOUS', 'F_CONTIGUOUS', 'OWNDATA':
+            ret.append(t % (name, self[name]))
+        return '\n'.join(ret)
diff --git a/cupy/_core/fusion.pyx b/cupy/_core/fusion.pyx
new file mode 100644
index 0000000..6d496e2
--- /dev/null
+++ b/cupy/_core/fusion.pyx
@@ -0,0 +1,1004 @@
+from cupy._core cimport _accelerator
+from cupy._core._accelerator cimport ACCELERATOR_CUB
+from cupy._core._scalar cimport get_typename, _get_cuda_scalar_repr
+
+import functools
+import string
+
+import numpy
+
+import cupy
+from cupy._core._dtype import get_dtype, _raise_if_invalid_cast
+from cupy._core import _kernel
+from cupy._core import _fusion_thread_local
+from cupy._core import _reduction
+from cupy._core import core
+from cupy._core import new_fusion
+
+
+_is_fusing = _fusion_thread_local.is_fusing  # NOQA
+_thread_local = _fusion_thread_local.thread_local
+
+cdef dict _kind_score = {
+    'b': 0,
+    'u': 1,
+    'i': 1,
+    'f': 2,
+    'c': 2,
+}
+
+cdef list _dtype_list = [numpy.dtype(_) for _ in '?bhilqBHILQefdFD']
+
+cdef tuple _acceptable_types = (
+    core.ndarray, numpy.ndarray, numpy.generic,
+    int, float, complex, bool, type(None))
+
+
+class _Submodule(object):
+    """Ufunc or elementwise kernel with types.
+
+    Attributes:
+       name (str): The name of submodule
+       in_params (list of tuples of dtype and str):
+         The tuple of dtype and name of input parameters.
+       out_params (list of tuples of dtype and str):
+         The tuple of dtype and name of output parameters.
+       op (str): The operation code.
+       preamble (str): The preamble code.
+       dtypes (list of dtypes): The list of dtypes of the parameters.
+    """
+
+    def __init__(self, ufunc, in_params, out_params, op):
+        self.name = ufunc.name
+        self.in_params = in_params
+        self.out_params = out_params
+        self.op = op
+        self.preamble = ufunc._preamble
+        self.dtypes = [dtype for dtype, _ in self.in_params + self.out_params]
+
+    def __repr__(self):
+        return '<_Submodule {}>'.format(self.name)
+
+    def fcall(self, args):
+        return self.name + '(' + ', '.join(args) + ');\n'
+
+    def key(self):
+        return (self.name, tuple(self.dtypes))
+
+    def code(self):
+        params = ', '.join('{} &{}'.format(get_typename(t), s)
+                           for t, s in self.in_params + self.out_params)
+        typedef = ''.join('typedef {} {}_type;\n'.format(get_typename(t), s)
+                          for t, s in self.in_params + self.out_params)
+        module_code = string.Template('''
+        __device__ void ${name}(${parameters}) {
+          ${typedef}
+          ${operation};
+        }
+        ''').substitute(
+            name=self.name,
+            parameters=params,
+            operation=self.op,
+            typedef=typedef)
+        return module_code + '\n'
+
+
+class _FusionVarCUDA(object):
+
+    """Local variable in CUDA program.
+
+    Attributes:
+        index (int): The name of the variable.
+        dtype (dtype): The dtype of the variable.
+        const_value (any of primitive types): The constant value (or None)
+    """
+
+    def __init__(self, index, dtype, const_value=None):
+        self.index = index
+        self.dtype = dtype
+        self.const_value = const_value
+        self.mutable = False
+
+    def __repr__(self):
+        return 'v{}'.format(self.index)
+
+    def mutate(self):
+        self.mutable = True
+
+    def declaration(self):
+        c = self.const_value
+        val = c.item() if hasattr(c, 'dtype') else c
+        ctype = get_typename(self.dtype)
+
+        if self.const_value is None:
+            return '{} v{};\n'.format(ctype, self.index)
+
+        code = _get_cuda_scalar_repr(val, self.dtype)
+        return 'const {} v{} = {};\n'.format(ctype, self.index, code)
+
+    def declaration_in_param(self):
+        non_const = '_non_const ' if self.mutable else ''
+        return '{}{} v{}'.format(non_const, self.dtype, self.index)
+
+    def declaration_out_param(self):
+        return '{} v{}'.format(self.dtype, self.index)
+
+
+class _FusionOp(object):
+
+    """Function call with arguments in CUDA program.
+
+    Attributes:
+        index (int): The index of this operation.
+        submodule (submodule): The submodules called in this operation.
+        args (list of _FusionVarCUDA): The arguments.
+        types (list of dtype): The types of parameters.
+    """
+
+    def __init__(self, index, submodule, args):
+        self.index = index
+        self.submodule = submodule
+        self.args = args
+        self.dtypes = submodule.dtypes
+
+    def __repr__(self):
+        return '<_FusionOp #{}, {} types=[{}]>'.format(
+            self.index, self.submodule.name, ', '.join(map(str, self.dtypes)))
+
+    def declaration_args(self):
+        return ' '.join('{} v{}_{};'.format(get_typename(t), self.index, j)
+                        for j, t in enumerate(self.dtypes)) + '\n'
+
+    def code(self):
+        args_sub = ['v{}_{}'.format(self.index, i)
+                    for i in range(len(self.args))]
+        ctypes = [get_typename(t) for t in self.dtypes]
+        args_list = list(zip(self.args, args_sub, ctypes))
+        code = '// op  # {}\n'.format(self.index)
+        code += ''.join('{} = static_cast< {} >(v{});\n'.format(s, t, v.index)
+                        for v, s, t in args_list)
+        code += self.submodule.fcall(args_sub)
+        code += ''.join('v{} = static_cast< {} >({});\n'.format(
+            v.index, get_typename(v.dtype), s)
+            for v, s, _ in
+            args_list[len(self.submodule.in_params):])
+        return code
+
+
+class _FusionVarScalar(object):
+
+    """The values of variables in target function of fusion.
+
+    Args:
+        var (_FusionVarCUDA)
+        ndim (int)
+        is_postmap (bool)
+
+    Attributes:
+        dtype (dtype): The data type.
+    """
+
+    def __init__(self, var, ndim, is_postmap):
+        self._var = var
+        self.dtype = var.dtype
+        self.ndim = ndim
+        self._is_postmap = is_postmap
+        assert ndim == -1
+
+    def __repr__(self):
+        return '<_FusionVar {} scalar>'.format(self.dtype)
+
+    def __neg__(self):
+        return cupy.negative(self)
+
+    def __add__(self, other):
+        return cupy.add(self, other)
+
+    def __radd__(self, other):
+        return cupy.add(other, self)
+
+    def __sub__(self, other):
+        return cupy.subtract(self, other)
+
+    def __rsub__(self, other):
+        return cupy.subtract(other, self)
+
+    def __mul__(self, other):
+        return cupy.multiply(self, other)
+
+    def __rmul__(self, other):
+        return cupy.multiply(other, self)
+
+    def __div__(self, other):
+        return cupy.divide(self, other)
+
+    def __rdiv__(self, other):
+        return cupy.divide(other, self)
+
+    def __truediv__(self, other):
+        return cupy.true_divide(self, other)
+
+    def __rtruediv__(self, other):
+        return cupy.true_divide(other, self)
+
+    def __floordiv__(self, other):
+        return cupy.floor_divide(self, other)
+
+    def __rfloordiv__(self, other):
+        return cupy.floor_divide(other, self)
+
+    def __mod__(self, other):
+        return cupy.remainder(self, other)
+
+    def __rmod__(self, other):
+        return cupy.remainder(other, self)
+
+    def __pow__(x, y):
+        return cupy.power(x, y)
+
+    def __lshift__(self, other):
+        return cupy.left_shift(self, other)
+
+    def __rlshift__(self, other):
+        return cupy.left_shift(other, self)
+
+    def __rshift__(self, other):
+        return cupy.right_shift(self, other)
+
+    def __rrshift__(self, other):
+        return cupy.right_shift(other, self)
+
+    def __and__(self, other):
+        return cupy.bitwise_and(self, other)
+
+    def __rand__(self, other):
+        return cupy.bitwise_and(other, self)
+
+    def __or__(self, other):
+        return cupy.bitwise_or(self, other)
+
+    def __ror__(self, other):
+        return cupy.bitwise_or(other, self)
+
+    def __xor__(self, other):
+        return cupy.bitwise_xor(self, other)
+
+    def __rxor__(self, other):
+        return cupy.bitwise_xor(other, self)
+
+    def __invert__(self):
+        return cupy.invert(self)
+
+    def __lt__(self, other):
+        return cupy.less(self, other)
+
+    def __le__(self, other):
+        return cupy.less_equal(self, other)
+
+    def __eq__(self, other):
+        return cupy.equal(self, other)
+
+    def __ne__(self, other):
+        return cupy.not_equal(self, other)
+
+    def __gt__(self, other):
+        return cupy.greater(self, other)
+
+    def __ge__(self, other):
+        return cupy.greater_equal(self, other)
+
+    def __nonzero__(self):
+        raise Exception('Can\'t cast to bool')
+
+    def __bool__(self):
+        raise Exception('Can\'t cast to bool')
+
+    def __setitem__(self, slices, value):
+        if slices is Ellipsis or (isinstance(slices, slice) and
+                                  slices == slice(None)):
+            _call_ufunc(core.elementwise_copy, value, out=self)
+        else:
+            raise ValueError('The fusion supports `[...]` or `[:]`.')
+
+    def copy(self):
+        return cupy.copy(self)
+
+    def astype(self, dtype, order=None, casting=None, subok=None, copy=True):
+        dtype = get_dtype(dtype)
+        if order is not None:
+            raise TypeError('order is not supported yet')
+        if casting is not None:
+            raise TypeError('casting is not supported yet')
+        if subok is not None:
+            raise TypeError('subok is not supported yet')
+        if not copy and self.dtype == dtype:
+            return self
+        return _dtype_to_astype(dtype)(self)
+
+
+class _FusionVarArray(_FusionVarScalar):
+
+    def __init__(self, var, ndim, is_postmap):
+        self._var = var
+        self.dtype = var.dtype
+        self.ndim = ndim
+        self._is_postmap = is_postmap
+        assert ndim >= 0
+
+    def __repr__(self):
+        return '<_FusionVar {} {}-dim array>'.format(self.dtype, self.ndim)
+
+    def __iadd__(self, other):
+        return cupy.add(self, other, self)
+
+    def __isub__(self, other):
+        return cupy.subtract(self, other, self)
+
+    def __imul__(self, other):
+        return cupy.multiply(self, other, self)
+
+    def __idiv__(self, other):
+        return cupy.divide(self, other, self)
+
+    def __itruediv__(self, other):
+        return cupy.true_divide(self, other, self)
+
+    def __ifloordiv__(self, other):
+        return cupy.floor_divide(self, other, self)
+
+    def __imod__(self, other):
+        return cupy.remainder(self, other, self)
+
+    def __ipow__(self, other):
+        return cupy.power(self, other, self)
+
+    def __ilshift__(self, other):
+        return cupy.left_shift(self, other, self)
+
+    def __irshift__(self, other):
+        return cupy.right_shift(self, other, self)
+
+    def __iand__(self, other):
+        return cupy.bitwise_and(self, other, self)
+
+    def __ior__(self, other):
+        return cupy.bitwise_or(self, other, self)
+
+    def __ixor__(self, other):
+        return cupy.bitwise_xor(self, other, self)
+
+
+class _FusionHistory(object):
+
+    """History of operation exectuted in the target function of fusion.
+
+    Attributes:
+        preamble_set (set of str): The preambles of submodules.
+        submodules (dict from str to submodule): The submodules.
+        count (int): The number of variables in the fused function.
+
+        op_list (list of _FusionOp): The map operations.
+        param_list (list of _FusionVarCUDA): The parameters
+        local_list (list of _FusionVarCUDA): The local variables.
+
+    Only when fusing the reduction, the following attributes are updated.
+
+        reduce_op (tuple): One of the element of reduction.***._raws._ops.
+        reduce_identity (any type): The identity value of the reduction.
+        reduce_kwargs (dict or None): kwargs of the reduction.
+
+        premap_ret (_FusionVarCUDA or None): The target of reduction
+        postmap_param (_FusionVarCUDA or None): The result of reduction
+        postmap_op_list (list of FuisonOp): The post-map operations.
+        postmap_local_list (list of _FusionVarCUDA): The local variables which
+    appears in the post-map operations
+    """
+
+    def __init__(self):
+        self.preamble_set = set()
+        self.submodules = dict()
+        self.count = 0
+
+        self.op_list = []
+        self.param_list = []
+        self.local_list = []
+
+        self.reduce_op = None
+        self.reduce_identity = None
+        self.reduce_kwargs = None
+
+        self.postmap_op_list = []
+        self.premap_ret = None
+        self.postmap_param = None
+        self.postmap_local_list = []
+
+    def __repr__(self):
+        return '<_FusionMem, op_list={}, param_list={}, local_list={}>'.format(
+            self.op_list, self.param_list, self.local_list)
+
+    def _has_reduction(self):
+        return self.reduce_op is not None
+
+    def _fresh_index(self):
+        res = self.count
+        self.count += 1
+        return res
+
+    def _fresh_premap_param(self, *args, **kwargs):
+        index = self._fresh_index()
+        var = _FusionVarCUDA(index, *args, **kwargs)
+        self.param_list.append(var)
+        return var
+
+    def _fresh_postmap_param(self, *args, **kwargs):
+        assert self.postmap_param is None
+        index = self._fresh_index()
+        var = _FusionVarCUDA(index, *args, **kwargs)
+        self.postmap_param = var
+        return var
+
+    def _fresh_premap_local(self, *args, **kwargs):
+        index = self._fresh_index()
+        var = _FusionVarCUDA(index, *args, **kwargs)
+        self.local_list.append(var)
+        return var
+
+    def _fresh_postmap_local(self, *args, **kwargs):
+        index = self._fresh_index()
+        var = _FusionVarCUDA(index, *args, **kwargs)
+        self.postmap_local_list.append(var)
+        return var
+
+    def _fresh_local(self, *args, **kwargs):
+        if self._has_reduction():
+            return self._fresh_postmap_local(*args, **kwargs)
+        else:
+            return self._fresh_premap_local(*args, **kwargs)
+
+    def _add_premap_op(self, *args, **kwargs):
+        op = _FusionOp(len(self.op_list), *args, **kwargs)
+        subm = op.submodule
+        self.submodules[subm.key()] = subm
+        self.op_list.append(op)
+        self._add_preamble(subm.preamble)
+        return op
+
+    def _add_postmap_op(self, *args, **kwargs):
+        op = _FusionOp(len(self.postmap_op_list), *args, **kwargs)
+        subm = op.submodule
+        self.submodules[subm.key()] = subm
+        self.postmap_op_list.append(op)
+        self._add_preamble(subm.preamble)
+        return op
+
+    def add_op(self, *args, **kwargs):
+        if self._has_reduction():
+            return self._add_postmap_op(*args, **kwargs)
+        else:
+            return self._add_premap_op(*args, **kwargs)
+
+    def set_reduce_op(self, raw, arg, kwargs):
+        assert self.reduce_op is None
+        for op in raw._ops.ops:
+            input_type, = op.in_types
+            output_type, = op.out_types
+            if numpy.can_cast(arg.dtype, input_type):
+                return_dtype = numpy.dtype(output_type)
+                self.premap_ret = self._get_fusion_var(arg)._var
+                self.reduce_op = op
+                self.reduce_identity = raw.identity
+                self.reduce_kwargs = kwargs
+                self._add_preamble(raw.preamble)
+                return self._fresh_postmap_param(return_dtype)
+        raise TypeError('Type is mismatched. {}(...), {}'.format(
+            self.raw._ops.name, arg.dtype.type))
+
+    def _add_preamble(self, preamble):
+        self.preamble_set.add(preamble)
+
+    def _get_fusion_var(self, arg):
+        """This converts `arg` to _FusionVarScalar or _FusionVarArray data.
+
+        Args:
+            arg (_FusionVarScalar, _FusionVarArray or a primitive type)
+
+        Return value:
+            _FusionVarScalar or _FusionVarArray
+        """
+        if isinstance(arg, (_FusionVarScalar, _FusionVarArray)):
+            if arg._is_postmap == self._has_reduction():
+                return arg
+            else:
+                # Map operation between pre-map variable and post-map variable
+                raise Exception('Shape mismatch')
+        if isinstance(arg, (int, float, bool, complex, numpy.generic)):
+            var = self._fresh_local(numpy.dtype(type(arg)), const_value=arg)
+            return _FusionVarScalar(var, -1, self._has_reduction())
+        raise TypeError('Unsupported type {}'.format(type(arg)))
+
+    def call_ufunc(self, ufunc, *args, **kwargs):
+        nin = ufunc.nin
+        nout = ufunc.nout
+
+        # Corresponds to _check_should_use_min_scalar in elementwise.pxi
+        # This function decides which typecast rule to use.
+        def _should_use_min_scalar(in_args):
+            max_array_kind = -2
+            max_scalar_kind = -1
+            for arg in in_args:
+                kind = _kind_score[arg.dtype.kind]
+                if isinstance(arg, _FusionVarArray):
+                    max_array_kind = max(max_array_kind, kind)
+                elif isinstance(arg, _FusionVarScalar):
+                    max_scalar_kind = max(max_scalar_kind, kind)
+                else:
+                    assert False
+            return (max_scalar_kind != -1 and
+                    max_array_kind >= max_scalar_kind)
+
+        def can_cast1(args, in_dtypes):
+            for i in range(nin):
+                arg = args[i]
+                if isinstance(arg, _FusionVarArray):
+                    if not numpy.can_cast(arg.dtype, in_dtypes[i]):
+                        return False
+                elif isinstance(arg, _FusionVarScalar):
+                    scalar_value = arg._var.const_value
+                    if scalar_value is None:
+                        # This typecast is not safe.
+                        # The result of a typecast of an element-wise operation
+                        # between a numpy ndarray and a numpy scalar is not
+                        # decidable statically, because it depends on the value
+                        # of the scalar variable.
+                        scalar_value = arg.dtype.type(0)
+                    if not numpy.can_cast(scalar_value, in_dtypes[i]):
+                        return False
+                else:
+                    assert False
+            return True
+
+        def can_cast2(args, in_dtypes):
+            for i in range(nin):
+                if not numpy.can_cast(args[i].dtype, in_dtypes[i]):
+                    return False
+            return True
+
+        def make_fusion_var(var, ndim):
+            if ndim == -1:
+                return _FusionVarScalar(var, ndim, self._has_reduction())
+            else:
+                return _FusionVarArray(var, ndim, self._has_reduction())
+
+        # Make FusionVar list
+        var_list = [self._get_fusion_var(arg) for arg in args]
+        in_vars = var_list[:nin]
+        out_vars = var_list[nin:]
+        if 'out' in kwargs:
+            out = kwargs.pop('out')
+            if out_vars:
+                raise ValueError('cannot specify \'out\' as both a positional '
+                                 'and keyword argument')
+            if isinstance(out, _FusionVarArray):
+                out_vars.append(self._get_fusion_var(out))
+            elif out is not None:
+                raise ValueError('The \'out\' tuple must have exactly one '
+                                 'entry per ufunc output')
+        if kwargs:
+            raise TypeError('Wrong arguments {}'.format(kwargs))
+        if len(in_vars) != nin or len(out_vars) > nout:
+            raise ValueError('invalid number of arguments')
+        if not all([isinstance(v, _FusionVarArray) for v in out_vars]):
+            raise TypeError('return arrays must be of ArrayType')
+        var_list = in_vars + out_vars
+
+        # Broadcast
+        ndim = max([v.ndim for v in in_vars])
+        if any([v.ndim < ndim for v in out_vars]):
+            raise ValueError('non-broadcastable output operand')
+
+        # Typecast and add an operation
+        can_cast = can_cast1 if _should_use_min_scalar(var_list) else can_cast2
+        # TODO(asi1024): Fix to use ``guess_routine``.
+        for op in ufunc._ops.ops:
+            in_dtypes = [numpy.dtype(t) for t in op.in_types]
+            out_dtypes = [numpy.dtype(t) for t in op.out_types]
+            if can_cast(var_list, in_dtypes):
+                ret = []
+                for i in range(nout):
+                    if i >= len(out_vars):
+                        out_var = self._fresh_local(out_dtypes[i])
+                        out_var = make_fusion_var(out_var, ndim)
+                        out_vars.append(out_var)
+                    else:
+                        _raise_if_invalid_cast(
+                            out_dtypes[i], out_vars[i].dtype, 'same_kind',
+                            lambda: f'output {i}')
+                        out_var = out_vars[i]
+
+                    out_var._var.mutate()
+                    ret.append(out_var)
+
+                in_params = [(in_dtypes[i], 'in{}'.format(i))
+                             for i, _ in enumerate(in_vars)]
+                out_params = [(out_dtypes[i], 'out{}'.format(i))
+                              for i, _ in enumerate(out_vars)]
+                subm = _Submodule(ufunc, in_params, out_params, op.routine)
+                self.add_op(subm, [v._var for v in in_vars + out_vars])
+                return ret[0] if len(ret) == 1 else tuple(ret)
+        in_dtypes = [v.dtype for v in in_vars]
+        out_dtypes = [v.dtype for v in out_vars]
+        raise TypeError('Invalid type cast in \'{}\': {} -> {}'.format(
+            ufunc.name, in_dtypes, out_dtypes))
+
+    def call_elementwise(self, f, args, kwargs):
+        raise NotImplementedError(
+            'Fusion for elementwise-kernel is not implemented yet')
+
+    def _emit_submodules_code(self):
+        res = ''.join(self.preamble_set)
+        res += '\n'.join([_.code() for _ in self.submodules.values()])
+        return res
+
+    def _emit_operation_code(self):
+        res = '// {} operations\n'.format(len(self.op_list))
+        res += ''.join(v.declaration() for v in self.local_list)
+        res += ''.join(op.declaration_args() for op in self.op_list)
+        res += ''.join(op.code() for op in self.op_list)
+        return res
+
+    def _emit_premap_code(self, in_params, operation):
+        return_var = self.premap_ret
+        module_code = string.Template('''
+        __device__ ${return_ctype} _pre_map(${in_params}) {
+        ${operation};
+        return ${return_var};
+        }
+        ''').substitute(
+            return_ctype=get_typename(return_var.dtype),
+            in_params=', '.join('{} v{}'.format(get_typename(v.dtype),
+                                                v.index)
+                                for v in in_params),
+            operation=operation,
+            return_var=return_var)
+        return module_code
+
+    def _emit_postmap_code(self, out_params, operation):
+        in_param = self.postmap_param
+        in_ctype = get_typename(in_param.dtype)
+        module_code = string.Template('''
+        __device__ void _post_map(${in_ctype} in, ${out_params}) {
+        ${in_param} = in;
+        ${operation};
+        }
+        ''').substitute(
+            in_ctype=in_ctype,
+            in_param='{} v{}'.format(in_ctype, in_param.index),
+            out_params=', '.join('{} &v{}'.format(get_typename(v.dtype),
+                                                  v.index)
+                                 for v in out_params),
+            operation=operation)
+        return module_code
+
+    def _emit_postmap_cast_code(self, reduce_ctype, postmap_dtype, operation):
+        module_code = string.Template('''
+        __device__ ${postmap_ctype} _postmap_cast(${reduce_ctype} a) {
+        ${postmap_ctype} out0;
+        ${operation};
+        return out0;
+        }
+        ''').substitute(
+            reduce_ctype=reduce_ctype,
+            postmap_ctype=get_typename(postmap_dtype),
+            operation=operation)
+        return module_code
+
+    def _gen_abstracted_args(self, a):
+        if isinstance(a, core.ndarray):
+            cuda_var = self._fresh_premap_param(a.dtype)
+            python_var = _FusionVarArray(cuda_var, a.ndim, False)
+        elif a is None:
+            cuda_var = None
+            python_var = None
+        else:
+            cuda_var = self._fresh_premap_param(numpy.dtype(type(a)))
+            python_var = _FusionVarScalar(cuda_var, -1, False)
+        return cuda_var, python_var
+
+    def get_fusion(self, func, args, name):
+        """This generates CUDA kernel from the given function and dtypes.
+
+        This function generates ElementwiseKernel or ReductioKernel from the
+        given function and the list of dtypes of parameters.
+
+        Args:
+            func (function): The function to be fused.
+            args (tuple): The tuple of arguments.
+            name (str): The name of the kernel.
+
+        Return value (tuple of ElementwiseKernel/ReductionKernel and dict):
+            The second element of return values is kwargs that will give into
+            the elementwise kernel or reduction kernel.
+        """
+        self.ndim = max([a.ndim for a in args if isinstance(a, core.ndarray)])
+
+        in_params = []
+        function_args = []
+        for a in args:
+            cuda_var, python_var = self._gen_abstracted_args(a)
+            if cuda_var is not None:
+                in_params.append(cuda_var)
+            function_args.append(python_var)
+
+        return_value = func(*function_args)
+
+        if isinstance(return_value, tuple):
+            return_tuple = True
+            no_return = False
+            out_pvars = return_value
+        elif isinstance(return_value, (_FusionVarScalar, _FusionVarArray)):
+            return_tuple = False
+            no_return = False
+            out_pvars = [return_value]
+        elif return_value is None:
+            return_tuple = False
+            no_return = True
+            out_pvars = []
+        else:
+            raise TypeError(
+                'Fusion function can\'t return {}'.format(type(return_value)))
+
+        out_pvars = [_ for _ in out_pvars if _ is not None]
+        out_cvars = [self._get_fusion_var(_)._var for _ in out_pvars]
+
+        out_dtypes = [_.dtype for _ in out_pvars]
+        out_params = [self._fresh_premap_param(t) for t in out_dtypes]
+
+        in_params_code = ', '.join(var.declaration_in_param()
+                                   for var in in_params)
+        out_params_code = ', '.join(var.declaration_out_param()
+                                    for var in out_params)
+
+        operation = self._emit_operation_code()
+        submodule_code = self._emit_submodules_code()
+
+        if self.reduce_op is None:
+            operation += ' '.join('{} = {};'.format(t, s)
+                                  for s, t in zip(out_cvars, out_params))
+            kernel = _kernel.ElementwiseKernel(
+                in_params_code, out_params_code, operation,
+                preamble=submodule_code,
+                return_tuple=return_tuple,
+                no_return=no_return,
+                name=name)
+            return kernel, {}
+        else:
+            _, reduce_expr, postmap_expr, reduce_ctype = self.reduce_op.routine
+            if reduce_ctype is None:
+                reduce_ctype = 'type_out0_raw'
+
+            postmap_type, = self.reduce_op.out_types
+            postmap_dtype = numpy.dtype(postmap_type)
+            postmap_ctype = get_typename(postmap_dtype)
+
+            postmap_code = '// {} operations\n'.format(
+                len(self.postmap_op_list))
+            postmap_code += ''.join(v.declaration()
+                                    for v in self.postmap_local_list)
+            postmap_code += ''.join(op.declaration_args()
+                                    for op in self.postmap_op_list)
+            postmap_code += ''.join(op.code() for op in self.postmap_op_list)
+            postmap_code += ' '.join('{} = {};'.format(t, s)
+                                     for s, t in zip(out_cvars, out_params))
+
+            submodule_code += self._emit_premap_code(in_params, operation)
+            use_cub = ACCELERATOR_CUB in _accelerator._reduction_accelerators
+            if not use_cub:
+                submodule_code += 'typedef {} type_in0_raw;\n'.format(
+                    postmap_ctype)
+            submodule_code += 'typedef {} type_out0_raw;\n'.format(
+                postmap_ctype)
+            submodule_code += self._emit_postmap_cast_code(
+                reduce_ctype, postmap_dtype, postmap_expr)
+            submodule_code += self._emit_postmap_code(out_params, postmap_code)
+
+            kernel = _reduction.ReductionKernel(
+                in_params_code,
+                out_params_code,
+                '_pre_map({})'.format(', '.join([repr(p) for p in in_params])),
+                reduce_expr,
+                '_post_map(_postmap_cast(a), {})'.format(
+                    ', '.join([repr(p) for p in out_params])),
+                self.reduce_identity,
+                name=name,
+                reduce_type=reduce_ctype,
+                preamble=submodule_code)
+            return kernel, self.reduce_kwargs
+
+
+class Fusion(object):
+
+    """Function class.
+
+    This class can be get by using `fuse` function and
+    works like `ElementwiseKernel` or `ReductionKernel`.
+
+    Attributes:
+        func (function): The function before fusing.
+        name (str): The name of the function.
+    """
+
+    def __init__(self, func, name=None):
+        self.func = func
+        self.name = name or func.__name__
+        self._memo = {}
+        self.new_fusion = None
+
+    def __repr__(self):
+        return '<Fusion \'{}\'>'.format(self.name)
+
+    def __call__(self, *args):
+        if self.new_fusion is not None:
+            return self.new_fusion(*args)
+
+        # Inner function of composition of multiple fused functions.
+        if _fusion_thread_local.is_old_fusing():
+            return self.func(*args)
+
+        exec_cupy = False
+        for a in args:
+            if isinstance(a, core.ndarray):
+                exec_cupy = True
+                break
+        if not exec_cupy:
+            # No cupy ndarray exists in the arguments
+            return self.func(*args)
+
+        # Invalid argument types
+        for arg in args:
+            if not isinstance(arg, _acceptable_types):
+                mes = 'Invalid argument type for \'{}\': ({})'
+                arg_types = ', '.join(repr(type(a)) for a in args)
+                raise TypeError(mes.format(self.name, arg_types))
+
+        # Cache the result of execution path analysis
+        cdef list params_info = []
+        for arg in args:
+            if isinstance(arg, core.ndarray):
+                params_info.append(arg.dtype.char)
+                params_info.append(arg.ndim)
+            elif isinstance(arg, numpy.generic):
+                params_info.append(arg.dtype.char)
+            elif arg is None:
+                params_info.append(None)
+            elif isinstance(arg, float):
+                params_info.append('d')
+            elif isinstance(arg, int):
+                params_info.append('l')
+            elif isinstance(arg, bool):
+                params_info.append('?')
+            elif isinstance(arg, complex):
+                params_info.append('D')
+            else:
+                assert False
+
+        cdef tuple key = tuple(params_info)
+        if key not in self._memo:
+            try:
+                history = _FusionHistory()
+                _thread_local.history = history
+                _thread_local.is_old_fusing = True
+                try:
+                    self._memo[key] = history.get_fusion(
+                        self.func, args, self.name)
+                except Exception:
+                    self.new_fusion = new_fusion.Fusion(self.func, self.name)
+                    _thread_local.history = None
+                    _thread_local.is_old_fusing = False
+                    return self.new_fusion(*args)
+            finally:
+                _thread_local.history = None
+                _thread_local.is_old_fusing = False
+        kernel, kwargs = self._memo[key]
+
+        return kernel(
+            *[a for a in args if a is not None],
+            **kwargs)
+
+    def clear_cache(self):
+        self._memo = {}
+
+
+def fuse(*args, **kwargs):
+    """Decorator that fuses a function.
+
+    This decorator can be used to define an elementwise or reduction kernel
+    more easily than :class:`~cupy.ElementwiseKernel` or
+    :class:`~cupy.ReductionKernel`.
+
+    Since the fused kernels are cached and reused, it is recommended to reuse
+    the same decorated functions instead of e.g. decorating local functions
+    that are defined multiple times.
+
+    Args:
+        kernel_name (str): Name of the fused kernel function.
+            If omitted, the name of the decorated function is used.
+
+    Example:
+
+        >>> @cupy.fuse(kernel_name='squared_diff')
+        ... def squared_diff(x, y):
+        ...     return (x - y) * (x - y)
+        ...
+        >>> x = cupy.arange(10)
+        >>> y = cupy.arange(10)[::-1]
+        >>> squared_diff(x, y)
+        array([81, 49, 25,  9,  1,  1,  9, 25, 49, 81])
+    """
+
+    def wrapper(f, kernel_name=None):
+        return Fusion(f, kernel_name)
+
+    if len(args) == 1 and len(kwargs) == 0 and callable(args[0]):
+        return functools.update_wrapper(wrapper(args[0]), args[0])
+    else:
+        return lambda f: functools.update_wrapper(
+            wrapper(f, *args, **kwargs), f)
+
+
+def _call_ufunc(fusion_op, *args, **kwargs):
+    return _thread_local.history.call_ufunc(fusion_op, *args, **kwargs)
+
+
+def _call_reduction(fusion_op, *args, **kwargs):
+    if len(args) != 1:
+        mes = '{}() takes 1 positional argument but {} were given'
+        raise TypeError(mes.format(fusion_op._ops.name, len(args)))
+
+    arg = args[0]
+    kwargs = dict([(key, value) for key, value in kwargs.items()
+                   if (key in ('axis', 'out') and value is not None)])
+
+    if arg._is_postmap:
+        # Multiple reduction
+        raise NotImplementedError(
+            'Multiple reduction is not implemented yet')
+
+    var = _thread_local.history.set_reduce_op(fusion_op, arg, kwargs)
+
+    src_ndim = max(0, arg.ndim)
+    if 'axis' in kwargs:
+        axis = kwargs['axis']
+        if isinstance(axis, (tuple, list)):
+            ndim = src_ndim - len(axis)
+        else:
+            ndim = src_ndim - 1
+    else:
+        ndim = 0
+    if ndim < 0:
+        raise numpy.AxisError(axis, src_ndim)
+
+    _thread_local.history.ndim = ndim
+    if ndim >= 1:
+        return _FusionVarArray(var, ndim, True)
+    else:
+        return _FusionVarScalar(var, -1, True)
+
+
+def _create_astype_ufunc(dtype):
+    name = 'astype_{}'.format(dtype)
+    rules = tuple(['{}->{}'.format(cast_from.char, dtype.char)
+                   for cast_from in _dtype_list])
+    command = 'out0 = static_cast< {} >(in0)'.format(get_typename(dtype))
+    return core.create_ufunc(name, rules, command)
+
+
+_dtype_to_astype_dict = None
+
+
+def _dtype_to_astype(dtype):
+    global _dtype_to_astype_dict
+    if _dtype_to_astype_dict is None:
+        _dtype_to_astype_dict = dict([
+            (dt, _create_astype_ufunc(dt))
+            for dt in _dtype_list])
+    return _dtype_to_astype_dict[dtype]
diff --git a/cupy/_core/halffloat.h b/cupy/_core/halffloat.h
new file mode 100644
index 0000000..ca0ebed
--- /dev/null
+++ b/cupy/_core/halffloat.h
@@ -0,0 +1,125 @@
+typedef unsigned short npy_uint16;
+typedef unsigned int npy_uint32;
+
+/*
+ ********************************************************************
+ *                     BIT-LEVEL CONVERSIONS                        *
+ ********************************************************************
+ */
+
+
+npy_uint16 npy_floatbits_to_halfbits(npy_uint32 f)
+{
+    npy_uint32 f_exp, f_sig;
+    npy_uint16 h_sgn, h_exp, h_sig;
+
+    h_sgn = (npy_uint16) ((f&0x80000000u) >> 16);
+    f_exp = (f&0x7f800000u);
+
+    /* Exponent overflow/NaN converts to signed inf/NaN */
+    if (f_exp >= 0x47800000u) {
+        if (f_exp == 0x7f800000u) {
+            /* Inf or NaN */
+            f_sig = (f&0x007fffffu);
+            if (f_sig != 0) {
+                /* NaN - propagate the flag in the significand... */
+                npy_uint16 ret = (npy_uint16) (0x7c00u + (f_sig >> 13));
+                /* ...but make sure it stays a NaN */
+                if (ret == 0x7c00u) {
+                    ret++;
+                }
+                return h_sgn + ret;
+            } else {
+                /* signed inf */
+                return (npy_uint16) (h_sgn + 0x7c00u);
+            }
+        } else {
+            /* overflow to signed inf */
+            return (npy_uint16) (h_sgn + 0x7c00u);
+        }
+    }
+
+    /* Exponent underflow converts to a subnormal half or signed zero */
+    if (f_exp <= 0x38000000u) {
+        /*
+         * Signed zeros, subnormal floats, and floats with small
+         * exponents all convert to signed zero half-floats.
+         */
+        if (f_exp < 0x33000000u) {
+            return h_sgn;
+        }
+        /* Make the subnormal significand */
+        f_exp >>= 23;
+        f_sig = (0x00800000u + (f&0x007fffffu));
+        f_sig >>= (113 - f_exp);
+        /* Handle rounding by adding 1 to the bit beyond half precision */
+        /*
+         * If the last bit in the half significand is 0 (already even), and
+         * the remaining bit pattern is 1000...0, then we do not add one
+         * to the bit after the half significand.  In all other cases, we do.
+         */
+        if ((f_sig&0x00003fffu) != 0x00001000u) {
+            f_sig += 0x00001000u;
+        }
+        h_sig = (npy_uint16) (f_sig >> 13);
+        /*
+         * If the rounding causes a bit to spill into h_exp, it will
+         * increment h_exp from zero to one and h_sig will be zero.
+         * This is the correct result.
+         */
+        return (npy_uint16) (h_sgn + h_sig);
+    }
+
+    /* Regular case with no overflow or underflow */
+    h_exp = (npy_uint16) ((f_exp - 0x38000000u) >> 13);
+    /* Handle rounding by adding 1 to the bit beyond half precision */
+    f_sig = (f&0x007fffffu);
+    /*
+     * If the last bit in the half significand is 0 (already even), and
+     * the remaining bit pattern is 1000...0, then we do not add one
+     * to the bit after the half significand.  In all other cases, we do.
+     */
+    if ((f_sig&0x00003fffu) != 0x00001000u) {
+        f_sig += 0x00001000u;
+    }
+    h_sig = (npy_uint16) (f_sig >> 13);
+    /*
+     * If the rounding causes a bit to spill into h_exp, it will
+     * increment h_exp by one and h_sig will be zero.  This is the
+     * correct result.  h_exp may increment to 15, at greatest, in
+     * which case the result overflows to a signed inf.
+     */
+    return h_sgn + h_exp + h_sig;
+}
+
+npy_uint32 npy_halfbits_to_floatbits(npy_uint16 h)
+{
+    npy_uint16 h_exp, h_sig;
+    npy_uint32 f_sgn, f_exp, f_sig;
+
+    h_exp = (h&0x7c00u);
+    f_sgn = ((npy_uint32)h&0x8000u) << 16;
+    switch (h_exp) {
+        case 0x0000u: /* 0 or subnormal */
+            h_sig = (h&0x03ffu);
+            /* Signed zero */
+            if (h_sig == 0) {
+                return f_sgn;
+            }
+            /* Subnormal */
+            h_sig <<= 1;
+            while ((h_sig&0x0400u) == 0) {
+                h_sig <<= 1;
+                h_exp++;
+            }
+            f_exp = ((npy_uint32)(127 - 15 - h_exp)) << 23;
+            f_sig = ((npy_uint32)(h_sig&0x03ffu)) << 13;
+            return f_sgn + f_exp + f_sig;
+        case 0x7c00u: /* inf or NaN */
+            /* All-ones exponent and a copy of the significand */
+            return f_sgn + 0x7f800000u + (((npy_uint32)(h&0x03ffu)) << 13);
+        default: /* normalized */
+            /* Just need to adjust the exponent and shift */
+            return f_sgn + (((npy_uint32)(h&0x7fffu) + 0x1c000u) << 13);
+    }
+}
diff --git a/cupy/_core/include/cupy/README.md b/cupy/_core/include/cupy/README.md
new file mode 100644
index 0000000..38121d8
--- /dev/null
+++ b/cupy/_core/include/cupy/README.md
@@ -0,0 +1,22 @@
+# "include" directory
+
+All files and directories in this directory will be copied to the distribution (sdist and wheel).
+Note that items starting with `.` (e.g., `cub/.git`) are excluded.
+See `setup.py` for details.
+
+## CUB
+
+The `cub` folder is a git submodule for the CUB project.
+Including the CUB headers as a submodule enables not only building the `cupy.cuda.cub` module,
+but also easier maintenance.
+For further information on CUB, see the [CUB Project Website](http://nvlabs.github.com/cub).
+
+## Jitify
+The `Jitify` folder is a git submodule for the Jitify project.
+Including the Jitify header as a submodule for building the `cupy.cuda.jitify` module.
+For further information on Jitify, see the [Jitify repo](https://github.com/NVIDIA/jitify).
+
+## DLPack
+The `dlpack` folder stores the DLPack header for building the `cupy._core.dlpack` module,
+see `README.md` therein.
+For further information on DLPack, see the [DLPack repo](https://github.com/dmlc/dlpack).
diff --git a/cupy/_core/include/cupy/_cuda/README.md b/cupy/_core/include/cupy/_cuda/README.md
new file mode 100644
index 0000000..bc5d93f
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/README.md
@@ -0,0 +1,9 @@
+These files are copied from CUDA Toolkit Distribution and redistributed under the following license:
+
+https://docs.nvidia.com/cuda/archive/9.2/eula/#nvidia-cuda-toolkit-license-agreement
+
+For CUDA 11.2+, we enable CUDA Enhanced Compatibility by hosting the fp16 headers from the latest
+CUDA Toolkit release.
+
+* ``cuda-11`` contains headers from CTK 11.8.0.
+* ``cuda-12`` contains headers from CTK 12.1.1.
diff --git a/cupy/_core/include/cupy/_cuda/cuda-10.2/cuda_fp16.h b/cupy/_core/include/cupy/_cuda/cuda-10.2/cuda_fp16.h
new file mode 100755
index 0000000..a04c111
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-10.2/cuda_fp16.h
@@ -0,0 +1,3052 @@
+/*
+* Copyright 1993-2014 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+/**
+* \defgroup CUDA_MATH_INTRINSIC_HALF Half Precision Intrinsics
+* This section describes half precision intrinsic functions that are
+* only supported in device code.
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_ARITHMETIC Half Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_ARITHMETIC Half2 Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_COMPARISON Half Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_COMPARISON Half2 Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_MISC Half Precision Conversion And Data Movement
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_FUNCTIONS Half Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_FUNCTIONS Half2 Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+#ifndef __CUDA_FP16_H__
+#define __CUDA_FP16_H__
+
+#if defined(__cplusplus)
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__CUDACC__) */
+
+#define __CUDA_FP16_TYPES_EXIST__
+
+/* Forward-declaration of structures defined in "cuda_fp16.hpp" */
+
+/**
+ * \brief half datatype 
+ * 
+ * \details This structure implements the datatype for storing 
+ * half-precision floating-point numbers. The structure implements 
+ * assignment operators and type conversions. 
+ * 16 bits are being used in total: 1 sign bit, 5 bits for the exponent, 
+ * and the significand is being stored in 10 bits. 
+ * The total precision is 11 bits. There are 15361 representable 
+ * numbers within the interval [0.0, 1.0], endpoints included. 
+ * On average we have log10(2**11) ≈ 3.311 decimal digits. 
+ * 
+ * \req IEEE 754-2008 compliant implementation of half-precision 
+ * floating-point numbers. 
+ */
+struct __half;
+
+/**
+ * \brief half2 datatype
+ * 
+ * \details This structure implements the datatype for storing two 
+ * half-precision floating-point numbers. 
+ * The structure implements assignment operators and type conversions. 
+ * 
+ * \req Vectorified version of half. 
+ */
+struct __half2;
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value. 
+* 
+* \details Converts float number \p a to half precision in round-to-nearest-even mode. 
+* \req 
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts float number \p a to half precision in round-to-nearest-even mode.
+* \req 
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-towards-zero mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-towards-zero mode.
+* \req 
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-down mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-down mode.
+* \req 
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* \retval \p a converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-up mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-up mode.
+* \req 
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* \retval \p a converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts \p half number to float.
+* 
+* \details Converts half number \p a to float.
+* \req 
+* \param[in] a - float. Is only being read. 
+* 
+* \returns float
+* \retval \p a converted to float. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts input to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+*
+* \details Converts input \p a to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+* \req 
+* \param[in] a - float. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 value with both halves equal to the converted half
+* precision number.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both input floats to half precision in round-to-nearest-even
+* mode and returns \p half2 with converted values.
+*
+* \details Converts both input floats to half precision in round-to-nearest-even mode
+* and combines the results into one \p half2 number. Low 16 bits of the return
+* value correspond to the input \p a, high 16 bits correspond to the input \p
+* b.
+* \req 
+* \param[in] a - float. Is only being read. 
+* \param[in] b - float. Is only being read. 
+* 
+* \returns half2
+* \retval The \p half2 value with corresponding halves equal to the
+* converted input floats.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts low 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts low 16 bits of \p half2 input \p a to 32 bit floating point number
+* and returns the result.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* \retval The low 16 bits of \p a converted to float.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts high 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts high 16 bits of \p half2 input \p a to 32 bit floating point number
+* and returns the result.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* \retval The high 16 bits of \p a converted to float.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a);
+
+#if defined(__CUDACC__)
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both components of float2 number to half precision in
+* round-to-nearest-even mode and returns \p half2 with converted values.
+* 
+* \details Converts both components of float2 to half precision in round-to-nearest
+* mode and combines the results into one \p half2 number. Low 16 bits of the
+* return value correspond to \p a.x and high 16 bits of the return value
+* correspond to \p a.y.
+* \req 
+* \param[in] a - float2. Is only being read. 
+*  
+* \returns half2
+* \retval The \p half2 which has corresponding halves equal to the
+* converted float2 components.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both halves of \p half2 to float2 and returns the result.
+* 
+* \details Converts both halves of \p half2 input \p a to float2 and returns the
+* result.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float2
+* \retval \p a converted to float2.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ int __half2int_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-towards-zero mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ int __half2int_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-down mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ int __half2int_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-up mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ int __half2int_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-to-nearest-even mode.
+* \req 
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __int2half_rn(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-towards-zero mode.
+* \req 
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __int2half_rz(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-down mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-down mode.
+* \req 
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __int2half_rd(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-up mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-up mode.
+* \req 
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __int2half_ru(int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-to-nearest-even mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ short int __half2short_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-towards-zero mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ short int __half2short_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-down mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ short int __half2short_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-up mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ short int __half2short_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \req 
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __short2half_rn(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \req 
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __short2half_rz(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-down mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \req 
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __short2half_rd(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-up mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \req 
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __short2half_ru(short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-to-nearest-even mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-towards-zero mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-down mode.
+*
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-down mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-up mode.
+*
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-up mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-to-nearest-even mode.
+* \req 
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rn(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-towards-zero mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-towards-zero mode.
+* \req 
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half.  
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rz(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-down mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-down mode.
+* \req 
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rd(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-up mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-up mode.
+* \req 
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __uint2half_ru(unsigned int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-to-nearest-even mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-towards-zero
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-towards-zero mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-down mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-up mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \req 
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rn(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \req 
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rz(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-down mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \req 
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rd(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-up mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \req 
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_ru(unsigned short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-to-nearest-even mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-towards-zero
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-towards-zero mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-down mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-up mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \req 
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rn(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \req 
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rz(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \req 
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half.  
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rd(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \req 
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ull2half_ru(unsigned long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-to-nearest-even mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-towards-zero mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-down mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-up mode.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \req 
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rn(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \req 
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rz(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \req 
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rd(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \req 
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ll2half_ru(long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Truncate input argument to the integral part.
+* 
+* \details Round \p h to the nearest integer value that does not exceed \p h in
+* magnitude.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The truncated integer value. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half htrunc(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate ceiling of the input argument.
+* 
+* \details Compute the smallest integer value not less than \p h.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The smallest integer value not less than \p h. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hceil(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details Calculate the largest integer value which is less than or equal to \p h.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The largest integer value which is less than or equal to \p h. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hfloor(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating point
+* number.
+* 
+* \details Round \p h to the nearest integer value in half-precision floating point
+* format, with halfway cases rounded to the nearest even integer value.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The nearest integer to \p h. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hrint(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Truncate \p half2 vector input argument to the integral part.
+* 
+* \details Round each component of vector \p h to the nearest integer value that does
+* not exceed \p h in magnitude.
+* \req 
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The truncated \p h. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate \p half2 vector ceiling of the input argument.
+* 
+* \details For each component of vector \p h compute the smallest integer value not less
+* than \p h.
+* \req 
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of smallest integers not less than \p h. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details For each component of vector \p h calculate the largest integer value which
+* is less than or equal to \p h.
+* \req 
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of largest integers which is less than or equal to \p h. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating point
+* number.
+* 
+* \details Round each component of \p half2 vector \p h to the nearest integer value in
+* half-precision floating point format, with halfway cases rounded to the
+* nearest even integer value.
+* \req 
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of rounded integer values. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns \p half2 with both halves equal to the input value.
+* 
+* \details Returns \p half2 number with both halves equal to the input \p a \p half
+* number.
+* \req 
+* \param[in] a - half. Is only being read. 
+* 
+* \returns half2
+* \retval The vector which has both its halves equal to the input \p a. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Swaps both halves of the \p half2 input.
+* 
+* \details Swaps both halves of the \p half2 input and returns a new \p half2 number
+* with swapped halves.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval \p a with its halves being swapped. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from each of the two \p half2 inputs and combines
+* into one \p half2 number. 
+* 
+* \details Extracts low 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. Low 16 bits from input \p a is stored in low 16 bits of
+* the return value, low 16 bits from input \p b is stored in high 16 bits of
+* the return value. 
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The low 16 bits of \p a and of \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from each of the two \p half2 inputs and
+* combines into one \p half2 number.
+* 
+* \details Extracts high 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. High 16 bits from input \p a is stored in low 16 bits of
+* the return value, high 16 bits from input \p b is stored in high 16 bits of
+* the return value.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The high 16 bits of \p a and of \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns high 16 bits of \p half2 input.
+*
+* \details Returns high 16 bits of \p half2 input \p a.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* \retval The high 16 bits of the input. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __high2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns low 16 bits of \p half2 input.
+*
+* \details Returns low 16 bits of \p half2 input \p a.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* \retval Returns \p half which contains low 16 bits of the input \p a. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __low2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Checks if the input \p half number is infinite.
+* 
+* \details Checks if the input \p half number \p a is infinite. 
+* \req 
+* \param[in] a - half. Is only being read. 
+* 
+* \returns int 
+* \retval -1 iff \p a is equal to negative infinity, 
+* \retval 1 iff \p a is equal to positive infinity, 
+* \retval 0 otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ int __hisinf(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Combines two \p half numbers into one \p half2 number.
+* 
+* \details Combines two input \p half number \p a and \p b into one \p half2 number.
+* Input \p a is stored in low 16 bits of the return value, input \p b is stored
+* in high 16 bits of the return value.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with one half equal to \p a and the other to \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from \p half2 input.
+* 
+* \details Extracts low 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with both halves equal to the low 16 bits of the input. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from \p half2 input.
+* 
+* \details Extracts high 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with both halves equal to the high 16 bits of the input. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 a);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as a signed short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating point number \p h
+* as a signed short integer. 
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval The reinterpreted value. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as an unsigned short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating point \p h
+* as an unsigned short number.
+* \req 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval The reinterpreted value.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a signed short integer as a \p half.
+* 
+* \details Reinterprets the bits in the signed short integer \p i as a
+* half-precision floating point number.
+* \req 
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval The reinterpreted value.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in an unsigned short integer as a \p half.
+* 
+* \details Reinterprets the bits in the unsigned short integer \p i as a
+* half-precision floating point number.
+* \req 
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval The reinterpreted value.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i);
+
+#if __CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)
+#if !defined warpSize && !defined __local_warpSize
+#define warpSize    32
+#define __local_warpSize
+#endif
+
+#if defined(_WIN32)
+# define __DEPRECATED__(msg) __declspec(deprecated(msg))
+#elif (defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 5 && !defined(__clang__))))
+# define __DEPRECATED__(msg) __attribute__((deprecated))
+#else
+# define __DEPRECATED__(msg) __attribute__((deprecated(msg)))
+#endif
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+#define __WSB_DEPRECATION_MESSAGE(x) #x"() is deprecated in favor of "#x"_sync() and may be removed in a future release (Use -Wno-deprecated-declarations to suppress this warning)."
+
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half2 __shfl(__half2 var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half2 __shfl_up(__half2 var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down))__half2 __shfl_down(__half2 var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half2 __shfl_xor(__half2 var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half __shfl(__half var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half __shfl_up(__half var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down)) __half __shfl_down(__half var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half __shfl_xor(__half var, int delta, int width = warpSize);
+#endif
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* ithin the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \req 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_sync(unsigned mask, __half2 var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(unsigned mask, __half2 var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(unsigned mask, __half2 var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(unsigned mask, __half2 var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* ithin the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+*/
+__CUDA_FP16_DECL__ __half __shfl_sync(unsigned mask, __half var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+*/
+__CUDA_FP16_DECL__ __half __shfl_up_sync(unsigned mask, __half var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+*/
+__CUDA_FP16_DECL__ __half __shfl_down_sync(unsigned mask, __half var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+*/
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(unsigned mask, __half var, int delta, int width = warpSize);
+
+#if defined(__local_warpSize)
+#undef warpSize
+#undef __local_warpSize
+#endif
+#endif /*__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__) */
+
+#if defined(__cplusplus) && ( __CUDA_ARCH__ >=320 || !defined(__CUDA_ARCH__) )
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *ptr);
+__CUDA_FP16_DECL__ __half __ldg(const __half *ptr);
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *ptr);
+__CUDA_FP16_DECL__ __half __ldcg(const __half *ptr);
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *ptr);
+__CUDA_FP16_DECL__ __half __ldca(const __half *ptr);
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *ptr);
+__CUDA_FP16_DECL__ __half __ldcs(const __half *ptr);
+#endif /*defined(__cplusplus) && ( __CUDA_ARCH__ >=320 || !defined(__CUDA_ARCH__) )*/
+
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs half2 vector if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of if-equal comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison.
+* 
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of not-equal comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 result of less-equal comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of greater-equal comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The half2 vector result of less-than comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison.
+* 
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of greater-than comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of unordered if-equal comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered not-equal comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison.
+*
+* Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered less-equal comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 vector result of unordered greater-equal comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered less-than comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 vector result of unordered greater-than comparison of vectors \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Determine whether \p half2 argument is a NaN.
+*
+* \details Determine whether each half of input \p half2 number \p a is a NaN.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The half2 with the corresponding \p half results set to
+* 1.0 for for NaN, 0.0 otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \req DEEPLEARN-SRM_REQ-95
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The sum of vectors \p a and \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode.
+* \req DEEPLEARN-SRM_REQ-104
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The subtraction of vector \p b from \p a. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode.
+* \req DEEPLEARN-SRM_REQ-102
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise multiplying the vectors \p a and \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector division in round-to-nearest-even mode.
+*
+* \details Divides \p half2 input vector \p a by input vector \p b in round-to-nearest
+* mode.
+* \req DEEPLEARN-SRM_REQ-103
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise division of \p a with \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __h2div(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+*
+* \details Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval Returns \p a with the absolute value of both halves. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the results to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The sum of \p a and \p b, with respect to saturation. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The subtraction of vector \p b from \p a, with respect to saturation.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise multiplication of vectors \p a and \p b, 
+* with respect to saturation. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \req DEEPLEARN-SRM_REQ-105
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode, with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the
+* results to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c, 
+* with respect to saturation. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Negates both halves of the input \p half2 number and returns the
+* result.
+*
+* \details Negates both halves of the input \p half2 number \p a and returns the result.
+* \req DEEPLEARN-SRM_REQ-101
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval Returns \p a with both halves negated. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Calculates the absolute value of input \p half number and returns the result.
+*
+* \details Calculates the absolute value of input \p half number and returns the result.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The absolute value of a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __habs(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode.
+*
+* \details Performs \p half addition of inputs \p a and \p b, in round-to-nearest-even
+* mode.
+* \req DEEPLEARN-SRM_REQ-94
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The sum of \p a and \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode.
+* \req DEEPLEARN-SRM_REQ-97
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of subtracting \p b from \p a. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \req DEEPLEARN-SRM_REQ-99
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of multiplying \p a and \p b. 
+*/
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half division in round-to-nearest-even mode.
+* 
+* \details Divides \p half input \p a by input \p b in round-to-nearest
+* mode.
+* \req DEEPLEARN-SRM_REQ-98
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half
+* \retval The result of dividing \p a by \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__  __half __hdiv(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half add of inputs \p a and \p b, in round-to-nearest-even mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The sum of \p a and \p b, with respect to saturation.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of subtraction of \p b from \p a, with respect to saturation.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the result to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of multiplying \p a and \p b, with respect to saturation.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \req DEEPLEARN-SRM_REQ-96
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* \retval The result of fused multiply-add operation on \p
+* a, \p b, and \p c. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the result
+* to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* \retval The result of fused multiply-add operation on \p
+* a, \p b, and \p c, with respect to saturation. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Negates input \p half number and returns the result.
+*
+* \details Negates input \p half number and returns the result.
+* \req DEEPLEARN-SRM_REQ-100
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval minus a
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half __hneg(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector if-equal comparison, and returns boolean true
+* iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of if-equal comparison
+* of vectors \p a and \p b are true;
+* \retval false, otherwise.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of not-equal comparison
+* of vectors \p a and \p b are true, 
+* \retval false otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of less-equal comparison
+* of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of greater-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of less-than comparison
+* of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns bool 
+* \retval true, if both \p half results of greater-than
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered if-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered not-equal
+* comparison of vectors \p a and \p b are true;
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered less-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison, and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered
+* greater-equal comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered less-than comparison of 
+* vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison, and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered
+* greater-than comparison of vectors \p a and \p b are true;
+* \retval false, otherwise. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of if-equal comparison of \p a and \p b. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of not-equal comparison of \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of less-equal comparison of \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of greater-equal comparison of \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of less-than comparison of \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of greater-than comparison of \p a and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered if-equal comparison of \p a and
+* \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered not-equal comparison of \p a and
+* \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered less-equal comparison of \p a and
+* \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered greater-equal comparison of \p a
+* and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered less-than comparison of \p a and
+* \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \req 
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered greater-than comparison of \p a
+* and \p b.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Determine whether \p half argument is a NaN.
+*
+* \details Determine whether \p half value \p a is a NaN.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns bool
+* \retval true iff argument is NaN. 
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ bool __hisnan(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half square root of input \p a in round-to-nearest-even mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The square root of \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal square root in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half reciprocal square root of input \p a in round-to-nearest
+* mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The reciprocal square root of \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half reciprocal of input \p a in round-to-nearest-even mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The reciprocal of \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hrcp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half natural logarithm of input \p a in round-to-nearest-even
+* mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The natural logarithm of \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hlog(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half binary logarithm of input \p a in round-to-nearest-even
+* mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The binary logarithm of \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hlog2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half decimal logarithm of input \p a in round-to-nearest-even
+* mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The decimal logarithm of \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hlog10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half natural exponential function of input \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The natural exponential function on \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hexp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half binary exponential function of input \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The binary exponential function on \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hexp2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half decimal exponential function of input \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The decimal exponential function on \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hexp10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half cosine in round-to-nearest-even mode.
+*
+* \details Calculates \p half cosine of input \p a in round-to-nearest-even mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The cosine of \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hcos(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half sine in round-to-nearest-even mode.
+*
+* \details Calculates \p half sine of input \p a in round-to-nearest-even mode.
+* \req 
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The sine of \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half hsin(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 square root of input vector \p a in round-to-nearest
+* mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise square root on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal square root in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 reciprocal square root of input vector \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise reciprocal square root on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 reciprocal of input vector \p a in round-to-nearest-even
+* mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise reciprocal on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector natural logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 natural logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise natural logarithm on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 binary logarithm of input vector \p a in round-to-nearest
+* mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise binary logarithm on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 decimal logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise decimal logarithm on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise exponential function on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary exponential function in
+* round-to-nearest-even mode.
+*
+* \details Calculates \p half2 binary exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise binary exponential function on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal exponential function in
+* round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 decimal exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise decimal exponential function on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector cosine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 cosine of input vector \p a in round-to-nearest-even
+* mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise cosine on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector sine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 sine of input vector \p a in round-to-nearest-even mode.
+* \req 
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise sine on vector \p a.
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+*/
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a);
+
+#endif /*if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)*/
+
+#if __CUDA_ARCH__ >= 600 || !defined(__CUDA_ARCH__)
+
+__CUDA_FP16_DECL__ __half2 atomicAdd(__half2 *address, __half2 val);
+
+#endif /*if __CUDA_ARCH__ >= 600 || !defined(__CUDA_ARCH__)*/
+
+#if __CUDA_ARCH__ >= 700 || !defined(__CUDA_ARCH__)
+
+__CUDA_FP16_DECL__ __half atomicAdd(__half *address, __half val);
+
+#endif /*if __CUDA_ARCH__ >= 700 || !defined(__CUDA_ARCH__)*/
+
+#endif /* defined(__CUDACC__) */
+
+#undef __CUDA_FP16_DECL__
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+
+#endif /* defined(__cplusplus) */
+
+/* Note the .hpp file is included even for host-side compilation, to capture the "half" & "half2" definitions */
+#include "cuda_fp16.hpp"
+
+#endif /* end of include guard: __CUDA_FP16_H__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-10.2/cuda_fp16.hpp b/cupy/_core/include/cupy/_cuda/cuda-10.2/cuda_fp16.hpp
new file mode 100755
index 0000000..a4403b5
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-10.2/cuda_fp16.hpp
@@ -0,0 +1,2071 @@
+/*
+* Copyright 1993-2019 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+#if !defined(__CUDA_FP16_HPP__)
+#define __CUDA_FP16_HPP__
+
+/* C++11 header for std::move. 
+ * In RTC mode, std::move is provided implicitly; don't include the header
+ */
+#if (__cplusplus >= 201103L) && !defined(__CUDACC_RTC__)
+#include <utility>
+#endif /* __cplusplus >= 201103L && !defined(__CUDACC_RTC__) */
+
+/* C++ header for std::memcpy (used for type punning in host-side implementations).
+ * When compiling as a CUDA source file memcpy is provided implicitly.
+ * !defined(__CUDACC__) implies !defined(__CUDACC_RTC__).
+ */
+#if defined(__cplusplus) && !defined(__CUDACC__)
+#include <cstring>
+#endif /* defined(__cplusplus) && !defined(__CUDACC__) */
+
+
+/* Set up function decorations */
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#define __VECTOR_FUNCTIONS_DECL__ static __inline__ __host__ __device__
+#define __CUDA_HOSTDEVICE__ __host__ __device__
+#else /* !defined(__CUDACC__) */
+#if defined(__GNUC__)
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __attribute__ ((unused))
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__GNUC__) */
+#define __CUDA_HOSTDEVICE__
+#endif /* defined(__CUDACC_) */
+
+/* Set up structure-alignment attribute */
+#if defined(__CUDACC__)
+#define __CUDA_ALIGN__(align) __align__(align)
+#else
+/* Define alignment macro based on compiler type (cannot assume C11 "_Alignas" is available) */
+#if __cplusplus >= 201103L
+#define __CUDA_ALIGN__(n) alignas(n)    /* C++11 kindly gives us a keyword for this */
+#else /* !(__cplusplus >= 201103L)*/
+#if defined(__GNUC__)
+#define __CUDA_ALIGN__(n) __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+#define __CUDA_ALIGN__(n) __declspec(align(n))
+#else
+#define __CUDA_ALIGN__(n)
+#endif /* defined(__GNUC__) */
+#endif /* __cplusplus >= 201103L */
+#endif /* defined(__CUDACC__) */
+
+/* Macros to allow half & half2 to be used by inline assembly */
+#define __HALF_TO_US(var) *(reinterpret_cast<unsigned short *>(&(var)))
+#define __HALF_TO_CUS(var) *(reinterpret_cast<const unsigned short *>(&(var)))
+#define __HALF2_TO_UI(var) *(reinterpret_cast<unsigned int *>(&(var)))
+#define __HALF2_TO_CUI(var) *(reinterpret_cast<const unsigned int *>(&(var)))
+
+/**
+* Types which allow static initialization of "half" and "half2" until
+* these become an actual builtin. Note this initialization is as a
+* bitfield representation of "half", and not a conversion from short->half.
+* Such a representation will be deprecated in a future version of CUDA. 
+* (Note these are visible to non-nvcc compilers, including C-only compilation)
+*/
+typedef struct __CUDA_ALIGN__(2) {
+    unsigned short x;
+} __half_raw;
+
+typedef struct __CUDA_ALIGN__(4) {
+    unsigned short x;
+    unsigned short y;
+} __half2_raw;
+
+/* All other definitions in this file are only visible to C++ compilers */
+#if defined(__cplusplus)
+
+/* Hide GCC member initialization list warnings because of host/device in-function init requirement */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstrict-aliasing"
+#pragma GCC diagnostic ignored "-Weffc++"
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+/* class' : multiple assignment operators specified
+   The class has multiple assignment operators of a single type. This warning is informational */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( push )
+#pragma warning( disable:4522 )
+#endif /* defined(__GNUC__) */
+
+struct __CUDA_ALIGN__(2) __half {
+protected:
+    unsigned short __x;
+
+public:
+#if __cplusplus >= 201103L
+    __half() = default;
+#else
+    __CUDA_HOSTDEVICE__ __half() { }
+#endif /* __cplusplus >= 201103L */
+
+    /* Convert to/from __half_raw */
+    __CUDA_HOSTDEVICE__ __half(const __half_raw &hr) : __x(hr.x) { }
+    __CUDA_HOSTDEVICE__ __half &operator=(const __half_raw &hr) { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const volatile __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const { __half_raw ret; ret.x = __x; return ret; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const volatile { __half_raw ret; ret.x = __x; return ret; }
+
+#if !defined(__CUDA_NO_HALF_CONVERSIONS__)
+
+    /* Construct from float/double */
+    __CUDA_HOSTDEVICE__ __half(const float f) { __x = __float2half(f).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const double f) { __x = __float2half(static_cast<float>(f)).__x;  }
+
+    __CUDA_HOSTDEVICE__ operator float() const { return __half2float(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const float f) { __x = __float2half(f).__x; return *this; }
+
+    /* We omit "cast to double" operator, so as to not be ambiguous about up-cast */
+    __CUDA_HOSTDEVICE__ __half &operator=(const double f) { __x = __float2half(static_cast<float>(f)).__x; return *this; }
+
+/* Member functions only available to nvcc compilation so far */
+#if defined(__CUDACC__)
+    /* Allow automatic construction from types supported natively in hardware */
+    /* Note we do avoid constructor init-list because of special host/device compilation rules */
+    __device__ __half(short val) { __x = __short2half_rn(val).__x;  }
+    __device__ __half(unsigned short val) { __x = __ushort2half_rn(val).__x;  }
+    __device__ __half(int val) { __x = __int2half_rn(val).__x;  }
+    __device__ __half(unsigned int val) { __x = __uint2half_rn(val).__x;  }
+    __device__ __half(long long val) { __x = __ll2half_rn(val).__x;  }
+    __device__ __half(unsigned long long val) { __x = __ull2half_rn(val).__x; }
+
+    /* Allow automatic casts to supported builtin types, matching all that are permitted with float */
+    __device__ operator short() const { return __half2short_rn(*this); }
+    __device__ __half &operator=(short val) { __x = __short2half_rn(val).__x; return *this; }
+
+    __device__ operator unsigned short() const { return __half2ushort_rn(*this); }
+    __device__ __half &operator=(unsigned short val) { __x = __ushort2half_rn(val).__x; return *this; }
+
+    __device__ operator int() const { return __half2int_rn(*this); }
+    __device__ __half &operator=(int val) { __x = __int2half_rn(val).__x; return *this; }
+
+    __device__ operator unsigned int() const { return __half2uint_rn(*this); }
+    __device__ __half &operator=(unsigned int val) { __x = __uint2half_rn(val).__x; return *this; }
+
+    __device__ operator long long() const { return __half2ll_rn(*this); }
+    __device__ __half &operator=(long long val) { __x = __ll2half_rn(val).__x; return *this; }
+
+    __device__ operator unsigned long long() const { return __half2ull_rn(*this); }
+    __device__ __half &operator=(unsigned long long val) { __x = __ull2half_rn(val).__x; return *this; }
+
+    /* Boolean conversion - note both 0 and -0 must return false */
+    __device__ operator bool() const { return (__x & 0x7FFF) != 0; }
+#endif /* defined(__CUDACC__) */
+#endif /* !defined(__CUDA_NO_HALF_CONVERSIONS__) */
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16 operations only supported on arch >= 5.3 */
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+#if !defined(__CUDA_NO_HALF_OPERATORS__)
+/* Some basic arithmetic operations expected of a builtin */
+__device__ __forceinline__ __half operator+(const __half &lh, const __half &rh) { return __hadd(lh, rh); }
+__device__ __forceinline__ __half operator-(const __half &lh, const __half &rh) { return __hsub(lh, rh); }
+__device__ __forceinline__ __half operator*(const __half &lh, const __half &rh) { return __hmul(lh, rh); }
+__device__ __forceinline__ __half operator/(const __half &lh, const __half &rh) { return __hdiv(lh, rh); }
+
+__device__ __forceinline__ __half &operator+=(__half &lh, const __half &rh) { lh = __hadd(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator-=(__half &lh, const __half &rh) { lh = __hsub(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator*=(__half &lh, const __half &rh) { lh = __hmul(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator/=(__half &lh, const __half &rh) { lh = __hdiv(lh, rh); return lh; }
+
+/* Note for increment and decrement we use the raw value 0x3C00 equating to half(1.0f), to avoid the extra conversion */
+__device__ __forceinline__ __half &operator++(__half &h)      { __half_raw one; one.x = 0x3C00; h += one; return h; }
+__device__ __forceinline__ __half &operator--(__half &h)      { __half_raw one; one.x = 0x3C00; h -= one; return h; }
+__device__ __forceinline__ __half  operator++(__half &h, int) { __half ret = h; __half_raw one; one.x = 0x3C00; h += one; return ret; }
+__device__ __forceinline__ __half  operator--(__half &h, int) { __half ret = h; __half_raw one; one.x = 0x3C00; h -= one; return ret; }
+
+/* Unary plus and inverse operators */
+__device__ __forceinline__ __half operator+(const __half &h) { return h; }
+__device__ __forceinline__ __half operator-(const __half &h) { return __hneg(h); }
+
+/* Some basic comparison operations to make it look like a builtin */
+__device__ __forceinline__ bool operator==(const __half &lh, const __half &rh) { return __heq(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half &lh, const __half &rh) { return __hne(lh, rh); }
+__device__ __forceinline__ bool operator> (const __half &lh, const __half &rh) { return __hgt(lh, rh); }
+__device__ __forceinline__ bool operator< (const __half &lh, const __half &rh) { return __hlt(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half &lh, const __half &rh) { return __hge(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half &lh, const __half &rh) { return __hle(lh, rh); }
+#endif /* !defined(__CUDA_NO_HALF_OPERATORS__) */
+#endif /* __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__) */
+#endif /* defined(__CUDACC__) */
+
+/* __half2 is visible to non-nvcc host compilers */
+struct __CUDA_ALIGN__(4) __half2 {
+    __half x;
+    __half y;
+
+    // All construct/copy/assign/move
+public:
+#if __cplusplus >= 201103L
+    __half2() = default;
+    __CUDA_HOSTDEVICE__ __half2(__half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(__half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); return *this; }
+#else
+    __CUDA_HOSTDEVICE__ __half2() { }
+#endif /* __cplusplus >= 201103L */
+    __CUDA_HOSTDEVICE__ __half2(const __half &a, const __half &b) : x(a), y(b) { }
+    __CUDA_HOSTDEVICE__ __half2(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); return *this; }
+
+    /* Convert to/from __half2_raw */
+    __CUDA_HOSTDEVICE__ __half2(const __half2_raw &h2r ) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2_raw &h2r) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); return *this; }
+    __CUDA_HOSTDEVICE__ operator __half2_raw() const { __half2_raw ret; __HALF2_TO_UI(ret) = __HALF2_TO_CUI(*this); return ret; }
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16x2 operations only supported on arch >= 5.3 */
+#if (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)) && !defined(__CUDA_NO_HALF2_OPERATORS__)
+
+__device__ __forceinline__ __half2 operator+(const __half2 &lh, const __half2 &rh) { return __hadd2(lh, rh); }
+__device__ __forceinline__ __half2 operator-(const __half2 &lh, const __half2 &rh) { return __hsub2(lh, rh); }
+__device__ __forceinline__ __half2 operator*(const __half2 &lh, const __half2 &rh) { return __hmul2(lh, rh); }
+__device__ __forceinline__ __half2 operator/(const __half2 &lh, const __half2 &rh) { return __h2div(lh, rh); }
+
+__device__ __forceinline__ __half2& operator+=(__half2 &lh, const __half2 &rh) { lh = __hadd2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator-=(__half2 &lh, const __half2 &rh) { lh = __hsub2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator*=(__half2 &lh, const __half2 &rh) { lh = __hmul2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator/=(__half2 &lh, const __half2 &rh) { lh = __h2div(lh, rh); return lh; }
+
+__device__ __forceinline__ __half2 &operator++(__half2 &h)      { __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hadd2(h, one); return h; }
+__device__ __forceinline__ __half2 &operator--(__half2 &h)      { __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hsub2(h, one); return h; }
+__device__ __forceinline__ __half2  operator++(__half2 &h, int) { __half2 ret = h; __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hadd2(h, one); return ret; }
+__device__ __forceinline__ __half2  operator--(__half2 &h, int) { __half2 ret = h; __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hsub2(h, one); return ret; }
+
+__device__ __forceinline__ __half2 operator+(const __half2 &h) { return h; }
+__device__ __forceinline__ __half2 operator-(const __half2 &h) { return __hneg2(h); }
+
+__device__ __forceinline__ bool operator==(const __half2 &lh, const __half2 &rh) { return __hbeq2(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half2 &lh, const __half2 &rh) { return __hbne2(lh, rh); }
+__device__ __forceinline__ bool operator>(const __half2 &lh, const __half2 &rh) { return __hbgt2(lh, rh); }
+__device__ __forceinline__ bool operator<(const __half2 &lh, const __half2 &rh) { return __hblt2(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half2 &lh, const __half2 &rh) { return __hbge2(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half2 &lh, const __half2 &rh) { return __hble2(lh, rh); }
+
+#endif /* __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__) */
+#endif /* defined(__CUDACC__) */
+
+/* Restore warning for multiple assignment operators */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( pop )
+#endif /* defined(_MSC_VER) && _MSC_VER >= 1500 */
+
+/* Restore -Weffc++ warnings from here on */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic pop
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+#undef __CUDA_HOSTDEVICE__
+#undef __CUDA_ALIGN__
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static unsigned short __internal_float2half(const float f, unsigned int &sign, unsigned int &remainder)
+{
+    unsigned int x;
+    unsigned int u;
+    unsigned int result = 0U;
+#if defined(__CUDACC__)
+    (void)memcpy(&x, &f, sizeof(f));
+#else
+    (void)std::memcpy(&x, &f, sizeof(f));
+#endif
+    u = (x & 0x7fffffffU);
+    sign = ((x >> 16U) & 0x8000U);
+    // NaN/+Inf/-Inf
+    if (u >= 0x7f800000U) {
+        remainder = 0U;
+        result = ((u == 0x7f800000U) ? (sign | 0x7c00U) : 0x7fffU);
+    } else if (u > 0x477fefffU) { // Overflows
+        remainder = 0x80000000U;
+        result = (sign | 0x7bffU);
+    } else if (u >= 0x38800000U) { // Normal numbers
+        remainder = u << 19U;
+        u -= 0x38000000U;
+        result = (sign | (u >> 13U));
+    } else if (u < 0x33000001U) { // +0/-0
+        remainder = u;
+        result = sign;
+    } else { // Denormal numbers
+        const unsigned int exponent = u >> 23U;
+        const unsigned int shift = 0x7eU - exponent;
+        unsigned int mantissa = (u & 0x7fffffU);
+        mantissa |= 0x800000U;
+        remainder = mantissa << (32U - shift);
+        result = (sign | (mantissa >> shift));
+    }
+    return static_cast<unsigned short>(result);
+}
+#endif  /* #if !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rz.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rm.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign != 0U)) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rp.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign == 0U)) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a)
+{
+    __half2 val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a));
+#else
+    val = __half2(__float2half_rn(a), __float2half_rn(a));
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b)
+{
+    __half2 val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  cvt.rn.f16.f32 high, %2;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a), "f"(b));
+#else
+    val = __half2(__float2half_rn(a), __float2half_rn(b));
+#endif
+    return val;
+}
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static float __internal_half2float(const unsigned short h)
+{
+    unsigned int sign = ((static_cast<unsigned int>(h) >> 15U) & 1U);
+    unsigned int exponent = ((static_cast<unsigned int>(h) >> 10U) & 0x1fU);
+    unsigned int mantissa = ((static_cast<unsigned int>(h) & 0x3ffU) << 13U);
+    float f;
+    if (exponent == 0x1fU) { /* NaN or Inf */
+        sign = ((mantissa != 0U) ? 0U : sign);
+        mantissa = ((mantissa != 0U) ? 0x7fffffU : 0U);
+        exponent = 0xffU;
+    } else if (exponent == 0U) { /* Denorm or Zero */
+        if (mantissa != 0U) {
+            unsigned int msb;
+            exponent = 0x71U;
+            do {
+                msb = (mantissa & 0x400000U);
+                mantissa <<= 1U; /* normalize */
+                --exponent;
+            } while (msb == 0U);
+            mantissa &= 0x7fffffU; /* 1.mantissa is implicit */
+        }
+    } else {
+        exponent += 0x70U;
+    }
+    unsigned int u = ((sign << 31U) | (exponent << 23U) | mantissa);
+#if defined(__CUDACC__)
+    (void)memcpy(&f, &u, sizeof(u));
+#else
+    (void)std::memcpy(&f, &u, sizeof(u));
+#endif
+    return f;
+}
+#endif  /* !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.f32.f16 %0, %1;}\n" : "=f"(val) : "h"(__HALF_TO_CUS(a)));
+#else
+    val = __internal_half2float(static_cast<__half_raw>(a).x);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+#else
+    val = __internal_half2float(static_cast<__half2_raw>(a).x);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+#else
+    val = __internal_half2float(static_cast<__half2_raw>(a).y);
+#endif
+    return val;
+}
+
+/* Intrinsic functions only available to nvcc compilers */
+#if defined(__CUDACC__)
+
+/* CUDA vector-types compatible vector creation function (note returns __half2, not half2) */
+__VECTOR_FUNCTIONS_DECL__ __half2 make_half2(__half x, __half y)
+{
+    __half2 t; t.x = x; t.y = y; return t;
+}
+#undef __VECTOR_FUNCTIONS_DECL__
+
+
+/* Definitions of intrinsics */
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 f)
+{
+    __half2 val = __floats2half2_rn(f.x, f.y);
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 l)
+{
+    float hi_float;
+    float lo_float;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(lo_float) : "r"(__HALF2_TO_CUI(l)));
+
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(hi_float) : "r"(__HALF2_TO_CUI(l)));
+#else
+    lo_float = __internal_half2float(((__half2_raw)l).x);
+    hi_float = __internal_half2float(((__half2_raw)l).y);
+#endif
+    return make_float2(lo_float, hi_float);
+}
+__CUDA_FP16_DECL__ int __half2int_rn(__half h)
+{
+    int i;
+    asm("cvt.rni.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_rz(__half h)
+{
+    int i;
+    asm("cvt.rzi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_rd(__half h)
+{
+    int i;
+    asm("cvt.rmi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_ru(__half h)
+{
+    int i;
+    asm("cvt.rpi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __int2half_rn(int i)
+{
+    __half h;
+    asm("cvt.rn.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rz(int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rd(int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_ru(int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ short int __half2short_rn(__half h)
+{
+    short int i;
+    asm("cvt.rni.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_rz(__half h)
+{
+    short int i;
+    asm("cvt.rzi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_rd(__half h)
+{
+    short int i;
+    asm("cvt.rmi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_ru(__half h)
+{
+    short int i;
+    asm("cvt.rpi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __short2half_rn(short int i)
+{
+    __half h;
+    asm("cvt.rn.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rz(short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rd(short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_ru(short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(__half h)
+{
+    unsigned int i;
+    asm("cvt.rni.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_rz(__half h)
+{
+    unsigned int i;
+    asm("cvt.rzi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(__half h)
+{
+    unsigned int i;
+    asm("cvt.rmi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(__half h)
+{
+    unsigned int i;
+    asm("cvt.rpi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rn(unsigned int i)
+{
+    __half h;
+    asm("cvt.rn.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rz(unsigned int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rd(unsigned int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_ru(unsigned int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rni.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rz(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rzi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rmi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rpi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rn(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rn.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rz(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rd(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_ru(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rni.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rz(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rzi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rmi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rpi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rn(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rn.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rz(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rd(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_ru(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ long long int __half2ll_rn(__half h)
+{
+    long long int i;
+    asm("cvt.rni.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_rz(__half h)
+{
+    long long int i;
+    asm("cvt.rzi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_rd(__half h)
+{
+    long long int i;
+    asm("cvt.rmi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_ru(__half h)
+{
+    long long int i;
+    asm("cvt.rpi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rn(long long int i)
+{
+    __half h;
+    asm("cvt.rn.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rz(long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rd(long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_ru(long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ __half htrunc(const __half h)
+{
+    __half r;
+    asm("cvt.rzi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hceil(const __half h)
+{
+    __half r;
+    asm("cvt.rpi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hfloor(const __half h)
+{
+    __half r;
+    asm("cvt.rmi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hrint(const __half h)
+{
+    __half r;
+    asm("cvt.rni.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rzi.f16.f16 low, low;\n"
+        "  cvt.rzi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rpi.f16.f16 low, low;\n"
+        "  cvt.rpi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rmi.f16.f16 low, low;\n"
+        "  cvt.rmi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rni.f16.f16 low, low;\n"
+        "  cvt.rni.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 l, const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {alow,blow};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)), "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 l, const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {ahigh,bhigh};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)), "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __low2half(const __half2 h)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, low;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(h)));
+    return ret;
+}
+__CUDA_FP16_DECL__ int __hisinf(const __half a)
+{
+    if (__HALF_TO_CUS(a) == 0xFC00) {
+        return -1;
+    }
+    if (__HALF_TO_CUS(a) == 0x7C00) {
+        return 1;
+    }
+    return 0;
+}
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 l)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 l)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __high2half(const __half2 h)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, high;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(h)));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half l, const __half h)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%2};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(l)), "h"(__HALF_TO_CUS(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half lh)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%1};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(lh)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 lh)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(lh)));
+    return val;
+}
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h)
+{
+    return (short int)__HALF_TO_CUS(h);
+}
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h)
+{
+    return __HALF_TO_CUS(h);
+}
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = (unsigned short int)i;
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = i;
+    return h;
+}
+
+#if __CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)
+/******************************************************************************
+*                           __half, __half2 warp shuffle                     *
+******************************************************************************/
+#define __SHUFFLE_HALF2_MACRO(name) do {\
+   __half2 r; \
+   asm volatile ("{"#name" %0,%1,%2,%3;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c)); \
+   return r; \
+} while(0)
+
+#define __SHUFFLE_SYNC_HALF2_MACRO(name) do {\
+   __half2 r; \
+   asm("{"#name" %0,%1,%2,%3,%4;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c), "r"(mask)); \
+   return r; \
+} while(0)
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+
+__CUDA_FP16_DECL__ __half2 __shfl(__half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.idx.b32);
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up(__half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = (warpSize - width) << 8;
+    __SHUFFLE_HALF2_MACRO(shfl.up.b32);
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down(__half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.down.b32);
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor(__half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.bfly.b32);
+}
+
+#endif /* !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700 */
+
+__CUDA_FP16_DECL__ __half2 __shfl_sync(unsigned mask, __half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.idx.b32);
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(unsigned mask, __half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = (warpSize - width) << 8;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.up.b32);
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(unsigned mask, __half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.down.b32);
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(unsigned mask, __half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.bfly.b32);
+}
+
+#undef __SHUFFLE_HALF2_MACRO
+#undef __SHUFFLE_SYNC_HALF2_MACRO
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+
+__CUDA_FP16_DECL__ __half __shfl(__half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up(__half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_up(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down(__half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_down(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor(__half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_xor(temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /* !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700 */
+
+__CUDA_FP16_DECL__ __half __shfl_sync(unsigned mask, __half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up_sync(unsigned mask, __half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_up_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down_sync(unsigned mask, __half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_down_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(unsigned mask, __half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_xor_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /*__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)*/
+/******************************************************************************
+*               __half and __half2 __ldg,__ldcg,__ldca,__ldcs                *
+******************************************************************************/
+
+#if defined(__cplusplus) && (__CUDA_ARCH__ >= 320 || !defined(__CUDA_ARCH__))
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __LDG_PTR   "l"
+#else
+#define __LDG_PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.nc.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldg(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.nc.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cg.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcg(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.cg.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.ca.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldca(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.ca.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cs.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcs(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.cs.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+#undef __LDG_PTR
+#endif /*defined(__cplusplus) && (__CUDA_ARCH__ >= 320 || !defined(__CUDA_ARCH__))*/
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+/******************************************************************************
+*                             __half2 comparison                             *
+******************************************************************************/
+#define __COMPARISON_OP_HALF2_MACRO(name) do {\
+   __half2 val; \
+   asm( "{ "#name".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} while(0)
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.eq);
+}
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ne);
+}
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.le);
+}
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ge);
+}
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.lt);
+}
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gt);
+}
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.equ);
+}
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.neu);
+}
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.leu);
+}
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.geu);
+}
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ltu);
+}
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gtu);
+}
+#undef __COMPARISON_OP_HALF2_MACRO
+#define __BOOL_COMPARISON_OP_HALF2_MACRO(name) do {\
+   __half2 val; \
+   asm( "{ "#name".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   if (__HALF2_TO_CUI(val) == 0x3C003C00) \
+      return true; \
+   else  \
+      return false; \
+} while(0)
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.eq);
+}
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ne);
+}
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.le);
+}
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ge);
+}
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.lt);
+}
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gt);
+}
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.equ);
+}
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.neu);
+}
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.leu);
+}
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.geu);
+}
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ltu);
+}
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gtu);
+}
+#undef __BOOL_COMPARISON_OP_HALF2_MACRO
+/******************************************************************************
+*                             __half comparison                              *
+******************************************************************************/
+#define __COMPARISON_OP_HALF_MACRO(name) do {\
+   unsigned short val; \
+   asm( "{ .reg .pred __$temp3;\n" \
+        "  setp."#name".f16  __$temp3, %1, %2;\n" \
+        "  selp.u16 %0, 1, 0, __$temp3;}" \
+        : "=h"(val) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(b))); \
+   return val ? true : false; \
+} while(0)
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(eq);
+}
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ne);
+}
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(le);
+}
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ge);
+}
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(lt);
+}
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gt);
+}
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(equ);
+}
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(neu);
+}
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(leu);
+}
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(geu);
+}
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ltu);
+}
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gtu);
+}
+#undef __COMPARISON_OP_HALF_MACRO
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+#define __BINARY_OP_HALF2_MACRO(name) do {\
+   __half2 val; \
+   asm( "{"#name".f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} while(0)
+
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add);
+}
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub);
+}
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul);
+}
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add.sat);
+}
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub.sat);
+}
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul.sat);
+}
+#undef __BINARY_OP_HALF2_MACRO
+#define __TERNARY_OP_HALF2_MACRO(name) do {\
+   __half2 val; \
+   asm( "{"#name".f16x2 %0,%1,%2,%3;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b)),"r"(__HALF2_TO_CUI(c))); \
+   return val; \
+} while(0)
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn);
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.sat);
+}
+#undef __TERNARY_OP_HALF2_MACRO
+__CUDA_FP16_DECL__ __half2 __h2div(__half2 a, __half2 b) {
+    __half ha, hb;
+
+    ha = __low2half(a);
+    hb = __low2half(b);
+
+    __half v1 = __hdiv(ha, hb);
+
+    ha = __high2half(a);
+    hb = __high2half(b);
+
+    __half v2 = __hdiv(ha, hb);
+
+    return __halves2half2(v1, v2);
+}
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+#define __BINARY_OP_HALF_MACRO(name) do {\
+   __half val; \
+   asm( "{"#name".f16 %0,%1,%2;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b))); \
+   return val; \
+} while(0)
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add);
+}
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub);
+}
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul);
+}
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add.sat);
+}
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub.sat);
+}
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul.sat);
+}
+#undef __BINARY_OP_HALF_MACRO
+#define __TERNARY_OP_HALF_MACRO(name) do {\
+   __half val; \
+   asm( "{"#name".f16 %0,%1,%2,%3;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b)),"h"(__HALF_TO_CUS(c))); \
+   return val; \
+} while(0)
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn);
+}
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.sat);
+}
+#undef __TERNARY_OP_HALF2_MACRO
+__CUDA_FP16_DECL__ __half __hdiv(__half a, __half b) {
+    __half v, abs, den;
+    __HALF_TO_US(den) = 0x008F;
+    float fa, fb, fv, rcp;
+
+    fa = __half2float(a);
+    fb = __half2float(b);
+
+    asm("{rcp.approx.f32 %0, %1;\n}" :"=f"(rcp) : "f"(fb));
+
+    fv = rcp * fa;
+
+    v = __float2half(fv);
+    __HALF_TO_US(abs) = (unsigned short)(((unsigned int)__HALF_TO_CUS(v)) & 0x00007FFF);
+    if (__hlt(abs, den) && (!(__HALF_TO_CUS(abs) == 0x0000))) {
+        float err = __fmaf_rn(-fb, fv, fa);
+        fv = __fmaf_rn(rcp, err, fv);
+        v = __float2half(fv);
+    }
+    return v;
+}
+
+/******************************************************************************
+*                             __half2 functions                  *
+******************************************************************************/
+#define __SPEC_CASE2(i,r, spc, ulp) \
+   "{.reg.b32 spc, ulp, p;\n"\
+   "  mov.b32 spc,"#spc";\n"\
+   "  mov.b32 ulp,"#ulp";\n"\
+   "  set.eq.f16x2.f16x2 p,"#i", spc;\n"\
+   "  fma.rn.f16x2 "#r",p,ulp,"#r";\n}\n"
+#define __SPEC_CASE(i,r, spc, ulp) \
+   "{.reg.b16 spc, ulp, p;\n"\
+   "  mov.b16 spc,"#spc";\n"\
+   "  mov.b16 ulp,"#ulp";\n"\
+   "  set.eq.f16.f16 p,"#i", spc;\n"\
+   "  fma.rn.f16 "#r",p,ulp,"#r";\n}\n"
+#define __APPROX_FCAST(fun) do {\
+   __half val;\
+   asm("{.reg.b32         f;        \n"\
+                " .reg.b16         r;        \n"\
+                "  mov.b16         r,%1;     \n"\
+                "  cvt.f32.f16     f,r;      \n"\
+                "  "#fun".approx.f32   f,f;  \n"\
+                "  cvt.rn.f16.f32      r,f;  \n"\
+                "  mov.b16         %0,r;     \n"\
+                "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));\
+   return val;\
+} while(0)
+#define __APPROX_FCAST2(fun) do {\
+   __half2 val;\
+   asm("{.reg.b16         hl, hu;         \n"\
+                " .reg.b32         fl, fu;         \n"\
+                "  mov.b32         {hl, hu}, %1;   \n"\
+                "  cvt.f32.f16     fl, hl;         \n"\
+                "  cvt.f32.f16     fu, hu;         \n"\
+                "  "#fun".approx.f32   fl, fl;     \n"\
+                "  "#fun".approx.f32   fu, fu;     \n"\
+                "  cvt.rn.f16.f32      hl, fl;     \n"\
+                "  cvt.rn.f16.f32      hu, fu;     \n"\
+                "  mov.b32         %0, {hl, hu};   \n"\
+                "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));       \
+   return val;\
+} while(0)
+static __device__ __forceinline__ float __float_simpl_sinf(float);
+static __device__ __forceinline__ float __float_simpl_cosf(float);
+__CUDA_FP16_DECL__ __half __hsin_internal(const __half a) {
+    float f = __half2float(a);
+    f = __float_simpl_sinf(f);
+    return __float2half_rn(f);
+}
+__CUDA_FP16_DECL__ __half hsin(const __half a) {
+    __half r = __hsin_internal(a);
+    asm("{\n\t"
+        "  .reg.b16 i,r,t;     \n\t"
+        "  mov.b16 r, %0;      \n\t"
+        "  mov.b16 i, %1;      \n\t"
+        "  mov.b16 t, 0x8000;  \n\t"
+        "  and.b16 t,r,t;      \n\t"
+        __SPEC_CASE(i, r, 0X32B3, 0x0800)
+        __SPEC_CASE(i, r, 0X5CB0, 0x1000)
+        __SPEC_CASE(i, r, 0XB2B3, 0x8800)
+        __SPEC_CASE(i, r, 0XDCB0, 0x9000)
+        "  or.b16  r,r,t;      \n\t"
+        "  mov.b16 %0, r;      \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a) {
+    __half l = __low2half(a);
+    __half h = __high2half(a);
+    __half2 r = __halves2half2(__hsin_internal(l), __hsin_internal(h));
+    asm("{\n\t"
+        "  .reg.b32 i,r,t;             \n\t"
+        "  mov.b32 r, %0;              \n\t"
+        "  mov.b32 i, %1;              \n\t"
+        "  and.b32 t, r, 0x80008000;   \n\t"
+        __SPEC_CASE2(i, r, 0X32B332B3, 0x08000800)
+        __SPEC_CASE2(i, r, 0X5CB05CB0, 0x10001000)
+        __SPEC_CASE2(i, r, 0XB2B3B2B3, 0x88008800)
+        __SPEC_CASE2(i, r, 0XDCB0DCB0, 0x90009000)
+        "  or.b32  r, r, t;            \n\t"
+        "  mov.b32 %0, r;              \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __hcos_internal(const __half a) {
+    float f = __half2float(a);
+    f = __float_simpl_cosf(f);
+    return __float2half_rn(f);
+}
+__CUDA_FP16_DECL__ __half hcos(const __half a) {
+    __half r = __hcos_internal(a);
+    asm("{\n\t"
+        "  .reg.b16 i,r;        \n\t"
+        "  mov.b16 r, %0;       \n\t"
+        "  mov.b16 i, %1;       \n\t"
+        __SPEC_CASE(i, r, 0X2B7C, 0x1000)
+        __SPEC_CASE(i, r, 0XAB7C, 0x1000)
+        "  mov.b16 %0, r;       \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a) {
+    __half l = __low2half(a);
+    __half h = __high2half(a);
+    __half2 r = __halves2half2(__hcos_internal(l), __hcos_internal(h));
+    asm("{\n\t"
+        "  .reg.b32 i,r;   \n\t"
+        "  mov.b32 r, %0;  \n\t"
+        "  mov.b32 i, %1;  \n\t"
+        __SPEC_CASE2(i, r, 0X2B7C2B7C, 0x10001000)
+        __SPEC_CASE2(i, r, 0XAB7CAB7C, 0x10001000)
+        "  mov.b32 %0, r;  \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+static __device__ __forceinline__ float __internal_trig_reduction_kernel(float a, int *quadrant)
+{
+    float j, t;
+    int q;
+    q = __float2int_rn(a * 0.636619772F);
+    j = (float)q;
+    t = __fmaf_rn(-j, 1.5707962512969971e+000F, a);
+    t = __fmaf_rn(-j, 7.5497894158615964e-008F, t);
+    *quadrant = q;
+    return t;
+}
+static __device__ __forceinline__ float __internal_sin_cos_kernel(float x, int i)
+{
+    float x2, z;
+    x2 = x*x;
+
+    if (i & 1) {
+        z = 2.44331571e-5F;
+        z = __fmaf_rn(z, x2, -1.38873163e-3F);
+    }
+    else {
+        z = -1.95152959e-4F;
+        z = __fmaf_rn(z, x2, 8.33216087e-3F);
+    }
+    if (i & 1) {
+        z = __fmaf_rn(z, x2, 4.16666457e-2F);
+        z = __fmaf_rn(z, x2, -5.00000000e-1F);
+    }
+    else {
+        z = __fmaf_rn(z, x2, -1.66666546e-1F);
+        z = __fmaf_rn(z, x2, 0.0F);
+    }
+    x = __fmaf_rn(z, x, x);
+    if (i & 1) {
+        x = __fmaf_rn(z, x2, 1.0F);
+    }
+    if (i & 2) {
+        x = __fmaf_rn(x, -1.0F, 0.0F);
+    }
+    return x;
+}
+static __device__ __forceinline__ float __float_simpl_sinf(float a)
+{
+    float z;
+    int i;
+    if (::isinf(a)) {
+        a = a * 0.0F;
+    }
+    a = __internal_trig_reduction_kernel(a, &i);
+    z = __internal_sin_cos_kernel(a, i);
+    return z;
+}
+static __device__ __forceinline__ float __float_simpl_cosf(float a)
+{
+    float z;
+    int i;
+    if (::isinf(a)) {
+        a = a * 0.0F;
+    }
+    a = __internal_trig_reduction_kernel(a, &i);
+    i++;
+    z = __internal_sin_cos_kernel(a, i);
+    return z;
+}
+
+__CUDA_FP16_DECL__ __half hexp(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C;           \n"
+        " .reg.b16         h,r;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  mov.b32         C, 0x3fb8aa3b;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  ex2.approx.f32      f,f;        \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X1F79, 0x9400)
+        __SPEC_CASE(h, r, 0X25CF, 0x9400)
+        __SPEC_CASE(h, r, 0XC13B, 0x0400)
+        __SPEC_CASE(h, r, 0XC1EF, 0x0200)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu, C;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x3fb8aa3b;  \n"
+        "  mul.f32         fl,fl,C;        \n"
+        "  mul.f32         fu,fu,C;        \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0X1F791F79, 0x94009400)
+        __SPEC_CASE2(h, r, 0X25CF25CF, 0x94009400)
+        __SPEC_CASE2(h, r, 0XC13BC13B, 0x04000400)
+        __SPEC_CASE2(h, r, 0XC1EFC1EF, 0x02000200)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp2(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, ULP;         \n"
+        " .reg.b16         r;              \n"
+        "  mov.b16         r,%1;           \n"
+        "  cvt.f32.f16     f,r;            \n"
+        "  ex2.approx.f32      f,f;        \n"
+        "  mov.b32         ULP, 0x33800000;\n"
+        "  fma.rn.f32      f,f,ULP,f;      \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, ULP;    \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  mov.b32         ULP, 0x33800000;\n"
+        "  fma.rn.f32      fl,fl,ULP,fl;   \n"
+        "  fma.rn.f32      fu,fu,ULP,fu;   \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         %0, {hl, hu};   \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h,r;            \n"
+        " .reg.b32         f, C;           \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  mov.b32         C, 0x40549A78;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  ex2.approx.f32      f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x34DE, 0x9800)
+        __SPEC_CASE(h, r, 0x9766, 0x9000)
+        __SPEC_CASE(h, r, 0x9972, 0x1000)
+        __SPEC_CASE(h, r, 0xA5C4, 0x1000)
+        __SPEC_CASE(h, r, 0xBF0A, 0x8100)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu, C;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x40549A78;  \n"
+        "  mul.f32         fl,fl,C;        \n"
+        "  mul.f32         fu,fu,C;        \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0x34DE34DE, 0x98009800)
+        __SPEC_CASE2(h, r, 0x97669766, 0x90009000)
+        __SPEC_CASE2(h, r, 0x99729972, 0x10001000)
+        __SPEC_CASE2(h, r, 0xA5C4A5C4, 0x10001000)
+        __SPEC_CASE2(h, r, 0xBF0ABF0A, 0x81008100)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog2(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f;              \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.f32      f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(r, r, 0xA2E2, 0x8080)
+        __SPEC_CASE(r, r, 0xBF46, 0x9400)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, r, p;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  lg2.approx.f32      fl, fl;     \n"
+        "  lg2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(r, r, 0xA2E2A2E2, 0x80808080)
+        __SPEC_CASE2(r, r, 0xBF46BF46, 0x94009400)
+        "  mov.b32         %0, r;          \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C;           \n"
+        " .reg.b16         r,h;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  lg2.approx.f32      f,f;        \n"
+        "  mov.b32         C, 0x3f317218;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X160D, 0x9C00)
+        __SPEC_CASE(h, r, 0X3BFE, 0x8010)
+        __SPEC_CASE(h, r, 0X3C0B, 0x8080)
+        __SPEC_CASE(h, r, 0X6051, 0x1C00)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.f32      fl, fl;         \n"
+        "  lg2.approx.f32      fu, fu;         \n"
+        "  mov.b32         C, 0x3f317218;      \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0X160D160D, 0x9C009C00)
+        __SPEC_CASE2(h, r, 0X3BFE3BFE, 0x80108010)
+        __SPEC_CASE2(h, r, 0X3C0B3C0B, 0x80808080)
+        __SPEC_CASE2(h, r, 0X60516051, 0x1C001C00)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f, C;           \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.f32      f, f;       \n"
+        "  mov.b32         C, 0x3E9A209B;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x338F, 0x1000)
+        __SPEC_CASE(h, r, 0x33F8, 0x9000)
+        __SPEC_CASE(h, r, 0x57E1, 0x9800)
+        __SPEC_CASE(h, r, 0x719D, 0x9C00)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.f32      fl, fl;         \n"
+        "  lg2.approx.f32      fu, fu;         \n"
+        "  mov.b32         C, 0x3E9A209B;      \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0x338F338F, 0x10001000)
+        __SPEC_CASE2(h, r, 0x33F833F8, 0x90009000)
+        __SPEC_CASE2(h, r, 0x57E157E1, 0x98009800)
+        __SPEC_CASE2(h, r, 0x719D719D, 0x9C009C00)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+#undef __SPEC_CASE2
+#undef __SPEC_CASE
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a) {
+    __APPROX_FCAST2(rcp);
+}
+__CUDA_FP16_DECL__ __half hrcp(const __half a) {
+    __APPROX_FCAST(rcp);
+}
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a) {
+    __APPROX_FCAST2(rsqrt);
+}
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a) {
+    __APPROX_FCAST(rsqrt);
+}
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a) {
+    __APPROX_FCAST2(sqrt);
+}
+__CUDA_FP16_DECL__ __half hsqrt(const __half a) {
+    __APPROX_FCAST(sqrt);
+}
+#undef __APPROX_FCAST
+#undef __APPROX_FCAST2
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a)
+{
+    __half2 r;
+    asm("{set.nan.f16x2.f16x2 %0,%1,%2;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ bool __hisnan(const __half a)
+{
+    __half r;
+    asm("{set.nan.f16.f16 %0,%1,%2;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(a)));
+    return __HALF_TO_CUS(r) != 0U;
+}
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a)
+{
+    __half2 zero = __float2half2_rn(0.0);
+    return __hsub2(zero, a);
+}
+__CUDA_FP16_DECL__ __half __hneg(const __half a)
+{
+    __half zero;
+    zero = __float2half(0.0);
+    return __hsub(zero, a);
+}
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a)
+{
+    __half2 r;
+    asm("{abs.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __habs(const __half a)
+{
+    __half r;
+    asm("{abs.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+#endif /*__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)*/
+
+/* Define __PTR for atomicAdd prototypes below, undef after done */
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __PTR   "l"
+#else
+#define __PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600
+
+__CUDA_FP16_DECL__  __half2 atomicAdd(__half2 *address, __half2 val) {
+    __half2 r;
+    asm volatile ("{ atom.add.noftz.f16x2 %0,[%1],%2; }\n"
+                  : "=r"(__HALF2_TO_UI(r)) : __PTR(address), "r"(__HALF2_TO_CUI(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600*/
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700
+
+__CUDA_FP16_DECL__  __half atomicAdd(__half *address, __half val) {
+    __half r;
+    asm volatile ("{ atom.add.noftz.f16 %0,[%1],%2; }\n"
+                  : "=h"(__HALF_TO_US(r))
+                  : __PTR(address), "h"(__HALF_TO_CUS(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700*/
+
+#undef __PTR
+
+#undef __CUDA_FP16_DECL__
+#endif /* defined(__CUDACC__) */
+#endif /* defined(__cplusplus) */
+
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+#undef __CUDA_FP16_DECL__
+
+/* Define first-class types "half" and "half2", unless user specifies otherwise via "#define CUDA_NO_HALF" */
+/* C cannot ever have these types defined here, because __half and __half2 are C++ classes */
+#if defined(__cplusplus) && !defined(CUDA_NO_HALF)
+typedef __half half;
+typedef __half2 half2;
+#endif /* defined(__cplusplus) && !defined(CUDA_NO_HALF) */
+
+#endif /* end of include guard: __CUDA_FP16_HPP__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-11.0/cuda_fp16.h b/cupy/_core/include/cupy/_cuda/cuda-11.0/cuda_fp16.h
new file mode 100755
index 0000000..4729425
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-11.0/cuda_fp16.h
@@ -0,0 +1,3612 @@
+/*
+* Copyright 1993-2020 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+/**
+* \defgroup CUDA_MATH_INTRINSIC_HALF Half Precision Intrinsics
+* This section describes half precision intrinsic functions that are
+* only supported in device code.
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_ARITHMETIC Half Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_ARITHMETIC Half2 Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_COMPARISON Half Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_COMPARISON Half2 Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_MISC Half Precision Conversion And Data Movement
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_FUNCTIONS Half Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_FUNCTIONS Half2 Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+#ifndef __CUDA_FP16_H__
+#define __CUDA_FP16_H__
+
+#if defined(__cplusplus)
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__CUDACC__) */
+
+#define __CUDA_FP16_TYPES_EXIST__
+
+/* Forward-declaration of structures defined in "cuda_fp16.hpp" */
+
+/**
+ * \brief half datatype 
+ * 
+ * \details This structure implements the datatype for storing 
+ * half-precision floating-point numbers. The structure implements 
+ * assignment operators and type conversions. 
+ * 16 bits are being used in total: 1 sign bit, 5 bits for the exponent, 
+ * and the significand is being stored in 10 bits. 
+ * The total precision is 11 bits. There are 15361 representable 
+ * numbers within the interval [0.0, 1.0], endpoints included. 
+ * On average we have log10(2**11) ~ 3.311 decimal digits. 
+ * 
+ * \internal
+ * \req IEEE 754-2008 compliant implementation of half-precision 
+ * floating-point numbers. 
+ * \endinternal
+ */
+struct __half;
+
+/**
+ * \brief half2 datatype
+ * 
+ * \details This structure implements the datatype for storing two 
+ * half-precision floating-point numbers. 
+ * The structure implements assignment operators and type conversions. 
+ * 
+ * \internal
+ * \req Vectorified version of half. 
+ * \endinternal
+ */
+struct __half2;
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts double number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts double number \p a to half precision in round-to-nearest-even mode.
+* \param[in] a - double. Is only being read.
+* \returns half
+* \retval \p a converted to half.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __double2half(const double a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value. 
+* 
+* \details Converts float number \p a to half precision in round-to-nearest-even mode. 
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts float number \p a to half precision in round-to-nearest-even mode.
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-towards-zero mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-towards-zero mode.
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-down mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-down mode.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-up mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-up mode.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts \p half number to float.
+* 
+* \details Converts half number \p a to float.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns float
+* \retval \p a converted to float. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts input to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+*
+* \details Converts input \p a to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+* \param[in] a - float. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 value with both halves equal to the converted half
+* precision number.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both input floats to half precision in round-to-nearest-even
+* mode and returns \p half2 with converted values.
+*
+* \details Converts both input floats to half precision in round-to-nearest-even mode
+* and combines the results into one \p half2 number. Low 16 bits of the return
+* value correspond to the input \p a, high 16 bits correspond to the input \p
+* b.
+* \param[in] a - float. Is only being read. 
+* \param[in] b - float. Is only being read. 
+* 
+* \returns half2
+* \retval The \p half2 value with corresponding halves equal to the
+* converted input floats.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts low 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts low 16 bits of \p half2 input \p a to 32 bit floating point number
+* and returns the result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* \retval The low 16 bits of \p a converted to float.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts high 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts high 16 bits of \p half2 input \p a to 32 bit floating point number
+* and returns the result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* \retval The high 16 bits of \p a converted to float.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a);
+
+#if defined(__CUDACC__)
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both components of float2 number to half precision in
+* round-to-nearest-even mode and returns \p half2 with converted values.
+* 
+* \details Converts both components of float2 to half precision in round-to-nearest
+* mode and combines the results into one \p half2 number. Low 16 bits of the
+* return value correspond to \p a.x and high 16 bits of the return value
+* correspond to \p a.y.
+* \param[in] a - float2. Is only being read. 
+*  
+* \returns half2
+* \retval The \p half2 which has corresponding halves equal to the
+* converted float2 components.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both halves of \p half2 to float2 and returns the result.
+* 
+* \details Converts both halves of \p half2 input \p a to float2 and returns the
+* result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float2
+* \retval \p a converted to float2.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-to-nearest-even mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rn(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-towards-zero mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rz(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-down mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-down mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rd(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-up mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-up mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_ru(int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rn(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rz(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-down mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rd(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-up mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_ru(short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-down mode.
+*
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-down mode.
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-up mode.
+*
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-up mode.
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-to-nearest-even mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rn(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-towards-zero mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-towards-zero mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half.  
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rz(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-down mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-down mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rd(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-up mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-up mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_ru(unsigned int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-towards-zero
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rn(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rz(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-down mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rd(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-up mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_ru(unsigned short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-towards-zero
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rn(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rz(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half.  
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rd(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_ru(unsigned long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rn(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rz(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rd(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_ru(long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Truncate input argument to the integral part.
+* 
+* \details Round \p h to the nearest integer value that does not exceed \p h in
+* magnitude.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The truncated integer value. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half htrunc(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate ceiling of the input argument.
+* 
+* \details Compute the smallest integer value not less than \p h.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The smallest integer value not less than \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hceil(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details Calculate the largest integer value which is less than or equal to \p h.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The largest integer value which is less than or equal to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hfloor(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating point
+* number.
+* 
+* \details Round \p h to the nearest integer value in half-precision floating point
+* format, with halfway cases rounded to the nearest even integer value.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The nearest integer to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrint(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Truncate \p half2 vector input argument to the integral part.
+* 
+* \details Round each component of vector \p h to the nearest integer value that does
+* not exceed \p h in magnitude.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The truncated \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate \p half2 vector ceiling of the input argument.
+* 
+* \details For each component of vector \p h compute the smallest integer value not less
+* than \p h.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of smallest integers not less than \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details For each component of vector \p h calculate the largest integer value which
+* is less than or equal to \p h.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of largest integers which is less than or equal to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating point
+* number.
+* 
+* \details Round each component of \p half2 vector \p h to the nearest integer value in
+* half-precision floating point format, with halfway cases rounded to the
+* nearest even integer value.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of rounded integer values. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns \p half2 with both halves equal to the input value.
+* 
+* \details Returns \p half2 number with both halves equal to the input \p a \p half
+* number.
+* \param[in] a - half. Is only being read. 
+* 
+* \returns half2
+* \retval The vector which has both its halves equal to the input \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Swaps both halves of the \p half2 input.
+* 
+* \details Swaps both halves of the \p half2 input and returns a new \p half2 number
+* with swapped halves.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval \p a with its halves being swapped. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from each of the two \p half2 inputs and combines
+* into one \p half2 number. 
+* 
+* \details Extracts low 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. Low 16 bits from input \p a is stored in low 16 bits of
+* the return value, low 16 bits from input \p b is stored in high 16 bits of
+* the return value. 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The low 16 bits of \p a and of \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from each of the two \p half2 inputs and
+* combines into one \p half2 number.
+* 
+* \details Extracts high 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. High 16 bits from input \p a is stored in low 16 bits of
+* the return value, high 16 bits from input \p b is stored in high 16 bits of
+* the return value.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The high 16 bits of \p a and of \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns high 16 bits of \p half2 input.
+*
+* \details Returns high 16 bits of \p half2 input \p a.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* \retval The high 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __high2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns low 16 bits of \p half2 input.
+*
+* \details Returns low 16 bits of \p half2 input \p a.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* \retval Returns \p half which contains low 16 bits of the input \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __low2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Checks if the input \p half number is infinite.
+* 
+* \details Checks if the input \p half number \p a is infinite. 
+* \param[in] a - half. Is only being read. 
+* 
+* \returns int 
+* \retval -1 iff \p a is equal to negative infinity, 
+* \retval 1 iff \p a is equal to positive infinity, 
+* \retval 0 otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __hisinf(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Combines two \p half numbers into one \p half2 number.
+* 
+* \details Combines two input \p half number \p a and \p b into one \p half2 number.
+* Input \p a is stored in low 16 bits of the return value, input \p b is stored
+* in high 16 bits of the return value.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with one half equal to \p a and the other to \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from \p half2 input.
+* 
+* \details Extracts low 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with both halves equal to the low 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from \p half2 input.
+* 
+* \details Extracts high 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with both halves equal to the high 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 a);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as a signed short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating point number \p h
+* as a signed short integer. 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval The reinterpreted value. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as an unsigned short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating point \p h
+* as an unsigned short number.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a signed short integer as a \p half.
+* 
+* \details Reinterprets the bits in the signed short integer \p i as a
+* half-precision floating point number.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in an unsigned short integer as a \p half.
+* 
+* \details Reinterprets the bits in the unsigned short integer \p i as a
+* half-precision floating point number.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i);
+
+#if __CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)
+#if !defined warpSize && !defined __local_warpSize
+#define warpSize    32
+#define __local_warpSize
+#endif
+
+#if defined(_WIN32)
+# define __DEPRECATED__(msg) __declspec(deprecated(msg))
+#elif (defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 5 && !defined(__clang__))))
+# define __DEPRECATED__(msg) __attribute__((deprecated))
+#else
+# define __DEPRECATED__(msg) __attribute__((deprecated(msg)))
+#endif
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+#define __WSB_DEPRECATION_MESSAGE(x) #x"() is deprecated in favor of "#x"_sync() and may be removed in a future release (Use -Wno-deprecated-declarations to suppress this warning)."
+
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half2 __shfl(__half2 var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half2 __shfl_up(__half2 var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down))__half2 __shfl_down(__half2 var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half2 __shfl_xor(__half2 var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half __shfl(__half var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half __shfl_up(__half var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down)) __half __shfl_down(__half var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half __shfl_xor(__half var, int delta, int width = warpSize);
+#endif
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* ithin the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_sync(unsigned mask, __half2 var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(unsigned mask, __half2 var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(unsigned mask, __half2 var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(unsigned mask, __half2 var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* ithin the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_sync(unsigned mask, __half var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_up_sync(unsigned mask, __half var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_down_sync(unsigned mask, __half var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(unsigned mask, __half var, int delta, int width = warpSize);
+
+#if defined(__local_warpSize)
+#undef warpSize
+#undef __local_warpSize
+#endif
+#endif /*__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__) */
+
+#if defined(__cplusplus) && ( __CUDA_ARCH__ >=320 || !defined(__CUDA_ARCH__) )
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.nc` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.nc` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldg(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cg` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cg` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcg(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.ca` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.ca` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldca(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cs` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cs` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcs(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.lu` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldlu(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.lu` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldlu(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cv` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcv(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cv` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcv(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wb` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwb(__half2 *ptr, __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wb` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwb(__half *ptr, __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cg` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcg(__half2 *ptr, __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cg` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcg(__half *ptr, __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cs` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcs(__half2 *ptr, __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cs` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcs(__half *ptr, __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wt` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwt(__half2 *ptr, __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wt` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwt(__half *ptr, __half value);
+#endif /*defined(__cplusplus) && ( __CUDA_ARCH__ >=320 || !defined(__CUDA_ARCH__) )*/
+
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs half2 vector if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison.
+* 
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 result of less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The half2 vector result of less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison.
+* 
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of unordered if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison.
+*
+* Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 vector result of unordered greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 vector result of unordered greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Determine whether \p half2 argument is a NaN.
+*
+* \details Determine whether each half of input \p half2 number \p a is a NaN.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The half2 with the corresponding \p half results set to
+* 1.0 for for NaN, 0.0 otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-95
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The sum of vectors \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-104
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The subtraction of vector \p b from \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-102
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise multiplying the vectors \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector division in round-to-nearest-even mode.
+*
+* \details Divides \p half2 input vector \p a by input vector \p b in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-103
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise division of \p a with \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __h2div(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+*
+* \details Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval Returns \p a with the absolute value of both halves. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the results to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The sum of \p a and \p b, with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The subtraction of vector \p b from \p a, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise multiplication of vectors \p a and \p b, 
+* with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-105
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode, with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the
+* results to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c, 
+* with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Negates both halves of the input \p half2 number and returns the
+* result.
+*
+* \details Negates both halves of the input \p half2 number \p a and returns the result.
+* \internal
+* \req DEEPLEARN-SRM_REQ-101
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval Returns \p a with both halves negated. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Calculates the absolute value of input \p half number and returns the result.
+*
+* \details Calculates the absolute value of input \p half number and returns the result.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The absolute value of a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __habs(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode.
+*
+* \details Performs \p half addition of inputs \p a and \p b, in round-to-nearest-even
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-94
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The sum of \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-97
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of subtracting \p b from \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-99
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of multiplying \p a and \p b. 
+*/
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half division in round-to-nearest-even mode.
+* 
+* \details Divides \p half input \p a by input \p b in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-98
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half
+* \retval The result of dividing \p a by \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__  __half __hdiv(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half add of inputs \p a and \p b, in round-to-nearest-even mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The sum of \p a and \p b, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of subtraction of \p b from \p a, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the result to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of multiplying \p a and \p b, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-96
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* \retval The result of fused multiply-add operation on \p
+* a, \p b, and \p c. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the result
+* to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* \retval The result of fused multiply-add operation on \p
+* a, \p b, and \p c, with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Negates input \p half number and returns the result.
+*
+* \details Negates input \p half number and returns the result.
+* \internal
+* \req DEEPLEARN-SRM_REQ-100
+* \endinternal
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval minus a
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hneg(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector if-equal comparison, and returns boolean true
+* iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of if-equal comparison
+* of vectors \p a and \p b are true;
+* \retval false, otherwise.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of not-equal comparison
+* of vectors \p a and \p b are true, 
+* \retval false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of less-equal comparison
+* of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of greater-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of less-than comparison
+* of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns bool 
+* \retval true, if both \p half results of greater-than
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered if-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered not-equal
+* comparison of vectors \p a and \p b are true;
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered less-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison, and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered
+* greater-equal comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered less-than comparison of 
+* vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison, and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered
+* greater-than comparison of vectors \p a and \p b are true;
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of if-equal comparison of \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of not-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of less-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of greater-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of less-than comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of greater-than comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered if-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered not-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered less-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered greater-equal comparison of \p a
+* and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered less-than comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered greater-than comparison of \p a
+* and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Determine whether \p half argument is a NaN.
+*
+* \details Determine whether \p half value \p a is a NaN.
+* \param[in] a - half. Is only being read. 
+*
+* \returns bool
+* \retval true iff argument is NaN. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hisnan(const __half a);
+#if __CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half maximum of two input values.
+*
+* \details Calculates \p half max(\p a, \p b)
+* defined as (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmax(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half minimum of two input values.
+*
+* \details Calculates \p half min(\p a, \p b)
+* defined as (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmin(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half maximum of two input values, NaNs pass through.
+*
+* \details Calculates \p half max(\p a, \p b)
+* defined as (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmax_nan(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half minimum of two input values, NaNs pass through.
+*
+* \details Calculates \p half min(\p a, \p b)
+* defined as (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmin_nan(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode with relu saturation.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* Then negative result is clamped to 0.
+* NaN result is converted to canonical NaN.
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+* \param[in] c - half. Is only being read.
+*
+* \returns half
+* \retval The result of fused multiply-add operation on \p
+* a, \p b, and \p c with relu saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma_relu(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector maximum of two inputs.
+*
+* \details Calculates \p half2 vector max(\p a, \p b)
+* Elementwise \p half operation is defined as
+* (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise maximum of vectors \p a  and \p b
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmax2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector minimum of two inputs.
+*
+* \details Calculates \p half2 vector min(\p a, \p b)
+* Elementwise \p half operation is defined as
+* (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise minimum of vectors \p a  and \p b
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmin2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector maximum of two inputs, NaNs pass through.
+*
+* \details Calculates \p half2 vector max(\p a, \p b)
+* Elementwise \p half operation is defined as
+* (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise maximum of vectors \p a  and \p b, with NaNs pass through
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmax2_nan(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector minimum of two inputs, NaNs pass through.
+*
+* \details Calculates \p half2 vector min(\p a, \p b)
+* Elementwise \p half operation is defined as
+* (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise minimum of vectors \p a  and \p b, with NaNs pass through
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmin2_nan(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode with relu saturation.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* Then negative result is clamped to 0.
+* NaN result is converted to canonical NaN.
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+* \param[in] c - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c with relu saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_relu(const __half2 a, const __half2 b, const __half2 c);
+#endif /*__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)*/
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half square root of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The square root of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal square root in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half reciprocal square root of input \p a in round-to-nearest
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The reciprocal square root of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half reciprocal of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The reciprocal of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrcp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half natural logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The natural logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half binary logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The binary logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half decimal logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The decimal logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half natural exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The natural exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half binary exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The binary exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half decimal exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The decimal exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half cosine in round-to-nearest-even mode.
+*
+* \details Calculates \p half cosine of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The cosine of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hcos(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half sine in round-to-nearest-even mode.
+*
+* \details Calculates \p half sine of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The sine of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hsin(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 square root of input vector \p a in round-to-nearest
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise square root on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal square root in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 reciprocal square root of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise reciprocal square root on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 reciprocal of input vector \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise reciprocal on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector natural logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 natural logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise natural logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 binary logarithm of input vector \p a in round-to-nearest
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise binary logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 decimal logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise decimal logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary exponential function in
+* round-to-nearest-even mode.
+*
+* \details Calculates \p half2 binary exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise binary exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal exponential function in
+* round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 decimal exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise decimal exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector cosine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 cosine of input vector \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise cosine on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector sine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 sine of input vector \p a in round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise sine on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a);
+
+#endif /*if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)*/
+
+#if __CUDA_ARCH__ >= 600 || !defined(__CUDA_ARCH__)
+
+__CUDA_FP16_DECL__ __half2 atomicAdd(__half2 *address, __half2 val);
+
+#endif /*if __CUDA_ARCH__ >= 600 || !defined(__CUDA_ARCH__)*/
+
+#if __CUDA_ARCH__ >= 700 || !defined(__CUDA_ARCH__)
+
+__CUDA_FP16_DECL__ __half atomicAdd(__half *address, __half val);
+
+#endif /*if __CUDA_ARCH__ >= 700 || !defined(__CUDA_ARCH__)*/
+
+#endif /* defined(__CUDACC__) */
+
+#undef __CUDA_FP16_DECL__
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+
+#endif /* defined(__cplusplus) */
+
+/* Note the .hpp file is included even for host-side compilation, to capture the "half" & "half2" definitions */
+#include "cuda_fp16.hpp"
+
+#endif /* end of include guard: __CUDA_FP16_H__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-11.0/cuda_fp16.hpp b/cupy/_core/include/cupy/_cuda/cuda-11.0/cuda_fp16.hpp
new file mode 100755
index 0000000..aeaa36b
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-11.0/cuda_fp16.hpp
@@ -0,0 +1,2285 @@
+/*
+* Copyright 1993-2020 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+#if !defined(__CUDA_FP16_HPP__)
+#define __CUDA_FP16_HPP__
+
+#if !defined(__CUDA_FP16_H__)
+#error "Do not include this file directly. Instead, include cuda_fp16.h."
+#endif
+
+#if !defined(_MSC_VER) && __cplusplus >= 201103L
+#   define __CPP_VERSION_AT_LEAST_11_FP16
+#elif _MSC_FULL_VER >= 190024210 && _MSVC_LANG >= 201103L
+#   define __CPP_VERSION_AT_LEAST_11_FP16
+#endif
+
+/* C++11 header for std::move. 
+ * In RTC mode, std::move is provided implicitly; don't include the header
+ */
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16) && !defined(__CUDACC_RTC__)
+#include <utility>
+#endif /* __cplusplus >= 201103L && !defined(__CUDACC_RTC__) */
+
+/* C++ header for std::memcpy (used for type punning in host-side implementations).
+ * When compiling as a CUDA source file memcpy is provided implicitly.
+ * !defined(__CUDACC__) implies !defined(__CUDACC_RTC__).
+ */
+#if defined(__cplusplus) && !defined(__CUDACC__)
+#include <cstring>
+#endif /* defined(__cplusplus) && !defined(__CUDACC__) */
+
+
+/* Set up function decorations */
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#define __VECTOR_FUNCTIONS_DECL__ static __inline__ __host__ __device__
+#define __CUDA_HOSTDEVICE__ __host__ __device__
+#else /* !defined(__CUDACC__) */
+#if defined(__GNUC__)
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __attribute__ ((unused))
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__GNUC__) */
+#define __CUDA_HOSTDEVICE__
+#endif /* defined(__CUDACC_) */
+
+/* Set up structure-alignment attribute */
+#if defined(__CUDACC__)
+#define __CUDA_ALIGN__(align) __align__(align)
+#else
+/* Define alignment macro based on compiler type (cannot assume C11 "_Alignas" is available) */
+#if __cplusplus >= 201103L
+#define __CUDA_ALIGN__(n) alignas(n)    /* C++11 kindly gives us a keyword for this */
+#else /* !defined(__CPP_VERSION_AT_LEAST_11_FP16)*/
+#if defined(__GNUC__)
+#define __CUDA_ALIGN__(n) __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+#define __CUDA_ALIGN__(n) __declspec(align(n))
+#else
+#define __CUDA_ALIGN__(n)
+#endif /* defined(__GNUC__) */
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+#endif /* defined(__CUDACC__) */
+
+/* Macros to allow half & half2 to be used by inline assembly */
+#define __HALF_TO_US(var) *(reinterpret_cast<unsigned short *>(&(var)))
+#define __HALF_TO_CUS(var) *(reinterpret_cast<const unsigned short *>(&(var)))
+#define __HALF2_TO_UI(var) *(reinterpret_cast<unsigned int *>(&(var)))
+#define __HALF2_TO_CUI(var) *(reinterpret_cast<const unsigned int *>(&(var)))
+
+/* Macros for half & half2 binary arithmetic */
+#define __BINARY_OP_HALF_MACRO(name) /* do */ {\
+   __half val; \
+   asm( "{"#name".f16 %0,%1,%2;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b))); \
+   return val; \
+} /* while(0) */
+#define __BINARY_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{"#name".f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+#define __TERNARY_OP_HALF_MACRO(name) /* do */ {\
+   __half val; \
+   asm( "{"#name".f16 %0,%1,%2,%3;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b)),"h"(__HALF_TO_CUS(c))); \
+   return val; \
+} /* while(0) */
+#define __TERNARY_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{"#name".f16x2 %0,%1,%2,%3;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b)),"r"(__HALF2_TO_CUI(c))); \
+   return val; \
+} /* while(0) */
+
+/**
+* Types which allow static initialization of "half" and "half2" until
+* these become an actual builtin. Note this initialization is as a
+* bitfield representation of "half", and not a conversion from short->half.
+* Such a representation will be deprecated in a future version of CUDA. 
+* (Note these are visible to non-nvcc compilers, including C-only compilation)
+*/
+typedef struct __CUDA_ALIGN__(2) {
+    unsigned short x;
+} __half_raw;
+
+typedef struct __CUDA_ALIGN__(4) {
+    unsigned short x;
+    unsigned short y;
+} __half2_raw;
+
+/* All other definitions in this file are only visible to C++ compilers */
+#if defined(__cplusplus)
+
+/* Hide GCC member initialization list warnings because of host/device in-function init requirement */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstrict-aliasing"
+#pragma GCC diagnostic ignored "-Weffc++"
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+/* class' : multiple assignment operators specified
+   The class has multiple assignment operators of a single type. This warning is informational */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( push )
+#pragma warning( disable:4522 )
+#endif /* defined(__GNUC__) */
+
+struct __CUDA_ALIGN__(2) __half {
+protected:
+    unsigned short __x;
+
+public:
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+    __half() = default;
+#else
+    __CUDA_HOSTDEVICE__ __half() { }
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+
+    /* Convert to/from __half_raw */
+    __CUDA_HOSTDEVICE__ __half(const __half_raw &hr) : __x(hr.x) { }
+    __CUDA_HOSTDEVICE__ __half &operator=(const __half_raw &hr) { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const volatile __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const { __half_raw ret; ret.x = __x; return ret; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const volatile { __half_raw ret; ret.x = __x; return ret; }
+
+#if !defined(__CUDA_NO_HALF_CONVERSIONS__)
+
+    /* Construct from float/double */
+    __CUDA_HOSTDEVICE__ __half(const float f) { __x = __float2half(f).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const double f) { __x = __double2half(f).__x;  }
+
+    __CUDA_HOSTDEVICE__ operator float() const { return __half2float(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const float f) { __x = __float2half(f).__x; return *this; }
+
+    /* We omit "cast to double" operator, so as to not be ambiguous about up-cast */
+    __CUDA_HOSTDEVICE__ __half &operator=(const double f) { __x = __double2half(f).__x; return *this; }
+
+/* Member functions only available to nvcc compilation so far */
+#if defined(__CUDACC__)
+    /* Allow automatic construction from types supported natively in hardware */
+    /* Note we do avoid constructor init-list because of special host/device compilation rules */
+    __device__ __half(short val) { __x = __short2half_rn(val).__x;  }
+    __device__ __half(unsigned short val) { __x = __ushort2half_rn(val).__x;  }
+    __device__ __half(int val) { __x = __int2half_rn(val).__x;  }
+    __device__ __half(unsigned int val) { __x = __uint2half_rn(val).__x;  }
+    __device__ __half(long long val) { __x = __ll2half_rn(val).__x;  }
+    __device__ __half(unsigned long long val) { __x = __ull2half_rn(val).__x; }
+
+    /* Allow automatic casts to supported builtin types, matching all that are permitted with float */
+    __device__ operator short() const { return __half2short_rz(*this); }
+    __device__ __half &operator=(short val) { __x = __short2half_rn(val).__x; return *this; }
+
+    __device__ operator unsigned short() const { return __half2ushort_rz(*this); }
+    __device__ __half &operator=(unsigned short val) { __x = __ushort2half_rn(val).__x; return *this; }
+
+    __device__ operator int() const { return __half2int_rz(*this); }
+    __device__ __half &operator=(int val) { __x = __int2half_rn(val).__x; return *this; }
+
+    __device__ operator unsigned int() const { return __half2uint_rz(*this); }
+    __device__ __half &operator=(unsigned int val) { __x = __uint2half_rn(val).__x; return *this; }
+
+    __device__ operator long long() const { return __half2ll_rz(*this); }
+    __device__ __half &operator=(long long val) { __x = __ll2half_rn(val).__x; return *this; }
+
+    __device__ operator unsigned long long() const { return __half2ull_rz(*this); }
+    __device__ __half &operator=(unsigned long long val) { __x = __ull2half_rn(val).__x; return *this; }
+
+    /* Boolean conversion - note both 0 and -0 must return false */
+    __device__ operator bool() const { return (__x & 0x7FFF) != 0; }
+#endif /* defined(__CUDACC__) */
+#endif /* !defined(__CUDA_NO_HALF_CONVERSIONS__) */
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16 operations only supported on arch >= 5.3 */
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+#if !defined(__CUDA_NO_HALF_OPERATORS__)
+/* Some basic arithmetic operations expected of a builtin */
+__device__ __forceinline__ __half operator+(const __half &lh, const __half &rh) { return __hadd(lh, rh); }
+__device__ __forceinline__ __half operator-(const __half &lh, const __half &rh) { return __hsub(lh, rh); }
+__device__ __forceinline__ __half operator*(const __half &lh, const __half &rh) { return __hmul(lh, rh); }
+__device__ __forceinline__ __half operator/(const __half &lh, const __half &rh) { return __hdiv(lh, rh); }
+
+__device__ __forceinline__ __half &operator+=(__half &lh, const __half &rh) { lh = __hadd(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator-=(__half &lh, const __half &rh) { lh = __hsub(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator*=(__half &lh, const __half &rh) { lh = __hmul(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator/=(__half &lh, const __half &rh) { lh = __hdiv(lh, rh); return lh; }
+
+/* Note for increment and decrement we use the raw value 0x3C00 equating to half(1.0f), to avoid the extra conversion */
+__device__ __forceinline__ __half &operator++(__half &h)      { __half_raw one; one.x = 0x3C00; h += one; return h; }
+__device__ __forceinline__ __half &operator--(__half &h)      { __half_raw one; one.x = 0x3C00; h -= one; return h; }
+__device__ __forceinline__ __half  operator++(__half &h, int) { __half ret = h; __half_raw one; one.x = 0x3C00; h += one; return ret; }
+__device__ __forceinline__ __half  operator--(__half &h, int) { __half ret = h; __half_raw one; one.x = 0x3C00; h -= one; return ret; }
+
+/* Unary plus and inverse operators */
+__device__ __forceinline__ __half operator+(const __half &h) { return h; }
+__device__ __forceinline__ __half operator-(const __half &h) { return __hneg(h); }
+
+/* Some basic comparison operations to make it look like a builtin */
+__device__ __forceinline__ bool operator==(const __half &lh, const __half &rh) { return __heq(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half &lh, const __half &rh) { return __hneu(lh, rh); }
+__device__ __forceinline__ bool operator> (const __half &lh, const __half &rh) { return __hgt(lh, rh); }
+__device__ __forceinline__ bool operator< (const __half &lh, const __half &rh) { return __hlt(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half &lh, const __half &rh) { return __hge(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half &lh, const __half &rh) { return __hle(lh, rh); }
+#endif /* !defined(__CUDA_NO_HALF_OPERATORS__) */
+#endif /* __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__) */
+#endif /* defined(__CUDACC__) */
+
+/* __half2 is visible to non-nvcc host compilers */
+struct __CUDA_ALIGN__(4) __half2 {
+    __half x;
+    __half y;
+
+    // All construct/copy/assign/move
+public:
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+    __half2() = default;
+    __CUDA_HOSTDEVICE__ __half2(__half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(__half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); return *this; }
+#else
+    __CUDA_HOSTDEVICE__ __half2() { }
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+    __CUDA_HOSTDEVICE__ __half2(const __half &a, const __half &b) : x(a), y(b) { }
+    __CUDA_HOSTDEVICE__ __half2(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); return *this; }
+
+    /* Convert to/from __half2_raw */
+    __CUDA_HOSTDEVICE__ __half2(const __half2_raw &h2r ) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2_raw &h2r) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); return *this; }
+    __CUDA_HOSTDEVICE__ operator __half2_raw() const { __half2_raw ret; __HALF2_TO_UI(ret) = __HALF2_TO_CUI(*this); return ret; }
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16x2 operations only supported on arch >= 5.3 */
+#if (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)) && !defined(__CUDA_NO_HALF2_OPERATORS__)
+
+__device__ __forceinline__ __half2 operator+(const __half2 &lh, const __half2 &rh) { return __hadd2(lh, rh); }
+__device__ __forceinline__ __half2 operator-(const __half2 &lh, const __half2 &rh) { return __hsub2(lh, rh); }
+__device__ __forceinline__ __half2 operator*(const __half2 &lh, const __half2 &rh) { return __hmul2(lh, rh); }
+__device__ __forceinline__ __half2 operator/(const __half2 &lh, const __half2 &rh) { return __h2div(lh, rh); }
+
+__device__ __forceinline__ __half2& operator+=(__half2 &lh, const __half2 &rh) { lh = __hadd2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator-=(__half2 &lh, const __half2 &rh) { lh = __hsub2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator*=(__half2 &lh, const __half2 &rh) { lh = __hmul2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator/=(__half2 &lh, const __half2 &rh) { lh = __h2div(lh, rh); return lh; }
+
+__device__ __forceinline__ __half2 &operator++(__half2 &h)      { __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hadd2(h, one); return h; }
+__device__ __forceinline__ __half2 &operator--(__half2 &h)      { __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hsub2(h, one); return h; }
+__device__ __forceinline__ __half2  operator++(__half2 &h, int) { __half2 ret = h; __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hadd2(h, one); return ret; }
+__device__ __forceinline__ __half2  operator--(__half2 &h, int) { __half2 ret = h; __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hsub2(h, one); return ret; }
+
+__device__ __forceinline__ __half2 operator+(const __half2 &h) { return h; }
+__device__ __forceinline__ __half2 operator-(const __half2 &h) { return __hneg2(h); }
+
+__device__ __forceinline__ bool operator==(const __half2 &lh, const __half2 &rh) { return __hbeq2(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half2 &lh, const __half2 &rh) { return __hbneu2(lh, rh); }
+__device__ __forceinline__ bool operator>(const __half2 &lh, const __half2 &rh) { return __hbgt2(lh, rh); }
+__device__ __forceinline__ bool operator<(const __half2 &lh, const __half2 &rh) { return __hblt2(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half2 &lh, const __half2 &rh) { return __hbge2(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half2 &lh, const __half2 &rh) { return __hble2(lh, rh); }
+
+#endif /* __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__) */
+#endif /* defined(__CUDACC__) */
+
+/* Restore warning for multiple assignment operators */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( pop )
+#endif /* defined(_MSC_VER) && _MSC_VER >= 1500 */
+
+/* Restore -Weffc++ warnings from here on */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic pop
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+#undef __CUDA_HOSTDEVICE__
+#undef __CUDA_ALIGN__
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static unsigned short __internal_float2half(const float f, unsigned int &sign, unsigned int &remainder)
+{
+    unsigned int x;
+    unsigned int u;
+    unsigned int result = 0U;
+#if defined(__CUDACC__)
+    (void)memcpy(&x, &f, sizeof(f));
+#else
+    (void)std::memcpy(&x, &f, sizeof(f));
+#endif
+    u = (x & 0x7fffffffU);
+    sign = ((x >> 16U) & 0x8000U);
+    // NaN/+Inf/-Inf
+    if (u >= 0x7f800000U) {
+        remainder = 0U;
+        result = ((u == 0x7f800000U) ? (sign | 0x7c00U) : 0x7fffU);
+    } else if (u > 0x477fefffU) { // Overflows
+        remainder = 0x80000000U;
+        result = (sign | 0x7bffU);
+    } else if (u >= 0x38800000U) { // Normal numbers
+        remainder = u << 19U;
+        u -= 0x38000000U;
+        result = (sign | (u >> 13U));
+    } else if (u < 0x33000001U) { // +0/-0
+        remainder = u;
+        result = sign;
+    } else { // Denormal numbers
+        const unsigned int exponent = u >> 23U;
+        const unsigned int shift = 0x7eU - exponent;
+        unsigned int mantissa = (u & 0x7fffffU);
+        mantissa |= 0x800000U;
+        remainder = mantissa << (32U - shift);
+        result = (sign | (mantissa >> shift));
+    }
+    return static_cast<unsigned short>(result);
+}
+#endif  /* #if !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __double2half(const double x)
+{
+#if defined(__CUDA_ARCH__)
+    __half val;
+    asm("{  cvt.rn.f16.f64 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "d"(x));
+    return val;
+#else
+    // Perform rounding to 11 bits of precision, convert value
+    // to float and call existing float to half conversion.
+    // By pre-rounding to 11 bits we avoid additional rounding
+    // in float to half conversion.
+    unsigned long long int absx;
+    unsigned long long int ux;
+#if defined(__CUDACC__)
+    (void)memcpy(&ux, &x, sizeof(x));
+#else
+    (void)std::memcpy(&ux, &x, sizeof(x));
+#endif
+    absx = (ux & 0x7fffffffffffffffull);
+    if ((absx >= 0x40f0000000000000ull) || (absx <= 0x3e60000000000000ull))
+    {
+        // |x| >= 2^16 or NaN or |x| <= 2^(-25)
+        // double-rounding is not a problem
+        return __float2half(static_cast<float>(x));
+    }
+
+    // here 2^(-25) < |x| < 2^16
+    // prepare shifter value such that x + shifter
+    // done in double precision performs round-to-nearest-even
+    // and (x + shifter) - shifter results in x rounded to
+    // 11 bits of precision. Shifter needs to have exponent of
+    // x plus 53 - 11 = 42 and a leading bit in mantissa to guard
+    // against negative values.
+    // So need to have |x| capped to avoid overflow in exponent.
+    // For inputs that are smaller than half precision minnorm
+    // we prepare fixed shifter exponent.
+    unsigned long long shifterBits = ux & 0x7ff0000000000000ull;
+    if (absx >= 0x3f10000000000000ull)
+    {   // |x| >= 2^(-14)
+        // add 42 to exponent bits
+        shifterBits += 42ull << 52;
+    }
+    else
+    {   // 2^(-25) < |x| < 2^(-14), potentially results in denormal
+        // set exponent bits to 42 - 14 + bias
+        shifterBits = ((42ull - 14 + 1023) << 52);
+    }
+    // set leading mantissa bit to protect against negative inputs
+    shifterBits |= 1ull << 51;
+    double shifter;
+#if defined(__CUDACC__)
+    (void)memcpy(&shifter, &shifterBits, sizeof(shifterBits));
+#else
+    (void)std::memcpy(&shifter, &shifterBits, sizeof(shifterBits));
+#endif
+    double xShiftRound = x + shifter;
+
+    // Prevent the compiler from optimizing away x + shifter - shifter
+    // by doing intermediate memcopy and harmless bitwize operation
+    unsigned long long int xShiftRoundBits;
+#if defined(__CUDACC__)
+    (void)memcpy(&xShiftRoundBits, &xShiftRound, sizeof(xShiftRound));
+#else
+    (void)std::memcpy(&xShiftRoundBits, &xShiftRound, sizeof(xShiftRound));
+#endif
+
+    // the value is positive, so this operation doesn't change anything
+    xShiftRoundBits &= 0x7fffffffffffffffull;
+
+#if defined(__CUDACC__)
+    (void)memcpy(&xShiftRound, &xShiftRoundBits, sizeof(xShiftRound));
+#else
+    (void)std::memcpy(&xShiftRound, &xShiftRoundBits, sizeof(xShiftRound));
+#endif
+
+    double xRounded = xShiftRound - shifter;
+    float  xRndFlt = static_cast<float>(xRounded);
+    __half res =  __float2half(xRndFlt);
+    return res;
+#endif
+}
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rz.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rm.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign != 0U)) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rp.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign == 0U)) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a)
+{
+    __half2 val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a));
+#else
+    val = __half2(__float2half_rn(a), __float2half_rn(a));
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b)
+{
+    __half2 val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  cvt.rn.f16.f32 high, %2;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a), "f"(b));
+#else
+    val = __half2(__float2half_rn(a), __float2half_rn(b));
+#endif
+    return val;
+}
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static float __internal_half2float(const unsigned short h)
+{
+    unsigned int sign = ((static_cast<unsigned int>(h) >> 15U) & 1U);
+    unsigned int exponent = ((static_cast<unsigned int>(h) >> 10U) & 0x1fU);
+    unsigned int mantissa = ((static_cast<unsigned int>(h) & 0x3ffU) << 13U);
+    float f;
+    if (exponent == 0x1fU) { /* NaN or Inf */
+        sign = ((mantissa != 0U) ? 0U : sign);
+        mantissa = ((mantissa != 0U) ? 0x7fffffU : 0U);
+        exponent = 0xffU;
+    } else if (exponent == 0U) { /* Denorm or Zero */
+        if (mantissa != 0U) {
+            unsigned int msb;
+            exponent = 0x71U;
+            do {
+                msb = (mantissa & 0x400000U);
+                mantissa <<= 1U; /* normalize */
+                --exponent;
+            } while (msb == 0U);
+            mantissa &= 0x7fffffU; /* 1.mantissa is implicit */
+        }
+    } else {
+        exponent += 0x70U;
+    }
+    unsigned int u = ((sign << 31U) | (exponent << 23U) | mantissa);
+#if defined(__CUDACC__)
+    (void)memcpy(&f, &u, sizeof(u));
+#else
+    (void)std::memcpy(&f, &u, sizeof(u));
+#endif
+    return f;
+}
+#endif  /* !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.f32.f16 %0, %1;}\n" : "=f"(val) : "h"(__HALF_TO_CUS(a)));
+#else
+    val = __internal_half2float(static_cast<__half_raw>(a).x);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+#else
+    val = __internal_half2float(static_cast<__half2_raw>(a).x);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+#else
+    val = __internal_half2float(static_cast<__half2_raw>(a).y);
+#endif
+    return val;
+}
+
+/* Intrinsic functions only available to nvcc compilers */
+#if defined(__CUDACC__)
+
+/* CUDA vector-types compatible vector creation function (note returns __half2, not half2) */
+__VECTOR_FUNCTIONS_DECL__ __half2 make_half2(__half x, __half y)
+{
+    __half2 t; t.x = x; t.y = y; return t;
+}
+#undef __VECTOR_FUNCTIONS_DECL__
+
+
+/* Definitions of intrinsics */
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 f)
+{
+    __half2 val = __floats2half2_rn(f.x, f.y);
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 l)
+{
+    float hi_float;
+    float lo_float;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(lo_float) : "r"(__HALF2_TO_CUI(l)));
+
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(hi_float) : "r"(__HALF2_TO_CUI(l)));
+#else
+    lo_float = __internal_half2float(((__half2_raw)l).x);
+    hi_float = __internal_half2float(((__half2_raw)l).y);
+#endif
+    return make_float2(lo_float, hi_float);
+}
+__CUDA_FP16_DECL__ int __half2int_rn(__half h)
+{
+    int i;
+    asm("cvt.rni.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_rz(__half h)
+{
+    int i;
+    asm("cvt.rzi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_rd(__half h)
+{
+    int i;
+    asm("cvt.rmi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_ru(__half h)
+{
+    int i;
+    asm("cvt.rpi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __int2half_rn(int i)
+{
+    __half h;
+    asm("cvt.rn.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rz(int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rd(int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_ru(int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ short int __half2short_rn(__half h)
+{
+    short int i;
+    asm("cvt.rni.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_rz(__half h)
+{
+    short int i;
+    asm("cvt.rzi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_rd(__half h)
+{
+    short int i;
+    asm("cvt.rmi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_ru(__half h)
+{
+    short int i;
+    asm("cvt.rpi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __short2half_rn(short int i)
+{
+    __half h;
+    asm("cvt.rn.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rz(short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rd(short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_ru(short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(__half h)
+{
+    unsigned int i;
+    asm("cvt.rni.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_rz(__half h)
+{
+    unsigned int i;
+    asm("cvt.rzi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(__half h)
+{
+    unsigned int i;
+    asm("cvt.rmi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(__half h)
+{
+    unsigned int i;
+    asm("cvt.rpi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rn(unsigned int i)
+{
+    __half h;
+    asm("cvt.rn.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rz(unsigned int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rd(unsigned int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_ru(unsigned int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rni.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rz(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rzi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rmi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rpi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rn(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rn.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rz(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rd(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_ru(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rni.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rz(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rzi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rmi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rpi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rn(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rn.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rz(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rd(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_ru(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ long long int __half2ll_rn(__half h)
+{
+    long long int i;
+    asm("cvt.rni.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_rz(__half h)
+{
+    long long int i;
+    asm("cvt.rzi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_rd(__half h)
+{
+    long long int i;
+    asm("cvt.rmi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_ru(__half h)
+{
+    long long int i;
+    asm("cvt.rpi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rn(long long int i)
+{
+    __half h;
+    asm("cvt.rn.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rz(long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rd(long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_ru(long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ __half htrunc(const __half h)
+{
+    __half r;
+    asm("cvt.rzi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hceil(const __half h)
+{
+    __half r;
+    asm("cvt.rpi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hfloor(const __half h)
+{
+    __half r;
+    asm("cvt.rmi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hrint(const __half h)
+{
+    __half r;
+    asm("cvt.rni.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rzi.f16.f16 low, low;\n"
+        "  cvt.rzi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rpi.f16.f16 low, low;\n"
+        "  cvt.rpi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rmi.f16.f16 low, low;\n"
+        "  cvt.rmi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rni.f16.f16 low, low;\n"
+        "  cvt.rni.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 l, const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {alow,blow};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)), "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 l, const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {ahigh,bhigh};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)), "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __low2half(const __half2 h)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, low;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(h)));
+    return ret;
+}
+__CUDA_FP16_DECL__ int __hisinf(const __half a)
+{
+    if (__HALF_TO_CUS(a) == 0xFC00) {
+        return -1;
+    }
+    if (__HALF_TO_CUS(a) == 0x7C00) {
+        return 1;
+    }
+    return 0;
+}
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 l)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 l)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __high2half(const __half2 h)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, high;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(h)));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half l, const __half h)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%2};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(l)), "h"(__HALF_TO_CUS(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half lh)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%1};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(lh)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 lh)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(lh)));
+    return val;
+}
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h)
+{
+    return (short int)__HALF_TO_CUS(h);
+}
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h)
+{
+    return __HALF_TO_CUS(h);
+}
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = (unsigned short int)i;
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = i;
+    return h;
+}
+
+#if __CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)
+/******************************************************************************
+*                           __half, __half2 warp shuffle                     *
+******************************************************************************/
+#define __SHUFFLE_HALF2_MACRO(name) /* do */ {\
+   __half2 r; \
+   asm volatile ("{"#name" %0,%1,%2,%3;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c)); \
+   return r; \
+} /* while(0) */
+
+#define __SHUFFLE_SYNC_HALF2_MACRO(name) /* do */ {\
+   __half2 r; \
+   asm volatile ("{"#name" %0,%1,%2,%3,%4;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c), "r"(mask)); \
+   return r; \
+} /* while(0) */
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+
+__CUDA_FP16_DECL__ __half2 __shfl(__half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.idx.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up(__half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = (warpSize - width) << 8;
+    __SHUFFLE_HALF2_MACRO(shfl.up.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down(__half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.down.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor(__half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.bfly.b32)
+}
+
+#endif /* !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700 */
+
+__CUDA_FP16_DECL__ __half2 __shfl_sync(unsigned mask, __half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.idx.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(unsigned mask, __half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = (warpSize - width) << 8;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.up.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(unsigned mask, __half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.down.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(unsigned mask, __half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.bfly.b32)
+}
+
+#undef __SHUFFLE_HALF2_MACRO
+#undef __SHUFFLE_SYNC_HALF2_MACRO
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+
+__CUDA_FP16_DECL__ __half __shfl(__half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up(__half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_up(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down(__half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_down(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor(__half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_xor(temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /* !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700 */
+
+__CUDA_FP16_DECL__ __half __shfl_sync(unsigned mask, __half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up_sync(unsigned mask, __half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_up_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down_sync(unsigned mask, __half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_down_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(unsigned mask, __half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_xor_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /*__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)*/
+/******************************************************************************
+*               __half and __half2 __ldg,__ldcg,__ldca,__ldcs                *
+******************************************************************************/
+
+#if defined(__cplusplus) && (__CUDA_ARCH__ >= 320 || !defined(__CUDA_ARCH__))
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __LDG_PTR   "l"
+#else
+#define __LDG_PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.nc.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldg(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.nc.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cg.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcg(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.cg.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.ca.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldca(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.ca.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cs.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcs(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.cs.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldlu(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.lu.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldlu(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.lu.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcv(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cv.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcv(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.cv.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ void __stwb(__half2 *ptr, __half2 value)
+{
+    asm ("st.global.wb.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwb(__half *ptr, __half value)
+{
+    asm ("st.global.wb.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcg(__half2 *ptr, __half2 value)
+{
+    asm ("st.global.cg.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcg(__half *ptr, __half value)
+{
+    asm ("st.global.cg.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcs(__half2 *ptr, __half2 value)
+{
+    asm ("st.global.cs.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcs(__half *ptr, __half value)
+{
+    asm ("st.global.cs.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwt(__half2 *ptr, __half2 value)
+{
+    asm ("st.global.wt.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwt(__half *ptr, __half value)
+{
+    asm ("st.global.wt.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+#undef __LDG_PTR
+#endif /*defined(__cplusplus) && (__CUDA_ARCH__ >= 320 || !defined(__CUDA_ARCH__))*/
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+/******************************************************************************
+*                             __half2 comparison                             *
+******************************************************************************/
+#define __COMPARISON_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{ "#name".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.eq)
+}
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ne)
+}
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.le)
+}
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ge)
+}
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.lt)
+}
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gt)
+}
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.equ)
+}
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.neu)
+}
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.leu)
+}
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.geu)
+}
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ltu)
+}
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gtu)
+}
+#undef __COMPARISON_OP_HALF2_MACRO
+#define __BOOL_COMPARISON_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{ "#name".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   if (__HALF2_TO_CUI(val) == 0x3C003C00) \
+      return true; \
+   else  \
+      return false; \
+} /* while(0) */
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.eq)
+}
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ne)
+}
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.le)
+}
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ge)
+}
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.lt)
+}
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gt)
+}
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.equ)
+}
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.neu)
+}
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.leu)
+}
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.geu)
+}
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ltu)
+}
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gtu)
+}
+#undef __BOOL_COMPARISON_OP_HALF2_MACRO
+/******************************************************************************
+*                             __half comparison                              *
+******************************************************************************/
+#define __COMPARISON_OP_HALF_MACRO(name) /* do */ {\
+   unsigned short val; \
+   asm( "{ .reg .pred __$temp3;\n" \
+        "  setp."#name".f16  __$temp3, %1, %2;\n" \
+        "  selp.u16 %0, 1, 0, __$temp3;}" \
+        : "=h"(val) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(b))); \
+   return val ? true : false; \
+} /* while(0) */
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(eq)
+}
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ne)
+}
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(le)
+}
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ge)
+}
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(lt)
+}
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gt)
+}
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(equ)
+}
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(neu)
+}
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(leu)
+}
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(geu)
+}
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ltu)
+}
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gtu)
+}
+#undef __COMPARISON_OP_HALF_MACRO
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul)
+}
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.sat)
+}
+__CUDA_FP16_DECL__ __half2 __h2div(__half2 a, __half2 b) {
+    __half ha, hb;
+
+    ha = __low2half(a);
+    hb = __low2half(b);
+
+    __half v1 = __hdiv(ha, hb);
+
+    ha = __high2half(a);
+    hb = __high2half(b);
+
+    __half v2 = __hdiv(ha, hb);
+
+    return __halves2half2(v1, v2);
+}
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add)
+}
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub)
+}
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul)
+}
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add.sat)
+}
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub.sat)
+}
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul.sat)
+}
+
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn)
+}
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.sat)
+}
+__CUDA_FP16_DECL__ __half __hdiv(__half a, __half b) {
+    __half v, abs, den;
+    __HALF_TO_US(den) = 0x008F;
+    float fa, fb, fv, rcp;
+
+    fa = __half2float(a);
+    fb = __half2float(b);
+
+    asm("{rcp.approx.ftz.f32 %0, %1;\n}" :"=f"(rcp) : "f"(fb));
+
+    fv = rcp * fa;
+
+    v = __float2half(fv);
+    __HALF_TO_US(abs) = (unsigned short)(((unsigned int)__HALF_TO_CUS(v)) & 0x00007FFF);
+    if (__hlt(abs, den) && (!(__HALF_TO_CUS(abs) == 0x0000))) {
+        float err = __fmaf_rn(-fb, fv, fa);
+        fv = __fmaf_rn(rcp, err, fv);
+        v = __float2half(fv);
+    }
+    return v;
+}
+
+/******************************************************************************
+*                             __half2 functions                  *
+******************************************************************************/
+#define __SPEC_CASE2(i,r, spc, ulp) \
+   "{.reg.b32 spc, ulp, p;\n"\
+   "  mov.b32 spc,"#spc";\n"\
+   "  mov.b32 ulp,"#ulp";\n"\
+   "  set.eq.f16x2.f16x2 p,"#i", spc;\n"\
+   "  fma.rn.f16x2 "#r",p,ulp,"#r";\n}\n"
+#define __SPEC_CASE(i,r, spc, ulp) \
+   "{.reg.b16 spc, ulp, p;\n"\
+   "  mov.b16 spc,"#spc";\n"\
+   "  mov.b16 ulp,"#ulp";\n"\
+   "  set.eq.f16.f16 p,"#i", spc;\n"\
+   "  fma.rn.f16 "#r",p,ulp,"#r";\n}\n"
+#define __APPROX_FCAST(fun) /* do */ {\
+   __half val;\
+   asm("{.reg.b32         f;        \n"\
+                " .reg.b16         r;        \n"\
+                "  mov.b16         r,%1;     \n"\
+                "  cvt.f32.f16     f,r;      \n"\
+                "  "#fun".approx.f32   f,f;  \n"\
+                "  cvt.rn.f16.f32      r,f;  \n"\
+                "  mov.b16         %0,r;     \n"\
+                "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));\
+   return val;\
+} /* while(0) */
+#define __APPROX_FCAST2(fun) /* do */ {\
+   __half2 val;\
+   asm("{.reg.b16         hl, hu;         \n"\
+                " .reg.b32         fl, fu;         \n"\
+                "  mov.b32         {hl, hu}, %1;   \n"\
+                "  cvt.f32.f16     fl, hl;         \n"\
+                "  cvt.f32.f16     fu, hu;         \n"\
+                "  "#fun".approx.f32   fl, fl;     \n"\
+                "  "#fun".approx.f32   fu, fu;     \n"\
+                "  cvt.rn.f16.f32      hl, fl;     \n"\
+                "  cvt.rn.f16.f32      hu, fu;     \n"\
+                "  mov.b32         %0, {hl, hu};   \n"\
+                "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));       \
+   return val;\
+} /* while(0) */
+static __device__ __forceinline__ float __float_simpl_sinf(float);
+static __device__ __forceinline__ float __float_simpl_cosf(float);
+__CUDA_FP16_DECL__ __half __hsin_internal(const __half a) {
+    float f = __half2float(a);
+    f = __float_simpl_sinf(f);
+    return __float2half_rn(f);
+}
+__CUDA_FP16_DECL__ __half hsin(const __half a) {
+    __half r = __hsin_internal(a);
+    asm("{\n\t"
+        "  .reg.b16 i,r,t;     \n\t"
+        "  mov.b16 r, %0;      \n\t"
+        "  mov.b16 i, %1;      \n\t"
+        "  mov.b16 t, 0x8000;  \n\t"
+        "  and.b16 t,r,t;      \n\t"
+        __SPEC_CASE(i, r, 0X32B3, 0x0800)
+        __SPEC_CASE(i, r, 0X5CB0, 0x1000)
+        __SPEC_CASE(i, r, 0XB2B3, 0x8800)
+        __SPEC_CASE(i, r, 0XDCB0, 0x9000)
+        "  or.b16  r,r,t;      \n\t"
+        "  mov.b16 %0, r;      \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a) {
+    __half l = __low2half(a);
+    __half h = __high2half(a);
+    __half2 r = __halves2half2(__hsin_internal(l), __hsin_internal(h));
+    asm("{\n\t"
+        "  .reg.b32 i,r,t;             \n\t"
+        "  mov.b32 r, %0;              \n\t"
+        "  mov.b32 i, %1;              \n\t"
+        "  and.b32 t, r, 0x80008000;   \n\t"
+        __SPEC_CASE2(i, r, 0X32B332B3, 0x08000800)
+        __SPEC_CASE2(i, r, 0X5CB05CB0, 0x10001000)
+        __SPEC_CASE2(i, r, 0XB2B3B2B3, 0x88008800)
+        __SPEC_CASE2(i, r, 0XDCB0DCB0, 0x90009000)
+        "  or.b32  r, r, t;            \n\t"
+        "  mov.b32 %0, r;              \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __hcos_internal(const __half a) {
+    float f = __half2float(a);
+    f = __float_simpl_cosf(f);
+    return __float2half_rn(f);
+}
+__CUDA_FP16_DECL__ __half hcos(const __half a) {
+    __half r = __hcos_internal(a);
+    asm("{\n\t"
+        "  .reg.b16 i,r;        \n\t"
+        "  mov.b16 r, %0;       \n\t"
+        "  mov.b16 i, %1;       \n\t"
+        __SPEC_CASE(i, r, 0X2B7C, 0x1000)
+        __SPEC_CASE(i, r, 0XAB7C, 0x1000)
+        "  mov.b16 %0, r;       \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a) {
+    __half l = __low2half(a);
+    __half h = __high2half(a);
+    __half2 r = __halves2half2(__hcos_internal(l), __hcos_internal(h));
+    asm("{\n\t"
+        "  .reg.b32 i,r;   \n\t"
+        "  mov.b32 r, %0;  \n\t"
+        "  mov.b32 i, %1;  \n\t"
+        __SPEC_CASE2(i, r, 0X2B7C2B7C, 0x10001000)
+        __SPEC_CASE2(i, r, 0XAB7CAB7C, 0x10001000)
+        "  mov.b32 %0, r;  \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+static __device__ __forceinline__ float __internal_trig_reduction_kernel(float a, int *quadrant)
+{
+    float j, t;
+    int q;
+    q = __float2int_rn(a * 0.636619772F);
+    j = (float)q;
+    t = __fmaf_rn(-j, 1.5707962512969971e+000F, a);
+    t = __fmaf_rn(-j, 7.5497894158615964e-008F, t);
+    *quadrant = q;
+    return t;
+}
+static __device__ __forceinline__ float __internal_sin_cos_kernel(float x, int i)
+{
+    float x2, z;
+    x2 = x*x;
+
+    if (i & 1) {
+        z = 2.44331571e-5F;
+        z = __fmaf_rn(z, x2, -1.38873163e-3F);
+    }
+    else {
+        z = -1.95152959e-4F;
+        z = __fmaf_rn(z, x2, 8.33216087e-3F);
+    }
+    if (i & 1) {
+        z = __fmaf_rn(z, x2, 4.16666457e-2F);
+        z = __fmaf_rn(z, x2, -5.00000000e-1F);
+    }
+    else {
+        z = __fmaf_rn(z, x2, -1.66666546e-1F);
+        z = __fmaf_rn(z, x2, 0.0F);
+    }
+    x = __fmaf_rn(z, x, x);
+    if (i & 1) {
+        x = __fmaf_rn(z, x2, 1.0F);
+    }
+    if (i & 2) {
+        x = __fmaf_rn(x, -1.0F, 0.0F);
+    }
+    return x;
+}
+static __device__ __forceinline__ float __float_simpl_sinf(float a)
+{
+    float z;
+    int i;
+    if (::isinf(a)) {
+        a = a * 0.0F;
+    }
+    a = __internal_trig_reduction_kernel(a, &i);
+    z = __internal_sin_cos_kernel(a, i);
+    return z;
+}
+static __device__ __forceinline__ float __float_simpl_cosf(float a)
+{
+    float z;
+    int i;
+    if (::isinf(a)) {
+        a = a * 0.0F;
+    }
+    a = __internal_trig_reduction_kernel(a, &i);
+    i++;
+    z = __internal_sin_cos_kernel(a, i);
+    return z;
+}
+
+__CUDA_FP16_DECL__ __half hexp(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C;           \n"
+        " .reg.b16         h,r;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  mov.b32         C, 0x3fb8aa3b;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  ex2.approx.f32      f,f;        \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X1F79, 0x9400)
+        __SPEC_CASE(h, r, 0X25CF, 0x9400)
+        __SPEC_CASE(h, r, 0XC13B, 0x0400)
+        __SPEC_CASE(h, r, 0XC1EF, 0x0200)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu, C;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x3fb8aa3b;  \n"
+        "  mul.f32         fl,fl,C;        \n"
+        "  mul.f32         fu,fu,C;        \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0X1F791F79, 0x94009400)
+        __SPEC_CASE2(h, r, 0X25CF25CF, 0x94009400)
+        __SPEC_CASE2(h, r, 0XC13BC13B, 0x04000400)
+        __SPEC_CASE2(h, r, 0XC1EFC1EF, 0x02000200)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp2(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, ULP;         \n"
+        " .reg.b16         r;              \n"
+        "  mov.b16         r,%1;           \n"
+        "  cvt.f32.f16     f,r;            \n"
+        "  ex2.approx.f32      f,f;        \n"
+        "  mov.b32         ULP, 0x33800000;\n"
+        "  fma.rn.f32      f,f,ULP,f;      \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, ULP;    \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  mov.b32         ULP, 0x33800000;\n"
+        "  fma.rn.f32      fl,fl,ULP,fl;   \n"
+        "  fma.rn.f32      fu,fu,ULP,fu;   \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         %0, {hl, hu};   \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h,r;            \n"
+        " .reg.b32         f, C;           \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  mov.b32         C, 0x40549A78;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  ex2.approx.f32      f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x34DE, 0x9800)
+        __SPEC_CASE(h, r, 0x9766, 0x9000)
+        __SPEC_CASE(h, r, 0x9972, 0x1000)
+        __SPEC_CASE(h, r, 0xA5C4, 0x1000)
+        __SPEC_CASE(h, r, 0xBF0A, 0x8100)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu, C;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x40549A78;  \n"
+        "  mul.f32         fl,fl,C;        \n"
+        "  mul.f32         fu,fu,C;        \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0x34DE34DE, 0x98009800)
+        __SPEC_CASE2(h, r, 0x97669766, 0x90009000)
+        __SPEC_CASE2(h, r, 0x99729972, 0x10001000)
+        __SPEC_CASE2(h, r, 0xA5C4A5C4, 0x10001000)
+        __SPEC_CASE2(h, r, 0xBF0ABF0A, 0x81008100)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog2(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f;              \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.f32      f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(r, r, 0xA2E2, 0x8080)
+        __SPEC_CASE(r, r, 0xBF46, 0x9400)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, r, p;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  lg2.approx.f32      fl, fl;     \n"
+        "  lg2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(r, r, 0xA2E2A2E2, 0x80808080)
+        __SPEC_CASE2(r, r, 0xBF46BF46, 0x94009400)
+        "  mov.b32         %0, r;          \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C;           \n"
+        " .reg.b16         r,h;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  lg2.approx.f32      f,f;        \n"
+        "  mov.b32         C, 0x3f317218;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X160D, 0x9C00)
+        __SPEC_CASE(h, r, 0X3BFE, 0x8010)
+        __SPEC_CASE(h, r, 0X3C0B, 0x8080)
+        __SPEC_CASE(h, r, 0X6051, 0x1C00)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.f32      fl, fl;         \n"
+        "  lg2.approx.f32      fu, fu;         \n"
+        "  mov.b32         C, 0x3f317218;      \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0X160D160D, 0x9C009C00)
+        __SPEC_CASE2(h, r, 0X3BFE3BFE, 0x80108010)
+        __SPEC_CASE2(h, r, 0X3C0B3C0B, 0x80808080)
+        __SPEC_CASE2(h, r, 0X60516051, 0x1C001C00)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f, C;           \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.f32      f, f;       \n"
+        "  mov.b32         C, 0x3E9A209B;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x338F, 0x1000)
+        __SPEC_CASE(h, r, 0x33F8, 0x9000)
+        __SPEC_CASE(h, r, 0x57E1, 0x9800)
+        __SPEC_CASE(h, r, 0x719D, 0x9C00)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.f32      fl, fl;         \n"
+        "  lg2.approx.f32      fu, fu;         \n"
+        "  mov.b32         C, 0x3E9A209B;      \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0x338F338F, 0x10001000)
+        __SPEC_CASE2(h, r, 0x33F833F8, 0x90009000)
+        __SPEC_CASE2(h, r, 0x57E157E1, 0x98009800)
+        __SPEC_CASE2(h, r, 0x719D719D, 0x9C009C00)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+#undef __SPEC_CASE2
+#undef __SPEC_CASE
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a) {
+    __APPROX_FCAST2(rcp)
+}
+__CUDA_FP16_DECL__ __half hrcp(const __half a) {
+    __APPROX_FCAST(rcp)
+}
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a) {
+    __APPROX_FCAST2(rsqrt)
+}
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a) {
+    __APPROX_FCAST(rsqrt)
+}
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a) {
+    __APPROX_FCAST2(sqrt)
+}
+__CUDA_FP16_DECL__ __half hsqrt(const __half a) {
+    __APPROX_FCAST(sqrt)
+}
+#undef __APPROX_FCAST
+#undef __APPROX_FCAST2
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a)
+{
+    __half2 r;
+    asm("{set.nan.f16x2.f16x2 %0,%1,%2;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ bool __hisnan(const __half a)
+{
+    __half r;
+    asm("{set.nan.f16.f16 %0,%1,%2;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(a)));
+    return __HALF_TO_CUS(r) != 0U;
+}
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a)
+{
+    __half2 r;
+    asm("{neg.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __hneg(const __half a)
+{
+    __half r;
+    asm("{neg.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a)
+{
+    __half2 r;
+    asm("{abs.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __habs(const __half a)
+{
+    __half r;
+    asm("{abs.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+#endif /*__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)*/
+
+#if __CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half __hmax(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(max)
+}
+__CUDA_FP16_DECL__ __half __hmin(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(min)
+}
+__CUDA_FP16_DECL__ __half __hmax_nan(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(max.NaN)
+}
+__CUDA_FP16_DECL__ __half __hmin_nan(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(min.NaN)
+}
+__CUDA_FP16_DECL__ __half __hfma_relu(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.relu)
+}
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half2 __hmax2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(max)
+}
+__CUDA_FP16_DECL__ __half2 __hmin2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(min)
+}
+__CUDA_FP16_DECL__ __half2 __hmax2_nan(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(max.NaN)
+}
+__CUDA_FP16_DECL__ __half2 __hmin2_nan(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(min.NaN)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_relu(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.relu)
+}
+#endif /*__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)*/
+
+/* Define __PTR for atomicAdd prototypes below, undef after done */
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __PTR   "l"
+#else
+#define __PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600
+
+__CUDA_FP16_DECL__  __half2 atomicAdd(__half2 *address, __half2 val) {
+    __half2 r;
+    asm volatile ("{ atom.add.noftz.f16x2 %0,[%1],%2; }\n"
+                  : "=r"(__HALF2_TO_UI(r)) : __PTR(address), "r"(__HALF2_TO_CUI(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600*/
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700
+
+__CUDA_FP16_DECL__  __half atomicAdd(__half *address, __half val) {
+    __half r;
+    asm volatile ("{ atom.add.noftz.f16 %0,[%1],%2; }\n"
+                  : "=h"(__HALF_TO_US(r))
+                  : __PTR(address), "h"(__HALF_TO_CUS(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700*/
+
+#undef __PTR
+
+#undef __CUDA_FP16_DECL__
+#endif /* defined(__CUDACC__) */
+#endif /* defined(__cplusplus) */
+
+#undef __TERNARY_OP_HALF2_MACRO
+#undef __TERNARY_OP_HALF_MACRO
+#undef __BINARY_OP_HALF2_MACRO
+#undef __BINARY_OP_HALF_MACRO
+
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+#undef __CUDA_FP16_DECL__
+ 
+/* Define first-class types "half" and "half2", unless user specifies otherwise via "#define CUDA_NO_HALF" */
+/* C cannot ever have these types defined here, because __half and __half2 are C++ classes */
+#if defined(__cplusplus) && !defined(CUDA_NO_HALF)
+typedef __half half;
+typedef __half2 half2;
+// for consistency with __nv_bfloat16
+typedef __half      __nv_half;
+typedef __half2     __nv_half2;
+typedef __half_raw  __nv_half_raw;
+typedef __half2_raw __nv_half2_raw;
+typedef __half        nv_half;
+typedef __half2       nv_half2;
+#endif /* defined(__cplusplus) && !defined(CUDA_NO_HALF) */
+
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+#undef __CPP_VERSION_AT_LEAST_11_FP16
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+
+#endif /* end of include guard: __CUDA_FP16_HPP__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-11.1/cuda_fp16.h b/cupy/_core/include/cupy/_cuda/cuda-11.1/cuda_fp16.h
new file mode 100755
index 0000000..57441c6
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-11.1/cuda_fp16.h
@@ -0,0 +1,3631 @@
+/*
+* Copyright 1993-2020 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+/**
+* \defgroup CUDA_MATH_INTRINSIC_HALF Half Precision Intrinsics
+* This section describes half precision intrinsic functions that are
+* only supported in device code.
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_ARITHMETIC Half Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_ARITHMETIC Half2 Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_COMPARISON Half Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_COMPARISON Half2 Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_MISC Half Precision Conversion And Data Movement
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_FUNCTIONS Half Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_FUNCTIONS Half2 Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions include the header file \p cuda_fp16.h in your program.
+*/
+
+#ifndef __CUDA_FP16_H__
+#define __CUDA_FP16_H__
+
+#if defined(__cplusplus)
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__CUDACC__) */
+
+#define __CUDA_FP16_TYPES_EXIST__
+
+/* Forward-declaration of structures defined in "cuda_fp16.hpp" */
+
+/**
+ * \brief half datatype 
+ * 
+ * \details This structure implements the datatype for storing 
+ * half-precision floating-point numbers. The structure implements 
+ * assignment operators and type conversions. 
+ * 16 bits are being used in total: 1 sign bit, 5 bits for the exponent, 
+ * and the significand is being stored in 10 bits. 
+ * The total precision is 11 bits. There are 15361 representable 
+ * numbers within the interval [0.0, 1.0], endpoints included. 
+ * On average we have log10(2**11) ~ 3.311 decimal digits. 
+ * 
+ * \internal
+ * \req IEEE 754-2008 compliant implementation of half-precision 
+ * floating-point numbers. 
+ * \endinternal
+ */
+struct __half;
+
+/**
+ * \brief half2 datatype
+ * 
+ * \details This structure implements the datatype for storing two 
+ * half-precision floating-point numbers. 
+ * The structure implements assignment operators and type conversions. 
+ * 
+ * \internal
+ * \req Vectorified version of half. 
+ * \endinternal
+ */
+struct __half2;
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts double number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts double number \p a to half precision in round-to-nearest-even mode.
+* \param[in] a - double. Is only being read.
+* \returns half
+* \retval \p a converted to half.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __double2half(const double a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value. 
+* 
+* \details Converts float number \p a to half precision in round-to-nearest-even mode. 
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts float number \p a to half precision in round-to-nearest-even mode.
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-towards-zero mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-towards-zero mode.
+* \param[in] a - float. Is only being read. 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-down mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-down mode.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-up mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-up mode.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* \retval \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts \p half number to float.
+* 
+* \details Converts half number \p a to float.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns float
+* \retval \p a converted to float. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts input to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+*
+* \details Converts input \p a to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+* \param[in] a - float. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 value with both halves equal to the converted half
+* precision number.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both input floats to half precision in round-to-nearest-even
+* mode and returns \p half2 with converted values.
+*
+* \details Converts both input floats to half precision in round-to-nearest-even mode
+* and combines the results into one \p half2 number. Low 16 bits of the return
+* value correspond to the input \p a, high 16 bits correspond to the input \p
+* b.
+* \param[in] a - float. Is only being read. 
+* \param[in] b - float. Is only being read. 
+* 
+* \returns half2
+* \retval The \p half2 value with corresponding halves equal to the
+* converted input floats.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts low 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts low 16 bits of \p half2 input \p a to 32 bit floating point number
+* and returns the result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* \retval The low 16 bits of \p a converted to float.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts high 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts high 16 bits of \p half2 input \p a to 32 bit floating point number
+* and returns the result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* \retval The high 16 bits of \p a converted to float.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a);
+
+#if defined(__CUDACC__)
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both components of float2 number to half precision in
+* round-to-nearest-even mode and returns \p half2 with converted values.
+* 
+* \details Converts both components of float2 to half precision in round-to-nearest
+* mode and combines the results into one \p half2 number. Low 16 bits of the
+* return value correspond to \p a.x and high 16 bits of the return value
+* correspond to \p a.y.
+* \param[in] a - float2. Is only being read. 
+*  
+* \returns half2
+* \retval The \p half2 which has corresponding halves equal to the
+* converted float2 components.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both halves of \p half2 to float2 and returns the result.
+* 
+* \details Converts both halves of \p half2 input \p a to float2 and returns the
+* result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float2
+* \retval \p a converted to float2.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ int __half2int_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed integer in
+* round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* \retval \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-to-nearest-even mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __int2half_rn(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-towards-zero mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rz(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-down mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-down mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rd(int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-up mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating point
+* value in round-up mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_ru(int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ short int __half2short_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed short
+* integer in round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __short2half_rn(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rz(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-down mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rd(short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-up mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_ru(short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned int __half2uint_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-down mode.
+*
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-down mode.
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-up mode.
+*
+* \details Convert the half-precision floating point value \p h to an unsigned integer
+* in round-up mode.
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* \retval \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-to-nearest-even mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __uint2half_rn(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-towards-zero mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-towards-zero mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half.  
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rz(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-down mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-down mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rd(unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-up mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating point
+* value in round-up mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_ru(unsigned int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-towards-zero
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned short int __half2ushort_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned short
+* integer in round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval \p h converted to an an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ushort2half_rn(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rz(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-down mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rd(unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-up mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_ru(unsigned short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-towards-zero
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned long long int __half2ull_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to an unsigned 64-bit
+* integer in round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* \retval \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ull2half_rn(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rz(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half.  
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rd(unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_ru(unsigned long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-to-nearest-even mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rn(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-towards-zero mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-towards-zero mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ long long int __half2ll_rz(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-down mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rd(__half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating point value \p h to a signed 64-bit
+* integer in round-up mode.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* \retval \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_ru(__half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-to-nearest-even mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ll2half_rn(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-towards-zero mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rz(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-down mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rd(long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating
+* point value in round-up mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* \retval \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_ru(long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Truncate input argument to the integral part.
+* 
+* \details Round \p h to the nearest integer value that does not exceed \p h in
+* magnitude.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The truncated integer value. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half htrunc(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate ceiling of the input argument.
+* 
+* \details Compute the smallest integer value not less than \p h.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The smallest integer value not less than \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hceil(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details Calculate the largest integer value which is less than or equal to \p h.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The largest integer value which is less than or equal to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hfloor(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating point
+* number.
+* 
+* \details Round \p h to the nearest integer value in half-precision floating point
+* format, with halfway cases rounded to the nearest even integer value.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* \retval The nearest integer to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrint(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Truncate \p half2 vector input argument to the integral part.
+* 
+* \details Round each component of vector \p h to the nearest integer value that does
+* not exceed \p h in magnitude.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The truncated \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate \p half2 vector ceiling of the input argument.
+* 
+* \details For each component of vector \p h compute the smallest integer value not less
+* than \p h.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of smallest integers not less than \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details For each component of vector \p h calculate the largest integer value which
+* is less than or equal to \p h.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of largest integers which is less than or equal to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating point
+* number.
+* 
+* \details Round each component of \p half2 vector \p h to the nearest integer value in
+* half-precision floating point format, with halfway cases rounded to the
+* nearest even integer value.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector of rounded integer values. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns \p half2 with both halves equal to the input value.
+* 
+* \details Returns \p half2 number with both halves equal to the input \p a \p half
+* number.
+* \param[in] a - half. Is only being read. 
+* 
+* \returns half2
+* \retval The vector which has both its halves equal to the input \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Swaps both halves of the \p half2 input.
+* 
+* \details Swaps both halves of the \p half2 input and returns a new \p half2 number
+* with swapped halves.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval \p a with its halves being swapped. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from each of the two \p half2 inputs and combines
+* into one \p half2 number. 
+* 
+* \details Extracts low 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. Low 16 bits from input \p a is stored in low 16 bits of
+* the return value, low 16 bits from input \p b is stored in high 16 bits of
+* the return value. 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The low 16 bits of \p a and of \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from each of the two \p half2 inputs and
+* combines into one \p half2 number.
+* 
+* \details Extracts high 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. High 16 bits from input \p a is stored in low 16 bits of
+* the return value, high 16 bits from input \p b is stored in high 16 bits of
+* the return value.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The high 16 bits of \p a and of \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns high 16 bits of \p half2 input.
+*
+* \details Returns high 16 bits of \p half2 input \p a.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* \retval The high 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __high2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns low 16 bits of \p half2 input.
+*
+* \details Returns low 16 bits of \p half2 input \p a.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* \retval Returns \p half which contains low 16 bits of the input \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __low2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Checks if the input \p half number is infinite.
+* 
+* \details Checks if the input \p half number \p a is infinite. 
+* \param[in] a - half. Is only being read. 
+* 
+* \returns int 
+* \retval -1 iff \p a is equal to negative infinity, 
+* \retval 1 iff \p a is equal to positive infinity, 
+* \retval 0 otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __hisinf(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Combines two \p half numbers into one \p half2 number.
+* 
+* \details Combines two input \p half number \p a and \p b into one \p half2 number.
+* Input \p a is stored in low 16 bits of the return value, input \p b is stored
+* in high 16 bits of the return value.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with one half equal to \p a and the other to \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from \p half2 input.
+* 
+* \details Extracts low 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with both halves equal to the low 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from \p half2 input.
+* 
+* \details Extracts high 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The half2 with both halves equal to the high 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 a);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as a signed short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating point number \p h
+* as a signed short integer. 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* \retval The reinterpreted value. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as an unsigned short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating point \p h
+* as an unsigned short number.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* \retval The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a signed short integer as a \p half.
+* 
+* \details Reinterprets the bits in the signed short integer \p i as a
+* half-precision floating point number.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* \retval The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in an unsigned short integer as a \p half.
+* 
+* \details Reinterprets the bits in the unsigned short integer \p i as a
+* half-precision floating point number.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* \retval The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i);
+
+#if __CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)
+#if !defined warpSize && !defined __local_warpSize
+#define warpSize    32
+#define __local_warpSize
+#endif
+
+#if defined(_WIN32)
+# define __DEPRECATED__(msg) __declspec(deprecated(msg))
+#elif (defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 5 && !defined(__clang__))))
+# define __DEPRECATED__(msg) __attribute__((deprecated))
+#else
+# define __DEPRECATED__(msg) __attribute__((deprecated(msg)))
+#endif
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+#define __WSB_DEPRECATION_MESSAGE(x) #x"() is deprecated in favor of "#x"_sync() and may be removed in a future release (Use -Wno-deprecated-declarations to suppress this warning)."
+
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half2 __shfl(__half2 var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half2 __shfl_up(__half2 var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down))__half2 __shfl_down(__half2 var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half2 __shfl_xor(__half2 var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half __shfl(__half var, int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half __shfl_up(__half var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down)) __half __shfl_down(__half var, unsigned int delta, int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half __shfl_xor(__half var, int delta, int width = warpSize);
+#endif
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* ithin the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_sync(unsigned mask, __half2 var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(unsigned mask, __half2 var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(unsigned mask, __half2 var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(unsigned mask, __half2 var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* ithin the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_sync(unsigned mask, __half var, int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_up_sync(unsigned mask, __half var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_down_sync(unsigned mask, __half var, unsigned int delta, int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(unsigned mask, __half var, int delta, int width = warpSize);
+
+#if defined(__local_warpSize)
+#undef warpSize
+#undef __local_warpSize
+#endif
+#endif /*__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__) */
+
+#if defined(__cplusplus) && ( __CUDA_ARCH__ >=320 || !defined(__CUDA_ARCH__) )
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.nc` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.nc` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldg(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cg` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cg` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcg(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.ca` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.ca` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldca(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cs` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cs` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcs(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.lu` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldlu(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.lu` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldlu(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cv` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcv(const  __half2 *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cv` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcv(const __half *ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wb` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwb(__half2 *ptr, __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wb` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwb(__half *ptr, __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cg` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcg(__half2 *ptr, __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cg` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcg(__half *ptr, __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cs` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcs(__half2 *ptr, __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cs` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcs(__half *ptr, __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wt` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwt(__half2 *ptr, __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wt` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwt(__half *ptr, __half value);
+#endif /*defined(__cplusplus) && ( __CUDA_ARCH__ >=320 || !defined(__CUDA_ARCH__) )*/
+
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs half2 vector if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison.
+* 
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 result of less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The half2 vector result of less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison.
+* 
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The vector result of unordered if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison.
+*
+* Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 vector result of unordered greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The vector result of unordered less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The \p half2 vector result of unordered greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Determine whether \p half2 argument is a NaN.
+*
+* \details Determine whether each half of input \p half2 number \p a is a NaN.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The half2 with the corresponding \p half results set to
+* 1.0 for for NaN, 0.0 otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-95
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The sum of vectors \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-104
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The subtraction of vector \p b from \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-102
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise multiplying the vectors \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector division in round-to-nearest-even mode.
+*
+* \details Divides \p half2 input vector \p a by input vector \p b in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-103
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise division of \p a with \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __h2div(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+*
+* \details Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval Returns \p a with the absolute value of both halves. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the results to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The sum of \p a and \p b, with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The subtraction of vector \p b from \p a, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise multiplication of vectors \p a and \p b, 
+* with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-105
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode, with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the
+* results to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* \retval The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c, 
+* with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Negates both halves of the input \p half2 number and returns the
+* result.
+*
+* \details Negates both halves of the input \p half2 number \p a and returns the result.
+* \internal
+* \req DEEPLEARN-SRM_REQ-101
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval Returns \p a with both halves negated. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Calculates the absolute value of input \p half number and returns the result.
+*
+* \details Calculates the absolute value of input \p half number and returns the result.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The absolute value of a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __habs(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode.
+*
+* \details Performs \p half addition of inputs \p a and \p b, in round-to-nearest-even
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-94
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The sum of \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-97
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of subtracting \p b from \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-99
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of multiplying \p a and \p b. 
+*/
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half division in round-to-nearest-even mode.
+* 
+* \details Divides \p half input \p a by input \p b in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-98
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half
+* \retval The result of dividing \p a by \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__  __half __hdiv(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half add of inputs \p a and \p b, in round-to-nearest-even mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The sum of \p a and \p b, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of subtraction of \p b from \p a, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the result to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* \retval The result of multiplying \p a and \p b, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-96
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* \retval The result of fused multiply-add operation on \p
+* a, \p b, and \p c. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the result
+* to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* \retval The result of fused multiply-add operation on \p
+* a, \p b, and \p c, with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Negates input \p half number and returns the result.
+*
+* \details Negates input \p half number and returns the result.
+* \internal
+* \req DEEPLEARN-SRM_REQ-100
+* \endinternal
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval minus a
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hneg(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector if-equal comparison, and returns boolean true
+* iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of if-equal comparison
+* of vectors \p a and \p b are true;
+* \retval false, otherwise.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of not-equal comparison
+* of vectors \p a and \p b are true, 
+* \retval false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of less-equal comparison
+* of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of greater-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of less-than comparison
+* of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison, and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns bool 
+* \retval true, if both \p half results of greater-than
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered if-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered not-equal
+* comparison of vectors \p a and \p b are true;
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered less-equal
+* comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison, and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered
+* greater-equal comparison of vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison, and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered less-than comparison of 
+* vectors \p a and \p b are true; 
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison, and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* \retval true, if both \p half results of unordered
+* greater-than comparison of vectors \p a and \p b are true;
+* \retval false, otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of if-equal comparison of \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of not-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of less-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of greater-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of less-than comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of greater-than comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered if-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered not-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered less-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered greater-equal comparison of \p a
+* and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered less-than comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* \retval The boolean result of unordered greater-than comparison of \p a
+* and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Determine whether \p half argument is a NaN.
+*
+* \details Determine whether \p half value \p a is a NaN.
+* \param[in] a - half. Is only being read. 
+*
+* \returns bool
+* \retval true iff argument is NaN. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hisnan(const __half a);
+#if __CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half maximum of two input values.
+*
+* \details Calculates \p half max(\p a, \p b)
+* defined as (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmax(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half minimum of two input values.
+*
+* \details Calculates \p half min(\p a, \p b)
+* defined as (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmin(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half maximum of two input values, NaNs pass through.
+*
+* \details Calculates \p half max(\p a, \p b)
+* defined as (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmax_nan(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half minimum of two input values, NaNs pass through.
+*
+* \details Calculates \p half min(\p a, \p b)
+* defined as (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmin_nan(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode with relu saturation.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* Then negative result is clamped to 0.
+* NaN result is converted to canonical NaN.
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+* \param[in] c - half. Is only being read.
+*
+* \returns half
+* \retval The result of fused multiply-add operation on \p
+* a, \p b, and \p c with relu saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma_relu(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector maximum of two inputs.
+*
+* \details Calculates \p half2 vector max(\p a, \p b)
+* Elementwise \p half operation is defined as
+* (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise maximum of vectors \p a  and \p b
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmax2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector minimum of two inputs.
+*
+* \details Calculates \p half2 vector min(\p a, \p b)
+* Elementwise \p half operation is defined as
+* (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise minimum of vectors \p a  and \p b
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmin2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector maximum of two inputs, NaNs pass through.
+*
+* \details Calculates \p half2 vector max(\p a, \p b)
+* Elementwise \p half operation is defined as
+* (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise maximum of vectors \p a  and \p b, with NaNs pass through
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmax2_nan(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector minimum of two inputs, NaNs pass through.
+*
+* \details Calculates \p half2 vector min(\p a, \p b)
+* Elementwise \p half operation is defined as
+* (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise minimum of vectors \p a  and \p b, with NaNs pass through
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmin2_nan(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode with relu saturation.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* Then negative result is clamped to 0.
+* NaN result is converted to canonical NaN.
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+* \param[in] c - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c with relu saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_relu(const __half2 a, const __half2 b, const __half2 c);
+#endif /*__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)*/
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs fast complex multiply-accumulate
+*
+* \details Interprets vector \p half2 input pairs \p a, \p b, and \p c as
+* complex numbers in \p half precision and performs
+* complex multiply-accumulate operation: a*b + c
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+* \param[in] c - half2. Is only being read.
+*
+* \returns half2
+* \retval The result of complex multiply-accumulate operation on complex numbers \p a, \p b, and \p c
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hcmadd(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half square root of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The square root of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal square root in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half reciprocal square root of input \p a in round-to-nearest
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The reciprocal square root of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half reciprocal of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The reciprocal of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrcp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half natural logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The natural logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half binary logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The binary logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half decimal logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The decimal logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half natural exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The natural exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half binary exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The binary exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half decimal exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The decimal exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half cosine in round-to-nearest-even mode.
+*
+* \details Calculates \p half cosine of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The cosine of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hcos(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half sine in round-to-nearest-even mode.
+*
+* \details Calculates \p half sine of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* \retval The sine of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hsin(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 square root of input vector \p a in round-to-nearest
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise square root on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal square root in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 reciprocal square root of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise reciprocal square root on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 reciprocal of input vector \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise reciprocal on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector natural logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 natural logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise natural logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 binary logarithm of input vector \p a in round-to-nearest
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise binary logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 decimal logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise decimal logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary exponential function in
+* round-to-nearest-even mode.
+*
+* \details Calculates \p half2 binary exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* \retval The elementwise binary exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal exponential function in
+* round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 decimal exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise decimal exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector cosine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 cosine of input vector \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise cosine on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector sine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 sine of input vector \p a in round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* \retval The elementwise sine on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a);
+
+#endif /*if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)*/
+
+#if __CUDA_ARCH__ >= 600 || !defined(__CUDA_ARCH__)
+
+__CUDA_FP16_DECL__ __half2 atomicAdd(__half2 *address, __half2 val);
+
+#endif /*if __CUDA_ARCH__ >= 600 || !defined(__CUDA_ARCH__)*/
+
+#if __CUDA_ARCH__ >= 700 || !defined(__CUDA_ARCH__)
+
+__CUDA_FP16_DECL__ __half atomicAdd(__half *address, __half val);
+
+#endif /*if __CUDA_ARCH__ >= 700 || !defined(__CUDA_ARCH__)*/
+
+#endif /* defined(__CUDACC__) */
+
+#undef __CUDA_FP16_DECL__
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+
+#endif /* defined(__cplusplus) */
+
+/* Note the .hpp file is included even for host-side compilation, to capture the "half" & "half2" definitions */
+#include "cuda_fp16.hpp"
+
+#endif /* end of include guard: __CUDA_FP16_H__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-11.1/cuda_fp16.hpp b/cupy/_core/include/cupy/_cuda/cuda-11.1/cuda_fp16.hpp
new file mode 100755
index 0000000..123c6e0
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-11.1/cuda_fp16.hpp
@@ -0,0 +1,2453 @@
+/*
+* Copyright 1993-2020 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+#if !defined(__CUDA_FP16_HPP__)
+#define __CUDA_FP16_HPP__
+
+#if !defined(__CUDA_FP16_H__)
+#error "Do not include this file directly. Instead, include cuda_fp16.h."
+#endif
+
+#if !defined(_MSC_VER) && __cplusplus >= 201103L
+#   define __CPP_VERSION_AT_LEAST_11_FP16
+#elif _MSC_FULL_VER >= 190024210 && _MSVC_LANG >= 201103L
+#   define __CPP_VERSION_AT_LEAST_11_FP16
+#endif
+
+/* C++11 header for std::move. 
+ * In RTC mode, std::move is provided implicitly; don't include the header
+ */
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16) && !defined(__CUDACC_RTC__)
+#include <utility>
+#endif /* __cplusplus >= 201103L && !defined(__CUDACC_RTC__) */
+
+/* C++ header for std::memcpy (used for type punning in host-side implementations).
+ * When compiling as a CUDA source file memcpy is provided implicitly.
+ * !defined(__CUDACC__) implies !defined(__CUDACC_RTC__).
+ */
+#if defined(__cplusplus) && !defined(__CUDACC__)
+#include <cstring>
+#endif /* defined(__cplusplus) && !defined(__CUDACC__) */
+
+
+/* Set up function decorations */
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#define __VECTOR_FUNCTIONS_DECL__ static __inline__ __host__ __device__
+#define __CUDA_HOSTDEVICE__ __host__ __device__
+#else /* !defined(__CUDACC__) */
+#if defined(__GNUC__)
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __attribute__ ((unused))
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__GNUC__) */
+#define __CUDA_HOSTDEVICE__
+#endif /* defined(__CUDACC_) */
+
+/* Set up structure-alignment attribute */
+#if defined(__CUDACC__)
+#define __CUDA_ALIGN__(align) __align__(align)
+#else
+/* Define alignment macro based on compiler type (cannot assume C11 "_Alignas" is available) */
+#if __cplusplus >= 201103L
+#define __CUDA_ALIGN__(n) alignas(n)    /* C++11 kindly gives us a keyword for this */
+#else /* !defined(__CPP_VERSION_AT_LEAST_11_FP16)*/
+#if defined(__GNUC__)
+#define __CUDA_ALIGN__(n) __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+#define __CUDA_ALIGN__(n) __declspec(align(n))
+#else
+#define __CUDA_ALIGN__(n)
+#endif /* defined(__GNUC__) */
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+#endif /* defined(__CUDACC__) */
+
+/* Macros to allow half & half2 to be used by inline assembly */
+#define __HALF_TO_US(var) *(reinterpret_cast<unsigned short *>(&(var)))
+#define __HALF_TO_CUS(var) *(reinterpret_cast<const unsigned short *>(&(var)))
+#define __HALF2_TO_UI(var) *(reinterpret_cast<unsigned int *>(&(var)))
+#define __HALF2_TO_CUI(var) *(reinterpret_cast<const unsigned int *>(&(var)))
+
+/* Macros for half & half2 binary arithmetic */
+#define __BINARY_OP_HALF_MACRO(name) /* do */ {\
+   __half val; \
+   asm( "{"#name".f16 %0,%1,%2;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b))); \
+   return val; \
+} /* while(0) */
+#define __BINARY_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{"#name".f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+#define __TERNARY_OP_HALF_MACRO(name) /* do */ {\
+   __half val; \
+   asm( "{"#name".f16 %0,%1,%2,%3;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b)),"h"(__HALF_TO_CUS(c))); \
+   return val; \
+} /* while(0) */
+#define __TERNARY_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{"#name".f16x2 %0,%1,%2,%3;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b)),"r"(__HALF2_TO_CUI(c))); \
+   return val; \
+} /* while(0) */
+
+/**
+* Types which allow static initialization of "half" and "half2" until
+* these become an actual builtin. Note this initialization is as a
+* bitfield representation of "half", and not a conversion from short->half.
+* Such a representation will be deprecated in a future version of CUDA. 
+* (Note these are visible to non-nvcc compilers, including C-only compilation)
+*/
+typedef struct __CUDA_ALIGN__(2) {
+    unsigned short x;
+} __half_raw;
+
+typedef struct __CUDA_ALIGN__(4) {
+    unsigned short x;
+    unsigned short y;
+} __half2_raw;
+
+/* All other definitions in this file are only visible to C++ compilers */
+#if defined(__cplusplus)
+
+/* Hide GCC member initialization list warnings because of host/device in-function init requirement */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstrict-aliasing"
+#pragma GCC diagnostic ignored "-Weffc++"
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+/* class' : multiple assignment operators specified
+   The class has multiple assignment operators of a single type. This warning is informational */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( push )
+#pragma warning( disable:4522 )
+#endif /* defined(__GNUC__) */
+
+struct __CUDA_ALIGN__(2) __half {
+protected:
+    unsigned short __x;
+
+public:
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+    __half() = default;
+#else
+    __CUDA_HOSTDEVICE__ __half() { }
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+
+    /* Convert to/from __half_raw */
+    __CUDA_HOSTDEVICE__ __half(const __half_raw &hr) : __x(hr.x) { }
+    __CUDA_HOSTDEVICE__ __half &operator=(const __half_raw &hr) { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const volatile __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const { __half_raw ret; ret.x = __x; return ret; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const volatile { __half_raw ret; ret.x = __x; return ret; }
+
+#if !defined(__CUDA_NO_HALF_CONVERSIONS__)
+
+    /* Construct from float/double */
+    __CUDA_HOSTDEVICE__ __half(const float f) { __x = __float2half(f).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const double f) { __x = __double2half(f).__x;  }
+
+    __CUDA_HOSTDEVICE__ operator float() const { return __half2float(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const float f) { __x = __float2half(f).__x; return *this; }
+
+    /* We omit "cast to double" operator, so as to not be ambiguous about up-cast */
+    __CUDA_HOSTDEVICE__ __half &operator=(const double f) { __x = __double2half(f).__x; return *this; }
+
+/* Member functions only available to nvcc compilation so far */
+#if defined(__CUDACC__)
+    /* Allow automatic construction from types supported natively in hardware */
+    /* Note we do avoid constructor init-list because of special host/device compilation rules */
+    __CUDA_HOSTDEVICE__ __half(short val) { __x = __short2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(unsigned short val) { __x = __ushort2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(int val) { __x = __int2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(unsigned int val) { __x = __uint2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(long long val) { __x = __ll2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(unsigned long long val) { __x = __ull2half_rn(val).__x; }
+
+    /* Allow automatic casts to supported builtin types, matching all that are permitted with float */
+    __CUDA_HOSTDEVICE__ operator short() const { return __half2short_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(short val) { __x = __short2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned short() const { return __half2ushort_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(unsigned short val) { __x = __ushort2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator int() const { return __half2int_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(int val) { __x = __int2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned int() const { return __half2uint_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(unsigned int val) { __x = __uint2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator long long() const { return __half2ll_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(long long val) { __x = __ll2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned long long() const { return __half2ull_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(unsigned long long val) { __x = __ull2half_rn(val).__x; return *this; }
+
+    /* Boolean conversion - note both 0 and -0 must return false */
+    __CUDA_HOSTDEVICE__ operator bool() const { return (__x & 0x7FFF) != 0; }
+#endif /* defined(__CUDACC__) */
+#endif /* !defined(__CUDA_NO_HALF_CONVERSIONS__) */
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16 operations only supported on arch >= 5.3 */
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+#if !defined(__CUDA_NO_HALF_OPERATORS__)
+/* Some basic arithmetic operations expected of a builtin */
+__device__ __forceinline__ __half operator+(const __half &lh, const __half &rh) { return __hadd(lh, rh); }
+__device__ __forceinline__ __half operator-(const __half &lh, const __half &rh) { return __hsub(lh, rh); }
+__device__ __forceinline__ __half operator*(const __half &lh, const __half &rh) { return __hmul(lh, rh); }
+__device__ __forceinline__ __half operator/(const __half &lh, const __half &rh) { return __hdiv(lh, rh); }
+
+__device__ __forceinline__ __half &operator+=(__half &lh, const __half &rh) { lh = __hadd(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator-=(__half &lh, const __half &rh) { lh = __hsub(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator*=(__half &lh, const __half &rh) { lh = __hmul(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator/=(__half &lh, const __half &rh) { lh = __hdiv(lh, rh); return lh; }
+
+/* Note for increment and decrement we use the raw value 0x3C00 equating to half(1.0f), to avoid the extra conversion */
+__device__ __forceinline__ __half &operator++(__half &h)      { __half_raw one; one.x = 0x3C00; h += one; return h; }
+__device__ __forceinline__ __half &operator--(__half &h)      { __half_raw one; one.x = 0x3C00; h -= one; return h; }
+__device__ __forceinline__ __half  operator++(__half &h, int) { __half ret = h; __half_raw one; one.x = 0x3C00; h += one; return ret; }
+__device__ __forceinline__ __half  operator--(__half &h, int) { __half ret = h; __half_raw one; one.x = 0x3C00; h -= one; return ret; }
+
+/* Unary plus and inverse operators */
+__device__ __forceinline__ __half operator+(const __half &h) { return h; }
+__device__ __forceinline__ __half operator-(const __half &h) { return __hneg(h); }
+
+/* Some basic comparison operations to make it look like a builtin */
+__device__ __forceinline__ bool operator==(const __half &lh, const __half &rh) { return __heq(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half &lh, const __half &rh) { return __hneu(lh, rh); }
+__device__ __forceinline__ bool operator> (const __half &lh, const __half &rh) { return __hgt(lh, rh); }
+__device__ __forceinline__ bool operator< (const __half &lh, const __half &rh) { return __hlt(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half &lh, const __half &rh) { return __hge(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half &lh, const __half &rh) { return __hle(lh, rh); }
+#endif /* !defined(__CUDA_NO_HALF_OPERATORS__) */
+#endif /* __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__) */
+#endif /* defined(__CUDACC__) */
+
+/* __half2 is visible to non-nvcc host compilers */
+struct __CUDA_ALIGN__(4) __half2 {
+    __half x;
+    __half y;
+
+    // All construct/copy/assign/move
+public:
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+    __half2() = default;
+    __CUDA_HOSTDEVICE__ __half2(__half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(__half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); return *this; }
+#else
+    __CUDA_HOSTDEVICE__ __half2() { }
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+    __CUDA_HOSTDEVICE__ __half2(const __half &a, const __half &b) : x(a), y(b) { }
+    __CUDA_HOSTDEVICE__ __half2(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); return *this; }
+
+    /* Convert to/from __half2_raw */
+    __CUDA_HOSTDEVICE__ __half2(const __half2_raw &h2r ) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2_raw &h2r) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); return *this; }
+    __CUDA_HOSTDEVICE__ operator __half2_raw() const { __half2_raw ret; ret.x = 0U; ret.y = 0U; __HALF2_TO_UI(ret) = __HALF2_TO_CUI(*this); return ret; }
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16x2 operations only supported on arch >= 5.3 */
+#if (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)) && !defined(__CUDA_NO_HALF2_OPERATORS__)
+
+__device__ __forceinline__ __half2 operator+(const __half2 &lh, const __half2 &rh) { return __hadd2(lh, rh); }
+__device__ __forceinline__ __half2 operator-(const __half2 &lh, const __half2 &rh) { return __hsub2(lh, rh); }
+__device__ __forceinline__ __half2 operator*(const __half2 &lh, const __half2 &rh) { return __hmul2(lh, rh); }
+__device__ __forceinline__ __half2 operator/(const __half2 &lh, const __half2 &rh) { return __h2div(lh, rh); }
+
+__device__ __forceinline__ __half2& operator+=(__half2 &lh, const __half2 &rh) { lh = __hadd2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator-=(__half2 &lh, const __half2 &rh) { lh = __hsub2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator*=(__half2 &lh, const __half2 &rh) { lh = __hmul2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator/=(__half2 &lh, const __half2 &rh) { lh = __h2div(lh, rh); return lh; }
+
+__device__ __forceinline__ __half2 &operator++(__half2 &h)      { __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hadd2(h, one); return h; }
+__device__ __forceinline__ __half2 &operator--(__half2 &h)      { __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hsub2(h, one); return h; }
+__device__ __forceinline__ __half2  operator++(__half2 &h, int) { __half2 ret = h; __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hadd2(h, one); return ret; }
+__device__ __forceinline__ __half2  operator--(__half2 &h, int) { __half2 ret = h; __half2_raw one; one.x = 0x3C00; one.y = 0x3C00; h = __hsub2(h, one); return ret; }
+
+__device__ __forceinline__ __half2 operator+(const __half2 &h) { return h; }
+__device__ __forceinline__ __half2 operator-(const __half2 &h) { return __hneg2(h); }
+
+__device__ __forceinline__ bool operator==(const __half2 &lh, const __half2 &rh) { return __hbeq2(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half2 &lh, const __half2 &rh) { return __hbneu2(lh, rh); }
+__device__ __forceinline__ bool operator>(const __half2 &lh, const __half2 &rh) { return __hbgt2(lh, rh); }
+__device__ __forceinline__ bool operator<(const __half2 &lh, const __half2 &rh) { return __hblt2(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half2 &lh, const __half2 &rh) { return __hbge2(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half2 &lh, const __half2 &rh) { return __hble2(lh, rh); }
+
+#endif /* __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__) */
+#endif /* defined(__CUDACC__) */
+
+/* Restore warning for multiple assignment operators */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( pop )
+#endif /* defined(_MSC_VER) && _MSC_VER >= 1500 */
+
+/* Restore -Weffc++ warnings from here on */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic pop
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+#undef __CUDA_HOSTDEVICE__
+#undef __CUDA_ALIGN__
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static unsigned short __internal_float2half(const float f, unsigned int &sign, unsigned int &remainder)
+{
+    unsigned int x;
+    unsigned int u;
+    unsigned int result = 0U;
+#if defined(__CUDACC__)
+    (void)memcpy(&x, &f, sizeof(f));
+#else
+    (void)std::memcpy(&x, &f, sizeof(f));
+#endif
+    u = (x & 0x7fffffffU);
+    sign = ((x >> 16U) & 0x8000U);
+    // NaN/+Inf/-Inf
+    if (u >= 0x7f800000U) {
+        remainder = 0U;
+        result = ((u == 0x7f800000U) ? (sign | 0x7c00U) : 0x7fffU);
+    } else if (u > 0x477fefffU) { // Overflows
+        remainder = 0x80000000U;
+        result = (sign | 0x7bffU);
+    } else if (u >= 0x38800000U) { // Normal numbers
+        remainder = u << 19U;
+        u -= 0x38000000U;
+        result = (sign | (u >> 13U));
+    } else if (u < 0x33000001U) { // +0/-0
+        remainder = u;
+        result = sign;
+    } else { // Denormal numbers
+        const unsigned int exponent = u >> 23U;
+        const unsigned int shift = 0x7eU - exponent;
+        unsigned int mantissa = (u & 0x7fffffU);
+        mantissa |= 0x800000U;
+        remainder = mantissa << (32U - shift);
+        result = (sign | (mantissa >> shift));
+    }
+    return static_cast<unsigned short>(result);
+}
+#endif  /* #if !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __double2half(const double x)
+{
+#if defined(__CUDA_ARCH__)
+    __half val;
+    asm("{  cvt.rn.f16.f64 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "d"(x));
+    return val;
+#else
+    // Perform rounding to 11 bits of precision, convert value
+    // to float and call existing float to half conversion.
+    // By pre-rounding to 11 bits we avoid additional rounding
+    // in float to half conversion.
+    unsigned long long int absx;
+    unsigned long long int ux;
+#if defined(__CUDACC__)
+    (void)memcpy(&ux, &x, sizeof(x));
+#else
+    (void)std::memcpy(&ux, &x, sizeof(x));
+#endif
+    absx = (ux & 0x7fffffffffffffffull);
+    if ((absx >= 0x40f0000000000000ull) || (absx <= 0x3e60000000000000ull))
+    {
+        // |x| >= 2^16 or NaN or |x| <= 2^(-25)
+        // double-rounding is not a problem
+        return __float2half(static_cast<float>(x));
+    }
+
+    // here 2^(-25) < |x| < 2^16
+    // prepare shifter value such that x + shifter
+    // done in double precision performs round-to-nearest-even
+    // and (x + shifter) - shifter results in x rounded to
+    // 11 bits of precision. Shifter needs to have exponent of
+    // x plus 53 - 11 = 42 and a leading bit in mantissa to guard
+    // against negative values.
+    // So need to have |x| capped to avoid overflow in exponent.
+    // For inputs that are smaller than half precision minnorm
+    // we prepare fixed shifter exponent.
+    unsigned long long shifterBits = ux & 0x7ff0000000000000ull;
+    if (absx >= 0x3f10000000000000ull)
+    {   // |x| >= 2^(-14)
+        // add 42 to exponent bits
+        shifterBits += 42ull << 52;
+    }
+    else
+    {   // 2^(-25) < |x| < 2^(-14), potentially results in denormal
+        // set exponent bits to 42 - 14 + bias
+        shifterBits = ((42ull - 14 + 1023) << 52);
+    }
+    // set leading mantissa bit to protect against negative inputs
+    shifterBits |= 1ull << 51;
+    double shifter;
+#if defined(__CUDACC__)
+    (void)memcpy(&shifter, &shifterBits, sizeof(shifterBits));
+#else
+    (void)std::memcpy(&shifter, &shifterBits, sizeof(shifterBits));
+#endif
+    double xShiftRound = x + shifter;
+
+    // Prevent the compiler from optimizing away x + shifter - shifter
+    // by doing intermediate memcopy and harmless bitwize operation
+    unsigned long long int xShiftRoundBits;
+#if defined(__CUDACC__)
+    (void)memcpy(&xShiftRoundBits, &xShiftRound, sizeof(xShiftRound));
+#else
+    (void)std::memcpy(&xShiftRoundBits, &xShiftRound, sizeof(xShiftRound));
+#endif
+
+    // the value is positive, so this operation doesn't change anything
+    xShiftRoundBits &= 0x7fffffffffffffffull;
+
+#if defined(__CUDACC__)
+    (void)memcpy(&xShiftRound, &xShiftRoundBits, sizeof(xShiftRound));
+#else
+    (void)std::memcpy(&xShiftRound, &xShiftRoundBits, sizeof(xShiftRound));
+#endif
+
+    double xRounded = xShiftRound - shifter;
+    float  xRndFlt = static_cast<float>(xRounded);
+    __half res =  __float2half(xRndFlt);
+    return res;
+#endif
+}
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rz.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rm.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign != 0U)) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rp.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign;
+    unsigned int remainder;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign == 0U)) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a)
+{
+    __half2 val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a));
+#else
+    val = __half2(__float2half_rn(a), __float2half_rn(a));
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b)
+{
+    __half2 val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  cvt.rn.f16.f32 high, %2;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a), "f"(b));
+#else
+    val = __half2(__float2half_rn(a), __float2half_rn(b));
+#endif
+    return val;
+}
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static float __internal_half2float(const unsigned short h)
+{
+    unsigned int sign = ((static_cast<unsigned int>(h) >> 15U) & 1U);
+    unsigned int exponent = ((static_cast<unsigned int>(h) >> 10U) & 0x1fU);
+    unsigned int mantissa = ((static_cast<unsigned int>(h) & 0x3ffU) << 13U);
+    float f;
+    if (exponent == 0x1fU) { /* NaN or Inf */
+        sign = ((mantissa != 0U) ? 0U : sign);
+        mantissa = ((mantissa != 0U) ? 0x7fffffU : 0U);
+        exponent = 0xffU;
+    } else if (exponent == 0U) { /* Denorm or Zero */
+        if (mantissa != 0U) {
+            unsigned int msb;
+            exponent = 0x71U;
+            do {
+                msb = (mantissa & 0x400000U);
+                mantissa <<= 1U; /* normalize */
+                --exponent;
+            } while (msb == 0U);
+            mantissa &= 0x7fffffU; /* 1.mantissa is implicit */
+        }
+    } else {
+        exponent += 0x70U;
+    }
+    unsigned int u = ((sign << 31U) | (exponent << 23U) | mantissa);
+#if defined(__CUDACC__)
+    (void)memcpy(&f, &u, sizeof(u));
+#else
+    (void)std::memcpy(&f, &u, sizeof(u));
+#endif
+    return f;
+}
+#endif  /* !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.f32.f16 %0, %1;}\n" : "=f"(val) : "h"(__HALF_TO_CUS(a)));
+#else
+    val = __internal_half2float(static_cast<__half_raw>(a).x);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+#else
+    val = __internal_half2float(static_cast<__half2_raw>(a).x);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+#else
+    val = __internal_half2float(static_cast<__half2_raw>(a).y);
+#endif
+    return val;
+}
+
+/* Intrinsic functions only available to nvcc compilers */
+#if defined(__CUDACC__)
+
+/* CUDA vector-types compatible vector creation function (note returns __half2, not half2) */
+__VECTOR_FUNCTIONS_DECL__ __half2 make_half2(__half x, __half y)
+{
+    __half2 t; t.x = x; t.y = y; return t;
+}
+#undef __VECTOR_FUNCTIONS_DECL__
+
+
+/* Definitions of intrinsics */
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 f)
+{
+    __half2 val = __floats2half2_rn(f.x, f.y);
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 l)
+{
+    float hi_float;
+    float lo_float;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(lo_float) : "r"(__HALF2_TO_CUI(l)));
+
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(hi_float) : "r"(__HALF2_TO_CUI(l)));
+#else
+    lo_float = __internal_half2float(((__half2_raw)l).x);
+    hi_float = __internal_half2float(((__half2_raw)l).y);
+#endif
+    return make_float2(lo_float, hi_float);
+}
+__CUDA_FP16_DECL__ int __half2int_rn(__half h)
+{
+    int i;
+    asm("cvt.rni.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ int __half2int_rz(__half h)
+{
+    int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+#else
+    const float f = __half2float(h);
+                i = static_cast<int>(f);
+    const int max_val = (int)0x7fffffffU;
+    const int min_val = (int)0x80000000U;
+    // saturation fixup
+    if (f != f) {
+        // NaN
+        i = 0;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    }
+#endif
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_rd(__half h)
+{
+    int i;
+    asm("cvt.rmi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_ru(__half h)
+{
+    int i;
+    asm("cvt.rpi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __int2half_rn(int i)
+{
+    __half h;
+#if defined(__CUDA_ARCH__)
+    asm("cvt.rn.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+#else
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rz(int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rd(int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_ru(int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ short int __half2short_rn(__half h)
+{
+    short int i;
+    asm("cvt.rni.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ short int __half2short_rz(__half h)
+{
+    short int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+#else
+    const float f = __half2float(h);
+                i = static_cast<short int>(f);
+    const short int max_val = (short int)0x7fffU;
+    const short int min_val = (short int)0x8000U;
+    // saturation fixup
+    if (f != f) {
+        // NaN
+        i = 0;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    }
+#endif
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_rd(__half h)
+{
+    short int i;
+    asm("cvt.rmi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_ru(__half h)
+{
+    short int i;
+    asm("cvt.rpi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __short2half_rn(short int i)
+{
+    __half h;
+#if defined __CUDA_ARCH__
+    asm("cvt.rn.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+#else
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rz(short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rd(short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_ru(short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(__half h)
+{
+    unsigned int i;
+    asm("cvt.rni.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned int __half2uint_rz(__half h)
+{
+    unsigned int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+#else
+    const float f = __half2float(h);
+                i = static_cast<unsigned int>(f);
+    const unsigned int max_val = 0xffffffffU;
+    const unsigned int min_val = 0U;
+    // saturation fixup
+    if (f != f) {
+        // NaN
+        i = 0U;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    }
+#endif
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(__half h)
+{
+    unsigned int i;
+    asm("cvt.rmi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(__half h)
+{
+    unsigned int i;
+    asm("cvt.rpi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __uint2half_rn(unsigned int i)
+{
+    __half h;
+#if defined __CUDA_ARCH__
+    asm("cvt.rn.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+#else
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rz(unsigned int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rd(unsigned int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_ru(unsigned int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rni.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned short int __half2ushort_rz(__half h)
+{
+    unsigned short int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+#else
+    const float f = __half2float(h);
+                i = static_cast<unsigned short int>(f);
+    const unsigned short int max_val = 0xffffU;
+    const unsigned short int min_val = 0U;
+    // saturation fixup
+    if (f != f) {
+        // NaN
+        i = 0U;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    }
+#endif
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rmi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(__half h)
+{
+    unsigned short int i;
+    asm("cvt.rpi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ushort2half_rn(unsigned short int i)
+{
+    __half h;
+#if defined __CUDA_ARCH__
+    asm("cvt.rn.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+#else
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rz(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rd(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_ru(unsigned short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rni.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned long long int __half2ull_rz(__half h)
+{
+    unsigned long long int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+#else
+    const float f = __half2float(h);
+                i = static_cast<unsigned long long int>(f);
+    const unsigned long long int max_val = 0xffffffffffffffffULL;
+    const unsigned long long int min_val = 0ULL;
+    // saturation fixup
+    if (f != f) {
+        // NaN
+        i = 0x8000000000000000ULL;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    }
+#endif
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rmi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(__half h)
+{
+    unsigned long long int i;
+    asm("cvt.rpi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ull2half_rn(unsigned long long int i)
+{
+    __half h;
+#if defined(__CUDA_ARCH__)
+    asm("cvt.rn.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+#else
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rz(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rd(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_ru(unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ long long int __half2ll_rn(__half h)
+{
+    long long int i;
+    asm("cvt.rni.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ long long int __half2ll_rz(__half h)
+{
+    long long int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+#else
+    const float f = __half2float(h);
+                i = static_cast<long long int>(f);
+    const long long int max_val = (long long int)0x7fffffffffffffffULL;
+    const long long int min_val = (long long int)0x8000000000000000ULL;
+    // saturation fixup
+    if (f != f) {
+        // NaN
+        i = min_val;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    }
+#endif
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_rd(__half h)
+{
+    long long int i;
+    asm("cvt.rmi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_ru(__half h)
+{
+    long long int i;
+    asm("cvt.rpi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_US(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ll2half_rn(long long int i)
+{
+    __half h;
+#if defined(__CUDA_ARCH__)
+    asm("cvt.rn.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+#else
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rz(long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rd(long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_ru(long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ __half htrunc(const __half h)
+{
+    __half r;
+    asm("cvt.rzi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hceil(const __half h)
+{
+    __half r;
+    asm("cvt.rpi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hfloor(const __half h)
+{
+    __half r;
+    asm("cvt.rmi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hrint(const __half h)
+{
+    __half r;
+    asm("cvt.rni.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rzi.f16.f16 low, low;\n"
+        "  cvt.rzi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rpi.f16.f16 low, low;\n"
+        "  cvt.rpi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rmi.f16.f16 low, low;\n"
+        "  cvt.rmi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rni.f16.f16 low, low;\n"
+        "  cvt.rni.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 l, const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {alow,blow};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)), "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 l, const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {ahigh,bhigh};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)), "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __low2half(const __half2 h)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, low;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(h)));
+    return ret;
+}
+__CUDA_FP16_DECL__ int __hisinf(const __half a)
+{
+    if (__HALF_TO_CUS(a) == 0xFC00) {
+        return -1;
+    }
+    if (__HALF_TO_CUS(a) == 0x7C00) {
+        return 1;
+    }
+    return 0;
+}
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 l)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 l)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(l)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __high2half(const __half2 h)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, high;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(h)));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half l, const __half h)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%2};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(l)), "h"(__HALF_TO_CUS(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half lh)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%1};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(lh)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 lh)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(lh)));
+    return val;
+}
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h)
+{
+    return (short int)__HALF_TO_CUS(h);
+}
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h)
+{
+    return __HALF_TO_CUS(h);
+}
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = (unsigned short int)i;
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = i;
+    return h;
+}
+
+#if __CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)
+/******************************************************************************
+*                           __half, __half2 warp shuffle                     *
+******************************************************************************/
+#define __SHUFFLE_HALF2_MACRO(name) /* do */ {\
+   __half2 r; \
+   asm volatile ("{"#name" %0,%1,%2,%3;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c)); \
+   return r; \
+} /* while(0) */
+
+#define __SHUFFLE_SYNC_HALF2_MACRO(name) /* do */ {\
+   __half2 r; \
+   asm volatile ("{"#name" %0,%1,%2,%3,%4;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c), "r"(mask)); \
+   return r; \
+} /* while(0) */
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+
+__CUDA_FP16_DECL__ __half2 __shfl(__half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.idx.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up(__half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = (warpSize - width) << 8;
+    __SHUFFLE_HALF2_MACRO(shfl.up.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down(__half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.down.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor(__half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_HALF2_MACRO(shfl.bfly.b32)
+}
+
+#endif /* !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700 */
+
+__CUDA_FP16_DECL__ __half2 __shfl_sync(unsigned mask, __half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.idx.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(unsigned mask, __half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = (warpSize - width) << 8;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.up.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(unsigned mask, __half2 var, unsigned int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.down.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(unsigned mask, __half2 var, int delta, int width)
+{
+    int warpSize;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warpSize));
+    int c = ((warpSize - width) << 8) | 0x1f;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.bfly.b32)
+}
+
+#undef __SHUFFLE_HALF2_MACRO
+#undef __SHUFFLE_SYNC_HALF2_MACRO
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+
+__CUDA_FP16_DECL__ __half __shfl(__half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up(__half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_up(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down(__half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_down(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor(__half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_xor(temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /* !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700 */
+
+__CUDA_FP16_DECL__ __half __shfl_sync(unsigned mask, __half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up_sync(unsigned mask, __half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_up_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down_sync(unsigned mask, __half var, unsigned int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_down_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(unsigned mask, __half var, int delta, int width)
+{
+    __half2 temp1 = __halves2half2(var, var);
+    __half2 temp2 = __shfl_xor_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /*__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)*/
+/******************************************************************************
+*               __half and __half2 __ldg,__ldcg,__ldca,__ldcs                *
+******************************************************************************/
+
+#if defined(__cplusplus) && (__CUDA_ARCH__ >= 320 || !defined(__CUDA_ARCH__))
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __LDG_PTR   "l"
+#else
+#define __LDG_PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.nc.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldg(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.nc.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cg.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcg(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.cg.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.ca.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldca(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.ca.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cs.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcs(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.cs.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldlu(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.lu.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldlu(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.lu.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcv(const  __half2 *ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cv.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcv(const __half *ptr)
+{
+    __half ret;
+    asm ("ld.global.cv.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ void __stwb(__half2 *ptr, __half2 value)
+{
+    asm ("st.global.wb.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwb(__half *ptr, __half value)
+{
+    asm ("st.global.wb.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcg(__half2 *ptr, __half2 value)
+{
+    asm ("st.global.cg.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcg(__half *ptr, __half value)
+{
+    asm ("st.global.cg.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcs(__half2 *ptr, __half2 value)
+{
+    asm ("st.global.cs.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcs(__half *ptr, __half value)
+{
+    asm ("st.global.cs.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwt(__half2 *ptr, __half2 value)
+{
+    asm ("st.global.wt.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwt(__half *ptr, __half value)
+{
+    asm ("st.global.wt.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+#undef __LDG_PTR
+#endif /*defined(__cplusplus) && (__CUDA_ARCH__ >= 320 || !defined(__CUDA_ARCH__))*/
+#if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
+/******************************************************************************
+*                             __half2 comparison                             *
+******************************************************************************/
+#define __COMPARISON_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{ "#name".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.eq)
+}
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ne)
+}
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.le)
+}
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ge)
+}
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.lt)
+}
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gt)
+}
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.equ)
+}
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.neu)
+}
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.leu)
+}
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.geu)
+}
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ltu)
+}
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gtu)
+}
+#undef __COMPARISON_OP_HALF2_MACRO
+#define __BOOL_COMPARISON_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{ "#name".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   if (__HALF2_TO_CUI(val) == 0x3C003C00) \
+      return true; \
+   else  \
+      return false; \
+} /* while(0) */
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.eq)
+}
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ne)
+}
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.le)
+}
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ge)
+}
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.lt)
+}
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gt)
+}
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.equ)
+}
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.neu)
+}
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.leu)
+}
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.geu)
+}
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ltu)
+}
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gtu)
+}
+#undef __BOOL_COMPARISON_OP_HALF2_MACRO
+/******************************************************************************
+*                             __half comparison                              *
+******************************************************************************/
+#define __COMPARISON_OP_HALF_MACRO(name) /* do */ {\
+   unsigned short val; \
+   asm( "{ .reg .pred __$temp3;\n" \
+        "  setp."#name".f16  __$temp3, %1, %2;\n" \
+        "  selp.u16 %0, 1, 0, __$temp3;}" \
+        : "=h"(val) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(b))); \
+   return val ? true : false; \
+} /* while(0) */
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(eq)
+}
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ne)
+}
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(le)
+}
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ge)
+}
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(lt)
+}
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gt)
+}
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(equ)
+}
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(neu)
+}
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(leu)
+}
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(geu)
+}
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ltu)
+}
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gtu)
+}
+#undef __COMPARISON_OP_HALF_MACRO
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul)
+}
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.sat)
+}
+__CUDA_FP16_DECL__ __half2 __h2div(__half2 a, __half2 b) {
+    __half ha, hb;
+
+    ha = __low2half(a);
+    hb = __low2half(b);
+
+    __half v1 = __hdiv(ha, hb);
+
+    ha = __high2half(a);
+    hb = __high2half(b);
+
+    __half v2 = __hdiv(ha, hb);
+
+    return __halves2half2(v1, v2);
+}
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add)
+}
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub)
+}
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul)
+}
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add.sat)
+}
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub.sat)
+}
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul.sat)
+}
+
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn)
+}
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.sat)
+}
+__CUDA_FP16_DECL__ __half __hdiv(__half a, __half b) {
+    __half v, abs, den;
+    __HALF_TO_US(den) = 0x008F;
+    float fa, fb, fv, rcp;
+
+    fa = __half2float(a);
+    fb = __half2float(b);
+
+    asm("{rcp.approx.ftz.f32 %0, %1;\n}" :"=f"(rcp) : "f"(fb));
+
+    fv = rcp * fa;
+
+    v = __float2half(fv);
+    __HALF_TO_US(abs) = (unsigned short)(((unsigned int)__HALF_TO_CUS(v)) & 0x00007FFF);
+    if (__hlt(abs, den) && (!(__HALF_TO_CUS(abs) == 0x0000))) {
+        float err = __fmaf_rn(-fb, fv, fa);
+        fv = __fmaf_rn(rcp, err, fv);
+        v = __float2half(fv);
+    }
+    return v;
+}
+
+/******************************************************************************
+*                             __half2 functions                  *
+******************************************************************************/
+#define __SPEC_CASE2(i,r, spc, ulp) \
+   "{.reg.b32 spc, ulp, p;\n"\
+   "  mov.b32 spc,"#spc";\n"\
+   "  mov.b32 ulp,"#ulp";\n"\
+   "  set.eq.f16x2.f16x2 p,"#i", spc;\n"\
+   "  fma.rn.f16x2 "#r",p,ulp,"#r";\n}\n"
+#define __SPEC_CASE(i,r, spc, ulp) \
+   "{.reg.b16 spc, ulp, p;\n"\
+   "  mov.b16 spc,"#spc";\n"\
+   "  mov.b16 ulp,"#ulp";\n"\
+   "  set.eq.f16.f16 p,"#i", spc;\n"\
+   "  fma.rn.f16 "#r",p,ulp,"#r";\n}\n"
+#define __APPROX_FCAST(fun) /* do */ {\
+   __half val;\
+   asm("{.reg.b32         f;        \n"\
+                " .reg.b16         r;        \n"\
+                "  mov.b16         r,%1;     \n"\
+                "  cvt.f32.f16     f,r;      \n"\
+                "  "#fun".approx.f32   f,f;  \n"\
+                "  cvt.rn.f16.f32      r,f;  \n"\
+                "  mov.b16         %0,r;     \n"\
+                "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));\
+   return val;\
+} /* while(0) */
+#define __APPROX_FCAST2(fun) /* do */ {\
+   __half2 val;\
+   asm("{.reg.b16         hl, hu;         \n"\
+                " .reg.b32         fl, fu;         \n"\
+                "  mov.b32         {hl, hu}, %1;   \n"\
+                "  cvt.f32.f16     fl, hl;         \n"\
+                "  cvt.f32.f16     fu, hu;         \n"\
+                "  "#fun".approx.f32   fl, fl;     \n"\
+                "  "#fun".approx.f32   fu, fu;     \n"\
+                "  cvt.rn.f16.f32      hl, fl;     \n"\
+                "  cvt.rn.f16.f32      hu, fu;     \n"\
+                "  mov.b32         %0, {hl, hu};   \n"\
+                "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));       \
+   return val;\
+} /* while(0) */
+static __device__ __forceinline__ float __float_simpl_sinf(float);
+static __device__ __forceinline__ float __float_simpl_cosf(float);
+__CUDA_FP16_DECL__ __half __hsin_internal(const __half a) {
+    float f = __half2float(a);
+    f = __float_simpl_sinf(f);
+    return __float2half_rn(f);
+}
+__CUDA_FP16_DECL__ __half hsin(const __half a) {
+    __half r = __hsin_internal(a);
+    asm("{\n\t"
+        "  .reg.b16 i,r,t;     \n\t"
+        "  mov.b16 r, %0;      \n\t"
+        "  mov.b16 i, %1;      \n\t"
+        "  mov.b16 t, 0x8000;  \n\t"
+        "  and.b16 t,r,t;      \n\t"
+        __SPEC_CASE(i, r, 0X32B3, 0x0800)
+        __SPEC_CASE(i, r, 0X5CB0, 0x1000)
+        __SPEC_CASE(i, r, 0XB2B3, 0x8800)
+        __SPEC_CASE(i, r, 0XDCB0, 0x9000)
+        "  or.b16  r,r,t;      \n\t"
+        "  mov.b16 %0, r;      \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a) {
+    __half l = __low2half(a);
+    __half h = __high2half(a);
+    __half2 r = __halves2half2(__hsin_internal(l), __hsin_internal(h));
+    asm("{\n\t"
+        "  .reg.b32 i,r,t;             \n\t"
+        "  mov.b32 r, %0;              \n\t"
+        "  mov.b32 i, %1;              \n\t"
+        "  and.b32 t, r, 0x80008000;   \n\t"
+        __SPEC_CASE2(i, r, 0X32B332B3, 0x08000800)
+        __SPEC_CASE2(i, r, 0X5CB05CB0, 0x10001000)
+        __SPEC_CASE2(i, r, 0XB2B3B2B3, 0x88008800)
+        __SPEC_CASE2(i, r, 0XDCB0DCB0, 0x90009000)
+        "  or.b32  r, r, t;            \n\t"
+        "  mov.b32 %0, r;              \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __hcos_internal(const __half a) {
+    float f = __half2float(a);
+    f = __float_simpl_cosf(f);
+    return __float2half_rn(f);
+}
+__CUDA_FP16_DECL__ __half hcos(const __half a) {
+    __half r = __hcos_internal(a);
+    asm("{\n\t"
+        "  .reg.b16 i,r;        \n\t"
+        "  mov.b16 r, %0;       \n\t"
+        "  mov.b16 i, %1;       \n\t"
+        __SPEC_CASE(i, r, 0X2B7C, 0x1000)
+        __SPEC_CASE(i, r, 0XAB7C, 0x1000)
+        "  mov.b16 %0, r;       \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a) {
+    __half l = __low2half(a);
+    __half h = __high2half(a);
+    __half2 r = __halves2half2(__hcos_internal(l), __hcos_internal(h));
+    asm("{\n\t"
+        "  .reg.b32 i,r;   \n\t"
+        "  mov.b32 r, %0;  \n\t"
+        "  mov.b32 i, %1;  \n\t"
+        __SPEC_CASE2(i, r, 0X2B7C2B7C, 0x10001000)
+        __SPEC_CASE2(i, r, 0XAB7CAB7C, 0x10001000)
+        "  mov.b32 %0, r;  \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+static __device__ __forceinline__ float __internal_trig_reduction_kernel(float a, int *quadrant)
+{
+    float j, t;
+    int q;
+    q = __float2int_rn(a * 0.636619772F);
+    j = (float)q;
+    t = __fmaf_rn(-j, 1.5707962512969971e+000F, a);
+    t = __fmaf_rn(-j, 7.5497894158615964e-008F, t);
+    *quadrant = q;
+    return t;
+}
+static __device__ __forceinline__ float __internal_sin_cos_kernel(float x, int i)
+{
+    float x2, z;
+    x2 = x*x;
+
+    if (i & 1) {
+        z = 2.44331571e-5F;
+        z = __fmaf_rn(z, x2, -1.38873163e-3F);
+    }
+    else {
+        z = -1.95152959e-4F;
+        z = __fmaf_rn(z, x2, 8.33216087e-3F);
+    }
+    if (i & 1) {
+        z = __fmaf_rn(z, x2, 4.16666457e-2F);
+        z = __fmaf_rn(z, x2, -5.00000000e-1F);
+    }
+    else {
+        z = __fmaf_rn(z, x2, -1.66666546e-1F);
+        z = __fmaf_rn(z, x2, 0.0F);
+    }
+    x = __fmaf_rn(z, x, x);
+    if (i & 1) {
+        x = __fmaf_rn(z, x2, 1.0F);
+    }
+    if (i & 2) {
+        x = __fmaf_rn(x, -1.0F, 0.0F);
+    }
+    return x;
+}
+static __device__ __forceinline__ float __float_simpl_sinf(float a)
+{
+    float z;
+    int i;
+    if (::isinf(a)) {
+        a = a * 0.0F;
+    }
+    a = __internal_trig_reduction_kernel(a, &i);
+    z = __internal_sin_cos_kernel(a, i);
+    return z;
+}
+static __device__ __forceinline__ float __float_simpl_cosf(float a)
+{
+    float z;
+    int i;
+    if (::isinf(a)) {
+        a = a * 0.0F;
+    }
+    a = __internal_trig_reduction_kernel(a, &i);
+    i++;
+    z = __internal_sin_cos_kernel(a, i);
+    return z;
+}
+
+__CUDA_FP16_DECL__ __half hexp(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C;           \n"
+        " .reg.b16         h,r;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  mov.b32         C, 0x3fb8aa3b;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  ex2.approx.f32      f,f;        \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X1F79, 0x9400)
+        __SPEC_CASE(h, r, 0X25CF, 0x9400)
+        __SPEC_CASE(h, r, 0XC13B, 0x0400)
+        __SPEC_CASE(h, r, 0XC1EF, 0x0200)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu, C;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x3fb8aa3b;  \n"
+        "  mul.f32         fl,fl,C;        \n"
+        "  mul.f32         fu,fu,C;        \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0X1F791F79, 0x94009400)
+        __SPEC_CASE2(h, r, 0X25CF25CF, 0x94009400)
+        __SPEC_CASE2(h, r, 0XC13BC13B, 0x04000400)
+        __SPEC_CASE2(h, r, 0XC1EFC1EF, 0x02000200)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp2(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, ULP;         \n"
+        " .reg.b16         r;              \n"
+        "  mov.b16         r,%1;           \n"
+        "  cvt.f32.f16     f,r;            \n"
+        "  ex2.approx.f32      f,f;        \n"
+        "  mov.b32         ULP, 0x33800000;\n"
+        "  fma.rn.f32      f,f,ULP,f;      \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, ULP;    \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  mov.b32         ULP, 0x33800000;\n"
+        "  fma.rn.f32      fl,fl,ULP,fl;   \n"
+        "  fma.rn.f32      fu,fu,ULP,fu;   \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         %0, {hl, hu};   \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h,r;            \n"
+        " .reg.b32         f, C;           \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  mov.b32         C, 0x40549A78;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  ex2.approx.f32      f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x34DE, 0x9800)
+        __SPEC_CASE(h, r, 0x9766, 0x9000)
+        __SPEC_CASE(h, r, 0x9972, 0x1000)
+        __SPEC_CASE(h, r, 0xA5C4, 0x1000)
+        __SPEC_CASE(h, r, 0xBF0A, 0x8100)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu, C;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x40549A78;  \n"
+        "  mul.f32         fl,fl,C;        \n"
+        "  mul.f32         fu,fu,C;        \n"
+        "  ex2.approx.f32      fl, fl;     \n"
+        "  ex2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0x34DE34DE, 0x98009800)
+        __SPEC_CASE2(h, r, 0x97669766, 0x90009000)
+        __SPEC_CASE2(h, r, 0x99729972, 0x10001000)
+        __SPEC_CASE2(h, r, 0xA5C4A5C4, 0x10001000)
+        __SPEC_CASE2(h, r, 0xBF0ABF0A, 0x81008100)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog2(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f;              \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.f32      f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(r, r, 0xA2E2, 0x8080)
+        __SPEC_CASE(r, r, 0xBF46, 0x9400)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, r, p;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  lg2.approx.f32      fl, fl;     \n"
+        "  lg2.approx.f32      fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(r, r, 0xA2E2A2E2, 0x80808080)
+        __SPEC_CASE2(r, r, 0xBF46BF46, 0x94009400)
+        "  mov.b32         %0, r;          \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C;           \n"
+        " .reg.b16         r,h;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  lg2.approx.f32      f,f;        \n"
+        "  mov.b32         C, 0x3f317218;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X160D, 0x9C00)
+        __SPEC_CASE(h, r, 0X3BFE, 0x8010)
+        __SPEC_CASE(h, r, 0X3C0B, 0x8080)
+        __SPEC_CASE(h, r, 0X6051, 0x1C00)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.f32      fl, fl;         \n"
+        "  lg2.approx.f32      fu, fu;         \n"
+        "  mov.b32         C, 0x3f317218;      \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0X160D160D, 0x9C009C00)
+        __SPEC_CASE2(h, r, 0X3BFE3BFE, 0x80108010)
+        __SPEC_CASE2(h, r, 0X3C0B3C0B, 0x80808080)
+        __SPEC_CASE2(h, r, 0X60516051, 0x1C001C00)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f, C;           \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.f32      f, f;       \n"
+        "  mov.b32         C, 0x3E9A209B;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x338F, 0x1000)
+        __SPEC_CASE(h, r, 0x33F8, 0x9000)
+        __SPEC_CASE(h, r, 0x57E1, 0x9800)
+        __SPEC_CASE(h, r, 0x719D, 0x9C00)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.f32      fl, fl;         \n"
+        "  lg2.approx.f32      fu, fu;         \n"
+        "  mov.b32         C, 0x3E9A209B;      \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0x338F338F, 0x10001000)
+        __SPEC_CASE2(h, r, 0x33F833F8, 0x90009000)
+        __SPEC_CASE2(h, r, 0x57E157E1, 0x98009800)
+        __SPEC_CASE2(h, r, 0x719D719D, 0x9C009C00)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+#undef __SPEC_CASE2
+#undef __SPEC_CASE
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a) {
+    __APPROX_FCAST2(rcp)
+}
+__CUDA_FP16_DECL__ __half hrcp(const __half a) {
+    __APPROX_FCAST(rcp)
+}
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a) {
+    __APPROX_FCAST2(rsqrt)
+}
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a) {
+    __APPROX_FCAST(rsqrt)
+}
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a) {
+    __APPROX_FCAST2(sqrt)
+}
+__CUDA_FP16_DECL__ __half hsqrt(const __half a) {
+    __APPROX_FCAST(sqrt)
+}
+#undef __APPROX_FCAST
+#undef __APPROX_FCAST2
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a)
+{
+    __half2 r;
+    asm("{set.nan.f16x2.f16x2 %0,%1,%2;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ bool __hisnan(const __half a)
+{
+    __half r;
+    asm("{set.nan.f16.f16 %0,%1,%2;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(a)));
+    return __HALF_TO_CUS(r) != 0U;
+}
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a)
+{
+    __half2 r;
+    asm("{neg.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __hneg(const __half a)
+{
+    __half r;
+    asm("{neg.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a)
+{
+    __half2 r;
+    asm("{abs.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __habs(const __half a)
+{
+    __half r;
+    asm("{abs.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+
+__CUDA_FP16_DECL__ __half2 __hcmadd(const __half2 a, const __half2 b, const __half2 c)
+{
+    // fast version of complex multiply-accumulate
+    // (a.re, a.im) * (b.re, b.im) + (c.re, c.im)
+    // acc.re = (c.re + a.re*b.re) - a.im*b.im
+    // acc.im = (c.im + a.re*b.im) + a.im*b.re
+    const __half2 a_re = __half2half2(a.x);
+          __half2 acc  = __hfma2(a_re, b, c);
+    const __half2 a_im = __half2half2(a.y);
+    const __half2 ib   = __halves2half2(__hneg(b.y), b.x);
+                  acc  = __hfma2(a_im, ib, acc);
+    return acc;
+}
+#endif /*__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)*/
+
+#if __CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half __hmax(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(max)
+}
+__CUDA_FP16_DECL__ __half __hmin(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(min)
+}
+__CUDA_FP16_DECL__ __half __hmax_nan(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(max.NaN)
+}
+__CUDA_FP16_DECL__ __half __hmin_nan(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(min.NaN)
+}
+__CUDA_FP16_DECL__ __half __hfma_relu(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.relu)
+}
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half2 __hmax2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(max)
+}
+__CUDA_FP16_DECL__ __half2 __hmin2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(min)
+}
+__CUDA_FP16_DECL__ __half2 __hmax2_nan(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(max.NaN)
+}
+__CUDA_FP16_DECL__ __half2 __hmin2_nan(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(min.NaN)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_relu(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.relu)
+}
+#endif /*__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)*/
+
+/* Define __PTR for atomicAdd prototypes below, undef after done */
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __PTR   "l"
+#else
+#define __PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600
+
+__CUDA_FP16_DECL__  __half2 atomicAdd(__half2 *address, __half2 val) {
+    __half2 r;
+    asm volatile ("{ atom.add.noftz.f16x2 %0,[%1],%2; }\n"
+                  : "=r"(__HALF2_TO_UI(r)) : __PTR(address), "r"(__HALF2_TO_CUI(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600*/
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700
+
+__CUDA_FP16_DECL__  __half atomicAdd(__half *address, __half val) {
+    __half r;
+    asm volatile ("{ atom.add.noftz.f16 %0,[%1],%2; }\n"
+                  : "=h"(__HALF_TO_US(r))
+                  : __PTR(address), "h"(__HALF_TO_CUS(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700*/
+
+#undef __PTR
+
+#undef __CUDA_FP16_DECL__
+#endif /* defined(__CUDACC__) */
+#endif /* defined(__cplusplus) */
+
+#undef __TERNARY_OP_HALF2_MACRO
+#undef __TERNARY_OP_HALF_MACRO
+#undef __BINARY_OP_HALF2_MACRO
+#undef __BINARY_OP_HALF_MACRO
+
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+#undef __CUDA_FP16_DECL__
+ 
+/* Define first-class types "half" and "half2", unless user specifies otherwise via "#define CUDA_NO_HALF" */
+/* C cannot ever have these types defined here, because __half and __half2 are C++ classes */
+#if defined(__cplusplus) && !defined(CUDA_NO_HALF)
+typedef __half half;
+typedef __half2 half2;
+// for consistency with __nv_bfloat16
+typedef __half      __nv_half;
+typedef __half2     __nv_half2;
+typedef __half_raw  __nv_half_raw;
+typedef __half2_raw __nv_half2_raw;
+typedef __half        nv_half;
+typedef __half2       nv_half2;
+#endif /* defined(__cplusplus) && !defined(CUDA_NO_HALF) */
+
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+#undef __CPP_VERSION_AT_LEAST_11_FP16
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+
+#endif /* end of include guard: __CUDA_FP16_HPP__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-11/cuda_fp16.h b/cupy/_core/include/cupy/_cuda/cuda-11/cuda_fp16.h
new file mode 100644
index 0000000..8dc1c29
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-11/cuda_fp16.h
@@ -0,0 +1,3794 @@
+/*
+* Copyright 1993-2021 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+/**
+* \defgroup CUDA_MATH_INTRINSIC_HALF Half Precision Intrinsics
+* This section describes half precision intrinsic functions that are
+* only supported in device code.
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_ARITHMETIC Half Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_ARITHMETIC Half2 Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_COMPARISON Half Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_COMPARISON Half2 Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_MISC Half Precision Conversion and Data Movement
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_FUNCTIONS Half Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_FUNCTIONS Half2 Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+#ifndef __CUDA_FP16_H__
+#define __CUDA_FP16_H__
+
+#define ___CUDA_FP16_STRINGIFY_INNERMOST(x) #x
+#define __CUDA_FP16_STRINGIFY(x) ___CUDA_FP16_STRINGIFY_INNERMOST(x)
+
+#if defined(__cplusplus)
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__CUDACC__) */
+
+#define __CUDA_FP16_TYPES_EXIST__
+
+/* Forward-declaration of structures defined in "cuda_fp16.hpp" */
+
+/**
+ * \brief half datatype 
+ * 
+ * \details This structure implements the datatype for storing 
+ * half-precision floating-point numbers. The structure implements 
+ * assignment operators and type conversions. 
+ * 16 bits are being used in total: 1 sign bit, 5 bits for the exponent, 
+ * and the significand is being stored in 10 bits. 
+ * The total precision is 11 bits. There are 15361 representable 
+ * numbers within the interval [0.0, 1.0], endpoints included. 
+ * On average we have log10(2**11) ~ 3.311 decimal digits. 
+ * 
+ * \internal
+ * \req IEEE 754-2008 compliant implementation of half-precision 
+ * floating-point numbers. 
+ * \endinternal
+ */
+struct __half;
+
+/**
+ * \brief half2 datatype
+ * 
+ * \details This structure implements the datatype for storing two 
+ * half-precision floating-point numbers. 
+ * The structure implements assignment operators and type conversions. 
+ * 
+ * \internal
+ * \req Vectorified version of half. 
+ * \endinternal
+ */
+struct __half2;
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts double number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts double number \p a to half precision in round-to-nearest-even mode.
+* \param[in] a - double. Is only being read.
+* \returns half
+* - \p a converted to half.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __double2half(const double a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value. 
+* 
+* \details Converts float number \p a to half precision in round-to-nearest-even mode. 
+* \param[in] a - float. Is only being read. 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts float number \p a to half precision in round-to-nearest-even mode.
+* \param[in] a - float. Is only being read. 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-towards-zero mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-towards-zero mode.
+* \param[in] a - float. Is only being read. 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-down mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-down mode.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-up mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-up mode.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts \p half number to float.
+* 
+* \details Converts half number \p a to float.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns float
+* - \p a converted to float. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts input to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+*
+* \details Converts input \p a to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+* \param[in] a - float. Is only being read. 
+*
+* \returns half2
+* - The \p half2 value with both halves equal to the converted half
+* precision number.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both input floats to half precision in round-to-nearest-even
+* mode and returns \p half2 with converted values.
+*
+* \details Converts both input floats to half precision in round-to-nearest-even mode
+* and combines the results into one \p half2 number. Low 16 bits of the return
+* value correspond to the input \p a, high 16 bits correspond to the input \p
+* b.
+* \param[in] a - float. Is only being read. 
+* \param[in] b - float. Is only being read. 
+* 
+* \returns half2
+* - The \p half2 value with corresponding halves equal to the
+* converted input floats.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts low 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts low 16 bits of \p half2 input \p a to 32-bit floating-point number
+* and returns the result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* - The low 16 bits of \p a converted to float.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts high 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts high 16 bits of \p half2 input \p a to 32-bit floating-point number
+* and returns the result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* - The high 16 bits of \p a converted to float.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-towards-zero mode.
+*
+* \details Convert the half-precision floating-point value \p h to a signed short
+* integer in round-towards-zero mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read.
+*
+* \returns short int
+* - \p h converted to a signed short integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ short int __half2short_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-towards-zero
+* mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned short
+* integer in round-towards-zero mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read.
+*
+* \returns unsigned short int
+* - \p h converted to an unsigned short integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned short int __half2ushort_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-towards-zero mode.
+*
+* \details Convert the half-precision floating-point value \p h to a signed integer in
+* round-towards-zero mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read.
+*
+* \returns int
+* - \p h converted to a signed integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ int __half2int_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-towards-zero mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned integer
+* in round-towards-zero mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read.
+*
+* \returns unsigned int
+* - \p h converted to an unsigned integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned int __half2uint_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-towards-zero mode.
+*
+* \details Convert the half-precision floating-point value \p h to a signed 64-bit
+* integer in round-towards-zero mode. NaN inputs return a long long int with hex value of 0x8000000000000000.
+* \param[in] h - half. Is only being read.
+*
+* \returns long long int
+* - \p h converted to a signed 64-bit integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ long long int __half2ll_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-towards-zero
+* mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned 64-bit
+* integer in round-towards-zero mode. NaN inputs return 0x8000000000000000.
+* \param[in] h - half. Is only being read.
+*
+* \returns unsigned long long int
+* - \p h converted to an unsigned 64-bit integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned long long int __half2ull_rz(const __half h);
+
+#if defined(__CUDACC__)
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both components of float2 number to half precision in
+* round-to-nearest-even mode and returns \p half2 with converted values.
+* 
+* \details Converts both components of float2 to half precision in round-to-nearest
+* mode and combines the results into one \p half2 number. Low 16 bits of the
+* return value correspond to \p a.x and high 16 bits of the return value
+* correspond to \p a.y.
+* \param[in] a - float2. Is only being read. 
+*  
+* \returns half2
+* - The \p half2 which has corresponding halves equal to the
+* converted float2 components.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both halves of \p half2 to float2 and returns the result.
+* 
+* \details Converts both halves of \p half2 input \p a to float2 and returns the
+* result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float2
+* - \p a converted to float2.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed integer in
+* round-to-nearest-even mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* - \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed integer in
+* round-down mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* - \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed integer in
+* round-up mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* - \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __int2half_rn(const int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rz(const int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-down mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rd(const int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-up mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_ru(const int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed short
+* integer in round-to-nearest-even mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* - \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed short
+* integer in round-down mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* - \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed short
+* integer in round-up mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* - \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __short2half_rn(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rz(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-down mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rd(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-up mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_ru(const short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned integer
+* in round-to-nearest-even mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned int
+* - \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-down mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned integer
+* in round-down mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* - \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-up mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned integer
+* in round-up mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* - \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __uint2half_rn(const unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-towards-zero mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half.  
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rz(const unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-down mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rd(const unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-up mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_ru(const unsigned int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned short
+* integer in round-to-nearest-even mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* - \p h converted to an unsigned short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned short
+* integer in round-down mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* - \p h converted to an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned short
+* integer in round-up mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* - \p h converted to an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ushort2half_rn(const unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rz(const unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-down mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rd(const unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-up mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_ru(const unsigned short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned 64-bit
+* integer in round-to-nearest-even mode. NaN inputs return 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* - \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned 64-bit
+* integer in round-down mode. NaN inputs return 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* - \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned 64-bit
+* integer in round-up mode. NaN inputs return 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* - \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ull2half_rn(const unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rz(const unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half.  
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rd(const unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_ru(const unsigned long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed 64-bit
+* integer in round-to-nearest-even mode. NaN inputs return a long long int with hex value of 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* - \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed 64-bit
+* integer in round-down mode. NaN inputs return a long long int with hex value of 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* - \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed 64-bit
+* integer in round-up mode. NaN inputs return a long long int with hex value of 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* - \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ll2half_rn(const long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rz(const long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rd(const long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_ru(const long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Truncate input argument to the integral part.
+* 
+* \details Round \p h to the nearest integer value that does not exceed \p h in
+* magnitude.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* - The truncated integer value. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half htrunc(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate ceiling of the input argument.
+* 
+* \details Compute the smallest integer value not less than \p h.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* - The smallest integer value not less than \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hceil(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details Calculate the largest integer value which is less than or equal to \p h.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* - The largest integer value which is less than or equal to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hfloor(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating-point
+* number.
+* 
+* \details Round \p h to the nearest integer value in half-precision floating-point
+* format, with halfway cases rounded to the nearest even integer value.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* - The nearest integer to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrint(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Truncate \p half2 vector input argument to the integral part.
+* 
+* \details Round each component of vector \p h to the nearest integer value that does
+* not exceed \p h in magnitude.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* - The truncated \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate \p half2 vector ceiling of the input argument.
+* 
+* \details For each component of vector \p h compute the smallest integer value not less
+* than \p h.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector of smallest integers not less than \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details For each component of vector \p h calculate the largest integer value which
+* is less than or equal to \p h.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector of largest integers which is less than or equal to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating-point
+* number.
+* 
+* \details Round each component of \p half2 vector \p h to the nearest integer value in
+* half-precision floating-point format, with halfway cases rounded to the
+* nearest even integer value.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector of rounded integer values. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns \p half2 with both halves equal to the input value.
+* 
+* \details Returns \p half2 number with both halves equal to the input \p a \p half
+* number.
+* \param[in] a - half. Is only being read. 
+* 
+* \returns half2
+* - The vector which has both its halves equal to the input \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Swaps both halves of the \p half2 input.
+* 
+* \details Swaps both halves of the \p half2 input and returns a new \p half2 number
+* with swapped halves.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - \p a with its halves being swapped. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from each of the two \p half2 inputs and combines
+* into one \p half2 number. 
+* 
+* \details Extracts low 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. Low 16 bits from input \p a is stored in low 16 bits of
+* the return value, low 16 bits from input \p b is stored in high 16 bits of
+* the return value. 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The low 16 bits of \p a and of \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from each of the two \p half2 inputs and
+* combines into one \p half2 number.
+* 
+* \details Extracts high 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. High 16 bits from input \p a is stored in low 16 bits of
+* the return value, high 16 bits from input \p b is stored in high 16 bits of
+* the return value.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The high 16 bits of \p a and of \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns high 16 bits of \p half2 input.
+*
+* \details Returns high 16 bits of \p half2 input \p a.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* - The high 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __high2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns low 16 bits of \p half2 input.
+*
+* \details Returns low 16 bits of \p half2 input \p a.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* - Returns \p half which contains low 16 bits of the input \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __low2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Checks if the input \p half number is infinite.
+* 
+* \details Checks if the input \p half number \p a is infinite. 
+* \param[in] a - half. Is only being read. 
+* 
+* \returns int 
+* - -1 iff \p a is equal to negative infinity, 
+* - 1 iff \p a is equal to positive infinity, 
+* - 0 otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __hisinf(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Combines two \p half numbers into one \p half2 number.
+* 
+* \details Combines two input \p half number \p a and \p b into one \p half2 number.
+* Input \p a is stored in low 16 bits of the return value, input \p b is stored
+* in high 16 bits of the return value.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half2
+* - The half2 with one half equal to \p a and the other to \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from \p half2 input.
+* 
+* \details Extracts low 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The half2 with both halves equal to the low 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from \p half2 input.
+* 
+* \details Extracts high 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The half2 with both halves equal to the high 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 a);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as a signed short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating-point number \p h
+* as a signed short integer. 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* - The reinterpreted value. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as an unsigned short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating-point \p h
+* as an unsigned short number.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* - The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a signed short integer as a \p half.
+* 
+* \details Reinterprets the bits in the signed short integer \p i as a
+* half-precision floating-point number.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in an unsigned short integer as a \p half.
+* 
+* \details Reinterprets the bits in the unsigned short integer \p i as a
+* half-precision floating-point number.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half maximum of two input values.
+*
+* \details Calculates \p half max(\p a, \p b)
+* defined as (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmax(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half minimum of two input values.
+*
+* \details Calculates \p half min(\p a, \p b)
+* defined as (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmin(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector maximum of two inputs.
+*
+* \details Calculates \p half2 vector max(\p a, \p b).
+* Elementwise \p half operation is defined as
+* (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise maximum of vectors \p a  and \p b
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmax2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector minimum of two inputs.
+*
+* \details Calculates \p half2 vector min(\p a, \p b).
+* Elementwise \p half operation is defined as
+* (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise minimum of vectors \p a  and \p b
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmin2(const __half2 a, const __half2 b);
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 300)
+#if !defined warpSize && !defined __local_warpSize
+#define warpSize    32
+#define __local_warpSize
+#endif
+
+#if defined(_WIN32)
+# define __DEPRECATED__(msg) __declspec(deprecated(msg))
+#elif (defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 5 && !defined(__clang__))))
+# define __DEPRECATED__(msg) __attribute__((deprecated))
+#else
+# define __DEPRECATED__(msg) __attribute__((deprecated(msg)))
+#endif
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+#define __WSB_DEPRECATION_MESSAGE(x) __CUDA_FP16_STRINGIFY(x) "() is deprecated in favor of " __CUDA_FP16_STRINGIFY(x) "_sync() and may be removed in a future release (Use -Wno-deprecated-declarations to suppress this warning)."
+
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half2 __shfl(const __half2 var, const int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half2 __shfl_up(const __half2 var, const unsigned int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down))__half2 __shfl_down(const __half2 var, const unsigned int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half2 __shfl_xor(const __half2 var, const int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half __shfl(const __half var, const int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half __shfl_up(const __half var, const unsigned int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down)) __half __shfl_down(const __half var, const unsigned int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half __shfl_xor(const __half var, const int delta, const int width = warpSize);
+#endif
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* within the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_sync(const unsigned mask, const __half2 var, const int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(const unsigned mask, const __half2 var, const unsigned int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(const unsigned mask, const __half2 var, const unsigned int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(const unsigned mask, const __half2 var, const int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* within the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_sync(const unsigned mask, const __half var, const int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_up_sync(const unsigned mask, const __half var, const unsigned int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_down_sync(const unsigned mask, const __half var, const unsigned int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(const unsigned mask, const __half var, const int delta, const int width = warpSize);
+
+#if defined(__local_warpSize)
+#undef warpSize
+#undef __local_warpSize
+#endif
+#endif /*!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 300) */
+
+#if defined(__cplusplus) && ( !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320) )
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.nc` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.nc` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldg(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cg` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cg` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcg(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.ca` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.ca` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldca(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cs` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cs` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcs(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.lu` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldlu(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.lu` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldlu(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cv` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcv(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cv` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcv(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wb` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwb(__half2 *const ptr, const __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wb` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwb(__half *const ptr, const __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cg` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcg(__half2 *const ptr, const __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cg` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcg(__half *const ptr, const __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cs` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcs(__half2 *const ptr, const __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cs` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcs(__half *const ptr, const __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wt` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwt(__half2 *const ptr, const __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wt` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwt(__half *const ptr, const __half value);
+#endif /*defined(__cplusplus) && ( !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320) )*/
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs half2 vector if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector result of if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison.
+* 
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector result of not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The \p half2 result of less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The vector result of greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The half2 vector result of less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison.
+* 
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector result of greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector result of unordered if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The vector result of unordered not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison.
+*
+* Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The vector result of unordered less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The \p half2 vector result of unordered greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The vector result of unordered less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The \p half2 vector result of unordered greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Determine whether \p half2 argument is a NaN.
+*
+* \details Determine whether each half of input \p half2 number \p a is a NaN.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The half2 with the corresponding \p half results set to
+* 1.0 for NaN, 0.0 otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-95
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The sum of vectors \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-104
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The subtraction of vector \p b from \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-102
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The result of elementwise multiplying the vectors \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode. Prevents floating-point contractions of mul+add into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-95
+* \endinternal
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The sum of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2_rn(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode. Prevents floating-point contractions of mul+sub
+* into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-104
+* \endinternal
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The subtraction of vector \p b from \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2_rn(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode. Prevents floating-point contractions of
+* mul+add or sub into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-102
+* \endinternal
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise multiplying the vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2_rn(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector division in round-to-nearest-even mode.
+*
+* \details Divides \p half2 input vector \p a by input vector \p b in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-103
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise division of \p a with \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __h2div(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+*
+* \details Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - Returns \p a with the absolute value of both halves. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the results to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The sum of \p a and \p b, with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The subtraction of vector \p b from \p a, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The result of elementwise multiplication of vectors \p a and \p b, 
+* with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-105
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* - The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode, with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the
+* results to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* - The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c, 
+* with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Negates both halves of the input \p half2 number and returns the
+* result.
+*
+* \details Negates both halves of the input \p half2 number \p a and returns the result.
+* \internal
+* \req DEEPLEARN-SRM_REQ-101
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - Returns \p a with both halves negated. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Calculates the absolute value of input \p half number and returns the result.
+*
+* \details Calculates the absolute value of input \p half number and returns the result.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The absolute value of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __habs(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode.
+*
+* \details Performs \p half addition of inputs \p a and \p b, in round-to-nearest-even
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-94
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The sum of \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-97
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The result of subtracting \p b from \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-99
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The result of multiplying \p a and \p b. 
+*/
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode.
+*
+* \details Performs \p half addition of inputs \p a and \p b, in round-to-nearest-even
+* mode. Prevents floating-point contractions of mul+add into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-94
+* \endinternal
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* - The sum of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd_rn(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode. Prevents floating-point contractions of mul+sub into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-97
+* \endinternal
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* - The result of subtracting \p b from \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub_rn(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode. Prevents floating-point contractions of mul+add or sub into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-99
+* \endinternal
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* - The result of multiplying \p a and \p b.
+*/
+__CUDA_FP16_DECL__ __half __hmul_rn(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half division in round-to-nearest-even mode.
+* 
+* \details Divides \p half input \p a by input \p b in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-98
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half
+* - The result of dividing \p a by \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__  __half __hdiv(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half add of inputs \p a and \p b, in round-to-nearest-even mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The sum of \p a and \p b, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The result of subtraction of \p b from \p a, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the result to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The result of multiplying \p a and \p b, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-96
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* - The result of fused multiply-add operation on \p
+* a, \p b, and \p c. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the result
+* to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* - The result of fused multiply-add operation on \p
+* a, \p b, and \p c, with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Negates input \p half number and returns the result.
+*
+* \details Negates input \p half number and returns the result.
+* \internal
+* \req DEEPLEARN-SRM_REQ-100
+* \endinternal
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - minus a
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hneg(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector if-equal comparison and returns boolean true
+* iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of if-equal comparison
+* of vectors \p a and \p b are true;
+* - false otherwise.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of not-equal comparison
+* of vectors \p a and \p b are true, 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of less-equal comparison
+* of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of greater-equal
+* comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of less-than comparison
+* of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns bool 
+* - true if both \p half results of greater-than
+* comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered if-equal
+* comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered not-equal
+* comparison of vectors \p a and \p b are true;
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered less-equal
+* comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered
+* greater-equal comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered less-than comparison of 
+* vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered
+* greater-than comparison of vectors \p a and \p b are true;
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of if-equal comparison of \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of not-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of less-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of greater-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of less-than comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of greater-than comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered if-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered not-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered less-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered greater-equal comparison of \p a
+* and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered less-than comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered greater-than comparison of \p a
+* and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Determine whether \p half argument is a NaN.
+*
+* \details Determine whether \p half value \p a is a NaN.
+* \param[in] a - half. Is only being read. 
+*
+* \returns bool
+* - true iff argument is NaN. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hisnan(const __half a);
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half maximum of two input values, NaNs pass through.
+*
+* \details Calculates \p half max(\p a, \p b)
+* defined as (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmax_nan(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half minimum of two input values, NaNs pass through.
+*
+* \details Calculates \p half min(\p a, \p b)
+* defined as (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmin_nan(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode with relu saturation.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* Then negative result is clamped to 0.
+* NaN result is converted to canonical NaN.
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+* \param[in] c - half. Is only being read.
+*
+* \returns half
+* - The result of fused multiply-add operation on \p
+* a, \p b, and \p c with relu saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma_relu(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector maximum of two inputs, NaNs pass through.
+*
+* \details Calculates \p half2 vector max(\p a, \p b).
+* Elementwise \p half operation is defined as
+* (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise maximum of vectors \p a  and \p b, with NaNs pass through
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmax2_nan(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector minimum of two inputs, NaNs pass through.
+*
+* \details Calculates \p half2 vector min(\p a, \p b).
+* Elementwise \p half operation is defined as
+* (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise minimum of vectors \p a  and \p b, with NaNs pass through
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmin2_nan(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode with relu saturation.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* Then negative result is clamped to 0.
+* NaN result is converted to canonical NaN.
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+* \param[in] c - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c with relu saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_relu(const __half2 a, const __half2 b, const __half2 c);
+#endif /* !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800) */
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs fast complex multiply-accumulate
+*
+* \details Interprets vector \p half2 input pairs \p a, \p b, and \p c as
+* complex numbers in \p half precision and performs
+* complex multiply-accumulate operation: a*b + c
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+* \param[in] c - half2. Is only being read.
+*
+* \returns half2
+* - The result of complex multiply-accumulate operation on complex numbers \p a, \p b, and \p c
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hcmadd(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half square root of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The square root of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal square root in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half reciprocal square root of input \p a in round-to-nearest
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The reciprocal square root of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half reciprocal of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The reciprocal of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrcp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half natural logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The natural logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half binary logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The binary logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half decimal logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The decimal logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half natural exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The natural exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half binary exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The binary exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half decimal exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The decimal exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half cosine in round-to-nearest-even mode.
+*
+* \details Calculates \p half cosine of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The cosine of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hcos(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half sine in round-to-nearest-even mode.
+*
+* \details Calculates \p half sine of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The sine of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hsin(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 square root of input vector \p a in round-to-nearest
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise square root on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal square root in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 reciprocal square root of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise reciprocal square root on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 reciprocal of input vector \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise reciprocal on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector natural logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 natural logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise natural logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 binary logarithm of input vector \p a in round-to-nearest
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise binary logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 decimal logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise decimal logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary exponential function in
+* round-to-nearest-even mode.
+*
+* \details Calculates \p half2 binary exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise binary exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal exponential function in
+* round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 decimal exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The elementwise decimal exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector cosine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 cosine of input vector \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The elementwise cosine on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector sine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 sine of input vector \p a in round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The elementwise sine on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a);
+
+#endif /*if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)*/
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 600)
+
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Vector add \p val to the value stored at \p address in global or shared memory, and writes this
+* value back to \p address. The atomicity of the add operation is guaranteed separately for each of the
+* two __half elements; the entire __half2 is not guaranteed to be atomic as a single 32-bit access.
+* 
+* \details The location of \p address must be in global or shared memory. This operation has undefined
+* behavior otherwise. This operation is only supported by devices of compute capability 6.x and higher.
+* 
+* \param[in] address - half2*. An address in global or shared memory.
+* \param[in] val - half2. The value to be added.
+* 
+* \returns half2
+* - The old value read from \p address.
+*
+* \note_ref_guide_atomic
+*/
+__CUDA_FP16_DECL__ __half2 atomicAdd(__half2 *const address, const __half2 val);
+
+#endif /*if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 600)*/
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 700)
+
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Adds \p val to the value stored at \p address in global or shared memory, and writes this value
+* back to \p address. This operation is performed in one atomic operation.
+* 
+* \details The location of \p address must be in global or shared memory. This operation has undefined
+* behavior otherwise. This operation is only supported by devices of compute capability 7.x and higher.
+* 
+* \param[in] address - half*. An address in global or shared memory.
+* \param[in] val - half. The value to be added.
+* 
+* \returns half
+* - The old value read from \p address.
+* 
+* \note_ref_guide_atomic
+*/
+__CUDA_FP16_DECL__ __half atomicAdd(__half *const address, const __half val);
+
+#endif /*if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 700)*/
+
+#endif /* defined(__CUDACC__) */
+
+#undef __CUDA_FP16_DECL__
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+
+#endif /* defined(__cplusplus) */
+
+/* Note the .hpp file is included even for host-side compilation, to capture the "half" & "half2" definitions */
+#include "cuda_fp16.hpp"
+#undef ___CUDA_FP16_STRINGIFY_INNERMOST
+#undef __CUDA_FP16_STRINGIFY
+
+#endif /* end of include guard: __CUDA_FP16_H__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-11/cuda_fp16.hpp b/cupy/_core/include/cupy/_cuda/cuda-11/cuda_fp16.hpp
new file mode 100644
index 0000000..e15f46b
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-11/cuda_fp16.hpp
@@ -0,0 +1,2614 @@
+/*
+* Copyright 1993-2021 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+#if !defined(__CUDA_FP16_HPP__)
+#define __CUDA_FP16_HPP__
+
+#if !defined(__CUDA_FP16_H__)
+#error "Do not include this file directly. Instead, include cuda_fp16.h."
+#endif
+
+#if !defined(_MSC_VER) && __cplusplus >= 201103L
+#   define __CPP_VERSION_AT_LEAST_11_FP16
+#elif _MSC_FULL_VER >= 190024210 && _MSVC_LANG >= 201103L
+#   define __CPP_VERSION_AT_LEAST_11_FP16
+#endif
+
+/* C++11 header for std::move. 
+ * In RTC mode, std::move is provided implicitly; don't include the header
+ */
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16) && !defined(__CUDACC_RTC__)
+#include <utility>
+#endif /* __cplusplus >= 201103L && !defined(__CUDACC_RTC__) */
+
+/* C++ header for std::memcpy (used for type punning in host-side implementations).
+ * When compiling as a CUDA source file memcpy is provided implicitly.
+ * !defined(__CUDACC__) implies !defined(__CUDACC_RTC__).
+ */
+#if defined(__cplusplus) && !defined(__CUDACC__)
+#include <cstring>
+#endif /* defined(__cplusplus) && !defined(__CUDACC__) */
+
+
+/* Set up function decorations */
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#define __VECTOR_FUNCTIONS_DECL__ static __inline__ __host__ __device__
+#define __CUDA_HOSTDEVICE__ __host__ __device__
+#else /* !defined(__CUDACC__) */
+#if defined(__GNUC__)
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __attribute__ ((unused))
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__GNUC__) */
+#define __CUDA_HOSTDEVICE__
+#endif /* defined(__CUDACC_) */
+
+/* Set up structure-alignment attribute */
+#if defined(__CUDACC__)
+#define __CUDA_ALIGN__(align) __align__(align)
+#else
+/* Define alignment macro based on compiler type (cannot assume C11 "_Alignas" is available) */
+#if __cplusplus >= 201103L
+#define __CUDA_ALIGN__(n) alignas(n)    /* C++11 kindly gives us a keyword for this */
+#else /* !defined(__CPP_VERSION_AT_LEAST_11_FP16)*/
+#if defined(__GNUC__)
+#define __CUDA_ALIGN__(n) __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+#define __CUDA_ALIGN__(n) __declspec(align(n))
+#else
+#define __CUDA_ALIGN__(n)
+#endif /* defined(__GNUC__) */
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+#endif /* defined(__CUDACC__) */
+
+/* Macros to allow half & half2 to be used by inline assembly */
+#define __HALF_TO_US(var) *(reinterpret_cast<unsigned short *>(&(var)))
+#define __HALF_TO_CUS(var) *(reinterpret_cast<const unsigned short *>(&(var)))
+#define __HALF2_TO_UI(var) *(reinterpret_cast<unsigned int *>(&(var)))
+#define __HALF2_TO_CUI(var) *(reinterpret_cast<const unsigned int *>(&(var)))
+
+/* Macros for half & half2 binary arithmetic */
+#define __BINARY_OP_HALF_MACRO(name) /* do */ {\
+   __half val; \
+   asm( "{" __CUDA_FP16_STRINGIFY(name) ".f16 %0,%1,%2;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b))); \
+   return val; \
+} /* while(0) */
+#define __BINARY_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{" __CUDA_FP16_STRINGIFY(name) ".f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+#define __TERNARY_OP_HALF_MACRO(name) /* do */ {\
+   __half val; \
+   asm( "{" __CUDA_FP16_STRINGIFY(name) ".f16 %0,%1,%2,%3;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b)),"h"(__HALF_TO_CUS(c))); \
+   return val; \
+} /* while(0) */
+#define __TERNARY_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{" __CUDA_FP16_STRINGIFY(name) ".f16x2 %0,%1,%2,%3;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b)),"r"(__HALF2_TO_CUI(c))); \
+   return val; \
+} /* while(0) */
+
+/**
+* Types which allow static initialization of "half" and "half2" until
+* these become an actual builtin. Note this initialization is as a
+* bitfield representation of "half", and not a conversion from short->half.
+* Such a representation will be deprecated in a future version of CUDA. 
+* (Note these are visible to non-nvcc compilers, including C-only compilation)
+*/
+typedef struct __CUDA_ALIGN__(2) {
+    unsigned short x;
+} __half_raw;
+
+typedef struct __CUDA_ALIGN__(4) {
+    unsigned short x;
+    unsigned short y;
+} __half2_raw;
+
+/* All other definitions in this file are only visible to C++ compilers */
+#if defined(__cplusplus)
+
+/* Hide GCC member initialization list warnings because of host/device in-function init requirement */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstrict-aliasing"
+#pragma GCC diagnostic ignored "-Weffc++"
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+/* class' : multiple assignment operators specified
+   The class has multiple assignment operators of a single type. This warning is informational */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( push )
+#pragma warning( disable:4522 )
+#endif /* defined(__GNUC__) */
+
+struct __CUDA_ALIGN__(2) __half {
+protected:
+    unsigned short __x;
+
+public:
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+    __half() = default;
+#else
+    __CUDA_HOSTDEVICE__ __half() { }
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+
+    /* Convert to/from __half_raw */
+    __CUDA_HOSTDEVICE__ __half(const __half_raw &hr) : __x(hr.x) { }
+    __CUDA_HOSTDEVICE__ __half &operator=(const __half_raw &hr) { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const volatile __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const { __half_raw ret; ret.x = __x; return ret; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const volatile { __half_raw ret; ret.x = __x; return ret; }
+
+#if !defined(__CUDA_NO_HALF_CONVERSIONS__)
+
+    /* Construct from float/double */
+    __CUDA_HOSTDEVICE__ __half(const float f) { __x = __float2half(f).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const double f) { __x = __double2half(f).__x;  }
+
+    __CUDA_HOSTDEVICE__ operator float() const { return __half2float(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const float f) { __x = __float2half(f).__x; return *this; }
+
+    /* We omit "cast to double" operator, so as to not be ambiguous about up-cast */
+    __CUDA_HOSTDEVICE__ __half &operator=(const double f) { __x = __double2half(f).__x; return *this; }
+
+/* Member functions only available to nvcc compilation so far */
+#if defined(__CUDACC__)
+    /* Allow automatic construction from types supported natively in hardware */
+    /* Note we do avoid constructor init-list because of special host/device compilation rules */
+    __CUDA_HOSTDEVICE__ __half(const short val) { __x = __short2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const unsigned short val) { __x = __ushort2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const int val) { __x = __int2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const unsigned int val) { __x = __uint2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const long long val) { __x = __ll2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const unsigned long long val) { __x = __ull2half_rn(val).__x; }
+
+    /* Allow automatic casts to supported builtin types, matching all that are permitted with float */
+    __CUDA_HOSTDEVICE__ operator short() const { return __half2short_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const short val) { __x = __short2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned short() const { return __half2ushort_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const unsigned short val) { __x = __ushort2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator int() const { return __half2int_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const int val) { __x = __int2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned int() const { return __half2uint_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const unsigned int val) { __x = __uint2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator long long() const { return __half2ll_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const long long val) { __x = __ll2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned long long() const { return __half2ull_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const unsigned long long val) { __x = __ull2half_rn(val).__x; return *this; }
+
+    /* Boolean conversion - note both 0 and -0 must return false */
+    __CUDA_HOSTDEVICE__ operator bool() const { return (__x & 0x7FFFU) != 0U; }
+#endif /* defined(__CUDACC__) */
+#endif /* !defined(__CUDA_NO_HALF_CONVERSIONS__) */
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16 operations only supported on arch >= 5.3 */
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)
+#if !defined(__CUDA_NO_HALF_OPERATORS__)
+/* Some basic arithmetic operations expected of a builtin */
+__device__ __forceinline__ __half operator+(const __half &lh, const __half &rh) { return __hadd(lh, rh); }
+__device__ __forceinline__ __half operator-(const __half &lh, const __half &rh) { return __hsub(lh, rh); }
+__device__ __forceinline__ __half operator*(const __half &lh, const __half &rh) { return __hmul(lh, rh); }
+__device__ __forceinline__ __half operator/(const __half &lh, const __half &rh) { return __hdiv(lh, rh); }
+
+__device__ __forceinline__ __half &operator+=(__half &lh, const __half &rh) { lh = __hadd(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator-=(__half &lh, const __half &rh) { lh = __hsub(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator*=(__half &lh, const __half &rh) { lh = __hmul(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator/=(__half &lh, const __half &rh) { lh = __hdiv(lh, rh); return lh; }
+
+/* Note for increment and decrement we use the raw value 0x3C00U equating to half(1.0F), to avoid the extra conversion */
+__device__ __forceinline__ __half &operator++(__half &h)      { __half_raw one; one.x = 0x3C00U; h += one; return h; }
+__device__ __forceinline__ __half &operator--(__half &h)      { __half_raw one; one.x = 0x3C00U; h -= one; return h; }
+__device__ __forceinline__ __half  operator++(__half &h, const int ignored)
+{
+    // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators.
+    static_cast<void>(ignored);
+
+    const __half ret = h;
+    __half_raw one;
+    one.x = 0x3C00U;
+    h += one;
+    return ret;
+}
+__device__ __forceinline__ __half  operator--(__half &h, const int ignored)
+{
+    // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators.
+    static_cast<void>(ignored);
+
+    const __half ret = h;
+    __half_raw one;
+    one.x = 0x3C00U;
+    h -= one;
+    return ret;
+}
+
+/* Unary plus and inverse operators */
+__device__ __forceinline__ __half operator+(const __half &h) { return h; }
+__device__ __forceinline__ __half operator-(const __half &h) { return __hneg(h); }
+
+/* Some basic comparison operations to make it look like a builtin */
+__device__ __forceinline__ bool operator==(const __half &lh, const __half &rh) { return __heq(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half &lh, const __half &rh) { return __hneu(lh, rh); }
+__device__ __forceinline__ bool operator> (const __half &lh, const __half &rh) { return __hgt(lh, rh); }
+__device__ __forceinline__ bool operator< (const __half &lh, const __half &rh) { return __hlt(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half &lh, const __half &rh) { return __hge(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half &lh, const __half &rh) { return __hle(lh, rh); }
+#endif /* !defined(__CUDA_NO_HALF_OPERATORS__) */
+#endif /* !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) */
+#endif /* defined(__CUDACC__) */
+
+/* __half2 is visible to non-nvcc host compilers */
+struct __CUDA_ALIGN__(4) __half2 {
+    __half x;
+    __half y;
+
+    // All construct/copy/assign/move
+public:
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+    __half2() = default;
+    __CUDA_HOSTDEVICE__ __half2(const __half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); return *this; }
+#else
+    __CUDA_HOSTDEVICE__ __half2() { }
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+    __CUDA_HOSTDEVICE__ __half2(const __half &a, const __half &b) : x(a), y(b) { }
+    __CUDA_HOSTDEVICE__ __half2(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); return *this; }
+
+    /* Convert to/from __half2_raw */
+    __CUDA_HOSTDEVICE__ __half2(const __half2_raw &h2r ) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2_raw &h2r) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); return *this; }
+    __CUDA_HOSTDEVICE__ operator __half2_raw() const { __half2_raw ret; ret.x = 0U; ret.y = 0U; __HALF2_TO_UI(ret) = __HALF2_TO_CUI(*this); return ret; }
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16x2 operations only supported on arch >= 5.3 */
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)) && !defined(__CUDA_NO_HALF2_OPERATORS__)
+
+__device__ __forceinline__ __half2 operator+(const __half2 &lh, const __half2 &rh) { return __hadd2(lh, rh); }
+__device__ __forceinline__ __half2 operator-(const __half2 &lh, const __half2 &rh) { return __hsub2(lh, rh); }
+__device__ __forceinline__ __half2 operator*(const __half2 &lh, const __half2 &rh) { return __hmul2(lh, rh); }
+__device__ __forceinline__ __half2 operator/(const __half2 &lh, const __half2 &rh) { return __h2div(lh, rh); }
+
+__device__ __forceinline__ __half2& operator+=(__half2 &lh, const __half2 &rh) { lh = __hadd2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator-=(__half2 &lh, const __half2 &rh) { lh = __hsub2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator*=(__half2 &lh, const __half2 &rh) { lh = __hmul2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator/=(__half2 &lh, const __half2 &rh) { lh = __h2div(lh, rh); return lh; }
+
+__device__ __forceinline__ __half2 &operator++(__half2 &h)      { __half2_raw one; one.x = 0x3C00U; one.y = 0x3C00U; h = __hadd2(h, one); return h; }
+__device__ __forceinline__ __half2 &operator--(__half2 &h)      { __half2_raw one; one.x = 0x3C00U; one.y = 0x3C00U; h = __hsub2(h, one); return h; }
+__device__ __forceinline__ __half2  operator++(__half2 &h, const int ignored)
+{
+    // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators.
+    static_cast<void>(ignored);
+
+    const __half2 ret = h;
+    __half2_raw one;
+    one.x = 0x3C00U;
+    one.y = 0x3C00U;
+    h = __hadd2(h, one);
+    return ret;
+}
+__device__ __forceinline__ __half2  operator--(__half2 &h, const int ignored)
+{
+    // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators.
+    static_cast<void>(ignored);
+
+    const __half2 ret = h;
+    __half2_raw one;
+    one.x = 0x3C00U;
+    one.y = 0x3C00U;
+    h = __hsub2(h, one);
+    return ret;
+}
+
+__device__ __forceinline__ __half2 operator+(const __half2 &h) { return h; }
+__device__ __forceinline__ __half2 operator-(const __half2 &h) { return __hneg2(h); }
+
+__device__ __forceinline__ bool operator==(const __half2 &lh, const __half2 &rh) { return __hbeq2(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half2 &lh, const __half2 &rh) { return __hbneu2(lh, rh); }
+__device__ __forceinline__ bool operator>(const __half2 &lh, const __half2 &rh) { return __hbgt2(lh, rh); }
+__device__ __forceinline__ bool operator<(const __half2 &lh, const __half2 &rh) { return __hblt2(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half2 &lh, const __half2 &rh) { return __hbge2(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half2 &lh, const __half2 &rh) { return __hble2(lh, rh); }
+
+#endif /* !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) */
+#endif /* defined(__CUDACC__) */
+
+/* Restore warning for multiple assignment operators */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( pop )
+#endif /* defined(_MSC_VER) && _MSC_VER >= 1500 */
+
+/* Restore -Weffc++ warnings from here on */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic pop
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+#undef __CUDA_HOSTDEVICE__
+#undef __CUDA_ALIGN__
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static inline unsigned short __internal_float2half(const float f, unsigned int &sign, unsigned int &remainder)
+{
+    unsigned int x;
+    unsigned int u;
+    unsigned int result;
+#if defined(__CUDACC__)
+    (void)memcpy(&x, &f, sizeof(f));
+#else
+    (void)std::memcpy(&x, &f, sizeof(f));
+#endif
+    u = (x & 0x7fffffffU);
+    sign = ((x >> 16U) & 0x8000U);
+    // NaN/+Inf/-Inf
+    if (u >= 0x7f800000U) {
+        remainder = 0U;
+        result = ((u == 0x7f800000U) ? (sign | 0x7c00U) : 0x7fffU);
+    } else if (u > 0x477fefffU) { // Overflows
+        remainder = 0x80000000U;
+        result = (sign | 0x7bffU);
+    } else if (u >= 0x38800000U) { // Normal numbers
+        remainder = u << 19U;
+        u -= 0x38000000U;
+        result = (sign | (u >> 13U));
+    } else if (u < 0x33000001U) { // +0/-0
+        remainder = u;
+        result = sign;
+    } else { // Denormal numbers
+        const unsigned int exponent = u >> 23U;
+        const unsigned int shift = 0x7eU - exponent;
+        unsigned int mantissa = (u & 0x7fffffU);
+        mantissa |= 0x800000U;
+        remainder = mantissa << (32U - shift);
+        result = (sign | (mantissa >> shift));
+        result &= 0x0000FFFFU;
+    }
+    return static_cast<unsigned short>(result);
+}
+#endif  /* #if !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __double2half(const double a)
+{
+#if defined(__CUDA_ARCH__)
+    __half val;
+    asm("{  cvt.rn.f16.f64 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "d"(a));
+    return val;
+#else
+    __half result;
+    /*
+    // Perform rounding to 11 bits of precision, convert value
+    // to float and call existing float to half conversion.
+    // By pre-rounding to 11 bits we avoid additional rounding
+    // in float to half conversion.
+    */
+    unsigned long long int absa;
+    unsigned long long int ua;
+    #if defined(__CUDACC__)
+        (void)memcpy(&ua, &a, sizeof(a));
+    #else
+        (void)std::memcpy(&ua, &a, sizeof(a));
+    #endif
+    absa = (ua & 0x7fffffffffffffffULL);
+    if ((absa >= 0x40f0000000000000ULL) || (absa <= 0x3e60000000000000ULL))
+    {
+        /*
+        // |a| >= 2^16 or NaN or |a| <= 2^(-25)
+        // double-rounding is not a problem
+        */
+        result = __float2half(static_cast<float>(a));
+    }
+    else
+    {
+        /*
+        // here 2^(-25) < |a| < 2^16
+        // prepare shifter value such that a + shifter
+        // done in double precision performs round-to-nearest-even
+        // and (a + shifter) - shifter results in a rounded to
+        // 11 bits of precision. Shifter needs to have exponent of
+        // a plus 53 - 11 = 42 and a leading bit in mantissa to guard
+        // against negative values.
+        // So need to have |a| capped to avoid overflow in exponent.
+        // For inputs that are smaller than half precision minnorm
+        // we prepare fixed shifter exponent.
+        */
+        unsigned long long shifterBits;
+        if (absa >= 0x3f10000000000000ULL)
+        {
+            /*
+            // Here if |a| >= 2^(-14)
+            // add 42 to exponent bits
+            */
+            shifterBits  = (ua & 0x7ff0000000000000ULL) + 0x02A0000000000000ULL;
+        }
+        else
+        {
+            /*
+            // 2^(-25) < |a| < 2^(-14), potentially results in denormal
+            // set exponent bits to 42 - 14 + bias
+            */
+            shifterBits = 0x41B0000000000000ULL;
+        }
+        // set leading mantissa bit to protect against negative inputs
+        shifterBits |= 0x0008000000000000ULL;
+        double shifter;
+        #if defined(__CUDACC__)
+            (void)memcpy(&shifter, &shifterBits, sizeof(shifterBits));
+        #else
+            (void)std::memcpy(&shifter, &shifterBits, sizeof(shifterBits));
+        #endif
+        double aShiftRound = a + shifter;
+
+        /*
+        // Prevent the compiler from optimizing away a + shifter - shifter
+        // by doing intermediate memcopy and harmless bitwize operation
+        */
+        unsigned long long int aShiftRoundBits;
+        #if defined(__CUDACC__)
+            (void)memcpy(&aShiftRoundBits, &aShiftRound, sizeof(aShiftRound));
+        #else
+            (void)std::memcpy(&aShiftRoundBits, &aShiftRound, sizeof(aShiftRound));
+        #endif
+
+        // the value is positive, so this operation doesn't change anything
+        aShiftRoundBits &= 0x7fffffffffffffffULL;
+
+        #if defined(__CUDACC__)
+            (void)memcpy(&aShiftRound, &aShiftRoundBits, sizeof(aShiftRound));
+        #else
+            (void)std::memcpy(&aShiftRound, &aShiftRoundBits, sizeof(aShiftRound));
+        #endif
+
+        result = __float2half(static_cast<float>(aShiftRound - shifter));
+    }
+
+    return result;
+#endif
+}
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rz.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rm.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign != 0U)) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a)
+{
+    __half val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.rp.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+#else
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign == 0U)) {
+        r.x++;
+    }
+    val = r;
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a)
+{
+    __half2 val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a));
+#else
+    val = __half2(__float2half_rn(a), __float2half_rn(a));
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b)
+{
+    __half2 val;
+#if defined(__CUDA_ARCH__)
+#if (__CUDA_ARCH__ >= 800)
+    asm("{ cvt.rn.f16x2.f32 %0, %2, %1; }\n"
+        : "=r"(__HALF2_TO_UI(val)) : "f"(a), "f"(b));
+#else
+    asm("{.reg .f16 low,high;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  cvt.rn.f16.f32 high, %2;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a), "f"(b));
+#endif
+#else
+    val = __half2(__float2half_rn(a), __float2half_rn(b));
+#endif
+    return val;
+}
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static inline float __internal_half2float(const unsigned short h)
+{
+    unsigned int sign = ((static_cast<unsigned int>(h) >> 15U) & 1U);
+    unsigned int exponent = ((static_cast<unsigned int>(h) >> 10U) & 0x1fU);
+    unsigned int mantissa = ((static_cast<unsigned int>(h) & 0x3ffU) << 13U);
+    float f;
+    if (exponent == 0x1fU) { /* NaN or Inf */
+        /* discard sign of a NaN */
+        sign = ((mantissa != 0U) ? (sign >> 1U) : sign);
+        mantissa = ((mantissa != 0U) ? 0x7fffffU : 0U);
+        exponent = 0xffU;
+    } else if (exponent == 0U) { /* Denorm or Zero */
+        if (mantissa != 0U) {
+            unsigned int msb;
+            exponent = 0x71U;
+            do {
+                msb = (mantissa & 0x400000U);
+                mantissa <<= 1U; /* normalize */
+                --exponent;
+            } while (msb == 0U);
+            mantissa &= 0x7fffffU; /* 1.mantissa is implicit */
+        }
+    } else {
+        exponent += 0x70U;
+    }
+    const unsigned int u = ((sign << 31U) | (exponent << 23U) | mantissa);
+#if defined(__CUDACC__)
+    (void)memcpy(&f, &u, sizeof(u));
+#else
+    (void)std::memcpy(&f, &u, sizeof(u));
+#endif
+    return f;
+}
+#endif  /* !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{  cvt.f32.f16 %0, %1;}\n" : "=f"(val) : "h"(__HALF_TO_CUS(a)));
+#else
+    val = __internal_half2float(static_cast<__half_raw>(a).x);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+#else
+    val = __internal_half2float(static_cast<__half2_raw>(a).x);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a)
+{
+    float val;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+#else
+    val = __internal_half2float(static_cast<__half2_raw>(a).y);
+#endif
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ short int __half2short_rz(const __half h)
+{
+    short int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+#else
+    const float f = __half2float(h);
+    const short int max_val = (short int)0x7fffU;
+    const short int min_val = (short int)0x8000U;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<short int>(f);
+    }
+#endif
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned short int __half2ushort_rz(const __half h)
+{
+    unsigned short int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+#else
+    const float f = __half2float(h);
+    const unsigned short int max_val = 0xffffU;
+    const unsigned short int min_val = 0U;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0U;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<unsigned short int>(f);
+    }
+#endif
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ int __half2int_rz(const __half h)
+{
+    int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+#else
+    const float f = __half2float(h);
+    const int max_val = (int)0x7fffffffU;
+    const int min_val = (int)0x80000000U;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<int>(f);
+    }
+#endif
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned int __half2uint_rz(const __half h)
+{
+    unsigned int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+#else
+    const float f = __half2float(h);
+    const unsigned int max_val = 0xffffffffU;
+    const unsigned int min_val = 0U;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0U;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<unsigned int>(f);
+    }
+#endif
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ long long int __half2ll_rz(const __half h)
+{
+    long long int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+#else
+    const float f = __half2float(h);
+    const long long int max_val = (long long int)0x7fffffffffffffffULL;
+    const long long int min_val = (long long int)0x8000000000000000ULL;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = min_val;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<long long int>(f);
+    }
+#endif
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned long long int __half2ull_rz(const __half h)
+{
+    unsigned long long int i;
+#if defined __CUDA_ARCH__
+    asm("cvt.rzi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+#else
+    const float f = __half2float(h);
+    const unsigned long long int max_val = 0xffffffffffffffffULL;
+    const unsigned long long int min_val = 0ULL;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0x8000000000000000ULL;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<unsigned long long int>(f);
+    }
+#endif
+    return i;
+}
+
+/* Intrinsic functions only available to nvcc compilers */
+#if defined(__CUDACC__)
+
+/* CUDA vector-types compatible vector creation function (note returns __half2, not half2) */
+__VECTOR_FUNCTIONS_DECL__ __half2 make_half2(const __half x, const __half y)
+{
+    __half2 t; t.x = x; t.y = y; return t;
+}
+#undef __VECTOR_FUNCTIONS_DECL__
+
+
+/* Definitions of intrinsics */
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 a)
+{
+    const __half2 val = __floats2half2_rn(a.x, a.y);
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 a)
+{
+    float hi_float;
+    float lo_float;
+#if defined(__CUDA_ARCH__)
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(lo_float) : "r"(__HALF2_TO_CUI(a)));
+
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(hi_float) : "r"(__HALF2_TO_CUI(a)));
+#else
+    lo_float = __internal_half2float(((__half2_raw)a).x);
+    hi_float = __internal_half2float(((__half2_raw)a).y);
+#endif
+    return make_float2(lo_float, hi_float);
+}
+__CUDA_FP16_DECL__ int __half2int_rn(const __half h)
+{
+    int i;
+    asm("cvt.rni.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_rd(const __half h)
+{
+    int i;
+    asm("cvt.rmi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_ru(const __half h)
+{
+    int i;
+    asm("cvt.rpi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __int2half_rn(const int i)
+{
+    __half h;
+#if defined(__CUDA_ARCH__)
+    asm("cvt.rn.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+#else
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rz(const int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rd(const int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_ru(const int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ short int __half2short_rn(const __half h)
+{
+    short int i;
+    asm("cvt.rni.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_rd(const __half h)
+{
+    short int i;
+    asm("cvt.rmi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_ru(const __half h)
+{
+    short int i;
+    asm("cvt.rpi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __short2half_rn(const short int i)
+{
+    __half h;
+#if defined __CUDA_ARCH__
+    asm("cvt.rn.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+#else
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rz(const short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rd(const short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_ru(const short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(const __half h)
+{
+    unsigned int i;
+    asm("cvt.rni.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(const __half h)
+{
+    unsigned int i;
+    asm("cvt.rmi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(const __half h)
+{
+    unsigned int i;
+    asm("cvt.rpi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __uint2half_rn(const unsigned int i)
+{
+    __half h;
+#if defined __CUDA_ARCH__
+    asm("cvt.rn.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+#else
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rz(const unsigned int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rd(const unsigned int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_ru(const unsigned int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(const __half h)
+{
+    unsigned short int i;
+    asm("cvt.rni.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(const __half h)
+{
+    unsigned short int i;
+    asm("cvt.rmi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(const __half h)
+{
+    unsigned short int i;
+    asm("cvt.rpi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ushort2half_rn(const unsigned short int i)
+{
+    __half h;
+#if defined __CUDA_ARCH__
+    asm("cvt.rn.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+#else
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rz(const unsigned short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rd(const unsigned short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_ru(const unsigned short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(const __half h)
+{
+    unsigned long long int i;
+    asm("cvt.rni.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(const __half h)
+{
+    unsigned long long int i;
+    asm("cvt.rmi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(const __half h)
+{
+    unsigned long long int i;
+    asm("cvt.rpi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ull2half_rn(const unsigned long long int i)
+{
+    __half h;
+#if defined(__CUDA_ARCH__)
+    asm("cvt.rn.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+#else
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rz(const unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rd(const unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_ru(const unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ long long int __half2ll_rn(const __half h)
+{
+    long long int i;
+    asm("cvt.rni.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_rd(const __half h)
+{
+    long long int i;
+    asm("cvt.rmi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_ru(const __half h)
+{
+    long long int i;
+    asm("cvt.rpi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ll2half_rn(const long long int i)
+{
+    __half h;
+#if defined(__CUDA_ARCH__)
+    asm("cvt.rn.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+#else
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+#endif
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rz(const long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rd(const long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_ru(const long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ __half htrunc(const __half h)
+{
+    __half r;
+    asm("cvt.rzi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hceil(const __half h)
+{
+    __half r;
+    asm("cvt.rpi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hfloor(const __half h)
+{
+    __half r;
+    asm("cvt.rmi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hrint(const __half h)
+{
+    __half r;
+    asm("cvt.rni.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rzi.f16.f16 low, low;\n"
+        "  cvt.rzi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rpi.f16.f16 low, low;\n"
+        "  cvt.rpi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rmi.f16.f16 low, low;\n"
+        "  cvt.rmi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rni.f16.f16 low, low;\n"
+        "  cvt.rni.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 a, const __half2 b)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {alow,blow};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(b)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 a, const __half2 b)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {ahigh,bhigh};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(b)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __low2half(const __half2 a)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, low;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(a)));
+    return ret;
+}
+__CUDA_FP16_DECL__ int __hisinf(const __half a)
+{
+    int retval;
+    if (__HALF_TO_CUS(a) == 0xFC00U) {
+        retval = -1;
+    } else if (__HALF_TO_CUS(a) == 0x7C00U) {
+        retval = 1;
+    } else {
+        retval = 0;
+    }
+    return retval;
+}
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 a)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 a)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __high2half(const __half2 a)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, high;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(a)));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half a, const __half b)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%2};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(b)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half a)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%1};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 a)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h)
+{
+    return static_cast<short int>(__HALF_TO_CUS(h));
+}
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h)
+{
+    return __HALF_TO_CUS(h);
+}
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = static_cast<unsigned short int>(i);
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = i;
+    return h;
+}
+
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half __hmax(const __half a, const __half b)
+{
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+    __BINARY_OP_HALF_MACRO(max)
+#else
+    const float fa = __half2float(a);
+    const float fb = __half2float(b);
+    float fr;
+    asm("{max.f32 %0,%1,%2;\n}"
+        :"=f"(fr) : "f"(fa), "f"(fb));
+    const __half hr = __float2half(fr);
+    return hr;
+#endif
+}
+__CUDA_FP16_DECL__ __half __hmin(const __half a, const __half b)
+{
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+    __BINARY_OP_HALF_MACRO(min)
+#else
+    const float fa = __half2float(a);
+    const float fb = __half2float(b);
+    float fr;
+    asm("{min.f32 %0,%1,%2;\n}"
+        :"=f"(fr) : "f"(fa), "f"(fb));
+    const __half hr = __float2half(fr);
+    return hr;
+#endif
+}
+
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half2 __hmax2(const __half2 a, const __half2 b)
+{
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+    __BINARY_OP_HALF2_MACRO(max)
+#else
+    const float2 fa = __half22float2(a);
+    const float2 fb = __half22float2(b);
+    float2 fr;
+    asm("{max.f32 %0,%1,%2;\n}"
+        :"=f"(fr.x) : "f"(fa.x), "f"(fb.x));
+    asm("{max.f32 %0,%1,%2;\n}"
+        :"=f"(fr.y) : "f"(fa.y), "f"(fb.y));
+    const __half2 hr = __float22half2_rn(fr);
+    return hr;
+#endif
+}
+__CUDA_FP16_DECL__ __half2 __hmin2(const __half2 a, const __half2 b)
+{
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+    __BINARY_OP_HALF2_MACRO(min)
+#else
+    const float2 fa = __half22float2(a);
+    const float2 fb = __half22float2(b);
+    float2 fr;
+    asm("{min.f32 %0,%1,%2;\n}"
+        :"=f"(fr.x) : "f"(fa.x), "f"(fb.x));
+    asm("{min.f32 %0,%1,%2;\n}"
+        :"=f"(fr.y) : "f"(fa.y), "f"(fb.y));
+    const __half2 hr = __float22half2_rn(fr);
+    return hr;
+#endif
+}
+
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 300)
+/******************************************************************************
+*                           __half, __half2 warp shuffle                     *
+******************************************************************************/
+#define __SHUFFLE_HALF2_MACRO(name) /* do */ {\
+   __half2 r; \
+   asm volatile ("{" __CUDA_FP16_STRINGIFY(name) " %0,%1,%2,%3;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c)); \
+   return r; \
+} /* while(0) */
+
+#define __SHUFFLE_SYNC_HALF2_MACRO(name) /* do */ {\
+   __half2 r; \
+   asm volatile ("{" __CUDA_FP16_STRINGIFY(name) " %0,%1,%2,%3,%4;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c), "r"(mask)); \
+   return r; \
+} /* while(0) */
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+
+__CUDA_FP16_DECL__ __half2 __shfl(const __half2 var, const int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_HALF2_MACRO(shfl.idx.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up(const __half2 var, const unsigned int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = (warp_size - static_cast<unsigned>(width)) << 8U;
+    __SHUFFLE_HALF2_MACRO(shfl.up.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down(const __half2 var, const unsigned int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_HALF2_MACRO(shfl.down.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor(const __half2 var, const int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_HALF2_MACRO(shfl.bfly.b32)
+}
+
+#endif /* !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700 */
+
+__CUDA_FP16_DECL__ __half2 __shfl_sync(const unsigned mask, const __half2 var, const int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.idx.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(const unsigned mask, const __half2 var, const unsigned int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = (warp_size - static_cast<unsigned>(width)) << 8U;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.up.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(const unsigned mask, const __half2 var, const unsigned int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.down.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(const unsigned mask, const __half2 var, const int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.bfly.b32)
+}
+
+#undef __SHUFFLE_HALF2_MACRO
+#undef __SHUFFLE_SYNC_HALF2_MACRO
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+
+__CUDA_FP16_DECL__ __half __shfl(const __half var, const int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up(const __half var, const unsigned int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_up(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down(const __half var, const unsigned int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_down(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor(const __half var, const int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_xor(temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /* !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700 */
+
+__CUDA_FP16_DECL__ __half __shfl_sync(const unsigned mask, const __half var, const int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up_sync(const unsigned mask, const __half var, const unsigned int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_up_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down_sync(const unsigned mask, const __half var, const unsigned int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_down_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(const unsigned mask, const __half var, const int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_xor_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /*!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 300)*/
+/******************************************************************************
+*               __half and __half2 __ldg,__ldcg,__ldca,__ldcs                *
+******************************************************************************/
+
+#if defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320))
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __LDG_PTR   "l"
+#else
+#define __LDG_PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.nc.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldg(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.nc.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cg.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcg(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.cg.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.ca.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldca(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.ca.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cs.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcs(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.cs.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldlu(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.lu.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldlu(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.lu.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcv(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cv.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcv(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.cv.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ void __stwb(__half2 *const ptr, const __half2 value)
+{
+    asm ("st.global.wb.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwb(__half *const ptr, const __half value)
+{
+    asm ("st.global.wb.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcg(__half2 *const ptr, const __half2 value)
+{
+    asm ("st.global.cg.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcg(__half *const ptr, const __half value)
+{
+    asm ("st.global.cg.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcs(__half2 *const ptr, const __half2 value)
+{
+    asm ("st.global.cs.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcs(__half *const ptr, const __half value)
+{
+    asm ("st.global.cs.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwt(__half2 *const ptr, const __half2 value)
+{
+    asm ("st.global.wt.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwt(__half *const ptr, const __half value)
+{
+    asm ("st.global.wt.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+#undef __LDG_PTR
+#endif /*defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320))*/
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)
+/******************************************************************************
+*                             __half2 comparison                             *
+******************************************************************************/
+#define __COMPARISON_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{ " __CUDA_FP16_STRINGIFY(name) ".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.eq)
+}
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ne)
+}
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.le)
+}
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ge)
+}
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.lt)
+}
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gt)
+}
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.equ)
+}
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.neu)
+}
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.leu)
+}
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.geu)
+}
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ltu)
+}
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gtu)
+}
+#undef __COMPARISON_OP_HALF2_MACRO
+#define __BOOL_COMPARISON_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   bool retval; \
+   asm( "{ " __CUDA_FP16_STRINGIFY(name) ".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   if (__HALF2_TO_CUI(val) == 0x3C003C00U) {\
+      retval = true; \
+   } else { \
+      retval = false; \
+   }\
+   return retval;\
+} /* while(0) */
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.eq)
+}
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ne)
+}
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.le)
+}
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ge)
+}
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.lt)
+}
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gt)
+}
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.equ)
+}
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.neu)
+}
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.leu)
+}
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.geu)
+}
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ltu)
+}
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gtu)
+}
+#undef __BOOL_COMPARISON_OP_HALF2_MACRO
+/******************************************************************************
+*                             __half comparison                              *
+******************************************************************************/
+#define __COMPARISON_OP_HALF_MACRO(name) /* do */ {\
+   unsigned short val; \
+   asm( "{ .reg .pred __$temp3;\n" \
+        "  setp." __CUDA_FP16_STRINGIFY(name) ".f16  __$temp3, %1, %2;\n" \
+        "  selp.u16 %0, 1, 0, __$temp3;}" \
+        : "=h"(val) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(b))); \
+   return (val != 0U) ? true : false; \
+} /* while(0) */
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(eq)
+}
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ne)
+}
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(le)
+}
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ge)
+}
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(lt)
+}
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gt)
+}
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(equ)
+}
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(neu)
+}
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(leu)
+}
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(geu)
+}
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ltu)
+}
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gtu)
+}
+#undef __COMPARISON_OP_HALF_MACRO
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul)
+}
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hadd2_rn(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2_rn(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2_rn(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.sat)
+}
+__CUDA_FP16_DECL__ __half2 __h2div(const __half2 a, const __half2 b) {
+    __half ha = __low2half(a);
+    __half hb = __low2half(b);
+
+    const __half v1 = __hdiv(ha, hb);
+
+    ha = __high2half(a);
+    hb = __high2half(b);
+
+    const __half v2 = __hdiv(ha, hb);
+
+    return __halves2half2(v1, v2);
+}
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add)
+}
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub)
+}
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul)
+}
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add.sat)
+}
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub.sat)
+}
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul.sat)
+}
+__CUDA_FP16_DECL__ __half __hadd_rn(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add.rn)
+}
+__CUDA_FP16_DECL__ __half __hsub_rn(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub.rn)
+}
+__CUDA_FP16_DECL__ __half __hmul_rn(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul.rn)
+}
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn)
+}
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.sat)
+}
+__CUDA_FP16_DECL__ __half __hdiv(const __half a, const __half b) {
+    __half v;
+    __half abs;
+    __half den;
+    __HALF_TO_US(den) = 0x008FU;
+
+    float rcp;
+    const float fa = __half2float(a);
+    const float fb = __half2float(b);
+
+    asm("{rcp.approx.ftz.f32 %0, %1;\n}" :"=f"(rcp) : "f"(fb));
+
+    float fv = rcp * fa;
+
+    v = __float2half(fv);
+    __HALF_TO_US(abs) = static_cast<unsigned short>(static_cast<unsigned int>(__HALF_TO_CUS(v)) & 0x00007FFFU);
+    if (__hlt(abs, den) && (!(__HALF_TO_CUS(abs) == 0x0000U))) {
+        const float err = __fmaf_rn(-fb, fv, fa);
+        fv = __fmaf_rn(rcp, err, fv);
+        v = __float2half(fv);
+    }
+    return v;
+}
+
+/******************************************************************************
+*                             __half2 functions                  *
+******************************************************************************/
+#define __SPEC_CASE2(i,r, spc, ulp) \
+   "{.reg.b32 spc, ulp, p;\n"\
+   "  mov.b32 spc," __CUDA_FP16_STRINGIFY(spc) ";\n"\
+   "  mov.b32 ulp," __CUDA_FP16_STRINGIFY(ulp) ";\n"\
+   "  set.eq.f16x2.f16x2 p," __CUDA_FP16_STRINGIFY(i) ", spc;\n"\
+   "  fma.rn.f16x2 " __CUDA_FP16_STRINGIFY(r) ",p,ulp," __CUDA_FP16_STRINGIFY(r) ";\n}\n"
+#define __SPEC_CASE(i,r, spc, ulp) \
+   "{.reg.b16 spc, ulp, p;\n"\
+   "  mov.b16 spc," __CUDA_FP16_STRINGIFY(spc) ";\n"\
+   "  mov.b16 ulp," __CUDA_FP16_STRINGIFY(ulp) ";\n"\
+   "  set.eq.f16.f16 p," __CUDA_FP16_STRINGIFY(i) ", spc;\n"\
+   "  fma.rn.f16 " __CUDA_FP16_STRINGIFY(r) ",p,ulp," __CUDA_FP16_STRINGIFY(r) ";\n}\n"
+#define __APPROX_FCAST(fun) /* do */ {\
+   __half val;\
+   asm("{.reg.b32         f;        \n"\
+                " .reg.b16         r;        \n"\
+                "  mov.b16         r,%1;     \n"\
+                "  cvt.f32.f16     f,r;      \n"\
+                "  " __CUDA_FP16_STRINGIFY(fun) ".approx.ftz.f32   f,f;  \n"\
+                "  cvt.rn.f16.f32      r,f;  \n"\
+                "  mov.b16         %0,r;     \n"\
+                "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));\
+   return val;\
+} /* while(0) */
+#define __APPROX_FCAST2(fun) /* do */ {\
+   __half2 val;\
+   asm("{.reg.b16         hl, hu;         \n"\
+                " .reg.b32         fl, fu;         \n"\
+                "  mov.b32         {hl, hu}, %1;   \n"\
+                "  cvt.f32.f16     fl, hl;         \n"\
+                "  cvt.f32.f16     fu, hu;         \n"\
+                "  " __CUDA_FP16_STRINGIFY(fun) ".approx.ftz.f32   fl, fl;     \n"\
+                "  " __CUDA_FP16_STRINGIFY(fun) ".approx.ftz.f32   fu, fu;     \n"\
+                "  cvt.rn.f16.f32      hl, fl;     \n"\
+                "  cvt.rn.f16.f32      hu, fu;     \n"\
+                "  mov.b32         %0, {hl, hu};   \n"\
+                "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));       \
+   return val;\
+} /* while(0) */
+static __device__ __forceinline__ float __float_simpl_sinf(float a);
+static __device__ __forceinline__ float __float_simpl_cosf(float a);
+__CUDA_FP16_DECL__ __half hsin(const __half a) {
+    const float sl = __float_simpl_sinf(__half2float(a));
+    __half r = __float2half_rn(sl);
+    asm("{\n\t"
+        "  .reg.b16 i,r,t;     \n\t"
+        "  mov.b16 r, %0;      \n\t"
+        "  mov.b16 i, %1;      \n\t"
+        "  and.b16 t, r, 0x8000U; \n\t"
+        "  abs.f16 r, r;   \n\t"
+        "  abs.f16 i, i;   \n\t"
+        __SPEC_CASE(i, r, 0X32B3U, 0x0800U)
+        __SPEC_CASE(i, r, 0X5CB0U, 0x9000U)
+        "  or.b16  r,r,t;      \n\t"
+        "  mov.b16 %0, r;      \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a) {
+    const float sl = __float_simpl_sinf(__half2float(a.x));
+    const float sh = __float_simpl_sinf(__half2float(a.y));
+    __half2 r = __floats2half2_rn(sl, sh);
+    asm("{\n\t"
+        "  .reg.b32 i,r,t;             \n\t"
+        "  mov.b32 r, %0;              \n\t"
+        "  mov.b32 i, %1;              \n\t"
+        "  and.b32 t, r, 0x80008000U;   \n\t"
+        "  abs.f16x2 r, r;   \n\t"
+        "  abs.f16x2 i, i;   \n\t"
+        __SPEC_CASE2(i, r, 0X32B332B3U, 0x08000800U)
+        __SPEC_CASE2(i, r, 0X5CB05CB0U, 0x90009000U)
+        "  or.b32  r, r, t;            \n\t"
+        "  mov.b32 %0, r;              \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hcos(const __half a) {
+    const float cl = __float_simpl_cosf(__half2float(a));
+    __half r = __float2half_rn(cl);
+    asm("{\n\t"
+        "  .reg.b16 i,r;        \n\t"
+        "  mov.b16 r, %0;       \n\t"
+        "  mov.b16 i, %1;       \n\t"
+        "  abs.f16 i, i;        \n\t"
+        __SPEC_CASE(i, r, 0X2B7CU, 0x1000U)
+        "  mov.b16 %0, r;       \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a) {
+    const float cl = __float_simpl_cosf(__half2float(a.x));
+    const float ch = __float_simpl_cosf(__half2float(a.y));
+    __half2 r = __floats2half2_rn(cl, ch);
+    asm("{\n\t"
+        "  .reg.b32 i,r;   \n\t"
+        "  mov.b32 r, %0;  \n\t"
+        "  mov.b32 i, %1;  \n\t"
+        "  abs.f16x2 i, i; \n\t"
+        __SPEC_CASE2(i, r, 0X2B7C2B7CU, 0x10001000U)
+        "  mov.b32 %0, r;  \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+static __device__ __forceinline__ float __internal_trig_reduction_kernel(const float a, unsigned int *const quadrant)
+{
+    const float ar = __fmaf_rn(a, 0.636619772F, 12582912.0F);
+    const unsigned q = __float_as_uint(ar);
+    const float j = __fsub_rn(ar, 12582912.0F);
+    float t = __fmaf_rn(j, -1.5707962512969971e+000F, a);
+    t = __fmaf_rn(j, -7.5497894158615964e-008F, t);
+    *quadrant = q;
+    return t;
+}
+static __device__ __forceinline__ float __internal_sin_cos_kernel(const float x, const unsigned int i)
+{
+    float z;
+    const float x2 = x*x;
+    float a8;
+    float a6;
+    float a4;
+    float a2;
+    float a1;
+    float a0;
+
+    if ((i & 1U) != 0U) {
+        // cos
+        a8 =  2.44331571e-5F;
+        a6 = -1.38873163e-3F;
+        a4 =  4.16666457e-2F;
+        a2 = -5.00000000e-1F;
+        a1 = x2;
+        a0 = 1.0F;
+    }
+    else {
+        // sin
+        a8 = -1.95152959e-4F;
+        a6 =  8.33216087e-3F;
+        a4 = -1.66666546e-1F;
+        a2 = 0.0F;
+        a1 = x;
+        a0 = x;
+    }
+
+    z = __fmaf_rn(a8, x2, a6);
+    z = __fmaf_rn(z, x2, a4);
+    z = __fmaf_rn(z, x2, a2);
+    z = __fmaf_rn(z, a1, a0);
+
+    if ((i & 2U) != 0U) {
+        z = -z;
+    }
+    return z;
+}
+static __device__ __forceinline__ float __float_simpl_sinf(float a)
+{
+    float z;
+    unsigned i;
+    a = __internal_trig_reduction_kernel(a, &i);
+    z = __internal_sin_cos_kernel(a, i);
+    return z;
+}
+static __device__ __forceinline__ float __float_simpl_cosf(float a)
+{
+    float z;
+    unsigned i;
+    a = __internal_trig_reduction_kernel(a, &i);
+    z = __internal_sin_cos_kernel(a, (i & 0x3U) + 1U);
+    return z;
+}
+
+__CUDA_FP16_DECL__ __half hexp(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C, nZ;       \n"
+        " .reg.b16         h,r;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  mov.b32         C, 0x3fb8aa3bU; \n"
+        "  mov.b32         nZ, 0x80000000U;\n"
+        "  fma.rn.f32      f,f,C,nZ;       \n"
+        "  ex2.approx.ftz.f32  f,f;        \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X1F79U, 0x9400U)
+        __SPEC_CASE(h, r, 0X25CFU, 0x9400U)
+        __SPEC_CASE(h, r, 0XC13BU, 0x0400U)
+        __SPEC_CASE(h, r, 0XC1EFU, 0x0200U)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu,C,nZ; \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x3fb8aa3bU; \n"
+        "  mov.b32         nZ, 0x80000000U;\n"
+        "  fma.rn.f32      fl,fl,C,nZ;     \n"
+        "  fma.rn.f32      fu,fu,C,nZ;     \n"
+        "  ex2.approx.ftz.f32  fl, fl;     \n"
+        "  ex2.approx.ftz.f32  fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0X1F791F79U, 0x94009400U)
+        __SPEC_CASE2(h, r, 0X25CF25CFU, 0x94009400U)
+        __SPEC_CASE2(h, r, 0XC13BC13BU, 0x04000400U)
+        __SPEC_CASE2(h, r, 0XC1EFC1EFU, 0x02000200U)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp2(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, ULP;         \n"
+        " .reg.b16         r;              \n"
+        "  mov.b16         r,%1;           \n"
+        "  cvt.f32.f16     f,r;            \n"
+        "  ex2.approx.ftz.f32      f,f;    \n"
+        "  mov.b32         ULP, 0x33800000U;\n"
+        "  fma.rn.f32      f,f,ULP,f;      \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, ULP;    \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  ex2.approx.ftz.f32  fl, fl;     \n"
+        "  ex2.approx.ftz.f32  fu, fu;     \n"
+        "  mov.b32         ULP, 0x33800000U;\n"
+        "  fma.rn.f32      fl,fl,ULP,fl;   \n"
+        "  fma.rn.f32      fu,fu,ULP,fu;   \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         %0, {hl, hu};   \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h,r;            \n"
+        " .reg.b32         f, C, nZ;       \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  mov.b32         C, 0x40549A78U; \n"
+        "  mov.b32         nZ, 0x80000000U;\n"
+        "  fma.rn.f32      f,f,C,nZ;       \n"
+        "  ex2.approx.ftz.f32  f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x34DEU, 0x9800U)
+        __SPEC_CASE(h, r, 0x9766U, 0x9000U)
+        __SPEC_CASE(h, r, 0x9972U, 0x1000U)
+        __SPEC_CASE(h, r, 0xA5C4U, 0x1000U)
+        __SPEC_CASE(h, r, 0xBF0AU, 0x8100U)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu,C,nZ; \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x40549A78U; \n"
+        "  mov.b32         nZ, 0x80000000U;\n"
+        "  fma.rn.f32      fl,fl,C,nZ;     \n"
+        "  fma.rn.f32      fu,fu,C,nZ;     \n"
+        "  ex2.approx.ftz.f32  fl, fl;     \n"
+        "  ex2.approx.ftz.f32  fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0x34DE34DEU, 0x98009800U)
+        __SPEC_CASE2(h, r, 0x97669766U, 0x90009000U)
+        __SPEC_CASE2(h, r, 0x99729972U, 0x10001000U)
+        __SPEC_CASE2(h, r, 0xA5C4A5C4U, 0x10001000U)
+        __SPEC_CASE2(h, r, 0xBF0ABF0AU, 0x81008100U)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog2(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f;              \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.ftz.f32  f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(r, r, 0xA2E2U, 0x8080U)
+        __SPEC_CASE(r, r, 0xBF46U, 0x9400U)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, r, p;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  lg2.approx.ftz.f32  fl, fl;     \n"
+        "  lg2.approx.ftz.f32  fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(r, r, 0xA2E2A2E2U, 0x80808080U)
+        __SPEC_CASE2(r, r, 0xBF46BF46U, 0x94009400U)
+        "  mov.b32         %0, r;          \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C;           \n"
+        " .reg.b16         r,h;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  lg2.approx.ftz.f32  f,f;        \n"
+        "  mov.b32         C, 0x3f317218U;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X160DU, 0x9C00U)
+        __SPEC_CASE(h, r, 0X3BFEU, 0x8010U)
+        __SPEC_CASE(h, r, 0X3C0BU, 0x8080U)
+        __SPEC_CASE(h, r, 0X6051U, 0x1C00U)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.ftz.f32  fl, fl;         \n"
+        "  lg2.approx.ftz.f32  fu, fu;         \n"
+        "  mov.b32         C, 0x3f317218U;     \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0X160D160DU, 0x9C009C00U)
+        __SPEC_CASE2(h, r, 0X3BFE3BFEU, 0x80108010U)
+        __SPEC_CASE2(h, r, 0X3C0B3C0BU, 0x80808080U)
+        __SPEC_CASE2(h, r, 0X60516051U, 0x1C001C00U)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f, C;           \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.ftz.f32  f, f;       \n"
+        "  mov.b32         C, 0x3E9A209BU; \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x338FU, 0x1000U)
+        __SPEC_CASE(h, r, 0x33F8U, 0x9000U)
+        __SPEC_CASE(h, r, 0x57E1U, 0x9800U)
+        __SPEC_CASE(h, r, 0x719DU, 0x9C00U)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.ftz.f32  fl, fl;         \n"
+        "  lg2.approx.ftz.f32  fu, fu;         \n"
+        "  mov.b32         C, 0x3E9A209BU;     \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0x338F338FU, 0x10001000U)
+        __SPEC_CASE2(h, r, 0x33F833F8U, 0x90009000U)
+        __SPEC_CASE2(h, r, 0x57E157E1U, 0x98009800U)
+        __SPEC_CASE2(h, r, 0x719D719DU, 0x9C009C00U)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+#undef __SPEC_CASE2
+#undef __SPEC_CASE
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a) {
+    __APPROX_FCAST2(rcp)
+}
+__CUDA_FP16_DECL__ __half hrcp(const __half a) {
+    __APPROX_FCAST(rcp)
+}
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a) {
+    __APPROX_FCAST2(rsqrt)
+}
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a) {
+    __APPROX_FCAST(rsqrt)
+}
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a) {
+    __APPROX_FCAST2(sqrt)
+}
+__CUDA_FP16_DECL__ __half hsqrt(const __half a) {
+    __APPROX_FCAST(sqrt)
+}
+#undef __APPROX_FCAST
+#undef __APPROX_FCAST2
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a)
+{
+    __half2 r;
+    asm("{set.nan.f16x2.f16x2 %0,%1,%2;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ bool __hisnan(const __half a)
+{
+    __half r;
+    asm("{set.nan.f16.f16 %0,%1,%2;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(a)));
+    return __HALF_TO_CUS(r) != 0U;
+}
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a)
+{
+    __half2 r;
+    asm("{neg.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __hneg(const __half a)
+{
+    __half r;
+    asm("{neg.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a)
+{
+    __half2 r;
+    asm("{abs.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __habs(const __half a)
+{
+    __half r;
+    asm("{abs.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+
+__CUDA_FP16_DECL__ __half2 __hcmadd(const __half2 a, const __half2 b, const __half2 c)
+{
+    // fast version of complex multiply-accumulate
+    // (a.re, a.im) * (b.re, b.im) + (c.re, c.im)
+    // acc.re = (c.re + a.re*b.re) - a.im*b.im
+    // acc.im = (c.im + a.re*b.im) + a.im*b.re
+    __half real_tmp =  __hfma(a.x, b.x, c.x);
+    __half img_tmp  =  __hfma(a.x, b.y, c.y);
+    real_tmp = __hfma(__hneg(a.y), b.y, real_tmp);
+    img_tmp  = __hfma(a.y,         b.x, img_tmp);
+    return make_half2(real_tmp, img_tmp);
+}
+
+#endif /*!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)*/
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+__CUDA_FP16_DECL__ __half __hmax_nan(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(max.NaN)
+}
+__CUDA_FP16_DECL__ __half __hmin_nan(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(min.NaN)
+}
+__CUDA_FP16_DECL__ __half __hfma_relu(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.relu)
+}
+
+__CUDA_FP16_DECL__ __half2 __hmax2_nan(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(max.NaN)
+}
+__CUDA_FP16_DECL__ __half2 __hmin2_nan(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(min.NaN)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_relu(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.relu)
+}
+#endif /*!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)*/
+
+/* Define __PTR for atomicAdd prototypes below, undef after done */
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __PTR   "l"
+#else
+#define __PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600
+
+__CUDA_FP16_DECL__  __half2 atomicAdd(__half2 *const address, const __half2 val) {
+    __half2 r;
+    asm volatile ("{ atom.add.noftz.f16x2 %0,[%1],%2; }\n"
+                  : "=r"(__HALF2_TO_UI(r)) : __PTR(address), "r"(__HALF2_TO_CUI(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600*/
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700
+
+__CUDA_FP16_DECL__  __half atomicAdd(__half *const address, const __half val) {
+    __half r;
+    asm volatile ("{ atom.add.noftz.f16 %0,[%1],%2; }\n"
+                  : "=h"(__HALF_TO_US(r))
+                  : __PTR(address), "h"(__HALF_TO_CUS(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700*/
+
+#undef __PTR
+
+#undef __CUDA_FP16_DECL__
+#endif /* defined(__CUDACC__) */
+#endif /* defined(__cplusplus) */
+
+#undef __TERNARY_OP_HALF2_MACRO
+#undef __TERNARY_OP_HALF_MACRO
+#undef __BINARY_OP_HALF2_MACRO
+#undef __BINARY_OP_HALF_MACRO
+
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+#undef __CUDA_FP16_DECL__
+
+#undef __HALF_TO_US
+#undef __HALF_TO_CUS
+#undef __HALF2_TO_UI
+#undef __HALF2_TO_CUI
+
+/* Define first-class types "half" and "half2", unless user specifies otherwise via "#define CUDA_NO_HALF" */
+/* C cannot ever have these types defined here, because __half and __half2 are C++ classes */
+#if defined(__cplusplus) && !defined(CUDA_NO_HALF)
+typedef __half half;
+typedef __half2 half2;
+// for consistency with __nv_bfloat16
+typedef __half      __nv_half;
+typedef __half2     __nv_half2;
+typedef __half_raw  __nv_half_raw;
+typedef __half2_raw __nv_half2_raw;
+typedef __half        nv_half;
+typedef __half2       nv_half2;
+#endif /* defined(__cplusplus) && !defined(CUDA_NO_HALF) */
+
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+#undef __CPP_VERSION_AT_LEAST_11_FP16
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+
+#endif /* end of include guard: __CUDA_FP16_HPP__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-12/cuda_fp16.h b/cupy/_core/include/cupy/_cuda/cuda-12/cuda_fp16.h
new file mode 100644
index 0000000..01981ed
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-12/cuda_fp16.h
@@ -0,0 +1,4023 @@
+/*
+* Copyright 1993-2021 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+/**
+* \defgroup CUDA_MATH_INTRINSIC_HALF Half Precision Intrinsics
+* This section describes half precision intrinsic functions that are
+* only supported in device code.
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+* The following macros are available to help users selectively enable/disable
+* various definitions present in the header file:
+* - \p CUDA_NO_HALF - If defined, this macro will prevent the definition of
+* additional type aliases in the global namespace, helping to avoid potential
+* conflicts with symbols defined in the user program.
+* - \p __CUDA_NO_HALF_CONVERSIONS__ - If defined, this macro will prevent the
+* use of the C++ type conversions (converting constructors and conversion
+* operators) that are common for built-in floating-point types, but may be
+* undesirable for \p half which is essentially a user-defined type.
+* - \p __CUDA_NO_HALF_OPERATORS__ and \p __CUDA_NO_HALF2_OPERATORS__ - If
+* defined, these macros will prevent the inadvertent use of usual arithmetic
+* and comparison operators. This enforces the storage-only type semantics and
+* prevents C++ style computations on \p half and \p half2 types.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_ARITHMETIC Half Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_ARITHMETIC Half2 Arithmetic Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_COMPARISON Half Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_COMPARISON Half2 Comparison Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_MISC Half Precision Conversion and Data Movement
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF_FUNCTIONS Half Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+/**
+* \defgroup CUDA_MATH__HALF2_FUNCTIONS Half2 Math Functions
+* \ingroup CUDA_MATH_INTRINSIC_HALF
+* To use these functions, include the header file \p cuda_fp16.h in your program.
+*/
+
+#ifndef __CUDA_FP16_H__
+#define __CUDA_FP16_H__
+
+#define ___CUDA_FP16_STRINGIFY_INNERMOST(x) #x
+#define __CUDA_FP16_STRINGIFY(x) ___CUDA_FP16_STRINGIFY_INNERMOST(x)
+
+#if defined(__cplusplus)
+#if defined(__CUDACC__)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__CUDACC__) */
+
+#define __CUDA_FP16_TYPES_EXIST__
+
+/* Forward-declaration of structures defined in "cuda_fp16.hpp" */
+
+/**
+ * \brief half datatype 
+ * 
+ * \details This structure implements the datatype for storing 
+ * half-precision floating-point numbers. The structure implements 
+ * assignment operators and type conversions. 
+ * 16 bits are being used in total: 1 sign bit, 5 bits for the exponent, 
+ * and the significand is being stored in 10 bits. 
+ * The total precision is 11 bits. There are 15361 representable 
+ * numbers within the interval [0.0, 1.0], endpoints included. 
+ * On average we have log10(2**11) ~ 3.311 decimal digits. 
+ * 
+ * \internal
+ * \req IEEE 754-2008 compliant implementation of half-precision 
+ * floating-point numbers. 
+ * \endinternal
+ */
+struct __half;
+
+/**
+ * \brief half2 datatype
+ * 
+ * \details This structure implements the datatype for storing two 
+ * half-precision floating-point numbers. 
+ * The structure implements assignment operators and type conversions. 
+ * 
+ * \internal
+ * \req Vectorified version of half. 
+ * \endinternal
+ */
+struct __half2;
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts double number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts double number \p a to half precision in round-to-nearest-even mode.
+* \param[in] a - double. Is only being read.
+* \returns half
+* - \p a converted to half.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __double2half(const double a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value. 
+* 
+* \details Converts float number \p a to half precision in round-to-nearest-even mode. 
+* \param[in] a - float. Is only being read. 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-to-nearest-even mode
+* and returns \p half with converted value.
+*
+* \details Converts float number \p a to half precision in round-to-nearest-even mode.
+* \param[in] a - float. Is only being read. 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-towards-zero mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-towards-zero mode.
+* \param[in] a - float. Is only being read. 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-down mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-down mode.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts float number to half precision in round-up mode
+* and returns \p half with converted value.
+* 
+* \details Converts float number \p a to half precision in round-up mode.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns half
+* - \p a converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts \p half number to float.
+* 
+* \details Converts half number \p a to float.
+* \param[in] a - float. Is only being read. 
+* 
+* \returns float
+* - \p a converted to float. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts input to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+*
+* \details Converts input \p a to half precision in round-to-nearest-even mode and
+* populates both halves of \p half2 with converted value.
+* \param[in] a - float. Is only being read. 
+*
+* \returns half2
+* - The \p half2 value with both halves equal to the converted half
+* precision number.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both input floats to half precision in round-to-nearest-even
+* mode and returns \p half2 with converted values.
+*
+* \details Converts both input floats to half precision in round-to-nearest-even mode
+* and combines the results into one \p half2 number. Low 16 bits of the return
+* value correspond to the input \p a, high 16 bits correspond to the input \p
+* b.
+* \param[in] a - float. Is only being read. 
+* \param[in] b - float. Is only being read. 
+* 
+* \returns half2
+* - The \p half2 value with corresponding halves equal to the
+* converted input floats.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts low 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts low 16 bits of \p half2 input \p a to 32-bit floating-point number
+* and returns the result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* - The low 16 bits of \p a converted to float.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts high 16 bits of \p half2 to float and returns the result
+* 
+* \details Converts high 16 bits of \p half2 input \p a to 32-bit floating-point number
+* and returns the result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float
+* - The high 16 bits of \p a converted to float.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-towards-zero mode.
+*
+* \details Convert the half-precision floating-point value \p h to a signed short
+* integer in round-towards-zero mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read.
+*
+* \returns short int
+* - \p h converted to a signed short integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ short int __half2short_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-towards-zero
+* mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned short
+* integer in round-towards-zero mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read.
+*
+* \returns unsigned short int
+* - \p h converted to an unsigned short integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned short int __half2ushort_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-towards-zero mode.
+*
+* \details Convert the half-precision floating-point value \p h to a signed integer in
+* round-towards-zero mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read.
+*
+* \returns int
+* - \p h converted to a signed integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ int __half2int_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-towards-zero mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned integer
+* in round-towards-zero mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read.
+*
+* \returns unsigned int
+* - \p h converted to an unsigned integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned int __half2uint_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-towards-zero mode.
+*
+* \details Convert the half-precision floating-point value \p h to a signed 64-bit
+* integer in round-towards-zero mode. NaN inputs return a long long int with hex value of 0x8000000000000000.
+* \param[in] h - half. Is only being read.
+*
+* \returns long long int
+* - \p h converted to a signed 64-bit integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ long long int __half2ll_rz(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-towards-zero
+* mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned 64-bit
+* integer in round-towards-zero mode. NaN inputs return 0x8000000000000000.
+* \param[in] h - half. Is only being read.
+*
+* \returns unsigned long long int
+* - \p h converted to an unsigned 64-bit integer.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned long long int __half2ull_rz(const __half h);
+
+#if defined(__CUDACC__)
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both components of float2 number to half precision in
+* round-to-nearest-even mode and returns \p half2 with converted values.
+* 
+* \details Converts both components of float2 to half precision in round-to-nearest
+* mode and combines the results into one \p half2 number. Low 16 bits of the
+* return value correspond to \p a.x and high 16 bits of the return value
+* correspond to \p a.y.
+* \param[in] a - float2. Is only being read. 
+*  
+* \returns half2
+* - The \p half2 which has corresponding halves equal to the
+* converted float2 components.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Converts both halves of \p half2 to float2 and returns the result.
+* 
+* \details Converts both halves of \p half2 input \p a to float2 and returns the
+* result.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns float2
+* - \p a converted to float2.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed integer in
+* round-to-nearest-even mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* - \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed integer in
+* round-down mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* - \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed integer in
+* round-up mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns int
+* - \p h converted to a signed integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __half2int_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __int2half_rn(const int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rz(const int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-down mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_rd(const int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed integer to a half in round-up mode.
+* 
+* \details Convert the signed integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __int2half_ru(const int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed short
+* integer in round-to-nearest-even mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* - \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed short
+* integer in round-down mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* - \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed short integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed short
+* integer in round-up mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* - \p h converted to a signed short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half2short_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __short2half_rn(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rz(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-down mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_rd(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed short integer to a half in round-up mode.
+* 
+* \details Convert the signed short integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short2half_ru(const short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-to-nearest-even mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned integer
+* in round-to-nearest-even mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned int
+* - \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-down mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned integer
+* in round-down mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* - \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned integer in round-up mode.
+*
+* \details Convert the half-precision floating-point value \p h to an unsigned integer
+* in round-up mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+*
+* \returns unsigned int
+* - \p h converted to an unsigned integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-to-nearest-even mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __uint2half_rn(const unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-towards-zero mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half.  
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rz(const unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-down mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_rd(const unsigned int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned integer to a half in round-up mode.
+* 
+* \details Convert the unsigned integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - unsigned int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __uint2half_ru(const unsigned int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned short
+* integer in round-to-nearest-even mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* - \p h converted to an unsigned short integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned short
+* integer in round-down mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* - \p h converted to an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned short integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned short
+* integer in round-up mode. NaN inputs are converted to 0.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* - \p h converted to an unsigned short integer. 
+*/
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ushort2half_rn(const unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rz(const unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-down mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_rd(const unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned short integer to a half in round-up mode.
+* 
+* \details Convert the unsigned short integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort2half_ru(const unsigned short int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned 64-bit
+* integer in round-to-nearest-even mode. NaN inputs return 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* - \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned 64-bit
+* integer in round-down mode. NaN inputs return 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* - \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to an unsigned 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to an unsigned 64-bit
+* integer in round-up mode. NaN inputs return 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned long long int
+* - \p h converted to an unsigned 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ull2half_rn(const unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-towards-zero
+* mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rz(const unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half.  
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_rd(const unsigned long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert an unsigned 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the unsigned 64-bit integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - unsigned long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ull2half_ru(const unsigned long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-to-nearest-even
+* mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed 64-bit
+* integer in round-to-nearest-even mode. NaN inputs return a long long int with hex value of 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* - \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rn(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-down mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed 64-bit
+* integer in round-down mode. NaN inputs return a long long int with hex value of 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* - \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_rd(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a half to a signed 64-bit integer in round-up mode.
+* 
+* \details Convert the half-precision floating-point value \p h to a signed 64-bit
+* integer in round-up mode. NaN inputs return a long long int with hex value of 0x8000000000000000.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns long long int
+* - \p h converted to a signed 64-bit integer. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ long long int __half2ll_ru(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-to-nearest-even
+* mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating-point
+* value in round-to-nearest-even mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ll2half_rn(const long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-towards-zero mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating-point
+* value in round-towards-zero mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rz(const long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-down mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating-point
+* value in round-down mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_rd(const long long int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Convert a signed 64-bit integer to a half in round-up mode.
+* 
+* \details Convert the signed 64-bit integer value \p i to a half-precision floating-point
+* value in round-up mode.
+* \param[in] i - long long int. Is only being read. 
+* 
+* \returns half
+* - \p i converted to half. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ll2half_ru(const long long int i);
+
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Truncate input argument to the integral part.
+* 
+* \details Round \p h to the nearest integer value that does not exceed \p h in
+* magnitude.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* - The truncated integer value. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half htrunc(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate ceiling of the input argument.
+* 
+* \details Compute the smallest integer value not less than \p h.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* - The smallest integer value not less than \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hceil(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details Calculate the largest integer value which is less than or equal to \p h.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* - The largest integer value which is less than or equal to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hfloor(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating-point
+* number.
+* 
+* \details Round \p h to the nearest integer value in half-precision floating-point
+* format, with halfway cases rounded to the nearest even integer value.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns half
+* - The nearest integer to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrint(const __half h);
+
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Truncate \p half2 vector input argument to the integral part.
+* 
+* \details Round each component of vector \p h to the nearest integer value that does
+* not exceed \p h in magnitude.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* - The truncated \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate \p half2 vector ceiling of the input argument.
+* 
+* \details For each component of vector \p h compute the smallest integer value not less
+* than \p h.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector of smallest integers not less than \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculate the largest integer less than or equal to \p h.
+* 
+* \details For each component of vector \p h calculate the largest integer value which
+* is less than or equal to \p h.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector of largest integers which is less than or equal to \p h. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Round input to nearest integer value in half-precision floating-point
+* number.
+* 
+* \details Round each component of \p half2 vector \p h to the nearest integer value in
+* half-precision floating-point format, with halfway cases rounded to the
+* nearest even integer value.
+* \param[in] h - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector of rounded integer values. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns \p half2 with both halves equal to the input value.
+* 
+* \details Returns \p half2 number with both halves equal to the input \p a \p half
+* number.
+* \param[in] a - half. Is only being read. 
+* 
+* \returns half2
+* - The vector which has both its halves equal to the input \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Swaps both halves of the \p half2 input.
+* 
+* \details Swaps both halves of the \p half2 input and returns a new \p half2 number
+* with swapped halves.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - \p a with its halves being swapped. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from each of the two \p half2 inputs and combines
+* into one \p half2 number. 
+* 
+* \details Extracts low 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. Low 16 bits from input \p a is stored in low 16 bits of
+* the return value, low 16 bits from input \p b is stored in high 16 bits of
+* the return value. 
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The low 16 bits of \p a and of \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from each of the two \p half2 inputs and
+* combines into one \p half2 number.
+* 
+* \details Extracts high 16 bits from each of the two \p half2 inputs and combines into
+* one \p half2 number. High 16 bits from input \p a is stored in low 16 bits of
+* the return value, high 16 bits from input \p b is stored in high 16 bits of
+* the return value.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The high 16 bits of \p a and of \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns high 16 bits of \p half2 input.
+*
+* \details Returns high 16 bits of \p half2 input \p a.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* - The high 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __high2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Returns low 16 bits of \p half2 input.
+*
+* \details Returns low 16 bits of \p half2 input \p a.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half
+* - Returns \p half which contains low 16 bits of the input \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __low2half(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Checks if the input \p half number is infinite.
+* 
+* \details Checks if the input \p half number \p a is infinite. 
+* \param[in] a - half. Is only being read. 
+* 
+* \returns int 
+* - -1 iff \p a is equal to negative infinity, 
+* - 1 iff \p a is equal to positive infinity, 
+* - 0 otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ int __hisinf(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Combines two \p half numbers into one \p half2 number.
+* 
+* \details Combines two input \p half number \p a and \p b into one \p half2 number.
+* Input \p a is stored in low 16 bits of the return value, input \p b is stored
+* in high 16 bits of the return value.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half2
+* - The half2 with one half equal to \p a and the other to \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts low 16 bits from \p half2 input.
+* 
+* \details Extracts low 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The half2 with both halves equal to the low 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Extracts high 16 bits from \p half2 input.
+* 
+* \details Extracts high 16 bits from \p half2 input \p a and returns a new \p half2
+* number which has both halves equal to the extracted bits.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The half2 with both halves equal to the high 16 bits of the input. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 a);
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as a signed short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating-point number \p h
+* as a signed short integer. 
+* \param[in] h - half. Is only being read. 
+* 
+* \returns short int
+* - The reinterpreted value. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a \p half as an unsigned short integer.
+* 
+* \details Reinterprets the bits in the half-precision floating-point \p h
+* as an unsigned short number.
+* \param[in] h - half. Is only being read. 
+* 
+* \returns unsigned short int
+* - The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in a signed short integer as a \p half.
+* 
+* \details Reinterprets the bits in the signed short integer \p i as a
+* half-precision floating-point number.
+* \param[in] i - short int. Is only being read. 
+* 
+* \returns half
+* - The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Reinterprets bits in an unsigned short integer as a \p half.
+* 
+* \details Reinterprets the bits in the unsigned short integer \p i as a
+* half-precision floating-point number.
+* \param[in] i - unsigned short int. Is only being read. 
+* 
+* \returns half
+* - The reinterpreted value.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half maximum of two input values.
+*
+* \details Calculates \p half max(\p a, \p b)
+* defined as (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmax(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half minimum of two input values.
+*
+* \details Calculates \p half min(\p a, \p b)
+* defined as (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmin(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector maximum of two inputs.
+*
+* \details Calculates \p half2 vector max(\p a, \p b).
+* Elementwise \p half operation is defined as
+* (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise maximum of vectors \p a  and \p b
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmax2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector minimum of two inputs.
+*
+* \details Calculates \p half2 vector min(\p a, \p b).
+* Elementwise \p half operation is defined as
+* (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, the other input is returned.
+* - If both inputs are NaNs, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise minimum of vectors \p a  and \p b
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmin2(const __half2 a, const __half2 b);
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 300)
+#if !defined warpSize && !defined __local_warpSize
+#define warpSize    32
+#define __local_warpSize
+#endif
+
+#if defined(_WIN32)
+# define __DEPRECATED__(msg) __declspec(deprecated(msg))
+#elif (defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 5 && !defined(__clang__))))
+# define __DEPRECATED__(msg) __attribute__((deprecated))
+#else
+# define __DEPRECATED__(msg) __attribute__((deprecated(msg)))
+#endif
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
+#define __WSB_DEPRECATION_MESSAGE(x) __CUDA_FP16_STRINGIFY(x) "() is deprecated in favor of " __CUDA_FP16_STRINGIFY(x) "_sync() and may be removed in a future release (Use -Wno-deprecated-declarations to suppress this warning)."
+
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half2 __shfl(const __half2 var, const int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half2 __shfl_up(const __half2 var, const unsigned int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down))__half2 __shfl_down(const __half2 var, const unsigned int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half2 __shfl_xor(const __half2 var, const int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl)) __half __shfl(const __half var, const int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_up)) __half __shfl_up(const __half var, const unsigned int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_down)) __half __shfl_down(const __half var, const unsigned int delta, const int width = warpSize);
+__CUDA_FP16_DECL__ __DEPRECATED__(__WSB_DEPRECATION_MESSAGE(__shfl_xor)) __half __shfl_xor(const __half var, const int delta, const int width = warpSize);
+#endif
+
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* within the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_sync(const unsigned mask, const __half2 var, const int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(const unsigned mask, const __half2 var, const unsigned int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(const unsigned mask, const __half2 var, const unsigned int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half2. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 4-byte word referenced by var from the source thread ID as half2. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(const unsigned mask, const __half2 var, const int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Direct copy from indexed thread. 
+* 
+* \details Returns the value of var held by the thread whose ID is given by delta. 
+* If width is less than warpSize then each subsection of the warp behaves as a separate 
+* entity with a starting logical thread ID of 0. If delta is outside the range [0:width-1], 
+* the value returned corresponds to the value of var held by the delta modulo width (i.e. 
+* within the same subsection). width must have a value which is a power of 2; 
+* results are undefined if width is not a power of 2, or is a number greater than 
+* warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_sync(const unsigned mask, const __half var, const int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with lower ID relative to the caller. 
+* \details Calculates a source thread ID by subtracting delta from the caller's lane ID. 
+* The value of var held by the resulting lane ID is returned: in effect, var is shifted up 
+* the warp by delta threads. If width is less than warpSize then each subsection of the warp 
+* behaves as a separate entity with a starting logical thread ID of 0. The source thread index 
+* will not wrap around the value of width, so effectively the lower delta threads will be unchanged. 
+* width must have a value which is a power of 2; results are undefined if width is not a power of 2, 
+* or is a number greater than warpSize. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_up_sync(const unsigned mask, const __half var, const unsigned int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread with higher ID relative to the caller. 
+* 
+* \details Calculates a source thread ID by adding delta to the caller's thread ID. 
+* The value of var held by the resulting thread ID is returned: this has the effect 
+* of shifting var down the warp by delta threads. If width is less than warpSize then 
+* each subsection of the warp behaves as a separate entity with a starting logical 
+* thread ID of 0. As for __shfl_up_sync(), the ID number of the source thread 
+* will not wrap around the value of width and so the upper delta threads 
+* will remain unchanged. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_down_sync(const unsigned mask, const __half var, const unsigned int delta, const int width = warpSize);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Exchange a variable between threads within a warp. Copy from a thread based on bitwise XOR of own thread ID. 
+* 
+* \details Calculates a source thread ID by performing a bitwise XOR of the caller's thread ID with mask: 
+* the value of var held by the resulting thread ID is returned. If width is less than warpSize then each 
+* group of width consecutive threads are able to access elements from earlier groups of threads, 
+* however if they attempt to access elements from later groups of threads their own value of var 
+* will be returned. This mode implements a butterfly addressing pattern such as is used in tree 
+* reduction and broadcast. 
+* \param[in] mask - unsigned int. Is only being read. 
+* \param[in] var - half. Is only being read. 
+* \param[in] delta - int. Is only being read. 
+* \param[in] width - int. Is only being read. 
+* 
+* \returns Returns the 2-byte word referenced by var from the source thread ID as half. 
+* If the source thread ID is out of range or the source thread has exited, the calling thread's own var is returned. 
+* \note_ref_guide_warp_shuffle
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior not reentrant, not thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(const unsigned mask, const __half var, const int delta, const int width = warpSize);
+
+#if defined(__local_warpSize)
+#undef warpSize
+#undef __local_warpSize
+#endif
+#endif /*!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 300) */
+
+#if defined(__cplusplus) && ( !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320) )
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.nc` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.nc` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldg(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cg` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cg` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcg(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.ca` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.ca` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldca(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cs` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cs` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcs(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.lu` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldlu(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.lu` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldlu(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cv` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half2 __ldcv(const  __half2 *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `ld.global.cv` load instruction.
+* \param[in] ptr - memory location
+* \returns The value pointed by `ptr`
+*/
+__CUDA_FP16_DECL__ __half __ldcv(const __half *const ptr);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wb` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwb(__half2 *const ptr, const __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wb` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwb(__half *const ptr, const __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cg` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcg(__half2 *const ptr, const __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cg` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcg(__half *const ptr, const __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cs` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcs(__half2 *const ptr, const __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.cs` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stcs(__half *const ptr, const __half value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wt` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwt(__half2 *const ptr, const __half2 value);
+/**
+* \ingroup CUDA_MATH__HALF_MISC
+* \brief Generates a `st.global.wt` store instruction.
+* \param[out] ptr - memory location
+* \param[in] value - the value to be stored
+*/
+__CUDA_FP16_DECL__ void __stwt(__half *const ptr, const __half value);
+#endif /*defined(__cplusplus) && ( !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320) )*/
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs half2 vector if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector result of if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison.
+* 
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector result of not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The \p half2 result of less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The vector result of greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The half2 vector result of less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison.
+* 
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector result of greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns half2
+* - The vector result of unordered if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The vector result of unordered not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison.
+*
+* Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The vector result of unordered less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The \p half2 vector result of unordered greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The vector result of unordered less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p half results are set to 1.0 for true, or 0.0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The \p half2 vector result of unordered greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs half2 vector if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns unsigned int
+* - The vector mask result of if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __heq2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison.
+* 
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns unsigned int
+* - The vector mask result of not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hne2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns unsigned int
+* - The vector mask result of less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hle2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns unsigned int
+* - The vector mask result of greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hge2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns unsigned int
+* - The vector mask result of less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hlt2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison.
+* 
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns unsigned int
+* - The vector mask result of greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hgt2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison.
+* 
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns unsigned int
+* - The vector mask result of unordered if-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hequ2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns unsigned int
+* - The vector mask result of unordered not-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hneu2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison.
+*
+* Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns unsigned int
+* - The vector mask result of unordered less-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hleu2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns unsigned int
+* - The vector mask result of unordered greater-equal comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hgeu2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns unsigned int
+* - The vector mask result of unordered less-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hltu2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The corresponding \p unsigned bits are set to 0xFFFF for true, or 0x0 for false.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns unsigned int
+* - The vector mask result of unordered greater-than comparison of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ unsigned __hgtu2_mask(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Determine whether \p half2 argument is a NaN.
+*
+* \details Determine whether each half of input \p half2 number \p a is a NaN.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The half2 with the corresponding \p half results set to
+* 1.0 for NaN, 0.0 otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-95
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The sum of vectors \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-104
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The subtraction of vector \p b from \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-102
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The result of elementwise multiplying the vectors \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode. Prevents floating-point contractions of mul+add into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-95
+* \endinternal
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The sum of vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2_rn(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode. Prevents floating-point contractions of mul+sub
+* into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-104
+* \endinternal
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The subtraction of vector \p b from \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2_rn(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode. Prevents floating-point contractions of
+* mul+add or sub into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-102
+* \endinternal
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise multiplying the vectors \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2_rn(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector division in round-to-nearest-even mode.
+*
+* \details Divides \p half2 input vector \p a by input vector \p b in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-103
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise division of \p a with \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __h2div(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+*
+* \details Calculates the absolute value of both halves of the input \p half2 number and
+* returns the result.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - Returns \p a with the absolute value of both halves. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector add of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the results to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The sum of \p a and \p b, with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector subtraction in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half2 input vector \p b from input vector \p a in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The subtraction of vector \p b from \p a, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector multiplication in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiplication of inputs \p a and \p b, in
+* round-to-nearest-even mode, and clamps the results to range [0.0, 1.0]. NaN
+* results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns half2
+* - The result of elementwise multiplication of vectors \p a and \p b, 
+* with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-105
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* - The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode, with saturation to [0.0, 1.0].
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the
+* results to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* \param[in] c - half2. Is only being read. 
+*
+* \returns half2
+* - The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c, 
+* with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Negates both halves of the input \p half2 number and returns the
+* result.
+*
+* \details Negates both halves of the input \p half2 number \p a and returns the result.
+* \internal
+* \req DEEPLEARN-SRM_REQ-101
+* \endinternal
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - Returns \p a with both halves negated. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Calculates the absolute value of input \p half number and returns the result.
+*
+* \details Calculates the absolute value of input \p half number and returns the result.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The absolute value of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __habs(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode.
+*
+* \details Performs \p half addition of inputs \p a and \p b, in round-to-nearest-even
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-94
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The sum of \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-97
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The result of subtracting \p b from \p a. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-99
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The result of multiplying \p a and \p b. 
+*/
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode.
+*
+* \details Performs \p half addition of inputs \p a and \p b, in round-to-nearest-even
+* mode. Prevents floating-point contractions of mul+add into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-94
+* \endinternal
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* - The sum of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd_rn(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode.
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode. Prevents floating-point contractions of mul+sub into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-97
+* \endinternal
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* - The result of subtracting \p b from \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub_rn(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode.
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode. Prevents floating-point contractions of mul+add or sub into fma.
+* \internal
+* \req DEEPLEARN-SRM_REQ-99
+* \endinternal
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* - The result of multiplying \p a and \p b.
+*/
+__CUDA_FP16_DECL__ __half __hmul_rn(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half division in round-to-nearest-even mode.
+* 
+* \details Divides \p half input \p a by input \p b in round-to-nearest
+* mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-98
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* 
+* \returns half
+* - The result of dividing \p a by \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__  __half __hdiv(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half addition in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half add of inputs \p a and \p b, in round-to-nearest-even mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The sum of \p a and \p b, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half subtraction in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Subtracts \p half input \p b from input \p a in round-to-nearest
+* mode,
+* and clamps the result to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The result of subtraction of \p b from \p a, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half multiplication in round-to-nearest-even mode, with
+* saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiplication of inputs \p a and \p b, in round-to-nearest
+* mode, and clamps the result to range [0.0, 1.0]. NaN results are flushed to
+* +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns half
+* - The result of multiplying \p a and \p b, with respect to saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* \internal
+* \req DEEPLEARN-SRM_REQ-96
+* \endinternal
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* - The result of fused multiply-add operation on \p
+* a, \p b, and \p c. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode,
+* with saturation to [0.0, 1.0].
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode, and clamps the result
+* to range [0.0, 1.0]. NaN results are flushed to +0.0.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+* \param[in] c - half. Is only being read. 
+*
+* \returns half
+* - The result of fused multiply-add operation on \p
+* a, \p b, and \p c, with respect to saturation. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Negates input \p half number and returns the result.
+*
+* \details Negates input \p half number and returns the result.
+* \internal
+* \req DEEPLEARN-SRM_REQ-100
+* \endinternal
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - minus a
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hneg(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector if-equal comparison and returns boolean true
+* iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of if-equal comparison
+* of vectors \p a and \p b are true;
+* - false otherwise.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector not-equal comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of not-equal comparison
+* of vectors \p a and \p b are true, 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-equal comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of less-equal comparison
+* of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-equal comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of greater-equal
+* comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector less-than comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of less-than comparison
+* of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector greater-than comparison and returns boolean
+* true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate false results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+* 
+* \returns bool 
+* - true if both \p half results of greater-than
+* comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered if-equal comparison and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector if-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half if-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered if-equal
+* comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered not-equal comparison and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector not-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half not-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered not-equal
+* comparison of vectors \p a and \p b are true;
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-equal comparison and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered less-equal
+* comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-equal comparison and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-equal comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-equal comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered
+* greater-equal comparison of vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered less-than comparison and returns
+* boolean true iff both \p half results are true, boolean false otherwise.
+*
+* \details Performs \p half2 vector less-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half less-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered less-than comparison of 
+* vectors \p a and \p b are true; 
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Performs \p half2 vector unordered greater-than comparison and
+* returns boolean true iff both \p half results are true, boolean false
+* otherwise.
+*
+* \details Performs \p half2 vector greater-than comparison of inputs \p a and \p b.
+* The bool result is set to true only if both \p half greater-than comparisons
+* evaluate to true, or false otherwise.
+* NaN inputs generate true results.
+* \param[in] a - half2. Is only being read. 
+* \param[in] b - half2. Is only being read. 
+*
+* \returns bool
+* - true if both \p half results of unordered
+* greater-than comparison of vectors \p a and \p b are true;
+* - false otherwise. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of if-equal comparison of \p a and \p b. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of not-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of less-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of greater-equal comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of less-than comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate false results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of greater-than comparison of \p a and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered if-equal comparison.
+*
+* \details Performs \p half if-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered if-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered not-equal comparison.
+*
+* \details Performs \p half not-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered not-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-equal comparison.
+*
+* \details Performs \p half less-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered less-equal comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-equal comparison.
+*
+* \details Performs \p half greater-equal comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered greater-equal comparison of \p a
+* and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered less-than comparison.
+*
+* \details Performs \p half less-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered less-than comparison of \p a and
+* \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Performs \p half unordered greater-than comparison.
+*
+* \details Performs \p half greater-than comparison of inputs \p a and \p b.
+* NaN inputs generate true results.
+* \param[in] a - half. Is only being read. 
+* \param[in] b - half. Is only being read. 
+*
+* \returns bool
+* - The boolean result of unordered greater-than comparison of \p a
+* and \p b.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Determine whether \p half argument is a NaN.
+*
+* \details Determine whether \p half value \p a is a NaN.
+* \param[in] a - half. Is only being read. 
+*
+* \returns bool
+* - true iff argument is NaN. 
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ bool __hisnan(const __half a);
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half maximum of two input values, NaNs pass through.
+*
+* \details Calculates \p half max(\p a, \p b)
+* defined as (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmax_nan(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_COMPARISON
+* \brief Calculates \p half minimum of two input values, NaNs pass through.
+*
+* \details Calculates \p half min(\p a, \p b)
+* defined as (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+*
+* \returns half
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hmin_nan(const __half a, const __half b);
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Performs \p half fused multiply-add in round-to-nearest-even mode with relu saturation.
+*
+* \details Performs \p half multiply on inputs \p a and \p b,
+* then performs a \p half add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* Then negative result is clamped to 0.
+* NaN result is converted to canonical NaN.
+* \param[in] a - half. Is only being read.
+* \param[in] b - half. Is only being read.
+* \param[in] c - half. Is only being read.
+*
+* \returns half
+* - The result of fused multiply-add operation on \p
+* a, \p b, and \p c with relu saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half __hfma_relu(const __half a, const __half b, const __half c);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector maximum of two inputs, NaNs pass through.
+*
+* \details Calculates \p half2 vector max(\p a, \p b).
+* Elementwise \p half operation is defined as
+* (\p a > \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise maximum of vectors \p a  and \p b, with NaNs pass through
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmax2_nan(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_COMPARISON
+* \brief Calculates \p half2 vector minimum of two inputs, NaNs pass through.
+*
+* \details Calculates \p half2 vector min(\p a, \p b).
+* Elementwise \p half operation is defined as
+* (\p a < \p b) ? \p a : \p b.
+* - If either of inputs is NaN, then canonical NaN is returned.
+* - If values of both inputs are 0.0, then +0.0 > -0.0
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise minimum of vectors \p a  and \p b, with NaNs pass through
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hmin2_nan(const __half2 a, const __half2 b);
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs \p half2 vector fused multiply-add in round-to-nearest-even
+* mode with relu saturation.
+*
+* \details Performs \p half2 vector multiply on inputs \p a and \p b,
+* then performs a \p half2 vector add of the result with \p c,
+* rounding the result once in round-to-nearest-even mode.
+* Then negative result is clamped to 0.
+* NaN result is converted to canonical NaN.
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+* \param[in] c - half2. Is only being read.
+*
+* \returns half2
+* - The result of elementwise fused multiply-add operation on vectors \p a, \p b, and \p c with relu saturation.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hfma2_relu(const __half2 a, const __half2 b, const __half2 c);
+#endif /* !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800) */
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Performs fast complex multiply-accumulate
+*
+* \details Interprets vector \p half2 input pairs \p a, \p b, and \p c as
+* complex numbers in \p half precision and performs
+* complex multiply-accumulate operation: a*b + c
+* \param[in] a - half2. Is only being read.
+* \param[in] b - half2. Is only being read.
+* \param[in] c - half2. Is only being read.
+*
+* \returns half2
+* - The result of complex multiply-accumulate operation on complex numbers \p a, \p b, and \p c
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 __hcmadd(const __half2 a, const __half2 b, const __half2 c);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half square root of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The square root of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal square root in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half reciprocal square root of input \p a in round-to-nearest
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The reciprocal square root of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half reciprocal of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The reciprocal of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hrcp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half natural logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The natural logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half binary logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The binary logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal logarithm in round-to-nearest-even mode.
+*
+* \details Calculates \p half decimal logarithm of input \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The decimal logarithm of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hlog10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half natural exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half natural exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The natural exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half binary exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half binary exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The binary exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp2(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half decimal exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half decimal exponential function of input \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The decimal exponential function on \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hexp10(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half cosine in round-to-nearest-even mode.
+*
+* \details Calculates \p half cosine of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The cosine of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hcos(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF_FUNCTIONS
+* \brief Calculates \p half sine in round-to-nearest-even mode.
+*
+* \details Calculates \p half sine of input \p a in round-to-nearest-even mode.
+* \param[in] a - half. Is only being read. 
+*
+* \returns half
+* - The sine of \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half hsin(const __half a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector square root in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 square root of input vector \p a in round-to-nearest
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise square root on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal square root in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 reciprocal square root of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise reciprocal square root on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector reciprocal in round-to-nearest-even mode.
+*
+* \details Calculates \p half2 reciprocal of input vector \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise reciprocal on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector natural logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 natural logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise natural logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 binary logarithm of input vector \p a in round-to-nearest
+* mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise binary logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal logarithm in round-to-nearest-even
+* mode.
+*
+* \details Calculates \p half2 decimal logarithm of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise decimal logarithm on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector exponential function in round-to-nearest
+* mode.
+*
+* \details Calculates \p half2 exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector binary exponential function in
+* round-to-nearest-even mode.
+*
+* \details Calculates \p half2 binary exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+*
+* \returns half2
+* - The elementwise binary exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector decimal exponential function in
+* round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 decimal exponential function of input vector \p a in
+* round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The elementwise decimal exponential function on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector cosine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 cosine of input vector \p a in round-to-nearest-even
+* mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The elementwise cosine on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a);
+/**
+* \ingroup CUDA_MATH__HALF2_FUNCTIONS
+* \brief Calculates \p half2 vector sine in round-to-nearest-even mode.
+* 
+* \details Calculates \p half2 sine of input vector \p a in round-to-nearest-even mode.
+* \param[in] a - half2. Is only being read. 
+* 
+* \returns half2
+* - The elementwise sine on vector \p a.
+* \internal
+* \exception-guarantee no-throw guarantee
+* \behavior reentrant, thread safe
+* \endinternal
+*/
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a);
+
+#endif /*if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)*/
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 600)
+
+/**
+* \ingroup CUDA_MATH__HALF2_ARITHMETIC
+* \brief Vector add \p val to the value stored at \p address in global or shared memory, and writes this
+* value back to \p address. The atomicity of the add operation is guaranteed separately for each of the
+* two __half elements; the entire __half2 is not guaranteed to be atomic as a single 32-bit access.
+* 
+* \details The location of \p address must be in global or shared memory. This operation has undefined
+* behavior otherwise. This operation is only supported by devices of compute capability 6.x and higher.
+* 
+* \param[in] address - half2*. An address in global or shared memory.
+* \param[in] val - half2. The value to be added.
+* 
+* \returns half2
+* - The old value read from \p address.
+*
+* \note_ref_guide_atomic
+*/
+__CUDA_FP16_DECL__ __half2 atomicAdd(__half2 *const address, const __half2 val);
+
+#endif /*if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 600)*/
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 700)
+
+/**
+* \ingroup CUDA_MATH__HALF_ARITHMETIC
+* \brief Adds \p val to the value stored at \p address in global or shared memory, and writes this value
+* back to \p address. This operation is performed in one atomic operation.
+* 
+* \details The location of \p address must be in global or shared memory. This operation has undefined
+* behavior otherwise. This operation is only supported by devices of compute capability 7.x and higher.
+* 
+* \param[in] address - half*. An address in global or shared memory.
+* \param[in] val - half. The value to be added.
+* 
+* \returns half
+* - The old value read from \p address.
+* 
+* \note_ref_guide_atomic
+*/
+__CUDA_FP16_DECL__ __half atomicAdd(__half *const address, const __half val);
+
+#endif /*if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 700)*/
+
+#endif /* defined(__CUDACC__) */
+
+#undef __CUDA_FP16_DECL__
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+
+#endif /* defined(__cplusplus) */
+
+/* Note the .hpp file is included even for host-side compilation, to capture the "half" & "half2" definitions */
+#include "cuda_fp16.hpp"
+#undef ___CUDA_FP16_STRINGIFY_INNERMOST
+#undef __CUDA_FP16_STRINGIFY
+
+#endif /* end of include guard: __CUDA_FP16_H__ */
diff --git a/cupy/_core/include/cupy/_cuda/cuda-12/cuda_fp16.hpp b/cupy/_core/include/cupy/_cuda/cuda-12/cuda_fp16.hpp
new file mode 100644
index 0000000..22fb0c0
--- /dev/null
+++ b/cupy/_core/include/cupy/_cuda/cuda-12/cuda_fp16.hpp
@@ -0,0 +1,2738 @@
+/*
+* Copyright 1993-2022 NVIDIA Corporation.  All rights reserved.
+*
+* NOTICE TO LICENSEE:
+*
+* This source code and/or documentation ("Licensed Deliverables") are
+* subject to NVIDIA intellectual property rights under U.S. and
+* international Copyright laws.
+*
+* These Licensed Deliverables contained herein is PROPRIETARY and
+* CONFIDENTIAL to NVIDIA and is being provided under the terms and
+* conditions of a form of NVIDIA software license agreement by and
+* between NVIDIA and Licensee ("License Agreement") or electronically
+* accepted by Licensee.  Notwithstanding any terms or conditions to
+* the contrary in the License Agreement, reproduction or disclosure
+* of the Licensed Deliverables to any third party without the express
+* written consent of NVIDIA is prohibited.
+*
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
+* SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
+* PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
+* NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
+* DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
+* NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
+* NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
+* LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
+* SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
+* DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+* WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+* OF THESE LICENSED DELIVERABLES.
+*
+* U.S. Government End Users.  These Licensed Deliverables are a
+* "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
+* 1995), consisting of "commercial computer software" and "commercial
+* computer software documentation" as such terms are used in 48
+* C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
+* only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
+* 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
+* U.S. Government End Users acquire the Licensed Deliverables with
+* only those rights set forth herein.
+*
+* Any use of the Licensed Deliverables in individual and commercial
+* software must include, in the user documentation and internal
+* comments to the code, the above Disclaimer and U.S. Government End
+* Users Notice.
+*/
+
+#if !defined(__CUDA_FP16_HPP__)
+#define __CUDA_FP16_HPP__
+
+#if !defined(__CUDA_FP16_H__)
+#error "Do not include this file directly. Instead, include cuda_fp16.h."
+#endif
+
+#if !defined(_MSC_VER) && __cplusplus >= 201103L
+#   define __CPP_VERSION_AT_LEAST_11_FP16
+#elif _MSC_FULL_VER >= 190024210 && _MSVC_LANG >= 201103L
+#   define __CPP_VERSION_AT_LEAST_11_FP16
+#endif
+
+// implicitly provided by NVRTC
+#if !defined(__CUDACC_RTC__)
+#include <nv/target>
+#endif  /* !defined(__CUDACC_RTC__) */
+
+
+#if !defined(IF_DEVICE_OR_CUDACC)
+#if defined(__CUDACC__)
+    #define IF_DEVICE_OR_CUDACC(d, c, f) NV_IF_ELSE_TARGET(NV_IS_DEVICE, d, c)
+#else
+    #define IF_DEVICE_OR_CUDACC(d, c, f) NV_IF_ELSE_TARGET(NV_IS_DEVICE, d, f)
+#endif
+#endif
+/* C++11 header for std::move. 
+ * In RTC mode, std::move is provided implicitly; don't include the header
+ */
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16) && !defined(__CUDACC_RTC__)
+#include <utility>
+#endif /* __cplusplus >= 201103L && !defined(__CUDACC_RTC__) */
+
+/* C++ header for std::memcpy (used for type punning in host-side implementations).
+ * When compiling as a CUDA source file memcpy is provided implicitly.
+ * !defined(__CUDACC__) implies !defined(__CUDACC_RTC__).
+ */
+#if defined(__cplusplus) && !defined(__CUDACC__)
+#include <cstring>
+#endif /* defined(__cplusplus) && !defined(__CUDACC__) */
+
+
+/* Set up function decorations */
+#if defined(__CUDACC__) || defined(_NVHPC_CUDA)
+#define __CUDA_FP16_DECL__ static __device__ __inline__
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __host__ __device__ __inline__
+#define __VECTOR_FUNCTIONS_DECL__ static __inline__ __host__ __device__
+#define __CUDA_HOSTDEVICE__ __host__ __device__
+#else /* !defined(__CUDACC__) */
+#if defined(__GNUC__)
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static __attribute__ ((unused))
+#else
+#define __CUDA_HOSTDEVICE_FP16_DECL__ static
+#endif /* defined(__GNUC__) */
+#define __CUDA_HOSTDEVICE__
+#endif /* defined(__CUDACC_) */
+
+/* Set up structure-alignment attribute */
+#if defined(__CUDACC__)
+#define __CUDA_ALIGN__(align) __align__(align)
+#else
+/* Define alignment macro based on compiler type (cannot assume C11 "_Alignas" is available) */
+#if __cplusplus >= 201103L
+#define __CUDA_ALIGN__(n) alignas(n)    /* C++11 kindly gives us a keyword for this */
+#else /* !defined(__CPP_VERSION_AT_LEAST_11_FP16)*/
+#if defined(__GNUC__)
+#define __CUDA_ALIGN__(n) __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+#define __CUDA_ALIGN__(n) __declspec(align(n))
+#else
+#define __CUDA_ALIGN__(n)
+#endif /* defined(__GNUC__) */
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+#endif /* defined(__CUDACC__) */
+
+/* Macros to allow half & half2 to be used by inline assembly */
+#define __HALF_TO_US(var) *(reinterpret_cast<unsigned short *>(&(var)))
+#define __HALF_TO_CUS(var) *(reinterpret_cast<const unsigned short *>(&(var)))
+#define __HALF2_TO_UI(var) *(reinterpret_cast<unsigned int *>(&(var)))
+#define __HALF2_TO_CUI(var) *(reinterpret_cast<const unsigned int *>(&(var)))
+
+/* Macros for half & half2 binary arithmetic */
+#define __BINARY_OP_HALF_MACRO(name) /* do */ {\
+   __half val; \
+   asm( "{" __CUDA_FP16_STRINGIFY(name) ".f16 %0,%1,%2;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b))); \
+   return val; \
+} /* while(0) */
+#define __BINARY_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{" __CUDA_FP16_STRINGIFY(name) ".f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+#define __TERNARY_OP_HALF_MACRO(name) /* do */ {\
+   __half val; \
+   asm( "{" __CUDA_FP16_STRINGIFY(name) ".f16 %0,%1,%2,%3;\n}" \
+        :"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)),"h"(__HALF_TO_CUS(b)),"h"(__HALF_TO_CUS(c))); \
+   return val; \
+} /* while(0) */
+#define __TERNARY_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{" __CUDA_FP16_STRINGIFY(name) ".f16x2 %0,%1,%2,%3;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b)),"r"(__HALF2_TO_CUI(c))); \
+   return val; \
+} /* while(0) */
+
+/**
+* Types which allow static initialization of "half" and "half2" until
+* these become an actual builtin. Note this initialization is as a
+* bitfield representation of "half", and not a conversion from short->half.
+* Such a representation will be deprecated in a future version of CUDA. 
+* (Note these are visible to non-nvcc compilers, including C-only compilation)
+*/
+typedef struct __CUDA_ALIGN__(2) {
+    unsigned short x;
+} __half_raw;
+
+typedef struct __CUDA_ALIGN__(4) {
+    unsigned short x;
+    unsigned short y;
+} __half2_raw;
+
+/* All other definitions in this file are only visible to C++ compilers */
+#if defined(__cplusplus)
+
+/* Hide GCC member initialization list warnings because of host/device in-function init requirement */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstrict-aliasing"
+#pragma GCC diagnostic ignored "-Weffc++"
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+/* class' : multiple assignment operators specified
+   The class has multiple assignment operators of a single type. This warning is informational */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( push )
+#pragma warning( disable:4522 )
+#endif /* defined(__GNUC__) */
+
+struct __CUDA_ALIGN__(2) __half {
+protected:
+    unsigned short __x;
+
+public:
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+    __half() = default;
+#else
+    __CUDA_HOSTDEVICE__ __half() { }
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+
+    /* Convert to/from __half_raw */
+    __CUDA_HOSTDEVICE__ __half(const __half_raw &hr) : __x(hr.x) { }
+    __CUDA_HOSTDEVICE__ __half &operator=(const __half_raw &hr) { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ volatile __half &operator=(const volatile __half_raw &hr) volatile { __x = hr.x; return *this; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const { __half_raw ret; ret.x = __x; return ret; }
+    __CUDA_HOSTDEVICE__ operator __half_raw() const volatile { __half_raw ret; ret.x = __x; return ret; }
+
+#if !defined(__CUDA_NO_HALF_CONVERSIONS__)
+
+    /* Construct from float/double */
+    __CUDA_HOSTDEVICE__ __half(const float f) { __x = __float2half(f).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const double f) { __x = __double2half(f).__x;  }
+
+    __CUDA_HOSTDEVICE__ operator float() const { return __half2float(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const float f) { __x = __float2half(f).__x; return *this; }
+
+    /* We omit "cast to double" operator, so as to not be ambiguous about up-cast */
+    __CUDA_HOSTDEVICE__ __half &operator=(const double f) { __x = __double2half(f).__x; return *this; }
+
+/* Member functions only available to nvcc compilation so far */
+#if defined(__CUDACC__)
+    /* Allow automatic construction from types supported natively in hardware */
+    /* Note we do avoid constructor init-list because of special host/device compilation rules */
+    __CUDA_HOSTDEVICE__ __half(const short val) { __x = __short2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const unsigned short val) { __x = __ushort2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const int val) { __x = __int2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const unsigned int val) { __x = __uint2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const long long val) { __x = __ll2half_rn(val).__x;  }
+    __CUDA_HOSTDEVICE__ __half(const unsigned long long val) { __x = __ull2half_rn(val).__x; }
+
+    /* Allow automatic casts to supported builtin types, matching all that are permitted with float */
+    __CUDA_HOSTDEVICE__ operator short() const { return __half2short_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const short val) { __x = __short2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned short() const { return __half2ushort_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const unsigned short val) { __x = __ushort2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator int() const { return __half2int_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const int val) { __x = __int2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned int() const { return __half2uint_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const unsigned int val) { __x = __uint2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator long long() const { return __half2ll_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const long long val) { __x = __ll2half_rn(val).__x; return *this; }
+
+    __CUDA_HOSTDEVICE__ operator unsigned long long() const { return __half2ull_rz(*this); }
+    __CUDA_HOSTDEVICE__ __half &operator=(const unsigned long long val) { __x = __ull2half_rn(val).__x; return *this; }
+
+    /* Boolean conversion - note both 0 and -0 must return false */
+    __CUDA_HOSTDEVICE__ operator bool() const { return (__x & 0x7FFFU) != 0U; }
+#endif /* defined(__CUDACC__) */
+#endif /* !defined(__CUDA_NO_HALF_CONVERSIONS__) */
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16 operations only supported on arch >= 5.3 */
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) || defined(_NVHPC_CUDA)
+#if !defined(__CUDA_NO_HALF_OPERATORS__)
+/* Some basic arithmetic operations expected of a builtin */
+__device__ __forceinline__ __half operator+(const __half &lh, const __half &rh) { return __hadd(lh, rh); }
+__device__ __forceinline__ __half operator-(const __half &lh, const __half &rh) { return __hsub(lh, rh); }
+__device__ __forceinline__ __half operator*(const __half &lh, const __half &rh) { return __hmul(lh, rh); }
+__device__ __forceinline__ __half operator/(const __half &lh, const __half &rh) { return __hdiv(lh, rh); }
+
+__device__ __forceinline__ __half &operator+=(__half &lh, const __half &rh) { lh = __hadd(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator-=(__half &lh, const __half &rh) { lh = __hsub(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator*=(__half &lh, const __half &rh) { lh = __hmul(lh, rh); return lh; }
+__device__ __forceinline__ __half &operator/=(__half &lh, const __half &rh) { lh = __hdiv(lh, rh); return lh; }
+
+/* Note for increment and decrement we use the raw value 0x3C00U equating to half(1.0F), to avoid the extra conversion */
+__device__ __forceinline__ __half &operator++(__half &h)      { __half_raw one; one.x = 0x3C00U; h += one; return h; }
+__device__ __forceinline__ __half &operator--(__half &h)      { __half_raw one; one.x = 0x3C00U; h -= one; return h; }
+__device__ __forceinline__ __half  operator++(__half &h, const int ignored)
+{
+    // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators.
+    static_cast<void>(ignored);
+
+    const __half ret = h;
+    __half_raw one;
+    one.x = 0x3C00U;
+    h += one;
+    return ret;
+}
+__device__ __forceinline__ __half  operator--(__half &h, const int ignored)
+{
+    // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators.
+    static_cast<void>(ignored);
+
+    const __half ret = h;
+    __half_raw one;
+    one.x = 0x3C00U;
+    h -= one;
+    return ret;
+}
+
+/* Unary plus and inverse operators */
+__device__ __forceinline__ __half operator+(const __half &h) { return h; }
+__device__ __forceinline__ __half operator-(const __half &h) { return __hneg(h); }
+
+/* Some basic comparison operations to make it look like a builtin */
+__device__ __forceinline__ bool operator==(const __half &lh, const __half &rh) { return __heq(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half &lh, const __half &rh) { return __hneu(lh, rh); }
+__device__ __forceinline__ bool operator> (const __half &lh, const __half &rh) { return __hgt(lh, rh); }
+__device__ __forceinline__ bool operator< (const __half &lh, const __half &rh) { return __hlt(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half &lh, const __half &rh) { return __hge(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half &lh, const __half &rh) { return __hle(lh, rh); }
+#endif /* !defined(__CUDA_NO_HALF_OPERATORS__) */
+#endif /* !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) || defined(_NVHPC_CUDA) */
+#endif /* defined(__CUDACC__) */
+
+/* __half2 is visible to non-nvcc host compilers */
+struct __CUDA_ALIGN__(4) __half2 {
+    __half x;
+    __half y;
+
+    // All construct/copy/assign/move
+public:
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+    __half2() = default;
+    __CUDA_HOSTDEVICE__ __half2(const __half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2 &&src) { __HALF2_TO_UI(*this) = std::move(__HALF2_TO_CUI(src)); return *this; }
+#else
+    __CUDA_HOSTDEVICE__ __half2() { }
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+    __CUDA_HOSTDEVICE__ __half2(const __half &a, const __half &b) : x(a), y(b) { }
+    __CUDA_HOSTDEVICE__ __half2(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2 &src) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(src); return *this; }
+
+    /* Convert to/from __half2_raw */
+    __CUDA_HOSTDEVICE__ __half2(const __half2_raw &h2r ) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); }
+    __CUDA_HOSTDEVICE__ __half2 &operator=(const __half2_raw &h2r) { __HALF2_TO_UI(*this) = __HALF2_TO_CUI(h2r); return *this; }
+    __CUDA_HOSTDEVICE__ operator __half2_raw() const { __half2_raw ret; ret.x = 0U; ret.y = 0U; __HALF2_TO_UI(ret) = __HALF2_TO_CUI(*this); return ret; }
+};
+
+/* Global-space operator functions are only available to nvcc compilation */
+#if defined(__CUDACC__)
+
+/* Arithmetic FP16x2 operations only supported on arch >= 5.3 */
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) || defined(_NVHPC_CUDA)) && !defined(__CUDA_NO_HALF2_OPERATORS__)
+
+__device__ __forceinline__ __half2 operator+(const __half2 &lh, const __half2 &rh) { return __hadd2(lh, rh); }
+__device__ __forceinline__ __half2 operator-(const __half2 &lh, const __half2 &rh) { return __hsub2(lh, rh); }
+__device__ __forceinline__ __half2 operator*(const __half2 &lh, const __half2 &rh) { return __hmul2(lh, rh); }
+__device__ __forceinline__ __half2 operator/(const __half2 &lh, const __half2 &rh) { return __h2div(lh, rh); }
+
+__device__ __forceinline__ __half2& operator+=(__half2 &lh, const __half2 &rh) { lh = __hadd2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator-=(__half2 &lh, const __half2 &rh) { lh = __hsub2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator*=(__half2 &lh, const __half2 &rh) { lh = __hmul2(lh, rh); return lh; }
+__device__ __forceinline__ __half2& operator/=(__half2 &lh, const __half2 &rh) { lh = __h2div(lh, rh); return lh; }
+
+__device__ __forceinline__ __half2 &operator++(__half2 &h)      { __half2_raw one; one.x = 0x3C00U; one.y = 0x3C00U; h = __hadd2(h, one); return h; }
+__device__ __forceinline__ __half2 &operator--(__half2 &h)      { __half2_raw one; one.x = 0x3C00U; one.y = 0x3C00U; h = __hsub2(h, one); return h; }
+__device__ __forceinline__ __half2  operator++(__half2 &h, const int ignored)
+{
+    // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators.
+    static_cast<void>(ignored);
+
+    const __half2 ret = h;
+    __half2_raw one;
+    one.x = 0x3C00U;
+    one.y = 0x3C00U;
+    h = __hadd2(h, one);
+    return ret;
+}
+__device__ __forceinline__ __half2  operator--(__half2 &h, const int ignored)
+{
+    // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators.
+    static_cast<void>(ignored);
+
+    const __half2 ret = h;
+    __half2_raw one;
+    one.x = 0x3C00U;
+    one.y = 0x3C00U;
+    h = __hsub2(h, one);
+    return ret;
+}
+
+__device__ __forceinline__ __half2 operator+(const __half2 &h) { return h; }
+__device__ __forceinline__ __half2 operator-(const __half2 &h) { return __hneg2(h); }
+
+__device__ __forceinline__ bool operator==(const __half2 &lh, const __half2 &rh) { return __hbeq2(lh, rh); }
+__device__ __forceinline__ bool operator!=(const __half2 &lh, const __half2 &rh) { return __hbneu2(lh, rh); }
+__device__ __forceinline__ bool operator>(const __half2 &lh, const __half2 &rh) { return __hbgt2(lh, rh); }
+__device__ __forceinline__ bool operator<(const __half2 &lh, const __half2 &rh) { return __hblt2(lh, rh); }
+__device__ __forceinline__ bool operator>=(const __half2 &lh, const __half2 &rh) { return __hbge2(lh, rh); }
+__device__ __forceinline__ bool operator<=(const __half2 &lh, const __half2 &rh) { return __hble2(lh, rh); }
+
+#endif /* (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) || defined(_NVHPC_CUDA)) && !defined(__CUDA_NO_HALF2_OPERATORS__) */
+#endif /* defined(__CUDACC__) */
+
+/* Restore warning for multiple assignment operators */
+#if defined(_MSC_VER) && _MSC_VER >= 1500
+#pragma warning( pop )
+#endif /* defined(_MSC_VER) && _MSC_VER >= 1500 */
+
+/* Restore -Weffc++ warnings from here on */
+#if defined(__GNUC__)
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)
+#pragma GCC diagnostic pop
+#endif /* __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) */
+#endif /* defined(__GNUC__) */
+
+#undef __CUDA_HOSTDEVICE__
+#undef __CUDA_ALIGN__
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static inline unsigned short __internal_float2half(const float f, unsigned int &sign, unsigned int &remainder)
+{
+    unsigned int x;
+    unsigned int u;
+    unsigned int result;
+#if defined(__CUDACC__)
+    (void)memcpy(&x, &f, sizeof(f));
+#else
+    (void)std::memcpy(&x, &f, sizeof(f));
+#endif
+    u = (x & 0x7fffffffU);
+    sign = ((x >> 16U) & 0x8000U);
+    // NaN/+Inf/-Inf
+    if (u >= 0x7f800000U) {
+        remainder = 0U;
+        result = ((u == 0x7f800000U) ? (sign | 0x7c00U) : 0x7fffU);
+    } else if (u > 0x477fefffU) { // Overflows
+        remainder = 0x80000000U;
+        result = (sign | 0x7bffU);
+    } else if (u >= 0x38800000U) { // Normal numbers
+        remainder = u << 19U;
+        u -= 0x38000000U;
+        result = (sign | (u >> 13U));
+    } else if (u < 0x33000001U) { // +0/-0
+        remainder = u;
+        result = sign;
+    } else { // Denormal numbers
+        const unsigned int exponent = u >> 23U;
+        const unsigned int shift = 0x7eU - exponent;
+        unsigned int mantissa = (u & 0x7fffffU);
+        mantissa |= 0x800000U;
+        remainder = mantissa << (32U - shift);
+        result = (sign | (mantissa >> shift));
+        result &= 0x0000FFFFU;
+    }
+    return static_cast<unsigned short>(result);
+}
+#endif  /* #if !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __double2half(const double a)
+{
+IF_DEVICE_OR_CUDACC(
+    __half val;
+    asm("{  cvt.rn.f16.f64 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "d"(a));
+    return val;
+,
+    __half result;
+    // Perform rounding to 11 bits of precision, convert value
+    // to float and call existing float to half conversion.
+    // By pre-rounding to 11 bits we avoid additional rounding
+    // in float to half conversion.
+    unsigned long long int absa;
+    unsigned long long int ua;
+    (void)memcpy(&ua, &a, sizeof(a));
+    absa = (ua & 0x7fffffffffffffffULL);
+    if ((absa >= 0x40f0000000000000ULL) || (absa <= 0x3e60000000000000ULL))
+    {
+        // |a| >= 2^16 or NaN or |a| <= 2^(-25)
+        // double-rounding is not a problem
+        result = __float2half(static_cast<float>(a));
+    }
+    else
+    {
+        // here 2^(-25) < |a| < 2^16
+        // prepare shifter value such that a + shifter
+        // done in double precision performs round-to-nearest-even
+        // and (a + shifter) - shifter results in a rounded to
+        // 11 bits of precision. Shifter needs to have exponent of
+        // a plus 53 - 11 = 42 and a leading bit in mantissa to guard
+        // against negative values.
+        // So need to have |a| capped to avoid overflow in exponent.
+        // For inputs that are smaller than half precision minnorm
+        // we prepare fixed shifter exponent.
+        unsigned long long shifterBits;
+        if (absa >= 0x3f10000000000000ULL)
+        {   // Here if |a| >= 2^(-14)
+            // add 42 to exponent bits
+            shifterBits  = (ua & 0x7ff0000000000000ULL) + 0x02A0000000000000ULL;
+        }
+        else
+        {   // 2^(-25) < |a| < 2^(-14), potentially results in denormal
+            // set exponent bits to 42 - 14 + bias
+            shifterBits = 0x41B0000000000000ULL;
+        }
+        // set leading mantissa bit to protect against negative inputs
+        shifterBits |= 0x0008000000000000ULL;
+        double shifter;
+        (void)memcpy(&shifter, &shifterBits, sizeof(shifterBits));
+        double aShiftRound = a + shifter;
+
+        // Prevent the compiler from optimizing away a + shifter - shifter
+        // by doing intermediate memcopy and harmless bitwize operation
+        unsigned long long int aShiftRoundBits;
+        (void)memcpy(&aShiftRoundBits, &aShiftRound, sizeof(aShiftRound));
+
+        // the value is positive, so this operation doesn't change anything
+        aShiftRoundBits &= 0x7fffffffffffffffULL;
+
+        (void)memcpy(&aShiftRound, &aShiftRoundBits, sizeof(aShiftRound));
+
+        result = __float2half(static_cast<float>(aShiftRound - shifter));
+    }
+
+    return result;
+,
+    __half result;
+    /*
+    // Perform rounding to 11 bits of precision, convert value
+    // to float and call existing float to half conversion.
+    // By pre-rounding to 11 bits we avoid additional rounding
+    // in float to half conversion.
+    */
+    unsigned long long int absa;
+    unsigned long long int ua;
+    (void)std::memcpy(&ua, &a, sizeof(a));
+    absa = (ua & 0x7fffffffffffffffULL);
+    if ((absa >= 0x40f0000000000000ULL) || (absa <= 0x3e60000000000000ULL))
+    {
+        /*
+        // |a| >= 2^16 or NaN or |a| <= 2^(-25)
+        // double-rounding is not a problem
+        */
+        result = __float2half(static_cast<float>(a));
+    }
+    else
+    {
+        /*
+        // here 2^(-25) < |a| < 2^16
+        // prepare shifter value such that a + shifter
+        // done in double precision performs round-to-nearest-even
+        // and (a + shifter) - shifter results in a rounded to
+        // 11 bits of precision. Shifter needs to have exponent of
+        // a plus 53 - 11 = 42 and a leading bit in mantissa to guard
+        // against negative values.
+        // So need to have |a| capped to avoid overflow in exponent.
+        // For inputs that are smaller than half precision minnorm
+        // we prepare fixed shifter exponent.
+        */
+        unsigned long long shifterBits;
+        if (absa >= 0x3f10000000000000ULL)
+        {
+            /*
+            // Here if |a| >= 2^(-14)
+            // add 42 to exponent bits
+            */
+            shifterBits  = (ua & 0x7ff0000000000000ULL) + 0x02A0000000000000ULL;
+        }
+        else
+        {
+            /*
+            // 2^(-25) < |a| < 2^(-14), potentially results in denormal
+            // set exponent bits to 42 - 14 + bias
+            */
+            shifterBits = 0x41B0000000000000ULL;
+        }
+        // set leading mantissa bit to protect against negative inputs
+        shifterBits |= 0x0008000000000000ULL;
+        double shifter;
+        (void)std::memcpy(&shifter, &shifterBits, sizeof(shifterBits));
+        double aShiftRound = a + shifter;
+
+        /*
+        // Prevent the compiler from optimizing away a + shifter - shifter
+        // by doing intermediate memcopy and harmless bitwize operation
+        */
+        unsigned long long int aShiftRoundBits;
+        (void)std::memcpy(&aShiftRoundBits, &aShiftRound, sizeof(aShiftRound));
+
+        // the value is positive, so this operation doesn't change anything
+        aShiftRoundBits &= 0x7fffffffffffffffULL;
+
+        (void)std::memcpy(&aShiftRound, &aShiftRoundBits, sizeof(aShiftRound));
+
+        result = __float2half(static_cast<float>(aShiftRound - shifter));
+    }
+
+    return result;
+)
+}
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half(const float a)
+{
+    __half val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+,
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+)
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rn(const float a)
+{
+    __half val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{  cvt.rn.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+,
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder > 0x80000000U) || ((remainder == 0x80000000U) && ((r.x & 0x1U) != 0U))) {
+        r.x++;
+    }
+    val = r;
+)
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rz(const float a)
+{
+    __half val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{  cvt.rz.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+,
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    val = r;
+)
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_rd(const float a)
+{
+    __half val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{  cvt.rm.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+,
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign != 0U)) {
+        r.x++;
+    }
+    val = r;
+)
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __float2half_ru(const float a)
+{
+    __half val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{  cvt.rp.f16.f32 %0, %1;}\n" : "=h"(__HALF_TO_US(val)) : "f"(a));
+,
+    __half_raw r;
+    unsigned int sign = 0U;
+    unsigned int remainder = 0U;
+    r.x = __internal_float2half(a, sign, remainder);
+    if ((remainder != 0U) && (sign == 0U)) {
+        r.x++;
+    }
+    val = r;
+)
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float2half2_rn(const float a)
+{
+    __half2 val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{.reg .f16 low;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a));
+,
+    val = __half2(__float2half_rn(a), __float2half_rn(a));
+)
+    return val;
+}
+
+#if defined(__CUDACC__) || defined(_NVHPC_CUDA)
+__CUDA_FP16_DECL__ __half2 __internal_device_float2_to_half2_rn(const float a, const float b) {
+    __half2 val;
+NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80,
+    asm("{ cvt.rn.f16x2.f32 %0, %2, %1; }\n"
+        : "=r"(__HALF2_TO_UI(val)) : "f"(a), "f"(b));
+,
+    asm("{.reg .f16 low,high;\n"
+        "  cvt.rn.f16.f32 low, %1;\n"
+        "  cvt.rn.f16.f32 high, %2;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "f"(a), "f"(b));
+)
+    return val;
+}
+
+#endif /* defined(__CUDACC__) || defined(_NVHPC_CUDA) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __floats2half2_rn(const float a, const float b)
+{
+    __half2 val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+   val = __internal_device_float2_to_half2_rn(a,b);
+,
+    val = __half2(__float2half_rn(a), __float2half_rn(b));
+)
+    return val;
+}
+
+#ifndef __CUDACC_RTC__  /* no host functions in NVRTC mode */
+static inline float __internal_half2float(const unsigned short h)
+{
+    unsigned int sign = ((static_cast<unsigned int>(h) >> 15U) & 1U);
+    unsigned int exponent = ((static_cast<unsigned int>(h) >> 10U) & 0x1fU);
+    unsigned int mantissa = ((static_cast<unsigned int>(h) & 0x3ffU) << 13U);
+    float f;
+    if (exponent == 0x1fU) { /* NaN or Inf */
+        /* discard sign of a NaN */
+        sign = ((mantissa != 0U) ? (sign >> 1U) : sign);
+        mantissa = ((mantissa != 0U) ? 0x7fffffU : 0U);
+        exponent = 0xffU;
+    } else if (exponent == 0U) { /* Denorm or Zero */
+        if (mantissa != 0U) {
+            unsigned int msb;
+            exponent = 0x71U;
+            do {
+                msb = (mantissa & 0x400000U);
+                mantissa <<= 1U; /* normalize */
+                --exponent;
+            } while (msb == 0U);
+            mantissa &= 0x7fffffU; /* 1.mantissa is implicit */
+        }
+    } else {
+        exponent += 0x70U;
+    }
+    const unsigned int u = ((sign << 31U) | (exponent << 23U) | mantissa);
+#if defined(__CUDACC__)
+    (void)memcpy(&f, &u, sizeof(u));
+#else
+    (void)std::memcpy(&f, &u, sizeof(u));
+#endif
+    return f;
+}
+#endif  /* !defined(__CUDACC_RTC__) */
+
+__CUDA_HOSTDEVICE_FP16_DECL__ float __half2float(const __half a)
+{
+    float val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{  cvt.f32.f16 %0, %1;}\n" : "=f"(val) : "h"(__HALF_TO_CUS(a)));
+,
+    val = __internal_half2float(static_cast<__half_raw>(a).x);
+)
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __low2float(const __half2 a)
+{
+    float val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+,
+    val = __internal_half2float(static_cast<__half2_raw>(a).x);
+)
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float __high2float(const __half2 a)
+{
+    float val;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(val) : "r"(__HALF2_TO_CUI(a)));
+,
+    val = __internal_half2float(static_cast<__half2_raw>(a).y);
+)
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ short int __half2short_rz(const __half h)
+{
+    short int i;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rzi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+,
+    const float f = __half2float(h);
+    const short int max_val = (short int)0x7fffU;
+    const short int min_val = (short int)0x8000U;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<short int>(f);
+    }
+)
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned short int __half2ushort_rz(const __half h)
+{
+    unsigned short int i;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rzi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+,
+    const float f = __half2float(h);
+    const unsigned short int max_val = 0xffffU;
+    const unsigned short int min_val = 0U;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0U;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<unsigned short int>(f);
+    }
+)
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ int __half2int_rz(const __half h)
+{
+    int i;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rzi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+,
+    const float f = __half2float(h);
+    const int max_val = (int)0x7fffffffU;
+    const int min_val = (int)0x80000000U;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<int>(f);
+    }
+)
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned int __half2uint_rz(const __half h)
+{
+    unsigned int i;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rzi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+,
+    const float f = __half2float(h);
+    const unsigned int max_val = 0xffffffffU;
+    const unsigned int min_val = 0U;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0U;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<unsigned int>(f);
+    }
+)
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ long long int __half2ll_rz(const __half h)
+{
+    long long int i;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rzi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+,
+    const float f = __half2float(h);
+    const long long int max_val = (long long int)0x7fffffffffffffffULL;
+    const long long int min_val = (long long int)0x8000000000000000ULL;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = min_val;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<long long int>(f);
+    }
+)
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ unsigned long long int __half2ull_rz(const __half h)
+{
+    unsigned long long int i;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rzi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+,
+    const float f = __half2float(h);
+    const unsigned long long int max_val = 0xffffffffffffffffULL;
+    const unsigned long long int min_val = 0ULL;
+    const unsigned short bits = static_cast<unsigned short>(static_cast<__half_raw>(h).x << 1U);
+    // saturation fixup
+    if (bits > (unsigned short)0xF800U) {
+        // NaN
+        i = 0x8000000000000000ULL;
+    } else if (f > static_cast<float>(max_val)) {
+        // saturate maximum
+        i = max_val;
+    } else if (f < static_cast<float>(min_val)) {
+        // saturate minimum
+        i = min_val;
+    } else {
+        // normal value, conversion is well-defined
+        i = static_cast<unsigned long long int>(f);
+    }
+)
+    return i;
+}
+
+/* Intrinsic functions only available to nvcc compilers */
+#if defined(__CUDACC__)
+
+/* CUDA vector-types compatible vector creation function (note returns __half2, not half2) */
+__VECTOR_FUNCTIONS_DECL__ __half2 make_half2(const __half x, const __half y)
+{
+    __half2 t; t.x = x; t.y = y; return t;
+}
+#undef __VECTOR_FUNCTIONS_DECL__
+
+
+/* Definitions of intrinsics */
+__CUDA_HOSTDEVICE_FP16_DECL__ __half2 __float22half2_rn(const float2 a)
+{
+    const __half2 val = __floats2half2_rn(a.x, a.y);
+    return val;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ float2 __half22float2(const __half2 a)
+{
+    float hi_float;
+    float lo_float;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, low;}\n" : "=f"(lo_float) : "r"(__HALF2_TO_CUI(a)));
+
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high},%1;\n"
+        "  cvt.f32.f16 %0, high;}\n" : "=f"(hi_float) : "r"(__HALF2_TO_CUI(a)));
+,
+    lo_float = __internal_half2float(((__half2_raw)a).x);
+    hi_float = __internal_half2float(((__half2_raw)a).y);
+)
+    return make_float2(lo_float, hi_float);
+}
+__CUDA_FP16_DECL__ int __half2int_rn(const __half h)
+{
+    int i;
+    asm("cvt.rni.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_rd(const __half h)
+{
+    int i;
+    asm("cvt.rmi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ int __half2int_ru(const __half h)
+{
+    int i;
+    asm("cvt.rpi.s32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __int2half_rn(const int i)
+{
+    __half h;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rn.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+,
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+)
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rz(const int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_rd(const int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __int2half_ru(const int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ short int __half2short_rn(const __half h)
+{
+    short int i;
+    asm("cvt.rni.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_rd(const __half h)
+{
+    short int i;
+    asm("cvt.rmi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ short int __half2short_ru(const __half h)
+{
+    short int i;
+    asm("cvt.rpi.s16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __short2half_rn(const short int i)
+{
+    __half h;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rn.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+,
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+)
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rz(const short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_rd(const short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __short2half_ru(const short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned int __half2uint_rn(const __half h)
+{
+    unsigned int i;
+    asm("cvt.rni.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_rd(const __half h)
+{
+    unsigned int i;
+    asm("cvt.rmi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned int __half2uint_ru(const __half h)
+{
+    unsigned int i;
+    asm("cvt.rpi.u32.f16 %0, %1;" : "=r"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __uint2half_rn(const unsigned int i)
+{
+    __half h;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rn.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+,
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+)
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rz(const unsigned int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_rd(const unsigned int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __uint2half_ru(const unsigned int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rn(const __half h)
+{
+    unsigned short int i;
+    asm("cvt.rni.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_rd(const __half h)
+{
+    unsigned short int i;
+    asm("cvt.rmi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned short int __half2ushort_ru(const __half h)
+{
+    unsigned short int i;
+    asm("cvt.rpi.u16.f16 %0, %1;" : "=h"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ushort2half_rn(const unsigned short int i)
+{
+    __half h;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rn.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+,
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+)
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rz(const unsigned short int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_rd(const unsigned short int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort2half_ru(const unsigned short int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u16 %0, %1;" : "=h"(__HALF_TO_US(h)) : "h"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rn(const __half h)
+{
+    unsigned long long int i;
+    asm("cvt.rni.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_rd(const __half h)
+{
+    unsigned long long int i;
+    asm("cvt.rmi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ unsigned long long int __half2ull_ru(const __half h)
+{
+    unsigned long long int i;
+    asm("cvt.rpi.u64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ull2half_rn(const unsigned long long int i)
+{
+    __half h;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rn.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+,
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+)
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rz(const unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_rd(const unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ull2half_ru(const unsigned long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.u64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ long long int __half2ll_rn(const __half h)
+{
+    long long int i;
+    asm("cvt.rni.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_rd(const __half h)
+{
+    long long int i;
+    asm("cvt.rmi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_FP16_DECL__ long long int __half2ll_ru(const __half h)
+{
+    long long int i;
+    asm("cvt.rpi.s64.f16 %0, %1;" : "=l"(i) : "h"(__HALF_TO_CUS(h)));
+    return i;
+}
+__CUDA_HOSTDEVICE_FP16_DECL__ __half __ll2half_rn(const long long int i)
+{
+    __half h;
+NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+    asm("cvt.rn.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+,
+    // double-rounding is not a problem here: if integer
+    // has more than 24 bits, it is already too large to
+    // be represented in half precision, and result will
+    // be infinity.
+    const float  f = static_cast<float>(i);
+                 h = __float2half_rn(f);
+)
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rz(const long long int i)
+{
+    __half h;
+    asm("cvt.rz.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_rd(const long long int i)
+{
+    __half h;
+    asm("cvt.rm.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ll2half_ru(const long long int i)
+{
+    __half h;
+    asm("cvt.rp.f16.s64 %0, %1;" : "=h"(__HALF_TO_US(h)) : "l"(i));
+    return h;
+}
+
+__CUDA_FP16_DECL__ __half htrunc(const __half h)
+{
+    __half r;
+    asm("cvt.rzi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hceil(const __half h)
+{
+    __half r;
+    asm("cvt.rpi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hfloor(const __half h)
+{
+    __half r;
+    asm("cvt.rmi.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hrint(const __half h)
+{
+    __half r;
+    asm("cvt.rni.f16.f16 %0, %1;" : "=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(h)));
+    return r;
+}
+
+__CUDA_FP16_DECL__ __half2 h2trunc(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rzi.f16.f16 low, low;\n"
+        "  cvt.rzi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2ceil(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rpi.f16.f16 low, low;\n"
+        "  cvt.rpi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2floor(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rmi.f16.f16 low, low;\n"
+        "  cvt.rmi.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2rint(const __half2 h)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  cvt.rni.f16.f16 low, low;\n"
+        "  cvt.rni.f16.f16 high, high;\n"
+        "  mov.b32 %0, {low,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(h)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lows2half2(const __half2 a, const __half2 b)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {alow,blow};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(b)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __highs2half2(const __half2 a, const __half2 b)
+{
+    __half2 val;
+    asm("{.reg .f16 alow,ahigh,blow,bhigh;\n"
+        "  mov.b32 {alow,ahigh}, %1;\n"
+        "  mov.b32 {blow,bhigh}, %2;\n"
+        "  mov.b32 %0, {ahigh,bhigh};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(b)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __low2half(const __half2 a)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, low;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(a)));
+    return ret;
+}
+__CUDA_FP16_DECL__ int __hisinf(const __half a)
+{
+    int retval;
+    if (__HALF_TO_CUS(a) == 0xFC00U) {
+        retval = -1;
+    } else if (__HALF_TO_CUS(a) == 0x7C00U) {
+        retval = 1;
+    } else {
+        retval = 0;
+    }
+    return retval;
+}
+__CUDA_FP16_DECL__ __half2 __low2half2(const __half2 a)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {low,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __high2half2(const __half2 a)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,high};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half __high2half(const __half2 a)
+{
+    __half ret;
+    asm("{.reg .f16 low,high;\n"
+        " mov.b32 {low,high}, %1;\n"
+        " mov.b16 %0, high;}" : "=h"(__HALF_TO_US(ret)) : "r"(__HALF2_TO_CUI(a)));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __halves2half2(const __half a, const __half b)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%2};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(b)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __half2half2(const __half a)
+{
+    __half2 val;
+    asm("{  mov.b32 %0, {%1,%1};}\n"
+        : "=r"(__HALF2_TO_UI(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 __lowhigh2highlow(const __half2 a)
+{
+    __half2 val;
+    asm("{.reg .f16 low,high;\n"
+        "  mov.b32 {low,high}, %1;\n"
+        "  mov.b32 %0, {high,low};}\n" : "=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ short int __half_as_short(const __half h)
+{
+    return static_cast<short int>(__HALF_TO_CUS(h));
+}
+__CUDA_FP16_DECL__ unsigned short int __half_as_ushort(const __half h)
+{
+    return __HALF_TO_CUS(h);
+}
+__CUDA_FP16_DECL__ __half __short_as_half(const short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = static_cast<unsigned short int>(i);
+    return h;
+}
+__CUDA_FP16_DECL__ __half __ushort_as_half(const unsigned short int i)
+{
+    __half h;
+    __HALF_TO_US(h) = i;
+    return h;
+}
+
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half __hmax(const __half a, const __half b)
+{
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+    __BINARY_OP_HALF_MACRO(max)
+#else
+    const float fa = __half2float(a);
+    const float fb = __half2float(b);
+    float fr;
+    asm("{max.f32 %0,%1,%2;\n}"
+        :"=f"(fr) : "f"(fa), "f"(fb));
+    const __half hr = __float2half(fr);
+    return hr;
+#endif
+}
+__CUDA_FP16_DECL__ __half __hmin(const __half a, const __half b)
+{
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+    __BINARY_OP_HALF_MACRO(min)
+#else
+    const float fa = __half2float(a);
+    const float fb = __half2float(b);
+    float fr;
+    asm("{min.f32 %0,%1,%2;\n}"
+        :"=f"(fr) : "f"(fa), "f"(fb));
+    const __half hr = __float2half(fr);
+    return hr;
+#endif
+}
+
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half2 __hmax2(const __half2 a, const __half2 b)
+{
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+    __BINARY_OP_HALF2_MACRO(max)
+#else
+    const float2 fa = __half22float2(a);
+    const float2 fb = __half22float2(b);
+    float2 fr;
+    asm("{max.f32 %0,%1,%2;\n}"
+        :"=f"(fr.x) : "f"(fa.x), "f"(fb.x));
+    asm("{max.f32 %0,%1,%2;\n}"
+        :"=f"(fr.y) : "f"(fa.y), "f"(fb.y));
+    const __half2 hr = __float22half2_rn(fr);
+    return hr;
+#endif
+}
+__CUDA_FP16_DECL__ __half2 __hmin2(const __half2 a, const __half2 b)
+{
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+    __BINARY_OP_HALF2_MACRO(min)
+#else
+    const float2 fa = __half22float2(a);
+    const float2 fb = __half22float2(b);
+    float2 fr;
+    asm("{min.f32 %0,%1,%2;\n}"
+        :"=f"(fr.x) : "f"(fa.x), "f"(fb.x));
+    asm("{min.f32 %0,%1,%2;\n}"
+        :"=f"(fr.y) : "f"(fa.y), "f"(fb.y));
+    const __half2 hr = __float22half2_rn(fr);
+    return hr;
+#endif
+}
+
+
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 300) || defined(_NVHPC_CUDA)
+/******************************************************************************
+*                           __half, __half2 warp shuffle                     *
+******************************************************************************/
+#define __SHUFFLE_HALF2_MACRO(name) /* do */ {\
+   __half2 r; \
+   asm volatile ("{" __CUDA_FP16_STRINGIFY(name) " %0,%1,%2,%3;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c)); \
+   return r; \
+} /* while(0) */
+
+#define __SHUFFLE_SYNC_HALF2_MACRO(name) /* do */ {\
+   __half2 r; \
+   asm volatile ("{" __CUDA_FP16_STRINGIFY(name) " %0,%1,%2,%3,%4;\n}" \
+       :"=r"(__HALF2_TO_UI(r)): "r"(__HALF2_TO_CUI(var)), "r"(delta), "r"(c), "r"(mask)); \
+   return r; \
+} /* while(0) */
+
+#if defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ < 700)
+
+__CUDA_FP16_DECL__ __half2 __shfl(const __half2 var, const int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_HALF2_MACRO(shfl.idx.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up(const __half2 var, const unsigned int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = (warp_size - static_cast<unsigned>(width)) << 8U;
+    __SHUFFLE_HALF2_MACRO(shfl.up.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down(const __half2 var, const unsigned int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_HALF2_MACRO(shfl.down.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor(const __half2 var, const int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_HALF2_MACRO(shfl.bfly.b32)
+}
+
+#endif /* defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ < 700) */
+
+__CUDA_FP16_DECL__ __half2 __shfl_sync(const unsigned mask, const __half2 var, const int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.idx.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_up_sync(const unsigned mask, const __half2 var, const unsigned int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = (warp_size - static_cast<unsigned>(width)) << 8U;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.up.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_down_sync(const unsigned mask, const __half2 var, const unsigned int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.down.b32)
+}
+__CUDA_FP16_DECL__ __half2 __shfl_xor_sync(const unsigned mask, const __half2 var, const int delta, const int width)
+{
+    unsigned int warp_size;
+    asm("{mov.u32 %0, WARP_SZ;\n}" : "=r"(warp_size));
+    const unsigned int c = ((warp_size - static_cast<unsigned>(width)) << 8U) | 0x1fU;
+    __SHUFFLE_SYNC_HALF2_MACRO(shfl.sync.bfly.b32)
+}
+
+#undef __SHUFFLE_HALF2_MACRO
+#undef __SHUFFLE_SYNC_HALF2_MACRO
+
+#if defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ < 700)
+
+__CUDA_FP16_DECL__ __half __shfl(const __half var, const int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up(const __half var, const unsigned int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_up(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down(const __half var, const unsigned int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_down(temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor(const __half var, const int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_xor(temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /* defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ < 700) */
+
+__CUDA_FP16_DECL__ __half __shfl_sync(const unsigned mask, const __half var, const int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_up_sync(const unsigned mask, const __half var, const unsigned int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_up_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_down_sync(const unsigned mask, const __half var, const unsigned int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_down_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+__CUDA_FP16_DECL__ __half __shfl_xor_sync(const unsigned mask, const __half var, const int delta, const int width)
+{
+    const __half2 temp1 = __halves2half2(var, var);
+    const __half2 temp2 = __shfl_xor_sync(mask, temp1, delta, width);
+    return __low2half(temp2);
+}
+
+#endif /* !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 300) || defined(_NVHPC_CUDA) */
+/******************************************************************************
+*               __half and __half2 __ldg,__ldcg,__ldca,__ldcs                *
+******************************************************************************/
+
+#if defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320) || defined(_NVHPC_CUDA))
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __LDG_PTR   "l"
+#else
+#define __LDG_PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+__CUDA_FP16_DECL__ __half2 __ldg(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.nc.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldg(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.nc.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcg(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cg.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcg(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.cg.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldca(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.ca.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldca(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.ca.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcs(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cs.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcs(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.cs.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr));
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldlu(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.lu.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldlu(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.lu.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half2 __ldcv(const  __half2 *const ptr)
+{
+    __half2 ret;
+    asm ("ld.global.cv.b32 %0, [%1];"  : "=r"(__HALF2_TO_UI(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ __half __ldcv(const __half *const ptr)
+{
+    __half ret;
+    asm ("ld.global.cv.b16 %0, [%1];"  : "=h"(__HALF_TO_US(ret)) : __LDG_PTR(ptr) : "memory");
+    return ret;
+}
+__CUDA_FP16_DECL__ void __stwb(__half2 *const ptr, const __half2 value)
+{
+    asm ("st.global.wb.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwb(__half *const ptr, const __half value)
+{
+    asm ("st.global.wb.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcg(__half2 *const ptr, const __half2 value)
+{
+    asm ("st.global.cg.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcg(__half *const ptr, const __half value)
+{
+    asm ("st.global.cg.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcs(__half2 *const ptr, const __half2 value)
+{
+    asm ("st.global.cs.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stcs(__half *const ptr, const __half value)
+{
+    asm ("st.global.cs.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwt(__half2 *const ptr, const __half2 value)
+{
+    asm ("st.global.wt.b32 [%0], %1;"  :: __LDG_PTR(ptr), "r"(__HALF2_TO_CUI(value)) : "memory");
+}
+__CUDA_FP16_DECL__ void __stwt(__half *const ptr, const __half value)
+{
+    asm ("st.global.wt.b16 [%0], %1;"  :: __LDG_PTR(ptr),  "h"(__HALF_TO_CUS(value)) : "memory");
+}
+#undef __LDG_PTR
+#endif /* defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320) || defined(_NVHPC_CUDA)) */
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) || defined(_NVHPC_CUDA)
+/******************************************************************************
+*                             __half2 comparison                             *
+******************************************************************************/
+#define __COMPARISON_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   asm( "{ " __CUDA_FP16_STRINGIFY(name) ".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+__CUDA_FP16_DECL__ __half2 __heq2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.eq)
+}
+__CUDA_FP16_DECL__ __half2 __hne2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ne)
+}
+__CUDA_FP16_DECL__ __half2 __hle2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.le)
+}
+__CUDA_FP16_DECL__ __half2 __hge2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ge)
+}
+__CUDA_FP16_DECL__ __half2 __hlt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.lt)
+}
+__CUDA_FP16_DECL__ __half2 __hgt2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gt)
+}
+__CUDA_FP16_DECL__ __half2 __hequ2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.equ)
+}
+__CUDA_FP16_DECL__ __half2 __hneu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.neu)
+}
+__CUDA_FP16_DECL__ __half2 __hleu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.leu)
+}
+__CUDA_FP16_DECL__ __half2 __hgeu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.geu)
+}
+__CUDA_FP16_DECL__ __half2 __hltu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.ltu)
+}
+__CUDA_FP16_DECL__ __half2 __hgtu2(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO(set.gtu)
+}
+#undef __COMPARISON_OP_HALF2_MACRO
+/******************************************************************************
+*                 __half2 comparison with mask output                        *
+******************************************************************************/
+#define __COMPARISON_OP_HALF2_MACRO_MASK(name) /* do */ {\
+   unsigned val; \
+   asm( "{ " __CUDA_FP16_STRINGIFY(name) ".u32.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   return val; \
+} /* while(0) */
+__CUDA_FP16_DECL__ unsigned __heq2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.eq)
+}
+__CUDA_FP16_DECL__ unsigned __hne2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.ne)
+}
+__CUDA_FP16_DECL__ unsigned __hle2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.le)
+}
+__CUDA_FP16_DECL__ unsigned __hge2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.ge)
+}
+__CUDA_FP16_DECL__ unsigned __hlt2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.lt)
+}
+__CUDA_FP16_DECL__ unsigned __hgt2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.gt)
+}
+__CUDA_FP16_DECL__ unsigned __hequ2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.equ)
+}
+__CUDA_FP16_DECL__ unsigned __hneu2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.neu)
+}
+__CUDA_FP16_DECL__ unsigned __hleu2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.leu)
+}
+__CUDA_FP16_DECL__ unsigned __hgeu2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.geu)
+}
+__CUDA_FP16_DECL__ unsigned __hltu2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.ltu)
+}
+__CUDA_FP16_DECL__ unsigned __hgtu2_mask(const __half2 a, const __half2 b)
+{
+    __COMPARISON_OP_HALF2_MACRO_MASK(set.gtu)
+}
+#undef __COMPARISON_OP_HALF2_MACRO_MASK
+#define __BOOL_COMPARISON_OP_HALF2_MACRO(name) /* do */ {\
+   __half2 val; \
+   bool retval; \
+   asm( "{ " __CUDA_FP16_STRINGIFY(name) ".f16x2.f16x2 %0,%1,%2;\n}" \
+        :"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)),"r"(__HALF2_TO_CUI(b))); \
+   if (__HALF2_TO_CUI(val) == 0x3C003C00U) {\
+      retval = true; \
+   } else { \
+      retval = false; \
+   }\
+   return retval;\
+} /* while(0) */
+__CUDA_FP16_DECL__ bool __hbeq2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.eq)
+}
+__CUDA_FP16_DECL__ bool __hbne2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ne)
+}
+__CUDA_FP16_DECL__ bool __hble2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.le)
+}
+__CUDA_FP16_DECL__ bool __hbge2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ge)
+}
+__CUDA_FP16_DECL__ bool __hblt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.lt)
+}
+__CUDA_FP16_DECL__ bool __hbgt2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gt)
+}
+__CUDA_FP16_DECL__ bool __hbequ2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.equ)
+}
+__CUDA_FP16_DECL__ bool __hbneu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.neu)
+}
+__CUDA_FP16_DECL__ bool __hbleu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.leu)
+}
+__CUDA_FP16_DECL__ bool __hbgeu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.geu)
+}
+__CUDA_FP16_DECL__ bool __hbltu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.ltu)
+}
+__CUDA_FP16_DECL__ bool __hbgtu2(const __half2 a, const __half2 b)
+{
+    __BOOL_COMPARISON_OP_HALF2_MACRO(set.gtu)
+}
+#undef __BOOL_COMPARISON_OP_HALF2_MACRO
+/******************************************************************************
+*                             __half comparison                              *
+******************************************************************************/
+#define __COMPARISON_OP_HALF_MACRO(name) /* do */ {\
+   unsigned short val; \
+   asm( "{ .reg .pred __$temp3;\n" \
+        "  setp." __CUDA_FP16_STRINGIFY(name) ".f16  __$temp3, %1, %2;\n" \
+        "  selp.u16 %0, 1, 0, __$temp3;}" \
+        : "=h"(val) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(b))); \
+   return (val != 0U) ? true : false; \
+} /* while(0) */
+__CUDA_FP16_DECL__ bool __heq(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(eq)
+}
+__CUDA_FP16_DECL__ bool __hne(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ne)
+}
+__CUDA_FP16_DECL__ bool __hle(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(le)
+}
+__CUDA_FP16_DECL__ bool __hge(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ge)
+}
+__CUDA_FP16_DECL__ bool __hlt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(lt)
+}
+__CUDA_FP16_DECL__ bool __hgt(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gt)
+}
+__CUDA_FP16_DECL__ bool __hequ(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(equ)
+}
+__CUDA_FP16_DECL__ bool __hneu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(neu)
+}
+__CUDA_FP16_DECL__ bool __hleu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(leu)
+}
+__CUDA_FP16_DECL__ bool __hgeu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(geu)
+}
+__CUDA_FP16_DECL__ bool __hltu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(ltu)
+}
+__CUDA_FP16_DECL__ bool __hgtu(const __half a, const __half b)
+{
+    __COMPARISON_OP_HALF_MACRO(gtu)
+}
+#undef __COMPARISON_OP_HALF_MACRO
+/******************************************************************************
+*                            __half2 arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half2 __hadd2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul)
+}
+__CUDA_FP16_DECL__ __half2 __hadd2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2_sat(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul.sat)
+}
+__CUDA_FP16_DECL__ __half2 __hadd2_rn(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(add.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hsub2_rn(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(sub.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hmul2_rn(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(mul.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_sat(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.sat)
+}
+__CUDA_FP16_DECL__ __half2 __h2div(const __half2 a, const __half2 b) {
+    __half ha = __low2half(a);
+    __half hb = __low2half(b);
+
+    const __half v1 = __hdiv(ha, hb);
+
+    ha = __high2half(a);
+    hb = __high2half(b);
+
+    const __half v2 = __hdiv(ha, hb);
+
+    return __halves2half2(v1, v2);
+}
+/******************************************************************************
+*                             __half arithmetic                             *
+******************************************************************************/
+__CUDA_FP16_DECL__ __half __hadd(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add)
+}
+__CUDA_FP16_DECL__ __half __hsub(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub)
+}
+__CUDA_FP16_DECL__ __half __hmul(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul)
+}
+__CUDA_FP16_DECL__ __half __hadd_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add.sat)
+}
+__CUDA_FP16_DECL__ __half __hsub_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub.sat)
+}
+__CUDA_FP16_DECL__ __half __hmul_sat(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul.sat)
+}
+__CUDA_FP16_DECL__ __half __hadd_rn(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(add.rn)
+}
+__CUDA_FP16_DECL__ __half __hsub_rn(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(sub.rn)
+}
+__CUDA_FP16_DECL__ __half __hmul_rn(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(mul.rn)
+}
+__CUDA_FP16_DECL__ __half __hfma(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn)
+}
+__CUDA_FP16_DECL__ __half __hfma_sat(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.sat)
+}
+__CUDA_FP16_DECL__ __half __hdiv(const __half a, const __half b) {
+    __half v;
+    __half abs;
+    __half den;
+    __HALF_TO_US(den) = 0x008FU;
+
+    float rcp;
+    const float fa = __half2float(a);
+    const float fb = __half2float(b);
+
+    asm("{rcp.approx.ftz.f32 %0, %1;\n}" :"=f"(rcp) : "f"(fb));
+
+    float fv = rcp * fa;
+
+    v = __float2half(fv);
+    abs = __habs(v);
+    if (__hlt(abs, den) && __hlt(__float2half(0.0f), abs))  {
+        const float err = __fmaf_rn(-fb, fv, fa);
+        fv = __fmaf_rn(rcp, err, fv);
+        v = __float2half(fv);
+    }
+    return v;
+}
+
+/******************************************************************************
+*                             __half2 functions                  *
+******************************************************************************/
+#define __SPEC_CASE2(i,r, spc, ulp) \
+   "{.reg.b32 spc, ulp, p;\n"\
+   "  mov.b32 spc," __CUDA_FP16_STRINGIFY(spc) ";\n"\
+   "  mov.b32 ulp," __CUDA_FP16_STRINGIFY(ulp) ";\n"\
+   "  set.eq.f16x2.f16x2 p," __CUDA_FP16_STRINGIFY(i) ", spc;\n"\
+   "  fma.rn.f16x2 " __CUDA_FP16_STRINGIFY(r) ",p,ulp," __CUDA_FP16_STRINGIFY(r) ";\n}\n"
+#define __SPEC_CASE(i,r, spc, ulp) \
+   "{.reg.b16 spc, ulp, p;\n"\
+   "  mov.b16 spc," __CUDA_FP16_STRINGIFY(spc) ";\n"\
+   "  mov.b16 ulp," __CUDA_FP16_STRINGIFY(ulp) ";\n"\
+   "  set.eq.f16.f16 p," __CUDA_FP16_STRINGIFY(i) ", spc;\n"\
+   "  fma.rn.f16 " __CUDA_FP16_STRINGIFY(r) ",p,ulp," __CUDA_FP16_STRINGIFY(r) ";\n}\n"
+#define __APPROX_FCAST(fun) /* do */ {\
+   __half val;\
+   asm("{.reg.b32         f;        \n"\
+                " .reg.b16         r;        \n"\
+                "  mov.b16         r,%1;     \n"\
+                "  cvt.f32.f16     f,r;      \n"\
+                "  " __CUDA_FP16_STRINGIFY(fun) ".approx.ftz.f32   f,f;  \n"\
+                "  cvt.rn.f16.f32      r,f;  \n"\
+                "  mov.b16         %0,r;     \n"\
+                "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));\
+   return val;\
+} /* while(0) */
+#define __APPROX_FCAST2(fun) /* do */ {\
+   __half2 val;\
+   asm("{.reg.b16         hl, hu;         \n"\
+                " .reg.b32         fl, fu;         \n"\
+                "  mov.b32         {hl, hu}, %1;   \n"\
+                "  cvt.f32.f16     fl, hl;         \n"\
+                "  cvt.f32.f16     fu, hu;         \n"\
+                "  " __CUDA_FP16_STRINGIFY(fun) ".approx.ftz.f32   fl, fl;     \n"\
+                "  " __CUDA_FP16_STRINGIFY(fun) ".approx.ftz.f32   fu, fu;     \n"\
+                "  cvt.rn.f16.f32      hl, fl;     \n"\
+                "  cvt.rn.f16.f32      hu, fu;     \n"\
+                "  mov.b32         %0, {hl, hu};   \n"\
+                "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));       \
+   return val;\
+} /* while(0) */
+static __device__ __forceinline__ float __float_simpl_sinf(float a);
+static __device__ __forceinline__ float __float_simpl_cosf(float a);
+__CUDA_FP16_DECL__ __half hsin(const __half a) {
+    const float sl = __float_simpl_sinf(__half2float(a));
+    __half r = __float2half_rn(sl);
+    asm("{\n\t"
+        "  .reg.b16 i,r,t;     \n\t"
+        "  mov.b16 r, %0;      \n\t"
+        "  mov.b16 i, %1;      \n\t"
+        "  and.b16 t, r, 0x8000U; \n\t"
+        "  abs.f16 r, r;   \n\t"
+        "  abs.f16 i, i;   \n\t"
+        __SPEC_CASE(i, r, 0X32B3U, 0x0800U)
+        __SPEC_CASE(i, r, 0X5CB0U, 0x9000U)
+        "  or.b16  r,r,t;      \n\t"
+        "  mov.b16 %0, r;      \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2sin(const __half2 a) {
+    const float sl = __float_simpl_sinf(__half2float(a.x));
+    const float sh = __float_simpl_sinf(__half2float(a.y));
+    __half2 r = __floats2half2_rn(sl, sh);
+    asm("{\n\t"
+        "  .reg.b32 i,r,t;             \n\t"
+        "  mov.b32 r, %0;              \n\t"
+        "  mov.b32 i, %1;              \n\t"
+        "  and.b32 t, r, 0x80008000U;   \n\t"
+        "  abs.f16x2 r, r;   \n\t"
+        "  abs.f16x2 i, i;   \n\t"
+        __SPEC_CASE2(i, r, 0X32B332B3U, 0x08000800U)
+        __SPEC_CASE2(i, r, 0X5CB05CB0U, 0x90009000U)
+        "  or.b32  r, r, t;            \n\t"
+        "  mov.b32 %0, r;              \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half hcos(const __half a) {
+    const float cl = __float_simpl_cosf(__half2float(a));
+    __half r = __float2half_rn(cl);
+    asm("{\n\t"
+        "  .reg.b16 i,r;        \n\t"
+        "  mov.b16 r, %0;       \n\t"
+        "  mov.b16 i, %1;       \n\t"
+        "  abs.f16 i, i;        \n\t"
+        __SPEC_CASE(i, r, 0X2B7CU, 0x1000U)
+        "  mov.b16 %0, r;       \n"
+        "}\n" : "+h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 h2cos(const __half2 a) {
+    const float cl = __float_simpl_cosf(__half2float(a.x));
+    const float ch = __float_simpl_cosf(__half2float(a.y));
+    __half2 r = __floats2half2_rn(cl, ch);
+    asm("{\n\t"
+        "  .reg.b32 i,r;   \n\t"
+        "  mov.b32 r, %0;  \n\t"
+        "  mov.b32 i, %1;  \n\t"
+        "  abs.f16x2 i, i; \n\t"
+        __SPEC_CASE2(i, r, 0X2B7C2B7CU, 0x10001000U)
+        "  mov.b32 %0, r;  \n"
+        "}\n" : "+r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+static __device__ __forceinline__ float __internal_trig_reduction_kernel(const float a, unsigned int *const quadrant)
+{
+    const float ar = __fmaf_rn(a, 0.636619772F, 12582912.0F);
+    const unsigned q = __float_as_uint(ar);
+    const float j = __fsub_rn(ar, 12582912.0F);
+    float t = __fmaf_rn(j, -1.5707962512969971e+000F, a);
+    t = __fmaf_rn(j, -7.5497894158615964e-008F, t);
+    *quadrant = q;
+    return t;
+}
+static __device__ __forceinline__ float __internal_sin_cos_kernel(const float x, const unsigned int i)
+{
+    float z;
+    const float x2 = x*x;
+    float a8;
+    float a6;
+    float a4;
+    float a2;
+    float a1;
+    float a0;
+
+    if ((i & 1U) != 0U) {
+        // cos
+        a8 =  2.44331571e-5F;
+        a6 = -1.38873163e-3F;
+        a4 =  4.16666457e-2F;
+        a2 = -5.00000000e-1F;
+        a1 = x2;
+        a0 = 1.0F;
+    }
+    else {
+        // sin
+        a8 = -1.95152959e-4F;
+        a6 =  8.33216087e-3F;
+        a4 = -1.66666546e-1F;
+        a2 = 0.0F;
+        a1 = x;
+        a0 = x;
+    }
+
+    z = __fmaf_rn(a8, x2, a6);
+    z = __fmaf_rn(z, x2, a4);
+    z = __fmaf_rn(z, x2, a2);
+    z = __fmaf_rn(z, a1, a0);
+
+    if ((i & 2U) != 0U) {
+        z = -z;
+    }
+    return z;
+}
+static __device__ __forceinline__ float __float_simpl_sinf(float a)
+{
+    float z;
+    unsigned i;
+    a = __internal_trig_reduction_kernel(a, &i);
+    z = __internal_sin_cos_kernel(a, i);
+    return z;
+}
+static __device__ __forceinline__ float __float_simpl_cosf(float a)
+{
+    float z;
+    unsigned i;
+    a = __internal_trig_reduction_kernel(a, &i);
+    z = __internal_sin_cos_kernel(a, (i & 0x3U) + 1U);
+    return z;
+}
+
+__CUDA_FP16_DECL__ __half hexp(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C, nZ;       \n"
+        " .reg.b16         h,r;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  mov.b32         C, 0x3fb8aa3bU; \n"
+        "  mov.b32         nZ, 0x80000000U;\n"
+        "  fma.rn.f32      f,f,C,nZ;       \n"
+        "  ex2.approx.ftz.f32  f,f;        \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X1F79U, 0x9400U)
+        __SPEC_CASE(h, r, 0X25CFU, 0x9400U)
+        __SPEC_CASE(h, r, 0XC13BU, 0x0400U)
+        __SPEC_CASE(h, r, 0XC1EFU, 0x0200U)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu,C,nZ; \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x3fb8aa3bU; \n"
+        "  mov.b32         nZ, 0x80000000U;\n"
+        "  fma.rn.f32      fl,fl,C,nZ;     \n"
+        "  fma.rn.f32      fu,fu,C,nZ;     \n"
+        "  ex2.approx.ftz.f32  fl, fl;     \n"
+        "  ex2.approx.ftz.f32  fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0X1F791F79U, 0x94009400U)
+        __SPEC_CASE2(h, r, 0X25CF25CFU, 0x94009400U)
+        __SPEC_CASE2(h, r, 0XC13BC13BU, 0x04000400U)
+        __SPEC_CASE2(h, r, 0XC1EFC1EFU, 0x02000200U)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp2(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, ULP;         \n"
+        " .reg.b16         r;              \n"
+        "  mov.b16         r,%1;           \n"
+        "  cvt.f32.f16     f,r;            \n"
+        "  ex2.approx.ftz.f32      f,f;    \n"
+        "  mov.b32         ULP, 0x33800000U;\n"
+        "  fma.rn.f32      f,f,ULP,f;      \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, ULP;    \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  ex2.approx.ftz.f32  fl, fl;     \n"
+        "  ex2.approx.ftz.f32  fu, fu;     \n"
+        "  mov.b32         ULP, 0x33800000U;\n"
+        "  fma.rn.f32      fl,fl,ULP,fl;   \n"
+        "  fma.rn.f32      fu,fu,ULP,fu;   \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         %0, {hl, hu};   \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hexp10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h,r;            \n"
+        " .reg.b32         f, C, nZ;       \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  mov.b32         C, 0x40549A78U; \n"
+        "  mov.b32         nZ, 0x80000000U;\n"
+        "  fma.rn.f32      f,f,C,nZ;       \n"
+        "  ex2.approx.ftz.f32  f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x34DEU, 0x9800U)
+        __SPEC_CASE(h, r, 0x9766U, 0x9000U)
+        __SPEC_CASE(h, r, 0x9972U, 0x1000U)
+        __SPEC_CASE(h, r, 0xA5C4U, 0x1000U)
+        __SPEC_CASE(h, r, 0xBF0AU, 0x8100U)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2exp10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         h,r,fl,fu,C,nZ; \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  mov.b32         h, %1;          \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  mov.b32         C, 0x40549A78U; \n"
+        "  mov.b32         nZ, 0x80000000U;\n"
+        "  fma.rn.f32      fl,fl,C,nZ;     \n"
+        "  fma.rn.f32      fu,fu,C,nZ;     \n"
+        "  ex2.approx.ftz.f32  fl, fl;     \n"
+        "  ex2.approx.ftz.f32  fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(h, r, 0x34DE34DEU, 0x98009800U)
+        __SPEC_CASE2(h, r, 0x97669766U, 0x90009000U)
+        __SPEC_CASE2(h, r, 0x99729972U, 0x10001000U)
+        __SPEC_CASE2(h, r, 0xA5C4A5C4U, 0x10001000U)
+        __SPEC_CASE2(h, r, 0xBF0ABF0AU, 0x81008100U)
+        "  mov.b32         %0, r;  \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog2(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f;              \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.ftz.f32  f, f;       \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(r, r, 0xA2E2U, 0x8080U)
+        __SPEC_CASE(r, r, 0xBF46U, 0x9400U)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log2(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;         \n"
+        " .reg.b32         fl, fu, r, p;   \n"
+        "  mov.b32         {hl, hu}, %1;   \n"
+        "  cvt.f32.f16     fl, hl;         \n"
+        "  cvt.f32.f16     fu, hu;         \n"
+        "  lg2.approx.ftz.f32  fl, fl;     \n"
+        "  lg2.approx.ftz.f32  fu, fu;     \n"
+        "  cvt.rn.f16.f32      hl, fl;     \n"
+        "  cvt.rn.f16.f32      hu, fu;     \n"
+        "  mov.b32         r, {hl, hu};    \n"
+        __SPEC_CASE2(r, r, 0xA2E2A2E2U, 0x80808080U)
+        __SPEC_CASE2(r, r, 0xBF46BF46U, 0x94009400U)
+        "  mov.b32         %0, r;          \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog(const __half a) {
+    __half val;
+    asm("{.reg.b32         f, C;           \n"
+        " .reg.b16         r,h;            \n"
+        "  mov.b16         h,%1;           \n"
+        "  cvt.f32.f16     f,h;            \n"
+        "  lg2.approx.ftz.f32  f,f;        \n"
+        "  mov.b32         C, 0x3f317218U;  \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r,f;        \n"
+        __SPEC_CASE(h, r, 0X160DU, 0x9C00U)
+        __SPEC_CASE(h, r, 0X3BFEU, 0x8010U)
+        __SPEC_CASE(h, r, 0X3C0BU, 0x8080U)
+        __SPEC_CASE(h, r, 0X6051U, 0x1C00U)
+        "  mov.b16         %0,r;           \n"
+        "}": "=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.ftz.f32  fl, fl;         \n"
+        "  lg2.approx.ftz.f32  fu, fu;         \n"
+        "  mov.b32         C, 0x3f317218U;     \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0X160D160DU, 0x9C009C00U)
+        __SPEC_CASE2(h, r, 0X3BFE3BFEU, 0x80108010U)
+        __SPEC_CASE2(h, r, 0X3C0B3C0BU, 0x80808080U)
+        __SPEC_CASE2(h, r, 0X60516051U, 0x1C001C00U)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half hlog10(const __half a) {
+    __half val;
+    asm("{.reg.b16         h, r;           \n"
+        " .reg.b32         f, C;           \n"
+        "  mov.b16         h, %1;          \n"
+        "  cvt.f32.f16     f, h;           \n"
+        "  lg2.approx.ftz.f32  f, f;       \n"
+        "  mov.b32         C, 0x3E9A209BU; \n"
+        "  mul.f32         f,f,C;          \n"
+        "  cvt.rn.f16.f32      r, f;       \n"
+        __SPEC_CASE(h, r, 0x338FU, 0x1000U)
+        __SPEC_CASE(h, r, 0x33F8U, 0x9000U)
+        __SPEC_CASE(h, r, 0x57E1U, 0x9800U)
+        __SPEC_CASE(h, r, 0x719DU, 0x9C00U)
+        "  mov.b16         %0, r;          \n"
+        "}":"=h"(__HALF_TO_US(val)) : "h"(__HALF_TO_CUS(a)));
+    return val;
+}
+__CUDA_FP16_DECL__ __half2 h2log10(const __half2 a) {
+    __half2 val;
+    asm("{.reg.b16         hl, hu;             \n"
+        " .reg.b32         r, fl, fu, C, h;    \n"
+        "  mov.b32         {hl, hu}, %1;       \n"
+        "  mov.b32         h, %1;              \n"
+        "  cvt.f32.f16     fl, hl;             \n"
+        "  cvt.f32.f16     fu, hu;             \n"
+        "  lg2.approx.ftz.f32  fl, fl;         \n"
+        "  lg2.approx.ftz.f32  fu, fu;         \n"
+        "  mov.b32         C, 0x3E9A209BU;     \n"
+        "  mul.f32         fl,fl,C;            \n"
+        "  mul.f32         fu,fu,C;            \n"
+        "  cvt.rn.f16.f32      hl, fl;         \n"
+        "  cvt.rn.f16.f32      hu, fu;         \n"
+        "  mov.b32         r, {hl, hu};        \n"
+        __SPEC_CASE2(h, r, 0x338F338FU, 0x10001000U)
+        __SPEC_CASE2(h, r, 0x33F833F8U, 0x90009000U)
+        __SPEC_CASE2(h, r, 0x57E157E1U, 0x98009800U)
+        __SPEC_CASE2(h, r, 0x719D719DU, 0x9C009C00U)
+        "  mov.b32         %0, r;              \n"
+        "}":"=r"(__HALF2_TO_UI(val)) : "r"(__HALF2_TO_CUI(a)));
+    return val;
+}
+#undef __SPEC_CASE2
+#undef __SPEC_CASE
+__CUDA_FP16_DECL__ __half2 h2rcp(const __half2 a) {
+    __APPROX_FCAST2(rcp)
+}
+__CUDA_FP16_DECL__ __half hrcp(const __half a) {
+    __APPROX_FCAST(rcp)
+}
+__CUDA_FP16_DECL__ __half2 h2rsqrt(const __half2 a) {
+    __APPROX_FCAST2(rsqrt)
+}
+__CUDA_FP16_DECL__ __half hrsqrt(const __half a) {
+    __APPROX_FCAST(rsqrt)
+}
+__CUDA_FP16_DECL__ __half2 h2sqrt(const __half2 a) {
+    __APPROX_FCAST2(sqrt)
+}
+__CUDA_FP16_DECL__ __half hsqrt(const __half a) {
+    __APPROX_FCAST(sqrt)
+}
+#undef __APPROX_FCAST
+#undef __APPROX_FCAST2
+__CUDA_FP16_DECL__ __half2 __hisnan2(const __half2 a)
+{
+    __half2 r;
+    asm("{set.nan.f16x2.f16x2 %0,%1,%2;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)), "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ bool __hisnan(const __half a)
+{
+    __half r;
+    asm("{set.nan.f16.f16 %0,%1,%2;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)), "h"(__HALF_TO_CUS(a)));
+    return __HALF_TO_CUS(r) != 0U;
+}
+__CUDA_FP16_DECL__ __half2 __hneg2(const __half2 a)
+{
+    __half2 r;
+    asm("{neg.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __hneg(const __half a)
+{
+    __half r;
+    asm("{neg.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half2 __habs2(const __half2 a)
+{
+    __half2 r;
+    asm("{abs.f16x2 %0,%1;\n}"
+        :"=r"(__HALF2_TO_UI(r)) : "r"(__HALF2_TO_CUI(a)));
+    return r;
+}
+__CUDA_FP16_DECL__ __half __habs(const __half a)
+{
+    __half r;
+    asm("{abs.f16 %0,%1;\n}"
+        :"=h"(__HALF_TO_US(r)) : "h"(__HALF_TO_CUS(a)));
+    return r;
+}
+
+__CUDA_FP16_DECL__ __half2 __hcmadd(const __half2 a, const __half2 b, const __half2 c)
+{
+    // fast version of complex multiply-accumulate
+    // (a.re, a.im) * (b.re, b.im) + (c.re, c.im)
+    // acc.re = (c.re + a.re*b.re) - a.im*b.im
+    // acc.im = (c.im + a.re*b.im) + a.im*b.re
+    __half real_tmp =  __hfma(a.x, b.x, c.x);
+    __half img_tmp  =  __hfma(a.x, b.y, c.y);
+    real_tmp = __hfma(__hneg(a.y), b.y, real_tmp);
+    img_tmp  = __hfma(a.y,         b.x, img_tmp);
+    return make_half2(real_tmp, img_tmp);
+}
+
+#endif /* !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) || defined(_NVHPC_CUDA) */
+
+#if defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)
+__CUDA_FP16_DECL__ __half __hmax_nan(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(max.NaN)
+}
+__CUDA_FP16_DECL__ __half __hmin_nan(const __half a, const __half b)
+{
+    __BINARY_OP_HALF_MACRO(min.NaN)
+}
+__CUDA_FP16_DECL__ __half __hfma_relu(const __half a, const __half b, const __half c)
+{
+    __TERNARY_OP_HALF_MACRO(fma.rn.relu)
+}
+
+__CUDA_FP16_DECL__ __half2 __hmax2_nan(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(max.NaN)
+}
+__CUDA_FP16_DECL__ __half2 __hmin2_nan(const __half2 a, const __half2 b)
+{
+    __BINARY_OP_HALF2_MACRO(min.NaN)
+}
+__CUDA_FP16_DECL__ __half2 __hfma2_relu(const __half2 a, const __half2 b, const __half2 c)
+{
+    __TERNARY_OP_HALF2_MACRO(fma.rn.relu)
+}
+#endif /*defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 800)*/
+
+/* Define __PTR for atomicAdd prototypes below, undef after done */
+#if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)
+#define __PTR   "l"
+#else
+#define __PTR   "r"
+#endif /*(defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__)*/
+
+#if defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 600)
+
+__CUDA_FP16_DECL__  __half2 atomicAdd(__half2 *const address, const __half2 val) {
+    __half2 r;
+    asm volatile ("{ atom.add.noftz.f16x2 %0,[%1],%2; }\n"
+                  : "=r"(__HALF2_TO_UI(r)) : __PTR(address), "r"(__HALF2_TO_CUI(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 600)*/
+
+#if defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 700)
+
+__CUDA_FP16_DECL__  __half atomicAdd(__half *const address, const __half val) {
+    __half r;
+    asm volatile ("{ atom.add.noftz.f16 %0,[%1],%2; }\n"
+                  : "=h"(__HALF_TO_US(r))
+                  : __PTR(address), "h"(__HALF_TO_CUS(val))
+                  : "memory");
+   return r;
+}
+
+#endif /*defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 700)*/
+
+#undef __PTR
+
+#undef __CUDA_FP16_DECL__
+#endif /* defined(__CUDACC__) */
+#endif /* defined(__cplusplus) */
+
+#undef __TERNARY_OP_HALF2_MACRO
+#undef __TERNARY_OP_HALF_MACRO
+#undef __BINARY_OP_HALF2_MACRO
+#undef __BINARY_OP_HALF_MACRO
+
+#undef __CUDA_HOSTDEVICE_FP16_DECL__
+#undef __CUDA_FP16_DECL__
+
+#undef __HALF_TO_US
+#undef __HALF_TO_CUS
+#undef __HALF2_TO_UI
+#undef __HALF2_TO_CUI
+
+/* Define first-class types "half" and "half2", unless user specifies otherwise via "#define CUDA_NO_HALF" */
+/* C cannot ever have these types defined here, because __half and __half2 are C++ classes */
+#if defined(__cplusplus) && !defined(CUDA_NO_HALF)
+typedef __half half;
+typedef __half2 half2;
+// for consistency with __nv_bfloat16
+typedef __half      __nv_half;
+typedef __half2     __nv_half2;
+typedef __half_raw  __nv_half_raw;
+typedef __half2_raw __nv_half2_raw;
+typedef __half        nv_half;
+typedef __half2       nv_half2;
+#endif /* defined(__cplusplus) && !defined(CUDA_NO_HALF) */
+
+#if defined(__CPP_VERSION_AT_LEAST_11_FP16)
+#undef __CPP_VERSION_AT_LEAST_11_FP16
+#endif /* defined(__CPP_VERSION_AT_LEAST_11_FP16) */
+
+#endif /* end of include guard: __CUDA_FP16_HPP__ */
diff --git a/cupy/_core/include/cupy/atomics.cuh b/cupy/_core/include/cupy/atomics.cuh
new file mode 100644
index 0000000..772c1d7
--- /dev/null
+++ b/cupy/_core/include/cupy/atomics.cuh
@@ -0,0 +1,114 @@
+#pragma once
+
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)
+
+__device__ double atomicAdd(double *address, double val)
+{
+    unsigned long long int* address_as_ull = (unsigned long long int*)address;
+    unsigned long long int old = *address_as_ull;
+    unsigned long long int assumed;
+    do {
+	assumed = old;
+	old = atomicCAS(address_as_ull, assumed,
+			__double_as_longlong(val + __longlong_as_double(assumed)));
+    } while (assumed != old);
+    return __longlong_as_double(old);
+}
+
+#endif // #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)
+
+__device__ float16 atomicAdd(float16* address, float16 val) {
+  unsigned int *aligned = (unsigned int*)((size_t)address - ((size_t)address & 2));
+  unsigned int old = *aligned;
+  unsigned int assumed;
+  unsigned short old_as_us;
+  do {
+    assumed = old;
+    old_as_us = (unsigned short)((size_t)address & 2 ? old >> 16 : old & 0xffff);
+#if __CUDACC_VER_MAJOR__ >= 9
+    half sum = __float2half_rn(__half2float(__ushort_as_half(old_as_us)) + float(val));
+    unsigned short sum_as_us = __half_as_ushort(sum);
+#else
+    unsigned short sum_as_us = __float2half_rn(__half2float(old_as_us) + float(val));
+#endif
+    unsigned int sum_as_ui = (size_t)address & 2 ? (sum_as_us << 16) | (old & 0xffff)
+                                                 : (old & 0xffff0000) | sum_as_us;
+    old = atomicCAS(aligned, assumed, sum_as_ui);
+  } while(assumed != old);
+  __half_raw raw;
+  raw.x = old_as_us;
+  return float16(raw);
+};
+
+
+__device__ long long atomicAdd(long long *address, long long val) {
+    return atomicAdd(reinterpret_cast<unsigned long long*>(address),
+                     static_cast<unsigned long long>(val));
+}
+
+
+#if __HIPCC__
+#include <hip/hip_version.h>
+#endif  // #if __HIPCC__
+
+// Skip if ROCm 4.5+ as it implements the following atomic functions.
+#if !defined(__HIPCC__) || HIP_VERSION < 40400000
+
+__device__ float atomicMax(float* address, float val) {
+  int* address_as_i = reinterpret_cast<int*>(address);
+  int old = *address_as_i, assumed;
+  do {
+    assumed = old;
+    old = atomicCAS(
+        reinterpret_cast<int*>(address), assumed,
+        __float_as_int(fmaxf(val, __int_as_float(assumed))));
+  } while (assumed != old);
+  return __int_as_float(old);
+}
+
+
+__device__ double atomicMax(double* address, double val) {
+  unsigned long long* address_as_i =
+    reinterpret_cast<unsigned long long*>(address);
+  unsigned long long old = *address_as_i, assumed;
+  do {
+    assumed = old;
+    const long long result = __double_as_longlong(
+      fmaxf(val, __longlong_as_double(reinterpret_cast<long long&>(assumed))));
+    old = atomicCAS(
+      address_as_i, assumed,
+      reinterpret_cast<const unsigned long long&>(result));
+  } while (assumed != old);
+  return __longlong_as_double(reinterpret_cast<long long&>(old));
+}
+
+
+__device__ float atomicMin(float* address, float val) {
+  int* address_as_i = reinterpret_cast<int*>(address);
+  int old = *address_as_i, assumed;
+  do {
+    assumed = old;
+    old = atomicCAS(
+        reinterpret_cast<int*>(address), assumed,
+        __float_as_int(fminf(val, __int_as_float(assumed))));
+  } while (assumed != old);
+  return __int_as_float(old);
+}
+
+
+__device__ double atomicMin(double* address, double val) {
+  unsigned long long* address_as_i =
+    reinterpret_cast<unsigned long long*>(address);
+  unsigned long long old = *address_as_i, assumed;
+  do {
+    assumed = old;
+    const long long result = __double_as_longlong(
+      fminf(val, __longlong_as_double(reinterpret_cast<long long&>(assumed))));
+    old = atomicCAS(
+      address_as_i, assumed,
+      reinterpret_cast<const unsigned long long&>(result));
+  } while (assumed != old);
+  return __longlong_as_double(reinterpret_cast<long long&>(old));
+}
+
+#endif  // #if !defined(__HIPCC__) || HIP_VERSION < 40400000
diff --git a/cupy/_core/include/cupy/carray.cuh b/cupy/_core/include/cupy/carray.cuh
new file mode 100644
index 0000000..e117a3a
--- /dev/null
+++ b/cupy/_core/include/cupy/carray.cuh
@@ -0,0 +1,849 @@
+#pragma once
+
+#if __cplusplus >= 201103 || (defined(_MSC_VER) && _MSC_VER >= 1900)
+#ifndef __CUDACC_RTC__
+// in NVRTC std:initializer_list is pre-defined (no need to include it)
+#include <initializer_list>
+#endif
+#endif
+
+// Basic implementation of std::type_traits
+// We use this regardless when C++ is requested, as NVRTC by default lacks many
+// C++ features like this. We need to wrap in a namespace in case Jitify kicks
+// in and/or users provide custom definitions.
+namespace cupy {
+  namespace type_traits {
+    template<bool B, class T, class F>
+    struct conditional { typedef T type; };
+    template<class T, class F>
+    struct conditional<false, T, F> { typedef F type; };
+
+    template<bool B, class T = void>
+    struct enable_if {};
+    template<class T>
+    struct enable_if<true, T> { typedef T type; };
+  }
+}
+
+// math
+#ifndef M_PI
+#define M_PI 3.1415926535897932384626433832795
+#endif
+
+#ifdef __HIPCC_RTC__
+
+#include <hip/hip_version.h>
+#if HIP_VERSION >= 40400000
+// HIP runtime headers can be no longer explicitly included since ROCm 4.5 so
+// we only include necessary standard headers.
+#include <cassert>
+#include <cstddef>
+
+// Confirmed to AMD, ROCm 5.0 doesn't recognize __forceinline__ and
+// __noinline__.
+#define __noinline__ __attribute__((noinline))
+#define __forceinline__ inline __attribute__((always_inline))
+
+#else
+#include <hip/hip_fp16.h>
+#endif  // #if HIP_VERSION >= 40400000
+
+#elif __HIPCC__
+
+#include <hip/hip_fp16.h>
+
+#elif __CUDACC_VER_MAJOR__ >= 9
+
+#include <cuda_fp16.h>
+
+#else  // #if __CUDACC_VER_MAJOR__ >= 9
+
+struct __half_raw {
+  unsigned short x;
+};
+
+struct half {
+private:
+  unsigned short data_;
+public:
+  __device__ half() {}
+  __device__ half(const half &v) : data_(v.data_) {}
+  __device__ half(float v) : data_(__float2half_rn(v)) {}
+
+  explicit __device__ half(const __half_raw &v) : data_(v.x) {}
+  explicit __device__ half(bool v) : data_(__float2half_rn(float(v))) {}
+  explicit __device__ half(double v) : data_(__float2half_rn(float(v))) {}
+  explicit __device__ half(int v) : data_(__float2half_rn(float(v))) {}
+  explicit __device__ half(unsigned int v) : data_(__float2half_rn(float(v))) {}
+  explicit __device__ half(long long v) : data_(__float2half_rn(float(v))) {}
+  explicit __device__ half(unsigned long long v) : data_(__float2half_rn(float(v))) {}
+
+  __device__ operator float() const {return __half2float(data_);}
+  __device__ operator __half_raw() const {__half_raw ret = {data_}; return ret;}
+};
+
+#endif  // #if __CUDACC_VER_MAJOR__ >= 9
+
+class float16 {
+private:
+  half  data_;
+public:
+  __device__ float16() {}
+  __device__ float16(float v) : data_(v) {}
+
+  explicit __device__ float16(bool v) : data_(float(v)) {}
+  explicit __device__ float16(double v) : data_(v) {}
+  explicit __device__ float16(int v) : data_(v) {}
+  explicit __device__ float16(unsigned int v) : data_(v) {}
+  explicit __device__ float16(long long v) : data_(v) {}
+  explicit __device__ float16(unsigned long long v) : data_(v) {}
+
+  explicit __device__ float16(const half &v): data_(v) {}
+  explicit __device__ float16(const __half_raw &v): data_(v) {}
+
+  __device__ operator float() const {return float(data_);}
+
+  static const unsigned short nan = 0x7e00u;
+
+  __device__ int iszero() const {
+    return (__half_raw(data_).x & 0x7fffu) == 0;
+  }
+
+  __device__ int isnan() const {
+    __half_raw raw_ = __half_raw(data_);
+    return (raw_.x & 0x7c00u) == 0x7c00u && (raw_.x & 0x03ffu) != 0x0000u;
+  }
+
+  __device__ int isinf() const {
+    return (__half_raw(data_).x & 0x7fffu) == 0x7c00u;
+  }
+
+  __device__ int isfinite() const {
+    return (__half_raw(data_).x & 0x7c00u) != 0x7c00u;
+  }
+
+  __device__ int signbit() const {
+    return (__half_raw(data_).x & 0x8000u) != 0;
+  }
+
+#ifdef __HIPCC__
+
+__device__ float16 operator-() {
+  return float16(-data_);
+}
+
+#endif
+
+  template<typename T>
+  inline __device__ float16& operator+=(const T& rhs) {
+    *this = *this + rhs;
+    return *this;
+  }
+
+  template<typename T>
+  inline __device__ float16& operator-=(const T& rhs) {
+    *this = *this - rhs;
+    return *this;
+  }
+
+  template<typename T>
+  inline __device__ float16& operator*=(const T& rhs) {
+    *this = *this * rhs;
+    return *this;
+  }
+
+  template<typename T>
+  inline __device__ float16& operator/=(const T& rhs) {
+    *this = *this / rhs;
+    return *this;
+  }
+
+  friend __device__ float16 copysign(float16 x, float16 y) {
+    __half_raw x_raw_ = __half_raw(x.data_);
+    __half_raw y_raw_ = __half_raw(y.data_);
+    __half_raw ret_raw_;
+    ret_raw_.x = (x_raw_.x & 0x7fffu) | (y_raw_.x & 0x8000u);
+    return float16(ret_raw_);
+  }
+
+  friend __device__ float16 nextafter(float16 x, float16 y) {
+    __half_raw x_raw_ = __half_raw(x.data_);
+    __half_raw y_raw_ = __half_raw(y.data_);
+    __half_raw ret_raw_;
+    if (!x.isfinite() || y.isnan()) {
+      ret_raw_.x = nan;
+    } else if (eq_nonan(x, y)) {
+      ret_raw_.x = x_raw_.x;
+    } else if (x.iszero()) {
+      ret_raw_.x = (y_raw_.x & 0x8000u) + 1;
+    } else if (!(x_raw_.x & 0x8000u)) {
+      if (static_cast<signed short>(x_raw_.x) > static_cast<signed short>(y_raw_.x)) {
+        ret_raw_.x = x_raw_.x - 1;
+      } else {
+        ret_raw_.x = x_raw_.x + 1;
+      }
+    } else if(!(y_raw_.x & 0x8000u) || (x_raw_.x & 0x7fffu) > (y_raw_.x & 0x7fffu)) {
+      ret_raw_.x = x_raw_.x - 1;
+    } else {
+      ret_raw_.x = x_raw_.x + 1;
+    }
+    return float16(ret_raw_);
+  }
+
+private:
+  static __device__ int eq_nonan(const float16 x, const float16 y) {
+    __half_raw x_raw_ = __half_raw(x.data_);
+    __half_raw y_raw_ = __half_raw(y.data_);
+    return (x_raw_.x == y_raw_.x || ((x_raw_.x | y_raw_.x) & 0x7fff) == 0);
+  }
+};
+
+
+__device__ float16 min(float16 x, float16 y) {
+  return float16(min(float(x), float(y)));
+}
+__device__ float16 max(float16 x, float16 y) {
+  return float16(max(float(x), float(y)));
+}
+__device__ float16 fmin(float16 x, float16 y) {
+  return float16(fmin(float(x), float(y)));
+}
+__device__ float16 fmax(float16 x, float16 y) {
+  return float16(fmax(float(x), float(y)));
+}
+__device__ int iszero(float16 x) {return x.iszero();}
+__device__ int isnan(float16 x) {return x.isnan();}
+__device__ int isinf(float16 x) {return x.isinf();}
+__device__ int isfinite(float16 x) {return x.isfinite();}
+__device__ int signbit(float16 x) {return x.signbit();}
+
+// CArray
+#define CUPY_FOR(i, n) \
+    for (ptrdiff_t i = \
+            static_cast<ptrdiff_t>(blockIdx.x) * blockDim.x + threadIdx.x; \
+         i < (n); \
+         i += static_cast<ptrdiff_t>(blockDim.x) * gridDim.x)
+
+#ifdef CUPY_JIT_MODE
+#ifdef CUPY_JIT_NVCC
+#include <thrust/swap.h>
+#include <thrust/tuple.h>
+#include <thrust/pair.h>
+#else
+#include <cupy/swap.cuh>
+#include <cupy/tuple.cuh>
+#include <cupy/pair.cuh>
+#endif  // CUPY_JIT_NVCC
+#endif  // CUPY_JIT_MODE
+
+#ifdef CUPY_JIT_MODE
+namespace cupy {
+
+/*
+ * param ndim: the size of the returned tuple
+ * param T: the type of the tuple elements
+ */
+template<int ndim, typename T>
+struct as_tuple {
+
+    template <int _ndim, typename... Args>
+    struct as_tuple_impl {
+        using ChildClass = as_tuple_impl<_ndim - 1, T, Args...>;
+        using type = typename ChildClass::type;
+
+        template <typename Ints>
+        __device__ static type call(Ints ints, Args... args) {
+            return ChildClass::call(ints, ints[_ndim - 1], args...);
+        }
+    };
+
+    template <typename... Args>
+    struct as_tuple_impl<0, Args...> {
+        using type = thrust::tuple<Args...>;
+
+        template <typename Ints>
+        __device__ static type call(Ints ints, Args... args) {
+            return thrust::make_tuple(args...);
+        }
+    };
+
+    using type = typename as_tuple_impl<ndim>::type;
+
+    template <typename Ints, typename... Args>
+    __device__ static type call(Ints ints, Args... args) {
+        return as_tuple_impl<ndim>::call(ints, args...);
+    }
+
+};
+
+}  // namespace cupy
+
+template <int dim>
+struct Dim {
+  __device__ Dim() {}
+};
+#endif  // CUPY_JIT_MODE
+
+template <typename T, typename index_t>
+class CArrayIterator {
+public:
+  typedef ptrdiff_t difference_type;
+  typedef T value_type;
+  typedef T* pointer;
+  typedef T& reference;
+#ifdef CUPY_JIT_NVCC
+  typedef std::random_access_iterator_tag iterator_category;
+#endif  // CUPY_JIT_NVCC
+
+private:
+  T* head_;
+  index_t step_;
+public:
+  __host__ __device__ CArrayIterator(T* head, index_t step) {
+    this->head_ = head;
+    this->step_ = step;
+  }
+  __host__ __device__ CArrayIterator(const CArrayIterator& itr) {
+    this->head_ = itr.head_;
+    this->step_ = itr.step_;
+  }
+  __host__ __device__ bool operator==(const CArrayIterator& itr) const {
+    return (this->head_ == itr.head_) && (this->step_ == itr.step_);
+  }
+  __host__ __device__ bool operator!=(const CArrayIterator& itr) const {
+    return !(*this == itr);
+  }
+  __host__ __device__ T& operator*() const {
+    return *(this->head_);
+  }
+  __host__ __device__ const T* operator->() const {
+    return this->head_;
+  }
+  __host__ __device__ CArrayIterator& operator++() {
+    this->head_ += this->step_;
+    return *this;
+  }
+  __host__ __device__ CArrayIterator operator++(int) {
+    CArrayIterator tmp = *this;
+    this->head_ += this->step_;
+    return tmp;
+  }
+  __host__ __device__ CArrayIterator& operator--() {
+    this->head_ -= this->step_;
+    return *this;
+  }
+  __host__ __device__ CArrayIterator operator--(int) {
+    CArrayIterator tmp = *this;
+    this->head_ -= this->step_;
+    return tmp;
+  }
+  __host__ __device__ CArrayIterator operator+(ptrdiff_t n) const {
+    CArrayIterator out = *this;
+    out.head_ += out.step_ * n;
+    return out;
+  }
+  __host__ __device__ difference_type operator-(const CArrayIterator& itr) const {
+    return (this->head_ - itr.head_) / this->step_;
+  }
+  __host__ __device__ CArrayIterator operator-(ptrdiff_t n) const {
+    CArrayIterator out = *this;
+    out.head_ -= out.step_ * n;
+    return out;
+  }
+  __host__ __device__ CArrayIterator& operator+=(ptrdiff_t n) {
+    this->head_ += this->step_ * n;
+    return *this;
+  }
+  __host__ __device__ CArrayIterator& operator-=(ptrdiff_t n) {
+    this->head_ -= this->step_ * n;
+    return *this;
+  }
+  __host__ __device__ T& operator[](index_t n) const {
+    return *(this->head_ + this->step_ * n);
+  }
+  __host__ __device__ bool operator<(const CArrayIterator& itr) const {
+    return this->head_ < itr.head_;
+  }
+  __host__ __device__ bool operator>(const CArrayIterator& itr) const {
+    return this->head_ > itr.head_;
+  }
+  __host__ __device__ bool operator<=(const CArrayIterator& itr) const {
+    return !(*this > itr);
+  }
+  __host__ __device__ bool operator>=(const CArrayIterator& itr) const {
+    return !(*this < itr);
+  }
+};
+
+template <typename T, int _ndim, bool _c_contiguous=false, bool _use_32bit_indexing=false>
+class CArray {
+public:
+  static const int ndim = _ndim;
+  static const bool c_contiguous = _c_contiguous;
+  typedef typename cupy::type_traits::conditional<_use_32bit_indexing, int, ptrdiff_t>::type index_t;
+  typedef typename cupy::type_traits::conditional<_c_contiguous, T*, CArrayIterator<T, index_t> >::type iterator;
+
+private:
+  T* data_;
+  ptrdiff_t size_;
+  ptrdiff_t shape_[ndim];
+  ptrdiff_t strides_[ndim];
+
+public:
+  // Constructor supports pointers or initializer lists and strides is optional
+  // as long as _c_contiguous=true.
+  //    CArray<T, 3> ca(data, shape, strides);
+  //    CArray<T, 3, true> ca(data, shape);
+  //    CArray<T, 3> ca(data, {1, 2, 3}, {48, 24, 8});
+  //    CArray<T, 3, true> ca(data, {1, 2, 3});
+  // Initializer lists and optional strides are only supported with -std=c++11
+  // or higher.
+
+  template <typename Int1, typename Int2>
+  __device__ CArray(T* data, const Int1* shape, const Int2* strides)
+      : data_(data), size_(1)
+  {
+    if (_c_contiguous) {
+      assert(strides[_ndim-1] == sizeof(T));
+      for (int i = _ndim-1; i > 0; i--) {
+        assert(strides[i-1] == shape[i] * strides[i]);
+      }
+    }
+    for (int i = 0; i < _ndim; i++) {
+      this->size_ *= shape[i];
+      this->shape_[i] = shape[i];
+      this->strides_[i] = strides[i];
+    }
+  } 
+
+#if __cplusplus >= 201103 || (defined(_MSC_VER) && _MSC_VER >= 1900)
+  template <typename Int, typename U=T>
+  __device__ CArray(typename cupy::type_traits::enable_if<_c_contiguous, U>::type* data,
+                    const Int* shape)
+      : data_(data), size_(1)
+  {
+    for (int i = 0; i < _ndim; i++) {
+      this->size_ *= shape[i];
+      this->shape_[i] = shape[i];
+    }
+    this->strides_[_ndim-1] = sizeof(T);
+    for (int i = _ndim-1; i > 0; i--) {
+      this->strides_[i-1] = shape[i] * this->strides_[i];
+    }
+  }
+
+  template <typename Int, typename U=T>
+  __device__ CArray(typename cupy::type_traits::enable_if<_c_contiguous, U>::type* data,
+                    const std::initializer_list<Int> shape)
+      : CArray(data, shape.begin())
+  {
+    assert(shape.size() == _ndim);
+  }
+
+  template <typename Int1, typename Int2>
+  __device__ CArray(T* data,
+                    const std::initializer_list<Int1> shape,
+                    const std::initializer_list<Int2> strides)
+      : CArray(data, shape.begin(), strides.begin())
+  {
+    assert(shape.size() == _ndim);
+    assert(strides.size() == _ndim);
+  }
+#endif
+
+  __device__ CArray() : data_(NULL), size_(1)
+  {
+    memset(this->shape_, 0, sizeof(this->shape_));
+    memset(this->strides_, 0, sizeof(this->strides_));
+  }
+
+  __device__ ptrdiff_t size() const {
+    return size_;
+  }
+
+  __device__ const ptrdiff_t* shape() const {
+    return shape_;
+  }
+
+  __device__ const ptrdiff_t* strides() const {
+    return strides_;
+  }
+
+#ifdef CUPY_JIT_MODE
+  __device__ typename cupy::as_tuple<_ndim, ptrdiff_t>::type get_shape() const {
+    return cupy::as_tuple<_ndim, ptrdiff_t>::call(shape_);
+  }
+
+  __device__ typename cupy::as_tuple<_ndim, ptrdiff_t>::type get_strides() const {
+    return cupy::as_tuple<_ndim, ptrdiff_t>::call(strides_);
+  }
+#endif  // CUPY_JIT_MODE
+
+#if __cplusplus >= 201103 || (defined(_MSC_VER) && _MSC_VER >= 1900)
+  template <typename Int>
+  __device__ T& operator[](const std::initializer_list<Int> idx_) {
+    assert(idx_.size() == _ndim);
+    Int idx[ndim];
+    memcpy(idx, idx_.begin(), ndim*sizeof(Int));
+    return this->operator[](idx);
+  }
+
+  template <typename Int>
+  __device__ const T& operator[](const std::initializer_list<Int> idx_) const {
+    assert(idx_.size() == _ndim);
+    Int idx[ndim];
+    memcpy(idx, idx_.begin(), ndim*sizeof(Int));
+    return this->operator[](idx);
+  }
+#endif
+
+  template <typename Int>
+  __device__ T& operator[](const Int (&idx)[ndim]) {
+    return const_cast<T&>(const_cast<const CArray&>(*this)[idx]);
+  }
+
+  template <typename Int>
+  __device__ const T& operator[](const Int (&idx)[ndim]) const {
+    index_t diff = 0;
+    for (int dim = 0; dim < ndim; ++dim) {
+      diff += static_cast<index_t>(strides_[dim]) * static_cast<index_t>(idx[dim]);
+    }
+    const char* ptr = reinterpret_cast<const char*>(data_);
+    return *reinterpret_cast<const T*>(ptr + diff);
+  }
+
+  __device__ T& operator[](ptrdiff_t i) {
+    return const_cast<T&>(const_cast<const CArray&>(*this)[i]);
+  }
+
+#ifdef CUPY_JIT_MODE
+  __forceinline__ __device__ iterator begin_ptr() const {
+    return reinterpret_cast<T*>(data_);
+  }
+
+  __forceinline__ __device__ iterator end_ptr() const {
+    return reinterpret_cast<T*>(data_) + size_;
+  }
+
+  __forceinline__ __device__ iterator begin() const {
+    return iterator(data_, strides_[0] / sizeof(T));
+  }
+
+  __forceinline__ __device__ iterator end() const {
+    return iterator(data_ + size_ * strides_[0] / sizeof(T), strides_[0] / sizeof(T));
+  }
+
+  template <typename Tuple, int dim>
+  __forceinline__ __device__ const T& _indexing(const Tuple &idx, Dim<dim>, const char* ptr) const {
+    index_t i = static_cast<index_t>(thrust::get<dim>(idx));
+    ptr += static_cast<index_t>(strides_[dim]) * i;
+    return _indexing(idx, Dim<dim + 1>(), ptr);
+  }
+
+  template <typename Tuple>
+  __forceinline__ __device__ const T& _indexing(const Tuple &idx, Dim<_ndim>, const char* ptr) const {
+    return *reinterpret_cast<const T*>(ptr);
+  }
+
+  template <typename Tuple>
+  __forceinline__ __device__ const T& _indexing(const Tuple &idx) const {
+    const char* ptr = reinterpret_cast<const char*>(data_);
+    return _indexing(idx, Dim<0>(), ptr);
+  }
+
+  template <typename Tuple>
+  __forceinline__ __device__ T& _indexing(const Tuple &idx) {
+    return const_cast<T&>(const_cast<const CArray&>(*this)._indexing(idx));
+  }
+
+  template <typename Tuple, int dim, int dimreduce>
+  __forceinline__ __device__ char* _slicing(const Tuple &idx, char* new_head_ptr, Dim<dim>, Dim<dimreduce>) const {
+    index_t i = static_cast<index_t>(thrust::get<dim>(idx));
+    new_head_ptr += static_cast<index_t>(strides_[dim]) * i;
+    return _slicing(idx, new_head_ptr, Dim<dim+1>(), Dim<dimreduce>());
+  }
+
+  template <typename Tuple, int dimreduce>
+  __forceinline__ __device__ char* _slicing(const Tuple &idx, char* new_head_ptr, Dim<dimreduce>, Dim<dimreduce>) const {
+    return new_head_ptr;
+  }
+
+  template <typename Tuple, int dimreduce>
+  __forceinline__ __device__ CArray<T, _ndim - dimreduce, _c_contiguous, _use_32bit_indexing> _slicing(const Tuple &idx, Dim<dimreduce>) {
+    char* new_head_ptr = reinterpret_cast<char*>(data_);
+    new_head_ptr = _slicing(idx, new_head_ptr, Dim<0>(), Dim<dimreduce>());
+    T* new_head = reinterpret_cast<T*>(new_head_ptr);
+    return CArray<T, _ndim - dimreduce, _c_contiguous, _use_32bit_indexing>(new_head, shape_ + dimreduce, strides_ + dimreduce);
+  }
+
+  __forceinline__ __device__ CArray<T, _ndim - 1, _c_contiguous, _use_32bit_indexing> _slicing(const int idx) {
+    char* new_head_ptr = reinterpret_cast<char*>(data_);
+    index_t i = static_cast<index_t>(idx);
+    new_head_ptr += static_cast<index_t>(strides_[0]) * i;
+    T* new_head = reinterpret_cast<T*>(new_head_ptr);
+    return CArray<T, _ndim - 1, _c_contiguous, _use_32bit_indexing>(new_head, shape_ + 1, strides_ + 1);
+  }
+#endif  // CUPY_JIT_MODE
+
+  __device__ const T& operator[](ptrdiff_t idx) const {
+    if (c_contiguous) {
+      // contiguous arrays can be directly addressed by the
+      // numeric value, avoiding expensive 64 bit operations in cuda
+      return data_[idx];
+    } else {
+      // 64-bit mults and divs are pretty expensive and can lead to severe
+      // performance degradation in computation bound kernels
+      index_t diff = 0;
+      index_t i = static_cast<index_t>(idx);
+      for (int dim = ndim; --dim > 0; ) {
+          index_t shape_dim = static_cast<index_t>(shape_[dim]);
+          diff += static_cast<index_t>(strides_[dim]) * (i % shape_dim);
+          i /= shape_dim;
+      }
+      diff += static_cast<index_t>(strides_[0]) * i;
+      const char* ptr = reinterpret_cast<const char*>(data_);
+      return *reinterpret_cast<const T*>(ptr + diff);
+    }
+  }
+};
+
+template <typename T, bool _use_32bit_indexing>
+
+class CArray<T, 0, true, _use_32bit_indexing> {
+private:
+  T* data_;
+  ptrdiff_t size_;
+
+public:
+  static const int ndim = 0;
+
+  __device__ CArray() : data_(NULL), size_(1) { }
+  
+  __device__ explicit CArray(T* data) : data_(data), size_(1) { }
+
+  template <typename Int>
+  __device__ CArray(T* data, Int size) : data_(data), size_(size) { }
+
+  // These constructors are just to match the non-0-dim constructors
+  template <typename Int1, typename Int2>
+  __device__ CArray(T* data, const Int1* shape, const Int2* strides)
+      : data_(data), size_(1) { }
+
+#if __cplusplus >= 201103 || (defined(_MSC_VER) && _MSC_VER >= 1900)
+  __device__ CArray(T* data,
+                    const std::initializer_list<int> shape,
+                    const std::initializer_list<int> strides)
+      : data_(data), size_(1)
+  {
+    assert(shape.size() == 0);
+    assert(strides.size() == 0);
+  }
+#endif
+
+
+  __device__ ptrdiff_t size() const {
+    return size_;
+  }
+
+  __device__ const ptrdiff_t* shape() const {
+    return NULL;
+  }
+
+  __device__ const ptrdiff_t* strides() const {
+    return NULL;
+  }
+
+  template <typename U>
+  __device__ T& operator[](const U&) {
+    return *data_;
+  }
+
+  template <typename U>
+  __device__ T operator[](const U&) const {
+    return *data_;
+  }
+};
+
+template <int _ndim, bool _use_32bit_indexing=false>
+class CIndexer {
+public:
+  static const int ndim = _ndim;
+private:
+  ptrdiff_t size_;
+  ptrdiff_t shape_[ndim];
+  ptrdiff_t index_[ndim];
+
+  typedef ptrdiff_t index_t[ndim];
+
+public:
+  // Constructor supports pointers or initializer lists and index is optional
+  //    CIndexer<3> ca(shape, index);
+  //    CIndexer<3> ca({1, 2, 3}, {0, 1, 1});
+  //    CIndexer<3> ca = {1, 2, 3};
+  // Initializer lists are only supported with -std=c++11 or higher.
+
+  template <typename Int>
+  __device__ explicit CIndexer(const Int* shape)
+      : size_(1) {
+    for (int i = 0; i < _ndim; i++) {
+      this->size_ *= shape[i];
+      this->shape_[i] = shape[i];
+      this->index_[i] = 0;
+    }
+  }
+
+  template <typename Int1, typename Int2>
+  __device__ CIndexer(const Int1* shape, const Int2* index)
+      : size_(1) {
+    for (int i = 0; i < _ndim; i++) {
+      this->size_ *= shape[i];
+      this->shape_[i] = shape[i];
+      this->index_[i] = index[i];
+    }
+  }
+
+#if __cplusplus >= 201103 || (defined(_MSC_VER) && _MSC_VER >= 1900)
+  template <typename Int>
+  __device__ CIndexer(const std::initializer_list<Int> shape)
+      : CIndexer(shape.begin())
+  {
+    assert(shape.size() == _ndim);
+  }
+
+  template <typename Int1, typename Int2>
+  __device__ CIndexer(const std::initializer_list<Int1> shape,
+                      const std::initializer_list<Int2> index)
+      : CIndexer(shape.begin(), index.begin())
+  {
+    assert(shape.size() == _ndim);
+    assert(index.size() == _ndim);
+  }
+#endif
+
+  __device__ CIndexer() : size_(1) 
+  {
+    memset(this->shape_, 0, sizeof(this->shape_));
+    memset(this->index_, 0, sizeof(this->index_));
+  }
+
+  __device__ ptrdiff_t size() const {
+    return size_;
+  }
+
+  __device__ const index_t& get() const {
+    return index_;
+  }
+
+  __device__ void set(ptrdiff_t i) {
+    // ndim == 0 case uses partial template specialization
+    if (ndim == 1) {
+      index_[0] = i;
+    } else if (!_use_32bit_indexing && size_ > 1LL << 31) {
+      // 64-bit division is very slow on GPU
+      this->_set(static_cast<unsigned long long int>(i));
+    } else {
+      this->_set(static_cast<unsigned int>(i));
+    }
+  }
+
+private:
+  template<typename index_t>
+  __device__ void _set(index_t i) {
+      for (int dim = ndim; --dim > 0; ) {
+        index_t s = static_cast<index_t>(shape_[dim]);
+        if (s & (s - 1)) {
+          index_t t = i / s;
+          index_[dim] = i - t * s;
+          i = t;
+        } else { // exp of 2
+          index_[dim] = i & (s - 1);
+          i >>= _log2(s);
+        }
+      }
+      index_[0] = i;
+  }
+
+  // can also be implemented as __ffs(x)-1 or 31-__clz(x)
+  static unsigned int __device__ _log2(unsigned int x) { return __popc(x-1); }
+  static unsigned long long int __device__ _log2(unsigned long long int x) { return __popcll(x-1); }
+};
+
+template <bool _use_32bit_indexing>
+class CIndexer<0, _use_32bit_indexing> {
+private:
+  ptrdiff_t size_;
+
+public:
+  static const int ndim = 0;
+
+  __device__ CIndexer() : size_(1) { }
+
+  template <typename Int>
+  __device__ explicit CIndexer(Int size) : size_(size) { }
+
+  // These constructors are just to match the non-0-dim constructors
+  template <typename Int>
+  __device__ explicit CIndexer(const Int* shape) : size_(1) { }
+
+  template <typename Int1, typename Int2>
+  __device__ CIndexer(const Int1* shape, const Int2* strides)
+      : size_(1) { }
+
+#if __cplusplus >= 201103 || (defined(_MSC_VER) && _MSC_VER >= 1900)
+  __device__ CIndexer(const std::initializer_list<int> shape)
+      : size_(1)
+  {
+    assert(shape.size() == 0);
+  }
+
+  __device__ CIndexer(const std::initializer_list<int> shape,
+                      const std::initializer_list<int> index)
+      : size_(1)
+  {
+    assert(shape.size() == 0);
+    assert(index.size() == 0);
+  }
+#endif
+
+  __device__ ptrdiff_t size() const {
+    return size_;
+  }
+
+  __device__ void set(ptrdiff_t i) {
+  }
+
+  __device__ const ptrdiff_t* get() const {
+    return NULL;
+  }
+};
+
+__device__ int _floor_divide(int x, int y) {
+  if (y == 0) return 0;
+  int q = x / y;
+  return q - (((x < 0) != (y < 0)) && q * y != x);
+}
+
+__device__ long long _floor_divide(long long x, long long y) {
+  if (y == 0) return 0;
+  long long q = x / y;
+  return q - (((x < 0) != (y < 0)) && q * y != x);
+}
+
+__device__ unsigned _floor_divide(unsigned x, unsigned y) {
+  if (y == 0) return 0;
+  return x / y;
+}
+
+__device__ unsigned long long _floor_divide(
+    unsigned long long x, unsigned long long y) {
+  if (y == 0) return 0;
+  return x / y;
+}
+
+__device__ float _floor_divide(float x, float y) {
+  return floor(x / y);
+}
+
+__device__ double _floor_divide(double x, double y) {
+  return floor(x / y);
+}
diff --git a/cupy/_core/include/cupy/complex.cuh b/cupy/_core/include/cupy/complex.cuh
new file mode 100644
index 0000000..fa4908b
--- /dev/null
+++ b/cupy/_core/include/cupy/complex.cuh
@@ -0,0 +1,100 @@
+#pragma once
+
+#include <cupy/complex/complex.h>
+
+using thrust::complex;
+using thrust::conj;
+using thrust::real;
+using thrust::imag;
+using thrust::arg;
+using thrust::exp;
+using thrust::log;
+using thrust::log10;
+using thrust::sin;
+using thrust::cos;
+using thrust::tan;
+using thrust::sinh;
+using thrust::cosh;
+using thrust::tanh;
+using thrust::asinh;
+using thrust::acosh;
+using thrust::atanh;
+using thrust::asin;
+using thrust::acos;
+using thrust::atan;
+
+template<typename T>
+__host__ __device__ bool isnan(complex<T> x) {
+    return isnan(x.real()) || isnan(x.imag());
+}
+
+template<typename T>
+__host__ __device__ bool isinf(complex<T> x) {
+    return isinf(x.real()) || isinf(x.imag());
+}
+
+template<typename T>
+__host__ __device__ bool isfinite(complex<T> x) {
+    return isfinite(x.real()) && isfinite(x.imag());
+}
+
+template<typename T>
+__host__ __device__ complex<T> log1p(complex<T> x) {
+    x += 1;
+    return log(x);
+}
+
+template<typename T>
+__host__ __device__ complex<T> log2(complex<T> x) {
+    complex<T> y = log(x);
+    y /= log(T(2));
+    return y;
+}
+
+template<typename T>
+__host__ __device__ complex<T> expm1(complex<T> x) {
+    complex<T> y = exp(x);
+    y -= 1;
+    return y;
+}
+
+template<typename T>
+__host__ __device__ complex<T> min(complex<T> x, complex<T> y) {
+    if (isnan(x)) {
+        return y;
+    } else if (isnan(y)) {
+        return x;
+    } else if (x.real() < y.real()) {
+        return x;
+    } else if (x.real() > y.real()) {
+        return y;
+    } else if (x.imag() < y.imag()) {
+        return x;
+    } else {
+        return y;
+    }
+}
+
+template<typename T>
+__host__ __device__ complex<T> max(complex<T> x, complex<T> y) {
+    if (isnan(x)) {
+        return y;
+    } else if (isnan(y)) {
+        return x;
+    } else if (x.real() < y.real()) {
+        return y;
+    } else if (x.real() > y.real()) {
+        return x;
+    } else if (x.imag() < y.imag()) {
+        return y;
+    } else {
+        return x;
+    }
+}
+
+template<typename T>
+__host__ __device__ complex<T> rint(complex<T> x) {
+    return complex<T>(rint(x.real()), rint(x.imag()));
+}
+
+// ToDo: assignment operator for complex<T> = T2 for T2 all types
diff --git a/cupy/_core/include/cupy/complex/README.md b/cupy/_core/include/cupy/complex/README.md
new file mode 100644
index 0000000..ee065ba
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/README.md
@@ -0,0 +1,3 @@
+These files are copied from thrust project and are modified.
+
+    http://thrust.github.io/
\ No newline at end of file
diff --git a/cupy/_core/include/cupy/complex/arithmetic.h b/cupy/_core/include/cupy/complex/arithmetic.h
new file mode 100644
index 0000000..0cc5293
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/arithmetic.h
@@ -0,0 +1,314 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+
+/* --- Binary Arithmetic Operators --- */
+
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const complex<T>& lhs,
+                                                const complex<T>& rhs) {
+  return complex<T>(lhs.real() + rhs.real(), lhs.imag() + rhs.imag());
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const volatile complex<T>& lhs,
+                                                const volatile complex<T>& rhs) {
+  return complex<T>(lhs.real() + rhs.real(), lhs.imag() + rhs.imag());
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const complex<T>& lhs,
+                                                const T& rhs) {
+  return complex<T>(lhs.real() + rhs, lhs.imag());
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const T& lhs,
+                                                const complex<T>& rhs) {
+  return complex<T>(rhs.real() + lhs, rhs.imag());
+}
+
+// TODO(leofang): support operator+ for (complex<T0> x, complex<T1> y)
+
+template <typename T>
+__host__ __device__ inline complex<T> operator-(const complex<T>& lhs,
+                                                const complex<T>& rhs) {
+  return complex<T>(lhs.real() - rhs.real(), lhs.imag() - rhs.imag());
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator-(const complex<T>& lhs,
+                                                const T& rhs) {
+  return complex<T>(lhs.real() - rhs, lhs.imag());
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator-(const T& lhs,
+                                                const complex<T>& rhs) {
+  return complex<T>(lhs - rhs.real(), -rhs.imag());
+}
+
+// TODO(leofang): support operator- for (complex<T0> x, complex<T1> y)
+
+template <typename T>
+__host__ __device__ inline complex<T> operator*(const complex<T>& lhs,
+                                                const complex<T>& rhs) {
+  return complex<T>(lhs.real() * rhs.real() - lhs.imag() * rhs.imag(),
+                            lhs.real() * rhs.imag() + lhs.imag() * rhs.real());
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator*(const complex<T>& lhs,
+                                                const T& rhs) {
+  return complex<T>(lhs.real() * rhs, lhs.imag() * rhs);
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator*(const T& lhs,
+                                                const complex<T>& rhs) {
+  return complex<T>(rhs.real() * lhs, rhs.imag() * lhs);
+}
+
+// TODO(leofang): support operator* for (complex<T0> x, complex<T1> y)
+
+template <typename T>
+__host__ __device__ inline complex<T> operator/(const complex<T>& lhs,
+                                                const complex<T>& rhs) {
+  T s = abs(rhs.real()) + abs(rhs.imag());
+  T oos = T(1.0) / s;
+  T ars = lhs.real() * oos;
+  T ais = lhs.imag() * oos;
+  T brs = rhs.real() * oos;
+  T bis = rhs.imag() * oos;
+  s = (brs * brs) + (bis * bis);
+  oos = T(1.0) / s;
+  complex<T> quot(((ars * brs) + (ais * bis)) * oos,
+                  ((ais * brs) - (ars * bis)) * oos);
+  return quot;
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator/(const complex<T>& lhs,
+                                                const T& rhs) {
+  return complex<T>(lhs.real() / rhs, lhs.imag() / rhs);
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator/(const T& lhs,
+                                                const complex<T>& rhs) {
+  return complex<T>(lhs) / rhs;
+}
+
+// TODO(leofang): support operator/ for (complex<T0> x, complex<T1> y)
+
+/* --- Unary comparison with Numpy logic. This means that a + bi > c + di if either
+ * a > c or a == c and b > d. --- */
+
+template <typename T>
+__host__ __device__ inline bool operator<(const complex<T>& lhs,
+                                          const complex<T>& rhs) {
+  if (lhs == rhs) {
+      return false;
+  } else if (lhs.real() < rhs.real()) {
+      return true;
+  } else if (lhs.real() == rhs.real()) {
+      return lhs.imag() < rhs.imag();
+  } else {
+      return false;
+  }
+}
+
+template <typename T>
+__host__ __device__ inline bool operator<=(const complex<T>& lhs,
+                                           const complex<T>& rhs) {
+  if (lhs == rhs || lhs < rhs) {
+      return true;
+  } else {
+      return false;
+  }
+}
+
+template <typename T>
+__host__ __device__ inline bool operator>(const complex<T>& lhs,
+                                          const complex<T>& rhs) {
+  if (lhs == rhs) {
+      return false;
+  } else if (lhs.real() > rhs.real()) {
+      return true;
+  } else if (lhs.real() == rhs.real()) {
+      return lhs.imag() > rhs.imag();
+  } else {
+      return false;
+  }
+}
+
+template <typename T>
+__host__ __device__ inline bool operator>=(const complex<T>& lhs,
+                                           const complex<T>& rhs) {
+  if (lhs == rhs || lhs > rhs) {
+      return true;
+  } else {
+      return false;
+  }
+}
+
+template <typename T>
+__host__ __device__ inline bool operator<(const T& lhs,
+                                          const complex<T>& rhs) {
+    return complex<T>(lhs) < rhs;
+}
+
+template <typename T>
+__host__ __device__ inline bool operator>(const T& lhs,
+                                          const complex<T>& rhs) {
+    return complex<T>(lhs) > rhs;
+}
+
+template <typename T>
+__host__ __device__ inline bool operator<(const complex<T>& lhs,
+                                          const T& rhs) {
+    return lhs < complex<T>(rhs);
+}
+
+template <typename T>
+__host__ __device__ inline bool operator>(const complex<T>& lhs,
+                                          const T& rhs) {
+    return lhs > complex<T>(rhs);
+}
+
+template <typename T>
+__host__ __device__ inline bool operator<=(const T& lhs,
+                                           const complex<T>& rhs) {
+    return complex<T>(lhs) <= rhs;
+}
+
+template <typename T>
+__host__ __device__ inline bool operator>=(const T& lhs,
+                                           const complex<T>& rhs) {
+    return complex<T>(lhs) >= rhs;
+}
+
+template <typename T>
+__host__ __device__ inline bool operator<=(const complex<T>& lhs,
+                                           const T& rhs) {
+    return lhs <= complex<T>(rhs);
+}
+
+template <typename T>
+__host__ __device__ inline bool operator>=(const complex<T>& lhs,
+                                           const T& rhs) {
+    return lhs >= complex<T>(rhs);
+}
+
+/* --- Unary Arithmetic Operators --- */
+
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const complex<T>& rhs) {
+  return rhs;
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> operator-(const complex<T>& rhs) {
+  return rhs * -T(1);
+}
+
+/* --- Other Basic Arithmetic Functions --- */
+
+// As hypot is only C++11 we have to use the C interface
+template <typename T>
+__host__ __device__ inline T abs(const complex<T>& z) {
+  return hypot(z.real(), z.imag());
+}
+
+namespace detail {
+namespace complex {
+__host__ __device__ inline float abs(const thrust::complex<float>& z) {
+  return hypotf(z.real(), z.imag());
+}
+
+__host__ __device__ inline double abs(const thrust::complex<double>& z) {
+  return hypot(z.real(), z.imag());
+}
+}
+}
+
+template <>
+__host__ __device__ inline float abs(const complex<float>& z) {
+  return detail::complex::abs(z);
+}
+template <>
+__host__ __device__ inline double abs(const complex<double>& z) {
+  return detail::complex::abs(z);
+}
+
+template <typename T>
+__host__ __device__ inline T arg(const complex<T>& z) {
+  return atan2(z.imag(), z.real());
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> conj(const complex<T>& z) {
+  return complex<T>(z.real(), -z.imag());
+}
+
+template <typename T>
+__host__ __device__ inline T real(const complex<T>& z) {
+  return z.real();
+}
+
+template <typename T>
+__host__ __device__ inline T imag(const complex<T>& z) {
+  return z.imag();
+}
+
+template <typename T>
+__host__ __device__ inline T norm(const complex<T>& z) {
+  return z.real() * z.real() + z.imag() * z.imag();
+}
+
+template <>
+__host__ __device__ inline float norm(const complex<float>& z) {
+  if (::abs(z.real()) < ::sqrtf(FLT_MIN) && ::abs(z.imag()) < ::sqrtf(FLT_MIN)) {
+    float a = z.real() * 4.0f;
+    float b = z.imag() * 4.0f;
+    return (a * a + b * b) / 16.0f;
+  }
+  return z.real() * z.real() + z.imag() * z.imag();
+}
+
+template <>
+__host__ __device__ inline double norm(const complex<double>& z) {
+  if (::abs(z.real()) < ::sqrt(DBL_MIN) && ::abs(z.imag()) < ::sqrt(DBL_MIN)) {
+    double a = z.real() * 4.0;
+    double b = z.imag() * 4.0;
+    return (a * a + b * b) / 16.0;
+  }
+  return z.real() * z.real() + z.imag() * z.imag();
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> polar(const T& m,
+                                           const T& theta) {
+  return complex<T>(m * cos(theta), m * sin(theta));
+}
+}
diff --git a/cupy/_core/include/cupy/complex/catrig.h b/cupy/_core/include/cupy/complex/catrig.h
new file mode 100644
index 0000000..2d29d78
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/catrig.h
@@ -0,0 +1,730 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2012 Stephen Montgomery-Smith <stephen@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Adapted from FreeBSD by Filipe Maia <filipe.c.maia@gmail.com>:
+ *    freebsd/lib/msun/src/catrig.c
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline void raise_inexact() {
+  const volatile float tiny = 7.888609052210118054117286e-31; /* 0x1p-100; */
+  // needs the volatile to prevent compiler from ignoring it
+  volatile float junk = 1 + tiny;
+  (void)junk;
+}
+
+__host__ __device__ inline complex<double> clog_for_large_values(complex<double> z);
+
+/*
+ * Testing indicates that all these functions are accurate up to 4 ULP.
+ * The functions casin(h) and cacos(h) are about 2.5 times slower than asinh.
+ * The functions catan(h) are a little under 2 times slower than atanh.
+ *
+ * The code for casinh, casin, cacos, and cacosh comes first.  The code is
+ * rather complicated, and the four functions are highly interdependent.
+ *
+ * The code for catanh and catan comes at the end.  It is much simpler than
+ * the other functions, and the code for these can be disconnected from the
+ * rest of the code.
+ */
+
+/*
+ *			================================
+ *			| casinh, casin, cacos, cacosh |
+ *			================================
+ */
+
+/*
+ * The algorithm is very close to that in "Implementing the complex arcsine
+ * and arccosine functions using exception handling" by T. E. Hull, Thomas F.
+ * Fairgrieve, and Ping Tak Peter Tang, published in ACM Transactions on
+ * Mathematical Software, Volume 23 Issue 3, 1997, Pages 299-335,
+ * http://dl.acm.org/citation.cfm?id=275324.
+ *
+ * Throughout we use the convention z = x + I*y.
+ *
+ * casinh(z) = sign(x)*log(A+sqrt(A*A-1)) + I*asin(B)
+ * where
+ * A = (|z+I| + |z-I|) / 2
+ * B = (|z+I| - |z-I|) / 2 = y/A
+ *
+ * These formulas become numerically unstable:
+ *   (a) for Re(casinh(z)) when z is close to the line segment [-I, I] (that
+ *       is, Re(casinh(z)) is close to 0);
+ *   (b) for Im(casinh(z)) when z is close to either of the intervals
+ *       [I, I*infinity) or (-I*infinity, -I] (that is, |Im(casinh(z))| is
+ *       close to PI/2).
+ *
+ * These numerical problems are overcome by defining
+ * f(a, b) = (hypot(a, b) - b) / 2 = a*a / (hypot(a, b) + b) / 2
+ * Then if A < A_crossover, we use
+ *   log(A + sqrt(A*A-1)) = log1p((A-1) + sqrt((A-1)*(A+1)))
+ *   A-1 = f(x, 1+y) + f(x, 1-y)
+ * and if B > B_crossover, we use
+ *   asin(B) = atan2(y, sqrt(A*A - y*y)) = atan2(y, sqrt((A+y)*(A-y)))
+ *   A-y = f(x, y+1) + f(x, y-1)
+ * where without loss of generality we have assumed that x and y are
+ * non-negative.
+ *
+ * Much of the difficulty comes because the intermediate computations may
+ * produce overflows or underflows.  This is dealt with in the paper by Hull
+ * et al by using exception handling.  We do this by detecting when
+ * computations risk underflow or overflow.  The hardest part is handling the
+ * underflows when computing f(a, b).
+ *
+ * Note that the function f(a, b) does not appear explicitly in the paper by
+ * Hull et al, but the idea may be found on pages 308 and 309.  Introducing the
+ * function f(a, b) allows us to concentrate many of the clever tricks in this
+ * paper into one function.
+ */
+
+/*
+ * Function f(a, b, hypot_a_b) = (hypot(a, b) - b) / 2.
+ * Pass hypot(a, b) as the third argument.
+ */
+__host__ __device__ inline double f(double a, double b, double hypot_a_b) {
+  if (b < 0) return ((hypot_a_b - b) / 2);
+  if (b == 0) return (a / 2);
+  return (a * a / (hypot_a_b + b) / 2);
+}
+
+/*
+ * All the hard work is contained in this function.
+ * x and y are assumed positive or zero, and less than RECIP_EPSILON.
+ * Upon return:
+ * rx = Re(casinh(z)) = -Im(cacos(y + I*x)).
+ * B_is_usable is set to 1 if the value of B is usable.
+ * If B_is_usable is set to 0, sqrt_A2my2 = sqrt(A*A - y*y), and new_y = y.
+ * If returning sqrt_A2my2 has potential to result in an underflow, it is
+ * rescaled, and new_y is similarly rescaled.
+ */
+__host__ __device__ inline void do_hard_work(double x, double y, double* rx,
+                                    int* B_is_usable, double* B,
+                                    double* sqrt_A2my2, double* new_y) {
+  double R, S, A;  /* A, B, R, and S are as in Hull et al. */
+  double Am1, Amy; /* A-1, A-y. */
+  const double A_crossover =
+      10; /* Hull et al suggest 1.5, but 10 works better */
+  const double FOUR_SQRT_MIN =
+      5.966672584960165394632772e-154; /* =0x1p-509; >= 4 * sqrt(DBL_MIN) */
+  const double B_crossover = 0.6417;   /* suggested by Hull et al */
+
+  R = hypot(x, y + 1); /* |z+I| */
+  S = hypot(x, y - 1); /* |z-I| */
+
+  /* A = (|z+I| + |z-I|) / 2 */
+  A = (R + S) / 2;
+  /*
+   * Mathematically A >= 1.  There is a small chance that this will not
+   * be so because of rounding errors.  So we will make certain it is
+   * so.
+   */
+  if (A < 1) A = 1;
+
+  if (A < A_crossover) {
+    /*
+     * Am1 = fp + fm, where fp = f(x, 1+y), and fm = f(x, 1-y).
+     * rx = log1p(Am1 + sqrt(Am1*(A+1)))
+     */
+    if (y == 1 && x < DBL_EPSILON * DBL_EPSILON / 128) {
+      /*
+       * fp is of order x^2, and fm = x/2.
+       * A = 1 (inexactly).
+       */
+      *rx = sqrt(x);
+    } else if (x >= DBL_EPSILON * fabs(y - 1)) {
+      /*
+       * Underflow will not occur because
+       * x >= DBL_EPSILON^2/128 >= FOUR_SQRT_MIN
+       */
+      Am1 = f(x, 1 + y, R) + f(x, 1 - y, S);
+      *rx = log1p(Am1 + sqrt(Am1 * (A + 1)));
+    } else if (y < 1) {
+      /*
+       * fp = x*x/(1+y)/4, fm = x*x/(1-y)/4, and
+       * A = 1 (inexactly).
+       */
+      *rx = x / sqrt((1 - y) * (1 + y));
+    } else { /* if (y > 1) */
+             /*
+              * A-1 = y-1 (inexactly).
+              */
+      *rx = log1p((y - 1) + sqrt((y - 1) * (y + 1)));
+    }
+  } else {
+    *rx = log(A + sqrt(A * A - 1));
+  }
+
+  *new_y = y;
+
+  if (y < FOUR_SQRT_MIN) {
+    /*
+     * Avoid a possible underflow caused by y/A.  For casinh this
+     * would be legitimate, but will be picked up by invoking atan2
+     * later on.  For cacos this would not be legitimate.
+     */
+    *B_is_usable = 0;
+    *sqrt_A2my2 = A * (2 / DBL_EPSILON);
+    *new_y = y * (2 / DBL_EPSILON);
+    return;
+  }
+
+  /* B = (|z+I| - |z-I|) / 2 = y/A */
+  *B = y / A;
+  *B_is_usable = 1;
+
+  if (*B > B_crossover) {
+    *B_is_usable = 0;
+    /*
+     * Amy = fp + fm, where fp = f(x, y+1), and fm = f(x, y-1).
+     * sqrt_A2my2 = sqrt(Amy*(A+y))
+     */
+    if (y == 1 && x < DBL_EPSILON / 128) {
+      /*
+       * fp is of order x^2, and fm = x/2.
+       * A = 1 (inexactly).
+       */
+      *sqrt_A2my2 = sqrt(x) * sqrt((A + y) / 2);
+    } else if (x >= DBL_EPSILON * fabs(y - 1)) {
+      /*
+       * Underflow will not occur because
+       * x >= DBL_EPSILON/128 >= FOUR_SQRT_MIN
+       * and
+       * x >= DBL_EPSILON^2 >= FOUR_SQRT_MIN
+       */
+      Amy = f(x, y + 1, R) + f(x, y - 1, S);
+      *sqrt_A2my2 = sqrt(Amy * (A + y));
+    } else if (y > 1) {
+      /*
+       * fp = x*x/(y+1)/4, fm = x*x/(y-1)/4, and
+       * A = y (inexactly).
+       *
+       * y < RECIP_EPSILON.  So the following
+       * scaling should avoid any underflow problems.
+       */
+      *sqrt_A2my2 =
+          x * (4 / DBL_EPSILON / DBL_EPSILON) * y / sqrt((y + 1) * (y - 1));
+      *new_y = y * (4 / DBL_EPSILON / DBL_EPSILON);
+    } else { /* if (y < 1) */
+             /*
+              * fm = 1-y >= DBL_EPSILON, fp is of order x^2, and
+              * A = 1 (inexactly).
+              */
+      *sqrt_A2my2 = sqrt((1 - y) * (1 + y));
+    }
+  }
+}
+
+/*
+ * casinh(z) = z + O(z^3)   as z -> 0
+ *
+ * casinh(z) = sign(x)*clog(sign(x)*z) + O(1/z^2)   as z -> infinity
+ * The above formula works for the imaginary part as well, because
+ * Im(casinh(z)) = sign(x)*atan2(sign(x)*y, fabs(x)) + O(y/z^3)
+ *    as z -> infinity, uniformly in y
+ */
+__host__ __device__ inline complex<double> casinh(complex<double> z) {
+  double x, y, ax, ay, rx, ry, B, sqrt_A2my2, new_y;
+  int B_is_usable;
+  complex<double> w;
+  const double RECIP_EPSILON = 1.0 / DBL_EPSILON;
+  const double m_ln2 = 6.9314718055994531e-1; /*  0x162e42fefa39ef.0p-53 */
+  x = z.real();
+  y = z.imag();
+  ax = fabs(x);
+  ay = fabs(y);
+
+  if (isnan(x) || isnan(y)) {
+    /* casinh(+-Inf + I*NaN) = +-Inf + I*NaN */
+    if (isinf(x)) return (complex<double>(x, y + y));
+    /* casinh(NaN + I*+-Inf) = opt(+-)Inf + I*NaN */
+    if (isinf(y)) return (complex<double>(y, x + x));
+    /* casinh(NaN + I*0) = NaN + I*0 */
+    if (y == 0) return (complex<double>(x + x, y));
+    /*
+     * All other cases involving NaN return NaN + I*NaN.
+     * C99 leaves it optional whether to raise invalid if one of
+     * the arguments is not NaN, so we opt not to raise it.
+     */
+    return (complex<double>(x + 0.0 + (y + 0.0), x + 0.0 + (y + 0.0)));
+  }
+
+  if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) {
+    /* clog...() will raise inexact unless x or y is infinite. */
+    if (signbit(x) == 0)
+      w = clog_for_large_values(z) + m_ln2;
+    else
+      w = clog_for_large_values(-z) + m_ln2;
+    return (complex<double>(copysign(w.real(), x), copysign(w.imag(), y)));
+  }
+
+  /* Avoid spuriously raising inexact for z = 0. */
+  if (x == 0 && y == 0) return (z);
+
+  /* All remaining cases are inexact. */
+  raise_inexact();
+
+  const double SQRT_6_EPSILON =
+      3.6500241499888571e-8; /*  0x13988e1409212e.0p-77 */
+  if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) return (z);
+
+  do_hard_work(ax, ay, &rx, &B_is_usable, &B, &sqrt_A2my2, &new_y);
+  if (B_is_usable)
+    ry = asin(B);
+  else
+    ry = atan2(new_y, sqrt_A2my2);
+  return (complex<double>(copysign(rx, x), copysign(ry, y)));
+}
+
+/*
+ * casin(z) = reverse(casinh(reverse(z)))
+ * where reverse(x + I*y) = y + I*x = I*conj(z).
+ */
+__host__ __device__ inline complex<double> casin(complex<double> z) {
+  complex<double> w = casinh(complex<double>(z.imag(), z.real()));
+
+  return (complex<double>(w.imag(), w.real()));
+}
+
+/*
+ * cacos(z) = PI/2 - casin(z)
+ * but do the computation carefully so cacos(z) is accurate when z is
+ * close to 1.
+ *
+ * cacos(z) = PI/2 - z + O(z^3)   as z -> 0
+ *
+ * cacos(z) = -sign(y)*I*clog(z) + O(1/z^2)   as z -> infinity
+ * The above formula works for the real part as well, because
+ * Re(cacos(z)) = atan2(fabs(y), x) + O(y/z^3)
+ *    as z -> infinity, uniformly in y
+ */
+__host__ __device__ inline complex<double> cacos(complex<double> z) {
+  double x, y, ax, ay, rx, ry, B, sqrt_A2mx2, new_x;
+  int sx, sy;
+  int B_is_usable;
+  complex<double> w;
+  const double pio2_hi = 1.5707963267948966e0; /*  0x1921fb54442d18.0p-52 */
+  const volatile double pio2_lo =
+      6.1232339957367659e-17;                 /*  0x11a62633145c07.0p-106 */
+  const double m_ln2 = 6.9314718055994531e-1; /*  0x162e42fefa39ef.0p-53 */
+
+  x = z.real();
+  y = z.imag();
+  sx = signbit(x);
+  sy = signbit(y);
+  ax = fabs(x);
+  ay = fabs(y);
+
+  if (isnan(x) || isnan(y)) {
+    /* cacos(+-Inf + I*NaN) = NaN + I*opt(-)Inf */
+    if (isinf(x)) return (complex<double>(y + y, -infinity<double>()));
+    /* cacos(NaN + I*+-Inf) = NaN + I*-+Inf */
+    if (isinf(y)) return (complex<double>(x + x, -y));
+    /* cacos(0 + I*NaN) = PI/2 + I*NaN with inexact */
+    if (x == 0) return (complex<double>(pio2_hi + pio2_lo, y + y));
+    /*
+     * All other cases involving NaN return NaN + I*NaN.
+     * C99 leaves it optional whether to raise invalid if one of
+     * the arguments is not NaN, so we opt not to raise it.
+     */
+    return (complex<double>(x + 0.0 + (y + 0), x + 0.0 + (y + 0)));
+  }
+
+  const double RECIP_EPSILON = 1.0 / DBL_EPSILON;
+  if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) {
+    /* clog...() will raise inexact unless x or y is infinite. */
+    w = clog_for_large_values(z);
+    rx = fabs(w.imag());
+    ry = w.real() + m_ln2;
+    if (sy == 0) ry = -ry;
+    return (complex<double>(rx, ry));
+  }
+
+  /* Avoid spuriously raising inexact for z = 1. */
+  if (x == 1.0 && y == 0.0) return (complex<double>(0, -y));
+
+  /* All remaining cases are inexact. */
+  raise_inexact();
+
+  const double SQRT_6_EPSILON =
+      3.6500241499888571e-8; /*  0x13988e1409212e.0p-77 */
+  if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4)
+    return (complex<double>(pio2_hi - (x - pio2_lo), -y));
+
+  do_hard_work(ay, ax, &ry, &B_is_usable, &B, &sqrt_A2mx2, &new_x);
+  if (B_is_usable) {
+    if (sx == 0)
+      rx = acos(B);
+    else
+      rx = acos(-B);
+  } else {
+    if (sx == 0)
+      rx = atan2(sqrt_A2mx2, new_x);
+    else
+      rx = atan2(sqrt_A2mx2, -new_x);
+  }
+  if (sy == 0) ry = -ry;
+  return (complex<double>(rx, ry));
+}
+
+/*
+ * cacosh(z) = I*cacos(z) or -I*cacos(z)
+ * where the sign is chosen so Re(cacosh(z)) >= 0.
+ */
+__host__ __device__ inline complex<double> cacosh(complex<double> z) {
+  complex<double> w;
+  double rx, ry;
+
+  w = cacos(z);
+  rx = w.real();
+  ry = w.imag();
+  /* cacosh(NaN + I*NaN) = NaN + I*NaN */
+  if (isnan(rx) && isnan(ry)) return (complex<double>(ry, rx));
+  /* cacosh(NaN + I*+-Inf) = +Inf + I*NaN */
+  /* cacosh(+-Inf + I*NaN) = +Inf + I*NaN */
+  if (isnan(rx)) return (complex<double>(fabs(ry), rx));
+  /* cacosh(0 + I*NaN) = NaN + I*NaN */
+  if (isnan(ry)) return (complex<double>(ry, ry));
+  return (complex<double>(fabs(ry), copysign(rx, z.imag())));
+}
+
+/*
+ * Optimized version of clog() for |z| finite and larger than ~RECIP_EPSILON.
+ */
+__host__ __device__ inline complex<double> clog_for_large_values(complex<double> z) {
+  double x, y;
+  double ax, ay, t;
+  const double m_e = 2.7182818284590452e0; /*  0x15bf0a8b145769.0p-51 */
+
+  x = z.real();
+  y = z.imag();
+  ax = fabs(x);
+  ay = fabs(y);
+  if (ax < ay) {
+    t = ax;
+    ax = ay;
+    ay = t;
+  }
+
+  /*
+   * Avoid overflow in hypot() when x and y are both very large.
+   * Divide x and y by E, and then add 1 to the logarithm.  This depends
+   * on E being larger than sqrt(2).
+   * Dividing by E causes an insignificant loss of accuracy; however
+   * this method is still poor since it is uneccessarily slow.
+   */
+  if (ax > DBL_MAX / 2)
+    return (complex<double>(log(hypot(x / m_e, y / m_e)) + 1, atan2(y, x)));
+
+  /*
+   * Avoid overflow when x or y is large.  Avoid underflow when x or
+   * y is small.
+   */
+  const double QUARTER_SQRT_MAX =
+      5.966672584960165394632772e-154; /* = 0x1p509; <= sqrt(DBL_MAX) / 4 */
+  const double SQRT_MIN =
+      1.491668146240041348658193e-154; /* = 0x1p-511; >= sqrt(DBL_MIN) */
+  if (ax > QUARTER_SQRT_MAX || ay < SQRT_MIN)
+    return (complex<double>(log(hypot(x, y)), atan2(y, x)));
+
+  return (complex<double>(log(ax * ax + ay * ay) / 2, atan2(y, x)));
+}
+
+/*
+ *				=================
+ *				| catanh, catan |
+ *				=================
+ */
+
+/*
+   * sum_squares(x,y) = x*x + y*y (or just x*x if y*y would underflow).
+   * Assumes x*x and y*y will not overflow.
+   * Assumes x and y are finite.
+   * Assumes y is non-negative.
+   * Assumes fabs(x) >= DBL_EPSILON.
+   */
+__host__ __device__ inline double sum_squares(double x, double y) {
+  const double SQRT_MIN =
+      1.491668146240041348658193e-154; /* = 0x1p-511; >= sqrt(DBL_MIN) */
+  /* Avoid underflow when y is small. */
+  if (y < SQRT_MIN) return (x * x);
+
+  return (x * x + y * y);
+}
+
+/*
+ * real_part_reciprocal(x, y) = Re(1/(x+I*y)) = x/(x*x + y*y).
+ * Assumes x and y are not NaN, and one of x and y is larger than
+ * RECIP_EPSILON.  We avoid unwarranted underflow.  It is important to not use
+ * the code creal(1/z), because the imaginary part may produce an unwanted
+ * underflow.
+ * This is only called in a context where inexact is always raised before
+ * the call, so no effort is made to avoid or force inexact.
+ */
+__host__ __device__ inline double real_part_reciprocal(double x, double y) {
+  double scale;
+  uint32_t hx, hy;
+  int32_t ix, iy;
+
+  /*
+   * This code is inspired by the C99 document n1124.pdf, Section G.5.1,
+   * example 2.
+   */
+  get_high_word(hx, x);
+  ix = hx & 0x7ff00000;
+  get_high_word(hy, y);
+  iy = hy & 0x7ff00000;
+  //#define	BIAS	(DBL_MAX_EXP - 1)
+  const int BIAS = DBL_MAX_EXP - 1;
+  /* XXX more guard digits are useful iff there is extra precision. */
+  //#define	CUTOFF	(DBL_MANT_DIG / 2 + 1)	/* just half or 1 guard digit */
+  const int CUTOFF = (DBL_MANT_DIG / 2 + 1);
+  if (ix - iy >= CUTOFF << 20 || isinf(x))
+    return (1 / x); /* +-Inf -> +-0 is special */
+  if (iy - ix >= CUTOFF << 20)
+    return (x / y / y); /* should avoid double div, but hard */
+  if (ix <= (BIAS + DBL_MAX_EXP / 2 - CUTOFF) << 20)
+    return (x / (x * x + y * y));
+  scale = 1;
+  set_high_word(scale, 0x7ff00000 - ix); /* 2**(1-ilogb(x)) */
+  x *= scale;
+  y *= scale;
+  return (x / (x * x + y * y) * scale);
+}
+
+/*
+ * catanh(z) = log((1+z)/(1-z)) / 2
+ *           = log1p(4*x / |z-1|^2) / 4
+ *             + I * atan2(2*y, (1-x)*(1+x)-y*y) / 2
+ *
+ * catanh(z) = z + O(z^3)   as z -> 0
+ *
+ * catanh(z) = 1/z + sign(y)*I*PI/2 + O(1/z^3)   as z -> infinity
+ * The above formula works for the real part as well, because
+ * Re(catanh(z)) = x/|z|^2 + O(x/z^4)
+ *    as z -> infinity, uniformly in x
+ */
+#if __cplusplus >= 201103L || !defined _MSC_VER
+__host__ __device__ inline complex<double> catanh(complex<double> z) {
+  double x, y, ax, ay, rx, ry;
+  const volatile double pio2_lo =
+      6.1232339957367659e-17;                  /*  0x11a62633145c07.0p-106 */
+  const double pio2_hi = 1.5707963267948966e0; /*  0x1921fb54442d18.0p-52 */
+
+  x = z.real();
+  y = z.imag();
+  ax = fabs(x);
+  ay = fabs(y);
+
+  /* This helps handle many cases. */
+  if (y == 0 && ax <= 1) return (complex<double>(atanh(x), y));
+
+  /* To ensure the same accuracy as atan(), and to filter out z = 0. */
+  if (x == 0) return (complex<double>(x, atan(y)));
+
+  if (isnan(x) || isnan(y)) {
+    /* catanh(+-Inf + I*NaN) = +-0 + I*NaN */
+    if (isinf(x)) return (complex<double>(copysign(0.0, x), y + y));
+    /* catanh(NaN + I*+-Inf) = sign(NaN)0 + I*+-PI/2 */
+    if (isinf(y))
+      return (
+          complex<double>(copysign(0.0, x), copysign(pio2_hi + pio2_lo, y)));
+    /*
+     * All other cases involving NaN return NaN + I*NaN.
+     * C99 leaves it optional whether to raise invalid if one of
+     * the arguments is not NaN, so we opt not to raise it.
+     */
+    return (complex<double>(x + 0.0 + (y + 0), x + 0.0 + (y + 0)));
+  }
+
+  const double RECIP_EPSILON = 1.0 / DBL_EPSILON;
+  if (ax > RECIP_EPSILON || ay > RECIP_EPSILON)
+    return (complex<double>(real_part_reciprocal(x, y),
+                            copysign(pio2_hi + pio2_lo, y)));
+
+  const double SQRT_3_EPSILON =
+      2.5809568279517849e-8; /*  0x1bb67ae8584caa.0p-78 */
+  if (ax < SQRT_3_EPSILON / 2 && ay < SQRT_3_EPSILON / 2) {
+    /*
+     * z = 0 was filtered out above.  All other cases must raise
+     * inexact, but this is the only only that needs to do it
+     * explicitly.
+     */
+    raise_inexact();
+    return (z);
+  }
+
+  const double m_ln2 = 6.9314718055994531e-1; /*  0x162e42fefa39ef.0p-53 */
+  if (ax == 1 && ay < DBL_EPSILON)
+    rx = (m_ln2 - log(ay)) / 2;
+  else
+    rx = log1p(4 * ax / sum_squares(ax - 1, ay)) / 4;
+
+  if (ax == 1)
+    ry = atan2(2.0, -ay) / 2;
+  else if (ay < DBL_EPSILON)
+    ry = atan2(2 * ay, (1 - ax) * (1 + ax)) / 2;
+  else
+    ry = atan2(2 * ay, (1 - ax) * (1 + ax) - ay * ay) / 2;
+
+  return (complex<double>(copysign(rx, x), copysign(ry, y)));
+}
+
+/*
+ * catan(z) = reverse(catanh(reverse(z)))
+ * where reverse(x + I*y) = y + I*x = I*conj(z).
+ */
+__host__ __device__ inline complex<double> catan(complex<double> z) {
+  complex<double> w = catanh(complex<double>(z.imag(), z.real()));
+  return (complex<double>(w.imag(), w.real()));
+}
+
+#endif
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> acos(const complex<ValueType>& z) {
+  const complex<ValueType> ret = thrust::asin(z);
+  const ValueType pi = ValueType(3.14159265358979323846);
+  return complex<ValueType>(pi / 2 - ret.real(), -ret.imag());
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> asin(const complex<ValueType>& z) {
+  const complex<ValueType> i(0, 1);
+  return -i * asinh(i * z);
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> atan(const complex<ValueType>& z) {
+  const complex<ValueType> i(0, 1);
+  return -i * thrust::atanh(i * z);
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> acosh(const complex<ValueType>& z) {
+  thrust::complex<ValueType> ret(
+      (z.real() - z.imag()) * (z.real() + z.imag()) - ValueType(1.0),
+      ValueType(2.0) * z.real() * z.imag());
+  ret = thrust::sqrt(ret);
+  if (z.real() < ValueType(0.0)) {
+    ret = -ret;
+  }
+  ret += z;
+  ret = thrust::log(ret);
+  if (ret.real() < ValueType(0.0)) {
+    ret = -ret;
+  }
+  return ret;
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> asinh(const complex<ValueType>& z) {
+  return thrust::log(thrust::sqrt(z * z + ValueType(1)) + z);
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> atanh(const complex<ValueType>& z) {
+  ValueType imag2 = z.imag() * z.imag();
+  ValueType n = ValueType(1.0) + z.real();
+  n = imag2 + n * n;
+
+  ValueType d = ValueType(1.0) - z.real();
+  d = imag2 + d * d;
+  complex<ValueType> ret(ValueType(0.25) * (::log(n) - ::log(d)), 0);
+
+  d = ValueType(1.0) - z.real() * z.real() - imag2;
+
+  ret.imag(ValueType(0.5) * ::atan2(ValueType(2.0) * z.imag(), d));
+  return ret;
+}
+
+template <>
+__host__ __device__ inline complex<double> acos(const complex<double>& z) {
+  return detail::complex::cacos(z);
+}
+
+template <>
+__host__ __device__ inline complex<double> asin(const complex<double>& z) {
+  return detail::complex::casin(z);
+}
+
+#if __cplusplus >= 201103L || !defined _MSC_VER
+template <>
+__host__ __device__ inline complex<double> atan(const complex<double>& z) {
+  return detail::complex::catan(z);
+}
+#endif
+
+template <>
+__host__ __device__ inline complex<double> acosh(const complex<double>& z) {
+  return detail::complex::cacosh(z);
+}
+
+template <>
+__host__ __device__ inline complex<double> asinh(const complex<double>& z) {
+  return detail::complex::casinh(z);
+}
+
+#if __cplusplus >= 201103L || !defined _MSC_VER
+template <>
+__host__ __device__ inline complex<double> atanh(const complex<double>& z) {
+  return detail::complex::catanh(z);
+}
+#endif
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/catrigf.h b/cupy/_core/include/cupy/complex/catrigf.h
new file mode 100644
index 0000000..410d06f
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/catrigf.h
@@ -0,0 +1,444 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2012 Stephen Montgomery-Smith <stephen@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Adapted from FreeBSD by Filipe Maia <filipe.c.maia@gmail.com>:
+ *    freebsd/lib/msun/src/catrig.c
+ */
+
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<float> clog_for_large_values(complex<float> z);
+
+/*
+ * The algorithm is very close to that in "Implementing the complex arcsine
+ * and arccosine functions using exception handling" by T. E. Hull, Thomas F.
+ * Fairgrieve, and Ping Tak Peter Tang, published in ACM Transactions on
+ * Mathematical Software, Volume 23 Issue 3, 1997, Pages 299-335,
+ * http://dl.acm.org/citation.cfm?id=275324.
+ *
+ * See catrig.c for complete comments.
+ *
+ * XXX comments were removed automatically, and even short ones on the right
+ * of statements were removed (all of them), contrary to normal style.  Only
+ * a few comments on the right of declarations remain.
+ */
+
+__host__ __device__ inline float f(float a, float b, float hypot_a_b) {
+  if (b < 0.0f) return ((hypot_a_b - b) / 2.0f);
+  if (b == 0.0f) return (a / 2.0f);
+  return (a * a / (hypot_a_b + b) / 2.0f);
+}
+
+/*
+ * All the hard work is contained in this function.
+ * x and y are assumed positive or zero, and less than RECIP_EPSILON.
+ * Upon return:
+ * rx = Re(casinh(z)) = -Im(cacos(y + I*x)).
+ * B_is_usable is set to 1 if the value of B is usable.
+ * If B_is_usable is set to 0, sqrt_A2my2 = sqrt(A*A - y*y), and new_y = y.
+ * If returning sqrt_A2my2 has potential to result in an underflow, it is
+ * rescaled, and new_y is similarly rescaled.
+ */
+__host__ __device__ inline void do_hard_work(float x, float y, float* rx,
+                                             int* B_is_usable, float* B,
+                                             float* sqrt_A2my2, float* new_y) {
+  float R, S, A;  /* A, B, R, and S are as in Hull et al. */
+  float Am1, Amy; /* A-1, A-y. */
+  const float A_crossover =
+      10; /* Hull et al suggest 1.5, but 10 works better */
+  const float FOUR_SQRT_MIN = 4.336808689942017736029811e-19f;
+  ;                                  /* =0x1p-61; >= 4 * sqrt(FLT_MIN) */
+  const float B_crossover = 0.6417f; /* suggested by Hull et al */
+  R = hypotf(x, y + 1);
+  S = hypotf(x, y - 1);
+
+  A = (R + S) / 2;
+  if (A < 1) A = 1;
+
+  if (A < A_crossover) {
+    if (y == 1 && x < FLT_EPSILON * FLT_EPSILON / 128) {
+      *rx = sqrtf(x);
+    } else if (x >= FLT_EPSILON * fabsf(y - 1)) {
+      Am1 = f(x, 1 + y, R) + f(x, 1 - y, S);
+      *rx = log1pf(Am1 + sqrtf(Am1 * (A + 1)));
+    } else if (y < 1) {
+      *rx = x / sqrtf((1 - y) * (1 + y));
+    } else {
+      *rx = log1pf((y - 1) + sqrtf((y - 1) * (y + 1)));
+    }
+  } else {
+    *rx = logf(A + sqrtf(A * A - 1));
+  }
+
+  *new_y = y;
+
+  if (y < FOUR_SQRT_MIN) {
+    *B_is_usable = 0;
+    *sqrt_A2my2 = A * (2 / FLT_EPSILON);
+    *new_y = y * (2 / FLT_EPSILON);
+    return;
+  }
+
+  *B = y / A;
+  *B_is_usable = 1;
+
+  if (*B > B_crossover) {
+    *B_is_usable = 0;
+    if (y == 1 && x < FLT_EPSILON / 128) {
+      *sqrt_A2my2 = sqrtf(x) * sqrtf((A + y) / 2);
+    } else if (x >= FLT_EPSILON * fabsf(y - 1)) {
+      Amy = f(x, y + 1, R) + f(x, y - 1, S);
+      *sqrt_A2my2 = sqrtf(Amy * (A + y));
+    } else if (y > 1) {
+      *sqrt_A2my2 =
+          x * (4 / FLT_EPSILON / FLT_EPSILON) * y / sqrtf((y + 1) * (y - 1));
+      *new_y = y * (4 / FLT_EPSILON / FLT_EPSILON);
+    } else {
+      *sqrt_A2my2 = sqrtf((1 - y) * (1 + y));
+    }
+  }
+}
+
+__host__ __device__ inline complex<float> casinhf(complex<float> z) {
+  float x, y, ax, ay, rx, ry, B, sqrt_A2my2, new_y;
+  int B_is_usable;
+  complex<float> w;
+  const float RECIP_EPSILON = 1.0 / FLT_EPSILON;
+  const float m_ln2 = 6.9314718055994531e-1f; /*  0x162e42fefa39ef.0p-53 */
+  x = z.real();
+  y = z.imag();
+  ax = fabsf(x);
+  ay = fabsf(y);
+
+  if (isnan(x) || isnan(y)) {
+    if (isinf(x)) return (complex<float>(x, y + y));
+    if (isinf(y)) return (complex<float>(y, x + x));
+    if (y == 0) return (complex<float>(x + x, y));
+    return (complex<float>(x + 0.0f + (y + 0), x + 0.0f + (y + 0)));
+  }
+
+  if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) {
+    if (signbit(x) == 0)
+      w = clog_for_large_values(z) + m_ln2;
+    else
+      w = clog_for_large_values(-z) + m_ln2;
+    return (complex<float>(copysignf(w.real(), x), copysignf(w.imag(), y)));
+  }
+
+  if (x == 0 && y == 0) return (z);
+
+  raise_inexact();
+
+  const float SQRT_6_EPSILON = 8.4572793338e-4f; /*  0xddb3d7.0p-34 */
+  if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) return (z);
+
+  do_hard_work(ax, ay, &rx, &B_is_usable, &B, &sqrt_A2my2, &new_y);
+  if (B_is_usable)
+    ry = asinf(B);
+  else
+    ry = atan2f(new_y, sqrt_A2my2);
+  return (complex<float>(copysignf(rx, x), copysignf(ry, y)));
+}
+
+__host__ __device__ inline complex<float> casinf(complex<float> z) {
+  complex<float> w = casinhf(complex<float>(z.imag(), z.real()));
+
+  return (complex<float>(w.imag(), w.real()));
+}
+
+__host__ __device__ inline complex<float> cacosf(complex<float> z) {
+  float x, y, ax, ay, rx, ry, B, sqrt_A2mx2, new_x;
+  int sx, sy;
+  int B_is_usable;
+  complex<float> w;
+  const float pio2_hi = 1.5707963267948966e0f; /*  0x1921fb54442d18.0p-52 */
+  const volatile float pio2_lo =
+      6.1232339957367659e-17f;                /*  0x11a62633145c07.0p-106 */
+  const float m_ln2 = 6.9314718055994531e-1f; /*  0x162e42fefa39ef.0p-53 */
+
+  x = z.real();
+  y = z.imag();
+  sx = signbit(x);
+  sy = signbit(y);
+  ax = fabsf(x);
+  ay = fabsf(y);
+
+  if (isnan(x) || isnan(y)) {
+    if (isinf(x)) return (complex<float>(y + y, -infinity<float>()));
+    if (isinf(y)) return (complex<float>(x + x, -y));
+    if (x == 0) return (complex<float>(pio2_hi + pio2_lo, y + y));
+    return (complex<float>(x + 0.0f + (y + 0), x + 0.0f + (y + 0)));
+  }
+
+  const float RECIP_EPSILON = 1.0 / FLT_EPSILON;
+  if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) {
+    w = clog_for_large_values(z);
+    rx = fabsf(w.imag());
+    ry = w.real() + m_ln2;
+    if (sy == 0) ry = -ry;
+    return (complex<float>(rx, ry));
+  }
+
+  if (x == 1 && y == 0) return (complex<float>(0, -y));
+
+  raise_inexact();
+
+  const float SQRT_6_EPSILON = 8.4572793338e-4f; /*  0xddb3d7.0p-34 */
+  if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4)
+    return (complex<float>(pio2_hi - (x - pio2_lo), -y));
+
+  do_hard_work(ay, ax, &ry, &B_is_usable, &B, &sqrt_A2mx2, &new_x);
+  if (B_is_usable) {
+    if (sx == 0)
+      rx = acosf(B);
+    else
+      rx = acosf(-B);
+  } else {
+    if (sx == 0)
+      rx = atan2f(sqrt_A2mx2, new_x);
+    else
+      rx = atan2f(sqrt_A2mx2, -new_x);
+  }
+  if (sy == 0) ry = -ry;
+  return (complex<float>(rx, ry));
+}
+
+__host__ __device__ inline complex<float> cacoshf(complex<float> z) {
+  complex<float> w;
+  float rx, ry;
+
+  w = cacosf(z);
+  rx = w.real();
+  ry = w.imag();
+  /* cacosh(NaN + I*NaN) = NaN + I*NaN */
+  if (isnan(rx) && isnan(ry)) return (complex<float>(ry, rx));
+  /* cacosh(NaN + I*+-Inf) = +Inf + I*NaN */
+  /* cacosh(+-Inf + I*NaN) = +Inf + I*NaN */
+  if (isnan(rx)) return (complex<float>(fabsf(ry), rx));
+  /* cacosh(0 + I*NaN) = NaN + I*NaN */
+  if (isnan(ry)) return (complex<float>(ry, ry));
+  return (complex<float>(fabsf(ry), copysignf(rx, z.imag())));
+}
+
+/*
+ * Optimized version of clog() for |z| finite and larger than ~RECIP_EPSILON.
+ */
+__host__ __device__ inline complex<float> clog_for_large_values(complex<float> z) {
+  float x, y;
+  float ax, ay, t;
+  const float m_e = 2.7182818284590452e0f; /*  0x15bf0a8b145769.0p-51 */
+
+  x = z.real();
+  y = z.imag();
+  ax = fabsf(x);
+  ay = fabsf(y);
+  if (ax < ay) {
+    t = ax;
+    ax = ay;
+    ay = t;
+  }
+
+  if (ax > FLT_MAX / 2)
+    return (complex<float>(logf(hypotf(x / m_e, y / m_e)) + 1, atan2f(y, x)));
+
+  const float QUARTER_SQRT_MAX =
+      2.3058430092136939520000000e+18f; /* = 0x1p61; <= sqrt(FLT_MAX) / 4 */
+  const float SQRT_MIN =
+      1.084202172485504434007453e-19f; /* 0x1p-63; >= sqrt(FLT_MIN) */
+  if (ax > QUARTER_SQRT_MAX || ay < SQRT_MIN)
+    return (complex<float>(logf(hypotf(x, y)), atan2f(y, x)));
+
+  return (complex<float>(logf(ax * ax + ay * ay) / 2, atan2f(y, x)));
+}
+
+/*
+ *				=================
+ *				| catanh, catan |
+ *				=================
+ */
+
+/*
+ * sum_squares(x,y) = x*x + y*y (or just x*x if y*y would underflow).
+ * Assumes x*x and y*y will not overflow.
+ * Assumes x and y are finite.
+ * Assumes y is non-negative.
+ * Assumes fabsf(x) >= FLT_EPSILON.
+ */
+__host__ __device__ inline float sum_squares(float x, float y) {
+  const float SQRT_MIN =
+      1.084202172485504434007453e-19f; /* 0x1p-63; >= sqrt(FLT_MIN) */
+  /* Avoid underflow when y is small. */
+  if (y < SQRT_MIN) return (x * x);
+
+  return (x * x + y * y);
+}
+
+__host__ __device__ inline float real_part_reciprocal(float x, float y) {
+  float scale;
+  uint32_t hx, hy;
+  int32_t ix, iy;
+
+  get_float_word(hx, x);
+  ix = hx & 0x7f800000;
+  get_float_word(hy, y);
+  iy = hy & 0x7f800000;
+  //#define	BIAS	(FLT_MAX_EXP - 1)
+  const int BIAS = FLT_MAX_EXP - 1;
+  //#define	CUTOFF	(FLT_MANT_DIG / 2 + 1)
+  const int CUTOFF = (FLT_MANT_DIG / 2 + 1);
+  if (ix - iy >= CUTOFF << 23 || isinf(x)) return (1 / x);
+  if (iy - ix >= CUTOFF << 23) return (x / y / y);
+  if (ix <= (BIAS + FLT_MAX_EXP / 2 - CUTOFF) << 23)
+    return (x / (x * x + y * y));
+  set_float_word(scale, 0x7f800000 - ix);
+  x *= scale;
+  y *= scale;
+  return (x / (x * x + y * y) * scale);
+}
+
+#if __cplusplus >= 201103L || !defined _MSC_VER
+__host__ __device__ inline complex<float> catanhf(complex<float> z) {
+  float x, y, ax, ay, rx, ry;
+  const volatile float pio2_lo =
+      6.1232339957367659e-17;                 /*  0x11a62633145c07.0p-106 */
+  const float pio2_hi = 1.5707963267948966e0; /*  0x1921fb54442d18.0p-52 */
+
+  x = z.real();
+  y = z.imag();
+  ax = fabsf(x);
+  ay = fabsf(y);
+
+  if (y == 0 && ax <= 1) return (complex<float>(atanhf(x), y));
+
+  if (x == 0) return (complex<float>(x, atanf(y)));
+
+  if (isnan(x) || isnan(y)) {
+    if (isinf(x)) return (complex<float>(copysignf(0, x), y + y));
+    if (isinf(y))
+      return (complex<float>(copysignf(0, x), copysignf(pio2_hi + pio2_lo, y)));
+    return (complex<float>(x + 0.0f + (y + 0.0f), x + 0.0f + (y + 0.0f)));
+  }
+
+  const float RECIP_EPSILON = 1.0f / FLT_EPSILON;
+  if (ax > RECIP_EPSILON || ay > RECIP_EPSILON)
+    return (complex<float>(real_part_reciprocal(x, y),
+                           copysignf(pio2_hi + pio2_lo, y)));
+
+  const float SQRT_3_EPSILON = 5.9801995673e-4; /*  0x9cc471.0p-34 */
+  if (ax < SQRT_3_EPSILON / 2 && ay < SQRT_3_EPSILON / 2) {
+    raise_inexact();
+    return (z);
+  }
+
+  const float m_ln2 = 6.9314718056e-1f; /*  0xb17218.0p-24 */
+  if (ax == 1 && ay < FLT_EPSILON)
+    rx = (m_ln2 - logf(ay)) / 2;
+  else
+    rx = log1pf(4 * ax / sum_squares(ax - 1, ay)) / 4;
+
+  if (ax == 1)
+    ry = atan2f(2, -ay) / 2;
+  else if (ay < FLT_EPSILON)
+    ry = atan2f(2 * ay, (1 - ax) * (1 + ax)) / 2;
+  else
+    ry = atan2f(2 * ay, (1 - ax) * (1 + ax) - ay * ay) / 2;
+
+  return (complex<float>(copysignf(rx, x), copysignf(ry, y)));
+}
+
+__host__ __device__ inline complex<float> catanf(complex<float> z) {
+  complex<float> w = catanhf(complex<float>(z.imag(), z.real()));
+  return (complex<float>(w.imag(), w.real()));
+}
+#endif
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <>
+__host__ __device__ inline complex<float> acos(const complex<float>& z) {
+  return detail::complex::cacosf(z);
+}
+
+template <>
+__host__ __device__ inline complex<float> asin(const complex<float>& z) {
+  return detail::complex::casinf(z);
+}
+
+#if __cplusplus >= 201103L || !defined _MSC_VER
+template <>
+__host__ __device__ inline complex<float> atan(const complex<float>& z) {
+  return detail::complex::catanf(z);
+}
+#endif
+
+template <>
+__host__ __device__ inline complex<float> acosh(const complex<float>& z) {
+  return detail::complex::cacoshf(z);
+}
+
+template <>
+__host__ __device__ inline complex<float> asinh(const complex<float>& z) {
+  return detail::complex::casinhf(z);
+}
+
+#if __cplusplus >= 201103L || !defined _MSC_VER
+template <>
+__host__ __device__ inline complex<float> atanh(const complex<float>& z) {
+  return detail::complex::catanhf(z);
+}
+#endif
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/ccosh.h b/cupy/_core/include/cupy/complex/ccosh.h
new file mode 100644
index 0000000..93af077
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/ccosh.h
@@ -0,0 +1,205 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2005 Bruce D. Evans and Steven G. Kargl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* adapted from FreeBSD:
+ *    lib/msun/src/s_ccosh.c
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+/*
+ * Hyperbolic cosine of a complex argument z = x + i y.
+ *
+ * cosh(z) = cosh(x+iy)
+ *         = cosh(x) cos(y) + i sinh(x) sin(y).
+ *
+ * Exceptional values are noted in the comments within the source code.
+ * These values and the return value were taken from n1124.pdf.
+ */
+
+__host__ __device__ inline thrust::complex<double> ccosh(
+    const thrust::complex<double>& z) {
+  const double huge = 8.98846567431157953864652595395e+307;  // 0x1p1023
+  double x, y, h;
+  uint32_t hx, hy, ix, iy, lx, ly;
+
+  x = z.real();
+  y = z.imag();
+
+  extract_words(hx, lx, x);
+  extract_words(hy, ly, y);
+
+  ix = 0x7fffffff & hx;
+  iy = 0x7fffffff & hy;
+
+  /* Handle the nearly-non-exceptional cases where x and y are finite. */
+  if (ix < 0x7ff00000 && iy < 0x7ff00000) {
+    if ((iy | ly) == 0) return (thrust::complex<double>(::cosh(x), x * y));
+    if (ix < 0x40360000) /* small x: normal case */
+      return (
+          thrust::complex<double>(::cosh(x) * ::cos(y), ::sinh(x) * ::sin(y)));
+
+    /* |x| >= 22, so cosh(x) ~= exp(|x|) */
+    if (ix < 0x40862e42) {
+      /* x < 710: exp(|x|) won't overflow */
+      h = ::exp(::fabs(x)) * 0.5;
+      return (thrust::complex<double>(h * cos(y), copysign(h, x) * sin(y)));
+    } else if (ix < 0x4096bbaa) {
+      /* x < 1455: scale to avoid overflow */
+      thrust::complex<double> z_;
+      z_ = ldexp_cexp(thrust::complex<double>(fabs(x), y), -1);
+      return (thrust::complex<double>(z_.real(), z_.imag() * copysign(1.0, x)));
+    } else {
+      /* x >= 1455: the result always overflows */
+      h = huge * x;
+      return (thrust::complex<double>(h * h * cos(y), h * sin(y)));
+    }
+  }
+
+  /*
+   * cosh(+-0 +- I Inf) = dNaN + I sign(d(+-0, dNaN))0.
+   * The sign of 0 in the result is unspecified.  Choice = normally
+   * the same as dNaN.  Raise the invalid floating-point exception.
+   *
+   * cosh(+-0 +- I NaN) = d(NaN) + I sign(d(+-0, NaN))0.
+   * The sign of 0 in the result is unspecified.  Choice = normally
+   * the same as d(NaN).
+   */
+  if ((ix | lx) == 0 && iy >= 0x7ff00000)
+    return (thrust::complex<double>(y - y, copysign(0.0, x * (y - y))));
+
+  /*
+   * cosh(+-Inf +- I 0) = +Inf + I (+-)(+-)0.
+   *
+   * cosh(NaN +- I 0)   = d(NaN) + I sign(d(NaN, +-0))0.
+   * The sign of 0 in the result is unspecified.
+   */
+  if ((iy | ly) == 0 && ix >= 0x7ff00000) {
+    if (((hx & 0xfffff) | lx) == 0)
+      return (thrust::complex<double>(x * x, copysign(0.0, x) * y));
+    return (thrust::complex<double>(x * x, copysign(0.0, (x + x) * y)));
+  }
+
+  /*
+   * cosh(x +- I Inf) = dNaN + I dNaN.
+   * Raise the invalid floating-point exception for finite nonzero x.
+   *
+   * cosh(x + I NaN) = d(NaN) + I d(NaN).
+   * Optionally raises the invalid floating-point exception for finite
+   * nonzero x.  Choice = don't raise (except for signaling NaNs).
+   */
+  if (ix < 0x7ff00000 && iy >= 0x7ff00000)
+    return (thrust::complex<double>(y - y, x * (y - y)));
+
+  /*
+   * cosh(+-Inf + I NaN)  = +Inf + I d(NaN).
+   *
+   * cosh(+-Inf +- I Inf) = +Inf + I dNaN.
+   * The sign of Inf in the result is unspecified.  Choice = always +.
+   * Raise the invalid floating-point exception.
+   *
+   * cosh(+-Inf + I y)   = +Inf cos(y) +- I Inf sin(y)
+   */
+  if (ix >= 0x7ff00000 && ((hx & 0xfffff) | lx) == 0) {
+    if (iy >= 0x7ff00000) return (thrust::complex<double>(x * x, x * (y - y)));
+    return (thrust::complex<double>((x * x) * cos(y), x * sin(y)));
+  }
+
+  /*
+   * cosh(NaN + I NaN)  = d(NaN) + I d(NaN).
+   *
+   * cosh(NaN +- I Inf) = d(NaN) + I d(NaN).
+   * Optionally raises the invalid floating-point exception.
+   * Choice = raise.
+   *
+   * cosh(NaN + I y)    = d(NaN) + I d(NaN).
+   * Optionally raises the invalid floating-point exception for finite
+   * nonzero y.  Choice = don't raise (except for signaling NaNs).
+   */
+  return (thrust::complex<double>((x * x) * (y - y), (x + x) * (y - y)));
+}
+
+__host__ __device__ inline thrust::complex<double> ccos(
+    const thrust::complex<double>& z) {
+  /* ccos(z) = ccosh(I * z) */
+  return (ccosh(thrust::complex<double>(-z.imag(), z.real())));
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> cos(const complex<ValueType>& z) {
+  const ValueType re = z.real();
+  const ValueType im = z.imag();
+  return complex<ValueType>(::cos(re) * ::cosh(im), -::sin(re) * ::sinh(im));
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> cosh(const complex<ValueType>& z) {
+  const ValueType re = z.real();
+  const ValueType im = z.imag();
+  return complex<ValueType>(::cosh(re) * ::cos(im), ::sinh(re) * ::sin(im));
+}
+
+template <>
+__host__ __device__ inline thrust::complex<double> cos(
+    const thrust::complex<double>& z) {
+  return detail::complex::ccos(z);
+}
+
+template <>
+__host__ __device__ inline thrust::complex<double> cosh(
+    const thrust::complex<double>& z) {
+  return detail::complex::ccosh(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/ccoshf.h b/cupy/_core/include/cupy/complex/ccoshf.h
new file mode 100644
index 0000000..744abab
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/ccoshf.h
@@ -0,0 +1,135 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2005 Bruce D. Evans and Steven G. Kargl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* adapted from FreeBSD:
+ *    lib/msun/src/s_ccoshf.c
+ */
+
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<float> ccoshf(const complex<float>& z) {
+  float x, y, h;
+  uint32_t hx, hy, ix, iy;
+  const float huge = 1.70141183460469231731687303716e+38;  // 0x1p127;
+
+  x = z.real();
+  y = z.imag();
+
+  get_float_word(hx, x);
+  get_float_word(hy, y);
+
+  ix = 0x7fffffff & hx;
+  iy = 0x7fffffff & hy;
+  if (ix < 0x7f800000 && iy < 0x7f800000) {
+    if (iy == 0) {
+      return (complex<float>(coshf(x), x * y));
+    }
+    if (ix < 0x41100000) { /* small x: normal case */
+      return (complex<float>(coshf(x) * cosf(y), sinhf(x) * sinf(y)));
+    }
+    /* |x| >= 9, so cosh(x) ~= exp(|x|) */
+    if (ix < 0x42b17218) {
+      /* x < 88.7: expf(|x|) won't overflow */
+      h = expf(fabsf(x)) * 0.5f;
+      return (complex<float>(h * cosf(y), copysignf(h, x) * sinf(y)));
+    } else if (ix < 0x4340b1e7) {
+      /* x < 192.7: scale to avoid overflow */
+      thrust::complex<float> z_;
+      z_ = ldexp_cexpf(complex<float>(fabsf(x), y), -1);
+      return (complex<float>(z_.real(), z_.imag() * copysignf(1.0f, x)));
+    } else {
+      /* x >= 192.7: the result always overflows */
+      h = huge * x;
+      return (complex<float>(h * h * cosf(y), h * sinf(y)));
+    }
+  }
+
+  if (ix == 0 && iy >= 0x7f800000) {
+    return (complex<float>(y - y, copysignf(0.0f, x * (y - y))));
+  }
+  if (iy == 0 && ix >= 0x7f800000) {
+    if ((hx & 0x7fffff) == 0)
+      return (complex<float>(x * x, copysignf(0.0f, x) * y));
+    return (complex<float>(x * x, copysignf(0.0f, (x + x) * y)));
+  }
+
+  if (ix < 0x7f800000 && iy >= 0x7f800000) {
+    return (complex<float>(y - y, x * (y - y)));
+  }
+
+  if (ix >= 0x7f800000 && (hx & 0x7fffff) == 0) {
+    if (iy >= 0x7f800000) return (complex<float>(x * x, x * (y - y)));
+    return (complex<float>((x * x) * cosf(y), x * sinf(y)));
+  }
+  return (complex<float>((x * x) * (y - y), (x + x) * (y - y)));
+}
+
+__host__ __device__ inline complex<float> ccosf(const complex<float>& z) {
+  return (ccoshf(complex<float>(-z.imag(), z.real())));
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <>
+__host__ __device__ inline complex<float> cos(const complex<float>& z) {
+  return detail::complex::ccosf(z);
+}
+
+template <>
+__host__ __device__ inline complex<float> cosh(const complex<float>& z) {
+  return detail::complex::ccoshf(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/cexp.h b/cupy/_core/include/cupy/complex/cexp.h
new file mode 100644
index 0000000..3bf1c79
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/cexp.h
@@ -0,0 +1,173 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2011 David Schultz <das@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* adapted from FreeBSD:
+ *    lib/msun/src/s_cexp.c
+ *    lib/msun/src/k_exp.c
+ *
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+/*
+ * Compute exp(x), scaled to avoid spurious overflow.  An exponent is
+ * returned separately in 'expt'.
+ *
+ * Input:  ln(DBL_MAX) <= x < ln(2 * DBL_MAX / DBL_MIN_DENORM) ~= 1454.91
+ * Output: 2**1023 <= y < 2**1024
+ */
+__host__ __device__ inline double frexp_exp(double x, int* expt) {
+  const uint32_t k = 1799;                    /* constant for reduction */
+  const double kln2 = 1246.97177782734161156; /* k * ln2 */
+
+  double exp_x;
+  uint32_t hx;
+
+  /*
+   * We use exp(x) = exp(x - kln2) * 2**k, carefully chosen to
+   * minimize |exp(kln2) - 2**k|.  We also scale the exponent of
+   * exp_x to MAX_EXP so that the result can be multiplied by
+   * a tiny number without losing accuracy due to denormalization.
+   */
+  exp_x = exp(x - kln2);
+  get_high_word(hx, exp_x);
+  *expt = (hx >> 20) - (0x3ff + 1023) + k;
+  set_high_word(exp_x, (hx & 0xfffff) | ((0x3ff + 1023) << 20));
+  return (exp_x);
+}
+
+__host__ __device__ inline complex<double> ldexp_cexp(complex<double> z, int expt) {
+  double x, y, exp_x, scale1, scale2;
+  int ex_expt, half_expt;
+
+  x = z.real();
+  y = z.imag();
+  exp_x = frexp_exp(x, &ex_expt);
+  expt += ex_expt;
+
+  /*
+   * Arrange so that scale1 * scale2 == 2**expt.  We use this to
+   * compensate for scalbn being horrendously slow.
+   */
+  half_expt = expt / 2;
+  insert_words(scale1, (0x3ff + half_expt) << 20, 0);
+  half_expt = expt - half_expt;
+  insert_words(scale2, (0x3ff + half_expt) << 20, 0);
+
+  return (complex<double>(::cos(y) * exp_x * scale1 * scale2,
+                          ::sin(y) * exp_x * scale1 * scale2));
+}
+
+__host__ __device__ inline complex<double> cexp(const complex<double>& z) {
+  double x, y, exp_x;
+  uint32_t hx, hy, lx, ly;
+
+  const uint32_t exp_ovfl = 0x40862e42, /* high bits of MAX_EXP * ln2 ~= 710 */
+      cexp_ovfl = 0x4096b8e4;           /* (MAX_EXP - MIN_DENORM_EXP) * ln2 */
+
+  x = z.real();
+  y = z.imag();
+
+  extract_words(hy, ly, y);
+  hy &= 0x7fffffff;
+
+  /* cexp(x + I 0) = exp(x) + I 0 */
+  if ((hy | ly) == 0) return (complex<double>(exp(x), y));
+  extract_words(hx, lx, x);
+  /* cexp(0 + I y) = cos(y) + I sin(y) */
+  if (((hx & 0x7fffffff) | lx) == 0) return (complex<double>(cos(y), sin(y)));
+
+  if (hy >= 0x7ff00000) {
+    if (lx != 0 || (hx & 0x7fffffff) != 0x7ff00000) {
+      /* cexp(finite|NaN +- I Inf|NaN) = NaN + I NaN */
+      return (complex<double>(y - y, y - y));
+    } else if (hx & 0x80000000) {
+      /* cexp(-Inf +- I Inf|NaN) = 0 + I 0 */
+      return (complex<double>(0.0, 0.0));
+    } else {
+      /* cexp(+Inf +- I Inf|NaN) = Inf + I NaN */
+      return (complex<double>(x, y - y));
+    }
+  }
+
+  if (hx >= exp_ovfl && hx <= cexp_ovfl) {
+    /*
+     * x is between 709.7 and 1454.3, so we must scale to avoid
+     * overflow in exp(x).
+     */
+    return (ldexp_cexp(z, 0));
+  } else {
+    /*
+     * Cases covered here:
+     *  -  x < exp_ovfl and exp(x) won't overflow (common case)
+     *  -  x > cexp_ovfl, so exp(x) * s overflows for all s > 0
+     *  -  x = +-Inf (generated by exp())
+     *  -  x = NaN (spurious inexact exception from y)
+     */
+    exp_x = ::exp(x);
+    return (complex<double>(exp_x * cos(y), exp_x * sin(y)));
+  }
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> exp(const complex<ValueType>& z) {
+  return polar(::exp(z.real()), z.imag());
+}
+
+template <>
+__host__ __device__ inline complex<double> exp(const complex<double>& z) {
+  return detail::complex::cexp(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/cexpf.h b/cupy/_core/include/cupy/complex/cexpf.h
new file mode 100644
index 0000000..e3f63c8
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/cexpf.h
@@ -0,0 +1,153 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2011 David Schultz <das@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* adapted from FreeBSD:
+ *    lib/msun/src/s_cexpf.c
+ *    lib/msun/src/k_exp.c
+ *
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+__host__ __device__ inline float frexp_expf(float x, int* expt) {
+  const uint32_t k = 235;           /* constant for reduction */
+  const float kln2 = 162.88958740F; /* k * ln2 */
+
+  // should this be a double instead?
+  float exp_x;
+  uint32_t hx;
+
+  exp_x = expf(x - kln2);
+  get_float_word(hx, exp_x);
+  *expt = (hx >> 23) - (0x7f + 127) + k;
+  set_float_word(exp_x, (hx & 0x7fffff) | ((0x7f + 127) << 23));
+  return (exp_x);
+}
+
+__host__ __device__ inline complex<float> ldexp_cexpf(complex<float> z, int expt) {
+  float x, y, exp_x, scale1, scale2;
+  int ex_expt, half_expt;
+
+  x = z.real();
+  y = z.imag();
+  exp_x = frexp_expf(x, &ex_expt);
+  expt += ex_expt;
+
+  half_expt = expt / 2;
+  set_float_word(scale1, (0x7f + half_expt) << 23);
+  half_expt = expt - half_expt;
+  set_float_word(scale2, (0x7f + half_expt) << 23);
+
+  return (complex<float>(cos(y) * exp_x * scale1 * scale2,
+                         sin(y) * exp_x * scale1 * scale2));
+}
+
+__host__ __device__ inline complex<float> cexpf(const complex<float>& z) {
+  float x, y, exp_x;
+  uint32_t hx, hy;
+
+  const uint32_t exp_ovfl = 0x42b17218, /* MAX_EXP * ln2 ~= 88.722839355 */
+      cexp_ovfl = 0x43400074;           /* (MAX_EXP - MIN_DENORM_EXP) * ln2 */
+
+  x = z.real();
+  y = z.imag();
+
+  get_float_word(hy, y);
+  hy &= 0x7fffffff;
+
+  /* cexp(x + I 0) = exp(x) + I 0 */
+  if (hy == 0) return (complex<float>(exp(x), y));
+  get_float_word(hx, x);
+  /* cexp(0 + I y) = cos(y) + I sin(y) */
+  if ((hx & 0x7fffffff) == 0) {
+    return (complex<float>(cos(y), sin(y)));
+  }
+  if (hy >= 0x7f800000) {
+    if ((hx & 0x7fffffff) != 0x7f800000) {
+      /* cexp(finite|NaN +- I Inf|NaN) = NaN + I NaN */
+      return (complex<float>(y - y, y - y));
+    } else if (hx & 0x80000000) {
+      /* cexp(-Inf +- I Inf|NaN) = 0 + I 0 */
+      return (complex<float>(0.0, 0.0));
+    } else {
+      /* cexp(+Inf +- I Inf|NaN) = Inf + I NaN */
+      return (complex<float>(x, y - y));
+    }
+  }
+
+  if (hx >= exp_ovfl && hx <= cexp_ovfl) {
+    /*
+     * x is between 88.7 and 192, so we must scale to avoid
+     * overflow in expf(x).
+     */
+    return (ldexp_cexpf(z, 0));
+  } else {
+    /*
+     * Cases covered here:
+     *  -  x < exp_ovfl and exp(x) won't overflow (common case)
+     *  -  x > cexp_ovfl, so exp(x) * s overflows for all s > 0
+     *  -  x = +-Inf (generated by exp())
+     *  -  x = NaN (spurious inexact exception from y)
+     */
+    exp_x = ::exp(x);
+    return (complex<float>(exp_x * ::cos(y), exp_x * ::sin(y)));
+  }
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <>
+__host__ __device__ inline complex<float> exp(const complex<float>& z) {
+  return detail::complex::cexpf(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/clog.h b/cupy/_core/include/cupy/complex/clog.h
new file mode 100644
index 0000000..96444d8
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/clog.h
@@ -0,0 +1,203 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2012 Stephen Montgomery-Smith <stephen@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* adapted from FreeBSDs msun:*/
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+/* round down to 18 = 54/3 bits */
+__host__ __device__ inline double trim(double x) {
+  uint32_t hi;
+  get_high_word(hi, x);
+  insert_words(x, hi & 0xfffffff8, 0);
+  return x;
+}
+
+__host__ __device__ inline complex<double> clog(const complex<double>& z) {
+  // Adapted from FreeBSDs msun
+  double x, y;
+  double ax, ay;
+  double x0, y0, x1, y1, x2, y2, t, hm1;
+  double val[12];
+  int i, sorted;
+  const double e = 2.7182818284590452354;
+
+  x = z.real();
+  y = z.imag();
+
+  /* Handle NaNs using the general formula to mix them right. */
+  if (x != x || y != y) {
+    return (complex<double>(::log(norm(z)), ::atan2(y, x)));
+  }
+
+  ax = ::abs(x);
+  ay = ::abs(y);
+  if (ax < ay) {
+    t = ax;
+    ax = ay;
+    ay = t;
+  }
+
+  /*
+   * To avoid unnecessary overflow, if x and y are very large, divide x
+   * and y by M_E, and then add 1 to the logarithm.  This depends on
+   * M_E being larger than sqrt(2).
+   * There is a potential loss of accuracy caused by dividing by M_E,
+   * but this case should happen extremely rarely.
+   */
+  //    if (ay > 5e307){
+  // For high values of ay -> hypotf(DBL_MAX,ay) = inf
+  // We expect that for values at or below ay = 5e307 this should not happen
+  if (ay > 5e307) {
+    return (complex<double>(::log(hypot(x / e, y / e)) + 1.0, ::atan2(y, x)));
+  }
+  if (ax == 1.) {
+    if (ay < 1e-150) {
+      return (complex<double>((ay * 0.5) * ay, ::atan2(y, x)));
+    }
+    return (complex<double>(log1p(ay * ay) * 0.5, ::atan2(y, x)));
+  }
+
+  /*
+   * Because atan2 and hypot conform to C99, this also covers all the
+   * edge cases when x or y are 0 or infinite.
+   */
+  if (ax < 1e-50 || ay < 1e-50 || ax > 1e50 || ay > 1e50) {
+    return (complex<double>(::log(hypot(x, y)), ::atan2(y, x)));
+  }
+
+  /*
+   * From this point on, we don't need to worry about underflow or
+   * overflow in calculating ax*ax or ay*ay.
+   */
+
+  /* Some easy cases. */
+
+  if (ax >= 1.0) {
+    return (complex<double>(log1p((ax - 1) * (ax + 1) + ay * ay) * 0.5,
+                            atan2(y, x)));
+  }
+
+  if (ax * ax + ay * ay <= 0.7) {
+    return (complex<double>(::log(ax * ax + ay * ay) * 0.5, ::atan2(y, x)));
+  }
+
+  /*
+   * Take extra care so that ULP of real part is small if hypot(x,y) is
+   * moderately close to 1.
+   */
+
+  x0 = trim(ax);
+  ax = ax - x0;
+  x1 = trim(ax);
+  x2 = ax - x1;
+  y0 = trim(ay);
+  ay = ay - y0;
+  y1 = trim(ay);
+  y2 = ay - y1;
+
+  val[0] = x0 * x0;
+  val[1] = y0 * y0;
+  val[2] = 2 * x0 * x1;
+  val[3] = 2 * y0 * y1;
+  val[4] = x1 * x1;
+  val[5] = y1 * y1;
+  val[6] = 2 * x0 * x2;
+  val[7] = 2 * y0 * y2;
+  val[8] = 2 * x1 * x2;
+  val[9] = 2 * y1 * y2;
+  val[10] = x2 * x2;
+  val[11] = y2 * y2;
+
+  /* Bubble sort. */
+
+  do {
+    sorted = 1;
+    for (i = 0; i < 11; i++) {
+      if (val[i] < val[i + 1]) {
+        sorted = 0;
+        t = val[i];
+        val[i] = val[i + 1];
+        val[i + 1] = t;
+      }
+    }
+  } while (!sorted);
+
+  hm1 = -1;
+  for (i = 0; i < 12; i++) {
+    hm1 += val[i];
+  }
+  return (complex<double>(0.5 * log1p(hm1), atan2(y, x)));
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> log(const complex<ValueType>& z) {
+  return complex<ValueType>(::log(thrust::abs(z)), thrust::arg(z));
+}
+
+template <>
+__host__ __device__ inline complex<double> log(const complex<double>& z) {
+  return detail::complex::clog(z);
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> log10(const complex<ValueType>& z) {
+  // Using the explicit literal prevents compile time warnings in
+  // devices that don't support doubles
+  return thrust::log(z) / ValueType(2.30258509299404568402);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/clogf.h b/cupy/_core/include/cupy/complex/clogf.h
new file mode 100644
index 0000000..4084c2b
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/clogf.h
@@ -0,0 +1,192 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2012 Stephen Montgomery-Smith <stephen@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* adapted from FreeBSDs msun:*/
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+/* round down to 8 = 24/3 bits */
+__host__ __device__ inline float trim(float x) {
+  uint32_t hx;
+  get_float_word(hx, x);
+  hx &= 0xffff0000;
+  float ret;
+  set_float_word(ret, hx);
+  return ret;
+}
+
+__host__ __device__ inline complex<float> clogf(const complex<float>& z) {
+  // Adapted from FreeBSDs msun
+  float x, y;
+  float ax, ay;
+  float x0, y0, x1, y1, x2, y2, t, hm1;
+  float val[12];
+  int i, sorted;
+  const float e = 2.7182818284590452354f;
+
+  x = z.real();
+  y = z.imag();
+
+  /* Handle NaNs using the general formula to mix them right. */
+  if (x != x || y != y) {
+    return (complex<float>(::log(norm(z)), ::atan2(y, x)));
+  }
+
+  ax = ::abs(x);
+  ay = ::abs(y);
+  if (ax < ay) {
+    t = ax;
+    ax = ay;
+    ay = t;
+  }
+
+  /*
+   * To avoid unnecessary overflow, if x and y are very large, divide x
+   * and y by M_E, and then add 1 to the logarithm.  This depends on
+   * M_E being larger than sqrt(2).
+   * There is a potential loss of accuracy caused by dividing by M_E,
+   * but this case should happen extremely rarely.
+   */
+  // For high values of ay -> hypotf(FLT_MAX,ay) = inf
+  // We expect that for values at or below ay = 1e34f this should not happen
+  if (ay > 1e34f) {
+    return (complex<float>(::log(hypotf(x / e, y / e)) + 1.0f, ::atan2(y, x)));
+  }
+  if (ax == 1.f) {
+    if (ay < 1e-19f) {
+      return (complex<float>((ay * 0.5f) * ay, ::atan2(y, x)));
+    }
+    return (complex<float>(log1pf(ay * ay) * 0.5f, ::atan2(y, x)));
+  }
+
+  /*
+   * Because atan2 and hypot conform to C99, this also covers all the
+   * edge cases when x or y are 0 or infinite.
+   */
+  if (ax < 1e-6f || ay < 1e-6f || ax > 1e6f || ay > 1e6f) {
+    return (complex<float>(::log(hypotf(x, y)), ::atan2(y, x)));
+  }
+
+  /*
+   * From this point on, we don't need to worry about underflow or
+   * overflow in calculating ax*ax or ay*ay.
+   */
+
+  /* Some easy cases. */
+
+  if (ax >= 1.0f) {
+    return (complex<float>(log1pf((ax - 1.f) * (ax + 1.f) + ay * ay) * 0.5f,
+                           atan2(y, x)));
+  }
+
+  if (ax * ax + ay * ay <= 0.7f) {
+    return (complex<float>(::log(ax * ax + ay * ay) * 0.5f, ::atan2(y, x)));
+  }
+
+  /*
+   * Take extra care so that ULP of real part is small if hypot(x,y) is
+   * moderately close to 1.
+   */
+
+  x0 = trim(ax);
+  ax = ax - x0;
+  x1 = trim(ax);
+  x2 = ax - x1;
+  y0 = trim(ay);
+  ay = ay - y0;
+  y1 = trim(ay);
+  y2 = ay - y1;
+
+  val[0] = x0 * x0;
+  val[1] = y0 * y0;
+  val[2] = 2 * x0 * x1;
+  val[3] = 2 * y0 * y1;
+  val[4] = x1 * x1;
+  val[5] = y1 * y1;
+  val[6] = 2 * x0 * x2;
+  val[7] = 2 * y0 * y2;
+  val[8] = 2 * x1 * x2;
+  val[9] = 2 * y1 * y2;
+  val[10] = x2 * x2;
+  val[11] = y2 * y2;
+
+  /* Bubble sort. */
+
+  do {
+    sorted = 1;
+    for (i = 0; i < 11; i++) {
+      if (val[i] < val[i + 1]) {
+        sorted = 0;
+        t = val[i];
+        val[i] = val[i + 1];
+        val[i + 1] = t;
+      }
+    }
+  } while (!sorted);
+
+  hm1 = -1;
+  for (i = 0; i < 12; i++) {
+    hm1 += val[i];
+  }
+  return (complex<float>(0.5f * log1pf(hm1), atan2(y, x)));
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <>
+__host__ __device__ inline complex<float> log(const complex<float>& z) {
+  return detail::complex::clogf(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/complex.h b/cupy/_core/include/cupy/complex/complex.h
new file mode 100644
index 0000000..dc5f5f1
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/complex.h
@@ -0,0 +1,674 @@
+/*  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*! \file complex.h
+ *  \brief Complex numbers
+ */
+
+#pragma once
+
+namespace thrust {
+
+template <typename T, typename U, bool x>
+struct _select_greater_type_impl {
+  typedef T type;
+};
+
+template <typename T, typename U>
+struct _select_greater_type_impl<T, U, false> {
+  typedef U type;
+};
+
+template <typename T, typename U>
+struct _select_greater_type
+    : _select_greater_type_impl<T, U, (sizeof(T) > sizeof(U))> {};
+
+/*
+ *  Calls to the standard math library from inside the thrust namespace
+ *  with real arguments require explicit scope otherwise they will fail
+ *  to resolve as it will find the equivalent complex function but then
+ *  fail to match the template, and give up looking for other scopes.
+ */
+
+/*! \addtogroup numerics
+ *  \{
+ */
+
+/*! \addtogroup complex_numbers Complex Numbers
+ *  \{
+ */
+
+/*! \p complex is the Thrust equivalent to <tt>std::complex</tt>. It is
+ * functionally
+ *  equivalent to it, but can also be used in device code which
+ * <tt>std::complex</tt> currently cannot.
+ *
+ *  \tparam T The type used to hold the real and imaginary parts. Should be
+ * <tt>float</tt>
+ *  or <tt>double</tt>. Others types are not supported.
+ *
+ */
+template <typename T>
+#if defined(__CUDACC__)
+struct __align__(sizeof(T)*2) complex {
+#else
+// ROCm (hipcc) does not support `__align__`
+struct complex {
+#endif
+ public:
+  /*! \p value_type is the type of \p complex's real and imaginary parts.
+   */
+  typedef T value_type;
+
+  /* --- Constructors --- */
+
+  /*! Construct a complex number with an imaginary part of 0.
+   *
+   *  \param re The real part of the number.
+   */
+  inline __host__ __device__ complex(const T& re);
+
+  /*! Construct a complex number from its real and imaginary parts.
+   *
+   *  \param re The real part of the number.
+   *  \param im The imaginary part of the number.
+   */
+  inline __host__ __device__ complex(const T& re, const T& im);
+
+#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1900)
+  /*! Default construct a complex number.
+   */
+  inline complex() = default;
+
+  /*! This copy constructor copies from a \p complex with a type that is
+   *  convertible to this \p complex's \c value_type.
+   *
+   *  \param z The \p complex to copy from.
+   */
+  inline complex(const complex<T>& z) = default;
+#else
+  /*! Default construct a complex number.
+   */
+  inline __host__ __device__ complex();
+
+  /*! This copy constructor copies from a \p complex with a type that is
+   *  convertible to this \p complex's \c value_type.
+   *
+   *  \param z The \p complex to copy from.
+   */
+  inline __host__ __device__ complex(const complex<T>& z);
+#endif // c++11
+
+  /*! This copy constructor copies from a \p complex with a type that
+   *  is convertible to this \p complex \c value_type.
+   *
+   *  \param z The \p complex to copy from.
+   *
+   *  \tparam X is convertible to \c value_type.
+   */
+  template <typename X>
+  inline __host__ __device__ complex(const complex<X>& z);
+
+  /* --- Assignment Operators --- */
+
+  /*! Assign `re` to the real part of this \p complex and set the imaginary part
+   *  to 0.
+   *
+   *  \param re The real part of the number.
+   */
+  inline __host__ __device__ complex& operator=(const T& re);
+
+  /*! Assign `z.real()` and `z.imag()` to the real and imaginary parts of this
+   *  \p complex respectively.
+   *
+   *  \param z The \p complex to copy from.
+   */
+  inline __host__ __device__ complex& operator=(const complex<T>& z);
+
+  /*! Assign `z.real()` and `z.imag()` to the real and imaginary parts of this
+   *  \p complex respectively.
+   *
+   *  \param z The \p complex to copy from.
+   *
+   *  \tparam U is convertible to \c value_type.
+   */
+  template <typename U>
+  inline __host__ __device__ complex& operator=(const complex<U>& z);
+
+  /* --- Compound Assignment Operators --- */
+
+  /*! Adds a \p complex to this \p complex and
+   *  assigns the result to this \p complex.
+   *
+   *  \param z The \p complex to be Added.
+   */
+  __host__ __device__ inline complex<T>& operator+=(const complex<T> z);
+
+  /*! Subtracts a \p complex from this \p complex and
+   *  assigns the result to this \p complex.
+   *
+   *  \param z The \p complex to be subtracted.
+   */
+  __host__ __device__ inline complex<T>& operator-=(const complex<T> z);
+
+  /*! Multiplies this \p complex by another \p complex and
+   *  assigns the result to this \p complex.
+   *
+   *  \param z The \p complex to be multiplied.
+   */
+  __host__ __device__ inline complex<T>& operator*=(const complex<T> z);
+
+  /*! Divides this \p complex by another \p complex and
+   *  assigns the result to this \p complex.
+   *
+   *  \param z The \p complex to be divided.
+   */
+  __host__ __device__ inline complex<T>& operator/=(const complex<T> z);
+
+  /* --- Getter functions ---
+   * The volatile ones are there to help for example
+   * with certain reductions optimizations
+   */
+
+  /*! Returns the real part of this \p complex.
+   */
+  __host__ __device__ inline T real() const volatile { return m_data[0]; }
+
+  /*! Returns the imaginary part of this \p complex.
+   */
+  __host__ __device__ inline T imag() const volatile { return m_data[1]; }
+
+  /*! Returns the real part of this \p complex.
+   */
+  __host__ __device__ inline T real() const { return m_data[0]; }
+
+  /*! Returns the imaginary part of this \p complex.
+   */
+  __host__ __device__ inline T imag() const { return m_data[1]; }
+
+  /* --- Setter functions ---
+   * The volatile ones are there to help for example
+   * with certain reductions optimizations
+   */
+
+  /*! Sets the real part of this \p complex.
+   *
+   *  \param re The new real part of this \p complex.
+   */
+  __host__ __device__ inline void real(T re) volatile { m_data[0] = re; }
+
+  /*! Sets the imaginary part of this \p complex.
+   *
+   *  \param im The new imaginary part of this \p complex.e
+   */
+  __host__ __device__ inline void imag(T im) volatile { m_data[1] = im; }
+
+  /*! Sets the real part of this \p complex.
+   *
+   *  \param re The new real part of this \p complex.
+   */
+  __host__ __device__ inline void real(T re) { m_data[0] = re; }
+
+  /*! Sets the imaginary part of this \p complex.
+   *
+   *  \param im The new imaginary part of this \p complex.
+   */
+  __host__ __device__ inline void imag(T im) { m_data[1] = im; }
+
+ private:
+  T m_data[2];
+};
+
+/* --- General Functions --- */
+
+/*! Returns the magnitude (also known as absolute value) of a \p complex.
+ *
+ *  \param z The \p complex from which to calculate the absolute value.
+ */
+template <typename T>
+__host__ __device__ inline T abs(const complex<T>& z);
+
+/*! Returns the phase angle (also known as argument) in radians of a \p complex.
+ *
+ *  \param z The \p complex from which to calculate the phase angle.
+ */
+template <typename T>
+__host__ __device__ inline T arg(const complex<T>& z);
+
+/*! Returns the square of the magnitude of a \p complex.
+ *
+ *  \param z The \p complex from which to calculate the norm.
+ */
+template <typename T>
+__host__ __device__ inline T norm(const complex<T>& z);
+
+/*! Returns the complex conjugate of a \p complex.
+ *
+ *  \param z The \p complex from which to calculate the complex conjugate.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> conj(const complex<T>& z);
+
+/*! Returns the real part of a \p complex.
+ *
+ *  \param z The \p complex from which to return the real part
+ */
+template <typename T>
+__host__ __device__ inline T real(const complex<T>& z);
+
+/*! Returns the imaginary part of a \p complex.
+ *
+ *  \param z The \p complex from which to return the imaginary part
+ */
+template <typename T>
+__host__ __device__ inline T imag(const complex<T>& z);
+
+/*! Returns a \p complex with the specified magnitude and phase.
+ *
+ *  \param m The magnitude of the returned \p complex.
+ *  \param theta The phase of the returned \p complex in radians.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> polar(const T& m, const T& theta = 0);
+
+/*! Returns the projection of a \p complex on the Riemann sphere.
+ *  For all finite \p complex it returns the argument. For \p complexs
+ *  with a non finite part returns (INFINITY,+/-0) where the sign of
+ *  the zero matches the sign of the imaginary part of the argument.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> proj(const T& z);
+
+/* --- Binary Arithmetic operators --- */
+
+/*! Multiplies two \p complex numbers.
+ *
+ *  \param lhs The first \p complex.
+ *  \param rhs The second \p complex.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator*(const complex<T>& lhs,
+                                       const complex<T>& rhs);
+
+/*! Multiplies a \p complex number by a scalar.
+ *
+ *  \param lhs The \p complex.
+ *  \param rhs The scalar.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator*(const complex<T>& lhs, const T& rhs);
+
+/*! Multiplies a scalar by a \p complex number.
+ *
+ *  \param lhs The scalar.
+ *  \param rhs The \p complex.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator*(const T& lhs, const complex<T>& rhs);
+
+/*! Divides two \p complex numbers.
+ *
+ *  \param lhs The numerator (dividend).
+ *  \param rhs The denomimator (divisor).
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator/(const complex<T>& lhs,
+                                       const complex<T>& rhs);
+
+/*! Divides a \p complex number by a scalar.
+ *
+ *  \param lhs The complex numerator (dividend).
+ *  \param rhs The scalar denomimator (divisor).
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator/(const complex<T>& lhs, const T& rhs);
+
+/*! Divides a scalar by a \p complex number.
+ *
+ *  \param lhs The scalar numerator (dividend).
+ *  \param rhs The complex denomimator (divisor).
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator/(const T& lhs, const complex<T>& rhs);
+
+/*! Adds two \p complex numbers.
+ *
+ *  \param lhs The first \p complex.
+ *  \param rhs The second \p complex.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const complex<T>& lhs,
+                                       const complex<T>& rhs);
+
+/*! Adds a scalar to a \p complex number.
+ *
+ *  \param lhs The \p complex.
+ *  \param rhs The scalar.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const complex<T>& lhs, const T& rhs);
+
+/*! Adds a \p complex number to a scalar.
+ *
+ *  \param lhs The scalar.
+ *  \param rhs The \p complex.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const T& lhs, const complex<T>& rhs);
+
+/*! Subtracts two \p complex numbers.
+ *
+ *  \param lhs The first \p complex (minuend).
+ *  \param rhs The second \p complex (subtrahend).
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator-(const complex<T>& lhs,
+                                       const complex<T>& rhs);
+
+/*! Subtracts a scalar from a \p complex number.
+ *
+ *  \param lhs The \p complex (minuend).
+ *  \param rhs The scalar (subtrahend).
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator-(const complex<T>& lhs, const T& rhs);
+
+/*! Subtracts a \p complex number from a scalar.
+ *
+ *  \param lhs The scalar (minuend).
+ *  \param rhs The \p complex (subtrahend).
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator-(const T& lhs, const complex<T>& rhs);
+
+/* --- Unary Arithmetic operators --- */
+
+/*! Unary plus, returns its \p complex argument.
+ *
+ *  \param rhs The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator+(const complex<T>& rhs);
+
+/*! Unary minus, returns the additive inverse (negation) of its \p complex
+ * argument.
+ *
+ *  \param rhs The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> operator-(const complex<T>& rhs);
+
+/* --- Exponential Functions --- */
+
+/*! Returns the complex exponential of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> exp(const complex<T>& z);
+
+/*! Returns the complex natural logarithm of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> log(const complex<T>& z);
+
+/*! Returns the complex base 10 logarithm of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ inline complex<T> log10(const complex<T>& z);
+
+/* --- Power Functions --- */
+
+/*! Returns a \p complex number raised to another.
+ *
+ *  \param x The base.
+ *  \param y The exponent.
+ */
+template <typename T>
+__host__ __device__ complex<T> pow(const complex<T>& x, const complex<T>& y);
+
+/*! Returns a \p complex number raised to a scalar.
+ *
+ *  \param x The \p complex base.
+ *  \param y The scalar exponent.
+ */
+template <typename T>
+__host__ __device__ complex<T> pow(const complex<T>& x, const T& y);
+
+/*! Returns a scalar raised to a \p complex number.
+ *
+ *  \param x The scalar base.
+ *  \param y The \p complex exponent.
+ */
+template <typename T>
+__host__ __device__ complex<T> pow(const T& x, const complex<T>& y);
+
+/*! Returns a \p complex number raised to another. The types of the two \p
+ * complex should be compatible
+ * and the type of the returned \p complex is the promoted type of the two
+ * arguments.
+ *
+ *  \param x The base.
+ *  \param y The exponent.
+ */
+template <typename T, typename U>
+__host__ __device__ complex<typename _select_greater_type<T, U>::type> pow(
+    const complex<T>& x, const complex<U>& y);
+
+/*! Returns a \p complex number raised to a scalar. The type of the \p complex
+ * should be compatible with the scalar
+ * and the type of the returned \p complex is the promoted type of the two
+ * arguments.
+ *
+ *  \param x The base.
+ *  \param y The exponent.
+ */
+template <typename T, typename U>
+__host__ __device__ complex<typename _select_greater_type<T, U>::type> pow(
+    const complex<T>& x, const U& y);
+
+/*! Returns a scalar raised to a \p complex number. The type of the \p complex
+ * should be compatible with the scalar
+ * and the type of the returned \p complex is the promoted type of the two
+ * arguments.
+ *
+ *  \param x The base.
+ *  \param y The exponent.
+ */
+template <typename T, typename U>
+__host__ __device__ complex<typename _select_greater_type<T, U>::type> pow(
+    const T& x, const complex<U>& y);
+
+/*! Returns the complex square root of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> sqrt(const complex<T>& z);
+
+/* --- Trigonometric Functions --- */
+
+/*! Returns the complex cosine of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> cos(const complex<T>& z);
+
+/*! Returns the complex sine of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> sin(const complex<T>& z);
+
+/*! Returns the complex tangent of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> tan(const complex<T>& z);
+
+/* --- Hyperbolic Functions --- */
+
+/*! Returns the complex hyperbolic cosine of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> cosh(const complex<T>& z);
+
+/*! Returns the complex hyperbolic sine of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> sinh(const complex<T>& z);
+
+/*! Returns the complex hyperbolic tangent of a \p complex number.
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> tanh(const complex<T>& z);
+
+/* --- Inverse Trigonometric Functions --- */
+
+/*! Returns the complex arc cosine of a \p complex number.
+ *
+ *  The range of the real part of the result is [0, Pi] and
+ *  the range of the imaginary part is [-inf, +inf]
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> acos(const complex<T>& z);
+
+/*! Returns the complex arc sine of a \p complex number.
+ *
+ *  The range of the real part of the result is [-Pi/2, Pi/2] and
+ *  the range of the imaginary part is [-inf, +inf]
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> asin(const complex<T>& z);
+
+/*! Returns the complex arc tangent of a \p complex number.
+ *
+ *  The range of the real part of the result is [-Pi/2, Pi/2] and
+ *  the range of the imaginary part is [-inf, +inf]
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> atan(const complex<T>& z);
+
+/* --- Inverse Hyperbolic Functions --- */
+
+/*! Returns the complex inverse hyperbolic cosine of a \p complex number.
+ *
+ *  The range of the real part of the result is [0, +inf] and
+ *  the range of the imaginary part is [-Pi, Pi]
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> acosh(const complex<T>& z);
+
+/*! Returns the complex inverse hyperbolic sine of a \p complex number.
+ *
+ *  The range of the real part of the result is [-inf, +inf] and
+ *  the range of the imaginary part is [-Pi/2, Pi/2]
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> asinh(const complex<T>& z);
+
+/*! Returns the complex inverse hyperbolic tangent of a \p complex number.
+ *
+ *  The range of the real part of the result is [-inf, +inf] and
+ *  the range of the imaginary part is [-Pi/2, Pi/2]
+ *
+ *  \param z The \p complex argument.
+ */
+template <typename T>
+__host__ __device__ complex<T> atanh(const complex<T>& z);
+
+/* --- Equality Operators --- */
+
+/*! Returns true if two \p complex numbers are equal and false otherwise.
+ *
+ *  \param lhs The first \p complex.
+ *  \param rhs The second \p complex.
+ */
+template <typename T>
+__host__ __device__ inline bool operator==(const complex<T>& lhs, const complex<T>& rhs);
+
+/*! Returns true if the imaginary part of the  \p complex number is zero and the
+ * real part is equal to the scalar. Returns false otherwise.
+ *
+ *  \param lhs The scalar.
+ *  \param rhs The \p complex.
+ */
+template <typename T>
+__host__ __device__ inline bool operator==(const T& lhs, const complex<T>& rhs);
+
+/*! Returns true if the imaginary part of the  \p complex number is zero and the
+ * real part is equal to the scalar. Returns false otherwise.
+ *
+ *  \param lhs The \p complex.
+ *  \param rhs The scalar.
+ */
+template <typename T>
+__host__ __device__ inline bool operator==(const complex<T>& lhs, const T& rhs);
+
+/*! Returns true if two \p complex numbers are different and false otherwise.
+ *
+ *  \param lhs The first \p complex.
+ *  \param rhs The second \p complex.
+ */
+template <typename T>
+__host__ __device__ inline bool operator!=(const complex<T>& lhs, const complex<T>& rhs);
+
+/*! Returns true if the imaginary part of the  \p complex number is not zero or
+ * the real part is different from the scalar. Returns false otherwise.
+ *
+ *  \param lhs The scalar.
+ *  \param rhs The \p complex.
+ */
+template <typename T>
+__host__ __device__ inline bool operator!=(const T& lhs, const complex<T>& rhs);
+
+/*! Returns true if the imaginary part of the \p complex number is not zero or
+ * the real part is different from the scalar. Returns false otherwise.
+ *
+ *  \param lhs The \p complex.
+ *  \param rhs The scalar.
+ */
+template <typename T>
+__host__ __device__ inline bool operator!=(const complex<T>& lhs, const T& rhs);
+
+}  // end namespace thrust
+
+#include <cupy/complex/complex_inl.h>
diff --git a/cupy/_core/include/cupy/complex/complex_inl.h b/cupy/_core/include/cupy/complex/complex_inl.h
new file mode 100644
index 0000000..e38cfbd
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/complex_inl.h
@@ -0,0 +1,164 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+
+namespace thrust {
+
+/* --- Constructors --- */
+template <typename T>
+inline __host__ __device__ complex<T>::complex(const T& re) {
+  real(re);
+  imag(T());
+}
+
+template <typename T>
+inline __host__ __device__ complex<T>::complex(const T& re, const T& im) {
+  real(re);
+  imag(im);
+}
+
+#if ((!defined(_MSC_VER) && __cplusplus < 201103L) || \
+     (defined(_MSC_VER) && _MSC_VER < 1900))
+template <typename T>
+inline __host__ __device__ complex<T>::complex() {
+  real(T());
+  imag(T());
+}
+
+template <typename T>
+inline __host__ __device__ complex<T>::complex(const complex<T>& z) {
+  real(z.real());
+  imag(z.imag());
+}
+#endif
+
+template <typename T>
+template <typename X>
+inline __host__ __device__ complex<T>::complex(const complex<X>& z) {
+  // The explicit T() is there no prevent Visual Studio from complaining
+  // about potential loss of precision
+  real(T(z.real()));
+  imag(T(z.imag()));
+}
+
+/* --- Assignment Operators --- */
+
+template <typename T>
+inline __host__ __device__ complex<T>& complex<T>::operator=(const T& re) {
+  real(re);
+  imag(T());
+  return *this;
+}
+
+template <typename T>
+inline __host__ __device__ complex<T>& complex<T>::operator=(const complex<T>& z) {
+  real(z.real());
+  imag(z.imag());
+  return *this;
+}
+
+template <typename T>
+template <typename U>
+inline __host__ __device__ complex<T>& complex<T>::operator=(const complex<U>& z) {
+  real(T(z.real()));
+  imag(T(z.imag()));
+  return *this;
+}
+
+/* --- Compound Assignment Operators --- */
+// TODO(leofang): support operators with argument of type T, see upstream
+
+template <typename T>
+__host__ __device__ inline complex<T>& complex<T>::operator+=(const complex<T> z) {
+  *this = *this + z;
+  return *this;
+}
+
+template <typename T>
+__host__ __device__ inline complex<T>& complex<T>::operator-=(const complex<T> z) {
+  *this = *this - z;
+  return *this;
+}
+
+template <typename T>
+__host__ __device__ inline complex<T>& complex<T>::operator*=(const complex<T> z) {
+  *this = *this * z;
+  return *this;
+}
+
+template <typename T>
+__host__ __device__ inline complex<T>& complex<T>::operator/=(const complex<T> z) {
+  *this = *this / z;
+  return *this;
+}
+
+/* --- Equality Operators --- */
+
+template <typename T>
+__host__ __device__ inline bool operator==(const complex<T>& lhs,
+                                           const complex<T>& rhs) {
+  return lhs.real() == rhs.real() && lhs.imag() == rhs.imag();
+}
+
+template <typename T>
+__host__ __device__ inline bool operator==(const T& lhs, const complex<T>& rhs) {
+  return lhs == rhs.real() && rhs.imag() == 0;
+}
+
+template <typename T>
+__host__ __device__ inline bool operator==(const complex<T>& lhs, const T& rhs) {
+  return lhs.real() == rhs && lhs.imag() == 0;
+}
+
+template <typename T>
+__host__ __device__ inline bool operator!=(const complex<T>& lhs,
+                                           const complex<T>& rhs) {
+  return !(lhs == rhs);
+}
+
+template <typename T>
+__host__ __device__ inline bool operator!=(const T& lhs, const complex<T>& rhs) {
+  return !(lhs == rhs);
+}
+
+template <typename T>
+__host__ __device__ inline bool operator!=(const complex<T>& lhs, const T& rhs) {
+  return !(lhs == rhs);
+}
+}
+
+
+#include <cupy/complex/arithmetic.h>
+#include <cupy/complex/cproj.h>
+#include <cupy/complex/cexp.h>
+#include <cupy/complex/cexpf.h>
+#include <cupy/complex/clog.h>
+#include <cupy/complex/clogf.h>
+#include <cupy/complex/cpow.h>
+#include <cupy/complex/ccosh.h>
+#include <cupy/complex/ccoshf.h>
+#include <cupy/complex/csinh.h>
+#include <cupy/complex/csinhf.h>
+#include <cupy/complex/ctanh.h>
+#include <cupy/complex/ctanhf.h>
+#include <cupy/complex/csqrt.h>
+#include <cupy/complex/csqrtf.h>
+#include <cupy/complex/catrig.h>
+#include <cupy/complex/catrigf.h>
diff --git a/cupy/_core/include/cupy/complex/cpow.h b/cupy/_core/include/cupy/complex/cpow.h
new file mode 100644
index 0000000..28c19c8
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/cpow.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include <cupy/complex/complex.h>
+
+namespace thrust {
+
+template <typename T>
+__host__ __device__ inline complex<T> pow(const complex<T>& z,
+                                          const complex<T>& exponent) {
+  return exp(log(complex<T>(z)) * complex<T>(exponent));
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> pow(const complex<T>& z, const T& exponent) {
+  return exp(log(complex<T>(z)) * T(exponent));
+}
+
+template <typename T>
+__host__ __device__ inline complex<T> pow(const T& x, const complex<T>& exponent) {
+  return exp(log(T(x)) * complex<T>(exponent));
+}
+
+template <typename T, typename U>
+__host__ __device__ inline complex<typename _select_greater_type<T, U>::type> pow(
+    const complex<T>& z, const complex<T>& exponent) {
+  typedef typename _select_greater_type<T, U>::type PromotedType;
+  return pow(complex<PromotedType>(z), complex<PromotedType>(exponent));
+}
+
+template <typename T, typename U>
+__host__ __device__ inline complex<typename _select_greater_type<T, U>::type> pow(
+    const complex<T>& z, const U& exponent) {
+  typedef typename _select_greater_type<T, U>::type PromotedType;
+  return pow(complex<PromotedType>(z), PromotedType(exponent));
+}
+
+template <typename T, typename U>
+__host__ __device__ inline complex<typename _select_greater_type<T, U>::type> pow(
+    const T& x, const complex<U>& exponent) {
+  typedef typename _select_greater_type<T, U>::type PromotedType;
+  return pow(PromotedType(x), complex<PromotedType>(exponent));
+}
+
+}
diff --git a/cupy/_core/include/cupy/complex/cproj.h b/cupy/_core/include/cupy/complex/cproj.h
new file mode 100644
index 0000000..f5e2d33
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/cproj.h
@@ -0,0 +1,64 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<float> cprojf(const complex<float>& z) {
+  if (!isinf(z.real()) && !isinf(z.imag())) {
+    return z;
+  } else {
+    // std::numeric_limits<T>::infinity() doesn't run on the GPU
+    return complex<float>(infinity<float>(), copysignf(0.0, z.imag()));
+  }
+}
+
+__host__ __device__ inline complex<double> cproj(const complex<double>& z) {
+  if (!isinf(z.real()) && !isinf(z.imag())) {
+    return z;
+  } else {
+    // numeric_limits<T>::infinity() doesn't run on the GPU
+    return complex<double>(infinity<double>(), copysign(0.0, z.imag()));
+  }
+}
+}
+}
+
+template <typename T>
+__host__ __device__ inline thrust::complex<T> proj(const thrust::complex<T>& z) {
+  return detail::complex::cproj(z);
+}
+
+template <>
+__host__ __device__ inline thrust::complex<double> proj(
+    const thrust::complex<double>& z) {
+  return detail::complex::cproj(z);
+}
+
+template <>
+__host__ __device__ inline thrust::complex<float> proj(const thrust::complex<float>& z) {
+  return detail::complex::cprojf(z);
+}
+}
diff --git a/cupy/_core/include/cupy/complex/csinh.h b/cupy/_core/include/cupy/complex/csinh.h
new file mode 100644
index 0000000..766004d
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/csinh.h
@@ -0,0 +1,192 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2005 Bruce D. Evans and Steven G. Kargl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* adapted from FreeBSD:
+ *    lib/msun/src/s_csinh.c
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<double> csinh(const complex<double>& z) {
+  double x, y, h;
+  uint32_t hx, hy, ix, iy, lx, ly;
+  const double huge = 8.98846567431157953864652595395e+307;  // 0x1p1023;
+
+  x = z.real();
+  y = z.imag();
+
+  extract_words(hx, lx, x);
+  extract_words(hy, ly, y);
+
+  ix = 0x7fffffff & hx;
+  iy = 0x7fffffff & hy;
+
+  /* Handle the nearly-non-exceptional cases where x and y are finite. */
+  if (ix < 0x7ff00000 && iy < 0x7ff00000) {
+    if ((iy | ly) == 0) return (complex<double>(sinh(x), y));
+    if (ix < 0x40360000) /* small x: normal case */
+      return (complex<double>(sinh(x) * cos(y), cosh(x) * sin(y)));
+
+    /* |x| >= 22, so cosh(x) ~= exp(|x|) */
+    if (ix < 0x40862e42) {
+      /* x < 710: exp(|x|) won't overflow */
+      h = exp(fabs(x)) * 0.5;
+      return (complex<double>(copysign(h, x) * cos(y), h * sin(y)));
+    } else if (ix < 0x4096bbaa) {
+      /* x < 1455: scale to avoid overflow */
+      complex<double> z_ = ldexp_cexp(complex<double>(fabs(x), y), -1);
+      return (complex<double>(z_.real() * copysign(1.0, x), z_.imag()));
+    } else {
+      /* x >= 1455: the result always overflows */
+      h = huge * x;
+      return (complex<double>(h * cos(y), h * h * sin(y)));
+    }
+  }
+
+  /*
+   * sinh(+-0 +- I Inf) = sign(d(+-0, dNaN))0 + I dNaN.
+   * The sign of 0 in the result is unspecified.  Choice = normally
+   * the same as dNaN.  Raise the invalid floating-point exception.
+   *
+   * sinh(+-0 +- I NaN) = sign(d(+-0, NaN))0 + I d(NaN).
+   * The sign of 0 in the result is unspecified.  Choice = normally
+   * the same as d(NaN).
+   */
+  if ((ix | lx) == 0 && iy >= 0x7ff00000)
+    return (complex<double>(copysign(0.0, x * (y - y)), y - y));
+
+  /*
+   * sinh(+-Inf +- I 0) = +-Inf + I +-0.
+   *
+   * sinh(NaN +- I 0)   = d(NaN) + I +-0.
+   */
+  if ((iy | ly) == 0 && ix >= 0x7ff00000) {
+    if (((hx & 0xfffff) | lx) == 0) return (complex<double>(x, y));
+    return (complex<double>(x, copysign(0.0, y)));
+  }
+
+  /*
+   * sinh(x +- I Inf) = dNaN + I dNaN.
+   * Raise the invalid floating-point exception for finite nonzero x.
+   *
+   * sinh(x + I NaN) = d(NaN) + I d(NaN).
+   * Optionally raises the invalid floating-point exception for finite
+   * nonzero x.  Choice = don't raise (except for signaling NaNs).
+   */
+  if (ix < 0x7ff00000 && iy >= 0x7ff00000)
+    return (complex<double>(y - y, x * (y - y)));
+
+  /*
+   * sinh(+-Inf + I NaN)  = +-Inf + I d(NaN).
+   * The sign of Inf in the result is unspecified.  Choice = normally
+   * the same as d(NaN).
+   *
+   * sinh(+-Inf +- I Inf) = +Inf + I dNaN.
+   * The sign of Inf in the result is unspecified.  Choice = always +.
+   * Raise the invalid floating-point exception.
+   *
+   * sinh(+-Inf + I y)   = +-Inf cos(y) + I Inf sin(y)
+   */
+  if (ix >= 0x7ff00000 && ((hx & 0xfffff) | lx) == 0) {
+    if (iy >= 0x7ff00000) return (complex<double>(x * x, x * (y - y)));
+    return (complex<double>(x * cos(y), infinity<double>() * sin(y)));
+  }
+
+  /*
+   * sinh(NaN + I NaN)  = d(NaN) + I d(NaN).
+   *
+   * sinh(NaN +- I Inf) = d(NaN) + I d(NaN).
+   * Optionally raises the invalid floating-point exception.
+   * Choice = raise.
+   *
+   * sinh(NaN + I y)    = d(NaN) + I d(NaN).
+   * Optionally raises the invalid floating-point exception for finite
+   * nonzero y.  Choice = don't raise (except for signaling NaNs).
+   */
+  return (complex<double>((x * x) * (y - y), (x + x) * (y - y)));
+}
+
+__host__ __device__ inline complex<double> csin(complex<double> z) {
+  /* csin(z) = -I * csinh(I * z) */
+  z = csinh(complex<double>(-z.imag(), z.real()));
+  return (complex<double>(z.imag(), -z.real()));
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> sin(const complex<ValueType>& z) {
+  const ValueType re = z.real();
+  const ValueType im = z.imag();
+  return complex<ValueType>(::sin(re) * ::cosh(im), ::cos(re) * ::sinh(im));
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> sinh(const complex<ValueType>& z) {
+  const ValueType re = z.real();
+  const ValueType im = z.imag();
+  return complex<ValueType>(::sinh(re) * ::cos(im), ::cosh(re) * ::sin(im));
+}
+
+template <>
+__host__ __device__ inline complex<double> sin(const complex<double>& z) {
+  return detail::complex::csin(z);
+}
+
+template <>
+__host__ __device__ inline complex<double> sinh(const complex<double>& z) {
+  return detail::complex::csinh(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/csinhf.h b/cupy/_core/include/cupy/complex/csinhf.h
new file mode 100644
index 0000000..cb12236
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/csinhf.h
@@ -0,0 +1,133 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2005 Bruce D. Evans and Steven G. Kargl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* adapted from FreeBSD:
+ *    lib/msun/src/s_csinhf.c
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<float> csinhf(const complex<float>& z) {
+  float x, y, h;
+  uint32_t hx, hy, ix, iy;
+
+  const float huge = 1.70141183460469231731687303716e+38;  // 0x1p127;
+
+  x = z.real();
+  y = z.imag();
+
+  get_float_word(hx, x);
+  get_float_word(hy, y);
+
+  ix = 0x7fffffff & hx;
+  iy = 0x7fffffff & hy;
+
+  if (ix < 0x7f800000 && iy < 0x7f800000) {
+    if (iy == 0) return (complex<float>(sinhf(x), y));
+    if (ix < 0x41100000) /* small x: normal case */
+      return (complex<float>(sinhf(x) * cosf(y), coshf(x) * sinf(y)));
+
+    /* |x| >= 9, so cosh(x) ~= exp(|x|) */
+    if (ix < 0x42b17218) {
+      /* x < 88.7: expf(|x|) won't overflow */
+      h = expf(fabsf(x)) * 0.5f;
+      return (complex<float>(copysignf(h, x) * cosf(y), h * sinf(y)));
+    } else if (ix < 0x4340b1e7) {
+      /* x < 192.7: scale to avoid overflow */
+      complex<float> z_ = ldexp_cexpf(complex<float>(fabsf(x), y), -1);
+      return (complex<float>(z_.real() * copysignf(1.0f, x), z_.imag()));
+    } else {
+      /* x >= 192.7: the result always overflows */
+      h = huge * x;
+      return (complex<float>(h * cosf(y), h * h * sinf(y)));
+    }
+  }
+
+  if (ix == 0 && iy >= 0x7f800000)
+    return (complex<float>(copysignf(0, x * (y - y)), y - y));
+
+  if (iy == 0 && ix >= 0x7f800000) {
+    if ((hx & 0x7fffff) == 0) return (complex<float>(x, y));
+    return (complex<float>(x, copysignf(0.0f, y)));
+  }
+
+  if (ix < 0x7f800000 && iy >= 0x7f800000)
+    return (complex<float>(y - y, x * (y - y)));
+
+  if (ix >= 0x7f800000 && (hx & 0x7fffff) == 0) {
+    if (iy >= 0x7f800000) return (complex<float>(x * x, x * (y - y)));
+    return (complex<float>(x * cosf(y), infinity<float>() * sinf(y)));
+  }
+
+  return (complex<float>((x * x) * (y - y), (x + x) * (y - y)));
+}
+
+__host__ __device__ inline complex<float> csinf(complex<float> z) {
+  z = csinhf(complex<float>(-z.imag(), z.real()));
+  return (complex<float>(z.imag(), -z.real()));
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <>
+__host__ __device__ inline complex<float> sin(const complex<float>& z) {
+  return detail::complex::csinf(z);
+}
+
+template <>
+__host__ __device__ inline complex<float> sinh(const complex<float>& z) {
+  return detail::complex::csinhf(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/csqrt.h b/cupy/_core/include/cupy/complex/csqrt.h
new file mode 100644
index 0000000..68d8a13
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/csqrt.h
@@ -0,0 +1,144 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2007 David Schultz <das@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Adapted from FreeBSD by Filipe Maia <filipe.c.maia@gmail.com>:
+ *    freebsd/lib/msun/src/s_csqrt.c
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<double> csqrt(const complex<double>& z) {
+  complex<double> result;
+  double a, b;
+  double t;
+  int scale;
+
+  /* We risk spurious overflow for components >= DBL_MAX / (1 + sqrt(2)). */
+  const double THRESH = 7.446288774449766337959726e+307;
+
+  a = z.real();
+  b = z.imag();
+
+  /* Handle special cases. */
+  if (z == 0.0) return (complex<double>(0.0, b));
+  if (isinf(b)) return (complex<double>(infinity<double>(), b));
+  if (isnan(a)) {
+    t = (b - b) / (b - b);          /* raise invalid if b is not a NaN */
+    return (complex<double>(a, t)); /* return NaN + NaN i */
+  }
+  if (isinf(a)) {
+    /*
+     * csqrt(inf + NaN i)  = inf +  NaN i
+     * csqrt(inf + y i)    = inf +  0 i
+     * csqrt(-inf + NaN i) = NaN +- inf i
+     * csqrt(-inf + y i)   = 0   +  inf i
+     */
+    if (signbit(a))
+      return (complex<double>(fabs(b - b), copysign(a, b)));
+    else
+      return (complex<double>(a, copysign(b - b, b)));
+  }
+  /*
+   * The remaining special case (b is NaN) is handled just fine by
+   * the normal code path below.
+   */
+
+  // DBL_MIN*2
+  const double low_thresh = 4.450147717014402766180465e-308;
+  scale = 0;
+
+  if (fabs(a) >= THRESH || fabs(b) >= THRESH) {
+    /* Scale to avoid overflow. */
+    a *= 0.25;
+    b *= 0.25;
+    scale = 1;
+  } else if (fabs(a) <= low_thresh && fabs(b) <= low_thresh) {
+    /* Scale to avoid underflow. */
+    a *= 4.0;
+    b *= 4.0;
+    scale = 2;
+  }
+
+  /* Algorithm 312, CACM vol 10, Oct 1967. */
+  if (a >= 0.0) {
+    t = sqrt((a + hypot(a, b)) * 0.5);
+    result = complex<double>(t, b / (2 * t));
+  } else {
+    t = sqrt((-a + hypot(a, b)) * 0.5);
+    result = complex<double>(fabs(b) / (2 * t), copysign(t, b));
+  }
+
+  /* Rescale. */
+  if (scale == 1)
+    return (result * 2.0);
+  else if (scale == 2)
+    return (result * 0.5);
+  else
+    return (result);
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> sqrt(const complex<ValueType>& z) {
+  return thrust::polar(::sqrt(thrust::abs(z)), thrust::arg(z) / ValueType(2));
+}
+
+template <>
+__host__ __device__ inline complex<double> sqrt(const complex<double>& z) {
+  return detail::complex::csqrt(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/csqrtf.h b/cupy/_core/include/cupy/complex/csqrtf.h
new file mode 100644
index 0000000..8a5a7d4
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/csqrtf.h
@@ -0,0 +1,141 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2007 David Schultz <das@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Adapted from FreeBSD by Filipe Maia <filipe.c.maia@gmail.com>:
+ *    freebsd/lib/msun/src/s_csqrt.c
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<float> csqrtf(const complex<float>& z) {
+  float a = z.real(), b = z.imag();
+  float t;
+  int scale;
+  complex<float> result;
+
+  /* We risk spurious overflow for components >= FLT_MAX / (1 + sqrt(2)). */
+  const float THRESH = 1.40949553037932e+38f;
+
+  /* Handle special cases. */
+  if (z == 0.0f) return (complex<float>(0, b));
+  if (isinf(b)) return (complex<float>(infinity<float>(), b));
+  if (isnan(a)) {
+    t = (b - b) / (b - b);         /* raise invalid if b is not a NaN */
+    return (complex<float>(a, t)); /* return NaN + NaN i */
+  }
+  if (isinf(a)) {
+    /*
+     * csqrtf(inf + NaN i)  = inf +  NaN i
+     * csqrtf(inf + y i)    = inf +  0 i
+     * csqrtf(-inf + NaN i) = NaN +- inf i
+     * csqrtf(-inf + y i)   = 0   +  inf i
+     */
+    if (signbit(a))
+      return (complex<float>(fabsf(b - b), copysignf(a, b)));
+    else
+      return (complex<float>(a, copysignf(b - b, b)));
+  }
+  /*
+   * The remaining special case (b is NaN) is handled just fine by
+   * the normal code path below.
+   */
+
+  /*
+   * Unlike in the FreeBSD code we'll avoid using double precision as
+   * not all hardware supports it.
+   */
+
+  // FLT_MIN*2
+  const float low_thresh = 2.35098870164458e-38f;
+  scale = 0;
+
+  if (fabsf(a) >= THRESH || fabsf(b) >= THRESH) {
+    /* Scale to avoid overflow. */
+    a *= 0.25f;
+    b *= 0.25f;
+    scale = 1;
+  } else if (fabsf(a) <= low_thresh && fabsf(b) <= low_thresh) {
+    /* Scale to avoid underflow. */
+    a *= 4.f;
+    b *= 4.f;
+    scale = 2;
+  }
+
+  /* Algorithm 312, CACM vol 10, Oct 1967. */
+  if (a >= 0.0f) {
+    t = sqrtf((a + hypotf(a, b)) * 0.5f);
+    result = complex<float>(t, b / (2.0f * t));
+  } else {
+    t = sqrtf((-a + hypotf(a, b)) * 0.5f);
+    result = complex<float>(fabsf(b) / (2.0f * t), copysignf(t, b));
+  }
+
+  /* Rescale. */
+  if (scale == 1)
+    return (result * 2.0f);
+  else if (scale == 2)
+    return (result * 0.5f);
+  else
+    return (result);
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <>
+__host__ __device__ inline complex<float> sqrt(const complex<float>& z) {
+  return detail::complex::csqrtf(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/ctanh.h b/cupy/_core/include/cupy/complex/ctanh.h
new file mode 100644
index 0000000..4929f29
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/ctanh.h
@@ -0,0 +1,191 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2011 David Schultz
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Adapted from FreeBSD by Filipe Maia <filipe.c.maia@gmail.com>:
+ *    freebsd/lib/msun/src/s_ctanh.c
+ */
+
+/*
+ * Hyperbolic tangent of a complex argument z = x + i y.
+ *
+ * The algorithm is from:
+ *
+ *   W. Kahan.  Branch Cuts for Complex Elementary Functions or Much
+ *   Ado About Nothing's Sign Bit.  In The State of the Art in
+ *   Numerical Analysis, pp. 165 ff.  Iserles and Powell, eds., 1987.
+ *
+ * Method:
+ *
+ *   Let t    = tan(x)
+ *       beta = 1/cos^2(y)
+ *       s    = sinh(x)
+ *       rho  = cosh(x)
+ *
+ *   We have:
+ *
+ *   tanh(z) = sinh(z) / cosh(z)
+ *
+ *             sinh(x) cos(y) + i cosh(x) sin(y)
+ *           = ---------------------------------
+ *             cosh(x) cos(y) + i sinh(x) sin(y)
+ *
+ *             cosh(x) sinh(x) / cos^2(y) + i tan(y)
+ *           = -------------------------------------
+ *                    1 + sinh^2(x) / cos^2(y)
+ *
+ *             beta rho s + i t
+ *           = ----------------
+ *               1 + beta s^2
+ *
+ * Modifications:
+ *
+ *   I omitted the original algorithm's handling of overflow in tan(x) after
+ *   verifying with nearpi.c that this can't happen in IEEE single or double
+ *   precision.  I also handle large x differently.
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<double> ctanh(const complex<double>& z) {
+  double x, y;
+  double t, beta, s, rho, denom;
+  uint32_t hx, ix, lx;
+
+  x = z.real();
+  y = z.imag();
+
+  extract_words(hx, lx, x);
+  ix = hx & 0x7fffffff;
+
+  /*
+   * ctanh(NaN + i 0) = NaN + i 0
+   *
+   * ctanh(NaN + i y) = NaN + i NaN		for y != 0
+   *
+   * The imaginary part has the sign of x*sin(2*y), but there's no
+   * special effort to get this right.
+   *
+   * ctanh(+-Inf +- i Inf) = +-1 +- 0
+   *
+   * ctanh(+-Inf + i y) = +-1 + 0 sin(2y)		for y finite
+   *
+   * The imaginary part of the sign is unspecified.  This special
+   * case is only needed to avoid a spurious invalid exception when
+   * y is infinite.
+   */
+  if (ix >= 0x7ff00000) {
+    if ((ix & 0xfffff) | lx) /* x is NaN */
+      return (complex<double>(x, (y == 0 ? y : x * y)));
+    set_high_word(x, hx - 0x40000000); /* x = copysign(1, x) */
+    return (complex<double>(x, copysign(0.0, isinf(y) ? y : sin(y) * cos(y))));
+  }
+
+  /*
+   * ctanh(x + i NAN) = NaN + i NaN
+   * ctanh(x +- i Inf) = NaN + i NaN
+   */
+  if (!isfinite(y)) return (complex<double>(y - y, y - y));
+
+  /*
+   * ctanh(+-huge + i +-y) ~= +-1 +- i 2sin(2y)/exp(2x), using the
+   * approximation sinh^2(huge) ~= exp(2*huge) / 4.
+   * We use a modified formula to avoid spurious overflow.
+   */
+  if (ix >= 0x40360000) { /* x >= 22 */
+    double exp_mx = exp(-fabs(x));
+    return (complex<double>(copysign(1.0, x),
+                            4.0 * sin(y) * cos(y) * exp_mx * exp_mx));
+  }
+
+  /* Kahan's algorithm */
+  t = tan(y);
+  beta = 1.0 + t * t; /* = 1 / cos^2(y) */
+  s = sinh(x);
+  rho = sqrt(1.0 + s * s); /* = cosh(x) */
+  denom = 1.0 + beta * s * s;
+  return (complex<double>((beta * rho * s) / denom, t / denom));
+}
+
+__host__ __device__ inline complex<double> ctan(complex<double> z) {
+  /* ctan(z) = -I * ctanh(I * z) */
+  z = ctanh(complex<double>(-z.imag(), z.real()));
+  return (complex<double>(z.imag(), -z.real()));
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> tan(const complex<ValueType>& z) {
+  return sin(z) / cos(z);
+}
+
+template <typename ValueType>
+__host__ __device__ inline complex<ValueType> tanh(const complex<ValueType>& z) {
+  // This implementation seems better than the simple sin/cos
+  return (thrust::exp(ValueType(2) * z) - ValueType(1)) /
+         (thrust::exp(ValueType(2) * z) + ValueType(1));
+}
+
+template <>
+__host__ __device__ inline complex<double> tan(const complex<double>& z) {
+  return detail::complex::ctan(z);
+}
+
+template <>
+__host__ __device__ inline complex<double> tanh(const complex<double>& z) {
+  return detail::complex::ctanh(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/ctanhf.h b/cupy/_core/include/cupy/complex/ctanhf.h
new file mode 100644
index 0000000..8a0acc7
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/ctanhf.h
@@ -0,0 +1,116 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2011 David Schultz
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Adapted from FreeBSD by Filipe Maia, filipe.c.maia@gmail.com:
+ *    freebsd/lib/msun/src/s_ctanhf.c
+ */
+
+/*
+ * Hyperbolic tangent of a complex argument z.  See ctanh.c for details.
+ */
+
+#pragma once
+
+#include <cupy/complex/complex.h>
+#include <cupy/complex/math_private.h>
+
+namespace thrust {
+namespace detail {
+namespace complex {
+
+using thrust::complex;
+
+__host__ __device__ inline complex<float> ctanhf(const complex<float>& z) {
+  float x, y;
+  float t, beta, s, rho, denom;
+  uint32_t hx, ix;
+
+  x = z.real();
+  y = z.imag();
+
+  get_float_word(hx, x);
+  ix = hx & 0x7fffffff;
+
+  if (ix >= 0x7f800000) {
+    if (ix & 0x7fffff) return (complex<float>(x, (y == 0.0f ? y : x * y)));
+    set_float_word(x, hx - 0x40000000);
+    return (complex<float>(x, copysignf(0, isinf(y) ? y : sinf(y) * cosf(y))));
+  }
+
+  if (!isfinite(y)) return (complex<float>(y - y, y - y));
+
+  if (ix >= 0x41300000) { /* x >= 11 */
+    float exp_mx = expf(-fabsf(x));
+    return (complex<float>(copysignf(1.0f, x),
+                           4.0f * sinf(y) * cosf(y) * exp_mx * exp_mx));
+  }
+
+  t = tanf(y);
+  beta = 1.0f + t * t;
+  s = sinhf(x);
+  rho = sqrtf(1.0f + s * s);
+  denom = 1.0f + beta * s * s;
+  return (complex<float>((beta * rho * s) / denom, t / denom));
+}
+
+__host__ __device__ inline complex<float> ctanf(complex<float> z) {
+  z = ctanhf(complex<float>(-z.imag(), z.real()));
+  return (complex<float>(z.imag(), -z.real()));
+}
+
+}  // namespace complex
+
+}  // namespace detail
+
+template <>
+__host__ __device__ inline complex<float> tan(const complex<float>& z) {
+  return detail::complex::ctanf(z);
+}
+
+template <>
+__host__ __device__ inline complex<float> tanh(const complex<float>& z) {
+  return detail::complex::ctanhf(z);
+}
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/complex/math_private.h b/cupy/_core/include/cupy/complex/math_private.h
new file mode 100644
index 0000000..7784ca1
--- /dev/null
+++ b/cupy/_core/include/cupy/complex/math_private.h
@@ -0,0 +1,192 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *  Copyright 2013 Filipe RNC Maia
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * ====================================================
+ * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+ *
+ * Developed at SunPro, a Sun Microsystems, Inc. business.
+ * Permission to use, copy, modify, and distribute this
+ * software is freely granted, provided that this notice
+ * is preserved.
+ * ====================================================
+ */
+
+/* adapted from FreeBSD:
+ *    lib/msun/src/math_private.h
+ */
+
+#pragma once
+
+#if defined(_MSC_VER)  // see #6823
+#include <cfloat>
+#endif  // defined(_MSC_VER)
+
+namespace thrust {
+
+#if !defined(_MSC_VER)  // see #6823
+const float FLT_MIN = 1.17549435e-38F;
+const float FLT_MAX = 3.40282347e+38F;
+const float FLT_EPSILON = 1.19209290e-07F;
+const int FLT_MAX_EXP = 128;
+const int FLT_MANT_DIG = 24;
+
+const double DBL_MIN = 2.2250738585072014e-308;
+const double DBL_MAX = 1.7976931348623157e+308;
+const double DBL_EPSILON = 2.2204460492503131e-16;
+const int DBL_MAX_EXP = 1024;
+const int DBL_MANT_DIG = 53;
+#endif  // !defined(_MSC_VER)
+
+namespace detail {
+namespace complex {
+
+typedef int int32_t;
+typedef unsigned int uint32_t;
+typedef long long int64_t;
+typedef unsigned long long uint64_t;
+
+typedef union {
+  float value;
+  uint32_t word;
+} ieee_float_shape_type;
+
+__host__ __device__ inline void get_float_word(uint32_t& i, float d) {
+  ieee_float_shape_type gf_u;
+  gf_u.value = (d);
+  (i) = gf_u.word;
+}
+
+__host__ __device__ inline void get_float_word(int32_t& i, float d) {
+  ieee_float_shape_type gf_u;
+  gf_u.value = (d);
+  (i) = gf_u.word;
+}
+
+__host__ __device__ inline void set_float_word(float& d, uint32_t i) {
+  ieee_float_shape_type sf_u;
+  sf_u.word = (i);
+  (d) = sf_u.value;
+}
+
+// Assumes little endian ordering
+typedef union {
+  double value;
+  struct {
+    uint32_t lsw;
+    uint32_t msw;
+  } parts;
+  struct {
+    uint64_t w;
+  } xparts;
+} ieee_double_shape_type;
+
+__host__ __device__ inline void get_high_word(uint32_t& i, double d) {
+  ieee_double_shape_type gh_u;
+  gh_u.value = (d);
+  (i) = gh_u.parts.msw;
+}
+
+/* Set the more significant 32 bits of a double from an int.  */
+__host__ __device__ inline void set_high_word(double& d, uint32_t v) {
+  ieee_double_shape_type sh_u;
+  sh_u.value = (d);
+  sh_u.parts.msw = (v);
+  (d) = sh_u.value;
+}
+
+__host__ __device__ inline void insert_words(double& d, uint32_t ix0, uint32_t ix1) {
+  ieee_double_shape_type iw_u;
+  iw_u.parts.msw = (ix0);
+  iw_u.parts.lsw = (ix1);
+  (d) = iw_u.value;
+}
+
+/* Get two 32 bit ints from a double.  */
+__host__ __device__ inline void extract_words(uint32_t& ix0, uint32_t& ix1, double d) {
+  ieee_double_shape_type ew_u;
+  ew_u.value = (d);
+  (ix0) = ew_u.parts.msw;
+  (ix1) = ew_u.parts.lsw;
+}
+
+/* Get two 32 bit ints from a double.  */
+__host__ __device__ inline void extract_words(int32_t& ix0, int32_t& ix1, double d) {
+  ieee_double_shape_type ew_u;
+  ew_u.value = (d);
+  (ix0) = ew_u.parts.msw;
+  (ix1) = ew_u.parts.lsw;
+}
+
+template <typename T>
+inline __host__ __device__ T infinity();
+
+template <>
+inline __host__ __device__ float infinity<float>() {
+  float res;
+  set_float_word(res, 0x7f800000);
+  return res;
+}
+
+template <>
+inline __host__ __device__ double infinity<double>() {
+  double res;
+  insert_words(res, 0x7ff00000, 0);
+  return res;
+}
+
+using ::abs;
+using ::log;
+using ::acos;
+using ::asin;
+using ::sqrt;
+using ::sinh;
+using ::tan;
+using ::cos;
+using ::sin;
+using ::exp;
+using ::cosh;
+using ::atan;
+using ::atanh;
+using ::isinf;
+using ::isnan;
+using ::signbit;
+using ::isfinite;
+
+}  // namespace complex
+
+}  // namespace detail
+
+using ::abs;
+using ::log;
+using ::acos;
+using ::asin;
+using ::sqrt;
+using ::sinh;
+using ::tan;
+using ::cos;
+using ::sin;
+using ::exp;
+using ::cosh;
+using ::atan;
+using ::atanh;
+using ::isinf;
+using ::isnan;
+using ::signbit;
+using ::isfinite;
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/cuComplex_bridge.h b/cupy/_core/include/cupy/cuComplex_bridge.h
new file mode 100644
index 0000000..6e57276
--- /dev/null
+++ b/cupy/_core/include/cupy/cuComplex_bridge.h
@@ -0,0 +1,34 @@
+/* 
+   This header is to support the "translate_cucomplex" option
+   that turns cuComplex function calls to their Thrust counterparts.
+*/ 
+
+/* ------------------- single complex ------------------- */
+#define cuFloatComplex                complex<float>
+#define cuComplex                     complex<float>
+#define cuCrealf                      real
+#define cuCimagf                      imag
+#define make_cuFloatComplex(A, B)     complex<float>(A, B)
+#define make_cuComplex(A, B)          complex<float>(A, B)
+#define cuConjf                       conj
+#define cuCaddf(A, B)                 (A + B)
+#define cuCsubf(A, B)                 (A - B)
+#define cuCmulf(A, B)                 (A * B)
+#define cuCdivf(A, B)                 (A / B)
+#define cuCabsf                       abs
+#define cuComplexDoubleToFloat        complex<float>
+#define cuCfmaf(A, B, C)              (A * B + C)
+
+/* ------------------- double complex ------------------- */
+#define cuDoubleComplex               complex<double>
+#define cuCreal                       real
+#define cuCimag                       imag
+#define make_cuDoubleComplex(A, B)    complex<double>(A, B)
+#define cuConj                        conj
+#define cuCadd(A, B)                  (A + B)
+#define cuCsub(A, B)                  (A - B)
+#define cuCmul(A, B)                  (A * B)
+#define cuCdiv(A, B)                  (A / B)
+#define cuCabs                        abs
+#define cuComplexFloatToDouble        complex<double>
+#define cuCfma(A, B, C)               (A * B + C)
diff --git a/cupy/_core/include/cupy/cub/.gitattributes b/cupy/_core/include/cupy/cub/.gitattributes
new file mode 100644
index 0000000..c6a031b
--- /dev/null
+++ b/cupy/_core/include/cupy/cub/.gitattributes
@@ -0,0 +1 @@
+cub symlink=dir
diff --git a/cupy/_core/include/cupy/cub/LICENSE.TXT b/cupy/_core/include/cupy/cub/LICENSE.TXT
new file mode 120000
index 0000000..d7c89da
--- /dev/null
+++ b/cupy/_core/include/cupy/cub/LICENSE.TXT
@@ -0,0 +1 @@
+../../../../../third_party/cub/LICENSE.TXT
\ No newline at end of file
diff --git a/cupy/_core/include/cupy/cub/cub b/cupy/_core/include/cupy/cub/cub
new file mode 120000
index 0000000..12d0d58
--- /dev/null
+++ b/cupy/_core/include/cupy/cub/cub
@@ -0,0 +1 @@
+../../../../../third_party/cub/cub
\ No newline at end of file
diff --git a/cupy/_core/include/cupy/cuda_workaround.h b/cupy/_core/include/cupy/cuda_workaround.h
new file mode 100644
index 0000000..74c2117
--- /dev/null
+++ b/cupy/_core/include/cupy/cuda_workaround.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#ifdef __CUDACC_RTC__
+// cudaDeviceSynchronize() is no longer supported by nvrtc in device code on
+// H100 GPUs or any GPUs in CUDA 12.x. cudaDeviceSynchronize() is used in CUB
+// bundled with CuPy, resulting in comiplation error when GPU is H100 or later,
+// or CUDA version is 12 or later.
+#if __CUDA_ARCH__ >= 900
+cudaError_t cudaDeviceSynchronize() { return cudaSuccess; }
+#elif __CUDACC_VER_MAJOR__ >= 12
+cudaError_t cudaDeviceSynchronize() { return cudaSuccess; }
+#endif
+#endif
diff --git a/cupy/_core/include/cupy/dlpack/README.md b/cupy/_core/include/cupy/dlpack/README.md
new file mode 100644
index 0000000..6d50eff
--- /dev/null
+++ b/cupy/_core/include/cupy/dlpack/README.md
@@ -0,0 +1,4 @@
+## DLPack header
+
+The header `dlpack.h` is downloaded from https://github.com/dmlc/dlpack/blob/main/include/dlpack/dlpack.h.
+The commit is [`365b823`](https://github.com/dmlc/dlpack/commit/365b823cedb281cd0240ca601aba9b78771f91a3).
diff --git a/cupy/_core/include/cupy/dlpack/dlpack.h b/cupy/_core/include/cupy/dlpack/dlpack.h
new file mode 100644
index 0000000..6d51801
--- /dev/null
+++ b/cupy/_core/include/cupy/dlpack/dlpack.h
@@ -0,0 +1,232 @@
+/*!
+ *  Copyright (c) 2017 by Contributors
+ * \file dlpack.h
+ * \brief The common header of DLPack.
+ */
+#ifndef DLPACK_DLPACK_H_
+#define DLPACK_DLPACK_H_
+
+/**
+ * \brief Compatibility with C++
+ */
+#ifdef __cplusplus
+#define DLPACK_EXTERN_C extern "C"
+#else
+#define DLPACK_EXTERN_C
+#endif
+
+/*! \brief The current version of dlpack */
+#define DLPACK_VERSION 80
+
+/*! \brief The current ABI version of dlpack */
+#define DLPACK_ABI_VERSION 1
+
+/*! \brief DLPACK_DLL prefix for windows */
+#ifdef _WIN32
+#ifdef DLPACK_EXPORTS
+#define DLPACK_DLL __declspec(dllexport)
+#else
+#define DLPACK_DLL __declspec(dllimport)
+#endif
+#else
+#define DLPACK_DLL
+#endif
+
+#include <stdint.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*!
+ * \brief The device type in DLDevice.
+ */
+#ifdef __cplusplus
+typedef enum : int32_t {
+#else
+typedef enum {
+#endif
+  /*! \brief CPU device */
+  kDLCPU = 1,
+  /*! \brief CUDA GPU device */
+  kDLCUDA = 2,
+  /*!
+   * \brief Pinned CUDA CPU memory by cudaMallocHost
+   */
+  kDLCUDAHost = 3,
+  /*! \brief OpenCL devices. */
+  kDLOpenCL = 4,
+  /*! \brief Vulkan buffer for next generation graphics. */
+  kDLVulkan = 7,
+  /*! \brief Metal for Apple GPU. */
+  kDLMetal = 8,
+  /*! \brief Verilog simulator buffer */
+  kDLVPI = 9,
+  /*! \brief ROCm GPUs for AMD GPUs */
+  kDLROCM = 10,
+  /*!
+   * \brief Pinned ROCm CPU memory allocated by hipMallocHost
+   */
+  kDLROCMHost = 11,
+  /*!
+   * \brief Reserved extension device type,
+   * used for quickly test extension device
+   * The semantics can differ depending on the implementation.
+   */
+  kDLExtDev = 12,
+  /*!
+   * \brief CUDA managed/unified memory allocated by cudaMallocManaged
+   */
+  kDLCUDAManaged = 13,
+  /*!
+   * \brief Unified shared memory allocated on a oneAPI non-partititioned
+   * device. Call to oneAPI runtime is required to determine the device
+   * type, the USM allocation type and the sycl context it is bound to.
+   *
+   */
+  kDLOneAPI = 14,
+  /*! \brief GPU support for next generation WebGPU standard. */
+  kDLWebGPU = 15,
+  /*! \brief Qualcomm Hexagon DSP */
+  kDLHexagon = 16,
+} DLDeviceType;
+
+/*!
+ * \brief A Device for Tensor and operator.
+ */
+typedef struct {
+  /*! \brief The device type used in the device. */
+  DLDeviceType device_type;
+  /*!
+   * \brief The device index.
+   * For vanilla CPU memory, pinned memory, or managed memory, this is set to 0.
+   */
+  int32_t device_id;
+} DLDevice;
+
+/*!
+ * \brief The type code options DLDataType.
+ */
+typedef enum {
+  /*! \brief signed integer */
+  kDLInt = 0U,
+  /*! \brief unsigned integer */
+  kDLUInt = 1U,
+  /*! \brief IEEE floating point */
+  kDLFloat = 2U,
+  /*!
+   * \brief Opaque handle type, reserved for testing purposes.
+   * Frameworks need to agree on the handle data type for the exchange to be well-defined.
+   */
+  kDLOpaqueHandle = 3U,
+  /*! \brief bfloat16 */
+  kDLBfloat = 4U,
+  /*!
+   * \brief complex number
+   * (C/C++/Python layout: compact struct per complex number)
+   */
+  kDLComplex = 5U,
+  /*! \brief boolean */
+  kDLBool = 6U,
+} DLDataTypeCode;
+
+/*!
+ * \brief The data type the tensor can hold. The data type is assumed to follow the
+ * native endian-ness. An explicit error message should be raised when attempting to
+ * export an array with non-native endianness
+ *
+ *  Examples
+ *   - float: type_code = 2, bits = 32, lanes = 1
+ *   - float4(vectorized 4 float): type_code = 2, bits = 32, lanes = 4
+ *   - int8: type_code = 0, bits = 8, lanes = 1
+ *   - std::complex<float>: type_code = 5, bits = 64, lanes = 1
+ *   - bool: type_code = 6, bits = 8, lanes = 1 (as per common array library convention, the underlying storage size of bool is 8 bits)
+ */
+typedef struct {
+  /*!
+   * \brief Type code of base types.
+   * We keep it uint8_t instead of DLDataTypeCode for minimal memory
+   * footprint, but the value should be one of DLDataTypeCode enum values.
+   * */
+  uint8_t code;
+  /*!
+   * \brief Number of bits, common choices are 8, 16, 32.
+   */
+  uint8_t bits;
+  /*! \brief Number of lanes in the type, used for vector types. */
+  uint16_t lanes;
+} DLDataType;
+
+/*!
+ * \brief Plain C Tensor object, does not manage memory.
+ */
+typedef struct {
+  /*!
+   * \brief The data pointer points to the allocated data. This will be CUDA
+   * device pointer or cl_mem handle in OpenCL. It may be opaque on some device
+   * types. This pointer is always aligned to 256 bytes as in CUDA. The
+   * `byte_offset` field should be used to point to the beginning of the data.
+   *
+   * Note that as of Nov 2021, multiply libraries (CuPy, PyTorch, TensorFlow,
+   * TVM, perhaps others) do not adhere to this 256 byte aligment requirement
+   * on CPU/CUDA/ROCm, and always use `byte_offset=0`.  This must be fixed
+   * (after which this note will be updated); at the moment it is recommended
+   * to not rely on the data pointer being correctly aligned.
+   *
+   * For given DLTensor, the size of memory required to store the contents of
+   * data is calculated as follows:
+   *
+   * \code{.c}
+   * static inline size_t GetDataSize(const DLTensor* t) {
+   *   size_t size = 1;
+   *   for (tvm_index_t i = 0; i < t->ndim; ++i) {
+   *     size *= t->shape[i];
+   *   }
+   *   size *= (t->dtype.bits * t->dtype.lanes + 7) / 8;
+   *   return size;
+   * }
+   * \endcode
+   */
+  void* data;
+  /*! \brief The device of the tensor */
+  DLDevice device;
+  /*! \brief Number of dimensions */
+  int32_t ndim;
+  /*! \brief The data type of the pointer*/
+  DLDataType dtype;
+  /*! \brief The shape of the tensor */
+  int64_t* shape;
+  /*!
+   * \brief strides of the tensor (in number of elements, not bytes)
+   *  can be NULL, indicating tensor is compact and row-majored.
+   */
+  int64_t* strides;
+  /*! \brief The offset in bytes to the beginning pointer to data */
+  uint64_t byte_offset;
+} DLTensor;
+
+/*!
+ * \brief C Tensor object, manage memory of DLTensor. This data structure is
+ *  intended to facilitate the borrowing of DLTensor by another framework. It is
+ *  not meant to transfer the tensor. When the borrowing framework doesn't need
+ *  the tensor, it should call the deleter to notify the host that the resource
+ *  is no longer needed.
+ */
+typedef struct DLManagedTensor {
+  /*! \brief DLTensor which is being memory managed */
+  DLTensor dl_tensor;
+  /*! \brief the context of the original host framework of DLManagedTensor in
+   *   which DLManagedTensor is used in the framework. It can also be NULL.
+   */
+  void * manager_ctx;
+  /*! \brief Destructor signature void (*)(void*) - this should be called
+   *   to destruct manager_ctx which holds the DLManagedTensor. It can be NULL
+   *   if there is no way for the caller to provide a reasonable destructor.
+   *   The destructors deletes the argument self as well.
+   */
+  void (*deleter)(struct DLManagedTensor * self);
+} DLManagedTensor;
+#ifdef __cplusplus
+}  // DLPACK_EXTERN_C
+#endif
+#endif  // DLPACK_DLPACK_H_
diff --git a/cupy/_core/include/cupy/hip_workaround.cuh b/cupy/_core/include/cupy/hip_workaround.cuh
new file mode 100644
index 0000000..883fb10
--- /dev/null
+++ b/cupy/_core/include/cupy/hip_workaround.cuh
@@ -0,0 +1,20 @@
+#ifndef INCLUDE_GUARD_CUPY_HIP_WORKAROUND_H
+#define INCLUDE_GUARD_CUPY_HIP_WORKAROUND_H
+
+#ifdef __HIP_DEVICE_COMPILE__
+
+// ignore mask
+#define __shfl_sync(mask, ...) __shfl(__VA_ARGS__)
+#define __shfl_up_sync(mask, ...) __shfl_up(__VA_ARGS__)
+#define __shfl_down_sync(mask, ...) __shfl_down(__VA_ARGS__)
+#define __shfl_xor_sync(mask, ...) __shfl_xor(__VA_ARGS__)
+
+// In ROCm, threads in a warp march in lock-step, so we don't need to
+// synchronize the threads. But it doesn't guarantee the memory order,
+// which still make us use memory fences.
+// https://rocmdocs.amd.com/en/latest/Programming_Guides/Kernel_language.html#warp-cross-lane-functions
+#define __syncwarp() { __threadfence_block(); }
+
+#endif  // __HIP_DEVICE_COMPILE__
+
+#endif  // INCLUDE_GUARD_CUPY_HIP_WORKAROUND_H
diff --git a/cupy/_core/include/cupy/jitify/.clang-format b/cupy/_core/include/cupy/jitify/.clang-format
new file mode 100644
index 0000000..4a88069
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/.clang-format
@@ -0,0 +1,149 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:   
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CommentPragmas:  '^\\.+'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:   
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories: 
+  - Regex:           '^<ext/.*\.h>'
+    Priority:        2
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Never
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+RawStringFormats: 
+  - Language:        Cpp
+    Delimiters:      
+      - cc
+      - CC
+      - cpp
+      - Cpp
+      - CPP
+      - 'c++'
+      - 'C++'
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+  - Language:        TextProto
+    Delimiters:      
+      - pb
+      - PB
+      - proto
+      - PROTO
+    EnclosingFunctions: 
+      - EqualsProto
+      - EquivToProto
+      - PARSE_PARTIAL_TEXT_PROTO
+      - PARSE_TEST_PROTO
+      - PARSE_TEXT_PROTO
+      - ParseTextOrDie
+      - ParseTextProtoOrDie
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
+
diff --git a/cupy/_core/include/cupy/jitify/.gitignore b/cupy/_core/include/cupy/jitify/.gitignore
new file mode 100644
index 0000000..c3a73d4
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/.gitignore
@@ -0,0 +1,10 @@
+
+jitify_example
+stringify
+*.jit
+*.o
+jitify_2nd_compilation_unit.cpp
+jitify_test
+googletest/
+# Backup files
+*~
diff --git a/cupy/_core/include/cupy/jitify/Doxyfile b/cupy/_core/include/cupy/jitify/Doxyfile
new file mode 100644
index 0000000..583bc0c
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/Doxyfile
@@ -0,0 +1,2427 @@
+# Doxyfile 1.8.11
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all text
+# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
+# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
+# for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING      = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = Jitify
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         =
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          =
+
+# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
+# in the documentation. The maximum height of the logo should not exceed 55
+# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
+# the logo to the output directory.
+
+PROJECT_LOGO           =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       = doc
+
+# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS         = NO
+
+# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
+# characters to appear in the names of generated files. If set to NO, non-ASCII
+# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
+# U+3044.
+# The default value is: NO.
+
+ALLOW_UNICODE_NAMES    = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE        = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF           = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF       =
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC    = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB  = NO
+
+# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES        = YES
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH        =
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH    =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES            = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF      = NO
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF           = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS           = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new
+# page for each member. If set to NO, the documentation of a member will be part
+# of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES  = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE               = 8
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines.
+
+ALIASES                =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C  = NO
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA   = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN   = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL   = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
+# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
+# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
+# Fortran. In the later case the parser tries to guess whether the code is fixed
+# or free formatted code, this is the default for Fortran type files), VHDL. For
+# instance to make doxygen treat .inc files as Fortran files (default is PHP),
+# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+#
+# Note: For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING      =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT       = YES
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT       = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT    = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT        = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT            = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT   = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC   = NO
+
+# If one adds a struct or class to a group and this option is enabled, then also
+# any nested class or struct is added to the same group. By default this option
+# is disabled and one has to add nested compounds explicitly via \ingroup.
+# The default value is: NO.
+
+GROUP_NESTED_COMPOUNDS = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING            = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS  = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT   = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE      = 0
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL            = NO
+
+# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE        = NO
+
+# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE        = NO
+
+# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC         = NO
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO,
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES  = YES
+
+# This flag is only useful for Objective-C code. If set to YES, local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO, only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS  = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES   = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES     = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# (class|struct|union) declarations. If set to NO, these declarations will be
+# included in the documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS  = YES
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO, these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS      = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS          = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES, upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# and Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES       = YES
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES, the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES       = NO
+
+# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will
+# append additional text to a page's title, such as Class Reference. If set to
+# YES the compound reference will be hidden.
+# The default value is: NO.
+
+HIDE_COMPOUND_REFERENCE= NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES     = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC  = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES   = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO            = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS       = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS        = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES       = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME     = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING  = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo
+# list. This list is created by putting \todo commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST      = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
+# list. This list is created by putting \test commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST      = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST       = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS       =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES  = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES, the
+# list will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES        = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES             = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES        = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER    =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE            =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. See also \cite for info how to create references.
+
+CITE_BIB_FILES         =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET                  = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS               = YES
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED   = YES
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR      = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO, doxygen will only warn about wrong or incomplete
+# parameter documentation, but not about the absence of documentation.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC       = NO
+
+# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
+# a warning is encountered.
+# The default value is: NO.
+
+WARN_AS_ERROR          = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT            = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE           =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+INPUT                  = jitify.hpp
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl,
+# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js.
+
+FILE_PATTERNS          =
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE              = NO
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       =
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS        =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH           =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS       =
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE      = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH             =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+INPUT_FILTER           =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+FILTER_PATTERNS        =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES    = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER         = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES         = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS    = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# function all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = YES
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION    = YES
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS        = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see http://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS              = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS       = YES
+
+# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the
+# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the
+# cost of reduced performance. This can be particularly helpful with template
+# rich C++ code for which doxygen's built-in parser lacks the necessary type
+# information.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse-libclang=ON option for CMake.
+# The default value is: NO.
+
+CLANG_ASSISTED_PARSING = NO
+
+# If clang assisted parsing is enabled you can provide the compiler with command
+# line options that you would normally use when invoking the compiler. Note that
+# the include paths will already be set by doxygen for the files and directories
+# specified with INPUT and INCLUDE_PATH.
+# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
+
+CLANG_OPTIONS          =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX     = NO
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX    = 5
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX          =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML          = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT            = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION    = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER            =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER            =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET        =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# cascading style sheets that are included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefore more robust against future updates.
+# Doxygen will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list). For an example see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET  =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES       =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the style sheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE    = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT    = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA  = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to YES can help to show when doxygen was last run and thus if the
+# documentation is up to date.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP         = NO
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS  = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: http://developer.apple.com/tools/xcode/), introduced with
+# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
+# for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET        = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME        = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID       = org.doxygen.Project
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID    = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME  = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP      = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE               =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler (hhc.exe). If non-empty,
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION           =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated
+# (YES) or that it should be included in the master .chm file (NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI           = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING     =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated
+# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
+# enables the Previous and Next buttons.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC             = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND             = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP           = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE               =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE          = org.doxygen.Project
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER     = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME   =
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS  =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS  =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION           =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP   = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID         = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX          = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW      = NO
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH         = 250
+
+# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW    = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE       = 10
+
+# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT    = YES
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# http://www.mathjax.org) which uses client side Javascript for the rendering
+# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX            = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT         = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from http://www.mathjax.org before deployment.
+# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS     =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE       =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE           = NO
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using Javascript. There
+# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
+# setting. When disabled, doxygen will generate a PHP script for searching and
+# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
+# and searching needs to be provided by external tools. See the section
+# "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH    = NO
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH        = NO
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL       =
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE        = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID     =
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS  =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = YES
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT           = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when enabling USE_PDFLATEX this option is only used for generating
+# bitmaps for formulas in the HTML output, but not in the Makefile that is
+# written to the output directory.
+# The default file is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME         = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME     = makeindex
+
+# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX          = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE             = a4wide
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. The package can be specified just
+# by its name or with the correct syntax as to be used with the LaTeX
+# \usepackage command. To get the times font for instance you can specify :
+# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times}
+# To use the option intlimits with the amsmath package you can specify:
+# EXTRA_PACKAGES=[intlimits]{amsmath}
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES         =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
+# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
+# string, for the replacement values of the other commands the user is referred
+# to HTML_HEADER.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER           =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer. See
+# LATEX_HEADER for more information on how to generate a default footer and what
+# special commands can be used inside the footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER           =
+
+# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# LaTeX style sheets that are included after the standard style sheets created
+# by doxygen. Using this option one can overrule certain style aspects. Doxygen
+# will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_STYLESHEET =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES      =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS         = NO
+
+# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES, to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX           = NO
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE        = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES     = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE      = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE        = plain
+
+# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_TIMESTAMP        = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF           = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT             = rtf
+
+# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF            = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS         = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's config
+# file, i.e. a series of assignments. You only have to provide replacements,
+# missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE    =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's config file. A template extensions file can be generated
+# using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE    =
+
+# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
+# with syntax highlighting in the RTF output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_SOURCE_CODE        = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN           = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT             = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION          = .3
+
+# The MAN_SUBDIR tag determines the name of the directory created within
+# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
+# MAN_EXTENSION with the initial . removed.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_SUBDIR             =
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS              = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML           = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT             = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING     = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK       = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT         = docbook
+
+# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
+# program listings (including syntax highlighting and cross-referencing
+# information) to the DOCBOOK output. Note that enabling this will significantly
+# increase the size of the DOCBOOK output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_PROGRAMLISTING = NO
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
+# AutoGen Definitions (see http://autogen.sf.net) file that captures the
+# structure of the code including all documentation. Note that this feature is
+# still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF   = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD       = NO
+
+# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX          = NO
+
+# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO, the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY         = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = NO
+
+# If the SEARCH_INCLUDES tag is set to YES, the include files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES        = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           =
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS  =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED             = __cplusplus=201103L
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have
+# an all uppercase name, and do not end with a semicolon. Such function macros
+# are typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS   = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have a unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES               =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE       =
+
+# If the ALLEXTERNALS tag is set to YES, all external class will be listed in
+# the class index. If set to NO, only the inherited external classes will be
+# listed.
+# The default value is: NO.
+
+ALLEXTERNALS           = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS        = YES
+
+# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES         = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script
+# interpreter (i.e. the result of 'which perl').
+# The default file (with absolute path) is: /usr/bin/perl.
+
+PERL_PATH              = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS         = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see:
+# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
+# default search path.
+
+MSCGEN_PATH            =
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH               =
+
+# If set to YES the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS   = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: YES.
+
+HAVE_DOT               = NO
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS        = 0
+
+# When you want a differently looking font in the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME           = Helvetica
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE           = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH           =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH            = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH    = YES
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS           = YES
+
+# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK               = NO
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS   = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS     = NO
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH          = YES
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH      = YES
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command. Disabling a call graph can be
+# accomplished by means of the command \hidecallgraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH             = NO
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command. Disabling a caller graph can be
+# accomplished by means of the command \hidecallergraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH           = NO
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY    = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH        = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot. For an explanation of the image formats see the section
+# output formats in the documentation of the dot tool (Graphviz (see:
+# http://www.graphviz.org/)).
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,
+# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,
+# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo,
+# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
+# png:gdiplus:gdiplus.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT       = png
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG        = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH               =
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS           =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS           =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS           =
+
+# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
+# path where java can find the plantuml.jar file. If left blank, it is assumed
+# PlantUML is not used or called during a preprocessing step. Doxygen will
+# generate a warning when it encounters a \startuml command in this case and
+# will not generate output for the diagram.
+
+PLANTUML_JAR_PATH      =
+
+# When using plantuml, the specified paths are searched for files specified by
+# the !include statement in a plantuml block.
+
+PLANTUML_INCLUDE_PATH  =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES    = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH    = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT        = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS      = NO
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND        = YES
+
+# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP            = YES
diff --git a/cupy/_core/include/cupy/jitify/LICENSE b/cupy/_core/include/cupy/jitify/LICENSE
new file mode 100644
index 0000000..76d9aee
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/LICENSE
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2017-2020, NVIDIA Corporation
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/cupy/_core/include/cupy/jitify/Makefile b/cupy/_core/include/cupy/jitify/Makefile
new file mode 100644
index 0000000..c683dcf
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/Makefile
@@ -0,0 +1,105 @@
+
+DOXYGEN ?= doxygen
+CXXFLAGS ?= -O3 -Wall -Wconversion -Wextra -Wshadow -g -fmessage-length=80
+
+CXX11 ?= 1
+
+CUDA_DIR ?= /usr/local/cuda
+CUDA_INC_DIR ?= $(CUDA_DIR)/include
+
+CXXFLAGS += -pthread
+
+ifeq ($(CXX11),1)
+	CXXFLAGS += -std=c++11
+endif
+
+EMBED_BEGIN = -rdynamic -Wl,-b,binary,
+EMBED_END   = ,-b,default
+
+NVCC_EMBED_BEGIN = -rdynamic -Wl\,-b\,binary\,
+NVCC_EMBED_END   = \,-b\,default
+
+EMBED =
+NVCC_EMBED =
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+	# Embed a header into the executable (only supported by gcc on Linux)
+	EMBED = $(EMBED_BEGIN)example_headers/my_header2.cuh$(EMBED_END)
+	NVCC_EMBED = $(NVCC_EMBED_BEGIN)example_headers/my_header2.cuh$(NVCC_EMBED_END)
+	CXXFLAGS += -D LINUX
+	CUDA_LIB_DIR = $(CUDA_DIR)/lib64
+else ifeq ($(UNAME_S),Darwin)
+	CUDA_LIB_DIR = $(CUDA_DIR)/lib
+endif
+
+INC += -I$(CUDA_INC_DIR)
+LIB += -ldl -L$(CUDA_LIB_DIR) -lcuda -lcudart -lnvrtc
+
+HEADERS = jitify.hpp \
+          example_headers/my_header1.cuh.jit \
+          example_headers/my_header2.cuh
+
+all: jitify_example
+
+JITIFY_EXAMPLE_DEFINES = -DCUDA_INC_DIR="\"$(CUDA_INC_DIR)\""
+
+jitify_example: jitify_example.cpp $(HEADERS)
+	$(CXX) -o $@ $< $(CXXFLAGS) $(INC) $(LIB) $(EMBED) $(JITIFY_EXAMPLE_DEFINES)
+
+%.jit: % stringify
+	./stringify $< > $@
+
+stringify: stringify.cpp
+	$(CXX) -o $@ $< -O3 -Wall
+
+get-deps:
+	sudo apt-get update
+	# CMake is needed to build gtest.
+	sudo apt-get install -y cmake
+.PHONY: get-deps
+
+GTEST_DIR = googletest
+GTEST_STATIC_LIB = $(GTEST_DIR)/build/googlemock/gtest/libgtest.a
+$(GTEST_STATIC_LIB):
+	rm -rf $(GTEST_DIR)
+	git clone https://github.com/google/googletest.git $(GTEST_DIR)
+	cd $(GTEST_DIR) && git checkout release-1.8.1 && rm -rf build && mkdir build && cd build && cmake .. && make -j8
+
+GTEST_INC = -I$(GTEST_DIR)/googletest/include
+GTEST_LIB = -L$(GTEST_DIR)/build/googlemock/gtest -lgtest -lgtest_main
+
+CUB_DIR ?= /tmp/cub
+CUB_HEADER = $(CUB_DIR)/cub/cub.cuh
+$(CUB_HEADER):
+	rm -rf $(CUB_DIR)
+	git clone https://github.com/NVlabs/cub.git $(CUB_DIR)
+	cd $(CUB_DIR) && git checkout v1.8.0
+
+CUB_INC = -I$(CUB_DIR)
+JITIFY_TEST_DEFINES = -DCUDA_INC_DIR="\"$(CUDA_INC_DIR)\"" -DCUB_DIR="\"$(CUB_DIR)\""
+
+jitify_test: jitify_test.cu $(HEADERS) $(GTEST_STATIC_LIB) $(CUB_HEADER) Makefile
+	# Link a 2nd compilation unit to ensure no multiple definition errors.
+	echo "#include \"jitify.hpp\"\n#include \"example_headers/my_header1.cuh.jit\"" \
+        > jitify_2nd_compilation_unit.cpp
+	nvcc -o $@ $< jitify_2nd_compilation_unit.cpp -rdc=true -std=c++11 -O3 \
+        -Xcompiler "$(CXXFLAGS) $(NVCC_EMBED) -pthread" \
+        $(JITIFY_TEST_DEFINES) $(INC) $(GTEST_INC) $(CUB_INC) $(LIB) $(GTEST_LIB)
+
+test: jitify_test
+	./jitify_test
+.PHONY: test
+
+doc: jitify.hpp Doxyfile
+	$(DOXYGEN) Doxyfile
+.PHONY: doc
+
+clean:
+	rm -f stringify
+	rm -f example_headers/*.jit
+	rm -f jitify_example
+	rm -f jitify_test
+	rm -rf $(GTEST_DIR)
+	rm -rf $(CUB_DIR)
+	rm -f *.o
+.PHONY: clean
diff --git a/cupy/_core/include/cupy/jitify/README.md b/cupy/_core/include/cupy/jitify/README.md
new file mode 100644
index 0000000..5731227
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/README.md
@@ -0,0 +1,116 @@
+
+# Jitify
+
+A single-header C++ library for simplifying the use of CUDA Runtime Compilation (NVRTC).
+
+## Rationale
+
+Integrating NVRTC into existing and/or templated CUDA code can be
+tricky. Jitify aims to simplify this process by hiding the
+complexities behind a simple, high-level interface.
+
+## Quick example
+
+```c++
+const char* program_source = "my_program\n"
+    "template<int N, typename T>\n"
+    "__global__\n"
+    "void my_kernel(T* data) {\n"
+    "    T data0 = data[0];\n"
+    "    for( int i=0; i<N-1; ++i ) {\n"
+    "        data[0] *= data0;\n"
+    "    }\n"
+    "}\n";
+static jitify::JitCache kernel_cache;
+jitify::Program program = kernel_cache.program(program_source);
+// ...set up data etc.
+dim3 grid(1);
+dim3 block(1);
+using jitify::reflection::type_of;
+program.kernel("my_kernel")
+       .instantiate(3, type_of(*data))
+       .configure(grid, block)
+       .launch(data);
+```
+
+## Features
+
+Jitify provides/takes care of the following things:
+
+ * All NVRTC and CUDA Driver API calls
+ * Simple kernel instantiation and launch syntax
+ * Caching compiled kernels
+ * Loading source code from strings, files, or embedded in an executable
+ * Ignoring host code in runtime-compiled sources
+ * Skipping unneeded headers
+ * Support for JIT-safe standard library headers (e.g., float.h, stdint.h etc.)
+ * Dealing with kernel name mangling
+ * Reflecting kernel template parameters into strings
+ * Compiling specifically for the current device's compute capability
+ * Linking to pre-compiled PTX/CUBIN/FATBIN/object/library files
+ * Support for CUDA versions 7.0, 7.5, 8.0, 9.x, 10.x, on both Linux and Windows
+ * Convenient parallel_for function and lambda support
+ * \*New\* jitify::experimental API provides serialization capabilities to enable [user-managed hashing and caching](https://github.com/rapidsai/cudf/blob/v0.12.0/cpp/src/jit/cache.h)
+
+Things you can do with Jitify and NVRTC:
+
+ * *Rapidly port existing code* to use CUDA Runtime Compilation
+ * *Dramatically reduce code volume* and offline-compilation times
+ * *Increase kernel performance* by baking in runtime constants and autotuning
+
+## How to build
+
+Jitify is just a single header file:
+
+```c++
+#include <jitify.hpp>
+```
+
+Compile with: `-pthread` (not needed if JITIFY_THREAD_SAFE is defined to 0)
+
+Link with: `-lcuda -lcudart -lnvrtc`
+
+A small utility called stringify is included for converting text files into
+C string literals, which provides a convenient way to integrate JIT-compiled
+sources into a build.
+
+### Running tests
+
+Tests can be run with the following command:
+
+```shell
+$ make test
+```
+
+This will automatically download and build the
+[GoogleTest](https://github.com/google/googletest) library, which
+requires [CMake](https://cmake.org) to be available on the system.
+
+## Documentation
+
+### Examples
+
+See [jitify_example.cpp](jitify_example.cpp) for some examples of how to use the library.
+The [Makefile](Makefile) also demonstrates how to use the provided stringify utility.
+
+[GTC 2017 Talk by Ben Barsdell and Kate Clark](http://on-demand.gputechconf.com/gtc/2017/video/s7716-barsdell-ben-jitify.mp4)
+
+### API documentation
+
+Doxygen documentation can be generated by running:
+
+```shell
+$ make doc
+```
+
+The HTML and LaTeX results are placed into the doc/ subdirectory.
+
+## License
+
+BSD-3-Clause
+
+## Authors
+
+Ben Barsdell (NVIDIA, bbarsdell at nvidia dot com)
+
+Kate Clark (NVIDIA, mclark at nvidia dot com)
diff --git a/cupy/_core/include/cupy/jitify/example_headers/class_arg_kernel.cuh b/cupy/_core/include/cupy/jitify/example_headers/class_arg_kernel.cuh
new file mode 100644
index 0000000..4059a26
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/example_headers/class_arg_kernel.cuh
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+class Managed {
+ public:
+  void* operator new(size_t len) {
+    void* ptr = nullptr;
+#ifndef __CUDACC_RTC__
+    cudaMallocManaged(&ptr, len);
+#endif
+    cudaDeviceSynchronize();
+    return ptr;
+  }
+
+  void operator delete(void* ptr) {
+    cudaDeviceSynchronize();
+    cudaFree(ptr);
+  }
+};
+
+struct Arg : public Managed {
+  const int x;
+  Arg(int x_) : x(x_) {}
+
+  // there can be no call to the copy constructor
+  Arg(const Arg& arg) = delete;
+};
+
+template <typename T>
+__global__ void class_arg_kernel(int* x, T arg) {
+  *x = arg.x;
+}
+
+template <typename T>
+__global__ void class_arg_ref_kernel(int* x, T& arg) {
+  *x = arg.x;
+}
+
+template <typename T>
+__global__ void class_arg_ptr_kernel(int* x, T* arg) {
+  *x = arg->x;
+}
diff --git a/cupy/_core/include/cupy/jitify/example_headers/constant_header.cuh b/cupy/_core/include/cupy/jitify/example_headers/constant_header.cuh
new file mode 100644
index 0000000..8879cb4
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/example_headers/constant_header.cuh
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+namespace {
+
+namespace b {
+__constant__ int a[3];
+__device__ int d[3];
+}  // namespace b
+
+}  // namespace
+
+__global__ void constant_test2(int* x) {
+  for (int i = 0; i < 3; i++) x[i] = (b::a[i]);
+  for (int i = 0; i < 3; i++) x[i + 3] = (b::d[i]);
+}
diff --git a/cupy/_core/include/cupy/jitify/example_headers/my_header1.cuh b/cupy/_core/include/cupy/jitify/example_headers/my_header1.cuh
new file mode 100644
index 0000000..92deddf
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/example_headers/my_header1.cuh
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+template <typename T>
+__device__ inline T square(T x) {
+  return x * x;
+}
diff --git a/cupy/_core/include/cupy/jitify/example_headers/my_header2.cuh b/cupy/_core/include/cupy/jitify/example_headers/my_header2.cuh
new file mode 100644
index 0000000..cd793ae
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/example_headers/my_header2.cuh
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+template <typename T>
+T negate(T x) {
+  return -x;
+}
diff --git a/cupy/_core/include/cupy/jitify/example_headers/my_header3.cuh b/cupy/_core/include/cupy/jitify/example_headers/my_header3.cuh
new file mode 100644
index 0000000..81fd018
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/example_headers/my_header3.cuh
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+template <typename T>
+T identity(T x) {
+  return x;
+}
diff --git a/cupy/_core/include/cupy/jitify/jitify.hpp b/cupy/_core/include/cupy/jitify/jitify.hpp
new file mode 100644
index 0000000..29de298
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/jitify.hpp
@@ -0,0 +1,4454 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+  -----------
+  Jitify 0.9
+  -----------
+  A C++ library for easy integration of CUDA runtime compilation into
+  existing codes.
+
+  --------------
+  How to compile
+  --------------
+  Compiler dependencies: <jitify.hpp>, -std=c++11
+  Linker dependencies:   dl cuda nvrtc
+
+  --------------------------------------
+  Embedding source files into executable
+  --------------------------------------
+  g++  ... -ldl -rdynamic -DJITIFY_ENABLE_EMBEDDED_FILES=1
+  -Wl,-b,binary,my_kernel.cu,include/my_header.cuh,-b,default nvcc ... -ldl
+  -Xcompiler "-rdynamic
+  -Wl\,-b\,binary\,my_kernel.cu\,include/my_header.cuh\,-b\,default"
+  JITIFY_INCLUDE_EMBEDDED_FILE(my_kernel_cu);
+  JITIFY_INCLUDE_EMBEDDED_FILE(include_my_header_cuh);
+
+  ----
+  TODO
+  ----
+  Extract valid compile options and pass the rest to cuModuleLoadDataEx
+  See if can have stringified headers automatically looked-up
+    by having stringify add them to a (static) global map.
+    The global map can be updated by creating a static class instance
+      whose constructor performs the registration.
+    Can then remove all headers from JitCache constructor in example code
+  See other TODOs in code
+*/
+
+/*! \file jitify.hpp
+ *  \brief The Jitify library header
+ */
+
+/*! \mainpage Jitify - A C++ library that simplifies the use of NVRTC
+ *  \p Use class jitify::JitCache to manage and launch JIT-compiled CUDA
+ *    kernels.
+ *
+ *  \p Use namespace jitify::reflection to reflect types and values into
+ *    code-strings.
+ *
+ *  \p Use JITIFY_INCLUDE_EMBEDDED_FILE() to declare files that have been
+ *  embedded into the executable using the GCC linker.
+ *
+ *  \p Use jitify::parallel_for and JITIFY_LAMBDA() to generate and launch
+ *  simple kernels.
+ */
+
+#pragma once
+
+#ifndef JITIFY_THREAD_SAFE
+#define JITIFY_THREAD_SAFE 1
+#endif
+
+#if JITIFY_ENABLE_EMBEDDED_FILES
+#include <dlfcn.h>
+#endif
+#include <stdint.h>
+#include <algorithm>
+#include <cctype>
+#include <cstring>  // For strtok_r etc.
+#include <deque>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <typeinfo>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+#if JITIFY_THREAD_SAFE
+#include <mutex>
+#endif
+
+#include <cuda.h>
+#include <cuda_runtime_api.h>  // For dim3, cudaStream_t
+#if CUDA_VERSION >= 8000
+#define NVRTC_GET_TYPE_NAME 1
+#endif
+#include <nvrtc.h>
+
+// For use by get_current_executable_path().
+#ifdef __linux__
+#include <linux/limits.h>  // For PATH_MAX
+
+#include <cstdlib>  // For realpath
+#define JITIFY_PATH_MAX PATH_MAX
+#elif defined(_WIN32) || defined(_WIN64)
+#include <windows.h>
+#define JITIFY_PATH_MAX MAX_PATH
+#else
+#error "Unsupported platform"
+#endif
+
+#ifdef _MSC_VER       // MSVC compiler
+#include <dbghelp.h>  // For UnDecorateSymbolName
+#else
+#include <cxxabi.h>  // For abi::__cxa_demangle
+#endif
+
+#if defined(_WIN32) || defined(_WIN64)
+// WAR for strtok_r being called strtok_s on Windows
+#pragma push_macro("strtok_r")
+#undef strtok_r
+#define strtok_r strtok_s
+// WAR for min and max possibly being macros defined by windows.h
+#pragma push_macro("min")
+#pragma push_macro("max")
+#undef min
+#undef max
+#endif
+
+#ifndef JITIFY_PRINT_LOG
+#define JITIFY_PRINT_LOG 1
+#endif
+
+#if JITIFY_PRINT_ALL
+#define JITIFY_PRINT_INSTANTIATION 1
+#define JITIFY_PRINT_SOURCE 1
+#define JITIFY_PRINT_LOG 1
+#define JITIFY_PRINT_PTX 1
+#define JITIFY_PRINT_LINKER_LOG 1
+#define JITIFY_PRINT_LAUNCH 1
+#define JITIFY_PRINT_HEADER_PATHS 1
+#endif
+
+#if JITIFY_ENABLE_EMBEDDED_FILES
+#define JITIFY_FORCE_UNDEFINED_SYMBOL(x) void* x##_forced = (void*)&x
+/*! Include a source file that has been embedded into the executable using the
+ *    GCC linker.
+ * \param name The name of the source file (<b>not</b> as a string), which must
+ * be sanitized by replacing non-alpha-numeric characters with underscores.
+ * E.g., \code{.cpp}JITIFY_INCLUDE_EMBEDDED_FILE(my_header_h)\endcode will
+ * include the embedded file "my_header.h".
+ * \note Files declared with this macro can be referenced using
+ * their original (unsanitized) filenames when creating a \p
+ * jitify::Program instance.
+ */
+#define JITIFY_INCLUDE_EMBEDDED_FILE(name)                                \
+  extern "C" uint8_t _jitify_binary_##name##_start[] asm("_binary_" #name \
+                                                         "_start");       \
+  extern "C" uint8_t _jitify_binary_##name##_end[] asm("_binary_" #name   \
+                                                       "_end");           \
+  JITIFY_FORCE_UNDEFINED_SYMBOL(_jitify_binary_##name##_start);           \
+  JITIFY_FORCE_UNDEFINED_SYMBOL(_jitify_binary_##name##_end)
+#endif  // JITIFY_ENABLE_EMBEDDED_FILES
+
+/*! Jitify library namespace
+ */
+namespace jitify {
+
+/*! Source-file load callback.
+ *
+ *  \param filename The name of the requested source file.
+ *  \param tmp_stream A temporary stream that can be used to hold source code.
+ *  \return A pointer to an input stream containing the source code, or NULL
+ *  to defer loading of the file to Jitify's file-loading mechanisms.
+ */
+typedef std::istream* (*file_callback_type)(std::string filename,
+                                            std::iostream& tmp_stream);
+
+// Exclude from Doxygen
+//! \cond
+
+class JitCache;
+
+// Simple cache using LRU discard policy
+template <typename KeyType, typename ValueType>
+class ObjectCache {
+ public:
+  typedef KeyType key_type;
+  typedef ValueType value_type;
+
+ private:
+  typedef std::map<key_type, value_type> object_map;
+  typedef std::deque<key_type> key_rank;
+  typedef typename key_rank::iterator rank_iterator;
+  object_map _objects;
+  key_rank _ranked_keys;
+  size_t _capacity;
+
+  inline void discard_old(size_t n = 0) {
+    if (n > _capacity) {
+      throw std::runtime_error("Insufficient capacity in cache");
+    }
+    while (_objects.size() > _capacity - n) {
+      key_type discard_key = _ranked_keys.back();
+      _ranked_keys.pop_back();
+      _objects.erase(discard_key);
+    }
+  }
+
+ public:
+  inline ObjectCache(size_t capacity = 8) : _capacity(capacity) {}
+  inline void resize(size_t capacity) {
+    _capacity = capacity;
+    this->discard_old();
+  }
+  inline bool contains(const key_type& k) const {
+    return (bool)_objects.count(k);
+  }
+  inline void touch(const key_type& k) {
+    if (!this->contains(k)) {
+      throw std::runtime_error("Key not found in cache");
+    }
+    rank_iterator rank = std::find(_ranked_keys.begin(), _ranked_keys.end(), k);
+    if (rank != _ranked_keys.begin()) {
+      // Move key to front of ranks
+      _ranked_keys.erase(rank);
+      _ranked_keys.push_front(k);
+    }
+  }
+  inline value_type& get(const key_type& k) {
+    if (!this->contains(k)) {
+      throw std::runtime_error("Key not found in cache");
+    }
+    this->touch(k);
+    return _objects[k];
+  }
+  inline value_type& insert(const key_type& k,
+                            const value_type& v = value_type()) {
+    this->discard_old(1);
+    _ranked_keys.push_front(k);
+    return _objects.insert(std::make_pair(k, v)).first->second;
+  }
+  template <typename... Args>
+  inline value_type& emplace(const key_type& k, Args&&... args) {
+    this->discard_old(1);
+    // Note: Use of piecewise_construct allows non-movable non-copyable types
+    auto iter = _objects
+                    .emplace(std::piecewise_construct, std::forward_as_tuple(k),
+                             std::forward_as_tuple(args...))
+                    .first;
+    _ranked_keys.push_front(iter->first);
+    return iter->second;
+  }
+};
+
+namespace detail {
+
+// Convenience wrapper for std::vector that provides handy constructors
+template <typename T>
+class vector : public std::vector<T> {
+  typedef std::vector<T> super_type;
+
+ public:
+  vector() : super_type() {}
+  vector(size_t n) : super_type(n) {}  // Note: Not explicit, allows =0
+  vector(std::vector<T> const& vals) : super_type(vals) {}
+  template <int N>
+  vector(T const (&vals)[N]) : super_type(vals, vals + N) {}
+  vector(std::vector<T>&& vals) : super_type(vals) {}
+  vector(std::initializer_list<T> vals) : super_type(vals) {}
+};
+
+// Helper functions for parsing/manipulating source code
+
+inline std::string replace_characters(std::string str,
+                                      std::string const& oldchars,
+                                      char newchar) {
+  size_t i = str.find_first_of(oldchars);
+  while (i != std::string::npos) {
+    str[i] = newchar;
+    i = str.find_first_of(oldchars, i + 1);
+  }
+  return str;
+}
+inline std::string sanitize_filename(std::string name) {
+  return replace_characters(name, "/\\.-: ?%*|\"<>", '_');
+}
+
+#if JITIFY_ENABLE_EMBEDDED_FILES
+class EmbeddedData {
+  void* _app;
+  EmbeddedData(EmbeddedData const&);
+  EmbeddedData& operator=(EmbeddedData const&);
+
+ public:
+  EmbeddedData() {
+    _app = dlopen(NULL, RTLD_LAZY);
+    if (!_app) {
+      throw std::runtime_error(std::string("dlopen failed: ") + dlerror());
+    }
+    dlerror();  // Clear any existing error
+  }
+  ~EmbeddedData() {
+    if (_app) {
+      dlclose(_app);
+    }
+  }
+  const uint8_t* operator[](std::string key) const {
+    key = sanitize_filename(key);
+    key = "_binary_" + key;
+    uint8_t const* data = (uint8_t const*)dlsym(_app, key.c_str());
+    if (!data) {
+      throw std::runtime_error(std::string("dlsym failed: ") + dlerror());
+    }
+    return data;
+  }
+  const uint8_t* begin(std::string key) const {
+    return (*this)[key + "_start"];
+  }
+  const uint8_t* end(std::string key) const { return (*this)[key + "_end"]; }
+};
+#endif  // JITIFY_ENABLE_EMBEDDED_FILES
+
+inline bool is_tokenchar(char c) {
+  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+         (c >= '0' && c <= '9') || c == '_';
+}
+inline std::string replace_token(std::string src, std::string token,
+                                 std::string replacement) {
+  size_t i = src.find(token);
+  while (i != std::string::npos) {
+    if (i == 0 || i == src.size() - token.size() ||
+        (!is_tokenchar(src[i - 1]) && !is_tokenchar(src[i + token.size()]))) {
+      src.replace(i, token.size(), replacement);
+      i += replacement.size();
+    } else {
+      i += token.size();
+    }
+    i = src.find(token, i);
+  }
+  return src;
+}
+inline std::string path_base(std::string p) {
+  // "/usr/local/myfile.dat" -> "/usr/local"
+  // "foo/bar"  -> "foo"
+  // "foo/bar/" -> "foo/bar"
+#if defined _WIN32 || defined _WIN64
+  const char* sep = "\\/";
+#else
+  char sep = '/';
+#endif
+  size_t i = p.find_last_of(sep);
+  if (i != std::string::npos) {
+    return p.substr(0, i);
+  } else {
+    return "";
+  }
+}
+inline std::string path_join(std::string p1, std::string p2) {
+#ifdef _WIN32
+  char sep = '\\';
+#else
+  char sep = '/';
+#endif
+  if (p1.size() && p2.size() && p2[0] == sep) {
+    throw std::invalid_argument("Cannot join to absolute path");
+  }
+  if (p1.size() && p1[p1.size() - 1] != sep) {
+    p1 += sep;
+  }
+  return p1 + p2;
+}
+// Elides "/." and "/.." tokens from path.
+inline std::string path_simplify(const std::string& path) {
+  std::vector<std::string> dirs;
+  std::string cur_dir;
+  bool after_slash = false;
+  for (int i = 0; i < (int)path.size(); ++i) {
+    if (path[i] == '/') {
+      if (after_slash) continue;  // Ignore repeat slashes
+      after_slash = true;
+      if (cur_dir == ".." && !dirs.empty() && dirs.back() != "..") {
+        if (dirs.size() == 1 && dirs.front().empty()) {
+          throw std::runtime_error(
+              "Invalid path: back-traversals exceed depth of absolute path");
+        }
+        dirs.pop_back();
+      } else if (cur_dir != ".") {  // Ignore /./
+        dirs.push_back(cur_dir);
+      }
+      cur_dir.clear();
+    } else {
+      after_slash = false;
+      cur_dir.push_back(path[i]);
+    }
+  }
+  if (!after_slash) {
+    dirs.push_back(cur_dir);
+  }
+  std::stringstream ss;
+  for (int i = 0; i < (int)dirs.size() - 1; ++i) {
+    ss << dirs[i] << "/";
+  }
+  if (!dirs.empty()) ss << dirs.back();
+  if (after_slash) ss << "/";
+  return ss.str();
+}
+inline unsigned long long hash_larson64(const char* s,
+                                        unsigned long long seed = 0) {
+  unsigned long long hash = seed;
+  while (*s) {
+    hash = hash * 101 + *s++;
+  }
+  return hash;
+}
+
+inline uint64_t hash_combine(uint64_t a, uint64_t b) {
+  // Note: The magic number comes from the golden ratio
+  return a ^ (0x9E3779B97F4A7C17ull + b + (b >> 2) + (a << 6));
+}
+
+inline bool extract_include_info_from_compile_error(std::string log,
+                                                    std::string& name,
+                                                    std::string& parent,
+                                                    int& line_num) {
+  static const std::vector<std::string> pattern = {
+      "could not open source file \"", "cannot open source file \""};
+
+  for (auto& p : pattern) {
+    size_t beg = log.find(p);
+    if (beg != std::string::npos) {
+      beg += p.size();
+      size_t end = log.find("\"", beg);
+      name = log.substr(beg, end - beg);
+
+      size_t line_beg = log.rfind("\n", beg);
+      if (line_beg == std::string::npos) {
+        line_beg = 0;
+      } else {
+        line_beg += 1;
+      }
+
+      size_t split = log.find("(", line_beg);
+      parent = log.substr(line_beg, split - line_beg);
+      line_num =
+          atoi(log.substr(split + 1, log.find(")", split + 1) - (split + 1))
+                   .c_str());
+
+      return true;
+    }
+  }
+
+  return false;
+}
+
+inline bool is_include_directive_with_quotes(const std::string& source,
+                                             int line_num) {
+  // TODO: Check each find() for failure.
+  size_t beg = 0;
+  for (int i = 1; i < line_num; ++i) {
+    beg = source.find("\n", beg) + 1;
+  }
+  beg = source.find("include", beg) + 7;
+  beg = source.find_first_of("\"<", beg);
+  return source[beg] == '"';
+}
+
+inline std::string comment_out_code_line(int line_num, std::string source) {
+  size_t beg = 0;
+  for (int i = 1; i < line_num; ++i) {
+    beg = source.find("\n", beg) + 1;
+  }
+  return (source.substr(0, beg) + "//" + source.substr(beg));
+}
+
+inline void print_with_line_numbers(std::string const& source) {
+  int linenum = 1;
+  std::stringstream source_ss(source);
+  std::stringstream output_ss;
+  output_ss.imbue(std::locale::classic());
+  for (std::string line; std::getline(source_ss, line); ++linenum) {
+    output_ss << std::setfill(' ') << std::setw(3) << linenum << " " << line
+              << std::endl;
+  }
+  std::cout << output_ss.str();
+}
+
+inline void print_compile_log(std::string program_name,
+                              std::string const& log) {
+  std::cout << "---------------------------------------------------"
+            << std::endl;
+  std::cout << "--- JIT compile log for " << program_name << " ---"
+            << std::endl;
+  std::cout << "---------------------------------------------------"
+            << std::endl;
+  std::cout << log << std::endl;
+  std::cout << "---------------------------------------------------"
+            << std::endl;
+}
+
+inline std::vector<std::string> split_string(std::string str,
+                                             long maxsplit = -1,
+                                             std::string delims = " \t") {
+  std::vector<std::string> results;
+  if (maxsplit == 0) {
+    results.push_back(str);
+    return results;
+  }
+  // Note: +1 to include NULL-terminator
+  std::vector<char> v_str(str.c_str(), str.c_str() + (str.size() + 1));
+  char* c_str = v_str.data();
+  char* saveptr = c_str;
+  char* token = nullptr;
+  for (long i = 0; i != maxsplit; ++i) {
+    token = ::strtok_r(c_str, delims.c_str(), &saveptr);
+    c_str = 0;
+    if (!token) {
+      return results;
+    }
+    results.push_back(token);
+  }
+  // Check if there's a final piece
+  token += ::strlen(token) + 1;
+  if (token - v_str.data() < (ptrdiff_t)str.size()) {
+    // Find the start of the final piece
+    token += ::strspn(token, delims.c_str());
+    if (*token) {
+      results.push_back(token);
+    }
+  }
+  return results;
+}
+
+static const std::map<std::string, std::string>& get_jitsafe_headers_map();
+
+inline bool load_source(
+    std::string filename, std::map<std::string, std::string>& sources,
+    std::string current_dir = "",
+    std::vector<std::string> include_paths = std::vector<std::string>(),
+    file_callback_type file_callback = 0, std::string* program_name = nullptr,
+    std::map<std::string, std::string>* fullpaths = nullptr,
+    bool search_current_dir = true) {
+  std::istream* source_stream = 0;
+  std::stringstream string_stream;
+  std::ifstream file_stream;
+  // First detect direct source-code string ("my_program\nprogram_code...")
+  size_t newline_pos = filename.find("\n");
+  if (newline_pos != std::string::npos) {
+    std::string source = filename.substr(newline_pos + 1);
+    filename = filename.substr(0, newline_pos);
+    string_stream << source;
+    source_stream = &string_stream;
+  }
+  if (program_name) {
+    *program_name = filename;
+  }
+  if (sources.count(filename)) {
+    // Already got this one
+    return true;
+  }
+  if (!source_stream) {
+    std::string fullpath = path_join(current_dir, filename);
+    // Try loading from callback
+    if (!file_callback ||
+        !((source_stream = file_callback(fullpath, string_stream)) != 0)) {
+#if JITIFY_ENABLE_EMBEDDED_FILES
+      // Try loading as embedded file
+      EmbeddedData embedded;
+      std::string source;
+      try {
+        source.assign(embedded.begin(fullpath), embedded.end(fullpath));
+        string_stream << source;
+        source_stream = &string_stream;
+      } catch (std::runtime_error const&)
+#endif  // JITIFY_ENABLE_EMBEDDED_FILES
+      {
+        // Try loading from filesystem
+        bool found_file = false;
+        if (search_current_dir) {
+          file_stream.open(fullpath.c_str());
+          if (file_stream) {
+            source_stream = &file_stream;
+            found_file = true;
+          }
+        }
+        // Search include directories
+        if (!found_file) {
+          for (int i = 0; i < (int)include_paths.size(); ++i) {
+            fullpath = path_join(include_paths[i], filename);
+            file_stream.open(fullpath.c_str());
+            if (file_stream) {
+              source_stream = &file_stream;
+              found_file = true;
+              break;
+            }
+          }
+          if (!found_file) {
+            // Try loading from builtin headers
+            fullpath = path_join("__jitify_builtin", filename);
+            auto it = get_jitsafe_headers_map().find(filename);
+            if (it != get_jitsafe_headers_map().end()) {
+              string_stream << it->second;
+              source_stream = &string_stream;
+            } else {
+              return false;
+            }
+          }
+        }
+      }
+    }
+    if (fullpaths) {
+      // Record the full file path corresponding to this include name.
+      (*fullpaths)[filename] = path_simplify(fullpath);
+    }
+  }
+  sources[filename] = std::string();
+  std::string& source = sources[filename];
+  std::string line;
+  size_t linenum = 0;
+  unsigned long long hash = 0;
+  bool pragma_once = false;
+  bool remove_next_blank_line = false;
+  while (std::getline(*source_stream, line)) {
+    ++linenum;
+
+    // HACK WAR for static variables not allowed on the device (unless
+    // __shared__)
+    // TODO: This breaks static member variables
+    // line = replace_token(line, "static const", "/*static*/ const");
+
+    // TODO: Need to watch out for /* */ comments too
+    std::string cleanline =
+        line.substr(0, line.find("//"));  // Strip line comments
+    // if( cleanline.back() == "\r" ) { // Remove Windows line ending
+    //	cleanline = cleanline.substr(0, cleanline.size()-1);
+    //}
+    // TODO: Should trim whitespace before checking .empty()
+    if (cleanline.empty() && remove_next_blank_line) {
+      remove_next_blank_line = false;
+      continue;
+    }
+    // Maintain a file hash for use in #pragma once WAR
+    hash = hash_larson64(line.c_str(), hash);
+    if (cleanline.find("#pragma once") != std::string::npos) {
+      pragma_once = true;
+      // Note: This is an attempt to recover the original line numbering,
+      //         which otherwise gets off-by-one due to the include guard.
+      remove_next_blank_line = true;
+      // line = "//" + line; // Comment out the #pragma once line
+      continue;
+    }
+
+    // HACK WAR for Thrust using "#define FOO #pragma bar"
+    // TODO: This is not robust to block comments, line continuations, or tabs.
+    size_t pragma_beg = cleanline.find("#pragma ");
+    if (pragma_beg != std::string::npos) {
+      std::string line_after_pragma = line.substr(pragma_beg + 8);
+      // TODO: Handle block comments (currently they cause a compilation error).
+      size_t comment_start = line_after_pragma.find("//");
+      std::string pragma_args = line_after_pragma.substr(0, comment_start);
+      std::string comment = comment_start != std::string::npos
+                                ? line_after_pragma.substr(comment_start)
+                                : "";
+      line = line.substr(0, pragma_beg) + "_Pragma(\"" + pragma_args + "\")" +
+             comment;
+    }
+
+    source += line + "\n";
+  }
+  // HACK TESTING (WAR for cub)
+  // source = "#define cudaDeviceSynchronize() cudaSuccess\n" + source;
+  ////source = "cudaError_t cudaDeviceSynchronize() { return cudaSuccess; }\n" +
+  /// source;
+
+  // WAR for #pragma once causing problems when there are multiple inclusions
+  //   of the same header from different paths.
+  if (pragma_once) {
+    std::stringstream ss;
+    ss.imbue(std::locale::classic());
+    ss << std::uppercase << std::hex << std::setw(8) << std::setfill('0')
+       << hash;
+    std::string include_guard_name = "_JITIFY_INCLUDE_GUARD_" + ss.str() + "\n";
+    std::string include_guard_header;
+    include_guard_header += "#ifndef " + include_guard_name;
+    include_guard_header += "#define " + include_guard_name;
+    std::string include_guard_footer;
+    include_guard_footer += "#endif // " + include_guard_name;
+    source = include_guard_header + source + "\n" + include_guard_footer;
+  }
+  // return filename;
+  return true;
+}
+
+}  // namespace detail
+
+//! \endcond
+
+/*! Jitify reflection utilities namespace
+ */
+namespace reflection {
+
+//  Provides type and value reflection via a function 'reflect':
+//    reflect<Type>()   -> "Type"
+//    reflect(value)    -> "(T)value"
+//    reflect<VAL>()    -> "VAL"
+//    reflect<Type,VAL> -> "VAL"
+//    reflect_template<float,NonType<int,7>,char>() -> "<float,7,char>"
+//    reflect_template({"float", "7", "char"}) -> "<float,7,char>"
+
+/*! A wrapper class for non-type template parameters.
+ */
+template <typename T, T VALUE_>
+struct NonType {
+  constexpr static T VALUE = VALUE_;
+};
+
+// Forward declaration
+template <typename T>
+inline std::string reflect(T const& value);
+
+//! \cond
+
+namespace detail {
+
+template <typename T>
+inline std::string value_string(const T& x) {
+  std::stringstream ss;
+  ss << x;
+  return ss.str();
+}
+// WAR for non-printable characters
+template <>
+inline std::string value_string<char>(const char& x) {
+  std::stringstream ss;
+  ss << (int)x;
+  return ss.str();
+}
+template <>
+inline std::string value_string<signed char>(const signed char& x) {
+  std::stringstream ss;
+  ss << (int)x;
+  return ss.str();
+}
+template <>
+inline std::string value_string<unsigned char>(const unsigned char& x) {
+  std::stringstream ss;
+  ss << (int)x;
+  return ss.str();
+}
+template <>
+inline std::string value_string<wchar_t>(const wchar_t& x) {
+  std::stringstream ss;
+  ss << (long)x;
+  return ss.str();
+}
+// Specialisation for bool true/false literals
+template <>
+inline std::string value_string<bool>(const bool& x) {
+  return x ? "true" : "false";
+}
+
+// Removes all tokens that start with double underscores.
+inline void strip_double_underscore_tokens(char* s) {
+  using jitify::detail::is_tokenchar;
+  char* w = s;
+  do {
+    if (*s == '_' && *(s + 1) == '_') {
+      while (is_tokenchar(*++s))
+        ;
+    }
+  } while ((*w++ = *s++));
+}
+
+//#if CUDA_VERSION < 8000
+#ifdef _MSC_VER  // MSVC compiler
+inline std::string demangle_cuda_symbol(const char* mangled_name) {
+  // We don't have a way to demangle CUDA symbol names under MSVC.
+  return mangled_name;
+}
+inline std::string demangle_native_type(const std::type_info& typeinfo) {
+  // Get the decorated name and skip over the leading '.'.
+  const char* decorated_name = typeinfo.raw_name() + 1;
+  char undecorated_name[4096];
+  if (UnDecorateSymbolName(
+          decorated_name, undecorated_name,
+          sizeof(undecorated_name) / sizeof(*undecorated_name),
+          UNDNAME_NO_ARGUMENTS |          // Treat input as a type name
+              UNDNAME_NAME_ONLY           // No "class" and "struct" prefixes
+          /*UNDNAME_NO_MS_KEYWORDS*/)) {  // No "__cdecl", "__ptr64" etc.
+    // WAR for UNDNAME_NO_MS_KEYWORDS messing up function types.
+    strip_double_underscore_tokens(undecorated_name);
+    return undecorated_name;
+  }
+  throw std::runtime_error("UnDecorateSymbolName failed");
+}
+#else  // not MSVC
+inline std::string demangle_cuda_symbol(const char* mangled_name) {
+  size_t bufsize = 0;
+  char* buf = nullptr;
+  std::string demangled_name;
+  int status;
+  auto demangled_ptr = std::unique_ptr<char, decltype(free)*>(
+      abi::__cxa_demangle(mangled_name, buf, &bufsize, &status), free);
+  if (status == 0) {
+    demangled_name = demangled_ptr.get();  // all worked as expected
+  } else if (status == -2) {
+    demangled_name = mangled_name;  // we interpret this as plain C name
+  } else if (status == -1) {
+    throw std::runtime_error(
+        std::string("memory allocation failure in __cxa_demangle"));
+  } else if (status == -3) {
+    throw std::runtime_error(std::string("invalid argument to __cxa_demangle"));
+  }
+  return demangled_name;
+}
+inline std::string demangle_native_type(const std::type_info& typeinfo) {
+  return demangle_cuda_symbol(typeinfo.name());
+}
+#endif  // not MSVC
+//#endif // CUDA_VERSION < 8000
+
+template <typename>
+class JitifyTypeNameWrapper_ {};
+
+template <typename T>
+struct type_reflection {
+  inline static std::string name() {
+    //#if CUDA_VERSION < 8000
+    // TODO: Use nvrtcGetTypeName once it has the same behavior as this.
+    // WAR for typeid discarding cv qualifiers on value-types
+    // Wrap type in dummy template class to preserve cv-qualifiers, then strip
+    // off the wrapper from the resulting string.
+    std::string wrapped_name =
+        demangle_native_type(typeid(JitifyTypeNameWrapper_<T>));
+    // Note: The reflected name of this class also has namespace prefixes.
+    const std::string wrapper_class_name = "JitifyTypeNameWrapper_<";
+    size_t start = wrapped_name.find(wrapper_class_name);
+    if (start == std::string::npos) {
+      throw std::runtime_error("Type reflection failed: " + wrapped_name);
+    }
+    start += wrapper_class_name.size();
+    std::string name =
+        wrapped_name.substr(start, wrapped_name.size() - (start + 1));
+    return name;
+    //#else
+    //         std::string ret;
+    //         nvrtcResult status = nvrtcGetTypeName<T>(&ret);
+    //         if( status != NVRTC_SUCCESS ) {
+    //                 throw std::runtime_error(std::string("nvrtcGetTypeName
+    // failed:
+    //")+ nvrtcGetErrorString(status));
+    //         }
+    //         return ret;
+    //#endif
+  }
+};  // namespace detail
+template <typename T, T VALUE>
+struct type_reflection<NonType<T, VALUE> > {
+  inline static std::string name() {
+    return jitify::reflection::reflect(VALUE);
+  }
+};
+
+}  // namespace detail
+
+//! \endcond
+
+/*! Create an Instance object that contains a const reference to the
+ *  value.  We use this to wrap abstract objects from which we want to extract
+ *  their type at runtime (e.g., derived type).  This is used to facilitate
+ *  templating on derived type when all we know at compile time is abstract
+ * type.
+ */
+template <typename T>
+struct Instance {
+  const T& value;
+  Instance(const T& value_arg) : value(value_arg) {}
+};
+
+/*! Create an Instance object from which we can extract the value's run-time
+ * type.
+ *  \param value The const value to be captured.
+ */
+template <typename T>
+inline Instance<T const> instance_of(T const& value) {
+  return Instance<T const>(value);
+}
+
+/*! A wrapper used for representing types as values.
+ */
+template <typename T>
+struct Type {};
+
+// Type reflection
+// E.g., reflect<float>() -> "float"
+// Note: This strips trailing const and volatile qualifiers
+/*! Generate a code-string for a type.
+ *  \code{.cpp}reflect<float>() --> "float"\endcode
+ */
+template <typename T>
+inline std::string reflect() {
+  return detail::type_reflection<T>::name();
+}
+// Value reflection
+// E.g., reflect(3.14f) -> "(float)3.14"
+/*! Generate a code-string for a value.
+ *  \code{.cpp}reflect(3.14f) --> "(float)3.14"\endcode
+ */
+template <typename T>
+inline std::string reflect(T const& value) {
+  return "(" + reflect<T>() + ")" + detail::value_string(value);
+}
+// Non-type template arg reflection (implicit conversion to int64_t)
+// E.g., reflect<7>() -> "(int64_t)7"
+/*! Generate a code-string for an integer non-type template argument.
+ *  \code{.cpp}reflect<7>() --> "(int64_t)7"\endcode
+ */
+template <int64_t N>
+inline std::string reflect() {
+  return reflect<NonType<int64_t, N> >();
+}
+// Non-type template arg reflection (explicit type)
+// E.g., reflect<int,7>() -> "(int)7"
+/*! Generate a code-string for a generic non-type template argument.
+ *  \code{.cpp} reflect<int,7>() --> "(int)7" \endcode
+ */
+template <typename T, T N>
+inline std::string reflect() {
+  return reflect<NonType<T, N> >();
+}
+// Type reflection via value
+// E.g., reflect(Type<float>()) -> "float"
+/*! Generate a code-string for a type wrapped as a Type instance.
+ *  \code{.cpp}reflect(Type<float>()) --> "float"\endcode
+ */
+template <typename T>
+inline std::string reflect(jitify::reflection::Type<T>) {
+  return reflect<T>();
+}
+
+/*! Generate a code-string for a type wrapped as an Instance instance.
+ *  \code{.cpp}reflect(Instance<float>(3.1f)) --> "float"\endcode
+ *  or more simply when passed to a instance_of helper
+ *  \code{.cpp}reflect(instance_of(3.1f)) --> "float"\endcodei
+ *  This is specifically for the case where we want to extract the run-time
+ * type, e.g., derived type, of an object pointer.
+ */
+template <typename T>
+inline std::string reflect(jitify::reflection::Instance<T>& value) {
+  return detail::demangle_native_type(typeid(value.value));
+}
+
+// Type from value
+// E.g., type_of(3.14f) -> Type<float>()
+/*! Create a Type object representing a value's type.
+ *  \param value The value whose type is to be captured.
+ */
+template <typename T>
+inline Type<T> type_of(T&) {
+  return Type<T>();
+}
+/*! Create a Type object representing a value's type.
+ *  \param value The const value whose type is to be captured.
+ */
+template <typename T>
+inline Type<T const> type_of(T const&) {
+  return Type<T const>();
+}
+
+// Multiple value reflections one call, returning list of strings
+template <typename... Args>
+inline std::vector<std::string> reflect_all(Args... args) {
+  return {reflect(args)...};
+}
+
+inline std::string reflect_list(jitify::detail::vector<std::string> const& args,
+                                std::string opener = "",
+                                std::string closer = "") {
+  std::stringstream ss;
+  ss << opener;
+  for (int i = 0; i < (int)args.size(); ++i) {
+    if (i > 0) ss << ",";
+    ss << args[i];
+  }
+  ss << closer;
+  return ss.str();
+}
+
+// Template instantiation reflection
+// inline std::string reflect_template(std::vector<std::string> const& args) {
+inline std::string reflect_template(
+    jitify::detail::vector<std::string> const& args) {
+  // Note: The space in " >" is a WAR to avoid '>>' appearing
+  return reflect_list(args, "<", " >");
+}
+// TODO: See if can make this evaluate completely at compile-time
+template <typename... Ts>
+inline std::string reflect_template() {
+  return reflect_template({reflect<Ts>()...});
+  // return reflect_template<sizeof...(Ts)>({reflect<Ts>()...});
+}
+
+}  // namespace reflection
+
+//! \cond
+
+namespace detail {
+
+// Demangles nested variable names using the PTX name mangling scheme
+// (which follows the Itanium64 ABI). E.g., _ZN1a3Foo2bcE -> a::Foo::bc.
+inline std::string demangle_ptx_variable_name(const char* name) {
+  std::stringstream ss;
+  const char* c = name;
+  if (*c++ != '_' || *c++ != 'Z') return name;  // Non-mangled name
+  if (*c++ != 'N') return "";  // Not a nested name, unsupported
+  while (true) {
+    // Parse identifier length.
+    int n = 0;
+    while (std::isdigit(*c)) {
+      n = n * 10 + (*c - '0');
+      c++;
+    }
+    if (!n) return "";  // Invalid or unsupported mangled name
+    // Parse identifier.
+    const char* c0 = c;
+    while (n-- && *c) c++;
+    if (!*c) return "";  // Mangled name is truncated
+    std::string id(c0, c);
+    // Identifiers starting with "_GLOBAL" are anonymous namespaces.
+    ss << (id.substr(0, 7) == "_GLOBAL" ? "(anonymous namespace)" : id);
+    // Nested name specifiers end with 'E'.
+    if (*c == 'E') break;
+    // There are more identifiers to come, add join token.
+    ss << "::";
+  }
+  return ss.str();
+}
+
+static const char* get_current_executable_path() {
+  static const char* path = []() -> const char* {
+    static char buffer[JITIFY_PATH_MAX] = {};
+#ifdef __linux__
+    if (!::realpath("/proc/self/exe", buffer)) return nullptr;
+#elif defined(_WIN32) || defined(_WIN64)
+    if (!GetModuleFileNameA(nullptr, buffer, JITIFY_PATH_MAX)) return nullptr;
+#endif
+    return buffer;
+  }();
+  return path;
+}
+
+inline bool endswith(const std::string& str, const std::string& suffix) {
+  return str.size() >= suffix.size() &&
+         str.substr(str.size() - suffix.size()) == suffix;
+}
+
+// Infers the JIT input type from the filename suffix. If no known suffix is
+// present, the filename is assumed to refer to a library, and the associated
+// suffix (and possibly prefix) is automatically added to the filename.
+inline CUjitInputType get_cuda_jit_input_type(std::string* filename) {
+  if (endswith(*filename, ".ptx")) {
+    return CU_JIT_INPUT_PTX;
+  } else if (endswith(*filename, ".cubin")) {
+    return CU_JIT_INPUT_CUBIN;
+  } else if (endswith(*filename, ".fatbin")) {
+    return CU_JIT_INPUT_FATBINARY;
+  } else if (endswith(*filename,
+#if defined _WIN32 || defined _WIN64
+                      ".obj"
+#else  // Linux
+                      ".o"
+#endif
+                      )) {
+    return CU_JIT_INPUT_OBJECT;
+  } else {  // Assume library
+#if defined _WIN32 || defined _WIN64
+    if (!endswith(*filename, ".lib")) {
+      *filename += ".lib";
+    }
+#else  // Linux
+    if (!endswith(*filename, ".a")) {
+      *filename = "lib" + *filename + ".a";
+    }
+#endif
+    return CU_JIT_INPUT_LIBRARY;
+  }
+}
+
+class CUDAKernel {
+  std::vector<std::string> _link_files;
+  std::vector<std::string> _link_paths;
+  CUlinkState _link_state;
+  CUmodule _module;
+  CUfunction _kernel;
+  std::string _func_name;
+  std::string _ptx;
+  std::map<std::string, std::string> _global_map;
+  std::vector<CUjit_option> _opts;
+  std::vector<void*> _optvals;
+#ifdef JITIFY_PRINT_LINKER_LOG
+  static const unsigned int _log_size = 8192;
+  char _error_log[_log_size];
+  char _info_log[_log_size];
+#endif
+
+  inline void cuda_safe_call(CUresult res) const {
+    if (res != CUDA_SUCCESS) {
+      const char* msg;
+      cuGetErrorName(res, &msg);
+      throw std::runtime_error(msg);
+    }
+  }
+  inline void create_module(std::vector<std::string> link_files,
+                            std::vector<std::string> link_paths) {
+    CUresult result;
+#ifndef JITIFY_PRINT_LINKER_LOG
+    // WAR since linker log does not seem to be constructed using a single call
+    // to cuModuleLoadDataEx.
+    if (link_files.empty()) {
+      result =
+          cuModuleLoadDataEx(&_module, _ptx.c_str(), (unsigned)_opts.size(),
+                             _opts.data(), _optvals.data());
+    } else
+#endif
+    {
+      cuda_safe_call(cuLinkCreate((unsigned)_opts.size(), _opts.data(),
+                                  _optvals.data(), &_link_state));
+      cuda_safe_call(cuLinkAddData(_link_state, CU_JIT_INPUT_PTX,
+                                   (void*)_ptx.c_str(), _ptx.size(),
+                                   "jitified_source.ptx", 0, 0, 0));
+      for (int i = 0; i < (int)link_files.size(); ++i) {
+        std::string link_file = link_files[i];
+        CUjitInputType jit_input_type;
+        if (link_file == ".") {
+          // Special case for linking to current executable.
+          link_file = get_current_executable_path();
+          jit_input_type = CU_JIT_INPUT_OBJECT;
+        } else {
+          // Infer based on filename.
+          jit_input_type = get_cuda_jit_input_type(&link_file);
+        }
+        result = cuLinkAddFile(_link_state, jit_input_type, link_file.c_str(),
+                               0, 0, 0);
+        int path_num = 0;
+        while (result == CUDA_ERROR_FILE_NOT_FOUND &&
+               path_num < (int)link_paths.size()) {
+          std::string filename = path_join(link_paths[path_num++], link_file);
+          result = cuLinkAddFile(_link_state, jit_input_type, filename.c_str(),
+                                 0, 0, 0);
+        }
+#if JITIFY_PRINT_LINKER_LOG
+        if (result == CUDA_ERROR_FILE_NOT_FOUND) {
+          std::cerr << "Linker error: Device library not found: " << link_file
+                    << std::endl;
+        } else if (result != CUDA_SUCCESS) {
+          std::cerr << "Linker error: Failed to add file: " << link_file
+                    << std::endl;
+          std::cerr << _error_log << std::endl;
+        }
+#endif
+        cuda_safe_call(result);
+      }
+      size_t cubin_size;
+      void* cubin;
+      result = cuLinkComplete(_link_state, &cubin, &cubin_size);
+      if (result == CUDA_SUCCESS) {
+        result = cuModuleLoadData(&_module, cubin);
+      }
+    }
+#ifdef JITIFY_PRINT_LINKER_LOG
+    std::cout << "---------------------------------------" << std::endl;
+    std::cout << "--- Linker for "
+              << reflection::detail::demangle_cuda_symbol(_func_name.c_str())
+              << " ---" << std::endl;
+    std::cout << "---------------------------------------" << std::endl;
+    std::cout << _info_log << std::endl;
+    std::cout << std::endl;
+    std::cout << _error_log << std::endl;
+    std::cout << "---------------------------------------" << std::endl;
+#endif
+    cuda_safe_call(result);
+    // Allow _func_name to be empty to support cases where we want to generate
+    // PTX containing extern symbol definitions but no kernels.
+    if (!_func_name.empty()) {
+      cuda_safe_call(
+          cuModuleGetFunction(&_kernel, _module, _func_name.c_str()));
+    }
+  }
+  inline void destroy_module() {
+    if (_link_state) {
+      cuda_safe_call(cuLinkDestroy(_link_state));
+    }
+    _link_state = 0;
+    if (_module) {
+      cuModuleUnload(_module);
+    }
+    _module = 0;
+  }
+
+  // create a map of __constant__ and __device__ variables in the ptx file
+  // mapping demangled to mangled name
+  inline void create_global_variable_map() {
+    size_t pos = 0;
+    while (pos < _ptx.size()) {
+      pos = std::min(_ptx.find(".const .align", pos),
+                     _ptx.find(".global .align", pos));
+      if (pos == std::string::npos) break;
+      size_t end = _ptx.find_first_of(";=", pos);
+      if (_ptx[end] == '=') --end;
+      std::string line = _ptx.substr(pos, end - pos);
+      pos = end;
+      size_t symbol_start = line.find_last_of(" ") + 1;
+      size_t symbol_end = line.find_last_of("[");
+      std::string entry = line.substr(symbol_start, symbol_end - symbol_start);
+      std::string key = detail::demangle_ptx_variable_name(entry.c_str());
+      // Skip unsupported mangled names. E.g., a static variable defined inside
+      // a function (such variables are not directly addressable from outside
+      // the function, so skipping them is the correct behavior).
+      if (key == "") continue;
+      _global_map[key] = entry;
+    }
+  }
+
+  inline void set_linker_log() {
+#ifdef JITIFY_PRINT_LINKER_LOG
+    _opts.push_back(CU_JIT_INFO_LOG_BUFFER);
+    _optvals.push_back((void*)_info_log);
+    _opts.push_back(CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES);
+    _optvals.push_back((void*)(long)_log_size);
+    _opts.push_back(CU_JIT_ERROR_LOG_BUFFER);
+    _optvals.push_back((void*)_error_log);
+    _opts.push_back(CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES);
+    _optvals.push_back((void*)(long)_log_size);
+    _opts.push_back(CU_JIT_LOG_VERBOSE);
+    _optvals.push_back((void*)1);
+#endif
+  }
+
+ public:
+  inline CUDAKernel() : _link_state(0), _module(0), _kernel(0) {}
+  inline CUDAKernel(const CUDAKernel& other) = delete;
+  inline CUDAKernel& operator=(const CUDAKernel& other) = delete;
+  inline CUDAKernel(CUDAKernel&& other) = delete;
+  inline CUDAKernel& operator=(CUDAKernel&& other) = delete;
+  inline CUDAKernel(const char* func_name, const char* ptx,
+                    std::vector<std::string> link_files,
+                    std::vector<std::string> link_paths, unsigned int nopts = 0,
+                    CUjit_option* opts = 0, void** optvals = 0)
+      : _link_files(link_files),
+        _link_paths(link_paths),
+        _link_state(0),
+        _module(0),
+        _kernel(0),
+        _func_name(func_name),
+        _ptx(ptx),
+        _opts(opts, opts + nopts),
+        _optvals(optvals, optvals + nopts) {
+    this->set_linker_log();
+    this->create_module(link_files, link_paths);
+    this->create_global_variable_map();
+  }
+
+  inline CUDAKernel& set(const char* func_name, const char* ptx,
+                         std::vector<std::string> link_files,
+                         std::vector<std::string> link_paths,
+                         unsigned int nopts = 0, CUjit_option* opts = 0,
+                         void** optvals = 0) {
+    this->destroy_module();
+    _func_name = func_name;
+    _ptx = ptx;
+    _link_files = link_files;
+    _link_paths = link_paths;
+    _opts.assign(opts, opts + nopts);
+    _optvals.assign(optvals, optvals + nopts);
+    this->set_linker_log();
+    this->create_module(link_files, link_paths);
+    this->create_global_variable_map();
+    return *this;
+  }
+  inline ~CUDAKernel() { this->destroy_module(); }
+  inline operator CUfunction() const { return _kernel; }
+
+  inline CUresult launch(dim3 grid, dim3 block, unsigned int smem,
+                         CUstream stream, std::vector<void*> arg_ptrs) const {
+    return cuLaunchKernel(_kernel, grid.x, grid.y, grid.z, block.x, block.y,
+                          block.z, smem, stream, arg_ptrs.data(), NULL);
+  }
+
+  inline void safe_launch(dim3 grid, dim3 block, unsigned int smem,
+                          CUstream stream, std::vector<void*> arg_ptrs) const {
+    return cuda_safe_call(cuLaunchKernel(_kernel, grid.x, grid.y, grid.z,
+                                         block.x, block.y, block.z, smem,
+                                         stream, arg_ptrs.data(), NULL));
+  }
+
+  inline int get_func_attribute(CUfunction_attribute attribute) const {
+    int value;
+    cuda_safe_call(cuFuncGetAttribute(&value, attribute, _kernel));
+    return value;
+  }
+
+  inline void set_func_attribute(CUfunction_attribute attribute,
+                                 int value) const {
+    cuda_safe_call(cuFuncSetAttribute(_kernel, attribute, value));
+  }
+
+  inline CUdeviceptr get_global_ptr(const char* name,
+                                    size_t* size = nullptr) const {
+    CUdeviceptr global_ptr = 0;
+    auto global = _global_map.find(name);
+    if (global != _global_map.end()) {
+      cuda_safe_call(cuModuleGetGlobal(&global_ptr, size, _module,
+                                       global->second.c_str()));
+    } else {
+      throw std::runtime_error(std::string("failed to look up global ") + name);
+    }
+    return global_ptr;
+  }
+
+  template <typename T>
+  inline CUresult get_global_data(const char* name, T* data, size_t count,
+                                  CUstream stream = 0) const {
+    size_t size_bytes;
+    CUdeviceptr ptr = get_global_ptr(name, &size_bytes);
+    size_t given_size_bytes = count * sizeof(T);
+    if (given_size_bytes != size_bytes) {
+      throw std::runtime_error(
+          std::string("Value for global variable ") + name +
+          " has wrong size: got " + std::to_string(given_size_bytes) +
+          " bytes, expected " + std::to_string(size_bytes));
+    }
+    return cuMemcpyDtoHAsync(data, ptr, size_bytes, stream);
+  }
+
+  template <typename T>
+  inline CUresult set_global_data(const char* name, const T* data, size_t count,
+                                  CUstream stream = 0) const {
+    size_t size_bytes;
+    CUdeviceptr ptr = get_global_ptr(name, &size_bytes);
+    size_t given_size_bytes = count * sizeof(T);
+    if (given_size_bytes != size_bytes) {
+      throw std::runtime_error(
+          std::string("Value for global variable ") + name +
+          " has wrong size: got " + std::to_string(given_size_bytes) +
+          " bytes, expected " + std::to_string(size_bytes));
+    }
+    return cuMemcpyHtoDAsync(ptr, data, size_bytes, stream);
+  }
+
+  const std::string& function_name() const { return _func_name; }
+  const std::string& ptx() const { return _ptx; }
+  const std::vector<std::string>& link_files() const { return _link_files; }
+  const std::vector<std::string>& link_paths() const { return _link_paths; }
+};
+
+static const char* jitsafe_header_preinclude_h = R"(
+//// WAR for Thrust (which appears to have forgotten to include this in result_of_adaptable_function.h
+//#include <type_traits>
+
+//// WAR for Thrust (which appear to have forgotten to include this in error_code.h)
+//#include <string>
+
+// WAR for generics/shfl.h
+#define THRUST_STATIC_ASSERT(x)
+
+// WAR for CUB
+#ifdef __host__
+#undef __host__
+#endif
+#define __host__
+
+// WAR to allow exceptions to be parsed
+#define try
+#define catch(...)
+)"
+#if defined(_WIN32) || defined(_WIN64)
+// WAR for NVRTC <= 11.0 not defining _WIN64.
+R"(
+#ifndef _WIN64
+#define _WIN64 1
+#endif
+)"
+#endif
+;
+
+static const char* jitsafe_header_float_h = R"(
+#pragma once
+
+#define FLT_RADIX       2
+#define FLT_MANT_DIG    24
+#define DBL_MANT_DIG    53
+#define FLT_DIG         6
+#define DBL_DIG         15
+#define FLT_MIN_EXP     -125
+#define DBL_MIN_EXP     -1021
+#define FLT_MIN_10_EXP  -37
+#define DBL_MIN_10_EXP  -307
+#define FLT_MAX_EXP     128
+#define DBL_MAX_EXP     1024
+#define FLT_MAX_10_EXP  38
+#define DBL_MAX_10_EXP  308
+#define FLT_MAX         3.4028234e38f
+#define DBL_MAX         1.7976931348623157e308
+#define FLT_EPSILON     1.19209289e-7f
+#define DBL_EPSILON     2.220440492503130e-16
+#define FLT_MIN         1.1754943e-38f
+#define DBL_MIN         2.2250738585072013e-308
+#define FLT_ROUNDS      1
+#if defined __cplusplus && __cplusplus >= 201103L
+#define FLT_EVAL_METHOD 0
+#define DECIMAL_DIG     21
+#endif
+)";
+
+static const char* jitsafe_header_limits_h = R"(
+#pragma once
+
+#if defined _WIN32 || defined _WIN64
+ #define __WORDSIZE 32
+#else
+ #if defined __x86_64__ && !defined __ILP32__
+  #define __WORDSIZE 64
+ #else
+  #define __WORDSIZE 32
+ #endif
+#endif
+#define MB_LEN_MAX  16
+#define CHAR_BIT    8
+#define SCHAR_MIN   (-128)
+#define SCHAR_MAX   127
+#define UCHAR_MAX   255
+enum {
+  _JITIFY_CHAR_IS_UNSIGNED = (char)-1 >= 0,
+  CHAR_MIN = _JITIFY_CHAR_IS_UNSIGNED ? 0 : SCHAR_MIN,
+  CHAR_MAX = _JITIFY_CHAR_IS_UNSIGNED ? UCHAR_MAX : SCHAR_MAX,
+};
+#define SHRT_MIN    (-32768)
+#define SHRT_MAX    32767
+#define USHRT_MAX   65535
+#define INT_MIN     (-INT_MAX - 1)
+#define INT_MAX     2147483647
+#define UINT_MAX    4294967295U
+#if __WORDSIZE == 64
+ # define LONG_MAX  9223372036854775807L
+#else
+ # define LONG_MAX  2147483647L
+#endif
+#define LONG_MIN    (-LONG_MAX - 1L)
+#if __WORDSIZE == 64
+ #define ULONG_MAX  18446744073709551615UL
+#else
+ #define ULONG_MAX  4294967295UL
+#endif
+#define LLONG_MAX  9223372036854775807LL
+#define LLONG_MIN  (-LLONG_MAX - 1LL)
+#define ULLONG_MAX 18446744073709551615ULL
+)";
+
+static const char* jitsafe_header_iterator = R"(
+#pragma once
+
+namespace std {
+struct output_iterator_tag {};
+struct input_iterator_tag {};
+struct forward_iterator_tag {};
+struct bidirectional_iterator_tag {};
+struct random_access_iterator_tag {};
+template<class Iterator>
+struct iterator_traits {
+  typedef typename Iterator::iterator_category iterator_category;
+  typedef typename Iterator::value_type        value_type;
+  typedef typename Iterator::difference_type   difference_type;
+  typedef typename Iterator::pointer           pointer;
+  typedef typename Iterator::reference         reference;
+};
+template<class T>
+struct iterator_traits<T*> {
+  typedef random_access_iterator_tag iterator_category;
+  typedef T                          value_type;
+  typedef ptrdiff_t                  difference_type;
+  typedef T*                         pointer;
+  typedef T&                         reference;
+};
+template<class T>
+struct iterator_traits<T const*> {
+  typedef random_access_iterator_tag iterator_category;
+  typedef T                          value_type;
+  typedef ptrdiff_t                  difference_type;
+  typedef T const*                   pointer;
+  typedef T const&                   reference;
+};
+}  // namespace std
+)";
+
+// TODO: This is incomplete; need floating point limits
+//   Joe Eaton: added IEEE float and double types, none of the smaller types
+//              using type specific structs since we can't template on floats.
+static const char* jitsafe_header_limits = R"(
+#pragma once
+#include <cfloat>
+#include <climits>
+#include <cstdint>
+// TODO: epsilon(), infinity(), etc
+namespace std {
+namespace __jitify_detail {
+#if __cplusplus >= 201103L
+#define JITIFY_CXX11_CONSTEXPR constexpr
+#define JITIFY_CXX11_NOEXCEPT noexcept
+#else
+#define JITIFY_CXX11_CONSTEXPR
+#define JITIFY_CXX11_NOEXCEPT
+#endif
+
+struct FloatLimits {
+#if __cplusplus >= 201103L
+   static JITIFY_CXX11_CONSTEXPR inline __host__ __device__ 
+          float lowest() JITIFY_CXX11_NOEXCEPT {   return -FLT_MAX;}
+   static JITIFY_CXX11_CONSTEXPR inline __host__ __device__ 
+          float min() JITIFY_CXX11_NOEXCEPT {      return FLT_MIN; }
+   static JITIFY_CXX11_CONSTEXPR inline __host__ __device__ 
+          float max() JITIFY_CXX11_NOEXCEPT {      return FLT_MAX; }
+#endif  // __cplusplus >= 201103L
+   enum {
+   is_specialized    = true,
+   is_signed         = true,
+   is_integer        = false,
+   is_exact          = false,
+   has_infinity      = true,
+   has_quiet_NaN     = true,
+   has_signaling_NaN = true,
+   has_denorm        = 1,
+   has_denorm_loss   = true,
+   round_style       = 1,
+   is_iec559         = true,
+   is_bounded        = true,
+   is_modulo         = false,
+   digits            = 24,
+   digits10          = 6,
+   max_digits10      = 9,
+   radix             = 2,
+   min_exponent      = -125,
+   min_exponent10    = -37,
+   max_exponent      = 128,
+   max_exponent10    = 38,
+   tinyness_before   = false,
+   traps             = false
+   };
+};
+struct DoubleLimits {
+#if __cplusplus >= 201103L
+   static JITIFY_CXX11_CONSTEXPR inline __host__ __device__ 
+          double lowest() noexcept { return -DBL_MAX; }
+   static JITIFY_CXX11_CONSTEXPR inline __host__ __device__ 
+          double min() noexcept { return DBL_MIN; }
+   static JITIFY_CXX11_CONSTEXPR inline __host__ __device__ 
+          double max() noexcept { return DBL_MAX; }
+#endif  // __cplusplus >= 201103L
+   enum {
+   is_specialized    = true,
+   is_signed         = true,
+   is_integer        = false,
+   is_exact          = false,
+   has_infinity      = true,
+   has_quiet_NaN     = true,
+   has_signaling_NaN = true,
+   has_denorm        = 1,
+   has_denorm_loss   = true,
+   round_style       = 1,
+   is_iec559         = true,
+   is_bounded        = true,
+   is_modulo         = false,
+   digits            = 53,
+   digits10          = 15,
+   max_digits10      = 17,
+   radix             = 2,
+   min_exponent      = -1021,
+   min_exponent10    = -307,
+   max_exponent      = 1024,
+   max_exponent10    = 308,
+   tinyness_before   = false,
+   traps             = false
+   };
+};
+template<class T, T Min, T Max, int Digits=-1>
+struct IntegerLimits {
+	static inline __host__ __device__ T min() { return Min; }
+	static inline __host__ __device__ T max() { return Max; }
+#if __cplusplus >= 201103L
+	static constexpr inline __host__ __device__ T lowest() noexcept {
+		return Min;
+	}
+#endif  // __cplusplus >= 201103L
+	enum {
+       is_specialized = true,
+       digits = (Digits == -1) ? (int)(sizeof(T)*8 - (Min != 0)) : Digits,
+       digits10   = (digits * 30103) / 100000,
+       is_signed  = ((T)(-1)<0),
+       is_integer = true,
+       is_exact   = true,
+       radix      = 2,
+       is_bounded = true,
+       is_modulo  = false
+	};
+};
+} // namespace __jitify_detail
+template<typename T> struct numeric_limits {
+    enum { is_specialized = false };
+};
+template<> struct numeric_limits<bool>               : public 
+__jitify_detail::IntegerLimits<bool,              false,    true,1> {};
+template<> struct numeric_limits<char>               : public 
+__jitify_detail::IntegerLimits<char,              CHAR_MIN, CHAR_MAX> 
+{};
+template<> struct numeric_limits<signed char>        : public 
+__jitify_detail::IntegerLimits<signed char,       SCHAR_MIN,SCHAR_MAX> 
+{};
+template<> struct numeric_limits<unsigned char>      : public 
+__jitify_detail::IntegerLimits<unsigned char,     0,        UCHAR_MAX> 
+{};
+template<> struct numeric_limits<wchar_t>            : public 
+__jitify_detail::IntegerLimits<wchar_t,           WCHAR_MIN, WCHAR_MAX> {};
+template<> struct numeric_limits<short>              : public 
+__jitify_detail::IntegerLimits<short,             SHRT_MIN, SHRT_MAX> 
+{};
+template<> struct numeric_limits<unsigned short>     : public 
+__jitify_detail::IntegerLimits<unsigned short,    0,        USHRT_MAX> 
+{};
+template<> struct numeric_limits<int>                : public 
+__jitify_detail::IntegerLimits<int,               INT_MIN,  INT_MAX> {};
+template<> struct numeric_limits<unsigned int>       : public 
+__jitify_detail::IntegerLimits<unsigned int,      0,        UINT_MAX> 
+{};
+template<> struct numeric_limits<long>               : public 
+__jitify_detail::IntegerLimits<long,              LONG_MIN, LONG_MAX> 
+{};
+template<> struct numeric_limits<unsigned long>      : public 
+__jitify_detail::IntegerLimits<unsigned long,     0,        ULONG_MAX> 
+{};
+template<> struct numeric_limits<long long>          : public 
+__jitify_detail::IntegerLimits<long long,         LLONG_MIN,LLONG_MAX> 
+{};
+template<> struct numeric_limits<unsigned long long> : public 
+__jitify_detail::IntegerLimits<unsigned long long,0,        ULLONG_MAX> 
+{};
+//template<typename T> struct numeric_limits { static const bool 
+//is_signed = ((T)(-1)<0); };
+template<> struct numeric_limits<float>              : public 
+__jitify_detail::FloatLimits 
+{};
+template<> struct numeric_limits<double>             : public 
+__jitify_detail::DoubleLimits 
+{};
+}  // namespace std
+)";
+
+// TODO: This is highly incomplete
+static const char* jitsafe_header_type_traits = R"(
+    #pragma once
+    #if __cplusplus >= 201103L
+    namespace std {
+
+    template<bool B, class T = void> struct enable_if {};
+    template<class T>                struct enable_if<true, T> { typedef T type; };
+    #if __cplusplus >= 201402L
+    template< bool B, class T = void > using enable_if_t = typename enable_if<B,T>::type;
+    #endif
+
+    struct true_type  {
+      enum { value = true };
+      operator bool() const { return true; }
+    };
+    struct false_type {
+      enum { value = false };
+      operator bool() const { return false; }
+    };
+
+    template<typename T> struct is_floating_point    : false_type {};
+    template<> struct is_floating_point<float>       :  true_type {};
+    template<> struct is_floating_point<double>      :  true_type {};
+    template<> struct is_floating_point<long double> :  true_type {};
+
+    template<class T> struct is_integral              : false_type {};
+    template<> struct is_integral<bool>               :  true_type {};
+    template<> struct is_integral<char>               :  true_type {};
+    template<> struct is_integral<signed char>        :  true_type {};
+    template<> struct is_integral<unsigned char>      :  true_type {};
+    template<> struct is_integral<short>              :  true_type {};
+    template<> struct is_integral<unsigned short>     :  true_type {};
+    template<> struct is_integral<int>                :  true_type {};
+    template<> struct is_integral<unsigned int>       :  true_type {};
+    template<> struct is_integral<long>               :  true_type {};
+    template<> struct is_integral<unsigned long>      :  true_type {};
+    template<> struct is_integral<long long>          :  true_type {};
+    template<> struct is_integral<unsigned long long> :  true_type {};
+
+    template<typename T> struct is_signed    : false_type {};
+    template<> struct is_signed<float>       :  true_type {};
+    template<> struct is_signed<double>      :  true_type {};
+    template<> struct is_signed<long double> :  true_type {};
+    template<> struct is_signed<signed char> :  true_type {};
+    template<> struct is_signed<short>       :  true_type {};
+    template<> struct is_signed<int>         :  true_type {};
+    template<> struct is_signed<long>        :  true_type {};
+    template<> struct is_signed<long long>   :  true_type {};
+
+    template<typename T> struct is_unsigned             : false_type {};
+    template<> struct is_unsigned<unsigned char>      :  true_type {};
+    template<> struct is_unsigned<unsigned short>     :  true_type {};
+    template<> struct is_unsigned<unsigned int>       :  true_type {};
+    template<> struct is_unsigned<unsigned long>      :  true_type {};
+    template<> struct is_unsigned<unsigned long long> :  true_type {};
+
+    template<typename T, typename U> struct is_same      : false_type {};
+    template<typename T>             struct is_same<T,T> :  true_type {};
+
+    template<class T> struct is_array : false_type {};
+    template<class T> struct is_array<T[]> : true_type {};
+    template<class T, size_t N> struct is_array<T[N]> : true_type {};
+
+    //partial implementation only of is_function
+    template<class> struct is_function : false_type { };
+    template<class Ret, class... Args> struct is_function<Ret(Args...)> : true_type {}; //regular
+    template<class Ret, class... Args> struct is_function<Ret(Args......)> : true_type {}; // variadic
+
+    template<class> struct result_of;
+    template<class F, typename... Args>
+    struct result_of<F(Args...)> {
+    // TODO: This is a hack; a proper implem is quite complicated.
+    typedef typename F::result_type type;
+    };
+
+    template <class T> struct remove_reference { typedef T type; };
+    template <class T> struct remove_reference<T&> { typedef T type; };
+    template <class T> struct remove_reference<T&&> { typedef T type; };
+    #if __cplusplus >= 201402L
+    template< class T > using remove_reference_t = typename remove_reference<T>::type;
+    #endif
+
+    template<class T> struct remove_extent { typedef T type; };
+    template<class T> struct remove_extent<T[]> { typedef T type; };
+    template<class T, size_t N> struct remove_extent<T[N]> { typedef T type; };
+    #if __cplusplus >= 201402L
+    template< class T > using remove_extent_t = typename remove_extent<T>::type;
+    #endif
+
+    template< class T > struct remove_const          { typedef T type; };
+    template< class T > struct remove_const<const T> { typedef T type; };
+    template< class T > struct remove_volatile             { typedef T type; };
+    template< class T > struct remove_volatile<volatile T> { typedef T type; };
+    template< class T > struct remove_cv { typedef typename remove_volatile<typename remove_const<T>::type>::type type; };
+    #if __cplusplus >= 201402L
+    template< class T > using remove_cv_t       = typename remove_cv<T>::type;
+    template< class T > using remove_const_t    = typename remove_const<T>::type;
+    template< class T > using remove_volatile_t = typename remove_volatile<T>::type;
+    #endif
+
+    template<bool B, class T, class F> struct conditional { typedef T type; };
+    template<class T, class F> struct conditional<false, T, F> { typedef F type; };
+    #if __cplusplus >= 201402L
+    template< bool B, class T, class F > using conditional_t = typename conditional<B,T,F>::type;
+    #endif
+
+    namespace __jitify_detail {
+    template< class T, bool is_function_type = false > struct add_pointer { using type = typename remove_reference<T>::type*; };
+    template< class T > struct add_pointer<T, true> { using type = T; };
+    template< class T, class... Args > struct add_pointer<T(Args...), true> { using type = T(*)(Args...); };
+    template< class T, class... Args > struct add_pointer<T(Args..., ...), true> { using type = T(*)(Args..., ...); };
+    }  // namespace __jitify_detail
+    template< class T > struct add_pointer : __jitify_detail::add_pointer<T, is_function<T>::value> {};
+    #if __cplusplus >= 201402L
+    template< class T > using add_pointer_t = typename add_pointer<T>::type;
+    #endif
+
+    template< class T > struct decay {
+    private:
+      typedef typename remove_reference<T>::type U;
+    public:
+      typedef typename conditional<is_array<U>::value, typename remove_extent<U>::type*,
+        typename conditional<is_function<U>::value,typename add_pointer<U>::type,typename remove_cv<U>::type
+        >::type>::type type;
+    };
+    #if __cplusplus >= 201402L
+    template< class T > using decay_t = typename decay<T>::type;
+    #endif
+
+    template<class T, T v>
+    struct integral_constant {
+    static constexpr T value = v;
+    typedef T value_type;
+    typedef integral_constant type; // using injected-class-name
+    constexpr operator value_type() const noexcept { return value; }
+    #if __cplusplus >= 201402L
+    constexpr value_type operator()() const noexcept { return value; }
+    #endif
+    };
+
+    template<class T> struct is_lvalue_reference : false_type {};
+    template<class T> struct is_lvalue_reference<T&> : true_type {};
+
+    template<class T> struct is_rvalue_reference : false_type {};
+    template<class T> struct is_rvalue_reference<T&&> : true_type {};
+
+    namespace __jitify_detail {
+    template <class T> struct type_identity { using type = T; };
+    template <class T> auto add_lvalue_reference(int) -> type_identity<T&>;
+    template <class T> auto add_lvalue_reference(...) -> type_identity<T>;
+    template <class T> auto add_rvalue_reference(int) -> type_identity<T&&>;
+    template <class T> auto add_rvalue_reference(...) -> type_identity<T>;
+    } // namespace _jitify_detail
+
+    template <class T> struct add_lvalue_reference : decltype(__jitify_detail::add_lvalue_reference<T>(0)) {};
+    template <class T> struct add_rvalue_reference : decltype(__jitify_detail::add_rvalue_reference<T>(0)) {};
+    #if __cplusplus >= 201402L
+    template <class T> using add_lvalue_reference_t = typename add_lvalue_reference<T>::type;
+    template <class T> using add_rvalue_reference_t = typename add_rvalue_reference<T>::type;
+    #endif
+
+    template<typename T> struct is_const          : public false_type {};
+    template<typename T> struct is_const<const T> : public true_type {};
+
+    template<typename T> struct is_volatile             : public false_type {};
+    template<typename T> struct is_volatile<volatile T> : public true_type {};
+
+    template<typename T> struct is_void             : public false_type {};
+    template<>           struct is_void<void>       : public true_type {};
+    template<>           struct is_void<const void> : public true_type {};
+
+    template<typename T> struct is_reference     : public false_type {};
+    template<typename T> struct is_reference<T&> : public true_type {};
+
+    template<typename _Tp, bool = (is_void<_Tp>::value || is_reference<_Tp>::value)>
+    struct __add_reference_helper { typedef _Tp&    type; };
+
+    template<typename _Tp> struct __add_reference_helper<_Tp, true> { typedef _Tp     type; };
+    template<typename _Tp> struct add_reference : public __add_reference_helper<_Tp>{};
+
+    namespace __jitify_detail {
+    template<typename T> struct is_int_or_cref {
+    typedef typename remove_reference<T>::type type_sans_ref;
+    static const bool value = (is_integral<T>::value || (is_integral<type_sans_ref>::value
+      && is_const<type_sans_ref>::value && !is_volatile<type_sans_ref>::value));
+    }; // end is_int_or_cref
+    template<typename From, typename To> struct is_convertible_sfinae {
+    private:
+    typedef char                          yes;
+    typedef struct { char two_chars[2]; } no;
+    static inline yes   test(To) { return yes(); }
+    static inline no    test(...) { return no(); }
+    static inline typename remove_reference<From>::type& from() { typename remove_reference<From>::type* ptr = 0; return *ptr; }
+    public:
+    static const bool value = sizeof(test(from())) == sizeof(yes);
+    }; // end is_convertible_sfinae
+    template<typename From, typename To> struct is_convertible_needs_simple_test {
+    static const bool from_is_void      = is_void<From>::value;
+    static const bool to_is_void        = is_void<To>::value;
+    static const bool from_is_float     = is_floating_point<typename remove_reference<From>::type>::value;
+    static const bool to_is_int_or_cref = is_int_or_cref<To>::value;
+    static const bool value = (from_is_void || to_is_void || (from_is_float && to_is_int_or_cref));
+    }; // end is_convertible_needs_simple_test
+    template<typename From, typename To, bool = is_convertible_needs_simple_test<From,To>::value>
+    struct is_convertible {
+    static const bool value = (is_void<To>::value || (is_int_or_cref<To>::value && !is_void<From>::value));
+    }; // end is_convertible
+    template<typename From, typename To> struct is_convertible<From, To, false> {
+    static const bool value = (is_convertible_sfinae<typename add_reference<From>::type, To>::value);
+    }; // end is_convertible
+    } // end __jitify_detail
+    // implementation of is_convertible taken from thrust's pre C++11 path
+    template<typename From, typename To> struct is_convertible
+    : public integral_constant<bool, __jitify_detail::is_convertible<From, To>::value>
+    { }; // end is_convertible
+
+    template<class A, class B> struct is_base_of { };
+
+    template<size_t len, size_t alignment> struct aligned_storage { struct type { alignas(alignment) char data[len]; }; };
+    template <class T> struct alignment_of : std::integral_constant<size_t,alignof(T)> {};
+
+    template <typename T> struct make_unsigned;
+    template <> struct make_unsigned<signed char>        { typedef unsigned char type; };
+    template <> struct make_unsigned<signed short>       { typedef unsigned short type; };
+    template <> struct make_unsigned<signed int>         { typedef unsigned int type; };
+    template <> struct make_unsigned<signed long>        { typedef unsigned long type; };
+    template <> struct make_unsigned<signed long long>   { typedef unsigned long long type; };
+    template <> struct make_unsigned<unsigned char>      { typedef unsigned char type; };
+    template <> struct make_unsigned<unsigned short>     { typedef unsigned short type; };
+    template <> struct make_unsigned<unsigned int>       { typedef unsigned int type; };
+    template <> struct make_unsigned<unsigned long>      { typedef unsigned long type; };
+    template <> struct make_unsigned<unsigned long long> { typedef unsigned long long type; };
+    template <> struct make_unsigned<char>               { typedef unsigned char type; };
+    #if defined _WIN32 || defined _WIN64
+    template <> struct make_unsigned<wchar_t>            { typedef unsigned short type; };
+    #else
+    template <> struct make_unsigned<wchar_t>            { typedef unsigned int type; };
+    #endif
+
+    template <typename T> struct make_signed;
+    template <> struct make_signed<signed char>        { typedef signed char type; };
+    template <> struct make_signed<signed short>       { typedef signed short type; };
+    template <> struct make_signed<signed int>         { typedef signed int type; };
+    template <> struct make_signed<signed long>        { typedef signed long type; };
+    template <> struct make_signed<signed long long>   { typedef signed long long type; };
+    template <> struct make_signed<unsigned char>      { typedef signed char type; };
+    template <> struct make_signed<unsigned short>     { typedef signed short type; };
+    template <> struct make_signed<unsigned int>       { typedef signed int type; };
+    template <> struct make_signed<unsigned long>      { typedef signed long type; };
+    template <> struct make_signed<unsigned long long> { typedef signed long long type; };
+    template <> struct make_signed<char>               { typedef signed char type; };
+    #if defined _WIN32 || defined _WIN64
+    template <> struct make_signed<wchar_t>            { typedef signed short type; };
+    #else
+    template <> struct make_signed<wchar_t>            { typedef signed int type; };
+    #endif
+
+    }  // namespace std
+    #endif // c++11
+)";
+
+// TODO: INT_FAST8_MAX et al. and a few other misc constants
+static const char* jitsafe_header_stdint_h =
+    "#pragma once\n"
+    "#include <climits>\n"
+    "namespace __jitify_stdint_ns {\n"
+    "typedef signed char      int8_t;\n"
+    "typedef signed short     int16_t;\n"
+    "typedef signed int       int32_t;\n"
+    "typedef signed long long int64_t;\n"
+    "typedef signed char      int_fast8_t;\n"
+    "typedef signed short     int_fast16_t;\n"
+    "typedef signed int       int_fast32_t;\n"
+    "typedef signed long long int_fast64_t;\n"
+    "typedef signed char      int_least8_t;\n"
+    "typedef signed short     int_least16_t;\n"
+    "typedef signed int       int_least32_t;\n"
+    "typedef signed long long int_least64_t;\n"
+    "typedef signed long long intmax_t;\n"
+    "typedef signed long      intptr_t; //optional\n"
+    "typedef unsigned char      uint8_t;\n"
+    "typedef unsigned short     uint16_t;\n"
+    "typedef unsigned int       uint32_t;\n"
+    "typedef unsigned long long uint64_t;\n"
+    "typedef unsigned char      uint_fast8_t;\n"
+    "typedef unsigned short     uint_fast16_t;\n"
+    "typedef unsigned int       uint_fast32_t;\n"
+    "typedef unsigned long long uint_fast64_t;\n"
+    "typedef unsigned char      uint_least8_t;\n"
+    "typedef unsigned short     uint_least16_t;\n"
+    "typedef unsigned int       uint_least32_t;\n"
+    "typedef unsigned long long uint_least64_t;\n"
+    "typedef unsigned long long uintmax_t;\n"
+    "#define INT8_MIN    SCHAR_MIN\n"
+    "#define INT16_MIN   SHRT_MIN\n"
+    "#if defined _WIN32 || defined _WIN64\n"
+    "#define WCHAR_MIN   0\n"
+    "#define WCHAR_MAX   USHRT_MAX\n"
+    "typedef unsigned long long uintptr_t; //optional\n"
+    "#else\n"
+    "#define WCHAR_MIN   INT_MIN\n"
+    "#define WCHAR_MAX   INT_MAX\n"
+    "typedef unsigned long      uintptr_t; //optional\n"
+    "#endif\n"
+    "#define INT32_MIN   INT_MIN\n"
+    "#define INT64_MIN   LLONG_MIN\n"
+    "#define INT8_MAX    SCHAR_MAX\n"
+    "#define INT16_MAX   SHRT_MAX\n"
+    "#define INT32_MAX   INT_MAX\n"
+    "#define INT64_MAX   LLONG_MAX\n"
+    "#define UINT8_MAX   UCHAR_MAX\n"
+    "#define UINT16_MAX  USHRT_MAX\n"
+    "#define UINT32_MAX  UINT_MAX\n"
+    "#define UINT64_MAX  ULLONG_MAX\n"
+    "#define INTPTR_MIN  LONG_MIN\n"
+    "#define INTMAX_MIN  LLONG_MIN\n"
+    "#define INTPTR_MAX  LONG_MAX\n"
+    "#define INTMAX_MAX  LLONG_MAX\n"
+    "#define UINTPTR_MAX ULONG_MAX\n"
+    "#define UINTMAX_MAX ULLONG_MAX\n"
+    "#define PTRDIFF_MIN INTPTR_MIN\n"
+    "#define PTRDIFF_MAX INTPTR_MAX\n"
+    "#define SIZE_MAX    UINT64_MAX\n"
+    "} // namespace __jitify_stdint_ns\n"
+    "namespace std { using namespace __jitify_stdint_ns; }\n"
+    "using namespace __jitify_stdint_ns;\n";
+
+// TODO: offsetof
+static const char* jitsafe_header_stddef_h =
+    "#pragma once\n"
+    "#include <climits>\n"
+    "namespace __jitify_stddef_ns {\n"
+    "#if __cplusplus >= 201103L\n"
+    "typedef decltype(nullptr) nullptr_t;\n"
+    "#if defined(_MSC_VER)\n"
+    "  typedef double max_align_t;\n"
+    "#elif defined(__APPLE__)\n"
+    "  typedef long double max_align_t;\n"
+    "#else\n"
+    "  // Define max_align_t to match the GCC definition.\n"
+    "  typedef struct {\n"
+    "    long long __jitify_max_align_nonce1\n"
+    "        __attribute__((__aligned__(__alignof__(long long))));\n"
+    "    long double __jitify_max_align_nonce2\n"
+    "        __attribute__((__aligned__(__alignof__(long double))));\n"
+    "  } max_align_t;\n"
+    "#endif\n"
+    "#endif  // __cplusplus >= 201103L\n"
+    "#if __cplusplus >= 201703L\n"
+    "enum class byte : unsigned char {};\n"
+    "#endif  // __cplusplus >= 201703L\n"
+    "} // namespace __jitify_stddef_ns\n"
+    "namespace std {\n"
+    "  // NVRTC provides built-in definitions of ::size_t and ::ptrdiff_t.\n"
+    "  using ::size_t;\n"
+    "  using ::ptrdiff_t;\n"
+    "  using namespace __jitify_stddef_ns;\n"
+    "} // namespace std\n"
+    "using namespace __jitify_stddef_ns;\n";
+
+static const char* jitsafe_header_stdlib_h =
+    "#pragma once\n"
+    "#include <stddef.h>\n";
+static const char* jitsafe_header_stdio_h =
+    "#pragma once\n"
+    "#include <stddef.h>\n"
+    "#define FILE int\n"
+    "int fflush ( FILE * stream );\n"
+    "int fprintf ( FILE * stream, const char * format, ... );\n";
+
+static const char* jitsafe_header_string_h =
+    "#pragma once\n"
+    "char* strcpy ( char * destination, const char * source );\n"
+    "int strcmp ( const char * str1, const char * str2 );\n"
+    "char* strerror( int errnum );\n";
+
+static const char* jitsafe_header_cstring =
+    "#pragma once\n"
+    "\n"
+    "namespace __jitify_cstring_ns {\n"
+    "char* strcpy ( char * destination, const char * source );\n"
+    "int strcmp ( const char * str1, const char * str2 );\n"
+    "char* strerror( int errnum );\n"
+    "} // namespace __jitify_cstring_ns\n"
+    "namespace std { using namespace __jitify_cstring_ns; }\n"
+    "using namespace __jitify_cstring_ns;\n";
+
+// HACK TESTING (WAR for cub)
+static const char* jitsafe_header_iostream =
+    "#pragma once\n"
+    "#include <ostream>\n"
+    "#include <istream>\n";
+// HACK TESTING (WAR for Thrust)
+static const char* jitsafe_header_ostream =
+    "#pragma once\n"
+    "\n"
+    "namespace std {\n"
+    "template<class CharT,class Traits=void>\n"  // = std::char_traits<CharT>
+                                                 // >\n"
+    "struct basic_ostream {\n"
+    "};\n"
+    "typedef basic_ostream<char> ostream;\n"
+    "ostream& endl(ostream& os);\n"
+    "ostream& operator<<( ostream&, ostream& (*f)( ostream& ) );\n"
+    "template< class CharT, class Traits > basic_ostream<CharT, Traits>& endl( "
+    "basic_ostream<CharT, Traits>& os );\n"
+    "template< class CharT, class Traits > basic_ostream<CharT, Traits>& "
+    "operator<<( basic_ostream<CharT,Traits>& os, const char* c );\n"
+    "#if __cplusplus >= 201103L\n"
+    "template< class CharT, class Traits, class T > basic_ostream<CharT, "
+    "Traits>& operator<<( basic_ostream<CharT,Traits>&& os, const T& value );\n"
+    "#endif  // __cplusplus >= 201103L\n"
+    "}  // namespace std\n";
+
+static const char* jitsafe_header_istream =
+    "#pragma once\n"
+    "\n"
+    "namespace std {\n"
+    "template<class CharT,class Traits=void>\n"  // = std::char_traits<CharT>
+                                                 // >\n"
+    "struct basic_istream {\n"
+    "};\n"
+    "typedef basic_istream<char> istream;\n"
+    "}  // namespace std\n";
+
+static const char* jitsafe_header_sstream =
+    "#pragma once\n"
+    "#include <ostream>\n"
+    "#include <istream>\n";
+
+static const char* jitsafe_header_utility =
+    "#pragma once\n"
+    "namespace std {\n"
+    "template<class T1, class T2>\n"
+    "struct pair {\n"
+    "	T1 first;\n"
+    "	T2 second;\n"
+    "	inline pair() {}\n"
+    "	inline pair(T1 const& first_, T2 const& second_)\n"
+    "		: first(first_), second(second_) {}\n"
+    "	// TODO: Standard includes many more constructors...\n"
+    "	// TODO: Comparison operators\n"
+    "};\n"
+    "template<class T1, class T2>\n"
+    "pair<T1,T2> make_pair(T1 const& first, T2 const& second) {\n"
+    "	return pair<T1,T2>(first, second);\n"
+    "}\n"
+    "}  // namespace std\n";
+
+// TODO: incomplete
+static const char* jitsafe_header_vector =
+    "#pragma once\n"
+    "namespace std {\n"
+    "template<class T, class Allocator=void>\n"  // = std::allocator> \n"
+    "struct vector {\n"
+    "};\n"
+    "}  // namespace std\n";
+
+// TODO: incomplete
+static const char* jitsafe_header_string =
+    "#pragma once\n"
+    "namespace std {\n"
+    "template<class CharT,class Traits=void,class Allocator=void>\n"
+    "struct basic_string {\n"
+    "basic_string();\n"
+    "basic_string( const CharT* s );\n"  //, const Allocator& alloc =
+                                         // Allocator() );\n"
+    "const CharT* c_str() const;\n"
+    "bool empty() const;\n"
+    "void operator+=(const char *);\n"
+    "void operator+=(const basic_string &);\n"
+    "};\n"
+    "typedef basic_string<char> string;\n"
+    "}  // namespace std\n";
+
+// TODO: incomplete
+static const char* jitsafe_header_stdexcept =
+    "#pragma once\n"
+    "namespace std {\n"
+    "struct runtime_error {\n"
+    "explicit runtime_error( const std::string& what_arg );"
+    "explicit runtime_error( const char* what_arg );"
+    "virtual const char* what() const;\n"
+    "};\n"
+    "}  // namespace std\n";
+
+// TODO: incomplete
+static const char* jitsafe_header_complex =
+    "#pragma once\n"
+    "namespace std {\n"
+    "template<typename T>\n"
+    "class complex {\n"
+    "	T _real;\n"
+    "	T _imag;\n"
+    "public:\n"
+    "	complex() : _real(0), _imag(0) {}\n"
+    "	complex(T const& real, T const& imag)\n"
+    "		: _real(real), _imag(imag) {}\n"
+    "	complex(T const& real)\n"
+    "               : _real(real), _imag(static_cast<T>(0)) {}\n"
+    "	T const& real() const { return _real; }\n"
+    "	T&       real()       { return _real; }\n"
+    "	void real(const T &r) { _real = r; }\n"
+    "	T const& imag() const { return _imag; }\n"
+    "	T&       imag()       { return _imag; }\n"
+    "	void imag(const T &i) { _imag = i; }\n"
+    "       complex<T>& operator+=(const complex<T> z)\n"
+    "         { _real += z.real(); _imag += z.imag(); return *this; }\n"
+    "};\n"
+    "template<typename T>\n"
+    "complex<T> operator*(const complex<T>& lhs, const complex<T>& rhs)\n"
+    "  { return complex<T>(lhs.real()*rhs.real()-lhs.imag()*rhs.imag(),\n"
+    "                      lhs.real()*rhs.imag()+lhs.imag()*rhs.real()); }\n"
+    "template<typename T>\n"
+    "complex<T> operator*(const complex<T>& lhs, const T & rhs)\n"
+    "  { return complexs<T>(lhs.real()*rhs,lhs.imag()*rhs); }\n"
+    "template<typename T>\n"
+    "complex<T> operator*(const T& lhs, const complex<T>& rhs)\n"
+    "  { return complexs<T>(rhs.real()*lhs,rhs.imag()*lhs); }\n"
+    "}  // namespace std\n";
+
+// TODO: This is incomplete (missing binary and integer funcs, macros,
+// constants, types)
+static const char* jitsafe_header_math_h =
+    "#pragma once\n"
+    "namespace __jitify_math_ns {\n"
+    "#if __cplusplus >= 201103L\n"
+    "#define DEFINE_MATH_UNARY_FUNC_WRAPPER(f) \\\n"
+    "	inline double      f(double x)         { return ::f(x); } \\\n"
+    "	inline float       f##f(float x)       { return ::f(x); } \\\n"
+    "	/*inline long double f##l(long double x) { return ::f(x); }*/ \\\n"
+    "	inline float       f(float x)          { return ::f(x); } \\\n"
+    "	/*inline long double f(long double x)    { return ::f(x); }*/\n"
+    "#else\n"
+    "#define DEFINE_MATH_UNARY_FUNC_WRAPPER(f) \\\n"
+    "	inline double      f(double x)         { return ::f(x); } \\\n"
+    "	inline float       f##f(float x)       { return ::f(x); } \\\n"
+    "	/*inline long double f##l(long double x) { return ::f(x); }*/\n"
+    "#endif\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(cos)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(sin)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(tan)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(acos)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(asin)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(atan)\n"
+    "template<typename T> inline T atan2(T y, T x) { return ::atan2(y, x); }\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(cosh)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(sinh)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(tanh)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(exp)\n"
+    "template<typename T> inline T frexp(T x, int* exp) { return ::frexp(x, "
+    "exp); }\n"
+    "template<typename T> inline T ldexp(T x, int  exp) { return ::ldexp(x, "
+    "exp); }\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(log)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(log10)\n"
+    "template<typename T> inline T modf(T x, T* intpart) { return ::modf(x, "
+    "intpart); }\n"
+    "template<typename T> inline T pow(T x, T y) { return ::pow(x, y); }\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(sqrt)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(ceil)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(floor)\n"
+    "template<typename T> inline T fmod(T n, T d) { return ::fmod(n, d); }\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(fabs)\n"
+    "template<typename T> inline T abs(T x) { return ::abs(x); }\n"
+    "#if __cplusplus >= 201103L\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(acosh)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(asinh)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(atanh)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(exp2)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(expm1)\n"
+    "template<typename T> inline int ilogb(T x) { return ::ilogb(x); }\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(log1p)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(log2)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(logb)\n"
+    "template<typename T> inline T scalbn (T x, int n)  { return ::scalbn(x, "
+    "n); }\n"
+    "template<typename T> inline T scalbln(T x, long n) { return ::scalbn(x, "
+    "n); }\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(cbrt)\n"
+    "template<typename T> inline T hypot(T x, T y) { return ::hypot(x, y); }\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(erf)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(erfc)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(tgamma)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(lgamma)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(trunc)\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(round)\n"
+    "template<typename T> inline long lround(T x) { return ::lround(x); }\n"
+    "template<typename T> inline long long llround(T x) { return ::llround(x); "
+    "}\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(rint)\n"
+    "template<typename T> inline long lrint(T x) { return ::lrint(x); }\n"
+    "template<typename T> inline long long llrint(T x) { return ::llrint(x); "
+    "}\n"
+    "DEFINE_MATH_UNARY_FUNC_WRAPPER(nearbyint)\n"
+    // TODO: remainder, remquo, copysign, nan, nextafter, nexttoward, fdim,
+    // fmax, fmin, fma
+    "#endif\n"
+    "#undef DEFINE_MATH_UNARY_FUNC_WRAPPER\n"
+    "} // namespace __jitify_math_ns\n"
+    "namespace std { using namespace __jitify_math_ns; }\n"
+    "#define M_PI 3.14159265358979323846\n"
+    // Note: Global namespace already includes CUDA math funcs
+    "//using namespace __jitify_math_ns;\n";
+
+static const char* jitsafe_header_memory_h = R"(
+    #pragma once
+    #include <string.h>
+ )";
+
+// TODO: incomplete
+static const char* jitsafe_header_mutex = R"(
+    #pragma once
+    #if __cplusplus >= 201103L
+    namespace std {
+    class mutex {
+    public:
+    void lock();
+    bool try_lock();
+    void unlock();
+    };
+    }  // namespace std
+    #endif
+ )";
+
+static const char* jitsafe_header_algorithm = R"(
+    #pragma once
+    #if __cplusplus >= 201103L
+    namespace std {
+
+    #if __cplusplus == 201103L
+    #define JITIFY_CXX14_CONSTEXPR
+    #else
+    #define JITIFY_CXX14_CONSTEXPR constexpr
+    #endif
+
+    template<class T> JITIFY_CXX14_CONSTEXPR const T& max(const T& a, const T& b)
+    {
+      return (b > a) ? b : a;
+    }
+    template<class T> JITIFY_CXX14_CONSTEXPR const T& min(const T& a, const T& b)
+    {
+      return (b < a) ? b : a;
+    }
+
+    }  // namespace std
+    #endif
+ )";
+
+static const char* jitsafe_header_time_h = R"(
+    #pragma once
+    #define NULL 0
+    #define CLOCKS_PER_SEC 1000000
+    namespace __jitify_time_ns {
+    typedef long time_t;
+    struct tm {
+      int tm_sec;
+      int tm_min;
+      int tm_hour;
+      int tm_mday;
+      int tm_mon;
+      int tm_year;
+      int tm_wday;
+      int tm_yday;
+      int tm_isdst;
+    };
+    #if __cplusplus >= 201703L
+    struct timespec {
+      time_t tv_sec;
+      long tv_nsec;
+    };
+    #endif
+    }  // namespace __jitify_time_ns
+    namespace std {
+      // NVRTC provides built-in definitions of ::size_t and ::clock_t.
+      using ::size_t;
+      using ::clock_t;
+      using namespace __jitify_time_ns;
+    }
+    using namespace __jitify_time_ns;
+ )";
+
+static const char* jitsafe_header_tuple = R"(
+    #pragma once
+    #if __cplusplus >= 201103L
+    namespace std {
+    template<class... Types > class tuple;
+    } // namespace std
+    #endif
+ )";
+
+static const char* jitsafe_header_assert = R"(
+    #pragma once
+ )";
+
+// WAR: These need to be pre-included as a workaround for NVRTC implicitly using
+// /usr/include as an include path. The other built-in headers will be included
+// lazily as needed.
+static const char* preinclude_jitsafe_header_names[] = {"jitify_preinclude.h",
+                                                        "limits.h",
+                                                        "math.h",
+                                                        "memory.h",
+                                                        "stdint.h",
+                                                        "stdlib.h",
+                                                        "stdio.h",
+                                                        "string.h",
+                                                        "time.h",
+                                                        "assert.h"};
+
+template <class T, int N>
+int array_size(T (&)[N]) {
+  return N;
+}
+const int preinclude_jitsafe_headers_count =
+    array_size(preinclude_jitsafe_header_names);
+
+static const std::map<std::string, std::string>& get_jitsafe_headers_map() {
+  static const std::map<std::string, std::string> jitsafe_headers_map = {
+      {"jitify_preinclude.h", jitsafe_header_preinclude_h},
+      {"float.h", jitsafe_header_float_h},
+      {"cfloat", jitsafe_header_float_h},
+      {"limits.h", jitsafe_header_limits_h},
+      {"climits", jitsafe_header_limits_h},
+      {"stdint.h", jitsafe_header_stdint_h},
+      {"cstdint", jitsafe_header_stdint_h},
+      {"stddef.h", jitsafe_header_stddef_h},
+      {"cstddef", jitsafe_header_stddef_h},
+      {"stdlib.h", jitsafe_header_stdlib_h},
+      {"cstdlib", jitsafe_header_stdlib_h},
+      {"stdio.h", jitsafe_header_stdio_h},
+      {"cstdio", jitsafe_header_stdio_h},
+      {"string.h", jitsafe_header_string_h},
+      {"cstring", jitsafe_header_cstring},
+      {"iterator", jitsafe_header_iterator},
+      {"limits", jitsafe_header_limits},
+      {"type_traits", jitsafe_header_type_traits},
+      {"utility", jitsafe_header_utility},
+      {"math.h", jitsafe_header_math_h},
+      {"cmath", jitsafe_header_math_h},
+      {"memory.h", jitsafe_header_memory_h},
+      {"complex", jitsafe_header_complex},
+      {"iostream", jitsafe_header_iostream},
+      {"ostream", jitsafe_header_ostream},
+      {"istream", jitsafe_header_istream},
+      {"sstream", jitsafe_header_sstream},
+      {"vector", jitsafe_header_vector},
+      {"string", jitsafe_header_string},
+      {"stdexcept", jitsafe_header_stdexcept},
+      {"mutex", jitsafe_header_mutex},
+      {"algorithm", jitsafe_header_algorithm},
+      {"time.h", jitsafe_header_time_h},
+      {"ctime", jitsafe_header_time_h},
+      {"tuple", jitsafe_header_tuple},
+      {"assert.h", jitsafe_header_assert},
+      {"cassert", jitsafe_header_assert}};
+  return jitsafe_headers_map;
+}
+
+inline void add_options_from_env(std::vector<std::string>& options) {
+  // Add options from environment variable
+  const char* env_options = std::getenv("JITIFY_OPTIONS");
+  if (env_options) {
+    std::stringstream ss;
+    ss << env_options;
+    std::string opt;
+    while (!(ss >> opt).fail()) {
+      options.push_back(opt);
+    }
+  }
+  // Add options from JITIFY_OPTIONS macro
+#ifdef JITIFY_OPTIONS
+#define JITIFY_TOSTRING_IMPL(x) #x
+#define JITIFY_TOSTRING(x) JITIFY_TOSTRING_IMPL(x)
+  std::stringstream ss;
+  ss << JITIFY_TOSTRING(JITIFY_OPTIONS);
+  std::string opt;
+  while (!(ss >> opt).fail()) {
+    options.push_back(opt);
+  }
+#undef JITIFY_TOSTRING
+#undef JITIFY_TOSTRING_IMPL
+#endif  // JITIFY_OPTIONS
+}
+
+inline void detect_and_add_cuda_arch(std::vector<std::string>& options) {
+  for (int i = 0; i < (int)options.size(); ++i) {
+    // Note that this will also match the middle of "--gpu-architecture".
+    if (options[i].find("-arch") != std::string::npos) {
+      // Arch already specified in options
+      return;
+    }
+  }
+  // Use the compute capability of the current device
+  // TODO: Check these API calls for errors
+  cudaError_t status;
+  int device;
+  status = cudaGetDevice(&device);
+  if (status != cudaSuccess) {
+    throw std::runtime_error(
+        std::string(
+            "Failed to detect GPU architecture: cudaGetDevice failed: ") +
+        cudaGetErrorString(status));
+  }
+  int cc_major;
+  cudaDeviceGetAttribute(&cc_major, cudaDevAttrComputeCapabilityMajor, device);
+  int cc_minor;
+  cudaDeviceGetAttribute(&cc_minor, cudaDevAttrComputeCapabilityMinor, device);
+  int cc = cc_major * 10 + cc_minor;
+  // Note: We must limit the architecture to the max supported by the current
+  //         version of NVRTC, otherwise newer hardware will cause errors
+  //         on older versions of CUDA.
+  // TODO: It would be better to detect this somehow, rather than hard-coding it
+
+  // Tegra chips do not have forwards compatibility so we need to special case
+  // them.
+  bool is_tegra = ((cc_major == 3 && cc_minor == 2) ||  // Logan
+                   (cc_major == 5 && cc_minor == 3) ||  // Erista
+                   (cc_major == 6 && cc_minor == 2) ||  // Parker
+                   (cc_major == 7 && cc_minor == 2));   // Xavier
+  if (!is_tegra) {
+    // ensure that future CUDA versions just work (even if suboptimal)
+    const int cuda_major = std::min(10, CUDA_VERSION / 1000);
+    // clang-format off
+    switch (cuda_major) {
+      case 10: cc = std::min(cc, 75); break; // Turing
+      case  9: cc = std::min(cc, 70); break; // Volta
+      case  8: cc = std::min(cc, 61); break; // Pascal
+      case  7: cc = std::min(cc, 52); break; // Maxwell
+      default:
+        throw std::runtime_error("Unexpected CUDA major version " +
+                                 std::to_string(cuda_major));
+    }
+    // clang-format on
+  }
+
+  std::stringstream ss;
+  ss << cc;
+  options.push_back("-arch=compute_" + ss.str());
+}
+
+inline void detect_and_add_cxx11_flag(std::vector<std::string>& options) {
+  // Reverse loop so we can erase on the fly.
+  for (int i = (int)options.size() - 1; i >= 0; --i) {
+    if (options[i].find("-std=c++98") != std::string::npos) {
+      // NVRTC doesn't support specifying c++98 explicitly, so we remove it.
+      options.erase(options.begin() + i);
+      return;
+    } else if (options[i].find("-std") != std::string::npos) {
+      // Some other standard was explicitly specified, don't change anything.
+      return;
+    }
+  }
+  // Jitify must be compiled with C++11 support, so we default to enabling it
+  // for the JIT-compiled code too.
+  options.push_back("-std=c++11");
+}
+
+inline void split_compiler_and_linker_options(
+    std::vector<std::string> options,
+    std::vector<std::string>* compiler_options,
+    std::vector<std::string>* linker_files,
+    std::vector<std::string>* linker_paths) {
+  for (int i = 0; i < (int)options.size(); ++i) {
+    std::string opt = options[i];
+    std::string flag = opt.substr(0, 2);
+    std::string value = opt.substr(2);
+    if (flag == "-l") {
+      linker_files->push_back(value);
+    } else if (flag == "-L") {
+      linker_paths->push_back(value);
+    } else {
+      compiler_options->push_back(opt);
+    }
+  }
+}
+
+inline bool pop_remove_unused_globals_flag(std::vector<std::string>* options) {
+  auto it = std::remove_if(
+      options->begin(), options->end(), [](const std::string& opt) {
+        return opt.find("-remove-unused-globals") != std::string::npos;
+      });
+  if (it != options->end()) {
+    options->resize(it - options->begin());
+    return true;
+  }
+  return false;
+}
+
+inline std::string ptx_parse_decl_name(const std::string& line) {
+  size_t name_end = line.find_first_of("[;");
+  if (name_end == std::string::npos) {
+    throw std::runtime_error(
+        "Failed to parse .global/.const declaration in PTX: expected a "
+        "semicolon");
+  }
+  size_t name_start_minus1 = line.find_last_of(" \t", name_end);
+  if (name_start_minus1 == std::string::npos) {
+    throw std::runtime_error(
+        "Failed to parse .global/.const declaration in PTX: expected "
+        "whitespace");
+  }
+  size_t name_start = name_start_minus1 + 1;
+  std::string name = line.substr(name_start, name_end - name_start);
+  return name;
+}
+
+inline void ptx_remove_unused_globals(std::string* ptx) {
+  std::istringstream iss(*ptx);
+  std::vector<std::string> lines;
+  std::unordered_map<size_t, std::string> line_num_to_global_name;
+  std::unordered_set<std::string> name_set;
+  for (std::string line; std::getline(iss, line);) {
+    size_t line_num = lines.size();
+    lines.push_back(line);
+    auto terms = split_string(line);
+    if (terms.size() <= 1) continue;  // Ignore lines with no arguments
+    if (terms[0].substr(0, 2) == "//") continue;  // Ignore comment lines
+    if (terms[0].substr(0, 7) == ".global" ||
+        terms[0].substr(0, 6) == ".const") {
+      line_num_to_global_name.emplace(line_num, ptx_parse_decl_name(line));
+      continue;
+    }
+    if (terms[0][0] == '.') continue;  // Ignore .version, .reg, .param etc.
+    // Note: The first term will always be an instruction name; starting at 1
+    // also allows unchecked inspection of the previous term.
+    for (int i = 1; i < (int)terms.size(); ++i) {
+      if (terms[i].substr(0, 2) == "//") break;  // Ignore comments
+      // Note: The characters '.' and '%' are not treated as delimiters.
+      const char* token_delims = " \t()[]{},;+-*/~&|^?:=!<>\"'\\";
+      for (auto token : split_string(terms[i], -1, token_delims)) {
+        if (  // Ignore non-names
+            !(std::isalpha(token[0]) || token[0] == '_' || token[0] == '$') ||
+            token.find('.') != std::string::npos ||
+            // Ignore variable/parameter declarations
+            terms[i - 1][0] == '.' ||
+            // Ignore branch instructions
+            (token == "bra" && terms[i - 1][0] == '@') ||
+            // Ignore branch labels
+            (token.substr(0, 2) == "BB" &&
+             terms[i - 1].substr(0, 3) == "bra")) {
+          continue;
+        }
+        name_set.insert(token);
+      }
+    }
+  }
+  std::ostringstream oss;
+  for (size_t line_num = 0; line_num < lines.size(); ++line_num) {
+    auto it = line_num_to_global_name.find(line_num);
+    if (it != line_num_to_global_name.end()) {
+      const std::string& name = it->second;
+      if (!name_set.count(name)) {
+        continue;  // Remove unused .global declaration.
+      }
+    }
+    oss << lines[line_num] << '\n';
+  }
+  *ptx = oss.str();
+}
+
+inline nvrtcResult compile_kernel(std::string program_name,
+                                  std::map<std::string, std::string> sources,
+                                  std::vector<std::string> options,
+                                  std::string instantiation = "",
+                                  std::string* log = 0, std::string* ptx = 0,
+                                  std::string* mangled_instantiation = 0) {
+  std::string program_source = sources[program_name];
+  // Build arrays of header names and sources
+  std::vector<const char*> header_names_c;
+  std::vector<const char*> header_sources_c;
+  int num_headers = (int)(sources.size() - 1);
+  header_names_c.reserve(num_headers);
+  header_sources_c.reserve(num_headers);
+  typedef std::map<std::string, std::string> source_map;
+  for (source_map::const_iterator iter = sources.begin(); iter != sources.end();
+       ++iter) {
+    std::string const& name = iter->first;
+    std::string const& code = iter->second;
+    if (name == program_name) {
+      continue;
+    }
+    header_names_c.push_back(name.c_str());
+    header_sources_c.push_back(code.c_str());
+  }
+
+  // TODO: This WAR is expected to be unnecessary as of CUDA > 10.2.
+  bool should_remove_unused_globals =
+      detail::pop_remove_unused_globals_flag(&options);
+
+  std::vector<const char*> options_c(options.size() + 2);
+  options_c[0] = "--device-as-default-execution-space";
+  options_c[1] = "--pre-include=jitify_preinclude.h";
+  for (int i = 0; i < (int)options.size(); ++i) {
+    options_c[i + 2] = options[i].c_str();
+  }
+
+#if CUDA_VERSION < 8000
+  std::string inst_dummy;
+  if (!instantiation.empty()) {
+    // WAR for no nvrtcAddNameExpression before CUDA 8.0
+    // Force template instantiation by adding dummy reference to kernel
+    inst_dummy = "__jitify_instantiation";
+    program_source +=
+        "\nvoid* " + inst_dummy + " = (void*)" + instantiation + ";\n";
+  }
+#endif
+
+#define CHECK_NVRTC(call)                         \
+  do {                                            \
+    nvrtcResult check_nvrtc_macro_ret = call;     \
+    if (check_nvrtc_macro_ret != NVRTC_SUCCESS) { \
+      return check_nvrtc_macro_ret;               \
+    }                                             \
+  } while (0)
+
+  nvrtcProgram nvrtc_program;
+  CHECK_NVRTC(nvrtcCreateProgram(
+      &nvrtc_program, program_source.c_str(), program_name.c_str(), num_headers,
+      header_sources_c.data(), header_names_c.data()));
+
+  // Ensure nvrtc_program gets destroyed.
+  struct ScopedNvrtcProgramDestroyer {
+    nvrtcProgram& nvrtc_program_;
+    ~ScopedNvrtcProgramDestroyer() { nvrtcDestroyProgram(&nvrtc_program_); }
+    ScopedNvrtcProgramDestroyer(const ScopedNvrtcProgramDestroyer&) = delete;
+    ScopedNvrtcProgramDestroyer& operator=(const ScopedNvrtcProgramDestroyer&) =
+        delete;
+  } nvrtc_program_scope_guard{nvrtc_program};
+
+#if CUDA_VERSION >= 8000
+  if (!instantiation.empty()) {
+    CHECK_NVRTC(nvrtcAddNameExpression(nvrtc_program, instantiation.c_str()));
+  }
+#endif
+
+  nvrtcResult ret = nvrtcCompileProgram(nvrtc_program, (int)options_c.size(),
+                                        options_c.data());
+  if (log) {
+    size_t logsize;
+    CHECK_NVRTC(nvrtcGetProgramLogSize(nvrtc_program, &logsize));
+    std::vector<char> vlog(logsize, 0);
+    CHECK_NVRTC(nvrtcGetProgramLog(nvrtc_program, vlog.data()));
+    log->assign(vlog.data(), logsize);
+  }
+  if (ret != NVRTC_SUCCESS) {
+    return ret;
+  }
+
+  if (ptx) {
+    size_t ptxsize;
+    CHECK_NVRTC(nvrtcGetPTXSize(nvrtc_program, &ptxsize));
+    std::vector<char> vptx(ptxsize);
+    CHECK_NVRTC(nvrtcGetPTX(nvrtc_program, vptx.data()));
+    ptx->assign(vptx.data(), ptxsize);
+    if (should_remove_unused_globals) {
+      detail::ptx_remove_unused_globals(ptx);
+    }
+  }
+
+  if (!instantiation.empty() && mangled_instantiation) {
+#if CUDA_VERSION >= 8000
+    const char* mangled_instantiation_cstr;
+    // Note: The returned string pointer becomes invalid after
+    //         nvrtcDestroyProgram has been called, so we save it.
+    CHECK_NVRTC(nvrtcGetLoweredName(nvrtc_program, instantiation.c_str(),
+                                    &mangled_instantiation_cstr));
+    *mangled_instantiation = mangled_instantiation_cstr;
+#else
+    // Extract mangled kernel template instantiation from PTX
+    inst_dummy += " = ";  // Note: This must match how the PTX is generated
+    int mi_beg = ptx->find(inst_dummy) + inst_dummy.size();
+    int mi_end = ptx->find(";", mi_beg);
+    *mangled_instantiation = ptx->substr(mi_beg, mi_end - mi_beg);
+#endif
+  }
+
+#undef CHECK_NVRTC
+  return NVRTC_SUCCESS;
+}
+
+inline void load_program(std::string const& cuda_source,
+                         std::vector<std::string> const& headers,
+                         file_callback_type file_callback,
+                         std::vector<std::string>* include_paths,
+                         std::map<std::string, std::string>* program_sources,
+                         std::vector<std::string>* program_options,
+                         std::string* program_name) {
+  // Extract include paths from compile options
+  std::vector<std::string>::iterator iter = program_options->begin();
+  while (iter != program_options->end()) {
+    std::string const& opt = *iter;
+    if (opt.substr(0, 2) == "-I") {
+      include_paths->push_back(opt.substr(2));
+      iter = program_options->erase(iter);
+    } else {
+      ++iter;
+    }
+  }
+
+  // Load program source
+  if (!detail::load_source(cuda_source, *program_sources, "", *include_paths,
+                           file_callback, program_name)) {
+    throw std::runtime_error("Source not found: " + cuda_source);
+  }
+
+  // Maps header include names to their full file paths.
+  std::map<std::string, std::string> header_fullpaths;
+
+  // Load header sources
+  for (std::string const& header : headers) {
+    if (!detail::load_source(header, *program_sources, "", *include_paths,
+                             file_callback, nullptr, &header_fullpaths)) {
+      // **TODO: Deal with source not found
+      throw std::runtime_error("Source not found: " + header);
+    }
+  }
+
+#if JITIFY_PRINT_SOURCE
+  std::string& program_source = (*program_sources)[*program_name];
+  std::cout << "---------------------------------------" << std::endl;
+  std::cout << "--- Source of " << *program_name << " ---" << std::endl;
+  std::cout << "---------------------------------------" << std::endl;
+  detail::print_with_line_numbers(program_source);
+  std::cout << "---------------------------------------" << std::endl;
+#endif
+
+  std::vector<std::string> compiler_options, linker_files, linker_paths;
+  detail::split_compiler_and_linker_options(*program_options, &compiler_options,
+                                            &linker_files, &linker_paths);
+
+  // If no arch is specified at this point we use whatever the current
+  // context is. This ensures we pick up the correct internal headers
+  // for arch-dependent compilation, e.g., some intrinsics are only
+  // present for specific architectures.
+  detail::detect_and_add_cuda_arch(compiler_options);
+  detail::detect_and_add_cxx11_flag(compiler_options);
+
+  // Iteratively try to compile the sources, and use the resulting errors to
+  // identify missing headers.
+  std::string log;
+  nvrtcResult ret;
+  while ((ret = detail::compile_kernel(*program_name, *program_sources,
+                                       compiler_options, "", &log)) ==
+         NVRTC_ERROR_COMPILATION) {
+    std::string include_name;
+    std::string include_parent;
+    int line_num = 0;
+    if (!detail::extract_include_info_from_compile_error(
+            log, include_name, include_parent, line_num)) {
+#if JITIFY_PRINT_LOG
+      detail::print_compile_log(*program_name, log);
+#endif
+      // There was a non include-related compilation error
+      // TODO: How to handle error?
+      throw std::runtime_error("Runtime compilation failed");
+    }
+
+    bool is_included_with_quotes = false;
+    if (program_sources->count(include_parent)) {
+      const std::string& parent_source = (*program_sources)[include_parent];
+      is_included_with_quotes =
+          is_include_directive_with_quotes(parent_source, line_num);
+    }
+
+    // Try to load the new header
+    // Note: This fullpath lookup is needed because the compiler error
+    // messages have the include name of the header instead of its full path.
+    std::string include_parent_fullpath = header_fullpaths[include_parent];
+    std::string include_path = detail::path_base(include_parent_fullpath);
+    if (detail::load_source(include_name, *program_sources, include_path,
+                            *include_paths, file_callback, nullptr,
+                            &header_fullpaths, is_included_with_quotes)) {
+#if JITIFY_PRINT_HEADER_PATHS
+      std::cout << "Found #include " << include_name << " from "
+                << include_parent << ":" << line_num << " ["
+                << include_parent_fullpath << "]"
+                << " at:\n  " << header_fullpaths[include_name] << std::endl;
+#endif
+    } else {  // Failed to find header file.
+      // Comment-out the include line and print a warning
+      if (!program_sources->count(include_parent)) {
+        // ***TODO: Unless there's another mechanism (e.g., potentially
+        //            the parent path vs. filename problem), getting
+        //            here means include_parent was found automatically
+        //            in a system include path.
+        //            We need a WAR to zap it from *its parent*.
+
+        typedef std::map<std::string, std::string> source_map;
+        for (source_map::const_iterator it = program_sources->begin();
+             it != program_sources->end(); ++it) {
+          std::cout << "  " << it->first << std::endl;
+        }
+        throw std::out_of_range(include_parent +
+                                " not in loaded sources!"
+                                " This may be due to a header being loaded by"
+                                " NVRTC without Jitify's knowledge.");
+      }
+      std::string& parent_source = (*program_sources)[include_parent];
+      parent_source = detail::comment_out_code_line(line_num, parent_source);
+#if JITIFY_PRINT_LOG
+      std::cout << include_parent << "(" << line_num
+                << "): warning: " << include_name << ": [jitify] File not found"
+                << std::endl;
+#endif
+    }
+  }
+  if (ret != NVRTC_SUCCESS) {
+#if JITIFY_PRINT_LOG
+    if (ret == NVRTC_ERROR_INVALID_OPTION) {
+      std::cout << "Compiler options: ";
+      for (int i = 0; i < (int)compiler_options.size(); ++i) {
+        std::cout << compiler_options[i] << " ";
+      }
+      std::cout << std::endl;
+    }
+#endif
+    throw std::runtime_error(std::string("NVRTC error: ") +
+                             nvrtcGetErrorString(ret));
+  }
+}
+
+inline void instantiate_kernel(
+    std::string const& program_name,
+    std::map<std::string, std::string> const& program_sources,
+    std::string const& instantiation, std::vector<std::string> const& options,
+    std::string* log, std::string* ptx, std::string* mangled_instantiation,
+    std::vector<std::string>* linker_files,
+    std::vector<std::string>* linker_paths) {
+  std::vector<std::string> compiler_options;
+  detail::split_compiler_and_linker_options(options, &compiler_options,
+                                            linker_files, linker_paths);
+
+  nvrtcResult ret =
+      detail::compile_kernel(program_name, program_sources, compiler_options,
+                             instantiation, log, ptx, mangled_instantiation);
+#if JITIFY_PRINT_LOG
+  if (log->size() > 1) {
+    detail::print_compile_log(program_name, *log);
+  }
+#endif
+  if (ret != NVRTC_SUCCESS) {
+    throw std::runtime_error(std::string("NVRTC error: ") +
+                             nvrtcGetErrorString(ret));
+  }
+
+#if JITIFY_PRINT_PTX
+  std::cout << "---------------------------------------" << std::endl;
+  std::cout << *mangled_instantiation << std::endl;
+  std::cout << "---------------------------------------" << std::endl;
+  std::cout << "--- PTX for " << mangled_instantiation << " in " << program_name
+            << " ---" << std::endl;
+  std::cout << "---------------------------------------" << std::endl;
+  std::cout << *ptx << std::endl;
+  std::cout << "---------------------------------------" << std::endl;
+#endif
+}
+
+inline void get_1d_max_occupancy(CUfunction func,
+                                 CUoccupancyB2DSize smem_callback,
+                                 unsigned int* smem, int max_block_size,
+                                 unsigned int flags, int* grid, int* block) {
+  if (!func) {
+    throw std::runtime_error(
+        "Kernel pointer is NULL; you may need to define JITIFY_THREAD_SAFE "
+        "1");
+  }
+  CUresult res = cuOccupancyMaxPotentialBlockSizeWithFlags(
+      grid, block, func, smem_callback, *smem, max_block_size, flags);
+  if (res != CUDA_SUCCESS) {
+    const char* msg;
+    cuGetErrorName(res, &msg);
+    throw std::runtime_error(msg);
+  }
+  if (smem_callback) {
+    *smem = (unsigned int)smem_callback(*block);
+  }
+}
+
+}  // namespace detail
+
+//! \endcond
+
+class KernelInstantiation;
+class Kernel;
+class Program;
+class JitCache;
+
+struct ProgramConfig {
+  std::vector<std::string> options;
+  std::vector<std::string> include_paths;
+  std::string name;
+  typedef std::map<std::string, std::string> source_map;
+  source_map sources;
+};
+
+class JitCache_impl {
+  friend class Program_impl;
+  friend class KernelInstantiation_impl;
+  friend class KernelLauncher_impl;
+  typedef uint64_t key_type;
+  jitify::ObjectCache<key_type, detail::CUDAKernel> _kernel_cache;
+  jitify::ObjectCache<key_type, ProgramConfig> _program_config_cache;
+  std::vector<std::string> _options;
+#if JITIFY_THREAD_SAFE
+  std::mutex _kernel_cache_mutex;
+  std::mutex _program_cache_mutex;
+#endif
+ public:
+  inline JitCache_impl(size_t cache_size)
+      : _kernel_cache(cache_size), _program_config_cache(cache_size) {
+    detail::add_options_from_env(_options);
+
+    // Bootstrap the cuda context to avoid errors
+    cudaFree(0);
+  }
+};
+
+class Program_impl {
+  // A friendly class
+  friend class Kernel_impl;
+  friend class KernelLauncher_impl;
+  friend class KernelInstantiation_impl;
+  // TODO: This can become invalid if JitCache is destroyed before the
+  //         Program object is. However, this can't happen if JitCache
+  //           instances are static.
+  JitCache_impl& _cache;
+  uint64_t _hash;
+  ProgramConfig* _config;
+  void load_sources(std::string source, std::vector<std::string> headers,
+                    std::vector<std::string> options,
+                    file_callback_type file_callback);
+
+ public:
+  inline Program_impl(JitCache_impl& cache, std::string source,
+                      jitify::detail::vector<std::string> headers = 0,
+                      jitify::detail::vector<std::string> options = 0,
+                      file_callback_type file_callback = 0);
+  inline Program_impl(Program_impl const&) = default;
+  inline Program_impl(Program_impl&&) = default;
+  inline std::vector<std::string> const& options() const {
+    return _config->options;
+  }
+  inline std::string const& name() const { return _config->name; }
+  inline ProgramConfig::source_map const& sources() const {
+    return _config->sources;
+  }
+  inline std::vector<std::string> const& include_paths() const {
+    return _config->include_paths;
+  }
+};
+
+class Kernel_impl {
+  friend class KernelLauncher_impl;
+  friend class KernelInstantiation_impl;
+  Program_impl _program;
+  std::string _name;
+  std::vector<std::string> _options;
+  uint64_t _hash;
+
+ public:
+  inline Kernel_impl(Program_impl const& program, std::string name,
+                     jitify::detail::vector<std::string> options = 0);
+  inline Kernel_impl(Kernel_impl const&) = default;
+  inline Kernel_impl(Kernel_impl&&) = default;
+};
+
+class KernelInstantiation_impl {
+  friend class KernelLauncher_impl;
+  Kernel_impl _kernel;
+  uint64_t _hash;
+  std::string _template_inst;
+  std::vector<std::string> _options;
+  detail::CUDAKernel* _cuda_kernel;
+  inline void print() const;
+  void build_kernel();
+
+ public:
+  inline KernelInstantiation_impl(
+      Kernel_impl const& kernel, std::vector<std::string> const& template_args);
+  inline KernelInstantiation_impl(KernelInstantiation_impl const&) = default;
+  inline KernelInstantiation_impl(KernelInstantiation_impl&&) = default;
+  detail::CUDAKernel const& cuda_kernel() const { return *_cuda_kernel; }
+};
+
+class KernelLauncher_impl {
+  KernelInstantiation_impl _kernel_inst;
+  dim3 _grid;
+  dim3 _block;
+  unsigned int _smem;
+  cudaStream_t _stream;
+
+ public:
+  inline KernelLauncher_impl(KernelInstantiation_impl const& kernel_inst,
+                             dim3 grid, dim3 block, unsigned int smem = 0,
+                             cudaStream_t stream = 0)
+      : _kernel_inst(kernel_inst),
+        _grid(grid),
+        _block(block),
+        _smem(smem),
+        _stream(stream) {}
+  inline KernelLauncher_impl(KernelLauncher_impl const&) = default;
+  inline KernelLauncher_impl(KernelLauncher_impl&&) = default;
+  inline CUresult launch(
+      jitify::detail::vector<void*> arg_ptrs,
+      jitify::detail::vector<std::string> arg_types = 0) const;
+  inline void safe_launch(
+      jitify::detail::vector<void*> arg_ptrs,
+      jitify::detail::vector<std::string> arg_types = 0) const;
+
+ private:
+  inline void pre_launch(
+      jitify::detail::vector<std::string> arg_types = 0) const;
+};
+
+/*! An object representing a configured and instantiated kernel ready
+ *    for launching.
+ */
+class KernelLauncher {
+  std::unique_ptr<KernelLauncher_impl const> _impl;
+
+ public:
+  inline KernelLauncher(KernelInstantiation const& kernel_inst, dim3 grid,
+                        dim3 block, unsigned int smem = 0,
+                        cudaStream_t stream = 0);
+
+  // Note: It's important that there is no implicit conversion required
+  //         for arg_ptrs, because otherwise the parameter pack version
+  //         below gets called instead (probably resulting in a segfault).
+  /*! Launch the kernel.
+   *
+   *  \param arg_ptrs  A vector of pointers to each function argument for the
+   *    kernel.
+   *  \param arg_types A vector of function argument types represented
+   *    as code-strings. This parameter is optional and is only used to print
+   *    out the function signature.
+   */
+  inline CUresult launch(
+      std::vector<void*> arg_ptrs = std::vector<void*>(),
+      jitify::detail::vector<std::string> arg_types = 0) const {
+    return _impl->launch(arg_ptrs, arg_types);
+  }
+
+  /*! Launch the kernel and check for cuda errors.
+   *
+   *  \see launch
+   */
+  inline void safe_launch(
+      std::vector<void*> arg_ptrs = std::vector<void*>(),
+      jitify::detail::vector<std::string> arg_types = 0) const {
+    _impl->safe_launch(arg_ptrs, arg_types);
+  }
+
+  // Regular function call syntax
+  /*! Launch the kernel.
+   *
+   *  \see launch
+   */
+  template <typename... ArgTypes>
+  inline CUresult operator()(const ArgTypes&... args) const {
+    return this->launch(args...);
+  }
+  /*! Launch the kernel.
+   *
+   *  \param args Function arguments for the kernel.
+   */
+  template <typename... ArgTypes>
+  inline CUresult launch(const ArgTypes&... args) const {
+    return this->launch(std::vector<void*>({(void*)&args...}),
+                        {reflection::reflect<ArgTypes>()...});
+  }
+  /*! Launch the kernel and check for cuda errors.
+   *
+   *  \param args Function arguments for the kernel.
+   */
+  template <typename... ArgTypes>
+  inline void safe_launch(const ArgTypes&... args) const {
+    this->safe_launch(std::vector<void*>({(void*)&args...}),
+                      {reflection::reflect<ArgTypes>()...});
+  }
+};
+
+/*! An object representing a kernel instantiation made up of a Kernel and
+ *    template arguments.
+ */
+class KernelInstantiation {
+  friend class KernelLauncher;
+  std::unique_ptr<KernelInstantiation_impl const> _impl;
+
+ public:
+  inline KernelInstantiation(Kernel const& kernel,
+                             std::vector<std::string> const& template_args);
+
+  /*! Implicit conversion to the underlying CUfunction object.
+   *
+   * \note This allows use of CUDA APIs like
+   *   cuOccupancyMaxActiveBlocksPerMultiprocessor.
+   */
+  inline operator CUfunction() const { return _impl->cuda_kernel(); }
+
+  /*! Configure the kernel launch.
+   *
+   *  \see configure
+   */
+  inline KernelLauncher operator()(dim3 grid, dim3 block, unsigned int smem = 0,
+                                   cudaStream_t stream = 0) const {
+    return this->configure(grid, block, smem, stream);
+  }
+  /*! Configure the kernel launch.
+   *
+   *  \param grid   The thread grid dimensions for the launch.
+   *  \param block  The thread block dimensions for the launch.
+   *  \param smem   The amount of shared memory to dynamically allocate, in
+   * bytes.
+   *  \param stream The CUDA stream to launch the kernel in.
+   */
+  inline KernelLauncher configure(dim3 grid, dim3 block, unsigned int smem = 0,
+                                  cudaStream_t stream = 0) const {
+    return KernelLauncher(*this, grid, block, smem, stream);
+  }
+  /*! Configure the kernel launch with a 1-dimensional block and grid chosen
+   *  automatically to maximise occupancy.
+   *
+   * \param max_block_size  The upper limit on the block size, or 0 for no
+   * limit.
+   * \param smem  The amount of shared memory to dynamically allocate, in bytes.
+   * \param smem_callback  A function returning smem for a given block size (overrides \p smem).
+   * \param stream The CUDA stream to launch the kernel in.
+   * \param flags The flags to pass to cuOccupancyMaxPotentialBlockSizeWithFlags.
+   */
+  inline KernelLauncher configure_1d_max_occupancy(
+      int max_block_size = 0, unsigned int smem = 0,
+      CUoccupancyB2DSize smem_callback = 0, cudaStream_t stream = 0,
+      unsigned int flags = 0) const {
+    int grid;
+    int block;
+    CUfunction func = _impl->cuda_kernel();
+    detail::get_1d_max_occupancy(func, smem_callback, &smem, max_block_size,
+                                 flags, &grid, &block);
+    return this->configure(grid, block, smem, stream);
+  }
+
+  /*
+   * Returns the function attribute requested from the kernel
+   */
+  inline int get_func_attribute(CUfunction_attribute attribute) const {
+    return _impl->cuda_kernel().get_func_attribute(attribute);
+  }
+
+  /*
+   * Set the function attribute requested for the kernel
+   */
+  inline void set_func_attribute(CUfunction_attribute attribute,
+                                 int value) const {
+    _impl->cuda_kernel().set_func_attribute(attribute, value);
+  }
+
+  /*
+   * \deprecated Use \p get_global_ptr instead.
+   */
+  inline CUdeviceptr get_constant_ptr(const char* name,
+                                      size_t* size = nullptr) const {
+    return get_global_ptr(name, size);
+  }
+
+  /*
+   * Get a device pointer to a global __constant__ or __device__ variable using
+   * its un-mangled name. If provided, *size is set to the size of the variable
+   * in bytes.
+   */
+  inline CUdeviceptr get_global_ptr(const char* name,
+                                    size_t* size = nullptr) const {
+    return _impl->cuda_kernel().get_global_ptr(name, size);
+  }
+
+  /*
+   * Copy data from a global __constant__ or __device__ array to the host using
+   * its un-mangled name.
+   */
+  template <typename T>
+  inline CUresult get_global_array(const char* name, T* data, size_t count,
+                                   CUstream stream = 0) const {
+    return _impl->cuda_kernel().get_global_data(name, data, count, stream);
+  }
+
+  /*
+   * Copy a value from a global __constant__ or __device__ variable to the host
+   * using its un-mangled name.
+   */
+  template <typename T>
+  inline CUresult get_global_value(const char* name, T* value,
+                                   CUstream stream = 0) const {
+    return get_global_array(name, value, 1, stream);
+  }
+
+  /*
+   * Copy data from the host to a global __constant__ or __device__ array using
+   * its un-mangled name.
+   */
+  template <typename T>
+  inline CUresult set_global_array(const char* name, const T* data,
+                                   size_t count, CUstream stream = 0) const {
+    return _impl->cuda_kernel().set_global_data(name, data, count, stream);
+  }
+
+  /*
+   * Copy a value from the host to a global __constant__ or __device__ variable
+   * using its un-mangled name.
+   */
+  template <typename T>
+  inline CUresult set_global_value(const char* name, const T& value,
+                                   CUstream stream = 0) const {
+    return set_global_array(name, &value, 1, stream);
+  }
+
+  const std::string& mangled_name() const {
+    return _impl->cuda_kernel().function_name();
+  }
+
+  const std::string& ptx() const { return _impl->cuda_kernel().ptx(); }
+
+  const std::vector<std::string>& link_files() const {
+    return _impl->cuda_kernel().link_files();
+  }
+
+  const std::vector<std::string>& link_paths() const {
+    return _impl->cuda_kernel().link_paths();
+  }
+};
+
+/*! An object representing a kernel made up of a Program, a name and options.
+ */
+class Kernel {
+  friend class KernelInstantiation;
+  std::unique_ptr<Kernel_impl const> _impl;
+
+ public:
+  Kernel(Program const& program, std::string name,
+         jitify::detail::vector<std::string> options = 0);
+
+  /*! Instantiate the kernel.
+   *
+   *  \param template_args A vector of template arguments represented as
+   *    code-strings. These can be generated using
+   *    \code{.cpp}jitify::reflection::reflect<type>()\endcode or
+   *    \code{.cpp}jitify::reflection::reflect(value)\endcode
+   *
+   *  \note Template type deduction is not possible, so all types must be
+   *    explicitly specified.
+   */
+  // inline KernelInstantiation instantiate(std::vector<std::string> const&
+  // template_args) const {
+  inline KernelInstantiation instantiate(
+      std::vector<std::string> const& template_args =
+          std::vector<std::string>()) const {
+    return KernelInstantiation(*this, template_args);
+  }
+
+  // Regular template instantiation syntax (note limited flexibility)
+  /*! Instantiate the kernel.
+   *
+   *  \note The template arguments specified on this function are
+   *    used to instantiate the kernel. Non-type template arguments must
+   *    be wrapped with
+   *    \code{.cpp}jitify::reflection::NonType<type,value>\endcode
+   *
+   *  \note Template type deduction is not possible, so all types must be
+   *    explicitly specified.
+   */
+  template <typename... TemplateArgs>
+  inline KernelInstantiation instantiate() const {
+    return this->instantiate(
+        std::vector<std::string>({reflection::reflect<TemplateArgs>()...}));
+  }
+  // Template-like instantiation syntax
+  //   E.g., instantiate(myvar,Type<MyType>())(grid,block)
+  /*! Instantiate the kernel.
+   *
+   *  \param targs The template arguments for the kernel, represented as
+   *    values. Types must be wrapped with
+   *    \code{.cpp}jitify::reflection::Type<type>()\endcode or
+   *    \code{.cpp}jitify::reflection::type_of(value)\endcode
+   *
+   *  \note Template type deduction is not possible, so all types must be
+   *    explicitly specified.
+   */
+  template <typename... TemplateArgs>
+  inline KernelInstantiation instantiate(TemplateArgs... targs) const {
+    return this->instantiate(
+        std::vector<std::string>({reflection::reflect(targs)...}));
+  }
+};
+
+/*! An object representing a program made up of source code, headers
+ *    and options.
+ */
+class Program {
+  friend class Kernel;
+  std::unique_ptr<Program_impl const> _impl;
+
+ public:
+  Program(JitCache& cache, std::string source,
+          jitify::detail::vector<std::string> headers = 0,
+          jitify::detail::vector<std::string> options = 0,
+          file_callback_type file_callback = 0);
+
+  /*! Select a kernel.
+   *
+   * \param name The name of the kernel (unmangled and without
+   * template arguments).
+   * \param options A vector of options to be passed to the NVRTC
+   * compiler when compiling this kernel.
+   */
+  inline Kernel kernel(std::string name,
+                       jitify::detail::vector<std::string> options = 0) const {
+    return Kernel(*this, name, options);
+  }
+  /*! Select a kernel.
+   *
+   *  \see kernel
+   */
+  inline Kernel operator()(
+      std::string name, jitify::detail::vector<std::string> options = 0) const {
+    return this->kernel(name, options);
+  }
+};
+
+/*! An object that manages a cache of JIT-compiled CUDA kernels.
+ *
+ */
+class JitCache {
+  friend class Program;
+  std::unique_ptr<JitCache_impl> _impl;
+
+ public:
+  /*! JitCache constructor.
+   *  \param cache_size The number of kernels to hold in the cache
+   *    before overwriting the least-recently-used ones.
+   */
+  enum { DEFAULT_CACHE_SIZE = 128 };
+  JitCache(size_t cache_size = DEFAULT_CACHE_SIZE)
+      : _impl(new JitCache_impl(cache_size)) {}
+
+  /*! Create a program.
+   *
+   *  \param source A string containing either the source filename or
+   *    the source itself; in the latter case, the first line must be
+   *    the name of the program.
+   *  \param headers A vector of strings representing the source of
+   *    each header file required by the program. Each entry can be
+   *    either the header filename or the header source itself; in
+   *    the latter case, the first line must be the name of the header
+   *    (i.e., the name by which the header is #included).
+   *  \param options A vector of options to be passed to the
+   *    NVRTC compiler. Include paths specified with \p -I
+   *    are added to the search paths used by Jitify. The environment
+   *    variable JITIFY_OPTIONS can also be used to define additional
+   *    options.
+   *  \param file_callback A pointer to a callback function that is
+   *    invoked whenever a source file needs to be loaded. Inside this
+   *    function, the user can either load/specify the source themselves
+   *    or defer to Jitify's file-loading mechanisms.
+   *  \note Program or header source files referenced by filename are
+   *  looked-up using the following mechanisms (in this order):
+   *  \note 1) By calling file_callback.
+   *  \note 2) By looking for the file embedded in the executable via the GCC
+   * linker.
+   *  \note 3) By looking for the file in the filesystem.
+   *
+   *  \note Jitify recursively scans all source files for \p #include
+   *  directives and automatically adds them to the set of headers needed
+   *  by the program.
+   *  If a \p #include directive references a header that cannot be found,
+   *  the directive is automatically removed from the source code to prevent
+   *  immediate compilation failure. This may result in compilation errors
+   *  if the header was required by the program.
+   *
+   *  \note Jitify automatically includes NVRTC-safe versions of some
+   *  standard library headers.
+   */
+  inline Program program(std::string source,
+                         jitify::detail::vector<std::string> headers = 0,
+                         jitify::detail::vector<std::string> options = 0,
+                         file_callback_type file_callback = 0) {
+    return Program(*this, source, headers, options, file_callback);
+  }
+};
+
+inline Program::Program(JitCache& cache, std::string source,
+                        jitify::detail::vector<std::string> headers,
+                        jitify::detail::vector<std::string> options,
+                        file_callback_type file_callback)
+    : _impl(new Program_impl(*cache._impl, source, headers, options,
+                             file_callback)) {}
+
+inline Kernel::Kernel(Program const& program, std::string name,
+                      jitify::detail::vector<std::string> options)
+    : _impl(new Kernel_impl(*program._impl, name, options)) {}
+
+inline KernelInstantiation::KernelInstantiation(
+    Kernel const& kernel, std::vector<std::string> const& template_args)
+    : _impl(new KernelInstantiation_impl(*kernel._impl, template_args)) {}
+
+inline KernelLauncher::KernelLauncher(KernelInstantiation const& kernel_inst,
+                                      dim3 grid, dim3 block, unsigned int smem,
+                                      cudaStream_t stream)
+    : _impl(new KernelLauncher_impl(*kernel_inst._impl, grid, block, smem,
+                                    stream)) {}
+
+inline std::ostream& operator<<(std::ostream& stream, dim3 d) {
+  if (d.y == 1 && d.z == 1) {
+    stream << d.x;
+  } else {
+    stream << "(" << d.x << "," << d.y << "," << d.z << ")";
+  }
+  return stream;
+}
+
+inline void KernelLauncher_impl::pre_launch(
+    jitify::detail::vector<std::string> arg_types) const {
+  (void)arg_types;
+#if JITIFY_PRINT_LAUNCH
+  Kernel_impl const& kernel = _kernel_inst._kernel;
+  std::string arg_types_string =
+      (arg_types.empty() ? "..." : reflection::reflect_list(arg_types));
+  std::cout << "Launching " << kernel._name << _kernel_inst._template_inst
+            << "<<<" << _grid << "," << _block << "," << _smem << "," << _stream
+            << ">>>"
+            << "(" << arg_types_string << ")" << std::endl;
+#endif
+  if (!_kernel_inst._cuda_kernel) {
+    throw std::runtime_error(
+        "Kernel pointer is NULL; you may need to define JITIFY_THREAD_SAFE 1");
+  }
+}
+
+inline CUresult KernelLauncher_impl::launch(
+    jitify::detail::vector<void*> arg_ptrs,
+    jitify::detail::vector<std::string> arg_types) const {
+  pre_launch(arg_types);
+  return _kernel_inst._cuda_kernel->launch(_grid, _block, _smem, _stream,
+                                           arg_ptrs);
+}
+
+inline void KernelLauncher_impl::safe_launch(
+    jitify::detail::vector<void*> arg_ptrs,
+    jitify::detail::vector<std::string> arg_types) const {
+  pre_launch(arg_types);
+  _kernel_inst._cuda_kernel->safe_launch(_grid, _block, _smem, _stream,
+                                         arg_ptrs);
+}
+
+inline KernelInstantiation_impl::KernelInstantiation_impl(
+    Kernel_impl const& kernel, std::vector<std::string> const& template_args)
+    : _kernel(kernel), _options(kernel._options) {
+  _template_inst =
+      (template_args.empty() ? ""
+                             : reflection::reflect_template(template_args));
+  using detail::hash_combine;
+  using detail::hash_larson64;
+  _hash = _kernel._hash;
+  _hash = hash_combine(_hash, hash_larson64(_template_inst.c_str()));
+  JitCache_impl& cache = _kernel._program._cache;
+  uint64_t cache_key = _hash;
+#if JITIFY_THREAD_SAFE
+  std::lock_guard<std::mutex> lock(cache._kernel_cache_mutex);
+#endif
+  if (cache._kernel_cache.contains(cache_key)) {
+#if JITIFY_PRINT_INSTANTIATION
+    std::cout << "Found ";
+    this->print();
+#endif
+    _cuda_kernel = &cache._kernel_cache.get(cache_key);
+  } else {
+#if JITIFY_PRINT_INSTANTIATION
+    std::cout << "Building ";
+    this->print();
+#endif
+    _cuda_kernel = &cache._kernel_cache.emplace(cache_key);
+    this->build_kernel();
+  }
+}
+
+inline void KernelInstantiation_impl::print() const {
+  std::string options_string = reflection::reflect_list(_options);
+  std::cout << _kernel._name << _template_inst << " [" << options_string << "]"
+            << std::endl;
+}
+
+inline void KernelInstantiation_impl::build_kernel() {
+  Program_impl const& program = _kernel._program;
+
+  std::string instantiation = _kernel._name + _template_inst;
+
+  std::string log, ptx, mangled_instantiation;
+  std::vector<std::string> linker_files, linker_paths;
+  detail::instantiate_kernel(program.name(), program.sources(), instantiation,
+                             _options, &log, &ptx, &mangled_instantiation,
+                             &linker_files, &linker_paths);
+
+  _cuda_kernel->set(mangled_instantiation.c_str(), ptx.c_str(), linker_files,
+                    linker_paths);
+}
+
+Kernel_impl::Kernel_impl(Program_impl const& program, std::string name,
+                         jitify::detail::vector<std::string> options)
+    : _program(program), _name(name), _options(options) {
+  // Merge options from parent
+  _options.insert(_options.end(), _program.options().begin(),
+                  _program.options().end());
+  detail::detect_and_add_cuda_arch(_options);
+  detail::detect_and_add_cxx11_flag(_options);
+  std::string options_string = reflection::reflect_list(_options);
+  using detail::hash_combine;
+  using detail::hash_larson64;
+  _hash = _program._hash;
+  _hash = hash_combine(_hash, hash_larson64(_name.c_str()));
+  _hash = hash_combine(_hash, hash_larson64(options_string.c_str()));
+}
+
+Program_impl::Program_impl(JitCache_impl& cache, std::string source,
+                           jitify::detail::vector<std::string> headers,
+                           jitify::detail::vector<std::string> options,
+                           file_callback_type file_callback)
+    : _cache(cache) {
+  // Compute hash of source, headers and options
+  std::string options_string = reflection::reflect_list(options);
+  using detail::hash_combine;
+  using detail::hash_larson64;
+  _hash = hash_combine(hash_larson64(source.c_str()),
+                       hash_larson64(options_string.c_str()));
+  for (size_t i = 0; i < headers.size(); ++i) {
+    _hash = hash_combine(_hash, hash_larson64(headers[i].c_str()));
+  }
+  _hash = hash_combine(_hash, (uint64_t)file_callback);
+  // Add pre-include built-in JIT-safe headers
+  for (int i = 0; i < detail::preinclude_jitsafe_headers_count; ++i) {
+    const char* hdr_name = detail::preinclude_jitsafe_header_names[i];
+    const std::string& hdr_source =
+        detail::get_jitsafe_headers_map().at(hdr_name);
+    headers.push_back(std::string(hdr_name) + "\n" + hdr_source);
+  }
+  // Merge options from parent
+  options.insert(options.end(), _cache._options.begin(), _cache._options.end());
+  // Load sources
+#if JITIFY_THREAD_SAFE
+  std::lock_guard<std::mutex> lock(cache._program_cache_mutex);
+#endif
+  if (!cache._program_config_cache.contains(_hash)) {
+    _config = &cache._program_config_cache.insert(_hash);
+    this->load_sources(source, headers, options, file_callback);
+  } else {
+    _config = &cache._program_config_cache.get(_hash);
+  }
+}
+
+inline void Program_impl::load_sources(std::string source,
+                                       std::vector<std::string> headers,
+                                       std::vector<std::string> options,
+                                       file_callback_type file_callback) {
+  _config->options = options;
+  detail::load_program(source, headers, file_callback, &_config->include_paths,
+                       &_config->sources, &_config->options, &_config->name);
+}
+
+enum Location { HOST, DEVICE };
+
+/*! Specifies location and parameters for execution of an algorithm.
+ *  \param stream        The CUDA stream on which to execute.
+ *  \param headers       A vector of headers to include in the code.
+ *  \param options       Options to pass to the NVRTC compiler.
+ *  \param file_callback See jitify::Program.
+ *  \param block_size    The size of the CUDA thread block with which to
+ * execute.
+ *  \param cache_size    The number of kernels to store in the cache
+ * before overwriting the least-recently-used ones.
+ */
+struct ExecutionPolicy {
+  /*! Location (HOST or DEVICE) on which to execute.*/
+  Location location;
+  /*! List of headers to include when compiling the algorithm.*/
+  std::vector<std::string> headers;
+  /*! List of compiler options.*/
+  std::vector<std::string> options;
+  /*! Optional callback for loading source files.*/
+  file_callback_type file_callback;
+  /*! CUDA stream on which to execute.*/
+  cudaStream_t stream;
+  /*! CUDA device on which to execute.*/
+  int device;
+  /*! CUDA block size with which to execute.*/
+  int block_size;
+  /*! The number of instantiations to store in the cache before overwriting
+   *  the least-recently-used ones.*/
+  size_t cache_size;
+  ExecutionPolicy(Location location_ = DEVICE,
+                  jitify::detail::vector<std::string> headers_ = 0,
+                  jitify::detail::vector<std::string> options_ = 0,
+                  file_callback_type file_callback_ = 0,
+                  cudaStream_t stream_ = 0, int device_ = 0,
+                  int block_size_ = 256,
+                  size_t cache_size_ = JitCache::DEFAULT_CACHE_SIZE)
+      : location(location_),
+        headers(headers_),
+        options(options_),
+        file_callback(file_callback_),
+        stream(stream_),
+        device(device_),
+        block_size(block_size_),
+        cache_size(cache_size_) {}
+};
+
+template <class Func>
+class Lambda;
+
+/*! An object that captures a set of variables for use in a parallel_for
+ *    expression. See JITIFY_CAPTURE().
+ */
+class Capture {
+ public:
+  std::vector<std::string> _arg_decls;
+  std::vector<void*> _arg_ptrs;
+
+ public:
+  template <typename... Args>
+  inline Capture(std::vector<std::string> arg_names, Args const&... args)
+      : _arg_ptrs{(void*)&args...} {
+    std::vector<std::string> arg_types = {reflection::reflect<Args>()...};
+    _arg_decls.resize(arg_names.size());
+    for (int i = 0; i < (int)arg_names.size(); ++i) {
+      _arg_decls[i] = arg_types[i] + " " + arg_names[i];
+    }
+  }
+};
+
+/*! An object that captures the instantiated Lambda function for use
+    in a parallel_for expression and the function string for NVRTC
+    compilation
+ */
+template <class Func>
+class Lambda {
+ public:
+  Capture _capture;
+  std::string _func_string;
+  Func _func;
+
+ public:
+  inline Lambda(Capture const& capture, std::string func_string, Func func)
+      : _capture(capture), _func_string(func_string), _func(func) {}
+};
+
+template <typename T>
+inline Lambda<T> make_Lambda(Capture const& capture, std::string func,
+                             T lambda) {
+  return Lambda<T>(capture, func, lambda);
+}
+
+#define JITIFY_CAPTURE(...)                                            \
+  jitify::Capture(jitify::detail::split_string(#__VA_ARGS__, -1, ","), \
+                  __VA_ARGS__)
+
+#define JITIFY_MAKE_LAMBDA(capture, x, ...)               \
+  jitify::make_Lambda(capture, std::string(#__VA_ARGS__), \
+                      [x](int i) { __VA_ARGS__; })
+
+#define JITIFY_ARGS(...) __VA_ARGS__
+
+#define JITIFY_LAMBDA_(x, ...) \
+  JITIFY_MAKE_LAMBDA(JITIFY_CAPTURE(x), JITIFY_ARGS(x), __VA_ARGS__)
+
+// macro sequence to strip surrounding brackets
+#define JITIFY_STRIP_PARENS(X) X
+#define JITIFY_PASS_PARAMETERS(X) JITIFY_STRIP_PARENS(JITIFY_ARGS X)
+
+/*! Creates a Lambda object with captured variables and a function
+ *    definition.
+ *  \param capture A bracket-enclosed list of variables to capture.
+ *  \param ...     The function definition.
+ *
+ *  \code{.cpp}
+ *  float* capture_me;
+ *  int    capture_me_too;
+ *  auto my_lambda = JITIFY_LAMBDA( (capture_me, capture_me_too),
+ *                                  capture_me[i] = i*capture_me_too );
+ *  \endcode
+ */
+#define JITIFY_LAMBDA(capture, ...)                            \
+  JITIFY_LAMBDA_(JITIFY_ARGS(JITIFY_PASS_PARAMETERS(capture)), \
+                 JITIFY_ARGS(__VA_ARGS__))
+
+// TODO: Try to implement for_each that accepts iterators instead of indices
+//       Add compile guard for NOCUDA compilation
+/*! Call a function for a range of indices
+ *
+ *  \param policy Determines the location and device parameters for
+ *  execution of the parallel_for.
+ *  \param begin  The starting index.
+ *  \param end    The ending index.
+ *  \param lambda A Lambda object created using the JITIFY_LAMBDA() macro.
+ *
+ *  \code{.cpp}
+ *  char const* in;
+ *  float*      out;
+ *  parallel_for(0, 100, JITIFY_LAMBDA( (in, out), {char x = in[i]; out[i] =
+ * x*x; } ); \endcode
+ */
+template <typename IndexType, class Func>
+CUresult parallel_for(ExecutionPolicy policy, IndexType begin, IndexType end,
+                      Lambda<Func> const& lambda) {
+  using namespace jitify;
+
+  if (policy.location == HOST) {
+#ifdef _OPENMP
+#pragma omp parallel for
+#endif
+    for (IndexType i = begin; i < end; i++) {
+      lambda._func(i);
+    }
+    return CUDA_SUCCESS;  // FIXME - replace with non-CUDA enum type?
+  }
+
+  thread_local static JitCache kernel_cache(policy.cache_size);
+
+  std::vector<std::string> arg_decls;
+  arg_decls.push_back("I begin, I end");
+  arg_decls.insert(arg_decls.end(), lambda._capture._arg_decls.begin(),
+                   lambda._capture._arg_decls.end());
+
+  std::stringstream source_ss;
+  source_ss << "parallel_for_program\n";
+  for (auto const& header : policy.headers) {
+    std::string header_name = header.substr(0, header.find("\n"));
+    source_ss << "#include <" << header_name << ">\n";
+  }
+  source_ss << "template<typename I>\n"
+               "__global__\n"
+               "void parallel_for_kernel("
+            << reflection::reflect_list(arg_decls)
+            << ") {\n"
+               "	I i0 = threadIdx.x + blockDim.x*blockIdx.x;\n"
+               "	for( I i=i0+begin; i<end; i+=blockDim.x*gridDim.x ) {\n"
+               "	"
+            << "\t" << lambda._func_string << ";\n"
+            << "	}\n"
+               "}\n";
+
+  Program program = kernel_cache.program(source_ss.str(), policy.headers,
+                                         policy.options, policy.file_callback);
+
+  std::vector<void*> arg_ptrs;
+  arg_ptrs.push_back(&begin);
+  arg_ptrs.push_back(&end);
+  arg_ptrs.insert(arg_ptrs.end(), lambda._capture._arg_ptrs.begin(),
+                  lambda._capture._arg_ptrs.end());
+
+  size_t n = end - begin;
+  dim3 block(policy.block_size);
+  dim3 grid((unsigned int)std::min((n - 1) / block.x + 1, size_t(65535)));
+  cudaSetDevice(policy.device);
+  return program.kernel("parallel_for_kernel")
+      .instantiate<IndexType>()
+      .configure(grid, block, 0, policy.stream)
+      .launch(arg_ptrs);
+}
+
+namespace experimental {
+
+using jitify::file_callback_type;
+
+namespace serialization {
+
+namespace detail {
+
+// This should be incremented whenever the serialization format changes in any
+// incompatible way.
+static constexpr const size_t kSerializationVersion = 2;
+
+inline void serialize(std::ostream& stream, size_t u) {
+  uint64_t u64 = u;
+  char bytes[8];
+  for (int i = 0; i < (int)sizeof(bytes); ++i) {
+    // Convert to little-endian bytes.
+    bytes[i] = (unsigned char)(u64 >> (i * CHAR_BIT));
+  }
+  stream.write(bytes, sizeof(bytes));
+}
+
+inline bool deserialize(std::istream& stream, size_t* size) {
+  char bytes[8];
+  stream.read(bytes, sizeof(bytes));
+  uint64_t u64 = 0;
+  for (int i = 0; i < (int)sizeof(bytes); ++i) {
+    // Convert from little-endian bytes.
+    u64 |= uint64_t((unsigned char)(bytes[i])) << (i * CHAR_BIT);
+  }
+  *size = u64;
+  return stream.good();
+}
+
+inline void serialize(std::ostream& stream, std::string const& s) {
+  serialize(stream, s.size());
+  stream.write(s.data(), s.size());
+}
+
+inline bool deserialize(std::istream& stream, std::string* s) {
+  size_t size;
+  if (!deserialize(stream, &size)) return false;
+  s->resize(size);
+  if (s->size()) {
+    stream.read(&(*s)[0], s->size());
+  }
+  return stream.good();
+}
+
+inline void serialize(std::ostream& stream, std::vector<std::string> const& v) {
+  serialize(stream, v.size());
+  for (auto const& s : v) {
+    serialize(stream, s);
+  }
+}
+
+inline bool deserialize(std::istream& stream, std::vector<std::string>* v) {
+  size_t size;
+  if (!deserialize(stream, &size)) return false;
+  v->resize(size);
+  for (auto& s : *v) {
+    if (!deserialize(stream, &s)) return false;
+  }
+  return true;
+}
+
+inline void serialize(std::ostream& stream,
+                      std::map<std::string, std::string> const& m) {
+  serialize(stream, m.size());
+  for (auto const& kv : m) {
+    serialize(stream, kv.first);
+    serialize(stream, kv.second);
+  }
+}
+
+inline bool deserialize(std::istream& stream,
+                        std::map<std::string, std::string>* m) {
+  size_t size;
+  if (!deserialize(stream, &size)) return false;
+  for (size_t i = 0; i < size; ++i) {
+    std::string key;
+    if (!deserialize(stream, &key)) return false;
+    if (!deserialize(stream, &(*m)[key])) return false;
+  }
+  return true;
+}
+
+template <typename T, typename... Rest>
+inline void serialize(std::ostream& stream, T const& value, Rest... rest) {
+  serialize(stream, value);
+  serialize(stream, rest...);
+}
+
+template <typename T, typename... Rest>
+inline bool deserialize(std::istream& stream, T* value, Rest... rest) {
+  if (!deserialize(stream, value)) return false;
+  return deserialize(stream, rest...);
+}
+
+inline void serialize_magic_number(std::ostream& stream) {
+  stream.write("JTFY", 4);
+  serialize(stream, kSerializationVersion);
+}
+
+inline bool deserialize_magic_number(std::istream& stream) {
+  char magic_number[4] = {0, 0, 0, 0};
+  stream.read(&magic_number[0], 4);
+  if (!(magic_number[0] == 'J' && magic_number[1] == 'T' &&
+        magic_number[2] == 'F' && magic_number[3] == 'Y')) {
+    return false;
+  }
+  size_t serialization_version;
+  if (!deserialize(stream, &serialization_version)) return false;
+  return serialization_version == kSerializationVersion;
+}
+
+}  // namespace detail
+
+template <typename... Values>
+inline std::string serialize(Values const&... values) {
+  std::ostringstream ss(std::stringstream::out | std::stringstream::binary);
+  detail::serialize_magic_number(ss);
+  detail::serialize(ss, values...);
+  return ss.str();
+}
+
+template <typename... Values>
+inline bool deserialize(std::string const& serialized, Values*... values) {
+  std::istringstream ss(serialized,
+                        std::stringstream::in | std::stringstream::binary);
+  if (!detail::deserialize_magic_number(ss)) return false;
+  return detail::deserialize(ss, values...);
+}
+
+}  // namespace serialization
+
+class Program;
+class Kernel;
+class KernelInstantiation;
+class KernelLauncher;
+
+/*! An object representing a program made up of source code, headers
+ *    and options.
+ */
+class Program {
+ private:
+  friend class KernelInstantiation;
+  std::string _name;
+  std::vector<std::string> _options;
+  std::map<std::string, std::string> _sources;
+
+  // Private constructor used by deserialize()
+  Program() {}
+
+ public:
+  /*! Create a program.
+   *
+   *  \param source A string containing either the source filename or
+   *    the source itself; in the latter case, the first line must be
+   *    the name of the program.
+   *  \param headers A vector of strings representing the source of
+   *    each header file required by the program. Each entry can be
+   *    either the header filename or the header source itself; in
+   *    the latter case, the first line must be the name of the header
+   *    (i.e., the name by which the header is #included).
+   *  \param options A vector of options to be passed to the
+   *    NVRTC compiler. Include paths specified with \p -I
+   *    are added to the search paths used by Jitify. The environment
+   *    variable JITIFY_OPTIONS can also be used to define additional
+   *    options.
+   *  \param file_callback A pointer to a callback function that is
+   *    invoked whenever a source file needs to be loaded. Inside this
+   *    function, the user can either load/specify the source themselves
+   *    or defer to Jitify's file-loading mechanisms.
+   *  \note Program or header source files referenced by filename are
+   *  looked-up using the following mechanisms (in this order):
+   *  \note 1) By calling file_callback.
+   *  \note 2) By looking for the file embedded in the executable via the GCC
+   * linker.
+   *  \note 3) By looking for the file in the filesystem.
+   *
+   *  \note Jitify recursively scans all source files for \p #include
+   *  directives and automatically adds them to the set of headers needed
+   *  by the program.
+   *  If a \p #include directive references a header that cannot be found,
+   *  the directive is automatically removed from the source code to prevent
+   *  immediate compilation failure. This may result in compilation errors
+   *  if the header was required by the program.
+   *
+   *  \note Jitify automatically includes NVRTC-safe versions of some
+   *  standard library headers.
+   */
+  Program(std::string const& cuda_source,
+          std::vector<std::string> const& given_headers = {},
+          std::vector<std::string> const& given_options = {},
+          file_callback_type file_callback = nullptr) {
+    // Add pre-include built-in JIT-safe headers
+    std::vector<std::string> headers = given_headers;
+    for (int i = 0; i < detail::preinclude_jitsafe_headers_count; ++i) {
+      const char* hdr_name = detail::preinclude_jitsafe_header_names[i];
+      const std::string& hdr_source =
+          detail::get_jitsafe_headers_map().at(hdr_name);
+      headers.push_back(std::string(hdr_name) + "\n" + hdr_source);
+    }
+
+    _options = given_options;
+    detail::add_options_from_env(_options);
+    std::vector<std::string> include_paths;
+    detail::load_program(cuda_source, headers, file_callback, &include_paths,
+                         &_sources, &_options, &_name);
+  }
+
+  /*! Restore a serialized program.
+   *
+   * \param serialized_program The serialized program to restore.
+   *
+   * \see serialize
+   */
+  static Program deserialize(std::string const& serialized_program) {
+    Program program;
+    if (!serialization::deserialize(serialized_program, &program._name,
+                                    &program._options, &program._sources)) {
+      throw std::runtime_error("Failed to deserialize program");
+    }
+    return program;
+  }
+
+  /*! Save the program.
+   *
+   * \see deserialize
+   */
+  std::string serialize() const {
+    // Note: Must update kSerializationVersion if this is changed.
+    return serialization::serialize(_name, _options, _sources);
+  };
+
+  /*! Select a kernel.
+   *
+   * \param name The name of the kernel (unmangled and without
+   * template arguments).
+   * \param options A vector of options to be passed to the NVRTC
+   * compiler when compiling this kernel.
+   */
+  Kernel kernel(std::string const& name,
+                std::vector<std::string> const& options = {}) const;
+};
+
+class Kernel {
+  friend class KernelInstantiation;
+  Program const* _program;
+  std::string _name;
+  std::vector<std::string> _options;
+
+ public:
+  Kernel(Program const* program, std::string const& name,
+         std::vector<std::string> const& options = {})
+      : _program(program), _name(name), _options(options) {}
+
+  /*! Instantiate the kernel.
+   *
+   *  \param template_args A vector of template arguments represented as
+   *    code-strings. These can be generated using
+   *    \code{.cpp}jitify::reflection::reflect<type>()\endcode or
+   *    \code{.cpp}jitify::reflection::reflect(value)\endcode
+   *
+   *  \note Template type deduction is not possible, so all types must be
+   *    explicitly specified.
+   */
+  KernelInstantiation instantiate(
+      std::vector<std::string> const& template_args =
+          std::vector<std::string>()) const;
+
+  // Regular template instantiation syntax (note limited flexibility)
+  /*! Instantiate the kernel.
+   *
+   *  \note The template arguments specified on this function are
+   *    used to instantiate the kernel. Non-type template arguments must
+   *    be wrapped with
+   *    \code{.cpp}jitify::reflection::NonType<type,value>\endcode
+   *
+   *  \note Template type deduction is not possible, so all types must be
+   *    explicitly specified.
+   */
+  template <typename... TemplateArgs>
+  KernelInstantiation instantiate() const;
+
+  // Template-like instantiation syntax
+  //   E.g., instantiate(myvar,Type<MyType>())(grid,block)
+  /*! Instantiate the kernel.
+   *
+   *  \param targs The template arguments for the kernel, represented as
+   *    values. Types must be wrapped with
+   *    \code{.cpp}jitify::reflection::Type<type>()\endcode or
+   *    \code{.cpp}jitify::reflection::type_of(value)\endcode
+   *
+   *  \note Template type deduction is not possible, so all types must be
+   *    explicitly specified.
+   */
+  template <typename... TemplateArgs>
+  KernelInstantiation instantiate(TemplateArgs... targs) const;
+};
+
+class KernelInstantiation {
+  friend class KernelLauncher;
+  std::unique_ptr<detail::CUDAKernel> _cuda_kernel;
+
+  // Private constructor used by deserialize()
+  KernelInstantiation(std::string const& func_name, std::string const& ptx,
+                      std::vector<std::string> const& link_files,
+                      std::vector<std::string> const& link_paths)
+      : _cuda_kernel(new detail::CUDAKernel(func_name.c_str(), ptx.c_str(),
+                                            link_files, link_paths)) {}
+
+ public:
+  KernelInstantiation(Kernel const& kernel,
+                      std::vector<std::string> const& template_args) {
+    Program const* program = kernel._program;
+
+    std::string template_inst =
+        (template_args.empty() ? ""
+                               : reflection::reflect_template(template_args));
+    std::string instantiation = kernel._name + template_inst;
+
+    std::vector<std::string> options;
+    options.insert(options.begin(), program->_options.begin(),
+                   program->_options.end());
+    options.insert(options.begin(), kernel._options.begin(),
+                   kernel._options.end());
+    detail::detect_and_add_cuda_arch(options);
+    detail::detect_and_add_cxx11_flag(options);
+
+    std::string log, ptx, mangled_instantiation;
+    std::vector<std::string> linker_files, linker_paths;
+    detail::instantiate_kernel(program->_name, program->_sources, instantiation,
+                               options, &log, &ptx, &mangled_instantiation,
+                               &linker_files, &linker_paths);
+
+    _cuda_kernel.reset(new detail::CUDAKernel(mangled_instantiation.c_str(),
+                                              ptx.c_str(), linker_files,
+                                              linker_paths));
+  }
+
+  /*! Implicit conversion to the underlying CUfunction object.
+   *
+   * \note This allows use of CUDA APIs like
+   *   cuOccupancyMaxActiveBlocksPerMultiprocessor.
+   */
+  operator CUfunction() const { return *_cuda_kernel; }
+
+  /*! Restore a serialized kernel instantiation.
+   *
+   * \param serialized_kernel_inst The serialized kernel instantiation to
+   * restore.
+   *
+   * \see serialize
+   */
+  static KernelInstantiation deserialize(
+      std::string const& serialized_kernel_inst) {
+    std::string func_name, ptx;
+    std::vector<std::string> link_files, link_paths;
+    if (!serialization::deserialize(serialized_kernel_inst, &func_name, &ptx,
+                                    &link_files, &link_paths)) {
+      throw std::runtime_error("Failed to deserialize kernel instantiation");
+    }
+    return KernelInstantiation(func_name, ptx, link_files, link_paths);
+  }
+
+  /*! Save the program.
+   *
+   * \see deserialize
+   */
+  std::string serialize() const {
+    // Note: Must update kSerializationVersion if this is changed.
+    return serialization::serialize(
+        _cuda_kernel->function_name(), _cuda_kernel->ptx(),
+        _cuda_kernel->link_files(), _cuda_kernel->link_paths());
+  }
+
+  /*! Configure the kernel launch.
+   *
+   *  \param grid   The thread grid dimensions for the launch.
+   *  \param block  The thread block dimensions for the launch.
+   *  \param smem   The amount of shared memory to dynamically allocate, in
+   * bytes.
+   *  \param stream The CUDA stream to launch the kernel in.
+   */
+  KernelLauncher configure(dim3 grid, dim3 block, unsigned int smem = 0,
+                           cudaStream_t stream = 0) const;
+
+  /*! Configure the kernel launch with a 1-dimensional block and grid chosen
+   *  automatically to maximise occupancy.
+   *
+   * \param max_block_size  The upper limit on the block size, or 0 for no
+   * limit.
+   * \param smem  The amount of shared memory to dynamically allocate, in bytes.
+   * \param smem_callback  A function returning smem for a given block size
+   * (overrides \p smem).
+   * \param stream The CUDA stream to launch the kernel in.
+   * \param flags The flags to pass to
+   * cuOccupancyMaxPotentialBlockSizeWithFlags.
+   */
+  KernelLauncher configure_1d_max_occupancy(
+      int max_block_size = 0, unsigned int smem = 0,
+      CUoccupancyB2DSize smem_callback = 0, cudaStream_t stream = 0,
+      unsigned int flags = 0) const;
+
+  /*
+   * Returns the function attribute requested from the kernel
+   */
+  inline int get_func_attribute(CUfunction_attribute attribute) const {
+    return _cuda_kernel->get_func_attribute(attribute);
+  }
+
+  /*
+   * Set the function attribute requested for the kernel
+   */
+  inline void set_func_attribute(CUfunction_attribute attribute,
+                                 int value) const {
+    _cuda_kernel->set_func_attribute(attribute, value);
+  }
+
+  /*
+   * \deprecated Use \p get_global_ptr instead.
+   */
+  CUdeviceptr get_constant_ptr(const char* name, size_t* size = nullptr) const {
+    return get_global_ptr(name, size);
+  }
+
+  /*
+   * Get a device pointer to a global __constant__ or __device__ variable using
+   * its un-mangled name. If provided, *size is set to the size of the variable
+   * in bytes.
+   */
+  CUdeviceptr get_global_ptr(const char* name, size_t* size = nullptr) const {
+    return _cuda_kernel->get_global_ptr(name, size);
+  }
+
+  /*
+   * Copy data from a global __constant__ or __device__ array to the host using
+   * its un-mangled name.
+   */
+  template <typename T>
+  CUresult get_global_array(const char* name, T* data, size_t count,
+                            CUstream stream = 0) const {
+    return _cuda_kernel->get_global_data(name, data, count, stream);
+  }
+
+  /*
+   * Copy a value from a global __constant__ or __device__ variable to the host
+   * using its un-mangled name.
+   */
+  template <typename T>
+  CUresult get_global_value(const char* name, T* value,
+                            CUstream stream = 0) const {
+    return get_global_array(name, value, 1, stream);
+  }
+
+  /*
+   * Copy data from the host to a global __constant__ or __device__ array using
+   * its un-mangled name.
+   */
+  template <typename T>
+  CUresult set_global_array(const char* name, const T* data, size_t count,
+                            CUstream stream = 0) const {
+    return _cuda_kernel->set_global_data(name, data, count, stream);
+  }
+
+  /*
+   * Copy a value from the host to a global __constant__ or __device__ variable
+   * using its un-mangled name.
+   */
+  template <typename T>
+  CUresult set_global_value(const char* name, const T& value,
+                            CUstream stream = 0) const {
+    return set_global_array(name, &value, 1, stream);
+  }
+
+  const std::string& mangled_name() const {
+    return _cuda_kernel->function_name();
+  }
+
+  const std::string& ptx() const { return _cuda_kernel->ptx(); }
+
+  const std::vector<std::string>& link_files() const {
+    return _cuda_kernel->link_files();
+  }
+
+  const std::vector<std::string>& link_paths() const {
+    return _cuda_kernel->link_paths();
+  }
+};
+
+class KernelLauncher {
+  KernelInstantiation const* _kernel_inst;
+  dim3 _grid;
+  dim3 _block;
+  unsigned int _smem;
+  cudaStream_t _stream;
+
+ private:
+  void pre_launch(std::vector<std::string> arg_types = {}) const {
+    (void)arg_types;
+#if JITIFY_PRINT_LAUNCH
+    std::string arg_types_string =
+        (arg_types.empty() ? "..." : reflection::reflect_list(arg_types));
+    std::cout << "Launching " << _kernel_inst->_cuda_kernel->function_name()
+              << "<<<" << _grid << "," << _block << "," << _smem << ","
+              << _stream << ">>>"
+              << "(" << arg_types_string << ")" << std::endl;
+#endif
+  }
+
+ public:
+  KernelLauncher(KernelInstantiation const* kernel_inst, dim3 grid, dim3 block,
+                 unsigned int smem = 0, cudaStream_t stream = 0)
+      : _kernel_inst(kernel_inst),
+        _grid(grid),
+        _block(block),
+        _smem(smem),
+        _stream(stream) {}
+
+  // Note: It's important that there is no implicit conversion required
+  //         for arg_ptrs, because otherwise the parameter pack version
+  //         below gets called instead (probably resulting in a segfault).
+  /*! Launch the kernel.
+   *
+   *  \param arg_ptrs  A vector of pointers to each function argument for the
+   *    kernel.
+   *  \param arg_types A vector of function argument types represented
+   *    as code-strings. This parameter is optional and is only used to print
+   *    out the function signature.
+   */
+  CUresult launch(std::vector<void*> arg_ptrs = {},
+                  std::vector<std::string> arg_types = {}) const {
+    pre_launch(arg_types);
+    return _kernel_inst->_cuda_kernel->launch(_grid, _block, _smem, _stream,
+                                              arg_ptrs);
+  }
+
+  void safe_launch(std::vector<void*> arg_ptrs = {},
+                   std::vector<std::string> arg_types = {}) const {
+    pre_launch(arg_types);
+    _kernel_inst->_cuda_kernel->safe_launch(_grid, _block, _smem, _stream,
+                                            arg_ptrs);
+  }
+
+  /*! Launch the kernel.
+   *
+   *  \param args Function arguments for the kernel.
+   */
+  template <typename... ArgTypes>
+  CUresult launch(const ArgTypes&... args) const {
+    return this->launch(std::vector<void*>({(void*)&args...}),
+                        {reflection::reflect<ArgTypes>()...});
+  }
+
+  /*! Launch the kernel and check for cuda errors.
+   *
+   *  \param args Function arguments for the kernel.
+   */
+  template <typename... ArgTypes>
+  void safe_launch(const ArgTypes&... args) const {
+    return this->safe_launch(std::vector<void*>({(void*)&args...}),
+                             {reflection::reflect<ArgTypes>()...});
+  }
+};
+
+inline Kernel Program::kernel(std::string const& name,
+                              std::vector<std::string> const& options) const {
+  return Kernel(this, name, options);
+}
+
+inline KernelInstantiation Kernel::instantiate(
+    std::vector<std::string> const& template_args) const {
+  return KernelInstantiation(*this, template_args);
+}
+
+template <typename... TemplateArgs>
+inline KernelInstantiation Kernel::instantiate() const {
+  return this->instantiate(
+      std::vector<std::string>({reflection::reflect<TemplateArgs>()...}));
+}
+
+template <typename... TemplateArgs>
+inline KernelInstantiation Kernel::instantiate(TemplateArgs... targs) const {
+  return this->instantiate(
+      std::vector<std::string>({reflection::reflect(targs)...}));
+}
+
+inline KernelLauncher KernelInstantiation::configure(
+    dim3 grid, dim3 block, unsigned int smem, cudaStream_t stream) const {
+  return KernelLauncher(this, grid, block, smem, stream);
+}
+
+inline KernelLauncher KernelInstantiation::configure_1d_max_occupancy(
+    int max_block_size, unsigned int smem, CUoccupancyB2DSize smem_callback,
+    cudaStream_t stream, unsigned int flags) const {
+  int grid;
+  int block;
+  CUfunction func = *_cuda_kernel;
+  detail::get_1d_max_occupancy(func, smem_callback, &smem, max_block_size,
+                               flags, &grid, &block);
+  return this->configure(grid, block, smem, stream);
+}
+
+}  // namespace experimental
+
+}  // namespace jitify
+
+#if defined(_WIN32) || defined(_WIN64)
+#pragma pop_macro("max")
+#pragma pop_macro("min")
+#pragma pop_macro("strtok_r")
+#endif
diff --git a/cupy/_core/include/cupy/jitify/jitify_example.cpp b/cupy/_core/include/cupy/jitify/jitify_example.cpp
new file mode 100644
index 0000000..8ef12eb
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/jitify_example.cpp
@@ -0,0 +1,359 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+  Simple examples demonstrating different ways to load source code
+    and call kernels.
+ */
+
+#ifdef LINUX  // Only supported by gcc on Linux (defined in Makefile)
+#define JITIFY_ENABLE_EMBEDDED_FILES 1
+#endif
+#define JITIFY_PRINT_INSTANTIATION 1
+#define JITIFY_PRINT_SOURCE 1
+#define JITIFY_PRINT_LOG 1
+#define JITIFY_PRINT_PTX 1
+#define JITIFY_PRINT_LINKER_LOG 1
+#define JITIFY_PRINT_LAUNCH 1
+#include "jitify.hpp"
+
+#include "example_headers/my_header1.cuh.jit"
+#ifdef LINUX  // Only supported by gcc on Linux (defined in Makefile)
+JITIFY_INCLUDE_EMBEDDED_FILE(example_headers_my_header2_cuh);
+#endif
+
+#include <cassert>
+#include <cmath>
+#include <iostream>
+
+#define CHECK_CUDA(call)                                                  \
+  do {                                                                    \
+    if (call != CUDA_SUCCESS) {                                           \
+      const char* str;                                                    \
+      cuGetErrorName(call, &str);                                         \
+      std::cout << "(CUDA) returned " << str;                             \
+      std::cout << " (" << __FILE__ << ":" << __LINE__ << ":" << __func__ \
+                << "())" << std::endl;                                    \
+      return false;                                                       \
+    }                                                                     \
+  } while (0)
+
+template <typename T>
+bool are_close(T in, T out) {
+  return fabs(in - out) <= 1e-5f * fabs(in);
+}
+
+std::istream* file_callback(std::string filename, std::iostream& tmp_stream) {
+  // User returns NULL or pointer to stream containing file source
+  // Note: tmp_stream is provided for convenience
+  if (filename == "example_headers/my_header4.cuh") {
+    tmp_stream << "#pragma once\n"
+                  "template<typename T>\n"
+                  "T pointless_func(T x) {\n"
+                  "	return x;\n"
+                  "}\n";
+    return &tmp_stream;
+  } else {
+    // Find this file through other mechanisms
+    return 0;
+  }
+}
+
+template <typename T>
+bool test_simple() {
+  const char* program_source =
+      "my_program\n"
+      "template<int N, typename T>\n"
+      "__global__\n"
+      "void my_kernel(T* data) {\n"
+      "    T data0 = data[0];\n"
+      "    for( int i=0; i<N-1; ++i ) {\n"
+      "        data[0] *= data0;\n"
+      "    }\n"
+      "}\n";
+  static jitify::JitCache kernel_cache;
+  jitify::Program program = kernel_cache.program(program_source, 0);
+  T h_data = 5;
+  T* d_data;
+  cudaMalloc((void**)&d_data, sizeof(T));
+  cudaMemcpy(d_data, &h_data, sizeof(T), cudaMemcpyHostToDevice);
+  dim3 grid(1);
+  dim3 block(1);
+  using jitify::reflection::type_of;
+  CHECK_CUDA(program.kernel("my_kernel")
+                 .instantiate(3, type_of(*d_data))
+                 .configure(grid, block)
+                 .launch(d_data));
+  cudaMemcpy(&h_data, d_data, sizeof(T), cudaMemcpyDeviceToHost);
+  cudaFree(d_data);
+  std::cout << h_data << std::endl;
+  return are_close(h_data, 125.f);
+}
+
+template <typename T>
+bool test_simple_experimental() {
+  const char* program_source =
+      "my_program\n"
+      "template<int N, typename T>\n"
+      "__global__\n"
+      "void my_kernel(T* data) {\n"
+      "    T data0 = data[0];\n"
+      "    for( int i=0; i<N-1; ++i ) {\n"
+      "        data[0] *= data0;\n"
+      "    }\n"
+      "}\n";
+  std::vector<std::string> opts;
+  jitify::experimental::Program program_orig(program_source, {}, opts);
+  auto program =
+      jitify::experimental::Program::deserialize(program_orig.serialize());
+  T h_data = 5;
+  T* d_data;
+  cudaMalloc((void**)&d_data, sizeof(T));
+  cudaMemcpy(d_data, &h_data, sizeof(T), cudaMemcpyHostToDevice);
+  dim3 grid(1);
+  dim3 block(1);
+  using jitify::reflection::type_of;
+  auto kernel_inst_orig =
+      program.kernel("my_kernel").instantiate(3, type_of(*d_data));
+  auto kernel_inst = jitify::experimental::KernelInstantiation::deserialize(
+      kernel_inst_orig.serialize());
+  CHECK_CUDA(kernel_inst.configure(grid, block).launch(d_data));
+  cudaMemcpy(&h_data, d_data, sizeof(T), cudaMemcpyDeviceToHost);
+  cudaFree(d_data);
+  std::cout << h_data << std::endl;
+  return are_close(h_data, 125.f);
+}
+
+template <typename T>
+bool test_kernels() {
+  // Note: The name is specified first, followed by a newline, then the code
+  const char* program1 =
+      "my_program1\n"
+      "#include \"example_headers/my_header1.cuh\"\n"
+      "#include \"example_headers/my_header2.cuh\"\n"
+      "#include \"example_headers/my_header3.cuh\"\n"
+      "#include \"example_headers/my_header4.cuh\"\n"
+      "\n"
+      "__global__\n"
+      "void my_kernel1(float const* indata, float* outdata) {\n"
+      "    outdata[0] = indata[0] + 1;\n"
+      "    outdata[0] -= 1;\n"
+      "}\n"
+      "\n"
+      "template<int C, typename T>\n"
+      "__global__\n"
+      "void my_kernel2(float const* indata, float* outdata) {\n"
+      "    for( int i=0; i<C; ++i ) {\n"
+      "        outdata[0] = "
+      "pointless_func(identity(sqrt(square(negate(indata[0])))));\n"
+      "    }\n"
+      "}\n";
+
+  using jitify::reflection::instance_of;
+  using jitify::reflection::NonType;
+  using jitify::reflection::reflect;
+  using jitify::reflection::Type;
+  using jitify::reflection::type_of;
+
+  thread_local static jitify::JitCache kernel_cache;
+  jitify::Program program = kernel_cache.program(
+      program1,                          // Code string specified above
+      {example_headers_my_header1_cuh},  // Code string generated by stringify
+      {"--use_fast_math", "-I" CUDA_INC_DIR}, file_callback);
+
+  T* indata;
+  T* outdata;
+  cudaMalloc((void**)&indata, sizeof(T));
+  cudaMalloc((void**)&outdata, sizeof(T));
+  T inval = 3.14159f;
+  cudaMemcpy(indata, &inval, sizeof(T), cudaMemcpyHostToDevice);
+
+  dim3 grid(1);
+  dim3 block(1);
+  CHECK_CUDA(program.kernel("my_kernel1")
+                 .instantiate()
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  enum { C = 123 };
+  // These invocations are all equivalent and will come from cache after the 1st
+  CHECK_CUDA((program.kernel("my_kernel2")
+                  .instantiate<NonType<int, C>, T>()
+                  .configure(grid, block)
+                  .launch(indata, outdata)));
+  CHECK_CUDA(program.kernel("my_kernel2")
+                 .instantiate({reflect((int)C), reflect<T>()})
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  // Recommended versions
+  CHECK_CUDA(program.kernel("my_kernel2")
+                 .instantiate((int)C, Type<T>())
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  CHECK_CUDA(program.kernel("my_kernel2")
+                 .instantiate((int)C, type_of(*indata))
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  CHECK_CUDA(program.kernel("my_kernel2")
+                 .instantiate((int)C, instance_of(*indata))
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+
+  T outval = 0;
+  cudaMemcpy(&outval, outdata, sizeof(T), cudaMemcpyDeviceToHost);
+  cudaFree(outdata);
+  cudaFree(indata);
+
+  std::cout << inval << " -> " << outval << std::endl;
+
+  return are_close(inval, outval);
+}
+
+bool test_constant() {
+  using jitify::reflection::Type;
+  thread_local static jitify::JitCache kernel_cache;
+
+  constexpr int n_const = 3;
+  int* outdata;
+  cudaMalloc((void**)&outdata, n_const * sizeof(int));
+
+  bool test = true;
+
+  dim3 grid(1);
+  dim3 block(1);
+  {  // test __constant__ look up in kernel string using diffrent namespaces
+    const char* const_program = R"(const_program
+    #pragma once
+
+    __constant__ int a;
+    namespace b { __constant__ int a; }
+    namespace c { namespace b { __constant__ int a; } }
+
+    __global__ void constant_test(int *x) {
+      x[0] = a;
+      x[1] = b::a;
+      x[2] = c::b::a;
+    }
+    )";
+    jitify::Program program = kernel_cache.program(
+        const_program, 0, {"--use_fast_math", "-I" CUDA_INC_DIR});
+    auto instance = program.kernel("constant_test").instantiate();
+    int inval[] = {2, 4, 8};
+    cuMemcpyHtoD(instance.get_constant_ptr("a"), &inval[0], sizeof(int));
+    cuMemcpyHtoD(instance.get_constant_ptr("b::a"), &inval[1], sizeof(int));
+    cuMemcpyHtoD(instance.get_constant_ptr("c::b::a"), &inval[2], sizeof(int));
+    CHECK_CUDA(instance.configure(grid, block).launch(outdata));
+    cudaDeviceSynchronize();
+    int outval[n_const];
+    cudaMemcpy(outval, outdata, sizeof(outval), cudaMemcpyDeviceToHost);
+
+    for (int i = 0; i < n_const; i++)
+      if (inval[i] != outval[i]) test = false;
+  }
+
+  {  // test __constant__ array look up in header nested in both anonymous and
+     // explicit namespace
+    jitify::Program program =
+        kernel_cache.program("example_headers/constant_header.cuh", 0,
+                             {"--use_fast_math", "-I" CUDA_INC_DIR});
+    auto instance = program.kernel("constant_test2").instantiate();
+    int inval[] = {3, 5, 9};
+    cuMemcpyHtoD(instance.get_constant_ptr("(anonymous namespace)::b::a"),
+                 inval, sizeof(inval));
+    CHECK_CUDA(instance.configure(grid, block).launch(outdata));
+
+    int outval[n_const];
+    cudaMemcpy(outval, outdata, sizeof(outval), cudaMemcpyDeviceToHost);
+
+    for (int i = 0; i < n_const; i++)
+      if (inval[i] != outval[i]) test = false;
+  }
+
+  cudaFree(outdata);
+
+  return test;
+}
+
+template <typename T>
+bool test_parallel_for() {
+  int n = 10000;
+  T* d_out;
+  cudaMalloc((void**)&d_out, n * sizeof(T));
+  T val = 3.14159f;
+
+  jitify::ExecutionPolicy policy(jitify::DEVICE);
+  auto lambda = JITIFY_LAMBDA((d_out, val),
+                              d_out[i] = static_cast<decltype(val)>(i) * val);
+  CHECK_CUDA(jitify::parallel_for(policy, 0, n, lambda));
+
+  std::vector<T> h_out(n);
+  cudaMemcpy(&h_out[0], d_out, n * sizeof(T), cudaMemcpyDeviceToHost);
+
+  cudaFree(d_out);
+
+  for (int i = 0; i < n; ++i) {
+    if (!are_close(h_out[i], (T)i * val)) {
+      std::cout << h_out[i] << " != " << (T)i * val << std::endl;
+      return false;
+    }
+  }
+  return true;
+}
+
+int main(int argc, char* argv[]) {
+  (void)argc;
+  (void)argv;
+#define TEST_RESULT(result) (result ? "PASSED" : "FAILED")
+
+  // Uncached
+  bool test_simple_result = test_simple<float>();
+  bool test_simple_experimental_result = test_simple_experimental<float>();
+  bool test_kernels_result = test_kernels<float>();
+  bool test_parallel_for_result = test_parallel_for<float>();
+  bool test_constant_result = test_constant();
+
+  // Cached
+  test_simple_result &= test_simple<float>();
+  test_kernels_result &= test_kernels<float>();
+  test_parallel_for_result &= test_parallel_for<float>();
+  test_constant_result &= test_constant();
+
+  std::cout << "test_simple<float>:       " << TEST_RESULT(test_simple_result)
+            << std::endl;
+  std::cout << "test_simple_experimental<float>:    "
+            << TEST_RESULT(test_simple_experimental_result) << std::endl;
+  std::cout << "test_kernels<float>:      " << TEST_RESULT(test_kernels_result)
+            << std::endl;
+  std::cout << "test_parallel_for<float>: "
+            << TEST_RESULT(test_parallel_for_result) << std::endl;
+  std::cout << "test_constant:            " << TEST_RESULT(test_constant_result)
+            << std::endl;
+
+  return (!test_simple_result + !test_simple_experimental_result +
+          !test_kernels_result + !test_parallel_for_result +
+          !test_constant_result);
+}
diff --git a/cupy/_core/include/cupy/jitify/jitify_test.cu b/cupy/_core/include/cupy/jitify/jitify_test.cu
new file mode 100644
index 0000000..9faf4ff
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/jitify_test.cu
@@ -0,0 +1,1081 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef LINUX  // Only supported by gcc on Linux (defined in Makefile)
+#define JITIFY_ENABLE_EMBEDDED_FILES 1
+#endif
+#define JITIFY_PRINT_INSTANTIATION 1
+#define JITIFY_PRINT_SOURCE 1
+#define JITIFY_PRINT_LOG 1
+#define JITIFY_PRINT_PTX 1
+#define JITIFY_PRINT_LINKER_LOG 1
+#define JITIFY_PRINT_LAUNCH 1
+#define JITIFY_PRINT_HEADER_PATHS 1
+#include "jitify.hpp"
+
+#include "example_headers/class_arg_kernel.cuh"
+#include "example_headers/my_header1.cuh.jit"
+
+#ifdef LINUX  // Only supported by gcc on Linux (defined in Makefile)
+JITIFY_INCLUDE_EMBEDDED_FILE(example_headers_my_header2_cuh);
+#endif
+
+#include "gtest/gtest.h"
+
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <memory>
+
+#define CHECK_CUDA(call)                                                  \
+  do {                                                                    \
+    CUresult status = call;                                               \
+    if (status != CUDA_SUCCESS) {                                         \
+      const char* str;                                                    \
+      cuGetErrorName(status, &str);                                       \
+      std::cout << "(CUDA) returned " << str;                             \
+      std::cout << " (" << __FILE__ << ":" << __LINE__ << ":" << __func__ \
+                << "())" << std::endl;                                    \
+      ASSERT_EQ(status, CUDA_SUCCESS);                                    \
+    }                                                                     \
+  } while (0)
+
+#define CHECK_CUDART(call)                                                \
+  do {                                                                    \
+    cudaError_t status = call;                                            \
+    if (status != cudaSuccess) {                                          \
+      std::cout << "(CUDART) returned " << cudaGetErrorString(status);    \
+      std::cout << " (" << __FILE__ << ":" << __LINE__ << ":" << __func__ \
+                << "())" << std::endl;                                    \
+      ASSERT_EQ(status, cudaSuccess);                                     \
+    }                                                                     \
+  } while (0)
+
+std::istream* file_callback(std::string filename, std::iostream& tmp_stream) {
+  // User returns NULL or pointer to stream containing file source
+  // Note: tmp_stream is provided for convenience
+  if (filename == "example_headers/my_header4.cuh") {
+    tmp_stream << "#pragma once\n"
+                  "template<typename T>\n"
+                  "T pointless_func(T x) {\n"
+                  "	return x;\n"
+                  "}\n";
+    return &tmp_stream;
+  } else {
+    // Find this file through other mechanisms
+    return 0;
+  }
+}
+
+static const char* const simple_program_source =
+    "my_program\n"
+    "template<int N, typename T>\n"
+    "__global__\n"
+    "void my_kernel(T* data) {\n"
+    "    if (blockIdx.x != 0 || threadIdx.x != 0) return;\n"
+    "    T data0 = data[0];\n"
+    "    for( int i=0; i<N-1; ++i ) {\n"
+    "        data[0] *= data0;\n"
+    "    }\n"
+    "}\n";
+
+TEST(JitifyTest, Simple) {
+  static jitify::JitCache kernel_cache;
+  jitify::Program program = kernel_cache.program(simple_program_source);
+  typedef float T;
+  T* d_data;
+  CHECK_CUDART(cudaMalloc((void**)&d_data, sizeof(T)));
+  dim3 grid(1);
+  dim3 block(1);
+  using jitify::reflection::type_of;
+  auto kernel_inst =
+      program.kernel("my_kernel").instantiate(3, type_of(*d_data));
+  T h_data = 5;
+  CHECK_CUDART(cudaMemcpy(d_data, &h_data, sizeof(T), cudaMemcpyHostToDevice));
+  CHECK_CUDA(kernel_inst.configure(grid, block).launch(d_data));
+  CHECK_CUDART(cudaMemcpy(&h_data, d_data, sizeof(T), cudaMemcpyDeviceToHost));
+  EXPECT_FLOAT_EQ(h_data, 125.f);
+
+  h_data = 5;
+  CHECK_CUDART(cudaMemcpy(d_data, &h_data, sizeof(T), cudaMemcpyHostToDevice));
+  CHECK_CUDA(kernel_inst.configure_1d_max_occupancy().launch(d_data));
+  CHECK_CUDART(cudaMemcpy(&h_data, d_data, sizeof(T), cudaMemcpyDeviceToHost));
+  EXPECT_FLOAT_EQ(h_data, 125.f);
+
+  CHECK_CUDART(cudaFree(d_data));
+}
+
+TEST(JitifyTest, Simple_experimental) {
+  std::vector<std::string> opts;
+  jitify::experimental::Program program_orig(simple_program_source, {}, opts);
+  auto program =
+      jitify::experimental::Program::deserialize(program_orig.serialize());
+  typedef float T;
+  T* d_data;
+  CHECK_CUDART(cudaMalloc((void**)&d_data, sizeof(T)));
+  dim3 grid(1);
+  dim3 block(1);
+  using jitify::reflection::type_of;
+  auto kernel_inst_orig =
+      program.kernel("my_kernel").instantiate(3, type_of(*d_data));
+  auto kernel_inst = jitify::experimental::KernelInstantiation::deserialize(
+      kernel_inst_orig.serialize());
+  T h_data = 5;
+  CHECK_CUDART(cudaMemcpy(d_data, &h_data, sizeof(T), cudaMemcpyHostToDevice));
+  CHECK_CUDA(kernel_inst.configure(grid, block).launch(d_data));
+  CHECK_CUDART(cudaMemcpy(&h_data, d_data, sizeof(T), cudaMemcpyDeviceToHost));
+  EXPECT_FLOAT_EQ(h_data, 125.f);
+
+  h_data = 5;
+  CHECK_CUDART(cudaMemcpy(d_data, &h_data, sizeof(T), cudaMemcpyHostToDevice));
+  CHECK_CUDA(kernel_inst.configure_1d_max_occupancy().launch(d_data));
+  CHECK_CUDART(cudaMemcpy(&h_data, d_data, sizeof(T), cudaMemcpyDeviceToHost));
+  EXPECT_FLOAT_EQ(h_data, 125.f);
+
+  CHECK_CUDART(cudaFree(d_data));
+}
+
+static const char* const multiple_kernels_program_source =
+    "my_program1\n"
+    "#include \"example_headers/my_header1.cuh\"\n"
+    "#include \"example_headers/my_header2.cuh\"\n"
+    "#include \"example_headers/my_header3.cuh\"\n"
+    "#include \"example_headers/my_header4.cuh\"\n"
+    "\n"
+    "__global__\n"
+    "void my_kernel1(float const* indata, float* outdata) {\n"
+    "    outdata[0] = indata[0] + 1;\n"
+    "    outdata[0] -= 1;\n"
+    "}\n"
+    "\n"
+    "template<int C, typename T>\n"
+    "__global__\n"
+    "void my_kernel2(float const* indata, float* outdata) {\n"
+    "    for( int i=0; i<C; ++i ) {\n"
+    "        outdata[0] = "
+    "pointless_func(identity(sqrt(square(negate(indata[0])))));\n"
+    "    }\n"
+    "}\n";
+
+TEST(JitifyTest, MultipleKernels) {
+  using jitify::reflection::instance_of;
+  using jitify::reflection::NonType;
+  using jitify::reflection::reflect;
+  using jitify::reflection::Type;
+  using jitify::reflection::type_of;
+
+  thread_local static jitify::JitCache kernel_cache;
+  jitify::Program program = kernel_cache.program(
+      multiple_kernels_program_source,   // Code string specified above
+      {example_headers_my_header1_cuh},  // Code string generated by stringify
+      {"--use_fast_math", "-I" CUDA_INC_DIR}, file_callback);
+
+  typedef float T;
+  T* indata;
+  T* outdata;
+  CHECK_CUDART(cudaMalloc((void**)&indata, sizeof(T)));
+  CHECK_CUDART(cudaMalloc((void**)&outdata, sizeof(T)));
+  T inval = 3.14159f;
+  CHECK_CUDART(cudaMemcpy(indata, &inval, sizeof(T), cudaMemcpyHostToDevice));
+
+  dim3 grid(1);
+  dim3 block(1);
+  CHECK_CUDA(program.kernel("my_kernel1")
+                 .instantiate()
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  enum { C = 123 };
+  // These invocations are all equivalent and will come from cache after the 1st
+  CHECK_CUDA((program.kernel("my_kernel2")
+                  .instantiate<NonType<int, C>, T>()
+                  .configure(grid, block)
+                  .launch(indata, outdata)));
+  CHECK_CUDA(program.kernel("my_kernel2")
+                 .instantiate({reflect((int)C), reflect<T>()})
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  // Recommended versions
+  CHECK_CUDA(program.kernel("my_kernel2")
+                 .instantiate((int)C, Type<T>())
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  CHECK_CUDA(program.kernel("my_kernel2")
+                 .instantiate((int)C, type_of(*indata))
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  CHECK_CUDA(program.kernel("my_kernel2")
+                 .instantiate((int)C, instance_of(*indata))
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+
+  T outval = 0;
+  CHECK_CUDART(cudaMemcpy(&outval, outdata, sizeof(T), cudaMemcpyDeviceToHost));
+  CHECK_CUDART(cudaFree(outdata));
+  CHECK_CUDART(cudaFree(indata));
+
+  EXPECT_FLOAT_EQ(inval, outval);
+}
+
+TEST(JitifyTest, MultipleKernels_experimental) {
+  using jitify::reflection::instance_of;
+  using jitify::reflection::NonType;
+  using jitify::reflection::reflect;
+  using jitify::reflection::Type;
+  using jitify::reflection::type_of;
+
+  jitify::experimental::Program program_orig(
+      multiple_kernels_program_source,   // Code string specified above
+      {example_headers_my_header1_cuh},  // Code string generated by stringify
+      {"--use_fast_math", "-I" CUDA_INC_DIR}, file_callback);
+  auto program =
+      jitify::experimental::Program::deserialize(program_orig.serialize());
+
+  typedef float T;
+  T* indata;
+  T* outdata;
+  CHECK_CUDART(cudaMalloc((void**)&indata, sizeof(T)));
+  CHECK_CUDART(cudaMalloc((void**)&outdata, sizeof(T)));
+  T inval = 3.14159f;
+  CHECK_CUDART(cudaMemcpy(indata, &inval, sizeof(T), cudaMemcpyHostToDevice));
+
+  dim3 grid(1);
+  dim3 block(1);
+  CHECK_CUDA(program.kernel("my_kernel1")
+                 .instantiate()
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  enum { C = 123 };
+  // These invocations are all equivalent.
+  CHECK_CUDA(jitify::experimental::KernelInstantiation::deserialize(
+                 program.kernel("my_kernel2")
+                     .instantiate<NonType<int, C>, T>()
+                     .serialize())
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  CHECK_CUDA(jitify::experimental::KernelInstantiation::deserialize(
+                 program.kernel("my_kernel2")
+                     .instantiate({reflect((int)C), reflect<T>()})
+                     .serialize())
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  // Recommended versions
+  CHECK_CUDA(jitify::experimental::KernelInstantiation::deserialize(
+                 program.kernel("my_kernel2")
+                     .instantiate((int)C, Type<T>())
+                     .serialize())
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  CHECK_CUDA(jitify::experimental::KernelInstantiation::deserialize(
+                 program.kernel("my_kernel2")
+                     .instantiate((int)C, type_of(*indata))
+                     .serialize())
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+  CHECK_CUDA(jitify::experimental::KernelInstantiation::deserialize(
+                 program.kernel("my_kernel2")
+                     .instantiate((int)C, instance_of(*indata))
+                     .serialize())
+                 .configure(grid, block)
+                 .launch(indata, outdata));
+
+  T outval = 0;
+  CHECK_CUDART(cudaMemcpy(&outval, outdata, sizeof(T), cudaMemcpyDeviceToHost));
+  CHECK_CUDART(cudaFree(outdata));
+  CHECK_CUDART(cudaFree(indata));
+
+  EXPECT_FLOAT_EQ(inval, outval);
+}
+
+static const char* const constmem_program_source =
+    "constmem_program\n"
+    "#pragma once\n"
+    "\n"
+    "__constant__ int a;\n"
+    "__device__ int d;\n"
+    "namespace b { __constant__ int a; __device__ int d; }\n"
+    "namespace c { namespace b { __constant__ int a; __device__ int d; } }\n"
+    "namespace x { __constant__ int a = 3; __device__ int d = 7; }\n"
+    "namespace y { __constant__ int a[] = {4, 5}; __device__ int d[] = {8, 9}; "
+    "}\n"
+    "\n"
+    "__global__ void constant_test(int *x) {\n"
+    "  x[0] = a;\n"
+    "  x[1] = b::a;\n"
+    "  x[2] = c::b::a;\n"
+    "  x[3] = d;\n"
+    "  x[4] = b::d;\n"
+    "  x[5] = c::b::d;\n"
+    "  x[6] = x::a;\n"
+    "  x[7] = x::d;\n"
+    "  x[8] = y::a[0];\n"
+    "  x[9] = y::a[1];\n"
+    "  x[10] = y::d[0];\n"
+    "  x[11] = y::d[1];\n"
+    "}\n";
+
+TEST(JitifyTest, ConstantMemory) {
+  using jitify::reflection::Type;
+  thread_local static jitify::JitCache kernel_cache;
+
+  constexpr int n_const = 12;
+  int* outdata;
+  CHECK_CUDART(cudaMalloc((void**)&outdata, n_const * sizeof(int)));
+
+  dim3 grid(1);
+  dim3 block(1);
+  {  // test __constant__ look up in kernel string using diffrent namespaces
+    jitify::Program program = kernel_cache.program(
+        constmem_program_source, 0, {"--use_fast_math", "-I" CUDA_INC_DIR});
+    auto instance = program.kernel("constant_test").instantiate();
+    int inval[] = {2, 4, 8, 12, 14, 18, 22, 26, 30, 34, 38, 42};
+    int dval;
+    CHECK_CUDA(instance.get_global_value("x::a", &dval));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    EXPECT_EQ(dval, 3);
+    CHECK_CUDA(instance.get_global_value("x::d", &dval));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    EXPECT_EQ(dval, 7);
+    int darr[2];
+    CHECK_CUDA(instance.get_global_array("y::a", &darr[0], 2));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    EXPECT_EQ(darr[0], 4);
+    EXPECT_EQ(darr[1], 5);
+    CHECK_CUDA(instance.get_global_value("y::d", &darr));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    EXPECT_EQ(darr[0], 8);
+    EXPECT_EQ(darr[1], 9);
+    CHECK_CUDA(instance.set_global_value("a", inval[0]));
+    CHECK_CUDA(instance.set_global_value("b::a", inval[1]));
+    CHECK_CUDA(instance.set_global_value("c::b::a", inval[2]));
+    CHECK_CUDA(instance.set_global_value("d", inval[3]));
+    CHECK_CUDA(instance.set_global_value("b::d", inval[4]));
+    CHECK_CUDA(instance.set_global_value("c::b::d", inval[5]));
+    CHECK_CUDA(instance.set_global_value("x::a", inval[6]));
+    CHECK_CUDA(instance.set_global_value("x::d", inval[7]));
+    CHECK_CUDA(instance.set_global_array("y::a", &inval[8], 2));
+    int inarr[] = {inval[10], inval[11]};
+    CHECK_CUDA(instance.set_global_value("y::d", inarr));
+    CHECK_CUDA(instance.configure(grid, block).launch(outdata));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    int outval[n_const];
+    CHECK_CUDART(
+        cudaMemcpy(outval, outdata, sizeof(outval), cudaMemcpyDeviceToHost));
+
+    for (int i = 0; i < n_const; i++) {
+      EXPECT_EQ(inval[i], outval[i]);
+    }
+  }
+
+  {  // test __constant__ array look up in header nested in both anonymous and
+     // explicit namespace
+    jitify::Program program =
+        kernel_cache.program("example_headers/constant_header.cuh", 0,
+                             {"--use_fast_math", "-I" CUDA_INC_DIR});
+    auto instance = program.kernel("constant_test2").instantiate();
+    constexpr int n_anon_const = 6;
+    int inval[] = {3, 5, 9, 13, 15, 19};
+    CHECK_CUDA(
+        cuMemcpyHtoD(instance.get_constant_ptr("(anonymous namespace)::b::a"),
+                     inval, sizeof(inval) / 2));
+    CHECK_CUDA(
+        cuMemcpyHtoD(instance.get_global_ptr("(anonymous namespace)::b::d"),
+                     inval + 3, sizeof(inval) / 2));
+    CHECK_CUDA(instance.configure(grid, block).launch(outdata));
+
+    int outval[n_anon_const];
+    CHECK_CUDART(
+        cudaMemcpy(outval, outdata, sizeof(outval), cudaMemcpyDeviceToHost));
+
+    for (int i = 0; i < n_anon_const; i++) {
+      EXPECT_EQ(inval[i], outval[i]);
+    }
+  }
+
+  CHECK_CUDART(cudaFree(outdata));
+}
+
+TEST(JitifyTest, ConstantMemory_experimental) {
+  using jitify::reflection::Type;
+
+  constexpr int n_const = 12;
+  int* outdata;
+  CHECK_CUDART(cudaMalloc((void**)&outdata, n_const * sizeof(int)));
+
+  dim3 grid(1);
+  dim3 block(1);
+  {  // test __constant__ look up in kernel string using different namespaces
+    jitify::experimental::Program program_orig(
+        constmem_program_source, {}, {"--use_fast_math", "-I" CUDA_INC_DIR});
+    auto program =
+        jitify::experimental::Program::deserialize(program_orig.serialize());
+    auto instance = jitify::experimental::KernelInstantiation::deserialize(
+        program.kernel("constant_test").instantiate().serialize());
+    int inval[] = {2, 4, 8, 12, 14, 18, 22, 26, 30, 34, 38, 42};
+    int dval;
+    CHECK_CUDA(instance.get_global_value("x::a", &dval));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    EXPECT_EQ(dval, 3);
+    CHECK_CUDA(instance.get_global_value("x::d", &dval));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    EXPECT_EQ(dval, 7);
+    int darr[2];
+    CHECK_CUDA(instance.get_global_array("y::a", &darr[0], 2));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    EXPECT_EQ(darr[0], 4);
+    EXPECT_EQ(darr[1], 5);
+    CHECK_CUDA(instance.get_global_value("y::d", &darr));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    EXPECT_EQ(darr[0], 8);
+    EXPECT_EQ(darr[1], 9);
+    CHECK_CUDA(instance.set_global_value("a", inval[0]));
+    CHECK_CUDA(instance.set_global_value("b::a", inval[1]));
+    CHECK_CUDA(instance.set_global_value("c::b::a", inval[2]));
+    CHECK_CUDA(instance.set_global_value("d", inval[3]));
+    CHECK_CUDA(instance.set_global_value("b::d", inval[4]));
+    CHECK_CUDA(instance.set_global_value("c::b::d", inval[5]));
+    CHECK_CUDA(instance.set_global_value("x::a", inval[6]));
+    CHECK_CUDA(instance.set_global_value("x::d", inval[7]));
+    CHECK_CUDA(instance.set_global_array("y::a", &inval[8], 2));
+    int inarr[] = {inval[10], inval[11]};
+    CHECK_CUDA(instance.set_global_value("y::d", inarr));
+    CHECK_CUDA(instance.configure(grid, block).launch(outdata));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    int outval[n_const];
+    CHECK_CUDART(
+        cudaMemcpy(outval, outdata, sizeof(outval), cudaMemcpyDeviceToHost));
+
+    for (int i = 0; i < n_const; i++) {
+      EXPECT_EQ(inval[i], outval[i]);
+    }
+  }
+
+  {  // test __constant__ array look up in header nested in both anonymous and
+     // explicit namespace
+    jitify::experimental::Program program_orig(
+        "example_headers/constant_header.cuh", {},
+        {"--use_fast_math", "-I" CUDA_INC_DIR});
+    auto program =
+        jitify::experimental::Program::deserialize(program_orig.serialize());
+    auto instance = jitify::experimental::KernelInstantiation::deserialize(
+        program.kernel("constant_test2").instantiate().serialize());
+    constexpr int n_anon_const = 6;
+    int inval[] = {3, 5, 9, 13, 15, 19};
+    CHECK_CUDA(
+        cuMemcpyHtoD(instance.get_constant_ptr("(anonymous namespace)::b::a"),
+                     inval, sizeof(inval) / 2));
+    CHECK_CUDA(
+        cuMemcpyHtoD(instance.get_global_ptr("(anonymous namespace)::b::d"),
+                     inval + 3, sizeof(inval) / 2));
+    CHECK_CUDA(instance.configure(grid, block).launch(outdata));
+
+    int outval[n_anon_const];
+    CHECK_CUDART(
+        cudaMemcpy(outval, outdata, sizeof(outval), cudaMemcpyDeviceToHost));
+
+    for (int i = 0; i < n_anon_const; i++) {
+      EXPECT_EQ(inval[i], outval[i]);
+    }
+  }
+
+  CHECK_CUDART(cudaFree(outdata));
+}
+
+TEST(JitifyTest, ParallelFor) {
+  int n = 10000;
+  typedef float T;
+  T* d_out;
+  CHECK_CUDART(cudaMalloc((void**)&d_out, n * sizeof(T)));
+  T val = 3.14159f;
+
+  jitify::ExecutionPolicy policy(jitify::DEVICE);
+  auto lambda = JITIFY_LAMBDA((d_out, val), d_out[i] = (float)i * val);
+  CHECK_CUDA(jitify::parallel_for(policy, 0, n, lambda));
+
+  std::vector<T> h_out(n);
+  CHECK_CUDART(
+      cudaMemcpy(&h_out[0], d_out, n * sizeof(T), cudaMemcpyDeviceToHost));
+
+  CHECK_CUDART(cudaFree(d_out));
+
+  for (int i = 0; i < n; ++i) {
+    EXPECT_FLOAT_EQ(h_out[i], (T)i * val);
+  }
+}
+
+TEST(JitifyTest, InvalidPrograms) {
+  jitify::JitCache kernel_cache;
+  auto program_v1 = kernel_cache.program("empty_program\n");  // OK
+  EXPECT_THROW(auto program_v2 = kernel_cache.program("missing_filename"),
+               std::runtime_error);
+  EXPECT_THROW(
+      auto program_v3 = kernel_cache.program("bad_program\nNOT CUDA C!"),
+      std::runtime_error);
+  jitify::experimental::Program program_v4("empty_program\n");  // OK
+  EXPECT_THROW(jitify::experimental::Program program_v5("missing_filename"),
+               std::runtime_error);
+  EXPECT_THROW(
+      jitify::experimental::Program program_v6("bad_program\nNOT CUDA C!"),
+      std::runtime_error);
+}
+
+static const char* const pragma_repl_program_source = R"(my_program
+template <int N, typename T>
+__global__ void my_kernel(T* data) {
+  if (blockIdx.x != 0 || threadIdx.x != 0) return;
+  T data0 = data[0];
+  #pragma unroll
+  for (int i = 0; i < N - 1; ++i) data[0] *= data0;
+  #pragma unroll 1
+  for (int i = 0; i < N - 1; ++i) data[0] *= data0;
+  #pragma unroll 1  // Make sure parsing works with comments
+  for (int i = 0; i < N - 1; ++i) data[0] *= data0;
+  // TODO: Add support for block comments.
+  //#pragma unroll 1  /* Make sure parsing works with comments */
+  //for (int i = 0; i < N - 1; ++i) data[0] *= data0;
+}
+)";
+
+TEST(JitifyTest, PragmaReplacement) {
+  static jitify::JitCache kernel_cache;
+  jitify::Program program = kernel_cache.program(pragma_repl_program_source);
+  typedef float T;
+  T* d_data = nullptr;
+  using jitify::reflection::type_of;
+  auto kernel_inst =
+      program.kernel("my_kernel").instantiate(3, type_of(*d_data));
+}
+
+// TODO: Expand this to include more Thrust code.
+static const char* const thrust_program_source =
+    "thrust_program\n"
+    "#include <thrust/iterator/counting_iterator.h>\n"
+    "__global__ void my_kernel(thrust::counting_iterator<int> begin,\n"
+    "                          thrust::counting_iterator<int> end) {\n"
+    "}\n";
+
+TEST(JitifyTest, ThrustHeaders) {
+  // Checks that basic Thrust headers can be compiled.
+  jitify::JitCache kernel_cache;
+#if CUDA_VERSION < 11000
+  const char* cppstd = "-std=c++98";
+#else
+  const char* cppstd = "-std=c++11";
+#endif
+  auto program_v1 = kernel_cache.program(thrust_program_source, {},
+                                         {"-I" CUDA_INC_DIR, cppstd});
+  auto program_v2 = jitify::experimental::Program(thrust_program_source, {},
+                                                  {"-I" CUDA_INC_DIR, cppstd});
+}
+
+static const char* const cub_program_source =
+    "cub_program\n"
+    "#include <cub/block/block_load.cuh>\n"
+    "#include <cub/block/block_radix_sort.cuh>\n"
+    "#include <cub/block/block_reduce.cuh>\n"
+    "#include <cub/block/block_store.cuh>\n"
+    "\n"
+    "template<int BLOCK_SIZE, int PER_THREAD>\n"
+    "__global__ void my_kernel(float* data) {\n"
+    "    typedef cub::BlockLoad<float, BLOCK_SIZE, PER_THREAD,\n"
+    "                           cub::BLOCK_LOAD_VECTORIZE> BlockLoad;\n"
+    "    typedef cub::BlockRadixSort<float, BLOCK_SIZE, PER_THREAD>\n"
+    "        BlockSort;\n"
+    "    typedef cub::BlockReduce<float, BLOCK_SIZE> BlockReduce;\n"
+    "    typedef cub::BlockStore<float, BLOCK_SIZE, PER_THREAD,\n"
+    "                            cub::BLOCK_STORE_VECTORIZE> BlockStore;\n"
+    "    __shared__ union {\n"
+    "        typename BlockLoad::TempStorage load;\n"
+    "        typename BlockSort::TempStorage sort;\n"
+    "        typename BlockReduce::TempStorage reduce;\n"
+    "        typename BlockStore::TempStorage store;\n"
+    "        float sum;\n"
+    "      } temp_storage;\n"
+    "    float thread_data[PER_THREAD];\n"
+    "    BlockLoad(temp_storage.load).Load(data, thread_data);\n"
+    "    __syncthreads();\n"
+    "    BlockSort(temp_storage.sort).Sort(thread_data);\n"
+    "    __syncthreads();\n"
+    "    float sum = BlockReduce(temp_storage.reduce).Sum(thread_data);\n"
+    "    __syncthreads();\n"
+    "    if (threadIdx.x == 0) {\n"
+    "      temp_storage.sum = sum;\n"
+    "    }\n"
+    "    __syncthreads();\n"
+    "    sum = temp_storage.sum;\n"
+    "    #pragma unroll\n"
+    "    for (int i = 0; i < PER_THREAD; ++i) {\n"
+    "        thread_data[i] *= 1.f / sum;\n"
+    "    }\n"
+    "    __syncthreads();\n"
+    "    BlockStore(temp_storage.store).Store(data, thread_data);\n"
+    "}\n";
+
+TEST(JitifyTest, CubBlockPrimitives) {
+  int block_size = 64;
+  int per_thread = 4;
+  int n = block_size * per_thread;
+  std::vector<float> h_data(n);
+  float sum = 0;
+  for (int i = 0; i < n; ++i) {
+    // Start with values sorted in reverse.
+    h_data[i] = (float)(n - 1 - i);
+    sum += h_data[i];
+  }
+  // Shuffle the values a bit.
+  std::swap(h_data[3], h_data[7]);
+  std::swap(h_data[10], h_data[20]);
+  std::vector<float> h_expected(n);
+  for (int i = 0; i < n; ++i) {
+    // Expected sorted and normalized.
+    h_expected[i] = (float)i / sum;
+  }
+  std::vector<float> h_result(n);
+  float* d_data;
+  CHECK_CUDART(cudaMalloc((void**)&d_data, n * sizeof(float)));
+
+  jitify::JitCache kernel_cache;
+  auto program_v1 = kernel_cache.program(cub_program_source, {},
+                                         {"-I" CUB_DIR, "-I" CUDA_INC_DIR});
+  CHECK_CUDART(cudaMemcpy(d_data, h_data.data(), n * sizeof(float),
+                          cudaMemcpyHostToDevice));
+  CHECK_CUDA(program_v1.kernel("my_kernel")
+                 .instantiate(block_size, per_thread)
+                 .configure(1, block_size)
+                 .launch(d_data));
+  CHECK_CUDART(cudaMemcpy(h_result.data(), d_data, n * sizeof(float),
+                          cudaMemcpyDeviceToHost));
+  for (int i = 0; i < n; ++i) {
+    EXPECT_FLOAT_EQ(h_result[i], h_expected[i]);
+  }
+
+  auto program_v2 = jitify::experimental::Program::deserialize(
+      jitify::experimental::Program(cub_program_source, {},
+                                    {"-I" CUB_DIR, "-I" CUDA_INC_DIR})
+          .serialize());
+  auto kernel_inst_v2 = jitify::experimental::KernelInstantiation::deserialize(
+      program_v2.kernel("my_kernel")
+          .instantiate(block_size, per_thread)
+          .serialize());
+  CHECK_CUDART(cudaMemcpy(d_data, h_data.data(), n * sizeof(float),
+                          cudaMemcpyHostToDevice));
+  CHECK_CUDA(kernel_inst_v2.configure(1, block_size).launch(d_data));
+  CHECK_CUDART(cudaMemcpy(h_result.data(), d_data, n * sizeof(float),
+                          cudaMemcpyDeviceToHost));
+  for (int i = 0; i < n; ++i) {
+    EXPECT_FLOAT_EQ(h_result[i], h_expected[i]);
+  }
+
+  CHECK_CUDART(cudaFree(d_data));
+}
+
+static const char* const unused_globals_source =
+    "unused_globals_program\n"
+    "struct Foo { static const int value = 7; };\n"
+    "struct Bar { int a; double b; };\n"
+    "__device__ float used_scalar;\n"
+    "__device__ float used_array[2];\n"
+    "__device__ Bar used_struct;\n"
+    "__device__ float unused_scalar;\n"
+    "__device__ float unused_array[3];\n"
+    "__device__ Bar unused_struct;\n"
+    "__device__ float reg, ret, bra;\n"  // Tricky names
+    "__global__ void foo_kernel(int* data) {\n"
+    "  if (blockIdx.x != 0 || threadIdx.x != 0) return;\n"
+    "  used_scalar = 1.f;\n"
+    "  used_array[1] = 2.f;\n"
+    "  used_struct.b = 3.f;\n"
+    "  __syncthreads();\n"
+    "  *data += Foo::value + used_scalar + used_array[1] + used_struct.b;\n"
+    "  printf(\"*data = %i\\n\", *data);\n"  // Produces global symbols named
+                                             // $str
+    "}\n";
+
+TEST(JitifyTest, RemoveUnusedGlobals) {
+  cudaFree(0);
+  auto program_v2 = jitify::experimental::Program(
+      unused_globals_source, {},
+      // Note: Flag added twice to test handling of repeats.
+      {"-remove-unused-globals", "--remove-unused-globals"});
+  auto kernel_inst_v2 = program_v2.kernel("foo_kernel").instantiate();
+  std::string ptx = kernel_inst_v2.ptx();
+  EXPECT_TRUE(ptx.find(".global .align 4 .f32 used_scalar;") !=
+              std::string::npos);
+  // Note: PTX represents arrays and structs as .b8 instead of the actual type.
+  EXPECT_TRUE(ptx.find(".global .align 4 .b8 used_array[8];") !=
+              std::string::npos);
+  EXPECT_TRUE(ptx.find(".global .align 8 .b8 used_struct[16];") !=
+              std::string::npos);
+  EXPECT_FALSE(ptx.find("_ZN3Foo5valueE") != std::string::npos);
+  EXPECT_FALSE(ptx.find("unused_scalar;") != std::string::npos);
+  EXPECT_FALSE(ptx.find("unused_array;") != std::string::npos);
+  EXPECT_FALSE(ptx.find("unused_struct;") != std::string::npos);
+  EXPECT_FALSE(ptx.find(".global .align 4 .f32 reg;") != std::string::npos);
+  EXPECT_FALSE(ptx.find(".global .align 4 .f32 ret;") != std::string::npos);
+  EXPECT_FALSE(ptx.find(".global .align 4 .f32 bra;") != std::string::npos);
+  int* d_data;
+  CHECK_CUDART(cudaMalloc((void**)&d_data, sizeof(int)));
+  int h_data = 3;
+  CHECK_CUDART(
+      cudaMemcpy(d_data, &h_data, sizeof(int), cudaMemcpyHostToDevice));
+  CHECK_CUDA(kernel_inst_v2.configure(1, 1).launch(d_data));
+  CHECK_CUDART(
+      cudaMemcpy(&h_data, d_data, sizeof(int), cudaMemcpyDeviceToHost));
+  EXPECT_EQ(h_data, 16);
+  CHECK_CUDART(cudaFree(d_data));
+}
+
+static const char* const curand_program_source =
+    "curand_program\n"
+    "#include <curand_kernel.h>\n"
+    "__global__ void my_kernel() {}\n"
+    "\n";
+
+TEST(JitifyTest, CuRandKernel) {
+  auto program_v2 = jitify::experimental::Program(
+      curand_program_source, {},
+      // Note: --remove-unused-globals is added to remove huge precomputed
+      // arrays that come from CURAND.
+      {"-I" CUDA_INC_DIR, "--remove-unused-globals"});
+  auto kernel_inst_v2 = program_v2.kernel("my_kernel").instantiate();
+  // TODO: Expand this test to actually call curand kernels and check outputs.
+}
+
+static const char* const linktest_program1_source =
+    "linktest_program1\n"
+    "__constant__ int c = 5;\n"
+    "__device__ int d = 7;\n"
+    "__device__ int f(int i) { return i + 11; }\n"
+    "\n";
+
+static const char* const linktest_program2_source =
+    "linktest_program2\n"
+    "extern __constant__ int c;\n"
+    "extern __device__ int d;\n"
+    "extern __device__ int f(int);\n"
+    "__global__ void my_kernel(int* data) {\n"
+    "  *data = f(*data + c + d);\n"
+    "}\n"
+    "\n";
+
+TEST(JitifyTest, LinkExternalFiles) {
+  cudaFree(0);
+  // Ensure temporary file is deleted at the end.
+  std::unique_ptr<const char, int (*)(const char*)> ptx_filename(
+      "example_headers/linktest.ptx", std::remove);
+  {
+    std::ofstream ptx_file(ptx_filename.get());
+    ptx_file.exceptions(std::ofstream::failbit | std::ofstream::badbit);
+    ptx_file << jitify::experimental::Program(linktest_program1_source, {},
+                                              {"-rdc=true"})
+                    .kernel("")
+                    .instantiate()
+                    .ptx();
+  }
+  auto program_v2 = jitify::experimental::Program(
+      linktest_program2_source, {},
+      {"-rdc=true", "-Lexample_headers", "-llinktest.ptx"});
+  auto kernel_inst_v2 = program_v2.kernel("my_kernel").instantiate();
+  int* d_data;
+  CHECK_CUDART(cudaMalloc((void**)&d_data, sizeof(int)));
+  int h_data = 3;
+  CHECK_CUDART(
+      cudaMemcpy(d_data, &h_data, sizeof(int), cudaMemcpyHostToDevice));
+  CHECK_CUDA(kernel_inst_v2.configure(1, 1).launch(d_data));
+  CHECK_CUDART(
+      cudaMemcpy(&h_data, d_data, sizeof(int), cudaMemcpyDeviceToHost));
+  EXPECT_EQ(h_data, 26);
+  CHECK_CUDART(cudaFree(d_data));
+}
+
+namespace a {
+__host__ __device__ int external_device_func(int i) { return i + 1; }
+}  // namespace a
+
+static const char* const selflink_program_source =
+    "selflink_program\n"
+    "namespace a {\n"
+    "extern __device__ int external_device_func(int);\n"
+    "}\n"
+    "__global__ void my_kernel(int* data) {\n"
+    "  *data = a::external_device_func(*data);\n"
+    "}\n"
+    "\n";
+
+TEST(JitifyTest, LinkCurrentExecutable) {
+  cudaFree(0);
+  using namespace jitify::experimental;
+  auto program = Program(selflink_program_source, {}, {"-l."});
+  auto kernel_inst = program.kernel("my_kernel").instantiate();
+  int* d_data;
+  CHECK_CUDART(cudaMalloc((void**)&d_data, sizeof(int)));
+  int h_data = 3;
+  CHECK_CUDART(
+      cudaMemcpy(d_data, &h_data, sizeof(int), cudaMemcpyHostToDevice));
+  CHECK_CUDA(kernel_inst.configure(1, 1).launch(d_data));
+  CHECK_CUDART(
+      cudaMemcpy(&h_data, d_data, sizeof(int), cudaMemcpyDeviceToHost));
+  EXPECT_EQ(h_data, 4);
+  CHECK_CUDART(cudaFree(d_data));
+}
+
+static const char* const reflection_program_source =
+    "reflection_program\n"
+    "struct Base { virtual ~Base() {} };\n"
+    "template <typename T>\n"
+    "struct Derived : public Base {};\n"
+    "template<typename T>\n"
+    "__global__ void type_kernel() {}\n"
+    "template<unsigned short N>\n"
+    "__global__ void nontype_kernel() {}\n"
+    "\n";
+
+struct Base {
+  virtual ~Base() {}
+};
+template <typename T>
+struct Derived : public Base {};
+
+TEST(JitifyTest, Reflection) {
+  cudaFree(0);
+  using namespace jitify::experimental;
+  using jitify::reflection::instance_of;
+  Program program(reflection_program_source);
+  auto type_kernel = program.kernel("type_kernel");
+
+#define JITIFY_TYPE_REFLECTION_TEST(T)                   \
+  EXPECT_EQ(type_kernel.instantiate<T>().mangled_name(), \
+            type_kernel.instantiate({#T}).mangled_name())
+  JITIFY_TYPE_REFLECTION_TEST(const volatile float);
+  JITIFY_TYPE_REFLECTION_TEST(const volatile float*);
+  JITIFY_TYPE_REFLECTION_TEST(const volatile float&);
+  JITIFY_TYPE_REFLECTION_TEST(Base * (const volatile float));
+  JITIFY_TYPE_REFLECTION_TEST(const volatile float[4]);
+#undef JITIFY_TYPE_REFLECTION_TEST
+
+  typedef Derived<float> derived_type;
+  const Base& base = derived_type();
+  EXPECT_EQ(type_kernel.instantiate(instance_of(base)).mangled_name(),
+            type_kernel.instantiate<derived_type>().mangled_name());
+
+  auto nontype_kernel = program.kernel("nontype_kernel");
+#define JITIFY_NONTYPE_REFLECTION_TEST(N)                 \
+  EXPECT_EQ(nontype_kernel.instantiate(N).mangled_name(), \
+            nontype_kernel.instantiate({#N}).mangled_name())
+  JITIFY_NONTYPE_REFLECTION_TEST(7);
+  JITIFY_NONTYPE_REFLECTION_TEST('J');
+#undef JITIFY_NONTYPE_REFLECTION_TEST
+}
+
+static const char* const builtin_numeric_limits_program_source =
+    "builtin_numeric_limits_program\n"
+    "#include <limits>\n"
+    "struct MyType {};\n"
+    "namespace std {\n"
+    "template<> class numeric_limits<MyType> {\n"
+    " public:\n"
+    "  static MyType min() { return {}; }\n"
+    "  static MyType max() { return {}; }\n"
+    "};\n"
+    "}  // namespace std\n"
+    "template <typename T>\n"
+    "__global__ void my_kernel(T* data) {\n"
+    "  data[0] = std::numeric_limits<T>::min();\n"
+    "  data[1] = std::numeric_limits<T>::max();\n"
+    "}\n";
+
+TEST(JitifyTest, BuiltinNumericLimitsHeader) {
+  cudaFree(0);
+  using namespace jitify::experimental;
+  auto program = Program(builtin_numeric_limits_program_source);
+  for (const auto& type :
+       {"float", "double", "char", "signed char", "unsigned char", "short",
+        "unsigned short", "int", "unsigned int", "long", "unsigned long",
+        "long long", "unsigned long long", "MyType"}) {
+    program.kernel("my_kernel").instantiate({type});
+  }
+}
+
+TEST(JitifyTest, ClassKernelArg) {
+  using jitify::reflection::Type;
+  thread_local static jitify::JitCache kernel_cache;
+
+  int h_data;
+  int* d_data;
+  CHECK_CUDART(cudaMalloc((void**)&d_data, sizeof(int)));
+
+  dim3 grid(1);
+  dim3 block(1);
+
+  jitify::Program program =
+      kernel_cache.program("example_headers/class_arg_kernel.cuh", 0,
+                           {"--use_fast_math", "-I" CUDA_INC_DIR});
+
+  {  // test that we can pass an arg object to a kernel
+    Arg arg(-1);
+    CHECK_CUDA(program.kernel("class_arg_kernel")
+                   .instantiate(Type<Arg>())
+                   .configure(grid, block)
+                   .launch(d_data, arg));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    CHECK_CUDART(
+        cudaMemcpy(&h_data, d_data, sizeof(int), cudaMemcpyDeviceToHost));
+    EXPECT_EQ(arg.x, h_data);
+  }
+
+  {  // test that we can pass an arg object rvalue to a kernel
+    int value = -2;
+    CHECK_CUDA(program.kernel("class_arg_kernel")
+                   .instantiate(Type<Arg>())
+                   .configure(grid, block)
+                   .launch(d_data, Arg(value)));
+    CHECK_CUDART(cudaDeviceSynchronize());
+    CHECK_CUDART(
+        cudaMemcpy(&h_data, d_data, sizeof(int), cudaMemcpyDeviceToHost));
+    EXPECT_EQ(value, h_data);
+  }
+
+  {  // test that we can pass an arg object reference to a kernel
+    Arg* arg = new Arg(-3);
+    // references are passed as pointers since refernces are just pointers from
+    // an ABI point of view
+    CHECK_CUDA(program.kernel("class_arg_ref_kernel")
+                   .instantiate(Type<Arg>())
+                   .configure(grid, block)
+                   .launch(d_data, arg));
+    CHECK_CUDART(
+        cudaMemcpy(&h_data, d_data, sizeof(int), cudaMemcpyDeviceToHost));
+    EXPECT_EQ(arg->x, h_data);
+    delete (arg);
+  }
+
+  {  // test that we can pass an arg object reference to a kernel
+    Arg* arg = new Arg(-4);
+    CHECK_CUDA(program.kernel("class_arg_ptr_kernel")
+                   .instantiate(Type<Arg>())
+                   .configure(grid, block)
+                   .launch(d_data, arg));
+    CHECK_CUDART(
+        cudaMemcpy(&h_data, d_data, sizeof(int), cudaMemcpyDeviceToHost));
+    EXPECT_EQ(arg->x, h_data);
+    delete (arg);
+  }
+
+  CHECK_CUDART(cudaFree(d_data));
+}
+
+static const char* const assert_program_source = R"(
+  #include <cassert>
+  __global__ void my_assert_kernel() {
+  assert(0 == 1);
+  }
+  )";
+
+static const char* const get_attribute_program_source = R"(
+  __global__ void get_attribute_kernel(int *out, int *in) {
+  __shared__ int buffer[4096];
+  buffer[threadIdx.x] = in[threadIdx.x];
+  __syncthreads();
+  out[threadIdx.y] = buffer[threadIdx.x];
+  }
+  )";
+
+TEST(JitifyTest, GetAttribute) {
+  // Checks that we can get function attributes
+  jitify::JitCache kernel_cache;
+  auto program = kernel_cache.program(get_attribute_program_source, {},
+                                      {"-I" CUDA_INC_DIR});
+  auto instance = program.kernel("get_attribute_kernel").instantiate();
+
+  EXPECT_EQ(4096 * (int)sizeof(int),
+            instance.get_func_attribute(CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES));
+}
+
+static const char* const set_attribute_program_source = R"(
+  __global__ void set_attribute_kernel(int *out, int *in) {
+  extern __shared__ int buffer[];
+  buffer[threadIdx.x] = in[threadIdx.x];
+  __syncthreads();
+  out[threadIdx.y] = buffer[threadIdx.x];
+  }
+  )";
+
+TEST(JitifyTest, SetAttribute) {
+  // Checks that we can set function attributes
+  jitify::JitCache kernel_cache;
+
+  int* in;
+  CHECK_CUDART(cudaMalloc((void**)&in, sizeof(int)));
+  int* out;
+  CHECK_CUDART(cudaMalloc((void**)&out, sizeof(int)));
+
+  // query the maximum supported shared bytes per block
+  CUdevice device;
+  CHECK_CUDA(cuDeviceGet(&device, 0));
+  int shared_bytes;
+  CHECK_CUDA(cuDeviceGetAttribute(
+      &shared_bytes, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN,
+      device));
+
+  auto program = kernel_cache.program(set_attribute_program_source, {},
+                                      {"-I" CUDA_INC_DIR});
+  auto instance = program.kernel("set_attribute_kernel").instantiate();
+  instance.set_func_attribute(CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES,
+                              shared_bytes);
+
+  dim3 grid(1);
+  dim3 block(1);
+  // this kernel will fail on Volta+ unless the set attribute succeeded
+  CHECK_CUDA(instance.configure(grid, block, shared_bytes).launch(out, in));
+
+  CHECK_CUDART(cudaFree(out));
+  CHECK_CUDART(cudaFree(in));
+}
+
+TEST(JitifyTest, EnvVarOptions) {
+  setenv("JITIFY_OPTIONS", "-bad_option", true);
+  EXPECT_THROW(jitify::JitCache kernel_cache;
+               auto program = kernel_cache.program(simple_program_source),
+               std::runtime_error);
+  EXPECT_THROW(jitify::experimental::Program program(simple_program_source),
+               std::runtime_error);
+  setenv("JITIFY_OPTIONS", "", true);
+}
+
+// NOTE: This MUST be the last test in the file, due to sticky CUDA error.
+TEST(JitifyTest, AssertHeader) {
+  // Checks that cassert works as expected
+  jitify::JitCache kernel_cache;
+  auto program =
+      kernel_cache.program(assert_program_source, {}, {"-I" CUDA_INC_DIR});
+  dim3 grid(1);
+  dim3 block(1);
+  CHECK_CUDA((program.kernel("my_assert_kernel")
+                  .instantiate<>()
+                  .configure(grid, block)
+                  .launch()));
+}
diff --git a/cupy/_core/include/cupy/jitify/stringify.cpp b/cupy/_core/include/cupy/jitify/stringify.cpp
new file mode 100644
index 0000000..6b81758
--- /dev/null
+++ b/cupy/_core/include/cupy/jitify/stringify.cpp
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+  Stringify is a simple utility to convert text files to C string literals.
+ */
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+// Replaces non-alphanumeric characters with '_' and
+//   prepends '_' if the string begins with a digit.
+std::string sanitize_varname(std::string const& s) {
+  std::string r = s;
+  if (std::isdigit(r[0])) {
+    r = '_' + r;
+  }
+  for (std::string::iterator it = r.begin(); it != r.end(); ++it) {
+    if (!std::isalnum(*it)) {
+      *it = '_';
+    }
+  }
+  return r;
+}
+// Replaces " with \"
+std::string sanitize_string_literal(std::string const& s) {
+  std::stringstream ss;
+  for (std::string::const_iterator it = s.begin(); it != s.end(); ++it) {
+    if (*it == '"' || *it == '\\') {
+      ss << '\\';
+    }
+    ss << *it;
+  }
+  return ss.str();
+}
+
+int main(int argc, char* argv[]) {
+  if (argc <= 1 || argv[1][0] == '-') {
+    std::cout << "Stringify - Converts text files to C string literals"
+              << std::endl;
+    std::cout << "Usage: " << argv[0] << " infile [varname] > outfile"
+              << std::endl;
+    return -1;
+  }
+  char* filename = argv[1];
+  std::string varname = (argc > 2) ? argv[2] : sanitize_varname(filename);
+  std::ifstream istream(filename);
+  std::ostream& ostream = std::cout;
+  std::string line;
+  // Note: This puts "filename\n" at the beginning of the string, which is
+  //         what jitify expects.
+  ostream << "const char* const " << varname << " = "
+          << "\"" << filename << "\\n\"" << std::endl;
+  while (std::getline(istream, line)) {
+    ostream << "\"" << sanitize_string_literal(line) << "\\n\"" << std::endl;
+  }
+  ostream << ";" << std::endl;
+  return 0;
+}
diff --git a/cupy/_core/include/cupy/math_constants.h b/cupy/_core/include/cupy/math_constants.h
new file mode 100644
index 0000000..9467145
--- /dev/null
+++ b/cupy/_core/include/cupy/math_constants.h
@@ -0,0 +1,20 @@
+#ifndef CUPY_MATH_CONSTANTS_H
+#define CUPY_MATH_CONSTANTS_H
+
+/*
+   We bundle some constants found in math_constants.h, so that code can be
+   JIT compiled in environments without a full CUDA installation. The constants
+   are added as needed to avoid copying the entire header.
+*/
+
+
+/* single precision constants */
+#define CUDART_INF_F            __int_as_float(0x7f800000)
+#define CUDART_NAN_F            __int_as_float(0x7fffffff)
+
+/* double precision constants */
+#define CUDART_INF              __longlong_as_double(0x7ff0000000000000ULL)
+#define CUDART_NAN              __longlong_as_double(0xfff8000000000000ULL)
+
+
+#endif  // CUPY_MATH_CONSTANTS_H
diff --git a/cupy/_core/include/cupy/pair.cuh b/cupy/_core/include/cupy/pair.cuh
new file mode 100644
index 0000000..7f2aeda
--- /dev/null
+++ b/cupy/_core/include/cupy/pair.cuh
@@ -0,0 +1,279 @@
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*! \file pair.h
+ *  \brief A type encapsulating a heterogeneous pair of elements
+ */
+
+#pragma once
+
+namespace thrust
+{
+
+/*! \addtogroup utility
+ *  \{
+ */
+
+/*! \addtogroup pair
+ *  \{
+ */
+
+/*! \p pair is a generic data structure encapsulating a heterogeneous
+ *  pair of values.
+ *
+ *  \tparam T1 The type of \p pair's first object type.  There are no
+ *          requirements on the type of \p T1. <tt>T1</tt>'s type is
+ *          provided by <tt>pair::first_type</tt>.
+ *
+ *  \tparam T2 The type of \p pair's second object type.  There are no
+ *          requirements on the type of \p T2. <tt>T2</tt>'s type is
+ *          provided by <tt>pair::second_type</tt>.
+ */
+template <typename T1, typename T2>
+  struct pair
+{
+  /*! \p first_type is the type of \p pair's first object type.
+   */
+  typedef T1 first_type;
+
+  /*! \p second_type is the type of \p pair's second object type.
+   */
+  typedef T2 second_type;
+
+  /*! The \p pair's first object.
+   */
+  first_type first;
+
+  /*! The \p pair's second object.
+   */
+  second_type second;
+
+  /*! \p pair's default constructor constructs \p first
+   *  and \p second using \c first_type & \c second_type's
+   *  default constructors, respectively.
+   */
+  __host__ __device__ pair(void);
+
+  /*! This constructor accepts two objects to copy into this \p pair.
+   *
+   *  \param x The object to copy into \p first.
+   *  \param y The object to copy into \p second.
+   */
+  inline __host__ __device__
+  pair(const T1 &x, const T2 &y);
+
+  /*! This copy constructor copies from a \p pair whose types are
+   *  convertible to this \p pair's \c first_type and \c second_type,
+   *  respectively.
+   *
+   *  \param p The \p pair to copy from.
+   *
+   *  \tparam U1 is convertible to \c first_type.
+   *  \tparam U2 is convertible to \c second_type.
+   */
+  template <typename U1, typename U2>
+  inline __host__ __device__
+  pair(const pair<U1,U2> &p);
+
+  /*! This copy constructor copies from a <tt>std::pair</tt> whose types are
+   *  convertible to this \p pair's \c first_type and \c second_type,
+   *  respectively.
+   *
+   *  \param p The <tt>std::pair</tt> to copy from.
+   *
+   *  \tparam U1 is convertible to \c first_type.
+   *  \tparam U2 is convertible to \c second_type.
+   */
+  // template <typename U1, typename U2>
+  // inline __host__ __device__
+  // pair(const std::pair<U1,U2> &p);
+
+  /*! \p swap swaps the elements of two <tt>pair</tt>s.
+   *  
+   *  \param p The other <tt>pair</tt> with which to swap.
+   */
+  inline __host__ __device__
+  void swap(pair &p);
+}; // end pair
+
+
+/*! This operator tests two \p pairs for equality.
+ *
+ *  \param x The first \p pair to compare.
+ *  \param y The second \p pair to compare.
+ *  \return \c true if and only if <tt>x.first == y.first && x.second == y.second</tt>.
+ *  
+ *  \tparam T1 is a model of <a href="https://en.cppreference.com/w/cpp/concepts/equality_comparable">Equality Comparable</a>.
+ *  \tparam T2 is a model of <a href="https://en.cppreference.com/w/cpp/concepts/equality_comparable">Equality Comparable</a>.
+ */
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator==(const pair<T1,T2> &x, const pair<T1,T2> &y);
+
+
+/*! This operator tests two pairs for ascending ordering.
+ *
+ *  \param x The first \p pair to compare.
+ *  \param y The second \p pair to compare.
+ *  \return \c true if and only if <tt>x.first < y.first || (!(y.first < x.first) && x.second < y.second)</tt>.
+ *
+ *  \tparam T1 is a model of <a href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">LessThan Comparable</a>.
+ *  \tparam T2 is a model of <a href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">LessThan Comparable</a>.
+ */
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator<(const pair<T1,T2> &x, const pair<T1,T2> &y);
+
+
+/*! This operator tests two pairs for inequality.
+ *
+ *  \param x The first \p pair to compare.
+ *  \param y The second \p pair to compare.
+ *  \return \c true if and only if <tt>!(x == y)</tt>.
+ *
+ *  \tparam T1 is a model of <a href="https://en.cppreference.com/w/cpp/concepts/equality_comparable">Equality Comparable</a>.
+ *  \tparam T2 is a model of <a href="https://en.cppreference.com/w/cpp/concepts/equality_comparable">Equality Comparable</a>.
+ */
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator!=(const pair<T1,T2> &x, const pair<T1,T2> &y);
+
+
+/*! This operator tests two pairs for descending ordering.
+ *
+ *  \param x The first \p pair to compare.
+ *  \param y The second \p pair to compare.
+ *  \return \c true if and only if <tt>y < x</tt>.
+ *
+ *  \tparam T1 is a model of <a href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">LessThan Comparable</a>.
+ *  \tparam T2 is a model of <a href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">LessThan Comparable</a>.
+ */
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator>(const pair<T1,T2> &x, const pair<T1,T2> &y);
+
+
+/*! This operator tests two pairs for ascending ordering or equivalence.
+ *
+ *  \param x The first \p pair to compare.
+ *  \param y The second \p pair to compare.
+ *  \return \c true if and only if <tt>!(y < x)</tt>.
+ *
+ *  \tparam T1 is a model of <a href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">LessThan Comparable</a>.
+ *  \tparam T2 is a model of <a href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">LessThan Comparable</a>.
+ */
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator<=(const pair<T1,T2> &x, const pair<T1,T2> &y);
+
+
+/*! This operator tests two pairs for descending ordering or equivalence.
+ *
+ *  \param x The first \p pair to compare.
+ *  \param y The second \p pair to compare.
+ *  \return \c true if and only if <tt>!(x < y)</tt>.
+ *
+ *  \tparam T1 is a model of <a href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">LessThan Comparable</a>.
+ *  \tparam T2 is a model of <a href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">LessThan Comparable</a>.
+ */
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator>=(const pair<T1,T2> &x, const pair<T1,T2> &y);
+
+
+/*! \p swap swaps the contents of two <tt>pair</tt>s.
+ *
+ *  \param x The first \p pair to swap.
+ *  \param y The second \p pair to swap.
+ */
+template <typename T1, typename T2>
+  inline __host__ __device__
+    void swap(pair<T1,T2> &x, pair<T1,T2> &y);
+
+
+/*! This convenience function creates a \p pair from two objects.
+ *
+ *  \param x The first object to copy from.
+ *  \param y The second object to copy from.
+ *  \return A newly-constructed \p pair copied from \p a and \p b.
+ *
+ *  \tparam T1 There are no requirements on the type of \p T1.
+ *  \tparam T2 There are no requirements on the type of \p T2.
+ */
+template <typename T1, typename T2>
+  inline __host__ __device__
+    pair<T1,T2> make_pair(T1 x, T2 y);
+
+
+/*! This convenience metafunction is included for compatibility with
+ *  \p tuple. It returns either the type of a \p pair's
+ *  \c first_type or \c second_type in its nested type, \c type.
+ *
+ *  \tparam N This parameter selects the member of interest.
+ *  \tparam T A \c pair type of interest.
+ */
+template<size_t N, class T> struct tuple_element;
+
+
+/*! This convenience metafunction is included for compatibility with
+ *  \p tuple. It returns \c 2, the number of elements of a \p pair,
+ *  in its nested data member, \c value.
+ *
+ *  \tparam Pair A \c pair type of interest.
+ */
+template<typename Pair> struct tuple_size;
+
+
+/*! This convenience function returns a reference to either the first or
+ *  second member of a \p pair.
+ *
+ *  \param p The \p pair of interest.
+ *  \return \c p.first or \c p.second, depending on the template
+ *          parameter.
+ *
+ *  \tparam N This parameter selects the member of interest.
+ */
+// XXX comment out these prototypes as a WAR to a problem on MSVC 2005
+//template<unsigned int N, typename T1, typename T2>
+//  inline __host__ __device__
+//    typename tuple_element<N, pair<T1,T2> >::type &
+//      get(pair<T1,T2> &p);
+
+
+/*! This convenience function returns a const reference to either the
+ *  first or second member of a \p pair.
+ *
+ *  \param p The \p pair of interest.
+ *  \return \c p.first or \c p.second, depending on the template
+ *          parameter.
+ *
+ *  \tparam i This parameter selects the member of interest.
+ */
+// XXX comment out these prototypes as a WAR to a problem on MSVC 2005
+//template<int N, typename T1, typename T2>
+//  inline __host__ __device__
+//    const typename tuple_element<N, pair<T1,T2> >::type &
+//      get(const pair<T1,T2> &p);
+
+/*! \} // pair
+ */
+
+/*! \} // utility
+ */
+
+}  // namespace thrust
+
+#include <cupy/tuple/pair.h>
diff --git a/cupy/_core/include/cupy/swap.cuh b/cupy/_core/include/cupy/swap.cuh
new file mode 100644
index 0000000..5c6efdc
--- /dev/null
+++ b/cupy/_core/include/cupy/swap.cuh
@@ -0,0 +1,37 @@
+// This file is modified from thrust/detail/swap.h
+
+/*
+ *  Copyright 2008-2013 NVIDIA Corporation
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+#pragma once
+
+#define __thrust_exec_check_disable__
+
+namespace thrust
+{
+
+__thrust_exec_check_disable__
+template<typename Assignable1, typename Assignable2>
+__host__ __device__
+inline void swap(Assignable1 &a, Assignable2 &b)
+{
+  Assignable1 temp = a;
+  a = b;
+  b = temp;
+} // end swap()
+
+} // end namespace thrust
diff --git a/cupy/_core/include/cupy/tuple.cuh b/cupy/_core/include/cupy/tuple.cuh
new file mode 100644
index 0000000..df8b16b
--- /dev/null
+++ b/cupy/_core/include/cupy/tuple.cuh
@@ -0,0 +1,568 @@
+/*
+ *  Copyright 2008-2018 NVIDIA Corporation
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*! \file tuple.h
+ *  \brief A type encapsulating a heterogeneous collection of elements
+ */
+
+/*
+ * Copyright (C) 1999, 2000 Jaakko Jarvi (jaakko.jarvi@cs.utu.fi)
+ * 
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying NOTICE file for the complete license)
+ *
+ * For more information, see http://www.boost.org
+ */
+
+#include <cupy/tuple/tuple.h>
+
+namespace thrust
+{
+
+/*! \addtogroup utility
+ *  \{
+ */
+
+/*! \addtogroup tuple
+ *  \{
+ */
+
+/*! \cond
+ */
+
+struct null_type;
+
+/*! \endcond
+ */
+
+/*! This metafunction returns the type of a
+ *  \p tuple's <tt>N</tt>th element.
+ *
+ *  \tparam N This parameter selects the element of interest.
+ *  \tparam T A \c tuple type of interest.
+ *
+ *  \see pair
+ *  \see tuple
+ */
+template<size_t N, class T>
+  struct tuple_element
+{
+  private:
+    typedef typename T::tail_type Next;
+
+  public:
+    /*! The result of this metafunction is returned in \c type.
+     */
+    typedef typename tuple_element<N-1, Next>::type type;
+}; // end tuple_element
+
+/*! This metafunction returns the number of elements
+ *  of a \p tuple type of interest.
+ *
+ *  \tparam T A \c tuple type of interest.
+ *
+ *  \see pair
+ *  \see tuple
+ */
+template<class T>
+  struct tuple_size
+{
+  /*! The result of this metafunction is returned in \c value.
+   */
+  static const int value = 1 + tuple_size<typename T::tail_type>::value;
+}; // end tuple_size
+
+// get function for non-const cons-lists, returns a reference to the element
+
+/*! The \p get function returns a reference to a \p tuple element of
+ *  interest.
+ *
+ *  \param t A reference to a \p tuple of interest.
+ *  \return A reference to \p t's <tt>N</tt>th element.
+ *
+ *  \tparam N The index of the element of interest.
+ *
+ *  The following code snippet demonstrates how to use \p get to print
+ *  the value of a \p tuple element.
+ *
+ *  \code
+ *  #include <thrust/tuple.h>
+ *  #include <iostream>
+ *  ...
+ *  thrust::tuple<int, const char *> t(13, "thrust");
+ *
+ *  std::cout << "The 1st value of t is " << thrust::get<0>(t) << std::endl;
+ *  \endcode
+ *
+ *  \see pair
+ *  \see tuple
+ */
+template<int N, class HT, class TT>
+__host__ __device__
+inline typename access_traits<
+                  typename tuple_element<N, detail::cons<HT, TT> >::type
+                >::non_const_type
+get(detail::cons<HT, TT>& t);
+
+
+/*! The \p get function returns a \c const reference to a \p tuple element of
+ *  interest.
+ *
+ *  \param t A reference to a \p tuple of interest.
+ *  \return A \c const reference to \p t's <tt>N</tt>th element.
+ *
+ *  \tparam N The index of the element of interest.
+ *
+ *  The following code snippet demonstrates how to use \p get to print
+ *  the value of a \p tuple element.
+ *
+ *  \code
+ *  #include <thrust/tuple.h>
+ *  #include <iostream>
+ *  ...
+ *  thrust::tuple<int, const char *> t(13, "thrust");
+ *
+ *  std::cout << "The 1st value of t is " << thrust::get<0>(t) << std::endl;
+ *  \endcode
+ *
+ *  \see pair
+ *  \see tuple
+ */
+template<int N, class HT, class TT>
+__host__ __device__
+inline typename access_traits<
+                  typename tuple_element<N, detail::cons<HT, TT> >::type
+                >::const_type
+get(const detail::cons<HT, TT>& t);
+
+
+
+/*! \p tuple is a class template that can be instantiated with up to ten arguments.
+ *  Each template argument specifies the type of element in the \p tuple.
+ *  Consequently, tuples are heterogeneous, fixed-size collections of values. An
+ *  instantiation of \p tuple with two arguments is similar to an instantiation
+ *  of \p pair with the same two arguments. Individual elements of a \p tuple may
+ *  be accessed with the \p get function.
+ *
+ *  \tparam TN The type of the <tt>N</tt> \c tuple element. Thrust's \p tuple
+ *          type currently supports up to ten elements.
+ *
+ *  The following code snippet demonstrates how to create a new \p tuple object
+ *  and inspect and modify the value of its elements.
+ *
+ *  \code
+ *  #include <thrust/tuple.h>
+ *  #include <iostream>
+ *  ...
+ *  // create a tuple containing an int, a float, and a string
+ *  thrust::tuple<int, float, const char*> t(13, 0.1f, "thrust");
+ *
+ *  // individual members are accessed with the free function get
+ *  std::cout << "The first element's value is " << thrust::get<0>(t) << std::endl; 
+ *
+ *  // or the member function get
+ *  std::cout << "The second element's value is " << t.get<1>() << std::endl;
+ *
+ *  // we can also modify elements with the same function
+ *  thrust::get<0>(t) += 10;
+ *  \endcode
+ *
+ *  \see pair
+ *  \see get
+ *  \see make_tuple
+ *  \see tuple_element
+ *  \see tuple_size
+ *  \see tie
+ */
+template <class T0, class T1, class T2, class T3, class T4,
+          class T5, class T6, class T7, class T8, class T9>
+  class tuple :
+    public detail::map_tuple_to_cons<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>::type
+{
+  /*! \cond
+   */
+
+  private:
+  typedef typename detail::map_tuple_to_cons<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>::type inherited;
+
+  /*! \endcond
+   */
+
+  public:
+  /*! \p tuple's no-argument constructor initializes each element.
+   */
+  inline __host__ __device__
+  tuple(void) {}
+
+  /*! \p tuple's one-argument constructor copy constructs the first element from the given parameter
+   *     and intializes all other elements.
+   *  \param t0 The value to assign to this \p tuple's first element.
+   */
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0)
+    : inherited(t0,
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type())) {}
+
+  /*! \p tuple's one-argument constructor copy constructs the first two elements from the given parameters
+   *     and intializes all other elements.
+   *  \param t0 The value to assign to this \p tuple's first element.
+   *  \param t1 The value to assign to this \p tuple's second element.
+   *  \note \p tuple's constructor has ten variants of this form, the rest of which are ommitted here for brevity.
+   */
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1)
+    : inherited(t0, t1,
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type())) {}
+
+  /*! \cond
+   */
+
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1,
+        typename access_traits<T2>::parameter_type t2)
+    : inherited(t0, t1, t2,
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type())) {}
+
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1,
+        typename access_traits<T2>::parameter_type t2,
+        typename access_traits<T3>::parameter_type t3)
+    : inherited(t0, t1, t2, t3,
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type())) {}
+
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1,
+        typename access_traits<T2>::parameter_type t2,
+        typename access_traits<T3>::parameter_type t3,
+        typename access_traits<T4>::parameter_type t4)
+    : inherited(t0, t1, t2, t3, t4,
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type())) {}
+
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1,
+        typename access_traits<T2>::parameter_type t2,
+        typename access_traits<T3>::parameter_type t3,
+        typename access_traits<T4>::parameter_type t4,
+        typename access_traits<T5>::parameter_type t5)
+    : inherited(t0, t1, t2, t3, t4, t5,
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type())) {}
+
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1,
+        typename access_traits<T2>::parameter_type t2,
+        typename access_traits<T3>::parameter_type t3,
+        typename access_traits<T4>::parameter_type t4,
+        typename access_traits<T5>::parameter_type t5,
+        typename access_traits<T6>::parameter_type t6)
+    : inherited(t0, t1, t2, t3, t4, t5, t6,
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type())) {}
+
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1,
+        typename access_traits<T2>::parameter_type t2,
+        typename access_traits<T3>::parameter_type t3,
+        typename access_traits<T4>::parameter_type t4,
+        typename access_traits<T5>::parameter_type t5,
+        typename access_traits<T6>::parameter_type t6,
+        typename access_traits<T7>::parameter_type t7)
+    : inherited(t0, t1, t2, t3, t4, t5, t6, t7,
+                static_cast<const null_type&>(null_type()),
+                static_cast<const null_type&>(null_type())) {}
+
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1,
+        typename access_traits<T2>::parameter_type t2,
+        typename access_traits<T3>::parameter_type t3,
+        typename access_traits<T4>::parameter_type t4,
+        typename access_traits<T5>::parameter_type t5,
+        typename access_traits<T6>::parameter_type t6,
+        typename access_traits<T7>::parameter_type t7,
+        typename access_traits<T8>::parameter_type t8)
+    : inherited(t0, t1, t2, t3, t4, t5, t6, t7, t8,
+                static_cast<const null_type&>(null_type())) {}
+
+  inline __host__ __device__ 
+  tuple(typename access_traits<T0>::parameter_type t0,
+        typename access_traits<T1>::parameter_type t1,
+        typename access_traits<T2>::parameter_type t2,
+        typename access_traits<T3>::parameter_type t3,
+        typename access_traits<T4>::parameter_type t4,
+        typename access_traits<T5>::parameter_type t5,
+        typename access_traits<T6>::parameter_type t6,
+        typename access_traits<T7>::parameter_type t7,
+        typename access_traits<T8>::parameter_type t8,
+        typename access_traits<T9>::parameter_type t9)
+    : inherited(t0, t1, t2, t3, t4, t5, t6, t7, t8, t9) {}
+
+
+  template<class U1, class U2>
+  inline __host__ __device__ 
+  tuple(const detail::cons<U1, U2>& p) : inherited(p) {}
+
+  __thrust_exec_check_disable__
+  template <class U1, class U2>
+  inline __host__ __device__ 
+  tuple& operator=(const detail::cons<U1, U2>& k)
+  {
+    inherited::operator=(k);
+    return *this;
+  }
+
+  /*! \endcond
+   */
+
+  /*! \p swap swaps the elements of two <tt>tuple</tt>s.
+   *
+   *  \param t The other <tt>tuple</tt> with which to swap.
+   */
+  inline __host__ __device__
+  void swap(tuple &t)
+  {
+    inherited::swap(t);
+  }
+};
+
+/*! \cond
+ */
+
+template <>
+class tuple<null_type, null_type, null_type, null_type, null_type, null_type, null_type, null_type, null_type, null_type>  :
+  public null_type
+{
+public:
+  typedef null_type inherited;
+};
+
+/*! \endcond
+ */
+
+
+/*! This version of \p make_tuple creates a new \c tuple object from a
+ *  single object.
+ *
+ *  \param t0 The object to copy from.
+ *  \return A \p tuple object with a single member which is a copy of \p t0.
+ */
+template<class T0>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0>::type
+    make_tuple(const T0& t0);
+
+/*! This version of \p make_tuple creates a new \c tuple object from two
+ *  objects.
+ *
+ *  \param t0 The first object to copy from.
+ *  \param t1 The second object to copy from.
+ *  \return A \p tuple object with two members which are copies of \p t0
+ *          and \p t1.
+ *
+ *  \note \p make_tuple has ten variants, the rest of which are omitted here
+ *        for brevity.
+ */
+template<class T0, class T1>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1>::type
+    make_tuple(const T0& t0, const T1& t1);
+
+/*! This version of \p tie creates a new \c tuple whose single element is
+ *  a reference which refers to this function's argument.
+ *
+ *  \param t0 The object to reference.
+ *  \return A \p tuple object with one member which is a reference to \p t0.
+ */
+template<typename T0>
+__host__ __device__ inline
+tuple<T0&> tie(T0& t0);
+
+/*! This version of \p tie creates a new \c tuple of references object which
+ *  refers to this function's arguments.
+ *
+ *  \param t0 The first object to reference.
+ *  \param t1 The second object to reference.
+ *  \return A \p tuple object with two members which are references to \p t0
+ *          and \p t1.
+ *
+ *  \note \p tie has ten variants, the rest of which are omitted here for
+ *           brevity.
+ */
+template<typename T0, typename T1>
+__host__ __device__ inline
+tuple<T0&,T1&> tie(T0& t0, T1& t1);
+
+/*! \p swap swaps the contents of two <tt>tuple</tt>s.
+ *
+ *  \param x The first \p tuple to swap.
+ *  \param y The second \p tuple to swap.
+ */
+template<
+  typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9,
+  typename U0, typename U1, typename U2, typename U3, typename U4, typename U5, typename U6, typename U7, typename U8, typename U9
+>
+inline __host__ __device__
+void swap(tuple<T0,T1,T2,T3,T4,T5,T6,T7,T8,T9> &x,
+          tuple<U0,U1,U2,U3,U4,U5,U6,U7,U8,U9> &y);
+
+
+
+/*! \cond
+ */
+
+template<class T0, class T1, class T2>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2);
+
+template<class T0, class T1, class T2, class T3>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3);
+
+template<class T0, class T1, class T2, class T3, class T4>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4);
+
+template<class T0, class T1, class T2, class T3, class T4, class T5>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5);
+
+template<class T0, class T1, class T2, class T3, class T4, class T5, class T6>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5, T6>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5, const T6& t6);
+
+template<class T0, class T1, class T2, class T3, class T4, class T5, class T6, class T7>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5, T6, T7>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5, const T6& t6, const T7& t7);
+
+template<class T0, class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5, T6, T7, T8>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5, const T6& t6, const T7& t7, const T8& t8);
+
+template<class T0, class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5, const T6& t6, const T7& t7, const T8& t8, const T9& t9);
+
+template<typename T0, typename T1, typename T2>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&> tie(T0 &t0, T1 &t1, T2 &t2);
+
+template<typename T0, typename T1, typename T2, typename T3>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3);
+
+template<typename T0, typename T1, typename T2, typename T3, typename T4>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4);
+
+template<typename T0, typename T1, typename T2, typename T3, typename T4, typename T5>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5);
+
+template<typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5, T6 &t6);
+
+template<typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5, T6 &t6, T7 &t7);
+
+template<typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&,T8&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5, T6 &t6, T7 &t7, T8 &t8);
+
+template<typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&,T8&,T9&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5, T6 &t6, T7 &t7, T8 &t8, T9 &t9);
+
+
+__host__ __device__ inline
+bool operator==(const null_type&, const null_type&);
+
+__host__ __device__ inline
+bool operator>=(const null_type&, const null_type&);
+
+__host__ __device__ inline
+bool operator<=(const null_type&, const null_type&);
+
+__host__ __device__ inline
+bool operator!=(const null_type&, const null_type&);
+
+__host__ __device__ inline
+bool operator<(const null_type&, const null_type&);
+
+__host__ __device__ inline
+bool operator>(const null_type&, const null_type&);
+
+/*! \endcond
+ */
+
+/*! \} // tuple
+ */
+
+/*! \} // utility
+ */
+
+} // end thrust
+
diff --git a/cupy/_core/include/cupy/tuple/pair.h b/cupy/_core/include/cupy/tuple/pair.h
new file mode 100644
index 0000000..09c63f2
--- /dev/null
+++ b/cupy/_core/include/cupy/tuple/pair.h
@@ -0,0 +1,228 @@
+/*
+ *  Copyright 2008-2021 NVIDIA Corporation
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#pragma once
+
+#include <cupy/tuple/type_traits.h>
+#include <cupy/pair.cuh>
+
+namespace thrust {
+
+template <typename T1, typename T2>
+  __host__ __device__
+  pair<T1,T2>
+    ::pair(void)
+      :first(),second()
+{
+  ;
+} // end pair::pair()
+
+
+template <typename T1, typename T2>
+  __host__ __device__
+  pair<T1,T2>
+    ::pair(const T1 &x, const T2 &y)
+      :first(x),second(y)
+{
+  ;
+} // end pair::pair()
+
+
+template <typename T1, typename T2>
+  template <typename U1, typename U2>
+    __host__ __device__
+    pair<T1,T2>
+      ::pair(const pair<U1,U2> &p)
+        :first(p.first),second(p.second)
+{
+  ;
+} // end pair::pair()
+
+
+// template <typename T1, typename T2>
+//   template <typename U1, typename U2>
+//     __host__ __device__
+//     pair<T1,T2>
+//       ::pair(const std::pair<U1,U2> &p)
+//         :first(p.first),second(p.second)
+// {
+//   ;
+// } // end pair::pair()
+
+
+template<typename T1, typename T2>
+  inline __host__ __device__
+    void pair<T1,T2>
+      ::swap(thrust::pair<T1,T2> &p)
+{
+  using thrust::swap;
+
+  swap(first, p.first);
+  swap(second, p.second);
+} // end pair::swap()
+
+
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator==(const pair<T1,T2> &x, const pair<T1,T2> &y)
+{
+  return x.first == y.first && x.second == y.second;
+} // end operator==()
+
+
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator<(const pair<T1,T2> &x, const pair<T1,T2> &y)
+{
+  return x.first < y.first || (!(y.first < x.first) && x.second < y.second);
+} // end operator<()
+
+
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator!=(const pair<T1,T2> &x, const pair<T1,T2> &y)
+{
+  return !(x == y);
+} // end operator==()
+
+
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator>(const pair<T1,T2> &x, const pair<T1,T2> &y)
+{
+  return y < x;
+} // end operator<()
+
+
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator<=(const pair<T1,T2> &x, const pair<T1,T2> &y)
+{
+  return !(y < x);
+} // end operator<=()
+
+
+template <typename T1, typename T2>
+  inline __host__ __device__
+    bool operator>=(const pair<T1,T2> &x, const pair<T1,T2> &y)
+{
+  return !(x < y);
+} // end operator>=()
+
+
+template <typename T1, typename T2>
+  inline __host__ __device__
+    void swap(pair<T1,T2> &x, pair<T1,T2> &y)
+{
+  return x.swap(y);
+} // end swap()
+
+
+template <typename T1, typename T2>
+  inline __host__ __device__
+    pair<T1,T2> make_pair(T1 x, T2 y)
+{
+  return pair<T1,T2>(x,y);
+} // end make_pair()
+
+
+// specializations of tuple_element for pair
+template<typename T1, typename T2>
+  struct tuple_element<0, pair<T1,T2>>
+{
+  typedef T1 type;
+}; // end tuple_element
+
+template<typename T1, typename T2>
+  struct tuple_element<1, pair<T1,T2>>
+{
+  typedef T2 type;
+}; // end tuple_element
+
+
+// specialization of tuple_size for pair
+template<typename T1, typename T2>
+  struct tuple_size<pair<T1,T2>>
+{
+  static const unsigned int value = 2;
+}; // end tuple_size
+
+
+
+namespace detail
+{
+
+
+template<int N, typename Pair> struct pair_get {};
+
+template<typename Pair>
+  struct pair_get<0, Pair>
+{
+  inline __host__ __device__
+    const typename tuple_element<0, Pair>::type &
+      operator()(const Pair &p) const
+  {
+    return p.first;
+  } // end operator()()
+
+  inline __host__ __device__
+    typename tuple_element<0, Pair>::type &
+      operator()(Pair &p) const
+  {
+    return p.first;
+  } // end operator()()
+}; // end pair_get
+
+
+template<typename Pair>
+  struct pair_get<1, Pair>
+{
+  inline __host__ __device__
+    const typename tuple_element<1, Pair>::type &
+      operator()(const Pair &p) const
+  {
+    return p.second;
+  } // end operator()()
+
+  inline __host__ __device__
+    typename tuple_element<1, Pair>::type &
+      operator()(Pair &p) const
+  {
+    return p.second;
+  } // end operator()()
+}; // end pair_get
+
+} // end detail
+
+
+
+template<unsigned int N, typename T1, typename T2>
+  inline __host__ __device__
+    typename tuple_element<N, pair<T1,T2> >::type &
+      get(pair<T1,T2> &p)
+{
+  return detail::pair_get<N, pair<T1,T2> >()(p);
+} // end get()
+
+template<unsigned int N, typename T1, typename T2>
+  inline __host__ __device__
+    const typename tuple_element<N, pair<T1,T2> >::type &
+      get(const pair<T1,T2> &p)
+{
+  return detail::pair_get<N, pair<T1,T2> >()(p);
+} // end get()
+
+}  // namespace thrust
diff --git a/cupy/_core/include/cupy/tuple/tuple.h b/cupy/_core/include/cupy/tuple/tuple.h
new file mode 100644
index 0000000..0a481fc
--- /dev/null
+++ b/cupy/_core/include/cupy/tuple/tuple.h
@@ -0,0 +1,960 @@
+/*
+ *  Copyright 2008-2018 NVIDIA Corporation
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <cupy/tuple/type_traits.h>
+
+#define __thrust_exec_check_disable__
+
+namespace thrust
+{
+
+// define null_type
+struct null_type {};
+
+// null_type comparisons
+__host__ __device__ inline
+bool operator==(const null_type&, const null_type&) { return true; }
+
+__host__ __device__ inline
+bool operator>=(const null_type&, const null_type&) { return true; }
+
+__host__ __device__ inline
+bool operator<=(const null_type&, const null_type&) { return true; }
+
+__host__ __device__ inline
+bool operator!=(const null_type&, const null_type&) { return false; }
+
+__host__ __device__ inline
+bool operator<(const null_type&, const null_type&) { return false; }
+
+__host__ __device__ inline
+bool operator>(const null_type&, const null_type&) { return false; }
+
+// forward declaration for tuple
+template <
+  class T0 = null_type, class T1 = null_type, class T2 = null_type,
+  class T3 = null_type, class T4 = null_type, class T5 = null_type,
+  class T6 = null_type, class T7 = null_type, class T8 = null_type,
+  class T9 = null_type>
+class tuple;
+
+// forward declaration of tuple_element
+template<size_t N, class T> struct tuple_element;
+
+// specializations for tuple_element
+template<class T>
+  struct tuple_element<0,T>
+{
+  typedef typename T::head_type type;
+}; // end tuple_element<0,T>
+
+template<size_t N, class T>
+  struct tuple_element<N, const T>
+{
+  private:
+    typedef typename T::tail_type Next;
+    typedef typename tuple_element<N-1, Next>::type unqualified_type;
+
+  public:
+    typedef typename thrust::detail::add_const<unqualified_type>::type type;
+}; // end tuple_element<N, const T>
+
+template<class T>
+  struct tuple_element<0,const T>
+{
+  typedef typename thrust::detail::add_const<typename T::head_type>::type type;
+}; // end tuple_element<0,const T>
+
+
+
+// forward declaration of tuple_size
+template<class T> struct tuple_size;
+
+// specializations for tuple_size
+template<>
+  struct tuple_size< tuple<> >
+{
+  static const int value = 0;
+}; // end tuple_size< tuple<> >
+
+template<>
+  struct tuple_size<null_type>
+{
+  static const int value = 0;
+}; // end tuple_size<null_type>
+
+
+
+// forward declaration of detail::cons
+namespace detail
+{
+
+template <class HT, class TT> struct cons;
+
+} // end detail
+
+
+// -- some traits classes for get functions
+template <class T> struct access_traits
+{
+  typedef const T& const_type;
+  typedef T& non_const_type;
+
+  typedef const typename thrust::detail::remove_cv<T>::type& parameter_type;
+
+// used as the tuple constructors parameter types
+// Rationale: non-reference tuple element types can be cv-qualified.
+// It should be possible to initialize such types with temporaries,
+// and when binding temporaries to references, the reference must
+// be non-volatile and const. 8.5.3. (5)
+}; // end access_traits
+
+template <class T> struct access_traits<T&>
+{
+  typedef T& const_type;
+  typedef T& non_const_type;
+
+  typedef T& parameter_type;
+}; // end access_traits<T&>
+
+// forward declarations of get()
+template<int N, class HT, class TT>
+__host__ __device__
+inline typename access_traits<
+                  typename tuple_element<N, detail::cons<HT, TT> >::type
+                >::non_const_type
+// XXX we probably don't need to do this for any compiler we care about -jph
+//get(cons<HT, TT>& c BOOST_APPEND_EXPLICIT_TEMPLATE_NON_TYPE(int, N));
+get(detail::cons<HT, TT>& c);
+
+template<int N, class HT, class TT>
+__host__ __device__
+inline typename access_traits<
+                  typename tuple_element<N, detail::cons<HT, TT> >::type
+                >::const_type
+// XXX we probably don't need to do this for any compiler we care about -jph
+//get(const cons<HT, TT>& c BOOST_APPEND_EXPLICIT_TEMPLATE_NON_TYPE(int, N));
+get(const detail::cons<HT, TT>& c);
+
+namespace detail
+{
+
+// -- generate error template, referencing to non-existing members of this
+// template is used to produce compilation errors intentionally
+template<class T>
+class generate_error;
+
+// - cons getters --------------------------------------------------------
+// called: get_class<N>::get<RETURN_TYPE>(aTuple)
+
+template< int N >
+struct get_class
+{
+  template<class RET, class HT, class TT >
+  __host__ __device__
+  inline static RET get(const cons<HT, TT>& t)
+  {
+    // XXX we may not need to deal with this for any compiler we care about -jph
+    //return get_class<N-1>::BOOST_NESTED_TEMPLATE get<RET>(t.tail);
+    return get_class<N-1>::template get<RET>(t.tail);
+    
+    // gcc 4.3 couldn't compile this:
+    //return get_class<N-1>::get<RET>(t.tail);
+  }
+
+  template<class RET, class HT, class TT >
+  __host__ __device__
+  inline static RET get(cons<HT, TT>& t)
+  {
+    // XXX we may not need to deal with this for any compiler we care about -jph
+    //return get_class<N-1>::BOOST_NESTED_TEMPLATE get<RET>(t.tail);
+    return get_class<N-1>::template get<RET>(t.tail);
+
+    // gcc 4.3 couldn't compile this:
+    //return get_class<N-1>::get<RET>(t.tail);
+  }
+}; // end get_class
+
+template<>
+struct get_class<0>
+{
+  template<class RET, class HT, class TT>
+  __host__ __device__
+  inline static RET get(const cons<HT, TT>& t)
+  {
+    return t.head;
+  }
+
+  template<class RET, class HT, class TT>
+  __host__ __device__
+  inline static RET get(cons<HT, TT>& t)
+  {
+    return t.head;
+  }
+}; // get get_class<0>
+
+
+template <bool If, class Then, class Else> struct IF
+{
+  typedef Then RET;
+};
+
+template <class Then, class Else> struct IF<false, Then, Else>
+{
+  typedef Else RET;
+};
+
+//  These helper templates wrap void types and plain function types.
+//  The rationale is to allow one to write tuple types with those types
+//  as elements, even though it is not possible to instantiate such object.
+//  E.g: typedef tuple<void> some_type; // ok
+//  but: some_type x; // fails
+
+template <class T> class non_storeable_type
+{
+  __host__ __device__
+  non_storeable_type();
+};
+
+template <class T> struct wrap_non_storeable_type
+{
+  // XXX is_function looks complicated; punt for now -jph
+  //typedef typename IF<
+  //  ::thrust::detail::is_function<T>::value, non_storeable_type<T>, T
+  //>::RET type;
+
+  typedef T type;
+};
+
+template <> struct wrap_non_storeable_type<void>
+{
+  typedef non_storeable_type<void> type;
+};
+
+
+template <class HT, class TT>
+  struct cons
+{
+  typedef HT head_type;
+  typedef TT tail_type;
+
+  typedef typename
+    wrap_non_storeable_type<head_type>::type stored_head_type;
+
+  stored_head_type head;
+  tail_type tail;
+
+  inline __host__ __device__
+  typename access_traits<stored_head_type>::non_const_type
+  get_head() { return head; }
+
+  inline __host__ __device__
+  typename access_traits<tail_type>::non_const_type
+  get_tail() { return tail; }
+
+  inline __host__ __device__
+  typename access_traits<stored_head_type>::const_type
+  get_head() const { return head; }
+
+  inline __host__ __device__
+  typename access_traits<tail_type>::const_type
+  get_tail() const { return tail; }
+
+  inline __host__ __device__
+  cons(void) : head(), tail() {}
+  //  cons() : head(detail::default_arg<HT>::f()), tail() {}
+
+  // the argument for head is not strictly needed, but it prevents
+  // array type elements. This is good, since array type elements
+  // cannot be supported properly in any case (no assignment,
+  // copy works only if the tails are exactly the same type, ...)
+
+  inline __host__ __device__
+  cons(typename access_traits<stored_head_type>::parameter_type h,
+       const tail_type& t)
+    : head (h), tail(t) {}
+
+  template <class T1, class T2, class T3, class T4, class T5,
+            class T6, class T7, class T8, class T9, class T10>
+  inline __host__ __device__
+  cons( T1& t1, T2& t2, T3& t3, T4& t4, T5& t5,
+        T6& t6, T7& t7, T8& t8, T9& t9, T10& t10 )
+    : head (t1),
+      tail (t2, t3, t4, t5, t6, t7, t8, t9, t10, static_cast<const null_type&>(null_type()))
+      {}
+
+  template <class T2, class T3, class T4, class T5,
+            class T6, class T7, class T8, class T9, class T10>
+  inline __host__ __device__
+  cons( const null_type& /*t1*/, T2& t2, T3& t3, T4& t4, T5& t5,
+        T6& t6, T7& t7, T8& t8, T9& t9, T10& t10 )
+    : head (),
+      tail (t2, t3, t4, t5, t6, t7, t8, t9, t10, static_cast<const null_type&>(null_type()))
+      {}
+
+
+  template <class HT2, class TT2>
+  inline __host__ __device__
+  cons( const cons<HT2, TT2>& u ) : head(u.head), tail(u.tail) {}
+
+#if THRUST_CPP_DIALECT >= 2011
+  cons(const cons &) = default;
+#endif
+
+  __thrust_exec_check_disable__
+  template <class HT2, class TT2>
+  inline __host__ __device__
+  cons& operator=( const cons<HT2, TT2>& u ) {
+    head=u.head; tail=u.tail; return *this;
+  }
+
+  // must define assignment operator explicitly, implicit version is
+  // illformed if HT is a reference (12.8. (12))
+  __thrust_exec_check_disable__
+  inline __host__ __device__
+  cons& operator=(const cons& u) {
+    head = u.head; tail = u.tail;  return *this;
+  }
+
+  // XXX enable when we support std::pair -jph
+  //template <class T1, class T2>
+  //__host__ __device__
+  //cons& operator=( const std::pair<T1, T2>& u ) {
+  //  //BOOST_STATIC_ASSERT(length<cons>::value == 2); // check length = 2
+  //  head = u.first; tail.head = u.second; return *this;
+  //}
+
+  // get member functions (non-const and const)
+  template <int N>
+  __host__ __device__
+  typename access_traits<
+             typename tuple_element<N, cons<HT, TT> >::type
+           >::non_const_type
+  get() {
+    return thrust::get<N>(*this); // delegate to non-member get
+  }
+
+  template <int N>
+  __host__ __device__
+  typename access_traits<
+             typename tuple_element<N, cons<HT, TT> >::type
+           >::const_type
+  get() const {
+    return thrust::get<N>(*this); // delegate to non-member get
+  }
+
+  inline __host__ __device__
+  void swap(cons &c)
+  {
+    using thrust::swap;
+
+    swap(head, c.head);
+    tail.swap(c.tail);
+  }
+};
+
+template <class HT>
+  struct cons<HT, null_type>
+{
+  typedef HT head_type;
+  typedef null_type tail_type;
+  typedef cons<HT, null_type> self_type;
+
+  typedef typename
+    wrap_non_storeable_type<head_type>::type stored_head_type;
+  stored_head_type head;
+
+  typename access_traits<stored_head_type>::non_const_type
+  inline __host__ __device__
+  get_head() { return head; }
+
+  inline __host__ __device__
+  null_type get_tail() { return null_type(); }
+
+  inline __host__ __device__
+  typename access_traits<stored_head_type>::const_type
+  get_head() const { return head; }
+
+  inline __host__ __device__
+  null_type get_tail() const { return null_type(); }
+
+  inline __host__ __device__
+  cons() : head() {}
+
+  inline __host__ __device__
+  cons(typename access_traits<stored_head_type>::parameter_type h,
+       const null_type& = null_type())
+    : head (h) {}
+
+  template<class T1>
+  inline __host__ __device__
+  cons(T1& t1, const null_type&, const null_type&, const null_type&,
+       const null_type&, const null_type&, const null_type&,
+       const null_type&, const null_type&, const null_type&)
+  : head (t1) {}
+
+  inline __host__ __device__
+  cons(const null_type&,
+       const null_type&, const null_type&, const null_type&,
+       const null_type&, const null_type&, const null_type&,
+       const null_type&, const null_type&, const null_type&)
+  : head () {}
+
+  template <class HT2>
+  inline __host__ __device__
+  cons( const cons<HT2, null_type>& u ) : head(u.head) {}
+
+#if THRUST_CPP_DIALECT >= 2011
+  cons(const cons &) = default;
+#endif
+
+  __thrust_exec_check_disable__
+  template <class HT2>
+  inline __host__ __device__
+  cons& operator=(const cons<HT2, null_type>& u )
+  {
+    head = u.head;
+    return *this;
+  }
+
+  // must define assignment operator explicitly, implicit version
+  // is illformed if HT is a reference
+  inline __host__ __device__
+  cons& operator=(const cons& u) { head = u.head; return *this; }
+
+  template <int N>
+  inline __host__ __device__
+  typename access_traits<
+             typename tuple_element<N, self_type>::type
+            >::non_const_type
+  // XXX we probably don't need this for the compilers we care about -jph
+  //get(BOOST_EXPLICIT_TEMPLATE_NON_TYPE(int, N))
+  get(void)
+  {
+    return thrust::get<N>(*this);
+  }
+
+  template <int N>
+  inline __host__ __device__
+  typename access_traits<
+             typename tuple_element<N, self_type>::type
+           >::const_type
+  // XXX we probably don't need this for the compilers we care about -jph
+  //get(BOOST_EXPLICIT_TEMPLATE_NON_TYPE(int, N)) const
+  get(void) const
+  {
+    return thrust::get<N>(*this);
+  }
+
+  inline __host__ __device__
+  void swap(cons &c)
+  {
+    using thrust::swap;
+
+    swap(head, c.head);
+  }
+}; // end cons
+
+template <class T0, class T1, class T2, class T3, class T4,
+          class T5, class T6, class T7, class T8, class T9>
+  struct map_tuple_to_cons
+{
+  typedef cons<T0,
+               typename map_tuple_to_cons<T1, T2, T3, T4, T5,
+                                          T6, T7, T8, T9, null_type>::type
+              > type;
+}; // end map_tuple_to_cons
+
+// The empty tuple is a null_type
+template <>
+  struct map_tuple_to_cons<null_type, null_type, null_type, null_type, null_type, null_type, null_type, null_type, null_type, null_type>
+{
+  typedef null_type type;
+}; // end map_tuple_to_cons<...>
+
+
+
+// ---------------------------------------------------------------------------
+// The call_traits for make_tuple
+
+// Must be instantiated with plain or const plain types (not with references)
+
+// from template<class T> foo(const T& t) : make_tuple_traits<const T>::type
+// from template<class T> foo(T& t) : make_tuple_traits<T>::type
+
+// Conversions:
+// T -> T,
+// references -> compile_time_error
+// array -> const ref array
+
+
+template<class T>
+struct make_tuple_traits {
+  typedef T type;
+
+  // commented away, see below  (JJ)
+  //  typedef typename IF<
+  //  boost::is_function<T>::value,
+  //  T&,
+  //  T>::RET type;
+
+};
+
+// The is_function test was there originally for plain function types,
+// which can't be stored as such (we must either store them as references or
+// pointers). Such a type could be formed if make_tuple was called with a
+// reference to a function.
+// But this would mean that a const qualified function type was formed in
+// the make_tuple function and hence make_tuple can't take a function
+// reference as a parameter, and thus T can't be a function type.
+// So is_function test was removed.
+// (14.8.3. says that type deduction fails if a cv-qualified function type
+// is created. (It only applies for the case of explicitly specifying template
+// args, though?)) (JJ)
+
+template<class T>
+struct make_tuple_traits<T&> {
+  typedef typename
+     detail::generate_error<T&>::
+       do_not_use_with_reference_type error;
+};
+
+// Arrays can't be stored as plain types; convert them to references.
+// All arrays are converted to const. This is because make_tuple takes its
+// parameters as const T& and thus the knowledge of the potential
+// non-constness of actual argument is lost.
+template<class T, int n>  struct make_tuple_traits <T[n]> {
+  typedef const T (&type)[n];
+};
+
+template<class T, int n>
+struct make_tuple_traits<const T[n]> {
+  typedef const T (&type)[n];
+};
+
+template<class T, int n>  struct make_tuple_traits<volatile T[n]> {
+  typedef const volatile T (&type)[n];
+};
+
+template<class T, int n>
+struct make_tuple_traits<const volatile T[n]> {
+  typedef const volatile T (&type)[n];
+};
+
+// XXX enable these if we ever care about reference_wrapper -jph
+//template<class T>
+//struct make_tuple_traits<reference_wrapper<T> >{
+//  typedef T& type;
+//};
+//
+//template<class T>
+//struct make_tuple_traits<const reference_wrapper<T> >{
+//  typedef T& type;
+//};
+
+
+// a helper traits to make the make_tuple functions shorter (Vesa Karvonen's
+// suggestion)
+template <
+  class T0 = null_type, class T1 = null_type, class T2 = null_type,
+  class T3 = null_type, class T4 = null_type, class T5 = null_type,
+  class T6 = null_type, class T7 = null_type, class T8 = null_type,
+  class T9 = null_type
+>
+struct make_tuple_mapper {
+  typedef
+    tuple<typename make_tuple_traits<T0>::type,
+          typename make_tuple_traits<T1>::type,
+          typename make_tuple_traits<T2>::type,
+          typename make_tuple_traits<T3>::type,
+          typename make_tuple_traits<T4>::type,
+          typename make_tuple_traits<T5>::type,
+          typename make_tuple_traits<T6>::type,
+          typename make_tuple_traits<T7>::type,
+          typename make_tuple_traits<T8>::type,
+          typename make_tuple_traits<T9>::type> type;
+};
+
+} // end detail
+
+
+template<int N, class HT, class TT>
+__host__ __device__
+inline typename access_traits<
+                  typename tuple_element<N, detail::cons<HT, TT> >::type
+                >::non_const_type
+get(detail::cons<HT, TT>& c)
+{
+  //return detail::get_class<N>::BOOST_NESTED_TEMPLATE
+  
+  // gcc 4.3 couldn't compile this:
+  //return detail::get_class<N>::
+
+  return detail::get_class<N>::template
+         get<
+           typename access_traits<
+             typename tuple_element<N, detail::cons<HT, TT> >::type
+           >::non_const_type,
+           HT,TT
+         >(c);
+}
+
+
+// get function for const cons-lists, returns a const reference to
+// the element. If the element is a reference, returns the reference
+// as such (that is, can return a non-const reference)
+template<int N, class HT, class TT>
+__host__ __device__
+inline typename access_traits<
+                  typename tuple_element<N, detail::cons<HT, TT> >::type
+                >::const_type
+get(const detail::cons<HT, TT>& c)
+{
+  //return detail::get_class<N>::BOOST_NESTED_TEMPLATE
+
+  // gcc 4.3 couldn't compile this:
+  //return detail::get_class<N>::
+
+  return detail::get_class<N>::template
+         get<
+           typename access_traits<
+             typename tuple_element<N, detail::cons<HT, TT> >::type
+           >::const_type,
+           HT,TT
+         >(c);
+}
+
+
+template<class T0>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0>::type
+    make_tuple(const T0& t0)
+{
+  typedef typename detail::make_tuple_mapper<T0>::type t;
+  return t(t0);
+} // end make_tuple()
+
+template<class T0, class T1>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1>::type
+    make_tuple(const T0& t0, const T1& t1)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1>::type t;
+  return t(t0,t1);
+} // end make_tuple()
+
+template<class T0, class T1, class T2>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1,T2>::type t;
+  return t(t0,t1,t2);
+} // end make_tuple()
+
+template<class T0, class T1, class T2, class T3>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1,T2,T3>::type t;
+  return t(t0,t1,t2,t3);
+} // end make_tuple()
+
+template<class T0, class T1, class T2, class T3, class T4>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1,T2,T3,T4>::type t;
+  return t(t0,t1,t2,t3,t4);
+} // end make_tuple()
+
+template<class T0, class T1, class T2, class T3, class T4, class T5>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1,T2,T3,T4,T5>::type t;
+  return t(t0,t1,t2,t3,t4,t5);
+} // end make_tuple()
+
+template<class T0, class T1, class T2, class T3, class T4, class T5, class T6>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5, T6>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5, const T6& t6)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1,T2,T3,T4,T5,T6>::type t;
+  return t(t0,t1,t2,t3,t4,t5,t6);
+} // end make_tuple()
+
+template<class T0, class T1, class T2, class T3, class T4, class T5, class T6, class T7>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5, T6, T7>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5, const T6& t6, const T7& t7)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1,T2,T3,T4,T5,T6,T7>::type t;
+  return t(t0,t1,t2,t3,t4,t5,t6,t7);
+} // end make_tuple()
+
+template<class T0, class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5, T6, T7, T8>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5, const T6& t6, const T7& t7, const T8& t8)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1,T2,T3,T4,T5,T6,T7,T8>::type t;
+  return t(t0,t1,t2,t3,t4,t5,t6,t7,t8);
+} // end make_tuple()
+
+template<class T0, class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9>
+__host__ __device__ inline
+  typename detail::make_tuple_mapper<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>::type
+    make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3, const T4& t4, const T5& t5, const T6& t6, const T7& t7, const T8& t8, const T9& t9)
+{
+  typedef typename detail::make_tuple_mapper<T0,T1,T2,T3,T4,T5,T6,T7,T8,T9>::type t;
+  return t(t0,t1,t2,t3,t4,t5,t6,t7,t8,t9);
+} // end make_tuple()
+
+
+template<typename T0>
+__host__ __device__ inline
+tuple<T0&> tie(T0 &t0)
+{
+  return tuple<T0&>(t0);
+}
+
+template<typename T0,typename T1>
+__host__ __device__ inline
+tuple<T0&,T1&> tie(T0 &t0, T1 &t1)
+{
+  return tuple<T0&,T1&>(t0,t1);
+}
+
+template<typename T0,typename T1, typename T2>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&> tie(T0 &t0, T1 &t1, T2 &t2)
+{
+  return tuple<T0&,T1&,T2&>(t0,t1,t2);
+}
+
+template<typename T0,typename T1, typename T2, typename T3>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3)
+{
+  return tuple<T0&,T1&,T2&,T3&>(t0,t1,t2,t3);
+}
+
+template<typename T0,typename T1, typename T2, typename T3, typename T4>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4)
+{
+  return tuple<T0&,T1&,T2&,T3&,T4&>(t0,t1,t2,t3,t4);
+}
+
+template<typename T0,typename T1, typename T2, typename T3, typename T4, typename T5>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5)
+{
+  return tuple<T0&,T1&,T2&,T3&,T4&,T5&>(t0,t1,t2,t3,t4,t5);
+}
+
+template<typename T0,typename T1, typename T2, typename T3, typename T4, typename T5, typename T6>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5, T6 &t6)
+{
+  return tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&>(t0,t1,t2,t3,t4,t5,t6);
+}
+
+template<typename T0,typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5, T6 &t6, T7 &t7)
+{
+  return tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&>(t0,t1,t2,t3,t4,t5,t6,t7);
+}
+
+template<typename T0,typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&,T8&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5, T6 &t6, T7 &t7, T8 &t8)
+{
+  return tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&,T8&>(t0,t1,t2,t3,t4,t5,t6,t7,t8);
+}
+
+template<typename T0,typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9>
+__host__ __device__ inline
+tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&,T8&,T9&> tie(T0 &t0, T1 &t1, T2 &t2, T3 &t3, T4 &t4, T5 &t5, T6 &t6, T7 &t7, T8 &t8, T9 &t9)
+{
+  return tuple<T0&,T1&,T2&,T3&,T4&,T5&,T6&,T7&,T8&,T9&>(t0,t1,t2,t3,t4,t5,t6,t7,t8,t9);
+}
+
+template<
+  typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9,
+  typename U0, typename U1, typename U2, typename U3, typename U4, typename U5, typename U6, typename U7, typename U8, typename U9
+>
+__host__ __device__ inline
+void swap(thrust::tuple<T0,T1,T2,T3,T4,T5,T6,T7,T8,T9> &x,
+          thrust::tuple<U0,U1,U2,U3,U4,U5,U6,U7,U8,U9> &y)
+{
+  return x.swap(y);
+}
+
+
+
+namespace detail
+{
+
+template<class T1, class T2>
+__host__ __device__
+inline bool eq(const T1& lhs, const T2& rhs) {
+  return lhs.get_head() == rhs.get_head() &&
+         eq(lhs.get_tail(), rhs.get_tail());
+}
+template<>
+__host__ __device__
+inline bool eq<null_type,null_type>(const null_type&, const null_type&) { return true; }
+
+template<class T1, class T2>
+__host__ __device__
+inline bool neq(const T1& lhs, const T2& rhs) {
+  return lhs.get_head() != rhs.get_head()  ||
+         neq(lhs.get_tail(), rhs.get_tail());
+}
+template<>
+__host__ __device__
+inline bool neq<null_type,null_type>(const null_type&, const null_type&) { return false; }
+
+template<class T1, class T2>
+__host__ __device__
+inline bool lt(const T1& lhs, const T2& rhs) {
+  return (lhs.get_head() < rhs.get_head())  ||
+            (!(rhs.get_head() < lhs.get_head()) &&
+             lt(lhs.get_tail(), rhs.get_tail()));
+}
+template<>
+__host__ __device__
+inline bool lt<null_type,null_type>(const null_type&, const null_type&) { return false; }
+
+template<class T1, class T2>
+__host__ __device__
+inline bool gt(const T1& lhs, const T2& rhs) {
+  return (lhs.get_head() > rhs.get_head())  ||
+            (!(rhs.get_head() > lhs.get_head()) &&
+             gt(lhs.get_tail(), rhs.get_tail()));
+}
+template<>
+__host__ __device__
+inline bool gt<null_type,null_type>(const null_type&, const null_type&) { return false; }
+
+template<class T1, class T2>
+__host__ __device__
+inline bool lte(const T1& lhs, const T2& rhs) {
+  return lhs.get_head() <= rhs.get_head()  &&
+          ( !(rhs.get_head() <= lhs.get_head()) ||
+            lte(lhs.get_tail(), rhs.get_tail()));
+}
+template<>
+__host__ __device__
+inline bool lte<null_type,null_type>(const null_type&, const null_type&) { return true; }
+
+template<class T1, class T2>
+__host__ __device__
+inline bool gte(const T1& lhs, const T2& rhs) {
+  return lhs.get_head() >= rhs.get_head()  &&
+          ( !(rhs.get_head() >= lhs.get_head()) ||
+            gte(lhs.get_tail(), rhs.get_tail()));
+}
+template<>
+__host__ __device__
+inline bool gte<null_type,null_type>(const null_type&, const null_type&) { return true; }
+
+} // end detail
+
+
+
+// equal ----
+
+template<class T1, class T2, class S1, class S2>
+__host__ __device__
+inline bool operator==(const detail::cons<T1, T2>& lhs, const detail::cons<S1, S2>& rhs)
+{
+  // XXX support this eventually -jph
+  //// check that tuple lengths are equal
+  //BOOST_STATIC_ASSERT(tuple_size<T2>::value == tuple_size<S2>::value);
+
+  return  detail::eq(lhs, rhs);
+} // end operator==()
+
+// not equal -----
+
+template<class T1, class T2, class S1, class S2>
+__host__ __device__
+inline bool operator!=(const detail::cons<T1, T2>& lhs, const detail::cons<S1, S2>& rhs)
+{
+  // XXX support this eventually -jph
+  //// check that tuple lengths are equal
+  //BOOST_STATIC_ASSERT(tuple_size<T2>::value == tuple_size<S2>::value);
+
+  return detail::neq(lhs, rhs);
+} // end operator!=()
+
+// <
+template<class T1, class T2, class S1, class S2>
+__host__ __device__
+inline bool operator<(const detail::cons<T1, T2>& lhs, const detail::cons<S1, S2>& rhs)
+{
+  // XXX support this eventually -jph
+  //// check that tuple lengths are equal
+  //BOOST_STATIC_ASSERT(tuple_size<T2>::value == tuple_size<S2>::value);
+
+  return detail::lt(lhs, rhs);
+} // end operator<()
+
+// >
+template<class T1, class T2, class S1, class S2>
+__host__ __device__
+inline bool operator>(const detail::cons<T1, T2>& lhs, const detail::cons<S1, S2>& rhs)
+{
+  // XXX support this eventually -jph
+  //// check that tuple lengths are equal
+  //BOOST_STATIC_ASSERT(tuple_size<T2>::value == tuple_size<S2>::value);
+
+  return detail::gt(lhs, rhs);
+} // end operator>()
+
+// <=
+template<class T1, class T2, class S1, class S2>
+__host__ __device__
+inline bool operator<=(const detail::cons<T1, T2>& lhs, const detail::cons<S1, S2>& rhs)
+{
+  // XXX support this eventually -jph
+  //// check that tuple lengths are equal
+  //BOOST_STATIC_ASSERT(tuple_size<T2>::value == tuple_size<S2>::value);
+
+  return detail::lte(lhs, rhs);
+} // end operator<=()
+
+// >=
+template<class T1, class T2, class S1, class S2>
+__host__ __device__
+inline bool operator>=(const detail::cons<T1, T2>& lhs, const detail::cons<S1, S2>& rhs)
+{
+  // XXX support this eventually -jph
+  //// check that tuple lengths are equal
+  //BOOST_STATIC_ASSERT(tuple_size<T2>::value == tuple_size<S2>::value);
+
+  return detail::gte(lhs, rhs);
+} // end operator>=()
+
+} // end thrust
diff --git a/cupy/_core/include/cupy/tuple/type_traits.h b/cupy/_core/include/cupy/tuple/type_traits.h
new file mode 100644
index 0000000..37eb099
--- /dev/null
+++ b/cupy/_core/include/cupy/tuple/type_traits.h
@@ -0,0 +1,70 @@
+/*
+ *  Copyright 2008-2018 NVIDIA Corporation
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*! \file type_traits.h
+ *  \brief Temporarily define some type traits
+ *         until nvcc can compile tr1::type_traits.
+ */
+
+#pragma once
+
+namespace thrust
+{
+
+namespace detail
+{
+ /// helper classes [4.3].
+
+template<typename T>
+  struct add_const
+{
+  typedef T const type;
+}; // end add_const
+
+template<typename T>
+  struct remove_const
+{
+  typedef T type;
+}; // end remove_const
+
+template<typename T>
+  struct remove_const<const T>
+{
+  typedef T type;
+}; // end remove_const
+
+template<typename T>
+  struct remove_volatile
+{
+  typedef T type;
+}; // end remove_volatile
+
+template<typename T>
+  struct remove_volatile<volatile T>
+{
+  typedef T type;
+}; // end remove_volatile
+
+template<typename T>
+  struct remove_cv
+{
+  typedef typename remove_const<typename remove_volatile<T>::type>::type type;
+}; // end remove_cv
+
+} // end detail
+
+} // end thrust
diff --git a/cupy/_core/include/cupy/type_dispatcher.cuh b/cupy/_core/include/cupy/type_dispatcher.cuh
new file mode 100644
index 0000000..8bfeb4a
--- /dev/null
+++ b/cupy/_core/include/cupy/type_dispatcher.cuh
@@ -0,0 +1,63 @@
+#ifndef INCLUDE_GUARD_CUPY_TYPE_DISPATCHER_H
+#define INCLUDE_GUARD_CUPY_TYPE_DISPATCHER_H
+
+#include <cupy/complex.cuh>
+#include <stdexcept>
+#include <cstdint>
+#if (__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+    && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))
+#include <cuda_fp16.h>
+#elif (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+#include <hip/hip_fp16.h>
+#endif
+
+
+#define CUPY_TYPE_INT8        0
+#define CUPY_TYPE_UINT8       1
+#define CUPY_TYPE_INT16       2
+#define CUPY_TYPE_UINT16      3
+#define CUPY_TYPE_INT32       4
+#define CUPY_TYPE_UINT32      5
+#define CUPY_TYPE_INT64       6
+#define CUPY_TYPE_UINT64      7
+#define CUPY_TYPE_FLOAT16     8
+#define CUPY_TYPE_FLOAT32     9
+#define CUPY_TYPE_FLOAT64    10
+#define CUPY_TYPE_COMPLEX64  11
+#define CUPY_TYPE_COMPLEX128 12
+#define CUPY_TYPE_BOOL       13
+
+
+//
+// **** dtype_dispatcher ****
+//
+// This is implemented with reference to the following implementation.
+// https://github.com/rapidsai/cudf/blob/branch-0.6/cpp/src/utilities/type_dispatcher.hpp
+//
+template <class functor_t, typename... Ts>
+void dtype_dispatcher(int dtype_id, functor_t f, Ts&&... args)
+{
+    switch (dtype_id) {
+    case CUPY_TYPE_INT8:       return f.template operator()<char>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_INT16:      return f.template operator()<short>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_INT32:      return f.template operator()<int>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_INT64:      return f.template operator()<int64_t>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_UINT8:      return f.template operator()<unsigned char>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_UINT16:     return f.template operator()<unsigned short>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_UINT32:     return f.template operator()<unsigned int>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_UINT64:     return f.template operator()<uint64_t>(std::forward<Ts>(args)...);
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+     && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+    case CUPY_TYPE_FLOAT16:    return f.template operator()<__half>(std::forward<Ts>(args)...);
+#endif
+    case CUPY_TYPE_FLOAT32:    return f.template operator()<float>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_FLOAT64:    return f.template operator()<double>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_COMPLEX64:  return f.template operator()<complex<float>>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_COMPLEX128: return f.template operator()<complex<double>>(std::forward<Ts>(args)...);
+    case CUPY_TYPE_BOOL:       return f.template operator()<bool>(std::forward<Ts>(args)...);
+    default:
+	    throw std::runtime_error("Unsupported dtype ID");
+    }
+}
+
+#endif  // #ifndef INCLUDE_GUARD_CUPY_TYPE_DISPATCHER_H
diff --git a/cupy/_core/internal.pxd b/cupy/_core/internal.pxd
new file mode 100644
index 0000000..11aad0f
--- /dev/null
+++ b/cupy/_core/internal.pxd
@@ -0,0 +1,66 @@
+from libcpp cimport bool as cpp_bool
+from libcpp cimport vector
+from libc.stdint cimport uint16_t
+
+from cupy._core._carray cimport shape_t
+from cupy._core._carray cimport strides_t
+
+
+cpdef Py_ssize_t prod(const vector.vector[Py_ssize_t]& args)
+
+cpdef Py_ssize_t prod_sequence(object args)
+
+cpdef bint is_in(const vector.vector[Py_ssize_t]& args, Py_ssize_t x)
+
+cpdef tuple get_size(object size)
+
+cpdef bint vector_equal(
+    const vector.vector[Py_ssize_t]& x, const vector.vector[Py_ssize_t]& y)
+
+cdef void get_reduced_dims(
+    shape_t& shape, strides_t& strides,
+    Py_ssize_t itemsize, shape_t& reduced_shape,
+    strides_t& reduced_strides)
+
+# Computes the contiguous strides given a shape and itemsize.
+# Returns the size (total number of elements).
+cdef Py_ssize_t get_contiguous_strides_inplace(
+    const shape_t& shape, strides_t& strides,
+    Py_ssize_t itemsize, bint is_c_contiguous, bint zeros_for_zerosize)
+
+cpdef bint get_c_contiguity(
+    shape_t& shape, strides_t& strides, Py_ssize_t itemsize)
+
+cpdef shape_t infer_unknown_dimension(
+    const shape_t& shape, Py_ssize_t size) except *
+
+cpdef slice complete_slice(slice slc, Py_ssize_t dim)
+
+cpdef tuple complete_slice_list(list slice_list, Py_ssize_t ndim)
+
+cpdef size_t clp2(size_t x)
+
+ctypedef unsigned short _float16
+
+cpdef uint16_t to_float16(float f)
+
+cpdef float from_float16(uint16_t v)
+
+cdef int _normalize_order(order, cpp_bool allow_k=*) except? 0
+
+cdef _broadcast_core(list arrays, shape_t& shape)
+
+cpdef bint _contig_axes(tuple axes)
+
+cpdef Py_ssize_t _normalize_axis_index(
+    Py_ssize_t axis, Py_ssize_t ndim) except -1
+
+cpdef tuple _normalize_axis_indices(
+    axes, Py_ssize_t ndim, cpp_bool sort_axes=*)
+
+cpdef strides_t _get_strides_for_order_K(x, dtype, shape=*)
+
+cpdef int _update_order_char(
+    bint is_c_contiguous, bint is_f_contiguous, int order_char)
+
+cpdef tuple _broadcast_shapes(shapes)
diff --git a/cupy/_core/internal.pyx b/cupy/_core/internal.pyx
new file mode 100644
index 0000000..c463dac
--- /dev/null
+++ b/cupy/_core/internal.pyx
@@ -0,0 +1,536 @@
+# distutils: language = c++
+
+cimport cpython  # NOQA
+cimport cython  # NOQA
+from libcpp cimport bool as cpp_bool
+from libc.stdint cimport uint32_t
+
+import warnings
+
+import numpy
+
+from cupy._core.core cimport _ndarray_base
+
+
+cdef extern from 'halffloat.h':
+    uint16_t npy_floatbits_to_halfbits(uint32_t f)
+    uint32_t npy_halfbits_to_floatbits(uint16_t h)
+
+
+@cython.profile(False)
+cpdef inline Py_ssize_t prod(const vector.vector[Py_ssize_t]& args):
+    cdef Py_ssize_t n = 1
+    for i in range(args.size()):
+        n *= args[i]
+    return n
+
+
+@cython.profile(False)
+cpdef inline Py_ssize_t prod_sequence(object args):
+    cdef Py_ssize_t i, n = 1
+    for i in args:
+        n *= i
+    return n
+
+
+@cython.profile(False)
+cpdef inline bint is_in(const vector.vector[Py_ssize_t]& args, Py_ssize_t x):
+    cdef Py_ssize_t i
+    for i in range(<Py_ssize_t>args.size()):
+        if args[i] == x:
+            return True
+    return False
+
+
+@cython.profile(False)
+cpdef inline void _check_not_bool(object x) except *:
+    if isinstance(x, (bool, numpy.bool_)):
+        raise TypeError('an integer is required')
+    if (
+        isinstance(x, numpy.ndarray)
+        and x.shape == ()
+        and x.dtype == numpy.bool_
+    ):
+        raise TypeError('an integer is required')
+
+
+@cython.profile(False)
+cpdef inline tuple get_size(object size):
+    if size is None:
+        warnings.warn(
+            'Passing None into shape arguments as an alias for () is '
+            'deprecated.',
+            DeprecationWarning,
+        )
+        return ()
+    if cpython.PySequence_Check(size):
+        # A numpy.ndarray unconditionally succeeds in PySequence_Check as
+        # it implements __getitem__, but zero-dim one is an unsized object
+        # and fails to make a tuple from it.
+        try:
+            ret = tuple(size)
+        except TypeError:
+            _check_not_bool(size)
+            return size,
+        for x in ret:
+            _check_not_bool(x)
+        return ret
+    _check_not_bool(size)
+    return size,
+
+
+@cython.profile(False)
+cpdef inline bint vector_equal(
+        const vector.vector[Py_ssize_t]& x,
+        const vector.vector[Py_ssize_t]& y):
+    cdef Py_ssize_t n = x.size()
+    if n != <Py_ssize_t>y.size():
+        return False
+    for i in range(n):
+        if x[i] != y[i]:
+            return False
+    return True
+
+
+@cython.profile(False)
+cdef void get_reduced_dims(
+        shape_t& shape, strides_t& strides, Py_ssize_t itemsize,
+        shape_t& reduced_shape, strides_t& reduced_strides):
+    cdef Py_ssize_t i, ndim, sh, st, prev_st, index
+    ndim = shape.size()
+    reduced_shape.clear()
+    reduced_strides.clear()
+    if ndim == 0:
+        return
+    reduced_shape.reserve(ndim)
+    reduced_strides.reserve(ndim)
+
+    prev_st = 0
+    index = -1
+    for i in range(ndim):
+        sh = shape[i]
+        if sh == 0:
+            reduced_shape.assign(1, 0)
+            reduced_strides.assign(1, itemsize)
+            return
+        if sh == 1:
+            continue
+        st = strides[i]
+        if index == -1 or prev_st != sh * st:
+            reduced_shape.push_back(sh)
+            reduced_strides.push_back(st)
+            index += 1
+        else:
+            reduced_shape[index] *= sh
+            reduced_strides[index] = st
+        prev_st = st
+
+
+@cython.profile(False)
+cdef inline Py_ssize_t get_contiguous_strides_inplace(
+        const shape_t& shape, strides_t& strides,
+        Py_ssize_t itemsize, bint is_c_contiguous, bint zeros_for_zerosize):
+    cdef Py_ssize_t st, sh
+    cdef Py_ssize_t is_nonzero_size = 1
+    cdef int i, ndim = shape.size()
+    cdef Py_ssize_t idx
+    strides.resize(ndim, 0)
+    st = 1
+
+    # For some cases, the strides are all set to zero if the shape is
+    # zero-sized since NumPy 1.23.
+    if zeros_for_zerosize:
+        for i in range(ndim):
+            sh = shape[i]
+            if sh == 0:
+                is_nonzero_size = 0
+                break
+
+        if is_nonzero_size == 0:
+            for i in range(ndim):
+                strides[i] = 0
+            return 0
+
+    for i in range(ndim):
+        if is_c_contiguous:
+            idx = ndim - 1 - i
+        else:
+            idx = i
+        strides[idx] = st * itemsize
+        sh = shape[idx]
+        if sh > 1:
+            st *= sh
+        elif sh == 0:
+            is_nonzero_size = 0
+    return st * is_nonzero_size
+
+
+@cython.profile(False)
+cpdef inline bint get_c_contiguity(
+        shape_t& shape, strides_t& strides, Py_ssize_t itemsize):
+    cdef Py_ssize_t i, prev_i, ndim, sh, st, index
+    ndim = strides.size()
+    if ndim == 0 or (ndim == 1 and strides[0] == itemsize):
+        return True
+    prev_i = -1
+    index = st = 0
+    for i in range(ndim):
+        sh = shape[i]
+        if sh == 0:
+            return True
+        if sh == 1:
+            continue
+        st = strides[i]
+        if prev_i == -1 or strides[prev_i] != sh * st:
+            index += 1
+        prev_i = i
+    return index == 0 or (index == 1 and st == itemsize)
+
+
+@cython.profile(False)
+cpdef shape_t infer_unknown_dimension(
+        const shape_t& shape, Py_ssize_t size) except *:
+    cdef shape_t ret = shape
+    cdef Py_ssize_t cnt=0, index=-1, new_size=1
+    for i in range(shape.size()):
+        if shape[i] < 0:
+            cnt += 1
+            index = i
+        else:
+            new_size *= shape[i]
+    if cnt == 0 and new_size == size:
+        return ret
+    if cnt > 1:
+        raise ValueError('can only specify one unknown dimension')
+    if cnt == 0 or new_size == 0 or size % new_size != 0:
+        # TODO(kataoka): print "newaxis" for unknown
+        raise ValueError(
+            f'cannot reshape array of size {size} into shape {tuple(shape)}')
+    ret[index] = size // new_size
+    return ret
+
+
+@cython.profile(False)
+cpdef inline Py_ssize_t _extract_slice_element(x) except? 0:
+    try:
+        return x.__index__()
+    except AttributeError:
+        return int(x)
+
+
+@cython.profile(False)
+cpdef slice complete_slice(slice slc, Py_ssize_t dim):
+    cdef Py_ssize_t start=0, stop=0, step=0
+    cdef bint start_none, stop_none
+    if slc.step is None:
+        step = 1
+    else:
+        try:
+            step = _extract_slice_element(slc.step)
+        except TypeError:
+            raise TypeError(
+                'slice.step must be int or None or have __index__ method: '
+                '{}'.format(slc))
+        if step == 0:
+            raise ValueError('Slice step must be nonzero.')
+
+    start_none = slc.start is None
+    if not start_none:
+        try:
+            start = _extract_slice_element(slc.start)
+        except TypeError:
+            raise TypeError(
+                'slice.start must be int or None or have __index__ method: '
+                '{}'.format(slc))
+
+        if start < 0:
+            start += dim
+
+    stop_none = slc.stop is None
+    if not stop_none:
+        try:
+            stop = _extract_slice_element(slc.stop)
+        except TypeError:
+            raise TypeError(
+                'slice.stop must be int or None or have __index__ method: '
+                '{}'.format(slc))
+
+        if stop < 0:
+            stop += dim
+
+    if step > 0:
+        start = 0 if start_none else max(0, min(dim, start))
+        stop = dim if stop_none else max(start, min(dim, stop))
+    else:
+        start = dim - 1 if start_none else max(-1, min(dim - 1, start))
+        stop = -1 if stop_none else max(-1, min(start, stop))
+
+    return slice(start, stop, step)
+
+
+@cython.profile(False)
+cpdef tuple complete_slice_list(list slice_list, Py_ssize_t ndim):
+    warnings.warn(
+        'complete_slice_list is deprecated because the function sometimes '
+        'returns wrong result (#1512, #4799).',
+        DeprecationWarning)
+    cdef Py_ssize_t i, n_newaxes, n_ellipses, ellipsis, n
+    slice_list = list(slice_list)  # copy list
+    # Expand ellipsis into empty slices
+    ellipsis = -1
+    n_newaxes = n_ellipses = 0
+    for i, s in enumerate(slice_list):
+        if s is None:
+            n_newaxes += 1
+        elif s is Ellipsis:
+            n_ellipses += 1
+            ellipsis = i
+    if n_ellipses > 1:
+        raise IndexError("an index can only have a single ellipsis ('...')")
+
+    n = ndim - <Py_ssize_t>len(slice_list) + n_newaxes
+    if n_ellipses > 0:
+        slice_list[ellipsis:ellipsis + 1] = [slice(None)] * (n + 1)
+    elif n > 0:
+        slice_list += [slice(None)] * n
+    return slice_list, n_newaxes
+
+
+@cython.profile(False)
+cpdef size_t clp2(size_t x):
+    x -= 1
+    x |= x >> 1
+    x |= x >> 2
+    x |= x >> 4
+    x |= x >> 8
+    x |= x >> 16
+    x |= x >> 32
+    return x + 1
+
+
+cdef union float32_int:
+    uint32_t n
+    float f
+
+
+cpdef uint16_t to_float16(float f):
+    cdef float32_int c
+    c.f = f
+    return npy_floatbits_to_halfbits(c.n)
+
+
+cpdef float from_float16(uint16_t v):
+    cdef float32_int c
+    c.n = npy_halfbits_to_floatbits(v)
+    return c.f
+
+
+@cython.profile(False)
+cdef inline int _normalize_order(order, cpp_bool allow_k=True) except? 0:
+    cdef int order_char
+    order_char = b'C' if len(order) == 0 else ord(order[0])
+    if order_char == b'K' or order_char == b'k':
+        if not allow_k:
+            raise ValueError('order \'K\' is not permitted')
+        order_char = b'K'
+    elif order_char == b'A' or order_char == b'a':
+        order_char = b'A'
+    elif order_char == b'C' or order_char == b'c':
+        order_char = b'C'
+    elif order_char == b'F' or order_char == b'f':
+        order_char = b'F'
+    else:
+        raise ValueError('order not understood')
+    return order_char
+
+
+cdef _broadcast_core(list arrays, shape_t& shape):
+    cdef Py_ssize_t i, j, s, a_ndim, a_sh, nd
+    cdef strides_t strides
+    cdef vector.vector[int] index
+    cdef _ndarray_base a
+
+    shape.clear()
+    index.reserve(len(arrays))
+    nd = 0
+    for i, x in enumerate(arrays):
+        if not isinstance(x, _ndarray_base):
+            continue
+        a = x
+        index.push_back(i)
+        nd = max(nd, <Py_ssize_t>a._shape.size())
+
+    if index.size() == 0:
+        return
+
+    shape.reserve(nd)
+    for i in range(nd):
+        s = 1
+        for j in index:
+            a = arrays[j]
+            a_ndim = <Py_ssize_t>a._shape.size()
+            if i < nd - a_ndim:
+                continue
+            a_sh = a._shape[i - (nd - a_ndim)]
+            if a_sh == s or a_sh == 1:
+                continue
+            if s == 1:
+                s = a_sh
+                continue
+            raise ValueError(
+                'operands could not be broadcast together with shapes {}'
+                .format(
+                    ' '.join([str(x.shape) if isinstance(x, _ndarray_base)
+                              else '()' for x in arrays])))
+        shape.push_back(s)
+
+    for i in index:
+        a = arrays[i]
+        if vector_equal(a._shape, shape):
+            continue
+
+        strides.assign(nd, <Py_ssize_t>0)
+        a_ndim = <Py_ssize_t>a._shape.size()
+        for j in range(a_ndim):
+            a_sh = a._shape[j]
+            if a_sh == shape[j + nd - a_ndim]:
+                strides[j + nd - a_ndim] = a._strides[j]
+
+        # TODO(niboshi): Confirm update_x_contiguity flags
+        arrays[i] = a._view(type(a), shape, strides, True, True, a)
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+cpdef bint _contig_axes(tuple axes):
+    # Indicate if the specified axes are in ascending order without gaps.
+    cdef Py_ssize_t n
+    cdef int n_ax = len(axes)
+    cdef bint contig = n_ax > 0
+    for n in range(1, n_ax):
+        contig = (axes[n] - axes[n - 1]) == 1
+        if not contig:
+            break
+    return contig
+
+
+cpdef Py_ssize_t _normalize_axis_index(
+        Py_ssize_t axis, Py_ssize_t ndim) except -1:
+    """
+    Normalizes an axis index, ``axis``, such that is a valid positive index
+    into the shape of array with ``ndim`` dimensions. Raises a ValueError
+    with an appropriate message if this is not possible.
+
+    Args:
+        axis (int):
+            The un-normalized index of the axis. Can be negative
+        ndim (int):
+            The number of dimensions of the array that ``axis`` should be
+            normalized against
+
+    Returns:
+        int:
+            The normalized axis index, such that `0 <= normalized_axis < ndim`
+
+    """
+    if not (-ndim <= axis < ndim):
+        raise numpy.AxisError(axis, ndim)
+    if axis < 0:
+        axis += ndim
+    return axis
+
+
+cpdef tuple _normalize_axis_indices(
+        axes, Py_ssize_t ndim, cpp_bool sort_axes=True):
+    """Normalize axis indices.
+
+    Args:
+        axis (int, tuple of int or None):
+            The un-normalized indices of the axis. Can be negative.
+        ndim (int):
+            The number of dimensions of the array that ``axis`` should be
+            normalized against
+        sort_axes (bool):
+            If provided as False will not sort the axes, default is to return
+            the sorted values.
+
+    Returns:
+        tuple of int:
+            The tuple of normalized axis indices.
+    """
+    if axes is None:
+        axes = tuple(range(ndim))
+    elif not isinstance(axes, tuple):
+        axes = axes,
+
+    res = []
+    for axis in axes:
+        axis = _normalize_axis_index(axis, ndim)
+        if axis in res:
+            # the message in `numpy/core/src/multiarray/conversion_utils.c`
+            raise ValueError('duplicate value in \'axis\'')
+        res.append(axis)
+
+    return tuple(sorted(res) if sort_axes else res)
+
+
+cpdef strides_t _get_strides_for_order_K(x, dtype, shape=None):
+    # x here can be either numpy.ndarray or cupy.ndarray
+    cdef strides_t strides
+    # strides used when order='K' for astype, empty_like, etc.
+
+    # Note that there is different semantics of order='K'.
+    # See also `_routines_manipulation._npyiter_k_order_axes`.
+    stride_and_index = [
+        (abs(s), -i) for i, s in enumerate(x.strides)]
+    stride_and_index.sort()
+    strides.resize(x.ndim)
+    stride = dtype.itemsize
+    for s, i in stride_and_index:
+        strides[-i] = stride
+        stride *= shape[-i] if shape else x.shape[-i]
+    return strides
+
+
+cpdef int _update_order_char(
+        bint is_c_contiguous, bint is_f_contiguous, int order_char):
+    # update order_char based on array contiguity
+    if order_char == b'A':
+        if is_f_contiguous and not is_c_contiguous:
+            order_char = b'F'
+        else:
+            order_char = b'C'
+    elif order_char == b'K':
+        if is_c_contiguous:
+            order_char = b'C'
+        elif is_f_contiguous:
+            order_char = b'F'
+    return order_char
+
+
+cpdef tuple _broadcast_shapes(shapes):
+    """Broadcast shapes together.
+
+    Args:
+        shapes (list of tuples of int):
+            shapes that will be broadcasted together.
+
+    Returns:
+        tuple of int:
+            Resulting shape of broadcasting shapes together.
+    """
+    out_ndim = max([len(shape) for shape in shapes])
+    padded_shapes = [
+        (1,) * (out_ndim - len(shape)) + shape for shape in shapes]
+
+    result_shape = []
+    for dims in zip(*padded_shapes):
+        dims = [dim for dim in dims if dim != 1]
+        out_dim = 1 if len(dims) == 0 else dims[0]
+        if any([dim != out_dim for dim in dims]):
+            raise ValueError(
+                'operands could not be broadcast together with shapes' +
+                ' '.join([str(shape) for shape in shapes]))
+        result_shape.append(out_dim)
+
+    return tuple(result_shape)
diff --git a/cupy/_core/new_fusion.pyx b/cupy/_core/new_fusion.pyx
new file mode 100644
index 0000000..e0c5d94
--- /dev/null
+++ b/cupy/_core/new_fusion.pyx
@@ -0,0 +1,176 @@
+import functools
+
+import numpy
+
+from cupy._core import core
+from cupy._core import _fusion_trace
+from cupy._core import _fusion_kernel
+from cupy._core import _fusion_thread_local
+from cupy._core._fusion_interface import _ScalarProxy  # no-cython-lint
+from cupy._core._fusion_interface import _ArrayProxy  # no-cython-lint
+from cupy._core._fusion_variable import _AbstractDim
+
+
+_thread_local = _fusion_thread_local.thread_local
+_is_fusing = _fusion_thread_local.is_new_fusing
+_call_ufunc = _fusion_thread_local.call_ufunc
+_call_reduction = _fusion_thread_local.call_reduction
+
+
+cdef tuple _fusion_argument_types = (
+    core.ndarray, numpy.ndarray, numpy.generic,
+    int, float, complex, bool, type(None))
+
+
+def _get_fused_kernel(name, func, args):
+    try:
+        _thread_local.is_new_fusing = True
+        trace_result = _fusion_trace.trace(func, args)
+        kernel = _fusion_kernel.FusedKernel(name, trace_result)
+    finally:
+        _thread_local.is_new_fusing = False
+    return kernel
+
+
+class Fusion:
+    """Function class.
+
+    This class can be get by using `fuse` function
+
+    Args:
+        func (function): The function before fusing.
+        name (str optional): The name of the function.
+    """
+    def __init__(self, func, name=None):
+        self.func = func
+        self.name = name or func.__name__
+        self._cache = {}
+        # TODO(asi1024): Support switch of optimization mode.
+
+    def __repr__(self):
+        return '<Fusion name={}>'.format(self.name)
+
+    def clear_cache(self):
+        self._cache = {}
+
+    def __call__(self, *args, **kwargs):
+        cdef int nargs = len(args)
+        cdef int i
+
+        if _is_fusing():
+            # Inner function of composition of multiple fused functions.
+            return self.func(*args)
+
+        exec_cupy = False
+        for i in range(nargs):
+            if isinstance(args[i], core.ndarray):
+                exec_cupy = True
+                break
+        if not exec_cupy:
+            # No cupy ndarray exists in the arguments
+            return self.func(*args)
+
+        # Check for invalid argument types
+        for i in range(nargs):
+            if not isinstance(args[i], _fusion_argument_types):
+                mes = 'Invalid argument type for \'{}\': ({})'
+                arg_types = ', '.join(repr(type(a)) for a in args)
+                raise TypeError(mes.format(self.name, arg_types))
+
+        # Cache the result of execution path trace
+        # TODO(asi1024): Some ndarrays may share the same memory space.
+        cdef list params_info = []
+        cdef list shape_info = []
+
+        # Create cache keys to find a kernel already emitted:
+        #     param_key: ndims and dtypes of the arguments.
+        #     shape_key: shapes of the arguments.
+        for i in range(nargs):
+            arg = args[i]
+            if isinstance(arg, core.ndarray):
+                params_info.append(arg.dtype.char)
+                params_info.append(arg.ndim)
+                shape_info.append(arg.shape)
+                base = arg.base
+                params_info.append(base is None)
+                for j in range(i):
+                    params_info.append(arg is args[j])
+                    if base is not None and isinstance(args[j], core.ndarray):
+                        # args[i] and args[j] may share the same memory space.
+                        params_info.append(base is args[j].base)
+            elif isinstance(arg, numpy.generic):
+                params_info.append(arg.dtype.char)
+            elif arg is None:
+                params_info.append(None)
+            elif isinstance(arg, float):
+                params_info.append('d')
+            elif isinstance(arg, int):
+                params_info.append('l')
+            elif isinstance(arg, bool):
+                params_info.append('?')
+            elif isinstance(arg, complex):
+                params_info.append('D')
+            else:
+                raise TypeError('Unsupported input type {}.'.format(type(arg)))
+
+        cdef tuple param_key = tuple(params_info)
+        cdef tuple shape_key = tuple(shape_info)
+
+        # self._cache(dict):
+        #     key:   Tuple of dtypes and ndims of the inputs (param_key).
+        #     value: Pair of cache_shape and kernel_list.
+        # cache_shape(dict):
+        #     key:   Tuple of shapes of the inputs (shape_key).
+        #     value: Pair of Runtime kernel and its performance cookie.
+        # kernel_list(list): List of Runtime kernel.
+        cache_shape, kernel_list = self._cache.get(param_key, (None, None))
+
+        if cache_shape is None:
+            # Initializes cache_shape and kernel_list.
+            cache_shape = {}
+            kernel_list = []
+            self._cache[param_key] = cache_shape, kernel_list
+
+        # Find a cached kernel from cache_shape with the key as actual shape.
+        kernel, shapes = cache_shape.get(shape_key, (None, None))
+
+        if kernel is None:
+            # Find a cached kernel from kernel_list.
+
+            # Create a dim_map: a dictionary from _AbstractDim to int.
+            dim_map = {}
+            for input_index, arg in enumerate(args):
+                if isinstance(arg, core.ndarray):
+                    for axis, dim in enumerate(arg.shape):
+                        dim_map[_AbstractDim(input_index, axis)] = dim
+
+            # Find a kernel that satisfies the shape constraints.
+            for cand_kernel in kernel_list:
+                if cand_kernel.shape_constraints.satisfy(dim_map):
+                    kernel = cand_kernel
+                    shapes = kernel.get_shapes_of_kernel_params(args)
+                    cache_shape[shape_key] = kernel, shapes
+                    break
+
+        if kernel is None:
+            # If not cached at all, analyze the target function.
+            kernel = _get_fused_kernel(self.name, self.func, args)
+            shapes = kernel.get_shapes_of_kernel_params(args)
+            cache_shape[shape_key] = kernel, shapes
+            kernel_list.append(kernel)
+
+        return kernel.execute(args, shapes)
+
+
+def fuse(*args, **kwargs):
+    """Decorator that fuses a function.
+    """
+
+    def wrapper(f, kernel_name=None):
+        return Fusion(f, kernel_name)
+
+    if len(args) == 1 and len(kwargs) == 0 and callable(args[0]):
+        return functools.update_wrapper(wrapper(args[0]), args[0])
+    else:
+        return lambda f: functools.update_wrapper(
+            wrapper(f, *args, **kwargs), f)
diff --git a/cupy/_core/raw.pxd b/cupy/_core/raw.pxd
new file mode 100644
index 0000000..59a557b
--- /dev/null
+++ b/cupy/_core/raw.pxd
@@ -0,0 +1,30 @@
+cdef class RawKernel:
+
+    cdef:
+        readonly str code
+        readonly str file_path
+        readonly str name
+        readonly tuple options
+        readonly str backend
+        readonly bint enable_cooperative_groups
+        object log_stream
+        tuple name_expressions
+        bint translate_cucomplex
+        bint jitify
+        list _kernel_cache
+        readonly str __name__
+        int raw_ver
+
+
+cdef class RawModule:
+
+    cdef:
+        readonly str code
+        readonly str file_path
+        readonly tuple options
+        readonly str backend
+        readonly bint enable_cooperative_groups
+        readonly tuple name_expressions
+        object log_stream
+        bint translate_cucomplex
+        bint jitify
diff --git a/cupy/_core/raw.pyx b/cupy/_core/raw.pyx
new file mode 100644
index 0000000..7fc8555
--- /dev/null
+++ b/cupy/_core/raw.pyx
@@ -0,0 +1,545 @@
+import pickle
+
+import cupy
+
+from cupy_backends.cuda.api cimport driver
+from cupy_backends.cuda.api cimport runtime
+from cupy.cuda.function cimport Function, Module
+
+
+cdef class RawKernel:
+
+    """User-defined custom kernel.
+
+    This class can be used to define a custom kernel using raw CUDA source.
+
+    The kernel is compiled at an invocation of the :meth:`~RawKernel.__call__`
+    method, which is cached for each device.
+    The compiled binary is also cached into a file under the
+    ``$HOME/.cupy/kernel_cache/`` directory with a hashed file name. The cached
+    binary is reused by other processes.
+
+    Args:
+        code (str): CUDA source code.
+        name (str): Name of the kernel function.
+        options (tuple of str): Compiler options passed to the backend (NVRTC
+            or NVCC). For details, see
+            https://docs.nvidia.com/cuda/nvrtc/index.html#group__options or
+            https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#command-option-description
+        backend (str): Either `nvrtc` or `nvcc`. Defaults to `nvrtc`
+        translate_cucomplex (bool): Whether the CUDA source includes the header
+            `cuComplex.h` or not. If set to ``True``, any code that uses the
+            functions from `cuComplex.h` will be translated to its Thrust
+            counterpart. Defaults to ``False``.
+        enable_cooperative_groups (bool): Whether to enable cooperative groups
+            in the CUDA source. If set to ``True``, compile options are
+            configured properly and the kernel is launched with
+            ``cuLaunchCooperativeKernel`` so that cooperative groups can be
+            used from the CUDA source.
+            This feature is only supported in CUDA 9 or later.
+        jitify (bool): Whether or not to use `Jitify`_ to assist NVRTC to
+            compile C++ kernels. Defaults to ``False``.
+
+    .. _Jitify:
+        https://github.com/NVIDIA/jitify
+
+    """
+    def __cinit__(self):
+        # this is only for pickling: if any change is made such that the old
+        # pickles cannot be reused, we bump this version number
+        self.raw_ver = 2
+
+    def __init__(self, str code, str name, tuple options=(),
+                 str backend='nvrtc', *, bint translate_cucomplex=False,
+                 bint enable_cooperative_groups=False, bint jitify=False):
+
+        self.code = code
+        self.name = name
+        self.options = options
+        self.backend = backend
+        self.translate_cucomplex = translate_cucomplex
+        self.enable_cooperative_groups = enable_cooperative_groups
+        self.jitify = jitify
+
+        # only used when RawKernels are produced from RawModule
+        self.file_path = None  # for cubin/ptx
+        self.name_expressions = None  # for C++ template
+
+        # per-device, per-instance cache, to be initialized on first call
+        self._kernel_cache = []
+
+        # This is for profiling mechanisms to auto infer a name
+        self.__name__ = name
+
+    def __call__(self, grid, block, args, **kwargs):
+        """__call__(self, grid, block, args, *, shared_mem=0)
+
+        Compiles and invokes the kernel.
+
+        The compilation runs only if the kernel is not cached.
+
+        Args:
+            grid (tuple): Size of grid in blocks.
+            block (tuple): Dimensions of each thread block.
+            args (tuple): Arguments of the kernel.
+            shared_mem (int): Dynamic shared-memory size per thread block in
+                bytes.
+
+        """
+        self.kernel(
+            grid, block, args,
+            enable_cooperative_groups=self.enable_cooperative_groups,
+            **kwargs)
+
+    @property
+    def kernel(self):
+        return self._kernel()
+
+    def _kernel(self, log_stream=None):
+        # The kernel is cached, so on the device where this has been called,
+        # we would just look up from the cache, and do recompiling only when
+        # switching to a different device
+        cdef Function ker
+        cdef Module mod
+
+        # We delay establishing the CUDA context until it's really needed
+        cdef int dev = runtime.getDevice()
+        if not self._kernel_cache:
+            self._kernel_cache = [None] * runtime.getDeviceCount()
+
+        ker = self._kernel_cache[dev]
+        if ker is None:
+            assert (self.code is None) != (self.file_path is None)
+            mod = _get_raw_module(
+                self.code, self.file_path, self.options, self.backend,
+                self.translate_cucomplex, self.enable_cooperative_groups,
+                self.jitify, self.name_expressions, log_stream)
+            ker = mod.get_function(self.name)
+            self._kernel_cache[dev] = ker
+        return ker
+
+    # It is not possible to implement __reduce__ for a cdef class. The
+    # two-tuple return cannot handle the keyword-only arguments, and
+    # the three-tuple return (for updating the object's internal state)
+    # does not work either, because cdef classes by default does not have
+    # __dict__. Therefore, the only way to handle keyword-only arguments
+    # for picking a cdef class is to define the following two special
+    # functions, which is in fact preferred over __reduce__.
+
+    def __getstate__(self):
+        cdef dict args
+        args = {'code': self.code,
+                'name': self.name,
+                'options': self.options,
+                'backend': self.backend,
+                'translate_cucomplex': self.translate_cucomplex,
+                'file_path': self.file_path,
+                'name_expressions': self.name_expressions,
+                'enable_cooperative_groups': self.enable_cooperative_groups,
+                'jitify': self.jitify,
+                'raw_ver': self.raw_ver}
+        return args
+
+    def __setstate__(self, dict args):
+        if args.get('raw_ver') != self.raw_ver:
+            raise pickle.UnpicklingError(
+                'The pickled RawKernel object is not supported by the current '
+                'CuPy version. It should not be used. Please recompile.')
+
+        self.code = args['code']
+        self.name = self.__name__ = args['name']
+        self.options = args['options']
+        self.backend = args['backend']
+        self.translate_cucomplex = args['translate_cucomplex']
+        self.enable_cooperative_groups = args['enable_cooperative_groups']
+        self.file_path = args['file_path']
+        self.name_expressions = args['name_expressions']
+        self.jitify = args['jitify']
+        self._kernel_cache = []  # to force recompiling
+
+    @property
+    def attributes(self):
+        """Returns a dictionary containing runtime kernel attributes. This is
+        a read-only property; to overwrite the attributes, use
+
+        .. code-block:: python
+
+            kernel = RawKernel(...)  # arguments omitted
+            kernel.max_dynamic_shared_size_bytes = ...
+            kernel.preferred_shared_memory_carveout = ...
+
+        Note that the two attributes shown in the above example are the only
+        two currently settable in CUDA.
+
+        Any attribute not existing in the present CUDA toolkit version will
+        have the value -1.
+
+        Returns:
+            dict: A dictionary containing the kernel's attributes.
+        """
+        cdef dict attrs = {}
+        cdef list keys = ['max_threads_per_block', 'shared_size_bytes',
+                          'const_size_bytes', 'local_size_bytes',
+                          'num_regs', 'ptx_version', 'binary_version',
+                          'cache_mode_ca', 'max_dynamic_shared_size_bytes',
+                          'preferred_shared_memory_carveout']
+        for attr in keys:
+            attrs[attr] = getattr(self, attr)
+        return attrs
+
+    @property
+    def max_threads_per_block(self):
+        """The maximum number of threads per block that can successfully
+        launch the function on the device.
+        """
+        attr = driver.CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @property
+    def shared_size_bytes(self):
+        """The size in bytes of the statically-allocated shared memory
+        used by the function. This is separate from any dynamically-allocated
+        shared memory, which must be specified when the function is called.
+        """
+        attr = driver.CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @property
+    def const_size_bytes(self):
+        """The size in bytes of constant memory used by the function."""
+        attr = driver.CU_FUNC_ATTRIBUTE_CONST_SIZE_BYTES
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @property
+    def local_size_bytes(self):
+        """The size in bytes of local memory used by the function."""
+        attr = driver.CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @property
+    def num_regs(self):
+        """The number of registers used by the function."""
+        attr = driver.CU_FUNC_ATTRIBUTE_NUM_REGS
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @property
+    def ptx_version(self):
+        """The PTX virtual architecture version that was used during
+        compilation, in the format: 10*major + minor.
+        """
+        attr = driver.CU_FUNC_ATTRIBUTE_PTX_VERSION
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @property
+    def binary_version(self):
+        """The binary architecture version that was used during compilation,
+        in the format: 10*major + minor.
+        """
+        attr = driver.CU_FUNC_ATTRIBUTE_BINARY_VERSION
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @property
+    def cache_mode_ca(self):
+        """Indicates whether option "-Xptxas --dlcm=ca" was set during
+        compilation.
+        """
+        attr = driver.CU_FUNC_ATTRIBUTE_CACHE_MODE_CA
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @property
+    def max_dynamic_shared_size_bytes(self):
+        """The maximum dynamically-allocated shared memory size in bytes that
+        can be used by the function. Can be set.
+        """
+        attr = driver.CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @max_dynamic_shared_size_bytes.setter
+    def max_dynamic_shared_size_bytes(self, bytes):
+        attr = driver.CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES
+        driver.funcSetAttribute(self.kernel.ptr, attr, bytes)
+
+    @property
+    def preferred_shared_memory_carveout(self):
+        """On devices that have a unified L1 cache and shared memory,
+        indicates the fraction to be used for shared memory as a
+        `percentage` of the total. If the fraction does not exactly equal a
+        supported shared memory capacity, then the next larger supported
+        capacity is used. Can be set.
+        """
+        attr = driver.CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT
+        return driver.funcGetAttribute(attr, self.kernel.ptr)
+
+    @preferred_shared_memory_carveout.setter
+    def preferred_shared_memory_carveout(self, fraction):
+        attr = driver.CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT
+        driver.funcSetAttribute(self.kernel.ptr, attr, fraction)
+
+    def compile(self, log_stream=None):
+        """Compile the current kernel.
+
+        In general, you don't have to call this method;
+        kernels are compiled implicitly on the first call.
+
+        Args:
+            log_stream (object): Pass either ``sys.stdout`` or a file object to
+                which the compiler output will be written.
+                Defaults to ``None``.
+        """
+        # Flush the cache when compilation is explicitly requested
+        self._kernel_cache = [None] * runtime.getDeviceCount()
+        self._kernel(log_stream=log_stream)
+
+
+cdef class RawModule:
+    """User-defined custom module.
+
+    This class can be used to either compile raw CUDA sources or load CUDA
+    modules (\\*.cubin, \\*.ptx). This class is useful when a number of CUDA
+    kernels in the same source need to be retrieved.
+
+    For the former case, the CUDA source code is compiled when any method is
+    called. For the latter case, an existing CUDA binary (\\*.cubin) or a PTX
+    file can be loaded by providing its path.
+
+    CUDA kernels in a :class:`RawModule` can be retrieved by calling
+    :meth:`get_function`, which will return an instance of :class:`RawKernel`.
+    (Same as in :class:`RawKernel`, the generated binary is also cached.)
+
+    Args:
+        code (str): CUDA source code. Mutually exclusive with ``path``.
+        path (str): Path to cubin/ptx. Mutually exclusive with ``code``.
+        options (tuple of str): Compiler options passed to the backend (NVRTC
+            or NVCC). For details, see
+            https://docs.nvidia.com/cuda/nvrtc/index.html#group__options or
+            https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#command-option-description.
+        backend (str): Either `nvrtc` or `nvcc`. Defaults to `nvrtc`
+        translate_cucomplex (bool): Whether the CUDA source includes the header
+            `cuComplex.h` or not. If set to ``True``, any code that uses the
+            functions from `cuComplex.h` will be translated to its Thrust
+            counterpart. Defaults to ``False``.
+        enable_cooperative_groups (bool): Whether to enable cooperative groups
+            in the CUDA source. If set to ``True``, compile options are
+            configured properly and the kernel is launched with
+            ``cuLaunchCooperativeKernel`` so that cooperative groups can be
+            used from the CUDA source.
+            This feature is only supported in CUDA 9 or later.
+        name_expressions (sequence of str): A sequence (e.g. list) of strings
+            referring to the names of C++ global/template kernels. For example,
+            ``name_expressions=['func1<int>', 'func1<double>', 'func2']`` for
+            the template kernel ``func1<T>`` and non-template kernel ``func2``.
+            Strings in this tuple must then be passed, one at a time, to
+            :meth:`get_function` to retrieve the corresponding kernel.
+        jitify (bool): Whether or not to use `Jitify`_ to assist NVRTC to
+            compile C++ kernels. Defaults to ``False``.
+
+    .. note::
+        Each kernel in ``RawModule`` possesses independent function attributes.
+
+    .. note::
+        Before CuPy v8.0.0, the compilation happens at initialization. Now, it
+        happens at the first time retrieving any object (kernels or pointers)
+        from the module.
+
+    .. _Jitify:
+        https://github.com/NVIDIA/jitify
+
+    """
+    def __init__(self, *, str code=None, str path=None, tuple options=(),
+                 str backend='nvrtc', bint translate_cucomplex=False,
+                 bint enable_cooperative_groups=False,
+                 name_expressions=None, bint jitify=False):
+        if (code is None) == (path is None):
+            raise TypeError(
+                'Exactly one of `code` and `path` keyword arguments must be '
+                'given.')
+        if name_expressions is not None:
+            if code is None:
+                raise ValueError('need CUDA C++ code for the requested '
+                                 'kernels')
+            if backend != 'nvrtc':
+                raise ValueError('only nvrtc supports retrieving the mangled '
+                                 'names for the given name expressions')
+            for option in options:
+                if '-std=c++' in option:  # both -std and --std are valid
+                    break
+            else:
+                raise ValueError('need to specify C++ standard for compiling '
+                                 'template code')
+            self.name_expressions = tuple(name_expressions)  # make it hashable
+        else:
+            self.name_expressions = None
+        if jitify:
+            if code is None:
+                raise ValueError('Jitify does not support precompiled objects')
+            if backend != 'nvrtc':  # TODO(leofang): how about hiprtc?
+                raise ValueError('Jitify only supports NVRTC')
+
+        self.code = code
+        self.file_path = path
+        self.enable_cooperative_groups = enable_cooperative_groups
+        self.jitify = jitify
+
+        if self.code is not None:
+            self.options = options
+            self.backend = backend
+            self.translate_cucomplex = translate_cucomplex
+        elif self.file_path is not None:
+            self.options = ()
+            self.backend = 'nvcc'
+            self.translate_cucomplex = False
+
+    @property
+    def module(self):
+        return self._module()
+
+    def _module(self, log_stream=None):
+        # The module is cached, so on the device where this has been called,
+        # we would just look up from the cache, and do recompiling only when
+        # switching to a different device
+        cdef Module mod
+
+        mod = _get_raw_module(
+            self.code, self.file_path, self.options, self.backend,
+            self.translate_cucomplex, self.enable_cooperative_groups,
+            self.jitify, self.name_expressions, log_stream)
+        return mod
+
+    def compile(self, log_stream=None):
+        """Compile the current module.
+
+        In general, you don't have to call this method;
+        kernels are compiled implicitly on the first call.
+
+        Args:
+            log_stream (object): Pass either ``sys.stdout`` or a file object to
+                which the compiler output will be written.
+                Defaults to ``None``.
+
+        .. note::
+            Calling :meth:`compile` will reset the internal state of
+            a :class:`RawKernel`.
+
+        """
+        self._module(log_stream)
+
+    def get_function(self, str name):
+        """Retrieve a CUDA kernel by its name from the module.
+
+        Args:
+            name (str): Name of the kernel function. For C++ global/template
+                kernels, ``name`` refers to one of the name expressions
+                specified when initializing the present :class:`RawModule`
+                instance.
+
+        Returns:
+            RawKernel: An ``RawKernel`` instance.
+
+        .. note::
+            The following example shows how to retrieve one of the specialized
+            C++ template kernels:
+
+            .. code-block:: python
+
+                code = r'''
+                template<typename T>
+                __global__ void func(T* in_arr) { /* do something */ }
+                '''
+
+                kers = ('func<int>', 'func<float>', 'func<double>')
+                mod = cupy.RawModule(code=code, options=('--std=c++11',),
+                                     name_expressions=kers)
+
+                // retrieve func<int>
+                ker_int = mod.get_function(kers[0])
+
+        .. seealso::
+            ``nvrtcAddNameExpression`` and ``nvrtcGetLoweredName`` from
+            `Accessing Lowered Names`_ of the NVRTC documentation.
+
+        .. _Accessing Lowered Names:
+            https://docs.nvidia.com/cuda/nvrtc/index.html#accessing-lowered-names
+
+        """
+        cdef RawKernel ker
+        cdef Function func
+        cdef str mangled_name
+
+        # check if the name is a valid C++ name expression
+        if self.name_expressions:
+            mangled_name = self.module.mapping.get(name)
+            if mangled_name is not None:
+                name = mangled_name
+
+        ker = RawKernel(
+            self.code, name, self.options, self.backend,
+            translate_cucomplex=self.translate_cucomplex,
+            enable_cooperative_groups=self.enable_cooperative_groups,
+            jitify=self.jitify)
+
+        # for lookup in case we loaded from cubin/ptx
+        ker.file_path = self.file_path
+        # for lookup in case we specialize a template
+        ker.name_expressions = self.name_expressions
+        # register the kernel in the cache
+        func = ker.kernel  # no-cython-lint
+        return ker
+
+    def get_global(self, name):
+        '''Retrieve a pointer to a global symbol by its name from the module.
+
+        Args:
+            name (str): Name of the global symbol.
+
+        Returns:
+            ~cupy.cuda.MemoryPointer: A handle to the global symbol.
+
+        .. note::
+            This method can be used to access, for example, constant memory:
+
+            .. code-block:: python
+
+                # to get a pointer to "arr" declared in the source like this:
+                # __constant__ float arr[10];
+                memptr = mod.get_global("arr")
+                # ...wrap it using cupy.ndarray with a known shape
+                arr_ndarray = cp.ndarray((10,), cp.float32, memptr)
+                # ...perform data transfer to initialize it
+                arr_ndarray[...] = cp.random.random((10,), dtype=cp.float32)
+                # ...and arr is ready to be accessed by RawKernels
+
+        '''
+        from cupy.cuda.memory import MemoryPointer, UnownedMemory
+        cdef Module mod = self.module
+        ptr = mod.get_global_var(name)
+        # 1. unable to retrieve size, plus it's not used anywhere, so set to 0
+        # 2. it is safe to call getDevice() since self.module is cached on a
+        #    per-device basis
+        # 3. in CUDA, passing the device id saves us a look-up of the pointer
+        #    attributes; in ROCm, this is a must because there's a bug when
+        #    looking up a pointer to constant memory (hipErrorInvalidDevice)
+        cdef int dev = runtime.getDevice()
+        mem = UnownedMemory(ptr, 0, mod, dev)
+        memptr = MemoryPointer(mem, 0)
+        return memptr
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_raw_module(str code, str path, tuple options, str backend,
+                    bint translate_cucomplex,
+                    bint enable_cooperative_groups,
+                    bint jitify,
+                    tuple name_expressions,
+                    object log_stream):
+    cdef Module mod
+    if code is not None:
+        mod = cupy._core.core.compile_with_cache(
+            code, options, prepend_cupy_headers=False, backend=backend,
+            translate_cucomplex=translate_cucomplex,
+            enable_cooperative_groups=enable_cooperative_groups,
+            name_expressions=name_expressions,
+            log_stream=log_stream, jitify=jitify)
+    elif path is not None:
+        mod = Module()
+        mod.load_file(path)
+    return mod
diff --git a/cupy/_core/syncdetect.py b/cupy/_core/syncdetect.py
new file mode 100644
index 0000000..a6b2ee3
--- /dev/null
+++ b/cupy/_core/syncdetect.py
@@ -0,0 +1,68 @@
+import contextlib
+import threading
+import warnings
+
+
+_thread_local = threading.local()
+
+
+class DeviceSynchronized(RuntimeError):
+    """Raised when device synchronization is detected while disallowed.
+
+    .. warning::
+
+       This API has been deprecated in CuPy v10 and will be removed in future
+       releases.
+
+    .. seealso:: :func:`cupyx.allow_synchronize`
+
+    """
+
+    def __init__(self, message=None):
+        if message is None:
+            message = 'Device synchronization was detected while disallowed.'
+        super().__init__(message)
+
+
+def _is_allowed():
+    # Returns whether device synchronization is allowed in the current thread.
+    try:
+        return _thread_local.allowed
+    except AttributeError:
+        _thread_local.allowed = True
+        return True
+
+
+def _declare_synchronize():
+    # Raises DeviceSynchronized if device synchronization is disallowed in
+    # the current thread.
+    if not _is_allowed():
+        raise DeviceSynchronized()
+
+
+@contextlib.contextmanager
+def allow_synchronize(allow):
+    """Allows or disallows device synchronization temporarily in the current \
+thread.
+
+    .. warning::
+
+       This API has been deprecated in CuPy v10 and will be removed in future
+       releases.
+
+    If device synchronization is detected, :class:`cupyx.DeviceSynchronized`
+    will be raised.
+
+    Note that there can be false negatives and positives.
+    Device synchronization outside CuPy will not be detected.
+    """
+    warnings.warn(
+        'cupyx.allow_synchronize will be removed in future releases as it '
+        'is not possible to reliably detect synchronizations.')
+
+    old = _is_allowed()
+    _thread_local.allowed = allow
+    try:
+        yield
+    finally:
+        _thread_local.allowed = old
diff --git a/cupy/_creation/__init__.py b/cupy/_creation/__init__.py
new file mode 100644
index 0000000..c08f0eb
--- /dev/null
+++ b/cupy/_creation/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.array-creation.html
diff --git a/cupy/_creation/basic.py b/cupy/_creation/basic.py
new file mode 100644
index 0000000..32a5220
--- /dev/null
+++ b/cupy/_creation/basic.py
@@ -0,0 +1,314 @@
+import numpy
+
+import cupy
+from cupy._core.internal import _get_strides_for_order_K, _update_order_char
+
+
+def empty(shape, dtype=float, order='C'):
+    """Returns an array without initializing the elements.
+
+    Args:
+        shape (int or tuple of ints): Dimensionalities of the array.
+        dtype: Data type specifier.
+        order ({'C', 'F'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Returns:
+        cupy.ndarray: A new array with elements not initialized.
+
+    .. seealso:: :func:`numpy.empty`
+
+    """
+    return cupy.ndarray(shape, dtype, order=order)
+
+
+def _new_like_order_and_strides(
+        a, dtype, order, shape=None, *, get_memptr=True):
+    """
+    Determine order and strides as in NumPy's PyArray_NewLikeArray.
+
+    (see: numpy/core/src/multiarray/ctors.c)
+    """
+    order = order.upper()
+    if order not in ['C', 'F', 'K', 'A']:
+        raise ValueError('order not understood: {}'.format(order))
+
+    if numpy.isscalar(shape):
+        shape = (shape,)
+
+    # Fallback to c_contiguous if keep order and number of dimensions
+    # of new shape mismatch
+    if order == 'K' and shape is not None and len(shape) != a.ndim:
+        return 'C', None, None
+
+    order = chr(_update_order_char(
+        a.flags.c_contiguous, a.flags.f_contiguous, ord(order)))
+
+    if order == 'K':
+        strides = _get_strides_for_order_K(a, numpy.dtype(dtype), shape)
+        order = 'C'
+        memptr = cupy.empty(a.size, dtype=dtype).data if get_memptr else None
+        return order, strides, memptr
+    else:
+        return order, None, None
+
+
+def empty_like(a, dtype=None, order='K', subok=None, shape=None):
+    """Returns a new array with same shape and dtype of a given array.
+
+    This function currently does not support ``subok`` option.
+
+    Args:
+        a (cupy.ndarray): Base array.
+        dtype: Data type specifier. The data type of ``a`` is used by default.
+        order ({'C', 'F', 'A', or 'K'}): Overrides the memory layout of the
+            result. ``'C'`` means C-order, ``'F'`` means F-order, ``'A'`` means
+            ``'F'`` if ``a`` is Fortran contiguous, ``'C'`` otherwise.
+            ``'K'`` means match the layout of ``a`` as closely as possible.
+        subok: Not supported yet, must be None.
+        shape (int or tuple of ints): Overrides the shape of the result. If
+            ``order='K'`` and the number of dimensions is unchanged, will try
+            to keep order, otherwise, ``order='C'`` is implied.
+
+    Returns:
+        cupy.ndarray: A new array with same shape and dtype of ``a`` with
+        elements not initialized.
+
+    .. seealso:: :func:`numpy.empty_like`
+
+    """
+    if subok is not None:
+        raise TypeError('subok is not supported yet')
+    if dtype is None:
+        dtype = a.dtype
+
+    order, strides, memptr = _new_like_order_and_strides(a, dtype, order,
+                                                         shape)
+    shape = shape if shape else a.shape
+    return cupy.ndarray(shape, dtype, memptr, strides, order)
+
+
+def eye(N, M=None, k=0, dtype=float, order='C'):
+    """Returns a 2-D array with ones on the diagonals and zeros elsewhere.
+
+    Args:
+        N (int): Number of rows.
+        M (int): Number of columns. ``M == N`` by default.
+        k (int): Index of the diagonal. Zero indicates the main diagonal,
+            a positive index an upper diagonal, and a negative index a lower
+            diagonal.
+        dtype: Data type specifier.
+        order ({'C', 'F'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Returns:
+        cupy.ndarray: A 2-D array with given diagonals filled with ones and
+        zeros elsewhere.
+
+    .. seealso:: :func:`numpy.eye`
+
+    """
+    if M is None:
+        M = N
+    ret = zeros((N, M), dtype=dtype, order=order)
+    if k <= -N or k >= M:
+        return ret
+    ret.diagonal(k).fill(1)
+    return ret
+
+
+def identity(n, dtype=float):
+    """Returns a 2-D identity array.
+
+    It is equivalent to ``eye(n, n, dtype)``.
+
+    Args:
+        n (int): Number of rows and columns.
+        dtype: Data type specifier.
+
+    Returns:
+        cupy.ndarray: A 2-D identity array.
+
+    .. seealso:: :func:`numpy.identity`
+
+    """
+    return eye(n, dtype=dtype)
+
+
+def ones(shape, dtype=float, order='C'):
+    """Returns a new array of given shape and dtype, filled with ones.
+
+    This function currently does not support ``order`` option.
+
+    Args:
+        shape (int or tuple of ints): Dimensionalities of the array.
+        dtype: Data type specifier.
+        order ({'C', 'F'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Returns:
+        cupy.ndarray: An array filled with ones.
+
+    .. seealso:: :func:`numpy.ones`
+
+    """
+    a = cupy.ndarray(shape, dtype, order=order)
+    a.fill(1)
+    return a
+
+
+def ones_like(a, dtype=None, order='K', subok=None, shape=None):
+    """Returns an array of ones with same shape and dtype as a given array.
+
+    This function currently does not support ``subok`` option.
+
+    Args:
+        a (cupy.ndarray): Base array.
+        dtype: Data type specifier. The dtype of ``a`` is used by default.
+        order ({'C', 'F', 'A', or 'K'}): Overrides the memory layout of the
+            result. ``'C'`` means C-order, ``'F'`` means F-order, ``'A'`` means
+            ``'F'`` if ``a`` is Fortran contiguous, ``'C'`` otherwise.
+            ``'K'`` means match the layout of ``a`` as closely as possible.
+        subok: Not supported yet, must be None.
+        shape (int or tuple of ints): Overrides the shape of the result. If
+            ``order='K'`` and the number of dimensions is unchanged, will try
+            to keep order, otherwise, ``order='C'`` is implied.
+
+    Returns:
+        cupy.ndarray: An array filled with ones.
+
+    .. seealso:: :func:`numpy.ones_like`
+
+    """
+    if subok is not None:
+        raise TypeError('subok is not supported yet')
+    if dtype is None:
+        dtype = a.dtype
+
+    order, strides, memptr = _new_like_order_and_strides(a, dtype, order,
+                                                         shape)
+    shape = shape if shape else a.shape
+    a = cupy.ndarray(shape, dtype, memptr, strides, order)
+    a.fill(1)
+    return a
+
+
+def zeros(shape, dtype=float, order='C'):
+    """Returns a new array of given shape and dtype, filled with zeros.
+
+    Args:
+        shape (int or tuple of ints): Dimensionalities of the array.
+        dtype: Data type specifier.
+        order ({'C', 'F'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Returns:
+        cupy.ndarray: An array filled with zeros.
+
+    .. seealso:: :func:`numpy.zeros`
+
+    """
+    a = cupy.ndarray(shape, dtype, order=order)
+    a.data.memset_async(0, a.nbytes)
+    return a
+
+
+def zeros_like(a, dtype=None, order='K', subok=None, shape=None):
+    """Returns an array of zeros with same shape and dtype as a given array.
+
+    This function currently does not support ``subok`` option.
+
+    Args:
+        a (cupy.ndarray): Base array.
+        dtype: Data type specifier. The dtype of ``a`` is used by default.
+        order ({'C', 'F', 'A', or 'K'}): Overrides the memory layout of the
+            result. ``'C'`` means C-order, ``'F'`` means F-order, ``'A'`` means
+            ``'F'`` if ``a`` is Fortran contiguous, ``'C'`` otherwise.
+            ``'K'`` means match the layout of ``a`` as closely as possible.
+        subok: Not supported yet, must be None.
+        shape (int or tuple of ints): Overrides the shape of the result. If
+            ``order='K'`` and the number of dimensions is unchanged, will try
+            to keep order, otherwise, ``order='C'`` is implied.
+
+    Returns:
+        cupy.ndarray: An array filled with zeros.
+
+    .. seealso:: :func:`numpy.zeros_like`
+
+    """
+    if subok is not None:
+        raise TypeError('subok is not supported yet')
+    if dtype is None:
+        dtype = a.dtype
+
+    order, strides, memptr = _new_like_order_and_strides(a, dtype, order,
+                                                         shape)
+    shape = shape if shape else a.shape
+    a = cupy.ndarray(shape, dtype, memptr, strides, order)
+    a.data.memset_async(0, a.nbytes)
+    return a
+
+
+def full(shape, fill_value, dtype=None, order='C'):
+    """Returns a new array of given shape and dtype, filled with a given value.
+
+    This function currently does not support ``order`` option.
+
+    Args:
+        shape (int or tuple of ints): Dimensionalities of the array.
+        fill_value: A scalar value to fill a new array.
+        dtype: Data type specifier.
+        order ({'C', 'F'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Returns:
+        cupy.ndarray: An array filled with ``fill_value``.
+
+    .. seealso:: :func:`numpy.full`
+
+    """
+    if dtype is None:
+        if isinstance(fill_value, cupy.ndarray):
+            dtype = fill_value.dtype
+        else:
+            dtype = numpy.array(fill_value).dtype
+    a = cupy.ndarray(shape, dtype, order=order)
+    cupy.copyto(a, fill_value, casting='unsafe')
+    return a
+
+
+def full_like(a, fill_value, dtype=None, order='K', subok=None, shape=None):
+    """Returns a full array with same shape and dtype as a given array.
+
+    This function currently does not support ``subok`` option.
+
+    Args:
+        a (cupy.ndarray): Base array.
+        fill_value: A scalar value to fill a new array.
+        dtype: Data type specifier. The dtype of ``a`` is used by default.
+        order ({'C', 'F', 'A', or 'K'}): Overrides the memory layout of the
+            result. ``'C'`` means C-order, ``'F'`` means F-order, ``'A'`` means
+            ``'F'`` if ``a`` is Fortran contiguous, ``'C'`` otherwise.
+            ``'K'`` means match the layout of ``a`` as closely as possible.
+        subok: Not supported yet, must be None.
+        shape (int or tuple of ints): Overrides the shape of the result. If
+            ``order='K'`` and the number of dimensions is unchanged, will try
+            to keep order, otherwise, ``order='C'`` is implied.
+
+    Returns:
+        cupy.ndarray: An array filled with ``fill_value``.
+
+    .. seealso:: :func:`numpy.full_like`
+
+    """
+    if subok is not None:
+        raise TypeError('subok is not supported yet')
+    if dtype is None:
+        dtype = a.dtype
+
+    order, strides, memptr = _new_like_order_and_strides(a, dtype, order,
+                                                         shape)
+    shape = shape if shape else a.shape
+    a = cupy.ndarray(shape, dtype, memptr, strides, order)
+    cupy.copyto(a, fill_value, casting='unsafe')
+    return a
diff --git a/cupy/_creation/from_data.py b/cupy/_creation/from_data.py
new file mode 100644
index 0000000..fe61983
--- /dev/null
+++ b/cupy/_creation/from_data.py
@@ -0,0 +1,226 @@
+import numpy
+
+from cupy import _core
+from cupy._core import fusion
+
+
+def array(obj, dtype=None, copy=True, order='K', subok=False, ndmin=0):
+    """Creates an array on the current device.
+
+    This function currently does not support the ``subok`` option.
+
+    Args:
+        obj: :class:`cupy.ndarray` object or any other object that can be
+            passed to :func:`numpy.array`.
+        dtype: Data type specifier.
+        copy (bool): If ``False``, this function returns ``obj`` if possible.
+            Otherwise this function always returns a new array.
+        order ({'C', 'F', 'A', 'K'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+            When ``order`` is ``'A'``, it uses ``'F'`` if ``a`` is column-major
+            and uses ``'C'`` otherwise.
+            And when ``order`` is ``'K'``, it keeps strides as closely as
+            possible.
+            If ``obj`` is :class:`numpy.ndarray`, the function returns ``'C'``
+            or ``'F'`` order array.
+        subok (bool): If ``True``, then sub-classes will be passed-through,
+            otherwise the returned array will be forced to be a base-class
+            array (default).
+        ndmin (int): Minimum number of dimensions. Ones are inserted to the
+            head of the shape if needed.
+
+    Returns:
+        cupy.ndarray: An array on the current device.
+
+    .. note::
+       This method currently does not support ``subok`` argument.
+
+    .. note::
+       If ``obj`` is an `numpy.ndarray` instance that contains big-endian data,
+       this function automatically swaps its byte order to little-endian,
+       which is the NVIDIA and AMD GPU architecture's native use.
+
+    .. seealso:: :func:`numpy.array`
+
+    """
+    return _core.array(obj, dtype, copy, order, subok, ndmin)
+
+
+def asarray(a, dtype=None, order=None):
+    """Converts an object to array.
+
+    This is equivalent to ``array(a, dtype, copy=False, order=order)``.
+
+    Args:
+        a: The source object.
+        dtype: Data type specifier. It is inferred from the input by default.
+        order ({'C', 'F', 'A', 'K'}):
+            Whether to use row-major (C-style) or column-major (Fortran-style)
+            memory representation. Defaults to ``'K'``. ``order`` is ignored
+            for objects that are not :class:`cupy.ndarray`, but have the
+            ``__cuda_array_interface__`` attribute.
+
+    Returns:
+        cupy.ndarray: An array on the current device. If ``a`` is already on
+        the device, no copy is performed.
+
+    .. note::
+       If ``a`` is an `numpy.ndarray` instance that contains big-endian data,
+       this function automatically swaps its byte order to little-endian,
+       which is the NVIDIA and AMD GPU architecture's native use.
+
+    .. seealso:: :func:`numpy.asarray`
+
+    """
+    return _core.array(a, dtype, False, order)
+
+
+def asanyarray(a, dtype=None, order=None):
+    """Converts an object to array.
+
+    This is currently equivalent to :func:`cupy.asarray`, since there is no
+    subclass of :class:`cupy.ndarray` in CuPy. Note that the original
+    :func:`numpy.asanyarray` returns the input array as is if it is an instance
+    of a subtype of :class:`numpy.ndarray`.
+
+    .. seealso:: :func:`cupy.asarray`, :func:`numpy.asanyarray`
+
+    """
+    return _core.array(a, dtype, False, order)
+
+
+def ascontiguousarray(a, dtype=None):
+    """Returns a C-contiguous array.
+
+    Args:
+        a (cupy.ndarray): Source array.
+        dtype: Data type specifier.
+
+    Returns:
+        cupy.ndarray: If no copy is required, it returns ``a``. Otherwise, it
+        returns a copy of ``a``.
+
+    .. seealso:: :func:`numpy.ascontiguousarray`
+
+    """
+    return _core.ascontiguousarray(a, dtype)
+
+# TODO(okuta): Implement asmatrix
+
+
+def copy(a, order='K'):
+    """Creates a copy of a given array on the current device.
+
+    This function allocates the new array on the current device. If the given
+    array is allocated on the different device, then this function tries to
+    copy the contents over the devices.
+
+    Args:
+        a (cupy.ndarray): The source array.
+        order ({'C', 'F', 'A', 'K'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+            When ``order`` is ``'A'``, it uses ``'F'`` if ``a`` is column-major
+            and uses ``'C'`` otherwise.
+            And when ``order`` is ``'K'``, it keeps strides as closely as
+            possible.
+
+    Returns:
+        cupy.ndarray: The copy of ``a`` on the current device.
+
+    .. seealso:: :func:`numpy.copy`, :meth:`cupy.ndarray.copy`
+
+    """
+    if fusion._is_fusing():
+        if order != 'K':
+            raise NotImplementedError(
+                'cupy.copy does not support `order` in fusion yet.')
+        return fusion._call_ufunc(_core.elementwise_copy, a)
+
+    # If the current device is different from the device of ``a``, then this
+    # function allocates a new array on the current device, and copies the
+    # contents over the devices.
+    return a.copy(order=order)
+
+
+def frombuffer(*args, **kwargs):
+    """Interpret a buffer as a 1-dimensional array.
+
+    .. note::
+        Uses NumPy's ``frombuffer`` and coerces the result to a CuPy array.
+
+    .. seealso:: :func:`numpy.frombuffer`
+
+    """
+    return asarray(numpy.frombuffer(*args, **kwargs))
+
+
+def fromfile(*args, **kwargs):
+    """Reads an array from a file.
+
+    .. note::
+        Uses NumPy's ``fromfile`` and coerces the result to a CuPy array.
+
+    .. note::
+       If you let NumPy's ``fromfile`` read the file in big-endian, CuPy
+       automatically swaps its byte order to little-endian, which is the NVIDIA
+       and AMD GPU architecture's native use.
+
+    .. seealso:: :func:`numpy.fromfile`
+
+    """
+    return asarray(numpy.fromfile(*args, **kwargs))
+
+
+def fromfunction(*args, **kwargs):
+    """Construct an array by executing a function over each coordinate.
+
+    .. note::
+        Uses NumPy's ``fromfunction`` and coerces the result to a CuPy array.
+
+    .. seealso:: :func:`numpy.fromfunction`
+    """
+    return asarray(numpy.fromfunction(*args, **kwargs))
+
+
+def fromiter(*args, **kwargs):
+    """Create a new 1-dimensional array from an iterable object.
+
+    .. note::
+        Uses NumPy's ``fromiter`` and coerces the result to a CuPy array.
+
+    .. seealso:: :func:`numpy.fromiter`
+    """
+    return asarray(numpy.fromiter(*args, **kwargs))
+
+
+def fromstring(*args, **kwargs):
+    """A new 1-D array initialized from text data in a string.
+
+    .. note::
+        Uses NumPy's ``fromstring`` and coerces the result to a CuPy array.
+
+    .. seealso:: :func:`numpy.fromstring`
+    """
+    return asarray(numpy.fromstring(*args, **kwargs))
+
+
+def loadtxt(*args, **kwargs):
+    """Load data from a text file.
+
+    .. note::
+        Uses NumPy's ``loadtxt`` and coerces the result to a CuPy array.
+
+    .. seealso:: :func:`numpy.loadtxt`
+    """
+    return asarray(numpy.loadtxt(*args, **kwargs))
+
+
+def genfromtxt(*args, **kwargs):
+    """Load data from text file, with missing values handled as specified.
+
+    .. note::
+        Uses NumPy's ``genfromtxt`` and coerces the result to a CuPy array.
+
+    .. seealso:: :func:`numpy.genfromtxt`
+    """
+    return asarray(numpy.genfromtxt(*args, **kwargs))
diff --git a/cupy/_creation/matrix.py b/cupy/_creation/matrix.py
new file mode 100644
index 0000000..f2315d2
--- /dev/null
+++ b/cupy/_creation/matrix.py
@@ -0,0 +1,178 @@
+import numpy
+
+import cupy
+from cupy import _core
+
+
+def diag(v, k=0):
+    """Returns a diagonal or a diagonal array.
+
+    Args:
+        v (array-like): Array or array-like object.
+        k (int): Index of diagonals. Zero indicates the main diagonal, a
+            positive value an upper diagonal, and a negative value a lower
+            diagonal.
+
+    Returns:
+        cupy.ndarray: If ``v`` indicates a 1-D array, then it returns a 2-D
+        array with the specified diagonal filled by ``v``. If ``v`` indicates a
+        2-D array, then it returns the specified diagonal of ``v``. In latter
+        case, if ``v`` is a :class:`cupy.ndarray` object, then its view is
+        returned.
+
+    .. seealso:: :func:`numpy.diag`
+
+    """
+    if isinstance(v, cupy.ndarray):
+        ndim = v.ndim
+    else:
+        ndim = numpy.ndim(v)
+        if ndim == 1:
+            v = cupy.array(v)
+        if ndim == 2:
+            # to save bandwidth, don't copy non-diag elements to GPU
+            v = numpy.array(v)
+
+    if ndim == 1:
+        size = v.size + abs(k)
+        ret = cupy.zeros((size, size), dtype=v.dtype)
+        ret.diagonal(k)[:] = v
+        return ret
+    elif ndim == 2:
+        return cupy.array(v.diagonal(k))
+    else:
+        raise ValueError('Input must be 1- or 2-d.')
+
+
+def diagflat(v, k=0):
+    """Creates a diagonal array from the flattened input.
+
+    Args:
+        v (array-like): Array or array-like object.
+        k (int): Index of diagonals. See :func:`cupy.diag` for detail.
+
+    Returns:
+        cupy.ndarray: A 2-D diagonal array with the diagonal copied from ``v``.
+
+    .. seealso:: :func:`numpy.diagflat`
+
+    """
+    if numpy.isscalar(v):
+        v = numpy.asarray(v)
+
+    return cupy.diag(v.ravel(), k)
+
+
+_tri_kernel = _core.ElementwiseKernel(
+    'int32 m, int32 k',
+    'T out',
+    '''
+    int row = i / m;
+    int col = i % m;
+    out = (col <= row + k);
+    ''',
+    'cupy_tri',
+)
+
+
+def tri(N, M=None, k=0, dtype=float):
+    """Creates an array with ones at and below the given diagonal.
+
+    Args:
+        N (int): Number of rows.
+        M (int): Number of columns. ``M == N`` by default.
+        k (int): The sub-diagonal at and below which the array is filled. Zero
+            is the main diagonal, a positive value is above it, and a negative
+            value is below.
+        dtype: Data type specifier.
+
+    Returns:
+        cupy.ndarray: An array with ones at and below the given diagonal.
+
+    .. seealso:: :func:`numpy.tri`
+
+    """
+    if M is None:
+        M = N
+    out = cupy.empty((N, M), dtype=dtype)
+
+    return _tri_kernel(M, k, out)
+
+
+def tril(m, k=0):
+    """Returns a lower triangle of an array.
+
+    Args:
+        m (array-like): Array or array-like object.
+        k (int): The diagonal above which to zero elements. Zero is the main
+            diagonal, a positive value is above it, and a negative value is
+            below.
+
+    Returns:
+        cupy.ndarray: A lower triangle of an array.
+
+    .. seealso:: :func:`numpy.tril`
+
+    """
+    m = cupy.asarray(m)
+    mask = tri(*m.shape[-2:], k=k, dtype=bool)
+
+    return cupy.where(mask, m, m.dtype.type(0))
+
+
+def triu(m, k=0):
+    """Returns an upper triangle of an array.
+
+    Args:
+        m (array-like): Array or array-like object.
+        k (int): The diagonal below which to zero elements. Zero is the main
+            diagonal, a positive value is above it, and a negative value is
+            below.
+
+    Returns:
+        cupy.ndarray: An upper triangle of an array.
+
+    .. seealso:: :func:`numpy.triu`
+
+    """
+    m = cupy.asarray(m)
+    mask = tri(*m.shape[-2:], k=k-1, dtype=bool)
+
+    return cupy.where(mask, m.dtype.type(0), m)
+
+
+def vander(x, N=None, increasing=False):
+    """Returns a Vandermonde matrix.
+
+    Args:
+        x (array-like): 1-D array or array-like object.
+        N (int, optional): Number of columns in the output.
+            ``N = len(x)`` by default.
+        increasing (bool, optional): Order of the powers of the columns.
+            If True, the powers increase from right to left,
+            if False (the default) they are reversed.
+
+    Returns:
+        cupy.ndarray: A Vandermonde matrix.
+
+    .. seealso:: :func:`numpy.vander`
+
+    """
+    x = cupy.asarray(x)
+    if x.ndim != 1:
+        raise ValueError("x must be a one-dimensional array or sequence.")
+    if N is None:
+        N = len(x)
+
+    v = cupy.empty((len(x), N), dtype=numpy.promote_types(x.dtype, int))
+    tmp = v[:, ::-1] if not increasing else v
+
+    cupy.power(x.reshape(-1, 1), cupy.arange(N), out=tmp)
+
+    return v
+
+
+# TODO(okuta): Implement mat
+
+
+# TODO(okuta): Implement bmat
diff --git a/cupy/_creation/ranges.py b/cupy/_creation/ranges.py
new file mode 100644
index 0000000..d01195f
--- /dev/null
+++ b/cupy/_creation/ranges.py
@@ -0,0 +1,442 @@
+import math
+
+import numpy
+
+import cupy
+from cupy import _core
+
+
+def arange(start, stop=None, step=1, dtype=None):
+    """Returns an array with evenly spaced values within a given interval.
+
+    Values are generated within the half-open interval [start, stop). The first
+    three arguments are mapped like the ``range`` built-in function, i.e. start
+    and step are optional.
+
+    Args:
+        start: Start of the interval.
+        stop: End of the interval.
+        step: Step width between each pair of consecutive values.
+        dtype: Data type specifier. It is inferred from other arguments by
+            default.
+
+    Returns:
+        cupy.ndarray: The 1-D array of range values.
+
+    .. seealso:: :func:`numpy.arange`
+
+    """
+    if dtype is None:
+        if any(numpy.dtype(type(val)).kind == 'f'
+               for val in (start, stop, step)):
+            dtype = float
+        else:
+            dtype = int
+
+    if stop is None:
+        stop = start
+        start = 0
+
+    if step is None:
+        step = 1
+
+    size = int(numpy.ceil((stop - start) / step))
+    if size <= 0:
+        return cupy.empty((0,), dtype=dtype)
+
+    if numpy.dtype(dtype).type == numpy.bool_:
+        if size > 2:
+            raise TypeError(
+                'arange() is only supported for booleans '
+                'when the result has at most length 2.'
+            )
+        if size == 2:
+            return cupy.array([start, start - step], dtype=numpy.bool_)
+        else:
+            return cupy.array([start], dtype=numpy.bool_)
+
+    ret = cupy.empty((size,), dtype=dtype)
+    typ = numpy.dtype(dtype).type
+    _arange_ufunc(typ(start), typ(step), ret, dtype=dtype)
+    return ret
+
+
+def _linspace_scalar(start, stop, num=50, endpoint=True, retstep=False,
+                     dtype=None):
+    """Returns an array with evenly-spaced values within a given interval.
+
+    Instead of specifying the step width like :func:`cupy.arange`, this
+    function requires the total number of elements specified.
+
+    Args:
+        start: Start of the interval.
+        stop: End of the interval.
+        num: Number of elements.
+        endpoint (bool): If ``True``, the stop value is included as the last
+            element. Otherwise, the stop value is omitted.
+        retstep (bool): If ``True``, this function returns (array, step).
+            Otherwise, it returns only the array.
+        dtype: Data type specifier. It is inferred from the start and stop
+            arguments by default.
+
+    Returns:
+        cupy.ndarray: The 1-D array of ranged values.
+
+    """
+    dt = cupy.result_type(start, stop, float(num))
+    if dtype is None:
+        # In actual implementation, only float is used
+        dtype = dt
+
+    ret = cupy.empty((num,), dtype=dt)
+    div = (num - 1) if endpoint else num
+    if div <= 0:
+        if num > 0:
+            ret.fill(start)
+        step = float('nan')
+    else:
+        step = float(stop - start) / div
+        stop = float(stop)
+
+        if step == 0.0:
+            # for underflow
+            _linspace_ufunc_underflow(start, stop - start, div, ret)
+        else:
+            _linspace_ufunc(start, step, ret)
+
+        if endpoint:
+            # Here num == div + 1 > 1 is ensured.
+            ret[-1] = stop
+
+    if cupy.issubdtype(dtype, cupy.integer):
+        cupy.floor(ret, out=ret)
+
+    ret = ret.astype(dtype, copy=False)
+
+    if retstep:
+        return ret, step
+    else:
+        return ret
+
+
+def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None,
+             axis=0):
+    """Returns an array with evenly-spaced values within a given interval.
+
+    Instead of specifying the step width like :func:`cupy.arange`, this
+    function requires the total number of elements specified.
+
+    Args:
+        start (scalar or array_like): Starting value(s) of the sequence.
+        stop (scalar or array_like): Ending value(s) of the sequence, unless
+            ``endpoint`` is set to ``False``. In that case, the sequence
+            consists of all but the last of ``num + 1`` evenly spaced samples,
+            so that ``stop`` is excluded.  Note that the step size changes when
+            ``endpoint`` is ``False``.
+        num: Number of elements.
+        endpoint (bool): If ``True``, the stop value is included as the last
+            element. Otherwise, the stop value is omitted.
+        retstep (bool): If ``True``, this function returns (array, step).
+            Otherwise, it returns only the array.
+        dtype: Data type specifier. It is inferred from the start and stop
+            arguments by default.
+        axis (int):  The axis in the result to store the samples.  Relevant
+            only if start or stop are array-like.  By default ``0``, the
+            samples will be along a new axis inserted at the beginning.
+            Use ``-1`` to get an axis at the end.
+
+    Returns:
+        cupy.ndarray: The 1-D array of ranged values.
+
+    .. seealso:: :func:`numpy.linspace`
+
+    """
+    if num < 0:
+        raise ValueError('linspace with num<0 is not supported')
+    div = (num - 1) if endpoint else num
+
+    scalar_start = cupy.isscalar(start)
+    scalar_stop = cupy.isscalar(stop)
+    if scalar_start and scalar_stop:
+        return _linspace_scalar(start, stop, num, endpoint, retstep, dtype)
+
+    if not scalar_start:
+        if not (isinstance(start, cupy.ndarray) and start.dtype.kind == 'f'):
+            start = cupy.asarray(start) * 1.0
+
+    if not scalar_stop:
+        if not (isinstance(stop, cupy.ndarray) and stop.dtype.kind == 'f'):
+            stop = cupy.asarray(stop) * 1.0
+
+    dt = cupy.result_type(start, stop, float(num))
+    if dtype is None:
+        # In actual implementation, only float is used
+        dtype = dt
+
+    delta = stop - start
+
+    # ret = cupy.arange(0, num, dtype=dt).reshape((-1,) + (1,) * delta.ndim)
+    ret = cupy.empty((num,), dtype=dt)
+    _arange_ufunc(0.0, 1.0, ret, dtype=dt)
+    ret = ret.reshape((-1,) + (1,) * delta.ndim)
+
+    # In-place multiplication y *= delta/div is faster, but prevents the
+    # multiplicant from overriding what class is produced, and thus prevents,
+    # e.g. use of Quantities, see numpy#7142. Hence, we multiply in place only
+    # for standard scalar types.
+    if num > 1:
+        step = delta / div
+        if cupy.any(step == 0):
+            # Special handling for denormal numbers, numpy#5437
+            ret /= div
+            ret = ret * delta
+        else:
+            ret = ret * step
+    else:
+        # 0 and 1 item long sequences have an undefined step
+        step = float('nan')
+        # Multiply with delta to allow possible override of output class.
+        ret = ret * delta
+
+    ret += start
+    if endpoint and num > 1:
+        ret[-1] = stop
+
+    if axis != 0:
+        ret = cupy.moveaxis(ret, 0, axis)
+
+    if cupy.issubdtype(dtype, cupy.integer):
+        cupy.floor(ret, out=ret)
+
+    ret = ret.astype(dtype, copy=False)
+
+    if retstep:
+        return ret, step
+    else:
+        return ret
+
+
+def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None,
+             axis=0):
+    """Returns an array with evenly-spaced values on a log-scale.
+
+    Instead of specifying the step width like :func:`cupy.arange`, this
+    function requires the total number of elements specified.
+
+    Args:
+        start: Start of the interval.
+        stop: End of the interval.
+        num: Number of elements.
+        endpoint (bool): If ``True``, the stop value is included as the last
+            element. Otherwise, the stop value is omitted.
+        base (float): Base of the log space. The step sizes between the
+            elements on a log-scale are the same as ``base``.
+        dtype: Data type specifier. It is inferred from the start and stop
+            arguments by default.
+        axis (int):  The axis in the result to store the samples.  Relevant
+            only if start or stop are array-like.  By default ``0``, the
+            samples will be along a new axis inserted at the beginning.
+            Use ``-1`` to get an axis at the end.
+    Returns:
+        cupy.ndarray: The 1-D array of ranged values.
+
+    .. seealso:: :func:`numpy.logspace`
+
+    """
+    y = linspace(start, stop, num=num, endpoint=endpoint, axis=axis)
+    if dtype is None:
+        return _core.power(base, y)
+    # This is to avoid cupy strange behaviors such as
+    # cupy.power(10.0, cupy.array(2.0, dtype=cupy.float32)) == 99.9999
+    # numpy.power(10.0, cupy.array(2.0, dtype=numpy.float32)) == 100.0
+    if cupy.dtype(dtype).kind in 'iu':
+        y = y.astype(cupy.float64)
+    return _core.power(base, y).astype(dtype)
+
+
+def meshgrid(*xi, **kwargs):
+    """Return coordinate matrices from coordinate vectors.
+
+    Given one-dimensional coordinate arrays ``x1, x2, ... , xn`` this
+    function makes N-D grids.
+
+    For one-dimensional arrays ``x1, x2, ... , xn`` with lengths
+    ``Ni = len(xi)``, this function returns ``(N1, N2, N3, ..., Nn)`` shaped
+    arrays if indexing='ij' or ``(N2, N1, N3, ..., Nn)`` shaped arrays
+    if indexing='xy'.
+
+    Unlike NumPy, CuPy currently only supports 1-D arrays as inputs.
+
+    Args:
+        xi (tuple of ndarrays): 1-D arrays representing the coordinates
+            of a grid.
+        indexing ({'xy', 'ij'}, optional): Cartesian ('xy', default) or
+            matrix ('ij') indexing of output.
+        sparse (bool, optional): If ``True``, a sparse grid is returned in
+            order to conserve memory. Default is ``False``.
+        copy (bool, optional): If ``False``, a view
+            into the original arrays are returned. Default is ``True``.
+
+    Returns:
+        list of cupy.ndarray
+
+    .. seealso:: :func:`numpy.meshgrid`
+
+    """
+    indexing = kwargs.pop('indexing', 'xy')
+    copy = bool(kwargs.pop('copy', True))
+    sparse = bool(kwargs.pop('sparse', False))
+    if kwargs:
+        raise TypeError(
+            'meshgrid() got an unexpected keyword argument \'{}\''.format(
+                list(kwargs)[0]))
+    if indexing not in ['xy', 'ij']:
+        raise ValueError('Valid values for `indexing` are \'xy\' and \'ij\'.')
+
+    for x in xi:
+        if x.ndim != 1:
+            raise ValueError('input has to be 1d')
+        if not isinstance(x, cupy.ndarray):
+            raise ValueError('input has to be cupy.ndarray')
+    if len(xi) <= 1:
+        return list(xi)
+
+    meshes = []
+    for i, x in enumerate(xi):
+        if indexing == 'xy' and i == 0:
+            left_none = 1
+        elif indexing == 'xy' and i == 1:
+            left_none = 0
+        else:
+            left_none = i
+
+        expand_slices = ((None,) * left_none +
+                         (slice(None),) +
+                         (None,) * (len(xi) - (left_none + 1)))
+        meshes.append(x[expand_slices])
+
+    if sparse:
+        meshes_br = meshes
+    else:
+        meshes_br = list(cupy.broadcast_arrays(*meshes))
+
+    if copy:
+        for i in range(len(meshes_br)):
+            meshes_br[i] = meshes_br[i].copy()
+    return meshes_br
+
+
+class nd_grid(object):
+    """Construct a multi-dimensional "meshgrid".
+
+    ``grid = nd_grid()`` creates an instance which will return a mesh-grid
+    when indexed.  The dimension and number of the output arrays are equal
+    to the number of indexing dimensions.  If the step length is not a
+    complex number, then the stop is not inclusive.
+
+    However, if the step length is a **complex number** (e.g. 5j), then the
+    integer part of its magnitude is interpreted as specifying the
+    number of points to create between the start and stop values, where
+    the stop value **is inclusive**.
+
+    If instantiated with an argument of ``sparse=True``, the mesh-grid is
+    open (or not fleshed out) so that only one-dimension of each returned
+    argument is greater than 1.
+
+    Args:
+        sparse (bool, optional): Whether the grid is sparse or not.
+            Default is False.
+
+    .. seealso:: :data:`numpy.mgrid` and :data:`numpy.ogrid`
+
+    """
+
+    def __init__(self, sparse=False):
+        self.sparse = sparse
+
+    def __getitem__(self, key):
+        if isinstance(key, slice):
+            step = key.step
+            stop = key.stop
+            start = key.start
+            if start is None:
+                start = 0
+            if isinstance(step, complex):
+                step = abs(step)
+                length = int(step)
+                if step != 1:
+                    step = (key.stop - start) / float(step - 1)
+                stop = key.stop + step
+                return cupy.arange(0, length, 1, float) * step + start
+            else:
+                return cupy.arange(start, stop, step)
+
+        size = []
+        typ = int
+        for k in range(len(key)):
+            step = key[k].step
+            start = key[k].start
+            if start is None:
+                start = 0
+            if step is None:
+                step = 1
+            if isinstance(step, complex):
+                size.append(int(abs(step)))
+                typ = float
+            else:
+                size.append(
+                    int(math.ceil((key[k].stop - start) / (step * 1.0))))
+            if (isinstance(step, float) or
+                    isinstance(start, float) or
+                    isinstance(key[k].stop, float)):
+                typ = float
+        if self.sparse:
+            nn = [cupy.arange(_x, dtype=_t)
+                  for _x, _t in zip(size, (typ,) * len(size))]
+        else:
+            nn = cupy.indices(size, typ)
+        for k in range(len(size)):
+            step = key[k].step
+            start = key[k].start
+            if start is None:
+                start = 0
+            if step is None:
+                step = 1
+            if isinstance(step, complex):
+                step = int(abs(step))
+                if step != 1:
+                    step = (key[k].stop - start) / float(step - 1)
+            nn[k] = (nn[k] * step + start)
+        if self.sparse:
+            slobj = [cupy.newaxis] * len(size)
+            for k in range(len(size)):
+                slobj[k] = slice(None, None)
+                nn[k] = nn[k][tuple(slobj)]
+                slobj[k] = cupy.newaxis
+        return nn
+
+    def __len__(self):
+        return 0
+
+
+mgrid = nd_grid(sparse=False)
+ogrid = nd_grid(sparse=True)
+
+
+_arange_ufunc = _core.create_ufunc(
+    'cupy_arange',
+    ('bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l', 'LL->L',
+     'qq->q', 'QQ->Q', 'ee->e', 'ff->f', 'dd->d',
+     ('FF->F', 'out0 = in0 + float(i) * in1'),
+     ('DD->D', 'out0 = in0 + double(i) * in1')),
+    'out0 = in0 + i * in1')
+
+_linspace_ufunc = _core.create_ufunc(
+    'cupy_linspace',
+    ('dd->d',),
+    'out0 = in0 + i * in1')
+
+_linspace_ufunc_underflow = _core.create_ufunc(
+    'cupy_linspace',
+    ('ddd->d',),
+    'out0 = in0 + i * in1 / in2')
diff --git a/cupy/_environment.py b/cupy/_environment.py
new file mode 100644
index 0000000..397cdff
--- /dev/null
+++ b/cupy/_environment.py
@@ -0,0 +1,529 @@
+"""
+This file must not depend on any other CuPy modules.
+"""
+
+import ctypes
+import json
+import os
+import os.path
+import shutil
+import sys
+from typing import Any, Dict, Optional
+import warnings
+
+
+# '' for uninitialized, None for non-existing
+_cuda_path = ''
+_nvcc_path = ''
+_rocm_path = ''
+_hipcc_path = ''
+_cub_path = ''
+
+"""
+Library Preloading
+------------------
+
+Wheel packages are built against specific versions of CUDA libraries
+(cuTENSOR/NCCL/cuDNN).
+To avoid loading wrong version, these shared libraries are manually
+preloaded.
+
+# TODO(kmaehashi): Support NCCL
+
+Example of `_preload_config` is as follows:
+
+{
+    # installation source
+    'packaging': 'pip',
+
+    # CUDA version string
+    'cuda': '11.0',
+
+    'cudnn': {
+        # cuDNN version string
+        'version': '8.0.0',
+
+        # names of the shared library
+        'filenames': ['libcudnn.so.X.Y.Z']  # or `cudnn64_X.dll` for Windows
+    }
+}
+
+The configuration file is intended solely for internal purposes and
+not expected to be parsed by end-users.
+"""
+
+_preload_config = None
+
+_preload_libs = {
+    'cudnn': None,
+    'nccl': None,
+    'cutensor': None,
+}
+
+_preload_logs = []
+
+
+def _log(msg):
+    # TODO(kmaehashi): replace with the standard logging
+    _preload_logs.append(msg)
+
+
+def get_cuda_path():
+    # Returns the CUDA installation path or None if not found.
+    global _cuda_path
+    if _cuda_path == '':
+        _cuda_path = _get_cuda_path()
+    return _cuda_path
+
+
+def get_nvcc_path():
+    # Returns the path to the nvcc command or None if not found.
+    global _nvcc_path
+    if _nvcc_path == '':
+        _nvcc_path = _get_nvcc_path()
+    return _nvcc_path
+
+
+def get_rocm_path():
+    # Returns the ROCm installation path or None if not found.
+    global _rocm_path
+    if _rocm_path == '':
+        _rocm_path = _get_rocm_path()
+    return _rocm_path
+
+
+def get_hipcc_path():
+    # Returns the path to the hipcc command or None if not found.
+    global _hipcc_path
+    if _hipcc_path == '':
+        _hipcc_path = _get_hipcc_path()
+    return _hipcc_path
+
+
+def get_cub_path():
+    # Returns the CUB header path or None if not found.
+    global _cub_path
+    if _cub_path == '':
+        _cub_path = _get_cub_path()
+    return _cub_path
+
+
+def _get_cuda_path():
+    # Use environment variable
+    cuda_path = os.environ.get('CUDA_PATH', '')  # Nvidia default on Windows
+    if os.path.exists(cuda_path):
+        return cuda_path
+
+    # Use nvcc path
+    nvcc_path = shutil.which('nvcc')
+    if nvcc_path is not None:
+        return os.path.dirname(os.path.dirname(nvcc_path))
+
+    # Use typical path
+    if os.path.exists('/usr/local/cuda'):
+        return '/usr/local/cuda'
+
+    return None
+
+
+def _get_nvcc_path():
+    # Honor the "NVCC" env var
+    nvcc_path = os.environ.get('NVCC', None)
+    if nvcc_path is not None:
+        return nvcc_path
+
+    # Lookup <CUDA>/bin
+    cuda_path = get_cuda_path()
+    if cuda_path is None:
+        return None
+
+    return shutil.which('nvcc', path=os.path.join(cuda_path, 'bin'))
+
+
+def _get_rocm_path():
+    # Use environment variable
+    rocm_path = os.environ.get('ROCM_HOME', '')
+    if os.path.exists(rocm_path):
+        return rocm_path
+
+    # Use hipcc path
+    hipcc_path = shutil.which('hipcc')
+    if hipcc_path is not None:
+        return os.path.dirname(os.path.dirname(hipcc_path))
+
+    # Use typical path
+    if os.path.exists('/opt/rocm'):
+        return '/opt/rocm'
+
+    return None
+
+
+def _get_hipcc_path():
+    # TODO(leofang): Introduce an env var HIPCC?
+
+    # Lookup <ROCM>/bin
+    rocm_path = get_rocm_path()
+    if rocm_path is None:
+        return None
+
+    return shutil.which('hipcc', path=os.path.join(rocm_path, 'bin'))
+
+
+def _get_cub_path():
+    # runtime discovery of CUB headers
+    from cupy_backends.cuda.api import runtime
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+
+    if not runtime.is_hip:
+        cuda_path = get_cuda_path()
+        if os.path.isdir(os.path.join(current_dir, '_core/include/cupy/cub')):
+            _cub_path = '<bundle>'
+        elif cuda_path is not None and os.path.isdir(
+                os.path.join(cuda_path, 'include/cub')):
+            # use built-in CUB for CUDA 11+
+            _cub_path = '<CUDA>'
+        else:
+            _cub_path = None
+    else:
+        # the bundled CUB does not work in ROCm
+        rocm_path = get_rocm_path()
+        if rocm_path is not None and os.path.isdir(
+                os.path.join(rocm_path, 'include/hipcub')):
+            # use hipCUB
+            _cub_path = '<ROCm>'
+        else:
+            _cub_path = None
+    return _cub_path
+
+
+def _setup_win32_dll_directory():
+    # Setup DLL directory to load CUDA Toolkit libs and shared libraries
+    # added during the build process.
+    if sys.platform.startswith('win32'):
+        # see _can_attempt_preload()
+        config = get_preload_config()
+        is_conda = (config is not None and (config['packaging'] == 'conda'))
+
+        # Path to the CUDA Toolkit binaries
+        cuda_path = get_cuda_path()
+        if cuda_path is not None:
+            if is_conda:
+                cuda_bin_path = cuda_path
+            else:
+                cuda_bin_path = os.path.join(cuda_path, 'bin')
+        else:
+            cuda_bin_path = None
+            warnings.warn(
+                'CUDA path could not be detected.'
+                ' Set CUDA_PATH environment variable if CuPy fails to load.')
+        _log('CUDA_PATH: {}'.format(cuda_path))
+
+        # Path to shared libraries in wheel
+        wheel_libdir = os.path.join(
+            get_cupy_install_path(), 'cupy', '.data', 'lib')
+        if os.path.isdir(wheel_libdir):
+            _log('Wheel shared libraries: {}'.format(wheel_libdir))
+        else:
+            _log('Not wheel distribution ({} not found)'.format(
+                wheel_libdir))
+            wheel_libdir = None
+
+        if (3, 8) <= sys.version_info:
+            if cuda_bin_path is not None:
+                _log('Adding DLL search path: {}'.format(cuda_bin_path))
+                os.add_dll_directory(cuda_bin_path)
+            if wheel_libdir is not None:
+                _log('Adding DLL search path: {}'.format(wheel_libdir))
+                os.add_dll_directory(wheel_libdir)
+        else:
+            # Users are responsible for adding `%CUDA_PATH%/bin` to PATH.
+            if wheel_libdir is not None:
+                _log('Adding to PATH: {}'.format(wheel_libdir))
+                path = os.environ.get('PATH', '')
+                os.environ['PATH'] = wheel_libdir + os.pathsep + path
+
+
+def get_cupy_install_path():
+    # Path to the directory where the package is installed.
+    return os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..'))
+
+
+def get_cupy_cuda_lib_path():
+    """Returns the directory where CUDA external libraries are installed.
+
+    This environment variable only affects wheel installations.
+
+    Shared libraries are looked up from
+    `$CUPY_CUDA_LIB_PATH/$CUDA_VER/$LIB_NAME/$LIB_VER/{lib,lib64,bin}`,
+    e.g., `~/.cupy/cuda_lib/11.2/cudnn/8.1.1/lib64/libcudnn.so.8.1.1`.
+
+    The default $CUPY_CUDA_LIB_PATH is `~/.cupy/cuda_lib`.
+    """
+    cupy_cuda_lib_path = os.environ.get('CUPY_CUDA_LIB_PATH', None)
+    if cupy_cuda_lib_path is None:
+        return os.path.expanduser('~/.cupy/cuda_lib')
+    return os.path.abspath(cupy_cuda_lib_path)
+
+
+def get_preload_config() -> Optional[Dict[str, Any]]:
+    global _preload_config
+    if _preload_config is None:
+        _preload_config = _get_json_data('_wheel.json')
+    return _preload_config
+
+
+def _get_json_data(name: str) -> Optional[Dict[str, Any]]:
+    config_path = os.path.join(
+        get_cupy_install_path(), 'cupy', '.data', name)
+    if not os.path.exists(config_path):
+        return None
+    with open(config_path) as f:
+        return json.load(f)
+
+
+def _can_attempt_preload(lib: str) -> bool:
+    """Returns if the preload can be attempted."""
+
+    config = get_preload_config()
+    if (config is None) or (config['packaging'] == 'conda'):
+        # We don't do preload if CuPy is installed from Conda-Forge, as we
+        # cannot guarantee the version pinned in _wheel.json, which is
+        # encoded in config[lib]['filenames'], is always available on
+        # Conda-Forge. See here for the configuration files used in
+        # Conda-Forge distributions.
+        # https://github.com/conda-forge/cupy-feedstock/blob/master/recipe/preload_config/
+        _log(f'Cannot preload {lib} as this is not a wheel installation')
+        return False
+
+    if lib not in _preload_libs:
+        raise AssertionError(f'Unknown preload library: {lib}')
+
+    if lib not in config:
+        _log(f'Preload {lib} not configured in wheel')
+        return False
+
+    if _preload_libs[lib] is not None:
+        _log(f'Preload already attempted: {lib}')
+        return False
+
+    return True
+
+
+def _preload_library(lib):
+    """Preload dependent shared libraries.
+
+    The preload configuration file (cupy/.data/_wheel.json) will be added
+    during the wheel build process.
+    """
+
+    _log(f'Preloading triggered for library: {lib}')
+
+    if not _can_attempt_preload(lib):
+        return
+    _preload_libs[lib] = {}
+
+    config = get_preload_config()
+    cuda_version = config['cuda']
+    _log('CuPy wheel package built for CUDA {}'.format(cuda_version))
+
+    cupy_cuda_lib_path = get_cupy_cuda_lib_path()
+    _log('CuPy CUDA library directory: {}'.format(cupy_cuda_lib_path))
+
+    version = config[lib]['version']
+    filenames = config[lib]['filenames']
+    for filename in filenames:
+        _log(f'Looking for {lib} version {version} ({filename})')
+
+        # "lib": cuTENSOR (Linux/Windows) / NCCL (Linux)
+        # "lib64": cuDNN (Linux)
+        # "bin": cuDNN (Windows)
+        libpath_cands = [
+            os.path.join(
+                cupy_cuda_lib_path, config['cuda'], lib, version, x,
+                filename)
+            for x in ['lib', 'lib64', 'bin']]
+        for libpath in libpath_cands:
+            if not os.path.exists(libpath):
+                _log('Rejected candidate (not found): {}'.format(libpath))
+                continue
+
+            try:
+                _log(f'Trying to load {libpath}')
+                # Keep reference to the preloaded module.
+                _preload_libs[lib][libpath] = ctypes.CDLL(libpath)
+                _log('Loaded')
+                break
+            except Exception as e:
+                e_type = type(e).__name__  # NOQA
+                msg = (
+                    f'CuPy failed to preload library ({libpath}): '
+                    f'{e_type} ({e})')
+                _log(msg)
+                warnings.warn(msg)
+        else:
+            _log('File {} could not be found'.format(filename))
+
+            # Lookup library with fully-qualified version (e.g.,
+            # `libcudnn.so.X.Y.Z`).
+            _log(f'Trying to load {filename} from default search path')
+            try:
+                _preload_libs[lib][filename] = ctypes.CDLL(filename)
+                _log('Loaded')
+            except Exception as e:
+                # Fallback to the standard shared library lookup which only
+                # uses the major version (e.g., `libcudnn.so.X`).
+                _log(f'Library {lib} could not be preloaded: {e}')
+
+
+def _get_preload_logs():
+    return '\n'.join(_preload_logs)
+
+
+def _preload_warning(lib, exc):
+    config = get_preload_config()
+    if config is not None and lib in config:
+        msg = '''
+{lib} library could not be loaded.
+
+Reason: {exc_type} ({exc})
+
+You can install the library by:
+'''
+        if config['packaging'] == 'pip':
+            msg += '''
+  $ python -m cupyx.tools.install_library --library {lib} --cuda {cuda}
+'''
+        elif config['packaging'] == 'conda':
+            msg += '''
+  $ conda install -c conda-forge {lib}
+'''
+        else:
+            raise AssertionError
+        msg = msg.format(
+            lib=lib, exc_type=type(exc).__name__, exc=str(exc),
+            cuda=config['cuda'])
+        warnings.warn(msg)
+
+
+def _detect_duplicate_installation():
+    # importlib.metadata only available in Python 3.8+.
+    if sys.version_info < (3, 8):
+        return
+    import importlib.metadata
+
+    # List of all CuPy packages, including out-dated ones.
+    known = [
+        'cupy',
+        'cupy-cuda80',
+        'cupy-cuda90',
+        'cupy-cuda91',
+        'cupy-cuda92',
+        'cupy-cuda100',
+        'cupy-cuda101',
+        'cupy-cuda102',
+        'cupy-cuda110',
+        'cupy-cuda111',
+        'cupy-cuda112',
+        'cupy-cuda113',
+        'cupy-cuda114',
+        'cupy-cuda115',
+        'cupy-cuda116',
+        'cupy-cuda117',
+        'cupy-cuda118',
+        'cupy-cuda11x',
+        'cupy-cuda12x',
+        'cupy-rocm-4-0',
+        'cupy-rocm-4-1',
+        'cupy-rocm-4-2',
+        'cupy-rocm-4-3',
+        'cupy-rocm-5-0',
+    ]
+    cupy_installed = [
+        name for name in known
+        if list(importlib.metadata.distributions(name=name))]
+    if 1 < len(cupy_installed):
+        cupy_packages_list = ', '.join(sorted(cupy_installed))
+        warnings.warn(f'''
+--------------------------------------------------------------------------------
+
+  CuPy may not function correctly because multiple CuPy packages are installed
+  in your environment:
+
+    {cupy_packages_list}
+
+  Follow these steps to resolve this issue:
+
+    1. For all packages listed above, run the following command to remove all
+       existing CuPy installations:
+
+         $ pip uninstall <package_name>
+
+      If you previously installed CuPy via conda, also run the following:
+
+         $ conda uninstall cupy
+
+    2. Install the appropriate CuPy package.
+       Refer to the Installation Guide for detailed instructions.
+
+         https://docs.cupy.dev/en/stable/install.html
+
+--------------------------------------------------------------------------------
+''')
+
+
+def _diagnose_import_error() -> str:
+    # TODO(kmaehashi): provide better diagnostics.
+    msg = '''\
+Failed to import CuPy.
+
+If you installed CuPy via wheels (cupy-cudaXXX or cupy-rocm-X-X), make sure that the package matches with the version of CUDA or ROCm installed.
+
+On Linux, you may need to set LD_LIBRARY_PATH environment variable depending on how you installed CUDA/ROCm.
+On Windows, try setting CUDA_PATH environment variable.
+
+Check the Installation Guide for details:
+  https://docs.cupy.dev/en/latest/install.html'''  # NOQA
+
+    if sys.platform == 'win32':
+        try:
+            msg += _diagnose_win32_dll_load()
+        except Exception as e:
+            msg += (
+                '\n\nThe cause could not be identified: '
+                f'{type(e).__name__}: {e}'
+            )
+
+    return msg
+
+
+def _diagnose_win32_dll_load() -> str:
+    depends = _get_json_data('_depends.json')
+    if depends is None:
+        return ''
+
+    from ctypes import wintypes
+    kernel32 = ctypes.CDLL('kernel32')
+    kernel32.GetModuleFileNameW.argtypes = [
+        wintypes.HANDLE, wintypes.LPWSTR, wintypes.DWORD]
+    kernel32.GetModuleFileNameW.restype = wintypes.DWORD
+
+    # Show dependents in similar form of ldd on Linux.
+    lines = [
+        '',
+        '',
+        f'CUDA Path: {get_cuda_path()}',
+        'DLL dependencies:'
+    ]
+    filepath = ctypes.create_unicode_buffer(2**15)
+    for name in depends['depends']:
+        try:
+            dll = ctypes.CDLL(name)
+            kernel32.GetModuleFileNameW(dll._handle, filepath, len(filepath))
+            lines.append(f'  {name} -> {filepath.value}')
+        except FileNotFoundError:
+            lines.append(f'  {name} -> not found')
+        except Exception as e:
+            lines.append(f'  {name} -> error ({type(e).__name__}: {e})')
+
+    return '\n'.join(lines)
diff --git a/cupy/_functional/__init__.py b/cupy/_functional/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cupy/_functional/piecewise.py b/cupy/_functional/piecewise.py
new file mode 100644
index 0000000..aecebfc
--- /dev/null
+++ b/cupy/_functional/piecewise.py
@@ -0,0 +1,61 @@
+import cupy
+
+from cupy import _core
+
+_piecewise_krnl = _core.ElementwiseKernel(
+    'bool cond, T value',
+    'T y',
+    'if (cond) y = value',
+    'cupy_piecewise_kernel'
+)
+
+
+def piecewise(x, condlist, funclist):
+    """Evaluate a piecewise-defined function.
+
+        Args:
+            x (cupy.ndarray): input domain
+            condlist (list of cupy.ndarray):
+                Each boolean array/ scalar corresponds to a function
+                in funclist. Length of funclist is equal to that of
+                condlist. If one extra function is given, it is used
+                as the default value when the otherwise condition is met
+            funclist (list of scalars): list of scalar functions.
+
+        Returns:
+            cupy.ndarray: the scalar values in funclist on portions of x
+            defined by condlist.
+
+        .. warning::
+
+            This function currently doesn't support callable functions,
+            args and kw parameters.
+
+        .. seealso:: :func:`numpy.piecewise`
+        """
+    if cupy.isscalar(condlist):
+        condlist = [condlist]
+
+    condlen = len(condlist)
+    funclen = len(funclist)
+    if condlen == funclen:
+        out = cupy.zeros(x.shape, x.dtype)
+    elif condlen + 1 == funclen:
+        func = funclist[-1]
+        funclist = funclist[:-1]
+        if callable(func):
+            raise NotImplementedError(
+                'Callable functions are not supported currently')
+        out = cupy.full(x.shape, func, x.dtype)
+    else:
+        raise ValueError('with {} condition(s), either {} or {} functions'
+                         ' are expected'.format(condlen, condlen, condlen + 1))
+
+    for condition, func in zip(condlist, funclist):
+        if callable(func):
+            raise NotImplementedError(
+                'Callable functions are not supported currently')
+        if isinstance(func, cupy.ndarray):
+            func = func.astype(x.dtype)
+        _piecewise_krnl(condition, func, out)
+    return out
diff --git a/cupy/_functional/vectorize.py b/cupy/_functional/vectorize.py
new file mode 100644
index 0000000..f0f37f9
--- /dev/null
+++ b/cupy/_functional/vectorize.py
@@ -0,0 +1,104 @@
+import numpy
+
+from cupy import _core
+from cupyx.jit import _interface
+from cupyx.jit import _cuda_types
+
+
+def _get_input_type(arg):
+    if isinstance(arg, int):
+        return 'l'
+    if isinstance(arg, float):
+        return 'd'
+    if isinstance(arg, complex):
+        return 'D'
+    return arg.dtype.char
+
+
+class vectorize(object):
+    """Generalized function class.
+
+    .. seealso:: :class:`numpy.vectorize`
+    """
+
+    def __init__(
+            self, pyfunc, otypes=None, doc=None, excluded=None,
+            cache=False, signature=None):
+        """
+        Args:
+            pyfunc (callable): The target python function.
+            otypes (str or list of dtypes, optional): The output data type.
+            doc (str or None): The docstring for the function.
+            excluded: Currently not supported.
+            cache: Currently Ignored.
+            signature: Currently not supported.
+        """
+
+        self.pyfunc = pyfunc
+        self.__doc__ = doc or pyfunc.__doc__
+        self.excluded = excluded
+        self.cache = cache
+        self.signature = signature
+        self._kernel_cache = {}
+
+        self.otypes = None
+        if otypes is not None:
+            self.otypes = ''.join([numpy.dtype(t).char for t in otypes])
+
+        if excluded is not None:
+            raise NotImplementedError(
+                'cupy.vectorize does not support `excluded` option currently.')
+
+        if signature is not None:
+            raise NotImplementedError(
+                'cupy.vectorize does not support `signature`'
+                ' option currently.')
+
+    @staticmethod
+    def _get_body(return_type, call):
+        if isinstance(return_type, _cuda_types.Scalar):
+            dtypes = [return_type.dtype]
+            code = f'out0 = {call};'
+        elif isinstance(return_type, _cuda_types.Tuple):
+            dtypes = []
+            code = f"auto out = {call};\n"
+            for i, t in enumerate(return_type.types):
+                if not isinstance(t, _cuda_types.Scalar):
+                    raise TypeError(f'Invalid return type: {return_type}')
+                dtypes.append(t.dtype)
+                code += f'out{i} = thrust::get<{i}>(out);\n'
+        else:
+            raise TypeError(f'Invalid return type: {return_type}')
+
+        out_params = [f'{dtype} out{i}' for i, dtype in enumerate(dtypes)]
+        return ', '.join(out_params), code
+
+    def __call__(self, *args):
+        itypes = ''.join([_get_input_type(x) for x in args])
+        kern = self._kernel_cache.get(itypes, None)
+
+        if kern is None:
+            in_types = [_cuda_types.Scalar(t) for t in itypes]
+            ret_type = None
+            if self.otypes is not None:
+                # TODO(asi1024): Implement
+                raise NotImplementedError
+
+            func = _interface._CudaFunction(self.pyfunc, 'numpy', device=True)
+            result = func._emit_code_from_types(in_types, ret_type)
+            in_params = ', '.join(
+                f'{t.dtype} in{i}' for i, t in enumerate(in_types))
+            in_args = ', '.join([f'in{i}' for i in range(len(in_types))])
+            call = f'{result.func_name}({in_args})'
+            out_params, body = self._get_body(result.return_type, call)
+            # note: we don't worry about -D not working on ROCm here, because
+            # we unroll all headers for HIP and so thrust::tuple et al are all
+            # defined regardless if CUPY_JIT_MODE is defined or not
+            kern = _core.ElementwiseKernel(
+                in_params, out_params, body, 'cupy_vectorize',
+                preamble=result.code,
+                options=('-DCUPY_JIT_MODE', '--std=c++14'),
+            )
+            self._kernel_cache[itypes] = kern
+
+        return kern(*args)
diff --git a/cupy/_indexing/__init__.py b/cupy/_indexing/__init__.py
new file mode 100644
index 0000000..637a20f
--- /dev/null
+++ b/cupy/_indexing/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.indexing.html
diff --git a/cupy/_indexing/generate.py b/cupy/_indexing/generate.py
new file mode 100644
index 0000000..eaaab36
--- /dev/null
+++ b/cupy/_indexing/generate.py
@@ -0,0 +1,588 @@
+# class s_(object):
+
+import functools
+import numbers
+import operator
+
+import numpy
+
+import cupy
+from cupy._creation import from_data
+from cupy._manipulation import join
+
+
+class AxisConcatenator(object):
+    """Translates slice objects to concatenation along an axis.
+
+    For detailed documentation on usage, see :func:`cupy.r_`.
+    This implementation is partially borrowed from NumPy's one.
+
+    """
+
+    def _output_obj(self, obj, ndim, ndmin, trans1d):
+        k2 = ndmin - ndim
+        if trans1d < 0:
+            trans1d += k2 + 1
+        defaxes = list(range(ndmin))
+        k1 = trans1d
+        axes = defaxes[:k1] + defaxes[k2:] + \
+            defaxes[k1:k2]
+        return obj.transpose(axes)
+
+    def __init__(self, axis=0, matrix=False, ndmin=1, trans1d=-1):
+        self.axis = axis
+        self.trans1d = trans1d
+        self.matrix = matrix
+        self.ndmin = ndmin
+
+    def __getitem__(self, key):
+        trans1d = self.trans1d
+        ndmin = self.ndmin
+        objs = []
+        arrays = []
+        scalars = []
+        if isinstance(key, str):
+            raise NotImplementedError
+        if not isinstance(key, tuple):
+            key = (key,)
+
+        for i, k in enumerate(key):
+            if isinstance(k, slice):
+                raise NotImplementedError
+            elif isinstance(k, str):
+                if i != 0:
+                    raise ValueError(
+                        'special directives must be the first entry.')
+                raise NotImplementedError
+            elif type(k) in numpy.ScalarType:
+                newobj = from_data.array(k, ndmin=ndmin)
+                scalars.append(i)
+            else:
+                newobj = from_data.array(k, copy=False, ndmin=ndmin)
+                if ndmin > 1:
+                    ndim = from_data.array(k, copy=False).ndim
+                    if trans1d != -1 and ndim < ndmin:
+                        newobj = self._output_obj(newobj, ndim, ndmin, trans1d)
+                arrays.append(newobj)
+
+            objs.append(newobj)
+
+        final_dtype = numpy.result_type(*arrays, *scalars)
+        if final_dtype is not None:
+            for k in scalars:
+                objs[k] = objs[k].astype(final_dtype)
+
+        return join.concatenate(tuple(objs), axis=self.axis)
+
+    def __len__(self):
+        return 0
+
+
+class CClass(AxisConcatenator):
+
+    def __init__(self):
+        super(CClass, self).__init__(-1, ndmin=2, trans1d=0)
+
+
+c_ = CClass()
+"""Translates slice objects to concatenation along the second axis.
+
+This is a CuPy object that corresponds to :obj:`cupy.r_`, which is
+useful because of its common occurrence. In particular, arrays will be
+stacked along their last axis after being upgraded to at least 2-D with
+1's post-pended to the shape (column vectors made out of 1-D arrays).
+
+For detailed documentation, see :obj:`r_`.
+
+This implementation is partially borrowed from NumPy's one.
+
+Returns:
+    cupy.ndarray: Joined array.
+
+.. seealso:: :obj:`numpy.c_`
+
+Examples
+--------
+>>> a = cupy.array([[1, 2, 3]], dtype=np.int32)
+>>> b = cupy.array([[4, 5, 6]], dtype=np.int32)
+>>> cupy.c_[a, 0, 0, b]
+array([[1, 2, 3, 0, 0, 4, 5, 6]], dtype=int32)
+
+"""
+
+
+class RClass(AxisConcatenator):
+
+    def __init__(self):
+        super(RClass, self).__init__()
+
+
+r_ = RClass()
+"""Translates slice objects to concatenation along the first axis.
+
+This is a simple way to build up arrays quickly.
+If the index expression contains comma separated arrays, then stack
+them along their first axis.
+
+This object can build up from normal CuPy arrays.
+Therefore, the other objects (e.g. writing strings like '2,3,4',
+or using imaginary numbers like [1,2,3j],
+or using string integers like '-1') are not implemented yet
+compared with NumPy.
+
+This implementation is partially borrowed from NumPy's one.
+
+Returns:
+    cupy.ndarray: Joined array.
+
+.. seealso:: :obj:`numpy.r_`
+
+Examples
+--------
+>>> a = cupy.array([1, 2, 3], dtype=np.int32)
+>>> b = cupy.array([4, 5, 6], dtype=np.int32)
+>>> cupy.r_[a, 0, 0, b]
+array([1, 2, 3, 0, 0, 4, 5, 6], dtype=int32)
+
+"""
+
+
+def indices(dimensions, dtype=int):
+    """Returns an array representing the indices of a grid.
+
+    Computes an array where the subarrays contain index values 0,1,...
+    varying only along the corresponding axis.
+
+    Args:
+        dimensions: The shape of the grid.
+        dtype: Data type specifier. It is int by default.
+
+    Returns:
+        ndarray:
+        The array of grid indices,
+        ``grid.shape = (len(dimensions),) + tuple(dimensions)``.
+
+    Examples
+    --------
+    >>> grid = cupy.indices((2, 3))
+    >>> grid.shape
+    (2, 2, 3)
+    >>> grid[0]        # row indices
+    array([[0, 0, 0],
+           [1, 1, 1]])
+    >>> grid[1]        # column indices
+    array([[0, 1, 2],
+           [0, 1, 2]])
+
+    .. seealso:: :func:`numpy.indices`
+
+    """
+    dimensions = tuple(dimensions)
+    N = len(dimensions)
+    shape = (1,) * N
+    res = cupy.empty((N,) + dimensions, dtype=dtype)
+    for i, dim in enumerate(dimensions):
+        res[i] = cupy.arange(dim, dtype=dtype).reshape(
+            shape[:i] + (dim,) + shape[i + 1:]
+        )
+    return res
+
+
+def ix_(*args):
+    """Construct an open mesh from multiple sequences.
+
+    This function takes N 1-D sequences and returns N outputs with N
+    dimensions each, such that the shape is 1 in all but one dimension
+    and the dimension with the non-unit shape value cycles through all
+    N dimensions.
+
+    Using `ix_` one can quickly construct index arrays that will index
+    the cross product. ``a[cupy.ix_([1,3],[2,5])]`` returns the array
+    ``[[a[1,2] a[1,5]], [a[3,2] a[3,5]]]``.
+
+    Args:
+        *args: 1-D sequences
+
+    Returns:
+        tuple of ndarrays:
+        N arrays with N dimensions each, with N the number of input sequences.
+        Together these arrays form an open mesh.
+
+    Examples
+    --------
+    >>> a = cupy.arange(10).reshape(2, 5)
+    >>> a
+    array([[0, 1, 2, 3, 4],
+           [5, 6, 7, 8, 9]])
+    >>> ixgrid = cupy.ix_([0,1], [2,4])
+    >>> ixgrid
+    (array([[0],
+           [1]]), array([[2, 4]]))
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.ix_`
+
+    """
+    # TODO(niboshi): Avoid nonzero which may synchronize the device.
+    out = []
+    nd = len(args)
+    for k, new in enumerate(args):
+        new = from_data.asarray(new)
+        if new.ndim != 1:
+            raise ValueError('Cross index must be 1 dimensional')
+        if new.size == 0:
+            # Explicitly type empty arrays to avoid float default
+            new = new.astype(numpy.intp)
+        if cupy.issubdtype(new.dtype, cupy.bool_):
+            new, = new.nonzero()  # may synchronize
+        new = new.reshape((1,) * k + (new.size,) + (1,) * (nd - k - 1))
+        out.append(new)
+    return tuple(out)
+
+
+def ravel_multi_index(multi_index, dims, mode='wrap', order='C'):
+    """
+    Converts a tuple of index arrays into an array of flat indices, applying
+    boundary modes to the multi-index.
+
+    Args:
+        multi_index (tuple of cupy.ndarray) : A tuple of integer arrays, one
+            array for each dimension.
+        dims (tuple of ints): The shape of array into which the indices from
+            ``multi_index`` apply.
+        mode ('raise', 'wrap' or 'clip'), optional: Specifies how out-of-bounds
+            indices are handled.  Can specify either one mode or a tuple of
+            modes, one mode per index:
+
+            - *'raise'* -- raise an error
+            - *'wrap'* -- wrap around (default)
+            - *'clip'* -- clip to the range
+
+            In 'clip' mode, a negative index which would normally wrap will
+            clip to 0 instead.
+        order ('C' or 'F'), optional: Determines whether the multi-index should
+            be viewed as indexing in row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Returns:
+        raveled_indices (cupy.ndarray): An array of indices into the flattened
+        version of an array of dimensions ``dims``.
+
+    .. warning::
+
+        This function may synchronize the device when ``mode == 'raise'``.
+
+    Notes
+    -----
+    Note that the default `mode` (``'wrap'``) is different than in NumPy. This
+    is done to avoid potential device synchronization.
+
+    Examples
+    --------
+    >>> cupy.ravel_multi_index(cupy.asarray([[3,6,6],[4,5,1]]), (7,6))
+    array([22, 41, 37])
+    >>> cupy.ravel_multi_index(cupy.asarray([[3,6,6],[4,5,1]]), (7,6),
+    ...                        order='F')
+    array([31, 41, 13])
+    >>> cupy.ravel_multi_index(cupy.asarray([[3,6,6],[4,5,1]]), (4,6),
+    ...                        mode='clip')
+    array([22, 23, 19])
+    >>> cupy.ravel_multi_index(cupy.asarray([[3,6,6],[4,5,1]]), (4,4),
+    ...                        mode=('clip', 'wrap'))
+    array([12, 13, 13])
+    >>> cupy.ravel_multi_index(cupy.asarray((3,1,4,1)), (6,7,8,9))
+    array(1621)
+
+    .. seealso:: :func:`numpy.ravel_multi_index`, :func:`unravel_index`
+    """
+
+    ndim = len(dims)
+    if len(multi_index) != ndim:
+        raise ValueError(
+            "parameter multi_index must be a sequence of "
+            "length {}".format(ndim))
+
+    for d in dims:
+        if not isinstance(d, numbers.Integral):
+            raise TypeError(
+                "{} object cannot be interpreted as an integer".format(
+                    type(d)))
+
+    if isinstance(mode, str):
+        mode = (mode, ) * ndim
+
+    if functools.reduce(operator.mul, dims) > cupy.iinfo(cupy.int64).max:
+        raise ValueError("invalid dims: array size defined by dims is larger "
+                         "than the maximum possible size")
+
+    s = 1
+    ravel_strides = [1] * ndim
+
+    order = 'C' if order is None else order.upper()
+    if order == 'C':
+        for i in range(ndim - 2, -1, -1):
+            s = s * dims[i + 1]
+            ravel_strides[i] = s
+    elif order == 'F':
+        for i in range(1, ndim):
+            s = s * dims[i - 1]
+            ravel_strides[i] = s
+    else:
+        raise ValueError('order not understood')
+
+    multi_index = cupy.broadcast_arrays(*multi_index)
+    raveled_indices = cupy.zeros(multi_index[0].shape, dtype=cupy.int64)
+    for d, stride, idx, _mode in zip(dims, ravel_strides, multi_index, mode):
+
+        if not isinstance(idx, cupy.ndarray):
+            raise TypeError("elements of multi_index must be cupy arrays")
+        if not cupy.can_cast(idx, cupy.int64, 'same_kind'):
+            raise TypeError(
+                'multi_index entries could not be cast from dtype(\'{}\') to '
+                'dtype(\'{}\') according to the rule \'same_kind\''.format(
+                    idx.dtype, cupy.int64().dtype))
+        idx = idx.astype(cupy.int64, copy=False)
+
+        if _mode == "raise":
+            if cupy.any(cupy.logical_or(idx >= d, idx < 0)):
+                raise ValueError("invalid entry in coordinates array")
+        elif _mode == "clip":
+            idx = cupy.clip(idx, 0, d - 1)
+        elif _mode == 'wrap':
+            idx = idx % d
+        else:
+            raise ValueError('Unrecognized mode: {}'.format(_mode))
+        raveled_indices += stride * idx
+    return raveled_indices
+
+
+def unravel_index(indices, dims, order='C'):
+    """Converts array of flat indices into a tuple of coordinate arrays.
+
+    Args:
+        indices (cupy.ndarray): An integer array whose elements are indices
+            into the flattened version of an array of dimensions :obj:`dims`.
+        dims (tuple of ints): The shape of the array to use for unraveling
+            indices.
+        order ('C' or 'F'): Determines whether the indices should be viewed as
+            indexing in row-major (C-style) or column-major (Fortran-style)
+            order.
+
+    Returns:
+        tuple of ndarrays:
+        Each array in the tuple has the same shape as the indices array.
+
+    Examples
+    --------
+    >>> cupy.unravel_index(cupy.array([22, 41, 37]), (7, 6))
+    (array([3, 6, 6]), array([4, 5, 1]))
+    >>> cupy.unravel_index(cupy.array([31, 41, 13]), (7, 6), order='F')
+    (array([3, 6, 6]), array([4, 5, 1]))
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.unravel_index`, :func:`ravel_multi_index`
+
+    """
+    order = 'C' if order is None else order.upper()
+    if order == 'C':
+        dims = reversed(dims)
+    elif order == 'F':
+        pass
+    else:
+        raise ValueError('order not understood')
+
+    if not cupy.can_cast(indices, cupy.int64, 'same_kind'):
+        raise TypeError(
+            'Iterator operand 0 dtype could not be cast '
+            'from dtype(\'{}\') to dtype(\'{}\') '
+            'according to the rule \'same_kind\''.format(
+                indices.dtype, cupy.int64().dtype))
+
+    if (indices < 0).any():  # synchronize!
+        raise ValueError('invalid entry in index array')
+
+    unraveled_coords = []
+    for dim in dims:
+        unraveled_coords.append(indices % dim)
+        indices = indices // dim
+
+    if (indices > 0).any():  # synchronize!
+        raise ValueError('invalid entry in index array')
+
+    if order == 'C':
+        unraveled_coords = reversed(unraveled_coords)
+    return tuple(unraveled_coords)
+
+
+def mask_indices(n, mask_func, k=0):
+    """
+    Return the indices to access (n, n) arrays, given a masking function.
+
+    Assume `mask_func` is a function that, for a square array a of
+    size ``(n, n)`` with a possible offset argument `k`, when called
+    as ``mask_func(a, k)`` returns a new array with zeros in certain
+    locations (functions like :func:`~cupy.triu` or :func:`~cupy.tril` do
+    precisely this). Then this function returns the indices where the non-zero
+    values would be located.
+
+    Args:
+        n (int): The returned indices will be valid to access arrays
+            of shape (n, n).
+        mask_func (callable): A function whose call signature is
+            similar to that of :func:`~cupy.triu`, :func:`~tril`.  That is,
+            ``mask_func(x, k)`` returns a boolean array, shaped like
+            `x`.  `k` is an optional argument to the function.
+        k (scalar): An optional argument which is passed through to
+            `mask_func`. Functions like :func:`~cupy.triu`, :func:`~cupy.tril`
+            take a second argument that is interpreted as an offset.
+
+    Returns:
+        tuple of arrays: The `n` arrays of indices corresponding to
+        the locations where ``mask_func(np.ones((n, n)), k)`` is
+        True.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.mask_indices`
+    """
+    a = cupy.ones((n, n), dtype=cupy.int8)
+    return mask_func(a, k).nonzero()
+
+
+# TODO(okuta): Implement diag_indices
+
+
+# TODO(okuta): Implement diag_indices_from
+
+
+def tril_indices(n, k=0, m=None):
+    """Returns the indices of the lower triangular matrix.
+    Here, the first group of elements contains row coordinates
+    of all indices and the second group of elements
+    contains column coordinates.
+
+    Parameters
+    ----------
+    n : int
+        The row dimension of the arrays for which the returned
+        indices will be valid.
+    k : int, optional
+        Diagonal above which to zero elements. `k = 0`
+        (the default) is the main diagonal, `k < 0` is
+        below it and `k > 0` is above.
+    m : int, optional
+        The column dimension of the arrays for which the
+        returned arrays will be valid. By default, `m = n`.
+
+    Returns
+    -------
+    y : tuple of ndarrays
+        The indices for the triangle. The returned tuple
+        contains two arrays, each with the indices along
+        one dimension of the array.
+
+    See Also
+    --------
+    numpy.tril_indices
+
+    """
+
+    tri_ = cupy.tri(n, m, k=k, dtype=bool)
+
+    return tuple(cupy.broadcast_to(inds, tri_.shape)[tri_]
+                 for inds in cupy.indices(tri_.shape, dtype=int))
+
+
+def tril_indices_from(arr, k=0):
+    """Returns the indices for the lower-triangle of arr.
+
+    Parameters
+    ----------
+    arr : cupy.ndarray
+          The indices are valid for square arrays
+          whose dimensions are the same as arr.
+    k : int, optional
+        Diagonal offset.
+
+    See Also
+    --------
+    numpy.tril_indices_from
+
+    """
+
+    if arr.ndim != 2:
+        raise ValueError("input array must be 2-d")
+    return tril_indices(arr.shape[-2], k=k, m=arr.shape[-1])
+
+
+def triu_indices(n, k=0, m=None):
+    """Returns the indices of the upper triangular matrix.
+    Here, the first group of elements contains row coordinates
+    of all indices and the second group of elements
+    contains column coordinates.
+
+    Parameters
+    ----------
+    n : int
+        The size of the arrays for which the returned indices will
+        be valid.
+    k : int, optional
+        Refers to the diagonal offset. By default, `k = 0` i.e.
+        the main dialogal. The positive value of `k`
+        denotes the diagonals above the main diagonal, while the negative
+        value includes the diagonals below the main diagonal.
+    m : int, optional
+        The column dimension of the arrays for which the
+        returned arrays will be valid. By default, `m = n`.
+
+    Returns
+    -------
+    y : tuple of ndarrays
+        The indices for the triangle. The returned tuple
+        contains two arrays, each with the indices along
+        one dimension of the array.
+
+    See Also
+    --------
+    numpy.triu_indices
+
+    """
+
+    tri_ = ~cupy.tri(n, m, k=k - 1, dtype=bool)
+
+    return tuple(cupy.broadcast_to(inds, tri_.shape)[tri_]
+                 for inds in cupy.indices(tri_.shape, dtype=int))
+
+
+def triu_indices_from(arr, k=0):
+    """Returns indices for the upper-triangle of arr.
+
+    Parameters
+    ----------
+    arr : cupy.ndarray
+          The indices are valid for square arrays.
+    k : int, optional
+        Diagonal offset (see 'triu_indices` for details).
+
+    Returns
+    -------
+    triu_indices_from : tuple of ndarrays
+        Indices for the upper-triangle of `arr`.
+
+    See Also
+    --------
+    numpy.triu_indices_from
+
+    """
+
+    if arr.ndim != 2:
+        raise ValueError("input array must be 2-d")
+    return triu_indices(arr.shape[-2], k=k, m=arr.shape[-1])
diff --git a/cupy/_indexing/indexing.py b/cupy/_indexing/indexing.py
new file mode 100644
index 0000000..37314af
--- /dev/null
+++ b/cupy/_indexing/indexing.py
@@ -0,0 +1,223 @@
+import cupy
+from cupy._core import internal
+
+
+def take(a, indices, axis=None, out=None):
+    """Takes elements of an array at specified indices along an axis.
+
+    This is an implementation of "fancy indexing" at single axis.
+
+    This function does not support ``mode`` option.
+
+    Args:
+        a (cupy.ndarray): Array to extract elements.
+        indices (int or array-like): Indices of elements that this function
+            takes.
+        axis (int): The axis along which to select indices. The flattened input
+            is used by default.
+        out (cupy.ndarray): Output array. If provided, it should be of
+            appropriate shape and dtype.
+
+    Returns:
+        cupy.ndarray: The result of fancy indexing.
+
+    .. seealso:: :func:`numpy.take`
+
+    """
+    # TODO(okuta): check type
+    return a.take(indices, axis, out)
+
+
+def take_along_axis(a, indices, axis):
+    """Take values from the input array by matching 1d index and data slices.
+
+    Args:
+        a (cupy.ndarray): Array to extract elements.
+        indices (cupy.ndarray): Indices to take along each 1d slice of ``a``.
+        axis (int): The axis to take 1d slices along.
+
+    Returns:
+        cupy.ndarray: The indexed result.
+
+    .. seealso:: :func:`numpy.take_along_axis`
+    """
+
+    if indices.dtype.kind not in ('i', 'u'):
+        raise IndexError('`indices` must be an integer array')
+
+    if axis is None:
+        a = a.ravel()
+        axis = 0
+
+    ndim = a.ndim
+
+    axis = internal._normalize_axis_index(axis, ndim)
+
+    if ndim != indices.ndim:
+        raise ValueError(
+            '`indices` and `a` must have the same number of dimensions')
+
+    fancy_index = []
+    for i, n in enumerate(a.shape):
+        if i == axis:
+            fancy_index.append(indices)
+        else:
+            ind_shape = (1,) * i + (-1,) + (1,) * (ndim - i - 1)
+            fancy_index.append(cupy.arange(n).reshape(ind_shape))
+
+    return a[tuple(fancy_index)]
+
+
+def choose(a, choices, out=None, mode='raise'):
+    return a.choose(choices, out, mode)
+
+
+def compress(condition, a, axis=None, out=None):
+    """Returns selected slices of an array along given axis.
+
+    Args:
+        condition (1-D array of bools): Array that selects which entries to
+            return. If len(condition) is less than the size of a along the
+            given axis, then output is truncated to the length of the condition
+            array.
+        a (cupy.ndarray): Array from which to extract a part.
+        axis (int): Axis along which to take slices. If None (default), work
+            on the flattened array.
+        out (cupy.ndarray): Output array. If provided, it should be of
+            appropriate shape and dtype.
+
+    Returns:
+        cupy.ndarray: A copy of a without the slices along axis for which
+        condition is false.
+
+    .. warning::
+
+            This function may synchronize the device.
+
+
+    .. seealso:: :func:`numpy.compress`
+
+    """
+    return a.compress(condition, axis, out)
+
+
+def diagonal(a, offset=0, axis1=0, axis2=1):
+    """Returns specified diagonals.
+
+    This function extracts the diagonals along two specified axes. The other
+    axes are not changed. This function returns a writable view of this array
+    as NumPy 1.10 will do.
+
+    Args:
+        a (cupy.ndarray): Array from which the diagonals are taken.
+        offset (int): Index of the diagonals. Zero indicates the main
+            diagonals, a positive value upper diagonals, and a negative value
+            lower diagonals.
+        axis1 (int): The first axis to take diagonals from.
+        axis2 (int): The second axis to take diagonals from.
+
+    Returns:
+        cupy.ndarray: A view of the diagonals of ``a``.
+
+    .. seealso:: :func:`numpy.diagonal`
+
+    """
+    # TODO(okuta): check type
+    return a.diagonal(offset, axis1, axis2)
+
+
+def extract(condition, a):
+    """Return the elements of an array that satisfy some condition.
+
+    This is equivalent to ``np.compress(ravel(condition), ravel(arr))``.
+    If ``condition`` is boolean, ``np.extract`` is equivalent to
+    ``arr[condition]``.
+
+    Args:
+        condition (int or array_like): An array whose nonzero or True entries
+            indicate the elements of array to extract.
+        a (cupy.ndarray): Input array of the same size as condition.
+
+    Returns:
+        cupy.ndarray: Rank 1 array of values from arr where condition is True.
+
+    .. warning::
+
+            This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.extract`
+    """
+
+    if not isinstance(a, cupy.ndarray):
+        raise TypeError('extract requires input array to be cupy.ndarray')
+
+    if not isinstance(condition, cupy.ndarray):
+        condition = cupy.array(condition)
+
+    a = a.ravel()
+    condition = condition.ravel()
+
+    return a.take(condition.nonzero()[0])
+
+
+def select(condlist, choicelist, default=0):
+    """Return an array drawn from elements in choicelist, depending on conditions.
+
+    Args:
+        condlist (list of bool arrays): The list of conditions which determine
+            from which array in `choicelist` the output elements are taken.
+            When multiple conditions are satisfied, the first one encountered
+            in `condlist` is used.
+        choicelist (list of cupy.ndarray): The list of arrays from which the
+            output elements are taken. It has to be of the same length
+            as `condlist`.
+        default (scalar) : If provided, will fill element inserted in `output`
+            when all conditions evaluate to False. default value is 0.
+
+    Returns:
+        cupy.ndarray: The output at position m is the m-th element of the
+        array in `choicelist` where the m-th element of the corresponding
+        array in `condlist` is True.
+
+    .. seealso:: :func:`numpy.select`
+    """  # NOQA
+
+    if len(condlist) != len(choicelist):
+        raise ValueError(
+            'list of cases must be same length as list of conditions')
+
+    if len(condlist) == 0:
+        raise ValueError("select with an empty condition list is not possible")
+
+    if not cupy.isscalar(default):
+        raise TypeError("default only accepts scalar values")
+
+    for i in range(len(choicelist)):
+        if not isinstance(choicelist[i], cupy.ndarray):
+            raise TypeError("choicelist only accepts lists of cupy ndarrays")
+        cond = condlist[i]
+        if cond.dtype.type is not cupy.bool_:
+            raise ValueError(
+                'invalid entry {} in condlist: should be boolean ndarray'
+                .format(i))
+
+    dtype = cupy.result_type(*choicelist)
+
+    condlist = cupy.broadcast_arrays(*condlist)
+    choicelist = cupy.broadcast_arrays(*choicelist, default)
+
+    if choicelist[0].ndim == 0:
+        result_shape = condlist[0].shape
+    else:
+        result_shape = cupy.broadcast_arrays(condlist[0],
+                                             choicelist[0])[0].shape
+
+    result = cupy.empty(result_shape, dtype)
+    cupy.copyto(result, default)
+
+    choicelist = choicelist[-2::-1]
+    condlist = condlist[::-1]
+    for choice, cond in zip(choicelist, condlist):
+        cupy.copyto(result, choice, where=cond)
+
+    return result
diff --git a/cupy/_indexing/insert.py b/cupy/_indexing/insert.py
new file mode 100644
index 0000000..75a95a7
--- /dev/null
+++ b/cupy/_indexing/insert.py
@@ -0,0 +1,260 @@
+import numpy
+
+import cupy
+from cupy import _core
+
+
+def place(arr, mask, vals):
+    """Change elements of an array based on conditional and input values.
+
+    This function uses the first N elements of `vals`, where N is the number
+    of true values in `mask`.
+
+    Args:
+        arr (cupy.ndarray): Array to put data into.
+        mask (array-like): Boolean mask array. Must have the same size as `a`.
+        vals (array-like): Values to put into `a`. Only the first
+            N elements are used, where N is the number of True values in
+            `mask`. If `vals` is smaller than N, it will be repeated, and if
+            elements of `a` are to be masked, this sequence must be non-empty.
+
+    Examples
+    --------
+    >>> arr = np.arange(6).reshape(2, 3)
+    >>> np.place(arr, arr>2, [44, 55])
+    >>> arr
+    array([[ 0,  1,  2],
+           [44, 55, 44]])
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.place`
+    """
+    # TODO(niboshi): Avoid nonzero which may synchronize the device.
+    mask = cupy.asarray(mask)
+    if arr.size != mask.size:
+        raise ValueError('Mask and data must be the same size.')
+    vals = cupy.asarray(vals)
+
+    mask_indices = mask.ravel().nonzero()[0]  # may synchronize
+    if mask_indices.size == 0:
+        return
+    if vals.size == 0:
+        raise ValueError('Cannot insert from an empty array.')
+    arr.put(mask_indices, vals, mode='wrap')
+
+
+def put(a, ind, v, mode='wrap'):
+    """Replaces specified elements of an array with given values.
+
+    Args:
+        a (cupy.ndarray): Target array.
+        ind (array-like): Target indices, interpreted as integers.
+        v (array-like): Values to place in `a` at target indices.
+            If `v` is shorter than `ind` it will be repeated as necessary.
+        mode (str): How out-of-bounds indices will behave. Its value must be
+            either `'raise'`, `'wrap'` or `'clip'`. Otherwise,
+            :class:`TypeError` is raised.
+
+    .. note::
+        Default `mode` is set to `'wrap'` to avoid unintended performance drop.
+        If you need NumPy's behavior, please pass `mode='raise'` manually.
+
+    .. seealso:: :func:`numpy.put`
+    """
+    a.put(ind, v, mode=mode)
+
+
+_putmask_kernel = _core.ElementwiseKernel(
+    'Q mask, raw S values, uint64 len_vals', 'T out',
+    '''
+    if (mask) out = (T) values[i % len_vals];
+    ''',
+    'cupy_putmask_kernel'
+)
+
+
+def putmask(a, mask, values):
+    """
+    Changes elements of an array inplace, based on a conditional mask and
+    input values.
+
+    Sets ``a.flat[n] = values[n]`` for each n where ``mask.flat[n]==True``.
+    If `values` is not the same size as `a` and `mask` then it will repeat.
+
+    Args:
+        a (cupy.ndarray): Target array.
+        mask (cupy.ndarray): Boolean mask array. It has to be
+            the same shape as `a`.
+        values (cupy.ndarray or scalar): Values to put into `a` where `mask`
+            is True. If `values` is smaller than `a`, then it will be
+            repeated.
+
+    Examples
+    --------
+    >>> x = cupy.arange(6).reshape(2, 3)
+    >>> cupy.putmask(x, x>2, x**2)
+    >>> x
+    array([[ 0,  1,  2],
+           [ 9, 16, 25]])
+
+    If `values` is smaller than `a` it is repeated:
+
+    >>> x = cupy.arange(6)
+    >>> cupy.putmask(x, x>2, cupy.array([-33, -44]))
+    >>> x
+    array([  0,   1,   2, -44, -33, -44])
+
+    .. seealso:: :func:`numpy.putmask`
+
+    """
+
+    if not isinstance(a, cupy.ndarray):
+        raise TypeError('`a` should be of type cupy.ndarray')
+    if not isinstance(mask, cupy.ndarray):
+        raise TypeError('`mask` should be of type cupy.ndarray')
+    if not (cupy.isscalar(values) or isinstance(values, cupy.ndarray)):
+        raise TypeError('`values` should be of type cupy.ndarray')
+
+    if not a.shape == mask.shape:
+        raise ValueError('mask and data must be the same size')
+
+    mask = mask.astype(numpy.bool_)
+
+    if cupy.isscalar(values):
+        a[mask] = values
+
+    elif not numpy.can_cast(values.dtype, a.dtype):
+        raise TypeError('Cannot cast array data from'
+                        ' {} to {} according to the rule \'safe\''
+                        .format(values.dtype, a.dtype))
+
+    elif a.shape == values.shape:
+        a[mask] = values[mask]
+
+    else:
+        values = values.ravel()
+        _putmask_kernel(mask, values, len(values), a)
+
+
+def fill_diagonal(a, val, wrap=False):
+    """Fills the main diagonal of the given array of any dimensionality.
+
+    For an array `a` with ``a.ndim > 2``, the diagonal is the list of
+    locations with indices ``a[i, i, ..., i]`` all identical. This function
+    modifies the input array in-place, it does not return a value.
+
+    Args:
+        a (cupy.ndarray): The array, at least 2-D.
+        val (scalar): The value to be written on the diagonal.
+            Its type must be compatible with that of the array a.
+        wrap (bool): If specified, the diagonal is "wrapped" after N columns.
+            This affects only tall matrices.
+
+    Examples
+    --------
+    >>> a = cupy.zeros((3, 3), int)
+    >>> cupy.fill_diagonal(a, 5)
+    >>> a
+    array([[5, 0, 0],
+           [0, 5, 0],
+           [0, 0, 5]])
+
+    .. seealso:: :func:`numpy.fill_diagonal`
+    """
+    # The followings are imported from the original numpy
+    if a.ndim < 2:
+        raise ValueError('array must be at least 2-d')
+    end = None
+    if a.ndim == 2:
+        step = a.shape[1] + 1
+        if not wrap:
+            end = a.shape[1] * a.shape[1]
+    else:
+        if not numpy.all(numpy.diff(a.shape) == 0):
+            raise ValueError('All dimensions of input must be of equal length')
+        step = 1 + numpy.cumprod(a.shape[:-1]).sum()
+
+    a.flat[:end:step] = val
+
+
+def diag_indices(n, ndim=2):
+    """Return the indices to access the main diagonal of an array.
+
+    Returns a tuple of indices that can be used to access the main
+    diagonal of an array with ``ndim >= 2`` dimensions and shape
+    (n, n, ..., n).
+
+    Args:
+        n (int): The size, along each dimension of the arrays for which
+            the indices are to be returned.
+        ndim (int): The number of dimensions. default `2`.
+
+    Examples
+    --------
+    Create a set of indices to access the diagonal of a (4, 4) array:
+
+    >>> di = cupy.diag_indices(4)
+    >>> di
+    (array([0, 1, 2, 3]), array([0, 1, 2, 3]))
+    >>> a = cupy.arange(16).reshape(4, 4)
+    >>> a
+    array([[ 0,  1,  2,  3],
+           [ 4,  5,  6,  7],
+           [ 8,  9, 10, 11],
+           [12, 13, 14, 15]])
+    >>> a[di] = 100
+    >>> a
+    array([[100,   1,   2,   3],
+           [  4, 100,   6,   7],
+           [  8,   9, 100,  11],
+           [ 12,  13,  14, 100]])
+
+    Create indices to manipulate a 3-D array:
+
+    >>> d3 = cupy.diag_indices(2, 3)
+    >>> d3
+    (array([0, 1]), array([0, 1]), array([0, 1]))
+
+    And use it to set the diagonal of an array of zeros to 1:
+
+    >>> a = cupy.zeros((2, 2, 2), dtype=int)
+    >>> a[d3] = 1
+    >>> a
+    array([[[1, 0],
+            [0, 0]],
+    <BLANKLINE>
+           [[0, 0],
+            [0, 1]]])
+
+    .. seealso:: :func:`numpy.diag_indices`
+
+    """
+    idx = cupy.arange(n)
+    return (idx,) * ndim
+
+
+def diag_indices_from(arr):
+    """
+    Return the indices to access the main diagonal of an n-dimensional array.
+    See `diag_indices` for full details.
+
+    Args:
+        arr (cupy.ndarray): At least 2-D.
+
+    .. seealso:: :func:`numpy.diag_indices_from`
+
+    """
+    if not isinstance(arr, cupy.ndarray):
+        raise TypeError("Argument must be cupy.ndarray")
+
+    if not arr.ndim >= 2:
+        raise ValueError("input array must be at least 2-d")
+    # For more than d=2, the strided formula is only valid for arrays with
+    # all dimensions equal, so we check first.
+    if not cupy.all(cupy.diff(arr.shape) == 0):
+        raise ValueError("All dimensions of input must be of equal length")
+
+    return diag_indices(arr.shape[0], arr.ndim)
diff --git a/cupy/_indexing/iterate.py b/cupy/_indexing/iterate.py
new file mode 100644
index 0000000..d8fbddc
--- /dev/null
+++ b/cupy/_indexing/iterate.py
@@ -0,0 +1,155 @@
+import numpy
+
+import cupy
+from cupy import _core
+from cupy._core import internal
+
+
+class flatiter:
+    """Flat iterator object to iterate over arrays.
+
+    A flatiter iterator is returned by ``x.flat`` for any array ``x``. It
+    allows iterating over the array as if it were a 1-D array, either in a
+    for-loop or by calling its ``next`` method.
+
+    Iteration is done in row-major, C-style order (the last index varying the
+    fastest).
+
+    Attributes:
+        base (cupy.ndarray): A reference to the array that is iterated over.
+
+    .. note::
+       Restricted support of basic slicing is currently supplied. Advanced
+       indexing is not supported yet.
+
+    .. seealso:: :func:`numpy.flatiter`
+
+    """
+
+    def __init__(self, a):
+        self._base = a
+        self._index = 0
+
+    def __setitem__(self, ind, value):
+        if ind is Ellipsis:
+            self[:] = value
+            return
+
+        if isinstance(ind, tuple):
+            raise IndexError('unsupported iterator index')
+
+        if isinstance(ind, bool):
+            raise IndexError('unsupported iterator index')
+
+        if numpy.isscalar(ind):
+            ind = int(ind)
+            base = self._base
+            size = base.size
+            indices = []
+            for s in base.shape:
+                size = size // s
+                indices.append(ind // size)
+                ind %= size
+            base[tuple(indices)] = value
+            return
+
+        if isinstance(ind, slice):
+            base = self._base
+            s = internal.complete_slice(ind, base.size)
+            s_start = s.start
+            s_step = s.step
+            size = s.stop - s.start
+            if s_step > 0:
+                size = (size - 1) // s_step + 1
+            else:
+                size = (size + 1) // s_step + 1
+            value = cupy.asarray(value, dtype=base.dtype)
+            _flatiter_setitem_slice(value, s_start, s_step, base, size=size)
+            return
+
+        raise IndexError('unsupported iterator index')
+
+    def __getitem__(self, ind):
+        if ind is Ellipsis:
+            return self[:]
+
+        if isinstance(ind, tuple):
+            raise IndexError('unsupported iterator index')
+
+        if isinstance(ind, bool):
+            raise IndexError('unsupported iterator index')
+
+        if numpy.isscalar(ind):
+            ind = int(ind)
+            base = self._base
+            size = base.size
+            indices = []
+            for s in base.shape:
+                size = size // s
+                indices.append(ind // size)
+                ind %= size
+            return base[tuple(indices)].copy()
+
+        if isinstance(ind, slice):
+            base = self._base
+            s = internal.complete_slice(ind, base.size)
+            s_start = s.start
+            s_step = s.step
+            size = s.stop - s.start
+            if s_step > 0:
+                size = (size - 1) // s_step + 1
+            else:
+                size = (size + 1) // s_step + 1
+            return _flatiter_getitem_slice(base, s_start, s_step, size=size)
+
+        raise IndexError('unsupported iterator index')
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        index = self._index
+        if index == len(self):
+            raise StopIteration()
+        self._index += 1
+        return self[index]
+
+    def copy(self):
+        """Get a copy of the iterator as a 1-D array."""
+        return self.base.flatten()
+
+    @property
+    def base(self):
+        """A reference to the array that is iterated over."""
+        return self._base
+
+    # TODO(Takagi): Implement coords
+
+    # TODO(Takagi): Implement index
+
+    # TODO(Takagi): Implement __lt__
+
+    # TODO(Takagi): Implement __le__
+
+    # TODO(Takagi): Implement __eq__
+
+    # TODO(Takagi): Implement __ne__
+
+    # TODO(Takagi): Implement __ge__
+
+    # TODO(Takagi): Implement __gt__
+
+    def __len__(self):
+        return self.base.size
+
+
+_flatiter_setitem_slice = _core.ElementwiseKernel(
+    'raw T val, int64 start, int64 step', 'raw T a',
+    'a[start + i * step] = val[i % val.size()]',
+    'cupy_flatiter_setitem_slice')
+
+
+_flatiter_getitem_slice = _core.ElementwiseKernel(
+    'raw T a, int64 start, int64 step', 'T o',
+    'o = a[start + i * step]',
+    'cupy_flatiter_getitem_slice')
diff --git a/cupy/_io/__init__.py b/cupy/_io/__init__.py
new file mode 100644
index 0000000..dad57e7
--- /dev/null
+++ b/cupy/_io/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.io.html
diff --git a/cupy/_io/formatting.py b/cupy/_io/formatting.py
new file mode 100644
index 0000000..e0ec5fa
--- /dev/null
+++ b/cupy/_io/formatting.py
@@ -0,0 +1,75 @@
+import cupy
+import numpy
+
+
+def array_repr(arr, max_line_width=None, precision=None, suppress_small=None):
+    """Returns the string representation of an array.
+
+    Args:
+        arr (array_like): Input array. It should be able to feed to
+            :func:`cupy.asnumpy`.
+        max_line_width (int): The maximum number of line lengths.
+        precision (int): Floating point precision. It uses the current printing
+            precision of NumPy.
+        suppress_small (bool): If ``True``, very small numbers are printed as
+            zeros
+
+    Returns:
+        str: The string representation of ``arr``.
+
+    .. seealso:: :func:`numpy.array_repr`
+
+    """
+    return numpy.array_repr(cupy.asnumpy(arr), max_line_width, precision,
+                            suppress_small)
+
+
+def array_str(arr, max_line_width=None, precision=None, suppress_small=None):
+    """Returns the string representation of the content of an array.
+
+    Args:
+        arr (array_like): Input array. It should be able to feed to
+            :func:`cupy.asnumpy`.
+        max_line_width (int): The maximum number of line lengths.
+        precision (int): Floating point precision. It uses the current printing
+            precision of NumPy.
+        suppress_small (bool): If ``True``, very small number are printed as
+            zeros.
+
+    .. seealso:: :func:`numpy.array_str`
+
+    """
+    return numpy.array_str(cupy.asnumpy(arr), max_line_width, precision,
+                           suppress_small)
+
+
+def array2string(a, *args, **kwargs):
+    """Return a string representation of an array.
+
+
+    .. seealso:: :func:`numpy.array2string`
+
+    """
+    return numpy.array2string(cupy.asnumpy(a), *args, **kwargs)
+
+
+def format_float_positional(x, *args, **kwargs):
+    """Format a floating-point scalar as a decimal string in positional notation.
+
+    See :func:`numpy.format_float_positional` for the list of arguments.
+
+    .. seealso:: :func:`numpy.format_float_positional`
+
+    """  # NOQA
+    return numpy.format_float_positional(cupy.asnumpy(x), *args, **kwargs)
+
+
+def format_float_scientific(x, *args, **kwargs):
+    """Format a floating-point scalar as a decimal string in scientific notation.
+
+    See :func:`numpy.format_float_scientific` for the list of arguments.
+
+    .. seealso:: :func:`numpy.format_float_scientific`
+
+    """  # NOQA
+    return numpy.format_float_scientific(cupy.asnumpy(x), *args, **kwargs)
diff --git a/cupy/_io/npz.py b/cupy/_io/npz.py
new file mode 100644
index 0000000..b1af591
--- /dev/null
+++ b/cupy/_io/npz.py
@@ -0,0 +1,146 @@
+import warnings
+
+import numpy
+
+import cupy
+
+
+_support_allow_pickle = (numpy.lib.NumpyVersion(numpy.__version__) >= '1.10.0')
+
+
+class NpzFile(object):
+
+    def __init__(self, npz_file):
+        self.npz_file = npz_file
+
+    def __enter__(self):
+        self.npz_file.__enter__()
+        return self
+
+    def __exit__(self, typ, val, traceback):
+        self.npz_file.__exit__(typ, val, traceback)
+
+    def __getitem__(self, key):
+        arr = self.npz_file[key]
+        return cupy.array(arr)
+
+    def close(self):
+        self.npz_file.close()
+
+
+def load(file, mmap_mode=None, allow_pickle=None):
+    """Loads arrays or pickled objects from ``.npy``, ``.npz`` or pickled file.
+
+    This function just calls ``numpy.load`` and then sends the arrays to the
+    current device. NPZ file is converted to NpzFile object, which defers the
+    transfer to the time of accessing the items.
+
+    Args:
+        file (file-like object or string): The file to read.
+        mmap_mode (None, 'r+', 'r', 'w+', 'c'): If not ``None``, memory-map the
+            file to construct an intermediate :class:`numpy.ndarray` object and
+            transfer it to the current device.
+        allow_pickle (bool): Allow loading pickled object arrays stored in npy
+            files. Reasons for disallowing pickles include security, as
+            loading pickled data can execute arbitrary code. If pickles are
+            disallowed, loading object arrays will fail.
+            Please be aware that CuPy does not support arrays with dtype of
+            `object`.
+            The default is False.
+            This option is available only for NumPy 1.10 or later.
+            In NumPy 1.9, this option cannot be specified (loading pickled
+            objects is always allowed).
+
+    Returns:
+        CuPy array or NpzFile object depending on the type of the file. NpzFile
+        object is a dictionary-like object with the context manager protocol
+        (which enables us to use *with* statement on it).
+
+    .. seealso:: :func:`numpy.load`
+
+    """
+    if _support_allow_pickle:
+        allow_pickle = False if allow_pickle is None else allow_pickle
+        obj = numpy.load(file, mmap_mode, allow_pickle)
+    else:
+        if allow_pickle is not None:
+            warnings.warn('allow_pickle option is not supported in NumPy 1.9')
+        obj = numpy.load(file, mmap_mode)
+
+    if isinstance(obj, numpy.ndarray):
+        return cupy.array(obj)
+    elif isinstance(obj, numpy.lib.npyio.NpzFile):
+        return NpzFile(obj)
+    else:
+        return obj
+
+
+def save(file, arr, allow_pickle=None):
+    """Saves an array to a binary file in ``.npy`` format.
+
+    Args:
+        file (file or str): File or filename to save.
+        arr (array_like): Array to save. It should be able to feed to
+            :func:`cupy.asnumpy`.
+        allow_pickle (bool): Allow saving object arrays using Python pickles.
+            Reasons for disallowing pickles include security (loading pickled
+            data can execute arbitrary code) and portability (pickled objects
+            may not be loadable on different Python installations, for example
+            if the stored objects require libraries that are not available,
+            and not all pickled data is compatible between Python 2 and Python
+            3).
+            The default is True.
+            This option is available only for NumPy 1.10 or later.
+            In NumPy 1.9, this option cannot be specified (saving objects
+            using pickles is always allowed).
+
+    .. seealso:: :func:`numpy.save`
+
+    """
+    if _support_allow_pickle:
+        allow_pickle = True if allow_pickle is None else allow_pickle
+        numpy.save(file, cupy.asnumpy(arr), allow_pickle)
+    else:
+        if allow_pickle is not None:
+            warnings.warn('allow_pickle option is not supported in NumPy 1.9')
+        numpy.save(file, cupy.asnumpy(arr))
+
+
+def savez(file, *args, **kwds):
+    """Saves one or more arrays into a file in uncompressed ``.npz`` format.
+
+    Arguments without keys are treated as arguments with automatic keys named
+    ``arr_0``, ``arr_1``, etc. corresponding to the positions in the argument
+    list. The keys of arguments are used as keys in the ``.npz`` file, which
+    are used for accessing NpzFile object when the file is read by
+    :func:`cupy.load` function.
+
+    Args:
+        file (file or str): File or filename to save.
+        *args: Arrays with implicit keys.
+        **kwds: Arrays with explicit keys.
+
+    .. seealso:: :func:`numpy.savez`
+
+    """
+    args = map(cupy.asnumpy, args)
+    for key in kwds:
+        kwds[key] = cupy.asnumpy(kwds[key])
+    numpy.savez(file, *args, **kwds)
+
+
+def savez_compressed(file, *args, **kwds):
+    """Saves one or more arrays into a file in compressed ``.npz`` format.
+
+    It is equivalent to :func:`cupy.savez` function except the output file is
+    compressed.
+
+    .. seealso::
+       :func:`cupy.savez` for more detail,
+       :func:`numpy.savez_compressed`
+
+    """
+    args = map(cupy.asnumpy, args)
+    for key in kwds:
+        kwds[key] = cupy.asnumpy(kwds[key])
+    numpy.savez_compressed(file, *args, **kwds)
diff --git a/cupy/_io/text.py b/cupy/_io/text.py
new file mode 100644
index 0000000..4f90acd
--- /dev/null
+++ b/cupy/_io/text.py
@@ -0,0 +1,13 @@
+import numpy
+import cupy
+
+
+def savetxt(fname, X, *args, **kwargs):
+    """Save an array to a text file.
+
+    .. note::
+        Uses NumPy's ``savetxt``.
+
+    .. seealso:: :func:`numpy.savetxt`
+    """
+    numpy.savetxt(fname, cupy.asnumpy(X), *args, **kwargs)
diff --git a/cupy/_logic/__init__.py b/cupy/_logic/__init__.py
new file mode 100644
index 0000000..f6502b4
--- /dev/null
+++ b/cupy/_logic/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.logic.html
diff --git a/cupy/_logic/comparison.py b/cupy/_logic/comparison.py
new file mode 100644
index 0000000..e75eff5
--- /dev/null
+++ b/cupy/_logic/comparison.py
@@ -0,0 +1,174 @@
+import numpy
+
+import cupy
+from cupy import _core
+from cupy._logic import content
+
+
+_is_close = _core.create_ufunc(
+    'cupy_is_close',
+    ('eeee?->?', 'ffff?->?', 'dddd?->?'),
+    '''
+    bool equal_nan = in4;
+    if (isfinite(in0) && isfinite(in1)) {
+      out0 = fabs(in0 - in1) <= in3 + in2 * fabs(in1);
+    } else if (equal_nan) {
+      out0 = (in0 == in1) || (isnan(in0) && isnan(in1));
+    } else {
+      out0 = (in0 == in1);
+    }
+    '''
+)
+
+# Note that in cupy/_core/include/cupy/complex.cuh, we already got isfinite and
+# isnan working for complex numbers, so just replace fabs above by abs (from
+# thrust) and we are ready to go
+_is_close_complex = _core.create_ufunc(
+    'cupy_is_close_complex',
+    ('FFff?->?', 'DDdd?->?'),
+    '''
+    bool equal_nan = in4;
+    if (isfinite(in0) && isfinite(in1)) {
+      out0 = abs(in0 - in1) <= in3 + in2 * abs(in1);
+    } else if (equal_nan) {
+      out0 = (in0 == in1) || (isnan(in0) && isnan(in1));
+    } else {
+      out0 = (in0 == in1);
+    }
+    '''
+)
+
+
+def array_equal(a1, a2, equal_nan=False):
+    """Returns ``True`` if two arrays are element-wise exactly equal.
+
+    Args:
+        a1 (cupy.ndarray): Input array to compare.
+        a2 (cupy.ndarray): Input array to compare.
+        equal_nan (bool): If ``True``, NaN's in ``a1`` will be considered equal
+            to NaN's in ``a2``.
+
+    Returns:
+        cupy.ndarray: A boolean 0-dim array.
+        If its value is ``True``, two arrays are element-wise equal.
+
+    .. seealso:: :func:`numpy.array_equal`
+
+    """
+    if a1.shape != a2.shape:
+        return cupy.array(False)
+    if not equal_nan:
+        return (a1 == a2).all()
+    # Handling NaN values if equal_nan is True
+    a1nan, a2nan = content.isnan(a1), content.isnan(a2)
+    # NaN's occur at different locations
+    if not (a1nan == a2nan).all():
+        return cupy.array(False)
+    # Shapes of a1, a2 and masks are guaranteed to be consistent by this point
+    return (a1[~a1nan] == a2[~a1nan]).all()
+
+
+def allclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False):
+    """Returns True if two arrays are element-wise equal within a tolerance.
+
+    Two values in ``a`` and ``b`` are  considiered equal when the following
+    equation is satisfied.
+
+    .. math::
+
+       |a - b| \\le \\mathrm{atol} + \\mathrm{rtol} |b|
+
+    Args:
+        a (cupy.ndarray): Input array to compare.
+        b (cupy.ndarray): Input array to compare.
+        rtol (float): The relative tolerance.
+        atol (float): The absolute tolerance.
+        equal_nan (bool): If ``True``, NaN's in ``a`` will be considered equal
+            to NaN's in ``b``.
+
+    Returns:
+        cupy.ndarray: A boolean 0-dim array.
+        If its value is ``True``, two arrays are element-wise equal within
+        a tolerance.
+
+    .. seealso:: :func:`numpy.allclose`
+
+    """
+    return isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan).all()
+
+
+def isclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False):
+    """Returns a boolean array where two arrays are equal within a tolerance.
+
+    Two values in ``a`` and ``b`` are  considiered equal when the following
+    equation is satisfied.
+
+    .. math::
+
+       |a - b| \\le \\mathrm{atol} + \\mathrm{rtol} |b|
+
+    Args:
+        a (cupy.ndarray): Input array to compare.
+        b (cupy.ndarray): Input array to compare.
+        rtol (float): The relative tolerance.
+        atol (float): The absolute tolerance.
+        equal_nan (bool): If ``True``, NaN's in ``a`` will be considered equal
+            to NaN's in ``b``.
+
+    Returns:
+        cupy.ndarray: A boolean array storing where ``a`` and ``b`` are equal.
+
+    .. seealso:: :func:`numpy.isclose`
+
+    """
+    a = cupy.asanyarray(a)
+    b = cupy.asanyarray(b)
+    if (a.dtype in [numpy.complex64, numpy.complex128]) or \
+       (b.dtype in [numpy.complex64, numpy.complex128]):
+        return _is_close_complex(a, b, rtol, atol, equal_nan)
+    else:
+        return _is_close(a, b, rtol, atol, equal_nan)
+
+
+def array_equiv(a1, a2):
+    """
+    Returns ``True`` if all elements are equal or shape consistent,
+    i.e., one input array can be broadcasted to create the same
+    shape as the other.
+
+    Args:
+        a1 (cupy.ndarray): Input array.
+        a2 (cupy.ndarray): Input array.
+
+    Returns:
+        cupy.ndarray: A boolean 0-dim array.
+            ``True`` if equivalent, otherwise ``False``.
+
+    .. seealso:: :func:`numpy.array_equiv`
+
+    """
+
+    try:
+        cupy.broadcast(a1, a2)
+    except Exception:
+        return cupy.array(False)
+
+    return (a1 == a2).all()
+
+
+greater: _core.ufunc = _core.greater
+
+
+greater_equal: _core.ufunc = _core.greater_equal
+
+
+less: _core.ufunc = _core.less
+
+
+less_equal: _core.ufunc = _core.less_equal
+
+
+equal: _core.ufunc = _core.equal
+
+
+not_equal: _core.ufunc = _core.not_equal
diff --git a/cupy/_logic/content.py b/cupy/_logic/content.py
new file mode 100644
index 0000000..851edc8
--- /dev/null
+++ b/cupy/_logic/content.py
@@ -0,0 +1,131 @@
+from cupy import _core
+
+import cupy
+
+
+def _create_float_test_ufunc(name, doc):
+    return _core.create_ufunc(
+        'cupy_' + name,
+        ('e->?', 'f->?', 'd->?', 'F->?', 'D->?',
+         ), 'out0 = %s(in0)' % name,
+        doc=doc)
+
+
+isfinite = _create_float_test_ufunc(
+    'isfinite',
+    '''Tests finiteness elementwise.
+
+    Each element of returned array is ``True`` only if the corresponding
+    element of the input is finite (i.e. not an infinity nor NaN).
+
+    .. seealso:: :data:`numpy.isfinite`
+
+    ''')
+
+
+isinf = _create_float_test_ufunc(
+    'isinf',
+    '''Tests if each element is the positive or negative infinity.
+
+    .. seealso:: :data:`numpy.isinf`
+
+    ''')
+
+
+isnan = _create_float_test_ufunc(
+    'isnan',
+    '''Tests if each element is a NaN.
+
+    .. seealso:: :data:`numpy.isnan`
+
+    ''')
+
+
+def isneginf(x, out=None):
+    """Test element-wise for negative infinity, return result as bool array.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        Input array.
+    out : cupy.ndarray, optional
+        A location into which the result is stored. If provided,
+        it should have a shape that input broadcasts to.
+        By default, None, a freshly- allocated boolean array,
+        is returned.
+
+    Returns
+    -------
+    y : cupy.ndarray
+        Boolean array of same shape as ``x``.
+
+    Examples
+    --------
+    >>> cupy.isneginf(0)
+    array(False)
+    >>> cupy.isneginf(-cupy.inf)
+    array(True)
+    >>> cupy.isneginf(cupy.array([-cupy.inf, -4, cupy.nan, 0, 4, cupy.inf]))
+    array([ True, False, False, False, False, False])
+
+    See Also
+    --------
+    numpy.isneginf
+
+    """
+
+    is_inf = isinf(x)
+    try:
+        signbit = cupy.signbit(x)
+    except TypeError as e:
+        dtype = x.dtype
+        raise TypeError(f'This operation is not supported for {dtype} values '
+                        'because it would be ambiguous.') from e
+
+    # TODO(khushi-411): Use `out` instead of `out=out` (see #6393)
+    return cupy.logical_and(is_inf, signbit, out=out)
+
+
+def isposinf(x, out=None):
+    """Test element-wise for positive infinity, return result as bool array.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        Input array.
+    out : cupy.ndarray
+        A location into which the result is stored. If provided,
+        it should have a shape that input broadcasts to.
+        By default, None, a freshly- allocated boolean array,
+        is returned.
+
+    Returns
+    -------
+    y : cupy.ndarray
+        Boolean array of same shape as ``x``.
+
+    Examples
+    --------
+    >>> cupy.isposinf(0)
+    array(False)
+    >>> cupy.isposinf(cupy.inf)
+    array(True)
+    >>> cupy.isposinf(cupy.array([-cupy.inf, -4, cupy.nan, 0, 4, cupy.inf]))
+    array([False, False, False, False, False,  True])
+
+    See Also
+    --------
+    numpy.isposinf
+
+    """
+
+    is_inf = isinf(x)
+    try:
+        signbit = ~cupy.signbit(x)
+    except TypeError as e:
+        dtype = x.dtype
+        raise TypeError(f'This operation is not supported for {dtype} values '
+                        'because it would be ambiguous.') from e
+
+    # TODO(khushi-411): Use `out` instead of `out=out` (see #6393)
+    return cupy.logical_and(is_inf, signbit, out=out)
diff --git a/cupy/_logic/ops.py b/cupy/_logic/ops.py
new file mode 100644
index 0000000..fee3916
--- /dev/null
+++ b/cupy/_logic/ops.py
@@ -0,0 +1,43 @@
+from cupy._core import _kernel
+from cupy._core import _routines_logic
+
+logical_and = _routines_logic.create_comparison(
+    'logical_and', '&&',
+    '''Computes the logical AND of two arrays.
+
+    .. seealso:: :data:`numpy.logical_and`
+
+    ''')
+
+
+logical_or = _routines_logic.create_comparison(
+    'logical_or', '||',
+    '''Computes the logical OR of two arrays.
+
+    .. seealso:: :data:`numpy.logical_or`
+
+    ''')
+
+
+logical_not = _kernel.create_ufunc(
+    'cupy_logical_not',
+    ('?->?', 'b->?', 'B->?', 'h->?', 'H->?', 'i->?', 'I->?', 'l->?', 'L->?',
+     'q->?', 'Q->?', 'e->?', 'f->?', 'd->?'),
+    'out0 = !in0',
+    doc='''Computes the logical NOT of an array.
+
+    .. seealso:: :data:`numpy.logical_not`
+
+    ''')
+
+
+logical_xor = _kernel.create_ufunc(
+    'cupy_logical_xor',
+    ('??->?', 'bb->?', 'BB->?', 'hh->?', 'HH->?', 'ii->?', 'II->?', 'll->?',
+     'LL->?', 'qq->?', 'QQ->?', 'ee->?', 'ff->?', 'dd->?'),
+    'out0 = !in0 != !in1',
+    doc='''Computes the logical XOR of two arrays.
+
+    .. seealso:: :data:`numpy.logical_xor`
+
+    ''')
diff --git a/cupy/_logic/truth.py b/cupy/_logic/truth.py
new file mode 100644
index 0000000..f88b61c
--- /dev/null
+++ b/cupy/_logic/truth.py
@@ -0,0 +1,316 @@
+import cupy
+from cupy._core import _routines_logic as _logic
+from cupy._core import _fusion_thread_local
+from cupy._sorting import search as _search
+from cupy import _util
+
+
+_setxorkernel = cupy._core.ElementwiseKernel(
+    'raw T X, int64 len',
+    'bool z',
+    'z = (i == 0 || X[i] != X[i-1]) && (i == len - 1 || X[i] != X[i+1])',
+    'setxorkernel'
+)
+
+
+def all(a, axis=None, out=None, keepdims=False):
+    """Tests whether all array elements along a given axis evaluate to True.
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        Input array.
+    axis : int or tuple of ints
+        Along which axis to compute all.
+        The flattened array is used by default.
+    out : cupy.ndarray
+        Output array.
+    keepdims : bool
+        If ``True``, the axis is remained as an axis of size one.
+
+    Returns
+    -------
+    y : cupy.ndarray
+        An array reduced of the input array along the axis.
+
+    See Also
+    --------
+    numpy.all
+
+    """
+    if _fusion_thread_local.is_fusing():
+        if keepdims:
+            raise NotImplementedError(
+                'cupy.all does not support `keepdims` in fusion yet.')
+        return _fusion_thread_local.call_reduction(
+            _logic.all, a, axis=axis, out=out)
+
+    _util.check_array(a, arg_name='a')
+
+    return a.all(axis=axis, out=out, keepdims=keepdims)
+
+
+def any(a, axis=None, out=None, keepdims=False):
+    """Tests whether any array elements along a given axis evaluate to True.
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        Input array.
+    axis : int or tuple of ints
+        Along which axis to compute all.
+        The flattened array is used by default.
+    out : cupy.ndarray
+        Output array.
+    keepdims : bool
+        If ``True``, the axis is remained as an axis of size one.
+
+    Returns
+    -------
+    y : cupy.ndarray
+        An array reduced of the input array along the axis.
+
+    See Also
+    --------
+    numpy.any
+
+    """
+    if _fusion_thread_local.is_fusing():
+        if keepdims:
+            raise NotImplementedError(
+                'cupy.any does not support `keepdims` in fusion yet.')
+        return _fusion_thread_local.call_reduction(
+            _logic.any, a, axis=axis, out=out)
+
+    _util.check_array(a, arg_name='a')
+
+    return a.any(axis=axis, out=out, keepdims=keepdims)
+
+
+def in1d(ar1, ar2, assume_unique=False, invert=False):
+    """Tests whether each element of a 1-D array is also present in a second
+    array.
+
+    Returns a boolean array the same length as ``ar1`` that is ``True``
+    where an element of ``ar1`` is in ``ar2`` and ``False`` otherwise.
+
+    Parameters
+    ----------
+    ar1 : cupy.ndarray
+        Input array.
+    ar2 : cupy.ndarray
+        The values against which to test each value of ``ar1``.
+    assume_unique : bool, optional
+        Ignored
+    invert : bool, optional
+        If ``True``, the values in the returned array
+        are inverted (that is, ``False`` where an element of ``ar1`` is in
+        ``ar2`` and ``True`` otherwise). Default is ``False``.
+
+    Returns
+    -------
+    y : cupy.ndarray, bool
+        The values ``ar1[in1d]`` are in ``ar2``.
+
+    """
+    # Ravel both arrays, behavior for the first array could be different
+    ar1 = ar1.ravel()
+    ar2 = ar2.ravel()
+    if ar1.size == 0 or ar2.size == 0:
+        if invert:
+            return cupy.ones(ar1.shape, dtype=cupy.bool_)
+        else:
+            return cupy.zeros(ar1.shape, dtype=cupy.bool_)
+    # Use brilliant searchsorted trick
+    # https://github.com/cupy/cupy/pull/4018#discussion_r495790724
+    ar2 = cupy.sort(ar2)
+    return _search._exists_kernel(ar1, ar2, ar2.size, invert)
+
+
+def intersect1d(arr1, arr2, assume_unique=False, return_indices=False):
+    """Find the intersection of two arrays.
+    Returns the sorted, unique values that are in both of the input arrays.
+
+    Parameters
+    ----------
+    arr1, arr2 : cupy.ndarray
+        Input arrays. Arrays will be flattened if they are not in 1D.
+    assume_unique : bool
+        By default, False. If set True, the input arrays will be
+        assumend to be unique, which speeds up the calculation. If set True,
+        but the arrays are not unique, incorrect results and out-of-bounds
+        indices could result.
+    return_indices : bool
+       By default, False. If True, the indices which correspond to the
+       intersection of the two arrays are returned.
+
+    Returns
+    -------
+    intersect1d : cupy.ndarray
+        Sorted 1D array of common and unique elements.
+    comm1 : cupy.ndarray
+        The indices of the first occurrences of the common values
+        in `arr1`. Only provided if `return_indices` is True.
+    comm2 : cupy.ndarray
+        The indices of the first occurrences of the common values
+        in `arr2`. Only provided if `return_indices` is True.
+
+    See Also
+    --------
+    numpy.intersect1d
+
+    """
+    if not assume_unique:
+        if return_indices:
+            arr1, ind1 = cupy.unique(arr1, return_index=True)
+            arr2, ind2 = cupy.unique(arr2, return_index=True)
+        else:
+            arr1 = cupy.unique(arr1)
+            arr2 = cupy.unique(arr2)
+    else:
+        arr1 = arr1.ravel()
+        arr2 = arr2.ravel()
+
+    if not return_indices:
+        mask = _search._exists_kernel(arr1, arr2, arr2.size, False)
+        return arr1[mask]
+
+    mask, v1 = _search._exists_and_searchsorted_kernel(
+        arr1, arr2, arr2.size, False)
+    int1d = arr1[mask]
+    arr1_indices = cupy.flatnonzero(mask)
+    arr2_indices = v1[mask]
+
+    if not assume_unique:
+        arr1_indices = ind1[arr1_indices]
+        arr2_indices = ind2[arr2_indices]
+
+    return int1d, arr1_indices, arr2_indices
+
+
+def isin(element, test_elements, assume_unique=False, invert=False):
+    """Calculates element in ``test_elements``, broadcasting over ``element``
+    only. Returns a boolean array of the same shape as ``element`` that is
+    ``True`` where an element of ``element`` is in ``test_elements`` and
+    ``False`` otherwise.
+
+    Parameters
+    ----------
+    element : cupy.ndarray
+        Input array.
+    test_elements : cupy.ndarray
+        The values against which to test each
+        value of ``element``. This argument is flattened if it is an
+        array or array_like.
+    assume_unique : bool, optional
+        Ignored
+    invert : bool, optional
+        If ``True``, the values in the returned array
+        are inverted, as if calculating element not in ``test_elements``.
+        Default is ``False``.
+
+    Returns
+    -------
+    y : cupy.ndarray, bool
+        Has the same shape as ``element``. The values ``element[isin]``
+        are in ``test_elements``.
+
+    """
+    return in1d(element, test_elements, assume_unique=assume_unique,
+                invert=invert).reshape(element.shape)
+
+
+def setdiff1d(ar1, ar2, assume_unique=False):
+    """Find the set difference of two arrays. It returns unique
+    values in `ar1` that are not in `ar2`.
+
+    Parameters
+    ----------
+    ar1 : cupy.ndarray
+        Input array
+    ar2 : cupy.ndarray
+        Input array for comparision
+    assume_unique : bool
+        By default, False, i.e. input arrays are not unique.
+        If True, input arrays are assumed to be unique. This can
+        speed up the calculation.
+
+    Returns
+    -------
+    setdiff1d : cupy.ndarray
+        Returns a 1D array of values in `ar1` that are not in `ar2`.
+        It always returns a sorted output for unsorted input only
+        if `assume_unique=False`.
+
+    See Also
+    --------
+    numpy.setdiff1d
+
+    """
+    if assume_unique:
+        ar1 = cupy.ravel(ar1)
+    else:
+        ar1 = cupy.unique(ar1)
+        ar2 = cupy.unique(ar2)
+    return ar1[in1d(ar1, ar2, assume_unique=True, invert=True)]
+
+
+def setxor1d(ar1, ar2, assume_unique=False):
+    """Find the set exclusive-or of two arrays.
+
+    Parameters
+    ----------
+    ar1, ar2 : cupy.ndarray
+        Input arrays. They are flattend if they are not already 1-D.
+    assume_unique : bool
+        By default, False, i.e. input arrays are not unique.
+        If True, input arrays are assumed to be unique. This can
+        speed up the calculation.
+
+    Returns
+    -------
+    setxor1d : cupy.ndarray
+        Return the sorted, unique values that are in only one
+        (not both) of the input arrays.
+
+    See Also
+    --------
+    numpy.setxor1d
+
+    """
+    if not assume_unique:
+        ar1 = cupy.unique(ar1)
+        ar2 = cupy.unique(ar2)
+
+    aux = cupy.concatenate((ar1, ar2), axis=None)
+    if aux.size == 0:
+        return aux
+
+    aux.sort()
+
+    return aux[_setxorkernel(aux, aux.size,
+                             cupy.zeros(aux.size, dtype=cupy.bool_))]
+
+
+def union1d(arr1, arr2):
+    """Find the union of two arrays.
+
+    Returns the unique, sorted array of values that are in either of
+    the two input arrays.
+
+    Parameters
+    ----------
+    arr1, arr2 : cupy.ndarray
+        Input arrays. They are flattend if they are not already 1-D.
+
+    Returns
+    -------
+    union1d : cupy.ndarray
+        Sorted union of the input arrays.
+
+    See Also
+    --------
+    numpy.union1d
+
+    """
+    return cupy.unique(cupy.concatenate((arr1, arr2), axis=None))
diff --git a/cupy/_logic/type_testing.py b/cupy/_logic/type_testing.py
new file mode 100644
index 0000000..68cb436
--- /dev/null
+++ b/cupy/_logic/type_testing.py
@@ -0,0 +1,183 @@
+import numpy
+
+import cupy
+
+
+def iscomplex(x):
+    """Returns a bool array, where True if input element is complex.
+
+    What is tested is whether the input has a non-zero imaginary part, not if
+    the input type is complex.
+
+    Args:
+        x (cupy.ndarray): Input array.
+
+    Returns:
+        cupy.ndarray: Boolean array of the same shape as ``x``.
+
+    .. seealso::
+        :func:`isreal`, :func:`iscomplexobj`
+
+    Examples
+    --------
+    >>> cupy.iscomplex(cupy.array([1+1j, 1+0j, 4.5, 3, 2, 2j]))
+    array([ True, False, False, False, False,  True])
+
+    """
+    if numpy.isscalar(x):
+        return numpy.iscomplex(x)
+    if not isinstance(x, cupy.ndarray):
+        return cupy.asarray(numpy.iscomplex(x))
+    if x.dtype.kind == 'c':
+        return x.imag != 0
+    return cupy.zeros(x.shape, bool)
+
+
+def iscomplexobj(x):
+    """Check for a complex type or an array of complex numbers.
+
+    The type of the input is checked, not the value. Even if the input
+    has an imaginary part equal to zero, `iscomplexobj` evaluates to True.
+
+    Args:
+        x (cupy.ndarray): Input array.
+
+    Returns:
+        bool: The return value, True if ``x`` is of a complex type or
+        has at least one complex element.
+
+    .. seealso::
+        :func:`isrealobj`, :func:`iscomplex`
+
+    Examples
+    --------
+    >>> cupy.iscomplexobj(cupy.array([3, 1+0j, True]))
+    True
+    >>> cupy.iscomplexobj(cupy.array([3, 1, True]))
+    False
+
+    """
+    if not isinstance(x, cupy.ndarray):
+        return numpy.iscomplexobj(x)
+    return x.dtype.kind == 'c'
+
+
+def isfortran(a):
+    """Returns True if the array is Fortran contiguous but *not* C contiguous.
+
+    If you only want to check if an array is Fortran contiguous use
+    ``a.flags.f_contiguous`` instead.
+
+    Args:
+        a (cupy.ndarray): Input array.
+
+    Returns:
+        bool: The return value, True if ``a`` is Fortran contiguous but not C
+        contiguous.
+
+    .. seealso::
+       :func:`~numpy.isfortran`
+
+    Examples
+    --------
+
+    cupy.array allows to specify whether the array is written in C-contiguous
+    order (last index varies the fastest), or FORTRAN-contiguous order in
+    memory (first index varies the fastest).
+
+    >>> a = cupy.array([[1, 2, 3], [4, 5, 6]], order='C')
+    >>> a
+    array([[1, 2, 3],
+           [4, 5, 6]])
+    >>> cupy.isfortran(a)
+    False
+
+    >>> b = cupy.array([[1, 2, 3], [4, 5, 6]], order='F')
+    >>> b
+    array([[1, 2, 3],
+           [4, 5, 6]])
+    >>> cupy.isfortran(b)
+    True
+
+    The transpose of a C-ordered array is a FORTRAN-ordered array.
+
+    >>> a = cupy.array([[1, 2, 3], [4, 5, 6]], order='C')
+    >>> a
+    array([[1, 2, 3],
+           [4, 5, 6]])
+    >>> cupy.isfortran(a)
+    False
+    >>> b = a.T
+    >>> b
+    array([[1, 4],
+           [2, 5],
+           [3, 6]])
+    >>> cupy.isfortran(b)
+    True
+
+    C-ordered arrays evaluate as False even if they are also FORTRAN-ordered.
+
+    >>> cupy.isfortran(np.array([1, 2], order='F'))
+    False
+
+    """
+    return a.flags.f_contiguous and not a.flags.c_contiguous
+
+
+def isreal(x):
+    """Returns a bool array, where True if input element is real.
+
+    If element has complex type with zero complex part, the return value
+    for that element is True.
+
+    Args:
+        x (cupy.ndarray): Input array.
+
+    Returns:
+        cupy.ndarray: Boolean array of same shape as ``x``.
+
+    .. seealso::
+        :func:`iscomplex`, :func:`isrealobj`
+
+    Examples
+    --------
+    >>> cupy.isreal(cp.array([1+1j, 1+0j, 4.5, 3, 2, 2j]))
+    array([False,  True,  True,  True,  True, False])
+
+    """
+    if numpy.isscalar(x):
+        return numpy.isreal(x)
+    if not isinstance(x, cupy.ndarray):
+        return cupy.asarray(numpy.isreal(x))
+    if x.dtype.kind == 'c':
+        return x.imag == 0
+    return cupy.ones(x.shape, bool)
+
+
+def isrealobj(x):
+    """Return True if x is a not complex type or an array of complex numbers.
+
+    The type of the input is checked, not the value. So even if the input
+    has an imaginary part equal to zero, `isrealobj` evaluates to False
+    if the data type is complex.
+
+    Args:
+        x (cupy.ndarray): The input can be of any type and shape.
+
+    Returns:
+        bool: The return value, False if ``x`` is of a complex type.
+
+    .. seealso::
+        :func:`iscomplexobj`, :func:`isreal`
+
+    Examples
+    --------
+    >>> cupy.isrealobj(cupy.array([3, 1+0j, True]))
+    False
+    >>> cupy.isrealobj(cupy.array([3, 1, True]))
+    True
+
+    """
+    if not isinstance(x, cupy.ndarray):
+        return numpy.isrealobj(x)
+    return x.dtype.kind != 'c'
diff --git a/cupy/_manipulation/__init__.py b/cupy/_manipulation/__init__.py
new file mode 100644
index 0000000..3112f86
--- /dev/null
+++ b/cupy/_manipulation/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.array-manipulation.html
diff --git a/cupy/_manipulation/add_remove.py b/cupy/_manipulation/add_remove.py
new file mode 100644
index 0000000..033b093
--- /dev/null
+++ b/cupy/_manipulation/add_remove.py
@@ -0,0 +1,213 @@
+import numpy
+
+import cupy
+from cupy import _core
+
+
+# TODO(okuta): Implement delete
+
+
+# TODO(okuta): Implement insert
+
+
+def append(arr, values, axis=None):
+    """
+    Append values to the end of an array.
+
+    Args:
+        arr (array_like):
+            Values are appended to a copy of this array.
+        values (array_like):
+            These values are appended to a copy of ``arr``.  It must be of the
+            correct shape (the same shape as ``arr``, excluding ``axis``).  If
+            ``axis`` is not specified, ``values`` can be any shape and will be
+            flattened before use.
+        axis (int or None):
+            The axis along which ``values`` are appended.  If ``axis`` is not
+            given, both ``arr`` and ``values`` are flattened before use.
+
+    Returns:
+        cupy.ndarray:
+            A copy of ``arr`` with ``values`` appended to ``axis``.  Note that
+            ``append`` does not occur in-place: a new array is allocated and
+            filled.  If ``axis`` is None, ``out`` is a flattened array.
+
+    .. seealso:: :func:`numpy.append`
+    """
+    # TODO(asi1024): Implement fast path for scalar inputs.
+    arr = cupy.asarray(arr)
+    values = cupy.asarray(values)
+    if axis is None:
+        return _core.concatenate_method(
+            (arr.ravel(), values.ravel()), 0).ravel()
+    return _core.concatenate_method((arr, values), axis)
+
+
+_resize_kernel = _core.ElementwiseKernel(
+    'raw T x, int64 size', 'T y',
+    'y = x[i % size]',
+    'cupy_resize',
+)
+
+
+def resize(a, new_shape):
+    """Return a new array with the specified shape.
+
+    If the new array is larger than the original array, then the new
+    array is filled with repeated copies of ``a``.  Note that this behavior
+    is different from a.resize(new_shape) which fills with zeros instead
+    of repeated copies of ``a``.
+
+    Args:
+        a (array_like): Array to be resized.
+        new_shape (int or tuple of int): Shape of resized array.
+
+    Returns:
+        cupy.ndarray:
+            The new array is formed from the data in the old array, repeated
+            if necessary to fill out the required number of elements.  The
+            data are repeated in the order that they are stored in memory.
+
+    .. seealso:: :func:`numpy.resize`
+    """
+    if numpy.isscalar(a):
+        return cupy.full(new_shape, a)
+    a = cupy.asarray(a)
+    if a.size == 0:
+        return cupy.zeros(new_shape, dtype=a.dtype)
+    out = cupy.empty(new_shape, a.dtype)
+    _resize_kernel(a, a.size, out)
+    return out
+
+
+_first_nonzero_krnl = _core.ReductionKernel(
+    'T data, int64 len',
+    'int64 y',
+    'data == T(0) ? len : _j',
+    'min(a, b)',
+    'y = a',
+    'len',
+    'first_nonzero'
+)
+
+
+def trim_zeros(filt, trim='fb'):
+    """Trim the leading and/or trailing zeros from a 1-D array or sequence.
+
+    Returns the trimmed array
+
+    Args:
+        filt(cupy.ndarray): Input array
+        trim(str, optional):
+            'fb' default option trims the array from both sides.
+            'f' option trim zeros from front.
+            'b' option trim zeros from back.
+
+    Returns:
+        cupy.ndarray: trimmed input
+
+    .. seealso:: :func:`numpy.trim_zeros`
+
+    """
+    if filt.ndim > 1:
+        raise ValueError('Multi-dimensional trim is not supported')
+    if not filt.ndim:
+        raise TypeError('0-d array cannot be trimmed')
+    start = 0
+    end = filt.size
+    trim = trim.upper()
+    if 'F' in trim:
+        start = _first_nonzero_krnl(filt, filt.size).item()
+    if 'B' in trim:
+        end = filt.size - _first_nonzero_krnl(filt[::-1], filt.size).item()
+    return filt[start:end]
+
+
+@_core.fusion.fuse()
+def _unique_update_mask_equal_nan(mask, x0):
+    mask1 = cupy.logical_not(cupy.isnan(x0))
+    mask[:] = cupy.logical_and(mask, mask1)
+
+
+def unique(ar, return_index=False, return_inverse=False,
+           return_counts=False, axis=None, *, equal_nan=True):
+    """Find the unique elements of an array.
+
+    Returns the sorted unique elements of an array. There are three optional
+    outputs in addition to the unique elements:
+
+    * the indices of the input array that give the unique values
+    * the indices of the unique array that reconstruct the input array
+    * the number of times each unique value comes up in the input array
+
+    Args:
+        ar(array_like): Input array. This will be flattened if it is not
+            already 1-D.
+        return_index(bool, optional): If True, also return the indices of `ar`
+            (along the specified axis, if provided, or in the flattened array)
+            that result in the unique array.
+        return_inverse(bool, optional): If True, also return the indices of the
+            unique array (for the specified axis, if provided) that can be used
+            to reconstruct `ar`.
+        return_counts(bool, optional): If True, also return the number of times
+            each unique item appears in `ar`.
+        axis(int or None, optional): Not supported yet.
+        equal_nan(bool, optional): If True, collapse multiple NaN values in the
+            return array into one.
+
+    Returns:
+        cupy.ndarray or tuple:
+            If there are no optional outputs, it returns the
+            :class:`cupy.ndarray` of the sorted unique values. Otherwise, it
+            returns the tuple which contains the sorted unique values and
+            followings.
+
+            * The indices of the first occurrences of the unique values in the
+              original array. Only provided if `return_index` is True.
+            * The indices to reconstruct the original array from the
+              unique array. Only provided if `return_inverse` is True.
+            * The number of times each of the unique values comes up in the
+              original array. Only provided if `return_counts` is True.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.unique`
+    """
+    if axis is not None:
+        raise NotImplementedError('axis option is not supported yet.')
+
+    ar = cupy.asarray(ar).flatten()
+
+    if return_index or return_inverse:
+        perm = ar.argsort()
+        aux = ar[perm]
+    else:
+        ar.sort()
+        aux = ar
+    mask = cupy.empty(aux.shape, dtype=cupy.bool_)
+    mask[:1] = True
+    mask[1:] = aux[1:] != aux[:-1]
+    if equal_nan:
+        _unique_update_mask_equal_nan(mask[1:], aux[:-1])
+
+    ret = aux[mask]
+    if not return_index and not return_inverse and not return_counts:
+        return ret
+
+    ret = ret,
+    if return_index:
+        ret += perm[mask],
+    if return_inverse:
+        imask = cupy.cumsum(mask) - 1
+        inv_idx = cupy.empty(mask.shape, dtype=cupy.intp)
+        inv_idx[perm] = imask
+        ret += inv_idx,
+    if return_counts:
+        nonzero = cupy.nonzero(mask)[0]  # may synchronize
+        idx = cupy.empty((nonzero.size + 1,), nonzero.dtype)
+        idx[:-1] = nonzero
+        idx[-1] = mask.size
+        ret += idx[1:] - idx[:-1],
+    return ret
diff --git a/cupy/_manipulation/basic.py b/cupy/_manipulation/basic.py
new file mode 100644
index 0000000..f2bb864
--- /dev/null
+++ b/cupy/_manipulation/basic.py
@@ -0,0 +1,114 @@
+import itertools
+
+import numpy
+
+from cupy import _core
+from cupy._core import _fusion_interface
+from cupy._core import fusion
+from cupy._sorting import search
+from cupy_backends.cuda.api import runtime
+
+
+def copyto(dst, src, casting='same_kind', where=None):
+    """Copies values from one array to another with broadcasting.
+
+    This function can be called for arrays on different devices. In this case,
+    casting, ``where``, and broadcasting is not supported, and an exception is
+    raised if these are used.
+
+    Args:
+        dst (cupy.ndarray): Target array.
+        src (cupy.ndarray): Source array.
+        casting (str): Casting rule. See :func:`numpy.can_cast` for detail.
+        where (cupy.ndarray of bool): If specified, this array acts as a mask,
+            and an element is copied only if the corresponding element of
+            ``where`` is True.
+
+    .. seealso:: :func:`numpy.copyto`
+
+    """
+    src_is_numpy_scalar = False
+
+    src_type = type(src)
+    src_is_python_scalar = src_type in (
+        int, bool, float, complex,
+        fusion._FusionVarScalar, _fusion_interface._ScalarProxy)
+    if src_is_python_scalar:
+        src_dtype = numpy.dtype(type(src))
+        can_cast = numpy.can_cast(src, dst.dtype, casting)
+    elif isinstance(src, numpy.ndarray) or numpy.isscalar(src):
+        if src.size != 1:
+            raise ValueError(
+                'non-scalar numpy.ndarray cannot be used for copyto')
+        src_dtype = src.dtype
+        can_cast = numpy.can_cast(src, dst.dtype, casting)
+        src = src.item()
+        src_is_numpy_scalar = True
+    else:
+        src_dtype = src.dtype
+        can_cast = numpy.can_cast(src_dtype, dst.dtype, casting)
+
+    if not can_cast:
+        raise TypeError('Cannot cast %s to %s in %s casting mode' %
+                        (src_dtype, dst.dtype, casting))
+
+    if fusion._is_fusing():
+        # TODO(kataoka): NumPy allows stripping leading unit dimensions.
+        # But fusion array proxy does not currently support
+        # `shape` and `squeeze`.
+
+        if where is None:
+            _core.elementwise_copy(src, dst)
+        else:
+            fusion._call_ufunc(search._where_ufunc, where, src, dst, dst)
+        return
+
+    if not src_is_python_scalar and not src_is_numpy_scalar:
+        # Check broadcast condition
+        # - for fast-paths and
+        # - for a better error message (than ufunc's).
+        # NumPy allows stripping leading unit dimensions.
+        if not all([
+            s in (d, 1)
+            for s, d in itertools.zip_longest(
+                reversed(src.shape), reversed(dst.shape), fillvalue=1)
+        ]):
+            raise ValueError(
+                "could not broadcast input array "
+                f"from shape {src.shape} into shape {dst.shape}")
+        squeeze_ndim = src.ndim - dst.ndim
+        if squeeze_ndim > 0:
+            # always succeeds because broadcast conition is checked.
+            src = src.squeeze(tuple(range(squeeze_ndim)))
+
+    if where is not None:
+        _core.elementwise_copy(src, dst, _where=where)
+        return
+
+    if dst.size == 0:
+        return
+
+    if src_is_python_scalar or src_is_numpy_scalar:
+        _core.elementwise_copy(src, dst)
+        return
+
+    if _can_memcpy(dst, src):
+        dst.data.copy_from_async(src.data, src.nbytes)
+        return
+
+    device = dst.device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        if src.device != device:
+            src = src.copy()
+        _core.elementwise_copy(src, dst)
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def _can_memcpy(dst, src):
+    c_contiguous = dst.flags.c_contiguous and src.flags.c_contiguous
+    f_contiguous = dst.flags.f_contiguous and src.flags.f_contiguous
+    return (c_contiguous or f_contiguous) and dst.dtype == src.dtype and \
+        dst.size == src.size
diff --git a/cupy/_manipulation/dims.py b/cupy/_manipulation/dims.py
new file mode 100644
index 0000000..65d8648
--- /dev/null
+++ b/cupy/_manipulation/dims.py
@@ -0,0 +1,167 @@
+import cupy
+from cupy import _core
+import cupy._core._routines_manipulation as _manipulation
+
+
+# Shape map for atleast_nd functions
+# (minimum dimension, input dimension) -> (output shape)
+_atleast_nd_shape_map = {
+    (1, 0): lambda shape: (1,),
+    (2, 0): lambda shape: (1, 1),
+    (2, 1): lambda shape: (1,) + shape,
+    (3, 0): lambda shape: (1, 1, 1),
+    (3, 1): lambda shape: (1,) + shape + (1,),
+    (3, 2): lambda shape: shape + (1,),
+}
+
+
+def _atleast_nd_helper(n, arys):
+    """Helper function for atleast_nd functions."""
+
+    res = []
+    for a in arys:
+        a = cupy.asarray(a)
+        if a.ndim < n:
+            new_shape = _atleast_nd_shape_map[(n, a.ndim)](a.shape)
+            a = a.reshape(*new_shape)
+        res.append(a)
+
+    if len(res) == 1:
+        res, = res
+    return res
+
+
+def atleast_1d(*arys):
+    """Converts arrays to arrays with dimensions >= 1.
+
+    Args:
+        arys (tuple of arrays): Arrays to be converted. All arguments must be
+            :class:`cupy.ndarray` objects. Only zero-dimensional array is
+            affected.
+
+    Returns:
+        If there are only one input, then it returns its converted version.
+        Otherwise, it returns a list of converted arrays.
+
+    .. seealso:: :func:`numpy.atleast_1d`
+
+    """
+    return _atleast_nd_helper(1, arys)
+
+
+def atleast_2d(*arys):
+    """Converts arrays to arrays with dimensions >= 2.
+
+    If an input array has dimensions less than two, then this function inserts
+    new axes at the head of dimensions to make it have two dimensions.
+
+    Args:
+        arys (tuple of arrays): Arrays to be converted. All arguments must be
+            :class:`cupy.ndarray` objects.
+
+    Returns:
+        If there are only one input, then it returns its converted version.
+        Otherwise, it returns a list of converted arrays.
+
+    .. seealso:: :func:`numpy.atleast_2d`
+
+    """
+    return _atleast_nd_helper(2, arys)
+
+
+def atleast_3d(*arys):
+    """Converts arrays to arrays with dimensions >= 3.
+
+    If an input array has dimensions less than three, then this function
+    inserts new axes to make it have three dimensions. The place of the new
+    axes are following:
+
+    - If its shape is ``()``, then the shape of output is ``(1, 1, 1)``.
+    - If its shape is ``(N,)``, then the shape of output is ``(1, N, 1)``.
+    - If its shape is ``(M, N)``, then the shape of output is ``(M, N, 1)``.
+    - Otherwise, the output is the input array itself.
+
+    Args:
+        arys (tuple of arrays): Arrays to be converted. All arguments must be
+            :class:`cupy.ndarray` objects.
+
+    Returns:
+        If there are only one input, then it returns its converted version.
+        Otherwise, it returns a list of converted arrays.
+
+    .. seealso:: :func:`numpy.atleast_3d`
+
+    """
+    return _atleast_nd_helper(3, arys)
+
+
+broadcast = _core.broadcast
+
+
+def broadcast_arrays(*args):
+    """Broadcasts given arrays.
+
+    Args:
+        args (tuple of arrays): Arrays to broadcast for each other.
+
+    Returns:
+        list: A list of broadcasted arrays.
+
+    .. seealso:: :func:`numpy.broadcast_arrays`
+
+    """
+    return list(broadcast(*args).values)
+
+
+def broadcast_to(array, shape):
+    """Broadcast an array to a given shape.
+
+    Args:
+        array (cupy.ndarray): Array to broadcast.
+        shape (tuple of int): The shape of the desired array.
+
+    Returns:
+        cupy.ndarray: Broadcasted view.
+
+    .. seealso:: :func:`numpy.broadcast_to`
+
+    """
+    return _core.broadcast_to(array, shape)
+
+
+def expand_dims(a, axis):
+    """Expands given arrays.
+
+    Args:
+        a (cupy.ndarray): Array to be expanded.
+        axis (int): Position where new axis is to be inserted.
+
+    Returns:
+        cupy.ndarray: The number of dimensions is one greater than that of
+        the input array.
+
+    .. seealso:: :func:`numpy.expand_dims`
+
+    """
+    if type(axis) not in (tuple, list):
+        axis = axis,
+    return _manipulation._expand_dims(a, axis)
+
+
+def squeeze(a, axis=None):
+    """Removes size-one axes from the shape of an array.
+
+    Args:
+        a (cupy.ndarray): Array to be reshaped.
+        axis (int or tuple of ints): Axes to be removed. This function removes
+            all size-one axes by default. If one of the specified axes is not
+            of size one, an exception is raised.
+
+    Returns:
+        cupy.ndarray: An array without (specified) size-one axes.
+
+    .. seealso:: :func:`numpy.squeeze`
+
+    """
+    # TODO(okuta): check type
+    return a.squeeze(axis)
diff --git a/cupy/_manipulation/join.py b/cupy/_manipulation/join.py
new file mode 100644
index 0000000..a1b82a6
--- /dev/null
+++ b/cupy/_manipulation/join.py
@@ -0,0 +1,135 @@
+import cupy
+from cupy import _core
+
+
+def column_stack(tup):
+    """Stacks 1-D and 2-D arrays as columns into a 2-D array.
+
+    A 1-D array is first converted to a 2-D column array. Then, the 2-D arrays
+    are concatenated along the second axis.
+
+    Args:
+        tup (sequence of arrays): 1-D or 2-D arrays to be stacked.
+
+    Returns:
+        cupy.ndarray: A new 2-D array of stacked columns.
+
+    .. seealso:: :func:`numpy.column_stack`
+
+    """
+    if any(not isinstance(a, cupy.ndarray) for a in tup):
+        raise TypeError('Only cupy arrays can be column stacked')
+
+    lst = list(tup)
+    for i, a in enumerate(lst):
+        if a.ndim == 1:
+            a = a[:, cupy.newaxis]
+            lst[i] = a
+        elif a.ndim != 2:
+            raise ValueError(
+                'Only 1 or 2 dimensional arrays can be column stacked')
+
+    return concatenate(lst, axis=1)
+
+
+def concatenate(tup, axis=0, out=None, *, dtype=None, casting='same_kind'):
+    """Joins arrays along an axis.
+
+    Args:
+        tup (sequence of arrays): Arrays to be joined. All of these should have
+            same dimensionalities except the specified axis.
+        axis (int or None): The axis to join arrays along.
+            If axis is None, arrays are flattened before use.
+            Default is 0.
+        out (cupy.ndarray): Output array.
+        dtype (str or dtype): If provided, the destination array will have this
+            dtype. Cannot be provided together with ``out``.
+        casting ({‘no’, ‘equiv’, ‘safe’, ‘same_kind’, ‘unsafe’}, optional):
+            Controls what kind of data casting may occur. Defaults to
+            ``'same_kind'``.
+
+    Returns:
+        cupy.ndarray: Joined array.
+
+    .. seealso:: :func:`numpy.concatenate`
+
+    """
+    if axis is None:
+        tup = [m.ravel() for m in tup]
+        axis = 0
+    return _core.concatenate_method(tup, axis, out, dtype, casting)
+
+
+def dstack(tup):
+    """Stacks arrays along the third axis.
+
+    Args:
+        tup (sequence of arrays): Arrays to be stacked. Each array is converted
+            by :func:`cupy.atleast_3d` before stacking.
+
+    Returns:
+        cupy.ndarray: Stacked array.
+
+    .. seealso:: :func:`numpy.dstack`
+
+    """
+    return concatenate([cupy.atleast_3d(m) for m in tup], 2)
+
+
+def hstack(tup):
+    """Stacks arrays horizontally.
+
+    If an input array has one dimension, then the array is treated as a
+    horizontal vector and stacked along the first axis. Otherwise, the array is
+    stacked along the second axis.
+
+    Args:
+        tup (sequence of arrays): Arrays to be stacked.
+
+    Returns:
+        cupy.ndarray: Stacked array.
+
+    .. seealso:: :func:`numpy.hstack`
+
+    """
+    arrs = [cupy.atleast_1d(a) for a in tup]
+    axis = 1
+    if arrs[0].ndim == 1:
+        axis = 0
+    return concatenate(arrs, axis)
+
+
+def vstack(tup):
+    """Stacks arrays vertically.
+
+    If an input array has one dimension, then the array is treated as a
+    horizontal vector and stacked along the additional axis at the head.
+    Otherwise, the array is stacked along the first axis.
+
+    Args:
+        tup (sequence of arrays): Arrays to be stacked. Each array is converted
+            by :func:`cupy.atleast_2d` before stacking.
+
+    Returns:
+        cupy.ndarray: Stacked array.
+
+    .. seealso:: :func:`numpy.dstack`
+
+    """
+    return concatenate([cupy.atleast_2d(m) for m in tup], 0)
+
+
+def stack(tup, axis=0, out=None):
+    """Stacks arrays along a new axis.
+
+    Args:
+        tup (sequence of arrays): Arrays to be stacked.
+        axis (int): Axis along which the arrays are stacked.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: Stacked array.
+
+    .. seealso:: :func:`numpy.stack`
+    """
+    return concatenate([cupy.expand_dims(x, axis) for x in tup], axis, out)
diff --git a/cupy/_manipulation/kind.py b/cupy/_manipulation/kind.py
new file mode 100644
index 0000000..0d1c3aa
--- /dev/null
+++ b/cupy/_manipulation/kind.py
@@ -0,0 +1,122 @@
+import cupy
+from cupy import _core
+
+
+def asarray_chkfinite(a, dtype=None, order=None):
+    """Converts the given input to an array,
+    and raises an error if the input contains NaNs or Infs.
+
+    Args:
+        a: array like.
+        dtype: data type, optional
+        order: {'C', 'F', 'A', 'K'}, optional
+
+    Returns:
+        cupy.ndarray: An array on the current device.
+
+    .. note::
+        This function performs device synchronization.
+
+    .. seealso:: :func:`numpy.asarray_chkfinite`
+
+    """
+
+    a = cupy.asarray(a, dtype=dtype, order=order)
+    if not cupy.isfinite(a).all():
+        raise ValueError("array must not contain Infs or NaNs")
+    return a
+
+
+def asfarray(a, dtype=cupy.float_):
+    """Converts array elements to float type.
+
+    Args:
+        a (cupy.ndarray): Source array.
+        dtype: str or dtype object, optional
+
+    Returns:
+        cupy.ndarray: The input array ``a`` as a float ndarray.
+
+    .. seealso:: :func:`numpy.asfarray`
+
+    """
+    if not cupy.issubdtype(dtype, cupy.inexact):
+        dtype = cupy.float_
+    return cupy.asarray(a, dtype=dtype)
+
+
+def asfortranarray(a, dtype=None):
+    """Return an array laid out in Fortran order in memory.
+
+    Args:
+        a (~cupy.ndarray): The input array.
+        dtype (str or dtype object, optional): By default, the data-type is
+            inferred from the input data.
+
+    Returns:
+        ~cupy.ndarray: The input `a` in Fortran, or column-major, order.
+
+    .. seealso:: :func:`numpy.asfortranarray`
+
+    """
+    return _core.asfortranarray(a, dtype)
+
+
+def require(a, dtype=None, requirements=None):
+    """Return an array which satisfies the requirements.
+
+    Args:
+        a (~cupy.ndarray): The input array.
+        dtype (str or dtype object, optional): The required data-type.
+            If None preserve the current dtype.
+        requirements (str or list of str): The requirements can be any
+            of the following
+
+            * 'F_CONTIGUOUS' ('F', 'FORTRAN') - ensure a Fortran-contiguous \
+                array. \
+
+            * 'C_CONTIGUOUS' ('C', 'CONTIGUOUS') - ensure a C-contiguous array.
+
+            * 'OWNDATA' ('O')      - ensure an array that owns its own data.
+
+    Returns:
+        ~cupy.ndarray: The input array ``a`` with specified requirements and
+        type if provided.
+
+    .. seealso:: :func:`numpy.require`
+
+    """
+
+    possible_flags = {'C': 'C', 'C_CONTIGUOUS': 'C', 'CONTIGUOUS': 'C',
+                      'F': 'F', 'F_CONTIGUOUS': 'F', 'FORTRAN': 'F',
+                      'O': 'OWNDATA', 'OWNDATA': 'OWNDATA'}
+
+    if not requirements:
+        try:
+            return cupy.asanyarray(a, dtype=dtype)
+        except TypeError:
+            raise ValueError("Incorrect dtype \"{}\" provided".format(dtype))
+    else:
+        try:
+            requirements = {possible_flags[x.upper()] for x in requirements}
+        except KeyError:
+            raise ValueError("Incorrect flag \"{}\" in requirements".format(
+                             (set(requirements) -
+                              set(possible_flags.keys())).pop()))
+
+    order = 'A'
+    if requirements >= {'C', 'F'}:
+        raise ValueError('Cannot specify both "C" and "F" order')
+    elif 'F' in requirements:
+        order = 'F_CONTIGUOUS'
+        requirements.remove('F')
+    elif 'C' in requirements:
+        order = 'C_CONTIGUOUS'
+        requirements.remove('C')
+
+    copy = 'OWNDATA' in requirements
+    try:
+        arr = cupy.array(a, dtype=dtype, order=order, copy=copy, subok=False)
+    except TypeError:
+        raise ValueError("Incorrect dtype \"{}\" provided".format(dtype))
+    return arr
diff --git a/cupy/_manipulation/rearrange.py b/cupy/_manipulation/rearrange.py
new file mode 100644
index 0000000..d81b3b1
--- /dev/null
+++ b/cupy/_manipulation/rearrange.py
@@ -0,0 +1,200 @@
+import itertools
+
+import numpy
+
+import cupy
+from cupy._core import internal
+
+
+def flip(a, axis=None):
+    """Reverse the order of elements in an array along the given axis.
+
+    Note that ``flip`` function has been introduced since NumPy v1.12.
+    The contents of this document is the same as the original one.
+
+    Args:
+        a (~cupy.ndarray): Input array.
+        axis (int or tuple of int or None): Axis or axes along which to flip
+            over. The default, ``axis=None``, will flip over all of the axes of
+            the input array. If axis is negative it counts from the last to the
+            first axis. If axis is a tuple of ints, flipping is performed on
+            all of the axes specified in the tuple.
+
+    Returns:
+        ~cupy.ndarray: Output array.
+
+    .. seealso:: :func:`numpy.flip`
+
+    """
+    axes = internal._normalize_axis_indices(axis, a.ndim)
+    return _flip(a, axes)
+
+
+def fliplr(a):
+    """Flip array in the left/right direction.
+
+    Flip the entries in each row in the left/right direction. Columns
+    are preserved, but appear in a different order than before.
+
+    Args:
+        a (~cupy.ndarray): Input array.
+
+    Returns:
+        ~cupy.ndarray: Output array.
+
+    .. seealso:: :func:`numpy.fliplr`
+
+    """
+    if a.ndim < 2:
+        raise ValueError('Input must be >= 2-d')
+    return a[::, ::-1]
+
+
+def flipud(a):
+    """Flip array in the up/down direction.
+
+    Flip the entries in each column in the up/down direction. Rows are
+    preserved, but appear in a different order than before.
+
+    Args:
+        a (~cupy.ndarray): Input array.
+
+    Returns:
+        ~cupy.ndarray: Output array.
+
+    .. seealso:: :func:`numpy.flipud`
+
+    """
+    if a.ndim < 1:
+        raise ValueError('Input must be >= 1-d')
+    return a[::-1]
+
+
+def roll(a, shift, axis=None):
+    """Roll array elements along a given axis.
+
+    Elements that roll beyond the last position are re-introduced at the first.
+
+    Args:
+        a (~cupy.ndarray): Array to be rolled.
+        shift (int or tuple of int): The number of places by which elements are
+            shifted. If a tuple, then `axis` must be a tuple of the same size,
+            and each of the given axes is shifted by the corresponding number.
+            If an int while `axis` is a tuple of ints, then the same value is
+            used for all given axes.
+        axis (int or tuple of int or None): The axis along which elements are
+            shifted. By default, the array is flattened before shifting, after
+            which the original shape is restored.
+
+    Returns:
+        ~cupy.ndarray: Output array.
+
+    .. seealso:: :func:`numpy.roll`
+
+    """
+    if axis is None:
+        return roll(a.ravel(), shift, 0).reshape(a.shape)
+
+    axes = (axis,) if numpy.isscalar(axis) else axis
+    axes = tuple([  # allow_duplicate
+        internal._normalize_axis_index(ax, a.ndim) for ax in axes
+    ])
+    if isinstance(shift, cupy.ndarray):
+        shift = shift.ravel()
+        n_axes = max(len(axes), shift.size)
+        axes = numpy.broadcast_to(axes, (n_axes,))
+        shift = cupy.broadcast_to(shift, (n_axes,))
+
+        # TODO(asi1024): Improve after issue #4799 is resolved.
+        indices = []
+        for ax in range(a.ndim):
+            ind_shape = [1] * a.ndim
+            ind_shape[ax] = a.shape[ax]
+            indices.append(cupy.arange(a.shape[ax]).reshape(ind_shape))
+
+        for ax, s in zip(axes, shift):
+            indices[ax] -= s
+            indices[ax] %= a.shape[ax]
+
+        for ax in range(a.ndim):
+            indices[ax] = cupy.broadcast_to(indices[ax], a.shape)
+
+        return a[tuple(indices)]
+    else:
+        broadcasted = numpy.broadcast(shift, axes)
+        if broadcasted.nd > 1:
+            raise ValueError(
+                '\'shift\' and \'axis\' should be scalars or 1D sequences')
+        shifts = {ax: 0 for ax in range(a.ndim)}
+        for sh, ax in broadcasted:
+            shifts[ax] += sh
+
+        rolls = [((slice(None), slice(None)),)] * a.ndim
+        for ax, offset in shifts.items():
+            offset %= a.shape[ax] or 1  # If `a` is empty, nothing matters.
+            if offset:
+                # (original, result), (original, result)
+                rolls[ax] = ((slice(None, -offset), slice(offset, None)),
+                             (slice(-offset, None), slice(None, offset)))
+
+        result = cupy.empty_like(a)
+        for indices in itertools.product(*rolls):
+            arr_index, res_index = zip(*indices)
+            result[res_index] = a[arr_index]
+
+        return result
+
+
+def rot90(a, k=1, axes=(0, 1)):
+    """Rotate an array by 90 degrees in the plane specified by axes.
+
+    Note that ``axes`` argument has been introduced since NumPy v1.12.
+    The contents of this document is the same as the original one.
+
+    Args:
+        a (~cupy.ndarray): Array of two or more dimensions.
+        k (int): Number of times the array is rotated by 90 degrees.
+        axes: (tuple of ints): The array is rotated in the plane defined by
+            the axes. Axes must be different.
+
+    Returns:
+        ~cupy.ndarray: Output array.
+
+    .. seealso:: :func:`numpy.rot90`
+
+    """
+    a_ndim = a.ndim
+    if a_ndim < 2:
+        raise ValueError('Input must be >= 2-d')
+
+    axes = tuple(axes)
+    if len(axes) != 2:
+        raise ValueError('len(axes) must be 2')
+    if axes[0] == axes[1] or abs(axes[0] - axes[1]) == a_ndim:
+        raise ValueError('axes must be different')
+    if not (-a_ndim <= axes[0] < a_ndim and -a_ndim <= axes[1] < a_ndim):
+        raise ValueError('axes must be >= %d and < %d' % (-a_ndim, a_ndim))
+
+    k = k % 4
+
+    if k == 0:
+        return a[:]
+    if k == 2:
+        return _flip(a, axes)
+
+    axes_t = list(range(0, a_ndim))
+    axes_t[axes[0]], axes_t[axes[1]] = axes_t[axes[1]], axes_t[axes[0]]
+
+    if k == 1:
+        return cupy.transpose(_flip(a, (axes[1],)), axes_t)
+    else:
+        return _flip(cupy.transpose(a, axes_t), (axes[1],))
+
+
+def _flip(a, axes):
+    # This function flips array without checking args.
+    indexer = [slice(None)] * a.ndim
+    for ax in axes:
+        indexer[ax] = slice(None, None, -1)
+
+    return a[tuple(indexer)]
diff --git a/cupy/_manipulation/shape.py b/cupy/_manipulation/shape.py
new file mode 100644
index 0000000..1fa4225
--- /dev/null
+++ b/cupy/_manipulation/shape.py
@@ -0,0 +1,89 @@
+import numpy
+
+import cupy
+
+
+def shape(a):
+    """Returns the shape of an array
+
+    Args:
+        a (array_like): Input array
+
+    Returns:
+        tuple of ints: The elements of the shape tuple give the lengths of the
+        corresponding array dimensions.
+
+    """
+    if isinstance(a, cupy.ndarray):
+        return a.shape
+    else:
+        return numpy.shape(a)
+
+
+def reshape(a, newshape, order='C'):
+    """Returns an array with new shape and same elements.
+
+    It tries to return a view if possible, otherwise returns a copy.
+
+    Args:
+        a (cupy.ndarray): Array to be reshaped.
+        newshape (int or tuple of ints): The new shape of the array to return.
+            If it is an integer, then it is treated as a tuple of length one.
+            It should be compatible with ``a.size``. One of the elements can be
+            -1, which is automatically replaced with the appropriate value to
+            make the shape compatible with ``a.size``.
+        order ({'C', 'F', 'A'}):
+            Read the elements of ``a`` using this index order, and place the
+            elements into the reshaped array using this index order.
+            'C' means to read / write the elements using C-like index order,
+            with the last axis index changing fastest, back to the first axis
+            index changing slowest. 'F' means to read / write the elements
+            using Fortran-like index order, with the first index changing
+            fastest, and the last index changing slowest. Note that the 'C'
+            and 'F' options take no account of the memory layout of the
+            underlying array, and only refer to the order of indexing. 'A'
+            means to read / write the elements in Fortran-like index order if
+            a is Fortran contiguous in memory, C-like order otherwise.
+
+    Returns:
+        cupy.ndarray: A reshaped view of ``a`` if possible, otherwise a copy.
+
+    .. seealso:: :func:`numpy.reshape`
+
+    """
+    # TODO(okuta): check type
+    return a.reshape(newshape, order=order)
+
+
+def ravel(a, order='C'):
+    """Returns a flattened array.
+
+    It tries to return a view if possible, otherwise returns a copy.
+
+    Args:
+        a (cupy.ndarray): Array to be flattened.
+        order ({'C', 'F', 'A', 'K'}):
+            The elements of ``a`` are read using this index order. 'C' means
+            to index the elements in row-major, C-style order,
+            with the last axis index changing fastest, back to the first
+            axis index changing slowest.  'F' means to index the elements
+            in column-major, Fortran-style order, with the
+            first index changing fastest, and the last index changing
+            slowest. Note that the 'C' and 'F' options take no account of
+            the memory layout of the underlying array, and only refer to
+            the order of axis indexing.  'A' means to read the elements in
+            Fortran-like index order if ``a`` is Fortran *contiguous* in
+            memory, C-like order otherwise.  'K' means to read the
+            elements in the order they occur in memory, except for
+            reversing the data when strides are negative.  By default, 'C'
+            index order is used.
+
+    Returns:
+        cupy.ndarray: A flattened view of ``a`` if possible, otherwise a copy.
+
+    .. seealso:: :func:`numpy.ravel`
+
+    """
+    # TODO(beam2d, grlee77): Support ordering option K
+    # TODO(okuta): check type
+    return a.ravel(order)
diff --git a/cupy/_manipulation/split.py b/cupy/_manipulation/split.py
new file mode 100644
index 0000000..8c2f8c4
--- /dev/null
+++ b/cupy/_manipulation/split.py
@@ -0,0 +1,91 @@
+import numpy
+
+from cupy import _core
+
+
+def array_split(ary, indices_or_sections, axis=0):
+    """Splits an array into multiple sub arrays along a given axis.
+
+    This function is almost equivalent to :func:`cupy.split`. The only
+    difference is that this function allows an integer sections that does not
+    evenly divide the axis.
+
+    .. seealso:: :func:`cupy.split` for more detail, :func:`numpy.array_split`
+
+    """
+    return _core.array_split(ary, indices_or_sections, axis)
+
+
+def dsplit(ary, indices_or_sections):
+    """Splits an array into multiple sub arrays along the third axis.
+
+    This is equivalent to ``split`` with ``axis=2``.
+
+    .. seealso:: :func:`cupy.split` for more detail, :func:`numpy.dsplit`
+
+    """
+    if ary.ndim <= 2:
+        raise ValueError('Cannot dsplit an array with less than 3 dimensions')
+    return split(ary, indices_or_sections, 2)
+
+
+def hsplit(ary, indices_or_sections):
+    """Splits an array into multiple sub arrays horizontally.
+
+    This is equivalent to ``split`` with ``axis=0`` if ``ary`` has one
+    dimension, and otherwise that with ``axis=1``.
+
+    .. seealso:: :func:`cupy.split` for more detail, :func:`numpy.hsplit`
+
+    """
+    if ary.ndim == 0:
+        raise ValueError('Cannot hsplit a zero-dimensional array')
+    if ary.ndim == 1:
+        return split(ary, indices_or_sections, 0)
+    else:
+        return split(ary, indices_or_sections, 1)
+
+
+def split(ary, indices_or_sections, axis=0):
+    """Splits an array into multiple sub arrays along a given axis.
+
+    Args:
+        ary (cupy.ndarray): Array to split.
+        indices_or_sections (int or sequence of ints): A value indicating how
+            to divide the axis. If it is an integer, then is treated as the
+            number of sections, and the axis is evenly divided. Otherwise,
+            the integers indicate indices to split at. Note that the sequence
+            on the device memory is not allowed.
+        axis (int): Axis along which the array is split.
+
+    Returns:
+        A list of sub arrays. Each array is a view of the corresponding input
+        array.
+
+    .. seealso:: :func:`numpy.split`
+
+    """
+    if ary.ndim <= axis:
+        raise IndexError('Axis exceeds ndim')
+    size = ary.shape[axis]
+
+    if numpy.isscalar(indices_or_sections):
+        if size % indices_or_sections != 0:
+            raise ValueError(
+                'indices_or_sections must divide the size along the axes.\n'
+                'If you want to split the array into non-equally-sized '
+                'arrays, use array_split instead.')
+    return array_split(ary, indices_or_sections, axis)
+
+
+def vsplit(ary, indices_or_sections):
+    """Splits an array into multiple sub arrays along the first axis.
+
+    This is equivalent to ``split`` with ``axis=0``.
+
+    .. seealso:: :func:`cupy.split` for more detail, :func:`numpy.dsplit`
+
+    """
+    if ary.ndim <= 1:
+        raise ValueError('Cannot vsplit an array with less than 2 dimensions')
+    return split(ary, indices_or_sections, 0)
diff --git a/cupy/_manipulation/tiling.py b/cupy/_manipulation/tiling.py
new file mode 100644
index 0000000..0d643c9
--- /dev/null
+++ b/cupy/_manipulation/tiling.py
@@ -0,0 +1,70 @@
+import cupy
+from cupy import _core
+
+
+def tile(A, reps):
+    """Construct an array by repeating A the number of times given by reps.
+
+    Args:
+        A (cupy.ndarray): Array to transform.
+        reps (int or tuple): The number of repeats.
+
+    Returns:
+        cupy.ndarray: Transformed array with repeats.
+
+    .. seealso:: :func:`numpy.tile`
+
+    """
+    try:
+        tup = tuple(reps)
+    except TypeError:
+        tup = (reps,)
+    d = len(tup)
+    if tup.count(1) == len(tup) and isinstance(A, cupy.ndarray):
+        # Fixes the problem that the function does not make a copy if A is a
+        # array and the repetitions are 1 in all dimensions
+        return cupy.array(A, copy=True, ndmin=d)
+    else:
+        # Note that no copy of zero-sized arrays is made. However since they
+        # have no data there is no risk of an inadvertent overwrite.
+        c = cupy.array(A, copy=False, ndmin=d)
+    if d < c.ndim:
+        tup = (1,) * (c.ndim - d) + tup
+    shape_out = tuple(s * t for s, t in zip(c.shape, tup))
+    if c.size == 0:
+        return cupy.empty(shape_out, dtype=c.dtype)
+    c_shape = []
+    ret_shape = []
+    for dim_in, nrep in zip(c.shape, tup):
+        if nrep == 1:
+            c_shape.append(dim_in)
+            ret_shape.append(dim_in)
+        elif dim_in == 1:
+            c_shape.append(dim_in)
+            ret_shape.append(nrep)
+        else:
+            c_shape.append(1)
+            c_shape.append(dim_in)
+            ret_shape.append(nrep)
+            ret_shape.append(dim_in)
+    ret = cupy.empty(ret_shape, dtype=c.dtype)
+    if ret.size:
+        _core.elementwise_copy(c.reshape(c_shape), ret)
+    return ret.reshape(shape_out)
+
+
+def repeat(a, repeats, axis=None):
+    """Repeat arrays along an axis.
+
+    Args:
+        a (cupy.ndarray): Array to transform.
+        repeats (int, list or tuple): The number of repeats.
+        axis (int): The axis to repeat.
+
+    Returns:
+        cupy.ndarray: Transformed array with repeats.
+
+    .. seealso:: :func:`numpy.repeat`
+
+    """
+    return a.repeat(repeats, axis)
diff --git a/cupy/_manipulation/transpose.py b/cupy/_manipulation/transpose.py
new file mode 100644
index 0000000..b2a4717
--- /dev/null
+++ b/cupy/_manipulation/transpose.py
@@ -0,0 +1,82 @@
+from cupy import _core
+from cupy._core import _routines_manipulation
+
+
+def rollaxis(a, axis, start=0):
+    """Moves the specified axis backwards to the given place.
+
+    Args:
+        a (cupy.ndarray): Array to move the axis.
+        axis (int): The axis to move.
+        start (int): The place to which the axis is moved.
+
+    Returns:
+        cupy.ndarray: A view of ``a`` that the axis is moved to ``start``.
+
+    .. seealso:: :func:`numpy.rollaxis`
+
+    """
+    return _core.rollaxis(a, axis, start)
+
+
+def swapaxes(a, axis1, axis2):
+    """Swaps the two axes.
+
+    Args:
+        a (cupy.ndarray): Array to swap the axes.
+        axis1 (int): The first axis to swap.
+        axis2 (int): The second axis to swap.
+
+    Returns:
+        cupy.ndarray: A view of ``a`` that the two axes are swapped.
+
+    .. seealso:: :func:`numpy.swapaxes`
+
+    """
+    # TODO(okuta): check type
+    return a.swapaxes(axis1, axis2)
+
+
+def moveaxis(a, source, destination):
+    """Moves axes of an array to new positions.
+
+    Other axes remain in their original order.
+
+    Args:
+        a (cupy.ndarray): Array whose axes should be reordered.
+        source (int or sequence of int):
+            Original positions of the axes to move. These must be unique.
+        destination (int or sequence of int):
+            Destination positions for each of the original axes. These must
+            also be unique.
+
+    Returns:
+        cupy.ndarray:
+        Array with moved axes. This array is a view of the input array.
+
+    .. seealso:: :func:`numpy.moveaxis`
+
+    """
+    # TODO(fukatani): check type
+    # checking __len__ attribute is faster than cupy.isscalar or isinstance
+    if not (hasattr(source, '__len__') or hasattr(destination, '__len__')):
+        return _routines_manipulation._move_single_axis(a, source, destination)
+    return _core.moveaxis(a, source, destination)
+
+
+def transpose(a, axes=None):
+    """Permutes the dimensions of an array.
+
+    Args:
+        a (cupy.ndarray): Array to permute the dimensions.
+        axes (tuple of ints): Permutation of the dimensions. This function
+            reverses the shape by default.
+
+    Returns:
+        cupy.ndarray: A view of ``a`` that the dimensions are permuted.
+
+    .. seealso:: :func:`numpy.transpose`
+
+    """
+    # TODO(okuta): check type
+    return a.transpose(axes)
diff --git a/cupy/_math/__init__.py b/cupy/_math/__init__.py
new file mode 100644
index 0000000..d14acd9
--- /dev/null
+++ b/cupy/_math/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.math.html
diff --git a/cupy/_math/arithmetic.py b/cupy/_math/arithmetic.py
new file mode 100644
index 0000000..5528466
--- /dev/null
+++ b/cupy/_math/arithmetic.py
@@ -0,0 +1,163 @@
+from cupy import _core
+from cupy._core import fusion
+
+
+add = _core.add
+
+
+reciprocal = _core.create_ufunc(
+    'cupy_reciprocal',
+    ('b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q',
+     ('e', 'out0 = 1 / in0'),
+     ('f', 'out0 = 1 / in0'),
+     ('d', 'out0 = 1 / in0'),
+     ('F', 'out0 = in0_type(1) / in0'),
+     ('D', 'out0 = in0_type(1) / in0')),
+    'out0 = in0 == 0 ? 0 : (1 / in0)',
+    doc='''Computes ``1 / x`` elementwise.
+
+    .. seealso:: :data:`numpy.reciprocal`
+
+    ''')
+
+
+positive = _core.positive
+
+
+negative = _core.negative
+
+
+conjugate = _core.conjugate
+
+
+# cupy.real is not a ufunc because it returns a view.
+# The ufunc implementation is used by fusion.
+_real_ufunc = _core.create_ufunc(
+    'cupy_real',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q', 'e->e', 'f->f', 'd->d',
+     ('F->f', 'out0 = in0.real()'),
+     ('D->d', 'out0 = in0.real()')),
+    'out0 = in0',
+    doc='''Returns the real part of the elements of the array.
+
+    .. seealso:: :func:`numpy.real`
+
+    ''')
+
+
+# cupy.imag is not a ufunc because it may return a view.
+# The ufunc implementation is used by fusion.
+_imag_ufunc = _core.create_ufunc(
+    'cupy_imag',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L',
+     'q->q', 'Q->Q', 'e->e', 'f->f', 'd->d',
+     ('F->f', 'out0 = in0.imag()'),
+     ('D->d', 'out0 = in0.imag()')),
+    'out0 = 0',
+    doc='''Returns the imaginary part of the elements of the array.
+
+    .. seealso:: :func:`numpy.imag`
+
+    ''')
+
+
+def angle(z, deg=False):
+    '''Returns the angle of the complex argument.
+
+    .. seealso:: :func:`numpy.angle`
+
+    '''
+    if deg:
+        return _core.angle_deg(z)
+    return _core.angle(z)
+
+
+def real(val):
+    '''Returns the real part of the elements of the array.
+
+    .. seealso:: :func:`numpy.real`
+
+    '''
+    if fusion._is_fusing():
+        return fusion._call_ufunc(_real_ufunc, val)
+    if not isinstance(val, _core.ndarray):
+        val = _core.array(val)
+    return val.real
+
+
+def imag(val):
+    '''Returns the imaginary part of the elements of the array.
+
+    .. seealso:: :func:`numpy.imag`
+
+    '''
+    if fusion._is_fusing():
+        return fusion._call_ufunc(_imag_ufunc, val)
+    if not isinstance(val, _core.ndarray):
+        val = _core.array(val)
+    return val.imag
+
+
+multiply = _core.multiply
+
+
+divide = _core.divide
+
+
+divmod = _core.divmod
+
+
+power = _core.power
+
+
+subtract = _core.subtract
+
+
+true_divide = _core.true_divide
+
+
+floor_divide = _core.floor_divide
+
+float_power = _core.create_ufunc(
+    'cupy_float_power',
+    ('dd->d', 'FF->D',
+     ('DD->D', 'out0 = in1 == in1_type(0) ? in1_type(1): pow(in0, in1)')),
+    'out0 = pow(in0, in1)',
+    doc='''First array elements raised to powers from second array, element-wise.
+
+    .. seealso:: :data:`numpy.float_power`
+
+    '''  # NOQA
+)
+
+fmod = _core.create_ufunc(
+    'cupy_fmod',
+    ('bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l', 'LL->L',
+     'qq->q', 'QQ->Q',
+     ('ee->e', 'out0 = fmodf(in0, in1)'),
+     ('ff->f', 'out0 = fmodf(in0, in1)'),
+     ('dd->d', 'out0 = fmod(in0, in1)')),
+    'out0 = in1 == 0 ? 0 : fmod((double)in0, (double)in1)',
+    doc='''Computes the remainder of C division elementwise.
+
+    .. seealso:: :data:`numpy.fmod`
+
+    ''')
+
+
+modf = _core.create_ufunc(
+    'cupy_modf',
+    ('e->ee', 'f->ff',
+     ('d->dd', 'double iptr; out0 = modf(in0, &iptr); out1 = iptr')),
+    'float iptr; out0 = modff(in0, &iptr); out1 = iptr',
+    doc='''Extracts the fractional and integral parts of an array elementwise.
+
+    This ufunc returns two arrays.
+
+    .. seealso:: :data:`numpy.modf`
+
+    ''')
+
+
+remainder = _core.remainder
diff --git a/cupy/_math/explog.py b/cupy/_math/explog.py
new file mode 100644
index 0000000..57342c7
--- /dev/null
+++ b/cupy/_math/explog.py
@@ -0,0 +1,103 @@
+from cupy import _core
+from cupy._math import ufunc
+
+
+exp = ufunc.create_math_ufunc(
+    'exp', 1, 'cupy_exp',
+    '''Elementwise exponential function.
+
+    .. seealso:: :data:`numpy.exp`
+
+    ''')
+
+
+expm1 = ufunc.create_math_ufunc(
+    'expm1', 1, 'cupy_expm1',
+    '''Computes ``exp(x) - 1`` elementwise.
+
+    .. seealso:: :data:`numpy.expm1`
+
+    ''')
+
+
+exp2 = _core.create_ufunc(
+    'cupy_exp2',
+    ('e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    'out0 = pow(in0_type(2), in0)',
+    doc='''Elementwise exponentiation with base 2.
+
+    .. seealso:: :data:`numpy.exp2`
+
+    ''')
+
+
+log = ufunc.create_math_ufunc(
+    'log', 1, 'cupy_log',
+    '''Elementwise natural logarithm function.
+
+    .. seealso:: :data:`numpy.log`
+
+    ''')
+
+
+log10 = ufunc.create_math_ufunc(
+    'log10', 1, 'cupy_log10',
+    '''Elementwise common logarithm function.
+
+    .. seealso:: :data:`numpy.log10`
+
+    ''')
+
+
+log2 = ufunc.create_math_ufunc(
+    'log2', 1, 'cupy_log2',
+    '''Elementwise binary logarithm function.
+
+    .. seealso:: :data:`numpy.log2`
+
+    ''')
+
+
+log1p = ufunc.create_math_ufunc(
+    'log1p', 1, 'cupy_log1p',
+    '''Computes ``log(1 + x)`` elementwise.
+
+    .. seealso:: :data:`numpy.log1p`
+
+    ''')
+
+
+logaddexp = _core.create_ufunc(
+    'cupy_logaddexp',
+    ('ee->e', 'ff->f', 'dd->d'),
+    '''
+    if (in0 == in1) {
+        /* Handles infinities of the same sign */
+        out0 = in0 + log(2.0);
+    } else {
+        out0 = fmax(in0, in1) + log1p(exp(-fabs(in0 - in1)));
+    }
+    ''',
+    doc='''Computes ``log(exp(x1) + exp(x2))`` elementwise.
+
+    .. seealso:: :data:`numpy.logaddexp`
+
+    ''')
+
+
+logaddexp2 = _core.create_ufunc(
+    'cupy_logaddexp2',
+    ('ee->e', 'ff->f', 'dd->d'),
+    '''
+    if (in0 == in1) {
+        /* Handles infinities of the same sign */
+        out0 = in0 + 1.0;
+    } else {
+        out0 = fmax(in0, in1) + log2(1 + exp2(-fabs(in0 - in1)));
+    }
+    ''',
+    doc='''Computes ``log2(exp2(x1) + exp2(x2))`` elementwise.
+
+    .. seealso:: :data:`numpy.logaddexp2`
+
+    ''')
diff --git a/cupy/_math/floating.py b/cupy/_math/floating.py
new file mode 100644
index 0000000..27f5174
--- /dev/null
+++ b/cupy/_math/floating.py
@@ -0,0 +1,62 @@
+from cupy import _core
+from cupy._math import ufunc
+from cupy_backends.cuda.api import runtime
+
+
+signbit = _core.create_ufunc(
+    'cupy_signbit',
+    ('e->?', 'f->?', 'd->?'),
+    'out0 = signbit(in0)',
+    doc='''Tests elementwise if the sign bit is set (i.e. less than zero).
+
+    .. seealso:: :data:`numpy.signbit`
+
+    ''')
+
+
+copysign = ufunc.create_math_ufunc(
+    'copysign', 2, 'cupy_copysign',
+    '''Returns the first argument with the sign bit of the second elementwise.
+
+    .. seealso:: :data:`numpy.copysign`
+
+    ''')
+
+
+ldexp = _core.create_ufunc(
+    'cupy_ldexp',
+    ('ei->e', 'fi->f', 'el->e', 'fl->f', 'di->d', 'dq->d'),
+    'out0 = ldexp(in0, in1)',
+    doc='''Computes ``x1 * 2 ** x2`` elementwise.
+
+    .. seealso:: :data:`numpy.ldexp`
+
+    ''')
+
+# HIP supports frexpf but not frexp ...
+frexp = _core.create_ufunc(
+    'cupy_frexp',
+    ('e->ei', 'f->fi', 'd->di'),
+    'int nptr; out0 = {}(in0, &nptr); out1 = nptr'.format(
+        'frexpf' if runtime.is_hip else 'frexp'),
+    doc='''Decomposes each element to mantissa and two's exponent.
+
+    This ufunc outputs two arrays of the input dtype and the ``int`` dtype.
+
+    .. seealso:: :data:`numpy.frexp`
+
+    ''')
+
+
+nextafter = ufunc.create_math_ufunc(
+    'nextafter', 2, 'cupy_nextafter',
+    '''Computes the nearest neighbor float values towards the second argument.
+
+    .. note::
+        For values that are close to zero (or denormal numbers),
+        results of :func:`cupy.nextafter` may be different from those of
+        :func:`numpy.nextafter`, because CuPy sets ``-ftz=true``.
+
+    .. seealso:: :data:`numpy.nextafter`
+
+    ''')
diff --git a/cupy/_math/hyperbolic.py b/cupy/_math/hyperbolic.py
new file mode 100644
index 0000000..2ea3228
--- /dev/null
+++ b/cupy/_math/hyperbolic.py
@@ -0,0 +1,55 @@
+from cupy._math import ufunc
+
+
+sinh = ufunc.create_math_ufunc(
+    'sinh', 1, 'cupy_sinh',
+    '''Elementwise hyperbolic sine function.
+
+    .. seealso:: :data:`numpy.sinh`
+
+    ''')
+
+
+cosh = ufunc.create_math_ufunc(
+    'cosh', 1, 'cupy_cosh',
+    '''Elementwise hyperbolic cosine function.
+
+    .. seealso:: :data:`numpy.cosh`
+
+    ''')
+
+
+tanh = ufunc.create_math_ufunc(
+    'tanh', 1, 'cupy_tanh',
+    '''Elementwise hyperbolic tangent function.
+
+    .. seealso:: :data:`numpy.tanh`
+
+    ''')
+
+
+arcsinh = ufunc.create_math_ufunc(
+    'asinh', 1, 'cupy_arcsinh',
+    '''Elementwise inverse of hyperbolic sine function.
+
+    .. seealso:: :data:`numpy.arcsinh`
+
+    ''')
+
+
+arccosh = ufunc.create_math_ufunc(
+    'acosh', 1, 'cupy_arccosh',
+    '''Elementwise inverse of hyperbolic cosine function.
+
+    .. seealso:: :data:`numpy.arccosh`
+
+    ''')
+
+
+arctanh = ufunc.create_math_ufunc(
+    'atanh', 1, 'cupy_arctanh',
+    '''Elementwise inverse of hyperbolic tangent function.
+
+    .. seealso:: :data:`numpy.arctanh`
+
+    ''')
diff --git a/cupy/_math/misc.py b/cupy/_math/misc.py
new file mode 100644
index 0000000..31f0492
--- /dev/null
+++ b/cupy/_math/misc.py
@@ -0,0 +1,569 @@
+import cupy
+import cupyx.scipy.fft
+
+from cupy import _core
+from cupy._core import _routines_math as _math
+from cupy._core import fusion
+from cupy.lib import stride_tricks
+
+import numpy
+
+
+_dot_kernel = _core.ReductionKernel(
+    'T x1, T x2',
+    'T y',
+    'x1 * x2',
+    'a + b',
+    'y = a',
+    '0',
+    'dot_product'
+)
+
+
+def _choose_conv_method(in1, in2, mode):
+    if in1.ndim != 1 or in2.ndim != 1:
+        raise NotImplementedError('Only 1d inputs are supported currently')
+
+    if in1.dtype.kind in 'bui' or in2.dtype.kind in 'bui':
+        return 'direct'
+
+    if _fftconv_faster(in1, in2, mode):
+        return 'fft'
+
+    return 'direct'
+
+
+def _fftconv_faster(x, h, mode):
+    """
+    .. seealso:: :func: `scipy.signal._signaltools._fftconv_faster`
+
+    """
+    # TODO(Dahlia-Chehata): replace with GPU-based constants.
+    return True
+
+
+def convolve(a, v, mode='full'):
+    """Returns the discrete, linear convolution of two one-dimensional sequences.
+
+    Args:
+        a (cupy.ndarray): first 1-dimensional input.
+        v (cupy.ndarray): second 1-dimensional input.
+        mode (str, optional): `valid`, `same`, `full`
+
+    Returns:
+        cupy.ndarray: Discrete, linear convolution of a and v.
+
+    .. seealso:: :func:`numpy.convolve`
+
+    """  # NOQA
+    if a.size == 0:
+        raise ValueError('a cannot be empty')
+    if v.size == 0:
+        raise ValueError('v cannot be empty')
+    if v.ndim > 1:
+        raise ValueError('v cannot be multidimensional array')
+    if v.size > a.size:
+        a, v = v, a
+    a = a.ravel()
+    v = v.ravel()
+
+    method = _choose_conv_method(a, v, mode)
+    if method == 'direct':
+        out = _dot_convolve(a, v, mode)
+    elif method == 'fft':
+        out = _fft_convolve(a, v, mode)
+    else:
+        raise ValueError('Unsupported method')
+    return out
+
+
+def _fft_convolve(a1, a2, mode):
+
+    offset = 0
+    if a1.shape[-1] < a2.shape[-1]:
+        a1, a2 = a2, a1
+        offset = 1 - a2.shape[-1] % 2
+
+    # if either of them is complex, the dtype after multiplication will also be
+    if a1.dtype.kind == 'c' or a2.dtype.kind == 'c':
+        fft, ifft = cupy.fft.fft, cupy.fft.ifft
+    else:
+        fft, ifft = cupy.fft.rfft, cupy.fft.irfft
+
+    dtype = cupy.result_type(a1, a2)
+    n1, n2 = a1.shape[-1], a2.shape[-1]
+    out_size = cupyx.scipy.fft.next_fast_len(n1 + n2 - 1)
+    fa1 = fft(a1, out_size)
+    fa2 = fft(a2, out_size)
+    out = ifft(fa1 * fa2, out_size)
+
+    if mode == 'full':
+        start, end = 0, n1 + n2 - 1
+    elif mode == 'same':
+        start = (n2 - 1) // 2 + offset
+        end = start + n1
+    elif mode == 'valid':
+        start, end = n2 - 1, n1
+    else:
+        raise ValueError(
+            'acceptable mode flags are `valid`, `same`, or `full`.')
+
+    out = out[..., start:end]
+
+    if dtype.kind in 'iu':
+        out = cupy.around(out)
+
+    return out.astype(dtype, copy=False)
+
+
+def _dot_convolve(a1, a2, mode):
+
+    offset = 0
+    if a1.size < a2.size:
+        a1, a2 = a2, a1
+        offset = 1 - a2.size % 2
+
+    dtype = cupy.result_type(a1, a2)
+    n1, n2 = a1.size, a2.size
+    a1 = a1.astype(dtype, copy=False)
+    a2 = a2.astype(dtype, copy=False)
+
+    if mode == 'full':
+        out_size = n1 + n2 - 1
+        a1 = cupy.pad(a1, n2 - 1)
+    elif mode == 'same':
+        out_size = n1
+        pad_size = (n2 - 1) // 2 + offset
+        a1 = cupy.pad(a1, (n2 - 1 - pad_size, pad_size))
+    elif mode == 'valid':
+        out_size = n1 - n2 + 1
+
+    stride = a1.strides[0]
+    a1 = stride_tricks.as_strided(a1, (out_size, n2), (stride, stride))
+    output = _dot_kernel(a1, a2[::-1], axis=1)
+    return output
+
+
+def clip(a, a_min, a_max, out=None):
+    """Clips the values of an array to a given interval.
+
+    This is equivalent to ``maximum(minimum(a, a_max), a_min)``, while this
+    function is more efficient.
+
+    Args:
+        a (cupy.ndarray): The source array.
+        a_min (scalar, cupy.ndarray or None): The left side of the interval.
+            When it is ``None``, it is ignored.
+        a_max (scalar, cupy.ndarray or None): The right side of the interval.
+            When it is ``None``, it is ignored.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: Clipped array.
+
+    .. seealso:: :func:`numpy.clip`
+
+    Notes
+    -----
+    When `a_min` is greater than `a_max`, `clip` returns an
+    array in which all values are equal to `a_max`.
+    """
+    if fusion._is_fusing():
+        return fusion._call_ufunc(_math.clip,
+                                  a, a_min, a_max, out=out)
+
+    # TODO(okuta): check type
+    return a.clip(a_min, a_max, out=out)
+
+
+# sqrt_fixed is deprecated.
+# numpy.sqrt is fixed in numpy 1.11.2.
+sqrt = sqrt_fixed = _core.sqrt
+
+
+cbrt = _core.create_ufunc(
+    'cupy_cbrt',
+    ('e->e', 'f->f', 'd->d'),
+    'out0 = cbrt(in0)',
+    doc='''Elementwise cube root function.
+
+    .. seealso:: :data:`numpy.cbrt`
+
+    ''')
+
+
+square = _core.create_ufunc(
+    'cupy_square',
+    ('b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I', 'l->l', 'L->L', 'q->q',
+     'Q->Q', 'e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    'out0 = in0 * in0',
+    doc='''Elementwise square function.
+
+    .. seealso:: :data:`numpy.square`
+
+    ''')
+
+
+absolute = _core.absolute
+
+
+fabs = _core.create_ufunc(
+    'cupy_fabs',
+    ('e->e', 'f->f', 'd->d'),
+    'out0 = abs(in0)',
+    doc='''Calculates absolute values element-wise.
+    Only real values are handled.
+
+    .. seealso:: :data:`numpy.fabs`
+
+    ''')
+
+
+_unsigned_sign = 'out0 = in0 > 0'
+_complex_sign = '''
+if (in0.real() == 0) {
+  out0 = (in0.imag() > 0) - (in0.imag() < 0);
+} else {
+  out0 = (in0.real() > 0) - (in0.real() < 0);
+}
+'''
+sign = _core.create_ufunc(
+    'cupy_sign',
+    ('b->b', ('B->B', _unsigned_sign), 'h->h', ('H->H', _unsigned_sign),
+     'i->i', ('I->I', _unsigned_sign), 'l->l', ('L->L', _unsigned_sign),
+     'q->q', ('Q->Q', _unsigned_sign), 'e->e', 'f->f', 'd->d',
+     ('F->F', _complex_sign), ('D->D', _complex_sign)),
+    'out0 = (in0 > 0) - (in0 < 0)',
+    doc='''Elementwise sign function.
+
+    It returns -1, 0, or 1 depending on the sign of the input.
+
+    .. seealso:: :data:`numpy.sign`
+
+    ''')
+
+
+heaviside = _core.create_ufunc(
+    'cupy_heaviside',
+    ('ee->e', 'ff->f', 'dd->d'),
+    '''
+    if (isnan(in0)) {
+        out0 = in0;
+    } else if (in0 == 0) {
+        out0 = in1;
+    } else {
+        out0 = (in0 > 0);
+    }
+    ''',
+    doc='''Compute the Heaviside step function.
+
+    .. seealso:: :data:`numpy.heaviside`
+
+    '''
+)
+
+
+_float_preamble = '''
+#ifndef NAN
+#define NAN __int_as_float(0x7fffffff)
+#endif
+'''
+_float_maximum = ('out0 = (isnan(in0) | isnan(in1)) ? out0_type(NAN) : '
+                  'out0_type(max(in0, in1))')
+maximum = _core.create_ufunc(
+    'cupy_maximum',
+    ('??->?', 'bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+     'LL->L', 'qq->q', 'QQ->Q',
+     ('ee->e', _float_maximum),
+     ('ff->f', _float_maximum),
+     ('dd->d', _float_maximum),
+     ('FF->F', _float_maximum),
+     ('DD->D', _float_maximum)),
+    'out0 = max(in0, in1)',
+    preamble=_float_preamble,
+    doc='''Takes the maximum of two arrays elementwise.
+
+    If NaN appears, it returns the NaN.
+
+    .. seealso:: :data:`numpy.maximum`
+
+    ''',
+    cutensor_op=('OP_MAX', 1, 1), scatter_op='max')
+
+
+_float_minimum = ('out0 = (isnan(in0) | isnan(in1)) ? out0_type(NAN) : '
+                  'out0_type(min(in0, in1))')
+minimum = _core.create_ufunc(
+    'cupy_minimum',
+    ('??->?', 'bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+     'LL->L', 'qq->q', 'QQ->Q',
+     ('ee->e', _float_minimum),
+     ('ff->f', _float_minimum),
+     ('dd->d', _float_minimum),
+     ('FF->F', _float_minimum),
+     ('DD->D', _float_minimum)),
+    'out0 = min(in0, in1)',
+    preamble=_float_preamble,
+    doc='''Takes the minimum of two arrays elementwise.
+
+    If NaN appears, it returns the NaN.
+
+    .. seealso:: :data:`numpy.minimum`
+
+    ''',
+    cutensor_op=('OP_MIN', 1, 1), scatter_op='min')
+
+
+fmax = _core.create_ufunc(
+    'cupy_fmax',
+    ('??->?', 'bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+     'LL->L', 'qq->q', 'QQ->Q',
+     ('ee->e', 'out0 = fmax(in0, in1)'),
+     ('ff->f', 'out0 = fmax(in0, in1)'),
+     ('dd->d', 'out0 = fmax(in0, in1)'),
+     'FF->F', 'DD->D'),
+    'out0 = max(in0, in1)',
+    doc='''Takes the maximum of two arrays elementwise.
+
+    If NaN appears, it returns the other operand.
+
+    .. seealso:: :data:`numpy.fmax`
+
+    ''')
+
+
+fmin = _core.create_ufunc(
+    'cupy_fmin',
+    ('??->?', 'bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+     'LL->L', 'qq->q', 'QQ->Q',
+     ('ee->e', 'out0 = fmin(in0, in1)'),
+     ('ff->f', 'out0 = fmin(in0, in1)'),
+     ('dd->d', 'out0 = fmin(in0, in1)'),
+     'FF->F', 'DD->D'),
+    'out0 = min(in0, in1)',
+    doc='''Takes the minimum of two arrays elementwise.
+
+    If NaN appears, it returns the other operand.
+
+    .. seealso:: :data:`numpy.fmin`
+
+    ''')
+
+
+_nan_to_num_preamble = '''
+template <class T>
+__device__ T nan_to_num(T x, T nan, T posinf, T neginf) {
+    if (isnan(x))
+        return nan;
+    if (isinf(x))
+        return x > 0 ? posinf : neginf;
+    return x;
+}
+
+template <class T>
+__device__ complex<T> nan_to_num(complex<T> x, T nan, T posinf, T neginf) {
+    T re = nan_to_num(x.real(), nan, posinf, neginf);
+    T im = nan_to_num(x.imag(), nan, posinf, neginf);
+    return complex<T>(re, im);
+}
+'''
+
+
+_nan_to_num = _core.create_ufunc(
+    'cupy_nan_to_num_',
+    ('????->?', 'bbbb->b', 'BBBB->B', 'hhhh->h', 'HHHH->H',
+     'iiii->i', 'IIII->I', 'llll->l', 'LLLL->L', 'qqqq->q', 'QQQQ->Q',
+     ('eeee->e',
+      'out0 = nan_to_num(in0, in1, in2, in3)'),
+     ('ffff->f',
+      'out0 = nan_to_num(in0, in1, in2, in3)'),
+     ('dddd->d',
+      'out0 = nan_to_num(in0, in1, in2, in3)'),
+     ('Ffff->F',
+      'out0 = nan_to_num(in0, in1, in2, in3)'),
+     ('Dddd->D',
+      'out0 = nan_to_num(in0, in1, in2, in3)')),
+    'out0 = in0',
+    preamble=_nan_to_num_preamble,
+    doc='''Elementwise nan_to_num function.
+
+    .. seealso:: :func:`numpy.nan_to_num`
+
+    ''')
+
+
+def _check_nan_inf(x, dtype, neg=None):
+    if dtype.char in 'FD':
+        dtype = cupy.dtype(dtype.char.lower())
+    if dtype.char not in 'efd':
+        x = 0
+    elif x is None and neg is not None:
+        x = cupy.finfo(dtype).min if neg else cupy.finfo(dtype).max
+    elif cupy.isnan(x):
+        x = cupy.nan
+    elif cupy.isinf(x):
+        x = cupy.inf * (-1)**(x < 0)
+    return cupy.asanyarray(x, dtype)
+
+
+def nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None):
+    """Replace NaN with zero and infinity with large finite numbers (default
+    behaviour) or with the numbers defined by the user using the `nan`,
+    `posinf` and/or `neginf` keywords.
+
+    .. seealso:: :func:`numpy.nan_to_num`
+
+    """
+    if not isinstance(x, cupy.ndarray):
+        out = cupy.full((), x)
+    else:
+        out = cupy.empty_like(x) if copy else x
+    dtype = out.dtype
+    nan = _check_nan_inf(nan, dtype)
+    posinf = _check_nan_inf(posinf, dtype, False)
+    neginf = _check_nan_inf(neginf, dtype, True)
+    return _nan_to_num(x, nan, posinf, neginf, out=out)
+
+
+def real_if_close(a, tol=100):
+    """If input is complex with all imaginary parts close to zero, return real
+    parts.
+    "Close to zero" is defined as `tol` * (machine epsilon of the type for
+    `a`).
+
+    .. warning::
+
+            This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.real_if_close`
+    """
+    if not issubclass(a.dtype.type, cupy.complexfloating):
+        return a
+    if tol > 1:
+        f = numpy.finfo(a.dtype.type)
+        tol = f.eps * tol
+    if cupy.all(cupy.absolute(a.imag) < tol):
+        a = a.real
+    return a
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_interp_kernel(is_complex):
+    in_params = 'raw V x, raw U idx, '
+    in_params += 'raw W fx, raw Y fy, U len, raw Y left, raw Y right'
+    out_params = 'Z y'  # output dtype follows NumPy's
+
+    if is_complex:
+        preamble = 'typedef double real_t;\n'
+    else:
+        preamble = 'typedef Z real_t;\n'
+    preamble += 'typedef Z value_t;\n'
+    preamble += cupy._sorting.search._preamble  # for _isnan
+
+    code = r'''
+        U x_idx = idx[i] - 1;
+
+        if ( _isnan<V>(x[i]) ) { y = x[i]; }
+        else if (x_idx < 0) { y = left[0]; }
+        else if (x[i] == fx[len - 1]) {
+            // searchsorted cannot handle both of the boundary points,
+            // so we must detect and correct ourselves...
+            y = fy[len - 1];
+        }
+        else if (x_idx >= len - 1) { y = right[0]; }
+        else {
+            const Z slope = (value_t)(fy[x_idx+1] - fy[x_idx]) / \
+                            ((real_t)fx[x_idx+1] - (real_t)fx[x_idx]);
+            Z out = slope * ((real_t)x[i] - (real_t)fx[x_idx]) \
+                    + (value_t)fy[x_idx];
+            if (_isnan<Z>(out)) {
+                out = slope * ((real_t)x[i] - (real_t)fx[x_idx+1]) \
+                      + (value_t)fy[x_idx+1];
+                if (_isnan<Z>(out) && (fy[x_idx] == fy[x_idx+1])) {
+                    out = fy[x_idx];
+                }
+            }
+            y = out;
+        }
+    '''
+    return cupy.ElementwiseKernel(
+        in_params, out_params, code, 'cupy_interp', preamble=preamble)
+
+
+def interp(x, xp, fp, left=None, right=None, period=None):
+    """ One-dimensional linear interpolation.
+
+    Args:
+        x (cupy.ndarray): a 1D array of points on which the interpolation
+            is performed.
+        xp (cupy.ndarray): a 1D array of points on which the function values
+            (``fp``) are known.
+        fp (cupy.ndarray): a 1D array containing the function values at the
+            the points ``xp``.
+        left (float or complex): value to return if ``x < xp[0]``. Default is
+            ``fp[0]``.
+        right (float or complex): value to return if ``x > xp[-1]``. Default is
+            ``fp[-1]``.
+        period (None or float): a period for the x-coordinates. Parameters
+            ``left`` and ``right`` are ignored if ``period`` is specified.
+            Default is ``None``.
+
+    Returns:
+        cupy.ndarray: The interpolated values, same shape as ``x``.
+
+    .. note::
+        This function may synchronize if ``left`` or ``right`` is not already
+        on the device.
+
+    .. seealso:: :func:`numpy.interp`
+
+    """
+
+    if xp.ndim != 1 or fp.ndim != 1:
+        raise ValueError('xp and fp must be 1D arrays')
+    if xp.size != fp.size:
+        raise ValueError('fp and xp are not of the same length')
+    if xp.size == 0:
+        raise ValueError('array of sample points is empty')
+    if not x.flags.c_contiguous:
+        raise NotImplementedError('Non-C-contiguous x is currently not '
+                                  'supported')
+    x_dtype = cupy.common_type(x, xp)
+    if not cupy.can_cast(x_dtype, cupy.float64):
+        raise TypeError('Cannot cast array data from'
+                        ' {} to {} according to the rule \'safe\''
+                        .format(x_dtype, cupy.float64))
+
+    if period is not None:
+        # The handling of "period" below is modified from NumPy's
+
+        if period == 0:
+            raise ValueError("period must be a non-zero value")
+        period = abs(period)
+        left = None
+        right = None
+
+        x = x.astype(cupy.float64)
+        xp = xp.astype(cupy.float64)
+
+        # normalizing periodic boundaries
+        x %= period
+        xp %= period
+        asort_xp = cupy.argsort(xp)
+        xp = xp[asort_xp]
+        fp = fp[asort_xp]
+        xp = cupy.concatenate((xp[-1:]-period, xp, xp[0:1]+period))
+        fp = cupy.concatenate((fp[-1:], fp, fp[0:1]))
+        assert xp.flags.c_contiguous
+        assert fp.flags.c_contiguous
+
+    # NumPy always returns float64 or complex128, so we upcast all values
+    # on the fly in the kernel
+    out_dtype = 'D' if fp.dtype.kind == 'c' else 'd'
+    output = cupy.empty(x.shape, dtype=out_dtype)
+    idx = cupy.searchsorted(xp, x, side='right')
+    left = fp[0] if left is None else cupy.array(left, fp.dtype)
+    right = fp[-1] if right is None else cupy.array(right, fp.dtype)
+    kern = _get_interp_kernel(out_dtype == 'D')
+    kern(x, idx, xp, fp, xp.size, left, right, output)
+    return output
diff --git a/cupy/_math/rational.py b/cupy/_math/rational.py
new file mode 100644
index 0000000..639d728
--- /dev/null
+++ b/cupy/_math/rational.py
@@ -0,0 +1,62 @@
+from cupy import _core
+
+
+def _negative_gcd_error():
+    raise TypeError("gcd cannot be computed with boolean arrays")
+
+
+def _negative_lcm_error():
+    raise TypeError("lcm cannot be computed with boolean arrays")
+
+
+_gcd_preamble = '''
+template <typename T> inline __device__ T gcd(T in0, T in1) {
+  T r;
+  while (in1 != 0) {
+    r = in0 % in1;
+    in0 = in1;
+    in1 = r;
+  }
+  if (in0 < 0)
+    return -in0;
+  return in0;
+}
+'''
+
+gcd = _core.create_ufunc(
+    'cupy_gcd',
+    (('??->?', _negative_gcd_error),
+     'bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+     'LL->L', 'qq->q', 'QQ->Q'),
+    'out0 = gcd(in0, in1)',
+    preamble=_gcd_preamble,
+    doc='''Computes gcd of ``x1`` and ``x2`` elementwise.
+
+    .. seealso:: :data:`numpy.gcd`
+
+    ''')
+
+_lcm_preamble = _gcd_preamble + '''
+template <typename T> inline __device__ T lcm(T in0, T in1) {
+  T r = gcd(in0, in1);
+  if (r == 0)
+    return 0;
+  r = in0 / r * in1;
+  if (r < 0)
+    return -r;
+  return r;
+}
+'''
+
+lcm = _core.create_ufunc(
+    'cupy_lcm',
+    (('??->?', _negative_lcm_error),
+     'bb->b', 'BB->B', 'hh->h', 'HH->H', 'ii->i', 'II->I', 'll->l',
+     'LL->L', 'qq->q', 'QQ->Q'),
+    'out0 = lcm(in0, in1)',
+    preamble=_lcm_preamble,
+    doc='''Computes lcm of ``x1`` and ``x2`` elementwise.
+
+    .. seealso:: :data:`numpy.lcm`
+
+    ''')
diff --git a/cupy/_math/rounding.py b/cupy/_math/rounding.py
new file mode 100644
index 0000000..beea1e9
--- /dev/null
+++ b/cupy/_math/rounding.py
@@ -0,0 +1,77 @@
+from cupy import _core
+from cupy._core import fusion
+from cupy._math import ufunc
+
+
+def around(a, decimals=0, out=None):
+    """Rounds to the given number of decimals.
+
+    Args:
+        a (cupy.ndarray): The source array.
+        decimals (int): Number of decimal places to round to (default: 0).
+            If decimals is negative, it specifies the number of positions to
+            the left of the decimal point.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: Rounded array.
+
+    .. seealso:: :func:`numpy.around`
+
+    """
+    if fusion._is_fusing():
+        return fusion._call_ufunc(
+            _core.core._round_ufunc, a, decimals, out=out)
+    a = _core.array(a, copy=False)
+    return a.round(decimals, out=out)
+
+
+def round_(a, decimals=0, out=None):
+    return around(a, decimals, out=out)
+
+
+rint = ufunc.create_math_ufunc(
+    'rint', 1, 'cupy_rint',
+    '''Rounds each element of an array to the nearest integer.
+
+    .. seealso:: :data:`numpy.rint`
+
+    ''')
+
+
+floor = ufunc.create_math_ufunc(
+    'floor', 1, 'cupy_floor',
+    '''Rounds each element of an array to its floor integer.
+
+    .. seealso:: :data:`numpy.floor`
+
+    ''', support_complex=False)
+
+
+ceil = ufunc.create_math_ufunc(
+    'ceil', 1, 'cupy_ceil',
+    '''Rounds each element of an array to its ceiling integer.
+
+    .. seealso:: :data:`numpy.ceil`
+
+    ''', support_complex=False)
+
+
+trunc = ufunc.create_math_ufunc(
+    'trunc', 1, 'cupy_trunc',
+    '''Rounds each element of an array towards zero.
+
+    .. seealso:: :data:`numpy.trunc`
+
+    ''', support_complex=False)
+
+
+fix = _core.create_ufunc(
+    'cupy_fix', ('e->e', 'f->f', 'd->d'),
+    'out0 = (in0 >= 0.0) ? floor(in0): ceil(in0)',
+    doc='''If given value x is positive, it return floor(x).
+    Else, it return ceil(x).
+
+    .. seealso:: :func:`numpy.fix`
+
+    ''')
diff --git a/cupy/_math/special.py b/cupy/_math/special.py
new file mode 100644
index 0000000..3b1bc45
--- /dev/null
+++ b/cupy/_math/special.py
@@ -0,0 +1,26 @@
+from cupy import _core
+from cupy._math import ufunc
+
+
+i0 = ufunc.create_math_ufunc(
+    'cyl_bessel_i0', 1, 'cupy_i0',
+    '''Modified Bessel function of the first kind, order 0.
+
+    .. seealso:: :func:`numpy.i0`
+
+    ''')
+
+
+sinc = _core.create_ufunc(
+    'cupy_sinc',
+    ('e->e', 'f->f', 'd->d',
+     ('F->F', 'in0_type pi_in0 = (in0_type) M_PI * in0;'
+              'out0 = abs(in0) > 1e-9 ? sin(pi_in0) / (pi_in0) : 1'),
+     ('D->D', 'in0_type pi_in0 = (in0_type) M_PI * in0;'
+              'out0 = abs(in0) > 1e-9 ? sin(pi_in0) / (pi_in0) : 1')),
+    'out0 = abs(in0) > 1e-9 ? sinpi(in0) / (M_PI * in0) : 1',
+    doc='''Elementwise sinc function.
+
+    .. seealso:: :func:`numpy.sinc`
+
+    ''')
diff --git a/cupy/_math/sumprod.py b/cupy/_math/sumprod.py
new file mode 100644
index 0000000..5f93e8b
--- /dev/null
+++ b/cupy/_math/sumprod.py
@@ -0,0 +1,626 @@
+import numpy
+
+import cupy
+from cupy._core import _routines_math as _math
+from cupy._core import _fusion_thread_local
+from cupy._core import internal
+
+
+def sum(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Returns the sum of an array along given axes.
+
+    Args:
+        a (cupy.ndarray): Array to take sum.
+        axis (int or sequence of ints): Axes along which the sum is taken.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the specified axes are remained as axes
+            of length one.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.sum`
+
+    """
+    if _fusion_thread_local.is_fusing():
+        if keepdims:
+            raise NotImplementedError(
+                'cupy.sum does not support `keepdims` in fusion yet.')
+        if dtype is None:
+            func = _math.sum_auto_dtype
+        else:
+            func = _math._sum_keep_dtype
+        return _fusion_thread_local.call_reduction(
+            func, a, axis=axis, dtype=dtype, out=out)
+
+    # TODO(okuta): check type
+    return a.sum(axis, dtype, out, keepdims)
+
+
+def prod(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Returns the product of an array along given axes.
+
+    Args:
+        a (cupy.ndarray): Array to take product.
+        axis (int or sequence of ints): Axes along which the product is taken.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the specified axes are remained as axes
+            of length one.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.prod`
+
+    """
+    if _fusion_thread_local.is_fusing():
+        if keepdims:
+            raise NotImplementedError(
+                'cupy.prod does not support `keepdims` in fusion yet.')
+        if dtype is None:
+            func = _math._prod_auto_dtype
+        else:
+            func = _math._prod_keep_dtype
+        return _fusion_thread_local.call_reduction(
+            func, a, axis=axis, dtype=dtype, out=out)
+
+    # TODO(okuta): check type
+    return a.prod(axis, dtype, out, keepdims)
+
+
+def nansum(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Returns the sum of an array along given axes treating Not a Numbers
+    (NaNs) as zero.
+
+    Args:
+        a (cupy.ndarray): Array to take sum.
+        axis (int or sequence of ints): Axes along which the sum is taken.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the specified axes are remained as axes
+            of length one.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.nansum`
+
+    """
+    if _fusion_thread_local.is_fusing():
+        if keepdims:
+            raise NotImplementedError(
+                'cupy.nansum does not support `keepdims` in fusion yet.')
+        if a.dtype.char in 'FD':
+            func = _math._nansum_complex_dtype
+        elif dtype is None:
+            func = _math._nansum_auto_dtype
+        else:
+            func = _math._nansum_keep_dtype
+        return _fusion_thread_local.call_reduction(
+            func, a, axis=axis, dtype=dtype, out=out)
+
+    # TODO(okuta): check type
+    return _math._nansum(a, axis, dtype, out, keepdims)
+
+
+def nanprod(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Returns the product of an array along given axes treating Not a Numbers
+    (NaNs) as zero.
+
+    Args:
+        a (cupy.ndarray): Array to take product.
+        axis (int or sequence of ints): Axes along which the product is taken.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the specified axes are remained as axes
+            of length one.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.nanprod`
+
+    """
+    if _fusion_thread_local.is_fusing():
+        if keepdims:
+            raise NotImplementedError(
+                'cupy.nanprod does not support `keepdims` in fusion yet.')
+        if dtype is None:
+            func = _math._nanprod_auto_dtype
+        else:
+            func = _math._nanprod_keep_dtype
+        return _fusion_thread_local.call_reduction(
+            func, a, axis=axis, dtype=dtype, out=out)
+
+    # TODO(okuta): check type
+    return _math._nanprod(a, axis, dtype, out, keepdims)
+
+
+def cumsum(a, axis=None, dtype=None, out=None):
+    """Returns the cumulative sum of an array along a given axis.
+
+    Args:
+        a (cupy.ndarray): Input array.
+        axis (int): Axis along which the cumulative sum is taken. If it is not
+            specified, the input is flattened.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.cumsum`
+
+    """
+    return _math.scan_core(a, axis, _math.scan_op.SCAN_SUM, dtype, out)
+
+
+def cumprod(a, axis=None, dtype=None, out=None):
+    """Returns the cumulative product of an array along a given axis.
+
+    Args:
+        a (cupy.ndarray): Input array.
+        axis (int): Axis along which the cumulative product is taken. If it is
+            not specified, the input is flattened.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.cumprod`
+
+    """
+    return _math.scan_core(a, axis, _math.scan_op.SCAN_PROD, dtype, out)
+
+
+def nancumsum(a, axis=None, dtype=None, out=None):
+    """Returns the cumulative sum of an array along a given axis treating Not a
+    Numbers (NaNs) as zero.
+
+    Args:
+        a (cupy.ndarray): Input array.
+        axis (int): Axis along which the cumulative sum is taken. If it is not
+            specified, the input is flattened.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.nancumsum`
+    """
+    a = _replace_nan(a, 0, out=out)
+    return cumsum(a, axis=axis, dtype=dtype, out=out)
+
+
+def nancumprod(a, axis=None, dtype=None, out=None):
+    """Returns the cumulative product of an array along a given axis treating
+    Not a Numbers (NaNs) as one.
+
+    Args:
+        a (cupy.ndarray): Input array.
+        axis (int): Axis along which the cumulative product is taken. If it is
+            not specified, the input is flattened.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.nancumprod`
+    """
+    a = _replace_nan(a, 1, out=out)
+    return cumprod(a, axis=axis, dtype=dtype, out=out)
+
+
+_replace_nan_kernel = cupy._core._kernel.ElementwiseKernel(
+    'T a, T val', 'T out', 'if (a == a) {out = a;} else {out = val;}',
+    'cupy_replace_nan')
+
+
+def _replace_nan(a, val, out=None):
+    if out is None or a.dtype != out.dtype:
+        out = cupy.empty_like(a)
+    _replace_nan_kernel(a, val, out)
+    return out
+
+
+def diff(a, n=1, axis=-1, prepend=None, append=None):
+    """Calculate the n-th discrete difference along the given axis.
+
+    Args:
+        a (cupy.ndarray): Input array.
+        n (int): The number of times values are differenced. If zero, the input
+            is returned as-is.
+        axis (int): The axis along which the difference is taken, default is
+            the last axis.
+        prepend (int, float, cupy.ndarray): Value to prepend to ``a``.
+        append (int, float, cupy.ndarray): Value to append to ``a``.
+
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.diff`
+    """
+
+    if n == 0:
+        return a
+    if n < 0:
+        raise ValueError(
+            "order must be non-negative but got " + repr(n))
+
+    a = cupy.asanyarray(a)
+    nd = a.ndim
+    axis = internal._normalize_axis_index(axis, nd)
+
+    combined = []
+
+    if prepend is not None:
+        prepend = cupy.asanyarray(prepend)
+        if prepend.ndim == 0:
+            shape = list(a.shape)
+            shape[axis] = 1
+            prepend = cupy.broadcast_to(prepend, tuple(shape))
+        combined.append(prepend)
+
+    combined.append(a)
+
+    if append is not None:
+        append = cupy.asanyarray(append)
+        if append.ndim == 0:
+            shape = list(a.shape)
+            shape[axis] = 1
+            append = cupy.broadcast_to(append, tuple(shape))
+        combined.append(append)
+
+    if len(combined) > 1:
+        a = cupy.concatenate(combined, axis)
+
+    slice1 = [slice(None)] * nd
+    slice2 = [slice(None)] * nd
+    slice1[axis] = slice(1, None)
+    slice2[axis] = slice(None, -1)
+    slice1 = tuple(slice1)
+    slice2 = tuple(slice2)
+
+    op = cupy.not_equal if a.dtype == numpy.bool_ else cupy.subtract
+    for _ in range(n):
+        a = op(a[slice1], a[slice2])
+
+    return a
+
+
+def gradient(f, *varargs, axis=None, edge_order=1):
+    """Return the gradient of an N-dimensional array.
+
+    The gradient is computed using second order accurate central differences
+    in the interior points and either first or second order accurate one-sides
+    (forward or backwards) differences at the boundaries.
+    The returned gradient hence has the same shape as the input array.
+
+    Args:
+        f (cupy.ndarray): An N-dimensional array containing samples of a scalar
+            function.
+        varargs (list of scalar or array, optional): Spacing between f values.
+            Default unitary spacing for all dimensions. Spacing can be
+            specified using:
+
+            1. single scalar to specify a sample distance for all dimensions.
+            2. N scalars to specify a constant sample distance for each
+               dimension. i.e. `dx`, `dy`, `dz`, ...
+            3. N arrays to specify the coordinates of the values along each
+               dimension of F. The length of the array must match the size of
+               the corresponding dimension
+            4. Any combination of N scalars/arrays with the meaning of 2. and
+               3.
+
+            If `axis` is given, the number of varargs must equal the number of
+            axes. Default: 1.
+        edge_order ({1, 2}, optional): The gradient is calculated using N-th
+            order accurate differences at the boundaries. Default: 1.
+        axis (None or int or tuple of ints, optional): The gradient is
+            calculated only along the given axis or axes. The default
+            (axis = None) is to calculate the gradient for all the axes of the
+            input array. axis may be negative, in which case it counts from the
+            last to the first axis.
+
+    Returns:
+        gradient (cupy.ndarray or list of cupy.ndarray): A set of ndarrays
+        (or a single ndarray if there is only one dimension) corresponding
+        to the derivatives of f with respect to each dimension. Each
+        derivative has the same shape as f.
+
+    .. seealso:: :func:`numpy.gradient`
+    """
+    f = cupy.asanyarray(f)
+    ndim = f.ndim  # number of dimensions
+    axes = internal._normalize_axis_indices(axis, ndim, sort_axes=False)
+
+    len_axes = len(axes)
+    n = len(varargs)
+    if n == 0:
+        # no spacing argument - use 1 in all axes
+        dx = [1.0] * len_axes
+    elif n == 1 and cupy.ndim(varargs[0]) == 0:
+        # single scalar for all axes
+        dx = varargs * len_axes
+    elif n == len_axes:
+        # scalar or 1d array for each axis
+        dx = list(varargs)
+        for i, distances in enumerate(dx):
+            if cupy.ndim(distances) == 0:
+                continue
+            elif cupy.ndim(distances) != 1:
+                raise ValueError("distances must be either scalars or 1d")
+            if len(distances) != f.shape[axes[i]]:
+                raise ValueError(
+                    "when 1d, distances must match "
+                    "the length of the corresponding dimension"
+                )
+            if numpy.issubdtype(distances.dtype, numpy.integer):
+                # Convert numpy integer types to float64 to avoid modular
+                # arithmetic in np.diff(distances).
+                distances = distances.astype(numpy.float64)
+            diffx = cupy.diff(distances)
+            # if distances are constant reduce to the scalar case
+            # since it brings a consistent speedup
+            if (diffx == diffx[0]).all():  # synchronize
+                diffx = diffx[0]
+            dx[i] = diffx
+    else:
+        raise TypeError("invalid number of arguments")
+
+    if edge_order > 2:
+        raise ValueError("'edge_order' greater than 2 not supported")
+
+    # use central differences on interior and one-sided differences on the
+    # endpoints. This preserves second order-accuracy over the full domain.
+
+    outvals = []
+
+    # create slice objects --- initially all are [:, :, ..., :]
+    slice1 = [slice(None)] * ndim
+    slice2 = [slice(None)] * ndim
+    slice3 = [slice(None)] * ndim
+    slice4 = [slice(None)] * ndim
+
+    otype = f.dtype
+    if numpy.issubdtype(otype, numpy.inexact):
+        pass
+    else:
+        # All other types convert to floating point.
+        # First check if f is a numpy integer type; if so, convert f to float64
+        # to avoid modular arithmetic when computing the changes in f.
+        if numpy.issubdtype(otype, numpy.integer):
+            f = f.astype(numpy.float64)
+        otype = numpy.float64
+
+    for axis, ax_dx in zip(axes, dx):
+        if f.shape[axis] < edge_order + 1:
+            raise ValueError(
+                "Shape of array too small to calculate a numerical gradient, "
+                "at least (edge_order + 1) elements are required."
+            )
+        # result allocation
+        out = cupy.empty_like(f, dtype=otype)
+
+        # spacing for the current axis
+        uniform_spacing = cupy.ndim(ax_dx) == 0
+
+        # Numerical differentiation: 2nd order interior
+        slice1[axis] = slice(1, -1)
+        slice2[axis] = slice(None, -2)
+        slice3[axis] = slice(1, -1)
+        slice4[axis] = slice(2, None)
+
+        if uniform_spacing:
+            out[tuple(slice1)] = (f[tuple(slice4)] - f[tuple(slice2)]) / (
+                2.0 * ax_dx
+            )
+        else:
+            dx1 = ax_dx[0:-1]
+            dx2 = ax_dx[1:]
+            dx_sum = dx1 + dx2
+            a = -(dx2) / (dx1 * dx_sum)
+            b = (dx2 - dx1) / (dx1 * dx2)
+            c = dx1 / (dx2 * dx_sum)
+            # fix the shape for broadcasting
+            shape = [1] * ndim
+            shape[axis] = -1
+            a.shape = b.shape = c.shape = tuple(shape)
+            # 1D equivalent -- out[1:-1] = a * f[:-2] + b * f[1:-1] + c * f[2:]
+            out[tuple(slice1)] = (a * f[tuple(slice2)] +
+                                  b * f[tuple(slice3)] +
+                                  c * f[tuple(slice4)])
+
+        # Numerical differentiation: 1st order edges
+        if edge_order == 1:
+            slice1[axis] = 0
+            slice2[axis] = 1
+            slice3[axis] = 0
+            dx_0 = ax_dx if uniform_spacing else ax_dx[0]
+            # 1D equivalent -- out[0] = (f[1] - f[0]) / (x[1] - x[0])
+            out[tuple(slice1)] = (f[tuple(slice2)] - f[tuple(slice3)]) / dx_0
+
+            slice1[axis] = -1
+            slice2[axis] = -1
+            slice3[axis] = -2
+            dx_n = ax_dx if uniform_spacing else ax_dx[-1]
+            # 1D equivalent -- out[-1] = (f[-1] - f[-2]) / (x[-1] - x[-2])
+            out[tuple(slice1)] = (f[tuple(slice2)] - f[tuple(slice3)]) / dx_n
+
+        # Numerical differentiation: 2nd order edges
+        else:
+            slice1[axis] = 0
+            slice2[axis] = 0
+            slice3[axis] = 1
+            slice4[axis] = 2
+            if uniform_spacing:
+                a = -1.5 / ax_dx
+                b = 2.0 / ax_dx
+                c = -0.5 / ax_dx
+            else:
+                dx1 = ax_dx[0]
+                dx2 = ax_dx[1]
+                dx_sum = dx1 + dx2
+                a = -(2.0 * dx1 + dx2) / (dx1 * (dx_sum))
+                b = dx_sum / (dx1 * dx2)
+                c = -dx1 / (dx2 * (dx_sum))
+            # 1D equivalent -- out[0] = a * f[0] + b * f[1] + c * f[2]
+            out[tuple(slice1)] = (a * f[tuple(slice2)] +
+                                  b * f[tuple(slice3)] +
+                                  c * f[tuple(slice4)])
+
+            slice1[axis] = -1
+            slice2[axis] = -3
+            slice3[axis] = -2
+            slice4[axis] = -1
+            if uniform_spacing:
+                a = 0.5 / ax_dx
+                b = -2.0 / ax_dx
+                c = 1.5 / ax_dx
+            else:
+                dx1 = ax_dx[-2]
+                dx2 = ax_dx[-1]
+                dx_sum = dx1 + dx2
+                a = (dx2) / (dx1 * (dx_sum))
+                b = -dx_sum / (dx1 * dx2)
+                c = (2.0 * dx2 + dx1) / (dx2 * (dx_sum))
+            # 1D equivalent -- out[-1] = a * f[-3] + b * f[-2] + c * f[-1]
+            out[tuple(slice1)] = (a * f[tuple(slice2)] +
+                                  b * f[tuple(slice3)] +
+                                  c * f[tuple(slice4)])
+        outvals.append(out)
+
+        # reset the slice object in this dimension to ":"
+        slice1[axis] = slice(None)
+        slice2[axis] = slice(None)
+        slice3[axis] = slice(None)
+        slice4[axis] = slice(None)
+
+    if len_axes == 1:
+        return outvals[0]
+    else:
+        return outvals
+
+
+def ediff1d(arr, to_end=None, to_begin=None):
+    """
+    Calculates the difference between consecutive elements of an array.
+
+    Args:
+        arr (cupy.ndarray): Input array.
+        to_end (cupy.ndarray, optional): Numbers to append at the end
+            of the returend differences.
+        to_begin (cupy.ndarray, optional): Numbers to prepend at the
+            beginning of the returned differences.
+
+    Returns:
+        cupy.ndarray: New array consisting differences among succeeding
+        elements.
+
+    .. seealso:: :func:`numpy.ediff1d`
+    """
+    if not isinstance(arr, cupy.ndarray):
+        raise TypeError('`arr` should be of type cupy.ndarray')
+
+    # to flattened array.
+    arr = arr.ravel()
+
+    # to ensure the dtype of the output array is same as that of input.
+    dtype_req = arr.dtype
+
+    # if none optional cases are given
+    if to_begin is None and to_end is None:
+        return arr[1:] - arr[:-1]
+
+    if to_begin is None:
+        l_begin = 0
+    else:
+        if not isinstance(to_begin, cupy.ndarray):
+            raise TypeError('`to_begin` should be of type cupy.ndarray')
+        if not cupy.can_cast(to_begin, dtype_req, casting="same_kind"):
+            raise TypeError("dtype of `to_begin` must be compatible "
+                            "with input `arr` under the `same_kind` rule.")
+
+        to_begin = to_begin.ravel()
+        l_begin = len(to_begin)
+
+    if to_end is None:
+        l_end = 0
+    else:
+        if not isinstance(to_end, cupy.ndarray):
+            raise TypeError('`to_end` should be of type cupy.ndarray')
+        if not cupy.can_cast(to_end, dtype_req, casting="same_kind"):
+            raise TypeError("dtype of `to_end` must be compatible "
+                            "with input `arr` under the `same_kind` rule.")
+
+        to_end = to_end.ravel()
+        l_end = len(to_end)
+
+    # calulating using in place operation
+    l_diff = max(len(arr) - 1, 0)
+    result = cupy.empty(l_diff + l_begin + l_end, dtype=arr.dtype)
+    # Cupy does not support subclassing a ndarray
+    # result = arr.__array_wrap__(result)
+    if l_begin > 0:
+        result[:l_begin] = to_begin
+    if l_end > 0:
+        result[l_begin + l_diff:] = to_end
+    cupy.subtract(arr[1:], arr[:-1], result[l_begin:l_begin + l_diff])
+    return result
+
+
+# TODO(okuta): Implement cross
+
+
+def trapz(y, x=None, dx=1.0, axis=-1):
+    """
+    Integrate along the given axis using the composite trapezoidal rule.
+    Integrate `y` (`x`) along the given axis.
+
+    Args:
+        y (cupy.ndarray): Input array to integrate.
+        x (cupy.ndarray): Sample points over which to integrate. If None equal
+            spacing `dx` is assumed.
+        dx (float): Spacing between sample points, used if `x` is None, default
+            is 1.
+        axis (int): The axis along which the integral is taken, default is
+            the last axis.
+
+    Returns:
+        cupy.ndarray: Definite integral as approximated by the trapezoidal
+        rule.
+
+    .. seealso:: :func:`numpy.trapz`
+    """
+    if not isinstance(y, cupy.ndarray):
+        raise TypeError('`y` should be of type cupy.ndarray')
+
+    if x is None:
+        d = dx
+    else:
+        if not isinstance(x, cupy.ndarray):
+            raise TypeError('`x` should be of type cupy.ndarray')
+        if x.ndim == 1:
+            d = diff(x)
+            # reshape to correct shape
+            shape = [1] * y.ndim
+            shape[axis] = d.shape[0]
+            d = d.reshape(shape)
+        else:
+            d = diff(x, axis=axis)
+    nd = y.ndim
+    slice1 = [slice(None)] * nd
+    slice2 = [slice(None)] * nd
+    slice1[axis] = slice(1, None)
+    slice2[axis] = slice(None, -1)
+    product = d * (y[tuple(slice1)] + y[tuple(slice2)]) / 2.0
+    try:
+        ret = product.sum(axis)
+    except ValueError:
+        ret = cupy.add.reduce(product, axis)
+    return ret
diff --git a/cupy/_math/trigonometric.py b/cupy/_math/trigonometric.py
new file mode 100644
index 0000000..4b4eb76
--- /dev/null
+++ b/cupy/_math/trigonometric.py
@@ -0,0 +1,163 @@
+import numpy
+
+import cupy
+from cupy import _core
+from cupy._math import sumprod
+from cupy._math import ufunc
+
+
+sin = ufunc.create_math_ufunc(
+    'sin', 1, 'cupy_sin',
+    '''Elementwise sine function.
+
+    .. seealso:: :data:`numpy.sin`
+
+    ''')
+
+
+cos = ufunc.create_math_ufunc(
+    'cos', 1, 'cupy_cos',
+    '''Elementwise cosine function.
+
+    .. seealso:: :data:`numpy.cos`
+
+    ''')
+
+
+tan = ufunc.create_math_ufunc(
+    'tan', 1, 'cupy_tan',
+    '''Elementwise tangent function.
+
+    .. seealso:: :data:`numpy.tan`
+
+    ''')
+
+
+arcsin = ufunc.create_math_ufunc(
+    'asin', 1, 'cupy_arcsin',
+    '''Elementwise inverse-sine function (a.k.a. arcsine function).
+
+    .. seealso:: :data:`numpy.arcsin`
+
+    ''')
+
+
+arccos = ufunc.create_math_ufunc(
+    'acos', 1, 'cupy_arccos',
+    '''Elementwise inverse-cosine function (a.k.a. arccosine function).
+
+    .. seealso:: :data:`numpy.arccos`
+
+    ''')
+
+
+arctan = ufunc.create_math_ufunc(
+    'atan', 1, 'cupy_arctan',
+    '''Elementwise inverse-tangent function (a.k.a. arctangent function).
+
+    .. seealso:: :data:`numpy.arctan`
+
+    ''')
+
+
+hypot = ufunc.create_math_ufunc(
+    'hypot', 2, 'cupy_hypot',
+    '''Computes the hypoteneous of orthogonal vectors of given length.
+
+    This is equivalent to ``sqrt(x1 **2 + x2 ** 2)``, while this function is
+    more efficient.
+
+    .. seealso:: :data:`numpy.hypot`
+
+    ''')
+
+
+arctan2 = ufunc.create_math_ufunc(
+    'atan2', 2, 'cupy_arctan2',
+    '''Elementwise inverse-tangent of the ratio of two arrays.
+
+    .. seealso:: :data:`numpy.arctan2`
+
+    ''')
+
+
+deg2rad = _core.create_ufunc(
+    'cupy_deg2rad',
+    ('e->e', 'f->f', 'd->d'),
+    'out0 = in0 * (out0_type)(M_PI / 180)',
+    doc='''Converts angles from degrees to radians elementwise.
+
+    .. seealso:: :data:`numpy.deg2rad`, :data:`numpy.radians`
+
+    ''')
+
+
+rad2deg = _core.create_ufunc(
+    'cupy_rad2deg',
+    ('e->e', 'f->f', 'd->d'),
+    'out0 = in0 * (out0_type)(180 / M_PI)',
+    doc='''Converts angles from radians to degrees elementwise.
+
+    .. seealso:: :data:`numpy.rad2deg`, :data:`numpy.degrees`
+
+    ''')
+
+
+def unwrap(p, discont=None, axis=-1, *, period=2*numpy.pi):
+    r"""Unwrap by taking the complement of large deltas w.r.t. the period.
+
+    This unwraps a signal `p` by changing elements which have an absolute
+    difference from their predecessor of more than ``max(discont, period/2)``
+    to their `period`-complementary values.
+
+    For the default case where `period` is :math:`2\pi` and is ``discont``
+    is :math:`\pi`, this unwraps a radian phase `p` such that adjacent
+    differences are never greater than :math:`\pi` by adding :math:`2k\pi`
+    for some integer :math:`k`.
+
+    Args:
+        p (cupy.ndarray): Input array.
+            discont (float): Maximum discontinuity between values, default is
+            ``period/2``. Values below ``period/2`` are treated as if they were
+            ``period/2``. To have an effect different from the default,
+            ``discont`` should be larger than ``period/2``.
+        axis (int): Axis along which unwrap will operate, default is the last
+            axis.
+        period: float, optional
+            Size of the range over which the input wraps. By default, it is
+            :math:`2\pi`.
+    Returns:
+        cupy.ndarray: The result array.
+
+    .. seealso:: :func:`numpy.unwrap`
+    """
+
+    p = cupy.asarray(p)
+    nd = p.ndim
+    dd = sumprod.diff(p, axis=axis)
+    if discont is None:
+        discont = period/2
+    slice1 = [slice(None, None)]*nd     # full slices
+    slice1[axis] = slice(1, None)
+    slice1 = tuple(slice1)
+    dtype = numpy.result_type(dd.dtype, period)
+    if numpy.issubdtype(dtype, numpy.integer):
+        interval_high, rem = divmod(period, 2)
+        boundary_ambiguous = rem == 0
+    else:
+        interval_high = period / 2
+        boundary_ambiguous = True
+    interval_low = -interval_high
+    ddmod = cupy.mod(dd - interval_low, period) + interval_low
+    if boundary_ambiguous:
+        cupy.copyto(ddmod, interval_high, where=(
+            ddmod == interval_low) & (dd > 0))
+    ph_correct = ddmod - dd
+    cupy.copyto(ph_correct, 0, where=abs(dd) < discont)
+    up = cupy.array(p, copy=True, dtype=dtype)
+    up[slice1] = p[slice1] + cupy.cumsum(ph_correct, axis=axis)
+    return up
+
+
+degrees = rad2deg
+radians = deg2rad
diff --git a/cupy/_math/ufunc.py b/cupy/_math/ufunc.py
new file mode 100644
index 0000000..0d54645
--- /dev/null
+++ b/cupy/_math/ufunc.py
@@ -0,0 +1,17 @@
+from cupy import _core
+
+
+def create_math_ufunc(math_name, nargs, name, doc, support_complex=True):
+    assert 1 <= nargs <= 2
+    if nargs == 1:
+        types = ('e->e', 'f->f', 'd->d')
+        if support_complex:
+            types += ('F->F', 'D->D')
+        return _core.create_ufunc(
+            name, types, 'out0 = %s(in0)' % math_name, doc=doc)
+    else:
+        types = ('ee->e', 'ff->f', 'dd->d')
+        if support_complex:
+            types += ('FF->F', 'DD->D')
+        return _core.create_ufunc(
+            name, types, 'out0 = %s(in0, in1)' % math_name, doc=doc)
diff --git a/cupy/_math/window.py b/cupy/_math/window.py
new file mode 100644
index 0000000..3b41fef
--- /dev/null
+++ b/cupy/_math/window.py
@@ -0,0 +1,197 @@
+import numpy
+
+import cupy
+from cupy import _core
+
+_blackman_kernel = _core.ElementwiseKernel(
+    "float32 alpha",
+    "float64 out",
+    """
+    out = 0.42 - 0.5 * cos(i * alpha) + 0.08 * cos(2 * alpha * i);
+    """, name="cupy_blackman")
+
+
+_bartlett_kernel = _core.ElementwiseKernel(
+    "float32 alpha",
+    "T arr",
+    """
+    if (i < alpha)
+        arr = i / alpha;
+    else
+        arr = 2.0 - i / alpha;
+    """, name="cupy_bartlett")
+
+
+def bartlett(M):
+    """Returns the Bartlett window.
+
+    The Bartlett window is defined as
+
+    .. math::
+            w(n) = \\frac{2}{M-1} \\left(
+            \\frac{M-1}{2} - \\left|n - \\frac{M-1}{2}\\right|
+            \\right)
+
+    Args:
+        M (int):
+            Number of points in the output window. If zero or less, an empty
+            array is returned.
+
+    Returns:
+        ~cupy.ndarray: Output ndarray.
+
+    .. seealso:: :func:`numpy.bartlett`
+    """
+    if M == 1:
+        return cupy.ones(1, dtype=cupy.float64)
+    if M <= 0:
+        return cupy.array([])
+    alpha = (M - 1) / 2.0
+    out = cupy.empty(M, dtype=cupy.float64)
+    return _bartlett_kernel(alpha, out)
+
+
+def blackman(M):
+    """Returns the Blackman window.
+
+    The Blackman window is defined as
+
+    .. math::
+        w(n) = 0.42 - 0.5 \\cos\\left(\\frac{2\\pi{n}}{M-1}\\right)
+        + 0.08 \\cos\\left(\\frac{4\\pi{n}}{M-1}\\right)
+        \\qquad 0 \\leq n \\leq M-1
+
+    Args:
+        M (:class:`~int`):
+            Number of points in the output window. If zero or less, an empty
+            array is returned.
+
+    Returns:
+        ~cupy.ndarray: Output ndarray.
+
+    .. seealso:: :func:`numpy.blackman`
+    """
+    if M == 1:
+        return cupy.ones(1, dtype=cupy.float64)
+    if M <= 0:
+        return cupy.array([])
+    alpha = numpy.pi * 2 / (M - 1)
+    out = cupy.empty(M, dtype=cupy.float64)
+    return _blackman_kernel(alpha, out)
+
+
+_hamming_kernel = _core.ElementwiseKernel(
+    "float32 alpha",
+    "float64 out",
+    """
+    out = 0.54 - 0.46 * cos(i * alpha);
+    """, name="cupy_hamming")
+
+
+def hamming(M):
+    """Returns the Hamming window.
+
+    The Hamming window is defined as
+
+    .. math::
+        w(n) = 0.54 - 0.46\\cos\\left(\\frac{2\\pi{n}}{M-1}\\right)
+        \\qquad 0 \\leq n \\leq M-1
+
+    Args:
+        M (:class:`~int`):
+            Number of points in the output window. If zero or less, an empty
+            array is returned.
+
+    Returns:
+        ~cupy.ndarray: Output ndarray.
+
+    .. seealso:: :func:`numpy.hamming`
+    """
+    if M == 1:
+        return cupy.ones(1, dtype=cupy.float64)
+    if M <= 0:
+        return cupy.array([])
+    alpha = numpy.pi * 2 / (M - 1)
+    out = cupy.empty(M, dtype=cupy.float64)
+    return _hamming_kernel(alpha, out)
+
+
+_hanning_kernel = _core.ElementwiseKernel(
+    "float32 alpha",
+    "float64 out",
+    """
+    out = 0.5 - 0.5 * cos(i * alpha);
+    """, name="cupy_hanning")
+
+
+def hanning(M):
+    """Returns the Hanning window.
+
+    The Hanning window is defined as
+
+    .. math::
+        w(n) = 0.5 - 0.5\\cos\\left(\\frac{2\\pi{n}}{M-1}\\right)
+        \\qquad 0 \\leq n \\leq M-1
+
+    Args:
+        M (:class:`~int`):
+            Number of points in the output window. If zero or less, an empty
+            array is returned.
+
+    Returns:
+        ~cupy.ndarray: Output ndarray.
+
+    .. seealso:: :func:`numpy.hanning`
+    """
+    if M == 1:
+        return cupy.ones(1, dtype=cupy.float64)
+    if M <= 0:
+        return cupy.array([])
+    alpha = numpy.pi * 2 / (M - 1)
+    out = cupy.empty(M, dtype=cupy.float64)
+    return _hanning_kernel(alpha, out)
+
+
+_kaiser_kernel = _core.ElementwiseKernel(
+    "float32 beta, float32 alpha",
+    "T arr",
+    """
+    float temp = (i - alpha) / alpha;
+    arr = cyl_bessel_i0(beta * sqrt(1 - (temp * temp)));
+    arr /= cyl_bessel_i0(beta);
+    """, name="cupy_kaiser")
+
+
+def kaiser(M, beta):
+    """Return the Kaiser window.
+    The Kaiser window is a taper formed by using a Bessel function.
+
+    .. math::  w(n) = I_0\\left( \\beta \\sqrt{1-\\frac{4n^2}{(M-1)^2}}
+               \\right)/I_0(\\beta)
+
+    with
+
+    .. math:: \\quad -\\frac{M-1}{2} \\leq n \\leq \\frac{M-1}{2}
+
+    where :math:`I_0` is the modified zeroth-order Bessel function.
+
+     Args:
+        M (int):
+            Number of points in the output window. If zero or less, an empty
+            array is returned.
+        beta (float):
+            Shape parameter for window
+
+    Returns:
+        ~cupy.ndarray:  The window, with the maximum value normalized to one
+        (the value one appears only if the number of samples is odd).
+
+    .. seealso:: :func:`numpy.kaiser`
+    """
+    if M == 1:
+        return cupy.array([1.])
+    if M <= 0:
+        return cupy.array([])
+    alpha = (M - 1) / 2.0
+    out = cupy.empty(M, dtype=cupy.float64)
+    return _kaiser_kernel(beta, alpha, out)
diff --git a/cupy/_misc/__init__.py b/cupy/_misc/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cupy/_misc/byte_bounds.py b/cupy/_misc/byte_bounds.py
new file mode 100644
index 0000000..02c3de6
--- /dev/null
+++ b/cupy/_misc/byte_bounds.py
@@ -0,0 +1,23 @@
+def byte_bounds(a):
+    """Returns pointers to the end-points of an array.
+
+    Args:
+        a: ndarray
+    Returns:
+        Tuple[int, int]: pointers to the end-points of an array
+
+    .. seealso:: :func:`numpy.byte_bounds`
+    """
+    a_low = a_high = a.data.ptr
+    a_strides = a.strides
+    a_shape = a.shape
+    a_item_bytes = a.itemsize
+
+    for shape, stride in zip(a_shape, a_strides):
+        if stride < 0:
+            a_low += (shape - 1) * stride
+        else:
+            a_high += (shape - 1) * stride
+
+    a_high += a_item_bytes
+    return a_low, a_high
diff --git a/cupy/_misc/memory_ranges.py b/cupy/_misc/memory_ranges.py
new file mode 100644
index 0000000..94821aa
--- /dev/null
+++ b/cupy/_misc/memory_ranges.py
@@ -0,0 +1,44 @@
+from cupy._core import _kernel
+from cupy._core import _memory_range
+from cupy._manipulation import join
+from cupy._sorting import search
+
+
+def may_share_memory(a, b, max_work=None):
+    if max_work is None:
+        return _memory_range.may_share_bounds(a, b)
+
+    raise NotImplementedError('Only supported for `max_work` is `None`')
+
+
+_get_memory_ptrs_kernel = _kernel.ElementwiseKernel(
+    'T x', 'uint64 out',
+    'out = (unsigned long long)(&x)',
+    'cupy_get_memory_ptrs'
+)
+
+
+def _get_memory_ptrs(x):
+    if x.dtype.kind != 'c':
+        return _get_memory_ptrs_kernel(x)
+    return join.concatenate([
+        _get_memory_ptrs_kernel(x.real),
+        _get_memory_ptrs_kernel(x.imag)
+    ])
+
+
+def shares_memory(a, b, max_work=None):
+    if a is b and a.size != 0:
+        return True
+    if max_work == 'MAY_SHARE_BOUNDS':
+        return _memory_range.may_share_bounds(a, b)
+
+    if max_work in (None, 'MAY_SHARE_EXACT'):
+        a_ptrs = _get_memory_ptrs(a).ravel()
+        b_ptrs = _get_memory_ptrs(b).reshape(-1, 1)
+        a_ptrs.sort()
+        x = search.searchsorted(a_ptrs, b_ptrs, 'left')
+        y = search.searchsorted(a_ptrs, b_ptrs, 'right')
+        return bool((x != y).any())
+
+    raise NotImplementedError('Not supported for integer `max_work`.')
diff --git a/cupy/_misc/who.py b/cupy/_misc/who.py
new file mode 100644
index 0000000..a10ae17
--- /dev/null
+++ b/cupy/_misc/who.py
@@ -0,0 +1,106 @@
+import sys
+import cupy
+
+
+def who(vardict=None):
+    """Print the CuPy arrays in the given dictionary.
+
+    Prints out the name, shape, bytes and type of all of the ndarrays
+    present in `vardict`.
+
+    If there is no dictionary passed in or `vardict` is None then returns
+    CuPy arrays in the globals() dictionary (all CuPy arrays in the
+    namespace).
+
+    Args:
+        vardict : (None or dict)  A dictionary possibly containing ndarrays.
+                  Default is globals() if `None` specified
+
+
+    .. admonition:: Example
+
+        >>> a = cupy.arange(10)
+        >>> b = cupy.ones(20)
+        >>> cupy.who()
+        Name            Shape            Bytes            Type
+        ===========================================================
+        <BLANKLINE>
+        a               10               80               int64
+        b               20               160              float64
+        <BLANKLINE>
+        Upper bound on total bytes  =       240
+        >>> d = {'x': cupy.arange(2.0),
+        ... 'y': cupy.arange(3.0), 'txt': 'Some str',
+        ... 'idx':5}
+        >>> cupy.who(d)
+        Name            Shape            Bytes            Type
+        ===========================================================
+        <BLANKLINE>
+        x               2                16               float64
+        y               3                24               float64
+        <BLANKLINE>
+        Upper bound on total bytes  =       40
+
+    """
+
+    # Implementation is largely copied from numpy.who()
+    if vardict is None:
+        frame = sys._getframe().f_back
+        vardict = frame.f_globals
+    sta = []
+    cache = {}
+    for name in sorted(vardict.keys()):
+        if isinstance(vardict[name], cupy.ndarray):
+            var = vardict[name]
+            idv = id(var)
+            if idv in cache.keys():
+                namestr = '{} ({})'.format(name, cache[idv])
+                original = 0
+            else:
+                cache[idv] = name
+                namestr = name
+                original = 1
+            shapestr = ' x '.join(map(str, var.shape))
+            bytestr = str(var.nbytes)
+            sta.append(
+                [namestr, shapestr, bytestr, var.dtype.name, original]
+            )
+
+    maxname = 0
+    maxshape = 0
+    maxbyte = 0
+    totalbytes = 0
+    for k in range(len(sta)):
+        val = sta[k]
+        if maxname < len(val[0]):
+            maxname = len(val[0])
+        if maxshape < len(val[1]):
+            maxshape = len(val[1])
+        if maxbyte < len(val[2]):
+            maxbyte = len(val[2])
+        if val[4]:
+            totalbytes += int(val[2])
+
+    if len(sta) > 0:
+        sp1 = max(10, maxname)
+        sp2 = max(10, maxshape)
+        sp3 = max(10, maxbyte)
+        prval = 'Name {} Shape {} Bytes {} Type'.format(
+            sp1 * ' ', sp2 * ' ', sp3 * ' '
+        )
+        print("{}\n{}\n".format(prval, "=" * (len(prval) + 5)))
+
+    for k in range(len(sta)):
+        val = sta[k]
+        print(
+            '{} {} {} {} {} {} {}'.format(
+                val[0],
+                ' ' * (sp1 - len(val[0]) + 4),
+                val[1],
+                ' ' * (sp2 - len(val[1]) + 5),
+                val[2],
+                ' ' * (sp3 - len(val[2]) + 5),
+                val[3],
+            )
+        )
+    print('\nUpper bound on total bytes  =       {}'.format(totalbytes))
diff --git a/cupy/_padding/__init__.py b/cupy/_padding/__init__.py
new file mode 100644
index 0000000..d9d95aa
--- /dev/null
+++ b/cupy/_padding/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.padding.html
diff --git a/cupy/_padding/pad.py b/cupy/_padding/pad.py
new file mode 100644
index 0000000..3a0f322
--- /dev/null
+++ b/cupy/_padding/pad.py
@@ -0,0 +1,752 @@
+import numbers
+
+import numpy
+
+import cupy
+
+
+###############################################################################
+# Private utility functions.
+
+
+def _round_if_needed(arr, dtype):
+    """Rounds arr inplace if the destination dtype is an integer.
+    """
+    if cupy.issubdtype(dtype, cupy.integer):
+        arr.round(out=arr)  # bug in round so use rint (cupy/cupy#2330)
+
+
+def _slice_at_axis(sl, axis):
+    """Constructs a tuple of slices to slice an array in the given dimension.
+
+    Args:
+      sl(slice): The slice for the given dimension.
+      axis(int): The axis to which `sl` is applied. All other dimensions are
+          left "unsliced".
+
+    Returns:
+      tuple of slices: A tuple with slices matching `shape` in length.
+    """
+    return (slice(None),) * axis + (sl,) + (Ellipsis,)
+
+
+def _view_roi(array, original_area_slice, axis):
+    """Gets a view of the current region of interest during iterative padding.
+
+    When padding multiple dimensions iteratively corner values are
+    unnecessarily overwritten multiple times. This function reduces the
+    working area for the first dimensions so that corners are excluded.
+
+    Args:
+      array(cupy.ndarray): The array with the region of interest.
+      original_area_slice(tuple of slices): Denotes the area with original
+          values of the unpadded array.
+      axis(int): The currently padded dimension assuming that `axis` is padded
+          before `axis` + 1.
+
+    Returns:
+    """
+    axis += 1
+    sl = (slice(None),) * axis + original_area_slice[axis:]
+    return array[sl]
+
+
+def _pad_simple(array, pad_width, fill_value=None):
+    """Pads an array on all sides with either a constant or undefined values.
+
+    Args:
+      array(cupy.ndarray): Array to grow.
+      pad_width(sequence of tuple[int, int]): Pad width on both sides for each
+          dimension in `arr`.
+      fill_value(scalar, optional): If provided the padded area is
+          filled with this value, otherwise the pad area left undefined.
+          (Default value = None)
+    """
+    # Allocate grown array
+    new_shape = tuple(
+        left + size + right
+        for size, (left, right) in zip(array.shape, pad_width)
+    )
+    order = 'F' if array.flags.fnc else 'C'  # Fortran and not also C-order
+    padded = cupy.empty(new_shape, dtype=array.dtype, order=order)
+
+    if fill_value is not None:
+        padded.fill(fill_value)
+
+    # Copy old array into correct space
+    original_area_slice = tuple(
+        slice(left, left + size)
+        for size, (left, right) in zip(array.shape, pad_width)
+    )
+    padded[original_area_slice] = array
+
+    return padded, original_area_slice
+
+
+def _set_pad_area(padded, axis, width_pair, value_pair):
+    """Set an empty-padded area in given dimension.
+    """
+    left_slice = _slice_at_axis(slice(None, width_pair[0]), axis)
+    padded[left_slice] = value_pair[0]
+
+    right_slice = _slice_at_axis(
+        slice(padded.shape[axis] - width_pair[1], None), axis
+    )
+    padded[right_slice] = value_pair[1]
+
+
+def _get_edges(padded, axis, width_pair):
+    """Retrieves edge values from an empty-padded array along a given axis.
+
+    Args:
+      padded(cupy.ndarray): Empty-padded array.
+      axis(int): Dimension in which the edges are considered.
+      width_pair((int, int)): Pair of widths that mark the pad area on both
+          sides in the given dimension.
+    """
+    left_index = width_pair[0]
+    left_slice = _slice_at_axis(slice(left_index, left_index + 1), axis)
+    left_edge = padded[left_slice]
+
+    right_index = padded.shape[axis] - width_pair[1]
+    right_slice = _slice_at_axis(slice(right_index - 1, right_index), axis)
+    right_edge = padded[right_slice]
+
+    return left_edge, right_edge
+
+
+def _get_linear_ramps(padded, axis, width_pair, end_value_pair):
+    """Constructs linear ramps for an empty-padded array along a given axis.
+
+    Args:
+      padded(cupy.ndarray): Empty-padded array.
+      axis(int): Dimension in which the ramps are constructed.
+      width_pair((int, int)): Pair of widths that mark the pad area on both
+          sides in the given dimension.
+      end_value_pair((scalar, scalar)): End values for the linear ramps which
+          form the edge of the fully padded array. These values are included in
+          the linear ramps.
+    """
+    edge_pair = _get_edges(padded, axis, width_pair)
+
+    left_ramp = cupy.linspace(
+        start=end_value_pair[0],
+        # squeeze axis replaced by linspace
+        stop=edge_pair[0].squeeze(axis),
+        num=width_pair[0],
+        endpoint=False,
+        dtype=padded.dtype,
+        axis=axis,
+    )
+
+    right_ramp = cupy.linspace(
+        start=end_value_pair[1],
+        # squeeze axis replaced by linspace
+        stop=edge_pair[1].squeeze(axis),
+        num=width_pair[1],
+        endpoint=False,
+        dtype=padded.dtype,
+        axis=axis,
+    )
+    # Reverse linear space in appropriate dimension
+    right_ramp = right_ramp[_slice_at_axis(slice(None, None, -1), axis)]
+
+    return left_ramp, right_ramp
+
+
+def _get_stats(padded, axis, width_pair, length_pair, stat_func):
+    """Calculates a statistic for an empty-padded array along a given axis.
+
+    Args:
+      padded(cupy.ndarray): Empty-padded array.
+      axis(int): Dimension in which the statistic is calculated.
+      width_pair((int, int)): Pair of widths that mark the pad area on both
+          sides in the given dimension.
+      length_pair(2-element sequence of None or int): Gives the number of
+          values in valid area from each side that is taken into account when
+          calculating the statistic. If None the entire valid area in `padded`
+          is considered.
+      stat_func(function): Function to compute statistic. The expected
+          signature is
+          ``stat_func(x: ndarray, axis: int, keepdims: bool) -> ndarray``.
+    """
+    # Calculate indices of the edges of the area with original values
+    left_index = width_pair[0]
+    right_index = padded.shape[axis] - width_pair[1]
+    # as well as its length
+    max_length = right_index - left_index
+
+    # Limit stat_lengths to max_length
+    left_length, right_length = length_pair
+    if left_length is None or max_length < left_length:
+        left_length = max_length
+    if right_length is None or max_length < right_length:
+        right_length = max_length
+
+    # Calculate statistic for the left side
+    left_slice = _slice_at_axis(
+        slice(left_index, left_index + left_length), axis
+    )
+    left_chunk = padded[left_slice]
+    left_stat = stat_func(left_chunk, axis=axis, keepdims=True)
+    _round_if_needed(left_stat, padded.dtype)
+
+    if left_length == right_length == max_length:
+        # return early as right_stat must be identical to left_stat
+        return left_stat, left_stat
+
+    # Calculate statistic for the right side
+    right_slice = _slice_at_axis(
+        slice(right_index - right_length, right_index), axis
+    )
+    right_chunk = padded[right_slice]
+    right_stat = stat_func(right_chunk, axis=axis, keepdims=True)
+    _round_if_needed(right_stat, padded.dtype)
+    return left_stat, right_stat
+
+
+def _set_reflect_both(padded, axis, width_pair, method, include_edge=False):
+    """Pads an `axis` of `arr` using reflection.
+
+    Args:
+      padded(cupy.ndarray): Input array of arbitrary shape.
+      axis(int): Axis along which to pad `arr`.
+      width_pair((int, int)): Pair of widths that mark the pad area on both
+          sides in the given dimension.
+      method(str): Controls method of reflection; options are 'even' or 'odd'.
+      include_edge(bool, optional): If true, edge value is included in
+          reflection, otherwise the edge value forms the symmetric axis to the
+          reflection. (Default value = False)
+    """
+    left_pad, right_pad = width_pair
+    old_length = padded.shape[axis] - right_pad - left_pad
+
+    if include_edge:
+        # Edge is included, we need to offset the pad amount by 1
+        edge_offset = 1
+    else:
+        edge_offset = 0  # Edge is not included, no need to offset pad amount
+        old_length -= 1  # but must be omitted from the chunk
+
+    if left_pad > 0:
+        # Pad with reflected values on left side:
+        # First limit chunk size which can't be larger than pad area
+        chunk_length = min(old_length, left_pad)
+        # Slice right to left, stop on or next to edge, start relative to stop
+        stop = left_pad - edge_offset
+        start = stop + chunk_length
+        left_slice = _slice_at_axis(slice(start, stop, -1), axis)
+        left_chunk = padded[left_slice]
+
+        if method == 'odd':
+            # Negate chunk and align with edge
+            edge_slice = _slice_at_axis(slice(left_pad, left_pad + 1), axis)
+            left_chunk = 2 * padded[edge_slice] - left_chunk
+
+        # Insert chunk into padded area
+        start = left_pad - chunk_length
+        stop = left_pad
+        pad_area = _slice_at_axis(slice(start, stop), axis)
+        padded[pad_area] = left_chunk
+        # Adjust pointer to left edge for next iteration
+        left_pad -= chunk_length
+
+    if right_pad > 0:
+        # Pad with reflected values on right side:
+        # First limit chunk size which can't be larger than pad area
+        chunk_length = min(old_length, right_pad)
+        # Slice right to left, start on or next to edge, stop relative to start
+        start = -right_pad + edge_offset - 2
+        stop = start - chunk_length
+        right_slice = _slice_at_axis(slice(start, stop, -1), axis)
+        right_chunk = padded[right_slice]
+
+        if method == 'odd':
+            # Negate chunk and align with edge
+            edge_slice = _slice_at_axis(
+                slice(-right_pad - 1, -right_pad), axis
+            )
+            right_chunk = 2 * padded[edge_slice] - right_chunk
+
+        # Insert chunk into padded area
+        start = padded.shape[axis] - right_pad
+        stop = start + chunk_length
+        pad_area = _slice_at_axis(slice(start, stop), axis)
+        padded[pad_area] = right_chunk
+        # Adjust pointer to right edge for next iteration
+        right_pad -= chunk_length
+
+    return left_pad, right_pad
+
+
+def _set_wrap_both(padded, axis, width_pair):
+    """Pads an `axis` of `arr` with wrapped values.
+
+    Args:
+      padded(cupy.ndarray): Input array of arbitrary shape.
+      axis(int): Axis along which to pad `arr`.
+      width_pair((int, int)): Pair of widths that mark the pad area on both
+          sides in the given dimension.
+    """
+    left_pad, right_pad = width_pair
+    period = padded.shape[axis] - right_pad - left_pad
+
+    # If the current dimension of `arr` doesn't contain enough valid values
+    # (not part of the undefined pad area) we need to pad multiple times.
+    # Each time the pad area shrinks on both sides which is communicated with
+    # these variables.
+    new_left_pad = 0
+    new_right_pad = 0
+
+    if left_pad > 0:
+        # Pad with wrapped values on left side
+        # First slice chunk from right side of the non-pad area.
+        # Use min(period, left_pad) to ensure that chunk is not larger than
+        # pad area
+        right_slice = _slice_at_axis(
+            slice(
+                -right_pad - min(period, left_pad),
+                -right_pad if right_pad != 0 else None,
+            ),
+            axis,
+        )
+        right_chunk = padded[right_slice]
+
+        if left_pad > period:
+            # Chunk is smaller than pad area
+            pad_area = _slice_at_axis(slice(left_pad - period, left_pad), axis)
+            new_left_pad = left_pad - period
+        else:
+            # Chunk matches pad area
+            pad_area = _slice_at_axis(slice(None, left_pad), axis)
+        padded[pad_area] = right_chunk
+
+    if right_pad > 0:
+        # Pad with wrapped values on right side
+        # First slice chunk from left side of the non-pad area.
+        # Use min(period, right_pad) to ensure that chunk is not larger than
+        # pad area
+        left_slice = _slice_at_axis(
+            slice(left_pad, left_pad + min(period, right_pad)), axis
+        )
+        left_chunk = padded[left_slice]
+
+        if right_pad > period:
+            # Chunk is smaller than pad area
+            pad_area = _slice_at_axis(
+                slice(-right_pad, -right_pad + period), axis
+            )
+            new_right_pad = right_pad - period
+        else:
+            # Chunk matches pad area
+            pad_area = _slice_at_axis(slice(-right_pad, None), axis)
+        padded[pad_area] = left_chunk
+
+    return new_left_pad, new_right_pad
+
+
+def _as_pairs(x, ndim, as_index=False):
+    """Broadcasts `x` to an array with shape (`ndim`, 2).
+
+    A helper function for `pad` that prepares and validates arguments like
+    `pad_width` for iteration in pairs.
+
+    Args:
+      x(scalar or array-like, optional): The object to broadcast to the shape
+          (`ndim`, 2).
+      ndim(int): Number of pairs the broadcasted `x` will have.
+      as_index(bool, optional): If `x` is not None, try to round each
+          element of `x` to an integer (dtype `cupy.intp`) and ensure every
+          element is positive. (Default value = False)
+
+    Returns:
+      nested iterables, shape (`ndim`, 2): The broadcasted version of `x`.
+    """
+    if x is None:
+        # Pass through None as a special case, otherwise cupy.round(x) fails
+        # with an AttributeError
+        return ((None, None),) * ndim
+    elif isinstance(x, numbers.Number):
+        if as_index:
+            x = round(x)
+        return ((x, x),) * ndim
+
+    x = numpy.array(x)
+    if as_index:
+        x = numpy.asarray(numpy.round(x), dtype=numpy.intp)
+
+    if x.ndim < 3:
+        # Optimization: Possibly use faster paths for cases where `x` has
+        # only 1 or 2 elements. `numpy.broadcast_to` could handle these as well
+        # but is currently slower
+
+        if x.size == 1:
+            # x was supplied as a single value
+            x = x.ravel()  # Ensure x[0] works for x.ndim == 0, 1, 2
+            if as_index and x < 0:
+                raise ValueError("index can't contain negative values")
+            return ((x[0], x[0]),) * ndim
+
+        if x.size == 2 and x.shape != (2, 1):
+            # x was supplied with a single value for each side
+            # but except case when each dimension has a single value
+            # which should be broadcasted to a pair,
+            # e.g. [[1], [2]] -> [[1, 1], [2, 2]] not [[1, 2], [1, 2]]
+            x = x.ravel()  # Ensure x[0], x[1] works
+            if as_index and (x[0] < 0 or x[1] < 0):
+                raise ValueError("index can't contain negative values")
+            return ((x[0], x[1]),) * ndim
+
+    if as_index and x.min() < 0:
+        raise ValueError("index can't contain negative values")
+
+    # Converting the array with `tolist` seems to improve performance
+    # when iterating and indexing the result (see usage in `pad`)
+    x_view = x.view()
+    x_view.shape = (ndim, 2)
+    return x_view.tolist()
+
+# def _pad_dispatcher(array, pad_width, mode=None, **kwargs):
+#    return (array,)
+
+
+###############################################################################
+# Public functions
+
+
+# @array_function_dispatch(_pad_dispatcher, module='numpy')
+def pad(array, pad_width, mode='constant', **kwargs):
+    """Pads an array with specified widths and values.
+
+    Args:
+      array(cupy.ndarray): The array to pad.
+      pad_width(sequence, array_like or int): Number of values padded to the
+          edges of each axis. ((before_1, after_1), ... (before_N, after_N))
+          unique pad widths for each axis. ((before, after),) yields same
+          before and after pad for each axis. (pad,) or int is a shortcut for
+          before = after = pad width for all axes. You cannot specify
+          ``cupy.ndarray``.
+      mode(str or function, optional): One of the following string values or a
+          user supplied function
+
+          'constant' (default)
+              Pads with a constant value.
+          'edge'
+              Pads with the edge values of array.
+          'linear_ramp'
+              Pads with the linear ramp between end_value and the array edge
+              value.
+          'maximum'
+              Pads with the maximum value of all or part of the vector along
+              each axis.
+          'mean'
+              Pads with the mean value of all or part of the vector along each
+              axis.
+          'median'
+              Pads with the median value of all or part of the vector along
+              each axis. (Not Implemented)
+          'minimum'
+              Pads with the minimum value of all or part of the vector along
+              each axis.
+          'reflect'
+              Pads with the reflection of the vector mirrored on the first and
+              last values of the vector along each axis.
+          'symmetric'
+               Pads with the reflection of the vector mirrored along the edge
+               of the array.
+          'wrap'
+              Pads with the wrap of the vector along the axis. The first
+              values are used to pad the end and the end values are used to
+              pad the beginning.
+          'empty'
+              Pads with undefined values.
+          <function>
+              Padding function, see Notes.
+      stat_length(sequence or int, optional): Used in 'maximum', 'mean',
+          'median', and 'minimum'.  Number of values at edge of each axis used
+          to calculate the statistic value.
+          ((before_1, after_1), ... (before_N, after_N)) unique statistic
+          lengths for each axis. ((before, after),) yields same before and
+          after statistic lengths for each axis. (stat_length,) or int is a
+          shortcut for before = after = statistic length for all axes.
+          Default is ``None``, to use the entire axis. You cannot specify
+          ``cupy.ndarray``.
+      constant_values(sequence or scalar, optional): Used in 'constant'. The
+          values to set the padded values for each axis.
+          ((before_1, after_1), ... (before_N, after_N)) unique pad constants
+          for each axis.
+          ((before, after),) yields same before and after constants for each
+          axis.
+          (constant,) or constant is a shortcut for before = after = constant
+          for all axes.
+          Default is 0. You cannot specify ``cupy.ndarray``.
+      end_values(sequence or scalar, optional): Used in 'linear_ramp'. The
+          values used for the ending value of the linear_ramp and that will
+          form the edge of the padded array.
+          ((before_1, after_1), ... (before_N, after_N)) unique end values
+          for each axis.
+          ((before, after),) yields same before and after end
+          values for each axis.
+          (constant,) or constant is a shortcut for before = after = constant
+          for all axes.
+          Default is 0. You cannot specify ``cupy.ndarray``.
+      reflect_type({'even', 'odd'}, optional): Used in 'reflect', and
+          'symmetric'.  The 'even' style is the default with an unaltered
+          reflection around the edge value.  For the 'odd' style, the extended
+          part of the array is created by subtracting the reflected values from
+          two times the edge value.
+
+    Returns:
+      cupy.ndarray: Padded array with shape extended by ``pad_width``.
+
+    .. note::
+        For an array with rank greater than 1, some of the padding of later
+        axes is calculated from padding of previous axes.  This is easiest to
+        think about with a rank 2 array where the corners of the padded array
+        are calculated by using padded values from the first axis.
+
+        The padding function, if used, should modify a rank 1 array in-place.
+        It has the following signature:
+
+        ``padding_func(vector, iaxis_pad_width, iaxis, kwargs)``
+
+        where
+
+        vector (cupy.ndarray)
+            A rank 1 array already padded with zeros.  Padded values are
+            ``vector[:iaxis_pad_width[0]]`` and
+            ``vector[-iaxis_pad_width[1]:]``.
+        iaxis_pad_width (tuple)
+            A 2-tuple of ints, ``iaxis_pad_width[0]`` represents the number of
+            values padded at the beginning of vector where
+            ``iaxis_pad_width[1]`` represents the number of values padded at
+            the end of vector.
+        iaxis (int)
+            The axis currently being calculated.
+        kwargs (dict)
+            Any keyword arguments the function requires.
+
+    Examples
+    --------
+    >>> a = cupy.array([1, 2, 3, 4, 5])
+    >>> cupy.pad(a, (2, 3), 'constant', constant_values=(4, 6))
+    array([4, 4, 1, ..., 6, 6, 6])
+
+    >>> cupy.pad(a, (2, 3), 'edge')
+    array([1, 1, 1, ..., 5, 5, 5])
+
+    >>> cupy.pad(a, (2, 3), 'linear_ramp', end_values=(5, -4))
+    array([ 5,  3,  1,  2,  3,  4,  5,  2, -1, -4])
+
+    >>> cupy.pad(a, (2,), 'maximum')
+    array([5, 5, 1, 2, 3, 4, 5, 5, 5])
+
+    >>> cupy.pad(a, (2,), 'mean')
+    array([3, 3, 1, 2, 3, 4, 5, 3, 3])
+
+    >>> a = cupy.array([[1, 2], [3, 4]])
+    >>> cupy.pad(a, ((3, 2), (2, 3)), 'minimum')
+    array([[1, 1, 1, 2, 1, 1, 1],
+           [1, 1, 1, 2, 1, 1, 1],
+           [1, 1, 1, 2, 1, 1, 1],
+           [1, 1, 1, 2, 1, 1, 1],
+           [3, 3, 3, 4, 3, 3, 3],
+           [1, 1, 1, 2, 1, 1, 1],
+           [1, 1, 1, 2, 1, 1, 1]])
+
+    >>> a = cupy.array([1, 2, 3, 4, 5])
+    >>> cupy.pad(a, (2, 3), 'reflect')
+    array([3, 2, 1, 2, 3, 4, 5, 4, 3, 2])
+
+    >>> cupy.pad(a, (2, 3), 'reflect', reflect_type='odd')
+    array([-1,  0,  1,  2,  3,  4,  5,  6,  7,  8])
+
+    >>> cupy.pad(a, (2, 3), 'symmetric')
+    array([2, 1, 1, 2, 3, 4, 5, 5, 4, 3])
+
+    >>> cupy.pad(a, (2, 3), 'symmetric', reflect_type='odd')
+    array([0, 1, 1, 2, 3, 4, 5, 5, 6, 7])
+
+    >>> cupy.pad(a, (2, 3), 'wrap')
+    array([4, 5, 1, 2, 3, 4, 5, 1, 2, 3])
+
+    >>> def pad_with(vector, pad_width, iaxis, kwargs):
+    ...     pad_value = kwargs.get('padder', 10)
+    ...     vector[:pad_width[0]] = pad_value
+    ...     vector[-pad_width[1]:] = pad_value
+    >>> a = cupy.arange(6)
+    >>> a = a.reshape((2, 3))
+    >>> cupy.pad(a, 2, pad_with)
+    array([[10, 10, 10, 10, 10, 10, 10],
+           [10, 10, 10, 10, 10, 10, 10],
+           [10, 10,  0,  1,  2, 10, 10],
+           [10, 10,  3,  4,  5, 10, 10],
+           [10, 10, 10, 10, 10, 10, 10],
+           [10, 10, 10, 10, 10, 10, 10]])
+    >>> cupy.pad(a, 2, pad_with, padder=100)
+    array([[100, 100, 100, 100, 100, 100, 100],
+           [100, 100, 100, 100, 100, 100, 100],
+           [100, 100,   0,   1,   2, 100, 100],
+           [100, 100,   3,   4,   5, 100, 100],
+           [100, 100, 100, 100, 100, 100, 100],
+           [100, 100, 100, 100, 100, 100, 100]])
+    """
+    if isinstance(pad_width, numbers.Integral):
+        pad_width = ((pad_width, pad_width),) * array.ndim
+    else:
+        pad_width = numpy.asarray(pad_width)
+
+        if not pad_width.dtype.kind == 'i':
+            raise TypeError('`pad_width` must be of integral type.')
+
+        # Broadcast to shape (array.ndim, 2)
+        pad_width = _as_pairs(pad_width, array.ndim, as_index=True)
+
+    if callable(mode):
+        # Old behavior: Use user-supplied function with numpy.apply_along_axis
+        function = mode
+        # Create a new zero padded array
+        padded, _ = _pad_simple(array, pad_width, fill_value=0)
+        # And apply along each axis
+
+        for axis in range(padded.ndim):
+            # Iterate using ndindex as in apply_along_axis, but assuming that
+            # function operates inplace on the padded array.
+
+            # view with the iteration axis at the end
+            view = cupy.moveaxis(padded, axis, -1)
+
+            # compute indices for the iteration axes, and append a trailing
+            # ellipsis to prevent 0d arrays decaying to scalars (gh-8642)
+            inds = numpy.ndindex(view.shape[:-1])
+            inds = (ind + (Ellipsis,) for ind in inds)
+            for ind in inds:
+                function(view[ind], pad_width[axis], axis, kwargs)
+
+        return padded
+
+    # Make sure that no unsupported keywords were passed for the current mode
+    allowed_kwargs = {
+        'empty': [],
+        'edge': [],
+        'wrap': [],
+        'constant': ['constant_values'],
+        'linear_ramp': ['end_values'],
+        'maximum': ['stat_length'],
+        'mean': ['stat_length'],
+        # 'median': ['stat_length'],
+        'minimum': ['stat_length'],
+        'reflect': ['reflect_type'],
+        'symmetric': ['reflect_type'],
+    }
+    try:
+        unsupported_kwargs = set(kwargs) - set(allowed_kwargs[mode])
+    except KeyError:
+        raise ValueError("mode '{}' is not supported".format(mode))
+    if unsupported_kwargs:
+        raise ValueError(
+            "unsupported keyword arguments for mode '{}': {}".format(
+                mode, unsupported_kwargs
+            )
+        )
+
+    if mode == 'constant':
+        values = kwargs.get('constant_values', 0)
+        if isinstance(values, numbers.Number) and values == 0 and (
+                array.ndim == 1 or array.size < 4e6):
+            # faster path for 1d arrays or small n-dimensional arrays
+            return _pad_simple(array, pad_width, 0)[0]
+
+    stat_functions = {
+        'maximum': cupy.max,
+        'minimum': cupy.min,
+        'mean': cupy.mean,
+        # 'median': cupy.median,
+    }
+
+    # Create array with final shape and original values
+    # (padded area is undefined)
+    padded, original_area_slice = _pad_simple(array, pad_width)
+    # And prepare iteration over all dimensions
+    # (zipping may be more readable than using enumerate)
+    axes = range(padded.ndim)
+
+    if mode == 'constant':
+        values = _as_pairs(values, padded.ndim)
+        for axis, width_pair, value_pair in zip(axes, pad_width, values):
+            roi = _view_roi(padded, original_area_slice, axis)
+            _set_pad_area(roi, axis, width_pair, value_pair)
+
+    elif mode == 'empty':
+        pass  # Do nothing as _pad_simple already returned the correct result
+
+    elif array.size == 0:
+        # Only modes 'constant' and 'empty' can extend empty axes, all other
+        # modes depend on `array` not being empty
+        # -> ensure every empty axis is only 'padded with 0'
+        for axis, width_pair in zip(axes, pad_width):
+            if array.shape[axis] == 0 and any(width_pair):
+                raise ValueError(
+                    "can't extend empty axis {} using modes other than "
+                    "'constant' or 'empty'".format(axis)
+                )
+        # passed, don't need to do anything more as _pad_simple already
+        # returned the correct result
+
+    elif mode == 'edge':
+        for axis, width_pair in zip(axes, pad_width):
+            roi = _view_roi(padded, original_area_slice, axis)
+            edge_pair = _get_edges(roi, axis, width_pair)
+            _set_pad_area(roi, axis, width_pair, edge_pair)
+
+    elif mode == 'linear_ramp':
+        end_values = kwargs.get('end_values', 0)
+        end_values = _as_pairs(end_values, padded.ndim)
+        for axis, width_pair, value_pair in zip(axes, pad_width, end_values):
+            roi = _view_roi(padded, original_area_slice, axis)
+            ramp_pair = _get_linear_ramps(roi, axis, width_pair, value_pair)
+            _set_pad_area(roi, axis, width_pair, ramp_pair)
+
+    elif mode in stat_functions:
+        func = stat_functions[mode]
+        length = kwargs.get('stat_length', None)
+        length = _as_pairs(length, padded.ndim, as_index=True)
+        for axis, width_pair, length_pair in zip(axes, pad_width, length):
+            roi = _view_roi(padded, original_area_slice, axis)
+            stat_pair = _get_stats(roi, axis, width_pair, length_pair, func)
+            _set_pad_area(roi, axis, width_pair, stat_pair)
+
+    elif mode in {'reflect', 'symmetric'}:
+        method = kwargs.get('reflect_type', 'even')
+        include_edge = True if mode == 'symmetric' else False
+        for axis, (left_index, right_index) in zip(axes, pad_width):
+            if array.shape[axis] == 1 and (left_index > 0 or right_index > 0):
+                # Extending singleton dimension for 'reflect' is legacy
+                # behavior; it really should raise an error.
+                edge_pair = _get_edges(padded, axis, (left_index, right_index))
+                _set_pad_area(
+                    padded, axis, (left_index, right_index), edge_pair
+                )
+                continue
+
+            roi = _view_roi(padded, original_area_slice, axis)
+            while left_index > 0 or right_index > 0:
+                # Iteratively pad until dimension is filled with reflected
+                # values. This is necessary if the pad area is larger than
+                # the length of the original values in the current dimension.
+                left_index, right_index = _set_reflect_both(
+                    roi, axis, (left_index, right_index), method, include_edge
+                )
+
+    elif mode == 'wrap':
+        for axis, (left_index, right_index) in zip(axes, pad_width):
+            roi = _view_roi(padded, original_area_slice, axis)
+            while left_index > 0 or right_index > 0:
+                # Iteratively pad until dimension is filled with wrapped
+                # values. This is necessary if the pad area is larger than
+                # the length of the original values in the current dimension.
+                left_index, right_index = _set_wrap_both(
+                    roi, axis, (left_index, right_index)
+                )
+
+    return padded
diff --git a/cupy/_sorting/__init__.py b/cupy/_sorting/__init__.py
new file mode 100644
index 0000000..597fd75
--- /dev/null
+++ b/cupy/_sorting/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.sort.html
diff --git a/cupy/_sorting/count.py b/cupy/_sorting/count.py
new file mode 100644
index 0000000..92b57ed
--- /dev/null
+++ b/cupy/_sorting/count.py
@@ -0,0 +1,31 @@
+from cupy import _core
+
+
+def count_nonzero(a, axis=None):
+    """Counts the number of non-zero values in the array.
+
+    .. note::
+
+       :func:`numpy.count_nonzero` returns `int` value when `axis=None`,
+       but :func:`cupy.count_nonzero` returns zero-dimensional array to reduce
+       CPU-GPU synchronization.
+
+    Args:
+        a (cupy.ndarray): The array for which to count non-zeros.
+        axis (int or tuple, optional): Axis or tuple of axes along which to
+            count non-zeros. Default is None, meaning that non-zeros will be
+            counted along a flattened version of ``a``
+    Returns:
+        cupy.ndarray of int: Number of non-zero values in the array
+        along a given axis. Otherwise, the total number of non-zero values
+        in the array is returned.
+    """
+
+    return _count_nonzero(a, axis=axis)
+
+
+_count_nonzero = _core.create_reduction_func(
+    'cupy_count_nonzero',
+    ('?->l', 'B->l', 'h->l', 'H->l', 'i->l', 'I->l', 'l->l', 'L->l',
+     'q->l', 'Q->l', 'e->l', 'f->l', 'd->l', 'F->l', 'D->l'),
+    ('in0 != type_in0_raw(0)', 'a + b', 'out0 = a', None), 0)
diff --git a/cupy/_sorting/search.py b/cupy/_sorting/search.py
new file mode 100644
index 0000000..0eb9506
--- /dev/null
+++ b/cupy/_sorting/search.py
@@ -0,0 +1,466 @@
+import cupy
+from cupy import _core
+from cupy._core import fusion
+from cupy import _util
+
+from cupy._core import _routines_indexing as _indexing
+from cupy._core import _routines_statistics as _statistics
+
+
+def argmax(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Returns the indices of the maximum along an axis.
+
+    Args:
+        a (cupy.ndarray): Array to take argmax.
+        axis (int): Along which axis to find the maximum. ``a`` is flattened by
+            default.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis ``axis`` is preserved as an axis
+            of length one.
+
+    Returns:
+        cupy.ndarray: The indices of the maximum of ``a`` along an axis.
+
+    .. note::
+       ``dtype`` and ``keepdim`` arguments are specific to CuPy. They are
+       not in NumPy.
+
+    .. note::
+       ``axis`` argument accepts a tuple of ints, but this is specific to
+       CuPy. NumPy does not support it.
+
+    .. seealso:: :func:`numpy.argmax`
+
+    """
+    # TODO(okuta): check type
+    return a.argmax(axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def nanargmax(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Return the indices of the maximum values in the specified axis ignoring
+    NaNs. For all-NaN slice ``-1`` is returned.
+    Subclass cannot be passed yet, subok=True still unsupported
+
+    Args:
+        a (cupy.ndarray): Array to take nanargmax.
+        axis (int): Along which axis to find the maximum. ``a`` is flattened by
+            default.
+
+    Returns:
+        cupy.ndarray: The indices of the maximum of ``a``
+        along an axis ignoring NaN values.
+
+    .. note:: For performance reasons, ``cupy.nanargmax`` returns
+            ``out of range values`` for all-NaN slice
+            whereas ``numpy.nanargmax`` raises ``ValueError``
+    .. seealso:: :func:`numpy.nanargmax`
+    """
+    if a.dtype.kind in 'biu':
+        return argmax(a, axis=axis)
+
+    return _statistics._nanargmax(a, axis, dtype, out, keepdims)
+
+
+def argmin(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Returns the indices of the minimum along an axis.
+
+    Args:
+        a (cupy.ndarray): Array to take argmin.
+        axis (int): Along which axis to find the minimum. ``a`` is flattened by
+            default.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis ``axis`` is preserved as an axis
+            of length one.
+
+    Returns:
+        cupy.ndarray: The indices of the minimum of ``a`` along an axis.
+
+    .. note::
+       ``dtype`` and ``keepdim`` arguments are specific to CuPy. They are
+       not in NumPy.
+
+    .. note::
+       ``axis`` argument accepts a tuple of ints, but this is specific to
+       CuPy. NumPy does not support it.
+
+    .. seealso:: :func:`numpy.argmin`
+
+    """
+    # TODO(okuta): check type
+    return a.argmin(axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def nanargmin(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Return the indices of the minimum values in the specified axis ignoring
+    NaNs. For all-NaN slice ``-1`` is returned.
+    Subclass cannot be passed yet, subok=True still unsupported
+
+    Args:
+        a (cupy.ndarray): Array to take nanargmin.
+        axis (int): Along which axis to find the minimum. ``a`` is flattened by
+            default.
+
+    Returns:
+        cupy.ndarray: The indices of the minimum of ``a``
+        along an axis ignoring NaN values.
+
+    .. note:: For performance reasons, ``cupy.nanargmin`` returns
+            ``out of range values`` for all-NaN slice
+            whereas ``numpy.nanargmin`` raises ``ValueError``
+    .. seealso:: :func:`numpy.nanargmin`
+    """
+    if a.dtype.kind in 'biu':
+        return argmin(a, axis=axis)
+
+    return _statistics._nanargmin(a, axis, dtype, out, keepdims)
+
+
+def nonzero(a):
+    """Return the indices of the elements that are non-zero.
+
+    Returns a tuple of arrays, one for each dimension of a,
+    containing the indices of the non-zero elements in that dimension.
+
+    Args:
+        a (cupy.ndarray): array
+
+    Returns:
+        tuple of arrays: Indices of elements that are non-zero.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.nonzero`
+
+    """
+    _util.check_array(a, arg_name='a')
+    return a.nonzero()
+
+
+def flatnonzero(a):
+    """Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to a.ravel().nonzero()[0].
+
+    Args:
+        a (cupy.ndarray): input array
+
+    Returns:
+        cupy.ndarray: Output array,
+        containing the indices of the elements of a.ravel() that are non-zero.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.flatnonzero`
+    """
+    _util.check_array(a, arg_name='a')
+    return a.ravel().nonzero()[0]
+
+
+_where_ufunc = _core.create_ufunc(
+    'cupy_where',
+    ('???->?', '?bb->b', '?BB->B', '?hh->h', '?HH->H', '?ii->i', '?II->I',
+     '?ll->l', '?LL->L', '?qq->q', '?QQ->Q', '?ee->e', '?ff->f',
+     # On CUDA 6.5 these combinations don't work correctly (on CUDA >=7.0, it
+     # works).
+     # See issue #551.
+     '?hd->d', '?Hd->d',
+     '?dd->d', '?FF->F', '?DD->D'),
+    'out0 = in0 ? in1 : in2')
+
+
+def where(condition, x=None, y=None):
+    """Return elements, either from x or y, depending on condition.
+
+    If only condition is given, return ``condition.nonzero()``.
+
+    Args:
+        condition (cupy.ndarray): When True, take x, otherwise take y.
+        x (cupy.ndarray): Values from which to choose on ``True``.
+        y (cupy.ndarray): Values from which to choose on ``False``.
+
+    Returns:
+        cupy.ndarray: Each element of output contains elements of ``x`` when
+        ``condition`` is ``True``, otherwise elements of ``y``. If only
+        ``condition`` is given, return the tuple ``condition.nonzero()``,
+        the indices where ``condition`` is True.
+
+    .. warning::
+
+        This function may synchronize the device if both ``x`` and ``y`` are
+        omitted.
+
+    .. seealso:: :func:`numpy.where`
+
+    """
+
+    missing = (x is None, y is None).count(True)
+
+    if missing == 1:
+        raise ValueError('Must provide both \'x\' and \'y\' or neither.')
+    if missing == 2:
+        return nonzero(condition)  # may synchronize
+
+    if fusion._is_fusing():
+        return fusion._call_ufunc(_where_ufunc, condition, x, y)
+    return _where_ufunc(condition.astype('?'), x, y)
+
+
+def argwhere(a):
+    """Return the indices of the elements that are non-zero.
+
+    Returns a (N, ndim) dimantional array containing the
+    indices of the non-zero elements. Where `N` is number of
+    non-zero elements and `ndim` is dimension of the given array.
+
+    Args:
+        a (cupy.ndarray): array
+
+    Returns:
+        cupy.ndarray: Indices of elements that are non-zero.
+
+    .. seealso:: :func:`numpy.argwhere`
+
+    """
+    _util.check_array(a, arg_name='a')
+    return _indexing._ndarray_argwhere(a)
+
+
+# This is to allow using the same kernels for all dtypes, ints & floats
+# as nan is a special case
+_preamble = '''
+template<typename T>
+__device__ bool _isnan(T val) {
+    return val != val;
+}
+'''
+
+
+_hip_preamble = r'''
+#ifdef __HIP_DEVICE_COMPILE__
+  #define no_thread_divergence(do_work, to_return) \
+    if (!is_done) {                                \
+      do_work;                                     \
+      is_done = true;                              \
+    }
+#else
+  #define no_thread_divergence(do_work, to_return) \
+    do_work;                                       \
+    if (to_return) { return; }
+#endif
+'''
+
+
+_searchsorted_code = '''
+    #ifdef __HIP_DEVICE_COMPILE__
+    bool is_done = false;
+    #endif
+
+    // Array is assumed to be monotonically
+    // increasing unless a check is requested with the
+    // `assume_increasing = False` parameter.
+    // `digitize` allows increasing and decreasing arrays.
+    bool inc = true;
+    if (!assume_increasing && n_bins >= 2) {
+        // In the case all the bins are nan the array is considered
+        // to be decreasing in numpy
+        inc = (bins[0] <= bins[n_bins-1])
+              || (!_isnan<T>(bins[0]) && _isnan<T>(bins[n_bins-1]));
+    }
+
+    if (_isnan<S>(x)) {
+        long long pos = (inc ? n_bins : 0);
+        if (!side_is_right) {
+            if (inc) {
+                while (pos > 0 && _isnan<T>(bins[pos-1])) {
+                    --pos;
+                }
+            } else {
+                while (pos < n_bins && _isnan<T>(bins[pos])) {
+                    ++pos;
+                }
+            }
+        }
+        no_thread_divergence( y = pos , true )
+    }
+
+    bool greater = false;
+    if (side_is_right) {
+        greater = inc && x >= bins[n_bins-1];
+    } else {
+        greater = (inc ? x > bins[n_bins-1] : x <= bins[n_bins-1]);
+    }
+    if (greater) {
+        no_thread_divergence( y = n_bins , true )
+    }
+
+    long long left = 0;
+    // In the case the bins is all NaNs, digitize
+    // needs to place all the valid values to the right
+    if (!inc) {
+        while (_isnan<T>(bins[left]) && left < n_bins) {
+            ++left;
+        }
+        if (left == n_bins) {
+            no_thread_divergence( y = n_bins , true )
+        }
+        if (side_is_right
+                && !_isnan<T>(bins[n_bins-1]) && !_isnan<S>(x)
+                && bins[n_bins-1] > x) {
+            no_thread_divergence( y = n_bins , true )
+        }
+    }
+
+    long long right = n_bins-1;
+    while (left < right) {
+        long long m = left + (right - left) / 2;
+        bool look_right = true;
+        if (side_is_right) {
+            look_right = (inc ? bins[m] <= x : bins[m] > x);
+        } else {
+            look_right = (inc ? bins[m] < x : bins[m] >= x);
+        }
+        if (look_right) {
+            left = m + 1;
+        } else {
+            right = m;
+        }
+    }
+    no_thread_divergence( y = right , false )
+'''
+
+
+_searchsorted_kernel = _core.ElementwiseKernel(
+    'S x, raw T bins, int64 n_bins, bool side_is_right, '
+    'bool assume_increasing',
+    'int64 y',
+    _searchsorted_code,
+    name='cupy_searchsorted_kernel', preamble=_preamble+_hip_preamble)
+
+
+_hip_preamble = r'''
+#ifdef __HIP_DEVICE_COMPILE__
+  #define no_thread_divergence(do_work, to_return) \
+    if (!is_done) {                                \
+      do_work;                                     \
+      is_done = true;                              \
+    }
+#else
+  #define no_thread_divergence(do_work, to_return) \
+    do_work;                                       \
+    if (to_return) {                               \
+      out = (y == n_bins ? false : bins[y] == x);  \
+      if (invert) out = !out;                      \
+      return;                                      \
+    }
+#endif
+'''
+
+
+_exists_kernel = _core.ElementwiseKernel(
+    'S x, raw T bins, int64 n_bins, bool invert',
+    'bool out',
+    '''
+    const bool assume_increasing = true;
+    const bool side_is_right = false;
+    long long y;
+    '''
+    + _searchsorted_code + '''
+    out = (y == n_bins ? false : bins[y] == x);
+    if (invert) out = !out;
+    ''', name='cupy_exists_kernel', preamble=_preamble+_hip_preamble)
+
+
+_exists_and_searchsorted_kernel = _core.ElementwiseKernel(
+    'S x, raw T bins, int64 n_bins, bool invert',
+    'bool out, int64 y',
+    '''
+    const bool assume_increasing = true;
+    const bool side_is_right = false;
+    '''
+    + _searchsorted_code + '''
+    out = (y == n_bins ? false : bins[y] == x);
+    if (invert) out = !out;
+    ''', name='cupy_exists_and_searchsorted_kernel',
+    preamble=_preamble+_hip_preamble)
+
+
+def searchsorted(a, v, side='left', sorter=None):
+    """Finds indices where elements should be inserted to maintain order.
+
+    Find the indices into a sorted array ``a`` such that,
+    if the corresponding elements in ``v`` were inserted before the indices,
+    the order of ``a`` would be preserved.
+
+    Args:
+        a (cupy.ndarray): Input array. If ``sorter`` is ``None``, then
+            it must be sorted in ascending order,
+            otherwise ``sorter`` must be an array of indices that sort it.
+        v (cupy.ndarray): Values to insert into ``a``.
+        side : {'left', 'right'}
+            If ``left``, return the index of the first suitable location found
+            If ``right``, return the last such index.
+            If there is no suitable index, return either 0 or length of ``a``.
+        sorter : 1-D array_like
+            Optional array of integer indices that sort array ``a`` into
+            ascending order. They are typically the result of
+            :func:`~cupy.argsort`.
+
+    Returns:
+        cupy.ndarray: Array of insertion points with the same shape as ``v``.
+
+    .. note:: When a is not in ascending order, behavior is undefined.
+
+    .. seealso:: :func:`numpy.searchsorted`
+
+    """
+    return _searchsorted(a, v, side, sorter, True)
+
+
+def _searchsorted(a, v, side, sorter, assume_increasing):
+    """`assume_increasing` is used in the kernel to
+    skip monotonically increasing or decreasing verification
+    inside the cuda kernel.
+    """
+    if not isinstance(a, cupy.ndarray):
+        raise NotImplementedError('Only int or ndarray are supported for a')
+
+    if not isinstance(v, cupy.ndarray):
+        raise NotImplementedError('Only int or ndarray are supported for v')
+
+    if a.ndim > 1:
+        raise ValueError('object too deep for desired array')
+    if a.ndim < 1:
+        raise ValueError('object of too small depth for desired array')
+    if a.size == 0:
+        return cupy.zeros(v.shape, dtype=cupy.int64)
+
+    a_iscomplex = a.dtype.kind == 'c'
+    v_iscomplex = v.dtype.kind == 'c'
+
+    if a_iscomplex and not v_iscomplex:
+        v = v.astype(a.dtype)
+    elif v_iscomplex and not a_iscomplex:
+        a = a.astype(v.dtype)
+
+    # Numpy does not check if the array is monotonic inside searchsorted
+    # which leads to undefined behavior in such cases.
+    if sorter is not None:
+        if sorter.dtype.kind not in ('i', 'u'):
+            raise TypeError('sorter must be of integer type')
+        if sorter.size != a.size:
+            raise ValueError('sorter.size must equal a.size')
+        a = a.take(sorter)
+
+    y = cupy.zeros(v.shape, dtype=cupy.int64)
+
+    _searchsorted_kernel(v, a, a.size, side == 'right', assume_increasing, y)
+    return y
+
+
+# TODO(okuta): Implement extract
diff --git a/cupy/_sorting/sort.py b/cupy/_sorting/sort.py
new file mode 100644
index 0000000..5f7cd3c
--- /dev/null
+++ b/cupy/_sorting/sort.py
@@ -0,0 +1,217 @@
+import warnings
+
+import cupy
+import numpy
+
+from cupy.cuda import thrust
+
+
+def sort(a, axis=-1, kind=None):
+    """Returns a sorted copy of an array with a stable sorting algorithm.
+
+    Args:
+        a (cupy.ndarray): Array to be sorted.
+        axis (int or None): Axis along which to sort. Default is -1, which
+            means sort along the last axis. If None is supplied, the array is
+            flattened before sorting.
+        kind: Default is `None`, which is equivalent to 'stable'. Unlike in
+            NumPy any other options are not accepted here.
+
+    Returns:
+        cupy.ndarray: Array of the same type and shape as ``a``.
+
+    .. note::
+       For its implementation reason, ``cupy.sort`` currently does not support
+       ``kind`` and ``order`` parameters that ``numpy.sort`` does
+       support.
+
+    .. seealso:: :func:`numpy.sort`
+
+    """
+    if kind is not None and kind != 'stable':
+        raise ValueError("kind can only be None or 'stable'")
+    if axis is None:
+        ret = a.flatten()
+        axis = -1
+    else:
+        ret = a.copy()
+    ret.sort(axis=axis)
+    return ret
+
+
+def lexsort(keys):
+    """Perform an indirect sort using an array of keys.
+
+    Args:
+        keys (cupy.ndarray): ``(k, N)`` array containing ``k`` ``(N,)``-shaped
+            arrays. The ``k`` different "rows" to be sorted. The last row is
+            the primary sort key.
+
+    Returns:
+        cupy.ndarray: Array of indices that sort the keys.
+
+    .. note::
+        For its implementation reason, ``cupy.lexsort`` currently supports only
+        keys with their rank of one or two and does not support ``axis``
+        parameter that ``numpy.lexsort`` supports.
+
+    .. seealso:: :func:`numpy.lexsort`
+
+    """
+
+    # TODO(takagi): Support axis argument.
+
+    if not cupy.cuda.thrust.available:
+        raise RuntimeError('Thrust is needed to use cupy.lexsort. Please '
+                           'install CUDA Toolkit with Thrust then reinstall '
+                           'CuPy after uninstalling it.')
+
+    if keys.ndim == ():
+        # as numpy.lexsort() raises
+        raise TypeError('need sequence of keys with len > 0 in lexsort')
+
+    if keys.ndim == 1:
+        return cupy.array(0, dtype=numpy.intp)
+
+    # TODO(takagi): Support ranks of three or more.
+    if keys.ndim > 2:
+        raise NotImplementedError('Keys with the rank of three or more is not '
+                                  'supported in lexsort')
+
+    # thrust.lexsort() assumes a C-contiguous array
+    if not keys.flags.c_contiguous:
+        keys = keys.copy('C')
+
+    idx_array = cupy.ndarray(keys._shape[1:], dtype=numpy.intp)
+    k = keys._shape[0]
+    n = keys._shape[1]
+    thrust.lexsort(keys.dtype, idx_array.data.ptr, keys.data.ptr, k, n)
+
+    return idx_array
+
+
+def argsort(a, axis=-1, kind=None):
+    """Returns the indices that would sort an array with a stable sorting.
+
+    Args:
+        a (cupy.ndarray): Array to sort.
+        axis (int or None): Axis along which to sort. Default is -1, which
+            means sort along the last axis. If None is supplied, the array is
+            flattened before sorting.
+        kind: Default is `None`, which is equivalent to 'stable'. Unlike in
+            NumPy any other options are not accepted here.
+
+    Returns:
+        cupy.ndarray: Array of indices that sort ``a``.
+
+    .. note::
+        For its implementation reason, ``cupy.argsort`` does not support
+        ``kind`` and ``order`` parameters.
+
+    .. seealso:: :func:`numpy.argsort`
+
+    """
+    if kind is not None and kind != 'stable':
+        raise ValueError("kind can only be None or 'stable'")
+    return a.argsort(axis=axis)
+
+
+def msort(a):
+    """Returns a copy of an array sorted along the first axis.
+
+    Args:
+        a (cupy.ndarray): Array to be sorted.
+
+    Returns:
+        cupy.ndarray: Array of the same type and shape as ``a``.
+
+    .. note:
+        ``cupy.msort(a)``, the CuPy counterpart of ``numpy.msort(a)``, is
+        equivalent to ``cupy.sort(a, axis=0)``.
+
+    .. seealso:: :func:`numpy.msort`
+
+    """
+    warnings.warn(
+        'msort is deprecated, use cupy.sort(a, axis=0) instead',
+        DeprecationWarning)
+    return sort(a, axis=0)
+
+
+def sort_complex(a):
+    """Sort a complex array using the real part first,
+    then the imaginary part.
+
+    Args:
+        a (cupy.ndarray): Array to be sorted.
+
+    Returns:
+        cupy.ndarray: sorted complex array.
+
+    .. seealso:: :func:`numpy.sort_complex`
+
+    """
+    if a.dtype.char in 'bhBHF':
+        a = a.astype('F')
+    else:
+        a = a.astype('D')
+    a.sort()
+    return a
+
+
+def partition(a, kth, axis=-1):
+    """Returns a partitioned copy of an array.
+
+    Creates a copy of the array whose elements are rearranged such that the
+    value of the element in k-th position would occur in that position in a
+    sorted array. All of the elements before the new k-th element are less
+    than or equal to the elements after the new k-th element.
+
+    Args:
+        a (cupy.ndarray): Array to be sorted.
+        kth (int or sequence of ints): Element index to partition by. If
+            supplied with a sequence of k-th it will partition all elements
+            indexed by k-th of them into their sorted position at once.
+        axis (int or None): Axis along which to sort. Default is -1, which
+            means sort along the last axis. If None is supplied, the array is
+            flattened before sorting.
+
+    Returns:
+        cupy.ndarray: Array of the same type and shape as ``a``.
+
+    .. seealso:: :func:`numpy.partition`
+
+    """
+    if axis is None:
+        ret = a.flatten()
+        axis = -1
+    else:
+        ret = a.copy()
+    ret.partition(kth, axis=axis)
+    return ret
+
+
+def argpartition(a, kth, axis=-1):
+    """Returns the indices that would partially sort an array.
+
+    Args:
+        a (cupy.ndarray): Array to be sorted.
+        kth (int or sequence of ints): Element index to partition by. If
+            supplied with a sequence of k-th it will partition all elements
+            indexed by k-th of them into their sorted position at once.
+        axis (int or None): Axis along which to sort. Default is -1, which
+            means sort along the last axis. If None is supplied, the array is
+            flattened before sorting.
+
+    Returns:
+        cupy.ndarray: Array of the same type and shape as ``a``.
+
+    .. note::
+        For its implementation reason, `cupy.argpartition` fully sorts the
+        given array as `cupy.argsort` does. It also does not support ``kind``
+        and ``order`` parameters that ``numpy.argpartition`` supports.
+
+    .. seealso:: :func:`numpy.argpartition`
+
+    """
+    return a.argpartition(kth, axis=axis)
diff --git a/cupy/_statistics/__init__.py b/cupy/_statistics/__init__.py
new file mode 100644
index 0000000..4eb0c01
--- /dev/null
+++ b/cupy/_statistics/__init__.py
@@ -0,0 +1,2 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.statistics.html
diff --git a/cupy/_statistics/correlation.py b/cupy/_statistics/correlation.py
new file mode 100644
index 0000000..b6ae419
--- /dev/null
+++ b/cupy/_statistics/correlation.py
@@ -0,0 +1,212 @@
+import functools
+import warnings
+
+import numpy
+
+import cupy
+from cupy import _core
+
+
+def corrcoef(a, y=None, rowvar=True, bias=None, ddof=None, *, dtype=None):
+    """Returns the Pearson product-moment correlation coefficients of an array.
+
+    Args:
+        a (cupy.ndarray): Array to compute the Pearson product-moment
+            correlation coefficients.
+        y (cupy.ndarray): An additional set of variables and observations.
+        rowvar (bool): If ``True``, then each row represents a variable, with
+            observations in the columns. Otherwise, the relationship is
+            transposed.
+        bias (None): Has no effect, do not use.
+        ddof (None): Has no effect, do not use.
+        dtype: Data type specifier. By default, the return data-type will have
+            at least `numpy.float64` precision.
+
+    Returns:
+        cupy.ndarray: The Pearson product-moment correlation coefficients of
+        the input array.
+
+    .. seealso:: :func:`numpy.corrcoef`
+
+    """
+    if bias is not None or ddof is not None:
+        warnings.warn('bias and ddof have no effect and are deprecated',
+                      DeprecationWarning)
+
+    out = cov(a, y, rowvar, dtype=dtype)
+    try:
+        d = cupy.diag(out)
+    except ValueError:
+        return out / out
+
+    stddev = cupy.sqrt(d.real)
+    out /= stddev[:, None]
+    out /= stddev[None, :]
+
+    cupy.clip(out.real, -1, 1, out=out.real)
+    if cupy.iscomplexobj(out):
+        cupy.clip(out.imag, -1, 1, out=out.imag)
+
+    return out
+
+
+def correlate(a, v, mode='valid'):
+    """Returns the cross-correlation of two 1-dimensional sequences.
+
+    Args:
+        a (cupy.ndarray): first 1-dimensional input.
+        v (cupy.ndarray): second 1-dimensional input.
+        mode (str, optional): `valid`, `same`, `full`
+
+    Returns:
+        cupy.ndarray: Discrete cross-correlation of a and v.
+
+    .. seealso:: :func:`numpy.correlate`
+
+    """
+    if a.size == 0 or v.size == 0:
+        raise ValueError('Array arguments cannot be empty')
+    if a.ndim != 1 or v.ndim != 1:
+        raise ValueError('object too deep for desired array')
+    # choose_conv_method does not choose from the values in
+    # the input array, so no need to apply conj.
+    method = cupy._math.misc._choose_conv_method(a, v, mode)
+    if method == 'direct':
+        out = cupy._math.misc._dot_convolve(a, v.conj()[::-1], mode)
+    elif method == 'fft':
+        out = cupy._math.misc._fft_convolve(a, v.conj()[::-1], mode)
+    else:
+        raise ValueError('Unsupported method')
+    return out
+
+
+def cov(a, y=None, rowvar=True, bias=False, ddof=None,
+        fweights=None, aweights=None, *, dtype=None):
+    """Returns the covariance matrix of an array.
+
+    This function currently does not support ``fweights`` and ``aweights``
+    options.
+
+    Args:
+        a (cupy.ndarray): Array to compute covariance matrix.
+        y (cupy.ndarray): An additional set of variables and observations.
+        rowvar (bool): If ``True``, then each row represents a variable, with
+            observations in the columns. Otherwise, the relationship is
+            transposed.
+        bias (bool): If ``False``, normalization is by ``(N - 1)``, where N is
+            the number of observations given (unbiased estimate). If ``True``,
+            then normalization is by ``N``.
+        ddof (int): If not ``None`` the default value implied by bias is
+            overridden. Note that ``ddof=1`` will return the unbiased estimate
+            and ``ddof=0`` will return the simple average.
+
+        fweights (cupy.ndarray, int): 1-D array of integer frequency weights.
+            the number of times each observation vector should be repeated.
+            It is required that fweights >= 0. However, the function will not
+            error when fweights < 0 for performance reasons.
+        aweights (cupy.ndarray): 1-D array of observation vector weights.
+            These relative weights are typically large for observations
+            considered "important" and smaller for observations considered
+            less "important". If ``ddof=0`` the array of weights can be used
+            to assign probabilities to observation vectors.
+            It is required that aweights >= 0. However, the function will not
+            error when aweights < 0 for performance reasons.
+        dtype: Data type specifier. By default, the return data-type will have
+            at least `numpy.float64` precision.
+
+    Returns:
+        cupy.ndarray: The covariance matrix of the input array.
+
+    .. seealso:: :func:`numpy.cov`
+
+    """
+    if ddof is not None and ddof != int(ddof):
+        raise ValueError('ddof must be integer')
+
+    if a.ndim > 2:
+        raise ValueError('Input must be <= 2-d')
+
+    if dtype is None:
+        if y is None:
+            dtype = numpy.promote_types(a.dtype, numpy.float64)
+        else:
+            if y.ndim > 2:
+                raise ValueError('y must be <= 2-d')
+            dtype = functools.reduce(
+                numpy.promote_types,
+                (a.dtype, y.dtype, numpy.float64)
+            )
+
+    X = cupy.array(a, ndmin=2, dtype=dtype)
+    if not rowvar and X.shape[0] != 1:
+        X = X.T
+    if X.shape[0] == 0:
+        return cupy.array([]).reshape(0, 0)
+    if y is not None:
+        y = cupy.array(y, copy=False, ndmin=2, dtype=dtype)
+        if not rowvar and y.shape[0] != 1:
+            y = y.T
+        X = _core.concatenate_method((X, y), axis=0)
+
+    if ddof is None:
+        ddof = 0 if bias else 1
+
+    w = None
+    if fweights is not None:
+        if not isinstance(fweights, cupy.ndarray):
+            raise TypeError(
+                "fweights must be a cupy.ndarray")
+        if fweights.dtype.char not in 'bBhHiIlLqQ':
+            raise TypeError(
+                "fweights must be integer")
+        fweights = fweights.astype(dtype=float)
+        if fweights.ndim > 1:
+            raise RuntimeError(
+                "cannot handle multidimensional fweights")
+        if fweights.shape[0] != X.shape[1]:
+            raise RuntimeError(
+                "incompatible numbers of samples and fweights")
+        w = fweights
+
+    if aweights is not None:
+        if not isinstance(aweights, cupy.ndarray):
+            raise TypeError(
+                "aweights must be a cupy.ndarray")
+        aweights = aweights.astype(dtype=float)
+        if aweights.ndim > 1:
+            raise RuntimeError(
+                "cannot handle multidimensional aweights")
+        if aweights.shape[0] != X.shape[1]:
+            raise RuntimeError(
+                "incompatible numbers of samples and aweights")
+        if w is None:
+            w = aweights
+        else:
+            w *= aweights
+
+    avg, w_sum = cupy.average(X, axis=1, weights=w, returned=True)
+    w_sum = w_sum[0]
+
+    # Determine the normalization
+    if w is None:
+        fact = X.shape[1] - ddof
+    elif ddof == 0:
+        fact = w_sum
+    elif aweights is None:
+        fact = w_sum - ddof
+    else:
+        fact = w_sum - ddof * sum(w*aweights) / w_sum
+
+    if fact <= 0:
+        warnings.warn('Degrees of freedom <= 0 for slice',
+                      RuntimeWarning, stacklevel=2)
+        fact = 0.0
+
+    X -= X.mean(axis=1)[:, None]
+    if w is None:
+        X_T = X.T
+    else:
+        X_T = (X * w).T
+    out = X.dot(X_T.conj()) * (1 / cupy.float64(fact))
+
+    return out.squeeze()
diff --git a/cupy/_statistics/histogram.py b/cupy/_statistics/histogram.py
new file mode 100644
index 0000000..47eeab0
--- /dev/null
+++ b/cupy/_statistics/histogram.py
@@ -0,0 +1,565 @@
+import operator
+import warnings
+
+import numpy
+
+import cupy
+from cupy import _core
+
+
+# rename builtin range for use in functions that take a range argument
+_range = range
+
+
+# TODO(unno): use searchsorted
+_histogram_kernel = _core.ElementwiseKernel(
+    'S x, raw T bins, int32 n_bins',
+    'raw U y',
+    '''
+    if (x < bins[0] or bins[n_bins - 1] < x) {
+        return;
+    }
+    int high = n_bins - 1;
+    int low = 0;
+
+    while (high - low > 1) {
+        int mid = (high + low) / 2;
+        if (bins[mid] <= x) {
+            low = mid;
+        } else {
+            high = mid;
+        }
+    }
+    atomicAdd(&y[low], U(1));
+    ''',
+    'cupy_histogram_kernel')
+
+
+_weighted_histogram_kernel = _core.ElementwiseKernel(
+    'S x, raw T bins, int32 n_bins, raw W weights',
+    'raw Y y',
+    '''
+    if (x < bins[0] or bins[n_bins - 1] < x) {
+        return;
+    }
+    int high = n_bins - 1;
+    int low = 0;
+
+    while (high - low > 1) {
+        int mid = (high + low) / 2;
+        if (bins[mid] <= x) {
+            low = mid;
+        } else {
+            high = mid;
+        }
+    }
+    atomicAdd(&y[low], (Y)weights[i]);
+    ''',
+    'cupy_weighted_histogram_kernel')
+
+
+def _ravel_and_check_weights(a, weights):
+    """ Check a and weights have matching shapes, and ravel both """
+
+    # Ensure that the array is a "subtractable" dtype
+    if a.dtype == cupy.bool_:
+        warnings.warn('Converting input from {} to {} for compatibility.'
+                      .format(a.dtype, cupy.uint8),
+                      RuntimeWarning, stacklevel=3)
+        a = a.astype(cupy.uint8)
+
+    if weights is not None:
+        if not isinstance(weights, cupy.ndarray):
+            raise ValueError('weights must be a cupy.ndarray')
+        if weights.shape != a.shape:
+            raise ValueError(
+                'weights should have the same shape as a.')
+        weights = weights.ravel()
+    a = a.ravel()
+    return a, weights
+
+
+def _get_outer_edges(a, range):
+    """
+    Determine the outer bin edges to use, from either the data or the range
+    argument
+    """
+    if range is not None:
+        first_edge, last_edge = range
+        if first_edge > last_edge:
+            raise ValueError(
+                'max must be larger than min in range parameter.')
+        if not (numpy.isfinite(first_edge) and numpy.isfinite(last_edge)):
+            raise ValueError(
+                'supplied range of [{}, {}] is not finite'.format(
+                    first_edge, last_edge))
+    elif a.size == 0:
+        first_edge = 0.0
+        last_edge = 1.0
+    else:
+        first_edge = float(a.min())
+        last_edge = float(a.max())
+        if not (cupy.isfinite(first_edge) and cupy.isfinite(last_edge)):
+            raise ValueError(
+                'autodetected range of [{}, {}] is not finite'.format(
+                    first_edge, last_edge))
+
+    # expand empty range to avoid divide by zero
+    if first_edge == last_edge:
+        first_edge = first_edge - 0.5
+        last_edge = last_edge + 0.5
+
+    return first_edge, last_edge
+
+
+def _get_bin_edges(a, bins, range):
+    """
+    Computes the bins used internally by `histogram`.
+
+    Args:
+        a (ndarray): Ravelled data array
+        bins (int or ndarray): Forwarded argument from `histogram`.
+        range (None or tuple): Forwarded argument from `histogram`.
+
+    Returns:
+        bin_edges (ndarray): Array of bin edges
+    """
+    # parse the overloaded bins argument
+    n_equal_bins = None
+    bin_edges = None
+
+    if isinstance(bins, str):
+        raise NotImplementedError(
+            'only integer and array bins are implemented')
+    elif isinstance(bins, cupy.ndarray) or numpy.ndim(bins) == 1:
+        # TODO(okuta): After #3060 is merged, `if cupy.ndim(bins) == 1:`.
+        if isinstance(bins, cupy.ndarray):
+            bin_edges = bins
+        else:
+            bin_edges = numpy.asarray(bins)
+
+        if (bin_edges[:-1] > bin_edges[1:]).any():  # synchronize! when CuPy
+            raise ValueError(
+                '`bins` must increase monotonically, when an array')
+        if isinstance(bin_edges, numpy.ndarray):
+            bin_edges = cupy.asarray(bin_edges)
+    elif numpy.ndim(bins) == 0:
+        try:
+            n_equal_bins = operator.index(bins)
+        except TypeError:
+            raise TypeError(
+                '`bins` must be an integer, a string, or an array')
+        if n_equal_bins < 1:
+            raise ValueError('`bins` must be positive, when an integer')
+
+        first_edge, last_edge = _get_outer_edges(a, range)
+    else:
+        raise ValueError('`bins` must be 1d, when an array')
+
+    if n_equal_bins is not None:
+        # numpy's gh-10322 means that type resolution rules are dependent on
+        # array shapes. To avoid this causing problems, we pick a type now and
+        # stick with it throughout.
+        bin_type = cupy.result_type(first_edge, last_edge, a)
+        if cupy.issubdtype(bin_type, cupy.integer):
+            bin_type = cupy.result_type(bin_type, float)
+
+        # bin edges must be computed
+        bin_edges = cupy.linspace(
+            first_edge, last_edge, n_equal_bins + 1,
+            endpoint=True, dtype=bin_type)
+    return bin_edges
+
+
+def histogram(x, bins=10, range=None, weights=None, density=False):
+    """Computes the histogram of a set of data.
+
+    Args:
+        x (cupy.ndarray): Input array.
+        bins (int or cupy.ndarray): If ``bins`` is an int, it represents the
+            number of bins. If ``bins`` is an :class:`~cupy.ndarray`, it
+            represents a bin edges.
+        range (2-tuple of float, optional): The lower and upper range of the
+            bins.  If not provided, range is simply ``(x.min(), x.max())``.
+            Values outside the range are ignored. The first element of the
+            range must be less than or equal to the second. `range` affects the
+            automatic bin computation as well. While bin width is computed to
+            be optimal based on the actual data within `range`, the bin count
+            will fill the entire range including portions containing no data.
+        density (bool, optional): If False, the default, returns the number of
+            samples in each bin. If True, returns the probability *density*
+            function at the bin, ``bin_count / sample_count / bin_volume``.
+        weights (cupy.ndarray, optional): An array of weights, of the same
+            shape as `x`.  Each value in `x` only contributes its associated
+            weight towards the bin count (instead of 1).
+    Returns:
+        tuple: ``(hist, bin_edges)`` where ``hist`` is a :class:`cupy.ndarray`
+        storing the values of the histogram, and ``bin_edges`` is a
+        :class:`cupy.ndarray` storing the bin edges.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.histogram`
+    """
+
+    if x.dtype.kind == 'c':
+        # TODO(unno): comparison between complex numbers is not implemented
+        raise NotImplementedError('complex number is not supported')
+
+    if not isinstance(x, cupy.ndarray):
+        raise ValueError('x must be a cupy.ndarray')
+
+    x, weights = _ravel_and_check_weights(x, weights)
+    bin_edges = _get_bin_edges(x, bins, range)
+
+    if weights is None:
+        y = cupy.zeros(bin_edges.size - 1, dtype=cupy.int64)
+        # TODO(leofang): we temporarily remove CUB histogram support for now,
+        # see cupy/cupy#7698. When it's ready, revert the commit that checked
+        # in this comment to restore the support.
+        _histogram_kernel(x, bin_edges, bin_edges.size, y)
+    else:
+        simple_weights = (
+            cupy.can_cast(weights.dtype, cupy.float64) or
+            cupy.can_cast(weights.dtype, cupy.complex128)
+        )
+        if not simple_weights:
+            # object dtype such as Decimal are supported in NumPy, but not here
+            raise NotImplementedError(
+                'only weights with dtype that can be cast to float or complex '
+                'are supported')
+        if weights.dtype.kind == 'c':
+            y = cupy.zeros(bin_edges.size - 1, dtype=cupy.complex128)
+            _weighted_histogram_kernel(
+                x, bin_edges, bin_edges.size, weights.real, y.real)
+            _weighted_histogram_kernel(
+                x, bin_edges, bin_edges.size, weights.imag, y.imag)
+        else:
+            if weights.dtype.kind in 'bui':
+                y = cupy.zeros(bin_edges.size - 1, dtype=int)
+            else:
+                y = cupy.zeros(bin_edges.size - 1, dtype=cupy.float64)
+            _weighted_histogram_kernel(
+                x, bin_edges, bin_edges.size, weights, y)
+
+    if density:
+        db = cupy.array(cupy.diff(bin_edges), cupy.float64)
+        return y / db / y.sum(), bin_edges
+    return y, bin_edges
+
+
+def histogramdd(sample, bins=10, range=None, weights=None, density=False):
+    """Compute the multidimensional histogram of some data.
+
+    Args:
+        sample (cupy.ndarray): The data to be histogrammed. (N, D) or (D, N)
+            array
+
+            Note the unusual interpretation of sample when an array_like:
+
+            * When an array, each row is a coordinate in a D-dimensional
+              space - such as ``histogramdd(cupy.array([p1, p2, p3]))``.
+            * When an array_like, each element is the list of values for single
+              coordinate - such as ``histogramdd((X, Y, Z))``.
+
+            The first form should be preferred.
+        bins (int or tuple of int or cupy.ndarray): The bin specification:
+
+            * A sequence of arrays describing the monotonically increasing bin
+              edges along each dimension.
+            * The number of bins for each dimension (nx, ny, ... =bins)
+            * The number of bins for all dimensions (nx=ny=...=bins).
+        range (sequence, optional): A sequence of length D, each an optional
+            (lower, upper) tuple giving the outer bin edges to be used if the
+            edges are not given explicitly in `bins`. An entry of None in the
+            sequence results in the minimum and maximum values being used for
+            the corresponding dimension. The default, None, is equivalent to
+            passing a tuple of D None values.
+        weights (cupy.ndarray): An array of values `w_i` weighing each sample
+            `(x_i, y_i, z_i, ...)`. The values of the returned histogram are
+            equal to the sum of the weights belonging to the samples falling
+            into each bin.
+        density (bool, optional): If False, the default, returns the number of
+            samples in each bin. If True, returns the probability *density*
+            function at the bin, ``bin_count / sample_count / bin_volume``.
+
+    Returns:
+        tuple:
+        H (cupy.ndarray):
+            The multidimensional histogram of sample x. See
+            normed and weights for the different possible semantics.
+        edges (list of cupy.ndarray):
+            A list of D arrays describing the bin
+            edges for each dimension.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.histogramdd`
+    """
+    if isinstance(sample, cupy.ndarray):
+        # Sample is an ND-array.
+        if sample.ndim == 1:
+            sample = sample[:, cupy.newaxis]
+        nsamples, ndim = sample.shape
+    else:
+        sample = cupy.stack(sample, axis=-1)
+        nsamples, ndim = sample.shape
+
+    nbin = numpy.empty(ndim, int)
+    edges = ndim * [None]
+    dedges = ndim * [None]
+    if weights is not None:
+        weights = cupy.asarray(weights)
+
+    try:
+        nbins = len(bins)
+        if nbins != ndim:
+            raise ValueError(
+                'The dimension of bins must be equal to the dimension of the '
+                ' sample x.'
+            )
+    except TypeError:
+        # bins is an integer
+        bins = ndim * [bins]
+
+    # normalize the range argument
+    if range is None:
+        range = (None,) * ndim
+    elif len(range) != ndim:
+        raise ValueError('range argument must have one entry per dimension')
+
+    # Create edge arrays
+    for i in _range(ndim):
+        if cupy.ndim(bins[i]) == 0:
+            if bins[i] < 1:
+                raise ValueError(
+                    '`bins[{}]` must be positive, when an integer'.format(i)
+                )
+            smin, smax = _get_outer_edges(sample[:, i], range[i])
+            num = int(bins[i] + 1)  # synchronize!
+            edges[i] = cupy.linspace(smin, smax, num)
+        elif cupy.ndim(bins[i]) == 1:
+            if not isinstance(bins[i], cupy.ndarray):
+                raise ValueError('array-like bins not supported')
+            edges[i] = bins[i]
+            if (edges[i][:-1] > edges[i][1:]).any():  # synchronize!
+                raise ValueError(
+                    '`bins[{}]` must be monotonically increasing, when an '
+                    'array'.format(i)
+                )
+        else:
+            raise ValueError(
+                '`bins[{}]` must be a scalar or 1d array'.format(i)
+            )
+
+        nbin[i] = len(edges[i]) + 1  # includes an outlier on each end
+        dedges[i] = cupy.diff(edges[i])
+
+    # Compute the bin number each sample falls into.
+    ncount = tuple(
+        # avoid cupy.digitize to work around NumPy issue gh-11022
+        cupy.searchsorted(edges[i], sample[:, i], side='right')
+        for i in _range(ndim)
+    )
+
+    # Using digitize, values that fall on an edge are put in the right bin.
+    # For the rightmost bin, we want values equal to the right edge to be
+    # counted in the last bin, and not as an outlier.
+    for i in _range(ndim):
+        # Find which points are on the rightmost edge.
+        on_edge = sample[:, i] == edges[i][-1]
+        # Shift these points one bin to the left.
+        ncount[i][on_edge] -= 1
+
+    # Compute the sample indices in the flattened histogram matrix.
+    # This raises an error if the array is too large.
+    xy = cupy.ravel_multi_index(ncount, nbin)
+
+    # Compute the number of repetitions in xy and assign it to the
+    # flattened histmat.
+    hist = cupy.bincount(xy, weights, minlength=numpy.prod(nbin))
+
+    # Shape into a proper matrix
+    hist = hist.reshape(nbin)
+
+    # This preserves the (bad) behavior observed in NumPy gh-7845, for now.
+    hist = hist.astype(float)  # Note: NumPy uses casting='safe' here too
+
+    # Remove outliers (indices 0 and -1 for each dimension).
+    core = ndim * (slice(1, -1),)
+    hist = hist[core]
+
+    if density:
+        # calculate the probability density function
+        s = hist.sum()
+        for i in _range(ndim):
+            shape = [1] * ndim
+            shape[i] = nbin[i] - 2
+            hist = hist / dedges[i].reshape(shape)
+        hist /= s
+
+    if any(hist.shape != numpy.asarray(nbin) - 2):
+        raise RuntimeError('Internal Shape Error')
+    return hist, edges
+
+
+def histogram2d(x, y, bins=10, range=None, weights=None, density=None):
+    """Compute the bi-dimensional histogram of two data samples.
+
+    Args:
+        x (cupy.ndarray): The first array of samples to be histogrammed.
+        y (cupy.ndarray): The second array of samples to be histogrammed.
+        bins (int or tuple of int or cupy.ndarray): The bin specification:
+
+            * A sequence of arrays describing the monotonically increasing bin
+              edges along each dimension.
+            * The number of bins for each dimension (nx, ny)
+            * The number of bins for all dimensions (nx=ny=bins).
+        range (sequence, optional): A sequence of length two, each an optional
+            (lower, upper) tuple giving the outer bin edges to be used if the
+            edges are not given explicitly in `bins`. An entry of None in the
+            sequence results in the minimum and maximum values being used for
+            the corresponding dimension. The default, None, is equivalent to
+            passing a tuple of two None values.
+        weights (cupy.ndarray): An array of values `w_i` weighing each sample
+            `(x_i, y_i)`. The values of the returned histogram are equal to the
+            sum of the weights belonging to the samples falling into each bin.
+        density (bool, optional): If False, the default, returns the number of
+            samples in each bin. If True, returns the probability *density*
+            function at the bin, ``bin_count / sample_count / bin_volume``.
+
+    Returns:
+        tuple:
+        H (cupy.ndarray):
+            The multidimensional histogram of sample x. See
+            normed and weights for the different possible semantics.
+        edges0 (tuple of cupy.ndarray):
+            A list of D arrays describing the bin
+            edges for the first dimension.
+        edges1 (tuple of cupy.ndarray):
+            A list of D arrays describing the bin
+            edges for the second dimension.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.histogram2d`
+    """
+    try:
+        n = len(bins)
+    except TypeError:
+        n = 1
+
+    if n != 1 and n != 2:
+        if isinstance(bins, cupy.ndarray):
+            xedges = yedges = bins
+            bins = [xedges, yedges]
+        else:
+            raise ValueError('array-like bins not supported in CuPy')
+
+    hist, edges = histogramdd([x, y], bins, range, weights, density)
+    return hist, edges[0], edges[1]
+
+
+_bincount_kernel = _core.ElementwiseKernel(
+    'S x', 'raw U bin',
+    'atomicAdd(&bin[x], U(1))',
+    'cupy_bincount_kernel')
+
+
+_bincount_with_weight_kernel = _core.ElementwiseKernel(
+    'S x, T w', 'raw U bin',
+    'atomicAdd(&bin[x], w)',
+    'cupy_bincount_with_weight_kernel')
+
+
+def bincount(x, weights=None, minlength=None):
+    """Count number of occurrences of each value in array of non-negative ints.
+
+    Args:
+        x (cupy.ndarray): Input array.
+        weights (cupy.ndarray): Weights array which has the same shape as
+            ``x``.
+        minlength (int): A minimum number of bins for the output array.
+
+    Returns:
+        cupy.ndarray: The result of binning the input array. The length of
+        output is equal to ``max(cupy.max(x) + 1, minlength)``.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.bincount`
+
+    """
+    if x.ndim > 1:
+        raise ValueError('object too deep for desired array')
+    if x.ndim < 1:
+        raise ValueError('object of too small depth for desired array')
+    if x.dtype.kind == 'f':
+        raise TypeError('x must be int array')
+    if (x < 0).any():  # synchronize!
+        raise ValueError('The first argument of bincount must be non-negative')
+    if weights is not None and x.shape != weights.shape:
+        raise ValueError('The weights and list don\'t have the same length.')
+    if minlength is not None:
+        minlength = int(minlength)
+        if minlength < 0:
+            raise ValueError('minlength must be non-negative')
+
+    size = int(cupy.max(x)) + 1  # synchronize!
+    if minlength is not None:
+        size = max(size, minlength)
+
+    if weights is None:
+        b = cupy.zeros((size,), dtype=numpy.intp)
+        # TODO(leofang): we temporarily remove CUB histogram support for now,
+        # see cupy/cupy#7698. When it's ready, revert the commit that checked
+        # in this comment to restore the support.
+        _bincount_kernel(x, b)
+    else:
+        b = cupy.zeros((size,), dtype=numpy.float64)
+        _bincount_with_weight_kernel(x, weights, b)
+
+    return b
+
+
+def digitize(x, bins, right=False):
+    """Finds the indices of the bins to which each value in input array belongs.
+
+    .. note::
+
+        In order to avoid device synchronization, digitize does not raise
+        an exception when the array is not monotonic
+
+    Args:
+        x (cupy.ndarray): Input array.
+        bins (cupy.ndarray): Array of bins.
+            It has to be 1-dimensional and monotonic increasing or decreasing.
+        right (bool):
+            Indicates whether the intervals include the right or the left bin
+            edge.
+
+    Returns:
+        cupy.ndarray: Output array of indices, of same shape as ``x``.
+
+    .. seealso:: :func:`numpy.digitize`
+    """  # NOQA
+    # This is for NumPy compat, although it works fine
+    if x.dtype.kind == 'c':
+        raise TypeError('x may not be complex')
+
+    if bins.ndim > 1:
+        raise ValueError('object too deep for desired array')
+    if bins.ndim < 1:
+        raise ValueError('object of too small depth for desired array')
+
+    # As the order of the arguments are reversed, the side must be too.
+    side = 'left' if right else 'right'
+    return cupy._sorting.search._searchsorted(bins, x, side, None, False)
diff --git a/cupy/_statistics/meanvar.py b/cupy/_statistics/meanvar.py
new file mode 100644
index 0000000..ac8070d
--- /dev/null
+++ b/cupy/_statistics/meanvar.py
@@ -0,0 +1,288 @@
+import functools
+import numpy
+
+import cupy
+from cupy._core import _routines_statistics as _statistics
+
+
+def median(a, axis=None, out=None, overwrite_input=False, keepdims=False):
+    """Compute the median along the specified axis.
+
+    Returns the median of the array elements.
+
+    Args:
+        a (cupy.ndarray): Array to compute the median.
+        axis (int, sequence of int or None): Axis along which the medians are
+             computed. The flattened array is used by default.
+        out (cupy.ndarray): Output array.
+        overwrite_input (bool): If ``True``, then allow use of memory of input
+            array a for calculations. The input array will be modified by the
+            call to median. This will save memory when you do not need to
+            preserve the contents of the input array. Treat the input as
+            undefined, but it will probably be fully or partially sorted.
+            Default is ``False``. If ``overwrite_input`` is ``True`` and ``a``
+            is not already an ndarray, an error will be raised.
+        keepdims (bool): If ``True``, the axis is remained as an axis of size
+            one.
+
+    Returns:
+        cupy.ndarray: The median of ``a``, along the axis if specified.
+
+    .. seealso:: :func:`numpy.median`
+
+    """
+    return _statistics._median(a, axis, out, overwrite_input, keepdims)
+
+
+def nanmedian(a, axis=None, out=None, overwrite_input=False, keepdims=False):
+    """Compute the median along the specified axis, while ignoring NaNs.
+
+    Returns the median of the array elements.
+
+    Args:
+        a (cupy.ndarray): Array to compute the median.
+        axis (int, sequence of int or None): Axis along which the medians are
+            computed. The flattened array is used by default.
+        out (cupy.ndarray): Output array.
+        overwrite_input (bool): If ``True``, then allow use of memory of input
+            array a for calculations. The input array will be modified by the
+            call to median. This will save memory when you do not need to
+            preserve the contents of the input array. Treat the input as
+            undefined, but it will probably be fully or partially sorted.
+            Default is ``False``. If ``overwrite_input`` is ``True`` and ``a``
+            is not already an ndarray, an error will be raised.
+        keepdims (bool): If ``True``, the axis is remained as an axis of size
+            one.
+
+    Returns:
+        cupy.ndarray: The median of ``a``, along the axis if specified.
+
+    .. seealso:: :func:`numpy.nanmedian`
+
+    """
+    if a.dtype.char in 'efdFD':
+        return _statistics._nanmedian(a, axis, out, overwrite_input, keepdims)
+    else:
+        return median(a, axis=axis, out=out, overwrite_input=overwrite_input,
+                      keepdims=keepdims)
+
+
+def average(a, axis=None, weights=None, returned=False, *, keepdims=False):
+    """Returns the weighted average along an axis.
+
+    Args:
+        a (cupy.ndarray): Array to compute average.
+        axis (int): Along which axis to compute average. The flattened array
+            is used by default.
+        weights (cupy.ndarray): Array of weights where each element
+            corresponds to the value in ``a``. If ``None``, all the values
+            in ``a`` have a weight equal to one.
+        returned (bool): If ``True``, a tuple of the average and the sum
+            of weights is returned, otherwise only the average is returned.
+        keepdims (bool): If ``True``, the axis is remained as an axis of size
+            one.
+
+    Returns:
+        cupy.ndarray or tuple of cupy.ndarray: The average of the input array
+        along the axis and the sum of weights.
+
+    .. warning::
+
+        This function may synchronize the device if ``weight`` is given.
+
+    .. seealso:: :func:`numpy.average`
+    """
+    # TODO(niboshi): Avoid synchronization.
+    a = cupy.asarray(a)
+
+    if weights is None:
+        avg = a.mean(axis=axis, keepdims=keepdims)
+        scl = avg.dtype.type(a.size / avg.size)
+    else:
+        wgt = cupy.asarray(weights)
+
+        if issubclass(a.dtype.type, (numpy.integer, numpy.bool_)):
+            result_dtype = functools.reduce(numpy.promote_types,
+                                            (a.dtype, wgt.dtype, 'f8'))
+        else:
+            result_dtype = numpy.promote_types(a.dtype, wgt.dtype)
+
+        # Sanity checks
+        if a.shape != wgt.shape:
+            if axis is None:
+                raise TypeError(
+                    'Axis must be specified when shapes of a and weights '
+                    'differ.')
+            if wgt.ndim != 1:
+                raise TypeError(
+                    '1D weights expected when shapes of a and weights differ.')
+            if wgt.shape[0] != a.shape[axis]:
+                raise ValueError(
+                    'Length of weights not compatible with specified axis.')
+
+            # setup wgt to broadcast along axis
+            wgt = cupy.broadcast_to(wgt, (a.ndim - 1) * (1,) + wgt.shape)
+            wgt = wgt.swapaxes(-1, axis)
+
+        scl = wgt.sum(axis=axis, dtype=result_dtype, keepdims=keepdims)
+        if cupy.any(scl == 0.0):  # synchronize!
+            raise ZeroDivisionError(
+                'Weights sum to zero, can\'t be normalized')
+
+        avg = cupy.multiply(a, wgt, dtype=result_dtype).sum(
+            axis, keepdims=keepdims) / scl
+
+    if returned:
+        if scl.shape != avg.shape:
+            scl = cupy.broadcast_to(cupy.array(scl), avg.shape).copy()
+        return avg, scl
+    else:
+        return avg
+
+
+def mean(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Returns the arithmetic mean along an axis.
+
+    Args:
+        a (cupy.ndarray): Array to compute mean.
+        axis (int, sequence of int or None): Along which axis to compute mean.
+            The flattened array is used by default.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The mean of the input array along the axis.
+
+    .. seealso:: :func:`numpy.mean`
+
+    """
+    # TODO(okuta): check type
+    return a.mean(axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
+    """Returns the variance along an axis.
+
+    Args:
+        a (cupy.ndarray): Array to compute variance.
+        axis (int): Along which axis to compute variance. The flattened array
+            is used by default.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The variance of the input array along the axis.
+
+    .. seealso:: :func:`numpy.var`
+
+    """
+    # TODO(okuta): check type
+    return a.var(axis=axis, dtype=dtype, out=out, ddof=ddof,
+                 keepdims=keepdims)
+
+
+def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
+    """Returns the standard deviation along an axis.
+
+    Args:
+        a (cupy.ndarray): Array to compute standard deviation.
+        axis (int): Along which axis to compute standard deviation. The
+            flattened array is used by default.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The standard deviation of the input array along the axis.
+
+    .. seealso:: :func:`numpy.std`
+
+    """
+    # TODO(okuta): check type
+    return a.std(axis=axis, dtype=dtype, out=out, ddof=ddof,
+                 keepdims=keepdims)
+
+
+def nanmean(a, axis=None, dtype=None, out=None, keepdims=False):
+    """Returns the arithmetic mean along an axis ignoring NaN values.
+
+    Args:
+        a (cupy.ndarray): Array to compute mean.
+        axis (int, sequence of int or None): Along which axis to compute mean.
+            The flattened array is used by default.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The mean of the input array along the axis ignoring NaNs.
+
+    .. seealso:: :func:`numpy.nanmean`
+
+    """
+    if a.dtype.kind in 'biu':
+        return a.mean(axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+    # TODO(okuta): check type
+    return _statistics._nanmean(
+        a, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
+    """Returns the variance along an axis ignoring NaN values.
+
+    Args:
+        a (cupy.ndarray): Array to compute variance.
+        axis (int): Along which axis to compute variance. The flattened array
+            is used by default.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The variance of the input array along the axis.
+
+    .. seealso:: :func:`numpy.nanvar`
+
+    """
+    if a.dtype.kind in 'biu':
+        return a.var(axis=axis, dtype=dtype, out=out, ddof=ddof,
+                     keepdims=keepdims)
+
+    # TODO(okuta): check type
+    return _statistics._nanvar(
+        a, axis=axis, dtype=dtype, out=out, ddof=ddof, keepdims=keepdims)
+
+
+def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
+    """Returns the standard deviation along an axis ignoring NaN values.
+
+    Args:
+        a (cupy.ndarray): Array to compute standard deviation.
+        axis (int): Along which axis to compute standard deviation. The
+            flattened array is used by default.
+        dtype: Data type specifier.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The standard deviation of the input array along the axis.
+
+    .. seealso:: :func:`numpy.nanstd`
+
+    """
+    if a.dtype.kind in 'biu':
+        return a.std(axis=axis, dtype=dtype, out=out, ddof=ddof,
+                     keepdims=keepdims)
+
+    # TODO(okuta): check type
+    return _statistics._nanstd(
+        a, axis=axis, dtype=dtype, out=out, ddof=ddof, keepdims=keepdims)
diff --git a/cupy/_statistics/order.py b/cupy/_statistics/order.py
new file mode 100644
index 0000000..afe3985
--- /dev/null
+++ b/cupy/_statistics/order.py
@@ -0,0 +1,401 @@
+import warnings
+
+import cupy
+from cupy import _core
+from cupy._core import _routines_statistics as _statistics
+from cupy._core import _fusion_thread_local
+from cupy._logic import content
+
+
+def amin(a, axis=None, out=None, keepdims=False):
+    """Returns the minimum of an array or the minimum along an axis.
+
+    .. note::
+
+       When at least one element is NaN, the corresponding min value will be
+       NaN.
+
+    Args:
+        a (cupy.ndarray): Array to take the minimum.
+        axis (int): Along which axis to take the minimum. The flattened array
+            is used by default.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The minimum of ``a``, along the axis if specified.
+
+    .. note::
+       When cuTENSOR accelerator is used, the output value might be collapsed
+       for reduction axes that have one or more NaN elements.
+
+    .. seealso:: :func:`numpy.amin`
+
+    """
+    if _fusion_thread_local.is_fusing():
+        if keepdims:
+            raise NotImplementedError(
+                'cupy.amin does not support `keepdims` in fusion yet.')
+        return _fusion_thread_local.call_reduction(
+            _statistics.amin, a, axis=axis, out=out)
+
+    # TODO(okuta): check type
+    return a.min(axis=axis, out=out, keepdims=keepdims)
+
+
+def amax(a, axis=None, out=None, keepdims=False):
+    """Returns the maximum of an array or the maximum along an axis.
+
+    .. note::
+
+       When at least one element is NaN, the corresponding min value will be
+       NaN.
+
+    Args:
+        a (cupy.ndarray): Array to take the maximum.
+        axis (int): Along which axis to take the maximum. The flattened array
+            is used by default.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The maximum of ``a``, along the axis if specified.
+
+    .. note::
+       When cuTENSOR accelerator is used, the output value might be collapsed
+       for reduction axes that have one or more NaN elements.
+
+    .. seealso:: :func:`numpy.amax`
+
+    """
+    if _fusion_thread_local.is_fusing():
+        if keepdims:
+            raise NotImplementedError(
+                'cupy.amax does not support `keepdims` in fusion yet.')
+        return _fusion_thread_local.call_reduction(
+            _statistics.amax, a, axis=axis, out=out)
+
+    # TODO(okuta): check type
+    return a.max(axis=axis, out=out, keepdims=keepdims)
+
+
+def nanmin(a, axis=None, out=None, keepdims=False):
+    """Returns the minimum of an array along an axis ignoring NaN.
+
+    When there is a slice whose elements are all NaN, a :class:`RuntimeWarning`
+    is raised and NaN is returned.
+
+    Args:
+        a (cupy.ndarray): Array to take the minimum.
+        axis (int): Along which axis to take the minimum. The flattened array
+            is used by default.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The minimum of ``a``, along the axis if specified.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.nanmin`
+
+    """
+    # TODO(niboshi): Avoid synchronization.
+    res = _core.nanmin(a, axis=axis, out=out, keepdims=keepdims)
+    if content.isnan(res).any():  # synchronize!
+        warnings.warn('All-NaN slice encountered', RuntimeWarning)
+    return res
+
+
+def nanmax(a, axis=None, out=None, keepdims=False):
+    """Returns the maximum of an array along an axis ignoring NaN.
+
+    When there is a slice whose elements are all NaN, a :class:`RuntimeWarning`
+    is raised and NaN is returned.
+
+    Args:
+        a (cupy.ndarray): Array to take the maximum.
+        axis (int): Along which axis to take the maximum. The flattened array
+            is used by default.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The maximum of ``a``, along the axis if specified.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`numpy.nanmax`
+
+    """
+    # TODO(niboshi): Avoid synchronization.
+    res = _core.nanmax(a, axis=axis, out=out, keepdims=keepdims)
+    if content.isnan(res).any():  # synchronize!
+        warnings.warn('All-NaN slice encountered', RuntimeWarning)
+    return res
+
+
+def ptp(a, axis=None, out=None, keepdims=False):
+    """Returns the range of values (maximum - minimum) along an axis.
+
+    .. note::
+
+       The name of the function comes from the acronym for 'peak to peak'.
+
+       When at least one element is NaN, the corresponding ptp value will be
+       NaN.
+
+    Args:
+        a (cupy.ndarray): Array over which to take the range.
+        axis (int): Axis along which to take the minimum. The flattened
+            array is used by default.
+        out (cupy.ndarray): Output array.
+        keepdims (bool): If ``True``, the axis is retained as an axis of
+            size one.
+
+    Returns:
+        cupy.ndarray: The minimum of ``a``, along the axis if specified.
+
+    .. note::
+       When cuTENSOR accelerator is used, the output value might be collapsed
+       for reduction axes that have one or more NaN elements.
+
+    .. seealso:: :func:`numpy.amin`
+
+    """
+    return a.ptp(axis=axis, out=out, keepdims=keepdims)
+
+
+def _quantile_unchecked(a, q, axis=None, out=None,
+                        overwrite_input=False,
+                        method='linear',
+                        keepdims=False):
+    if q.ndim == 0:
+        q = q[None]
+        zerod = True
+    else:
+        zerod = False
+    if q.ndim > 1:
+        raise ValueError('Expected q to have a dimension of 1.\n'
+                         'Actual: {0} != 1'.format(q.ndim))
+    if keepdims:
+        if axis is None:
+            keepdim = (1,) * a.ndim
+        else:
+            keepdim = list(a.shape)
+            for ax in axis:
+                keepdim[ax % a.ndim] = 1
+            keepdim = tuple(keepdim)
+
+    if isinstance(axis, int):
+        axis = axis,
+    if axis is None:
+        if overwrite_input:
+            ap = a.ravel()
+        else:
+            ap = a.flatten()
+        nkeep = 0
+    else:
+        # Reduce axes from a and put them last
+        axis = tuple(ax % a.ndim for ax in axis)
+        keep = set(range(a.ndim)) - set(axis)
+        nkeep = len(keep)
+        for i, s in enumerate(sorted(keep)):
+            a = a.swapaxes(i, s)
+        if overwrite_input:
+            ap = a.reshape(a.shape[:nkeep] + (-1,))
+        else:
+            ap = a.reshape(a.shape[:nkeep] + (-1,)).copy()
+
+    axis = -1
+    ap.sort(axis=axis)
+    Nx = ap.shape[axis]
+    indices = q * (Nx - 1.)
+
+    if method in ['inverted_cdf', 'averaged_inverted_cdf',
+                  'closest_observation', 'interpolated_inverted_cdf',
+                  'hazen', 'weibull', 'median_unbiased', 'normal_unbiased']:
+        # TODO(takagi) Implement new methods introduced in NumPy 1.22
+        raise ValueError(f'\'{method}\' method is not yet supported. '
+                         'Please use any other method.')
+    elif method == 'lower':
+        indices = cupy.floor(indices).astype(cupy.int32)
+    elif method == 'higher':
+        indices = cupy.ceil(indices).astype(cupy.int32)
+    elif method == 'midpoint':
+        indices = 0.5 * (cupy.floor(indices) + cupy.ceil(indices))
+    elif method == 'nearest':
+        # TODO(hvy): Implement nearest using around
+        raise ValueError('\'nearest\' method is not yet supported. '
+                         'Please use any other method.')
+    elif method == 'linear':
+        pass
+    else:
+        raise ValueError('Unexpected interpolation method.\n'
+                         'Actual: \'{0}\' not in (\'linear\', \'lower\', '
+                         '\'higher\', \'midpoint\')'.format(method))
+
+    if indices.dtype == cupy.int32:
+        ret = cupy.rollaxis(ap, axis)
+        ret = ret.take(indices, axis=0, out=out)
+    else:
+        if out is None:
+            ret = cupy.empty(ap.shape[:-1] + q.shape, dtype=cupy.float64)
+        else:
+            ret = cupy.rollaxis(out, 0, out.ndim)
+
+        cupy.ElementwiseKernel(
+            'S idx, raw T a, raw int32 offset, raw int32 size', 'U ret',
+            '''
+            ptrdiff_t idx_below = floor(idx);
+            U weight_above = idx - idx_below;
+
+            ptrdiff_t max_idx = size - 1;
+            ptrdiff_t offset_bottom = _ind.get()[0] * offset + idx_below;
+            ptrdiff_t offset_top = min(offset_bottom + 1, max_idx);
+
+            U diff = a[offset_top] - a[offset_bottom];
+
+            if (weight_above < 0.5) {
+                ret = a[offset_bottom] + diff * weight_above;
+            } else {
+                ret = a[offset_top] - diff * (1 - weight_above);
+            }
+            ''',
+            'cupy_percentile_weightnening'
+        )(indices, ap, ap.shape[-1] if ap.ndim > 1 else 0, ap.size, ret)
+        ret = cupy.rollaxis(ret, -1)  # Roll q dimension back to first axis
+
+    if zerod:
+        ret = ret.squeeze(0)
+    if keepdims:
+        if q.size > 1:
+            keepdim = (-1,) + keepdim
+        ret = ret.reshape(keepdim)
+
+    return _core._internal_ascontiguousarray(ret)
+
+
+def _quantile_is_valid(q):
+    if cupy.count_nonzero(q < 0.0) or cupy.count_nonzero(q > 1.0):
+        return False
+    return True
+
+
+def percentile(a, q, axis=None, out=None,
+               overwrite_input=False,
+               method='linear',
+               keepdims=False,
+               *,
+               interpolation=None):
+    """Computes the q-th percentile of the data along the specified axis.
+
+    Args:
+        a (cupy.ndarray): Array for which to compute percentiles.
+        q (float, tuple of floats or cupy.ndarray): Percentiles to compute
+            in the range between 0 and 100 inclusive.
+        axis (int or tuple of ints): Along which axis or axes to compute the
+            percentiles. The flattened array is used by default.
+        out (cupy.ndarray): Output array.
+        overwrite_input (bool): If True, then allow the input array `a`
+            to be modified by the intermediate calculations, to save
+            memory. In this case, the contents of the input `a` after this
+            function completes is undefined.
+        method (str): Interpolation method when a quantile lies between
+            two data points. ``linear`` interpolation is used by default.
+            Supported interpolations are``lower``, ``higher``, ``midpoint``,
+            ``nearest`` and ``linear``.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+        interpolation (str): Deprecated name for the method keyword argument.
+
+    Returns:
+        cupy.ndarray: The percentiles of ``a``, along the axis if specified.
+
+    .. seealso:: :func:`numpy.percentile`
+    """
+    if interpolation is not None:
+        method = _check_interpolation_as_method(
+            method, interpolation, 'percentile')
+    if not isinstance(q, cupy.ndarray):
+        q = cupy.asarray(q, dtype='d')
+    q = cupy.true_divide(q, 100)
+    if not _quantile_is_valid(q):  # synchronize
+        raise ValueError('Percentiles must be in the range [0, 100]')
+    return _quantile_unchecked(
+        a, q, axis=axis, out=out,
+        overwrite_input=overwrite_input,
+        method=method,
+        keepdims=keepdims)
+
+
+def quantile(a, q, axis=None, out=None,
+             overwrite_input=False,
+             method='linear',
+             keepdims=False,
+             *,
+             interpolation=None):
+    """Computes the q-th quantile of the data along the specified axis.
+
+    Args:
+        a (cupy.ndarray): Array for which to compute quantiles.
+        q (float, tuple of floats or cupy.ndarray): Quantiles to compute
+            in the range between 0 and 1 inclusive.
+        axis (int or tuple of ints): Along which axis or axes to compute the
+            quantiles. The flattened array is used by default.
+        out (cupy.ndarray): Output array.
+        overwrite_input (bool): If True, then allow the input array `a`
+            to be modified by the intermediate calculations, to save
+            memory. In this case, the contents of the input `a` after this
+            function completes is undefined.
+        method (str): Interpolation method when a quantile lies between
+            two data points. ``linear`` interpolation is used by default.
+            Supported interpolations are``lower``, ``higher``, ``midpoint``,
+            ``nearest`` and ``linear``.
+        keepdims (bool): If ``True``, the axis is remained as an axis of
+            size one.
+        interpolation (str): Deprecated name for the method keyword argument.
+
+    Returns:
+        cupy.ndarray: The quantiles of ``a``, along the axis if specified.
+
+    .. seealso:: :func:`numpy.quantile`
+    """
+    if interpolation is not None:
+        method = _check_interpolation_as_method(
+            method, interpolation, 'quantile')
+    if not isinstance(q, cupy.ndarray):
+        q = cupy.asarray(q, dtype='d')
+    if not _quantile_is_valid(q):  # synchronize
+        raise ValueError('Quantiles must be in the range [0, 1]')
+    return _quantile_unchecked(
+        a, q, axis=axis, out=out,
+        overwrite_input=overwrite_input,
+        method=method,
+        keepdims=keepdims)
+
+
+# Borrowd from NumPy
+def _check_interpolation_as_method(method, interpolation, fname):
+    # Deprecated NumPy 1.22, 2021-11-08
+    warnings.warn(
+        f"the `interpolation=` argument to {fname} was renamed to "
+        "`method=`, which has additional options.\n"
+        "Users of the modes 'nearest', 'lower', 'higher', or "
+        "'midpoint' are encouraged to review the method they. "
+        "(Deprecated NumPy 1.22)",
+        DeprecationWarning, stacklevel=3)
+    if method != "linear":
+        # sanity check, we assume this basically never happens
+        raise TypeError(
+            "You shall not pass both `method` and `interpolation`!\n"
+            "(`interpolation` is Deprecated in favor of `method`)")
+    return interpolation
diff --git a/cupy/_util.pyi b/cupy/_util.pyi
new file mode 100644
index 0000000..e69de29
diff --git a/cupy/_util.pyx b/cupy/_util.pyx
new file mode 100644
index 0000000..2190429
--- /dev/null
+++ b/cupy/_util.pyx
@@ -0,0 +1,218 @@
+# distutils: language = c++
+
+import atexit
+import functools
+import os
+import warnings
+
+import cupy
+from cupy.cuda cimport device
+
+
+DEF CYTHON_BUILD_VER = CUPY_CYTHON_VERSION
+cython_build_ver = CYTHON_BUILD_VER
+
+
+ENABLE_SLICE_COPY = bool(
+    int(os.environ.get('CUPY_EXPERIMENTAL_SLICE_COPY', 0)))
+
+
+CUDA_ARRAY_INTERFACE_SYNC = bool(
+    int(os.environ.get('CUPY_CUDA_ARRAY_INTERFACE_SYNC', 1)))
+CUDA_ARRAY_INTERFACE_EXPORT_VERSION = int(
+    os.environ.get('CUPY_CUDA_ARRAY_INTERFACE_EXPORT_VERSION', 3))
+
+
+DLPACK_EXPORT_VERSION = tuple(
+    [
+        int(x)
+        for x in os.environ.get('CUPY_DLPACK_EXPORT_VERSION', '0.6').split('.')
+    ]
+)
+
+
+cdef list _memos = []
+
+
+def memoize(bint for_each_device=False):
+    """Makes a function memoizing the result for each argument and device.
+
+    This decorator provides automatic memoization of the function result.
+
+    Args:
+        for_each_device (bool): If ``True``, it memoizes the results for each
+            device. Otherwise, it memoizes the results only based on the
+            arguments.
+
+    """
+    def decorator(f):
+        memo = {}
+        _memos.append(memo)
+
+        @functools.wraps(f)
+        def ret(*args, **kwargs):
+            cdef int id = -1
+            cdef dict m = memo
+            if for_each_device:
+                id = device.get_device_id()
+            if len(kwargs):
+                arg_key = (id, args, frozenset(kwargs.items()))
+            else:
+                arg_key = (id, args)
+            result = m.get(arg_key, m)
+            if result is m:
+                result = f(*args, **kwargs)
+                m[arg_key] = result
+            return result
+
+        return ret
+
+    return decorator
+
+
+@atexit.register
+def clear_memo():
+    """Clears the memoized results for all functions decorated by memoize."""
+    for memo in _memos:
+        memo.clear()
+
+
+def experimental(api_name):
+    """Declares that user is using an experimental feature.
+
+    The developer of an API can mark it as *experimental* by calling
+    this function. When users call experimental APIs, :class:`FutureWarning`
+    is issued.
+    The presentation of :class:`FutureWarning` is disabled by setting
+    ``cupy.disable_experimental_warning`` to ``True``,
+    which is ``False`` by default.
+
+    The basic usage is to call it in the function or method we want to
+    mark as experimental along with the API name.
+
+    .. testsetup::
+
+        import sys
+        import warnings
+
+        warnings.simplefilter('always')
+
+        def wrapper(message, category, filename, lineno, file=None, line=None):
+            sys.stdout.write(warnings.formatwarning(
+                message, category, filename, lineno))
+
+        showwarning_orig = warnings.showwarning
+        warnings.showwarning = wrapper
+
+    .. testcleanup::
+
+        warnings.showwarning = showwarning_orig
+
+    .. testcode::
+
+        from cupy import _util
+
+        def f(x):
+            _util.experimental('cupy.foo.bar.f')
+            # concrete implementation of f follows
+
+        f(1)
+
+    .. testoutput::
+        :options: +ELLIPSIS, +NORMALIZE_WHITESPACE
+
+        ... FutureWarning: cupy.foo.bar.f is experimental. \
+The interface can change in the future. ...
+
+    We can also make a whole class experimental. In that case,
+    we should call this function in its ``__init__`` method.
+
+    .. testcode::
+
+        class C():
+            def __init__(self):
+              _util.experimental('cupy.foo.C')
+
+        C()
+
+    .. testoutput::
+        :options: +ELLIPSIS, +NORMALIZE_WHITESPACE
+
+        ... FutureWarning: cupy.foo.C is experimental. \
+The interface can change in the future. ...
+
+    If we want to mark ``__init__`` method only, rather than class itself,
+    it is recommended that we explicitly feed its API name.
+
+    .. testcode::
+
+        class D():
+            def __init__(self):
+                _util.experimental('D.__init__')
+
+        D()
+
+    .. testoutput::
+        :options: +ELLIPSIS, +NORMALIZE_WHITESPACE
+
+        ...  FutureWarning: D.__init__ is experimental. \
+The interface can change in the future. ...
+
+    Currently, we do not have any sophisticated way to mark some usage of
+    non-experimental function as experimental.
+    But we can support such usage by explicitly branching it.
+
+    .. testcode::
+
+        def g(x, experimental_arg=None):
+            if experimental_arg is not None:
+                _util.experimental('experimental_arg of cupy.foo.g')
+
+    Args:
+        api_name(str): The name of an API marked as experimental.
+    """
+
+    if not cupy.disable_experimental_feature_warning:
+        warnings.warn('{} is experimental. '
+                      'The interface can change in the future.'.format(
+                          api_name),
+                      FutureWarning)
+
+
+class PerformanceWarning(RuntimeWarning):
+    """Warning that indicates possible performance issues."""
+
+
+def check_array(obj, *, arg_name):
+    """Checks if the given object is an array.
+
+    This function raises :class:`TypeError` if ``obj`` is not an instance
+    of :type:`cupy.ndarray`\\ .
+    """
+    if not isinstance(obj, cupy.ndarray):
+        raise TypeError(
+            '\'{}\' must be a cupy.ndarray object, not {}.'.format(
+                arg_name, type(obj)))
+
+
+# This code is to signal when the interpreter is in shutdown mode
+# to prevent using globals that could be already deleted in
+# objects `__del__` method
+#
+# This solution is taken from the Numba/llvmlite code
+_shutting_down = [False]
+
+
+@atexit.register
+def _at_shutdown():
+    _shutting_down[0] = True
+
+
+def is_shutting_down(_shutting_down=_shutting_down):
+    """
+    Whether the interpreter is currently shutting down.
+    For use in finalizers, __del__ methods, and similar; it is advised
+    to early bind this function rather than look it up when calling it,
+    since at shutdown module globals may be cleared.
+    """
+    return _shutting_down[0]
diff --git a/cupy/_version.py b/cupy/_version.py
new file mode 100644
index 0000000..2abdf87
--- /dev/null
+++ b/cupy/_version.py
@@ -0,0 +1 @@
+__version__ = '12.3.0'
diff --git a/cupy/array_api/__init__.py b/cupy/array_api/__init__.py
new file mode 100644
index 0000000..2c5d689
--- /dev/null
+++ b/cupy/array_api/__init__.py
@@ -0,0 +1,388 @@
+"""
+A NumPy sub-namespace that conforms to the Python array API standard.
+
+This submodule accompanies NEP 47, which proposes its inclusion in NumPy. It
+is still considered experimental, and will issue a warning when imported.
+
+This is a proof-of-concept namespace that wraps the corresponding NumPy
+functions to give a conforming implementation of the Python array API standard
+(https://data-apis.github.io/array-api/latest/). The standard is currently in
+an RFC phase and comments on it are both welcome and encouraged. Comments
+should be made either at https://github.com/data-apis/array-api or at
+https://github.com/data-apis/consortium-feedback/discussions.
+
+NumPy already follows the proposed spec for the most part, so this module
+serves mostly as a thin wrapper around it. However, NumPy also implements a
+lot of behavior that is not included in the spec, so this serves as a
+restricted subset of the API. Only those functions that are part of the spec
+are included in this namespace, and all functions are given with the exact
+signature given in the spec, including the use of position-only arguments, and
+omitting any extra keyword arguments implemented by NumPy but not part of the
+spec. The behavior of some functions is also modified from the NumPy behavior
+to conform to the standard. Note that the underlying array object itself is
+wrapped in a wrapper Array() class, but is otherwise unchanged. This submodule
+is implemented in pure Python with no C extensions.
+
+The array API spec is designed as a "minimal API subset" and explicitly allows
+libraries to include behaviors not specified by it. But users of this module
+that intend to write portable code should be aware that only those behaviors
+that are listed in the spec are guaranteed to be implemented across libraries.
+Consequently, the NumPy implementation was chosen to be both conforming and
+minimal, so that users can use this implementation of the array API namespace
+and be sure that behaviors that it defines will be available in conforming
+namespaces from other libraries.
+
+A few notes about the current state of this submodule:
+
+- There is a test suite that tests modules against the array API standard at
+  https://github.com/data-apis/array-api-tests. The test suite is still a work
+  in progress, but the existing tests pass on this module, with a few
+  exceptions:
+
+  - DLPack support (see https://github.com/data-apis/array-api/pull/106) is
+    not included here, as it requires a full implementation in NumPy proper
+    first.
+
+  The test suite is not yet complete, and even the tests that exist are not
+  guaranteed to give a comprehensive coverage of the spec. Therefore, when
+  reviewing and using this submodule, you should refer to the standard
+  documents themselves. There are some tests in numpy.array_api.tests, but
+  they primarily focus on things that are not tested by the official array API
+  test suite.
+
+- There is a custom array object, numpy.array_api.Array, which is returned by
+  all functions in this module. All functions in the array API namespace
+  implicitly assume that they will only receive this object as input. The only
+  way to create instances of this object is to use one of the array creation
+  functions. It does not have a public constructor on the object itself. The
+  object is a small wrapper class around numpy.ndarray. The main purpose of it
+  is to restrict the namespace of the array object to only those dtypes and
+  only those methods that are required by the spec, as well as to limit/change
+  certain behavior that differs in the spec. In particular:
+
+  - The array API namespace does not have scalar objects, only 0-D arrays.
+    Operations on Array that would create a scalar in NumPy create a 0-D
+    array.
+
+  - Indexing: Only a subset of indices supported by NumPy are required by the
+    spec. The Array object restricts indexing to only allow those types of
+    indices that are required by the spec. See the docstring of the
+    numpy.array_api.Array._validate_indices helper function for more
+    information.
+
+  - Type promotion: Some type promotion rules are different in the spec. In
+    particular, the spec does not have any value-based casting. The spec also
+    does not require cross-kind casting, like integer -> floating-point. Only
+    those promotions that are explicitly required by the array API
+    specification are allowed in this module. See NEP 47 for more info.
+
+  - Functions do not automatically call asarray() on their input, and will not
+    work if the input type is not Array. The exception is array creation
+    functions, and Python operators on the Array object, which accept Python
+    scalars of the same type as the array dtype.
+
+- All functions include type annotations, corresponding to those given in the
+  spec (see _typing.py for definitions of some custom types). These do not
+  currently fully pass mypy due to some limitations in mypy.
+
+- Dtype objects are just the NumPy dtype objects, e.g., float64 =
+  np.dtype('float64'). The spec does not require any behavior on these dtype
+  objects other than that they be accessible by name and be comparable by
+  equality, but it was considered too much extra complexity to create custom
+  objects to represent dtypes.
+
+- All places where the implementations in this submodule are known to deviate
+  from their corresponding functions in NumPy are marked with "# Note:"
+  comments.
+
+Still TODO in this module are:
+
+- DLPack support for numpy.ndarray is still in progress. See
+  https://github.com/numpy/numpy/pull/19083.
+
+- The copy=False keyword argument to asarray() is not yet implemented. This
+  requires support in numpy.asarray() first.
+
+- Some functions are not yet fully tested in the array API test suite, and may
+  require updates that are not yet known until the tests are written.
+
+- The spec is still in an RFC phase and may still have minor updates, which
+  will need to be reflected here.
+
+- Complex number support in array API spec is planned but not yet finalized,
+  as are the fft extension and certain linear algebra functions such as eig
+  that require complex dtypes.
+
+"""
+
+# CuPy-specific: still need to support Python 3.7.
+import sys
+
+if sys.version_info < (3, 8):
+    raise RuntimeError('cupy.array_api requires Python 3.8+')
+
+
+import warnings
+
+warnings.warn(
+    "The cupy.array_api submodule is still experimental. See NEP 47.", stacklevel=2
+)
+
+__array_api_version__ = "2021.12"
+
+__all__ = ["__array_api_version__"]
+
+from ._constants import e, inf, nan, pi
+
+__all__ += ["e", "inf", "nan", "pi"]
+
+from ._creation_functions import (
+    asarray,
+    arange,
+    empty,
+    empty_like,
+    eye,
+    from_dlpack,
+    full,
+    full_like,
+    linspace,
+    meshgrid,
+    ones,
+    ones_like,
+    tril,
+    triu,
+    zeros,
+    zeros_like,
+)
+
+__all__ += [
+    "asarray",
+    "arange",
+    "empty",
+    "empty_like",
+    "eye",
+    "from_dlpack",
+    "full",
+    "full_like",
+    "linspace",
+    "meshgrid",
+    "ones",
+    "ones_like",
+    "tril",
+    "triu",
+    "zeros",
+    "zeros_like",
+]
+
+from ._data_type_functions import (
+    astype,
+    broadcast_arrays,
+    broadcast_to,
+    can_cast,
+    finfo,
+    iinfo,
+    result_type,
+)
+
+__all__ += [
+    "astype",
+    "broadcast_arrays",
+    "broadcast_to",
+    "can_cast",
+    "finfo",
+    "iinfo",
+    "result_type",
+]
+
+from ._dtypes import (
+    int8,
+    int16,
+    int32,
+    int64,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    float32,
+    float64,
+    bool,
+)
+
+__all__ += [
+    "int8",
+    "int16",
+    "int32",
+    "int64",
+    "uint8",
+    "uint16",
+    "uint32",
+    "uint64",
+    "float32",
+    "float64",
+    "bool",
+]
+
+from ._elementwise_functions import (
+    abs,
+    acos,
+    acosh,
+    add,
+    asin,
+    asinh,
+    atan,
+    atan2,
+    atanh,
+    bitwise_and,
+    bitwise_left_shift,
+    bitwise_invert,
+    bitwise_or,
+    bitwise_right_shift,
+    bitwise_xor,
+    ceil,
+    cos,
+    cosh,
+    divide,
+    equal,
+    exp,
+    expm1,
+    floor,
+    floor_divide,
+    greater,
+    greater_equal,
+    isfinite,
+    isinf,
+    isnan,
+    less,
+    less_equal,
+    log,
+    log1p,
+    log2,
+    log10,
+    logaddexp,
+    logical_and,
+    logical_not,
+    logical_or,
+    logical_xor,
+    multiply,
+    negative,
+    not_equal,
+    positive,
+    pow,
+    remainder,
+    round,
+    sign,
+    sin,
+    sinh,
+    square,
+    sqrt,
+    subtract,
+    tan,
+    tanh,
+    trunc,
+)
+
+__all__ += [
+    "abs",
+    "acos",
+    "acosh",
+    "add",
+    "asin",
+    "asinh",
+    "atan",
+    "atan2",
+    "atanh",
+    "bitwise_and",
+    "bitwise_left_shift",
+    "bitwise_invert",
+    "bitwise_or",
+    "bitwise_right_shift",
+    "bitwise_xor",
+    "ceil",
+    "cos",
+    "cosh",
+    "divide",
+    "equal",
+    "exp",
+    "expm1",
+    "floor",
+    "floor_divide",
+    "greater",
+    "greater_equal",
+    "isfinite",
+    "isinf",
+    "isnan",
+    "less",
+    "less_equal",
+    "log",
+    "log1p",
+    "log2",
+    "log10",
+    "logaddexp",
+    "logical_and",
+    "logical_not",
+    "logical_or",
+    "logical_xor",
+    "multiply",
+    "negative",
+    "not_equal",
+    "positive",
+    "pow",
+    "remainder",
+    "round",
+    "sign",
+    "sin",
+    "sinh",
+    "square",
+    "sqrt",
+    "subtract",
+    "tan",
+    "tanh",
+    "trunc",
+]
+
+from ._indexing_functions import take
+
+__all__ += ["take"]
+
+# linalg is an extension in the array API spec, which is a sub-namespace. Only
+# a subset of functions in it are imported into the top-level namespace.
+from . import linalg
+
+__all__ += ["linalg"]
+
+from .linalg import matmul, tensordot, matrix_transpose, vecdot
+
+__all__ += ["matmul", "tensordot", "matrix_transpose", "vecdot"]
+
+from ._manipulation_functions import (
+    concat,
+    expand_dims,
+    flip,
+    permute_dims,
+    reshape,
+    roll,
+    squeeze,
+    stack,
+)
+
+__all__ += ["concat", "expand_dims", "flip", "permute_dims", "reshape", "roll", "squeeze", "stack"]
+
+from ._searching_functions import argmax, argmin, nonzero, where
+
+__all__ += ["argmax", "argmin", "nonzero", "where"]
+
+from ._set_functions import unique_all, unique_counts, unique_inverse, unique_values
+
+__all__ += ["unique_all", "unique_counts", "unique_inverse", "unique_values"]
+
+from ._sorting_functions import argsort, sort
+
+__all__ += ["argsort", "sort"]
+
+from ._statistical_functions import max, mean, min, prod, std, sum, var
+
+__all__ += ["max", "mean", "min", "prod", "std", "sum", "var"]
+
+from ._utility_functions import all, any
+
+__all__ += ["all", "any"]
diff --git a/cupy/array_api/_array_object.py b/cupy/array_api/_array_object.py
new file mode 100644
index 0000000..25a12bf
--- /dev/null
+++ b/cupy/array_api/_array_object.py
@@ -0,0 +1,1138 @@
+"""
+Wrapper class around the ndarray object for the array API standard.
+
+The array API standard defines some behaviors differently than ndarray, in
+particular, type promotion rules are different (the standard has no
+value-based casting). The standard also specifies a more limited subset of
+array methods and functionalities than are implemented on ndarray. Since the
+goal of the array_api namespace is to be a minimal implementation of the array
+API standard, we need to define a separate wrapper class for the array_api
+namespace.
+
+The standard compliant class is only a wrapper class. It is *not* a subclass
+of ndarray.
+"""
+
+from __future__ import annotations
+
+import operator
+from enum import IntEnum
+from ._creation_functions import asarray
+from ._dtypes import (
+    _all_dtypes,
+    _boolean_dtypes,
+    _integer_dtypes,
+    _integer_or_boolean_dtypes,
+    _floating_dtypes,
+    _numeric_dtypes,
+    _result_type,
+    _dtype_categories,
+)
+
+from typing import TYPE_CHECKING, Optional, Tuple, Union, Any, SupportsIndex
+import types
+
+if TYPE_CHECKING:
+    from ._typing import Any, PyCapsule, Device, Dtype
+    import numpy.typing as npt
+
+import cupy as np
+from cupy.cuda import Device as _Device
+from cupy.cuda import stream as stream_module
+from cupy_backends.cuda.api import runtime
+
+from cupy import array_api
+
+
+class Array:
+    """
+    n-d array object for the array API namespace.
+
+    See the docstring of :py:obj:`np.ndarray <numpy.ndarray>` for more
+    information.
+
+    This is a wrapper around numpy.ndarray that restricts the usage to only
+    those things that are required by the array API namespace. Note,
+    attributes on this object that start with a single underscore are not part
+    of the API specification and should only be used internally. This object
+    should not be constructed directly. Rather, use one of the creation
+    functions, such as asarray().
+
+    """
+    _array: np.ndarray
+
+    # Use a custom constructor instead of __init__, as manually initializing
+    # this class is not supported API.
+    @classmethod
+    def _new(cls, x: Union[np.ndarray, np.generic], /) -> Array:
+        """
+        This is a private method for initializing the array API Array
+        object.
+
+        Functions outside of the array_api submodule should not use this
+        method. Use one of the creation functions instead, such as
+        ``asarray``.
+
+        """
+        obj = super().__new__(cls)
+        # Note: The spec does not have array scalars, only 0-D arrays.
+        if isinstance(x, np.generic):
+            # Convert the array scalar to a 0-D array
+            x = np.asarray(x)
+        if x.dtype not in _all_dtypes:
+            raise TypeError(
+                f"The array_api namespace does not support the dtype '{x.dtype}'"
+            )
+        obj._array = x
+        return obj
+
+    # Prevent Array() from working
+    def __new__(cls, *args, **kwargs):
+        raise TypeError(
+            "The array_api Array object should not be instantiated directly. Use an array creation function, such as asarray(), instead."
+        )
+
+    # These functions are not required by the spec, but are implemented for
+    # the sake of usability.
+
+    def __str__(self: Array, /) -> str:
+        """
+        Performs the operation __str__.
+        """
+        return self._array.__str__().replace("array", "Array")
+
+    def __repr__(self: Array, /) -> str:
+        """
+        Performs the operation __repr__.
+        """
+        suffix = f", dtype={self.dtype.name})"
+        if 0 in self.shape:
+            prefix = "empty("
+            mid = str(self.shape)
+        else:
+            prefix = "Array("
+            mid = np.array2string(np.asnumpy(self._array), separator=', ', prefix=prefix, suffix=suffix)
+        return prefix + mid + suffix
+
+    def __cupy_get_ndarray__(self):
+        return self._array
+
+    # These are various helper functions to make the array behavior match the
+    # spec in places where it either deviates from or is more strict than
+    # NumPy behavior
+
+    def _check_allowed_dtypes(self, other: Union[bool, int, float, Array], dtype_category: str, op: str) -> Array:
+        """
+        Helper function for operators to only allow specific input dtypes
+
+        Use like
+
+            other = self._check_allowed_dtypes(other, 'numeric', '__add__')
+            if other is NotImplemented:
+                return other
+        """
+
+        if self.dtype not in _dtype_categories[dtype_category]:
+            raise TypeError(f"Only {dtype_category} dtypes are allowed in {op}")
+        if isinstance(other, (int, float, bool)):
+            other = self._promote_scalar(other)
+        elif isinstance(other, Array):
+            if other.dtype not in _dtype_categories[dtype_category]:
+                raise TypeError(f"Only {dtype_category} dtypes are allowed in {op}")
+        else:
+            return NotImplemented
+
+        # This will raise TypeError for type combinations that are not allowed
+        # to promote in the spec (even if the NumPy array operator would
+        # promote them).
+        res_dtype = _result_type(self.dtype, other.dtype)
+        if op.startswith("__i"):
+            # Note: NumPy will allow in-place operators in some cases where
+            # the type promoted operator does not match the left-hand side
+            # operand. For example,
+
+            # >>> a = np.array(1, dtype=np.int8)
+            # >>> a += np.array(1, dtype=np.int16)
+
+            # The spec explicitly disallows this.
+            if res_dtype != self.dtype:
+                raise TypeError(
+                    f"Cannot perform {op} with dtypes {self.dtype} and {other.dtype}"
+                )
+
+        return other
+
+    # Helper function to match the type promotion rules in the spec
+    def _promote_scalar(self, scalar: Union[bool, int, float]) -> Array:
+        """
+        Returns a promoted version of a Python scalar appropriate for use with
+        operations on self.
+
+        This may raise an OverflowError in cases where the scalar is an
+        integer that is too large to fit in a NumPy integer dtype, or
+        TypeError when the scalar type is incompatible with the dtype of self.
+        """
+        # Note: Only Python scalar types that match the array dtype are
+        # allowed.
+        if isinstance(scalar, bool):
+            if self.dtype not in _boolean_dtypes:
+                raise TypeError(
+                    "Python bool scalars can only be promoted with bool arrays"
+                )
+        elif isinstance(scalar, int):
+            if self.dtype in _boolean_dtypes:
+                raise TypeError(
+                    "Python int scalars cannot be promoted with bool arrays"
+                )
+        elif isinstance(scalar, float):
+            if self.dtype not in _floating_dtypes:
+                raise TypeError(
+                    "Python float scalars can only be promoted with floating-point arrays."
+                )
+        else:
+            raise TypeError("'scalar' must be a Python scalar")
+
+        # Note: scalars are unconditionally cast to the same dtype as the
+        # array.
+
+        # Note: the spec only specifies integer-dtype/int promotion
+        # behavior for integers within the bounds of the integer dtype.
+        # Outside of those bounds we use the default NumPy behavior (either
+        # cast or raise OverflowError).
+        return Array._new(np.array(scalar, self.dtype))
+
+    @staticmethod
+    def _normalize_two_args(x1, x2) -> Tuple[Array, Array]:
+        """
+        Normalize inputs to two arg functions to fix type promotion rules
+
+        NumPy deviates from the spec type promotion rules in cases where one
+        argument is 0-dimensional and the other is not. For example:
+
+        >>> import numpy as np
+        >>> a = np.array([1.0], dtype=np.float32)
+        >>> b = np.array(1.0, dtype=np.float64)
+        >>> np.add(a, b) # The spec says this should be float64
+        array([2.], dtype=float32)
+
+        To fix this, we add a dimension to the 0-dimension array before passing it
+        through. This works because a dimension would be added anyway from
+        broadcasting, so the resulting shape is the same, but this prevents NumPy
+        from not promoting the dtype.
+        """
+        # Another option would be to use signature=(x1.dtype, x2.dtype, None),
+        # but that only works for ufuncs, so we would have to call the ufuncs
+        # directly in the operator methods. One should also note that this
+        # sort of trick wouldn't work for functions like searchsorted, which
+        # don't do normal broadcasting, but there aren't any functions like
+        # that in the array API namespace.
+        if x1.ndim == 0 and x2.ndim != 0:
+            # The _array[None] workaround was chosen because it is relatively
+            # performant. broadcast_to(x1._array, x2.shape) is much slower. We
+            # could also manually type promote x2, but that is more complicated
+            # and about the same performance as this.
+            x1 = Array._new(x1._array[None])
+        elif x2.ndim == 0 and x1.ndim != 0:
+            x2 = Array._new(x2._array[None])
+        return (x1, x2)
+
+    # Note: A large fraction of allowed indices are disallowed here (see the
+    # docstring below)
+    def _validate_index(self, key):
+        """
+        Validate an index according to the array API.
+
+        The array API specification only requires a subset of indices that are
+        supported by NumPy. This function will reject any index that is
+        allowed by NumPy but not required by the array API specification. We
+        always raise ``IndexError`` on such indices (the spec does not require
+        any specific behavior on them, but this makes the NumPy array API
+        namespace a minimal implementation of the spec). See
+        https://data-apis.org/array-api/latest/API_specification/indexing.html
+        for the full list of required indexing behavior
+
+        This function raises IndexError if the index ``key`` is invalid. It
+        only raises ``IndexError`` on indices that are not already rejected by
+        NumPy, as NumPy will already raise the appropriate error on such
+        indices. ``shape`` may be None, in which case, only cases that are
+        independent of the array shape are checked.
+
+        The following cases are allowed by NumPy, but not specified by the array
+        API specification:
+
+        - Indices to not include an implicit ellipsis at the end. That is,
+          every axis of an array must be explicitly indexed or an ellipsis
+          included. This behaviour is sometimes referred to as flat indexing.
+
+        - The start and stop of a slice may not be out of bounds. In
+          particular, for a slice ``i:j:k`` on an axis of size ``n``, only the
+          following are allowed:
+
+          - ``i`` or ``j`` omitted (``None``).
+          - ``-n <= i <= max(0, n - 1)``.
+          - For ``k > 0`` or ``k`` omitted (``None``), ``-n <= j <= n``.
+          - For ``k < 0``, ``-n - 1 <= j <= max(0, n - 1)``.
+
+        - Boolean array indices are not allowed as part of a larger tuple
+          index.
+
+        - Integer array indices are not allowed (with the exception of 0-D
+          arrays, which are treated the same as scalars).
+
+        Additionally, it should be noted that indices that would return a
+        scalar in NumPy will return a 0-D array. Array scalars are not allowed
+        in the specification, only 0-D arrays. This is done in the
+        ``Array._new`` constructor, not this function.
+
+        """
+        _key = key if isinstance(key, tuple) else (key,)
+        for i in _key:
+            if isinstance(i, bool) or not (
+                isinstance(i, SupportsIndex)  # i.e. ints
+                or isinstance(i, Array)
+                or isinstance(i, np.ndarray)
+                or isinstance(i, slice)
+                or i == Ellipsis
+                or i is None
+            ):
+                raise IndexError(
+                    f"Single-axes index {i} has {type(i)=}, but only "
+                    "integers, slices (:), ellipsis (...), newaxis (None), "
+                    "zero-dimensional integer arrays and boolean arrays "
+                    "are specified in the Array API."
+                )
+
+        nonexpanding_key = []
+        single_axes = []
+        n_ellipsis = 0
+        key_has_mask = False
+        for i in _key:
+            if i is not None:
+                nonexpanding_key.append(i)
+                if isinstance(i, Array) or isinstance(i, np.ndarray):
+                    if i.dtype in _boolean_dtypes:
+                        key_has_mask = True
+                    single_axes.append(i)
+                else:
+                    # i must not be an array here, to avoid elementwise equals
+                    if i == Ellipsis:
+                        n_ellipsis += 1
+                    else:
+                        single_axes.append(i)
+
+        n_single_axes = len(single_axes)
+        if n_ellipsis > 1:
+            return  # handled by ndarray
+        elif n_ellipsis == 0:
+            # Note boolean masks must be the sole index, which we check for
+            # later on.
+            if not key_has_mask and n_single_axes < self.ndim:
+                raise IndexError(
+                    f"{self.ndim=}, but the multi-axes index only specifies "
+                    f"{n_single_axes} dimensions. If this was intentional, "
+                    "add a trailing ellipsis (...) which expands into as many "
+                    "slices (:) as necessary - this is what np.ndarray arrays "
+                    "implicitly do, but such flat indexing behaviour is not "
+                    "specified in the Array API."
+                )
+
+        if n_ellipsis == 0:
+            indexed_shape = self.shape
+        else:
+            ellipsis_start = None
+            for pos, i in enumerate(nonexpanding_key):
+                if not (isinstance(i, Array) or isinstance(i, np.ndarray)):
+                    if i == Ellipsis:
+                        ellipsis_start = pos
+                        break
+            assert ellipsis_start is not None  # sanity check
+            ellipsis_end = self.ndim - (n_single_axes - ellipsis_start)
+            indexed_shape = (
+                self.shape[:ellipsis_start] + self.shape[ellipsis_end:]
+            )
+        for i, side in zip(single_axes, indexed_shape):
+            if isinstance(i, slice):
+                if side == 0:
+                    f_range = "0 (or None)"
+                else:
+                    f_range = f"between -{side} and {side - 1} (or None)"
+                if i.start is not None:
+                    try:
+                        start = operator.index(i.start)
+                    except TypeError:
+                        pass  # handled by ndarray
+                    else:
+                        if not (-side <= start <= side):
+                            raise IndexError(
+                                f"Slice {i} contains {start=}, but should be "
+                                f"{f_range} for an axis of size {side} "
+                                "(out-of-bounds starts are not specified in "
+                                "the Array API)"
+                            )
+                if i.stop is not None:
+                    try:
+                        stop = operator.index(i.stop)
+                    except TypeError:
+                        pass  # handled by ndarray
+                    else:
+                        if not (-side <= stop <= side):
+                            raise IndexError(
+                                f"Slice {i} contains {stop=}, but should be "
+                                f"{f_range} for an axis of size {side} "
+                                "(out-of-bounds stops are not specified in "
+                                "the Array API)"
+                            )
+            elif isinstance(i, Array):
+                if i.dtype in _boolean_dtypes and len(_key) != 1:
+                    assert isinstance(key, tuple)  # sanity check
+                    raise IndexError(
+                        f"Single-axes index {i} is a boolean array and "
+                        f"{len(key)=}, but masking is only specified in the "
+                        "Array API when the array is the sole index."
+                    )
+                elif i.dtype in _integer_dtypes and i.ndim != 0:
+                    raise IndexError(
+                        f"Single-axes index {i} is a non-zero-dimensional "
+                        "integer array, but advanced integer indexing is not "
+                        "specified in the Array API."
+                    )
+            elif isinstance(i, tuple):
+                raise IndexError(
+                    f"Single-axes index {i} is a tuple, but nested tuple "
+                    "indices are not specified in the Array API."
+                )
+
+    # Everything below this line is required by the spec.
+
+    def __abs__(self: Array, /) -> Array:
+        """
+        Performs the operation __abs__.
+        """
+        if self.dtype not in _numeric_dtypes:
+            raise TypeError("Only numeric dtypes are allowed in __abs__")
+        res = self._array.__abs__()
+        return self.__class__._new(res)
+
+    def __add__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __add__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__add__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__add__(other._array)
+        return self.__class__._new(res)
+
+    def __and__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __and__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__and__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__and__(other._array)
+        return self.__class__._new(res)
+
+    def __array_namespace__(
+        self: Array, /, *, api_version: Optional[str] = None
+    ) -> types.ModuleType:
+        if api_version is not None and not api_version.startswith("2021."):
+            raise ValueError(f"Unrecognized array API version: {api_version!r}")
+        return array_api
+
+    def __bool__(self: Array, /) -> bool:
+        """
+        Performs the operation __bool__.
+        """
+        # Note: This is an error here.
+        if self._array.ndim != 0:
+            raise TypeError("bool is only allowed on arrays with 0 dimensions")
+        if self.dtype not in _boolean_dtypes:
+            raise ValueError("bool is only allowed on boolean arrays")
+        res = self._array.__bool__()
+        return res
+
+    def __dlpack__(self: Array, /, *, stream=None) -> PyCapsule:
+        """
+        Performs the operation __dlpack__.
+        """
+        return self._array.__dlpack__(stream=stream)
+
+    def __dlpack_device__(self: Array, /) -> Tuple[IntEnum, int]:
+        """
+        Performs the operation __dlpack_device__.
+        """
+        return self._array.__dlpack_device__()
+
+    def __eq__(self: Array, other: Union[int, float, bool, Array], /) -> Array:  # type: ignore
+        """
+        Performs the operation __eq__.
+        """
+        # Even though "all" dtypes are allowed, we still require them to be
+        # promotable with each other.
+        other = self._check_allowed_dtypes(other, "all", "__eq__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__eq__(other._array)
+        return self.__class__._new(res)
+
+    def __float__(self: Array, /) -> float:
+        """
+        Performs the operation __float__.
+        """
+        # Note: This is an error here.
+        if self._array.ndim != 0:
+            raise TypeError("float is only allowed on arrays with 0 dimensions")
+        if self.dtype not in _floating_dtypes:
+            raise ValueError("float is only allowed on floating-point arrays")
+        res = self._array.__float__()
+        return res
+
+    def __floordiv__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __floordiv__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__floordiv__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__floordiv__(other._array)
+        return self.__class__._new(res)
+
+    def __ge__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __ge__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__ge__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__ge__(other._array)
+        return self.__class__._new(res)
+
+    def __getitem__(
+        self: Array,
+        key: Union[
+            int, slice, ellipsis, Tuple[Union[int, slice, ellipsis], ...], Array
+        ],
+        /,
+    ) -> Array:
+        """
+        Performs the operation __getitem__.
+        """
+        # Note: Only indices required by the spec are allowed. See the
+        # docstring of _validate_index
+        self._validate_index(key)
+        if isinstance(key, Array):
+            # Indexing self._array with array_api arrays can be erroneous
+            key = key._array
+        res = self._array.__getitem__(key)
+        return self._new(res)
+
+    def __gt__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __gt__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__gt__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__gt__(other._array)
+        return self.__class__._new(res)
+
+    def __int__(self: Array, /) -> int:
+        """
+        Performs the operation __int__.
+        """
+        # Note: This is an error here.
+        if self._array.ndim != 0:
+            raise TypeError("int is only allowed on arrays with 0 dimensions")
+        if self.dtype not in _integer_dtypes:
+            raise ValueError("int is only allowed on integer arrays")
+        res = self._array.__int__()
+        return res
+
+    def __index__(self: Array, /) -> int:
+        """
+        Performs the operation __index__.
+        """
+        # TODO(leofang): just do this when CuPy is ready:
+        # res = self._array.__index__()
+        if self.ndim != 0 or self.dtype not in _integer_dtypes:
+            raise TypeError("only integer scalar arrays can be converted to a scalar index")
+        return int(self._array)
+
+    def __invert__(self: Array, /) -> Array:
+        """
+        Performs the operation __invert__.
+        """
+        if self.dtype not in _integer_or_boolean_dtypes:
+            raise TypeError("Only integer or boolean dtypes are allowed in __invert__")
+        res = self._array.__invert__()
+        return self.__class__._new(res)
+
+    def __le__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __le__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__le__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__le__(other._array)
+        return self.__class__._new(res)
+
+    def __lshift__(self: Array, other: Union[int, Array], /) -> Array:
+        """
+        Performs the operation __lshift__.
+        """
+        other = self._check_allowed_dtypes(other, "integer", "__lshift__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__lshift__(other._array)
+        return self.__class__._new(res)
+
+    def __lt__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __lt__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__lt__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__lt__(other._array)
+        return self.__class__._new(res)
+
+    def __matmul__(self: Array, other: Array, /) -> Array:
+        """
+        Performs the operation __matmul__.
+        """
+        # matmul is not defined for scalars, but without this, we may get
+        # the wrong error message from asarray.
+        other = self._check_allowed_dtypes(other, "numeric", "__matmul__")
+        if other is NotImplemented:
+            return other
+        res = self._array.__matmul__(other._array)
+        return self.__class__._new(res)
+
+    def __mod__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __mod__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__mod__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__mod__(other._array)
+        return self.__class__._new(res)
+
+    def __mul__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __mul__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__mul__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__mul__(other._array)
+        return self.__class__._new(res)
+
+    def __ne__(self: Array, other: Union[int, float, bool, Array], /) -> Array:  # type: ignore
+        """
+        Performs the operation __ne__.
+        """
+        other = self._check_allowed_dtypes(other, "all", "__ne__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__ne__(other._array)
+        return self.__class__._new(res)
+
+    def __neg__(self: Array, /) -> Array:
+        """
+        Performs the operation __neg__.
+        """
+        if self.dtype not in _numeric_dtypes:
+            raise TypeError("Only numeric dtypes are allowed in __neg__")
+        res = self._array.__neg__()
+        return self.__class__._new(res)
+
+    def __or__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __or__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__or__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__or__(other._array)
+        return self.__class__._new(res)
+
+    def __pos__(self: Array, /) -> Array:
+        """
+        Performs the operation __pos__.
+        """
+        if self.dtype not in _numeric_dtypes:
+            raise TypeError("Only numeric dtypes are allowed in __pos__")
+        res = self._array.__pos__()
+        return self.__class__._new(res)
+
+    # PEP 484 requires int to be a subtype of float, but __pow__ should not
+    # accept int.
+    def __pow__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __pow__.
+        """
+        from ._elementwise_functions import pow
+
+        other = self._check_allowed_dtypes(other, "numeric", "__pow__")
+        if other is NotImplemented:
+            return other
+        # Note: NumPy's __pow__ does not follow type promotion rules for 0-d
+        # arrays, so we use pow() here instead.
+        return pow(self, other)
+
+    def __rshift__(self: Array, other: Union[int, Array], /) -> Array:
+        """
+        Performs the operation __rshift__.
+        """
+        other = self._check_allowed_dtypes(other, "integer", "__rshift__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rshift__(other._array)
+        return self.__class__._new(res)
+
+    def __setitem__(
+        self,
+        key: Union[
+            int, slice, ellipsis, Tuple[Union[int, slice, ellipsis], ...], Array
+        ],
+        value: Union[int, float, bool, Array],
+        /,
+    ) -> None:
+        """
+        Performs the operation __setitem__.
+        """
+        # Note: Only indices required by the spec are allowed. See the
+        # docstring of _validate_index
+        self._validate_index(key)
+        if isinstance(key, Array):
+            # Indexing self._array with array_api arrays can be erroneous
+            key = key._array
+        self._array.__setitem__(key, asarray(value)._array)
+
+    def __sub__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __sub__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__sub__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__sub__(other._array)
+        return self.__class__._new(res)
+
+    # PEP 484 requires int to be a subtype of float, but __truediv__ should
+    # not accept int.
+    def __truediv__(self: Array, other: Union[float, Array], /) -> Array:
+        """
+        Performs the operation __truediv__.
+        """
+        other = self._check_allowed_dtypes(other, "floating-point", "__truediv__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__truediv__(other._array)
+        return self.__class__._new(res)
+
+    def __xor__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __xor__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__xor__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__xor__(other._array)
+        return self.__class__._new(res)
+
+    def __iadd__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __iadd__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__iadd__")
+        if other is NotImplemented:
+            return other
+        self._array.__iadd__(other._array)
+        return self
+
+    def __radd__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __radd__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__radd__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__radd__(other._array)
+        return self.__class__._new(res)
+
+    def __iand__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __iand__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__iand__")
+        if other is NotImplemented:
+            return other
+        self._array.__iand__(other._array)
+        return self
+
+    def __rand__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __rand__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__rand__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rand__(other._array)
+        return self.__class__._new(res)
+
+    def __ifloordiv__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __ifloordiv__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__ifloordiv__")
+        if other is NotImplemented:
+            return other
+        self._array.__ifloordiv__(other._array)
+        return self
+
+    def __rfloordiv__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __rfloordiv__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__rfloordiv__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rfloordiv__(other._array)
+        return self.__class__._new(res)
+
+    def __ilshift__(self: Array, other: Union[int, Array], /) -> Array:
+        """
+        Performs the operation __ilshift__.
+        """
+        other = self._check_allowed_dtypes(other, "integer", "__ilshift__")
+        if other is NotImplemented:
+            return other
+        self._array.__ilshift__(other._array)
+        return self
+
+    def __rlshift__(self: Array, other: Union[int, Array], /) -> Array:
+        """
+        Performs the operation __rlshift__.
+        """
+        other = self._check_allowed_dtypes(other, "integer", "__rlshift__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rlshift__(other._array)
+        return self.__class__._new(res)
+
+    def __imatmul__(self: Array, other: Array, /) -> Array:
+        """
+        Performs the operation __imatmul__.
+        """
+        # Note: NumPy does not implement __imatmul__.
+
+        # matmul is not defined for scalars, but without this, we may get
+        # the wrong error message from asarray.
+        other = self._check_allowed_dtypes(other, "numeric", "__imatmul__")
+        if other is NotImplemented:
+            return other
+
+        # __imatmul__ can only be allowed when it would not change the shape
+        # of self.
+        other_shape = other.shape
+        if self.shape == () or other_shape == ():
+            raise ValueError("@= requires at least one dimension")
+        if len(other_shape) == 1 or other_shape[-1] != other_shape[-2]:
+            raise ValueError("@= cannot change the shape of the input array")
+        self._array[:] = self._array.__matmul__(other._array)
+        return self
+
+    def __rmatmul__(self: Array, other: Array, /) -> Array:
+        """
+        Performs the operation __rmatmul__.
+        """
+        # matmul is not defined for scalars, but without this, we may get
+        # the wrong error message from asarray.
+        other = self._check_allowed_dtypes(other, "numeric", "__rmatmul__")
+        if other is NotImplemented:
+            return other
+        res = self._array.__rmatmul__(other._array)
+        return self.__class__._new(res)
+
+    def __imod__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __imod__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__imod__")
+        if other is NotImplemented:
+            return other
+        self._array.__imod__(other._array)
+        return self
+
+    def __rmod__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __rmod__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__rmod__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rmod__(other._array)
+        return self.__class__._new(res)
+
+    def __imul__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __imul__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__imul__")
+        if other is NotImplemented:
+            return other
+        self._array.__imul__(other._array)
+        return self
+
+    def __rmul__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __rmul__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__rmul__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rmul__(other._array)
+        return self.__class__._new(res)
+
+    def __ior__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __ior__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__ior__")
+        if other is NotImplemented:
+            return other
+        self._array.__ior__(other._array)
+        return self
+
+    def __ror__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __ror__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__ror__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__ror__(other._array)
+        return self.__class__._new(res)
+
+    def __ipow__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __ipow__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__ipow__")
+        if other is NotImplemented:
+            return other
+        self._array.__ipow__(other._array)
+        return self
+
+    def __rpow__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __rpow__.
+        """
+        from ._elementwise_functions import pow
+
+        other = self._check_allowed_dtypes(other, "numeric", "__rpow__")
+        if other is NotImplemented:
+            return other
+        # Note: NumPy's __pow__ does not follow the spec type promotion rules
+        # for 0-d arrays, so we use pow() here instead.
+        return pow(other, self)
+
+    def __irshift__(self: Array, other: Union[int, Array], /) -> Array:
+        """
+        Performs the operation __irshift__.
+        """
+        other = self._check_allowed_dtypes(other, "integer", "__irshift__")
+        if other is NotImplemented:
+            return other
+        self._array.__irshift__(other._array)
+        return self
+
+    def __rrshift__(self: Array, other: Union[int, Array], /) -> Array:
+        """
+        Performs the operation __rrshift__.
+        """
+        other = self._check_allowed_dtypes(other, "integer", "__rrshift__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rrshift__(other._array)
+        return self.__class__._new(res)
+
+    def __isub__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __isub__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__isub__")
+        if other is NotImplemented:
+            return other
+        self._array.__isub__(other._array)
+        return self
+
+    def __rsub__(self: Array, other: Union[int, float, Array], /) -> Array:
+        """
+        Performs the operation __rsub__.
+        """
+        other = self._check_allowed_dtypes(other, "numeric", "__rsub__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rsub__(other._array)
+        return self.__class__._new(res)
+
+    def __itruediv__(self: Array, other: Union[float, Array], /) -> Array:
+        """
+        Performs the operation __itruediv__.
+        """
+        other = self._check_allowed_dtypes(other, "floating-point", "__itruediv__")
+        if other is NotImplemented:
+            return other
+        self._array.__itruediv__(other._array)
+        return self
+
+    def __rtruediv__(self: Array, other: Union[float, Array], /) -> Array:
+        """
+        Performs the operation __rtruediv__.
+        """
+        other = self._check_allowed_dtypes(other, "floating-point", "__rtruediv__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rtruediv__(other._array)
+        return self.__class__._new(res)
+
+    def __ixor__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __ixor__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__ixor__")
+        if other is NotImplemented:
+            return other
+        self._array.__ixor__(other._array)
+        return self
+
+    def __rxor__(self: Array, other: Union[int, bool, Array], /) -> Array:
+        """
+        Performs the operation __rxor__.
+        """
+        other = self._check_allowed_dtypes(other, "integer or boolean", "__rxor__")
+        if other is NotImplemented:
+            return other
+        self, other = self._normalize_two_args(self, other)
+        res = self._array.__rxor__(other._array)
+        return self.__class__._new(res)
+
+    def to_device(self: Array, device: Device, /, stream=None) -> Array:
+        if device == self.device:
+            return self
+        elif not isinstance(device, _Device):
+            raise ValueError(f"Unsupported device {device!r}")
+        else:
+            # see cupy/cupy#5985 for the reason how we handle device/stream here
+            prev_device = runtime.getDevice()
+            prev_stream: stream_module.Stream = None
+            if stream is not None:
+                prev_stream = stream_module.get_current_stream()
+                # stream can be an int as specified in __dlpack__, or a CuPy stream
+                if isinstance(stream, int):
+                    stream = np.cuda.ExternalStream(stream)
+                elif isinstance(stream, np.cuda.Stream):
+                    pass
+                else:
+                    raise ValueError('the input stream is not recognized')
+                stream.use()
+            try:
+                runtime.setDevice(device.id)
+                arr = self._array.copy()
+            finally:
+                runtime.setDevice(prev_device)
+                if stream is not None:
+                    prev_stream.use()
+            return Array._new(arr)
+
+    @property
+    def dtype(self) -> Dtype:
+        """
+        Array API compatible wrapper for :py:meth:`np.ndarray.dtype <numpy.ndarray.dtype>`.
+
+        See its docstring for more information.
+        """
+        return self._array.dtype
+
+    @property
+    def device(self) -> Device:
+        return self._array.device
+
+    # Note: mT is new in array API spec (see matrix_transpose)
+    @property
+    def mT(self) -> Array:
+        from .linalg import matrix_transpose
+        return matrix_transpose(self)
+
+    @property
+    def ndim(self) -> int:
+        """
+        Array API compatible wrapper for :py:meth:`np.ndarray.ndim <numpy.ndarray.ndim>`.
+
+        See its docstring for more information.
+        """
+        return self._array.ndim
+
+    @property
+    def shape(self) -> Tuple[int, ...]:
+        """
+        Array API compatible wrapper for :py:meth:`np.ndarray.shape <numpy.ndarray.shape>`.
+
+        See its docstring for more information.
+        """
+        return self._array.shape
+
+    @property
+    def size(self) -> int:
+        """
+        Array API compatible wrapper for :py:meth:`np.ndarray.size <numpy.ndarray.size>`.
+
+        See its docstring for more information.
+        """
+        return self._array.size
+
+    @property
+    def T(self) -> Array:
+        """
+        Array API compatible wrapper for :py:meth:`np.ndarray.T <numpy.ndarray.T>`.
+
+        See its docstring for more information.
+        """
+        # Note: T only works on 2-dimensional arrays. See the corresponding
+        # note in the specification:
+        # https://data-apis.org/array-api/latest/API_specification/array_object.html#t
+        if self.ndim != 2:
+            raise ValueError("x.T requires x to have 2 dimensions. Use x.mT to transpose stacks of matrices and permute_dims() to permute dimensions.")
+        return self.__class__._new(self._array.T)
diff --git a/cupy/array_api/_constants.py b/cupy/array_api/_constants.py
new file mode 100644
index 0000000..c3ada27
--- /dev/null
+++ b/cupy/array_api/_constants.py
@@ -0,0 +1,6 @@
+import cupy as np
+
+e = np.e
+inf = np.inf
+nan = np.nan
+pi = np.pi
diff --git a/cupy/array_api/_creation_functions.py b/cupy/array_api/_creation_functions.py
new file mode 100644
index 0000000..6d7369a
--- /dev/null
+++ b/cupy/array_api/_creation_functions.py
@@ -0,0 +1,448 @@
+from __future__ import annotations
+
+
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+
+if TYPE_CHECKING:
+    from ._typing import (
+        Array,
+        Device,
+        Dtype,
+        NestedSequence,
+        SupportsBufferProtocol,
+    )
+    from collections.abc import Sequence
+from ._dtypes import _all_dtypes
+
+import cupy as np
+from cupy.cuda import Device as _Device
+from cupy_backends.cuda.api import runtime
+
+
+def _check_valid_dtype(dtype):
+    # Note: Only spelling dtypes as the dtype objects is supported.
+
+    # We use this instead of "dtype in _all_dtypes" because the dtype objects
+    # define equality with the sorts of things we want to disallow.
+    for d in (None,) + _all_dtypes:
+        if dtype is d:
+            return
+    raise ValueError("dtype must be one of the supported dtypes")
+
+
+def asarray(
+    obj: Union[
+        Array,
+        bool,
+        int,
+        float,
+        NestedSequence[bool | int | float],
+        SupportsBufferProtocol,
+    ],
+    /,
+    *,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+    copy: Optional[bool] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.asarray <numpy.asarray>`.
+
+    See its docstring for more information.
+    """
+    # _array_object imports in this file are inside the functions to avoid
+    # circular imports
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    if copy is False:
+        # Note: copy=False is not yet implemented in np.asarray
+        raise NotImplementedError("copy=False is not yet implemented")
+    if isinstance(obj, Array):
+        if dtype is not None and obj.dtype != dtype:
+            copy = True
+        if copy is True:
+            prev_device = runtime.getDevice()
+            try:
+                runtime.setDevice(device.id)
+                obj = Array._new(np.array(obj._array, copy=True, dtype=dtype))
+            finally:
+                runtime.setDevice(prev_device)
+        return obj
+    if dtype is None and isinstance(obj, int) and (obj > 2 ** 64 or obj < -(2 ** 63)):
+        # Give a better error message in this case. NumPy would convert this
+        # to an object array. TODO: This won't handle large integers in lists.
+        raise OverflowError("Integer out of bounds for array dtypes")
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        res = np.asarray(obj, dtype=dtype)
+    finally:
+        runtime.setDevice(prev_device)
+    return Array._new(res)
+
+
+def arange(
+    start: Union[int, float],
+    /,
+    stop: Optional[Union[int, float]] = None,
+    step: Union[int, float] = 1,
+    *,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.arange <numpy.arange>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.arange(start, stop=stop, step=step, dtype=dtype))
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def empty(
+    shape: Union[int, Tuple[int, ...]],
+    *,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.empty <numpy.empty>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.empty(shape, dtype=dtype))
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def empty_like(
+    x: Array, /, *, dtype: Optional[Dtype] = None, device: Optional[Device] = None
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.empty_like <numpy.empty_like>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.empty_like(x._array, dtype=dtype))
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def eye(
+    n_rows: int,
+    n_cols: Optional[int] = None,
+    /,
+    *,
+    k: int = 0,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.eye <numpy.eye>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.eye(n_rows, M=n_cols, k=k, dtype=dtype))
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def from_dlpack(x: object, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.from_dlpack <numpy.from_dlpack>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+    return Array._new(np.from_dlpack(x))
+
+
+def full(
+    shape: Union[int, Tuple[int, ...]],
+    fill_value: Union[int, float],
+    *,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.full <numpy.full>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    if isinstance(fill_value, Array) and fill_value.ndim == 0:
+        fill_value = fill_value._array
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        res = np.full(shape, fill_value, dtype=dtype)
+    finally:
+        runtime.setDevice(prev_device)
+    if res.dtype not in _all_dtypes:
+        # This will happen if the fill value is not something that NumPy
+        # coerces to one of the acceptable dtypes.
+        raise TypeError("Invalid input to full")
+    return Array._new(res)
+
+
+def full_like(
+    x: Array,
+    /,
+    fill_value: Union[int, float],
+    *,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.full_like <numpy.full_like>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    if isinstance(fill_value, Array) and fill_value.ndim == 0:
+        fill_value = fill_value._array
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        res = np.full_like(x._array, fill_value, dtype=dtype)
+    finally:
+        runtime.setDevice(prev_device)
+    if res.dtype not in _all_dtypes:
+        # This will happen if the fill value is not something that NumPy
+        # coerces to one of the acceptable dtypes.
+        raise TypeError("Invalid input to full_like")
+    return Array._new(res)
+
+
+def linspace(
+    start: Union[int, float],
+    stop: Union[int, float],
+    /,
+    num: int,
+    *,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+    endpoint: bool = True,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linspace <numpy.linspace>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is None:
+        device = _Device()  # current device
+    elif not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.linspace(start, stop, num, dtype=dtype, endpoint=endpoint))
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def meshgrid(*arrays: Array, indexing: str = "xy") -> List[Array]:
+    """
+    Array API compatible wrapper for :py:func:`np.meshgrid <numpy.meshgrid>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    # Note: unlike np.meshgrid, only inputs with all the same dtype are
+    # allowed
+
+    if len({a.dtype for a in arrays}) > 1:
+        raise ValueError("meshgrid inputs must all have the same dtype")
+
+    return [
+        Array._new(array)
+        for array in np.meshgrid(*[a._array for a in arrays], indexing=indexing)
+    ]
+
+
+def ones(
+    shape: Union[int, Tuple[int, ...]],
+    *,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.ones <numpy.ones>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.ones(shape, dtype=dtype))
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def ones_like(
+    x: Array, /, *, dtype: Optional[Dtype] = None, device: Optional[Device] = None
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.ones_like <numpy.ones_like>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.ones_like(x._array, dtype=dtype))
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def tril(x: Array, /, *, k: int = 0) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.tril <numpy.tril>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    if x.ndim < 2:
+        # Note: Unlike np.tril, x must be at least 2-D
+        raise ValueError("x must be at least 2-dimensional for tril")
+    return Array._new(np.tril(x._array, k=k))
+
+
+def triu(x: Array, /, *, k: int = 0) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.triu <numpy.triu>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    if x.ndim < 2:
+        # Note: Unlike np.triu, x must be at least 2-D
+        raise ValueError("x must be at least 2-dimensional for triu")
+    return Array._new(np.triu(x._array, k=k))
+
+
+def zeros(
+    shape: Union[int, Tuple[int, ...]],
+    *,
+    dtype: Optional[Dtype] = None,
+    device: Optional[Device] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.zeros <numpy.zeros>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.zeros(shape, dtype=dtype))
+    finally:
+        runtime.setDevice(prev_device)
+
+
+def zeros_like(
+    x: Array, /, *, dtype: Optional[Dtype] = None, device: Optional[Device] = None
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.zeros_like <numpy.zeros_like>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    _check_valid_dtype(dtype)
+    if device is not None and not isinstance(device, _Device):
+        raise ValueError(f"Unsupported device {device!r}")
+    if device is None:
+        device = _Device()  # current device
+    prev_device = runtime.getDevice()
+    try:
+        runtime.setDevice(device.id)
+        return Array._new(np.zeros_like(x._array, dtype=dtype))
+    finally:
+        runtime.setDevice(prev_device)
diff --git a/cupy/array_api/_data_type_functions.py b/cupy/array_api/_data_type_functions.py
new file mode 100644
index 0000000..954e84a
--- /dev/null
+++ b/cupy/array_api/_data_type_functions.py
@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from ._array_object import Array
+from ._dtypes import _all_dtypes, _result_type
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Tuple, Union
+
+if TYPE_CHECKING:
+    from ._typing import Dtype
+    from collections.abc import Sequence
+
+import cupy as np
+
+
+# Note: astype is a function, not an array method as in NumPy.
+def astype(x: Array, dtype: Dtype, /, *, copy: bool = True) -> Array:
+    if not copy and dtype == x.dtype:
+        return x
+    return Array._new(x._array.astype(dtype=dtype, copy=copy))
+
+
+def broadcast_arrays(*arrays: Array) -> List[Array]:
+    """
+    Array API compatible wrapper for :py:func:`np.broadcast_arrays <numpy.broadcast_arrays>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    return [
+        Array._new(array) for array in np.broadcast_arrays(*[a._array for a in arrays])
+    ]
+
+
+def broadcast_to(x: Array, /, shape: Tuple[int, ...]) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.broadcast_to <numpy.broadcast_to>`.
+
+    See its docstring for more information.
+    """
+    from ._array_object import Array
+
+    return Array._new(np.broadcast_to(x._array, shape))
+
+
+def can_cast(from_: Union[Dtype, Array], to: Dtype, /) -> bool:
+    """
+    Array API compatible wrapper for :py:func:`np.can_cast <numpy.can_cast>`.
+
+    See its docstring for more information.
+    """
+    if isinstance(from_, Array):
+        from_ = from_.dtype
+    elif from_ not in _all_dtypes:
+        raise TypeError(f"{from_=}, but should be an array_api array or dtype")
+    if to not in _all_dtypes:
+        raise TypeError(f"{to=}, but should be a dtype")
+    # Note: We avoid np.can_cast() as it has discrepancies with the array API,
+    # since NumPy allows cross-kind casting (e.g., NumPy allows bool -> int8).
+    # See https://github.com/numpy/numpy/issues/20870
+    try:
+        # We promote `from_` and `to` together. We then check if the promoted
+        # dtype is `to`, which indicates if `from_` can (up)cast to `to`.
+        dtype = _result_type(from_, to)
+        return to == dtype
+    except TypeError:
+        # _result_type() raises if the dtypes don't promote together
+        return False
+
+
+# These are internal objects for the return types of finfo and iinfo, since
+# the NumPy versions contain extra data that isn't part of the spec.
+@dataclass
+class finfo_object:
+    bits: int
+    # Note: The types of the float data here are float, whereas in NumPy they
+    # are scalars of the corresponding float dtype.
+    eps: float
+    max: float
+    min: float
+    smallest_normal: float
+
+
+@dataclass
+class iinfo_object:
+    bits: int
+    max: int
+    min: int
+
+
+def finfo(type: Union[Dtype, Array], /) -> finfo_object:
+    """
+    Array API compatible wrapper for :py:func:`np.finfo <numpy.finfo>`.
+
+    See its docstring for more information.
+    """
+    fi = np.finfo(type)  # type: ignore
+    # Note: The types of the float data here are float, whereas in NumPy they
+    # are scalars of the corresponding float dtype.
+    try:
+        tiny = fi.smallest_normal
+    except AttributeError:  # for backward compatibility
+        tiny = fi.tiny
+    return finfo_object(
+        fi.bits,
+        float(fi.eps),
+        float(fi.max),
+        float(fi.min),
+        float(tiny),
+    )
+
+
+def iinfo(type: Union[Dtype, Array], /) -> iinfo_object:
+    """
+    Array API compatible wrapper for :py:func:`np.iinfo <numpy.iinfo>`.
+
+    See its docstring for more information.
+    """
+    ii = np.iinfo(type)  # type: ignore
+    return iinfo_object(ii.bits, ii.max, ii.min)
+
+
+def result_type(*arrays_and_dtypes: Union[Array, Dtype]) -> Dtype:
+    """
+    Array API compatible wrapper for :py:func:`np.result_type <numpy.result_type>`.
+
+    See its docstring for more information.
+    """
+    # Note: we use a custom implementation that gives only the type promotions
+    # required by the spec rather than using np.result_type. NumPy implements
+    # too many extra type promotions like int64 + uint64 -> float64, and does
+    # value-based casting on scalar arrays.
+    A = []
+    for a in arrays_and_dtypes:
+        if isinstance(a, Array):
+            a = a.dtype
+        elif isinstance(a, np.ndarray) or a not in _all_dtypes:
+            raise TypeError("result_type() inputs must be array_api arrays or dtypes")
+        A.append(a)
+
+    if len(A) == 0:
+        raise ValueError("at least one array or dtype is required")
+    elif len(A) == 1:
+        return A[0]
+    else:
+        t = A[0]
+        for t2 in A[1:]:
+            t = _result_type(t, t2)
+        return t
diff --git a/cupy/array_api/_dtypes.py b/cupy/array_api/_dtypes.py
new file mode 100644
index 0000000..8853510
--- /dev/null
+++ b/cupy/array_api/_dtypes.py
@@ -0,0 +1,143 @@
+import cupy as np
+
+# Note: we use dtype objects instead of dtype classes. The spec does not
+# require any behavior on dtypes other than equality.
+int8 = np.dtype("int8")
+int16 = np.dtype("int16")
+int32 = np.dtype("int32")
+int64 = np.dtype("int64")
+uint8 = np.dtype("uint8")
+uint16 = np.dtype("uint16")
+uint32 = np.dtype("uint32")
+uint64 = np.dtype("uint64")
+float32 = np.dtype("float32")
+float64 = np.dtype("float64")
+# Note: This name is changed
+bool = np.dtype("bool")
+
+_all_dtypes = (
+    int8,
+    int16,
+    int32,
+    int64,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    float32,
+    float64,
+    bool,
+)
+_boolean_dtypes = (bool,)
+_floating_dtypes = (float32, float64)
+_integer_dtypes = (int8, int16, int32, int64, uint8, uint16, uint32, uint64)
+_integer_or_boolean_dtypes = (
+    bool,
+    int8,
+    int16,
+    int32,
+    int64,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+)
+_numeric_dtypes = (
+    float32,
+    float64,
+    int8,
+    int16,
+    int32,
+    int64,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+)
+
+_dtype_categories = {
+    "all": _all_dtypes,
+    "numeric": _numeric_dtypes,
+    "integer": _integer_dtypes,
+    "integer or boolean": _integer_or_boolean_dtypes,
+    "boolean": _boolean_dtypes,
+    "floating-point": _floating_dtypes,
+}
+
+
+# Note: the spec defines a restricted type promotion table compared to NumPy.
+# In particular, cross-kind promotions like integer + float or boolean +
+# integer are not allowed, even for functions that accept both kinds.
+# Additionally, NumPy promotes signed integer + uint64 to float64, but this
+# promotion is not allowed here. To be clear, Python scalar int objects are
+# allowed to promote to floating-point dtypes, but only in array operators
+# (see Array._promote_scalar) method in _array_object.py.
+_promotion_table = {
+    (int8, int8): int8,
+    (int8, int16): int16,
+    (int8, int32): int32,
+    (int8, int64): int64,
+    (int16, int8): int16,
+    (int16, int16): int16,
+    (int16, int32): int32,
+    (int16, int64): int64,
+    (int32, int8): int32,
+    (int32, int16): int32,
+    (int32, int32): int32,
+    (int32, int64): int64,
+    (int64, int8): int64,
+    (int64, int16): int64,
+    (int64, int32): int64,
+    (int64, int64): int64,
+    (uint8, uint8): uint8,
+    (uint8, uint16): uint16,
+    (uint8, uint32): uint32,
+    (uint8, uint64): uint64,
+    (uint16, uint8): uint16,
+    (uint16, uint16): uint16,
+    (uint16, uint32): uint32,
+    (uint16, uint64): uint64,
+    (uint32, uint8): uint32,
+    (uint32, uint16): uint32,
+    (uint32, uint32): uint32,
+    (uint32, uint64): uint64,
+    (uint64, uint8): uint64,
+    (uint64, uint16): uint64,
+    (uint64, uint32): uint64,
+    (uint64, uint64): uint64,
+    (int8, uint8): int16,
+    (int8, uint16): int32,
+    (int8, uint32): int64,
+    (int16, uint8): int16,
+    (int16, uint16): int32,
+    (int16, uint32): int64,
+    (int32, uint8): int32,
+    (int32, uint16): int32,
+    (int32, uint32): int64,
+    (int64, uint8): int64,
+    (int64, uint16): int64,
+    (int64, uint32): int64,
+    (uint8, int8): int16,
+    (uint16, int8): int32,
+    (uint32, int8): int64,
+    (uint8, int16): int16,
+    (uint16, int16): int32,
+    (uint32, int16): int64,
+    (uint8, int32): int32,
+    (uint16, int32): int32,
+    (uint32, int32): int64,
+    (uint8, int64): int64,
+    (uint16, int64): int64,
+    (uint32, int64): int64,
+    (float32, float32): float32,
+    (float32, float64): float64,
+    (float64, float32): float64,
+    (float64, float64): float64,
+    (bool, bool): bool,
+}
+
+
+def _result_type(type1, type2):
+    if (type1, type2) in _promotion_table:
+        return _promotion_table[type1, type2]
+    raise TypeError(f"{type1} and {type2} cannot be type promoted together")
diff --git a/cupy/array_api/_elementwise_functions.py b/cupy/array_api/_elementwise_functions.py
new file mode 100644
index 0000000..063fa33
--- /dev/null
+++ b/cupy/array_api/_elementwise_functions.py
@@ -0,0 +1,729 @@
+from __future__ import annotations
+
+from ._dtypes import (
+    _boolean_dtypes,
+    _floating_dtypes,
+    _integer_dtypes,
+    _integer_or_boolean_dtypes,
+    _numeric_dtypes,
+    _result_type,
+)
+from ._array_object import Array
+
+import cupy as np
+
+
+def abs(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.abs <numpy.abs>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in abs")
+    return Array._new(np.abs(x._array))
+
+
+# Note: the function name is different here
+def acos(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.arccos <numpy.arccos>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in acos")
+    return Array._new(np.arccos(x._array))
+
+
+# Note: the function name is different here
+def acosh(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.arccosh <numpy.arccosh>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in acosh")
+    return Array._new(np.arccosh(x._array))
+
+
+def add(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.add <numpy.add>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in add")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.add(x1._array, x2._array))
+
+
+# Note: the function name is different here
+def asin(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.arcsin <numpy.arcsin>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in asin")
+    return Array._new(np.arcsin(x._array))
+
+
+# Note: the function name is different here
+def asinh(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.arcsinh <numpy.arcsinh>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in asinh")
+    return Array._new(np.arcsinh(x._array))
+
+
+# Note: the function name is different here
+def atan(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.arctan <numpy.arctan>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in atan")
+    return Array._new(np.arctan(x._array))
+
+
+# Note: the function name is different here
+def atan2(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.arctan2 <numpy.arctan2>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _floating_dtypes or x2.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in atan2")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.arctan2(x1._array, x2._array))
+
+
+# Note: the function name is different here
+def atanh(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.arctanh <numpy.arctanh>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in atanh")
+    return Array._new(np.arctanh(x._array))
+
+
+def bitwise_and(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.bitwise_and <numpy.bitwise_and>`.
+
+    See its docstring for more information.
+    """
+    if (
+        x1.dtype not in _integer_or_boolean_dtypes
+        or x2.dtype not in _integer_or_boolean_dtypes
+    ):
+        raise TypeError("Only integer or boolean dtypes are allowed in bitwise_and")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.bitwise_and(x1._array, x2._array))
+
+
+# Note: the function name is different here
+def bitwise_left_shift(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.left_shift <numpy.left_shift>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _integer_dtypes or x2.dtype not in _integer_dtypes:
+        raise TypeError("Only integer dtypes are allowed in bitwise_left_shift")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    # Note: bitwise_left_shift is only defined for x2 nonnegative.
+    if np.any(x2._array < 0):
+        raise ValueError("bitwise_left_shift(x1, x2) is only defined for x2 >= 0")
+    return Array._new(np.left_shift(x1._array, x2._array))
+
+
+# Note: the function name is different here
+def bitwise_invert(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.invert <numpy.invert>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _integer_or_boolean_dtypes:
+        raise TypeError("Only integer or boolean dtypes are allowed in bitwise_invert")
+    return Array._new(np.invert(x._array))
+
+
+def bitwise_or(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.bitwise_or <numpy.bitwise_or>`.
+
+    See its docstring for more information.
+    """
+    if (
+        x1.dtype not in _integer_or_boolean_dtypes
+        or x2.dtype not in _integer_or_boolean_dtypes
+    ):
+        raise TypeError("Only integer or boolean dtypes are allowed in bitwise_or")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.bitwise_or(x1._array, x2._array))
+
+
+# Note: the function name is different here
+def bitwise_right_shift(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.right_shift <numpy.right_shift>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _integer_dtypes or x2.dtype not in _integer_dtypes:
+        raise TypeError("Only integer dtypes are allowed in bitwise_right_shift")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    # Note: bitwise_right_shift is only defined for x2 nonnegative.
+    if np.any(x2._array < 0):
+        raise ValueError("bitwise_right_shift(x1, x2) is only defined for x2 >= 0")
+    return Array._new(np.right_shift(x1._array, x2._array))
+
+
+def bitwise_xor(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.bitwise_xor <numpy.bitwise_xor>`.
+
+    See its docstring for more information.
+    """
+    if (
+        x1.dtype not in _integer_or_boolean_dtypes
+        or x2.dtype not in _integer_or_boolean_dtypes
+    ):
+        raise TypeError("Only integer or boolean dtypes are allowed in bitwise_xor")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.bitwise_xor(x1._array, x2._array))
+
+
+def ceil(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.ceil <numpy.ceil>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in ceil")
+    if x.dtype in _integer_dtypes:
+        # Note: The return dtype of ceil is the same as the input
+        return x
+    return Array._new(np.ceil(x._array))
+
+
+def cos(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.cos <numpy.cos>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in cos")
+    return Array._new(np.cos(x._array))
+
+
+def cosh(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.cosh <numpy.cosh>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in cosh")
+    return Array._new(np.cosh(x._array))
+
+
+def divide(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.divide <numpy.divide>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _floating_dtypes or x2.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in divide")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.divide(x1._array, x2._array))
+
+
+def equal(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.equal <numpy.equal>`.
+
+    See its docstring for more information.
+    """
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.equal(x1._array, x2._array))
+
+
+def exp(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.exp <numpy.exp>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in exp")
+    return Array._new(np.exp(x._array))
+
+
+def expm1(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.expm1 <numpy.expm1>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in expm1")
+    return Array._new(np.expm1(x._array))
+
+
+def floor(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.floor <numpy.floor>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in floor")
+    if x.dtype in _integer_dtypes:
+        # Note: The return dtype of floor is the same as the input
+        return x
+    return Array._new(np.floor(x._array))
+
+
+def floor_divide(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.floor_divide <numpy.floor_divide>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in floor_divide")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.floor_divide(x1._array, x2._array))
+
+
+def greater(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.greater <numpy.greater>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in greater")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.greater(x1._array, x2._array))
+
+
+def greater_equal(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.greater_equal <numpy.greater_equal>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in greater_equal")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.greater_equal(x1._array, x2._array))
+
+
+def isfinite(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.isfinite <numpy.isfinite>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in isfinite")
+    return Array._new(np.isfinite(x._array))
+
+
+def isinf(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.isinf <numpy.isinf>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in isinf")
+    return Array._new(np.isinf(x._array))
+
+
+def isnan(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.isnan <numpy.isnan>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in isnan")
+    return Array._new(np.isnan(x._array))
+
+
+def less(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.less <numpy.less>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in less")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.less(x1._array, x2._array))
+
+
+def less_equal(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.less_equal <numpy.less_equal>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in less_equal")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.less_equal(x1._array, x2._array))
+
+
+def log(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.log <numpy.log>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in log")
+    return Array._new(np.log(x._array))
+
+
+def log1p(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.log1p <numpy.log1p>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in log1p")
+    return Array._new(np.log1p(x._array))
+
+
+def log2(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.log2 <numpy.log2>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in log2")
+    return Array._new(np.log2(x._array))
+
+
+def log10(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.log10 <numpy.log10>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in log10")
+    return Array._new(np.log10(x._array))
+
+
+def logaddexp(x1: Array, x2: Array) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.logaddexp <numpy.logaddexp>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _floating_dtypes or x2.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in logaddexp")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.logaddexp(x1._array, x2._array))
+
+
+def logical_and(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.logical_and <numpy.logical_and>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _boolean_dtypes or x2.dtype not in _boolean_dtypes:
+        raise TypeError("Only boolean dtypes are allowed in logical_and")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.logical_and(x1._array, x2._array))
+
+
+def logical_not(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.logical_not <numpy.logical_not>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _boolean_dtypes:
+        raise TypeError("Only boolean dtypes are allowed in logical_not")
+    return Array._new(np.logical_not(x._array))
+
+
+def logical_or(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.logical_or <numpy.logical_or>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _boolean_dtypes or x2.dtype not in _boolean_dtypes:
+        raise TypeError("Only boolean dtypes are allowed in logical_or")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.logical_or(x1._array, x2._array))
+
+
+def logical_xor(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.logical_xor <numpy.logical_xor>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _boolean_dtypes or x2.dtype not in _boolean_dtypes:
+        raise TypeError("Only boolean dtypes are allowed in logical_xor")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.logical_xor(x1._array, x2._array))
+
+
+def multiply(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.multiply <numpy.multiply>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in multiply")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.multiply(x1._array, x2._array))
+
+
+def negative(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.negative <numpy.negative>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in negative")
+    return Array._new(np.negative(x._array))
+
+
+def not_equal(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.not_equal <numpy.not_equal>`.
+
+    See its docstring for more information.
+    """
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.not_equal(x1._array, x2._array))
+
+
+def positive(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.positive <numpy.positive>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in positive")
+    return Array._new(np.positive(x._array))
+
+
+# Note: the function name is different here
+def pow(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.power <numpy.power>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in pow")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.power(x1._array, x2._array))
+
+
+def remainder(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.remainder <numpy.remainder>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in remainder")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.remainder(x1._array, x2._array))
+
+
+def round(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.round <numpy.round>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in round")
+    return Array._new(np.round(x._array))
+
+
+def sign(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.sign <numpy.sign>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in sign")
+    return Array._new(np.sign(x._array))
+
+
+def sin(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.sin <numpy.sin>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in sin")
+    return Array._new(np.sin(x._array))
+
+
+def sinh(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.sinh <numpy.sinh>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in sinh")
+    return Array._new(np.sinh(x._array))
+
+
+def square(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.square <numpy.square>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in square")
+    return Array._new(np.square(x._array))
+
+
+def sqrt(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.sqrt <numpy.sqrt>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in sqrt")
+    return Array._new(np.sqrt(x._array))
+
+
+def subtract(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.subtract <numpy.subtract>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in subtract")
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.subtract(x1._array, x2._array))
+
+
+def tan(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.tan <numpy.tan>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in tan")
+    return Array._new(np.tan(x._array))
+
+
+def tanh(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.tanh <numpy.tanh>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in tanh")
+    return Array._new(np.tanh(x._array))
+
+
+def trunc(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.trunc <numpy.trunc>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in trunc")
+    if x.dtype in _integer_dtypes:
+        # Note: The return dtype of trunc is the same as the input
+        return x
+    return Array._new(np.trunc(x._array))
diff --git a/cupy/array_api/_indexing_functions.py b/cupy/array_api/_indexing_functions.py
new file mode 100644
index 0000000..be38fcc
--- /dev/null
+++ b/cupy/array_api/_indexing_functions.py
@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from ._array_object import Array
+from ._dtypes import _integer_dtypes
+
+import cupy as np
+
+def take(x: Array, indices: Array, /, *, axis: int) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.take <numpy.take>`.
+    See its docstring for more information.
+    """    
+    if indices.dtype not in _integer_dtypes:
+        raise TypeError("Only integer dtypes are allowed in indexing")
+    if indices.ndim != 1: 
+        raise ValueError("Only 1-dim indices array is supported")
+    return Array._new(np.take(x._array, indices._array, axis=axis))
diff --git a/cupy/array_api/_manipulation_functions.py b/cupy/array_api/_manipulation_functions.py
new file mode 100644
index 0000000..bbe3850
--- /dev/null
+++ b/cupy/array_api/_manipulation_functions.py
@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from ._array_object import Array
+from ._data_type_functions import result_type
+
+from typing import List, Optional, Tuple, Union
+
+import cupy as np
+
+# Note: the function name is different here
+def concat(
+    arrays: Union[Tuple[Array, ...], List[Array]], /, *, axis: Optional[int] = 0
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.concatenate <numpy.concatenate>`.
+
+    See its docstring for more information.
+    """
+    # Note: Casting rules here are different from the np.concatenate default
+    # (no for scalars with axis=None, no cross-kind casting)
+    dtype = result_type(*arrays)
+    arrays = tuple(a._array for a in arrays)
+    return Array._new(np.concatenate(arrays, axis=axis, dtype=dtype))
+
+
+def expand_dims(x: Array, /, *, axis: int) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.expand_dims <numpy.expand_dims>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.expand_dims(x._array, axis))
+
+
+def flip(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.flip <numpy.flip>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.flip(x._array, axis=axis))
+
+
+# Note: The function name is different here (see also matrix_transpose).
+# Unlike transpose(), the axes argument is required.
+def permute_dims(x: Array, /, axes: Tuple[int, ...]) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.transpose <numpy.transpose>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.transpose(x._array, axes))
+
+
+# Note: the optional argument is called 'shape', not 'newshape'
+def reshape(x: Array, /, shape: Tuple[int, ...]) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.reshape <numpy.reshape>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.reshape(x._array, shape))
+
+
+def roll(
+    x: Array,
+    /,
+    shift: Union[int, Tuple[int, ...]],
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.roll <numpy.roll>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.roll(x._array, shift, axis=axis))
+
+
+def squeeze(x: Array, /, axis: Union[int, Tuple[int, ...]]) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.squeeze <numpy.squeeze>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.squeeze(x._array, axis=axis))
+
+
+def stack(arrays: Union[Tuple[Array, ...], List[Array]], /, *, axis: int = 0) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.stack <numpy.stack>`.
+
+    See its docstring for more information.
+    """
+    # Call result type here just to raise on disallowed type combinations
+    result_type(*arrays)
+    arrays = tuple(a._array for a in arrays)
+    return Array._new(np.stack(arrays, axis=axis))
diff --git a/cupy/array_api/_searching_functions.py b/cupy/array_api/_searching_functions.py
new file mode 100644
index 0000000..6657ae8
--- /dev/null
+++ b/cupy/array_api/_searching_functions.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from ._array_object import Array
+from ._dtypes import _result_type
+
+from typing import Optional, Tuple
+
+import cupy as np
+
+
+def argmax(x: Array, /, *, axis: Optional[int] = None, keepdims: bool = False) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.argmax <numpy.argmax>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.asarray(np.argmax(x._array, axis=axis, keepdims=keepdims)))
+
+
+def argmin(x: Array, /, *, axis: Optional[int] = None, keepdims: bool = False) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.argmin <numpy.argmin>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.asarray(np.argmin(x._array, axis=axis, keepdims=keepdims)))
+
+
+def nonzero(x: Array, /) -> Tuple[Array, ...]:
+    """
+    Array API compatible wrapper for :py:func:`np.nonzero <numpy.nonzero>`.
+
+    See its docstring for more information.
+    """
+    return tuple(Array._new(i) for i in np.nonzero(x._array))
+
+
+def where(condition: Array, x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.where <numpy.where>`.
+
+    See its docstring for more information.
+    """
+    # Call result type here just to raise on disallowed type combinations
+    _result_type(x1.dtype, x2.dtype)
+    x1, x2 = Array._normalize_two_args(x1, x2)
+    return Array._new(np.where(condition._array, x1._array, x2._array))
diff --git a/cupy/array_api/_set_functions.py b/cupy/array_api/_set_functions.py
new file mode 100644
index 0000000..faf122c
--- /dev/null
+++ b/cupy/array_api/_set_functions.py
@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from ._array_object import Array
+
+from typing import NamedTuple
+
+import cupy as np
+
+# Note: np.unique() is split into four functions in the array API:
+# unique_all, unique_counts, unique_inverse, and unique_values (this is done
+# to remove polymorphic return types).
+
+# Note: The various unique() functions are supposed to return multiple NaNs.
+# This does not match the NumPy behavior, however, this is currently left as a
+# TODO in this implementation as this behavior may be reverted in np.unique().
+# See https://github.com/numpy/numpy/issues/20326.
+
+# Note: The functions here return a namedtuple (np.unique() returns a normal
+# tuple).
+
+class UniqueAllResult(NamedTuple):
+    values: Array
+    indices: Array
+    inverse_indices: Array
+    counts: Array
+
+
+class UniqueCountsResult(NamedTuple):
+    values: Array
+    counts: Array
+
+
+class UniqueInverseResult(NamedTuple):
+    values: Array
+    inverse_indices: Array
+
+
+def unique_all(x: Array, /) -> UniqueAllResult:
+    """
+    Array API compatible wrapper for :py:func:`np.unique <numpy.unique>`.
+
+    See its docstring for more information.
+    """
+    values, indices, inverse_indices, counts = np.unique(
+        x._array,
+        return_counts=True,
+        return_index=True,
+        return_inverse=True,
+        equal_nan=False,
+    )
+    # np.unique() flattens inverse indices, but they need to share x's shape
+    # See https://github.com/numpy/numpy/issues/20638
+    inverse_indices = inverse_indices.reshape(x.shape)
+    return UniqueAllResult(
+        Array._new(values),
+        Array._new(indices),
+        Array._new(inverse_indices),
+        Array._new(counts),
+    )
+
+
+def unique_counts(x: Array, /) -> UniqueCountsResult:
+    res = np.unique(
+        x._array,
+        return_counts=True,
+        return_index=False,
+        return_inverse=False,
+        equal_nan=False,
+    )
+
+    return UniqueCountsResult(*[Array._new(i) for i in res])
+
+
+def unique_inverse(x: Array, /) -> UniqueInverseResult:
+    """
+    Array API compatible wrapper for :py:func:`np.unique <numpy.unique>`.
+
+    See its docstring for more information.
+    """
+    values, inverse_indices = np.unique(
+        x._array,
+        return_counts=False,
+        return_index=False,
+        return_inverse=True,
+        equal_nan=False,
+    )
+    # np.unique() flattens inverse indices, but they need to share x's shape
+    # See https://github.com/numpy/numpy/issues/20638
+    inverse_indices = inverse_indices.reshape(x.shape)
+    return UniqueInverseResult(Array._new(values), Array._new(inverse_indices))
+
+
+def unique_values(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.unique <numpy.unique>`.
+
+    See its docstring for more information.
+    """
+    res = np.unique(
+        x._array,
+        return_counts=False,
+        return_index=False,
+        return_inverse=False,
+        equal_nan=False,
+    )
+    return Array._new(res)
diff --git a/cupy/array_api/_sorting_functions.py b/cupy/array_api/_sorting_functions.py
new file mode 100644
index 0000000..ce62a2a
--- /dev/null
+++ b/cupy/array_api/_sorting_functions.py
@@ -0,0 +1,53 @@
+# mypy: ignore-errors
+
+from __future__ import annotations
+
+from ._array_object import Array
+
+import cupy as np
+
+
+# Note: the descending keyword argument is new in this function
+def argsort(
+    x: Array, /, *, axis: int = -1, descending: bool = False, stable: bool = True
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.argsort <numpy.argsort>`.
+
+    See its docstring for more information.
+    """
+    # Note: Unlike in NumPy we only support kind={None, 'stable'}, but the standard
+    # does *not* require we need to support unstable sort.
+    kind = None
+    if not descending:
+        res = np.argsort(x._array, axis=axis, kind=kind)
+    else:
+        # As NumPy has no native descending sort, we imitate it here. Note that
+        # simply flipping the results of np.argsort(x._array, ...) would not
+        # respect the relative order like it would in native descending sorts.
+        res = np.flip(
+            np.argsort(np.flip(x._array, axis=axis), axis=axis, kind=kind),
+            axis=axis,
+        )
+        # Rely on flip()/argsort() to validate axis
+        normalised_axis = axis if axis >= 0 else x.ndim + axis
+        max_i = x.shape[normalised_axis] - 1
+        res = max_i - res
+    return Array._new(res)
+
+# Note: the descending keyword argument is new in this function
+def sort(
+    x: Array, /, *, axis: int = -1, descending: bool = False, stable: bool = True
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.sort <numpy.sort>`.
+
+    See its docstring for more information.
+    """
+    # Note: Unlike in NumPy we only support kind={None, 'stable'}, but the standard
+    # does *not* require we need to support unstable sort.
+    kind = None
+    res = np.sort(x._array, axis=axis, kind=kind)
+    if descending:
+        res = np.flip(res, axis=axis)
+    return Array._new(res)
diff --git a/cupy/array_api/_statistical_functions.py b/cupy/array_api/_statistical_functions.py
new file mode 100644
index 0000000..6f9c4b7
--- /dev/null
+++ b/cupy/array_api/_statistical_functions.py
@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from ._dtypes import (
+    _floating_dtypes,
+    _numeric_dtypes,
+)
+from ._array_object import Array
+from ._creation_functions import asarray
+from ._dtypes import float32, float64
+
+from typing import TYPE_CHECKING, Optional, Tuple, Union
+
+if TYPE_CHECKING:
+    from ._typing import Dtype
+
+import cupy as np
+
+
+def max(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+) -> Array:
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in max")
+    return Array._new(np.max(x._array, axis=axis, keepdims=keepdims))
+
+
+def mean(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+) -> Array:
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in mean")
+    return Array._new(np.mean(x._array, axis=axis, keepdims=keepdims))
+
+
+def min(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+) -> Array:
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in min")
+    return Array._new(np.min(x._array, axis=axis, keepdims=keepdims))
+
+
+def prod(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    dtype: Optional[Dtype] = None,
+    keepdims: bool = False,
+) -> Array:
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in prod")
+    # Note: sum() and prod() always upcast float32 to float64 for dtype=None
+    # We need to do so here before computing the product to avoid overflow
+    if dtype is None and x.dtype == float32:
+        dtype = float64
+    return Array._new(np.prod(x._array, dtype=dtype, axis=axis, keepdims=keepdims))
+
+
+def std(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    correction: Union[int, float] = 0.0,
+    keepdims: bool = False,
+) -> Array:
+    # Note: the keyword argument correction is different here
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in std")
+    return Array._new(np.std(x._array, axis=axis, ddof=correction, keepdims=keepdims))
+
+
+def sum(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    dtype: Optional[Dtype] = None,
+    keepdims: bool = False,
+) -> Array:
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError("Only numeric dtypes are allowed in sum")
+    # Note: sum() and prod() always upcast integers to (u)int64 and float32 to
+    # float64 for dtype=None. `np.sum` does that too for integers, but not for
+    # float32, so we need to special-case it here
+    if dtype is None and x.dtype == float32:
+        dtype = float64
+    return Array._new(np.sum(x._array, axis=axis, dtype=dtype, keepdims=keepdims))
+
+
+def var(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    correction: Union[int, float] = 0.0,
+    keepdims: bool = False,
+) -> Array:
+    # Note: the keyword argument correction is different here
+    if x.dtype not in _floating_dtypes:
+        raise TypeError("Only floating-point dtypes are allowed in var")
+    return Array._new(np.var(x._array, axis=axis, ddof=correction, keepdims=keepdims))
diff --git a/cupy/array_api/_typing.py b/cupy/array_api/_typing.py
new file mode 100644
index 0000000..e5289b7
--- /dev/null
+++ b/cupy/array_api/_typing.py
@@ -0,0 +1,75 @@
+"""
+This file defines the types for type annotations.
+
+These names aren't part of the module namespace, but they are used in the
+annotations in the function signatures. The functions in the module are only
+valid for inputs that match the given type annotations.
+"""
+
+from __future__ import annotations
+from cupy.cuda import Device
+
+
+__all__ = [
+    "Array",
+    "Device",
+    "Dtype",
+    "SupportsDLPack",
+    "SupportsBufferProtocol",
+    "PyCapsule",
+]
+
+import sys
+from typing import (
+    Any,
+    Literal,
+    Sequence,
+    Type,
+    Union,
+    TYPE_CHECKING,
+    TypeVar,
+    Protocol,
+)
+
+from ._array_object import Array
+from numpy import (
+    dtype,
+    int8,
+    int16,
+    int32,
+    int64,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    float32,
+    float64,
+)
+
+_T_co = TypeVar("_T_co", covariant=True)
+
+class NestedSequence(Protocol[_T_co]):
+    def __getitem__(self, key: int, /) -> _T_co | NestedSequence[_T_co]: ...
+    def __len__(self, /) -> int: ...
+
+if TYPE_CHECKING or sys.version_info >= (3, 9):
+    Dtype = dtype[Union[
+        int8,
+        int16,
+        int32,
+        int64,
+        uint8,
+        uint16,
+        uint32,
+        uint64,
+        float32,
+        float64,
+    ]]
+else:
+    Dtype = dtype
+
+SupportsBufferProtocol = Any
+PyCapsule = Any
+
+class SupportsDLPack(Protocol):
+    def __dlpack__(self, /, *, stream: None = ...) -> PyCapsule: ...
diff --git a/cupy/array_api/_utility_functions.py b/cupy/array_api/_utility_functions.py
new file mode 100644
index 0000000..cdcbbc8
--- /dev/null
+++ b/cupy/array_api/_utility_functions.py
@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from ._array_object import Array
+
+from typing import Optional, Tuple, Union
+
+import cupy as np
+
+
+def all(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.all <numpy.all>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.asarray(np.all(x._array, axis=axis, keepdims=keepdims)))
+
+
+def any(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.any <numpy.any>`.
+
+    See its docstring for more information.
+    """
+    return Array._new(np.asarray(np.any(x._array, axis=axis, keepdims=keepdims)))
diff --git a/cupy/array_api/linalg.py b/cupy/array_api/linalg.py
new file mode 100644
index 0000000..9eaacd1
--- /dev/null
+++ b/cupy/array_api/linalg.py
@@ -0,0 +1,445 @@
+from __future__ import annotations
+
+import functools
+
+from ._dtypes import _floating_dtypes, _numeric_dtypes
+from ._manipulation_functions import reshape
+from ._array_object import Array
+
+from .._core.internal import _normalize_axis_indices as normalize_axis_tuple
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from typing import Literal, Optional, Sequence, Tuple, Union
+
+from typing import NamedTuple
+
+import cupy as np
+
+class EighResult(NamedTuple):
+    eigenvalues: Array
+    eigenvectors: Array
+
+class QRResult(NamedTuple):
+    Q: Array
+    R: Array
+
+class SlogdetResult(NamedTuple):
+    sign: Array
+    logabsdet: Array
+
+class SVDResult(NamedTuple):
+    U: Array
+    S: Array
+    Vh: Array
+
+# Note: the inclusion of the upper keyword is different from
+# np.linalg.cholesky, which does not have it.
+def cholesky(x: Array, /, *, upper: bool = False) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.cholesky <numpy.linalg.cholesky>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.cholesky.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in cholesky')
+    L = np.linalg.cholesky(x._array)
+    if upper:
+        return Array._new(L).mT
+    return Array._new(L)
+
+# Note: cross is the numpy top-level namespace, not np.linalg
+def cross(x1: Array, x2: Array, /, *, axis: int = -1) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.cross <numpy.cross>`.
+
+    See its docstring for more information.
+    """
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError('Only numeric dtypes are allowed in cross')
+    # Note: this is different from np.cross(), which broadcasts
+    if x1.shape != x2.shape:
+        raise ValueError('x1 and x2 must have the same shape')
+    if x1.ndim == 0:
+        raise ValueError('cross() requires arrays of dimension at least 1')
+    # Note: this is different from np.cross(), which allows dimension 2
+    if x1.shape[axis] != 3:
+        raise ValueError('cross() dimension must equal 3')
+    return Array._new(np.cross(x1._array, x2._array, axis=axis))
+
+def det(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.det <numpy.linalg.det>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.det.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in det')
+    return Array._new(np.linalg.det(x._array))
+
+# Note: diagonal is the numpy top-level namespace, not np.linalg
+def diagonal(x: Array, /, *, offset: int = 0) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.diagonal <numpy.diagonal>`.
+
+    See its docstring for more information.
+    """
+    # Note: diagonal always operates on the last two axes, whereas np.diagonal
+    # operates on the first two axes by default
+    return Array._new(np.diagonal(x._array, offset=offset, axis1=-2, axis2=-1))
+
+
+def eigh(x: Array, /) -> EighResult:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.eigh <numpy.linalg.eigh>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.eigh.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in eigh')
+
+    # Note: the return type here is a namedtuple, which is different from
+    # np.eigh, which only returns a tuple.
+    return EighResult(*map(Array._new, np.linalg.eigh(x._array)))
+
+
+def eigvalsh(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.eigvalsh <numpy.linalg.eigvalsh>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.eigvalsh.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in eigvalsh')
+
+    return Array._new(np.linalg.eigvalsh(x._array))
+
+def inv(x: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.inv <numpy.linalg.inv>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.inv.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in inv')
+
+    return Array._new(np.linalg.inv(x._array))
+
+
+# Note: matmul is the numpy top-level namespace but not in np.linalg
+def matmul(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.matmul <numpy.matmul>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to numeric dtypes only is different from
+    # np.matmul.
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError('Only numeric dtypes are allowed in matmul')
+
+    return Array._new(np.matmul(x1._array, x2._array))
+
+
+# Note: the name here is different from norm(). The array API norm is split
+# into matrix_norm and vector_norm().
+
+# The type for ord should be Optional[Union[int, float, Literal[np.inf,
+# -np.inf, 'fro', 'nuc']]], but Literal does not support floating-point
+# literals.
+def matrix_norm(x: Array, /, *, keepdims: bool = False, ord: Optional[Union[int, float, Literal['fro', 'nuc']]] = 'fro') -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.norm <numpy.linalg.norm>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.norm.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in matrix_norm')
+
+    return Array._new(np.linalg.norm(x._array, axis=(-2, -1), keepdims=keepdims, ord=ord))
+
+
+def matrix_power(x: Array, n: int, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.matrix_power <numpy.matrix_power>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.matrix_power.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed for the first argument of matrix_power')
+
+    # np.matrix_power already checks if n is an integer
+    return Array._new(np.linalg.matrix_power(x._array, n))
+
+# Note: the keyword argument name rtol is different from np.linalg.matrix_rank
+def matrix_rank(x: Array, /, *, rtol: Optional[Union[float, Array]] = None) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.matrix_rank <numpy.matrix_rank>`.
+
+    See its docstring for more information.
+    """
+    # Note: this is different from np.linalg.matrix_rank, which supports 1
+    # dimensional arrays.
+    if x.ndim < 2:
+        raise np.linalg.LinAlgError("1-dimensional array given. Array must be at least two-dimensional")
+    S = np.linalg.svd(x._array, compute_uv=False)
+    if rtol is None:
+        tol = S.max(axis=-1, keepdims=True) * max(x.shape[-2:]) * np.finfo(S.dtype).eps
+    else:
+        if isinstance(rtol, Array):
+            rtol = rtol._array
+        # Note: this is different from np.linalg.matrix_rank, which does not multiply
+        # the tolerance by the largest singular value.
+        tol = S.max(axis=-1, keepdims=True)*np.asarray(rtol)[..., np.newaxis]
+    return Array._new(np.count_nonzero(S > tol, axis=-1))
+
+
+# Note: this function is new in the array API spec. Unlike transpose, it only
+# transposes the last two axes.
+def matrix_transpose(x: Array, /) -> Array:
+    if x.ndim < 2:
+        raise ValueError("x must be at least 2-dimensional for matrix_transpose")
+    return Array._new(np.swapaxes(x._array, -1, -2))
+
+# Note: outer is the numpy top-level namespace, not np.linalg
+def outer(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.outer <numpy.outer>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to numeric dtypes only is different from
+    # np.outer.
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError('Only numeric dtypes are allowed in outer')
+
+    # Note: the restriction to only 1-dim arrays is different from np.outer
+    if x1.ndim != 1 or x2.ndim != 1:
+        raise ValueError('The input arrays to outer must be 1-dimensional')
+
+    return Array._new(np.outer(x1._array, x2._array))
+
+# Note: the keyword argument name rtol is different from np.linalg.pinv
+def pinv(x: Array, /, *, rtol: Optional[Union[float, Array]] = None) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.pinv <numpy.linalg.pinv>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.pinv.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in pinv')
+
+    # Note: this is different from np.linalg.pinv, which does not multiply the
+    # default tolerance by max(M, N).
+    if rtol is None:
+        rtol = max(x.shape[-2:]) * np.finfo(x.dtype).eps  # type: ignore
+    return Array._new(np.linalg.pinv(x._array, rcond=rtol))
+
+def qr(x: Array, /, *, mode: Literal['reduced', 'complete'] = 'reduced') -> QRResult:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.qr <numpy.linalg.qr>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.qr.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in qr')
+
+    # Note: the return type here is a namedtuple, which is different from
+    # np.linalg.qr, which only returns a tuple.
+    return QRResult(*map(Array._new, np.linalg.qr(x._array, mode=mode)))
+
+def slogdet(x: Array, /) -> SlogdetResult:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.slogdet <numpy.linalg.slogdet>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.slogdet.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in slogdet')
+
+    # Note: the return type here is a namedtuple, which is different from
+    # np.linalg.slogdet, which only returns a tuple.
+    return SlogdetResult(*map(Array._new, np.linalg.slogdet(x._array)))
+
+# Note: unlike np.linalg.solve, the array API solve() only accepts x2 as a
+# vector when it is exactly 1-dimensional. All other cases treat x2 as a stack
+# of matrices. The np.linalg.solve behavior of allowing stacks of both
+# matrices and vectors is ambiguous c.f.
+# https://github.com/numpy/numpy/issues/15349 and
+# https://github.com/data-apis/array-api/issues/285.
+
+# Note: The impl below is deviated from numpy.array_api's.
+def _solve(a, b):
+    from cupy.linalg._util import (
+        _assert_stacked_2d, _assert_stacked_square, linalg_common_type)
+
+    _commonType = functools.partial(linalg_common_type, reject_float16=False)
+
+    _assert_stacked_2d(a)
+    _assert_stacked_square(a)
+    _, result_t = _commonType(a, b)
+
+    if b.ndim == 1:
+        # (M,) -> (M, 1)
+        old_shape = b.shape
+        b = b.reshape(-1, 1)
+    elif b.ndim == a.ndim - 1:
+        # x1 has shape (M, M) or (..., M, M)
+        raise ValueError('x2 must have shape (M,) or (..., M, K); '
+                         '(..., M) is not allowed')
+    else:
+        # (..., M, K) => no change
+        old_shape = None
+
+    r = np.linalg.solve(a, b).astype(result_t, copy=False)
+    r = r.reshape(old_shape) if old_shape else r
+    return r
+
+def solve(x1: Array, x2: Array, /) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.solve <numpy.linalg.solve>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.solve.
+    if x1.dtype not in _floating_dtypes or x2.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in solve')
+
+    return Array._new(_solve(x1._array, x2._array))
+
+def svd(x: Array, /, *, full_matrices: bool = True) -> SVDResult:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.svd <numpy.linalg.svd>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.svd.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in svd')
+
+    # Note: the return type here is a namedtuple, which is different from
+    # np.svd, which only returns a tuple.
+    return SVDResult(*map(Array._new, np.linalg.svd(x._array, full_matrices=full_matrices)))
+
+# Note: svdvals is not in NumPy (but it is in SciPy). It is equivalent to
+# np.linalg.svd(compute_uv=False).
+def svdvals(x: Array, /) -> Union[Array, Tuple[Array, ...]]:
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in svdvals')
+    return Array._new(np.linalg.svd(x._array, compute_uv=False))
+
+# Note: tensordot is the numpy top-level namespace but not in np.linalg
+
+# Note: axes must be a tuple, unlike np.tensordot where it can be an array or array-like.
+def tensordot(x1: Array, x2: Array, /, *, axes: Union[int, Tuple[Sequence[int], Sequence[int]]] = 2) -> Array:
+    # Note: the restriction to numeric dtypes only is different from
+    # np.tensordot.
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError('Only numeric dtypes are allowed in tensordot')
+
+    return Array._new(np.tensordot(x1._array, x2._array, axes=axes))
+
+# Note: trace is the numpy top-level namespace, not np.linalg
+def trace(x: Array, /, *, offset: int = 0) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.trace <numpy.trace>`.
+
+    See its docstring for more information.
+    """
+    if x.dtype not in _numeric_dtypes:
+        raise TypeError('Only numeric dtypes are allowed in trace')
+    # Note: trace always operates on the last two axes, whereas np.trace
+    # operates on the first two axes by default
+    return Array._new(np.asarray(np.trace(x._array, offset=offset, axis1=-2, axis2=-1)))
+
+# Note: vecdot is not in NumPy
+def vecdot(x1: Array, x2: Array, /, *, axis: int = -1) -> Array:
+    if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes:
+        raise TypeError('Only numeric dtypes are allowed in vecdot')
+    ndim = max(x1.ndim, x2.ndim)
+    x1_shape = (1,)*(ndim - x1.ndim) + tuple(x1.shape)
+    x2_shape = (1,)*(ndim - x2.ndim) + tuple(x2.shape)
+    if x1_shape[axis] != x2_shape[axis]:
+        raise ValueError("x1 and x2 must have the same size along the given axis")
+
+    x1_, x2_ = np.broadcast_arrays(x1._array, x2._array)
+    x1_ = np.moveaxis(x1_, axis, -1)
+    x2_ = np.moveaxis(x2_, axis, -1)
+
+    res = x1_[..., None, :] @ x2_[..., None]
+    return Array._new(res[..., 0, 0])
+
+
+# Note: the name here is different from norm(). The array API norm is split
+# into matrix_norm and vector_norm().
+
+# The type for ord should be Optional[Union[int, float, Literal[np.inf,
+# -np.inf]]] but Literal does not support floating-point literals.
+def vector_norm(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None, keepdims: bool = False, ord: Optional[Union[int, float]] = 2) -> Array:
+    """
+    Array API compatible wrapper for :py:func:`np.linalg.norm <numpy.linalg.norm>`.
+
+    See its docstring for more information.
+    """
+    # Note: the restriction to floating-point dtypes only is different from
+    # np.linalg.norm.
+    if x.dtype not in _floating_dtypes:
+        raise TypeError('Only floating-point dtypes are allowed in norm')
+
+    # np.linalg.norm tries to do a matrix norm whenever axis is a 2-tuple or
+    # when axis=None and the input is 2-D, so to force a vector norm, we make
+    # it so the input is 1-D (for axis=None), or reshape so that norm is done
+    # on a single dimension.
+    a = x._array
+    if axis is None:
+        # Note: np.linalg.norm() doesn't handle 0-D arrays
+        a = a.ravel()
+        _axis = 0
+    elif isinstance(axis, tuple):
+        # Note: The axis argument supports any number of axes, whereas
+        # np.linalg.norm() only supports a single axis for vector norm.
+        normalized_axis = normalize_axis_tuple(axis, x.ndim)
+        rest = tuple(i for i in range(a.ndim) if i not in normalized_axis)
+        newshape = axis + rest
+        a = np.transpose(a, newshape).reshape(
+            (np.prod([a.shape[i] for i in axis], dtype=int), *[a.shape[i] for i in rest]))
+        _axis = 0
+    else:
+        _axis = axis
+
+    res = Array._new(np.linalg.norm(a, axis=_axis, ord=ord))
+
+    if keepdims:
+        # We can't reuse np.linalg.norm(keepdims) because of the reshape hacks
+        # above to avoid matrix norm logic.
+        shape = list(x.shape)
+        _axis = normalize_axis_tuple(range(x.ndim) if axis is None else axis, x.ndim)
+        for i in _axis:
+            shape[i] = 1
+        res = reshape(res, tuple(shape))
+
+    return res
+
+__all__ = ['cholesky', 'cross', 'det', 'diagonal', 'eigh', 'eigvalsh', 'inv', 'matmul', 'matrix_norm', 'matrix_power', 'matrix_rank', 'matrix_transpose', 'outer', 'pinv', 'qr', 'slogdet', 'solve', 'svd', 'svdvals', 'tensordot', 'trace', 'vecdot', 'vector_norm']
diff --git a/cupy/cublas.py b/cupy/cublas.py
new file mode 100644
index 0000000..e5b2149
--- /dev/null
+++ b/cupy/cublas.py
@@ -0,0 +1,1012 @@
+import numpy
+from numpy import linalg
+
+import warnings
+
+import cupy
+from cupy import _core
+from cupy_backends.cuda.libs import cublas
+from cupy.cuda import device
+from cupy.linalg import _util
+
+_batched_gesv_limit = 256
+
+
+def get_batched_gesv_limit():
+    global _batched_gesv_limit
+    return _batched_gesv_limit
+
+
+def set_batched_gesv_limit(limit):
+    global _batched_gesv_limit
+    _batched_gesv_limit = limit
+
+
+def batched_gesv(a, b):
+    """Solves multiple linear matrix equations using cublas<t>getr[fs]Batched().
+
+    Computes the solution to system of linear equation ``ax = b``.
+
+    Args:
+        a (cupy.ndarray): The matrix with dimension ``(..., M, M)``.
+        b (cupy.ndarray): The matrix with dimension ``(..., M)`` or
+            ``(..., M, K)``.
+
+    Returns:
+        cupy.ndarray:
+            The matrix with dimension ``(..., M)`` or ``(..., M, K)``.
+    """  # NOQA
+    _util._assert_cupy_array(a, b)
+    _util._assert_stacked_2d(a)
+    _util._assert_stacked_square(a)
+
+    # TODO(kataoka): Support broadcast
+    if not (
+        (a.ndim == b.ndim or a.ndim == b.ndim + 1)
+        and a.shape[:-1] == b.shape[:a.ndim - 1]
+    ):
+        raise ValueError(
+            'a must have (..., M, M) shape and b must have (..., M) '
+            'or (..., M, K)')
+
+    dtype, out_dtype = _util.linalg_common_type(a, b)
+    if b.size == 0:
+        return cupy.empty(b.shape, out_dtype)
+
+    if dtype == 'f':
+        t = 's'
+    elif dtype == 'd':
+        t = 'd'
+    elif dtype == 'F':
+        t = 'c'
+    elif dtype == 'D':
+        t = 'z'
+    else:
+        raise TypeError('invalid dtype')
+    getrf = getattr(cublas, t + 'getrfBatched')
+    getrs = getattr(cublas, t + 'getrsBatched')
+
+    bs = numpy.prod(a.shape[:-2]) if a.ndim > 2 else 1
+    n = a.shape[-1]
+    nrhs = b.shape[-1] if a.ndim == b.ndim else 1
+    b_shape = b.shape
+    a_data_ptr = a.data.ptr
+    b_data_ptr = b.data.ptr
+    a = cupy.ascontiguousarray(a.reshape(bs, n, n).transpose(0, 2, 1),
+                               dtype=dtype)
+    b = cupy.ascontiguousarray(b.reshape(bs, n, nrhs).transpose(0, 2, 1),
+                               dtype=dtype)
+    if a.data.ptr == a_data_ptr:
+        a = a.copy()
+    if b.data.ptr == b_data_ptr:
+        b = b.copy()
+
+    if n > get_batched_gesv_limit():
+        warnings.warn('The matrix size ({}) exceeds the set limit ({})'.
+                      format(n, get_batched_gesv_limit()))
+
+    handle = device.get_cublas_handle()
+    lda = n
+    a_step = lda * n * a.itemsize
+    a_array = cupy.arange(a.data.ptr, a.data.ptr + a_step * bs, a_step,
+                          dtype=cupy.uintp)
+    ldb = n
+    b_step = ldb * nrhs * b.itemsize
+    b_array = cupy.arange(b.data.ptr, b.data.ptr + b_step * bs, b_step,
+                          dtype=cupy.uintp)
+    pivot = cupy.empty((bs, n), dtype=numpy.int32)
+    dinfo = cupy.empty((bs,), dtype=numpy.int32)
+    info = numpy.empty((1,), dtype=numpy.int32)
+    # LU factorization (A = L * U)
+    getrf(handle, n, a_array.data.ptr, lda, pivot.data.ptr, dinfo.data.ptr, bs)
+    _util._check_cublas_info_array_if_synchronization_allowed(getrf, dinfo)
+    # Solves Ax = b
+    getrs(handle, cublas.CUBLAS_OP_N, n, nrhs, a_array.data.ptr, lda,
+          pivot.data.ptr, b_array.data.ptr, ldb, info.ctypes.data, bs)
+    if info[0] != 0:
+        msg = 'Error reported by {} in cuBLAS. '.format(getrs.__name__)
+        if info[0] < 0:
+            msg += 'The {}-th parameter had an illegal value.'.format(-info[0])
+        raise linalg.LinAlgError(msg)
+
+    return b.transpose(0, 2, 1).reshape(b_shape).astype(out_dtype, copy=False)
+
+
+def iamax(x, out=None):
+    """Finds the (smallest) index of the element with the maximum magnitude.
+
+    Note: The result index is 1-based index (not 0-based index).
+    """
+    return _iamaxmin(x, out, 'amax')
+
+
+def iamin(x, out=None):
+    """Finds the (smallest) index of the element with the minimum magnitude.
+
+    Note: The result index is 1-based index (not 0-based index).
+    """
+    return _iamaxmin(x, out, 'amin')
+
+
+def _iamaxmin(x, out, name):
+    if x.ndim != 1:
+        raise ValueError('x must be a 1D array (actual: {})'.format(x.ndim))
+
+    dtype = x.dtype.char
+    if dtype == 'f':
+        t = 's'
+    elif dtype == 'd':
+        t = 'd'
+    elif dtype == 'F':
+        t = 'c'
+    elif dtype == 'D':
+        t = 'z'
+    else:
+        raise TypeError('invalid dtype')
+    func = getattr(cublas, 'i' + t + name)
+
+    handle = device.get_cublas_handle()
+    result_dtype = 'i'
+    result_ptr, result, orig_mode = _setup_result_ptr(
+        handle, out, result_dtype)
+    try:
+        func(handle, x.size, x.data.ptr, 1, result_ptr)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+    if out is None:
+        out = result
+    elif out.dtype != result_dtype:
+        _core.elementwise_copy(result, out)
+    return out
+
+
+def asum(x, out=None):
+    """Computes the sum of the absolute of x."""
+    if x.ndim != 1:
+        raise ValueError('x must be a 1D array (actual: {})'.format(x.ndim))
+
+    dtype = x.dtype.char
+    if dtype == 'f':
+        func = cublas.sasum
+    elif dtype == 'd':
+        func = cublas.dasum
+    elif dtype == 'F':
+        func = cublas.scasum
+    elif dtype == 'D':
+        func = cublas.dzasum
+    else:
+        raise TypeError('invalid dtype')
+
+    handle = device.get_cublas_handle()
+    result_dtype = dtype.lower()
+    result_ptr, result, orig_mode = _setup_result_ptr(
+        handle, out, result_dtype)
+    try:
+        func(handle, x.size, x.data.ptr, 1, result_ptr)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+    if out is None:
+        out = result
+    elif out.dtype != result_dtype:
+        _core.elementwise_copy(result, out)
+    return out
+
+
+def axpy(a, x, y):
+    """Computes y += a * x.
+
+    (*) y will be updated.
+    """
+    _check_two_vectors(x, y)
+
+    dtype = x.dtype.char
+    if dtype == 'f':
+        func = cublas.saxpy
+    elif dtype == 'd':
+        func = cublas.daxpy
+    elif dtype == 'F':
+        func = cublas.caxpy
+    elif dtype == 'D':
+        func = cublas.zaxpy
+    else:
+        raise TypeError('invalid dtype')
+
+    handle = device.get_cublas_handle()
+    a, a_ptr, orig_mode = _setup_scalar_ptr(handle, a, dtype)
+    try:
+        func(handle, x.size, a_ptr, x.data.ptr, 1, y.data.ptr, 1)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+
+def dot(x, y, out=None):
+    """Computes the dot product of x and y."""
+    dtype = x.dtype.char
+    if dtype == 'f':
+        func = cublas.sdot
+    elif dtype == 'd':
+        func = cublas.ddot
+    elif dtype in 'FD':
+        raise TypeError('Use dotu() or dotc() for complex dtype')
+    else:
+        raise TypeError('invalid dtype')
+    _check_two_vectors(x, y)
+
+    handle = device.get_cublas_handle()
+    result_dtype = dtype
+    result_ptr, result, orig_mode = _setup_result_ptr(
+        handle, out, result_dtype)
+    try:
+        func(handle, x.size, x.data.ptr, 1, y.data.ptr, 1, result_ptr)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+    if out is None:
+        out = result
+    elif out.dtype != result_dtype:
+        _core.elementwise_copy(result, out)
+    return out
+
+
+def dotu(x, y, out=None):
+    """Computes the dot product of x and y."""
+    dtype = x.dtype.char
+    if dtype in 'fd':
+        return dot(x, y, out=out)
+    elif dtype == 'F':
+        func = cublas.cdotu
+    elif dtype == 'D':
+        func = cublas.zdotu
+    else:
+        raise TypeError('invalid dtype')
+    _check_two_vectors(x, y)
+
+    handle = device.get_cublas_handle()
+    result_dtype = dtype
+    result_ptr, result, orig_mode = _setup_result_ptr(
+        handle, out, result_dtype)
+    try:
+        func(handle, x.size, x.data.ptr, 1, y.data.ptr, 1, result_ptr)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+    if out is None:
+        out = result
+    elif out.dtype != result_dtype:
+        _core.elementwise_copy(result, out)
+    return out
+
+
+def dotc(x, y, out=None):
+    """Computes the dot product of x.conj() and y."""
+    dtype = x.dtype.char
+    if dtype in 'fd':
+        return dot(x, y, out=out)
+    elif dtype == 'F':
+        func = cublas.cdotc
+    elif dtype == 'D':
+        func = cublas.zdotc
+    else:
+        raise TypeError('invalid dtype')
+    _check_two_vectors(x, y)
+
+    handle = device.get_cublas_handle()
+    result_dtype = dtype
+    result_ptr, result, orig_mode = _setup_result_ptr(
+        handle, out, result_dtype)
+    try:
+        func(handle, x.size, x.data.ptr, 1, y.data.ptr, 1, result_ptr)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+    if out is None:
+        out = result
+    elif out.dtype != result_dtype:
+        _core.elementwise_copy(result, out)
+    return out
+
+
+def nrm2(x, out=None):
+    """Computes the Euclidean norm of vector x."""
+    if x.ndim != 1:
+        raise ValueError('x must be a 1D array (actual: {})'.format(x.ndim))
+
+    dtype = x.dtype.char
+    if dtype == 'f':
+        func = cublas.snrm2
+    elif dtype == 'd':
+        func = cublas.dnrm2
+    elif dtype == 'F':
+        func = cublas.scnrm2
+    elif dtype == 'D':
+        func = cublas.dznrm2
+    else:
+        raise TypeError('invalid dtype')
+
+    handle = device.get_cublas_handle()
+    result_dtype = dtype.lower()
+    result_ptr, result, orig_mode = _setup_result_ptr(
+        handle, out, result_dtype)
+    try:
+        func(handle, x.size, x.data.ptr, 1, result_ptr)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+    if out is None:
+        out = result
+    elif out.dtype != result_dtype:
+        _core.elementwise_copy(result, out)
+    return out
+
+
+def scal(a, x):
+    """Computes x *= a.
+
+    (*) x will be updated.
+    """
+    if x.ndim != 1:
+        raise ValueError('x must be a 1D array (actual: {})'.format(x.ndim))
+
+    dtype = x.dtype.char
+    if dtype == 'f':
+        func = cublas.sscal
+    elif dtype == 'd':
+        func = cublas.dscal
+    elif dtype == 'F':
+        func = cublas.cscal
+    elif dtype == 'D':
+        func = cublas.zscal
+    else:
+        raise TypeError('invalid dtype')
+
+    handle = device.get_cublas_handle()
+    a, a_ptr, orig_mode = _setup_scalar_ptr(handle, a, dtype)
+    try:
+        func(handle, x.size, a_ptr, x.data.ptr, 1)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+
+def _check_two_vectors(x, y):
+    if x.ndim != 1:
+        raise ValueError('x must be a 1D array (actual: {})'.format(x.ndim))
+    if y.ndim != 1:
+        raise ValueError('y must be a 1D array (actual: {})'.format(y.ndim))
+    if x.size != y.size:
+        raise ValueError('x and y must be the same size (actual: {} and {})'
+                         ''.format(x.size, y.size))
+    if x.dtype != y.dtype:
+        raise TypeError('x and y must be the same dtype (actual: {} and {})'
+                        ''.format(x.dtype, y.dtype))
+
+
+def _setup_result_ptr(handle, out, dtype):
+    mode = cublas.getPointerMode(handle)
+    if out is None or isinstance(out, cupy.ndarray):
+        if out is None or out.dtype != dtype:
+            result = cupy.empty([], dtype=dtype)
+        else:
+            result = out
+        result_ptr = result.data.ptr
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_DEVICE)
+    elif isinstance(out, numpy.ndarray):
+        if out.dtype != dtype:
+            result = numpy.empty([], dtype=dtype)
+        else:
+            result = out
+        result_ptr = result.ctypes.data
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_HOST)
+    else:
+        raise TypeError('out must be either cupy or numpy ndarray')
+    return result_ptr, result, mode
+
+
+def _setup_scalar_ptr(handle, a, dtype):
+    a, a_ptr = _get_scalar_ptr(a, dtype)
+    mode = cublas.getPointerMode(handle)
+    if isinstance(a, cupy.ndarray):
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_DEVICE)
+    else:
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_HOST)
+    return a, a_ptr, mode
+
+
+def _get_scalar_ptr(a, dtype):
+    if isinstance(a, cupy.ndarray):
+        if a.dtype != dtype:
+            a = cupy.array(a, dtype=dtype)
+        a_ptr = a.data.ptr
+    else:
+        if not (isinstance(a, numpy.ndarray) and a.dtype == dtype):
+            a = numpy.array(a, dtype=dtype)
+        a_ptr = a.ctypes.data
+    return a, a_ptr
+
+
+def gemv(transa, alpha, a, x, beta, y):
+    """Computes y = alpha * op(a) @ x + beta * y
+
+    op(a) = a if transa is 'N', op(a) = a.T if transa is 'T',
+    op(a) = a.T.conj() if transa is 'H'.
+
+    Note: ''y'' will be updated.
+    """
+    dtype = a.dtype.char
+    if dtype == 'f':
+        func = cublas.sgemv
+    elif dtype == 'd':
+        func = cublas.dgemv
+    elif dtype == 'F':
+        func = cublas.cgemv
+    elif dtype == 'D':
+        func = cublas.zgemv
+    else:
+        raise TypeError('invalid dtype')
+    assert a.ndim == 2
+    assert x.ndim == y.ndim == 1
+    assert a.dtype == x.dtype == y.dtype
+    m, n = a.shape
+    transa = _trans_to_cublas_op(transa)
+    if transa == cublas.CUBLAS_OP_N:
+        xlen, ylen = n, m
+    else:
+        xlen, ylen = m, n
+    assert x.shape[0] == xlen
+    assert y.shape[0] == ylen
+
+    alpha, alpha_ptr = _get_scalar_ptr(alpha, a.dtype)
+    beta, beta_ptr = _get_scalar_ptr(beta, a.dtype)
+    handle = device.get_cublas_handle()
+    orig_mode = cublas.getPointerMode(handle)
+    if isinstance(alpha, cupy.ndarray) or isinstance(beta, cupy.ndarray):
+        if not isinstance(alpha, cupy.ndarray):
+            alpha = cupy.array(alpha)
+            alpha_ptr = alpha.data.ptr
+        if not isinstance(beta, cupy.ndarray):
+            beta = cupy.array(beta)
+            beta_ptr = beta.data.ptr
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_DEVICE)
+    else:
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_HOST)
+
+    try:
+        if a._f_contiguous:
+            func(handle, transa, m, n, alpha_ptr, a.data.ptr, m, x.data.ptr, 1,
+                 beta_ptr, y.data.ptr, 1)
+        elif a._c_contiguous and transa != cublas.CUBLAS_OP_C:
+            if transa == cublas.CUBLAS_OP_N:
+                transa = cublas.CUBLAS_OP_T
+            else:
+                transa = cublas.CUBLAS_OP_N
+            func(handle, transa, n, m, alpha_ptr, a.data.ptr, n, x.data.ptr, 1,
+                 beta_ptr, y.data.ptr, 1)
+        else:
+            a = a.copy(order='F')
+            func(handle, transa, m, n, alpha_ptr, a.data.ptr, m, x.data.ptr, 1,
+                 beta_ptr, y.data.ptr, 1)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+
+def ger(alpha, x, y, a):
+    """Computes a += alpha * x @ y.T
+
+    Note: ''a'' will be updated.
+    """
+    dtype = a.dtype.char
+    if dtype == 'f':
+        func = cublas.sger
+    elif dtype == 'd':
+        func = cublas.dger
+    elif dtype in 'FD':
+        raise TypeError('Use geru or gerc for complex dtypes')
+    else:
+        raise TypeError('invalid dtype')
+
+    assert a.ndim == 2
+    assert x.ndim == y.ndim == 1
+    assert a.dtype == x.dtype == y.dtype
+    m, n = a.shape
+    assert x.shape[0] == m
+    assert y.shape[0] == n
+
+    handle = device.get_cublas_handle()
+    alpha, alpha_ptr, orig_mode = _setup_scalar_ptr(handle, alpha, dtype)
+    x_ptr, y_ptr = x.data.ptr, y.data.ptr
+    try:
+        if a._f_contiguous:
+            func(handle, m, n, alpha_ptr, x_ptr, 1, y_ptr, 1, a.data.ptr, m)
+        elif a._c_contiguous:
+            func(handle, n, m, alpha_ptr, y_ptr, 1, x_ptr, 1, a.data.ptr, n)
+        else:
+            aa = a.copy(order='F')
+            func(handle, m, n, alpha_ptr, x_ptr, 1, y_ptr, 1, aa.data.ptr, m)
+            _core.elementwise_copy(aa, a)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+
+def geru(alpha, x, y, a):
+    """Computes a += alpha * x @ y.T
+
+    Note: ''a'' will be updated.
+    """
+    dtype = a.dtype.char
+    if dtype in 'fd':
+        return ger(alpha, x, y, a)
+    elif dtype == 'F':
+        func = cublas.cgeru
+    elif dtype == 'D':
+        func = cublas.zgeru
+    else:
+        raise TypeError('invalid dtype')
+    assert a.ndim == 2
+    assert x.ndim == y.ndim == 1
+    assert a.dtype == x.dtype == y.dtype
+    m, n = a.shape
+    assert x.shape[0] == m
+    assert y.shape[0] == n
+
+    handle = device.get_cublas_handle()
+    alpha, alpha_ptr, orig_mode = _setup_scalar_ptr(handle, alpha, dtype)
+    x_ptr, y_ptr = x.data.ptr, y.data.ptr
+    try:
+        if a._f_contiguous:
+            func(handle, m, n, alpha_ptr, x_ptr, 1, y_ptr, 1, a.data.ptr, m)
+        elif a._c_contiguous:
+            func(handle, n, m, alpha_ptr, y_ptr, 1, x_ptr, 1, a.data.ptr, n)
+        else:
+            aa = a.copy(order='F')
+            func(handle, m, n, alpha_ptr, x_ptr, 1, y_ptr, 1, aa.data.ptr, m)
+            _core.elementwise_copy(aa, a)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+
+def gerc(alpha, x, y, a):
+    """Computes a += alpha * x @ y.T.conj()
+
+    Note: ''a'' will be updated.
+    """
+    dtype = a.dtype.char
+    if dtype in 'fd':
+        return ger(alpha, x, y, a)
+    elif dtype == 'F':
+        func = cublas.cgerc
+    elif dtype == 'D':
+        func = cublas.zgerc
+    else:
+        raise TypeError('invalid dtype')
+    assert a.ndim == 2
+    assert x.ndim == y.ndim == 1
+    assert a.dtype == x.dtype == y.dtype
+    m, n = a.shape
+    assert x.shape[0] == m
+    assert y.shape[0] == n
+
+    handle = device.get_cublas_handle()
+    alpha, alpha_ptr, orig_mode = _setup_scalar_ptr(handle, alpha, dtype)
+    x_ptr, y_ptr = x.data.ptr, y.data.ptr
+    try:
+        if a._f_contiguous:
+            func(handle, m, n, alpha_ptr, x_ptr, 1, y_ptr, 1, a.data.ptr, m)
+        else:
+            aa = a.copy(order='F')
+            func(handle, m, n, alpha_ptr, x_ptr, 1, y_ptr, 1, aa.data.ptr, m)
+            _core.elementwise_copy(aa, a)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+
+def sbmv(k, alpha, a, x, beta, y, lower=False):
+    """Computes y = alpha*A @ x + beta * y
+
+    """
+    dtype = a.dtype.char
+    if dtype == 'f':
+        func = cublas.ssbmv
+    elif dtype == 'd':
+        func = cublas.dsbmv
+    else:
+        raise TypeError('Complex dtypes not supported')
+
+    assert a.ndim == 2
+    assert x.ndim == y.ndim == 1
+    assert a.dtype == x.dtype == y.dtype
+    m, n = a.shape
+    assert x.shape[0] == n
+    assert y.shape[0] == n
+
+    if not a._f_contiguous:
+        a = a.copy(order='F')
+
+    alpha, alpha_ptr = _get_scalar_ptr(alpha, a.dtype)
+    beta, beta_ptr = _get_scalar_ptr(beta, a.dtype)
+    handle = device.get_cublas_handle()
+    orig_mode = cublas.getPointerMode(handle)
+    if isinstance(alpha, cupy.ndarray) or isinstance(beta, cupy.ndarray):
+        if not isinstance(alpha, cupy.ndarray):
+            alpha = cupy.array(alpha)
+            alpha_ptr = alpha.data.ptr
+        if not isinstance(beta, cupy.ndarray):
+            beta = cupy.array(beta)
+            beta_ptr = beta.data.ptr
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_DEVICE)
+    else:
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_HOST)
+
+    if lower:
+        uplo = cublas.CUBLAS_FILL_MODE_LOWER
+    else:
+        uplo = cublas.CUBLAS_FILL_MODE_UPPER
+
+    handle = device.get_cublas_handle()
+    try:
+        func(handle, uplo, n, k,
+             alpha_ptr, a.data.ptr, m, x.data.ptr, 1,
+             beta_ptr, y.data.ptr, 1)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+
+    return y
+
+
+def _trans_to_cublas_op(trans):
+    if trans == 'N' or trans == cublas.CUBLAS_OP_N:
+        trans = cublas.CUBLAS_OP_N
+    elif trans == 'T' or trans == cublas.CUBLAS_OP_T:
+        trans = cublas.CUBLAS_OP_T
+    elif trans == 'H' or trans == cublas.CUBLAS_OP_C:
+        trans = cublas.CUBLAS_OP_C
+    else:
+        raise TypeError('invalid trans (actual: {})'.format(trans))
+    return trans
+
+
+def _decide_ld_and_trans(a, trans):
+    ld = None
+    if trans in (cublas.CUBLAS_OP_N, cublas.CUBLAS_OP_T):
+        if a._f_contiguous:
+            ld = a.shape[0]
+        elif a._c_contiguous:
+            ld = a.shape[1]
+            trans = 1 - trans
+    return ld, trans
+
+
+def _change_order_if_necessary(a, lda):
+    if lda is None:
+        lda = a.shape[0]
+        if not a._f_contiguous:
+            a = a.copy(order='F')
+    return a, lda
+
+
+def gemm(transa, transb, a, b, out=None, alpha=1.0, beta=0.0):
+    """Computes out = alpha * op(a) @ op(b) + beta * out
+
+    op(a) = a if transa is 'N', op(a) = a.T if transa is 'T',
+    op(a) = a.T.conj() if transa is 'H'.
+    op(b) = b if transb is 'N', op(b) = b.T if transb is 'T',
+    op(b) = b.T.conj() if transb is 'H'.
+    """
+    assert a.ndim == b.ndim == 2
+    assert a.dtype == b.dtype
+    dtype = a.dtype.char
+    if dtype == 'f':
+        func = cublas.sgemm
+    elif dtype == 'd':
+        func = cublas.dgemm
+    elif dtype == 'F':
+        func = cublas.cgemm
+    elif dtype == 'D':
+        func = cublas.zgemm
+    else:
+        raise TypeError('invalid dtype')
+
+    transa = _trans_to_cublas_op(transa)
+    transb = _trans_to_cublas_op(transb)
+    if transa == cublas.CUBLAS_OP_N:
+        m, k = a.shape
+    else:
+        k, m = a.shape
+    if transb == cublas.CUBLAS_OP_N:
+        n = b.shape[1]
+        assert b.shape[0] == k
+    else:
+        n = b.shape[0]
+        assert b.shape[1] == k
+    if out is None:
+        out = cupy.empty((m, n), dtype=dtype, order='F')
+        beta = 0.0
+    else:
+        assert out.ndim == 2
+        assert out.shape == (m, n)
+        assert out.dtype == dtype
+
+    alpha, alpha_ptr = _get_scalar_ptr(alpha, a.dtype)
+    beta, beta_ptr = _get_scalar_ptr(beta, a.dtype)
+    handle = device.get_cublas_handle()
+    orig_mode = cublas.getPointerMode(handle)
+    if isinstance(alpha, cupy.ndarray) or isinstance(beta, cupy.ndarray):
+        if not isinstance(alpha, cupy.ndarray):
+            alpha = cupy.array(alpha)
+            alpha_ptr = alpha.data.ptr
+        if not isinstance(beta, cupy.ndarray):
+            beta = cupy.array(beta)
+            beta_ptr = beta.data.ptr
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_DEVICE)
+    else:
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_HOST)
+
+    lda, transa = _decide_ld_and_trans(a, transa)
+    ldb, transb = _decide_ld_and_trans(b, transb)
+    if not (lda is None or ldb is None):
+        if out._f_contiguous:
+            try:
+                func(handle, transa, transb, m, n, k, alpha_ptr,
+                     a.data.ptr, lda, b.data.ptr, ldb, beta_ptr, out.data.ptr,
+                     m)
+            finally:
+                cublas.setPointerMode(handle, orig_mode)
+            return out
+        elif out._c_contiguous:
+            # Computes out.T = alpha * b.T @ a.T + beta * out.T
+            try:
+                func(handle, 1 - transb, 1 - transa, n, m, k, alpha_ptr,
+                     b.data.ptr, ldb, a.data.ptr, lda, beta_ptr, out.data.ptr,
+                     n)
+            finally:
+                cublas.setPointerMode(handle, orig_mode)
+            return out
+
+    a, lda = _change_order_if_necessary(a, lda)
+    b, ldb = _change_order_if_necessary(b, ldb)
+    c = out
+    if not out._f_contiguous:
+        c = out.copy(order='F')
+    try:
+        func(handle, transa, transb, m, n, k, alpha_ptr, a.data.ptr, lda,
+             b.data.ptr, ldb, beta_ptr, c.data.ptr, m)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+    if not out._f_contiguous:
+        _core.elementwise_copy(c, out)
+    return out
+
+
+def geam(transa, transb, alpha, a, beta, b, out=None):
+    """Computes alpha * op(a) + beta * op(b)
+
+    op(a) = a if transa is 'N', op(a) = a.T if transa is 'T',
+    op(a) = a.T.conj() if transa is 'H'.
+    op(b) = b if transb is 'N', op(b) = b.T if transb is 'T',
+    op(b) = b.T.conj() if transb is 'H'.
+    """
+    assert a.ndim == b.ndim == 2
+    assert a.dtype == b.dtype
+    dtype = a.dtype.char
+    if dtype == 'f':
+        func = cublas.sgeam
+    elif dtype == 'd':
+        func = cublas.dgeam
+    elif dtype == 'F':
+        func = cublas.cgeam
+    elif dtype == 'D':
+        func = cublas.zgeam
+    else:
+        raise TypeError('invalid dtype')
+
+    transa = _trans_to_cublas_op(transa)
+    transb = _trans_to_cublas_op(transb)
+    if transa == cublas.CUBLAS_OP_N:
+        m, n = a.shape
+    else:
+        n, m = a.shape
+    if transb == cublas.CUBLAS_OP_N:
+        assert b.shape == (m, n)
+    else:
+        assert b.shape == (n, m)
+    if out is None:
+        out = cupy.empty((m, n), dtype=dtype, order='F')
+    else:
+        assert out.ndim == 2
+        assert out.shape == (m, n)
+        assert out.dtype == dtype
+
+    alpha, alpha_ptr = _get_scalar_ptr(alpha, a.dtype)
+    beta, beta_ptr = _get_scalar_ptr(beta, a.dtype)
+    handle = device.get_cublas_handle()
+    orig_mode = cublas.getPointerMode(handle)
+    if isinstance(alpha, cupy.ndarray) or isinstance(beta, cupy.ndarray):
+        if not isinstance(alpha, cupy.ndarray):
+            alpha = cupy.array(alpha)
+            alpha_ptr = alpha.data.ptr
+        if not isinstance(beta, cupy.ndarray):
+            beta = cupy.array(beta)
+            beta_ptr = beta.data.ptr
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_DEVICE)
+    else:
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_HOST)
+
+    lda, transa = _decide_ld_and_trans(a, transa)
+    ldb, transb = _decide_ld_and_trans(b, transb)
+    if not (lda is None or ldb is None):
+        if out._f_contiguous:
+            try:
+                func(handle, transa, transb, m, n, alpha_ptr, a.data.ptr,
+                     lda, beta_ptr, b.data.ptr, ldb, out.data.ptr, m)
+            finally:
+                cublas.setPointerMode(handle, orig_mode)
+            return out
+        elif out._c_contiguous:
+            # Computes alpha * a.T + beta * b.T
+            try:
+                func(handle, 1-transa, 1-transb, n, m, alpha_ptr, a.data.ptr,
+                     lda, beta_ptr, b.data.ptr, ldb, out.data.ptr, n)
+            finally:
+                cublas.setPointerMode(handle, orig_mode)
+            return out
+
+    a, lda = _change_order_if_necessary(a, lda)
+    b, ldb = _change_order_if_necessary(b, ldb)
+    c = out
+    if not out._f_contiguous:
+        c = out.copy(order='F')
+    try:
+        func(handle, transa, transb, m, n, alpha_ptr, a.data.ptr, lda,
+             beta_ptr, b.data.ptr, ldb, c.data.ptr, m)
+    finally:
+        cublas.setPointerMode(handle, orig_mode)
+    if not out._f_contiguous:
+        _core.elementwise_copy(c, out)
+    return out
+
+
+def dgmm(side, a, x, out=None, incx=1):
+    """Computes diag(x) @ a or a @ diag(x)
+
+    Computes diag(x) @ a if side is 'L', a @ diag(x) if side is 'R'.
+    """
+    assert a.ndim == 2
+    assert 0 <= x.ndim <= 2
+    assert a.dtype == x.dtype
+    dtype = a.dtype.char
+    if dtype == 'f':
+        func = cublas.sdgmm
+    elif dtype == 'd':
+        func = cublas.ddgmm
+    elif dtype == 'F':
+        func = cublas.cdgmm
+    elif dtype == 'D':
+        func = cublas.zdgmm
+    else:
+        raise TypeError('invalid dtype')
+    if side == 'L' or side == cublas.CUBLAS_SIDE_LEFT:
+        side = cublas.CUBLAS_SIDE_LEFT
+    elif side == 'R' or side == cublas.CUBLAS_SIDE_RIGHT:
+        side = cublas.CUBLAS_SIDE_RIGHT
+    else:
+        raise ValueError('invalid side (actual: {})'.format(side))
+    m, n = a.shape
+    if side == cublas.CUBLAS_SIDE_LEFT:
+        assert x.size >= (m - 1) * abs(incx) + 1
+    else:
+        assert x.size >= (n - 1) * abs(incx) + 1
+    if out is None:
+        if a._c_contiguous:
+            order = 'C'
+        else:
+            order = 'F'
+        out = cupy.empty((m, n), dtype=dtype, order=order)
+    else:
+        assert out.ndim == 2
+        assert out.shape == a.shape
+        assert out.dtype == a.dtype
+
+    handle = device.get_cublas_handle()
+    if out._c_contiguous:
+        if not a._c_contiguous:
+            a = a.copy(order='C')
+        func(handle, 1 - side, n, m, a.data.ptr, n, x.data.ptr, incx,
+             out.data.ptr, n)
+    else:
+        if not a._f_contiguous:
+            a = a.copy(order='F')
+        c = out
+        if not out._f_contiguous:
+            c = out.copy(order='F')
+        func(handle, side, m, n, a.data.ptr, m, x.data.ptr, incx,
+             c.data.ptr, m)
+        if not out._f_contiguous:
+            _core.elementwise_copy(c, out)
+    return out
+
+
+def syrk(trans, a, out=None, alpha=1.0, beta=0.0, lower=False):
+    """Computes out := alpha*op1(a)*op2(a) + beta*out
+
+    op1(a) = a if trans is 'N', op2(a) = a.T if transa is 'N'
+    op1(a) = a.T if trans is 'T', op2(a) = a if transa is 'T'
+    lower specifies  whether  the  upper  or  lower triangular
+    part  of the  array  out  is to be  referenced
+    """
+    assert a.ndim == 2
+    dtype = a.dtype.char
+    if dtype == 'f':
+        func = cublas.ssyrk
+    elif dtype == 'd':
+        func = cublas.dsyrk
+    elif dtype == 'F':
+        func = cublas.csyrk
+    elif dtype == 'D':
+        func = cublas.zsyrk
+    else:
+        raise TypeError('invalid dtype')
+
+    trans = _trans_to_cublas_op(trans)
+    if trans == cublas.CUBLAS_OP_N:
+        n, k = a.shape
+    else:
+        k, n = a.shape
+    if out is None:
+        out = cupy.zeros((n, n), dtype=dtype, order='F')
+        beta = 0.0
+    else:
+        assert out.ndim == 2
+        assert out.shape == (n, n)
+        assert out.dtype == dtype
+
+    if lower:
+        uplo = cublas.CUBLAS_FILL_MODE_LOWER
+    else:
+        uplo = cublas.CUBLAS_FILL_MODE_UPPER
+
+    alpha, alpha_ptr = _get_scalar_ptr(alpha, a.dtype)
+    beta, beta_ptr = _get_scalar_ptr(beta, a.dtype)
+    handle = device.get_cublas_handle()
+    orig_mode = cublas.getPointerMode(handle)
+    if isinstance(alpha, cupy.ndarray) or isinstance(beta, cupy.ndarray):
+        if not isinstance(alpha, cupy.ndarray):
+            alpha = cupy.array(alpha)
+            alpha_ptr = alpha.data.ptr
+        if not isinstance(beta, cupy.ndarray):
+            beta = cupy.array(beta)
+            beta_ptr = beta.data.ptr
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_DEVICE)
+    else:
+        cublas.setPointerMode(handle, cublas.CUBLAS_POINTER_MODE_HOST)
+
+    lda, trans = _decide_ld_and_trans(a, trans)
+    ldo, _ = _decide_ld_and_trans(out, trans)
+    if out._c_contiguous:
+        if not a._c_contiguous:
+            a = a.copy(order='C')
+            trans = 1 - trans
+            lda = a.shape[1]
+        try:
+            func(handle, 1 - uplo, trans, n, k,
+                 alpha_ptr, a.data.ptr, lda,
+                 beta_ptr, out.data.ptr, ldo)
+        finally:
+            cublas.setPointerMode(handle, orig_mode)
+
+    else:
+        if not a._f_contiguous:
+            a = a.copy(order='F')
+            lda = a.shape[0]
+            trans = 1 - trans
+        c = out
+        if not out._f_contiguous:
+            c = out.copy(order='F')
+        try:
+            func(handle, uplo, trans, n, k,
+                 alpha_ptr, a.data.ptr, lda,
+                 beta_ptr, out.data.ptr, ldo)
+        finally:
+            cublas.setPointerMode(handle, orig_mode)
+        if not out._f_contiguous:
+            out[...] = c
+    return out
diff --git a/cupy/cuda/__init__.pxd b/cupy/cuda/__init__.pxd
new file mode 100644
index 0000000..e69de29
diff --git a/cupy/cuda/__init__.py b/cupy/cuda/__init__.py
new file mode 100644
index 0000000..1d68a55
--- /dev/null
+++ b/cupy/cuda/__init__.py
@@ -0,0 +1,184 @@
+import contextlib
+import warnings
+
+from cupy._environment import get_cuda_path  # NOQA
+from cupy._environment import get_nvcc_path  # NOQA
+from cupy._environment import get_rocm_path  # NOQA
+from cupy._environment import get_hipcc_path  # NOQA
+from cupy.cuda import compiler  # NOQA
+from cupy.cuda import device  # NOQA
+from cupy.cuda import function  # NOQA
+from cupy.cuda import memory  # NOQA
+from cupy.cuda import memory_hook  # NOQA
+from cupy.cuda import memory_hooks  # NOQA
+from cupy.cuda import pinned_memory  # NOQA
+from cupy.cuda import stream  # NOQA
+from cupy.cuda import texture  # NOQA
+from cupy_backends.cuda.api import driver  # NOQA
+from cupy_backends.cuda.api import runtime  # NOQA
+from cupy_backends.cuda.libs import cublas  # NOQA
+from cupy_backends.cuda.libs import curand  # NOQA
+from cupy_backends.cuda.libs import cusolver  # NOQA
+from cupy_backends.cuda.libs import cusparse  # NOQA
+from cupy_backends.cuda.libs import nvrtc  # NOQA
+from cupy_backends.cuda.libs import profiler  # NOQA
+
+
+_available = None
+
+
+class _UnavailableModule():
+    available = False
+
+    def __init__(self, name):
+        self.__name__ = name
+
+
+from cupy.cuda import cub  # NOQA
+
+if not runtime.is_hip and driver.get_build_version() > 0:
+    from cupy.cuda import jitify  # NOQA
+else:
+    jitify = None
+
+try:
+    from cupy_backends.cuda.libs import nvtx  # NOQA
+except ImportError:
+    nvtx = _UnavailableModule('cupy.cuda.nvtx')
+
+try:
+    from cupy.cuda import thrust  # NOQA
+except ImportError:
+    thrust = _UnavailableModule('cupy.cuda.thrust')
+
+
+def __getattr__(key):
+    # `*_enabled` flags are kept for backward compatibility.
+    # Note: module-level getattr only runs on Python 3.7+.
+    if key == 'cusolver_enabled':
+        # cuSOLVER is always available in CUDA 8.0+.
+        warnings.warn('''
+cupy.cuda.cusolver_enabled has been deprecated in CuPy v8 and will be removed in the future release.
+This flag always returns True as cuSOLVER is always available in CUDA 8.0 or later.
+            ''', DeprecationWarning)  # NOQA
+        return True
+
+    for mod in [nvtx, thrust, cub]:
+        flag = '{}_enabled'.format(mod.__name__.split('.')[-1])
+        if key == flag:
+            warnings.warn('''
+cupy.cuda.{} has been deprecated in CuPy v8 and will be removed in the future release.
+Use {}.available instead.
+                '''.format(flag, mod.__name__), DeprecationWarning)  # NOQA
+            return not isinstance(mod, _UnavailableModule)
+
+    raise AttributeError(
+        "module '{}' has no attribute '{}'".format(__name__, key))
+
+
+def is_available():
+    global _available
+    if _available is None:
+        _available = False
+        try:
+            _available = runtime.getDeviceCount() > 0
+        except Exception as e:
+            if (not runtime.is_hip and e.args[0] !=
+                    'cudaErrorNoDevice: no CUDA-capable device is detected'):
+                raise
+            elif runtime.is_hip and 'hipErrorNoDevice' not in e.args[0]:
+                raise
+    return _available
+
+
+# import class and function
+from cupy.cuda.device import Device  # NOQA
+from cupy.cuda.device import get_cublas_handle  # NOQA
+from cupy.cuda.device import get_device_id  # NOQA
+from cupy.cuda.function import Function  # NOQA
+from cupy.cuda.function import Module  # NOQA
+from cupy.cuda.memory import alloc  # NOQA
+from cupy.cuda.memory import BaseMemory  # NOQA
+from cupy.cuda.memory import malloc_managed  # NOQA
+from cupy.cuda.memory import malloc_async  # NOQA
+from cupy.cuda.memory import ManagedMemory  # NOQA
+from cupy.cuda.memory import Memory  # NOQA
+from cupy.cuda.memory import MemoryAsync  # NOQA
+from cupy.cuda.memory import MemoryPointer  # NOQA
+from cupy.cuda.memory import MemoryPool  # NOQA
+from cupy.cuda.memory import MemoryAsyncPool  # NOQA
+from cupy.cuda.memory import PythonFunctionAllocator  # NOQA
+from cupy.cuda.memory import CFunctionAllocator  # NOQA
+from cupy.cuda.memory import set_allocator  # NOQA
+from cupy.cuda.memory import get_allocator  # NOQA
+from cupy.cuda.memory import UnownedMemory  # NOQA
+from cupy.cuda.memory_hook import MemoryHook  # NOQA
+from cupy.cuda.pinned_memory import alloc_pinned_memory  # NOQA
+from cupy.cuda.pinned_memory import PinnedMemory  # NOQA
+from cupy.cuda.pinned_memory import PinnedMemoryPointer  # NOQA
+from cupy.cuda.pinned_memory import PinnedMemoryPool  # NOQA
+from cupy.cuda.pinned_memory import set_pinned_memory_allocator  # NOQA
+from cupy.cuda.stream import Event  # NOQA
+from cupy.cuda.stream import get_current_stream  # NOQA
+from cupy.cuda.stream import get_elapsed_time  # NOQA
+from cupy.cuda.stream import Stream  # NOQA
+from cupy.cuda.stream import ExternalStream  # NOQA
+from cupy.cuda.graph import Graph  # NOQA
+
+
+# Importing only for backward compatibility:
+from cupy.cuda.compiler import compile_with_cache  # NOQA
+
+
+@contextlib.contextmanager
+def using_allocator(allocator=None):
+    """Sets a thread-local allocator for GPU memory inside
+       context manager
+
+    Args:
+        allocator (function): CuPy memory allocator. It must have the same
+            interface as the :func:`cupy.cuda.alloc` function, which takes the
+            buffer size as an argument and returns the device buffer of that
+            size. When ``None`` is specified, raw memory allocator will be
+            used (i.e., memory pool is disabled).
+    """
+    # Note: cupy/memory.pyx would be the better place to implement this
+    # function but `contextmanager` decoration doesn't behave well in Cython.
+    if allocator is None:
+        allocator = memory._malloc
+    previous_allocator = memory._get_thread_local_allocator()
+    memory._set_thread_local_allocator(allocator)
+    try:
+        yield
+    finally:
+        memory._set_thread_local_allocator(previous_allocator)
+
+
+@contextlib.contextmanager
+def profile():
+    """Enable CUDA profiling during with statement.
+
+    This function enables profiling on entering a with statement, and disables
+    profiling on leaving the statement.
+
+    >>> with cupy.cuda.profile():
+    ...    # do something you want to measure
+    ...    pass
+
+    .. note::
+        When starting ``nvprof`` from the command line, manually setting
+        ``--profile-from-start off`` may be required for the desired behavior.
+
+    .. warning:: This context manager is deprecated. Please use
+        :class:`cupyx.profiler.profile` instead.
+    """
+    warnings.warn(
+        'cupy.cuda.profile has been deprecated since CuPy v10 '
+        'and will be removed in the future. Use cupyx.profiler.profile '
+        'instead.')
+
+    profiler.start()
+    try:
+        yield
+    finally:
+        profiler.stop()
diff --git a/cupy/cuda/common.pxd b/cupy/cuda/common.pxd
new file mode 100644
index 0000000..a5dbaf1
--- /dev/null
+++ b/cupy/cuda/common.pxd
@@ -0,0 +1,22 @@
+# distutils: language = c++
+
+# this mirrors the macro defined in cupy/_core/include/cupy/type_dispatcher.cuh
+cdef enum:
+    CUPY_TYPE_INT8 = 0
+    CUPY_TYPE_UINT8 = 1
+    CUPY_TYPE_INT16 = 2
+    CUPY_TYPE_UINT16 = 3
+    CUPY_TYPE_INT32 = 4
+    CUPY_TYPE_UINT32 = 5
+    CUPY_TYPE_INT64 = 6
+    CUPY_TYPE_UINT64 = 7
+    CUPY_TYPE_FLOAT16 = 8
+    CUPY_TYPE_FLOAT32 = 9
+    CUPY_TYPE_FLOAT64 = 10
+    CUPY_TYPE_COMPLEX64 = 11
+    CUPY_TYPE_COMPLEX128 = 12
+    CUPY_TYPE_BOOL = 13
+
+
+cpdef int _get_dtype_id(dtype) except -1
+cpdef int _is_fp16_supported() except -2
diff --git a/cupy/cuda/common.pyx b/cupy/cuda/common.pyx
new file mode 100644
index 0000000..120c239
--- /dev/null
+++ b/cupy/cuda/common.pyx
@@ -0,0 +1,60 @@
+from cupy_backends.cuda.api cimport runtime
+from cupy.cuda cimport device
+
+import numpy
+
+
+cpdef int _get_dtype_id(dtype) except -1:
+    cdef int ret
+
+    if dtype == numpy.int8:
+        ret = CUPY_TYPE_INT8
+    elif dtype == numpy.uint8:
+        ret = CUPY_TYPE_UINT8
+    elif dtype == numpy.int16:
+        ret = CUPY_TYPE_INT16
+    elif dtype == numpy.uint16:
+        ret = CUPY_TYPE_UINT16
+    elif dtype == numpy.int32:
+        ret = CUPY_TYPE_INT32
+    elif dtype == numpy.uint32:
+        ret = CUPY_TYPE_UINT32
+    elif dtype == numpy.int64:
+        ret = CUPY_TYPE_INT64
+    elif dtype == numpy.uint64:
+        ret = CUPY_TYPE_UINT64
+    elif dtype == numpy.float16:
+        ret = CUPY_TYPE_FLOAT16
+    elif dtype == numpy.float32:
+        ret = CUPY_TYPE_FLOAT32
+    elif dtype == numpy.float64:
+        ret = CUPY_TYPE_FLOAT64
+    elif dtype == numpy.complex64:
+        ret = CUPY_TYPE_COMPLEX64
+    elif dtype == numpy.complex128:
+        ret = CUPY_TYPE_COMPLEX128
+    elif dtype == numpy.bool_:
+        ret = CUPY_TYPE_BOOL
+    else:
+        raise ValueError('Unsupported dtype ({})'.format(dtype))
+    return ret
+
+
+cdef int _has_fp16 = -1
+
+
+cpdef int _is_fp16_supported() except -2:
+    global _has_fp16
+
+    if _has_fp16 != -1:
+        return _has_fp16
+
+    # TODO(leofang): make sure if this is really OK
+    if runtime._is_hip_environment:
+        _has_fp16 = 1  # tested on ROCm 3.5.0 + gfx906
+    elif (int(device.get_compute_capability()) < 53
+          or runtime.runtimeGetVersion() < 9020):
+        _has_fp16 = 0
+    else:
+        _has_fp16 = 1
+    return _has_fp16
diff --git a/cupy/cuda/compiler.py b/cupy/cuda/compiler.py
new file mode 100644
index 0000000..fd03c1a
--- /dev/null
+++ b/cupy/cuda/compiler.py
@@ -0,0 +1,943 @@
+import copy
+import hashlib
+import math
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import warnings
+
+from cupy.cuda import device
+from cupy.cuda import function
+from cupy.cuda import get_rocm_path
+from cupy_backends.cuda.api import driver
+from cupy_backends.cuda.api import runtime
+from cupy_backends.cuda.libs import nvrtc
+from cupy import _util
+
+_cuda_hip_version = driver.get_build_version()
+if not runtime.is_hip and _cuda_hip_version > 0:
+    from cupy.cuda import jitify
+
+
+_nvrtc_version = None
+_win32 = sys.platform.startswith('win32')
+_rdc_flags = ('--device-c', '-dc', '-rdc=true',
+              '--relocatable-device-code=true')
+_cudadevrt = None
+
+
+class NVCCException(Exception):
+    pass
+
+
+class HIPCCException(Exception):
+    pass
+
+
+class JitifyException(Exception):
+    pass
+
+
+def _run_cc(cmd, cwd, backend, log_stream=None):
+    # backend in ('nvcc', 'hipcc')
+    try:
+        # Inherit the environment variable as NVCC refers to PATH, TMPDIR/TMP,
+        # NVCC_PREPEND_FLAGS, NVCC_APPEND_FLAGS.
+        env = os.environ
+        if _win32:
+            # Adds the extra PATH for NVCC invocation.
+            # When running NVCC, a host compiler must be available in PATH,
+            # but this is not true in general Windows environment unless
+            # running inside the SDK Tools command prompt.
+            # To mitigate the situation CuPy automatically adds a path to
+            # the VC++ compiler (cl.exe) found via setuptools, if it is not
+            # on the PATH.
+            extra_path = _get_extra_path_for_msvc()
+            if extra_path is not None:
+                path = extra_path + os.pathsep + os.environ.get('PATH', '')
+                env = copy.deepcopy(env)
+                env['PATH'] = path
+        log = subprocess.check_output(cmd, cwd=cwd, env=env,
+                                      stderr=subprocess.STDOUT,
+                                      universal_newlines=True)
+        if log_stream is not None:
+            log_stream.write(log)
+        return log
+    except subprocess.CalledProcessError as e:
+        msg = ('`{0}` command returns non-zero exit status. \n'
+               'command: {1}\n'
+               'return-code: {2}\n'
+               'stdout/stderr: \n'
+               '{3}'.format(backend,
+                            e.cmd,
+                            e.returncode,
+                            e.output))
+        if backend == 'nvcc':
+            raise NVCCException(msg)
+        elif backend == 'hipcc':
+            raise HIPCCException(msg)
+        else:
+            raise RuntimeError(msg)
+    except OSError as e:
+        msg = 'Failed to run `{0}` command. ' \
+              'Check PATH environment variable: ' \
+              + str(e)
+        raise OSError(msg.format(backend))
+
+
+@_util.memoize()
+def _get_extra_path_for_msvc():
+    cl_exe = shutil.which('cl.exe')
+    if cl_exe:
+        # The compiler is already on PATH, no extra path needed.
+        return None
+
+    try:
+        import setuptools
+        vctools = setuptools.msvc.EnvironmentInfo(platform.machine()).VCTools
+    except Exception as e:
+        warnings.warn(f'Failed to auto-detect cl.exe path: {type(e)}: {e}')
+        return None
+
+    for path in vctools:
+        cl_exe = os.path.join(path, 'cl.exe')
+        if os.path.exists(cl_exe):
+            return path
+    warnings.warn(f'cl.exe could not be found in {vctools}')
+    return None
+
+
+def _get_nvrtc_version():
+    global _nvrtc_version
+    if _nvrtc_version is None:
+        _nvrtc_version = nvrtc.getVersion()
+
+    return _nvrtc_version
+
+
+# Known archs for Tegra/Jetson/Xavier/etc
+_tegra_archs = ('32', '53', '62', '72', '87')
+
+
+@_util.memoize()
+def _get_max_compute_capability():
+    major, minor = _get_nvrtc_version()
+    if major < 11:
+        # CUDA 10.2
+        nvrtc_max_compute_capability = '75'
+    elif major == 11 and minor == 0:
+        # CUDA 11.0
+        nvrtc_max_compute_capability = '80'
+    elif major == 11 and minor < 8:
+        # CUDA 11.1 - 11.7
+        # Note: 87 is for Jetson Orin
+        nvrtc_max_compute_capability = '86'
+    else:
+        # CUDA 11.8+
+        nvrtc_max_compute_capability = '90'
+
+    return nvrtc_max_compute_capability
+
+
+@_util.memoize(for_each_device=True)
+def _get_arch():
+    # See Supported Compile Options section of NVRTC User Guide for
+    # the maximum value allowed for `--gpu-architecture`.
+    nvrtc_max_compute_capability = _get_max_compute_capability()
+
+    arch = device.Device().compute_capability
+    if arch in _tegra_archs:
+        return arch
+    else:
+        return min(arch, nvrtc_max_compute_capability)
+
+
+@_util.memoize(for_each_device=True)
+def _get_arch_for_options_for_nvrtc(arch=None):
+    # NVRTC in CUDA 11.3+ generates PTX that cannot be run an earlier driver
+    # version than the one included in the used CUDA version, as
+    # documented in:
+    # https://docs.nvidia.com/cuda/archive/11.3.0/nvrtc/index.html#versioning
+    # Here we use `-arch=sm_*` instead of `-arch=compute_*` to directly
+    # generate cubin (SASS) instead of PTX. See #5097 for details.
+    if arch is None:
+        arch = _get_arch()
+    if driver._is_cuda_python():
+        version = runtime.runtimeGetVersion()
+    else:
+        version = _cuda_hip_version
+    if (
+        not _use_ptx and version >= 11010
+        and arch <= _get_max_compute_capability()
+    ):
+        return f'-arch=sm_{arch}', 'cubin'
+    return f'-arch=compute_{arch}', 'ptx'
+
+
+def _is_cudadevrt_needed(options):
+    return any(o for o in options if o in _rdc_flags)
+
+
+def _get_cudadevrt_path():
+    global _cudadevrt
+    if _cudadevrt is not None:
+        return _cudadevrt
+
+    # defer import to here to avoid circular dependency
+    from cupy.cuda import get_cuda_path
+    global _win32
+
+    cudadevrt = get_cuda_path()
+    if cudadevrt is None:
+        raise RuntimeError('CUDA is not found.')
+
+    if _win32:
+        # rely on os.altsep
+        cudadevrt += '/lib/x64/cudadevrt.lib'
+    else:  # linux & osx: search twice as in cupy/install/build.py
+        cudadevrt64 = cudadevrt + '/lib64/libcudadevrt.a'
+        if not os.path.isfile(cudadevrt64):
+            cudadevrt += '/lib/libcudadevrt.a'
+        else:
+            cudadevrt = cudadevrt64
+    if not os.path.isfile(cudadevrt):
+        raise RuntimeError(
+            'Relocatable PTX code is requested, but cudadevrt '
+            'is not found.')
+    return cudadevrt
+
+
+def _remove_rdc_option(options):
+    return tuple(o for o in options if o not in _rdc_flags)
+
+
+def _get_bool_env_variable(name, default):
+    val = os.environ.get(name)
+    if val is None or len(val) == 0:
+        return default
+    try:
+        return int(val) == 1
+    except ValueError:
+        return False
+
+
+_use_ptx = _get_bool_env_variable('CUPY_COMPILE_WITH_PTX', False)
+_jitify_header_source_map_populated = False
+
+
+def _jitify_prep(source, options, cu_path):
+    # TODO(leofang): refactor this?
+    global _jitify_header_source_map_populated
+    if not _jitify_header_source_map_populated:
+        from cupy._core import core
+        jitify._add_sources(core._get_header_source_map())
+        _jitify_header_source_map_populated = True
+
+    # jitify requires the 1st line to be the program name
+    old_source = source
+    source = cu_path + '\n' + source
+
+    # Upon failure, in addition to throw an error Jitify also prints the log
+    # to stdout. In principle we could intercept that by hijacking stdout's
+    # file descriptor (tested locally), but the problem is pytest also does
+    # the same thing internally, causing strange errors when running the tests.
+    # As a result, we currently maintain Jitify's default behavior for easy
+    # debugging, and wait for the upstream to address this issue
+    # (NVIDIA/jitify#79).
+
+    try:
+        name, options, headers, include_names = jitify.jitify(source, options)
+    except Exception as e:  # C++ could throw all kinds of errors
+        cex = CompileException(str(e), old_source, cu_path, options, 'jitify')
+        dump = _get_bool_env_variable(
+            'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+        if dump:
+            cex.dump(sys.stderr)
+        raise JitifyException(str(cex))
+    assert name == cu_path
+
+    return options, headers, include_names
+
+
+_has_usedforsecurity = (sys.version_info >= (3, 9))
+
+
+def _hash_hexdigest(value):
+    if _has_usedforsecurity:
+        hashobj = hashlib.sha1(value, usedforsecurity=False)
+    else:
+        hashobj = hashlib.sha1(value)
+    return hashobj.hexdigest()
+
+
+_hash_length = len(_hash_hexdigest(b''))  # 40 for SHA1
+
+
+def compile_using_nvrtc(source, options=(), arch=None, filename='kern.cu',
+                        name_expressions=None, log_stream=None,
+                        cache_in_memory=False, jitify=False):
+    def _compile(
+            source, options, cu_path, name_expressions, log_stream, jitify):
+
+        if jitify:
+            options, headers, include_names = _jitify_prep(
+                source, options, cu_path)
+        else:
+            headers = include_names = ()
+            major_version, minor_version = _get_nvrtc_version()
+            if major_version >= 12:
+                # Starting with CUDA 12.0, even without using jitify, some
+                # tests cause an error if the following option is not included.
+                options += ('--device-as-default-execution-space',)
+
+        if not runtime.is_hip:
+            arch_opt, method = _get_arch_for_options_for_nvrtc(arch)
+            options += (arch_opt,)
+        else:
+            method = 'ptx'
+
+        prog = _NVRTCProgram(source, cu_path, headers, include_names,
+                             name_expressions=name_expressions, method=method)
+        try:
+            compiled_obj, mapping = prog.compile(options, log_stream)
+        except CompileException as e:
+            dump = _get_bool_env_variable(
+                'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+            if dump:
+                e.dump(sys.stderr)
+            raise
+        return compiled_obj, mapping
+
+    if not cache_in_memory:
+        with tempfile.TemporaryDirectory() as root_dir:
+            cu_path = os.path.join(root_dir, filename)
+
+            with open(cu_path, 'w') as cu_file:
+                cu_file.write(source)
+
+            return _compile(source, options, cu_path,
+                            name_expressions, log_stream, jitify)
+    else:
+        cu_path = '' if not jitify else filename
+        return _compile(source, options, cu_path, name_expressions,
+                        log_stream, jitify)
+
+
+def compile_using_nvcc(source, options=(), arch=None,
+                       filename='kern.cu', code_type='cubin',
+                       separate_compilation=False, log_stream=None):
+    # defer import to here to avoid circular dependency
+    from cupy.cuda import get_nvcc_path
+
+    if not arch:
+        arch = _get_arch()
+
+    if code_type not in ('cubin', 'ptx'):
+        raise ValueError('Invalid code_type %s. Should be cubin or ptx')
+    if code_type == 'ptx':
+        assert not separate_compilation
+
+    arch_str = '-gencode=arch=compute_{cc},code=sm_{cc}'.format(cc=arch)
+    _nvcc = get_nvcc_path()
+    # split() is needed because _nvcc could come from the env var NVCC
+    cmd = _nvcc.split()
+    cmd.append(arch_str)
+
+    with tempfile.TemporaryDirectory() as root_dir:
+        first_part = filename.split('.')[0]
+
+        path = os.path.join(root_dir, first_part)
+        cu_path = '%s.cu' % path
+        result_path = '%s.%s' % (path, code_type)
+
+        with open(cu_path, 'w') as cu_file:
+            cu_file.write(source)
+
+        if not separate_compilation:  # majority cases
+            cmd.append('--%s' % code_type)
+            cmd += list(options)
+            cmd.append(cu_path)
+
+            try:
+                _run_cc(cmd, root_dir, 'nvcc', log_stream)
+            except NVCCException as e:
+                cex = CompileException(str(e), source, cu_path, options,
+                                       'nvcc')
+
+                dump = _get_bool_env_variable(
+                    'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+                if dump:
+                    cex.dump(sys.stderr)
+
+                raise cex
+        else:  # two steps: compile to object and device-link
+            cmd_partial = cmd.copy()
+            cmd_partial.append('--cubin')
+
+            obj = path + '.o'
+            cmd += list(options + ('-o', obj))
+            cmd.append(cu_path)
+
+            try:
+                _run_cc(cmd, root_dir, 'nvcc', log_stream)
+            except NVCCException as e:
+                cex = CompileException(str(e), source, cu_path, options,
+                                       'nvcc')
+
+                dump = _get_bool_env_variable(
+                    'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+                if dump:
+                    cex.dump(sys.stderr)
+
+                raise cex
+
+            options = _remove_rdc_option(options)
+            options += ('--device-link', obj, '-o', path + '.cubin')
+            cmd = cmd_partial + list(options)
+
+            try:
+                _run_cc(cmd, root_dir, 'nvcc', log_stream)
+            except NVCCException as e:
+                cex = CompileException(str(e), '', '', options, 'nvcc')
+                raise cex
+
+        if code_type == 'ptx':
+            with open(result_path, 'rb') as ptx_file:
+                return ptx_file.read()
+        elif code_type == 'cubin':
+            with open(result_path, 'rb') as bin_file:
+                return bin_file.read()
+        else:
+            assert False, code_type
+
+
+def _preprocess(source, options, arch, backend):
+    if backend == 'nvrtc':
+        # For the preprocess it is enough to use PTX method
+        # we don't need to explicitly obtain a CUBIN file.
+        options += ('-arch=compute_{}'.format(arch),)
+        prog = _NVRTCProgram(source)
+        try:
+            result, _ = prog.compile(options)
+        except CompileException as e:
+            dump = _get_bool_env_variable(
+                'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+            if dump:
+                e.dump(sys.stderr)
+            raise
+    elif backend == 'nvcc':
+        try:
+            result = compile_using_nvcc(source, options, arch, 'preprocess.cu',
+                                        code_type='ptx')
+        except CompileException as e:
+            dump = _get_bool_env_variable(
+                'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+            if dump:
+                e.dump(sys.stderr)
+            raise
+    else:
+        raise ValueError('Invalid backend %s' % backend)
+
+    assert isinstance(result, bytes)
+
+    # Extract the part containing version information.
+    return '\n'.join(
+        x for x in result.decode().splitlines() if x.startswith('//'))
+
+
+_default_cache_dir = os.path.expanduser('~/.cupy/kernel_cache')
+
+
+def get_cache_dir():
+    return os.environ.get('CUPY_CACHE_DIR', _default_cache_dir)
+
+
+_empty_file_preprocess_cache: dict = {}
+
+
+def compile_with_cache(*args, **kwargs):
+    # TODO(asi1024): Remove in CuPy v13+.
+    warnings.warn(
+        'cupy.cuda.compile_with_cache has been deprecated in CuPy v10, and'
+        ' will be removed in the future. Use cupy.RawModule or cupy.RawKernel'
+        ' instead.', UserWarning)
+    return _compile_module_with_cache(*args, **kwargs)
+
+
+def _compile_module_with_cache(
+        source, options=(), arch=None, cache_dir=None, extra_source=None,
+        backend='nvrtc', *, enable_cooperative_groups=False,
+        name_expressions=None, log_stream=None, jitify=False):
+
+    if enable_cooperative_groups:
+        if runtime.is_hip:
+            raise ValueError(
+                'Cooperative groups is not supported in HIP.')
+
+    if name_expressions is not None and backend != 'nvrtc':
+        raise NotImplementedError
+
+    # We silently ignore CUPY_CACHE_IN_MEMORY if nvcc/hipcc are in use, because
+    # they must dump files to disk.
+    cache_in_memory = (
+        _get_bool_env_variable('CUPY_CACHE_IN_MEMORY', False)
+        and backend == 'nvrtc')
+
+    if runtime.is_hip:
+        backend = 'hiprtc' if backend == 'nvrtc' else 'hipcc'
+        return _compile_with_cache_hip(
+            source, options, arch, cache_dir, extra_source, backend,
+            name_expressions, log_stream, cache_in_memory)
+    else:
+        return _compile_with_cache_cuda(
+            source, options, arch, cache_dir, extra_source, backend,
+            enable_cooperative_groups, name_expressions, log_stream,
+            cache_in_memory, jitify)
+
+
+def _compile_with_cache_cuda(
+        source, options, arch, cache_dir, extra_source=None, backend='nvrtc',
+        enable_cooperative_groups=False, name_expressions=None,
+        log_stream=None, cache_in_memory=False, jitify=False):
+    # NVRTC does not use extra_source. extra_source is used for cache key.
+    global _empty_file_preprocess_cache
+    if cache_dir is None:
+        cache_dir = get_cache_dir()
+    if arch is None:
+        arch = _get_arch()
+
+    options += ('-ftz=true',)
+
+    if enable_cooperative_groups:
+        # `cooperative_groups` requires relocatable device code.
+        options += ('--device-c',)
+
+    if _get_bool_env_variable('CUPY_CUDA_COMPILE_WITH_DEBUG', False):
+        options += ('--device-debug', '--generate-line-info')
+
+    is_jitify_requested = ('-DCUPY_USE_JITIFY' in options)
+    if jitify and not is_jitify_requested:
+        # jitify is set in RawKernel/RawModule, translate it to an option
+        # that is useless to the compiler, but can be used as part of the
+        # hash key
+        options += ('-DCUPY_USE_JITIFY',)
+    elif is_jitify_requested and not jitify:
+        # jitify is requested internally, just set the flag
+        jitify = True
+    if jitify and backend != 'nvrtc':
+        raise ValueError('jitify only works with NVRTC')
+
+    env = ((arch, options, _get_nvrtc_version(), backend)
+           + _get_arch_for_options_for_nvrtc(arch))
+    base = _empty_file_preprocess_cache.get(env, None)
+    if base is None:
+        # This is for checking NVRTC/NVCC compiler internal version
+        base = _preprocess('', options, arch, backend)
+        _empty_file_preprocess_cache[env] = base
+
+    key_src = '%s %s %s %s' % (env, base, source, extra_source)
+    key_src = key_src.encode('utf-8')
+    name = _hash_hexdigest(key_src) + '.cubin'
+
+    mod = function.Module()
+
+    if not cache_in_memory:
+        # Read from disk cache
+        if not os.path.isdir(cache_dir):
+            os.makedirs(cache_dir, exist_ok=True)
+
+        # To handle conflicts in concurrent situation, we adopt lock-free
+        # method to avoid performance degradation.
+        # We force recompiling to retrieve C++ mangled names if so desired.
+        path = os.path.join(cache_dir, name)
+        if os.path.exists(path) and not name_expressions:
+            with open(path, 'rb') as file:
+                data = file.read()
+            if len(data) >= _hash_length:
+                hash = data[:_hash_length]
+                cubin = data[_hash_length:]
+                cubin_hash = _hash_hexdigest(cubin).encode('ascii')
+                if hash == cubin_hash:
+                    mod.load(cubin)
+                    return mod
+    else:
+        # Enforce compiling -- the resulting kernel will be cached elsewhere,
+        # so we do nothing
+        pass
+
+    if backend == 'nvrtc':
+        cu_name = '' if cache_in_memory else name + '.cu'
+        ptx, mapping = compile_using_nvrtc(
+            source, options, arch, cu_name, name_expressions,
+            log_stream, cache_in_memory, jitify)
+        if _is_cudadevrt_needed(options):
+            # for separate compilation
+            ls = function.LinkState()
+            ls.add_ptr_data(ptx, 'cupy.ptx')
+            _cudadevrt = _get_cudadevrt_path()
+            ls.add_ptr_file(_cudadevrt)
+            cubin = ls.complete()
+        else:
+            cubin = ptx
+        mod._set_mapping(mapping)
+    elif backend == 'nvcc':
+        rdc = _is_cudadevrt_needed(options)
+        cubin = compile_using_nvcc(source, options, arch,
+                                   name + '.cu', code_type='cubin',
+                                   separate_compilation=rdc,
+                                   log_stream=log_stream)
+    else:
+        raise ValueError('Invalid backend %s' % backend)
+
+    if not cache_in_memory:
+        # Write to disk cache
+        cubin_hash = _hash_hexdigest(cubin).encode('ascii')
+
+        # shutil.move is not atomic operation, so it could result in a
+        # corrupted file. We detect it by appending a hash at the beginning
+        # of each cache file. If the file is corrupted, it will be ignored
+        # next time it is read.
+        with tempfile.NamedTemporaryFile(dir=cache_dir, delete=False) as tf:
+            tf.write(cubin_hash)
+            tf.write(cubin)
+            temp_path = tf.name
+        shutil.move(temp_path, path)
+
+        # Save .cu source file along with .cubin
+        if _get_bool_env_variable('CUPY_CACHE_SAVE_CUDA_SOURCE', False):
+            with open(path + '.cu', 'w') as f:
+                f.write(source)
+    else:
+        # we don't do any disk I/O
+        pass
+
+    mod.load(cubin)
+    return mod
+
+
+class CompileException(Exception):
+
+    def __init__(self, msg, source, name, options, backend='nvrtc'):
+        self._msg = msg
+        self.source = source
+        self.name = name
+        self.options = options
+        self.backend = backend
+        super(CompileException, self).__init__()
+
+    def __reduce__(self):
+        return (type(self), (self._msg, self.source, self.name,
+                             self.options, self.backend))
+
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self):
+        return self.get_message()
+
+    def get_message(self):
+        return self._msg
+
+    def dump(self, f):
+        lines = self.source.split('\n')
+        digits = int(math.floor(math.log10(len(lines)))) + 1
+        linum_fmt = '{{:0{}d}} '.format(digits)
+        f.write('{} '.format(self.backend.upper()))
+        f.write('compilation error: {}\n'.format(self))
+        f.write('-----\n')
+        f.write('Name: {}\n'.format(self.name))
+        f.write('Options: {}\n'.format(' '.join(self.options)))
+        f.write('CUDA source:\n')
+        for i, line in enumerate(lines):
+            f.write(linum_fmt.format(i + 1) + line.rstrip() + '\n')
+        f.write('-----\n')
+        f.flush()
+
+
+class _NVRTCProgram(object):
+
+    def __init__(self, src, name='default_program', headers=(),
+                 include_names=(), name_expressions=None, method='ptx'):
+        self.ptr = None
+
+        if isinstance(src, bytes):
+            src = src.decode('UTF-8')
+        if isinstance(name, bytes):
+            name = name.decode('UTF-8')
+
+        self.src = src
+        self.name = name
+        self.ptr = nvrtc.createProgram(src, name, headers, include_names)
+        self.name_expressions = name_expressions
+        self.method = method
+
+    def __del__(self, is_shutting_down=_util.is_shutting_down):
+        if is_shutting_down():
+            return
+        if self.ptr:
+            nvrtc.destroyProgram(self.ptr)
+
+    def compile(self, options=(), log_stream=None):
+        try:
+            if self.name_expressions:
+                for ker in self.name_expressions:
+                    nvrtc.addNameExpression(self.ptr, ker)
+            nvrtc.compileProgram(self.ptr, options)
+            mapping = None
+            if self.name_expressions:
+                mapping = {}
+                for ker in self.name_expressions:
+                    mapping[ker] = nvrtc.getLoweredName(self.ptr, ker)
+            if log_stream is not None:
+                log_stream.write(nvrtc.getProgramLog(self.ptr))
+            # This is to ensure backwards compatibility with nvrtc
+            if self.method == 'cubin':
+                return nvrtc.getCUBIN(self.ptr), mapping
+            elif self.method == 'ptx':
+                return nvrtc.getPTX(self.ptr), mapping
+            # TODO(leofang): support JIT LTO using nvrtc.getNVVM()?
+            # need -dlto and -arch=compute_XX
+            else:
+                raise RuntimeError('Unknown NVRTC compile method')
+        except nvrtc.NVRTCError:
+            log = nvrtc.getProgramLog(self.ptr)
+            raise CompileException(log, self.src, self.name, options,
+                                   'nvrtc' if not runtime.is_hip else 'hiprtc')
+
+
+def is_valid_kernel_name(name):
+    return re.match('^[a-zA-Z_][a-zA-Z_0-9]*$', name) is not None
+
+
+def compile_using_hipcc(source, options, arch, log_stream=None):
+    # As of ROCm 3.5.0 hiprtc/hipcc can automatically pick up the
+    # right arch without setting HCC_AMDGPU_TARGET, so we don't need
+    # to set arch here
+    cmd = ['hipcc', '--genco'] + list(options)
+
+    with tempfile.TemporaryDirectory() as root_dir:
+        path = os.path.join(root_dir, 'kern')
+        in_path = path + '.cpp'
+        out_path = path + '.hsaco'
+
+        with open(in_path, 'w') as f:
+            f.write(source)
+
+        cmd += [in_path, '-o', out_path]
+
+        try:
+            output = _run_cc(cmd, root_dir, 'hipcc', log_stream)
+        except HIPCCException as e:
+            cex = CompileException(str(e), source, in_path, options,
+                                   'hipcc')
+
+            dump = _get_bool_env_variable(
+                'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+            if dump:
+                cex.dump(sys.stderr)
+
+            raise cex
+        if not os.path.isfile(out_path):
+            raise HIPCCException(
+                '`hipcc` command does not generate output file. \n'
+                'command: {0}\n'
+                'stdout/stderr: \n'
+                '{1}'.format(cmd, output))
+        with open(out_path, 'rb') as f:
+            return f.read()
+
+
+# TODO(leofang): consider merge _preprocess_hipcc with _preprocess_hiprtc,
+# perhaps also with _preprocess?
+def _preprocess_hipcc(source, options):
+    cmd = ['hipcc', '--preprocess'] + list(options)
+    with tempfile.TemporaryDirectory() as root_dir:
+        path = os.path.join(root_dir, 'kern')
+        cu_path = '%s.cpp' % path
+
+        with open(cu_path, 'w') as cu_file:
+            cu_file.write(source)
+
+        cmd.append(cu_path)
+        pp_src = _run_cc(cmd, root_dir, 'hipcc')
+        assert isinstance(pp_src, str)
+        return re.sub('(?m)^#.*$', '', pp_src)
+
+
+def _preprocess_hiprtc(source, options):
+    # source is ignored
+    if _cuda_hip_version >= 40400000:
+        # HIP runtime headers can be no longer explicitly included on ROCm 4.5+
+        code = '''
+        // hiprtc segfaults if the input code is empty
+        __global__ void _cupy_preprocess_dummy_kernel_() { }
+        '''
+    else:
+        code = '''
+        // hiprtc segfaults if the input code is empty
+        #include <hip/hip_runtime.h>
+        __global__ void _cupy_preprocess_dummy_kernel_() { }
+        '''
+
+    prog = _NVRTCProgram(code)
+    try:
+        result, _ = prog.compile(options)
+    except CompileException as e:
+        dump = _get_bool_env_variable(
+            'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+        if dump:
+            e.dump(sys.stderr)
+        raise
+    assert isinstance(result, bytes)
+    return result
+
+
+_hip_extra_source = None
+
+
+def _convert_to_hip_source(source, extra_source, is_hiprtc):
+    if not is_hiprtc:
+        return '#include <hip/hip_runtime.h>\n' + source
+    if _cuda_hip_version >= 40400000:
+        # HIP runtime headers can be no longer explicitly included on ROCm 4.5+
+        return source
+    if _cuda_hip_version >= 402:
+        # "-I" is fixed on ROCm 4.2.0+
+        return '#include <hip/hip_runtime.h>\n' + source
+
+    # Workaround for hiprtc: it does not follow the -I option to search
+    # headers (as of ROCm 3.5.0), so we must prepend all CuPy's headers
+    global _hip_extra_source
+    if _hip_extra_source is None:
+        if extra_source is not None:
+            extra_source = extra_source.split('\n')
+            extra_source = [line for line in extra_source if (
+                not line.startswith('#include')
+                and not line.startswith('#pragma once'))]
+            _hip_extra_source = extra_source = '\n'.join(extra_source)
+
+    source = source.split('\n')
+    source = [line for line in source if not line.startswith('#include')]
+    source = ('#include <hip/hip_runtime.h>\n#include <hip/hip_fp16.h>\n'
+              + _hip_extra_source + '\n'.join(source))
+
+    return source
+
+
+# TODO(leofang): evaluate if this can be merged with _compile_with_cache_cuda()
+def _compile_with_cache_hip(source, options, arch, cache_dir, extra_source,
+                            backend='hiprtc', name_expressions=None,
+                            log_stream=None, cache_in_memory=False,
+                            use_converter=True):
+    global _empty_file_preprocess_cache
+
+    # TODO(leofang): this might be possible but is currently undocumented
+    if _is_cudadevrt_needed(options):
+        raise ValueError('separate compilation is not supported in HIP')
+
+    # HIP's equivalent of -ftz=true, see ROCm-Developer-Tools/HIP#2252
+    # Notes:
+    # - For hipcc, this should just work, as invalid options would cause errors
+    #   See https://clang.llvm.org/docs/ClangCommandLineReference.html.
+    # - For hiprtc, this is a no-op until the compiler options like -D and -I
+    #   are accepted, see ROCm-Developer-Tools/HIP#2182 and
+    #   ROCm-Developer-Tools/HIP#2248
+    options += ('-fcuda-flush-denormals-to-zero',)
+
+    # Workaround ROCm 4.3 LLVM_PATH issue in hipRTC #5689
+    rocm_build_version = driver.get_build_version()
+    if rocm_build_version >= 40300000 and rocm_build_version < 40500000:
+        options += (
+            '-I' + get_rocm_path() + '/llvm/lib/clang/13.0.0/include/',)
+
+    if cache_dir is None:
+        cache_dir = get_cache_dir()
+    # As of ROCm 3.5.0 hiprtc/hipcc can automatically pick up the
+    # right arch without setting HCC_AMDGPU_TARGET, so we don't need
+    # to tell the compiler which arch we are targeting. But, we still
+    # need to know arch as part of the cache key:
+    if arch is None:
+        # On HIP, gcnArch is computed from "compute capability":
+        # https://github.com/ROCm-Developer-Tools/HIP/blob/rocm-4.0.0/rocclr/hip_device.cpp#L202
+        arch = device.Device().compute_capability
+    if use_converter:
+        source = _convert_to_hip_source(source, extra_source,
+                                        is_hiprtc=(backend == 'hiprtc'))
+
+    env = (arch, options, _get_nvrtc_version(), backend)
+    base = _empty_file_preprocess_cache.get(env, None)
+    if base is None:
+        # This is for checking HIPRTC/HIPCC compiler internal version
+        if backend == 'hiprtc':
+            base = _preprocess_hiprtc('', options)
+        else:
+            base = _preprocess_hipcc('', options)
+        _empty_file_preprocess_cache[env] = base
+
+    key_src = '%s %s %s %s' % (env, base, source, extra_source)
+    key_src = key_src.encode('utf-8')
+    name = _hash_hexdigest(key_src) + '.hsaco'
+
+    mod = function.Module()
+
+    if not cache_in_memory:
+        # Read from disk cache
+        if not os.path.isdir(cache_dir):
+            os.makedirs(cache_dir, exist_ok=True)
+
+        # To handle conflicts in concurrent situation, we adopt lock-free
+        # method to avoid performance degradation.
+        # We force recompiling to retrieve C++ mangled names if so desired.
+        path = os.path.join(cache_dir, name)
+        if os.path.exists(path) and not name_expressions:
+            with open(path, 'rb') as f:
+                data = f.read()
+            if len(data) >= _hash_length:
+                hash_value = data[:_hash_length]
+                binary = data[_hash_length:]
+                binary_hash = _hash_hexdigest(binary).encode('ascii')
+                if hash_value == binary_hash:
+                    mod.load(binary)
+                    return mod
+    else:
+        # Enforce compiling -- the resulting kernel will be cached elsewhere,
+        # so we do nothing
+        pass
+
+    if backend == 'hiprtc':
+        # compile_using_nvrtc calls hiprtc for hip builds
+        binary, mapping = compile_using_nvrtc(
+            source, options, arch, name + '.cu', name_expressions,
+            log_stream, cache_in_memory)
+        mod._set_mapping(mapping)
+    else:
+        binary = compile_using_hipcc(source, options, arch, log_stream)
+
+    if not cache_in_memory:
+        # Write to disk cache
+        binary_hash = _hash_hexdigest(binary).encode('ascii')
+
+        # shutil.move is not atomic operation, so it could result in a
+        # corrupted file. We detect it by appending a hash at the beginning
+        # of each cache file. If the file is corrupted, it will be ignored
+        # next time it is read.
+        with tempfile.NamedTemporaryFile(dir=cache_dir, delete=False) as tf:
+            tf.write(binary_hash)
+            tf.write(binary)
+            temp_path = tf.name
+        shutil.move(temp_path, path)
+
+        # Save .cu source file along with .hsaco
+        if _get_bool_env_variable('CUPY_CACHE_SAVE_CUDA_SOURCE', False):
+            with open(path + '.cpp', 'w') as f:
+                f.write(source)
+    else:
+        # we don't do any disk I/O
+        pass
+
+    mod.load(binary)
+    return mod
diff --git a/cupy/cuda/cub.pxd b/cupy/cuda/cub.pxd
new file mode 100644
index 0000000..9276084
--- /dev/null
+++ b/cupy/cuda/cub.pxd
@@ -0,0 +1,20 @@
+from cupy._core.core cimport _ndarray_base
+
+
+cpdef enum cupy_cub_op:
+    CUPY_CUB_SUM = 0
+    CUPY_CUB_MIN = 1
+    CUPY_CUB_MAX = 2
+    CUPY_CUB_ARGMIN = 3
+    CUPY_CUB_ARGMAX = 4
+    CUPY_CUB_CUMSUM = 5
+    CUPY_CUB_CUMPROD = 6
+    CUPY_CUB_PROD = 7
+
+
+# TODO(leofang): cimport these in other modules?
+cpdef cub_reduction(_ndarray_base arr, op,
+                    axis=*, dtype=*, _ndarray_base out=*, keepdims=*)
+cpdef cub_scan(_ndarray_base arr, op)
+
+cpdef bint _cub_device_segmented_reduce_axis_compatible(tuple, Py_ssize_t, str)
diff --git a/cupy/cuda/cub.pyx b/cupy/cuda/cub.pyx
new file mode 100644
index 0000000..8b62e7b
--- /dev/null
+++ b/cupy/cuda/cub.pyx
@@ -0,0 +1,585 @@
+# distutils: language = c++
+
+"""Wrapper of CUB functions for CuPy API."""
+
+from cpython cimport sequence
+from libc.stdint cimport intptr_t
+
+from cupy_backends.cuda.api cimport runtime
+from cupy._core.core cimport _internal_ascontiguousarray
+from cupy._core.internal cimport _contig_axes, is_in
+from cupy.cuda cimport common
+from cupy.cuda cimport device
+from cupy.cuda cimport memory
+from cupy.cuda cimport stream
+
+import cupy
+import cupy._core as _core
+from cupy.cuda import memory as _memory
+import numpy
+
+
+###############################################################################
+# Const
+###############################################################################
+
+CUB_support_dtype_without_half = [numpy.int8, numpy.uint8,
+                                  numpy.int16, numpy.uint16,
+                                  numpy.int32, numpy.uint32,
+                                  numpy.int64, numpy.uint64,
+                                  numpy.float32, numpy.float64,
+                                  numpy.complex64, numpy.complex128]
+
+CUB_support_dtype_with_half = CUB_support_dtype_without_half + [numpy.float16]
+
+CUB_support_dtype = {}
+
+CUB_sum_support_dtype_without_half = [numpy.int64, numpy.uint64,
+                                      numpy.float32, numpy.float64,
+                                      numpy.complex64, numpy.complex128]
+
+CUB_sum_support_dtype_with_half = \
+    CUB_sum_support_dtype_without_half + [numpy.float16]
+
+CUB_sum_support_dtype = {}
+
+
+###############################################################################
+# Extern
+###############################################################################
+
+cdef extern from 'cupy_cub.h' nogil:
+    ctypedef void* Stream_t 'cudaStream_t'
+
+    void cub_device_reduce(void*, size_t&, void*, void*, int, Stream_t,
+                           int, int)
+    void cub_device_segmented_reduce(void*, size_t&, void*, void*, int, int,
+                                     Stream_t, int, int)
+    void cub_device_spmv(void*, size_t&, void*, void*, void*, void*, void*,
+                         int, int, int, Stream_t, int)
+    void cub_device_scan(void*, size_t&, void*, void*, int, Stream_t, int, int)
+    void cub_device_histogram_range(void*, size_t&, void*, void*, int, void*,
+                                    size_t, Stream_t, int)
+    void cub_device_histogram_even(void*, size_t&, void*, void*, int, int, int,
+                                   size_t, Stream_t, int)
+    size_t cub_device_reduce_get_workspace_size(void*, void*, int, Stream_t,
+                                                int, int)
+    size_t cub_device_segmented_reduce_get_workspace_size(
+        void*, void*, int, int, Stream_t, int, int)
+    size_t cub_device_spmv_get_workspace_size(
+        void*, void*, void*, void*, void*, int, int, int, Stream_t, int)
+    size_t cub_device_scan_get_workspace_size(
+        void*, void*, int, Stream_t, int, int)
+    size_t cub_device_histogram_range_get_workspace_size(
+        void*, void*, int, void*, size_t, Stream_t, int)
+    size_t cub_device_histogram_even_get_workspace_size(
+        void*, void*, int, int, int, size_t, Stream_t, int)
+
+    # Build-time version
+    int CUPY_CUB_VERSION_CODE
+
+
+###############################################################################
+# Python interface
+###############################################################################
+
+available = True
+
+
+def get_build_version():
+    if CUPY_CUB_VERSION_CODE == -1:
+        return '<unknown>'
+    return CUPY_CUB_VERSION_CODE
+
+
+cpdef _get_cuda_build_version():
+    if CUPY_CUDA_VERSION > 0:
+        return CUPY_CUDA_VERSION
+    elif CUPY_HIP_VERSION > 0:
+        return CUPY_HIP_VERSION
+    else:
+        return 0
+
+
+cdef tuple _get_output_shape(_ndarray_base arr, tuple out_axis, bint keepdims):
+    cdef tuple out_shape
+
+    if not keepdims:
+        out_shape = tuple([arr.shape[axis] for axis in out_axis])
+    else:
+        out_shape = tuple([arr.shape[axis] if axis in out_axis else 1
+                           for axis in range(arr.ndim)])
+    return out_shape
+
+
+cpdef Py_ssize_t _preprocess_array(tuple arr_shape, tuple reduce_axis,
+                                   tuple out_axis, str order) except -1:
+    '''
+    This function more or less follows the logic of _get_permuted_args() in
+    reduction.pxi. The input array arr is C- or F- contiguous along axis.
+    '''
+    cdef tuple axis_permutes
+    cdef Py_ssize_t contiguous_size = 1
+
+    # one more sanity check?
+    if order == 'C':
+        axis_permutes = out_axis + reduce_axis
+    elif order == 'F':
+        axis_permutes = reduce_axis + out_axis
+    elif order == 'CF':
+        axis_permutes = tuple(set(out_axis + reduce_axis))
+    assert axis_permutes == tuple(range(len(arr_shape)))
+
+    for axis in reduce_axis:
+        contiguous_size *= arr_shape[axis]
+    return contiguous_size
+
+
+def device_reduce(_ndarray_base x, op, tuple reduce_axis, tuple out_axis,
+                  out=None, bint keepdims=False):
+    cdef _ndarray_base y
+    cdef memory.MemoryPointer ws
+    cdef int dtype_id, ndim_out, kv_bytes, x_size, op_code
+    cdef size_t ws_size
+    cdef void *x_ptr
+    cdef void *y_ptr
+    cdef void *ws_ptr
+    cdef Stream_t s
+    cdef tuple out_shape
+
+    if keepdims:
+        out_shape = _get_output_shape(x, out_axis, keepdims)
+        ndim_out = len(out_shape)
+    else:
+        ndim_out = 0
+    reduce_shape = _get_output_shape(x, reduce_axis, False)
+
+    if out is not None and out.ndim != ndim_out:
+        raise ValueError(
+            'output parameter for reduction operation has the wrong number of '
+            'dimensions')
+    if op not in (CUPY_CUB_SUM, CUPY_CUB_PROD, CUPY_CUB_MIN, CUPY_CUB_MAX,
+                  CUPY_CUB_ARGMIN, CUPY_CUB_ARGMAX):
+        raise ValueError('only CUPY_CUB_SUM, CUPY_CUB_PROD, CUPY_CUB_MIN, '
+                         'CUPY_CUB_MAX, CUPY_CUB_ARGMIN, and CUPY_CUB_ARGMAX '
+                         'are supported.')
+    if op not in (CUPY_CUB_SUM, CUPY_CUB_PROD) and is_in(reduce_shape, 0):
+        raise ValueError('zero-size array to reduction operation {} which has '
+                         'no identity'.format(op.name))
+    x = _internal_ascontiguousarray(x)
+
+    if op in (CUPY_CUB_SUM, CUPY_CUB_PROD, CUPY_CUB_MIN, CUPY_CUB_MAX):
+        y = _core.ndarray((), x.dtype)
+    else:  # argmin and argmax
+        # cub::KeyValuePair has 1 int + 1 arbitrary type
+        kv_bytes = (4 + x.dtype.itemsize)
+        y = _core.ndarray((kv_bytes,), numpy.int8)
+    x_ptr = <void *>x.data.ptr
+    y_ptr = <void *>y.data.ptr
+    dtype_id = common._get_dtype_id(x.dtype)
+    s = <Stream_t>stream.get_current_stream_ptr()
+    x_size = <int>x.size
+    ws_size = cub_device_reduce_get_workspace_size(x_ptr, y_ptr, x.size, s,
+                                                   op, dtype_id)
+    ws = memory.alloc(ws_size)
+    ws_ptr = <void *>ws.ptr
+    op_code = <int>op
+    with nogil:
+        cub_device_reduce(ws_ptr, ws_size, x_ptr, y_ptr, x_size, s, op_code,
+                          dtype_id)
+    if op in (CUPY_CUB_ARGMIN, CUPY_CUB_ARGMAX):
+        # get key from KeyValuePair: need to reinterpret the first 4 bytes
+        # and then cast it
+        y = y[0:4].view(numpy.int32).astype(numpy.int64)[0]
+        y = y.reshape(())
+
+    if keepdims:
+        y = y.reshape(out_shape)
+    if out is not None:
+        cupy._core.elementwise_copy(y, out)
+        y = out
+    return y
+
+
+def device_segmented_reduce(_ndarray_base x, op, tuple reduce_axis,
+                            tuple out_axis, out=None, bint keepdims=False,
+                            Py_ssize_t contiguous_size=0):
+    cdef _ndarray_base y
+    cdef str order
+    cdef memory.MemoryPointer ws
+    cdef void* x_ptr
+    cdef void* y_ptr
+    cdef void* ws_ptr
+    cdef int dtype_id, n_segments, op_code
+    cdef size_t ws_size
+    cdef tuple out_shape
+    cdef Stream_t s
+
+    if op not in (CUPY_CUB_SUM, CUPY_CUB_PROD, CUPY_CUB_MIN, CUPY_CUB_MAX):
+        raise ValueError('only CUPY_CUB_SUM, CUPY_CUB_PROD, CUPY_CUB_MIN, '
+                         'and CUPY_CUB_MAX are supported.')
+    reduce_shape = _get_output_shape(x, reduce_axis, False)
+    if op not in (CUPY_CUB_SUM, CUPY_CUB_PROD) and is_in(reduce_shape, 0):
+        raise ValueError('zero-size array to reduction operation {} which has '
+                         'no identity'.format(op.name))
+    if x.flags.c_contiguous:
+        order = 'C'
+    elif x.flags.f_contiguous:
+        order = 'F'
+    else:  # impossible at this point, just in case
+        raise RuntimeError('input is neither C- nor F- contiguous.')
+
+    # prepare input
+    out_shape = _get_output_shape(x, out_axis, keepdims)
+    x_ptr = <void*>x.data.ptr
+    y = _core.ndarray(out_shape, dtype=x.dtype, order=order)
+    y_ptr = <void*>y.data.ptr
+    if out is not None and out.shape != out_shape:
+        raise ValueError(
+            'output parameter for reduction operation has the wrong shape')
+    if x.size == 0:  # for CUPY_CUB_SUM & CUPY_CUB_PROD
+        if out is not None:
+            y = out
+        if op == CUPY_CUB_SUM:
+            y.fill(0)
+        elif op == CUPY_CUB_PROD:
+            y.fill(1)
+        return y
+    n_segments = x.size//contiguous_size
+    s = <Stream_t>stream.get_current_stream_ptr()
+    dtype_id = common._get_dtype_id(x.dtype)
+
+    # get workspace size and then fire up
+    ws_size = cub_device_segmented_reduce_get_workspace_size(
+        x_ptr, y_ptr, n_segments, contiguous_size, s, op, dtype_id)
+    ws = memory.alloc(ws_size)
+    ws_ptr = <void*>ws.ptr
+    op_code = <int>op
+    with nogil:
+        cub_device_segmented_reduce(ws_ptr, ws_size, x_ptr, y_ptr, n_segments,
+                                    contiguous_size, s, op_code, dtype_id)
+
+    if out is not None:
+        cupy._core.elementwise_copy(y, out)
+        y = out
+    return y
+
+
+def device_csrmv(int n_rows, int n_cols, int nnz, _ndarray_base values,
+                 _ndarray_base indptr, _ndarray_base indices, _ndarray_base x):
+    cdef _ndarray_base y
+    cdef memory.MemoryPointer ws
+    cdef void* values_ptr
+    cdef void* row_offsets_ptr
+    cdef void* col_indices_ptr
+    cdef void* x_ptr
+    cdef void* y_ptr
+    cdef void* ws_ptr
+    cdef int dtype_id
+    cdef size_t ws_size
+    cdef Stream_t s
+
+    if x.ndim != 1:
+        raise ValueError('array must be 1d')
+    if x.size != n_cols:
+        raise ValueError("size of array does not match the CSR matrix")
+    if runtime._is_hip_environment:
+        raise RuntimeError("hipCUB does not support SpMV")
+
+    if values.dtype == x.dtype:
+        dtype = values.dtype
+    else:
+        dtype = numpy.promote_types(values.dtype, x.dtype)
+        values = values.astype(dtype, "C", None, None, False)
+        x = x.astype(dtype, "C", None, None, False)
+
+    # CSR matrix attributes
+    values_ptr = <void*>values.data.ptr
+    row_offsets_ptr = <void*>indptr.data.ptr
+    col_indices_ptr = <void*>indices.data.ptr
+
+    x_ptr = <void*>x.data.ptr
+
+    # prepare output array
+    y = _core.ndarray((n_rows,), dtype=dtype)
+    y_ptr = <void*>y.data.ptr
+
+    s = <Stream_t>stream.get_current_stream_ptr()
+    dtype_id = common._get_dtype_id(dtype)
+
+    # get workspace size and then fire up
+    ws_size = cub_device_spmv_get_workspace_size(
+        values_ptr, row_offsets_ptr, col_indices_ptr, x_ptr, y_ptr, n_rows,
+        n_cols, nnz, s, dtype_id)
+    ws = memory.alloc(ws_size)
+    ws_ptr = <void *>ws.ptr
+    with nogil:
+        cub_device_spmv(ws_ptr, ws_size, values_ptr, row_offsets_ptr,
+                        col_indices_ptr, x_ptr, y_ptr, n_rows, n_cols, nnz, s,
+                        dtype_id)
+
+    return y
+
+
+def device_scan(_ndarray_base x, op):
+    cdef memory.MemoryPointer ws
+    cdef int dtype_id, x_size, op_code
+    cdef size_t ws_size
+    cdef void *x_ptr
+    cdef void *ws_ptr
+    cdef Stream_t s
+
+    if op not in (CUPY_CUB_CUMSUM, CUPY_CUB_CUMPROD):
+        raise ValueError('only CUPY_CUB_CUMSUM and CUPY_CUB_CUMPROD '
+                         'are supported.')
+
+    # determine shape: x is either 1D (with axis=None,0) or ND but ravelled.
+    x_size = <int>x.size
+    if x_size == 0:
+        return x
+
+    x = _internal_ascontiguousarray(x)
+    x_ptr = <void *>x.data.ptr
+    s = <Stream_t>stream.get_current_stream_ptr()
+    dtype_id = common._get_dtype_id(x.dtype)
+    ws_size = cub_device_scan_get_workspace_size(x_ptr, x_ptr, x_size, s,
+                                                 op, dtype_id)
+    ws = memory.alloc(ws_size)
+    ws_ptr = <void *>ws.ptr
+    op_code = <int>op
+    with nogil:
+        # the scan is in-place
+        cub_device_scan(ws_ptr, ws_size, x_ptr, x_ptr, x_size, s,
+                        op_code, dtype_id)
+    return x
+
+
+def device_histogram(_ndarray_base x, _ndarray_base y, bins):
+    cdef memory.MemoryPointer ws
+    cdef size_t ws_size, n_samples
+    cdef int dtype_id, n_bins
+    cdef void* x_ptr
+    cdef void* bins_ptr
+    cdef void* y_ptr
+    cdef void* ws_ptr
+    cdef Stream_t s
+    cdef bint is_even
+
+    # TODO(leofang): perhaps not needed?
+    # y is guaranteed contiguous
+    x = _internal_ascontiguousarray(x)
+    if isinstance(bins, _ndarray_base):
+        bins = _internal_ascontiguousarray(bins)
+        bins_ptr = <void*><intptr_t>bins.data.ptr
+        n_bins = bins.size
+        is_even = False
+    else:
+        n_bins = bins
+        is_even = True
+        if runtime._is_hip_environment:
+            raise RuntimeError("not supported yet")
+        if x.dtype.kind not in 'bui':
+            raise ValueError("only integer input is supported")
+    assert y.size == n_bins - 1
+
+    x_ptr = <void*>x.data.ptr
+    y_ptr = <void*>y.data.ptr
+    n_samples = x.size
+    s = <Stream_t>stream.get_current_stream_ptr()
+    dtype_id = common._get_dtype_id(x.dtype)
+
+    if is_even:
+        ws_size = cub_device_histogram_even_get_workspace_size(
+            x_ptr, y_ptr, n_bins, 0, n_bins-1, n_samples, s, dtype_id)
+    else:
+        ws_size = cub_device_histogram_range_get_workspace_size(
+            x_ptr, y_ptr, n_bins, bins_ptr, n_samples, s, dtype_id)
+    ws = memory.alloc(ws_size)
+    ws_ptr = <void*>ws.ptr
+
+    with nogil:
+        if is_even:
+            cub_device_histogram_even(
+                ws_ptr, ws_size, x_ptr, y_ptr, n_bins, 0, n_bins-1,
+                n_samples, s, dtype_id)
+        else:
+            cub_device_histogram_range(
+                ws_ptr, ws_size, x_ptr, y_ptr, n_bins,
+                bins_ptr, n_samples, s, dtype_id)
+    return y
+
+
+cpdef bint _cub_device_segmented_reduce_axis_compatible(
+        tuple cub_axis, Py_ssize_t ndim, str order):
+    # This function checks if the reduced axes are C- or F- contiguous.
+    if _contig_axes(cub_axis):
+        if order == 'C':
+            return (cub_axis[-1] == (ndim - 1))
+        elif order == 'F':
+            return (cub_axis[0] == 0)
+    return False
+
+
+cdef bint can_use_device_reduce(
+        _ndarray_base x, int op, tuple out_axis, dtype=None) except*:
+    return (
+        out_axis is ()
+        and _cub_reduce_dtype_compatible(x.dtype, op, dtype)
+        and x.size <= 0x7fffffff)  # until we resolve cupy/cupy#3309
+
+
+cdef (bint, Py_ssize_t) can_use_device_segmented_reduce(  # noqa: E211
+        _ndarray_base x, int op, tuple reduce_axis, tuple out_axis,
+        dtype=None, str order='C') except*:
+    if not _cub_reduce_dtype_compatible(x.dtype, op, dtype):
+        return (False, 0)
+    # NumPy/CuPy quirk: zero-size arrays are both C- & F- contig...
+    if x.size != 0:
+        if not _cub_device_segmented_reduce_axis_compatible(
+                reduce_axis, x.ndim, order):
+            return (False, 0)
+    else:
+        order = 'CF'  # for computing the contig size
+    # until we resolve cupy/cupy#3309
+    cdef Py_ssize_t contiguous_size = _preprocess_array(
+        x.shape, reduce_axis, out_axis, order)
+    return (contiguous_size <= 0x7fffffff, contiguous_size)
+
+
+cdef _cub_support_dtype(bint sum_mode, int dev_id):
+    if sum_mode:
+        support_dtype_dict = CUB_sum_support_dtype
+        with_half = CUB_sum_support_dtype_with_half
+        without_half = CUB_sum_support_dtype_without_half
+    else:
+        support_dtype_dict = CUB_support_dtype
+        with_half = CUB_support_dtype_with_half
+        without_half = CUB_support_dtype_without_half
+
+    if dev_id not in support_dtype_dict:
+        if common._is_fp16_supported():
+            support_dtype = with_half
+        else:
+            support_dtype = without_half
+
+        support_dtype_dict[dev_id] = support_dtype
+
+    return support_dtype_dict[dev_id]
+
+
+cdef _cub_reduce_dtype_compatible(x_dtype, int op, dtype=None):
+    cdef int dev_id = device.get_device_id()
+
+    if dtype is None:
+        if op in (CUPY_CUB_SUM, CUPY_CUB_PROD):
+            # auto dtype:
+            # CUB reduce_sum does not support dtype promotion.
+            # See _sum_auto_dtype in cupy/_core/_routines_math.pyx for which
+            # dtypes are promoted.
+            support_dtype = _cub_support_dtype(True, dev_id)
+        else:
+            support_dtype = _cub_support_dtype(False, dev_id)
+    elif dtype == x_dtype:
+        support_dtype = _cub_support_dtype(False, dev_id)
+    else:
+        return False
+
+    if x_dtype not in support_dtype:
+        return False
+    return True
+
+
+cpdef cub_reduction(
+        _ndarray_base arr, op,
+        axis=None, dtype=None, _ndarray_base out=None, keepdims=False):
+    """Perform a reduction using CUB.
+
+    If the specified reduction is not possible, None is returned.
+    """
+    # if import at the top level, a segfault would happen when import cupy!
+    from cupy._core._reduction import _get_axis
+    cdef bint enforce_numpy_API = False, is_ok
+    cdef str order
+    cdef tuple reduce_axis, out_axis
+    cdef Py_ssize_t contiguous_size
+
+    if op in (CUPY_CUB_ARGMIN, CUPY_CUB_ARGMAX):
+        # For argmin and argmax, NumPy does not allow a tuple for axis.
+        # Also, the keepdims and dtype kwargs are not provided.
+        #
+        # For now we don't enforce these for consistency with existing CuPy
+        # non-CUB reduction behavior.
+        # https://github.com/cupy/cupy/issues/2595
+        enforce_numpy_API = False
+        if enforce_numpy_API:
+            # numpy's argmin and argmax do not support a tuple of axes
+            if sequence.PySequence_Check(axis):
+                raise TypeError(
+                    "'tuple' object cannot be interpreted as an integer")
+            if keepdims:
+                raise TypeError(
+                    "'keepdims' is an invalid keyword argument for "
+                    "argmin or argmax.")
+            if dtype is not None:
+                raise TypeError(
+                    "'dtype' is an invalid keyword argument for "
+                    "argmin or argmax.")
+        else:
+            if dtype is not None:
+                # fallback to existing non-CUB behavior
+                return None
+
+    if arr._c_contiguous:
+        order = 'C'
+    elif arr._f_contiguous:
+        order = 'F'
+    else:
+        return None
+
+    reduce_axis, out_axis = _get_axis(axis, arr.ndim)
+    if can_use_device_reduce(arr, op, out_axis, dtype):
+        return device_reduce(arr, op, reduce_axis, out_axis, out, keepdims)
+
+    if op in (CUPY_CUB_ARGMIN, CUPY_CUB_ARGMAX):
+        # segmented reduction not currently implemented for argmax, argmin
+        return None
+
+    is_ok, contiguous_size = can_use_device_segmented_reduce(
+        arr, op, reduce_axis, out_axis, dtype, order)
+    if is_ok and contiguous_size > 0:
+        return device_segmented_reduce(arr, op, reduce_axis, out_axis,
+                                       out, keepdims, contiguous_size)
+    return None
+
+
+cpdef cub_scan(_ndarray_base arr, op):
+    """Perform an (in-place) prefix scan using CUB.
+
+    If the specified scan is not possible, None is returned.
+    """
+    if op not in (CUPY_CUB_CUMSUM, CUPY_CUB_CUMPROD):
+        return None
+
+    x_dtype = arr.dtype
+    if x_dtype == numpy.complex128:
+        # cub_device_scan seems buggy for complex128:
+        # https://github.com/cupy/cupy/pull/2919#issuecomment-574633590
+        return None
+
+    cdef int dev_id = device.get_device_id()
+    if x_dtype in _cub_support_dtype(False, dev_id):
+        return device_scan(arr, op)
+
+    return None
+
+
+cpdef cub_histogram(_ndarray_base x, _ndarray_base y, bins):
+    """Check if the required workspace size is too much, if not then proceed
+    to compute the histogram, otherwise return None. This is a workaround for
+    NVIDIA/cub#613.
+    """
+    try:
+        out = device_histogram(x, y, bins)
+    except _memory.OutOfMemoryError:
+        return None
+    return out
diff --git a/cupy/cuda/cudnn.py b/cupy/cuda/cudnn.py
new file mode 100644
index 0000000..95c8eed
--- /dev/null
+++ b/cupy/cuda/cudnn.py
@@ -0,0 +1,17 @@
+"""
+cuDNN Wrapper
+
+Use `cupy_backends.cuda.libs.cudnn` directly in CuPy codebase.
+"""
+
+from cupy import _environment
+
+
+available = True
+
+try:
+    _environment._preload_library('cudnn')
+    from cupy_backends.cuda.libs.cudnn import *  # NOQA
+except ImportError as e:
+    available = False
+    _environment._preload_warning('cudnn', e)
diff --git a/cupy/cuda/cufft.pxd b/cupy/cuda/cufft.pxd
new file mode 100644
index 0000000..2c397a2
--- /dev/null
+++ b/cupy/cuda/cufft.pxd
@@ -0,0 +1,99 @@
+# Note: nothing exposed in this pxd header is considered public API;
+# we copy this to sdist only because it's needed at runtime to support
+# cuFFT callbacks
+
+from libc.stdint cimport intptr_t
+
+cdef extern from *:
+    ctypedef float Float 'cufftReal'
+    ctypedef double Double 'cufftDoubleReal'
+    ctypedef int Result 'cufftResult_t'
+
+    IF CUPY_HIP_VERSION > 0:
+        ctypedef int Handle 'cufftHandle'
+    ELSE:
+        ctypedef struct hipHandle 'hipfftHandle_t':
+            pass
+        ctypedef hipHandle* Handle 'cufftHandle'
+
+    ctypedef enum Type 'cufftType_t':
+        pass
+
+
+cpdef enum:
+    CUFFT_C2C = 0x29
+    CUFFT_R2C = 0x2a
+    CUFFT_C2R = 0x2c
+    CUFFT_Z2Z = 0x69
+    CUFFT_D2Z = 0x6a
+    CUFFT_Z2D = 0x6c
+
+    CUFFT_FORWARD = -1
+    CUFFT_INVERSE = 1
+
+    CUFFT_CB_LD_COMPLEX = 0x0,
+    CUFFT_CB_LD_COMPLEX_DOUBLE = 0x1,
+    CUFFT_CB_LD_REAL = 0x2,
+    CUFFT_CB_LD_REAL_DOUBLE = 0x3,
+    CUFFT_CB_ST_COMPLEX = 0x4,
+    CUFFT_CB_ST_COMPLEX_DOUBLE = 0x5,
+    CUFFT_CB_ST_REAL = 0x6,
+    CUFFT_CB_ST_REAL_DOUBLE = 0x7,
+
+
+cpdef get_current_plan()
+cpdef int getVersion() except? -1
+
+
+cdef class Plan1d:
+    cdef:
+        readonly intptr_t handle
+        readonly object work_area  # can be MemoryPointer or a list of it
+        readonly int nx
+        readonly int batch
+        readonly Type fft_type
+
+        readonly list gpus
+        list batch_share
+        list gather_streams
+        list gather_events
+        dict scatter_streams
+        dict scatter_events
+        intptr_t xtArr
+        list xtArr_buffer
+
+        void _single_gpu_get_plan(
+            self, Handle plan, int nx, int fft_type, int batch) except*
+        void _multi_gpu_get_plan(
+            self, Handle plan, int nx, int fft_type, int batch,
+            devices, out) except*
+
+
+cdef class PlanNd:
+    cdef:
+        readonly intptr_t handle
+        readonly object work_area  # memory.MemoryPointer
+        readonly tuple shape
+        readonly Type fft_type
+        readonly str order
+        readonly int last_axis
+        readonly object last_size
+
+        # TODO(leofang): support multi-GPU transforms
+        readonly list gpus
+
+
+cdef class XtPlanNd:
+    cdef:
+        readonly intptr_t handle
+        readonly object work_area  # memory.MemoryPointer
+        readonly tuple shape
+        readonly int itype
+        readonly int otype
+        readonly int etype
+        readonly str order
+        readonly int last_axis
+        readonly object last_size
+
+        # TODO(leofang): support multi-GPU transforms
+        readonly list gpus
diff --git a/cupy/cuda/cufft.pyx b/cupy/cuda/cufft.pyx
new file mode 100644
index 0000000..fd22983
--- /dev/null
+++ b/cupy/cuda/cufft.pyx
@@ -0,0 +1,1205 @@
+cimport cython  # NOQA
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from libc.string cimport memset as c_memset
+from libcpp cimport vector
+
+import numpy
+import threading
+
+import cupy
+from cupy.cuda import device
+from cupy.cuda import memory
+from cupy.cuda import runtime
+from cupy.cuda import stream
+
+
+cdef object _thread_local = threading.local()
+
+
+cpdef get_current_plan():
+    """Get current cuFFT plan.
+
+    Returns:
+        None or cupy.cuda.cufft.Plan1d or cupy.cuda.cufft.PlanNd
+    """
+    if not hasattr(_thread_local, '_current_plan'):
+        _thread_local._current_plan = None
+    return _thread_local._current_plan
+
+
+cdef enum:
+    # Actually, this is 64, but it's undocumented. For the sake
+    # of safety, let us use 16, which agrees with the cuFFT doc.
+    MAX_CUDA_DESCRIPTOR_GPUS = 16
+
+
+cdef extern from 'cupy_cufft.h' nogil:
+    # we duplicate some types here to avoid cimporting from driver/runtime,
+    # as we don't include their .pxd files in the sdist
+    ctypedef void* Stream 'cudaStream_t'
+    ctypedef int DataType 'cudaDataType'
+
+    ctypedef struct Complex 'cufftComplex':
+        float x, y
+    ctypedef struct DoubleComplex 'cufftDoubleComplex':
+        double x, y
+
+    # cuFFT Helper Function
+    Result cufftCreate(Handle *plan)
+    Result cufftDestroy(Handle plan)
+    Result cufftSetAutoAllocation(Handle plan, int autoAllocate)
+    Result cufftSetWorkArea(Handle plan, void *workArea)
+
+    # cuFFT Stream Function
+    Result cufftSetStream(Handle plan, Stream streamId)
+
+    # cuFFT Plan Functions
+    Result cufftMakePlan1d(Handle plan, int nx, Type type, int batch,
+                           size_t *workSize)
+    Result cufftMakePlanMany(Handle plan, int rank, int *n, int *inembed,
+                             int istride, int idist, int *onembed, int ostride,
+                             int odist, Type type, int batch,
+                             size_t *workSize)
+
+    # cuFFT Exec Function
+    Result cufftExecC2C(Handle plan, Complex *idata, Complex *odata,
+                        int direction)
+    Result cufftExecR2C(Handle plan, Float *idata, Complex *odata)
+    Result cufftExecC2R(Handle plan, Complex *idata, Float *odata)
+    Result cufftExecZ2Z(Handle plan, DoubleComplex *idata,
+                        DoubleComplex *odata, int direction)
+    Result cufftExecD2Z(Handle plan, Double *idata, DoubleComplex *odata)
+    Result cufftExecZ2D(Handle plan, DoubleComplex *idata, Double *odata)
+
+    # Version
+    Result cufftGetVersion(int* version)
+
+    # cufftXt data types
+    ctypedef struct XtArrayDesc 'cudaXtDesc':
+        int version
+        int nGPUs
+        int GPUs[MAX_CUDA_DESCRIPTOR_GPUS]
+        void* data[MAX_CUDA_DESCRIPTOR_GPUS]
+        size_t size[MAX_CUDA_DESCRIPTOR_GPUS]
+        void* cudaXtState
+
+    ctypedef enum XtSubFormat 'cufftXtSubFormat':
+        CUFFT_XT_FORMAT_INPUT
+        CUFFT_XT_FORMAT_OUTPUT
+        CUFFT_XT_FORMAT_INPLACE
+        CUFFT_XT_FORMAT_INPLACE_SHUFFLED
+        CUFFT_XT_FORMAT_1D_INPUT_SHUFFLED
+
+    ctypedef struct XtArray 'cudaLibXtDesc':
+        int version
+        XtArrayDesc* descriptor
+        int library
+        XtSubFormat subFormat
+        void* libDescriptor
+
+    ctypedef enum XtCopyType 'cufftXtCopyType':
+        CUFFT_COPY_HOST_TO_DEVICE = 0x00
+        CUFFT_COPY_DEVICE_TO_HOST = 0x01
+        CUFFT_COPY_DEVICE_TO_DEVICE = 0x02
+
+    # cufftXt functions
+    Result cufftXtSetGPUs(Handle plan, int nGPUs, int* gpus)
+    Result cufftXtSetWorkArea(Handle plan, void** workArea)
+    Result cufftXtMemcpy(Handle plan, void *dst, void *src, XtCopyType type)
+    Result cufftXtExecDescriptorC2C(Handle plan, XtArray* idata,
+                                    XtArray* odata, int direction)
+    Result cufftXtExecDescriptorZ2Z(Handle plan, XtArray* idata,
+                                    XtArray* odata, int direction)
+    Result cufftXtMakePlanMany(Handle plan, int rank, long long int* n,
+                               long long int* inembed,
+                               long long int istride,
+                               long long int idist,
+                               DataType inputtype,
+                               long long int* onembed,
+                               long long int ostride,
+                               long long int odist,
+                               DataType outputtype,
+                               long long int batch, size_t* workSize,
+                               DataType executiontype)
+    Result cufftXtExec(Handle plan, void* inarr, void* outarr, int d)
+
+
+IF CUPY_CUFFT_STATIC:
+    # cuFFT callback
+    cdef extern from 'cupy_cufftXt.h' nogil:
+        ctypedef enum callbackType 'cufftXtCallbackType':
+            pass
+        Result set_callback(Handle, callbackType, bint, void**)
+
+
+cdef dict RESULT = {
+    0: 'CUFFT_SUCCESS',
+    1: 'CUFFT_INVALID_PLAN',
+    2: 'CUFFT_ALLOC_FAILED',
+    3: 'CUFFT_INVALID_TYPE',
+    4: 'CUFFT_INVALID_VALUE',
+    5: 'CUFFT_INTERNAL_ERROR',
+    6: 'CUFFT_EXEC_FAILED',
+    7: 'CUFFT_SETUP_FAILED',
+    8: 'CUFFT_INVALID_SIZE',
+    9: 'CUFFT_UNALIGNED_DATA',
+    10: 'CUFFT_INCOMPLETE_PARAMETER_LIST',
+    11: 'CUFFT_INVALID_DEVICE',
+    12: 'CUFFT_PARSE_ERROR',
+    13: 'CUFFT_NO_WORKSPACE',
+    14: 'CUFFT_NOT_IMPLEMENTED',
+    15: 'CUFFT_LICENSE_ERROR',
+    16: 'CUFFT_NOT_SUPPORTED',
+}
+
+
+class CuFFTError(RuntimeError):
+
+    def __init__(self, int result):
+        self.result = result
+        super(CuFFTError, self).__init__('%s' % (RESULT[result]))
+
+    def __reduce__(self):
+        return (type(self), (self.result,))
+
+
+@cython.profile(False)
+cpdef inline void check_result(int result) except *:
+    if result != 0:
+        raise CuFFTError(result)
+
+
+cpdef int getVersion() except? -1:
+    cdef int version, result
+    result = cufftGetVersion(&version)
+    check_result(result)
+    return version
+
+
+# This is necessary for single-batch transforms: "when batch is one, data is
+# left in the GPU memory in a permutation of the natural output", see
+# https://docs.nvidia.com/cuda/cufft/index.html#multiple-GPU-cufft-intermediate-helper  # NOQA
+cdef _reorder_buffers(Handle plan, intptr_t xtArr, list xtArr_buffer):
+    cdef int i, result, nGPUs
+    cdef intptr_t temp_xtArr
+    cdef XtArray* temp_arr
+    cdef XtArray* arr
+    cdef list gpus = []
+    cdef list sizes = []
+    cdef list temp_xtArr_buffer
+
+    arr = <XtArray*>xtArr
+    nGPUs = len(xtArr_buffer)
+    assert nGPUs == arr.descriptor.nGPUs
+
+    # allocate another buffer to prepare for order conversion
+    for i in range(nGPUs):
+        gpus.append(arr.descriptor.GPUs[i])
+        sizes.append(arr.descriptor.size[i])
+    temp_xtArr, temp_xtArr_buffer = _XtMalloc(gpus, sizes,
+                                              CUFFT_XT_FORMAT_INPLACE)
+    temp_arr = <XtArray*>temp_xtArr
+
+    # Make a device copy to bring the data from the permuted order back to
+    # the natural order. Note that this works because after FFT
+    # arr.subFormat is silently changed to CUFFT_XT_FORMAT_INPLACE_SHUFFLED
+    with nogil:
+        result = cufftXtMemcpy(plan, <void*>temp_arr, <void*>arr,
+                               CUFFT_COPY_DEVICE_TO_DEVICE)
+    check_result(result)
+
+    for i in range(nGPUs):
+        # swap MemoryPointer in xtArr_buffer
+        temp = temp_xtArr_buffer[i]
+        temp_xtArr_buffer[i] = xtArr_buffer[i]
+        xtArr_buffer[i] = temp
+
+        # swap pointer in xtArr
+        arr.descriptor.data[i] = temp_arr.descriptor.data[i]
+        assert arr.descriptor.size[i] == temp_arr.descriptor.size[i]
+        temp_arr.descriptor.data[i] = NULL
+        temp_arr.descriptor.size[i] = 0
+
+    # temp_xtArr now points to the old data, which is now in temp_xtArr_buffer
+    # and will be deallocated after this line (out of scope)
+    _XtFree(temp_xtArr)
+
+
+# This is meant to replace cufftXtMalloc().
+# We need to manage the buffers ourselves in order to 1. avoid excessive,
+# uncessary memory usage, and 2. use CuPy's memory pool.
+cdef _XtMalloc(list gpus, list sizes, XtSubFormat fmt):
+    cdef XtArrayDesc* xtArr_desc
+    cdef XtArray* xtArr
+    cdef list xtArr_buffer = []
+    cdef int i, nGPUs
+    cdef size_t size
+
+    nGPUs = len(gpus)
+    assert nGPUs == len(sizes)
+    xtArr_desc = <XtArrayDesc*>PyMem_Malloc(sizeof(XtArrayDesc))
+    xtArr = <XtArray*>PyMem_Malloc(sizeof(XtArray))
+    c_memset(xtArr_desc, 0, sizeof(XtArrayDesc))
+    c_memset(xtArr, 0, sizeof(XtArray))
+
+    xtArr_desc.nGPUs = nGPUs
+    for i, (gpu, size) in enumerate(zip(gpus, sizes)):
+        prev_device = runtime.getDevice()
+        runtime.setDevice(gpu)
+        try:
+            buf = memory.alloc(size)
+        finally:
+            runtime.setDevice(prev_device)
+        assert gpu == buf.device_id
+        xtArr_buffer.append(buf)
+        xtArr_desc.GPUs[i] = gpu
+        xtArr_desc.data[i] = <void*><intptr_t>(buf.ptr)
+        xtArr_desc.size[i] = size
+
+    xtArr.descriptor = xtArr_desc
+    xtArr.subFormat = fmt
+
+    return <intptr_t>xtArr, xtArr_buffer
+
+
+# This is meant to replace cufftXtFree().
+# We only free the C structs. The underlying GPU buffers are deallocated when
+# going out of scope.
+cdef _XtFree(intptr_t ptr):
+    cdef XtArray* xtArr = <XtArray*>ptr
+    cdef XtArrayDesc* xtArr_desc = xtArr.descriptor
+    PyMem_Free(xtArr_desc)
+    PyMem_Free(xtArr)
+
+
+cdef class Plan1d:
+    def __init__(self, int nx, int fft_type, int batch, *,
+                 devices=None, out=None):
+        cdef Handle plan
+        cdef bint use_multi_gpus = 0 if devices is None else 1
+        cdef int result
+
+        self.handle = <intptr_t>0
+        self.xtArr = <intptr_t>0  # pointer to metadata for multi-GPU buffer
+        self.xtArr_buffer = None  # actual multi-GPU intermediate buffer
+
+        with nogil:
+            result = cufftCreate(&plan)
+            if result == 0:
+                result = cufftSetAutoAllocation(plan, 0)
+        check_result(result)
+
+        self.handle = <intptr_t>plan
+        self.work_area = None
+        self.gpus = None
+
+        self.gather_streams = None
+        self.gather_events = None
+        self.scatter_streams = None
+        self.scatter_events = None
+
+        if batch != 0:
+            # set plan, work_area, gpus, streams, and events
+            if not use_multi_gpus:
+                self._single_gpu_get_plan(plan, nx, fft_type, batch)
+            else:
+                self._multi_gpu_get_plan(
+                    plan, nx, fft_type, batch, devices, out)
+        else:
+            if use_multi_gpus:
+                # multi-GPU FFT cannot transform 0-size arrays, and attempting
+                # to create such a plan will error out, but we still need this
+                # for bookkeeping
+                if isinstance(devices, (tuple, list)):
+                    self.gpus = list(devices)
+                elif isinstance(devices, int) and devices > 0:
+                    self.gpus = [i for i in range(int)]
+                else:
+                    raise ValueError
+
+        self.nx = nx
+        self.fft_type = <Type>fft_type
+        self.batch = batch
+        self.batch_share = None
+
+    cdef void _single_gpu_get_plan(self, Handle plan, int nx, int fft_type,
+                                   int batch) except*:
+        cdef int result
+        cdef size_t work_size
+        cdef intptr_t ptr
+
+        with nogil:
+            result = cufftMakePlan1d(plan, nx, <Type>fft_type, batch,
+                                     &work_size)
+
+        # cufftMakePlan1d uses large memory when nx has large divisor.
+        # See https://github.com/cupy/cupy/issues/1063
+        if result == 2:
+            cupy.get_default_memory_pool().free_all_blocks()
+            with nogil:
+                result = cufftMakePlan1d(plan, nx, <Type>fft_type, batch,
+                                         &work_size)
+        check_result(result)
+
+        work_area = memory.alloc(work_size)
+        ptr = <intptr_t>(work_area.ptr)
+        with nogil:
+            result = cufftSetWorkArea(plan, <void*>(ptr))
+        check_result(result)
+
+        self.work_area = work_area  # this is for cuFFT plan
+
+    cdef void _multi_gpu_get_plan(self, Handle plan, int nx, int fft_type,
+                                  int batch, devices, out) except*:
+        cdef int nGPUs, min_len, result
+        cdef vector.vector[int] gpus
+        cdef vector.vector[size_t] work_size
+        cdef list work_area = []
+        cdef list gather_streams = []
+        cdef list gather_events = []
+        cdef vector.vector[void*] work_area_ptr
+
+        # some sanity checks
+        if runtime.is_hip:
+            raise RuntimeError('hipFFT/rocFFT does not support multi-GPU FFT')
+        if fft_type != CUFFT_C2C and fft_type != CUFFT_Z2Z:
+            raise ValueError('Currently for multiple GPUs only C2C and Z2Z are'
+                             ' supported.')
+        if isinstance(devices, (tuple, list)):
+            nGPUs = len(devices)
+            for i in range(nGPUs):
+                gpus.push_back(devices[i])
+        elif isinstance(devices, int):
+            nGPUs = devices
+            for i in range(nGPUs):
+                gpus.push_back(i)
+        else:
+            raise ValueError('\"devices\" should be an int or an iterable '
+                             'of int.')
+        if batch == 1:
+            if (nx & (nx - 1)) != 0:
+                raise ValueError('For multi-GPU FFT with batch = 1, the array '
+                                 'size must be a power of 2.')
+            if nGPUs not in (2, 4, 8, 16):
+                raise ValueError('For multi-GPU FFT with batch = 1, the number'
+                                 ' of devices must be 2, 4, 8, or 16.')
+            if nGPUs in (2, 4):
+                min_len = 64
+            elif nGPUs == 8:
+                min_len = 128
+            else:  # nGPU = 16
+                min_len = 1024
+            if nx < min_len:
+                raise ValueError('For {} GPUs, the array length must be at '
+                                 'least {} (you have {}).'
+                                 .format(nGPUs, min_len, nx))
+        work_size.resize(nGPUs)
+
+        with nogil:
+            result = cufftXtSetGPUs(plan, nGPUs, gpus.data())
+            if result == 0:
+                result = cufftMakePlan1d(plan, nx, <Type>fft_type, batch,
+                                         work_size.data())
+
+        # cufftMakePlan1d uses large memory when nx has large divisor.
+        # See https://github.com/cupy/cupy/issues/1063
+        if result == 2:
+            cupy.get_default_memory_pool().free_all_blocks()
+            with nogil:
+                result = cufftMakePlan1d(plan, nx, <Type>fft_type, batch,
+                                         work_size.data())
+        check_result(result)
+
+        for i in range(nGPUs):
+            prev_device = runtime.getDevice()
+            runtime.setDevice(gpus[i])
+            try:
+                buf = memory.alloc(work_size[i])
+                s = stream.Stream()
+                e = stream.Event()
+            finally:
+                runtime.setDevice(prev_device)
+            work_area.append(buf)
+            work_area_ptr.push_back(<void*><intptr_t>(buf.ptr))
+            gather_streams.append(s)
+            gather_events.append(e)
+        with nogil:
+            result = cufftXtSetWorkArea(plan, work_area_ptr.data())
+        check_result(result)
+
+        self.work_area = work_area  # this is for cuFFT plan
+        self.gpus = list(gpus)
+
+        # For async, overlapped copies. We need to distinguish scatter and
+        # gather because for async memcpy, the stream is on the source device
+        self.gather_streams = gather_streams
+        self.gather_events = gather_events
+        self.scatter_streams = {}
+        self.scatter_events = {}
+        self._multi_gpu_get_scatter_streams_events(runtime.getDevice())
+
+    def _multi_gpu_get_scatter_streams_events(self, int curr_device):
+        '''
+        create a list of streams and events on the current device
+        '''
+        cdef int i
+        cdef list scatter_streams = []
+        cdef list scatter_events = []
+
+        assert curr_device in self.gpus
+        prev_device = runtime.getDevice()
+        runtime.setDevice(curr_device)
+        try:
+            for i in self.gpus:
+                scatter_streams.append(stream.Stream())
+                scatter_events.append(stream.Event())
+        finally:
+            runtime.setDevice(prev_device)
+        self.scatter_streams[curr_device] = scatter_streams
+        self.scatter_events[curr_device] = scatter_events
+
+    def __dealloc__(self):
+        cdef Handle plan = <Handle>self.handle
+        cdef int dev, result
+
+        if self.xtArr != 0:
+            _XtFree(self.xtArr)
+            self.xtArr = 0
+
+        try:
+            dev = runtime.getDevice()
+        except Exception as e:
+            # hack: the runtime module is purged at interpreter shutdown,
+            # since this is not a __del__ method, we can't use
+            # cupy._util.is_shutting_down()...
+            return
+
+        if plan != <Handle>0:
+            with nogil:
+                result = cufftDestroy(plan)
+            check_result(result)
+            self.handle = <intptr_t>0
+
+        # cuFFT bug: after cufftDestroy(), the current device is mistakenly
+        # set to the last device in self.gpus, so we must correct it. See
+        # https://github.com/cupy/cupy/pull/2644#discussion_r347567899 and
+        # NVIDIA internal ticket 2761341.
+        runtime.setDevice(dev)
+
+    def __enter__(self):
+        _thread_local._current_plan = self
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        _thread_local._current_plan = None
+
+    def fft(self, a, out, direction):
+        if self.gpus is not None:
+            self._multi_gpu_fft(a, out, direction)
+        else:
+            self._single_gpu_fft(a, out, direction)
+
+    def _single_gpu_fft(self, a, out, direction):
+        cdef intptr_t plan = self.handle
+        cdef intptr_t s = stream.get_current_stream().ptr
+        cdef int result
+
+        with nogil:
+            result = cufftSetStream(<Handle>plan, <Stream>s)
+        check_result(result)
+
+        if self.fft_type == CUFFT_C2C:
+            execC2C(plan, a.data.ptr, out.data.ptr, direction)
+        elif self.fft_type == CUFFT_R2C:
+            execR2C(plan, a.data.ptr, out.data.ptr)
+        elif self.fft_type == CUFFT_C2R:
+            execC2R(plan, a.data.ptr, out.data.ptr)
+        elif self.fft_type == CUFFT_Z2Z:
+            execZ2Z(plan, a.data.ptr, out.data.ptr, direction)
+        elif self.fft_type == CUFFT_D2Z:
+            execD2Z(plan, a.data.ptr, out.data.ptr)
+        elif self.fft_type == CUFFT_Z2D:
+            execZ2D(plan, a.data.ptr, out.data.ptr)
+        else:
+            raise ValueError
+
+    def _multi_gpu_setup_buffer(self, a):
+        cdef XtArrayDesc* xtArr_desc
+        cdef XtArray* xtArr
+        cdef intptr_t ptr
+        cdef list xtArr_buffer, share, sizes
+        cdef int i, nGPUs
+        cdef XtSubFormat fmt
+
+        # First, get the buffers:
+        # We need to manage the buffers ourselves in order to avoid excessive,
+        # uncessary memory usage. Note that these buffers are used for in-place
+        # transforms, and are re-used (lifetime tied to the plan).
+
+        if isinstance(a, cupy.ndarray) or isinstance(a, numpy.ndarray):
+            if self.xtArr == 0 and self.xtArr_buffer is None:
+                nGPUs = len(self.gpus)
+
+                # this is the rule for distributing the workload
+                if self.batch > 1:
+                    share = [self.batch // nGPUs] * nGPUs
+                    for i in range(self.batch % nGPUs):
+                        share[i] += 1
+                else:
+                    share = [1.0 / nGPUs] * nGPUs
+                sizes = [int(share[i] * self.nx * a.dtype.itemsize)
+                         for i in range(nGPUs)]
+
+                # get buffer
+                if isinstance(a, cupy.ndarray):
+                    fmt = CUFFT_XT_FORMAT_INPLACE
+                else:  # from numpy
+                    fmt = CUFFT_XT_FORMAT_1D_INPUT_SHUFFLED
+                ptr, xtArr_buffer = _XtMalloc(self.gpus, sizes, fmt)
+
+                xtArr = <XtArray*>ptr
+                xtArr_desc = xtArr.descriptor
+                assert xtArr_desc.nGPUs == nGPUs
+
+                self.batch_share = share
+                self.xtArr = ptr
+                self.xtArr_buffer = xtArr_buffer  # kept to ensure lifetime
+            else:
+                # After FFT the subFormat flag is silently changed to
+                # CUFFT_XT_FORMAT_INPLACE_SHUFFLED. For reuse we must correct
+                # it, otherwise in the next run we would encounter
+                # CUFFT_INVALID_TYPE!
+                ptr = self.xtArr
+                xtArr = <XtArray*>ptr
+                if self.batch == 1:
+                    if isinstance(a, cupy.ndarray):
+                        fmt = CUFFT_XT_FORMAT_INPLACE
+                    else:  # from numpy
+                        fmt = CUFFT_XT_FORMAT_1D_INPUT_SHUFFLED
+                    xtArr.subFormat = fmt
+        elif isinstance(a, list):
+            # TODO(leofang): For users running Plan1d.fft() (bypassing all
+            # checks in cupy.fft.fft), they are allowed to send in a list of
+            # ndarrays, each of which is on a different GPU. Then, no data
+            # copy is needed, just replace the pointers in the descriptor.
+            raise NotImplementedError('User-managed buffer area is not yet '
+                                      'supported.')
+        else:
+            raise ValueError('Impossible to reach.')
+
+    def _multi_gpu_memcpy(self, a, str action):
+        cdef Handle plan = <Handle>self.handle
+        cdef list xtArr_buffer, share
+        cdef int nGPUs, dev, s_device, start, count, result
+        cdef XtArray* arr
+        cdef intptr_t ptr, ptr2
+        cdef size_t size
+
+        assert isinstance(a, (cupy.ndarray, numpy.ndarray))
+
+        start = 0
+        assert a.flags.c_contiguous  # NumPy does not have _c_contiguous
+        b = a.ravel()
+        assert b.flags['OWNDATA'] is False
+        assert self.xtArr_buffer is not None
+        ptr = self.xtArr
+        arr = <XtArray*>ptr
+        xtArr_buffer = self.xtArr_buffer
+        nGPUs = len(self.gpus)
+        share = self.batch_share
+
+        if action == 'scatter':
+            if isinstance(a, cupy.ndarray):
+                s_device = b.data.device_id
+                if s_device not in self.scatter_streams:
+                    self._multi_gpu_get_scatter_streams_events(s_device)
+
+                # When we come here, another stream could still be
+                # copying data for us, so we wait patiently...
+                outer_stream = stream.get_current_stream()
+                outer_stream.synchronize()
+
+                for dev in range(nGPUs):
+                    count = int(share[dev] * self.nx)
+                    size = count * b.dtype.itemsize
+                    curr_stream = self.scatter_streams[s_device][dev]
+                    curr_event = self.scatter_events[s_device][dev]
+                    xtArr_buffer[dev].copy_from_device_async(
+                        b[start:start+count].data, size, curr_stream)
+                    if dev != 0:
+                        prev_event = self.scatter_events[s_device][dev-1]
+                        curr_stream.wait_event(prev_event)
+                    curr_event.record(curr_stream)
+                    start += count
+                assert start == b.size
+                self.scatter_events[s_device][-1].synchronize()
+            else:  # numpy
+                ptr2 = b.ctypes.data
+                with nogil:
+                    result = cufftXtMemcpy(
+                        plan, <void*>arr, <void*>ptr2,
+                        CUFFT_COPY_HOST_TO_DEVICE)
+                check_result(result)
+        elif action == 'gather':
+            if isinstance(a, cupy.ndarray):
+                if self.batch == 1:
+                    _reorder_buffers(plan, self.xtArr, xtArr_buffer)
+
+                # When we come here, another stream could still be
+                # copying data for us, so we wait patiently...
+                outer_stream = stream.get_current_stream()
+                outer_stream.synchronize()
+
+                for i in range(nGPUs):
+                    count = int(share[i] * self.nx)
+                    size = count * b.dtype.itemsize
+                    curr_stream = self.gather_streams[i]
+                    curr_event = self.gather_events[i]
+                    b[start:start+count].data.copy_from_device_async(
+                        xtArr_buffer[i], size, curr_stream)
+                    if i != 0:
+                        prev_event = self.gather_events[i-1]
+                        curr_stream.wait_event(prev_event)
+                    curr_event.record(curr_stream)
+                    start += count
+                assert start == b.size
+                self.gather_events[-1].synchronize()
+            else:  # numpy
+                ptr2 = b.ctypes.data
+                with nogil:
+                    result = cufftXtMemcpy(
+                        plan, <void*>ptr2, <void*>arr,
+                        CUFFT_COPY_DEVICE_TO_HOST)
+                check_result(result)
+        else:
+            raise ValueError
+
+    def _multi_gpu_fft(self, a, out, direction):
+        # When we arrive here, the normal CuPy call path ensures a and out
+        # reside on the same GPU -> must distribute a to all of the GPUs
+        self._multi_gpu_setup_buffer(a)
+
+        # Next, copy data to buffer
+        self._multi_gpu_memcpy(a, 'scatter')
+
+        # Actual workhorses
+        # Note: mult-GPU plans cannot set stream
+        cdef intptr_t plan = self.handle
+        if self.fft_type == CUFFT_C2C:
+            multi_gpu_execC2C(plan, self.xtArr, self.xtArr, direction)
+        elif self.fft_type == CUFFT_Z2Z:
+            multi_gpu_execZ2Z(plan, self.xtArr, self.xtArr, direction)
+        else:
+            raise ValueError
+
+        # Gather the distributed outputs
+        self._multi_gpu_memcpy(out, 'gather')
+
+    def _output_dtype_and_shape(self, a):
+        shape = list(a.shape)
+        if self.fft_type == CUFFT_C2C:
+            dtype = numpy.complex64
+        elif self.fft_type == CUFFT_R2C:
+            shape[-1] = shape[-1] // 2 + 1
+            dtype = numpy.complex64
+        elif self.fft_type == CUFFT_C2R:
+            shape[-1] = self.nx
+            dtype = numpy.float32
+        elif self.fft_type == CUFFT_Z2Z:
+            dtype = numpy.complex128
+        elif self.fft_type == CUFFT_D2Z:
+            shape[-1] = shape[-1] // 2 + 1
+            dtype = numpy.complex128
+        else:
+            shape[-1] = self.nx
+            dtype = numpy.float64
+        return tuple(shape), dtype
+
+    def get_output_array(self, a):
+        shape, dtype = self._output_dtype_and_shape(a)
+        return cupy.empty(shape, dtype)
+
+    def check_output_array(self, a, out):
+        """Verify shape and dtype of the output array.
+
+        Parameters
+        ----------
+        a : cupy.array
+            The input to the transform
+        out : cupy.array
+            The array where the output of the transform will be stored.
+        """
+        shape, dtype = self._output_dtype_and_shape(a)
+        if out.shape != shape:
+            raise ValueError(
+                ('out must have shape {}.').format(shape))
+        if out.dtype != dtype:
+            raise ValueError(
+                'out dtype mismatch: found {}, expected {}'.format(
+                    out.dtype, dtype))
+
+
+cdef class PlanNd:
+    def __init__(self, object shape, object inembed, int istride,
+                 int idist, object onembed, int ostride, int odist,
+                 int fft_type, int batch, str order, int last_axis, last_size):
+        cdef Handle plan
+        cdef size_t work_size
+        cdef int ndim, result
+        cdef vector.vector[int] shape_arr = shape
+        cdef vector.vector[int] inembed_arr
+        cdef vector.vector[int] onembed_arr
+        cdef int* shape_ptr = shape_arr.data()
+        cdef int* inembed_ptr
+        cdef int* onembed_ptr
+        cdef intptr_t ptr
+
+        self.handle = <intptr_t>0
+        ndim = len(shape)
+
+        if inembed is None:
+            inembed_ptr = NULL  # ignore istride and use default strides
+        else:
+            inembed_arr = inembed
+            inembed_ptr = inembed_arr.data()
+
+        if onembed is None:
+            onembed_ptr = NULL  # ignore ostride and use default strides
+        else:
+            onembed_arr = onembed
+            onembed_ptr = onembed_arr.data()
+
+        with nogil:
+            result = cufftCreate(&plan)
+            if result == 0:
+                result = cufftSetAutoAllocation(plan, 0)
+        check_result(result)
+
+        self.handle = <intptr_t>plan
+        self.gpus = None  # TODO(leofang): support multi-GPU PlanNd
+
+        if batch == 0:
+            work_size = 0
+        else:
+            with nogil:
+                result = cufftMakePlanMany(plan, ndim, shape_ptr,
+                                           inembed_ptr, istride, idist,
+                                           onembed_ptr, ostride, odist,
+                                           <Type>fft_type, batch,
+                                           &work_size)
+
+            # cufftMakePlanMany could use a large amount of memory
+            if result == 2:
+                cupy.get_default_memory_pool().free_all_blocks()
+                with nogil:
+                    result = cufftMakePlanMany(plan, ndim, shape_ptr,
+                                               inembed_ptr, istride, idist,
+                                               onembed_ptr, ostride, odist,
+                                               <Type>fft_type, batch,
+                                               &work_size)
+            check_result(result)
+
+        # TODO: for CUDA>=9.2 could also allow setting a work area policy
+        # result = cufftXtSetWorkAreaPolicy(plan, policy, &work_size)
+
+        work_area = memory.alloc(work_size)
+        ptr = <intptr_t>(work_area.ptr)
+        with nogil:
+            result = cufftSetWorkArea(plan, <void*>(ptr))
+        check_result(result)
+
+        self.shape = tuple(shape)
+        self.fft_type = <Type>fft_type
+        self.work_area = work_area
+        self.order = order  # either 'C' or 'F'
+        self.last_axis = last_axis  # ignored for C2C
+        self.last_size = last_size  # = None (and ignored) for C2C
+
+    def __dealloc__(self):
+        cdef Handle plan = <Handle>self.handle
+        cdef int result
+
+        if plan != <Handle>0:
+            with nogil:
+                result = cufftDestroy(plan)
+            check_result(result)
+            self.handle = <intptr_t>0
+
+    def __enter__(self):
+        _thread_local._current_plan = self
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        _thread_local._current_plan = None
+
+    def fft(self, a, out, direction):
+        cdef intptr_t plan = self.handle
+        cdef intptr_t s = stream.get_current_stream().ptr
+        cdef int result
+
+        with nogil:
+            result = cufftSetStream(<Handle>plan, <Stream>s)
+        check_result(result)
+
+        if self.fft_type == CUFFT_C2C:
+            execC2C(plan, a.data.ptr, out.data.ptr, direction)
+        elif self.fft_type == CUFFT_R2C:
+            execR2C(plan, a.data.ptr, out.data.ptr)
+        elif self.fft_type == CUFFT_C2R:
+            execC2R(plan, a.data.ptr, out.data.ptr)
+        elif self.fft_type == CUFFT_Z2Z:
+            execZ2Z(plan, a.data.ptr, out.data.ptr, direction)
+        elif self.fft_type == CUFFT_D2Z:
+            execD2Z(plan, a.data.ptr, out.data.ptr)
+        elif self.fft_type == CUFFT_Z2D:
+            execZ2D(plan, a.data.ptr, out.data.ptr)
+        else:
+            raise ValueError
+
+    def _output_dtype_and_shape(self, a):
+        shape = list(a.shape)
+        if self.fft_type == CUFFT_C2C:
+            dtype = numpy.complex64
+        elif self.fft_type == CUFFT_R2C:
+            shape[self.last_axis] = self.last_size
+            dtype = numpy.complex64
+        elif self.fft_type == CUFFT_C2R:
+            shape[self.last_axis] = self.last_size
+            dtype = numpy.float32
+        elif self.fft_type == CUFFT_Z2Z:
+            dtype = numpy.complex128
+        elif self.fft_type == CUFFT_D2Z:
+            shape[self.last_axis] = self.last_size
+            dtype = numpy.complex128
+        else:  # CUFFT_Z2D
+            shape[self.last_axis] = self.last_size
+            dtype = numpy.float64
+        return tuple(shape), dtype
+
+    def get_output_array(self, a, order='C'):
+        shape, dtype = self._output_dtype_and_shape(a)
+        return cupy.empty(shape, dtype, order=order)
+
+    def check_output_array(self, a, out):
+        if out is a:
+            # TODO(leofang): think about in-place transforms for C2R & R2C
+            return
+        if self.fft_type in (CUFFT_C2C, CUFFT_Z2Z):
+            if out.shape != a.shape:
+                raise ValueError('output shape mismatch')
+            if out.dtype != a.dtype:
+                raise ValueError('output dtype mismatch')
+        else:
+            if out.ndim != a.ndim:
+                raise ValueError('output dimension mismatch')
+            for i, size in enumerate(out.shape):
+                if (i != self.last_axis and size != a.shape[i]) or \
+                   (i == self.last_axis and size != self.last_size):
+                    raise ValueError('output shape is incorrecct')
+            if self.fft_type in (CUFFT_R2C, CUFFT_D2Z):
+                if out.dtype != cupy.dtype(a.dtype.char.upper()):
+                    raise ValueError('output dtype is unexpected')
+            else:  # CUFFT_C2R or CUFFT_Z2D
+                if out.dtype != cupy.dtype(a.dtype.char.lower()):
+                    raise ValueError('output dtype is unexpected')
+        if not ((out.flags.f_contiguous == a.flags.f_contiguous) and
+                (out.flags.c_contiguous == a.flags.c_contiguous)):
+            raise ValueError('output contiguity mismatch')
+
+
+# TODO(leofang): Unify with PlanND?!
+# TODO(leofang): support cufftXtSetGPUs?
+cdef class XtPlanNd:
+    def __init__(self, shape,
+                 inembed, long long int istride, long long int idist, idtype,
+                 onembed, long long int ostride, long long int odist, odtype,
+                 long long int batch, edtype, *,
+                 str order, int last_axis, last_size):
+        # Note: we don't pass in fft_type here because it's redundant and
+        # does not cover exotic types like complex32 or bf16
+
+        cdef Handle plan
+        cdef size_t work_size
+        cdef int ndim, result
+        cdef vector.vector[long long int] shape_arr = shape
+        cdef vector.vector[long long int] inembed_arr
+        cdef vector.vector[long long int] onembed_arr
+        cdef long long int* shape_ptr = shape_arr.data()
+        cdef long long int* inembed_ptr
+        cdef long long int* onembed_ptr
+
+        self.handle = <intptr_t>0
+        ndim = len(shape)
+
+        if inembed is None:
+            inembed_ptr = NULL  # ignore istride and use default strides
+        else:
+            inembed_arr = inembed
+            inembed_ptr = inembed_arr.data()
+
+        if onembed is None:
+            onembed_ptr = NULL  # ignore ostride and use default strides
+        else:
+            onembed_arr = onembed
+            onembed_ptr = onembed_arr.data()
+
+        with nogil:
+            result = cufftCreate(&plan)
+            if result == 0:
+                result = cufftSetAutoAllocation(plan, 0)
+        check_result(result)
+
+        self.handle = <intptr_t>plan
+        self.gpus = None  # TODO(leofang): support multi-GPU plans
+
+        # determine input/output/execution types here; note that we don't
+        # cimport to_cuda_dtype due to circular dependency
+        from cupy._core._dtype import to_cuda_dtype
+        cdef int itype = to_cuda_dtype(idtype, True)
+        cdef int otype = to_cuda_dtype(odtype, True)
+        cdef int etype = to_cuda_dtype(edtype, True)
+
+        cdef long long int length
+        cdef long long int full = 1
+        for length in shape:
+            full *= length
+        length = last_size if last_size is not None else shape[-1]
+        try:
+            self._sanity_checks(itype, otype, etype, length, full)
+        except AssertionError:
+            raise ValueError('input/output/execution types mismatch')
+
+        if batch == 0:
+            work_size = 0
+        else:
+            with nogil:
+                result = cufftXtMakePlanMany(
+                    plan, ndim, shape_ptr,
+                    inembed_ptr, istride, idist, <DataType>itype,
+                    onembed_ptr, ostride, odist, <DataType>otype,
+                    batch, &work_size, <DataType>etype)
+
+            # cufftMakePlanMany could use a large amount of memory
+            if result == 2:
+                cupy.get_default_memory_pool().free_all_blocks()
+                with nogil:
+                    result = cufftXtMakePlanMany(
+                        plan, ndim, shape_ptr,
+                        inembed_ptr, istride, idist, <DataType>itype,
+                        onembed_ptr, ostride, odist, <DataType>otype,
+                        batch, &work_size, <DataType>etype)
+            check_result(result)
+
+        work_area = memory.alloc(work_size)
+        cdef intptr_t ptr = <intptr_t>(work_area.ptr)
+        with nogil:
+            result = cufftSetWorkArea(plan, <void*>(ptr))
+        check_result(result)
+
+        self.shape = tuple(shape)
+        self.itype = itype
+        self.otype = otype
+        self.etype = etype
+        self.work_area = work_area
+        self.order = order  # either 'C' or 'F'
+        self.last_axis = last_axis  # ignored for C2C
+        self.last_size = last_size  # = None (and ignored) for C2C
+
+    def __dealloc__(self):
+        cdef Handle plan = <Handle>self.handle
+        cdef int result
+
+        if plan != <Handle>0:
+            with nogil:
+                result = cufftDestroy(plan)
+            check_result(result)
+            self.handle = <intptr_t>0
+
+    def __enter__(self):
+        _thread_local._current_plan = self
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        _thread_local._current_plan = None
+
+    def fft(self, a, out, direction):
+        cdef intptr_t plan = self.handle
+        cdef intptr_t s = stream.get_current_stream().ptr
+        cdef int result
+
+        with nogil:
+            result = cufftSetStream(<Handle>plan, <Stream>s)
+        check_result(result)
+        XtExec(plan, a.data.ptr, out.data.ptr, direction)
+
+    def _sanity_checks(self, int itype, int otype, int etype,
+                       long long int last_size, long long int full_size):
+        # not every possible type combination is legit
+        # TODO(leofang): support bf16?
+        # C2C
+        if itype == runtime.CUDA_C_16F and otype == runtime.CUDA_C_16F:
+            assert etype == runtime.CUDA_C_16F
+        elif itype == runtime.CUDA_C_32F and otype == runtime.CUDA_C_32F:
+            assert etype == runtime.CUDA_C_32F
+        elif itype == runtime.CUDA_C_64F and otype == runtime.CUDA_C_64F:
+            assert etype == runtime.CUDA_C_64F
+        # C2R
+        elif itype == runtime.CUDA_C_16F and otype == runtime.CUDA_R_16F:
+            assert etype == runtime.CUDA_C_16F
+        elif itype == runtime.CUDA_C_32F and otype == runtime.CUDA_R_32F:
+            assert etype == runtime.CUDA_C_32F
+        elif itype == runtime.CUDA_C_64F and otype == runtime.CUDA_R_64F:
+            assert etype == runtime.CUDA_C_64F
+        # R2C
+        elif itype == runtime.CUDA_R_16F and otype == runtime.CUDA_C_16F:
+            assert etype == runtime.CUDA_C_16F
+        elif itype == runtime.CUDA_R_32F and otype == runtime.CUDA_C_32F:
+            assert etype == runtime.CUDA_C_32F
+        elif itype == runtime.CUDA_R_64F and otype == runtime.CUDA_C_64F:
+            assert etype == runtime.CUDA_C_64F
+        else:
+            assert False
+
+        # check fp16 runtime constraints
+        # https://docs.nvidia.com/cuda/cufft/index.html#half-precision-transforms
+        if etype == runtime.CUDA_C_16F:
+            if int(device.get_compute_capability()) < 53:
+                raise RuntimeError("this device doesn't support complex32 FFT")
+            if (last_size & (last_size - 1)) != 0:
+                raise ValueError('size must be power of 2')
+            if full_size > 4000000000:
+                raise ValueError('input array too large')
+            # TODO(leofang): check if multi-GPU is requested
+        # TODO(leofang): also check for bf16?
+        # https://docs.nvidia.com/cuda/cufft/index.html#bfloat16-precision-transforms
+
+    def _output_dtype_and_shape(self, a):
+        shape = list(a.shape)
+        if self.itype != self.otype:  # R2C or C2R
+            shape[self.last_axis] = self.last_size
+        if self.otype == runtime.CUDA_C_16F:
+            # dtype = numpy.complex32
+            raise NotImplementedError('complex32 is not supported yet, please '
+                                      'allocate the output array manually')
+        elif self.otype == runtime.CUDA_C_32F:
+            dtype = numpy.complex64
+        elif self.otype == runtime.CUDA_C_64F:
+            dtype = numpy.complex128
+        elif self.otype == runtime.CUDA_R_16F:
+            dtype = numpy.float16
+        elif self.otype == runtime.CUDA_R_32F:
+            dtype = numpy.float32
+        elif self.otype == runtime.CUDA_R_64F:
+            dtype = numpy.float64
+        return tuple(shape), dtype
+
+    def get_output_array(self, a, order='C'):
+        shape, dtype = self._output_dtype_and_shape(a)
+        return cupy.empty(shape, dtype, order=order)
+
+
+cpdef execC2C(intptr_t plan, intptr_t idata, intptr_t odata, int direction):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftExecC2C(h, <Complex*>idata, <Complex*>odata,
+                              direction)
+    check_result(result)
+
+
+cpdef execR2C(intptr_t plan, intptr_t idata, intptr_t odata):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftExecR2C(h, <Float*>idata, <Complex*>odata)
+    check_result(result)
+
+
+cpdef execC2R(intptr_t plan, intptr_t idata, intptr_t odata):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftExecC2R(h, <Complex*>idata, <Float*>odata)
+    check_result(result)
+
+
+cpdef execZ2Z(intptr_t plan, intptr_t idata, intptr_t odata, int direction):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftExecZ2Z(h, <DoubleComplex*>idata,
+                              <DoubleComplex*>odata, direction)
+    check_result(result)
+
+
+cpdef execD2Z(intptr_t plan, intptr_t idata, intptr_t odata):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftExecD2Z(h, <Double*>idata, <DoubleComplex*>odata)
+    check_result(result)
+
+
+cpdef execZ2D(intptr_t plan, intptr_t idata, intptr_t odata):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftExecZ2D(h, <DoubleComplex*>idata, <Double*>odata)
+    check_result(result)
+
+
+cpdef multi_gpu_execC2C(intptr_t plan, intptr_t idata, intptr_t odata,
+                        int direction):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftXtExecDescriptorC2C(h, <XtArray*>idata,
+                                          <XtArray*>odata, direction)
+    check_result(result)
+
+
+cpdef multi_gpu_execZ2Z(intptr_t plan, intptr_t idata, intptr_t odata,
+                        int direction):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftXtExecDescriptorZ2Z(h, <XtArray*>idata,
+                                          <XtArray*>odata, direction)
+    check_result(result)
+
+
+cpdef XtExec(intptr_t plan, intptr_t idata, intptr_t odata, int direction):
+    cdef Handle h = <Handle>plan
+    cdef int result
+
+    with nogil:
+        result = cufftXtExec(h, <void*>idata, <void*>odata, direction)
+    check_result(result)
+
+
+cpdef intptr_t setCallback(
+        intptr_t plan, int cb_type, bint is_load, intptr_t aux_arr=0):
+    cdef Handle h = <Handle>plan  # no-cython-lint
+    cdef int result  # no-cython-lint
+    cdef void** callerInfo  # no-cython-lint
+
+    IF CUPY_CUFFT_STATIC:
+        with nogil:
+            if aux_arr > 0:
+                callerInfo = (<void**>(&aux_arr))
+            else:
+                callerInfo = NULL
+            result = set_callback(
+                h, <callbackType>cb_type, is_load, callerInfo)
+        check_result(result)
+    ELSE:
+        raise RuntimeError('cuFFT is dynamically linked and thus does not '
+                           'support callback')
diff --git a/cupy/cuda/cupy_cub.cu b/cupy/cuda/cupy_cub.cu
new file mode 100644
index 0000000..71a4f15
--- /dev/null
+++ b/cupy/cuda/cupy_cub.cu
@@ -0,0 +1,1030 @@
+#include "cupy_cub.h"  // need to make atomicAdd visible to CUB templates early
+#include <cupy/type_dispatcher.cuh>
+
+#ifndef CUPY_USE_HIP
+#include <cub/device/device_reduce.cuh>
+#include <cub/device/device_segmented_reduce.cuh>
+#include <cub/device/device_spmv.cuh>
+#include <cub/device/device_scan.cuh>
+#include <cub/device/device_histogram.cuh>
+#include <cub/iterator/counting_input_iterator.cuh>
+#include <cub/iterator/transform_input_iterator.cuh>
+#else
+#include <hipcub/device/device_reduce.hpp>
+#include <hipcub/device/device_segmented_reduce.hpp>
+#include <hipcub/device/device_scan.hpp>
+#include <hipcub/device/device_histogram.hpp>
+#include <rocprim/iterator/counting_iterator.hpp>
+#include <hipcub/iterator/transform_input_iterator.hpp>
+#endif
+
+
+/* ------------------------------------ Minimum boilerplate to support complex numbers ------------------------------------ */
+#ifndef CUPY_USE_HIP
+// - This works only because all data fields in the *Traits struct are not
+//   used in <cub/device/device_reduce.cuh>.
+// - The Max() and Lowest() below are chosen to comply with NumPy's lexical
+//   ordering; note that std::numeric_limits<T> does not support complex
+//   numbers as in general the comparison is ill defined.
+// - DO NOT USE THIS STUB for supporting CUB sorting!!!!!!
+using namespace cub;
+
+template <>
+struct FpLimits<complex<float>>
+{
+    static __host__ __device__ __forceinline__ complex<float> Max() {
+        return (complex<float>(FLT_MAX, FLT_MAX));
+    }
+
+    static __host__ __device__ __forceinline__ complex<float> Lowest() {
+        return (complex<float>(FLT_MAX * float(-1), FLT_MAX * float(-1)));
+    }
+};
+
+template <>
+struct FpLimits<complex<double>>
+{
+    static __host__ __device__ __forceinline__ complex<double> Max() {
+        return (complex<double>(DBL_MAX, DBL_MAX));
+    }
+
+    static __host__ __device__ __forceinline__ complex<double> Lowest() {
+        return (complex<double>(DBL_MAX * double(-1), DBL_MAX * double(-1)));
+    }
+};
+
+template <> struct NumericTraits<complex<float>>  : BaseTraits<FLOATING_POINT, true, false, unsigned int, complex<float>> {};
+template <> struct NumericTraits<complex<double>> : BaseTraits<FLOATING_POINT, true, false, unsigned long long, complex<double>> {};
+
+#else
+
+// hipCUB internally uses std::numeric_limits, so we should provide specializations for the complex numbers.
+// Note that there's std::complex, so to avoid name collision we must use the full decoration (thrust::complex)!
+// TODO(leofang): wrap CuPy's thrust namespace with another one (say, cupy::thrust) for safer scope resolution?
+
+namespace std {
+template <>
+class numeric_limits<thrust::complex<float>> {
+  public:
+    static __host__ __device__ thrust::complex<float> max() noexcept {
+        return thrust::complex<float>(std::numeric_limits<float>::max(), std::numeric_limits<float>::max());
+    }
+
+    static __host__ __device__ thrust::complex<float> lowest() noexcept {
+        return thrust::complex<float>(-std::numeric_limits<float>::max(), -std::numeric_limits<float>::max());
+    }
+};
+
+template <>
+class numeric_limits<thrust::complex<double>> {
+  public:
+    static __host__ __device__ thrust::complex<double> max() noexcept {
+        return thrust::complex<double>(std::numeric_limits<double>::max(), std::numeric_limits<double>::max());
+    }
+
+    static __host__ __device__ thrust::complex<double> lowest() noexcept {
+        return thrust::complex<double>(-std::numeric_limits<double>::max(), -std::numeric_limits<double>::max());
+    }
+};
+
+// Copied from https://github.com/ROCmSoftwarePlatform/hipCUB/blob/master-rocm-3.5/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp
+// (For some reason the specialization for __half defined in the above file does not work, so we have to go
+// through the same route as we did above for complex numbers.)
+template <>
+class numeric_limits<__half> {
+  public:
+    static __host__ __device__ __half max() noexcept {
+        unsigned short max_half = 0x7bff;
+        __half max_value = *reinterpret_cast<__half*>(&max_half);
+        return max_value;
+    }
+
+    static __host__ __device__ __half lowest() noexcept {
+        unsigned short lowest_half = 0xfbff;
+        __half lowest_value = *reinterpret_cast<__half*>(&lowest_half);
+        return lowest_value;
+    }
+};
+}  // namespace std
+
+using namespace hipcub;
+
+#endif  // ifndef CUPY_USE_HIP
+/* ------------------------------------ end of boilerplate ------------------------------------ */
+
+
+/* ------------------------------------ "Patches" to CUB ------------------------------------
+   This stub is needed because CUB does not have a built-in "prod" operator
+*/
+
+//
+// product functor
+//
+struct _multiply
+{
+    template <typename T>
+    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
+    {
+        return a * b;
+    }
+};
+
+//
+// arange functor: arange(0, n+1) -> arange(0, n+1, step_size)
+//
+struct _arange
+{
+    private:
+        int step_size;
+
+    public:
+    __host__ __device__ __forceinline__ _arange(int i): step_size(i) {}
+    __host__ __device__ __forceinline__ int operator()(const int &in) const {
+        return step_size * in;
+    }
+};
+
+#ifndef CUPY_USE_HIP
+typedef TransformInputIterator<int, _arange, CountingInputIterator<int>> seg_offset_itr;
+#else
+typedef TransformInputIterator<int, _arange, rocprim::counting_iterator<int>> seg_offset_itr;
+#endif
+
+/*
+   These stubs are needed because CUB does not handle NaNs properly, while NumPy has certain
+   behaviors with which we must comply.
+*/
+
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+    && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+__host__ __device__ __forceinline__ bool half_isnan(const __half& x) {
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+    return __hisnan(x);
+#else
+    // TODO: avoid cast to float
+    return isnan(__half2float(x));
+#endif
+}
+
+__host__ __device__ __forceinline__ bool half_less(const __half& l, const __half& r) {
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+    return l < r;
+#else
+    // TODO: avoid cast to float
+    return __half2float(l) < __half2float(r);
+#endif
+}
+
+__host__ __device__ __forceinline__ bool half_equal(const __half& l, const __half& r) {
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+    return l == r;
+#else
+    // TODO: avoid cast to float
+    return __half2float(l) == __half2float(r);
+#endif
+}
+#endif
+
+//
+// Max()
+//
+
+// specialization for float for handling NaNs
+template <>
+__host__ __device__ __forceinline__ float Max::operator()(const float &a, const float &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? b : a;}
+}
+
+// specialization for double for handling NaNs
+template <>
+__host__ __device__ __forceinline__ double Max::operator()(const double &a, const double &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? b : a;}
+}
+
+// specialization for complex<float> for handling NaNs
+template <>
+__host__ __device__ __forceinline__ complex<float> Max::operator()(const complex<float> &a, const complex<float> &b) const
+{
+    // - TODO(leofang): just call max() here when the bug in cupy/complex.cuh is fixed
+    // - NumPy behavior: If both a and b contain NaN, the first argument is chosen
+    // - isnan() and max() are defined in cupy/complex.cuh
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? b : a;}
+}
+
+// specialization for complex<double> for handling NaNs
+template <>
+__host__ __device__ __forceinline__ complex<double> Max::operator()(const complex<double> &a, const complex<double> &b) const
+{
+    // - TODO(leofang): just call max() here when the bug in cupy/complex.cuh is fixed
+    // - NumPy behavior: If both a and b contain NaN, the first argument is chosen
+    // - isnan() and max() are defined in cupy/complex.cuh
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? b : a;}
+}
+
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+    && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+// specialization for half for handling NaNs
+template <>
+__host__ __device__ __forceinline__ __half Max::operator()(const __half &a, const __half &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (half_isnan(a)) {return a;}
+    else if (half_isnan(b)) {return b;}
+    else { return half_less(a, b) ? b : a; }
+}
+#endif
+
+//
+// Min()
+//
+
+// specialization for float for handling NaNs
+template <>
+__host__ __device__ __forceinline__ float Min::operator()(const float &a, const float &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? a : b;}
+}
+
+// specialization for double for handling NaNs
+template <>
+__host__ __device__ __forceinline__ double Min::operator()(const double &a, const double &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? a : b;}
+}
+
+// specialization for complex<float> for handling NaNs
+template <>
+__host__ __device__ __forceinline__ complex<float> Min::operator()(const complex<float> &a, const complex<float> &b) const
+{
+    // - TODO(leofang): just call min() here when the bug in cupy/complex.cuh is fixed
+    // - NumPy behavior: If both a and b contain NaN, the first argument is chosen
+    // - isnan() and min() are defined in cupy/complex.cuh
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? a : b;}
+}
+
+// specialization for complex<double> for handling NaNs
+template <>
+__host__ __device__ __forceinline__ complex<double> Min::operator()(const complex<double> &a, const complex<double> &b) const
+{
+    // - TODO(leofang): just call min() here when the bug in cupy/complex.cuh is fixed
+    // - NumPy behavior: If both a and b contain NaN, the first argument is chosen
+    // - isnan() and min() are defined in cupy/complex.cuh
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? a : b;}
+}
+
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+    && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+// specialization for half for handling NaNs
+template <>
+__host__ __device__ __forceinline__ __half Min::operator()(const __half &a, const __half &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (half_isnan(a)) {return a;}
+    else if (half_isnan(b)) {return b;}
+    else { return half_less(a, b) ? a : b; }
+}
+#endif
+
+//
+// ArgMax()
+//
+
+// specialization for float for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, float> ArgMax::operator()(
+    const KeyValuePair<int, float> &a,
+    const KeyValuePair<int, float> &b) const
+{
+    if (isnan(a.value))
+        return a;
+    else if (isnan(b.value))
+        return b;
+    else if ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+
+// specialization for double for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, double> ArgMax::operator()(
+    const KeyValuePair<int, double> &a,
+    const KeyValuePair<int, double> &b) const
+{
+    if (isnan(a.value))
+        return a;
+    else if (isnan(b.value))
+        return b;
+    else if ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+
+// specialization for complex<float> for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, complex<float>> ArgMax::operator()(
+    const KeyValuePair<int, complex<float>> &a,
+    const KeyValuePair<int, complex<float>> &b) const
+{
+    if (isnan(a.value))
+        return a;
+    else if (isnan(b.value))
+        return b;
+    else if ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+
+// specialization for complex<double> for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, complex<double>> ArgMax::operator()(
+    const KeyValuePair<int, complex<double>> &a,
+    const KeyValuePair<int, complex<double>> &b) const
+{
+    if (isnan(a.value))
+        return a;
+    else if (isnan(b.value))
+        return b;
+    else if ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+    && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+// specialization for half for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, __half> ArgMax::operator()(
+    const KeyValuePair<int, __half> &a,
+    const KeyValuePair<int, __half> &b) const
+{
+    if (half_isnan(a.value))
+        return a;
+    else if (half_isnan(b.value))
+        return b;
+    else if ((half_less(a.value, b.value)) ||
+             (half_equal(a.value, b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+#endif
+
+//
+// ArgMin()
+//
+
+// specialization for float for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, float> ArgMin::operator()(
+    const KeyValuePair<int, float> &a,
+    const KeyValuePair<int, float> &b) const
+{
+    if (isnan(a.value))
+        return a;
+    else if (isnan(b.value))
+        return b;
+    else if ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+
+// specialization for double for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, double> ArgMin::operator()(
+    const KeyValuePair<int, double> &a,
+    const KeyValuePair<int, double> &b) const
+{
+    if (isnan(a.value))
+        return a;
+    else if (isnan(b.value))
+        return b;
+    else if ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+
+// specialization for complex<float> for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, complex<float>> ArgMin::operator()(
+    const KeyValuePair<int, complex<float>> &a,
+    const KeyValuePair<int, complex<float>> &b) const
+{
+    if (isnan(a.value))
+        return a;
+    else if (isnan(b.value))
+        return b;
+    else if ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+
+// specialization for complex<double> for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, complex<double>> ArgMin::operator()(
+    const KeyValuePair<int, complex<double>> &a,
+    const KeyValuePair<int, complex<double>> &b) const
+{
+    if (isnan(a.value))
+        return a;
+    else if (isnan(b.value))
+        return b;
+    else if ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+}
+
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+    && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+// specialization for half for handling NaNs
+template <>
+__host__ __device__ __forceinline__ KeyValuePair<int, __half> ArgMin::operator()(
+    const KeyValuePair<int, __half> &a,
+    const KeyValuePair<int, __half> &b) const
+{
+    if (half_isnan(a.value))
+        return a;
+    else if (half_isnan(b.value))
+        return b;
+    else if ((half_less(b.value, a.value)) ||
+             (half_equal(a.value, b.value) && (b.key < a.key)))
+        return b;
+    else
+        return a;
+
+}
+#endif
+
+#if __CUDACC_VER_MAJOR__ >= 12
+//
+// Specialization for cub operators Max() and Min() in CUDA 12.x. Why is this
+// necessary? Because the signatures have changed in CUDA 12.0.
+//
+
+//
+// Max()
+//
+template<>
+__host__ __device__ __forceinline__ float Max::operator()(float &a, float &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? b : a;}
+}
+
+template<>
+__host__ __device__ __forceinline__ double Max::operator()(double &a, double &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? b : a;}
+}
+
+template<>
+__host__ __device__ __forceinline__ complex<float> Max::operator()(complex<float> &a, complex<float> &b) const
+{
+    // - TODO(leofang): just call max() here when the bug in cupy/complex.cuh is fixed
+    // - NumPy behavior: If both a and b contain NaN, the first argument is chosen
+    // - isnan() and max() are defined in cupy/complex.cuh
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? b : a;}
+}
+
+template<>
+__host__ __device__ __forceinline__ complex<double> Max::operator()(complex<double> &a, complex<double> &b) const
+{
+    // - TODO(leofang): just call max() here when the bug in cupy/complex.cuh is fixed
+    // - NumPy behavior: If both a and b contain NaN, the first argument is chosen
+    // - isnan() and max() are defined in cupy/complex.cuh
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? b : a;}
+}
+
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+    && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+template<>
+__host__ __device__ __forceinline__ __half Max::operator()(__half &a, __half &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (half_isnan((const __half)a)) {return a;}
+    else if (half_isnan((const __half)b)) {return b;}
+    else { return half_less((const __half)a, (const __half)b) ? b : a; }
+}
+#endif
+
+//
+// Min()
+//
+template<>
+__host__ __device__ __forceinline__ float Min::operator()(float &a, float &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? a : b;}
+}
+
+template<>
+__host__ __device__ __forceinline__ double Min::operator()(double &a, double &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? a : b;}
+}
+
+template<>
+__host__ __device__ __forceinline__ complex<float> Min::operator()(complex<float> &a, complex<float> &b) const
+{
+    // - TODO(leofang): just call min() here when the bug in cupy/complex.cuh is fixed
+    // - NumPy behavior: If both a and b contain NaN, the first argument is chosen
+    // - isnan() and min() are defined in cupy/complex.cuh
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? a : b;}
+}
+
+template<>
+__host__ __device__ __forceinline__ complex<double> Min::operator()(complex<double> &a, complex<double> &b) const
+{
+    // - TODO(leofang): just call min() here when the bug in cupy/complex.cuh is fixed
+    // - NumPy behavior: If both a and b contain NaN, the first argument is chosen
+    // - isnan() and min() are defined in cupy/complex.cuh
+    if (isnan(a)) {return a;}
+    else if (isnan(b)) {return b;}
+    else {return a < b ? a : b;}
+}
+
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+    && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+template<>
+__host__ __device__ __forceinline__ __half Min::operator()(__half &a, __half &b) const
+{
+    // NumPy behavior: NaN is always chosen!
+    if (half_isnan((const __half)a)) {return a;}
+    else if (half_isnan((const __half)b)) {return b;}
+    else { return half_less((const __half)a, (const __half)b) ? a: b; }
+}
+#endif
+
+#endif // #if __CUDACC_VER_MAJOR__ == 12
+
+/* ------------------------------------ End of "patches" ------------------------------------ */
+
+//
+// **** CUB Sum ****
+//
+struct _cub_reduce_sum {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_items, cudaStream_t s)
+    {
+        DeviceReduce::Sum(workspace, workspace_size, static_cast<T*>(x),
+            static_cast<T*>(y), num_items, s);
+    }
+};
+
+struct _cub_segmented_reduce_sum {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_segments, seg_offset_itr offset_start, cudaStream_t s)
+    {
+        DeviceSegmentedReduce::Sum(workspace, workspace_size,
+            static_cast<T*>(x), static_cast<T*>(y), num_segments,
+            offset_start, offset_start+1, s);
+    }
+};
+
+//
+// **** CUB Prod ****
+//
+struct _cub_reduce_prod {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_items, cudaStream_t s)
+    {
+        _multiply product_op;
+        // the init value is cast from 1.0f because on host __half can only be
+        // initialized by float or double; static_cast<__half>(1) = 0 on host.
+        DeviceReduce::Reduce(workspace, workspace_size, static_cast<T*>(x),
+            static_cast<T*>(y), num_items, product_op, static_cast<T>(1.0f), s);
+    }
+};
+
+struct _cub_segmented_reduce_prod {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_segments, seg_offset_itr offset_start, cudaStream_t s)
+    {
+        _multiply product_op;
+        // the init value is cast from 1.0f because on host __half can only be
+        // initialized by float or double; static_cast<__half>(1) = 0 on host.
+        DeviceSegmentedReduce::Reduce(workspace, workspace_size,
+            static_cast<T*>(x), static_cast<T*>(y), num_segments,
+            offset_start, offset_start+1,
+            product_op, static_cast<T>(1.0f), s);
+    }
+};
+
+//
+// **** CUB Min ****
+//
+struct _cub_reduce_min {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_items, cudaStream_t s)
+    {
+        DeviceReduce::Min(workspace, workspace_size, static_cast<T*>(x),
+            static_cast<T*>(y), num_items, s);
+    }
+};
+
+struct _cub_segmented_reduce_min {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_segments, seg_offset_itr offset_start, cudaStream_t s)
+    {
+        DeviceSegmentedReduce::Min(workspace, workspace_size,
+            static_cast<T*>(x), static_cast<T*>(y), num_segments,
+            offset_start, offset_start+1, s);
+    }
+};
+
+//
+// **** CUB Max ****
+//
+struct _cub_reduce_max {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_items, cudaStream_t s)
+    {
+        DeviceReduce::Max(workspace, workspace_size, static_cast<T*>(x),
+            static_cast<T*>(y), num_items, s);
+    }
+};
+
+struct _cub_segmented_reduce_max {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_segments, seg_offset_itr offset_start, cudaStream_t s)
+    {
+        DeviceSegmentedReduce::Max(workspace, workspace_size,
+            static_cast<T*>(x), static_cast<T*>(y), num_segments,
+            offset_start, offset_start+1, s);
+    }
+};
+
+//
+// **** CUB ArgMin ****
+//
+struct _cub_reduce_argmin {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_items, cudaStream_t s)
+    {
+        DeviceReduce::ArgMin(workspace, workspace_size, static_cast<T*>(x),
+            static_cast<KeyValuePair<int, T>*>(y), num_items, s);
+    }
+};
+
+// TODO(leofang): add _cub_segmented_reduce_argmin
+
+//
+// **** CUB ArgMax ****
+//
+struct _cub_reduce_argmax {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* x, void* y,
+        int num_items, cudaStream_t s)
+    {
+        DeviceReduce::ArgMax(workspace, workspace_size, static_cast<T*>(x),
+            static_cast<KeyValuePair<int, T>*>(y), num_items, s);
+    }
+};
+
+// TODO(leofang): add _cub_segmented_reduce_argmax
+
+//
+// **** CUB SpMV ****
+//
+struct _cub_device_spmv {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* values,
+        void* row_offsets, void* column_indices, void* x, void* y,
+        int num_rows, int num_cols, int num_nonzeros, cudaStream_t stream)
+    {
+        #ifndef CUPY_USE_HIP
+        DeviceSpmv::CsrMV(workspace, workspace_size, static_cast<T*>(values),
+            static_cast<int*>(row_offsets), static_cast<int*>(column_indices),
+            static_cast<T*>(x), static_cast<T*>(y), num_rows, num_cols,
+            num_nonzeros, stream);
+        #endif
+    }
+};
+
+//
+// **** CUB InclusiveSum  ****
+//
+struct _cub_inclusive_sum {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* input, void* output,
+        int num_items, cudaStream_t s)
+    {
+        DeviceScan::InclusiveSum(workspace, workspace_size, static_cast<T*>(input),
+            static_cast<T*>(output), num_items, s);
+    }
+};
+
+//
+// **** CUB inclusive product  ****
+//
+struct _cub_inclusive_product {
+    template <typename T>
+    void operator()(void* workspace, size_t& workspace_size, void* input, void* output,
+        int num_items, cudaStream_t s)
+    {
+        _multiply product_op;
+        DeviceScan::InclusiveScan(workspace, workspace_size, static_cast<T*>(input),
+            static_cast<T*>(output), product_op, num_items, s);
+    }
+};
+
+//
+// **** CUB histogram range ****
+//
+struct _cub_histogram_range {
+    template <typename sampleT,
+              typename binT = typename If<std::is_integral<sampleT>::value, double, sampleT>::Type>
+    void operator()(void* workspace, size_t& workspace_size, void* input, void* output,
+        int n_bins, void* bins, size_t n_samples, cudaStream_t s) const
+    {
+        // Ugly hack to avoid specializing complex types, which cub::DeviceHistogram does not support.
+        // The If and Equals templates are from cub/util_type.cuh.
+        // TODO(leofang): revisit this part when complex support is added to cupy.histogram()
+        #ifndef CUPY_USE_HIP
+        typedef typename If<(Equals<sampleT, complex<float>>::VALUE || Equals<sampleT, complex<double>>::VALUE),
+                            double,
+                            sampleT>::Type h_sampleT;
+        typedef typename If<(Equals<binT, complex<float>>::VALUE || Equals<binT, complex<double>>::VALUE),
+                            double,
+                            binT>::Type h_binT;
+        #else
+        typedef typename std::conditional<(std::is_same<sampleT, complex<float>>::value || std::is_same<sampleT, complex<double>>::value),
+                                          double,
+                                          sampleT>::type h_sampleT;
+        typedef typename std::conditional<(std::is_same<binT, complex<float>>::value || std::is_same<binT, complex<double>>::value),
+                                          double,
+                                          binT>::type h_binT;
+        #endif
+
+        // TODO(leofang): CUB has a bug that when specializing n_samples with type size_t,
+        // it would error out. Before the fix (thrust/cub#38) is merged we disable the code
+        // path splitting for now. A type/range check must be done in the caller.
+        // TODO(leofang): check if hipCUB has the same bug or not
+
+        // if (n_samples < (1ULL << 31)) {
+            int num_samples = n_samples;
+            DeviceHistogram::HistogramRange(workspace, workspace_size, static_cast<h_sampleT*>(input),
+                #ifndef CUPY_USE_HIP
+                static_cast<long long*>(output), n_bins, static_cast<h_binT*>(bins), num_samples, s);
+                #else
+                // rocPRIM looks up atomic_add() from the namespace rocprim::detail; there's no way we can
+                // inject a "long long" version as we did for CUDA, so we must do it in "unsigned long long"
+                // and convert later...
+                static_cast<unsigned long long*>(output), n_bins, static_cast<h_binT*>(bins), num_samples, s);
+                #endif
+        // } else {
+        //     DeviceHistogram::HistogramRange(workspace, workspace_size, static_cast<h_sampleT*>(input),
+        //         static_cast<long long*>(output), n_bins, static_cast<h_binT*>(bins), n_samples, s);
+        // }
+    }
+};
+
+//
+// **** CUB histogram even ****
+//
+struct _cub_histogram_even {
+    template <typename sampleT>
+    void operator()(void* workspace, size_t& workspace_size, void* input, void* output,
+        int& n_bins, int& lower, int& upper, size_t n_samples, cudaStream_t s) const
+    {
+        #ifndef CUPY_USE_HIP
+        // Ugly hack to avoid specializing numerical types
+        typedef typename If<std::is_integral<sampleT>::value, sampleT, int>::Type h_sampleT;
+        int num_samples = n_samples;
+        static_assert(sizeof(long long) == sizeof(intptr_t), "not supported");
+        DeviceHistogram::HistogramEven(workspace, workspace_size, static_cast<h_sampleT*>(input),
+            static_cast<long long*>(output), n_bins, lower, upper, num_samples, s);
+        #else
+        throw std::runtime_error("HIP is not supported yet");
+        #endif
+    }
+};
+
+//
+// APIs exposed to CuPy
+//
+
+/* -------- device reduce -------- */
+
+void cub_device_reduce(void* workspace, size_t& workspace_size, void* x, void* y,
+    int num_items, cudaStream_t stream, int op, int dtype_id)
+{
+    switch(op) {
+    case CUPY_CUB_SUM:      return dtype_dispatcher(dtype_id, _cub_reduce_sum(),
+                                workspace, workspace_size, x, y, num_items, stream);
+    case CUPY_CUB_MIN:      return dtype_dispatcher(dtype_id, _cub_reduce_min(),
+                                workspace, workspace_size, x, y, num_items, stream);
+    case CUPY_CUB_MAX:      return dtype_dispatcher(dtype_id, _cub_reduce_max(),
+                                workspace, workspace_size, x, y, num_items, stream);
+    case CUPY_CUB_ARGMIN:   return dtype_dispatcher(dtype_id, _cub_reduce_argmin(),
+                                workspace, workspace_size, x, y, num_items, stream);
+    case CUPY_CUB_ARGMAX:   return dtype_dispatcher(dtype_id, _cub_reduce_argmax(),
+                                workspace, workspace_size, x, y, num_items, stream);
+    case CUPY_CUB_PROD:     return dtype_dispatcher(dtype_id, _cub_reduce_prod(),
+                                workspace, workspace_size, x, y, num_items, stream);
+    default:            throw std::runtime_error("Unsupported operation");
+    }
+}
+
+size_t cub_device_reduce_get_workspace_size(void* x, void* y, int num_items,
+    cudaStream_t stream, int op, int dtype_id)
+{
+    size_t workspace_size = 0;
+    cub_device_reduce(NULL, workspace_size, x, y, num_items, stream,
+                      op, dtype_id);
+    return workspace_size;
+}
+
+/* -------- device segmented reduce -------- */
+
+void cub_device_segmented_reduce(void* workspace, size_t& workspace_size,
+    void* x, void* y, int num_segments, int segment_size,
+    cudaStream_t stream, int op, int dtype_id)
+{
+    // CUB internally use int for offset...
+    // This iterates over [0, segment_size, 2*segment_size, 3*segment_size, ...]
+    #ifndef CUPY_USE_HIP
+    CountingInputIterator<int> count_itr(0);
+    #else
+    rocprim::counting_iterator<int> count_itr(0);
+    #endif
+    _arange scaling(segment_size);
+    seg_offset_itr itr(count_itr, scaling);
+
+    switch(op) {
+    case CUPY_CUB_SUM:
+        return dtype_dispatcher(dtype_id, _cub_segmented_reduce_sum(),
+                   workspace, workspace_size, x, y, num_segments, itr, stream);
+    case CUPY_CUB_MIN:
+        return dtype_dispatcher(dtype_id, _cub_segmented_reduce_min(),
+                   workspace, workspace_size, x, y, num_segments, itr, stream);
+    case CUPY_CUB_MAX:
+        return dtype_dispatcher(dtype_id, _cub_segmented_reduce_max(),
+                   workspace, workspace_size, x, y, num_segments, itr, stream);
+    case CUPY_CUB_PROD:
+        return dtype_dispatcher(dtype_id, _cub_segmented_reduce_prod(),
+                   workspace, workspace_size, x, y, num_segments, itr, stream);
+    default:
+        throw std::runtime_error("Unsupported operation");
+    }
+}
+
+size_t cub_device_segmented_reduce_get_workspace_size(void* x, void* y,
+    int num_segments, int segment_size,
+    cudaStream_t stream, int op, int dtype_id)
+{
+    size_t workspace_size = 0;
+    cub_device_segmented_reduce(NULL, workspace_size, x, y,
+                                num_segments, segment_size, stream,
+                                op, dtype_id);
+    return workspace_size;
+}
+
+/*--------- device spmv (sparse-matrix dense-vector multiply) ---------*/
+
+void cub_device_spmv(void* workspace, size_t& workspace_size, void* values,
+    void* row_offsets, void* column_indices, void* x, void* y, int num_rows,
+    int num_cols, int num_nonzeros, cudaStream_t stream,
+    int dtype_id)
+{
+    #ifndef CUPY_USE_HIP
+    return dtype_dispatcher(dtype_id, _cub_device_spmv(),
+                            workspace, workspace_size, values, row_offsets,
+                            column_indices, x, y, num_rows, num_cols,
+                            num_nonzeros, stream);
+    #endif
+}
+
+size_t cub_device_spmv_get_workspace_size(void* values, void* row_offsets,
+    void* column_indices, void* x, void* y, int num_rows, int num_cols,
+    int num_nonzeros, cudaStream_t stream, int dtype_id)
+{
+    size_t workspace_size = 0;
+    #ifndef CUPY_USE_HIP
+    cub_device_spmv(NULL, workspace_size, values, row_offsets, column_indices,
+                    x, y, num_rows, num_cols, num_nonzeros, stream, dtype_id);
+    #endif
+    return workspace_size;
+}
+
+/* -------- device scan -------- */
+
+void cub_device_scan(void* workspace, size_t& workspace_size, void* x, void* y,
+    int num_items, cudaStream_t stream, int op, int dtype_id)
+{
+    switch(op) {
+    case CUPY_CUB_CUMSUM:
+        return dtype_dispatcher(dtype_id, _cub_inclusive_sum(),
+                                workspace, workspace_size, x, y, num_items, stream);
+    case CUPY_CUB_CUMPROD:
+        return dtype_dispatcher(dtype_id, _cub_inclusive_product(),
+                                workspace, workspace_size, x, y, num_items, stream);
+    default:
+        throw std::runtime_error("Unsupported operation");
+    }
+}
+
+size_t cub_device_scan_get_workspace_size(void* x, void* y, int num_items,
+    cudaStream_t stream, int op, int dtype_id)
+{
+    size_t workspace_size = 0;
+    cub_device_scan(NULL, workspace_size, x, y, num_items, stream,
+                    op, dtype_id);
+    return workspace_size;
+}
+
+/* -------- device histogram -------- */
+
+void cub_device_histogram_range(void* workspace, size_t& workspace_size, void* x, void* y,
+    int n_bins, void* bins, size_t n_samples, cudaStream_t stream, int dtype_id)
+{
+    // TODO(leofang): support complex
+    if (dtype_id == CUPY_TYPE_COMPLEX64 || dtype_id == CUPY_TYPE_COMPLEX128) {
+	    throw std::runtime_error("complex dtype is not yet supported");
+    }
+
+    // TODO(leofang): n_samples is of type size_t, but if it's < 2^31 we cast it to int later
+    return dtype_dispatcher(dtype_id, _cub_histogram_range(),
+                            workspace, workspace_size, x, y, n_bins, bins, n_samples, stream);
+}
+
+size_t cub_device_histogram_range_get_workspace_size(void* x, void* y, int n_bins,
+    void* bins, size_t n_samples, cudaStream_t stream, int dtype_id)
+{
+    size_t workspace_size = 0;
+    cub_device_histogram_range(NULL, workspace_size, x, y, n_bins, bins, n_samples,
+                               stream, dtype_id);
+    return workspace_size;
+}
+
+void cub_device_histogram_even(void* workspace, size_t& workspace_size, void* x, void* y,
+    int n_bins, int lower, int upper, size_t n_samples, cudaStream_t stream, int dtype_id)
+{
+    #ifndef CUPY_USE_HIP
+    return dtype_dispatcher(dtype_id, _cub_histogram_even(),
+                            workspace, workspace_size, x, y, n_bins, lower, upper, n_samples, stream);
+    #endif
+}
+
+size_t cub_device_histogram_even_get_workspace_size(void* x, void* y, int n_bins,
+    int lower, int upper, size_t n_samples, cudaStream_t stream, int dtype_id)
+{
+    size_t workspace_size = 0;
+    cub_device_histogram_even(NULL, workspace_size, x, y, n_bins, lower, upper, n_samples,
+                              stream, dtype_id);
+    return workspace_size;
+}
diff --git a/cupy/cuda/cupy_cub.h b/cupy/cuda/cupy_cub.h
new file mode 100644
index 0000000..e7125ed
--- /dev/null
+++ b/cupy/cuda/cupy_cub.h
@@ -0,0 +1,97 @@
+#ifndef INCLUDE_GUARD_CUPY_CUDA_CUB_H
+#define INCLUDE_GUARD_CUPY_CUDA_CUB_H
+
+#define CUPY_CUB_SUM     0
+#define CUPY_CUB_MIN     1
+#define CUPY_CUB_MAX     2
+#define CUPY_CUB_ARGMIN  3
+#define CUPY_CUB_ARGMAX  4
+#define CUPY_CUB_CUMSUM  5
+#define CUPY_CUB_CUMPROD 6
+#define CUPY_CUB_PROD    7
+
+// this is defined during the build process
+#ifndef CUPY_CUB_VERSION_CODE
+#define CUPY_CUB_VERSION_CODE 0
+#endif
+
+#ifndef CUPY_NO_CUDA
+
+// for cudaStream_t
+#ifndef CUPY_USE_HIP
+#include <cuda_runtime.h>
+#else
+#include <hip/hip_runtime.h>
+#define cudaStream_t hipStream_t
+#endif
+
+void cub_device_reduce(void*, size_t&, void*, void*, int, cudaStream_t, int, int);
+void cub_device_segmented_reduce(void*, size_t&, void*, void*, int, int, cudaStream_t, int, int);
+void cub_device_spmv(void*, size_t&, void*, void*, void*, void*, void*, int, int, int, cudaStream_t, int);
+void cub_device_scan(void*, size_t&, void*, void*, int, cudaStream_t, int, int);
+void cub_device_histogram_range(void*, size_t&, void*, void*, int, void*, size_t, cudaStream_t, int);
+void cub_device_histogram_even(void*, size_t&, void*, void*, int, int, int, size_t, cudaStream_t, int);
+size_t cub_device_reduce_get_workspace_size(void*, void*, int, cudaStream_t, int, int);
+size_t cub_device_segmented_reduce_get_workspace_size(void*, void*, int, int, cudaStream_t, int, int);
+size_t cub_device_spmv_get_workspace_size(void*, void*, void*, void*, void*, int, int, int, cudaStream_t, int);
+size_t cub_device_scan_get_workspace_size(void*, void*, int, cudaStream_t, int, int);
+size_t cub_device_histogram_range_get_workspace_size(void*, void*, int, void*, size_t, cudaStream_t, int);
+size_t cub_device_histogram_even_get_workspace_size(void*, void*, int, int, int, size_t, cudaStream_t, int);
+
+// This is for CUB's HistogramRange; hipCUB does not need this (see comment in cupy_cub.cu)
+#ifdef __CUDA_ARCH__
+__device__ long long atomicAdd(long long *address, long long val) {
+    return atomicAdd(reinterpret_cast<unsigned long long*>(address),
+                     static_cast<unsigned long long>(val));
+}
+#endif // __CUDA_ARCH__
+
+#else // CUPY_NO_CUDA
+
+typedef struct CUstream_st *cudaStream_t;
+
+void cub_device_reduce(...) {
+}
+
+void cub_device_segmented_reduce(...) {
+}
+
+void cub_device_spmv(...) {
+}
+
+void cub_device_scan(...) {
+}
+
+void cub_device_histogram_range(...) {
+}
+
+void cub_device_histogram_even(...) {
+}
+
+size_t cub_device_reduce_get_workspace_size(...) {
+    return 0;
+}
+
+size_t cub_device_segmented_reduce_get_workspace_size(...) {
+    return 0;
+}
+
+size_t cub_device_spmv_get_workspace_size(...) {
+    return 0;
+}
+
+size_t cub_device_scan_get_workspace_size(...) {
+    return 0;
+}
+
+size_t cub_device_histogram_range_get_workspace_size(...) {
+    return 0;
+}
+
+size_t cub_device_histogram_even_get_workspace_size(...) {
+    return 0;
+}
+
+#endif // #ifndef CUPY_NO_CUDA
+
+#endif // #ifndef INCLUDE_GUARD_CUPY_CUDA_CUB_H
diff --git a/cupy/cuda/cupy_cufft.h b/cupy/cuda/cupy_cufft.h
new file mode 100644
index 0000000..1ccecde
--- /dev/null
+++ b/cupy/cuda/cupy_cufft.h
@@ -0,0 +1,324 @@
+#ifndef INCLUDE_GUARD_CUPY_CUFFT_H
+#define INCLUDE_GUARD_CUPY_CUFFT_H
+
+/*
+ * Note: this file should *not* be split into 3 and moved under cupy_backends/,
+ * because we need to copy this header to sdist and use it at runtime for cuFFT
+ * callbacks.
+ */
+
+#if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+#include <cufft.h>
+#include <cufftXt.h>
+
+#elif defined(CUPY_USE_HIP)
+#include <hipfft.h>
+
+extern "C" {
+
+typedef hipfftComplex cufftComplex;
+typedef hipfftDoubleComplex cufftDoubleComplex;
+typedef hipfftReal cufftReal;
+typedef hipfftDoubleReal cufftDoubleReal;
+
+typedef hipfftResult_t cufftResult_t;
+typedef hipfftHandle cufftHandle;
+typedef hipfftType_t cufftType_t;
+typedef hipStream_t cudaStream_t;
+
+// cuFFT Helper Function
+cufftResult_t cufftCreate(cufftHandle* plan) {
+    return hipfftCreate(plan);
+}
+
+cufftResult_t cufftDestroy(cufftHandle plan) {
+    return hipfftDestroy(plan);
+}
+
+cufftResult_t cufftSetAutoAllocation(cufftHandle plan, int autoAllocate) {
+    return hipfftSetAutoAllocation(plan, autoAllocate);
+}
+
+cufftResult_t cufftSetWorkArea(cufftHandle plan, void *workArea) {
+    return hipfftSetWorkArea(plan, workArea);
+}
+
+// cuFFT Stream Function
+cufftResult_t cufftSetStream(cufftHandle plan, cudaStream_t stream) {
+    return hipfftSetStream(plan, stream);
+}
+
+// cuFFT Plan Functions
+cufftResult_t cufftMakePlan1d(cufftHandle plan,
+                              int nx,
+                              cufftType_t type,
+                              int batch,
+                              size_t *workSize) {
+    return hipfftMakePlan1d(plan, nx, type, batch, workSize);
+}
+
+cufftResult_t cufftMakePlanMany(cufftHandle plan,
+                                int rank,
+                                int *n,
+                                int *inembed, int istride, int idist,
+                                int *onembed, int ostride, int odist,
+                                cufftType_t type,
+                                int batch,
+                                size_t *workSize) {
+    return hipfftMakePlanMany(plan, rank, n,
+                              inembed, istride, idist,
+                              onembed, ostride, odist,
+                              type, batch, workSize);
+}
+
+// cuFFT Exec Function
+cufftResult_t cufftExecC2C(cufftHandle plan,
+                           cufftComplex *idata,
+                           cufftComplex *odata,
+                           int direction) {
+    return hipfftExecC2C(plan, idata, odata, direction);
+}
+
+cufftResult_t cufftExecR2C(cufftHandle plan,
+                           cufftReal *idata,
+                           cufftComplex *odata) {
+    return hipfftExecR2C(plan, idata, odata);
+}
+
+cufftResult_t cufftExecC2R(cufftHandle plan,
+                           cufftComplex *idata,
+                           cufftReal *odata) {
+    return hipfftExecC2R(plan, idata, odata);
+}
+
+cufftResult_t cufftExecZ2Z(cufftHandle plan,
+                           cufftDoubleComplex *idata,
+                           cufftDoubleComplex *odata,
+                           int direction) {
+    return hipfftExecZ2Z(plan, idata, odata, direction);
+}
+
+cufftResult_t cufftExecD2Z(cufftHandle plan,
+                           cufftDoubleReal *idata,
+                           cufftDoubleComplex *odata) {
+    return hipfftExecD2Z(plan, idata, odata);
+}
+
+cufftResult_t cufftExecZ2D(cufftHandle plan,
+                           cufftDoubleComplex *idata,
+                           cufftDoubleReal *odata) {
+    return hipfftExecZ2D(plan, idata, odata);
+}
+
+// cuFFT Version
+cufftResult_t cufftGetVersion(int *version) {
+    return hipfftGetVersion(version);
+}
+
+// TODO(leofang): move this header to cupy_backends/ and include hip/cupy_hip_common.h
+typedef enum {} cudaDataType;
+
+// cufftXt functions
+cufftResult_t cufftXtSetGPUs(...) {
+    return HIPFFT_NOT_IMPLEMENTED;
+}
+
+cufftResult_t cufftXtSetWorkArea(...) {
+    return HIPFFT_NOT_IMPLEMENTED;
+}
+
+cufftResult_t cufftXtMemcpy(...) {
+    return HIPFFT_NOT_IMPLEMENTED;
+}
+
+cufftResult_t cufftXtMakePlanMany(...) {
+    return HIPFFT_NOT_IMPLEMENTED;
+}
+
+cufftResult_t cufftXtExec(...) {
+    return HIPFFT_NOT_IMPLEMENTED;
+}
+
+cufftResult_t cufftXtExecDescriptorC2C(...) {
+    return HIPFFT_NOT_IMPLEMENTED;
+}
+
+cufftResult_t cufftXtExecDescriptorZ2Z(...) {
+    return HIPFFT_NOT_IMPLEMENTED;
+}
+
+} // extern "C"
+
+#else  // defined(CUPY_NO_CUDA)
+
+#include "../../cupy_backends/stub/cupy_cuda_common.h"
+#include "../../cupy_backends/stub/cupy_cuComplex.h"
+
+extern "C" {
+
+typedef float cufftReal;
+typedef double cufftDoubleReal;
+typedef cuComplex cufftComplex;
+typedef cuDoubleComplex cufftDoubleComplex;
+
+typedef enum {
+    CUFFT_SUCCESS = 0,
+    CUFFT_INVALID_PLAN = 1,
+    CUFFT_ALLOC_FAILED = 2,
+    CUFFT_INVALID_TYPE = 3,
+    CUFFT_INVALID_VALUE = 4,
+    CUFFT_INTERNAL_ERROR = 5,
+    CUFFT_EXEC_FAILED = 6,
+    CUFFT_SETUP_FAILED = 7,
+    CUFFT_INVALID_SIZE = 8,
+    CUFFT_UNALIGNED_DATA = 9,
+    CUFFT_INCOMPLETE_PARAMETER_LIST = 10,
+    CUFFT_INVALID_DEVICE = 11,
+    CUFFT_PARSE_ERROR = 12,
+    CUFFT_NO_WORKSPACE = 13,
+    CUFFT_NOT_IMPLEMENTED = 14,
+    CUFFT_LICENSE_ERROR = 15,
+    CUFFT_NOT_SUPPORTED = 16,
+} cufftResult_t;
+
+typedef int cufftHandle;
+
+typedef enum {} cufftType_t;
+
+// cuFFT Helper Function
+cufftResult_t cufftCreate(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftDestroy(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftSetAutoAllocation(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftSetWorkArea(...) {
+    return CUFFT_SUCCESS;
+}
+
+// cuFFT Stream Function
+cufftResult_t cufftSetStream(...) {
+    return CUFFT_SUCCESS;
+}
+
+// cuFFT Plan Functions
+cufftResult_t cufftMakePlan1d(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftMakePlanMany(...) {
+    return CUFFT_SUCCESS;
+}
+
+// cuFFT Exec Function
+cufftResult_t cufftExecC2C(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftExecR2C(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftExecC2R(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftExecZ2Z(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftExecD2Z(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftExecZ2D(...) {
+    return CUFFT_SUCCESS;
+}
+
+// cuFFT Version
+cufftResult_t cufftGetVersion(...) {
+    return CUFFT_SUCCESS;
+}
+
+// cufftXt functions
+cufftResult_t cufftXtSetGPUs(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftXtSetWorkArea(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftXtMemcpy(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftXtMakePlanMany(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftXtExec(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftXtExecDescriptorC2C(...) {
+    return CUFFT_SUCCESS;
+}
+
+cufftResult_t cufftXtExecDescriptorZ2Z(...) {
+    return CUFFT_SUCCESS;
+}
+
+}  // extern "C"
+
+#endif  // #if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+
+#if defined(CUPY_NO_CUDA) || defined(CUPY_USE_HIP)
+// common stubs for both no-cuda and hip environments
+
+extern "C" {
+// cufftXt relavant data structs
+typedef struct cudaXtDesc_t {
+   int version;
+   int nGPUs;
+   int GPUs[64];
+   void* data[64];
+   size_t size[64];
+   void* cudaXtState;
+} cudaXtDesc;
+
+typedef enum cufftXtSubFormat_t {
+    CUFFT_XT_FORMAT_INPUT = 0x00,
+    CUFFT_XT_FORMAT_OUTPUT = 0x01,
+    CUFFT_XT_FORMAT_INPLACE = 0x02,
+    CUFFT_XT_FORMAT_INPLACE_SHUFFLED = 0x03,
+    CUFFT_XT_FORMAT_1D_INPUT_SHUFFLED = 0x04,
+    CUFFT_FORMAT_UNDEFINED = 0x05
+} cufftXtSubFormat;
+
+typedef struct cudaLibXtDesc_t{
+    int version;
+    cudaXtDesc *descriptor;
+    int library;  // libFormat is an undoumented type, so use int here
+    int subFormat;
+    void *libDescriptor;
+} cudaLibXtDesc;
+
+typedef enum cufftXtCopyType_t {
+    CUFFT_COPY_HOST_TO_DEVICE = 0x00,
+    CUFFT_COPY_DEVICE_TO_HOST = 0x01,
+    CUFFT_COPY_DEVICE_TO_DEVICE = 0x02,
+    CUFFT_COPY_UNDEFINED = 0x03
+} cufftXtCopyType;
+
+} // extern "C"
+
+#endif // #if defined(CUPY_NO_CUDA) || defined(CUPY_USE_HIP)
+
+#endif  // INCLUDE_GUARD_CUPY_CUFFT_H
diff --git a/cupy/cuda/cupy_cufftXt.cu b/cupy/cuda/cupy_cufftXt.cu
new file mode 100644
index 0000000..3e04d50
--- /dev/null
+++ b/cupy/cuda/cupy_cufftXt.cu
@@ -0,0 +1,68 @@
+#include "cupy_cufftXt.h"
+
+
+// this must define d_loadCallbackPtr
+${dev_load_callback_ker}
+
+// this must define d_storeCallbackPtr
+${dev_store_callback_ker}
+
+cufftResult set_callback(cufftHandle plan, cufftXtCallbackType type, bool cb_load, void** callerInfo) {
+    if (cb_load) {
+        switch (type) {
+            #ifdef HAS_LOAD_CALLBACK
+            case CUFFT_CB_LD_COMPLEX: {
+                cufftCallbackLoadC h_ptr;
+                cudaMemcpyFromSymbol(&h_ptr, d_loadCallbackPtr, sizeof(h_ptr));
+                return cufftXtSetCallback(plan, (void**)&h_ptr, type, callerInfo);
+            }
+            case CUFFT_CB_LD_COMPLEX_DOUBLE: {
+                cufftCallbackLoadZ h_ptr;
+                cudaMemcpyFromSymbol(&h_ptr, d_loadCallbackPtr, sizeof(h_ptr));
+                return cufftXtSetCallback(plan, (void**)&h_ptr, type, callerInfo);
+            }
+            case CUFFT_CB_LD_REAL: {
+                cufftCallbackLoadR h_ptr;
+                cudaMemcpyFromSymbol(&h_ptr, d_loadCallbackPtr, sizeof(h_ptr));
+                return cufftXtSetCallback(plan, (void**)&h_ptr, type, callerInfo);
+            }
+            case CUFFT_CB_LD_REAL_DOUBLE: {
+                cufftCallbackLoadD h_ptr;
+                cudaMemcpyFromSymbol(&h_ptr, d_loadCallbackPtr, sizeof(h_ptr));
+                return cufftXtSetCallback(plan, (void**)&h_ptr, type, callerInfo);
+            }
+            #endif  // HAS_LOAD_CALLBACK
+            default: {
+                throw std::runtime_error("unrecognized callback");
+            }
+        }
+    } else {
+        switch (type) {
+            #ifdef HAS_STORE_CALLBACK
+            case CUFFT_CB_ST_COMPLEX: {
+                cufftCallbackStoreC h_ptr;
+                cudaMemcpyFromSymbol(&h_ptr, d_storeCallbackPtr, sizeof(h_ptr));
+                return cufftXtSetCallback(plan, (void**)&h_ptr, type, callerInfo);
+            }
+            case CUFFT_CB_ST_COMPLEX_DOUBLE: {
+                cufftCallbackStoreZ h_ptr;
+                cudaMemcpyFromSymbol(&h_ptr, d_storeCallbackPtr, sizeof(h_ptr));
+                return cufftXtSetCallback(plan, (void**)&h_ptr, type, callerInfo);
+            }
+            case CUFFT_CB_ST_REAL: {
+                cufftCallbackStoreR h_ptr;
+                cudaMemcpyFromSymbol(&h_ptr, d_storeCallbackPtr, sizeof(h_ptr));
+                return cufftXtSetCallback(plan, (void**)&h_ptr, type, callerInfo);
+            }
+            case CUFFT_CB_ST_REAL_DOUBLE: {
+                cufftCallbackStoreD h_ptr;
+                cudaMemcpyFromSymbol(&h_ptr, d_storeCallbackPtr, sizeof(h_ptr));
+                return cufftXtSetCallback(plan, (void**)&h_ptr, type, callerInfo);
+            }
+            #endif  // HAS_STORE_CALLBACK
+            default: {
+                throw std::runtime_error("unrecognized callback");
+            }
+        }
+    }
+}
diff --git a/cupy/cuda/cupy_cufftXt.h b/cupy/cuda/cupy_cufftXt.h
new file mode 100644
index 0000000..0906230
--- /dev/null
+++ b/cupy/cuda/cupy_cufftXt.h
@@ -0,0 +1,10 @@
+#include <cufftXt.h>
+#include <cstdint>
+#include <stdexcept>
+
+
+extern "C" {
+
+cufftResult set_callback(cufftHandle plan, cufftXtCallbackType cbType, bool cb_load, void** callerInfo=NULL);
+
+}
diff --git a/cupy/cuda/cupy_jitify.h b/cupy/cuda/cupy_jitify.h
new file mode 100644
index 0000000..0157615
--- /dev/null
+++ b/cupy/cuda/cupy_jitify.h
@@ -0,0 +1,41 @@
+#ifndef INCLUDE_GUARD_CUPY_JITIFY_H
+#define INCLUDE_GUARD_CUPY_JITIFY_H
+
+#define _str_(s) #s
+#define _xstr_(s) _str_(s)
+
+#if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+
+#include <cupy/jitify/jitify.hpp>
+namespace jitify {
+namespace detail {
+const char* jitify_ver = _xstr_(CUPY_JITIFY_VERSION_CODE);
+}  // namespace detail
+}  // namespace jitify
+
+#else
+
+namespace jitify {
+namespace detail {
+
+const char* jitify_ver = _xstr_(CUPY_JITIFY_VERSION_CODE);
+std::map<std::string, std::string>& get_jitsafe_headers_map();
+const int preinclude_jitsafe_headers_count = 0;
+const char* preinclude_jitsafe_header_names[] = {};
+void load_program(std::string&,
+                  std::vector<std::string>&,
+                  void*,
+                  std::vector<std::string>*,
+                  std::map<std::string, std::string>*,
+                  std::vector<std::string>*,
+                  std::string*);
+
+}  // namespace detail
+}  // namespace jitify
+
+#endif
+
+#undef _xstr_
+#undef _str_
+
+#endif  // INCLUDE_GUARD_CUPY_JITIFY_H
diff --git a/cupy/cuda/cupy_thrust.cu b/cupy/cuda/cupy_thrust.cu
new file mode 100644
index 0000000..ff39b40
--- /dev/null
+++ b/cupy/cuda/cupy_thrust.cu
@@ -0,0 +1,526 @@
+#include <cupy/type_dispatcher.cuh>
+#include <thrust/device_ptr.h>
+#include <thrust/device_vector.h>
+#include <thrust/iterator/zip_iterator.h>
+#include <thrust/iterator/constant_iterator.h>
+#include <thrust/sequence.h>
+#include <thrust/sort.h>
+#include <thrust/tuple.h>
+#include <thrust/execution_policy.h>
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+// This is used to avoid a problem with constexpr in functions declarations introduced in
+// cuda 11.2, MSVC 15 does not fully support it so we need a dummy constexpr declaration
+// that is provided by this header. However optional.h is only available
+// starting CUDA 10.1
+#include <thrust/optional.h>
+
+#ifdef _MSC_VER
+#define THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS constexpr
+#else
+#define THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS THRUST_OPTIONAL_CPP11_CONSTEXPR
+#endif
+
+#endif
+#include "cupy_thrust.h"
+
+
+#if CUPY_USE_HIP
+typedef hipStream_t cudaStream_t;
+namespace cuda {
+    using thrust::hip::par;
+}
+#else // #if CUPY_USE_HIP
+namespace cuda {
+    using thrust::cuda::par;
+}
+#endif // #if CUPY_USE_HIP
+
+
+extern "C" char *cupy_malloc(void *, size_t);
+extern "C" void cupy_free(void *, char *);
+
+
+class cupy_allocator {
+private:
+    void* memory;
+
+public:
+    typedef char value_type;
+
+    cupy_allocator(void* memory) : memory(memory) {}
+
+    char *allocate(size_t num_bytes) {
+        return cupy_malloc(memory, num_bytes);
+    }
+
+    void deallocate(char *ptr, size_t n) {
+        cupy_free(memory, ptr);
+    }
+};
+
+
+/*
+ * ------------------------------------- Minimum boilerplate for NumPy compatibility --------------------------------------
+ * We need a specialized operator< here in order to match the NumPy behavior:
+ * "The sort order for complex numbers is lexicographic. If both the real and imaginary parts are non-nan then the order is
+ * determined by the real parts except when they are equal, in which case the order is determined by the imaginary parts.
+ *
+ * In numpy versions >= 1.4.0 nan values are sorted to the end. The extended sort order is:
+ *     Real: [R, nan]
+ *     Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj]
+ * where R is a non-nan real value. Complex values with the same nan placements are sorted according to the non-nan part if
+ * it exists. Non-nan values are sorted as before."
+ * Ref: https://numpy.org/doc/stable/reference/generated/numpy.sort.html
+ */
+
+template <typename T>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2))
+THRUST_OPTIONAL_CPP11_CONSTEXPR
+#endif
+bool _tuple_less(const thrust::tuple<size_t, T>& lhs,
+		 const thrust::tuple<size_t, T>& rhs) {
+    const size_t& lhs_k = lhs.template get<0>();
+    const size_t& rhs_k = rhs.template get<0>();
+    const T& lhs_v = lhs.template get<1>();
+    const T& rhs_v = rhs.template get<1>();
+    const thrust::less<T> _less;
+
+    // tuple's comparison rule: compare the 1st member, then 2nd, then 3rd, ...,
+    // which should be respected
+    if (lhs_k < rhs_k) {
+        return true;
+    } else if (lhs_k == rhs_k) {
+        // same key, compare values
+        // note that we can't rely on native operator< due to NaN, so we rely on
+        // thrust::less() to be specialized shortly
+        return _less(lhs_v, rhs_v);
+    } else {
+        return false;
+    }
+}
+
+/*
+ * ********** complex numbers **********
+ * We need to specialize thrust::less because obviously we can't overload operator< for complex numbers...
+ */
+
+template <typename T>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2))
+THRUST_OPTIONAL_CPP11_CONSTEXPR
+#endif
+bool _cmp_less(const T& lhs, const T& rhs) {
+    bool lhsRe = isnan(lhs.real());
+    bool lhsIm = isnan(lhs.imag());
+    bool rhsRe = isnan(rhs.real());
+    bool rhsIm = isnan(rhs.imag());
+
+    // neither side has nan
+    if (!lhsRe && !lhsIm && !rhsRe && !rhsIm) {
+        return lhs < rhs;
+    }
+
+    // one side has nan, and the other does not
+    if (!lhsRe && !lhsIm && (rhsRe || rhsIm)) {
+        return true;
+    }
+    if ((lhsRe || lhsIm) && !rhsRe && !rhsIm) {
+        return false;
+    }
+
+    // pick 2 from 3 possibilities (R + nanj, nan + Rj, nan + nanj)
+    if (lhsRe && !rhsRe) {
+        return false;
+    }
+    if (!lhsRe && rhsRe) {
+        return true;
+    }
+    if (lhsIm && !rhsIm) {
+        return false;
+    }
+    if (!lhsIm && rhsIm) {
+        return true;
+    }
+
+    // pick 1 from 3 and compare the numerical values (nan+nan*I compares to itself as false)
+    return (((lhsIm && rhsIm) && (lhs.real() < rhs.real())) || ((lhsRe && rhsRe) && (lhs.imag() < rhs.imag())));
+}
+
+// specialize thrust::less for single complex
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less<complex<float>>::operator() (
+    const complex<float>& lhs, const complex<float>& rhs) const {
+
+    return _cmp_less<complex<float>>(lhs, rhs);
+}
+
+// specialize thrust::less for double complex
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less<complex<double>>::operator() (
+    const complex<double>& lhs, const complex<double>& rhs) const {
+
+    return _cmp_less<complex<double>>(lhs, rhs);
+}
+
+// specialize thrust::less for tuple<size_t, complex<float>>
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less< thrust::tuple<size_t, complex<float>> >::operator() (
+    const thrust::tuple<size_t, complex<float>>& lhs, const thrust::tuple<size_t, complex<float>>& rhs) const {
+
+    return _tuple_less<complex<float>>(lhs, rhs);
+}
+
+// specialize thrust::less for tuple<size_t, complex<double>>
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less< thrust::tuple<size_t, complex<double>> >::operator() (
+    const thrust::tuple<size_t, complex<double>>& lhs, const thrust::tuple<size_t, complex<double>>& rhs) const {
+
+    return _tuple_less<complex<double>>(lhs, rhs);
+}
+
+/*
+ * ********** real numbers (templates) **********
+ * We need to specialize thrust::less because obviously we can't overload operator< for floating point numbers...
+ */
+
+template <typename T>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2))
+THRUST_OPTIONAL_CPP11_CONSTEXPR
+#endif
+bool _real_less(const T& lhs, const T& rhs) {
+    #if  (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
+    if (isnan(lhs)) {
+        return false;
+    } else if (isnan(rhs)) {
+        return true;
+    } else {
+        return lhs < rhs;
+    }
+    #else
+    return false;  // This will be never executed in the host
+    #endif
+}
+
+/*
+ * ********** real numbers (specializations for single & double precisions) **********
+ */
+
+// specialize thrust::less for float
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less<float>::operator() (
+    const float& lhs, const float& rhs) const {
+
+    return _real_less<float>(lhs, rhs);
+}
+
+// specialize thrust::less for double
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less<double>::operator() (
+    const double& lhs, const double& rhs) const {
+
+    return _real_less<double>(lhs, rhs);
+}
+
+// specialize thrust::less for tuple<size_t, float>
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less< thrust::tuple<size_t, float> >::operator() (
+    const thrust::tuple<size_t, float>& lhs, const thrust::tuple<size_t, float>& rhs) const {
+
+    return _tuple_less<float>(lhs, rhs);
+}
+
+// specialize thrust::less for tuple<size_t, double>
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less< thrust::tuple<size_t, double> >::operator() (
+    const thrust::tuple<size_t, double>& lhs, const thrust::tuple<size_t, double>& rhs) const {
+
+    return _tuple_less<double>(lhs, rhs);
+}
+
+/*
+ * ********** real numbers (specializations for half precision) **********
+ */
+
+#if ((__CUDACC_VER_MAJOR__ > 9 || (__CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2)) \
+     && (__CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__))) || (defined(__HIPCC__) || defined(CUPY_USE_HIP))
+
+// it seems Thrust doesn't care the code path on host, so we just need a wrapper for device
+__host__ __device__ __forceinline__ bool isnan(const __half& x) {
+    #if (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
+    return __hisnan(x);
+    #else
+    return false;  // This will never be called on the host
+    #endif
+}
+
+// specialize thrust::less for __half
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less<__half>::operator() (const __half& lhs, const __half& rhs) const {
+    return _real_less<__half>(lhs, rhs);
+}
+
+// specialize thrust::less for tuple<size_t, __half>
+template <>
+__host__ __device__ __forceinline__
+#if (__CUDACC_VER_MAJOR__ >11 || (__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ >= 2) || HIP_VERSION >= 402)
+THRUST_OPTIONAL_CPP11_CONSTEXPR_LESS
+#endif
+bool thrust::less< thrust::tuple<size_t, __half> >::operator() (
+    const thrust::tuple<size_t, __half>& lhs, const thrust::tuple<size_t, __half>& rhs) const {
+
+    return _tuple_less<__half>(lhs, rhs);
+}
+
+#endif  // include cupy_fp16.h
+
+/*
+ * -------------------------------------------------- end of boilerplate --------------------------------------------------
+ */
+
+
+/*
+ * sort
+ */
+
+struct _sort {
+    template <typename T>
+    __forceinline__ void operator()(void *data_start, size_t *keys_start,
+                                    const std::vector<ptrdiff_t>& shape, intptr_t stream,
+                                    void* memory) {
+        size_t ndim = shape.size();
+        ptrdiff_t size;
+	thrust::device_ptr<T> dp_data_first, dp_data_last;
+        thrust::device_ptr<size_t> dp_keys_first, dp_keys_last;
+        cudaStream_t stream_ = (cudaStream_t)stream;
+        cupy_allocator alloc(memory);
+
+        // Compute the total size of the array.
+        size = shape[0];
+        for (size_t i = 1; i < ndim; ++i) {
+            size *= shape[i];
+        }
+
+        dp_data_first = thrust::device_pointer_cast(static_cast<T*>(data_start));
+        dp_data_last  = thrust::device_pointer_cast(static_cast<T*>(data_start) + size);
+
+        if (ndim == 1) {
+            stable_sort(cuda::par(alloc).on(stream_), dp_data_first, dp_data_last, thrust::less<T>());
+        } else {
+            // Generate key indices.
+            dp_keys_first = thrust::device_pointer_cast(keys_start);
+            dp_keys_last  = thrust::device_pointer_cast(keys_start + size);
+            transform(cuda::par(alloc).on(stream_),
+                      #ifdef __HIP_PLATFORM_HCC__
+                      rocprim::make_counting_iterator<size_t>(0),
+                      rocprim::make_counting_iterator<size_t>(size),
+                      rocprim::make_constant_iterator<ptrdiff_t>(shape[ndim-1]),
+                      #else
+                      thrust::make_counting_iterator<size_t>(0),
+                      thrust::make_counting_iterator<size_t>(size),
+                      thrust::make_constant_iterator<ptrdiff_t>(shape[ndim-1]),
+                      #endif
+                      dp_keys_first,
+                      thrust::divides<size_t>());
+
+            stable_sort(
+                cuda::par(alloc).on(stream_),
+                make_zip_iterator(make_tuple(dp_keys_first, dp_data_first)),
+                make_zip_iterator(make_tuple(dp_keys_last, dp_data_last)),
+                thrust::less< thrust::tuple<size_t, T> >());
+        }
+    }
+};
+
+
+/*
+ * lexsort
+ */
+
+template <typename T>
+class elem_less {
+public:
+    elem_less(const T *data):_data(data) {}
+    __device__ __forceinline__ bool operator()(size_t i, size_t j) const {
+        return thrust::less<T>()(_data[i], _data[j]);
+    }
+private:
+    const T *_data;
+};
+
+struct _lexsort {
+    template <typename T>
+    __forceinline__ void operator()(size_t *idx_start, void *keys_start, size_t k,
+                                    size_t n, intptr_t stream, void *memory) {
+        /* idx_start is the beginning of the output array where the indexes that
+           would sort the data will be placed. The original contents of idx_start
+           will be destroyed. */
+        thrust::device_ptr<size_t> dp_first = thrust::device_pointer_cast(idx_start);
+        thrust::device_ptr<size_t> dp_last  = thrust::device_pointer_cast(idx_start + n);
+        cudaStream_t stream_ = (cudaStream_t)stream;
+        cupy_allocator alloc(memory);
+        sequence(cuda::par(alloc).on(stream_), dp_first, dp_last);
+        for (size_t i = 0; i < k; ++i) {
+            T *key_start = static_cast<T*>(keys_start) + i * n;
+            stable_sort(
+                cuda::par(alloc).on(stream_),
+                dp_first,
+                dp_last,
+                elem_less<T>(key_start)
+            );
+        }
+    }
+};
+
+
+/*
+ * argsort
+ */
+
+struct _argsort {
+    template <typename T>
+    __forceinline__ void operator()(size_t *idx_start, void *data_start,
+                                    void *keys_start,
+                                    const std::vector<ptrdiff_t>& shape,
+                                    intptr_t stream, void *memory) {
+        /* idx_start is the beginning of the output array where the indexes that
+           would sort the data will be placed. The original contents of idx_start
+           will be destroyed. */
+
+        size_t ndim = shape.size();
+        ptrdiff_t size;
+        cudaStream_t stream_ = (cudaStream_t)stream;
+        cupy_allocator alloc(memory);
+
+        thrust::device_ptr<size_t> dp_idx_first, dp_idx_last;
+        thrust::device_ptr<T> dp_data_first, dp_data_last;
+        thrust::device_ptr<size_t> dp_keys_first, dp_keys_last;
+
+        // Compute the total size of the data array.
+        size = shape[0];
+        for (size_t i = 1; i < ndim; ++i) {
+            size *= shape[i];
+        }
+
+        // Cast device pointers of data.
+        dp_data_first = thrust::device_pointer_cast(static_cast<T*>(data_start));
+        dp_data_last  = thrust::device_pointer_cast(static_cast<T*>(data_start) + size);
+
+        // Generate an index sequence.
+        dp_idx_first = thrust::device_pointer_cast(static_cast<size_t*>(idx_start));
+        dp_idx_last  = thrust::device_pointer_cast(static_cast<size_t*>(idx_start) + size);
+        transform(cuda::par(alloc).on(stream_),
+                  #ifdef __HIP_PLATFORM_HCC__
+                  rocprim::make_counting_iterator<size_t>(0),
+                  rocprim::make_counting_iterator<size_t>(size),
+                  rocprim::make_constant_iterator<ptrdiff_t>(shape[ndim-1]),
+                  #else
+                  thrust::make_counting_iterator<size_t>(0),
+                  thrust::make_counting_iterator<size_t>(size),
+                  thrust::make_constant_iterator<ptrdiff_t>(shape[ndim-1]),
+                  #endif
+                  dp_idx_first,
+                  thrust::modulus<size_t>());
+
+        if (ndim == 1) {
+            // Sort the index sequence by data.
+            stable_sort_by_key(cuda::par(alloc).on(stream_),
+                               dp_data_first,
+                               dp_data_last,
+                               dp_idx_first);
+        } else {
+            // Generate key indices.
+            dp_keys_first = thrust::device_pointer_cast(static_cast<size_t*>(keys_start));
+            dp_keys_last  = thrust::device_pointer_cast(static_cast<size_t*>(keys_start) + size);
+            transform(cuda::par(alloc).on(stream_),
+                      #ifdef __HIP_PLATFORM_HCC__
+                      rocprim::make_counting_iterator<size_t>(0),
+                      rocprim::make_counting_iterator<size_t>(size),
+                      rocprim::make_constant_iterator<ptrdiff_t>(shape[ndim-1]),
+                      #else
+                      thrust::make_counting_iterator<size_t>(0),
+                      thrust::make_counting_iterator<size_t>(size),
+                      thrust::make_constant_iterator<ptrdiff_t>(shape[ndim-1]),
+                      #endif
+                      dp_keys_first,
+                      thrust::divides<size_t>());
+
+            stable_sort_by_key(
+                cuda::par(alloc).on(stream_),
+                make_zip_iterator(make_tuple(dp_keys_first, dp_data_first)),
+                make_zip_iterator(make_tuple(dp_keys_last, dp_data_last)),
+                dp_idx_first);
+        }
+    }
+};
+
+
+//
+// APIs exposed to CuPy
+//
+
+/* -------- sort -------- */
+
+void thrust_sort(int dtype_id, void *data_start, size_t *keys_start,
+    const std::vector<ptrdiff_t>& shape, intptr_t stream, void* memory) {
+
+    _sort op;
+    return dtype_dispatcher(dtype_id, op, data_start, keys_start, shape, stream, memory);
+}
+
+
+/* -------- lexsort -------- */
+void thrust_lexsort(int dtype_id, size_t *idx_start, void *keys_start, size_t k,
+    size_t n, intptr_t stream, void *memory) {
+
+    _lexsort op;
+    return dtype_dispatcher(dtype_id, op, idx_start, keys_start, k, n, stream, memory);
+}
+
+
+/* -------- argsort -------- */
+void thrust_argsort(int dtype_id, size_t *idx_start, void *data_start,
+    void *keys_start, const std::vector<ptrdiff_t>& shape, intptr_t stream, void *memory) {
+
+    _argsort op;
+    return dtype_dispatcher(dtype_id, op, idx_start, data_start, keys_start, shape,
+                            stream, memory);
+}
+
diff --git a/cupy/cuda/cupy_thrust.h b/cupy/cuda/cupy_thrust.h
new file mode 100644
index 0000000..b981ccf
--- /dev/null
+++ b/cupy/cuda/cupy_thrust.h
@@ -0,0 +1,26 @@
+#ifndef INCLUDE_GUARD_CUPY_CUDA_THRUST_H
+#define INCLUDE_GUARD_CUPY_CUDA_THRUST_H
+
+#ifndef CUPY_NO_CUDA
+#include <thrust/version.h>  // for THRUST_VERSION
+
+void thrust_sort(int, void *, size_t *, const std::vector<ptrdiff_t>&, intptr_t, void *);
+void thrust_lexsort(int, size_t *, void *, size_t, size_t, intptr_t, void *);
+void thrust_argsort(int, size_t *, void *, void *, const std::vector<ptrdiff_t>&, intptr_t, void *);
+
+#else // CUPY_NO_CUDA
+
+#define THRUST_VERSION 0
+
+void thrust_sort(...) {
+}
+
+void thrust_lexsort(...) {
+}
+
+void thrust_argsort(...) {
+}
+
+#endif // #ifndef CUPY_NO_CUDA
+
+#endif // INCLUDE_GUARD_CUPY_CUDA_THRUST_H
diff --git a/cupy/cuda/cutensor.py b/cupy/cuda/cutensor.py
new file mode 100644
index 0000000..aa1dc21
--- /dev/null
+++ b/cupy/cuda/cutensor.py
@@ -0,0 +1,14 @@
+"""
+cuTENSOR Wrapper
+
+Use `cupy_backends.cuda.libs.cutensor` directly in CuPy codebase.
+"""
+
+available = True
+
+try:
+    from cupy_backends.cuda.libs.cutensor import *  # NOQA
+except ImportError as e:
+    available = False
+    from cupy._environment import _preload_warning
+    _preload_warning('cutensor', e)
diff --git a/cupy/cuda/device.pxd b/cupy/cuda/device.pxd
new file mode 100644
index 0000000..f69afdd
--- /dev/null
+++ b/cupy/cuda/device.pxd
@@ -0,0 +1,23 @@
+from libc.stdint cimport intptr_t
+
+
+cpdef int get_device_id() except? -1
+cpdef intptr_t get_cublas_handle() except? 0
+cpdef intptr_t get_cusolver_handle() except? 0
+cpdef intptr_t get_cusolver_sp_handle() except? 0
+cpdef intptr_t get_cusparse_handle() except? 0
+cpdef str get_compute_capability()
+cdef bint _enable_peer_access(int device, int peer) except -1
+
+cdef class Handle:
+    cdef:
+        public size_t handle
+        object _destroy_func
+
+cdef class Device:
+    cdef:
+        public int id
+        list _device_stack
+
+    cpdef use(self)
+    cpdef synchronize(self)
diff --git a/cupy/cuda/device.pyx b/cupy/cuda/device.pyx
new file mode 100644
index 0000000..06ab58d
--- /dev/null
+++ b/cupy/cuda/device.pyx
@@ -0,0 +1,368 @@
+# distutils: language = c++
+
+import threading
+
+from cupy._core import syncdetect
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda.api import runtime as runtime_module
+from cupy_backends.cuda.libs import cublas
+from cupy_backends.cuda.libs import cusolver
+from cupy_backends.cuda.libs import cusparse
+from cupy import _util
+
+
+# This flag is kept for backward compatibility.
+# It is always True as cuSOLVER library is always available in CUDA 8.0+.
+cusolver_enabled = True
+
+cdef object _thread_local = threading.local()
+
+cdef dict _devices = {}
+cdef dict _compute_capabilities = {}
+
+
+cdef class _ThreadLocalStack:
+    cdef list _devices
+
+    def __init__(self):
+        self._devices = []
+
+    @staticmethod
+    cdef _ThreadLocalStack get():
+        try:
+            stack = _thread_local._device_stack
+        except AttributeError:
+            stack = _ThreadLocalStack()
+            _thread_local._device_stack = stack
+        return <_ThreadLocalStack>stack
+
+    cdef void push(self, int device_id) except *:
+        self._devices.append(device_id)
+
+    cdef int pop(self) except -1:
+        return <int>self._devices.pop()
+
+
+cpdef int get_device_id() except? -1:
+    return runtime.getDevice()
+
+
+cpdef Device _get_device():
+    dev_id = runtime.getDevice()
+    ret = _devices.get(dev_id, None)
+    if ret is None:
+        ret = Device()
+        _devices[dev_id] = ret
+    return ret
+
+
+cdef class Handle:
+    def __init__(self, handle, destroy_func):
+        self.handle = handle
+        self._destroy_func = destroy_func
+
+    def __dealloc__(self):
+        self._destroy_func(self.handle)
+
+
+cpdef intptr_t get_cublas_handle() except? 0:
+    return _get_device().cublas_handle
+
+
+cpdef intptr_t get_cusolver_handle() except? 0:
+    return _get_device().cusolver_handle
+
+
+cpdef intptr_t get_cusolver_sp_handle() except? 0:
+    return _get_device().cusolver_sp_handle
+
+
+cpdef intptr_t get_cusparse_handle() except? 0:
+    return _get_device().cusparse_handle
+
+
+cpdef str get_compute_capability():
+    dev_id = get_device_id()
+    ret = _compute_capabilities.get(dev_id, None)
+    if ret is not None:
+        return ret
+    return Device().compute_capability
+
+
+cdef bint _enable_peer_access(int device, int peer) except -1:
+    """Enable accessing memory allocated on `peer` from `device`."""
+    if device == peer:
+        return True
+
+    cdef int can_access = runtime.deviceCanAccessPeer(device, peer)
+    if can_access == 0:
+        return False
+
+    cdef int current = runtime.getDevice()
+    runtime.setDevice(device)
+    try:
+        # Note: external libraries may disable the peer access, so we need to
+        # call this everytime. See #5496.
+        runtime._deviceEnsurePeerAccess(peer)
+    finally:
+        runtime.setDevice(current)
+    return True
+
+
+@_util.memoize()
+def _get_attributes(device_id):
+    """Return a dict containing all device attributes."""
+    d = {}
+    for k, v in runtime_module.__dict__.items():
+        if k.startswith('cudaDevAttr'):
+            try:
+                name = k.replace('cudaDevAttr', '', 1)
+                d[name] = runtime.deviceGetAttribute(v, device_id)
+            except runtime_module.CUDARuntimeError as e:
+                if e.status != runtime.errorInvalidValue:
+                    raise
+    return d
+
+
+cdef class Device:
+
+    """Object that represents a CUDA device.
+
+    This class provides some basic manipulations on CUDA devices.
+
+    It supports the context protocol. For example, the following code is an
+    example of temporarily switching the current device::
+
+       with Device(0):
+           do_something_on_device_0()
+
+    After the *with* statement gets done, the current device is reset to the
+    original one.
+
+    Args:
+        device (int or cupy.cuda.Device): Index of the device to manipulate. Be
+            careful that the device ID (a.k.a. GPU ID) is zero origin. If it is
+            a Device object, then its ID is used. The current device is
+            selected by default.
+
+    Attributes:
+        id (int): ID of this device.
+
+    """
+
+    def __init__(self, device=None):
+        if device is None:
+            self.id = runtime.getDevice()
+        else:
+            self.id = int(device)
+
+    @classmethod
+    def from_pci_bus_id(cls, pci_bus_id):
+        """Returns a new device instance based on a PCI Bus ID
+
+        Args:
+            pci_bus_id (str):
+                The string for a device in the following format
+                [domain]:[bus]:[device].[function] where domain, bus, device,
+                and function are all hexadecimal values.
+        Returns:
+            device (Device):
+                An instance of the Device class that has the PCI Bus ID as
+                given by the argument pci_bus_id.
+        """
+        device_id = runtime.deviceGetByPCIBusId(pci_bus_id)
+        return cls(device_id)
+
+    def __int__(self):
+        return self.id
+
+    def __enter__(self):
+        cdef int id = runtime.getDevice()
+        _ThreadLocalStack.get().push(id)
+        if self.id != id:
+            runtime.setDevice(self.id)
+        return self
+
+    def __exit__(self, *args):
+        runtime.setDevice(_ThreadLocalStack.get().pop())
+
+    def __repr__(self):
+        return '<CUDA Device %d>' % self.id
+
+    cpdef use(self):
+        """Makes this device current.
+
+        If you want to switch a device temporarily, use the *with* statement.
+
+        """
+        runtime.setDevice(self.id)
+        return self
+
+    cpdef synchronize(self):
+        """Synchronizes the current thread to the device."""
+        syncdetect._declare_synchronize()
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(self.id)
+            runtime.deviceSynchronize()
+        finally:
+            runtime.setDevice(prev_device)
+
+    @property
+    def compute_capability(self):
+        """Compute capability of this device.
+
+        The capability is represented by a string containing the major index
+        and the minor index. For example, compute capability 3.5 is represented
+        by the string '35'.
+
+        """
+        if self.id in _compute_capabilities:
+            return _compute_capabilities[self.id]
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(self.id)
+            major = runtime.deviceGetAttribute(
+                runtime.deviceAttributeComputeCapabilityMajor, self.id)
+            minor = runtime.deviceGetAttribute(
+                runtime.deviceAttributeComputeCapabilityMinor, self.id)
+            cc = '%d%d' % (major, minor)
+            _compute_capabilities[self.id] = cc
+            return cc
+        finally:
+            runtime.setDevice(prev_device)
+
+    def _get_handle(self, name, create_func, destroy_func):
+        handles = getattr(_thread_local, name, None)
+        if handles is None:
+            handles = {}
+            setattr(_thread_local, name, handles)
+        handle = handles.get(self.id, None)
+        if handle is not None:
+            return handle.handle
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(self.id)
+            handle = create_func()
+            handles[self.id] = Handle(handle, destroy_func)
+            return handle
+        finally:
+            runtime.setDevice(prev_device)
+
+    @property
+    def cublas_handle(self):
+        """The cuBLAS handle for this device.
+
+        The same handle is used for the same device even if the Device instance
+        itself is different.
+
+        """
+        return self._get_handle(
+            'cublas_handles', cublas.create, cublas.destroy)
+
+    @property
+    def cusolver_handle(self):
+        """The cuSOLVER handle for this device.
+
+        The same handle is used for the same device even if the Device instance
+        itself is different.
+
+        """
+        return self._get_handle(
+            'cusolver_handles', cusolver.create, cusolver.destroy)
+
+    @property
+    def cusolver_sp_handle(self):
+        """The cuSOLVER Sphandle for this device.
+
+        The same handle is used for the same device even if the Device instance
+        itself is different.
+
+        """
+        return self._get_handle(
+            'cusolver_sp_handles', cusolver.spCreate, cusolver.spDestroy)
+
+    @property
+    def cusparse_handle(self):
+        """The cuSPARSE handle for this device.
+
+        The same handle is used for the same device even if the Device instance
+        itself is different.
+
+        """
+        return self._get_handle(
+            'cusparse_sp_handles', cusparse.create, cusparse.destroy)
+
+    @property
+    def mem_info(self):
+        """The device memory info.
+
+        Returns:
+            free: The amount of free memory, in bytes.
+            total: The total amount of memory, in bytes.
+        """
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(self.id)
+            return runtime.memGetInfo()
+        finally:
+            runtime.setDevice(prev_device)
+
+    @property
+    def attributes(self):
+        """A dictionary of device attributes.
+
+        Returns:
+            attributes (dict):
+                Dictionary of attribute values with the names as keys.
+                The string `cudaDevAttr` has been trimmed from the names.
+                For example, the attribute corresponding to the enumerated
+                value `cudaDevAttrMaxThreadsPerBlock` will have key
+                `MaxThreadsPerBlock`.
+        """
+        return _get_attributes(self.id)
+
+    @property
+    def pci_bus_id(self):
+        """A string of the PCI Bus ID
+
+        Returns:
+            pci_bus_id (str):
+                Returned identifier string for the device in the following
+                format [domain]:[bus]:[device].[function] where domain, bus,
+                device, and function are all hexadecimal values.
+        """
+        return runtime.deviceGetPCIBusId(self.id)
+
+    def __richcmp__(Device self, object other, int op):
+        if op == 2:
+            return isinstance(other, Device) and self.id == other.id
+        if op == 3:
+            return not (isinstance(other, Device) and self.id == other.id)
+        if not isinstance(other, Device):
+            return NotImplemented
+        if op == 0:
+            return self.id < other.id
+        if op == 1:
+            return self.id <= other.id
+        if op == 4:
+            return self.id > other.id
+        if op == 5:
+            return self.id >= other.id
+        return NotImplemented
+
+
+def from_pointer(ptr):
+    """Extracts a Device object from a device pointer.
+
+    Args:
+        ptr (int): Pointer to the device memory.
+
+    Returns:
+        Device: The device whose memory the pointer refers to.
+
+    """
+    # Initialize a context to workaround a bug in CUDA 10.2+. (#3991)
+    runtime._ensure_context()
+    attrs = runtime.pointerGetAttributes(ptr)
+    return Device(attrs.device)
diff --git a/cupy/cuda/function.pxd b/cupy/cuda/function.pxd
new file mode 100644
index 0000000..269a8c5
--- /dev/null
+++ b/cupy/cuda/function.pxd
@@ -0,0 +1,39 @@
+from libc.stdint cimport intptr_t
+
+
+cdef class CPointer:
+    cdef void* ptr
+
+
+cdef class Function:
+
+    cdef:
+        public Module module
+        public intptr_t ptr
+
+    cpdef linear_launch(self, size_t size, args, size_t shared_mem=*,
+                        size_t block_max_size=*, stream=*,
+                        bint enable_cooperative_groups=*)
+
+
+cdef class Module:
+
+    cdef:
+        public intptr_t ptr
+        readonly dict mapping
+
+    cpdef load_file(self, filename)
+    cpdef load(self, bytes cubin)
+    cpdef get_global_var(self, name)
+    cpdef get_function(self, name)
+    cpdef _set_mapping(self, dict mapping)
+
+
+cdef class LinkState:
+
+    cdef:
+        public intptr_t ptr
+
+    cpdef add_ptr_data(self, bytes data, unicode name)
+    cpdef add_ptr_file(self, unicode path)
+    cpdef bytes complete(self)
diff --git a/cupy/cuda/function.pyx b/cupy/cuda/function.pyx
new file mode 100644
index 0000000..9d7c4db
--- /dev/null
+++ b/cupy/cuda/function.pyx
@@ -0,0 +1,302 @@
+# distutils: language = c++
+
+import numpy
+import warnings
+
+from libc.stdint cimport int8_t
+from libc.stdint cimport int16_t
+from libc.stdint cimport int32_t
+from libc.stdint cimport int64_t
+from libc.stdint cimport intptr_t
+from libc.stdint cimport uintmax_t
+from libcpp cimport vector
+
+from cupy._core cimport _carray
+from cupy._core.core cimport _ndarray_base
+from cupy_backends.cuda.api cimport driver
+from cupy_backends.cuda.api cimport runtime
+from cupy.cuda cimport stream as stream_module
+from cupy.cuda.memory cimport MemoryPointer
+from cupy.cuda.texture cimport TextureObject, SurfaceObject
+from cupy.cuda import device
+
+
+cdef class CPointer:
+    def __init__(self, p=0):
+        self.ptr = <void*>p
+
+
+cdef class CInt8(CPointer):
+    cdef:
+        int8_t val
+
+    def __init__(self, int8_t v):
+        self.val = v
+        self.ptr = <void*>&self.val
+
+
+cdef class CInt16(CPointer):
+    cdef:
+        int16_t val
+
+    def __init__(self, int16_t v):
+        self.val = v
+        self.ptr = <void*>&self.val
+
+
+cdef class CInt32(CPointer):
+    cdef:
+        int32_t val
+
+    def __init__(self, int32_t v):
+        self.val = v
+        self.ptr = <void*>&self.val
+
+
+cdef class CInt64(CPointer):
+    cdef:
+        int64_t val
+
+    def __init__(self, int64_t v):
+        self.val = v
+        self.ptr = <void*>&self.val
+
+
+cdef class CInt128(CPointer):
+    cdef:
+        double complex val
+
+    def __init__(self, double complex v):
+        self.val = v
+        self.ptr = <void*>&self.val
+
+
+cdef class CUIntMax(CPointer):
+    cdef:
+        uintmax_t val
+
+    def __init__(self, uintmax_t v):
+        self.val = v
+        self.ptr = <void*>&self.val
+
+
+cdef class CIntptr(CPointer):
+    cdef:
+        intptr_t val
+
+    def __init__(self, intptr_t v):
+        self.val = v
+        self.ptr = <void*>&self.val
+
+
+cdef class CNumpyArray(CPointer):
+    cdef:
+        object val
+
+    def __init__(self, v):
+        self.val = v
+        self.ptr = <void*><size_t>v.__array_interface__['data'][0]
+
+
+cdef set _pointer_numpy_types = {numpy.dtype(i).type
+                                 for i in '?bhilqBHILQefdFD'}
+
+
+cdef inline CPointer _pointer(x):
+    cdef Py_ssize_t itemsize
+
+    if x is None:
+        return CPointer()
+    if isinstance(x, _ndarray_base):
+        return (<_ndarray_base>x).get_pointer()
+    if isinstance(x, _carray.Indexer):
+        return (<_carray.Indexer>x).get_pointer()
+    if isinstance(x, MemoryPointer):
+        return CIntptr(x.ptr)
+    if isinstance(x, CPointer):
+        return x
+    if isinstance(x, (TextureObject, SurfaceObject)):
+        return CUIntMax(x.ptr)
+    if isinstance(x, numpy.ndarray):
+        # All numpy.ndarray work with CNumpyArray to pass a kernel argument by
+        # value. Here we allow only arrays of size one so that users do not
+        # mistakenly send numpy.ndarrays instead of cupy.ndarrays to kernels.
+        # This may happen if they forget to convert numpy arrays to cupy arrays
+        # prior to kernel call and would pass silently without this check.
+        if (x.size == 1):
+            return CNumpyArray(x)
+        else:
+            msg = ('You are trying to pass a numpy.ndarray of shape {} as a '
+                   'kernel parameter. Only numpy.ndarrays of size one can be '
+                   'passed by value. If you meant to pass a pointer to __glob'
+                   'al__ memory, you need to pass a cupy.ndarray instead.')
+            raise TypeError(msg.format(x.shape))
+
+    if type(x) not in _pointer_numpy_types:
+        if isinstance(x, int):
+            x = numpy.int64(x)
+        elif isinstance(x, float):
+            x = numpy.float64(x)
+        elif isinstance(x, bool):
+            x = numpy.bool_(x)
+        elif isinstance(x, complex):
+            x = numpy.complex128(x)
+        else:
+            raise TypeError('Unsupported type %s' % type(x))
+
+    itemsize = x.itemsize
+    if itemsize == 1:
+        return CInt8(x.view(numpy.int8))
+    if itemsize == 2:
+        return CInt16(x.view(numpy.int16))
+    if itemsize == 4:
+        return CInt32(x.view(numpy.int32))
+    if itemsize == 8:
+        return CInt64(x.view(numpy.int64))
+    if itemsize == 16:
+        return CInt128(x.view(numpy.complex128))
+    raise TypeError('Unsupported type %s. (size=%d)', type(x), itemsize)
+
+
+cdef inline size_t _get_stream(stream) except *:
+    if stream is None:
+        return stream_module.get_current_stream_ptr()
+    else:
+        return stream.ptr
+
+
+cdef _launch(intptr_t func, Py_ssize_t grid0, int grid1, int grid2,
+             Py_ssize_t block0, int block1, int block2,
+             args, Py_ssize_t shared_mem, size_t stream,
+             bint enable_cooperative_groups=False):
+    cdef list pargs = []
+    cdef vector.vector[void*] kargs
+    cdef CPointer cp
+    kargs.reserve(len(args))
+    for a in args:
+        cp = _pointer(a)
+        pargs.append(cp)  # keep the CPointer objects alive
+        kargs.push_back(cp.ptr)
+
+    runtime._ensure_context()
+
+    cdef int dev_id
+    cdef int num_sm
+    cdef int max_grid_size
+    if enable_cooperative_groups:
+        dev_id = device.get_device_id()
+        num_sm = device._get_attributes(dev_id)['MultiProcessorCount']
+        max_grid_size = driver.occupancyMaxActiveBlocksPerMultiprocessor(
+            func, block0 * block1 * block2, shared_mem) * num_sm
+        if grid0 * grid1 * grid2 > max_grid_size:
+            if grid1 == grid2 == 1:
+                warnings.warn('The grid size will be reduced from {} to {}, '
+                              'as the specified grid size exceeds the limit.'.
+                              format(grid0, max_grid_size))
+                grid0 = max_grid_size
+            else:
+                raise ValueError('The specified grid size ({} * {} * {}) '
+                                 'exceeds the limit ({}).'.
+                                 format(grid0, grid1, grid2, max_grid_size))
+        driver.launchCooperativeKernel(
+            func, <int>grid0, grid1, grid2, <int>block0, block1, block2,
+            <int>shared_mem, stream, <intptr_t>kargs.data())
+    else:
+        driver.launchKernel(
+            func, <int>grid0, grid1, grid2, <int>block0, block1, block2,
+            <int>shared_mem, stream, <intptr_t>kargs.data(), <intptr_t>0)
+
+
+cdef class Function:
+
+    """CUDA kernel function."""
+
+    def __init__(self, Module module, str funcname):
+        self.module = module  # to keep module loaded
+        self.ptr = driver.moduleGetFunction(module.ptr, funcname)
+
+    def __call__(self, tuple grid, tuple block, args, size_t shared_mem=0,
+                 stream=None, enable_cooperative_groups=False):
+        grid = (grid + (1, 1))[:3]
+        block = (block + (1, 1))[:3]
+        s = _get_stream(stream)
+        _launch(
+            self.ptr,
+            max(1, grid[0]), max(1, grid[1]), max(1, grid[2]),
+            max(1, block[0]), max(1, block[1]), max(1, block[2]),
+            args, shared_mem, s, enable_cooperative_groups)
+
+    cpdef linear_launch(self, size_t size, args, size_t shared_mem=0,
+                        size_t block_max_size=128, stream=None,
+                        bint enable_cooperative_groups=False):
+        # TODO(beam2d): Tune it
+        cdef size_t gridx = min(
+            0x7fffffffUL, (size + block_max_size - 1) // block_max_size)
+        cdef size_t blockx = min(block_max_size, size)
+        s = _get_stream(stream)
+        _launch(
+            self.ptr,
+            gridx, 1, 1, blockx, 1, 1,
+            args,
+            shared_mem, s, enable_cooperative_groups)
+
+
+cdef class Module:
+
+    """CUDA kernel module."""
+
+    def __init__(self):
+        self.ptr = 0
+        self.mapping = None
+
+    def __dealloc__(self):
+        if self.ptr:
+            driver.moduleUnload(self.ptr)
+            self.ptr = 0
+
+    cpdef load_file(self, filename):
+        if isinstance(filename, bytes):
+            filename = filename.decode()
+        runtime._ensure_context()
+        self.ptr = driver.moduleLoad(filename)
+
+    cpdef load(self, bytes cubin):
+        runtime._ensure_context()
+        self.ptr = driver.moduleLoadData(cubin)
+
+    cpdef get_global_var(self, name):
+        if isinstance(name, bytes):
+            name = name.decode()
+        return driver.moduleGetGlobal(self.ptr, name)
+
+    cpdef get_function(self, name):
+        if isinstance(name, bytes):
+            name = name.decode()
+        return Function(self, name)
+
+    cpdef _set_mapping(self, dict mapping):
+        self.mapping = mapping
+
+
+cdef class LinkState:
+
+    """CUDA link state."""
+
+    def __init__(self):
+        runtime._ensure_context()
+        self.ptr = driver.linkCreate()
+
+    def __dealloc__(self):
+        if self.ptr:
+            driver.linkDestroy(self.ptr)
+            self.ptr = 0
+
+    cpdef add_ptr_data(self, bytes data, unicode name):
+        driver.linkAddData(self.ptr, driver.CU_JIT_INPUT_PTX, data, name)
+
+    cpdef add_ptr_file(self, unicode path):
+        driver.linkAddFile(self.ptr, driver.CU_JIT_INPUT_LIBRARY, path)
+
+    cpdef bytes complete(self):
+        cubin = driver.linkComplete(self.ptr)
+        return cubin
diff --git a/cupy/cuda/graph.pxd b/cupy/cuda/graph.pxd
new file mode 100644
index 0000000..194eb5c
--- /dev/null
+++ b/cupy/cuda/graph.pxd
@@ -0,0 +1,15 @@
+from libc.stdint cimport intptr_t
+
+
+cdef class Graph:
+    cdef:
+        readonly intptr_t graph  # cudaGraph_t
+        readonly intptr_t graphExec  # cudaGraphExec_t
+
+    cdef void _init(self, intptr_t g, intptr_t ge) except*
+
+    @staticmethod
+    cdef Graph from_stream(intptr_t g)
+
+    cpdef launch(self, stream=*)
+    cpdef upload(self, stream=*)
diff --git a/cupy/cuda/graph.pyx b/cupy/cuda/graph.pyx
new file mode 100644
index 0000000..005cf0b
--- /dev/null
+++ b/cupy/cuda/graph.pyx
@@ -0,0 +1,80 @@
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda cimport stream as stream_module
+
+
+cdef class Graph:
+    """The CUDA graph object.
+
+    Currently this class cannot be initiated by the user and must be created
+    via stream capture. See :meth:`~cupy.cuda.Stream.begin_capture` for detail.
+
+    """
+
+    cdef void _init(self, intptr_t graph, intptr_t graphExec) except*:
+        if graph > 0:
+            # at this point cudaGraphExec_t has been instantiated, so we no
+            # longer need to hold the cudaGraph_t
+            runtime.graphDestroy(graph)
+        self.graph = 0
+        self.graphExec = graphExec
+
+    def __dealloc__(self):
+        if self.graph > 0:
+            runtime.graphDestroy(self.graph)
+        if self.graphExec > 0:
+            runtime.graphExecDestroy(self.graphExec)
+
+    def __init__(self, *args, **kwargs):
+        raise NotImplementedError(
+            'currently this class cannot be initiated by the user and must '
+            'be created via stream capture')
+
+    @staticmethod
+    cdef Graph from_stream(intptr_t g):
+        # TODO(leofang): optionally print out the error log?
+        cdef intptr_t ge = runtime.graphInstantiate(g)
+        cdef Graph graph = Graph.__new__(Graph)
+        graph._init(g, ge)
+        return graph
+
+    cpdef launch(self, stream=None):
+        """Launch the CUDA graph on the given stream.
+
+        Args:
+            stream (:class:`~cupy.cuda.Stream`): A CuPy stream object. If not
+                specified (using the default value `None`), the graph is
+                launched on the current stream.
+
+        .. seealso:: `cudaGraphLaunch()`_
+
+        .. _cudaGraphLaunch():
+            https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__GRAPH.html#group__CUDART__GRAPH_1g1accfe1da0c605a577c22d9751a09597
+
+        """
+        cdef intptr_t stream_ptr
+        if stream is None:
+            stream_ptr = stream_module.get_current_stream_ptr()
+        else:
+            stream_ptr = stream.ptr
+        runtime.graphLaunch(self.graphExec, stream_ptr)
+
+    cpdef upload(self, stream=None):
+        """Upload the CUDA graph to the given stream.
+
+        Args:
+            stream (:class:`~cupy.cuda.Stream`): A CuPy stream object. If not
+                specified (using the default value `None`), the graph is
+                uploaded the current stream.
+
+        .. seealso:: `cudaGraphUpload()`_
+
+        .. _cudaGraphUpload():
+            https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__GRAPH.html#group__CUDART__GRAPH_1ge546432e411b4495b93bdcbf2fc0b2bd
+
+        """
+        cdef intptr_t stream_ptr
+        if stream is None:
+            stream_ptr = stream_module.get_current_stream_ptr()
+        else:
+            stream_ptr = stream.ptr
+        runtime.graphUpload(self.graphExec, stream_ptr)
diff --git a/cupy/cuda/jitify.pyx b/cupy/cuda/jitify.pyx
new file mode 100644
index 0000000..b9e1b4d
--- /dev/null
+++ b/cupy/cuda/jitify.pyx
@@ -0,0 +1,108 @@
+# distutils: language = c++
+
+"""Wrapper of Jitify utilities for CuPy API."""
+
+from libcpp cimport nullptr
+from libcpp.map cimport map as cpp_map
+from libcpp.string cimport string as cpp_str
+from libcpp.vector cimport vector
+
+
+###############################################################################
+# Extern
+###############################################################################
+
+cdef extern from 'cupy_jitify.h' namespace "jitify::detail" nogil:
+    cpp_map[cpp_str, cpp_str]& get_jitsafe_headers_map()
+    const int preinclude_jitsafe_headers_count
+    const char* preinclude_jitsafe_header_names[]
+    void load_program(cpp_str&,
+                      vector[cpp_str]&,
+                      void*,
+                      vector[cpp_str]*,
+                      cpp_map[cpp_str, cpp_str]*,
+                      vector[cpp_str]*,
+                      cpp_str*) except +
+
+    const char* jitify_ver  # set at build time
+
+
+###############################################################################
+# API
+###############################################################################
+
+def get_build_version():
+    if jitify_ver == b'-1':
+        return '<unknown>'
+    return jitify_ver.decode()
+
+
+# We cache headers found by Jitify. This is initialized with a few built-in
+# JIT-safe headers, and expands as needed to help reduce compile time.
+cdef cpp_map[cpp_str, cpp_str] cupy_headers
+cdef inline void init_cupy_headers():
+    cdef int i
+    for i in range(preinclude_jitsafe_headers_count):
+        hdr_name = preinclude_jitsafe_header_names[i]
+        hdr_source = get_jitsafe_headers_map().at(hdr_name)
+        cupy_headers[hdr_name] = hdr_source
+
+
+init_cupy_headers()
+
+
+cpdef _add_sources(dict sources):
+    for hdr_name, hdr_source in sources.items():
+        cupy_headers[hdr_name] = hdr_source
+
+
+# Use Jitify's internal mechanism to search all included headers, and return
+# the modified options and the header mapping (as two lists). This roughly
+# follows the constructor of jitify::Program(). The found headers are cached
+# to accelerate Jitify's search loop.
+cpdef jitify(str code, tuple opt):
+
+    # input
+    cdef cpp_str cuda_source
+    cdef vector[cpp_str] headers
+
+    # output
+    cdef vector[cpp_str] include_paths
+    cdef cpp_map[cpp_str, cpp_str] _sources = cupy_headers  # copy
+    cdef vector[cpp_str] _options  # the input gets modified
+    cdef cpp_str _name
+    cdef list new_opt = None
+    cdef list hdr_codes = []
+    cdef list hdr_names = []
+
+    # dummy
+    cdef cpp_str h
+    cdef str s
+    cdef bytes k, v
+
+    cuda_source = code.encode()
+    _options = [s.encode() for s in opt]
+
+    with nogil:
+        # Where the real magic happens: a compile-fail-search loop
+        load_program(cuda_source, headers, nullptr, &include_paths,
+                     &_sources, &_options, &_name)
+
+        # Remove input code from header cache
+        _sources.erase(_name)
+
+    # Get updated options. The two additions are from
+    # jitify::detail::compile_kernel()
+    new_opt = [h.decode() for h in _options]
+    new_opt += ['--device-as-default-execution-space',
+                '--pre-include=jitify_preinclude.h']
+
+    # Collect header names and contents (as bytes)
+    for itr in _sources:  # itr is an iterator of std::map
+        k = itr.first
+        v = itr.second
+        hdr_codes.append(v)
+        hdr_names.append(k)
+        cupy_headers[k] = v
+
+    return _name.decode(), tuple(new_opt), hdr_codes, hdr_names
diff --git a/cupy/cuda/memory.pxd b/cupy/cuda/memory.pxd
new file mode 100644
index 0000000..187feb9
--- /dev/null
+++ b/cupy/cuda/memory.pxd
@@ -0,0 +1,95 @@
+cimport cython  # NOQA
+
+from libc.stdint cimport intptr_t
+from libcpp cimport vector
+from libcpp cimport map
+
+from cupy.cuda cimport device
+
+
+@cython.no_gc
+cdef class BaseMemory:
+
+    cdef:
+        public intptr_t ptr
+        public size_t size
+        public int device_id
+
+
+cdef class MemoryPointer:
+
+    cdef:
+        readonly intptr_t ptr
+        readonly int device_id
+        readonly BaseMemory mem
+
+    cdef _init(self, BaseMemory mem, ptrdiff_t offset)
+
+    cpdef copy_from_device(self, MemoryPointer src, size_t size)
+    cpdef copy_from_device_async(self, MemoryPointer src, size_t size,
+                                 stream=?)
+    cpdef copy_from_host(self, mem, size_t size)
+    cpdef copy_from_host_async(self, mem, size_t size, stream=?)
+    cpdef copy_from(self, mem, size_t size)
+    cpdef copy_from_async(self, mem, size_t size, stream=?)
+    cpdef copy_to_host(self, mem, size_t size)
+    cpdef copy_to_host_async(self, mem, size_t size, stream=?)
+    cpdef memset(self, int value, size_t size)
+    cpdef memset_async(self, int value, size_t size, stream=?)
+
+
+cpdef MemoryPointer alloc(size)
+
+
+cpdef set_allocator(allocator=*)
+cpdef get_allocator()
+
+
+cdef class MemoryPool:
+
+    cdef:
+        object _pools
+
+    cpdef MemoryPointer malloc(self, size_t size)
+    cpdef free_all_blocks(self, stream=?)
+    cpdef free_all_free(self)
+    cpdef size_t n_free_blocks(self)
+    cpdef size_t used_bytes(self)
+    cpdef size_t free_bytes(self)
+    cpdef size_t total_bytes(self)
+    cpdef set_limit(self, size=?, fraction=?)
+    cpdef size_t get_limit(self)
+
+
+@cython.no_gc
+cdef class CFunctionAllocatorMemory(BaseMemory):
+
+    cdef:
+        intptr_t _param
+        intptr_t _free_func
+
+
+cdef class CFunctionAllocator:
+
+    cdef:
+        intptr_t _param
+        intptr_t _malloc_func
+        intptr_t _free_func
+        object _owner
+
+    cpdef MemoryPointer malloc(self, size_t size)
+
+
+cdef class PythonFunctionAllocatorMemory(BaseMemory):
+
+    cdef:
+        object _free_func
+
+
+cdef class PythonFunctionAllocator:
+
+    cdef:
+        object _malloc_func
+        object _free_func
+
+    cpdef MemoryPointer malloc(self, size_t size)
diff --git a/cupy/cuda/memory.pyx b/cupy/cuda/memory.pyx
new file mode 100644
index 0000000..e17cbcd
--- /dev/null
+++ b/cupy/cuda/memory.pyx
@@ -0,0 +1,1982 @@
+# distutils: language = c++
+cimport cpython  # NOQA
+cimport cython  # NOQA
+
+import atexit
+import collections
+import gc
+import os
+import threading
+import warnings
+import weakref
+
+from fastrlock cimport rlock
+from libc.stdint cimport int8_t
+from libc.stdint cimport intptr_t
+from libc.stdint cimport UINT64_MAX
+from libcpp cimport algorithm
+
+from cupy.cuda cimport device
+from cupy.cuda cimport memory_hook
+from cupy.cuda cimport stream as stream_module
+from cupy_backends.cuda.api cimport driver
+from cupy_backends.cuda.api cimport runtime
+
+from cupy_backends.cuda.api.runtime import CUDARuntimeError
+from cupy import _util
+
+
+cdef bint _exit_mode = False
+
+
+@atexit.register
+def _exit():
+    _exit_mode = True
+
+
+class OutOfMemoryError(MemoryError):
+    """Out-of-memory error.
+
+    Args:
+        size (int): Size of memory about to be allocated.
+        total (int): Size of memory successfully allocated so far.
+        limit (int): Allocation limit.
+    """
+
+    def __init__(self, size, total, limit=0):
+        self._size = size
+        self._total = total
+        self._limit = limit
+
+        if limit == 0:
+            msg = 'Out of memory allocating {:,} bytes'.format(size)
+            if total != -1:
+                msg += ' (allocated so far: {:,} bytes).'.format(total)
+        else:
+            msg = (
+                'Out of memory allocating {:,} bytes '
+                '(allocated so far: {:,} bytes, '
+                'limit set to: {:,} bytes).'.format(size, total, limit))
+        super(OutOfMemoryError, self).__init__(msg)
+
+    def __reduce__(self):
+        return (type(self), (self._size, self._total, self._limit))
+
+
+@cython.no_gc
+cdef class BaseMemory:
+    """Memory on a CUDA device.
+
+    Attributes:
+        ~Memory.ptr (int): Pointer to the place within the buffer.
+        ~Memory.size (int): Size of the memory allocation in bytes.
+        ~Memory.device (~cupy.cuda.Device): Device whose memory the pointer
+            refers to.
+    """
+
+    def __int__(self):
+        """Returns the pointer value to the head of the allocation."""
+        return self.ptr
+
+    @property
+    def device(self):
+        return device.Device(self.device_id)
+
+
+@cython.no_gc
+cdef class Memory(BaseMemory):
+    """Memory allocation on a CUDA device.
+
+    This class provides an RAII interface of the CUDA memory allocation.
+
+    Args:
+        size (int): Size of the memory allocation in bytes.
+    """
+
+    def __init__(self, size_t size):
+        self.size = size
+        self.device_id = device.get_device_id()
+        self.ptr = 0
+        if size > 0:
+            self.ptr = runtime.malloc(size)
+
+    def __dealloc__(self):
+        # Note: Cannot raise in the destructor! (cython/cython#1613)
+        if self.ptr:
+            runtime.free(self.ptr)
+
+
+cdef inline void check_async_alloc_supported(int device_id) except*:
+    if runtime._is_hip_environment:
+        raise RuntimeError('HIP does not support memory_async')
+    if runtime.runtimeGetVersion() < 11020:
+        raise RuntimeError("memory_async is supported since CUDA 11.2")
+    cdef int dev_id
+    cdef list support
+    try:
+        is_supported = _thread_local.device_support_async_alloc[device_id]
+    except AttributeError:
+        support = [runtime.deviceGetAttribute(
+            runtime.cudaDevAttrMemoryPoolsSupported, dev_id)
+            for dev_id in range(runtime.getDeviceCount())]
+        _thread_local.device_support_async_alloc = support
+        is_supported = support[device_id]
+    if not is_supported:
+        raise RuntimeError('Device {} does not support '
+                           'malloc_async'.format(device_id))
+
+
+@cython.no_gc
+cdef class MemoryAsync(BaseMemory):
+    """Asynchronous memory allocation on a CUDA device.
+
+    This class provides an RAII interface of the CUDA memory allocation.
+
+    Args:
+        size (int): Size of the memory allocation in bytes.
+        stream (Stream): The stream on which the memory is allocated and freed.
+    """
+
+    cdef:
+        readonly object stream_ref
+
+    def __init__(self, size_t size, stream):
+        self.size = size
+        self.device_id = device.get_device_id()
+        # The stream is allowed to be destroyed before the memory is freed, so
+        # we don't need to hold a strong reference to the stream.
+        self.stream_ref = weakref.ref(stream)
+        check_async_alloc_supported(self.device_id)
+        if size > 0:
+            self.ptr = runtime.mallocAsync(size, stream.ptr)
+
+    def __dealloc__(self):
+        # Free on the stream on which this memory was allocated.
+        # If the stream is already destroyed, free on the current stream. In
+        # this case, we trust the user has established a correct stream order.
+        if self.ptr == 0:
+            return
+        stream = self.stream_ref()
+        if stream is None:
+            stream = stream_module.get_current_stream()
+        runtime.freeAsync(self.ptr, stream.ptr)
+
+
+cdef class UnownedMemory(BaseMemory):
+    """CUDA memory that is not owned by CuPy.
+
+    Args:
+        ptr (int): Pointer to the buffer.
+        size (int): Size of the buffer.
+        owner (object): Reference to the owner object to keep the memory
+            alive.
+        device_id (int): CUDA device ID of the buffer. If omitted, the device
+            associated to the pointer is retrieved.
+    """
+
+    cdef:
+        readonly object _owner
+
+    def __init__(self, intptr_t ptr, size_t size, object owner,
+                 int device_id=-1):
+        cdef runtime.PointerAttributes ptr_attrs
+        # ptr=0 for 0-size arrays from __cuda_array_interface__ v2:
+        # we need a valid device id as null ptr can't be looked up
+        if device_id < 0:
+            if ptr == 0:
+                raise RuntimeError('UnownedMemory requires explicit'
+                                   ' device ID for a null pointer.')
+            # Initialize a context to workaround a bug in CUDA 10.2+. (#3991)
+            runtime._ensure_context()
+            ptr_attrs = runtime.pointerGetAttributes(ptr)
+            device_id = ptr_attrs.device
+        self.size = size
+        self.device_id = device_id
+        self.ptr = ptr
+        self._owner = owner
+
+
+@cython.no_gc
+cdef class ManagedMemory(BaseMemory):
+    """Managed memory (Unified memory) allocation on a CUDA device.
+
+    This class provides an RAII interface of the CUDA managed memory
+    allocation.
+
+    Args:
+        size (int): Size of the memory allocation in bytes.
+
+    """
+
+    def __init__(self, size_t size):
+        if (
+            runtime._is_hip_environment and
+            driver.get_build_version() < 40300000
+        ):
+            raise RuntimeError('Managed memory requires ROCm 4.3+')
+        self.size = size
+        self.device_id = device.get_device_id()
+        self.ptr = 0
+        if size > 0:
+            self.ptr = runtime.mallocManaged(size)
+
+    def prefetch(self, stream):
+        """(experimental) Prefetch memory.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream.
+        """
+        runtime.memPrefetchAsync(self.ptr, self.size, self.device_id,
+                                 stream.ptr)
+
+    def advise(self, int advise, device.Device dev):
+        """(experimental) Advise about the usage of this memory.
+
+        Args:
+            advics (int): Advise to be applied for this memory.
+            dev (cupy.cuda.Device): Device to apply the advice for.
+
+        """
+        runtime.memAdvise(self.ptr, self.size, advise, dev.id)
+
+    def __dealloc__(self):
+        # Note: Cannot raise in the destructor! (cython/cython#1613)
+        if self.ptr:
+            runtime.free(self.ptr)
+
+
+@cython.final
+cdef class _Chunk:
+
+    """A chunk points to a device memory.
+
+    A chunk might be a splitted memory block from a larger allocation.
+    The prev/next pointers contruct a doubly-linked list of memory addresses
+    sorted by base address that must be contiguous.
+
+    Args:
+        mem (~cupy.cuda.Memory): The device memory buffer.
+        offset (int): An offset bytes from the head of the buffer.
+        size (int): Chunk size in bytes.
+        stream_ident (intptr_t): Value to uniquely identify the stream.
+
+    Attributes:
+        mem (Memory): The device memory buffer.
+        ptr (int): Memory address.
+        offset (int): An offset bytes from the head of the buffer.
+        size (int): Chunk size in bytes.
+        prev (Chunk): prev memory pointer if split from a larger allocation
+        next (Chunk): next memory pointer if split from a larger allocation
+        stream_ident (intptr_t): Value to uniquely identify the stream.
+    """
+
+    cdef:
+        readonly BaseMemory mem
+        readonly ptrdiff_t offset
+        readonly size_t size
+        readonly intptr_t stream_ident
+        public _Chunk prev
+        public _Chunk next
+
+    def __init__(self, *args):
+        # For debug
+        mem, offset, size, stream_ident = args
+        self._init(mem, offset, size, stream_ident)
+
+    cdef _init(self, BaseMemory mem, ptrdiff_t offset,
+               size_t size, intptr_t stream_ident):
+        assert mem.ptr != 0 or offset == 0
+        self.mem = mem
+        self.offset = offset
+        self.size = size
+        self.stream_ident = stream_ident
+
+    cpdef intptr_t ptr(self):
+        return self.mem.ptr + self.offset
+
+    cpdef _Chunk split(self, size_t size):
+        """Split contiguous block of a larger allocation"""
+        cdef _Chunk remaining
+        assert self.size >= size
+        if self.size == size:
+            return None
+        remaining = _Chunk.__new__(_Chunk)
+        remaining._init(self.mem, self.offset + size, self.size - size,
+                        self.stream_ident)
+        self.size = size
+
+        if self.next is not None:
+            remaining.next = self.next
+            remaining.next.prev = remaining
+        self.next = remaining
+        remaining.prev = self
+        return remaining
+
+    cpdef merge(self, _Chunk remaining):
+        """Merge previously splitted block (chunk)"""
+        assert self.stream_ident == remaining.stream_ident
+        self.size += remaining.size
+        self.next = remaining.next
+        if remaining.next is not None:
+            self.next.prev = self
+
+
+cdef class MemoryPointer:
+    """Pointer to a point on a device memory.
+
+    An instance of this class holds a reference to the original memory buffer
+    and a pointer to a place within this buffer.
+
+    Args:
+        mem (~cupy.cuda.BaseMemory): The device memory buffer.
+        offset (int): An offset from the head of the buffer to the place this
+            pointer refers.
+
+    Attributes:
+        ~MemoryPointer.device (~cupy.cuda.Device): Device whose memory the
+            pointer refers to.
+        ~MemoryPointer.mem (~cupy.cuda.BaseMemory): The device memory buffer.
+        ~MemoryPointer.ptr (int): Pointer to the place within the buffer.
+    """
+
+    def __init__(self, BaseMemory mem, ptrdiff_t offset):
+        self._init(mem, offset)
+
+    cdef _init(self, BaseMemory mem, ptrdiff_t offset):
+        assert mem.ptr != 0 or offset == 0
+        self.ptr = mem.ptr + offset
+        self.device_id = mem.device_id
+        self.mem = mem
+
+    def __int__(self):
+        """Returns the pointer value."""
+        return self.ptr
+
+    def __repr__(self):
+        return '<{} 0x{:x} device={} mem={!r}>'.format(
+            self.__class__.__name__,
+            self.ptr, self.device_id, self.mem)
+
+    @property
+    def device(self):
+        return device.Device(self.device_id)
+
+    def __add__(x, y):
+        """Adds an offset to the pointer."""
+        cdef MemoryPointer self
+        cdef ptrdiff_t offset
+        if isinstance(x, MemoryPointer):
+            self = x
+            offset = <ptrdiff_t?>y
+        else:
+            self = <MemoryPointer?>y
+            offset = <ptrdiff_t?>x
+        assert self.ptr != 0 or offset == 0
+        return MemoryPointer(self.mem,
+                             self.ptr - self.mem.ptr + offset)
+
+    def __iadd__(self, ptrdiff_t offset):
+        """Adds an offset to the pointer in place."""
+        assert self.ptr != 0 or offset == 0
+        self.ptr += offset
+        return self
+
+    def __sub__(self, offset):
+        """Subtracts an offset from the pointer."""
+        return self + -offset
+
+    def __isub__(self, ptrdiff_t offset):
+        """Subtracts an offset from the pointer in place."""
+        return self.__iadd__(-offset)
+
+    cpdef copy_from_device(self, MemoryPointer src, size_t size):
+        """Copies a memory sequence from a (possibly different) device.
+
+        Args:
+            src (cupy.cuda.MemoryPointer): Source memory pointer.
+            size (int): Size of the sequence in bytes.
+
+        .. warning::
+
+            This function always uses the legacy default stream and does not
+            honor the current stream. Use `copy_from_device_async` instead
+            if you are using streams in your code, or have PTDS enabled.
+
+        """
+        stream_ptr = stream_module.get_current_stream_ptr()
+        if (
+            not runtime._is_hip_environment
+            and runtime.streamIsCapturing(stream_ptr)
+        ):
+            raise RuntimeError(
+                'the current stream is capturing, so synchronous API calls '
+                'are disallowed')
+        if size > 0:
+            device._enable_peer_access(src.device_id, self.device_id)
+            runtime.memcpy(self.ptr, src.ptr, size,
+                           runtime.memcpyDefault)
+
+    cpdef copy_from_device_async(self, MemoryPointer src, size_t size,
+                                 stream=None):
+        """Copies a memory from a (possibly different) device asynchronously.
+
+        Args:
+            src (cupy.cuda.MemoryPointer): Source memory pointer.
+            size (int): Size of the sequence in bytes.
+            stream (cupy.cuda.Stream): CUDA stream.
+                The default uses CUDA stream of the current context.
+
+        """
+        if stream is None:
+            stream_ptr = stream_module.get_current_stream_ptr()
+        else:
+            stream_ptr = stream.ptr
+        if size > 0:
+            device._enable_peer_access(src.device_id, self.device_id)
+            runtime.memcpyAsync(self.ptr, src.ptr, size,
+                                runtime.memcpyDefault, stream_ptr)
+
+    cpdef copy_from_host(self, mem, size_t size):
+        """Copies a memory sequence from the host memory.
+
+        Args:
+            mem (int or ctypes.c_void_p): Source memory pointer.
+            size (int): Size of the sequence in bytes.
+
+        .. warning::
+
+            This function always uses the legacy default stream and does not
+            honor the current stream. Use `copy_from_host_async` instead
+            if you are using streams in your code, or have PTDS enabled.
+
+        """
+        stream_ptr = stream_module.get_current_stream_ptr()
+        if (
+            not runtime._is_hip_environment
+            and runtime.streamIsCapturing(stream_ptr)
+        ):
+            raise RuntimeError(
+                'the current stream is capturing, so synchronous API calls '
+                'are disallowed')
+        if size > 0:
+            ptr = mem if isinstance(mem, int) else mem.value
+            runtime.memcpy(self.ptr, ptr, size,
+                           runtime.memcpyHostToDevice)
+
+    cpdef copy_from_host_async(self, mem, size_t size, stream=None):
+        """Copies a memory sequence from the host memory asynchronously.
+
+        Args:
+            mem (int or ctypes.c_void_p): Source memory pointer. It must point
+                to pinned memory.
+            size (int): Size of the sequence in bytes.
+            stream (cupy.cuda.Stream): CUDA stream.
+                The default uses CUDA stream of the current context.
+
+        """
+        if stream is None:
+            stream_ptr = stream_module.get_current_stream_ptr()
+        else:
+            stream_ptr = stream.ptr
+        if (
+            not runtime._is_hip_environment
+            and runtime.streamIsCapturing(stream_ptr)
+        ):
+            raise RuntimeError(
+                'the current stream is capturing, so H2D transfers '
+                'are disallowed')
+        if size > 0:
+            ptr = mem if isinstance(mem, int) else mem.value
+            runtime.memcpyAsync(self.ptr, ptr, size,
+                                runtime.memcpyHostToDevice, stream_ptr)
+
+    cpdef copy_from(self, mem, size_t size):
+        """Copies a memory sequence from a (possibly different) device or host.
+
+        This function is a useful interface that selects appropriate one from
+        :meth:`~cupy.cuda.MemoryPointer.copy_from_device` and
+        :meth:`~cupy.cuda.MemoryPointer.copy_from_host`.
+
+        Args:
+            mem (int or ctypes.c_void_p or cupy.cuda.MemoryPointer):
+                Source memory pointer.
+            size (int): Size of the sequence in bytes.
+
+        .. warning::
+
+            This function always uses the legacy default stream and does not
+            honor the current stream. Use `copy_from_async` instead
+            if you are using streams in your code, or have PTDS enabled.
+
+        """
+        if isinstance(mem, MemoryPointer):
+            self.copy_from_device(mem, size)
+        else:
+            self.copy_from_host(mem, size)
+
+    cpdef copy_from_async(self, mem, size_t size, stream=None):
+        """Copies a memory sequence from an arbitrary place asynchronously.
+
+        This function is a useful interface that selects appropriate one from
+        :meth:`~cupy.cuda.MemoryPointer.copy_from_device_async` and
+        :meth:`~cupy.cuda.MemoryPointer.copy_from_host_async`.
+
+        Args:
+            mem (int or ctypes.c_void_p or cupy.cuda.MemoryPointer):
+                Source memory pointer.
+            size (int): Size of the sequence in bytes.
+            stream (cupy.cuda.Stream): CUDA stream.
+                The default uses CUDA stream of the current context.
+
+        """
+        if isinstance(mem, MemoryPointer):
+            self.copy_from_device_async(mem, size, stream)
+        else:
+            self.copy_from_host_async(mem, size, stream)
+
+    cpdef copy_to_host(self, mem, size_t size):
+        """Copies a memory sequence to the host memory.
+
+        Args:
+            mem (int or ctypes.c_void_p): Target memory pointer.
+            size (int): Size of the sequence in bytes.
+
+        .. warning::
+
+            This function always uses the legacy default stream and does not
+            honor the current stream. Use `copy_to_host_async` instead
+            if you are using streams in your code, or have PTDS enabled.
+
+        """
+        stream_ptr = stream_module.get_current_stream_ptr()
+        if (
+            not runtime._is_hip_environment
+            and runtime.streamIsCapturing(stream_ptr)
+        ):
+            raise RuntimeError(
+                'the current stream is capturing, so synchronous API calls '
+                'are disallowed')
+        if size > 0:
+            ptr = mem if isinstance(mem, int) else mem.value
+            runtime.memcpy(ptr, self.ptr, size,
+                           runtime.memcpyDeviceToHost)
+
+    cpdef copy_to_host_async(self, mem, size_t size, stream=None):
+        """Copies a memory sequence to the host memory asynchronously.
+
+        Args:
+            mem (int or ctypes.c_void_p): Target memory pointer. It must point
+                to pinned memory.
+            size (int): Size of the sequence in bytes.
+            stream (cupy.cuda.Stream): CUDA stream.
+                The default uses CUDA stream of the current context.
+
+        """
+        if stream is None:
+            stream_ptr = stream_module.get_current_stream_ptr()
+        else:
+            stream_ptr = stream.ptr
+        if (
+            not runtime._is_hip_environment
+            and runtime.streamIsCapturing(stream_ptr)
+        ):
+            raise RuntimeError(
+                'the current stream is capturing, so D2H transfers '
+                'are disallowed')
+        if size > 0:
+            ptr = mem if isinstance(mem, int) else mem.value
+            runtime.memcpyAsync(ptr, self.ptr, size,
+                                runtime.memcpyDeviceToHost, stream_ptr)
+
+    cpdef memset(self, int value, size_t size):
+        """Fills a memory sequence by constant byte value.
+
+        Args:
+            value (int): Value to fill.
+            size (int): Size of the sequence in bytes.
+
+        .. warning::
+
+            This function always uses the legacy default stream and does not
+            honor the current stream. Use `memset_async` instead
+            if you are using streams in your code, or have PTDS enabled.
+
+        """
+        stream_ptr = stream_module.get_current_stream_ptr()
+        if (
+            not runtime._is_hip_environment
+            and runtime.streamIsCapturing(stream_ptr)
+        ):
+            raise RuntimeError(
+                'the current stream is capturing, so synchronous API calls '
+                'are disallowed')
+        if size > 0:
+            runtime.memset(self.ptr, value, size)
+
+    cpdef memset_async(self, int value, size_t size, stream=None):
+        """Fills a memory sequence by constant byte value asynchronously.
+
+        Args:
+            value (int): Value to fill.
+            size (int): Size of the sequence in bytes.
+            stream (cupy.cuda.Stream): CUDA stream.
+                The default uses CUDA stream of the current context.
+
+        """
+        if stream is None:
+            stream_ptr = stream_module.get_current_stream_ptr()
+        else:
+            stream_ptr = stream.ptr
+        if size > 0:
+            runtime.memsetAsync(self.ptr, value, size, stream_ptr)
+
+
+# cpdef because unit-tested
+cpdef MemoryPointer _malloc(size_t size):
+    mem = Memory(size)
+    return MemoryPointer(mem, 0)
+
+
+cpdef MemoryPointer malloc_async(size_t size):
+    """(Experimental) Allocate memory from Stream Ordered Memory Allocator.
+
+    This method can be used as a CuPy memory allocator. The simplest way to
+    use CUDA's Stream Ordered Memory Allocator as the default allocator is
+    the following code::
+
+        set_allocator(malloc_async)
+
+    Using this feature requires CUDA >= 11.2 with a supported GPU and platform.
+    If it is not supported, an error will be raised.
+
+    The current CuPy stream is used to allocate/free the memory.
+
+    Args:
+        size (int): Size of the memory allocation in bytes.
+
+    Returns:
+        ~cupy.cuda.MemoryPointer: Pointer to the allocated buffer.
+
+    .. warning::
+        This feature is currently experimental and subject to change.
+
+    .. seealso:: `Stream Ordered Memory Allocator`_
+
+    .. _Stream Ordered Memory Allocator:
+        https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#stream-ordered-memory-allocator
+    """
+    mem = MemoryAsync(size, stream_module.get_current_stream())
+    return MemoryPointer(mem, 0)
+
+
+cpdef MemoryPointer malloc_managed(size_t size):
+    """Allocate managed memory (unified memory).
+
+    This method can be used as a CuPy memory allocator. The simplest way to
+    use a managed memory as the default allocator is the following code::
+
+        set_allocator(malloc_managed)
+
+    The advantage using managed memory in CuPy is that device memory
+    oversubscription is possible for GPUs that have a non-zero value for the
+    device attribute cudaDevAttrConcurrentManagedAccess.
+    CUDA >= 8.0 with GPUs later than or equal to Pascal is preferrable.
+
+    Read more at: https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__MEMORY.html#axzz4qygc1Ry1  # NOQA
+
+    Args:
+        size (int): Size of the memory allocation in bytes.
+
+    Returns:
+        ~cupy.cuda.MemoryPointer: Pointer to the allocated buffer.
+    """
+    mem = ManagedMemory(size)
+    return MemoryPointer(mem, 0)
+
+
+cdef object _current_allocator = _malloc
+cdef object _thread_local = threading.local()
+
+
+def _get_thread_local_allocator():
+    try:
+        allocator = _thread_local.allocator
+    except AttributeError:
+        allocator = _thread_local.allocator = None
+    return allocator
+
+
+def _set_thread_local_allocator(allocator):
+    _thread_local.allocator = allocator
+
+
+cdef inline intptr_t _get_stream_identifier(intptr_t stream_ptr):
+    # When PTDS is enabled, return an ID to uniquely identify the default
+    # stream for each thread. (#5069)
+    if stream_ptr != runtime.streamPerThread:
+        return stream_ptr
+
+    cdef intptr_t tid
+    try:
+        tid = _thread_local._tid
+    except AttributeError:
+        _thread_local._tid_obj = tid_obj = object()
+        _thread_local._tid = tid = id(tid_obj)
+    return -tid
+
+
+cpdef MemoryPointer alloc(size):
+    """Calls the current allocator.
+
+    Use :func:`~cupy.cuda.set_allocator` to change the current allocator.
+
+    Args:
+        size (int): Size of the memory allocation.
+
+    Returns:
+        ~cupy.cuda.MemoryPointer: Pointer to the allocated buffer.
+
+    """
+    return get_allocator()(size)
+
+
+cpdef set_allocator(allocator=None):
+    """Sets the current allocator for GPU memory.
+
+    Args:
+        allocator (function): CuPy memory allocator. It must have the same
+            interface as the :func:`cupy.cuda.alloc` function, which takes the
+            buffer size as an argument and returns the device buffer of that
+            size. When ``None`` is specified, raw memory allocator will be
+            used (i.e., memory pool is disabled).
+
+    """
+    global _current_allocator
+    if allocator is None:
+        allocator = _malloc
+    if getattr(_thread_local, 'allocator', None) is not None:
+        raise ValueError('Can\'t change the global allocator inside '
+                         '`using_allocator` context manager')
+    if allocator is malloc_async:
+        _util.experimental('cupy.cuda.malloc_async')
+    _current_allocator = allocator
+
+
+cpdef get_allocator():
+    """Returns the current allocator for GPU memory.
+
+    Returns:
+        function: CuPy memory allocator.
+    """
+    try:
+        allocator = _thread_local.allocator
+    except AttributeError:
+        _thread_local.allocator = allocator = None
+    if allocator is None:
+        return _current_allocator
+    else:
+        return allocator
+
+
+@cython.final
+@cython.no_gc
+cdef class PooledMemory(BaseMemory):
+
+    """Memory allocation for a memory pool.
+
+    The instance of this class is created by memory pool allocator, so user
+    should not instantiate it by hand.
+
+    """
+
+    cdef:
+        readonly object pool
+
+    def __init__(self, _Chunk chunk, pool):
+        self._init(chunk, pool)
+
+    cdef _init(self, _Chunk chunk, pool):
+        self.ptr = chunk.ptr()
+        self.size = chunk.size
+        self.device_id = chunk.mem.device_id
+        self.pool = pool
+
+    cpdef free(self):
+        """Frees the memory buffer and returns it to the memory pool.
+
+        This function actually does not free the buffer. It just returns the
+        buffer to the memory pool for reuse.
+
+        """
+        cdef intptr_t ptr
+        ptr = self.ptr
+        if ptr == 0:
+            return
+        self.ptr = 0
+        pool = self.pool()
+        if pool is None:
+            return
+
+        size = self.size
+        if memory_hook._has_memory_hooks():
+            hooks = memory_hook.get_memory_hooks()
+            if hooks:
+                device_id = self.device_id
+                pmem_id = id(self)
+
+                for hook in hooks.values():
+                    hook.free_preprocess(device_id=device_id,
+                                         mem_size=size,
+                                         mem_ptr=ptr,
+                                         pmem_id=pmem_id)
+                try:
+                    (<SingleDeviceMemoryPool>pool).free(ptr, size)
+                finally:
+                    for hook in hooks.values():
+                        hook.free_postprocess(device_id=device_id,
+                                              mem_size=size,
+                                              mem_ptr=ptr,
+                                              pmem_id=pmem_id)
+                return
+        (<SingleDeviceMemoryPool>pool).free(ptr, size)
+
+    def __dealloc__(self):
+        if _exit_mode:
+            return  # To avoid error at exit
+        self.free()
+
+
+cdef size_t _index_compaction_threshold = 512
+
+
+# cudaMalloc() is aligned to at least 512 bytes
+# cf. https://gist.github.com/sonots/41daaa6432b1c8b27ef782cd14064269
+DEF ALLOCATION_UNIT_SIZE = 512
+# for test
+_allocation_unit_size = ALLOCATION_UNIT_SIZE
+
+
+cpdef inline size_t _round_size(size_t size):
+    """Rounds up the memory size to fit memory alignment of cudaMalloc."""
+    # avoid 0 div checking
+    size = (size + ALLOCATION_UNIT_SIZE - 1) // ALLOCATION_UNIT_SIZE
+    return size * ALLOCATION_UNIT_SIZE
+
+cpdef size_t _bin_index_from_size(size_t size):
+    """Returns appropriate bins index from the memory size."""
+    # avoid 0 div checking
+    return (size - 1) // ALLOCATION_UNIT_SIZE
+
+
+cdef _gc_isenabled = gc.isenabled
+cdef _gc_disable = gc.disable
+cdef _gc_enable = gc.enable
+
+
+cdef bint _lock_no_gc(lock):
+    """Lock to ensure single thread execution and no garbage collection.
+
+    Returns:
+        bool: Whether GC is disabled.
+    """
+    rlock.lock_fastrlock(lock, -1, True)
+
+    # This function may be called from the context of finalizer
+    # (e.g., `__dealloc__` of PooledMemory class).
+    # If the process is going to be terminated, the module itself may
+    # already been unavailable.
+    if not _exit_mode and _gc_isenabled():
+        _gc_disable()
+        return True
+    return False
+
+
+cdef _unlock_no_gc(lock, bint gc_mode):
+    if gc_mode:
+        _gc_enable()
+    rlock.unlock_fastrlock(lock)
+
+
+cdef class LockAndNoGc:
+    """A context manager that ensures single-thread execution
+    and no garbage collection in the wrapped code.
+    The purpose of disabling GC is to prevent unexpected recursion.
+    See gh-2074 for details.
+    """
+
+    cdef object _lock
+    cdef bint _gc
+
+    def __cinit__(self, lock):
+        self._lock = lock
+
+    def __enter__(self):
+        self._gc = _lock_no_gc(self._lock)
+
+    def __exit__(self, t, v, tb):
+        _unlock_no_gc(self._lock, self._gc)
+
+
+@cython.final
+cdef class _Arena:
+
+    cdef:
+        # `_free_lock` must be acquired to access it.
+        list _free
+        # `_free_lock` must be acquired to access it.
+        vector.vector[size_t] _index
+        # `_free_lock` must be acquired to access it.
+        vector.vector[int8_t] _flag
+
+    def __init__(self):
+        self._free = []
+
+    cdef append_to_free_list(self, _Chunk chunk):
+        # need self._free_lock
+        cdef size_t index, bin_index
+        cdef set free_list
+        cdef vector.vector[size_t].iterator it
+
+        bin_index = _bin_index_from_size(chunk.size)
+        it = algorithm.lower_bound(
+            self._index.begin(), self._index.end(), bin_index)
+        index = <size_t>(it - self._index.begin())
+        if index < self._index.size() and self._index.at(index) == bin_index:
+            free_list = self._free[index]
+            if free_list is None:
+                self._free[index] = free_list = set()
+        else:
+            free_list = set()
+            self._index.insert(self._index.begin() + index, bin_index)
+            self._flag.insert(self._flag.begin() + index, 0)
+            self._free.insert(index, free_list)
+        free_list.add(chunk)
+        self._flag[index] = 1
+
+    cdef bint remove_from_free_list(self, _Chunk chunk):
+        """Removes the chunk from the free list (need self._free_lock).
+
+        Returns:
+            bool: ``True`` if the chunk can successfully be removed from
+            the free list. ``False`` otherwise (e.g., the chunk could not
+            be found in the free list as the chunk is allocated.)
+        """
+
+        cdef size_t index, bin_index
+        cdef set free_list
+        cdef vector.vector[size_t].iterator it
+
+        bin_index = _bin_index_from_size(chunk.size)
+        if self._index.size() == 0:
+            return False
+        it = algorithm.lower_bound(
+            self._index.begin(), self._index.end(), bin_index)
+        index = <size_t>(it - self._index.begin())
+        if index == self._index.size():
+            # Bin does not exist for the given chunk size.
+            return False
+        if self._index.at(index) != bin_index or self._flag.at(index) == 0:
+            return False
+        free_list = self._free[index]
+        if chunk in free_list:
+            free_list.remove(chunk)
+            if len(free_list) == 0:
+                self._free[index] = None
+                self._flag[index] = 0
+            return True
+        return False
+
+
+# cpdef because uint-tested
+# module-level function can be inlined
+cpdef inline dict _parse_limit_string(limit=None):
+    if limit is None:
+        limit = os.environ.get('CUPY_GPU_MEMORY_LIMIT')
+    size = None
+    fraction = None
+    if limit is not None:
+        if limit.endswith('%'):
+            fraction = float(limit[:-1]) / 100.0
+        else:
+            size = int(limit)
+    return {'size': size, 'fraction': fraction}
+
+
+@cython.final
+cdef class SingleDeviceMemoryPool:
+    """Memory pool implementation for single device.
+
+    - The allocator attempts to find the smallest cached block that will fit
+      the requested size. If the block is larger than the requested size,
+      it may be split. If no block is found, the allocator will delegate to
+      cudaMalloc.
+    - If the cudaMalloc fails, the allocator will free all cached blocks that
+      are not split and retry the allocation.
+    """
+
+    cdef:
+        object _allocator
+
+        # Map from memory pointer of the chunk (intptr_t) to the corresponding
+        # Chunk object. All chunks currently allocated to the application from
+        # this pool are stored.
+        # `_in_use_lock` must be acquired to access it.
+        dict _in_use
+
+        # Map from stream identifier to its arena for the stream.
+        # `_free_lock` must be acquired to access it.
+        dict _arenas
+
+        # Number of total bytes actually allocated on GPU.
+        # `_total_bytes_lock` must be acquired to access it.
+        size_t _total_bytes
+
+        # Upper limit of the amount to be allocated by this pool.
+        # `_total_bytes_lock` must be acquired to access it.
+        size_t _total_bytes_limit
+
+        object __weakref__
+        object _weakref
+        object _free_lock
+        object _in_use_lock
+        object _total_bytes_lock
+        readonly int _device_id
+
+    def __init__(self, allocator=None):
+        if allocator is None:
+            allocator = _malloc
+        self._in_use = {}
+        self._arenas = {}
+        self._allocator = allocator
+        self._weakref = weakref.ref(self)
+        self._device_id = device.get_device_id()
+        self._free_lock = rlock.create_fastrlock()
+        self._in_use_lock = rlock.create_fastrlock()
+        self._total_bytes_lock = rlock.create_fastrlock()
+
+        self.set_limit(**(_parse_limit_string()))
+
+    cdef _Arena _arena(self, intptr_t stream_ident):
+        """Returns appropriate arena of a given stream.
+
+        All free chunks in the stream belong to one of the bin in the arena.
+
+        Caller is responsible to acquire `_free_lock`.
+        """
+        ret = self._arenas.get(stream_ident, None)
+        if ret is None:
+            self._arenas[stream_ident] = ret = _Arena()
+        return ret
+
+    cdef MemoryPointer _alloc(self, Py_ssize_t rounded_size):
+        if memory_hook._has_memory_hooks():
+            hooks = memory_hook.get_memory_hooks()
+            if hooks:
+                memptr = None
+                device_id = self._device_id
+                for hook in hooks.values():
+                    hook.alloc_preprocess(device_id=device_id,
+                                          mem_size=rounded_size)
+                try:
+                    memptr = self._allocator(rounded_size)
+                finally:
+                    for hook in hooks.values():
+                        mem_ptr = memptr.ptr if memptr is not None else 0
+                        hook.alloc_postprocess(device_id=device_id,
+                                               mem_size=rounded_size,
+                                               mem_ptr=mem_ptr)
+                return memptr
+        return self._allocator(rounded_size)
+
+    cpdef MemoryPointer malloc(self, size_t size):
+        rounded_size = _round_size(size)
+        if memory_hook._has_memory_hooks():
+            hooks = memory_hook.get_memory_hooks()
+            if hooks:
+                memptr = None
+                device_id = self._device_id
+                for hook in hooks.values():
+                    hook.malloc_preprocess(device_id=device_id,
+                                           size=size,
+                                           mem_size=rounded_size)
+                try:
+                    memptr = self._malloc(rounded_size)
+                finally:
+                    if memptr is None:
+                        mem_ptr = 0
+                        pmem_id = 0
+                    else:
+                        mem_ptr = memptr.ptr
+                        pmem_id = id(memptr.mem)
+                    for hook in hooks.values():
+                        hook.malloc_postprocess(device_id=device_id,
+                                                size=size,
+                                                mem_size=rounded_size,
+                                                mem_ptr=mem_ptr,
+                                                pmem_id=pmem_id)
+                return memptr
+        return self._malloc(rounded_size)
+
+    cdef MemoryPointer _malloc(self, size_t size):
+        cdef _Chunk chunk
+        cdef BaseMemory mem
+        cdef PooledMemory pmem
+        cdef MemoryPointer ret
+        if size == 0:
+            return MemoryPointer(Memory(0), 0)
+
+        stream_ident = _get_stream_identifier(
+            stream_module.get_current_stream_ptr())
+
+        # find best-fit, or a smallest larger allocation
+        gc_mode = _lock_no_gc(self._free_lock)
+        try:
+            chunk = self._get_chunk(size, stream_ident)
+        finally:
+            _unlock_no_gc(self._free_lock, gc_mode)
+
+        if chunk is None:
+            mem = self._try_malloc(size)
+            chunk = _Chunk.__new__(_Chunk)
+            # cudaMalloc if a cache is not found
+            chunk._init(mem, 0, size, stream_ident)
+
+        rlock.lock_fastrlock(self._in_use_lock, -1, True)
+        try:
+            self._in_use[chunk.ptr()] = chunk
+        finally:
+            rlock.unlock_fastrlock(self._in_use_lock)
+        pmem = PooledMemory.__new__(PooledMemory)
+        pmem._init(chunk, self._weakref)
+        ret = MemoryPointer.__new__(MemoryPointer)
+        ret._init(pmem, 0)
+        return ret
+
+    cpdef free(self, intptr_t ptr, size_t size):
+        cdef _Chunk chunk, c
+
+        rlock.lock_fastrlock(self._in_use_lock, -1, True)
+        try:
+            chunk = self._in_use.pop(ptr)
+        except KeyError:
+            raise RuntimeError('Cannot free out-of-pool memory')
+        finally:
+            rlock.unlock_fastrlock(self._in_use_lock)
+        stream_ident = chunk.stream_ident
+
+        gc_mode = _lock_no_gc(self._free_lock)
+        try:
+            arena = self._arena(stream_ident)
+
+            c = chunk.next
+            if c is not None and arena.remove_from_free_list(c):
+                chunk.merge(c)
+
+            c = chunk.prev
+            if c is not None and arena.remove_from_free_list(c):
+                c.merge(chunk)
+                chunk = c
+
+            arena.append_to_free_list(chunk)
+        finally:
+            _unlock_no_gc(self._free_lock, gc_mode)
+
+    cpdef free_all_blocks(self, stream=None):
+        """Free all **non-split** chunks"""
+        cdef intptr_t stream_ident
+
+        with LockAndNoGc(self._free_lock):
+            # free blocks in all arenas
+            if stream is None:
+                for stream_ident in list(self._arenas.iterkeys()):
+                    self._compact_index(stream_ident, True)
+            else:
+                self._compact_index(_get_stream_identifier(stream.ptr), True)
+
+    cpdef free_all_free(self):
+        warnings.warn(
+            'free_all_free is deprecated. Use free_all_blocks instead.',
+            DeprecationWarning)
+        self.free_all_blocks()
+
+    cpdef size_t n_free_blocks(self):
+        cdef size_t n = 0
+        cdef _Arena arena
+        rlock.lock_fastrlock(self._free_lock, -1, True)
+        try:
+            for arena in self._arenas.itervalues():
+                for v in arena._free:
+                    if v is not None:
+                        n += len(v)
+        finally:
+            rlock.unlock_fastrlock(self._free_lock)
+        return n
+
+    cpdef size_t used_bytes(self):
+        cdef size_t size = 0
+        cdef _Chunk chunk
+        rlock.lock_fastrlock(self._in_use_lock, -1, True)
+        try:
+            for chunk in self._in_use.itervalues():
+                size += chunk.size
+        finally:
+            rlock.unlock_fastrlock(self._in_use_lock)
+        return size
+
+    cpdef size_t free_bytes(self):
+        cdef size_t size = 0
+        cdef set free_list
+        cdef _Chunk chunk
+        cdef _Arena arena
+        rlock.lock_fastrlock(self._free_lock, -1, True)
+        try:
+            for arena in self._arenas.itervalues():
+                for free_list in arena._free:
+                    if free_list is None:
+                        continue
+                    for chunk in free_list:
+                        size += chunk.size
+        finally:
+            rlock.unlock_fastrlock(self._free_lock)
+        return size
+
+    cpdef size_t total_bytes(self):
+        with LockAndNoGc(self._total_bytes_lock):
+            return self._total_bytes
+
+    cpdef set_limit(self, size=None, fraction=None):
+        if size is None:
+            if fraction is None:
+                size = 0
+            else:
+                if not 0 <= fraction <= 1:
+                    raise ValueError(
+                        'memory limit fraction out of range: {}'.format(
+                            fraction))
+                _, total = runtime.memGetInfo()
+                size = fraction * total
+            self.set_limit(size=size)
+            return
+
+        if fraction is not None:
+            raise ValueError('size and fraction cannot be specified at '
+                             'one time')
+        if size < 0:
+            raise ValueError(
+                'memory limit size out of range: {}'.format(size))
+
+        with LockAndNoGc(self._total_bytes_lock):
+            self._total_bytes_limit = size
+
+    cpdef size_t get_limit(self):
+        with LockAndNoGc(self._total_bytes_lock):
+            return self._total_bytes_limit
+
+    cdef _compact_index(self, intptr_t stream_ident, bint free):
+        # need self._free_lock
+        cdef _Arena arena
+        cdef list new_free
+        cdef set free_list, keep_list
+        cdef vector.vector[size_t] new_index
+        cdef size_t index
+        cdef size_t size_to_free = 0
+
+        if stream_ident not in self._arenas:
+            return
+        new_free = []
+        arena = self._arenas[stream_ident]
+
+        for index, free_list in enumerate(arena._free):
+            if not free_list:
+                continue
+            if free:
+                keep_list = set()
+                for chunk in free_list:
+                    if chunk.prev is not None or chunk.next is not None:
+                        keep_list.add(chunk)
+                    else:
+                        size_to_free += chunk.size
+                if len(keep_list) == 0:
+                    continue
+                free_list = keep_list
+
+            new_index.push_back(arena._index.at(index))
+            new_free.append(free_list)
+        if free and len(new_free) == 0:
+            del self._arenas[stream_ident]
+        else:
+            arena._free = new_free
+            arena._index.swap(new_index)
+            arena._flag.assign(new_index.size(), <int8_t>1)
+        if size_to_free > 0:
+            with LockAndNoGc(self._total_bytes_lock):
+                self._total_bytes -= size_to_free
+
+    cdef object _get_chunk(self, size_t size, intptr_t stream_ident):
+        # need self._free_lock
+        cdef set free_list
+        cdef size_t i, index, length
+        cdef _Chunk chunk
+        cdef size_t bin_index = _bin_index_from_size(size)
+        cdef _Arena a = self._arena(stream_ident)
+        index = <size_t>(
+            algorithm.lower_bound(a._index.begin(), a._index.end(), bin_index)
+            - a._index.begin())
+        length = a._index.size()
+        for i in range(index, length):
+            if a._flag.at(i) == 0:
+                continue
+            free_list = a._free[i]
+            chunk = free_list.pop()
+            if len(free_list) == 0:
+                a._flag[i] = 0
+                a._free[i] = None
+            if i - index >= _index_compaction_threshold:
+                self._compact_index(stream_ident, False)
+            remaining = chunk.split(size)
+            if remaining is not None:
+                a.append_to_free_list(remaining)
+            assert chunk.stream_ident == stream_ident
+            return chunk
+        return None
+
+    cdef BaseMemory _try_malloc(self, size_t size):
+        cdef size_t limit
+        cdef bint limit_ok
+        with LockAndNoGc(self._total_bytes_lock):
+            limit = self._total_bytes_limit
+            if limit != 0:
+                limit_ok = (self._total_bytes + size) <= limit
+                if not limit_ok:
+                    self.free_all_blocks()
+                    limit_ok = (self._total_bytes + size) <= limit
+                if not limit_ok:
+                    gc.collect()
+                    self.free_all_blocks()
+                    limit_ok = (self._total_bytes + size) <= limit
+                if not limit_ok:
+                    raise OutOfMemoryError(size, self._total_bytes, limit)
+            self._total_bytes += size
+
+        mem = None
+        oom_error = False
+        try:
+            mem = self._alloc(size).mem
+        except CUDARuntimeError as e:
+            if e.status != runtime.errorMemoryAllocation:
+                raise
+            self.free_all_blocks()
+            try:
+                mem = self._alloc(size).mem
+            except CUDARuntimeError as e:
+                if e.status != runtime.errorMemoryAllocation:
+                    raise
+                gc.collect()
+                self.free_all_blocks()
+                try:
+                    mem = self._alloc(size).mem
+                except CUDARuntimeError as e:
+                    if e.status != runtime.errorMemoryAllocation:
+                        raise
+                    oom_error = True
+        finally:
+            if mem is None:
+                with LockAndNoGc(self._total_bytes_lock):
+                    self._total_bytes -= size
+                    if oom_error:
+                        raise OutOfMemoryError(size, self._total_bytes, limit)
+
+        return mem
+
+
+cdef class MemoryPool:
+
+    """Memory pool for all GPU devices on the host.
+
+    A memory pool preserves any allocations even if they are freed by the user.
+    Freed memory buffers are held by the memory pool as *free blocks*, and they
+    are reused for further memory allocations of the same sizes. The allocated
+    blocks are managed for each device, so one instance of this class can be
+    used for multiple devices.
+
+    .. note::
+       When the allocation is skipped by reusing the pre-allocated block, it
+       does not call ``cudaMalloc`` and therefore CPU-GPU synchronization does
+       not occur. It makes interleaves of memory allocations and kernel
+       invocations very fast.
+
+    .. note::
+       The memory pool holds allocated blocks without freeing as much as
+       possible. It makes the program hold most of the device memory, which may
+       make other CUDA programs running in parallel out-of-memory situation.
+
+    Args:
+        allocator (function): The base CuPy memory allocator. It is used for
+            allocating new blocks when the blocks of the required size are all
+            in use.
+
+    """
+
+    def __init__(self, allocator=None):
+        if allocator is None:
+            allocator = _malloc
+        self._pools = collections.defaultdict(
+            lambda: SingleDeviceMemoryPool(allocator))
+
+    cpdef MemoryPointer malloc(self, size_t size):
+        """Allocates the memory, from the pool if possible.
+
+        This method can be used as a CuPy memory allocator. The simplest way to
+        use a memory pool as the default allocator is the following code::
+
+            set_allocator(MemoryPool().malloc)
+
+        Also, the way to use a memory pool of Managed memory (Unified memory)
+        as the default allocator is the following code::
+
+            set_allocator(MemoryPool(malloc_managed).malloc)
+
+        Args:
+            size (int): Size of the memory buffer to allocate in bytes.
+
+        Returns:
+            ~cupy.cuda.MemoryPointer: Pointer to the allocated buffer.
+
+        """
+        mp = <SingleDeviceMemoryPool>self._pools[device.get_device_id()]
+        return mp.malloc(size)
+
+    cpdef free_all_blocks(self, stream=None):
+        """Releases free blocks.
+
+        Args:
+            stream (cupy.cuda.Stream): Release free blocks in the arena
+                of the given stream. The default releases blocks in all
+                arenas.
+
+        .. note::
+            A memory pool may split a free block for space efficiency. A split
+            block is not released until all its parts are merged back into one
+            even if :meth:`free_all_blocks` is called.
+        """
+        mp = <SingleDeviceMemoryPool>self._pools[device.get_device_id()]
+        mp.free_all_blocks(stream=stream)
+
+    cpdef free_all_free(self):
+        """(Deprecated) Use :meth:`free_all_blocks` instead."""
+        warnings.warn(
+            'free_all_free is deprecated. Use free_all_blocks instead.',
+            DeprecationWarning)
+        self.free_all_blocks()
+
+    cpdef size_t n_free_blocks(self):
+        """Counts the total number of free blocks.
+
+        Returns:
+            int: The total number of free blocks.
+        """
+        mp = <SingleDeviceMemoryPool>self._pools[device.get_device_id()]
+        return mp.n_free_blocks()
+
+    cpdef size_t used_bytes(self):
+        """Gets the total number of bytes used by the pool.
+
+        Returns:
+            int: The total number of bytes used by the pool.
+        """
+        mp = <SingleDeviceMemoryPool>self._pools[device.get_device_id()]
+        return mp.used_bytes()
+
+    cpdef size_t free_bytes(self):
+        """Gets the total number of bytes acquired but not used by the pool.
+
+        Returns:
+            int: The total number of bytes acquired but not used by the pool.
+        """
+        mp = <SingleDeviceMemoryPool>self._pools[device.get_device_id()]
+        return mp.free_bytes()
+
+    cpdef size_t total_bytes(self):
+        """Gets the total number of bytes acquired by the pool.
+
+        Returns:
+            int: The total number of bytes acquired by the pool.
+        """
+        mp = <SingleDeviceMemoryPool>self._pools[device.get_device_id()]
+        return mp.total_bytes()
+
+    cpdef set_limit(self, size=None, fraction=None):
+        """Sets the upper limit of memory allocation of the current device.
+
+        When `fraction` is specified, its value will become a fraction of the
+        amount of GPU memory that is available for allocation.
+        For example, if you have a GPU with 2 GiB memory, you can either use
+        ``set_limit(fraction=0.5)`` or ``set_limit(size=1024**3)`` to limit
+        the memory size to 1 GiB.
+
+        ``size`` and ``fraction`` cannot be specified at the same time.
+        If both of them are **not** specified or ``0`` is specified, the
+        limit will be disabled.
+
+        .. note::
+            You can also set the limit by using ``CUPY_GPU_MEMORY_LIMIT``
+            environment variable, see :ref:`environment` for the details.
+            The limit set by this method supersedes the value specified in
+            the environment variable.
+
+            Also note that this method only changes the limit for the current
+            device, whereas the environment variable sets the default limit for
+            all devices.
+
+        Args:
+            size (int): Limit size in bytes.
+            fraction (float): Fraction in the range of ``[0, 1]``.
+        """
+        mp = <SingleDeviceMemoryPool>self._pools[device.get_device_id()]
+        mp.set_limit(size, fraction)
+
+    cpdef size_t get_limit(self):
+        """Gets the upper limit of memory allocation of the current device.
+
+        Returns:
+            int: The number of bytes
+        """
+        mp = <SingleDeviceMemoryPool>self._pools[device.get_device_id()]
+        return mp.get_limit()
+
+
+cdef class MemoryAsyncPool:
+    """(Experimental) CUDA memory pool for all GPU devices on the host.
+
+    A memory pool preserves any allocations even if they are freed by the user.
+    One instance of this class can be used for multiple devices. This class
+    uses CUDA's Stream Ordered Memory Allocator (supported on CUDA 11.2+).
+    The simplest way to use this pool as CuPy's default allocator is the
+    following code::
+
+        set_allocator(MemoryAsyncPool().malloc)
+
+    Using this feature requires CUDA >= 11.2 with a supported GPU and platform.
+    If it is not supported, an error will be raised.
+
+    The current CuPy stream is used to allocate/free the memory.
+
+    Args:
+        pool_handles (str or int): A flag to indicate which mempool to use.
+            `'default'` is for the device's default mempool, `'current'` is for
+            the current mempool (which could be the default one), and an `int`
+            that represents ``cudaMemPool_t`` created from elsewhere for an
+            external mempool. A list consisting of these flags can also be
+            accepted, in which case the list length must equal to the total
+            number of visible devices so that the mempools for each device can
+            be set independently.
+
+    .. warning::
+        This feature is currently experimental and subject to change.
+
+    .. note::
+        :class:`MemoryAsyncPool` currently cannot work with memory hooks.
+
+    .. seealso:: `Stream Ordered Memory Allocator`_
+
+    .. _Stream Ordered Memory Allocator:
+        https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#stream-ordered-memory-allocator
+    """
+    # This is an analogous to SingleDeviceMemoryPool + MemoryPool, but for
+    # CUDA's async allocator. The main purpose is to provide a memory pool
+    # interface for multiple devices. Given that CUDA's mempool is implemented
+    # at the driver level, the same pool can be shared by many applications
+    # in the same process.
+    #
+    # Internally (as of driver v11.3) the pool starts with size 0. The first
+    # allocation will bump the size to 32n MiB to accommodate the requested
+    # amount (n is integer). The size is increased only if it is not enough
+    # to meet later allocation needs. The size is decreased to 32m MiB (m<n)
+    # if enough memory is returned (freed) to the pool when free_all_blocks()
+    # (that is, sync + cudaMemPoolTrimTo) is called. This observation may
+    # vary with future driver updates, so the MemoryAsyncPool API does not
+    # rely on any internal behavior, but only on the Programming Guide and
+    # sane assumptions.
+
+    cdef:
+        # A list of cudaMemPool_t to each device's mempool
+        readonly list _pools
+        readonly bint memoryAsyncHasStat
+
+    def __init__(self, pool_handles='current'):
+        _util.experimental('cupy.cuda.MemoryAsyncPool')
+        cdef int dev_id, prev_dev_id, dev_counts
+        cdef dict limit = _parse_limit_string()
+        dev_counts = runtime.getDeviceCount()
+        self._pools = []
+        self.memoryAsyncHasStat = (runtime.driverGetVersion() >= 11030)
+        prev_dev_id = runtime.getDevice()
+        if (cpython.PySequence_Check(pool_handles)
+                and not isinstance(pool_handles, str)):
+            # allow different kinds of handles on each device
+            for dev_id in range(dev_counts):
+                try:
+                    runtime.setDevice(dev_id)
+                    self._pools.append(self.set_pool(
+                        pool_handles[dev_id], dev_id))
+                    self.set_limit(**limit)
+                finally:
+                    runtime.setDevice(prev_dev_id)
+        else:
+            # use the same argument for all devices
+            for dev_id in range(dev_counts):
+                try:
+                    runtime.setDevice(dev_id)
+                    self._pools.append(self.set_pool(pool_handles, dev_id))
+                    self.set_limit(**limit)
+                finally:
+                    runtime.setDevice(prev_dev_id)
+
+    cdef intptr_t set_pool(self, handle, int dev_id) except? 0:
+        cdef intptr_t pool
+        if handle == 'default':
+            # Use the device's default pool
+            pool = runtime.deviceGetDefaultMemPool(dev_id)
+        elif handle == 'current':
+            # Use the device's current pool
+            pool = runtime.deviceGetMemPool(dev_id)
+        elif handle == 'create':
+            # TODO(leofang): Support cudaMemPoolCreate
+            raise NotImplementedError('cudaMemPoolCreate is not yet supported')
+        elif isinstance(handle, int):
+            # Use an existing pool (likely from other applications?)
+            pool = <intptr_t>(handle)
+        else:
+            raise ValueError("handle must be "
+                             "'default' (for the device's default pool), "
+                             "'current' (for the device's current pool), "
+                             "or int (a pointer to cudaMemPool_t)")
+        runtime.deviceSetMemPool(dev_id, pool)
+        return pool
+
+    cpdef MemoryPointer malloc(self, size_t size):
+        """Allocate memory from the current device's pool on the current
+        stream.
+
+        This method can be used as a CuPy memory allocator. The simplest way to
+        use a memory pool as the default allocator is the following code::
+
+            set_allocator(MemoryAsyncPool().malloc)
+
+        Args:
+            size (int): Size of the memory buffer to allocate in bytes.
+
+        Returns:
+            ~cupy.cuda.MemoryPointer: Pointer to the allocated buffer.
+
+        """
+        cdef size_t rounded_size = _round_size(size)
+        mem = None
+        oom_error = False
+
+        # CUDA does not allow us to set a hard limit, so the best we can do is
+        # to prevent CuPy from drawing too much memory from the pool; we cannot
+        # do anything if other applications oversubscribe the pool.
+        cdef size_t curr_total=-1, curr_free=0, total_limit=0
+        if self.memoryAsyncHasStat:
+            curr_total = self.total_bytes()
+            curr_free = curr_total - self.used_bytes()
+            if curr_free < rounded_size:  # need to increase pool size
+                total_limit = self.get_limit()
+                if max(total_limit, curr_total) < curr_total + rounded_size:
+                    raise OutOfMemoryError(size, curr_total, total_limit)
+
+        try:
+            mem = malloc_async(rounded_size)
+        except CUDARuntimeError as e:
+            if e.status != runtime.errorMemoryAllocation:
+                raise
+            stream = stream_module.get_current_stream()
+            stream.synchronize()
+            try:
+                mem = malloc_async(rounded_size)
+            except CUDARuntimeError as e:
+                if e.status != runtime.errorMemoryAllocation:
+                    raise
+                stream.synchronize()
+                try:
+                    mem = malloc_async(rounded_size)
+                except CUDARuntimeError as e:
+                    if e.status != runtime.errorMemoryAllocation:
+                        raise
+                    oom_error = True
+        finally:
+            if mem is None:
+                assert oom_error
+                # Set total to -1 as we do not have access to the mempool usage
+                raise OutOfMemoryError(size, curr_total, total_limit)
+        return mem
+
+    cpdef free_all_blocks(self, stream=None):
+        """Releases free memory.
+
+        Args:
+            stream (cupy.cuda.Stream): Release memory freed on the given
+                ``stream``. If ``stream`` is ``None``, the current stream is
+                used.
+
+        .. seealso:: `Physical Page Caching Behavior`_
+
+        .. _Physical Page Caching Behavior:
+            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#stream-ordered-physical-page-caching-behavior
+        """
+        # We don't have access to the mempool internal, but if there are
+        # any memory asynchronously freed, a synchonization will make sure
+        # they become visible (to both cudaMalloc and cudaMallocAsync). See
+        # https://github.com/cupy/cupy/issues/3777#issuecomment-758890450
+        if stream is None:
+            stream = stream_module.get_current_stream()
+        stream.synchronize()
+        cdef intptr_t pool = self._pools[device.get_device_id()]
+        # We don't care the actual limit; putting 0 here means we guarantee
+        # to reserve at least 0 bytes
+        runtime.memPoolTrimTo(pool, 0)
+
+    cpdef size_t n_free_blocks(self):
+        raise NotImplementedError(
+            'This function is not supported in MemoryAsyncPool')
+
+    cpdef size_t used_bytes(self) except*:
+        """Gets the total number of bytes used by the pool.
+
+        Returns:
+            int: The total number of bytes used by the pool.
+        """
+        if not self.memoryAsyncHasStat:
+            raise RuntimeError(
+                'The driver version is insufficient for this query')
+        cdef intptr_t pool = self._pools[device.get_device_id()]
+        return runtime.memPoolGetAttribute(
+            pool, runtime.cudaMemPoolAttrUsedMemCurrent)
+
+    cpdef size_t free_bytes(self) except*:
+        """Gets the total number of bytes acquired but not used by the pool.
+
+        Returns:
+            int: The total number of bytes acquired but not used by the pool.
+        """
+        return self.total_bytes() - self.used_bytes()
+
+    cpdef size_t total_bytes(self) except*:
+        """Gets the total number of bytes acquired by the pool.
+
+        Returns:
+            int: The total number of bytes acquired by the pool.
+        """
+        if not self.memoryAsyncHasStat:
+            raise RuntimeError(
+                'The driver version is insufficient for this query')
+        cdef intptr_t pool = self._pools[device.get_device_id()]
+        return runtime.memPoolGetAttribute(
+            pool, runtime.cudaMemPoolAttrReservedMemCurrent)
+
+    cpdef set_limit(self, size=None, fraction=None):
+        """Sets the upper limit of memory allocation of the current device.
+
+        When `fraction` is specified, its value will become a fraction of the
+        amount of GPU memory that is available for allocation.
+        For example, if you have a GPU with 2 GiB memory, you can either use
+        ``set_limit(fraction=0.5)`` or ``set_limit(size=1024**3)`` to limit
+        the memory size to 1 GiB.
+
+        ``size`` and ``fraction`` cannot be specified at the same time.
+        If both of them are **not** specified or ``0`` is specified, the
+        limit will be disabled.
+
+        .. note::
+            Unlike with :class:`MemoryPool`, :class:`MemoryAsyncPool`'s
+            :meth:`set_limit` method can only impose a *soft* limit. If other
+            (non-CuPy) applications are also allocating memory from the same
+            mempool, this limit may not be respected. Internally, this limit
+            is set via the ``cudaMemPoolAttrReleaseThreshold`` attribute.
+
+        .. note::
+            You can also set the limit by using ``CUPY_GPU_MEMORY_LIMIT``
+            environment variable, see :ref:`environment` for the details.
+            The limit set by this method supersedes the value specified in
+            the environment variable.
+
+            Also note that this method only changes the limit for the current
+            device, whereas the environment variable sets the default limit for
+            all devices.
+
+        Args:
+            size (int): Limit size in bytes.
+            fraction (float): Fraction in the range of ``[0, 1]``.
+        """
+        if size is None:
+            if fraction is None:
+                size = 0
+            else:
+                if not 0 <= fraction <= 1:
+                    raise ValueError(
+                        'memory limit fraction out of range: {}'.format(
+                            fraction))
+                _, total = runtime.memGetInfo()
+                size = fraction * total
+            self.set_limit(size=size)
+            return
+
+        if fraction is not None:
+            raise ValueError('size and fraction cannot be specified at '
+                             'one time')
+        if size < 0:
+            raise ValueError(
+                'memory limit size out of range: {}'.format(size))
+
+        if size == 0:
+            size = UINT64_MAX  # ensure pool size is never shrunk
+        cdef intptr_t pool = self._pools[device.get_device_id()]
+        runtime.memPoolSetAttribute(
+            pool, runtime.cudaMemPoolAttrReleaseThreshold, size)
+
+    cpdef size_t get_limit(self):
+        """Gets the upper limit of memory allocation of the current device.
+
+        Returns:
+            int: The number of bytes
+
+        .. note::
+            Unlike with :class:`MemoryPool`, :class:`MemoryAsyncPool`'s
+            :meth:`set_limit` method can only impose a *soft* limit. If other
+            (non-CuPy) applications are also allocating memory from the same
+            mempool, this limit may not be respected.
+        """
+        cdef intptr_t pool = self._pools[device.get_device_id()]
+        return runtime.memPoolGetAttribute(
+            pool, runtime.cudaMemPoolAttrReleaseThreshold)
+
+
+ctypedef void*(*malloc_func_type)(void*, size_t, int)
+ctypedef void(*free_func_type)(void*, void*, int)
+
+
+cdef intptr_t _call_malloc(
+        intptr_t param, intptr_t malloc_func, Py_ssize_t size, int device_id):
+    return <intptr_t>(
+        (<malloc_func_type>malloc_func)(<void*>param, size, device_id))
+
+
+cdef void _call_free(
+        intptr_t param, intptr_t free_func, intptr_t ptr, int device_id):
+    (<free_func_type>free_func)(<void*>param, <void*>ptr, device_id)
+
+
+@cython.no_gc
+cdef class CFunctionAllocatorMemory(BaseMemory):
+
+    def __init__(self, size_t size, intptr_t param,
+                 intptr_t malloc_func, intptr_t free_func,
+                 int device_id):
+        self._param = param
+        self._free_func = free_func
+        self.device_id = device_id
+        self.size = size
+        self.ptr = 0
+        if size > 0:
+            self.ptr = _call_malloc(param, malloc_func, size, device_id)
+
+    def __dealloc__(self):
+        if self.ptr:
+            _call_free(self._param, self._free_func, self.ptr, self.device_id)
+
+
+cdef class CFunctionAllocator:
+
+    """Allocator with C function pointers to allocation routines.
+
+    This allocator keeps raw pointers to a *param* object along with functions
+    pointers to *malloc* and *free*, delegating the actual allocation to
+    external sources while only handling the timing of the resource allocation
+    and deallocation.
+
+    *malloc* should follow the signature ``void*(*malloc)(void*, size_t, int)``
+    returning the pointer to the allocated memory given the pointer to
+    *param*, the number of bytes to allocate and the device id on which the
+    allocation should take place.
+
+    Similarly, *free* should follow the signature
+    ``void(*free)(void*, void*, int)`` with no return, taking the pointer to
+    *param*, the pointer to the allocated memory and the device id on which the
+    memory was allocated.
+
+    Args:
+        param (int): Address of *param*.
+        malloc_func (int): Address of *malloc*.
+        free_func (int): Address of *free*.
+        owner (object): Reference to the owner object to keep the param and
+            the functions alive.
+
+    """
+
+    def __init__(self, intptr_t param, intptr_t malloc_func,
+                 intptr_t free_func, object owner):
+        self._param = param
+        self._malloc_func = malloc_func
+        self._free_func = free_func
+        self._owner = owner
+
+    cpdef MemoryPointer malloc(self, size_t size):
+        mem = CFunctionAllocatorMemory(size, self._param, self._malloc_func,
+                                       self._free_func, device.get_device_id())
+        return MemoryPointer(mem, 0)
+
+
+cdef class PythonFunctionAllocatorMemory(BaseMemory):
+
+    def __init__(self, size_t size, malloc_func, free_func,
+                 int device_id):
+        self._free_func = free_func
+        self.device_id = device_id
+        self.size = size
+        self.ptr = 0
+        if size > 0:
+            self.ptr = malloc_func(size, device_id)
+
+    def __dealloc__(self):
+        if self.ptr:
+            self._free_func(self.ptr, self.device_id)
+
+
+cdef class PythonFunctionAllocator:
+
+    """Allocator with python functions to perform memory allocation.
+
+    This allocator keeps functions corresponding to *malloc* and *free*,
+    delegating the actual allocation to external sources while only
+    handling the timing of the resource allocation and deallocation.
+
+    *malloc* should follow the signature ``malloc(int, int) -> int``
+    returning the pointer to the allocated memory given the *param* object,
+    the number of bytes to allocate and the device id on which the
+    allocation should take place.
+
+    Similarly, *free* should follow the signature
+    ``free(int, int)`` with no return, taking the pointer to the
+    allocated memory and the device id on which the memory was allocated.
+
+    If the external memory management supports asynchronous operations,
+    the current CuPy stream can be retrieved inside ``malloc_func`` and
+    ``free_func`` by calling :func:`cupy.cuda.get_current_stream()`. To
+    use external streams, wrap them with :func:`cupy.cuda.ExternalStream`.
+
+    Args:
+        malloc_func (function): *malloc* function to be called.
+        free_func (function): *free* function to be called.
+
+    """
+
+    def __init__(self, malloc_func, free_func):
+        self._malloc_func = malloc_func
+        self._free_func = free_func
+
+    cpdef MemoryPointer malloc(self, size_t size):
+        mem = PythonFunctionAllocatorMemory(
+            size, self._malloc_func,
+            self._free_func, device.get_device_id())
+        return MemoryPointer(mem, 0)
diff --git a/cupy/cuda/memory_hook.pxd b/cupy/cuda/memory_hook.pxd
new file mode 100644
index 0000000..6a01543
--- /dev/null
+++ b/cupy/cuda/memory_hook.pxd
@@ -0,0 +1,2 @@
+cpdef bint _has_memory_hooks()
+cpdef get_memory_hooks()
diff --git a/cupy/cuda/memory_hook.pyx b/cupy/cuda/memory_hook.pyx
new file mode 100644
index 0000000..b6cfc05
--- /dev/null
+++ b/cupy/cuda/memory_hook.pyx
@@ -0,0 +1,178 @@
+import collections
+import threading
+
+cdef object _thread_local = threading.local()
+
+
+cdef class _ThreadLocal:
+
+    cdef object memory_hooks
+
+    def __init__(self):
+        self.memory_hooks = None
+
+    @staticmethod
+    cdef _ThreadLocal get():
+        try:
+            tls = _thread_local.tls
+        except AttributeError:
+            tls = _thread_local.tls = _ThreadLocal()
+        return <_ThreadLocal>tls
+
+
+cpdef bint _has_memory_hooks():
+    tls = _ThreadLocal.get()
+    return tls.memory_hooks is not None
+
+
+cpdef get_memory_hooks():
+    tls = _ThreadLocal.get()
+    if tls.memory_hooks is None:
+        tls.memory_hooks = collections.OrderedDict()
+    return tls.memory_hooks
+
+
+class MemoryHook(object):
+    """Base class of hooks for Memory allocations.
+
+    :class:`~cupy.cuda.MemoryHook` is an callback object.
+    Registered memory hooks are invoked before and after
+    memory is allocated from GPU device, and
+    memory is retrieved from memory pool, and
+    memory is released to memory pool.
+
+    Memory hooks that derive :class:`MemoryHook` are required
+    to implement six methods:
+    :meth:`~cupy.cuda.MemoryHook.alloc_preprocess`,
+    :meth:`~cupy.cuda.MemoryHook.alloc_postprocess`,
+    :meth:`~cupy.cuda.MemoryHook.malloc_preprocess`,
+    :meth:`~cupy.cuda.MemoryHook.malloc_postprocess`,
+    :meth:`~cupy.cuda.MemoryHook.free_preprocess`, and
+    :meth:`~cupy.cuda.MemoryHook.free_postprocess`,
+    By default, these methods do nothing.
+
+    Specifically, :meth:`~cupy.cuda.MemoryHook.alloc_preprocess`
+    (resp. :meth:`~cupy.cuda.MemoryHook.alloc_postprocess`)
+    of all memory hooks registered are called before (resp. after)
+    memory is allocated from GPU device.
+
+    Likewise, :meth:`~cupy.cuda.MemoryHook.malloc_preprocess`
+    (resp. :meth:`~cupy.cuda.MemoryHook.malloc_postprocess`)
+    of all memory hooks registered are called before (resp. after)
+    memory is retrieved from memory pool.
+
+    Below is a pseudo code to descirbe how malloc and hooks work.
+    Please note that :meth:`~cupy.cuda.MemoryHook.alloc_preprocess` and
+    :meth:`~cupy.cuda.MemoryHook.alloc_postprocess` are not invoked if a cached
+    free chunk is found::
+
+        def malloc(size):
+            Call malloc_preprocess of all memory hooks
+            Try to find a cached free chunk from memory pool
+            if chunk is not found:
+                Call alloc_preprocess for all memory hooks
+                Invoke actual memory allocation to get a new chunk
+                Call alloc_postprocess for all memory hooks
+            Call malloc_postprocess for all memory hooks
+
+    Moreover, :meth:`~cupy.cuda.MemoryHook.free_preprocess`
+    (resp. :meth:`~cupy.cuda.MemoryHook.free_postprocess`)
+    of all memory hooks registered are called before (resp. after)
+    memory is released to memory pool.
+
+    Below is a pseudo code to descirbe how free and hooks work::
+
+        def free(ptr):
+            Call free_preprocess of all memory hooks
+            Push a memory chunk of a given pointer back to memory pool
+            Call free_postprocess for all memory hooks
+
+    To register a memory hook, use ``with`` statement. Memory hooks
+    are registered to all method calls within ``with`` statement
+    and are unregistered at the end of ``with`` statement.
+
+    .. note::
+
+       CuPy stores the dictionary of registered function hooks
+       as a thread local object. So, memory hooks registered
+       can be different depending on threads.
+    """
+
+    name = 'MemoryHook'
+
+    def __enter__(self):
+        memory_hooks = get_memory_hooks()
+        if self.name in memory_hooks:
+            raise KeyError('memory hook %s already exists' % self.name)
+
+        memory_hooks[self.name] = self
+        return self
+
+    def __exit__(self, *_):
+        del get_memory_hooks()[self.name]
+
+    def alloc_preprocess(self, **kwargs):
+        """Callback function invoked before allocating memory from GPU device.
+
+        Keyword Args:
+            device_id(int): CUDA device ID
+            mem_size(int): Rounded memory bytesize to be allocated
+        """
+        pass
+
+    def alloc_postprocess(self, **kwargs):
+        """Callback function invoked after allocating memory from GPU device.
+
+        Keyword Args:
+            device_id(int): CUDA device ID
+            mem_size(int): Rounded memory bytesize allocated
+            mem_ptr(int): Obtained memory pointer.
+                0 if an error occurred in allocation.
+        """
+        pass
+
+    def malloc_preprocess(self, **kwargs):
+        """Callback function invoked before retrieving memory from memory pool.
+
+        Keyword Args:
+            device_id(int): CUDA device ID
+            size(int): Requested memory bytesize to allocate
+            mem_size(int): Rounded memory bytesize to be allocated
+        """
+        pass
+
+    def malloc_postprocess(self, **kwargs):
+        """Callback function invoked after retrieving memory from memory pool.
+
+        Keyword Args:
+            device_id(int): CUDA device ID
+            size(int): Requested memory bytesize to allocate
+            mem_size(int): Rounded memory bytesize allocated
+            mem_ptr(int): Obtained memory pointer.
+                0 if an error occurred in ``malloc``.
+            pmem_id(int): Pooled memory object ID.
+                0 if an error occurred in ``malloc``.
+        """
+        pass
+
+    def free_preprocess(self, **kwargs):
+        """Callback function invoked before releasing memory to memory pool.
+
+        Keyword Args:
+            device_id(int): CUDA device ID
+            mem_size(int): Memory bytesize
+            mem_ptr(int): Memory pointer to free
+            pmem_id(int): Pooled memory object ID.
+        """
+        pass
+
+    def free_postprocess(self, **kwargs):
+        """Callback function invoked after releasing memory to memory pool.
+
+        Keyword Args:
+            device_id(int): CUDA device ID
+            mem_size(int): Memory bytesize
+            mem_ptr(int): Memory pointer to free
+            pmem_id(int): Pooled memory object ID.
+        """
+        pass
diff --git a/cupy/cuda/memory_hooks/__init__.py b/cupy/cuda/memory_hooks/__init__.py
new file mode 100644
index 0000000..94036ce
--- /dev/null
+++ b/cupy/cuda/memory_hooks/__init__.py
@@ -0,0 +1,6 @@
+from cupy.cuda.memory_hooks import debug_print  # NOQA
+from cupy.cuda.memory_hooks import line_profile  # NOQA
+
+# import class and function
+from cupy.cuda.memory_hooks.debug_print import DebugPrintHook  # NOQA
+from cupy.cuda.memory_hooks.line_profile import LineProfileHook  # NOQA
diff --git a/cupy/cuda/memory_hooks/debug_print.py b/cupy/cuda/memory_hooks/debug_print.py
new file mode 100644
index 0000000..19d711e
--- /dev/null
+++ b/cupy/cuda/memory_hooks/debug_print.py
@@ -0,0 +1,78 @@
+import sys
+
+from cupy.cuda import memory_hook
+
+
+class DebugPrintHook(memory_hook.MemoryHook):
+    """Memory hook that prints debug information.
+
+    This memory hook outputs the debug information of input arguments of
+    ``malloc`` and ``free`` methods involved in the hooked functions
+    at postprocessing time (that is, just after each method is called).
+
+    Example:
+        The basic usage is to use it with ``with`` statement.
+
+        Code example::
+
+            >>> import cupy
+            >>> from cupy.cuda import memory_hooks
+            >>>
+            >>> cupy.cuda.set_allocator(cupy.cuda.MemoryPool().malloc)
+            >>> with memory_hooks.DebugPrintHook():
+            ...     x = cupy.array([1, 2, 3])
+            ...     del x  # doctest:+SKIP
+
+        Output example::
+
+            {"hook":"alloc","device_id":0,"mem_size":512,"mem_ptr":150496608256}
+            {"hook":"malloc","device_id":0,"size":24,"mem_size":512,"mem_ptr":150496608256,"pmem_id":"0x7f39200c5278"}
+            {"hook":"free","device_id":0,"mem_size":512,"mem_ptr":150496608256,"pmem_id":"0x7f39200c5278"}
+
+        where the output format is JSONL (JSON Lines) and
+        ``hook`` is the name of hook point, and
+        ``device_id`` is the CUDA Device ID, and
+        ``size`` is the requested memory size to allocate, and
+        ``mem_size`` is the rounded memory size to be allocated, and
+        ``mem_ptr`` is the memory pointer, and
+        ``pmem_id`` is the pooled memory object ID.
+
+    Attributes:
+        file: Output file_like object that redirect to.
+        flush: If ``True``, this hook forcibly flushes the text stream
+            at the end of print. The default is ``True``.
+
+    """
+
+    name = 'DebugPrintHook'
+
+    def __init__(self, file=sys.stdout, flush=True):
+        self.file = file
+        self.flush = flush
+
+    def _print(self, msg):
+        self.file.write(msg)
+        self.file.write('\n')
+        if self.flush:
+            self.file.flush()
+
+    def alloc_postprocess(self, **kwargs):
+        msg = '{"hook":"%s","device_id":%d,' \
+              '"mem_size":%d,"mem_ptr":%d}'
+        msg %= ('alloc', kwargs['device_id'],
+                kwargs['mem_size'], kwargs['mem_ptr'])
+        self._print(msg)
+
+    def malloc_postprocess(self, **kwargs):
+        msg = '{"hook":"%s","device_id":%d,"size":%d,' \
+              '"mem_size":%d,"mem_ptr":%d,"pmem_id":"%s"}'
+        msg %= ('malloc', kwargs['device_id'], kwargs['size'],
+                kwargs['mem_size'], kwargs['mem_ptr'], hex(kwargs['pmem_id']))
+        self._print(msg)
+
+    def free_postprocess(self, **kwargs):
+        msg = '{"hook":"%s","device_id":%d,' \
+              '"mem_size":%d,"mem_ptr":%d,"pmem_id":"%s"}'
+        msg %= ('free', kwargs['device_id'],
+                kwargs['mem_size'], kwargs['mem_ptr'], hex(kwargs['pmem_id']))
+        self._print(msg)
diff --git a/cupy/cuda/memory_hooks/line_profile.py b/cupy/cuda/memory_hooks/line_profile.py
new file mode 100644
index 0000000..a0151e6
--- /dev/null
+++ b/cupy/cuda/memory_hooks/line_profile.py
@@ -0,0 +1,171 @@
+from os import path
+import sys
+import traceback
+
+from cupy.cuda import memory_hook
+
+
+class LineProfileHook(memory_hook.MemoryHook):
+    """Code line CuPy memory profiler.
+
+    This profiler shows line-by-line GPU memory consumption using traceback
+    module. But, note that it can trace only CPython level, no Cython level.
+    ref. https://github.com/cython/cython/issues/1755
+
+    Example:
+        Code example::
+
+            from cupy.cuda import memory_hooks
+            hook = memory_hooks.LineProfileHook()
+            with hook:
+                # some CuPy codes
+            hook.print_report()
+
+        Output example::
+
+            _root (4.00KB, 4.00KB)
+              lib/python3.6/unittest/__main__.py:18:<module> (4.00KB, 4.00KB)
+                lib/python3.6/unittest/main.py:255:runTests (4.00KB, 4.00KB)
+                  tests/cupy_tests/test.py:37:test (1.00KB, 1.00KB)
+                  tests/cupy_tests/test.py:38:test (1.00KB, 1.00KB)
+                  tests/cupy_tests/test.py:39:test (2.00KB, 2.00KB)
+
+        Each line shows::
+
+            {filename}:{lineno}:{func_name} ({used_bytes}, {acquired_bytes})
+
+        where *used_bytes* is the memory bytes used from CuPy memory pool, and
+        *acquired_bytes* is the actual memory bytes the CuPy memory pool
+        acquired from GPU device.
+        *_root* is a root node of the stack trace to show total memory usage.
+
+    Args:
+        max_depth (int): maximum depth to follow stack traces.
+            Default is 0 (no limit).
+    """
+
+    name = 'LineProfileHook'
+
+    def __init__(self, max_depth=0):
+        self._memory_frames = {}
+        self._root = MemoryFrame(None, None)
+        self._filename = path.abspath(__file__)
+        self._max_depth = max_depth
+
+    # callback
+    def malloc_preprocess(self, device_id, size, mem_size):
+        self._cretate_frame_tree(used_bytes=mem_size)
+
+    # callback
+    def alloc_preprocess(self, device_id, mem_size):
+        self._cretate_frame_tree(acquired_bytes=mem_size)
+
+    def _cretate_frame_tree(self, used_bytes=0, acquired_bytes=0):
+        self._root.used_bytes += used_bytes
+        self._root.acquired_bytes += acquired_bytes
+        parent = self._root
+        for depth, stackframe in enumerate(self._extract_stackframes()):
+            if 0 < self._max_depth <= depth + 1:
+                break
+            memory_frame = self._add_frame(parent, stackframe)
+            memory_frame.used_bytes += used_bytes
+            memory_frame.acquired_bytes += acquired_bytes
+            parent = memory_frame
+
+    def _extract_stackframes(self):
+        stackframes = traceback.extract_stack()
+        stackframes = [StackFrame(st) for st in stackframes]
+        stackframes = [
+            st for st in stackframes if st.filename != self._filename]
+        return stackframes
+
+    def _key_frame(self, parent, stackframe):
+        return (parent,
+                stackframe.filename,
+                stackframe.lineno,
+                stackframe.name)
+
+    def _add_frame(self, parent, stackframe):
+        key = self._key_frame(parent, stackframe)
+        if key in self._memory_frames:
+            memory_frame = self._memory_frames[key]
+        else:
+            memory_frame = MemoryFrame(parent, stackframe)
+            self._memory_frames[key] = memory_frame
+        return memory_frame
+
+    def print_report(self, file=sys.stdout):
+        """Prints a report of line memory profiling."""
+        line = '_root (%s, %s)\n' % self._root.humanized_bytes()
+        file.write(line)
+        for child in self._root.children:
+            self._print_frame(child, depth=1, file=file)
+        file.flush()
+
+    def _print_frame(self, memory_frame, depth=0, file=sys.stdout):
+        indent = ' ' * (depth * 2)
+        st = memory_frame.stackframe
+        used_bytes, acquired_bytes = memory_frame.humanized_bytes()
+        line = '%s%s:%s:%s (%s, %s)\n' % (
+            indent, st.filename, st.lineno, st.name,
+            used_bytes, acquired_bytes)
+        file.write(line)
+        for child in memory_frame.children:
+            self._print_frame(child, depth=depth + 1, file=file)
+
+
+class StackFrame(object):
+    """Compatibility layer for outputs of traceback.extract_stack().
+
+    Attributes:
+        filename (string): filename
+        lineno (int): line number
+        name (string): function name
+    """
+
+    def __init__(self, obj):
+        if isinstance(obj, tuple):  # < 3.5
+            self.filename = obj[0]
+            self.lineno = obj[1]
+            self.name = obj[2]
+        else:  # >= 3.5 FrameSummary
+            self.filename = obj.filename
+            self.lineno = obj.lineno
+            self.name = obj.name
+
+
+class MemoryFrame(object):
+    """A single stack frame along with sum of memory usage at the frame.
+
+    Attributes:
+        stackframe (FrameSummary): stackframe from traceback.extract_stack().
+        parent (MemoryFrame): parent frame, that is, caller.
+        children (list of MemoryFrame): child frames, that is, callees.
+        used_bytes (int): memory bytes that users used from CuPy memory pool.
+        acquired_bytes (int): memory bytes that CuPy memory pool acquired
+            from GPU device.
+    """
+
+    def __init__(self, parent, stackframe):
+        self.stackframe = stackframe
+        self.children = []
+        self._set_parent(parent)
+        self.used_bytes = 0
+        self.acquired_bytes = 0
+
+    def humanized_bytes(self):
+        used_bytes = self._humanized_size(self.used_bytes)
+        acquired_bytes = self._humanized_size(self.acquired_bytes)
+        return (used_bytes, acquired_bytes)
+
+    def _set_parent(self, parent):
+        if parent and parent not in parent.children:
+            self.parent = parent
+            parent.children.append(self)
+
+    def _humanized_size(self, size):
+        for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E']:
+            if size < 1024.0:
+                return '%3.2f%sB' % (size, unit)
+            size /= 1024.0
+        return '%.2f%sB' % (size, 'Z')
diff --git a/cupy/cuda/nccl.py b/cupy/cuda/nccl.py
new file mode 100644
index 0000000..98a169e
--- /dev/null
+++ b/cupy/cuda/nccl.py
@@ -0,0 +1,14 @@
+"""
+NCCL Wrapper
+
+Use `cupy_backends.cuda.libs.nccl` directly in CuPy codebase.
+"""
+
+available = True
+
+try:
+    from cupy_backends.cuda.libs.nccl import *  # NOQA
+except ImportError as e:
+    available = False
+    from cupy._environment import _preload_warning
+    _preload_warning('nccl', e)
diff --git a/cupy/cuda/nvtx.py b/cupy/cuda/nvtx.py
new file mode 100644
index 0000000..61fe633
--- /dev/null
+++ b/cupy/cuda/nvtx.py
@@ -0,0 +1 @@
+from cupy_backends.cuda.libs.nvtx import *  # NOQA
diff --git a/cupy/cuda/pinned_memory.pxd b/cupy/cuda/pinned_memory.pxd
new file mode 100644
index 0000000..9f968a7
--- /dev/null
+++ b/cupy/cuda/pinned_memory.pxd
@@ -0,0 +1,38 @@
+from libc.stdint cimport intptr_t
+
+
+cdef class PinnedMemoryPointer:
+
+    cdef:
+        readonly object mem
+        readonly intptr_t ptr
+        Py_ssize_t _shape[1]
+        Py_ssize_t _strides[1]
+
+    cpdef size_t size(self)
+
+
+cpdef _add_to_watch_list(event, obj)
+
+
+cpdef PinnedMemoryPointer alloc_pinned_memory(size_t size)
+
+
+cpdef set_pinned_memory_allocator(allocator=*)
+
+
+cdef class PinnedMemoryPool:
+
+    cdef:
+        object _alloc
+        dict _in_use
+        object _free
+        object __weakref__
+        object _weakref
+        object _lock
+        size_t _allocation_unit_size
+
+    cpdef PinnedMemoryPointer malloc(self, size_t size)
+    cpdef free(self, intptr_t ptr, size_t size)
+    cpdef free_all_blocks(self)
+    cpdef n_free_blocks(self)
diff --git a/cupy/cuda/pinned_memory.pyx b/cupy/cuda/pinned_memory.pyx
new file mode 100644
index 0000000..3e88580
--- /dev/null
+++ b/cupy/cuda/pinned_memory.pyx
@@ -0,0 +1,352 @@
+# distutils: language = c++
+
+import collections
+import weakref
+
+from fastrlock cimport rlock
+
+from cupy_backends.cuda.api import runtime
+
+from cupy._core cimport internal
+from cupy_backends.cuda.api cimport runtime
+from cupy import _util
+
+
+class PinnedMemory(object):
+
+    """Pinned memory allocation on host.
+
+    This class provides a RAII interface of the pinned memory allocation.
+
+    Args:
+        size (int): Size of the memory allocation in bytes.
+
+    """
+
+    def __init__(self, size_t size, unsigned int flags=0):
+        self.size = size
+        self.ptr = 0
+        if size > 0:
+            self.ptr = runtime.hostAlloc(size, flags)
+
+    def __del__(self, is_shutting_down=_util.is_shutting_down):
+        if is_shutting_down():
+            return
+        if self.ptr:
+            runtime.freeHost(self.ptr)
+
+    def __int__(self):
+        """Returns the pointer value to the head of the allocation."""
+        return self.ptr
+
+
+cdef class PinnedMemoryPointer:
+
+    """Pointer of a pinned memory.
+
+    An instance of this class holds a reference to the original memory buffer
+    and a pointer to a place within this buffer.
+
+    Args:
+        mem (PinnedMemory): The device memory buffer.
+        offset (int): An offset from the head of the buffer to the place this
+            pointer refers.
+
+    Attributes:
+        ~PinnedMemoryPointer.mem (PinnedMemory): The device memory buffer.
+        ~PinnedMemoryPointer.ptr (int): Pointer to the place within the buffer.
+    """
+
+    def __init__(self, mem, ptrdiff_t offset):
+        self.mem = mem
+        self.ptr = mem.ptr + offset
+
+    def __int__(self):
+        """Returns the pointer value."""
+        return self.ptr
+
+    def __add__(x, y):
+        """Adds an offset to the pointer."""
+        cdef PinnedMemoryPointer self
+        cdef ptrdiff_t offset
+        if isinstance(x, PinnedMemoryPointer):
+            self = x
+            offset = <ptrdiff_t?>y
+        else:
+            self = <PinnedMemoryPointer?>y
+            offset = <ptrdiff_t?>x
+        return PinnedMemoryPointer(
+            self.mem, self.ptr - self.mem.ptr + offset)
+
+    def __iadd__(self, ptrdiff_t offset):
+        """Adds an offset to the pointer in place."""
+        self.ptr += offset
+        return self
+
+    def __sub__(self, offset):
+        """Subtracts an offset from the pointer."""
+        return self + -offset
+
+    def __isub__(self, ptrdiff_t offset):
+        """Subtracts an offset from the pointer in place."""
+        return self.__iadd__(-offset)
+
+    cpdef size_t size(self):
+        return self.mem.size - (self.ptr - self.mem.ptr)
+
+    def __getbuffer__(self, Py_buffer *buffer, int flags):
+        size = self.size()
+
+        self._shape[0] = size
+        self._strides[0] = 1
+
+        buffer.buf = <void*>self.ptr
+        buffer.format = 'b'
+        buffer.internal = NULL
+        buffer.itemsize = 1
+        buffer.len = size
+        buffer.ndim = 1
+        buffer.obj = self
+        buffer.readonly = 0
+        buffer.shape = self._shape
+        buffer.strides = self._strides
+        buffer.suboffsets = NULL
+
+    def __releasebuffer__(self, Py_buffer *buffer):
+        pass
+
+    def __getsegcount__(self, Py_ssize_t *lenp):
+        if lenp != NULL:
+            lenp[0] = self.size()
+        return 1
+
+    def __getreadbuffer__(self, Py_ssize_t idx, void **p):
+        if idx != 0:
+            raise SystemError('accessing non-existent buffer segment')
+        p[0] = <void*>self.ptr
+        return self.size()
+
+    def __getwritebuffer__(self, Py_ssize_t idx, void **p):
+        if idx != 0:
+            raise SystemError('accessing non-existent buffer segment')
+        p[0] = <void*>self.ptr
+        return self.size()
+
+
+cdef class _EventWatcher:
+    cdef:
+        cdef list events
+        cdef object _lock
+
+    def __init__(self):
+        self.events = []
+        self._lock = rlock.create_fastrlock()
+
+    cpdef add(self, event, obj):
+        """ Add event to be monitored.
+
+        The ``obj`` are automatically released when the event done.
+
+        Args:
+            event (cupy.cuda.Event): The CUDA event to be monitored.
+            obj: The object to be held.
+        """
+        rlock.lock_fastrlock(self._lock, -1, True)
+        try:
+            self._check_and_release_without_lock()
+            if event.done:
+                return
+            self.events.append((event, obj))
+        finally:
+            rlock.unlock_fastrlock(self._lock)
+
+    cpdef check_and_release(self):
+        """ Check and release completed events.
+
+        """
+        if not self.events:
+            return
+        rlock.lock_fastrlock(self._lock, -1, True)
+        try:
+            self._check_and_release_without_lock()
+        finally:
+            rlock.unlock_fastrlock(self._lock)
+
+    cpdef _check_and_release_without_lock(self):
+        while self.events and self.events[0][0].done:
+            del self.events[0]
+
+
+cpdef PinnedMemoryPointer _malloc(size_t size):
+    mem = PinnedMemory(size, runtime.hostAllocPortable)
+    return PinnedMemoryPointer(mem, 0)
+
+
+cdef object _current_allocator = _malloc
+cdef _EventWatcher _watcher = _EventWatcher()
+
+
+cpdef _add_to_watch_list(event, obj):
+    """ Add event to be monitored.
+
+    The ``obj`` are automatically released when the event done.
+
+    Args:
+        event (cupy.cuda.Event): The CUDA event to be monitored.
+        obj: The object to be held.
+    """
+    _watcher.add(event, obj)
+
+
+cpdef PinnedMemoryPointer alloc_pinned_memory(size_t size):
+    """Calls the current allocator.
+
+    Use :func:`~cupy.cuda.set_pinned_memory_allocator` to change the current
+    allocator.
+
+    Args:
+        size (int): Size of the memory allocation.
+
+    Returns:
+        ~cupy.cuda.PinnedMemoryPointer: Pointer to the allocated buffer.
+
+    """
+    _watcher.check_and_release()
+    return _current_allocator(size)
+
+
+cpdef set_pinned_memory_allocator(allocator=None):
+    """Sets the current allocator for the pinned memory.
+
+    Args:
+        allocator (function): CuPy pinned memory allocator. It must have the
+            same interface as the :func:`cupy.cuda.alloc_pinned_memory`
+            function, which takes the buffer size as an argument and returns
+            the device buffer of that size. When ``None`` is specified, raw
+            memory allocator is used (i.e., memory pool is disabled).
+
+    """
+    global _current_allocator
+    if allocator is None:
+        allocator = _malloc
+    _current_allocator = allocator
+
+
+class PooledPinnedMemory(PinnedMemory):
+
+    """Memory allocation for a memory pool.
+
+    As the instance of this class is created by memory pool allocator, users
+    should not instantiate it manually.
+
+    """
+
+    def __init__(self, mem, pool):
+        self.ptr = mem.ptr
+        self.size = mem.size
+        self.pool = pool
+
+    def free(self):
+        """Releases the memory buffer and sends it to the memory pool.
+
+        This function actually does not free the buffer. It just returns the
+        buffer to the memory pool for reuse.
+
+        """
+        pool = self.pool()
+        if pool and self.ptr != 0:
+            pool.free(self.ptr, self.size)
+        self.ptr = 0
+        self.size = 0
+
+    __del__ = free
+
+
+cdef class PinnedMemoryPool:
+
+    """Memory pool for pinned memory on the host.
+
+    Note that it preserves all allocated memory buffers even if the user
+    explicitly release the one. Those released memory buffers are held by the
+    memory pool as *free blocks*, and reused for further memory allocations of
+    the same size.
+
+    Args:
+        allocator (function): The base CuPy pinned memory allocator. It is
+            used for allocating new blocks when the blocks of the required
+            size are all in use.
+
+    """
+
+    def __init__(self, allocator=_malloc):
+        self._in_use = {}
+        self._free = collections.defaultdict(list)
+        self._alloc = allocator
+        self._weakref = weakref.ref(self)
+        self._lock = rlock.create_fastrlock()
+        self._allocation_unit_size = 512
+
+    cpdef PinnedMemoryPointer malloc(self, size_t size):
+        cdef list free
+        cdef size_t unit
+
+        if size == 0:
+            return PinnedMemoryPointer(PinnedMemory(0), 0)
+
+        # Round up the memory size to fit memory alignment of cudaHostAlloc
+        unit = self._allocation_unit_size
+        size = internal.clp2(((size + unit - 1) // unit) * unit)
+        rlock.lock_fastrlock(self._lock, -1, True)
+        try:
+            free = self._free[size]
+            if free:
+                mem = free.pop()
+            else:
+                try:
+                    mem = self._alloc(size).mem
+                except runtime.CUDARuntimeError as e:
+                    if e.status != runtime.errorMemoryAllocation:
+                        raise
+                    self.free_all_blocks()
+                    mem = self._alloc(size).mem
+
+            self._in_use[mem.ptr] = mem
+        finally:
+            rlock.unlock_fastrlock(self._lock)
+        pmem = PooledPinnedMemory(mem, self._weakref)
+        return PinnedMemoryPointer(pmem, 0)
+
+    cpdef free(self, intptr_t ptr, size_t size):
+        cdef list free
+        rlock.lock_fastrlock(self._lock, -1, True)
+        try:
+            mem = self._in_use.pop(ptr, None)
+            if mem is None:
+                raise RuntimeError('Cannot free out-of-pool memory')
+            free = self._free[size]
+            free.append(mem)
+        finally:
+            rlock.unlock_fastrlock(self._lock)
+
+    cpdef free_all_blocks(self):
+        """Release free all blocks."""
+        rlock.lock_fastrlock(self._lock, -1, True)
+        try:
+            self._free.clear()
+        finally:
+            rlock.unlock_fastrlock(self._lock)
+
+    cpdef n_free_blocks(self):
+        """Count the total number of free blocks.
+
+        Returns:
+            int: The total number of free blocks.
+        """
+        cdef Py_ssize_t n = 0
+        rlock.lock_fastrlock(self._lock, -1, True)
+        try:
+            for v in self._free.values():
+                n += len(v)
+        finally:
+            rlock.unlock_fastrlock(self._lock)
+        return n
diff --git a/cupy/cuda/profiler.py b/cupy/cuda/profiler.py
new file mode 100644
index 0000000..fab38ba
--- /dev/null
+++ b/cupy/cuda/profiler.py
@@ -0,0 +1 @@
+from cupy_backends.cuda.libs.profiler import *  # NOQA
diff --git a/cupy/cuda/runtime.py b/cupy/cuda/runtime.py
new file mode 100644
index 0000000..6d27998
--- /dev/null
+++ b/cupy/cuda/runtime.py
@@ -0,0 +1 @@
+from cupy_backends.cuda.api.runtime import *  # NOQA
diff --git a/cupy/cuda/stream.pxd b/cupy/cuda/stream.pxd
new file mode 100644
index 0000000..1e396b6
--- /dev/null
+++ b/cupy/cuda/stream.pxd
@@ -0,0 +1,5 @@
+from libc.stdint cimport intptr_t
+
+
+cdef intptr_t get_current_stream_ptr()
+cpdef get_current_stream()
diff --git a/cupy/cuda/stream.pyx b/cupy/cuda/stream.pyx
new file mode 100644
index 0000000..10e7a12
--- /dev/null
+++ b/cupy/cuda/stream.pyx
@@ -0,0 +1,524 @@
+import threading
+
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda cimport stream as backends_stream
+from cupy.cuda cimport graph
+
+from cupy import _util
+
+
+cdef object _thread_local = threading.local()
+
+
+cdef class _ThreadLocal:
+    # We keep both current_stream and current_stream_stack because the
+    # former is also used when calling stream.use(). This bookkeeping enables
+    # correct rewinding when "with" blocks are mixed with ".use()" (though
+    # this is considered an anti-pattern).
+
+    cdef list current_stream  # list of object
+    cdef list current_device_id_stack  # list of int
+    cdef list current_stream_stack  # list of list
+
+    def __init__(self):
+        cdef int i, num_devices = runtime.getDeviceCount()
+        self.current_stream = []
+        self.current_device_id_stack = []
+        self.current_stream_stack = []
+        for i in range(num_devices):
+            default_stream = get_default_stream()
+            self.current_stream.append(default_stream)
+            self.current_stream_stack.append([default_stream])
+
+    @staticmethod
+    cdef _ThreadLocal get():
+        try:
+            tls = _thread_local.tls
+        except AttributeError:
+            tls = _thread_local.tls = _ThreadLocal()
+        return <_ThreadLocal>tls
+
+    cdef void push_stream(self, stream, int device_id) except*:
+        assert device_id >= 0
+        self.current_stream_stack[device_id].append(stream)
+        # record device_id to prevent from popping the wrong stream at exit
+        self.current_device_id_stack.append(device_id)
+        self.set_current_stream(stream)
+
+    cdef void pop_stream(self) except*:
+        cdef int device_id = self.current_device_id_stack.pop()
+        self.current_stream_stack[device_id].pop()
+        prev_stream = self.current_stream_stack[device_id][-1]
+        self.set_current_stream(prev_stream)
+        assert len(self.current_stream_stack[device_id]) >= 1
+
+    cdef set_current_stream(self, stream):
+        cdef intptr_t ptr = <intptr_t>stream.ptr
+        cdef int device_id = stream.device_id
+        if device_id == -1:
+            device_id = runtime.getDevice()
+        backends_stream.set_current_stream_ptr(ptr, device_id)
+        self.current_stream[device_id] = stream
+
+    cdef get_current_stream(self, int device_id=-1):
+        if device_id == -1:
+            device_id = runtime.getDevice()
+        stream_ref = self.current_stream[device_id]
+        return stream_ref
+
+    cdef intptr_t get_current_stream_ptr(self):
+        return backends_stream.get_current_stream_ptr()
+
+
+cdef get_default_stream():
+    return Stream.ptds if backends_stream.is_ptds_enabled() else Stream.null
+
+
+cdef intptr_t get_current_stream_ptr():
+    """C API to get current CUDA stream pointer.
+
+    Returns:
+        intptr_t: The current CUDA stream pointer.
+    """
+    tls = _ThreadLocal.get()
+    return <intptr_t>tls.get_current_stream_ptr()
+
+
+cpdef get_current_stream():
+    """Gets current CUDA stream.
+
+    Returns:
+        cupy.cuda.Stream: The current CUDA stream.
+    """
+    tls = _ThreadLocal.get()
+    return tls.get_current_stream()
+
+
+class Event(object):
+
+    """CUDA event, a synchronization point of CUDA streams.
+
+    This class handles the CUDA event handle in RAII way, i.e., when an Event
+    instance is destroyed by the GC, its handle is also destroyed.
+
+    Args:
+        block (bool): If ``True``, the event blocks on the
+            :meth:`~cupy.cuda.Event.synchronize` method.
+        disable_timing (bool): If ``True``, the event does not prepare the
+            timing data.
+        interprocess (bool): If ``True``, the event can be passed to other
+            processes.
+
+    Attributes:
+        ~Event.ptr (intptr_t): Raw event handle.
+
+    """
+
+    def __init__(self, block=False, disable_timing=False, interprocess=False):
+        self.ptr = 0
+
+        if interprocess and not disable_timing:
+            raise ValueError('Timing must be disabled for interprocess events')
+        flag = ((block and runtime.eventBlockingSync) |
+                (disable_timing and runtime.eventDisableTiming) |
+                (interprocess and runtime.eventInterprocess))
+        self.ptr = runtime.eventCreateWithFlags(flag)
+
+    def __del__(self, is_shutting_down=_util.is_shutting_down):
+        if is_shutting_down():
+            return
+        if self.ptr:
+            runtime.eventDestroy(self.ptr)
+
+    @property
+    def done(self):
+        """True if the event is done."""
+        return runtime.eventQuery(self.ptr) == 0  # cudaSuccess
+
+    def record(self, stream=None):
+        """Records the event to a stream.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream to record event. The null
+                stream is used by default.
+
+        .. seealso:: :meth:`cupy.cuda.Stream.record`
+
+        """
+        if stream is None:
+            stream_ptr = get_current_stream_ptr()
+        else:
+            stream_ptr = stream.ptr
+        runtime.eventRecord(self.ptr, stream_ptr)
+
+    def synchronize(self):
+        """Synchronizes all device work to the event.
+
+        If the event is created as a blocking event, it also blocks the CPU
+        thread until the event is done.
+
+        """
+        runtime.eventSynchronize(self.ptr)
+
+
+def get_elapsed_time(start_event, end_event):
+    """Gets the elapsed time between two events.
+
+    Args:
+        start_event (Event): Earlier event.
+        end_event (Event): Later event.
+
+    Returns:
+        float: Elapsed time in milliseconds.
+
+    """
+    return runtime.eventElapsedTime(start_event.ptr, end_event.ptr)
+
+
+cdef int check_stream_device_match(int device_id) except? -1:
+    """Check if the stream was created on the current device."""
+    cdef int curr_dev = runtime.getDevice()
+    if device_id == -1:
+        device_id = curr_dev
+    if device_id != curr_dev:
+        raise RuntimeError(
+            f'The device that the stream is created on ({device_id})'
+            f' does not match with the current device ({curr_dev})')
+    return device_id
+
+
+class _BaseStream:
+
+    """CUDA stream.
+
+    Attributes:
+        ~Stream.ptr (intptr_t): Raw stream handle.
+        ~Stream.device_id (int): The ID of the device that the stream was
+            created on.
+
+    """
+
+    def __init__(self, ptr, device_id):
+        self.ptr = ptr
+        self.device_id = device_id
+
+    def __eq__(self, other):
+        # This operator needed as the ptr may be shared between multiple Stream
+        # instances (e.g, `Stream.null` singleton and `Stream(null=True)` or
+        # `ExternalStream`s).
+        if isinstance(other, _BaseStream):
+            return self.ptr == other.ptr
+        return False
+
+    def __enter__(self):
+        # N.B. for maintainers: do not use this context manager in CuPy
+        # codebase. See #5943 and #5963.
+        tls = _ThreadLocal.get()
+        cdef int device_id = self.device_id
+        device_id = check_stream_device_match(device_id)
+        tls.push_stream(self, device_id)
+        return self
+
+    def __exit__(self, *args):
+        tls = _ThreadLocal.get()
+        tls.pop_stream()
+
+    def __repr__(self):
+        return '<{} {} (device {})>'.format(
+            type(self).__name__, self.ptr, self.device_id)
+
+    def __hash__(self):
+        return self.ptr
+
+    def use(self):
+        """Makes this stream current.
+
+        If you want to switch a stream temporarily, use the *with* statement.
+        """
+        tls = _ThreadLocal.get()
+        cdef int device_id = self.device_id
+        check_stream_device_match(device_id)
+        tls.set_current_stream(self)
+        return self
+
+    @property
+    def done(self):
+        """True if all work on this stream has been done."""
+        return runtime.streamQuery(self.ptr) == 0  # cudaSuccess
+
+    def synchronize(self):
+        """Waits for the stream completing all queued work."""
+        runtime.streamSynchronize(self.ptr)
+
+    def add_callback(self, callback, arg):
+        """Adds a callback that is called when all queued work is done.
+
+        Args:
+            callback (function): Callback function. It must take three
+                arguments (Stream object, int error status, and user data
+                object), and returns nothing.
+            arg (object): Argument to the callback.
+
+        .. note::
+            Whenever possible, use the :meth:`launch_host_func` method
+            instead of this one, as it may be deprecated and removed from
+            CUDA at some point.
+
+        """
+        def f(stream, status, dummy):
+            callback(self, status, arg)
+
+        runtime.streamAddCallback(self.ptr, f, 0)
+
+    def launch_host_func(self, callback, arg):
+        """Launch a callback on host when all queued work is done.
+
+        Args:
+            callback (function): Callback function. It must take only one
+                argument (user data object), and returns nothing.
+            arg (object): Argument to the callback.
+
+        .. note::
+            Whenever possible, this method is recommended over
+            :meth:`add_callback`, which may be deprecated and removed from
+            CUDA at some point.
+
+        .. seealso:: `cudaLaunchHostFunc()`_
+
+        .. _cudaLaunchHostFunc():
+            https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__EXECUTION.html#group__CUDART__EXECUTION_1g05841eaa5f90f27124241baafb3e856f
+
+        """
+        def f(dummy):
+            callback(arg)
+
+        runtime.launchHostFunc(self.ptr, f, 0)
+
+    def record(self, event=None):
+        """Records an event on the stream.
+
+        Args:
+            event (None or cupy.cuda.Event): CUDA event. If ``None``, then a
+                new plain event is created and used.
+
+        Returns:
+            cupy.cuda.Event: The recorded event.
+
+        .. seealso:: :meth:`cupy.cuda.Event.record`
+
+        """
+        if event is None:
+            event = Event()
+        runtime.eventRecord(event.ptr, self.ptr)
+        return event
+
+    def wait_event(self, event):
+        """Makes the stream wait for an event.
+
+        The future work on this stream will be done after the event.
+
+        Args:
+            event (cupy.cuda.Event): CUDA event.
+
+        """
+        runtime.streamWaitEvent(self.ptr, event.ptr)
+
+    def begin_capture(self, mode=None):
+        """Begin stream capture to construct a CUDA graph.
+
+        A call to this function must be paired with a call to
+        :meth:`end_capture` to complete the capture.
+
+        .. code-block:: python
+
+            # create a non-blocking stream for the purpose of capturing
+            s1 = cp.cuda.Stream(non_blocking=True)
+            with s1:
+                s1.begin_capture()
+                # ... perform operations to construct a graph ...
+                g = s1.end_capture()
+
+            # the returned graph can be launched on any stream (including s1)
+            g.launch(stream=s1)
+            s1.synchronize()
+
+            s2 = cp.cuda.Stream()
+            with s2:
+                g.launch()
+            s2.synchronize()
+
+        Args:
+            mode (int): The stream capture mode. Default is
+                :data:`~cupy.cuda.runtime.streamCaptureModeRelaxed`.
+
+        .. note:: During the stream capture, synchronous device-host transfers
+            are not allowed. This has a particular implication for CuPy APIs,
+            as some functions that internally require synchronous transfer
+            would not work as expected and an exception would be raised. For
+            further constraints of CUDA stream capture, please refer to the
+            CUDA Programming Guide.
+
+        .. note:: Currently this capability is not supported on HIP.
+
+        .. seealso:: `cudaStreamBeginCapture()`_
+
+        .. _cudaStreamBeginCapture():
+            https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html#group__CUDART__STREAM_1g793d7d4e474388ddfda531603dc34aa3
+
+        """
+        if runtime._is_hip_environment:
+            raise RuntimeError('This function is not supported on HIP')
+        if self.ptr == 0 or self.ptr == 1:
+            raise RuntimeError('cannot capture on the default (legacy) stream')
+        if mode is None:
+            # We default to the relaxed mode for the following reason: During
+            # the capture the memory pool might need to increase size. If it's
+            # backed by cudaMalloc, which is an "unsafe" API, other modes would
+            # not permit such a (legitimate) operation. See the API doc for
+            # "cudaThreadExchangeStreamCaptureMode" for further detail.
+            # (Though, ideally stream capture should be used together with
+            # the async APIs, such as cudaMallocAsync.)
+            mode = runtime.streamCaptureModeRelaxed
+        runtime.streamBeginCapture(self.ptr, mode)
+
+    def end_capture(self):
+        """End stream capture and retrieve the constructed CUDA graph.
+
+        Returns:
+            cupy.cuda.Graph:
+                A CUDA graph object that encapsulates the captured work.
+
+        .. note:: Currently this capability is not supported on HIP.
+
+        .. seealso:: `cudaStreamEndCapture()`_
+
+        .. _cudaStreamEndCapture():
+            https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html#group__CUDART__STREAM_1gf5a0efebc818054ceecd1e3e5e76d93e
+
+        """
+        if runtime._is_hip_environment:
+            raise RuntimeError('This function is not supported on HIP')
+        cdef intptr_t g = runtime.streamEndCapture(self.ptr)
+        return graph.Graph.from_stream(g)
+
+    def is_capturing(self):
+        """Check if the stream is capturing.
+
+        Returns:
+            bool:
+                If the capturing status is successfully queried, the returned
+                value indicates the capturing status. An exception could be
+                raised if such a query is illegal, please refer to the CUDA
+                Programming Guide for detail.
+
+        """
+        # TODO(leofang): is it better to be a property?
+        if runtime._is_hip_environment:
+            raise RuntimeError('This function is not supported on HIP')
+        try:
+            return runtime.streamIsCapturing(self.ptr)
+        except RuntimeError:  # can be RuntimeError or CUDARuntimeError
+            raise
+
+
+class Stream(_BaseStream):
+
+    """CUDA stream.
+
+    This class handles the CUDA stream handle in RAII way, i.e., when an Stream
+    instance is destroyed by the GC, its handle is also destroyed.
+
+    Note that if both ``null`` and ``ptds`` are ``False``, a plain new
+    stream is created.
+
+    Args:
+        null (bool): If ``True``, the stream is a null stream (i.e. the default
+            stream that synchronizes with all streams). Note that you can also
+            use the ``Stream.null`` singleton object instead of creating a new
+            null stream object.
+        ptds (bool): If ``True`` and ``null`` is ``False``, the per-thread
+            default stream is used. Note that you can also use the
+            ``Stream.ptds`` singleton object instead of creating a new
+            per-thread default stream object.
+        non_blocking (bool): If ``True`` and both ``null`` and ``ptds`` are
+            ``False``, the stream does not synchronize with the NULL stream.
+
+    Attributes:
+        ~Stream.ptr (intptr_t): Raw stream handle.
+        ~Stream.device_id (int): The ID of the device that the stream was
+            created on. The value ``-1`` is used for the singleton stream
+            objects.
+
+    """
+
+    null = None
+    ptds = None
+
+    def __init__(self, null=False, non_blocking=False, ptds=False):
+        if null:
+            # TODO(pentschev): move to streamLegacy. This wasn't possible
+            # because of a NCCL bug that should be fixed in the version
+            # following 2.8.3-1.
+            ptr = 0
+            device_id = -1
+        elif ptds:
+            if runtime._is_hip_environment:
+                raise ValueError('HIP does not support per-thread '
+                                 'default stream (ptds)')
+            ptr = runtime.streamPerThread
+            device_id = -1
+        elif non_blocking:
+            ptr = runtime.streamCreateWithFlags(runtime.streamNonBlocking)
+            device_id = runtime.getDevice()
+        else:
+            ptr = runtime.streamCreate()
+            device_id = runtime.getDevice()
+        super().__init__(ptr, device_id)
+
+    def __del__(self, is_shutting_down=_util.is_shutting_down):
+        if is_shutting_down():
+            return
+        if self.ptr not in (0, runtime.streamLegacy, runtime.streamPerThread):
+            runtime.streamDestroy(self.ptr)
+        # Note that we can not release memory pool of the stream held in CPU
+        # because the memory would still be used in kernels executed in GPU.
+
+
+class ExternalStream(_BaseStream):
+
+    """CUDA stream not managed by CuPy.
+
+    This class allows to use external streams in CuPy by providing the
+    stream pointer obtained from the CUDA runtime call.
+    The user is in charge of managing the life-cycle of the stream.
+
+    Args:
+        ptr (intptr_t): Address of the `cudaStream_t` object.
+        device_id (int): The ID of the device that the stream was created on.
+            Default is ``-1``, indicating it is unknown.
+
+    Attributes:
+        ~Stream.ptr (intptr_t): Raw stream handle.
+        ~Stream.device_id (int): The ID of the device that the stream was
+            created on. The value ``-1`` is used to indicate it is unknown.
+
+    .. warning::
+        If ``device_id`` is not specified, the user is required to ensure legal
+        operations of the stream. Specifically, the stream must be used on the
+        device that it was created on.
+
+    """
+
+    def __init__(self, ptr, device_id=-1):
+        # It is in theory unsafe to just call runtime.getDevice() here, as the
+        # stream pointer could come from a different device (although
+        # unlikely). While we could use driver API combos cuStreamGetCtx ->
+        # cuCtxSetCurrent -> cuCtxGetDevice -> ... to retrieve the device ID
+        # associated with the stream, it is way too complicated and does not
+        # work with HIP. Let us keep this as thin as possible.
+        super().__init__(ptr, device_id)
+
+
+Stream.null = Stream(null=True)
+if not runtime._is_hip_environment:
+    Stream.ptds = Stream(ptds=True)
diff --git a/cupy/cuda/texture.pxd b/cupy/cuda/texture.pxd
new file mode 100644
index 0000000..e4a4188
--- /dev/null
+++ b/cupy/cuda/texture.pxd
@@ -0,0 +1,47 @@
+from libc.stdint cimport intptr_t, uintmax_t
+
+from cupy._core.core cimport _ndarray_base
+
+
+cdef class ChannelFormatDescriptor:
+    cdef:
+        readonly intptr_t ptr
+
+
+cdef class ResourceDescriptor:
+    cdef:
+        readonly intptr_t ptr
+        readonly ChannelFormatDescriptor chDesc
+        readonly CUDAarray cuArr
+        readonly _ndarray_base arr
+
+
+cdef class TextureDescriptor:
+    cdef:
+        readonly intptr_t ptr
+
+
+cdef class CUDAarray:
+    cdef:
+        readonly intptr_t ptr
+        readonly ChannelFormatDescriptor desc
+        readonly size_t width, height, depth
+        readonly unsigned int flags
+        readonly int ndim
+
+        int _get_memory_kind(self, src, dst)
+        void* _make_cudaMemcpy3DParms(self, src, dst)
+        void _prepare_copy(self, arr, stream, direction) except*
+
+
+cdef class TextureObject:
+    cdef:
+        readonly uintmax_t ptr
+        readonly ResourceDescriptor ResDesc
+        readonly TextureDescriptor TexDesc
+
+
+cdef class SurfaceObject:
+    cdef:
+        readonly uintmax_t ptr
+        readonly ResourceDescriptor ResDesc
diff --git a/cupy/cuda/texture.pyx b/cupy/cuda/texture.pyx
new file mode 100644
index 0000000..d66cbb1
--- /dev/null
+++ b/cupy/cuda/texture.pyx
@@ -0,0 +1,561 @@
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from libc.string cimport memset as c_memset
+
+import numpy
+
+from cupy._core.core cimport _ndarray_base
+from cupy._core.core cimport _internal_ascontiguousarray
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda.api.runtime cimport Array, \
+    ChannelFormatDesc, ChannelFormatKind, \
+    Memcpy3DParms, MemoryKind, PitchedPtr, ResourceDesc, ResourceType, \
+    TextureAddressMode, TextureDesc, TextureFilterMode, TextureReadMode
+from cupy.cuda cimport stream as stream_module
+
+from cupy_backends.cuda.api.runtime import CUDARuntimeError
+
+
+cdef extern from '../../cupy_backends/cupy_backend.h':
+    pass
+
+cdef extern from '../../cupy_backends/cupy_backend_runtime.h':
+    pass
+
+
+cdef class ChannelFormatDescriptor:
+    '''A class that holds the channel format description. Equivalent to
+    ``cudaChannelFormatDesc``.
+
+    Args:
+        x (int): the number of bits for the x channel.
+        y (int): the number of bits for the y channel.
+        z (int): the number of bits for the z channel.
+        w (int): the number of bits for the w channel.
+        f (int): the channel format. Use one of the values in ``cudaChannelFormat*``,
+            such as :const:`cupy.cuda.runtime.cudaChannelFormatKindFloat`.
+
+    .. seealso:: `cudaCreateChannelDesc()`_
+
+    .. _cudaCreateChannelDesc():
+        https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TEXTURE__OBJECT.html#group__CUDART__TEXTURE__OBJECT_1g39df9e3b6edc41cd6f189d2109672ca5
+    '''  # noqa
+    def __init__(self, int x, int y, int z, int w, int f):
+        # We don't call cudaCreateChannelDesc() here for two reasons: 1. to
+        # avoid out of scope; 2. it doesn't do input verification for us.
+        #
+        # WARNING: don't use [0] or cython.operator.dereference to dereference
+        # a pointer to struct for writing to its members !!! (But read is OK.)
+        # Turns out while there's no arrow operator '->' in Cython, one can
+        # just treat the ptr as a real object and access the struct attributes.
+        # This applies to several classes below.
+        self.ptr = <intptr_t>PyMem_Malloc(sizeof(ChannelFormatDesc))
+        cdef ChannelFormatDesc* desc = (<ChannelFormatDesc*>self.ptr)
+        desc.x = x
+        desc.y = y
+        desc.z = z
+        desc.w = w
+        desc.f = <ChannelFormatKind>f
+
+    def __dealloc__(self):
+        if self.ptr != 0:
+            PyMem_Free(<ChannelFormatDesc*>self.ptr)
+            self.ptr = 0
+
+    def get_channel_format(self):
+        '''Returns a dict containing the input.'''
+        cdef dict desc = {}
+        desc['x'] = (<ChannelFormatDesc*>self.ptr).x
+        desc['y'] = (<ChannelFormatDesc*>self.ptr).y
+        desc['z'] = (<ChannelFormatDesc*>self.ptr).z
+        desc['w'] = (<ChannelFormatDesc*>self.ptr).w
+        desc['f'] = (<ChannelFormatDesc*>self.ptr).f
+        return desc
+
+
+cdef class ResourceDescriptor:
+    '''A class that holds the resource description. Equivalent to
+    ``cudaResourceDesc``.
+
+    Args:
+        restype (int): the resource type. Use one of the values in
+            ``cudaResourceType*``, such as
+            :const:`cupy.cuda.runtime.cudaResourceTypeArray`.
+        cuArr (CUDAarray, optional): An instance of :class:`CUDAarray`,
+            required if ``restype`` is set to
+            :const:`cupy.cuda.runtime.cudaResourceTypeArray`.
+        arr (cupy.ndarray, optional): An instance of :class:`~cupy.ndarray`,
+            required if ``restype`` is set to
+            :const:`cupy.cuda.runtime.cudaResourceTypeLinear` or
+            :const:`cupy.cuda.runtime.cudaResourceTypePitch2D`.
+        chDesc (ChannelFormatDescriptor, optional): an instance of
+            :class:`ChannelFormatDescriptor`, required if ``restype`` is set to
+            :const:`cupy.cuda.runtime.cudaResourceTypeLinear` or
+            :const:`cupy.cuda.runtime.cudaResourceTypePitch2D`.
+        sizeInBytes (int, optional): total bytes in the linear memory, required
+            if ``restype`` is set to
+            :const:`cupy.cuda.runtime.cudaResourceTypeLinear`.
+        width (int, optional): the width (in elements) of the 2D array,
+            required if ``restype`` is set to
+            :const:`cupy.cuda.runtime.cudaResourceTypePitch2D`.
+        height (int, optional): the height (in elements) of the 2D array,
+            required if ``restype`` is set to
+            :const:`cupy.cuda.runtime.cudaResourceTypePitch2D`.
+        pitchInBytes (int, optional): the number of bytes per pitch-aligned row,
+            required if ``restype`` is set to
+            :const:`cupy.cuda.runtime.cudaResourceTypePitch2D`.
+
+    .. note::
+        A texture backed by `mipmap` arrays is currently not supported in CuPy.
+
+    .. seealso:: `cudaCreateTextureObject()`_
+
+    .. _cudaCreateTextureObject():
+        https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TEXTURE__OBJECT.html#group__CUDART__TEXTURE__OBJECT_1g16ac75814780c3a16e4c63869feb9ad3
+    '''  # noqa
+    def __init__(self, int restype, CUDAarray cuArr=None,
+                 _ndarray_base arr=None, ChannelFormatDescriptor chDesc=None,
+                 size_t sizeInBytes=0, size_t width=0, size_t height=0,
+                 size_t pitchInBytes=0):
+        if restype == runtime.cudaResourceTypeMipmappedArray:
+            # TODO(leofang): support this?
+            raise NotImplementedError('cudaResourceTypeMipmappedArray is '
+                                      'currently not supported.')
+
+        self.ptr = <intptr_t>PyMem_Malloc(sizeof(ResourceDesc))
+        cdef ResourceDesc* desc = (<ResourceDesc*>self.ptr)
+        c_memset(desc, 0, sizeof(ResourceDesc))
+
+        desc.resType = <ResourceType>restype
+        if restype == runtime.cudaResourceTypeArray:
+            desc.res.array.array = <Array>(cuArr.ptr)
+        elif restype == runtime.cudaResourceTypeLinear:
+            desc.res.linear.devPtr = <void*>(arr.data.ptr)
+            desc.res.linear.desc = (<ChannelFormatDesc*>chDesc.ptr)[0]
+            desc.res.linear.sizeInBytes = sizeInBytes
+        elif restype == runtime.cudaResourceTypePitch2D:
+            desc.res.pitch2D.devPtr = <void*>(arr.data.ptr)
+            desc.res.pitch2D.desc = (<ChannelFormatDesc*>chDesc.ptr)[0]
+            desc.res.pitch2D.width = width
+            desc.res.pitch2D.height = height
+            desc.res.pitch2D.pitchInBytes = pitchInBytes
+
+        self.chDesc = chDesc
+        self.cuArr = cuArr
+        self.arr = arr
+
+    def __dealloc__(self):
+        if self.ptr != 0:
+            PyMem_Free(<ResourceDesc*>self.ptr)
+            self.ptr = 0
+
+    def get_resource_desc(self):
+        '''Returns a dict containing the input.'''
+        cdef dict desc = {}
+        cdef intptr_t ptr
+        cdef ResourceDesc* resPtr = <ResourceDesc*>(self.ptr)
+
+        # For texture memory, print the underlying pointer address so that
+        # it can be used for verification by the caller. Note that resPtr.res
+        # is a union, so ptr is always there for every category, which could
+        # be confusing and thus need a logic to make selections.
+        desc['resType'] = resPtr.resType
+        if resPtr.resType == 0:
+            ptr = <intptr_t>(resPtr.res.array.array)
+            desc['array'] = {'array': ptr}
+        elif resPtr.resType == 2:
+            ptr = <intptr_t>(resPtr.res.linear.devPtr)
+            desc['linear'] = {'devPtr': ptr,
+                              'desc': self.chDesc.get_channel_format(),
+                              'sizeInBytes': resPtr.res.linear.sizeInBytes}
+        elif resPtr.resType == 3:
+            ptr = <intptr_t>(resPtr.res.pitch2D.devPtr)
+            desc['pitch2D'] = {'devPtr': ptr,
+                               'desc': self.chDesc.get_channel_format(),
+                               'width': resPtr.res.pitch2D.width,
+                               'height': resPtr.res.pitch2D.height,
+                               'pitchInBytes': resPtr.res.pitch2D.pitchInBytes}
+        return desc
+
+
+cdef class TextureDescriptor:
+    '''A class that holds the texture description. Equivalent to
+    ``cudaTextureDesc``.
+
+    Args:
+        addressModes (tuple or list): an iterable with length up to 3, each
+            element is one of the values in ``cudaAddressMode*``, such as
+            :const:`cupy.cuda.runtime.cudaAddressModeWrap`.
+        filterMode (int): the filter mode. Use one of the values in
+            ``cudaFilterMode*``, such as
+            :const:`cupy.cuda.runtime.cudaFilterModePoint`.
+        readMode (int): the read mode. Use one of the values in
+            ``cudaReadMode*``, such as
+            :const:`cupy.cuda.runtime.cudaReadModeElementType`.
+        normalizedCoords (int): whether coordinates are normalized or not.
+        sRGB (int, optional)
+        borderColors (tuple or list, optional): an iterable with length up to
+            4.
+        maxAnisotropy (int, optional)
+
+    .. note::
+        A texture backed by `mipmap` arrays is currently not supported in CuPy.
+
+    .. seealso:: `cudaCreateTextureObject()`_
+
+    .. _cudaCreateTextureObject():
+        https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TEXTURE__OBJECT.html#group__CUDART__TEXTURE__OBJECT_1g16ac75814780c3a16e4c63869feb9ad3
+    '''  # noqa
+    def __init__(self, addressModes=None, int filterMode=0, int readMode=0,
+                 sRGB=None, borderColors=None, normalizedCoords=None,
+                 maxAnisotropy=None):
+        self.ptr = <intptr_t>PyMem_Malloc(sizeof(TextureDesc))
+        cdef TextureDesc* desc = (<TextureDesc*>self.ptr)
+        c_memset(desc, 0, sizeof(TextureDesc))
+
+        if addressModes is not None:
+            assert len(addressModes) <= 3
+            for i, mode in enumerate(addressModes):
+                desc.addressMode[i] = <TextureAddressMode>mode
+        desc.filterMode = <TextureFilterMode>filterMode
+        desc.readMode = <TextureReadMode>readMode
+        if normalizedCoords is not None:
+            desc.normalizedCoords = normalizedCoords
+        if sRGB is not None:
+            desc.sRGB = sRGB
+        if borderColors is not None:
+            assert len(borderColors) <= 4
+            for i, color in enumerate(borderColors):
+                desc.borderColor[i] = color
+        if maxAnisotropy is not None:
+            desc.maxAnisotropy = maxAnisotropy
+        # TODO(leofang): support mipmap?
+
+    def __dealloc__(self):
+        if self.ptr != 0:
+            PyMem_Free(<TextureDesc*>self.ptr)
+            self.ptr = 0
+
+    def get_texture_desc(self):
+        '''Returns a dict containing the input.'''
+        cdef dict desc = {}
+        cdef TextureDesc* ptr = <TextureDesc*>(self.ptr)
+        desc['addressMode'] = (ptr.addressMode[0],
+                               ptr.addressMode[1],
+                               ptr.addressMode[2])
+        desc['filterMode'] = ptr.filterMode
+        desc['readMode'] = ptr.readMode
+        desc['sRGB'] = ptr.sRGB
+        desc['borderColor']= (ptr.borderColor[0],
+                              ptr.borderColor[1],
+                              ptr.borderColor[2],
+                              ptr.borderColor[3])
+        desc['normalizedCoords']= ptr.normalizedCoords
+        desc['maxAnisotropy'] = ptr.maxAnisotropy
+        # TODO(leofang): support these?
+        # desc['mipmapFilterMode'] = ptr.mipmapFilterMode
+        # desc['mipmapLevelBias'] = ptr.mipmapLevelBias
+        # desc['minMipmapLevelClamp'] = ptr.minMipmapLevelClamp
+        # desc['maxMipmapLevelClamp'] = ptr.maxMipmapLevelClamp
+        return desc
+
+
+cdef class CUDAarray:
+    '''Allocate a CUDA array (`cudaArray_t`) that can be used as texture memory.
+    Depending on the input, either 1D, 2D, or 3D CUDA array is returned.
+
+    Args:
+        desc (ChannelFormatDescriptor): an instance of
+            :class:`ChannelFormatDescriptor`.
+        width (int): the width (in elements) of the array.
+        height (int, optional): the height (in elements) of the array.
+        depth (int, optional): the depth (in elements) of the array.
+        flags (int, optional): the flag for extensions. Use one of the values
+            in ``cudaArray*``, such as
+            :const:`cupy.cuda.runtime.cudaArrayDefault`.
+
+    .. warning::
+        The memory allocation of :class:`CUDAarray` is done outside of CuPy's
+        memory management (enabled by default) due to CUDA's limitation. Users
+        of :class:`CUDAarray` should be cautious about any out-of-memory
+        possibilities.
+
+    .. seealso:: `cudaMalloc3DArray()`_
+
+    .. _cudaMalloc3DArray():
+        https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__MEMORY.html#group__CUDART__MEMORY_1g948143cf2423a072ac6a31fb635efd88
+    '''  # noqa
+    # TODO(leofang): perhaps this wrapper is not needed when cupy.ndarray
+    # can be backed by texture memory/CUDA arrays?
+    def __init__(self, ChannelFormatDescriptor desc, size_t width,
+                 size_t height=0, size_t depth=0, unsigned int flags=0):
+        if width == 0:
+            raise ValueError('To create a CUDA array, width must be nonzero.')
+        elif height == 0 and depth > 0:
+            raise ValueError('To create a 2D CUDA array, height must be '
+                             'nonzero.')
+        else:
+            # malloc3DArray handles all possibilities (1D, 2D, 3D)
+            self.ptr = runtime.malloc3DArray(desc.ptr, width, height, depth,
+                                             flags)
+
+        # bookkeeping
+        self.desc = desc
+        self.width = width
+        self.height = height
+        self.depth = depth
+        self.flags = flags
+        self.ndim = 3 if depth > 0 else 2 if height > 0 else 1
+
+    def __dealloc__(self):
+        if self.ptr != 0:
+            runtime.freeArray(self.ptr)
+            self.ptr = 0
+
+    cdef int _get_memory_kind(self, src, dst):
+        cdef int kind
+        if isinstance(src, _ndarray_base) and dst is self:
+            kind = runtime.memcpyDeviceToDevice
+        elif src is self and isinstance(dst, _ndarray_base):
+            kind = runtime.memcpyDeviceToDevice
+        elif isinstance(src, numpy.ndarray) and dst is self:
+            kind = runtime.memcpyHostToDevice
+        elif src is self and isinstance(dst, numpy.ndarray):
+            kind = runtime.memcpyDeviceToHost
+        else:
+            raise
+        return kind
+
+    cdef void* _make_cudaMemcpy3DParms(self, src, dst):
+        '''Private helper for data transfer. Supports all dimensions.'''
+        cdef Memcpy3DParms* param = \
+            <Memcpy3DParms*>PyMem_Malloc(sizeof(Memcpy3DParms))
+        c_memset(param, 0, sizeof(Memcpy3DParms))
+        cdef PitchedPtr srcPitchedPtr, dstPitchedPtr
+        cdef intptr_t ptr
+
+        # get kind
+        param.kind = <MemoryKind>self._get_memory_kind(src, dst)
+
+        # get src
+        if src is self:
+            # Important: cannot convert from src.ptr!
+            param.srcArray = <Array>(self.ptr)
+            param.extent = runtime.make_Extent(self.width, self.height,
+                                               self.depth)
+        else:
+            width = src.shape[-1]
+            if src.ndim >= 2:
+                height = src.shape[-2]
+            else:
+                height = 1  # same "one-stride" trick here
+
+            if isinstance(src, _ndarray_base):
+                ptr = src.data.ptr
+            else:  # numpy.ndarray
+                ptr = src.ctypes.data
+
+            srcPitchedPtr = runtime.make_PitchedPtr(
+                ptr, width*src.dtype.itemsize, width, height)
+            param.srcPtr = srcPitchedPtr
+
+        # get dst
+        if dst is self:
+            # Important: cannot convert from dst.ptr!
+            param.dstArray = <Array>(self.ptr)
+            param.extent = runtime.make_Extent(self.width, self.height,
+                                               self.depth)
+        else:
+            width = dst.shape[-1]
+            if dst.ndim >= 2:
+                height = dst.shape[-2]
+            else:
+                height = 1  # same "one-stride" trick here
+
+            if isinstance(dst, _ndarray_base):
+                ptr = dst.data.ptr
+            else:  # numpy.ndarray
+                ptr = dst.ctypes.data
+
+            dstPitchedPtr = runtime.make_PitchedPtr(
+                ptr, width*dst.dtype.itemsize, width, height)
+            param.dstPtr = dstPitchedPtr
+
+        return <void*>param
+
+    cdef void _prepare_copy(self, arr, stream, direction) except*:
+        cdef dict ch = self.desc.get_channel_format()
+
+        # sanity checks:
+        # - check shape
+        cdef int num_channels = 0
+        for key in ['x', 'y', 'z', 'w']:
+            if ch[key] > 0:
+                num_channels += 1
+
+        if self.ndim == 3:
+            if arr.shape != (self.depth, self.height, num_channels*self.width):
+                raise ValueError("shape mismatch")
+        elif self.ndim == 2:
+            if arr.shape != (self.height, num_channels*self.width):
+                raise ValueError("shape mismatch")
+        else:  # ndim = 1
+            if arr.shape != (num_channels*self.width,):
+                raise ValueError("shape mismatch")
+
+        # - check dtype
+        ch_kind = ch['f']
+        if ch_kind == runtime.cudaChannelFormatKindSigned:
+            if arr.dtype not in (numpy.int8, numpy.int16, numpy.int32):
+                raise ValueError("dtype mismatch")
+        elif ch_kind == runtime.cudaChannelFormatKindUnsigned:
+            if arr.dtype not in (numpy.uint8, numpy.uint16, numpy.uint32):
+                raise ValueError("dtype mismatch")
+        elif ch_kind == runtime.cudaChannelFormatKindFloat:
+            if arr.dtype not in (numpy.float16, numpy.float32):
+                raise ValueError("dtype mismatch")
+        else:
+            raise ValueError("dtype not supported")
+
+        cdef Memcpy3DParms* param = NULL
+
+        # Trick: For 1D or 2D CUDA arrays, we need to "fool" memcpy3D so that
+        # at least one stride gets copied. This is not properly documented in
+        # Runtime API unfortunately. See, e.g.,
+        # https://stackoverflow.com/a/39217379/2344149
+        if self.ndim == 1:
+            self.height = 1
+            self.depth = 1
+        elif self.ndim == 2:
+            self.depth = 1
+
+        if direction == 'in':
+            param = <Memcpy3DParms*>self._make_cudaMemcpy3DParms(arr, self)
+        elif direction == 'out':
+            param = <Memcpy3DParms*>self._make_cudaMemcpy3DParms(self, arr)
+
+        # we need to serialize on the current or given stream to ensure
+        # data is ready
+        cdef intptr_t stream_ptr
+        try:
+            if stream is None:
+                stream_ptr = stream_module.get_current_stream_ptr()
+            else:
+                stream_ptr = <intptr_t>(stream.ptr)
+            runtime.memcpy3DAsync(<intptr_t>param, stream_ptr)
+        except CUDARuntimeError as ex:
+            raise ex
+        finally:
+            PyMem_Free(param)
+
+            # restore old config
+            if self.ndim == 1:
+                self.height = 0
+                self.depth = 0
+            elif self.ndim == 2:
+                self.depth = 0
+
+    def copy_from(self, in_arr, stream=None):
+        '''Copy data from device or host array to CUDA array.
+
+        Args:
+            in_arr (cupy.ndarray or numpy.ndarray)
+            stream (cupy.cuda.Stream): if not ``None``, an asynchronous copy is
+                performed.
+
+        .. note::
+            For CUDA arrays with different dimensions, the requirements for the
+            shape of the input array are given as follows:
+
+                - 1D: ``(nch * width,)``
+                - 2D: ``(height, nch * width)``
+                - 3D: ``(depth, height, nch * width)``
+
+            where ``nch`` is the number of channels specified in
+            :attr:`~CUDAarray.desc`.
+        '''
+        # ensure the array is C-contiguous
+        if isinstance(in_arr, _ndarray_base):
+            prev_stream = None
+            if stream is not None:
+                prev_stream = stream_module.get_current_stream()
+                stream.use()
+            try:
+                in_arr = _internal_ascontiguousarray(in_arr)
+            finally:
+                if stream is not None:
+                    prev_stream.use()
+        elif isinstance(in_arr, numpy.ndarray):
+            in_arr = numpy.ascontiguousarray(in_arr)
+        self._prepare_copy(in_arr, stream, direction='in')
+
+    def copy_to(self, out_arr, stream=None):
+        '''Copy data from CUDA array to device or host array.
+
+        Args:
+            out_arr (cupy.ndarray or numpy.ndarray): must be C-contiguous
+            stream (cupy.cuda.Stream): if not ``None``, an asynchronous copy is
+                performed.
+
+        .. note::
+            For CUDA arrays with different dimensions, the requirements for the
+            shape of the output array are given as follows:
+
+                - 1D: ``(nch * width,)``
+                - 2D: ``(height, nch * width)``
+                - 3D: ``(depth, height, nch * width)``
+
+            where ``nch`` is the number of channels specified in
+            :attr:`~CUDAarray.desc`.
+        '''
+        if not out_arr.flags.c_contiguous:
+            raise ValueError('The output array must be C-contiguous')
+        self._prepare_copy(out_arr, stream, direction='out')
+
+
+cdef class TextureObject:
+    '''A class that holds a texture object. Equivalent to
+    ``cudaTextureObject_t``. The returned :class:`TextureObject` instance can
+    be passed as a argument when launching :class:`~cupy.RawKernel` or
+    :class:`~cupy.ElementwiseKernel`.
+
+    Args:
+        ResDesc (ResourceDescriptor): an intance of the resource descriptor.
+        TexDesc (TextureDescriptor): an instance of the texture descriptor.
+
+    .. seealso:: `cudaCreateTextureObject()`_
+
+    .. _cudaCreateTextureObject():
+        https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TEXTURE__OBJECT.html#group__CUDART__TEXTURE__OBJECT_1g16ac75814780c3a16e4c63869feb9ad3
+    '''  # noqa
+    def __init__(self, ResourceDescriptor ResDesc, TextureDescriptor TexDesc):
+        self.ptr = runtime.createTextureObject(ResDesc.ptr, TexDesc.ptr)
+        self.ResDesc = ResDesc
+        self.TexDesc = TexDesc
+
+    def __dealloc__(self):
+        if self.ptr != 0:
+            runtime.destroyTextureObject(self.ptr)
+            self.ptr = 0
+
+
+cdef class SurfaceObject:
+    '''A class that holds a surface object. Equivalent to
+    ``cudaSurfaceObject_t``. The returned :class:`SurfaceObject` instance can
+    be passed as a argument when launching :class:`~cupy.RawKernel`.
+
+    Args:
+        ResDesc (ResourceDescriptor): an intance of the resource descriptor.
+
+    .. seealso:: `cudaCreateSurfaceObject()`_
+
+    .. _cudaCreateSurfaceObject():
+        https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__SURFACE__OBJECT.html#group__CUDART__SURFACE__OBJECT_1g958899474ab2c5f40d233b524d6c5a01
+    '''  # noqa
+    def __init__(self, ResourceDescriptor ResDesc):
+        self.ptr = runtime.createSurfaceObject(ResDesc.ptr)
+        self.ResDesc = ResDesc
+
+    def __dealloc__(self):
+        if self.ptr != 0:
+            runtime.destroySurfaceObject(self.ptr)
+            self.ptr = 0
diff --git a/cupy/cuda/thrust.pyx b/cupy/cuda/thrust.pyx
new file mode 100644
index 0000000..380fef8
--- /dev/null
+++ b/cupy/cuda/thrust.pyx
@@ -0,0 +1,134 @@
+# distutils: language = c++
+
+"""Thin wrapper of Thrust implementations for CuPy API."""
+
+from libc.stdint cimport intptr_t
+from libcpp cimport vector
+
+from cupy.cuda cimport common
+from cupy.cuda cimport memory
+from cupy.cuda cimport stream
+
+
+###############################################################################
+# Memory Management
+###############################################################################
+
+# Before attempting to refactor this part, read the discussion in #3212 first.
+
+cdef class _MemoryManager:
+    cdef:
+        dict memory
+
+    def __init__(self):
+        self.memory = dict()
+
+
+cdef public char* cupy_malloc(void *m, size_t size) with gil:
+    if size == 0:
+        return <char *>0
+    cdef _MemoryManager mm = <_MemoryManager>m
+    mem = memory.alloc(size)
+    mm.memory[mem.ptr] = mem
+    return <char *>mem.ptr
+
+
+cdef public void cupy_free(void *m, char* ptr) with gil:
+    if ptr == <char *>0:
+        return
+    cdef _MemoryManager mm = <_MemoryManager>m
+    del mm.memory[<size_t>ptr]
+
+
+###############################################################################
+# Extern
+###############################################################################
+
+cdef extern from 'cupy_thrust.h':
+    void thrust_sort(int, void *, size_t *, const vector.vector[ptrdiff_t]&,
+                     intptr_t, void *)
+    void thrust_lexsort(
+        int, size_t *, void *, size_t, size_t, intptr_t, void *)
+    void thrust_argsort(int, size_t *, void *, void *,
+                        const vector.vector[ptrdiff_t]&, intptr_t, void *)
+
+    # Build-time version
+    int THRUST_VERSION
+
+
+###############################################################################
+# Python interface
+###############################################################################
+
+
+available = True
+
+
+def get_build_version():
+    return THRUST_VERSION
+
+
+cpdef sort(dtype, intptr_t data_start, intptr_t keys_start,
+           const vector.vector[ptrdiff_t]& shape) except +:
+    cdef void* _data_start = <void*>data_start
+    cdef size_t* _keys_start = <size_t*>keys_start
+    cdef intptr_t _strm = stream.get_current_stream_ptr()
+    cdef _MemoryManager mem_obj = _MemoryManager()
+    cdef void* mem = <void*>mem_obj
+
+    cdef int dtype_id
+    try:
+        dtype_id = common._get_dtype_id(dtype)
+    except ValueError:
+        raise NotImplementedError('Sorting arrays with dtype \'{}\' is not '
+                                  'supported'.format(dtype))
+    if dtype_id == 8 and not common._is_fp16_supported():
+        raise RuntimeError('either the GPU or the CUDA Toolkit does not '
+                           'support fp16')
+
+    thrust_sort(dtype_id, _data_start, _keys_start, shape, _strm, mem)
+
+
+cpdef lexsort(dtype, intptr_t idx_start, intptr_t keys_start,
+              size_t k, size_t n) except +:
+    cdef size_t* idx_ptr = <size_t*>idx_start
+    cdef void* keys_ptr = <void*>keys_start
+    cdef intptr_t _strm = stream.get_current_stream_ptr()
+    cdef _MemoryManager mem_obj = _MemoryManager()
+    cdef void* mem = <void*>mem_obj
+
+    cdef int dtype_id
+    try:
+        dtype_id = common._get_dtype_id(dtype)
+    except ValueError:
+        raise TypeError('Sorting keys with dtype \'{}\' is not '
+                        'supported'.format(dtype))
+    if dtype_id == 8 and not common._is_fp16_supported():
+        raise RuntimeError('either the GPU or the CUDA Toolkit does not '
+                           'support fp16')
+
+    thrust_lexsort(dtype_id, idx_ptr, keys_ptr, k, n, _strm, mem)
+
+
+cpdef argsort(dtype, intptr_t idx_start, intptr_t data_start,
+              intptr_t keys_start,
+              const vector.vector[ptrdiff_t]& shape) except +:
+    cdef size_t*_idx_start = <size_t*>idx_start
+    cdef void* _data_start = <void*>data_start
+    cdef size_t* _keys_start = <size_t*>keys_start
+    cdef intptr_t _strm = stream.get_current_stream_ptr()
+    cdef _MemoryManager mem_obj = _MemoryManager()
+    cdef void* mem = <void *>mem_obj
+
+    cdef int dtype_id
+    try:
+        dtype_id = common._get_dtype_id(dtype)
+    except ValueError:
+        raise NotImplementedError('Sorting arrays with dtype \'{}\' is not '
+                                  'supported'.format(dtype))
+    if dtype_id == 8 and not common._is_fp16_supported():
+        raise RuntimeError('either the GPU or the CUDA Toolkit does not '
+                           'support fp16')
+
+    thrust_argsort(
+        dtype_id, _idx_start, _data_start, _keys_start, shape, _strm, mem)
diff --git a/cupy/cudnn.py b/cupy/cudnn.py
new file mode 100644
index 0000000..a4767a7
--- /dev/null
+++ b/cupy/cudnn.py
@@ -0,0 +1,8 @@
+import warnings
+
+from cupyx.cudnn import *  # NOQA
+
+
+warnings.warn(
+    'cupy.cudnn is deprecated. Use cupyx.cudnn instead',
+    DeprecationWarning)
diff --git a/cupy/cusolver.py b/cupy/cusolver.py
new file mode 100644
index 0000000..2fea5e9
--- /dev/null
+++ b/cupy/cusolver.py
@@ -0,0 +1,8 @@
+import warnings
+
+from cupyx.cusolver import *  # NOQA
+
+
+warnings.warn(
+    'cupy.cusolver is deprecated. Use cupyx.cusolver instead',
+    DeprecationWarning)
diff --git a/cupy/cusparse.py b/cupy/cusparse.py
new file mode 100644
index 0000000..1bb90e1
--- /dev/null
+++ b/cupy/cusparse.py
@@ -0,0 +1,8 @@
+import warnings
+
+from cupyx.cusparse import *  # NOQA
+
+
+warnings.warn(
+    'cupy.cusparse is deprecated. Use cupyx.cusparse instead',
+    DeprecationWarning)
diff --git a/cupy/cutensor.py b/cupy/cutensor.py
new file mode 100644
index 0000000..312b633
--- /dev/null
+++ b/cupy/cutensor.py
@@ -0,0 +1,8 @@
+import warnings
+
+from cupyx.cutensor import *  # NOQA
+
+
+warnings.warn(
+    'cupy.cutensor is deprecated. Use cupyx.cutensor instead',
+    DeprecationWarning)
diff --git a/cupy/fft/__init__.pxd b/cupy/fft/__init__.pxd
new file mode 100644
index 0000000..e69de29
diff --git a/cupy/fft/__init__.py b/cupy/fft/__init__.py
new file mode 100644
index 0000000..47cc5b8
--- /dev/null
+++ b/cupy/fft/__init__.py
@@ -0,0 +1,19 @@
+from cupy.fft._fft import fft  # NOQA
+from cupy.fft._fft import fft2  # NOQA
+from cupy.fft._fft import fftfreq  # NOQA
+from cupy.fft._fft import fftn  # NOQA
+from cupy.fft._fft import fftshift  # NOQA
+from cupy.fft._fft import hfft  # NOQA
+from cupy.fft._fft import ifft  # NOQA
+from cupy.fft._fft import ifft2  # NOQA
+from cupy.fft._fft import ifftn  # NOQA
+from cupy.fft._fft import ifftshift  # NOQA
+from cupy.fft._fft import ihfft  # NOQA
+from cupy.fft._fft import irfft  # NOQA
+from cupy.fft._fft import irfft2  # NOQA
+from cupy.fft._fft import irfftn  # NOQA
+from cupy.fft._fft import rfft  # NOQA
+from cupy.fft._fft import rfft2  # NOQA
+from cupy.fft._fft import rfftfreq  # NOQA
+from cupy.fft._fft import rfftn  # NOQA
+from cupy.fft import config  # NOQA
diff --git a/cupy/fft/_cache.pyx b/cupy/fft/_cache.pyx
new file mode 100644
index 0000000..b1448e3
--- /dev/null
+++ b/cupy/fft/_cache.pyx
@@ -0,0 +1,682 @@
+# distutils: language = c++
+
+import gc
+import warnings
+import weakref
+
+from cupy_backends.cuda.api cimport runtime
+
+import threading
+
+from cupy import _util
+from cupy.cuda import cufft
+
+
+#####################################################################
+# Internal implementation                                           #
+#####################################################################
+
+cdef object _thread_local = threading.local()
+
+
+cdef class _ThreadLocal:
+
+    cdef list per_device_cufft_cache
+
+    def __init__(self):
+        cdef int i
+        self.per_device_cufft_cache = [
+            None for i in range(runtime.getDeviceCount())]
+
+    @staticmethod
+    cdef _ThreadLocal get():
+        cdef _ThreadLocal tls
+        tls = getattr(_thread_local, 'tls', None)
+        if tls is None:
+            tls = _ThreadLocal()
+            setattr(_thread_local, 'tls', tls)
+        return tls
+
+
+cdef inline Py_ssize_t _get_plan_memsize(plan, int curr_dev=-1) except -1:
+    cdef Py_ssize_t memsize = 0
+    cdef int dev
+
+    # work_area could be None for "empty" plans...
+    if plan is not None and plan.work_area is not None:
+        if plan.gpus is not None:
+            # multi-GPU plan
+            if curr_dev == -1:
+                curr_dev = runtime.getDevice()
+            # ptr is memory.MemoryPointer, but we can't type it here
+            for dev, ptr in zip(plan.gpus, plan.work_area):
+                if dev == curr_dev:
+                    memsize = <Py_ssize_t>(ptr.mem.size)
+                    break
+            else:
+                raise RuntimeError('invalid multi-GPU plan')
+        else:
+            # single-GPU plan
+            ptr = plan.work_area
+            memsize = <Py_ssize_t>(ptr.mem.size)
+
+    return memsize
+
+
+cdef class _Node:
+    # Unfortunately cython cdef class cannot be nested, so the node class
+    # has to live outside of the linked list...
+
+    # data
+    cdef readonly tuple key
+    cdef readonly object plan
+    cdef readonly Py_ssize_t memsize
+    cdef readonly list gpus
+
+    # link
+    cdef _Node prev
+    cdef _Node next
+
+    def __init__(self, tuple key, plan=None, int curr_dev=-1):
+        self.key = key
+        self.plan = plan
+        self.memsize = _get_plan_memsize(plan, curr_dev)
+        self.gpus = plan.gpus if plan is not None else None
+
+        self.prev = None
+        self.next = None
+
+    def __repr__(self):
+        cdef str output
+        cdef str plan_type = str(type(self.plan))
+        if isinstance(self.plan, cufft.Plan1d):
+            plan_type = 'Plan1d'
+        elif isinstance(self.plan, cufft.PlanNd):
+            plan_type = 'PlanNd'
+        elif 'cupy_callback' in plan_type:
+            # <class 'cupy_callback.Plan1d'> or PlanNd
+            plan_type = plan_type.split('.')[1]
+            plan_type = plan_type[:6]
+            plan_type += ' (static)'
+        else:
+            raise TypeError('unrecognized plan type: {}'.format(
+                type(self.plan)))
+        output = 'key: {0}, plan type: {1}, memory usage: {2}'.format(
+            self.key, plan_type, self.memsize)
+        return output
+
+
+cpdef void _clear_LinkedList(_LinkedList ll):
+    """ Delete all the nodes to ensure they are cleaned up.
+
+    This serves for the purpose of destructor and is invoked by weakref's
+    finalizer, as __del__ has no effect for cdef classes (cupy/cupy#3999).
+    """
+    cdef _Node curr = ll.head
+
+    while curr.next is not ll.tail:
+        ll.remove_node(curr.next)
+    assert ll.count == 0
+
+    # remove head and tail too
+    ll.head.next = None
+    ll.tail.prev = None
+    ll.head = None
+    ll.tail = None
+
+    # make the memory released asap
+    gc.collect()
+
+
+cdef class _LinkedList:
+    # link
+    cdef _Node head
+    cdef _Node tail
+
+    # bookkeeping
+    cdef readonly size_t count
+
+    # for clean-up
+    cdef object __weakref__
+    cdef object _finalizer
+
+    def __init__(self):
+        """ A doubly linked list to be used as an LRU cache. """
+        self.head = _Node(None, None)
+        self.tail = _Node(None, None)
+        self.count = 0
+        self.head.next = self.tail
+        self.tail.prev = self.head
+
+        # the finalizer is called when clearing the cache or at exit
+        self._finalizer = weakref.finalize(self, _clear_LinkedList, self)
+
+    cdef void remove_node(self, _Node node):
+        """ Remove the node from the linked list. """
+        cdef _Node p = node.prev
+        cdef _Node n = node.next
+        p.next = n
+        n.prev = p
+        node.prev = None
+        node.next = None
+        self.count -= 1
+
+    cdef void append_node(self, _Node node):
+        """ Add a node to the tail of the linked list. """
+        cdef _Node t = self.tail
+        cdef _Node p = t.prev
+        p.next = node
+        t.prev = node
+        node.prev = p
+        node.next = t
+        self.count += 1
+
+
+#####################################################################
+# cuFFT plan cache                                                  #
+#####################################################################
+
+cdef class PlanCache:
+    """A per-thread, per-device, least recently used (LRU) cache for cuFFT
+    plans.
+
+    Args:
+        size (int): The number of plans that the cache can accommodate. The
+            default is 16. Setting this to ``-1`` will make this limit ignored.
+        memsize (int): The amount of GPU memory, in bytes, that the plans in
+            the cache will use for their work areas. Default is ``-1``, meaning
+            it is unlimited.
+        dev (int): The ID of the device that the cache targets.
+
+    .. note::
+        1. By setting either ``size`` to ``0`` (by calling :meth:`set_size`) or
+           ``memsize`` to ``0`` (by calling :meth:`set_memsize`), the cache is
+           disabled, and any operation is no-op. To re-enable it, simply set
+           a nonzero ``size`` and/or ``memsize``.
+
+        2. This class can be instantiated by users, but it is discouraged.
+           Instead, we expect the following canonical usage pattern to
+           retrieve a handle to the cache through
+           :func:`~cupy.fft.config.get_plan_cache`:
+
+           .. code-block:: python
+
+               from cupy.cuda import Device
+               from cupy.fft.config import get_plan_cache
+
+               # get the cache for device n
+               with Device(n):
+                   cache = get_plan_cache()
+                   cache.set_size(0)  # disable the cache
+
+           In particular, the cache for device ``n`` should be manipulated
+           under device ``n``'s context.
+
+        3. This class is thread-safe since by default it is created on a
+           per-thread basis. When starting a new thread, a new cache is not
+           initialized until :func:`~cupy.fft.config.get_plan_cache` is
+           called or when the constructor is manually invoked.
+
+        4. For multi-GPU plans, the plan will be added to each participating
+           GPU's cache. Upon removal (by any of the caches), the plan will
+           be removed from each participating GPU's cache.
+
+        5. This cache supports the iterator protocol, and returns a 2-tuple:
+           ``(key, node)`` starting from the most recently used plan.
+
+    """
+    # total number of plans, regardless of plan type
+    # -1: unlimited/ignored, cache size is restricted by "memsize"
+    # 0: disable cache
+    cdef Py_ssize_t size
+
+    # current number of cached plans
+    cdef Py_ssize_t curr_size
+
+    # total amount of memory for all of the cached plans
+    # -1: unlimited/ignored, cache size is restricted by "size"
+    # 0: disable cache
+    cdef Py_ssize_t memsize
+
+    # current amount of memory used by cached plans
+    cdef Py_ssize_t curr_memsize
+
+    # for collecting statistics
+    cdef size_t hits
+    cdef size_t misses
+
+    # whether the cache is enabled (True) or disabled (False)
+    cdef bint is_enabled
+
+    # the ID of the device on which the cached plans are allocated
+    cdef int dev
+
+    # key: all arguments used to construct Plan1d or PlanNd
+    # value: the node that holds the plan corresponding to the key
+    cdef dict cache
+
+    # for keeping track of least recently used plans
+    # lru.head: least recent used
+    # lru.tail: most recent used
+    cdef _LinkedList lru
+
+    # ---------------------- Python methods ---------------------- #
+
+    def __init__(self, Py_ssize_t size=16, Py_ssize_t memsize=-1, int dev=-1):
+        if runtime.runtimeGetVersion() == 11010:
+            warnings.warn('cuFFT plan cache is disabled on CUDA 11.1 due to a '
+                          'known bug, so performance may be degraded. The bug '
+                          'is fixed on CUDA 11.2+.')
+            size = 0
+        self._validate_size_memsize(size, memsize)
+        self._set_size_memsize(size, memsize)
+        self._reset()
+        self.dev = dev if dev != -1 else runtime.getDevice()
+
+    def __dealloc__(self):
+        self._cleanup()
+
+    def __getitem__(self, tuple key):
+        # no-op if cache is disabled
+        if not self.is_enabled:
+            assert (self.size == 0 or self.memsize == 0)
+            return
+
+        cdef _Node node
+        cdef list gpus
+
+        node = self.cache.get(key)
+        if node is not None:
+            # hit, move the node to the end
+            gpus = node.gpus
+            if gpus is None:
+                self._move_plan_to_end(key=None, node=node)
+            else:
+                _remove_append_multi_gpu_plan(gpus, key)
+            self.hits += 1
+            return node.plan
+        else:
+            self.misses += 1
+            raise KeyError('plan not found for key: {}'.format(key))
+
+    def __setitem__(self, tuple key, plan):
+        # no-op if cache is disabled
+        if not self.is_enabled:
+            assert (self.size == 0 or self.memsize == 0)
+            return
+
+        # First, check for the worst case: the plan is too large to fit in
+        # the cache. In this case, we leave the cache intact and return early.
+        # If we have the budget, then try to squeeze in.
+        cdef list gpus = plan.gpus
+        if gpus is None:
+            self._check_plan_fit(plan)
+            self._add_plan(key, plan)
+        else:
+            # check all device's caches
+            _check_multi_gpu_plan_fit(gpus, plan)
+            # collectively add the plan to all devices' caches
+            _add_multi_gpu_plan(gpus, key, plan)
+
+    def __delitem__(self, tuple key):
+        cdef _Node node
+        cdef list gpus
+
+        # no-op if cache is disabled
+        if not self.is_enabled:
+            assert (self.size == 0 or self.memsize == 0)
+            return
+
+        node = self.cache.get(key)
+        if node is not None:
+            gpus = node.gpus
+            if gpus is None:
+                self._remove_plan(key=None, node=node)
+            else:
+                _remove_multi_gpu_plan(gpus, key)
+            self.hits += 1
+        else:
+            self.misses += 1
+            raise KeyError('plan not found for key: {}'.format(key))
+
+    def __repr__(self):
+        # we also validate data when the cache information is needed
+        assert len(self.cache) == int(self.lru.count) == self.curr_size
+        if self.size >= 0:
+            assert self.curr_size <= self.size
+        if self.memsize >= 0:
+            assert self.curr_memsize <= self.memsize
+
+        cdef str output = ''
+        output += '------------------- cuFFT plan cache '
+        output += '(device {}) -------------------\n'.format(self.dev)
+        output += 'cache enabled? {}\n'.format(self.is_enabled)
+        output += 'current / max size   : {0} / {1} (counts)\n'.format(
+            self.curr_size,
+            '(unlimited)' if self.size == -1 else self.size)
+        output += 'current / max memsize: {0} / {1} (bytes)\n'.format(
+            self.curr_memsize,
+            '(unlimited)' if self.memsize == -1 else self.memsize)
+        output += 'hits / misses: {0} / {1} (counts)\n'.format(
+            self.hits, self.misses)
+        output += '\ncached plans (most recently used first):\n'
+
+        cdef tuple key
+        cdef _Node node
+        cdef size_t count = 0
+        for key, node in self:
+            output += str(node) + '\n'
+            count += 1
+        assert count == self.lru.count
+        return output
+
+    def __iter__(self):
+        # Traverse from the end (LRU). Unlike dict and other map-like
+        # containers, we also return the node (value) here for inspecting
+        # and testing the data structure without accidentally changing the
+        # cache order.
+        cdef _Node node = self.lru.tail
+
+        while node.prev is not self.lru.head:
+            node = node.prev
+            yield (node.key, node)
+
+    # --------------------- internal helpers --------------------- #
+
+    cdef void _reset(self):
+        self.curr_size = 0
+        self.curr_memsize = 0
+        self.hits = 0
+        self.misses = 0
+        self.cache = {}
+        self.lru = _LinkedList()
+
+    cdef void _cleanup(self):
+        # remove circular reference and kick off garbage collection by
+        # invoking the finalizer
+        self.cache.clear()
+        self.lru._finalizer()
+
+    cdef void _validate_size_memsize(
+            self, Py_ssize_t size, Py_ssize_t memsize) except*:
+        if size < -1 or memsize < -1:
+            raise ValueError('invalid input')
+
+    cdef void _set_size_memsize(self, Py_ssize_t size, Py_ssize_t memsize):
+        self.size = size
+        self.memsize = memsize
+        self.is_enabled = (size != 0 and memsize != 0)
+
+    cdef void _check_plan_fit(self, plan) except*:
+        cdef Py_ssize_t memsize = _get_plan_memsize(plan, self.dev)
+        if (memsize > self.memsize > 0):
+            raise RuntimeError('the plan memsize is too large')
+
+    # The four helpers below (_move_plan_to_end, _add_plan, _remove_plan, and
+    # _eject_until_fit) most of the time only change the internal state of the
+    # current device's cache (self); the only exception is when removing a
+    # multi-GPU plan from the caches (in _eject_until_fit).
+
+    cdef void _move_plan_to_end(self, tuple key=None, _Node node=None) except*:
+        # either key is None or node is None
+        assert (key is None) == (node is not None)
+        if node is None:
+            node = self.cache.get(key)
+
+        self.lru.remove_node(node)
+        self.lru.append_node(node)
+
+    cdef void _add_plan(self, tuple key, plan) except*:
+        cdef _Node node = _Node(key, plan, self.dev)
+        cdef _Node unwanted_node
+
+        # Now we ensure we have room to insert, check if the key already exists
+        unwanted_node = self.cache.get(key)
+        if unwanted_node is not None:
+            self._remove_plan(key=None, node=unwanted_node)
+
+        # See if the plan can fit in, if not we remove least used ones
+        self._eject_until_fit(
+            self.size - 1 if self.size != -1 else -1,
+            self.memsize - node.memsize if self.memsize != -1 else -1)
+
+        # At this point we ensure we have room to insert
+        self.lru.append_node(node)
+        self.cache[node.key] = node
+        self.curr_size += 1
+        self.curr_memsize += node.memsize
+
+    cdef void _remove_plan(self, tuple key=None, _Node node=None) except*:
+        # either key is None or node is None
+        assert (key is None) == (node is not None)
+        if node is None:
+            node = self.cache.get(key)
+        elif key is None:
+            key = node.key
+
+        self.lru.remove_node(node)
+        del self.cache[key]
+        self.curr_size -= 1
+        self.curr_memsize -= node.memsize
+
+    cdef void _eject_until_fit(
+            self, Py_ssize_t size, Py_ssize_t memsize):
+        cdef _Node unwanted_node
+        cdef list gpus
+
+        while True:
+            if (self.curr_size == 0
+                or ((self.curr_size <= size or size == -1)
+                    and (self.curr_memsize <= memsize or memsize == -1))):
+                break
+            else:
+                # remove from the front to free up space
+                unwanted_node = self.lru.head.next
+                if unwanted_node is not self.lru.tail:
+                    gpus = unwanted_node.gpus
+                    if gpus is None:
+                        self._remove_plan(key=None, node=unwanted_node)
+                    else:
+                        _remove_multi_gpu_plan(gpus, unwanted_node.key)
+
+    # -------------- helpers also exposed to Python -------------- #
+
+    cpdef set_size(self, Py_ssize_t size):
+        self._validate_size_memsize(size, self.memsize)
+        self._eject_until_fit(size, self.memsize)
+        self._set_size_memsize(size, self.memsize)
+
+    cpdef Py_ssize_t get_size(self):
+        return self.size
+
+    cpdef Py_ssize_t get_curr_size(self):
+        return self.curr_size
+
+    cpdef set_memsize(self, Py_ssize_t memsize):
+        self._validate_size_memsize(self.size, memsize)
+        self._eject_until_fit(self.size, memsize)
+        self._set_size_memsize(self.size, memsize)
+
+    cpdef Py_ssize_t get_memsize(self):
+        return self.memsize
+
+    cpdef Py_ssize_t get_curr_memsize(self):
+        return self.curr_memsize
+
+    cpdef get(self, tuple key, default=None):
+        # behaves as if calling dict.get()
+        try:
+            plan = self[key]
+        except KeyError:
+            plan = default
+        else:
+            # if cache is disabled, plan can be None
+            if plan is None:
+                plan = default
+        return plan
+
+    cpdef clear(self):
+        self._cleanup()
+        self._reset()
+
+    cpdef show_info(self):
+        print(self)
+
+
+# The three functions below are used to collectively add, remove, or move a
+# a multi-GPU plan in all devices' caches (per thread). Therefore, they're
+# not PlanCache's methods, which focus on the current (device's) cache. This
+# module-level definition has an additional benefit that "cdef inline ..."
+# can work.
+
+cdef inline void _add_multi_gpu_plan(list gpus, tuple key, plan) except*:
+    cdef int dev
+    cdef PlanCache cache
+    cdef list insert_ok = []
+
+    try:
+        for dev in gpus:
+            prev_device = runtime.getDevice()
+            try:
+                runtime.setDevice(dev)
+                cache = get_plan_cache()
+                cache._add_plan(key, plan)
+            finally:
+                runtime.setDevice(prev_device)
+            insert_ok.append(dev)
+    except Exception as e:
+        # clean up and raise
+        _remove_multi_gpu_plan(insert_ok, key)
+        x = RuntimeError('Insert succeeded only on devices {0}:\n'
+                         '{1}'.format(insert_ok, e))
+        raise x.with_traceback(e.__traceback__)
+    assert len(insert_ok) == len(gpus)
+
+
+cdef inline void _remove_multi_gpu_plan(list gpus, tuple key) except*:
+    """ Removal of a multi-GPU plan is triggered when any of the participating
+    devices removes the plan from its cache.
+    """
+    cdef int dev
+    cdef PlanCache cache
+
+    for dev in gpus:
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(dev)
+            cache = get_plan_cache()
+            cache._remove_plan(key=key)
+        finally:
+            runtime.setDevice(prev_device)
+
+
+cdef inline void _remove_append_multi_gpu_plan(list gpus, tuple key) except *:
+    cdef int dev
+    cdef PlanCache cache
+
+    for dev in gpus:
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(dev)
+            cache = get_plan_cache()
+            cache._move_plan_to_end(key=key)
+        finally:
+            runtime.setDevice(prev_device)
+
+
+cdef inline void _check_multi_gpu_plan_fit(list gpus, plan) except*:
+    cdef int dev
+    cdef PlanCache cache
+
+    try:
+        for dev in gpus:
+            prev_device = runtime.getDevice()
+            try:
+                runtime.setDevice(dev)
+                cache = get_plan_cache()
+                cache._check_plan_fit(plan)
+            finally:
+                runtime.setDevice(prev_device)
+    except RuntimeError as e:
+        e.args = (e.args[0] + ' for device {}'.format(cache.dev),)
+        raise e
+
+
+#####################################################################
+# Public API                                                        #
+#####################################################################
+
+cpdef inline PlanCache get_plan_cache():
+    """Get the per-thread, per-device plan cache, or create one if not found.
+
+    .. seealso::
+        :class:`~cupy.fft._cache.PlanCache`
+
+    """
+    cdef _ThreadLocal tls = _ThreadLocal.get()
+    cdef int dev = runtime.getDevice()
+    cdef PlanCache cache = tls.per_device_cufft_cache[dev]
+    if cache is None:
+        # not found, do a default initialization
+        cache = PlanCache(dev=dev)
+        tls.per_device_cufft_cache[dev] = cache
+    return cache
+
+
+# TODO(leofang): remove experimental warning when scipy/scipy#12512 is merged
+cpdef Py_ssize_t get_plan_cache_size():
+    _util.experimental('cupy.fft.cache.get_plan_cache_size')
+    cdef PlanCache cache = get_plan_cache()
+    return cache.get_size()
+
+
+# TODO(leofang): remove experimental warning when scipy/scipy#12512 is merged
+cpdef set_plan_cache_size(size):
+    _util.experimental('cupy.fft.cache.set_plan_cache_size')
+    cdef PlanCache cache = get_plan_cache()
+    cache.set_size(size)
+
+
+# TODO(leofang): remove experimental warning when scipy/scipy#12512 is merged
+cpdef Py_ssize_t get_plan_cache_max_memsize():
+    _util.experimental('cupy.fft.cache.get_plan_cache_max_memsize')
+    cdef PlanCache cache = get_plan_cache()
+    return cache.get_memsize()
+
+
+# TODO(leofang): remove experimental warning when scipy/scipy#12512 is merged
+cpdef set_plan_cache_max_memsize(size):
+    _util.experimental('cupy.fft.cache.set_plan_cache_max_memsize')
+    cdef PlanCache cache = get_plan_cache()
+    cache.set_memsize(size)
+
+
+# TODO(leofang): remove experimental warning when scipy/scipy#12512 is merged
+cpdef clear_plan_cache():
+    _util.experimental('cupy.fft.cache.clear_plan_cache')
+    cdef PlanCache cache = get_plan_cache()
+    cache.clear()
+
+
+cpdef show_plan_cache_info():
+    """Show all of the plan caches' info on this thread.
+
+    .. seealso::
+        :class:`~cupy.fft._cache.PlanCache`
+
+    """
+
+    cdef _ThreadLocal tls = _ThreadLocal.get()
+    cdef list caches = tls.per_device_cufft_cache
+    cdef int dev
+    cdef PlanCache cache
+
+    print('=============== cuFFT plan cache info (all devices) '
+          '===============')
+    for dev, cache in enumerate(caches):
+        if cache is None:
+            print('------------------- cuFFT plan cache '
+                  '(device {}) -------------------'.format(dev))
+            print('(uninitialized)\n')
+        else:
+            cache.show_info()
diff --git a/cupy/fft/_callback.pyx b/cupy/fft/_callback.pyx
new file mode 100644
index 0000000..08cb824
--- /dev/null
+++ b/cupy/fft/_callback.pyx
@@ -0,0 +1,605 @@
+from libc.stdint cimport intptr_t
+
+from cupy_backends.cuda.api cimport runtime
+from cupy._core.core cimport _ndarray_base
+from cupy.cuda.device cimport get_compute_capability
+
+import hashlib
+import importlib
+import os
+import shutil
+import string
+import subprocess
+import sys
+import sysconfig
+import tempfile
+import threading
+import warnings
+
+from cupy import __version__ as _cupy_ver
+from cupy._environment import (get_nvcc_path, get_cuda_path)
+from cupy.cuda.compiler import (_get_bool_env_variable, CompileException)
+# for some reason we can't cimport stuff from cupy.cuda.cufft...
+from cupy.cuda.cufft import (CUFFT_C2C, CUFFT_C2R, CUFFT_R2C,
+                             CUFFT_Z2Z, CUFFT_Z2D, CUFFT_D2Z,
+                             CUFFT_CB_LD_COMPLEX, CUFFT_CB_LD_COMPLEX_DOUBLE,
+                             CUFFT_CB_LD_REAL, CUFFT_CB_LD_REAL_DOUBLE,
+                             CUFFT_CB_ST_COMPLEX, CUFFT_CB_ST_COMPLEX_DOUBLE,
+                             CUFFT_CB_ST_REAL, CUFFT_CB_ST_REAL_DOUBLE,)
+from cupy.cuda.cufft import getVersion as get_cufft_version
+
+
+# information needed for building an external module
+cdef list _cc = []
+cdef str _python_include = None
+cdef list _nvcc = []
+cdef str _cuda_path = None
+cdef str _cuda_include = None
+cdef str _nvprune = None
+cdef str _build_ver = None
+cdef int _cufft_ver = 0
+cdef str _cupy_root = None
+cdef str _cupy_include = None
+cdef str _source_dir = None
+cdef str _ext_suffix = None
+
+
+# callback related stuff
+cdef bint _is_init = False
+cdef str _callback_dev_code = None
+cdef str _callback_cache_dir = None
+cdef dict _callback_mgr = {}  # keep the Python modules alive
+cdef object _callback_thread_local = threading.local()
+
+
+cdef class _ThreadLocal:
+    cdef _CallbackManager _current_cufft_callback
+
+    def __init__(self):
+        self._current_cufft_callback = None
+
+    @staticmethod
+    cdef _ThreadLocal get():
+        cdef _ThreadLocal tls
+        try:
+            tls = _callback_thread_local.tls
+        except AttributeError:
+            tls = _callback_thread_local.tls = _ThreadLocal()
+        return tls
+
+
+cdef inline void _set_vars() except*:
+    global _cc, _python_include, _cuda_path, _cuda_include, _nvprune
+    global _build_ver, _cufft_ver, _ext_suffix, _is_init
+
+    cdef str cxx = sysconfig.get_config_var('CXX')
+    if cxx is not None:
+        _cc = cxx.split(' ')
+    elif 'CXX' in os.environ:
+        _cc = os.environ.get('CXX').split(' ')
+    else:
+        _cc = None
+
+    _python_include = sysconfig.get_path('include')
+
+    _cuda_path = get_cuda_path()
+    if _cuda_path is not None:
+        _cuda_include = os.path.join(_cuda_path, 'include')
+        _nvprune = os.path.join(_cuda_path, 'bin/nvprune')
+        if not os.path.isfile(_nvprune):
+            _nvprune = None
+
+    _build_ver = str(runtime.runtimeGetVersion())
+    _cufft_ver = get_cufft_version()
+    _ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
+
+    _set_cupy_paths()
+    _set_nvcc_path()
+    _is_init = True
+
+
+cdef inline void _set_cupy_paths() except*:
+    # Workaround: older Cython cannot use __file__ in global scope
+    global _cupy_root, _cupy_include, _source_dir
+    if _cupy_root is None:
+        _cupy_root = os.path.join(os.path.dirname(__file__), '..')
+        _cupy_include = os.path.join(_cupy_root, '_core/include')
+        _source_dir = os.path.join(_cupy_root, 'cuda')
+
+
+cdef inline void _set_nvcc_path() except*:
+    # get_nvcc_path() could be a long string like "ccache nvcc ..." from NVCC
+    cdef str nvcc = None
+    cdef str f
+    cdef int i
+
+    global _nvcc
+    if _nvcc == []:
+        nvcc = get_nvcc_path()
+        if nvcc is not None:
+            _nvcc = nvcc.split(' ')
+            # if the host compiler is set in NVCC, we force it to align with
+            # the one reported by sysconfig
+            for i, f in enumerate(_nvcc):
+                if f in ('-ccbin', '--compiler-bindir'):
+                    _nvcc[i+1] = _cc[0]
+                    break
+            else:
+                _nvcc += ['-ccbin', _cc[0]]
+        else:
+            _nvcc = None
+
+
+cdef inline void _sanity_checks(
+        str cb_load, str cb_store,
+        _ndarray_base cb_load_aux_arr, _ndarray_base cb_store_aux_arr) except*:
+    if runtime._is_hip_environment:
+        raise RuntimeError('hipFFT does not support callbacks')
+    if not sys.platform.startswith('linux'):
+        raise RuntimeError('cuFFT callbacks are only available on Linux')
+    if not (sys.maxsize > 2**32):
+        raise RuntimeError('cuFFT callbacks require 64 bit OS')
+    if not cb_load and not cb_store:
+        raise ValueError('need to specify either cb_load or cb_store, '
+                         'or both')
+    if cb_load and 'd_loadCallbackPtr' not in cb_load:
+        raise ValueError('need to specify d_loadCallbackPtr in cb_load')
+    if cb_store and 'd_storeCallbackPtr' not in cb_store:
+        raise ValueError('need to specify d_storeCallbackPtr in cb_store')
+    if _cc is None:
+        raise RuntimeError('a C++ compiler is required but not found, '
+                           'please set the environment variable CXX')
+    if _nvcc is None:
+        raise RuntimeError('nvcc is required but not found')
+    if _nvprune is None:
+        warnings.warn('nvprune is not found', RuntimeWarning)
+    if cb_load_aux_arr is not None:
+        if not cb_load:
+            raise ValueError('load callback is not given')
+    if cb_store_aux_arr is not None:
+        if not cb_store:
+            raise ValueError('store callback is not given')
+
+
+cdef inline void _cythonize(str tempdir, str mod_name) except*:
+    shutil.copyfile(os.path.join(_source_dir, 'cupy_cufft.h'),
+                    os.path.join(tempdir, 'cupy_cufft.h'))
+    shutil.copyfile(os.path.join(_source_dir, 'cufft.pxd'),
+                    os.path.join(tempdir, mod_name + '.pxd'))
+    shutil.copyfile(os.path.join(_source_dir, 'cufft.pyx'),
+                    os.path.join(tempdir, mod_name + '.pyx'))
+    p = subprocess.run(['cython', '-3', '--cplus',
+                        '-E', 'CUPY_HIP_VERSION=0',
+                        '-E', 'CUPY_CUDA_VERSION=' + _build_ver,
+                        '-E', 'CUPY_CUFFT_STATIC=True',
+                        os.path.join(tempdir, mod_name + '.pyx'),
+                        '-o', os.path.join(tempdir, mod_name + '.cpp')],
+                       env=os.environ, cwd=tempdir)
+    p.check_returncode()
+
+
+cdef inline void _mod_compile(str tempdir, str mod_name, str obj_host) except*:
+    shutil.copyfile(os.path.join(_source_dir, 'cupy_cufftXt.h'),
+                    os.path.join(tempdir, 'cupy_cufftXt.h'))
+    p = subprocess.run(_cc + [
+                       '-I' + _python_include,
+                       '-I' + _cuda_include,
+                       '-I' + _cupy_include,
+                       '-fPIC', '-O2', '-std=c++11',
+                       '-c', os.path.join(tempdir, mod_name + '.cpp'),
+                       '-o', obj_host],
+                       env=os.environ, cwd=tempdir)
+    p.check_returncode()
+
+
+# Archs with the same major CC version could share the implementations, so for
+# the purpose of pruning we should dump all SMs with the same major ver.
+#
+# In practice, this only happens with sm_86 (cupy/cupy#5508) as of CUDA 11.4;
+# other SMs always use sm_X0's symbols only and do not have their own
+# specializations.
+cdef dict _cc_major_map = {
+    '9': ('90',),
+    '8': ('80', '86', '87'),
+    '7': ('70', '72', '75'),
+    '6': ('60', '61', '62'),
+    '5': ('50', '52', '53'),
+    '3': ('35', '37'),
+}
+
+
+cdef inline str _prune(str temp_dir, str cache_dir, str _cufft_ver, str arch):
+    cdef str cufft_lib_full, cufft_lib_pruned, cufft_lib_temp, cufft_lib_cached
+
+    if _nvprune:
+        cufft_lib_full = os.path.join(_cuda_path, 'lib64/libcufft_static.a')
+        cufft_lib_pruned = f'cufft_static_{_cufft_ver}_sm{arch[0]}'
+        cufft_lib_temp = os.path.join(temp_dir,
+                                      'lib' + cufft_lib_pruned + '.a')
+        cufft_lib_cached = os.path.join(cache_dir,
+                                        'lib' + cufft_lib_pruned + '.a')
+        if not os.path.isfile(cufft_lib_cached):
+            comm = [_nvprune]
+            for cc in _cc_major_map[arch[0]]:
+                comm.append(f'--generate-code=arch=compute_{cc},code=sm_{cc}')
+            comm += [cufft_lib_full, '-o', cufft_lib_temp]
+            p = subprocess.run(comm,
+                               env=os.environ, cwd=temp_dir,
+                               stderr=subprocess.PIPE)
+            p.check_returncode()
+            if p.stderr:
+                # maybe this is a new arch that we haven't seen or unsupported
+                # by the current CUDA ver?
+                err = f'nvprune failed with sm_{arch}:\n{p.stderr}'
+                raise RuntimeError(err)
+            else:
+                # atomic move with the destination guaranteed to be overwritten
+                # (using os.replace() is also ok here)
+                os.rename(cufft_lib_temp, cufft_lib_cached)
+    else:
+        # nvprune is not found, just link against the full static lib
+        cufft_lib_pruned = None
+
+    return cufft_lib_pruned
+
+
+cdef inline void _nvcc_compile(
+        str tempdir, str mod_name, str cb_load, str cb_store,
+        str obj_dev, str arch) except*:
+    cdef str support
+    cdef list cmd
+    global _callback_dev_code
+
+    if _callback_dev_code is None:
+        with open(os.path.join(_source_dir, 'cupy_cufftXt.cu')) as f:
+            support = _callback_dev_code = f.read()
+    else:
+        support = _callback_dev_code
+    with open(os.path.join(tempdir, 'cupy_cufftXt.cu'), 'w') as f:
+        support = string.Template(support).substitute(
+            dev_load_callback_ker=cb_load,
+            dev_store_callback_ker=cb_store)
+        f.write(support)
+    cmd = _nvcc + ['-arch=sm_'+arch, '-dc',
+                   '-I' + _cupy_include,
+                   '-c', os.path.join(tempdir, 'cupy_cufftXt.cu'),
+                   '-Xcompiler', '-fPIC', '-O2', '-std=c++11']
+    if cb_load:
+        cmd.append('-DHAS_LOAD_CALLBACK')
+    if cb_store:
+        cmd.append('-DHAS_STORE_CALLBACK')
+    p = subprocess.run(cmd + ['-o', obj_dev],
+                       env=os.environ, cwd=tempdir,
+                       stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE)
+    try:
+        p.check_returncode()
+    except subprocess.CalledProcessError as e:
+        cex = CompileException(
+            str(e) + '\nStderr: ' + e.stderr.decode(), support,
+            os.path.join(tempdir, 'cupy_cufftXt.cu'),
+            cmd[1:], 'nvcc')
+        dump = _get_bool_env_variable(
+            'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+        if dump:
+            cex.dump(sys.stderr)
+        raise cex
+
+
+cdef inline void _nvcc_link(
+        str tempdir, str obj_host, str obj_dev, str arch, str mod_filename,
+        str cache_dir, str cufft_lib_pruned) except*:
+    # WARNING: CANNOT use host compiler to link!
+    cdef list cmd
+    cdef str mod_temp, mod_cached
+
+    mod_temp = os.path.join(tempdir, mod_filename)
+    mod_cached = os.path.join(cache_dir, mod_filename)
+    cmd = _nvcc + ['-shared', '-arch=sm_'+arch, obj_dev, obj_host]
+    if cufft_lib_pruned:
+        cmd += ['-L'+cache_dir, '-l'+cufft_lib_pruned]
+    else:
+        cmd.append('-lcufft_static')
+    cmd += ['-lculibos', '-lpthread', '-o', mod_temp]
+
+    p = subprocess.run(cmd, env=os.environ, cwd=tempdir)
+    p.check_returncode()
+    # atomic move with the destination guaranteed to be overwritten;
+    # using os.replace() is also ok here
+    os.rename(mod_temp, mod_cached)
+
+
+# make it a plain Python function so that we can mock-test it
+def get_cache_dir():
+    global _callback_cache_dir
+    if _callback_cache_dir is None:
+        _callback_cache_dir = os.environ.get(
+            'CUPY_CACHE_DIR', os.path.expanduser('~/.cupy/callback_cache'))
+    return _callback_cache_dir
+
+
+cpdef get_current_callback_manager():
+    cdef _ThreadLocal tls = _ThreadLocal.get()
+    cdef _CallbackManager mgr = tls._current_cufft_callback
+    return mgr
+
+
+cdef class _CallbackManager:
+    cdef:
+        readonly str cb_load
+        readonly str cb_store
+        readonly _ndarray_base cb_load_aux_arr
+        readonly _ndarray_base cb_store_aux_arr
+        object mod
+
+    def __init__(self,
+                 str cb_load='',
+                 str cb_store='',
+                 _ndarray_base cb_load_aux_arr=None,
+                 _ndarray_base cb_store_aux_arr=None):
+        if not _is_init:
+            _set_vars()
+        _sanity_checks(cb_load, cb_store,
+                       cb_load_aux_arr, cb_store_aux_arr)
+
+        self.cb_load = cb_load
+        self.cb_store = cb_store
+        self.cb_load_aux_arr = cb_load_aux_arr
+        self.cb_store_aux_arr = cb_store_aux_arr
+
+        # Set up some variables...
+        cdef str arch = get_compute_capability()
+        cdef str tempdir
+        cdef str obj_host
+        cdef str obj_dev
+        cdef str mod_name
+        cdef str mod_filename
+        cdef str cache_dir
+        cdef str path
+        cdef str cufft_lib_pruned
+
+        # For hash; note this is independent of the plan to be created, and
+        # only depends on the given load/store callbacks and the runtime
+        # environment
+        keys = (_cc, _nvcc, arch, _build_ver, _cufft_ver, _cupy_ver,
+                _python_include, cb_load, cb_store)
+        keys = '%s %s %s %s %s %s %s %s %s' % keys
+
+        # Generate module filename: all modules with the identical callbacks
+        # are considered identical regardless of which plan is actually
+        # executed at the time of generation
+        mod_name = 'cupy_callback_'
+        mod_name += hashlib.md5(keys.encode()).hexdigest()
+        mod_name = mod_name.replace('.', '')
+        mod_filename = mod_name + _ext_suffix
+
+        # Check if the module is already cached on disk. If not, we compile.
+        cache_dir = get_cache_dir()
+        if not os.path.isdir(cache_dir):
+            os.makedirs(cache_dir, exist_ok=True)
+        path = os.path.join(cache_dir, mod_filename)
+        if not os.path.isfile(path):
+            # Set up temp directory; it must be under the cache directory so
+            # that atomic moves within the same filesystem can be guaranteed
+            tempdir_obj = tempfile.TemporaryDirectory(dir=cache_dir)
+            tempdir = tempdir_obj.name
+
+            # Cythonize the Cython code to produce a c++ source file
+            _cythonize(tempdir, mod_name)
+
+            # Compile the Python module
+            obj_host = os.path.join(tempdir, mod_name + '.o')
+            _mod_compile(tempdir, mod_name, obj_host)
+
+            # Prune libcufft_static.a for the target arch and cache it
+            cufft_lib_pruned = _prune(
+                tempdir, cache_dir, str(_cufft_ver), arch)
+
+            # Dump and compile device code using nvcc
+            obj_dev = os.path.join(tempdir, mod_name + '_dev.o')
+            _nvcc_compile(tempdir, mod_name, cb_load, cb_store, obj_dev, arch)
+
+            # Use nvcc to link and generate a shared library, and place it in
+            # the disk cache
+            _nvcc_link(tempdir, obj_host, obj_dev, arch, mod_filename,
+                       cache_dir, cufft_lib_pruned)
+
+            # Clean up build directory
+            tempdir_obj.cleanup()
+
+        # Load the Python module
+        spec = importlib.util.spec_from_file_location(mod_name, path)
+        self.mod = module = importlib.util.module_from_spec(spec)
+        sys.modules[mod_name] = module
+        spec.loader.exec_module(module)
+
+    cpdef create_plan(self, tuple plan_info):
+        cdef str plan_type
+        cdef tuple plan_args
+
+        plan_type, plan_args = plan_info
+        plan = getattr(self.mod, plan_type)(*plan_args)
+        return plan
+
+    cpdef set_callbacks(self, plan):
+        '''Set the load/store callbacks by making calls to
+        ``cufftXtSetCallback``.
+
+        Args:
+            plan (:class:`~cupy.cuda.cufft.Plan1d` or
+                :class:`~cupy.cuda.cufft.PlanNd`, optional): A cuFFT plan
+                against which the load/store callbacks are set.
+
+        .. note::
+            If :meth:`set_caller_infos` is called, a call to this method must
+            follow.
+
+        '''
+        cdef _ndarray_base cb_load_aux_arr = self.cb_load_aux_arr
+        cdef _ndarray_base cb_store_aux_arr = self.cb_store_aux_arr
+        cdef intptr_t cb_load_ptr=0, cb_store_ptr=0
+
+        fft_type = plan.fft_type
+        if fft_type == CUFFT_C2C:
+            cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
+            cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
+        elif fft_type == CUFFT_R2C:
+            cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
+            cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
+        elif fft_type == CUFFT_C2R:
+            cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
+            cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
+        elif fft_type == CUFFT_Z2Z:
+            cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
+            cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
+        elif fft_type == CUFFT_D2Z:
+            cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
+            cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
+        elif fft_type == CUFFT_Z2D:
+            cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
+            cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
+        else:
+            raise ValueError
+
+        if self.cb_load:
+            if cb_load_aux_arr is not None:
+                cb_load_ptr = cb_load_aux_arr.data.ptr
+            self.mod.setCallback(
+                plan.handle, cb_load_type, True, cb_load_ptr)
+        if self.cb_store:
+            if cb_store_aux_arr is not None:
+                cb_store_ptr = cb_store_aux_arr.data.ptr
+            self.mod.setCallback(
+                plan.handle, cb_store_type, False, cb_store_ptr)
+
+    cdef set_caller_infos(self,
+                          _ndarray_base cb_load_aux_arr=None,
+                          _ndarray_base cb_store_aux_arr=None):
+        '''Set the auxilliary arrays to be used by the load/store callbacks.
+        Corresponding to the ``callerInfo`` field in cuFFT's callback API.
+
+        Args:
+            cb_load_aux_arr (:class:`cupy.ndarray`, optional): A CuPy array
+                containing data to be used in the load callback.
+            cb_store_aux_arr (:class:`cupy.ndarray`, optional): A CuPy array
+                containing data to be used in the store callback.
+
+        .. note::
+            After this method is called, a call to :meth:`set_callbacks` must
+            follow. This is for internal use.
+
+        '''
+        self.cb_load_aux_arr = cb_load_aux_arr
+        self.cb_store_aux_arr = cb_store_aux_arr
+
+
+cdef class set_cufft_callbacks:
+    """A context manager for setting up load and/or store callbacks.
+
+    Args:
+        cb_load (str): A string contains the device kernel for the load
+            callback. It must define ``d_loadCallbackPtr``.
+        cb_store (str): A string contains the device kernel for the store
+            callback. It must define ``d_storeCallbackPtr``.
+        cb_load_aux_arr (cupy.ndarray, optional): A CuPy array containing
+            data to be used in the load callback.
+        cb_store_aux_arr (cupy.ndarray, optional): A CuPy array containing
+            data to be used in the store callback.
+
+    .. note::
+        Any FFT calls living in this context will have callbacks set up. An
+        example for a load callback is shown below:
+
+        .. code-block:: python
+
+            code = r'''
+            __device__ cufftComplex CB_ConvertInputC(
+                void *dataIn,
+                size_t offset,
+                void *callerInfo,
+                void *sharedPtr) {
+              // implementation
+            }
+
+            __device__ cufftCallbackLoadC d_loadCallbackPtr = CB_ConvertInputC;
+            '''
+
+            with cp.fft.config.set_cufft_callbacks(cb_load=code):
+                out_arr = cp.fft.fft(in_arr, ...)
+
+    .. note::
+        Below are the *runtime* requirements for using this feature:
+
+            * cython >= 0.29.0
+            * A host compiler that supports C++11 and above; might need to set
+              up the ``CXX`` environment variable.
+            * ``nvcc`` and the full CUDA Toolkit. Note that the ``cudatoolkit``
+              package from Conda-Forge is not enough, as it does not contain
+              static libraries.
+
+    .. note::
+        Callbacks only work for transforms over contiguous axes; the behavior
+        for non-contiguous transforms is in general undefined.
+
+    .. warning::
+        Using cuFFT callbacks requires compiling and loading a Python module at
+        runtime as well as static linking for each distinct transform and
+        callback, so the first invocation for each combination will be very
+        slow. This is a limitation of cuFFT, so use this feature only when the
+        callback-enabled transform is known more performant and can be reused
+        to amortize the cost.
+
+    .. warning::
+        The generated Python modules are by default cached in
+        ``~/.cupy/callback_cache`` for possible reuse (with the same set of
+        load/store callbacks). Due to static linking, however, the file sizes
+        can be excessive! The cache position can be changed via setting
+        ``CUPY_CACHE_DIR``.
+
+    .. seealso:: `cuFFT Callback Routines`_
+
+    .. _cuFFT Callback Routines:
+        https://docs.nvidia.com/cuda/cufft/index.html#callback-routines
+
+    """
+    # this class should have been a simple function decorated by @contextlib.
+    # contextmanager that yields...
+
+    cdef:
+        _CallbackManager mgr
+        _CallbackManager mgr_prev
+
+    def __init__(self,
+                 str cb_load='',
+                 str cb_store='',
+                 *,
+                 _ndarray_base cb_load_aux_arr=None,
+                 _ndarray_base cb_store_aux_arr=None):
+        # For every distinct pair of load & store callbacks, we compile an
+        # external Python module and cache it.
+        cdef tuple key = (cb_load, cb_store)
+        cdef _CallbackManager mgr = _callback_mgr.get(key)
+        if mgr is None:
+            mgr = _CallbackManager(
+                cb_load=cb_load,
+                cb_store=cb_store,
+                cb_load_aux_arr=cb_load_aux_arr,
+                cb_store_aux_arr=cb_store_aux_arr)
+            _callback_mgr[key] = mgr  # keep the Python module alive
+        else:
+            mgr.set_caller_infos(
+                cb_load_aux_arr=cb_load_aux_arr,
+                cb_store_aux_arr=cb_store_aux_arr)
+        self.mgr = mgr
+
+    def __enter__(self):
+        cdef _ThreadLocal tls = _ThreadLocal.get()
+        self.mgr_prev = tls._current_cufft_callback
+        tls._current_cufft_callback = self.mgr
+        return self.mgr
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        cdef _ThreadLocal tls = _ThreadLocal.get()
+        tls._current_cufft_callback = self.mgr_prev
+        # do not remove mgr from _callback_mgr, as one might still wanna use
+        # plans generated by it in the same Python session
diff --git a/cupy/fft/_fft.py b/cupy/fft/_fft.py
new file mode 100644
index 0000000..2ecf1b4
--- /dev/null
+++ b/cupy/fft/_fft.py
@@ -0,0 +1,1103 @@
+import functools
+import math
+import warnings
+
+import numpy as np
+
+import cupy
+from cupy.cuda import cufft
+from cupy.fft import config
+from cupy.fft._cache import get_plan_cache
+
+
+_reduce = functools.reduce
+_prod = cupy._core.internal.prod
+
+
+@cupy._util.memoize()
+def _output_dtype(dtype, value_type):
+    if value_type != 'R2C':
+        if dtype in [np.float16, np.float32]:
+            return np.complex64
+        elif dtype not in [np.complex64, np.complex128]:
+            return np.complex128
+    else:
+        if dtype in [np.complex64, np.complex128]:
+            return np.dtype(dtype.char.lower())
+        elif dtype == np.float16:
+            return np.float32
+        elif dtype not in [np.float32, np.float64]:
+            return np.float64
+    return dtype
+
+
+def _convert_dtype(a, value_type):
+    out_dtype = _output_dtype(a.dtype, value_type)
+    if out_dtype != a.dtype:
+        a = a.astype(out_dtype)
+    return a
+
+
+def _cook_shape(a, s, axes, value_type, order='C'):
+    if s is None or s == a.shape:
+        return a
+    if (value_type == 'C2R') and (s[-1] is not None):
+        s = list(s)
+        s[-1] = s[-1] // 2 + 1
+    for sz, axis in zip(s, axes):
+        if (sz is not None) and (sz != a.shape[axis]):
+            shape = list(a.shape)
+            if shape[axis] > sz:
+                index = [slice(None)] * a.ndim
+                index[axis] = slice(0, sz)
+                a = a[tuple(index)]
+            else:
+                index = [slice(None)] * a.ndim
+                index[axis] = slice(0, shape[axis])
+                shape[axis] = sz
+                z = cupy.zeros(shape, a.dtype.char, order=order)
+                z[tuple(index)] = a
+                a = z
+    return a
+
+
+def _convert_fft_type(dtype, value_type):
+    if value_type == 'C2C' and dtype == np.complex64:
+        return cufft.CUFFT_C2C
+    elif value_type == 'R2C' and dtype == np.float32:
+        return cufft.CUFFT_R2C
+    elif value_type == 'C2R' and dtype == np.complex64:
+        return cufft.CUFFT_C2R
+    elif value_type == 'C2C' and dtype == np.complex128:
+        return cufft.CUFFT_Z2Z
+    elif value_type == 'R2C' and dtype == np.float64:
+        return cufft.CUFFT_D2Z
+    elif value_type == 'C2R' and dtype == np.complex128:
+        return cufft.CUFFT_Z2D
+    else:
+        raise ValueError
+
+
+def _exec_fft(a, direction, value_type, norm, axis, overwrite_x,
+              out_size=None, out=None, plan=None):
+    fft_type = _convert_fft_type(a.dtype, value_type)
+
+    if axis % a.ndim != a.ndim - 1:
+        a = a.swapaxes(axis, -1)
+
+    if a.base is not None or not a.flags.c_contiguous:
+        a = a.copy()
+    elif (
+        not cupy.cuda.runtime.is_hip and
+        value_type == 'C2R' and not overwrite_x and
+        10010 <= cupy.cuda.runtime.runtimeGetVersion()
+    ):
+        # The input array may be modified in CUDA 10.1 and above.
+        # See #3763 for the discussion.
+        a = a.copy()
+    elif cupy.cuda.runtime.is_hip and value_type != 'C2C':
+        # hipFFT's R2C would overwrite input
+        # hipFFT's C2R needs a workaround (see below)
+        a = a.copy()
+
+    n = a.shape[-1]
+    if n < 1:
+        raise ValueError(
+            'Invalid number of FFT data points (%d) specified.' % n)
+
+    # Workaround for hipFFT/rocFFT:
+    # Both cuFFT and hipFFT/rocFFT have this requirement that 0-th and
+    # N/2-th element must be real, but cuFFT internally simply ignores it
+    # while hipFFT handles it badly in both Plan1d and PlanNd, so we must
+    # do the correction ourselves to ensure the condition is met.
+    if cupy.cuda.runtime.is_hip and value_type == 'C2R':
+        a[..., 0].imag = 0
+        if out_size is None:
+            a[..., -1].imag = 0
+        elif out_size % 2 == 0:
+            a[..., out_size // 2].imag = 0
+
+    if out_size is None:
+        out_size = n
+
+    batch = a.size // n
+
+    # plan search precedence:
+    # 1. plan passed in as an argument
+    # 2. plan as context manager
+    # 3. cached plan
+    # 4. create a new one
+    curr_plan = cufft.get_current_plan()
+    if curr_plan is not None:
+        if plan is None:
+            plan = curr_plan
+        else:
+            raise RuntimeError('Use the cuFFT plan either as a context manager'
+                               ' or as an argument.')
+
+    if plan is None:
+        devices = None if not config.use_multi_gpus else config._devices
+        # TODO(leofang): do we need to add the current stream to keys?
+        keys = (out_size, fft_type, batch, devices)
+        mgr = config.get_current_callback_manager()
+        if mgr is not None:
+            # to avoid a weird segfault, we generate and cache distinct plans
+            # for every possible (load_aux, store_aux) pairs; the plans are
+            # still generated from the same external Python module
+            load_aux = mgr.cb_load_aux_arr
+            store_aux = mgr.cb_store_aux_arr
+            keys += (mgr.cb_load, mgr.cb_store,
+                     0 if load_aux is None else load_aux.data.ptr,
+                     0 if store_aux is None else store_aux.data.ptr)
+        cache = get_plan_cache()
+        cached_plan = cache.get(keys)
+        if cached_plan is not None:
+            plan = cached_plan
+        elif mgr is None:
+            plan = cufft.Plan1d(out_size, fft_type, batch, devices=devices)
+            cache[keys] = plan
+        else:  # has callback
+            # TODO(leofang): support multi-GPU callback (devices is ignored)
+            if devices:
+                raise NotImplementedError('multi-GPU cuFFT callbacks are not '
+                                          'yet supported')
+            plan = mgr.create_plan(('Plan1d', keys[:-5]))
+            mgr.set_callbacks(plan)
+            cache[keys] = plan
+    else:
+        # check plan validity
+        if not isinstance(plan, cufft.Plan1d):
+            raise ValueError('expected plan to have type cufft.Plan1d')
+        if fft_type != plan.fft_type:
+            raise ValueError('cuFFT plan dtype mismatch.')
+        if out_size != plan.nx:
+            raise ValueError('Target array size does not match the plan.',
+                             out_size, plan.nx)
+        if batch != plan.batch:
+            raise ValueError('Batch size does not match the plan.')
+        if config.use_multi_gpus != (plan.gpus is not None):
+            raise ValueError('Unclear if multiple GPUs are to be used or not.')
+
+    if overwrite_x and value_type == 'C2C':
+        out = a
+    elif out is not None:
+        # verify that out has the expected shape and dtype
+        plan.check_output_array(a, out)
+    else:
+        out = plan.get_output_array(a)
+
+    if batch != 0:
+        plan.fft(a, out, direction)
+
+    sz = out.shape[-1]
+    if fft_type == cufft.CUFFT_R2C or fft_type == cufft.CUFFT_D2Z:
+        sz = n
+    if norm == 'backward' and direction == cufft.CUFFT_INVERSE:
+        out /= sz
+    elif norm == 'ortho':
+        out /= math.sqrt(sz)
+    elif norm == 'forward' and direction == cufft.CUFFT_FORWARD:
+        out /= sz
+
+    if axis % a.ndim != a.ndim - 1:
+        out = out.swapaxes(axis, -1)
+
+    return out
+
+
+def _fft_c2c(a, direction, norm, axes, overwrite_x, plan=None):
+    for axis in axes:
+        a = _exec_fft(a, direction, 'C2C', norm, axis, overwrite_x, plan=plan)
+    return a
+
+
+def _fft(a, s, axes, norm, direction, value_type='C2C', overwrite_x=False,
+         plan=None):
+    if not isinstance(a, cupy.ndarray):
+        raise TypeError('The input array a must be a cupy.ndarray')
+    if (s is not None) and (axes is not None) and len(s) != len(axes):
+        raise ValueError('Shape and axes have different lengths.')
+    if axes is None:
+        if s is None:
+            dim = a.ndim
+        else:
+            dim = len(s)
+        axes = [i for i in range(-dim, 0)]
+    else:
+        axes = tuple(axes)
+    if not axes:
+        if value_type == 'C2C':
+            return a
+        else:
+            raise IndexError('list index out of range')
+    if norm is None:  # for backward compatibility
+        norm = 'backward'
+    # it is important that we check norm after validating axes for NumPy
+    # compatibility: if axes=(), early return is triggered and norm is not
+    # checked...
+    if norm not in ('backward', 'ortho', 'forward'):
+        raise ValueError('Invalid norm value %s, should be "backward", '
+                         '"ortho", or "forward".' % norm)
+    a = _convert_dtype(a, value_type)
+    a = _cook_shape(a, s, axes, value_type)
+
+    if value_type == 'C2C':
+        a = _fft_c2c(a, direction, norm, axes, overwrite_x, plan=plan)
+    elif value_type == 'R2C':
+        a = _exec_fft(a, direction, value_type, norm, axes[-1], overwrite_x)
+        a = _fft_c2c(a, direction, norm, axes[:-1], overwrite_x)
+    else:  # C2R
+        a = _fft_c2c(a, direction, norm, axes[:-1], overwrite_x)
+        # _cook_shape tells us input shape only, and no output shape
+        out_size = _get_fftn_out_size(a.shape, s, axes[-1], value_type)
+        a = _exec_fft(a, direction, value_type, norm, axes[-1], overwrite_x,
+                      out_size)
+
+    return a
+
+
+def _prep_fftn_axes(ndim, s=None, axes=None, value_type='C2C'):
+    """Configure axes argument for an n-dimensional FFT.
+
+    The axes to be transformed are returned in ascending order.
+    """
+
+    # compatibility checks for cupy.cuda.cufft.PlanNd
+    if (s is not None) and (axes is not None) and len(s) != len(axes):
+        raise ValueError("Shape and axes have different lengths.")
+
+    if axes is None:
+        if s is None:
+            dim = ndim
+        else:
+            dim = len(s)
+        axes = tuple([i + ndim for i in range(-dim, 0)])
+        axes_sorted = axes
+    else:
+        axes = tuple(axes)
+        if not axes:
+            return (), ()
+        if _reduce(min, axes) < -ndim or _reduce(max, axes) > ndim - 1:
+            raise ValueError("The specified axes exceed the array dimensions.")
+        if value_type == 'C2C':
+            axes_sorted = tuple(sorted([ax % ndim for ax in axes]))
+        else:  # C2R or R2C
+            # The last axis is special, need to isolate it and append
+            # to the rest of (sorted) axes
+            axes_sorted = sorted([ax % ndim for ax in axes[:-1]])
+            axes_sorted.append(axes[-1] % ndim)
+            axes_sorted = tuple(axes_sorted)
+
+    # unsorted axes for _cook_shape, sorted ones are otherwise used
+    return axes, axes_sorted
+
+
+def _nd_plan_is_possible(axes_sorted, ndim):
+    # PlanNd supports 1D, 2D and 3D batch transforms over contiguous axes
+    # Axes must be contiguous and the first or last axis must be in the axes.
+    return (0 < len(axes_sorted) <= 3
+            and (0 in axes_sorted or (ndim - 1) in axes_sorted)
+            and all((axes_sorted[n + 1] - axes_sorted[n]) == 1
+                    for n in range(len(axes_sorted) - 1)))
+
+
+def _get_cufft_plan_nd(
+        shape, fft_type, axes=None, order='C', out_size=None, to_cache=True):
+    """Generate a CUDA FFT plan for transforming up to three axes.
+
+    Args:
+        shape (tuple of int): The shape of the array to transform
+        fft_type (int): The FFT type to perform. Supported values are:
+            `cufft.CUFFT_C2C`, `cufft.CUFFT_C2R`, `cufft.CUFFT_R2C`,
+            `cufft.CUFFT_Z2Z`, `cufft.CUFFT_Z2D`, and `cufft.CUFFT_D2Z`.
+        axes (None or int or tuple of int):  The axes of the array to
+            transform. Currently, these must be a set of up to three adjacent
+            axes and must include either the first or the last axis of the
+            array.  If `None`, it is assumed that all axes are transformed.
+        order ({'C', 'F'}): Specify whether the data to be transformed has C or
+            Fortran ordered data layout.
+        out_size (int): The output length along the last axis for R2C/C2R FFTs.
+            For C2C FFT, this is ignored (and set to `None`).
+        to_cache (bool): Whether to cache the generated plan. Default is
+            ``True``.
+
+    Returns:
+        plan (cufft.PlanNd): A cuFFT Plan for the chosen `fft_type`.
+    """
+    ndim = len(shape)
+
+    if fft_type in (cufft.CUFFT_C2C, cufft.CUFFT_Z2Z):
+        value_type = 'C2C'
+    elif fft_type in (cufft.CUFFT_C2R, cufft.CUFFT_Z2D):
+        value_type = 'C2R'
+    else:  # CUFFT_R2C or CUFFT_D2Z
+        value_type = 'R2C'
+
+    if axes is None:
+        # transform over all axes
+        fft_axes = tuple(range(ndim))
+    else:
+        _, fft_axes = _prep_fftn_axes(ndim, s=None, axes=axes,
+                                      value_type=value_type)
+
+    if not _nd_plan_is_possible(fft_axes, ndim):
+        raise ValueError(
+            "An n-dimensional cuFFT plan could not be created. The axes must "
+            "be contiguous and non-repeating. Between one and three axes can "
+            "be transformed and either the first or last axis must be "
+            "included in axes.")
+
+    if order not in ['C', 'F']:
+        raise ValueError('order must be \'C\' or \'F\'')
+
+    """
+    For full details on idist, istride, iembed, etc. see:
+    http://docs.nvidia.com/cuda/cufft/index.html#advanced-data-layout
+
+    in 1D:
+    input[b * idist + x * istride]
+    output[b * odist + x * ostride]
+
+    in 2D:
+    input[b * idist + (x * inembed[1] + y) * istride]
+    output[b * odist + (x * onembed[1] + y) * ostride]
+
+    in 3D:
+    input[b * idist + ((x * inembed[1] + y) * inembed[2] + z) * istride]
+    output[b * odist + ((x * onembed[1] + y) * onembed[2] + z) * ostride]
+    """
+    # At this point, _default_fft_func() guarantees that for F-order arrays
+    # we only need to consider C2C, and not C2R or R2C.
+    # TODO(leofang): figure out if we really have to skip F-order?
+    in_dimensions = [shape[d] for d in fft_axes]
+    if order == 'F':
+        in_dimensions = in_dimensions[::-1]
+    in_dimensions = tuple(in_dimensions)
+    if fft_type in (cufft.CUFFT_C2C, cufft.CUFFT_Z2Z):
+        out_dimensions = in_dimensions
+        plan_dimensions = in_dimensions
+    else:
+        out_dimensions = list(in_dimensions)
+        if out_size is not None:  # for C2R & R2C
+            out_dimensions[-1] = out_size  # only valid for C order!
+        out_dimensions = tuple(out_dimensions)
+        if fft_type in (cufft.CUFFT_R2C, cufft.CUFFT_D2Z):
+            plan_dimensions = in_dimensions
+        else:  # CUFFT_C2R or CUFFT_Z2D
+            plan_dimensions = out_dimensions
+    inembed = in_dimensions
+    onembed = out_dimensions
+
+    if fft_axes == tuple(range(ndim)):
+        # tranfsorm over all axes
+        nbatch = 1
+        idist = odist = 1  # doesn't matter since nbatch = 1
+        istride = ostride = 1
+    else:
+        # batch along the first or the last axis
+        if 0 not in fft_axes:
+            # don't FFT along the first min_axis_fft axes
+            min_axis_fft = _reduce(min, fft_axes)
+            nbatch = _prod(shape[:min_axis_fft])
+            if order == 'C':
+                # C-ordered GPU array with batch along first dim
+                idist = _prod(in_dimensions)
+                odist = _prod(out_dimensions)
+                istride = 1
+                ostride = 1
+            elif order == 'F':
+                # F-ordered GPU array with batch along first dim
+                idist = 1
+                odist = 1
+                istride = nbatch
+                ostride = nbatch
+        elif (ndim - 1) not in fft_axes:
+            # don't FFT along the last axis
+            num_axes_batch = ndim - len(fft_axes)
+            nbatch = _prod(shape[-num_axes_batch:])
+            if order == 'C':
+                # C-ordered GPU array with batch along last dim
+                idist = 1
+                odist = 1
+                istride = nbatch
+                ostride = nbatch
+            elif order == 'F':
+                # F-ordered GPU array with batch along last dim
+                idist = _prod(in_dimensions)
+                odist = _prod(out_dimensions)
+                istride = 1
+                ostride = 1
+        else:
+            raise ValueError(
+                'General subsets of FFT axes not currently supported for '
+                'GPU case (Can only batch FFT over the first or last '
+                'spatial axes).')
+
+    for n in plan_dimensions:
+        if n < 1:
+            raise ValueError(
+                'Invalid number of FFT data points specified.')
+
+    keys = (plan_dimensions, inembed, istride,
+            idist, onembed, ostride, odist,
+            fft_type, nbatch, order, fft_axes[-1], out_size)
+    mgr = config.get_current_callback_manager()
+    if mgr is not None:
+        # to avoid a weird segfault, we generate and cache distinct plans
+        # for every possible (load_aux, store_aux) pairs; the plans are
+        # still generated from the same external Python module
+        load_aux = mgr.cb_load_aux_arr
+        store_aux = mgr.cb_store_aux_arr
+        keys += (mgr.cb_load, mgr.cb_store,
+                 0 if load_aux is None else load_aux.data.ptr,
+                 0 if store_aux is None else store_aux.data.ptr)
+    cache = get_plan_cache()
+    cached_plan = cache.get(keys)
+    if cached_plan is not None:
+        plan = cached_plan
+    elif mgr is None:
+        plan = cufft.PlanNd(*keys)
+        if to_cache:
+            cache[keys] = plan
+    else:  # has callback
+        plan = mgr.create_plan(('PlanNd', keys[:-4]))
+        mgr.set_callbacks(plan)
+        if to_cache:
+            cache[keys] = plan
+
+    return plan
+
+
+def _get_fftn_out_size(in_shape, s, last_axis, value_type):
+    if value_type == 'C2R':
+        if (s is None) or (s[-1] is None):
+            out_size = 2 * (in_shape[last_axis] - 1)
+        else:
+            out_size = s[-1]
+    elif value_type == 'R2C':
+        out_size = in_shape[last_axis] // 2 + 1
+    else:  # C2C
+        out_size = None
+    return out_size
+
+
+def _exec_fftn(a, direction, value_type, norm, axes, overwrite_x,
+               plan=None, out=None, out_size=None):
+
+    fft_type = _convert_fft_type(a.dtype, value_type)
+
+    if a.flags.c_contiguous:
+        order = 'C'
+    elif a.flags.f_contiguous:
+        order = 'F'
+    else:
+        raise ValueError('a must be contiguous')
+
+    if (value_type == 'C2R' and not overwrite_x and
+            10010 <= cupy.cuda.runtime.runtimeGetVersion()):
+        # The input array may be modified in CUDA 10.1 and above.
+        # See #3763 for the discussion.
+        a = a.copy()
+    elif cupy.cuda.runtime.is_hip and value_type != 'C2C':
+        # hipFFT's R2C would overwrite input
+        # hipFFT's C2R PlanNd is actually not in use so it's fine here
+        a = a.copy()
+
+    # plan search precedence:
+    # 1. plan passed in as an argument
+    # 2. plan as context manager
+    # 3. cached plan
+    # 4. create a new one
+    curr_plan = cufft.get_current_plan()
+    if curr_plan is not None:
+        plan = curr_plan
+        # don't check repeated usage; it's done in _default_fft_func()
+    if plan is None:
+        # search from cache, and generate a plan if not found
+        plan = _get_cufft_plan_nd(a.shape, fft_type, axes=axes, order=order,
+                                  out_size=out_size)
+    else:
+        if not isinstance(plan, cufft.PlanNd):
+            raise ValueError('expected plan to have type cufft.PlanNd')
+        if order != plan.order:
+            raise ValueError('array orders mismatch (plan: {}, input: {})'
+                             .format(plan.order, order))
+        if a.flags.c_contiguous:
+            expected_shape = [a.shape[ax] for ax in axes]
+            if value_type == 'C2R':
+                expected_shape[-1] = out_size
+        else:
+            # plan.shape will be reversed for Fortran-ordered inputs
+            expected_shape = [a.shape[ax] for ax in axes[::-1]]
+            # TODO(leofang): modify the shape for C2R
+        expected_shape = tuple(expected_shape)
+        if expected_shape != plan.shape:
+            raise ValueError(
+                'The cuFFT plan and a.shape do not match: '
+                'plan.shape = {}, expected_shape={}, a.shape = {}'.format(
+                    plan.shape, expected_shape, a.shape))
+        if fft_type != plan.fft_type:
+            raise ValueError('cuFFT plan dtype mismatch.')
+        if value_type != 'C2C':
+            if axes[-1] != plan.last_axis:
+                raise ValueError('The last axis for R2C/C2R mismatch')
+            if out_size != plan.last_size:
+                raise ValueError('The size along the last R2C/C2R axis '
+                                 'mismatch')
+
+    # TODO(leofang): support in-place transform for R2C/C2R
+    if overwrite_x and value_type == 'C2C':
+        out = a
+    elif out is None:
+        out = plan.get_output_array(a, order=order)
+    else:
+        plan.check_output_array(a, out)
+
+    if out.size != 0:
+        plan.fft(a, out, direction)
+
+    # normalize by the product of the shape along the transformed axes
+    arr = a if fft_type in (cufft.CUFFT_R2C, cufft.CUFFT_D2Z) else out
+    sz = _prod([arr.shape[ax] for ax in axes])
+    if norm == 'backward' and direction == cufft.CUFFT_INVERSE:
+        out /= sz
+    elif norm == 'ortho':
+        out /= math.sqrt(sz)
+    elif norm == 'forward' and direction == cufft.CUFFT_FORWARD:
+        out /= sz
+
+    return out
+
+
+def _fftn(a, s, axes, norm, direction, value_type='C2C', order='A', plan=None,
+          overwrite_x=False, out=None):
+    if not isinstance(a, cupy.ndarray):
+        raise TypeError('The input array a must be a cupy.ndarray')
+    if norm is None:  # for backward compatibility
+        norm = 'backward'
+    if norm not in ('backward', 'ortho', 'forward'):
+        raise ValueError('Invalid norm value %s, should be "backward", '
+                         '"ortho", or "forward".' % norm)
+
+    axes, axes_sorted = _prep_fftn_axes(a.ndim, s, axes, value_type)
+    if not axes_sorted:
+        if value_type == 'C2C':
+            return a
+        else:
+            raise IndexError('list index out of range')
+    a = _convert_dtype(a, value_type)
+
+    if order == 'A':
+        if a.flags.f_contiguous:
+            order = 'F'
+        elif a.flags.c_contiguous:
+            order = 'C'
+        else:
+            a = cupy.ascontiguousarray(a)
+            order = 'C'
+    elif order not in ['C', 'F']:
+        raise ValueError('Unsupported order: {}'.format(order))
+
+    # Note: need to call _cook_shape prior to sorting the axes
+    a = _cook_shape(a, s, axes, value_type, order=order)
+
+    for n in a.shape:
+        if n < 1:
+            raise ValueError(
+                'Invalid number of FFT data points (%d) specified.' % n)
+
+    if order == 'C' and not a.flags.c_contiguous:
+        a = cupy.ascontiguousarray(a)
+    elif order == 'F' and not a.flags.f_contiguous:
+        a = cupy.asfortranarray(a)
+
+    # _cook_shape tells us input shape only, and not output shape
+    out_size = _get_fftn_out_size(a.shape, s, axes_sorted[-1], value_type)
+
+    a = _exec_fftn(a, direction, value_type, norm=norm, axes=axes_sorted,
+                   overwrite_x=overwrite_x, plan=plan, out=out,
+                   out_size=out_size)
+    return a
+
+
+def _default_fft_func(a, s=None, axes=None, plan=None, value_type='C2C'):
+    curr_plan = cufft.get_current_plan()
+    if curr_plan is not None:
+        if plan is None:
+            plan = curr_plan
+        else:
+            raise RuntimeError('Use the cuFFT plan either as a context manager'
+                               ' or as an argument.')
+
+    if isinstance(plan, cufft.PlanNd):  # a shortcut for using _fftn
+        return _fftn
+    elif (isinstance(plan, cufft.Plan1d) or
+          a.ndim == 1 or not config.enable_nd_planning):
+        return _fft
+
+    # cuFFT's N-D C2R/R2C transforms may not agree with NumPy's outcomes
+    if a.flags.f_contiguous and value_type != 'C2C':
+        return _fft
+
+    _, axes_sorted = _prep_fftn_axes(a.ndim, s, axes, value_type)
+    if len(axes_sorted) > 1 and _nd_plan_is_possible(axes_sorted, a.ndim):
+        # circumvent two potential hipFFT/rocFFT bugs as of ROCm 3.5.0
+        # TODO(leofang): understand hipFFT better and test newer ROCm versions
+        if cupy.cuda.runtime.is_hip:
+            if (0 == axes_sorted[0] and len(axes_sorted) != a.ndim
+                    and a.flags.c_contiguous):
+                return _fft
+
+            # For C2R, we don't use PlanNd; see the workaround in _exec_fft()
+            if value_type == 'C2R':
+                return _fft
+
+        # prefer Plan1D in the 1D case
+        return _fftn
+    return _fft
+
+
+def fft(a, n=None, axis=-1, norm=None):
+    """Compute the one-dimensional FFT.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if the input is other.
+
+    .. seealso:: :func:`numpy.fft.fft`
+    """
+    return _fft(a, (n,), (axis,), norm, cupy.cuda.cufft.CUFFT_FORWARD)
+
+
+def ifft(a, n=None, axis=-1, norm=None):
+    """Compute the one-dimensional inverse FFT.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if the input is other.
+
+    .. seealso:: :func:`numpy.fft.ifft`
+    """
+    return _fft(a, (n,), (axis,), norm, cufft.CUFFT_INVERSE)
+
+
+def fft2(a, s=None, axes=(-2, -1), norm=None):
+    """Compute the two-dimensional FFT.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``s`` is not given, the lengths of the input along the
+            axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other.
+
+    .. seealso:: :func:`numpy.fft.fft2`
+    """
+    func = _default_fft_func(a, s, axes)
+    return func(a, s, axes, norm, cufft.CUFFT_FORWARD)
+
+
+def ifft2(a, s=None, axes=(-2, -1), norm=None):
+    """Compute the two-dimensional inverse FFT.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``s`` is not given, the lengths of the input along the
+            axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other.
+
+    .. seealso:: :func:`numpy.fft.ifft2`
+    """
+    func = _default_fft_func(a, s, axes)
+    return func(a, s, axes, norm, cufft.CUFFT_INVERSE)
+
+
+def fftn(a, s=None, axes=None, norm=None):
+    """Compute the N-dimensional FFT.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``s`` is not given, the lengths of the input along the
+            axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other.
+
+    .. seealso:: :func:`numpy.fft.fftn`
+    """
+    func = _default_fft_func(a, s, axes)
+    return func(a, s, axes, norm, cufft.CUFFT_FORWARD)
+
+
+def ifftn(a, s=None, axes=None, norm=None):
+    """Compute the N-dimensional inverse FFT.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``s`` is not given, the lengths of the input along the
+            axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other.
+
+    .. seealso:: :func:`numpy.fft.ifftn`
+    """
+    func = _default_fft_func(a, s, axes)
+    return func(a, s, axes, norm, cufft.CUFFT_INVERSE)
+
+
+def rfft(a, n=None, axis=-1, norm=None):
+    """Compute the one-dimensional FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        n (None or int): Number of points along transformation axis in the
+            input to use. If ``n`` is not given, the length of the input along
+            the axis specified by ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if the input is other. The length of the
+            transformed axis is ``n//2+1``.
+
+    .. seealso:: :func:`numpy.fft.rfft`
+    """
+    return _fft(a, (n,), (axis,), norm, cufft.CUFFT_FORWARD, 'R2C')
+
+
+def irfft(a, n=None, axis=-1, norm=None):
+    """Compute the one-dimensional inverse FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        n (None or int): Length of the transformed axis of the output. For
+            ``n`` output points, ``n//2+1`` input points are necessary. If
+            ``n`` is not given, it is determined from the length of the input
+            along the axis specified by ``axis``.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if the input is other. If ``n`` is not
+            given, the length of the transformed axis is`2*(m-1)` where `m`
+            is the length of the transformed axis of the input.
+
+    .. seealso:: :func:`numpy.fft.irfft`
+    """
+    return _fft(a, (n,), (axis,), norm, cufft.CUFFT_INVERSE, 'C2R')
+
+
+def rfft2(a, s=None, axes=(-2, -1), norm=None):
+    """Compute the two-dimensional FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape to use from the input. If ``s`` is not
+            given, the lengths of the input along the axes specified by
+            ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other. The length of the
+            last axis transformed will be ``s[-1]//2+1``.
+
+    .. seealso:: :func:`numpy.fft.rfft2`
+    """
+    func = _default_fft_func(a, s, axes, value_type='R2C')
+    return func(a, s, axes, norm, cufft.CUFFT_FORWARD, 'R2C')
+
+
+def irfft2(a, s=None, axes=(-2, -1), norm=None):
+    """Compute the two-dimensional inverse FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape of the output. If ``s`` is not given,
+            they are determined from the lengths of the input along the axes
+            specified by ``axes``.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other. If ``s`` is not
+            given, the length of final transformed axis of output will be
+            `2*(m-1)` where `m` is the length of the final transformed axis of
+            the input.
+
+    .. seealso:: :func:`numpy.fft.irfft2`
+    """
+    func = _default_fft_func(a, s, axes, value_type='C2R')
+    return func(a, s, axes, norm, cufft.CUFFT_INVERSE, 'C2R')
+
+
+def rfftn(a, s=None, axes=None, norm=None):
+    """Compute the N-dimensional FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape to use from the input. If ``s`` is not
+            given, the lengths of the input along the axes specified by
+            ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other. The length of the
+            last axis transformed will be ``s[-1]//2+1``.
+
+    .. seealso:: :func:`numpy.fft.rfftn`
+    """
+    func = _default_fft_func(a, s, axes, value_type='R2C')
+    return func(a, s, axes, norm, cufft.CUFFT_FORWARD, 'R2C')
+
+
+def _size_last_transform_axis(shape, s, axes):
+    if s is not None:
+        if s[-1] is not None:
+            return s[-1]
+    elif axes is not None:
+        return shape[axes[-1]]
+    return shape[-1]
+
+
+def irfftn(a, s=None, axes=None, norm=None):
+    """Compute the N-dimensional inverse FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape of the output. If ``s`` is not given,
+            they are determined from the lengths of the input along the axes
+            specified by ``axes``.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other. If ``s`` is not
+            given, the length of final transformed axis of output will be
+            ``2*(m-1)`` where `m` is the length of the final transformed axis
+            of the input.
+
+    .. seealso:: :func:`numpy.fft.irfftn`
+    """
+    if (10020 >= cupy.cuda.runtime.runtimeGetVersion() >= 10010
+            and int(cupy.cuda.device.get_compute_capability()) < 70
+            and _size_last_transform_axis(a.shape, s, axes) == 2):
+        warnings.warn('Output of irfftn might not be correct due to issue '
+                      'of cuFFT in CUDA 10.1/10.2 on Pascal or older GPUs.')
+
+    func = _default_fft_func(a, s, axes, value_type='C2R')
+    return func(a, s, axes, norm, cufft.CUFFT_INVERSE, 'C2R')
+
+
+def _swap_direction(norm):
+    if norm in (None, 'backward'):
+        norm = 'forward'
+    elif norm == 'forward':
+        norm = 'backward'
+    elif norm != 'ortho':
+        raise ValueError('Invalid norm value %s; should be "backward", '
+                         '"ortho", or "forward".' % norm)
+    return norm
+
+
+def hfft(a, n=None, axis=-1, norm=None):
+    """Compute the FFT of a signal that has Hermitian symmetry.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        n (None or int): Length of the transformed axis of the output. For
+            ``n`` output points, ``n//2+1`` input points are necessary. If
+            ``n`` is not given, it is determined from the length of the input
+            along the axis specified by ``axis``.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if the input is other. If ``n`` is not
+            given, the length of the transformed axis is ``2*(m-1)`` where `m`
+            is the length of the transformed axis of the input.
+
+    .. seealso:: :func:`numpy.fft.hfft`
+    """
+    return irfft(a.conj(), n, axis, _swap_direction(norm))
+
+
+def ihfft(a, n=None, axis=-1, norm=None):
+    """Compute the FFT of a signal that has Hermitian symmetry.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        n (None or int): Number of points along transformation axis in the
+            input to use. If ``n`` is not given, the length of the input along
+            the axis specified by ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if the input is other. The length of the
+            transformed axis is ``n//2+1``.
+
+    .. seealso:: :func:`numpy.fft.ihfft`
+    """
+    return rfft(a, n, axis, _swap_direction(norm)).conj()
+
+
+def fftfreq(n, d=1.0):
+    """Return the FFT sample frequencies.
+
+    Args:
+        n (int): Window length.
+        d (scalar): Sample spacing.
+
+    Returns:
+        cupy.ndarray: Array of length ``n`` containing the sample frequencies.
+
+    .. seealso:: :func:`numpy.fft.fftfreq`
+    """
+    return cupy.hstack((cupy.arange(0, (n - 1) // 2 + 1, dtype=np.float64),
+                        cupy.arange(-(n // 2), 0, dtype=np.float64))) / (n * d)
+
+
+def rfftfreq(n, d=1.0):
+    """Return the FFT sample frequencies for real input.
+
+    Args:
+        n (int): Window length.
+        d (scalar): Sample spacing.
+
+    Returns:
+        cupy.ndarray:
+            Array of length ``n//2+1`` containing the sample frequencies.
+
+    .. seealso:: :func:`numpy.fft.rfftfreq`
+    """
+    return cupy.arange(0, n // 2 + 1, dtype=np.float64) / (n * d)
+
+
+def fftshift(x, axes=None):
+    """Shift the zero-frequency component to the center of the spectrum.
+
+    Args:
+        x (cupy.ndarray): Input array.
+        axes (int or tuple of ints): Axes over which to shift. Default is
+            ``None``, which shifts all axes.
+
+    Returns:
+        cupy.ndarray: The shifted array.
+
+    .. seealso:: :func:`numpy.fft.fftshift`
+    """
+    x = cupy.asarray(x)
+    if axes is None:
+        axes = list(range(x.ndim))
+    elif isinstance(axes, np.compat.integer_types):
+        axes = (axes,)
+    return cupy.roll(x, [x.shape[axis] // 2 for axis in axes], axes)
+
+
+def ifftshift(x, axes=None):
+    """The inverse of :meth:`fftshift`.
+
+    Args:
+        x (cupy.ndarray): Input array.
+        axes (int or tuple of ints): Axes over which to shift. Default is
+            ``None``, which shifts all axes.
+
+    Returns:
+        cupy.ndarray: The shifted array.
+
+    .. seealso:: :func:`numpy.fft.ifftshift`
+    """
+    x = cupy.asarray(x)
+    if axes is None:
+        axes = list(range(x.ndim))
+    elif isinstance(axes, np.compat.integer_types):
+        axes = (axes,)
+    return cupy.roll(x, [-(x.shape[axis] // 2) for axis in axes], axes)
diff --git a/cupy/fft/config.py b/cupy/fft/config.py
new file mode 100644
index 0000000..fcf0518
--- /dev/null
+++ b/cupy/fft/config.py
@@ -0,0 +1,61 @@
+from cupy import _util
+
+# expose cache handles to this module
+from cupy.fft._cache import get_plan_cache  # NOQA
+from cupy.fft._cache import clear_plan_cache  # NOQA
+from cupy.fft._cache import get_plan_cache_size  # NOQA
+from cupy.fft._cache import set_plan_cache_size  # NOQA
+from cupy.fft._cache import get_plan_cache_max_memsize  # NOQA
+from cupy.fft._cache import set_plan_cache_max_memsize  # NOQA
+from cupy.fft._cache import show_plan_cache_info  # NOQA
+
+# on Linux, expose callback handles to this module
+import sys as _sys
+if _sys.platform.startswith('linux'):
+    from cupy.fft._callback import get_current_callback_manager  # NOQA
+    from cupy.fft._callback import set_cufft_callbacks  # NOQA
+else:
+    def get_current_callback_manager(*args, **kwargs):
+        return None
+
+    class set_cufft_callbacks:  # type: ignore
+        def __init__(self, *args, **kwargs):
+            raise RuntimeError('cuFFT callback is only available on Linux')
+
+
+enable_nd_planning = True
+use_multi_gpus = False
+_devices = None
+
+
+def set_cufft_gpus(gpus):
+    '''Set the GPUs to be used in multi-GPU FFT.
+
+    Args:
+        gpus (int or list of int): The number of GPUs or a list of GPUs
+            to be used. For the former case, the first ``gpus`` GPUs
+            will be used.
+
+    .. warning::
+        This API is currently experimental and may be changed in the future
+        version.
+
+    .. seealso:: `Multiple GPU cuFFT Transforms`_
+
+    .. _Multiple GPU cuFFT Transforms:
+        https://docs.nvidia.com/cuda/cufft/index.html#multiple-GPU-cufft-transforms
+    '''
+    _util.experimental('cupy.fft.config.set_cufft_gpus')
+    global _devices
+
+    if isinstance(gpus, int):
+        devs = [i for i in range(gpus)]
+    elif isinstance(gpus, list):
+        devs = gpus
+    else:
+        raise ValueError("gpus must be an int or a list of int.")
+    if len(devs) <= 1:
+        raise ValueError("Must use at least 2 GPUs.")
+
+    # make it hashable
+    _devices = tuple(devs)
diff --git a/cupy/lib/__init__.py b/cupy/lib/__init__.py
new file mode 100644
index 0000000..cd39775
--- /dev/null
+++ b/cupy/lib/__init__.py
@@ -0,0 +1 @@
+from cupy.lib import stride_tricks  # NOQA
diff --git a/cupy/lib/_polynomial.pyx b/cupy/lib/_polynomial.pyx
new file mode 100644
index 0000000..19dbdc9
--- /dev/null
+++ b/cupy/lib/_polynomial.pyx
@@ -0,0 +1,257 @@
+import numbers
+
+import numpy
+
+from cupy._core.core cimport _ndarray_base
+
+import cupy
+from cupy.lib import _routines_poly
+
+
+cdef class poly1d:
+    """A one-dimensional polynomial class.
+
+    .. note::
+        This is a counterpart of an old polynomial class in NumPy.
+        Note that the new NumPy polynomial API
+        (:mod:`numpy.polynomial.polynomial`)
+        has different convention, e.g. order of coefficients is reversed.
+
+    Args:
+        c_or_r (array_like): The polynomial's
+         coefficients in decreasing powers
+        r (bool, optional): If True, ``c_or_r`` specifies the
+            polynomial's roots; the default is False.
+        variable (str, optional): Changes the variable used when
+            printing the polynomial from ``x`` to ``variable``
+
+    .. seealso:: :class:`numpy.poly1d`
+
+    """
+    __hash__ = None
+
+    cdef:
+        readonly _ndarray_base _coeffs
+        readonly str _variable
+        readonly bint _trimmed
+
+    @property
+    def coeffs(self):
+        if self._trimmed:
+            return self._coeffs
+        self._coeffs = cupy.trim_zeros(self._coeffs, trim='f')
+        if self._coeffs.size == 0:
+            self._coeffs = cupy.array([0.], self._coeffs.dtype)
+        self._trimmed = True
+        return self._coeffs
+
+    @coeffs.setter
+    def coeffs(self, value):
+        if value is not self._coeffs:
+            raise AttributeError('Cannot set attribute')
+
+    @property
+    def variable(self):
+        return self._variable
+
+    @property
+    def order(self):
+        return self.coeffs.size - 1
+
+    @property
+    def roots(self):
+        return _routines_poly.roots(self._coeffs)
+
+    @property
+    def r(self):
+        return self.roots
+
+    @property
+    def c(self):
+        return self.coeffs
+
+    @property
+    def coef(self):
+        return self.coeffs
+
+    @property
+    def coefficients(self):
+        return self.coeffs
+
+    @property
+    def o(self):
+        return self.order
+
+    def __init__(self, c_or_r, r=False, variable=None):
+        if isinstance(c_or_r, (numpy.poly1d, poly1d)):
+            self._coeffs = cupy.asarray(c_or_r.coeffs)
+            self._variable = c_or_r._variable
+            self._trimmed = True
+            if variable is not None:
+                self._variable = variable
+            return
+        if r:
+            c_or_r = _routines_poly.poly(c_or_r)
+        c_or_r = cupy.atleast_1d(c_or_r)
+        if c_or_r.ndim > 1:
+            raise ValueError('Polynomial must be 1d only.')
+        self._coeffs = c_or_r
+        self._trimmed = False
+        if variable is None:
+            variable = 'x'
+        self._variable = variable
+
+    def __cupy_get_ndarray__(self):
+        return self.coeffs
+
+    def __array__(self, dtype=None):
+        raise TypeError(
+            'Implicit conversion to a NumPy array is not allowed. '
+            'Please use `.get()` to construct a NumPy array explicitly.')
+
+    def __repr__(self):
+        return repr(self.get())
+
+    def __len__(self):
+        return self.order
+
+    def __str__(self):
+        return str(self.get())
+
+    def __call__(self, val):
+        return _routines_poly.polyval(self.coeffs, val)
+
+    def __neg__(self):
+        return poly1d(-self.coeffs)
+
+    def __pos__(self):
+        return self
+
+    # TODO(kataoka): Rename `self`. It is misleading for cdef class.
+    def __mul__(self, other):
+        if cupy.isscalar(other):
+            # case: poly1d * python scalar
+            # the return type of cupy.polymul output is
+            # inconsistent with NumPy's output for this case.
+            return poly1d(self.coeffs * other)
+        if isinstance(self, numpy.generic):
+            # case: numpy scalar * poly1d
+            # poly1d addition and subtraction don't support this case
+            # so it is not supported here for consistency purposes
+            # between polyarithmetic routines
+            raise TypeError('Numpy scalar and poly1d multiplication'
+                            ' is not supported currently.')
+        if cupy.isscalar(self) and isinstance(other, poly1d):
+            # case: python scalar * poly1d
+            # the return type of cupy.polymul output is
+            # inconsistent with NumPy's output for this case
+            # So casting python scalar is required.
+            self = other._coeffs.dtype.type(self)
+        return _routines_poly.polymul(self, other)
+
+    def __add__(self, other):
+        if isinstance(self, numpy.generic):
+            # for the case: numpy scalar + poly1d
+            raise TypeError('Numpy scalar and poly1d '
+                            'addition is not supported')
+        return _routines_poly.polyadd(self, other)
+
+    def __pow__(self, val, modulo):
+        if not cupy.isscalar(val) or int(val) != val or val < 0:
+            raise ValueError('Power to non-negative integers only.')
+        if not isinstance(val, numbers.Integral):
+            raise TypeError('float object cannot be interpreted as an integer')
+
+        base = self.coeffs
+        dtype = base.dtype
+
+        if dtype.kind == 'c':
+            base = base.astype(numpy.complex128, copy=False)
+        elif dtype.kind == 'f' or dtype == numpy.uint64:
+            base = base.astype(numpy.float64, copy=False)
+        else:
+            # use Python int here for cross-platform portability
+            int_dtype = numpy.promote_types(dtype, int)
+            base = base.astype(int_dtype, copy=False)
+        return poly1d(_routines_poly._polypow(base, val))
+
+    def __sub__(self, other):
+        if isinstance(self, numpy.generic):
+            # for the case: numpy scalar - poly1d
+            raise TypeError('Numpy scalar and poly1d '
+                            'subtraction is not supported')
+        return _routines_poly.polysub(self, other)
+
+    # TODO(Dahlia-Chehata): use polydiv for non-scalars
+    def __truediv__(self, other):
+        if cupy.isscalar(other):
+            return poly1d(self.coeffs / other)
+        raise NotImplementedError
+
+    def __eq__(self, other):
+        if not isinstance(other, poly1d):
+            raise NotImplementedError
+        if self.coeffs.shape != other.coeffs.shape:
+            return False
+        return (self.coeffs == other.coeffs).all().get()[()]
+
+    def __ne__(self, other):
+        if not isinstance(other, poly1d):
+            raise NotImplementedError
+        return not self.__eq__(other)
+
+    def __getitem__(self, val):
+        if 0 <= val < self._coeffs.size:
+            return self._coeffs[-val-1]
+        return 0
+
+    def __setitem__(self, key, val):
+        if key < 0:
+            raise ValueError('Negative powers are not supported.')
+        if key >= self._coeffs.size:
+            self._coeffs = cupy.pad(self._coeffs,
+                                    (key - (self._coeffs.size - 1), 0))
+        self._coeffs[-key-1] = val
+        return
+
+    def __iter__(self):
+        return iter(self.coeffs)
+
+    def integ(self, m=1, k=0):
+        raise NotImplementedError
+
+    def deriv(self, m=1):
+        raise NotImplementedError
+
+    # -------------------------------------------------------------------------
+    # Cupy specific attributes and methods
+    # -------------------------------------------------------------------------
+    cpdef get(self, stream=None):
+        """Returns a copy of poly1d object on host memory.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+                The default uses CUDA stream object of the current context.
+
+        Returns:
+            numpy.poly1d: Copy of poly1d object on host memory.
+
+        """
+        return numpy.poly1d(self.coeffs.get(stream=stream),
+                            variable=self.variable)
+
+    cpdef set(self, polyin, stream=None):
+        """Copies a poly1d object on the host memory to :class:`cupy.poly1d`.
+
+        Args:
+            polyin (numpy.poly1d): The source object on the host memory.
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+                The default uses CUDA stream object of the current context.
+
+        """
+        if not isinstance(polyin, numpy.poly1d):
+            raise TypeError('Only numpy.poly1d can be set to cupy.poly1d')
+        self._variable = polyin.variable
+        self.coeffs.set(polyin.coeffs, stream)
diff --git a/cupy/lib/_routines_poly.py b/cupy/lib/_routines_poly.py
new file mode 100644
index 0000000..65fe572
--- /dev/null
+++ b/cupy/lib/_routines_poly.py
@@ -0,0 +1,396 @@
+import functools
+import warnings
+
+import numpy
+
+import cupy
+import cupyx.scipy.fft
+
+
+def _wraps_polyroutine(func):
+    def _get_coeffs(x):
+        if isinstance(x, cupy.poly1d):
+            return x._coeffs
+        if cupy.isscalar(x):
+            return cupy.atleast_1d(x)
+        if isinstance(x, cupy.ndarray):
+            x = cupy.atleast_1d(x)
+            if x.ndim == 1:
+                return x
+            raise ValueError('Multidimensional inputs are not supported')
+        raise TypeError('Unsupported type')
+
+    def wrapper(*args):
+        coeffs = [_get_coeffs(x) for x in args]
+        out = func(*coeffs)
+
+        if all(not isinstance(x, cupy.poly1d) for x in args):
+            return out
+        if isinstance(out, cupy.ndarray):
+            return cupy.poly1d(out)
+        if isinstance(out, tuple):
+            return tuple([cupy.poly1d(x) for x in out])
+        assert False  # Never reach
+
+    return functools.update_wrapper(wrapper, func)
+
+
+def poly(seq_of_zeros):
+    """Computes the coefficients of a polynomial with the given roots sequence.
+
+    Args:
+        seq_of_zeros (cupy.ndarray): a sequence of polynomial roots.
+
+    Returns:
+        cupy.ndarray: polynomial coefficients from highest to lowest degree.
+
+    .. warning::
+
+        This function doesn't support general 2d square arrays currently.
+        Only complex Hermitian and real symmetric 2d arrays are allowed.
+
+    .. seealso:: :func:`numpy.poly`
+
+    """
+    x = seq_of_zeros
+    if x.ndim == 2 and x.shape[0] == x.shape[1] and x.shape[0] != 0:
+        if cupy.array_equal(x, x.conj().T):
+            x = cupy.linalg.eigvalsh(x)
+        else:
+            raise NotImplementedError('Only complex Hermitian and real '
+                                      'symmetric 2d arrays are supported '
+                                      'currently')
+    elif x.ndim == 1:
+        x = x.astype(cupy.mintypecode(x.dtype.char), copy=False)
+    else:
+        raise ValueError('Input must be 1d or non-empty square 2d array.')
+
+    if x.size == 0:
+        return 1.0
+
+    size = 2 ** (x.size - 1).bit_length()
+    a = cupy.zeros((size, 2), x.dtype)
+    a[:, 0].fill(1)
+    cupy.negative(x, out=a[:x.size, 1])
+    while size > 1:
+        size = size // 2
+        a = cupy._math.misc._fft_convolve(a[:size], a[size:], 'full')
+    return a[0, :x.size + 1]
+
+
+@_wraps_polyroutine
+def polyadd(a1, a2):
+    """Computes the sum of two polynomials.
+
+    Args:
+        a1 (scalar, cupy.ndarray or cupy.poly1d): first input polynomial.
+        a2 (scalar, cupy.ndarray or cupy.poly1d): second input polynomial.
+
+    Returns:
+        cupy.ndarray or cupy.poly1d: The sum of the inputs.
+
+    .. seealso:: :func:`numpy.polyadd`
+
+    """
+    if a1.size < a2.size:
+        a1, a2 = a2, a1
+    out = cupy.pad(a2, (a1.size - a2.size, 0))
+    out = out.astype(cupy.result_type(a1, a2), copy=False)
+    out += a1
+    return out
+
+
+@_wraps_polyroutine
+def polysub(a1, a2):
+    """Computes the difference of two polynomials.
+
+    Args:
+        a1 (scalar, cupy.ndarray or cupy.poly1d): first input polynomial.
+        a2 (scalar, cupy.ndarray or cupy.poly1d): second input polynomial.
+
+    Returns:
+        cupy.ndarray or cupy.poly1d: The difference of the inputs.
+
+    .. seealso:: :func:`numpy.polysub`
+
+    """
+    if a1.shape[0] <= a2.shape[0]:
+        out = cupy.pad(a1, (a2.shape[0] - a1.shape[0], 0))
+        out = out.astype(cupy.result_type(a1, a2), copy=False)
+        out -= a2
+    else:
+        out = cupy.pad(a2, (a1.shape[0] - a2.shape[0], 0))
+        out = out.astype(cupy.result_type(a1, a2), copy=False)
+        out -= 2 * out - a1
+    return out
+
+
+@_wraps_polyroutine
+def polymul(a1, a2):
+    """Computes the product of two polynomials.
+
+    Args:
+        a1 (scalar, cupy.ndarray or cupy.poly1d): first input polynomial.
+        a2 (scalar, cupy.ndarray or cupy.poly1d): second input polynomial.
+
+    Returns:
+        cupy.ndarray or cupy.poly1d: The product of the inputs.
+
+    .. seealso:: :func:`numpy.polymul`
+
+    """
+    a1 = cupy.trim_zeros(a1, trim='f')
+    a2 = cupy.trim_zeros(a2, trim='f')
+    if a1.size == 0:
+        a1 = cupy.array([0.], a1.dtype)
+    if a2.size == 0:
+        a2 = cupy.array([0.], a2.dtype)
+    return cupy.convolve(a1, a2)
+
+
+def _polypow_direct(x, n):
+    if n == 0:
+        return 1
+    if n == 1:
+        return x
+    if n % 2 == 0:
+        return _polypow(cupy.convolve(x, x), n // 2)
+    return cupy.convolve(x, _polypow(cupy.convolve(x, x), (n - 1) // 2))
+
+
+def _polypow(x, n):
+    if n == 0:
+        return 1
+    if n == 1:
+        return x
+
+    method = cupy._math.misc._choose_conv_method(x, x, 'full')
+
+    if method == 'direct':
+        return _polypow_direct(x, n)
+    elif method == 'fft':
+        if x.dtype.kind == 'c':
+            fft, ifft = cupy.fft.fft, cupy.fft.ifft
+        else:
+            fft, ifft = cupy.fft.rfft, cupy.fft.irfft
+        out_size = (x.size - 1) * n + 1
+        size = cupyx.scipy.fft.next_fast_len(out_size)
+        fx = fft(x, size)
+        fy = cupy.power(fx, n, fx)
+        y = ifft(fy, size)
+        return y[:out_size]
+    else:
+        assert False
+
+
+def _polyfit_typecast(x):
+    if x.dtype.kind == 'c':
+        return x.astype(numpy.complex128, copy=False)
+    return x.astype(numpy.float64, copy=False)
+
+
+def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False):
+    """Returns the least squares fit of polynomial of degree deg
+    to the data y sampled at x.
+
+    Args:
+        x (cupy.ndarray): x-coordinates of the sample points of shape (M,).
+        y (cupy.ndarray): y-coordinates of the sample points of shape
+            (M,) or (M, K).
+        deg (int): degree of the fitting polynomial.
+        rcond (float, optional): relative condition number of the fit.
+            The default value is ``len(x) * eps``.
+        full (bool, optional): indicator of the return value nature.
+            When False (default), only the coefficients are returned.
+            When True, diagnostic information is also returned.
+        w (cupy.ndarray, optional): weights applied to the y-coordinates
+            of the sample points of shape (M,).
+        cov (bool or str, optional): if given, returns the coefficients
+            along with the covariance matrix.
+
+    Returns:
+        cupy.ndarray or tuple:
+        p (cupy.ndarray of shape (deg + 1,) or (deg + 1, K)):
+            Polynomial coefficients from highest to lowest degree
+        residuals, rank, singular_values, rcond \
+        (cupy.ndarray, int, cupy.ndarray, float):
+            Present only if ``full=True``.
+            Sum of squared residuals of the least-squares fit,
+            rank of the scaled Vandermonde coefficient matrix,
+            its singular values, and the specified value of ``rcond``.
+        V (cupy.ndarray of shape (M, M) or (M, M, K)):
+            Present only if ``full=False`` and ``cov=True``.
+            The covariance matrix of the polynomial coefficient estimates.
+
+    .. warning::
+
+        numpy.RankWarning: The rank of the coefficient matrix in the
+        least-squares fit is deficient. It is raised if ``full=False``.
+
+    .. seealso:: :func:`numpy.polyfit`
+
+    """
+    if x.dtype.char == 'e' and y.dtype.kind == 'b':
+        raise NotImplementedError('float16 x and bool y are not'
+                                  ' currently supported')
+    if y.dtype == numpy.float16:
+        raise TypeError('float16 y are not supported')
+
+    x = _polyfit_typecast(x)
+    y = _polyfit_typecast(y)
+    deg = int(deg)
+
+    if deg < 0:
+        raise ValueError('expected deg >= 0')
+    if x.ndim != 1:
+        raise TypeError('expected 1D vector for x')
+    if x.size == 0:
+        raise TypeError('expected non-empty vector for x')
+    if y.ndim < 1 or y.ndim > 2:
+        raise TypeError('expected 1D or 2D array for y')
+    if x.size != y.shape[0]:
+        raise TypeError('expected x and y to have same length')
+
+    lhs = cupy.polynomial.polynomial.polyvander(x, deg)[:, ::-1]
+    rhs = y
+
+    if w is not None:
+        w = _polyfit_typecast(w)
+        if w.ndim != 1:
+            raise TypeError('expected a 1-d array for weights')
+        if w.size != x.size:
+            raise TypeError('expected w and y to have the same length')
+
+        lhs *= w[:, None]
+        if rhs.ndim == 2:
+            w = w[:, None]
+        rhs *= w
+
+    if rcond is None:
+        rcond = x.size * cupy.finfo(x.dtype).eps
+
+    scale = cupy.sqrt((cupy.square(lhs)).sum(axis=0))
+    lhs /= scale
+    c, resids, rank, s = cupy.linalg.lstsq(lhs, rhs, rcond)
+    if y.ndim > 1:
+        scale = scale.reshape(-1, 1)
+    c /= scale
+
+    order = deg + 1
+    if rank != order and not full:
+        msg = 'Polyfit may be poorly conditioned'
+        warnings.warn(msg, numpy.RankWarning, stacklevel=4)
+
+    if full:
+        if resids.dtype.kind == 'c':
+            resids = cupy.absolute(resids)
+        return c, resids, rank, s, rcond
+    if cov:
+        base = cupy.linalg.inv(cupy.dot(lhs.T, lhs))
+        base /= cupy.outer(scale, scale)
+
+        if cov == 'unscaled':
+            factor = 1
+        elif x.size > order:
+            factor = resids / (x.size - order)
+        else:
+            raise ValueError('the number of data points must exceed order'
+                             ' to scale the covariance matrix')
+
+        if y.ndim != 1:
+            base = base[..., None]
+        return c, base * factor
+
+    return c
+
+
+def polyval(p, x):
+    """Evaluates a polynomial at specific values.
+
+    Args:
+        p (cupy.ndarray or cupy.poly1d): input polynomial.
+        x (scalar, cupy.ndarray): values at which the polynomial
+        is evaluated.
+
+    Returns:
+        cupy.ndarray or cupy.poly1d: polynomial evaluated at x.
+
+    .. warning::
+
+        This function doesn't currently support poly1d values to evaluate.
+
+    .. seealso:: :func:`numpy.polyval`
+
+    """
+    if isinstance(p, cupy.poly1d):
+        p = p.coeffs
+    if not isinstance(p, cupy.ndarray) or p.ndim == 0:
+        raise TypeError('p must be 1d ndarray or poly1d object')
+    if p.ndim > 1:
+        raise ValueError('p must be 1d array')
+    if isinstance(x, cupy.poly1d):
+        # TODO(asi1024): Needs performance improvement.
+        dtype = numpy.result_type(x.coeffs, 1)
+        res = cupy.poly1d(cupy.array([0], dtype=dtype))
+        prod = cupy.poly1d(cupy.array([1], dtype=dtype))
+        for c in p[::-1]:
+            res = res + prod * c
+            prod = prod * x
+        return res
+    dtype = numpy.result_type(p.dtype.type(0), x)
+    p = p.astype(dtype, copy=False)
+    if p.size == 0:
+        return cupy.zeros(x.shape, dtype)
+    if dtype == numpy.bool_:
+        return p.any() * x + p[-1]
+    if not cupy.isscalar(x):
+        x = cupy.asarray(x, dtype=dtype)[..., None]
+    x = x ** cupy.arange(p.size, dtype=dtype)
+    return (p[::-1] * x).sum(axis=-1, dtype=dtype)
+
+
+def roots(p):
+    """Computes the roots of a polynomial with given coefficients.
+
+    Args:
+        p (cupy.ndarray or cupy.poly1d): polynomial coefficients.
+
+    Returns:
+        cupy.ndarray: polynomial roots.
+
+    .. warning::
+
+        This function doesn't support currently polynomial coefficients
+        whose companion matrices are general 2d square arrays. Only those
+        with complex Hermitian or real symmetric 2d arrays are allowed.
+
+        The current `cupy.roots` doesn't guarantee the order of results.
+
+    .. seealso:: :func:`numpy.roots`
+
+    """
+    if isinstance(p, cupy.poly1d):
+        p = p.coeffs
+    if p.dtype.kind == 'b':
+        raise NotImplementedError('boolean inputs are not supported')
+    if p.ndim == 0:
+        raise TypeError('0-dimensional input is not allowed')
+    if p.size < 2:
+        return cupy.array([])
+    [p] = cupy.polynomial.polyutils.as_series([p[::-1]])
+    if p.size < 2:
+        return cupy.array([])
+    if p.size == 2:
+        out = (-p[0] / p[1])[None]
+        if p[0] == 0:
+            out = out.real.astype(numpy.float64)
+        return out
+    cmatrix = cupy.polynomial.polynomial.polycompanion(p)
+    # TODO(Dahlia-Chehata): Support after cupy.linalg.eigvals is supported
+    if cupy.array_equal(cmatrix, cmatrix.conj().T):
+        out = cupy.linalg.eigvalsh(cmatrix)
+    else:
+        raise NotImplementedError('Only complex Hermitian and real '
+                                  'symmetric 2d arrays are supported '
+                                  'currently')
+    return out.astype(p.dtype)
diff --git a/cupy/lib/_shape_base.py b/cupy/lib/_shape_base.py
new file mode 100644
index 0000000..8d948f3
--- /dev/null
+++ b/cupy/lib/_shape_base.py
@@ -0,0 +1,63 @@
+from numpy.lib import index_tricks
+
+import cupy
+from cupy._core import internal
+
+
+def apply_along_axis(func1d, axis, arr, *args, **kwargs):
+    """Apply a function to 1-D slices along the given axis.
+
+    Args:
+        func1d (function (M,) -> (Nj...)): This function should accept 1-D
+            arrays. It is applied to 1-D slices of ``arr`` along the specified
+            axis. It must return a 1-D ``cupy.ndarray``.
+        axis (integer): Axis along which ``arr`` is sliced.
+        arr (cupy.ndarray (Ni..., M, Nk...)): Input array.
+        args: Additional arguments for ``func1d``.
+        kwargs: Additional keyword arguments for ``func1d``.
+
+    Returns:
+        cupy.ndarray: The output array. The shape of ``out`` is identical to
+        the shape of ``arr``, except along the ``axis`` dimension. This
+        axis is removed, and replaced with new dimensions equal to the
+        shape of the return value of ``func1d``. So if ``func1d`` returns a
+        scalar ``out`` will have one fewer dimensions than ``arr``.
+
+    .. seealso:: :func:`numpy.apply_along_axis`
+    """
+    ndim = arr.ndim
+    axis = internal._normalize_axis_index(axis, ndim)
+    inarr_view = cupy.moveaxis(arr, axis, -1)
+
+    # compute indices for the iteration axes, and append a trailing ellipsis to
+    # prevent 0d arrays decaying to scalars
+    inds = index_tricks.ndindex(inarr_view.shape[:-1])
+    inds = (ind + (Ellipsis,) for ind in inds)
+
+    # invoke the function on the first item
+    try:
+        ind0 = next(inds)
+    except StopIteration:
+        raise ValueError(
+            'Cannot apply_along_axis when any iteration dimensions are 0'
+        )
+    res = func1d(inarr_view[ind0], *args, **kwargs)
+    res = cupy.asarray(res)
+
+    # build a buffer for storing evaluations of func1d.
+    # remove the requested axis, and add the new ones on the end.
+    # laid out so that each write is contiguous.
+    # for a tuple index inds, buff[inds] = func1d(inarr_view[inds])
+    buff = cupy.empty(inarr_view.shape[:-1] + res.shape, res.dtype)
+
+    # save the first result, then compute and save all remaining results
+    buff[ind0] = res
+    for ind in inds:
+        out = func1d(inarr_view[ind], *args, **kwargs)
+        buff[ind] = cupy.asarray(out)
+
+    # restore the inserted axes back to where they belong
+    for i in range(res.ndim):
+        buff = cupy.moveaxis(buff, -1, axis)
+
+    return buff
diff --git a/cupy/lib/stride_tricks.py b/cupy/lib/stride_tricks.py
new file mode 100644
index 0000000..04c9139
--- /dev/null
+++ b/cupy/lib/stride_tricks.py
@@ -0,0 +1,39 @@
+import cupy as _cupy
+
+
+def as_strided(x, shape=None, strides=None):
+    """
+    Create a view into the array with the given shape and strides.
+
+    .. warning:: This function has to be used with extreme care, see notes.
+
+    Parameters
+    ----------
+    x : ndarray
+        Array to create a new.
+    shape : sequence of int, optional
+        The shape of the new array. Defaults to ``x.shape``.
+    strides : sequence of int, optional
+        The strides of the new array. Defaults to ``x.strides``.
+
+    Returns
+    -------
+    view : ndarray
+
+    See also
+    --------
+    numpy.lib.stride_tricks.as_strided
+    reshape : reshape an array.
+
+    Notes
+    -----
+    ``as_strided`` creates a view into the array given the exact strides
+    and shape. This means it manipulates the internal data structure of
+    ndarray and, if done incorrectly, the array elements can point to
+    invalid memory and can corrupt results or crash your program.
+    """
+    shape = x.shape if shape is None else tuple(shape)
+    strides = x.strides if strides is None else tuple(strides)
+
+    return _cupy.ndarray(shape=shape, dtype=x.dtype,
+                         memptr=x.data, strides=strides)
diff --git a/cupy/linalg/__init__.py b/cupy/linalg/__init__.py
new file mode 100644
index 0000000..392377b
--- /dev/null
+++ b/cupy/linalg/__init__.py
@@ -0,0 +1,43 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.linalg.html
+
+# -----------------------------------------------------------------------------
+# Matrix and vector products
+# -----------------------------------------------------------------------------
+from cupy.linalg._product import matrix_power  # NOQA
+
+# -----------------------------------------------------------------------------
+# Decompositions
+# -----------------------------------------------------------------------------
+from cupy.linalg._decomposition import cholesky  # NOQA
+from cupy.linalg._decomposition import qr  # NOQA
+from cupy.linalg._decomposition import svd  # NOQA
+
+# -----------------------------------------------------------------------------
+# Matrix eigenvalues
+# -----------------------------------------------------------------------------
+from cupy.linalg._eigenvalue import eigh  # NOQA
+from cupy.linalg._eigenvalue import eigvalsh  # NOQA
+
+# -----------------------------------------------------------------------------
+# Norms and other numbers
+# -----------------------------------------------------------------------------
+from cupy.linalg._norms import norm  # NOQA
+from cupy.linalg._norms import det  # NOQA
+from cupy.linalg._norms import matrix_rank  # NOQA
+from cupy.linalg._norms import slogdet  # NOQA
+
+# -----------------------------------------------------------------------------
+# Solving equations and inverting matrices
+# -----------------------------------------------------------------------------
+from cupy.linalg._solve import solve  # NOQA
+from cupy.linalg._solve import tensorsolve  # NOQA
+from cupy.linalg._solve import lstsq  # NOQA
+from cupy.linalg._solve import inv  # NOQA
+from cupy.linalg._solve import pinv  # NOQA
+from cupy.linalg._solve import tensorinv  # NOQA
+
+# -----------------------------------------------------------------------------
+# Exceptions
+# -----------------------------------------------------------------------------
+from numpy.linalg import LinAlgError  # NOQA
diff --git a/cupy/linalg/_decomposition.py b/cupy/linalg/_decomposition.py
new file mode 100644
index 0000000..fbf4f54
--- /dev/null
+++ b/cupy/linalg/_decomposition.py
@@ -0,0 +1,585 @@
+import numpy
+
+import cupy
+from cupy_backends.cuda.api import runtime
+from cupy_backends.cuda.libs import cublas
+from cupy_backends.cuda.libs import cusolver
+from cupy._core import internal
+from cupy.cuda import device
+from cupyx.cusolver import check_availability
+from cupyx.cusolver import _gesvdj_batched, _gesvd_batched
+from cupyx.cusolver import _geqrf_orgqr_batched
+from cupy.linalg import _util
+
+
+def _lu_factor(a_t, dtype):
+    """Compute pivoted LU decomposition.
+
+    Decompose a given batch of square matrices. Inputs and outputs are
+    transposed.
+
+    Args:
+        a_t (cupy.ndarray): The input matrix with dimension ``(..., N, N)``.
+            The dimension condition is not checked.
+        dtype (numpy.dtype): float32, float64, complex64, or complex128.
+
+    Returns:
+        tuple:
+        lu_t (cupy.ndarray):
+            ``L`` without its unit diagonal and ``U`` with
+            dimension ``(..., N, N)``.
+        piv (cupy.ndarray):
+            1-origin pivot indices with dimension
+            ``(..., N)``.
+        dev_info (cupy.ndarray):
+            ``getrf`` info with dimension ``(...)``.
+
+    .. seealso:: :func:`scipy.linalg.lu_factor`
+
+    """
+    orig_shape = a_t.shape
+    n = orig_shape[-2]
+
+    # copy is necessary to present `a` to be overwritten.
+    a_t = a_t.astype(dtype, order='C').reshape(-1, n, n)
+    batch_size = a_t.shape[0]
+    ipiv = cupy.empty((batch_size, n), dtype=numpy.int32)
+    dev_info = cupy.empty((batch_size,), dtype=numpy.int32)
+
+    # Heuristic condition from some performance test.
+    # TODO(kataoka): autotune
+    use_batched = batch_size * 65536 >= n * n
+
+    if use_batched:
+        handle = device.get_cublas_handle()
+        lda = n
+        step = n * lda * a_t.itemsize
+        start = a_t.data.ptr
+        stop = start + step * batch_size
+        a_array = cupy.arange(start, stop, step, dtype=cupy.uintp)
+
+        if dtype == numpy.float32:
+            getrfBatched = cupy.cuda.cublas.sgetrfBatched
+        elif dtype == numpy.float64:
+            getrfBatched = cupy.cuda.cublas.dgetrfBatched
+        elif dtype == numpy.complex64:
+            getrfBatched = cupy.cuda.cublas.cgetrfBatched
+        elif dtype == numpy.complex128:
+            getrfBatched = cupy.cuda.cublas.zgetrfBatched
+        else:
+            assert False
+
+        getrfBatched(
+            handle, n, a_array.data.ptr, lda, ipiv.data.ptr,
+            dev_info.data.ptr, batch_size)
+
+    else:
+        handle = device.get_cusolver_handle()
+        if dtype == numpy.float32:
+            getrf_bufferSize = cusolver.sgetrf_bufferSize
+            getrf = cusolver.sgetrf
+        elif dtype == numpy.float64:
+            getrf_bufferSize = cusolver.dgetrf_bufferSize
+            getrf = cusolver.dgetrf
+        elif dtype == numpy.complex64:
+            getrf_bufferSize = cusolver.cgetrf_bufferSize
+            getrf = cusolver.cgetrf
+        elif dtype == numpy.complex128:
+            getrf_bufferSize = cusolver.zgetrf_bufferSize
+            getrf = cusolver.zgetrf
+        else:
+            assert False
+
+        for i in range(batch_size):
+            a_ptr = a_t[i].data.ptr
+            buffersize = getrf_bufferSize(handle, n, n, a_ptr, n)
+            workspace = cupy.empty(buffersize, dtype=dtype)
+            getrf(
+                handle, n, n, a_ptr, n, workspace.data.ptr,
+                ipiv[i].data.ptr, dev_info[i].data.ptr)
+
+    return (
+        a_t.reshape(orig_shape),
+        ipiv.reshape(orig_shape[:-1]),
+        dev_info.reshape(orig_shape[:-2]),
+    )
+
+
+def _potrf_batched(a):
+    """Batched Cholesky decomposition.
+
+    Decompose a given array of two-dimensional square matrices into
+    ``L * L.T``, where ``L`` is a lower-triangular matrix and ``.T``
+    is a conjugate transpose operator.
+
+    Args:
+        a (cupy.ndarray): The input array of matrices
+            with dimension ``(..., N, N)``
+
+    Returns:
+        cupy.ndarray: The lower-triangular matrix.
+    """
+    if not check_availability('potrfBatched'):
+        raise RuntimeError('potrfBatched is not available')
+
+    dtype, out_dtype = _util.linalg_common_type(a)
+    if a.size == 0:
+        return cupy.empty(a.shape, out_dtype)
+
+    if dtype == 'f':
+        potrfBatched = cusolver.spotrfBatched
+    elif dtype == 'd':
+        potrfBatched = cusolver.dpotrfBatched
+    elif dtype == 'F':
+        potrfBatched = cusolver.cpotrfBatched
+    else:  # dtype == 'D':
+        potrfBatched = cusolver.zpotrfBatched
+
+    x = a.astype(dtype, order='C', copy=True)
+    xp = cupy._core._mat_ptrs(x)
+    n = x.shape[-1]
+    ldx = x.strides[-2] // x.dtype.itemsize
+    handle = device.get_cusolver_handle()
+    batch_size = internal.prod(x.shape[:-2])
+    dev_info = cupy.empty(batch_size, dtype=numpy.int32)
+
+    potrfBatched(
+        handle, cublas.CUBLAS_FILL_MODE_UPPER, n, xp.data.ptr, ldx,
+        dev_info.data.ptr, batch_size)
+    cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        potrfBatched, dev_info)
+
+    return cupy.tril(x).astype(out_dtype, copy=False)
+
+
+def cholesky(a):
+    """Cholesky decomposition.
+
+    Decompose a given two-dimensional square matrix into ``L * L.H``,
+    where ``L`` is a lower-triangular matrix and ``.H`` is a conjugate
+    transpose operator.
+
+    Args:
+        a (cupy.ndarray): Hermitian (symmetric if all elements are real),
+            positive-definite input matrix with dimension ``(..., M, M)``.
+
+    Returns:
+        cupy.ndarray: The lower-triangular matrix of shape ``(..., M, M)``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.cholesky`
+    """
+    _util._assert_cupy_array(a)
+    _util._assert_stacked_2d(a)
+    _util._assert_stacked_square(a)
+
+    if a.ndim > 2:
+        return _potrf_batched(a)
+
+    dtype, out_dtype = _util.linalg_common_type(a)
+    if a.size == 0:
+        return cupy.empty(a.shape, out_dtype)
+
+    x = a.astype(dtype, order='C', copy=True)
+    n = len(a)
+    handle = device.get_cusolver_handle()
+    dev_info = cupy.empty(1, dtype=numpy.int32)
+
+    if dtype == 'f':
+        potrf = cusolver.spotrf
+        potrf_bufferSize = cusolver.spotrf_bufferSize
+    elif dtype == 'd':
+        potrf = cusolver.dpotrf
+        potrf_bufferSize = cusolver.dpotrf_bufferSize
+    elif dtype == 'F':
+        potrf = cusolver.cpotrf
+        potrf_bufferSize = cusolver.cpotrf_bufferSize
+    else:  # dtype == 'D':
+        potrf = cusolver.zpotrf
+        potrf_bufferSize = cusolver.zpotrf_bufferSize
+
+    buffersize = potrf_bufferSize(
+        handle, cublas.CUBLAS_FILL_MODE_UPPER, n, x.data.ptr, n)
+    workspace = cupy.empty(buffersize, dtype=dtype)
+    potrf(
+        handle, cublas.CUBLAS_FILL_MODE_UPPER, n, x.data.ptr, n,
+        workspace.data.ptr, buffersize, dev_info.data.ptr)
+    cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        potrf, dev_info)
+
+    _util._tril(x, k=0)
+    return x.astype(out_dtype, copy=False)
+
+
+def _qr_batched(a, mode):
+    batch_shape = a.shape[:-2]
+    batch_size = internal.prod(batch_shape)
+    m, n = a.shape[-2:]
+    k = min(m, n)
+
+    # first handle any 0-size inputs
+    if batch_size == 0 or k == 0:
+        # support float32, float64, complex64, and complex128
+        dtype, out_dtype = _util.linalg_common_type(a)
+
+        if mode == 'reduced':
+            return (cupy.empty(batch_shape + (m, k), out_dtype),
+                    cupy.empty(batch_shape + (k, n), out_dtype))
+        elif mode == 'complete':
+            q = _util.stacked_identity(batch_shape, m, out_dtype)
+            return (q, cupy.empty(batch_shape + (m, n), out_dtype))
+        elif mode == 'r':
+            return cupy.empty(batch_shape + (k, n), out_dtype)
+        elif mode == 'raw':
+            return (cupy.empty(batch_shape + (n, m), out_dtype),
+                    cupy.empty(batch_shape + (k,), out_dtype))
+
+    # ...then delegate real computation to cuSOLVER/rocSOLVER
+    a = a.reshape(-1, *(a.shape[-2:]))
+    out = _geqrf_orgqr_batched(a, mode)
+
+    if mode == 'r':
+        return out.reshape(batch_shape + out.shape[-2:])
+    q, r = out
+    q = q.reshape(batch_shape + q.shape[-2:])
+    idx = -1 if mode == 'raw' else -2
+    r = r.reshape(batch_shape + r.shape[idx:])
+    return (q, r)
+
+
+def qr(a, mode='reduced'):
+    """QR decomposition.
+
+    Decompose a given two-dimensional matrix into ``Q * R``, where ``Q``
+    is an orthonormal and ``R`` is an upper-triangular matrix.
+
+    Args:
+        a (cupy.ndarray): The input matrix.
+        mode (str): The mode of decomposition. Currently 'reduced',
+            'complete', 'r', and 'raw' modes are supported. The default mode
+            is 'reduced', in which matrix ``A = (..., M, N)`` is decomposed
+            into ``Q``, ``R`` with dimensions ``(..., M, K)``, ``(..., K, N)``,
+            where ``K = min(M, N)``.
+
+    Returns:
+        cupy.ndarray, or tuple of ndarray:
+            Although the type of returned object depends on the mode,
+            it returns a tuple of ``(Q, R)`` by default.
+            For details, please see the document of :func:`numpy.linalg.qr`.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.qr`
+    """
+    _util._assert_cupy_array(a)
+
+    if mode not in ('reduced', 'complete', 'r', 'raw'):
+        if mode in ('f', 'full', 'e', 'economic'):
+            msg = 'The deprecated mode \'{}\' is not supported'.format(mode)
+        else:
+            msg = 'Unrecognized mode \'{}\''.format(mode)
+        raise ValueError(msg)
+    if a.ndim > 2:
+        return _qr_batched(a, mode)
+
+    # support float32, float64, complex64, and complex128
+    dtype, out_dtype = _util.linalg_common_type(a)
+
+    m, n = a.shape
+    k = min(m, n)
+    if k == 0:
+        if mode == 'reduced':
+            return cupy.empty((m, 0), out_dtype), cupy.empty((0, n), out_dtype)
+        elif mode == 'complete':
+            return cupy.identity(m, out_dtype), cupy.empty((m, n), out_dtype)
+        elif mode == 'r':
+            return cupy.empty((0, n), out_dtype)
+        else:  # mode == 'raw'
+            return cupy.empty((n, m), out_dtype), cupy.empty((0,), out_dtype)
+
+    x = a.transpose().astype(dtype, order='C', copy=True)
+    handle = device.get_cusolver_handle()
+    dev_info = cupy.empty(1, dtype=numpy.int32)
+
+    if dtype == 'f':
+        geqrf_bufferSize = cusolver.sgeqrf_bufferSize
+        geqrf = cusolver.sgeqrf
+    elif dtype == 'd':
+        geqrf_bufferSize = cusolver.dgeqrf_bufferSize
+        geqrf = cusolver.dgeqrf
+    elif dtype == 'F':
+        geqrf_bufferSize = cusolver.cgeqrf_bufferSize
+        geqrf = cusolver.cgeqrf
+    elif dtype == 'D':
+        geqrf_bufferSize = cusolver.zgeqrf_bufferSize
+        geqrf = cusolver.zgeqrf
+    else:
+        msg = ('dtype must be float32, float64, complex64 or complex128'
+               ' (actual: {})'.format(a.dtype))
+        raise ValueError(msg)
+
+    # compute working space of geqrf and solve R
+    buffersize = geqrf_bufferSize(handle, m, n, x.data.ptr, n)
+    workspace = cupy.empty(buffersize, dtype=dtype)
+    tau = cupy.empty(k, dtype=dtype)
+    geqrf(handle, m, n, x.data.ptr, m,
+          tau.data.ptr, workspace.data.ptr, buffersize, dev_info.data.ptr)
+    cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        geqrf, dev_info)
+
+    if mode == 'r':
+        r = x[:, :k].transpose()
+        return _util._triu(r).astype(out_dtype, copy=False)
+
+    if mode == 'raw':
+        return (
+            x.astype(out_dtype, copy=False),
+            tau.astype(out_dtype, copy=False))
+
+    if mode == 'complete' and m > n:
+        mc = m
+        q = cupy.empty((m, m), dtype)
+    else:
+        mc = k
+        q = cupy.empty((n, m), dtype)
+    q[:n] = x
+
+    # compute working space of orgqr and solve Q
+    if dtype == 'f':
+        orgqr_bufferSize = cusolver.sorgqr_bufferSize
+        orgqr = cusolver.sorgqr
+    elif dtype == 'd':
+        orgqr_bufferSize = cusolver.dorgqr_bufferSize
+        orgqr = cusolver.dorgqr
+    elif dtype == 'F':
+        orgqr_bufferSize = cusolver.cungqr_bufferSize
+        orgqr = cusolver.cungqr
+    elif dtype == 'D':
+        orgqr_bufferSize = cusolver.zungqr_bufferSize
+        orgqr = cusolver.zungqr
+
+    buffersize = orgqr_bufferSize(
+        handle, m, mc, k, q.data.ptr, m, tau.data.ptr)
+    workspace = cupy.empty(buffersize, dtype=dtype)
+    orgqr(
+        handle, m, mc, k, q.data.ptr, m, tau.data.ptr, workspace.data.ptr,
+        buffersize, dev_info.data.ptr)
+    cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        orgqr, dev_info)
+
+    q = q[:mc].transpose()
+    r = x[:, :mc].transpose()
+    return (
+        q.astype(out_dtype, copy=False),
+        _util._triu(r).astype(out_dtype, copy=False))
+
+
+def _svd_batched(a, full_matrices, compute_uv):
+    batch_shape = a.shape[:-2]
+    batch_size = internal.prod(batch_shape)
+    n, m = a.shape[-2:]
+
+    dtype, uv_dtype = _util.linalg_common_type(a)
+    s_dtype = uv_dtype.char.lower()
+
+    # first handle any 0-size inputs
+    if batch_size == 0:
+        k = min(m, n)
+        s = cupy.empty(batch_shape + (k,), s_dtype)
+        if compute_uv:
+            if full_matrices:
+                u = cupy.empty(batch_shape + (n, n), dtype=uv_dtype)
+                vt = cupy.empty(batch_shape + (m, m), dtype=uv_dtype)
+            else:
+                u = cupy.empty(batch_shape + (n, k), dtype=uv_dtype)
+                vt = cupy.empty(batch_shape + (k, m), dtype=uv_dtype)
+            return u, s, vt
+        else:
+            return s
+    elif m == 0 or n == 0:
+        s = cupy.empty(batch_shape + (0,), s_dtype)
+        if compute_uv:
+            if full_matrices:
+                u = _util.stacked_identity(batch_shape, n, uv_dtype)
+                vt = _util.stacked_identity(batch_shape, m, uv_dtype)
+            else:
+                u = cupy.empty(batch_shape + (n, 0), dtype=uv_dtype)
+                vt = cupy.empty(batch_shape + (0, m), dtype=uv_dtype)
+            return u, s, vt
+        else:
+            return s
+
+    # ...then delegate real computation to cuSOLVER
+    a = a.reshape(-1, *(a.shape[-2:]))
+    if runtime.is_hip or (m <= 32 and n <= 32):
+        # copy is done in _gesvdj_batched, so let's try not to do it here
+        a = a.astype(dtype, order='C', copy=False)
+        out = _gesvdj_batched(a, full_matrices, compute_uv, False)
+    else:
+        # manually loop over cusolverDn<t>gesvd()
+        # copy (via possible type casting) is done in _gesvd_batched
+        # note: _gesvd_batched returns V, not V^H
+        out = _gesvd_batched(a, dtype.char, full_matrices, compute_uv, False)
+
+    if compute_uv:
+        u, s, v = out
+        u = u.astype(uv_dtype, copy=False)
+        u = u.reshape(*batch_shape, *(u.shape[-2:]))
+        s = s.astype(s_dtype, copy=False)
+        s = s.reshape(*batch_shape, *(s.shape[-1:]))
+        v = v.astype(uv_dtype, copy=False)
+        v = v.reshape(*batch_shape, *(v.shape[-2:]))
+        return u, s, v.swapaxes(-2, -1).conj()
+    else:
+        s = out
+        s = s.astype(s_dtype, copy=False)
+        s = s.reshape(*batch_shape, *(s.shape[-1:]))
+        return s
+
+
+# TODO(leofang): support the hermitian keyword?
+def svd(a, full_matrices=True, compute_uv=True):
+    """Singular Value Decomposition.
+
+    Factorizes the matrix ``a`` as ``u * np.diag(s) * v``, where ``u`` and
+    ``v`` are unitary and ``s`` is an one-dimensional array of ``a``'s
+    singular values.
+
+    Args:
+        a (cupy.ndarray): The input matrix with dimension ``(..., M, N)``.
+        full_matrices (bool): If True, it returns u and v with dimensions
+            ``(..., M, M)`` and ``(..., N, N)``. Otherwise, the dimensions
+            of u and v are ``(..., M, K)`` and ``(..., K, N)``, respectively,
+            where ``K = min(M, N)``.
+        compute_uv (bool): If ``False``, it only returns singular values.
+
+    Returns:
+        tuple of :class:`cupy.ndarray`:
+            A tuple of ``(u, s, v)`` such that ``a = u * np.diag(s) * v``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. note::
+        On CUDA, when ``a.ndim > 2`` and the matrix dimensions <= 32, a fast
+        code path based on Jacobian method (``gesvdj``) is taken. Otherwise,
+        a QR method (``gesvd``) is used.
+
+        On ROCm, there is no such a fast code path that switches the underlying
+        algorithm.
+
+    .. seealso:: :func:`numpy.linalg.svd`
+    """
+    _util._assert_cupy_array(a)
+    if a.ndim > 2:
+        return _svd_batched(a, full_matrices, compute_uv)
+
+    # Cast to float32 or float64
+    dtype, uv_dtype = _util.linalg_common_type(a)
+    real_dtype = dtype.char.lower()
+    s_dtype = uv_dtype.char.lower()
+
+    # Remark 1: gesvd only supports m >= n (WHAT?)
+    # Remark 2: gesvd returns matrix U and V^H
+    n, m = a.shape
+
+    if m == 0 or n == 0:
+        s = cupy.empty((0,), s_dtype)
+        if compute_uv:
+            if full_matrices:
+                u = cupy.eye(n, dtype=uv_dtype)
+                vt = cupy.eye(m, dtype=uv_dtype)
+            else:
+                u = cupy.empty((n, 0), dtype=uv_dtype)
+                vt = cupy.empty((0, m), dtype=uv_dtype)
+            return u, s, vt
+        else:
+            return s
+
+    # `a` must be copied because xgesvd destroys the matrix
+    if m >= n:
+        x = a.astype(dtype, order='C', copy=True)
+        trans_flag = False
+    else:
+        m, n = a.shape
+        x = a.transpose().astype(dtype, order='C', copy=True)
+        trans_flag = True
+
+    k = n  # = min(m, n) where m >= n is ensured above
+    if compute_uv:
+        if full_matrices:
+            u = cupy.empty((m, m), dtype=dtype)
+            vt = x[:, :n]
+            job_u = ord('A')
+            job_vt = ord('O')
+        else:
+            u = x
+            vt = cupy.empty((k, n), dtype=dtype)
+            job_u = ord('O')
+            job_vt = ord('S')
+        u_ptr, vt_ptr = u.data.ptr, vt.data.ptr
+    else:
+        u_ptr, vt_ptr = 0, 0  # Use nullptr
+        job_u = ord('N')
+        job_vt = ord('N')
+    s = cupy.empty(k, dtype=real_dtype)
+    handle = device.get_cusolver_handle()
+    dev_info = cupy.empty(1, dtype=numpy.int32)
+
+    if dtype == 'f':
+        gesvd = cusolver.sgesvd
+        gesvd_bufferSize = cusolver.sgesvd_bufferSize
+    elif dtype == 'd':
+        gesvd = cusolver.dgesvd
+        gesvd_bufferSize = cusolver.dgesvd_bufferSize
+    elif dtype == 'F':
+        gesvd = cusolver.cgesvd
+        gesvd_bufferSize = cusolver.cgesvd_bufferSize
+    else:  # dtype == 'D':
+        gesvd = cusolver.zgesvd
+        gesvd_bufferSize = cusolver.zgesvd_bufferSize
+
+    buffersize = gesvd_bufferSize(handle, m, n)
+    workspace = cupy.empty(buffersize, dtype=dtype)
+    if not runtime.is_hip:
+        # rwork can be NULL if the information from supperdiagonal isn't needed
+        # https://docs.nvidia.com/cuda/cusolver/index.html#cuSolverDN-lt-t-gt-gesvd  # noqa
+        rwork_ptr = 0
+    else:
+        rwork = cupy.empty(min(m, n)-1, dtype=s_dtype)
+        rwork_ptr = rwork.data.ptr
+    gesvd(
+        handle, job_u, job_vt, m, n, x.data.ptr, m, s.data.ptr, u_ptr, m,
+        vt_ptr, n, workspace.data.ptr, buffersize, rwork_ptr,
+        dev_info.data.ptr)
+    cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        gesvd, dev_info)
+
+    s = s.astype(s_dtype, copy=False)
+
+    # Note that the returned array may need to be transposed
+    # depending on the structure of an input
+    if compute_uv:
+        u = u.astype(uv_dtype, copy=False)
+        vt = vt.astype(uv_dtype, copy=False)
+        if trans_flag:
+            return u.transpose(), s, vt.transpose()
+        else:
+            return vt, s, u
+    else:
+        return s
diff --git a/cupy/linalg/_eigenvalue.py b/cupy/linalg/_eigenvalue.py
new file mode 100644
index 0000000..ea69f82
--- /dev/null
+++ b/cupy/linalg/_eigenvalue.py
@@ -0,0 +1,190 @@
+import numpy
+
+import cupy
+from cupy_backends.cuda.libs import cublas
+from cupy_backends.cuda.libs import cusolver
+from cupy.cuda import device
+from cupy.cuda import runtime
+from cupy.linalg import _util
+from cupy._core import _dtype
+import cupyx.cusolver
+
+
+_cuda_runtime_version = -1
+
+
+def _syevd(a, UPLO, with_eigen_vector, overwrite_a=False):
+    if UPLO not in ('L', 'U'):
+        raise ValueError('UPLO argument must be \'L\' or \'U\'')
+
+    # reject_float16=False for backward compatibility
+    dtype, v_dtype = _util.linalg_common_type(a, reject_float16=False)
+    real_dtype = dtype.char.lower()
+    w_dtype = v_dtype.char.lower()
+
+    # Note that cuSolver assumes fortran array
+    v = a.astype(dtype, order='F', copy=not overwrite_a)
+
+    m, lda = a.shape
+    w = cupy.empty(m, real_dtype)
+    dev_info = cupy.empty((), numpy.int32)
+    handle = device.Device().cusolver_handle
+
+    if with_eigen_vector:
+        jobz = cusolver.CUSOLVER_EIG_MODE_VECTOR
+    else:
+        jobz = cusolver.CUSOLVER_EIG_MODE_NOVECTOR
+
+    if UPLO == 'L':
+        uplo = cublas.CUBLAS_FILL_MODE_LOWER
+    else:  # UPLO == 'U'
+        uplo = cublas.CUBLAS_FILL_MODE_UPPER
+
+    global _cuda_runtime_version
+    if _cuda_runtime_version < 0:
+        _cuda_runtime_version = runtime.runtimeGetVersion()
+
+    if not runtime.is_hip and _cuda_runtime_version >= 11010:
+        if dtype.char not in 'fdFD':
+            raise RuntimeError('Only float32, float64, complex64, and '
+                               'complex128 are supported')
+        type_v = _dtype.to_cuda_dtype(dtype)
+        type_w = _dtype.to_cuda_dtype(real_dtype)
+        params = cusolver.createParams()
+        try:
+            work_device_size, work_host_sizse = cusolver.xsyevd_bufferSize(
+                handle, params, jobz, uplo, m, type_v, v.data.ptr, lda,
+                type_w, w.data.ptr, type_v)
+            work_device = cupy.empty(work_device_size, 'b')
+            work_host = numpy.empty(work_host_sizse, 'b')
+            cusolver.xsyevd(
+                handle, params, jobz, uplo, m, type_v, v.data.ptr, lda,
+                type_w, w.data.ptr, type_v,
+                work_device.data.ptr, work_device_size,
+                work_host.ctypes.data, work_host_sizse, dev_info.data.ptr)
+        finally:
+            cusolver.destroyParams(params)
+        cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+            cusolver.xsyevd, dev_info)
+    else:
+        if dtype == 'f':
+            buffer_size = cupy.cuda.cusolver.ssyevd_bufferSize
+            syevd = cupy.cuda.cusolver.ssyevd
+        elif dtype == 'd':
+            buffer_size = cupy.cuda.cusolver.dsyevd_bufferSize
+            syevd = cupy.cuda.cusolver.dsyevd
+        elif dtype == 'F':
+            buffer_size = cupy.cuda.cusolver.cheevd_bufferSize
+            syevd = cupy.cuda.cusolver.cheevd
+        elif dtype == 'D':
+            buffer_size = cupy.cuda.cusolver.zheevd_bufferSize
+            syevd = cupy.cuda.cusolver.zheevd
+        else:
+            raise RuntimeError('Only float32, float64, complex64, and '
+                               'complex128 are supported')
+
+        work_size = buffer_size(
+            handle, jobz, uplo, m, v.data.ptr, lda, w.data.ptr)
+        work = cupy.empty(work_size, dtype)
+        syevd(
+            handle, jobz, uplo, m, v.data.ptr, lda,
+            w.data.ptr, work.data.ptr, work_size, dev_info.data.ptr)
+        cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+            syevd, dev_info)
+
+    return w.astype(w_dtype, copy=False), v.astype(v_dtype, copy=False)
+
+
+# TODO(okuta): Implement eig
+
+
+def eigh(a, UPLO='L'):
+    """
+    Return the eigenvalues and eigenvectors of a complex Hermitian
+    (conjugate symmetric) or a real symmetric matrix.
+
+    Returns two objects, a 1-D array containing the eigenvalues of `a`, and
+    a 2-D square array or matrix (depending on the input type) of the
+    corresponding eigenvectors (in columns).
+
+    Args:
+        a (cupy.ndarray): A symmetric 2-D square matrix ``(M, M)`` or a batch
+            of symmetric 2-D square matrices ``(..., M, M)``.
+        UPLO (str): Select from ``'L'`` or ``'U'``. It specifies which
+            part of ``a`` is used. ``'L'`` uses the lower triangular part of
+            ``a``, and ``'U'`` uses the upper triangular part of ``a``.
+    Returns:
+        tuple of :class:`~cupy.ndarray`:
+            Returns a tuple ``(w, v)``. ``w`` contains eigenvalues and
+            ``v`` contains eigenvectors. ``v[:, i]`` is an eigenvector
+            corresponding to an eigenvalue ``w[i]``. For batch input,
+            ``v[k, :, i]`` is an eigenvector corresponding to an eigenvalue
+            ``w[k, i]`` of ``a[k]``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.eigh`
+    """
+    _util._assert_stacked_2d(a)
+    _util._assert_stacked_square(a)
+
+    if a.size == 0:
+        _, v_dtype = _util.linalg_common_type(a)
+        w_dtype = v_dtype.char.lower()
+        w = cupy.empty(a.shape[:-1], w_dtype)
+        v = cupy.empty(a.shape, v_dtype)
+        return w, v
+
+    if a.ndim > 2 or runtime.is_hip:
+        w, v = cupyx.cusolver.syevj(a, UPLO, True)
+        return w, v
+    else:
+        return _syevd(a, UPLO, True)
+
+
+# TODO(okuta): Implement eigvals
+
+
+def eigvalsh(a, UPLO='L'):
+    """
+    Compute the eigenvalues of a complex Hermitian or real symmetric matrix.
+
+    Main difference from eigh: the eigenvectors are not computed.
+
+    Args:
+        a (cupy.ndarray): A symmetric 2-D square matrix ``(M, M)`` or a batch
+            of symmetric 2-D square matrices ``(..., M, M)``.
+        UPLO (str): Select from ``'L'`` or ``'U'``. It specifies which
+            part of ``a`` is used. ``'L'`` uses the lower triangular part of
+            ``a``, and ``'U'`` uses the upper triangular part of ``a``.
+    Returns:
+        cupy.ndarray:
+            Returns eigenvalues as a vector ``w``. For batch input,
+            ``w[k]`` is a vector of eigenvalues of matrix ``a[k]``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.eigvalsh`
+    """
+    _util._assert_stacked_2d(a)
+    _util._assert_stacked_square(a)
+
+    if a.size == 0:
+        _, v_dtype = _util.linalg_common_type(a)
+        w_dtype = v_dtype.char.lower()
+        return cupy.empty(a.shape[:-1], w_dtype)
+
+    if a.ndim > 2 or runtime.is_hip:
+        return cupyx.cusolver.syevj(a, UPLO, False)
+    else:
+        return _syevd(a, UPLO, False)[0]
diff --git a/cupy/linalg/_einsum.py b/cupy/linalg/_einsum.py
new file mode 100644
index 0000000..1a27f44
--- /dev/null
+++ b/cupy/linalg/_einsum.py
@@ -0,0 +1,696 @@
+import copy
+import itertools
+import operator
+import string
+import warnings
+
+import cupy
+from cupy._core import _accelerator
+from cupy import _util
+from cupy.linalg._einsum_opt import _greedy_path
+from cupy.linalg._einsum_opt import _optimal_path
+from cupy.linalg._einsum_cutn import _try_use_cutensornet
+
+
+try:
+    import cupy_backends.cuda.libs.cutensor  # NOQA
+    from cupyx import cutensor
+except ImportError:
+    cutensor = None
+
+
+options = {
+    'sum_ellipsis': False,
+    'broadcast_diagonal': False,
+}
+
+
+einsum_symbols = string.ascii_uppercase + string.ascii_lowercase
+
+
+def _transpose_ex(a, axeses):
+    """Transpose and diagonal
+
+    Args:
+        a
+        axeses (sequence of sequences of ints)
+
+    Returns:
+        ndarray: a with its axes permutated. A writeable view is returned
+        whenever possible.
+    """
+
+    shape = []
+    strides = []
+    for axes in axeses:
+        shape.append(a.shape[axes[0]] if axes else 1)
+        stride = sum(a.strides[axis] for axis in axes)
+        strides.append(stride)
+    a = a.view()
+    # TODO(niboshi): Confirm update_x_contiguity flags
+    a._set_shape_and_strides(shape, strides, True, True)
+    return a
+
+
+def _parse_int_subscript(list_subscript):
+    str_subscript = ''
+    for s in list_subscript:
+        if s is Ellipsis:
+            str_subscript += '@'
+        else:
+            try:
+                s = operator.index(s)
+            except TypeError as e:
+                raise TypeError(
+                    'For this input type lists must contain '
+                    'either int or Ellipsis') from e
+            str_subscript += einsum_symbols[s]
+    return str_subscript
+
+
+def _parse_einsum_input(args):
+    """Parse einsum operands.
+
+    This function is based on `numpy.core.einsumfunc._parse_einsum_input`
+    function in NumPy 1.14.
+
+    Parameters
+    ----------
+    args : tuple
+        The non-keyword arguments to einsum
+
+    Returns
+    -------
+    input_strings : str
+        Parsed input strings
+    output_string : str
+        Parsed output string
+    operands : list of array_like
+        The operands to use in the contraction
+
+    Examples
+    --------
+    The operand list is simplified to reduce printing:
+
+    >>> a = np.random.rand(4, 4)
+    >>> b = np.random.rand(4, 4, 4)
+    >>> _parse_einsum_input(('...a,...a->...', a, b))
+    (['@a, @a'], 'xz', [a, b])
+
+    >>> _parse_einsum_input((a, [Ellipsis, 0], b, [Ellipsis, 0]))
+    (['@a, @a'], 'xz', [a, b])
+    """
+
+    if len(args) == 0:
+        raise ValueError(
+            'must specify the einstein sum subscripts string and at least one '
+            'operand, or at least one operand and its corresponding '
+            'subscripts list')
+
+    if isinstance(args[0], str):
+        subscripts = args[0]
+        operands = list(args[1:])
+
+        # Ensure all characters are valid
+        for s in subscripts:
+            if s in '.,-> ':
+                continue
+            if s not in einsum_symbols:
+                raise ValueError(
+                    'invalid subscript \'%s\' in einstein sum subscripts '
+                    'string, subscripts must be letters' % s)
+
+        # Parse '...'
+        subscripts = subscripts.replace('...', '@')
+        if '.' in subscripts:
+            raise ValueError(
+                'einstein sum subscripts string contains a \'.\' that is not '
+                'part of an ellipsis (\'...\')')
+
+        # Parse '->'
+        if ('-' in subscripts) or ('>' in subscripts):
+            # Check for proper '->'
+            invalid = subscripts.count('-') > 1 or subscripts.count('>') > 1
+            subscripts = subscripts.split('->')
+            if invalid or len(subscripts) != 2:
+                raise ValueError(
+                    'einstein sum subscript string does not contain proper '
+                    '\'->\' output specified')
+            input_subscripts, output_subscript = subscripts
+            output_subscript = output_subscript.replace(' ', '')
+
+        else:
+            input_subscripts = subscripts
+            output_subscript = None
+
+        input_subscripts = input_subscripts.replace(' ', '').split(',')
+        if len(input_subscripts) != len(operands):
+            msg = 'more' if len(operands) > len(input_subscripts) else 'fewer'
+            raise ValueError(
+                msg + ' operands provided to einstein sum function than '
+                'specified in the subscripts string')
+
+    else:
+        args = list(args)
+        operands = []
+        input_subscripts = []
+        while len(args) >= 2:
+            operands.append(args.pop(0))
+            input_subscripts.append(_parse_int_subscript(args.pop(0)))
+        if args:
+            output_subscript = _parse_int_subscript(args[0])
+        else:
+            output_subscript = None
+
+    return input_subscripts, output_subscript, operands
+
+
+def _chr(label):
+    if label < 0:
+        return '...[%d]' % label
+    else:
+        return chr(label)
+
+
+def _parse_ellipsis_subscript(subscript, idx, ndim=None, ellipsis_len=None):
+    """Parse a subscript that may contain ellipsis
+
+    Args:
+        subscript (str): An einsum subscript of an operand or an output. '...'
+            should be replaced by '@'.
+        idx (int or None): For error messages, give int idx for the idx-th
+            operand or None for the output.
+        ndim (int, optional): ndim of the operand
+        ellipsis_len (int, optional): number of broadcast dimensions of the
+            output.
+
+    Returns:
+        list of ints: The parsed subscript
+
+    """
+    subs = subscript.split('@')
+    if len(subs) == 1:
+        sub, = subs
+        if ndim is not None and len(sub) != ndim:
+            if len(sub) > ndim:
+                raise ValueError(
+                    'einstein sum subscripts string %s contains too many '
+                    'subscripts for operand %d' % (sub, idx))
+            raise ValueError(
+                'operand %d has more dimensions than subscripts string %s '
+                'given in einstein sum, but no \'...\' ellipsis provided to '
+                'broadcast the extra dimensions.' % (idx, sub))
+        return [ord(label) for label in sub]
+    elif len(subs) == 2:
+        left_sub, right_sub = subs
+        if ndim is not None:
+            ellipsis_len = ndim - (len(left_sub) + len(right_sub))
+        if ellipsis_len < 0:
+            raise ValueError(
+                'einstein sum subscripts string %s...%s contains too many '
+                'subscripts for operand %d' % (left_sub, right_sub, idx))
+        ret = []
+        ret.extend(ord(label) for label in left_sub)
+        ret.extend(range(-ellipsis_len, 0))
+        ret.extend(ord(label) for label in right_sub)
+        return ret
+    else:
+        # >= 2 ellipses for an operand
+        raise ValueError(
+            'einstein sum subscripts string contains a \'.\' that is not '
+            'part of an ellipsis (\'...\') ' +
+            ('in the output' if idx is None else 'for operand %d' % idx))
+
+
+def _einsum_diagonals(input_subscripts, operands):
+    """Compute diagonal for each operand
+
+    This function mutates args.
+    """
+    for idx in range(len(input_subscripts)):
+        sub = input_subscripts[idx]
+        arr = operands[idx]
+
+        if len(set(sub)) < len(sub):
+            axeses = {}
+            for axis, label in enumerate(sub):
+                axeses.setdefault(label, []).append(axis)
+
+            axeses = list(axeses.items())
+
+            for label, axes in axeses:
+                if options['broadcast_diagonal']:
+                    axes = [axis for axis in axes if arr.shape[axis] != 1]
+                dims = {arr.shape[axis] for axis in axes}
+                if len(dims) >= 2:
+                    dim0 = dims.pop()
+                    dim1 = dims.pop()
+                    raise ValueError(
+                        'dimensions in operand %d'
+                        ' for collapsing index \'%s\' don\'t match (%d != %d)'
+                        % (idx, _chr(label), dim0, dim1)
+                    )
+
+            sub, axeses = zip(*axeses)  # axeses is not empty
+            input_subscripts[idx] = list(sub)
+            operands[idx] = _transpose_ex(arr, axeses)
+
+
+def _iter_path_pairs(path):
+    """Decompose path into binary path
+
+    Args:
+        path (sequence of tuples of ints)
+
+    Yields:
+        tuple of ints: pair (idx0, idx1) that represents the operation
+            {pop(idx0); pop(idx1); append();}
+    """
+
+    for indices in path:
+        assert all(idx >= 0 for idx in indices)
+        # [3, 1, 4, 9] -> [(9, 4), (-1, 3), (-1, 1)]
+        if len(indices) >= 2:
+            indices = sorted(indices, reverse=True)
+            yield indices[0], indices[1]
+            for idx in indices[2:]:
+                yield -1, idx
+
+
+def _flatten_transpose(a, axeses):
+    """Transpose and flatten each
+
+    Args:
+        a
+        axeses (sequence of sequences of ints)
+
+    Returns:
+        aT: a with its axes permutated and flatten
+        shapes: flattened shapes
+    """
+
+    transpose_axes = []
+    shapes = []
+    for axes in axeses:
+        transpose_axes.extend(axes)
+        shapes.append([a.shape[axis] for axis in axes])
+    return (
+        a.transpose(transpose_axes).reshape(
+            tuple([cupy._core.internal.prod(shape) for shape in shapes])),
+        shapes
+    )
+
+
+def _use_cutensor(dtype0, sub0, dtype1, sub1, batch_dims, contract_dims):
+    if not cutensor.check_availability('contraction'):
+        return False
+    if dtype0 != dtype1:
+        return False
+    if dtype0 not in (cupy.float32, cupy.float64,
+                      cupy.complex64, cupy.complex128):
+        return False
+    return True
+
+
+def _get_out_shape(shape0, sub0, shape1, sub1, sub_out):
+    extent = {}
+    for size, i in zip(shape0 + shape1, sub0 + sub1):
+        extent[i] = size
+    out_shape = [extent[i] for i in sub_out]
+    return out_shape
+
+
+def _expand_dims_transpose(arr, mode, mode_out):
+    """Return a reshaped and transposed array.
+
+    The input array ``arr`` having ``mode`` as its modes is reshaped and
+    transposed so that modes of the output becomes ``mode_out``.
+
+    Example
+        >>> import cupy
+        >>> a = cupy.zeros((10, 20))
+        >>> mode_a = ('A', 'B')
+        >>> mode_out = ('B', 'C', 'A')
+        >>> out = cupy.linalg.einsum._expand_dims_transpose(a, mode_a,
+        ...                                                 mode_out)
+        >>> out.shape
+        (20, 1, 10)
+
+    Args:
+        arr (cupy.ndarray):
+        mode (tuple or list): The modes of input array.
+        mode_out (tuple or list): The modes of output array.
+
+    Returns:
+        cupy.ndarray: The reshaped and transposed array.
+
+    """
+    mode = list(mode)
+    shape = list(arr.shape)
+    axes = []
+    for i in mode_out:
+        if i not in mode:
+            mode.append(i)
+            shape.append(1)
+        axes.append(mode.index(i))
+    return cupy.transpose(arr.reshape(shape), axes)
+
+
+def reduced_binary_einsum(arr0, sub0, arr1, sub1, sub_others):
+    set0 = set(sub0)
+    set1 = set(sub1)
+    assert len(set0) == len(sub0), 'operand 0 should be reduced: diagonal'
+    assert len(set1) == len(sub1), 'operand 1 should be reduced: diagonal'
+
+    if len(sub0) == 0 or len(sub1) == 0:
+        return arr0 * arr1, sub0 + sub1
+
+    set_others = set(sub_others)
+    shared = set0 & set1
+    batch_dims = shared & set_others
+    contract_dims = shared - batch_dims
+
+    bs0, cs0, ts0 = _make_transpose_axes(sub0, batch_dims, contract_dims)
+    bs1, cs1, ts1 = _make_transpose_axes(sub1, batch_dims, contract_dims)
+
+    sub_b = [sub0[axis] for axis in bs0]
+    assert sub_b == [sub1[axis] for axis in bs1]
+    sub_l = [sub0[axis] for axis in ts0]
+    sub_r = [sub1[axis] for axis in ts1]
+
+    sub_out = sub_b + sub_l + sub_r
+    assert set(sub_out) <= set_others, 'operands should be reduced: unary sum'
+
+    if len(contract_dims) == 0:
+        # Use element-wise multiply when no contraction is needed
+        if len(sub_out) == len(sub_others):
+            # to assure final output of einsum is C-contiguous
+            sub_out = sub_others
+        arr0 = _expand_dims_transpose(arr0, sub0, sub_out)
+        arr1 = _expand_dims_transpose(arr1, sub1, sub_out)
+        return arr0 * arr1, sub_out
+
+    for accelerator in _accelerator.get_routine_accelerators():
+        if (accelerator == _accelerator.ACCELERATOR_CUTENSOR and
+                cutensor is not None):
+            if _use_cutensor(arr0.dtype, sub0, arr1.dtype, sub1,
+                             batch_dims, contract_dims):
+                if len(sub_out) == len(sub_others):
+                    # to assure final output of einsum is C-contiguous
+                    sub_out = sub_others
+                out_shape = _get_out_shape(
+                    arr0.shape, sub0, arr1.shape, sub1, sub_out)
+                arr_out = cupy.empty(out_shape, arr0.dtype)
+                arr0 = cupy.ascontiguousarray(arr0)
+                arr1 = cupy.ascontiguousarray(arr1)
+                desc_0 = cutensor.create_tensor_descriptor(arr0)
+                desc_1 = cutensor.create_tensor_descriptor(arr1)
+                desc_out = cutensor.create_tensor_descriptor(arr_out)
+                arr_out = cutensor.contraction(
+                    1.0,
+                    arr0, desc_0, sub0,
+                    arr1, desc_1, sub1,
+                    0.0,
+                    arr_out, desc_out, sub_out)
+                return arr_out, sub_out
+
+    tmp0, shapes0 = _flatten_transpose(arr0, [bs0, ts0, cs0])
+    tmp1, shapes1 = _flatten_transpose(arr1, [bs1, cs1, ts1])
+    shapes_out = shapes0[0] + shapes0[1] + shapes1[2]
+    assert shapes0[0] == shapes1[0]
+    arr_out = cupy.matmul(tmp0, tmp1).reshape(shapes_out)
+    return arr_out, sub_out
+
+
+def _make_transpose_axes(sub, b_dims, c_dims):
+    bs = []
+    cs = []
+    ts = []
+    for axis, label in enumerate(sub):
+        if label in b_dims:
+            bs.append((label, axis))
+        elif label in c_dims:
+            cs.append((label, axis))
+        else:
+            ts.append((label, axis))
+    return (
+        _tuple_sorted_by_0(bs),
+        _tuple_sorted_by_0(cs),
+        _tuple_sorted_by_0(ts),
+    )
+
+
+def _tuple_sorted_by_0(zs):
+    return tuple(i for _, i in sorted(zs))
+
+
+def einsum(*operands, **kwargs):
+    """einsum(subscripts, *operands, dtype=None, optimize=False)
+
+    Evaluates the Einstein summation convention on the operands.
+    Using the Einstein summation convention, many common multi-dimensional
+    array operations can be represented in a simple fashion. This function
+    provides a way to compute such summations.
+
+    .. note::
+
+       - Memory contiguity of the returned array is not always compatible with
+         that of :func:`numpy.einsum`.
+       - ``out``, ``order``, and ``casting`` options are not supported.
+       - If :envvar:`CUPY_ACCELERATORS` includes ``cutensornet``, the `einsum`
+         calculation will be performed by the cuTensorNet backend if possible.
+
+           - The support of the ``optimize`` option is limited (currently, only
+             `False`, 'cutensornet', or a custom path for pairwise contraction
+             is supported, and the maximum intermediate size is ignored). If
+             you need finer control for path optimization, consider replacing
+             :func:`cupy.einsum` by :func:`cuquantum.contract` instead.
+           - Requires `cuQuantum Python`_ (v22.03+).
+
+       - If :envvar:`CUPY_ACCELERATORS` includes ``cutensor``, `einsum` will be
+         accelerated by the cuTENSOR backend whenever possible.
+
+    Args:
+        subscripts (str): Specifies the subscripts for summation.
+        operands (sequence of arrays): These are the arrays for the operation.
+        dtype: If provided, forces the calculation to use the data type
+            specified. Default is None.
+        optimize: Valid options include {`False`, `True`, 'greedy', 'optimal'}.
+            Controls if intermediate optimization should occur. No optimization
+            will occur if `False`, and `True` will default to the 'greedy'
+            algorithm. Also accepts an explicit contraction list from
+            :func:`numpy.einsum_path`. Defaults to `False`. If a pair is
+            supplied, the second argument is assumed to be the maximum
+            intermediate size created.
+
+    Returns:
+        cupy.ndarray:
+            The calculation based on the Einstein summation convention.
+
+    .. seealso:: :func:`numpy.einsum`
+    .. _cuQuantum Python: https://docs.nvidia.com/cuda/cuquantum/python/
+    """
+    out = _try_use_cutensornet(*operands, **kwargs)
+    if out is not None:
+        return out
+
+    input_subscripts, output_subscript, operands = \
+        _parse_einsum_input(operands)
+    assert isinstance(input_subscripts, list)
+    assert isinstance(operands, list)
+
+    dtype = kwargs.pop('dtype', None)
+
+    # casting = kwargs.pop('casting', 'safe')
+    casting_kwargs = {}  # casting is not supported yet in astype
+
+    optimize = kwargs.pop('optimize', False)
+    if optimize is True:
+        optimize = 'greedy'
+    if kwargs:
+        raise TypeError('Did not understand the following kwargs: %s'
+                        % list(kwargs.keys()))
+
+    result_dtype = cupy.result_type(*operands) if dtype is None else dtype
+    operands = [
+        cupy.asanyarray(arr)
+        for arr in operands
+    ]
+
+    input_subscripts = [
+        _parse_ellipsis_subscript(sub, idx, ndim=arr.ndim)
+        for idx, (sub, arr) in enumerate(zip(input_subscripts, operands))
+    ]
+
+    # Get length of each unique dimension and ensure all dimensions are correct
+    dimension_dict = {}
+    for idx, sub in enumerate(input_subscripts):
+        sh = operands[idx].shape
+        for axis, label in enumerate(sub):
+            dim = sh[axis]
+            if label in dimension_dict.keys():
+                # For broadcasting cases we always want the largest dim size
+                if dimension_dict[label] == 1:
+                    dimension_dict[label] = dim
+                elif dim not in (1, dimension_dict[label]):
+                    dim_old = dimension_dict[label]
+                    raise ValueError(
+                        'Size of label \'%s\' for operand %d (%d) '
+                        'does not match previous terms (%d).'
+                        % (_chr(label), idx, dim, dim_old))
+            else:
+                dimension_dict[label] = dim
+
+    if output_subscript is None:
+        # Build output subscripts
+        tmp_subscripts = list(itertools.chain.from_iterable(input_subscripts))
+        output_subscript = [
+            label
+            for label in sorted(set(tmp_subscripts))
+            if label < 0 or tmp_subscripts.count(label) == 1
+        ]
+    else:
+        if not options['sum_ellipsis']:
+            if '@' not in output_subscript and -1 in dimension_dict:
+                raise ValueError(
+                    'output has more dimensions than subscripts '
+                    'given in einstein sum, but no \'...\' ellipsis '
+                    'provided to broadcast the extra dimensions.')
+        output_subscript = _parse_ellipsis_subscript(
+            output_subscript, None,
+            ellipsis_len=sum(label < 0 for label in dimension_dict.keys())
+        )
+
+        # Make sure output subscripts are in the input
+        tmp_subscripts = set(itertools.chain.from_iterable(input_subscripts))
+        for label in output_subscript:
+            if label not in tmp_subscripts:
+                raise ValueError(
+                    'einstein sum subscripts string included output subscript '
+                    '\'%s\' which never appeared in an input' % _chr(label))
+        if len(output_subscript) != len(set(output_subscript)):
+            for label in output_subscript:
+                if output_subscript.count(label) >= 2:
+                    raise ValueError(
+                        'einstein sum subscripts string includes output '
+                        'subscript \'%s\' multiple times' % _chr(label))
+
+    _einsum_diagonals(input_subscripts, operands)
+
+    # no more raises
+
+    if len(operands) >= 2:
+        if any(arr.size == 0 for arr in operands):
+            return cupy.zeros(
+                tuple(dimension_dict[label] for label in output_subscript),
+                dtype=result_dtype
+            )
+
+        # Don't squeeze if unary, because this affects later (in trivial sum)
+        # whether the return is a writeable view.
+        for idx in range(len(operands)):
+            arr = operands[idx]
+            if 1 in arr.shape:
+                squeeze_indices = []
+                sub = []
+                for axis, label in enumerate(input_subscripts[idx]):
+                    if arr.shape[axis] == 1:
+                        squeeze_indices.append(axis)
+                    else:
+                        sub.append(label)
+                input_subscripts[idx] = sub
+                operands[idx] = cupy.squeeze(arr, axis=tuple(squeeze_indices))
+                assert operands[idx].ndim == len(input_subscripts[idx])
+            del arr
+
+    # unary einsum without summation should return a (writeable) view
+    returns_view = len(operands) == 1
+
+    # unary sum
+    for idx, sub in enumerate(input_subscripts):
+        other_subscripts = copy.copy(input_subscripts)
+        other_subscripts[idx] = output_subscript
+        other_subscripts = set(itertools.chain.from_iterable(other_subscripts))
+        sum_axes = tuple(
+            axis
+            for axis, label in enumerate(sub)
+            if label not in other_subscripts
+        )
+        if sum_axes:
+            returns_view = False
+            input_subscripts[idx] = [
+                label
+                for axis, label in enumerate(sub)
+                if axis not in sum_axes
+            ]
+
+            operands[idx] = operands[idx].sum(
+                axis=sum_axes, dtype=result_dtype)
+
+    if returns_view:
+        operands = [a.view() for a in operands]
+    else:
+        operands = [
+            a.astype(result_dtype, copy=False, **casting_kwargs)
+            for a in operands
+        ]
+
+    # no more casts
+
+    optimize_algorithms = {
+        'greedy': _greedy_path,
+        'optimal': _optimal_path,
+    }
+    if optimize is False:
+        path = [tuple(range(len(operands)))]
+    elif len(optimize) and (optimize[0] == 'einsum_path'):
+        path = optimize[1:]
+    else:
+        try:
+            if len(optimize) == 2 and isinstance(optimize[1], (int, float)):
+                algo = optimize_algorithms[optimize[0]]
+                memory_limit = int(optimize[1])
+            else:
+                algo = optimize_algorithms[optimize]
+                memory_limit = 2 ** 31  # TODO(kataoka): fix?
+        except (TypeError, KeyError):  # unhashable type or not found
+            raise TypeError('Did not understand the path (optimize): %s'
+                            % str(optimize))
+        input_sets = [set(sub) for sub in input_subscripts]
+        output_set = set(output_subscript)
+        path = algo(input_sets, output_set, dimension_dict, memory_limit)
+        if any(len(indices) > 2 for indices in path):
+            warnings.warn(
+                'memory efficient einsum is not supported yet',
+                _util.PerformanceWarning)
+
+    for idx0, idx1 in _iter_path_pairs(path):
+        # "reduced" binary einsum
+        arr0 = operands.pop(idx0)
+        sub0 = input_subscripts.pop(idx0)
+        arr1 = operands.pop(idx1)
+        sub1 = input_subscripts.pop(idx1)
+        sub_others = list(itertools.chain(
+            output_subscript,
+            itertools.chain.from_iterable(input_subscripts)))
+        arr_out, sub_out = reduced_binary_einsum(
+            arr0, sub0, arr1, sub1, sub_others)
+        operands.append(arr_out)
+        input_subscripts.append(sub_out)
+        del arr0, arr1
+
+    # unary einsum at last
+    arr0, = operands
+    sub0, = input_subscripts
+
+    transpose_axes = []
+    for label in output_subscript:
+        if label in sub0:
+            transpose_axes.append(sub0.index(label))
+
+    arr_out = arr0.transpose(transpose_axes).reshape([
+        dimension_dict[label]
+        for label in output_subscript
+    ])
+    assert returns_view or arr_out.dtype == result_dtype
+    return arr_out
diff --git a/cupy/linalg/_einsum_cutn.py b/cupy/linalg/_einsum_cutn.py
new file mode 100644
index 0000000..f710abc
--- /dev/null
+++ b/cupy/linalg/_einsum_cutn.py
@@ -0,0 +1,168 @@
+import threading
+import warnings
+
+try:
+    import cuquantum
+    from cuquantum import cutensornet
+except ImportError:
+    cuquantum = cutensornet = None
+
+import cupy
+from cupy import _util
+from cupy._core import _accelerator
+from cupy.cuda.device import Handle
+
+
+_tls = threading.local()
+
+
+@_util.memoize()
+def _is_cuqnt_22_11_or_higher():
+    ver = [int(i) for i in cuquantum.__version__.split('.')]
+    if (ver[0] > 22) or (ver[0] == 22 and ver[1] >= 11):
+        return True
+    return False
+
+
+def _is_nonblocking_supported():
+    return _is_cuqnt_22_11_or_higher()
+
+
+def _get_einsum_operands(args):
+    """Parse & retrieve einsum operands, assuming ``args`` is in either
+    "subscript" or "interleaved" format.
+    """
+
+    if len(args) == 0:
+        raise ValueError(
+            'must specify the einstein sum subscripts string and at least one '
+            'operand, or at least one operand and its corresponding '
+            'subscripts list')
+
+    if isinstance(args[0], str):
+        expr = args[0]
+        operands = list(args[1:])
+        return expr, operands
+    else:
+        args = list(args)
+        operands = []
+        inputs = []
+        output = None
+        while len(args) >= 2:
+            operands.append(args.pop(0))
+            inputs.append(args.pop(0))
+        if len(args) == 1:
+            output = args.pop(0)
+        assert not args
+        return inputs, operands, output
+
+
+def _try_use_cutensornet(*args, **kwargs):
+    if cupy.cuda.runtime.is_hip:
+        return None
+
+    if (_accelerator.ACCELERATOR_CUTENSORNET not in
+            _accelerator.get_routine_accelerators()):
+        return None
+
+    if cutensornet is None:
+        warnings.warn(
+            'using the cuTensorNet backend was requested but it cannot be '
+            'imported -- maybe you forgot to install cuQuantum Python? '
+            'Please do "pip install cuquantum-python" or "conda install '
+            '-c conda-forge cuquantum-python" and retry',
+            stacklevel=2)
+        return None
+
+    # cannot pop as we might still need kwargs later
+    dtype = kwargs.get('dtype', None)
+    path = kwargs.get('optimize', False)
+    if path is True:
+        path = 'greedy'
+
+    # we do very lightweight pre-processing here just to inspect the
+    # operands; the actual input verification is deferred to cuTensorNet
+    # which can generate far better diagonostic messages
+    args = _get_einsum_operands(args)
+    operands = [cupy.asarray(op) for op in args[1]]
+
+    if len(operands) == 1:
+        # As of cuTENSOR 1.5.0 it still chokes with some common operations
+        # like trace ("ii->") so it's easier to just skip all single-operand
+        # cases instead of whitelisting what could be done explicitly
+        return None
+
+    if (any(op.size == 0 for op in operands) or
+            any(len(op.shape) == 0 for op in operands)):
+        # To cuTensorNet the shape is invalid
+        return None
+
+    # all input dtypes must be identical (to a numerical dtype)
+    result_dtype = cupy.result_type(*operands) if dtype is None else dtype
+    if result_dtype not in (
+            cupy.float32, cupy.float64, cupy.complex64, cupy.complex128):
+        return None
+    operands = [op.astype(result_dtype, copy=False) for op in operands]
+
+    # prepare cutn inputs
+    device = cupy.cuda.runtime.getDevice()
+    if not hasattr(_tls, "cutn_handle_cache"):
+        cutn_handle_cache = _tls.cutn_handle_cache = {}
+    else:
+        cutn_handle_cache = _tls.cutn_handle_cache
+    handle = cutn_handle_cache.get(device)
+    if handle is None:
+        handle = cutensornet.create()
+        cutn_handle_cache[device] = Handle(handle, cutensornet.destroy)
+    else:
+        handle = handle.handle
+    cutn_options = {'device_id': device, 'handle': handle}
+    if _is_nonblocking_supported():
+        cutn_options['blocking'] = "auto"
+
+    # TODO(leofang): support all valid combinations:
+    # - path from user, contract with cutn (done)
+    # - path from cupy, contract with cutn (not yet)
+    # - path from cutn, contract with cutn (done)
+    # - path from cutn, contract with cupy (not yet)
+    raise_warning = False
+    if path is False:
+        # following the same convention (contracting from the right) as would
+        # be produced by _iter_path_pairs(), but converting to a list of pairs
+        # due to cuTensorNet's requirement
+        path = [(i-1, i-2) for i in range(len(operands), 1, -1)]
+    elif len(path) and path[0] == 'einsum_path':
+        # let cuTensorNet check if the format is correct
+        path = path[1:]
+    elif len(path) == 2:
+        if isinstance(path[1], (int, float)):
+            raise_warning = True
+        if path[0] != 'cutensornet':
+            raise_warning = True
+        path = None
+    else:  # path is a string
+        if path != 'cutensornet':
+            raise_warning = True
+        path = None
+    if raise_warning:
+        warnings.warn(
+            'the cuTensorNet backend ignores the "optimize" option '
+            'except when an explicit contraction path is provided '
+            'or when optimize=False (disable optimization); also, '
+            'the maximum intermediate size, if set, is ignored',
+            stacklevel=2)
+    cutn_optimizer = {'path': path} if path else None
+
+    if len(args) == 2:
+        out = cutensornet.contract(
+            args[0], *operands, options=cutn_options, optimize=cutn_optimizer)
+    elif len(args) == 3:
+        inputs = [i for pair in zip(operands, args[0]) for i in pair]
+        if args[2] is not None:
+            inputs.append(args[2])
+        out = cutensornet.contract(
+            *inputs, options=cutn_options, optimize=cutn_optimizer)
+    else:
+        assert False
+
+    return out
diff --git a/cupy/linalg/_einsum_opt.py b/cupy/linalg/_einsum_opt.py
new file mode 100644
index 0000000..09e0900
--- /dev/null
+++ b/cupy/linalg/_einsum_opt.py
@@ -0,0 +1,412 @@
+import itertools
+
+
+def _flop_count(idx_contraction, inner, num_terms, size_dictionary):
+    """Copied from _flop_count in numpy/core/einsumfunc.py
+
+    Computes the number of FLOPS in the contraction.
+
+    Parameters
+    ----------
+    idx_contraction : iterable
+        The indices involved in the contraction
+    inner : bool
+        Does this contraction require an inner product?
+    num_terms : int
+        The number of terms in a contraction
+    size_dictionary : dict
+        The size of each of the indices in idx_contraction
+
+    Returns
+    -------
+    flop_count : int
+        The total number of FLOPS required for the contraction.
+
+    Examples
+    --------
+
+    >>> _flop_count('abc', False, 1, {'a': 2, 'b':3, 'c':5})
+    90
+
+    >>> _flop_count('abc', True, 2, {'a': 2, 'b':3, 'c':5})
+    270
+
+    """
+
+    overall_size = _compute_size_by_dict(idx_contraction, size_dictionary)
+    op_factor = max(1, num_terms - 1)
+    if inner:
+        op_factor += 1
+
+    return overall_size * op_factor
+
+
+def _compute_size_by_dict(indices, idx_dict):
+    """Copied from _compute_size_by_dict in numpy/core/einsumfunc.py
+
+    Computes the product of the elements in indices based on the dictionary
+    idx_dict.
+
+    Parameters
+    ----------
+    indices : iterable
+        Indices to base the product on.
+    idx_dict : dictionary
+        Dictionary of index sizes
+
+    Returns
+    -------
+    ret : int
+        The resulting product.
+
+    Examples
+    --------
+    >>> _compute_size_by_dict('abbc', {'a': 2, 'b':3, 'c':5})
+    90
+
+    """
+    ret = 1
+    for i in indices:
+        ret *= idx_dict[i]
+    return ret
+
+
+def _find_contraction(positions, input_sets, output_set):
+    """Copied from _find_contraction in numpy/core/einsumfunc.py
+
+    Finds the contraction for a given set of input and output sets.
+
+    Parameters
+    ----------
+    positions : iterable
+        Integer positions of terms used in the contraction.
+    input_sets : list
+        List of sets that represent the lhs side of the einsum subscript
+    output_set : set
+        Set that represents the rhs side of the overall einsum subscript
+
+    Returns
+    -------
+    new_result : set
+        The indices of the resulting contraction
+    remaining : list
+        List of sets that have not been contracted, the new set is appended to
+        the end of this list
+    idx_removed : set
+        Indices removed from the entire contraction
+    idx_contraction : set
+        The indices used in the current contraction
+
+    Examples
+    --------
+
+    # A simple dot product test case
+    >>> pos = (0, 1)
+    >>> isets = [set('ab'), set('bc')]
+    >>> oset = set('ac')
+    >>> _find_contraction(pos, isets, oset)
+    ({'a', 'c'}, [{'a', 'c'}], {'b'}, {'a', 'b', 'c'})
+
+    # A more complex case with additional terms in the contraction
+    >>> pos = (0, 2)
+    >>> isets = [set('abd'), set('ac'), set('bdc')]
+    >>> oset = set('ac')
+    >>> _find_contraction(pos, isets, oset)
+    ({'a', 'c'}, [{'a', 'c'}, {'a', 'c'}], {'b', 'd'}, {'a', 'b', 'c', 'd'})
+    """
+
+    idx_contract = set()
+    idx_remain = output_set.copy()
+    remaining = []
+    for ind, value in enumerate(input_sets):
+        if ind in positions:
+            idx_contract |= value
+        else:
+            remaining.append(value)
+            idx_remain |= value
+
+    new_result = idx_remain & idx_contract
+    idx_removed = (idx_contract - new_result)
+    remaining.append(new_result)
+
+    return (new_result, remaining, idx_removed, idx_contract)
+
+
+def _optimal_path(input_sets, output_set, idx_dict, memory_limit):
+    """Copied from _optimal_path in numpy/core/einsumfunc.py
+
+    Computes all possible pair contractions, sieves the results based
+    on ``memory_limit`` and returns the lowest cost path. This algorithm
+    scales factorial with respect to the elements in the list ``input_sets``.
+
+    Parameters
+    ----------
+    input_sets : list
+        List of sets that represent the lhs side of the einsum subscript
+    output_set : set
+        Set that represents the rhs side of the overall einsum subscript
+    idx_dict : dictionary
+        Dictionary of index sizes
+    memory_limit : int
+        The maximum number of elements in a temporary array
+
+    Returns
+    -------
+    path : list
+        The optimal contraction order within the memory limit constraint.
+
+    Examples
+    --------
+    >>> isets = [set('abd'), set('ac'), set('bdc')]
+    >>> oset = set('')
+    >>> idx_sizes = {'a': 1, 'b':2, 'c':3, 'd':4}
+    >>> _optimal_path(isets, oset, idx_sizes, 5000)
+    [(0, 2), (0, 1)]
+    """
+
+    full_results = [(0, [], input_sets)]
+    for iteration in range(len(input_sets) - 1):
+        iter_results = []
+
+        # Compute all unique pairs
+        for curr in full_results:
+            cost, positions, remaining = curr
+            for con in itertools.combinations(range(len(input_sets) - iteration), 2):  # NOQA
+
+                # Find the contraction
+                cont = _find_contraction(con, remaining, output_set)
+                new_result, new_input_sets, idx_removed, idx_contract = cont
+
+                # Sieve the results based on memory_limit
+                new_size = _compute_size_by_dict(new_result, idx_dict)
+                if new_size > memory_limit:
+                    continue
+
+                # Build (total_cost, positions, indices_remaining)
+                total_cost = cost + \
+                    _flop_count(idx_contract, idx_removed, len(con), idx_dict)
+                new_pos = positions + [con]
+                iter_results.append((total_cost, new_pos, new_input_sets))
+
+        # Update combinatorial list, if we did not find anything return best
+        # path + remaining contractions
+        if iter_results:
+            full_results = iter_results
+        else:
+            path = min(full_results, key=lambda x: x[0])[1]
+            path += [tuple(range(len(input_sets) - iteration))]
+            return path
+
+    # If we have not found anything return single einsum contraction
+    if len(full_results) == 0:
+        return [tuple(range(len(input_sets)))]
+
+    path = min(full_results, key=lambda x: x[0])[1]
+    return path
+
+
+def _parse_possible_contraction(positions, input_sets, output_set, idx_dict, memory_limit, path_cost, naive_cost):  # NOQA
+    """Copied from _parse_possible_contraction in numpy/core/einsumfunc.py
+
+    Compute the cost (removed size + flops) and resultant indices for
+    performing the contraction specified by ``positions``.
+
+    Parameters
+    ----------
+    positions : tuple of int
+        The locations of the proposed tensors to contract.
+    input_sets : list of sets
+        The indices found on each tensors.
+    output_set : set
+        The output indices of the expression.
+    idx_dict : dict
+        Mapping of each index to its size.
+    memory_limit : int
+        The total allowed size for an intermediary tensor.
+    path_cost : int
+        The contraction cost so far.
+    naive_cost : int
+        The cost of the unoptimized expression.
+
+    Returns
+    -------
+    cost : (int, int)
+        A tuple containing the size of any indices removed, and the flop cost.
+    positions : tuple of int
+        The locations of the proposed tensors to contract.
+    new_input_sets : list of sets
+        The resulting new list of indices if this proposed contraction is performed.
+
+    """  # NOQA
+
+    # Find the contraction
+    contract = _find_contraction(positions, input_sets, output_set)
+    idx_result, new_input_sets, idx_removed, idx_contract = contract
+
+    # Sieve the results based on memory_limit
+    new_size = _compute_size_by_dict(idx_result, idx_dict)
+    if new_size > memory_limit:
+        return None
+
+    # Build sort tuple
+    old_sizes = (_compute_size_by_dict(
+        input_sets[p], idx_dict) for p in positions)
+    removed_size = sum(old_sizes) - new_size
+
+    # NB: removed_size used to be just the size of any removed indices i.e.:
+    #     helpers.compute_size_by_dict(idx_removed, idx_dict)
+    cost = _flop_count(idx_contract, idx_removed, len(positions), idx_dict)
+    sort = (-removed_size, cost)
+
+    # Sieve based on total cost as well
+    if (path_cost + cost) > naive_cost:
+        return None
+
+    # Add contraction to possible choices
+    return [sort, positions, new_input_sets]
+
+
+def _update_other_results(results, best):
+    """Copied from _update_other_results in numpy/core/einsumfunc.py
+
+    Update the positions and provisional input_sets of ``results`` based on
+    performing the contraction result ``best``. Remove any involving the tensors
+    contracted.
+
+    Parameters
+    ----------
+    results : list
+        List of contraction results produced by ``_parse_possible_contraction``.
+    best : list
+        The best contraction of ``results`` i.e. the one that will be performed.
+
+    Returns
+    -------
+    mod_results : list
+        The list of modifed results, updated with outcome of ``best`` contraction.  # NOQA
+    """
+
+    best_con = best[1]
+    bx, by = best_con
+    mod_results = []
+
+    for cost, (x, y), con_sets in results:
+
+        # Ignore results involving tensors just contracted
+        if x in best_con or y in best_con:
+            continue
+
+        # Update the input_sets
+        del con_sets[by - int(by > x) - int(by > y)]
+        del con_sets[bx - int(bx > x) - int(bx > y)]
+        con_sets.insert(-1, best[2][-1])
+
+        # Update the position indices
+        mod_con = x - int(x > bx) - int(x > by), y - int(y > bx) - int(y > by)
+        mod_results.append((cost, mod_con, con_sets))
+
+    return mod_results
+
+
+def _greedy_path(input_sets, output_set, idx_dict, memory_limit):
+    """Copied from _greedy_path in numpy/core/einsumfunc.py
+
+    Finds the path by contracting the best pair until the input list is
+    exhausted. The best pair is found by minimizing the tuple
+    ``(-prod(indices_removed), cost)``.  What this amounts to is prioritizing
+    matrix multiplication or inner product operations, then Hadamard like
+    operations, and finally outer operations. Outer products are limited by
+    ``memory_limit``. This algorithm scales cubically with respect to the
+    number of elements in the list ``input_sets``.
+
+    Parameters
+    ----------
+    input_sets : list
+        List of sets that represent the lhs side of the einsum subscript
+    output_set : set
+        Set that represents the rhs side of the overall einsum subscript
+    idx_dict : dictionary
+        Dictionary of index sizes
+    memory_limit_limit : int
+        The maximum number of elements in a temporary array
+
+    Returns
+    -------
+    path : list
+        The greedy contraction order within the memory limit constraint.
+
+    Examples
+    --------
+    >>> isets = [set('abd'), set('ac'), set('bdc')]
+    >>> oset = set('')
+    >>> idx_sizes = {'a': 1, 'b':2, 'c':3, 'd':4}
+    >>> _greedy_path(isets, oset, idx_sizes, 5000)
+    [(0, 2), (0, 1)]
+    """
+
+    # Handle trivial cases that leaked through
+    if len(input_sets) == 1:
+        return [(0,)]
+    elif len(input_sets) == 2:
+        return [(0, 1)]
+
+    # Build up a naive cost
+    contract = _find_contraction(
+        range(len(input_sets)), input_sets, output_set)
+    idx_result, new_input_sets, idx_removed, idx_contract = contract
+    naive_cost = _flop_count(idx_contract, idx_removed,
+                             len(input_sets), idx_dict)
+
+    # Initially iterate over all pairs
+    comb_iter = itertools.combinations(range(len(input_sets)), 2)
+    known_contractions = []
+
+    path_cost = 0
+    path = []
+
+    for iteration in range(len(input_sets) - 1):
+
+        # Iterate over all pairs on first step, only previously found pairs on subsequent steps  # NOQA
+        for positions in comb_iter:
+
+            # Always initially ignore outer products
+            if input_sets[positions[0]].isdisjoint(input_sets[positions[1]]):
+                continue
+
+            result = _parse_possible_contraction(positions, input_sets, output_set, idx_dict, memory_limit, path_cost,  # NOQA
+                                                 naive_cost)
+            if result is not None:
+                known_contractions.append(result)
+
+        # If we do not have a inner contraction, rescan pairs including outer products  # NOQA
+        if len(known_contractions) == 0:
+
+            # Then check the outer products
+            for positions in itertools.combinations(range(len(input_sets)), 2):
+                result = _parse_possible_contraction(positions, input_sets, output_set, idx_dict, memory_limit,  # NOQA
+                                                     path_cost, naive_cost)
+                if result is not None:
+                    known_contractions.append(result)
+
+            # If we still did not find any remaining contractions, default back to einsum like behavior  # NOQA
+            if len(known_contractions) == 0:
+                path.append(tuple(range(len(input_sets))))
+                break
+
+        # Sort based on first index
+        best = min(known_contractions, key=lambda x: x[0])
+
+        # Now propagate as many unused contractions as possible to next iteration  # NOQA
+        known_contractions = _update_other_results(known_contractions, best)
+
+        # Next iteration only compute contractions with the new tensor
+        # All other contractions have been accounted for
+        input_sets = best[2]
+        new_tensor_pos = len(input_sets) - 1
+        comb_iter = ((i, new_tensor_pos) for i in range(new_tensor_pos))
+
+        # Update path and total cost
+        path.append(best[1])
+        path_cost += best[0][1]
+
+    return path
diff --git a/cupy/linalg/_norms.py b/cupy/linalg/_norms.py
new file mode 100644
index 0000000..0628b52
--- /dev/null
+++ b/cupy/linalg/_norms.py
@@ -0,0 +1,304 @@
+import numpy
+
+import cupy
+from cupy import _core
+from cupy.linalg import _decomposition
+from cupy.linalg import _util
+
+import functools
+
+
+def _multi_svd_norm(x, row_axis, col_axis, op):
+    y = cupy.moveaxis(x, (row_axis, col_axis), (-2, -1))
+    result = op(_decomposition.svd(y, compute_uv=False), axis=-1)
+    return result
+
+
+_norm_ord2 = _core.create_reduction_func(
+    '_norm_ord2',
+    ('?->l', 'b->l', 'B->L', 'h->l', 'H->L', 'i->l', 'I->L', 'l->l', 'L->L',
+     'q->q', 'Q->Q',
+     ('e->e', (None, None, None, 'float')),
+     'f->f', 'd->d'),
+    ('in0 * in0', 'a + b', 'out0 = sqrt(type_out0_raw(a))', None), 0)
+_norm_ord2_complex = _core.create_reduction_func(
+    '_norm_ord2_complex',
+    ('F->f', 'D->d'),
+    ('in0.real() * in0.real() + in0.imag() * in0.imag()',
+     'a + b', 'out0 = sqrt(type_out0_raw(a))', None), 0)
+
+
+def norm(x, ord=None, axis=None, keepdims=False):
+    """Returns one of matrix norms specified by ``ord`` parameter.
+
+    See numpy.linalg.norm for more detail.
+
+    Args:
+        x (cupy.ndarray): Array to take norm. If ``axis`` is None,
+            ``x`` must be 1-D or 2-D.
+        ord (non-zero int, inf, -inf, 'fro'): Norm type.
+        axis (int, 2-tuple of ints, None): 1-D or 2-D norm is cumputed over
+            ``axis``.
+        keepdims (bool): If this is set ``True``, the axes which are normed
+            over are left.
+
+    Returns:
+        cupy.ndarray
+
+    """
+    if not issubclass(x.dtype.type, numpy.inexact):
+        x = x.astype(float)
+
+    # Immediately handle some default, simple, fast, and common cases.
+    if axis is None:
+        ndim = x.ndim
+        if (ord is None or (ndim == 1 and ord == 2) or
+                (ndim == 2 and ord in ('f', 'fro'))):
+            if x.dtype.kind == 'c':
+                s = abs(x.ravel())
+                s *= s
+                ret = cupy.sqrt(s.sum())
+            else:
+                ret = cupy.sqrt((x * x).sum())
+            if keepdims:
+                ret = ret.reshape((1,) * ndim)
+            return ret
+
+    # Normalize the `axis` argument to a tuple.
+    nd = x.ndim
+    if axis is None:
+        axis = tuple(range(nd))
+    elif not isinstance(axis, tuple):
+        try:
+            axis = int(axis)
+        except Exception:
+            raise TypeError(
+                '\'axis\' must be None, an integer or a tuple of integers')
+        axis = (axis,)
+
+    if len(axis) == 1:
+        if ord == numpy.inf:
+            return abs(x).max(axis=axis, keepdims=keepdims)
+        elif ord == -numpy.inf:
+            return abs(x).min(axis=axis, keepdims=keepdims)
+        elif ord == 0:
+            # Zero norm
+            # Convert to Python float in accordance with NumPy
+            return (x != 0).astype(x.real.dtype).sum(
+                axis=axis, keepdims=keepdims)
+        elif ord == 1:
+            # special case for speedup
+            return abs(x).sum(axis=axis, keepdims=keepdims)
+        elif ord is None or ord == 2:
+            # special case for speedup
+            if x.dtype.kind == 'c':
+                return _norm_ord2_complex(x, axis=axis, keepdims=keepdims)
+            return _norm_ord2(x, axis=axis, keepdims=keepdims)
+        else:
+            try:
+                float(ord)
+            except TypeError:
+                raise ValueError('Invalid norm order for vectors.')
+
+            absx = abs(x)
+            absx **= ord
+            ret = absx.sum(axis=axis, keepdims=keepdims)
+            ret **= cupy.reciprocal(ord, dtype=ret.dtype)
+            return ret
+    elif len(axis) == 2:
+        row_axis, col_axis = axis
+        if row_axis < 0:
+            row_axis += nd
+        if col_axis < 0:
+            col_axis += nd
+        if not (0 <= row_axis < nd and 0 <= col_axis < nd):
+            raise ValueError('Invalid axis %r for an array with shape %r' %
+                             (axis, x.shape))
+        if row_axis == col_axis:
+            raise ValueError('Duplicate axes given.')
+        if ord == 2:
+            op_max = functools.partial(cupy.take, indices=0)
+            ret = _multi_svd_norm(x, row_axis, col_axis, op_max)
+        elif ord == -2:
+            op_min = functools.partial(cupy.take, indices=-1)
+            ret = _multi_svd_norm(x, row_axis, col_axis, op_min)
+        elif ord == 1:
+            if col_axis > row_axis:
+                col_axis -= 1
+            ret = abs(x).sum(axis=row_axis).max(axis=col_axis)
+        elif ord == numpy.inf:
+            if row_axis > col_axis:
+                row_axis -= 1
+            ret = abs(x).sum(axis=col_axis).max(axis=row_axis)
+        elif ord == -1:
+            if col_axis > row_axis:
+                col_axis -= 1
+            ret = abs(x).sum(axis=row_axis).min(axis=col_axis)
+        elif ord == -numpy.inf:
+            if row_axis > col_axis:
+                row_axis -= 1
+            ret = abs(x).sum(axis=col_axis).min(axis=row_axis)
+        elif ord in [None, 'fro', 'f']:
+            if x.dtype.kind == 'c':
+                ret = _norm_ord2_complex(x, axis=axis)
+            else:
+                ret = _norm_ord2(x, axis=axis)
+        elif ord == 'nuc':
+            ret = _multi_svd_norm(x, row_axis, col_axis, cupy.sum)
+        else:
+            raise ValueError('Invalid norm order for matrices.')
+        if keepdims:
+            ret_shape = list(x.shape)
+            ret_shape[axis[0]] = 1
+            ret_shape[axis[1]] = 1
+            ret = ret.reshape(ret_shape)
+        return ret
+    else:
+        raise ValueError('Improper number of dimensions to norm.')
+
+
+# TODO(okuta): Implement cond
+
+
+def det(a):
+    """Returns the determinant of an array.
+
+    Args:
+        a (cupy.ndarray): The input matrix with dimension ``(..., N, N)``.
+
+    Returns:
+        cupy.ndarray: Determinant of ``a``. Its shape is ``a.shape[:-2]``.
+
+    .. seealso:: :func:`numpy.linalg.det`
+    """
+    sign, logdet = slogdet(a)
+    return sign * cupy.exp(logdet)
+
+
+def matrix_rank(M, tol=None):
+    """Return matrix rank of array using SVD method
+
+    Args:
+        M (cupy.ndarray): Input array. Its `ndim` must be less than or equal to
+            2.
+        tol (None or float): Threshold of singular value of `M`.
+            When `tol` is `None`, and `eps` is the epsilon value for datatype
+            of `M`, then `tol` is set to `S.max() * max(M.shape) * eps`,
+            where `S` is the singular value of `M`.
+            It obeys :func:`numpy.linalg.matrix_rank`.
+
+    Returns:
+        cupy.ndarray: Rank of `M`.
+
+    .. seealso:: :func:`numpy.linalg.matrix_rank`
+    """
+    if M.ndim < 2:
+        return (M != 0).any().astype(int)
+    S = _decomposition.svd(M, compute_uv=False)
+    if tol is None:
+        tol = (S.max(axis=-1, keepdims=True) * max(M.shape[-2:]) *
+               numpy.finfo(S.dtype).eps)
+    return (S > tol).sum(axis=-1, dtype=numpy.intp)
+
+
+def slogdet(a):
+    """Returns sign and logarithm of the determinant of an array.
+
+    It calculates the natural logarithm of the determinant of a given value.
+
+    Args:
+        a (cupy.ndarray): The input matrix with dimension ``(..., N, N)``.
+
+    Returns:
+        tuple of :class:`~cupy.ndarray`:
+            It returns a tuple ``(sign, logdet)``. ``sign`` represents each
+            sign of the determinant as a real number ``0``, ``1`` or ``-1``.
+            'logdet' represents the natural logarithm of the absolute of the
+            determinant.
+            If the determinant is zero, ``sign`` will be ``0`` and ``logdet``
+            will be ``-inf``.
+            The shapes of both ``sign`` and ``logdet`` are equal to
+            ``a.shape[:-2]``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. warning::
+        To produce the same results as :func:`numpy.linalg.slogdet` for
+        singular inputs, set the `linalg` configuration to `raise`.
+
+    .. seealso:: :func:`numpy.linalg.slogdet`
+    """
+    _util._assert_stacked_2d(a)
+    _util._assert_stacked_square(a)
+
+    dtype, sign_dtype = _util.linalg_common_type(a)
+    logdet_dtype = numpy.dtype(sign_dtype.char.lower())
+
+    a_shape = a.shape
+    shape = a_shape[:-2]
+    n = a_shape[-2]
+
+    if a.size == 0:
+        # empty batch (result is empty, too) or empty matrices det([[]]) == 1
+        sign = cupy.ones(shape, sign_dtype)
+        logdet = cupy.zeros(shape, logdet_dtype)
+        return sign, logdet
+
+    lu, ipiv, dev_info = _decomposition._lu_factor(a, dtype)
+
+    # dev_info < 0 means illegal value (in dimensions, strides, and etc.) that
+    # should never happen even if the matrix contains nan or inf.
+    # TODO(kataoka): assert dev_info >= 0 if synchronization is allowed for
+    # debugging purposes.
+
+    diag = cupy.diagonal(lu, axis1=-2, axis2=-1)
+
+    logdet = cupy.log(cupy.abs(diag)).sum(axis=-1)
+
+    # ipiv is 1-origin
+    non_zero = cupy.count_nonzero(ipiv != cupy.arange(1, n + 1), axis=-1)
+    if dtype.kind == "f":
+        non_zero += cupy.count_nonzero(diag < 0, axis=-1)
+
+    # Note: sign == -1 ** (non_zero % 2)
+    sign = (non_zero % 2) * -2 + 1
+    if dtype.kind == "c":
+        sign = sign * cupy.prod(diag / cupy.abs(diag), axis=-1)
+
+    sign = sign.astype(dtype)
+    logdet = logdet.astype(logdet_dtype, copy=False)
+    singular = dev_info > 0
+    return (
+        cupy.where(singular, sign_dtype.type(0), sign).reshape(shape),
+        cupy.where(singular, logdet_dtype.type('-inf'), logdet).reshape(shape),
+    )
+
+
+def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None):
+    """Returns the sum along the diagonals of an array.
+
+    It computes the sum along the diagonals at ``axis1`` and ``axis2``.
+
+    Args:
+        a (cupy.ndarray): Array to take trace.
+        offset (int): Index of diagonals. Zero indicates the main diagonal, a
+            positive value an upper diagonal, and a negative value a lower
+            diagonal.
+        axis1 (int): The first axis along which the trace is taken.
+        axis2 (int): The second axis along which the trace is taken.
+        dtype: Data type specifier of the output.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: The trace of ``a`` along axes ``(axis1, axis2)``.
+
+    .. seealso:: :func:`numpy.trace`
+
+    """
+    # TODO(okuta): check type
+    return a.trace(offset, axis1, axis2, dtype, out)
diff --git a/cupy/linalg/_product.py b/cupy/linalg/_product.py
new file mode 100644
index 0000000..934f645
--- /dev/null
+++ b/cupy/linalg/_product.py
@@ -0,0 +1,427 @@
+import collections.abc
+
+import numpy
+
+import cupy
+from cupy import _core
+from cupy._core import internal
+from cupy._core._gufuncs import _GUFunc
+from cupy.linalg import _solve
+from cupy.linalg import _util
+
+
+matmul = _GUFunc(
+    _core.matmul,
+    '(n?,k),(k,m?)->(n?,m?)',
+    supports_batched=True,
+    supports_out=True,
+    doc="""matmul(x1, x2, /, out=None, \\*\\*kwargs)
+
+    Matrix product of two arrays.
+
+    Returns the matrix product of two arrays and is the implementation of
+    the `@` operator introduced in Python 3.5 following PEP465.
+
+    The main difference against cupy.dot are the handling of arrays with more
+    than 2 dimensions. For more information see :func:`numpy.matmul`.
+
+    Args:
+        x1 (cupy.ndarray): The left argument.
+        x2 (cupy.ndarray): The right argument.
+        out (cupy.ndarray, optional): Output array.
+        \\*\\*kwargs: ufunc keyword arguments.
+
+    Returns:
+        cupy.ndarray: Output array.
+
+    .. seealso:: :func:`numpy.matmul`
+    """
+)
+
+
+def dot(a, b, out=None):
+    """Returns a dot product of two arrays.
+
+    For arrays with more than one axis, it computes the dot product along the
+    last axis of ``a`` and the second-to-last axis of ``b``. This is just a
+    matrix product if the both arrays are 2-D. For 1-D arrays, it uses their
+    unique axis as an axis to take dot product over.
+
+    Args:
+        a (cupy.ndarray): The left argument.
+        b (cupy.ndarray): The right argument.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: The dot product of ``a`` and ``b``.
+
+    .. seealso:: :func:`numpy.dot`
+
+    """
+    # TODO(okuta): check type
+    return a.dot(b, out)
+
+
+def vdot(a, b):
+    """Returns the dot product of two vectors.
+
+    The input arrays are flattened into 1-D vectors and then it performs inner
+    product of these vectors.
+
+    Args:
+        a (cupy.ndarray): The first argument.
+        b (cupy.ndarray): The second argument.
+
+    Returns:
+        cupy.ndarray: Zero-dimensional array of the dot product result.
+
+    .. seealso:: :func:`numpy.vdot`
+
+    """
+    if a.size != b.size:
+        raise ValueError('Axis dimension mismatch')
+    if a.dtype.kind == 'c':
+        a = a.conj()
+
+    return _core.tensordot_core(a, b, None, 1, 1, a.size, ())
+
+
+def cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None):
+    """Returns the cross product of two vectors.
+
+    The cross product of ``a`` and ``b`` in :math:`R^3` is a vector
+    perpendicular to both ``a`` and ``b``.  If ``a`` and ``b`` are arrays
+    of vectors, the vectors are defined by the last axis of ``a`` and ``b``
+    by default, and these axes can have dimensions 2 or 3.  Where the
+    dimension of either ``a`` or ``b`` is 2, the third component of the input
+    vector is assumed to be zero and the cross product calculated accordingly.
+    In cases where both input vectors have dimension 2, the z-component of
+    the cross product is returned.
+
+    Args:
+        a (cupy.ndarray): Components of the first vector(s).
+        b (cupy.ndarray): Components of the second vector(s).
+        axisa (int, optional):
+            Axis of ``a`` that defines the vector(s).
+            By default, the last axis.
+        axisb (int, optional):
+            Axis of ``b`` that defines the vector(s).
+            By default, the last axis.
+        axisc (int, optional):
+            Axis of ``c`` containing the cross product vector(s).  Ignored if
+            both input vectors have dimension 2, as the return is scalar.
+            By default, the last axis.
+        axis (int, optional):
+            If defined, the axis of ``a``, ``b`` and ``c``
+            that defines the vector(s) and cross product(s).
+            Overrides ``axisa``, ``axisb`` and ``axisc``.
+
+    Returns:
+        cupy.ndarray :
+            Vector cross product(s).
+
+    .. seealso:: :func:`numpy.cross`
+
+    """
+
+    if axis is not None:
+        axisa, axisb, axisc = (axis,) * 3
+    a = cupy.asarray(a)
+    b = cupy.asarray(b)
+    # Check axisa and axisb are within bounds
+    axisa = internal._normalize_axis_index(axisa, a.ndim)
+    axisb = internal._normalize_axis_index(axisb, b.ndim)
+
+    # Move working axis to the end of the shape
+    a = cupy.moveaxis(a, axisa, -1)
+    b = cupy.moveaxis(b, axisb, -1)
+    if a.shape[-1] not in (2, 3) or b.shape[-1] not in (2, 3):
+        msg = ('incompatible dimensions for cross product\n'
+               '(dimension must be 2 or 3)')
+        raise ValueError(msg)
+
+    # Create the output array
+    shape = cupy.broadcast(a[..., 0], b[..., 0]).shape
+    if a.shape[-1] == 3 or b.shape[-1] == 3:
+        shape += (3,)
+        # Check axisc is within bounds
+        axisc = internal._normalize_axis_index(axisc, len(shape))
+    dtype = cupy.promote_types(a.dtype, b.dtype)
+    cp = cupy.empty(shape, dtype)
+
+    # create local aliases for readability
+    a0 = a[..., 0]
+    a1 = a[..., 1]
+    if a.shape[-1] == 3:
+        a2 = a[..., 2]
+    b0 = b[..., 0]
+    b1 = b[..., 1]
+    if b.shape[-1] == 3:
+        b2 = b[..., 2]
+    if cp.ndim != 0 and cp.shape[-1] == 3:
+        cp0 = cp[..., 0]
+        cp1 = cp[..., 1]
+        cp2 = cp[..., 2]
+
+    if a.shape[-1] == 2:
+        if b.shape[-1] == 2:
+            # a0 * b1 - a1 * b0
+            cupy.multiply(a0, b1, out=cp)
+            cp -= a1 * b0
+            return cp
+        else:
+            assert b.shape[-1] == 3
+            # cp0 = a1 * b2 - 0  (a2 = 0)
+            # cp1 = 0 - a0 * b2  (a2 = 0)
+            # cp2 = a0 * b1 - a1 * b0
+            cupy.multiply(a1, b2, out=cp0)
+            cupy.multiply(a0, b2, out=cp1)
+            cupy.negative(cp1, out=cp1)
+            cupy.multiply(a0, b1, out=cp2)
+            cp2 -= a1 * b0
+    else:
+        assert a.shape[-1] == 3
+        if b.shape[-1] == 3:
+            # cp0 = a1 * b2 - a2 * b1
+            # cp1 = a2 * b0 - a0 * b2
+            # cp2 = a0 * b1 - a1 * b0
+            cupy.multiply(a1, b2, out=cp0)
+            tmp = a2 * b1
+            cp0 -= tmp
+            cupy.multiply(a2, b0, out=cp1)
+            cupy.multiply(a0, b2, out=tmp)
+            cp1 -= tmp
+            cupy.multiply(a0, b1, out=cp2)
+            cupy.multiply(a1, b0, out=tmp)
+            cp2 -= tmp
+        else:
+            assert b.shape[-1] == 2
+            # cp0 = 0 - a2 * b1  (b2 = 0)
+            # cp1 = a2 * b0 - 0  (b2 = 0)
+            # cp2 = a0 * b1 - a1 * b0
+            cupy.multiply(a2, b1, out=cp0)
+            cupy.negative(cp0, out=cp0)
+            cupy.multiply(a2, b0, out=cp1)
+            cupy.multiply(a0, b1, out=cp2)
+            cp2 -= a1 * b0
+
+    return cupy.moveaxis(cp, -1, axisc)
+
+
+def inner(a, b):
+    """Returns the inner product of two arrays.
+
+    It uses the last axis of each argument to take sum product.
+
+    Args:
+        a (cupy.ndarray): The first argument.
+        b (cupy.ndarray): The second argument.
+
+    Returns:
+        cupy.ndarray: The inner product of ``a`` and ``b``.
+
+    .. seealso:: :func:`numpy.inner`
+
+    """
+    a_ndim = a.ndim
+    b_ndim = b.ndim
+    if a_ndim == 0 or b_ndim == 0:
+        return cupy.multiply(a, b)
+
+    a_axis = a_ndim - 1
+    b_axis = b_ndim - 1
+
+    if a.shape[-1] != b.shape[-1]:
+        raise ValueError('Axis dimension mismatch')
+
+    if a_axis:
+        a = cupy.rollaxis(a, a_axis, 0)
+    if b_axis:
+        b = cupy.rollaxis(b, b_axis, 0)
+
+    ret_shape = a.shape[1:] + b.shape[1:]
+
+    k = a.shape[0]
+    n = a.size // k
+    m = b.size // k
+
+    return _core.tensordot_core(a, b, None, n, m, k, ret_shape)
+
+
+def outer(a, b, out=None):
+    """Returns the outer product of two vectors.
+
+    The input arrays are flattened into 1-D vectors and then it performs outer
+    product of these vectors.
+
+    Args:
+        a (cupy.ndarray): The first argument.
+        b (cupy.ndarray): The second argument.
+        out (cupy.ndarray): Output array.
+
+    Returns:
+        cupy.ndarray: 2-D array of the outer product of ``a`` and ``b``.
+
+    .. seealso:: :func:`numpy.outer`
+
+    """
+    return cupy.multiply(a.ravel()[:, None], b.ravel()[None, :], out=out)
+
+
+def tensordot(a, b, axes=2):
+    """Returns the tensor dot product of two arrays along specified axes.
+
+    This is equivalent to compute dot product along the specified axes which
+    are treated as one axis by reshaping.
+
+    Args:
+        a (cupy.ndarray): The first argument.
+        b (cupy.ndarray): The second argument.
+        axes:
+            - If it is an integer, then ``axes`` axes at the last of ``a`` and
+              the first of ``b`` are used.
+            - If it is a pair of sequences of integers, then these two
+              sequences specify the list of axes for ``a`` and ``b``. The
+              corresponding axes are paired for sum-product.
+
+    Returns:
+        cupy.ndarray: The tensor dot product of ``a`` and ``b`` along the
+        axes specified by ``axes``.
+
+    .. seealso:: :func:`numpy.tensordot`
+
+    """
+    a_ndim = a.ndim
+    b_ndim = b.ndim
+    if a_ndim == 0 or b_ndim == 0:
+        if axes != 0 and axes != ((), ()):
+            raise ValueError('An input is zero-dim while axes has dimensions')
+        return cupy.multiply(a, b)
+
+    if isinstance(axes, collections.abc.Sequence):
+        if len(axes) != 2:
+            raise ValueError('Axes must consist of two arrays.')
+        a_axes, b_axes = axes
+        if numpy.isscalar(a_axes):
+            a_axes = a_axes,
+        if numpy.isscalar(b_axes):
+            b_axes = b_axes,
+    else:
+        a_axes = tuple(range(a_ndim - axes, a_ndim))
+        b_axes = tuple(range(axes))
+
+    sum_ndim = len(a_axes)
+    if sum_ndim != len(b_axes):
+        raise ValueError('Axes length mismatch')
+
+    for a_axis, b_axis in zip(a_axes, b_axes):
+        if a.shape[a_axis] != b.shape[b_axis]:
+            raise ValueError('Axis dimension mismatch')
+
+    # Make the axes non-negative
+    a = _move_axes_to_head(a, [axis % a_ndim for axis in a_axes])
+    b = _move_axes_to_head(b, [axis % b_ndim for axis in b_axes])
+
+    ret_shape = a.shape[sum_ndim:] + b.shape[sum_ndim:]
+
+    k = internal.prod(a.shape[:sum_ndim])
+    # Avoid division by zero: _core.tensordot_core returns zeros without
+    # checking n, m consistency, thus allowing 0-length dimensions to work
+    n = a.size // k if k != 0 else 0
+    m = b.size // k if k != 0 else 0
+
+    return _core.tensordot_core(a, b, None, n, m, k, ret_shape)
+
+
+# TODO: rename `M` to `a`
+def matrix_power(M, n):
+    """Raise a square matrix to the (integer) power `n`.
+
+    Args:
+        M (~cupy.ndarray): Matrix to raise by power n.
+        n (~int): Power to raise matrix to.
+
+    Returns:
+        ~cupy.ndarray: Output array.
+
+    ..seealso:: :func:`numpy.linalg.matrix_power`
+    """
+    _util._assert_cupy_array(M)
+    _util._assert_stacked_2d(M)
+    _util._assert_stacked_square(M)
+    if not isinstance(n, int):
+        raise TypeError('exponent must be an integer')
+
+    if n == 0:
+        return _util.stacked_identity_like(M)
+    elif n < 0:
+        M = _solve.inv(M)
+        n *= -1
+
+    # short-cuts
+    if n <= 3:
+        if n == 1:
+            return M
+        elif n == 2:
+            return cupy.matmul(M, M)
+        else:
+            return cupy.matmul(cupy.matmul(M, M), M)
+
+    # binary decomposition to reduce the number of Matrix
+    # multiplications for n > 3.
+    result, Z = None, None
+    for b in cupy.binary_repr(n)[::-1]:
+        Z = M if Z is None else cupy.matmul(Z, Z)
+        if b == '1':
+            result = Z if result is None else cupy.matmul(result, Z)
+
+    return result
+
+
+def kron(a, b):
+    """Returns the kronecker product of two arrays.
+
+    Args:
+        a (~cupy.ndarray): The first argument.
+        b (~cupy.ndarray): The second argument.
+
+    Returns:
+        ~cupy.ndarray: Output array.
+
+    .. seealso:: :func:`numpy.kron`
+
+    """
+    a_ndim = a.ndim
+    b_ndim = b.ndim
+    if a_ndim == 0 or b_ndim == 0:
+        return cupy.multiply(a, b)
+
+    ndim = b_ndim
+    a_shape = a.shape
+    b_shape = b.shape
+    if a_ndim != b_ndim:
+        if b_ndim > a_ndim:
+            a_shape = (1,) * (b_ndim - a_ndim) + a_shape
+        else:
+            b_shape = (1,) * (a_ndim - b_ndim) + b_shape
+            ndim = a_ndim
+
+    axis = ndim - 1
+    out = _core.tensordot_core(
+        a, b, None, a.size, b.size, 1, a_shape + b_shape)
+    for _ in range(ndim):
+        out = _core.concatenate_method(out, axis=axis)
+
+    return out
+
+
+def _move_axes_to_head(a, axes):
+    # This function moves the axes of ``s`` to the head of the shape.
+    for idx, axis in enumerate(axes):
+        if idx != axis:
+            break
+    else:
+        return a
+
+    return a.transpose(
+        axes + [i for i in range(a.ndim) if i not in axes])
diff --git a/cupy/linalg/_solve.py b/cupy/linalg/_solve.py
new file mode 100644
index 0000000..8090e74
--- /dev/null
+++ b/cupy/linalg/_solve.py
@@ -0,0 +1,407 @@
+import warnings
+
+import numpy
+from numpy import linalg
+
+import cupy
+from cupy._core import internal
+from cupy.cuda import device
+from cupy.linalg import _decomposition
+from cupy.linalg import _util
+from cupy.cublas import batched_gesv, get_batched_gesv_limit
+import cupyx
+
+
+def solve(a, b):
+    """Solves a linear matrix equation.
+
+    It computes the exact solution of ``x`` in ``ax = b``,
+    where ``a`` is a square and full rank matrix.
+
+    Args:
+        a (cupy.ndarray): The matrix with dimension ``(..., M, M)``.
+        b (cupy.ndarray): The matrix with dimension ``(..., M)`` or
+            ``(..., M, K)``.
+
+    Returns:
+        cupy.ndarray:
+            The matrix with dimension ``(..., M)`` or ``(..., M, K)``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.solve`
+    """
+    if a.ndim > 2 and a.shape[-1] <= get_batched_gesv_limit():
+        # Note: There is a low performance issue in batched_gesv when matrix is
+        # large, so it is not used in such cases.
+        return batched_gesv(a, b)
+
+    # TODO(kataoka): Move the checks to the beginning
+    _util._assert_cupy_array(a, b)
+    _util._assert_stacked_2d(a)
+    _util._assert_stacked_square(a)
+
+    # TODO(kataoka): Support broadcast
+    if not (
+        (a.ndim == b.ndim or a.ndim == b.ndim + 1)
+        and a.shape[:-1] == b.shape[:a.ndim - 1]
+    ):
+        raise ValueError(
+            'a must have (..., M, M) shape and b must have (..., M) '
+            'or (..., M, K)')
+
+    dtype, out_dtype = _util.linalg_common_type(a, b)
+    if b.size == 0:
+        return cupy.empty(b.shape, out_dtype)
+
+    if a.ndim == 2:
+        # prevent 'a' and 'b' to be overwritten
+        a = a.astype(dtype, copy=True, order='F')
+        b = b.astype(dtype, copy=True, order='F')
+        cupyx.lapack.gesv(a, b)
+        return b.astype(out_dtype, copy=False)
+
+    # prevent 'a' to be overwritten
+    a = a.astype(dtype, copy=True, order='C')
+    x = cupy.empty_like(b, dtype=out_dtype)
+    shape = a.shape[:-2]
+    for i in range(numpy.prod(shape)):
+        index = numpy.unravel_index(i, shape)
+        # prevent 'b' to be overwritten
+        bi = b[index].astype(dtype, copy=True, order='F')
+        cupyx.lapack.gesv(a[index], bi)
+        x[index] = bi
+    return x
+
+
+def tensorsolve(a, b, axes=None):
+    """Solves tensor equations denoted by ``ax = b``.
+
+    Suppose that ``b`` is equivalent to ``cupy.tensordot(a, x)``.
+    This function computes tensor ``x`` from ``a`` and ``b``.
+
+    Args:
+        a (cupy.ndarray): The tensor with ``len(shape) >= 1``
+        b (cupy.ndarray): The tensor with ``len(shape) >= 1``
+        axes (tuple of ints): Axes in ``a`` to reorder to the right
+            before inversion.
+
+    Returns:
+        cupy.ndarray:
+            The tensor with shape ``Q`` such that ``b.shape + Q == a.shape``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.tensorsolve`
+    """
+    if axes is not None:
+        allaxes = list(range(a.ndim))
+        for k in axes:
+            allaxes.remove(k)
+            allaxes.insert(a.ndim, k)
+        a = a.transpose(allaxes)
+
+    oldshape = a.shape[-(a.ndim - b.ndim):]
+    prod = internal.prod(oldshape)
+
+    a = a.reshape(-1, prod)
+    b = b.ravel()
+    result = solve(a, b)
+    return result.reshape(oldshape)
+
+
+def _nrm2_last_axis(x):
+    real_dtype = x.dtype.char.lower()
+    x = cupy.ascontiguousarray(x)
+    return cupy.sum(cupy.square(x.view(real_dtype)), axis=-1)
+
+
+def lstsq(a, b, rcond='warn'):
+    """Return the least-squares solution to a linear matrix equation.
+
+    Solves the equation `a x = b` by computing a vector `x` that
+    minimizes the Euclidean 2-norm `|| b - a x ||^2`.  The equation may
+    be under-, well-, or over- determined (i.e., the number of
+    linearly independent rows of `a` can be less than, equal to, or
+    greater than its number of linearly independent columns).  If `a`
+    is square and of full rank, then `x` (but for round-off error) is
+    the "exact" solution of the equation.
+
+    Args:
+        a (cupy.ndarray): "Coefficient" matrix with dimension ``(M, N)``
+        b (cupy.ndarray): "Dependent variable" values with dimension ``(M,)``
+            or ``(M, K)``
+        rcond (float): Cutoff parameter for small singular values.
+            For stability it computes the largest singular value denoted by
+            ``s``, and sets all singular values smaller than ``s`` to zero.
+
+    Returns:
+        tuple:
+            A tuple of ``(x, residuals, rank, s)``. Note ``x`` is the
+            least-squares solution with shape ``(N,)`` or ``(N, K)`` depending
+            if ``b`` was two-dimensional. The sums of ``residuals`` is the
+            squared Euclidean 2-norm for each column in b - a*x. The
+            ``residuals`` is an empty array if the rank of a is < N or M <= N,
+            but  iff b is 1-dimensional, this is a (1,) shape array, Otherwise
+            the shape is (K,). The ``rank`` of matrix ``a`` is an integer. The
+            singular values of ``a`` are ``s``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.lstsq`
+    """
+    if rcond == 'warn':
+        warnings.warn(
+            '`rcond` parameter will change to the default of '
+            'machine precision times ``max(M, N)`` where M and N '
+            'are the input matrix dimensions.\n'
+            'To use the future default and silence this warning '
+            'we advise to pass `rcond=None`, to keep using the old, '
+            'explicitly pass `rcond=-1`.',
+            FutureWarning)
+        rcond = -1
+
+    _util._assert_cupy_array(a, b)
+    _util._assert_2d(a)
+    # TODO(kataoka): Fix 0-dim
+    if b.ndim > 2:
+        raise linalg.LinAlgError('{}-dimensional array given. Array must be at'
+                                 ' most two-dimensional'.format(b.ndim))
+    m, n = a.shape[-2:]
+    m2 = b.shape[0]
+    if m != m2:
+        raise linalg.LinAlgError('Incompatible dimensions')
+
+    u, s, vh = cupy.linalg.svd(a, full_matrices=False)
+
+    if rcond is None:
+        rcond = numpy.finfo(s.dtype).eps * max(m, n)
+    elif rcond <= 0 or rcond >= 1:
+        # some doc of gelss/gelsd says "rcond < 0", but it's not true!
+        rcond = numpy.finfo(s.dtype).eps
+
+    # number of singular values and matrix rank
+    s1 = 1 / s
+    rank = cupy.array(s.size, numpy.int32)
+    if s.size > 0:
+        cutoff = rcond * s.max()
+        sing_vals = s <= cutoff
+        s1[sing_vals] = 0
+        rank -= sing_vals.sum(dtype=numpy.int32)
+
+    # Solve the least-squares solution
+    # x = vh.T.conj() @ diag(s1) @ u.T.conj() @ b
+    z = (cupy.dot(b.T, u.conj()) * s1).T
+    x = cupy.dot(vh.T.conj(), z)
+    # Calculate squared Euclidean 2-norm for each column in b - a*x
+    if m <= n or rank != n:
+        resids = cupy.empty((0,), dtype=s.dtype)
+    else:
+        e = b - a.dot(x)
+        resids = cupy.atleast_1d(_nrm2_last_axis(e.T))
+    return x, resids, rank, s
+
+
+def inv(a):
+    """Computes the inverse of a matrix.
+
+    This function computes matrix ``a_inv`` from n-dimensional regular matrix
+    ``a`` such that ``dot(a, a_inv) == eye(n)``.
+
+    Args:
+        a (cupy.ndarray): The regular matrix
+
+    Returns:
+        cupy.ndarray: The inverse of a matrix.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.inv`
+    """
+    _util._assert_cupy_array(a)
+    _util._assert_stacked_2d(a)
+    _util._assert_stacked_square(a)
+
+    if a.ndim >= 3:
+        return _batched_inv(a)
+
+    dtype, out_dtype = _util.linalg_common_type(a)
+    if a.size == 0:
+        return cupy.empty(a.shape, out_dtype)
+
+    order = 'F' if a._f_contiguous else 'C'
+    # prevent 'a' to be overwritten
+    a = a.astype(dtype, copy=True, order=order)
+    b = cupy.eye(a.shape[0], dtype=dtype, order=order)
+    if order == 'F':
+        cupyx.lapack.gesv(a, b)
+    else:
+        cupyx.lapack.gesv(a.T, b.T)
+    return b.astype(out_dtype, copy=False)
+
+
+def _batched_inv(a):
+    # a.ndim must be >= 3
+    dtype, out_dtype = _util.linalg_common_type(a)
+    if a.size == 0:
+        return cupy.empty(a.shape, out_dtype)
+
+    if dtype == cupy.float32:
+        getrf = cupy.cuda.cublas.sgetrfBatched
+        getri = cupy.cuda.cublas.sgetriBatched
+    elif dtype == cupy.float64:
+        getrf = cupy.cuda.cublas.dgetrfBatched
+        getri = cupy.cuda.cublas.dgetriBatched
+    elif dtype == cupy.complex64:
+        getrf = cupy.cuda.cublas.cgetrfBatched
+        getri = cupy.cuda.cublas.cgetriBatched
+    elif dtype == cupy.complex128:
+        getrf = cupy.cuda.cublas.zgetrfBatched
+        getri = cupy.cuda.cublas.zgetriBatched
+    else:
+        msg = ('dtype must be float32, float64, complex64 or complex128'
+               ' (actual: {})'.format(a.dtype))
+        raise ValueError(msg)
+
+    if 0 in a.shape:
+        return cupy.empty_like(a, dtype=out_dtype)
+    a_shape = a.shape
+
+    # copy is necessary to present `a` to be overwritten.
+    a = a.astype(dtype, order='C').reshape(-1, a_shape[-2], a_shape[-1])
+
+    handle = device.get_cublas_handle()
+    batch_size = a.shape[0]
+    n = a.shape[1]
+    lda = n
+    step = n * lda * a.itemsize
+    start = a.data.ptr
+    stop = start + step * batch_size
+    a_array = cupy.arange(start, stop, step, dtype=cupy.uintp)
+    pivot_array = cupy.empty((batch_size, n), dtype=cupy.int32)
+    info_array = cupy.empty((batch_size,), dtype=cupy.int32)
+
+    getrf(handle, n, a_array.data.ptr, lda, pivot_array.data.ptr,
+          info_array.data.ptr, batch_size)
+    cupy.linalg._util._check_cublas_info_array_if_synchronization_allowed(
+        getrf, info_array)
+
+    c = cupy.empty_like(a)
+    ldc = lda
+    step = n * ldc * c.itemsize
+    start = c.data.ptr
+    stop = start + step * batch_size
+    c_array = cupy.arange(start, stop, step, dtype=cupy.uintp)
+
+    getri(handle, n, a_array.data.ptr, lda, pivot_array.data.ptr,
+          c_array.data.ptr, ldc, info_array.data.ptr, batch_size)
+    cupy.linalg._util._check_cublas_info_array_if_synchronization_allowed(
+        getri, info_array)
+
+    return c.reshape(a_shape).astype(out_dtype, copy=False)
+
+
+# TODO(leofang): support the hermitian keyword?
+def pinv(a, rcond=1e-15):
+    """Compute the Moore-Penrose pseudoinverse of a matrix.
+
+    It computes a pseudoinverse of a matrix ``a``, which is a generalization
+    of the inverse matrix with Singular Value Decomposition (SVD).
+    Note that it automatically removes small singular values for stability.
+
+    Args:
+        a (cupy.ndarray): The matrix with dimension ``(..., M, N)``
+        rcond (float or cupy.ndarray): Cutoff parameter for small singular
+            values. For stability it computes the largest singular value
+            denoted by ``s``, and sets all singular values smaller than
+            ``rcond * s`` to zero. Broadcasts against the stack of matrices.
+
+    Returns:
+        cupy.ndarray: The pseudoinverse of ``a`` with dimension
+        ``(..., N, M)``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.pinv`
+    """
+    _util._assert_cupy_array(a)
+    if a.size == 0:
+        _, out_dtype = _util.linalg_common_type(a)
+        m, n = a.shape[-2:]
+        if m == 0 or n == 0:
+            out_dtype = a.dtype  # NumPy bug?
+        return cupy.empty(a.shape[:-2] + (n, m), dtype=out_dtype)
+
+    u, s, vt = _decomposition.svd(a.conj(), full_matrices=False)
+
+    # discard small singular values
+    cutoff = rcond * cupy.amax(s, axis=-1)
+    leq = s <= cutoff[..., None]
+    cupy.reciprocal(s, out=s)
+    s[leq] = 0
+
+    return cupy.matmul(vt.swapaxes(-2, -1), s[..., None] * u.swapaxes(-2, -1))
+
+
+def tensorinv(a, ind=2):
+    """Computes the inverse of a tensor.
+
+    This function computes tensor ``a_inv`` from tensor ``a`` such that
+    ``tensordot(a_inv, a, ind) == I``, where ``I`` denotes the identity tensor.
+
+    Args:
+        a (cupy.ndarray):
+            The tensor such that
+            ``prod(a.shape[:ind]) == prod(a.shape[ind:])``.
+        ind (int):
+            The positive number used in ``axes`` option of ``tensordot``.
+
+    Returns:
+        cupy.ndarray:
+            The inverse of a tensor whose shape is equivalent to
+            ``a.shape[ind:] + a.shape[:ind]``.
+
+    .. warning::
+        This function calls one or more cuSOLVER routine(s) which may yield
+        invalid results if input conditions are not met.
+        To detect these invalid results, you can set the `linalg`
+        configuration to a value that is not `ignore` in
+        :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+    .. seealso:: :func:`numpy.linalg.tensorinv`
+    """
+    _util._assert_cupy_array(a)
+
+    if ind <= 0:
+        raise ValueError('Invalid ind argument')
+    oldshape = a.shape
+    invshape = oldshape[ind:] + oldshape[:ind]
+    prod = internal.prod(oldshape[ind:])
+    a = a.reshape(prod, -1)
+    a_inv = inv(a)
+    return a_inv.reshape(*invshape)
diff --git a/cupy/linalg/_util.py b/cupy/linalg/_util.py
new file mode 100644
index 0000000..3dce6cc
--- /dev/null
+++ b/cupy/linalg/_util.py
@@ -0,0 +1,195 @@
+import os
+
+import numpy
+from numpy import linalg
+
+import cupy
+import cupy._util
+from cupy import _core
+import cupyx
+
+
+_default_precision = os.getenv('CUPY_DEFAULT_PRECISION')
+
+
+# The helper functions raise LinAlgError if the conditions are not met.
+
+def _assert_cupy_array(*arrays):
+    for a in arrays:
+        if not isinstance(a, cupy._core.ndarray):
+            raise linalg.LinAlgError(
+                'cupy.linalg only supports cupy.ndarray')
+
+
+def _assert_2d(*arrays):
+    for a in arrays:
+        if a.ndim != 2:
+            raise linalg.LinAlgError(
+                '{}-dimensional array given. Array must be '
+                'two-dimensional'.format(a.ndim))
+
+
+def _assert_stacked_2d(*arrays):
+    for a in arrays:
+        if a.ndim < 2:
+            raise linalg.LinAlgError(
+                '{}-dimensional array given. Array must be '
+                'at least two-dimensional'.format(a.ndim))
+
+
+def _assert_stacked_square(*arrays):
+    """Assert that stacked matrices are square matrices
+
+    Precondition: `arrays` are at least 2d. The caller should assert it
+    beforehand. For example,
+
+    >>> def det(a):
+    ...     _assert_stacked_2d(a)
+    ...     _assert_stacked_square(a)
+    ...     ...
+
+    """
+
+    for a in arrays:
+        m, n = a.shape[-2:]
+        if m != n:
+            raise linalg.LinAlgError(
+                'Last 2 dimensions of the array must be square')
+
+
+def linalg_common_type(*arrays, reject_float16=True):
+    """Common type for linalg
+
+    The logic is intended to be equivalent with
+    `numpy.linalg.linalg._commonType`.
+    The differences from `numpy.common_type` are
+    - to accept ``bool_`` arrays, and
+    - to reject ``float16`` arrays.
+
+    Args:
+        *arrays (ndarray): Input arrays.
+        reject_float16 (bool): Flag to follow NumPy to raise TypeError for
+            ``float16`` inputs.
+
+    Returns:
+        compute_dtype (dtype): The precision to be used in linalg calls.
+        result_dtype (dtype): The dtype of (possibly complex) output(s).
+    """
+    dtypes = [arr.dtype for arr in arrays]
+    if reject_float16 and 'float16' in dtypes:
+        raise TypeError('float16 is unsupported in linalg')
+
+    if _default_precision is not None:
+        cupy._util.experimental('CUPY_DEFAULT_PRECISION')
+        if _default_precision not in ('32', '64'):
+            raise ValueError(
+                'invalid CUPY_DEFAULT_PRECISION: {}'.format(
+                    _default_precision))
+        default = 'float' + _default_precision
+    else:
+        default = 'float64'
+    compute_dtype = _common_type_internal(default, *dtypes)
+    # No fp16 cuSOLVER routines
+    if compute_dtype == 'float16':
+        compute_dtype = numpy.dtype('float32')
+
+    # numpy casts integer types to float64
+    result_dtype = _common_type_internal('float64', *dtypes)
+
+    return compute_dtype, result_dtype
+
+
+def _common_type_internal(default_dtype, *dtypes):
+    inexact_dtypes = [
+        dtype if dtype.kind in 'fc' else default_dtype
+        for dtype in dtypes]
+    return numpy.result_type(*inexact_dtypes)
+
+
+def _check_cusolver_dev_info_if_synchronization_allowed(routine, dev_info):
+    # `dev_info` contains integers, the status code of a cuSOLVER
+    # routine call. It is referred to as "infoArray" or "devInfo" in the
+    # official cuSOLVER documentation.
+    assert isinstance(dev_info, _core.ndarray)
+    config_linalg = cupyx._ufunc_config.get_config_linalg()
+    # Only 'ignore' and 'raise' are currently supported.
+    if config_linalg == 'ignore':
+        return
+
+    try:
+        name = routine.__name__
+    except AttributeError:
+        name = routine  # routine is a str
+
+    assert config_linalg == 'raise'
+    if (dev_info != 0).any():
+        raise linalg.LinAlgError(
+            'Error reported by {} in cuSOLVER. devInfo = {}. Please refer'
+            ' to the cuSOLVER documentation.'.format(
+                name, dev_info))
+
+
+def _check_cublas_info_array_if_synchronization_allowed(routine, info_array):
+    # `info_array` contains integers, the status codes of a cuBLAS routine
+    # call. It is referrd to as "infoArray" or "devInfoArray" in the official
+    # cuBLAS documentation.
+    assert isinstance(info_array, _core.ndarray)
+    assert info_array.ndim == 1
+
+    config_linalg = cupyx._ufunc_config.get_config_linalg()
+    # Only 'ignore' and 'raise' are currently supported.
+    if config_linalg == 'ignore':
+        return
+
+    assert config_linalg == 'raise'
+    if (info_array != 0).any():
+        raise linalg.LinAlgError(
+            'Error reported by {} in cuBLAS. infoArray/devInfoArray = {}.'
+            ' Please refer to the cuBLAS documentation.'.format(
+                routine.__name__, info_array))
+
+
+_tril_kernel = _core.ElementwiseKernel(
+    'int64 k', 'S x',
+    'x = (_ind.get()[1] - _ind.get()[0] <= k) ? x : 0',
+    'cupy_tril_kernel',
+    reduce_dims=False
+)
+
+
+def _tril(x, k=0):
+    _tril_kernel(k, x)
+    return x
+
+
+# support a batch of matrices
+_triu_kernel = _core.ElementwiseKernel(
+    'int64 k', 'S x',
+    'x = (_ind.get()[_ind.ndim - 1] - _ind.get()[_ind.ndim - 2] >= k) ? x : 0',
+    'cupy_triu_kernel',
+    reduce_dims=False
+)
+
+
+def _triu(x, k=0):
+    _triu_kernel(k, x)
+    return x
+
+
+def stacked_identity(batch_shape, n, dtype):
+    shape = batch_shape + (n, n)
+    idx = cupy.arange(n)
+    x = cupy.zeros(shape, dtype)
+    x[..., idx, idx] = 1
+    return x
+
+
+def stacked_identity_like(x):
+    """
+    Precondition: ``x`` is `cupy.ndarray` of shape ``(..., N, N)``
+    """
+    n = x.shape[-1]
+    idx = cupy.arange(n)
+    x = cupy.zeros_like(x)
+    x[..., idx, idx] = 1
+    return x
diff --git a/cupy/polynomial/__init__.py b/cupy/polynomial/__init__.py
new file mode 100644
index 0000000..271f72d
--- /dev/null
+++ b/cupy/polynomial/__init__.py
@@ -0,0 +1,6 @@
+# Functions from the following NumPy document
+# https://numpy.org/doc/stable/reference/routines.polynomials.html
+
+# "NOQA" to suppress flake8 warning
+from cupy.polynomial import polynomial  # NOQA
+from cupy.polynomial import polyutils  # NOQA
diff --git a/cupy/polynomial/polynomial.py b/cupy/polynomial/polynomial.py
new file mode 100644
index 0000000..783617e
--- /dev/null
+++ b/cupy/polynomial/polynomial.py
@@ -0,0 +1,46 @@
+import cupy
+
+
+def polyvander(x, deg):
+    """Computes the Vandermonde matrix of given degree.
+
+    Args:
+        x (cupy.ndarray): array of points
+        deg (int): degree of the resulting matrix.
+
+    Returns:
+        cupy.ndarray: The Vandermonde matrix
+
+    .. seealso:: :func:`numpy.polynomial.polynomial.polyvander`
+
+    """
+    deg = cupy.polynomial.polyutils._deprecate_as_int(deg, 'deg')
+    if deg < 0:
+        raise ValueError('degree must be non-negative')
+    if x.ndim == 0:
+        x = x.ravel()
+    dtype = cupy.float64 if x.dtype.kind in 'biu' else x.dtype
+    out = x ** cupy.arange(deg + 1, dtype=dtype).reshape((-1,) + (1,) * x.ndim)
+    return cupy.moveaxis(out, 0, -1)
+
+
+def polycompanion(c):
+    """Computes the companion matrix of c.
+
+    Args:
+        c (cupy.ndarray): 1-D array of polynomial coefficients
+            ordered from low to high degree.
+
+    Returns:
+        cupy.ndarray: Companion matrix of dimensions (deg, deg).
+
+    .. seealso:: :func:`numpy.polynomial.polynomial.polycompanion`
+
+    """
+    [c] = cupy.polynomial.polyutils.as_series([c])
+    deg = c.size - 1
+    if deg == 0:
+        raise ValueError('Series must have maximum degree of at least 1.')
+    matrix = cupy.eye(deg, k=-1, dtype=c.dtype)
+    matrix[:, -1] -= c[:-1] / c[-1]
+    return matrix
diff --git a/cupy/polynomial/polyutils.py b/cupy/polynomial/polyutils.py
new file mode 100644
index 0000000..a617758
--- /dev/null
+++ b/cupy/polynomial/polyutils.py
@@ -0,0 +1,110 @@
+import cupy
+
+import operator
+import warnings
+
+
+def _deprecate_as_int(x, desc):
+    try:
+        return operator.index(x)
+    except TypeError as e:
+        try:
+            ix = int(x)
+        except TypeError:
+            pass
+        else:
+            if ix == x:
+                warnings.warn(
+                    'In future, this will raise TypeError, as {} will '
+                    'need to be an integer not just an integral float.'
+                    .format(desc),
+                    DeprecationWarning,
+                    stacklevel=3
+                )
+                return ix
+
+        raise TypeError('{} must be an integer'.format(desc)) from e
+
+
+def trimseq(seq):
+    """Removes small polynomial series coefficients.
+
+    Args:
+        seq (cupy.ndarray): input array.
+
+    Returns:
+        cupy.ndarray: input array with trailing zeros removed. If the
+        resulting output is empty, it returns the first element.
+
+    .. seealso:: :func:`numpy.polynomial.polyutils.trimseq`
+
+    """
+    if seq.size == 0:
+        return seq
+    ret = cupy.trim_zeros(seq, trim='b')
+    if ret.size > 0:
+        return ret
+    return seq[:1]
+
+
+def as_series(alist, trim=True):
+    """Returns argument as a list of 1-d arrays.
+
+    Args:
+        alist (cupy.ndarray or list of cupy.ndarray): 1-D or 2-D input array.
+        trim (bool, optional): trim trailing zeros.
+
+    Returns:
+        list of cupy.ndarray: list of 1-D arrays.
+
+    .. seealso:: :func:`numpy.polynomial.polyutils.as_series`
+
+    """
+    arrays = []
+    for a in alist:
+        if a.size == 0:
+            raise ValueError('Coefficient array is empty')
+        if a.ndim > 1:
+            raise ValueError('Coefficient array is not 1-d')
+        if a.dtype.kind == 'b':
+            raise ValueError('Coefficient arrays have no common type')
+        a = a.ravel()
+        if trim:
+            a = trimseq(a)
+        arrays.append(a)
+    dtype = cupy.common_type(*arrays)
+    ret = [a.astype(dtype, copy=False) for a in arrays]
+    return ret
+
+
+def trimcoef(c, tol=0):
+    """Removes small trailing coefficients from a polynomial.
+
+    Args:
+        c(cupy.ndarray): 1d array of coefficients from lowest to highest order.
+        tol(number, optional): trailing coefficients whose absolute value are
+            less than or equal to ``tol`` are trimmed.
+
+    Returns:
+        cupy.ndarray: trimmed 1d array.
+
+    .. seealso:: :func:`numpy.polynomial.polyutils.trimcoef`
+
+    """
+    if tol < 0:
+        raise ValueError('tol must be non-negative')
+    if c.size == 0:
+        raise ValueError('Coefficient array is empty')
+    if c.ndim > 1:
+        raise ValueError('Coefficient array is not 1-d')
+    if c.dtype.kind == 'b':
+        raise ValueError('bool inputs are not allowed')
+    if c.ndim == 0:
+        c = c.ravel()
+    c = c.astype(cupy.common_type(c), copy=False)
+    filt = (cupy.abs(c) > tol)[::-1]
+    ind = c.size - cupy._manipulation.add_remove._first_nonzero_krnl(
+        filt, c.size).item()
+    if ind == 0:
+        return cupy.zeros_like(c[:1])
+    return c[: ind]
diff --git a/cupy/prof/__init__.py b/cupy/prof/__init__.py
new file mode 100644
index 0000000..011cceb
--- /dev/null
+++ b/cupy/prof/__init__.py
@@ -0,0 +1,2 @@
+from cupy.prof._time_range import time_range  # NOQA
+from cupy.prof._time_range import TimeRangeDecorator  # NOQA
diff --git a/cupy/prof/_time_range.py b/cupy/prof/_time_range.py
new file mode 100644
index 0000000..ba50d62
--- /dev/null
+++ b/cupy/prof/_time_range.py
@@ -0,0 +1,78 @@
+import contextlib
+import warnings
+
+from cupyx.profiler import time_range as _time_range
+
+
+@contextlib.contextmanager
+def time_range(message, color_id=None, argb_color=None, sync=False):
+    """A context manager to describe the enclosed block as a nested range
+
+    >>> from cupy import prof
+    >>> with cupy.prof.time_range('some range in green', color_id=0):
+    ...    # do something you want to measure
+    ...    pass
+
+    Args:
+        message: Name of a range.
+        color_id: range color ID
+        argb_color: range color in ARGB (e.g. 0xFF00FF00 for green)
+        sync (bool): If ``True``, waits for completion of all outstanding
+            processing on GPU before calling :func:`cupy.cuda.nvtx.RangePush()`
+            or :func:`cupy.cuda.nvtx.RangePop()`
+
+    .. seealso:: :func:`cupy.cuda.nvtx.RangePush`
+        :func:`cupy.cuda.nvtx.RangePop`
+
+    .. warning:: This context manager is deprecated. Please use
+        :class:`cupyx.profiler.time_range` instead.
+    """
+    warnings.warn(
+        'cupy.prof.time_range has been deprecated since CuPy v10 '
+        'and will be removed in the future. Use cupyx.profiler.time_range '
+        'instead.')
+
+    with _time_range(message, color_id, argb_color, sync):
+        yield
+
+
+class TimeRangeDecorator:
+    """Decorator to mark function calls with range in NVIDIA profiler
+
+    Decorated function calls are marked as ranges in NVIDIA profiler timeline.
+
+    >>> from cupy import prof
+    >>> @cupy.prof.TimeRangeDecorator()
+    ... def function_to_profile():
+    ...     pass
+
+    Args:
+        message (str): Name of a range, default use ``func.__name__``.
+        color_id: range color ID
+        argb_color: range color in ARGB (e.g. 0xFF00FF00 for green)
+        sync (bool): If ``True``, waits for completion of all outstanding
+            processing on GPU before calling :func:`cupy.cuda.nvtx.RangePush()`
+            or :func:`cupy.cuda.nvtx.RangePop()`
+
+    .. seealso:: :func:`cupy.cuda.nvtx.RangePush`
+        :func:`cupy.cuda.nvtx.RangePop`
+
+    .. warning:: This decorator is deprecated. Please use
+        :class:`cupyx.profiler.time_range` instead.
+    """
+
+    _init = _time_range.__init__
+    __enter__ = _time_range.__enter__
+    __exit__ = _time_range.__exit__
+    __call__ = _time_range.__call__
+    _recreate_cm = _time_range._recreate_cm
+
+    def __init__(self, message=None, color_id=None, argb_color=None,
+                 sync=False):
+
+        warnings.warn(
+            'cupy.prof.TimeRangeDecorator has been deprecated since CuPy v10 '
+            'and will be removed in the future. Use cupyx.profiler.time_range '
+            'instead.')
+
+        self._init(message, color_id, argb_color, sync)
diff --git a/cupy/random/LICENSE b/cupy/random/LICENSE
new file mode 100644
index 0000000..73c4aa6
--- /dev/null
+++ b/cupy/random/LICENSE
@@ -0,0 +1,109 @@
+Random kit 1.3
+
+Copyright (c) 2003-2005, Jean-Sebastien Roy (js@jeannot.org)
+
+The rk_random and rk_seed functions algorithms and the original design of
+the Mersenne Twister RNG:
+
+  Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+  1. Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+  3. The names of its contributors may not be used to endorse or promote
+  products derived from this software without specific prior written
+  permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Original algorithm for the implementation of rk_interval function from
+Richard J. Wagner's implementation of the Mersenne Twister RNG, optimised by
+Magnus Jonsson.
+
+Constants used in the rk_double implementation by Isaku Wada.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+The kernels for distributions are based on
+numpy/random/mtrand/distributions.c
+with the following licenses:
+
+/* Copyright 2005 Robert Kern (robert.kern@gmail.com)
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/* The implementations of rk_hypergeometric_hyp(), rk_hypergeometric_hrua(),
+ * and rk_triangular() were adapted from Ivan Frohne's rv.py which has this
+ * license:
+ *
+ *            Copyright 1998 by Ivan Frohne; Wasilla, Alaska, U.S.A.
+ *                            All Rights Reserved
+ *
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation for any purpose, free of charge, is granted subject to the
+ * following conditions:
+ *   The above copyright notice and this permission notice shall be included in
+ *   all copies or substantial portions of the software.
+ *
+ *   THE SOFTWARE AND DOCUMENTATION IS PROVIDED WITHOUT WARRANTY OF ANY KIND,
+ *   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO MERCHANTABILITY, FITNESS
+ *   FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR
+ *   OR COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM OR DAMAGES IN A CONTRACT
+ *   ACTION, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ *   SOFTWARE OR ITS DOCUMENTATION.
+ */
diff --git a/cupy/random/__init__.py b/cupy/random/__init__.py
new file mode 100644
index 0000000..167e494
--- /dev/null
+++ b/cupy/random/__init__.py
@@ -0,0 +1,101 @@
+import numpy as _numpy
+
+from cupy_backends.cuda.api import runtime
+
+
+def bytes(length):
+    """Returns random bytes.
+
+    .. note:: This function is just a wrapper for :obj:`numpy.random.bytes`.
+        The resulting bytes are generated on the host (NumPy), not GPU.
+
+    .. seealso:: :meth:`numpy.random.bytes
+                 <numpy.random.mtrand.RandomState.bytes>`
+    """
+    # TODO(kmaehashi): should it be provided in CuPy?
+    return _numpy.random.bytes(length)
+
+
+def default_rng(seed=None):  # NOQA  avoid redefinition of seed
+    """Construct a new Generator with the default BitGenerator (XORWOW).
+
+    Args:
+        seed (None, int, array_like[ints], numpy.random.SeedSequence, cupy.random.BitGenerator, cupy.random.Generator, optional):
+            A seed to initialize the :class:`cupy.random.BitGenerator`. If an
+            ``int`` or ``array_like[ints]`` or None is passed, then it will be
+            passed to :class:`numpy.random.SeedSequence` to detive the initial
+            :class:`BitGenerator` state. One may also pass in a `SeedSequence
+            instance. Adiditionally, when passed :class:`BitGenerator`, it will
+            be wrapped by :class:`Generator`. If passed a :class:`Generator`,
+            it will be returned unaltered.
+
+    Returns:
+        Generator: The initialized generator object.
+    """  # NOQA, list of types need to be in one line for sphinx
+    if runtime.is_hip and int(str(runtime.runtimeGetVersion())[:3]) < 403:
+        raise RuntimeError('Generator API not supported in ROCm<4.3,'
+                           ' please use the legacy one or update ROCm.')
+    if isinstance(seed, BitGenerator):
+        return Generator(seed)
+    elif isinstance(seed, Generator):
+        return seed
+    return Generator(XORWOW(seed))
+
+
+# import class and function
+from cupy.random._distributions import beta  # NOQA
+from cupy.random._distributions import binomial  # NOQA
+from cupy.random._distributions import chisquare  # NOQA
+from cupy.random._distributions import dirichlet  # NOQA
+from cupy.random._distributions import exponential  # NOQA
+from cupy.random._distributions import f  # NOQA
+from cupy.random._distributions import gamma  # NOQA
+from cupy.random._distributions import geometric  # NOQA
+from cupy.random._distributions import gumbel  # NOQA
+from cupy.random._distributions import hypergeometric  # NOQA
+from cupy.random._distributions import laplace  # NOQA
+from cupy.random._distributions import logistic  # NOQA
+from cupy.random._distributions import lognormal  # NOQA
+from cupy.random._distributions import logseries  # NOQA
+from cupy.random._distributions import multivariate_normal  # NOQA
+from cupy.random._distributions import negative_binomial  # NOQA
+from cupy.random._distributions import noncentral_chisquare  # NOQA
+from cupy.random._distributions import noncentral_f  # NOQA
+from cupy.random._distributions import normal  # NOQA
+from cupy.random._distributions import pareto  # NOQA
+from cupy.random._distributions import poisson  # NOQA
+from cupy.random._distributions import power  # NOQA
+from cupy.random._distributions import rayleigh  # NOQA
+from cupy.random._distributions import standard_cauchy  # NOQA
+from cupy.random._distributions import standard_exponential  # NOQA
+from cupy.random._distributions import standard_gamma  # NOQA
+from cupy.random._distributions import standard_normal  # NOQA
+from cupy.random._distributions import standard_t  # NOQA
+from cupy.random._distributions import triangular  # NOQA
+from cupy.random._distributions import uniform  # NOQA
+from cupy.random._distributions import vonmises  # NOQA
+from cupy.random._distributions import wald  # NOQA
+from cupy.random._distributions import weibull  # NOQA
+from cupy.random._distributions import zipf  # NOQA
+from cupy.random._generator import get_random_state  # NOQA
+from cupy.random._generator import RandomState  # NOQA
+from cupy.random._generator import reset_states  # NOQA
+from cupy.random._generator import seed  # NOQA
+from cupy.random._generator import set_random_state  # NOQA
+from cupy.random._permutations import permutation  # NOQA
+from cupy.random._permutations import shuffle  # NOQA
+from cupy.random._sample import choice  # NOQA
+from cupy.random._sample import multinomial  # NOQA
+from cupy.random._sample import rand  # NOQA
+from cupy.random._sample import randint  # NOQA
+from cupy.random._sample import randn  # NOQA
+from cupy.random._sample import random_integers  # NOQA
+from cupy.random._sample import random_sample  # NOQA
+from cupy.random._sample import random_sample as random  # NOQA
+from cupy.random._sample import random_sample as ranf  # NOQA
+from cupy.random._sample import random_sample as sample  # NOQA
+from cupy.random._bit_generator import BitGenerator  # NOQA
+from cupy.random._bit_generator import XORWOW  # NOQA
+from cupy.random._bit_generator import MRG32k3a  # NOQA
+from cupy.random._bit_generator import Philox4x3210  # NOQA
+from cupy.random._generator_api import Generator  # NOQA
diff --git a/cupy/random/_bit_generator.pyx b/cupy/random/_bit_generator.pyx
new file mode 100644
index 0000000..c73f4b0
--- /dev/null
+++ b/cupy/random/_bit_generator.pyx
@@ -0,0 +1,184 @@
+# distutils: language = c++
+import threading
+
+import numpy
+
+from libc.stdint cimport uint64_t
+
+import cupy
+from cupy.random._generator_api import init_curand, random_raw
+from cupy_backends.cuda.api import runtime
+
+# We need access to the sizes here, so this is why we have this header
+# in here instead of cupy backends
+cdef extern from 'cupy_distributions.cuh' nogil:
+    cppclass curandState:
+        pass
+    cppclass curandStateMRG32k3a:
+        pass
+    cppclass curandStatePhilox4_32_10_t:
+        pass
+
+    cdef enum _RandGenerators 'RandGenerators':
+        CURAND_XOR_WOW
+        CURAND_MRG32k3a
+        CURAND_PHILOX_4x32_10
+
+
+class BitGenerator:
+    """Generic BitGenerator.
+
+    Base Class for generic BitGenerators, which provide a stream
+    of random bits based on different algorithms. Must be overridden.
+
+    Args:
+        seed (int, array_like[ints], numpy.random.SeedSequence, optional):
+            A seed to initialize the `BitGenerator`. If None, then fresh,
+            unpredictable entropy will be pulled from the OS. If an ``int`` or
+            ``array_like[ints]`` is passed, then it will be passed to
+            ~`numpy.random.SeedSequence` to derive the initial `BitGenerator`
+            state. One may also pass in a `SeedSequence` instance.
+    """
+    def __init__(self, seed=None):
+        if runtime.is_hip and int(str(runtime.runtimeGetVersion())[:3]) < 403:
+            raise RuntimeError('Generator API not supported in ROCm<4.3, '
+                               'please use the legacy one or update ROCm.')
+        self.lock = threading.Lock()
+        # TODO(ecastill) port SeedSequence
+        if isinstance(seed, numpy.random.SeedSequence):
+            self._seed_seq = seed
+        else:
+            if isinstance(seed, cupy.ndarray):
+                seed = cupy.asnumpy(seed)
+            self._seed_seq = numpy.random.SeedSequence(seed)
+        self._current_device_id = cupy.cuda.get_device_id()
+
+    def random_raw(self, size=None, output=True):
+        raise NotImplementedError(
+            'Not implemented in base BitGenerator')
+
+    def _state_size(self):
+        """Maximum number of samples that can be generated at once
+        """
+        return 0
+
+    def _check_device(self):
+        if cupy.cuda.get_device_id() != self._current_device_id:
+            raise RuntimeError(
+                'This Generator state is allocated in a different device')
+
+
+class _cuRANDGenerator(BitGenerator):
+    # Size is the number of threads that will be initialized
+    def __init__(self, seed=None, *, size=-1):
+        super().__init__(seed)
+        # Raw kernel has problems with integers with the 64th bit set
+        self._seed = self._seed_seq.generate_state(1, numpy.uint32)[0]
+        if size < 0:
+            size = 8 * 256 * cupy.cuda.runtime.cudaDevAttrMultiProcessorCount
+        self._size = size
+        cdef uint64_t b_size = self._type_size() * size
+        self._state = cupy.zeros(b_size, dtype=numpy.int8)
+        ptr = self._state.data.ptr
+        # Initialize the state
+        init_curand(self.generator, ptr, self._seed, size)
+
+    def random_raw(self, size=None, output=True):
+        """Return randoms as generated by the underlying BitGenerator.
+
+        Args:
+            size (int or tuple of ints, optional):
+                Output shape.  If the given shape is, e.g., ``(m, n, k)``, then
+                ``m * n * k`` samples are drawn.  Default is None, in which
+                case a single value is returned.
+            output (bool, optional):
+                Output values.  Used for performance testing since the
+                generated values are not returned.
+
+        Returns:
+            cupy.ndarray: Drawn samples.
+
+        .. note::
+            This method directly exposes the the raw underlying pseudo-random
+            number generator. All values are returned as unsigned 64-bit
+            values irrespective of the number of bits produced by the PRNG.
+            See the class docstring for the number of bits returned.
+
+        """
+        shape = size if size is not None else ()
+        y = cupy.zeros(shape, dtype=numpy.int32)
+        random_raw(self, y)
+        return y if output else None
+
+    def state(self):
+        self._check_device()
+        return self._state.data.ptr
+
+    def _state_size(self):
+        return self._size
+
+    def _type_size(self):
+        return 0
+
+
+class XORWOW(_cuRANDGenerator):
+    """BitGenerator that uses cuRAND XORWOW device generator.
+
+    This generator allocates the state using the cuRAND device API.
+
+    Args:
+        seed (None, int, array_like[ints], numpy.random.SeedSequence):
+            A seed to initialize the `BitGenerator`. If None, then fresh,
+            unpredictable entropy will be pulled from the OS. If an ``int`` or
+            ``array_like[ints]`` is passed, then it will be passed to
+            ~`numpy.random.SeedSequence` to derive the initial `BitGenerator`
+            state. One may also pass in a `SeedSequence` instance.
+        size (int): Maximum number of samples that can be generated at once.
+            defaults to 1000 * 256.
+    """
+    generator = CURAND_XOR_WOW  # Use The Enum
+
+    def _type_size(self):
+        return sizeof(curandState)
+
+
+class MRG32k3a(_cuRANDGenerator):
+    """BitGenerator that uses cuRAND MRG32k3a device generator.
+
+    This generator allocates the state using the cuRAND device API.
+
+    Args:
+        seed (int, array_like[ints], numpy.random.SeedSequence, optional):
+            A seed to initialize the `BitGenerator`. If None, then fresh,
+            unpredictable entropy will be pulled from the OS. If an ``int`` or
+            ``array_like[ints]`` is passed, then it will be passed to
+            ~`numpy.random.SeedSequence` to derive the initial `BitGenerator`
+            state. One may also pass in a `SeedSequence` instance.
+        size (int): Maximum number of samples that can be generated at once.
+            defaults to 1000 * 256.
+    """
+    generator = CURAND_MRG32k3a
+
+    def _type_size(self):
+        return sizeof(curandStateMRG32k3a)
+
+
+class Philox4x3210(_cuRANDGenerator):
+    """BitGenerator that uses cuRAND Philox4x3210 device generator.
+
+    This generator allocates the state using the cuRAND device API.
+
+    Args:
+        seed (int, array_like[ints], numpy.random.SeedSequence, optional):
+            A seed to initialize the `BitGenerator`. If None, then fresh,
+            unpredictable entropy will be pulled from the OS. If an ``int`` or
+            ``array_like[ints]`` is passed, then it will be passed to
+            ~`numpy.random.SeedSequence` to derive the initial `BitGenerator`
+            state. One may also pass in a `SeedSequence` instance.
+        size (int): Maximum number of samples that can be generated at once.
+            defaults to 1000 * 256.
+    """
+    generator = CURAND_PHILOX_4x32_10
+
+    def _type_size(self):
+        return sizeof(curandStatePhilox4_32_10_t)
diff --git a/cupy/random/_distributions.py b/cupy/random/_distributions.py
new file mode 100644
index 0000000..e778372
--- /dev/null
+++ b/cupy/random/_distributions.py
@@ -0,0 +1,952 @@
+from cupy.random import _generator
+from cupy import _util
+
+
+# TODO(beam2d): Implement many distributions
+
+
+def beta(a, b, size=None, dtype=float):
+    """Beta distribution.
+
+    Returns an array of samples drawn from the beta distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{x^{\\alpha-1}(1-x)^{\\beta-1}}{B(\\alpha,\\beta)}.
+
+    Args:
+        a (float): Parameter of the beta distribution :math:`\\alpha`.
+        b (float): Parameter of the beta distribution :math:`\\beta`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the beta distribution.
+
+    .. seealso::
+        :func:`numpy.random.beta`
+    """
+    rs = _generator.get_random_state()
+    return rs.beta(a, b, size, dtype)
+
+
+def binomial(n, p, size=None, dtype=int):
+    """Binomial distribution.
+
+    Returns an array of samples drawn from the binomial distribution. Its
+    probability mass function is defined as
+
+    .. math::
+        f(x) = \\binom{n}{x}p^x(1-p)^{n-x}.
+
+    Args:
+        n (int): Trial number of the binomial distribution.
+        p (float): Success probability of the binomial distribution.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.int32` and
+            :class:`numpy.int64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the binomial distribution.
+
+    .. seealso::
+        :func:`numpy.random.binomial`
+    """
+    rs = _generator.get_random_state()
+    return rs.binomial(n, p, size, dtype)
+
+
+def chisquare(df, size=None, dtype=float):
+    """Chi-square distribution.
+
+    Returns an array of samples drawn from the chi-square distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{(1/2)^{k/2}}{\\Gamma(k/2)}x^{k/2-1}e^{-x/2}.
+
+    Args:
+        df (int or array_like of ints): Degree of freedom :math:`k`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the chi-square distribution.
+
+    .. seealso::
+        :func:`numpy.random.chisquare`
+    """
+    rs = _generator.get_random_state()
+    return rs.chisquare(df, size, dtype)
+
+
+def dirichlet(alpha, size=None, dtype=float):
+    """Dirichlet distribution.
+
+    Returns an array of samples drawn from the dirichlet distribution. Its
+    probability density function is defined as
+
+    .. math::
+        f(x) = \\frac{\\Gamma(\\sum_{i=1}^K\\alpha_i)} \
+            {\\prod_{i=1}^{K}\\Gamma(\\alpha_i)} \
+            \\prod_{i=1}^Kx_i^{\\alpha_i-1}.
+
+    Args:
+        alpha (array): Parameters of the dirichlet distribution
+            :math:`\\alpha`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the dirichlet distribution.
+
+    .. seealso::
+        :func:`numpy.random.dirichlet`
+    """
+    rs = _generator.get_random_state()
+    return rs.dirichlet(alpha, size, dtype)
+
+
+def exponential(scale, size=None, dtype=float):
+    """Exponential distribution.
+
+    Returns an array of samples drawn from the exponential distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{1}{\\beta}\\exp (-\\frac{x}{\\beta}).
+
+    Args:
+        scale (float or array_like of floats): The scale parameter
+            :math:`\\beta`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the exponential distribution.
+
+    .. seealso::
+        :func:`numpy.random.exponential`
+    """
+    rs = _generator.get_random_state()
+    return rs.exponential(scale, size, dtype)
+
+
+def f(dfnum, dfden, size=None, dtype=float):
+    """F distribution.
+
+    Returns an array of samples drawn from the f distribution. Its probability
+    density function is defined as
+
+    .. math::
+        f(x) = \\frac{1}{B(\\frac{d_1}{2},\\frac{d_2}{2})} \
+            \\left(\\frac{d_1}{d_2}\\right)^{\\frac{d_1}{2}} \
+            x^{\\frac{d_1}{2}-1} \
+            \\left(1+\\frac{d_1}{d_2}x\\right) \
+            ^{-\\frac{d_1+d_2}{2}}.
+
+    Args:
+        dfnum (float or array_like of floats): Parameter of the f distribution
+            :math:`d_1`.
+        dfden (float or array_like of floats): Parameter of the f distribution
+            :math:`d_2`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the f distribution.
+
+    .. seealso::
+        :func:`numpy.random.f`
+    """
+    rs = _generator.get_random_state()
+    return rs.f(dfnum, dfden, size, dtype)
+
+
+def gamma(shape, scale=1.0, size=None, dtype=float):
+    """Gamma distribution.
+
+    Returns an array of samples drawn from the gamma distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{1}{\\Gamma(k)\\theta^k}x^{k-1}e^{-x/\\theta}.
+
+    Args:
+        shape (array): Parameter of the gamma distribution :math:`k`.
+        scale (array): Parameter of the gamma distribution :math:`\\theta`
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:cupy.ndarray: Samples drawn from the gamma distribution.
+
+    .. seealso::
+        :func:`numpy.random.gamma`
+    """
+    rs = _generator.get_random_state()
+    return rs.gamma(shape, scale, size, dtype)
+
+
+def geometric(p, size=None, dtype=int):
+    """Geometric distribution.
+
+    Returns an array of samples drawn from the geometric distribution. Its
+    probability mass function is defined as
+
+    .. math::
+        f(x) = p(1-p)^{k-1}.
+
+    Args:
+        p (float): Success probability of the geometric distribution.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.int32` and
+            :class:`numpy.int64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the geometric distribution.
+
+    .. seealso::
+        :func:`numpy.random.geometric`
+    """
+    rs = _generator.get_random_state()
+    return rs.geometric(p, size, dtype)
+
+
+def gumbel(loc=0.0, scale=1.0, size=None, dtype=float):
+    """Returns an array of samples drawn from a Gumbel distribution.
+
+    The samples are drawn from a Gumbel distribution with location ``loc``
+    and scale ``scale``.
+    Its probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{1}{\\eta} \
+           \\exp\\left\\{ - \\frac{x - \\mu}{\\eta} \\right\\} \
+           \\exp\\left[-\\exp\\left\\{-\\frac{x - \\mu}{\\eta} \
+           \\right\\}\\right],
+
+    where :math:`\\mu` is ``loc`` and :math:`\\eta` is ``scale``.
+
+    Args:
+        loc (float): The location of the mode :math:`\\mu`.
+        scale (float): The scale parameter :math:`\\eta`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the Gumbel distribution.
+
+    .. seealso::
+        :func:`numpy.random.gumbel`
+    """
+    rs = _generator.get_random_state()
+    return rs.gumbel(loc, scale, size, dtype)
+
+
+def hypergeometric(ngood, nbad, nsample, size=None, dtype=int):
+    """hypergeometric distribution.
+
+    Returns an array of samples drawn from the hypergeometric distribution. Its
+    probability mass function is defined as
+
+    .. math::
+        f(x) = \\frac{\\binom{m}{n}\\binom{N-m}{n-x}}{\\binom{N}{n}}.
+
+    Args:
+        ngood (int or array_like of ints): Parameter of the hypergeometric
+            distribution :math:`n`.
+        nbad (int or array_like of ints): Parameter of the hypergeometric
+            distribution :math:`m`.
+        nsample (int or array_like of ints): Parameter of the hypergeometric
+            distribution :math:`N`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.int32` and
+            :class:`numpy.int64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the hypergeometric distribution.
+
+    .. seealso::
+        :func:`numpy.random.hypergeometric`
+    """
+    rs = _generator.get_random_state()
+    return rs.hypergeometric(ngood, nbad, nsample, size, dtype)
+
+
+def logistic(loc=0.0, scale=1.0, size=None, dtype=float):
+    """Logistic distribution.
+
+    Returns an array of samples drawn from the logistic distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{e^{-(x-\\mu)/s}}{s(1+e^{-(x-\\mu)/s})^2}.
+
+    Args:
+        loc (float): The location of the mode :math:`\\mu`.
+        scale (float): The scale parameter :math:`s`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the logistic distribution.
+
+    .. seealso::
+        :func:`numpy.random.logistic`
+    """
+    rs = _generator.get_random_state()
+    return rs.logistic(loc, scale, size, dtype)
+
+
+def laplace(loc=0.0, scale=1.0, size=None, dtype=float):
+    """Laplace distribution.
+
+    Returns an array of samples drawn from the laplace distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{1}{2b}\\exp\\left(-\\frac{|x-\\mu|}{b}\\right).
+
+    Args:
+        loc (float): The location of the mode :math:`\\mu`.
+        scale (float): The scale parameter :math:`b`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the laplace distribution.
+
+    .. seealso::
+        :func:`numpy.random.laplace`
+    """
+    rs = _generator.get_random_state()
+    return rs.laplace(loc, scale, size, dtype)
+
+
+def lognormal(mean=0.0, sigma=1.0, size=None, dtype=float):
+    """Returns an array of samples drawn from a log normal distribution.
+
+    The samples are natural log of samples drawn from a normal distribution
+    with mean ``mean`` and deviation ``sigma``.
+
+    Args:
+        mean (float): Mean of the normal distribution.
+        sigma (float): Standard deviation of the normal distribution.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the log normal distribution.
+
+    .. seealso:: :func:`numpy.random.lognormal`
+
+    """
+    rs = _generator.get_random_state()
+    return rs.lognormal(mean, sigma, size=size, dtype=dtype)
+
+
+def logseries(p, size=None, dtype=int):
+    """Log series distribution.
+
+    Returns an array of samples drawn from the log series distribution. Its
+    probability mass function is defined as
+
+    .. math::
+       f(x) = \\frac{-p^x}{x\\ln(1-p)}.
+
+    Args:
+        p (float): Parameter of the log series distribution :math:`p`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.int32` and
+            :class:`numpy.int64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the log series distribution.
+
+    .. seealso:: :func:`numpy.random.logseries`
+
+    """
+    rs = _generator.get_random_state()
+    return rs.logseries(p, size=size, dtype=dtype)
+
+
+def negative_binomial(n, p, size=None, dtype=int):
+    """Negative binomial distribution.
+
+    Returns an array of samples drawn from the negative binomial distribution.
+    Its probability mass function is defined as
+
+    .. math::
+        f(x) = \\binom{x + n - 1}{n - 1}p^n(1-p)^{x}.
+
+    Args:
+        n (int): Parameter of the negative binomial distribution :math:`n`.
+        p (float): Parameter of the negative binomial distribution :math:`p`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.int32` and
+            :class:`numpy.int64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the negative binomial distribution.
+
+    .. seealso::
+        :func:`numpy.random.negative_binomial`
+    """
+    rs = _generator.get_random_state()
+    return rs.negative_binomial(n, p, size=size, dtype=dtype)
+
+
+def multivariate_normal(mean, cov, size=None, check_valid='ignore',
+                        tol=1e-08, method='cholesky', dtype=float):
+    """Multivariate normal distribution.
+
+    Returns an array of samples drawn from the multivariate normal
+    distribution. Its probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{1}{(2\\pi|\\Sigma|)^(n/2)} \
+           \\exp\\left(-\\frac{1}{2} \
+           (x-\\mu)^{\\top}\\Sigma^{-1}(x-\\mu)\\right).
+
+    Args:
+        mean (1-D array_like, of length N): Mean of the multivariate normal
+            distribution :math:`\\mu`.
+        cov (2-D array_like, of shape (N, N)): Covariance matrix
+            :math:`\\Sigma` of the multivariate normal distribution. It must be
+            symmetric and positive-semidefinite for proper sampling.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        check_valid ('warn', 'raise', 'ignore'): Behavior when the covariance
+            matrix is not positive semidefinite.
+        tol (float): Tolerance when checking the singular values in
+            covariance matrix.
+        method : { 'cholesky', 'eigh', 'svd'}, optional
+            The cov input is used to compute a factor matrix A such that
+            ``A @ A.T = cov``. This argument is used to select the method
+            used to compute the factor matrix A. The default method 'cholesky'
+            is the fastest, while 'svd' is the slowest but more robust than
+            the fastest method. The method `eigh` uses eigen decomposition to
+            compute A and is faster than svd but slower than cholesky.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the multivariate normal distribution.
+
+    .. note:: Default `method` is set to fastest, 'cholesky', unlike numpy
+        which defaults to 'svd'. Cholesky decomposition in CuPy will fail
+        silently if the input covariance matrix is not positive definite and
+        give invalid results, unlike in numpy, where an invalid covariance
+        matrix will raise an exception. Setting `check_valid` to 'raise' will
+        replicate numpy behavior by checking the input, but will also force
+        device synchronization. If validity of input is unknown, setting
+        `method` to 'einh' or 'svd' and `check_valid` to 'warn' will use
+        cholesky decomposition for positive definite matrices, and fallback to
+        the specified `method` for other matrices (i.e., not positive
+        semi-definite), and will warn if decomposition is suspect.
+
+    .. seealso:: :func:`numpy.random.multivariate_normal`
+
+    """
+    _util.experimental('cupy.random.multivariate_normal')
+    rs = _generator.get_random_state()
+    return rs.multivariate_normal(
+        mean, cov, size, check_valid, tol, method, dtype)
+
+
+def normal(loc=0.0, scale=1.0, size=None, dtype=float):
+    """Returns an array of normally distributed samples.
+
+    Args:
+        loc (float or array_like of floats): Mean of the normal distribution.
+        scale (float or array_like of floats):
+            Standard deviation of the normal distribution.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Normally distributed samples.
+
+    .. seealso:: :func:`numpy.random.normal`
+
+    """
+    rs = _generator.get_random_state()
+    return rs.normal(loc, scale, size, dtype)
+
+
+def pareto(a, size=None, dtype=float):
+    """Pareto II or Lomax distribution.
+
+    Returns an array of samples drawn from the Pareto II distribution. Its
+    probability density function is defined as
+
+    .. math::
+        f(x) = \\alpha(1+x)^{-(\\alpha+1)}.
+
+    Args:
+        a (float or array_like of floats): Parameter of the Pareto II
+            distribution :math:`\\alpha`.
+        size (int or tuple of ints): The shape of the array. If ``None``, this
+            function generate an array whose shape is `a.shape`.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the Pareto II distribution.
+
+    .. seealso:: :func:`numpy.random.pareto`
+    """
+    rs = _generator.get_random_state()
+    return rs.pareto(a, size, dtype)
+
+
+def noncentral_chisquare(df, nonc, size=None, dtype=float):
+    """Noncentral chisquare distribution.
+
+    Returns an array of samples drawn from the noncentral chisquare
+    distribution. Its probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{1}{2}e^{-(x+\\lambda)/2} \\
+        \\left(\\frac{x}{\\lambda}\\right)^{k/4 - 1/2} \\
+        I_{k/2 - 1}(\\sqrt{\\lambda x}),
+
+    where :math:`I` is the modified Bessel function of the first kind.
+
+    Args:
+        df (float): Parameter of the noncentral chisquare distribution
+            :math:`k`.
+        nonc (float): Parameter of the noncentral chisquare distribution
+            :math:`\\lambda`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the noncentral chisquare distribution.
+
+    .. seealso::
+        :func:`numpy.random.noncentral_chisquare`
+    """
+    rs = _generator.get_random_state()
+    return rs.noncentral_chisquare(df, nonc, size=size, dtype=dtype)
+
+
+def noncentral_f(dfnum, dfden, nonc, size=None, dtype=float):
+    """Noncentral F distribution.
+
+    Returns an array of samples drawn from the noncentral F
+    distribution.
+
+    Reference: https://en.wikipedia.org/wiki/Noncentral_F-distribution
+
+    Args:
+        dfnum (float): Parameter of the noncentral F distribution.
+        dfden (float): Parameter of the noncentral F distribution.
+        nonc (float): Parameter of the noncentral F distribution.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the noncentral F distribution.
+
+    .. seealso::
+        :func:`numpy.random.noncentral_f`
+    """
+    rs = _generator.get_random_state()
+    return rs.noncentral_f(dfnum, dfden, nonc, size=size, dtype=dtype)
+
+
+def poisson(lam=1.0, size=None, dtype=int):
+    """Poisson distribution.
+
+    Returns an array of samples drawn from the poisson distribution. Its
+    probability mass function is defined as
+
+    .. math::
+        f(x) = \\frac{\\lambda^xe^{-\\lambda}}{k!}.
+
+    Args:
+        lam (array_like of floats): Parameter of the poisson distribution
+            :math:`\\lambda`.
+        size (int or tuple of ints): The shape of the array. If ``None``, this
+            function generate an array whose shape is `lam.shape`.
+        dtype: Data type specifier. Only :class:`numpy.int32` and
+            :class:`numpy.int64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the poisson distribution.
+
+    .. seealso:: :func:`numpy.random.poisson`
+    """
+    rs = _generator.get_random_state()
+    return rs.poisson(lam, size, dtype)
+
+
+def power(a, size=None, dtype=float):
+    """Power distribution.
+
+    Returns an array of samples drawn from the power distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = ax^{a-1}.
+
+    Args:
+        a (float): Parameter of the power distribution :math:`a`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the power distribution.
+
+    .. seealso::
+        :func:`numpy.random.power`
+    """
+    rs = _generator.get_random_state()
+    return rs.power(a, size, dtype)
+
+
+def rayleigh(scale=1.0, size=None, dtype=float):
+    """Rayleigh distribution.
+
+    Returns an array of samples drawn from the rayleigh distribution.
+    Its probability density function is defined as
+
+      .. math::
+         f(x) = \\frac{x}{\\sigma^2}e^{\\frac{-x^2}{2-\\sigma^2}}, x \\ge 0.
+
+    Args:
+        scale (array): Parameter of the rayleigh distribution :math:`\\sigma`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the rayleigh distribution.
+
+    .. seealso:: :func:`numpy.random.rayleigh`
+    """
+    rs = _generator.get_random_state()
+    return rs.rayleigh(scale, size, dtype)
+
+
+def standard_cauchy(size=None, dtype=float):
+    """Standard cauchy distribution.
+
+    Returns an array of samples drawn from the standard cauchy distribution.
+    Its probability density function is defined as
+
+      .. math::
+         f(x) = \\frac{1}{\\pi(1+x^2)}.
+
+    Args:
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the standard cauchy distribution.
+
+    .. seealso:: :func:`numpy.random.standard_cauchy`
+    """
+    rs = _generator.get_random_state()
+    return rs.standard_cauchy(size, dtype)
+
+
+def standard_exponential(size=None, dtype=float):
+    """Standard exponential distribution.
+
+    Returns an array of samples drawn from the standard exponential
+    distribution. Its probability density function is defined as
+
+      .. math::
+         f(x) = e^{-x}.
+
+    Args:
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the standard exponential distribution.
+
+    .. seealso:: :func:`numpy.random.standard_exponential`
+    """
+    rs = _generator.get_random_state()
+    return rs.standard_exponential(size, dtype)
+
+
+def standard_gamma(shape, size=None, dtype=float):
+    """Standard gamma distribution.
+
+    Returns an array of samples drawn from the standard gamma distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{1}{\\Gamma(k)}x^{k-1}e^{-x}.
+
+    Args:
+        shape (array): Parameter of the gamma distribution :math:`k`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the standard gamma distribution.
+
+    .. seealso::
+        :func:`numpy.random.standard_gamma`
+    """
+    rs = _generator.get_random_state()
+    return rs.standard_gamma(shape, size, dtype)
+
+
+def standard_normal(size=None, dtype=float):
+    """Returns an array of samples drawn from the standard normal distribution.
+
+    This is a variant of :func:`cupy.random.randn`.
+
+    Args:
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the standard normal distribution.
+
+    .. seealso:: :func:`numpy.random.standard_normal`
+
+    """
+    rs = _generator.get_random_state()
+    return rs.standard_normal(size, dtype)
+
+
+def standard_t(df, size=None, dtype=float):
+    """Standard Student's t distribution.
+
+    Returns an array of samples drawn from the standard Student's t
+    distribution. Its probability density function is defined as
+
+    .. math::
+        f(x) = \\frac{\\Gamma(\\frac{\\nu+1}{2})} \
+            {\\sqrt{\\nu\\pi}\\Gamma(\\frac{\\nu}{2})} \
+            \\left(1 + \\frac{x^2}{\\nu} \\right)^{-(\\frac{\\nu+1}{2})}.
+
+    Args:
+        df (float or array_like of floats): Degree of freedom :math:`\\nu`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the standard Student's t distribution.
+
+    .. seealso::
+        :func:`numpy.random.standard_t`
+    """
+    rs = _generator.get_random_state()
+    return rs.standard_t(df, size, dtype)
+
+
+def triangular(left, mode, right, size=None, dtype=float):
+    """Triangular distribution.
+
+    Returns an array of samples drawn from the triangular distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\begin{cases}
+            \\frac{2(x-l)}{(r-l)(m-l)} & \\text{for } l \\leq x \\leq m, \\\\
+            \\frac{2(r-x)}{(r-l)(r-m)} & \\text{for } m \\leq x \\leq r, \\\\
+            0 & \\text{otherwise}.
+          \\end{cases}
+
+    Args:
+        left (float): Lower limit :math:`l`.
+        mode (float): The value where the peak of the distribution occurs.
+            :math:`m`.
+        right (float): Higher Limit :math:`r`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the triangular distribution.
+
+    .. seealso::
+        :func:`numpy.random.triangular`
+    """
+    rs = _generator.get_random_state()
+    return rs.triangular(left, mode, right, size, dtype)
+
+
+def uniform(low=0.0, high=1.0, size=None, dtype=float):
+    """Returns an array of uniformly-distributed samples over an interval.
+
+    Samples are drawn from a uniform distribution over the half-open interval
+    ``[low, high)``. The samples may contain the ``high`` limit due to
+    floating-point rounding.
+
+    Args:
+        low (float): Lower end of the interval.
+        high (float): Upper end of the interval.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the uniform distribution.
+
+    .. seealso:: :func:`numpy.random.uniform`
+
+    """
+    rs = _generator.get_random_state()
+    return rs.uniform(low, high, size=size, dtype=dtype)
+
+
+def vonmises(mu, kappa, size=None, dtype=float):
+    """von Mises distribution.
+
+    Returns an array of samples drawn from the von Mises distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\frac{e^{\\kappa \\cos(x-\\mu)}}{2\\pi I_0(\\kappa)}.
+
+    Args:
+        mu (float): Parameter of the von Mises distribution :math:`\\mu`.
+        kappa (float): Parameter of the von Mises distribution :math:`\\kappa`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the von Mises distribution.
+
+    .. seealso::
+        :func:`numpy.random.vonmises`
+    """
+    rs = _generator.get_random_state()
+    return rs.vonmises(mu, kappa, size=size, dtype=dtype)
+
+
+def wald(mean, scale, size=None, dtype=float):
+    """Wald distribution.
+
+    Returns an array of samples drawn from the Wald distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = \\sqrt{\\frac{\\lambda}{2\\pi x^3}}\\
+           e^{\\frac{-\\lambda(x-\\mu)^2}{2\\mu^2x}}.
+
+    Args:
+        mean (float): Parameter of the wald distribution :math:`\\mu`.
+        scale (float): Parameter of the wald distribution :math:`\\lambda`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the wald distribution.
+
+    .. seealso::
+        :func:`numpy.random.wald`
+    """
+    rs = _generator.get_random_state()
+    return rs.wald(mean, scale, size, dtype)
+
+
+def weibull(a, size=None, dtype=float):
+    """weibull distribution.
+
+    Returns an array of samples drawn from the weibull distribution. Its
+    probability density function is defined as
+
+    .. math::
+       f(x) = ax^{(a-1)}e^{-x^a}.
+
+    Args:
+        a (float): Parameter of the weibull distribution :math:`a`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the weibull distribution.
+
+    .. seealso::
+        :func:`numpy.random.weibull`
+    """
+    rs = _generator.get_random_state()
+    return rs.weibull(a, size=size, dtype=dtype)
+
+
+def zipf(a, size=None, dtype=int):
+    """Zipf distribution.
+
+    Returns an array of samples drawn from the Zipf distribution. Its
+    probability mass function is defined as
+
+    .. math::
+        f(x) = \\frac{x^{-a}}{ \\zeta (a)},
+
+    where :math:`\\zeta` is the Riemann Zeta function.
+
+    Args:
+        a (float): Parameter of the beta distribution :math:`a`.
+        size (int or tuple of ints): The shape of the array. If ``None``, a
+            zero-dimensional array is generated.
+        dtype: Data type specifier. Only :class:`numpy.int32` and
+            :class:`numpy.int64` types are allowed.
+
+    Returns:
+        cupy.ndarray: Samples drawn from the Zipf distribution.
+
+    .. seealso::
+        :func:`numpy.random.zipf`
+    """
+    rs = _generator.get_random_state()
+    return rs.zipf(a, size=size, dtype=dtype)
diff --git a/cupy/random/_generator.py b/cupy/random/_generator.py
new file mode 100644
index 0000000..e59ebed
--- /dev/null
+++ b/cupy/random/_generator.py
@@ -0,0 +1,1309 @@
+import atexit
+import binascii
+import functools
+import hashlib
+import operator
+import os
+import time
+
+import numpy
+import warnings
+from numpy.linalg import LinAlgError
+
+import cupy
+from cupy import _core
+from cupy import cuda
+from cupy.cuda import curand
+from cupy.cuda import device
+from cupy.random import _kernels
+from cupy import _util
+
+import cupyx
+
+
+_UINT32_MAX = 0xffffffff
+_UINT64_MAX = 0xffffffffffffffff
+
+
+class RandomState(object):
+
+    """Portable container of a pseudo-random number generator.
+
+    An instance of this class holds the state of a random number generator. The
+    state is available only on the device which has been current at the
+    initialization of the instance.
+
+    Functions of :mod:`cupy.random` use global instances of this class.
+    Different instances are used for different devices. The global state for
+    the current device can be obtained by the
+    :func:`cupy.random.get_random_state` function.
+
+    Args:
+        seed (None or int): Seed of the random number generator. See the
+            :meth:`~cupy.random.RandomState.seed` method for detail.
+        method (int): Method of the random number generator. Following values
+            are available::
+
+               cupy.cuda.curand.CURAND_RNG_PSEUDO_DEFAULT
+               cupy.cuda.curand.CURAND_RNG_PSEUDO_XORWOW
+               cupy.cuda.curand.CURAND_RNG_PSEUDO_MRG32K3A
+               cupy.cuda.curand.CURAND_RNG_PSEUDO_MTGP32
+               cupy.cuda.curand.CURAND_RNG_PSEUDO_MT19937
+               cupy.cuda.curand.CURAND_RNG_PSEUDO_PHILOX4_32_10
+
+    """
+
+    def __init__(self, seed=None, method=curand.CURAND_RNG_PSEUDO_DEFAULT):
+        self._generator = curand.createGenerator(method)
+        self.method = method
+        self.seed(seed)
+
+    def __del__(self, is_shutting_down=_util.is_shutting_down):
+        # When createGenerator raises an error, _generator is not initialized
+        if is_shutting_down():
+            return
+        if hasattr(self, '_generator'):
+            curand.destroyGenerator(self._generator)
+
+    def _update_seed(self, size):
+        self._rk_seed = (self._rk_seed + size) % _UINT64_MAX
+
+    def _generate_normal(self, func, size, dtype, *args):
+        # curand functions below don't support odd size.
+        # * curand.generateNormal
+        # * curand.generateNormalDouble
+        # * curand.generateLogNormal
+        # * curand.generateLogNormalDouble
+        size = _core.get_size(size)
+        element_size = _core.internal.prod(size)
+        if element_size % 2 == 0:
+            out = cupy.empty(size, dtype=dtype)
+            func(self._generator, out.data.ptr, out.size, *args)
+            return out
+        else:
+            out = cupy.empty((element_size + 1,), dtype=dtype)
+            func(self._generator, out.data.ptr, out.size, *args)
+            return out[:element_size].reshape(size)
+
+    # NumPy compatible functions
+
+    def beta(self, a, b, size=None, dtype=float):
+        """Returns an array of samples drawn from the beta distribution.
+
+        .. seealso::
+            - :func:`cupy.random.beta` for full documentation
+            - :meth:`numpy.random.RandomState.beta`
+        """
+        a, b = cupy.asarray(a), cupy.asarray(b)
+        if size is None:
+            size = cupy.broadcast(a, b).shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.beta_kernel(a, b, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def binomial(self, n, p, size=None, dtype=int):
+        """Returns an array of samples drawn from the binomial distribution.
+
+        .. seealso::
+            - :func:`cupy.random.binomial` for full documentation
+            - :meth:`numpy.random.RandomState.binomial`
+        """
+        n, p = cupy.asarray(n), cupy.asarray(p)
+        if size is None:
+            size = cupy.broadcast(n, p).shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.binomial_kernel(n, p, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def chisquare(self, df, size=None, dtype=float):
+        """Returns an array of samples drawn from the chi-square distribution.
+
+        .. seealso::
+            - :func:`cupy.random.chisquare` for full documentation
+            - :meth:`numpy.random.RandomState.chisquare`
+        """
+        df = cupy.asarray(df)
+        if size is None:
+            size = df.shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.chisquare_kernel(df, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def dirichlet(self, alpha, size=None, dtype=float):
+        """Returns an array of samples drawn from the dirichlet distribution.
+
+        .. seealso::
+            - :func:`cupy.random.dirichlet` for full documentation
+            - :meth:`numpy.random.RandomState.dirichlet`
+        """
+        alpha = cupy.asarray(alpha)
+        if size is None:
+            size = alpha.shape
+        elif isinstance(size, (int, cupy.integer)):
+            size = (size,) + alpha.shape
+        else:
+            size += alpha.shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.standard_gamma_kernel(alpha, self._rk_seed, y)
+        y /= y.sum(axis=-1, keepdims=True)
+        self._update_seed(y.size)
+        return y
+
+    def exponential(self, scale=1.0, size=None, dtype=float):
+        """Returns an array of samples drawn from a exponential distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.exponential` for full documentation
+            - :meth:`numpy.random.RandomState.exponential`
+        """
+        scale = cupy.asarray(scale, dtype)
+        if (scale < 0).any():  # synchronize!
+            raise ValueError('scale < 0')
+        if size is None:
+            size = scale.shape
+        x = self.standard_exponential(size, dtype)
+        x *= scale
+        return x
+
+    def f(self, dfnum, dfden, size=None, dtype=float):
+        """Returns an array of samples drawn from the f distribution.
+
+        .. seealso::
+            - :func:`cupy.random.f` for full documentation
+            - :meth:`numpy.random.RandomState.f`
+        """
+        dfnum, dfden = cupy.asarray(dfnum), cupy.asarray(dfden)
+        if size is None:
+            size = cupy.broadcast(dfnum, dfden).shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.f_kernel(dfnum, dfden, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def gamma(self, shape, scale=1.0, size=None, dtype=float):
+        """Returns an array of samples drawn from a gamma distribution.
+
+        .. seealso::
+            - :func:`cupy.random.gamma` for full documentation
+            - :meth:`numpy.random.RandomState.gamma`
+        """
+        shape, scale = cupy.asarray(shape), cupy.asarray(scale)
+        if size is None:
+            size = cupy.broadcast(shape, scale).shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.standard_gamma_kernel(shape, self._rk_seed, y)
+        y *= scale
+        self._update_seed(y.size)
+        return y
+
+    def geometric(self, p, size=None, dtype=int):
+        """Returns an array of samples drawn from the geometric distribution.
+
+        .. seealso::
+            - :func:`cupy.random.geometric` for full documentation
+            - :meth:`numpy.random.RandomState.geometric`
+        """
+        p = cupy.asarray(p)
+        if size is None:
+            size = p.shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.geometric_kernel(p, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def hypergeometric(self, ngood, nbad, nsample, size=None, dtype=int):
+        """Returns an array of samples drawn from the hypergeometric distribution.
+
+        .. seealso::
+            - :func:`cupy.random.hypergeometric` for full documentation
+            - :meth:`numpy.random.RandomState.hypergeometric`
+        """  # NOQA
+        ngood, nbad, nsample = \
+            cupy.asarray(ngood), cupy.asarray(nbad), cupy.asarray(nsample)
+        if size is None:
+            size = cupy.broadcast(ngood, nbad, nsample).shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.hypergeometric_kernel(ngood, nbad, nsample, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    _laplace_kernel = _core.ElementwiseKernel(
+        'T x, T loc, T scale', 'T y',
+        'y = loc + scale * ((x <= 0.5) ? log(x + x): -log(x + x - 1.0))',
+        'cupy_laplace_kernel')
+
+    def laplace(self, loc=0.0, scale=1.0, size=None, dtype=float):
+        """Returns an array of samples drawn from the laplace distribution.
+
+        .. seealso::
+            - :func:`cupy.random.laplace` for full documentation
+            - :meth:`numpy.random.RandomState.laplace`
+        """
+        loc = cupy.asarray(loc, dtype)
+        scale = cupy.asarray(scale, dtype)
+        if size is None:
+            size = cupy.broadcast(loc, scale).shape
+        x = self._random_sample_raw(size, dtype)
+        RandomState._laplace_kernel(x, loc, scale, x)
+        return x
+
+    def logistic(self, loc=0.0, scale=1.0, size=None, dtype=float):
+        """Returns an array of samples drawn from the logistic distribution.
+
+        .. seealso::
+            - :func:`cupy.random.logistic` for full documentation
+            - :meth:`numpy.random.RandomState.logistic`
+        """
+        loc, scale = cupy.asarray(loc), cupy.asarray(scale)
+        if size is None:
+            size = cupy.broadcast(loc, scale).shape
+        x = cupy.empty(shape=size, dtype=dtype)
+        _kernels.open_uniform_kernel(self._rk_seed, x)
+        self._update_seed(x.size)
+        x = (1.0 - x) / x
+        cupy.log(x, out=x)
+        cupy.multiply(x, scale, out=x)
+        cupy.add(x, loc, out=x)
+        return x
+
+    def lognormal(self, mean=0.0, sigma=1.0, size=None, dtype=float):
+        """Returns an array of samples drawn from a log normal distribution.
+
+        .. seealso::
+            - :func:`cupy.random.lognormal` for full documentation
+            - :meth:`numpy.random.RandomState.lognormal`
+
+        """
+        if any(isinstance(arg, cupy.ndarray) for arg in (mean, sigma)):
+            x = self.normal(mean, sigma, size, dtype)
+            cupy.exp(x, out=x)
+            return x
+        if size is None:
+            size = ()
+        dtype = _check_and_get_dtype(dtype)
+        if dtype.char == 'f':
+            func = curand.generateLogNormal
+        else:
+            func = curand.generateLogNormalDouble
+        return self._generate_normal(func, size, dtype, mean, sigma)
+
+    def logseries(self, p, size=None, dtype=int):
+        """Returns an array of samples drawn from a log series distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.logseries` for full documentation
+            - :meth:`numpy.random.RandomState.logseries`
+
+        """
+        p = cupy.asarray(p)
+        if cupy.any(p <= 0):  # synchronize!
+            raise ValueError('p <= 0.0')
+        if cupy.any(p >= 1):  # synchronize!
+            raise ValueError('p >= 1.0')
+        if size is None:
+            size = p.shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.logseries_kernel(p, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def multivariate_normal(self, mean, cov, size=None, check_valid='ignore',
+                            tol=1e-08, method='cholesky', dtype=float):
+        """Returns an array of samples drawn from the multivariate normal
+        distribution.
+
+        .. warning::
+            This function calls one or more cuSOLVER routine(s) which may yield
+            invalid results if input conditions are not met.
+            To detect these invalid results, you can set the `linalg`
+            configuration to a value that is not `ignore` in
+            :func:`cupyx.errstate` or :func:`cupyx.seterr`.
+
+        .. seealso::
+            - :func:`cupy.random.multivariate_normal` for full documentation
+            - :meth:`numpy.random.RandomState.multivariate_normal`
+        """
+        _util.experimental('cupy.random.RandomState.multivariate_normal')
+        mean = cupy.asarray(mean, dtype=dtype)
+        cov = cupy.asarray(cov, dtype=dtype)
+        if size is None:
+            shape = []
+        elif isinstance(size, (int, cupy.integer)):
+            shape = [size]
+        else:
+            shape = size
+
+        if len(mean.shape) != 1:
+            raise ValueError('mean must be 1 dimensional')
+        if (len(cov.shape) != 2) or (cov.shape[0] != cov.shape[1]):
+            raise ValueError('cov must be 2 dimensional and square')
+        if mean.shape[0] != cov.shape[0]:
+            raise ValueError('mean and cov must have same length')
+
+        final_shape = list(shape[:])
+        final_shape.append(mean.shape[0])
+
+        if method not in {'eigh', 'svd', 'cholesky'}:
+            raise ValueError(
+                "method must be one of {'eigh', 'svd', 'cholesky'}")
+
+        if check_valid != 'ignore':
+            if check_valid != 'warn' and check_valid != 'raise':
+                raise ValueError(
+                    "check_valid must equal 'warn', 'raise', or 'ignore'")
+
+        if check_valid == 'warn':
+            with cupyx.errstate(linalg='raise'):
+                try:
+                    decomp = cupy.linalg.cholesky(cov)
+                except LinAlgError:
+                    with cupyx.errstate(linalg='ignore'):
+                        if method != 'cholesky':
+                            if method == 'eigh':
+                                (s, u) = cupy.linalg.eigh(cov)
+                                psd = not cupy.any(s < -tol)
+                            if method == 'svd':
+                                (u, s, vh) = cupy.linalg.svd(cov)
+                                psd = cupy.allclose(cupy.dot(vh.T * s, vh),
+                                                    cov, rtol=tol, atol=tol)
+                            decomp = u * cupy.sqrt(cupy.abs(s))
+                            if not psd:
+                                warnings.warn("covariance is not positive-" +
+                                              "semidefinite, output may be " +
+                                              "invalid.", RuntimeWarning)
+
+                        else:
+                            warnings.warn("covariance is not positive-" +
+                                          "semidefinite, output *is* " +
+                                          "invalid.", RuntimeWarning)
+                            decomp = cupy.linalg.cholesky(cov)
+
+        else:
+            with cupyx.errstate(linalg=check_valid):
+                try:
+                    if method == 'cholesky':
+                        decomp = cupy.linalg.cholesky(cov)
+                    elif method == 'eigh':
+                        (s, u) = cupy.linalg.eigh(cov)
+                        decomp = u * cupy.sqrt(cupy.abs(s))
+                    elif method == 'svd':
+                        (u, s, vh) = cupy.linalg.svd(cov)
+                        decomp = u * cupy.sqrt(cupy.abs(s))
+
+                except LinAlgError:
+                    raise LinAlgError("Matrix is not positive definite; if " +
+                                      "matrix is positive-semidefinite, set" +
+                                      "'check_valid' to 'warn'")
+
+        x = self.standard_normal(final_shape,
+                                 dtype=dtype).reshape(-1, mean.shape[0])
+        x = cupy.dot(decomp, x.T)
+        x = x.T
+        x += mean
+        x.shape = tuple(final_shape)
+        return x
+
+    def negative_binomial(self, n, p, size=None, dtype=int):
+        """Returns an array of samples drawn from the negative binomial distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.negative_binomial` for full documentation
+            - :meth:`numpy.random.RandomState.negative_binomial`
+        """  # NOQA
+        n = cupy.asarray(n)
+        p = cupy.asarray(p)
+        if cupy.any(n <= 0):  # synchronize!
+            raise ValueError('n <= 0')
+        if cupy.any(p < 0):  # synchronize!
+            raise ValueError('p < 0')
+        if cupy.any(p > 1):  # synchronize!
+            raise ValueError('p > 1')
+        y = self.gamma(n, (1-p)/p, size)
+        return self.poisson(y, dtype=dtype)
+
+    def normal(self, loc=0.0, scale=1.0, size=None, dtype=float):
+        """Returns an array of normally distributed samples.
+
+        .. seealso::
+            - :func:`cupy.random.normal` for full documentation
+            - :meth:`numpy.random.RandomState.normal`
+
+        """
+        dtype = _check_and_get_dtype(dtype)
+        if size is None:
+            size = cupy.broadcast(loc, scale).shape
+        if dtype.char == 'f':
+            func = curand.generateNormal
+        else:
+            func = curand.generateNormalDouble
+        if isinstance(scale, cupy.ndarray):
+            x = self._generate_normal(func, size, dtype, 0.0, 1.0)
+            cupy.multiply(x, scale, out=x)
+            cupy.add(x, loc, out=x)
+        elif isinstance(loc, cupy.ndarray):
+            x = self._generate_normal(func, size, dtype, 0.0, scale)
+            cupy.add(x, loc, out=x)
+        else:
+            x = self._generate_normal(func, size, dtype, loc, scale)
+        return x
+
+    def pareto(self, a, size=None, dtype=float):
+        """Returns an array of samples drawn from the pareto II distribution.
+
+        .. seealso::
+            - :func:`cupy.random.pareto` for full documentation
+            - :meth:`numpy.random.RandomState.pareto`
+        """
+        a = cupy.asarray(a)
+        if size is None:
+            size = a.shape
+        x = self._random_sample_raw(size, dtype)
+        cupy.log(x, out=x)
+        cupy.exp(-x/a, out=x)
+        return x - 1
+
+    def noncentral_chisquare(self, df, nonc, size=None, dtype=float):
+        """Returns an array of samples drawn from the noncentral chi-square
+        distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.noncentral_chisquare` for full documentation
+            - :meth:`numpy.random.RandomState.noncentral_chisquare`
+        """
+        df, nonc = cupy.asarray(df), cupy.asarray(nonc)
+        if cupy.any(df <= 0):  # synchronize!
+            raise ValueError('df <= 0')
+        if cupy.any(nonc < 0):  # synchronize!
+            raise ValueError('nonc < 0')
+        if size is None:
+            size = cupy.broadcast(df, nonc).shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.noncentral_chisquare_kernel(df, nonc, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def noncentral_f(self, dfnum, dfden, nonc, size=None, dtype=float):
+        """Returns an array of samples drawn from the noncentral F distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.noncentral_f` for full documentation
+            - :meth:`numpy.random.RandomState.noncentral_f`
+        """  # NOQA
+        dfnum, dfden, nonc = \
+            cupy.asarray(dfnum), cupy.asarray(dfden), cupy.asarray(nonc)
+        if cupy.any(dfnum <= 0):  # synchronize!
+            raise ValueError('dfnum <= 0')
+        if cupy.any(dfden <= 0):  # synchronize!
+            raise ValueError('dfden <= 0')
+        if cupy.any(nonc < 0):  # synchronize!
+            raise ValueError('nonc < 0')
+        if size is None:
+            size = cupy.broadcast(dfnum, dfden, nonc).shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.noncentral_f_kernel(dfnum, dfden, nonc, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def poisson(self, lam=1.0, size=None, dtype=int):
+        """Returns an array of samples drawn from the poisson distribution.
+
+        .. seealso::
+            - :func:`cupy.random.poisson` for full documentation
+            - :meth:`numpy.random.RandomState.poisson`
+        """
+        lam = cupy.asarray(lam)
+        if size is None:
+            size = lam.shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.poisson_kernel(lam, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def power(self, a, size=None, dtype=float):
+        """Returns an array of samples drawn from the power distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.power` for full documentation
+            - :meth:`numpy.random.RandomState.power`
+        """
+        a = cupy.asarray(a)
+        if cupy.any(a < 0):  # synchronize!
+            raise ValueError('a < 0')
+        if size is None:
+            size = a.shape
+        x = self.standard_exponential(size=size, dtype=dtype)
+        cupy.exp(-x, out=x)
+        cupy.add(1, -x, out=x)
+        cupy.power(x, 1./a, out=x)
+        return x
+
+    def rand(self, *size, **kwarg):
+        """Returns uniform random values over the interval ``[0, 1)``.
+
+        .. seealso::
+            - :func:`cupy.random.rand` for full documentation
+            - :meth:`numpy.random.RandomState.rand`
+
+        """
+        dtype = kwarg.pop('dtype', float)
+        if kwarg:
+            raise TypeError('rand() got unexpected keyword arguments %s'
+                            % ', '.join(kwarg.keys()))
+        return self.random_sample(size=size, dtype=dtype)
+
+    def randn(self, *size, **kwarg):
+        """Returns an array of standard normal random values.
+
+        .. seealso::
+            - :func:`cupy.random.randn` for full documentation
+            - :meth:`numpy.random.RandomState.randn`
+
+        """
+        dtype = kwarg.pop('dtype', float)
+        if kwarg:
+            raise TypeError('randn() got unexpected keyword arguments %s'
+                            % ', '.join(kwarg.keys()))
+        return self.normal(size=size, dtype=dtype)
+
+    _mod1_kernel = _core.ElementwiseKernel(
+        '', 'T x', 'x = (x == (T)1) ? 0 : x', 'cupy_random_x_mod_1')
+
+    def _random_sample_raw(self, size, dtype):
+        dtype = _check_and_get_dtype(dtype)
+        out = cupy.empty(size, dtype=dtype)
+        if dtype.char == 'f':
+            func = curand.generateUniform
+        else:
+            func = curand.generateUniformDouble
+        func(self._generator, out.data.ptr, out.size)
+        return out
+
+    def random_sample(self, size=None, dtype=float):
+        """Returns an array of random values over the interval ``[0, 1)``.
+
+        .. seealso::
+            - :func:`cupy.random.random_sample` for full documentation
+            - :meth:`numpy.random.RandomState.random_sample`
+
+        """
+        if size is None:
+            size = ()
+        out = self._random_sample_raw(size, dtype)
+        RandomState._mod1_kernel(out)
+        return out
+
+    def rayleigh(self, scale=1.0, size=None, dtype=float):
+        """Returns an array of samples drawn from a rayleigh distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.rayleigh` for full documentation
+            - :meth:`numpy.random.RandomState.rayleigh`
+        """
+        scale = cupy.asarray(scale)
+        if size is None:
+            size = scale.shape
+        if cupy.any(scale < 0):  # synchronize!
+            raise ValueError('scale < 0')
+        x = self._random_sample_raw(size, dtype)
+        x = cupy.log(x, out=x)
+        x = cupy.multiply(x, -2., out=x)
+        x = cupy.sqrt(x, out=x)
+        x = cupy.multiply(x, scale, out=x)
+        return x
+
+    _interval_upper_limit = _core.ElementwiseKernel(
+        'T max, T mx', 'T out',
+        'out = max - (mx != max ? (max - mx) % (mx + 1) : 0)',
+        'cupy_random_interval_upper_limit')
+
+    _interval_sample_modulo = _core.ElementwiseKernel(
+        'T max, T mx', 'T sample',
+        'if (mx != max) { sample %= mx + 1; }',
+        'cupy_random_interval_sample_modulo')
+
+    def _interval(self, mx, size):
+        """Generate multiple integers independently sampled uniformly from ``[0, mx]``.
+
+        Args:
+            mx (int): Upper bound of the interval
+            size (None or int or tuple): Shape of the array or the scalar
+                returned.
+        Returns:
+            int or cupy.ndarray: If ``None``, an :class:`cupy.ndarray` with
+            shape ``()`` is returned.
+            If ``int``, 1-D array of length size is returned.
+            If ``tuple``, multi-dimensional array with shape
+            ``size`` is returned.
+            Currently, only 32 bit or 64 bit integers can be sampled.
+        """  # NOQA
+        if size is None:
+            size = ()
+        elif isinstance(size, int):
+            size = size,
+
+        is_mx_scalar = numpy.isscalar(mx)
+        if is_mx_scalar:
+            if mx == 0:
+                return cupy.zeros(size, dtype=numpy.uint32)
+
+            if mx < 0:
+                raise ValueError(
+                    'mx must be non-negative (actual: {})'.format(mx))
+            elif mx <= _UINT32_MAX:
+                dtype = numpy.uint32
+                upper_limit = _UINT32_MAX - (1 << 32) % (mx + 1)
+            elif mx <= _UINT64_MAX:
+                dtype = numpy.uint64
+                upper_limit = _UINT64_MAX - (1 << 64) % (mx + 1)
+            else:
+                raise ValueError(
+                    'mx must be within uint64 range (actual: {})'.format(mx))
+        else:
+            dtype = mx.dtype
+            if dtype == cupy.int32 or dtype == cupy.uint32:
+                dtype = numpy.uint32
+                mx = mx.astype(dtype, copy=False)
+                upper_limit = self._interval_upper_limit(_UINT32_MAX, mx)
+            elif dtype == cupy.int64 or dtype == cupy.uint64:
+                dtype = numpy.uint64
+                mx = mx.astype(dtype, copy=False)
+                upper_limit = self._interval_upper_limit(_UINT64_MAX, mx)
+            else:
+                raise ValueError(
+                    'dtype must be integer, got: {}'.format(dtype))
+
+        n_sample = functools.reduce(operator.mul, size, 1)
+        if n_sample == 0:
+            return cupy.empty(size, dtype=dtype)
+        sample = self._curand_generate(n_sample, dtype)
+
+        if is_mx_scalar:
+            mx1 = mx + 1
+            if mx1 == 1 << (mx1.bit_length() - 1):
+                mask = (1 << mx.bit_length()) - 1
+                sample &= mask
+                return sample.reshape(size)
+
+        # Get index of samples that exceed the upper limit
+        ng_indices = self._get_indices(sample, upper_limit, False)
+        n_ng = ng_indices.size
+
+        if n_ng > 0 and not numpy.isscalar(mx):
+            upper_limit = upper_limit[ng_indices]
+
+        while n_ng > 0:
+            n_supplement = (max(n_ng * 2, 1024)
+                            if is_mx_scalar else upper_limit.size)
+            supplement = self._curand_generate(n_supplement, dtype)
+
+            # Get index of supplements that are within the upper limit
+            ok_indices = self._get_indices(supplement, upper_limit, True)
+            n_ok = ok_indices.size
+
+            # Replace the values that exceed the upper limit
+            if n_ok >= n_ng:
+                sample[ng_indices] = supplement[ok_indices[:n_ng]]
+                n_ng = 0
+            else:
+                sample[ng_indices[:n_ok]] = supplement[ok_indices]
+                ng_indices = ng_indices[n_ok:]
+                if not is_mx_scalar:
+                    upper_limit = upper_limit[n_ok:]
+                n_ng -= n_ok
+
+        if is_mx_scalar:
+            sample %= mx1
+        elif dtype == cupy.uint32:
+            sample = self._interval_sample_modulo(_UINT32_MAX, mx, sample)
+        else:  # dtype == cupy.uint64
+            sample = self._interval_sample_modulo(_UINT64_MAX, mx, sample)
+
+        return sample.reshape(size)
+
+    def _curand_generate(self, num, dtype):
+        sample = cupy.empty((num,), dtype=dtype)
+        # Call 32-bit RNG to fill 32-bit or 64-bit `sample`
+        size32 = sample.view(dtype=numpy.uint32).size
+        curand.generate(self._generator, sample.data.ptr, size32)
+        return sample
+
+    def _get_indices(self, sample, upper_limit, cond):
+        dtype = numpy.uint32 if sample.size < 2**32 else numpy.uint64
+        flags = (sample <= upper_limit) if cond else (sample > upper_limit)
+        csum = cupy.cumsum(flags, dtype=dtype)
+        del flags
+        indices = cupy.empty((int(csum[-1]),), dtype=dtype)
+        self._kernel_get_indices(csum, indices, size=csum.size)
+        return indices
+
+    _kernel_get_indices = _core.ElementwiseKernel(
+        'raw U csum', 'raw U indices',
+        '''
+        int j = 0;
+        if (i > 0) { j = csum[i-1]; }
+        if (csum[i] > j) { indices[j] = i; }
+        ''',
+        'cupy_get_indices')
+
+    def seed(self, seed=None):
+        """Resets the state of the random number generator with a seed.
+
+        .. seealso::
+            - :func:`cupy.random.seed` for full documentation
+            - :meth:`numpy.random.RandomState.seed`
+
+        """
+        if seed is None:
+            try:
+                seed_str = binascii.hexlify(os.urandom(8))
+                seed = int(seed_str, 16)
+            except NotImplementedError:
+                seed = (time.time() * 1000000) % _UINT64_MAX
+        else:
+            if isinstance(seed, numpy.ndarray):
+                seed = int(hashlib.md5(seed).hexdigest()[:16], 16)
+            else:
+                seed = int(
+                    numpy.asarray(seed).astype(numpy.uint64, casting='safe'))
+
+        curand.setPseudoRandomGeneratorSeed(self._generator, seed)
+        if (self.method not in (curand.CURAND_RNG_PSEUDO_MT19937,
+                                curand.CURAND_RNG_PSEUDO_MTGP32)):
+            curand.setGeneratorOffset(self._generator, 0)
+
+        self._rk_seed = seed
+
+    def standard_cauchy(self, size=None, dtype=float):
+        """Returns an array of samples drawn from the standard cauchy distribution.
+
+        .. seealso::
+            - :func:`cupy.random.standard_cauchy` for full documentation
+            - :meth:`numpy.random.RandomState.standard_cauchy`
+        """  # NOQA
+        x = self.uniform(size=size, dtype=dtype)
+        return cupy.tan(cupy.pi * (x - 0.5))
+
+    def standard_exponential(self, size=None, dtype=float):
+        """Returns an array of samples drawn from the standard exp distribution.
+
+         .. seealso::
+            - :func:`cupy.random.standard_exponential` for full documentation
+            - :meth:`numpy.random.RandomState.standard_exponential`
+        """  # NOQA
+        if size is None:
+            size = ()
+        x = self._random_sample_raw(size, dtype)
+        return -cupy.log(x, out=x)
+
+    def standard_gamma(self, shape, size=None, dtype=float):
+        """Returns an array of samples drawn from a standard gamma distribution.
+
+        .. seealso::
+            - :func:`cupy.random.standard_gamma` for full documentation
+            - :meth:`numpy.random.RandomState.standard_gamma`
+        """  # NOQA
+        shape = cupy.asarray(shape)
+        if size is None:
+            size = shape.shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.standard_gamma_kernel(shape, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def standard_normal(self, size=None, dtype=float):
+        """Returns samples drawn from the standard normal distribution.
+
+        .. seealso::
+            - :func:`cupy.random.standard_normal` for full documentation
+            - :meth:`numpy.random.RandomState.standard_normal`
+
+        """
+        return self.normal(size=size, dtype=dtype)
+
+    def standard_t(self, df, size=None, dtype=float):
+        """Returns an array of samples drawn from the standard t distribution.
+
+        .. seealso::
+            - :func:`cupy.random.standard_t` for full documentation
+            - :meth:`numpy.random.RandomState.standard_t`
+        """
+        df = cupy.asarray(df)
+        if size is None:
+            size = df.shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.standard_t_kernel(df, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def tomaxint(self, size=None):
+        """Draws integers between 0 and max integer inclusive.
+
+        Return a sample of uniformly distributed random integers in the
+        interval [0, ``np.iinfo(np.int_).max``]. The `np.int_` type translates
+        to the C long integer type and its precision is platform dependent.
+
+        Args:
+            size (int or tuple of ints): Output shape.
+
+        Returns:
+            cupy.ndarray: Drawn samples.
+
+        .. seealso::
+            :meth:`numpy.random.RandomState.tomaxint`
+
+        """
+        if size is None:
+            size = ()
+        sample = cupy.empty(size, dtype=cupy.int_)
+        # cupy.random only uses int32 random generator
+        size_in_int = sample.dtype.itemsize // 4
+        curand.generate(
+            self._generator, sample.data.ptr, sample.size * size_in_int)
+
+        # Disable sign bit
+        sample &= cupy.iinfo(cupy.int_).max
+        return sample
+
+    _triangular_kernel = _core.ElementwiseKernel(
+        'L left, M mode, R right', 'T x',
+        """
+        T base, leftbase, ratio, leftprod, rightprod;
+
+        base = right - left;
+        leftbase = mode - left;
+        ratio = leftbase / base;
+        leftprod = leftbase*base;
+        rightprod = (right - mode)*base;
+
+        if (x <= ratio)
+        {
+            x = left + sqrt(x*leftprod);
+        } else
+        {
+            x = right - sqrt((1.0 - x) * rightprod);
+        }
+        """,
+        'cupy_triangular_kernel'
+    )
+
+    def triangular(self, left, mode, right, size=None, dtype=float):
+        """Returns an array of samples drawn from the triangular distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.triangular` for full documentation
+            - :meth:`numpy.random.RandomState.triangular`
+        """
+        left, mode, right = \
+            cupy.asarray(left), cupy.asarray(mode), cupy.asarray(right)
+        if cupy.any(left > mode):  # synchronize!
+            raise ValueError('left > mode')
+        if cupy.any(mode > right):  # synchronize!
+            raise ValueError('mode > right')
+        if cupy.any(left == right):  # synchronize!
+            raise ValueError('left == right')
+        if size is None:
+            size = cupy.broadcast(left, mode, right).shape
+        x = self.random_sample(size=size, dtype=dtype)
+        return RandomState._triangular_kernel(left, mode, right, x)
+
+    _scale_kernel = _core.ElementwiseKernel(
+        'T low, T high', 'T x',
+        'x = T(low) + x * T(high - low)',
+        'cupy_scale')
+
+    def uniform(self, low=0.0, high=1.0, size=None, dtype=float):
+        """Returns an array of uniformly-distributed samples over an interval.
+
+        .. seealso::
+            - :func:`cupy.random.uniform` for full documentation
+            - :meth:`numpy.random.RandomState.uniform`
+
+        """
+        if not numpy.isscalar(low):
+            low = cupy.asarray(low, dtype)
+        if not numpy.isscalar(high):
+            high = cupy.asarray(high, dtype)
+
+        if size is None:
+            size = cupy.broadcast(low, high).shape
+
+        dtype = numpy.dtype(dtype)
+        rand = self.random_sample(size=size, dtype=dtype)
+        return RandomState._scale_kernel(low, high, rand)
+
+    def vonmises(self, mu, kappa, size=None, dtype=float):
+        """Returns an array of samples drawn from the von Mises distribution.
+
+        .. seealso::
+            - :func:`cupy.random.vonmises` for full documentation
+            - :meth:`numpy.random.RandomState.vonmises`
+        """
+        mu, kappa = cupy.asarray(mu), cupy.asarray(kappa)
+        if size is None:
+            size = cupy.broadcast(mu, kappa).shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.vonmises_kernel(mu, kappa, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    _wald_kernel = _core.ElementwiseKernel(
+        'T mean, T scale, T U', 'T X',
+        """
+            T mu_2l;
+            T Y;
+            mu_2l = mean / (2*scale);
+            Y = mean*X*X;
+            X = mean + mu_2l*(Y - sqrt(4*scale*Y + Y*Y));
+            if (U > mean/(mean+X))
+            {
+                X = mean*mean/X;
+            }
+        """,
+        'cupy_wald_scale')
+
+    def wald(self, mean, scale, size=None, dtype=float):
+        """Returns an array of samples drawn from the Wald distribution.
+
+         .. seealso::
+            - :func:`cupy.random.wald` for full documentation
+            - :meth:`numpy.random.RandomState.wald`
+        """
+        mean, scale = \
+            cupy.asarray(mean, dtype=dtype), cupy.asarray(scale, dtype=dtype)
+        if size is None:
+            size = cupy.broadcast(mean, scale).shape
+        x = self.normal(size=size, dtype=dtype)
+        u = self.random_sample(size=size, dtype=dtype)
+        return RandomState._wald_kernel(mean, scale, u, x)
+
+    def weibull(self, a, size=None, dtype=float):
+        """Returns an array of samples drawn from the weibull distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.weibull` for full documentation
+            - :meth:`numpy.random.RandomState.weibull`
+        """
+        a = cupy.asarray(a)
+        if cupy.any(a < 0):  # synchronize!
+            raise ValueError('a < 0')
+
+        if size is None:
+            size = a.shape
+
+        x = self.standard_exponential(size, dtype)
+        cupy.power(x, 1./a, out=x)
+        return x
+
+    def zipf(self, a, size=None, dtype=int):
+        """Returns an array of samples drawn from the Zipf distribution.
+
+        .. warning::
+
+            This function may synchronize the device.
+
+        .. seealso::
+            - :func:`cupy.random.zipf` for full documentation
+            - :meth:`numpy.random.RandomState.zipf`
+        """
+        a = cupy.asarray(a)
+        if cupy.any(a <= 1.0):  # synchronize!
+            raise ValueError('\'a\' must be a valid float > 1.0')
+        if size is None:
+            size = a.shape
+        y = cupy.empty(shape=size, dtype=dtype)
+        _kernels.zipf_kernel(a, self._rk_seed, y)
+        self._update_seed(y.size)
+        return y
+
+    def choice(self, a, size=None, replace=True, p=None):
+        """Returns an array of random values from a given 1-D array.
+
+        .. seealso::
+            - :func:`cupy.random.choice` for full documentation
+            - :meth:`numpy.random.choice`
+
+        """
+        if a is None:
+            raise ValueError('a must be 1-dimensional or an integer')
+        if isinstance(a, cupy.ndarray) and a.ndim == 0:
+            raise NotImplementedError
+        if isinstance(a, int):
+            a_size = a
+            if a_size < 0:
+                raise ValueError('a must be greater than or equal to 0')
+        else:
+            a = cupy.array(a, copy=False)
+            if a.ndim != 1:
+                raise ValueError('a must be 1-dimensional or an integer')
+            a_size = len(a)
+
+        if p is not None:
+            p = cupy.array(p)
+            if p.ndim != 1:
+                raise ValueError('p must be 1-dimensional')
+            if len(p) != a_size:
+                raise ValueError('a and p must have same size')
+            if not (p >= 0).all():
+                raise ValueError('probabilities are not non-negative')
+            p_sum = cupy.sum(p).get()
+            if not numpy.allclose(p_sum, 1):
+                raise ValueError('probabilities do not sum to 1')
+
+        if size is None:
+            raise NotImplementedError(
+                'choice() without specifying size is not supported yet')
+        shape = size
+        size = numpy.prod(shape)
+
+        if a_size == 0 and size > 0:
+            raise ValueError('a cannot be empty unless no samples are taken')
+
+        if not replace and p is None:
+            if a_size < size:
+                raise ValueError(
+                    'Cannot take a larger sample than population when '
+                    '\'replace=False\'')
+            if isinstance(a, int):
+                indices = cupy.arange(a, dtype='l')
+            else:
+                indices = a.copy()
+            self.shuffle(indices)
+            return indices[:size].reshape(shape)
+
+        if not replace:
+            raise NotImplementedError
+
+        if p is not None:
+            p = cupy.broadcast_to(p, (size, a_size))
+            index = cupy.argmax(cupy.log(p) +
+                                self.gumbel(size=(size, a_size)),
+                                axis=1)
+            if not isinstance(shape, int):
+                index = cupy.reshape(index, shape)
+        else:
+            if a_size == 0:  # TODO: (#4511) Fix `randint` instead
+                a_size = 1
+            index = self.randint(0, a_size, size=shape)
+            # Align the dtype with NumPy
+            index = index.astype(cupy.int64, copy=False)
+
+        if isinstance(a, int):
+            return index
+
+        if index.ndim == 0:
+            return cupy.array(a[index], dtype=a.dtype)
+
+        return a[index]
+
+    def shuffle(self, a):
+        """Returns a shuffled array.
+
+        .. seealso::
+            - :func:`cupy.random.shuffle` for full documentation
+            - :meth:`numpy.random.shuffle`
+
+        """
+        if not isinstance(a, cupy.ndarray):
+            raise TypeError('The array must be cupy.ndarray')
+
+        if a.ndim == 0:
+            raise TypeError('An array whose ndim is 0 is not supported')
+
+        a[:] = a[self._permutation(len(a))]
+
+    def permutation(self, a):
+        """Returns a permuted range or a permutation of an array."""
+        if isinstance(a, int):
+            return self._permutation(a)
+        else:
+            return a[self._permutation(len(a))]
+
+    def _permutation(self, num):
+        """Returns a permuted range."""
+        sample = cupy.empty((num,), dtype=numpy.int32)
+        curand.generate(self._generator, sample.data.ptr, num)
+        array = cupy.argsort(sample)
+        return array
+
+    _gumbel_kernel = _core.ElementwiseKernel(
+        'T x, T loc, T scale', 'T y',
+        'y = T(loc) - log(-log(x)) * T(scale)',
+        'cupy_gumbel_kernel')
+
+    def gumbel(self, loc=0.0, scale=1.0, size=None, dtype=float):
+        """Returns an array of samples drawn from a Gumbel distribution.
+
+        .. seealso::
+            - :func:`cupy.random.gumbel` for full documentation
+            - :meth:`numpy.random.RandomState.gumbel`
+        """
+        if not numpy.isscalar(loc):
+            loc = cupy.asarray(loc, dtype)
+        if not numpy.isscalar(scale):
+            scale = cupy.asarray(scale, dtype)
+        if size is None:
+            size = cupy.broadcast(loc, scale).shape
+        x = self._random_sample_raw(size=size, dtype=dtype)
+        RandomState._gumbel_kernel(x, loc, scale, x)
+        return x
+
+    def randint(self, low, high=None, size=None, dtype=int):
+        """Returns a scalar or an array of integer values over ``[low, high)``.
+
+        .. seealso::
+            - :func:`cupy.random.randint` for full documentation
+            - :meth:`numpy.random.RandomState.randint`
+        """
+        if not numpy.isscalar(low):
+            low = cupy.asarray(low)
+            if high is None:
+                lo = cupy.zeros_like(low)
+                hi = low - 1
+            else:
+                lo = low
+                hi = cupy.asarray(high) - 1
+
+            if size is None:
+                size = cupy.broadcast(lo, hi).shape
+
+            diff = hi - lo
+            total_elems = functools.reduce(operator.mul, size, 1)
+            out = self._interval(diff.flatten(), total_elems)
+            out = out.astype(dtype)
+            out = cupy.reshape(out, size)
+            lo = lo.astype(dtype, copy=False)
+            cupy.add(out, lo, out=out)
+            return out
+        else:
+            if high is None:
+                lo = 0
+                hi1 = int(low) - 1
+            else:
+                lo = int(low)
+                hi1 = int(high) - 1
+
+            if lo > hi1:
+                raise ValueError('low >= high')
+            if lo < cupy.iinfo(dtype).min:
+                raise ValueError(
+                    'low is out of bounds for {}'.format(
+                        cupy.dtype(dtype).name))
+            if hi1 > cupy.iinfo(dtype).max:
+                raise ValueError(
+                    'high is out of bounds for {}'.format(
+                        cupy.dtype(dtype).name))
+
+            diff = hi1 - lo
+            x = self._interval(diff, size).astype(dtype, copy=False)
+            cupy.add(x, lo, out=x)
+            return x
+
+
+def seed(seed=None):
+    """Resets the state of the random number generator with a seed.
+
+    This function resets the state of the global random number generator for
+    the current device. Be careful that generators for other devices are not
+    affected.
+
+    Args:
+        seed (None or int): Seed for the random number generator. If ``None``,
+            it uses :func:`os.urandom` if available or :func:`time.time`
+            otherwise. Note that this function does not support seeding by
+            an integer array.
+
+    """
+    get_random_state().seed(seed)
+
+
+# CuPy specific functions
+
+_random_states = {}
+
+
+@atexit.register
+def reset_states():
+    global _random_states
+    _random_states = {}
+
+
+def get_random_state():
+    """Gets the state of the random number generator for the current device.
+
+    If the state for the current device is not created yet, this function
+    creates a new one, initializes it, and stores it as the state for the
+    current device.
+
+    Returns:
+        RandomState: The state of the random number generator for the
+        device.
+
+    """
+    dev = cuda.Device()
+    rs = _random_states.get(dev.id, None)
+    if rs is None:
+        seed = os.getenv('CUPY_SEED')
+        if seed is not None:
+            seed = numpy.uint64(int(seed))
+        rs = RandomState(seed)
+        rs = _random_states.setdefault(dev.id, rs)
+    return rs
+
+
+def set_random_state(rs):
+    """Sets the state of the random number generator for the current device.
+
+    Args:
+        state(RandomState): Random state to set for the current device.
+    """
+    if not isinstance(rs, RandomState):
+        raise TypeError(
+            'Random state must be an instance of RandomState. '
+            'Actual: {}'.format(type(rs)))
+    _random_states[device.get_device_id()] = rs
+
+
+def _check_and_get_dtype(dtype):
+    dtype = numpy.dtype(dtype)
+    if dtype.char not in ('f', 'd'):
+        raise TypeError('cupy.random only supports float32 and float64')
+    return dtype
diff --git a/cupy/random/_generator_api.pyx b/cupy/random/_generator_api.pyx
new file mode 100644
index 0000000..f4a76f6
--- /dev/null
+++ b/cupy/random/_generator_api.pyx
@@ -0,0 +1,1054 @@
+# distutils: language = c++
+import numpy
+
+from libc.stdint cimport intptr_t, uint64_t, int32_t, int64_t
+
+import cupy
+from cupy import _core
+from cupy.cuda cimport stream
+from cupy._core.core cimport _ndarray_base
+from cupy_backends.cuda.api import runtime
+
+_UINT32_MAX = 0xffffffff
+_UINT64_MAX = 0xffffffffffffffff
+
+cdef extern from 'cupy_distributions.cuh' nogil:
+    cppclass rk_binomial_state:
+        pass
+    void init_curand_generator(
+        int generator, intptr_t state_ptr, uint64_t seed,
+        ssize_t size, intptr_t stream)
+    void random_uniform(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream)
+    void random_uniform_float(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream)
+    void raw(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream)
+    void interval_32(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream, int32_t mx, int32_t mask)
+    void interval_64(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream, int64_t mx, int64_t mask)
+    void beta(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream, intptr_t a, intptr_t b)
+    void exponential(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream)
+    void geometric(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream, intptr_t arg1)
+    void hypergeometric(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream,
+        intptr_t arg1, intptr_t arg2, intptr_t arg3)
+    void logseries(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream, intptr_t arg1)
+    void standard_normal(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream)
+    void standard_normal_float(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream)
+    # if the types are the same, but the names are different
+    # cython will fail when trying to create a PyObj wrapper
+    # to use these functions from python
+    # arg1 is shape
+    void standard_gamma(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream, intptr_t arg1)
+    # arg1 is lam
+    void poisson(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream, intptr_t arg1)
+    void binomial(
+        int generator, intptr_t state, ssize_t state_size, intptr_t out,
+        ssize_t size, intptr_t stream,
+        intptr_t arg1, intptr_t arg2, intptr_t arg3)
+
+
+cdef _ndarray_base _array_data(_ndarray_base x):
+    return cupy.array((x.data.ptr, x.ndim) + x.shape + x.strides)
+
+
+class Generator:
+    """Container for the BitGenerators.
+
+    ``Generator`` exposes a number of methods for generating random
+    numbers drawn from a variety of probability distributions. In addition to
+    the distribution-specific arguments, each method takes a keyword argument
+    `size` that defaults to ``None``. If `size` is ``None``, then a single
+    value is generated and returned. If `size` is an integer, then a 1-D
+    array filled with generated values is returned. If `size` is a tuple,
+    then an array with that shape is filled and returned.
+    The function :func:`numpy.random.default_rng` will instantiate
+    a `Generator` with numpy's default `BitGenerator`.
+    **No Compatibility Guarantee**
+    ``Generator`` does not provide a version compatibility guarantee. In
+    particular, as better algorithms evolve the bit stream may change.
+
+    Args:
+        bit_generator : (cupy.random.BitGenerator): BitGenerator to use
+            as the core generator.
+
+    """
+    def __init__(self, bit_generator):
+        if runtime.is_hip and int(str(runtime.runtimeGetVersion())[:3]) < 403:
+            raise RuntimeError('Generator API not supported in ROCm<4.3,'
+                               ' please use the legacy one or update ROCm.')
+        self.bit_generator = bit_generator
+        self._binomial_state = None
+
+    def _check_output_array(self, dtype, size, out, check_only_c_cont=False):
+        # Checks borrowed from NumPy
+        # https://github.com/numpy/numpy/blob/cb557b79fa0ce467c881830f8e8e042c484ccfaa/numpy/random/_common.pyx#L235-L251
+        dtype = numpy.dtype(dtype)
+        if out.dtype.char != dtype.char:
+            raise TypeError(
+                f'Supplied output array has the wrong type. '
+                f'Expected {dtype.name}, got {out.dtype.name}')
+        if not out.flags.c_contiguous:
+            if check_only_c_cont or not out.flags.f_contiguous:
+                raise ValueError(
+                    'Supplied output array is not contiguous,'
+                    ' writable or aligned.')
+        if size is not None:
+            try:
+                tup_size = tuple(size)
+            except TypeError:
+                tup_size = tuple([size])
+            if tup_size != out.shape:
+                raise ValueError(
+                    'size must match out.shape when used together')
+
+    def random(self, size=None, dtype=numpy.float64, out=None):
+        """Return random floats in the half-open interval [0.0, 1.0).
+
+        Results are from the "continuous uniform" distribution over the
+        stated interval.  To sample :math:`Unif[a, b), b > a` multiply
+        the output of `random` by `(b-a)` and add `a`::
+
+          (b - a) * random() + a
+
+        Args:
+            size (None or int or tuple of ints): The shape of returned value.
+            dtype: Data type specifier.
+            out (cupy.ndarray, optional): If specified, values will be written
+                to this array
+
+        Returns:
+            cupy.ndarray: Samples uniformly drawn from the [0, 1) interval
+
+        .. seealso::
+            - :meth:`numpy.random.Generator.random`
+        """
+        cdef _ndarray_base y
+
+        if out is not None:
+            self._check_output_array(dtype, size, out)
+
+        y = _core.ndarray(size if size is not None else (), dtype)
+        if y.dtype.char == 'd':
+            _launch_dist(self.bit_generator, random_uniform, y, ())
+        else:
+            _launch_dist(self.bit_generator, random_uniform_float, y, ())
+        if out is not None:
+            _core.elementwise_copy(y, out)
+            y = out
+        # we cast the array to a python object because
+        # cython cant call astype with the default values for
+        # omitted args.
+        return (<object>y).astype(dtype, copy=False)
+
+    def uniform(self, low=0.0, high=1.0, size=None, dtype=numpy.float64):
+        """
+        Draw samples from a uniform distribution.
+        Samples are uniformly distributed over the half-open interval
+        ``[low, high)`` (includes low, but excludes high).  In other words,
+        any value within the given interval is equally likely to be drawn
+        by `uniform`.
+
+        Parameters
+        ----------
+        low : float or array_like of floats, optional
+            Lower boundary of the output interval.  All values generated will
+            be greater than or equal to low.  The default value is 0.
+        high : float or array_like of floats
+            Upper boundary of the output interval.  All values generated will
+            be less than high.  The high limit may be included in the returned
+            array of floats due to floating-point rounding in the equation
+            ``low + (high-low) * random()``.  high - low must be
+            non-negative.  The default value is 1.0.
+        size : int or tuple of ints, optional
+            Output shape.  If the given shape is, e.g., ``(m, n, k)``, then
+            ``m * n * k`` samples are drawn.  If size is ``None`` (default),
+            a single value is returned if ``low`` and ``high`` are both
+            scalars.  Otherwise, ``cupy.broadcast(low, high).size`` samples are
+            drawn.
+
+        Returns
+        -------
+        out : ndarray or scalar
+            Drawn samples from the parameterized uniform distribution.
+        See Also
+        --------
+        - :meth:`numpy.random.Generator.uniform`
+        - :meth:`integers`: Discrete uniform distribution, yielding integers.
+        - :meth:`random`: Floats uniformly distributed over ``[0, 1)``.
+        """
+
+        cdef _ndarray_base y
+
+        low = cupy.asarray(low)
+        high = cupy.asarray(high)
+
+        if size is None:
+            size = cupy.broadcast(low, high).shape
+
+        y = _core.ndarray(size, dtype)
+        low = cupy.broadcast_to(low, y.shape)
+        high = cupy.broadcast_to(high, y.shape)
+
+        if y.dtype.char == 'd':
+            _launch_dist(self.bit_generator, random_uniform, y, ())
+        else:
+            _launch_dist(self.bit_generator, random_uniform_float, y, ())
+
+        y = low + (high - low) * y
+
+        # we cast the array to a python object because
+        # cython cant call astype with the default values for
+        # omitted args.
+        return (<object>y).astype(dtype, copy=False)
+
+    def integers(
+            self, low, high=None, size=None,
+            dtype=numpy.int64, endpoint=False):
+        """Returns a scalar or an array of integer values over an interval.
+
+        Each element of returned values are independently sampled from
+        uniform distribution over the ``[low, high)`` or ``[low, high]``
+        intervals.
+
+        Args:
+            low (int): If ``high`` is not ``None``,
+                it is the lower bound of the interval.
+                Otherwise, it is the **upper** bound of the interval
+                and lower bound of the interval is set to ``0``.
+            high (int): Upper bound of the interval.
+            size (None or int or tuple of ints): The shape of returned value.
+            dtype: Data type specifier.
+            endpoint (bool): If ``True``, sample from ``[low, high]``.
+                Defaults to ``False``
+
+        Returns:
+            int or cupy.ndarray of ints: If size is ``None``,
+            it is single integer sampled.
+            If size is integer, it is the 1D-array of length ``size`` element.
+            Otherwise, it is the array whose shape specified by ``size``.
+
+        .. seealso::
+            - :meth:`numpy.random.Generator.integers`
+        """
+        cdef _ndarray_base y
+        if high is None:
+            lo = 0
+            hi1 = int(low)
+        else:
+            lo = int(low)
+            hi1 = int(high)
+
+        if not endpoint:
+            hi1 -= 1
+
+        if lo > hi1:
+            raise ValueError('low >= high')
+        if lo < cupy.iinfo(dtype).min:
+            raise ValueError(
+                'low is out of bounds for {}'.format(cupy.dtype(dtype).name))
+        if hi1 > cupy.iinfo(dtype).max:
+            raise ValueError(
+                'high is out of bounds for {}'.format(cupy.dtype(dtype).name))
+
+        diff = hi1 - lo
+
+        cdef uint64_t mask = (1 << diff.bit_length()) - 1
+        # TODO adjust dtype
+        if diff <= _UINT32_MAX:
+            pdtype = numpy.uint32
+        elif diff <= _UINT64_MAX:
+            pdtype = numpy.uint64
+        else:
+            raise ValueError(
+                f'high - low must be within uint64 range (actual: {diff})')
+
+        y = _core.ndarray(size if size is not None else (), pdtype)
+        if pdtype is numpy.uint32:
+            _launch_dist(self.bit_generator, interval_32, y, (diff, mask))
+        else:
+            _launch_dist(self.bit_generator, interval_64, y, (diff, mask))
+        return cupy.add(lo, y, dtype=dtype)
+
+    def beta(self, a, b, size=None, dtype=numpy.float64):
+        """Beta distribution.
+
+        Returns an array of samples drawn from the beta distribution. Its
+        probability density function is defined as
+
+        .. math::
+           f(x) = \\frac{x^{\\alpha-1}(1-x)^{\\beta-1}}{B(\\alpha,\\beta)}.
+
+        Args:
+            a (float): Parameter of the beta distribution :math:`\\alpha`.
+            b (float): Parameter of the beta distribution :math:`\\beta`.
+            size (int or tuple of ints): The shape of the array. If ``None``, a
+                zero-dimensional array is generated.
+            dtype: Data type specifier. Only :class:`numpy.float32` and
+                :class:`numpy.float64` types are allowed.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the beta distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.beta`
+        """
+        cdef _ndarray_base y
+
+        if not isinstance(a, _ndarray_base):
+            if type(a) in (float, int):
+                a = cupy.asarray(a, numpy.float64)
+            else:
+                raise TypeError('a is required to be a cupy.ndarray'
+                                ' or a scalar')
+        if not isinstance(b, _ndarray_base):
+            if type(b) in (float, int):
+                b = cupy.asarray(b, numpy.float64)
+            else:
+                raise TypeError('b is required to be a cupy.ndarray'
+                                ' or a scalar')
+
+        if size is not None and not isinstance(size, tuple):
+            size = (size, )
+        elif size is None:
+            size = cupy.broadcast(a, b).shape
+
+        y = _core.ndarray(size, numpy.float64)
+
+        a = cupy.broadcast_to(a, y.shape)
+        b = cupy.broadcast_to(b, y.shape)
+
+        _launch_dist(self.bit_generator, beta, y, (a, b))
+        # we cast the array to a python object because
+        # cython cant call astype with the default values for
+        # omitted args.
+        return (<object>y).astype(dtype, copy=False)
+
+    def chisquare(self, df, size=None):
+        """Chi-square distribution.
+
+        Returns an array of samples drawn from the chi-square distribution. Its
+        probability density function is defined as
+
+        .. math::
+           f(x) = \\frac{(1/2)^{k/2}}{\\Gamma(k/2)}x^{k/2-1}e^{-x/2}.
+
+        Args:
+            df (float or array_like of floats): Degree of freedom :math:`k`.
+            size (int or tuple of ints): The shape of the array. If ``None``, a
+                zero-dimensional array is generated.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the chi-square distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.chisquare`
+        """
+
+        cdef _ndarray_base y
+
+        if not isinstance(df, _ndarray_base):
+            if type(df) in (float, int):
+                df = cupy.asarray(df, numpy.float64)
+            else:
+                raise TypeError('df is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            df = df.astype('d', copy=False)
+
+        if size is not None and not isinstance(size, tuple):
+            size = (size,)
+        if size is None:
+            size = df.shape
+
+        y = _core.ndarray(size, numpy.float64)
+
+        df = cupy.broadcast_to(df, y.shape)
+        y = self.standard_gamma(df / 2)
+        y *= 2
+        return y
+
+    def dirichlet(self, alpha, size=None):
+        """Dirichlet distribution.
+
+        Returns an array of samples drawn from the dirichlet distribution. Its
+        probability density function is defined as
+
+        .. math::
+            f(x) = \\frac{\\Gamma(\\sum_{i=1}^K\\alpha_i)} \
+                {\\prod_{i=1}^{K}\\Gamma(\\alpha_i)} \
+                \\prod_{i=1}^Kx_i^{\\alpha_i-1}.
+
+        Args:
+            alpha (array): Parameters of the dirichlet distribution
+                :math:`\\alpha`.
+            size (int or tuple of ints): The shape of the array. If ``None``,
+                array of ``alpha.shape`` is generated
+
+        Returns:
+            cupy.ndarray: Samples drawn from the dirichlet distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.dirichlet`
+        """
+
+        if not isinstance(alpha, _ndarray_base):
+            if type(alpha) in (float, int):
+                alpha = cupy.asarray(alpha, numpy.float64)
+            else:
+                raise TypeError('alpha is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            alpha = alpha.astype('d', copy=False)
+
+        if size is not None:
+            if not isinstance(size, tuple):
+                size = (size,)
+            size += alpha.shape
+
+        elif size is None:
+            size = alpha.shape
+
+        y = self.standard_gamma(alpha, size)
+        y /= y.sum(axis=-1, keepdims=True)
+        return y
+
+    def exponential(self, scale=1.0, size=None):
+        """Exponential distribution.
+
+        Returns an array of samples drawn from the exponential distribution.
+        Its probability density function is defined as
+
+        .. math::
+           f(x) = \\frac{1}{\\beta}\\exp (-\\frac{x}{\\beta}).
+
+        Args:
+            scale (float or array_like of floats): The scale parameter
+                :math:`\\beta`.
+            size (int or tuple of ints): The shape of the array. If ``None``, a
+                zero-dimensional array is generated.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the exponential distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.exponential`
+        """
+        return self.standard_exponential(size) * scale
+
+    def f(self, dfnum, dfden, size=None):
+        """F distribution.
+
+        Returns an array of samples drawn from the f distribution. Its
+        probability density function is defined as
+
+        .. math::
+            f(x) = \\frac{1}{B(\\frac{d_1}{2},\\frac{d_2}{2})} \
+                \\left(\\frac{d_1}{d_2}\\right)^{\\frac{d_1}{2}} \
+                x^{\\frac{d_1}{2}-1} \
+                \\left(1+\\frac{d_1}{d_2}x\\right) \
+                ^{-\\frac{d_1+d_2}{2}}.
+
+        Args:
+            dfnum (float or array_like of floats): Degrees of freedom in
+                numerator, :math:`d_1`.
+            dfden (float or array_like of floats): Degrees of freedom in
+                denominator, :math:`d_2`.
+            size (int or tuple of ints): The shape of the array. If ``None``, a
+                zero-dimensional array is generated.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the f distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.f`
+        """
+        if not isinstance(dfnum, _ndarray_base):
+            if type(dfnum) in (float, int):
+                dfnum = cupy.asarray(dfnum, numpy.float64)
+            else:
+                raise TypeError('dfnum is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            dfnum = dfnum.astype('d', copy=False)
+
+        if not isinstance(dfden, _ndarray_base):
+            if type(dfden) in (float, int):
+                dfden = cupy.asarray(dfden, numpy.float64)
+            else:
+                raise TypeError('dfden is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            dfden = dfden.astype('d', copy=False)
+
+        if size is None:
+            size = cupy.broadcast(dfnum, dfden).shape
+
+        y = (self.chisquare(dfnum, size) * dfden) / (
+            self.chisquare(dfden, size) * dfnum)
+        return y
+
+    def geometric(self, p, size=None):
+        """Geometric distribution.
+
+        Returns an array of samples drawn from the geometric distribution. Its
+        probability mass function is defined as
+
+        .. math::
+            f(x) = p(1-p)^{k-1}.
+
+        Args:
+            p (float or cupy.ndarray of floats): Success probability of
+                the geometric distribution.
+            size (int or tuple of ints, optional): The shape of the output
+                array. If ``None`` (default), a single value is returned if
+                ``p`` is scalar. Otherwise, ``p.size`` samples are drawn.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the geometric distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.geometric`
+        """
+        cdef _ndarray_base y
+        cdef _ndarray_base p_a
+
+        if not isinstance(p, _ndarray_base):
+            if type(p) in (float, int):
+                p_a = _core.ndarray((), numpy.float64)
+                p_a.fill(p)
+                p = p_a
+            else:
+                raise TypeError('p is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            p = p.astype('d', copy=False)
+
+        if size is not None and not isinstance(size, tuple):
+            size = (size,)
+        elif size is None:
+            size = p.shape
+        y = _core.ndarray(size if size is not None else (), numpy.int64)
+
+        p = cupy.broadcast_to(p, y.shape)
+
+        _launch_dist(self.bit_generator, geometric, y, (p,))
+        return y
+
+    def hypergeometric(self, ngood, nbad, nsample, size=None):
+        """Hypergeometric distribution.
+
+        Returns an array of samples drawn from the hypergeometric distribution.
+        Its probability mass function is defined as
+
+        .. math::
+            f(x) = \\frac{\\binom{m}{n}\\binom{N-m}{n-x}}{\\binom{N}{n}}.
+
+        Args:
+            ngood (int or array_like of ints): Parameter of the hypergeometric
+                distribution :math:`n`.
+            nbad (int or array_like of ints): Parameter of the hypergeometric
+                distribution :math:`m`.
+            nsample (int or array_like of ints): Parameter of the
+                hypergeometric distribution :math:`N`.
+            size (int or tuple of ints): The shape of the array. If ``None``, a
+                zero-dimensional array is generated.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the hypergeometric distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.hypergeometric`
+        """
+        cdef _ndarray_base y
+        cdef _ndarray_base ngood_a
+        cdef _ndarray_base nbad_a
+        cdef _ndarray_base nsample_a
+
+        if not isinstance(ngood, _ndarray_base):
+            if type(ngood) in (float, int):
+                ngood_a = _core.ndarray((), numpy.int64)
+                ngood_a.fill(ngood)
+                ngood = ngood_a
+            else:
+                raise TypeError('ngood is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            ngood = ngood.astype(numpy.int64, copy=False)
+
+        if not isinstance(nbad, _ndarray_base):
+            if type(nbad) in (float, int):
+                nbad_a = _core.ndarray((), numpy.int64)
+                nbad_a.fill(nbad)
+                nbad = nbad_a
+            else:
+                raise TypeError('nbad is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            nbad = nbad.astype(numpy.int64, copy=False)
+
+        if not isinstance(nsample, _ndarray_base):
+            if type(nsample) in (float, int):
+                nsample_a = _core.ndarray((), numpy.int64)
+                nsample_a.fill(nsample)
+                nsample = nsample_a
+            else:
+                raise TypeError('nsample is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            nsample = nsample.astype(numpy.int64, copy=False)
+
+        if size is not None and not isinstance(size, tuple):
+            size = (size,)
+        if size is None:
+            size = cupy.broadcast(ngood, nbad, nsample).shape
+        y = _core.ndarray(size, numpy.int64)
+
+        ngood = cupy.broadcast_to(ngood, y.shape)
+        nbad = cupy.broadcast_to(nbad, y.shape)
+        nsample = cupy.broadcast_to(nsample, y.shape)
+
+        _launch_dist(self.bit_generator, hypergeometric, y,
+                     (ngood, nbad, nsample))
+        return y
+
+    def logseries(self, p, size=None):
+        """Log series distribution.
+
+        Returns an array of samples drawn from the log series distribution.
+        Its probability mass function is defined as
+
+        .. math::
+           f(x) = \\frac{-p^x}{x\\ln(1-p)}.
+
+        Args:
+            p (float or cupy.ndarray of floats): Parameter of the log series
+                distribution. Must be in the range (0, 1).
+            size (int or tuple of ints, optional): The shape of the output
+                array. If ``None`` (default), a single value is returned if
+                ``p`` is scalar. Otherwise, ``p.size`` samples are drawn.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the log series distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.logseries`
+        """
+        cdef _ndarray_base y
+
+        if not isinstance(p, _ndarray_base):
+            if type(p) in (float, int):
+                p = cupy.asarray(p, numpy.float64)
+            else:
+                raise TypeError('p is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            p = p.astype('d', copy=False)
+
+        if size is not None and not isinstance(size, tuple):
+            size = (size,)
+        elif size is None:
+            size = p.shape
+
+        y = _core.ndarray(size, numpy.int64)
+
+        p = cupy.broadcast_to(p, y.shape)
+        _launch_dist(self.bit_generator, logseries, y, (p,))
+        return y
+
+    def standard_exponential(
+            self, size=None, dtype=numpy.float64,
+            method='inv', out=None):
+        """Standard exponential distribution.
+
+        Returns an array of samples drawn from the standard exponential
+        distribution. Its probability density function is defined as
+
+          .. math::
+             f(x) = e^{-x}.
+
+        Args:
+            size (int or tuple of ints): The shape of the array. If ``None``,
+                a zero-dimensional array is generated.
+            dtype: Data type specifier. Only :class:`numpy.float32` and
+                :class:`numpy.float64` types are allowed.
+            method (str): Method to sample. Currently only ``'inv'``, sampling
+                from the default inverse CDF, is supported.
+            out (cupy.ndarray, optional): If specified, values will be written
+                to this array
+        Returns:
+            cupy.ndarray: Samples drawn from the standard exponential
+            distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.standard_exponential`
+        """
+        cdef _ndarray_base y
+
+        if method == 'zig':
+            raise NotImplementedError('Ziggurat method is not supported')
+
+        if out is not None:
+            self._check_output_array(dtype, size, out)
+
+        y = _core.ndarray(size if size is not None else (), numpy.float64)
+        _launch_dist(self.bit_generator, exponential, y, ())
+        if out is not None:
+            _core.elementwise_copy(y, out)
+            y = out
+        # we cast the array to a python object because
+        # cython cant call astype with the default values for
+        # omitted args.
+        return (<object>y).astype(dtype, copy=False)
+
+    def poisson(self, lam=1.0, size=None):
+        """Poisson distribution.
+
+        Returns an array of samples drawn from the poisson distribution. Its
+        probability mass function is defined as
+
+        .. math::
+            f(x) = \\frac{\\lambda^xe^{-\\lambda}}{x!}.
+
+        Args:
+            lam (float or array_like of floats): Parameter of
+                the poisson distribution
+                :math:`\\lambda`.
+            size (int or tuple of ints): The shape of the array. If ``None``,
+                this function generate an array whose shape is ``lam.shape``.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the poisson distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.poisson`
+        """
+        cdef _ndarray_base y
+        cdef _ndarray_base lam_a
+
+        if not isinstance(lam, _ndarray_base):
+            if type(lam) in (float, int):
+                lam_a = _core.ndarray((), numpy.float64)
+                lam_a.fill(lam)
+                lam = lam_a
+            else:
+                raise TypeError('lam is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            # Check if size is broadcastable to shape
+            # but size determines the output
+            lam = lam.astype('d', copy=False)
+
+        if size is not None and not isinstance(size, tuple):
+            size = (size,)
+        elif size is None:
+            size = lam.shape
+
+        y = _core.ndarray(size if size is not None else (), numpy.int64)
+
+        lam = cupy.broadcast_to(lam, y.shape)
+        _launch_dist(self.bit_generator, poisson, y, (lam,))
+        return y
+
+    def power(self, a, size=None):
+        """Power distribution.
+
+        Returns an array of samples drawn from the power distribution. Its
+        probability density function is defined as
+
+        .. math::
+           f(x) = ax^{a-1}.
+
+        Args:
+            a (float or array_like of floats): Parameter of the power
+                distribution :math:`a`.
+            size (int or tuple of ints): The shape of the array. If ``None``, a
+                zero-dimensional array is generated.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the power distribution.
+
+        .. seealso::
+            :meth:`numpy.random.Generator.power`
+        """
+
+        if not isinstance(a, _ndarray_base):
+            if type(a) in (float, int):
+                a = cupy.asarray(a, numpy.float64)
+            else:
+                raise TypeError('a is required to be a cupy.ndarray'
+                                ' or a scalar')
+        else:
+            a = a.astype('d', copy=False)
+
+        if size is not None and not isinstance(size, tuple):
+            size = (size, )
+        elif size is None:
+            size = a.shape
+
+        x = self.standard_exponential(size)
+        cupy.exp(-x, out=x)
+        cupy.add(1, -x, out=x)
+        cupy.power(x, 1./a, out=x)
+        return x
+
+    def standard_normal(self, size=None, dtype=numpy.float64, out=None):
+        """Standard normal distribution.
+
+        Returns an array of samples drawn from the standard normal
+        distribution.
+
+        Args:
+            size (int or tuple of ints): The shape of the array. If ``None``, a
+                zero-dimensional array is generated.
+            dtype: Data type specifier.
+
+            out (cupy.ndarray, optional): If specified, values will be written
+                to this array
+
+        Returns:
+            cupy.ndarray: Samples drawn from the standard normal distribution.
+
+        .. seealso::
+            - :meth:`numpy.random.Generator.standard_normal`
+        """
+        cdef _ndarray_base y
+
+        if out is not None:
+            self._check_output_array(dtype, size, out)
+            y = out
+        else:
+            y = _core.ndarray(size if size is not None else (), dtype)
+
+        if y.dtype.char not in ('f', 'd'):
+            raise TypeError(
+                f'Unsupported dtype {y.dtype.name} for standard_normal')
+
+        if y.dtype.char == 'd':
+            _launch_dist(self.bit_generator, standard_normal, y, ())
+        else:
+            _launch_dist(self.bit_generator, standard_normal_float, y, ())
+
+        return y
+
+    def gamma(self, shape, scale=1.0, size=None):
+        """Gamma distribution.
+
+        Returns an array of samples drawn from the gamma distribution. Its
+        probability density function is defined as
+
+        .. math::
+           f(x) = \\frac{1}{\\Gamma(k)\\theta^k}x^{k-1}e^{-x/\\theta}.
+
+
+        Args:
+            shape (float or array_like of float): The shape of the
+                gamma distribution.  Must be non-negative.
+            scale (float or array_like of float): The scale of the
+                gamma distribution.  Must be non-negative.
+                Default equals to 1
+            size (int or tuple of ints): The shape of the array.
+                If ``None``, a zero-dimensional array is generated.
+
+        .. seealso::
+            - :meth:`numpy.random.Generator.gamma`
+        """
+        if size is None:
+            size = cupy.broadcast(shape, scale).shape
+        y = self.standard_gamma(shape, size)
+        y *= scale
+        return y
+
+    def standard_gamma(self, shape, size=None, dtype=numpy.float64, out=None):
+        """Standard gamma distribution.
+
+        Returns an array of samples drawn from the standard gamma distribution.
+        Its probability density function is defined as
+
+        .. math::
+           f(x) = \\frac{1}{\\Gamma(k)}x^{k-1}e^{-x}.
+
+
+        Args:
+            shape (float or array_like of float): The shape of the
+                gamma distribution.  Must be non-negative.
+            size (int or tuple of ints): The shape of the array.
+                If ``None``, a zero-dimensional array is generated.
+            dtype: Data type specifier.
+            out (cupy.ndarray, optional): If specified, values will be written
+                to this array
+
+        .. seealso::
+            - :meth:`numpy.random.Generator.standard_gamma`
+        """
+        cdef _ndarray_base y
+        cdef _ndarray_base shape_a
+
+        if not isinstance(shape, _ndarray_base):
+            if type(shape) in (float, int):
+                shape_a = _core.ndarray((), numpy.float64)
+                shape_a.fill(shape)
+                shape = shape_a
+            else:
+                if shape is None:
+                    raise TypeError('shape must be real number, not NoneType')
+                raise ValueError('shape is required to be a cupy.ndarray'
+                                 ' or a scalar')
+        else:
+            # Check if size is broadcastable to shape
+            # but size determines the output
+            shape = shape.astype('d', copy=False)
+
+        if size is not None and not isinstance(size, tuple):
+            size = (size,)
+        elif size is None:
+            size = shape.shape if out is None else out.shape
+
+        y = None
+        if out is not None:
+            self._check_output_array(dtype, size, out, True)
+            if out.dtype.char == 'd':
+                y = out
+
+        if y is None:
+            y = _core.ndarray(size if size is not None else (), numpy.float64)
+
+        if numpy.dtype(dtype).char not in ('f', 'd'):
+            raise TypeError(
+                f'Unsupported dtype {y.dtype.name} for standard_gamma')
+
+        shape = cupy.broadcast_to(shape, y.shape)
+
+        _launch_dist(self.bit_generator, standard_gamma, y, (shape,))
+        if out is not None and y is not out:
+            _core.elementwise_copy(y, out)
+            y = out
+        # we cast the array to a python object because
+        # cython cant call astype with the default values for
+        # omitted args.
+        return (<object>y).astype(dtype, copy=False)
+
+    def binomial(self, n, p, size=None):
+        """Binomial distribution.
+
+        Returns an array of samples drawn from the binomial distribution. Its
+        probability mass function is defined as
+
+        .. math::
+           f(x) = \\binom{n}{x}p^x(1-p)^(n-x).
+
+        Args:
+            n (int or cupy.ndarray of ints): Parameter of the distribution,
+                >= 0. Floats are also accepted, but they will be truncated to
+                integers.
+            p (float or cupy.ndarray of floats): Parameter of the distribution,
+                >= 0 and <= 1.
+            size (int or tuple of ints, optional): The shape of the output
+                array. If ``None`` (default), a single value is returned if
+                ``n`` and ``p`` are both scalars. Otherwise,
+                ``cupy.broadcast(n, p).size`` samples are drawn.
+
+        Returns:
+            cupy.ndarray: Samples drawn from the binomial distribution.
+
+        .. seealso::
+           :meth:`numpy.random.Generator.binomial`
+        """
+        cdef _ndarray_base y
+        cdef intptr_t binomial_state_ptr
+
+        if isinstance(n, _ndarray_base):
+            n = n.astype(numpy.int64, copy=False)
+        elif type(n) in (float, int):
+            n = cupy.asarray(n, numpy.int64)
+        else:
+            raise TypeError('n is required to be a cupy.ndarray or a scalar')
+
+        if isinstance(p, _ndarray_base):
+            p = p.astype(numpy.float64, copy=False)
+        elif type(p) is float:
+            p = cupy.asarray(p, numpy.float64)
+        else:
+            raise TypeError('p is required to be a cupy.ndarray or a scalar')
+
+        if size is None:
+            size = cupy.broadcast(n, p).shape
+
+        y = _core.ndarray(size if size is not None else (), numpy.int64)
+
+        n = cupy.broadcast_to(n, y.shape)
+        p = cupy.broadcast_to(p, y.shape)
+
+        if self._binomial_state is None:
+            state_size = self.bit_generator._state_size()
+            self._binomial_state = cupy.zeros(
+                sizeof(rk_binomial_state) * state_size, dtype=numpy.int8)
+        binomial_state_ptr = <intptr_t>self._binomial_state.data.ptr
+        _launch_dist(
+            self.bit_generator, binomial, y,
+            (n, p, binomial_state_ptr))
+        return y
+
+
+def init_curand(generator, state, seed, size):
+    init_curand_generator(
+        <int>generator,
+        <intptr_t>state,
+        <uint64_t>seed,
+        <uint64_t>size,
+        <intptr_t>stream.get_current_stream_ptr()
+    )
+
+
+def random_raw(generator, out):
+    _launch_dist(generator, raw, out, ())
+
+
+cdef void _launch(
+        func, int generator, intptr_t state, intptr_t strm,
+        int bsize, out, args):
+    cdef ssize_t size = out.size
+    if size == 0:
+        # Avoid issues launching empty grids in CUDA 10.2
+        return
+    nargs = [
+        _array_data(a)
+        if isinstance(a, cupy.ndarray) else a for a in args]
+    args_ptr = [
+        <intptr_t>a.data.ptr
+        if isinstance(a, cupy.ndarray) else a for a in nargs]
+
+    func(generator, state, bsize,
+         <intptr_t>out.data.ptr, size, strm, *args_ptr)
+
+
+cdef void _launch_dist(bit_generator, func, out, args) except*:
+    cdef intptr_t strm = stream.get_current_stream_ptr()
+    cdef intptr_t state = <intptr_t>bit_generator.state()
+    cdef int generator = bit_generator.generator
+    cdef bsize = bit_generator._state_size()
+    _launch(func, generator, state, strm, bsize, out, args)
diff --git a/cupy/random/_kernels.py b/cupy/random/_kernels.py
new file mode 100644
index 0000000..3592e3f
--- /dev/null
+++ b/cupy/random/_kernels.py
@@ -0,0 +1,1087 @@
+"""
+Random kit 1.3
+
+Copyright (c) 2003-2005, Jean-Sebastien Roy (js@jeannot.org)
+
+The rk_random and rk_seed functions algorithms and the original design of
+the Mersenne Twister RNG:
+
+  Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+  1. Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+  3. The names of its contributors may not be used to endorse or promote
+  products derived from this software without specific prior written
+  permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Original algorithm for the implementation of rk_interval function from
+Richard J. Wagner's implementation of the Mersenne Twister RNG, optimised by
+Magnus Jonsson.
+
+Constants used in the rk_double implementation by Isaku Wada.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""  # NOQA
+
+from cupy import _core
+
+
+rk_use_binominal = '''
+#define CUPY_USE_BINOMIAL
+'''
+
+rk_basic_definition = '''
+typedef struct {
+    unsigned int xor128[4];
+    double gauss;
+    int has_gauss; // !=0: gauss contains a gaussian deviate
+
+#ifdef CUPY_USE_BINOMIAL
+    int has_binomial; // !=0: following parameters initialized for binomial
+    /* The rk_state structure has been extended to store the following
+     * information for the binomial generator. If the input values of n or p
+     * are different than nsave and psave, then the other parameters will be
+     * recomputed. RTK 2005-09-02 */
+    int nsave, m;
+    double psave, r, q, fm, p1, xm, xl, xr, c, laml, lamr, p2, p3, p4;
+#endif
+} rk_state;
+
+
+__device__ void rk_seed(unsigned long long s, rk_state *state) {
+    for (int i = 1; i <= 4; i++) {
+        s = 1812433253U * (s ^ (s >> 30)) + i;
+        state->xor128[i - 1] = s;
+    }
+    state->has_gauss = 0;
+#ifdef CUPY_USE_BINOMIAL
+    state->has_binomial = 0;
+#endif
+}
+
+
+__device__ unsigned long rk_random(rk_state *state) {
+    unsigned int *xor128 = state->xor128;
+    unsigned int t = xor128[0] ^ (xor128[0] << 11);
+    xor128[0] = xor128[1];
+    xor128[1] = xor128[2];
+    xor128[2] = xor128[3];
+    return xor128[3] ^= (xor128[3] >> 19) ^ t ^ (t >> 8);
+}
+
+
+__device__ double rk_double(rk_state *state) {
+    /* shifts : 67108864 = 0x4000000, 9007199254740992 = 0x20000000000000 */
+    int a = rk_random(state) >> 5, b = rk_random(state) >> 6;
+    return (a * 67108864.0 + b) / 9007199254740992.0;
+}
+'''
+
+
+# The kernels for distributions are based on
+# numpy/random/mtrand/distributions.c
+# with the following licenses:
+"""
+/* Copyright 2005 Robert Kern (robert.kern@gmail.com)
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/* The implementations of rk_hypergeometric_hyp(), rk_hypergeometric_hrua(),
+ * and rk_triangular() were adapted from Ivan Frohne's rv.py which has this
+ * license:
+ *
+ *            Copyright 1998 by Ivan Frohne; Wasilla, Alaska, U.S.A.
+ *                            All Rights Reserved
+ *
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation for any purpose, free of charge, is granted subject to the
+ * following conditions:
+ *   The above copyright notice and this permission notice shall be included in
+ *   all copies or substantial portions of the software.
+ *
+ *   THE SOFTWARE AND DOCUMENTATION IS PROVIDED WITHOUT WARRANTY OF ANY KIND,
+ *   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO MERCHANTABILITY, FITNESS
+ *   FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR
+ *   OR COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM OR DAMAGES IN A CONTRACT
+ *   ACTION, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ *   SOFTWARE OR ITS DOCUMENTATION.
+ */
+"""  # NOQA
+
+loggam_definition = '''
+/*
+ * log-gamma function to support some of these distributions. The
+ * algorithm comes from SPECFUN by Shanjie Zhang and Jianming Jin and their
+ * book "Computation of Special Functions", 1996, John Wiley & Sons, Inc.
+ */
+static __device__ double loggam(double x) {
+    double x0, x2, xp, gl, gl0;
+    long k, n;
+    double a[10] = {8.333333333333333e-02,-2.777777777777778e-03,
+         7.936507936507937e-04,-5.952380952380952e-04,
+         8.417508417508418e-04,-1.917526917526918e-03,
+         6.410256410256410e-03,-2.955065359477124e-02,
+         1.796443723688307e-01,-1.39243221690590e+00};
+    x0 = x;
+    n = 0;
+    if ((x == 1.0) || (x == 2.0)) {
+        return 0.0;
+    } else if (x <= 7.0) {
+        n = (long)(7 - x);
+        x0 = x + n;
+    }
+    x2 = 1.0/(x0*x0);
+    xp = 2*M_PI;
+    gl0 = a[9];
+    for (k=8; k>=0; k--) {
+        gl0 *= x2;
+        gl0 += a[k];
+    }
+    gl = gl0/x0 + 0.5*log(xp) + (x0-0.5)*log(x0) - x0;
+    if (x <= 7.0) {
+        for (k=1; k<=n; k++) {
+            gl -= log(x0-1.0);
+            x0 -= 1.0;
+        }
+    }
+    return gl;
+}
+'''
+
+rk_standard_exponential_definition = '''
+__device__ double rk_standard_exponential(rk_state *state) {
+    /* We use -log(1-U) since U is [0, 1) */
+    return -log(1.0 - rk_double(state));
+}
+'''
+
+rk_standard_gamma_definition = '''
+__device__ double rk_standard_gamma(rk_state *state, double shape) {
+    double b, c;
+    double U, V, X, Y;
+    if (shape == 1.0) {
+        return rk_standard_exponential(state);
+    } else if (shape < 1.0) {
+        for (;;) {
+            U = rk_double(state);
+            V = rk_standard_exponential(state);
+            if (U <= 1.0 - shape) {
+                X = pow(U, 1./shape);
+                if (X <= V) {
+                    return X;
+                }
+            } else {
+                Y = -log((1-U)/shape);
+                X = pow(1.0 - shape + shape*Y, 1./shape);
+                if (X <= (V + Y)) {
+                    return X;
+                }
+            }
+        }
+    } else {
+        b = shape - 1./3.;
+        c = 1./sqrt(9*b);
+        for (;;) {
+            do {
+                X = rk_gauss(state);
+                V = 1.0 + c*X;
+            } while (V <= 0.0);
+            V = V*V*V;
+            U = rk_double(state);
+            if (U < 1.0 - 0.0331*(X*X)*(X*X)) return (b*V);
+            if (log(U) < 0.5*X*X + b*(1. - V + log(V))) return (b*V);
+        }
+    }
+}
+'''
+
+rk_beta_definition = '''
+__device__ double rk_beta(rk_state *state, double a, double b) {
+    double Ga, Gb;
+    if ((a <= 1.0) && (b <= 1.0)) {
+        double U, V, X, Y;
+        /* Use Johnk's algorithm */
+        while (1) {
+            U = rk_double(state);
+            V = rk_double(state);
+            X = pow(U, 1.0/a);
+            Y = pow(V, 1.0/b);
+            if ((X + Y) <= 1.0) {
+                if (X +Y > 0) {
+                    return X / (X + Y);
+                } else {
+                    double logX = log(U) / a;
+                    double logY = log(V) / b;
+                    double logM = logX > logY ? logX : logY;
+                    logX -= logM;
+                    logY -= logM;
+                    return exp(logX - log(exp(logX) + exp(logY)));
+                }
+            }
+        }
+    } else {
+        Ga = rk_standard_gamma(state, a);
+        Gb = rk_standard_gamma(state, b);
+        return Ga/(Ga + Gb);
+    }
+}
+'''
+
+rk_chisquare_definition = '''
+__device__ double rk_chisquare(rk_state *state, double df) {
+    return 2.0*rk_standard_gamma(state, df/2.0);
+}
+'''
+
+rk_noncentral_chisquare_definition = '''
+__device__ double rk_noncentral_chisquare(
+    rk_state *state, double df, double nonc)
+{
+    if (nonc == 0){
+        return rk_chisquare(state, df);
+    }
+    if(1 < df)
+    {
+        const double Chi2 = rk_chisquare(state, df - 1);
+        const double N = rk_gauss(state) + sqrt(nonc);
+        return Chi2 + N*N;
+    }
+    else
+    {
+        const long i = rk_poisson(state, nonc / 2.0);
+        return rk_chisquare(state, df + 2 * i);
+    }
+}
+'''
+
+rk_f_definition = '''
+__device__ double rk_f(rk_state *state, double dfnum, double dfden) {
+    return ((rk_chisquare(state, dfnum) * dfden) /
+            (rk_chisquare(state, dfden) * dfnum));
+}
+'''
+
+rk_noncentral_f_definition = '''
+__device__ double rk_noncentral_f(
+    rk_state *state, double dfnum, double dfden, double nonc)
+{
+    double t = rk_noncentral_chisquare(state, dfnum, nonc) * dfden;
+    return t / (rk_chisquare(state, dfden) * dfnum);
+}
+'''
+
+rk_binomial_definition = '''
+__device__ long rk_binomial_btpe(rk_state *state, long n, double p) {
+    double r,q,fm,p1,xm,xl,xr,c,laml,lamr,p2,p3,p4;
+    double a,u,v,s,F,rho,t,A,nrq,x1,x2,f1,f2,z,z2,w,w2,x;
+    int m,y,k,i;
+    if (!(state->has_binomial) ||
+         (state->nsave != n) ||
+         (state->psave != p)) {
+        /* initialize */
+        state->nsave = n;
+        state->psave = p;
+        state->has_binomial = 1;
+        state->r = r = min(p, 1.0-p);
+        state->q = q = 1.0 - r;
+        state->fm = fm = n*r+r;
+        state->m = m = (long)floor(state->fm);
+        state->p1 = p1 = floor(2.195*sqrt(n*r*q)-4.6*q) + 0.5;
+        state->xm = xm = m + 0.5;
+        state->xl = xl = xm - p1;
+        state->xr = xr = xm + p1;
+        state->c = c = 0.134 + 20.5/(15.3 + m);
+        a = (fm - xl)/(fm-xl*r);
+        state->laml = laml = a*(1.0 + a/2.0);
+        a = (xr - fm)/(xr*q);
+        state->lamr = lamr = a*(1.0 + a/2.0);
+        state->p2 = p2 = p1*(1.0 + 2.0*c);
+        state->p3 = p3 = p2 + c/laml;
+        state->p4 = p4 = p3 + c/lamr;
+    } else {
+        r = state->r;
+        q = state->q;
+        fm = state->fm;
+        m = state->m;
+        p1 = state->p1;
+        xm = state->xm;
+        xl = state->xl;
+        xr = state->xr;
+        c = state->c;
+        laml = state->laml;
+        lamr = state->lamr;
+        p2 = state->p2;
+        p3 = state->p3;
+        p4 = state->p4;
+    }
+  /* sigh ... */
+  Step10:
+    nrq = n*r*q;
+    u = rk_double(state)*p4;
+    v = rk_double(state);
+    if (u > p1) goto Step20;
+    y = (long)floor(xm - p1*v + u);
+    goto Step60;
+  Step20:
+    if (u > p2) goto Step30;
+    x = xl + (u - p1)/c;
+    v = v*c + 1.0 - fabs(m - x + 0.5)/p1;
+    if (v > 1.0) goto Step10;
+    y = (long)floor(x);
+    goto Step50;
+  Step30:
+    if (u > p3) goto Step40;
+    y = (long)floor(xl + log(v)/laml);
+    if (y < 0) goto Step10;
+    v = v*(u-p2)*laml;
+    goto Step50;
+  Step40:
+    y = (long)floor(xr - log(v)/lamr);
+    if (y > n) goto Step10;
+    v = v*(u-p3)*lamr;
+  Step50:
+    k = labs(y - m);
+    if ((k > 20) && (k < ((nrq)/2.0 - 1))) goto Step52;
+    s = r/q;
+    a = s*(n+1);
+    F = 1.0;
+    if (m < y) {
+        for (i=m+1; i<=y; i++) {
+            F *= (a/i - s);
+        }
+    } else if (m > y) {
+        for (i=y+1; i<=m; i++) {
+            F /= (a/i - s);
+        }
+    }
+    if (v > F) goto Step10;
+    goto Step60;
+    Step52:
+    rho = (k/(nrq))*((k*(k/3.0 + 0.625) + 0.16666666666666666)/nrq + 0.5);
+    t = -k*k/(2*nrq);
+    A = log(v);
+    if (A < (t - rho)) goto Step60;
+    if (A > (t + rho)) goto Step10;
+    x1 = y+1;
+    f1 = m+1;
+    z = n+1-m;
+    w = n-y+1;
+    x2 = x1*x1;
+    f2 = f1*f1;
+    z2 = z*z;
+    w2 = w*w;
+    if (A > (xm*log(f1/x1)
+           + (n-m+0.5)*log(z/w)
+           + (y-m)*log(w*r/(x1*q))
+           + (13680.-(462.-(132.-(99.-140./f2)/f2)/f2)/f2)/f1/166320.
+           + (13680.-(462.-(132.-(99.-140./z2)/z2)/z2)/z2)/z/166320.
+           + (13680.-(462.-(132.-(99.-140./x2)/x2)/x2)/x2)/x1/166320.
+           + (13680.-(462.-(132.-(99.-140./w2)/w2)/w2)/w2)/w/166320.)) {
+        goto Step10;
+    }
+  Step60:
+    if (p > 0.5) {
+        y = n - y;
+    }
+    return y;
+}
+
+__device__ long rk_binomial_inversion(rk_state *state, int n, double p) {
+    double q, qn, np, px, U;
+    int X, bound;
+    if (!(state->has_binomial) ||
+         (state->nsave != n) ||
+         (state->psave != p)) {
+        state->nsave = n;
+        state->psave = p;
+        state->has_binomial = 1;
+        state->q = q = 1.0 - p;
+        state->r = qn = exp(n * log(q));
+        state->c = np = n*p;
+        state->m = bound = min((double)n, np + 10.0*sqrt(np*q + 1));
+    } else {
+        q = state->q;
+        qn = state->r;
+        np = state->c;
+        bound = state->m;
+    }
+    X = 0;
+    px = qn;
+    U = rk_double(state);
+    while (U > px) {
+        X++;
+        if (X > bound) {
+            X = 0;
+            px = qn;
+            U = rk_double(state);
+        } else {
+            U -= px;
+            px  = ((n-X+1) * p * px)/(X*q);
+        }
+    }
+    return X;
+}
+
+__device__ long rk_binomial(rk_state *state, int n, double p) {
+    double q;
+    if (p <= 0.5) {
+        if (p*n <= 30.0) {
+            return rk_binomial_inversion(state, n, p);
+        } else {
+            return rk_binomial_btpe(state, n, p);
+        }
+    } else {
+        q = 1.0-p;
+        if (q*n <= 30.0) {
+            return n - rk_binomial_inversion(state, n, q);
+        } else {
+            return n - rk_binomial_btpe(state, n, q);
+        }
+    }
+}
+'''
+
+rk_poisson_mult_definition = '''
+__device__ long rk_poisson_mult(rk_state *state, double lam) {
+    long X;
+    double prod, U, enlam;
+    enlam = exp(-lam);
+    X = 0;
+    prod = 1.0;
+    while (1) {
+        U = rk_double(state);
+        prod *= U;
+        if (prod > enlam) {
+            X += 1;
+        } else {
+            return X;
+        }
+    }
+}
+'''
+
+rk_poisson_ptrs_definition = '''
+/*
+ * The transformed rejection method for generating Poisson random variables
+ * W. Hoermann
+ * Insurance: Mathematics and Economics 12, 39-45 (1993)
+ */
+#define LS2PI 0.91893853320467267
+#define TWELFTH 0.083333333333333333333333
+__device__ long rk_poisson_ptrs(rk_state *state, double lam) {
+    long k;
+    double U, V, slam, loglam, a, b, invalpha, vr, us;
+    slam = sqrt(lam);
+    loglam = log(lam);
+    b = 0.931 + 2.53*slam;
+    a = -0.059 + 0.02483*b;
+    invalpha = 1.1239 + 1.1328/(b-3.4);
+    vr = 0.9277 - 3.6224/(b-2);
+    while (1) {
+        U = rk_double(state) - 0.5;
+        V = rk_double(state);
+        us = 0.5 - fabs(U);
+        k = (long)floor((2*a/us + b)*U + lam + 0.43);
+        if ((us >= 0.07) && (V <= vr)) {
+            return k;
+        }
+        if ((k < 0) ||
+            ((us < 0.013) && (V > us))) {
+            continue;
+        }
+        if ((log(V) + log(invalpha) - log(a/(us*us)+b)) <=
+            (-lam + k*loglam - loggam(k+1))) {
+            return k;
+        }
+    }
+}
+'''
+
+rk_poisson_definition = '''
+__device__ long rk_poisson(rk_state *state, double lam) {
+    if (lam >= 10) {
+        return rk_poisson_ptrs(state, lam);
+    } else if (lam == 0) {
+        return 0;
+    } else {
+        return rk_poisson_mult(state, lam);
+    }
+}
+'''
+
+rk_standard_t_definition = '''
+__device__ double rk_standard_t(rk_state *state, double df) {
+    return sqrt(df/2)*rk_gauss(state)/sqrt(rk_standard_gamma(state, df/2));
+}
+'''
+
+rk_vonmises_definition = '''
+__device__ double rk_vonmises(rk_state *state, double mu, double kappa)
+{
+    double s;
+    double U, V, W, Y, Z;
+    double result, mod;
+    int neg;
+
+    if (kappa < 1e-8)
+    {
+        return M_PI * (2*rk_double(state)-1);
+    }
+    else
+    {
+        /* with double precision rho is zero until 1.4e-8 */
+        if (kappa < 1e-5) {
+            /*
+             * second order taylor expansion around kappa = 0
+             * precise until relatively large kappas as second order is 0
+             */
+            s = (1./kappa + kappa);
+        }
+        else {
+            double r = 1 + sqrt(1 + 4*kappa*kappa);
+            double rho = (r - sqrt(2*r)) / (2*kappa);
+            s = (1 + rho*rho)/(2*rho);
+        }
+
+        while (1)
+        {
+        U = rk_double(state);
+            Z = cos(M_PI*U);
+            W = (1 + s*Z)/(s + Z);
+            Y = kappa * (s - W);
+            V = rk_double(state);
+            if ((Y*(2-Y) - V >= 0) || (log(Y/V)+1 - Y >= 0))
+            {
+                break;
+            }
+        }
+
+        U = rk_double(state);
+
+        result = acos(W);
+        if (U < 0.5)
+        {
+        result = -result;
+        }
+        result += mu;
+        neg = (result < 0);
+        mod = fabs(result);
+        mod = (fmod(mod+M_PI, 2*M_PI)-M_PI);
+        if (neg)
+        {
+            mod *= -1;
+        }
+
+        return mod;
+    }
+}
+'''
+
+rk_zipf_definition = '''
+__device__ long rk_zipf(rk_state *state, double a)
+{
+    double am1, b;
+
+    am1 = a - 1.0;
+    b = pow(2.0, am1);
+    while (1) {
+        double T, U, V, X;
+
+        U = 1.0 - rk_double(state);
+        V = rk_double(state);
+        X = floor(pow(U, -1.0/am1));
+
+        if (X < 1.0) {
+            continue;
+        }
+
+        T = pow(1.0 + 1.0/X, am1);
+        if (V*X*(T - 1.0)/(b - 1.0) <= T/b) {
+            return (long)X;
+        }
+    }
+}
+'''
+
+rk_geometric_definition = '''
+__device__ long rk_geometric_search(rk_state *state, double p) {
+    double U;
+    long X;
+    double sum, prod, q;
+    X = 1;
+    sum = prod = p;
+    q = 1.0 - p;
+    U = rk_double(state);
+    while (U > sum) {
+        prod *= q;
+        sum += prod;
+        X++;
+    }
+    return X;
+}
+
+__device__ long rk_geometric_inversion(rk_state *state, double p) {
+    return (long)ceil(log(1.0-rk_double(state))/log(1.0-p));
+}
+
+__device__ long rk_geometric(rk_state *state, double p) {
+    if (p >= 0.333333333333333333333333) {
+        return rk_geometric_search(state, p);
+    } else {
+        return rk_geometric_inversion(state, p);
+    }
+}
+'''
+
+# min and max for the long type are not defined in cuda90 but in cuda75.
+long_min_max_definition = '''
+__device__ long long_min(long a, long b)
+{
+    return a < b ? a : b;
+}
+
+__device__ long long_max(long a, long b)
+{
+    return a > b ? a : b;
+}
+'''
+
+rk_hypergeometric_definition = '''
+__device__ long rk_hypergeometric_hyp(
+    rk_state *state, long good, long bad, long sample)
+{
+    long d1, K, Z;
+    double d2, U, Y;
+
+    d1 = bad + good - sample;
+    d2 = (double)long_min(bad, good);
+
+    Y = d2;
+    K = sample;
+    while (Y > 0.0)
+    {
+        U = rk_double(state);
+        Y -= (long)floor(U + Y/(d1 + K));
+        K--;
+        if (K == 0) break;
+    }
+    Z = (long)(d2 - Y);
+    if (good > bad) Z = sample - Z;
+    return Z;
+}
+
+/* D1 = 2*sqrt(2/e) */
+/* D2 = 3 - 2*sqrt(3/e) */
+#define D1 1.7155277699214135
+#define D2 0.8989161620588988
+__device__ long rk_hypergeometric_hrua(
+    rk_state *state, long good, long bad, long sample)
+{
+    long mingoodbad, maxgoodbad, popsize, m, d9;
+    double d4, d5, d6, d7, d8, d10, d11;
+    long Z;
+    double T, W, X, Y;
+
+    mingoodbad = long_min(good, bad);
+    popsize = good + bad;
+    maxgoodbad = long_max(good, bad);
+    m = long_min(sample, popsize - sample);
+    d4 = ((double)mingoodbad) / popsize;
+    d5 = 1.0 - d4;
+    d6 = m*d4 + 0.5;
+    d7 = sqrt((double)(popsize - m) * sample * d4 * d5 / (popsize - 1) + 0.5);
+    d8 = D1*d7 + D2;
+    d9 = (long)floor((double)(m + 1) * (mingoodbad + 1) / (popsize + 2));
+    d10 = (loggam(d9+1) + loggam(mingoodbad-d9+1) + loggam(m-d9+1) +
+           loggam(maxgoodbad-m+d9+1));
+    d11 = min(long_min(m, mingoodbad)+1.0, floor(d6+16*d7));
+    /* 16 for 16-decimal-digit precision in D1 and D2 */
+
+    while (1)
+    {
+        X = rk_double(state);
+        Y = rk_double(state);
+        W = d6 + d8*(Y- 0.5)/X;
+
+        /* fast rejection: */
+        if ((W < 0.0) || (W >= d11)) continue;
+
+        Z = (long)floor(W);
+        T = d10 - (loggam(Z+1) + loggam(mingoodbad-Z+1) + loggam(m-Z+1) +
+                   loggam(maxgoodbad-m+Z+1));
+
+        /* fast acceptance: */
+        if ((X*(4.0-X)-3.0) <= T) break;
+
+        /* fast rejection: */
+        if (X*(X-T) >= 1) continue;
+
+        if (2.0*log(X) <= T) break;  /* acceptance */
+    }
+
+    /* this is a correction to HRUA* by Ivan Frohne in rv.py */
+    if (good > bad) Z = m - Z;
+
+    /* another fix from rv.py to allow sample to exceed popsize/2 */
+    if (m < sample) Z = good - Z;
+
+    return Z;
+}
+#undef D1
+#undef D2
+
+__device__ long rk_hypergeometric(
+    rk_state *state, long good, long bad, long sample)
+{
+    if (sample > 10)
+    {
+        return rk_hypergeometric_hrua(state, good, bad, sample);
+    } else
+    {
+        return rk_hypergeometric_hyp(state, good, bad, sample);
+    }
+}
+'''
+
+rk_logseries_definition = '''
+__device__ long rk_logseries(rk_state *state, double p)
+{
+    double q, r, U, V;
+    long result;
+
+    r = log(1.0 - p);
+
+    while (1) {
+        V = rk_double(state);
+        if (V >= p) {
+            return 1;
+        }
+        U = rk_double(state);
+        q = 1.0 - exp(r*U);
+        if (V <= q*q) {
+            result = (long)floor(1 + log(V)/log(q));
+            if (result < 1) {
+                continue;
+            }
+            else {
+                return result;
+            }
+        }
+        if (V >= q) {
+            return 1;
+        }
+        return 2;
+    }
+}
+'''
+
+rk_gauss_definition = '''
+__device__ double rk_gauss(rk_state *state) {
+    if (state->has_gauss) {
+        const double tmp = state->gauss;
+        state->gauss = 0;
+        state->has_gauss = 0;
+        return tmp;
+    } else {
+        double f, x1, x2, r2;
+        do {
+            x1 = 2.0*rk_double(state) - 1.0;
+            x2 = 2.0*rk_double(state) - 1.0;
+            r2 = x1*x1 + x2*x2;
+        }
+        while (r2 >= 1.0 || r2 == 0.0);
+        /* Box-Muller transform */
+        f = sqrt(-2.0*log(r2)/r2);
+        /* Keep for next call */
+        state->gauss = f*x1;
+        state->has_gauss = 1;
+        return f*x2;
+    }
+}
+'''
+
+open_uniform_definition = '''
+__device__ void open_uniform(rk_state *state, double *U) {
+    do {
+        *U = rk_double(state);
+    } while (*U <= 0.0 || *U >= 1.0);
+}
+
+__device__ void open_uniform(rk_state *state, float *U) {
+    do {
+        *U = rk_double(state);
+    } while (*U <= 0.0 || *U >= 1.0);
+}
+'''
+
+definitions = [
+    rk_basic_definition, rk_gauss_definition,
+    rk_standard_exponential_definition, rk_standard_gamma_definition,
+    rk_beta_definition]
+beta_kernel = _core.ElementwiseKernel(
+    'S a, T b, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_beta(&internal_state, a, b);
+    ''',
+    'cupy_beta_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = [
+    rk_use_binominal, rk_basic_definition, rk_binomial_definition]
+binomial_kernel = _core.ElementwiseKernel(
+    'S n, T p, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_binomial(&internal_state, n, p);
+    ''',
+    'cupy_binomial_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = \
+    [rk_basic_definition, rk_gauss_definition,
+     rk_standard_exponential_definition, rk_standard_gamma_definition,
+     rk_standard_t_definition]
+standard_t_kernel = _core.ElementwiseKernel(
+    'S df, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_standard_t(&internal_state, df);
+    ''',
+    'cupy_standard_t_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = \
+    [rk_basic_definition, rk_gauss_definition,
+     rk_standard_exponential_definition, rk_standard_gamma_definition,
+     rk_chisquare_definition]
+chisquare_kernel = _core.ElementwiseKernel(
+    'T df, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_chisquare(&internal_state, df);
+    ''',
+    'cupy_chisquare_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = \
+    [rk_basic_definition, rk_gauss_definition,
+     rk_standard_exponential_definition, rk_standard_gamma_definition,
+     rk_chisquare_definition, rk_f_definition]
+f_kernel = _core.ElementwiseKernel(
+    'S dfnum, T dfden, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_f(&internal_state, dfnum, dfden);
+    ''',
+    'cupy_f_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = \
+    [rk_basic_definition, rk_geometric_definition]
+geometric_kernel = _core.ElementwiseKernel(
+    'T p, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_geometric(&internal_state, p);
+    ''',
+    'cupy_geometric_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = \
+    [rk_basic_definition, loggam_definition, long_min_max_definition,
+     rk_hypergeometric_definition]
+hypergeometric_kernel = _core.ElementwiseKernel(
+    'S good, T bad, U sample, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_hypergeometric(&internal_state, good, bad, sample);
+    ''',
+    'cupy_hypergeometric_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = \
+    [rk_basic_definition, rk_logseries_definition]
+logseries_kernel = _core.ElementwiseKernel(
+    'T p, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_logseries(&internal_state, p);
+    ''',
+    'cupy_logseries_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = [
+    rk_basic_definition, loggam_definition, rk_gauss_definition,
+    rk_standard_exponential_definition, rk_standard_gamma_definition,
+    rk_chisquare_definition, rk_poisson_mult_definition,
+    rk_poisson_ptrs_definition, rk_poisson_definition,
+    rk_noncentral_chisquare_definition]
+noncentral_chisquare_kernel = _core.ElementwiseKernel(
+    'S df, T nonc, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_noncentral_chisquare(&internal_state, df, nonc);
+    ''',
+    'cupy_noncentral_chisquare_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = [
+    rk_basic_definition, loggam_definition, rk_gauss_definition,
+    rk_standard_exponential_definition, rk_standard_gamma_definition,
+    rk_chisquare_definition, rk_poisson_mult_definition,
+    rk_poisson_ptrs_definition, rk_poisson_definition,
+    rk_noncentral_chisquare_definition, rk_noncentral_f_definition]
+noncentral_f_kernel = _core.ElementwiseKernel(
+    'S dfnum, T dfden, U nonc, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_noncentral_f(&internal_state, dfnum, dfden, nonc);
+    ''',
+    'cupy_noncentral_f_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = \
+    [rk_basic_definition, loggam_definition,
+     rk_poisson_mult_definition, rk_poisson_ptrs_definition,
+     rk_poisson_definition]
+poisson_kernel = _core.ElementwiseKernel(
+    'T lam, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_poisson(&internal_state, lam);
+    ''',
+    'cupy_poisson_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = [
+    rk_basic_definition, rk_gauss_definition,
+    rk_standard_exponential_definition, rk_standard_gamma_definition]
+standard_gamma_kernel = _core.ElementwiseKernel(
+    'T shape, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_standard_gamma(&internal_state, shape);
+    ''',
+    'cupy_standard_gamma_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = [
+    rk_basic_definition, rk_vonmises_definition]
+vonmises_kernel = _core.ElementwiseKernel(
+    'S mu, T kappa, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_vonmises(&internal_state, mu, kappa);
+    ''',
+    'cupy_vonmises_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = [
+    rk_basic_definition, rk_zipf_definition]
+zipf_kernel = _core.ElementwiseKernel(
+    'T a, uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    y = rk_zipf(&internal_state, a);
+    ''',
+    'cupy_zipf_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
+
+definitions = [
+    rk_basic_definition, open_uniform_definition]
+open_uniform_kernel = _core.ElementwiseKernel(
+    'uint64 seed', 'Y y',
+    '''
+    rk_seed(seed + i, &internal_state);
+    open_uniform(&internal_state, &y);
+    ''',
+    'cupy_open_uniform_kernel',
+    preamble=''.join(definitions),
+    loop_prep='rk_state internal_state;'
+)
diff --git a/cupy/random/_permutations.py b/cupy/random/_permutations.py
new file mode 100644
index 0000000..d9ca931
--- /dev/null
+++ b/cupy/random/_permutations.py
@@ -0,0 +1,31 @@
+from cupy.random import _generator
+
+
+def shuffle(a):
+    """Shuffles an array.
+
+    Args:
+        a (cupy.ndarray): The array to be shuffled.
+
+    .. seealso:: :meth:`numpy.random.shuffle`
+
+    """
+    rs = _generator.get_random_state()
+    return rs.shuffle(a)
+
+
+def permutation(a):
+    """Returns a permuted range or a permutation of an array.
+
+    Args:
+        a (int or cupy.ndarray): The range or the array to be shuffled.
+
+    Returns:
+        cupy.ndarray: If `a` is an integer, it is permutation range between 0
+        and `a` - 1.
+        Otherwise, it is a permutation of `a`.
+
+    .. seealso:: :meth:`numpy.random.permutation`
+    """
+    rs = _generator.get_random_state()
+    return rs.permutation(a)
diff --git a/cupy/random/_sample.py b/cupy/random/_sample.py
new file mode 100644
index 0000000..5dc63d1
--- /dev/null
+++ b/cupy/random/_sample.py
@@ -0,0 +1,239 @@
+from cupy import _core
+from cupy._creation import basic
+from cupy.random import _distributions
+from cupy.random import _generator
+
+
+def rand(*size, **kwarg):
+    """Returns an array of uniform random values over the interval ``[0, 1)``.
+
+    Each element of the array is uniformly distributed on the half-open
+    interval ``[0, 1)``. All elements are identically and independently
+    distributed (i.i.d.).
+
+    Args:
+        size (ints): The shape of the array.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed. The default is
+            :class:`numpy.float64`.
+
+    Returns:
+        cupy.ndarray: A random array.
+
+    .. seealso:: :meth:`numpy.random.rand`
+
+    .. admonition:: Example
+
+       .. code-block:: python
+
+          >>> cupy.random.rand(3, 2)
+          array([[0.86476479, 0.05633727],   # random
+                 [0.27283185, 0.38255354],   # random
+                 [0.16592278, 0.75150313]])  # random
+
+          >>> cupy.random.rand(3, 2, dtype=cupy.float32)
+          array([[0.9672306 , 0.9590486 ],                  # random
+                 [0.6851264 , 0.70457625],                  # random
+                 [0.22382522, 0.36055237]], dtype=float32)  # random
+
+    """
+    dtype = kwarg.pop('dtype', float)
+    if kwarg:
+        raise TypeError('rand() got unexpected keyword arguments %s'
+                        % ', '.join(kwarg.keys()))
+    return random_sample(size=size, dtype=dtype)
+
+
+def randn(*size, **kwarg):
+    """Returns an array of standard normal random values.
+
+    Each element of the array is normally distributed with zero mean and unit
+    variance. All elements are identically and independently distributed
+    (i.i.d.).
+
+    Args:
+        size (ints): The shape of the array.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+            The default is :class:`numpy.float64`.
+
+    Returns:
+        cupy.ndarray: An array of standard normal random values.
+
+    .. seealso:: :meth:`numpy.random.randn`
+
+    .. admonition:: Example
+
+       .. code-block:: python
+
+          >>> cupy.random.randn(3, 2)
+          array([[0.41193321, 1.59579542],   # random
+                 [0.47904589, 0.18566376],   # random
+                 [0.59748424, 2.32602829]])  # random
+
+          >>> cupy.random.randn(3, 2, dtype=cupy.float32)
+          array([[ 0.1373886 ,  2.403238  ],                  # random
+                 [ 0.84020025,  1.5089266 ],                  # random
+                 [-1.2268474 , -0.48219103]], dtype=float32)  # random
+
+    """
+    dtype = kwarg.pop('dtype', float)
+    if kwarg:
+        raise TypeError('randn() got unexpected keyword arguments %s'
+                        % ', '.join(kwarg.keys()))
+    return _distributions.normal(size=size, dtype=dtype)
+
+
+def randint(low, high=None, size=None, dtype='l'):
+    """Returns a scalar or an array of integer values over ``[low, high)``.
+
+    Each element of returned values are independently sampled from
+    uniform distribution over left-close and right-open interval
+    ``[low, high)``.
+
+    Args:
+        low (int): If ``high`` is not ``None``,
+            it is the lower bound of the interval.
+            Otherwise, it is the **upper** bound of the interval
+            and lower bound of the interval is set to ``0``.
+        high (int): Upper bound of the interval.
+        size (None or int or tuple of ints): The shape of returned value.
+        dtype: Data type specifier.
+
+    Returns:
+        int or cupy.ndarray of ints: If size is ``None``,
+        it is single integer sampled.
+        If size is integer, it is the 1D-array of length ``size`` element.
+        Otherwise, it is the array whose shape specified by ``size``.
+    """
+    rs = _generator.get_random_state()
+    return rs.randint(low, high, size, dtype)
+
+
+def random_integers(low, high=None, size=None):
+    """Return a scalar or an array of integer values over ``[low, high]``
+
+    Each element of returned values are independently sampled from
+    uniform distribution over closed interval ``[low, high]``.
+
+    Args:
+        low (int): If ``high`` is not ``None``,
+            it is the lower bound of the interval.
+            Otherwise, it is the **upper** bound of the interval
+            and the lower bound is set to ``1``.
+        high (int): Upper bound of the interval.
+        size (None or int or tuple of ints): The shape of returned value.
+
+    Returns:
+        int or cupy.ndarray of ints: If size is ``None``,
+        it is single integer sampled.
+        If size is integer, it is the 1D-array of length ``size`` element.
+        Otherwise, it is the array whose shape specified by ``size``.
+    """
+    if high is None:
+        high = low
+        low = 1
+    return randint(low, high + 1, size)
+
+
+def random_sample(size=None, dtype=float):
+    """Returns an array of random values over the interval ``[0, 1)``.
+
+    This is a variant of :func:`cupy.random.rand`.
+
+    Args:
+        size (int or tuple of ints): The shape of the array.
+        dtype: Data type specifier. Only :class:`numpy.float32` and
+            :class:`numpy.float64` types are allowed.
+
+    Returns:
+        cupy.ndarray: An array of uniformly distributed random values.
+
+    .. seealso:: :meth:`numpy.random.random_sample`
+
+    """
+    rs = _generator.get_random_state()
+    return rs.random_sample(size=size, dtype=dtype)
+
+
+def choice(a, size=None, replace=True, p=None):
+    """Returns an array of random values from a given 1-D array.
+
+    Each element of the returned array is independently sampled
+    from ``a`` according to ``p`` or uniformly.
+
+    .. note::
+
+       Currently ``p`` is not supported when ``replace=False``.
+
+    Args:
+        a (1-D array-like or int):
+            If an array-like,
+            a random sample is generated from its elements.
+            If an int, the random sample is generated as if ``a`` was
+            ``cupy.arange(n)``
+        size (int or tuple of ints): The shape of the array.
+        replace (boolean): Whether the sample is with or without replacement.
+        p (1-D array-like):
+            The probabilities associated with each entry in ``a``.
+            If not given the sample assumes a uniform distribution over all
+            entries in ``a``.
+
+    Returns:
+        cupy.ndarray: An array of ``a`` values distributed according to
+        ``p`` or uniformly.
+
+    .. seealso:: :meth:`numpy.random.choice`
+
+    """
+    rs = _generator.get_random_state()
+    return rs.choice(a, size, replace, p)
+
+
+_multinominal_kernel = _core.ElementwiseKernel(
+    'int64 x, int32 p, int32 n', 'raw U ys',
+    'atomicAdd(&ys[i / n * p + x], U(1))',
+    'cupy_random_multinomial')
+
+
+def multinomial(n, pvals, size=None):
+    """Returns an array from multinomial distribution.
+
+    Args:
+        n (int): Number of trials.
+        pvals (cupy.ndarray): Probabilities of each of the ``p`` different
+            outcomes. The sum of these values must be 1.
+        size (int or tuple of ints or None): Shape of a sample in each trial.
+            For example when ``size`` is ``(a, b)``, shape of returned value is
+            ``(a, b, p)`` where ``p`` is ``len(pvals)``.
+            If ``size`` is ``None``, it is treated as ``()``. So, shape of
+            returned value is ``(p,)``.
+
+    Returns:
+        cupy.ndarray: An array drawn from multinomial distribution.
+
+    .. note::
+       It does not support ``sum(pvals) < 1`` case.
+
+    .. seealso:: :meth:`numpy.random.multinomial`
+    """
+
+    if size is None:
+        m = 1
+        size = ()
+    elif isinstance(size, int):
+        m = size
+        size = (size,)
+    else:
+        size = tuple(size)
+        m = 1
+        for x in size:
+            m *= x
+
+    p = len(pvals)
+    shape = size + (p,)
+    ys = basic.zeros(shape, 'l')
+    if ys.size > 0:
+        xs = choice(p, p=pvals, size=n * m)
+        _multinominal_kernel(xs, p, n, ys)
+    return ys
diff --git a/cupy/random/cupy_distributions.cu b/cupy/random/cupy_distributions.cu
new file mode 100644
index 0000000..11c7dfc
--- /dev/null
+++ b/cupy/random/cupy_distributions.cu
@@ -0,0 +1,908 @@
+#include <stdio.h>
+#include <stdexcept>
+#include <utility>
+#include <iostream>
+#include <stdint.h>
+#include <type_traits>
+
+
+#include "cupy_distributions.cuh"
+
+
+// avoid explicit compilation for hip<4.3
+#if !defined(CUPY_USE_HIP) || defined(COMPILE_FOR_HIP)
+
+
+template<typename CURAND_TYPE>
+struct curand_pseudo_state {
+    // Valid for  XORWOW and MRG32k3a
+    CURAND_TYPE _state;
+    CURAND_TYPE* _state_ptr;
+
+    __device__ curand_pseudo_state(int id, intptr_t state) {
+        _state_ptr = reinterpret_cast<CURAND_TYPE*>(state) + id;
+        _state = *_state_ptr;
+    }
+
+    __device__ ~curand_pseudo_state() {
+        *_state_ptr = _state;
+    }
+
+    __device__ uint32_t rk_int() {
+        return curand(&_state);
+    }
+
+    __device__ double rk_double() {
+        // Curand returns (0, 1] while the functions
+        // below rely on [0, 1)
+#ifdef CUPY_USE_HIP
+        double r = curand_uniform(&_state);
+#else
+        double r = curand_uniform_double(&_state);
+#endif
+        if (r >= 1.0) { 
+           r = 0.0;
+        }
+        return r;
+    }
+
+    __device__ float rk_float() {
+        // Curand returns (0, 1] while the functions
+        // below rely on [0, 1)
+        float r = curand_uniform(&_state);
+        if (r >= 1.0) { 
+           r = 0.0;
+        }
+        return r;
+    }
+
+    __device__ double rk_normal() {
+        return curand_normal_double(&_state);
+    }
+
+    __device__ float rk_normal_float() {
+        return curand_normal(&_state);
+    }
+};
+
+
+// This design is the same as the dtypes one
+template <typename F, typename... Ts>
+void generator_dispatcher(int generator_id, F f, Ts&&... args) {
+   switch(generator_id) {
+       case CURAND_XOR_WOW: return f.template operator()<curand_pseudo_state<curandState>>(std::forward<Ts>(args)...);
+       case CURAND_MRG32k3a: return f.template operator()<curand_pseudo_state<curandStateMRG32k3a>>(std::forward<Ts>(args)...);
+       case CURAND_PHILOX_4x32_10: return f.template operator()<curand_pseudo_state<curandStatePhilox4_32_10_t>>(std::forward<Ts>(args)...);
+       default: throw std::runtime_error("Unknown random generator");
+   }
+}
+
+
+template<typename T>
+__global__ void init_curand(intptr_t state, uint64_t seed, ssize_t size) {
+    int id = threadIdx.x + blockIdx.x * blockDim.x;
+    /* Each thread gets same seed, a different sequence
+       number, no offset */
+    T curand_state(id, state);
+    if (id < size) {
+        curand_init(seed, id, 0, &curand_state._state);    
+    }
+}
+
+struct initialize_launcher {
+    initialize_launcher(ssize_t size, cudaStream_t stream) : _size(size), _stream(stream) {
+    }
+    template<typename T, typename... Args>
+    void operator()(Args&&... args) { 
+        int tpb = 256;
+        int bpg =  (_size + tpb - 1) / tpb;
+        init_curand<T><<<bpg, tpb, 0, _stream>>>(std::forward<Args>(args)...);
+    }
+    ssize_t _size;
+    cudaStream_t _stream;
+};
+
+void init_curand_generator(int generator, intptr_t state_ptr, uint64_t seed, ssize_t size, intptr_t stream) {
+    // state_ptr is a device ptr
+    initialize_launcher launcher(size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state_ptr, seed, size);
+}
+
+template<typename T>
+__device__ double rk_standard_exponential(T& state) {
+    /* We use -log(1-U) since U is [0, 1) */
+    return -log(1.0 - state.rk_double());
+}
+
+template<typename T>
+__device__ double rk_standard_normal(T& state) {
+    return state.rk_normal();
+}
+
+template<typename T>
+__device__ float rk_standard_normal_float(T& state) {
+    return state.rk_normal_float();
+}
+
+template<typename T>
+__device__ double rk_standard_gamma(T& state, double shape) {
+    double b, c;
+    double U, V, X, Y;
+    if (shape == 1.0) {
+        return rk_standard_exponential(state);
+    } else if (shape < 0.0) {
+        return 0.0;
+    } else if (shape < 1.0) {
+        for (;;) {
+            U = state.rk_double();
+            V = rk_standard_exponential(state);
+            if (U <= 1.0 - shape) {
+                X = pow(U, 1./shape);
+                if (X <= V) {
+                    return X;
+                }
+            } else {
+                Y = -log((1-U)/shape);
+                X = pow(1.0 - shape + shape*Y, 1./shape);
+                if (X <= (V + Y)) {
+                    return X;
+                }
+            }
+        }
+    } else {
+        b = shape - 1./3.;
+        c = 1./sqrt(9*b);
+        for (;;) {
+            do {
+                X = state.rk_normal();
+                V = 1.0 + c*X;
+            } while (V <= 0.0);
+            V = V*V*V;
+            U = state.rk_double();
+            if (U < 1.0 - 0.0331*(X*X)*(X*X)) return (b*V);
+            if (log(U) < 0.5*X*X + b*(1. - V + log(V))) return (b*V);
+        }
+    }
+}
+
+template<typename T>
+__device__ double rk_beta(T& state, double a, double b) {
+    double Ga, Gb;
+    if ((a <= 1.0) && (b <= 1.0)) {
+        double U, V, X, Y;
+        /* Use Johnk's algorithm */
+        while (1) {
+            U = state.rk_double();
+            V = state.rk_double();
+            X = pow(U, 1.0/a);
+            Y = pow(V, 1.0/b);
+            if ((X + Y) <= 1.0) {
+                if (X +Y > 0) {
+                    return X / (X + Y);
+                } else {
+                    double logX = log(U) / a;
+                    double logY = log(V) / b;
+                    double logM = logX > logY ? logX : logY;
+                    logX -= logM;
+                    logY -= logM;
+                    return exp(logX - log(exp(logX) + exp(logY)));
+                }
+            }
+        }
+    } else {
+        Ga = rk_standard_gamma(state, a);
+        Gb = rk_standard_gamma(state, b);
+        return Ga/(Ga + Gb);
+    }
+}
+
+template<typename T>
+__device__ int64_t rk_geometric_search(T& state, double p) {
+    double U, sum, prod, q;
+    int64_t X;
+    X = 1;
+    sum = prod = p;
+    q = 1.0 - p;
+    U = state.rk_double();
+    while (U > sum) {
+        prod *= q;
+        sum += prod;
+        X++;
+    }
+    return X;
+}
+
+template<typename T>
+__device__ int64_t rk_geometric_inversion(T& state, double p) {
+    return ceil(log(1.0-state.rk_double())/log(1.0-p));
+}
+
+template<typename T>
+__device__ int64_t rk_geometric(T& state, double p) {
+    if (p >= 0.333333333333333333333333) {
+        return rk_geometric_search(state, p);
+    } else {
+        return rk_geometric_inversion(state, p);
+    }
+}
+
+__device__ double loggam(double x) {
+    double x0, x2, xp, gl, gl0;
+    long k, n;
+    double a[10] = {8.333333333333333e-02,-2.777777777777778e-03,
+         7.936507936507937e-04,-5.952380952380952e-04,
+         8.417508417508418e-04,-1.917526917526918e-03,
+         6.410256410256410e-03,-2.955065359477124e-02,
+         1.796443723688307e-01,-1.39243221690590e+00};
+    x0 = x;
+    n = 0;
+    if ((x == 1.0) || (x == 2.0)) {
+        return 0.0;
+    } else if (x <= 7.0) {
+        n = (long)(7 - x);
+        x0 = x + n;
+    }
+    x2 = 1.0/(x0*x0);
+    xp = 2*M_PI;
+    gl0 = a[9];
+    for (k=8; k>=0; k--) {
+        gl0 *= x2;
+        gl0 += a[k];
+    }
+    gl = gl0/x0 + 0.5*log(xp) + (x0-0.5)*log(x0) - x0;
+    if (x <= 7.0) {
+        for (k=1; k<=n; k++) {
+            gl -= log(x0-1.0);
+            x0 -= 1.0;
+        }
+    }
+    return gl;
+}
+
+template<typename T>
+__device__ int64_t rk_hypergeometric_hyp(T& state, int64_t good, int64_t bad, int64_t sample) {
+    int64_t d1, K, Z;
+    double d2, U, Y;
+
+    d1 = bad + good - sample;
+    d2 = min(bad, good);
+
+    Y = d2;
+    K = sample;
+    while (Y > 0.0)
+    {
+        U = state.rk_double();
+        Y -= (int64_t)floor(U + Y/(d1 + K));
+        K--;
+        if (K == 0) break;
+    }
+    Z = (int64_t)(d2 - Y);
+    if (good > bad) Z = sample - Z;
+    return Z;
+}
+
+
+template<typename T>
+__device__ int64_t rk_hypergeometric_hrua(T& state, int64_t good, int64_t bad, int64_t sample) {
+
+    int64_t mingoodbad, maxgoodbad, popsize, m, d9;
+    int64_t Z;
+    double d4, d5, d6, d7, d8, d10, d11, U, W, X, Y;
+    double D1=1.7155277699214135, D2=0.8989161620588988;
+
+    mingoodbad = min(good, bad);
+    popsize = good + bad;
+    maxgoodbad = max(good, bad);
+    m = min(sample, popsize - sample);
+    d4 = ((double)mingoodbad) / popsize;
+    d5 = 1.0 - d4;
+    d6 = m*d4 + 0.5;
+    d7 = sqrt((double)(popsize - m) * sample * d4 * d5 / (popsize - 1) + 0.5);
+    d8 = D1*d7 + D2;
+    d9 = (int64_t)floor((double)(m + 1) * (mingoodbad + 1) / (popsize + 2));
+    d10 = (loggam(d9+1) + loggam(mingoodbad-d9+1) + loggam(m-d9+1) +
+	   loggam(maxgoodbad-m+d9+1));
+    d11 = min(min(m, mingoodbad)+1.0, floor(d6+16*d7));
+    /* 16 for 16-decimal-digit precision in D1 and D2 */
+
+    while (1) {
+        X = state.rk_double();
+        Y = state.rk_double();
+        W = d6 + d8*(Y- 0.5)/X;
+
+        if ((W < 0.0) || (W >= d11)) continue;
+
+        Z = (int64_t)floor(W);
+        U = d10 - (loggam(Z+1) + loggam(mingoodbad-Z+1) + loggam(m-Z+1) +
+                   loggam(maxgoodbad-m+Z+1));
+
+        if ((X*(4.0-X)-3.0) <= U) break;
+
+        if (X*(X-U) >= 1) continue;
+
+        if (2.0*log(X) <= U) break;
+    }
+
+    if (good > bad) Z = m - Z;
+    if (m < sample) Z = good - Z;
+
+    return Z;
+}
+
+template<typename T>
+__device__ int64_t rk_hypergeometric(T& state, int64_t good, int64_t bad, int64_t sample) {
+    if (sample > 10) {
+        return rk_hypergeometric_hrua(state, good, bad, sample);
+    }
+    else {
+        return rk_hypergeometric_hyp(state, good, bad, sample);
+    }
+}
+
+template<typename T>
+__device__ int64_t rk_logseries(T& state, double p)
+{
+    double q, r, U, V;
+    int64_t result;
+
+    r = log(1.0 - p);
+
+    while (1) {
+        V = state.rk_double();
+        if (V >= p) {
+            return 1;
+        }
+        U = state.rk_double();
+        q = 1.0 - exp(r*U);
+        if (V <= q*q) {
+            result = floor(1 + log(V)/log(q));
+            if (result < 1) {
+                continue;
+            }
+            else {
+                return result;
+            }
+        }
+        if (V >= q) {
+            return 1;
+        }
+        return 2;
+    }
+}
+
+template<typename T>
+__device__ int64_t rk_poisson_mult(T& state, double lam) {
+    int64_t X;
+    double prod, U, enlam;
+    enlam = exp(-lam);
+    X = 0;
+    prod = 1.0;
+    while (1) {
+        U = state.rk_double();
+        prod *= U;
+        if (prod > enlam) {
+            X += 1;
+        } else {
+            return X;
+        }
+    }
+}
+
+template<typename T>
+__device__ int64_t rk_poisson_ptrs(T& state, double lam) {
+    int64_t k;
+    double U, V, slam, loglam, a, b, invalpha, vr, us;
+    slam = sqrt(lam);
+    loglam = log(lam);
+    b = 0.931 + 2.53*slam;
+    a = -0.059 + 0.02483*b;
+    invalpha = 1.1239 + 1.1328/(b-3.4);
+    vr = 0.9277 - 3.6224/(b-2);
+    while (1) {
+        U = state.rk_double() - 0.5;
+        V = state.rk_double();
+        us = 0.5 - fabs(U);
+        k = (int64_t)floor((2*a/us + b)*U + lam + 0.43);
+        if ((us >= 0.07) && (V <= vr)) {
+            return k;
+        }
+        if ((k < 0) ||
+            ((us < 0.013) && (V > us))) {
+            continue;
+        }
+        if ((log(V) + log(invalpha) - log(a/(us*us)+b)) <=
+            (-lam + k*loglam - loggam(k+1))) {
+            return k;
+        }
+    }
+}
+
+template<typename T>
+__device__ int64_t rk_poisson(T& state, double lam) {
+    if (lam >= 10) {
+        return rk_poisson_ptrs(state, lam);
+    } else if (lam == 0) {
+        return 0;
+    } else {
+        return rk_poisson_mult(state, lam);
+    }
+}
+
+template<typename T>
+__device__ uint32_t rk_raw(T& state) {
+    return state.rk_int();
+}
+
+template<typename T>
+__device__ double rk_random_uniform(T& state) {
+    return state.rk_double();
+}
+
+template<typename T>
+__device__ float rk_random_uniform_float(T& state) {
+    return state.rk_float();
+}
+
+template<typename T>
+__device__ uint32_t rk_interval_32(T& state, uint32_t mx, uint32_t mask) {
+    uint32_t sampled = state.rk_int() & mask;
+    while(sampled > mx)  {
+        sampled = state.rk_int() & mask;
+    }
+    return sampled;
+}
+
+template<typename T>
+__device__ uint64_t rk_interval_64(T& state, uint64_t  mx, uint64_t mask) {
+    uint32_t hi= state.rk_int();
+    uint32_t lo= state.rk_int();
+    uint64_t sampled = (static_cast<uint64_t>(hi) << 32 | lo)  & mask;
+    while(sampled > mx)  {
+        hi= state.rk_int();
+        lo= state.rk_int();
+        sampled = (static_cast<uint64_t>(hi) << 32 | lo) & mask;
+    }
+    return sampled;
+}
+
+template<typename T>
+__device__ int64_t rk_binomial_btpe(T& state, long n, double p, rk_binomial_state *binomial_state) {
+    double r,q,fm,p1,xm,xl,xr,c,laml,lamr,p2,p3,p4;
+    double a,u,v,s,F,rho,t,A,nrq,x1,x2,f1,f2,z,z2,w,w2,x;
+    int m,y,k,i;
+
+    if (!(binomial_state->initialized) ||
+         (binomial_state->nsave != n) ||
+         (binomial_state->psave != p)) {
+        /* initialize */
+        binomial_state->nsave = n;
+        binomial_state->psave = p;
+        binomial_state->initialized = 1;
+        binomial_state->r = r = min(p, 1.0-p);
+        binomial_state->q = q = 1.0 - r;
+        binomial_state->fm = fm = n*r+r;
+        binomial_state->m = m = (long)floor(binomial_state->fm);
+        binomial_state->p1 = p1 = floor(2.195*sqrt(n*r*q)-4.6*q) + 0.5;
+        binomial_state->xm = xm = m + 0.5;
+        binomial_state->xl = xl = xm - p1;
+        binomial_state->xr = xr = xm + p1;
+        binomial_state->c = c = 0.134 + 20.5/(15.3 + m);
+        a = (fm - xl)/(fm-xl*r);
+        binomial_state->laml = laml = a*(1.0 + a/2.0);
+        a = (xr - fm)/(xr*q);
+        binomial_state->lamr = lamr = a*(1.0 + a/2.0);
+        binomial_state->p2 = p2 = p1*(1.0 + 2.0*c);
+        binomial_state->p3 = p3 = p2 + c/laml;
+        binomial_state->p4 = p4 = p3 + c/lamr;
+    } else {
+        r = binomial_state->r;
+        q = binomial_state->q;
+        fm = binomial_state->fm;
+        m = binomial_state->m;
+        p1 = binomial_state->p1;
+        xm = binomial_state->xm;
+        xl = binomial_state->xl;
+        xr = binomial_state->xr;
+        c = binomial_state->c;
+        laml = binomial_state->laml;
+        lamr = binomial_state->lamr;
+        p2 = binomial_state->p2;
+        p3 = binomial_state->p3;
+        p4 = binomial_state->p4;
+    }
+  /* sigh ... */
+  Step10:
+    nrq = n*r*q;
+    u = state.rk_double()*p4;
+    v = state.rk_double();
+    if (u > p1) goto Step20;
+    y = (long)floor(xm - p1*v + u);
+    goto Step60;
+  Step20:
+    if (u > p2) goto Step30;
+    x = xl + (u - p1)/c;
+    v = v*c + 1.0 - fabs(m - x + 0.5)/p1;
+    if (v > 1.0) goto Step10;
+    y = (long)floor(x);
+    goto Step50;
+  Step30:
+    if (u > p3) goto Step40;
+    y = (long)floor(xl + log(v)/laml);
+    if (y < 0) goto Step10;
+    v = v*(u-p2)*laml;
+    goto Step50;
+  Step40:
+    y = (long)floor(xr - log(v)/lamr);
+    if (y > n) goto Step10;
+    v = v*(u-p3)*lamr;
+  Step50:
+    k = labs(y - m);
+    if ((k > 20) && (k < ((nrq)/2.0 - 1))) goto Step52;
+    s = r/q;
+    a = s*(n+1);
+    F = 1.0;
+    if (m < y) {
+        for (i=m+1; i<=y; i++) {
+            F *= (a/i - s);
+        }
+    } else if (m > y) {
+        for (i=y+1; i<=m; i++) {
+            F /= (a/i - s);
+        }
+    }
+    if (v > F) goto Step10;
+    goto Step60;
+    Step52:
+    rho = (k/(nrq))*((k*(k/3.0 + 0.625) + 0.16666666666666666)/nrq + 0.5);
+    t = -k*k/(2*nrq);
+    A = log(v);
+    if (A < (t - rho)) goto Step60;
+    if (A > (t + rho)) goto Step10;
+    x1 = y+1;
+    f1 = m+1;
+    z = n+1-m;
+    w = n-y+1;
+    x2 = x1*x1;
+    f2 = f1*f1;
+    z2 = z*z;
+    w2 = w*w;
+    if (A > (xm*log(f1/x1)
+           + (n-m+0.5)*log(z/w)
+           + (y-m)*log(w*r/(x1*q))
+           + (13680.-(462.-(132.-(99.-140./f2)/f2)/f2)/f2)/f1/166320.
+           + (13680.-(462.-(132.-(99.-140./z2)/z2)/z2)/z2)/z/166320.
+           + (13680.-(462.-(132.-(99.-140./x2)/x2)/x2)/x2)/x1/166320.
+           + (13680.-(462.-(132.-(99.-140./w2)/w2)/w2)/w2)/w/166320.)) {
+        goto Step10;
+    }
+  Step60:
+    if (p > 0.5) {
+        y = n - y;
+    }
+    return y;
+}
+
+template<typename T>
+__device__ int64_t rk_binomial_inversion(T& state, int n, double p, rk_binomial_state *binomial_state) {
+    double q, qn, np, px, U;
+    int X, bound;
+
+    if (!(binomial_state->initialized) ||
+         (binomial_state->nsave != n) ||
+         (binomial_state->psave != p)) {
+        binomial_state->nsave = n;
+        binomial_state->psave = p;
+        binomial_state->initialized = 1;
+        binomial_state->q = q = 1.0 - p;
+        binomial_state->r = qn = exp(n * log(q));
+        binomial_state->c = np = n*p;
+        binomial_state->m = bound = min((double)n, np + 10.0*sqrt(np*q + 1));
+    } else {
+        q = binomial_state->q;
+        qn = binomial_state->r;
+        np = binomial_state->c;
+        bound = binomial_state->m;
+    }
+    X = 0;
+    px = qn;
+    U = state.rk_double();
+    while (U > px) {
+        X++;
+        if (X > bound) {
+            X = 0;
+            px = qn;
+            U = state.rk_double();
+        } else {
+            U -= px;
+            px  = ((n-X+1) * p * px)/(X*q);
+        }
+    }
+    return X;
+}
+
+template<typename T>
+__device__ int64_t rk_binomial(T& state, int n, double p, rk_binomial_state *binomial_state) {
+    double q;
+
+    if (p <= 0.5) {
+        if (p*n <= 30.0) {
+            return rk_binomial_inversion(state, n, p, binomial_state);
+        } else {
+            return rk_binomial_btpe(state, n, p, binomial_state);
+        }
+    } else {
+        q = 1.0-p;
+        if (q*n <= 30.0) {
+            return n - rk_binomial_inversion(state, n, q, binomial_state);
+        } else {
+            return n - rk_binomial_btpe(state, n, q, binomial_state);
+        }
+    }
+}
+
+
+struct raw_functor {
+    template<typename... Args>
+    __device__ uint32_t operator () (Args&&... args) {
+        return rk_raw(args...);
+    }
+};
+
+
+struct random_uniform_functor {
+    template<typename... Args>
+    __device__ double operator () (Args&&... args) {
+        return rk_random_uniform(args...);
+    }
+};
+
+struct random_uniform_float_functor {
+    template<typename... Args>
+    __device__ float operator () (Args&&... args) {
+        return rk_random_uniform_float(args...);
+    }
+};
+
+
+struct interval_32_functor {
+    template<typename... Args>
+    __device__ uint32_t operator () (Args&&... args) {
+        return rk_interval_32(args...);
+    }
+};
+
+struct interval_64_functor {
+    template<typename... Args>
+    __device__ uint64_t operator () (Args&&... args) {
+        return rk_interval_64(args...);
+    }
+};
+
+struct beta_functor {
+    template<typename... Args>
+    __device__ double operator () (Args&&... args) {
+        return rk_beta(args...);
+    }
+};
+
+struct poisson_functor {
+    template<typename... Args>
+    __device__ int64_t operator () (Args&&... args) {
+        return rk_poisson(args...);
+    }
+};
+
+// There are several errors when trying to do this a full template
+struct exponential_functor {
+    template<typename... Args>
+    __device__ double operator () (Args&&... args) {
+        return rk_standard_exponential(args...);
+    }
+};
+
+struct geometric_functor {
+    template<typename... Args>
+    __device__ int64_t operator () (Args&&... args) {
+        return rk_geometric(args...);
+    }
+};
+
+struct hypergeometric_functor {
+    template<typename... Args>
+    __device__ int64_t operator () (Args&&... args) {
+        return rk_hypergeometric(args...);
+    }
+};
+
+struct logseries_functor {
+    template<typename... Args>
+    __device__ int64_t operator () (Args&&... args) {
+        return rk_logseries(args...);
+    }
+};
+
+struct standard_normal_functor {
+    template<typename... Args>
+    __device__ double operator () (Args&&... args) {
+        return rk_standard_normal(args...);
+    }
+};
+
+struct standard_normal_float_functor {
+    template<typename... Args>
+    __device__ float operator () (Args&&... args) {
+        return rk_standard_normal_float(args...);
+    }
+};
+
+// There are several errors when trying to do this a full template
+struct standard_gamma_functor {
+    template<typename... Args>
+    __device__ double operator () (Args&&... args) {
+        return rk_standard_gamma(args...);
+    }
+};
+
+struct binomial_functor {
+    template<typename... Args>
+    __device__ int64_t operator () (Args&&... args) {
+        return rk_binomial(args...);
+    }
+};
+
+// The following templates are used to unwrap arrays into an elementwise
+// approach, the array is `_array_data` in `cupy/random/_generator_api.pyx`.
+// When a pointer to `array_data<T>` is present in the variadic Args, it will
+// be replaced by the value of pointer[thread_id]
+
+template<typename T>
+struct array_data {};  // opaque type always used as a pointer type
+
+template<typename T>
+__device__ T get_index(array_data<T> *value, int id, ssize_t state_size) {
+    int64_t* data = reinterpret_cast<int64_t*>(value);
+    intptr_t ptr = reinterpret_cast<intptr_t>(data[0]);
+    int ndim = data[1];
+    ptrdiff_t offset = 0;
+    for (int dim = ndim; --dim >= 0; ) {
+        offset += data[ndim + dim + 2] * (id % data[dim + 2]);
+        id /= data[dim + 2];
+    }
+    return *reinterpret_cast<T*>(ptr + offset);
+}
+
+template<typename T>
+__device__ typename std::enable_if<std::is_arithmetic<T>::value, T>::type get_index(T value, int id, ssize_t state_size) {
+    return value;
+}
+
+__device__ rk_binomial_state* get_index(rk_binomial_state *value, int id, ssize_t state_size) {
+    return (value + id % state_size);
+}
+
+template<typename F, typename T, typename R, typename... Args>
+__global__ void execute_dist( intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, Args... args) {
+    R* out_ptr = reinterpret_cast<R*>(out);
+    F func;
+    T random(blockIdx.x * blockDim.x + threadIdx.x, state);
+    for (int id = blockIdx.x * blockDim.x + threadIdx.x; 
+             id < size; 
+             id += state_size) {
+        out_ptr[id] = func(random, (get_index(args, id, state_size))...);
+    }
+    return;
+}
+
+template <typename F, typename R>
+struct kernel_launcher {
+    kernel_launcher(ssize_t size, cudaStream_t stream) : _size(size), _stream(stream) {
+    }
+    template<typename T, typename... Args>
+    void operator()(Args&&... args) { 
+        int tpb = 256;
+        // Launch one thread per state size
+        int bpg = (_size + tpb - 1) / tpb; 
+        execute_dist<F, T, R><<<bpg, tpb, 0, _stream>>>(std::forward<Args>(args)...);
+    }
+    ssize_t _size;
+    cudaStream_t _stream;
+};
+
+//These functions will take the generator_id as a parameter
+void raw(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {
+    kernel_launcher<raw_functor, int32_t> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size);
+}
+
+void random_uniform(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {
+    kernel_launcher<random_uniform_functor, double> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size);
+}
+
+void random_uniform_float(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {
+    kernel_launcher<random_uniform_float_functor, float> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size);
+}
+
+//These functions will take the generator_id as a parameter
+void interval_32(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, int32_t mx, int32_t mask) {
+    kernel_launcher<interval_32_functor, int32_t> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, static_cast<uint32_t>(mx), static_cast<uint32_t>(mask));
+}
+
+void interval_64(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, int64_t mx, int64_t mask) {
+    kernel_launcher<interval_64_functor, int64_t> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, static_cast<uint64_t>(mx), static_cast<uint64_t>(mask));
+}
+
+void beta(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t a, intptr_t b) {
+    kernel_launcher<beta_functor, double> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, reinterpret_cast<array_data<double>*>(a), reinterpret_cast<array_data<double>*>(b));
+}
+
+void exponential(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {
+    kernel_launcher<exponential_functor, double> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size);
+}
+
+void geometric(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t p) {
+    kernel_launcher<geometric_functor, int64_t> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, reinterpret_cast<array_data<double>*>(p));
+}
+
+void hypergeometric(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t ngood, intptr_t nbad, intptr_t nsample) {
+    kernel_launcher<hypergeometric_functor, int64_t> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, reinterpret_cast<array_data<int64_t>*>(ngood), reinterpret_cast<array_data<int64_t>*>(nbad), reinterpret_cast<array_data<int64_t>*>(nsample));
+}
+
+void logseries(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t p) {
+    kernel_launcher<logseries_functor, int64_t> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, reinterpret_cast<array_data<double>*>(p));
+}
+
+void poisson(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t lam) {
+    kernel_launcher<poisson_functor, int64_t> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, reinterpret_cast<array_data<double>*>(lam));
+}
+
+void standard_normal(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {
+    kernel_launcher<standard_normal_functor, double> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size);
+}
+
+void standard_normal_float(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {
+    kernel_launcher<standard_normal_float_functor, float> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size);
+}
+
+void standard_gamma(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t shape) {
+    kernel_launcher<standard_gamma_functor, double> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, reinterpret_cast<array_data<double>*>(shape));
+}
+
+void binomial(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t n, intptr_t p, intptr_t binomial_state) {
+    kernel_launcher<binomial_functor, int64_t> launcher(state_size, reinterpret_cast<cudaStream_t>(stream));
+    generator_dispatcher(generator, launcher, state, state_size, out, size, reinterpret_cast<array_data<int>*>(n), reinterpret_cast<array_data<double>*>(p), reinterpret_cast<rk_binomial_state*>(binomial_state));
+}
+
+#else
+// the stubs need to be redeclared here for HIP versions less than 4.3 to avoid redeclarations in cython when importing the headers
+// No cuda will not compile the .cu file, so the definition needs to be done here explicitly
+void init_curand_generator(int generator, intptr_t state_ptr, ssize_t state_size, uint64_t seed, ssize_t size, intptr_t stream) {}
+void random_uniform(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void random_uniform_float(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void raw(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void interval_32(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, int32_t mx, int32_t mask) {}
+void interval_64(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, int64_t mx, int64_t mask) {}
+void beta(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t a, intptr_t b) {}
+void exponential(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void geometric(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t p) {}
+void hypergeometric(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t ngood, intptr_t nbad, intptr_t nsample) {}
+void logseries(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t p) {}
+void poisson(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t lam) {}
+void standard_normal(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void standard_normal_float(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void standard_gamma(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t shape) {}
+void binomial(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t n, intptr_t p, intptr_t binomial_state) {}
+
+#endif
diff --git a/cupy/random/cupy_distributions.cuh b/cupy/random/cupy_distributions.cuh
new file mode 100644
index 0000000..f010bff
--- /dev/null
+++ b/cupy/random/cupy_distributions.cuh
@@ -0,0 +1,103 @@
+#ifndef _CUPY_RANDOM_H
+#define _CUPY_RANDOM_H
+
+
+#if defined(_MSC_VER)
+#include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#endif
+
+
+// This enum holds the generators, we can't fully templatize the generators
+// because the dynamic design of BitGenerators in the python side does not allow us
+// to determine the correct type at compile time
+// We could use Jitify to avoid this code, but cuRAND EULA does not allow us to 
+// redistribute cuRAND header
+enum RandGenerators{
+   CURAND_XOR_WOW,
+   CURAND_MRG32k3a,
+   CURAND_PHILOX_4x32_10
+};
+
+struct rk_binomial_state {
+    int initialized;
+    int nsave, m;
+    double psave, r, q, fm, p1, xm, xl, xr, c, laml, lamr, p2, p3, p4;
+};
+
+#ifdef CUPY_USE_HIP
+#include <hip/hip_version.h>
+#if HIP_VERSION >= 403
+#define COMPILE_FOR_HIP
+#endif
+
+// When compiling cython extensions with hip 4.0
+// gcc will be used, but the hiprand_kernel can only be compiled with llvm
+// so we need to explicitly declare stubs for the functions
+#if HIP_VERSION > 400
+#include <hiprand_kernel.h>
+#else
+#include <hiprand.h>
+typedef struct {} hiprandState;
+typedef struct {} hiprandStateMRG32k3a;
+typedef struct {} hiprandStatePhilox4_32_10_t;
+#endif
+#define cudaStream_t hipStream_t
+#define curandState hiprandState
+#define curandStateMRG32k3a hiprandStateMRG32k3a
+#define curandStatePhilox4_32_10_t hiprandStatePhilox4_32_10_t
+#define curand_init hiprand_init
+#define curand hiprand
+#define curand_uniform hiprand_uniform
+#define curand_normal_double hiprand_normal_double
+#define curand_normal hiprand_normal
+
+#endif
+
+#if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+#include <curand_kernel.h>
+#endif
+
+#if !defined(CUPY_NO_CUDA)
+void init_curand_generator(int generator, intptr_t state_ptr, uint64_t seed, ssize_t size, intptr_t stream);
+void random_uniform(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream);
+void random_uniform_float(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream);
+void raw(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream);
+void interval_32(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, int32_t mx, int32_t mask);
+void interval_64(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, int64_t mx, int64_t mask);
+void beta(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t a, intptr_t b);
+void exponential(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream);
+void geometric(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t p);
+void hypergeometric(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t ngood, intptr_t nbad, intptr_t nsample);
+void logseries(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t p);
+void poisson(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t lam);
+void standard_normal(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream);
+void standard_normal_float(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream);
+void standard_gamma(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t shape);
+void binomial(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t n, intptr_t p, intptr_t binomial_state);
+
+#else
+// --no -cuda will not compile the .cu file, so the definition needs to be done here explicitly
+typedef struct {} curandState;
+typedef struct {} curandStateMRG32k3a;
+typedef struct {} curandStatePhilox4_32_10_t;
+void init_curand_generator(int generator, intptr_t state_ptr, uint64_t seed, ssize_t size, intptr_t stream) {}
+void random_uniform(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void random_uniform_float(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void raw(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void interval_32(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, int32_t mx, int32_t mask) {}
+void interval_64(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, int64_t mx, int64_t mask) {}
+void beta(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t a, intptr_t b) {}
+void exponential(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void geometric(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t p) {}
+void hypergeometric(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t ngood, intptr_t nbad, intptr_t nsample) {}
+void logseries(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t p) {}
+void poisson(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t lam) {}
+void standard_normal(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void standard_normal_float(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream) {}
+void standard_gamma(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t shape) {}
+void binomial(int generator, intptr_t state, ssize_t state_size, intptr_t out, ssize_t size, intptr_t stream, intptr_t n, intptr_t p, intptr_t binomial_state) {}
+
+#endif
+
+#endif
diff --git a/cupy/sparse/__init__.py b/cupy/sparse/__init__.py
new file mode 100644
index 0000000..9f00810
--- /dev/null
+++ b/cupy/sparse/__init__.py
@@ -0,0 +1,22 @@
+import sys
+import warnings
+
+import cupyx.scipy.sparse
+
+
+# Raise a `DeprecationWarning` for `cupy.sparse` submodule when its functions
+# are called. We could raise the warning on importing the submodule, but we
+# use module level `__getattr__` function here as the submodule is also
+# imported in cupy/__init__.py. Unfortunately, module level `__getattr__` is
+# supported on Python 3.7 and higher, so we need to keep the explicit import
+# list for older Python.
+if (3, 7) <= sys.version_info:
+    def __getattr__(name):
+        if hasattr(cupyx.scipy.sparse, name):
+            msg = 'cupy.sparse is deprecated. Use cupyx.scipy.sparse instead.'
+            warnings.warn(msg, DeprecationWarning)
+            return getattr(cupyx.scipy.sparse, name)
+        raise AttributeError(
+            "module 'cupy.sparse' has no attribute {!r}".format(name))
+else:
+    from cupyx.scipy.sparse import *  # NOQA
diff --git a/cupy/sparse/linalg/__init__.py b/cupy/sparse/linalg/__init__.py
new file mode 100644
index 0000000..67f89d5
--- /dev/null
+++ b/cupy/sparse/linalg/__init__.py
@@ -0,0 +1,17 @@
+import sys
+import warnings
+
+import cupyx.scipy.sparse.linalg
+
+
+if (3, 7) <= sys.version_info:
+    def __getattr__(name):
+        if hasattr(cupyx.scipy.sparse.linalg, name):
+            warnings.warn(
+                'cupy.sparse.linalg is deprecated.'
+                ' Use cupyx.scipy.sparse.linalg instead.', DeprecationWarning)
+            return getattr(cupyx.scipy.sparse.linalg, name)
+        raise AttributeError(
+            "module 'cupy.sparse.linalg' has no attribute {!r}".format(name))
+else:
+    from cupyx.scipy.sparse.linalg import *  # NOQA
diff --git a/cupy/testing/__init__.py b/cupy/testing/__init__.py
new file mode 100644
index 0000000..f4b8aca
--- /dev/null
+++ b/cupy/testing/__init__.py
@@ -0,0 +1,50 @@
+from cupy.testing._array import assert_allclose  # NOQA
+from cupy.testing._array import assert_array_almost_equal  # NOQA
+from cupy.testing._array import assert_array_almost_equal_nulp  # NOQA
+from cupy.testing._array import assert_array_equal  # NOQA
+from cupy.testing._array import assert_array_less  # NOQA
+from cupy.testing._array import assert_array_list_equal  # NOQA
+from cupy.testing._array import assert_array_max_ulp  # NOQA
+from cupy.testing._attr import multi_gpu  # NOQA
+from cupy.testing._attr import slow  # NOQA
+from cupy.testing._helper import assert_warns  # NOQA
+from cupy.testing._helper import numpy_satisfies  # NOQA
+from cupy.testing._helper import NumpyAliasBasicTestBase  # NOQA
+from cupy.testing._helper import NumpyAliasValuesTestBase  # NOQA
+from cupy.testing._helper import AssertFunctionIsCalled  # NOQA
+from cupy.testing._helper import shaped_arange  # NOQA
+from cupy.testing._helper import shaped_sparse_random  # NOQA
+from cupy.testing._helper import shaped_random  # NOQA
+from cupy.testing._helper import generate_matrix  # NOQA
+from cupy.testing._helper import shaped_reverse_arange  # NOQA
+from cupy.testing._helper import with_requires  # NOQA
+from cupy.testing._helper import installed  # NOQA
+from cupy.testing._loops import for_all_dtypes  # NOQA
+from cupy.testing._loops import for_all_dtypes_combination  # NOQA
+from cupy.testing._loops import for_CF_orders  # NOQA
+from cupy.testing._loops import for_complex_dtypes  # NOQA
+from cupy.testing._loops import for_contiguous_axes  # NOQA
+from cupy.testing._loops import for_dtypes  # NOQA
+from cupy.testing._loops import for_dtypes_combination  # NOQA
+from cupy.testing._loops import for_float_dtypes  # NOQA
+from cupy.testing._loops import for_int_dtypes  # NOQA
+from cupy.testing._loops import for_int_dtypes_combination  # NOQA
+from cupy.testing._loops import for_orders  # NOQA
+from cupy.testing._loops import for_signed_dtypes  # NOQA
+from cupy.testing._loops import for_signed_dtypes_combination  # NOQA
+from cupy.testing._loops import for_unsigned_dtypes  # NOQA
+from cupy.testing._loops import for_unsigned_dtypes_combination  # NOQA
+from cupy.testing._loops import numpy_cupy_allclose  # NOQA
+from cupy.testing._loops import numpy_cupy_array_almost_equal  # NOQA
+from cupy.testing._loops import numpy_cupy_array_almost_equal_nulp  # NOQA
+from cupy.testing._loops import numpy_cupy_array_equal  # NOQA
+from cupy.testing._loops import numpy_cupy_array_less  # NOQA
+from cupy.testing._loops import numpy_cupy_array_list_equal  # NOQA
+from cupy.testing._loops import numpy_cupy_array_max_ulp  # NOQA
+from cupy.testing._loops import numpy_cupy_equal  # NOQA
+from cupy.testing._loops import numpy_cupy_raises  # NOQA
+from cupy.testing._parameterized import parameterize  # NOQA
+from cupy.testing._parameterized import product  # NOQA
+from cupy.testing._parameterized import product_dict  # NOQA
+from cupy.testing._random import fix_random  # NOQA
+from cupy.testing._random import generate_seed  # NOQA
diff --git a/cupy/testing/_array.py b/cupy/testing/_array.py
new file mode 100644
index 0000000..df083d9
--- /dev/null
+++ b/cupy/testing/_array.py
@@ -0,0 +1,157 @@
+import numpy.testing
+
+import cupy
+
+
+# NumPy-like assertion functions that accept both NumPy and CuPy arrays
+
+def assert_allclose(actual, desired, rtol=1e-7, atol=0, err_msg='',
+                    verbose=True):
+    """Raises an AssertionError if objects are not equal up to desired tolerance.
+
+    Args:
+         actual(numpy.ndarray or cupy.ndarray): The actual object to check.
+         desired(numpy.ndarray or cupy.ndarray): The desired, expected object.
+         rtol(float): Relative tolerance.
+         atol(float): Absolute tolerance.
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting
+             values are appended to the error message.
+
+    .. seealso:: :func:`numpy.testing.assert_allclose`
+
+    """  # NOQA
+    numpy.testing.assert_allclose(
+        cupy.asnumpy(actual), cupy.asnumpy(desired),
+        rtol=rtol, atol=atol, err_msg=err_msg, verbose=verbose)
+
+
+def assert_array_almost_equal(x, y, decimal=6, err_msg='', verbose=True):
+    """Raises an AssertionError if objects are not equal up to desired precision.
+
+    Args:
+         x(numpy.ndarray or cupy.ndarray): The actual object to check.
+         y(numpy.ndarray or cupy.ndarray): The desired, expected object.
+         decimal(int): Desired precision.
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting
+             values are appended to the error message.
+
+    .. seealso:: :func:`numpy.testing.assert_array_almost_equal`
+    """  # NOQA
+    numpy.testing.assert_array_almost_equal(
+        cupy.asnumpy(x), cupy.asnumpy(y), decimal=decimal,
+        err_msg=err_msg, verbose=verbose)
+
+
+def assert_array_almost_equal_nulp(x, y, nulp=1):
+    """Compare two arrays relatively to their spacing.
+
+    Args:
+         x(numpy.ndarray or cupy.ndarray): The actual object to check.
+         y(numpy.ndarray or cupy.ndarray): The desired, expected object.
+         nulp(int): The maximum number of unit in the last place for tolerance.
+
+    .. seealso:: :func:`numpy.testing.assert_array_almost_equal_nulp`
+    """
+    numpy.testing.assert_array_almost_equal_nulp(
+        cupy.asnumpy(x), cupy.asnumpy(y), nulp=nulp)
+
+
+def assert_array_max_ulp(a, b, maxulp=1, dtype=None):
+    """Check that all items of arrays differ in at most N Units in the Last Place.
+
+    Args:
+         a(numpy.ndarray or cupy.ndarray): The actual object to check.
+         b(numpy.ndarray or cupy.ndarray): The desired, expected object.
+         maxulp(int): The maximum number of units in the last place
+             that elements of ``a`` and ``b`` can differ.
+         dtype(numpy.dtype): Data-type to convert ``a`` and ``b`` to if given.
+
+    .. seealso:: :func:`numpy.testing.assert_array_max_ulp`
+    """  # NOQA
+    numpy.testing.assert_array_max_ulp(
+        cupy.asnumpy(a), cupy.asnumpy(b), maxulp=maxulp, dtype=dtype)
+
+
+def assert_array_equal(x, y, err_msg='', verbose=True, strides_check=False):
+    """Raises an AssertionError if two array_like objects are not equal.
+
+    Args:
+         x(numpy.ndarray or cupy.ndarray): The actual object to check.
+         y(numpy.ndarray or cupy.ndarray): The desired, expected object.
+         strides_check(bool): If ``True``, consistency of strides is also
+             checked.
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values
+             are appended to the error message.
+
+    .. seealso:: :func:`numpy.testing.assert_array_equal`
+    """
+    numpy.testing.assert_array_equal(
+        cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
+        verbose=verbose)
+
+    if strides_check:
+        if x.strides != y.strides:
+            msg = ['Strides are not equal:']
+            if err_msg:
+                msg = [msg[0] + ' ' + err_msg]
+            if verbose:
+                msg.append(' x: {}'.format(x.strides))
+                msg.append(' y: {}'.format(y.strides))
+            raise AssertionError('\n'.join(msg))
+
+
+def assert_array_list_equal(xlist, ylist, err_msg='', verbose=True):
+    """Compares lists of arrays pairwise with ``assert_array_equal``.
+
+    Args:
+         x(array_like): Array of the actual objects.
+         y(array_like): Array of the desired, expected objects.
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values
+             are appended to the error message.
+
+    Each element of ``x`` and ``y`` must be either :class:`numpy.ndarray`
+    or :class:`cupy.ndarray`. ``x`` and ``y`` must have same length.
+    Otherwise, this function raises ``AssertionError``.
+    It compares elements of ``x`` and ``y`` pairwise
+    with :func:`assert_array_equal` and raises error if at least one
+    pair is not equal.
+
+    .. seealso:: :func:`numpy.testing.assert_array_equal`
+    """
+    x_type = type(xlist)
+    y_type = type(ylist)
+    if x_type is not y_type:
+        raise AssertionError(
+            'Matching types of list or tuple are expected, '
+            'but were different types '
+            '(xlist:{} ylist:{})'.format(x_type, y_type))
+    if x_type not in (list, tuple):
+        raise AssertionError(
+            'List or tuple is expected, but was {}'.format(x_type))
+    if len(xlist) != len(ylist):
+        raise AssertionError('List size is different')
+    for x, y in zip(xlist, ylist):
+        numpy.testing.assert_array_equal(
+            cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
+            verbose=verbose)
+
+
+def assert_array_less(x, y, err_msg='', verbose=True):
+    """Raises an AssertionError if array_like objects are not ordered by less than.
+
+    Args:
+         x(numpy.ndarray or cupy.ndarray): The smaller object to check.
+         y(numpy.ndarray or cupy.ndarray): The larger object to compare.
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values
+             are appended to the error message.
+
+    .. seealso:: :func:`numpy.testing.assert_array_less`
+    """  # NOQA
+    numpy.testing.assert_array_less(
+        cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
+        verbose=verbose)
diff --git a/cupy/testing/_attr.py b/cupy/testing/_attr.py
new file mode 100644
index 0000000..6e1e708
--- /dev/null
+++ b/cupy/testing/_attr.py
@@ -0,0 +1,43 @@
+import os
+
+
+from cupy.testing._pytest_impl import is_available, check_available
+
+
+if is_available():
+    import pytest
+
+    _gpu_limit = int(os.getenv('CUPY_TEST_GPU_LIMIT', '-1'))
+
+    def slow(*args, **kwargs):
+        return pytest.mark.slow(*args, **kwargs)
+
+else:
+    def _dummy_callable(*args, **kwargs):
+        check_available('pytest attributes')
+        assert False  # Not reachable
+
+    slow = _dummy_callable
+
+
+def multi_gpu(gpu_num):
+    """Decorator to indicate number of GPUs required to run the test.
+
+    Tests can be annotated with this decorator (e.g., ``@multi_gpu(2)``) to
+    declare number of GPUs required to run. When running tests, if
+    ``CUPY_TEST_GPU_LIMIT`` environment variable is set to value greater
+    than or equals to 0, test cases that require GPUs more than the limit will
+    be skipped.
+    """
+
+    check_available('multi_gpu attribute')
+    # at this point we know pytest is available for sure
+
+    assert 1 < gpu_num
+
+    def _wrapper(f):
+        return pytest.mark.skipif(
+            0 <= _gpu_limit < gpu_num,
+            reason='{} GPUs required'.format(gpu_num))(
+                pytest.mark.multi_gpu(f))
+    return _wrapper
diff --git a/cupy/testing/_bundle.py b/cupy/testing/_bundle.py
new file mode 100644
index 0000000..a4f7563
--- /dev/null
+++ b/cupy/testing/_bundle.py
@@ -0,0 +1,47 @@
+import inspect
+import sys
+
+
+def make_decorator(test_case_generator):
+    # `test_case_generator` is a callable that receives the source test class
+    # (typically a subclass of unittest.TestCase) and returns an iterable of
+    # generated test cases.
+    # Each element of the iterable is a 3-element tuple:
+    # [0] Generated class name
+    # [1] Dict of members
+    # [2] Method generator
+    # The method generator is also a callable that receives an original test
+    # method and returns a new test method.
+
+    def f(cls):
+        # The input is a bare test case
+        module_name = cls.__module__
+
+        # Generate parameterized test cases out of the input test case.
+        module = sys.modules[module_name]
+        assert module.__name__ == module_name
+        for cls_name, members, method_generator in test_case_generator(cls):
+            _generate_case(
+                cls, module, cls_name, members, method_generator)
+
+        # Remove original base class
+        return None
+    return f
+
+
+def _generate_case(base, module, cls_name, mb, method_generator):
+    members = mb.copy()
+    base_methods = inspect.getmembers(base, predicate=inspect.isfunction)
+    for name, value in base_methods:
+        if not name.startswith('test_'):
+            continue
+        value = method_generator(value)
+        # If the return value of method_generator is None, None is assigned
+        # as the value of the test method and it is ignored by pytest.
+        members[name] = value
+
+    cls = type(cls_name, (base,), members)
+
+    # Add new test class to module
+    cls.__module__ = module.__name__
+    setattr(module, cls_name, cls)
diff --git a/cupy/testing/_condition.py b/cupy/testing/_condition.py
new file mode 100644
index 0000000..d5f1e8d
--- /dev/null
+++ b/cupy/testing/_condition.py
@@ -0,0 +1,124 @@
+import functools
+import os
+import unittest
+
+
+class QuietTestRunner(object):
+
+    def run(self, suite):
+        result = unittest.TestResult()
+        suite(result)
+        return result
+
+
+def repeat_with_success_at_least(times, min_success):
+    """Decorator for multiple trial of the test case.
+
+    The decorated test case is launched multiple times.
+    The case is judged as passed at least specified number of trials.
+    If the number of successful trials exceeds `min_success`,
+    the remaining trials are skipped.
+
+    Args:
+        times(int): The number of trials.
+        min_success(int): Threshold that the decorated test
+            case is regarded as passed.
+
+    """
+
+    assert times >= min_success
+
+    def _repeat_with_success_at_least(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            assert len(args) > 0
+            instance = args[0]
+            assert isinstance(instance, unittest.TestCase)
+            success_counter = 0
+            failure_counter = 0
+            results = []
+
+            def fail():
+                msg = '\nFail: {0}, Success: {1}'.format(
+                    failure_counter, success_counter)
+                if len(results) > 0:
+                    first = results[0]
+                    errs = first.failures + first.errors
+                    if len(errs) > 0:
+                        err_msg = '\n'.join(fail[1] for fail in errs)
+                        msg += '\n\nThe first error message:\n' + err_msg
+                instance.fail(msg)
+
+            for _ in range(times):
+                suite = unittest.TestSuite()
+                # Create new instance to call the setup and the teardown only
+                # once.
+                ins = type(instance)(instance._testMethodName)
+                suite.addTest(
+                    unittest.FunctionTestCase(
+                        lambda: f(ins, *args[1:], **kwargs),
+                        setUp=ins.setUp,
+                        tearDown=ins.tearDown))
+
+                result = QuietTestRunner().run(suite)
+                if len(result.skipped) == 1:
+                    # "Skipped" is a special case of "Successful".
+                    # When the test has been skipped, immedeately quit the
+                    # test regardleess of `times` and `min_success` by raising
+                    # SkipTest exception using the original reason.
+                    instance.skipTest(result.skipped[0][1])
+                elif result.wasSuccessful():
+                    success_counter += 1
+                else:
+                    results.append(result)
+                    failure_counter += 1
+                if success_counter >= min_success:
+                    instance.assertTrue(True)
+                    return
+                if failure_counter > times - min_success:
+                    fail()
+                    return
+            fail()
+        return wrapper
+    return _repeat_with_success_at_least
+
+
+def repeat(times, intensive_times=None):
+    """Decorator that imposes the test to be successful in a row.
+
+    Decorated test case is launched multiple times.
+    The case is regarded as passed only if it is successful
+    specified times in a row.
+
+    .. note::
+        In current implementation, this decorator grasps the
+        failure information of each trial.
+
+    Args:
+        times(int): The number of trials in casual test.
+        intensive_times(int or None): The number of trials in more intensive
+            test. If ``None``, the same number as `times` is used.
+    """
+    if intensive_times is None:
+        return repeat_with_success_at_least(times, times)
+
+    casual_test = bool(int(os.environ.get('CUPY_TEST_CASUAL', '0')))
+    times_ = times if casual_test else intensive_times
+    return repeat_with_success_at_least(times_, times_)
+
+
+def retry(times):
+    """Decorator that imposes the test to be successful at least once.
+
+    Decorated test case is launched multiple times.
+    The case is regarded as passed if it is successful
+    at least once.
+
+    .. note::
+        In current implementation, this decorator grasps the
+        failure information of each trial.
+
+    Args:
+        times(int): The number of trials.
+    """
+    return repeat_with_success_at_least(times, 1)
diff --git a/cupy/testing/_helper.py b/cupy/testing/_helper.py
new file mode 100644
index 0000000..f0b5a64
--- /dev/null
+++ b/cupy/testing/_helper.py
@@ -0,0 +1,316 @@
+import contextlib
+import inspect
+from typing import Callable
+import unittest
+from unittest import mock
+import warnings
+
+import numpy
+
+import cupy
+from cupy._core import internal
+import cupyx
+import cupyx.scipy.sparse
+
+from cupy.testing._pytest_impl import is_available
+
+
+if is_available():
+    import pytest
+    _skipif: Callable[..., Callable[[Callable], Callable]] = pytest.mark.skipif
+else:
+    _skipif = unittest.skipIf
+
+
+def with_requires(*requirements):
+    """Run a test case only when given requirements are satisfied.
+
+    .. admonition:: Example
+
+       This test case runs only when `numpy>=1.18` is installed.
+
+       >>> from cupy import testing
+       ... class Test(unittest.TestCase):
+       ...     @testing.with_requires('numpy>=1.18')
+       ...     def test_for_numpy_1_18(self):
+       ...         pass
+
+    Args:
+        requirements: A list of string representing requirement condition to
+            run a given test case.
+
+    """
+    msg = 'requires: {}'.format(','.join(requirements))
+    return _skipif(not installed(requirements), reason=msg)
+
+
+def installed(*specifiers):
+    """Returns True if the current environment satisfies the specified
+    package requirement.
+
+    Args:
+        specifiers: Version specifiers (e.g., `numpy>=1.20.0`).
+    """
+    # Delay import of pkg_resources because it is excruciatingly slow.
+    # See https://github.com/pypa/setuptools/issues/510
+    import pkg_resources
+
+    for spec in specifiers:
+        try:
+            pkg_resources.require(spec)
+        except pkg_resources.ResolutionError:
+            return False
+    return True
+
+
+def numpy_satisfies(version_range):
+    """Returns True if numpy version satisfies the specified criteria.
+
+    Args:
+        version_range: A version specifier (e.g., `>=1.13.0`).
+    """
+    return installed('numpy{}'.format(version_range))
+
+
+def shaped_arange(shape, xp=cupy, dtype=numpy.float32, order='C'):
+    """Returns an array with given shape, array module, and dtype.
+
+    Args:
+         shape(tuple of int): Shape of returned ndarray.
+         xp(numpy or cupy): Array module to use.
+         dtype(dtype): Dtype of returned ndarray.
+         order({'C', 'F'}): Order of returned ndarray.
+
+    Returns:
+         numpy.ndarray or cupy.ndarray:
+         The array filled with :math:`1, \\cdots, N` with specified dtype
+         with given shape, array module. Here, :math:`N` is
+         the size of the returned array.
+         If ``dtype`` is ``numpy.bool_``, evens (resp. odds) are converted to
+         ``True`` (resp. ``False``).
+
+    """
+    dtype = numpy.dtype(dtype)
+    a = numpy.arange(1, internal.prod(shape) + 1, 1)
+    if dtype == '?':
+        a = a % 2 == 0
+    elif dtype.kind == 'c':
+        a = a + a * 1j
+    return xp.array(a.astype(dtype).reshape(shape), order=order)
+
+
+def shaped_reverse_arange(shape, xp=cupy, dtype=numpy.float32):
+    """Returns an array filled with decreasing numbers.
+
+    Args:
+         shape(tuple of int): Shape of returned ndarray.
+         xp(numpy or cupy): Array module to use.
+         dtype(dtype): Dtype of returned ndarray.
+
+    Returns:
+         numpy.ndarray or cupy.ndarray:
+         The array filled with :math:`N, \\cdots, 1` with specified dtype
+         with given shape, array module.
+         Here, :math:`N` is the size of the returned array.
+         If ``dtype`` is ``numpy.bool_``, evens (resp. odds) are converted to
+         ``True`` (resp. ``False``).
+    """
+    dtype = numpy.dtype(dtype)
+    size = internal.prod(shape)
+    a = numpy.arange(size, 0, -1)
+    if dtype == '?':
+        a = a % 2 == 0
+    elif dtype.kind == 'c':
+        a = a + a * 1j
+    return xp.array(a.astype(dtype).reshape(shape))
+
+
+def shaped_random(
+        shape, xp=cupy, dtype=numpy.float32, scale=10, seed=0, order='C'):
+    """Returns an array filled with random values.
+
+    Args:
+         shape(tuple): Shape of returned ndarray.
+         xp(numpy or cupy): Array module to use.
+         dtype(dtype): Dtype of returned ndarray.
+         scale(float): Scaling factor of elements.
+         seed(int): Random seed.
+
+    Returns:
+         numpy.ndarray or cupy.ndarray: The array with
+         given shape, array module,
+
+    If ``dtype`` is ``numpy.bool_``, the elements are
+    independently drawn from ``True`` and ``False``
+    with same probabilities.
+    Otherwise, the array is filled with samples
+    independently and identically drawn
+    from uniform distribution over :math:`[0, scale)`
+    with specified dtype.
+    """
+    numpy.random.seed(seed)
+    dtype = numpy.dtype(dtype)
+    if dtype == '?':
+        a = numpy.random.randint(2, size=shape)
+    elif dtype.kind == 'c':
+        a = numpy.random.rand(*shape) + 1j * numpy.random.rand(*shape)
+        a *= scale
+    else:
+        a = numpy.random.rand(*shape) * scale
+    return xp.asarray(a, dtype=dtype, order=order)
+
+
+def shaped_sparse_random(
+        shape, sp=cupyx.scipy.sparse, dtype=numpy.float32,
+        density=0.01, format='coo', seed=0):
+    """Returns an array filled with random values.
+
+    Args:
+        shape (tuple): Shape of returned sparse matrix.
+        sp (scipy.sparse or cupyx.scipy.sparse): Sparce matrix module to use.
+        dtype (dtype): Dtype of returned sparse matrix.
+        density (float): Density of returned sparse matrix.
+        format (str): Format of returned sparse matrix.
+        seed (int): Random seed.
+
+    Returns:
+        The sparse matrix with given shape, array module,
+    """
+    import scipy.sparse
+    n_rows, n_cols = shape
+    numpy.random.seed(seed)
+    a = scipy.sparse.random(n_rows, n_cols, density).astype(dtype)
+
+    if sp is cupyx.scipy.sparse:
+        a = cupyx.scipy.sparse.coo_matrix(a)
+    elif sp is not scipy.sparse:
+        raise ValueError('Unknown module: {}'.format(sp))
+
+    return a.asformat(format)
+
+
+def generate_matrix(
+        shape, xp=cupy, dtype=numpy.float32, *, singular_values=None):
+    r"""Returns a matrix with specified singular values.
+
+    Generates a random matrix with given singular values.
+    This function generates a random NumPy matrix (or a stack of matrices) that
+    has specified singular values. It can be used to generate the inputs for a
+    test that can be instable when the input value behaves bad.
+    Notation: denote the shape of the generated array by :math:`(B..., M, N)`,
+    and :math:`K = min\{M, N\}`. :math:`B...` may be an empty sequence.
+
+    Args:
+        shape (tuple of int): Shape of the generated array, i.e.,
+            :math:`(B..., M, N)`.
+        xp (numpy or cupy): Array module to use.
+        dtype: Dtype of the generated array.
+        singular_values (array-like): Singular values of the generated
+            matrices. It must be broadcastable to shape :math:`(B..., K)`.
+
+    Returns:
+        numpy.ndarray or cupy.ndarray: A random matrix that has specifiec
+        singular values.
+    """
+
+    if len(shape) <= 1:
+        raise ValueError(
+            'shape {} is invalid for matrices: too few axes'.format(shape)
+        )
+
+    if singular_values is None:
+        raise TypeError('singular_values is not given')
+    singular_values = xp.asarray(singular_values)
+
+    dtype = numpy.dtype(dtype)
+    if dtype.kind not in 'fc':
+        raise TypeError('dtype {} is not supported'.format(dtype))
+
+    if not xp.isrealobj(singular_values):
+        raise TypeError('singular_values is not real')
+    if (singular_values < 0).any():
+        raise ValueError('negative singular value is given')
+
+    # Generate random matrices with given singular values. We simply generate
+    # orthogonal vectors using SVD on random matrices and then combine them
+    # with the given singular values.
+    a = xp.random.randn(*shape)
+    if dtype.kind == 'c':
+        a = a + 1j * xp.random.randn(*shape)
+    u, s, vh = xp.linalg.svd(a, full_matrices=False)
+    sv = xp.broadcast_to(singular_values, s.shape)
+    a = xp.einsum('...ik,...k,...kj->...ij', u, sv, vh)
+    return a.astype(dtype)
+
+
+@contextlib.contextmanager
+def assert_warns(expected):
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter('always')
+        yield
+
+    if any(isinstance(m.message, expected) for m in w):
+        return
+
+    try:
+        exc_name = expected.__name__
+    except AttributeError:
+        exc_name = str(expected)
+
+    raise AssertionError('%s not triggerred' % exc_name)
+
+
+class NumpyAliasTestBase(unittest.TestCase):
+
+    @property
+    def func(self):
+        raise NotImplementedError()
+
+    @property
+    def cupy_func(self):
+        return getattr(cupy, self.func)
+
+    @property
+    def numpy_func(self):
+        return getattr(numpy, self.func)
+
+
+class NumpyAliasBasicTestBase(NumpyAliasTestBase):
+
+    def test_argspec(self):
+        f = inspect.signature
+        assert f(self.cupy_func) == f(self.numpy_func)
+
+    def test_docstring(self):
+        cupy_func = self.cupy_func
+        numpy_func = self.numpy_func
+        assert hasattr(cupy_func, '__doc__')
+        assert cupy_func.__doc__ is not None
+        assert cupy_func.__doc__ != ''
+        assert cupy_func.__doc__ is not numpy_func.__doc__
+
+
+class NumpyAliasValuesTestBase(NumpyAliasTestBase):
+
+    def test_values(self):
+        assert self.cupy_func(*self.args) == self.numpy_func(*self.args)
+
+
+@contextlib.contextmanager
+def assert_function_is_called(*args, times_called=1, **kwargs):
+    """A handy wrapper for unittest.mock to check if a function is called.
+
+    Args:
+        *args: Arguments of `mock.patch`.
+        times_called (int): The number of times the function should be
+            called. Default is ``1``.
+        **kwargs: Keyword arguments of `mock.patch`.
+
+    """
+    with mock.patch(*args, **kwargs) as handle:
+        yield
+        assert handle.call_count == times_called
+
+
+# TODO(kataoka): remove this alias
+AssertFunctionIsCalled = assert_function_is_called
diff --git a/cupy/testing/_hypothesis.py b/cupy/testing/_hypothesis.py
new file mode 100644
index 0000000..fe9ccce
--- /dev/null
+++ b/cupy/testing/_hypothesis.py
@@ -0,0 +1,137 @@
+import numpy
+
+
+def chi_square_test(observed, expected, alpha=0.05, df=None):
+    """Testing Goodness-of-fit Test with Pearson's Chi-squared Test.
+
+    Args:
+        observed (list of ints): List of # of counts each element is observed.
+        expected (list of floats): List of # of counts each element is expected
+            to be observed.
+        alpha (float): Significance level. Currently,
+            only 0.05 and 0.01 are acceptable.
+        df (int): Degree of freedom. If ``None``,
+            it is set to the length of ``observed`` minus 1.
+
+    Returns:
+        bool: ``True`` if null hypothesis is **NOT** reject.
+        Otherwise, ``False``.
+    """
+    if df is None:
+        df = observed.size - 1
+
+    if alpha == 0.01:
+        alpha_idx = 0
+    elif alpha == 0.05:
+        alpha_idx = 1
+    else:
+        raise ValueError('support only alpha == 0.05 or 0.01')
+
+    chi_square = numpy.sum((observed - expected) ** 2 / expected)
+    return chi_square < chi_square_table[alpha_idx][df]
+
+
+# https://www.medcalc.org/manual/chi-square-table.php
+chi_square_table = [
+    [None,
+     6.635, 9.210, 11.345, 13.277, 15.086,
+     16.812, 18.475, 20.090, 21.666, 23.209,
+     24.725, 26.217, 27.688, 29.141, 30.578,
+     32.000, 33.409, 34.805, 36.191, 37.566,
+     38.932, 40.289, 41.638, 42.980, 44.314,
+     45.642, 46.963, 48.278, 49.588, 50.892,
+     52.191, 53.486, 54.776, 56.061, 57.342,
+     58.619, 59.893, 61.162, 62.428, 63.691,
+     64.950, 66.206, 67.459, 68.710, 69.957,
+     71.201, 72.443, 73.683, 74.919, 76.154,
+     77.386, 78.616, 79.843, 81.069, 82.292,
+     83.513, 84.733, 85.950, 87.166, 88.379,
+     89.591, 90.802, 92.010, 93.217, 94.422,
+     95.626, 96.828, 98.028, 99.228, 100.425,
+     101.621, 102.816, 104.010, 105.202, 106.393,
+     107.583, 108.771, 109.958, 111.144, 112.329,
+     113.512, 114.695, 115.876, 117.057, 118.236,
+     119.414, 120.591, 121.767, 122.942, 124.116,
+     125.289, 126.462, 127.633, 128.803, 129.973,
+     131.141, 132.309, 133.476, 134.642, 135.807,
+     136.971, 138.134, 139.297, 140.459, 141.620,
+     142.780, 143.940, 145.099, 146.257, 147.414,
+     148.571, 149.727, 150.882, 152.037, 153.191,
+     154.344, 155.496, 156.648, 157.800, 158.950,
+     160.100, 161.250, 162.398, 163.546, 164.694,
+     165.841, 166.987, 168.133, 169.278, 170.423,
+     171.567, 172.711, 173.854, 174.996, 176.138,
+     177.280, 178.421, 179.561, 180.701, 181.840,
+     182.979, 184.118, 185.256, 186.393, 187.530,
+     188.666, 189.802, 190.938, 192.073, 193.208,
+     194.342, 195.476, 196.609, 197.742, 198.874,
+     200.006, 201.138, 202.269, 203.400, 204.530,
+     205.660, 206.790, 207.919, 209.047, 210.176,
+     211.304, 212.431, 213.558, 214.685, 215.812,
+     216.938, 218.063, 219.189, 220.314, 221.438,
+     222.563, 223.687, 224.810, 225.933, 227.056,
+     228.179, 229.301, 230.423, 231.544, 232.665,
+     233.786, 234.907, 236.027, 237.147, 238.266,
+     239.386, 240.505, 241.623, 242.742, 243.860,
+     244.977, 246.095, 247.212, 248.329, 249.445,
+     250.561, 251.677, 252.793, 253.908, 255.023,
+     256.138, 257.253, 258.367, 259.481, 260.595,
+     261.708, 262.821, 263.934, 265.047, 266.159,
+     267.271, 268.383, 269.495, 270.606, 271.717,
+     272.828, 273.939, 275.049, 276.159, 277.269,
+     278.379, 279.488, 280.597, 281.706, 282.814,
+     283.923, 285.031, 286.139, 287.247, 288.354,
+     289.461, 290.568, 291.675, 292.782, 293.888,
+     294.994, 296.100, 297.206, 298.311, 299.417,
+     300.522, 301.626, 302.731, 303.835, 304.940],
+    [None,
+     3.841, 5.991, 7.815, 9.488, 11.070,
+     12.592, 14.067, 15.507, 16.919, 18.307,
+     19.675, 21.026, 22.362, 23.685, 24.996,
+     26.296, 27.587, 28.869, 30.144, 31.410,
+     32.671, 33.924, 35.172, 36.415, 37.652,
+     38.885, 40.113, 41.337, 42.557, 43.773,
+     44.985, 46.194, 47.400, 48.602, 49.802,
+     50.998, 52.192, 53.384, 54.572, 55.758,
+     56.942, 58.124, 59.304, 60.481, 61.656,
+     62.830, 64.001, 65.171, 66.339, 67.505,
+     68.669, 69.832, 70.993, 72.153, 73.311,
+     74.468, 75.624, 76.778, 77.931, 79.082,
+     80.232, 81.381, 82.529, 83.675, 84.821,
+     85.965, 87.108, 88.250, 89.391, 90.531,
+     91.670, 92.808, 93.945, 95.081, 96.217,
+     97.351, 98.484, 99.617, 100.749, 101.879,
+     103.010, 104.139, 105.267, 106.395, 107.522,
+     108.648, 109.773, 110.898, 112.022, 113.145,
+     114.268, 115.390, 116.511, 117.632, 118.752,
+     119.871, 120.990, 122.108, 123.225, 124.342,
+     125.458, 126.574, 127.689, 128.804, 129.918,
+     131.031, 132.144, 133.257, 134.369, 135.480,
+     136.591, 137.701, 138.811, 139.921, 141.030,
+     142.138, 143.246, 144.354, 145.461, 146.567,
+     147.674, 148.779, 149.885, 150.989, 152.094,
+     153.198, 154.302, 155.405, 156.508, 157.610,
+     158.712, 159.814, 160.915, 162.016, 163.116,
+     164.216, 165.316, 166.415, 167.514, 168.613,
+     169.711, 170.809, 171.907, 173.004, 174.101,
+     175.198, 176.294, 177.390, 178.485, 179.581,
+     180.676, 181.770, 182.865, 183.959, 185.052,
+     186.146, 187.239, 188.332, 189.424, 190.516,
+     191.608, 192.700, 193.791, 194.883, 195.973,
+     197.064, 198.154, 199.244, 200.334, 201.423,
+     202.513, 203.602, 204.690, 205.779, 206.867,
+     207.955, 209.042, 210.130, 211.217, 212.304,
+     213.391, 214.477, 215.563, 216.649, 217.735,
+     218.820, 219.906, 220.991, 222.076, 223.160,
+     224.245, 225.329, 226.413, 227.496, 228.580,
+     229.663, 230.746, 231.829, 232.912, 233.994,
+     235.077, 236.159, 237.240, 238.322, 239.403,
+     240.485, 241.566, 242.647, 243.727, 244.808,
+     245.888, 246.968, 248.048, 249.128, 250.207,
+     251.286, 252.365, 253.444, 254.523, 255.602,
+     256.680, 257.758, 258.837, 259.914, 260.992,
+     262.070, 263.147, 264.224, 265.301, 266.378,
+     267.455, 268.531, 269.608, 270.684, 271.760,
+     272.836, 273.911, 274.987, 276.062, 277.138,
+     278.213, 279.288, 280.362, 281.437, 282.511,
+     283.586, 284.660, 285.734, 286.808, 287.882]]
diff --git a/cupy/testing/_loops.py b/cupy/testing/_loops.py
new file mode 100644
index 0000000..15edb6f
--- /dev/null
+++ b/cupy/testing/_loops.py
@@ -0,0 +1,1247 @@
+import functools
+import inspect
+import os
+import random
+from typing import Tuple, Type
+import traceback
+import unittest
+import warnings
+
+import numpy
+
+import cupy
+from cupy.testing import _array
+from cupy.testing import _parameterized
+import cupyx
+import cupyx.scipy.sparse
+
+from cupy.testing._pytest_impl import is_available
+
+
+if is_available():
+    import _pytest.outcomes
+    _is_pytest_available = True
+    _skip_classes: Tuple[Type, ...] = (
+        unittest.SkipTest, _pytest.outcomes.Skipped)
+else:
+    _is_pytest_available = False
+    _skip_classes = unittest.SkipTest,
+
+
+def _format_exception(exc):
+    if exc is None:
+        return None
+    # TODO(kataoka): Use traceback.format_exception(exc) in Python 3.10
+    return ''.join(traceback.TracebackException.from_exception(exc).format())
+
+
+def _call_func(impl, args, kw):
+    # Note that `_pytest.outcomes.Skipped` is derived from BaseException.
+    exceptions = Exception,
+    if _is_pytest_available:
+        exceptions += _pytest.outcomes.Skipped,
+
+    try:
+        result = impl(*args, **kw)
+        error = None
+    except exceptions as e:
+        tb = e.__traceback__
+        if tb.tb_next is None:
+            # failed before impl is called, e.g. invalid kw
+            raise e
+        result = None
+        error = e
+
+    return result, error
+
+
+def _call_func_cupy(impl, args, kw, name, sp_name, scipy_name):
+    assert isinstance(name, str)
+    assert sp_name is None or isinstance(sp_name, str)
+    assert scipy_name is None or isinstance(scipy_name, str)
+    kw = kw.copy()
+
+    # Run cupy
+    if sp_name:
+        kw[sp_name] = cupyx.scipy.sparse
+    if scipy_name:
+        kw[scipy_name] = cupyx.scipy
+    kw[name] = cupy
+    result, error = _call_func(impl, args, kw)
+    return result, error
+
+
+def _call_func_numpy(impl, args, kw, name, sp_name, scipy_name):
+    assert isinstance(name, str)
+    assert sp_name is None or isinstance(sp_name, str)
+    assert scipy_name is None or isinstance(scipy_name, str)
+    kw = kw.copy()
+
+    # Run numpy
+    kw[name] = numpy
+    if sp_name:
+        import scipy.sparse
+        kw[sp_name] = scipy.sparse
+    if scipy_name:
+        import scipy
+        kw[scipy_name] = scipy
+    result, error = _call_func(impl, args, kw)
+    return result, error
+
+
+def _call_func_numpy_cupy(impl, args, kw, name, sp_name, scipy_name):
+    # Run cupy
+    cupy_result, cupy_error = _call_func_cupy(
+        impl, args, kw, name, sp_name, scipy_name)
+
+    # Run numpy
+    numpy_result, numpy_error = _call_func_numpy(
+        impl, args, kw, name, sp_name, scipy_name)
+
+    return (
+        cupy_result, cupy_error,
+        numpy_result, numpy_error)
+
+
+_numpy_errors = [
+    AttributeError, Exception, IndexError, TypeError, ValueError,
+    NotImplementedError, DeprecationWarning,
+    numpy.AxisError, numpy.linalg.LinAlgError,
+]
+
+
+def _check_numpy_cupy_error_compatible(cupy_error, numpy_error):
+    """Checks if try/except blocks are equivalent up to public error classes
+    """
+
+    return all(isinstance(cupy_error, err) == isinstance(numpy_error, err)
+               for err in _numpy_errors)
+
+
+def _fail_test_with_unexpected_errors(
+        tb, msg_format, cupy_error, numpy_error):
+    # Fails the test due to unexpected errors raised from the test.
+    # msg_format may include format placeholders:
+    # '{cupy_error}' '{numpy_error}'
+
+    msg = msg_format.format(
+        cupy_error=_format_exception(cupy_error),
+        numpy_error=_format_exception(numpy_error))
+
+    # Fail the test with the traceback of the error (for pytest --pdb)
+    raise AssertionError(msg).with_traceback(tb)
+
+
+def _check_cupy_numpy_error(cupy_error, numpy_error,
+                            accept_error=False):
+    # Skip the test if both raised SkipTest.
+    if (isinstance(cupy_error, _skip_classes)
+            and isinstance(numpy_error, _skip_classes)):
+        if cupy_error.__class__ is not numpy_error.__class__:
+            raise AssertionError(
+                'Both numpy and cupy were skipped but with different '
+                'exceptions.')
+        if cupy_error.args != numpy_error.args:
+            raise AssertionError(
+                'Both numpy and cupy were skipped but with different causes.')
+        raise numpy_error  # reraise SkipTest
+
+    # Check if the error was not raised from test code.
+    if os.environ.get('CUPY_CI', '') != '' and cupy_error is not None:
+        frame = traceback.extract_tb(cupy_error.__traceback__)[-1]
+        filename = os.path.basename(frame.filename)
+        if filename == 'test_helper.py':
+            # Allows errors from the test code for testing helpers.
+            pass
+        elif filename.startswith('test_'):
+            _fail_test_with_unexpected_errors(
+                cupy_error.__traceback__,
+                'Error was raised from test code.\n\n{cupy_error}',
+                cupy_error, None)
+
+    # For backward compatibility
+    if accept_error is True:
+        accept_error = Exception
+    elif not accept_error:
+        accept_error = ()
+    # TODO(oktua): expected_regexp like numpy.testing.assert_raises_regex
+    if cupy_error is None and numpy_error is None:
+        raise AssertionError(
+            'Both cupy and numpy are expected to raise errors, but not')
+    elif cupy_error is None:
+        _fail_test_with_unexpected_errors(
+            numpy_error.__traceback__,
+            'Only numpy raises error\n\n{numpy_error}',
+            None, numpy_error)
+    elif numpy_error is None:
+        _fail_test_with_unexpected_errors(
+            cupy_error.__traceback__,
+            'Only cupy raises error\n\n{cupy_error}',
+            cupy_error, None)
+
+    elif not _check_numpy_cupy_error_compatible(cupy_error, numpy_error):
+        _fail_test_with_unexpected_errors(
+            cupy_error.__traceback__,
+            '''Different types of errors occurred
+
+cupy
+{cupy_error}
+
+numpy
+{numpy_error}
+''',
+            cupy_error, numpy_error)
+
+    elif not (isinstance(cupy_error, accept_error)
+              and isinstance(numpy_error, accept_error)):
+        _fail_test_with_unexpected_errors(
+            cupy_error.__traceback__,
+            '''Both cupy and numpy raise exceptions
+
+cupy
+{cupy_error}
+
+numpy
+{numpy_error}
+''',
+            cupy_error, numpy_error)
+
+
+def _signed_counterpart(dtype):
+    return numpy.dtype(numpy.dtype(dtype).char.lower()).type
+
+
+def _make_positive_masks(impl, args, kw, name, sp_name, scipy_name):
+    # Returns a mask of output arrays that indicates valid elements for
+    # comparison. See the comment at the caller.
+    ks = [k for k, v in kw.items() if v in _unsigned_dtypes]
+    for k in ks:
+        kw[k] = _signed_counterpart(kw[k])
+    result, error = _call_func_cupy(
+        impl, args, kw, name, sp_name, scipy_name)
+    assert error is None
+    if not isinstance(result, (tuple, list)):
+        result = result,
+    return [cupy.asnumpy(r) >= 0 for r in result]
+
+
+def _contains_signed_and_unsigned(kw):
+    def isdtype(v):
+        if isinstance(v, numpy.dtype):
+            return True
+        elif isinstance(v, str):  # expecting dtype character codes
+            return True
+        elif isinstance(v, type) and issubclass(v, numpy.number):
+            return True
+        else:
+            return False
+    vs = set(v for v in kw.values() if isdtype(v))
+    return any(d in vs for d in _unsigned_dtypes) and \
+        any(d in vs for d in _float_dtypes + _signed_dtypes)
+
+
+def _wraps_partial(wrapped, *names):
+    # Only `wrapped` function have args of `names`.
+    def decorator(impl):
+        impl = functools.wraps(wrapped)(impl)
+        impl.__signature__ = inspect.signature(
+            functools.partial(wrapped, **{name: None for name in names}))
+        return impl
+    return decorator
+
+
+def _wraps_partial_xp(wrapped, name, sp_name, scipy_name):
+    names = [name, sp_name, scipy_name]
+    names = [n for n in names if n is not None]
+    return _wraps_partial(wrapped, *names)
+
+
+def _make_decorator(check_func, name, type_check, contiguous_check,
+                    accept_error, sp_name=None, scipy_name=None,
+                    check_sparse_format=True):
+    assert isinstance(name, str)
+    assert sp_name is None or isinstance(sp_name, str)
+    assert scipy_name is None or isinstance(scipy_name, str)
+
+    def decorator(impl):
+        @_wraps_partial_xp(impl, name, sp_name, scipy_name)
+        def test_func(*args, **kw):
+            # Run cupy and numpy
+            (
+                cupy_result, cupy_error,
+                numpy_result, numpy_error) = (
+                    _call_func_numpy_cupy(
+                        impl, args, kw, name, sp_name, scipy_name))
+            assert cupy_result is not None or cupy_error is not None
+            assert numpy_result is not None or numpy_error is not None
+
+            # Check errors raised
+            if cupy_error or numpy_error:
+                _check_cupy_numpy_error(cupy_error,
+                                        numpy_error,
+                                        accept_error=accept_error)
+                return
+
+            # Check returned arrays
+
+            if not isinstance(cupy_result, (tuple, list)):
+                cupy_result = cupy_result,
+            if not isinstance(numpy_result, (tuple, list)):
+                numpy_result = numpy_result,
+
+            assert len(cupy_result) == len(numpy_result)
+
+            # Check types
+            cupy_numpy_result_ndarrays = [
+                _convert_output_to_ndarray(
+                    cupy_r, numpy_r, sp_name, check_sparse_format)
+                for cupy_r, numpy_r in zip(cupy_result, numpy_result)]
+
+            # Check dtypes
+            if type_check:
+                for cupy_r, numpy_r in cupy_numpy_result_ndarrays:
+                    if cupy_r.dtype != numpy_r.dtype:
+                        raise AssertionError(
+                            '''ndarrays of different dtypes are returned.
+cupy: {}
+numpy: {}'''.format(cupy_r.dtype, numpy_r.dtype))
+
+            # Check contiguities
+            if contiguous_check:
+                for cupy_r, numpy_r in zip(cupy_result, numpy_result):
+                    if isinstance(numpy_r, numpy.ndarray):
+                        if (numpy_r.flags.c_contiguous
+                                and not cupy_r.flags.c_contiguous):
+                            raise AssertionError(
+                                'The state of c_contiguous flag is false. '
+                                '(cupy_result:{} numpy_result:{})'.format(
+                                    cupy_r.flags.c_contiguous,
+                                    numpy_r.flags.c_contiguous))
+                        if (numpy_r.flags.f_contiguous
+                                and not cupy_r.flags.f_contiguous):
+                            raise AssertionError(
+                                'The state of f_contiguous flag is false. '
+                                '(cupy_result:{} numpy_result:{})'.format(
+                                    cupy_r.flags.f_contiguous,
+                                    numpy_r.flags.f_contiguous))
+
+            # Check shapes
+            for cupy_r, numpy_r in cupy_numpy_result_ndarrays:
+                assert cupy_r.shape == numpy_r.shape
+
+            masks = [None] * len(cupy_result)
+            if _contains_signed_and_unsigned(kw):
+                needs_mask = [
+                    cupy_r.dtype in _unsigned_dtypes
+                    for cupy_r in cupy_result]
+                if any(needs_mask):
+                    masks = _make_positive_masks(
+                        impl, args, kw, name, sp_name, scipy_name)
+                    for i, flag in enumerate(needs_mask):
+                        if not flag:
+                            masks[i] = None
+
+            # Check item values
+            for (cupy_r, numpy_r), mask in zip(
+                    cupy_numpy_result_ndarrays, masks):
+                # Behavior of assigning a negative value to an unsigned integer
+                # variable is undefined.
+                # nVidia GPUs and Intel CPUs behave differently.
+                # To avoid this difference, we need to ignore dimensions whose
+                # values are negative.
+
+                skip = False
+                if mask is not None:
+                    if cupy_r.shape == ():
+                        skip = (mask == 0).all()
+                    else:
+                        cupy_r = cupy_r[mask].get()
+                        numpy_r = numpy_r[mask]
+
+                if not skip:
+                    check_func(cupy_r, numpy_r)
+        return test_func
+    return decorator
+
+
+def _convert_output_to_ndarray(c_out, n_out, sp_name, check_sparse_format):
+    """Checks type of cupy/numpy results and returns cupy/numpy ndarrays.
+
+    Args:
+        c_out (cupy.ndarray, cupyx.scipy.sparse matrix, cupy.poly1d or scalar):
+            cupy result
+        n_out (numpy.ndarray, scipy.sparse matrix, numpy.poly1d or scalar):
+            numpy result
+        sp_name(str or None): Argument name whose value is either
+            ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+            argument is given for the modules.
+        check_sparse_format (bool): If ``True``, consistency of format of
+            sparse matrix is also checked. Default is ``True``.
+
+    Returns:
+        The tuple of cupy.ndarray and numpy.ndarray.
+    """
+    if sp_name is not None and cupyx.scipy.sparse.issparse(c_out):
+        # Sparse output case.
+        import scipy.sparse
+        assert scipy.sparse.issparse(n_out)
+        if check_sparse_format:
+            assert c_out.format == n_out.format
+        return c_out.A, n_out.A
+    if (isinstance(c_out, cupy.ndarray)
+            and isinstance(n_out, (numpy.ndarray, numpy.generic))):
+        # ndarray output case.
+        return c_out, n_out
+    if isinstance(c_out, cupy.poly1d) and isinstance(n_out, numpy.poly1d):
+        # poly1d output case.
+        assert c_out.variable == n_out.variable
+        return c_out.coeffs, n_out.coeffs
+    if isinstance(c_out, numpy.generic) and isinstance(n_out, numpy.generic):
+        # numpy scalar output case.
+        return c_out, n_out
+    if numpy.isscalar(c_out) and numpy.isscalar(n_out):
+        # python scalar output case.
+        return cupy.array(c_out), numpy.array(n_out)
+    raise AssertionError(
+        'numpy and cupy returns different type of return value:\n'
+        'cupy: {}\nnumpy: {}'.format(
+            type(c_out), type(n_out)))
+
+
+def _check_tolerance_keys(rtol, atol):
+    def _check(tol):
+        if isinstance(tol, dict):
+            for k in tol.keys():
+                if type(k) is type:
+                    continue
+                if type(k) is str and k == 'default':
+                    continue
+                msg = ('Keys of the tolerance dictionary need to be type '
+                       'objects as `numpy.float32` and `cupy.float32` or '
+                       '`\'default\'` string.')
+                raise TypeError(msg)
+    _check(rtol)
+    _check(atol)
+
+
+def _resolve_tolerance(type_check, result, rtol, atol):
+    def _resolve(dtype, tol):
+        if isinstance(tol, dict):
+            tol1 = tol.get(dtype.type)
+            if tol1 is None:
+                tol1 = tol.get('default')
+                if tol1 is None:
+                    raise TypeError(
+                        'Can not find tolerance for {}'.format(dtype.type))
+            return tol1
+        else:
+            return tol
+
+    dtype = result.dtype
+    rtol1 = _resolve(dtype, rtol)
+    atol1 = _resolve(dtype, atol)
+    return rtol1, atol1
+
+
+def numpy_cupy_allclose(rtol=1e-7, atol=0, err_msg='', verbose=True,
+                        name='xp', type_check=True, accept_error=False,
+                        sp_name=None, scipy_name=None, contiguous_check=True,
+                        *, _check_sparse_format=True):
+    """Decorator that checks NumPy results and CuPy ones are close.
+
+    Args:
+         rtol(float or dict): Relative tolerance. Besides a float value, a
+             dictionary that maps a dtypes to a float value can be supplied to
+             adjust tolerance per dtype. If the dictionary has ``'default'``
+             string as its key, its value is used as the default tolerance in
+             case any dtype keys do not match.
+         atol(float or dict): Absolute tolerance. Besides a float value, a
+             dictionary can be supplied as ``rtol``.
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values are
+             appended to the error message.
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         type_check(bool): If ``True``, consistency of dtype is also checked.
+         accept_error(bool, Exception or tuple of Exception): Specify
+             acceptable errors. When both NumPy test and CuPy test raises the
+             same type of errors, and the type of the errors is specified with
+             this argument, the errors are ignored and not raised.
+             If it is ``True`` all error types are acceptable.
+             If it is ``False`` no error is acceptable.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+         contiguous_check(bool): If ``True``, consistency of contiguity is
+             also checked.
+
+    Decorated test fixture is required to return the arrays whose values are
+    close between ``numpy`` case and ``cupy`` case.
+    For example, this test case checks ``numpy.zeros`` and ``cupy.zeros``
+    should return same value.
+
+    >>> import unittest
+    >>> from cupy import testing
+    >>> class TestFoo(unittest.TestCase):
+    ...
+    ...     @testing.numpy_cupy_allclose()
+    ...     def test_foo(self, xp):
+    ...         # ...
+    ...         # Prepare data with xp
+    ...         # ...
+    ...
+    ...         xp_result = xp.zeros(10)
+    ...         return xp_result
+
+    .. seealso:: :func:`cupy.testing.assert_allclose`
+    """
+    _check_tolerance_keys(rtol, atol)
+
+    # When `type_check` is `False`, cupy result and numpy result may have
+    # different dtypes so we can not determine the dtype to use from the
+    # tolerance associations.
+    if not type_check:
+        if isinstance(rtol, dict) or isinstance(atol, dict):
+            raise TypeError('When `type_check` is `False`, `rtol` and `atol` '
+                            'must be supplied as float.')
+
+    def check_func(c, n):
+        rtol1, atol1 = _resolve_tolerance(type_check, c, rtol, atol)
+        _array.assert_allclose(c, n, rtol1, atol1, err_msg, verbose)
+    return _make_decorator(check_func, name, type_check, contiguous_check,
+                           accept_error, sp_name, scipy_name,
+                           _check_sparse_format)
+
+
+def numpy_cupy_array_almost_equal(decimal=6, err_msg='', verbose=True,
+                                  name='xp', type_check=True,
+                                  accept_error=False, sp_name=None,
+                                  scipy_name=None):
+    """Decorator that checks NumPy results and CuPy ones are almost equal.
+
+    Args:
+         decimal(int): Desired precision.
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values
+             are appended to the error message.
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         type_check(bool): If ``True``, consistency of dtype is also checked.
+         accept_error(bool, Exception or tuple of Exception): Specify
+             acceptable errors. When both NumPy test and CuPy test raises the
+             same type of errors, and the type of the errors is specified with
+             this argument, the errors are ignored and not raised.
+             If it is ``True`` all error types are acceptable.
+             If it is ``False`` no error is acceptable.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+
+    Decorated test fixture is required to return the same arrays
+    in the sense of :func:`cupy.testing.assert_array_almost_equal`
+    (except the type of array module) even if ``xp`` is ``numpy`` or ``cupy``.
+
+    .. seealso:: :func:`cupy.testing.assert_array_almost_equal`
+    """
+    def check_func(x, y):
+        _array.assert_array_almost_equal(x, y, decimal, err_msg, verbose)
+    return _make_decorator(check_func, name, type_check, False,
+                           accept_error, sp_name, scipy_name)
+
+
+def numpy_cupy_array_almost_equal_nulp(nulp=1, name='xp', type_check=True,
+                                       accept_error=False, sp_name=None,
+                                       scipy_name=None):
+    """Decorator that checks results of NumPy and CuPy are equal w.r.t. spacing.
+
+    Args:
+         nulp(int): The maximum number of unit in the last place for tolerance.
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         type_check(bool): If ``True``, consistency of dtype is also checked.
+         accept_error(bool, Exception or tuple of Exception): Specify
+             acceptable errors. When both NumPy test and CuPy test raises the
+             same type of errors, and the type of the errors is specified with
+             this argument, the errors are ignored and not raised.
+             If it is ``True``, all error types are acceptable.
+             If it is ``False``, no error is acceptable.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+
+    Decorated test fixture is required to return the same arrays
+    in the sense of :func:`cupy.testing.assert_array_almost_equal_nulp`
+    (except the type of array module) even if ``xp`` is ``numpy`` or ``cupy``.
+
+    .. seealso:: :func:`cupy.testing.assert_array_almost_equal_nulp`
+    """  # NOQA
+    def check_func(x, y):
+        _array.assert_array_almost_equal_nulp(x, y, nulp)
+    return _make_decorator(check_func, name, type_check, False,
+                           accept_error, sp_name, scipy_name=None)
+
+
+def numpy_cupy_array_max_ulp(maxulp=1, dtype=None, name='xp', type_check=True,
+                             accept_error=False, sp_name=None,
+                             scipy_name=None):
+    """Decorator that checks results of NumPy and CuPy ones are equal w.r.t. ulp.
+
+    Args:
+         maxulp(int): The maximum number of units in the last place
+             that elements of resulting two arrays can differ.
+         dtype(numpy.dtype): Data-type to convert the resulting
+             two array to if given.
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         type_check(bool): If ``True``, consistency of dtype is also checked.
+         accept_error(bool, Exception or tuple of Exception): Specify
+             acceptable errors. When both NumPy test and CuPy test raises the
+             same type of errors, and the type of the errors is specified with
+             this argument, the errors are ignored and not raised.
+             If it is ``True`` all error types are acceptable.
+             If it is ``False`` no error is acceptable.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+
+    Decorated test fixture is required to return the same arrays
+    in the sense of :func:`assert_array_max_ulp`
+    (except the type of array module) even if ``xp`` is ``numpy`` or ``cupy``.
+
+    .. seealso:: :func:`cupy.testing.assert_array_max_ulp`
+
+    """  # NOQA
+    def check_func(x, y):
+        _array.assert_array_max_ulp(x, y, maxulp, dtype)
+    return _make_decorator(check_func, name, type_check, False,
+                           accept_error, sp_name, scipy_name)
+
+
+def numpy_cupy_array_equal(err_msg='', verbose=True, name='xp',
+                           type_check=True, accept_error=False, sp_name=None,
+                           scipy_name=None, strides_check=False):
+    """Decorator that checks NumPy results and CuPy ones are equal.
+
+    Args:
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values are
+             appended to the error message.
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         type_check(bool): If ``True``, consistency of dtype is also checked.
+         accept_error(bool, Exception or tuple of Exception): Specify
+             acceptable errors. When both NumPy test and CuPy test raises the
+             same type of errors, and the type of the errors is specified with
+             this argument, the errors are ignored and not raised.
+             If it is ``True`` all error types are acceptable.
+             If it is ``False`` no error is acceptable.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+         strides_check(bool): If ``True``, consistency of strides is also
+             checked.
+
+    Decorated test fixture is required to return the same arrays
+    in the sense of :func:`numpy_cupy_array_equal`
+    (except the type of array module) even if ``xp`` is ``numpy`` or ``cupy``.
+
+    .. seealso:: :func:`cupy.testing.assert_array_equal`
+    """
+    def check_func(x, y):
+        _array.assert_array_equal(x, y, err_msg, verbose, strides_check)
+    return _make_decorator(check_func, name, type_check, False,
+                           accept_error, sp_name, scipy_name)
+
+
+def numpy_cupy_array_list_equal(
+        err_msg='', verbose=True, name='xp', sp_name=None, scipy_name=None):
+    """Decorator that checks the resulting lists of NumPy and CuPy's one are equal.
+
+    Args:
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values are appended
+             to the error message.
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+
+    Decorated test fixture is required to return the same list of arrays
+    (except the type of array module) even if ``xp`` is ``numpy`` or ``cupy``.
+
+    .. seealso:: :func:`cupy.testing.assert_array_list_equal`
+    """  # NOQA
+    warnings.warn(
+        'numpy_cupy_array_list_equal is deprecated.'
+        ' Use numpy_cupy_array_equal instead.',
+        DeprecationWarning)
+
+    def check_func(x, y):
+        _array.assert_array_equal(x, y, err_msg, verbose)
+    return _make_decorator(check_func, name, False, False,
+                           False, sp_name, scipy_name)
+
+
+def numpy_cupy_array_less(err_msg='', verbose=True, name='xp',
+                          type_check=True, accept_error=False, sp_name=None,
+                          scipy_name=None):
+    """Decorator that checks the CuPy result is less than NumPy result.
+
+    Args:
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values are
+             appended to the error message.
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         type_check(bool): If ``True``, consistency of dtype is also checked.
+         accept_error(bool, Exception or tuple of Exception): Specify
+             acceptable errors. When both NumPy test and CuPy test raises the
+             same type of errors, and the type of the errors is specified with
+             this argument, the errors are ignored and not raised.
+             If it is ``True`` all error types are acceptable.
+             If it is ``False`` no error is acceptable.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+
+    Decorated test fixture is required to return the smaller array
+    when ``xp`` is ``cupy`` than the one when ``xp`` is ``numpy``.
+
+    .. seealso:: :func:`cupy.testing.assert_array_less`
+    """
+    def check_func(x, y):
+        _array.assert_array_less(x, y, err_msg, verbose)
+    return _make_decorator(check_func, name, type_check, False,
+                           accept_error, sp_name, scipy_name)
+
+
+def numpy_cupy_equal(name='xp', sp_name=None, scipy_name=None):
+    """Decorator that checks NumPy results are equal to CuPy ones.
+
+    Args:
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+
+    Decorated test fixture is required to return the same results
+    even if ``xp`` is ``numpy`` or ``cupy``.
+    """
+    def decorator(impl):
+        @_wraps_partial_xp(impl, name, sp_name, scipy_name)
+        def test_func(*args, **kw):
+            # Run cupy and numpy
+            (
+                cupy_result, cupy_error,
+                numpy_result, numpy_error) = (
+                    _call_func_numpy_cupy(
+                        impl, args, kw, name, sp_name, scipy_name))
+
+            if cupy_error or numpy_error:
+                _check_cupy_numpy_error(
+                    cupy_error, numpy_error,
+                    accept_error=False)
+                return
+
+            if cupy_result != numpy_result:
+                message = '''Results are not equal:
+cupy: %s
+numpy: %s''' % (str(cupy_result), str(numpy_result))
+                raise AssertionError(message)
+        return test_func
+    return decorator
+
+
+def numpy_cupy_raises(name='xp', sp_name=None, scipy_name=None,
+                      accept_error=Exception):
+    """Decorator that checks the NumPy and CuPy throw same errors.
+
+    Args:
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+         accept_error(bool, Exception or tuple of Exception): Specify
+             acceptable errors. When both NumPy test and CuPy test raises the
+             same type of errors, and the type of the errors is specified with
+             this argument, the errors are ignored and not raised.
+             If it is ``True`` all error types are acceptable.
+             If it is ``False`` no error is acceptable.
+
+    Decorated test fixture is required throw same errors
+    even if ``xp`` is ``numpy`` or ``cupy``.
+    """
+    warnings.warn(
+        'cupy.testing.numpy_cupy_raises is deprecated.',
+        DeprecationWarning)
+
+    def decorator(impl):
+        @_wraps_partial_xp(impl, name, sp_name, scipy_name)
+        def test_func(*args, **kw):
+            # Run cupy and numpy
+            (
+                cupy_result, cupy_error,
+                numpy_result, numpy_error) = (
+                    _call_func_numpy_cupy(
+                        impl, args, kw, name, sp_name, scipy_name))
+
+            _check_cupy_numpy_error(cupy_error,
+                                    numpy_error,
+                                    accept_error=accept_error)
+        return test_func
+    return decorator
+
+
+def for_dtypes(dtypes, name='dtype'):
+    """Decorator for parameterized dtype test.
+
+    Args:
+         dtypes(list of dtypes): dtypes to be tested.
+         name(str): Argument name to which specified dtypes are passed.
+
+    This decorator adds a keyword argument specified by ``name``
+    to the test fixture. Then, it runs the fixtures in parallel
+    by passing the each element of ``dtypes`` to the named
+    argument.
+    """
+    def decorator(impl):
+        @_wraps_partial(impl, name)
+        def test_func(*args, **kw):
+            for dtype in dtypes:
+                try:
+                    kw[name] = numpy.dtype(dtype).type
+                    impl(*args, **kw)
+                except _skip_classes as e:
+                    print('skipped: {} = {} ({})'.format(name, dtype, e))
+                except Exception:
+                    print(name, 'is', dtype)
+                    raise
+
+        return test_func
+    return decorator
+
+
+_complex_dtypes = (numpy.complex64, numpy.complex128)
+_regular_float_dtypes = (numpy.float64, numpy.float32)
+_float_dtypes = _regular_float_dtypes + (numpy.float16,)
+_signed_dtypes = tuple(numpy.dtype(i).type for i in 'bhilq')
+_unsigned_dtypes = tuple(numpy.dtype(i).type for i in 'BHILQ')
+_int_dtypes = _signed_dtypes + _unsigned_dtypes
+_int_bool_dtypes = _int_dtypes + (numpy.bool_,)
+_regular_dtypes = _regular_float_dtypes + _int_bool_dtypes
+_dtypes = _float_dtypes + _int_bool_dtypes
+
+
+def _make_all_dtypes(no_float16, no_bool, no_complex):
+    if no_float16:
+        dtypes = _regular_float_dtypes
+    else:
+        dtypes = _float_dtypes
+
+    if no_bool:
+        dtypes += _int_dtypes
+    else:
+        dtypes += _int_bool_dtypes
+
+    if not no_complex:
+        dtypes += _complex_dtypes
+
+    return dtypes
+
+
+def for_all_dtypes(name='dtype', no_float16=False, no_bool=False,
+                   no_complex=False):
+    """Decorator that checks the fixture with all dtypes.
+
+    Args:
+         name(str): Argument name to which specified dtypes are passed.
+         no_float16(bool): If ``True``, ``numpy.float16`` is
+             omitted from candidate dtypes.
+         no_bool(bool): If ``True``, ``numpy.bool_`` is
+             omitted from candidate dtypes.
+         no_complex(bool): If ``True``, ``numpy.complex64`` and
+             ``numpy.complex128`` are omitted from candidate dtypes.
+
+    dtypes to be tested: ``numpy.complex64`` (optional),
+    ``numpy.complex128`` (optional),
+    ``numpy.float16`` (optional), ``numpy.float32``,
+    ``numpy.float64``, ``numpy.dtype('b')``, ``numpy.dtype('h')``,
+    ``numpy.dtype('i')``, ``numpy.dtype('l')``, ``numpy.dtype('q')``,
+    ``numpy.dtype('B')``, ``numpy.dtype('H')``, ``numpy.dtype('I')``,
+    ``numpy.dtype('L')``, ``numpy.dtype('Q')``, and ``numpy.bool_`` (optional).
+
+    The usage is as follows.
+    This test fixture checks if ``cPickle`` successfully reconstructs
+    :class:`cupy.ndarray` for various dtypes.
+    ``dtype`` is an argument inserted by the decorator.
+
+    >>> import unittest
+    >>> from cupy import testing
+    >>> class TestNpz(unittest.TestCase):
+    ...
+    ...     @testing.for_all_dtypes()
+    ...     def test_pickle(self, dtype):
+    ...         a = testing.shaped_arange((2, 3, 4), dtype=dtype)
+    ...         s = pickle.dumps(a)
+    ...         b = pickle.loads(s)
+    ...         testing.assert_array_equal(a, b)
+
+    Typically, we use this decorator in combination with
+    decorators that check consistency between NumPy and CuPy like
+    :func:`cupy.testing.numpy_cupy_allclose`.
+    The following is such an example.
+
+    >>> import unittest
+    >>> from cupy import testing
+    >>> class TestMean(unittest.TestCase):
+    ...
+    ...     @testing.for_all_dtypes()
+    ...     @testing.numpy_cupy_allclose()
+    ...     def test_mean_all(self, xp, dtype):
+    ...         a = testing.shaped_arange((2, 3), xp, dtype)
+    ...         return a.mean()
+
+    .. seealso:: :func:`cupy.testing.for_dtypes`
+    """
+    return for_dtypes(_make_all_dtypes(no_float16, no_bool, no_complex),
+                      name=name)
+
+
+def for_float_dtypes(name='dtype', no_float16=False):
+    """Decorator that checks the fixture with float dtypes.
+
+    Args:
+         name(str): Argument name to which specified dtypes are passed.
+         no_float16(bool): If ``True``, ``numpy.float16`` is
+             omitted from candidate dtypes.
+
+    dtypes to be tested are ``numpy.float16`` (optional), ``numpy.float32``,
+    and ``numpy.float64``.
+
+    .. seealso:: :func:`cupy.testing.for_dtypes`,
+        :func:`cupy.testing.for_all_dtypes`
+    """
+    if no_float16:
+        return for_dtypes(_regular_float_dtypes, name=name)
+    else:
+        return for_dtypes(_float_dtypes, name=name)
+
+
+def for_signed_dtypes(name='dtype'):
+    """Decorator that checks the fixture with signed dtypes.
+
+    Args:
+         name(str): Argument name to which specified dtypes are passed.
+
+    dtypes to be tested are ``numpy.dtype('b')``, ``numpy.dtype('h')``,
+    ``numpy.dtype('i')``, ``numpy.dtype('l')``, and ``numpy.dtype('q')``.
+
+    .. seealso:: :func:`cupy.testing.for_dtypes`,
+        :func:`cupy.testing.for_all_dtypes`
+    """
+    return for_dtypes(_signed_dtypes, name=name)
+
+
+def for_unsigned_dtypes(name='dtype'):
+    """Decorator that checks the fixture with unsinged dtypes.
+
+    Args:
+         name(str): Argument name to which specified dtypes are passed.
+
+    dtypes to be tested are ``numpy.dtype('B')``, ``numpy.dtype('H')``,
+
+     ``numpy.dtype('I')``, ``numpy.dtype('L')``, and ``numpy.dtype('Q')``.
+
+    .. seealso:: :func:`cupy.testing.for_dtypes`,
+        :func:`cupy.testing.for_all_dtypes`
+    """
+    return for_dtypes(_unsigned_dtypes, name=name)
+
+
+def for_int_dtypes(name='dtype', no_bool=False):
+    """Decorator that checks the fixture with integer and optionally bool dtypes.
+
+    Args:
+         name(str): Argument name to which specified dtypes are passed.
+         no_bool(bool): If ``True``, ``numpy.bool_`` is
+             omitted from candidate dtypes.
+
+    dtypes to be tested are ``numpy.dtype('b')``, ``numpy.dtype('h')``,
+    ``numpy.dtype('i')``, ``numpy.dtype('l')``, ``numpy.dtype('q')``,
+    ``numpy.dtype('B')``, ``numpy.dtype('H')``, ``numpy.dtype('I')``,
+    ``numpy.dtype('L')``, ``numpy.dtype('Q')``, and ``numpy.bool_`` (optional).
+
+    .. seealso:: :func:`cupy.testing.for_dtypes`,
+        :func:`cupy.testing.for_all_dtypes`
+    """  # NOQA
+    if no_bool:
+        return for_dtypes(_int_dtypes, name=name)
+    else:
+        return for_dtypes(_int_bool_dtypes, name=name)
+
+
+def for_complex_dtypes(name='dtype'):
+    """Decorator that checks the fixture with complex dtypes.
+
+    Args:
+         name(str): Argument name to which specified dtypes are passed.
+
+    dtypes to be tested are ``numpy.complex64`` and ``numpy.complex128``.
+
+    .. seealso:: :func:`cupy.testing.for_dtypes`,
+        :func:`cupy.testing.for_all_dtypes`
+    """
+    return for_dtypes(_complex_dtypes, name=name)
+
+
+def for_dtypes_combination(types, names=('dtype',), full=None):
+    """Decorator that checks the fixture with a product set of dtypes.
+
+    Args:
+         types(list of dtypes): dtypes to be tested.
+         names(list of str): Argument names to which dtypes are passed.
+         full(bool): If ``True``, then all combinations
+             of dtypes will be tested.
+             Otherwise, the subset of combinations will be tested
+             (see the description below).
+
+    Decorator adds the keyword arguments specified by ``names``
+    to the test fixture. Then, it runs the fixtures in parallel
+    with passing (possibly a subset of) the product set of dtypes.
+    The range of dtypes is specified by ``types``.
+
+    The combination of dtypes to be tested changes depending
+    on the option ``full``. If ``full`` is ``True``,
+    all combinations of ``types`` are tested.
+    Sometimes, such an exhaustive test can be costly.
+    So, if ``full`` is ``False``, only a subset of possible combinations
+    is randomly sampled. If ``full`` is ``None``, the behavior is
+    determined by an environment variable ``CUPY_TEST_FULL_COMBINATION``.
+    If the value is set to ``'1'``, it behaves as if ``full=True``, and
+    otherwise ``full=False``.
+    """
+    types = list(types)
+
+    if len(types) == 1:
+        name, = names
+        return for_dtypes(types, name)
+
+    if full is None:
+        full = int(os.environ.get('CUPY_TEST_FULL_COMBINATION', '0')) != 0
+
+    if full:
+        combination = _parameterized.product({name: types for name in names})
+    else:
+        ts = []
+        for _ in range(len(names)):
+            # Make shuffled list of types for each name
+            shuffled_types = types[:]
+            random.shuffle(shuffled_types)
+            ts.append(types + shuffled_types)
+
+        combination = [tuple(zip(names, typs)) for typs in zip(*ts)]
+        # Remove duplicate entries
+        combination = [dict(assoc_list) for assoc_list in set(combination)]
+
+    def decorator(impl):
+        @_wraps_partial(impl, *names)
+        def test_func(*args, **kw):
+            for dtypes in combination:
+                kw_copy = kw.copy()
+                kw_copy.update(dtypes)
+
+                try:
+                    impl(*args, **kw_copy)
+                except _skip_classes as e:
+                    msg = ', '.join(
+                        '{} = {}'.format(name, dtype)
+                        for name, dtype in dtypes.items())
+                    print('skipped: {} ({})'.format(msg, e))
+                except Exception:
+                    print(dtypes)
+                    raise
+
+        return test_func
+    return decorator
+
+
+def for_all_dtypes_combination(names=('dtyes',),
+                               no_float16=False, no_bool=False, full=None,
+                               no_complex=False):
+    """Decorator that checks the fixture with a product set of all dtypes.
+
+    Args:
+         names(list of str): Argument names to which dtypes are passed.
+         no_float16(bool): If ``True``, ``numpy.float16`` is
+             omitted from candidate dtypes.
+         no_bool(bool): If ``True``, ``numpy.bool_`` is
+             omitted from candidate dtypes.
+         full(bool): If ``True``, then all combinations of dtypes
+             will be tested.
+             Otherwise, the subset of combinations will be tested
+             (see description in :func:`cupy.testing.for_dtypes_combination`).
+         no_complex(bool): If, True, ``numpy.complex64`` and
+             ``numpy.complex128`` are omitted from candidate dtypes.
+
+    .. seealso:: :func:`cupy.testing.for_dtypes_combination`
+    """
+    types = _make_all_dtypes(no_float16, no_bool, no_complex)
+    return for_dtypes_combination(types, names, full)
+
+
+def for_signed_dtypes_combination(names=('dtype',), full=None):
+    """Decorator for parameterized test w.r.t. the product set of signed dtypes.
+
+    Args:
+         names(list of str): Argument names to which dtypes are passed.
+         full(bool): If ``True``, then all combinations of dtypes
+             will be tested.
+             Otherwise, the subset of combinations will be tested
+             (see description in :func:`cupy.testing.for_dtypes_combination`).
+
+    .. seealso:: :func:`cupy.testing.for_dtypes_combination`
+    """  # NOQA
+    return for_dtypes_combination(_signed_dtypes, names=names, full=full)
+
+
+def for_unsigned_dtypes_combination(names=('dtype',), full=None):
+    """Decorator for parameterized test w.r.t. the product set of unsigned dtypes.
+
+    Args:
+         names(list of str): Argument names to which dtypes are passed.
+         full(bool): If ``True``, then all combinations of dtypes
+             will be tested.
+             Otherwise, the subset of combinations will be tested
+             (see description in :func:`cupy.testing.for_dtypes_combination`).
+
+    .. seealso:: :func:`cupy.testing.for_dtypes_combination`
+    """  # NOQA
+    return for_dtypes_combination(_unsigned_dtypes, names=names, full=full)
+
+
+def for_int_dtypes_combination(names=('dtype',), no_bool=False, full=None):
+    """Decorator for parameterized test w.r.t. the product set of int and boolean.
+
+    Args:
+         names(list of str): Argument names to which dtypes are passed.
+         no_bool(bool): If ``True``, ``numpy.bool_`` is
+             omitted from candidate dtypes.
+         full(bool): If ``True``, then all combinations of dtypes
+             will be tested.
+             Otherwise, the subset of combinations will be tested
+             (see description in :func:`cupy.testing.for_dtypes_combination`).
+
+    .. seealso:: :func:`cupy.testing.for_dtypes_combination`
+    """  # NOQA
+    if no_bool:
+        types = _int_dtypes
+    else:
+        types = _int_bool_dtypes
+    return for_dtypes_combination(types, names, full)
+
+
+def for_orders(orders, name='order'):
+    """Decorator to parameterize tests with order.
+
+    Args:
+         orders(list of order): orders to be tested.
+         name(str): Argument name to which the specified order is passed.
+
+    This decorator adds a keyword argument specified by ``name``
+    to the test fixtures. Then, the fixtures run by passing each element of
+    ``orders`` to the named argument.
+
+    """
+    def decorator(impl):
+        @_wraps_partial(impl, name)
+        def test_func(*args, **kw):
+            for order in orders:
+                try:
+                    kw[name] = order
+                    impl(*args, **kw)
+                except Exception:
+                    print(name, 'is', order)
+                    raise
+
+        return test_func
+    return decorator
+
+
+def for_CF_orders(name='order'):
+    """Decorator that checks the fixture with orders 'C' and 'F'.
+
+    Args:
+         name(str): Argument name to which the specified order is passed.
+
+    .. seealso:: :func:`cupy.testing.for_all_dtypes`
+
+    """
+    return for_orders([None, 'C', 'F', 'c', 'f'], name)
+
+
+def for_contiguous_axes(name='axis'):
+    '''Decorator for parametrizing tests with possible contiguous axes.
+
+    Args:
+        name(str): Argument name to which specified axis are passed.
+
+    .. note::
+        1. Adapted from tests/cupy_tests/fft_tests/test_fft.py.
+        2. Example: for ``shape = (1, 2, 3)``, the tested axes are
+            ``[(2,), (1, 2), (0, 1, 2)]`` for the C order, and
+            ``[(0,), (0, 1), (0, 1, 2)]`` for the F order.
+    '''
+    def decorator(impl):
+        @_wraps_partial(impl, name)
+        def test_func(self, *args, **kw):
+            ndim = len(self.shape)
+            order = self.order
+            for i in range(ndim):
+                a = ()
+                if order in ('c', 'C'):
+                    for j in range(ndim-1, i-1, -1):
+                        a = (j,) + a
+                elif order in ('f', 'F'):
+                    for j in range(0, i+1):
+                        a = a + (j,)
+                else:
+                    raise ValueError('Please specify the array order.')
+                try:
+                    kw[name] = a
+                    impl(self, *args, **kw)
+                except Exception:
+                    print(name, 'is', a, ', ndim is', ndim, ', shape is',
+                          self.shape, ', order is', order)
+                    raise
+        return test_func
+    return decorator
diff --git a/cupy/testing/_parameterized.py b/cupy/testing/_parameterized.py
new file mode 100644
index 0000000..203f78c
--- /dev/null
+++ b/cupy/testing/_parameterized.py
@@ -0,0 +1,115 @@
+import itertools
+import types
+import unittest
+
+from cupy.testing import _bundle
+from cupy.testing import _pytest_impl
+
+
+def _param_to_str(obj):
+    if isinstance(obj, type):
+        return obj.__name__
+    elif hasattr(obj, '__name__') and isinstance(obj.__name__, str):
+        # print __name__ attribute for classes, functions and modules
+        return obj.__name__
+    return repr(obj)
+
+
+def _shorten(s, maxlen):
+    # Shortens the string down to maxlen, by replacing the middle part with
+    # a 3-dots string '...'.
+    ellipsis = '...'
+    if len(s) <= maxlen:
+        return s
+    n1 = (maxlen - len(ellipsis)) // 2
+    n2 = maxlen - len(ellipsis) - n1
+    s = s[:n1] + ellipsis + s[-n2:]
+    assert len(s) == maxlen
+    return s
+
+
+def _make_class_name(base_class_name, i_param, param):
+    # Creates a class name for a single combination of parameters.
+    SINGLE_PARAM_MAXLEN = 100  # Length limit of a single parameter value
+    PARAMS_MAXLEN = 5000  # Length limit of the whole parameters part
+    param_strs = [
+        '{}={}'.format(k, _shorten(_param_to_str(v), SINGLE_PARAM_MAXLEN))
+        for k, v in sorted(param.items())]
+    param_strs = _shorten(', '.join(param_strs), PARAMS_MAXLEN)
+    cls_name = '{}_param_{}_{{{}}}'.format(
+        base_class_name, i_param, param_strs)
+    return cls_name
+
+
+def _parameterize_test_case_generator(base, params):
+    # Defines the logic to generate parameterized test case classes.
+
+    for i, param in enumerate(params):
+        yield _parameterize_test_case(base, i, param)
+
+
+def _parameterize_test_case(base, i, param):
+    cls_name = _make_class_name(base.__name__, i, param)
+
+    def __repr__(self):
+        name = base.__repr__(self)
+        return '<%s  parameter: %s>' % (name, param)
+
+    mb = {'__repr__': __repr__}
+    for k, v in sorted(param.items()):
+        if isinstance(v, types.FunctionType):
+
+            def create_new_v():
+                f = v
+
+                def new_v(self, *args, **kwargs):
+                    return f(*args, **kwargs)
+                return new_v
+
+            mb[k] = create_new_v()
+        else:
+            mb[k] = v
+
+    return (cls_name, mb, lambda method: method)
+
+
+def parameterize(*params, **kwargs):
+    """Generates test classes with given sets of additional attributes
+
+    >>> @parameterize({"a": 1}, {"b": 2, "c": 3})
+    ... class TestX(unittest.TestCase):
+    ...     def test_y(self):
+    ...         pass
+
+    generates two classes `TestX_param_0_...`, `TestX_param_1_...` and
+    removes the original class `TestX`.
+
+    The specification is subject to change, which applies to all the non-NumPy
+    `testing` features.
+
+    """
+    def f(cls):
+        if issubclass(cls, unittest.TestCase):
+            assert not kwargs
+            deco = _bundle.make_decorator(
+                lambda base: _parameterize_test_case_generator(base, params))
+        else:
+            deco = _pytest_impl.parameterize(*params, **kwargs)
+        return deco(cls)
+    return f
+
+
+def product(parameter):
+    # TODO(kataoka): Add documentation
+    assert isinstance(parameter, dict)
+    keys = sorted(parameter)
+    values = [parameter[key] for key in keys]
+    values_product = itertools.product(*values)
+    return [dict(zip(keys, vals)) for vals in values_product]
+
+
+def product_dict(*parameters):
+    # TODO(kataoka): Add documentation
+    return [
+        {k: v for dic in dicts for k, v in dic.items()}
+        for dicts in itertools.product(*parameters)]
diff --git a/cupy/testing/_pytest_impl.py b/cupy/testing/_pytest_impl.py
new file mode 100644
index 0000000..d88adf3
--- /dev/null
+++ b/cupy/testing/_pytest_impl.py
@@ -0,0 +1,70 @@
+import unittest
+
+import cupy.testing._parameterized
+
+try:
+    import pytest
+    import _pytest
+    _error = None
+except ImportError as e:
+    pytest = None  # type: ignore
+    _pytest = None  # type: ignore
+    _error = e
+
+
+def is_available():
+    return _error is None and hasattr(pytest, 'fixture')
+
+
+def check_available(feature):
+    if not is_available():
+        raise RuntimeError('''\
+cupy.testing: {} is not available.
+
+Reason: {}: {}'''.format(feature, type(_error).__name__, _error))
+
+
+if is_available():
+
+    class _TestingParameterizeMixin:
+
+        def __repr__(self):
+            return '<{}  parameter: {}>'.format(
+                super().__repr__(),
+                self.__dict__,
+            )
+
+        @pytest.fixture(autouse=True)
+        def _cupy_testing_parameterize(self, _cupy_testing_param):
+            assert not self.__dict__, \
+                'There should not be another hack with instance attribute.'
+            self.__dict__.update(_cupy_testing_param)
+
+
+def parameterize(*params, _ids=True):
+    check_available('parameterize')
+    if _ids:
+        param_name = cupy.testing._parameterized._make_class_name
+    else:
+        def param_name(_, i, param):
+            return str(i)
+
+    # TODO(kataoka): Give better names (`id`).
+    # For now, use legacy `_make_class_name` just for consistency. Here,
+    # a generated name is `TestFoo::test_bar[_param_0_{...}]`, whereas
+    # a legacy name is `TestFoo_param_0_{...}::test_bar
+    params = [
+        pytest.param(param, id=param_name('', i, param))
+        for i, param in enumerate(params)
+    ]
+
+    def f(cls):
+        assert not issubclass(cls, unittest.TestCase)
+        if issubclass(cls, _TestingParameterizeMixin):
+            raise RuntimeError('do not `@testing.parameterize` twice')
+        module_name = cls.__module__
+        cls = type(cls.__name__, (_TestingParameterizeMixin, cls), {})
+        cls.__module__ = module_name
+        cls = pytest.mark.parametrize('_cupy_testing_param', params)(cls)
+        return cls
+    return f
diff --git a/cupy/testing/_random.py b/cupy/testing/_random.py
new file mode 100644
index 0000000..723f1b8
--- /dev/null
+++ b/cupy/testing/_random.py
@@ -0,0 +1,142 @@
+import atexit
+import functools
+import numpy
+import os
+import random
+import types
+import unittest
+
+import cupy
+
+
+_old_python_random_state = None
+_old_numpy_random_state = None
+_old_cupy_random_states = None
+
+
+def do_setup(deterministic=True):
+    global _old_python_random_state
+    global _old_numpy_random_state
+    global _old_cupy_random_states
+    _old_python_random_state = random.getstate()
+    _old_numpy_random_state = numpy.random.get_state()
+    _old_cupy_random_states = cupy.random._generator._random_states
+    cupy.random.reset_states()
+    # Check that _random_state has been recreated in
+    # cupy.random.reset_states(). Otherwise the contents of
+    # _old_cupy_random_states would be overwritten.
+    assert (cupy.random._generator._random_states is not
+            _old_cupy_random_states)
+
+    if not deterministic:
+        random.seed()
+        numpy.random.seed()
+        cupy.random.seed()
+    else:
+        random.seed(99)
+        numpy.random.seed(100)
+        cupy.random.seed(101)
+
+
+def do_teardown():
+    global _old_python_random_state
+    global _old_numpy_random_state
+    global _old_cupy_random_states
+    random.setstate(_old_python_random_state)
+    numpy.random.set_state(_old_numpy_random_state)
+    cupy.random._generator._random_states = _old_cupy_random_states
+    _old_python_random_state = None
+    _old_numpy_random_state = None
+    _old_cupy_random_states = None
+
+
+# In some tests (which utilize condition.repeat or condition.retry),
+# setUp/tearDown is nested. _setup_random() and _teardown_random() do their
+# work only in the outermost setUp/tearDown pair.
+_nest_count = 0
+
+
+@atexit.register
+def _check_teardown():
+    assert _nest_count == 0, ('_setup_random() and _teardown_random() '
+                              'must be called in pairs.')
+
+
+def _setup_random():
+    """Sets up the deterministic random states of ``numpy`` and ``cupy``.
+
+    """
+    global _nest_count
+    if _nest_count == 0:
+        nondeterministic = bool(int(os.environ.get(
+            'CUPY_TEST_RANDOM_NONDETERMINISTIC', '0')))
+        do_setup(not nondeterministic)
+    _nest_count += 1
+
+
+def _teardown_random():
+    """Tears down the deterministic random states set up by ``_setup_random``.
+
+    """
+    global _nest_count
+    assert _nest_count > 0, '_setup_random has not been called'
+    _nest_count -= 1
+    if _nest_count == 0:
+        do_teardown()
+
+
+def generate_seed():
+    assert _nest_count > 0, 'random is not set up'
+    return numpy.random.randint(0x7fffffff)
+
+
+def fix_random():
+    """Decorator that fixes random numbers in a test.
+
+    This decorator can be applied to either a test case class or a test method.
+    It should not be applied within ``condition.retry`` or
+    ``condition.repeat``.
+    """
+
+    # TODO(niboshi): Prevent this decorator from being applied within
+    #    condition.repeat or condition.retry decorators. That would repeat
+    #    tests with the same random seeds. It's okay to apply this outside
+    #    these decorators.
+
+    def decorator(impl):
+        if (isinstance(impl, types.FunctionType) and
+                impl.__name__.startswith('test_')):
+            # Applied to test method
+            @functools.wraps(impl)
+            def test_func(self, *args, **kw):
+                _setup_random()
+                try:
+                    impl(self, *args, **kw)
+                finally:
+                    _teardown_random()
+            return test_func
+        elif isinstance(impl, type) and issubclass(impl, unittest.TestCase):
+            # Applied to test case class
+            klass = impl
+
+            def wrap_setUp(f):
+                def func(self):
+                    _setup_random()
+                    f(self)
+                return func
+
+            def wrap_tearDown(f):
+                def func(self):
+                    try:
+                        f(self)
+                    finally:
+                        _teardown_random()
+                return func
+
+            klass.setUp = wrap_setUp(klass.setUp)
+            klass.tearDown = wrap_tearDown(klass.tearDown)
+            return klass
+        else:
+            raise ValueError('Can\'t apply fix_random to {}'.format(impl))
+
+    return decorator
diff --git a/cupy/typing/__init__.py b/cupy/typing/__init__.py
new file mode 100644
index 0000000..9b09a52
--- /dev/null
+++ b/cupy/typing/__init__.py
@@ -0,0 +1,4 @@
+from cupy.typing._generic_alias import ArrayLike  # NOQA
+from cupy.typing._generic_alias import DTypeLike  # NOQA
+from cupy.typing._generic_alias import NBitBase  # NOQA
+from cupy.typing._generic_alias import NDArray  # NOQA
diff --git a/cupy/typing/_generic_alias.py b/cupy/typing/_generic_alias.py
new file mode 100644
index 0000000..14324ad
--- /dev/null
+++ b/cupy/typing/_generic_alias.py
@@ -0,0 +1,43 @@
+import sys
+from typing import Any, TypeVar
+
+import numpy
+
+import cupy
+
+
+if numpy.lib.NumpyVersion(numpy.__version__) >= '1.20.0':
+    from numpy.typing import ArrayLike  # NOQA
+    from numpy.typing import DTypeLike  # NOQA
+    from numpy.typing import NBitBase  # NOQA
+else:
+    ArrayLike = Any  # type: ignore
+    DTypeLike = Any  # type: ignore
+    NBitBase = Any  # type: ignore
+
+
+if sys.version_info >= (3, 9):
+    from types import GenericAlias
+else:
+    try:
+        # NumPy 1.23+
+        import numpy._typing as _numpy_typing
+    except Exception:
+        try:
+            # NumPy 1.21 & 1.22
+            import numpy.typing as _numpy_typing  # type: ignore
+        except Exception:
+            _numpy_typing = None  # type: ignore
+
+    _GenericAlias = getattr(_numpy_typing, '_GenericAlias', None)
+
+    if _GenericAlias is not None:
+        GenericAlias = _GenericAlias
+    else:
+        def GenericAlias(*args):
+            return Any
+
+
+_ScalarType = TypeVar('_ScalarType', bound=numpy.generic, covariant=True)
+_DType = GenericAlias(numpy.dtype, (_ScalarType,))
+NDArray = GenericAlias(cupy.ndarray, (Any, _DType))
diff --git a/cupy_backends/__init__.pxd b/cupy_backends/__init__.pxd
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/__init__.py b/cupy_backends/__init__.py
new file mode 100755
index 0000000..e69de29
diff --git a/cupy_backends/cuda/__init__.pxd b/cupy_backends/cuda/__init__.pxd
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/cuda/__init__.py b/cupy_backends/cuda/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/cuda/_softlink.pxd b/cupy_backends/cuda/_softlink.pxd
new file mode 100644
index 0000000..3e8302d
--- /dev/null
+++ b/cupy_backends/cuda/_softlink.pxd
@@ -0,0 +1,7 @@
+ctypedef int (*func_ptr)(...) nogil  # NOQA
+
+cdef class SoftLink:
+    cdef:
+        object _cdll
+        str _prefix
+        func_ptr get(self, str name)
diff --git a/cupy_backends/cuda/_softlink.pyx b/cupy_backends/cuda/_softlink.pyx
new file mode 100644
index 0000000..5bd06e0
--- /dev/null
+++ b/cupy_backends/cuda/_softlink.pyx
@@ -0,0 +1,51 @@
+import ctypes
+import warnings
+
+from libc.stdint cimport intptr_t
+cimport cython
+
+
+cdef class SoftLink:
+    def __init__(self, object libname, str prefix):
+        self._cdll = None
+        if libname is not None:
+            try:
+                self._cdll = ctypes.CDLL(libname)
+            except Exception as e:
+                warnings.warn(
+                    f'Warning: CuPy failed to load "{libname}": '
+                    f'({type(e).__name__}: {e})')
+        self._prefix = prefix
+
+    cdef func_ptr get(self, str name):
+        """
+        Returns a function pointer for the API.
+        """
+        if self._cdll is None:
+            return <func_ptr>_fail_unsupported
+        cdef str funcname = f'{self._prefix}{name}'
+        cdef object func = getattr(self._cdll, funcname, None)
+        if func is None:
+            return <func_ptr>_fail_not_found
+        cdef intptr_t ptr = ctypes.addressof(func)
+        return cython.operator.dereference(<func_ptr*>ptr)
+
+
+cdef int _fail_unsupported() nogil except -1:
+    with gil:
+        raise AssertionError('''
+*** The requested function is not supported in the current version of
+*** the toolkit installed in your environment.
+***
+*** This is likely a bug in CuPy. Please report this issue to:
+***   https://github.com/cupy/cupy/issues
+''')
+
+cdef int _fail_not_found() nogil except -1:
+    with gil:
+        raise AssertionError('''
+*** The requested function could not be found in the library.
+***
+*** This is likely a bug in CuPy. Please report this issue to:
+***   https://github.com/cupy/cupy/issues
+''')
diff --git a/cupy_backends/cuda/api/__init__.pxd b/cupy_backends/cuda/api/__init__.pxd
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/cuda/api/__init__.py b/cupy_backends/cuda/api/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/cuda/api/_driver_enum.pxd b/cupy_backends/cuda/api/_driver_enum.pxd
new file mode 100644
index 0000000..e8df201
--- /dev/null
+++ b/cupy_backends/cuda/api/_driver_enum.pxd
@@ -0,0 +1,78 @@
+cpdef enum:
+    # CUjitInputType
+    CU_JIT_INPUT_CUBIN = 0
+    CU_JIT_INPUT_PTX = 1
+    CU_JIT_INPUT_FATBINARY = 2
+    CU_JIT_INPUT_OBJECT = 3
+    CU_JIT_INPUT_LIBRARY = 4
+    CU_JIT_INPUT_NVVM = 5
+
+    # CUjit_option
+    CU_JIT_MAX_REGISTERS = 0
+    CU_JIT_THREADS_PER_BLOCK = 1
+    CU_JIT_WALL_TIME = 2
+    CU_JIT_INFO_LOG_BUFFER = 3
+    CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES = 4
+    CU_JIT_ERROR_LOG_BUFFER = 5
+    CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES = 6
+    CU_JIT_OPTIMIZATION_LEVEL = 7
+    CU_JIT_TARGET_FROM_CUCONTEXT = 8
+    CU_JIT_TARGET = 9
+    CU_JIT_FALLBACK_STRATEGY = 10
+    CU_JIT_GENERATE_DEBUG_INFO = 11
+    CU_JIT_LOG_VERBOSE = 12
+    CU_JIT_GENERATE_LINE_INFO = 13
+    CU_JIT_CACHE_MODE = 14
+    CU_JIT_NEW_SM3X_OPT = 15
+    CU_JIT_FAST_COMPILE = 16
+    CU_JIT_GLOBAL_SYMBOL_NAMES = 17
+    CU_JIT_GLOBAL_SYMBOL_ADDRESSES = 18
+    CU_JIT_GLOBAL_SYMBOL_COUNT = 19
+    CU_JIT_LTO = 20
+    CU_JIT_FTZ = 21
+    CU_JIT_PREC_DIV = 22
+    CU_JIT_PREC_SQRT = 23
+    CU_JIT_FMA = 24
+    CU_JIT_NUM_OPTIONS = 25
+
+    # CUfunction_attribute
+    CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK = 0
+    CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES = 1
+    CU_FUNC_ATTRIBUTE_CONST_SIZE_BYTES = 2
+    CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES = 3
+    CU_FUNC_ATTRIBUTE_NUM_REGS = 4
+    CU_FUNC_ATTRIBUTE_PTX_VERSION = 5
+    CU_FUNC_ATTRIBUTE_BINARY_VERSION = 6
+    CU_FUNC_ATTRIBUTE_CACHE_MODE_CA = 7
+    CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES = 8
+    CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 9
+
+    # CUresult
+    CUDA_ERROR_INVALID_VALUE = 1
+
+    # CUarray_format
+    CU_AD_FORMAT_UNSIGNED_INT8 = 0x01
+    CU_AD_FORMAT_UNSIGNED_INT16 = 0x02
+    CU_AD_FORMAT_UNSIGNED_INT32 = 0x03
+    CU_AD_FORMAT_SIGNED_INT8 = 0x08
+    CU_AD_FORMAT_SIGNED_INT16 = 0x09
+    CU_AD_FORMAT_SIGNED_INT32 = 0x0a
+    CU_AD_FORMAT_HALF = 0x10
+    CU_AD_FORMAT_FLOAT = 0x20
+
+    # CUaddress_mode
+    CU_TR_ADDRESS_MODE_WRAP = 0
+    CU_TR_ADDRESS_MODE_CLAMP = 1
+    CU_TR_ADDRESS_MODE_MIRROR = 2
+    CU_TR_ADDRESS_MODE_BORDER = 3
+
+    # CUfilter_mode
+    CU_TR_FILTER_MODE_POINT = 0
+    CU_TR_FILTER_MODE_LINEAR = 1
+
+    # Constants
+    CU_TRSA_OVERRIDE_FORMAT = 0x01
+
+    CU_TRSF_READ_AS_INTEGER = 0x01
+    CU_TRSF_NORMALIZED_COORDINATES = 0x02
+    CU_TRSF_SRGB = 0x10
diff --git a/cupy_backends/cuda/api/_driver_enum.pyx b/cupy_backends/cuda/api/_driver_enum.pyx
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/cuda/api/_driver_extern.pxi b/cupy_backends/cuda/api/_driver_extern.pxi
new file mode 100644
index 0000000..913972b
--- /dev/null
+++ b/cupy_backends/cuda/api/_driver_extern.pxi
@@ -0,0 +1,66 @@
+cdef extern from '../../cupy_backend.h' nogil:
+    # Error handling
+    int cuGetErrorName(Result error, const char** pStr)
+    int cuGetErrorString(Result error, const char** pStr)
+
+    # Primary context management
+    int cuDevicePrimaryCtxRelease(Device dev)
+
+    # Context management
+    int cuCtxGetCurrent(Context* pctx)
+    int cuCtxSetCurrent(Context ctx)
+    int cuCtxCreate(Context* pctx, unsigned int flags, Device dev)
+    int cuCtxDestroy(Context ctx)
+    int cuCtxGetDevice(Device*)
+
+    # Module load and kernel execution
+    int cuLinkCreate(unsigned int numOptions, CUjit_option* options,
+                     void** optionValues, LinkState* stateOut)
+    int cuLinkAddData(LinkState state, CUjitInputType type, void* data,
+                      size_t size, const char* name, unsigned int  numOptions,
+                      CUjit_option* options, void** optionValues)
+    int cuLinkAddFile(LinkState state, CUjitInputType type, const char* path,
+                      unsigned int numOptions, CUjit_option* options, void**
+                      optionValues)
+    int cuLinkComplete(LinkState state, void** cubinOut, size_t* sizeOut)
+    int cuLinkDestroy(LinkState state)
+    int cuModuleLoad(Module* module, char* fname)
+    int cuModuleLoadData(Module* module, void* image)
+    int cuModuleUnload(Module hmod)
+    int cuModuleGetFunction(Function* hfunc, Module hmod,
+                            char* name)
+    int cuModuleGetGlobal(Deviceptr* dptr, size_t* bytes, Module hmod,
+                          char* name)
+    int cuLaunchKernel(
+        Function f, unsigned int gridDimX, unsigned int gridDimY,
+        unsigned int gridDimZ, unsigned int blockDimX,
+        unsigned int blockDimY, unsigned int blockDimZ,
+        unsigned int sharedMemBytes, Stream hStream,
+        void** kernelParams, void** extra)
+
+    int cuLaunchCooperativeKernel(
+        Function f, unsigned int gridDimX, unsigned int gridDimY,
+        unsigned int gridDimZ, unsigned int blockDimX,
+        unsigned int blockDimY, unsigned int blockDimZ,
+        unsigned int sharedMemBytes, Stream hStream,
+        void** kernelParams)
+
+    # Kernel attributes
+    int cuFuncGetAttribute(int *pi, CUfunction_attribute attrib,
+                           Function hfunc)
+
+    int cuFuncSetAttribute(Function hfunc, CUfunction_attribute attrib,
+                           int value)
+
+    # Occupancy
+    int cuOccupancyMaxActiveBlocksPerMultiprocessor(
+        int* numBlocks, Function func, int blockSize, size_t dynamicSMemSize)
+    int cuOccupancyMaxPotentialBlockSize(
+        int* minGridSize, int* blockSize, Function func, CUoccupancyB2DSize
+        block2shmem, size_t dynamicSMemSize, int blockSizeLimit)
+
+    # Stream
+    int cuStreamGetCtx(Stream hStream, Context* pctx)
+
+    # Build-time version
+    enum: CUDA_VERSION
diff --git a/cupy_backends/cuda/api/_driver_typedef.pxi b/cupy_backends/cuda/api/_driver_typedef.pxi
new file mode 100644
index 0000000..0cc5fbf
--- /dev/null
+++ b/cupy_backends/cuda/api/_driver_typedef.pxi
@@ -0,0 +1,30 @@
+# Keep in sync with typenames exported in `driver.pxd`.
+
+cdef extern from *:
+    ctypedef int Device 'CUdevice'
+    ctypedef int Result 'CUresult'
+
+    ctypedef void* Context 'CUcontext'
+    ctypedef void* Deviceptr 'CUdeviceptr'
+    ctypedef void* Event 'cudaEvent_t'
+    ctypedef void* Function 'CUfunction'
+    ctypedef void* Module 'CUmodule'
+    ctypedef void* Stream 'cudaStream_t'
+    ctypedef void* LinkState 'CUlinkState'
+
+    ctypedef int CUjit_option 'CUjit_option'
+    ctypedef int CUjitInputType 'CUjitInputType'
+    ctypedef int CUfunction_attribute 'CUfunction_attribute'
+
+    ctypedef size_t(*CUoccupancyB2DSize)(int)
+
+    # For Texture Reference
+    ctypedef void* Array 'CUarray_st*'  # = cupy.cuda.runtime.Array
+    ctypedef int Array_format 'CUarray_format'
+    ctypedef struct Array_desc 'CUDA_ARRAY_DESCRIPTOR':
+        Array_format Format
+        size_t Height
+        unsigned int NumChannels
+        size_t Width
+    ctypedef int Address_mode 'CUaddress_mode'
+    ctypedef int Filter_mode 'CUfilter_mode'
diff --git a/cupy_backends/cuda/api/_runtime_enum.pxd b/cupy_backends/cuda/api/_runtime_enum.pxd
new file mode 100644
index 0000000..5b3ce92
--- /dev/null
+++ b/cupy_backends/cuda/api/_runtime_enum.pxd
@@ -0,0 +1,342 @@
+cpdef enum:
+    cudaMemoryTypeHost = 1
+    cudaMemoryTypeDevice = 2
+
+    cudaIpcMemLazyEnablePeerAccess = 1
+
+    cudaMemAttachGlobal = 1
+    cudaMemAttachHost = 2
+    cudaMemAttachSingle = 4
+
+    cudaMemAdviseSetReadMostly = 1
+    cudaMemAdviseUnsetReadMostly = 2
+    cudaMemAdviseSetPreferredLocation = 3
+    cudaMemAdviseUnsetPreferredLocation = 4
+    cudaMemAdviseSetAccessedBy = 5
+    cudaMemAdviseUnsetAccessedBy = 6
+
+    CUDA_R_32F = 0  # 32 bit real
+    CUDA_R_64F = 1  # 64 bit real
+    CUDA_R_16F = 2  # 16 bit real
+    CUDA_R_8I = 3  # 8 bit real as a signed integer
+    CUDA_C_32F = 4  # 32 bit complex
+    CUDA_C_64F = 5  # 64 bit complex
+    CUDA_C_16F = 6  # 16 bit complex
+    CUDA_C_8I = 7  # 8 bit complex as a pair of signed integers
+    CUDA_R_8U = 8  # 8 bit real as a signed integer
+    CUDA_C_8U = 9  # 8 bit complex as a pair of signed integers
+
+    # CUDA Limits
+    cudaLimitStackSize = 0x00
+    cudaLimitPrintfFifoSize = 0x01
+    cudaLimitMallocHeapSize = 0x02
+    cudaLimitDevRuntimeSyncDepth = 0x03
+    cudaLimitDevRuntimePendingLaunchCount = 0x04
+    cudaLimitMaxL2FetchGranularity = 0x05
+
+    # cudaChannelFormatKind
+    cudaChannelFormatKindSigned = 0
+    cudaChannelFormatKindUnsigned = 1
+    cudaChannelFormatKindFloat = 2
+    cudaChannelFormatKindNone = 3
+
+    # CUDA array flags
+    cudaArrayDefault = 0
+    # cudaArrayLayered = 1
+    cudaArraySurfaceLoadStore = 2
+    # cudaArrayCubemap = 4
+    # cudaArrayTextureGather = 8
+
+    # cudaResourceType
+    cudaResourceTypeArray = 0
+    cudaResourceTypeMipmappedArray = 1
+    cudaResourceTypeLinear = 2
+    cudaResourceTypePitch2D = 3
+
+    # cudaTextureAddressMode
+    cudaAddressModeWrap = 0
+    cudaAddressModeClamp = 1
+    cudaAddressModeMirror = 2
+    cudaAddressModeBorder = 3
+
+    # cudaTextureFilterMode
+    cudaFilterModePoint = 0
+    cudaFilterModeLinear = 1
+
+    # cudaTextureReadMode
+    cudaReadModeElementType = 0
+    cudaReadModeNormalizedFloat = 1
+
+    # cudaMemPoolAttr
+    # ----- added since 11.2 -----
+    cudaMemPoolReuseFollowEventDependencies = 0x1
+    cudaMemPoolReuseAllowOpportunistic = 0x2
+    cudaMemPoolReuseAllowInternalDependencies = 0x3
+    cudaMemPoolAttrReleaseThreshold = 0x4
+    # ----- added since 11.3 -----
+    cudaMemPoolAttrReservedMemCurrent = 0x5
+    cudaMemPoolAttrReservedMemHigh = 0x6
+    cudaMemPoolAttrUsedMemCurrent = 0x7
+    cudaMemPoolAttrUsedMemHigh = 0x8
+
+    # cudaMemAllocationType
+    cudaMemAllocationTypePinned = 0x1
+
+    # cudaMemAllocationHandleType
+    cudaMemHandleTypeNone = 0x0
+    cudaMemHandleTypePosixFileDescriptor = 0x1
+    # cudaMemHandleTypeWin32 = 0x2
+    # cudaMemHandleTypeWin32Kmt = 0x4
+
+    # cudaMemLocationType
+    cudaMemLocationTypeDevice = 1
+
+
+# This was a legacy mistake: the prefix "cuda" should have been removed
+# so that we can directly assign their C counterparts here. Now because
+# of backward compatibility and no flexible Cython macro (IF/ELSE), we
+# have to duplicate the enum. (CUDA and HIP use different values!)
+IF CUPY_HIP_VERSION > 0:
+    # separate in groups of 10 for easier counting...
+    cpdef enum:
+        cudaDevAttrMaxThreadsPerBlock = 0
+        cudaDevAttrMaxBlockDimX
+        cudaDevAttrMaxBlockDimY
+        cudaDevAttrMaxBlockDimZ
+        cudaDevAttrMaxGridDimX
+        cudaDevAttrMaxGridDimY
+        cudaDevAttrMaxGridDimZ
+        cudaDevAttrMaxSharedMemoryPerBlock
+        cudaDevAttrTotalConstantMemory
+        cudaDevAttrWarpSize
+
+        cudaDevAttrMaxRegistersPerBlock
+        cudaDevAttrClockRate
+        cudaDevAttrMemoryClockRate
+        cudaDevAttrGlobalMemoryBusWidth
+        cudaDevAttrMultiProcessorCount
+        cudaDevAttrComputeMode
+        cudaDevAttrL2CacheSize
+        cudaDevAttrMaxThreadsPerMultiProcessor
+        # The following are exposed as "deviceAttributeCo..."
+        # cudaDevAttrComputeCapabilityMajor
+        # cudaDevAttrComputeCapabilityMinor
+
+        cudaDevAttrConcurrentKernels = 20
+        cudaDevAttrPciBusId
+        cudaDevAttrPciDeviceId
+        cudaDevAttrMaxSharedMemoryPerMultiprocessor
+        cudaDevAttrIsMultiGpuBoard
+        cudaDevAttrIntegrated
+        cudaDevAttrCooperativeLaunch
+        cudaDevAttrCooperativeMultiDeviceLaunch
+        cudaDevAttrMaxTexture1DWidth
+        cudaDevAttrMaxTexture2DWidth
+
+        cudaDevAttrMaxTexture2DHeight
+        cudaDevAttrMaxTexture3DWidth
+        cudaDevAttrMaxTexture3DHeight
+        cudaDevAttrMaxTexture3DDepth
+        # The following attributes do not exist in CUDA and cause segfualts
+        # if we try to access them
+        # hipDeviceAttributeHdpMemFlushCntl
+        # hipDeviceAttributeHdpRegFlushCntl
+        cudaDevAttrMaxPitch = 36
+        cudaDevAttrTextureAlignment
+        cudaDevAttrTexturePitchAlignment
+        cudaDevAttrKernelExecTimeout
+
+        cudaDevAttrCanMapHostMemory
+        cudaDevAttrEccEnabled
+        cudaDevAttrMemoryPoolsSupported = 0
+        # The following attributes do not exist in CUDA
+        # hipDeviceAttributeCooperativeMultiDeviceUnmatchedFunc
+        # hipDeviceAttributeCooperativeMultiDeviceUnmatchedGridDim
+        # hipDeviceAttributeCooperativeMultiDeviceUnmatchedBlockDim
+        # hipDeviceAttributeCooperativeMultiDeviceUnmatchedSharedMem
+
+        # The rest do not have HIP correspondence...
+        # TODO(leofang): should we expose them anyway, with a value -1 to
+        # indicate they cannot be used in HIP?
+        # cudaDevAttrGpuOverlap
+        # cudaDevAttrMaxTexture2DLayeredWidth
+        # cudaDevAttrMaxTexture2DLayeredHeight
+        # cudaDevAttrMaxTexture2DLayeredLayers
+        # cudaDevAttrSurfaceAlignment
+        # cudaDevAttrTccDriver
+        # cudaDevAttrAsyncEngineCount
+        # cudaDevAttrUnifiedAddressing
+        # cudaDevAttrMaxTexture1DLayeredWidth
+        # cudaDevAttrMaxTexture1DLayeredLayers
+        # cudaDevAttrMaxTexture2DGatherWidth
+        # cudaDevAttrMaxTexture2DGatherHeight
+        # cudaDevAttrMaxTexture3DWidthAlt
+        # cudaDevAttrMaxTexture3DHeightAlt
+        # cudaDevAttrMaxTexture3DDepthAlt
+        # cudaDevAttrPciDomainId
+        # cudaDevAttrMaxTextureCubemapWidth
+        # cudaDevAttrMaxTextureCubemapLayeredWidth
+        # cudaDevAttrMaxTextureCubemapLayeredLayers
+        # cudaDevAttrMaxSurface1DWidth
+        # cudaDevAttrMaxSurface2DWidth
+        # cudaDevAttrMaxSurface2DHeight
+        # cudaDevAttrMaxSurface3DWidth
+        # cudaDevAttrMaxSurface3DHeight
+        # cudaDevAttrMaxSurface3DDepth
+        # cudaDevAttrMaxSurface1DLayeredWidth
+        # cudaDevAttrMaxSurface1DLayeredLayers
+        # cudaDevAttrMaxSurface2DLayeredWidth
+        # cudaDevAttrMaxSurface2DLayeredHeight
+        # cudaDevAttrMaxSurface2DLayeredLayers
+        # cudaDevAttrMaxSurfaceCubemapWidth
+        # cudaDevAttrMaxSurfaceCubemapLayeredWidth
+        # cudaDevAttrMaxSurfaceCubemapLayeredLayers
+        # cudaDevAttrMaxTexture1DLinearWidth
+        # cudaDevAttrMaxTexture2DLinearWidth
+        # cudaDevAttrMaxTexture2DLinearHeight
+        # cudaDevAttrMaxTexture2DLinearPitch
+        # cudaDevAttrMaxTexture2DMipmappedWidth
+        # cudaDevAttrMaxTexture2DMipmappedHeight
+        # cudaDevAttrMaxTexture1DMipmappedWidth
+        # cudaDevAttrStreamPrioritiesSupported
+        # cudaDevAttrGlobalL1CacheSupported
+        # cudaDevAttrLocalL1CacheSupported
+        # cudaDevAttrMaxRegistersPerMultiprocessor
+        # cudaDevAttrMultiGpuBoardGroupID
+        # cudaDevAttrHostNativeAtomicSupported
+        # cudaDevAttrSingleToDoublePrecisionPerfRatio
+        # cudaDevAttrComputePreemptionSupported
+        # cudaDevAttrCanUseHostPointerForRegisteredMem
+        # cudaDevAttrReserved92
+        # cudaDevAttrReserved93
+        # cudaDevAttrReserved94
+        # cudaDevAttrMaxSharedMemoryPerBlockOptin
+        # cudaDevAttrCanFlushRemoteWrites
+        # cudaDevAttrHostRegisterSupported
+    IF CUPY_HIP_VERSION >= 310:
+        cpdef enum:
+            # hipDeviceAttributeAsicRevision  # does not exist in CUDA
+            cudaDevAttrManagedMemory = 47
+            cudaDevAttrDirectManagedMemAccessFromHost
+            cudaDevAttrConcurrentManagedAccess
+
+            cudaDevAttrPageableMemoryAccess
+            cudaDevAttrPageableMemoryAccessUsesHostPageTables
+ELSE:
+    # For CUDA/RTD
+    cpdef enum:
+        cudaDevAttrMaxThreadsPerBlock = 1
+        cudaDevAttrMaxBlockDimX
+        cudaDevAttrMaxBlockDimY
+        cudaDevAttrMaxBlockDimZ
+        cudaDevAttrMaxGridDimX
+        cudaDevAttrMaxGridDimY
+        cudaDevAttrMaxGridDimZ
+        cudaDevAttrMaxSharedMemoryPerBlock
+        cudaDevAttrTotalConstantMemory
+        cudaDevAttrWarpSize
+        cudaDevAttrMaxPitch
+        cudaDevAttrMaxRegistersPerBlock
+        cudaDevAttrClockRate
+        cudaDevAttrTextureAlignment
+        cudaDevAttrGpuOverlap
+        cudaDevAttrMultiProcessorCount
+        cudaDevAttrKernelExecTimeout
+        cudaDevAttrIntegrated
+        cudaDevAttrCanMapHostMemory
+        cudaDevAttrComputeMode
+        cudaDevAttrMaxTexture1DWidth
+        cudaDevAttrMaxTexture2DWidth
+        cudaDevAttrMaxTexture2DHeight
+        cudaDevAttrMaxTexture3DWidth
+        cudaDevAttrMaxTexture3DHeight
+        cudaDevAttrMaxTexture3DDepth
+        cudaDevAttrMaxTexture2DLayeredWidth
+        cudaDevAttrMaxTexture2DLayeredHeight
+        cudaDevAttrMaxTexture2DLayeredLayers
+        cudaDevAttrSurfaceAlignment
+        cudaDevAttrConcurrentKernels
+        cudaDevAttrEccEnabled
+        cudaDevAttrPciBusId
+        cudaDevAttrPciDeviceId
+        cudaDevAttrTccDriver
+        cudaDevAttrMemoryClockRate
+        cudaDevAttrGlobalMemoryBusWidth
+        cudaDevAttrL2CacheSize
+        cudaDevAttrMaxThreadsPerMultiProcessor
+        cudaDevAttrAsyncEngineCount
+        cudaDevAttrUnifiedAddressing
+        cudaDevAttrMaxTexture1DLayeredWidth
+        cudaDevAttrMaxTexture1DLayeredLayers  # = 43; 44 is missing
+        cudaDevAttrMaxTexture2DGatherWidth = 45
+        cudaDevAttrMaxTexture2DGatherHeight
+        cudaDevAttrMaxTexture3DWidthAlt
+        cudaDevAttrMaxTexture3DHeightAlt
+        cudaDevAttrMaxTexture3DDepthAlt
+        cudaDevAttrPciDomainId
+        cudaDevAttrTexturePitchAlignment
+        cudaDevAttrMaxTextureCubemapWidth
+        cudaDevAttrMaxTextureCubemapLayeredWidth
+        cudaDevAttrMaxTextureCubemapLayeredLayers
+        cudaDevAttrMaxSurface1DWidth
+        cudaDevAttrMaxSurface2DWidth
+        cudaDevAttrMaxSurface2DHeight
+        cudaDevAttrMaxSurface3DWidth
+        cudaDevAttrMaxSurface3DHeight
+        cudaDevAttrMaxSurface3DDepth
+        cudaDevAttrMaxSurface1DLayeredWidth
+        cudaDevAttrMaxSurface1DLayeredLayers
+        cudaDevAttrMaxSurface2DLayeredWidth
+        cudaDevAttrMaxSurface2DLayeredHeight
+        cudaDevAttrMaxSurface2DLayeredLayers
+        cudaDevAttrMaxSurfaceCubemapWidth
+        cudaDevAttrMaxSurfaceCubemapLayeredWidth
+        cudaDevAttrMaxSurfaceCubemapLayeredLayers
+        cudaDevAttrMaxTexture1DLinearWidth
+        cudaDevAttrMaxTexture2DLinearWidth
+        cudaDevAttrMaxTexture2DLinearHeight
+        cudaDevAttrMaxTexture2DLinearPitch
+        cudaDevAttrMaxTexture2DMipmappedWidth
+        cudaDevAttrMaxTexture2DMipmappedHeight
+        # The following are exposed as "deviceAttributeCo..."
+        # cudaDevAttrComputeCapabilityMajor  # = 75
+        # cudaDevAttrComputeCapabilityMinor  # = 76
+        cudaDevAttrMaxTexture1DMipmappedWidth = 77
+        cudaDevAttrStreamPrioritiesSupported
+        cudaDevAttrGlobalL1CacheSupported
+        cudaDevAttrLocalL1CacheSupported
+        cudaDevAttrMaxSharedMemoryPerMultiprocessor
+        cudaDevAttrMaxRegistersPerMultiprocessor
+        cudaDevAttrManagedMemory
+        cudaDevAttrIsMultiGpuBoard
+        cudaDevAttrMultiGpuBoardGroupID
+        cudaDevAttrHostNativeAtomicSupported
+        cudaDevAttrSingleToDoublePrecisionPerfRatio
+        cudaDevAttrPageableMemoryAccess
+        cudaDevAttrConcurrentManagedAccess
+        cudaDevAttrComputePreemptionSupported
+        cudaDevAttrCanUseHostPointerForRegisteredMem
+        cudaDevAttrReserved92
+        cudaDevAttrReserved93
+        cudaDevAttrReserved94
+        cudaDevAttrCooperativeLaunch
+        cudaDevAttrCooperativeMultiDeviceLaunch
+        cudaDevAttrMaxSharedMemoryPerBlockOptin
+        cudaDevAttrCanFlushRemoteWrites
+        cudaDevAttrHostRegisterSupported
+        cudaDevAttrPageableMemoryAccessUsesHostPageTables
+        cudaDevAttrDirectManagedMemAccessFromHost  # = 101
+        # added since CUDA 11.0
+        cudaDevAttrMaxBlocksPerMultiprocessor = 106
+        cudaDevAttrReservedSharedMemoryPerBlock = 111
+        # added since CUDA 11.1
+        cudaDevAttrSparseCudaArraySupported = 112
+        cudaDevAttrHostRegisterReadOnlySupported = 113
+        # added since CUDA 11.2
+        cudaDevAttrMaxTimelineSemaphoreInteropSupported = 114
+        cudaDevAttrMemoryPoolsSupported = 115
+        # added since CUDA 11.3
+        cudaDevAttrGPUDirectRDMASupported
+        cudaDevAttrGPUDirectRDMAFlushWritesOptions
+        cudaDevAttrGPUDirectRDMAWritesOrdering
+        cudaDevAttrMemoryPoolSupportedHandleTypes
diff --git a/cupy_backends/cuda/api/_runtime_enum.pyx b/cupy_backends/cuda/api/_runtime_enum.pyx
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/cuda/api/_runtime_extern.pxi b/cupy_backends/cuda/api/_runtime_extern.pxi
new file mode 100644
index 0000000..4ca1984
--- /dev/null
+++ b/cupy_backends/cuda/api/_runtime_extern.pxi
@@ -0,0 +1,163 @@
+from cupy_backends.cuda.api cimport driver
+
+
+cdef extern from '../../cupy_backend_runtime.h' nogil:
+    # Error handling
+    const char* cudaGetErrorName(Error error)
+    const char* cudaGetErrorString(Error error)
+    int cudaGetLastError()
+
+    # Initialization
+    int cudaDriverGetVersion(int* driverVersion)
+    int cudaRuntimeGetVersion(int* runtimeVersion)
+
+    # Device operations
+    int cudaGetDevice(int* device)
+    int cudaDeviceGetAttribute(int* value, DeviceAttr attr, int device)
+    int cudaDeviceGetByPCIBusId(int* device, const char* pciBusId)
+    int cudaDeviceGetPCIBusId(char* pciBusId, int len, int device)
+    int cudaGetDeviceProperties(DeviceProp* prop, int device)
+    int cudaGetDeviceCount(int* count)
+    int cudaSetDevice(int device)
+    int cudaDeviceSynchronize()
+
+    int cudaDeviceCanAccessPeer(int* canAccessPeer, int device,
+                                int peerDevice)
+    int cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags)
+    int cudaDeviceDisablePeerAccess(int peerDevice)
+
+    int cudaDeviceGetLimit(size_t* value, Limit limit)
+    int cudaDeviceSetLimit(Limit limit, size_t value)
+
+    # IPC
+    int cudaIpcCloseMemHandle(void* devPtr)
+    int cudaIpcGetEventHandle(IpcEventHandle* handle, driver.Event event)
+    int cudaIpcGetMemHandle(IpcMemHandle*, void* devPtr)
+    int cudaIpcOpenEventHandle(driver.Event* event, IpcEventHandle handle)
+    int cudaIpcOpenMemHandle(void** devPtr, IpcMemHandle handle,
+                             unsigned int  flags)
+
+    # Memory management
+    int cudaMalloc(void** devPtr, size_t size)
+    int cudaMallocManaged(void** devPtr, size_t size, unsigned int flags)
+    int cudaMalloc3DArray(Array* array, const ChannelFormatDesc* desc,
+                          Extent extent, unsigned int flags)
+    int cudaMallocArray(Array* array, const ChannelFormatDesc* desc,
+                        size_t width, size_t height, unsigned int flags)
+    int cudaMallocAsync(void**, size_t, driver.Stream)
+    int cudaMallocFromPoolAsync(void**, size_t, MemPool, driver.Stream)
+    int cudaHostAlloc(void** ptr, size_t size, unsigned int flags)
+    int cudaHostRegister(void *ptr, size_t size, unsigned int flags)
+    int cudaHostUnregister(void *ptr)
+    int cudaFree(void* devPtr)
+    int cudaFreeHost(void* ptr)
+    int cudaFreeArray(Array array)
+    int cudaFreeAsync(void*, driver.Stream)
+    int cudaMemGetInfo(size_t* free, size_t* total)
+    int cudaMemcpy(void* dst, const void* src, size_t count,
+                   MemoryKind kind)
+    int cudaMemcpyAsync(void* dst, const void* src, size_t count,
+                        MemoryKind kind, driver.Stream stream)
+    int cudaMemcpyPeer(void* dst, int dstDevice, const void* src,
+                       int srcDevice, size_t count)
+    int cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src,
+                            int srcDevice, size_t count,
+                            driver.Stream stream)
+    int cudaMemcpy2DFromArray(void* dst, size_t dpitch, Array src,
+                              size_t wOffset, size_t hOffset, size_t width,
+                              size_t height, MemoryKind kind)
+    int cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, Array src,
+                                   size_t wOffset, size_t hOffset,
+                                   size_t width, size_t height,
+                                   MemoryKind kind, driver.Stream stream)
+    int cudaMemcpy2DToArray(Array dst, size_t wOffset, size_t hOffset,
+                            const void* src, size_t spitch, size_t width,
+                            size_t height, MemoryKind kind)
+    int cudaMemcpy2DToArrayAsync(Array dst, size_t wOffset, size_t hOffset,
+                                 const void* src, size_t spitch, size_t width,
+                                 size_t height, MemoryKind kind,
+                                 driver.Stream stream)
+    int cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch,
+                     size_t width, size_t height, MemoryKind kind)
+    int cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src,
+                          size_t spitch, size_t width, size_t height,
+                          MemoryKind kind, driver.Stream stream)
+    int cudaMemcpy3D(Memcpy3DParms* Memcpy3DParmsPtr)
+    int cudaMemcpy3DAsync(Memcpy3DParms* Memcpy3DParmsPtr,
+                          driver.Stream stream)
+    int cudaMemset(void* devPtr, int value, size_t count)
+    int cudaMemsetAsync(void* devPtr, int value, size_t count,
+                        driver.Stream stream)
+    int cudaMemPrefetchAsync(const void *devPtr, size_t count, int dstDevice,
+                             driver.Stream stream)
+    int cudaMemAdvise(const void *devPtr, size_t count,
+                      MemoryAdvise advice, int device)
+    int cudaDeviceGetDefaultMemPool(MemPool*, int)
+    int cudaDeviceGetMemPool(MemPool*, int)
+    int cudaDeviceSetMemPool(int, MemPool)
+    int cudaMemPoolCreate(MemPool*, _MemPoolProps*)
+    int cudaMemPoolDestroy(MemPool)
+    int cudaMemPoolTrimTo(MemPool, size_t)
+    int cudaMemPoolGetAttribute(MemPool, MemPoolAttr, void*)
+    int cudaMemPoolSetAttribute(MemPool, MemPoolAttr, void*)
+    int cudaPointerGetAttributes(_PointerAttributes* attributes,
+                                 const void* ptr)
+    Extent make_cudaExtent(size_t w, size_t h, size_t d)
+    Pos make_cudaPos(size_t x, size_t y, size_t z)
+    PitchedPtr make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz)
+
+    # Stream and Event
+    int cudaStreamCreate(driver.Stream* pStream)
+    int cudaStreamCreateWithFlags(driver.Stream* pStream,
+                                  unsigned int flags)
+    int cudaStreamDestroy(driver.Stream stream)
+    int cudaStreamSynchronize(driver.Stream stream)
+    int cudaStreamAddCallback(driver.Stream stream, StreamCallback callback,
+                              void* userData, unsigned int flags)
+    int cudaLaunchHostFunc(driver.Stream stream, HostFn fn, void* userData)
+    int cudaStreamQuery(driver.Stream stream)
+    int cudaStreamWaitEvent(driver.Stream stream, driver.Event event,
+                            unsigned int flags)
+    int cudaStreamBeginCapture(driver.Stream stream, StreamCaptureMode mode)
+    int cudaStreamEndCapture(driver.Stream stream, Graph*)
+    int cudaStreamIsCapturing(driver.Stream stream, StreamCaptureStatus*)
+    int cudaEventCreate(driver.Event* event)
+    int cudaEventCreateWithFlags(driver.Event* event, unsigned int flags)
+    int cudaEventDestroy(driver.Event event)
+    int cudaEventElapsedTime(float* ms, driver.Event start,
+                             driver.Event end)
+    int cudaEventQuery(driver.Event event)
+    int cudaEventRecord(driver.Event event, driver.Stream stream)
+    int cudaEventSynchronize(driver.Event event)
+
+    # Texture
+    int cudaCreateTextureObject(TextureObject* pTexObject,
+                                const ResourceDesc* pResDesc,
+                                const TextureDesc* pTexDesc,
+                                const ResourceViewDesc* pResViewDesc)
+    int cudaDestroyTextureObject(TextureObject texObject)
+    int cudaGetChannelDesc(ChannelFormatDesc* desc, Array array)
+    int cudaGetTextureObjectResourceDesc(ResourceDesc* desc, TextureObject obj)
+    int cudaGetTextureObjectTextureDesc(TextureDesc* desc, TextureObject obj)
+
+    # Surface
+    int cudaCreateSurfaceObject(SurfaceObject* pSurObject,
+                                const ResourceDesc* pResDesc)
+    int cudaDestroySurfaceObject(SurfaceObject surObject)
+
+    # Graph
+    int cudaGraphDestroy(Graph graph)
+    int cudaGraphExecDestroy(GraphExec graph)
+    int cudaGraphInstantiate(GraphExec*, Graph, GraphNode*, char*, size_t)
+    int cudaGraphLaunch(GraphExec, driver.Stream)
+    int cudaGraphUpload(GraphExec, driver.Stream)
+
+    int cudaDevAttrComputeCapabilityMajor
+    int cudaDevAttrComputeCapabilityMinor
+
+    # Error code
+    int cudaErrorMemoryAllocation
+    int cudaErrorInvalidValue
+    int cudaErrorPeerAccessAlreadyEnabled
+    int cudaErrorContextIsDestroyed
+    int cudaErrorInvalidResourceHandle
diff --git a/cupy_backends/cuda/api/_runtime_typedef.pxi b/cupy_backends/cuda/api/_runtime_typedef.pxi
new file mode 100644
index 0000000..9560b82
--- /dev/null
+++ b/cupy_backends/cuda/api/_runtime_typedef.pxi
@@ -0,0 +1,449 @@
+# Keep in sync with typenames exported in `runtime.pxd`.
+
+from cupy_backends.cuda.api cimport driver
+
+
+cdef extern from *:
+    ctypedef int Error 'cudaError_t'
+    ctypedef int DataType 'cudaDataType'
+
+    ctypedef int DeviceAttr 'cudaDeviceAttr'
+    ctypedef int MemoryAdvise 'cudaMemoryAdvise'
+
+    ctypedef void _StreamCallbackDef(
+        driver.Stream stream, Error status, void* userData)
+    ctypedef _StreamCallbackDef* StreamCallback 'cudaStreamCallback_t'
+    ctypedef void* StreamCaptureStatus 'cudaStreamCaptureStatus'
+    ctypedef void* GraphNode 'cudaGraphNode_t'
+
+    ctypedef void _HostFnDef(void* userData)
+    ctypedef _HostFnDef* HostFn 'cudaHostFn_t'
+
+    ctypedef int ChannelFormatKind 'cudaChannelFormatKind'
+    ctypedef struct ChannelFormatDesc 'cudaChannelFormatDesc':
+        int x, y, z, w
+        ChannelFormatKind f
+    ctypedef uintmax_t TextureObject 'cudaTextureObject_t'
+    ctypedef uintmax_t SurfaceObject 'cudaSurfaceObject_t'
+    ctypedef int ResourceType 'cudaResourceType'
+    ctypedef int TextureAddressMode 'cudaTextureAddressMode'
+    ctypedef int TextureFilterMode 'cudaTextureFilterMode'
+    ctypedef int TextureReadMode 'cudaTextureReadMode'
+    ctypedef struct ResourceViewDesc 'cudaResourceViewDesc'
+    ctypedef void* Array 'cudaArray_t'
+    ctypedef struct Extent 'cudaExtent':
+        size_t width, height, depth
+    ctypedef struct Pos 'cudaPos':
+        size_t x, y, z
+    ctypedef struct PitchedPtr 'cudaPitchedPtr':
+        size_t pitch
+        void* ptr
+        size_t xsize, ysize
+    ctypedef int MemoryKind 'cudaMemcpyKind'
+    ctypedef void* MipmappedArray 'cudaMipmappedArray_t'
+
+    ctypedef int Limit 'cudaLimit'
+
+    ctypedef int StreamCaptureMode 'cudaStreamCaptureMode'
+    ctypedef void* Graph 'cudaGraph_t'
+    ctypedef void* GraphExec 'cudaGraphExec_t'
+
+    # This is for the annoying nested struct cudaResourceDesc, which is not
+    # perfectly supprted in Cython
+    ctypedef struct _array:
+        Array array
+
+    ctypedef struct _mipmap:
+        MipmappedArray mipmap
+
+    ctypedef struct _linear:
+        void* devPtr
+        ChannelFormatDesc desc
+        size_t sizeInBytes
+
+    ctypedef struct _pitch2D:
+        void* devPtr
+        ChannelFormatDesc desc
+        size_t width
+        size_t height
+        size_t pitchInBytes
+
+    ctypedef union _res:
+        _array array
+        _mipmap mipmap
+        _linear linear
+        _pitch2D pitch2D
+
+    ctypedef struct ResourceDesc 'cudaResourceDesc':
+        int resType
+        _res res
+    # typedef cudaResourceDesc done
+
+    ctypedef struct Memcpy3DParms 'cudaMemcpy3DParms':
+        Array srcArray
+        Pos srcPos
+        PitchedPtr srcPtr
+
+        Array dstArray
+        Pos dstPos
+        PitchedPtr dstPtr
+
+        Extent extent
+        MemoryKind kind
+
+    ctypedef struct TextureDesc 'cudaTextureDesc':
+        int addressMode[3]
+        int filterMode
+        int readMode
+        int sRGB
+        float borderColor[4]
+        int normalizedCoords
+        unsigned int maxAnisotropy
+        # TODO(leofang): support mipmap?
+
+    ctypedef struct IpcMemHandle 'cudaIpcMemHandle_t':
+        unsigned char[64] reserved
+
+    ctypedef struct IpcEventHandle 'cudaIpcEventHandle_t':
+        unsigned char[64] reserved
+
+    ctypedef struct cudaUUID 'cudaUUID_t':
+        char bytes[16]
+
+    ctypedef void* MemPool 'cudaMemPool_t'
+    ctypedef int MemPoolAttr 'cudaMemPoolAttr'
+
+    ctypedef int MemAllocationType 'cudaMemAllocationType'
+    ctypedef int MemAllocationHandleType 'cudaMemAllocationHandleType'
+    ctypedef int MemLocationType 'cudaMemLocationType'
+    IF CUPY_CUDA_VERSION > 0:
+        # This is for the annoying nested struct, which is not
+        # perfectly supprted in Cython
+        ctypedef struct _MemLocation 'cudaMemLocation':
+            MemLocationType type
+            int id
+
+        ctypedef struct _MemPoolProps 'cudaMemPoolProps':
+            MemAllocationType allocType
+            MemAllocationHandleType handleTypes
+            _MemLocation location
+    ELSE:
+        ctypedef struct _MemPoolProps 'cudaMemPoolProps':
+            pass  # for HIP & RTD
+
+    IF CUPY_CUDA_VERSION > 0:
+        ctypedef struct _PointerAttributes 'cudaPointerAttributes':
+            int type
+            int device
+            void* devicePointer
+            void* hostPointer
+    ELIF CUPY_HIP_VERSION > 0:
+        ctypedef struct _PointerAttributes 'cudaPointerAttributes':
+            int memoryType
+            int device
+            void* devicePointer
+            void* hostPointer
+    ELSE:
+        ctypedef struct _PointerAttributes 'cudaPointerAttributes':
+            pass  # for RTD
+
+    IF CUPY_CUDA_VERSION >= 11000:
+        # We can't use IF in the middle of structs declaration
+        # to add or ignore fields in compile time so we have to
+        # replicate the struct definition
+        ctypedef struct DeviceProp 'cudaDeviceProp':
+            char         name[256]
+            cudaUUID     uuid
+            char         luid[8]
+            unsigned int luidDeviceNodeMask
+            size_t       totalGlobalMem
+            size_t       sharedMemPerBlock
+            int          regsPerBlock
+            int          warpSize
+            size_t       memPitch
+            int          maxThreadsPerBlock
+            int          maxThreadsDim[3]
+            int          maxGridSize[3]
+            int          clockRate
+            size_t       totalConstMem
+            int          major
+            int          minor
+            size_t       textureAlignment
+            size_t       texturePitchAlignment
+            int          deviceOverlap
+            int          multiProcessorCount
+            int          kernelExecTimeoutEnabled
+            int          integrated
+            int          canMapHostMemory
+            int          computeMode
+            int          maxTexture1D
+            int          maxTexture1DMipmap
+            int          maxTexture1DLinear
+            int          maxTexture2D[2]
+            int          maxTexture2DMipmap[2]
+            int          maxTexture2DLinear[3]
+            int          maxTexture2DGather[2]
+            int          maxTexture3D[3]
+            int          maxTexture3DAlt[3]
+            int          maxTextureCubemap
+            int          maxTexture1DLayered[2]
+            int          maxTexture2DLayered[3]
+            int          maxTextureCubemapLayered[2]
+            int          maxSurface1D
+            int          maxSurface2D[2]
+            int          maxSurface3D[3]
+            int          maxSurface1DLayered[2]
+            int          maxSurface2DLayered[3]
+            int          maxSurfaceCubemap
+            int          maxSurfaceCubemapLayered[2]
+            size_t       surfaceAlignment
+            int          concurrentKernels
+            int          ECCEnabled
+            int          pciBusID
+            int          pciDeviceID
+            int          pciDomainID
+            int          tccDriver
+            int          asyncEngineCount
+            int          unifiedAddressing
+            int          memoryClockRate
+            int          memoryBusWidth
+            int          l2CacheSize
+            int          persistingL2CacheMaxSize  # CUDA 11.0 field
+            int          maxThreadsPerMultiProcessor
+            int          streamPrioritiesSupported
+            int          globalL1CacheSupported
+            int          localL1CacheSupported
+            size_t       sharedMemPerMultiprocessor
+            int          regsPerMultiprocessor
+            int          managedMemory
+            int          isMultiGpuBoard
+            int          multiGpuBoardGroupID
+            int          hostNativeAtomicSupported
+            int          singleToDoublePrecisionPerfRatio
+            int          pageableMemoryAccess
+            int          concurrentManagedAccess
+            int          computePreemptionSupported
+            int          canUseHostPointerForRegisteredMem
+            int          cooperativeLaunch
+            int          cooperativeMultiDeviceLaunch
+            size_t       sharedMemPerBlockOptin
+            int          pageableMemoryAccessUsesHostPageTables
+            int          directManagedMemAccessFromHost
+            int          maxBlocksPerMultiProcessor  # CUDA 11.0 field
+            int          accessPolicyMaxWindowSize  # CUDA 11.0 field
+            size_t       reservedSharedMemPerBlock  # CUDA 11.0 field
+    ELIF CUPY_CUDA_VERSION >= 10000:
+        ctypedef struct DeviceProp 'cudaDeviceProp':
+            char         name[256]
+            cudaUUID     uuid
+            char         luid[8]
+            unsigned int luidDeviceNodeMask
+            size_t       totalGlobalMem
+            size_t       sharedMemPerBlock
+            int          regsPerBlock
+            int          warpSize
+            size_t       memPitch
+            int          maxThreadsPerBlock
+            int          maxThreadsDim[3]
+            int          maxGridSize[3]
+            int          clockRate
+            size_t       totalConstMem
+            int          major
+            int          minor
+            size_t       textureAlignment
+            size_t       texturePitchAlignment
+            int          deviceOverlap
+            int          multiProcessorCount
+            int          kernelExecTimeoutEnabled
+            int          integrated
+            int          canMapHostMemory
+            int          computeMode
+            int          maxTexture1D
+            int          maxTexture1DMipmap
+            int          maxTexture1DLinear
+            int          maxTexture2D[2]
+            int          maxTexture2DMipmap[2]
+            int          maxTexture2DLinear[3]
+            int          maxTexture2DGather[2]
+            int          maxTexture3D[3]
+            int          maxTexture3DAlt[3]
+            int          maxTextureCubemap
+            int          maxTexture1DLayered[2]
+            int          maxTexture2DLayered[3]
+            int          maxTextureCubemapLayered[2]
+            int          maxSurface1D
+            int          maxSurface2D[2]
+            int          maxSurface3D[3]
+            int          maxSurface1DLayered[2]
+            int          maxSurface2DLayered[3]
+            int          maxSurfaceCubemap
+            int          maxSurfaceCubemapLayered[2]
+            size_t       surfaceAlignment
+            int          concurrentKernels
+            int          ECCEnabled
+            int          pciBusID
+            int          pciDeviceID
+            int          pciDomainID
+            int          tccDriver
+            int          asyncEngineCount
+            int          unifiedAddressing
+            int          memoryClockRate
+            int          memoryBusWidth
+            int          l2CacheSize
+            int          maxThreadsPerMultiProcessor
+            int          streamPrioritiesSupported
+            int          globalL1CacheSupported
+            int          localL1CacheSupported
+            size_t       sharedMemPerMultiprocessor
+            int          regsPerMultiprocessor
+            int          managedMemory
+            int          isMultiGpuBoard
+            int          multiGpuBoardGroupID
+            int          hostNativeAtomicSupported
+            int          singleToDoublePrecisionPerfRatio
+            int          pageableMemoryAccess
+            int          concurrentManagedAccess
+            int          computePreemptionSupported
+            int          canUseHostPointerForRegisteredMem
+            int          cooperativeLaunch
+            int          cooperativeMultiDeviceLaunch
+            size_t       sharedMemPerBlockOptin
+            int          pageableMemoryAccessUsesHostPageTables
+            int          directManagedMemAccessFromHost
+    ELIF CUPY_HIP_VERSION > 0:
+        ctypedef struct deviceArch 'hipDeviceArch_t':
+            unsigned hasGlobalInt32Atomics
+            unsigned hasGlobalFloatAtomicExch
+            unsigned hasSharedInt32Atomics
+            unsigned hasSharedFloatAtomicExch
+            unsigned hasFloatAtomicAdd
+
+            unsigned hasGlobalInt64Atomics
+            unsigned hasSharedInt64Atomics
+
+            unsigned hasDoubles
+
+            unsigned hasWarpVote
+            unsigned hasWarpBallot
+            unsigned hasWarpShuffle
+            unsigned hasFunnelShift
+
+            unsigned hasThreadFenceSystem
+            unsigned hasSyncThreadsExt
+
+            unsigned hasSurfaceFuncs
+            unsigned has3dGrid
+            unsigned hasDynamicParallelism
+
+        IF CUPY_HIP_VERSION >= 310:
+            ctypedef struct DeviceProp 'cudaDeviceProp':
+                char name[256]
+                size_t totalGlobalMem
+                size_t sharedMemPerBlock
+                int regsPerBlock
+                int warpSize
+                int maxThreadsPerBlock
+                int maxThreadsDim[3]
+                int maxGridSize[3]
+                int clockRate
+                int memoryClockRate
+                int memoryBusWidth
+                size_t totalConstMem
+                int major
+                int minor
+                int multiProcessorCount
+                int l2CacheSize
+                int maxThreadsPerMultiProcessor
+                int computeMode
+                int clockInstructionRate
+                deviceArch arch
+                int concurrentKernels
+                int pciDomainID
+                int pciBusID
+                int pciDeviceID
+                size_t maxSharedMemoryPerMultiProcessor
+                int isMultiGpuBoard
+                int canMapHostMemory
+                int gcnArch
+                # gcnArchName is added since ROCm 3.6, but given it's just
+                # 'gfx'+str(gcnArch), in order not to duplicate another struct
+                # we add it here
+                char gcnArchName[256]
+                int integrated
+                int cooperativeLaunch
+                int cooperativeMultiDeviceLaunch
+                int maxTexture1D
+                int maxTexture2D[2]
+                int maxTexture3D[3]
+                unsigned int* hdpMemFlushCntl
+                unsigned int* hdpRegFlushCntl
+                size_t memPitch
+                size_t textureAlignment
+                size_t texturePitchAlignment
+                int kernelExecTimeoutEnabled
+                int ECCEnabled
+                int tccDriver
+                int cooperativeMultiDeviceUnmatchedFunc
+                int cooperativeMultiDeviceUnmatchedGridDim
+                int cooperativeMultiDeviceUnmatchedBlockDim
+                int cooperativeMultiDeviceUnmatchedSharedMem
+                int isLargeBar
+                # New since ROCm 3.10.0
+                int asicRevision
+                int managedMemory
+                int directManagedMemAccessFromHost
+                int concurrentManagedAccess
+                int pageableMemoryAccess
+                int pageableMemoryAccessUsesHostPageTables
+        ELSE:
+            ctypedef struct DeviceProp 'cudaDeviceProp':
+                char name[256]
+                size_t totalGlobalMem
+                size_t sharedMemPerBlock
+                int regsPerBlock
+                int warpSize
+                int maxThreadsPerBlock
+                int maxThreadsDim[3]
+                int maxGridSize[3]
+                int clockRate
+                int memoryClockRate
+                int memoryBusWidth
+                size_t totalConstMem
+                int major
+                int minor
+                int multiProcessorCount
+                int l2CacheSize
+                int maxThreadsPerMultiProcessor
+                int computeMode
+                int clockInstructionRate
+                deviceArch arch
+                int concurrentKernels
+                int pciDomainID
+                int pciBusID
+                int pciDeviceID
+                size_t maxSharedMemoryPerMultiProcessor
+                int isMultiGpuBoard
+                int canMapHostMemory
+                int gcnArch
+                int integrated
+                int cooperativeLaunch
+                int cooperativeMultiDeviceLaunch
+                int maxTexture1D
+                int maxTexture2D[2]
+                int maxTexture3D[3]
+                unsigned int* hdpMemFlushCntl
+                unsigned int* hdpRegFlushCntl
+                size_t memPitch
+                size_t textureAlignment
+                size_t texturePitchAlignment
+                int kernelExecTimeoutEnabled
+                int ECCEnabled
+                int tccDriver
+                int cooperativeMultiDeviceUnmatchedFunc
+                int cooperativeMultiDeviceUnmatchedGridDim
+                int cooperativeMultiDeviceUnmatchedBlockDim
+                int cooperativeMultiDeviceUnmatchedSharedMem
+                int isLargeBar
+    ELSE:  # for RTD
+        ctypedef struct DeviceProp 'cudaDeviceProp':
+            char         name[256]
diff --git a/cupy_backends/cuda/api/driver.pxd b/cupy_backends/cuda/api/driver.pxd
new file mode 100644
index 0000000..e6a70c4
--- /dev/null
+++ b/cupy_backends/cuda/api/driver.pxd
@@ -0,0 +1,101 @@
+from libc.stdint cimport intptr_t
+
+
+###############################################################################
+# Types and Enums
+###############################################################################
+
+IF CUPY_USE_CUDA_PYTHON:
+    from cuda.ccuda cimport *
+    # Aliases for compatibillity with existing CuPy codebase.
+    # Keep in sync with names defined in `_driver_typedef.pxi`.
+    # TODO(kmaehashi): Remove these aliases.
+    ctypedef CUdevice Device
+    ctypedef CUresult Result
+    ctypedef CUcontext Context
+    ctypedef CUdeviceptr Deviceptr
+    ctypedef CUevent Event
+    ctypedef CUfunction Function
+    ctypedef CUmodule Module
+    ctypedef CUstream Stream
+    ctypedef CUlinkState LinkState
+    ctypedef CUarray_st* Array
+    ctypedef CUarray_format Array_format
+    ctypedef CUDA_ARRAY_DESCRIPTOR Array_desc
+    ctypedef CUaddress_mode Address_mode
+    ctypedef CUfilter_mode Filter_mode
+ELSE:
+    include "_driver_typedef.pxi"
+    from cupy_backends.cuda.api._driver_enum cimport *
+
+
+###############################################################################
+# Build-time version
+###############################################################################
+
+cpdef get_build_version()
+
+###############################################################################
+# Primary context management
+###############################################################################
+
+cpdef devicePrimaryCtxRelease(Device dev)
+
+###############################################################################
+# Context management
+###############################################################################
+
+cpdef intptr_t ctxGetCurrent() except? 0
+cpdef ctxSetCurrent(intptr_t ctx)
+cpdef intptr_t ctxCreate(Device dev) except? 0
+cpdef ctxDestroy(intptr_t ctx)
+cpdef int ctxGetDevice() except? -1
+
+###############################################################################
+# Module load and kernel execution
+###############################################################################
+
+cpdef intptr_t linkCreate() except? 0
+cpdef linkAddData(intptr_t state, int input_type, bytes data, unicode name)
+cpdef linkAddFile(intptr_t state, int input_type, unicode path)
+cpdef bytes linkComplete(intptr_t state)
+cpdef linkDestroy(intptr_t state)
+cpdef intptr_t moduleLoad(str filename) except? 0
+cpdef intptr_t moduleLoadData(bytes image) except? 0
+cpdef moduleUnload(intptr_t module)
+cpdef intptr_t moduleGetFunction(intptr_t module, str funcname) except? 0
+cpdef intptr_t moduleGetGlobal(intptr_t module, str varname) except? 0
+cpdef launchKernel(
+    intptr_t f, unsigned int grid_dim_x, unsigned int grid_dim_y,
+    unsigned int grid_dim_z, unsigned int block_dim_x,
+    unsigned int block_dim_y, unsigned int block_dim_z,
+    unsigned int shared_mem_bytes, intptr_t stream, intptr_t kernel_params,
+    intptr_t extra)
+cpdef launchCooperativeKernel(
+    intptr_t f, unsigned int grid_dim_x, unsigned int grid_dim_y,
+    unsigned int grid_dim_z, unsigned int block_dim_x,
+    unsigned int block_dim_y, unsigned int block_dim_z,
+    unsigned int shared_mem_bytes, intptr_t stream, intptr_t kernel_params)
+
+###############################################################################
+# Kernel attributes
+###############################################################################
+
+cpdef int funcGetAttribute(int attribute, intptr_t func) except? -2
+cpdef funcSetAttribute(intptr_t func, int attribute, int value)
+
+###############################################################################
+# Occupancy
+###############################################################################
+
+cpdef int occupancyMaxActiveBlocksPerMultiprocessor(
+    intptr_t func, int blockSize, size_t dynamicSMemSize)
+
+cpdef occupancyMaxPotentialBlockSize(intptr_t func, size_t dynamicSMemSize,
+                                     int blockSizeLimit)
+
+###############################################################################
+# Stream management
+###############################################################################
+
+cpdef intptr_t streamGetCtx(intptr_t stream) except? 0
diff --git a/cupy_backends/cuda/api/driver.pyx b/cupy_backends/cuda/api/driver.pyx
new file mode 100644
index 0000000..67feca7
--- /dev/null
+++ b/cupy_backends/cuda/api/driver.pyx
@@ -0,0 +1,331 @@
+# distutils: language = c++
+
+"""Thin wrapper of CUDA Driver API.
+
+There are four differences compared to the original C API.
+
+1. Not all functions are ported.
+2. Errors are translated into CUDADriverError exceptions.
+3. The 'cu' prefix of each API is omitted and the next character is set to
+   lower case.
+4. The resulting values are returned directly instead of references.
+
+"""
+cimport cython  # NOQA
+from libc.stdint cimport intptr_t
+
+
+###############################################################################
+# Extern and Constants
+###############################################################################
+
+IF CUPY_USE_CUDA_PYTHON:
+    from cuda.ccuda cimport *
+ELSE:
+    include '_driver_extern.pxi'
+    pass  # for cython-lint
+
+cdef extern from '../../cupy_backend.h' nogil:
+    # Build-time version
+    # Note: CUDA_VERSION is defined either in CUDA Python or _driver_extern.pxi
+    enum: HIP_VERSION
+
+# Provide access to constants from Python.
+from cupy_backends.cuda.api._driver_enum import *
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+class CUDADriverError(RuntimeError):
+
+    def __init__(self, Result status):
+        self.status = status
+        cdef const char *name
+        cdef const char *msg
+        cuGetErrorName(status, &name)
+        cuGetErrorString(status, &msg)
+        cdef bytes s_name = name, s_msg = msg
+        super(CUDADriverError, self).__init__(
+            '%s: %s' % (s_name.decode(), s_msg.decode()))
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        raise CUDADriverError(status)
+
+
+@cython.profile(False)
+cdef inline void check_attribute_status(int status, int* pi) except *:
+    # set attribute to -1 on older versions of CUDA where it was undefined
+    if status == CUDA_ERROR_INVALID_VALUE:
+        pi[0] = -1
+    elif status != 0:
+        raise CUDADriverError(status)
+
+
+###############################################################################
+# Build-time version
+###############################################################################
+
+cpdef get_build_version():
+    """Returns the CUDA_VERSION / HIP_VERSION constant.
+
+    Note that when built with CUDA Python support, CUDA_VERSION will become a
+    constant:
+
+    https://github.com/NVIDIA/cuda-python/blob/v11.4.0/cuda/ccuda.pxd#L2268
+
+    In CuPy codebase, use `runtime.runtimeGetVersion()` instead of
+    this function to change the behavior based on the target CUDA version.
+    """
+
+    # The versions are mutually exclusive
+    if CUPY_CUDA_VERSION > 0:
+        return CUDA_VERSION
+    elif CUPY_HIP_VERSION > 0:
+        return HIP_VERSION
+    else:
+        return 0
+
+
+cpdef bint _is_cuda_python():
+    return CUPY_USE_CUDA_PYTHON
+
+
+###############################################################################
+# Primary context management
+###############################################################################
+
+cpdef devicePrimaryCtxRelease(Device dev):
+    with nogil:
+        status = cuDevicePrimaryCtxRelease(dev)
+    check_status(status)
+
+###############################################################################
+# Context management
+###############################################################################
+
+cpdef intptr_t ctxGetCurrent() except? 0:
+    cdef Context ctx
+    with nogil:
+        status = cuCtxGetCurrent(&ctx)
+    check_status(status)
+    return <intptr_t>ctx
+
+cpdef ctxSetCurrent(intptr_t ctx):
+    with nogil:
+        status = cuCtxSetCurrent(<Context>ctx)
+    check_status(status)
+
+cpdef intptr_t ctxCreate(Device dev) except? 0:
+    cdef Context ctx
+    cdef unsigned int flags = 0
+    with nogil:
+        status = cuCtxCreate(&ctx, flags, dev)
+    check_status(status)
+    return <intptr_t>ctx
+
+cpdef ctxDestroy(intptr_t ctx):
+    with nogil:
+        status = cuCtxDestroy(<Context>ctx)
+    check_status(status)
+
+cpdef int ctxGetDevice() except? -1:
+    cdef Device dev
+    with nogil:
+        status = cuCtxGetDevice(&dev)
+    check_status(status)
+    return dev
+
+
+###############################################################################
+# Module load and kernel execution
+###############################################################################
+
+cpdef intptr_t linkCreate() except? 0:
+    cdef LinkState state
+    with nogil:
+        status = cuLinkCreate(0, <CUjit_option*>0, <void**>0, &state)
+    check_status(status)
+    return <intptr_t>state
+
+
+cpdef linkAddData(intptr_t state, int input_type, bytes data, unicode name):
+    cdef const char* data_ptr = data
+    cdef size_t data_size = len(data) + 1
+    cdef bytes b_name = name.encode()
+    cdef const char* b_name_ptr = b_name
+    with nogil:
+        status = cuLinkAddData(
+            <LinkState>state, <CUjitInputType>input_type, <void*>data_ptr,
+            data_size, b_name_ptr, 0, <CUjit_option*>0, <void**>0)
+    check_status(status)
+
+
+cpdef linkAddFile(intptr_t state, int input_type, unicode path):
+    cdef bytes b_path = path.encode()
+    cdef const char* b_path_ptr = b_path
+    with nogil:
+        status = cuLinkAddFile(<LinkState>state, <CUjitInputType>input_type,
+                               b_path_ptr, 0, <CUjit_option*>0, <void**>0)
+    check_status(status)
+
+
+cpdef bytes linkComplete(intptr_t state):
+    cdef void* cubinOut
+    cdef size_t sizeOut
+    with nogil:
+        status = cuLinkComplete(<LinkState>state, &cubinOut, &sizeOut)
+    check_status(status)
+    return bytes((<char*>cubinOut)[:sizeOut])
+
+
+cpdef linkDestroy(intptr_t state):
+    with nogil:
+        status = cuLinkDestroy(<LinkState>state)
+    check_status(status)
+
+
+cpdef intptr_t moduleLoad(str filename) except? 0:
+    cdef Module module
+    cdef bytes b_filename = filename.encode()
+    cdef char* b_filename_ptr = b_filename
+    with nogil:
+        status = cuModuleLoad(&module, b_filename_ptr)
+    check_status(status)
+    return <intptr_t>module
+
+
+cpdef intptr_t moduleLoadData(bytes image) except? 0:
+    cdef Module module
+    cdef char* image_ptr = image
+    with nogil:
+        status = cuModuleLoadData(&module, image_ptr)
+    check_status(status)
+    return <intptr_t>module
+
+
+cpdef moduleUnload(intptr_t module):
+    with nogil:
+        status = cuModuleUnload(<Module>module)
+    check_status(status)
+
+
+cpdef intptr_t moduleGetFunction(intptr_t module, str funcname) except? 0:
+    cdef Function func
+    cdef bytes b_funcname = funcname.encode()
+    cdef char* b_funcname_ptr = b_funcname
+    with nogil:
+        status = cuModuleGetFunction(&func, <Module>module, b_funcname_ptr)
+    check_status(status)
+    return <intptr_t>func
+
+
+cpdef intptr_t moduleGetGlobal(intptr_t module, str varname) except? 0:
+    cdef Deviceptr var
+    cdef size_t size
+    cdef bytes b_varname = varname.encode()
+    cdef char* b_varname_ptr = b_varname
+    with nogil:
+        status = cuModuleGetGlobal(&var, &size, <Module>module, b_varname_ptr)
+    check_status(status)
+    return <intptr_t>var
+
+
+cpdef launchKernel(
+        intptr_t f, unsigned int grid_dim_x, unsigned int grid_dim_y,
+        unsigned int grid_dim_z, unsigned int block_dim_x,
+        unsigned int block_dim_y, unsigned int block_dim_z,
+        unsigned int shared_mem_bytes, intptr_t stream, intptr_t kernel_params,
+        intptr_t extra):
+    with nogil:
+        status = cuLaunchKernel(
+            <Function>f, grid_dim_x, grid_dim_y, grid_dim_z,
+            block_dim_x, block_dim_y, block_dim_z,
+            shared_mem_bytes, <Stream>stream,
+            <void**>kernel_params, <void**>extra)
+    check_status(status)
+
+
+cpdef launchCooperativeKernel(
+        intptr_t f, unsigned int grid_dim_x, unsigned int grid_dim_y,
+        unsigned int grid_dim_z, unsigned int block_dim_x,
+        unsigned int block_dim_y, unsigned int block_dim_z,
+        unsigned int shared_mem_bytes, intptr_t stream,
+        intptr_t kernel_params):
+    with nogil:
+        status = cuLaunchCooperativeKernel(
+            <Function>f, grid_dim_x, grid_dim_y, grid_dim_z,
+            block_dim_x, block_dim_y, block_dim_z,
+            shared_mem_bytes, <Stream>stream, <void**>kernel_params)
+    check_status(status)
+
+
+###############################################################################
+# Function attributes
+###############################################################################
+
+# -1 is reserved by check_attribute_status
+cpdef int funcGetAttribute(int attribute, intptr_t f) except? -2:
+    cdef int pi
+    with nogil:
+        status = cuFuncGetAttribute(
+            &pi,
+            <CUfunction_attribute> attribute,
+            <Function> f)
+    check_attribute_status(status, &pi)
+    return pi
+
+
+cpdef funcSetAttribute(intptr_t f, int attribute, int value):
+    with nogil:
+        status = cuFuncSetAttribute(
+            <Function> f,
+            <CUfunction_attribute> attribute,
+            value)
+    check_status(status)
+
+
+###############################################################################
+# Occupancy
+###############################################################################
+
+cpdef int occupancyMaxActiveBlocksPerMultiprocessor(
+        intptr_t func, int blockSize, size_t dynamicSMemSize):
+    cdef int numBlocks
+    with nogil:
+        status = cuOccupancyMaxActiveBlocksPerMultiprocessor(
+            &numBlocks, <Function> func, blockSize, dynamicSMemSize)
+    check_status(status)
+    return numBlocks
+
+
+cpdef occupancyMaxPotentialBlockSize(intptr_t func, size_t dynamicSMemSize,
+                                     int blockSizeLimit):
+    # CUoccupancyB2DSize is set to NULL as there is no way to pass in a
+    # unary function from Python.
+    cdef int minGridSize, blockSize
+    with nogil:
+        status = cuOccupancyMaxPotentialBlockSize(
+            &minGridSize, &blockSize, <Function> func, <CUoccupancyB2DSize>
+            NULL, dynamicSMemSize, blockSizeLimit)
+    check_status(status)
+    return minGridSize, blockSize
+
+
+###############################################################################
+# Stream management
+###############################################################################
+
+cpdef intptr_t streamGetCtx(intptr_t stream) except? 0:
+    cdef Context ctx
+    with nogil:
+        status = cuStreamGetCtx(<Stream>stream, &ctx)
+    check_status(status)
+    return <intptr_t>ctx
diff --git a/cupy_backends/cuda/api/runtime.pxd b/cupy_backends/cuda/api/runtime.pxd
new file mode 100644
index 0000000..40f2968
--- /dev/null
+++ b/cupy_backends/cuda/api/runtime.pxd
@@ -0,0 +1,321 @@
+from libc.stdint cimport intptr_t, uintmax_t
+
+
+###############################################################################
+# Classes
+###############################################################################
+
+cdef class PointerAttributes:
+    cdef:
+        readonly int device
+        readonly intptr_t devicePointer
+        readonly intptr_t hostPointer
+        readonly int type
+
+cdef class MemPoolProps:
+    # flatten the struct & list meaningful members
+    cdef:
+        int allocType
+        int handleType
+        int locationType
+        int devId
+
+
+###############################################################################
+# Types and Enums
+###############################################################################
+
+IF CUPY_USE_CUDA_PYTHON:
+    from cuda.ccudart cimport *
+    # Aliases for compatibillity with existing CuPy codebase.
+    # Keep in sync with names defined in `_runtime_typedef.pxi`.
+    # TODO(kmaehashi): Remove these aliases.
+    ctypedef cudaError_t Error
+    ctypedef cudaDataType_t DataType
+
+    ctypedef cudaDeviceAttr DeviceAttr
+    ctypedef cudaMemoryAdvise MemoryAdvise
+
+    ctypedef cudaStreamCallback_t StreamCallback
+    ctypedef cudaHostFn_t HostFn
+
+    ctypedef cudaChannelFormatKind ChannelFormatKind
+    ctypedef cudaChannelFormatDesc ChannelFormatDesc
+    ctypedef cudaTextureObject_t TextureObject
+    ctypedef cudaSurfaceObject_t SurfaceObject
+    ctypedef cudaResourceType ResourceType
+    ctypedef cudaTextureAddressMode TextureAddressMode
+    ctypedef cudaTextureFilterMode TextureFilterMode
+    ctypedef cudaTextureReadMode TextureReadMode
+    ctypedef cudaResourceViewDesc ResourceViewDesc
+    ctypedef cudaArray_t Array
+    ctypedef cudaExtent Extent
+    ctypedef cudaPos Pos
+    ctypedef cudaPitchedPtr PitchedPtr
+    ctypedef cudaMemcpyKind MemoryKind
+    ctypedef cudaMipmappedArray_t MipmappedArray
+
+    ctypedef cudaLimit Limit
+
+    ctypedef cudaResourceDesc ResourceDesc
+
+    ctypedef cudaMemcpy3DParms Memcpy3DParms
+
+    ctypedef cudaTextureDesc TextureDesc
+
+    ctypedef cudaIpcMemHandle_t IpcMemHandle
+
+    ctypedef cudaIpcEventHandle_t IpcEventHandle
+
+    ctypedef cudaUUID_t cudaUUID
+
+    ctypedef cudaMemPool_t MemPool
+
+    ctypedef cudaMemPoolAttr MemPoolAttr
+
+    ctypedef cudaMemPoolProps _MemPoolProps
+
+    ctypedef cudaPointerAttributes _PointerAttributes
+
+    ctypedef cudaDeviceProp DeviceProp
+
+    ctypedef cudaStreamCaptureStatus StreamCaptureStatus
+    ctypedef cudaStreamCaptureMode StreamCaptureMode
+    ctypedef cudaGraph_t Graph
+    ctypedef cudaGraphExec_t GraphExec
+    ctypedef cudaGraphNode_t GraphNode
+
+    ctypedef cudaMemAllocationType MemAllocationType
+    ctypedef cudaMemAllocationHandleType MemAllocationHandleType
+    ctypedef cudaMemLocationType MemLocationType
+
+ELSE:
+    include "_runtime_typedef.pxi"
+    from cupy_backends.cuda.api._runtime_enum cimport *
+
+
+# For backward compatibility, keep APIs not prefixed with "cuda".
+cpdef enum:
+    memcpyHostToHost = 0
+    memcpyHostToDevice = 1
+    memcpyDeviceToHost = 2
+    memcpyDeviceToDevice = 3
+    memcpyDefault = 4
+
+    hostAllocDefault = 0
+    hostAllocPortable = 1
+    hostAllocMapped = 2
+    hostAllocWriteCombined = 4
+
+    # cudaStream flags
+    streamDefault = 0
+    streamNonBlocking = 1
+
+    # cudaStream handles
+    streamLegacy = 1
+    streamPerThread = 2
+
+    eventDefault = 0
+    eventBlockingSync = 1
+    eventDisableTiming = 2
+    eventInterprocess = 4
+
+    # cudaStreamCaptureMode
+    streamCaptureModeGlobal = 0
+    streamCaptureModeThreadLocal = 1
+    streamCaptureModeRelaxed = 2
+
+    # cudaStreamCaptureStatus
+    streamCaptureStatusNone = 0
+    streamCaptureStatusActive = 1
+    streamCaptureStatusInvalidated = 2
+
+    # cudaMemoryType
+    memoryTypeUnregistered = 0
+    memoryTypeHost = 1
+    memoryTypeDevice = 2
+    memoryTypeManaged = 3
+
+
+###############################################################################
+# Constants
+###############################################################################
+
+# TODO(kmaehashi): Deprecate these aliases and use `cuda*`.
+cdef int errorMemoryAllocation
+cdef int errorInvalidValue
+cdef int errorPeerAccessAlreadyEnabled
+cdef int errorContextIsDestroyed
+cdef int errorInvalidResourceHandle
+
+cdef int deviceAttributeComputeCapabilityMajor
+cdef int deviceAttributeComputeCapabilityMinor
+
+
+###############################################################################
+# Constants (CuPy)
+###############################################################################
+
+cdef bint _is_hip_environment
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+cpdef check_status(int status)
+
+
+###############################################################################
+# Initialization
+###############################################################################
+
+cpdef int driverGetVersion() except? -1
+cpdef int runtimeGetVersion() except? -1
+
+
+###############################################################################
+# Device and context operations
+###############################################################################
+
+cpdef int getDevice() except? -1
+cpdef int deviceGetAttribute(int attrib, int device) except? -1
+cpdef int deviceGetByPCIBusId(str pci_bus_id) except? -1
+cpdef str deviceGetPCIBusId(int device)
+cpdef int getDeviceCount() except? -1
+cpdef setDevice(int device)
+cpdef deviceSynchronize()
+cpdef getDeviceProperties(int device)
+
+cpdef int deviceCanAccessPeer(int device, int peerDevice) except? -1
+cpdef deviceEnablePeerAccess(int peerDevice)
+cpdef _deviceEnsurePeerAccess(int peerDevice)
+
+cpdef size_t deviceGetLimit(int limit) except? -1
+cpdef deviceSetLimit(int limit, size_t value)
+
+
+###############################################################################
+# Memory management
+###############################################################################
+
+cpdef intptr_t malloc(size_t size) except? 0
+cpdef intptr_t mallocManaged(size_t size, unsigned int flags=*) except? 0
+cpdef intptr_t malloc3DArray(intptr_t desc, size_t width, size_t height,
+                             size_t depth, unsigned int flags=*) except? 0
+cpdef intptr_t mallocArray(intptr_t desc, size_t width, size_t height,
+                           unsigned int flags=*) except? 0
+cpdef intptr_t mallocAsync(size_t size, intptr_t stream) except? 0
+cpdef intptr_t mallocFromPoolAsync(size_t, intptr_t, intptr_t) except? 0
+cpdef intptr_t hostAlloc(size_t size, unsigned int flags) except? 0
+cpdef hostRegister(intptr_t ptr, size_t size, unsigned int flags)
+cpdef hostUnregister(intptr_t ptr)
+cpdef free(intptr_t ptr)
+cpdef freeHost(intptr_t ptr)
+cpdef freeArray(intptr_t ptr)
+cpdef freeAsync(intptr_t ptr, intptr_t stream)
+cpdef memGetInfo()
+cpdef memcpy(intptr_t dst, intptr_t src, size_t size, int kind)
+cpdef memcpyAsync(intptr_t dst, intptr_t src, size_t size, int kind,
+                  intptr_t stream)
+cpdef memcpyPeer(intptr_t dst, int dstDevice, intptr_t src, int srcDevice,
+                 size_t size)
+cpdef memcpyPeerAsync(intptr_t dst, int dstDevice,
+                      intptr_t src, int srcDevice,
+                      size_t size, intptr_t stream)
+cpdef memcpy2D(intptr_t dst, size_t dpitch, intptr_t src, size_t spitch,
+               size_t width, size_t height, MemoryKind kind)
+cpdef memcpy2DAsync(intptr_t dst, size_t dpitch, intptr_t src, size_t spitch,
+                    size_t width, size_t height, MemoryKind kind,
+                    intptr_t stream)
+cpdef memcpy2DFromArray(intptr_t dst, size_t dpitch, intptr_t src,
+                        size_t wOffset, size_t hOffset, size_t width,
+                        size_t height, int kind)
+cpdef memcpy2DFromArrayAsync(intptr_t dst, size_t dpitch, intptr_t src,
+                             size_t wOffset, size_t hOffset, size_t width,
+                             size_t height, int kind, intptr_t stream)
+cpdef memcpy2DToArray(intptr_t dst, size_t wOffset, size_t hOffset,
+                      intptr_t src, size_t spitch, size_t width, size_t height,
+                      int kind)
+cpdef memcpy2DToArrayAsync(intptr_t dst, size_t wOffset, size_t hOffset,
+                           intptr_t src, size_t spitch, size_t width,
+                           size_t height, int kind, intptr_t stream)
+cpdef memcpy3D(intptr_t Memcpy3DParmsPtr)
+cpdef memcpy3DAsync(intptr_t Memcpy3DParmsPtr, intptr_t stream)
+cpdef memset(intptr_t ptr, int value, size_t size)
+cpdef memsetAsync(intptr_t ptr, int value, size_t size, intptr_t stream)
+cpdef memPrefetchAsync(intptr_t devPtr, size_t count, int dstDevice,
+                       intptr_t stream)
+cpdef memAdvise(intptr_t devPtr, size_t count, int advice, int device)
+cpdef PointerAttributes pointerGetAttributes(intptr_t ptr)
+cpdef intptr_t deviceGetDefaultMemPool(int) except? 0
+cpdef intptr_t deviceGetMemPool(int) except? 0
+cpdef deviceSetMemPool(int, intptr_t)
+cpdef intptr_t memPoolCreate(MemPoolProps) except? 0
+cpdef memPoolDestroy(intptr_t)
+cpdef memPoolTrimTo(intptr_t, size_t)
+cpdef memPoolGetAttribute(intptr_t, int)
+cpdef memPoolSetAttribute(intptr_t, int, object)
+
+
+###############################################################################
+# Stream and Event
+###############################################################################
+
+cpdef intptr_t streamCreate() except? 0
+cpdef intptr_t streamCreateWithFlags(unsigned int flags) except? 0
+cpdef streamDestroy(intptr_t stream)
+cpdef streamSynchronize(intptr_t stream)
+cpdef streamAddCallback(intptr_t stream, callback, intptr_t arg,
+                        unsigned int flags=*)
+cpdef launchHostFunc(intptr_t stream, callback, intptr_t arg)
+cpdef streamQuery(intptr_t stream)
+cpdef streamWaitEvent(intptr_t stream, intptr_t event, unsigned int flags=*)
+cpdef streamBeginCapture(intptr_t stream, int mode=*)
+cpdef intptr_t streamEndCapture(intptr_t stream) except? 0
+cpdef bint streamIsCapturing(intptr_t stream) except*
+cpdef intptr_t eventCreate() except? 0
+cpdef intptr_t eventCreateWithFlags(unsigned int flags) except? 0
+cpdef eventDestroy(intptr_t event)
+cpdef float eventElapsedTime(intptr_t start, intptr_t end) except? 0
+cpdef eventQuery(intptr_t event)
+cpdef eventRecord(intptr_t event, intptr_t stream)
+cpdef eventSynchronize(intptr_t event)
+
+
+##############################################################################
+# util
+##############################################################################
+
+cdef _ensure_context()
+
+
+##############################################################################
+# Texture
+##############################################################################
+
+cpdef uintmax_t createTextureObject(
+    intptr_t ResDesc, intptr_t TexDesc) except? 0
+cpdef destroyTextureObject(uintmax_t texObject)
+cdef ChannelFormatDesc getChannelDesc(intptr_t array) except*
+cdef ResourceDesc getTextureObjectResourceDesc(uintmax_t texobj) except*
+cdef TextureDesc getTextureObjectTextureDesc(uintmax_t texobj) except*
+cdef Extent make_Extent(size_t w, size_t h, size_t d) except*
+cdef Pos make_Pos(size_t x, size_t y, size_t z) except*
+cdef PitchedPtr make_PitchedPtr(
+    intptr_t d, size_t p, size_t xsz, size_t ysz) except*
+
+cpdef uintmax_t createSurfaceObject(intptr_t ResDesc) except? 0
+cpdef destroySurfaceObject(uintmax_t surfObject)
+# TODO(leofang): add cudaGetSurfaceObjectResourceDesc
+
+
+##############################################################################
+# Graph
+##############################################################################
+
+cpdef graphDestroy(intptr_t graph)
+cpdef graphExecDestroy(intptr_t graphExec)
+cpdef intptr_t graphInstantiate(intptr_t graph) except? 0
+cpdef graphLaunch(intptr_t graphExec, intptr_t stream)
+cpdef graphUpload(intptr_t graphExec, intptr_t stream)
diff --git a/cupy_backends/cuda/api/runtime.pyx b/cupy_backends/cuda/api/runtime.pyx
new file mode 100644
index 0000000..69db512
--- /dev/null
+++ b/cupy_backends/cuda/api/runtime.pyx
@@ -0,0 +1,1097 @@
+"""Thin wrapper of CUDA Runtime API.
+
+There are four differences compared to the original C API.
+
+1. Not all functions are ported.
+2. Errors are translated into CUDARuntimeError exceptions.
+3. The 'cuda' prefix of each API is omitted and the next character is set to
+   lower case.
+4. The resulting values are returned directly instead of references.
+
+"""
+from libc.stdint cimport uint64_t
+from libc.string cimport memset as c_memset
+
+import threading as _threading
+
+cimport cpython  # NOQA
+cimport cython  # NOQA
+
+from cupy_backends.cuda.api cimport driver  # NOQA
+from cupy_backends.cuda.libs cimport nvrtc  # no-cython-lint
+
+
+###############################################################################
+# Classes
+###############################################################################
+
+cdef class PointerAttributes:
+
+    def __init__(self, int device, intptr_t devicePointer,
+                 intptr_t hostPointer, int type=-1):
+        self.type = type
+        self.device = device
+        self.devicePointer = devicePointer
+        self.hostPointer = hostPointer
+
+cdef class MemPoolProps:
+
+    def __init__(
+            self, int allocType, int handleType, int locationType, int devId):
+        self.allocType = allocType
+        self.handleType = handleType
+        self.locationType = locationType
+        self.devId = devId
+
+
+###############################################################################
+# Thread-local storage
+###############################################################################
+
+cdef object _thread_local = _threading.local()
+
+
+cdef class _ThreadLocal:
+
+    cdef list context_initialized
+
+    def __init__(self):
+        cdef int i
+        self.context_initialized = [False for i in range(getDeviceCount())]
+
+    @staticmethod
+    cdef _ThreadLocal get():
+        try:
+            tls = _thread_local.tls
+        except AttributeError:
+            tls = _thread_local.tls = _ThreadLocal()
+        return <_ThreadLocal>tls
+
+
+###############################################################################
+# Extern
+###############################################################################
+
+IF CUPY_USE_CUDA_PYTHON:
+    from cuda.ccudart cimport *
+ELSE:
+    include '_runtime_extern.pxi'
+    pass  # for cython-lint
+
+cdef extern from '../../cupy_backend_runtime.h' nogil:
+    bint hip_environment
+
+
+###############################################################################
+# Constants
+###############################################################################
+
+# TODO(kmaehashi): Deprecate these aliases and use `cuda*`.
+errorInvalidValue = cudaErrorInvalidValue
+errorMemoryAllocation = cudaErrorMemoryAllocation
+errorPeerAccessAlreadyEnabled = cudaErrorPeerAccessAlreadyEnabled
+errorContextIsDestroyed = cudaErrorContextIsDestroyed
+errorInvalidResourceHandle = cudaErrorInvalidResourceHandle
+deviceAttributeComputeCapabilityMajor = cudaDevAttrComputeCapabilityMajor
+deviceAttributeComputeCapabilityMinor = cudaDevAttrComputeCapabilityMinor
+
+
+# Provide access to constants from Python.
+# TODO(kmaehashi): Deprecate aliases above so that we can just do:
+# from cupy_backends.cuda.api._runtime_enum import *
+def _export_enum():
+    import sys
+    import cupy_backends.cuda.api._runtime_enum as _runtime_enum
+    this = sys.modules[__name__]
+    for key in dir(_runtime_enum):
+        if not key.startswith('_'):
+            setattr(this, key, getattr(_runtime_enum, key))
+
+
+_export_enum()
+
+
+###############################################################################
+# Constants (CuPy)
+###############################################################################
+
+_is_hip_environment = hip_environment  # for runtime being cimport'd
+is_hip = hip_environment  # for runtime being import'd
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+class CUDARuntimeError(RuntimeError):
+
+    def __init__(self, status):
+        self.status = status
+        cdef bytes name = cudaGetErrorName(<Error>status)
+        cdef bytes msg = cudaGetErrorString(<Error>status)
+        super(CUDARuntimeError, self).__init__(
+            '%s: %s' % (name.decode(), msg.decode()))
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        # to reset error status
+        cudaGetLastError()
+        raise CUDARuntimeError(status)
+
+
+###############################################################################
+# Initialization
+###############################################################################
+
+cpdef int driverGetVersion() except? -1:
+    cdef int version
+    status = cudaDriverGetVersion(&version)
+    check_status(status)
+    return version
+
+cpdef int runtimeGetVersion() except? -1:
+    cdef int version
+    IF CUPY_USE_CUDA_PYTHON:
+        # Workarounds an issue that cuda-python returns its version instead of
+        # the real runtime version.
+        # https://github.com/NVIDIA/cuda-python/issues/16
+        cdef int major, minor
+        (major, minor) = nvrtc.getVersion()
+        version = major * 1000 + minor * 10
+    ELSE:
+        status = cudaRuntimeGetVersion(&version)
+        check_status(status)
+    return version
+
+
+###############################################################################
+# Device and context operations
+###############################################################################
+
+cpdef int getDevice() except? -1:
+    cdef int device
+    status = cudaGetDevice(&device)
+    check_status(status)
+    return device
+
+cpdef int deviceGetAttribute(int attrib, int device) except? -1:
+    cdef int ret
+    status = cudaDeviceGetAttribute(&ret, <DeviceAttr>attrib, device)
+    check_status(status)
+    return ret
+
+cpdef getDeviceProperties(int device):
+    cdef DeviceProp props
+    cdef int status = cudaGetDeviceProperties(&props, device)
+    check_status(status)
+
+    cdef dict properties = {'name': b'UNAVAILABLE'}  # for RTD
+
+    # Common properties to CUDA 9.0, 9.2, 10.x, 11.x, and HIP
+    IF CUPY_CUDA_VERSION > 0 or CUPY_HIP_VERSION > 0:
+        properties = {
+            'name': props.name,
+            'totalGlobalMem': props.totalGlobalMem,
+            'sharedMemPerBlock': props.sharedMemPerBlock,
+            'regsPerBlock': props.regsPerBlock,
+            'warpSize': props.warpSize,
+            'maxThreadsPerBlock': props.maxThreadsPerBlock,
+            'maxThreadsDim': tuple(props.maxThreadsDim),
+            'maxGridSize': tuple(props.maxGridSize),
+            'clockRate': props.clockRate,
+            'totalConstMem': props.totalConstMem,
+            'major': props.major,
+            'minor': props.minor,
+            'textureAlignment': props.textureAlignment,
+            'texturePitchAlignment': props.texturePitchAlignment,
+            'multiProcessorCount': props.multiProcessorCount,
+            'kernelExecTimeoutEnabled': props.kernelExecTimeoutEnabled,
+            'integrated': props.integrated,
+            'canMapHostMemory': props.canMapHostMemory,
+            'computeMode': props.computeMode,
+            'maxTexture1D': props.maxTexture1D,
+            'maxTexture2D': tuple(props.maxTexture2D),
+            'maxTexture3D': tuple(props.maxTexture3D),
+            'concurrentKernels': props.concurrentKernels,
+            'ECCEnabled': props.ECCEnabled,
+            'pciBusID': props.pciBusID,
+            'pciDeviceID': props.pciDeviceID,
+            'pciDomainID': props.pciDomainID,
+            'tccDriver': props.tccDriver,
+            'memoryClockRate': props.memoryClockRate,
+            'memoryBusWidth': props.memoryBusWidth,
+            'l2CacheSize': props.l2CacheSize,
+            'maxThreadsPerMultiProcessor': props.maxThreadsPerMultiProcessor,
+            'isMultiGpuBoard': props.isMultiGpuBoard,
+            'cooperativeLaunch': props.cooperativeLaunch,
+            'cooperativeMultiDeviceLaunch': props.cooperativeMultiDeviceLaunch,
+        }
+    IF CUPY_USE_CUDA_PYTHON or CUPY_CUDA_VERSION >= 9020:
+        properties['deviceOverlap'] = props.deviceOverlap
+        properties['maxTexture1DMipmap'] = props.maxTexture1DMipmap
+        properties['maxTexture1DLinear'] = props.maxTexture1DLinear
+        properties['maxTexture1DLayered'] = tuple(props.maxTexture1DLayered)
+        properties['maxTexture2DMipmap'] = tuple(props.maxTexture2DMipmap)
+        properties['maxTexture2DLinear'] = tuple(props.maxTexture2DLinear)
+        properties['maxTexture2DLayered'] = tuple(props.maxTexture2DLayered)
+        properties['maxTexture2DGather'] = tuple(props.maxTexture2DGather)
+        properties['maxTexture3DAlt'] = tuple(props.maxTexture3DAlt)
+        properties['maxTextureCubemap'] = props.maxTextureCubemap
+        properties['maxTextureCubemapLayered'] = tuple(
+            props.maxTextureCubemapLayered)
+        properties['maxSurface1D'] = props.maxSurface1D
+        properties['maxSurface1DLayered'] = tuple(props.maxSurface1DLayered)
+        properties['maxSurface2D'] = tuple(props.maxSurface2D)
+        properties['maxSurface2DLayered'] = tuple(props.maxSurface2DLayered)
+        properties['maxSurface3D'] = tuple(props.maxSurface3D)
+        properties['maxSurfaceCubemap'] = props.maxSurfaceCubemap
+        properties['maxSurfaceCubemapLayered'] = tuple(
+            props.maxSurfaceCubemapLayered)
+        properties['surfaceAlignment'] = props.surfaceAlignment
+        properties['asyncEngineCount'] = props.asyncEngineCount
+        properties['unifiedAddressing'] = props.unifiedAddressing
+        properties['streamPrioritiesSupported'] = (
+            props.streamPrioritiesSupported)
+        properties['globalL1CacheSupported'] = props.globalL1CacheSupported
+        properties['localL1CacheSupported'] = props.localL1CacheSupported
+        properties['sharedMemPerMultiprocessor'] = (
+            props.sharedMemPerMultiprocessor)
+        properties['regsPerMultiprocessor'] = props.regsPerMultiprocessor
+        properties['managedMemory'] = props.managedMemory
+        properties['multiGpuBoardGroupID'] = props.multiGpuBoardGroupID
+        properties['hostNativeAtomicSupported'] = (
+            props.hostNativeAtomicSupported)
+        properties['singleToDoublePrecisionPerfRatio'] = (
+            props.singleToDoublePrecisionPerfRatio)
+        properties['pageableMemoryAccess'] = props.pageableMemoryAccess
+        properties['concurrentManagedAccess'] = props.concurrentManagedAccess
+        properties['computePreemptionSupported'] = (
+            props.computePreemptionSupported)
+        properties['canUseHostPointerForRegisteredMem'] = (
+            props.canUseHostPointerForRegisteredMem)
+        properties['sharedMemPerBlockOptin'] = props.sharedMemPerBlockOptin
+        properties['pageableMemoryAccessUsesHostPageTables'] = (
+            props.pageableMemoryAccessUsesHostPageTables)
+        properties['directManagedMemAccessFromHost'] = (
+            props.directManagedMemAccessFromHost)
+    if CUPY_USE_CUDA_PYTHON or CUPY_CUDA_VERSION >=10000:
+        properties['uuid'] = props.uuid.bytes
+        properties['luid'] = props.luid
+        properties['luidDeviceNodeMask'] = props.luidDeviceNodeMask
+    if CUPY_USE_CUDA_PYTHON or CUPY_CUDA_VERSION >= 11000:
+        properties['persistingL2CacheMaxSize'] = props.persistingL2CacheMaxSize
+        properties['maxBlocksPerMultiProcessor'] = (
+            props.maxBlocksPerMultiProcessor)
+        properties['accessPolicyMaxWindowSize'] = (
+            props.accessPolicyMaxWindowSize)
+        properties['reservedSharedMemPerBlock'] = (
+            props.reservedSharedMemPerBlock)
+    IF CUPY_HIP_VERSION > 0:  # HIP-only props
+        properties['clockInstructionRate'] = props.clockInstructionRate
+        properties['maxSharedMemoryPerMultiProcessor'] = (
+            props.maxSharedMemoryPerMultiProcessor)
+        properties['gcnArch'] = props.gcnArch
+        properties['hdpMemFlushCntl'] = <intptr_t>(props.hdpMemFlushCntl)
+        properties['hdpRegFlushCntl'] = <intptr_t>(props.hdpRegFlushCntl)
+        properties['memPitch'] = props.memPitch
+        properties['cooperativeMultiDeviceUnmatchedFunc'] = (
+            props.cooperativeMultiDeviceUnmatchedFunc)
+        properties['cooperativeMultiDeviceUnmatchedGridDim'] = (
+            props.cooperativeMultiDeviceUnmatchedGridDim)
+        properties['cooperativeMultiDeviceUnmatchedBlockDim'] = (
+            props.cooperativeMultiDeviceUnmatchedBlockDim)
+        properties['cooperativeMultiDeviceUnmatchedSharedMem'] = (
+            props.cooperativeMultiDeviceUnmatchedSharedMem)
+        properties['isLargeBar'] = props.isLargeBar
+
+        cdef dict arch = {}  # for hipDeviceArch_t
+        arch['hasGlobalInt32Atomics'] = props.arch.hasGlobalInt32Atomics
+        arch['hasGlobalFloatAtomicExch'] = props.arch.hasGlobalFloatAtomicExch
+        arch['hasSharedInt32Atomics'] = props.arch.hasSharedInt32Atomics
+        arch['hasSharedFloatAtomicExch'] = props.arch.hasSharedFloatAtomicExch
+        arch['hasFloatAtomicAdd'] = props.arch.hasFloatAtomicAdd
+        arch['hasGlobalInt64Atomics'] = props.arch.hasGlobalInt64Atomics
+        arch['hasSharedInt64Atomics'] = props.arch.hasSharedInt64Atomics
+        arch['hasDoubles'] = props.arch.hasDoubles
+        arch['hasWarpVote'] = props.arch.hasWarpVote
+        arch['hasWarpBallot'] = props.arch.hasWarpBallot
+        arch['hasWarpShuffle'] = props.arch.hasWarpShuffle
+        arch['hasFunnelShift'] = props.arch.hasFunnelShift
+        arch['hasThreadFenceSystem'] = props.arch.hasThreadFenceSystem
+        arch['hasSyncThreadsExt'] = props.arch.hasSyncThreadsExt
+        arch['hasSurfaceFuncs'] = props.arch.hasSurfaceFuncs
+        arch['has3dGrid'] = props.arch.has3dGrid
+        arch['hasDynamicParallelism'] = props.arch.hasDynamicParallelism
+        properties['arch'] = arch
+    IF CUPY_HIP_VERSION >= 310:
+        properties['gcnArchName'] = props.gcnArchName
+        properties['asicRevision'] = props.asicRevision
+        properties['managedMemory'] = props.managedMemory
+        properties['directManagedMemAccessFromHost'] = (
+            props.directManagedMemAccessFromHost)
+        properties['concurrentManagedAccess'] = props.concurrentManagedAccess
+        properties['pageableMemoryAccess'] = props.pageableMemoryAccess
+        properties['pageableMemoryAccessUsesHostPageTables'] = (
+            props.pageableMemoryAccessUsesHostPageTables)
+    return properties
+
+cpdef int deviceGetByPCIBusId(str pci_bus_id) except? -1:
+    # Encode the python string before passing to native code
+    byte_pci_bus_id = pci_bus_id.encode('ascii')
+    cdef const char* c_pci_bus_id = byte_pci_bus_id
+
+    cdef int device = -1
+    cdef int status
+    status = cudaDeviceGetByPCIBusId(&device, c_pci_bus_id)
+    check_status(status)
+    # on ROCm, it might fail silently, so we also need to check if the
+    # device is meaningful or not
+    if hip_environment and device == -1:
+        check_status(cudaErrorInvalidValue)
+    return device
+
+cpdef str deviceGetPCIBusId(int device):
+    # The PCI Bus ID string must be able to store 13 characters including the
+    # NULL-terminator according to the CUDA documentation.
+    # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__DEVICE.html
+    cdef char pci_bus_id[13]
+    status = cudaDeviceGetPCIBusId(pci_bus_id, 13, device)
+    check_status(status)
+    return pci_bus_id.decode('ascii')
+
+cpdef int getDeviceCount() except? -1:
+    cdef int count
+    status = cudaGetDeviceCount(&count)
+    check_status(status)
+    return count
+
+cpdef setDevice(int device):
+    status = cudaSetDevice(device)
+    check_status(status)
+
+cpdef deviceSynchronize():
+    with nogil:
+        status = cudaDeviceSynchronize()
+    check_status(status)
+
+cpdef int deviceCanAccessPeer(int device, int peerDevice) except? -1:
+    cdef int ret
+    status = cudaDeviceCanAccessPeer(&ret, device, peerDevice)
+    check_status(status)
+    return ret
+
+cpdef deviceEnablePeerAccess(int peerDevice):
+    status = cudaDeviceEnablePeerAccess(peerDevice, 0)
+    check_status(status)
+
+cpdef deviceDisablePeerAccess(int peerDevice):
+    status = cudaDeviceDisablePeerAccess(peerDevice)
+    check_status(status)
+
+cpdef _deviceEnsurePeerAccess(int peerDevice):
+    status = cudaDeviceEnablePeerAccess(peerDevice, 0)
+    if status == 0:
+        return
+    elif status == errorPeerAccessAlreadyEnabled:
+        cudaGetLastError()  # clear error status
+        return
+    check_status(status)
+
+cpdef size_t deviceGetLimit(int limit) except? -1:
+    cdef size_t value
+    status = cudaDeviceGetLimit(&value, <Limit>limit)
+    check_status(status)
+    return value
+
+cpdef deviceSetLimit(int limit, size_t value):
+    status = cudaDeviceSetLimit(<Limit>limit, value)
+    check_status(status)
+
+
+###############################################################################
+# IPC operations
+###############################################################################
+
+cpdef ipcCloseMemHandle(intptr_t devPtr):
+    status = cudaIpcCloseMemHandle(<void*>devPtr)
+    check_status(status)
+
+cpdef ipcGetEventHandle(intptr_t event):
+    cdef IpcEventHandle handle
+    status = cudaIpcGetEventHandle(&handle, <driver.Event>event)
+    check_status(status)
+    # We need to do this due to a bug in Cython that
+    # cuts out the 0 bytes in an array of chars when
+    # constructing the python object
+    # resulting in different sizes assignment errors
+    # when recreating the struct from the python
+    # array of bytes
+    reserved = [<unsigned char>handle.reserved[i] for i in range(64)]
+    return bytes(reserved)
+
+cpdef ipcGetMemHandle(intptr_t devPtr):
+    cdef IpcMemHandle handle
+    status = cudaIpcGetMemHandle(&handle, <void*>devPtr)
+    check_status(status)
+    # We need to do this due to a bug in Cython that
+    # when converting an array of chars in C to a python object
+    # it discards the data after the first 0 value
+    # resulting in a loss of data, as this is not a string
+    # but a buffer of bytes
+    reserved = [<unsigned char>handle.reserved[i] for i in range(64)]
+    return bytes(reserved)
+
+cpdef ipcOpenEventHandle(bytes handle):
+    cdef driver.Event event
+    cdef IpcEventHandle handle_
+    handle_.reserved = handle
+    status = cudaIpcOpenEventHandle(&event, handle_)
+    check_status(status)
+    return <intptr_t>event
+
+cpdef ipcOpenMemHandle(bytes handle,
+                       unsigned int flags=cudaIpcMemLazyEnablePeerAccess):
+    cdef void* devPtr
+    cdef IpcMemHandle handle_
+    handle_.reserved = handle
+    status = cudaIpcOpenMemHandle(&devPtr, handle_, flags)
+    check_status(status)
+    return <intptr_t>devPtr
+
+
+###############################################################################
+# Memory management
+###############################################################################
+
+cpdef intptr_t malloc(size_t size) except? 0:
+    cdef void* ptr
+    with nogil:
+        status = cudaMalloc(&ptr, size)
+    check_status(status)
+    return <intptr_t>ptr
+
+cpdef intptr_t mallocManaged(
+        size_t size, unsigned int flags=cudaMemAttachGlobal) except? 0:
+    if 0 < CUPY_HIP_VERSION < 40300000:
+        raise RuntimeError('Managed memory requires ROCm 4.3+')
+    cdef void* ptr
+    with nogil:
+        status = cudaMallocManaged(&ptr, size, flags)
+    check_status(status)
+    return <intptr_t>ptr
+
+cpdef intptr_t malloc3DArray(intptr_t descPtr, size_t width, size_t height,
+                             size_t depth, unsigned int flags=0) except? 0:
+    cdef Array ptr
+    cdef Extent extent = make_cudaExtent(width, height, depth)
+    with nogil:
+        status = cudaMalloc3DArray(&ptr, <ChannelFormatDesc*>descPtr, extent,
+                                   flags)
+    check_status(status)
+    return <intptr_t>ptr
+
+cpdef intptr_t mallocArray(intptr_t descPtr, size_t width, size_t height,
+                           unsigned int flags=0) except? 0:
+    cdef Array ptr
+    with nogil:
+        status = cudaMallocArray(&ptr, <ChannelFormatDesc*>descPtr, width,
+                                 height, flags)
+    check_status(status)
+    return <intptr_t>ptr
+
+cpdef intptr_t mallocAsync(size_t size, intptr_t stream) except? 0:
+    cdef void* ptr
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support mallocAsync')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('mallocAsync is supported since CUDA 11.2')
+    with nogil:
+        status = cudaMallocAsync(&ptr, size, <driver.Stream>stream)
+    check_status(status)
+    return <intptr_t>ptr
+
+cpdef intptr_t mallocFromPoolAsync(
+        size_t size, intptr_t pool, intptr_t stream) except? 0:
+    cdef void* ptr
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support mallocFromPoolAsync')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('mallocFromPoolAsync is supported since CUDA 11.2')
+    with nogil:
+        status = cudaMallocFromPoolAsync(
+            &ptr, size, <MemPool>pool, <driver.Stream>stream)
+    check_status(status)
+    return <intptr_t>ptr
+
+cpdef intptr_t hostAlloc(size_t size, unsigned int flags) except? 0:
+    cdef void* ptr
+    with nogil:
+        status = cudaHostAlloc(&ptr, size, flags)
+    check_status(status)
+    return <intptr_t>ptr
+
+cpdef hostRegister(intptr_t ptr, size_t size, unsigned int flags):
+    with nogil:
+        status = cudaHostRegister(<void*>ptr, size, flags)
+    check_status(status)
+
+cpdef hostUnregister(intptr_t ptr):
+    with nogil:
+        status = cudaHostUnregister(<void*>ptr)
+    check_status(status)
+
+cpdef free(intptr_t ptr):
+    with nogil:
+        status = cudaFree(<void*>ptr)
+    check_status(status)
+
+cpdef freeHost(intptr_t ptr):
+    with nogil:
+        status = cudaFreeHost(<void*>ptr)
+    check_status(status)
+
+cpdef freeArray(intptr_t ptr):
+    with nogil:
+        status = cudaFreeArray(<Array>ptr)
+    check_status(status)
+
+cpdef freeAsync(intptr_t ptr, intptr_t stream):
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support freeAsync')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('freeAsync is supported since CUDA 11.2')
+    with nogil:
+        status = cudaFreeAsync(<void*>ptr, <driver.Stream>stream)
+    check_status(status)
+
+cpdef memGetInfo():
+    cdef size_t free, total
+    status = cudaMemGetInfo(&free, &total)
+    check_status(status)
+    return free, total
+
+cpdef memcpy(intptr_t dst, intptr_t src, size_t size, int kind):
+    with nogil:
+        status = cudaMemcpy(<void*>dst, <void*>src, size, <MemoryKind>kind)
+    check_status(status)
+
+cpdef memcpyAsync(intptr_t dst, intptr_t src, size_t size, int kind,
+                  intptr_t stream):
+    with nogil:
+        status = cudaMemcpyAsync(
+            <void*>dst, <void*>src, size, <MemoryKind>kind,
+            <driver.Stream>stream)
+    check_status(status)
+
+cpdef memcpyPeer(intptr_t dst, int dstDevice, intptr_t src, int srcDevice,
+                 size_t size):
+    with nogil:
+        status = cudaMemcpyPeer(<void*>dst, dstDevice, <void*>src, srcDevice,
+                                size)
+    check_status(status)
+
+cpdef memcpyPeerAsync(intptr_t dst, int dstDevice, intptr_t src, int srcDevice,
+                      size_t size, intptr_t stream):
+    with nogil:
+        status = cudaMemcpyPeerAsync(<void*>dst, dstDevice, <void*>src,
+                                     srcDevice, size, <driver.Stream> stream)
+    check_status(status)
+
+cpdef memcpy2D(intptr_t dst, size_t dpitch, intptr_t src, size_t spitch,
+               size_t width, size_t height, MemoryKind kind):
+    with nogil:
+        status = cudaMemcpy2D(<void*>dst, dpitch, <void*>src, spitch, width,
+                              height, kind)
+    check_status(status)
+
+cpdef memcpy2DAsync(intptr_t dst, size_t dpitch, intptr_t src, size_t spitch,
+                    size_t width, size_t height, MemoryKind kind,
+                    intptr_t stream):
+    with nogil:
+        status = cudaMemcpy2DAsync(<void*>dst, dpitch, <void*>src, spitch,
+                                   width, height, kind, <driver.Stream>stream)
+    check_status(status)
+
+cpdef memcpy2DFromArray(intptr_t dst, size_t dpitch, intptr_t src,
+                        size_t wOffset, size_t hOffset, size_t width,
+                        size_t height, int kind):
+    with nogil:
+        status = cudaMemcpy2DFromArray(<void*>dst, dpitch, <Array>src, wOffset,
+                                       hOffset, width, height,
+                                       <MemoryKind>kind)
+    check_status(status)
+
+cpdef memcpy2DFromArrayAsync(intptr_t dst, size_t dpitch, intptr_t src,
+                             size_t wOffset, size_t hOffset, size_t width,
+                             size_t height, int kind, intptr_t stream):
+    with nogil:
+        status = cudaMemcpy2DFromArrayAsync(<void*>dst, dpitch, <Array>src,
+                                            wOffset, hOffset, width, height,
+                                            <MemoryKind>kind,
+                                            <driver.Stream>stream)
+    check_status(status)
+
+cpdef memcpy2DToArray(intptr_t dst, size_t wOffset, size_t hOffset,
+                      intptr_t src, size_t spitch, size_t width, size_t height,
+                      int kind):
+    with nogil:
+        status = cudaMemcpy2DToArray(<Array>dst, wOffset, hOffset, <void*>src,
+                                     spitch, width, height, <MemoryKind>kind)
+    check_status(status)
+
+cpdef memcpy2DToArrayAsync(intptr_t dst, size_t wOffset, size_t hOffset,
+                           intptr_t src, size_t spitch, size_t width,
+                           size_t height, int kind, intptr_t stream):
+    with nogil:
+        status = cudaMemcpy2DToArrayAsync(<Array>dst, wOffset, hOffset,
+                                          <void*>src, spitch, width, height,
+                                          <MemoryKind>kind,
+                                          <driver.Stream>stream)
+    check_status(status)
+
+cpdef memcpy3D(intptr_t Memcpy3DParmsPtr):
+    with nogil:
+        status = cudaMemcpy3D(<Memcpy3DParms*>Memcpy3DParmsPtr)
+    check_status(status)
+
+cpdef memcpy3DAsync(intptr_t Memcpy3DParmsPtr, intptr_t stream):
+    with nogil:
+        status = cudaMemcpy3DAsync(<Memcpy3DParms*>Memcpy3DParmsPtr,
+                                   <driver.Stream> stream)
+    check_status(status)
+
+cpdef memset(intptr_t ptr, int value, size_t size):
+    with nogil:
+        status = cudaMemset(<void*>ptr, value, size)
+    check_status(status)
+
+cpdef memsetAsync(intptr_t ptr, int value, size_t size, intptr_t stream):
+    with nogil:
+        status = cudaMemsetAsync(<void*>ptr, value, size,
+                                 <driver.Stream> stream)
+    check_status(status)
+
+cpdef memPrefetchAsync(intptr_t devPtr, size_t count, int dstDevice,
+                       intptr_t stream):
+    if 0 < CUPY_HIP_VERSION < 40300000:
+        raise RuntimeError('Managed memory requires ROCm 4.3+')
+    with nogil:
+        status = cudaMemPrefetchAsync(<void*>devPtr, count, dstDevice,
+                                      <driver.Stream> stream)
+    check_status(status)
+
+cpdef memAdvise(intptr_t devPtr, size_t count, int advice, int device):
+    if 0 < CUPY_HIP_VERSION < 40300000:
+        raise RuntimeError('Managed memory requires ROCm 4.3+')
+    with nogil:
+        status = cudaMemAdvise(<void*>devPtr, count,
+                               <MemoryAdvise>advice, device)
+    check_status(status)
+
+cpdef PointerAttributes pointerGetAttributes(intptr_t ptr):
+    cdef _PointerAttributes attrs
+    status = cudaPointerGetAttributes(&attrs, <void*>ptr)
+    check_status(status)
+    IF CUPY_CUDA_VERSION > 0:
+        return PointerAttributes(
+            attrs.device,
+            <intptr_t>attrs.devicePointer,
+            <intptr_t>attrs.hostPointer,
+            attrs.type)
+    ELIF CUPY_HIP_VERSION > 0:
+        return PointerAttributes(
+            attrs.device,
+            <intptr_t>attrs.devicePointer,
+            <intptr_t>attrs.hostPointer,
+            attrs.memoryType)
+    ELSE:  # for RTD
+        return None
+
+cpdef intptr_t deviceGetDefaultMemPool(int device) except? 0:
+    '''Get the default mempool on the current device.'''
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support deviceGetDefaultMemPool')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('deviceGetDefaultMemPool is supported since '
+                           'CUDA 11.2')
+    cdef MemPool pool
+    with nogil:
+        status = cudaDeviceGetDefaultMemPool(&pool, device)
+    check_status(status)
+    return <intptr_t>(pool)
+
+cpdef intptr_t deviceGetMemPool(int device) except? 0:
+    '''Get the current mempool on the current device.'''
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support deviceGetMemPool')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('deviceGetMemPool is supported since '
+                           'CUDA 11.2')
+    cdef MemPool pool
+    with nogil:
+        status = cudaDeviceGetMemPool(&pool, device)
+    check_status(status)
+    return <intptr_t>(pool)
+
+cpdef deviceSetMemPool(int device, intptr_t pool):
+    '''Set the current mempool on the current device to pool.'''
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support deviceSetMemPool')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('deviceSetMemPool is supported since '
+                           'CUDA 11.2')
+    with nogil:
+        status = cudaDeviceSetMemPool(device, <MemPool>pool)
+    check_status(status)
+
+cpdef intptr_t memPoolCreate(MemPoolProps props) except? 0:
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support memPoolCreate')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('memPoolCreate is supported since CUDA 11.2')
+
+    cdef MemPool pool
+    cdef _MemPoolProps props_c
+    c_memset(&props_c, 0, sizeof(_MemPoolProps))
+    props_c.allocType = <MemAllocationType>props.allocType
+    props_c.handleTypes = <MemAllocationHandleType>props.handleType
+    props_c.location.type = <MemLocationType>props.locationType
+    props_c.location.id = props.devId
+
+    with nogil:
+        status = cudaMemPoolCreate(&pool, &props_c)
+    check_status(status)
+    return <intptr_t>pool
+
+cpdef memPoolDestroy(intptr_t pool):
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support memPoolDestroy')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('memPoolDestroy is supported since CUDA 11.2')
+    with nogil:
+        status = cudaMemPoolDestroy(<MemPool>pool)
+    check_status(status)
+
+cpdef memPoolTrimTo(intptr_t pool, size_t size):
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support memPoolTrimTo')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('memPoolTrimTo is supported since CUDA 11.2')
+    with nogil:
+        status = cudaMemPoolTrimTo(<MemPool>pool, size)
+    check_status(status)
+
+cpdef memPoolGetAttribute(intptr_t pool, int attr):
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support memPoolGetAttribute')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('memPoolGetAttribute is supported since CUDA 11.2')
+    cdef int val1
+    cdef uint64_t val2
+    cdef void* out
+    # TODO(leofang): check this hack when more cudaMemPoolAttr are added!
+    out = <void*>(&val1) if attr <= 0x3 else <void*>(&val2)
+    with nogil:
+        status = cudaMemPoolGetAttribute(<MemPool>pool, <MemPoolAttr>attr, out)
+    check_status(status)
+    # TODO(leofang): check this hack when more cudaMemPoolAttr are added!
+    # cast to Python int regardless of C types
+    return val1 if attr <= 0x3 else val2
+
+cpdef memPoolSetAttribute(intptr_t pool, int attr, object value):
+    if _is_hip_environment:
+        raise RuntimeError('HIP does not support memPoolSetAttribute')
+    if runtimeGetVersion() < 11020:
+        raise RuntimeError('memPoolSetAttribute is supported since CUDA 11.2')
+    cdef int val1
+    cdef uint64_t val2
+    cdef void* out
+    # TODO(leofang): check this hack when more cudaMemPoolAttr are added!
+    if attr <= 0x3:
+        val1 = value
+        out = <void*>(&val1)
+    else:
+        val2 = value
+        out = <void*>(&val2)
+    with nogil:
+        status = cudaMemPoolSetAttribute(<MemPool>pool, <MemPoolAttr>attr, out)
+    check_status(status)
+
+
+###############################################################################
+# Stream and Event
+###############################################################################
+
+cpdef intptr_t streamCreate() except? 0:
+    cdef driver.Stream stream
+    status = cudaStreamCreate(&stream)
+    check_status(status)
+    return <intptr_t>stream
+
+
+cpdef intptr_t streamCreateWithFlags(unsigned int flags) except? 0:
+    cdef driver.Stream stream
+    status = cudaStreamCreateWithFlags(&stream, flags)
+    check_status(status)
+    return <intptr_t>stream
+
+
+cpdef streamDestroy(intptr_t stream):
+    status = cudaStreamDestroy(<driver.Stream>stream)
+    check_status(status)
+
+
+cpdef streamSynchronize(intptr_t stream):
+    with nogil:
+        status = cudaStreamSynchronize(<driver.Stream>stream)
+    check_status(status)
+
+
+cdef _streamCallbackFunc(driver.Stream hStream, int status,
+                         void* func_arg) with gil:
+    obj = <object>func_arg
+    func, arg = obj
+    func(<intptr_t>hStream, status, arg)
+    cpython.Py_DECREF(obj)
+
+
+cdef _HostFnFunc(void* func_arg) with gil:
+    obj = <object>func_arg
+    func, arg = obj
+    func(arg)
+    cpython.Py_DECREF(obj)
+
+
+cpdef streamAddCallback(intptr_t stream, callback, intptr_t arg,
+                        unsigned int flags=0):
+    if _is_hip_environment and stream == 0:
+        raise RuntimeError('HIP does not allow adding callbacks to the '
+                           'default (null) stream')
+    func_arg = (callback, arg)
+    cpython.Py_INCREF(func_arg)
+    with nogil:
+        status = cudaStreamAddCallback(
+            <driver.Stream>stream, <StreamCallback>_streamCallbackFunc,
+            <void*>func_arg, flags)
+    check_status(status)
+
+
+cpdef launchHostFunc(intptr_t stream, callback, intptr_t arg):
+    if _is_hip_environment:
+        raise RuntimeError('This feature is not supported on HIP')
+
+    func_arg = (callback, arg)
+    cpython.Py_INCREF(func_arg)
+    with nogil:
+        status = cudaLaunchHostFunc(
+            <driver.Stream>stream, <HostFn>_HostFnFunc,
+            <void*>func_arg)
+    check_status(status)
+
+
+cpdef streamQuery(intptr_t stream):
+    return cudaStreamQuery(<driver.Stream>stream)
+
+
+cpdef streamWaitEvent(intptr_t stream, intptr_t event, unsigned int flags=0):
+    with nogil:
+        status = cudaStreamWaitEvent(<driver.Stream>stream,
+                                     <driver.Event>event, flags)
+    check_status(status)
+
+
+cpdef streamBeginCapture(intptr_t stream, int mode=streamCaptureModeRelaxed):
+    if _is_hip_environment:
+        raise RuntimeError('streamBeginCapture is not supported in ROCm')
+    # TODO(leofang): check and raise if stream == 0?
+    with nogil:
+        status = cudaStreamBeginCapture(<driver.Stream>stream,
+                                        <StreamCaptureMode>mode)
+    check_status(status)
+
+
+cpdef intptr_t streamEndCapture(intptr_t stream) except? 0:
+    # TODO(leofang): check and raise if stream == 0?
+    cdef Graph g
+    if _is_hip_environment:
+        raise RuntimeError('streamEndCapture is not supported in ROCm')
+    with nogil:
+        status = cudaStreamEndCapture(<driver.Stream>stream, &g)
+    check_status(status)
+    return <intptr_t>g
+
+
+cpdef bint streamIsCapturing(intptr_t stream) except*:
+    cdef StreamCaptureStatus s
+    if _is_hip_environment:
+        raise RuntimeError('streamIsCapturing is not supported in ROCm')
+    with nogil:
+        status = cudaStreamIsCapturing(<driver.Stream>stream, &s)
+    check_status(status)  # cudaErrorStreamCaptureImplicit could be raised here
+    if s == <StreamCaptureStatus>streamCaptureStatusInvalidated:
+        raise RuntimeError('the stream was capturing, but an error has '
+                           'invalidated the capture sequence')
+    return <bint>s
+
+
+cpdef intptr_t eventCreate() except? 0:
+    cdef driver.Event event
+    status = cudaEventCreate(&event)
+    check_status(status)
+    return <intptr_t>event
+
+cpdef intptr_t eventCreateWithFlags(unsigned int flags) except? 0:
+    cdef driver.Event event
+    status = cudaEventCreateWithFlags(&event, flags)
+    check_status(status)
+    return <intptr_t>event
+
+
+cpdef eventDestroy(intptr_t event):
+    status = cudaEventDestroy(<driver.Event>event)
+    check_status(status)
+
+
+cpdef float eventElapsedTime(intptr_t start, intptr_t end) except? 0:
+    cdef float ms
+    status = cudaEventElapsedTime(&ms, <driver.Event>start, <driver.Event>end)
+    check_status(status)
+    return ms
+
+
+cpdef eventQuery(intptr_t event):
+    return cudaEventQuery(<driver.Event>event)
+
+
+cpdef eventRecord(intptr_t event, intptr_t stream):
+    status = cudaEventRecord(<driver.Event>event, <driver.Stream>stream)
+    check_status(status)
+
+
+cpdef eventSynchronize(intptr_t event):
+    with nogil:
+        status = cudaEventSynchronize(<driver.Event>event)
+    check_status(status)
+
+
+##############################################################################
+# util
+##############################################################################
+
+cdef _ensure_context():
+    """Ensure that CUcontext bound to the calling host thread exists.
+
+    See discussion on https://github.com/cupy/cupy/issues/72 for details.
+    """
+    tls = _ThreadLocal.get()
+    cdef int dev = getDevice()
+    if not tls.context_initialized[dev]:
+        # Call Runtime API to establish context on this host thread.
+        memGetInfo()
+        tls.context_initialized[dev] = True
+
+
+##############################################################################
+# Texture
+##############################################################################
+
+cpdef uintmax_t createTextureObject(
+        intptr_t ResDescPtr, intptr_t TexDescPtr) except? 0:
+    cdef uintmax_t texobj = 0
+    with nogil:
+        status = cudaCreateTextureObject(<TextureObject*>(&texobj),
+                                         <ResourceDesc*>ResDescPtr,
+                                         <TextureDesc*>TexDescPtr,
+                                         <ResourceViewDesc*>NULL)
+    check_status(status)
+    return texobj
+
+cpdef destroyTextureObject(uintmax_t texObject):
+    with nogil:
+        status = cudaDestroyTextureObject(<TextureObject>texObject)
+    check_status(status)
+
+cpdef uintmax_t createSurfaceObject(intptr_t ResDescPtr) except? 0:
+    cdef uintmax_t surfobj = 0
+    with nogil:
+        status = cudaCreateSurfaceObject(<SurfaceObject*>(&surfobj),
+                                         <ResourceDesc*>ResDescPtr)
+    check_status(status)
+    return surfobj
+
+cpdef destroySurfaceObject(uintmax_t surfObject):
+    with nogil:
+        status = cudaDestroySurfaceObject(<SurfaceObject>surfObject)
+    check_status(status)
+
+cdef ChannelFormatDesc getChannelDesc(intptr_t array) except*:
+    cdef ChannelFormatDesc desc
+    with nogil:
+        status = cudaGetChannelDesc(&desc, <Array>array)
+    check_status(status)
+    return desc
+
+cdef ResourceDesc getTextureObjectResourceDesc(uintmax_t obj) except*:
+    cdef ResourceDesc desc
+    with nogil:
+        status = cudaGetTextureObjectResourceDesc(&desc, <TextureObject>obj)
+    check_status(status)
+    return desc
+
+cdef TextureDesc getTextureObjectTextureDesc(uintmax_t obj) except*:
+    cdef TextureDesc desc
+    with nogil:
+        status = cudaGetTextureObjectTextureDesc(&desc, <TextureObject>obj)
+    check_status(status)
+    return desc
+
+cdef Extent make_Extent(size_t w, size_t h, size_t d) except*:
+    return make_cudaExtent(w, h, d)
+
+cdef Pos make_Pos(size_t x, size_t y, size_t z) except*:
+    return make_cudaPos(x, y, z)
+
+cdef PitchedPtr make_PitchedPtr(
+        intptr_t d, size_t p, size_t xsz, size_t ysz) except*:
+    return make_cudaPitchedPtr(<void*>d, p, xsz, ysz)
+
+
+##############################################################################
+# Graph
+##############################################################################
+
+cpdef graphDestroy(intptr_t graph):
+    with nogil:
+        status = cudaGraphDestroy(<Graph>graph)
+    check_status(status)
+
+cpdef graphExecDestroy(intptr_t graphExec):
+    with nogil:
+        status = cudaGraphExecDestroy(<GraphExec>graphExec)
+    check_status(status)
+
+cpdef intptr_t graphInstantiate(intptr_t graph) except? 0:
+    # TODO(leofang): support reporting error log?
+    cdef GraphExec ge
+    with nogil:
+        status = cudaGraphInstantiate(<GraphExec*>(&ge), <Graph>graph,
+                                      NULL, NULL, 0)
+    check_status(status)
+    return <intptr_t>ge
+
+cpdef graphLaunch(intptr_t graphExec, intptr_t stream):
+    with nogil:
+        status = cudaGraphLaunch(<GraphExec>(graphExec), <driver.Stream>stream)
+    check_status(status)
+
+cpdef graphUpload(intptr_t graphExec, intptr_t stream):
+    if runtimeGetVersion() < 11010:
+        raise RuntimeError('graphUpload is supported since CUDA 11.1+')
+    with nogil:
+        status = cudaGraphUpload(<GraphExec>(graphExec), <driver.Stream>stream)
+    check_status(status)
diff --git a/cupy_backends/cuda/cupy_cublas.h b/cupy_backends/cuda/cupy_cublas.h
new file mode 100644
index 0000000..3cca8e3
--- /dev/null
+++ b/cupy_backends/cuda/cupy_cublas.h
@@ -0,0 +1,24 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_CUBLAS_H
+#define INCLUDE_GUARD_CUDA_CUPY_CUBLAS_H
+
+#include <cuda.h>
+#include <cublas_v2.h>
+
+#if CUDA_VERSION >= 11000
+
+#define cublasGemmEx_v11 cublasGemmEx
+#define cublasGemmStridedBatchedEx_v11 cublasGemmStridedBatchedEx
+
+#else
+
+typedef enum{} cublasComputeType_t;
+cublasStatus_t cublasGemmEx_v11(...) {
+    return CUBLAS_STATUS_NOT_SUPPORTED;
+}
+cublasStatus_t cublasGemmStridedBatchedEx_v11(...) {
+    return CUBLAS_STATUS_NOT_SUPPORTED;
+}
+
+#endif // if CUDA_VERSION >= 11000
+
+#endif // #ifndef INCLUDE_GUARD_CUDA_CUPY_CUBLAS_H
diff --git a/cupy_backends/cuda/cupy_cuda.h b/cupy_backends/cuda/cupy_cuda.h
new file mode 100644
index 0000000..86d32e7
--- /dev/null
+++ b/cupy_backends/cuda/cupy_cuda.h
@@ -0,0 +1,7 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_CUDA_H
+#define INCLUDE_GUARD_CUDA_CUPY_CUDA_H
+
+#include <cuda.h>
+#include <cuda_runtime.h>
+
+#endif // #ifndef INCLUDE_GUARD_CUDA_CUPY_CUDA_H
diff --git a/cupy_backends/cuda/cupy_cuda_profiler_api.h b/cupy_backends/cuda/cupy_cuda_profiler_api.h
new file mode 100644
index 0000000..19e039c
--- /dev/null
+++ b/cupy_backends/cuda/cupy_cuda_profiler_api.h
@@ -0,0 +1,18 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_PROFILER_H
+#define INCLUDE_GUARD_CUDA_CUPY_PROFILER_H
+
+#include <cuda.h>
+#include <cuda_profiler_api.h>
+
+#if CUDA_VERSION >= 12000
+// Functions removed in CUDA 12.0
+
+enum cudaOutputMode_t {};
+
+cudaError_t cudaProfilerInitialize(...) {
+    return cudaErrorUnknown;
+}
+
+#endif // #if CUDA_VERSION >= 12
+
+#endif // #ifndef INCLUDE_GUARD_CUDA_CUPY_PROFILER_H
diff --git a/cupy_backends/cuda/cupy_cuda_runtime.h b/cupy_backends/cuda/cupy_cuda_runtime.h
new file mode 100644
index 0000000..61c5bc7
--- /dev/null
+++ b/cupy_backends/cuda/cupy_cuda_runtime.h
@@ -0,0 +1,89 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_CUDA_RUNTIME_H
+#define INCLUDE_GUARD_CUDA_CUPY_CUDA_RUNTIME_H
+
+#include <cuda.h>  // for CUDA_VERSION
+#include <cuda_runtime.h>
+
+extern "C" {
+
+bool hip_environment = false;
+
+#if CUDA_VERSION < 10010
+const int cudaErrorContextIsDestroyed = 709;
+#endif
+
+#if CUDA_VERSION < 11010
+// APIs added in CUDA 11.1
+cudaError_t cudaGraphUpload(...) {
+    return cudaErrorUnknown;
+}
+#endif
+
+#if CUDA_VERSION < 11020
+// APIs added in CUDA 11.2
+typedef void* cudaMemPool_t;
+enum cudaMemPoolAttr {};
+enum cudaMemAllocationType {};
+enum cudaMemAllocationHandleType {};
+enum cudaMemLocationType {};
+struct cudaMemLocation {
+    int id;
+    cudaMemLocationType type;
+};
+struct cudaMemPoolProps {
+    cudaMemAllocationType allocType;
+    cudaMemAllocationHandleType handleTypes;
+    struct cudaMemLocation location;
+    unsigned char reserved[64];
+    void* win32SecurityAttributes;
+};
+
+cudaError_t cudaMallocAsync(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaMallocFromPoolAsync(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaFreeAsync(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaDeviceSetMemPool(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaDeviceGetDefaultMemPool(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaDeviceGetMemPool(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaMemPoolCreate(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaMemPoolDestroy(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaMemPoolTrimTo(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaMemPoolSetAttribute(...) {
+    return cudaErrorUnknown;
+}
+
+cudaError_t cudaMemPoolGetAttribute(...) {
+    return cudaErrorUnknown;
+}
+
+#endif
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_CUDA_CUPY_CUDA_RUNTIME_H
diff --git a/cupy_backends/cuda/cupy_cudnn.h b/cupy_backends/cuda/cupy_cudnn.h
new file mode 100644
index 0000000..a3a2829
--- /dev/null
+++ b/cupy_backends/cuda/cupy_cudnn.h
@@ -0,0 +1,425 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_CUDNN_H
+#define INCLUDE_GUARD_CUDA_CUPY_CUDNN_H
+
+#include <cudnn.h>
+
+
+///////////////////////////////////////////////////////////////////////////////
+// Definitions are for compatibility with cuDNN v5 and v6.
+///////////////////////////////////////////////////////////////////////////////
+
+extern "C" {
+
+#if defined(CUPY_NO_CUDA) || (CUDNN_VERSION < 6000)
+
+typedef enum {} cudnnRNNAlgo_t;
+typedef enum {} cudnnReduceTensorOp_t;
+typedef enum {} cudnnReduceTensorIndices_t;
+typedef enum {} cudnnIndicesType_t;
+
+typedef void* cudnnPersistentRNNPlan_t;
+typedef void* cudnnReduceTensorDescriptor_t;
+
+cudnnStatus_t cudnnCreatePersistentRNNPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetPersistentRNNPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyPersistentRNNPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetRNNDescriptor_v6(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+
+// Tensor reductions
+cudnnStatus_t cudnnCreateReduceTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetReduceTensorDescriptor(...){
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetReduceTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnDestroyReduceTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetReductionIndicesSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetReductionWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnReduceTensor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+#endif // #if defined(CUPY_NO_CUDA) || (CUDNN_VERSION < 6000)
+
+
+#if !defined(CUPY_NO_CUDA)
+// Some functions are renamed in cuDNN v5.
+// Following definitions are for compatibility with cuDNN v5 and higher.
+
+#define cudnnAddTensor_v3 cudnnAddTensor
+#define cudnnConvolutionBackwardData_v3 cudnnConvolutionBackwardData
+#define cudnnConvolutionBackwardFilter_v3 cudnnConvolutionBackwardFilter
+#define cudnnSetConvolutionNdDescriptor_v3 cudnnSetConvolutionNdDescriptor
+
+#endif // #if !defined(CUPY_NO_CUDA)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 6000)
+
+#define cudnnSetConvolution2dDescriptor_v4 cudnnSetConvolution2dDescriptor
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 6000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 6000)
+// Some functions are renamed in cuDNN v6.
+// Following definitions are for compatibility with cuDNN v6 and higher.
+
+#define cudnnSetFilter4dDescriptor_v4 cudnnSetFilter4dDescriptor
+#define cudnnSetFilterNdDescriptor_v4 cudnnSetFilterNdDescriptor
+#define cudnnGetFilterNdDescriptor_v4 cudnnGetFilterNdDescriptor
+#define cudnnSetPooling2dDescriptor_v4 cudnnSetPooling2dDescriptor
+#define cudnnSetPoolingNdDescriptor_v4 cudnnSetPoolingNdDescriptor
+#define cudnnActivationForward_v4 cudnnActivationForward
+#define cudnnActivationBackward_v4 cudnnActivationBackward
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 6000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+#define cudnnSetRNNDescriptor_v5 cudnnSetRNNDescriptor
+
+typedef enum {} cudnnMathType_t;
+typedef enum {} cudnnCTCLossAlgo_t;
+typedef void* cudnnCTCLossDescriptor_t;
+
+// CTC
+cudnnStatus_t cudnnCreateCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnDestroyCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnSetCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnGetCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnGetCTCLossWorkspaceSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnCTCLoss(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetConvolutionMathType(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionMathType(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetConvolutionGroupCount(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionGroupCount(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionForwardAlgorithm_v7(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardDataAlgorithm_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 7000)
+
+#define cudnnSetConvolution2dDescriptor_v5 cudnnSetConvolution2dDescriptor
+
+cudnnStatus_t cudnnSetConvolution2dDescriptor_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 7000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 6000)
+
+typedef enum {} cudnnDeterminism_t;
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 6000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+cudnnStatus_t cudnnFindConvolutionForwardAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnFindConvolutionBackwardFilterAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnFindConvolutionBackwardDataAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+typedef struct {
+  cudnnConvolutionFwdAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionFwdAlgoPerf_v7_t;
+
+typedef struct {
+  cudnnConvolutionBwdFilterAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionBwdFilterAlgoPerf_v7_t;
+
+typedef struct {
+  cudnnConvolutionBwdDataAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionBwdDataAlgoPerf_v7_t;
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 7000)
+
+#define cudnnFindConvolutionForwardAlgorithmEx_v7 cudnnFindConvolutionForwardAlgorithmEx
+#define cudnnFindConvolutionBackwardFilterAlgorithmEx_v7 cudnnFindConvolutionBackwardFilterAlgorithmEx
+#define cudnnFindConvolutionBackwardDataAlgorithmEx_v7 cudnnFindConvolutionBackwardDataAlgorithmEx
+
+#define cudnnConvolutionFwdAlgoPerf_v7_t cudnnConvolutionFwdAlgoPerf_t
+#define cudnnConvolutionBwdFilterAlgoPerf_v7_t cudnnConvolutionBwdFilterAlgoPerf_t
+#define cudnnConvolutionBwdDataAlgoPerf_v7_t cudnnConvolutionBwdDataAlgoPerf_t
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 7000)
+
+
+#if defined(CUPY_NO_CUDA) || (CUDNN_VERSION < 7200)
+
+typedef void* cudnnRNNDataDescriptor_t;
+
+typedef enum {} cudnnRNNDataLayout_t;
+typedef enum {} cudnnRNNPaddingMode_t;
+
+cudnnStatus_t cudnnSetRNNPaddingMode(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNPaddingMode(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateRNNDataDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyRNNDataDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetRNNDataDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNDataDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNForwardInferenceEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNForwardTrainingEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNBackwardDataEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNBackwardWeightsEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+#endif // defined(CUPY_NO_CUDA) || (CUDNN_VERSION < 7200)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 8000)
+// TODO: check function names when cuDNN 8 is released.
+
+#define cudnnGetConvolutionForwardAlgorithm_v6 cudnnGetConvolutionForwardAlgorithm
+#define cudnnGetConvolutionBackwardFilterAlgorithm_v6 cudnnGetConvolutionBackwardFilterAlgorithm
+#define cudnnGetConvolutionBackwardDataAlgorithm_v6 cudnnGetConvolutionBackwardDataAlgorithm
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 8000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+typedef enum {} cudnnErrQueryMode_t;
+typedef struct cudnnRuntimeTag_t cudnnRuntimeTag_t;
+
+cudnnStatus_t cudnnQueryRuntimeError(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+#endif // !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7400)
+
+typedef enum {} cudnnBatchNormOps_t;
+
+cudnnStatus_t cudnnBatchNormalizationForwardTrainingEx(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnBatchNormalizationBackwardEx(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationBackwardExWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationTrainingExReserveSpaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+#endif // !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7400)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7600)
+
+// fused ops
+typedef void* cudnnFusedOpsConstParamPack_t;
+typedef void* cudnnFusedOpsVariantParamPack_t;
+typedef void* cudnnFusedOpsPlan_t;
+
+typedef enum {} cudnnFusedOps_t;
+typedef enum {} cudnnFusedOpsConstParamLabel_t;
+typedef enum {} cudnnFusedOpsPointerPlaceHolder_t;
+typedef enum {} cudnnFusedOpsVariantParamLabel_t;
+
+cudnnStatus_t cudnnCreateFusedOpsConstParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsConstParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetFusedOpsConstParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetFusedOpsConstParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateFusedOpsVariantParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsVariantParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetFusedOpsVariantParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetFusedOpsVariantParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnMakeFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFusedOpsExecute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+#endif // !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7600)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 8000)
+
+typedef enum {} cudnnConvolutionFwdPreference_t;
+typedef enum {} cudnnConvolutionBwdFilterPreference_t;
+typedef enum {} cudnnConvolutionBwdDataPreference_t;
+
+cudnnStatus_t cudnnGetConvolutionForwardAlgorithm_v6(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm_v6(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardDataAlgorithm_v6(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetRNNDescriptor_v5(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+#endif // !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 8000)
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_CUDA_CUPY_CUDNN_H
diff --git a/cupy_backends/cuda/cupy_cusolver.h b/cupy_backends/cuda/cupy_cusolver.h
new file mode 100644
index 0000000..2c255b4
--- /dev/null
+++ b/cupy_backends/cuda/cupy_cusolver.h
@@ -0,0 +1,245 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_CUSOLVER_H
+#define INCLUDE_GUARD_CUDA_CUPY_CUSOLVER_H
+
+#include <cuda.h>
+#include <cusolverDn.h>
+#include <cusolverSp.h>
+
+extern "C" {
+
+#if CUDA_VERSION < 10010
+// Functions added in CUDA 10.1
+cusolverStatus_t cusolverDnSgesvdaStridedBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvdaStridedBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvdaStridedBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvdaStridedBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgesvdaStridedBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvdaStridedBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvdaStridedBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvdaStridedBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+#endif // #if CUDA_VERSION < 10010
+
+#if CUDA_VERSION < 10020
+// Functions added in CUDA 10.2
+cusolverStatus_t cusolverDnZZgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZCgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZKgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCCgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCKgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDDgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDSgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDHgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSSgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSHgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZZgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZCgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZKgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCCgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCKgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDDgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDSgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDHgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSSgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSHgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+#endif // #if CUDA_VERSION < 10020
+
+#if CUDA_VERSION < 11000
+
+typedef enum cusolverDnParams_t {};
+
+// Functions added in CUDA 11.0
+cusolverStatus_t cusolverDnCreateParams(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDestroyParams(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZYgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCYgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDXgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSXgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZYgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCYgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDXgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSXgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZZgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZCgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZYgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZKgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCCgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCYgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCKgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDDgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDSgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDXgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDHgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSSgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSXgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSHgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZZgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZCgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZYgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZKgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCCgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCYgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCKgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDDgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDSgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDXgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDHgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSSgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSXgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSHgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+#endif // #if CUDA_VERSION < 11000
+
+#if CUDA_VERSION < 11010
+// Functions added in CUDA 11.1
+cusolverStatus_t cusolverDnXsyevd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXsyevd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+#endif // #if CUDA_VERSION < 11010
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_CUDA_CUPY_CUSOLVER_H
diff --git a/cupy_backends/cuda/cupy_cusparse.h b/cupy_backends/cuda/cupy_cusparse.h
new file mode 100644
index 0000000..980112f
--- /dev/null
+++ b/cupy_backends/cuda/cupy_cusparse.h
@@ -0,0 +1,910 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_CUSPARSE_H
+#define INCLUDE_GUARD_CUDA_CUPY_CUSPARSE_H
+
+#include <cuda.h>
+#include <cusparse.h>
+
+#if !defined(CUSPARSE_VERSION)
+  // CUSPARSE_VERSION introduced in CUDA 10.1 Update 2 (10.1.243).
+  #if CUDA_VERSION < 10000
+    #define CUSPARSE_VERSION CUDA_VERSION // CUDA_VERSION used instead
+  #elif CUDA_VERSION < 10010  // CUDA 10.0.x
+    # define CUSPARSE_VERSION 10000
+  #elif CUDA_VERSION < 10020  // CUDA 10.1.x
+    // CUSPARSE_VER_MAJOR introduced in CUDA 10.1 Update 1 (10.1.168).
+    #if !defined(CUSPARSE_VER_MAJOR)
+      // CUDA 10.1 (10.1.105) contains cuSPARSE 10.1
+      #define CUSPARSE_VERSION 10010
+    #else
+      // CUDA 10.1 Update 1 (10.1.168) contains cuSPARSE 10.2.0.0
+      #define CUSPARSE_VERSION (CUSPARSE_VER_MAJOR * 1000 + \
+                                CUSPARSE_VER_MINOR *  100 + \
+                                CUSPARSE_VER_PATCH)
+    #endif
+  #endif
+#endif  // #if !defined(CUSPARSE_VERSION)
+
+/*
+ * Generic APIs are not suppoted in CUDA 10.1/10.2 on Windows.
+ * These APIs are available in headers in CUDA 10.1 and 10.1 Update 1,
+ * but hidden in 10.1 Update 2 and 10.2 on Windows.
+ */
+
+#if defined(_WIN32) && (CUSPARSE_VERSION < 11000)
+  #if 10200 < CUSPARSE_VERSION
+    #define WIN32_EXPOSE_SPMM_STUB_DECL 1
+  #else
+    #define WIN32_EXPOSE_SPMM_STUB_DECL 0
+  #endif
+  #define WIN32_EXPOSE_SPMM_STUB_IMPL 1
+#else
+  #define WIN32_EXPOSE_SPMM_STUB_DECL 0
+  #define WIN32_EXPOSE_SPMM_STUB_IMPL 0
+#endif
+
+#if CUSPARSE_VERSION < 10010
+// Added in cuSPARSE 10.1 (CUDA 10.1.105)
+
+// CSR2CSC
+typedef enum {} cusparseCsr2CscAlg_t;
+
+#endif  // #if CUSPARSE_VERSION < 10010
+
+#if CUSPARSE_VERSION < 10010 || WIN32_EXPOSE_SPMM_STUB_DECL
+// Generic API types added in cuSPARSE 10.1 (CUDA 10.1.105)
+
+typedef void* cusparseSpMatDescr_t;
+typedef void* cusparseDnMatDescr_t;
+typedef enum {} cusparseIndexType_t;
+typedef enum {} cusparseFormat_t;
+typedef enum {} cusparseOrder_t;
+typedef enum {} cusparseSpMMAlg_t;
+
+#endif  // #if CUSPARSE_VERSION < 10010 || WIN32_EXPOSE_SPMM_STUB_DECL
+
+#if CUSPARSE_VERSION < 10200
+// Added in cuSPARSE 10.2 (CUDA 10.1.168)
+
+// CSR2CSC
+cusparseStatus_t cusparseCsr2cscEx2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsr2cscEx2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+#endif  // #if CUSPARSE_VERSION < 10200
+
+#if CUSPARSE_VERSION < 10200 || WIN32_EXPOSE_SPMM_STUB_DECL
+// Generic API types added in cuSPARSE 10.2 (CUDA 10.1.168)
+
+typedef void* cusparseSpVecDescr_t;
+typedef void* cusparseDnVecDescr_t;
+typedef enum {} cusparseSpMVAlg_t;
+
+#endif  // #if CUSPARSE_VERSION < 10200 || WIN32_EXPOSE_SPMM_STUB_DECL
+
+#if CUSPARSE_VERSION < 10200 || WIN32_EXPOSE_SPMM_STUB_IMPL
+// Generic APIs added in cuSPARSE 10.2 (CUDA 10.1.168)
+
+/*
+ * On Windows, implementations are not exposed from DLL in CUDA 10.1/10.2
+ * although it is declared in the header. So we have to provide a stub
+ * implementation using the full signature to match with the signature
+ * declared in cusparse.h, instead of using (...).
+ */
+
+cusparseStatus_t cusparseCreateSpVec(
+    cusparseSpVecDescr_t* spVecDescr,
+    int64_t               size,
+    int64_t               nnz,
+    void*                 indices,
+    void*                 values,
+    cusparseIndexType_t   idxType,
+    cusparseIndexBase_t   idxBase,
+    cudaDataType          valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroySpVec(cusparseSpVecDescr_t spVecDescr) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVecGet(
+    const cusparseSpVecDescr_t spVecDescr,
+    int64_t*                   size,
+    int64_t*                   nnz,
+    void**                     indices,
+    void**                     values,
+    cusparseIndexType_t*       idxType,
+    cusparseIndexBase_t*       idxBase,
+    cudaDataType*              valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVecGetIndexBase(
+    const cusparseSpVecDescr_t spVecDescr,
+    cusparseIndexBase_t*       idxBase) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVecGetValues(
+    const cusparseSpVecDescr_t spVecDescr,
+    void**                     values) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVecSetValues(
+    cusparseSpVecDescr_t spVecDescr,
+    void*                values) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCoo(
+    cusparseSpMatDescr_t* spMatDescr,
+    int64_t               rows,
+    int64_t               cols,
+    int64_t               nnz,
+    void*                 cooRowInd,
+    void*                 cooColInd,
+    void*                 cooValues,
+    cusparseIndexType_t   cooIdxType,
+    cusparseIndexBase_t   idxBase,
+    cudaDataType          valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCooAoS(
+    cusparseSpMatDescr_t* spMatDescr,
+    int64_t               rows,
+    int64_t               cols,
+    int64_t               nnz,
+    void*                 cooInd,
+    void*                 cooValues,
+    cusparseIndexType_t   cooIdxType,
+    cusparseIndexBase_t   idxBase,
+    cudaDataType          valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+
+cusparseStatus_t cusparseCreateCsr(
+    cusparseSpMatDescr_t* spMatDescr,
+    int64_t               rows,
+    int64_t               cols,
+    int64_t               nnz,
+    void*                 csrRowOffsets,
+    void*                 csrColInd,
+    void*                 csrValues,
+    cusparseIndexType_t   csrRowOffsetsType,
+    cusparseIndexType_t   csrColIndType,
+    cusparseIndexBase_t   idxBase,
+    cudaDataType          valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroySpMat(cusparseSpMatDescr_t spMatDescr) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCooGet(
+    const cusparseSpMatDescr_t spMatDescr,
+    int64_t*                   rows,
+    int64_t*                   cols,
+    int64_t*                   nnz,
+    void**                     cooRowInd,  // COO row indices
+    void**                     cooColInd,  // COO column indices
+    void**                     cooValues,  // COO values
+    cusparseIndexType_t*       idxType,
+    cusparseIndexBase_t*       idxBase,
+    cudaDataType*              valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCooAoSGet(
+    const cusparseSpMatDescr_t spMatDescr,
+    int64_t*                   rows,
+    int64_t*                   cols,
+    int64_t*                   nnz,
+    void**                     cooInd,     // COO indices
+    void**                     cooValues,  // COO values
+    cusparseIndexType_t*       idxType,
+    cusparseIndexBase_t*       idxBase,
+    cudaDataType*              valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsrGet(
+    const cusparseSpMatDescr_t spMatDescr,
+    int64_t*                   rows,
+    int64_t*                   cols,
+    int64_t*                   nnz,
+    void**                     csrRowOffsets,
+    void**                     csrColInd,
+    void**                     csrValues,
+    cusparseIndexType_t*       csrRowOffsetsType,
+    cusparseIndexType_t*       csrColIndType,
+    cusparseIndexBase_t*       idxBase,
+    cudaDataType*              valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetFormat(
+    const cusparseSpMatDescr_t spMatDescr,
+    cusparseFormat_t*          format) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetIndexBase(
+    const cusparseSpMatDescr_t spMatDescr,
+    cusparseIndexBase_t*       idxBase) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetValues(
+    const cusparseSpMatDescr_t spMatDescr,
+    void**                     values) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatSetValues(
+    cusparseSpMatDescr_t spMatDescr,
+    void*                values) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetStridedBatch(
+    const cusparseSpMatDescr_t spMatDescr,
+    int*                       batchCount) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatSetStridedBatch(
+    cusparseSpMatDescr_t spMatDescr,
+    int                  batchCount) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateDnVec(
+    cusparseDnVecDescr_t* dnVecDescr,
+    int64_t               size,
+    void*                 values,
+    cudaDataType          valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyDnVec(cusparseDnVecDescr_t dnVecDescr) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnVecGet(
+    const cusparseDnVecDescr_t dnVecDescr,
+    int64_t*                   size,
+    void**                     values,
+    cudaDataType*              valueType) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnVecGetValues(
+    const cusparseDnVecDescr_t dnVecDescr,
+    void**                     values) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnVecSetValues(
+    cusparseDnVecDescr_t dnVecDescr,
+    void*                values) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateDnMat(
+    cusparseDnMatDescr_t* dnMatDescr,
+    int64_t               rows,
+    int64_t               cols,
+    int64_t               ld,
+    void*                 values,
+    cudaDataType          valueType,
+    cusparseOrder_t       order) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyDnMat(cusparseDnMatDescr_t dnMatDescr) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatGet(
+    const cusparseDnMatDescr_t dnMatDescr,
+    int64_t*                   rows,
+    int64_t*                   cols,
+    int64_t*                   ld,
+    void**                     values,
+    cudaDataType*              type,
+    cusparseOrder_t*           order) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatGetValues(
+    const cusparseDnMatDescr_t dnMatDescr,
+    void**                     values) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatSetValues(
+    cusparseDnMatDescr_t dnMatDescr,
+    void*                values) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatGetStridedBatch(
+    const cusparseDnMatDescr_t dnMatDescr,
+    int*                       batchCount,
+    int64_t*                   batchStride) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatSetStridedBatch(
+    cusparseDnMatDescr_t dnMatDescr,
+    int                  batchCount,
+    int64_t              batchStride) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVV_bufferSize(
+    cusparseHandle_t           handle,
+    cusparseOperation_t        opX,
+    const cusparseSpVecDescr_t vecX,
+    const cusparseDnVecDescr_t vecY,
+    const void*                result,
+    cudaDataType               computeType,
+    size_t*                    bufferSize) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVV(
+    cusparseHandle_t           handle,
+    cusparseOperation_t        opX,
+    const cusparseSpVecDescr_t vecX,
+    const cusparseDnVecDescr_t vecY,
+    void*                      result,
+    cudaDataType               computeType,
+    void*                      externalBuffer) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMV_bufferSize(
+    cusparseHandle_t           handle,
+    cusparseOperation_t        opA,
+    const void*                alpha,
+    const cusparseSpMatDescr_t matA,
+    const cusparseDnVecDescr_t vecX,
+    const void*                beta,
+    const cusparseDnVecDescr_t vecY,
+    cudaDataType               computeType,
+    cusparseSpMVAlg_t          alg,
+    size_t*                    bufferSize) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMV(
+    cusparseHandle_t           handle,
+    cusparseOperation_t        opA,
+    const void*                alpha,
+    const cusparseSpMatDescr_t matA,
+    const cusparseDnVecDescr_t vecX,
+    const void*                beta,
+    const cusparseDnVecDescr_t vecY,
+    cudaDataType               computeType,
+    cusparseSpMVAlg_t          alg,
+    void*                      externalBuffer) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMM_bufferSize(
+    cusparseHandle_t           handle,
+    cusparseOperation_t        opA,
+    cusparseOperation_t        opB,
+    const void*                alpha,
+    const cusparseSpMatDescr_t matA,
+    const cusparseDnMatDescr_t matB,
+    const void*                beta,
+    cusparseDnMatDescr_t       matC,
+    cudaDataType               computeType,
+    cusparseSpMMAlg_t          alg,
+    size_t*                    bufferSize) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMM(
+    cusparseHandle_t           handle,
+    cusparseOperation_t        opA,
+    cusparseOperation_t        opB,
+    const void*                alpha,
+    const cusparseSpMatDescr_t matA,
+    const cusparseDnMatDescr_t matB,
+    const void*                beta,
+    cusparseDnMatDescr_t       matC,
+    cudaDataType               computeType,
+    cusparseSpMMAlg_t          alg,
+    void*                      externalBuffer) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+#endif  // #if CUSPARSE_VERSION < 10200 || WIN32_EXPOSE_SPMM_STUB_IMPL
+
+#if CUSPARSE_VERSION < 10300 || WIN32_EXPOSE_SPMM_STUB_IMPL
+// Generic APIs added in cuSPARSE 10.3 (CUDA 10.1.243)
+
+cusparseStatus_t cusparseConstrainedGeMM_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseConstrainedGeMM(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+#endif  // #if CUSPARSE_VERSION < 10300 || WIN32_EXPOSE_SPMM_STUB_IMPL
+
+#if CUSPARSE_VERSION >= 11000
+// Functions deleted in cuSparse 11.0
+
+// cuSPARSE Level2 Function
+cusparseStatus_t cusparseScsrmv(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrmv(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrmv(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrmv(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE Level3 Function
+cusparseStatus_t cusparseScsrmm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrmm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrmm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrmm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrmm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrmm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrmm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrmm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE Extra Function
+cusparseStatus_t cusparseXcsrgeamNnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgeam(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgeam(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgeam(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgeam(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+
+cusparseStatus_t cusparseXcsrgemmNnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgemm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgemm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgemm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgemm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE Format Convrsion
+cusparseStatus_t cusparseXcsr2coo(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsr2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsr2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsr2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsr2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+#endif // #if CUSPARSE_VERSION >= 11000
+
+#if CUSPARSE_VERSION < 11100
+// Functions added in cuSparse 11.1 (CUDA 11.0)
+
+cusparseStatus_t cusparseCsrSetPointers(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+typedef void* cusparseSpGEMMDescr_t;
+typedef enum {} cusparseSpGEMMAlg_t;
+
+cusparseStatus_t cusparseSpGEMM_createDescr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpGEMM_destroyDescr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpGEMM_workEstimation(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpGEMM_compute(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpGEMM_copy(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseGather(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+#endif // #if CUSPARSE_VERSION < 11100
+
+#if CUSPARSE_VERSION < 11300
+// Types and macros added in cuSparse 11.3 (CUDA 11.1.1)
+
+typedef enum {} cusparseSparseToDenseAlg_t;
+typedef enum {} cusparseDenseToSparseAlg_t;
+
+#endif // CUSPARSE_VERSION < 11300
+
+#if CUSPARSE_VERSION < 11500
+// API added in cuSparse 11.5 (CUDA 11.3.0)
+
+typedef enum {} cusparseSpMatAttribute_t;
+
+#endif // CUSPARSE_VERSION < 11500
+
+#if CUSPARSE_VERSION < 11600
+// Generic APIs added in cuSPARSE 11.6 (CUDA 11.3.1)
+
+typedef void* cusparseSpSMDescr_t;
+typedef enum {} cusparseSpSMAlg_t;
+
+#endif // #if CUSPARSE_VERSION < 11600
+
+#if CUSPARSE_VERSION >= 12000
+// Types and functions deleted in cuSPARSE 12.0 (CUDA 12.0)
+
+typedef enum {} cusparseAlgMode_t;
+typedef void* csrgemm2Info_t;
+typedef void* csrsm2Info_t;
+typedef void* csrsv2Info_t;
+
+cusparseStatus_t cusparseCreateCsrsv2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyCsrsv2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCsrsm2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyCsrsm2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCsrgemm2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyCsrgemm2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgthr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgthr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgthr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgthr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsrmvEx_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsrmvEx(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsv2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsv2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsv2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsv2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsv2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsv2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsv2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsv2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsv2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsv2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsv2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsv2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrsv2_zeroPivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsm2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsm2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsm2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsm2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsm2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsm2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsm2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsm2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrsm2_zeroPivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgemm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgemm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgemm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgemm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrgemm2Nnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgemm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgemm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgemm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgemm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsc2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsc2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsc2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsc2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsr2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsr2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsr2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsr2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSdense2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDdense2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCdense2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZdense2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSdense2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDdense2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCdense2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZdense2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatSetStridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseConstrainedGeMM(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseConstrainedGeMM_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCooAoS(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCooAoSGet(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+#endif // #if CUSPARSE_VERSION >= 12000
+
+#endif  // INCLUDE_GUARD_CUDA_CUPY_CUSPARSE_H
diff --git a/cupy_backends/cuda/cupy_cutensor.h b/cupy_backends/cuda/cupy_cutensor.h
new file mode 100644
index 0000000..c05513b
--- /dev/null
+++ b/cupy_backends/cuda/cupy_cutensor.h
@@ -0,0 +1,19 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_CUTENSOR_H
+#define INCLUDE_GUARD_CUDA_CUPY_CUTENSOR_H
+
+#include <library_types.h>
+#include <cutensor.h>
+
+#if CUTENSOR_VERSION < 10500
+
+cutensorStatus_t cutensorContractionGetWorkspaceSize(...) {
+    return CUTENSOR_STATUS_NOT_SUPPORTED;
+}
+
+cutensorStatus_t cutensorReductionGetWorkspaceSize(...) {
+    return CUTENSOR_STATUS_NOT_SUPPORTED;
+}
+
+#endif  // CUTENSOR_VERSION < 10500
+
+#endif  // INCLUDE_GUARD_CUDA_CUPY_CUTENSOR_H
diff --git a/cupy_backends/cuda/cupy_nccl.h b/cupy_backends/cuda/cupy_nccl.h
new file mode 100644
index 0000000..83aa060
--- /dev/null
+++ b/cupy_backends/cuda/cupy_nccl.h
@@ -0,0 +1,12 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_NCCL_H
+#define INCLUDE_GUARD_CUDA_CUPY_NCCL_H
+
+#include <nccl.h>
+
+#ifndef NCCL_MAJOR
+#ifndef CUDA_HAS_HALF
+#define ncclHalf ((ncclDataType_t)2)
+#endif
+#endif
+
+#endif
diff --git a/cupy_backends/cuda/cupy_nvrtc.h b/cupy_backends/cuda/cupy_nvrtc.h
new file mode 100644
index 0000000..f9d3f27
--- /dev/null
+++ b/cupy_backends/cuda/cupy_nvrtc.h
@@ -0,0 +1,22 @@
+#ifndef INCLUDE_GUARD_CUDA_CUPY_NVRTC_H
+#define INCLUDE_GUARD_CUDA_CUPY_NVRTC_H
+
+#include <cuda.h>  // for CUDA_VERSION
+#include <nvrtc.h>
+
+extern "C" {
+
+#if CUDA_VERSION < 11010
+// functions added in CUDA 11.1
+nvrtcResult nvrtcGetCUBINSize(...) {
+    return NVRTC_ERROR_COMPILATION;
+}
+
+nvrtcResult nvrtcGetCUBIN(...) {
+    return NVRTC_ERROR_COMPILATION;
+}
+#endif
+
+}
+
+#endif // #ifndef INCLUDE_GUARD_CUDA_CUPY_NVRTC_H
diff --git a/cupy_backends/cuda/libs/__init__.pxd b/cupy_backends/cuda/libs/__init__.pxd
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/cuda/libs/__init__.py b/cupy_backends/cuda/libs/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cupy_backends/cuda/libs/cublas.pxd b/cupy_backends/cuda/libs/cublas.pxd
new file mode 100644
index 0000000..987b1a9
--- /dev/null
+++ b/cupy_backends/cuda/libs/cublas.pxd
@@ -0,0 +1,332 @@
+"""Thin wrapper of CUBLAS."""
+from libc.stdint cimport intptr_t
+
+
+###############################################################################
+# Types
+###############################################################################
+
+cdef extern from *:
+    ctypedef void* cuComplexPtr 'cuComplex*'
+    ctypedef void* cuDoubleComplexPtr 'cuDoubleComplex*'
+
+
+cdef extern from *:
+    ctypedef void* Handle 'cublasHandle_t'
+
+    ctypedef int DiagType 'cublasDiagType_t'
+    ctypedef int FillMode 'cublasFillMode_t'
+    ctypedef int Operation 'cublasOperation_t'
+    ctypedef int PointerMode 'cublasPointerMode_t'
+    ctypedef int SideMode 'cublasSideMode_t'
+    ctypedef int GemmAlgo 'cublasGemmAlgo_t'
+    ctypedef int Math 'cublasMath_t'
+    ctypedef int ComputeType 'cublasComputeType_t'
+
+
+###############################################################################
+# Enum
+###############################################################################
+
+cpdef enum:
+    CUBLAS_OP_N = 0
+    CUBLAS_OP_T = 1
+    CUBLAS_OP_C = 2
+
+    CUBLAS_POINTER_MODE_HOST = 0
+    CUBLAS_POINTER_MODE_DEVICE = 1
+
+    CUBLAS_SIDE_LEFT = 0
+    CUBLAS_SIDE_RIGHT = 1
+
+    CUBLAS_FILL_MODE_LOWER = 0
+    CUBLAS_FILL_MODE_UPPER = 1
+
+    CUBLAS_DIAG_NON_UNIT = 0
+    CUBLAS_DIAG_UNIT = 1
+
+    CUBLAS_GEMM_DEFAULT = -1
+    CUBLAS_GEMM_DEFAULT_TENSOR_OP = 99
+
+    # The following two are left for backward compatibility; renamed from
+    # `DFALT` to `DEFAULT` in CUDA 9.1.
+    CUBLAS_GEMM_DFALT = -1
+    CUBLAS_GEMM_DFALT_TENSOR_OP = 99
+
+    CUBLAS_DEFAULT_MATH = 0
+    CUBLAS_TENSOR_OP_MATH = 1
+
+    # cublasComputeType_t added in CUDA 11.0
+    CUBLAS_COMPUTE_16F = 64            # half - default
+    CUBLAS_COMPUTE_16F_PEDANTIC = 65   # half - pedantic
+    CUBLAS_COMPUTE_32F = 68            # float - default
+    CUBLAS_COMPUTE_32F_PEDANTIC = 69   # float - pedantic
+    CUBLAS_COMPUTE_32F_FAST_16F = 74   # float - fast, allows down-converting inputs to half or TF32  # NOQA
+    CUBLAS_COMPUTE_32F_FAST_16BF = 75  # float - fast, allows down-converting inputs to bfloat16 or TF32  # NOQA
+    CUBLAS_COMPUTE_32F_FAST_TF32 = 77  # float - fast, allows down-converting inputs to TF32  # NOQA
+    CUBLAS_COMPUTE_64F = 70            # double - default
+    CUBLAS_COMPUTE_64F_PEDANTIC = 71   # double - pedantic
+    CUBLAS_COMPUTE_32I = 72            # signed 32-bit int - default
+    CUBLAS_COMPUTE_32I_PEDANTIC = 73   # signed 32-bit int - pedantic
+
+###############################################################################
+# Context
+###############################################################################
+
+cpdef intptr_t create() except? 0
+cpdef destroy(intptr_t handle)
+cpdef int getVersion(intptr_t handle) except? -1
+cpdef int getPointerMode(intptr_t handle) except? -1
+cpdef setPointerMode(intptr_t handle, int mode)
+
+
+###############################################################################
+# Stream
+###############################################################################
+
+cpdef setStream(intptr_t handle, size_t stream)
+cpdef size_t getStream(intptr_t handle) except? 0
+
+
+###############################################################################
+# Math Mode
+###############################################################################
+
+cpdef setMathMode(intptr_t handle, int mode)
+cpdef int getMathMode(intptr_t handle) except? -1
+
+
+###############################################################################
+# BLAS Level 1
+###############################################################################
+
+cpdef isamax(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef idamax(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef icamax(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef izamax(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef isamin(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef idamin(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef icamin(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef izamin(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef sasum(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef dasum(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef scasum(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef dzasum(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef saxpy(intptr_t handle, int n, size_t alpha, size_t x, int incx, size_t y,
+            int incy)
+cpdef daxpy(intptr_t handle, int n, size_t alpha, size_t x, int incx, size_t y,
+            int incy)
+cpdef caxpy(intptr_t handle, int n, size_t alpha, size_t x, int incx, size_t y,
+            int incy)
+cpdef zaxpy(intptr_t handle, int n, size_t alpha, size_t x, int incx, size_t y,
+            int incy)
+cpdef sdot(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+           size_t result)
+cpdef ddot(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+           size_t result)
+cpdef cdotu(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+            size_t result)
+cpdef cdotc(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+            size_t result)
+cpdef zdotu(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+            size_t result)
+cpdef zdotc(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+            size_t result)
+cpdef snrm2(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef dnrm2(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef scnrm2(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef dznrm2(intptr_t handle, int n, size_t x, int incx, size_t result)
+cpdef sscal(intptr_t handle, int n, size_t alpha, size_t x, int incx)
+cpdef dscal(intptr_t handle, int n, size_t alpha, size_t x, int incx)
+cpdef cscal(intptr_t handle, int n, size_t alpha, size_t x, int incx)
+cpdef csscal(intptr_t handle, int n, size_t alpha, size_t x, int incx)
+cpdef zscal(intptr_t handle, int n, size_t alpha, size_t x, int incx)
+cpdef zdscal(intptr_t handle, int n, size_t alpha, size_t x, int incx)
+
+
+###############################################################################
+# BLAS Level 2
+###############################################################################
+
+cpdef sgemv(intptr_t handle, int trans, int m, int n, size_t alpha, size_t A,
+            int lda, size_t x, int incx, size_t beta, size_t y, int incy)
+cpdef dgemv(intptr_t handle, int trans, int m, int n, size_t alpha, size_t A,
+            int lda, size_t x, int incx, size_t beta, size_t y, int incy)
+cpdef cgemv(intptr_t handle, int trans, int m, int n, size_t alpha, size_t A,
+            int lda, size_t x, int incx, size_t beta, size_t y, int incy)
+cpdef zgemv(intptr_t handle, int trans, int m, int n, size_t alpha, size_t A,
+            int lda, size_t x, int incx, size_t beta, size_t y, int incy)
+cpdef sger(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+           size_t y, int incy, size_t A, int lda)
+cpdef dger(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+           size_t y, int incy, size_t A, int lda)
+cpdef cgeru(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+            size_t y, int incy, size_t A, int lda)
+cpdef cgerc(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+            size_t y, int incy, size_t A, int lda)
+cpdef zgeru(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+            size_t y, int incy, size_t A, int lda)
+cpdef zgerc(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+            size_t y, int incy, size_t A, int lda)
+cpdef ssbmv(intptr_t handle, int uplo, int n, int k,
+            size_t alpha, size_t A, int lda,
+            size_t x, int incx, size_t beta, size_t y, int incy)
+cpdef dsbmv(intptr_t handle, int uplo, int n, int k,
+            size_t alpha, size_t A, int lda,
+            size_t x, int incx, size_t beta, size_t y, int incy)
+
+
+###############################################################################
+# BLAS Level 3
+###############################################################################
+
+cpdef sgemm(intptr_t handle, int transa, int transb,
+            int m, int n, int k, size_t alpha, size_t A, int lda,
+            size_t B, int ldb, size_t beta, size_t C, int ldc)
+cpdef dgemm(intptr_t handle, int transa, int transb,
+            int m, int n, int k, size_t alpha, size_t A, int lda,
+            size_t B, int ldb, size_t beta, size_t C, int ldc)
+cpdef cgemm(intptr_t handle, int transa, int transb,
+            int m, int n, int k, size_t alpha, size_t A, int lda,
+            size_t B, int ldb, size_t beta, size_t C, int ldc)
+cpdef zgemm(intptr_t handle, int transa, int transb,
+            int m, int n, int k, size_t alpha, size_t A, int lda,
+            size_t B, int ldb, size_t beta, size_t C, int ldc)
+cpdef sgemmBatched(intptr_t handle, int transa, int transb,
+                   int m, int n, int k, size_t alpha, size_t Aarray, int lda,
+                   size_t Barray, int ldb, size_t beta, size_t Carray, int ldc,
+                   int batchCount)
+cpdef dgemmBatched(intptr_t handle, int transa, int transb,
+                   int m, int n, int k, size_t alpha, size_t Aarray, int lda,
+                   size_t Barray, int ldb, size_t beta, size_t Carray, int ldc,
+                   int batchCount)
+cpdef cgemmBatched(intptr_t handle, int transa, int transb,
+                   int m, int n, int k, size_t alpha, size_t Aarray, int lda,
+                   size_t Barray, int ldb, size_t beta, size_t Carray, int ldc,
+                   int batchCount)
+cpdef zgemmBatched(intptr_t handle, int transa, int transb,
+                   int m, int n, int k, size_t alpha, size_t Aarray, int lda,
+                   size_t Barray, int ldb, size_t beta, size_t Carray, int ldc,
+                   int batchCount)
+cpdef sgemmStridedBatched(intptr_t handle, int transa, int transb, int m,
+                          int n, int k, size_t alpha, size_t A, int lda,
+                          long long strideA, size_t B, int ldb,
+                          long long strideB, size_t beta, size_t C, int ldc,
+                          long long strideC, int batchCount)
+cpdef dgemmStridedBatched(intptr_t handle, int transa, int transb, int m,
+                          int n, int k, size_t alpha, size_t A, int lda,
+                          long long strideA, size_t B, int ldb,
+                          long long strideB, size_t beta, size_t C, int ldc,
+                          long long strideC, int batchCount)
+cpdef cgemmStridedBatched(intptr_t handle, int transa, int transb, int m,
+                          int n, int k, size_t alpha, size_t A, int lda,
+                          long long strideA, size_t B, int ldb,
+                          long long strideB, size_t beta, size_t C, int ldc,
+                          long long strideC, int batchCount)
+cpdef zgemmStridedBatched(intptr_t handle, int transa, int transb, int m,
+                          int n, int k, size_t alpha, size_t A, int lda,
+                          long long strideA, size_t B, int ldb,
+                          long long strideB, size_t beta, size_t C, int ldc,
+                          long long strideC, int batchCount)
+cpdef strsm(intptr_t handle, int side, int uplo, int trans, int diag,
+            int m, int n, size_t alpha, size_t Aarray, int lda,
+            size_t Barray, int ldb)
+cpdef dtrsm(intptr_t handle, int side, int uplo, int trans, int diag,
+            int m, int n, size_t alpha, size_t Aarray, int lda,
+            size_t Barray, int ldb)
+cpdef ctrsm(intptr_t handle, int side, int uplo, int trans, int diag,
+            int m, int n, size_t alpha, size_t Aarray, int lda,
+            size_t Barray, int ldb)
+cpdef ztrsm(intptr_t handle, int side, int uplo, int trans, int diag,
+            int m, int n, size_t alpha, size_t Aarray, int lda,
+            size_t Barray, int ldb)
+cpdef ssyrk(intptr_t handle, int uplo, int trans, int n, int k,
+            size_t alpha, size_t A, int lda, size_t beta, size_t C, int ldc)
+cpdef dsyrk(intptr_t handle, int uplo, int trans, int n, int k,
+            size_t alpha, size_t A, int lda, size_t beta, size_t C, int ldc)
+cpdef csyrk(intptr_t handle, int uplo, int trans, int n, int k,
+            size_t alpha, size_t A, int lda, size_t beta, size_t C, int ldc)
+cpdef zsyrk(intptr_t handle, int uplo, int trans, int n, int k,
+            size_t alpha, size_t A, int lda, size_t beta, size_t C, int ldc)
+
+
+###############################################################################
+# BLAS extension
+###############################################################################
+
+cpdef sgeam(intptr_t handle, int transa, int transb, int m, int n,
+            size_t alpha, size_t A, int lda, size_t beta, size_t B, int ldb,
+            size_t C, int ldc)
+cpdef dgeam(intptr_t handle, int transa, int transb, int m, int n,
+            size_t alpha, size_t A, int lda, size_t beta, size_t B, int ldb,
+            size_t C, int ldc)
+cpdef cgeam(intptr_t handle, int transa, int transb, int m, int n,
+            size_t alpha, size_t A, int lda, size_t beta, size_t B, int ldb,
+            size_t C, int ldc)
+cpdef zgeam(intptr_t handle, int transa, int transb, int m, int n,
+            size_t alpha, size_t A, int lda, size_t beta, size_t B, int ldb,
+            size_t C, int ldc)
+cpdef sdgmm(intptr_t handle, int mode, int m, int n, size_t A, int lda,
+            size_t x, int incx, size_t C, int ldc)
+cpdef ddgmm(intptr_t handle, int mode, int m, int n, size_t A, int lda,
+            size_t x, int incx, size_t C, int ldc)
+cpdef cdgmm(intptr_t handle, int mode, int m, int n, size_t A, int lda,
+            size_t x, int incx, size_t C, int ldc)
+cpdef zdgmm(intptr_t handle, int mode, int m, int n, size_t A, int lda,
+            size_t x, int incx, size_t C, int ldc)
+cpdef sgemmEx(intptr_t handle, int transa, int transb, int m, int n, int k,
+              size_t alpha, size_t A, int Atype, int lda, size_t B,
+              int Btype, int ldb, size_t beta, size_t C, int Ctype,
+              int ldc)
+cpdef sgetrfBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t infoArray, int batchSize)
+cpdef dgetrfBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t infoArray, int batchSize)
+cpdef cgetrfBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t infoArray, int batchSize)
+cpdef zgetrfBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t infoArray, int batchSize)
+
+cpdef int sgetrsBatched(intptr_t handle, int trans, int n, int nrhs,
+                        size_t Aarray, int lda, size_t devIpiv,
+                        size_t Barray, int ldb, size_t info, int batchSize)
+cpdef int dgetrsBatched(intptr_t handle, int trans, int n, int nrhs,
+                        size_t Aarray, int lda, size_t devIpiv,
+                        size_t Barray, int ldb, size_t info, int batchSize)
+cpdef int cgetrsBatched(intptr_t handle, int trans, int n, int nrhs,
+                        size_t Aarray, int lda, size_t devIpiv,
+                        size_t Barray, int ldb, size_t info, int batchSize)
+cpdef int zgetrsBatched(intptr_t handle, int trans, int n, int nrhs,
+                        size_t Aarray, int lda, size_t devIpiv,
+                        size_t Barray, int ldb, size_t info, int batchSize)
+
+cpdef sgetriBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t Carray, int ldc,
+                    size_t infoArray, int batchSize)
+cpdef dgetriBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t Carray, int ldc,
+                    size_t infoArray, int batchSize)
+cpdef cgetriBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t Carray, int ldc,
+                    size_t infoArray, int batchSize)
+cpdef zgetriBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t Carray, int ldc,
+                    size_t infoArray, int batchSize)
+cpdef gemmEx(intptr_t handle, int transa, int transb, int m, int n, int k,
+             size_t alpha, size_t A, int Atype, int lda, size_t B,
+             int Btype, int ldb, size_t beta, size_t C, int Ctype,
+             int ldc, int computeType, int algo)
+cpdef gemmStridedBatchedEx(
+    intptr_t handle, int transa, int transb, int m, int n, int k,
+    size_t alpha,
+    size_t A, int Atype, int lda, long long strideA,
+    size_t B, int Btype, int ldb, long long strideB,
+    size_t beta,
+    size_t C, int Ctype, int ldc, long long strideC,
+    int batchCount, int computeType, int algo)
+
+cpdef stpttr(intptr_t handle, int uplo, int n, size_t AP, size_t A, int lda)
+cpdef dtpttr(intptr_t handle, int uplo, int n, size_t AP, size_t A, int lda)
+
+cpdef strttp(intptr_t handle, int uplo, int n, size_t A, int lda, size_t AP)
+cpdef dtrttp(intptr_t handle, int uplo, int n, size_t A, int lda, size_t AP)
diff --git a/cupy_backends/cuda/libs/cublas.pyx b/cupy_backends/cuda/libs/cublas.pyx
new file mode 100644
index 0000000..6952b56
--- /dev/null
+++ b/cupy_backends/cuda/libs/cublas.pyx
@@ -0,0 +1,1514 @@
+# distutils: language = c++
+
+"""Thin wrapper of CUBLAS."""
+
+cimport cython  # NOQA
+
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda cimport stream as stream_module
+
+###############################################################################
+# Extern
+###############################################################################
+
+cdef extern from '../../cupy_complex.h':
+    ctypedef struct cuComplex 'cuComplex':
+        float x, y
+
+    ctypedef struct cuDoubleComplex 'cuDoubleComplex':
+        double x, y
+
+cdef extern from '../../cupy_blas.h' nogil:
+    ctypedef void* Stream 'cudaStream_t'
+    ctypedef int DataType 'cudaDataType'
+
+    # Context
+    int cublasCreate(Handle* handle)
+    int cublasDestroy(Handle handle)
+    int cublasGetVersion(Handle handle, int* version)
+    int cublasGetPointerMode(Handle handle, PointerMode* mode)
+    int cublasSetPointerMode(Handle handle, PointerMode mode)
+
+    # Stream
+    int cublasSetStream(Handle handle, Stream streamId)
+    int cublasGetStream(Handle handle, Stream* streamId)
+
+    # Math Mode
+    int cublasSetMathMode(Handle handle, Math mode)
+    int cublasGetMathMode(Handle handle, Math* mode)
+
+    # BLAS Level 1
+    int cublasIsamax(Handle handle, int n, float* x, int incx,
+                     int* result)
+    int cublasIdamax(Handle handle, int n, double* x, int incx,
+                     int* result)
+    int cublasIcamax(Handle handle, int n, cuComplex* x, int incx,
+                     int* result)
+    int cublasIzamax(Handle handle, int n, cuDoubleComplex* x, int incx,
+                     int* result)
+    int cublasIsamin(Handle handle, int n, float* x, int incx,
+                     int* result)
+    int cublasIdamin(Handle handle, int n, double* x, int incx,
+                     int* result)
+    int cublasIcamin(Handle handle, int n, cuComplex* x, int incx,
+                     int* result)
+    int cublasIzamin(Handle handle, int n, cuDoubleComplex* x, int incx,
+                     int* result)
+    int cublasSasum(Handle handle, int n, float* x, int incx,
+                    float* result)
+    int cublasDasum(Handle handle, int n, double* x, int incx,
+                    double* result)
+    int cublasScasum(Handle handle, int n, cuComplex* x, int incx,
+                     float* result)
+    int cublasDzasum(Handle handle, int n, cuDoubleComplex* x, int incx,
+                     double* result)
+    int cublasSaxpy(Handle handle, int n, float* alpha, float* x,
+                    int incx, float* y, int incy)
+    int cublasDaxpy(Handle handle, int n, double* alpha, double* x,
+                    int incx, double* y, int incy)
+    int cublasCaxpy(Handle handle, int n, cuComplex* alpha, cuComplex* x,
+                    int incx, cuComplex* y, int incy)
+    int cublasZaxpy(Handle handle, int n, cuDoubleComplex* alpha,
+                    cuDoubleComplex* x, int incx, cuDoubleComplex* y, int incy)
+    int cublasSdot(Handle handle, int n, float* x, int incx,
+                   float* y, int incy, float* result)
+    int cublasDdot(Handle handle, int n, double* x, int incx,
+                   double* y, int incy, double* result)
+    int cublasCdotu(Handle handle, int n, cuComplex* x, int incx,
+                    cuComplex* y, int incy, cuComplex* result)
+    int cublasCdotc(Handle handle, int n, cuComplex* x, int incx,
+                    cuComplex* y, int incy, cuComplex* result)
+    int cublasZdotu(Handle handle, int n, cuDoubleComplex* x, int incx,
+                    cuDoubleComplex* y, int incy,
+                    cuDoubleComplex* result)
+    int cublasZdotc(Handle handle, int n, cuDoubleComplex* x, int incx,
+                    cuDoubleComplex* y, int incy,
+                    cuDoubleComplex* result)
+    int cublasSnrm2(Handle handle, int n, float* x, int incx, float* result)
+    int cublasDnrm2(Handle handle, int n, double* x, int incx, double* result)
+    int cublasScnrm2(Handle handle, int n, cuComplex* x, int incx,
+                     float* result)
+    int cublasDznrm2(Handle handle, int n, cuDoubleComplex* x, int incx,
+                     double* result)
+    int cublasSscal(Handle handle, int n, float* alpha, float* x, int incx)
+    int cublasDscal(Handle handle, int n, double* alpha, double* x, int incx)
+    int cublasCscal(Handle handle, int n, cuComplex* alpha,
+                    cuComplex* x, int incx)
+    int cublasCsscal(Handle handle, int n, float* alpha,
+                     cuComplex* x, int incx)
+    int cublasZscal(Handle handle, int n, cuDoubleComplex* alpha,
+                    cuDoubleComplex* x, int incx)
+    int cublasZdscal(Handle handle, int n, double* alpha,
+                     cuDoubleComplex* x, int incx)
+
+    # BLAS Level 2
+    int cublasSgemv(
+        Handle handle, Operation trans, int m, int n, float* alpha,
+        float* A, int lda, float* x, int incx, float* beta,
+        float* y, int incy)
+    int cublasDgemv(
+        Handle handle, Operation trans, int m, int n, double* alpha,
+        double* A, int lda, double* x, int incx, double* beta,
+        double* y, int incy)
+    int cublasCgemv(
+        Handle handle, Operation trans, int m, int n, cuComplex* alpha,
+        cuComplex* A, int lda, cuComplex* x, int incx, cuComplex* beta,
+        cuComplex* y, int incy)
+    int cublasZgemv(
+        Handle handle, Operation trans, int m, int n, cuDoubleComplex* alpha,
+        cuDoubleComplex* A, int lda, cuDoubleComplex* x, int incx,
+        cuDoubleComplex* beta, cuDoubleComplex* y, int incy)
+    int cublasSger(
+        Handle handle, int m, int n, float* alpha, float* x, int incx,
+        float* y, int incy, float* A, int lda)
+    int cublasDger(
+        Handle handle, int m, int n, double* alpha, double* x,
+        int incx, double* y, int incy, double* A, int lda)
+    int cublasCgeru(
+        Handle handle, int m, int n, cuComplex* alpha, cuComplex* x,
+        int incx, cuComplex* y, int incy, cuComplex* A, int lda)
+    int cublasCgerc(
+        Handle handle, int m, int n, cuComplex* alpha, cuComplex* x,
+        int incx, cuComplex* y, int incy, cuComplex* A, int lda)
+    int cublasZgeru(
+        Handle handle, int m, int n, cuDoubleComplex* alpha,
+        cuDoubleComplex* x, int incx, cuDoubleComplex* y, int incy,
+        cuDoubleComplex* A, int lda)
+    int cublasZgerc(
+        Handle handle, int m, int n, cuDoubleComplex* alpha,
+        cuDoubleComplex* x, int incx, cuDoubleComplex* y, int incy,
+        cuDoubleComplex* A, int lda)
+    int cublasSsbmv(
+        Handle handle, FillMode uplo, int n, int k,
+        const float* alpha, const float* A, int lda,
+        const float* x, int incx, const float* beta, float* y, int incy)
+    int cublasDsbmv(
+        Handle handle, FillMode uplo, int n, int k,
+        const double* alpha, const double* A, int lda,
+        const double* x, int incx, const double* beta, double* y, int incy)
+
+    # BLAS Level 3
+    int cublasSgemm(
+        Handle handle, Operation transa, Operation transb, int m,
+        int n, int k, float* alpha, float* A, int lda, float* B,
+        int ldb, float* beta, float* C, int ldc)
+    int cublasDgemm(
+        Handle handle, Operation transa, Operation transb, int m,
+        int n, int k, double* alpha, double* A, int lda, double* B,
+        int ldb, double* beta, double* C, int ldc)
+    int cublasCgemm(
+        Handle handle, Operation transa, Operation transb, int m,
+        int n, int k, cuComplex* alpha, cuComplex* A, int lda,
+        cuComplex* B, int ldb, cuComplex* beta, cuComplex* C,
+        int ldc)
+    int cublasZgemm(
+        Handle handle, Operation transa, Operation transb, int m,
+        int n, int k, cuDoubleComplex* alpha, cuDoubleComplex* A, int lda,
+        cuDoubleComplex* B, int ldb, cuDoubleComplex* beta,
+        cuDoubleComplex* C, int ldc)
+    int cublasSgemmBatched(
+        Handle handle, Operation transa, Operation transb, int m,
+        int n, int k, const float* alpha, const float** Aarray,
+        int lda, const float** Barray, int ldb, const float* beta,
+        float** Carray, int ldc, int batchCount)
+    int cublasDgemmBatched(
+        Handle handle, Operation transa, Operation transb, int m,
+        int n, int k, const double* alpha, const double** Aarray,
+        int lda, const double** Barray, int ldb, const double* beta,
+        double** Carray, int ldc, int batchCount)
+    int cublasCgemmBatched(
+        Handle handle, Operation transa, Operation transb, int m,
+        int n, int k, const cuComplex* alpha, const cuComplex** Aarray,
+        int lda, const cuComplex** Barray, int ldb, const cuComplex* beta,
+        cuComplex** Carray, int ldc, int batchCount)
+    int cublasZgemmBatched(
+        Handle handle, Operation transa, Operation transb, int m,
+        int n, int k, const cuDoubleComplex* alpha,
+        const cuDoubleComplex** Aarray, int lda,
+        const cuDoubleComplex** Barray, int ldb,
+        const cuDoubleComplex* beta, cuDoubleComplex** Carray, int ldc,
+        int batchCount)
+    int cublasSgemmStridedBatched(
+        Handle handle, Operation transa, Operation transb,
+        int m, int n, int k, const float* alpha,
+        const float* A, int lda, long long strideA,
+        const float* B, int ldb, long long strideB,
+        const float* beta,
+        float* C, int ldc, long long strideC, int batchCount)
+    int cublasDgemmStridedBatched(
+        Handle handle, Operation transa, Operation transb,
+        int m, int n, int k, const double* alpha,
+        const double* A, int lda, long long strideA,
+        const double* B, int ldb, long long strideB,
+        const double* beta,
+        double* C, int ldc, long long strideC, int batchCount)
+    int cublasCgemmStridedBatched(
+        Handle handle, Operation transa, Operation transb,
+        int m, int n, int k, const cuComplex* alpha,
+        const cuComplex* A, int lda, long long strideA,
+        const cuComplex* B, int ldb, long long strideB,
+        const cuComplex* beta,
+        cuComplex* C, int ldc, long long strideC, int batchCount)
+    int cublasZgemmStridedBatched(
+        Handle handle, Operation transa, Operation transb,
+        int m, int n, int k, const cuDoubleComplex* alpha,
+        const cuDoubleComplex* A, int lda, long long strideA,
+        const cuDoubleComplex* B, int ldb, long long strideB,
+        const cuDoubleComplex* beta,
+        cuDoubleComplex* C, int ldc, long long strideC, int batchCount)
+    int cublasStrsm(
+        Handle handle, SideMode size, FillMode uplo, Operation trans,
+        DiagType diag, int m, int n, const float* alpha,
+        const float* A, int lda, float* B, int ldb)
+    int cublasDtrsm(
+        Handle handle, SideMode size, FillMode uplo, Operation trans,
+        DiagType diag, int m, int n, const double* alpha,
+        const double* A, int lda, double* B, int ldb)
+    int cublasCtrsm(
+        Handle handle, SideMode size, FillMode uplo, Operation trans,
+        DiagType diag, int m, int n, const cuComplex* alpha,
+        const cuComplex* A, int lda, cuComplex* B, int ldb)
+    int cublasZtrsm(
+        Handle handle, SideMode size, FillMode uplo, Operation trans,
+        DiagType diag, int m, int n, const cuDoubleComplex* alpha,
+        const cuDoubleComplex* A, int lda, cuDoubleComplex* B, int ldb)
+    int cublasSsyrk(
+        Handle handle, FillMode uplo, Operation trans, int n, int k,
+        float* alpha, float* A, int lda,
+        float* beta, float* C, int ldc)
+    int cublasDsyrk(
+        Handle handle, FillMode uplo, Operation trans, int n, int k,
+        double* alpha, double* A, int lda,
+        double* beta, double* C, int ldc)
+    int cublasCsyrk(
+        Handle handle, FillMode uplo, Operation trans, int n, int k,
+        cuComplex* alpha, cuComplex* A, int lda,
+        cuComplex* beta, cuComplex* C, int ldc)
+    int cublasZsyrk(
+        Handle handle, FillMode uplo, Operation trans, int n, int k,
+        cuDoubleComplex* alpha, cuDoubleComplex* A, int lda,
+        cuDoubleComplex* beta, cuDoubleComplex* C, int ldc)
+
+    # BLAS extension
+    int cublasSgeam(
+        Handle handle, Operation transa, Operation transb, int m, int n,
+        const float* alpha, const float* A, int lda,
+        const float* beta, const float* B, int ldb,
+        float* C, int ldc)
+    int cublasDgeam(
+        Handle handle, Operation transa, Operation transb, int m, int n,
+        const double* alpha, const double* A, int lda,
+        const double* beta, const double* B, int ldb,
+        double* C, int ldc)
+    int cublasCgeam(
+        Handle handle, Operation transa, Operation transb, int m, int n,
+        const cuComplex* alpha, const cuComplex* A, int lda,
+        const cuComplex* beta, const cuComplex* B, int ldb,
+        cuComplex* C, int ldc)
+    int cublasZgeam(
+        Handle handle, Operation transa, Operation transb, int m, int n,
+        const cuDoubleComplex* alpha, const cuDoubleComplex* A, int lda,
+        const cuDoubleComplex* beta, const cuDoubleComplex* B, int ldb,
+        cuDoubleComplex* C, int ldc)
+    int cublasSdgmm(
+        Handle handle, SideMode mode, int m, int n, const float* A, int lda,
+        const float* x, int incx, float* C, int ldc)
+    int cublasDdgmm(
+        Handle handle, SideMode mode, int m, int n, const double* A, int lda,
+        const double* x, int incx, double* C, int ldc)
+    int cublasCdgmm(
+        Handle handle, SideMode mode, int m, int n, const cuComplex* A,
+        int lda, const cuComplex* x, int incx, cuComplex* C, int ldc)
+    int cublasZdgmm(
+        Handle handle, SideMode mode, int m, int n, const cuDoubleComplex* A,
+        int lda, const cuDoubleComplex* x, int incx, cuDoubleComplex* C,
+        int ldc)
+    int cublasSgemmEx(
+        Handle handle, Operation transa,
+        Operation transb, int m, int n, int k,
+        const float *alpha, const void *A, DataType Atype,
+        int lda, const void *B, DataType Btype, int ldb,
+        const float *beta, void *C, DataType Ctype, int ldc)
+    int cublasSgetrfBatched(
+        Handle handle, int n, float **Aarray, int lda,
+        int *PivotArray, int *infoArray, int batchSize)
+    int cublasDgetrfBatched(
+        Handle handle, int n, double **Aarray, int lda,
+        int *PivotArray, int *infoArray, int batchSize)
+    int cublasCgetrfBatched(
+        Handle handle, int n, cuComplex **Aarray, int lda,
+        int *PivotArray, int *infoArray, int batchSize)
+    int cublasZgetrfBatched(
+        Handle handle, int n, cuDoubleComplex **Aarray, int lda,
+        int *PivotArray, int *infoArray, int batchSize)
+
+    int cublasSgetrsBatched(
+        Handle handle, Operation trans, int n, int nrhs,
+        const float **Aarray, int lda, const int *devIpiv,
+        float **Barray, int ldb, int *info, int batchSize)
+    int cublasDgetrsBatched(
+        Handle handle, Operation trans, int n, int nrhs,
+        const double **Aarray, int lda, const int *devIpiv,
+        double **Barray, int ldb, int *info, int batchSize)
+    int cublasCgetrsBatched(
+        Handle handle, Operation trans, int n, int nrhs,
+        const cuComplex **Aarray, int lda, const int *devIpiv,
+        cuComplex **Barray, int ldb, int *info, int batchSize)
+    int cublasZgetrsBatched(
+        Handle handle, Operation trans, int n, int nrhs,
+        const cuDoubleComplex **Aarray, int lda, const int *devIpiv,
+        cuDoubleComplex **Barray, int ldb, int *info, int batchSize)
+
+    int cublasSgetriBatched(
+        Handle handle, int n, const float **Aarray, int lda,
+        int *PivotArray, float *Carray[], int ldc, int *infoArray,
+        int batchSize)
+    int cublasDgetriBatched(
+        Handle handle, int n, const double **Aarray, int lda,
+        int *PivotArray, double *Carray[], int ldc, int *infoArray,
+        int batchSize)
+    int cublasCgetriBatched(
+        Handle handle, int n, const cuComplex **Aarray, int lda,
+        int *PivotArray, cuComplex *Carray[], int ldc, int *infoArray,
+        int batchSize)
+    int cublasZgetriBatched(
+        Handle handle, int n, const cuDoubleComplex **Aarray, int lda,
+        int *PivotArray, cuDoubleComplex *Carray[], int ldc, int *infoArray,
+        int batchSize)
+    int cublasGemmEx(
+        Handle handle, Operation transa, Operation transb,
+        int m, int n, int k,
+        const void *alpha,
+        const void *A, DataType Atype, int lda,
+        const void *B, DataType Btype, int ldb,
+        const void *beta,
+        void *C, DataType Ctype, int ldc,
+        DataType computetype, GemmAlgo algo)
+    int cublasGemmEx_v11(
+        Handle handle, Operation transa, Operation transb,
+        int m, int n, int k,
+        const void *alpha,
+        const void *A, DataType Atype, int lda,
+        const void *B, DataType Btype, int ldb,
+        const void *beta,
+        void *C, DataType Ctype, int ldc,
+        ComputeType computetype, GemmAlgo algo)
+    int cublasGemmStridedBatchedEx(
+        Handle handle, Operation transa, Operation transb,
+        int m, int n, int k,
+        const void *alpha,
+        const void *A, DataType Atype, int lda, long long strideA,
+        const void *B, DataType Btype, int ldb, long long strideB,
+        const void *beta,
+        void *C, DataType Ctype, int ldc, long long strideC,
+        int batchCount, DataType computetype, GemmAlgo algo)
+    int cublasGemmStridedBatchedEx_v11(
+        Handle handle, Operation transa, Operation transb,
+        int m, int n, int k,
+        const void *alpha,
+        const void *A, DataType Atype, int lda, long long strideA,
+        const void *B, DataType Btype, int ldb, long long strideB,
+        const void *beta,
+        void *C, DataType Ctype, int ldc, long long strideC,
+        int batchCount, ComputeType computetype, GemmAlgo algo)
+    int cublasStpttr(
+        Handle handle, FillMode uplo, int n, const float *AP, float *A,
+        int lda)
+    int cublasDtpttr(
+        Handle handle, FillMode uplo, int n, const double *AP, double *A,
+        int lda)
+    int cublasStrttp(
+        Handle handle, FillMode uplo, int n, const float *A, int lda,
+        float *AP)
+    int cublasDtrttp(
+        Handle handle, FillMode uplo, int n, const double *A, int lda,
+        double *AP)
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+cdef dict STATUS = {
+    0: 'CUBLAS_STATUS_SUCCESS',
+    1: 'CUBLAS_STATUS_NOT_INITIALIZED',
+    3: 'CUBLAS_STATUS_ALLOC_FAILED',
+    7: 'CUBLAS_STATUS_INVALID_VALUE',
+    8: 'CUBLAS_STATUS_ARCH_MISMATCH',
+    11: 'CUBLAS_STATUS_MAPPING_ERROR',
+    13: 'CUBLAS_STATUS_EXECUTION_FAILED',
+    14: 'CUBLAS_STATUS_INTERNAL_ERROR',
+    15: 'CUBLAS_STATUS_NOT_SUPPORTED',
+    16: 'CUBLAS_STATUS_LICENSE_ERROR',
+}
+
+
+cdef dict HIP_STATUS = {
+    0: 'HIPBLAS_STATUS_SUCCESS',
+    1: 'HIPBLAS_STATUS_NOT_INITIALIZED',
+    2: 'HIPBLAS_STATUS_ALLOC_FAILED',
+    3: 'HIPBLAS_STATUS_INVALID_VALUE',
+    4: 'HIPBLAS_STATUS_MAPPING_ERROR',
+    5: 'HIPBLAS_STATUS_EXECUTION_FAILED',
+    6: 'HIPBLAS_STATUS_INTERNAL_ERROR',
+    7: 'HIPBLAS_STATUS_NOT_SUPPORTED',
+    8: 'HIPBLAS_STATUS_ARCH_MISMATCH',
+    9: 'HIPBLAS_STATUS_HANDLE_IS_NULLPTR',
+}
+
+
+class CUBLASError(RuntimeError):
+
+    def __init__(self, status):
+        self.status = status
+        cdef str err
+        if runtime._is_hip_environment:
+            err = HIP_STATUS[status]
+        else:
+            err = STATUS[status]
+        super(CUBLASError, self).__init__(err)
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        raise CUBLASError(status)
+
+
+###############################################################################
+# Context
+###############################################################################
+
+cpdef intptr_t create() except? 0:
+    cdef Handle handle
+    with nogil:
+        status = cublasCreate(&handle)
+    check_status(status)
+    return <intptr_t>handle
+
+
+cpdef destroy(intptr_t handle):
+    with nogil:
+        status = cublasDestroy(<Handle>handle)
+    check_status(status)
+
+
+cpdef int getVersion(intptr_t handle) except? -1:
+    cdef int version
+    with nogil:
+        status = cublasGetVersion(<Handle>handle, &version)
+    check_status(status)
+    return version
+
+
+cpdef int getPointerMode(intptr_t handle) except? -1:
+    cdef PointerMode mode
+    with nogil:
+        status = cublasGetPointerMode(<Handle>handle, &mode)
+    check_status(status)
+    return mode
+
+
+cpdef setPointerMode(intptr_t handle, int mode):
+    with nogil:
+        status = cublasSetPointerMode(<Handle>handle, <PointerMode>mode)
+    check_status(status)
+
+
+###############################################################################
+# Stream
+###############################################################################
+
+cpdef setStream(intptr_t handle, size_t stream):
+    # TODO(leofang): It seems most of cuBLAS APIs support stream capture (as of
+    # CUDA 11.5) under certain conditions, see
+    # https://docs.nvidia.com/cuda/cublas/index.html#CUDA-graphs
+    # Before we come up with a robust strategy to test the support conditions,
+    # we disable this functionality.
+    if not runtime._is_hip_environment and runtime.streamIsCapturing(stream):
+        raise NotImplementedError(
+            'calling cuBLAS API during stream capture is currently '
+            'unsupported')
+
+    with nogil:
+        status = cublasSetStream(<Handle>handle, <Stream>stream)
+    check_status(status)
+
+
+cpdef size_t getStream(intptr_t handle) except? 0:
+    cdef Stream stream
+    with nogil:
+        status = cublasGetStream(<Handle>handle, &stream)
+    check_status(status)
+    return <size_t>stream
+
+
+cdef _setStream(intptr_t handle):
+    """Set current stream"""
+    setStream(handle, stream_module.get_current_stream_ptr())
+
+###############################################################################
+# Math Mode
+###############################################################################
+
+cpdef setMathMode(intptr_t handle, int mode):
+    with nogil:
+        status = cublasSetMathMode(<Handle>handle, <Math>mode)
+    check_status(status)
+
+
+cpdef int getMathMode(intptr_t handle) except? -1:
+    cdef Math mode
+    with nogil:
+        status = cublasGetMathMode(<Handle>handle, &mode)
+    check_status(status)
+    return <int>mode
+
+
+###############################################################################
+# BLAS Level 1
+###############################################################################
+
+cpdef isamax(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasIsamax(
+            <Handle>handle, n, <float*>x, incx, <int*>result)
+    check_status(status)
+
+cpdef idamax(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasIdamax(
+            <Handle>handle, n, <double*>x, incx, <int*>result)
+    check_status(status)
+
+cpdef icamax(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasIcamax(
+            <Handle>handle, n, <cuComplex*>x, incx, <int*>result)
+    check_status(status)
+
+cpdef izamax(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasIzamax(
+            <Handle>handle, n, <cuDoubleComplex*>x, incx, <int*>result)
+    check_status(status)
+
+
+cpdef isamin(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasIsamin(
+            <Handle>handle, n, <float*>x, incx, <int*>result)
+    check_status(status)
+
+cpdef idamin(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasIdamin(
+            <Handle>handle, n, <double*>x, incx, <int*>result)
+    check_status(status)
+
+cpdef icamin(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasIcamin(
+            <Handle>handle, n, <cuComplex*>x, incx, <int*>result)
+    check_status(status)
+
+cpdef izamin(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasIzamin(
+            <Handle>handle, n, <cuDoubleComplex*>x, incx, <int*>result)
+    check_status(status)
+
+
+cpdef sasum(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasSasum(
+            <Handle>handle, n, <float*>x, incx, <float*>result)
+    check_status(status)
+
+cpdef dasum(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasDasum(
+            <Handle>handle, n, <double*>x, incx, <double*>result)
+    check_status(status)
+
+cpdef scasum(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasScasum(
+            <Handle>handle, n, <cuComplex*>x, incx, <float*>result)
+    check_status(status)
+
+cpdef dzasum(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasDzasum(
+            <Handle>handle, n, <cuDoubleComplex*>x, incx, <double*>result)
+    check_status(status)
+
+
+cpdef saxpy(intptr_t handle, int n, size_t alpha, size_t x, int incx, size_t y,
+            int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasSaxpy(
+            <Handle>handle, n, <float*>alpha, <float*>x, incx, <float*>y, incy)
+    check_status(status)
+
+cpdef daxpy(intptr_t handle, int n, size_t alpha, size_t x, int incx, size_t y,
+            int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasDaxpy(
+            <Handle>handle, n, <double*>alpha, <double*>x, incx, <double*>y,
+            incy)
+    check_status(status)
+
+cpdef caxpy(intptr_t handle, int n, size_t alpha, size_t x, int incx, size_t y,
+            int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasCaxpy(
+            <Handle>handle, n, <cuComplex*>alpha, <cuComplex*>x, incx,
+            <cuComplex*>y, incy)
+    check_status(status)
+
+cpdef zaxpy(intptr_t handle, int n, size_t alpha, size_t x, int incx, size_t y,
+            int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasZaxpy(
+            <Handle>handle, n, <cuDoubleComplex*>alpha, <cuDoubleComplex*>x,
+            incx, <cuDoubleComplex*>y, incy)
+    check_status(status)
+
+
+cpdef sdot(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+           size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasSdot(
+            <Handle>handle, n, <float*>x, incx, <float*>y, incy,
+            <float*>result)
+    check_status(status)
+
+cpdef ddot(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+           size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasDdot(
+            <Handle>handle, n, <double*>x, incx, <double*>y, incy,
+            <double*>result)
+    check_status(status)
+
+cpdef cdotu(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+            size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasCdotu(
+            <Handle>handle, n, <cuComplex*>x, incx, <cuComplex*>y, incy,
+            <cuComplex*>result)
+    check_status(status)
+
+cpdef cdotc(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+            size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasCdotc(
+            <Handle>handle, n, <cuComplex*>x, incx, <cuComplex*>y, incy,
+            <cuComplex*>result)
+    check_status(status)
+
+cpdef zdotu(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+            size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasZdotu(
+            <Handle>handle, n, <cuDoubleComplex*>x, incx,
+            <cuDoubleComplex*>y, incy, <cuDoubleComplex*>result)
+    check_status(status)
+
+cpdef zdotc(intptr_t handle, int n, size_t x, int incx, size_t y, int incy,
+            size_t result):
+    with nogil:
+        status = cublasZdotc(
+            <Handle>handle, n, <cuDoubleComplex*>x, incx,
+            <cuDoubleComplex*>y, incy, <cuDoubleComplex*>result)
+    check_status(status)
+
+
+cpdef snrm2(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasSnrm2(<Handle>handle, n, <float*>x, incx,
+                             <float*>result)
+    check_status(status)
+
+cpdef dnrm2(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasDnrm2(<Handle>handle, n, <double*>x, incx,
+                             <double*>result)
+    check_status(status)
+
+cpdef scnrm2(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasScnrm2(<Handle>handle, n, <cuComplex*>x, incx,
+                              <float*>result)
+    check_status(status)
+
+cpdef dznrm2(intptr_t handle, int n, size_t x, int incx, size_t result):
+    _setStream(handle)
+    with nogil:
+        status = cublasDznrm2(<Handle>handle, n, <cuDoubleComplex*>x, incx,
+                              <double*>result)
+    check_status(status)
+
+
+cpdef sscal(intptr_t handle, int n, size_t alpha, size_t x, int incx):
+    _setStream(handle)
+    with nogil:
+        status = cublasSscal(<Handle>handle, n, <float*>alpha,
+                             <float*>x, incx)
+    check_status(status)
+
+cpdef dscal(intptr_t handle, int n, size_t alpha, size_t x, int incx):
+    _setStream(handle)
+    with nogil:
+        status = cublasDscal(<Handle>handle, n, <double*>alpha,
+                             <double*>x, incx)
+    check_status(status)
+
+cpdef cscal(intptr_t handle, int n, size_t alpha, size_t x, int incx):
+    _setStream(handle)
+    with nogil:
+        status = cublasCscal(<Handle>handle, n, <cuComplex*>alpha,
+                             <cuComplex*>x, incx)
+    check_status(status)
+
+cpdef csscal(intptr_t handle, int n, size_t alpha, size_t x, int incx):
+    _setStream(handle)
+    with nogil:
+        status = cublasCsscal(<Handle>handle, n, <float*>alpha,
+                              <cuComplex*>x, incx)
+    check_status(status)
+
+cpdef zscal(intptr_t handle, int n, size_t alpha, size_t x, int incx):
+    _setStream(handle)
+    with nogil:
+        status = cublasZscal(<Handle>handle, n, <cuDoubleComplex*>alpha,
+                             <cuDoubleComplex*>x, incx)
+    check_status(status)
+
+cpdef zdscal(intptr_t handle, int n, size_t alpha, size_t x, int incx):
+    _setStream(handle)
+    with nogil:
+        status = cublasZdscal(<Handle>handle, n, <double*>alpha,
+                              <cuDoubleComplex*>x, incx)
+    check_status(status)
+
+
+###############################################################################
+# BLAS Level 2
+###############################################################################
+
+cpdef sgemv(intptr_t handle, int trans, int m, int n, size_t alpha, size_t A,
+            int lda, size_t x, int incx, size_t beta, size_t y, int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgemv(
+            <Handle>handle, <Operation>trans, m, n, <float*>alpha,
+            <float*>A, lda, <float*>x, incx, <float*>beta, <float*>y, incy)
+    check_status(status)
+
+
+cpdef dgemv(intptr_t handle, int trans, int m, int n, size_t alpha, size_t A,
+            int lda, size_t x, int incx, size_t beta, size_t y, int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasDgemv(
+            <Handle>handle, <Operation>trans, m, n, <double*>alpha,
+            <double*>A, lda, <double*>x, incx, <double*>beta, <double*>y, incy)
+    check_status(status)
+
+
+cpdef cgemv(intptr_t handle, int trans, int m, int n, size_t alpha, size_t A,
+            int lda, size_t x, int incx, size_t beta, size_t y, int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgemv(
+            <Handle>handle, <Operation>trans, m, n, <cuComplex*>alpha,
+            <cuComplex*>A, lda, <cuComplex*>x, incx, <cuComplex*>beta,
+            <cuComplex*>y, incy)
+    check_status(status)
+
+
+cpdef zgemv(intptr_t handle, int trans, int m, int n, size_t alpha, size_t A,
+            int lda, size_t x, int incx, size_t beta, size_t y, int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgemv(
+            <Handle>handle, <Operation>trans, m, n, <cuDoubleComplex*>alpha,
+            <cuDoubleComplex*>A, lda, <cuDoubleComplex*>x, incx,
+            <cuDoubleComplex*>beta, <cuDoubleComplex*>y, incy)
+    check_status(status)
+
+
+cpdef sger(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+           size_t y, int incy, size_t A, int lda):
+    _setStream(handle)
+    with nogil:
+        status = cublasSger(
+            <Handle>handle, m, n, <float*>alpha, <float*>x, incx, <float*>y,
+            incy, <float*>A, lda)
+    check_status(status)
+
+
+cpdef dger(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+           size_t y, int incy, size_t A, int lda):
+    _setStream(handle)
+    with nogil:
+        status = cublasDger(
+            <Handle>handle, m, n, <double*>alpha, <double*>x, incx, <double*>y,
+            incy, <double*>A, lda)
+    check_status(status)
+
+
+cpdef cgeru(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+            size_t y, int incy, size_t A, int lda):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgeru(
+            <Handle>handle, m, n, <cuComplex*>alpha, <cuComplex*>x, incx,
+            <cuComplex*>y, incy, <cuComplex*>A, lda)
+    check_status(status)
+
+
+cpdef cgerc(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+            size_t y, int incy, size_t A, int lda):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgerc(
+            <Handle>handle, m, n, <cuComplex*>alpha, <cuComplex*>x, incx,
+            <cuComplex*>y, incy, <cuComplex*>A, lda)
+    check_status(status)
+
+
+cpdef zgeru(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+            size_t y, int incy, size_t A, int lda):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgeru(
+            <Handle>handle, m, n, <cuDoubleComplex*>alpha,
+            <cuDoubleComplex*>x, incx, <cuDoubleComplex*>y, incy,
+            <cuDoubleComplex*>A, lda)
+    check_status(status)
+
+
+cpdef zgerc(intptr_t handle, int m, int n, size_t alpha, size_t x, int incx,
+            size_t y, int incy, size_t A, int lda):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgerc(
+            <Handle>handle, m, n, <cuDoubleComplex*>alpha,
+            <cuDoubleComplex*>x, incx, <cuDoubleComplex*>y, incy,
+            <cuDoubleComplex*>A, lda)
+    check_status(status)
+
+
+cpdef ssbmv(intptr_t handle, int uplo, int n, int k,
+            size_t alpha, size_t A, int lda,
+            size_t x, int incx, size_t beta, size_t y, int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasSsbmv(
+            <Handle>handle, <FillMode>uplo, n, k,
+            <float*>alpha, <float*>A, lda,
+            <float*>x, incx, <float*>beta, <float*>y, incy)
+    check_status(status)
+
+
+cpdef dsbmv(intptr_t handle, int uplo, int n, int k,
+            size_t alpha, size_t A, int lda,
+            size_t x, int incx, size_t beta, size_t y, int incy):
+    _setStream(handle)
+    with nogil:
+        status = cublasDsbmv(
+            <Handle>handle, <FillMode>uplo, n, k,
+            <double*>alpha, <double*>A, lda,
+            <double*>x, incx, <double*>beta, <double*>y, incy)
+    check_status(status)
+
+
+###############################################################################
+# BLAS Level 3
+###############################################################################
+
+cpdef sgemm(intptr_t handle, int transa, int transb,
+            int m, int n, int k, size_t alpha, size_t A, int lda,
+            size_t B, int ldb, size_t beta, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgemm(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <float*>alpha, <float*>A, lda, <float*>B, ldb, <float*>beta,
+            <float*>C, ldc)
+    check_status(status)
+
+
+cpdef dgemm(intptr_t handle, int transa, int transb,
+            int m, int n, int k, size_t alpha, size_t A, int lda,
+            size_t B, int ldb, size_t beta, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasDgemm(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <double*>alpha, <double*>A, lda, <double*>B, ldb, <double*>beta,
+            <double*>C, ldc)
+    check_status(status)
+
+
+cpdef cgemm(intptr_t handle, int transa, int transb,
+            int m, int n, int k, size_t alpha, size_t A, int lda,
+            size_t B, int ldb, size_t beta, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgemm(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <cuComplex*>alpha, <cuComplex*>A, lda, <cuComplex*>B, ldb,
+            <cuComplex*>beta, <cuComplex*>C, ldc)
+    check_status(status)
+
+
+cpdef zgemm(intptr_t handle, int transa, int transb,
+            int m, int n, int k, size_t alpha, size_t A, int lda,
+            size_t B, int ldb, size_t beta, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgemm(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <cuDoubleComplex*>alpha, <cuDoubleComplex*>A, lda,
+            <cuDoubleComplex*>B, ldb, <cuDoubleComplex*>beta,
+            <cuDoubleComplex*>C, ldc)
+    check_status(status)
+
+
+cpdef sgemmBatched(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t Aarray, int lda, size_t Barray, int ldb,
+        size_t beta, size_t Carray, int ldc, int batchCount):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgemmBatched(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <float*>alpha, <const float**>Aarray, lda, <const float**>Barray,
+            ldb, <float*>beta, <float**>Carray, ldc, batchCount)
+    check_status(status)
+
+
+cpdef dgemmBatched(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t Aarray, int lda, size_t Barray, int ldb,
+        size_t beta, size_t Carray, int ldc, int batchCount):
+    _setStream(handle)
+    with nogil:
+        status = cublasDgemmBatched(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <double*>alpha, <const double**>Aarray, lda,
+            <const double**>Barray, ldb, <double*>beta,
+            <double**>Carray, ldc, batchCount)
+    check_status(status)
+
+
+cpdef cgemmBatched(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t Aarray, int lda, size_t Barray, int ldb,
+        size_t beta, size_t Carray, int ldc, int batchCount):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgemmBatched(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <cuComplex*>alpha, <const cuComplex**>Aarray, lda,
+            <const cuComplex**>Barray, ldb, <cuComplex*>beta,
+            <cuComplex**>Carray, ldc, batchCount)
+    check_status(status)
+
+
+cpdef zgemmBatched(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t Aarray, int lda, size_t Barray, int ldb,
+        size_t beta, size_t Carray, int ldc, int batchCount):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgemmBatched(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <cuDoubleComplex*>alpha, <const cuDoubleComplex**>Aarray, lda,
+            <const cuDoubleComplex**>Barray, ldb, <cuDoubleComplex*>beta,
+            <cuDoubleComplex**>Carray, ldc, batchCount)
+    check_status(status)
+
+
+cpdef sgemmStridedBatched(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t A, int lda, long long strideA, size_t B, int ldb,
+        long long strideB, size_t beta, size_t C, int ldc, long long strideC,
+        int batchCount):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgemmStridedBatched(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <const float*>alpha,
+            <const float*>A, lda, <long long>strideA,
+            <const float*>B, ldb, <long long>strideB,
+            <const float*>beta,
+            <float*>C, ldc, <long long>strideC,
+            batchCount)
+    check_status(status)
+
+
+cpdef dgemmStridedBatched(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t A, int lda, long long strideA, size_t B, int ldb,
+        long long strideB, size_t beta, size_t C, int ldc, long long strideC,
+        int batchCount):
+    _setStream(handle)
+    with nogil:
+        status = cublasDgemmStridedBatched(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <const double*>alpha,
+            <const double*>A, lda, <long long>strideA,
+            <const double*>B, ldb, <long long>strideB,
+            <const double*>beta,
+            <double*>C, ldc, <long long>strideC,
+            batchCount)
+    check_status(status)
+
+
+cpdef cgemmStridedBatched(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t A, int lda, long long strideA, size_t B, int ldb,
+        long long strideB, size_t beta, size_t C, int ldc, long long strideC,
+        int batchCount):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgemmStridedBatched(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <const cuComplex*>alpha,
+            <const cuComplex*>A, lda, <long long>strideA,
+            <const cuComplex*>B, ldb, <long long>strideB,
+            <const cuComplex*>beta,
+            <cuComplex*>C, ldc, <long long>strideC,
+            batchCount)
+    check_status(status)
+
+
+cpdef zgemmStridedBatched(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t A, int lda, long long strideA, size_t B, int ldb,
+        long long strideB, size_t beta, size_t C, int ldc, long long strideC,
+        int batchCount):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgemmStridedBatched(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <const cuDoubleComplex*>alpha,
+            <const cuDoubleComplex*>A, lda, <long long>strideA,
+            <const cuDoubleComplex*>B, ldb, <long long>strideB,
+            <const cuDoubleComplex*>beta,
+            <cuDoubleComplex*>C, ldc, <long long>strideC,
+            batchCount)
+    check_status(status)
+
+
+cpdef strsm(
+        intptr_t handle, int side, int uplo, int trans, int diag,
+        int m, int n, size_t alpha, size_t Aarray, int lda,
+        size_t Barray, int ldb):
+    _setStream(handle)
+    with nogil:
+        status = cublasStrsm(
+            <Handle>handle, <SideMode>side, <FillMode>uplo, <Operation>trans,
+            <DiagType>diag, m, n, <const float*>alpha, <const float*>Aarray,
+            lda, <float*>Barray, ldb)
+    check_status(status)
+
+
+cpdef dtrsm(
+        intptr_t handle, int side, int uplo, int trans, int diag,
+        int m, int n, size_t alpha, size_t Aarray, int lda,
+        size_t Barray, int ldb):
+    _setStream(handle)
+    with nogil:
+        status = cublasDtrsm(
+            <Handle>handle, <SideMode>side, <FillMode>uplo, <Operation>trans,
+            <DiagType>diag, m, n, <const double*>alpha, <const double*>Aarray,
+            lda, <double*>Barray, ldb)
+    check_status(status)
+
+
+cpdef ctrsm(
+        intptr_t handle, int side, int uplo, int trans, int diag,
+        int m, int n, size_t alpha, size_t Aarray, int lda,
+        size_t Barray, int ldb):
+    _setStream(handle)
+    with nogil:
+        status = cublasCtrsm(
+            <Handle>handle, <SideMode>side, <FillMode>uplo, <Operation>trans,
+            <DiagType>diag, m, n, <const cuComplex*>alpha,
+            <const cuComplex*>Aarray, lda, <cuComplex*>Barray, ldb)
+    check_status(status)
+
+
+cpdef ztrsm(
+        intptr_t handle, int side, int uplo, int trans, int diag,
+        int m, int n, size_t alpha, size_t Aarray, int lda,
+        size_t Barray, int ldb):
+    _setStream(handle)
+    with nogil:
+        status = cublasZtrsm(
+            <Handle>handle, <SideMode>side, <FillMode>uplo, <Operation>trans,
+            <DiagType>diag, m, n, <const cuDoubleComplex*>alpha,
+            <const cuDoubleComplex*>Aarray, lda, <cuDoubleComplex*>Barray, ldb)
+    check_status(status)
+
+
+cpdef ssyrk(intptr_t handle, int uplo, int trans, int n, int k,
+            size_t alpha, size_t A, int lda, size_t beta, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasSsyrk(
+            <Handle>handle, <FillMode>uplo, <Operation>trans, n, k,
+            <const float*>alpha, <const float*>A, lda,
+            <const float*>beta, <float*>C, ldc)
+    check_status(status)
+
+
+cpdef dsyrk(intptr_t handle, int uplo, int trans, int n, int k,
+            size_t alpha, size_t A, int lda, size_t beta, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasDsyrk(
+            <Handle>handle, <FillMode>uplo, <Operation>trans, n, k,
+            <const double*>alpha, <const double*>A, lda,
+            <const double*>beta, <double*>C, ldc)
+    check_status(status)
+
+
+cpdef csyrk(intptr_t handle, int uplo, int trans, int n, int k,
+            size_t alpha, size_t A, int lda, size_t beta, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasCsyrk(
+            <Handle>handle, <FillMode>uplo, <Operation>trans, n, k,
+            <const cuComplex*>alpha, <const cuComplex*>A, lda,
+            <const cuComplex*>beta, <cuComplex*>C, ldc)
+    check_status(status)
+
+
+cpdef zsyrk(intptr_t handle, int uplo, int trans, int n, int k,
+            size_t alpha, size_t A, int lda, size_t beta, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasZsyrk(
+            <Handle>handle, <FillMode>uplo, <Operation>trans, n, k,
+            <const cuDoubleComplex*>alpha, <const cuDoubleComplex*>A, lda,
+            <const cuDoubleComplex*>beta, <cuDoubleComplex*>C, ldc)
+    check_status(status)
+
+
+###############################################################################
+# BLAS extension
+###############################################################################
+
+cpdef sgeam(intptr_t handle, int transa, int transb, int m, int n,
+            size_t alpha, size_t A, int lda, size_t beta, size_t B, int ldb,
+            size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgeam(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n,
+            <const float*>alpha, <const float*>A, lda, <const float*>beta,
+            <const float*>B, ldb, <float*>C, ldc)
+    check_status(status)
+
+cpdef dgeam(intptr_t handle, int transa, int transb, int m, int n,
+            size_t alpha, size_t A, int lda, size_t beta, size_t B, int ldb,
+            size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasDgeam(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n,
+            <const double*>alpha, <const double*>A, lda, <const double*>beta,
+            <const double*>B, ldb, <double*>C, ldc)
+    check_status(status)
+
+cpdef cgeam(intptr_t handle, int transa, int transb, int m, int n,
+            size_t alpha, size_t A, int lda, size_t beta, size_t B, int ldb,
+            size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgeam(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n,
+            <const cuComplex*>alpha, <const cuComplex*>A, lda,
+            <const cuComplex*>beta, <const cuComplex*>B, ldb,
+            <cuComplex*>C, ldc)
+    check_status(status)
+
+cpdef zgeam(intptr_t handle, int transa, int transb, int m, int n,
+            size_t alpha, size_t A, int lda, size_t beta, size_t B, int ldb,
+            size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgeam(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n,
+            <const cuDoubleComplex*>alpha, <const cuDoubleComplex*>A, lda,
+            <const cuDoubleComplex*>beta, <const cuDoubleComplex*>B, ldb,
+            <cuDoubleComplex*>C, ldc)
+    check_status(status)
+
+
+cpdef sdgmm(intptr_t handle, int mode, int m, int n, size_t A, int lda,
+            size_t x, int incx, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasSdgmm(
+            <Handle>handle, <SideMode>mode, m, n, <const float*>A, lda,
+            <const float*>x, incx, <float*>C, ldc)
+    check_status(status)
+
+cpdef ddgmm(intptr_t handle, int mode, int m, int n, size_t A, int lda,
+            size_t x, int incx, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasDdgmm(
+            <Handle>handle, <SideMode>mode, m, n, <const double*>A, lda,
+            <const double*>x, incx, <double*>C, ldc)
+    check_status(status)
+
+cpdef cdgmm(intptr_t handle, int mode, int m, int n, size_t A, int lda,
+            size_t x, int incx, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasCdgmm(
+            <Handle>handle, <SideMode>mode, m, n, <const cuComplex*>A, lda,
+            <const cuComplex*>x, incx, <cuComplex*>C, ldc)
+    check_status(status)
+
+cpdef zdgmm(intptr_t handle, int mode, int m, int n, size_t A, int lda,
+            size_t x, int incx, size_t C, int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasZdgmm(
+            <Handle>handle, <SideMode>mode, m, n, <const cuDoubleComplex*>A,
+            lda, <const cuDoubleComplex*>x, incx, <cuDoubleComplex*>C, ldc)
+    check_status(status)
+
+
+cpdef sgemmEx(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t A, int Atype, int lda, size_t B,
+        int Btype, int ldb, size_t beta, size_t C, int Ctype,
+        int ldc):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgemmEx(
+            <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+            <const float*>alpha, <const void*>A, <DataType>Atype, lda,
+            <const void*>B, <DataType>Btype, ldb, <const float*>beta,
+            <void*>C, <DataType>Ctype, ldc)
+    check_status(status)
+
+
+cpdef sgetrfBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t infoArray, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgetrfBatched(
+            <Handle>handle, n, <float**>Aarray, lda,
+            <int*>PivotArray, <int*>infoArray, batchSize)
+    check_status(status)
+
+
+cpdef dgetrfBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t infoArray, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasDgetrfBatched(
+            <Handle>handle, n, <double**>Aarray, lda,
+            <int*>PivotArray, <int*>infoArray, batchSize)
+    check_status(status)
+
+
+cpdef cgetrfBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t infoArray, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgetrfBatched(
+            <Handle>handle, n, <cuComplex**>Aarray, lda,
+            <int*>PivotArray, <int*>infoArray, batchSize)
+    check_status(status)
+
+
+cpdef zgetrfBatched(intptr_t handle, int n, size_t Aarray, int lda,
+                    size_t PivotArray, size_t infoArray, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgetrfBatched(
+            <Handle>handle, n, <cuDoubleComplex**>Aarray, lda,
+            <int*>PivotArray, <int*>infoArray, batchSize)
+    check_status(status)
+
+
+cpdef int sgetrsBatched(intptr_t handle, int trans, int n, int nrhs,
+                        size_t Aarray, int lda, size_t devIpiv,
+                        size_t Barray, int ldb, size_t info, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgetrsBatched(
+            <Handle>handle, <Operation>trans, n, nrhs,
+            <const float**>Aarray, lda, <const int*>devIpiv,
+            <float**>Barray, ldb, <int*>info, batchSize)
+    check_status(status)
+
+cpdef int dgetrsBatched(intptr_t handle, int trans, int n, int nrhs,
+                        size_t Aarray, int lda, size_t devIpiv,
+                        size_t Barray, int ldb, size_t info, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasDgetrsBatched(
+            <Handle>handle, <Operation>trans, n, nrhs,
+            <const double**>Aarray, lda, <const int*>devIpiv,
+            <double**>Barray, ldb, <int*>info, batchSize)
+    check_status(status)
+
+cpdef int cgetrsBatched(intptr_t handle, int trans, int n, int nrhs,
+                        size_t Aarray, int lda, size_t devIpiv,
+                        size_t Barray, int ldb, size_t info, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgetrsBatched(
+            <Handle>handle, <Operation>trans, n, nrhs,
+            <const cuComplex**>Aarray, lda, <const int*>devIpiv,
+            <cuComplex**>Barray, ldb, <int*>info, batchSize)
+    check_status(status)
+
+cpdef int zgetrsBatched(intptr_t handle, int trans, int n, int nrhs,
+                        size_t Aarray, int lda, size_t devIpiv,
+                        size_t Barray, int ldb, size_t info, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgetrsBatched(
+            <Handle>handle, <Operation>trans, n, nrhs,
+            <const cuDoubleComplex**>Aarray, lda, <const int*>devIpiv,
+            <cuDoubleComplex**>Barray, ldb, <int*>info, batchSize)
+    check_status(status)
+
+
+cpdef sgetriBatched(
+        intptr_t handle, int n, size_t Aarray, int lda, size_t PivotArray,
+        size_t Carray, int ldc, size_t infoArray, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasSgetriBatched(
+            <Handle>handle, n, <const float**>Aarray, lda, <int*>PivotArray,
+            <float**>Carray, ldc, <int*>infoArray, batchSize)
+    check_status(status)
+
+
+cpdef dgetriBatched(
+        intptr_t handle, int n, size_t Aarray, int lda, size_t PivotArray,
+        size_t Carray, int ldc, size_t infoArray, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasDgetriBatched(
+            <Handle>handle, n, <const double**>Aarray, lda, <int*>PivotArray,
+            <double**>Carray, ldc, <int*>infoArray, batchSize)
+    check_status(status)
+
+
+cpdef cgetriBatched(
+        intptr_t handle, int n, size_t Aarray, int lda, size_t PivotArray,
+        size_t Carray, int ldc, size_t infoArray, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasCgetriBatched(
+            <Handle>handle, n, <const cuComplex**>Aarray, lda,
+            <int*>PivotArray,
+            <cuComplex**>Carray, ldc, <int*>infoArray, batchSize)
+    check_status(status)
+
+
+cpdef zgetriBatched(
+        intptr_t handle, int n, size_t Aarray, int lda, size_t PivotArray,
+        size_t Carray, int ldc, size_t infoArray, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cublasZgetriBatched(
+            <Handle>handle, n, <const cuDoubleComplex**>Aarray, lda,
+            <int*>PivotArray,
+            <cuDoubleComplex**>Carray, ldc, <int*>infoArray, batchSize)
+    check_status(status)
+
+
+cpdef gemmEx(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha, size_t A, int Atype, int lda, size_t B,
+        int Btype, int ldb, size_t beta, size_t C, int Ctype,
+        int ldc, int computeType, int algo):
+    _setStream(handle)
+    with nogil:
+        if computeType >= CUBLAS_COMPUTE_16F:
+            status = cublasGemmEx_v11(
+                <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+                <const void*>alpha,
+                <const void*>A, <DataType>Atype, lda,
+                <const void*>B, <DataType>Btype, ldb,
+                <const void*>beta,
+                <void*>C, <DataType>Ctype, ldc,
+                <ComputeType>computeType, <GemmAlgo>algo)
+        else:
+            status = cublasGemmEx(
+                <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+                <const void*>alpha,
+                <const void*>A, <DataType>Atype, lda,
+                <const void*>B, <DataType>Btype, ldb,
+                <const void*>beta,
+                <void*>C, <DataType>Ctype, ldc,
+                <DataType>computeType, <GemmAlgo>algo)
+    check_status(status)
+
+
+cpdef gemmStridedBatchedEx(
+        intptr_t handle, int transa, int transb, int m, int n, int k,
+        size_t alpha,
+        size_t A, int Atype, int lda, long long strideA,
+        size_t B, int Btype, int ldb, long long strideB,
+        size_t beta,
+        size_t C, int Ctype, int ldc, long long strideC,
+        int batchCount, int computeType, int algo):
+    _setStream(handle)
+    with nogil:
+        if computeType >= CUBLAS_COMPUTE_16F:
+            status = cublasGemmStridedBatchedEx_v11(
+                <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+                <const void*>alpha,
+                <const void*>A, <DataType>Atype, lda, <long long>strideA,
+                <const void*>B, <DataType>Btype, ldb, <long long>strideB,
+                <const void*>beta,
+                <void*>C, <DataType>Ctype, ldc, <long long>strideC,
+                batchCount, <ComputeType>computeType, <GemmAlgo>algo)
+        else:
+            status = cublasGemmStridedBatchedEx(
+                <Handle>handle, <Operation>transa, <Operation>transb, m, n, k,
+                <const void*>alpha,
+                <const void*>A, <DataType>Atype, lda, <long long>strideA,
+                <const void*>B, <DataType>Btype, ldb, <long long>strideB,
+                <const void*>beta,
+                <void*>C, <DataType>Ctype, ldc, <long long>strideC,
+                batchCount, <DataType>computeType, <GemmAlgo>algo)
+    check_status(status)
+
+
+cpdef stpttr(intptr_t handle, int uplo, int n, size_t AP, size_t A, int lda):
+    _setStream(handle)
+    with nogil:
+        status = cublasStpttr(<Handle>handle, <FillMode>uplo, n,
+                              <const float*>AP, <float*>A, lda)
+    check_status(status)
+
+
+cpdef dtpttr(intptr_t handle, int uplo, int n, size_t AP, size_t A, int lda):
+    _setStream(handle)
+    with nogil:
+        status = cublasDtpttr(<Handle>handle, <FillMode>uplo, n,
+                              <const double*>AP, <double*>A, lda)
+    check_status(status)
+
+
+cpdef strttp(intptr_t handle, int uplo, int n, size_t A, int lda, size_t AP):
+    _setStream(handle)
+    with nogil:
+        status = cublasStrttp(<Handle>handle, <FillMode>uplo, n,
+                              <const float*>A, lda, <float*>AP)
+    check_status(status)
+
+
+cpdef dtrttp(intptr_t handle, int uplo, int n, size_t A, int lda, size_t AP):
+    _setStream(handle)
+    with nogil:
+        status = cublasDtrttp(<Handle>handle, <FillMode>uplo, n,
+                              <const double*>A, lda, <double*>AP)
+    check_status(status)
diff --git a/cupy_backends/cuda/libs/cudnn.pxd b/cupy_backends/cuda/libs/cudnn.pxd
new file mode 100644
index 0000000..5f7430a
--- /dev/null
+++ b/cupy_backends/cuda/libs/cudnn.pxd
@@ -0,0 +1,778 @@
+from libc.stdint cimport intptr_t
+
+
+###############################################################################
+# Enum
+###############################################################################
+
+cpdef enum:
+    CUDNN_DATA_FLOAT = 0
+    CUDNN_DATA_DOUBLE = 1
+    CUDNN_DATA_HALF = 2
+
+    CUDNN_DEFAULT_MATH = 0
+    CUDNN_TENSOR_OP_MATH = 1
+
+    CUDNN_NOT_PROPAGATE_NAN = 0
+    CUDNN_PROPAGATE_NAN = 1
+
+    CUDNN_NON_DETERMINISTIC = 0
+    CUDNN_DETERMINISTIC = 1
+
+    CUDNN_TENSOR_NCHW = 0
+    CUDNN_TENSOR_NHWC = 1
+
+    CUDNN_OP_TENSOR_ADD = 0
+    CUDNN_OP_TENSOR_MUL = 1
+    CUDNN_OP_TENSOR_MIN = 2
+    CUDNN_OP_TENSOR_MAX = 3
+    CUDNN_OP_TENSOR_SQRT = 4
+    CUDNN_OP_TENSOR_NOT = 5
+
+    CUDNN_REDUCE_TENSOR_ADD = 0
+    CUDNN_REDUCE_TENSOR_MUL = 1
+    CUDNN_REDUCE_TENSOR_MIN = 2
+    CUDNN_REDUCE_TENSOR_MAX = 3
+    CUDNN_REDUCE_TENSOR_AMAX = 4
+    CUDNN_REDUCE_TENSOR_AVG = 5
+    CUDNN_REDUCE_TENSOR_NORM1 = 6
+    CUDNN_REDUCE_TENSOR_NORM2 = 7
+    CUDNN_REDUCE_TENSOR_MUL_NO_ZEROS = 8
+
+    CUDNN_REDUCE_TENSOR_NO_INDICES = 0
+    CUDNN_REDUCE_TENSOR_FLATTENED_INDICES = 1
+
+    CUDNN_32BIT_INDICES = 0
+    CUDNN_64BIT_INDICES = 1
+    CUDNN_16BIT_INDICES = 2
+    CUDNN_8BIT_INDICES = 3
+
+    CUDNN_ADD_IMAGE = 0
+    CUDNN_ADD_SAME_HW = 0
+    CUDNN_ADD_FEATURE_MAP = 1
+    CUDNN_ADD_SAME_CHW = 1
+    CUDNN_ADD_SAME_C = 2
+    CUDNN_ADD_FULL_TENSOR = 3
+
+    CUDNN_CONVOLUTION = 0
+    CUDNN_CROSS_CORRELATION = 1
+
+    CUDNN_CONVOLUTION_FWD_NO_WORKSPACE = 0
+    CUDNN_CONVOLUTION_FWD_PREFER_FASTEST = 1
+    CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT = 2
+
+    CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM = 0
+    CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM = 1
+    CUDNN_CONVOLUTION_FWD_ALGO_GEMM = 2
+    CUDNN_CONVOLUTION_FWD_ALGO_DIRECT = 3
+    CUDNN_CONVOLUTION_FWD_ALGO_FFT = 4
+    CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING = 5
+    CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD = 6
+    CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED = 7
+
+    CUDNN_CONVOLUTION_BWD_FILTER_NO_WORKSPACE = 0
+    CUDNN_CONVOLUTION_BWD_FILTER_PREFER_FASTEST = 1
+    CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT = 2
+
+    CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0 = 0
+    CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 = 1
+    CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT = 2
+    CUDNN_CONVOLUTION_BWD_FILTER_ALGO_3 = 3
+    CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD = 4
+    CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED = 5
+
+    CUDNN_CONVOLUTION_BWD_DATA_NO_WORKSPACE = 0
+    CUDNN_CONVOLUTION_BWD_DATA_PREFER_FASTEST = 1
+    CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT = 2
+
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_0 = 0
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 = 1
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_FFT = 2
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_FFT_TILING = 3
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD = 4
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED = 5
+
+    CUDNN_SOFTMAX_FAST = 0
+    CUDNN_SOFTMAX_ACCURATE = 1
+    CUDNN_SOFTMAX_LOG = 2
+
+    CUDNN_SOFTMAX_MODE_INSTANCE = 0
+    CUDNN_SOFTMAX_MODE_CHANNEL = 1
+
+    CUDNN_POOLING_MAX = 0
+    CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING = 1
+    CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING = 2
+    CUDNN_POOLING_MAX_DETERMINISTIC = 3
+
+    CUDNN_ACTIVATION_SIGMOID = 0
+    CUDNN_ACTIVATION_RELU = 1
+    CUDNN_ACTIVATION_TANH = 2
+    CUDNN_ACTIVATION_CLIPPED_RELU = 3
+    CUDNN_ACTIVATION_ELU = 4
+    CUDNN_ACTIVATION_IDENTITY = 5
+
+    CUDNN_LRN_CROSS_CHANNEL_DIM1 = 0
+
+    CUDNN_DIVNORM_PRECOMPUTED_MEANS = 0
+
+    CUDNN_BATCHNORM_PER_ACTIVATION = 0
+    CUDNN_BATCHNORM_SPATIAL = 1
+    CUDNN_BATCHNORM_SPATIAL_PERSISTENT = 2
+
+    CUDNN_CTC_LOSS_ALGO_DETERMINISTIC = 0
+    CUDNN_CTC_LOSS_ALGO_NON_DETERMINISTIC = 1
+
+    CUDNN_BATCHNORM_OPS_BN = 0
+    CUDNN_BATCHNORM_OPS_BN_ACTIVATION = 1
+    CUDNN_BATCHNORM_OPS_BN_ADD_ACTIVATION = 2
+
+    CUDNN_RNN_RELU = 0
+    CUDNN_RNN_TANH = 1
+    CUDNN_LSTM = 2
+    CUDNN_GRU = 3
+
+    CUDNN_UNIDIRECTIONAL = 0
+    CUDNN_BIDIRECTIONAL = 1
+
+    CUDNN_RNN_ALGO_STANDARD = 0
+    CUDNN_RNN_ALGO_PERSIST_STATIC = 1
+    CUDNN_RNN_ALGO_PERSIST_DYNAMIC = 2
+
+    CUDNN_RNN_DATA_LAYOUT_SEQ_MAJOR_UNPACKED = 0
+    CUDNN_RNN_DATA_LAYOUT_SEQ_MAJOR_PACKED = 1
+    CUDNN_RNN_DATA_LAYOUT_BATCH_MAJOR_UNPACKED = 2
+
+    CUDNN_RNN_PADDED_IO_DISABLED = 0
+    CUDNN_RNN_PADDED_IO_ENABLED = 1
+
+    CUDNN_LINEAR_INPUT = 0
+    CUDNN_SKIP_INPUT = 1
+
+    CUDNN_SAMPLER_BILINEAR = 0
+
+    CUDNN_STATUS_SUCCESS = 0
+    CUDNN_STATUS_RUNTIME_PREREQUISITE_MISSING = 11
+    CUDNN_STATUS_RUNTIME_IN_PROGRESS = 12
+    CUDNN_STATUS_RUNTIME_FP_OVERFLOW = 13
+
+    CUDNN_ERRQUERY_RAWCODE = 0
+    CUDNN_ERRQUERY_NONBLOCKING = 1
+    CUDNN_ERRQUERY_BLOCKING = 2
+
+    # cudnnFusedOps_t
+    CUDNN_FUSED_SCALE_BIAS_ACTIVATION_CONV_BNSTATS = 0
+    CUDNN_FUSED_SCALE_BIAS_ACTIVATION_WGRAD = 1
+    CUDNN_FUSED_BN_FINALIZE_STATISTICS_TRAINING = 2
+    CUDNN_FUSED_BN_FINALIZE_STATISTICS_INFERENCE = 3
+    CUDNN_FUSED_CONV_SCALE_BIAS_ADD_ACTIVATION = 4
+    CUDNN_FUSED_SCALE_BIAS_ADD_ACTIVATION_GEN_BITMASK = 5
+    CUDNN_FUSED_DACTIVATION_FORK_DBATCHNORM = 6
+
+    # cudnnFusedOpsConstParamLabel_t
+    CUDNN_PARAM_XDESC = 0
+    CUDNN_PARAM_XDATA_PLACEHOLDER = 1
+    CUDNN_PARAM_BN_MODE = 2
+    CUDNN_PARAM_BN_EQSCALEBIAS_DESC = 3
+    CUDNN_PARAM_BN_EQSCALE_PLACEHOLDER = 4
+    CUDNN_PARAM_BN_EQBIAS_PLACEHOLDER = 5
+    CUDNN_PARAM_ACTIVATION_DESC = 6
+    CUDNN_PARAM_CONV_DESC = 7
+    CUDNN_PARAM_WDESC = 8
+    CUDNN_PARAM_WDATA_PLACEHOLDER = 9
+    CUDNN_PARAM_DWDESC = 10
+    CUDNN_PARAM_DWDATA_PLACEHOLDER = 11
+    CUDNN_PARAM_YDESC = 12
+    CUDNN_PARAM_YDATA_PLACEHOLDER = 13
+    CUDNN_PARAM_DYDESC = 14
+    CUDNN_PARAM_DYDATA_PLACEHOLDER = 15
+    CUDNN_PARAM_YSTATS_DESC = 16
+    CUDNN_PARAM_YSUM_PLACEHOLDER = 17
+    CUDNN_PARAM_YSQSUM_PLACEHOLDER = 18
+    CUDNN_PARAM_BN_SCALEBIAS_MEANVAR_DESC = 19
+    CUDNN_PARAM_BN_SCALE_PLACEHOLDER = 20
+    CUDNN_PARAM_BN_BIAS_PLACEHOLDER = 21
+    CUDNN_PARAM_BN_SAVED_MEAN_PLACEHOLDER = 22
+    CUDNN_PARAM_BN_SAVED_INVSTD_PLACEHOLDER = 23
+    CUDNN_PARAM_BN_RUNNING_MEAN_PLACEHOLDER = 24
+    CUDNN_PARAM_BN_RUNNING_VAR_PLACEHOLDER = 25
+    CUDNN_PARAM_ZDESC = 26
+    CUDNN_PARAM_ZDATA_PLACEHOLDER = 27
+    CUDNN_PARAM_BN_Z_EQSCALEBIAS_DESC = 28
+    CUDNN_PARAM_BN_Z_EQSCALE_PLACEHOLDER = 29
+    CUDNN_PARAM_BN_Z_EQBIAS_PLACEHOLDER = 30
+    CUDNN_PARAM_ACTIVATION_BITMASK_DESC = 31
+    CUDNN_PARAM_ACTIVATION_BITMASK_PLACEHOLDER = 32
+    CUDNN_PARAM_DXDESC = 33
+    CUDNN_PARAM_DXDATA_PLACEHOLDER = 34
+    CUDNN_PARAM_DZDESC = 35
+    CUDNN_PARAM_DZDATA_PLACEHOLDER = 36
+    CUDNN_PARAM_BN_DSCALE_PLACEHOLDER = 37
+    CUDNN_PARAM_BN_DBIAS_PLACEHOLDER = 38
+
+    # cudnnFusedOpsPointerPlaceHolder_t
+    CUDNN_PTR_NULL = 0
+    CUDNN_PTR_ELEM_ALIGNED = 1
+    CUDNN_PTR_16B_ALIGNED = 2
+
+    # cudnnFusedOpsVariantParamLabel_t
+    CUDNN_PTR_XDATA = 0
+    CUDNN_PTR_BN_EQSCALE = 1
+    CUDNN_PTR_BN_EQBIAS = 2
+    CUDNN_PTR_WDATA = 3
+    CUDNN_PTR_DWDATA = 4
+    CUDNN_PTR_YDATA = 5
+    CUDNN_PTR_DYDATA = 6
+    CUDNN_PTR_YSUM = 7
+    CUDNN_PTR_YSQSUM = 8
+    CUDNN_PTR_WORKSPACE = 9
+    CUDNN_PTR_BN_SCALE = 10
+    CUDNN_PTR_BN_BIAS = 11
+    CUDNN_PTR_BN_SAVED_MEAN = 12
+    CUDNN_PTR_BN_SAVED_INVSTD = 13
+    CUDNN_PTR_BN_RUNNING_MEAN = 14
+    CUDNN_PTR_BN_RUNNING_VAR = 15
+    CUDNN_PTR_ZDATA = 16
+    CUDNN_PTR_BN_Z_EQSCALE = 17
+    CUDNN_PTR_BN_Z_EQBIAS = 18
+    CUDNN_PTR_ACTIVATION_BITMASK = 19
+    CUDNN_PTR_DXDATA = 20
+    CUDNN_PTR_DZDATA = 21
+    CUDNN_PTR_BN_DSCALE = 22
+    CUDNN_PTR_BN_DBIAS = 23
+    CUDNN_SCALAR_SIZE_T_WORKSPACE_SIZE_IN_BYTES = 100
+    CUDNN_SCALAR_INT64_T_BN_ACCUMULATION_COUNT = 101
+    CUDNN_SCALAR_DOUBLE_BN_EXP_AVG_FACTOR = 102
+    CUDNN_SCALAR_DOUBLE_BN_EPSILON = 103
+
+
+###############################################################################
+# Class
+###############################################################################
+
+cdef class CuDNNAlgoPerf:
+    cdef:
+        int algo
+        int status
+        float time
+        size_t memory
+        int determinism
+        int mathType
+
+
+###############################################################################
+# Version
+###############################################################################
+
+cpdef size_t getVersion() except? 0
+
+###############################################################################
+# Runtime error checking
+###############################################################################
+cpdef queryRuntimeError(intptr_t handle, int mode)
+
+###############################################################################
+# Initialization and CUDA cooperation
+###############################################################################
+
+cpdef intptr_t create() except? 0
+cpdef destroy(intptr_t handle)
+cpdef setStream(intptr_t handle, size_t stream)
+cpdef size_t getStream(intptr_t handle) except? 0
+
+
+###############################################################################
+# Tensor manipulation
+###############################################################################
+
+cpdef size_t createTensorDescriptor() except? 0
+cpdef setTensor4dDescriptor(size_t tensorDesc, int format, int dataType,
+                            int n, int c, int h, int w)
+cpdef setTensor4dDescriptorEx(size_t tensorDesc, int dataType,
+                              int n, int c, int h, int w, int nStride,
+                              int cStride, int hStride, int wStride)
+cpdef tuple getTensor4dDescriptor(size_t tensorDesc)
+cpdef setTensorNdDescriptor(size_t tensorDesc, int dataType, int nbDims,
+                            size_t dimA, size_t strideA)
+cpdef destroyTensorDescriptor(size_t tensorDesc)
+cpdef addTensor_v3(intptr_t handle, size_t alpha, size_t bDesc,
+                   size_t b, size_t beta, size_t yDesc, size_t y)
+
+
+###############################################################################
+# Tensor operations
+###############################################################################
+
+cpdef size_t createOpTensorDescriptor() except? 0
+cpdef setOpTensorDescriptor(size_t opTensorDesc, int opTensorOp,
+                            int opTensorCompType, int opTensorNanOpt)
+cpdef getOpTensorDescriptor(size_t opTensorDesc)
+cpdef destroyOpTensorDescriptor(size_t opTensorDesc)
+cpdef opTensor(intptr_t handle, size_t opTensorDesc, size_t alpha1,
+               size_t aDesc, size_t A, size_t alpha2, size_t bDesc,
+               size_t B, size_t beta, size_t cDesc, size_t C)
+
+
+###############################################################################
+# Tensor reductions
+###############################################################################
+
+cpdef size_t createReduceTensorDescriptor() except? 0
+cpdef setReduceTensorDescriptor(
+    size_t reduceTensorDesc, int reduceTensorOp,
+    int reduceTensorCompType, int reduceTensorNanOpt,
+    int reduceTensorIndices, int reduceTensorIndicesType)
+cpdef getReduceTensorDescriptor(size_t reduceTensorDesc)
+cpdef destroyReduceTensorDescriptor(size_t reduceTensorDesc)
+cpdef size_t getReductionIndicesSize(
+    intptr_t handle, size_t reduceTensorDesc, size_t aDesc,
+    size_t cDesc) except? 0
+cpdef size_t getReductionWorkspaceSize(
+    intptr_t handle, size_t reduceTensorDesc, size_t aDesc,
+    size_t cDesc) except? 0
+cpdef reduceTensor(
+    intptr_t handle, size_t reduceTensorDesc, size_t indices,
+    size_t indicesSizeInBytes, size_t workspace,
+    size_t workspaceSizeInBytes, size_t alpha, size_t aDesc,
+    size_t A, size_t beta, size_t cDesc, size_t C)
+cpdef setTensor(intptr_t handle, size_t yDesc, size_t y, size_t valuePtr)
+cpdef scaleTensor(intptr_t handle, size_t yDesc, size_t y, size_t alpha)
+
+
+###############################################################################
+# Filter manipulation
+###############################################################################
+
+cpdef size_t createFilterDescriptor() except? 0
+cpdef setFilter4dDescriptor_v4(
+    size_t filterDesc, int dataType, int format, int k, int c, int h, int w)
+cpdef setFilterNdDescriptor_v4(
+    size_t filterDesc, int dataType, int format, int nbDims, size_t filterDimA)
+cpdef getFilterNdDescriptor(size_t wDesc, int nbDimsRequested)
+cpdef destroyFilterDescriptor(size_t filterDesc)
+
+
+###############################################################################
+# Convolution
+###############################################################################
+
+cpdef size_t createConvolutionDescriptor() except? 0
+cpdef setConvolutionMathType(
+    size_t convDesc, size_t mathType)
+cpdef size_t getConvolutionMathType(size_t convDesc) except? 0
+cpdef setConvolutionGroupCount(
+    size_t convDesc, int groupCount)
+cpdef int getConvolutionGroupCount(size_t convDesc) except? -1
+cpdef setConvolution2dDescriptor_v4(
+    size_t convDesc, int pad_h, int pad_w, int u, int v, int dilation_h,
+    int dilation_w, int mode)
+cpdef setConvolution2dDescriptor_v5(
+    size_t convDesc, int pad_h, int pad_w, int u, int v, int dilation_h,
+    int dilation_w, int mode, size_t computeType)
+cpdef setConvolutionNdDescriptor_v3(
+    size_t convDesc, int arrayLength, size_t padA, size_t filterStrideA,
+    size_t dilationA, int mode, int dataType)
+cpdef destroyConvolutionDescriptor(size_t convDesc)
+cpdef findConvolutionForwardAlgorithm(
+    intptr_t handle, size_t xDesc, size_t wDesc, size_t convDesc, size_t yDesc,
+    int requestedAlgoCount)
+cpdef list findConvolutionForwardAlgorithmEx(
+    intptr_t handle, size_t xDesc, size_t x, size_t wDesc, size_t w,
+    size_t convDesc, size_t yDesc, size_t y, int requestedAlgoCount,
+    size_t workSpace, size_t workSpaceSizeInBytes)
+cpdef list findConvolutionForwardAlgorithmEx_v7(
+    intptr_t handle, size_t xDesc, size_t x, size_t wDesc, size_t w,
+    size_t convDesc, size_t yDesc, size_t y, int requestedAlgoCount,
+    size_t workSpace, size_t workSpaceSizeInBytes)
+cpdef int getConvolutionForwardAlgorithm_v6(
+    intptr_t handle, size_t srcDesc, size_t filterDesc, size_t convDesc,
+    size_t destDesc, int preference, size_t memoryLimitInbytes) except? -1
+cpdef list getConvolutionForwardAlgorithm_v7(
+    intptr_t handle, size_t srcDesc, size_t filterDesc, size_t convDesc,
+    size_t destDesc, int requestedAlgoCount)
+cpdef Py_ssize_t getConvolutionForwardWorkspaceSize(
+    intptr_t handle, size_t srcDesc, size_t filterDesc, size_t convDesc,
+    size_t destDesc, int algo) except? -1
+cpdef convolutionForward(
+    intptr_t handle, size_t alpha, size_t srcDesc, size_t srcData,
+    size_t filterDesc, size_t filterData, size_t convDesc, int algo,
+    size_t workSpace, size_t workSpaceSizeInBytes, size_t beta,
+    size_t destDesc, size_t destData)
+cpdef convolutionBackwardBias(
+    intptr_t handle, size_t alpha, size_t srcDesc, size_t srcData,
+    size_t beta, size_t destDesc, size_t destData)
+cpdef findConvolutionBackwardFilterAlgorithm(
+    intptr_t handle, size_t xDesc, size_t dyDesc, size_t convDesc,
+    size_t dwDesc, int requestedAlgoCount)
+cpdef list findConvolutionBackwardFilterAlgorithmEx(
+    intptr_t handle, size_t xDesc, size_t x, size_t dyDesc, size_t dy,
+    size_t convDesc, size_t dwDesc, size_t dw, int requestedAlgoCount,
+    size_t workSpace, size_t workSpaceSizeInBytes)
+cpdef list findConvolutionBackwardFilterAlgorithmEx_v7(
+    intptr_t handle, size_t xDesc, size_t x, size_t dyDesc, size_t dy,
+    size_t convDesc, size_t dwDesc, size_t dw, int requestedAlgoCount,
+    size_t workSpace, size_t workSpaceSizeInBytes)
+cpdef int getConvolutionBackwardFilterAlgorithm_v6(
+    intptr_t handle, size_t srcDesc, size_t diffDesc, size_t convDesc,
+    size_t filterDesc, int preference, size_t memoryLimitInbytes) except? -1
+cpdef list getConvolutionBackwardFilterAlgorithm_v7(
+    intptr_t handle, size_t srcDesc, size_t diffDesc, size_t convDesc,
+    size_t gradDesc, int requestedAlgoCount)
+cpdef Py_ssize_t getConvolutionBackwardFilterWorkspaceSize(
+    intptr_t handle, size_t srcDesc, size_t diffDesc, size_t convDesc,
+    size_t filterDesc, int algo) except? -1
+cpdef convolutionBackwardFilter_v3(
+    intptr_t handle, size_t alpha, size_t srcDesc, size_t srcData,
+    size_t diffDesc, size_t diffData, size_t convDesc, int algo,
+    size_t workSpace, size_t workSpaceSizeInBytes, size_t beta,
+    size_t gradDesc, size_t gradData)
+cpdef findConvolutionBackwardDataAlgorithm(
+    intptr_t handle, size_t wDesc, size_t dyDesc, size_t convDesc,
+    size_t dxDesc, int requestedAlgoCount)
+cpdef list findConvolutionBackwardDataAlgorithmEx(
+    intptr_t handle, size_t wDesc, size_t w, size_t dyDesc, size_t dy,
+    size_t convDesc, size_t dxDesc, size_t dx,
+    int requestedAlgoCount, size_t workSpace, size_t workSpaceSizeInBytes)
+cpdef list findConvolutionBackwardDataAlgorithmEx_v7(
+    intptr_t handle, size_t wDesc, size_t w, size_t dyDesc, size_t dy,
+    size_t convDesc, size_t dxDesc, size_t dx,
+    int requestedAlgoCount, size_t workSpace, size_t workSpaceSizeInBytes)
+cpdef int getConvolutionBackwardDataAlgorithm_v6(
+    intptr_t handle, size_t filterDesc, size_t diffDesc, size_t convDesc,
+    size_t gradDesc, size_t preference,
+    size_t memoryLimitInbytes) except? -1
+cpdef list getConvolutionBackwardDataAlgorithm_v7(
+    intptr_t handle, size_t filterDesc, size_t diffDesc, size_t convDesc,
+    size_t gradDesc, int requestedAlgoCount)
+cpdef Py_ssize_t getConvolutionBackwardDataWorkspaceSize(
+    intptr_t handle, size_t filterDesc, size_t diffDesc, size_t convDesc,
+    size_t gradDesc, int algo) except? -1
+cpdef convolutionBackwardData_v3(
+    intptr_t handle, size_t alpha, size_t filterDesc, size_t filterData,
+    size_t diffDesc, size_t diffData, size_t convDesc, int algo,
+    size_t workSpace, size_t workSpaceSizeInBytes, size_t beta,
+    size_t gradDesc, size_t gradData)
+
+
+###############################################################################
+# Pooling
+###############################################################################
+
+cpdef size_t createPoolingDescriptor() except? 0
+cpdef setPooling2dDescriptor_v4(
+    size_t poolingDesc, int mode, int maxpoolingNanOpt, int windowHeight,
+    int windowWidth, int verticalPadding, int horizontalPadding,
+    int verticalStride, int horizontalStride)
+cpdef setPoolingNdDescriptor_v4(
+    size_t poolingDesc, int mode, int maxpoolingNanOpt, int nbDims,
+    size_t windowDimA, size_t paddingA, size_t strideA)
+cpdef destroyPoolingDescriptor(size_t poolingDesc)
+cpdef poolingForward(
+    intptr_t handle, size_t poolingDesc, size_t alpha, size_t srcDesc,
+    size_t srcData, size_t beta, size_t dstDesc, size_t dstData)
+cpdef poolingBackward(
+    intptr_t handle, size_t poolingDesc, size_t alpha, size_t srcDesc,
+    size_t srcData, size_t srcDiffDesc, size_t srcDiffData,
+    size_t destDesc, size_t destData, size_t beta, size_t destDiffDesc,
+    size_t destDiffData)
+
+###############################################################################
+# Batch Normalization
+###############################################################################
+
+cpdef deriveBNTensorDescriptor(
+    size_t derivedBnDesc, size_t xDesc, int mode)
+
+cpdef batchNormalizationForwardTraining(
+    intptr_t handle, int mode,
+    size_t alpha, size_t beta, size_t xDesc,
+    size_t x, size_t yDesc, size_t y,
+    size_t bnScaleBiasMeanVarDesc, size_t bnScale,
+    size_t bnBias, double exponentialAverageFactor,
+    size_t resultRunningMean, size_t resultRunningVariance,
+    double epsilon, size_t resultSaveMean, size_t resultSaveInvVariance)
+
+cpdef batchNormalizationForwardInference(
+    intptr_t handle, int mode,
+    size_t alpha, size_t beta, size_t xDesc,
+    size_t x, size_t yDesc, size_t y,
+    size_t bnScaleBiasMeanVarDesc, size_t bnScale,
+    size_t bnBias, size_t estimatedMean, size_t estimatedVariance,
+    double epsilon)
+
+cpdef batchNormalizationBackward(
+    intptr_t handle, int mode,
+    size_t alphaDataDiff, size_t betaDataDiff,
+    size_t alphaParamDiff, size_t betaParamDiff,
+    size_t xDesc, size_t x, size_t dyDesc,
+    size_t dy, size_t dxDesc, size_t dx,
+    size_t dBnScaleBiasDesc, size_t bnScale,
+    size_t dBnScaleResult, size_t dBnBiasResult,
+    double epsilon, size_t savedMean, size_t savedInvVariance)
+
+cpdef batchNormalizationForwardTrainingEx(
+    intptr_t handle, int mode, int bnOps,
+    size_t alpha, size_t beta,
+    size_t xDesc, size_t x,
+    size_t zDesc, size_t z,
+    size_t yDesc, size_t y,
+    size_t bnScaleBiasMeanVarDesc,
+    size_t bnScale, size_t bnBias,
+    double exponentialAverageFactor,
+    size_t resultRunningMean, size_t resultRunningVariance,
+    double epsilon, size_t resultSaveMean, size_t resultSaveInvVariance,
+    size_t activationDesc,
+    size_t workSpace, size_t workSpaceSizeInBytes,
+    size_t reserveSpace, size_t reserveSpaceSizeInBytes)
+
+cpdef size_t getBatchNormalizationForwardTrainingExWorkspaceSize(
+    intptr_t handle, int mode, int bnOps,
+    size_t xDesc,
+    size_t zDesc,
+    size_t yDesc,
+    size_t bnScaleBiasMeanVarDesc,
+    size_t activationDesc) except? 0
+
+cpdef batchNormalizationBackwardEx(
+    intptr_t handle, int mode, int bnops,
+    size_t alphaDataDiff, size_t betaDataDiff,
+    size_t alphaParamDiff, size_t betaParamDiff,
+    size_t xDesc, size_t x,
+    size_t yDesc, size_t y,
+    size_t dyDesc, size_t dy,
+    size_t dzDesc, size_t dz,
+    size_t dxDesc, size_t dx,
+    size_t dBnScaleBiasDesc,
+    size_t bnScaleData, size_t bnBiasData,
+    size_t dBnScaleData, size_t dBnBiasData,
+    double epsilon,
+    size_t savedMean, size_t savedInvVariance,
+    size_t activationDesc,
+    size_t workSpace, size_t workSpaceSizeInBytes,
+    size_t reserveSpace, size_t reserveSpaceSizeInBytes)
+
+cpdef size_t getBatchNormalizationBackwardExWorkspaceSize(
+    intptr_t handle, int mode, int bnOps,
+    size_t xDesc,
+    size_t yDesc,
+    size_t dyDesc,
+    size_t dzDesc,
+    size_t dxDesc,
+    size_t dBnScaleBiasDesc,
+    size_t activationDesc) except? 0
+
+cpdef size_t getBatchNormalizationTrainingExReserveSpaceSize(
+    intptr_t handle, int mode, int bnOps,
+    size_t activationDesc,
+    size_t xDesc) except? 0
+
+
+###############################################################################
+# Activation
+###############################################################################
+
+cpdef size_t createActivationDescriptor() except? 0
+cpdef setActivationDescriptor(
+    size_t activationDesc, int mode, int reluNanOpt, double reluCeiling)
+cpdef destroyActivationDescriptor(size_t activationDesc)
+cpdef softmaxForward(
+    intptr_t handle, int algorithm, int mode, size_t alpha, size_t srcDesc,
+    size_t srcData, size_t beta, size_t dstDesc, size_t dstData)
+cpdef softmaxBackward(
+    intptr_t handle, int algorithm, int mode, size_t alpha, size_t srcDesc,
+    size_t srcData, size_t srcDiffDesc, size_t srcDiffData, size_t beta,
+    size_t destDiffDesc, size_t destDiffData)
+cpdef activationForward_v4(
+    intptr_t handle, size_t activationDesc, size_t alpha, size_t srcDesc,
+    size_t srcData, size_t beta, size_t dstDesc, size_t dstData)
+cpdef activationBackward_v4(
+    intptr_t handle, size_t activationDesc, size_t alpha, size_t srcDesc,
+    size_t srcData, size_t srcDiffDesc, size_t srcDiffData,
+    size_t destDesc, size_t destData, size_t beta, size_t destDiffDesc,
+    size_t destDiffData)
+
+
+###############################################################################
+# Dropout
+###############################################################################
+cpdef size_t createDropoutDescriptor() except? 0
+cpdef destroyDropoutDescriptor(size_t dropoutDesc)
+cpdef Py_ssize_t dropoutGetStatesSize(intptr_t handle) except? -1
+cpdef setDropoutDescriptor(
+    size_t dropoutDesc, intptr_t handle, float dropout,
+    size_t states, size_t stateSizeInBytes, unsigned long long seed)
+cpdef size_t getDropoutReserveSpaceSize(size_t xDesc) except? 0
+cpdef dropoutForward(
+    intptr_t handle, size_t dropoutDesc,
+    size_t srcDesc, size_t srcData,
+    size_t dstDesc, size_t dstData,
+    size_t reserveSpace, size_t reserveSpaceSizeInBytes)
+cpdef dropoutBackward(
+    intptr_t handle, size_t dropoutDesc,
+    size_t dyDesc, size_t dyData,
+    size_t dxtDesc, size_t dxData,
+    size_t reserveSpace, size_t reserveSpaceSizeInBytes)
+
+
+###############################################################################
+# CTC
+###############################################################################
+
+cpdef size_t createCTCLossDescriptor() except? 0
+cpdef destroyCTCLossDescriptor(size_t ctcLossDesc)
+cpdef setCTCLossDescriptor(size_t ctcLossDesc, int dataType)
+cpdef getCTCLossDescriptor(size_t ctcLossDesc)
+cpdef size_t getCTCLossWorkspaceSize(
+    intptr_t handle, size_t probsDesc, size_t gradientsDesc,
+    size_t labels, size_t labelLengths, size_t inputLengths,
+    int algo, size_t ctcLossDesc) except? 0
+cpdef CTCLoss(
+    intptr_t handle, size_t probsDesc,
+    size_t probs, size_t labels, size_t labelLengths, size_t inputLengths,
+    size_t costs, size_t gradientsDesc, size_t gradients, int algo,
+    size_t ctcLossDesc, size_t workspace, size_t workSpaceSizeInBytes)
+
+
+###############################################################################
+# RNN
+###############################################################################
+
+cpdef size_t createRNNDescriptor() except? 0
+cpdef destroyRNNDescriptor(size_t rnnDesc)
+cpdef size_t createPersistentRNNPlan(
+    size_t rnnDesc, int minibatch, int dataType) except? 0
+cpdef setPersistentRNNPlan(size_t rnnDesc, size_t plan)
+cpdef destroyPersistentRNNPlan(size_t plan)
+cpdef setRNNDescriptor_v5(
+    size_t rnnDesc, int hiddenSize, int numLayers,
+    size_t dropoutDesc, int inputMode, int direction, int mode,
+    int dataType)
+cpdef setRNNDescriptor_v6(
+    intptr_t handle, size_t rnnDesc, int hiddenSize, int numLayers,
+    size_t dropoutDesc, int inputMode, int direction, int mode,
+    int algo, int dataType)
+cpdef setRNNPaddingMode(size_t rnnDesc, int paddingMode)
+cpdef getRNNPaddingMode(size_t rnnDesc)
+cpdef size_t createRNNDataDescriptor() except? 0
+cpdef destroyRNNDataDescriptor(size_t RNNDataDesc)
+cpdef setRNNDataDescriptor(
+    size_t RNNDataDesc, int dataType, size_t layout,
+    int maxSeqLength, int batchSize, int vectorSize,
+    size_t seqLengthArray, size_t paddingFill)
+cpdef getRNNDataDescriptor(
+    size_t RNNDataDesc, size_t dataType,
+    size_t layout, size_t maxSeqLength, size_t batchSize,
+    size_t vectorSize, int arrayLengthRequested, size_t seqLengthArray,
+    size_t paddingFill)
+cpdef getRNNWorkspaceSize(
+    intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc)
+cpdef getRNNTrainingReserveSize(
+    intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc)
+cpdef getRNNParamsSize(
+    intptr_t handle, size_t rnnDesc, size_t xDesc, int dataType)
+cpdef getRNNLinLayerMatrixParams(
+    intptr_t handle, size_t rnnDesc, int layer, size_t xDesc, size_t wDesc,
+    size_t w, int linLayerID, size_t linLayerMatDesc, size_t linLayerMat)
+cpdef getRNNLinLayerBiasParams(
+    intptr_t handle, size_t rnnDesc, int layer, size_t xDesc, size_t wDesc,
+    size_t w, int linLayerID, size_t linLayerBiasDesc,
+    size_t linLayerBias)
+cpdef RNNForwardInference(
+    intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc,
+    size_t x, size_t hxDesc, size_t hx, size_t cxDesc,
+    size_t cx, size_t wDesc, size_t w, size_t yDesc,
+    size_t y, size_t hyDesc, size_t hy, size_t cyDesc,
+    size_t cy, size_t workspace, size_t workSpaceSizeInBytes)
+cpdef RNNForwardTraining(
+    intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc, size_t x,
+    size_t hxDesc, size_t hx, size_t cxDesc, size_t cx,
+    size_t wDesc, size_t w, size_t yDesc, size_t y,
+    size_t hyDesc, size_t hy, size_t cyDesc, size_t cy,
+    size_t workspace, size_t workSpaceSizeInBytes, size_t reserveSpace,
+    size_t reserveSpaceSizeInBytes)
+cpdef RNNBackwardData(
+    intptr_t handle, size_t rnnDesc, int seqLength, size_t yDesc, size_t y,
+    size_t dyDesc, size_t dy, size_t dhyDesc, size_t dhy,
+    size_t dcyDesc, size_t dcy, size_t wDesc, size_t w,
+    size_t hxDesc, size_t hx, size_t cxDesc, size_t cx,
+    size_t dxDesc, size_t dx, size_t dhxDesc, size_t dhx,
+    size_t dcxDesc, size_t dcx, size_t workspace,
+    size_t workSpaceSizeInBytes, size_t reserveSpace,
+    size_t reserveSpaceSizeInBytes)
+cpdef RNNBackwardWeights(
+    intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc, size_t x,
+    size_t hxDesc, size_t hx, size_t yDesc, size_t y,
+    size_t workspace, size_t workSpaceSizeInBytes, size_t dwDesc,
+    size_t dw, size_t reserveSpace, size_t reserveSpaceSizeInBytes)
+cpdef RNNForwardInferenceEx(
+    intptr_t handle, size_t rnnDesc, size_t xDesc, size_t x, size_t hxDesc,
+    size_t hx, size_t cxDesc, size_t cx, size_t wDesc, size_t w,
+    size_t yDesc, size_t y, size_t hyDesc, size_t hy, size_t cyDesc,
+    size_t cy, size_t kDesc, size_t keys, size_t cDesc, size_t cAttn,
+    size_t iDesc, size_t iAttn, size_t qDesc, size_t queries,
+    size_t workSpace, size_t workSpaceSizeInBytes)
+cpdef RNNForwardTrainingEx(
+    intptr_t handle, size_t rnnDesc, size_t xDesc, size_t x, size_t hxDesc,
+    size_t hx, size_t cxDesc, size_t cx, size_t wDesc, size_t w,
+    size_t yDesc, size_t y, size_t hyDesc, size_t hy, size_t cyDesc,
+    size_t cy, size_t kDesc, size_t keys, size_t cDesc, size_t cAttn,
+    size_t iDesc, size_t iAttn, size_t qDesc, size_t queries,
+    size_t workSpace, size_t workSpaceSizeInBytes,
+    size_t reserveSpace, size_t reserveSpaceSizeInBytes)
+cpdef RNNBackwardDataEx(
+    intptr_t handle, size_t rnnDesc, size_t yDesc, size_t y, size_t dyDesc,
+    size_t dy, size_t dcDesc, size_t dcAttn, size_t dhyDesc, size_t dhy,
+    size_t dcyDesc, size_t dcy, size_t wDesc, size_t w, size_t hxDesc,
+    size_t hx, size_t cxDesc, size_t cx, size_t dxDesc, size_t dx,
+    size_t dhxDesc, size_t dhx, size_t dcxDesc, size_t dcx,
+    size_t dkDesc, size_t dkeys,
+    size_t workSpace, size_t workSpaceSizeInBytes,
+    size_t reserveSpace, size_t reserveSpaceSizeInBytes)
+cpdef RNNBackwardWeightsEx(
+    intptr_t handle, size_t rnnDesc, size_t xDesc, size_t x,
+    size_t hxDesc, size_t hx, size_t yDesc, size_t y,
+    size_t workSpace, size_t workSpaceSizeInBytes,
+    size_t dwDesc, size_t dw,
+    size_t reserveSpace, size_t reserveSpaceSizeInBytes)
+
+
+###############################################################################
+# Spatial Transformer
+###############################################################################
+
+cpdef size_t createSpatialTransformerDescriptor() except? 0
+cpdef destroySpatialTransformerDescriptor(size_t stDesc)
+cpdef setSpatialTransformerDescriptor(
+    size_t stDesc, size_t samplerType, int dataType,
+    int nbDims, size_t dimA)
+cpdef spatialTfGridGeneratorForward(
+    intptr_t handle, size_t stDesc, size_t theta, size_t grid)
+cpdef spatialTfGridGeneratorBackward(
+    intptr_t handle, size_t stDesc, size_t dgrid, size_t dtheta)
+cpdef spatialTfSamplerForward(
+    intptr_t handle, size_t stDesc, size_t alpha, size_t xDesc,
+    size_t x, size_t grid, size_t beta, size_t yDesc, size_t y)
+cpdef spatialTfSamplerBackward(
+    intptr_t handle, size_t stDesc, size_t alpha, size_t xDesc,
+    size_t x, size_t beta, size_t dxDesc, size_t dx, size_t alphaDgrid,
+    size_t dyDesc, size_t dy, size_t grid, size_t betaDgrid, size_t dgrid)
+
+###############################################################################
+# Fused Ops
+###############################################################################
+
+cpdef createFusedOpsConstParamPack(int ops)
+cpdef destroyFusedOpsConstParamPack(size_t constPack)
+cpdef setFusedOpsConstParamPackAttribute(size_t constPack, int paramLabel,
+                                         size_t param)
+cpdef getFusedOpsConstParamPackAttribute(size_t constPack, int paramLabel,
+                                         size_t param)
+cpdef createFusedOpsVariantParamPack(int ops)
+cpdef destroyFusedOpsVariantParamPack(size_t varPack)
+cpdef setFusedOpsVariantParamPackAttribute(size_t varPack, int paramLabel,
+                                           size_t ptr)
+cpdef getFusedOpsVariantParamPackAttribute(size_t varPack, int paramLabel,
+                                           size_t ptr)
+cpdef createFusedOpsPlan(int ops)
+cpdef destroyFusedOpsPlan(size_t plan)
+cpdef makeFusedOpsPlan(intptr_t handle, size_t plan, size_t constPack)
+cpdef fusedOpsExecute(intptr_t handle, size_t plan, size_t varPack)
diff --git a/cupy_backends/cuda/libs/cudnn.pyx b/cupy_backends/cuda/libs/cudnn.pyx
new file mode 100644
index 0000000..464c59d
--- /dev/null
+++ b/cupy_backends/cuda/libs/cudnn.pyx
@@ -0,0 +1,2541 @@
+# distutils: language = c++
+
+"""Thin wrapper of cuDNN."""
+# NOTE: This wrapper does not cover all APIs of cuDNN v4.
+cimport cython  # NOQA
+from libcpp cimport vector
+
+from cupy_backends.cuda.api cimport driver
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda cimport stream as stream_module
+
+###############################################################################
+# Extern
+###############################################################################
+
+cdef extern from '../../cupy_cudnn.h' nogil:
+    # Types
+    ctypedef int ActivationMode 'cudnnActivationMode_t'
+    ctypedef int AddMode 'cudnnAddMode_t'
+    ctypedef int BatchNormMode 'cudnnBatchNormMode_t'
+    ctypedef int BatchNormOps 'cudnnBatchNormOps_t'
+    ctypedef int ConvolutionBwdDataAlgo 'cudnnConvolutionBwdDataAlgo_t'
+    ctypedef int ConvolutionBwdDataPreference \
+        'cudnnConvolutionBwdDataPreference_t'
+    ctypedef struct ConvolutionBwdDataAlgoPerf \
+        'cudnnConvolutionBwdDataAlgoPerf_t':  # NOQA: E125
+        int algo
+        int status
+        float time
+        size_t memory
+    ctypedef struct ConvolutionBwdDataAlgoPerf_v7 \
+        'cudnnConvolutionBwdDataAlgoPerf_v7_t':  # NOQA: E125
+        int algo
+        int status
+        float time
+        size_t memory
+        int determinism
+        int mathType
+    ctypedef int ConvolutionBwdFilterAlgo 'cudnnConvolutionBwdFilterAlgo_t'
+    ctypedef int ConvolutionBwdFilterPreference \
+        'cudnnConvolutionBwdFilterPreference_t'
+    ctypedef struct ConvolutionBwdFilterAlgoPerf \
+        'cudnnConvolutionBwdFilterAlgoPerf_t':  # NOQA: E125
+        int algo
+        int status
+        float time
+        size_t memory
+    ctypedef struct ConvolutionBwdFilterAlgoPerf_v7 \
+        'cudnnConvolutionBwdFilterAlgoPerf_v7_t':  # NOQA: E125
+        int algo
+        int status
+        float time
+        size_t memory
+        int determinism
+        int mathType
+    ctypedef int ConvolutionFwdAlgo 'cudnnConvolutionFwdAlgo_t'
+    ctypedef int ConvolutionFwdPreference 'cudnnConvolutionFwdPreference_t'
+    ctypedef struct ConvolutionFwdAlgoPerf 'cudnnConvolutionFwdAlgoPerf_t':
+        int algo
+        int status
+        float time
+        size_t memory
+    ctypedef struct ConvolutionFwdAlgoPerf_v7 \
+        'cudnnConvolutionFwdAlgoPerf_v7_t':  # NOQA: E125
+        int algo
+        int status
+        float time
+        size_t memory
+        int determinism
+        int mathType
+    ctypedef int ConvolutionMode 'cudnnConvolutionMode_t'
+    ctypedef int DataType 'cudnnDataType_t'
+    ctypedef int MathType 'cudnnMathType_t'
+    ctypedef int DirectionMode 'cudnnDirectionMode_t'
+    ctypedef int NanPropagation 'cudnnNanPropagation_t'
+    ctypedef int PoolingMode 'cudnnPoolingMode_t'
+    ctypedef int RNNInputMode 'cudnnRNNInputMode_t'
+    ctypedef int CTCLossAlgo 'cudnnCTCLossAlgo_t'
+    ctypedef int RNNMode 'cudnnRNNMode_t'
+    ctypedef int RNNAlgo 'cudnnRNNAlgo_t'
+    ctypedef int RNNDataLayout 'cudnnRNNDataLayout_t'
+    ctypedef int RNNPaddingMode 'cudnnRNNPaddingMode_t'
+    ctypedef int SoftmaxAlgorithm 'cudnnSoftmaxAlgorithm_t'
+    ctypedef int SoftmaxMode 'cudnnSoftmaxMode_t'
+    ctypedef int Status 'cudnnStatus_t'
+    ctypedef int TensorFormat 'cudnnTensorFormat_t'
+    ctypedef int OpTensorOp 'cudnnOpTensorOp_t'
+    ctypedef int ReduceTensorOp 'cudnnReduceTensorOp_t'
+    ctypedef int ReduceTensorIndices 'cudnnReduceTensorIndices_t'
+    ctypedef int IndicesType 'cudnnIndicesType_t'
+    ctypedef int ErrQueryMode 'cudnnErrQueryMode_t'
+    ctypedef int FusedOps 'cudnnFusedOps_t'
+    ctypedef int FusedOpsConstParamLabel 'cudnnFusedOpsConstParamLabel_t'
+    ctypedef int FusedOpsPointerPlaceHolder 'cudnnFusedOpsPointerPlaceHolder_t'
+    ctypedef int FusedOpsVariantParamLabel 'cudnnFusedOpsVariantParamLabel_t'
+    ctypedef struct RuntimeTag 'cudnnRuntimeTag_t'
+
+    ctypedef void* ActivationDescriptor 'cudnnActivationDescriptor_t'
+    ctypedef void* ConvolutionDescriptor 'cudnnConvolutionDescriptor_t'
+    ctypedef void* DropoutDescriptor 'cudnnDropoutDescriptor_t'
+    ctypedef void* FilterDescriptor 'cudnnFilterDescriptor_t'
+    ctypedef void* Handle 'cudnnHandle_t'
+    ctypedef void* PoolingDescriptor 'cudnnPoolingDescriptor_t'
+    ctypedef void* CTCLossDescriptor 'cudnnCTCLossDescriptor_t'
+    ctypedef void* RNNDescriptor 'cudnnRNNDescriptor_t'
+    ctypedef void* RNNDataDescriptor 'cudnnRNNDataDescriptor_t'
+    ctypedef void* PersistentRNNPlan 'cudnnPersistentRNNPlan_t'
+    ctypedef void* TensorDescriptor 'cudnnTensorDescriptor_t'
+    ctypedef void* OpTensorDescriptor 'cudnnOpTensorDescriptor_t'
+    ctypedef void* ReduceTensorDescriptor 'cudnnReduceTensorDescriptor_t'
+    ctypedef void* SpatialTransformerDescriptor \
+        'cudnnSpatialTransformerDescriptor_t'
+    ctypedef void* SamplerType 'cudnnSamplerType_t'
+    ctypedef void* FusedOpsConstParamPack 'cudnnFusedOpsConstParamPack_t'
+    ctypedef void* FusedOpsVariantParamPack 'cudnnFusedOpsVariantParamPack_t'
+    ctypedef void* FusedOpsPlan 'cudnnFusedOpsPlan_t'
+
+    # Error handling
+    const char* cudnnGetErrorString(Status status)
+
+    # Version
+    size_t cudnnGetVersion()
+
+    # Runtime error checking
+    int cudnnQueryRuntimeError(Handle handle, Status *rstatus,
+                               ErrQueryMode mode, RuntimeTag *tag)
+
+    # Initialization and CUDA cooperation
+    int cudnnCreate(Handle* handle)
+    int cudnnDestroy(Handle handle)
+    int cudnnSetStream(Handle handle, driver.Stream stream)
+    int cudnnGetStream(Handle handle, driver.Stream* stream)
+
+    # Tensor manipulation
+    int cudnnCreateTensorDescriptor(TensorDescriptor* descriptor)
+    int cudnnSetTensor4dDescriptor(
+        TensorDescriptor tensorDesc, TensorFormat format,
+        DataType dataType, int n, int c, int h, int w)
+    int cudnnSetTensor4dDescriptorEx(
+        TensorDescriptor tensorDesc, DataType dataType,
+        int n, int c, int h, int w,
+        int nStride, int cStride, int hStride, int wStride)
+    int cudnnGetTensor4dDescriptor(
+        TensorDescriptor tensorDesc, DataType* dataType,
+        int* n, int* c, int* h, int* w,
+        int* nStride, int* cStride, int* hStride, int* wStride)
+    int cudnnSetTensorNdDescriptor(
+        TensorDescriptor tensorDesc, DataType dataType, int nbDims,
+        int* dimA, int* strideA)
+    int cudnnDestroyTensorDescriptor(TensorDescriptor tensorDesc)
+    int cudnnAddTensor_v3(
+        Handle handle, void* alpha, TensorDescriptor bDesc,
+        void* b, void* beta, TensorDescriptor yDesc, void* y)
+
+    # Tensor operations
+    int cudnnCreateOpTensorDescriptor(OpTensorDescriptor* opTensorDesc)
+    int cudnnSetOpTensorDescriptor(
+        OpTensorDescriptor opTensorDesc, OpTensorOp opTensorOp,
+        DataType opTensorCompType, NanPropagation opTensorNanOpt)
+    int cudnnGetOpTensorDescriptor(
+        OpTensorDescriptor opTensorDesc, OpTensorOp* opTensorOp,
+        DataType* opTensorCompType, NanPropagation* opTensorNanOpt)
+    int cudnnDestroyOpTensorDescriptor(OpTensorDescriptor opTensorDesc)
+    int cudnnOpTensor(
+        Handle handle, OpTensorDescriptor opTensorDesc, void* alpha1,
+        TensorDescriptor aDesc, void* A, void* alpha2,
+        TensorDescriptor bDesc, void* B, void* beta,
+        TensorDescriptor cDesc, void* C)
+
+    # Tensor reductions
+    int cudnnCreateReduceTensorDescriptor(
+        ReduceTensorDescriptor* reduceTensorDesc)
+    int cudnnSetReduceTensorDescriptor(
+        ReduceTensorDescriptor reduceTensorDesc, ReduceTensorOp reduceTensorOp,
+        DataType reduceTensorCompType, NanPropagation reduceTensorNanOpt,
+        ReduceTensorIndices reduceTensorIndices,
+        IndicesType reduceTensorIndicesType)
+    int cudnnGetReduceTensorDescriptor(
+        ReduceTensorDescriptor reduceTensorDesc,
+        ReduceTensorOp* reduceTensorOp, DataType* reduceTensorCompType,
+        NanPropagation* reduceTensorNanOpt,
+        ReduceTensorIndices* reduceTensorIndices,
+        IndicesType* reduceTensorIndicesType)
+    int cudnnDestroyReduceTensorDescriptor(
+        ReduceTensorDescriptor reduceTensorDesc)
+    int cudnnGetReductionIndicesSize(
+        Handle handle, ReduceTensorDescriptor reduceTensorDesc,
+        TensorDescriptor aDesc, TensorDescriptor cDesc, size_t* sizeInBytes)
+    int cudnnGetReductionWorkspaceSize(
+        Handle handle, ReduceTensorDescriptor reduceTensorDesc,
+        TensorDescriptor aDesc, TensorDescriptor cDesc, size_t* sizeInBytes)
+    int cudnnReduceTensor(
+        Handle handle, ReduceTensorDescriptor reduceTensorDesc, void* indices,
+        size_t indicesSizeInBytes, void* workspace,
+        size_t workspaceSizeInBytes, void* alpha, TensorDescriptor aDesc,
+        void* A, void* beta, TensorDescriptor cDesc, void* c)
+    int cudnnSetTensor(
+        Handle handle, TensorDescriptor yDesc, void* y, void* valuePtr)
+    int cudnnScaleTensor(
+        Handle handle, TensorDescriptor yDesc, void* y, void* alpha)
+
+    # Filter manipulation
+    int cudnnCreateFilterDescriptor(FilterDescriptor* filterDesc)
+    int cudnnSetFilter4dDescriptor_v4(
+        FilterDescriptor filterDesc, DataType dataType,
+        TensorFormat format, int k, int c, int h, int w)
+    int cudnnSetFilterNdDescriptor_v4(
+        FilterDescriptor filterDesc, DataType dataType,
+        TensorFormat format, int nbDims, const int filterDimA[])
+    int cudnnGetFilterNdDescriptor_v4(
+        FilterDescriptor wDesc, int nbDimsRequested, DataType* dataType,
+        TensorFormat* format, int* nbDims, int filterDimA[])
+    int cudnnDestroyFilterDescriptor(FilterDescriptor filterDesc)
+
+    # Convolution
+    int cudnnCreateConvolutionDescriptor(ConvolutionDescriptor* convDesc)
+    int cudnnSetConvolutionMathType(
+        ConvolutionDescriptor convDesc, MathType mathType)
+    int cudnnGetConvolutionMathType(
+        ConvolutionDescriptor convDesc, MathType *mathType)
+    int cudnnSetConvolutionGroupCount(
+        ConvolutionDescriptor convDesc, int groupCount)
+    int cudnnGetConvolutionGroupCount(
+        ConvolutionDescriptor convDesc, int *groupCount)
+    int cudnnSetConvolution2dDescriptor_v4(
+        ConvolutionDescriptor convDesc, int pad_h, int pad_w, int u,
+        int v, int dilation_h, int dilation_w, ConvolutionMode mode)
+    int cudnnSetConvolution2dDescriptor_v5(
+        ConvolutionDescriptor convDesc, int pad_h, int pad_w, int u,
+        int v, int dilation_h, int dilation_w, ConvolutionMode mode,
+        DataType computeType)
+    int cudnnSetConvolutionNdDescriptor_v3(
+        ConvolutionDescriptor convDesc, int arrayLength, int* padA,
+        int* filterStrideA, int* dilationA, ConvolutionMode mode,
+        DataType dataType)
+    int cudnnDestroyConvolutionDescriptor(ConvolutionDescriptor conDesc)
+    int cudnnFindConvolutionForwardAlgorithm(
+        Handle handle, TensorDescriptor xDesc, FilterDescriptor wDesc,
+        ConvolutionDescriptor convDesc, TensorDescriptor yDesc,
+        int requestedAlgoCount, int* returnedAlgoCount,
+        ConvolutionFwdAlgoPerf* perfResults)
+    int cudnnFindConvolutionForwardAlgorithmEx(
+        Handle handle, TensorDescriptor xDesc, void* x,
+        FilterDescriptor wDesc, void* w, ConvolutionDescriptor convDesc,
+        TensorDescriptor yDesc, void* y, int requestedAlgoCount,
+        int* returnedAlgoCount, ConvolutionFwdAlgoPerf* perfResults,
+        void* workSpace, size_t workSpaceSizeInBytes)
+    int cudnnFindConvolutionForwardAlgorithmEx_v7(
+        Handle handle, TensorDescriptor xDesc, void* x,
+        FilterDescriptor wDesc, void* w, ConvolutionDescriptor convDesc,
+        TensorDescriptor yDesc, void* y, int requestedAlgoCount,
+        int* returnedAlgoCount, ConvolutionFwdAlgoPerf_v7* perfResults,
+        void* workSpace, size_t workSpaceSizeInBytes)
+    int cudnnGetConvolutionForwardAlgorithm_v6(
+        Handle handle, TensorDescriptor srcDesc,
+        FilterDescriptor filterDesc, ConvolutionDescriptor convDesc,
+        TensorDescriptor destDesc, ConvolutionFwdPreference preference,
+        size_t memoryLimitInbytes, ConvolutionFwdAlgo* algo)
+    int cudnnGetConvolutionForwardAlgorithm_v7(
+        Handle handle, TensorDescriptor srcDesc,
+        FilterDescriptor filterDesc, ConvolutionDescriptor convDesc,
+        TensorDescriptor destDesc, int requestedAlgoCount,
+        int* returnedAlgoCount, ConvolutionFwdAlgoPerf_v7* perfResults)
+    int cudnnGetConvolutionForwardWorkspaceSize(
+        Handle handle, TensorDescriptor srcDesc,
+        FilterDescriptor filterDesc, ConvolutionDescriptor convDesc,
+        TensorDescriptor destDesc, ConvolutionFwdAlgo algo,
+        size_t* sizeInBytes)
+    int cudnnConvolutionForward(
+        Handle handle, void* alpha, TensorDescriptor srcDesc,
+        void* srcData, FilterDescriptor filterDesc, void* filterData,
+        ConvolutionDescriptor convDesc, ConvolutionFwdAlgo algo,
+        void* workSpace, size_t workSpaceSizeInBytes, void* beta,
+        TensorDescriptor destDesc, void* destData)
+    int cudnnConvolutionBackwardBias(
+        Handle handle, void* alpha,
+        TensorDescriptor srcDesc, void* srcData, void* beta,
+        TensorDescriptor destDesc, void* destData)
+    int cudnnFindConvolutionBackwardFilterAlgorithm(
+        Handle handle, TensorDescriptor xDesc, TensorDescriptor dyDesc,
+        ConvolutionDescriptor convDesc, FilterDescriptor dwDesc,
+        int requestedAlgoCount, int* returnedAlgoCount,
+        ConvolutionBwdFilterAlgoPerf* perfResults)
+    int cudnnFindConvolutionBackwardFilterAlgorithmEx(
+        Handle handle, TensorDescriptor xDesc, void* x,
+        TensorDescriptor dyDesc, void* dy, ConvolutionDescriptor convDesc,
+        FilterDescriptor dwDesc, void* dw, int requestedAlgoCount,
+        int* returnedAlgoCount, ConvolutionBwdFilterAlgoPerf* perfResults,
+        void* workSpace, size_t workSpaceSizeInBytes)
+    int cudnnFindConvolutionBackwardFilterAlgorithmEx_v7(
+        Handle handle, TensorDescriptor xDesc, void* x,
+        TensorDescriptor dyDesc, void* dy, ConvolutionDescriptor convDesc,
+        FilterDescriptor dwDesc, void* dw, int requestedAlgoCount,
+        int* returnedAlgoCount, ConvolutionBwdFilterAlgoPerf_v7* perfResults,
+        void* workSpace, size_t workSpaceSizeInBytes)
+    int cudnnGetConvolutionBackwardFilterAlgorithm_v6(
+        Handle handle, TensorDescriptor srcDesc, TensorDescriptor diffDesc,
+        ConvolutionDescriptor convDesc, FilterDescriptor filterDesc,
+        ConvolutionBwdFilterPreference preference,
+        size_t memoryLimitInbytes, ConvolutionBwdFilterAlgo* algo)
+    int cudnnGetConvolutionBackwardFilterAlgorithm_v7(
+        Handle handle, TensorDescriptor srcDesc, TensorDescriptor diffDesc,
+        ConvolutionDescriptor convDesc, FilterDescriptor gradDesc,
+        int requestedAlgoCount, int* returnedAlgoCount,
+        ConvolutionBwdFilterAlgoPerf_v7* perfResults)
+    int cudnnGetConvolutionBackwardFilterWorkspaceSize(
+        Handle handle, TensorDescriptor srcDesc, TensorDescriptor diffDesc,
+        ConvolutionDescriptor convDesc, FilterDescriptor filterDesc,
+        ConvolutionBwdFilterAlgo algo, size_t* sizeInBytes)
+    int cudnnConvolutionBackwardFilter_v3(
+        Handle handle, void* alpha,
+        TensorDescriptor srcDesc, void* srcData,
+        TensorDescriptor diffDesc, void* diffData,
+        ConvolutionDescriptor convDesc, ConvolutionBwdFilterAlgo algo,
+        void* workSpace, size_t workSpaceSizeInBytes, void* beta,
+        FilterDescriptor gradDesc, void* gradData)
+    int cudnnGetConvolutionBackwardDataAlgorithm_v6(
+        Handle handle, FilterDescriptor filterDesc,
+        TensorDescriptor diffDesc,
+        ConvolutionDescriptor convDesc, TensorDescriptor gradDesc,
+        ConvolutionBwdDataPreference preference,
+        size_t memoryLimitInbytes, ConvolutionBwdDataAlgo* algo)
+    int cudnnGetConvolutionBackwardDataAlgorithm_v7(
+        Handle handle, TensorDescriptor filterDesc, TensorDescriptor diffDesc,
+        ConvolutionDescriptor convDesc, FilterDescriptor gradDesc,
+        int requestedAlgoCount, int* returnedAlgoCount,
+        ConvolutionBwdDataAlgoPerf_v7* perfResults)
+    int cudnnFindConvolutionBackwardDataAlgorithm(
+        Handle handle, TensorDescriptor wDesc, TensorDescriptor dyDesc,
+        ConvolutionDescriptor convDesc, FilterDescriptor dxDesc,
+        int requestedAlgoCount, int* returnedAlgoCount,
+        ConvolutionBwdDataAlgoPerf* perfResults)
+    int cudnnFindConvolutionBackwardDataAlgorithmEx(
+        Handle handle, FilterDescriptor wDesc, void* w,
+        TensorDescriptor dyDesc, void* dy, ConvolutionDescriptor convDesc,
+        TensorDescriptor dxDesc, void* dx, int requestedAlgoCount,
+        int* returnedAlgoCount, ConvolutionBwdDataAlgoPerf* perfResults,
+        void* workSpace, size_t workSpaceSizeInBytes)
+    int cudnnFindConvolutionBackwardDataAlgorithmEx_v7(
+        Handle handle, FilterDescriptor wDesc, void* w,
+        TensorDescriptor dyDesc, void* dy, ConvolutionDescriptor convDesc,
+        TensorDescriptor dxDesc, void* dx, int requestedAlgoCount,
+        int* returnedAlgoCount, ConvolutionBwdDataAlgoPerf_v7* perfResults,
+        void* workSpace, size_t workSpaceSizeInBytes)
+    int cudnnGetConvolutionBackwardDataWorkspaceSize(
+        Handle handle, FilterDescriptor filterDesc,
+        TensorDescriptor diffDesc,
+        ConvolutionDescriptor convDesc, TensorDescriptor gradDesc,
+        ConvolutionBwdDataAlgo algo, size_t* sizeInBytes)
+    int cudnnConvolutionBackwardData_v3(
+        Handle handle, void* alpha,
+        FilterDescriptor filterDesc, void* filterData,
+        TensorDescriptor diffDesc, void* diffData,
+        ConvolutionDescriptor convDesc, ConvolutionBwdDataAlgo algo,
+        void* workSpace, size_t workSpaceSizeInBytes, void* beta,
+        TensorDescriptor gradDesc, void* gradData)
+
+    # Pooling
+    int cudnnCreatePoolingDescriptor(PoolingDescriptor* desc)
+    int cudnnSetPooling2dDescriptor_v4(
+        PoolingDescriptor poolingDesc, PoolingMode mode,
+        NanPropagation maxpoolingNanOpt, int windowHeight, int windowWidth,
+        int verticalPadding, int horizontalPadding, int verticalStride,
+        int horizontalStride)
+    int cudnnSetPoolingNdDescriptor_v4(
+        PoolingDescriptor poolingDesc, PoolingMode mode,
+        NanPropagation maxpoolingNanOpt, int nbDims,
+        int* windowDimA, int* paddingA, int* strideA)
+    int cudnnDestroyPoolingDescriptor(PoolingDescriptor poolingDesc)
+    int cudnnPoolingForward(
+        Handle handle, PoolingDescriptor poolingDesc, void* alpha,
+        TensorDescriptor srcDesc, void* srcData, void* beta,
+        TensorDescriptor dstDesc, void* dstData)
+    int cudnnPoolingBackward(
+        Handle handle, PoolingDescriptor poolingDesc, void* alpha,
+        TensorDescriptor srcDesc, void* srcData,
+        TensorDescriptor srcDiffDesc, void* srcDiffData,
+        TensorDescriptor destDesc, void* destData, void* beta,
+        TensorDescriptor destDiffDesc, void* destDiffData)
+
+    # Batch Normalization
+    int cudnnDeriveBNTensorDescriptor(
+        TensorDescriptor derivedBnDesc, TensorDescriptor xDesc,
+        BatchNormMode mode)
+    int cudnnBatchNormalizationForwardTraining(
+        Handle handle, BatchNormMode mode,
+        void* alpha, void* beta, TensorDescriptor xDesc,
+        void* x, TensorDescriptor yDesc, void* y,
+        TensorDescriptor bnScaleBiasMeanVarDesc, void* bnScale,
+        void* bnBias, double exponentialAverageFactor,
+        void* resultRunningMean, void* resultRunningVariance,
+        double epsilon, void* resultSaveMean,
+        void* resultSaveInvVariance)
+    int cudnnBatchNormalizationForwardInference(
+        Handle handle, BatchNormMode mode,
+        void* alpha, void* beta, TensorDescriptor xDesc,
+        void* x, TensorDescriptor yDesc, void* y,
+        TensorDescriptor bnScaleBiasMeanVarDesc, void* bnScale,
+        void* bnBias, void* estimatedMean, void* estimatedVariance,
+        double epsilon)
+    int cudnnBatchNormalizationBackward(
+        Handle handle, BatchNormMode mode,
+        void* alphaDataDiff, void* betaDataDiff,
+        void* alphaParamDiff, void* betaParamDiff,
+        TensorDescriptor xDesc, void* x,
+        TensorDescriptor dyDesc, void* dy,
+        TensorDescriptor dxDesc, void* dx,
+        TensorDescriptor dBnScaleBiasDesc, void* bnScale,
+        void* dBnScaleResult, void* dBnBiasResult,
+        double epsilon, void* savedMean, void* savedInvVariance)
+
+    int cudnnBatchNormalizationForwardTrainingEx(
+        Handle handle,
+        BatchNormMode mode, BatchNormOps bnOps,
+        void* alpha, void* beta,
+        TensorDescriptor xDesc, void* x,
+        TensorDescriptor zDesc, void* z,
+        TensorDescriptor yDesc, void* y,
+        TensorDescriptor bnScaleBiasMeanVarDesc,
+        void* bnScale, void* bnBias,
+        double exponentialAverageFactor,
+        void* resultRunningMean, void* resultRunningVariance,
+        double epsilon,
+        void* resultSaveMean, void* resultSaveInvVariance,
+        ActivationDescriptor activationDesc,
+        void* workspace, size_t workSpaceSizeInBytes,
+        void* reserveSpace, size_t reserveSpaceSizeInBytes)
+    int cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(
+        Handle handle,
+        BatchNormMode mode, BatchNormOps bnOps,
+        TensorDescriptor xDesc,
+        TensorDescriptor zDesc,
+        TensorDescriptor yDesc,
+        TensorDescriptor bnScaleBiasMeanVarDesc,
+        ActivationDescriptor activationDesc,
+        size_t* sizeInBytes)
+    int cudnnBatchNormalizationBackwardEx(
+        Handle handle,
+        BatchNormMode mode, BatchNormOps bnops,
+        void* alphaDataDiff, void* betaDataDiff,
+        void* alphaParamDiff, void* betaParamDiff,
+        TensorDescriptor xDesc, void* x,
+        TensorDescriptor yDesc, void* y,
+        TensorDescriptor dyDesc, void* dy,
+        TensorDescriptor dzDesc, void* dz,
+        TensorDescriptor dxDesc, void* dx,
+        TensorDescriptor dBnScaleBiasDesc,
+        void* bnScaleData, void* bnBiasData,
+        void* dBnScaleData, void* dBnBiasData,
+        double epsilon,
+        void* savedMean, void* savedInvVariance,
+        ActivationDescriptor activationDesc,
+        void* workspace, size_t workSpaceSizeInBytes,
+        void* reserveSpace, size_t reserveSpaceSizeInBytes)
+    int cudnnGetBatchNormalizationBackwardExWorkspaceSize(
+        Handle handle,
+        BatchNormMode mode,
+        BatchNormOps bnOps,
+        TensorDescriptor xDesc,
+        TensorDescriptor yDesc,
+        TensorDescriptor dyDesc,
+        TensorDescriptor dzDesc,
+        TensorDescriptor dxDesc,
+        TensorDescriptor dBnScaleBiasDesc,
+        ActivationDescriptor activationDesc,
+        size_t* sizeInBytes)
+    int cudnnGetBatchNormalizationTrainingExReserveSpaceSize(
+        Handle handle,
+        BatchNormMode mode,
+        BatchNormOps bnOps,
+        ActivationDescriptor activationDesc,
+        TensorDescriptor xDesc,
+        size_t* sizeInBytes)
+
+    # Activation
+    int cudnnCreateActivationDescriptor(
+        ActivationDescriptor* activationDesc)
+    int cudnnSetActivationDescriptor(
+        ActivationDescriptor activationDesc, ActivationMode mode,
+        NanPropagation reluNanOpt, double reluCeiling)
+    int cudnnDestroyActivationDescriptor(
+        ActivationDescriptor activationDesc)
+    int cudnnSoftmaxForward(
+        Handle handle, SoftmaxAlgorithm algorithm, SoftmaxMode mode,
+        void* alpha, TensorDescriptor srcDesc, void* srcData,
+        void* beta, TensorDescriptor dstDesc, void* dstData)
+    int cudnnSoftmaxBackward(
+        Handle handle, SoftmaxAlgorithm algorithm, SoftmaxMode mode,
+        void* alpha, TensorDescriptor srcDesc, void* srcData,
+        TensorDescriptor srcDiffDesc, void* srcDiffData, void* beta,
+        TensorDescriptor destDiffDesc, void* destDiffData)
+    int cudnnActivationForward_v4(
+        Handle handle, ActivationDescriptor activationDesc, void* alpha,
+        TensorDescriptor srcDesc, void* srcData, void* beta,
+        TensorDescriptor dstDesc, void* dstData)
+    int cudnnActivationBackward_v4(
+        Handle handle, ActivationDescriptor activationDesc, void* alpha,
+        TensorDescriptor srcDesc, void* srcData,
+        TensorDescriptor srcDiffDesc, void* srcDiffData,
+        TensorDescriptor destDesc, void* destData, void* beta,
+        TensorDescriptor destDiffDesc, void* destDiffData)
+
+    # Dropout
+    int cudnnCreateDropoutDescriptor(DropoutDescriptor* desc)
+    int cudnnDestroyDropoutDescriptor(DropoutDescriptor dropoutDesc)
+    int cudnnDropoutGetStatesSize(Handle handle, size_t* sizeInBytes)
+    int cudnnDropoutGetReserveSpaceSize(
+        TensorDescriptor xDesc, size_t* sizeInBytes)
+    int cudnnSetDropoutDescriptor(
+        DropoutDescriptor dropoutDesc, Handle handle, float dropout,
+        void* states, size_t stateSizeInBytes, unsigned long long seed)
+    int cudnnDropoutForward(
+        Handle handle, DropoutDescriptor dropoutDesc,
+        TensorDescriptor srcDesc, void* srcData,
+        TensorDescriptor dstDesc, void* dstData,
+        void* reserveSpace, size_t reserveSpaceSizeInBytes)
+    int cudnnDropoutBackward(
+        Handle handle, DropoutDescriptor dropoutDesc,
+        TensorDescriptor dydesc, void* dy, TensorDescriptor dxdesc,
+        void* dx, void* reserveSpace, size_t reserveSpaceSizeInBytes)
+
+    # CTC
+    int cudnnCreateCTCLossDescriptor(CTCLossDescriptor* ctcLossDesc)
+    int cudnnDestroyCTCLossDescriptor(CTCLossDescriptor ctcLossDesc)
+    int cudnnSetCTCLossDescriptor(
+        CTCLossDescriptor ctcLossDesc, DataType dataType)
+    int cudnnGetCTCLossDescriptor(
+        CTCLossDescriptor ctcLossDesc, DataType* dataType)
+    int cudnnGetCTCLossWorkspaceSize(
+        Handle handle, TensorDescriptor probsDesc,
+        TensorDescriptor gradientsDesc, int* labels,
+        int* labelLengths, int* inputLengths, CTCLossAlgo algo,
+        CTCLossDescriptor ctcLossDesc, size_t* sizeInBytes)
+    int cudnnCTCLoss(
+        Handle handle, TensorDescriptor probsDesc,
+        void* probs, int* labels, int* labelLengths, int* inputLengths,
+        void* costs, TensorDescriptor gradientsDesc, void* gradients,
+        CTCLossAlgo algo, CTCLossDescriptor ctcLossDesc,
+        void* workspace, size_t workSpaceSizeInBytes)
+    # RNN
+    int cudnnCreateRNNDescriptor(RNNDescriptor* rnnDesc)
+    int cudnnDestroyRNNDescriptor(RNNDescriptor rnnDesc)
+    int cudnnCreatePersistentRNNPlan(
+        RNNDescriptor rnnDesc,
+        const int minibatch, DataType dataType,
+        PersistentRNNPlan* plan)
+    int cudnnSetPersistentRNNPlan(
+        RNNDescriptor rnnDesc, PersistentRNNPlan plan)
+    int cudnnDestroyPersistentRNNPlan(PersistentRNNPlan plan)
+    int cudnnSetRNNDescriptor_v5(
+        RNNDescriptor rnnDesc, int hiddenSize,
+        int numLayers, DropoutDescriptor dropoutDesc, RNNInputMode inputMode,
+        DirectionMode direction, RNNMode mode, DataType dataType)
+    int cudnnSetRNNDescriptor_v6(
+        Handle handle, RNNDescriptor rnnDesc, int hiddenSize,
+        int numLayers, DropoutDescriptor dropoutDesc, RNNInputMode inputMode,
+        DirectionMode direction, RNNMode mode, RNNAlgo algo, DataType dataType)
+    int cudnnSetRNNPaddingMode(
+        RNNDescriptor rnnDesc, RNNPaddingMode paddingMode)
+    int cudnnGetRNNPaddingMode(
+        RNNDescriptor rnnDesc, RNNPaddingMode* paddingMode)
+    int cudnnCreateRNNDataDescriptor(RNNDataDescriptor* RNNDataDesc)
+    int cudnnDestroyRNNDataDescriptor(RNNDataDescriptor RNNDataDesc)
+    int cudnnSetRNNDataDescriptor(
+        RNNDataDescriptor RNNDataDesc, DataType dataType, RNNDataLayout layout,
+        int maxSeqLength, int batchSize, int vectorSize,
+        const int seqLengthArray[], void *paddingFill)
+    int cudnnGetRNNDataDescriptor(
+        RNNDataDescriptor RNNDataDesc, DataType* dataType,
+        RNNDataLayout* layout, int* maxSeqLength, int* batchSize,
+        int* vectorSize, int arrayLengthRequested, int seqLengthArray[],
+        void* paddingFill)
+    int cudnnGetRNNWorkspaceSize(
+        Handle handle, RNNDescriptor rnnDesc, int seqLength,
+        TensorDescriptor* xDesc, size_t* sizeInBytes)
+    int cudnnGetRNNTrainingReserveSize(
+        Handle handle, RNNDescriptor rnnDesc, int seqLength,
+        TensorDescriptor* xDesc, size_t* sizeInBytes)
+    int cudnnGetRNNParamsSize(
+        Handle handle, RNNDescriptor rnnDesc, TensorDescriptor xDesc,
+        size_t* sizeInBytes, DataType dataType)
+    int cudnnGetRNNLinLayerMatrixParams(
+        Handle handle, RNNDescriptor rnnDesc, int layer,
+        TensorDescriptor xDesc, FilterDescriptor wDesc, void* w,
+        int linLayerID, FilterDescriptor linLayerMatDesc,
+        void** linLayerMat)
+    int cudnnGetRNNLinLayerBiasParams(
+        Handle handle, RNNDescriptor rnnDesc, int layer,
+        TensorDescriptor xDesc, FilterDescriptor wDesc, void* w,
+        int linLayerID, FilterDescriptor linLayerBiasDesc,
+        void** linLayerBias)
+    int cudnnRNNForwardInference(
+        Handle handle, RNNDescriptor rnnDesc, int seqLength,
+        TensorDescriptor* xDesc,
+        void* x, TensorDescriptor hxDesc, void* hx, TensorDescriptor cxDesc,
+        void* cx, FilterDescriptor wDesc, void* w, TensorDescriptor* yDesc,
+        void* y, TensorDescriptor hyDesc, void* hy, TensorDescriptor cyDesc,
+        void* cy, void* workspace, size_t workSpaceSizeInBytes)
+    int cudnnRNNForwardTraining(
+        Handle handle, RNNDescriptor rnnDesc, int seqLength,
+        TensorDescriptor* xDesc, void* x,
+        TensorDescriptor hxDesc, void* hx, TensorDescriptor cxDesc, void* cx,
+        FilterDescriptor wDesc, void* w, TensorDescriptor* yDesc, void* y,
+        TensorDescriptor hyDesc, void* hy, TensorDescriptor cyDesc, void* cy,
+        void* workspace, size_t workSpaceSizeInBytes, void* reserveSpace,
+        size_t reserveSpaceSizeInBytes)
+    int cudnnRNNBackwardData(
+        Handle handle, RNNDescriptor rnnDesc, int seqLength,
+        TensorDescriptor* yDesc, void* y,
+        TensorDescriptor* dyDesc, void* dy,
+        TensorDescriptor dhyDesc, void* dhy,
+        TensorDescriptor dcyDesc, void* dcy,
+        FilterDescriptor wDesc, void* w,
+        TensorDescriptor hxDesc, void* hx,
+        TensorDescriptor cxDesc, void* cx,
+        TensorDescriptor* dxDesc, void* dx,
+        TensorDescriptor dhxDesc, void* dhx,
+        TensorDescriptor dcxDesc, void* dcx, void* workspace,
+        size_t workSpaceSizeInBytes, void* reserveSpace,
+        size_t reserveSpaceSizeInBytes)
+    int cudnnRNNBackwardWeights(
+        Handle handle, RNNDescriptor rnnDesc, int seqLength,
+        TensorDescriptor* xDesc, void* x, TensorDescriptor hxDesc, void* hx,
+        TensorDescriptor* yDesc, void* y,
+        void* workspace, size_t workSpaceSizeInBytes, FilterDescriptor dwDesc,
+        void* dw, void* reserveSpace, size_t reserveSpaceSizeInBytes)
+
+    int cudnnRNNForwardInferenceEx(
+        Handle handle, RNNDescriptor rnnDesc,
+        RNNDataDescriptor xDesc, const void* x,
+        TensorDescriptor hxDesc, const void* hx,
+        TensorDescriptor cxDesc, const void* cx,
+        FilterDescriptor wDesc, const void* w,
+        RNNDataDescriptor yDesc, void* y,
+        TensorDescriptor hyDesc, void* hy,
+        TensorDescriptor cyDesc, void* cy,
+        RNNDataDescriptor kDesc, const void* keys,
+        RNNDataDescriptor cDesc, void* cAttn,
+        RNNDataDescriptor iDesc, void* iAttn,
+        RNNDataDescriptor qDesc, void* queries,
+        void* workSpace, size_t workSpaceSizeInBytes)
+    int cudnnRNNForwardTrainingEx(
+        Handle handle, RNNDescriptor rnnDesc,
+        RNNDataDescriptor xDesc, const void* x,
+        TensorDescriptor hxDesc, const void* hx,
+        TensorDescriptor cxDesc, const void* cx,
+        FilterDescriptor wDesc, const void* w,
+        RNNDataDescriptor yDesc, void* y,
+        TensorDescriptor hyDesc, void* hy,
+        TensorDescriptor cyDesc, void* cy,
+        RNNDataDescriptor kDesc, const void* keys,
+        RNNDataDescriptor cDesc, void* cAttn,
+        RNNDataDescriptor iDesc, void* iAttn,
+        RNNDataDescriptor qDesc, void* queries,
+        void* workSpace, size_t workSpaceSizeInBytes,
+        void* reserveSpace, size_t reserveSpaceSizeInBytes)
+    int cudnnRNNBackwardDataEx(
+        Handle handle, RNNDescriptor rnnDesc,
+        RNNDataDescriptor yDesc, const void* y,
+        RNNDataDescriptor dyDesc, const void* dy,
+        RNNDataDescriptor dcDesc, const void* dcAttn,
+        TensorDescriptor dhyDesc, const void* dhy,
+        TensorDescriptor dcyDesc, const void* dcy,
+        FilterDescriptor wDesc, const void* w,
+        TensorDescriptor hxDesc, const void* hx,
+        TensorDescriptor cxDesc, const void* cx,
+        RNNDataDescriptor dxDesc, void* dx,
+        TensorDescriptor dhxDesc, void* dhx,
+        TensorDescriptor dcxDesc, void* dcx,
+        RNNDataDescriptor dkDesc, void* dkeys,
+        void* workSpace, size_t workSpaceSizeInBytes,
+        void* reserveSpace, size_t reserveSpaceSizeInBytes)
+    int cudnnRNNBackwardWeightsEx(
+        Handle handle, RNNDescriptor rnnDesc,
+        RNNDataDescriptor xDesc, const void* x,
+        TensorDescriptor hxDesc, const void* hx,
+        RNNDataDescriptor yDesc, const void* y,
+        void* workSpace, size_t workSpaceSizeInBytes,
+        FilterDescriptor dwDesc, void* dw,
+        void* reserveSpace, size_t reserveSpaceSizeInBytes)
+
+    # Spatial Transformer
+    int cudnnCreateSpatialTransformerDescriptor(
+        SpatialTransformerDescriptor* stDesc)
+    int cudnnDestroySpatialTransformerDescriptor(
+        SpatialTransformerDescriptor stDesc)
+    int cudnnSetSpatialTransformerNdDescriptor(
+        SpatialTransformerDescriptor stDesc, SamplerType samplerType,
+        DataType dataType, int nbDims, int dimA[])
+    int cudnnSpatialTfGridGeneratorForward(
+        Handle handle, SpatialTransformerDescriptor stDesc,
+        void* theta, void* grid)
+    int cudnnSpatialTfGridGeneratorBackward(
+        Handle handle, SpatialTransformerDescriptor stDesc,
+        void* dgrid, void* dtheta)
+    int cudnnSpatialTfSamplerForward(
+        Handle handle, SpatialTransformerDescriptor stDesc,
+        void* alpha, TensorDescriptor xDesc, void* x,
+        void* grid, void* beta, TensorDescriptor yDesc, void* y)
+    int cudnnSpatialTfSamplerBackward(
+        Handle handle, SpatialTransformerDescriptor stDesc,
+        void* alpha, TensorDescriptor xDesc, void* x, void* beta,
+        TensorDescriptor dxDesc, void* dx, void* alphaDgrid,
+        TensorDescriptor dyDesc, void* dy, void* grid,
+        void* betaDgrid, void* dgrid)
+
+    # Fused Ops
+    int cudnnCreateFusedOpsConstParamPack(
+        FusedOpsConstParamPack* constPack, int ops)
+    int cudnnDestroyFusedOpsConstParamPack(FusedOpsConstParamPack constPack)
+    int cudnnSetFusedOpsConstParamPackAttribute(
+        FusedOpsConstParamPack constPack, FusedOpsConstParamLabel paramLabel,
+        const void *param)
+    int cudnnGetFusedOpsConstParamPackAttribute(
+        const FusedOpsConstParamPack constPack,
+        FusedOpsConstParamLabel paramLabel, void *param, int *isNULL)
+    int cudnnCreateFusedOpsVariantParamPack(
+        FusedOpsVariantParamPack *varPack, FusedOps ops)
+    int cudnnDestroyFusedOpsVariantParamPack(FusedOpsVariantParamPack varPack)
+    int cudnnSetFusedOpsVariantParamPackAttribute(
+        FusedOpsVariantParamPack varPack, FusedOpsVariantParamLabel paramLabel,
+        void *ptr)
+    int cudnnGetFusedOpsVariantParamPackAttribute(
+        const FusedOpsVariantParamPack varPack,
+        FusedOpsVariantParamLabel paramLabel, void *ptr)
+    int cudnnCreateFusedOpsPlan(FusedOpsPlan *plan, FusedOps ops)
+    int cudnnDestroyFusedOpsPlan(FusedOpsPlan plan)
+    int cudnnMakeFusedOpsPlan(
+        Handle handle, FusedOpsPlan plan,
+        const FusedOpsConstParamPack constPack, size_t *workspaceSizeInBytes)
+    int cudnnFusedOpsExecute(
+        Handle handle, const FusedOpsPlan plan,
+        FusedOpsVariantParamPack varPack)
+
+    # Build-time version
+    int CUDNN_VERSION
+
+    # Constants
+    double _CUDNN_BN_MIN_EPSILON 'CUDNN_BN_MIN_EPSILON'
+
+
+cdef class CuDNNAlgoPerf:
+
+    def __init__(self, algo, status, time, memory, determinism, mathType):
+        self.algo = algo
+        self.status = status
+        self.time = time
+        self.memory = memory
+        self.determinism = determinism
+        self.mathType = mathType
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+class CuDNNError(RuntimeError):
+
+    def __init__(self, int status):
+        self.status = status
+        msg = cudnnGetErrorString(<Status>status)
+        super(CuDNNError, self).__init__(
+            'cuDNN Error: {}'.format(msg.decode()))
+        self._infos = []
+
+    def add_info(self, info):
+        assert isinstance(info, str)
+        self._infos.append(info)
+
+    def add_infos(self, infos):
+        assert isinstance(infos, list)
+        self._infos.extend(infos)
+
+    def __str__(self):
+        base = super(CuDNNError, self).__str__()
+        return base + ''.join(
+            '\n  ' + info for info in self._infos)
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        raise CuDNNError(status)
+
+
+###############################################################################
+# Build-time version
+###############################################################################
+
+def get_build_version():
+    return CUDNN_VERSION
+
+
+###############################################################################
+# Version
+###############################################################################
+
+cpdef size_t getVersion() except? 0:
+    return cudnnGetVersion()
+
+
+###############################################################################
+# Runtime error checking
+###############################################################################
+
+cpdef queryRuntimeError(intptr_t handle, int mode):
+    cdef Status rstatus
+    with nogil:
+        status = cudnnQueryRuntimeError(<Handle>handle, &rstatus,
+                                        <ErrQueryMode>mode, <RuntimeTag*>0)
+    check_status(status)
+    return rstatus
+
+
+###############################################################################
+# Initialization and CUDA cooperation
+###############################################################################
+
+cpdef intptr_t create() except? 0:
+    cdef Handle handle
+    with nogil:
+        status = cudnnCreate(&handle)
+    check_status(status)
+    return <intptr_t>handle
+
+
+cpdef destroy(intptr_t handle):
+    with nogil:
+        status = cudnnDestroy(<Handle>handle)
+    check_status(status)
+
+
+cpdef setStream(intptr_t handle, size_t stream):
+    # TODO(leofang): The support of stream capture is not mentioned at all in
+    # the cuDNN docs (as of CUDA 11.5), so we disable this functionality.
+    if not runtime._is_hip_environment and runtime.streamIsCapturing(stream):
+        raise NotImplementedError(
+            'calling cuDNN API during stream capture is currently '
+            'unsupported')
+
+    status = cudnnSetStream(<Handle>handle, <driver.Stream>stream)
+    check_status(status)
+
+
+cpdef size_t getStream(intptr_t handle) except? 0:
+    cdef driver.Stream stream
+    status = cudnnGetStream(<Handle>handle, &stream)
+    check_status(status)
+    return <size_t>stream
+
+
+cdef _setStream(intptr_t handle):
+    """Set current stream"""
+    setStream(handle, stream_module.get_current_stream_ptr())
+
+###############################################################################
+# Tensor manipulation
+###############################################################################
+
+cpdef size_t createTensorDescriptor() except? 0:
+    cdef TensorDescriptor descriptor
+    status = cudnnCreateTensorDescriptor(&descriptor)
+    check_status(status)
+    return <size_t>descriptor
+
+
+cpdef setTensor4dDescriptor(size_t tensorDesc, int format, int dataType,
+                            int n, int c, int h, int w):
+    status = cudnnSetTensor4dDescriptor(
+        <TensorDescriptor>tensorDesc, <TensorFormat>format,
+        <DataType>dataType, n, c, h, w)
+    check_status(status)
+
+
+cpdef setTensor4dDescriptorEx(size_t tensorDesc, int dataType,
+                              int n, int c, int h, int w, int nStride,
+                              int cStride, int hStride, int wStride):
+    status = cudnnSetTensor4dDescriptorEx(
+        <TensorDescriptor>tensorDesc, <DataType>dataType, n, c, h, w,
+        nStride, cStride, hStride, wStride)
+    check_status(status)
+
+
+cpdef tuple getTensor4dDescriptor(size_t tensorDesc):
+    cdef DataType dataType
+    cdef int n, c, h, w, nStride, cStride, hStride, wStride
+    status = cudnnGetTensor4dDescriptor(
+        <TensorDescriptor>tensorDesc, &dataType,
+        &n, &c, &h, &w, &nStride, &cStride, &hStride, &wStride)
+    check_status(status)
+    return dataType, n, c, h, w, nStride, cStride, hStride, wStride
+
+
+cpdef setTensorNdDescriptor(size_t tensorDesc, int dataType, int nbDims,
+                            size_t dimA, size_t strideA):
+    status = cudnnSetTensorNdDescriptor(
+        <TensorDescriptor>tensorDesc, <DataType>dataType, nbDims,
+        <int*>dimA, <int*>strideA)
+    check_status(status)
+
+
+cpdef destroyTensorDescriptor(size_t tensorDesc):
+    status = cudnnDestroyTensorDescriptor(<TensorDescriptor>tensorDesc)
+    check_status(status)
+
+
+cpdef addTensor_v3(intptr_t handle, size_t alpha, size_t bDesc,
+                   size_t b, size_t beta, size_t yDesc, size_t y):
+    _setStream(handle)
+    with nogil:
+        status = cudnnAddTensor_v3(
+            <Handle>handle, <void*>alpha, <TensorDescriptor>bDesc,
+            <void*>b, <void*>beta, <TensorDescriptor>yDesc, <void*>y)
+    check_status(status)
+
+
+###############################################################################
+# Tensor operations
+###############################################################################
+
+cpdef size_t createOpTensorDescriptor() except? 0:
+    cdef OpTensorDescriptor opTensorDesc
+    status = cudnnCreateOpTensorDescriptor(&opTensorDesc)
+    check_status(status)
+    return <size_t>opTensorDesc
+
+
+cpdef setOpTensorDescriptor(size_t opTensorDesc, int opTensorOp,
+                            int opTensorCompType, int opTensorNanOpt):
+    status = cudnnSetOpTensorDescriptor(
+        <OpTensorDescriptor>opTensorDesc, <OpTensorOp>opTensorOp,
+        <DataType>opTensorCompType, <NanPropagation>opTensorNanOpt)
+    check_status(status)
+
+
+cpdef getOpTensorDescriptor(size_t opTensorDesc):
+    cdef OpTensorOp opTensorOp
+    cdef DataType opTensorCompType
+    cdef NanPropagation opTensorNanOpt
+    status = cudnnGetOpTensorDescriptor(
+        <OpTensorDescriptor>opTensorDesc, &opTensorOp, &opTensorCompType,
+        &opTensorNanOpt)
+    check_status(status)
+    return opTensorOp, opTensorCompType, opTensorNanOpt
+
+
+cpdef destroyOpTensorDescriptor(size_t opTensorDesc):
+    status = cudnnDestroyOpTensorDescriptor(<OpTensorDescriptor>opTensorDesc)
+    check_status(status)
+
+
+cpdef opTensor(intptr_t handle, size_t opTensorDesc, size_t alpha1,
+               size_t aDesc, size_t A, size_t alpha2, size_t bDesc,
+               size_t B, size_t beta, size_t cDesc, size_t C):
+    _setStream(handle)
+    with nogil:
+        status = cudnnOpTensor(
+            <Handle>handle, <OpTensorDescriptor>opTensorDesc, <void*>alpha1,
+            <TensorDescriptor>aDesc, <void*>A, <void*>alpha2,
+            <TensorDescriptor>bDesc, <void*>B, <void*>beta,
+            <TensorDescriptor>cDesc, <void*>C)
+    check_status(status)
+
+
+###############################################################################
+# Tensor reductions
+###############################################################################
+
+cpdef size_t createReduceTensorDescriptor() except? 0:
+    cdef ReduceTensorDescriptor reduceTensorDesc
+    status = cudnnCreateReduceTensorDescriptor(&reduceTensorDesc)
+    check_status(status)
+    return <size_t>reduceTensorDesc
+
+cpdef setReduceTensorDescriptor(
+        size_t reduceTensorDesc, int reduceTensorOp, int reduceTensorCompType,
+        int reduceTensorNanOpt, int reduceTensorIndices,
+        int reduceTensorIndicesType):
+    status = cudnnSetReduceTensorDescriptor(
+        <ReduceTensorDescriptor>reduceTensorDesc,
+        <ReduceTensorOp>reduceTensorOp,
+        <DataType>reduceTensorCompType, <NanPropagation>reduceTensorNanOpt,
+        <ReduceTensorIndices>reduceTensorIndices,
+        <IndicesType>reduceTensorIndicesType)
+    check_status(status)
+
+
+cpdef getReduceTensorDescriptor(size_t reduceTensorDesc):
+    cdef ReduceTensorOp redOp
+    cdef DataType redCompType
+    cdef NanPropagation redNanOpt
+    cdef ReduceTensorIndices redIndices
+    cdef IndicesType redIndicesType
+    status = cudnnGetReduceTensorDescriptor(
+        <ReduceTensorDescriptor>reduceTensorDesc, &redOp,
+        &redCompType, &redNanOpt, &redIndices, &redIndicesType)
+    check_status(status)
+    return redOp, redCompType, redNanOpt, redIndices, redIndicesType
+
+
+cpdef destroyReduceTensorDescriptor(size_t reduceTensorDesc):
+    status = cudnnDestroyReduceTensorDescriptor(
+        <ReduceTensorDescriptor>reduceTensorDesc)
+    check_status(status)
+
+
+cpdef size_t getReductionIndicesSize(intptr_t handle, size_t reduceTensorDesc,
+                                     size_t aDesc, size_t cDesc) except? 0:
+    cdef size_t sizeInBytes
+    status = cudnnGetReductionIndicesSize(
+        <Handle>handle, <ReduceTensorDescriptor>reduceTensorDesc,
+        <TensorDescriptor>aDesc, <TensorDescriptor>cDesc, &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+
+cpdef size_t getReductionWorkspaceSize(intptr_t handle,
+                                       size_t reduceTensorDesc,
+                                       size_t aDesc, size_t cDesc) except? 0:
+    cdef size_t sizeInBytes
+    status = cudnnGetReductionWorkspaceSize(
+        <Handle>handle, <ReduceTensorDescriptor>reduceTensorDesc,
+        <TensorDescriptor>aDesc, <TensorDescriptor>cDesc,
+        &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+
+cpdef reduceTensor(intptr_t handle, size_t reduceTensorDesc, size_t indices,
+                   size_t indicesSizeInBytes, size_t workspace,
+                   size_t workspaceSizeInBytes, size_t alpha, size_t aDesc,
+                   size_t A, size_t beta, size_t cDesc, size_t C):
+    _setStream(handle)
+    with nogil:
+        status = cudnnReduceTensor(
+            <Handle>handle, <ReduceTensorDescriptor>reduceTensorDesc,
+            <void*>indices, indicesSizeInBytes, <void*>workspace,
+            workspaceSizeInBytes, <void*>alpha, <TensorDescriptor>aDesc,
+            <void*>A, <void*>beta, <TensorDescriptor>cDesc, <void*>C)
+    check_status(status)
+
+
+cpdef setTensor(intptr_t handle, size_t yDesc, size_t y, size_t valuePtr):
+    _setStream(handle)
+    with nogil:
+        status = cudnnSetTensor(
+            <Handle>handle, <TensorDescriptor>yDesc, <void*>y,
+            <void*>valuePtr)
+    check_status(status)
+
+
+cpdef scaleTensor(intptr_t handle, size_t yDesc, size_t y, size_t alpha):
+    _setStream(handle)
+    with nogil:
+        status = cudnnScaleTensor(
+            <Handle>handle, <TensorDescriptor>yDesc, <void*> y,
+            <void*>alpha)
+    check_status(status)
+
+
+###############################################################################
+# Filter manipulation
+###############################################################################
+
+cpdef size_t createFilterDescriptor() except? 0:
+    cdef FilterDescriptor desc
+    status = cudnnCreateFilterDescriptor(&desc)
+    check_status(status)
+    return <size_t>desc
+
+
+cpdef setFilter4dDescriptor_v4(
+        size_t filterDesc, int dataType,
+        int format, int k, int c, int h, int w):
+    status = cudnnSetFilter4dDescriptor_v4(
+        <FilterDescriptor>filterDesc, <DataType> dataType,
+        <TensorFormat> format, k, c, h, w)
+    check_status(status)
+
+
+cpdef setFilterNdDescriptor_v4(
+        size_t filterDesc, int dataType,
+        int format, int nbDims, size_t filterDimA):
+    status = cudnnSetFilterNdDescriptor_v4(
+        <FilterDescriptor>filterDesc, <DataType>dataType,
+        <TensorFormat>format, nbDims, <int*>filterDimA)
+    check_status(status)
+
+
+cpdef getFilterNdDescriptor(size_t wDesc, int nbDimsRequested):
+    cdef DataType dataType
+    cdef TensorFormat format
+    cdef int nbDims
+    cdef vector.vector[int] filterDimA
+    filterDimA.resize(nbDimsRequested)
+
+    status = cudnnGetFilterNdDescriptor_v4(
+        <FilterDescriptor>wDesc, nbDimsRequested, &dataType,
+        &format, &nbDims, filterDimA.data())
+    check_status(status)
+    return dataType, format, nbDims, tuple(filterDimA)
+
+
+cpdef destroyFilterDescriptor(size_t filterDesc):
+    status = cudnnDestroyFilterDescriptor(<FilterDescriptor>filterDesc)
+    check_status(status)
+
+
+###############################################################################
+# Convolution
+###############################################################################
+
+cpdef size_t createConvolutionDescriptor() except? 0:
+    cdef ConvolutionDescriptor desc
+    status = cudnnCreateConvolutionDescriptor(&desc)
+    check_status(status)
+    return <size_t>desc
+
+
+cpdef setConvolutionMathType(size_t convDesc, size_t mathType):
+    status = cudnnSetConvolutionMathType(
+        <ConvolutionDescriptor>convDesc, <MathType>mathType)
+    check_status(status)
+
+
+cpdef size_t getConvolutionMathType(size_t convDesc) except? 0:
+    cdef MathType mathType
+    status = cudnnGetConvolutionMathType(
+        <ConvolutionDescriptor>convDesc, &mathType)
+    check_status(status)
+    return <size_t>mathType
+
+
+cpdef setConvolutionGroupCount(size_t convDesc, int groupCount):
+    status = cudnnSetConvolutionGroupCount(
+        <ConvolutionDescriptor>convDesc, groupCount)
+    check_status(status)
+
+
+cpdef int getConvolutionGroupCount(size_t convDesc) except? -1:
+    cdef int groupCount
+    status = cudnnGetConvolutionGroupCount(
+        <ConvolutionDescriptor>convDesc, &groupCount)
+    check_status(status)
+    return groupCount
+
+
+cpdef setConvolution2dDescriptor_v4(
+        size_t convDesc, int pad_h, int pad_w, int u, int v, int dilation_h,
+        int dilation_w, int mode):
+    status = cudnnSetConvolution2dDescriptor_v4(
+        <ConvolutionDescriptor>convDesc, pad_h, pad_w, u, v, dilation_h,
+        dilation_w, <ConvolutionMode>mode)
+    check_status(status)
+
+
+cpdef setConvolution2dDescriptor_v5(
+        size_t convDesc, int pad_h, int pad_w, int u, int v, int dilation_h,
+        int dilation_w, int mode, size_t computeType):
+    status = cudnnSetConvolution2dDescriptor_v5(
+        <ConvolutionDescriptor>convDesc, pad_h, pad_w, u, v, dilation_h,
+        dilation_w, <ConvolutionMode>mode, <DataType>computeType)
+    check_status(status)
+
+
+cpdef setConvolutionNdDescriptor_v3(
+        size_t convDesc, int arrayLength, size_t padA, size_t filterStrideA,
+        size_t dilationA, int mode, int dataType):
+    status = cudnnSetConvolutionNdDescriptor_v3(
+        <ConvolutionDescriptor>convDesc, arrayLength, <int*>padA,
+        <int*>filterStrideA, <int*>dilationA, <ConvolutionMode>mode,
+        <DataType>dataType)
+    check_status(status)
+
+
+cpdef destroyConvolutionDescriptor(size_t convDesc):
+    status = cudnnDestroyConvolutionDescriptor(
+        <ConvolutionDescriptor>convDesc)
+    check_status(status)
+
+
+cpdef findConvolutionForwardAlgorithm(
+        intptr_t handle, size_t xDesc, size_t wDesc, size_t convDesc,
+        size_t yDesc, int requestedAlgoCount):
+    cdef vector.vector[ConvolutionFwdAlgoPerf] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionForwardAlgorithm(
+        <Handle> handle, <TensorDescriptor>xDesc, <FilterDescriptor>wDesc,
+        <ConvolutionDescriptor>convDesc, <TensorDescriptor>yDesc,
+        requestedAlgoCount, &returnedAlgoCount, perfResults.data())
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return perfResults
+
+
+cpdef list findConvolutionForwardAlgorithmEx(
+        intptr_t handle, size_t xDesc, size_t x, size_t wDesc, size_t w,
+        size_t convDesc, size_t yDesc, size_t y, int requestedAlgoCount,
+        size_t workSpace, size_t workSpaceSizeInBytes):
+    cdef vector.vector[ConvolutionFwdAlgoPerf] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionForwardAlgorithmEx(
+        <Handle> handle, <TensorDescriptor>xDesc, <void*>x,
+        <FilterDescriptor>wDesc, <void*>w, <ConvolutionDescriptor>convDesc,
+        <TensorDescriptor>yDesc, <void*>y, requestedAlgoCount,
+        &returnedAlgoCount, perfResults.data(), <void*>workSpace,
+        workSpaceSizeInBytes)
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory, -1, -1)
+            for p in perfResults]
+
+
+cpdef list findConvolutionForwardAlgorithmEx_v7(
+        intptr_t handle, size_t xDesc, size_t x, size_t wDesc, size_t w,
+        size_t convDesc, size_t yDesc, size_t y, int requestedAlgoCount,
+        size_t workSpace, size_t workSpaceSizeInBytes):
+    cdef vector.vector[ConvolutionFwdAlgoPerf_v7] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionForwardAlgorithmEx_v7(
+        <Handle> handle, <TensorDescriptor>xDesc, <void*>x,
+        <FilterDescriptor>wDesc, <void*>w, <ConvolutionDescriptor>convDesc,
+        <TensorDescriptor>yDesc, <void*>y, requestedAlgoCount,
+        &returnedAlgoCount, perfResults.data(), <void*>workSpace,
+        workSpaceSizeInBytes)
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory,
+                          p.determinism, p.mathType)
+            for p in perfResults]
+
+
+cpdef int getConvolutionForwardAlgorithm_v6(
+        intptr_t handle, size_t srcDesc, size_t filterDesc, size_t convDesc,
+        size_t destDesc, int preference, size_t memoryLimitInbytes) except? -1:
+    cdef ConvolutionFwdAlgo algo
+    status = cudnnGetConvolutionForwardAlgorithm_v6(
+        <Handle>handle, <TensorDescriptor>srcDesc,
+        <FilterDescriptor>filterDesc, <ConvolutionDescriptor>convDesc,
+        <TensorDescriptor>destDesc, <ConvolutionFwdPreference>preference,
+        memoryLimitInbytes, &algo)
+    check_status(status)
+    return algo
+
+
+cpdef list getConvolutionForwardAlgorithm_v7(
+        intptr_t handle, size_t srcDesc, size_t filterDesc, size_t convDesc,
+        size_t destDesc, int requestedAlgoCount):
+    cdef vector.vector[ConvolutionFwdAlgoPerf_v7] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnGetConvolutionForwardAlgorithm_v7(
+        <Handle> handle, <TensorDescriptor>srcDesc,
+        <FilterDescriptor>filterDesc, <ConvolutionDescriptor>convDesc,
+        <TensorDescriptor>destDesc, requestedAlgoCount,
+        &returnedAlgoCount, perfResults.data())
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory,
+                          p.determinism, p.mathType)
+            for p in perfResults]
+
+
+cpdef Py_ssize_t getConvolutionForwardWorkspaceSize(
+        intptr_t handle, size_t srcDesc, size_t filterDesc, size_t convDesc,
+        size_t destDesc, int algo) except? -1:
+    cdef size_t sizeInBytes
+    status = cudnnGetConvolutionForwardWorkspaceSize(
+        <Handle>handle, <TensorDescriptor>srcDesc,
+        <FilterDescriptor>filterDesc, <ConvolutionDescriptor> convDesc,
+        <TensorDescriptor>destDesc, <ConvolutionFwdAlgo>algo, &sizeInBytes)
+    check_status(status)
+    return <Py_ssize_t>sizeInBytes
+
+
+cpdef convolutionForward(
+        intptr_t handle, size_t alpha, size_t srcDesc, size_t srcData,
+        size_t filterDesc, size_t filterData, size_t convDesc, int algo,
+        size_t workSpace, size_t workSpaceSizeInBytes, size_t beta,
+        size_t destDesc, size_t destData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnConvolutionForward(
+            <Handle>handle, <void*>alpha,
+            <TensorDescriptor>srcDesc, <void*>srcData,
+            <FilterDescriptor>filterDesc, <void*>filterData,
+            <ConvolutionDescriptor>convDesc, <ConvolutionFwdAlgo>algo,
+            <void*>workSpace, workSpaceSizeInBytes, <void*>beta,
+            <TensorDescriptor>destDesc, <void*>destData)
+    check_status(status)
+
+
+cpdef convolutionBackwardBias(
+        intptr_t handle, size_t alpha, size_t srcDesc, size_t srcData,
+        size_t beta, size_t destDesc, size_t destData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnConvolutionBackwardBias(
+            <Handle>handle, <void*>alpha,
+            <TensorDescriptor>srcDesc, <void*>srcData, <void*>beta,
+            <TensorDescriptor>destDesc, <void*>destData)
+    check_status(status)
+
+
+cpdef findConvolutionBackwardFilterAlgorithm(
+        intptr_t handle, size_t xDesc, size_t dyDesc, size_t convDesc,
+        size_t dwDesc, int requestedAlgoCount):
+    cdef vector.vector[ConvolutionBwdFilterAlgoPerf] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionBackwardFilterAlgorithm(
+        <Handle> handle, <TensorDescriptor>xDesc, <TensorDescriptor>dyDesc,
+        <ConvolutionDescriptor>convDesc, <FilterDescriptor>dwDesc,
+        requestedAlgoCount, &returnedAlgoCount, perfResults.data())
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return perfResults
+
+
+cpdef list findConvolutionBackwardFilterAlgorithmEx(
+        intptr_t handle, size_t xDesc, size_t x, size_t dyDesc, size_t dy,
+        size_t convDesc, size_t dwDesc, size_t dw, int requestedAlgoCount,
+        size_t workSpace, size_t workSpaceSizeInBytes):
+    cdef vector.vector[ConvolutionBwdFilterAlgoPerf] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionBackwardFilterAlgorithmEx(
+        <Handle> handle, <TensorDescriptor>xDesc, <void*>x,
+        <TensorDescriptor>dyDesc, <void*>dy, <ConvolutionDescriptor>convDesc,
+        <FilterDescriptor>dwDesc, <void*>dw,
+        requestedAlgoCount, &returnedAlgoCount, perfResults.data(),
+        <void*>workSpace, workSpaceSizeInBytes)
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory, -1, -1)
+            for p in perfResults]
+
+
+cpdef list findConvolutionBackwardFilterAlgorithmEx_v7(
+        intptr_t handle, size_t xDesc, size_t x, size_t dyDesc, size_t dy,
+        size_t convDesc, size_t dwDesc, size_t dw, int requestedAlgoCount,
+        size_t workSpace, size_t workSpaceSizeInBytes):
+    cdef vector.vector[ConvolutionBwdFilterAlgoPerf_v7] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionBackwardFilterAlgorithmEx_v7(
+        <Handle> handle, <TensorDescriptor>xDesc, <void*>x,
+        <TensorDescriptor>dyDesc, <void*>dy, <ConvolutionDescriptor>convDesc,
+        <FilterDescriptor>dwDesc, <void*>dw,
+        requestedAlgoCount, &returnedAlgoCount, perfResults.data(),
+        <void*>workSpace, workSpaceSizeInBytes)
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory,
+                          p.determinism, p.mathType)
+            for p in perfResults]
+
+
+cpdef int getConvolutionBackwardFilterAlgorithm_v6(
+        intptr_t handle, size_t srcDesc, size_t diffDesc, size_t convDesc,
+        size_t filterDesc, int preference,
+        size_t memoryLimitInbytes) except? -1:
+    cdef ConvolutionBwdFilterAlgo algo
+    status = cudnnGetConvolutionBackwardFilterAlgorithm_v6(
+        <Handle>handle, <TensorDescriptor>srcDesc,
+        <TensorDescriptor>diffDesc, <ConvolutionDescriptor>convDesc,
+        <FilterDescriptor>filterDesc,
+        <ConvolutionBwdFilterPreference>preference,
+        memoryLimitInbytes, &algo)
+    check_status(status)
+    return algo
+
+
+cpdef list getConvolutionBackwardFilterAlgorithm_v7(
+        intptr_t handle, size_t srcDesc, size_t diffDesc, size_t convDesc,
+        size_t gradDesc, int requestedAlgoCount):
+    cdef vector.vector[ConvolutionBwdFilterAlgoPerf_v7] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnGetConvolutionBackwardFilterAlgorithm_v7(
+        <Handle>handle, <TensorDescriptor>srcDesc, <TensorDescriptor>diffDesc,
+        <ConvolutionDescriptor>convDesc, <FilterDescriptor>gradDesc,
+        requestedAlgoCount, &returnedAlgoCount, perfResults.data())
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory,
+                          p.determinism, p.mathType)
+            for p in perfResults]
+
+
+cpdef Py_ssize_t getConvolutionBackwardFilterWorkspaceSize(
+        intptr_t handle, size_t srcDesc, size_t diffDesc, size_t convDesc,
+        size_t filterDesc, int algo) except? -1:
+    cdef size_t sizeInBytes
+    status = cudnnGetConvolutionBackwardFilterWorkspaceSize(
+        <Handle>handle, <TensorDescriptor>srcDesc,
+        <TensorDescriptor>diffDesc, <ConvolutionDescriptor> convDesc,
+        <FilterDescriptor>filterDesc, <ConvolutionBwdFilterAlgo>algo,
+        &sizeInBytes)
+    check_status(status)
+    return <Py_ssize_t>sizeInBytes
+
+
+cpdef convolutionBackwardFilter_v3(
+        intptr_t handle, size_t alpha, size_t srcDesc, size_t srcData,
+        size_t diffDesc, size_t diffData, size_t convDesc, int algo,
+        size_t workSpace, size_t workSpaceSizeInBytes, size_t beta,
+        size_t gradDesc, size_t gradData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnConvolutionBackwardFilter_v3(
+            <Handle>handle, <void*>alpha,
+            <TensorDescriptor>srcDesc, <void*>srcData,
+            <TensorDescriptor>diffDesc, <void*>diffData,
+            <ConvolutionDescriptor>convDesc, <ConvolutionBwdFilterAlgo>algo,
+            <void*>workSpace, workSpaceSizeInBytes, <void*>beta,
+            <FilterDescriptor>gradDesc, <void*>gradData)
+    check_status(status)
+
+
+cpdef findConvolutionBackwardDataAlgorithm(
+        intptr_t handle, size_t wDesc, size_t dyDesc, size_t convDesc,
+        size_t dxDesc, int requestedAlgoCount):
+    cdef vector.vector[ConvolutionBwdDataAlgoPerf] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionBackwardDataAlgorithm(
+        <Handle> handle, <FilterDescriptor>wDesc, <TensorDescriptor>dyDesc,
+        <ConvolutionDescriptor>convDesc, <TensorDescriptor>dxDesc,
+        requestedAlgoCount, &returnedAlgoCount, perfResults.data())
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return perfResults
+
+
+cpdef list findConvolutionBackwardDataAlgorithmEx(
+        intptr_t handle, size_t wDesc, size_t w, size_t dyDesc, size_t dy,
+        size_t convDesc, size_t dxDesc, size_t dx,
+        int requestedAlgoCount, size_t workSpace, size_t workSpaceSizeInBytes):
+    cdef vector.vector[ConvolutionBwdDataAlgoPerf] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionBackwardDataAlgorithmEx(
+        <Handle> handle, <FilterDescriptor>wDesc, <void*>w,
+        <TensorDescriptor>dyDesc, <void*>dy, <ConvolutionDescriptor>convDesc,
+        <TensorDescriptor>dxDesc, <void*>dx,
+        requestedAlgoCount, &returnedAlgoCount, perfResults.data(),
+        <void*>workSpace, workSpaceSizeInBytes)
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory, -1, -1)
+            for p in perfResults]
+
+
+cpdef list findConvolutionBackwardDataAlgorithmEx_v7(
+        intptr_t handle, size_t wDesc, size_t w, size_t dyDesc, size_t dy,
+        size_t convDesc, size_t dxDesc, size_t dx,
+        int requestedAlgoCount, size_t workSpace, size_t workSpaceSizeInBytes):
+    cdef vector.vector[ConvolutionBwdDataAlgoPerf_v7] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnFindConvolutionBackwardDataAlgorithmEx_v7(
+        <Handle> handle, <FilterDescriptor>wDesc, <void*>w,
+        <TensorDescriptor>dyDesc, <void*>dy, <ConvolutionDescriptor>convDesc,
+        <TensorDescriptor>dxDesc, <void*>dx,
+        requestedAlgoCount, &returnedAlgoCount, perfResults.data(),
+        <void*>workSpace, workSpaceSizeInBytes)
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory,
+                          p.determinism, p.mathType)
+            for p in perfResults]
+
+
+cpdef int getConvolutionBackwardDataAlgorithm_v6(
+        intptr_t handle, size_t filterDesc, size_t diffDesc, size_t convDesc,
+        size_t gradDesc, size_t preference,
+        size_t memoryLimitInbytes) except? -1:
+    cdef ConvolutionBwdDataAlgo algo
+    status = cudnnGetConvolutionBackwardDataAlgorithm_v6(
+        <Handle>handle, <FilterDescriptor>filterDesc,
+        <TensorDescriptor>diffDesc, <ConvolutionDescriptor>convDesc,
+        <TensorDescriptor>gradDesc, <ConvolutionBwdDataPreference>preference,
+        memoryLimitInbytes, &algo)
+    check_status(status)
+    return algo
+
+
+cpdef list getConvolutionBackwardDataAlgorithm_v7(
+        intptr_t handle, size_t filterDesc, size_t diffDesc, size_t convDesc,
+        size_t gradDesc, int requestedAlgoCount):
+    cdef vector.vector[ConvolutionBwdDataAlgoPerf_v7] perfResults
+    cdef int returnedAlgoCount
+    perfResults.resize(requestedAlgoCount)
+    status = cudnnGetConvolutionBackwardDataAlgorithm_v7(
+        <Handle>handle, <FilterDescriptor>filterDesc,
+        <TensorDescriptor>diffDesc, <ConvolutionDescriptor>convDesc,
+        <TensorDescriptor>gradDesc, requestedAlgoCount,
+        &returnedAlgoCount, perfResults.data())
+    check_status(status)
+    perfResults.resize(returnedAlgoCount)
+    return [CuDNNAlgoPerf(p.algo, p.status, p.time, p.memory,
+                          p.determinism, p.mathType)
+            for p in perfResults]
+
+
+cpdef Py_ssize_t getConvolutionBackwardDataWorkspaceSize(
+        intptr_t handle, size_t filterDesc, size_t diffDesc, size_t convDesc,
+        size_t gradDesc, int algo) except? -1:
+    cdef size_t sizeInBytes
+    status = cudnnGetConvolutionBackwardDataWorkspaceSize(
+        <Handle>handle, <FilterDescriptor>filterDesc,
+        <TensorDescriptor>diffDesc,
+        <ConvolutionDescriptor>convDesc, <TensorDescriptor>gradDesc,
+        <ConvolutionBwdDataAlgo>algo, &sizeInBytes)
+    check_status(status)
+    return <Py_ssize_t>sizeInBytes
+
+
+cpdef convolutionBackwardData_v3(
+        intptr_t handle, size_t alpha, size_t filterDesc, size_t filterData,
+        size_t diffDesc, size_t diffData, size_t convDesc, int algo,
+        size_t workSpace, size_t workSpaceSizeInBytes, size_t beta,
+        size_t gradDesc, size_t gradData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnConvolutionBackwardData_v3(
+            <Handle>handle, <void*>alpha,
+            <FilterDescriptor>filterDesc, <void*>filterData,
+            <TensorDescriptor>diffDesc, <void*>diffData,
+            <ConvolutionDescriptor>convDesc, <ConvolutionBwdDataAlgo>algo,
+            <void*>workSpace, workSpaceSizeInBytes, <void*>beta,
+            <TensorDescriptor>gradDesc, <void*>gradData)
+    check_status(status)
+
+###############################################################################
+# Pooling
+###############################################################################
+
+cpdef size_t createPoolingDescriptor() except? 0:
+    cdef PoolingDescriptor desc
+    status = cudnnCreatePoolingDescriptor(&desc)
+    check_status(status)
+    return <size_t>desc
+
+
+cpdef setPooling2dDescriptor_v4(
+        size_t poolingDesc, int mode, int maxpoolingNanOpt, int windowHeight,
+        int windowWidth, int verticalPadding, int horizontalPadding,
+        int verticalStride, int horizontalStride):
+    status = cudnnSetPooling2dDescriptor_v4(
+        <PoolingDescriptor>poolingDesc, <PoolingMode>mode,
+        <NanPropagation>maxpoolingNanOpt, windowHeight, windowWidth,
+        verticalPadding, horizontalPadding, verticalStride, horizontalStride)
+    check_status(status)
+
+
+cpdef setPoolingNdDescriptor_v4(
+        size_t poolingDesc, int mode, int maxpoolingNanOpt, int nbDims,
+        size_t windowDimA, size_t paddingA, size_t strideA):
+    status = cudnnSetPoolingNdDescriptor_v4(
+        <PoolingDescriptor>poolingDesc, <PoolingMode>mode,
+        <NanPropagation>maxpoolingNanOpt, nbDims,
+        <int*>windowDimA, <int*>paddingA, <int*>strideA)
+    check_status(status)
+
+
+cpdef destroyPoolingDescriptor(size_t poolingDesc):
+    status = cudnnDestroyPoolingDescriptor(<PoolingDescriptor>poolingDesc)
+    check_status(status)
+
+
+cpdef poolingForward(
+        intptr_t handle, size_t poolingDesc, size_t alpha, size_t srcDesc,
+        size_t srcData, size_t beta, size_t dstDesc, size_t dstData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnPoolingForward(
+            <Handle>handle, <PoolingDescriptor>poolingDesc, <void*>alpha,
+            <TensorDescriptor>srcDesc, <void*>srcData, <void*>beta,
+            <TensorDescriptor>dstDesc, <void*>dstData)
+    check_status(status)
+
+
+cpdef poolingBackward(
+        intptr_t handle, size_t poolingDesc, size_t alpha, size_t srcDesc,
+        size_t srcData, size_t srcDiffDesc, size_t srcDiffData,
+        size_t destDesc, size_t destData, size_t beta, size_t destDiffDesc,
+        size_t destDiffData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnPoolingBackward(
+            <Handle>handle, <PoolingDescriptor>poolingDesc, <void*>alpha,
+            <TensorDescriptor>srcDesc, <void*>srcData,
+            <TensorDescriptor>srcDiffDesc, <void*>srcDiffData,
+            <TensorDescriptor>destDesc, <void*>destData, <void*>beta,
+            <TensorDescriptor>destDiffDesc, <void*>destDiffData)
+    check_status(status)
+
+###############################################################################
+# Batch Normalization
+###############################################################################
+
+CUDNN_BN_MIN_EPSILON = _CUDNN_BN_MIN_EPSILON
+
+cpdef deriveBNTensorDescriptor(
+        size_t derivedBnDesc, size_t xDesc, int mode):
+    status = cudnnDeriveBNTensorDescriptor(
+        <TensorDescriptor>derivedBnDesc, <TensorDescriptor>xDesc,
+        <BatchNormMode> mode)
+    check_status(status)
+
+
+cpdef batchNormalizationForwardTraining(
+        intptr_t handle, int mode,
+        size_t alpha, size_t beta, size_t xDesc,
+        size_t x, size_t yDesc, size_t y,
+        size_t bnScaleBiasMeanVarDesc, size_t bnScale,
+        size_t bnBias, double exponentialAverageFactor,
+        size_t resultRunningMean, size_t resultRunningVariance,
+        double epsilon, size_t resultSaveMean, size_t resultSaveInvVariance):
+    _setStream(handle)
+    with nogil:
+        status = cudnnBatchNormalizationForwardTraining(
+            <Handle>handle, <BatchNormMode> mode,
+            <void*>alpha, <void*>beta, <TensorDescriptor>xDesc,
+            <void*>x, <TensorDescriptor>yDesc, <void*>y,
+            <TensorDescriptor>bnScaleBiasMeanVarDesc, <void*>bnScale,
+            <void*>bnBias, exponentialAverageFactor,
+            <void*>resultRunningMean, <void*>resultRunningVariance,
+            epsilon, <void*>resultSaveMean, <void*>resultSaveInvVariance)
+    check_status(status)
+
+
+cpdef batchNormalizationForwardInference(
+        intptr_t handle, int mode,
+        size_t alpha, size_t beta, size_t xDesc,
+        size_t x, size_t yDesc, size_t y,
+        size_t bnScaleBiasMeanVarDesc, size_t bnScale,
+        size_t bnBias, size_t estimatedMean, size_t estimatedVariance,
+        double epsilon):
+    _setStream(handle)
+    with nogil:
+        status = cudnnBatchNormalizationForwardInference(
+            <Handle>handle, <BatchNormMode> mode,
+            <void*>alpha, <void*>beta, <TensorDescriptor>xDesc,
+            <void*>x, <TensorDescriptor>yDesc, <void*>y,
+            <TensorDescriptor>bnScaleBiasMeanVarDesc, <void*>bnScale,
+            <void*>bnBias, <void*>estimatedMean, <void*>estimatedVariance,
+            epsilon)
+    check_status(status)
+
+
+cpdef batchNormalizationBackward(
+        intptr_t handle, int mode,
+        size_t alphaDataDiff, size_t betaDataDiff,
+        size_t alphaParamDiff, size_t betaParamDiff,
+        size_t xDesc, size_t x, size_t dyDesc,
+        size_t dy, size_t dxDesc, size_t dx,
+        size_t dBnScaleBiasDesc, size_t bnScale,
+        size_t dBnScaleResult, size_t dBnBiasResult,
+        double epsilon, size_t savedMean, size_t savedInvVariance):
+    _setStream(handle)
+    with nogil:
+        status = cudnnBatchNormalizationBackward(
+            <Handle>handle, <BatchNormMode>mode,
+            <void*>alphaDataDiff, <void*>betaDataDiff,
+            <void*>alphaParamDiff, <void*>betaParamDiff,
+            <TensorDescriptor>xDesc, <void*>x,
+            <TensorDescriptor>dyDesc, <void*>dy,
+            <TensorDescriptor>dxDesc, <void*>dx,
+            <TensorDescriptor>dBnScaleBiasDesc, <void*>bnScale,
+            <void*>dBnScaleResult, <void*>dBnBiasResult,
+            epsilon, <void*>savedMean, <void*>savedInvVariance)
+    check_status(status)
+
+
+cpdef batchNormalizationForwardTrainingEx(
+        intptr_t handle, int mode, int bnOps,
+        size_t alpha, size_t beta,
+        size_t xDesc, size_t x,
+        size_t zDesc, size_t z,
+        size_t yDesc, size_t y,
+        size_t bnScaleBiasMeanVarDesc,
+        size_t bnScale, size_t bnBias,
+        double exponentialAverageFactor,
+        size_t resultRunningMean, size_t resultRunningVariance,
+        double epsilon, size_t resultSaveMean, size_t resultSaveInvVariance,
+        size_t activationDesc,
+        size_t workSpace, size_t workSpaceSizeInBytes,
+        size_t reserveSpace, size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnBatchNormalizationForwardTrainingEx(
+            <Handle>handle, <BatchNormMode> mode, <BatchNormOps> bnOps,
+            <void*>alpha, <void*>beta,
+            <TensorDescriptor>xDesc, <void*>x,
+            <TensorDescriptor>zDesc, <void*>z,
+            <TensorDescriptor>yDesc, <void*>y,
+            <TensorDescriptor>bnScaleBiasMeanVarDesc,
+            <void*>bnScale, <void*>bnBias,
+            exponentialAverageFactor,
+            <void*>resultRunningMean, <void*>resultRunningVariance,
+            epsilon, <void*>resultSaveMean, <void*>resultSaveInvVariance,
+            <ActivationDescriptor>activationDesc,
+            <void*>workSpace, workSpaceSizeInBytes,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef size_t getBatchNormalizationForwardTrainingExWorkspaceSize(
+        intptr_t handle, int mode, int bnOps,
+        size_t xDesc,
+        size_t zDesc,
+        size_t yDesc,
+        size_t bnScaleBiasMeanVarDesc,
+        size_t activationDesc) except? 0:
+    cdef size_t sizeInBytes
+    status = cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(
+        <Handle> handle,
+        <BatchNormMode> mode, <BatchNormOps> bnOps,
+        <TensorDescriptor> xDesc,
+        <TensorDescriptor> zDesc,
+        <TensorDescriptor> yDesc,
+        <TensorDescriptor> bnScaleBiasMeanVarDesc,
+        <ActivationDescriptor> activationDesc,
+        &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+
+cpdef batchNormalizationBackwardEx(
+        intptr_t handle, int mode, int bnops,
+        size_t alphaDataDiff, size_t betaDataDiff,
+        size_t alphaParamDiff, size_t betaParamDiff,
+        size_t xDesc, size_t x,
+        size_t yDesc, size_t y,
+        size_t dyDesc, size_t dy,
+        size_t dzDesc, size_t dz,
+        size_t dxDesc, size_t dx,
+        size_t dBnScaleBiasDesc,
+        size_t bnScaleData, size_t bnBiasData,
+        size_t dBnScaleData, size_t dBnBiasData,
+        double epsilon,
+        size_t savedMean, size_t savedInvVariance,
+        size_t activationDesc,
+        size_t workSpace, size_t workSpaceSizeInBytes,
+        size_t reserveSpace, size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnBatchNormalizationBackwardEx(
+            <Handle> handle,
+            <BatchNormMode> mode, <BatchNormOps> bnops,
+            <void*> alphaDataDiff, <void*> betaDataDiff,
+            <void*> alphaParamDiff, <void*> betaParamDiff,
+            <TensorDescriptor> xDesc, <void*> x,
+            <TensorDescriptor> yDesc, <void*> y,
+            <TensorDescriptor> dyDesc, <void*> dy,
+            <TensorDescriptor> dzDesc, <void*> dz,
+            <TensorDescriptor> dxDesc, <void*> dx,
+            <TensorDescriptor> dBnScaleBiasDesc,
+            <void*> bnScaleData, <void*> bnBiasData,
+            <void*> dBnScaleData, <void*> dBnBiasData,
+            epsilon,
+            <void*> savedMean, <void*> savedInvVariance,
+            <ActivationDescriptor> activationDesc,
+            <void*> workSpace, workSpaceSizeInBytes,
+            <void*> reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef size_t getBatchNormalizationBackwardExWorkspaceSize(
+        intptr_t handle, int mode, int bnOps,
+        size_t xDesc,
+        size_t yDesc,
+        size_t dyDesc,
+        size_t dzDesc,
+        size_t dxDesc,
+        size_t dBnScaleBiasDesc,
+        size_t activationDesc) except? 0:
+    cdef size_t sizeInBytes
+    status = cudnnGetBatchNormalizationBackwardExWorkspaceSize(
+        <Handle> handle,
+        <BatchNormMode> mode,
+        <BatchNormOps> bnOps,
+        <TensorDescriptor> xDesc,
+        <TensorDescriptor> yDesc,
+        <TensorDescriptor> dyDesc,
+        <TensorDescriptor> dzDesc,
+        <TensorDescriptor> dxDesc,
+        <TensorDescriptor> dBnScaleBiasDesc,
+        <ActivationDescriptor> activationDesc,
+        &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+
+cpdef size_t getBatchNormalizationTrainingExReserveSpaceSize(
+        intptr_t handle, int mode, int bnOps,
+        size_t activationDesc,
+        size_t xDesc) except? 0:
+    cdef size_t sizeInBytes
+    status = cudnnGetBatchNormalizationTrainingExReserveSpaceSize(
+        <Handle> handle,
+        <BatchNormMode> mode,
+        <BatchNormOps> bnOps,
+        <ActivationDescriptor> activationDesc,
+        <TensorDescriptor> xDesc,
+        &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+
+###############################################################################
+# Activation
+###############################################################################
+
+cpdef size_t createActivationDescriptor() except? 0:
+    cdef ActivationDescriptor activationDesc
+    status = cudnnCreateActivationDescriptor(&activationDesc)
+    check_status(status)
+    return <size_t>activationDesc
+
+
+cpdef setActivationDescriptor(
+        size_t activationDesc, int mode, int reluNanOpt, double reluCeiling):
+    status = cudnnSetActivationDescriptor(
+        <ActivationDescriptor>activationDesc, <ActivationMode>mode,
+        <NanPropagation>reluNanOpt, reluCeiling)
+    check_status(status)
+
+
+cpdef destroyActivationDescriptor(size_t activationDesc):
+    status = cudnnDestroyActivationDescriptor(
+        <ActivationDescriptor>activationDesc)
+    check_status(status)
+
+
+cpdef softmaxForward(
+        intptr_t handle, int algorithm, int mode, size_t alpha, size_t srcDesc,
+        size_t srcData, size_t beta, size_t dstDesc, size_t dstData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnSoftmaxForward(
+            <Handle>handle, <SoftmaxAlgorithm>algorithm, <SoftmaxMode>mode,
+            <void*>alpha, <TensorDescriptor>srcDesc, <void*>srcData,
+            <void*>beta, <TensorDescriptor>dstDesc, <void*>dstData)
+    check_status(status)
+
+
+cpdef softmaxBackward(
+        intptr_t handle, int algorithm, int mode, size_t alpha, size_t srcDesc,
+        size_t srcData, size_t srcDiffDesc, size_t srcDiffData, size_t beta,
+        size_t destDiffDesc, size_t destDiffData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnSoftmaxBackward(
+            <Handle>handle, <SoftmaxAlgorithm>algorithm, <SoftmaxMode>mode,
+            <void*>alpha, <TensorDescriptor>srcDesc, <void*>srcData,
+            <TensorDescriptor>srcDiffDesc, <void*>srcDiffData, <void*>beta,
+            <TensorDescriptor>destDiffDesc, <void*>destDiffData)
+    check_status(status)
+
+
+cpdef activationForward_v4(
+        intptr_t handle, size_t activationDesc, size_t alpha, size_t srcDesc,
+        size_t srcData, size_t beta, size_t dstDesc, size_t dstData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnActivationForward_v4(
+            <Handle>handle, <ActivationDescriptor>activationDesc, <void*>alpha,
+            <TensorDescriptor>srcDesc, <void*>srcData, <void*>beta,
+            <TensorDescriptor>dstDesc, <void*>dstData)
+    check_status(status)
+
+
+cpdef activationBackward_v4(
+        intptr_t handle, size_t activationDesc, size_t alpha, size_t srcDesc,
+        size_t srcData, size_t srcDiffDesc, size_t srcDiffData,
+        size_t destDesc, size_t destData, size_t beta, size_t destDiffDesc,
+        size_t destDiffData):
+    _setStream(handle)
+    with nogil:
+        status = cudnnActivationBackward_v4(
+            <Handle>handle, <ActivationDescriptor>activationDesc, <void*>alpha,
+            <TensorDescriptor>srcDesc, <void*>srcData,
+            <TensorDescriptor>srcDiffDesc, <void*>srcDiffData,
+            <TensorDescriptor>destDesc, <void*>destData, <void*>beta,
+            <TensorDescriptor>destDiffDesc, <void*>destDiffData)
+    check_status(status)
+
+
+###############################################################################
+# Dropout
+###############################################################################
+
+cpdef size_t createDropoutDescriptor() except? 0:
+    cdef DropoutDescriptor desc
+    status = cudnnCreateDropoutDescriptor(&desc)
+    check_status(status)
+    return <size_t>desc
+
+
+cpdef destroyDropoutDescriptor(size_t dropoutDesc):
+    status = cudnnDestroyDropoutDescriptor(<DropoutDescriptor>dropoutDesc)
+    check_status(status)
+
+
+cpdef Py_ssize_t dropoutGetStatesSize(intptr_t handle) except? -1:
+    cdef size_t sizeInBytes
+    status = cudnnDropoutGetStatesSize(
+        <Handle>handle, &sizeInBytes)
+    check_status(status)
+    return <Py_ssize_t>sizeInBytes
+
+
+cpdef setDropoutDescriptor(
+        size_t dropoutDesc, intptr_t handle, float dropout,
+        size_t states, size_t stateSizeInBytes, unsigned long long seed):
+    status = cudnnSetDropoutDescriptor(
+        <DropoutDescriptor>dropoutDesc, <Handle>handle, dropout,
+        <void*>states, stateSizeInBytes, seed)
+    check_status(status)
+
+
+cpdef size_t getDropoutReserveSpaceSize(size_t xDesc) except? 0:
+    cdef size_t sizeInBytes
+    status = cudnnDropoutGetReserveSpaceSize(
+        <TensorDescriptor>xDesc, &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+
+cpdef dropoutForward(
+        intptr_t handle, size_t dropoutDesc,
+        size_t srcDesc, size_t srcData,
+        size_t dstDesc, size_t dstData,
+        size_t reserveSpace, size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnDropoutForward(
+            <Handle>handle, <DropoutDescriptor>dropoutDesc,
+            <TensorDescriptor>srcDesc, <void*>srcData,
+            <TensorDescriptor>dstDesc, <void*>dstData,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef dropoutBackward(
+        intptr_t handle, size_t dropoutDesc,
+        size_t dyDesc, size_t dyData,
+        size_t dxDesc, size_t dxData,
+        size_t reserveSpace, size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnDropoutBackward(
+            <Handle>handle, <DropoutDescriptor>dropoutDesc,
+            <TensorDescriptor>dyDesc, <void*>dyData,
+            <TensorDescriptor>dxDesc, <void*>dxData,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+###############################################################################
+# CTC
+###############################################################################
+cpdef size_t createCTCLossDescriptor() except? 0:
+    cdef CTCLossDescriptor desc
+    status = cudnnCreateCTCLossDescriptor(&desc)
+    check_status(status)
+    return <size_t>desc
+
+cpdef destroyCTCLossDescriptor(size_t ctcLossDesc):
+    status = cudnnDestroyCTCLossDescriptor(<CTCLossDescriptor>ctcLossDesc)
+    check_status(status)
+
+cpdef setCTCLossDescriptor(size_t ctcLossDesc, int dataType):
+    status = cudnnSetCTCLossDescriptor(
+        <CTCLossDescriptor>ctcLossDesc, <DataType>dataType)
+    check_status(status)
+
+cpdef getCTCLossDescriptor(size_t ctcLossDesc):
+    cdef DataType compType
+    status = cudnnGetCTCLossDescriptor(
+        <CTCLossDescriptor>ctcLossDesc, &compType)
+    check_status(status)
+    return compType
+
+cpdef size_t getCTCLossWorkspaceSize(
+        intptr_t handle, size_t probsDesc, size_t gradientsDesc,
+        size_t labels, size_t labelLengths, size_t inputLengths,
+        int algo, size_t ctcLossDesc) except? 0:
+    cdef size_t sizeInBytes
+    status = cudnnGetCTCLossWorkspaceSize(
+        <Handle>handle, <TensorDescriptor>probsDesc,
+        <TensorDescriptor>gradientsDesc,
+        <int*>labels, <int*>labelLengths, <int*>inputLengths,
+        <CTCLossAlgo>algo, <CTCLossDescriptor>ctcLossDesc, &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+cpdef CTCLoss(
+        intptr_t handle, size_t probsDesc,
+        size_t probs, size_t labels, size_t labelLengths, size_t inputLengths,
+        size_t costs, size_t gradientsDesc, size_t gradients,
+        int algo, size_t ctcLossDesc,
+        size_t workspace, size_t workSpaceSizeInBytes):
+    status = cudnnCTCLoss(
+        <Handle>handle, <TensorDescriptor>probsDesc, <void*>probs,
+        <int*>labels, <int*>labelLengths, <int*>inputLengths,
+        <void*>costs, <TensorDescriptor>gradientsDesc, <void*>gradients,
+        <CTCLossAlgo>algo, <CTCLossDescriptor>ctcLossDesc,
+        <void*>workspace, <size_t>workSpaceSizeInBytes)
+    check_status(status)
+
+
+###############################################################################
+# RNN
+###############################################################################
+
+cpdef size_t createRNNDescriptor() except? 0:
+    cdef RNNDescriptor desc
+    status = cudnnCreateRNNDescriptor(&desc)
+    check_status(status)
+    return <size_t>desc
+
+
+cpdef destroyRNNDescriptor(size_t rnnDesc):
+    status = cudnnDestroyRNNDescriptor(<RNNDescriptor>rnnDesc)
+    check_status(status)
+
+
+cpdef size_t createPersistentRNNPlan(size_t rnnDesc, int minibatch,
+                                     int dataType) except? 0:
+    cdef PersistentRNNPlan plan
+    status = cudnnCreatePersistentRNNPlan(
+        <RNNDescriptor>rnnDesc,
+        <int>minibatch, <DataType>dataType, &plan)
+    check_status(status)
+    return <size_t>plan
+
+
+cpdef setPersistentRNNPlan(size_t rnnDesc, size_t plan):
+    status = cudnnSetPersistentRNNPlan(
+        <RNNDescriptor>rnnDesc, <PersistentRNNPlan>plan)
+    check_status(status)
+
+
+cpdef destroyPersistentRNNPlan(size_t plan):
+    status = cudnnDestroyPersistentRNNPlan(<PersistentRNNPlan>plan)
+    check_status(status)
+
+
+cpdef setRNNDescriptor_v5(
+        size_t rnnDesc, int hiddenSize, int numLayers,
+        size_t dropoutDesc, int inputMode, int direction, int mode,
+        int dataType):
+    status = cudnnSetRNNDescriptor_v5(
+        <RNNDescriptor>rnnDesc, hiddenSize, numLayers,
+        <DropoutDescriptor>dropoutDesc, <RNNInputMode>inputMode,
+        <DirectionMode>direction, <RNNMode>mode, <DataType>dataType)
+    check_status(status)
+
+
+cpdef setRNNDescriptor_v6(
+        intptr_t handle, size_t rnnDesc, int hiddenSize, int numLayers,
+        size_t dropoutDesc, int inputMode, int direction, int mode,
+        int algo, int dataType):
+    status = cudnnSetRNNDescriptor_v6(
+        <Handle>handle, <RNNDescriptor>rnnDesc, hiddenSize, numLayers,
+        <DropoutDescriptor>dropoutDesc, <RNNInputMode>inputMode,
+        <DirectionMode>direction, <RNNMode>mode, <RNNAlgo>algo,
+        <DataType>dataType)
+    check_status(status)
+
+
+cpdef setRNNPaddingMode(
+        size_t rnnDesc, int paddingMode):
+    status = cudnnSetRNNPaddingMode(
+        <RNNDescriptor>rnnDesc, <RNNPaddingMode>paddingMode)
+    check_status(status)
+
+
+cpdef getRNNPaddingMode(size_t rnnDesc):
+    cdef RNNPaddingMode paddingMode
+    status = cudnnGetRNNPaddingMode(
+        <RNNDescriptor>rnnDesc, &paddingMode)
+    check_status(status)
+    return paddingMode
+
+
+cpdef size_t createRNNDataDescriptor() except? 0:
+    cdef RNNDataDescriptor desc
+    status = cudnnCreateRNNDataDescriptor(&desc)
+    check_status(status)
+    return <size_t>desc
+
+
+cpdef destroyRNNDataDescriptor(size_t RNNDataDesc):
+    status = cudnnDestroyRNNDataDescriptor(<RNNDataDescriptor>RNNDataDesc)
+    check_status(status)
+
+
+cpdef setRNNDataDescriptor(
+        size_t RNNDataDesc, int dataType, size_t layout,
+        int maxSeqLength, int batchSize, int vectorSize,
+        size_t seqLengthArray, size_t paddingFill):
+    status = cudnnSetRNNDataDescriptor(
+        <RNNDataDescriptor>RNNDataDesc, <DataType>dataType,
+        <RNNDataLayout>layout, maxSeqLength, batchSize, vectorSize,
+        <const int*>seqLengthArray, <void*>paddingFill)
+    check_status(status)
+
+
+cpdef getRNNDataDescriptor(
+        size_t RNNDataDesc, size_t dataType,
+        size_t layout, size_t maxSeqLength, size_t batchSize,
+        size_t vectorSize, int arrayLengthRequested, size_t seqLengthArray,
+        size_t paddingFill):
+    status = cudnnGetRNNDataDescriptor(
+        <RNNDataDescriptor>RNNDataDesc, <DataType*>dataType,
+        <RNNDataLayout*>layout, <int*>maxSeqLength, <int*>batchSize,
+        <int*>vectorSize, arrayLengthRequested, <int*>seqLengthArray,
+        <void*>paddingFill)
+    check_status(status)
+
+
+cpdef getRNNWorkspaceSize(
+        intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc):
+    cdef size_t sizeInBytes
+    status = cudnnGetRNNWorkspaceSize(
+        <Handle>handle, <RNNDescriptor>rnnDesc, seqLength,
+        <TensorDescriptor*>xDesc, &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+
+cpdef getRNNTrainingReserveSize(
+        intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc):
+    cdef size_t sizeInBytes
+    status = cudnnGetRNNTrainingReserveSize(
+        <Handle>handle, <RNNDescriptor>rnnDesc, seqLength,
+        <TensorDescriptor*>xDesc, &sizeInBytes)
+    check_status(status)
+    return sizeInBytes
+
+
+cpdef getRNNParamsSize(
+        intptr_t handle, size_t rnnDesc, size_t xDesc, int dataType):
+    cdef size_t sizeInBytes
+    status = cudnnGetRNNParamsSize(
+        <Handle>handle, <RNNDescriptor>rnnDesc, <TensorDescriptor>xDesc,
+        &sizeInBytes, <DataType>dataType)
+    check_status(status)
+    return sizeInBytes
+
+
+cpdef getRNNLinLayerMatrixParams(
+        intptr_t handle, size_t rnnDesc, int layer, size_t xDesc, size_t wDesc,
+        size_t w, int linLayerID, size_t linLayerMatDesc, size_t linLayerMat):
+    status = cudnnGetRNNLinLayerMatrixParams(
+        <Handle>handle, <RNNDescriptor>rnnDesc, layer,
+        <TensorDescriptor>xDesc, <FilterDescriptor>wDesc, <void*>w,
+        linLayerID, <FilterDescriptor>linLayerMatDesc, <void**>linLayerMat)
+    check_status(status)
+
+
+cpdef getRNNLinLayerBiasParams(
+        intptr_t handle, size_t rnnDesc, int layer, size_t xDesc, size_t wDesc,
+        size_t w, int linLayerID, size_t linLayerBiasDesc,
+        size_t linLayerBias):
+    status = cudnnGetRNNLinLayerBiasParams(
+        <Handle>handle, <RNNDescriptor>rnnDesc, layer,
+        <TensorDescriptor>xDesc, <FilterDescriptor>wDesc, <void*>w,
+        linLayerID, <FilterDescriptor>linLayerBiasDesc, <void**>linLayerBias)
+    check_status(status)
+
+
+cpdef RNNForwardInference(
+        intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc,
+        size_t x, size_t hxDesc, size_t hx, size_t cxDesc,
+        size_t cx, size_t wDesc, size_t w, size_t yDesc,
+        size_t y, size_t hyDesc, size_t hy, size_t cyDesc,
+        size_t cy, size_t workspace, size_t workSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnRNNForwardInference(
+            <Handle>handle, <RNNDescriptor>rnnDesc, seqLength,
+            <TensorDescriptor*>xDesc, <void*>x,
+            <TensorDescriptor>hxDesc, <void*>hx,
+            <TensorDescriptor>cxDesc, <void*>cx,
+            <FilterDescriptor>wDesc, <void*>w,
+            <TensorDescriptor*>yDesc, <void*>y,
+            <TensorDescriptor>hyDesc, <void*>hy,
+            <TensorDescriptor>cyDesc, <void*>cy,
+            <void*>workspace, workSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef RNNForwardTraining(
+        intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc, size_t x,
+        size_t hxDesc, size_t hx, size_t cxDesc, size_t cx,
+        size_t wDesc, size_t w, size_t yDesc, size_t y,
+        size_t hyDesc, size_t hy, size_t cyDesc, size_t cy,
+        size_t workspace, size_t workSpaceSizeInBytes, size_t reserveSpace,
+        size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnRNNForwardTraining(
+            <Handle>handle, <RNNDescriptor>rnnDesc, seqLength,
+            <TensorDescriptor*>xDesc, <void*>x,
+            <TensorDescriptor>hxDesc, <void*>hx,
+            <TensorDescriptor>cxDesc, <void*>cx,
+            <FilterDescriptor>wDesc, <void*>w,
+            <TensorDescriptor*>yDesc, <void*>y,
+            <TensorDescriptor>hyDesc, <void*> hy,
+            <TensorDescriptor>cyDesc, <void*>cy,
+            <void*>workspace, workSpaceSizeInBytes,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef RNNBackwardData(
+        intptr_t handle, size_t rnnDesc, int seqLength, size_t yDesc, size_t y,
+        size_t dyDesc, size_t dy, size_t dhyDesc, size_t dhy,
+        size_t dcyDesc, size_t dcy, size_t wDesc, size_t w,
+        size_t hxDesc, size_t hx, size_t cxDesc, size_t cx,
+        size_t dxDesc, size_t dx, size_t dhxDesc, size_t dhx,
+        size_t dcxDesc, size_t dcx, size_t workspace,
+        size_t workSpaceSizeInBytes, size_t reserveSpace,
+        size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnRNNBackwardData(
+            <Handle>handle, <RNNDescriptor>rnnDesc, seqLength,
+            <TensorDescriptor*>yDesc, <void*>y,
+            <TensorDescriptor*>dyDesc, <void*>dy,
+            <TensorDescriptor>dhyDesc, <void*>dhy,
+            <TensorDescriptor>dcyDesc, <void*>dcy,
+            <FilterDescriptor>wDesc, <void*>w,
+            <TensorDescriptor>hxDesc, <void*>hx,
+            <TensorDescriptor>cxDesc, <void*>cx,
+            <TensorDescriptor*>dxDesc, <void*>dx,
+            <TensorDescriptor>dhxDesc, <void*>dhx,
+            <TensorDescriptor>dcxDesc, <void*>dcx,
+            <void*>workspace, workSpaceSizeInBytes,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef RNNBackwardWeights(
+        intptr_t handle, size_t rnnDesc, int seqLength, size_t xDesc, size_t x,
+        size_t hxDesc, size_t hx, size_t yDesc, size_t y,
+        size_t workspace, size_t workSpaceSizeInBytes, size_t dwDesc,
+        size_t dw, size_t reserveSpace, size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnRNNBackwardWeights(
+            <Handle>handle, <RNNDescriptor>rnnDesc, seqLength,
+            <TensorDescriptor*>xDesc, <void*>x,
+            <TensorDescriptor>hxDesc, <void*>hx,
+            <TensorDescriptor*>yDesc, <void*>y,
+            <void*>workspace, workSpaceSizeInBytes,
+            <FilterDescriptor>dwDesc, <void*>dw,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef RNNForwardInferenceEx(
+        intptr_t handle, size_t rnnDesc, size_t xDesc, size_t x, size_t hxDesc,
+        size_t hx, size_t cxDesc, size_t cx, size_t wDesc, size_t w,
+        size_t yDesc, size_t y, size_t hyDesc, size_t hy, size_t cyDesc,
+        size_t cy, size_t kDesc, size_t keys, size_t cDesc, size_t cAttn,
+        size_t iDesc, size_t iAttn, size_t qDesc, size_t queries,
+        size_t workSpace, size_t workSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnRNNForwardInferenceEx(
+            <Handle>handle, <RNNDescriptor>rnnDesc,
+            <RNNDataDescriptor>xDesc, <const void*>x,
+            <TensorDescriptor>hxDesc, <const void*>hx,
+            <TensorDescriptor>cxDesc, <const void*>cx,
+            <FilterDescriptor>wDesc, <const void*>w,
+            <RNNDataDescriptor>yDesc, <void*>y,
+            <TensorDescriptor>hyDesc, <void*>hy,
+            <TensorDescriptor>cyDesc, <void*>cy,
+            <RNNDataDescriptor>kDesc, <const void*>keys,
+            <RNNDataDescriptor>cDesc, <void*>cAttn,
+            <RNNDataDescriptor>iDesc, <void*>iAttn,
+            <RNNDataDescriptor>qDesc, <void*>queries,
+            <void*>workSpace, workSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef RNNForwardTrainingEx(
+        intptr_t handle, size_t rnnDesc, size_t xDesc, size_t x, size_t hxDesc,
+        size_t hx, size_t cxDesc, size_t cx, size_t wDesc, size_t w,
+        size_t yDesc, size_t y, size_t hyDesc, size_t hy, size_t cyDesc,
+        size_t cy, size_t kDesc, size_t keys, size_t cDesc, size_t cAttn,
+        size_t iDesc, size_t iAttn, size_t qDesc, size_t queries,
+        size_t workSpace, size_t workSpaceSizeInBytes,
+        size_t reserveSpace, size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnRNNForwardTrainingEx(
+            <Handle>handle, <RNNDescriptor>rnnDesc,
+            <RNNDataDescriptor>xDesc, <const void*>x,
+            <TensorDescriptor>hxDesc, <const void*>hx,
+            <TensorDescriptor>cxDesc, <const void*>cx,
+            <FilterDescriptor>wDesc, <const void*>w,
+            <RNNDataDescriptor>yDesc, <void*>y,
+            <TensorDescriptor>hyDesc, <void*>hy,
+            <TensorDescriptor>cyDesc, <void*>cy,
+            <RNNDataDescriptor>kDesc, <const void*>keys,
+            <RNNDataDescriptor>cDesc, <void*>cAttn,
+            <RNNDataDescriptor>iDesc, <void*>iAttn,
+            <RNNDataDescriptor>qDesc, <void*>queries,
+            <void*>workSpace, workSpaceSizeInBytes,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef RNNBackwardDataEx(
+        intptr_t handle, size_t rnnDesc, size_t yDesc, size_t y, size_t dyDesc,
+        size_t dy, size_t dcDesc, size_t dcAttn, size_t dhyDesc, size_t dhy,
+        size_t dcyDesc, size_t dcy, size_t wDesc, size_t w, size_t hxDesc,
+        size_t hx, size_t cxDesc, size_t cx, size_t dxDesc, size_t dx,
+        size_t dhxDesc, size_t dhx, size_t dcxDesc, size_t dcx,
+        size_t dkDesc, size_t dkeys,
+        size_t workSpace, size_t workSpaceSizeInBytes,
+        size_t reserveSpace, size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnRNNBackwardDataEx(
+            <Handle>handle, <RNNDescriptor>rnnDesc,
+            <RNNDataDescriptor>yDesc, <const void*>y,
+            <RNNDataDescriptor>dyDesc, <const void*>dy,
+            <RNNDataDescriptor>dcDesc, <const void*>dcAttn,
+            <TensorDescriptor>dhyDesc, <const void*>dhy,
+            <TensorDescriptor>dcyDesc, <const void*>dcy,
+            <FilterDescriptor>wDesc, <const void*>w,
+            <TensorDescriptor>hxDesc, <const void*>hx,
+            <TensorDescriptor>cxDesc, <const void*>cx,
+            <RNNDataDescriptor>dxDesc, <void*>dx,
+            <TensorDescriptor>dhxDesc, <void*>dhx,
+            <TensorDescriptor>dcxDesc, <void*>dcx,
+            <RNNDataDescriptor>dkDesc, <void*>dkeys,
+            <void*>workSpace, workSpaceSizeInBytes,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+cpdef RNNBackwardWeightsEx(
+        intptr_t handle, size_t rnnDesc, size_t xDesc, size_t x,
+        size_t hxDesc, size_t hx, size_t yDesc, size_t y,
+        size_t workSpace, size_t workSpaceSizeInBytes,
+        size_t dwDesc, size_t dw,
+        size_t reserveSpace, size_t reserveSpaceSizeInBytes):
+    _setStream(handle)
+    with nogil:
+        status = cudnnRNNBackwardWeightsEx(
+            <Handle>handle, <RNNDescriptor>rnnDesc,
+            <RNNDataDescriptor>xDesc, <const void*>x,
+            <TensorDescriptor>hxDesc, <const void*>hx,
+            <RNNDataDescriptor>yDesc, <const void*>y,
+            <void*>workSpace, workSpaceSizeInBytes,
+            <FilterDescriptor>dwDesc, <void*>dw,
+            <void*>reserveSpace, reserveSpaceSizeInBytes)
+    check_status(status)
+
+
+###############################################################################
+# Spatial Transformer
+###############################################################################
+
+cpdef size_t createSpatialTransformerDescriptor() except? 0:
+    cdef SpatialTransformerDescriptor stDesc
+    status = cudnnCreateSpatialTransformerDescriptor(&stDesc)
+    check_status(status)
+    return <size_t>stDesc
+
+
+cpdef destroySpatialTransformerDescriptor(size_t stDesc):
+    status = cudnnDestroySpatialTransformerDescriptor(
+        <SpatialTransformerDescriptor>stDesc)
+    check_status(status)
+
+
+cpdef setSpatialTransformerDescriptor(
+        size_t stDesc, size_t samplerType, int dataType,
+        int nbDims, size_t dimA):
+    status = cudnnSetSpatialTransformerNdDescriptor(
+        <SpatialTransformerDescriptor>stDesc, <SamplerType>samplerType,
+        <DataType>dataType, nbDims, <int*>dimA)
+    check_status(status)
+
+
+cpdef spatialTfGridGeneratorForward(
+        intptr_t handle, size_t stDesc, size_t theta, size_t grid):
+    _setStream(handle)
+    with nogil:
+        status = cudnnSpatialTfGridGeneratorForward(
+            <Handle>handle, <SpatialTransformerDescriptor> stDesc,
+            <void*>theta, <void*>grid)
+    check_status(status)
+
+
+cpdef spatialTfGridGeneratorBackward(
+        intptr_t handle, size_t stDesc, size_t dgrid, size_t dtheta):
+    _setStream(handle)
+    with nogil:
+        status = cudnnSpatialTfGridGeneratorBackward(
+            <Handle>handle, <SpatialTransformerDescriptor>stDesc,
+            <void*>dgrid, <void*>dtheta)
+    check_status(status)
+
+
+cpdef spatialTfSamplerForward(
+        intptr_t handle, size_t stDesc, size_t alpha, size_t xDesc,
+        size_t x, size_t grid, size_t beta, size_t yDesc, size_t y):
+    _setStream(handle)
+    with nogil:
+        status = cudnnSpatialTfSamplerForward(
+            <Handle>handle, <SpatialTransformerDescriptor>stDesc,
+            <void*>alpha, <TensorDescriptor>xDesc, <void*>x, <void*>grid,
+            <void*>beta, <TensorDescriptor>yDesc, <void*>y)
+    check_status(status)
+
+
+cpdef spatialTfSamplerBackward(
+        intptr_t handle, size_t stDesc, size_t alpha, size_t xDesc,
+        size_t x, size_t beta, size_t dxDesc, size_t dx, size_t alphaDgrid,
+        size_t dyDesc, size_t dy, size_t grid, size_t betaDgrid, size_t dgrid):
+    _setStream(handle)
+    with nogil:
+        status = cudnnSpatialTfSamplerBackward(
+            <Handle>handle, <SpatialTransformerDescriptor>stDesc,
+            <void*>alpha, <TensorDescriptor>xDesc, <void*>x, <void*>beta,
+            <TensorDescriptor>dxDesc, <void*>dx, <void*>alphaDgrid,
+            <TensorDescriptor>dyDesc, <void*>dy, <void*>grid,
+            <void*>betaDgrid, <void*>dgrid)
+    check_status(status)
+
+###############################################################################
+# Fused Ops
+###############################################################################
+
+cpdef createFusedOpsConstParamPack(int ops):
+    cdef FusedOpsConstParamPack constPack
+    with nogil:
+        status = cudnnCreateFusedOpsConstParamPack(&constPack, <FusedOps>ops)
+    check_status(status)
+    return <size_t>constPack
+
+cpdef destroyFusedOpsConstParamPack(size_t constPack):
+    with nogil:
+        status = cudnnDestroyFusedOpsConstParamPack(
+            <FusedOpsConstParamPack>constPack)
+    check_status(status)
+
+cpdef setFusedOpsConstParamPackAttribute(size_t constPack, int paramLabel,
+                                         size_t param):
+    with nogil:
+        status = cudnnSetFusedOpsConstParamPackAttribute(
+            <FusedOpsConstParamPack>constPack,
+            <FusedOpsConstParamLabel>paramLabel, <const void*>param)
+    check_status(status)
+
+cpdef getFusedOpsConstParamPackAttribute(size_t constPack, int paramLabel,
+                                         size_t param):
+    cdef int isNULL = 0
+    with nogil:
+        status = cudnnGetFusedOpsConstParamPackAttribute(
+            <const FusedOpsConstParamPack>constPack,
+            <FusedOpsConstParamLabel>paramLabel, <void*>param, &isNULL)
+    check_status(status)
+    return isNULL
+
+cpdef createFusedOpsVariantParamPack(int ops):
+    cdef FusedOpsVariantParamPack varPack
+    with nogil:
+        status = cudnnCreateFusedOpsVariantParamPack(&varPack, <FusedOps>ops)
+    check_status(status)
+    return <size_t>varPack
+
+cpdef destroyFusedOpsVariantParamPack(size_t varPack):
+    with nogil:
+        status = cudnnDestroyFusedOpsVariantParamPack(
+            <FusedOpsVariantParamPack>varPack)
+    check_status(status)
+
+cpdef setFusedOpsVariantParamPackAttribute(size_t varPack, int paramLabel,
+                                           size_t ptr):
+    with nogil:
+        status = cudnnSetFusedOpsVariantParamPackAttribute(
+            <FusedOpsVariantParamPack>varPack,
+            <FusedOpsVariantParamLabel>paramLabel, <void*>ptr)
+    check_status(status)
+
+cpdef getFusedOpsVariantParamPackAttribute(size_t varPack, int paramLabel,
+                                           size_t ptr):
+    with nogil:
+        status = cudnnGetFusedOpsVariantParamPackAttribute(
+            <const FusedOpsVariantParamPack>varPack,
+            <FusedOpsVariantParamLabel> paramLabel, <void*>ptr)
+    check_status(status)
+
+cpdef createFusedOpsPlan(int ops):
+    cdef FusedOpsPlan plan
+    with nogil:
+        status = cudnnCreateFusedOpsPlan(&plan, <FusedOps>ops)
+    check_status(status)
+    return <size_t>plan
+
+cpdef destroyFusedOpsPlan(size_t plan):
+    with nogil:
+        status = cudnnDestroyFusedOpsPlan(<FusedOpsPlan>plan)
+    check_status(status)
+
+cpdef makeFusedOpsPlan(intptr_t handle, size_t plan, size_t constPack):
+    cdef size_t workspaceSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cudnnMakeFusedOpsPlan(<Handle>handle, <FusedOpsPlan>plan,
+                                       <const FusedOpsConstParamPack>constPack,
+                                       &workspaceSizeInBytes)
+    check_status(status)
+    return workspaceSizeInBytes
+
+cpdef fusedOpsExecute(intptr_t handle, size_t plan, size_t varPack):
+    _setStream(handle)
+    with nogil:
+        status = cudnnFusedOpsExecute(<Handle>handle, <const FusedOpsPlan>plan,
+                                      <FusedOpsVariantParamPack>varPack)
+    check_status(status)
diff --git a/cupy_backends/cuda/libs/curand.pxd b/cupy_backends/cuda/libs/curand.pxd
new file mode 100644
index 0000000..33a4fa4
--- /dev/null
+++ b/cupy_backends/cuda/libs/curand.pxd
@@ -0,0 +1,32 @@
+###############################################################################
+# Types
+###############################################################################
+
+cdef extern from *:
+    ctypedef int Ordering 'curandOrdering_t'
+    ctypedef int RngType 'curandRngType_t'
+
+    ctypedef void* Generator 'curandGenerator_t'
+
+
+###############################################################################
+# Enum
+###############################################################################
+
+cpdef enum:
+    CURAND_RNG_PSEUDO_DEFAULT = 100
+    CURAND_RNG_PSEUDO_XORWOW = 101
+    CURAND_RNG_PSEUDO_MRG32K3A = 121
+    CURAND_RNG_PSEUDO_MTGP32 = 141
+    CURAND_RNG_PSEUDO_MT19937 = 142
+    CURAND_RNG_PSEUDO_PHILOX4_32_10 = 161
+    CURAND_RNG_QUASI_DEFAULT = 200
+    CURAND_RNG_QUASI_SOBOL32 = 201
+    CURAND_RNG_QUASI_SCRAMBLED_SOBOL32 = 202
+    CURAND_RNG_QUASI_SOBOL64 = 203
+    CURAND_RNG_QUASI_SCRAMBLED_SOBOL64 = 204
+
+    CURAND_ORDERING_PSEUDO_BEST = 100
+    CURAND_ORDERING_PSEUDO_DEFAULT = 101
+    CURAND_ORDERING_PSEUDO_SEEDED = 102
+    CURAND_ORDERING_QUASI_DEFAULT = 201
diff --git a/cupy_backends/cuda/libs/curand.pyx b/cupy_backends/cuda/libs/curand.pyx
new file mode 100644
index 0000000..d8e8df9
--- /dev/null
+++ b/cupy_backends/cuda/libs/curand.pyx
@@ -0,0 +1,233 @@
+# distutils: language = c++
+
+"""Thin wrapper of cuRAND."""
+cimport cython  # NOQA
+
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda cimport stream as stream_module
+
+###############################################################################
+# Extern
+###############################################################################
+
+cdef extern from '../../cupy_rand.h' nogil:
+    ctypedef void* Stream 'cudaStream_t'
+
+    # Generator
+    int curandCreateGenerator(Generator* generator, int rng_type)
+    int curandDestroyGenerator(Generator generator)
+    int curandGetVersion(int* version)
+
+    # Stream
+    int curandSetStream(Generator generator, Stream stream)
+    int curandSetPseudoRandomGeneratorSeed(
+        Generator generator, unsigned long long seed)
+    int curandSetGeneratorOffset(
+        Generator generator, unsigned long long offset)
+    int curandSetGeneratorOrdering(Generator generator, Ordering order)
+
+    # Generation functions
+    int curandGenerate(
+        Generator generator, unsigned int* outputPtr, size_t num)
+    int curandGenerateLongLong(
+        Generator generator, unsigned long long* outputPtr, size_t num)
+    int curandGenerateUniform(
+        Generator generator, float* outputPtr, size_t num)
+    int curandGenerateUniformDouble(
+        Generator generator, double* outputPtr, size_t num)
+    int curandGenerateNormal(
+        Generator generator, float* outputPtr, size_t num,
+        float mean, float stddev)
+    int curandGenerateNormalDouble(
+        Generator generator, double* outputPtr, size_t n,
+        double mean, double stddev)
+    int curandGenerateLogNormal(
+        Generator generator, float* outputPtr, size_t n,
+        float mean, float stddev)
+    int curandGenerateLogNormalDouble(
+        Generator generator, double* outputPtr, size_t n,
+        double mean, double stddev)
+    int curandGeneratePoisson(
+        Generator generator, unsigned int* outputPtr, size_t n, double lam)
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+STATUS = {
+    0: 'CURAND_STATUS_SUCCESS',
+    100: 'CURAND_STATUS_VERSION_MISMATCH',
+    101: 'CURAND_STATUS_NOT_INITIALIZED',
+    102: 'CURAND_STATUS_ALLOCATION_FAILED',
+    103: 'CURAND_STATUS_TYPE_ERROR',
+    104: 'CURAND_STATUS_OUT_OF_RANGE',
+    105: 'CURAND_STATUS_LENGTH_NOT_MULTIPLE',
+    106: 'CURAND_STATUS_DOUBLE_PRECISION_REQUIRED',
+    201: 'CURAND_STATUS_LAUNCH_FAILURE',
+    202: 'CURAND_STATUS_PREEXISTING_FAILURE',
+    203: 'CURAND_STATUS_INITIALIZATION_FAILED',
+    204: 'CURAND_STATUS_ARCH_MISMATCH',
+    999: 'CURAND_STATUS_INTERNAL_ERROR',
+}
+
+
+class CURANDError(RuntimeError):
+
+    def __init__(self, status):
+        self.status = status
+        super(CURANDError, self).__init__(STATUS[status])
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        raise CURANDError(status)
+
+
+###############################################################################
+# Generator
+###############################################################################
+
+cpdef size_t createGenerator(int rng_type) except? 0:
+    cdef Generator generator
+    with nogil:
+        status = curandCreateGenerator(&generator, <RngType>rng_type)
+    check_status(status)
+    return <size_t>generator
+
+
+cpdef destroyGenerator(size_t generator):
+    status = curandDestroyGenerator(<Generator>generator)
+    check_status(status)
+
+
+cpdef int getVersion() except? -1:
+    cdef int version
+    status = curandGetVersion(&version)
+    check_status(status)
+    return version
+
+
+cpdef setStream(size_t generator, size_t stream):
+    # TODO(leofang): The support of stream capture is not mentioned at all in
+    # the cuRAND docs (as of CUDA 11.5), so we disable this functionality.
+    if not runtime._is_hip_environment and runtime.streamIsCapturing(stream):
+        raise NotImplementedError(
+            'calling cuRAND API during stream capture is currently '
+            'unsupported')
+
+    status = curandSetStream(<Generator>generator, <Stream>stream)
+    check_status(status)
+
+
+cdef _setStream(size_t generator):
+    """Set current stream"""
+    setStream(generator, stream_module.get_current_stream_ptr())
+
+
+cpdef setPseudoRandomGeneratorSeed(size_t generator, unsigned long long seed):
+    status = curandSetPseudoRandomGeneratorSeed(<Generator>generator, seed)
+    check_status(status)
+
+
+cpdef setGeneratorOffset(size_t generator, unsigned long long offset):
+    status = curandSetGeneratorOffset(<Generator>generator, offset)
+    check_status(status)
+
+
+cpdef setGeneratorOrdering(size_t generator, int order):
+    status = curandSetGeneratorOrdering(<Generator>generator, <Ordering>order)
+    check_status(status)
+
+
+###############################################################################
+# Generation functions
+###############################################################################
+
+cpdef generate(size_t generator, size_t outputPtr, size_t num):
+    _setStream(generator)
+    status = curandGenerate(
+        <Generator>generator, <unsigned int*>outputPtr, num)
+    check_status(status)
+
+
+cpdef generateLongLong(size_t generator, size_t outputPtr, size_t num):
+    _setStream(generator)
+    status = curandGenerateLongLong(
+        <Generator>generator, <unsigned long long*>outputPtr, num)
+    check_status(status)
+
+
+cpdef generateUniform(size_t generator, size_t outputPtr, size_t num):
+    _setStream(generator)
+    status = curandGenerateUniform(
+        <Generator>generator, <float*>outputPtr, num)
+    check_status(status)
+
+
+cpdef generateUniformDouble(size_t generator, size_t outputPtr, size_t num):
+    _setStream(generator)
+    status = curandGenerateUniformDouble(
+        <Generator>generator, <double*>outputPtr, num)
+    check_status(status)
+
+
+cpdef generateNormal(size_t generator, size_t outputPtr, size_t n,
+                     float mean, float stddev):
+    if n % 2 == 1:
+        msg = ('curandGenerateNormal can only generate even number of '
+               'random variables simultaneously. See issue #390 for detail.')
+        raise ValueError(msg)
+    _setStream(generator)
+    status = curandGenerateNormal(
+        <Generator>generator, <float*>outputPtr, n, mean, stddev)
+    check_status(status)
+
+
+cpdef generateNormalDouble(size_t generator, size_t outputPtr, size_t n,
+                           float mean, float stddev):
+    if n % 2 == 1:
+        msg = ('curandGenerateNormalDouble can only generate even number of '
+               'random variables simultaneously. See issue #390 for detail.')
+        raise ValueError(msg)
+    _setStream(generator)
+    status = curandGenerateNormalDouble(
+        <Generator>generator, <double*>outputPtr, n, mean, stddev)
+    check_status(status)
+
+
+def generateLogNormal(size_t generator, size_t outputPtr, size_t n,
+                      float mean, float stddev):
+    if n % 2 == 1:
+        msg = ('curandGenerateLogNormal can only generate even number of '
+               'random variables simultaneously. See issue #390 for detail.')
+        raise ValueError(msg)
+    _setStream(generator)
+    status = curandGenerateLogNormal(
+        <Generator>generator, <float*>outputPtr, n, mean, stddev)
+    check_status(status)
+
+
+cpdef generateLogNormalDouble(size_t generator, size_t outputPtr, size_t n,
+                              float mean, float stddev):
+    if n % 2 == 1:
+        msg = ('curandGenerateLogNormalDouble can only generate even number '
+               'of random variables simultaneously. See issue #390 for '
+               'detail.')
+        raise ValueError(msg)
+    _setStream(generator)
+    status = curandGenerateLogNormalDouble(
+        <Generator>generator, <double*>outputPtr, n, mean, stddev)
+    check_status(status)
+
+
+cpdef generatePoisson(size_t generator, size_t outputPtr, size_t n,
+                      double lam):
+    _setStream(generator)
+    status = curandGeneratePoisson(
+        <Generator>generator, <unsigned int*>outputPtr, n, lam)
+    check_status(status)
diff --git a/cupy_backends/cuda/libs/cusolver.pxd b/cupy_backends/cuda/libs/cusolver.pxd
new file mode 100644
index 0000000..c88507e
--- /dev/null
+++ b/cupy_backends/cuda/libs/cusolver.pxd
@@ -0,0 +1,727 @@
+"""Thin wrapper of CUSOLVER."""
+from libc.stdint cimport intptr_t, int64_t
+
+cpdef _get_cuda_build_version()
+
+###############################################################################
+# Types
+###############################################################################
+cdef extern from *:
+    ctypedef int DataType 'cudaDataType'
+
+    ctypedef void* LibraryPropertyType 'libraryPropertyType_t'
+
+    ctypedef void* Handle 'cusolverDnHandle_t'
+    ctypedef void* SpHandle 'cusolverSpHandle_t'
+
+    ctypedef void* Params 'cusolverDnParams_t'
+
+    ctypedef int Operation 'cublasOperation_t'
+    ctypedef int SideMode 'cublasSideMode_t'
+    ctypedef int FillMode 'cublasFillMode_t'
+
+    ctypedef int EigType 'cusolverEigType_t'
+    ctypedef int EigMode 'cusolverEigMode_t'
+
+    ctypedef void* MatDescr 'cusparseMatDescr_t'
+
+    ctypedef void* cuComplex 'cuComplex'
+    ctypedef void* cuDoubleComplex 'cuDoubleComplex'
+
+    ctypedef void* GesvdjInfo 'gesvdjInfo_t'
+    ctypedef void* SyevjInfo 'syevjInfo_t'
+
+###############################################################################
+# Enum
+###############################################################################
+
+cpdef enum:
+    CUSOLVER_EIG_TYPE_1 = 1
+    CUSOLVER_EIG_TYPE_2 = 2
+    CUSOLVER_EIG_TYPE_3 = 3
+
+    CUSOLVER_EIG_MODE_NOVECTOR = 0
+    CUSOLVER_EIG_MODE_VECTOR = 1
+
+###############################################################################
+# Library Attributes
+###############################################################################
+
+cpdef int getProperty(int type) except? -1
+cpdef tuple _getVersion()
+
+###############################################################################
+# Context
+###############################################################################
+
+cpdef intptr_t create() except? 0
+cpdef intptr_t spCreate() except? 0
+cpdef destroy(intptr_t handle)
+cpdef spDestroy(intptr_t handle)
+
+###############################################################################
+# Stream
+###############################################################################
+
+cpdef setStream(intptr_t handle, size_t stream)
+cpdef size_t getStream(intptr_t handle) except? 0
+
+###############################################################################
+# Dense LAPACK Functions (Linear Solver)
+###############################################################################
+
+# Cholesky factorization
+cpdef int spotrf_bufferSize(intptr_t handle, int uplo,
+                            int n, size_t A, int lda) except? -1
+cpdef int dpotrf_bufferSize(intptr_t handle, int uplo,
+                            int n, size_t A, int lda) except? -1
+cpdef int cpotrf_bufferSize(intptr_t handle, int uplo,
+                            int n, size_t A, int lda) except? -1
+cpdef int zpotrf_bufferSize(intptr_t handle, int uplo,
+                            int n, size_t A, int lda) except? -1
+
+cpdef spotrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t work, int lwork, size_t devInfo)
+cpdef dpotrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t work, int lwork, size_t devInfo)
+cpdef cpotrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t work, int lwork, size_t devInfo)
+cpdef zpotrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t work, int lwork, size_t devInfo)
+
+cpdef spotrs(intptr_t handle, int uplo, int n, int nrhs,
+             size_t A, int lda, size_t B, int ldb, size_t devInfo)
+cpdef dpotrs(intptr_t handle, int uplo, int n, int nrhs,
+             size_t A, int lda, size_t B, int ldb, size_t devInfo)
+cpdef cpotrs(intptr_t handle, int uplo, int n, int nrhs,
+             size_t A, int lda, size_t B, int ldb, size_t devInfo)
+cpdef zpotrs(intptr_t handle, int uplo, int n, int nrhs,
+             size_t A, int lda, size_t B, int ldb, size_t devInfo)
+
+cpdef spotrfBatched(intptr_t handle, int uplo, int n, size_t Aarray, int lda,
+                    size_t infoArray, int batchSize)
+cpdef dpotrfBatched(intptr_t handle, int uplo, int n, size_t Aarray, int lda,
+                    size_t infoArray, int batchSize)
+cpdef cpotrfBatched(intptr_t handle, int uplo, int n, size_t Aarray, int lda,
+                    size_t infoArray, int batchSize)
+cpdef zpotrfBatched(intptr_t handle, int uplo, int n, size_t Aarray, int lda,
+                    size_t infoArray, int batchSize)
+
+cpdef spotrsBatched(intptr_t handle, int uplo, int n, int nrhs, size_t Aarray,
+                    int lda, size_t Barray, int ldb, size_t devInfo,
+                    int batchSize)
+cpdef dpotrsBatched(intptr_t handle, int uplo, int n, int nrhs, size_t Aarray,
+                    int lda, size_t Barray, int ldb, size_t devInfo,
+                    int batchSize)
+cpdef cpotrsBatched(intptr_t handle, int uplo, int n, int nrhs, size_t Aarray,
+                    int lda, size_t Barray, int ldb, size_t devInfo,
+                    int batchSize)
+cpdef zpotrsBatched(intptr_t handle, int uplo, int n, int nrhs, size_t Aarray,
+                    int lda, size_t Barray, int ldb, size_t devInfo,
+                    int batchSize)
+
+# LU factorization
+cpdef int sgetrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1
+cpdef int dgetrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1
+cpdef int cgetrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1
+cpdef int zgetrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1
+
+cpdef sgetrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t work, size_t devIpiv, size_t devInfo)
+cpdef dgetrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t work, size_t devIpiv, size_t devInfo)
+cpdef cgetrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t work, size_t devIpiv, size_t devInfo)
+cpdef zgetrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t work, size_t devIpiv, size_t devInfo)
+
+# TODO(anaruse): laswp
+
+# LU solve
+cpdef sgetrs(intptr_t handle, int trans, int n, int nrhs,
+             size_t A, int lda, size_t devIpiv,
+             size_t B, int ldb, size_t devInfo)
+cpdef dgetrs(intptr_t handle, int trans, int n, int nrhs,
+             size_t A, int lda, size_t devIpiv,
+             size_t B, int ldb, size_t devInfo)
+cpdef cgetrs(intptr_t handle, int trans, int n, int nrhs,
+             size_t A, int lda, size_t devIpiv,
+             size_t B, int ldb, size_t devInfo)
+cpdef zgetrs(intptr_t handle, int trans, int n, int nrhs,
+             size_t A, int lda, size_t devIpiv,
+             size_t B, int ldb, size_t devInfo)
+
+# QR factorization
+cpdef int sgeqrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1
+cpdef int dgeqrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1
+cpdef int cgeqrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1
+cpdef int zgeqrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1
+
+cpdef sgeqrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo)
+cpdef dgeqrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo)
+cpdef cgeqrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo)
+cpdef zgeqrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo)
+
+# Generate unitary matrix Q from QR factorization
+cpdef int sorgqr_bufferSize(intptr_t handle, int m, int n, int k,
+                            size_t A, int lda, size_t tau) except? -1
+cpdef int dorgqr_bufferSize(intptr_t handle, int m, int n, int k,
+                            size_t A, int lda, size_t tau) except? -1
+cpdef int cungqr_bufferSize(intptr_t handle, int m, int n, int k,
+                            size_t A, int lda, size_t tau) except? -1
+cpdef int zungqr_bufferSize(intptr_t handle, int m, int n, int k,
+                            size_t A, int lda, size_t tau) except? -1
+
+cpdef sorgqr(intptr_t handle, int m, int n, int k, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo)
+cpdef dorgqr(intptr_t handle, int m, int n, int k, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo)
+cpdef cungqr(intptr_t handle, int m, int n, int k, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo)
+cpdef zungqr(intptr_t handle, int m, int n, int k, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo)
+
+# Compute Q**T*b in solve min||A*x = b||
+cpdef int sormqr_bufferSize(intptr_t handle, int side, int trans,
+                            int m, int n, int k, size_t A, int lda, size_t tau,
+                            size_t C, int ldc) except? -1
+cpdef int dormqr_bufferSize(intptr_t handle, int side, int trans,
+                            int m, int n, int k, size_t A, int lda, size_t tau,
+                            size_t C, int ldc) except? -1
+cpdef int cunmqr_bufferSize(intptr_t handle, int side, int trans,
+                            int m, int n, int k, size_t A, int lda, size_t tau,
+                            size_t C, int ldc) except? -1
+cpdef int zunmqr_bufferSize(intptr_t handle, int side, int trans,
+                            int m, int n, int k, size_t A, int lda, size_t tau,
+                            size_t C, int ldc) except? -1
+
+cpdef sormqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau, size_t C,
+             int ldc, size_t work, int lwork, size_t devInfo)
+cpdef dormqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau, size_t C,
+             int ldc, size_t work, int lwork, size_t devInfo)
+cpdef cunmqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau, size_t C,
+             int ldc, size_t work, int lwork, size_t devInfo)
+cpdef zunmqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau, size_t C,
+             int ldc, size_t work, int lwork, size_t devInfo)
+cpdef cormqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau, size_t C,
+             int ldc, size_t work, int lwork, size_t devInfo)  # (obsoleted)
+cpdef zormqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau, size_t C,
+             int ldc, size_t work, int lwork, size_t devInfo)  # (obsoleted)
+
+# L*D*L**T,U*D*U**T factorization
+cpdef int ssytrf_bufferSize(intptr_t handle, int n, size_t A,
+                            int lda) except? -1
+cpdef int dsytrf_bufferSize(intptr_t handle, int n, size_t A,
+                            int lda) except? -1
+cpdef int csytrf_bufferSize(intptr_t handle, int n, size_t A,
+                            int lda) except? -1
+cpdef int zsytrf_bufferSize(intptr_t handle, int n, size_t A,
+                            int lda) except? -1
+
+cpdef ssytrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t ipiv, size_t work, int lwork, size_t devInfo)
+cpdef dsytrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t ipiv, size_t work, int lwork, size_t devInfo)
+cpdef csytrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t ipiv, size_t work, int lwork, size_t devInfo)
+cpdef zsytrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t ipiv, size_t work, int lwork, size_t devInfo)
+
+# Solve A * X = B using iterative refinement
+cpdef size_t zzgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t zcgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t zygesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t zkgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t ccgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t cygesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t ckgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t ddgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t dsgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t dxgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t dhgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t ssgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t sxgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t shgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+
+cpdef int zzgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int zcgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int zygesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int zkgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int ccgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int ckgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int cygesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int ddgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int dsgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int dxgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int dhgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int ssgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int sxgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int shgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+
+# Compute least-saure solution of A * X = B using iterative refinement
+cpdef size_t zzgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t zcgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t zygels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t zkgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t ccgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t cygels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t ckgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t ddgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t dsgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t dxgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t dhgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t ssgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t sxgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+cpdef size_t shgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1
+
+cpdef int zzgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int zcgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int zygels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int zkgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int ccgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int ckgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int cygels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int ddgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int dsgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int dxgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int dhgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int ssgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int sxgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+cpdef int shgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork_bytes, size_t dInfo)
+
+###############################################################################
+# Dense LAPACK Functions (Eigenvalue Solver)
+###############################################################################
+
+# Bidiagonal factorization
+cpdef int sgebrd_bufferSize(intptr_t handle, int m, int n) except? -1
+cpdef int dgebrd_bufferSize(intptr_t handle, int m, int n) except? -1
+cpdef int cgebrd_bufferSize(intptr_t handle, int m, int n) except? -1
+cpdef int zgebrd_bufferSize(intptr_t handle, int m, int n) except? -1
+
+cpdef sgebrd(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t D, size_t E, size_t tauQ, size_t tauP,
+             size_t Work, int lwork, size_t devInfo)
+cpdef dgebrd(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t D, size_t E, size_t tauQ, size_t tauP,
+             size_t Work, int lwork, size_t devInfo)
+cpdef cgebrd(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t D, size_t E, size_t tauQ, size_t tauP,
+             size_t Work, int lwork, size_t devInfo)
+cpdef zgebrd(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t D, size_t E, size_t tauQ, size_t tauP,
+             size_t Work, int lwork, size_t devInfo)
+
+# TODO(anaruse): orgbr/ungbr, sytrd/hetrd, orgtr/ungtr, ormtr/unmtr
+
+# Singular value decomposition, A = U * Sigma * V^H
+cpdef int sgesvd_bufferSize(intptr_t handle, int m, int n) except? -1
+cpdef int dgesvd_bufferSize(intptr_t handle, int m, int n) except? -1
+cpdef int cgesvd_bufferSize(intptr_t handle, int m, int n) except? -1
+cpdef int zgesvd_bufferSize(intptr_t handle, int m, int n) except? -1
+
+cpdef sgesvd(intptr_t handle, char jobu, char jobvt, int m, int n, size_t A,
+             int lda, size_t S, size_t U, int ldu, size_t VT, int ldvt,
+             size_t Work, int lwork, size_t rwork, size_t devInfo)
+cpdef dgesvd(intptr_t handle, char jobu, char jobvt, int m, int n, size_t A,
+             int lda, size_t S, size_t U, int ldu, size_t VT, int ldvt,
+             size_t Work, int lwork, size_t rwork, size_t devInfo)
+cpdef cgesvd(intptr_t handle, char jobu, char jobvt, int m, int n, size_t A,
+             int lda, size_t S, size_t U, int ldu, size_t VT, int ldvt,
+             size_t Work, int lwork, size_t rwork, size_t devInfo)
+cpdef zgesvd(intptr_t handle, char jobu, char jobvt, int m, int n, size_t A,
+             int lda, size_t S, size_t U, int ldu, size_t VT, int ldvt,
+             size_t Work, int lwork, size_t rwork, size_t devInfo)
+
+# gesvdj ... Singular value decomposition using Jacobi mathod
+cpdef intptr_t createGesvdjInfo() except? 0
+cpdef destroyGesvdjInfo(intptr_t info)
+
+cpdef xgesvdjSetTolerance(intptr_t info, double tolerance)
+cpdef xgesvdjSetMaxSweeps(intptr_t info, int max_sweeps)
+cpdef xgesvdjSetSortEig(intptr_t info, int sort_svd)
+cpdef double xgesvdjGetResidual(intptr_t handle, intptr_t info)
+cpdef int xgesvdjGetSweeps(intptr_t handle, intptr_t info)
+
+cpdef int sgesvdj_bufferSize(intptr_t handle, int jobz, int econ, int m, int n,
+                             intptr_t A, int lda, intptr_t S, intptr_t U,
+                             int ldu, intptr_t V, int ldv, intptr_t params)
+cpdef int dgesvdj_bufferSize(intptr_t handle, int jobz, int econ, int m, int n,
+                             intptr_t A, int lda, intptr_t S, intptr_t U,
+                             int ldu, intptr_t V, int ldv, intptr_t params)
+cpdef int cgesvdj_bufferSize(intptr_t handle, int jobz, int econ, int m, int n,
+                             intptr_t A, int lda, intptr_t S, intptr_t U,
+                             int ldu, intptr_t V, int ldv, intptr_t params)
+cpdef int zgesvdj_bufferSize(intptr_t handle, int jobz, int econ, int m, int n,
+                             intptr_t A, int lda, intptr_t S, intptr_t U,
+                             int ldu, intptr_t V, int ldv, intptr_t params)
+
+cpdef sgesvdj(intptr_t handle, int jobz, int econ, int m, int n, intptr_t A,
+              int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+              intptr_t work, int lwork, intptr_t info, intptr_t params)
+cpdef dgesvdj(intptr_t handle, int jobz, int econ, int m, int n, intptr_t A,
+              int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+              intptr_t work, int lwork, intptr_t info, intptr_t params)
+cpdef cgesvdj(intptr_t handle, int jobz, int econ, int m, int n, intptr_t A,
+              int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+              intptr_t work, int lwork, intptr_t info, intptr_t params)
+cpdef zgesvdj(intptr_t handle, int jobz, int econ, int m, int n, intptr_t A,
+              int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+              intptr_t work, int lwork, intptr_t info, intptr_t params)
+
+cpdef int sgesvdjBatched_bufferSize(
+    intptr_t handle, int jobz, int m, int n,
+    intptr_t A, int lda, intptr_t S, intptr_t U,
+    int ldu, intptr_t V, int ldv, intptr_t params,
+    int batchSize) except? -1
+cpdef int dgesvdjBatched_bufferSize(
+    intptr_t handle, int jobz, int m, int n,
+    intptr_t A, int lda, intptr_t S, intptr_t U,
+    int ldu, intptr_t V, int ldv, intptr_t params,
+    int batchSize) except? -1
+cpdef int cgesvdjBatched_bufferSize(
+    intptr_t handle, int jobz, int m, int n,
+    intptr_t A, int lda, intptr_t S, intptr_t U,
+    int ldu, intptr_t V, int ldv, intptr_t params,
+    int batchSize) except? -1
+cpdef int zgesvdjBatched_bufferSize(
+    intptr_t handle, int jobz, int m, int n,
+    intptr_t A, int lda, intptr_t S, intptr_t U,
+    int ldu, intptr_t V, int ldv, intptr_t params,
+    int batchSize) except? -1
+
+cpdef sgesvdjBatched(
+    intptr_t handle, int jobz, int m, int n, intptr_t A,
+    int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+    intptr_t work, int lwork, intptr_t info, intptr_t params, int batchSize)
+cpdef dgesvdjBatched(
+    intptr_t handle, int jobz, int m, int n, intptr_t A,
+    int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+    intptr_t work, int lwork, intptr_t info, intptr_t params, int batchSize)
+cpdef cgesvdjBatched(
+    intptr_t handle, int jobz, int m, int n, intptr_t A,
+    int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+    intptr_t work, int lwork, intptr_t info, intptr_t params, int batchSize)
+cpdef zgesvdjBatched(
+    intptr_t handle, int jobz, int m, int n, intptr_t A,
+    int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+    intptr_t work, int lwork, intptr_t info, intptr_t params, int batchSize)
+
+# gesvda ... Approximate singular value decomposition
+cpdef int sgesvdaStridedBatched_bufferSize(
+    intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+    int lda, long long int strideA, intptr_t d_S, long long int strideS,
+    intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+    long long int strideV, int batchSize)
+cpdef int dgesvdaStridedBatched_bufferSize(
+    intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+    int lda, long long int strideA, intptr_t d_S, long long int strideS,
+    intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+    long long int strideV, int batchSize)
+cpdef int cgesvdaStridedBatched_bufferSize(
+    intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+    int lda, long long int strideA, intptr_t d_S, long long int strideS,
+    intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+    long long int strideV, int batchSize)
+cpdef int zgesvdaStridedBatched_bufferSize(
+    intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+    int lda, long long int strideA, intptr_t d_S, long long int strideS,
+    intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+    long long int strideV, int batchSize)
+
+cpdef sgesvdaStridedBatched(
+    intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+    int lda, long long int strideA, intptr_t d_S, long long int strideS,
+    intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+    long long int strideV, intptr_t d_work, int lwork, intptr_t d_info,
+    intptr_t h_R_nrmF, int batchSize)
+cpdef dgesvdaStridedBatched(
+    intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+    int lda, long long int strideA, intptr_t d_S, long long int strideS,
+    intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+    long long int strideV, intptr_t d_work, int lwork, intptr_t d_info,
+    intptr_t h_R_nrmF, int batchSize)
+cpdef cgesvdaStridedBatched(
+    intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+    int lda, long long int strideA, intptr_t d_S, long long int strideS,
+    intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+    long long int strideV, intptr_t d_work, int lwork, intptr_t d_info,
+    intptr_t h_R_nrmF, int batchSize)
+cpdef zgesvdaStridedBatched(
+    intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+    int lda, long long int strideA, intptr_t d_S, long long int strideS,
+    intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+    long long int strideV, intptr_t d_work, int lwork, intptr_t d_info,
+    intptr_t h_R_nrmF, int batchSize)
+
+# Standard symmetric eigenvalue solver
+cpdef int ssyevd_bufferSize(intptr_t handle, int jobz, int uplo, int n,
+                            size_t A, int lda, size_t W) except? -1
+cpdef int dsyevd_bufferSize(intptr_t handle, int jobz, int uplo, int n,
+                            size_t A, int lda, size_t W) except? -1
+cpdef int cheevd_bufferSize(intptr_t handle, int jobz, int uplo, int n,
+                            size_t A, int lda, size_t W) except? -1
+cpdef int zheevd_bufferSize(intptr_t handle, int jobz, int uplo, int n,
+                            size_t A, int lda, size_t W) except? -1
+
+cpdef ssyevd(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info)
+cpdef dsyevd(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info)
+cpdef cheevd(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info)
+cpdef zheevd(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info)
+
+# TODO(anaruse); sygvd/hegvd, sygvd/hegvd
+
+# syevj ... Symmetric eigenvalue solver via Jacobi method
+cpdef intptr_t createSyevjInfo() except? 0
+cpdef destroySyevjInfo(intptr_t info)
+
+cpdef xsyevjSetTolerance(intptr_t info, double tolerance)
+cpdef xsyevjSetMaxSweeps(intptr_t info, int max_sweeps)
+cpdef xsyevjSetSortEig(intptr_t info, int sort_eig)
+cpdef double xsyevjGetResidual(intptr_t handle, intptr_t info)
+cpdef int xsyevjGetSweeps(intptr_t handle, intptr_t info)
+
+cpdef int ssyevj_bufferSize(
+    intptr_t handle, int jobz, int uplo, int n,
+    size_t A, int lda, size_t W, intptr_t params) except? -1
+cpdef int dsyevj_bufferSize(
+    intptr_t handle, int jobz, int uplo, int n,
+    size_t A, int lda, size_t W, intptr_t params) except? -1
+cpdef int cheevj_bufferSize(
+    intptr_t handle, int jobz, int uplo, int n,
+    size_t A, int lda, size_t W, intptr_t params) except? -1
+cpdef int zheevj_bufferSize(
+    intptr_t handle, int jobz, int uplo, int n,
+    size_t A, int lda, size_t W, intptr_t params) except? -1
+cpdef ssyevj(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info, intptr_t params)
+cpdef dsyevj(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info, intptr_t params)
+cpdef cheevj(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info, intptr_t params)
+cpdef zheevj(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info, intptr_t params)
+
+cpdef int ssyevjBatched_bufferSize(
+    intptr_t handle, int jobz, int uplo, int n,
+    size_t A, int lda, size_t W, intptr_t params, int batchSize) except? -1
+cpdef int dsyevjBatched_bufferSize(
+    intptr_t handle, int jobz, int uplo, int n,
+    size_t A, int lda, size_t W, intptr_t params, int batchSize) except? -1
+cpdef int cheevjBatched_bufferSize(
+    intptr_t handle, int jobz, int uplo, int n,
+    size_t A, int lda, size_t W, intptr_t params, int batchSize) except? -1
+cpdef int zheevjBatched_bufferSize(
+    intptr_t handle, int jobz, int uplo, int n,
+    size_t A, int lda, size_t W, intptr_t params, int batchSize) except? -1
+cpdef ssyevjBatched(
+    intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+    size_t W, size_t work, int lwork, size_t info, intptr_t params,
+    int batchSize)
+cpdef dsyevjBatched(
+    intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+    size_t W, size_t work, int lwork, size_t info, intptr_t params,
+    int batchSize)
+cpdef cheevjBatched(
+    intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+    size_t W, size_t work, int lwork, size_t info, intptr_t params,
+    int batchSize)
+cpdef zheevjBatched(
+    intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+    size_t W, size_t work, int lwork, size_t info, intptr_t params,
+    int batchSize)
+
+# dense eigenvalue solver (64bit)
+cpdef (size_t, size_t) xsyevd_bufferSize(  # noqa
+    intptr_t handle, intptr_t params, int jobz, int uplo,
+    int64_t n, int dataTypeA, intptr_t A, int64_t lda,
+    int dataTypeW, intptr_t W, int computeType) except *
+cpdef xsyevd(
+    intptr_t handle, intptr_t params, int jobz, int uplo,
+    int64_t n, int dataTypeA, intptr_t A, int64_t lda,
+    int dataTypeW, intptr_t W, int computeType, intptr_t bufferOnDevice,
+    size_t workspaceInBytesOnDevice, intptr_t bufferOnHost,
+    size_t workspaceInBytesOnHost, intptr_t info)
+
+###############################################################################
+# Sparse LAPACK Functions
+###############################################################################
+
+cpdef scsrlsvchol(intptr_t handle, int m, int nnz, size_t descrA,
+                  size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                  size_t b, float tol, int reorder, size_t x,
+                  size_t singularity)
+cpdef dcsrlsvchol(intptr_t handle, int m, int nnz, size_t descrA,
+                  size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                  size_t b, double tol, int reorder, size_t x,
+                  size_t singularity)
+cpdef ccsrlsvchol(intptr_t handle, int m, int nnz, size_t descrA,
+                  size_t csrVal, size_t csrRowPtr, size_t csrColInd, size_t b,
+                  float tol, int reorder, size_t x, size_t singularity)
+cpdef zcsrlsvchol(intptr_t handle, int m, int nnz, size_t descrA,
+                  size_t csrVal, size_t csrRowPtr, size_t csrColInd, size_t b,
+                  double tol, int reorder, size_t x, size_t singularity)
+
+cpdef scsrlsvqr(intptr_t handle, int m, int nnz, size_t descrA, size_t csrValA,
+                size_t csrRowPtrA, size_t csrColIndA, size_t b, float tol,
+                int reorder, size_t x, size_t singularity)
+cpdef dcsrlsvqr(intptr_t handle, int m, int nnz, size_t descrA, size_t csrValA,
+                size_t csrRowPtrA, size_t csrColIndA, size_t b, double tol,
+                int reorder, size_t x, size_t singularity)
+cpdef ccsrlsvqr(intptr_t handle, int m, int nnz, size_t descrA, size_t csrVal,
+                size_t csrRowPtr, size_t csrColInd, size_t b, float tol,
+                int reorder, size_t x, size_t singularity)
+cpdef zcsrlsvqr(intptr_t handle, int m, int nnz, size_t descrA, size_t csrVal,
+                size_t csrRowPtr, size_t csrColInd, size_t b, double tol,
+                int reorder, size_t x, size_t singularity)
+
+cpdef scsreigvsi(intptr_t handle, int m, int nnz, size_t descrA,
+                 size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                 float mu0, size_t x0, int maxite, float eps, size_t mu,
+                 size_t x)
+cpdef dcsreigvsi(intptr_t handle, int m, int nnz, size_t descrA,
+                 size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                 double mu0, size_t x0, int maxite, double eps, size_t mu,
+                 size_t x)
+cpdef ccsreigvsi(intptr_t handle, int m, int nnz, size_t descrA,
+                 size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                 size_t mu0, size_t x0, int maxite, float eps, size_t mu,
+                 size_t x)
+cpdef zcsreigvsi(intptr_t handle, int m, int nnz, size_t descrA,
+                 size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                 size_t mu0, size_t x0, int maxite, double eps, size_t mu,
+                 size_t x)
diff --git a/cupy_backends/cuda/libs/cusolver.pyx b/cupy_backends/cuda/libs/cusolver.pyx
new file mode 100644
index 0000000..c994d4d
--- /dev/null
+++ b/cupy_backends/cuda/libs/cusolver.pyx
@@ -0,0 +1,3656 @@
+# distutils: language = c++
+
+"""Thin wrapper of CUSOLVER."""
+
+cimport cython  # NOQA
+
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda cimport stream as stream_module
+
+
+cpdef _get_cuda_build_version():
+    if CUPY_CUDA_VERSION > 0:
+        return CUPY_CUDA_VERSION
+    elif CUPY_HIP_VERSION > 0:
+        return CUPY_HIP_VERSION
+    else:
+        return 0
+
+
+###############################################################################
+# Extern
+###############################################################################
+
+cdef extern from '../../cupy_complex.h':
+    ctypedef struct cuComplex 'cuComplex':
+        float x, y
+
+    ctypedef struct cuDoubleComplex 'cuDoubleComplex':
+        double x, y
+
+cdef extern from '../../cupy_lapack.h' nogil:
+    ctypedef void* Stream 'cudaStream_t'
+
+    # Context
+    int cusolverDnCreate(Handle* handle)
+    int cusolverSpCreate(SpHandle* handle)
+    int cusolverDnDestroy(Handle handle)
+    int cusolverSpDestroy(SpHandle handle)
+
+    # Stream
+    int cusolverDnGetStream(Handle handle, Stream* streamId)
+    int cusolverSpGetStream(SpHandle handle, Stream* streamId)
+    int cusolverDnSetStream(Handle handle, Stream streamId)
+    int cusolverSpSetStream(SpHandle handle, Stream streamId)
+
+    # Params
+    int cusolverDnCreateParams(Params* params)
+    int cusolverDnDestroyParams(Params params)
+
+    # Library Property
+    int cusolverGetProperty(LibraryPropertyType type, int* value)
+
+    # libraryPropertyType_t
+    int MAJOR_VERSION
+    int MINOR_VERSION
+    int PATCH_LEVEL
+
+    ###########################################################################
+    # Dense LAPACK Functions (Linear Solver)
+    ###########################################################################
+
+    # Cholesky factorization
+    int cusolverDnSpotrf_bufferSize(Handle handle, FillMode uplo, int n,
+                                    float* A, int lda, int* lwork)
+    int cusolverDnDpotrf_bufferSize(Handle handle, FillMode uplo, int n,
+                                    double* A, int lda, int* lwork)
+    int cusolverDnCpotrf_bufferSize(Handle handle, FillMode uplo, int n,
+                                    cuComplex* A, int lda, int* lwork)
+    int cusolverDnZpotrf_bufferSize(Handle handle, FillMode uplo, int n,
+                                    cuDoubleComplex* A, int lda, int* lwork)
+
+    int cusolverDnSpotrf(Handle handle, FillMode uplo, int n,
+                         float* A, int lda,
+                         float* work, int lwork, int* devInfo)
+    int cusolverDnDpotrf(Handle handle, FillMode uplo, int n,
+                         double* A, int lda,
+                         double* work, int lwork, int* devInfo)
+    int cusolverDnCpotrf(Handle handle, FillMode uplo, int n,
+                         cuComplex* A, int lda,
+                         cuComplex* work, int lwork, int* devInfo)
+    int cusolverDnZpotrf(Handle handle, FillMode uplo, int n,
+                         cuDoubleComplex* A, int lda,
+                         cuDoubleComplex* work, int lwork, int* devInfo)
+
+    int cusolverDnSpotrs(Handle handle, FillMode uplo, int n, int nrhs,
+                         const float* A, int lda,
+                         float* B, int ldb, int* devInfo)
+    int cusolverDnDpotrs(Handle handle, FillMode uplo, int n, int nrhs,
+                         const double* A, int lda,
+                         double* B, int ldb, int* devInfo)
+    int cusolverDnCpotrs(Handle handle, FillMode uplo, int n, int nrhs,
+                         const cuComplex* A, int lda,
+                         cuComplex* B, int ldb, int* devInfo)
+    int cusolverDnZpotrs(Handle handle, FillMode uplo, int n, int nrhs,
+                         const cuDoubleComplex* A, int lda,
+                         cuDoubleComplex* B, int ldb, int* devInfo)
+
+    int cusolverDnSpotrfBatched(Handle handle, FillMode uplo, int n,
+                                float** Aarray, int lda,
+                                int* infoArray, int batchSize)
+    int cusolverDnDpotrfBatched(Handle handle, FillMode uplo, int n,
+                                double** Aarray, int lda,
+                                int* infoArray, int batchSize)
+    int cusolverDnCpotrfBatched(Handle handle, FillMode uplo, int n,
+                                cuComplex** Aarray, int lda,
+                                int* infoArray, int batchSize)
+    int cusolverDnZpotrfBatched(Handle handle, FillMode uplo, int n,
+                                cuDoubleComplex** Aarray, int lda,
+                                int* infoArray, int batchSize)
+
+    int cusolverDnSpotrsBatched(Handle handle, FillMode uplo, int n,
+                                int nrhs, float** Aarray, int lda,
+                                float** Barray, int ldb,
+                                int* devInfo, int batchSize)
+    int cusolverDnDpotrsBatched(Handle handle, FillMode uplo, int n,
+                                int nrhs, double** Aarray, int lda,
+                                double** Barray, int ldb,
+                                int* devInfo, int batchSize)
+    int cusolverDnCpotrsBatched(Handle handle, FillMode uplo, int n,
+                                int nrhs, cuComplex** Aarray, int lda,
+                                cuComplex** Barray, int ldb,
+                                int* devInfo, int batchSize)
+    int cusolverDnZpotrsBatched(Handle handle, FillMode uplo, int n,
+                                int nrhs, cuDoubleComplex** Aarray, int lda,
+                                cuDoubleComplex** Barray, int ldb,
+                                int* devInfo, int batchSize)
+
+    # LU factorization
+    int cusolverDnSgetrf_bufferSize(Handle handle, int m, int n,
+                                    float* A, int lda, int* lwork)
+    int cusolverDnDgetrf_bufferSize(Handle handle, int m, int n,
+                                    double* A, int lda, int* lwork)
+    int cusolverDnCgetrf_bufferSize(Handle handle, int m, int n,
+                                    cuComplex* A, int lda, int* lwork)
+    int cusolverDnZgetrf_bufferSize(Handle handle, int m, int n,
+                                    cuDoubleComplex* A, int lda, int* lwork)
+
+    int cusolverDnSgetrf(Handle handle, int m, int n,
+                         float* A, int lda,
+                         float* work, int* devIpiv, int* devInfo)
+    int cusolverDnDgetrf(Handle handle, int m, int n,
+                         double* A, int lda,
+                         double* work, int* devIpiv, int* devInfo)
+    int cusolverDnCgetrf(Handle handle, int m, int n,
+                         cuComplex* A, int lda,
+                         cuComplex* work, int* devIpiv, int* devInfo)
+    int cusolverDnZgetrf(Handle handle, int m, int n,
+                         cuDoubleComplex* A, int lda,
+                         cuDoubleComplex* work, int* devIpiv, int* devInfo)
+
+    # TODO(anaruse): laswp
+
+    # LU solve
+    int cusolverDnSgetrs(Handle handle, Operation trans, int n, int nrhs,
+                         const float* A, int lda, const int* devIpiv,
+                         float* B, int ldb, int* devInfo)
+    int cusolverDnDgetrs(Handle handle, Operation trans, int n, int nrhs,
+                         const double* A, int lda, const int* devIpiv,
+                         double* B, int ldb, int* devInfo)
+    int cusolverDnCgetrs(Handle handle, Operation trans, int n, int nrhs,
+                         const cuComplex* A, int lda, const int* devIpiv,
+                         cuComplex* B, int ldb, int* devInfo)
+    int cusolverDnZgetrs(Handle handle, Operation trans, int n, int nrhs,
+                         const cuDoubleComplex* A, int lda, const int* devIpiv,
+                         cuDoubleComplex* B, int ldb, int* devInfo)
+
+    # QR factorization
+    int cusolverDnSgeqrf_bufferSize(Handle handle, int m, int n,
+                                    float* A, int lda, int* lwork)
+    int cusolverDnDgeqrf_bufferSize(Handle handle, int m, int n,
+                                    double* A, int lda, int* lwork)
+    int cusolverDnCgeqrf_bufferSize(Handle handle, int m, int n,
+                                    cuComplex* A, int lda, int* lwork)
+    int cusolverDnZgeqrf_bufferSize(Handle handle, int m, int n,
+                                    cuDoubleComplex* A, int lda, int* lwork)
+
+    int cusolverDnSgeqrf(Handle handle, int m, int n,
+                         float* A, int lda, float* tau,
+                         float* work, int lwork, int* devInfo)
+    int cusolverDnDgeqrf(Handle handle, int m, int n,
+                         double* A, int lda, double* tau,
+                         double* work, int lwork, int* devInfo)
+    int cusolverDnCgeqrf(Handle handle, int m, int n,
+                         cuComplex* A, int lda, cuComplex* tau,
+                         cuComplex* work, int lwork, int* devInfo)
+    int cusolverDnZgeqrf(Handle handle, int m, int n,
+                         cuDoubleComplex* A, int lda, cuDoubleComplex* tau,
+                         cuDoubleComplex* work, int lwork, int* devInfo)
+
+    # Generate unitary matrix Q from QR factorization.
+    int cusolverDnSorgqr_bufferSize(Handle handle, int m, int n, int k,
+                                    const float* A, int lda,
+                                    const float* tau, int* lwork)
+    int cusolverDnDorgqr_bufferSize(Handle handle, int m, int n, int k,
+                                    const double* A, int lda,
+                                    const double* tau, int* lwork)
+    int cusolverDnCungqr_bufferSize(Handle handle, int m, int n, int k,
+                                    const cuComplex* A, int lda,
+                                    const cuComplex* tau, int* lwork)
+    int cusolverDnZungqr_bufferSize(Handle handle, int m, int n, int k,
+                                    const cuDoubleComplex* A, int lda,
+                                    const cuDoubleComplex* tau, int* lwork)
+
+    int cusolverDnSorgqr(Handle handle, int m, int n, int k,
+                         float* A, int lda,
+                         const float* tau,
+                         float* work, int lwork, int* devInfo)
+    int cusolverDnDorgqr(Handle handle, int m, int n, int k,
+                         double* A, int lda,
+                         const double* tau,
+                         double* work, int lwork, int* devInfo)
+    int cusolverDnCungqr(Handle handle, int m, int n, int k,
+                         cuComplex* A, int lda,
+                         const cuComplex* tau,
+                         cuComplex* work, int lwork, int* devInfo)
+    int cusolverDnZungqr(Handle handle, int m, int n, int k,
+                         cuDoubleComplex* A, int lda,
+                         const cuDoubleComplex* tau,
+                         cuDoubleComplex* work, int lwork, int* devInfo)
+
+    # Compute Q**T*b in solve min||A*x = b||
+    int cusolverDnSormqr_bufferSize(Handle handle, SideMode side,
+                                    Operation trans, int m, int n, int k,
+                                    const float* A, int lda,
+                                    const float* tau,
+                                    const float* C, int ldc,
+                                    int* lwork)
+    int cusolverDnDormqr_bufferSize(Handle handle, SideMode side,
+                                    Operation trans, int m, int n, int k,
+                                    const double* A, int lda,
+                                    const double* tau,
+                                    const double* C, int ldc,
+                                    int* lwork)
+    int cusolverDnCunmqr_bufferSize(Handle handle, SideMode side,
+                                    Operation trans, int m, int n, int k,
+                                    const cuComplex* A, int lda,
+                                    const cuComplex* tau,
+                                    const cuComplex* C, int ldc,
+                                    int* lwork)
+    int cusolverDnZunmqr_bufferSize(Handle handle, SideMode side,
+                                    Operation trans, int m, int n, int k,
+                                    const cuDoubleComplex* A, int lda,
+                                    const cuDoubleComplex* tau,
+                                    const cuDoubleComplex* C, int ldc,
+                                    int* lwork)
+
+    int cusolverDnSormqr(Handle handle, SideMode side, Operation trans,
+                         int m, int n, int k,
+                         const float* A, int lda,
+                         const float* tau,
+                         float* C, int ldc, float* work,
+                         int lwork, int* devInfo)
+    int cusolverDnDormqr(Handle handle, SideMode side, Operation trans,
+                         int m, int n, int k,
+                         const double* A, int lda,
+                         const double* tau,
+                         double* C, int ldc, double* work,
+                         int lwork, int* devInfo)
+    int cusolverDnCunmqr(Handle handle, SideMode side, Operation trans,
+                         int m, int n, int k,
+                         const cuComplex* A, int lda,
+                         const cuComplex* tau,
+                         cuComplex* C, int ldc, cuComplex* work,
+                         int lwork, int* devInfo)
+    int cusolverDnZunmqr(Handle handle, SideMode side, Operation trans,
+                         int m, int n, int k,
+                         const cuDoubleComplex* A, int lda,
+                         const cuDoubleComplex* tau,
+                         cuDoubleComplex* C, int ldc, cuDoubleComplex* work,
+                         int lwork, int* devInfo)
+
+    # L*D*L**T,U*D*U**T factorization
+    int cusolverDnSsytrf_bufferSize(Handle handle, int n,
+                                    float* A, int lda, int* lwork)
+    int cusolverDnDsytrf_bufferSize(Handle handle, int n,
+                                    double* A, int lda, int* lwork)
+    int cusolverDnCsytrf_bufferSize(Handle handle, int n,
+                                    cuComplex* A, int lda, int* lwork)
+    int cusolverDnZsytrf_bufferSize(Handle handle, int n,
+                                    cuDoubleComplex* A, int lda, int* lwork)
+
+    int cusolverDnSsytrf(Handle handle, FillMode uplo, int n,
+                         float* A, int lda, int* ipiv,
+                         float* work, int lwork, int* devInfo)
+    int cusolverDnDsytrf(Handle handle, FillMode uplo, int n,
+                         double* A, int lda, int* ipiv,
+                         double* work, int lwork, int* devInfo)
+    int cusolverDnCsytrf(Handle handle, FillMode uplo, int n,
+                         cuComplex* A, int lda, int* ipiv,
+                         cuComplex* work, int lwork, int* devInfo)
+    int cusolverDnZsytrf(Handle handle, FillMode uplo, int n,
+                         cuDoubleComplex* A, int lda, int* ipiv,
+                         cuDoubleComplex* work, int lwork, int* devInfo)
+
+    # Solve A * X = B using iterative refinement
+    int cusolverDnZZgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    cuDoubleComplex *dA, int ldda, int *dipiv,
+                                    cuDoubleComplex *dB, int lddb,
+                                    cuDoubleComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnZCgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    cuDoubleComplex *dA, int ldda, int *dipiv,
+                                    cuDoubleComplex *dB, int lddb,
+                                    cuDoubleComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnZYgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    cuDoubleComplex *dA, int ldda, int *dipiv,
+                                    cuDoubleComplex *dB, int lddb,
+                                    cuDoubleComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnZKgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    cuDoubleComplex *dA, int ldda, int *dipiv,
+                                    cuDoubleComplex *dB, int lddb,
+                                    cuDoubleComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnCCgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    cuComplex *dA, int ldda, int *dipiv,
+                                    cuComplex *dB, int lddb,
+                                    cuComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnCYgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    cuComplex *dA, int ldda, int *dipiv,
+                                    cuComplex *dB, int lddb,
+                                    cuComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnCKgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    cuComplex *dA, int ldda, int *dipiv,
+                                    cuComplex *dB, int lddb,
+                                    cuComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnDDgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    double *dA, int ldda, int *dipiv,
+                                    double *dB, int lddb,
+                                    double *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnDSgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    double *dA, int ldda, int *dipiv,
+                                    double *dB, int lddb,
+                                    double *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnDXgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    double *dA, int ldda, int *dipiv,
+                                    double *dB, int lddb,
+                                    double *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnDHgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    double *dA, int ldda, int *dipiv,
+                                    double *dB, int lddb,
+                                    double *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnSSgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    float *dA, int ldda, int *dipiv,
+                                    float *dB, int lddb,
+                                    float *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnSXgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    float *dA, int ldda, int *dipiv,
+                                    float *dB, int lddb,
+                                    float *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnSHgesv_bufferSize(Handle handle, int n, int nrhs,
+                                    float *dA, int ldda, int *dipiv,
+                                    float *dB, int lddb,
+                                    float *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+
+    int cusolverDnZZgesv(Handle handle, int n, int nrhs,
+                         cuDoubleComplex *dA, int ldda, int *dipiv,
+                         cuDoubleComplex *dB, int lddb,
+                         cuDoubleComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnZCgesv(Handle handle, int n, int nrhs,
+                         cuDoubleComplex *dA, int ldda, int *dipiv,
+                         cuDoubleComplex *dB, int lddb,
+                         cuDoubleComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnZYgesv(Handle handle, int n, int nrhs,
+                         cuDoubleComplex *dA, int ldda, int *dipiv,
+                         cuDoubleComplex *dB, int lddb,
+                         cuDoubleComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnZKgesv(Handle handle, int n, int nrhs,
+                         cuDoubleComplex *dA, int ldda, int *dipiv,
+                         cuDoubleComplex *dB, int lddb,
+                         cuDoubleComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnCCgesv(Handle handle, int n, int nrhs,
+                         cuComplex *dA, int ldda, int *dipiv,
+                         cuComplex *dB, int lddb,
+                         cuComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnCYgesv(Handle handle, int n, int nrhs,
+                         cuComplex *dA, int ldda, int *dipiv,
+                         cuComplex *dB, int lddb,
+                         cuComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnCKgesv(Handle handle, int n, int nrhs,
+                         cuComplex *dA, int ldda, int *dipiv,
+                         cuComplex *dB, int lddb,
+                         cuComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnDDgesv(Handle handle, int n, int nrhs,
+                         double *dA, int ldda, int *dipiv,
+                         double *dB, int lddb,
+                         double *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnDSgesv(Handle handle, int n, int nrhs,
+                         double *dA, int ldda, int *dipiv,
+                         double *dB, int lddb,
+                         double *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnDXgesv(Handle handle, int n, int nrhs,
+                         double *dA, int ldda, int *dipiv,
+                         double *dB, int lddb,
+                         double *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnDHgesv(Handle handle, int n, int nrhs,
+                         double *dA, int ldda, int *dipiv,
+                         double *dB, int lddb,
+                         double *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnSSgesv(Handle handle, int n, int nrhs,
+                         float *dA, int ldda, int *dipiv,
+                         float *dB, int lddb,
+                         float *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnSXgesv(Handle handle, int n, int nrhs,
+                         float *dA, int ldda, int *dipiv,
+                         float *dB, int lddb,
+                         float *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnSHgesv(Handle handle, int n, int nrhs,
+                         float *dA, int ldda, int *dipiv,
+                         float *dB, int lddb,
+                         float *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+
+    # Compute least square solution to A * X = B using iterative refinement
+    int cusolverDnZZgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    cuDoubleComplex *dA, int ldda,
+                                    cuDoubleComplex *dB, int lddb,
+                                    cuDoubleComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnZCgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    cuDoubleComplex *dA, int ldda,
+                                    cuDoubleComplex *dB, int lddb,
+                                    cuDoubleComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnZYgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    cuDoubleComplex *dA, int ldda,
+                                    cuDoubleComplex *dB, int lddb,
+                                    cuDoubleComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnZKgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    cuDoubleComplex *dA, int ldda,
+                                    cuDoubleComplex *dB, int lddb,
+                                    cuDoubleComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnCCgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    cuComplex *dA, int ldda,
+                                    cuComplex *dB, int lddb,
+                                    cuComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnCYgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    cuComplex *dA, int ldda,
+                                    cuComplex *dB, int lddb,
+                                    cuComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnCKgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    cuComplex *dA, int ldda,
+                                    cuComplex *dB, int lddb,
+                                    cuComplex *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnDDgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    double *dA, int ldda,
+                                    double *dB, int lddb,
+                                    double *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnDSgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    double *dA, int ldda,
+                                    double *dB, int lddb,
+                                    double *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnDXgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    double *dA, int ldda,
+                                    double *dB, int lddb,
+                                    double *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnDHgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    double *dA, int ldda,
+                                    double *dB, int lddb,
+                                    double *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnSSgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    float *dA, int ldda,
+                                    float *dB, int lddb,
+                                    float *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnSXgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    float *dA, int ldda,
+                                    float *dB, int lddb,
+                                    float *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+    int cusolverDnSHgels_bufferSize(Handle handle, int m, int n, int nrhs,
+                                    float *dA, int ldda,
+                                    float *dB, int lddb,
+                                    float *dX, int lddx,
+                                    void *dWorkspace, size_t *lwork_bytes)
+
+    int cusolverDnZZgels(Handle handle, int m, int n, int nrhs,
+                         cuDoubleComplex *dA, int ldda,
+                         cuDoubleComplex *dB, int lddb,
+                         cuDoubleComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnZCgels(Handle handle, int m, int n, int nrhs,
+                         cuDoubleComplex *dA, int ldda,
+                         cuDoubleComplex *dB, int lddb,
+                         cuDoubleComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnZYgels(Handle handle, int m, int n, int nrhs,
+                         cuDoubleComplex *dA, int ldda,
+                         cuDoubleComplex *dB, int lddb,
+                         cuDoubleComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnZKgels(Handle handle, int m, int n, int nrhs,
+                         cuDoubleComplex *dA, int ldda,
+                         cuDoubleComplex *dB, int lddb,
+                         cuDoubleComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnCCgels(Handle handle, int m, int n, int nrhs,
+                         cuComplex *dA, int ldda,
+                         cuComplex *dB, int lddb,
+                         cuComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnCYgels(Handle handle, int m, int n, int nrhs,
+                         cuComplex *dA, int ldda,
+                         cuComplex *dB, int lddb,
+                         cuComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnCKgels(Handle handle, int m, int n, int nrhs,
+                         cuComplex *dA, int ldda,
+                         cuComplex *dB, int lddb,
+                         cuComplex *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnDDgels(Handle handle, int m, int n, int nrhs,
+                         double *dA, int ldda,
+                         double *dB, int lddb,
+                         double *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnDSgels(Handle handle, int m, int n, int nrhs,
+                         double *dA, int ldda,
+                         double *dB, int lddb,
+                         double *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnDXgels(Handle handle, int m, int n, int nrhs,
+                         double *dA, int ldda,
+                         double *dB, int lddb,
+                         double *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnDHgels(Handle handle, int m, int n, int nrhs,
+                         double *dA, int ldda,
+                         double *dB, int lddb,
+                         double *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnSSgels(Handle handle, int m, int n, int nrhs,
+                         float *dA, int ldda,
+                         float *dB, int lddb,
+                         float *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnSXgels(Handle handle, int m, int n, int nrhs,
+                         float *dA, int ldda,
+                         float *dB, int lddb,
+                         float *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+    int cusolverDnSHgels(Handle handle, int m, int n, int nrhs,
+                         float *dA, int ldda,
+                         float *dB, int lddb,
+                         float *dX, int lddx,
+                         void *dWorkspace, size_t lwork_bytes,
+                         int *iter, int *dInfo)
+
+    ###########################################################################
+    # Dense LAPACK Functions (Eigenvalue Solver)
+    ###########################################################################
+
+    # Bidiagonal factorization
+    int cusolverDnSgebrd_bufferSize(Handle handle, int m, int n, int* lwork)
+    int cusolverDnDgebrd_bufferSize(Handle handle, int m, int n, int* lwork)
+    int cusolverDnCgebrd_bufferSize(Handle handle, int m, int n, int* lwork)
+    int cusolverDnZgebrd_bufferSize(Handle handle, int m, int n, int* lwork)
+
+    int cusolverDnSgebrd(Handle handle, int m, int n,
+                         float* A, int lda,
+                         float* D, float* E,
+                         float* tauQ, float* tauP,
+                         float* Work, int lwork, int* devInfo)
+    int cusolverDnDgebrd(Handle handle, int m, int n,
+                         double* A, int lda,
+                         double* D, double* E,
+                         double* tauQ, double* tauP,
+                         double* Work, int lwork, int* devInfo)
+    int cusolverDnCgebrd(Handle handle, int m, int n,
+                         cuComplex* A, int lda,
+                         float* D, float* E,
+                         cuComplex* tauQ, cuComplex* tauP,
+                         cuComplex* Work, int lwork, int* devInfo)
+    int cusolverDnZgebrd(Handle handle, int m, int n,
+                         cuDoubleComplex* A, int lda,
+                         double* D, double* E,
+                         cuDoubleComplex* tauQ, cuDoubleComplex* tauP,
+                         cuDoubleComplex* Work, int lwork, int* devInfo)
+
+    # Singular value decomposition, A = U * Sigma * V^H
+    int cusolverDnSgesvd_bufferSize(Handle handle, int m, int n, int* lwork)
+    int cusolverDnDgesvd_bufferSize(Handle handle, int m, int n, int* lwork)
+    int cusolverDnCgesvd_bufferSize(Handle handle, int m, int n, int* lwork)
+    int cusolverDnZgesvd_bufferSize(Handle handle, int m, int n, int* lwork)
+
+    int cusolverDnSgesvd(Handle handle, char jobu, char jobvt, int m, int n,
+                         float* A, int lda, float* S,
+                         float* U, int ldu,
+                         float* VT, int ldvt,
+                         float* Work, int lwork,
+                         float* rwork, int* devInfo)
+    int cusolverDnDgesvd(Handle handle, char jobu, char jobvt, int m, int n,
+                         double* A, int lda, double* S,
+                         double* U, int ldu,
+                         double* VT, int ldvt,
+                         double* Work, int lwork,
+                         double* rwork, int* devInfo)
+    int cusolverDnCgesvd(Handle handle, char jobu, char jobvt, int m, int n,
+                         cuComplex* A, int lda, float* S,
+                         cuComplex* U, int ldu,
+                         cuComplex* VT, int ldvt,
+                         cuComplex* Work, int lwork,
+                         float* rwork, int* devInfo)
+    int cusolverDnZgesvd(Handle handle, char jobu, char jobvt, int m, int n,
+                         cuDoubleComplex* A, int lda, double* S,
+                         cuDoubleComplex* U, int ldu,
+                         cuDoubleComplex* VT, int ldvt,
+                         cuDoubleComplex* Work, int lwork,
+                         double* rwork, int* devInfo)
+
+    # gesvdj ... Singular value decomposition using Jacobi mathod
+    int cusolverDnCreateGesvdjInfo(GesvdjInfo *info)
+    int cusolverDnDestroyGesvdjInfo(GesvdjInfo info)
+
+    int cusolverDnXgesvdjSetTolerance(GesvdjInfo info, double tolerance)
+    int cusolverDnXgesvdjSetMaxSweeps(GesvdjInfo info, int max_sweeps)
+    int cusolverDnXgesvdjSetSortEig(GesvdjInfo info, int sort_svd)
+    int cusolverDnXgesvdjGetResidual(Handle handle, GesvdjInfo info,
+                                     double* residual)
+    int cusolverDnXgesvdjGetSweeps(Handle handle, GesvdjInfo info,
+                                   int* executed_sweeps)
+
+    int cusolverDnSgesvdj_bufferSize(Handle handle, EigMode jobz, int econ,
+                                     int m, int n, const float* A, int lda,
+                                     const float* S, const float* U, int ldu,
+                                     const float* V, int ldv, int* lwork,
+                                     GesvdjInfo params)
+    int cusolverDnDgesvdj_bufferSize(Handle handle, EigMode jobz, int econ,
+                                     int m, int n, const double* A, int lda,
+                                     const double* S, const double* U, int ldu,
+                                     const double* V, int ldv, int* lwork,
+                                     GesvdjInfo params)
+    int cusolverDnCgesvdj_bufferSize(Handle handle, EigMode jobz, int econ,
+                                     int m, int n, const cuComplex* A, int lda,
+                                     const float* S, const cuComplex* U,
+                                     int ldu, const cuComplex* V, int ldv,
+                                     int* lwork, GesvdjInfo params)
+    int cusolverDnZgesvdj_bufferSize(Handle handle, EigMode jobz, int econ,
+                                     int m, int n, const cuDoubleComplex* A,
+                                     int lda, const double* S,
+                                     const cuDoubleComplex* U, int ldu,
+                                     const cuDoubleComplex* V, int ldv,
+                                     int* lwork, GesvdjInfo params)
+
+    int cusolverDnSgesvdj(Handle handle, EigMode jobz, int econ, int m, int n,
+                          float *A, int lda, float *S, float *U, int ldu,
+                          float *V, int ldv, float *work, int lwork, int *info,
+                          GesvdjInfo params)
+    int cusolverDnDgesvdj(Handle handle, EigMode jobz, int econ, int m, int n,
+                          double *A, int lda, double *S, double *U, int ldu,
+                          double *V, int ldv, double *work, int lwork,
+                          int *info, GesvdjInfo params)
+    int cusolverDnCgesvdj(Handle handle, EigMode jobz, int econ, int m, int n,
+                          cuComplex *A, int lda, float *S, cuComplex *U,
+                          int ldu, cuComplex *V, int ldv, cuComplex *work,
+                          int lwork, int *info, GesvdjInfo params)
+    int cusolverDnZgesvdj(Handle handle, EigMode jobz, int econ, int m, int n,
+                          cuDoubleComplex *A, int lda, double *S,
+                          cuDoubleComplex *U, int ldu, cuDoubleComplex *V,
+                          int ldv, cuDoubleComplex *work, int lwork, int *info,
+                          GesvdjInfo params)
+
+    int cusolverDnSgesvdjBatched_bufferSize(
+        Handle handle, EigMode jobz, int m, int n, float* A, int lda,
+        float* S, float* U, int ldu, float* V, int ldv,
+        int* lwork, GesvdjInfo params, int batchSize)
+    int cusolverDnDgesvdjBatched_bufferSize(
+        Handle handle, EigMode jobz, int m, int n, double* A, int lda,
+        double* S, double* U, int ldu, double* V, int ldv,
+        int* lwork, GesvdjInfo params, int batchSize)
+    int cusolverDnCgesvdjBatched_bufferSize(
+        Handle handle, EigMode jobz, int m, int n, cuComplex* A, int lda,
+        float* S, cuComplex* U, int ldu, cuComplex* V, int ldv,
+        int* lwork, GesvdjInfo params, int batchSize)
+    int cusolverDnZgesvdjBatched_bufferSize(
+        Handle handle, EigMode jobz, int m, int n, cuDoubleComplex* A, int lda,
+        double* S, cuDoubleComplex* U, int ldu, cuDoubleComplex* V, int ldv,
+        int* lwork, GesvdjInfo params, int batchSize)
+    int cusolverDnSgesvdjBatched(
+        Handle handle, EigMode jobz, int m, int n, float* A, int lda, float* S,
+        float* U, int ldu, float* V, int ldv, float* work, int lwork,
+        int* info, GesvdjInfo params, int batchSize)
+    int cusolverDnDgesvdjBatched(
+        Handle handle, EigMode jobz, int m, int n, double* A, int lda,
+        double* S, double* U, int ldu, double* V, int ldv,
+        double* work, int lwork,
+        int* info, GesvdjInfo params, int batchSize)
+    int cusolverDnCgesvdjBatched(
+        Handle handle, EigMode jobz, int m, int n, cuComplex* A, int lda,
+        float* S, cuComplex* U, int ldu, cuComplex* V, int ldv,
+        cuComplex* work, int lwork,
+        int* info, GesvdjInfo params, int batchSize)
+    int cusolverDnZgesvdjBatched(
+        Handle handle, EigMode jobz, int m, int n, cuDoubleComplex* A, int lda,
+        double* S, cuDoubleComplex* U, int ldu, cuDoubleComplex* V, int ldv,
+        cuDoubleComplex* work, int lwork,
+        int* info, GesvdjInfo params, int batchSize)
+
+    # gesvda ... Approximate singular value decomposition
+    int cusolverDnSgesvdaStridedBatched_bufferSize(
+        Handle handle, EigMode jobz, int rank, int m, int n, const float *d_A,
+        int lda, long long int strideA, const float *d_S,
+        long long int strideS, const float *d_U, int ldu,
+        long long int strideU, const float *d_V, int ldv,
+        long long int strideV, int *lwork, int batchSize)
+
+    int cusolverDnDgesvdaStridedBatched_bufferSize(
+        Handle handle, EigMode jobz, int rank, int m, int n, const double *d_A,
+        int lda, long long int strideA, const double *d_S,
+        long long int strideS, const double *d_U, int ldu,
+        long long int strideU, const double *d_V, int ldv,
+        long long int strideV, int *lwork, int batchSize)
+
+    int cusolverDnCgesvdaStridedBatched_bufferSize(
+        Handle handle, EigMode jobz, int rank, int m, int n,
+        const cuComplex *d_A, int lda, long long int strideA, const float *d_S,
+        long long int strideS, const cuComplex *d_U, int ldu,
+        long long int strideU, const cuComplex *d_V, int ldv,
+        long long int strideV, int *lwork, int batchSize)
+
+    int cusolverDnZgesvdaStridedBatched_bufferSize(
+        Handle handle, EigMode jobz, int rank, int m, int n,
+        const cuDoubleComplex *d_A, int lda, long long int strideA,
+        const double *d_S, long long int strideS, const cuDoubleComplex *d_U,
+        int ldu, long long int strideU, const cuDoubleComplex *d_V, int ldv,
+        long long int strideV, int *lwork, int batchSize)
+
+    int cusolverDnSgesvdaStridedBatched(
+        Handle handle, EigMode jobz, int rank, int m, int n, const float *d_A,
+        int lda, long long int strideA, float *d_S, long long int strideS,
+        float *d_U, int ldu, long long int strideU, float *d_V, int ldv,
+        long long int strideV, float *d_work, int lwork, int *d_info,
+        double *h_R_nrmF, int batchSize)
+
+    int cusolverDnDgesvdaStridedBatched(
+        Handle handle, EigMode jobz, int rank, int m, int n, const double *d_A,
+        int lda, long long int strideA, double *d_S, long long int strideS,
+        double *d_U, int ldu, long long int strideU, double *d_V, int ldv,
+        long long int strideV, double *d_work, int lwork, int *d_info,
+        double *h_R_nrmF, int batchSize)
+
+    int cusolverDnCgesvdaStridedBatched(
+        Handle handle, EigMode jobz, int rank, int m, int n,
+        const cuComplex *d_A, int lda, long long int strideA, float *d_S,
+        long long int strideS, cuComplex *d_U, int ldu, long long int strideU,
+        cuComplex *d_V, int ldv, long long int strideV, cuComplex *d_work,
+        int lwork, int *d_info, double *h_R_nrmF, int batchSize)
+
+    int cusolverDnZgesvdaStridedBatched(
+        Handle handle, EigMode jobz, int rank, int m, int n,
+        const cuDoubleComplex *d_A, int lda, long long int strideA,
+        double *d_S, long long int strideS, cuDoubleComplex *d_U, int ldu,
+        long long int strideU, cuDoubleComplex *d_V, int ldv,
+        long long int strideV, cuDoubleComplex *d_work, int lwork, int *d_info,
+        double *h_R_nrmF, int batchSize)
+
+    # Standard symmetric eigenvalue solver
+    int cusolverDnSsyevd_bufferSize(Handle handle,
+                                    EigMode jobz, FillMode uplo, int n,
+                                    const float* A, int lda,
+                                    const float* W, int* lwork)
+    int cusolverDnDsyevd_bufferSize(Handle handle,
+                                    EigMode jobz, FillMode uplo, int n,
+                                    const double* A, int lda,
+                                    const double* W, int* lwork)
+    int cusolverDnCheevd_bufferSize(Handle handle,
+                                    EigMode jobz, FillMode uplo, int n,
+                                    const cuComplex* A, int lda,
+                                    const float* W, int* lwork)
+    int cusolverDnZheevd_bufferSize(Handle handle,
+                                    EigMode jobz, FillMode uplo, int n,
+                                    const cuDoubleComplex* A, int lda,
+                                    const double* W, int* lwork)
+
+    int cusolverDnSsyevd(Handle handle, EigMode jobz, FillMode uplo, int n,
+                         float* A, int lda, float* W,
+                         float* work, int lwork, int* info)
+    int cusolverDnDsyevd(Handle handle, EigMode jobz, FillMode uplo, int n,
+                         double* A, int lda, double* W,
+                         double* work, int lwork, int* info)
+    int cusolverDnCheevd(Handle handle, EigMode jobz, FillMode uplo, int n,
+                         cuComplex* A, int lda, float* W,
+                         cuComplex* work, int lwork, int* info)
+    int cusolverDnZheevd(Handle handle, EigMode jobz, FillMode uplo, int n,
+                         cuDoubleComplex* A, int lda, double* W,
+                         cuDoubleComplex* work, int lwork, int* info)
+
+    # Symmetric eigenvalue solver using Jacobi method
+    int cusolverDnCreateSyevjInfo(SyevjInfo *info)
+    int cusolverDnDestroySyevjInfo(SyevjInfo info)
+
+    int cusolverDnXsyevjSetTolerance(SyevjInfo info, double tolerance)
+    int cusolverDnXsyevjSetMaxSweeps(SyevjInfo info, int max_sweeps)
+    int cusolverDnXsyevjSetSortEig(SyevjInfo info, int sort_eig)
+    int cusolverDnXsyevjGetResidual(
+        Handle handle, SyevjInfo info, double* residual)
+    int cusolverDnXsyevjGetSweeps(
+        Handle handle, SyevjInfo info, int* executed_sweeps)
+
+    int cusolverDnSsyevj_bufferSize(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        const float *A, int lda, const float *W, int *lwork,
+        SyevjInfo params)
+    int cusolverDnDsyevj_bufferSize(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        const double *A, int lda, const double *W, int *lwork,
+        SyevjInfo params)
+    int cusolverDnCheevj_bufferSize(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        const cuComplex *A, int lda, const float *W, int *lwork,
+        SyevjInfo params)
+    int cusolverDnZheevj_bufferSize(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        const cuDoubleComplex *A, int lda, const double *W, int *lwork,
+        SyevjInfo params)
+
+    int cusolverDnSsyevj(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        float *A, int lda, float *W, float *work,
+        int lwork, int *info, SyevjInfo params)
+    int cusolverDnDsyevj(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        double *A, int lda, double *W, double *work,
+        int lwork, int *info, SyevjInfo params)
+    int cusolverDnCheevj(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        cuComplex *A, int lda, float *W, cuComplex *work,
+        int lwork, int *info, SyevjInfo params)
+    int cusolverDnZheevj(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        cuDoubleComplex *A, int lda, double *W, cuDoubleComplex *work,
+        int lwork, int *info, SyevjInfo params)
+
+    int cusolverDnSsyevjBatched_bufferSize(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        const float *A, int lda, const float *W, int *lwork,
+        SyevjInfo params, int batchSize)
+
+    int cusolverDnDsyevjBatched_bufferSize(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        const double *A, int lda, const double *W, int *lwork,
+        SyevjInfo params, int batchSize)
+
+    int cusolverDnCheevjBatched_bufferSize(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        const cuComplex *A, int lda, const float *W, int *lwork,
+        SyevjInfo params, int batchSize)
+
+    int cusolverDnZheevjBatched_bufferSize(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        const cuDoubleComplex *A, int lda, const double *W, int *lwork,
+        SyevjInfo params, int batchSize)
+
+    int cusolverDnSsyevjBatched(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        float *A, int lda, float *W, float *work, int lwork,
+        int *info, SyevjInfo params, int batchSize)
+
+    int cusolverDnDsyevjBatched(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        double *A, int lda, double *W, double *work, int lwork,
+        int *info, SyevjInfo params, int batchSize)
+
+    int cusolverDnCheevjBatched(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        cuComplex *A, int lda, float *W, cuComplex *work, int lwork,
+        int *info, SyevjInfo params, int batchSize)
+
+    int cusolverDnZheevjBatched(
+        Handle handle, EigMode jobz, FillMode uplo, int n,
+        cuDoubleComplex *A, int lda, double *W, cuDoubleComplex *work,
+        int lwork, int *info, SyevjInfo params, int batchSize)
+
+    # 64bit
+    int cusolverDnXsyevd_bufferSize(
+        Handle handle, Params params, EigMode jobz, FillMode uplo, int64_t n,
+        DataType dataTypeA, void *A, int64_t lda,
+        DataType dataTypeW, void *W, DataType computeType,
+        size_t *workspaceInBytesOnDevice, size_t *workspaceInBytesOnHost)
+    int cusolverDnXsyevd(
+        Handle handle, Params params, EigMode jobz, FillMode uplo, int64_t n,
+        DataType dataTypeA, void *A, int64_t lda,
+        DataType dataTypeW, void *W, DataType computeType,
+        void *bufferOnDevice, size_t workspaceInBytesOnDevice,
+        void *bufferOnHost, size_t workspaceInBytesOnHost, int *info)
+
+    ###########################################################################
+    # Sparse LAPACK Functions
+    ###########################################################################
+
+    int cusolverSpScsrlsvchol(
+        SpHandle handle, int m, int nnz, const MatDescr descrA,
+        const float* csrValA, const int* csrRowPtrA, const int* csrColIndA,
+        const float* b, float tol, int reorder, float* x, int* singularity)
+    int cusolverSpDcsrlsvchol(
+        SpHandle handle, int m, int nnz, const MatDescr descrA,
+        const double* csrValA, const int* csrRowPtrA, const int* csrColIndA,
+        const double* b, double tol, int reorder, double* x, int* singularity)
+    int cusolverSpCcsrlsvchol(
+        SpHandle handle, int m, int nnz,
+        const MatDescr descrA, const cuComplex *csrVal,
+        const int *csrRowPtr, const int *csrColInd, const cuComplex *b,
+        float tol, int reorder, cuComplex *x, int *singularity)
+    int cusolverSpZcsrlsvchol(
+        SpHandle handle, int m, int nnz,
+        const MatDescr descrA, const cuDoubleComplex *csrVal,
+        const int *csrRowPtr, const int *csrColInd, const cuDoubleComplex *b,
+        double tol, int reorder, cuDoubleComplex *x, int *singularity)
+
+    int cusolverSpScsrlsvqr(
+        SpHandle handle, int m, int nnz, const MatDescr descrA,
+        const float* csrValA, const int* csrRowPtrA, const int* csrColIndA,
+        const float* b, float tol, int reorder, float* x, int* singularity)
+    int cusolverSpDcsrlsvqr(
+        SpHandle handle, int m, int nnz, const MatDescr descrA,
+        const double* csrValA, const int* csrRowPtrA, const int* csrColIndA,
+        const double* b, double tol, int reorder, double* x, int* singularity)
+    int cusolverSpCcsrlsvqr(
+        SpHandle handle, int m, int nnz,
+        const MatDescr descrA, const cuComplex *csrVal,
+        const int *csrRowPtr, const int *csrColInd, const cuComplex *b,
+        float tol, int reorder, cuComplex *x, int *singularity)
+    int cusolverSpZcsrlsvqr(
+        SpHandle handle, int m, int nnz,
+        const MatDescr descrA, const cuDoubleComplex *csrVal,
+        const int *csrRowPtr, const int *csrColInd, const cuDoubleComplex *b,
+        double tol, int reorder, cuDoubleComplex *x, int *singularity)
+
+    int cusolverSpScsreigvsi(
+        SpHandle handle, int m, int nnz,
+        const MatDescr descrA, const float *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, float mu0,
+        const float *x0, int maxite, float eps, float *mu, float *x)
+    int cusolverSpDcsreigvsi(
+        SpHandle handle, int m, int nnz,
+        const MatDescr descrA, const double *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, double mu0,
+        const double *x0, int maxite, double eps, double *mu, double *x)
+    int cusolverSpCcsreigvsi(
+        SpHandle handle, int m, int nnz,
+        const MatDescr descrA, const cuComplex *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, cuComplex mu0,
+        const cuComplex *x0, int maxite, float eps, cuComplex *mu,
+        cuComplex *x)
+    int cusolverSpZcsreigvsi(
+        SpHandle handle, int m, int nnz,
+        const MatDescr descrA, const cuDoubleComplex *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, cuDoubleComplex mu0,
+        const cuDoubleComplex *x0, int maxite, double eps, cuDoubleComplex *mu,
+        cuDoubleComplex *x)
+
+###############################################################################
+# Error handling
+###############################################################################
+
+cdef dict STATUS = {
+    0: 'CUSOLVER_STATUS_SUCCESS',
+    1: 'CUSOLVER_STATUS_NOT_INITIALIZED',
+    2: 'CUSOLVER_STATUS_ALLOC_FAILED',
+    3: 'CUSOLVER_STATUS_INVALID_VALUE',
+    4: 'CUSOLVER_STATUS_ARCH_MISMATCH',
+    5: 'CUSOLVER_STATUS_MAPPING_ERROR',
+    6: 'CUSOLVER_STATUS_EXECUTION_FAILED',
+    7: 'CUSOLVER_STATUS_INTERNAL_ERROR',
+    8: 'CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED',
+    9: 'CUSOLVER_STATUS_NOT_SUPPORTED',
+    10: 'CUSOLVER_STATUS_ZERO_PIVOT',
+    11: 'CUSOLVER_STATUS_INVALID_LICENSE',
+    12: 'CUSOLVER_STATUS_IRS_PARAMS_NOT_INITIALIZED',
+    13: 'CUSOLVER_STATUS_IRS_PARAMS_INVALID',
+    14: 'CUSOLVER_STATUS_IRS_PARAMS_INVALID_PREC',
+    15: 'CUSOLVER_STATUS_IRS_PARAMS_INVALID_REFINE',
+    16: 'CUSOLVER_STATUS_IRS_PARAMS_INVALID_MAXITER',
+    20: 'CUSOLVER_STATUS_IRS_INTERNAL_ERROR',
+    21: 'CUSOLVER_STATUS_IRS_NOT_SUPPORTED',
+    22: 'CUSOLVER_STATUS_IRS_OUT_OF_RANGE',
+    23: 'CUSOLVER_STATUS_IRS_NRHS_NOT_SUPPORTED_FOR_REFINE_GMRES',
+    25: 'CUSOLVER_STATUS_IRS_INFOS_NOT_INITIALIZED',
+    26: 'CUSOLVER_STATUS_IRS_INFOS_NOT_DESTROYED',
+    30: 'CUSOLVER_STATUS_IRS_MATRIX_SINGULAR',
+    31: 'CUSOLVER_STATUS_INVALID_WORKSPACE',
+}
+
+# for rocBLAS and rocSOLVER
+cdef dict ROC_STATUS = {
+    0: 'rocblas_status_success',
+    1: 'rocblas_status_invalid_handle',
+    2: 'rocblas_status_not_implemented',
+    3: 'rocblas_status_invalid_pointer',
+    4: 'rocblas_status_invalid_size',
+    5: 'rocblas_status_memory_error',
+    6: 'rocblas_status_internal_error',
+    7: 'rocblas_status_perf_degraded',
+    8: 'rocblas_status_size_query_mismatch',
+    9: 'rocblas_status_size_increased',
+    10: 'rocblas_status_size_unchanged',
+    11: 'rocblas_status_invalid_value',
+    12: 'rocblas_status_continue',
+}
+
+
+class CUSOLVERError(RuntimeError):
+
+    def __init__(self, status):
+        self.status = status
+        if runtime._is_hip_environment:
+            err = ROC_STATUS
+        else:
+            err = STATUS
+        super(CUSOLVERError, self).__init__(err[status])
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        raise CUSOLVERError(status)
+
+
+###############################################################################
+# Library Attributes
+###############################################################################
+
+cpdef int getProperty(int type) except? -1:
+    cdef int value
+    with nogil:
+        status = cusolverGetProperty(<LibraryPropertyType>type, &value)
+    check_status(status)
+    return value
+
+
+cpdef tuple _getVersion():
+    return (getProperty(MAJOR_VERSION),
+            getProperty(MINOR_VERSION),
+            getProperty(PATCH_LEVEL))
+
+
+###############################################################################
+# Context
+###############################################################################
+
+cpdef intptr_t create() except? 0:
+    cdef Handle handle
+    with nogil:
+        status = cusolverDnCreate(&handle)
+    check_status(status)
+    return <intptr_t>handle
+
+
+cpdef intptr_t spCreate() except? 0:
+    cdef SpHandle handle
+    with nogil:
+        status = cusolverSpCreate(&handle)
+    check_status(status)
+    return <intptr_t>handle
+
+
+cpdef destroy(intptr_t handle):
+    with nogil:
+        status = cusolverDnDestroy(<Handle>handle)
+    check_status(status)
+
+
+cpdef spDestroy(intptr_t handle):
+    with nogil:
+        status = cusolverSpDestroy(<SpHandle>handle)
+    check_status(status)
+
+
+###############################################################################
+# Stream
+###############################################################################
+
+cpdef setStream(intptr_t handle, size_t stream):
+    # TODO(leofang): The support of stream capture is not mentioned at all in
+    # the cuSOLVER docs (as of CUDA 11.5), so we disable this functionality.
+    if not runtime._is_hip_environment and runtime.streamIsCapturing(stream):
+        raise NotImplementedError(
+            'calling cuSOLVER API during stream capture is currently '
+            'unsupported')
+
+    with nogil:
+        status = cusolverDnSetStream(<Handle>handle, <Stream>stream)
+    check_status(status)
+
+
+cpdef size_t getStream(intptr_t handle) except? 0:
+    cdef Stream stream
+    with nogil:
+        status = cusolverDnGetStream(<Handle>handle, &stream)
+    check_status(status)
+    return <size_t>stream
+
+
+cpdef spSetStream(intptr_t handle, size_t stream):
+    with nogil:
+        status = cusolverSpSetStream(<SpHandle>handle, <Stream>stream)
+    check_status(status)
+
+
+cpdef size_t spGetStream(intptr_t handle) except *:
+    cdef Stream stream
+    with nogil:
+        status = cusolverSpGetStream(<SpHandle>handle, &stream)
+    check_status(status)
+    return <size_t>stream
+
+
+cdef _setStream(intptr_t handle):
+    """Set current stream"""
+    setStream(handle, stream_module.get_current_stream_ptr())
+
+
+cdef _spSetStream(intptr_t handle):
+    """Set current stream"""
+    spSetStream(handle, stream_module.get_current_stream_ptr())
+
+
+###############################################################################
+# Params
+###############################################################################
+
+cpdef intptr_t createParams() except? 0:
+    cdef Params params
+    with nogil:
+        status = cusolverDnCreateParams(&params)
+    check_status(status)
+    return <intptr_t>params
+
+cpdef destroyParams(intptr_t params):
+    with nogil:
+        status = cusolverDnDestroyParams(<Params>params)
+    check_status(status)
+
+
+###########################################################################
+# Dense LAPACK Functions (Linear Solver)
+###########################################################################
+
+# Cholesky factorization
+cpdef int spotrf_bufferSize(intptr_t handle, int uplo,
+                            int n, size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSpotrf_bufferSize(
+            <Handle>handle, <FillMode>uplo, n,
+            <float*>A, <int>lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dpotrf_bufferSize(intptr_t handle, int uplo,
+                            int n, size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDpotrf_bufferSize(
+            <Handle>handle, <FillMode>uplo, n,
+            <double*>A, <int>lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int cpotrf_bufferSize(intptr_t handle, int uplo,
+                            int n, size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCpotrf_bufferSize(
+            <Handle>handle, <FillMode>uplo, n,
+            <cuComplex*>A, <int>lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zpotrf_bufferSize(intptr_t handle, int uplo,
+                            int n, size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZpotrf_bufferSize(
+            <Handle>handle, <FillMode>uplo, n,
+            <cuDoubleComplex*>A, <int>lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef spotrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSpotrf(
+            <Handle>handle, <FillMode>uplo, n, <float*>A,
+            lda, <float*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef dpotrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDpotrf(
+            <Handle>handle, <FillMode>uplo, n, <double*>A,
+            lda, <double*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef cpotrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCpotrf(
+            <Handle>handle, <FillMode>uplo, n, <cuComplex*>A,
+            lda, <cuComplex*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef zpotrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZpotrf(
+            <Handle>handle, <FillMode>uplo, n, <cuDoubleComplex*>A,
+            lda, <cuDoubleComplex*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef spotrs(intptr_t handle, int uplo, int n, int nrhs,
+             size_t A, int lda, size_t B, int ldb, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSpotrs(
+            <Handle>handle, <FillMode>uplo, n, nrhs,
+            <const float*>A, lda, <float*>B, ldb,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef dpotrs(intptr_t handle, int uplo, int n, int nrhs,
+             size_t A, int lda, size_t B, int ldb, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDpotrs(
+            <Handle>handle, <FillMode>uplo, n, nrhs,
+            <const double*>A, lda, <double*>B, ldb,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef cpotrs(intptr_t handle, int uplo, int n, int nrhs,
+             size_t A, int lda, size_t B, int ldb, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCpotrs(
+            <Handle>handle, <FillMode>uplo, n, nrhs,
+            <const cuComplex*>A, lda, <cuComplex*>B, ldb,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef zpotrs(intptr_t handle, int uplo, int n, int nrhs,
+             size_t A, int lda, size_t B, int ldb, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZpotrs(
+            <Handle>handle, <FillMode>uplo, n, nrhs,
+            <const cuDoubleComplex*>A, lda, <cuDoubleComplex*>B, ldb,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef spotrfBatched(intptr_t handle, int uplo, int n, size_t Aarray, int lda,
+                    size_t infoArray, int batchSize):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnSpotrfBatched(
+            <Handle>handle, <FillMode>uplo, n, <float**>Aarray,
+            lda, <int*>infoArray, batchSize)
+    check_status(status)
+
+cpdef dpotrfBatched(intptr_t handle, int uplo, int n, size_t Aarray, int lda,
+                    size_t infoArray, int batchSize):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnDpotrfBatched(
+            <Handle>handle, <FillMode>uplo, n, <double**>Aarray,
+            lda, <int*>infoArray, batchSize)
+    check_status(status)
+
+cpdef cpotrfBatched(intptr_t handle, int uplo, int n, size_t Aarray, int lda,
+                    size_t infoArray, int batchSize):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnCpotrfBatched(
+            <Handle>handle, <FillMode>uplo, n, <cuComplex**>Aarray,
+            lda, <int*>infoArray, batchSize)
+    check_status(status)
+
+cpdef zpotrfBatched(intptr_t handle, int uplo, int n, size_t Aarray, int lda,
+                    size_t infoArray, int batchSize):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnZpotrfBatched(
+            <Handle>handle, <FillMode>uplo, n, <cuDoubleComplex**>Aarray,
+            lda, <int*>infoArray, batchSize)
+    check_status(status)
+
+cpdef spotrsBatched(intptr_t handle, int uplo, int n, int nrhs, size_t Aarray,
+                    int lda, size_t Barray, int ldb, size_t devInfo,
+                    int batchSize):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnSpotrsBatched(
+            <Handle>handle, <FillMode>uplo, n, nrhs,
+            <float**>Aarray, lda, <float**>Barray, ldb,
+            <int*>devInfo, batchSize)
+    check_status(status)
+
+cpdef dpotrsBatched(intptr_t handle, int uplo, int n, int nrhs, size_t Aarray,
+                    int lda, size_t Barray, int ldb, size_t devInfo,
+                    int batchSize):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnDpotrsBatched(
+            <Handle>handle, <FillMode>uplo, n, nrhs,
+            <double**>Aarray, lda, <double**>Barray, ldb,
+            <int*>devInfo, batchSize)
+    check_status(status)
+
+cpdef cpotrsBatched(intptr_t handle, int uplo, int n, int nrhs, size_t Aarray,
+                    int lda, size_t Barray, int ldb, size_t devInfo,
+                    int batchSize):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnCpotrsBatched(
+            <Handle>handle, <FillMode>uplo, n, nrhs,
+            <cuComplex**>Aarray, lda, <cuComplex**>Barray, ldb,
+            <int*>devInfo, batchSize)
+    check_status(status)
+
+cpdef zpotrsBatched(intptr_t handle, int uplo, int n, int nrhs, size_t Aarray,
+                    int lda, size_t Barray, int ldb, size_t devInfo,
+                    int batchSize):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnZpotrsBatched(
+            <Handle>handle, <FillMode>uplo, n, nrhs,
+            <cuDoubleComplex**>Aarray, lda, <cuDoubleComplex**>Barray, ldb,
+            <int*>devInfo, batchSize)
+    check_status(status)
+
+# LU factorization
+cpdef int sgetrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgetrf_bufferSize(
+            <Handle>handle, m, n, <float*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dgetrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgetrf_bufferSize(
+            <Handle>handle, m, n, <double*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int cgetrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgetrf_bufferSize(
+            <Handle>handle, m, n, <cuComplex*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zgetrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgetrf_bufferSize(
+            <Handle>handle, m, n, <cuDoubleComplex*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef sgetrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t work, size_t devIpiv, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgetrf(
+            <Handle>handle, m, n, <float*>A, lda,
+            <float*>work, <int*>devIpiv, <int*>devInfo)
+    check_status(status)
+
+cpdef dgetrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t work, size_t devIpiv, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgetrf(
+            <Handle>handle, m, n, <double*>A, lda,
+            <double*>work, <int*>devIpiv, <int*>devInfo)
+    check_status(status)
+
+cpdef cgetrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t work, size_t devIpiv, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgetrf(
+            <Handle>handle, m, n, <cuComplex*>A, lda,
+            <cuComplex*>work, <int*>devIpiv, <int*>devInfo)
+    check_status(status)
+
+cpdef zgetrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t work, size_t devIpiv, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgetrf(
+            <Handle>handle, m, n, <cuDoubleComplex*>A, lda,
+            <cuDoubleComplex*>work, <int*>devIpiv, <int*>devInfo)
+    check_status(status)
+
+
+# LU solve
+cpdef sgetrs(intptr_t handle, int trans, int n, int nrhs,
+             size_t A, int lda, size_t devIpiv,
+             size_t B, int ldb, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgetrs(
+            <Handle>handle, <Operation>trans, n, nrhs,
+            <const float*> A, lda, <const int*>devIpiv,
+            <float*>B, ldb, <int*> devInfo)
+    check_status(status)
+
+cpdef dgetrs(intptr_t handle, int trans, int n, int nrhs,
+             size_t A, int lda, size_t devIpiv,
+             size_t B, int ldb, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgetrs(
+            <Handle>handle, <Operation>trans, n, nrhs,
+            <const double*> A, lda, <const int*>devIpiv,
+            <double*>B, ldb, <int*> devInfo)
+    check_status(status)
+
+cpdef cgetrs(intptr_t handle, int trans, int n, int nrhs,
+             size_t A, int lda, size_t devIpiv,
+             size_t B, int ldb, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgetrs(
+            <Handle>handle, <Operation>trans, n, nrhs,
+            <const cuComplex*> A, lda, <const int*>devIpiv,
+            <cuComplex*>B, ldb, <int*> devInfo)
+    check_status(status)
+
+cpdef zgetrs(intptr_t handle, int trans, int n, int nrhs,
+             size_t A, int lda, size_t devIpiv,
+             size_t B, int ldb, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgetrs(
+            <Handle>handle, <Operation>trans, n, nrhs,
+            <const cuDoubleComplex*> A, lda, <const int*>devIpiv,
+            <cuDoubleComplex*>B, ldb, <int*> devInfo)
+    check_status(status)
+
+
+# QR factorization
+cpdef int sgeqrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgeqrf_bufferSize(
+            <Handle>handle, m, n, <float*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dgeqrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgeqrf_bufferSize(
+            <Handle>handle, m, n, <double*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int cgeqrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgeqrf_bufferSize(
+            <Handle>handle, m, n, <cuComplex*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zgeqrf_bufferSize(intptr_t handle, int m, int n,
+                            size_t A, int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgeqrf_bufferSize(
+            <Handle>handle, m, n, <cuDoubleComplex*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef sgeqrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgeqrf(
+            <Handle>handle, m, n, <float*>A, lda,
+            <float*>tau, <float*>work, lwork,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef dgeqrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgeqrf(
+            <Handle>handle, m, n, <double*>A, lda,
+            <double*>tau, <double*>work, lwork,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef cgeqrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgeqrf(
+            <Handle>handle, m, n, <cuComplex*>A, lda,
+            <cuComplex*>tau, <cuComplex*>work, lwork,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef zgeqrf(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgeqrf(
+            <Handle>handle, m, n, <cuDoubleComplex*>A, lda,
+            <cuDoubleComplex*>tau, <cuDoubleComplex*>work, lwork,
+            <int*>devInfo)
+    check_status(status)
+
+
+# Generate unitary matrix Q from QR factorization
+cpdef int sorgqr_bufferSize(intptr_t handle, int m, int n, int k,
+                            size_t A, int lda, size_t tau) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSorgqr_bufferSize(
+            <Handle>handle, m, n, k, <const float*>A, lda,
+            <const float*>tau, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dorgqr_bufferSize(intptr_t handle, int m, int n, int k,
+                            size_t A, int lda, size_t tau) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDorgqr_bufferSize(
+            <Handle>handle, m, n, k, <const double*>A, lda,
+            <const double*>tau, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int cungqr_bufferSize(intptr_t handle, int m, int n, int k,
+                            size_t A, int lda, size_t tau) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCungqr_bufferSize(
+            <Handle>handle, m, n, k, <const cuComplex*>A, lda,
+            <const cuComplex*>tau, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zungqr_bufferSize(intptr_t handle, int m, int n, int k,
+                            size_t A, int lda, size_t tau) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZungqr_bufferSize(
+            <Handle>handle, m, n, k, <const cuDoubleComplex*>A, lda,
+            <const cuDoubleComplex*>tau, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef sorgqr(intptr_t handle, int m, int n, int k, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSorgqr(
+            <Handle>handle, m, n, k, <float*>A, lda,
+            <const float*>tau, <float*>work, lwork,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef dorgqr(intptr_t handle, int m, int n, int k, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDorgqr(
+            <Handle>handle, m, n, k, <double*>A, lda,
+            <const double*>tau, <double*>work, lwork,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef cungqr(intptr_t handle, int m, int n, int k, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCungqr(
+            <Handle>handle, m, n, k, <cuComplex*>A, lda,
+            <const cuComplex*>tau, <cuComplex*>work, lwork,
+            <int*>devInfo)
+    check_status(status)
+
+cpdef zungqr(intptr_t handle, int m, int n, int k, size_t A, int lda,
+             size_t tau, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZungqr(
+            <Handle>handle, m, n, k, <cuDoubleComplex*>A, lda,
+            <const cuDoubleComplex*>tau, <cuDoubleComplex*>work, lwork,
+            <int*>devInfo)
+    check_status(status)
+
+
+# Compute Q**T*b in solve min||A*x = b||
+cpdef int sormqr_bufferSize(intptr_t handle, int side, int trans,
+                            int m, int n, int k, size_t A, int lda, size_t tau,
+                            size_t C, int ldc) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSormqr_bufferSize(
+            <Handle>handle, <SideMode>side, <Operation>trans, m, n, k,
+            <const float*>A, lda, <const float*>tau,
+            <float*>C, ldc, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dormqr_bufferSize(intptr_t handle, int side, int trans,
+                            int m, int n, int k, size_t A, int lda, size_t tau,
+                            size_t C, int ldc) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDormqr_bufferSize(
+            <Handle>handle, <SideMode>side, <Operation>trans, m, n, k,
+            <const double*>A, lda, <const double*>tau,
+            <double*>C, ldc, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int cunmqr_bufferSize(intptr_t handle, int side, int trans,
+                            int m, int n, int k, size_t A, int lda, size_t tau,
+                            size_t C, int ldc) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCunmqr_bufferSize(
+            <Handle>handle, <SideMode>side, <Operation>trans, m, n, k,
+            <const cuComplex*>A, lda, <const cuComplex*>tau,
+            <cuComplex*>C, ldc, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zunmqr_bufferSize(intptr_t handle, int side, int trans,
+                            int m, int n, int k, size_t A, int lda, size_t tau,
+                            size_t C, int ldc) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZunmqr_bufferSize(
+            <Handle>handle, <SideMode>side, <Operation>trans, m, n, k,
+            <const cuDoubleComplex*>A, lda, <const cuDoubleComplex*>tau,
+            <cuDoubleComplex*>C, ldc, &lwork)
+    check_status(status)
+    return lwork
+
+
+cpdef sormqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau,
+             size_t C, int ldc, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSormqr(
+            <Handle>handle, <SideMode>side, <Operation>trans, m, n, k,
+            <const float*>A, lda, <const float*>tau,
+            <float*>C, ldc,
+            <float*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef dormqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau,
+             size_t C, int ldc, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDormqr(
+            <Handle>handle, <SideMode>side, <Operation>trans, m, n, k,
+            <const double*>A, lda, <const double*>tau,
+            <double*>C, ldc,
+            <double*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef cunmqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau,
+             size_t C, int ldc, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCunmqr(
+            <Handle>handle, <SideMode>side, <Operation>trans, m, n, k,
+            <const cuComplex*>A, lda, <const cuComplex*>tau,
+            <cuComplex*>C, ldc,
+            <cuComplex*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef zunmqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau,
+             size_t C, int ldc, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZunmqr(
+            <Handle>handle, <SideMode>side, <Operation>trans, m, n, k,
+            <const cuDoubleComplex*>A, lda, <const cuDoubleComplex*>tau,
+            <cuDoubleComplex*>C, ldc,
+            <cuDoubleComplex*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+# (obsoleted)
+cpdef cormqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau,
+             size_t C, int ldc, size_t work, int lwork, size_t devInfo):
+    return cunmqr(handle, side, trans, m, n, k, A, lda, tau,
+                  C, ldc, work, lwork, devInfo)
+
+# (obsoleted)
+cpdef zormqr(intptr_t handle, int side, int trans,
+             int m, int n, int k, size_t A, int lda, size_t tau,
+             size_t C, int ldc, size_t work, int lwork, size_t devInfo):
+    return zunmqr(handle, side, trans, m, n, k, A, lda, tau,
+                  C, ldc, work, lwork, devInfo)
+
+
+# L*D*L**T,U*D*U**T factorization
+cpdef int ssytrf_bufferSize(intptr_t handle, int n, size_t A,
+                            int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSsytrf_bufferSize(
+            <Handle>handle, n, <float*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dsytrf_bufferSize(intptr_t handle, int n, size_t A,
+                            int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDsytrf_bufferSize(
+            <Handle>handle, n, <double*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int csytrf_bufferSize(intptr_t handle, int n, size_t A,
+                            int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCsytrf_bufferSize(
+            <Handle>handle, n, <cuComplex*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zsytrf_bufferSize(intptr_t handle, int n, size_t A,
+                            int lda) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZsytrf_bufferSize(
+            <Handle>handle, n, <cuDoubleComplex*>A, lda, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef ssytrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t ipiv, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSsytrf(
+            <Handle>handle, <FillMode>uplo, n, <float*>A, lda,
+            <int*>ipiv, <float*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef dsytrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t ipiv, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDsytrf(
+            <Handle>handle, <FillMode>uplo, n, <double*>A, lda,
+            <int*>ipiv, <double*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef csytrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t ipiv, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCsytrf(
+            <Handle>handle, <FillMode>uplo, n, <cuComplex*>A, lda,
+            <int*>ipiv, <cuComplex*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef zsytrf(intptr_t handle, int uplo, int n, size_t A, int lda,
+             size_t ipiv, size_t work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZsytrf(
+            <Handle>handle, <FillMode>uplo, n, <cuDoubleComplex*>A, lda,
+            <int*>ipiv, <cuDoubleComplex*>work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef size_t zzgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZZgesv_bufferSize(
+            <Handle>handle, n, nrhs, <cuDoubleComplex*>dA, ldda, <int*>dipiv,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t zcgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZCgesv_bufferSize(
+            <Handle>handle, n, nrhs, <cuDoubleComplex*>dA, ldda, <int*>dipiv,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t zygesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZYgesv_bufferSize(
+            <Handle>handle, n, nrhs, <cuDoubleComplex*>dA, ldda, <int*>dipiv,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t zkgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZKgesv_bufferSize(
+            <Handle>handle, n, nrhs, <cuDoubleComplex*>dA, ldda, <int*>dipiv,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t ccgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCCgesv_bufferSize(
+            <Handle>handle, n, nrhs, <cuComplex*>dA, ldda, <int*>dipiv,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t cygesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCYgesv_bufferSize(
+            <Handle>handle, n, nrhs, <cuComplex*>dA, ldda, <int*>dipiv,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t ckgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCKgesv_bufferSize(
+            <Handle>handle, n, nrhs, <cuComplex*>dA, ldda, <int*>dipiv,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t ddgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDDgesv_bufferSize(
+            <Handle>handle, n, nrhs, <double*>dA, ldda, <int*>dipiv,
+            <double*>dB, lddb, <double*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t dsgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDSgesv_bufferSize(
+            <Handle>handle, n, nrhs, <double*>dA, ldda, <int*>dipiv,
+            <double*>dB, lddb, <double*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t dxgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDXgesv_bufferSize(
+            <Handle>handle, n, nrhs, <double*>dA, ldda, <int*>dipiv,
+            <double*>dB, lddb, <double*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t dhgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDHgesv_bufferSize(
+            <Handle>handle, n, nrhs, <double*>dA, ldda, <int*>dipiv,
+            <double*>dB, lddb, <double*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t ssgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSSgesv_bufferSize(
+            <Handle>handle, n, nrhs, <float*>dA, ldda, <int*>dipiv,
+            <float*>dB, lddb, <float*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t sxgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSXgesv_bufferSize(
+            <Handle>handle, n, nrhs, <float*>dA, ldda, <int*>dipiv,
+            <float*>dB, lddb, <float*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t shgesv_bufferSize(intptr_t handle, int n, int nrhs, size_t dA,
+                               int ldda, size_t dipiv, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSHgesv_bufferSize(
+            <Handle>handle, n, nrhs, <float*>dA, ldda, <int*>dipiv,
+            <float*>dB, lddb, <float*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zzgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZZgesv(
+            <Handle>handle, n, nrhs, <cuDoubleComplex*>dA, ldda, <int*>dipiv,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int zcgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZCgesv(
+            <Handle>handle, n, nrhs, <cuDoubleComplex*>dA, ldda, <int*>dipiv,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int zygesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZYgesv(
+            <Handle>handle, n, nrhs, <cuDoubleComplex*>dA, ldda, <int*>dipiv,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int zkgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZKgesv(
+            <Handle>handle, n, nrhs, <cuDoubleComplex*>dA, ldda, <int*>dipiv,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int ccgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCCgesv(
+            <Handle>handle, n, nrhs, <cuComplex*>dA, ldda, <int*>dipiv,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int cygesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCYgesv(
+            <Handle>handle, n, nrhs, <cuComplex*>dA, ldda, <int*>dipiv,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int ckgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCKgesv(
+            <Handle>handle, n, nrhs, <cuComplex*>dA, ldda, <int*>dipiv,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int ddgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDDgesv(
+            <Handle>handle, n, nrhs, <double*>dA, ldda, <int*>dipiv,
+            <double*>dB, lddb, <double*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int dsgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDSgesv(
+            <Handle>handle, n, nrhs, <double*>dA, ldda, <int*>dipiv,
+            <double*>dB, lddb, <double*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int dxgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDXgesv(
+            <Handle>handle, n, nrhs, <double*>dA, ldda, <int*>dipiv,
+            <double*>dB, lddb, <double*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int dhgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDHgesv(
+            <Handle>handle, n, nrhs, <double*>dA, ldda, <int*>dipiv,
+            <double*>dB, lddb, <double*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int ssgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSSgesv(
+            <Handle>handle, n, nrhs, <float*>dA, ldda, <int*>dipiv,
+            <float*>dB, lddb, <float*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int sxgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSXgesv(
+            <Handle>handle, n, nrhs, <float*>dA, ldda, <int*>dipiv,
+            <float*>dB, lddb, <float*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int shgesv(intptr_t handle, int n, int nrhs, size_t dA, int ldda,
+                 size_t dipiv, size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSHgesv(
+            <Handle>handle, n, nrhs, <float*>dA, ldda, <int*>dipiv,
+            <float*>dB, lddb, <float*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef size_t zzgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZZgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <cuDoubleComplex*>dA, ldda,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t zcgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZCgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <cuDoubleComplex*>dA, ldda,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t zygels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZYgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <cuDoubleComplex*>dA, ldda,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t zkgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZKgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <cuDoubleComplex*>dA, ldda,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t ccgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCCgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <cuComplex*>dA, ldda,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t cygels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCYgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <cuComplex*>dA, ldda,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t ckgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCKgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <cuComplex*>dA, ldda,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t ddgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDDgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <double*>dA, ldda,
+            <double*>dB, lddb, <double*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t dsgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDSgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <double*>dA, ldda,
+            <double*>dB, lddb, <double*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t dxgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDXgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <double*>dA, ldda,
+            <double*>dB, lddb, <double*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t dhgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDHgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <double*>dA, ldda,
+            <double*>dB, lddb, <double*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t ssgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSSgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <float*>dA, ldda,
+            <float*>dB, lddb, <float*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t sxgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSXgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <float*>dA, ldda,
+            <float*>dB, lddb, <float*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef size_t shgels_bufferSize(intptr_t handle, int m, int n, int nrhs,
+                               size_t dA, int ldda, size_t dB, int lddb,
+                               size_t dX, int lddx, size_t dwork) except? -1:
+    cdef size_t lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSHgels_bufferSize(
+            <Handle>handle, m, n, nrhs, <float*>dA, ldda,
+            <float*>dB, lddb, <float*>dX, lddx, <void*>dwork, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zzgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZZgels(
+            <Handle>handle, m, n, nrhs, <cuDoubleComplex*>dA, ldda,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int zcgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZCgels(
+            <Handle>handle, m, n, nrhs, <cuDoubleComplex*>dA, ldda,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int zygels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZYgels(
+            <Handle>handle, m, n, nrhs, <cuDoubleComplex*>dA, ldda,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int zkgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZKgels(
+            <Handle>handle, m, n, nrhs, <cuDoubleComplex*>dA, ldda,
+            <cuDoubleComplex*>dB, lddb, <cuDoubleComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int ccgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCCgels(
+            <Handle>handle, m, n, nrhs, <cuComplex*>dA, ldda,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int cygels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCYgels(
+            <Handle>handle, m, n, nrhs, <cuComplex*>dA, ldda,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int ckgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCKgels(
+            <Handle>handle, m, n, nrhs, <cuComplex*>dA, ldda,
+            <cuComplex*>dB, lddb, <cuComplex*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int ddgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDDgels(
+            <Handle>handle, m, n, nrhs, <double*>dA, ldda,
+            <double*>dB, lddb, <double*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int dsgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDSgels(
+            <Handle>handle, m, n, nrhs, <double*>dA, ldda,
+            <double*>dB, lddb, <double*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int dxgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDXgels(
+            <Handle>handle, m, n, nrhs, <double*>dA, ldda,
+            <double*>dB, lddb, <double*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int dhgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDHgels(
+            <Handle>handle, m, n, nrhs, <double*>dA, ldda,
+            <double*>dB, lddb, <double*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int ssgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSSgels(
+            <Handle>handle, m, n, nrhs, <float*>dA, ldda,
+            <float*>dB, lddb, <float*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int sxgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSXgels(
+            <Handle>handle, m, n, nrhs, <float*>dA, ldda,
+            <float*>dB, lddb, <float*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+cpdef int shgels(intptr_t handle, int m, int n, int nrhs, size_t dA, int ldda,
+                 size_t dB, int lddb, size_t dX, int lddx,
+                 size_t dwork, size_t lwork, size_t dInfo):
+    cdef int iter
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSHgels(
+            <Handle>handle, m, n, nrhs, <float*>dA, ldda,
+            <float*>dB, lddb, <float*>dX, lddx,
+            <void*>dwork, lwork, &iter, <int*>dInfo)
+    check_status(status)
+    return iter
+
+###############################################################################
+# Dense LAPACK Functions (Eigenvalue Solver)
+###############################################################################
+
+# Bidiagonal factorization
+cpdef int sgebrd_bufferSize(intptr_t handle, int m, int n) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgebrd_bufferSize(<Handle>handle, m, n, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dgebrd_bufferSize(intptr_t handle, int m, int n) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgebrd_bufferSize(<Handle>handle, m, n, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int cgebrd_bufferSize(intptr_t handle, int m, int n) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgebrd_bufferSize(<Handle>handle, m, n, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zgebrd_bufferSize(intptr_t handle, int m, int n) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgebrd_bufferSize(<Handle>handle, m, n, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef sgebrd(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t D, size_t E, size_t tauQ, size_t tauP,
+             size_t Work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgebrd(
+            <Handle>handle, m, n,
+            <float*>A, lda,
+            <float*>D, <float*>E,
+            <float*>tauQ, <float*>tauP,
+            <float*>Work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef dgebrd(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t D, size_t E, size_t tauQ, size_t tauP,
+             size_t Work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgebrd(
+            <Handle>handle, m, n,
+            <double*>A, lda,
+            <double*>D, <double*>E,
+            <double*>tauQ, <double*>tauP,
+            <double*>Work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef cgebrd(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t D, size_t E, size_t tauQ, size_t tauP,
+             size_t Work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgebrd(
+            <Handle>handle, m, n,
+            <cuComplex*>A, lda,
+            <float*>D, <float*>E,
+            <cuComplex*>tauQ, <cuComplex*>tauP,
+            <cuComplex*>Work, lwork, <int*>devInfo)
+    check_status(status)
+
+cpdef zgebrd(intptr_t handle, int m, int n, size_t A, int lda,
+             size_t D, size_t E, size_t tauQ, size_t tauP,
+             size_t Work, int lwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgebrd(
+            <Handle>handle, m, n,
+            <cuDoubleComplex*>A, lda,
+            <double*>D, <double*>E,
+            <cuDoubleComplex*>tauQ, <cuDoubleComplex*>tauP,
+            <cuDoubleComplex*>Work, lwork, <int*>devInfo)
+    check_status(status)
+
+
+# Singular value decomposition, A = U * Sigma * V^H
+cpdef int sgesvd_bufferSize(intptr_t handle, int m, int n) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgesvd_bufferSize(<Handle>handle, m, n, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dgesvd_bufferSize(intptr_t handle, int m, int n) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgesvd_bufferSize(<Handle>handle, m, n, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int cgesvd_bufferSize(intptr_t handle, int m, int n) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgesvd_bufferSize(<Handle>handle, m, n, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zgesvd_bufferSize(intptr_t handle, int m, int n) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgesvd_bufferSize(<Handle>handle, m, n, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef sgesvd(intptr_t handle, char jobu, char jobvt, int m, int n, size_t A,
+             int lda, size_t S, size_t U, int ldu, size_t VT, int ldvt,
+             size_t Work, int lwork, size_t rwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgesvd(
+            <Handle>handle, jobu, jobvt, m, n, <float*>A, lda,
+            <float*>S, <float*>U, ldu, <float*>VT, ldvt,
+            <float*>Work, lwork, <float*>rwork, <int*>devInfo)
+    check_status(status)
+
+cpdef dgesvd(intptr_t handle, char jobu, char jobvt, int m, int n, size_t A,
+             int lda, size_t S, size_t U, int ldu, size_t VT, int ldvt,
+             size_t Work, int lwork, size_t rwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgesvd(
+            <Handle>handle, jobu, jobvt, m, n, <double*>A, lda,
+            <double*>S, <double*>U, ldu, <double*>VT, ldvt,
+            <double*>Work, lwork, <double*>rwork, <int*>devInfo)
+    check_status(status)
+
+cpdef cgesvd(intptr_t handle, char jobu, char jobvt, int m, int n, size_t A,
+             int lda, size_t S, size_t U, int ldu, size_t VT, int ldvt,
+             size_t Work, int lwork, size_t rwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgesvd(
+            <Handle>handle, jobu, jobvt, m, n, <cuComplex*>A, lda,
+            <float*>S, <cuComplex*>U, ldu, <cuComplex*>VT, ldvt,
+            <cuComplex*>Work, lwork, <float*>rwork, <int*>devInfo)
+    check_status(status)
+
+cpdef zgesvd(intptr_t handle, char jobu, char jobvt, int m, int n, size_t A,
+             int lda, size_t S, size_t U, int ldu, size_t VT, int ldvt,
+             size_t Work, int lwork, size_t rwork, size_t devInfo):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgesvd(
+            <Handle>handle, jobu, jobvt, m, n, <cuDoubleComplex*>A, lda,
+            <double*>S, <cuDoubleComplex*>U, ldu, <cuDoubleComplex*>VT, ldvt,
+            <cuDoubleComplex*>Work, lwork, <double*>rwork, <int*>devInfo)
+    check_status(status)
+
+# gesvdj ... Singular value decomposition using Jacobi mathod
+cpdef intptr_t createGesvdjInfo() except? 0:
+    cdef GesvdjInfo info
+    status = cusolverDnCreateGesvdjInfo(&info)
+    check_status(status)
+    return <intptr_t>info
+
+cpdef destroyGesvdjInfo(intptr_t info):
+    status = cusolverDnDestroyGesvdjInfo(<GesvdjInfo>info)
+    check_status(status)
+
+cpdef xgesvdjSetTolerance(intptr_t info, double tolerance):
+    status = cusolverDnXgesvdjSetTolerance(<GesvdjInfo>info, tolerance)
+    check_status(status)
+
+cpdef xgesvdjSetMaxSweeps(intptr_t info, int max_sweeps):
+    status = cusolverDnXgesvdjSetMaxSweeps(<GesvdjInfo>info, max_sweeps)
+    check_status(status)
+
+cpdef xgesvdjSetSortEig(intptr_t info, int sort_svd):
+    status = cusolverDnXgesvdjSetSortEig(<GesvdjInfo>info, sort_svd)
+    check_status(status)
+
+cpdef double xgesvdjGetResidual(intptr_t handle, intptr_t info):
+    cdef double residual
+    status = cusolverDnXgesvdjGetResidual(<Handle>handle, <GesvdjInfo>info,
+                                          &residual)
+    check_status(status)
+    return residual
+
+cpdef int xgesvdjGetSweeps(intptr_t handle, intptr_t info):
+    cdef int executed_sweeps
+    status = cusolverDnXgesvdjGetSweeps(<Handle>handle, <GesvdjInfo>info,
+                                        &executed_sweeps)
+    check_status(status)
+    return executed_sweeps
+
+cpdef int sgesvdj_bufferSize(intptr_t handle, int jobz, int econ, int m, int n,
+                             intptr_t A, int lda, intptr_t S, intptr_t U,
+                             int ldu, intptr_t V, int ldv, intptr_t params):
+    cdef int lwork, status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgesvdj_bufferSize(
+            <Handle>handle, <EigMode>jobz, econ, m, n, <const float*>A, lda,
+            <const float*>S, <const float*>U, ldu, <const float*>V, ldv,
+            &lwork, <GesvdjInfo>params)
+    check_status(status)
+    return lwork
+
+cpdef int dgesvdj_bufferSize(intptr_t handle, int jobz, int econ, int m, int n,
+                             intptr_t A, int lda, intptr_t S, intptr_t U,
+                             int ldu, intptr_t V, int ldv, intptr_t params):
+    cdef int lwork, status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgesvdj_bufferSize(
+            <Handle>handle, <EigMode>jobz, econ, m, n, <const double*>A, lda,
+            <const double*>S, <const double*>U, ldu, <const double*>V, ldv,
+            &lwork, <GesvdjInfo>params)
+    check_status(status)
+    return lwork
+
+cpdef int cgesvdj_bufferSize(intptr_t handle, int jobz, int econ, int m, int n,
+                             intptr_t A, int lda, intptr_t S, intptr_t U,
+                             int ldu, intptr_t V, int ldv, intptr_t params):
+    cdef int lwork, status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgesvdj_bufferSize(
+            <Handle>handle, <EigMode>jobz, econ, m, n, <const cuComplex*>A,
+            lda, <const float*>S, <const cuComplex*>U, ldu,
+            <const cuComplex*>V, ldv, &lwork, <GesvdjInfo>params)
+    check_status(status)
+    return lwork
+
+cpdef int zgesvdj_bufferSize(intptr_t handle, int jobz, int econ, int m, int n,
+                             intptr_t A, int lda, intptr_t S, intptr_t U,
+                             int ldu, intptr_t V, int ldv, intptr_t params):
+    cdef int lwork, status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgesvdj_bufferSize(
+            <Handle>handle, <EigMode>jobz, econ, m, n,
+            <const cuDoubleComplex*>A, lda, <const double*>S,
+            <const cuDoubleComplex*>U, ldu, <const cuDoubleComplex*>V,
+            ldv, &lwork, <GesvdjInfo>params)
+    check_status(status)
+    return lwork
+
+cpdef sgesvdj(intptr_t handle, int jobz, int econ, int m, int n, intptr_t A,
+              int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+              intptr_t work, int lwork, intptr_t info, intptr_t params):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgesvdj(<Handle>handle, <EigMode>jobz, econ, m, n,
+                                   <float*>A, lda, <float*>S, <float*>U, ldu,
+                                   <float*>V, ldv, <float*>work, lwork,
+                                   <int*>info, <GesvdjInfo>params)
+    check_status(status)
+
+cpdef dgesvdj(intptr_t handle, int jobz, int econ, int m, int n, intptr_t A,
+              int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+              intptr_t work, int lwork, intptr_t info, intptr_t params):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgesvdj(<Handle>handle, <EigMode>jobz, econ, m, n,
+                                   <double*>A, lda, <double*>S, <double*>U,
+                                   ldu, <double*>V, ldv, <double*>work, lwork,
+                                   <int*>info, <GesvdjInfo>params)
+    check_status(status)
+
+cpdef cgesvdj(intptr_t handle, int jobz, int econ, int m, int n, intptr_t A,
+              int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+              intptr_t work, int lwork, intptr_t info, intptr_t params):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgesvdj(
+            <Handle>handle, <EigMode>jobz, econ, m, n, <cuComplex*>A, lda,
+            <float*>S, <cuComplex*>U, ldu, <cuComplex*>V, ldv,
+            <cuComplex*>work, lwork, <int*>info, <GesvdjInfo>params)
+    check_status(status)
+
+cpdef zgesvdj(intptr_t handle, int jobz, int econ, int m, int n, intptr_t A,
+              int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+              intptr_t work, int lwork, intptr_t info, intptr_t params):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgesvdj(
+            <Handle>handle, <EigMode>jobz, econ, m, n, <cuDoubleComplex*>A,
+            lda, <double*>S, <cuDoubleComplex*>U, ldu, <cuDoubleComplex*>V,
+            ldv, <cuDoubleComplex*>work, lwork, <int*>info, <GesvdjInfo>params)
+    check_status(status)
+
+cpdef int sgesvdjBatched_bufferSize(
+        intptr_t handle, int jobz, int m, int n, intptr_t A,
+        int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+        intptr_t params, int batchSize) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgesvdjBatched_bufferSize(
+            <Handle>handle, <EigMode>jobz, m, n, <float*>A, lda,
+            <float*>S, <float*>U, ldu, <float*>V, ldv, &lwork,
+            <GesvdjInfo>params, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int dgesvdjBatched_bufferSize(
+        intptr_t handle, int jobz, int m, int n, intptr_t A,
+        int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+        intptr_t params, int batchSize) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgesvdjBatched_bufferSize(
+            <Handle>handle, <EigMode>jobz, m, n, <double*>A, lda,
+            <double*>S, <double*>U, ldu, <double*>V, ldv, &lwork,
+            <GesvdjInfo>params, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int cgesvdjBatched_bufferSize(
+        intptr_t handle, int jobz, int m, int n, intptr_t A,
+        int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+        intptr_t params, int batchSize) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgesvdjBatched_bufferSize(
+            <Handle>handle, <EigMode>jobz, m, n, <cuComplex*>A, lda,
+            <float*>S, <cuComplex*>U, ldu, <cuComplex*>V, ldv, &lwork,
+            <GesvdjInfo>params, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int zgesvdjBatched_bufferSize(
+        intptr_t handle, int jobz, int m, int n, intptr_t A,
+        int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+        intptr_t params, int batchSize) except? -1:
+    cdef int lwork
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgesvdjBatched_bufferSize(
+            <Handle>handle, <EigMode>jobz, m, n, <cuDoubleComplex*>A, lda,
+            <double*>S, <cuDoubleComplex*>U, ldu, <cuDoubleComplex*>V, ldv,
+            &lwork,
+            <GesvdjInfo>params, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef sgesvdjBatched(
+        intptr_t handle, int jobz, int m, int n, intptr_t A,
+        int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+        intptr_t work, int lwork, intptr_t info,
+        intptr_t params, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgesvdjBatched(
+            <Handle>handle, <EigMode>jobz, m, n, <float*>A, lda,
+            <float*>S, <float*>U, ldu, <float*>V, ldv,
+            <float*>work, lwork, <int*>info,
+            <GesvdjInfo>params, batchSize)
+    check_status(status)
+
+cpdef dgesvdjBatched(
+        intptr_t handle, int jobz, int m, int n, intptr_t A,
+        int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+        intptr_t work, int lwork, intptr_t info,
+        intptr_t params, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgesvdjBatched(
+            <Handle>handle, <EigMode>jobz, m, n, <double*>A, lda,
+            <double*>S, <double*>U, ldu, <double*>V, ldv,
+            <double*>work, lwork, <int*>info,
+            <GesvdjInfo>params, batchSize)
+    check_status(status)
+
+cpdef cgesvdjBatched(
+        intptr_t handle, int jobz, int m, int n, intptr_t A,
+        int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+        intptr_t work, int lwork, intptr_t info,
+        intptr_t params, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgesvdjBatched(
+            <Handle>handle, <EigMode>jobz, m, n, <cuComplex*>A, lda,
+            <float*>S, <cuComplex*>U, ldu, <cuComplex*>V, ldv,
+            <cuComplex*>work, lwork, <int*>info,
+            <GesvdjInfo>params, batchSize)
+    check_status(status)
+
+cpdef zgesvdjBatched(
+        intptr_t handle, int jobz, int m, int n, intptr_t A,
+        int lda, intptr_t S, intptr_t U, int ldu, intptr_t V, int ldv,
+        intptr_t work, int lwork, intptr_t info,
+        intptr_t params, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgesvdjBatched(
+            <Handle>handle, <EigMode>jobz, m, n, <cuDoubleComplex*>A, lda,
+            <double*>S, <cuDoubleComplex*>U, ldu, <cuDoubleComplex*>V, ldv,
+            <cuDoubleComplex*>work, lwork, <int*>info,
+            <GesvdjInfo>params, batchSize)
+    check_status(status)
+
+# gesvda ... Approximate singular value decomposition
+cpdef int sgesvdaStridedBatched_bufferSize(
+        intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+        int lda, long long int strideA, intptr_t d_S, long long int strideS,
+        intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+        long long int strideV, int batchSize):
+    cdef int lwork
+    status = cusolverDnSgesvdaStridedBatched_bufferSize(
+        <Handle>handle, <EigMode>jobz, rank, m, n, <const float*>d_A, lda,
+        strideA, <const float*>d_S, strideS, <const float*>d_U, ldu, strideU,
+        <const float*>d_V, ldv, strideV, &lwork, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int dgesvdaStridedBatched_bufferSize(
+        intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+        int lda, long long int strideA, intptr_t d_S, long long int strideS,
+        intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+        long long int strideV, int batchSize):
+    cdef int lwork
+    status = cusolverDnDgesvdaStridedBatched_bufferSize(
+        <Handle>handle, <EigMode>jobz, rank, m, n, <const double*>d_A, lda,
+        strideA, <const double*>d_S, strideS, <const double*>d_U, ldu, strideU,
+        <const double*>d_V, ldv, strideV, &lwork, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int cgesvdaStridedBatched_bufferSize(
+        intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+        int lda, long long int strideA, intptr_t d_S, long long int strideS,
+        intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+        long long int strideV, int batchSize):
+    cdef int lwork
+    status = cusolverDnCgesvdaStridedBatched_bufferSize(
+        <Handle>handle, <EigMode>jobz, rank, m, n, <const cuComplex*>d_A, lda,
+        strideA, <const float*>d_S, strideS, <const cuComplex*>d_U, ldu,
+        strideU, <const cuComplex*>d_V, ldv, strideV, &lwork, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int zgesvdaStridedBatched_bufferSize(
+        intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+        int lda, long long int strideA, intptr_t d_S, long long int strideS,
+        intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+        long long int strideV, int batchSize):
+    cdef int lwork
+    status = cusolverDnZgesvdaStridedBatched_bufferSize(
+        <Handle>handle, <EigMode>jobz, rank, m, n, <const cuDoubleComplex*>d_A,
+        lda, strideA, <const double*>d_S, strideS, <const cuDoubleComplex*>d_U,
+        ldu, strideU, <const cuDoubleComplex*>d_V, ldv, strideV, &lwork,
+        batchSize)
+    check_status(status)
+    return lwork
+
+cpdef sgesvdaStridedBatched(
+        intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+        int lda, long long int strideA, intptr_t d_S, long long int strideS,
+        intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+        long long int strideV, intptr_t d_work, int lwork, intptr_t d_info,
+        intptr_t h_R_nrmF, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSgesvdaStridedBatched(
+            <Handle>handle, <EigMode>jobz, rank, m, n, <const float*>d_A, lda,
+            strideA, <float*>d_S, strideS, <float*>d_U, ldu, strideU,
+            <float*>d_V, ldv, strideV, <float*>d_work, lwork, <int*>d_info,
+            <double*>h_R_nrmF, batchSize)
+    check_status(status)
+
+cpdef dgesvdaStridedBatched(
+        intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+        int lda, long long int strideA, intptr_t d_S, long long int strideS,
+        intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+        long long int strideV, intptr_t d_work, int lwork, intptr_t d_info,
+        intptr_t h_R_nrmF, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDgesvdaStridedBatched(
+            <Handle>handle, <EigMode>jobz, rank, m, n, <const double*>d_A, lda,
+            strideA, <double*>d_S, strideS, <double*>d_U, ldu, strideU,
+            <double*>d_V, ldv, strideV, <double*>d_work, lwork, <int*>d_info,
+            <double*>h_R_nrmF, batchSize)
+    check_status(status)
+
+cpdef cgesvdaStridedBatched(
+        intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+        int lda, long long int strideA, intptr_t d_S, long long int strideS,
+        intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+        long long int strideV, intptr_t d_work, int lwork, intptr_t d_info,
+        intptr_t h_R_nrmF, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCgesvdaStridedBatched(
+            <Handle>handle, <EigMode>jobz, rank, m, n, <const cuComplex*>d_A,
+            lda, strideA, <float*>d_S, strideS, <cuComplex*>d_U, ldu, strideU,
+            <cuComplex*>d_V, ldv, strideV, <cuComplex*>d_work, lwork,
+            <int*>d_info, <double*>h_R_nrmF, batchSize)
+    check_status(status)
+
+cpdef zgesvdaStridedBatched(
+        intptr_t handle, int jobz, int rank, int m, int n, intptr_t d_A,
+        int lda, long long int strideA, intptr_t d_S, long long int strideS,
+        intptr_t d_U, int ldu, long long int strideU, intptr_t d_V, int ldv,
+        long long int strideV, intptr_t d_work, int lwork, intptr_t d_info,
+        intptr_t h_R_nrmF, int batchSize):
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZgesvdaStridedBatched(
+            <Handle>handle, <EigMode>jobz, rank, m, n,
+            <const cuDoubleComplex*>d_A, lda, strideA, <double*>d_S, strideS,
+            <cuDoubleComplex*>d_U, ldu, strideU, <cuDoubleComplex*>d_V, ldv,
+            strideV, <cuDoubleComplex*>d_work, lwork, <int*>d_info,
+            <double*>h_R_nrmF, batchSize)
+    check_status(status)
+
+# Standard symmetric eigenvalue solver
+cpdef int ssyevd_bufferSize(intptr_t handle, int jobz, int uplo, int n,
+                            size_t A, int lda, size_t W) except? -1:
+    cdef int lwork, status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSsyevd_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const float*>A,
+            lda, <const float*>W, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int dsyevd_bufferSize(intptr_t handle, int jobz, int uplo, int n,
+                            size_t A, int lda, size_t W) except? -1:
+    cdef int lwork, status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDsyevd_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const double*>A,
+            lda, <const double*>W, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int cheevd_bufferSize(intptr_t handle, int jobz, int uplo, int n,
+                            size_t A, int lda, size_t W) except? -1:
+    cdef int lwork, status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCheevd_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const cuComplex*>A,
+            lda, <const float*>W, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef int zheevd_bufferSize(intptr_t handle, int jobz, int uplo, int n,
+                            size_t A, int lda, size_t W) except? -1:
+    cdef int lwork, status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZheevd_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const cuDoubleComplex*>A,
+            lda, <const double*>W, &lwork)
+    check_status(status)
+    return lwork
+
+cpdef ssyevd(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info):
+    cdef int status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnSsyevd(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <float*>A, lda, <float*>W,
+            <float*>work, lwork, <int*>info)
+    check_status(status)
+
+cpdef dsyevd(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info):
+    cdef int status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnDsyevd(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <double*>A, lda, <double*>W,
+            <double*>work, lwork, <int*>info)
+    check_status(status)
+
+cpdef cheevd(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info):
+    cdef int status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnCheevd(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <cuComplex*>A, lda, <float*>W,
+            <cuComplex*>work, lwork, <int*>info)
+    check_status(status)
+
+cpdef zheevd(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info):
+    cdef int status
+    _setStream(handle)
+    with nogil:
+        status = cusolverDnZheevd(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <cuDoubleComplex*>A, lda, <double*>W,
+            <cuDoubleComplex*>work, lwork, <int*>info)
+    check_status(status)
+
+# Symmetric eigenvalue solver via Jacobi method
+cpdef intptr_t createSyevjInfo() except? 0:
+    cdef SyevjInfo info
+    status = cusolverDnCreateSyevjInfo(&info)
+    check_status(status)
+    return <intptr_t>info
+
+cpdef destroySyevjInfo(intptr_t info):
+    status = cusolverDnDestroySyevjInfo(<SyevjInfo>info)
+    check_status(status)
+
+cpdef xsyevjSetTolerance(intptr_t info, double tolerance):
+    status = cusolverDnXsyevjSetTolerance(<SyevjInfo>info, tolerance)
+    check_status(status)
+
+cpdef xsyevjSetMaxSweeps(intptr_t info, int max_sweeps):
+    status = cusolverDnXsyevjSetMaxSweeps(<SyevjInfo>info, max_sweeps)
+    check_status(status)
+
+cpdef xsyevjSetSortEig(intptr_t info, int sort_eig):
+    status = cusolverDnXsyevjSetSortEig(<SyevjInfo>info, sort_eig)
+    check_status(status)
+
+cpdef double xsyevjGetResidual(intptr_t handle, intptr_t info):
+    cdef double residual
+    status = cusolverDnXsyevjGetResidual(
+        <Handle>handle, <SyevjInfo>info, &residual)
+    check_status(status)
+    return residual
+
+cpdef int xsyevjGetSweeps(intptr_t handle, intptr_t info):
+    cdef int executed_sweeps
+    status = cusolverDnXsyevjGetSweeps(
+        <Handle>handle, <SyevjInfo>info, &executed_sweeps)
+    check_status(status)
+    return executed_sweeps
+
+cpdef int ssyevj_bufferSize(intptr_t handle, int jobz, int uplo,
+                            int n, size_t A, int lda, size_t W,
+                            intptr_t params) except? -1:
+    cdef int lwork, status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnSsyevj_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const float*>A,
+            lda, <const float*>W, &lwork, <SyevjInfo>params)
+    check_status(status)
+    return lwork
+
+cpdef int dsyevj_bufferSize(intptr_t handle, int jobz, int uplo,
+                            int n, size_t A, int lda, size_t W,
+                            intptr_t params) except? -1:
+    cdef int lwork, status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnDsyevj_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const double*>A,
+            lda, <const double*>W, &lwork, <SyevjInfo>params)
+    check_status(status)
+    return lwork
+
+cpdef int cheevj_bufferSize(intptr_t handle, int jobz, int uplo,
+                            int n, size_t A, int lda, size_t W,
+                            intptr_t params) except? -1:
+    cdef int lwork, status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnCheevj_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const cuComplex*>A,
+            lda, <const float*>W, &lwork, <SyevjInfo>params)
+    check_status(status)
+    return lwork
+
+cpdef int zheevj_bufferSize(intptr_t handle, int jobz, int uplo,
+                            int n, size_t A, int lda, size_t W,
+                            intptr_t params) except? -1:
+    cdef int lwork, status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnZheevj_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const cuDoubleComplex*>A,
+            lda, <const double*>W, &lwork, <SyevjInfo>params)
+    check_status(status)
+    return lwork
+
+cpdef ssyevj(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info, intptr_t params):
+    cdef int status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnSsyevj(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <float*>A, lda, <float*>W,
+            <float*>work, lwork, <int*>info, <SyevjInfo>params)
+    check_status(status)
+
+cpdef dsyevj(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info, intptr_t params):
+    cdef int status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnDsyevj(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <double*>A, lda, <double*>W,
+            <double*>work, lwork, <int*>info, <SyevjInfo>params)
+    check_status(status)
+
+cpdef cheevj(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info, intptr_t params):
+    cdef int status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnCheevj(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <cuComplex*>A, lda, <float*>W,
+            <cuComplex*>work, lwork, <int*>info, <SyevjInfo>params)
+    check_status(status)
+
+cpdef zheevj(intptr_t handle, int jobz, int uplo, int n, size_t A, int lda,
+             size_t W, size_t work, int lwork, size_t info, intptr_t params):
+    cdef int status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnZheevj(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <cuDoubleComplex*>A, lda, <double*>W,
+            <cuDoubleComplex*>work, lwork, <int*>info, <SyevjInfo>params)
+    check_status(status)
+
+# Batched symmetric eigenvalue solver via Jacobi method
+
+cpdef int ssyevjBatched_bufferSize(
+        intptr_t handle, int jobz, int uplo, int n,
+        size_t A, int lda, size_t W, intptr_t params,
+        int batchSize) except? -1:
+    cdef int lwork, status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnSsyevjBatched_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const float *>A, lda, <const float *>W, &lwork,
+            <SyevjInfo>params, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int dsyevjBatched_bufferSize(
+        intptr_t handle, int jobz, int uplo, int n,
+        size_t A, int lda, size_t W, intptr_t params,
+        int batchSize) except? -1:
+    cdef int lwork, status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnDsyevjBatched_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const double *>A, lda, <const double *>W, &lwork,
+            <SyevjInfo>params, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int cheevjBatched_bufferSize(
+        intptr_t handle, int jobz, int uplo, int n,
+        size_t A, int lda, size_t W, intptr_t params,
+        int batchSize) except? -1:
+    cdef int lwork, status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnCheevjBatched_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const cuComplex *>A, lda, <const float *>W, &lwork,
+            <SyevjInfo>params, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef int zheevjBatched_bufferSize(
+        intptr_t handle, int jobz, int uplo, int n,
+        size_t A, int lda, size_t W, intptr_t params,
+        int batchSize) except? -1:
+    cdef int lwork, status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnZheevjBatched_bufferSize(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <const cuDoubleComplex *>A, lda, <const double *>W, &lwork,
+            <SyevjInfo>params, batchSize)
+    check_status(status)
+    return lwork
+
+cpdef ssyevjBatched(intptr_t handle, int jobz, int uplo, int n,
+                    size_t A, int lda, size_t W, size_t work, int lwork,
+                    size_t info, intptr_t params, int batchSize):
+    cdef int status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnSsyevjBatched(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <float*>A, lda, <float*>W,
+            <float*>work, lwork, <int*>info, <SyevjInfo>params, batchSize)
+    check_status(status)
+
+cpdef dsyevjBatched(intptr_t handle, int jobz, int uplo, int n,
+                    size_t A, int lda, size_t W, size_t work, int lwork,
+                    size_t info, intptr_t params, int batchSize):
+    cdef int status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnDsyevjBatched(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <double*>A, lda, <double*>W,
+            <double*>work, lwork, <int*>info, <SyevjInfo>params, batchSize)
+    check_status(status)
+
+cpdef cheevjBatched(intptr_t handle, int jobz, int uplo, int n,
+                    size_t A, int lda, size_t W, size_t work, int lwork,
+                    size_t info, intptr_t params, int batchSize):
+    cdef int status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnCheevjBatched(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <cuComplex*>A, lda, <float*>W,
+            <cuComplex*>work, lwork, <int*>info, <SyevjInfo>params, batchSize)
+    check_status(status)
+
+cpdef zheevjBatched(intptr_t handle, int jobz, int uplo, int n,
+                    size_t A, int lda, size_t W, size_t work, int lwork,
+                    size_t info, intptr_t params, int batchSize):
+    cdef int status
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnZheevjBatched(
+            <Handle>handle, <EigMode>jobz, <FillMode>uplo, n,
+            <cuDoubleComplex*>A, lda, <double*>W,
+            <cuDoubleComplex*>work, lwork, <int*>info,
+            <SyevjInfo>params, batchSize)
+    check_status(status)
+
+# dense eigenvalue solver (64bit)
+cpdef (size_t, size_t) xsyevd_bufferSize(  # noqa
+        intptr_t handle, intptr_t params, int jobz, int uplo,
+        int64_t n, int dataTypeA, intptr_t A, int64_t lda,
+        int dataTypeW, intptr_t W, int computeType) except *:
+    cdef size_t workspaceInBytesOnDevice, workspaceInBytesOnHost
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnXsyevd_bufferSize(
+            <Handle>handle, <Params>params, <EigMode> jobz, <FillMode> uplo, n,
+            <DataType>dataTypeA, <void*>A, lda,
+            <DataType>dataTypeW, <void*>W, <DataType>computeType,
+            &workspaceInBytesOnDevice, &workspaceInBytesOnHost)
+    check_status(status)
+    return workspaceInBytesOnDevice, workspaceInBytesOnHost
+
+cpdef xsyevd(
+        intptr_t handle, intptr_t params, int jobz, int uplo,
+        int64_t n, int dataTypeA, intptr_t A, int64_t lda,
+        int dataTypeW, intptr_t W, int computeType, intptr_t bufferOnDevice,
+        size_t workspaceInBytesOnDevice, intptr_t bufferOnHost,
+        size_t workspaceInBytesOnHost, intptr_t info):
+    setStream(handle, stream_module.get_current_stream_ptr())
+    with nogil:
+        status = cusolverDnXsyevd(
+            <Handle>handle, <Params>params, <EigMode>jobz, <FillMode>uplo, n,
+            <DataType>dataTypeA, <void*>A, lda,
+            <DataType>dataTypeW, <void*>W, <DataType>computeType,
+            <void*>bufferOnDevice, workspaceInBytesOnDevice,
+            <void*>bufferOnHost, workspaceInBytesOnHost, <int*>info)
+    check_status(status)
+
+
+###############################################################################
+# Sparse LAPACK Functions
+###############################################################################
+cpdef scsrlsvchol(intptr_t handle, int m, int nnz, size_t descrA,
+                  size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                  size_t b, float tol, int reorder, size_t x,
+                  size_t singularity):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpScsrlsvchol(
+            <SpHandle>handle, m, nnz, <const MatDescr> descrA,
+            <const float*> csrValA, <const int*> csrRowPtrA,
+            <const int*> csrColIndA, <const float*> b,
+            tol, reorder, <float*> x, <int*> singularity)
+    check_status(status)
+
+cpdef dcsrlsvchol(intptr_t handle, int m, int nnz, size_t descrA,
+                  size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                  size_t b, double tol, int reorder, size_t x,
+                  size_t singularity):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpDcsrlsvchol(
+            <SpHandle>handle, m, nnz, <const MatDescr> descrA,
+            <const double*> csrValA, <const int*> csrRowPtrA,
+            <const int*> csrColIndA, <const double*> b,
+            tol, reorder, <double*> x, <int*> singularity)
+    check_status(status)
+
+cpdef ccsrlsvchol(intptr_t handle, int m, int nnz, size_t descrA,
+                  size_t csrVal, size_t csrRowPtr, size_t csrColInd, size_t b,
+                  float tol, int reorder, size_t x, size_t singularity):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpCcsrlsvchol(
+            <SpHandle>handle, m, nnz, <const MatDescr>descrA,
+            <const cuComplex*>csrVal, <const int*>csrRowPtr,
+            <const int*>csrColInd, <const cuComplex*>b, tol, reorder,
+            <cuComplex*>x, <int*>singularity)
+    check_status(status)
+
+cpdef zcsrlsvchol(intptr_t handle, int m, int nnz, size_t descrA,
+                  size_t csrVal, size_t csrRowPtr, size_t csrColInd, size_t b,
+                  double tol, int reorder, size_t x, size_t singularity):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpZcsrlsvchol(
+            <SpHandle>handle, m, nnz, <const MatDescr>descrA,
+            <const cuDoubleComplex*>csrVal, <const int*>csrRowPtr,
+            <const int*>csrColInd, <const cuDoubleComplex*>b, tol, reorder,
+            <cuDoubleComplex*>x, <int*>singularity)
+    check_status(status)
+
+cpdef scsrlsvqr(intptr_t handle, int m, int nnz, size_t descrA, size_t csrValA,
+                size_t csrRowPtrA, size_t csrColIndA, size_t b, float tol,
+                int reorder, size_t x, size_t singularity):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpScsrlsvqr(
+            <SpHandle>handle, m, nnz, <const MatDescr> descrA,
+            <const float*> csrValA, <const int*> csrRowPtrA,
+            <const int*> csrColIndA, <const float*> b,
+            tol, reorder, <float*> x, <int*> singularity)
+    check_status(status)
+
+cpdef dcsrlsvqr(intptr_t handle, int m, int nnz, size_t descrA, size_t csrValA,
+                size_t csrRowPtrA, size_t csrColIndA, size_t b, double tol,
+                int reorder, size_t x, size_t singularity):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpDcsrlsvqr(
+            <SpHandle>handle, m, nnz, <const MatDescr> descrA,
+            <const double*> csrValA, <const int*> csrRowPtrA,
+            <const int*> csrColIndA, <const double*> b,
+            tol, reorder, <double*> x, <int*> singularity)
+    check_status(status)
+
+cpdef ccsrlsvqr(intptr_t handle, int m, int nnz, size_t descrA, size_t csrVal,
+                size_t csrRowPtr, size_t csrColInd, size_t b, float tol,
+                int reorder, size_t x, size_t singularity):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpCcsrlsvqr(
+            <SpHandle>handle, m, nnz, <const MatDescr>descrA,
+            <const cuComplex*>csrVal, <const int*>csrRowPtr,
+            <const int*>csrColInd, <const cuComplex*>b, tol, reorder,
+            <cuComplex*>x, <int*>singularity)
+    check_status(status)
+
+cpdef zcsrlsvqr(intptr_t handle, int m, int nnz, size_t descrA, size_t csrVal,
+                size_t csrRowPtr, size_t csrColInd, size_t b, double tol,
+                int reorder, size_t x, size_t singularity):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpZcsrlsvqr(
+            <SpHandle>handle, m, nnz, <const MatDescr>descrA,
+            <const cuDoubleComplex*>csrVal, <const int*>csrRowPtr,
+            <const int*>csrColInd, <const cuDoubleComplex*>b, tol, reorder,
+            <cuDoubleComplex*>x, <int*>singularity)
+    check_status(status)
+
+cpdef scsreigvsi(intptr_t handle, int m, int nnz, size_t descrA,
+                 size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                 float mu0, size_t x0, int maxite, float eps, size_t mu,
+                 size_t x):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpScsreigvsi(
+            <SpHandle>handle, m, nnz, <const MatDescr>descrA,
+            <const float*>csrValA, <const int*>csrRowPtrA,
+            <const int*>csrColIndA, mu0, <const float*>x0, maxite, eps,
+            <float*>mu, <float*>x)
+    check_status(status)
+
+cpdef dcsreigvsi(intptr_t handle, int m, int nnz, size_t descrA,
+                 size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                 double mu0, size_t x0, int maxite, double eps, size_t mu,
+                 size_t x):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpDcsreigvsi(
+            <SpHandle>handle, m, nnz, <const MatDescr>descrA,
+            <const double*>csrValA, <const int*>csrRowPtrA,
+            <const int*>csrColIndA, mu0, <const double*>x0, maxite, eps,
+            <double*>mu, <double*>x)
+    check_status(status)
+
+cpdef ccsreigvsi(intptr_t handle, int m, int nnz, size_t descrA,
+                 size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                 size_t mu0, size_t x0, int maxite, float eps, size_t mu,
+                 size_t x):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpCcsreigvsi(
+            <SpHandle>handle, m, nnz, <const MatDescr>descrA,
+            <const cuComplex*>csrValA, <const int*>csrRowPtrA,
+            <const int*>csrColIndA, (<cuComplex*>mu0)[0], <const cuComplex*>x0,
+            maxite, eps, <cuComplex*>mu, <cuComplex*>x)
+    check_status(status)
+
+cpdef zcsreigvsi(intptr_t handle, int m, int nnz, size_t descrA,
+                 size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+                 size_t mu0, size_t x0, int maxite, double eps, size_t mu,
+                 size_t x):
+    cdef int status
+    _spSetStream(handle)
+    with nogil:
+        status = cusolverSpZcsreigvsi(
+            <SpHandle>handle, m, nnz, <const MatDescr>descrA,
+            <const cuDoubleComplex*>csrValA, <const int*>csrRowPtrA,
+            <const int*>csrColIndA, (<cuDoubleComplex*>mu0)[0],
+            <const cuDoubleComplex*>x0, maxite,
+            eps, <cuDoubleComplex*>mu, <cuDoubleComplex*>x)
+    check_status(status)
diff --git a/cupy_backends/cuda/libs/cusparse.pxd b/cupy_backends/cuda/libs/cusparse.pxd
new file mode 100644
index 0000000..2adf0c8
--- /dev/null
+++ b/cupy_backends/cuda/libs/cusparse.pxd
@@ -0,0 +1,220 @@
+
+from libc.stdint cimport intptr_t, int64_t
+
+cdef extern from *:
+    ctypedef int IndexBase 'cusparseIndexBase_t'
+    ctypedef int Status 'cusparseStatus_t'
+
+    ctypedef void* Handle 'cusparseHandle_t'
+
+    ctypedef void* MatDescr 'cusparseMatDescr_t'
+
+    ctypedef int Direction 'cusparseDirection_t'
+
+    ctypedef int MatrixType 'cusparseMatrixType_t'
+    ctypedef int FillMode 'cusparseFillMode_t'
+    ctypedef int DiagType 'cusparseDiagType_t'
+
+    ctypedef int Operation 'cusparseOperation_t'
+
+    ctypedef int PointerMode 'cusparsePointerMode_t'
+
+    ctypedef int Action 'cusparseAction_t'
+    ctypedef int AlgMode 'cusparseAlgMode_t'
+
+    ctypedef void* cusparseHandle_t
+    ctypedef void* cusparseMatDescr_t
+    ctypedef void* csrsv2Info_t
+    ctypedef void* csrsm2Info_t
+    ctypedef void* csric02Info_t
+    ctypedef void* bsric02Info_t
+    ctypedef void* csrilu02Info_t
+    ctypedef void* bsrilu02Info_t
+    ctypedef void* csrgemm2Info_t
+
+    # Declarations for cuSparse generic API
+    ctypedef int cusparseStatus_t
+    ctypedef int cusparseDirection_t
+    ctypedef int cusparseSolvePolicy_t
+
+    ctypedef int IndexType 'cusparseIndexType_t'
+    ctypedef int Format 'cusparseFormat_t'
+    ctypedef int Order 'cusparseOrder_t'
+    ctypedef int SpMVAlg 'cusparseSpMVAlg_t'
+    ctypedef int SpSMAlg 'cusparseSpSMAlg_t'
+    ctypedef int SpMMAlg 'cusparseSpMMAlg_t'
+    ctypedef int SpGEMMAlg 'cusparseSpGEMMAlg_t'
+    ctypedef int DataType 'cudaDataType'
+    ctypedef int SpMatAttribute 'cusparseSpMatAttribute_t'
+
+    ctypedef void* SpVecDescr 'cusparseSpVecDescr_t'
+    ctypedef void* DnVecDescr 'cusparseDnVecDescr_t'
+    ctypedef void* SpMatDescr 'cusparseSpMatDescr_t'
+    ctypedef void* DnMatDescr 'cusparseDnMatDescr_t'
+    ctypedef void* SpSMDescr 'cusparseSpSMDescr_t'
+    ctypedef void* SpGEMMDescr 'cusparseSpGEMMDescr_t'
+
+    ctypedef void* cusparseSpVecDescr_t
+    ctypedef void* cusparseDnVecDescr_t
+    ctypedef void* cusparseSpMatDescr_t
+    ctypedef void* cusparseDnMatDescr_t
+    ctypedef void* cusparseSpSMDescr_t
+    ctypedef void* cusparseSpGEMMDescr_t
+
+    ctypedef int cusparseSparseToDenseAlg_t
+    ctypedef int cusparseDenseToSparseAlg_t
+    ctypedef int cusparseSpSMAlg_t
+    ctypedef int cusparseSpGEMMAlg_t
+
+    # CSR2CSC
+    ctypedef int Csr2CscAlg 'cusparseCsr2CscAlg_t'
+
+cpdef enum:
+    CUSPARSE_POINTER_MODE_HOST = 0
+    CUSPARSE_POINTER_MODE_DEVICE = 1
+
+    CUSPARSE_ACTION_SYMBOLIC = 0
+    CUSPARSE_ACTION_NUMERIC = 1
+
+    # cusparseIndexBase_t
+    CUSPARSE_INDEX_BASE_ZERO = 0
+    CUSPARSE_INDEX_BASE_ONE = 1
+
+    CUSPARSE_MATRIX_TYPE_GENERAL = 0
+    CUSPARSE_MATRIX_TYPE_SYMMETRIC = 1
+    CUSPARSE_MATRIX_TYPE_HERMITIAN = 2
+    CUSPARSE_MATRIX_TYPE_TRIANGULAR = 3
+
+    # cusparseFillMode_t
+    CUSPARSE_FILL_MODE_LOWER = 0
+    CUSPARSE_FILL_MODE_UPPER = 1
+
+    # cusparseDiagType_t;
+    CUSPARSE_DIAG_TYPE_NON_UNIT = 0
+    CUSPARSE_DIAG_TYPE_UNIT = 1
+
+    CUSPARSE_OPERATION_NON_TRANSPOSE = 0
+    CUSPARSE_OPERATION_TRANSPOSE = 1
+    CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE = 2
+
+    CUSPARSE_DIRECTION_ROW = 0
+    CUSPARSE_DIRECTION_COLUMN = 1
+
+    CUSPARSE_SOLVE_POLICY_NO_LEVEL = 0
+    CUSPARSE_SOLVE_POLICY_USE_LEVEL = 1
+
+    CUSPARSE_ALG_NAIVE = 0
+    CUSPARSE_ALG_MERGE_PATH = 1
+
+    # Enums for cuSparse generic API
+    CUSPARSE_FORMAT_CSR = 1  # Compressed Sparse Row (CSR)
+    CUSPARSE_FORMAT_CSC = 2  # Compressed Sparse Column (CSC)
+    CUSPARSE_FORMAT_COO = 3  # Coordinate (COO) - Structure of Arrays
+    CUSPARSE_FORMAT_COO_AOS = 4  # Coordinate (COO) - Array of Structures
+
+    CUSPARSE_ORDER_COL = 1  # Column-Major Order - Matrix memory layout
+    CUSPARSE_ORDER_ROW = 2  # Row-Major Order - Matrix memory layout
+
+    CUSPARSE_MV_ALG_DEFAULT = 0
+    CUSPARSE_COOMV_ALG = 1
+    CUSPARSE_CSRMV_ALG1 = 2
+    CUSPARSE_CSRMV_ALG2 = 3
+
+    CUSPARSE_MM_ALG_DEFAULT = 0
+    CUSPARSE_COOMM_ALG1 = 1  # non-deterministc results
+    CUSPARSE_COOMM_ALG2 = 2  # deterministic results
+    CUSPARSE_COOMM_ALG3 = 3  # non-deterministc results, for large matrices
+    CUSPARSE_CSRMM_ALG1 = 4
+
+    CUSPARSE_INDEX_16U = 1  # 16-bit unsigned integer
+    CUSPARSE_INDEX_32I = 2  # 32-bit signed integer
+    CUSPARSE_INDEX_64I = 3  # 64-bit signed integer
+
+    # cusparseSpMatAttribute_t
+    CUSPARSE_SPMAT_FILL_MODE = 0
+    CUSPARSE_SPMAT_DIAG_TYPE = 1
+
+    # CSR2CSC
+    CUSPARSE_CSR2CSC_ALG1 = 1  # faster than ALG2 (in general), deterministc
+    CUSPARSE_CSR2CSC_ALG2 = 2  # low memory requirement, non-deterministc
+
+    # cusparseSpSMAlg_t
+    CUSPARSE_SPSM_ALG_DEFAULT = 0
+
+    # ...
+    CUSPARSE_SPGEMM_DEFAULT = 0
+
+    # cusparseSparseToDenseAlg_t
+    CUSPARSE_SPARSETODENSE_ALG_DEFAULT = 0
+
+    # cusparseDenseToSparseAlg_t
+    CUSPARSE_DENSETOSPARSE_ALG_DEFAULT = 0
+
+cdef class SpVecAttributes:
+    cdef:
+        public int64_t size
+        public int64_t nnz
+        public intptr_t idx
+        public intptr_t values
+        public IndexType idxType
+        public IndexBase idxBase
+        public DataType valueType
+
+cdef class CooAttributes:
+    cdef:
+        public int64_t rows
+        public int64_t cols
+        public int64_t nnz
+        public intptr_t rowIdx
+        public intptr_t colIdx
+        public intptr_t values
+        public IndexType idxType
+        public IndexBase idxBase
+        public DataType valueType
+
+cdef class CooAoSAttributes:
+    cdef:
+        public int64_t rows
+        public int64_t cols
+        public int64_t nnz
+        public intptr_t ind
+        public intptr_t values
+        public IndexType idxType
+        public IndexBase idxBase
+        public DataType valueType
+
+cdef class CsrAttributes:
+    cdef:
+        public int64_t rows
+        public int64_t cols
+        public int64_t nnz
+        public intptr_t rowOffsets
+        public intptr_t colIdx
+        public intptr_t values
+        public IndexType rowOffsetType
+        public IndexType colIdxType
+        public IndexBase idxBase
+        public DataType valueType
+
+cdef class DnVecAttributes:
+    cdef:
+        public int64_t size
+        public intptr_t values
+        public DataType valueType
+
+cdef class DnMatAttributes:
+    cdef:
+        public int64_t rows
+        public int64_t cols
+        public int64_t ld
+        public intptr_t values
+        public DataType valueType
+        public Order order
+
+cdef class DnMatBatchAttributes:
+    cdef:
+        public int count
+        public int64_t stride
+
+cpdef intptr_t create() except? 0
+cpdef void destroy(intptr_t handle) except *
diff --git a/cupy_backends/cuda/libs/cusparse.pyx b/cupy_backends/cuda/libs/cusparse.pyx
new file mode 100644
index 0000000..0b7a99c
--- /dev/null
+++ b/cupy_backends/cuda/libs/cusparse.pyx
@@ -0,0 +1,5173 @@
+import sys as _sys  # no-cython-lint
+cimport cython  # NOQA
+
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda.api.runtime cimport _is_hip_environment
+from cupy_backends.cuda cimport stream as stream_module
+from cupy_backends.cuda._softlink cimport SoftLink
+
+
+cdef extern from '../../cupy_complex.h':
+    ctypedef struct cuComplex 'cuComplex':
+        float x, y
+
+    ctypedef struct cuDoubleComplex 'cuDoubleComplex':
+        double x, y
+
+cdef extern from '../../cupy_sparse.h' nogil:
+    ctypedef void* Stream 'cudaStream_t'
+
+    # Version
+    cusparseStatus_t cusparseGetVersion(cusparseHandle_t handle, int* version)
+
+    # Error handling
+    const char* cusparseGetErrorName(Status status)
+    const char* cusparseGetErrorString(Status status)
+
+    # cuSPARSE Helper Function
+    Status cusparseCreate(Handle *handle)
+    Status cusparseCreateMatDescr(MatDescr descr)
+    Status cusparseDestroy(Handle handle)
+    Status cusparseDestroyMatDescr(MatDescr descr)
+    Status cusparseSetMatIndexBase(MatDescr descr, IndexBase base)
+    Status cusparseSetMatType(MatDescr descr, MatrixType type)
+    Status cusparseSetMatFillMode(MatDescr descrA, FillMode fillMode)
+    Status cusparseSetMatDiagType(MatDescr descrA, DiagType diagType)
+    Status cusparseSetPointerMode(Handle handle, PointerMode mode)
+
+    # Stream
+    Status cusparseSetStream(Handle handle, Stream streamId)
+    Status cusparseGetStream(Handle handle, Stream* streamId)
+
+    # cuSPARSE Level1 Function
+    Status cusparseSgthr(
+        Handle handle, int nnz, const float *y, float *xVal, const int *xInd,
+        IndexBase idxBase)
+
+    Status cusparseDgthr(
+        Handle handle, int nnz, const double *y, double *xVal, const int *xInd,
+        IndexBase idxBase)
+
+    Status cusparseCgthr(
+        Handle handle, int nnz, const cuComplex *y, cuComplex *xVal,
+        const int *xInd,
+        IndexBase idxBase)
+
+    Status cusparseZgthr(
+        Handle handle, int nnz, const cuDoubleComplex *y,
+        cuDoubleComplex *xVal, const int *xInd,
+        IndexBase idxBase)
+
+    # cuSPARSE Level2 Function
+    Status cusparseScsrmv(
+        Handle handle, Operation transA, int m, int n, int nnz,
+        const float *alpha, MatDescr descrA, const float *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        const float *x, const float *beta, float *y)
+
+    Status cusparseDcsrmv(
+        Handle handle, Operation transA, int m, int n, int nnz,
+        const double *alpha, MatDescr descrA, const double *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        const double *x, const double *beta, double *y)
+
+    Status cusparseCcsrmv(
+        Handle handle, Operation transA, int m, int n, int nnz,
+        const cuComplex *alpha, MatDescr descrA,
+        const cuComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        const cuComplex *x, const cuComplex *beta, cuComplex *y)
+
+    Status cusparseZcsrmv(
+        Handle handle, Operation transA, int m, int n, int nnz,
+        const cuDoubleComplex *alpha, MatDescr descrA,
+        const cuDoubleComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        const cuDoubleComplex *x, const cuDoubleComplex *beta,
+        cuDoubleComplex *y)
+
+    Status cusparseCsrmvEx_bufferSize(
+        Handle handle, AlgMode alg, Operation transA, int m, int n,
+        int nnz, const void *alpha, DataType alphatype,
+        MatDescr descrA, const void *csrValA, DataType csrValAtype,
+        const int *csrRowPtrA, const int *csrColIndA,
+        const void *x, DataType xtype, const void *beta,
+        DataType betatype, void *y, DataType ytype,
+        DataType executiontype, size_t *bufferSizeInBytes)
+
+    Status cusparseCsrmvEx(
+        Handle handle, AlgMode alg, Operation transA, int m, int n,
+        int nnz, const void *alpha, DataType alphatype,
+        MatDescr descrA, const void *csrValA, DataType csrValAtype,
+        const int *csrRowPtrA, const int *csrColIndA,
+        const void *x, DataType xtype, const void *beta,
+        DataType betatype, void *y, DataType ytype,
+        DataType executiontype, void* buffer)
+
+    Status cusparseCreateCsrsv2Info(csrsv2Info_t* info)
+    Status cusparseDestroyCsrsv2Info(csrsv2Info_t info)
+
+    Status cusparseScsrsv2_bufferSize(
+        Handle handle, Operation transA, int m, int nnz, const MatDescr descrA,
+        float* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        int* pBufferSizeInBytes)
+    Status cusparseDcsrsv2_bufferSize(
+        Handle handle, Operation transA, int m, int nnz, const MatDescr descrA,
+        double* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        int* pBufferSizeInBytes)
+    Status cusparseCcsrsv2_bufferSize(
+        Handle handle, Operation transA, int m, int nnz, const MatDescr descrA,
+        cuComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        int* pBufferSizeInBytes)
+    Status cusparseZcsrsv2_bufferSize(
+        Handle handle, Operation transA, int m, int nnz, const MatDescr descrA,
+        cuDoubleComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        int* pBufferSizeInBytes)
+
+    Status cusparseScsrsv2_analysis(
+        Handle handle, Operation transA, int m, int nnz, const MatDescr descrA,
+        const float* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        cusparseSolvePolicy_t policy, void* pBuffer)
+    Status cusparseDcsrsv2_analysis(
+        Handle handle, Operation transA, int m, int nnz, const MatDescr descrA,
+        const double* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        cusparseSolvePolicy_t policy, void* pBuffer)
+    Status cusparseCcsrsv2_analysis(
+        Handle handle, Operation transA, int m, int nnz, const MatDescr descrA,
+        const cuComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        cusparseSolvePolicy_t policy, void* pBuffer)
+    Status cusparseZcsrsv2_analysis(
+        Handle handle, Operation transA, int m, int nnz, const MatDescr descrA,
+        const cuDoubleComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        cusparseSolvePolicy_t policy, void* pBuffer)
+
+    Status cusparseScsrsv2_solve(
+        Handle handle, Operation transA, int m, int nnz,
+        const float* alpha, const MatDescr descrA,
+        const float* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        const float* x, float* y,
+        cusparseSolvePolicy_t policy, void* pBuffer)
+    Status cusparseDcsrsv2_solve(
+        Handle handle, Operation transA, int m, int nnz,
+        const double* alpha, const MatDescr descrA,
+        const double* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        const double* x, double* y,
+        cusparseSolvePolicy_t policy, void* pBuffer)
+    Status cusparseCcsrsv2_solve(
+        Handle handle, Operation transA, int m, int nnz,
+        const cuComplex* alpha, const MatDescr descrA,
+        const cuComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        const cuComplex* x, cuComplex* y,
+        cusparseSolvePolicy_t policy, void* pBuffer)
+    Status cusparseZcsrsv2_solve(
+        Handle handle, Operation transA, int m, int nnz,
+        const cuDoubleComplex* alpha, const MatDescr descrA,
+        const cuDoubleComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, csrsv2Info_t info,
+        const cuDoubleComplex* x, cuDoubleComplex* y,
+        cusparseSolvePolicy_t policy, void* pBuffer)
+
+    Status cusparseXcsrsv2_zeroPivot(
+        Handle handle, csrsv2Info_t info, int* position)
+
+    # cuSPARSE Level3 Function
+    Status cusparseScsrmm(
+        Handle handle, Operation transA, int m, int n, int k, int nnz,
+        const float *alpha, const MatDescr descrA, const float *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        const float *B, int ldb, const float *beta, float *C, int ldc)
+
+    Status cusparseDcsrmm(
+        Handle handle, Operation transA, int m, int n, int k, int nnz,
+        const double *alpha, const MatDescr descrA,
+        const double *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        const double *B, int ldb, const double *beta, double *C, int ldc)
+
+    Status cusparseCcsrmm(
+        Handle handle, Operation transA, int m, int n, int k, int nnz,
+        const cuComplex *alpha, const MatDescr descrA,
+        const cuComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        const cuComplex *B, int ldb, const cuComplex *beta,
+        cuComplex *C, int ldc)
+
+    Status cusparseZcsrmm(
+        Handle handle, Operation transA, int m, int n, int k, int nnz,
+        const cuDoubleComplex *alpha, const MatDescr descrA,
+        const cuDoubleComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        const cuDoubleComplex *B, int ldb,
+        const cuDoubleComplex *beta, cuDoubleComplex *C, int ldc)
+
+    Status cusparseScsrmm2(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        int nnz, const float *alpha, const MatDescr descrA,
+        const float *csrValA, const int *csrRowPtrA, const int *csrColIndA,
+        const float *B, int ldb, const float *beta, float *C, int ldc)
+
+    Status cusparseDcsrmm2(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        int nnz, const double *alpha, const MatDescr descrA,
+        const double *csrValA, const int *csrRowPtrA, const int *csrColIndA,
+        const double *B, int ldb, const double *beta, double *C, int ldc)
+
+    Status cusparseCcsrmm2(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        int nnz, const cuComplex *alpha, const MatDescr descrA,
+        const cuComplex *csrValA, const int *csrRowPtrA, const int *csrColIndA,
+        const cuComplex *B, int ldb, const cuComplex *beta,
+        cuComplex *C, int ldc)
+
+    Status cusparseZcsrmm2(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        int nnz, const cuDoubleComplex *alpha, const MatDescr descrA,
+        const cuDoubleComplex *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA,
+        const cuDoubleComplex *B, int ldb, const cuDoubleComplex *beta,
+        cuDoubleComplex *C, int ldc)
+
+    Status cusparseCreateCsrsm2Info(csrsm2Info_t* info)
+    Status cusparseDestroyCsrsm2Info(csrsm2Info_t info)
+
+    Status cusparseScsrsm2_bufferSizeExt(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const float* alpha, const MatDescr descrA,
+        const float* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, const float* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        size_t* pBufferSize)
+    Status cusparseDcsrsm2_bufferSizeExt(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const double* alpha, const MatDescr descrA,
+        const double* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, const double* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        size_t* pBufferSize)
+    Status cusparseCcsrsm2_bufferSizeExt(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const cuComplex* alpha, const MatDescr descrA,
+        const cuComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, const cuComplex* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        size_t* pBufferSize)
+    Status cusparseZcsrsm2_bufferSizeExt(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const cuDoubleComplex* alpha, const MatDescr descrA,
+        const cuDoubleComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, const cuDoubleComplex* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        size_t* pBufferSize)
+
+    Status cusparseScsrsm2_analysis(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const float* alpha, const MatDescr descrA,
+        const float* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, const float* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        void* pBuffer)
+    Status cusparseDcsrsm2_analysis(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const double* alpha, const MatDescr descrA,
+        const double* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, const double* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        void* pBuffer)
+    Status cusparseCcsrsm2_analysis(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const cuComplex* alpha, const MatDescr descrA,
+        const cuComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, const cuComplex* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        void* pBuffer)
+    Status cusparseZcsrsm2_analysis(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const cuDoubleComplex* alpha, const MatDescr descrA,
+        const cuDoubleComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, const cuDoubleComplex* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        void* pBuffer)
+
+    Status cusparseScsrsm2_solve(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const float* alpha, const MatDescr descrA,
+        const float* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, float* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        void* pBuffer)
+    Status cusparseDcsrsm2_solve(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const double* alpha, const MatDescr descrA,
+        const double* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, double* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        void* pBuffer)
+    Status cusparseCcsrsm2_solve(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const cuComplex* alpha, const MatDescr descrA,
+        const cuComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, cuComplex* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        void* pBuffer)
+    Status cusparseZcsrsm2_solve(
+        Handle handle, int algo, Operation transA, Operation transB, int m,
+        int nrhs, int nnz, const cuDoubleComplex* alpha, const MatDescr descrA,
+        const cuDoubleComplex* csrSortedValA, const int* csrSortedRowPtrA,
+        const int* csrSortedColIndA, cuDoubleComplex* B,
+        int ldb, csrsm2Info_t info, cusparseSolvePolicy_t policy,
+        void* pBuffer)
+
+    Status cusparseXcsrsm2_zeroPivot(
+        Handle handle, csrsm2Info_t info, int* position)
+
+    # cuSPARSE Extra Function
+    Status cusparseXcsrgeamNnz(
+        Handle handle, int m, int n, const MatDescr descrA, int nnzA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        int nnzB, const int *csrRowPtrB, const int *csrColIndB,
+        const MatDescr descrC, int *csrRowPtrC, int *nnzTotalDevHostPtr)
+
+    Status cusparseScsrgeam(
+        Handle handle, int m, int n, const float *alpha, const MatDescr descrA,
+        int nnzA, const float *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const float *beta, const MatDescr descrB,
+        int nnzB, const float *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, float *csrValC,
+        int *csrRowPtrC, int *csrColIndC)
+
+    Status cusparseDcsrgeam(
+        Handle handle, int m, int n, const double *alpha,
+        const MatDescr descrA,
+        int nnzA, const double *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const double *beta, const MatDescr descrB,
+        int nnzB, const double *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, double *csrValC,
+        int *csrRowPtrC, int *csrColIndC)
+
+    Status cusparseCcsrgeam(
+        Handle handle, int m, int n, const cuComplex *alpha,
+        const MatDescr descrA,
+        int nnzA, const cuComplex *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const cuComplex *beta, const MatDescr descrB,
+        int nnzB, const cuComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, cuComplex *csrValC,
+        int *csrRowPtrC, int *csrColIndC)
+
+    Status cusparseZcsrgeam(
+        Handle handle, int m, int n, const cuDoubleComplex *alpha,
+        const MatDescr descrA,
+        int nnzA, const cuDoubleComplex *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const cuDoubleComplex *beta,
+        const MatDescr descrB,
+        int nnzB, const cuDoubleComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC,
+        cuDoubleComplex *csrValC, int *csrRowPtrC, int *csrColIndC)
+
+    Status cusparseScsrgeam2_bufferSizeExt(
+        Handle handle, int m, int n, const float *alpha, const MatDescr descrA,
+        int nnzA, const float *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const float *beta, const MatDescr descrB,
+        int nnzB, const float *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, float *csrValC,
+        int *csrRowPtrC, int *csrColIndC, size_t *pBufferSize)
+
+    Status cusparseDcsrgeam2_bufferSizeExt(
+        Handle handle, int m, int n, const double *alpha,
+        const MatDescr descrA,
+        int nnzA, const double *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const double *beta, const MatDescr descrB,
+        int nnzB, const double *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, double *csrValC,
+        int *csrRowPtrC, int *csrColIndC, size_t *pBufferSize)
+
+    Status cusparseCcsrgeam2_bufferSizeExt(
+        Handle handle, int m, int n, const cuComplex *alpha,
+        const MatDescr descrA,
+        int nnzA, const cuComplex *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const cuComplex *beta, const MatDescr descrB,
+        int nnzB, const cuComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, cuComplex *csrValC,
+        int *csrRowPtrC, int *csrColIndC, size_t *pBufferSize)
+
+    Status cusparseZcsrgeam2_bufferSizeExt(
+        Handle handle, int m, int n, const cuDoubleComplex *alpha,
+        const MatDescr descrA,
+        int nnzA, const cuDoubleComplex *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const cuDoubleComplex *beta,
+        const MatDescr descrB,
+        int nnzB, const cuDoubleComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC,
+        cuDoubleComplex *csrValC, int *csrRowPtrC, int *csrColIndC,
+        size_t *pBufferSize)
+
+    Status cusparseXcsrgeam2Nnz(
+        Handle handle, int m, int n, const MatDescr descrA, int nnzA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        int nnzB, const int *csrRowPtrB, const int *csrColIndB,
+        const MatDescr descrC, int *csrRowPtrC, int *nnzTotalDevHostPtr,
+        void *workspace)
+
+    Status cusparseScsrgeam2(
+        Handle handle, int m, int n, const float *alpha, const MatDescr descrA,
+        int nnzA, const float *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const float *beta, const MatDescr descrB,
+        int nnzB, const float *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, float *csrValC,
+        int *csrRowPtrC, int *csrColIndC, void *pBuffer)
+
+    Status cusparseDcsrgeam2(
+        Handle handle, int m, int n, const double *alpha,
+        const MatDescr descrA,
+        int nnzA, const double *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const double *beta, const MatDescr descrB,
+        int nnzB, const double *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, double *csrValC,
+        int *csrRowPtrC, int *csrColIndC, void *pBuffer)
+
+    Status cusparseCcsrgeam2(
+        Handle handle, int m, int n, const cuComplex *alpha,
+        const MatDescr descrA,
+        int nnzA, const cuComplex *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const cuComplex *beta, const MatDescr descrB,
+        int nnzB, const cuComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, cuComplex *csrValC,
+        int *csrRowPtrC, int *csrColIndC, void *pBuffer)
+
+    Status cusparseZcsrgeam2(
+        Handle handle, int m, int n, const cuDoubleComplex *alpha,
+        const MatDescr descrA,
+        int nnzA, const cuDoubleComplex *csrValA, const int *csrRowPtrA,
+        const int *csrColIndA, const cuDoubleComplex *beta,
+        const MatDescr descrB,
+        int nnzB, const cuDoubleComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC,
+        cuDoubleComplex *csrValC, int *csrRowPtrC, int *csrColIndC,
+        void *pBuffer)
+
+    Status cusparseXcsrgemmNnz(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        const MatDescr descrA, const int nnzA, const int *csrRowPtrA,
+        const int *csrColIndA, const MatDescr descrB, const int nnzB,
+        const int *csrRowPtrB, const int *csrColIndB,
+        const MatDescr descrC, int *csrRowPtrC, int *nnzTotalDevHostPtr)
+
+    Status cusparseScsrgemm(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        const MatDescr descrA, const int nnzA, const float *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        const int nnzB, const float *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, float *csrValC,
+        const int *csrRowPtrC, int *csrColIndC)
+
+    Status cusparseDcsrgemm(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        const MatDescr descrA, const int nnzA, const double *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        const int nnzB, const double *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, double *csrValC,
+        const int *csrRowPtrC, int *csrColIndC)
+
+    Status cusparseCcsrgemm(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        const MatDescr descrA, const int nnzA, const cuComplex *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        const int nnzB, const cuComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, cuComplex *csrValC,
+        const int *csrRowPtrC, int *csrColIndC)
+
+    Status cusparseZcsrgemm(
+        Handle handle, Operation transA, Operation transB, int m, int n, int k,
+        const MatDescr descrA, const int nnzA, const cuDoubleComplex *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        const int nnzB, const cuDoubleComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const MatDescr descrC, cuDoubleComplex *csrValC,
+        const int *csrRowPtrC, int *csrColIndC)
+
+    cusparseStatus_t cusparseCreateCsrgemm2Info(csrgemm2Info_t *info)
+    cusparseStatus_t cusparseDestroyCsrgemm2Info(csrgemm2Info_t info)
+
+    Status cusparseScsrgemm2_bufferSizeExt(
+        Handle handle, int m, int n, int k, const float *alpha,
+        const MatDescr descrA, int nnzA, const int *csrRowPtrA,
+        const int *csrColIndA, const MatDescr descrB, int nnzB,
+        const int *csrRowPtrB, const int *csrColIndB, const float *beta,
+        const MatDescr descrD, int nnzD, const int *csrRowPtrD,
+        const int *csrColIndD, const csrgemm2Info_t info, size_t* pBufferSize)
+
+    Status cusparseDcsrgemm2_bufferSizeExt(
+        Handle handle, int m, int n, int k, const double *alpha,
+        const MatDescr descrA, int nnzA, const int *csrRowPtrA,
+        const int *csrColIndA, const MatDescr descrB, int nnzB,
+        const int *csrRowPtrB, const int *csrColIndB, const double *beta,
+        const MatDescr descrD, int nnzD, const int *csrRowPtrD,
+        const int *csrColIndD, const csrgemm2Info_t info, size_t* pBufferSize)
+
+    Status cusparseCcsrgemm2_bufferSizeExt(
+        Handle handle, int m, int n, int k, const cuComplex *alpha,
+        const MatDescr descrA, int nnzA, const int *csrRowPtrA,
+        const int *csrColIndA, const MatDescr descrB, int nnzB,
+        const int *csrRowPtrB, const int *csrColIndB, const cuComplex *beta,
+        const MatDescr descrD, int nnzD, const int *csrRowPtrD,
+        const int *csrColIndD, const csrgemm2Info_t info, size_t* pBufferSize)
+
+    Status cusparseZcsrgemm2_bufferSizeExt(
+        Handle handle, int m, int n, int k, const cuDoubleComplex *alpha,
+        const MatDescr descrA, int nnzA, const int *csrRowPtrA,
+        const int *csrColIndA, const MatDescr descrB, int nnzB,
+        const int *csrRowPtrB, const int *csrColIndB,
+        const cuDoubleComplex *beta, const MatDescr descrD, int nnzD,
+        const int *csrRowPtrD, const int *csrColIndD,
+        const csrgemm2Info_t info, size_t* pBufferSize)
+
+    Status cusparseXcsrgemm2Nnz(
+        Handle handle, int m, int n, int k, const MatDescr descrA, int nnzA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        int nnzB, const int *csrRowPtrB, const int *csrColIndB,
+        const MatDescr descrD, int nnzD, const int *csrRowPtrD,
+        const int *csrColIndD, const MatDescr descrC, int *csrRowPtrC,
+        int *nnzTotalDevHostPtr, const csrgemm2Info_t info, void* pBuffer)
+
+    Status cusparseScsrgemm2(
+        Handle handle, int m, int n, int k, const float *alpha,
+        const MatDescr descrA, int nnzA, const float *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        int nnzB, const float *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const float *beta, const MatDescr descrD,
+        int nnzD, const float *csrValD, const int *csrRowPtrD,
+        const int *csrColIndD, const MatDescr descrC, float *csrValC,
+        const int *csrRowPtrC, int *csrColIndC, const csrgemm2Info_t info,
+        void* pBuffer)
+
+    Status cusparseDcsrgemm2(
+        Handle handle, int m, int n, int k, const double *alpha,
+        const MatDescr descrA, int nnzA, const double *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        int nnzB, const double *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const double *beta, const MatDescr descrD,
+        int nnzD, const double *csrValD, const int *csrRowPtrD,
+        const int *csrColIndD, const MatDescr descrC, double *csrValC,
+        const int *csrRowPtrC, int *csrColIndC, const csrgemm2Info_t info,
+        void* pBuffer)
+
+    Status cusparseCcsrgemm2(
+        Handle handle, int m, int n, int k, const cuComplex *alpha,
+        const MatDescr descrA, int nnzA, const cuComplex *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        int nnzB, const cuComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const cuComplex *beta, const MatDescr descrD,
+        int nnzD, const cuComplex *csrValD, const int *csrRowPtrD,
+        const int *csrColIndD, const MatDescr descrC, cuComplex *csrValC,
+        const int *csrRowPtrC, int *csrColIndC, const csrgemm2Info_t info,
+        void* pBuffer)
+
+    Status cusparseZcsrgemm2(
+        Handle handle, int m, int n, int k, const cuDoubleComplex *alpha,
+        const MatDescr descrA, int nnzA, const cuDoubleComplex *csrValA,
+        const int *csrRowPtrA, const int *csrColIndA, const MatDescr descrB,
+        int nnzB, const cuDoubleComplex *csrValB, const int *csrRowPtrB,
+        const int *csrColIndB, const cuDoubleComplex *beta,
+        const MatDescr descrD, int nnzD, const cuDoubleComplex *csrValD,
+        const int *csrRowPtrD, const int *csrColIndD, const MatDescr descrC,
+        cuDoubleComplex *csrValC, const int *csrRowPtrC, int *csrColIndC,
+        const csrgemm2Info_t info, void* pBuffer)
+
+    # cuSPARSE Format Convrsion
+    Status cusparseXcoo2csr(
+        Handle handle, const int *cooRowInd, int nnz, int m, int *csrRowPtr,
+        IndexBase idxBase)
+
+    Status cusparseScsc2dense(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const float *cscSortedValA, const int *cscSortedRowIndA,
+        const int *cscSortedColPtrA, float *A, int lda)
+
+    Status cusparseDcsc2dense(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const double *cscSortedValA, const int *cscSortedRowIndA,
+        const int *cscSortedColPtrA, double *A, int lda)
+
+    Status cusparseCcsc2dense(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuComplex *cscSortedValA, const int *cscSortedRowIndA,
+        const int *cscSortedColPtrA, cuComplex *A, int lda)
+
+    Status cusparseZcsc2dense(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuDoubleComplex *cscSortedValA, const int *cscSortedRowIndA,
+        const int *cscSortedColPtrA, cuDoubleComplex *A, int lda)
+
+    Status cusparseXcsr2coo(
+        Handle handle, const int *csrRowPtr, int nnz, int m, int *cooRowInd,
+        IndexBase idxBase)
+
+    Status cusparseScsr2csc(
+        Handle handle, int m, int n, int nnz, const float *csrVal,
+        const int *csrRowPtr, const int *csrColInd, float *cscVal,
+        int *cscRowInd, int *cscColPtr, Action copyValues, IndexBase idxBase)
+
+    Status cusparseDcsr2csc(
+        Handle handle, int m, int n, int nnz, const double *csrVal,
+        const int *csrRowPtr, const int *csrColInd, double *cscVal,
+        int *cscRowInd, int *cscColPtr, Action copyValues, IndexBase idxBase)
+
+    Status cusparseCcsr2csc(
+        Handle handle, int m, int n, int nnz, const cuComplex *csrVal,
+        const int *csrRowPtr, const int *csrColInd, cuComplex *cscVal,
+        int *cscRowInd, int *cscColPtr, Action copyValues, IndexBase idxBase)
+
+    Status cusparseZcsr2csc(
+        Handle handle, int m, int n, int nnz, const cuDoubleComplex *csrVal,
+        const int *csrRowPtr, const int *csrColInd, cuDoubleComplex *cscVal,
+        int *cscRowInd, int *cscColPtr, Action copyValues, IndexBase idxBase)
+
+    Status cusparseScsr2dense(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const float *csrSortedValA, const int *csrSortedRowPtrA,
+        const int *csrSortedColIndA, float *A, int lda)
+
+    Status cusparseDcsr2dense(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const double *csrSortedValA, const int *csrSortedRowPtrA,
+        const int *csrSortedColIndA, double *A, int lda)
+
+    Status cusparseCcsr2dense(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuComplex *csrSortedValA, const int *csrSortedRowPtrA,
+        const int *csrSortedColIndA, cuComplex *A, int lda)
+
+    Status cusparseZcsr2dense(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuDoubleComplex *csrSortedValA, const int *csrSortedRowPtrA,
+        const int *csrSortedColIndA, cuDoubleComplex *A, int lda)
+
+    Status cusparseSnnz_compress(
+        Handle handle, int m, const MatDescr descr,
+        const float *values, const int *rowPtr, int *nnzPerRow,
+        int *nnzTotal, float tol)
+
+    Status cusparseDnnz_compress(
+        Handle handle, int m, const MatDescr descr,
+        const double *values, const int *rowPtr, int *nnzPerRow,
+        int *nnzTotal, double tol)
+
+    Status cusparseCnnz_compress(
+        Handle handle, int m, const MatDescr descr,
+        const cuComplex *values, const int *rowPtr, int *nnzPerRow,
+        int *nnzTotal, cuComplex tol)
+
+    Status cusparseZnnz_compress(
+        Handle handle, int m, const MatDescr descr,
+        const cuDoubleComplex *values, const int *rowPtr, int *nnzPerRow,
+        int *nnzTotal, cuDoubleComplex tol)
+
+    Status cusparseScsr2csr_compress(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const float *inVal, const int *inColInd, const int *inRowPtr,
+        int inNnz, int *nnzPerRow, float *outVal, int *outColInd,
+        int *outRowPtr, float tol)
+
+    Status cusparseDcsr2csr_compress(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const double *inVal, const int *inColInd, const int *inRowPtr,
+        int inNnz, int *nnzPerRow, double *outVal, int *outColInd,
+        int *outRowPtr, double tol)
+
+    Status cusparseCcsr2csr_compress(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuComplex *inVal, const int *inColInd, const int *inRowPtr,
+        int inNnz, int *nnzPerRow, cuComplex *outVal, int *outColInd,
+        int *outRowPtr, cuComplex tol)
+
+    Status cusparseZcsr2csr_compress(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuDoubleComplex *inVal, const int *inColInd, const int *inRowPtr,
+        int inNnz, int *nnzPerRow, cuDoubleComplex *outVal, int *outColInd,
+        int *outRowPtr, cuDoubleComplex tol)
+
+    Status cusparseSdense2csc(
+        Handle handle, int m, int n, const MatDescr descrA, const float *A,
+        int lda, const int *nnzPerCol, float *cscValA, int *cscRowIndA,
+        int *cscColPtrA)
+
+    Status cusparseDdense2csc(
+        Handle handle, int m, int n, const MatDescr descrA, const double *A,
+        int lda, const int *nnzPerCol, double *cscValA, int *cscRowIndA,
+        int *cscColPtrA)
+
+    Status cusparseCdense2csc(
+        Handle handle, int m, int n, const MatDescr descrA, const cuComplex *A,
+        int lda, const int *nnzPerCol, cuComplex *cscValA, int *cscRowIndA,
+        int *cscColPtrA)
+
+    Status cusparseZdense2csc(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuDoubleComplex *A,
+        int lda, const int *nnzPerCol, cuDoubleComplex *cscValA,
+        int *cscRowIndA, int *cscColPtrA)
+
+    Status cusparseSdense2csr(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const float *A, int lda, const int *nnzPerRow, float *csrValA,
+        int *csrRowPtrA, int *csrColIndA)
+
+    Status cusparseDdense2csr(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const double *A, int lda, const int *nnzPerRow, double *csrValA,
+        int *csrRowPtrA, int *csrColIndA)
+
+    Status cusparseCdense2csr(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuComplex *A, int lda, const int *nnzPerRow, cuComplex *csrValA,
+        int *csrRowPtrA, int *csrColIndA)
+
+    Status cusparseZdense2csr(
+        Handle handle, int m, int n, const MatDescr descrA,
+        const cuDoubleComplex *A, int lda, const int *nnzPerRow,
+        cuDoubleComplex *csrValA,
+        int *csrRowPtrA, int *csrColIndA)
+
+    Status cusparseSnnz(
+        Handle handle, Direction dirA, int m, int n, const MatDescr descrA,
+        const float *A, int lda, int *nnzPerRowColumn, int *nnzTotalDevHostPtr)
+
+    Status cusparseDnnz(
+        Handle handle, Direction dirA, int m, int n, const MatDescr descrA,
+        const double *A, int lda, int *nnzPerRowColumn,
+        int *nnzTotalDevHostPtr)
+
+    Status cusparseCnnz(
+        Handle handle, Direction dirA, int m, int n, const MatDescr descrA,
+        const cuComplex *A, int lda, int *nnzPerRowColumn,
+        int *nnzTotalDevHostPtr)
+
+    Status cusparseZnnz(
+        Handle handle, Direction dirA, int m, int n, const MatDescr descrA,
+        const cuDoubleComplex *A, int lda, int *nnzPerRowColumn,
+        int *nnzTotalDevHostPtr)
+
+    Status cusparseCreateIdentityPermutation(
+        Handle handle, int n, int *p)
+
+    Status cusparseXcoosort_bufferSizeExt(
+        Handle handle, int m, int n, int nnz, const int *cooRows,
+        const int *cooCols, size_t *pBufferSizeInBytes)
+
+    Status cusparseXcoosortByRow(
+        Handle handle, int m, int n, int nnz, int *cooRows, int *cooCols,
+        int *P, void *pBuffer)
+
+    Status cusparseXcoosortByColumn(
+        Handle handle, int m, int n, int nnz, int *cooRows, int *cooCols,
+        int *P, void *pBuffer)
+
+    Status cusparseXcsrsort_bufferSizeExt(
+        Handle handle, int m, int n, int nnz, const int *csrRowPtr,
+        const int *csrColInd, size_t *pBufferSizeInBytes)
+
+    Status cusparseXcsrsort(
+        Handle handle, int m, int n, int nnz, const MatDescr descrA,
+        const int *csrRowPtr, int *csrColInd, int *P, void *pBuffer)
+
+    Status cusparseXcscsort_bufferSizeExt(
+        Handle handle, int m, int n, int nnz, const int *cscColPtr,
+        const int *cscRowInd, size_t *pBufferSizeInBytes)
+
+    Status cusparseXcscsort(
+        Handle handle, int m, int n, int nnz, const MatDescr descrA,
+        const int *cscColPtr, int *cscRowInd, int *P, void *pBuffer)
+
+    # cuSparse PRECONDITIONERS
+    cusparseStatus_t cusparseCreateCsrilu02Info(csrilu02Info_t *info)
+    cusparseStatus_t cusparseDestroyCsrilu02Info(csrilu02Info_t info)
+    cusparseStatus_t cusparseCreateBsrilu02Info(bsrilu02Info_t *info)
+    cusparseStatus_t cusparseDestroyBsrilu02Info(bsrilu02Info_t info)
+    cusparseStatus_t cusparseCreateCsric02Info(csric02Info_t *info)
+    cusparseStatus_t cusparseDestroyCsric02Info(csric02Info_t info)
+    cusparseStatus_t cusparseCreateBsric02Info(bsric02Info_t *info)
+    cusparseStatus_t cusparseDestroyBsric02Info(bsric02Info_t info)
+    cusparseStatus_t cusparseScsrilu02_numericBoost(
+        cusparseHandle_t handle, csrilu02Info_t info, int enable_boost,
+        double *tol, float *boost_val)
+    cusparseStatus_t cusparseDcsrilu02_numericBoost(
+        cusparseHandle_t handle, csrilu02Info_t info, int enable_boost,
+        double *tol, double *boost_val)
+    cusparseStatus_t cusparseCcsrilu02_numericBoost(
+        cusparseHandle_t handle, csrilu02Info_t info, int enable_boost,
+        double *tol, cuComplex *boost_val)
+    cusparseStatus_t cusparseZcsrilu02_numericBoost(
+        cusparseHandle_t handle, csrilu02Info_t info, int enable_boost,
+        double *tol, cuDoubleComplex *boost_val)
+    cusparseStatus_t cusparseXcsrilu02_zeroPivot(
+        cusparseHandle_t handle, csrilu02Info_t info, int *position)
+    cusparseStatus_t cusparseScsrilu02_bufferSize(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, float *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseDcsrilu02_bufferSize(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, double *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseCcsrilu02_bufferSize(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, cuComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseZcsrilu02_bufferSize(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseScsrilu02_analysis(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, const float *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseDcsrilu02_analysis(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, const double *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseCcsrilu02_analysis(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, const cuComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseZcsrilu02_analysis(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, const cuDoubleComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseScsrilu02(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, float *csrSortedValA_valM,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseDcsrilu02(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, double *csrSortedValA_valM,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseCcsrilu02(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, cuComplex *csrSortedValA_valM,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseZcsrilu02(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *csrSortedValA_valM,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseSbsrilu02_numericBoost(
+        cusparseHandle_t handle, bsrilu02Info_t info, int enable_boost,
+        double *tol, float *boost_val)
+    cusparseStatus_t cusparseDbsrilu02_numericBoost(
+        cusparseHandle_t handle, bsrilu02Info_t info, int enable_boost,
+        double *tol, double *boost_val)
+    cusparseStatus_t cusparseCbsrilu02_numericBoost(
+        cusparseHandle_t handle, bsrilu02Info_t info, int enable_boost,
+        double *tol, cuComplex *boost_val)
+    cusparseStatus_t cusparseZbsrilu02_numericBoost(
+        cusparseHandle_t handle, bsrilu02Info_t info, int enable_boost,
+        double *tol, cuDoubleComplex *boost_val)
+    cusparseStatus_t cusparseXbsrilu02_zeroPivot(
+        cusparseHandle_t handle, bsrilu02Info_t info, int *position)
+    cusparseStatus_t cusparseSbsrilu02_bufferSize(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, float *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseDbsrilu02_bufferSize(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, double *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseCbsrilu02_bufferSize(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseZbsrilu02_bufferSize(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseSbsrilu02_analysis(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, float *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseDbsrilu02_analysis(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, double *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseCbsrilu02_analysis(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseZbsrilu02_analysis(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseSbsrilu02(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, float *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseDbsrilu02(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, double *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseCbsrilu02(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseZbsrilu02(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsrilu02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseXcsric02_zeroPivot(
+        cusparseHandle_t handle, csric02Info_t info, int *position)
+    cusparseStatus_t cusparseScsric02_bufferSize(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, float *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseDcsric02_bufferSize(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, double *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseCcsric02_bufferSize(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, cuComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseZcsric02_bufferSize(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseScsric02_analysis(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, const float *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseDcsric02_analysis(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, const double *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseCcsric02_analysis(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, const cuComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseZcsric02_analysis(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, const cuDoubleComplex *csrSortedValA,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseScsric02(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, float *csrSortedValA_valM,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseDcsric02(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, double *csrSortedValA_valM,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseCcsric02(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, cuComplex *csrSortedValA_valM,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseZcsric02(
+        cusparseHandle_t handle, int m, int nnz,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *csrSortedValA_valM,
+        const int *csrSortedRowPtrA, const int *csrSortedColIndA,
+        csric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseXbsric02_zeroPivot(
+        cusparseHandle_t handle, bsric02Info_t info, int *position)
+    cusparseStatus_t cusparseSbsric02_bufferSize(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, float *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseDbsric02_bufferSize(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, double *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseCbsric02_bufferSize(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseZbsric02_bufferSize(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, int *pBufferSizeInBytes)
+    cusparseStatus_t cusparseSbsric02_analysis(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, const float *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, cusparseSolvePolicy_t policy, void *pInputBuffer)
+    cusparseStatus_t cusparseDbsric02_analysis(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, const double *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, cusparseSolvePolicy_t policy, void *pInputBuffer)
+    cusparseStatus_t cusparseCbsric02_analysis(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, const cuComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, cusparseSolvePolicy_t policy, void *pInputBuffer)
+    cusparseStatus_t cusparseZbsric02_analysis(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, const cuDoubleComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, cusparseSolvePolicy_t policy, void *pInputBuffer)
+    cusparseStatus_t cusparseSbsric02(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, float *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseDbsric02(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, double *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseCbsric02(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseZbsric02(
+        cusparseHandle_t handle, cusparseDirection_t dirA, int mb, int nnzb,
+        const cusparseMatDescr_t descrA, cuDoubleComplex *bsrSortedVal,
+        const int *bsrSortedRowPtr, const int *bsrSortedColInd, int blockDim,
+        bsric02Info_t info, cusparseSolvePolicy_t policy, void *pBuffer)
+    cusparseStatus_t cusparseSgtsv2_bufferSizeExt(
+        cusparseHandle_t handle, int m, int n, const float *dl, const float *d,
+        const float *du, const float *B, int ldb, size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseDgtsv2_bufferSizeExt(
+        cusparseHandle_t handle, int m, int n, const double *dl,
+        const double *d, const double *du, const double *B, int ldb,
+        size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseCgtsv2_bufferSizeExt(
+        cusparseHandle_t handle, int m, int n, const cuComplex *dl,
+        const cuComplex *d, const cuComplex *du, const cuComplex *B, int ldb,
+        size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseZgtsv2_bufferSizeExt(
+        cusparseHandle_t handle, int m, int n, const cuDoubleComplex *dl,
+        const cuDoubleComplex *d, const cuDoubleComplex *du,
+        const cuDoubleComplex *B, int ldb, size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseSgtsv2(
+        cusparseHandle_t handle, int m, int n, const float *dl, const float *d,
+        const float *du, float *B, int ldb, void *pBuffer)
+    cusparseStatus_t cusparseDgtsv2(
+        cusparseHandle_t handle, int m, int n, const double *dl,
+        const double *d, const double *du, double *B, int ldb, void *pBuffer)
+    cusparseStatus_t cusparseCgtsv2(cusparseHandle_t handle, int m, int n,
+                                    const cuComplex *dl, const cuComplex *d,
+                                    const cuComplex *du, cuComplex *B, int ldb,
+                                    void *pBuffer)
+    cusparseStatus_t cusparseZgtsv2(
+        cusparseHandle_t handle, int m, int n, const cuDoubleComplex *dl,
+        const cuDoubleComplex *d, const cuDoubleComplex *du,
+        cuDoubleComplex *B, int ldb, void *pBuffer)
+    cusparseStatus_t cusparseSgtsv2_nopivot_bufferSizeExt(
+        cusparseHandle_t handle, int m, int n, const float *dl, const float *d,
+        const float *du, const float *B, int ldb, size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseDgtsv2_nopivot_bufferSizeExt(
+        cusparseHandle_t handle, int m, int n, const double *dl,
+        const double *d, const double *du, const double *B, int ldb,
+        size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseCgtsv2_nopivot_bufferSizeExt(
+        cusparseHandle_t handle, int m, int n, const cuComplex *dl,
+        const cuComplex *d, const cuComplex *du, const cuComplex *B, int ldb,
+        size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseZgtsv2_nopivot_bufferSizeExt(
+        cusparseHandle_t handle, int m, int n, const cuDoubleComplex *dl,
+        const cuDoubleComplex *d, const cuDoubleComplex *du,
+        const cuDoubleComplex *B, int ldb, size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseSgtsv2_nopivot(
+        cusparseHandle_t handle, int m, int n, const float *dl, const float *d,
+        const float *du, float *B, int ldb, void *pBuffer)
+    cusparseStatus_t cusparseDgtsv2_nopivot(
+        cusparseHandle_t handle, int m, int n, const double *dl,
+        const double *d, const double *du, double *B, int ldb, void *pBuffer)
+    cusparseStatus_t cusparseCgtsv2_nopivot(
+        cusparseHandle_t handle, int m, int n, const cuComplex *dl,
+        const cuComplex *d, const cuComplex *du, cuComplex *B, int ldb,
+        void *pBuffer)
+    cusparseStatus_t cusparseZgtsv2_nopivot(
+        cusparseHandle_t handle, int m, int n, const cuDoubleComplex *dl,
+        const cuDoubleComplex *d, const cuDoubleComplex *du,
+        cuDoubleComplex *B, int ldb, void *pBuffer)
+    cusparseStatus_t cusparseSgtsv2StridedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int m, const float *dl, const float *d,
+        const float *du, const float *x, int batchCount, int batchStride,
+        size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseDgtsv2StridedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int m, const double *dl, const double *d,
+        const double *du, const double *x, int batchCount, int batchStride,
+        size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseCgtsv2StridedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int m, const cuComplex *dl,
+        const cuComplex *d, const cuComplex *du, const cuComplex *x,
+        int batchCount, int batchStride, size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseZgtsv2StridedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int m, const cuDoubleComplex *dl,
+        const cuDoubleComplex *d, const cuDoubleComplex *du,
+        const cuDoubleComplex *x, int batchCount, int batchStride,
+        size_t *bufferSizeInBytes)
+    cusparseStatus_t cusparseSgtsv2StridedBatch(
+        cusparseHandle_t handle, int m, const float *dl, const float *d,
+        const float *du, float *x, int batchCount, int batchStride,
+        void *pBuffer)
+    cusparseStatus_t cusparseDgtsv2StridedBatch(
+        cusparseHandle_t handle, int m, const double *dl, const double *d,
+        const double *du, double *x, int batchCount, int batchStride,
+        void *pBuffer)
+    cusparseStatus_t cusparseCgtsv2StridedBatch(
+        cusparseHandle_t handle, int m, const cuComplex *dl,
+        const cuComplex *d, const cuComplex *du, cuComplex *x, int batchCount,
+        int batchStride, void *pBuffer)
+    cusparseStatus_t cusparseZgtsv2StridedBatch(
+        cusparseHandle_t handle, int m, const cuDoubleComplex *dl,
+        const cuDoubleComplex *d, const cuDoubleComplex *du,
+        cuDoubleComplex *x, int batchCount, int batchStride, void *pBuffer)
+    cusparseStatus_t cusparseSgtsvInterleavedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int algo, int m, const float *dl,
+        const float *d, const float *du, const float *x, int batchCount,
+        size_t *pBufferSizeInBytes)
+    cusparseStatus_t cusparseDgtsvInterleavedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int algo, int m, const double *dl,
+        const double *d, const double *du, const double *x, int batchCount,
+        size_t *pBufferSizeInBytes)
+    cusparseStatus_t cusparseCgtsvInterleavedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int algo, int m, const cuComplex *dl,
+        const cuComplex *d, const cuComplex *du, const cuComplex *x,
+        int batchCount, size_t *pBufferSizeInBytes)
+    cusparseStatus_t cusparseZgtsvInterleavedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int algo, int m, const cuDoubleComplex *dl,
+        const cuDoubleComplex *d, const cuDoubleComplex *du,
+        const cuDoubleComplex *x, int batchCount, size_t *pBufferSizeInBytes)
+    cusparseStatus_t cusparseSgtsvInterleavedBatch(
+        cusparseHandle_t handle, int algo, int m, float *dl, float *d,
+        float *du, float *x, int batchCount, void *pBuffer)
+    cusparseStatus_t cusparseDgtsvInterleavedBatch(
+        cusparseHandle_t handle, int algo, int m, double *dl, double *d,
+        double *du, double *x, int batchCount, void *pBuffer)
+    cusparseStatus_t cusparseCgtsvInterleavedBatch(
+        cusparseHandle_t handle, int algo, int m, cuComplex *dl, cuComplex *d,
+        cuComplex *du, cuComplex *x, int batchCount, void *pBuffer)
+    cusparseStatus_t cusparseZgtsvInterleavedBatch(
+        cusparseHandle_t handle, int algo, int m, cuDoubleComplex *dl,
+        cuDoubleComplex *d, cuDoubleComplex *du, cuDoubleComplex *x,
+        int batchCount, void *pBuffer)
+    cusparseStatus_t cusparseSgpsvInterleavedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int algo, int m, const float *ds,
+        const float *dl, const float *d, const float *du, const float *dw,
+        const float *x, int batchCount, size_t *pBufferSizeInBytes)
+    cusparseStatus_t cusparseDgpsvInterleavedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int algo, int m, const double *ds,
+        const double *dl, const double *d, const double *du, const double *dw,
+        const double *x, int batchCount, size_t *pBufferSizeInBytes)
+    cusparseStatus_t cusparseCgpsvInterleavedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int algo, int m, const cuComplex *ds,
+        const cuComplex *dl, const cuComplex *d, const cuComplex *du,
+        const cuComplex *dw, const cuComplex *x, int batchCount,
+        size_t *pBufferSizeInBytes)
+    cusparseStatus_t cusparseZgpsvInterleavedBatch_bufferSizeExt(
+        cusparseHandle_t handle, int algo, int m, const cuDoubleComplex *ds,
+        const cuDoubleComplex *dl, const cuDoubleComplex *d,
+        const cuDoubleComplex *du, const cuDoubleComplex *dw,
+        const cuDoubleComplex *x, int batchCount, size_t *pBufferSizeInBytes)
+    cusparseStatus_t cusparseSgpsvInterleavedBatch(
+        cusparseHandle_t handle, int algo, int m, float *ds, float *dl,
+        float *d, float *du, float *dw, float *x, int batchCount,
+        void *pBuffer)
+    cusparseStatus_t cusparseDgpsvInterleavedBatch(
+        cusparseHandle_t handle, int algo, int m, double *ds, double *dl,
+        double *d, double *du, double *dw, double *x, int batchCount,
+        void *pBuffer)
+    cusparseStatus_t cusparseCgpsvInterleavedBatch(
+        cusparseHandle_t handle, int algo, int m, cuComplex *ds, cuComplex *dl,
+        cuComplex *d, cuComplex *du, cuComplex *dw, cuComplex *x,
+        int batchCount, void *pBuffer)
+    cusparseStatus_t cusparseZgpsvInterleavedBatch(
+        cusparseHandle_t handle, int algo, int m, cuDoubleComplex *ds,
+        cuDoubleComplex *dl, cuDoubleComplex *d, cuDoubleComplex *du,
+        cuDoubleComplex *dw, cuDoubleComplex *x, int batchCount, void *pBuffer)
+
+    # Sparse Vector APIs
+    Status cusparseCreateSpVec(SpVecDescr* spVecDescr, int64_t size,
+                               int64_t nnz, void* indices, void* values,
+                               IndexType idxType, IndexBase idxBase,
+                               DataType valueType)
+    Status cusparseDestroySpVec(SpVecDescr spVecDescr)
+    Status cusparseSpVecGet(SpVecDescr spVecDescr, int64_t* size, int64_t* nnz,
+                            void** indices, void** values, IndexType* idxType,
+                            IndexBase* idxBase, DataType* valueType)
+    Status cusparseSpVecGetIndexBase(SpVecDescr spVecDescr, IndexBase* idxBae)
+    Status cusparseSpVecGetValues(SpVecDescr spVecDescr, void** values)
+    Status cusparseSpVecSetValues(SpVecDescr spVecDescr, void* values)
+
+    # Sparse Matrix APIs
+    Status cusparseCreateCoo(SpMatDescr* spMatDescr, int64_t rows,
+                             int64_t cols, int64_t nnz, void* cooRowInd,
+                             void* cooColInd, void* cooValues,
+                             IndexType cooIdxType, IndexBase idxBase,
+                             DataType valueType)
+    Status cusparseCreateCooAoS(SpMatDescr* spMatDescr, int64_t rows,
+                                int64_t cols, int64_t nnz, void* cooInd,
+                                void* cooValues, IndexType cooIdxType,
+                                IndexBase idxBase, DataType valueType)
+    Status cusparseCreateCsr(SpMatDescr* spMatDescr, int64_t rows,
+                             int64_t cols, int64_t nnz, void* csrRowOffsets,
+                             void* csrColind, void* csrValues,
+                             IndexType csrRowOffsetsType,
+                             IndexType csrColIndType, IndexBase idxBase,
+                             DataType valueType)
+    Status cusparseDestroySpMat(SpMatDescr spMatDescr)
+    Status cusparseCooGet(SpMatDescr spMatDescr, int64_t* rows, int64_t* cols,
+                          int64_t* nnz, void** cooRowInd, void** cooColInd,
+                          void** cooValues, IndexType* idxType,
+                          IndexBase* idxBase, DataType* valueType)
+    Status cusparseCooAoSGet(SpMatDescr spMatDescr, int64_t* rows,
+                             int64_t* cols, int64_t* nnz, void** cooInd,
+                             void** cooValues, IndexType* idxType,
+                             IndexBase* idxBase, DataType* valueType)
+    Status cusparseCsrGet(SpMatDescr spMatDescr, int64_t* rows, int64_t* cols,
+                          int64_t* nnz, void** csrRowOffsets, void** csrColInd,
+                          void** csrValues, IndexType* csrRowOffsetsType,
+                          IndexType* csrColIndType, IndexBase* idxBase,
+                          DataType* valueType)
+    Status cusparseCsrSetPointers(SpMatDescr spMatDescr, void* csrRowOffsets,
+                                  void* csrColInd, void* csrValues)
+    Status cusparseSpMatGetFormat(SpMatDescr spMatDescr, Format* format)
+    Status cusparseSpMatGetIndexBase(SpMatDescr spMatDescr, IndexBase* idxBase)
+    Status cusparseSpMatGetValues(SpMatDescr spMatDescr, void** values)
+    Status cusparseSpMatSetValues(SpMatDescr spMatDescr, void* values)
+    Status cusparseSpMatGetSize(SpMatDescr spMatDescr, int64_t* rows,
+                                int64_t* cols, int64_t* nnz)
+    Status cusparseSpMatGetStridedBatch(SpMatDescr spMatDescr, int* batchCount)
+    Status cusparseSpMatSetStridedBatch(SpMatDescr spMatDescr, int batchCount)
+
+    # Dense Vector APIs
+    Status cusparseCreateDnVec(DnVecDescr *dnVecDescr, int64_t size,
+                               void* values, DataType valueType)
+    Status cusparseDestroyDnVec(DnVecDescr dnVecDescr)
+    Status cusparseDnVecGet(DnVecDescr dnVecDescr, int64_t* size,
+                            void** values, DataType* valueType)
+    Status cusparseDnVecGetValues(DnVecDescr dnVecDescr, void** values)
+    Status cusparseDnVecSetValues(DnVecDescr dnVecDescr, void* values)
+
+    # Dense Matrix APIs
+    Status cusparseCreateDnMat(DnMatDescr* dnMatDescr, int64_t rows,
+                               int64_t cols, int64_t ld, void* values,
+                               DataType valueType, Order order)
+    Status cusparseDestroyDnMat(DnMatDescr dnVecDescr)
+    Status cusparseDnMatGet(DnMatDescr dnMatDescr, int64_t* rows,
+                            int64_t* cols, int64_t* ld, void** values,
+                            DataType* valueType, Order* order)
+    Status cusparseDnMatGetValues(DnMatDescr spMatDescr, void** values)
+    Status cusparseDnMatSetValues(DnMatDescr spMatDescr, void* values)
+    Status cusparseDnMatGetStridedBatch(DnMatDescr dnMatDescr, int* batchCount,
+                                        int64_t *batchStride)
+    Status cusparseDnMatSetStridedBatch(DnMatDescr dnMatDescr, int batchCount,
+                                        int64_t batchStride)
+
+    # Generic API Functions
+    Status cusparseSpVV_bufferSize(Handle handle, Operation opX,
+                                   SpVecDescr vecX, DnVecDescr vecY,
+                                   void* result, DataType computeType,
+                                   size_t* bufferSize)
+    Status cusparseSpVV(Handle handle, Operation opX, SpVecDescr vecX,
+                        DnVecDescr vecY, void* result, DataType computeType,
+                        void* externalBuffer)
+    Status cusparseSpMV_bufferSize(Handle handle, Operation opA, void* alpha,
+                                   SpMatDescr matA, DnVecDescr vecX,
+                                   void* beta, DnVecDescr vecY,
+                                   DataType computeType, SpMVAlg alg,
+                                   size_t* bufferSize)
+    Status cusparseSpMV(Handle handle, Operation opA, void* alpha,
+                        SpMatDescr matA, DnVecDescr vecX, void* beta,
+                        DnVecDescr vecY, DataType computeType, SpMVAlg alg,
+                        void* externalBuffer)
+    Status cusparseSpMM_bufferSize(Handle handle, Operation opA, Operation opB,
+                                   void* alpha, SpMatDescr matA,
+                                   DnMatDescr matB, void* beta,
+                                   DnMatDescr matC, DataType computeType,
+                                   SpMMAlg alg, size_t* bufferSize)
+    Status cusparseSpMM(Handle handle, Operation opA, Operation opB,
+                        void* alpha, SpMatDescr matA, DnMatDescr matB,
+                        void* beta, DnMatDescr matC, DataType computeType,
+                        SpMMAlg alg, void* externalBuffer)
+    Status cusparseConstrainedGeMM_bufferSize(
+        Handle handle, Operation opA, Operation opB, void* alpha,
+        DnMatDescr matA, DnMatDescr matB, void* beta, SpMatDescr matC,
+        DataType computeType, size_t* bufferSize)
+    Status cusparseConstrainedGeMM(
+        Handle handle, Operation opA, Operation opB, void* alpha,
+        DnMatDescr matA, DnMatDescr matB, void* beta, SpMatDescr matC,
+        DataType computeType, void* externalBuffer)
+
+    Status cusparseSpGEMM_createDescr(SpGEMMDescr* spgemmDescr)
+    Status cusparseSpGEMM_destroyDescr(SpGEMMDescr spgemmDescr)
+    Status cusparseSpGEMM_workEstimation(
+        Handle handle, Operation opA, Operation opB, const void* alpha,
+        SpMatDescr matA, SpMatDescr matB, const void* beta, SpMatDescr matC,
+        DataType computeType, SpGEMMAlg alg, SpGEMMDescr spgemmDescr,
+        size_t* bufferSize1, void* externalBuffer1)
+    Status cusparseSpGEMM_compute(
+        Handle handle, Operation opA, Operation opB, const void* alpha,
+        SpMatDescr matA, SpMatDescr matB, const void* beta, SpMatDescr matC,
+        DataType computeType, SpGEMMAlg alg, SpGEMMDescr spgemmDescr,
+        size_t* bufferSize2, void* externalBuffer2)
+    Status cusparseSpGEMM_copy(
+        Handle handle, Operation opA, Operation opB, const void* alpha,
+        SpMatDescr matA, SpMatDescr matB, const void* beta, SpMatDescr matC,
+        DataType computeType, SpGEMMAlg alg, SpGEMMDescr spgemmDescr)
+    Status cusparseGather(Handle handle, DnVecDescr vecY, SpVecDescr vecX)
+
+    # CSR2CSC
+    Status cusparseCsr2cscEx2_bufferSize(
+        Handle handle, int m, int n, int nnz, const void* csrVal,
+        const int* csrRowPtr, const int* csrColInd, void* cscVal,
+        int* cscColPtr, int* cscRowInd, DataType valType, Action copyValues,
+        IndexBase idxBase, Csr2CscAlg alg, size_t* bufferSize)
+
+    Status cusparseCsr2cscEx2(
+        Handle handle, int m, int n, int nnz, const void* csrVal,
+        const int* csrRowPtr, const int* csrColInd, void* cscVal,
+        int* cscColPtr, int* cscRowInd, DataType valType, Action copyValues,
+        IndexBase idxBase, Csr2CscAlg alg, void* buffer)
+
+    # Build-time version
+    int CUSPARSE_VERSION
+
+ctypedef Status (*f_type)(...) nogil  # NOQA
+IF 11010 <= CUPY_CUDA_VERSION < 12000:
+    if _sys.platform == 'linux':
+        _libname = 'libcusparse.so.11'
+    else:
+        _libname = 'cusparse64_11.dll'
+ELIF 12000 <= CUPY_CUDA_VERSION < 13000:
+    if _sys.platform == 'linux':
+        _libname = 'libcusparse.so.12'
+    else:
+        _libname = 'cusparse64_12.dll'
+ELIF 0 < CUPY_HIP_VERSION:
+    _libname = __file__
+ELSE:
+    _libname = None
+
+cdef SoftLink _lib = SoftLink(_libname, 'cusparse')
+# cuSPARSE 11.6+ (CUDA 11.3.1+)
+cdef f_type cusparseSpSM_createDescr = <f_type>_lib.get('SpSM_createDescr')
+cdef f_type cusparseSpSM_destroyDescr = <f_type>_lib.get('SpSM_destroyDescr')
+cdef f_type cusparseSpSM_bufferSize = <f_type>_lib.get('SpSM_bufferSize')
+cdef f_type cusparseSpSM_analysis = <f_type>_lib.get('SpSM_analysis')
+cdef f_type cusparseSpSM_solve = <f_type>_lib.get('SpSM_solve')
+# cuSPARSE 11.5+ (CUDA 11.3.0+)
+cdef f_type cusparseSpMatSetAttribute = <f_type>_lib.get('SpMatSetAttribute')
+# cuSPARSE 11.3.1+ (CUDA 11.2.0+)
+cdef f_type cusparseCreateCsc = <f_type>_lib.get('CreateCsc')
+# cuSPARSE 11.3+ (CUDA 11.1.1+)
+# Note: CUDA 11.1.0 contains cuSPARSE 11.2.0.275
+cdef f_type cusparseSparseToDense_bufferSize = <f_type>_lib.get('SparseToDense_bufferSize')  # NOQA
+cdef f_type cusparseSparseToDense = <f_type>_lib.get('SparseToDense')
+cdef f_type cusparseDenseToSparse_bufferSize = <f_type>_lib.get('DenseToSparse_bufferSize')  # NOQA
+cdef f_type cusparseDenseToSparse_analysis = <f_type>_lib.get('DenseToSparse_analysis')  # NOQA
+cdef f_type cusparseDenseToSparse_convert = <f_type>_lib.get('DenseToSparse_convert')  # NOQA
+
+cdef dict HIP_STATUS = {
+    0: b'HIPSPARSE_STATUS_SUCCESS',
+    1: b'HIPSPARSE_STATUS_NOT_INITIALIZED',
+    2: b'HIPSPARSE_STATUS_ALLOC_FAILED',
+    3: b'HIPSPARSE_STATUS_INVALID_VALUE',
+    4: b'HIPSPARSE_STATUS_ARCH_MISMATCH',
+    5: b'HIPSPARSE_STATUS_MAPPING_ERROR',
+    6: b'HIPSPARSE_STATUS_EXECUTION_FAILED',
+    7: b'HIPSPARSE_STATUS_INTERNAL_ERROR',
+    8: b'HIPSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED',
+    9: b'HIPSPARSE_STATUS_ZERO_PIVOT',
+    10: b'HIPSPARSE_STATUS_NOT_SUPPORTED',
+    11: b'HIPSPARSE_STATUS_INSUFFICIENT_RESOURCES',
+}
+
+
+cdef class SpVecAttributes:
+
+    def __init__(self, int64_t size, int64_t nnz,
+                 intptr_t idx, intptr_t values,
+                 IndexType idxType, IndexBase idxBase, DataType valueType):
+        self.size = size
+        self.nnz = nnz
+        self.idx = idx
+        self.values = values
+        self.idxType = idxType
+        self.idxBase = idxBase
+        self.valueType = valueType
+
+
+cdef class CooAttributes:
+
+    def __init__(self, int64_t rows, int64_t cols, int64_t nnz,
+                 intptr_t rowIdx, intptr_t colIdx, intptr_t values,
+                 IndexType idxType, IndexBase idxBase, DataType valueType):
+        self.rows = rows
+        self.cols = cols
+        self.nnz = nnz
+        self.rowIdx = rowIdx
+        self.colIdx = colIdx
+        self.values = values
+        self.idxType = idxType
+        self.idxBase = idxBase
+        self.valueType = valueType
+
+
+cdef class CooAoSAttributes:
+
+    def __init__(self, int64_t rows, int64_t cols, int64_t nnz,
+                 intptr_t ind, intptr_t values,
+                 IndexType idxType, IndexBase idxBase, DataType valueType):
+        self.rows = rows
+        self.cols = cols
+        self.nnz = nnz
+        self.ind = ind
+        self.values = values
+        self.idxType = idxType
+        self.idxBase = idxBase
+        self.valueType = valueType
+
+
+cdef class CsrAttributes:
+
+    def __init__(self, int64_t rows, int64_t cols, int64_t nnz,
+                 intptr_t rowOffsets, intptr_t colIdx, intptr_t values,
+                 IndexType rowOffsetType, IndexType colIdxType,
+                 IndexBase idxBase, DataType valueType):
+        self.rows = rows
+        self.cols = cols
+        self.nnz = nnz
+        self.rowOffsets = rowOffsets
+        self.colIdx = colIdx
+        self.values = values
+        self.rowOffsetType = rowOffsetType
+        self.colIdxType = colIdxType
+        self.idxBase = idxBase
+        self.valueType = valueType
+
+
+cdef class DnVecAttributes:
+
+    def __init__(self, int64_t size, intptr_t values, DataType valueType):
+        self.size = size
+        self.values = values
+        self.valueType = valueType
+
+
+cdef class DnMatAttributes:
+
+    def __init__(self, int64_t rows, int64_t cols, int64_t ld,
+                 intptr_t values, DataType valueType, Order order):
+        self.rows = rows
+        self.cols = cols
+        self.ld = ld
+        self.values = values
+        self.valueType = valueType
+        self.order = order
+
+
+cdef class DnMatBatchAttributes:
+
+    def __init__(self, int count, int64_t stride):
+        self.count = count
+        self.stride = stride
+
+
+class CuSparseError(RuntimeError):
+
+    def __init__(self, Status status):
+        self.status = status
+        cdef bytes name
+        cdef bytes msg
+        if _is_hip_environment:
+            name = HIP_STATUS[status]
+            msg = name
+        else:
+            name = cusparseGetErrorName(status)
+            msg = cusparseGetErrorString(status)
+        super().__init__(f'{name.decode()}: {msg.decode()}')
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        raise CuSparseError(status)
+
+
+@cython.profile(False)
+cdef inline cuComplex complex_to_cuda(complex value):
+    cdef cuComplex value_cuda
+    value_cuda.x = value.real
+    value_cuda.y = value.imag
+    return value_cuda
+
+
+@cython.profile(False)
+cdef inline cuDoubleComplex double_complex_to_cuda(double complex value):
+    cdef cuDoubleComplex value_cuda
+    value_cuda.x = value.real
+    value_cuda.y = value.imag
+    return value_cuda
+
+
+cpdef int getVersion(intptr_t handle) except? -1:
+    cdef int version
+    status = cusparseGetVersion(<Handle>handle, &version)
+    check_status(status)
+    return version
+
+
+def get_build_version():
+    return CUSPARSE_VERSION
+
+
+########################################
+# cuSPARSE Helper Function
+
+cpdef intptr_t create() except? 0:
+    cdef Handle handle
+    status = cusparseCreate(& handle)
+    check_status(status)
+    return <intptr_t>handle
+
+
+cpdef size_t createMatDescr() except? -1:
+    cdef MatDescr desc
+    status = cusparseCreateMatDescr(& desc)
+    check_status(status)
+    return <size_t>desc
+
+
+cpdef void destroy(intptr_t handle) except *:
+    status = cusparseDestroy(<Handle >handle)
+    check_status(status)
+
+
+cpdef void destroyMatDescr(size_t descr) except *:
+    status = cusparseDestroyMatDescr(<MatDescr>descr)
+    check_status(status)
+
+
+cpdef void setMatIndexBase(size_t descr, base) except *:
+    status = cusparseSetMatIndexBase(<MatDescr>descr, base)
+    check_status(status)
+
+
+cpdef void setMatType(size_t descr, typ) except *:
+    status = cusparseSetMatType(<MatDescr>descr, typ)
+    check_status(status)
+
+cpdef void setMatFillMode(size_t descrA, int fillMode) except *:
+    status = cusparseSetMatFillMode(<MatDescr>descrA, <FillMode>fillMode)
+    check_status(status)
+
+cpdef void setMatDiagType(size_t descrA, int diagType) except *:
+    status = cusparseSetMatDiagType(<MatDescr>descrA, <DiagType>diagType)
+    check_status(status)
+
+cpdef void setPointerMode(intptr_t handle, int mode) except *:
+    status = cusparseSetPointerMode(<Handle>handle, <PointerMode>mode)
+    check_status(status)
+
+cpdef void spMatSetAttribute(
+        size_t spMatDescr, int attribute, int data) except *:
+    # Assuming the value of attribute is an enum value, whose underlying type
+    # As for CUDA 11.7, the types of all the sparse matrix descriptor
+    # attributes are enums, whose underlying type is always int in C.
+    status = cusparseSpMatSetAttribute(
+        <SpMatDescr>spMatDescr, <SpMatAttribute>attribute, <void*>&data,
+        sizeof(int))
+    check_status(status)
+
+
+########################################
+# Stream
+
+cpdef void setStream(intptr_t handle, size_t stream) except *:
+    # TODO(leofang): It seems most of cuSPARSE APIs support stream capture (as
+    # of CUDA 11.5) under certain conditions, see
+    # https://docs.nvidia.com/cuda/cusparse/index.html#optimization-notes
+    # Before we come up with a robust strategy to test the support conditions,
+    # we disable this functionality.
+    if not runtime._is_hip_environment and runtime.streamIsCapturing(stream):
+        raise NotImplementedError(
+            'calling cuSPARSE API during stream capture is currently '
+            'unsupported')
+
+    status = cusparseSetStream(<Handle>handle, <Stream>stream)
+    check_status(status)
+
+
+cpdef size_t getStream(intptr_t handle) except? -1:
+    cdef Stream stream
+    status = cusparseGetStream(<Handle>handle, &stream)
+    check_status(status)
+    return <size_t>stream
+
+
+cdef void _setStream(intptr_t handle) except *:
+    """Set current stream"""
+    setStream(handle, stream_module.get_current_stream_ptr())
+
+
+########################################
+# cuSPARSE Level1 Function
+
+cpdef void sgthr(
+        intptr_t handle, int nnz, size_t y, size_t xVal, size_t xInd,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseSgthr(
+        <Handle>handle, nnz, <const float *>y, <float *>xVal,
+        <const int *>xInd, <IndexBase>idxBase)
+    check_status(status)
+
+cpdef void dgthr(
+        intptr_t handle, int nnz, size_t y, size_t xVal, size_t xInd,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseDgthr(
+        <Handle>handle, nnz, <const double *>y, <double *>xVal,
+        <const int *>xInd, <IndexBase>idxBase)
+    check_status(status)
+
+cpdef void cgthr(
+        intptr_t handle, int nnz, size_t y, size_t xVal, size_t xInd,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseCgthr(
+        <Handle>handle, nnz, <const cuComplex *>y, <cuComplex *>xVal,
+        <const int *>xInd, <IndexBase>idxBase)
+    check_status(status)
+
+cpdef void zgthr(
+        intptr_t handle, int nnz, size_t y, size_t xVal, size_t xInd,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseZgthr(
+        <Handle>handle, nnz, <const cuDoubleComplex *>y,
+        <cuDoubleComplex *>xVal, <const int *>xInd, <IndexBase>idxBase)
+    check_status(status)
+
+########################################
+# cuSPARSE Level2 Function
+
+cpdef void scsrmv(
+        intptr_t handle, int transA, int m, int n, int nnz,
+        size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA,
+        size_t x, size_t beta, size_t y) except *:
+    _setStream(handle)
+    status = cusparseScsrmv(
+        <Handle>handle, <Operation>transA, m, n, nnz,
+        <const float *>alpha, <MatDescr>descrA, <const float *>csrSortedValA,
+        <const int *>csrSortedRowPtrA, <const int *>csrSortedColIndA,
+        <const float *>x, <const float *>beta, <float *>y)
+    check_status(status)
+
+cpdef void dcsrmv(
+        intptr_t handle, int transA, int m, int n, int nnz,
+        size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA,
+        size_t x, size_t beta, size_t y) except *:
+    _setStream(handle)
+    status = cusparseDcsrmv(
+        <Handle>handle, <Operation>transA, m, n, nnz,
+        <const double *>alpha, <MatDescr>descrA, <const double *>csrSortedValA,
+        <const int *>csrSortedRowPtrA, <const int *>csrSortedColIndA,
+        <const double *>x, <const double *>beta, <double *>y)
+    check_status(status)
+
+cpdef void ccsrmv(
+        intptr_t handle, int transA, int m, int n, int nnz,
+        size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA,
+        size_t x, size_t beta, size_t y) except *:
+    _setStream(handle)
+    status = cusparseCcsrmv(
+        <Handle>handle, <Operation>transA, m, n, nnz,
+        <const cuComplex *>alpha, <MatDescr>descrA,
+        <const cuComplex *>csrSortedValA,
+        <const int *>csrSortedRowPtrA, <const int *>csrSortedColIndA,
+        <const cuComplex *>x, <const cuComplex *>beta, <cuComplex *>y)
+    check_status(status)
+
+cpdef void zcsrmv(
+        intptr_t handle, int transA, int m, int n, int nnz,
+        size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA,
+        size_t x, size_t beta, size_t y) except *:
+    _setStream(handle)
+    status = cusparseZcsrmv(
+        <Handle>handle, <Operation>transA, m, n, nnz,
+        <const cuDoubleComplex *>alpha, <MatDescr>descrA,
+        <const cuDoubleComplex *>csrSortedValA,
+        <const int *>csrSortedRowPtrA, <const int *>csrSortedColIndA,
+        <const cuDoubleComplex *>x, <const cuDoubleComplex *>beta,
+        <cuDoubleComplex *>y)
+    check_status(status)
+
+cpdef size_t csrmvEx_bufferSize(
+        intptr_t handle, int alg, int transA, int m, int n,
+        int nnz, size_t alpha, int alphatype, size_t descrA,
+        size_t csrValA, int csrValAtype, size_t csrRowPtrA,
+        size_t csrColIndA, size_t x, int xtype, size_t beta,
+        int betatype, size_t y, int ytype, int executiontype) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    status = cusparseCsrmvEx_bufferSize(
+        <Handle>handle, <AlgMode>alg, <Operation>transA, m,
+        n, nnz, <const void *>alpha, <DataType>alphatype,
+        <MatDescr>descrA, <const void *>csrValA, <DataType>csrValAtype,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const void *>x, <DataType>xtype, <const void *>beta,
+        <DataType>betatype, <void *>y, <DataType>ytype,
+        <DataType>executiontype, &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef void csrmvEx(
+        intptr_t handle, int alg, int transA, int m, int n,
+        int nnz, size_t alpha, int alphatype, size_t descrA,
+        size_t csrValA, int csrValAtype, size_t csrRowPtrA,
+        size_t csrColIndA, size_t x, int xtype, size_t beta,
+        int betatype, size_t y, int ytype, int executiontype,
+        size_t buffer) except *:
+    _setStream(handle)
+    status = cusparseCsrmvEx(
+        <Handle>handle, <AlgMode>alg, <Operation>transA, m,
+        n, nnz, <const void *>alpha, <DataType>alphatype,
+        <MatDescr>descrA, <const void *>csrValA, <DataType>csrValAtype,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const void *>x, <DataType>xtype, <const void *>beta,
+        <DataType>betatype, <void *>y, <DataType>ytype,
+        <DataType>executiontype, <void *>buffer)
+    check_status(status)
+
+cpdef size_t createCsrsv2Info() except? -1:
+    cdef csrsv2Info_t info
+    status = cusparseCreateCsrsv2Info(&info)
+    check_status(status)
+    return <size_t>info
+
+cpdef void destroyCsrsv2Info(size_t info) except *:
+    status = cusparseDestroyCsrsv2Info(<csrsv2Info_t>info)
+    check_status(status)
+
+cpdef int scsrsv2_bufferSize(
+        intptr_t handle, int transA, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int bufferSize
+    _setStream(handle)
+    status = cusparseScsrsv2_bufferSize(
+        <Handle>handle, <Operation>transA, m, nnz, <const MatDescr>descrA,
+        <float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef int dcsrsv2_bufferSize(
+        intptr_t handle, int transA, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int bufferSize
+    _setStream(handle)
+    status = cusparseDcsrsv2_bufferSize(
+        <Handle>handle, <Operation>transA, m, nnz, <const MatDescr>descrA,
+        <double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef int ccsrsv2_bufferSize(
+        intptr_t handle, int transA, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int bufferSize
+    _setStream(handle)
+    status = cusparseCcsrsv2_bufferSize(
+        <Handle>handle, <Operation>transA, m, nnz, <const MatDescr>descrA,
+        <cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef int zcsrsv2_bufferSize(
+        intptr_t handle, int transA, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int bufferSize
+    _setStream(handle)
+    status = cusparseZcsrsv2_bufferSize(
+        <Handle>handle, <Operation>transA, m, nnz, <const MatDescr>descrA,
+        <cuDoubleComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void scsrsv2_analysis(
+        intptr_t handle, int transA, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseScsrsv2_analysis(
+        <Handle>handle, <Operation>transA, m, nnz, <const MatDescr>descrA,
+        <const float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info,
+        <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsrsv2_analysis(
+        intptr_t handle, int transA, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseDcsrsv2_analysis(
+        <Handle>handle, <Operation>transA, m, nnz, <const MatDescr>descrA,
+        <const double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info,
+        <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsrsv2_analysis(
+        intptr_t handle, int transA, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseCcsrsv2_analysis(
+        <Handle>handle, <Operation>transA, m, nnz, <const MatDescr>descrA,
+        <const cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info,
+        <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsrsv2_analysis(
+        intptr_t handle, int transA, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseZcsrsv2_analysis(
+        <Handle>handle, <Operation>transA, m, nnz, <const MatDescr>descrA,
+        <const cuDoubleComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info,
+        <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void scsrsv2_solve(
+        intptr_t handle, int transA, int m, int nnz, size_t alpha,
+        size_t descrA, size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, size_t x, size_t y, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseScsrsv2_solve(
+        <Handle>handle, <Operation>transA, m, nnz,
+        <const float*>alpha, <const MatDescr>descrA,
+        <const float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info,
+        <const float*>x, <float*>y,
+        <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsrsv2_solve(
+        intptr_t handle, int transA, int m, int nnz, size_t alpha,
+        size_t descrA, size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, size_t x, size_t y, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseDcsrsv2_solve(
+        <Handle>handle, <Operation>transA, m, nnz,
+        <const double*>alpha, <const MatDescr>descrA,
+        <const double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info,
+        <const double*>x, <double*>y,
+        <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsrsv2_solve(
+        intptr_t handle, int transA, int m, int nnz, size_t alpha,
+        size_t descrA, size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, size_t x, size_t y, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseCcsrsv2_solve(
+        <Handle>handle, <Operation>transA, m, nnz,
+        <const cuComplex*>alpha, <const MatDescr>descrA,
+        <const cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info,
+        <const cuComplex*>x, <cuComplex*>y,
+        <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsrsv2_solve(
+        intptr_t handle, int transA, int m, int nnz, size_t alpha,
+        size_t descrA, size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, size_t x, size_t y, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseZcsrsv2_solve(
+        <Handle>handle, <Operation>transA, m, nnz,
+        <const cuDoubleComplex*>alpha, <const MatDescr>descrA,
+        <const cuDoubleComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <csrsv2Info_t>info,
+        <const cuDoubleComplex*>x, <cuDoubleComplex*>y,
+        <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void xcsrsv2_zeroPivot(
+        intptr_t handle, size_t info, size_t position) except *:
+    _setStream(handle)
+    status = cusparseXcsrsv2_zeroPivot(
+        <Handle>handle, <csrsv2Info_t>info, <int*>position)
+    check_status(status)
+
+########################################
+# cuSPARSE Level3 Function
+
+cpdef void scsrmm(
+        intptr_t handle, int transA, int m, int n, int k, int nnz,
+        size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA,
+        size_t B, int ldb, size_t beta, size_t C, int ldc) except *:
+    _setStream(handle)
+    status = cusparseScsrmm(
+        <Handle>handle, <Operation>transA, m, n, k, nnz,
+        <const float *>alpha, <MatDescr>descrA, <const float *>csrSortedValA,
+        <const int *>csrSortedRowPtrA, <const int *>csrSortedColIndA,
+        <const float *>B, ldb, <const float *>beta, <float *>C, ldc)
+    check_status(status)
+
+cpdef void dcsrmm(
+        intptr_t handle, int transA, int m, int n, int k, int nnz,
+        size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA,
+        size_t B, int ldb, size_t beta, size_t C, int ldc) except *:
+    _setStream(handle)
+    status = cusparseDcsrmm(
+        <Handle>handle, <Operation>transA, m, n, k, nnz,
+        <const double *>alpha, <MatDescr>descrA, <const double *>csrSortedValA,
+        <const int *>csrSortedRowPtrA, <const int *>csrSortedColIndA,
+        <const double *>B, ldb, <const double *>beta, <double *>C, ldc)
+    check_status(status)
+
+cpdef void ccsrmm(
+        intptr_t handle, int transA, int m, int n, int k, int nnz,
+        size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA,
+        size_t B, int ldb, size_t beta, size_t C, int ldc) except *:
+    _setStream(handle)
+    status = cusparseCcsrmm(
+        <Handle>handle, <Operation>transA, m, n, k, nnz,
+        <const cuComplex *>alpha, <MatDescr>descrA,
+        <const cuComplex *>csrSortedValA,
+        <const int *>csrSortedRowPtrA, <const int *>csrSortedColIndA,
+        <const cuComplex *>B, ldb, <const cuComplex *>beta,
+        <cuComplex *>C, ldc)
+    check_status(status)
+
+cpdef void zcsrmm(
+        intptr_t handle, int transA, int m, int n, int k, int nnz,
+        size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA,
+        size_t B, int ldb, size_t beta, size_t C, int ldc) except *:
+    _setStream(handle)
+    status = cusparseZcsrmm(
+        <Handle>handle, <Operation>transA, m, n, k, nnz,
+        <const cuDoubleComplex *>alpha, <MatDescr>descrA,
+        <const cuDoubleComplex *>csrSortedValA,
+        <const int *>csrSortedRowPtrA, <const int *>csrSortedColIndA,
+        <const cuDoubleComplex *>B, ldb,
+        <const cuDoubleComplex *>beta, <cuDoubleComplex *>C, ldc)
+    check_status(status)
+
+cpdef void scsrmm2(
+        intptr_t handle, int transA, int transB, int m, int n, int k, int nnz,
+        size_t alpha, size_t descrA, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA,
+        size_t B, int ldb, size_t beta, size_t C, int ldc) except *:
+    _setStream(handle)
+    status = cusparseScsrmm2(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k, nnz,
+        <const float *>alpha, <MatDescr>descrA, <const float *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const float *>B, ldb, <const float *>beta, <float *>C, ldc)
+    check_status(status)
+
+cpdef void dcsrmm2(
+        intptr_t handle, int transA, int transB, int m, int n, int k, int nnz,
+        size_t alpha, size_t descrA, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA,
+        size_t B, int ldb, size_t beta, size_t C, int ldc) except *:
+    _setStream(handle)
+    status = cusparseDcsrmm2(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k, nnz,
+        <const double *>alpha, <MatDescr>descrA, <const double *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const double *>B, ldb, <const double *>beta, <double *>C, ldc)
+    check_status(status)
+
+cpdef void ccsrmm2(
+        intptr_t handle, int transA, int transB, int m, int n, int k, int nnz,
+        size_t alpha, size_t descrA, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA,
+        size_t B, int ldb, size_t beta, size_t C, int ldc) except *:
+    _setStream(handle)
+    status = cusparseCcsrmm2(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k, nnz,
+        <const cuComplex *>alpha, <MatDescr>descrA, <const cuComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const cuComplex *>B, ldb, <const cuComplex *>beta,
+        <cuComplex *>C, ldc)
+    check_status(status)
+
+cpdef void zcsrmm2(
+        intptr_t handle, int transA, int transB, int m, int n, int k, int nnz,
+        size_t alpha, size_t descrA, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA,
+        size_t B, int ldb, size_t beta, size_t C, int ldc) except *:
+    _setStream(handle)
+    status = cusparseZcsrmm2(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k, nnz,
+        <const cuDoubleComplex *>alpha, <MatDescr>descrA,
+        <const cuDoubleComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const cuDoubleComplex *>B, ldb,
+        <const cuDoubleComplex *>beta, <cuDoubleComplex *>C, ldc)
+    check_status(status)
+
+cpdef size_t createCsrsm2Info() except? -1:
+    cdef csrsm2Info_t info
+    status = cusparseCreateCsrsm2Info(&info)
+    check_status(status)
+    return <size_t>info
+
+cpdef void destroyCsrsm2Info(size_t info) except *:
+    status = cusparseDestroyCsrsm2Info(<csrsm2Info_t>info)
+    check_status(status)
+
+cpdef size_t scsrsm2_bufferSizeExt(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy) except? -1:
+    cdef size_t bufferSize
+    _setStream(handle)
+    status = cusparseScsrsm2_bufferSizeExt(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const float*>alpha, <const MatDescr>descrA,
+        <const float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <float*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t dcsrsm2_bufferSizeExt(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy) except? -1:
+    cdef size_t bufferSize
+    _setStream(handle)
+    status = cusparseDcsrsm2_bufferSizeExt(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const double*>alpha, <const MatDescr>descrA,
+        <const double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <double*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t ccsrsm2_bufferSizeExt(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy) except? -1:
+    cdef size_t bufferSize
+    _setStream(handle)
+    status = cusparseCcsrsm2_bufferSizeExt(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const cuComplex*>alpha, <const MatDescr>descrA,
+        <const cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <cuComplex*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t zcsrsm2_bufferSizeExt(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy) except? -1:
+    cdef size_t bufferSize
+    _setStream(handle)
+    status = cusparseZcsrsm2_bufferSizeExt(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const cuDoubleComplex*>alpha, <const MatDescr>descrA,
+        <const cuDoubleComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <cuDoubleComplex*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void scsrsm2_analysis(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseScsrsm2_analysis(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const float*>alpha, <const MatDescr>descrA,
+        <const float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <float*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsrsm2_analysis(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseDcsrsm2_analysis(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const double*>alpha, <const MatDescr>descrA,
+        <const double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <double*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsrsm2_analysis(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseCcsrsm2_analysis(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const cuComplex*>alpha, <const MatDescr>descrA,
+        <const cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <cuComplex*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsrsm2_analysis(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseZcsrsm2_analysis(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const cuDoubleComplex*>alpha, <const MatDescr>descrA,
+        <const cuDoubleComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <cuDoubleComplex*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void scsrsm2_solve(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseScsrsm2_solve(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const float*>alpha, <const MatDescr>descrA,
+        <const float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <float*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsrsm2_solve(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseDcsrsm2_solve(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const double*>alpha, <const MatDescr>descrA,
+        <const double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <double*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsrsm2_solve(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseCcsrsm2_solve(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const cuComplex*>alpha, <const MatDescr>descrA,
+        <const cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <cuComplex*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsrsm2_solve(
+        intptr_t handle, int algo, int transA, int transB, int m, int nrhs,
+        int nnz, size_t alpha, size_t descrA, size_t csrSortedValA,
+        size_t csrSortedRowPtrA, size_t csrSortedColIndA, size_t B, int ldb,
+        size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseZcsrsm2_solve(
+        <Handle>handle, algo, <Operation>transA, <Operation>transB, m, nrhs,
+        nnz, <const cuDoubleComplex*>alpha, <const MatDescr>descrA,
+        <const cuDoubleComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+        <const int*>csrSortedColIndA, <cuDoubleComplex*>B, ldb,
+        <csrsm2Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void xcsrsm2_zeroPivot(
+        intptr_t handle, size_t info, size_t position) except *:
+    _setStream(handle)
+    status = cusparseXcsrsm2_zeroPivot(
+        <Handle>handle, <csrsm2Info_t>info, <int*>position)
+    check_status(status)
+
+########################################
+# cuSPARSE Extra Function
+
+cpdef void xcsrgeamNnz(
+        intptr_t handle, int m, int n, size_t descrA, int nnzA,
+        size_t csrRowPtrA, size_t csrColIndA, size_t descrB,
+        int nnzB, size_t csrRowPtrB, size_t csrColIndB,
+        size_t descrC, size_t csrRowPtrC, size_t nnzTotalDevHostPtr) except *:
+    _setStream(handle)
+    status = cusparseXcsrgeamNnz(
+        <Handle>handle, m, n, <const MatDescr>descrA, nnzA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const MatDescr>descrB, nnzB, <const int *>csrRowPtrB,
+        <const int *>csrColIndB, <const MatDescr>descrC, <int *>csrRowPtrC,
+        <int *>nnzTotalDevHostPtr)
+    check_status(status)
+
+cpdef void scsrgeam(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except *:
+    _setStream(handle)
+    status = cusparseScsrgeam(
+        <Handle>handle, m, n, <const float *>alpha,
+        <const MatDescr>descrA, nnzA, <const float *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA, <const float *>beta,
+        <const MatDescr>descrB, nnzB, <const float *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <float *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC)
+    check_status(status)
+
+cpdef void dcsrgeam(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except *:
+    _setStream(handle)
+    status = cusparseDcsrgeam(
+        <Handle>handle, m, n, <const double *>alpha,
+        <const MatDescr>descrA, nnzA, <const double *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA, <const double *>beta,
+        <const MatDescr>descrB, nnzB, <const double *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <double *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC)
+    check_status(status)
+
+cpdef void ccsrgeam(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except *:
+    _setStream(handle)
+    status = cusparseCcsrgeam(
+        <Handle>handle, m, n, <const cuComplex *>alpha,
+        <const MatDescr>descrA, nnzA, <const cuComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const cuComplex *>beta,
+        <const MatDescr>descrB, nnzB, <const cuComplex *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <cuComplex *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC)
+    check_status(status)
+
+cpdef void zcsrgeam(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except *:
+    _setStream(handle)
+    status = cusparseZcsrgeam(
+        <Handle>handle, m, n, <const cuDoubleComplex *>alpha,
+        <const MatDescr>descrA, nnzA, <const cuDoubleComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const cuDoubleComplex *>beta,
+        <const MatDescr>descrB, nnzB, <const cuDoubleComplex *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <cuDoubleComplex *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC)
+    check_status(status)
+
+cpdef size_t scsrgeam2_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except? -1:
+    cdef size_t bufferSize
+    _setStream(handle)
+    status = cusparseScsrgeam2_bufferSizeExt(
+        <Handle>handle, m, n, <const float *>alpha,
+        <const MatDescr>descrA, nnzA, <const float *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA, <const float *>beta,
+        <const MatDescr>descrB, nnzB, <const float *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <float *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t dcsrgeam2_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except? -1:
+    cdef size_t bufferSize
+    _setStream(handle)
+    status = cusparseDcsrgeam2_bufferSizeExt(
+        <Handle>handle, m, n, <const double *>alpha,
+        <const MatDescr>descrA, nnzA, <const double *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA, <const double *>beta,
+        <const MatDescr>descrB, nnzB, <const double *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <double *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t ccsrgeam2_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except? -1:
+    cdef size_t bufferSize
+    _setStream(handle)
+    status = cusparseCcsrgeam2_bufferSizeExt(
+        <Handle>handle, m, n, <const cuComplex *>alpha,
+        <const MatDescr>descrA, nnzA, <const cuComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const cuComplex *>beta,
+        <const MatDescr>descrB, nnzB, <const cuComplex *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <cuComplex *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t zcsrgeam2_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except? -1:
+    cdef size_t bufferSize
+    _setStream(handle)
+    status = cusparseZcsrgeam2_bufferSizeExt(
+        <Handle>handle, m, n, <const cuDoubleComplex *>alpha,
+        <const MatDescr>descrA, nnzA, <const cuDoubleComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const cuDoubleComplex *>beta,
+        <const MatDescr>descrB, nnzB, <const cuDoubleComplex *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <cuDoubleComplex *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void xcsrgeam2Nnz(
+        intptr_t handle, int m, int n, size_t descrA, int nnzA,
+        size_t csrRowPtrA, size_t csrColIndA, size_t descrB,
+        int nnzB, size_t csrRowPtrB, size_t csrColIndB,
+        size_t descrC, size_t csrRowPtrC, size_t nnzTotalDevHostPtr,
+        size_t workspace) except *:
+    _setStream(handle)
+    status = cusparseXcsrgeam2Nnz(
+        <Handle>handle, m, n, <const MatDescr>descrA, nnzA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const MatDescr>descrB, nnzB, <const int *>csrRowPtrB,
+        <const int *>csrColIndB, <const MatDescr>descrC, <int *>csrRowPtrC,
+        <int *>nnzTotalDevHostPtr, <void*> workspace)
+    check_status(status)
+
+cpdef void scsrgeam2(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC, size_t buffer) except *:
+    _setStream(handle)
+    status = cusparseScsrgeam2(
+        <Handle>handle, m, n, <const float *>alpha,
+        <const MatDescr>descrA, nnzA, <const float *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA, <const float *>beta,
+        <const MatDescr>descrB, nnzB, <const float *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <float *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC, <void*>buffer)
+    check_status(status)
+
+cpdef void dcsrgeam2(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC, size_t buffer) except *:
+    _setStream(handle)
+    status = cusparseDcsrgeam2(
+        <Handle>handle, m, n, <const double *>alpha,
+        <const MatDescr>descrA, nnzA, <const double *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA, <const double *>beta,
+        <const MatDescr>descrB, nnzB, <const double *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <double *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC, <void*>buffer)
+    check_status(status)
+
+cpdef void ccsrgeam2(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC, size_t buffer) except *:
+    _setStream(handle)
+    status = cusparseCcsrgeam2(
+        <Handle>handle, m, n, <const cuComplex *>alpha,
+        <const MatDescr>descrA, nnzA, <const cuComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const cuComplex *>beta,
+        <const MatDescr>descrB, nnzB, <const cuComplex *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <cuComplex *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC, <void*>buffer)
+    check_status(status)
+
+cpdef void zcsrgeam2(
+        intptr_t handle, int m, int n, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t beta, size_t descrB,
+        int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC, size_t buffer) except *:
+    _setStream(handle)
+    status = cusparseZcsrgeam2(
+        <Handle>handle, m, n, <const cuDoubleComplex *>alpha,
+        <const MatDescr>descrA, nnzA, <const cuDoubleComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const cuDoubleComplex *>beta,
+        <const MatDescr>descrB, nnzB, <const cuDoubleComplex *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <cuDoubleComplex *>csrValC, <int *>csrRowPtrC,
+        <int *>csrColIndC, <void*>buffer)
+    check_status(status)
+
+cpdef void xcsrgemmNnz(
+        intptr_t handle, int transA, int transB, int m, int n, int k,
+        size_t descrA, int nnzA, size_t csrRowPtrA,
+        size_t csrColIndA, size_t descrB, int nnzB,
+        size_t csrRowPtrB, size_t csrColIndB,
+        size_t descrC, size_t csrRowPtrC, size_t nnzTotalDevHostPtr) except *:
+    _setStream(handle)
+    status = cusparseXcsrgemmNnz(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k,
+        <const MatDescr>descrA, nnzA, <const int *>csrRowPtrA,
+        <const int *>csrColIndA, <const MatDescr>descrB, nnzB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <int *>csrRowPtrC, <int *>nnzTotalDevHostPtr)
+    check_status(status)
+
+cpdef void scsrgemm(
+        intptr_t handle, int transA, int transB, int m, int n, int k,
+        size_t descrA, const int nnzA, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA, size_t descrB,
+        const int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except *:
+    _setStream(handle)
+    status = cusparseScsrgemm(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k,
+        <const MatDescr>descrA, nnzA, <const float *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const MatDescr>descrB, nnzB, <const float *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <float *>csrValC,
+        <const int *>csrRowPtrC, <int *>csrColIndC)
+    check_status(status)
+
+cpdef void dcsrgemm(
+        intptr_t handle, int transA, int transB, int m, int n, int k,
+        size_t descrA, const int nnzA, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA, size_t descrB,
+        const int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except *:
+    _setStream(handle)
+    status = cusparseDcsrgemm(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k,
+        <const MatDescr>descrA, nnzA, <const double *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const MatDescr>descrB, nnzB, <const double *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <double *>csrValC,
+        <const int *>csrRowPtrC, <int *>csrColIndC)
+    check_status(status)
+
+cpdef void ccsrgemm(
+        intptr_t handle, int transA, int transB, int m, int n, int k,
+        size_t descrA, const int nnzA, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA, size_t descrB,
+        const int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except *:
+    _setStream(handle)
+    status = cusparseCcsrgemm(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k,
+        <const MatDescr>descrA, nnzA, <const cuComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const MatDescr>descrB, nnzB, <const cuComplex *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <cuComplex *>csrValC,
+        <const int *>csrRowPtrC, <int *>csrColIndC)
+    check_status(status)
+
+cpdef void zcsrgemm(
+        intptr_t handle, int transA, int transB, int m, int n, int k,
+        size_t descrA, const int nnzA, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA, size_t descrB,
+        const int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t descrC, size_t csrValC,
+        size_t csrRowPtrC, size_t csrColIndC) except *:
+    _setStream(handle)
+    status = cusparseZcsrgemm(
+        <Handle>handle, <Operation>transA, <Operation>transB, m, n, k,
+        <const MatDescr>descrA, nnzA, <const cuDoubleComplex *>csrValA,
+        <const int *>csrRowPtrA, <const int *>csrColIndA,
+        <const MatDescr>descrB, nnzB, <const cuDoubleComplex *>csrValB,
+        <const int *>csrRowPtrB, <const int *>csrColIndB,
+        <const MatDescr>descrC, <cuDoubleComplex *>csrValC,
+        <const int *>csrRowPtrC, <int *>csrColIndC)
+    check_status(status)
+
+cpdef size_t createCsrgemm2Info() except? -1:
+    cdef csrgemm2Info_t info
+    with nogil:
+        status = cusparseCreateCsrgemm2Info(&info)
+    check_status(status)
+    return <size_t>info
+
+cpdef void destroyCsrgemm2Info(size_t info) except *:
+    with nogil:
+        status = cusparseDestroyCsrgemm2Info(<csrgemm2Info_t>info)
+    check_status(status)
+
+cpdef size_t scsrgemm2_bufferSizeExt(
+        intptr_t handle, int m, int n, int k,
+        size_t alpha,
+        size_t descrA, int nnzA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrRowPtrB, size_t csrColIndB,
+        size_t beta,
+        size_t descrD, int nnzD, size_t csrRowPtrD, size_t csrColIndD,
+        size_t info) except? -1:
+    cdef size_t bufferSize
+    status = cusparseScsrgemm2_bufferSizeExt(
+        <Handle>handle, m, n, k,
+        <float*>alpha,
+        <MatDescr>descrA, nnzA, <int*>csrRowPtrA, <int*>csrColIndA,
+        <MatDescr>descrB, nnzB, <int*>csrRowPtrB, <int*>csrColIndB,
+        <float*>beta,
+        <MatDescr>descrD, nnzD, <int*>csrRowPtrD, <int*>csrColIndD,
+        <csrgemm2Info_t>info, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t dcsrgemm2_bufferSizeExt(
+        intptr_t handle, int m, int n, int k,
+        size_t alpha,
+        size_t descrA, int nnzA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrRowPtrB, size_t csrColIndB,
+        size_t beta,
+        size_t descrD, int nnzD, size_t csrRowPtrD, size_t csrColIndD,
+        size_t info) except? -1:
+    cdef size_t bufferSize
+    status = cusparseDcsrgemm2_bufferSizeExt(
+        <Handle>handle, m, n, k,
+        <double*>alpha,
+        <MatDescr>descrA, nnzA, <int*>csrRowPtrA, <int*>csrColIndA,
+        <MatDescr>descrB, nnzB, <int*>csrRowPtrB, <int*>csrColIndB,
+        <double*>beta,
+        <MatDescr>descrD, nnzD, <int*>csrRowPtrD, <int*>csrColIndD,
+        <csrgemm2Info_t>info, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t ccsrgemm2_bufferSizeExt(
+        intptr_t handle, int m, int n, int k,
+        size_t alpha,
+        size_t descrA, int nnzA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrRowPtrB, size_t csrColIndB,
+        size_t beta,
+        size_t descrD, int nnzD, size_t csrRowPtrD, size_t csrColIndD,
+        size_t info) except? -1:
+    cdef size_t bufferSize
+    status = cusparseCcsrgemm2_bufferSizeExt(
+        <Handle>handle, m, n, k,
+        <cuComplex*>alpha,
+        <MatDescr>descrA, nnzA, <int*>csrRowPtrA, <int*>csrColIndA,
+        <MatDescr>descrB, nnzB, <int*>csrRowPtrB, <int*>csrColIndB,
+        <cuComplex*>beta,
+        <MatDescr>descrD, nnzD, <int*>csrRowPtrD, <int*>csrColIndD,
+        <csrgemm2Info_t>info, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef size_t zcsrgemm2_bufferSizeExt(
+        intptr_t handle, int m, int n, int k,
+        size_t alpha,
+        size_t descrA, int nnzA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrRowPtrB, size_t csrColIndB,
+        size_t beta,
+        size_t descrD, int nnzD, size_t csrRowPtrD, size_t csrColIndD,
+        size_t info) except? -1:
+    cdef size_t bufferSize
+    status = cusparseZcsrgemm2_bufferSizeExt(
+        <Handle>handle, m, n, k,
+        <cuDoubleComplex*>alpha,
+        <MatDescr>descrA, nnzA, <int*>csrRowPtrA, <int*>csrColIndA,
+        <MatDescr>descrB, nnzB, <int*>csrRowPtrB, <int*>csrColIndB,
+        <cuDoubleComplex*>beta,
+        <MatDescr>descrD, nnzD, <int*>csrRowPtrD, <int*>csrColIndD,
+        <csrgemm2Info_t>info, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void xcsrgemm2Nnz(
+        intptr_t handle, int m, int n, int k,
+        size_t descrA, int nnzA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrRowPtrB, size_t csrColIndB,
+        size_t descrD, int nnzD, size_t csrRowPtrD, size_t csrColIndD,
+        size_t descrC, size_t csrRowPtrC,
+        intptr_t nnzTotalDevHostPtr, size_t info, intptr_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseXcsrgemm2Nnz(
+        <Handle>handle, m, n, k,
+        <MatDescr>descrA, nnzA, <int*>csrRowPtrA, <int*>csrColIndA,
+        <MatDescr>descrB, nnzB, <int*>csrRowPtrB, <int*>csrColIndB,
+        <MatDescr>descrD, nnzD, <int*>csrRowPtrD, <int*>csrColIndD,
+        <MatDescr>descrC, <int*>csrRowPtrC,
+        <int*>nnzTotalDevHostPtr, <csrgemm2Info_t>info, <void*>pBuffer)
+    check_status(status)
+
+cpdef void scsrgemm2(
+        intptr_t handle, int m, int n, int k, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t beta, size_t descrD, int nnzD,
+        size_t csrValD, size_t csrRowPtrD, size_t csrColIndD, size_t descrC,
+        size_t csrValC, size_t csrRowPtrC, size_t csrColIndC, size_t info,
+        intptr_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseScsrgemm2(
+        <Handle>handle, m, n, k, <float*>alpha, <MatDescr>descrA, nnzA,
+        <float*>csrValA, <int*>csrRowPtrA, <int*>csrColIndA, <MatDescr>descrB,
+        nnzB, <float*>csrValB, <int*>csrRowPtrB, <int*>csrColIndB,
+        <float*>beta, <MatDescr>descrD, nnzD, <float*>csrValD,
+        <int*>csrRowPtrD, <int*>csrColIndD, <MatDescr>descrC, <float*>csrValC,
+        <int*>csrRowPtrC, <int*>csrColIndC, <csrgemm2Info_t>info,
+        <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsrgemm2(
+        intptr_t handle, int m, int n, int k, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t beta, size_t descrD, int nnzD,
+        size_t csrValD, size_t csrRowPtrD, size_t csrColIndD, size_t descrC,
+        size_t csrValC, size_t csrRowPtrC, size_t csrColIndC, size_t info,
+        intptr_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseDcsrgemm2(
+        <Handle>handle, m, n, k, <double*>alpha, <MatDescr>descrA, nnzA,
+        <double*>csrValA, <int*>csrRowPtrA, <int*>csrColIndA, <MatDescr>descrB,
+        nnzB, <double*>csrValB, <int*>csrRowPtrB, <int*>csrColIndB,
+        <double*>beta, <MatDescr>descrD, nnzD, <double*>csrValD,
+        <int*>csrRowPtrD, <int*>csrColIndD, <MatDescr>descrC, <double*>csrValC,
+        <int*>csrRowPtrC, <int*>csrColIndC, <csrgemm2Info_t>info,
+        <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsrgemm2(
+        intptr_t handle, int m, int n, int k, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t beta, size_t descrD, int nnzD,
+        size_t csrValD, size_t csrRowPtrD, size_t csrColIndD, size_t descrC,
+        size_t csrValC, size_t csrRowPtrC, size_t csrColIndC, size_t info,
+        intptr_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseCcsrgemm2(
+        <Handle>handle, m, n, k, <cuComplex*>alpha, <MatDescr>descrA, nnzA,
+        <cuComplex*>csrValA, <int*>csrRowPtrA, <int*>csrColIndA,
+        <MatDescr>descrB, nnzB, <cuComplex*>csrValB, <int*>csrRowPtrB,
+        <int*>csrColIndB, <cuComplex*>beta, <MatDescr>descrD, nnzD,
+        <cuComplex*>csrValD, <int*>csrRowPtrD, <int*>csrColIndD,
+        <MatDescr>descrC, <cuComplex*>csrValC, <int*>csrRowPtrC,
+        <int*>csrColIndC, <csrgemm2Info_t>info, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsrgemm2(
+        intptr_t handle, int m, int n, int k, size_t alpha, size_t descrA,
+        int nnzA, size_t csrValA, size_t csrRowPtrA, size_t csrColIndA,
+        size_t descrB, int nnzB, size_t csrValB, size_t csrRowPtrB,
+        size_t csrColIndB, size_t beta, size_t descrD, int nnzD,
+        size_t csrValD, size_t csrRowPtrD, size_t csrColIndD, size_t descrC,
+        size_t csrValC, size_t csrRowPtrC, size_t csrColIndC, size_t info,
+        intptr_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseZcsrgemm2(
+        <Handle>handle, m, n, k, <cuDoubleComplex*>alpha, <MatDescr>descrA,
+        nnzA, <cuDoubleComplex*>csrValA, <int*>csrRowPtrA, <int*>csrColIndA,
+        <MatDescr>descrB, nnzB, <cuDoubleComplex*>csrValB, <int*>csrRowPtrB,
+        <int*>csrColIndB, <cuDoubleComplex*>beta, <MatDescr>descrD, nnzD,
+        <cuDoubleComplex*>csrValD, <int*>csrRowPtrD, <int*>csrColIndD,
+        <MatDescr>descrC, <cuDoubleComplex*>csrValC, <int*>csrRowPtrC,
+        <int*>csrColIndC, <csrgemm2Info_t>info, <void*>pBuffer)
+    check_status(status)
+
+########################################
+# cuSPARSE Format Convrsion
+
+cpdef void xcoo2csr(
+        intptr_t handle, size_t cooRowInd, int nnz, int m, size_t csrRowPtr,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseXcoo2csr(
+        <Handle>handle, <const int *>cooRowInd, nnz, m, <int *>csrRowPtr,
+        <IndexBase>idxBase)
+    check_status(status)
+
+
+cpdef void scsc2dense(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t cscSortedValA, size_t cscSortedRowIndA,
+        size_t cscSortedColPtrA, size_t A, int lda) except *:
+    _setStream(handle)
+    status = cusparseScsc2dense(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const float *>cscSortedValA, <const int *>cscSortedRowIndA,
+        <const int *>cscSortedColPtrA, <float *>A, lda)
+    check_status(status)
+
+
+cpdef void dcsc2dense(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t cscSortedValA, size_t cscSortedRowIndA,
+        size_t cscSortedColPtrA, size_t A, int lda) except *:
+    _setStream(handle)
+    status = cusparseDcsc2dense(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const double *>cscSortedValA, <const int *>cscSortedRowIndA,
+        <const int *>cscSortedColPtrA, <double *>A, lda)
+    check_status(status)
+
+cpdef void ccsc2dense(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t cscSortedValA, size_t cscSortedRowIndA,
+        size_t cscSortedColPtrA, size_t A, int lda) except *:
+    _setStream(handle)
+    status = cusparseCcsc2dense(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const cuComplex *>cscSortedValA, <const int *>cscSortedRowIndA,
+        <const int *>cscSortedColPtrA, <cuComplex *>A, lda)
+    check_status(status)
+
+cpdef void zcsc2dense(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t cscSortedValA, size_t cscSortedRowIndA,
+        size_t cscSortedColPtrA, size_t A, int lda) except *:
+    _setStream(handle)
+    status = cusparseZcsc2dense(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const cuDoubleComplex *>cscSortedValA, <const int *>cscSortedRowIndA,
+        <const int *>cscSortedColPtrA, <cuDoubleComplex *>A, lda)
+    check_status(status)
+
+cpdef void xcsr2coo(
+        intptr_t handle, size_t csrRowPtr, int nnz, int m, size_t cooRowInd,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseXcsr2coo(
+        <Handle>handle, <const int *>csrRowPtr, nnz, m, <int *>cooRowInd,
+        <IndexBase>idxBase)
+    check_status(status)
+
+
+cpdef void scsr2csc(
+        intptr_t handle, int m, int n, int nnz, size_t csrVal,
+        size_t csrRowPtr, size_t csrColInd, size_t cscVal,
+        size_t cscRowInd, size_t cscColPtr, int copyValues,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseScsr2csc(
+        <Handle>handle, m, n, nnz, <const float *>csrVal,
+        <const int *>csrRowPtr, <const int *>csrColInd, <float *>cscVal,
+        <int *>cscRowInd, <int *>cscColPtr, <Action>copyValues,
+        <IndexBase>idxBase)
+    check_status(status)
+
+
+cpdef void dcsr2csc(
+        intptr_t handle, int m, int n, int nnz, size_t csrVal,
+        size_t csrRowPtr, size_t csrColInd, size_t cscVal,
+        size_t cscRowInd, size_t cscColPtr, int copyValues,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseDcsr2csc(
+        <Handle>handle, m, n, nnz, <const double *>csrVal,
+        <const int *>csrRowPtr, <const int *>csrColInd, <double *>cscVal,
+        <int *>cscRowInd, <int *>cscColPtr, <Action>copyValues,
+        <IndexBase>idxBase)
+    check_status(status)
+
+cpdef void ccsr2csc(
+        intptr_t handle, int m, int n, int nnz, size_t csrVal,
+        size_t csrRowPtr, size_t csrColInd, size_t cscVal,
+        size_t cscRowInd, size_t cscColPtr, int copyValues,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseCcsr2csc(
+        <Handle>handle, m, n, nnz, <const cuComplex *>csrVal,
+        <const int *>csrRowPtr, <const int *>csrColInd, <cuComplex *>cscVal,
+        <int *>cscRowInd, <int *>cscColPtr, <Action>copyValues,
+        <IndexBase>idxBase)
+    check_status(status)
+
+cpdef void zcsr2csc(
+        intptr_t handle, int m, int n, int nnz, size_t csrVal,
+        size_t csrRowPtr, size_t csrColInd, size_t cscVal,
+        size_t cscRowInd, size_t cscColPtr, int copyValues,
+        int idxBase) except *:
+    _setStream(handle)
+    status = cusparseZcsr2csc(
+        <Handle>handle, m, n, nnz, <const cuDoubleComplex *>csrVal,
+        <const int *>csrRowPtr, <const int *>csrColInd,
+        <cuDoubleComplex *>cscVal,
+        <int *>cscRowInd, <int *>cscColPtr, <Action>copyValues,
+        <IndexBase>idxBase)
+    check_status(status)
+
+cpdef void scsr2dense(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t A, int lda) except *:
+    _setStream(handle)
+    status = cusparseScsr2dense(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const float *>csrSortedValA, <const int *>csrSortedRowPtrA,
+        <const int *>csrSortedColIndA, <float *>A, lda)
+    check_status(status)
+
+cpdef void dcsr2dense(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t A, int lda) except *:
+    _setStream(handle)
+    status = cusparseDcsr2dense(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const double *>csrSortedValA, <const int *>csrSortedRowPtrA,
+        <const int *>csrSortedColIndA, <double *>A, lda)
+    check_status(status)
+
+cpdef void ccsr2dense(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t A, int lda) except *:
+    _setStream(handle)
+    status = cusparseCcsr2dense(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const cuComplex *>csrSortedValA, <const int *>csrSortedRowPtrA,
+        <const int *>csrSortedColIndA, <cuComplex *>A, lda)
+    check_status(status)
+
+cpdef void zcsr2dense(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t A, int lda) except *:
+    _setStream(handle)
+    status = cusparseZcsr2dense(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const cuDoubleComplex *>csrSortedValA, <const int *>csrSortedRowPtrA,
+        <const int *>csrSortedColIndA, <cuDoubleComplex *>A, lda)
+    check_status(status)
+
+cpdef int snnz_compress(
+        intptr_t handle, int m, size_t descr,
+        size_t values, size_t rowPtr, size_t nnzPerRow,
+        float tol) except? -1:
+    cdef int nnz_total
+    _setStream(handle)
+    status = cusparseSnnz_compress(
+        <Handle>handle, m, <const MatDescr>descr,
+        <const float *>values, <const int *>rowPtr, <int *>nnzPerRow,
+        &nnz_total, tol)
+    check_status(status)
+    return nnz_total
+
+cpdef int dnnz_compress(
+        intptr_t handle, int m, size_t descr,
+        size_t values, size_t rowPtr, size_t nnzPerRow,
+        double tol) except? -1:
+    cdef int nnz_total
+    _setStream(handle)
+    status = cusparseDnnz_compress(
+        <Handle>handle, m, <const MatDescr>descr,
+        <const double *>values, <const int *>rowPtr, <int *>nnzPerRow,
+        &nnz_total, tol)
+    check_status(status)
+    return nnz_total
+
+cpdef int cnnz_compress(
+        intptr_t handle, int m, size_t descr,
+        size_t values, size_t rowPtr, size_t nnzPerRow,
+        complex tol) except? -1:
+    cdef int nnz_total
+    _setStream(handle)
+    status = cusparseCnnz_compress(
+        <Handle>handle, m, <const MatDescr>descr,
+        <const cuComplex *>values, <const int *>rowPtr, <int *>nnzPerRow,
+        &nnz_total, complex_to_cuda(tol))
+    check_status(status)
+    return nnz_total
+
+cpdef int znnz_compress(
+        intptr_t handle, int m, size_t descr,
+        size_t values, size_t rowPtr, size_t nnzPerRow,
+        double complex tol) except? -1:
+    cdef int nnz_total
+    _setStream(handle)
+    status = cusparseZnnz_compress(
+        <Handle>handle, m, <const MatDescr>descr,
+        <const cuDoubleComplex *>values, <const int *>rowPtr, <int *>nnzPerRow,
+        &nnz_total, double_complex_to_cuda(tol))
+    check_status(status)
+    return nnz_total
+
+cpdef void scsr2csr_compress(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t inVal, size_t inColInd, size_t inRowPtr,
+        int inNnz, size_t nnzPerRow, size_t outVal, size_t outColInd,
+        size_t outRowPtr, float tol) except *:
+    _setStream(handle)
+    status = cusparseScsr2csr_compress(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const float *>inVal, <const int *>inColInd, <const int *>inRowPtr,
+        inNnz, <int *>nnzPerRow, <float *>outVal, <int *>outColInd,
+        <int *>outRowPtr, tol)
+    check_status(status)
+
+
+cpdef void dcsr2csr_compress(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t inVal, size_t inColInd, size_t inRowPtr,
+        int inNnz, size_t nnzPerRow, size_t outVal, size_t outColInd,
+        size_t outRowPtr, float tol) except *:
+    _setStream(handle)
+    status = cusparseDcsr2csr_compress(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const double *>inVal, <const int *>inColInd, <const int *>inRowPtr,
+        inNnz, <int *>nnzPerRow, <double *>outVal, <int *>outColInd,
+        <int *>outRowPtr, tol)
+    check_status(status)
+
+cpdef void ccsr2csr_compress(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t inVal, size_t inColInd, size_t inRowPtr,
+        int inNnz, size_t nnzPerRow, size_t outVal, size_t outColInd,
+        size_t outRowPtr, complex tol) except *:
+    _setStream(handle)
+    status = cusparseCcsr2csr_compress(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const cuComplex *>inVal, <const int *>inColInd, <const int *>inRowPtr,
+        inNnz, <int *>nnzPerRow, <cuComplex *>outVal, <int *>outColInd,
+        <int *>outRowPtr, complex_to_cuda(tol))
+    check_status(status)
+
+cpdef void zcsr2csr_compress(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t inVal, size_t inColInd, size_t inRowPtr,
+        int inNnz, size_t nnzPerRow, size_t outVal, size_t outColInd,
+        size_t outRowPtr, double complex tol) except *:
+    _setStream(handle)
+    status = cusparseZcsr2csr_compress(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const cuDoubleComplex *>inVal, <const int *>inColInd,
+        <const int *>inRowPtr,
+        inNnz, <int *>nnzPerRow, <cuDoubleComplex *>outVal, <int *>outColInd,
+        <int *>outRowPtr, double_complex_to_cuda(tol))
+    check_status(status)
+
+cpdef void sdense2csc(
+        intptr_t handle, int m, int n, size_t descrA, size_t A,
+        int lda, size_t nnzPerCol, size_t cscValA, size_t cscRowIndA,
+        size_t cscColPtrA) except *:
+    _setStream(handle)
+    status = cusparseSdense2csc(
+        <Handle>handle, m, n, <const MatDescr>descrA, <const float *>A,
+        lda, <const int *>nnzPerCol, <float *>cscValA, <int *>cscRowIndA,
+        <int *>cscColPtrA)
+    check_status(status)
+
+
+cpdef void ddense2csc(
+        intptr_t handle, int m, int n, size_t descrA, size_t A,
+        int lda, size_t nnzPerCol, size_t cscValA, size_t cscRowIndA,
+        size_t cscColPtrA) except *:
+    _setStream(handle)
+    status = cusparseDdense2csc(
+        <Handle>handle, m, n, <const MatDescr>descrA, <const double *>A,
+        lda, <const int *>nnzPerCol, <double *>cscValA, <int *>cscRowIndA,
+        <int *>cscColPtrA)
+    check_status(status)
+
+cpdef void cdense2csc(
+        intptr_t handle, int m, int n, size_t descrA, size_t A,
+        int lda, size_t nnzPerCol, size_t cscValA, size_t cscRowIndA,
+        size_t cscColPtrA) except *:
+    _setStream(handle)
+    status = cusparseCdense2csc(
+        <Handle>handle, m, n, <const MatDescr>descrA, <const cuComplex *>A,
+        lda, <const int *>nnzPerCol, <cuComplex *>cscValA, <int *>cscRowIndA,
+        <int *>cscColPtrA)
+    check_status(status)
+
+cpdef void zdense2csc(
+        intptr_t handle, int m, int n, size_t descrA, size_t A,
+        int lda, size_t nnzPerCol, size_t cscValA, size_t cscRowIndA,
+        size_t cscColPtrA) except *:
+    _setStream(handle)
+    status = cusparseZdense2csc(
+        <Handle>handle, m, n,
+        <const MatDescr>descrA, <const cuDoubleComplex *>A,
+        lda, <const int *>nnzPerCol,
+        <cuDoubleComplex *>cscValA, <int *>cscRowIndA,
+        <int *>cscColPtrA)
+    check_status(status)
+
+cpdef void sdense2csr(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t A, int lda, size_t nnzPerRow, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA) except *:
+    _setStream(handle)
+    status = cusparseSdense2csr(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const float *>A, lda, <const int *>nnzPerRow, <float *>csrValA,
+        <int *>csrRowPtrA, <int *>csrColIndA)
+    check_status(status)
+
+
+cpdef void ddense2csr(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t A, int lda, size_t nnzPerRow, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA) except *:
+    _setStream(handle)
+    status = cusparseDdense2csr(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const double *>A, lda, <const int *>nnzPerRow, <double *>csrValA,
+        <int *>csrRowPtrA, <int *>csrColIndA)
+    check_status(status)
+
+cpdef void cdense2csr(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t A, int lda, size_t nnzPerRow, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA) except *:
+    _setStream(handle)
+    status = cusparseCdense2csr(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const cuComplex *>A, lda, <const int *>nnzPerRow,
+        <cuComplex *>csrValA, <int *>csrRowPtrA, <int *>csrColIndA)
+    check_status(status)
+
+cpdef void zdense2csr(
+        intptr_t handle, int m, int n, size_t descrA,
+        size_t A, int lda, size_t nnzPerRow, size_t csrValA,
+        size_t csrRowPtrA, size_t csrColIndA) except *:
+    _setStream(handle)
+    status = cusparseZdense2csr(
+        <Handle>handle, m, n, <MatDescr>descrA,
+        <const cuDoubleComplex *>A, lda, <const int *>nnzPerRow,
+        <cuDoubleComplex *>csrValA, <int *>csrRowPtrA, <int *>csrColIndA)
+    check_status(status)
+
+cpdef void snnz(
+        intptr_t handle, int dirA, int m, int n, size_t descrA,
+        size_t A, int lda, size_t nnzPerRowColumn,
+        size_t nnzTotalDevHostPtr) except *:
+    _setStream(handle)
+    status = cusparseSnnz(
+        <Handle>handle, <Direction>dirA, m, n, <const MatDescr>descrA,
+        <const float *>A, lda, <int *>nnzPerRowColumn,
+        <int *>nnzTotalDevHostPtr)
+    check_status(status)
+
+
+cpdef void dnnz(
+        intptr_t handle, int dirA, int m, int n, size_t descrA,
+        size_t A, int lda, size_t nnzPerRowColumn,
+        size_t nnzTotalDevHostPtr) except *:
+    _setStream(handle)
+    status = cusparseDnnz(
+        <Handle>handle, <Direction>dirA, m, n, <const MatDescr>descrA,
+        <const double *>A, lda, <int *>nnzPerRowColumn,
+        <int *>nnzTotalDevHostPtr)
+    check_status(status)
+
+cpdef void cnnz(
+        intptr_t handle, int dirA, int m, int n, size_t descrA,
+        size_t A, int lda, size_t nnzPerRowColumn,
+        size_t nnzTotalDevHostPtr) except *:
+    _setStream(handle)
+    status = cusparseCnnz(
+        <Handle>handle, <Direction>dirA, m, n, <const MatDescr>descrA,
+        <const cuComplex *>A, lda, <int *>nnzPerRowColumn,
+        <int *>nnzTotalDevHostPtr)
+    check_status(status)
+
+cpdef void znnz(
+        intptr_t handle, int dirA, int m, int n, size_t descrA,
+        size_t A, int lda, size_t nnzPerRowColumn,
+        size_t nnzTotalDevHostPtr) except *:
+    _setStream(handle)
+    status = cusparseZnnz(
+        <Handle>handle, <Direction>dirA, m, n, <const MatDescr>descrA,
+        <const cuDoubleComplex *>A, lda, <int *>nnzPerRowColumn,
+        <int *>nnzTotalDevHostPtr)
+    check_status(status)
+
+cpdef void createIdentityPermutation(
+        intptr_t handle, int n, size_t p) except *:
+    _setStream(handle)
+    status = cusparseCreateIdentityPermutation(
+        <Handle>handle, n, <int *>p)
+    check_status(status)
+
+
+cpdef size_t xcoosort_bufferSizeExt(
+        intptr_t handle, int m, int n, int nnz, size_t cooRows,
+        size_t cooCols) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    status = cusparseXcoosort_bufferSizeExt(
+        <Handle>handle, m, n, nnz, <const int *>cooRows,
+        <const int *>cooCols, &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+
+cpdef void xcoosortByRow(
+        intptr_t handle, int m, int n, int nnz, size_t cooRows, size_t cooCols,
+        size_t P, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseXcoosortByRow(
+        <Handle>handle, m, n, nnz, <int *>cooRows, <int *>cooCols,
+        <int *>P, <void *>pBuffer)
+    check_status(status)
+
+
+cpdef void xcoosortByColumn(
+        intptr_t handle, int m, int n, int nnz, size_t cooRows, size_t cooCols,
+        size_t P, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseXcoosortByColumn(
+        <Handle>handle, m, n, nnz, <int *>cooRows, <int *>cooCols,
+        <int *>P, <void *>pBuffer)
+    check_status(status)
+
+
+cpdef size_t xcsrsort_bufferSizeExt(
+        intptr_t handle, int m, int n, int nnz, size_t csrRowPtr,
+        size_t csrColInd) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    status = cusparseXcsrsort_bufferSizeExt(
+        <Handle>handle, m, n, nnz, <const int *>csrRowPtr,
+        <const int *>csrColInd, &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+
+cpdef void xcsrsort(
+        intptr_t handle, int m, int n, int nnz, size_t descrA,
+        size_t csrRowPtr, size_t csrColInd, size_t P, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseXcsrsort(
+        <Handle>handle, m, n, nnz, <const MatDescr>descrA,
+        <const int *>csrRowPtr, <int *>csrColInd, <int *>P, <void *>pBuffer)
+    check_status(status)
+
+
+cpdef size_t xcscsort_bufferSizeExt(
+        intptr_t handle, int m, int n, int nnz, size_t cscColPtr,
+        size_t cscRowInd) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    status = cusparseXcscsort_bufferSizeExt(
+        <Handle>handle, m, n, nnz, <const int *>cscColPtr,
+        <const int *>cscRowInd, &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+
+cpdef void xcscsort(
+        intptr_t handle, int m, int n, int nnz, size_t descrA,
+        size_t cscColPtr, size_t cscRowInd, size_t P, size_t pBuffer) except *:
+    _setStream(handle)
+    status = cusparseXcscsort(
+        <Handle>handle, m, n, nnz, <const MatDescr>descrA,
+        <const int *>cscColPtr, <int *>cscRowInd, <int *>P, <void *>pBuffer)
+    check_status(status)
+
+########################################
+# cuSPARSE PRECONDITIONERS
+
+cpdef size_t createCsrilu02Info() except? -1:
+    cdef csrilu02Info_t info
+    with nogil:
+        status = cusparseCreateCsrilu02Info(&info)
+    check_status(status)
+    return <size_t>info
+
+cpdef void destroyCsrilu02Info(size_t info) except *:
+    with nogil:
+        status = cusparseDestroyCsrilu02Info(<csrilu02Info_t>info)
+    check_status(status)
+
+cpdef size_t createBsrilu02Info() except? -1:
+    cdef bsrilu02Info_t info
+    with nogil:
+        status = cusparseCreateBsrilu02Info(&info)
+    check_status(status)
+    return <size_t>info
+
+cpdef void destroyBsrilu02Info(size_t info) except *:
+    with nogil:
+        status = cusparseDestroyBsrilu02Info(<bsrilu02Info_t>info)
+    check_status(status)
+
+cpdef size_t createCsric02Info() except? -1:
+    cdef csric02Info_t info
+    with nogil:
+        status = cusparseCreateCsric02Info(&info)
+    check_status(status)
+    return <size_t>info
+
+cpdef void destroyCsric02Info(size_t info) except *:
+    with nogil:
+        status = cusparseDestroyCsric02Info(<csric02Info_t>info)
+    check_status(status)
+
+cpdef size_t createBsric02Info() except? -1:
+    cdef bsric02Info_t info
+    with nogil:
+        status = cusparseCreateBsric02Info(&info)
+    check_status(status)
+    return <size_t>info
+
+cpdef void destroyBsric02Info(size_t info) except *:
+    with nogil:
+        status = cusparseDestroyBsric02Info(<bsric02Info_t>info)
+    check_status(status)
+
+cpdef void scsrilu02_numericBoost(
+        intptr_t handle, size_t info, int enable_boost,
+        size_t tol, size_t boost_val) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseScsrilu02_numericBoost(
+            <cusparseHandle_t>handle, <csrilu02Info_t>info, enable_boost,
+            <double*>tol, <float*>boost_val)
+    check_status(status)
+
+cpdef void dcsrilu02_numericBoost(
+        intptr_t handle, size_t info, int enable_boost,
+        size_t tol, size_t boost_val) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDcsrilu02_numericBoost(
+            <cusparseHandle_t>handle, <csrilu02Info_t>info, enable_boost,
+            <double*>tol, <double*>boost_val)
+    check_status(status)
+
+cpdef void ccsrilu02_numericBoost(
+        intptr_t handle, size_t info, int enable_boost,
+        size_t tol, size_t boost_val) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCcsrilu02_numericBoost(
+            <cusparseHandle_t>handle, <csrilu02Info_t>info, enable_boost,
+            <double*>tol, <cuComplex*>boost_val)
+    check_status(status)
+
+cpdef void zcsrilu02_numericBoost(
+        intptr_t handle, size_t info, int enable_boost,
+        size_t tol, size_t boost_val) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZcsrilu02_numericBoost(
+            <cusparseHandle_t>handle, <csrilu02Info_t>info, enable_boost,
+            <double*>tol, <cuDoubleComplex*>boost_val)
+    check_status(status)
+
+cpdef void xcsrilu02_zeroPivot(
+        intptr_t handle, size_t info, size_t position) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseXcsrilu02_zeroPivot(
+            <cusparseHandle_t>handle, <csrilu02Info_t>info, <int*>position)
+    check_status(status)
+
+cpdef int scsrilu02_bufferSize(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseScsrilu02_bufferSize(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int dcsrilu02_bufferSize(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDcsrilu02_bufferSize(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int ccsrilu02_bufferSize(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCcsrilu02_bufferSize(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int zcsrilu02_bufferSize(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZcsrilu02_bufferSize(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <cuDoubleComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef void scsrilu02_analysis(intptr_t handle, int m, int nnz, size_t descrA,
+                              size_t csrSortedValA, size_t csrSortedRowPtrA,
+                              size_t csrSortedColIndA, size_t info, int policy,
+                              size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseScsrilu02_analysis(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <const float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsrilu02_analysis(intptr_t handle, int m, int nnz, size_t descrA,
+                              size_t csrSortedValA, size_t csrSortedRowPtrA,
+                              size_t csrSortedColIndA, size_t info, int policy,
+                              size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDcsrilu02_analysis(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <const double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsrilu02_analysis(intptr_t handle, int m, int nnz, size_t descrA,
+                              size_t csrSortedValA, size_t csrSortedRowPtrA,
+                              size_t csrSortedColIndA, size_t info, int policy,
+                              size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCcsrilu02_analysis(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <const cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsrilu02_analysis(intptr_t handle, int m, int nnz, size_t descrA,
+                              size_t csrSortedValA, size_t csrSortedRowPtrA,
+                              size_t csrSortedColIndA, size_t info, int policy,
+                              size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZcsrilu02_analysis(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <const cuDoubleComplex*>csrSortedValA,
+            <const int*>csrSortedRowPtrA, <const int*>csrSortedColIndA,
+            <csrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void scsrilu02(intptr_t handle, int m, int nnz, size_t descrA,
+                     size_t csrSortedValA_valM, size_t csrSortedRowPtrA,
+                     size_t csrSortedColIndA, size_t info, int policy,
+                     size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseScsrilu02(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <float*>csrSortedValA_valM, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsrilu02(intptr_t handle, int m, int nnz, size_t descrA,
+                     size_t csrSortedValA_valM, size_t csrSortedRowPtrA,
+                     size_t csrSortedColIndA, size_t info, int policy,
+                     size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDcsrilu02(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <double*>csrSortedValA_valM, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsrilu02(intptr_t handle, int m, int nnz, size_t descrA,
+                     size_t csrSortedValA_valM, size_t csrSortedRowPtrA,
+                     size_t csrSortedColIndA, size_t info, int policy,
+                     size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCcsrilu02(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <cuComplex*>csrSortedValA_valM, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsrilu02(intptr_t handle, int m, int nnz, size_t descrA,
+                     size_t csrSortedValA_valM, size_t csrSortedRowPtrA,
+                     size_t csrSortedColIndA, size_t info, int policy,
+                     size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZcsrilu02(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <cuDoubleComplex*>csrSortedValA_valM, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csrilu02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void sbsrilu02_numericBoost(
+        intptr_t handle, size_t info, int enable_boost,
+        size_t tol, size_t boost_val) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSbsrilu02_numericBoost(
+            <cusparseHandle_t>handle, <bsrilu02Info_t>info, enable_boost,
+            <double*>tol, <float*>boost_val)
+    check_status(status)
+
+cpdef void dbsrilu02_numericBoost(
+        intptr_t handle, size_t info, int enable_boost,
+        size_t tol, size_t boost_val) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDbsrilu02_numericBoost(
+            <cusparseHandle_t>handle, <bsrilu02Info_t>info, enable_boost,
+            <double*>tol, <double*>boost_val)
+    check_status(status)
+
+cpdef void cbsrilu02_numericBoost(
+        intptr_t handle, size_t info, int enable_boost,
+        size_t tol, size_t boost_val) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCbsrilu02_numericBoost(
+            <cusparseHandle_t>handle, <bsrilu02Info_t>info, enable_boost,
+            <double*>tol, <cuComplex*>boost_val)
+    check_status(status)
+
+cpdef void zbsrilu02_numericBoost(
+        intptr_t handle, size_t info, int enable_boost,
+        size_t tol, size_t boost_val) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZbsrilu02_numericBoost(
+            <cusparseHandle_t>handle, <bsrilu02Info_t>info, enable_boost,
+            <double*>tol, <cuDoubleComplex*>boost_val)
+    check_status(status)
+
+cpdef void xbsrilu02_zeroPivot(
+        intptr_t handle, size_t info, size_t position) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseXbsrilu02_zeroPivot(
+            <cusparseHandle_t>handle, <bsrilu02Info_t>info, <int*>position)
+    check_status(status)
+
+cpdef int sbsrilu02_bufferSize(intptr_t handle, int dirA, int mb, int nnzb,
+                               size_t descrA, size_t bsrSortedVal,
+                               size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+                               int blockDim, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseSbsrilu02_bufferSize(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <float*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int dbsrilu02_bufferSize(intptr_t handle, int dirA, int mb, int nnzb,
+                               size_t descrA, size_t bsrSortedVal,
+                               size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+                               int blockDim, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDbsrilu02_bufferSize(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <double*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int cbsrilu02_bufferSize(intptr_t handle, int dirA, int mb, int nnzb,
+                               size_t descrA, size_t bsrSortedVal,
+                               size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+                               int blockDim, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCbsrilu02_bufferSize(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int zbsrilu02_bufferSize(intptr_t handle, int dirA, int mb, int nnzb,
+                               size_t descrA, size_t bsrSortedVal,
+                               size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+                               int blockDim, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZbsrilu02_bufferSize(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuDoubleComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef void sbsrilu02_analysis(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+        int blockDim, size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSbsrilu02_analysis(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <float*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void dbsrilu02_analysis(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+        int blockDim, size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDbsrilu02_analysis(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <double*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void cbsrilu02_analysis(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+        int blockDim, size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCbsrilu02_analysis(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void zbsrilu02_analysis(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+        int blockDim, size_t info, int policy, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZbsrilu02_analysis(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuDoubleComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void sbsrilu02(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr,
+        size_t bsrSortedColInd, int blockDim, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSbsrilu02(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <float*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void dbsrilu02(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr,
+        size_t bsrSortedColInd, int blockDim, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDbsrilu02(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <double*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void cbsrilu02(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr,
+        size_t bsrSortedColInd, int blockDim, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCbsrilu02(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void zbsrilu02(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr,
+        size_t bsrSortedColInd, int blockDim, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZbsrilu02(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuDoubleComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsrilu02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void xcsric02_zeroPivot(
+        intptr_t handle, size_t info, size_t position) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseXcsric02_zeroPivot(
+            <cusparseHandle_t>handle, <csric02Info_t>info, <int*>position)
+    check_status(status)
+
+cpdef int scsric02_bufferSize(intptr_t handle, int m, int nnz, size_t descrA,
+                              size_t csrSortedValA, size_t csrSortedRowPtrA,
+                              size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseScsric02_bufferSize(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int dcsric02_bufferSize(intptr_t handle, int m, int nnz, size_t descrA,
+                              size_t csrSortedValA, size_t csrSortedRowPtrA,
+                              size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDcsric02_bufferSize(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int ccsric02_bufferSize(intptr_t handle, int m, int nnz, size_t descrA,
+                              size_t csrSortedValA, size_t csrSortedRowPtrA,
+                              size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCcsric02_bufferSize(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int zcsric02_bufferSize(intptr_t handle, int m, int nnz, size_t descrA,
+                              size_t csrSortedValA, size_t csrSortedRowPtrA,
+                              size_t csrSortedColIndA, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZcsric02_bufferSize(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <cuDoubleComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef void scsric02_analysis(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseScsric02_analysis(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <const float*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsric02_analysis(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDcsric02_analysis(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <const double*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsric02_analysis(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCcsric02_analysis(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <const cuComplex*>csrSortedValA, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsric02_analysis(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZcsric02_analysis(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <const cuDoubleComplex*>csrSortedValA,
+            <const int*>csrSortedRowPtrA, <const int*>csrSortedColIndA,
+            <csric02Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void scsric02(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA_valM, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseScsric02(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <float*>csrSortedValA_valM, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dcsric02(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA_valM, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDcsric02(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <double*>csrSortedValA_valM, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void ccsric02(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA_valM, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCcsric02(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <cuComplex*>csrSortedValA_valM, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zcsric02(
+        intptr_t handle, int m, int nnz, size_t descrA,
+        size_t csrSortedValA_valM, size_t csrSortedRowPtrA,
+        size_t csrSortedColIndA, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZcsric02(
+            <cusparseHandle_t>handle, m, nnz, <const cusparseMatDescr_t>descrA,
+            <cuDoubleComplex*>csrSortedValA_valM, <const int*>csrSortedRowPtrA,
+            <const int*>csrSortedColIndA, <csric02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void xbsric02_zeroPivot(
+        intptr_t handle, size_t info, size_t position) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseXbsric02_zeroPivot(
+            <cusparseHandle_t>handle, <bsric02Info_t>info, <int*>position)
+    check_status(status)
+
+cpdef int sbsric02_bufferSize(intptr_t handle, int dirA, int mb, int nnzb,
+                              size_t descrA, size_t bsrSortedVal,
+                              size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+                              int blockDim, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseSbsric02_bufferSize(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <float*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int dbsric02_bufferSize(intptr_t handle, int dirA, int mb, int nnzb,
+                              size_t descrA, size_t bsrSortedVal,
+                              size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+                              int blockDim, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDbsric02_bufferSize(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <double*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int cbsric02_bufferSize(intptr_t handle, int dirA, int mb, int nnzb,
+                              size_t descrA, size_t bsrSortedVal,
+                              size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+                              int blockDim, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCbsric02_bufferSize(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef int zbsric02_bufferSize(intptr_t handle, int dirA, int mb, int nnzb,
+                              size_t descrA, size_t bsrSortedVal,
+                              size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+                              int blockDim, size_t info) except? -1:
+    cdef int pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZbsric02_bufferSize(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuDoubleComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, &pBufferSizeInBytes)
+    check_status(status)
+    return <int>pBufferSizeInBytes
+
+cpdef void sbsric02_analysis(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+        int blockDim, size_t info, int policy, size_t pInputBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSbsric02_analysis(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <const float*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pInputBuffer)
+    check_status(status)
+
+cpdef void dbsric02_analysis(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+        int blockDim, size_t info, int policy, size_t pInputBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDbsric02_analysis(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <const double*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pInputBuffer)
+    check_status(status)
+
+cpdef void cbsric02_analysis(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+        int blockDim, size_t info, int policy, size_t pInputBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCbsric02_analysis(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <const cuComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, <cusparseSolvePolicy_t>policy,
+            <void*>pInputBuffer)
+    check_status(status)
+
+cpdef void zbsric02_analysis(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr, size_t bsrSortedColInd,
+        int blockDim, size_t info, int policy, size_t pInputBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZbsric02_analysis(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA,
+            <const cuDoubleComplex*>bsrSortedVal, <const int*>bsrSortedRowPtr,
+            <const int*>bsrSortedColInd, blockDim, <bsric02Info_t>info,
+            <cusparseSolvePolicy_t>policy, <void*>pInputBuffer)
+    check_status(status)
+
+cpdef void sbsric02(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr,
+        size_t bsrSortedColInd, int blockDim, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSbsric02(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <float*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dbsric02(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr,
+        size_t bsrSortedColInd, int blockDim, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDbsric02(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <double*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void cbsric02(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr,
+        size_t bsrSortedColInd, int blockDim, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCbsric02(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zbsric02(
+        intptr_t handle, int dirA, int mb, int nnzb, size_t descrA,
+        size_t bsrSortedVal, size_t bsrSortedRowPtr,
+        size_t bsrSortedColInd, int blockDim, size_t info, int policy,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZbsric02(
+            <cusparseHandle_t>handle, <cusparseDirection_t>dirA, mb, nnzb,
+            <const cusparseMatDescr_t>descrA, <cuDoubleComplex*>bsrSortedVal,
+            <const int*>bsrSortedRowPtr, <const int*>bsrSortedColInd, blockDim,
+            <bsric02Info_t>info, <cusparseSolvePolicy_t>policy, <void*>pBuffer)
+    check_status(status)
+
+cpdef size_t sgtsv2_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t dl, size_t d,
+        size_t du, size_t B, int ldb) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgtsv2_bufferSizeExt(
+            <cusparseHandle_t>handle, m, n, <const float*>dl, <const float*>d,
+            <const float*>du, <const float*>B, ldb, &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef size_t dgtsv2_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t dl, size_t d,
+        size_t du, size_t B, int ldb) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgtsv2_bufferSizeExt(
+            <cusparseHandle_t>handle, m, n, <const double*>dl,
+            <const double*>d, <const double*>du, <const double*>B, ldb,
+            &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef size_t cgtsv2_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t dl, size_t d,
+        size_t du, size_t B, int ldb) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgtsv2_bufferSizeExt(
+            <cusparseHandle_t>handle, m, n, <const cuComplex*>dl,
+            <const cuComplex*>d, <const cuComplex*>du, <const cuComplex*>B,
+            ldb, &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef size_t zgtsv2_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t dl, size_t d,
+        size_t du, size_t B, int ldb) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgtsv2_bufferSizeExt(
+            <cusparseHandle_t>handle, m, n, <const cuDoubleComplex*>dl,
+            <const cuDoubleComplex*>d, <const cuDoubleComplex*>du,
+            <const cuDoubleComplex*>B, ldb, &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef void sgtsv2(
+        intptr_t handle, int m, int n, size_t dl, size_t d, size_t du,
+        size_t B, int ldb, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgtsv2(
+            <cusparseHandle_t>handle, m, n, <const float*>dl, <const float*>d,
+            <const float*>du, <float*>B, ldb, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dgtsv2(
+        intptr_t handle, int m, int n, size_t dl, size_t d, size_t du,
+        size_t B, int ldb, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgtsv2(<cusparseHandle_t>handle, m, n,
+                                <const double*>dl, <const double*>d,
+                                <const double*>du, <double*>B, ldb,
+                                <void*>pBuffer)
+    check_status(status)
+
+cpdef void cgtsv2(
+        intptr_t handle, int m, int n, size_t dl, size_t d, size_t du,
+        size_t B, int ldb, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgtsv2(<cusparseHandle_t>handle, m, n,
+                                <const cuComplex*>dl, <const cuComplex*>d,
+                                <const cuComplex*>du, <cuComplex*>B, ldb,
+                                <void*>pBuffer)
+    check_status(status)
+
+cpdef void zgtsv2(
+        intptr_t handle, int m, int n, size_t dl, size_t d, size_t du,
+        size_t B, int ldb, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgtsv2(
+            <cusparseHandle_t>handle, m, n, <const cuDoubleComplex*>dl,
+            <const cuDoubleComplex*>d, <const cuDoubleComplex*>du,
+            <cuDoubleComplex*>B, ldb, <void*>pBuffer)
+    check_status(status)
+
+cpdef size_t sgtsv2_nopivot_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t dl,
+        size_t d, size_t du, size_t B, int ldb) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgtsv2_nopivot_bufferSizeExt(
+            <cusparseHandle_t>handle, m, n, <const float*>dl, <const float*>d,
+            <const float*>du, <const float*>B, ldb, &pBufferSizeInBytes)
+    check_status(status)
+    return pBufferSizeInBytes
+
+cpdef size_t dgtsv2_nopivot_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t dl,
+        size_t d, size_t du, size_t B, int ldb) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgtsv2_nopivot_bufferSizeExt(
+            <cusparseHandle_t>handle, m, n, <const double*>dl,
+            <const double*>d, <const double*>du, <const double*>B, ldb,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return pBufferSizeInBytes
+
+cpdef size_t cgtsv2_nopivot_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t dl,
+        size_t d, size_t du, size_t B, int ldb) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgtsv2_nopivot_bufferSizeExt(
+            <cusparseHandle_t>handle, m, n, <const cuComplex*>dl,
+            <const cuComplex*>d, <const cuComplex*>du, <const cuComplex*>B,
+            ldb, &pBufferSizeInBytes)
+    check_status(status)
+    return pBufferSizeInBytes
+
+cpdef size_t zgtsv2_nopivot_bufferSizeExt(
+        intptr_t handle, int m, int n, size_t dl,
+        size_t d, size_t du, size_t B, int ldb) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgtsv2_nopivot_bufferSizeExt(
+            <cusparseHandle_t>handle, m, n, <const cuDoubleComplex*>dl,
+            <const cuDoubleComplex*>d, <const cuDoubleComplex*>du,
+            <const cuDoubleComplex*>B, ldb, &pBufferSizeInBytes)
+    check_status(status)
+    return pBufferSizeInBytes
+
+cpdef void sgtsv2_nopivot(
+        intptr_t handle, int m, int n, size_t dl, size_t d,
+        size_t du, size_t B, int ldb, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgtsv2_nopivot(
+            <cusparseHandle_t>handle, m, n, <const float*>dl, <const float*>d,
+            <const float*>du, <float*>B, ldb, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dgtsv2_nopivot(
+        intptr_t handle, int m, int n, size_t dl, size_t d,
+        size_t du, size_t B, int ldb, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgtsv2_nopivot(<cusparseHandle_t>handle, m, n,
+                                        <const double*>dl, <const double*>d,
+                                        <const double*>du, <double*>B, ldb,
+                                        <void*>pBuffer)
+    check_status(status)
+
+cpdef void cgtsv2_nopivot(
+        intptr_t handle, int m, int n, size_t dl, size_t d,
+        size_t du, size_t B, int ldb, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgtsv2_nopivot(
+            <cusparseHandle_t>handle, m, n, <const cuComplex*>dl,
+            <const cuComplex*>d, <const cuComplex*>du, <cuComplex*>B, ldb,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void zgtsv2_nopivot(
+        intptr_t handle, int m, int n, size_t dl, size_t d,
+        size_t du, size_t B, int ldb, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgtsv2_nopivot(
+            <cusparseHandle_t>handle, m, n, <const cuDoubleComplex*>dl,
+            <const cuDoubleComplex*>d, <const cuDoubleComplex*>du,
+            <cuDoubleComplex*>B, ldb, <void*>pBuffer)
+    check_status(status)
+
+cpdef size_t sgtsv2StridedBatch_bufferSizeExt(
+        intptr_t handle, int m, size_t dl, size_t d, size_t du, size_t x,
+        int batchCount, int batchStride) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgtsv2StridedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, m, <const float*>dl, <const float*>d,
+            <const float*>du, <const float*>x, batchCount, batchStride,
+            &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef size_t dgtsv2StridedBatch_bufferSizeExt(
+        intptr_t handle, int m, size_t dl, size_t d, size_t du, size_t x,
+        int batchCount, int batchStride):
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgtsv2StridedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, m, <const double*>dl, <const double*>d,
+            <const double*>du, <const double*>x, batchCount, batchStride,
+            &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef size_t cgtsv2StridedBatch_bufferSizeExt(
+        intptr_t handle, int m, size_t dl, size_t d, size_t du, size_t x,
+        int batchCount, int batchStride) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgtsv2StridedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, m, <const cuComplex*>dl,
+            <const cuComplex*>d, <const cuComplex*>du, <const cuComplex*>x,
+            batchCount, batchStride, &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef size_t zgtsv2StridedBatch_bufferSizeExt(
+        intptr_t handle, int m, size_t dl, size_t d, size_t du, size_t x,
+        int batchCount, int batchStride) except? -1:
+    cdef size_t bufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgtsv2StridedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, m, <const cuDoubleComplex*>dl,
+            <const cuDoubleComplex*>d, <const cuDoubleComplex*>du,
+            <const cuDoubleComplex*>x, batchCount, batchStride,
+            &bufferSizeInBytes)
+    check_status(status)
+    return bufferSizeInBytes
+
+cpdef void sgtsv2StridedBatch(
+        intptr_t handle, int m, size_t dl, size_t d,
+        size_t du, size_t x, int batchCount, int batchStride,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgtsv2StridedBatch(
+            <cusparseHandle_t>handle, m, <const float*>dl, <const float*>d,
+            <const float*>du, <float*>x, batchCount, batchStride,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void dgtsv2StridedBatch(
+        intptr_t handle, int m, size_t dl, size_t d,
+        size_t du, size_t x, int batchCount, int batchStride,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgtsv2StridedBatch(
+            <cusparseHandle_t>handle, m, <const double*>dl, <const double*>d,
+            <const double*>du, <double*>x, batchCount, batchStride,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void cgtsv2StridedBatch(
+        intptr_t handle, int m, size_t dl, size_t d,
+        size_t du, size_t x, int batchCount, int batchStride,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgtsv2StridedBatch(
+            <cusparseHandle_t>handle, m, <const cuComplex*>dl,
+            <const cuComplex*>d, <const cuComplex*>du, <cuComplex*>x,
+            batchCount, batchStride, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zgtsv2StridedBatch(
+        intptr_t handle, int m, size_t dl, size_t d,
+        size_t du, size_t x, int batchCount, int batchStride,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgtsv2StridedBatch(
+            <cusparseHandle_t>handle, m, <const cuDoubleComplex*>dl,
+            <const cuDoubleComplex*>d, <const cuDoubleComplex*>du,
+            <cuDoubleComplex*>x, batchCount, batchStride, <void*>pBuffer)
+    check_status(status)
+
+cpdef size_t sgtsvInterleavedBatch_bufferSizeExt(
+        intptr_t handle, int algo, int m, size_t dl, size_t d, size_t du,
+        size_t x, int batchCount) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgtsvInterleavedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, algo, m, <const float*>dl,
+            <const float*>d, <const float*>du, <const float*>x, batchCount,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <size_t>pBufferSizeInBytes
+
+cpdef size_t dgtsvInterleavedBatch_bufferSizeExt(
+        intptr_t handle, int algo, int m, size_t dl, size_t d, size_t du,
+        size_t x, int batchCount) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgtsvInterleavedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, algo, m, <const double*>dl,
+            <const double*>d, <const double*>du, <const double*>x, batchCount,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <size_t>pBufferSizeInBytes
+
+cpdef size_t cgtsvInterleavedBatch_bufferSizeExt(
+        intptr_t handle, int algo, int m, size_t dl, size_t d, size_t du,
+        size_t x, int batchCount) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgtsvInterleavedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, algo, m, <const cuComplex*>dl,
+            <const cuComplex*>d, <const cuComplex*>du, <const cuComplex*>x,
+            batchCount, &pBufferSizeInBytes)
+    check_status(status)
+    return <size_t>pBufferSizeInBytes
+
+cpdef size_t zgtsvInterleavedBatch_bufferSizeExt(
+        intptr_t handle, int algo, int m, size_t dl, size_t d, size_t du,
+        size_t x, int batchCount) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgtsvInterleavedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, algo, m, <const cuDoubleComplex*>dl,
+            <const cuDoubleComplex*>d, <const cuDoubleComplex*>du,
+            <const cuDoubleComplex*>x, batchCount, &pBufferSizeInBytes)
+    check_status(status)
+    return <size_t>pBufferSizeInBytes
+
+cpdef void sgtsvInterleavedBatch(
+        intptr_t handle, int algo, int m, size_t dl,
+        size_t d, size_t du, size_t x, int batchCount,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgtsvInterleavedBatch(
+            <cusparseHandle_t>handle, algo, m, <float*>dl, <float*>d,
+            <float*>du, <float*>x, batchCount, <void*>pBuffer)
+    check_status(status)
+
+cpdef void dgtsvInterleavedBatch(
+        intptr_t handle, int algo, int m, size_t dl,
+        size_t d, size_t du, size_t x, int batchCount,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgtsvInterleavedBatch(
+            <cusparseHandle_t>handle, algo, m, <double*>dl, <double*>d,
+            <double*>du, <double*>x, batchCount, <void*>pBuffer)
+    check_status(status)
+
+cpdef void cgtsvInterleavedBatch(
+        intptr_t handle, int algo, int m, size_t dl,
+        size_t d, size_t du, size_t x, int batchCount,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgtsvInterleavedBatch(
+            <cusparseHandle_t>handle, algo, m, <cuComplex*>dl, <cuComplex*>d,
+            <cuComplex*>du, <cuComplex*>x, batchCount, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zgtsvInterleavedBatch(
+        intptr_t handle, int algo, int m, size_t dl,
+        size_t d, size_t du, size_t x, int batchCount,
+        size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgtsvInterleavedBatch(
+            <cusparseHandle_t>handle, algo, m, <cuDoubleComplex*>dl,
+            <cuDoubleComplex*>d, <cuDoubleComplex*>du, <cuDoubleComplex*>x,
+            batchCount, <void*>pBuffer)
+    check_status(status)
+
+cpdef size_t sgpsvInterleavedBatch_bufferSizeExt(
+        intptr_t handle, int algo, int m, size_t ds, size_t dl, size_t d,
+        size_t du, size_t dw, size_t x, int batchCount) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgpsvInterleavedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, algo, m, <const float*>ds,
+            <const float*>dl, <const float*>d, <const float*>du,
+            <const float*>dw, <const float*>x, batchCount, &pBufferSizeInBytes)
+    check_status(status)
+    return <size_t>pBufferSizeInBytes
+
+cpdef size_t dgpsvInterleavedBatch_bufferSizeExt(
+        intptr_t handle, int algo, int m, size_t ds, size_t dl, size_t d,
+        size_t du, size_t dw, size_t x, int batchCount) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgpsvInterleavedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, algo, m, <const double*>ds,
+            <const double*>dl, <const double*>d, <const double*>du,
+            <const double*>dw, <const double*>x, batchCount,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <size_t>pBufferSizeInBytes
+
+cpdef size_t cgpsvInterleavedBatch_bufferSizeExt(
+        intptr_t handle, int algo, int m, size_t ds, size_t dl, size_t d,
+        size_t du, size_t dw, size_t x, int batchCount) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgpsvInterleavedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, algo, m, <const cuComplex*>ds,
+            <const cuComplex*>dl, <const cuComplex*>d, <const cuComplex*>du,
+            <const cuComplex*>dw, <const cuComplex*>x, batchCount,
+            &pBufferSizeInBytes)
+    check_status(status)
+    return <size_t>pBufferSizeInBytes
+
+cpdef size_t zgpsvInterleavedBatch_bufferSizeExt(
+        intptr_t handle, int algo, int m, size_t ds, size_t dl, size_t d,
+        size_t du, size_t dw, size_t x, int batchCount) except? -1:
+    cdef size_t pBufferSizeInBytes
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgpsvInterleavedBatch_bufferSizeExt(
+            <cusparseHandle_t>handle, algo, m, <const cuDoubleComplex*>ds,
+            <const cuDoubleComplex*>dl, <const cuDoubleComplex*>d,
+            <const cuDoubleComplex*>du, <const cuDoubleComplex*>dw,
+            <const cuDoubleComplex*>x, batchCount, &pBufferSizeInBytes)
+    check_status(status)
+    return <size_t>pBufferSizeInBytes
+
+cpdef void sgpsvInterleavedBatch(
+        intptr_t handle, int algo, int m, size_t ds,
+        size_t dl, size_t d, size_t du, size_t dw,
+        size_t x, int batchCount, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseSgpsvInterleavedBatch(
+            <cusparseHandle_t>handle, algo, m, <float*>ds, <float*>dl,
+            <float*>d, <float*>du, <float*>dw, <float*>x, batchCount,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void dgpsvInterleavedBatch(
+        intptr_t handle, int algo, int m, size_t ds,
+        size_t dl, size_t d, size_t du, size_t dw,
+        size_t x, int batchCount, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseDgpsvInterleavedBatch(
+            <cusparseHandle_t>handle, algo, m, <double*>ds, <double*>dl,
+            <double*>d, <double*>du, <double*>dw, <double*>x, batchCount,
+            <void*>pBuffer)
+    check_status(status)
+
+cpdef void cgpsvInterleavedBatch(
+        intptr_t handle, int algo, int m, size_t ds,
+        size_t dl, size_t d, size_t du, size_t dw,
+        size_t x, int batchCount, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseCgpsvInterleavedBatch(
+            <cusparseHandle_t>handle, algo, m, <cuComplex*>ds, <cuComplex*>dl,
+            <cuComplex*>d, <cuComplex*>du, <cuComplex*>dw, <cuComplex*>x,
+            batchCount, <void*>pBuffer)
+    check_status(status)
+
+cpdef void zgpsvInterleavedBatch(
+        intptr_t handle, int algo, int m, size_t ds,
+        size_t dl, size_t d, size_t du, size_t dw,
+        size_t x, int batchCount, size_t pBuffer) except *:
+    _setStream(handle)
+    with nogil:
+        status = cusparseZgpsvInterleavedBatch(
+            <cusparseHandle_t>handle, algo, m, <cuDoubleComplex*>ds,
+            <cuDoubleComplex*>dl, <cuDoubleComplex*>d, <cuDoubleComplex*>du,
+            <cuDoubleComplex*>dw, <cuDoubleComplex*>x, batchCount,
+            <void*>pBuffer)
+    check_status(status)
+
+############################################################
+# Sparse Vector APIs
+
+cpdef size_t createSpVec(int64_t size, int64_t nnz, intptr_t indices,
+                         intptr_t values, IndexType idxType, IndexBase idxBase,
+                         DataType valueType) except? -1:
+    cdef SpVecDescr desc
+    status = cusparseCreateSpVec(&desc, size, nnz, <void*>indices,
+                                 <void*>values, idxType, idxBase, valueType)
+    check_status(status)
+    return <size_t>desc
+
+cpdef void destroySpVec(size_t desc) except *:
+    status = cusparseDestroySpVec(<SpVecDescr>desc)
+    check_status(status)
+
+cpdef SpVecAttributes spVecGet(size_t desc):
+    cdef int64_t size, nnz
+    cdef intptr_t indices, values
+    cdef IndexType idxType
+    cdef IndexBase idxBase
+    cdef DataType valueType
+    status = cusparseSpVecGet(<SpVecDescr>desc, &size, &nnz,
+                              <void**>&indices, <void**>&values,
+                              &idxType, &idxBase, &valueType)
+    check_status(status)
+    return SpVecAttributes(size, nnz, indices, values, idxType, idxBase,
+                           valueType)
+
+cpdef int spVecGetIndexBase(size_t desc) except? -1:
+    cdef IndexBase idxBase
+    status = cusparseSpVecGetIndexBase(<SpVecDescr>desc, &idxBase)
+    check_status(status)
+    return <int>idxBase
+
+cpdef intptr_t spVecGetValues(size_t desc) except? -1:
+    cdef intptr_t values
+    status = cusparseSpVecGetValues(<SpVecDescr>desc, <void**>&values)
+    check_status(status)
+    return values
+
+cpdef void spVecSetValues(size_t desc, intptr_t values) except *:
+    status = cusparseSpVecSetValues(<SpVecDescr>desc, <void*>values)
+    check_status(status)
+
+############################################################
+# Sparse Matrix APIs
+
+cpdef size_t createCoo(int64_t rows, int64_t cols, int64_t nnz,
+                       intptr_t cooRowInd, intptr_t cooColInd,
+                       intptr_t cooValues, IndexType cooIdxType,
+                       IndexBase idxBase, DataType valueType) except? -1:
+    cdef SpMatDescr desc
+    status = cusparseCreateCoo(&desc, rows, cols, nnz, <void*>cooRowInd,
+                               <void*>cooColInd, <void*>cooValues,
+                               cooIdxType, idxBase, valueType)
+    check_status(status)
+    return <size_t>desc
+
+cpdef size_t createCooAoS(int64_t rows, int64_t cols, int64_t nnz,
+                          intptr_t cooInd, intptr_t cooValues,
+                          IndexType cooIdxType, IndexBase idxBase,
+                          DataType valueType) except? -1:
+    cdef SpMatDescr desc
+    status = cusparseCreateCooAoS(&desc, rows, cols, nnz, <void*>cooInd,
+                                  <void*>cooValues, cooIdxType, idxBase,
+                                  valueType)
+    check_status(status)
+    return <size_t>desc
+
+cpdef size_t createCsr(int64_t rows, int64_t cols, int64_t nnz,
+                       intptr_t csrRowOffsets, intptr_t csrColind,
+                       intptr_t csrValues, IndexType csrRowOffsetsType,
+                       IndexType csrColIndType, IndexBase idxBase,
+                       DataType valueType) except? -1:
+    cdef SpMatDescr desc
+    status = cusparseCreateCsr(&desc, rows, cols, nnz,
+                               <void*>csrRowOffsets, <void*>csrColind,
+                               <void*>csrValues, csrRowOffsetsType,
+                               csrColIndType, idxBase, valueType)
+    check_status(status)
+    return <size_t>desc
+
+cpdef size_t createCsc(int64_t rows, int64_t cols, int64_t nnz,
+                       intptr_t cscColOffsets, intptr_t cscRowInd,
+                       intptr_t cscValues, IndexType cscColOffsetsType,
+                       IndexType cscRowIndType, IndexBase idxBase,
+                       DataType valueType) except? -1:
+    cdef SpMatDescr desc
+    status = cusparseCreateCsc(&desc, rows, cols, nnz,
+                               <void*>cscColOffsets, <void*>cscRowInd,
+                               <void*>cscValues, cscColOffsetsType,
+                               cscRowIndType, idxBase, valueType)
+    check_status(status)
+    return <size_t>desc
+
+cpdef void destroySpMat(size_t desc) except *:
+    status = cusparseDestroySpMat(<SpMatDescr>desc)
+    check_status(status)
+
+cpdef CooAttributes cooGet(size_t desc):
+    cdef int64_t rows, cols, nnz
+    cdef intptr_t rowInd, colInd, values,
+    cdef IndexType idxType,
+    cdef IndexBase idxBase,
+    cdef DataType valueType
+    status = cusparseCooGet(<SpMatDescr>desc, &rows, &cols, &nnz,
+                            <void**>&rowInd, <void**>&colInd, <void**>&values,
+                            &idxType, &idxBase, &valueType)
+    check_status(status)
+    return CooAttributes(rows, cols, nnz, rowInd, colInd, values,
+                         idxType, idxBase, valueType)
+
+cpdef CooAoSAttributes cooAoSGet(size_t desc):
+    cdef int64_t rows, cols, nnz
+    cdef intptr_t ind, values,
+    cdef IndexType idxType,
+    cdef IndexBase idxBase,
+    cdef DataType valueType
+    status = cusparseCooAoSGet(<SpMatDescr>desc, &rows, &cols, &nnz,
+                               <void**>&ind, <void**>&values,
+                               &idxType, &idxBase, &valueType)
+    check_status(status)
+    return CooAoSAttributes(rows, cols, nnz, ind, values,
+                            idxType, idxBase, valueType)
+
+cpdef CsrAttributes csrGet(size_t desc):
+    cdef int64_t rows, cols, nnz
+    cdef intptr_t rowOffsets, colInd, values,
+    cdef IndexType rowOffsetsType, colIndType
+    cdef IndexBase idxBase
+    cdef DataType valueType
+    status = cusparseCsrGet(<SpMatDescr>desc, &rows, &cols, &nnz,
+                            <void**>&rowOffsets, <void**>&colInd,
+                            <void**>&values, &rowOffsetsType, &colIndType,
+                            &idxBase, &valueType)
+    check_status(status)
+    return CsrAttributes(rows, cols, nnz, rowOffsets, colInd, values,
+                         rowOffsetsType, colIndType, idxBase, valueType)
+
+cpdef void csrSetPointers(
+        size_t desc, size_t csrRowOffsets, size_t csrColInd,
+        size_t csrValues) except *:
+    status = cusparseCsrSetPointers(<SpMatDescr>desc, <void*>csrRowOffsets,
+                                    <void*>csrColInd, <void*>csrValues)
+    check_status(status)
+
+cpdef int spMatGetFormat(size_t desc) except? -1:
+    cdef Format format
+    status = cusparseSpMatGetFormat(<SpMatDescr>desc, &format)
+    check_status(status)
+    return <int>format
+
+cpdef int spMatGetIndexBase(size_t desc) except? -1:
+    cdef IndexBase idxBase
+    status = cusparseSpMatGetIndexBase(<SpMatDescr>desc, &idxBase)
+    check_status(status)
+    return <int>idxBase
+
+cpdef intptr_t spMatGetValues(size_t desc) except? -1:
+    cdef intptr_t values
+    status = cusparseSpMatGetValues(<SpMatDescr>desc, <void**>&values)
+    check_status(status)
+    return values
+
+cpdef void spMatSetValues(size_t desc, intptr_t values) except *:
+    status = cusparseSpMatSetValues(<SpMatDescr>desc, <void*>values)
+    check_status(status)
+
+cpdef void spMatGetSize(
+        size_t desc, size_t rows, size_t cols, size_t nnz) except *:
+    status = cusparseSpMatGetSize(<SpMatDescr>desc, <int64_t*>rows,
+                                  <int64_t*>cols, <int64_t*>nnz)
+    check_status(status)
+
+cpdef int spMatGetStridedBatch(size_t desc) except? -1:
+    cdef int batchCount
+    status = cusparseSpMatGetStridedBatch(<SpMatDescr>desc, &batchCount)
+    check_status(status)
+    return batchCount
+
+cpdef void spMatSetStridedBatch(size_t desc, int batchCount) except *:
+    status = cusparseSpMatSetStridedBatch(<SpMatDescr>desc, batchCount)
+    check_status(status)
+
+############################################################
+# Dense Vector APIs
+
+cpdef size_t createDnVec(int64_t size, intptr_t values,
+                         DataType valueType) except? -1:
+    cdef DnVecDescr desc
+    status = cusparseCreateDnVec(&desc, size, <void*>values, valueType)
+    check_status(status)
+    return <size_t>desc
+
+cpdef void destroyDnVec(size_t desc) except *:
+    status = cusparseDestroyDnVec(<DnVecDescr>desc)
+    check_status(status)
+
+cpdef DnVecAttributes dnVecGet(size_t desc):
+    cdef int64_t size
+    cdef intptr_t values
+    cdef DataType valueType
+    status = cusparseDnVecGet(<DnVecDescr>desc, &size, <void**>&values,
+                              &valueType)
+    check_status(status)
+    return DnVecAttributes(size, values, valueType)
+
+cpdef intptr_t dnVecGetValues(size_t desc) except? -1:
+    cdef intptr_t values
+    status = cusparseDnVecGetValues(<DnVecDescr>desc, <void**>&values)
+    check_status(status)
+    return values
+
+cpdef void dnVecSetValues(size_t desc, intptr_t values) except *:
+    status = cusparseDnVecSetValues(<DnVecDescr>desc, <void*>values)
+    check_status(status)
+
+############################################################
+# Dense Matrix APIs
+
+cpdef size_t createDnMat(int64_t rows, int64_t cols, int64_t ld,
+                         intptr_t values, DataType valueType,
+                         Order order) except? -1:
+    cdef DnMatDescr desc
+    status = cusparseCreateDnMat(&desc, rows, cols, ld, <void*>values,
+                                 valueType, order)
+    check_status(status)
+    return <size_t>desc
+
+cpdef void destroyDnMat(size_t desc) except *:
+    status = cusparseDestroyDnMat(<DnMatDescr>desc)
+    check_status(status)
+
+cpdef DnMatAttributes dnMatGet(size_t desc):
+    cdef int64_t rows, cols, ld
+    cdef intptr_t values,
+    cdef DataType valueType
+    cdef Order order
+    status = cusparseDnMatGet(<DnMatDescr>desc, &rows, &cols, &ld,
+                              <void**>&values, &valueType, &order)
+    check_status(status)
+    return DnMatAttributes(rows, cols, ld, values, valueType, order)
+
+cpdef intptr_t dnMatGetValues(size_t desc) except? -1:
+    cdef intptr_t values
+    status = cusparseDnMatGetValues(<DnMatDescr>desc, <void**>&values)
+    check_status(status)
+    return values
+
+cpdef void dnMatSetValues(size_t desc, intptr_t values) except *:
+    status = cusparseDnMatSetValues(<DnMatDescr>desc, <void*>values)
+    check_status(status)
+
+cpdef DnMatBatchAttributes dnMatGetStridedBatch(size_t desc):
+    cdef int batchCount
+    cdef int64_t batchStride
+    status = cusparseDnMatGetStridedBatch(<DnMatDescr>desc, &batchCount,
+                                          &batchStride)
+    check_status(status)
+    return DnMatBatchAttributes(batchCount, batchStride)
+
+cpdef void dnMatSetStridedBatch(
+        size_t desc, int batchCount, int64_t batchStride) except *:
+    status = cusparseDnMatSetStridedBatch(<DnMatDescr>desc, batchCount,
+                                          batchStride)
+    check_status(status)
+
+############################################################
+# Generic API Functions
+
+cpdef size_t spVV_bufferSize(intptr_t handle, Operation opX,
+                             size_t vecX, size_t vecY,
+                             intptr_t result, DataType computeType) except? -1:
+    cdef size_t bufferSize
+    status = cusparseSpVV_bufferSize(<Handle>handle, opX,
+                                     <SpVecDescr>vecX, <DnVecDescr>vecY,
+                                     <void*>result, computeType, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void spVV(
+        intptr_t handle, Operation opX, size_t vecX, size_t vecY,
+        intptr_t result, DataType computeType,
+        intptr_t externalBuffer) except *:
+    _setStream(handle)
+    status = cusparseSpVV(<Handle>handle, opX, <SpVecDescr>vecX,
+                          <DnVecDescr>vecY, <void*>result, computeType,
+                          <void*>externalBuffer)
+    check_status(status)
+
+cpdef size_t spMV_bufferSize(intptr_t handle, Operation opA, intptr_t alpha,
+                             size_t matA, size_t vecX, intptr_t beta,
+                             size_t vecY, DataType computeType,
+                             SpMVAlg alg) except? -1:
+    cdef size_t bufferSize
+    status = cusparseSpMV_bufferSize(<Handle>handle, opA, <void*>alpha,
+                                     <SpMatDescr>matA, <DnVecDescr>vecX,
+                                     <void*>beta, <DnVecDescr>vecY,
+                                     computeType, alg, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void spMV(
+        intptr_t handle, Operation opA, intptr_t alpha, size_t matA,
+        size_t vecX, intptr_t beta, size_t vecY, DataType computeType,
+        SpMVAlg alg, intptr_t externalBuffer) except *:
+    _setStream(handle)
+    status = cusparseSpMV(<Handle>handle, opA, <void*>alpha, <SpMatDescr>matA,
+                          <DnVecDescr>vecX, <void*>beta, <DnVecDescr>vecY,
+                          computeType, alg, <void*>externalBuffer)
+    check_status(status)
+
+cpdef size_t spSM_createDescr() except? -1:
+    cdef SpSMDescr descr
+    status = cusparseSpSM_createDescr(&descr)
+    check_status(status)
+    return <size_t>descr
+
+cpdef void spSM_destroyDescr(size_t descr) except *:
+    status = cusparseSpSM_destroyDescr(<SpSMDescr>descr)
+    check_status(status)
+
+cpdef size_t spSM_bufferSize(intptr_t handle, Operation opA, Operation opB,
+                             intptr_t alpha, size_t matA, size_t matB,
+                             size_t matC, DataType computeType, SpSMAlg alg,
+                             size_t spsmDescr) except? -1:
+    cdef size_t bufferSize
+    status = cusparseSpSM_bufferSize(<Handle>handle, opA, opB, <void*>alpha,
+                                     <SpMatDescr>matA, <DnMatDescr>matB,
+                                     <DnMatDescr>matC, computeType, alg,
+                                     <SpSMDescr>spsmDescr, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void spSM_analysis(
+        intptr_t handle, Operation opA, Operation opB,
+        intptr_t alpha, size_t matA, size_t matB, size_t matC,
+        DataType computeType, SpSMAlg alg, size_t spsmDescr,
+        intptr_t externalBuffer) except *:
+    _setStream(handle)
+    status = cusparseSpSM_analysis(<Handle> handle, opA, opB, <void*>alpha,
+                                   <SpMatDescr>matA, <DnMatDescr>matB,
+                                   <DnMatDescr>matC, computeType, alg,
+                                   <SpSMDescr>spsmDescr, <void*>externalBuffer)
+    check_status(status)
+
+cpdef void spSM_solve(
+        intptr_t handle, Operation opA, Operation opB, intptr_t alpha,
+        size_t matA, size_t matB, size_t matC, DataType computeType,
+        SpSMAlg alg, size_t spsmDescr, intptr_t externalBuffer=0) except *:
+    _setStream(handle)
+    IF CUPY_HIP_VERSION > 0:
+        # hipsparseSpSM_solve has the extra `externalBuffer` parameter that
+        # cusparseSpSM_solve does not require.
+        status = cusparseSpSM_solve(<Handle> handle, opA, opB, <void*>alpha,
+                                    <SpMatDescr>matA, <DnMatDescr>matB,
+                                    <DnMatDescr>matC, computeType, alg,
+                                    <SpSMDescr>spsmDescr,
+                                    <void*>externalBuffer)
+    ELSE:
+        status = cusparseSpSM_solve(<Handle> handle, opA, opB, <void*>alpha,
+                                    <SpMatDescr>matA, <DnMatDescr>matB,
+                                    <DnMatDescr>matC, computeType, alg,
+                                    <SpSMDescr>spsmDescr)
+    check_status(status)
+
+cpdef size_t spMM_bufferSize(intptr_t handle, Operation opA, Operation opB,
+                             intptr_t alpha, size_t matA, size_t matB,
+                             intptr_t beta, size_t matC, DataType computeType,
+                             SpMMAlg alg) except? -1:
+    cdef size_t bufferSize
+    status = cusparseSpMM_bufferSize(<Handle>handle, opA, opB, <void*>alpha,
+                                     <SpMatDescr>matA, <DnMatDescr>matB,
+                                     <void*>beta, <DnMatDescr>matC,
+                                     computeType, alg, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void spMM(
+        intptr_t handle, Operation opA, Operation opB, intptr_t alpha,
+        size_t matA, size_t matB, intptr_t beta, size_t matC,
+        DataType computeType, SpMMAlg alg, intptr_t externalBuffer) except *:
+    _setStream(handle)
+    status = cusparseSpMM(<Handle>handle, opA, opB, <void*>alpha,
+                          <SpMatDescr>matA, <DnMatDescr>matB, <void*>beta,
+                          <DnMatDescr>matC, computeType, alg,
+                          <void*>externalBuffer)
+    check_status(status)
+
+cpdef size_t constrainedGeMM_bufferSize(intptr_t handle, Operation opA,
+                                        Operation opB, intptr_t alpha,
+                                        size_t matA, size_t matB,
+                                        intptr_t beta, size_t matC,
+                                        DataType computeType) except? -1:
+    cdef size_t bufferSize
+    status = cusparseConstrainedGeMM_bufferSize(
+        <Handle>handle, opA, opB, <void*>alpha, <DnMatDescr>matA,
+        <DnMatDescr>matB, <void*>beta, <SpMatDescr>matC, computeType,
+        &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void constrainedGeMM(
+        intptr_t handle, Operation opA, Operation opB,
+        intptr_t alpha, size_t matA, size_t matB, intptr_t beta,
+        size_t matC, DataType computeType,
+        intptr_t externalBuffer) except *:
+    _setStream(handle)
+    status = cusparseConstrainedGeMM(
+        <Handle>handle, opA, opB, <void*>alpha, <DnMatDescr>matA,
+        <DnMatDescr>matB, <void*>beta, <SpMatDescr>matC, computeType,
+        <void*>externalBuffer)
+    check_status(status)
+
+cpdef size_t spGEMM_createDescr() except? -1:
+    cdef SpGEMMDescr descr
+    status = cusparseSpGEMM_createDescr(&descr)
+    check_status(status)
+    return <size_t>descr
+
+cpdef void spGEMM_destroyDescr(size_t descr) except *:
+    status = cusparseSpGEMM_destroyDescr(<SpGEMMDescr>descr)
+    check_status(status)
+
+cpdef size_t spGEMM_workEstimation(
+        intptr_t handle, Operation opA, Operation opB, intptr_t alpha,
+        size_t matA, size_t matB, intptr_t beta, size_t matC,
+        DataType computeType, int alg, size_t spgemmDescr,
+        size_t bufferSize, intptr_t externalBuffer1) except? -1:
+    cdef size_t bufferSize1 = bufferSize
+    status = cusparseSpGEMM_workEstimation(
+        <Handle>handle, opA, opB, <const void*>alpha, <SpMatDescr>matA,
+        <SpMatDescr>matB, <const void*>beta, <SpMatDescr>matC, computeType,
+        <SpGEMMAlg>alg, <SpGEMMDescr>spgemmDescr, &bufferSize1,
+        <void*>externalBuffer1)
+    check_status(status)
+    return bufferSize1
+
+cpdef size_t spGEMM_compute(
+        intptr_t handle, Operation opA, Operation opB, intptr_t alpha,
+        size_t matA, size_t matB, intptr_t beta, size_t matC,
+        DataType computeType, int alg, size_t spgemmDescr,
+        size_t bufferSize, intptr_t externalBuffer2) except? -1:
+    cdef size_t bufferSize2 = bufferSize
+    status = cusparseSpGEMM_compute(
+        <Handle>handle, opA, opB, <const void*>alpha, <SpMatDescr>matA,
+        <SpMatDescr>matB, <const void*>beta, <SpMatDescr>matC, computeType,
+        <SpGEMMAlg>alg, <SpGEMMDescr>spgemmDescr, &bufferSize2,
+        <void*>externalBuffer2)
+    check_status(status)
+    return bufferSize2
+
+cpdef void spGEMM_copy(
+        intptr_t handle, Operation opA, Operation opB, intptr_t alpha,
+        size_t matA, size_t matB, intptr_t beta, size_t matC,
+        DataType computeType, int alg, size_t spgemmDescr) except *:
+    status = cusparseSpGEMM_copy(
+        <Handle>handle, opA, opB, <const void*>alpha, <SpMatDescr>matA,
+        <SpMatDescr>matB, <const void*>beta, <SpMatDescr>matC, computeType,
+        <SpGEMMAlg>alg, <SpGEMMDescr>spgemmDescr)
+    check_status(status)
+
+cpdef void gather(intptr_t handle, size_t vecY, size_t vecX) except *:
+    status = cusparseGather(<Handle>handle, <DnVecDescr>vecY, <SpVecDescr>vecX)
+    check_status(status)
+
+cpdef size_t sparseToDense_bufferSize(intptr_t handle, size_t matA,
+                                      size_t matB, int alg) except? -1:
+    cdef size_t bufferSize
+    status = cusparseSparseToDense_bufferSize(
+        <Handle>handle, <SpMatDescr>matA, <DnMatDescr>matB,
+        <cusparseSparseToDenseAlg_t>alg, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void sparseToDense(
+        intptr_t handle, size_t matA, size_t matB, int alg,
+        intptr_t buffer) except *:
+    _setStream(handle)
+    status = cusparseSparseToDense(
+        <Handle>handle, <SpMatDescr>matA, <DnMatDescr>matB,
+        <cusparseSparseToDenseAlg_t>alg, <void*>buffer)
+    check_status(status)
+
+cpdef size_t denseToSparse_bufferSize(intptr_t handle, size_t matA,
+                                      size_t matB, int alg) except? -1:
+    cdef size_t bufferSize
+    status = cusparseDenseToSparse_bufferSize(
+        <Handle>handle, <DnMatDescr>matA, <SpMatDescr>matB,
+        <cusparseDenseToSparseAlg_t>alg, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void denseToSparse_analysis(
+        intptr_t handle, size_t matA, size_t matB,
+        int alg, intptr_t buffer) except *:
+    _setStream(handle)
+    status = cusparseDenseToSparse_analysis(
+        <Handle>handle, <DnMatDescr>matA, <SpMatDescr>matB,
+        <cusparseDenseToSparseAlg_t>alg, <void*>buffer)
+    check_status(status)
+
+cpdef void denseToSparse_convert(
+        intptr_t handle, size_t matA, size_t matB,
+        int alg, intptr_t buffer) except *:
+    _setStream(handle)
+    status = cusparseDenseToSparse_convert(
+        <Handle>handle, <DnMatDescr>matA, <SpMatDescr>matB,
+        <cusparseDenseToSparseAlg_t>alg, <void*>buffer)
+    check_status(status)
+
+# CSR2CSC
+cpdef size_t csr2cscEx2_bufferSize(
+        intptr_t handle, int m, int n, int nnz, intptr_t csrVal,
+        intptr_t csrRowPtr, intptr_t csrColInd, intptr_t cscVal,
+        intptr_t cscColPtr, intptr_t cscRowInd, DataType valType,
+        Action copyValues, IndexBase idxBase, Csr2CscAlg alg) except? -1:
+    cdef size_t bufferSize
+    status = cusparseCsr2cscEx2_bufferSize(
+        <Handle>handle, m, n, nnz, <const void*>csrVal, <const int*>csrRowPtr,
+        <const int*>csrColInd, <void*>cscVal, <int*>cscColPtr, <int*>cscRowInd,
+        valType, copyValues, idxBase, alg, &bufferSize)
+    check_status(status)
+    return bufferSize
+
+cpdef void csr2cscEx2(
+        intptr_t handle, int m, int n, int nnz, intptr_t csrVal,
+        intptr_t csrRowPtr, intptr_t csrColInd, intptr_t cscVal,
+        intptr_t cscColPtr, intptr_t cscRowInd, DataType valType,
+        Action copyValues, IndexBase idxBase, Csr2CscAlg alg,
+        intptr_t buffer) except *:
+    setStream(handle, stream_module.get_current_stream_ptr())
+    status = cusparseCsr2cscEx2(
+        <Handle>handle, m, n, nnz, <const void*>csrVal, <const int*>csrRowPtr,
+        <const int*>csrColInd, <void*>cscVal, <int*>cscColPtr, <int*>cscRowInd,
+        valType, copyValues, idxBase, alg, <void*>buffer)
+    check_status(status)
diff --git a/cupy_backends/cuda/libs/cusparselt.pxd b/cupy_backends/cuda/libs/cusparselt.pxd
new file mode 100644
index 0000000..8215cab
--- /dev/null
+++ b/cupy_backends/cuda/libs/cusparselt.pxd
@@ -0,0 +1,39 @@
+from libc.stdint cimport int32_t, uint32_t, int64_t, uint64_t, intptr_t
+
+###############################################################################
+# Enum
+###############################################################################
+
+cpdef enum:
+    # cusparseLtSparsity_t
+    CUSPARSELT_SPARSITY_50_PERCENT = 0
+
+    # cusparseLtMatDescAttribute_t
+    CUSPARSELT_MAT_NUM_BATCHES = 0   # READ/WRITE
+    CUSPARSELT_MAT_BATCH_STRIDE = 1  # READ/WRITE
+
+    # cusparseComputeType
+    CUSPARSE_COMPUTE_16F = 0
+    CUSPARSE_COMPUTE_32I = 1
+    CUSPARSE_COMPUTE_TF32 = 2
+    CUSPARSE_COMPUTE_TF32_FAST = 3
+
+    # cusparseLtMatmulDescAttribute_t
+    CUSPARSELT_MATMUL_ACTIVATION_RELU = 0             # READ/WRITE
+    CUSPARSELT_MATMUL_ACTIVATION_RELU_UPPERBOUND = 1  # READ/WRITE
+    CUSPARSELT_MATMUL_ACTIVATION_RELU_THRESHOLD = 2   # READ/WRITE
+    CUSPARSELT_MATMUL_ACTIVATION_GELU = 3             # READ/WRITE
+    CUSPARSELT_MATMUL_BIAS_STRIDE = 4                 # READ/WRITE
+    CUSPARSELT_MATMUL_BIAS_POINTER = 5                # READ/WRITE
+
+    # cusparseLtMatmulAlg_t
+    CUSPARSELT_MATMUL_ALG_DEFAULT = 0
+
+    # cusparseLtMatmulAlgAttribute_t
+    CUSPARSELT_MATMUL_ALG_CONFIG_ID = 0      # NOQA, READ/WRITE
+    CUSPARSELT_MATMUL_ALG_CONFIG_MAX_ID = 1  # NOQA, READ-ONLY
+    CUSPARSELT_MATMUL_SEARCH_ITERATIONS = 2  # NOQA, READ/WRITE
+
+    # cusparseLtPruneAlg_t
+    CUSPARSELT_PRUNE_SPMMA_TILE = 0
+    CUSPARSELT_PRUNE_SPMMA_STRIP = 1
diff --git a/cupy_backends/cuda/libs/cusparselt.pyx b/cupy_backends/cuda/libs/cusparselt.pyx
new file mode 100644
index 0000000..896d98f
--- /dev/null
+++ b/cupy_backends/cuda/libs/cusparselt.pyx
@@ -0,0 +1,505 @@
+"""Thin wrapper for cuSPARSELt"""
+
+cimport cython  # NOQA
+
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from libc.stdint cimport int32_t, uint32_t, int64_t, intptr_t
+
+from cupy_backends.cuda cimport stream as stream_module
+from cupy_backends.cuda.api cimport driver
+
+from cupy_backends.cuda.libs import cusparse as _cusparse
+
+
+cdef extern from '../../cupy_cusparselt.h' nogil:
+    ctypedef int cusparseStatus_t 'cusparseStatus_t'
+    ctypedef int cusparseOrder_t 'cusparseOrder_t'
+    ctypedef int cudaDataType 'cudaDataType'
+    ctypedef int cusparseComputeType 'cusparseComputeType'
+
+    # Opaque Data Structures
+    ctypedef struct cusparseLtHandle_t 'cusparseLtHandle_t':
+        pass
+    ctypedef struct cusparseLtMatDescriptor_t 'cusparseLtMatDescriptor_t':
+        pass
+    ctypedef struct cusparseLtMatmulDescriptor_t 'cusparseLtMatmulDescriptor_t':  # NOQA
+        pass
+    ctypedef struct cusparseLtMatmulAlgSelection_t 'cusparseLtMatmulAlgSelection_t':  # NOQA
+        pass
+    ctypedef struct cusparseLtMatmulPlan_t 'cusparseLtMatmulPlan_t':
+        pass
+
+    # Enumerators
+    ctypedef int cusparseLtSparsity_t 'cusparseLtSparsity_t'
+    ctypedef int cusparseLtMatDescAttribute_t 'cusparseLtMatDescAttribute_t'
+    ctypedef int cusparseLtMatmulDescAttribute_t 'cusparseLtMatmulDescAttribute_t'  # NOQA
+    ctypedef int cusparseOperation_t 'cusparseOperation_t'
+    ctypedef int cusparseLtMatmulAlg_t 'cusparseLtMatmulAlg_t'
+    ctypedef int cusparseLtMatmulAlgAttribute_t 'cusparseLtMatmulAlgAttribute_t'  # NOQA
+    ctypedef int cusparseLtPruneAlg_t 'cusparseLtPruneAlg_t'
+
+    # Management Functions
+    cusparseStatus_t cusparseLtInit(cusparseLtHandle_t* handle)
+    cusparseStatus_t cusparseLtDestroy(const cusparseLtHandle_t* handle)
+
+    # Matmul Functions
+    cusparseStatus_t cusparseLtDenseDescriptorInit(
+        const cusparseLtHandle_t* handle,
+        cusparseLtMatDescriptor_t* matDescr,
+        int64_t rows, int64_t cols, int64_t ld, uint32_t alignment,
+        cudaDataType valueType, cusparseOrder_t order)
+    cusparseStatus_t cusparseLtStructuredDescriptorInit(
+        const cusparseLtHandle_t* handle,
+        cusparseLtMatDescriptor_t* matDescr,
+        int64_t rows, int64_t cols, int64_t ld, uint32_t alignment,
+        cudaDataType valueType, cusparseOrder_t order,
+        cusparseLtSparsity_t sparsity)
+    cusparseStatus_t cusparseLtMatDescriptorDestroy(
+        const cusparseLtMatDescriptor_t* matDescr)
+    cusparseStatus_t cusparseLtMatDescSetAttribute(
+        const cusparseLtHandle_t* handle,
+        cusparseLtMatDescriptor_t* matDescr,
+        cusparseLtMatDescAttribute_t matAttribute,
+        const void* data, size_t dataSize)
+    cusparseStatus_t cusparseLtMatDescGetAttribute(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatDescriptor_t* matDescr,
+        cusparseLtMatDescAttribute_t matAttribute,
+        void* data, size_t dataSize)
+    cusparseStatus_t cusparseLtMatmulDescriptorInit(
+        const cusparseLtHandle_t* handle,
+        cusparseLtMatmulDescriptor_t* matMulDescr,
+        cusparseOperation_t opA,
+        cusparseOperation_t opB,
+        const cusparseLtMatDescriptor_t* matA,
+        const cusparseLtMatDescriptor_t* matB,
+        const cusparseLtMatDescriptor_t* matC,
+        const cusparseLtMatDescriptor_t* matD,
+        cusparseComputeType computeType)
+    cusparseStatus_t cusparseLtMatmulDescSetAttribute(
+        const cusparseLtHandle_t* handle,
+        cusparseLtMatmulDescriptor_t* matmulDescr,
+        cusparseLtMatmulDescAttribute_t matmulAttribute,
+        const void* data, size_t dataSize)
+    cusparseStatus_t cusparseLtMatmulDescGetAttribute(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatmulDescriptor_t* matmulDescr,
+        cusparseLtMatmulDescAttribute_t matmulAttribute,
+        void* data, size_t dataSize)
+    cusparseStatus_t cusparseLtMatmulAlgSelectionInit(
+        const cusparseLtHandle_t* handle,
+        cusparseLtMatmulAlgSelection_t* algSelection,
+        const cusparseLtMatmulDescriptor_t* matmulDescr,
+        cusparseLtMatmulAlg_t alg)
+    cusparseStatus_t cusparseLtMatmulAlgSetAttribute(
+        const cusparseLtHandle_t* handle,
+        cusparseLtMatmulAlgSelection_t* algSelection,
+        cusparseLtMatmulAlgAttribute_t attribute,
+        const void* data, size_t ataSize)
+    cusparseStatus_t cusparseLtMatmulGetWorkspace(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatmulAlgSelection_t* algSelection,
+        size_t* workspaceSize)
+    cusparseStatus_t cusparseLtMatmulPlanInit(
+        const cusparseLtHandle_t* handle,
+        cusparseLtMatmulPlan_t* plan,
+        const cusparseLtMatmulDescriptor_t* matmulDescr,
+        const cusparseLtMatmulAlgSelection_t* algSelection,
+        size_t workspaceSize)
+    cusparseStatus_t cusparseLtMatmulPlanDestroy(
+        const cusparseLtMatmulPlan_t* plan)
+    cusparseStatus_t cusparseLtMatmul(
+        const cusparseLtHandle_t* handle, const cusparseLtMatmulPlan_t* plan,
+        const void* alpha, const void* d_A, const void* d_B,
+        const void* beta, const void* d_C, void* d_D,
+        void* workspace, driver.Stream* streams, int32_t numStreams)
+
+    # Helper Functions
+    cusparseStatus_t cusparseLtSpMMAPrune(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatmulDescriptor_t* matmulDescr,
+        const void* d_in, void* d_out,
+        cusparseLtPruneAlg_t pruneAlg, driver.Stream stream)
+    cusparseStatus_t cusparseLtSpMMAPruneCheck(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatmulDescriptor_t* matmulDescr,
+        const void* d_in, int* valid, driver.Stream stream)
+    cusparseStatus_t cusparseLtSpMMAPrune2(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatDescriptor_t* sparseMatDescr,
+        int isSparseA, cusparseOperation_t op, const void* d_in,
+        void* d_out, cusparseLtPruneAlg_t pruneAlg, driver.Stream stream)
+    cusparseStatus_t cusparseLtSpMMAPruneCheck2(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatDescriptor_t* sparseMatDescr,
+        int isSparseA, cusparseOperation_t op, const void* d_in, int* d_valid,
+        driver.Stream stream)
+    cusparseStatus_t cusparseLtSpMMACompressedSize(
+        const cusparseLtHandle_t* handle, const cusparseLtMatmulPlan_t* plan,
+        size_t* compressedSize)
+    cusparseStatus_t cusparseLtSpMMACompress(
+        const cusparseLtHandle_t* handle, const cusparseLtMatmulPlan_t* plan,
+        const void* d_dense, void* d_compressed, driver.Stream stream)
+    cusparseStatus_t cusparseLtSpMMACompressedSize2(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatDescriptor_t* sparseMatDescr,
+        size_t* compressedSize)
+    cusparseStatus_t cusparseLtSpMMACompress2(
+        const cusparseLtHandle_t* handle,
+        const cusparseLtMatDescriptor_t* sparseMatDescr,
+        int isSparseA, cusparseOperation_t op, const void* d_dense,
+        void* d_compressed, driver.Stream stream)
+
+    # Build-time version
+    int CUSPARSELT_VERSION
+
+
+###############################################################################
+# Classes
+###############################################################################
+
+cdef class Handle:
+    cdef void * _ptr
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cusparseLtHandle_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+cdef class MatDescriptor:
+    cdef void * _ptr
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cusparseLtMatDescriptor_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+cdef class MatmulDescriptor:
+    cdef void * _ptr
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cusparseLtMatmulDescriptor_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+cdef class MatmulAlgSelection:
+    cdef void * _ptr
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cusparseLtMatmulAlgSelection_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+cdef class MatmulPlan:
+    cdef void * _ptr
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cusparseLtMatmulPlan_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        raise _cusparse.CuSparseError(status)
+
+###############################################################################
+# cuSPARSELt: Library Management Functions
+###############################################################################
+
+cpdef init(Handle handle):
+    """Initializes the cuSPARSELt library handle"""
+    status = cusparseLtInit(<cusparseLtHandle_t*> handle._ptr)
+    check_status(status)
+
+
+cpdef destroy(Handle handle):
+    """Releases hardware resources used by the cuSPARSELt library"""
+    status = cusparseLtDestroy(<cusparseLtHandle_t*> handle._ptr)
+    check_status(status)
+
+###############################################################################
+# cuSPARSELt: Matmul Functions
+###############################################################################
+
+cpdef denseDescriptorInit(Handle handle, MatDescriptor matDescr,
+                          rows, cols, ld, alignment, valueType, order):
+    """Initializes the descriptor of a dense matrix"""
+    status = cusparseLtDenseDescriptorInit(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <cusparseLtMatDescriptor_t*> matDescr._ptr,
+        <int64_t> rows, <int64_t> cols, <int64_t> ld, <uint32_t> alignment,
+        <cudaDataType> valueType, <cusparseOrder_t> order)
+    check_status(status)
+
+cpdef structuredDescriptorInit(Handle handle, MatDescriptor matDescr,
+                               rows, cols, ld, alignment, valueType, order,
+                               sparsity):
+    """Initializes the descriptor of a structured matrix."""
+    status = cusparseLtStructuredDescriptorInit(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <cusparseLtMatDescriptor_t*> matDescr._ptr,
+        <int64_t> rows, <int64_t> cols, <int64_t> ld, <uint32_t> alignment,
+        <cudaDataType> valueType, <cusparseOrder_t> order,
+        <cusparseLtSparsity_t> sparsity)
+    check_status(status)
+
+cpdef matDescriptorDestroy(MatDescriptor matDescr):
+    """Releases the resources used by an instance of a matrix descriptor."""
+    status = cusparseLtMatDescriptorDestroy(
+        <const cusparseLtMatDescriptor_t*> matDescr._ptr)
+    check_status(status)
+
+cpdef matDescSetAttribute(Handle handle, MatDescriptor matDescr,
+                          matAttribute, size_t data, size_t dataSize):
+    """Sets the attribute related to matrix descriptor."""
+    status = cusparseLtMatDescSetAttribute(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <cusparseLtMatDescriptor_t*> matDescr._ptr,
+        <cusparseLtMatDescAttribute_t> matAttribute,
+        <const void*> data, dataSize)
+    check_status(status)
+
+cpdef matDescGetAttribute(Handle handle, MatDescriptor matDescr,
+                          matAttribute, size_t data, size_t dataSize):
+    """Gets the attribute related to matrix descriptor."""
+    status = cusparseLtMatDescGetAttribute(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatDescriptor_t*> matDescr._ptr,
+        <cusparseLtMatDescAttribute_t> matAttribute,
+        <void*> data, dataSize)
+    check_status(status)
+
+cpdef matmulDescriptorInit(Handle handle,
+                           MatmulDescriptor matMulDescr,
+                           opA, opB,
+                           MatDescriptor matA,
+                           MatDescriptor matB,
+                           MatDescriptor matC,
+                           MatDescriptor matD,
+                           computeType):
+    """Initializes the matrix multiplication descriptor."""
+    status = cusparseLtMatmulDescriptorInit(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <cusparseLtMatmulDescriptor_t*> matMulDescr._ptr,
+        <cusparseOperation_t> opA,
+        <cusparseOperation_t> opB,
+        <const cusparseLtMatDescriptor_t*> matA._ptr,
+        <const cusparseLtMatDescriptor_t*> matB._ptr,
+        <const cusparseLtMatDescriptor_t*> matC._ptr,
+        <const cusparseLtMatDescriptor_t*> matD._ptr,
+        <cusparseComputeType> computeType)
+    check_status(status)
+
+cpdef matmulDescSetAttribute(Handle handle, MatmulDescriptor matmulDescr,
+                             matmulAttribute, size_t data, size_t dataSize):
+    """Sets the attribute related to matmul descriptor."""
+    status = cusparseLtMatmulDescSetAttribute(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <cusparseLtMatmulDescriptor_t*> matmulDescr._ptr,
+        <cusparseLtMatmulDescAttribute_t> matmulAttribute,
+        <const void*> data, dataSize)
+    check_status(status)
+
+cpdef matmulDescGetAttribute(Handle handle, MatmulDescriptor matmulDescr,
+                             matmulAttribute, size_t data, size_t dataSize):
+    """Gets the attribute related to matmul descriptor."""
+    status = cusparseLtMatmulDescGetAttribute(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatmulDescriptor_t*> matmulDescr._ptr,
+        <cusparseLtMatmulDescAttribute_t> matmulAttribute,
+        <void*> data, dataSize)
+    check_status(status)
+
+cpdef matmulAlgSelectionInit(Handle handle, MatmulAlgSelection algSelection,
+                             MatmulDescriptor matmulDescr, alg):
+    """Initializes the algorithm selection descriptor."""
+    status = cusparseLtMatmulAlgSelectionInit(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <cusparseLtMatmulAlgSelection_t*> algSelection._ptr,
+        <const cusparseLtMatmulDescriptor_t*> matmulDescr._ptr,
+        <cusparseLtMatmulAlg_t> alg)
+    check_status(status)
+
+cpdef matmulAlgSetAttribute(Handle handle, MatmulAlgSelection algSelection,
+                            attribute, size_t data, size_t dataSize):
+    """Sets the attribute related to algorithm selection descriptor."""
+    status = cusparseLtMatmulAlgSetAttribute(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <cusparseLtMatmulAlgSelection_t*> algSelection._ptr,
+        <cusparseLtMatmulAlgAttribute_t> attribute,
+        <const void*> data, <size_t> dataSize)
+    check_status(status)
+
+cpdef size_t matmulGetWorkspace(Handle handle,
+                                MatmulAlgSelection algSelection):
+    """Determines the required workspace size"""
+    cdef size_t workspaceSize
+    status = cusparseLtMatmulGetWorkspace(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatmulAlgSelection_t*> algSelection._ptr,
+        &workspaceSize)
+    check_status(status)
+    return workspaceSize
+
+cpdef matmulPlanInit(Handle handle, MatmulPlan plan,
+                     MatmulDescriptor matmulDescr,
+                     MatmulAlgSelection algSelection,
+                     size_t workspaceSize):
+    """Initializes the plan."""
+    status = cusparseLtMatmulPlanInit(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <cusparseLtMatmulPlan_t*> plan._ptr,
+        <const cusparseLtMatmulDescriptor_t*> matmulDescr._ptr,
+        <const cusparseLtMatmulAlgSelection_t*> algSelection._ptr,
+        <size_t> workspaceSize)
+    check_status(status)
+
+cpdef matmulPlanDestroy(MatmulPlan plan):
+    """Destroys plan"""
+    status = cusparseLtMatmulPlanDestroy(
+        <const cusparseLtMatmulPlan_t*> plan._ptr)
+    check_status(status)
+
+cpdef matmul(Handle handle, MatmulPlan plan,
+             size_t alpha, size_t d_A, size_t d_B,
+             size_t beta, size_t d_C, size_t d_D, size_t workspace):
+    """Computes the matrix multiplication"""
+    status = cusparseLtMatmul(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatmulPlan_t*> plan._ptr,
+        <const void*> alpha, <const void*> d_A, <const void*> d_B,
+        <const void*> beta, <const void*> d_C, <void*> d_D,
+        <void*> workspace, <driver.Stream*> NULL, <int32_t> 0)
+    check_status(status)
+
+###############################################################################
+# cuSPARSELt: Helper Functions
+###############################################################################
+
+cpdef spMMAPrune(Handle handle, MatmulDescriptor matmulDescr,
+                 size_t d_in, size_t d_out, pruneAlg):
+    """Prunes a dense matrix d_in"""
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cusparseLtSpMMAPrune(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatmulDescriptor_t*> matmulDescr._ptr,
+        <const void*> d_in, <void*> d_out,
+        <cusparseLtPruneAlg_t> pruneAlg, <driver.Stream> stream)
+    check_status(status)
+
+cpdef spMMAPruneCheck(Handle handle, MatmulDescriptor matmulDescr,
+                      size_t d_in, size_t valid):
+    """Checks the correctness of the pruning structure"""
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cusparseLtSpMMAPruneCheck(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatmulDescriptor_t*> matmulDescr._ptr,
+        <const void*> d_in, <int*> valid, <driver.Stream> stream)
+    check_status(status)
+
+cpdef spMMAPrune2(Handle handle, MatDescriptor sparseMatDescr, isSparseA,
+                  op, size_t d_in, size_t d_out, pruneAlg):
+    """Prunes a dense matrix d_in"""
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cusparseLtSpMMAPrune2(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatDescriptor_t*> sparseMatDescr._ptr,
+        <int> isSparseA, <cusparseOperation_t> op, <const void*> d_in,
+        <void*> d_out, <cusparseLtPruneAlg_t> pruneAlg, <driver.Stream> stream)
+    check_status(status)
+
+cpdef spMMAPruneCheck2(Handle handle, MatDescriptor sparseMatDescr, isSparseA,
+                       op, size_t d_in, size_t d_valid):
+    """Checks the correctness of the pruning structure"""
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cusparseLtSpMMAPruneCheck2(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatDescriptor_t*> sparseMatDescr._ptr,
+        <int> isSparseA, <cusparseOperation_t> op, <const void*> d_in,
+        <int*> d_valid, <driver.Stream> stream)
+    check_status(status)
+
+cpdef size_t spMMACompressedSize(Handle handle, MatmulPlan plan):
+    """Provides the size of the compressed matrix"""
+    cdef size_t compressedSize
+    status = cusparseLtSpMMACompressedSize(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatmulPlan_t*> plan._ptr,
+        &compressedSize)
+    check_status(status)
+    return compressedSize
+
+cpdef spMMACompress(Handle handle, MatmulPlan plan,
+                    size_t d_dense, size_t d_compressed):
+    """Compresses a dense matrix d_dense."""
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cusparseLtSpMMACompress(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatmulPlan_t*> plan._ptr,
+        <const void*> d_dense, <void*> d_compressed, <driver.Stream> stream)
+    check_status(status)
+
+cpdef size_t spMMACompressedSize2(Handle handle, MatDescriptor sparseMatDescr):
+    """Provides the size of the compressed matrix"""
+    cdef size_t compressedSize
+    status = cusparseLtSpMMACompressedSize2(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatDescriptor_t*> sparseMatDescr._ptr,
+        &compressedSize)
+    check_status(status)
+    return compressedSize
+
+cpdef spMMACompress2(Handle handle, MatDescriptor sparseMatDescr,
+                     isSparseA, op, size_t d_dense, size_t d_compressed):
+    """Compresses a dense matrix d_dense."""
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cusparseLtSpMMACompress2(
+        <const cusparseLtHandle_t*> handle._ptr,
+        <const cusparseLtMatDescriptor_t*> sparseMatDescr._ptr,
+        <int> isSparseA, <cusparseOperation_t> op, <const void*> d_dense,
+        <void*> d_compressed, <driver.Stream> stream)
+    check_status(status)
+
+
+def get_build_version():
+    return CUSPARSELT_VERSION
diff --git a/cupy_backends/cuda/libs/cutensor.pxd b/cupy_backends/cuda/libs/cutensor.pxd
new file mode 100644
index 0000000..3a2c1bd
--- /dev/null
+++ b/cupy_backends/cuda/libs/cutensor.pxd
@@ -0,0 +1,288 @@
+from libc.stdint cimport int32_t, uint32_t, int64_t, uint64_t, intptr_t
+
+
+###############################################################################
+# Enum
+###############################################################################
+
+cpdef enum:
+    # cutensorAlgo_t (values > 0 correspond to certain algorithms of GETT)
+    ALGO_DEFAULT_PATIENT = -6  # NOQA, Uses the more accurate but also more time-consuming performance model
+    ALGO_GETT = -4             # NOQA, Choose the GETT algorithm
+    ALGO_TGETT = -3            # NOQA, Transpose (A or B) + GETT
+    ALGO_TTGT = -2             # NOQA, Transpose-Transpose-GEMM-Transpose (requires additional memory)
+    ALGO_DEFAULT = -1          # NOQA, Lets the internal heuristic choose
+
+    # cutensorWorksizePreference_t
+    WORKSPACE_MIN = 1          # NOQA, At least one algorithm will be available
+    WORKSPACE_RECOMMENDED = 2  # NOQA, The most suitable algorithm will be available
+    WORKSPACE_MAX = 3          # NOQA, All algorithms will be available
+
+    # cutensorOperator_t (Unary)
+    OP_IDENTITY = 1  # NOQA, Identity operator (i.e., elements are not changed)
+    OP_SQRT = 2      # NOQA, Square root
+    OP_RELU = 8      # NOQA, Rectified linear unit
+    OP_CONJ = 9      # NOQA, Complex conjugate
+    OP_RCP = 10      # NOQA, Reciprocal
+    OP_SIGMOID = 11  # NOQA, y=1/(1+exp(-x))
+    OP_TANH = 12     # NOQA, y=tanh(x)
+    OP_EXP = 22      # NOQA, Exponentiation.
+    OP_LOG = 23      # NOQA, Log (base e).
+    OP_ABS = 24      # NOQA, Absolute value.
+    OP_NEG = 25      # NOQA, Negation.
+    OP_SIN = 26      # NOQA, Sine.
+    OP_COS = 27      # NOQA, Cosine.
+    OP_TAN = 28      # NOQA, Tangent.
+    OP_SINH = 29     # NOQA, Hyperbolic sine.
+    OP_COSH = 30     # NOQA, Hyperbolic cosine.
+    OP_ASIN = 31     # NOQA, Inverse sine.
+    OP_ACOS = 32     # NOQA, Inverse cosine.
+    OP_ATAN = 33     # NOQA, Inverse tangent.
+    OP_ASINH = 34    # NOQA, Inverse hyperbolic sine.
+    OP_ACOSH = 35    # NOQA, Inverse hyperbolic cosine.
+    OP_ATANH = 36    # NOQA, Inverse hyperbolic tangent.
+    OP_CEIL = 37     # NOQA, Ceiling.
+    OP_FLOOR = 38    # NOQA, Floor.
+
+    # cutensorOperator_t (Binary)
+    OP_ADD = 3  # NOQA, Addition of two elements
+    OP_MUL = 5  # NOQA, Multiplication of two elements
+    OP_MAX = 6  # NOQA, Maximum of two elements
+    OP_MIN = 7  # NOQA, Minimum of two elements
+
+    # cutensorStatus_t
+    STATUS_SUCCESS = 0
+    STATUS_NOT_INITIALIZED = 1
+    STATUS_ALLOC_FAILED = 3
+    STATUS_INVALID_VALUE = 7
+    STATUS_ARCH_MISMATCH = 8  # NOQA, Indicates that the device is either not ready, or the target architecture is not supported.
+    STATUS_MAPPING_ERROR = 11
+    STATUS_EXECUTION_FAILED = 13
+    STATUS_INTERNAL_ERROR = 14
+    STATUS_NOT_SUPPORTED = 15
+    STATUS_LICENSE_ERROR = 16
+    STATUS_CUBLAS_ERROR = 17
+    STATUS_CUDA_ERROR = 18
+    STATUS_INSUFFICIENT_WORKSPACE = 19
+    STATUS_INSUFFICIENT_DRIVER = 20  # NOQA, Indicates that the driver version is insufficient.
+    STATUS_IO_ERROR = 21
+
+    # cutensorComputeType_t
+    # (*) compute types added in versoin 1.2
+    COMPUTE_16F = 1     # NOQA, half
+    COMPUTE_16BF = 1024  # NOQA, bfloat
+    COMPUTE_TF32 = 4096  # NOQA, tensor-float-32
+    COMPUTE_32F = 4     # NOQA, float
+    COMPUTE_64F = 16    # NOQA, double
+    COMPUTE_8U = 64    # NOQA, uint8
+    COMPUTE_8I = 256   # NOQA, int8
+    COMPUTE_32U = 128   # NOQA, uint32
+    COMPUTE_32I = 512   # NOQA, int32
+    # (*) compute types below will be deprecated in the furture release.
+    R_MIN_16F = 1    # NOQA, real as a half
+    C_MIN_16F = 2    # NOQA, complex as a half
+    R_MIN_32F = 4    # NOQA, real as a float
+    C_MIN_32F = 8    # NOQA, complex as a float
+    R_MIN_64F = 16   # NOQA, real as a double
+    C_MIN_64F = 32   # NOQA, complex as a double
+    R_MIN_8U = 64   # NOQA, real as a uint8
+    R_MIN_32U = 128  # NOQA, real as a uint32
+    R_MIN_8I = 256  # NOQA, real as a int8
+    R_MIN_32I = 512  # NOQA, real as a int32
+    R_MIN_16BF = 1024  # NOQA, real as a bfloat16
+    R_MIN_TF32 = 2048  # NOQA, real as a tensorfloat32
+    C_MIN_TF32 = 4096  # NOQA, complex as a tensorfloat32
+
+
+cpdef size_t get_version()
+
+
+cdef class Handle:
+
+    cdef void* _ptr
+
+
+cdef class TensorDescriptor:
+
+    cdef void* _ptr
+
+
+cdef class ContractionDescriptor:
+
+    cdef void* _ptr
+
+
+cdef class ContractionFind:
+
+    cdef void* _ptr
+
+
+cdef class ContractionPlan:
+
+    cdef void* _ptr
+
+
+cpdef init(Handle handle)
+
+cpdef initTensorDescriptor(
+    Handle handle,
+    TensorDescriptor desc,
+    uint32_t numModes,
+    intptr_t extent,
+    intptr_t stride,
+    int dataType,
+    int unaryOp)
+
+cpdef elementwiseTrinary(
+    Handle handle,
+    intptr_t alpha,
+    intptr_t A,
+    TensorDescriptor descA,
+    intptr_t modeA,
+    intptr_t beta,
+    intptr_t B,
+    TensorDescriptor descB,
+    intptr_t modeB,
+    intptr_t gamma,
+    intptr_t C,
+    TensorDescriptor descC,
+    intptr_t modeC,
+    intptr_t D,
+    TensorDescriptor descD,
+    intptr_t modeD,
+    int opAB,
+    int opABC,
+    int typeScalar)
+
+cpdef elementwiseBinary(
+    Handle handle,
+    intptr_t alpha,
+    intptr_t A,
+    TensorDescriptor descA,
+    intptr_t modeA,
+    intptr_t gamma,
+    intptr_t C,
+    TensorDescriptor descC,
+    intptr_t modeC,
+    intptr_t D,
+    TensorDescriptor descD,
+    intptr_t modeD,
+    int opAC,
+    int typeScalar)
+
+cpdef permutation(
+    Handle handle,
+    intptr_t alpha,
+    intptr_t A,
+    TensorDescriptor descA,
+    intptr_t modeA,
+    intptr_t B,
+    TensorDescriptor descB,
+    intptr_t modeB,
+    int typeScalar)
+
+cpdef initContractionDescriptor(
+    Handle handle,
+    ContractionDescriptor desc,
+    TensorDescriptor descA,
+    intptr_t modeA,
+    uint32_t alignmentRequirementA,
+    TensorDescriptor descB,
+    intptr_t modeB,
+    uint32_t alignmentRequirementB,
+    TensorDescriptor descC,
+    intptr_t modeC,
+    uint32_t alignmentRequirementC,
+    TensorDescriptor descD,
+    intptr_t modeD,
+    uint32_t alignmentRequirementD,
+    int computeType)
+
+cpdef initContractionFind(
+    Handle handle,
+    ContractionFind find,
+    int algo)
+
+cpdef initContractionPlan(
+    Handle handle,
+    ContractionPlan plan,
+    ContractionDescriptor desc,
+    ContractionFind find,
+    uint64_t workspaceSize)
+
+cpdef contraction(
+    Handle handle,
+    ContractionPlan plan,
+    intptr_t alpha,
+    intptr_t A,
+    intptr_t B,
+    intptr_t beta,
+    intptr_t C,
+    intptr_t D,
+    intptr_t workspace,
+    uint64_t workspaceSize)
+
+cpdef uint64_t contractionGetWorkspaceSize(
+    Handle handle,
+    ContractionDescriptor desc,
+    ContractionFind find,
+    int pref)
+
+cpdef uint64_t contractionGetWorkspace(
+    Handle handle,
+    ContractionDescriptor desc,
+    ContractionFind find,
+    int pref)
+
+cpdef int32_t contractionMaxAlgos()
+
+cpdef reduction(
+    Handle handle,
+    intptr_t alpha,
+    intptr_t A,
+    TensorDescriptor descA,
+    intptr_t modeA,
+    intptr_t beta,
+    intptr_t C,
+    TensorDescriptor descC,
+    intptr_t modeC,
+    intptr_t D,
+    TensorDescriptor descD,
+    intptr_t modeD,
+    int opReduce,
+    int minTypeCompute,
+    intptr_t workspace,
+    uint64_t workspaceSize)
+
+cpdef uint64_t reductionGetWorkspaceSize(
+    Handle handle,
+    intptr_t A,
+    TensorDescriptor descA,
+    intptr_t modeA,
+    intptr_t C,
+    TensorDescriptor descC,
+    intptr_t modeC,
+    intptr_t D,
+    TensorDescriptor descD,
+    intptr_t modeD,
+    int opReduce,
+    int typeCompute)
+
+cpdef uint64_t reductionGetWorkspace(
+    Handle handle,
+    intptr_t A,
+    TensorDescriptor descA,
+    intptr_t modeA,
+    intptr_t C,
+    TensorDescriptor descC,
+    intptr_t modeC,
+    intptr_t D,
+    TensorDescriptor descD,
+    intptr_t modeD,
+    int opReduce,
+    int typeCompute)
+
+cpdef uint32_t getAlignmentRequirement(
+    Handle handle,
+    intptr_t ptr,
+    TensorDescriptor desc)
diff --git a/cupy_backends/cuda/libs/cutensor.pyx b/cupy_backends/cuda/libs/cutensor.pyx
new file mode 100644
index 0000000..7a8df25
--- /dev/null
+++ b/cupy_backends/cuda/libs/cutensor.pyx
@@ -0,0 +1,1000 @@
+# distutils: language = c++
+
+"""Thin wrapper of cuTENSOR."""
+
+cimport cython  # NOQA
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from libc.stdint cimport int32_t, uint32_t, int64_t, uint64_t, intptr_t
+
+from cupy_backends.cuda cimport stream as stream_module
+from cupy_backends.cuda.api cimport driver
+
+cdef extern from '../../cupy_cutensor.h' nogil:
+    ctypedef int Status 'cutensorStatus_t'
+    ctypedef int Algo 'cutensorAlgo_t'
+    ctypedef int Operator 'cutensorOperator_t'
+    ctypedef int WorksizePreference 'cutensorWorksizePreference_t'
+    ctypedef int DataType 'cudaDataType_t'
+    ctypedef int ComputeType 'cutensorComputeType_t'
+    ctypedef struct cutensorHandle_t 'cutensorHandle_t':
+        int64_t fields[512]
+    ctypedef struct cutensorTensorDescriptor_t 'cutensorTensorDescriptor_t':
+        int64_t fields[64]
+    ctypedef struct cutensorContractionDescriptor_t \
+        'cutensorContractionDescriptor_t':  # NOQA: E125
+        int64_t fields[256]
+    ctypedef struct cutensorContractionPlan_t 'cutensorContractionPlan_t':
+        int64_t fields[640]
+    ctypedef struct cutensorContractionFind_t 'cutensorContractionFind_t':
+        int64_t fields[64]
+
+    const char* cutensorGetErrorString(Status status)
+
+    int cutensorInit(cutensorHandle_t* handle)
+
+    # TODO(niboshi): Add const to input pointer parameters.
+
+    int cutensorInitTensorDescriptor(
+        cutensorHandle_t* handle,
+        cutensorTensorDescriptor_t* desc,
+        uint32_t numModes,
+        int64_t* extent,
+        int64_t* stride,
+        DataType dataType,
+        Operator unaryOp)
+
+    int cutensorElementwiseTrinary(
+        cutensorHandle_t* handle,
+        void* alpha,
+        void* A, cutensorTensorDescriptor_t* descA, int32_t* modeA,
+        void* beta,
+        void* B, cutensorTensorDescriptor_t* descB, int32_t* modeB,
+        void* gamma,
+        void* C, cutensorTensorDescriptor_t* descC, int32_t* modeC,
+        void* D, cutensorTensorDescriptor_t* descD, int32_t* modeD,
+        Operator otAB, Operator otABC,
+        DataType typeScalar, driver.Stream stream)
+
+    int cutensorElementwiseBinary(
+        cutensorHandle_t* handle,
+        void* alpha,
+        void* A, cutensorTensorDescriptor_t* descA, int32_t* modeA,
+        void* gamma,
+        void* C, cutensorTensorDescriptor_t* descC, int32_t* modeC,
+        void* D, cutensorTensorDescriptor_t* descD, int32_t* modeD,
+        Operator otAC,
+        DataType typeScalar, driver.Stream stream)
+
+    int cutensorPermutation(
+        cutensorHandle_t* handle,
+        void* alpha,
+        void* A, cutensorTensorDescriptor_t* descA, int32_t* modeA,
+        void* B, cutensorTensorDescriptor_t* descB, int32_t* modeB,
+        DataType typeScalar, driver.Stream stream)
+
+    int cutensorInitContractionDescriptor(
+        cutensorHandle_t* handle,
+        cutensorContractionDescriptor_t* desc,
+        cutensorTensorDescriptor_t* descA,
+        int32_t* modeA,
+        uint32_t alignmentReqA,
+        cutensorTensorDescriptor_t* descB,
+        int32_t* modeB,
+        uint32_t alignmentReqB,
+        cutensorTensorDescriptor_t* descC,
+        int32_t* modeC,
+        uint32_t alignmentReqC,
+        cutensorTensorDescriptor_t* descD,
+        int32_t* modeD,
+        uint32_t alignmentReqD,
+        ComputeType typeCompute)
+
+    int cutensorInitContractionFind(
+        cutensorHandle_t* handle,
+        cutensorContractionFind_t* find,
+        Algo algo)
+
+    int cutensorContractionGetWorkspaceSize(
+        cutensorHandle_t* handle,
+        cutensorContractionDescriptor_t* desc,
+        cutensorContractionFind_t* find,
+        WorksizePreference pref,
+        uint64_t *workspaceSize)
+
+    int cutensorContractionGetWorkspace(
+        cutensorHandle_t* handle,
+        cutensorContractionDescriptor_t* desc,
+        cutensorContractionFind_t* find,
+        WorksizePreference pref,
+        uint64_t *workspaceSize)
+
+    int cutensorInitContractionPlan(
+        cutensorHandle_t* handle,
+        cutensorContractionPlan_t* plan,
+        cutensorContractionDescriptor_t* desc,
+        cutensorContractionFind_t* find,
+        uint64_t workspaceSize)
+
+    int cutensorContraction(
+        cutensorHandle_t* handle,
+        cutensorContractionPlan_t* plan,
+        void* alpha, void* A, void* B, void* beta, void* C, void* D,
+        void *workspace, uint64_t workspaceSize, driver.Stream stream)
+
+    int cutensorContractionMaxAlgos(int32_t* maxNumAlgos)
+
+    int cutensorReduction(
+        cutensorHandle_t* handle,
+        void* alpha,
+        void* A, cutensorTensorDescriptor_t* descA, int32_t* modeA,
+        void* beta,
+        void* C, cutensorTensorDescriptor_t* descC, int32_t* modeC,
+        void* D, cutensorTensorDescriptor_t* descD, int32_t* modeD,
+        Operator opReduce, ComputeType typeCompute,
+        void* workspace, uint64_t workspaceSize,
+        driver.Stream stream)
+
+    int cutensorReductionGetWorkspaceSize(
+        cutensorHandle_t* handle,
+        void* A, cutensorTensorDescriptor_t* descA, int32_t* modeA,
+        void* C, cutensorTensorDescriptor_t* descC, int32_t* modeC,
+        void* D, cutensorTensorDescriptor_t* descD, int32_t* modeD,
+        Operator opReduce, ComputeType typeCompute,
+        uint64_t* workspaceSize)
+
+    int cutensorReductionGetWorkspace(
+        cutensorHandle_t* handle,
+        void* A, cutensorTensorDescriptor_t* descA, int32_t* modeA,
+        void* C, cutensorTensorDescriptor_t* descC, int32_t* modeC,
+        void* D, cutensorTensorDescriptor_t* descD, int32_t* modeD,
+        Operator opReduce, ComputeType typeCompute,
+        uint64_t* workspaceSize)
+
+    int cutensorGetAlignmentRequirement(
+        cutensorHandle_t* handle,
+        void* ptr,
+        cutensorTensorDescriptor_t* desc,
+        uint32_t* alignmentReq)
+
+    size_t cutensorGetVersion()
+
+    # build-time version
+    int CUTENSOR_VERSION
+
+
+available = True
+
+
+###############################################################################
+# Version information
+###############################################################################
+cpdef size_t get_version():
+    return cutensorGetVersion()
+
+
+###############################################################################
+# Classes
+###############################################################################
+
+cdef class Handle:
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cutensorHandle_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+cdef class TensorDescriptor:
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cutensorTensorDescriptor_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+cdef class ContractionDescriptor:
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cutensorContractionDescriptor_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+cdef class ContractionFind:
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cutensorContractionFind_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+cdef class ContractionPlan:
+
+    def __init__(self):
+        self._ptr = PyMem_Malloc(sizeof(cutensorContractionPlan_t))
+
+    def __dealloc__(self):
+        PyMem_Free(self._ptr)
+        self._ptr = NULL
+
+    @property
+    def ptr(self):
+        return <intptr_t>self._ptr
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+class CuTensorError(RuntimeError):
+
+    def __init__(self, int status):
+        self.status = status
+        msg = cutensorGetErrorString(<Status>status)
+        super(CuTensorError, self).__init__(msg.decode())
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cdef inline check_status(int status):
+    if status != STATUS_SUCCESS:
+        raise CuTensorError(status)
+
+
+###############################################################################
+# Handle initialization
+###############################################################################
+
+cpdef init(Handle handle):
+    """Initializes the cuTENSOR library"""
+    status = cutensorInit(<cutensorHandle_t*> handle._ptr)
+    check_status(status)
+
+
+###############################################################################
+# Tensor descriptor initialization
+###############################################################################
+
+cpdef initTensorDescriptor(
+        Handle handle,
+        TensorDescriptor desc,
+        uint32_t numModes,
+        intptr_t extent,
+        intptr_t stride,
+        int dataType,
+        int unaryOp):
+    """Initializes a tesnor descriptor
+
+    Args:
+        handle (Handle):
+            Opaque handle holding cuTENSOR's library context.
+        desc (TensorDescriptor):
+            Tensor descriptor.
+        numModes (uint32_t): number of modes
+        extent (int64_t*): extent of each mode (must be larger than zero)
+        stride (int64_t*): stride[i] denotes the displacement (stride)
+            between two consecutive elements in the ith-mode. This value may be
+            NULL, in which case a generalized column-major memory layout is
+            assumed (i.e., the strides increase monotonically from left to
+            right). Each stride must be larger than zero (a zero stride is
+            identical to removing this mode entirely).
+        dataType (cudaDataType_t): data type of stored entries.
+        unaryOp (cutensorOperator_t): Unary operator that will be applied to
+            each element of the corresponding tensor in a lazy fashion (i.e.,
+            the algorithm uses this tensor as its operand only once). The
+            original data of this tensor remains unchanged.
+    """
+    status = cutensorInitTensorDescriptor(
+        <cutensorHandle_t*> handle._ptr,
+        <cutensorTensorDescriptor_t*> desc._ptr,
+        numModes, <int64_t*> extent, <int64_t*> stride,
+        <DataType> dataType, <Operator> unaryOp)
+    check_status(status)
+
+
+###############################################################################
+# Tensor elementwise operations
+###############################################################################
+
+cpdef elementwiseTrinary(
+        Handle handle,
+        intptr_t alpha,
+        intptr_t A,
+        TensorDescriptor descA,
+        intptr_t modeA,
+        intptr_t beta,
+        intptr_t B,
+        TensorDescriptor descB,
+        intptr_t modeB,
+        intptr_t gamma,
+        intptr_t C,
+        TensorDescriptor descC,
+        intptr_t modeC,
+        intptr_t D,
+        TensorDescriptor descD,
+        intptr_t modeD,
+        int opAB,
+        int opABC,
+        int typeScalar):
+    """Element-wise tensor operation for three input tensors
+
+    This function performs a element-wise tensor operation of the form:
+
+        D_{Pi^C(i_0,i_1,...,i_nc)} =
+            Phi_ABC(Phi_AB(alpha * Psi_A(A_{Pi^A(i_0,i_1,...,i_na)}),
+                           beta  * Psi_B(B_{Pi^B(i_0,i_1,...,i_nb)})),
+                           gamma * Psi_C(C_{Pi^C(i_0,i_1,...,i_nc)}))
+
+    Where
+     - A,B,C,D are multi-mode tensors (of arbitrary data types).
+     - Pi^A, Pi^B, Pi^C are permutation operators that permute the modes of A,
+       B, and C respectively.
+     - Psi_A, Psi_B, Psi_C are unary element-wise operators (e.g., IDENTITY,
+       SQR, CONJUGATE).
+     - Phi_ABC, Phi_AB are binary element-wise operators (e.g., ADD, MUL, MAX,
+       MIN).
+
+    Notice that the broadcasting (of a mode) can be achieved by simply omitting
+    that mode from the respective tensor.
+
+    Moreover, modes may appear in any order giving the user a greater
+    flexibility. The only restrictions are:
+     - modes that appear in A or B must also appear in the output tensor as
+       such a case would correspond to a tensor contraction.
+     - each mode may appear in each tensor at most once.
+
+    It is guaranteed that an input tensor will not be read if the corresponding
+    scalar is zero.
+
+    Finally, the output tensor is padded with zeros, if the extent of the
+    vectorized mode ---of the output tensor--- is not a multiple of the
+    vector-width (the user has to ensure that the output tensor has sufficient
+    memory available to facilitate such a padding). Let the i'th mode be
+    vectorized with a vector-width w, then the additional padding is limited to
+    (w - (extent[i] % w)) many elements in total.
+
+    Examples:
+     - B_{a,b,c,d} = A_{b,d,a,c}
+     - C_{a,b,c,d} = 2.2 * A_{b,d,a,c} + 1.3 * B_{c,b,d,a}
+     - D_{a,b,c,d} = 2.2 * A_{b,d,a,c} + 1.3 * B_{c,b,d,a} + C_{a,b,c,d}
+     - D_{a,b,c,d} = min((2.2 * A_{b,d,a,c} + 1.3 * B_{c,b,d,a}), C_{a,b,c,d})
+
+    Args:
+        handle (Handle): Opaque handle holding CUTENSOR's library
+            context.
+        alpha (void*): Scaling factor for A (see equation above) of the type
+            typeCompute. Pointer to the host memory. Note that A is not read if
+            alpha is equal to zero, and the corresponding unary operator will
+            not be performed.
+        A (void*): Multi-mode tensor of type typeA with nmodeA modes. Pointer
+            to the GPU-accessable memory (while a host memory pointer is
+            acceptable, support for it remains an experimental feature).
+        descA (TensorDescriptor): A descriptor that holds the information
+            about the data type, modes, and strides of A.
+        modeA (int32_t*): Array (in host memory) of size descA->numModes that
+            holds the labels of the modes of A (e.g., if A_{a,b,c} => modeA =
+            {'a','b','c'}). The modeA[i] corresponds to extent[i] and stride[i]
+            w.r.t. the arguments provided to cutensorCreateTensorDescriptor.
+        beta (void*): Scaling factor for B (see equation above) of the type
+            typeCompute. Pointer to the host memory. Note that B is not read if
+            beta is equal to zero, and the corresponding unary operator will
+            not be performed.
+        B (void*): Multi-mode tensor of type typeB with nmodeB many modes.
+            Pointer to the GPU-accessable memory (while a host memory pointer
+            is acceptable, support for it remains an experimental feature).
+        descB (TensorDescriptor): The B descriptor that holds information
+            about the data type, modes, and strides of B.
+        modeB (int32_t*): Array (in host memory) of size descB->numModes that
+            holds the names of the modes of B. modeB[i] corresponds to
+            extent[i] and stride[i] of the cutensorCreateTensorDescriptor
+        gamma (void*): Scaling factor for C (see equation above) of type
+            typeCompute. Pointer to the host memory. Note that C is not read if
+            gamma is equal to zero, and the corresponding unary operator will
+            not be performed.
+        C (void*): Multi-mode tensor of type typeC with nmodeC many modes.
+            Pointer to the GPU-accessable memory (while a host memory pointer
+            is acceptable, support for it remains an experimental feature).
+        descC (TensorDescriptor): The C descriptor that holds information
+            about the data type, modes, and strides of C.
+        modeC (int32_t*): Array (in host memory) of size descC->numModes that
+            holds the names of the modes of C. The modeC[i] corresponds to
+            extent[i] and stride[i] of the cutensorCreateTensorDescriptor.
+        D (void*): Multi-mode output tensor of type typeC with nmodeC modes
+            that are ordered according to modeD. Pointer to the GPU-accessable
+            memory (while a host memory pointer is acceptable, support for it
+            remains an experimental feature). Notice that D may alias any input
+            tensor if they share the same memory layout (i.e., same tensor
+            descriptor).
+        descD (TensorDescriptor): The D descriptor that holds information
+            about the data type, modes, and strides of D. Notice that we
+            currently request descD and descC to be identical.
+        modeD (int32_t*): Array (in host memory) of size descD->numModes that
+            holds the names of the modes of D. The modeD[i] corresponds to
+            extent[i] and stride[i] of the cutensorCreateTensorDescriptor.
+        opAB (cutensorOperator_t): Element-wise binary operator
+            (see Phi_AB above).
+        opABC (cutensorOperator_t): Element-wise binary operator
+            (see Phi_ABC above).
+        typeScalar (cudaDataType_t): Compute type for the intermediate
+            computation.
+    """
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cutensorElementwiseTrinary(
+        <cutensorHandle_t*> handle._ptr,
+        <void*>alpha,
+        <void*>A,
+        <cutensorTensorDescriptor_t*> descA._ptr,
+        <int32_t*>modeA,
+        <void*>beta,
+        <void*>B,
+        <cutensorTensorDescriptor_t*> descB._ptr,
+        <int32_t*>modeB,
+        <void*>gamma,
+        <void*>C,
+        <cutensorTensorDescriptor_t*> descC._ptr,
+        <int32_t*>modeC,
+        <void*>D,
+        <cutensorTensorDescriptor_t*> descD._ptr,
+        <int32_t*>modeD,
+        <Operator>opAB,
+        <Operator>opABC,
+        <DataType>typeScalar,
+        <driver.Stream>stream)
+    check_status(status)
+
+
+cpdef elementwiseBinary(
+        Handle handle,
+        intptr_t alpha,
+        intptr_t A,
+        TensorDescriptor descA,
+        intptr_t modeA,
+        intptr_t gamma,
+        intptr_t C,
+        TensorDescriptor descC,
+        intptr_t modeC,
+        intptr_t D,
+        TensorDescriptor descD,
+        intptr_t modeD,
+        int opAC,
+        int typeScalar):
+    """Element-wise tensor operation for two input tensors
+
+    This function performs a element-wise tensor operation of the form:
+
+        D_{Pi^C(i_0,i_1,...,i_n)} =
+            Phi_AC(alpha * Psi_A(A_{Pi^A(i_0,i_1,...,i_n)}),
+                   gamma * Psi_C(C_{Pi^C(i_0,i_1,...,i_n)}))
+
+    See elementwiseTrinary() for details.
+    """
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cutensorElementwiseBinary(
+        <cutensorHandle_t*> handle._ptr,
+        <void*> alpha,
+        <void*> A,
+        <cutensorTensorDescriptor_t*> descA._ptr,
+        <int32_t*> modeA,
+        <void*> gamma,
+        <void*> C,
+        <cutensorTensorDescriptor_t*> descC._ptr,
+        <int32_t*> modeC,
+        <void*> D,
+        <cutensorTensorDescriptor_t*> descD._ptr,
+        <int32_t*> modeD,
+        <Operator> opAC,
+        <DataType> typeScalar,
+        <driver.Stream> stream)
+    check_status(status)
+
+
+cpdef permutation(
+        Handle handle,
+        intptr_t alpha,
+        intptr_t A,
+        TensorDescriptor descA,
+        intptr_t modeA,
+        intptr_t B,
+        TensorDescriptor descB,
+        intptr_t modeB,
+        int typeScalar):
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cutensorPermutation(
+        <cutensorHandle_t*> handle._ptr,
+        <void*> alpha,
+        <void*> A,
+        <cutensorTensorDescriptor_t*> descA._ptr,
+        <int32_t*> modeA,
+        <void*> B,
+        <cutensorTensorDescriptor_t*> descB._ptr,
+        <int32_t*> modeB,
+        <DataType> typeScalar,
+        <driver.Stream> stream)
+    check_status(status)
+
+
+###############################################################################
+# Tensor contraction
+###############################################################################
+
+cpdef initContractionDescriptor(
+        Handle handle,
+        ContractionDescriptor desc,
+        TensorDescriptor descA,
+        intptr_t modeA,
+        uint32_t alignmentRequirementA,
+        TensorDescriptor descB,
+        intptr_t modeB,
+        uint32_t alignmentRequirementB,
+        TensorDescriptor descC,
+        intptr_t modeC,
+        uint32_t alignmentRequirementC,
+        TensorDescriptor descD,
+        intptr_t modeD,
+        uint32_t alignmentRequirementD,
+        int computeType):
+    """Initializes tensor contraction descriptor.
+
+    Args:
+        handle (Handle): Opaque handle holding cuTENSOR's library
+            context.
+        desc (ContractionDescriptor): This opaque struct gets filled
+            with the information that encodes the tensor contraction problem.
+        descA (TensorDescriptor): A descriptor that holds the
+            information about the data type, modes and strides of A.
+        modeA (int32_t*): Array with 'nmodeA' entries that represent the modes
+            of A. The modeA[i] corresponds to extent[i] and stride[i] w.r.t.
+            the arguments provided to cutensorInitTensorDescriptor.
+        alighmentReqA (uint32_t): Alignment that cuTENSOR may require for
+            A's pointer (in bytes); you can use the helper function
+            cutensorGetAlignmentRequirement() to determine the best value for a
+            given pointer.
+        descB (TensorDescriptor): A descriptor that holds the
+            information about the data type, modes and strides of B.
+        modeB (int32_t*): Array with 'nmodeB' entries that represent the modes
+            of B. The modeB[i] corresponds to extent[i] and stride[i] w.r.t.
+            the arguments provided to cutensorInitTensorDescriptor.
+        alighmentReqB (uint32_t): Alignment that cuTENSOR may require for
+            B's pointer (in bytes); you can use the helper function
+            cutensorGetAlignmentRequirement() to determine the best value for a
+            given pointer.
+        descC (TensorDescriptor): A descriptor that holds the
+            information about the data type, modes and strides of C.
+        modeC (int32_t*): Array with 'nmodeC' entries that represent the modes
+            of C. The modeC[i] corresponds to extent[i] and stride[i] w.r.t.
+            the arguments provided to cutensorInitTensorDescriptor.
+        alighmentReqC (uint32_t): Alignment that cuTENSOR may require for
+            C's pointer (in bytes); you can use the helper function
+            cutensorGetAlignmentRequirement() to determine the best value for a
+            given pointer.
+        descD (TensorDescriptor): A descriptor that holds the
+            information about the data type, modes and strides of D.
+            (*) must be identical to descC for now.
+        modeD (int32_t*): Array with 'nmodeD' entries that represent the modes
+            of D. The modeD[i] corresponds to extent[i] and stride[i] w.r.t.
+            the arguments provided to cutensorInitTensorDescriptor.
+            (*) must be identical to modeD for now.
+        alighmentReqD (uint32_t): Alignment that cuTENSOR may require for
+            D's pointer (in bytes); you can use the helper function
+            cutensorGetAlignmentRequirement() to determine the best value for a
+            given pointer.
+        computeType (cutensorComputeType_t): Datatype of for the intermediate
+            computation of typeCompute T = A * B
+    """
+    status = cutensorInitContractionDescriptor(
+        <cutensorHandle_t*> handle._ptr,
+        <cutensorContractionDescriptor_t*> desc._ptr,
+        <cutensorTensorDescriptor_t*> descA._ptr,
+        <int32_t*> modeA,
+        alignmentRequirementA,
+        <cutensorTensorDescriptor_t*> descB._ptr,
+        <int32_t*> modeB,
+        alignmentRequirementB,
+        <cutensorTensorDescriptor_t*> descC._ptr,
+        <int32_t*> modeC,
+        alignmentRequirementC,
+        <cutensorTensorDescriptor_t*> descD._ptr,
+        <int32_t*> modeD,
+        alignmentRequirementD,
+        <ComputeType> computeType)
+    check_status(status)
+
+
+cpdef initContractionFind(
+        Handle handle,
+        ContractionFind find,
+        int algo):
+    """Limits the search space of viable candidates
+
+    This function gives the user finer control over the candidates that the
+    subsequent call to cutensorInitContractionPlan() is allowed to evaluate.
+
+    Args:
+        handle (Handle): Opaque handle holding cuTENSOR's library
+            context.
+        algo (cutensorAlgo_t): Allows users to select a specific algorithm.
+            CUTENSOR_ALGO_DEFAULT lets the heuristic choose the algorithm.
+            Any value >= 0 selects a specific GEMM-like algorithm and
+            deactivates the heuristic. If a specified algorithm is not
+            supported CUTENSOR_STATUS_NOT_SUPPORTED is returned.
+            See cutensorAlgo_t for additional choices.
+
+    Return:
+        find (cutensorContractionFind_t*):
+    """
+    status = cutensorInitContractionFind(
+        <cutensorHandle_t*> handle._ptr,
+        <cutensorContractionFind_t*> find._ptr,
+        <Algo> algo)
+    check_status(status)
+
+
+cpdef initContractionPlan(
+        Handle handle,
+        ContractionPlan plan,
+        ContractionDescriptor desc,
+        ContractionFind find,
+        uint64_t workspaceSize):
+    """Initializes the contraction plan
+
+    Args:
+        handle (Handle): Opaque handle holding cuTENSOR's library
+            context.
+        plan (ContractionPlan) Opaque handle holding the contraction
+            execution plan (i.e., the candidate that will be executed as well
+            as all it's runtime parameters for the given tensor contraction
+            problem).
+        desc (ContractionDescriptor) This opaque struct encodes the
+            given tensor contraction problem.
+        find (ContractionFind) This opaque struct is used to
+            restrict the search space of viable candidates.
+        workspaceSize (uint64_t) Available workspace size (in bytes).
+    """
+    status = cutensorInitContractionPlan(
+        <cutensorHandle_t*> handle._ptr,
+        <cutensorContractionPlan_t*> plan._ptr,
+        <cutensorContractionDescriptor_t*> desc._ptr,
+        <cutensorContractionFind_t*> find._ptr,
+        workspaceSize)
+    check_status(status)
+
+
+cpdef contraction(
+        Handle handle,
+        ContractionPlan plan,
+        intptr_t alpha,
+        intptr_t A,
+        intptr_t B,
+        intptr_t beta,
+        intptr_t C,
+        intptr_t D,
+        intptr_t workspace,
+        uint64_t workspaceSize):
+    """General tensor contraction
+
+    This routine computes the tensor contraction
+    D = alpha * Psi_A(A) * Psi_B(B) + beta * Psi_C(C).
+
+    Example:
+     - D_{a,b,c,d} = 1.3 * A_{b,e,d,f} * B_{f,e,a,c}
+
+    Args:
+        handle (Handle): Opaque handle holding CUTENSOR's library
+            context.
+        plan (ContractionPlan): Opaque handle holding the
+            contraction execution plan.
+        alpha (void*): Scaling for A*B. The data_type_t is determined by
+            'typeCompute'. Pointer to the host memory.
+        A (void*): Pointer to the data corresponding to A. Pointer to the
+            GPU-accessable memory.
+        B (void*): Pointer to the data corresponding to B. Pointer to the
+            GPU-accessable memory.
+        beta (void*): Scaling for C. The data_type_t is determined by
+            'typeCompute'. Pointer to the host memory.
+        C (void*): Pointer to the data corresponding to C. Pointer to the
+            GPU-accessable memory.
+        D (void*): Pointer to the data corresponding to D (must be identical
+            to C for now). Pointer to the GPU-accessable memory.
+        workspace (void*): Optional parameter that may be NULL. This pointer
+            provides additional workspace, in device memory, to the library for
+            additional optimizations.
+        workspaceSize (uint64_t): Size of the workspace array in bytes.
+    """
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cutensorContraction(
+        <cutensorHandle_t*> handle._ptr,
+        <cutensorContractionPlan_t*> plan._ptr,
+        <void*> alpha, <void*> A, <void*> B,
+        <void*> beta, <void*> C, <void*> D,
+        <void*> workspace, workspaceSize, <driver.Stream> stream)
+    check_status(status)
+
+
+cpdef uint64_t contractionGetWorkspaceSize(
+        Handle handle,
+        ContractionDescriptor desc,
+        ContractionFind find,
+        int pref):
+    """Determines the required workspaceSize for a given tensor contraction
+
+    Args:
+        handle (Handle): Opaque handle holding CUTENSOR's library
+            context.
+        desc (ContractionDescriptor): This opaque struct encodes the
+            given tensor contraction problem.
+        find (ContractionFind): This opaque struct restricts the
+            search space of viable candidates.
+        pref (cutensorWorksizePreference_t): User preference for the workspace.
+
+    Return:
+        workspaceSize (uint64_t): The workspace size (in bytes) that is
+            required for the given tensor contraction.
+    """
+    if CUTENSOR_VERSION < 10500:
+        raise RuntimeError(
+            'cutensorContractionGetWorkspaceSize is supported '
+            'since cuTENSOR 1.5')
+    cdef uint64_t workspaceSize = 0
+    status = cutensorContractionGetWorkspaceSize(
+        <cutensorHandle_t*> handle._ptr,
+        <cutensorContractionDescriptor_t*> desc._ptr,
+        <cutensorContractionFind_t*> find._ptr,
+        <WorksizePreference> pref,
+        &workspaceSize)
+    check_status(status)
+    return workspaceSize
+
+
+cpdef uint64_t contractionGetWorkspace(
+        Handle handle,
+        ContractionDescriptor desc,
+        ContractionFind find,
+        int pref):
+    """DEPRECATED: Determines the required workspaceSize for a given tensor contraction
+
+    Args:
+        handle (Handle): Opaque handle holding CUTENSOR's library
+            context.
+        desc (ContractionDescriptor): This opaque struct encodes the
+            given tensor contraction problem.
+        find (ContractionFind): This opaque struct restricts the
+            search space of viable candidates.
+        pref (cutensorWorksizePreference_t): User preference for the workspace.
+
+    Return:
+        workspaceSize (uint64_t): The workspace size (in bytes) that is
+            required for the given tensor contraction.
+    """  # NOQA
+    cdef uint64_t workspaceSize = 0
+    status = cutensorContractionGetWorkspace(
+        <cutensorHandle_t*> handle._ptr,
+        <cutensorContractionDescriptor_t*> desc._ptr,
+        <cutensorContractionFind_t*> find._ptr,
+        <WorksizePreference> pref,
+        &workspaceSize)
+    check_status(status)
+    return workspaceSize
+
+
+cpdef int32_t contractionMaxAlgos():
+    """Returns the maximum number of algorithms for cutensorContraction()
+
+    You can use the returned integer for auto-tuning purposes (i.e., iterate
+    overe all algorithms up to the returned value). Not all algorithms might be
+    applicable to your specific problem. cutensorContraction() will return
+    CUTENSOR_STATUS_NOT_SUPPORTED if an algorithm is not applicable.
+
+    Returns:
+        maxNumAlgos (int32_t): The maximum number of algorithms available for
+        cutensorContraction().
+    """
+    cdef int32_t maxNumAlgos = 0
+    status = cutensorContractionMaxAlgos(&maxNumAlgos)
+    check_status(status)
+    return maxNumAlgos
+
+
+###############################################################################
+# Tensor reduction
+###############################################################################
+
+cpdef reduction(
+        Handle handle,
+        intptr_t alpha,
+        intptr_t A,
+        TensorDescriptor descA,
+        intptr_t modeA,
+        intptr_t beta,
+        intptr_t C,
+        TensorDescriptor descC,
+        intptr_t modeC,
+        intptr_t D,
+        TensorDescriptor descD,
+        intptr_t modeD,
+        int opReduce,
+        int minTypeCompute,
+        intptr_t workspace,
+        uint64_t workspaceSize):
+    """Tensor reduction
+
+    This routine computes the tensor reduction of the form
+    D = alpha * opReduce(opA(A)) + beta * opC(C).
+
+    Example:
+     - D_{a,d} = 0.9 * A_{a,b,c,d} + 0.1 * C_{a,d}
+
+    Args:
+        handle (Handle handle): Opaque handle holding CUTENSOR's library
+            context.
+        alpha (void*): Scaling for A. The data_type_t is determined by
+            'typeCompute'. Pointer to the host memory.
+        A (void*): Pointer to the data corresponding to A. Pointer to the
+            GPU-accessable memory.
+        descA (TensorDescriptor): A descriptor that holds the information
+            about the data type, modes, strides and unary operator (opA) of A.
+        modeA (int32_t*): Array with 'nmodeA' entries that represent the modes
+            of A. The modeA[i] corresponds to extent[i] and stride[i] w.r.t.
+            the arguments provided to cutensorCreateTensorDescriptor.
+        beta (void*): Scaling for C. The data_type_t is determined by
+            'typeCompute'. Pointer to the host memory.
+        C (void*): Pointer to the data corresponding to C. Pointer to the
+            GPU-accessable memory.
+        modeC (int32_t*): Array with 'nmodeC' entries that represent the modes
+            of C. The modeC[i] corresponds to extent[i] and stride[i] w.r.t.
+            the arguments provided to cutensorCreateTensorDescriptor.
+        descC (TensorDescriptor): The C descriptor that holds information
+            about the data type, modes, strides and unary operator (opC) of C.
+        D (void*): Pointer to the data corresponding to D (must be identical
+            to C for now). Pointer to the GPU-accessable memory.
+        modeD (int32_t*): Array with 'nmodeD' entries that represent the modes
+            of D (must be identical to modeC for now). The modeD[i] corresponds
+            to extent[i] and stride[i] w.r.t. the arguments provided to
+            cutensorCreateTensorDescriptor.
+        descD (TensorDescriptor): The D descriptor that holds information
+            about the data type, modes, and strides of D (must be identical to
+            descC for now).
+        opReduce (cutensorOperator_t): Binary operator used to reduce elements
+            of A.
+        minTypeCompute (cutensorComputeType_t): All arithmetic is performed
+            usingthis data type (i.e., it affects the accuracy and
+            performance).
+        workspace (void*): Scratchpad (device) memory.
+        workspaceSize (uint64_t): Please use
+            cutensorReductionGetWorkspaceSize() to query the required
+            workspace. While lower values, including zero, are valid, they may
+            lead to grossly suboptimal performance.
+    """
+    cdef intptr_t stream = stream_module.get_current_stream_ptr()
+    status = cutensorReduction(
+        <cutensorHandle_t*> handle._ptr,
+        <void*> alpha,
+        <void*> A, <cutensorTensorDescriptor_t*> descA._ptr, <int32_t*> modeA,
+        <void*> beta,
+        <void*> C, <cutensorTensorDescriptor_t*> descC._ptr, <int32_t*> modeC,
+        <void*> D, <cutensorTensorDescriptor_t*> descD._ptr, <int32_t*> modeD,
+        <Operator> opReduce, <ComputeType> minTypeCompute,
+        <void*> workspace, workspaceSize, <driver.Stream> stream)
+    check_status(status)
+
+
+cpdef uint64_t reductionGetWorkspaceSize(
+        Handle handle,
+        intptr_t A,
+        TensorDescriptor descA,
+        intptr_t modeA,
+        intptr_t C,
+        TensorDescriptor descC,
+        intptr_t modeC,
+        intptr_t D,
+        TensorDescriptor descD,
+        intptr_t modeD,
+        int opReduce,
+        int typeCompute):
+    """Determines the required workspaceSize for a given tensor reduction
+
+    Args:
+        See reduction() about args.
+
+    Returns:
+        workspaceSize (uint64_t): The workspace size (in bytes) that is
+        required for the given tensor reduction.
+    """
+    if CUTENSOR_VERSION < 10500:
+        raise RuntimeError(
+            'cutensorReductionGetWorkspaceSize is supported '
+            'since cuTENSOR 1.5')
+    cdef uint64_t workspaceSize = 0
+    status = cutensorReductionGetWorkspace(
+        <cutensorHandle_t*> handle._ptr,
+        <void*> A, <cutensorTensorDescriptor_t*> descA._ptr, <int32_t*> modeA,
+        <void*> C, <cutensorTensorDescriptor_t*> descC._ptr, <int32_t*> modeC,
+        <void*> D, <cutensorTensorDescriptor_t*> descD._ptr, <int32_t*> modeD,
+        <Operator> opReduce, <ComputeType> typeCompute, &workspaceSize)
+    check_status(status)
+    return workspaceSize
+
+
+cpdef uint64_t reductionGetWorkspace(
+        Handle handle,
+        intptr_t A,
+        TensorDescriptor descA,
+        intptr_t modeA,
+        intptr_t C,
+        TensorDescriptor descC,
+        intptr_t modeC,
+        intptr_t D,
+        TensorDescriptor descD,
+        intptr_t modeD,
+        int opReduce,
+        int typeCompute):
+    """DEPRECATED: Determines the required workspaceSize for a given tensor reduction
+
+    Args:
+        See reduction() about args.
+
+    Returns:
+        workspaceSize (uint64_t): The workspace size (in bytes) that is
+        required for the given tensor reduction.
+    """  # NOQA
+    cdef uint64_t workspaceSize = 0
+    status = cutensorReductionGetWorkspace(
+        <cutensorHandle_t*> handle._ptr,
+        <void*> A, <cutensorTensorDescriptor_t*> descA._ptr, <int32_t*> modeA,
+        <void*> C, <cutensorTensorDescriptor_t*> descC._ptr, <int32_t*> modeC,
+        <void*> D, <cutensorTensorDescriptor_t*> descD._ptr, <int32_t*> modeD,
+        <Operator> opReduce, <ComputeType> typeCompute, &workspaceSize)
+    check_status(status)
+    return workspaceSize
+
+
+cpdef uint32_t getAlignmentRequirement(
+        Handle handle,
+        intptr_t ptr,
+        TensorDescriptor desc):
+    """Computes the minimal alignment requirement for a given pointer and
+       descriptor
+
+    Args:
+        handle (Handle): Opaque handle holding CUTENSOR's library
+            context.
+        ptr (const void*): Raw pointer to the data of the respective tensor.
+        desc (TensorDescriptor): Tensor descriptor for ptr.
+
+    Return:
+        alignmentRequirement (uint32_t): Largest alignment requirement that ptr
+            can fulfill (in bytes).
+    """
+    cdef uint32_t alignmentRequirement = 0
+    status = cutensorGetAlignmentRequirement(
+        <cutensorHandle_t*> handle._ptr,
+        <void*> ptr,
+        <cutensorTensorDescriptor_t*> desc._ptr,
+        &alignmentRequirement)
+    check_status(status)
+    return alignmentRequirement
diff --git a/cupy_backends/cuda/libs/nccl.pxd b/cupy_backends/cuda/libs/nccl.pxd
new file mode 100644
index 0000000..742887b
--- /dev/null
+++ b/cupy_backends/cuda/libs/nccl.pxd
@@ -0,0 +1,25 @@
+"""
+Wrapper for NCCL: Optimized primiteive for collective multi-GPU communication
+"""
+cpdef enum:
+    NCCL_SUM = 0
+    NCCL_PROD = 1
+    NCCL_MAX = 2
+    NCCL_MIN = 3
+    NCCL_AVG = 4
+
+    NCCL_INT8 = 0
+    NCCL_CHAR = 0
+    NCCL_UINT8 = 1
+    NCCL_INT32 = 2
+    NCCL_INT = 2
+    NCCL_UINT32 = 3
+    NCCL_INT64 = 4
+    NCCL_UINT64 = 5
+    NCCL_FLOAT16 = 6
+    NCCL_HALF = 6
+    NCCL_FLOAT32 = 7
+    NCCL_FLOAT = 7
+    NCCL_FLOAT64 = 8
+    NCCL_DOUBLE = 8
+    NCCL_BFLOAT16 = 9
diff --git a/cupy_backends/cuda/libs/nccl.pyx b/cupy_backends/cuda/libs/nccl.pyx
new file mode 100644
index 0000000..5dd2b9c
--- /dev/null
+++ b/cupy_backends/cuda/libs/nccl.pyx
@@ -0,0 +1,494 @@
+# distutils: language = c++
+
+"""
+Wrapper for NCCL: Optimized primiteive for collective multi-GPU communication
+"""
+cimport cython  # NOQA
+
+from libc.stdint cimport intptr_t
+from libcpp cimport vector
+
+from cupy_backends.cuda.api cimport driver
+from cupy_backends.cuda.api cimport runtime
+
+cdef extern from '../../cupy_nccl.h':
+    ctypedef struct ncclComm:
+        pass
+    ctypedef ncclComm* ncclComm_t
+    cdef enum:
+        NCCL_UNIQUE_ID_BYTES = 128
+    ctypedef struct ncclUniqueId:
+        char internal[NCCL_UNIQUE_ID_BYTES]
+    ctypedef enum ncclResult_t:
+        ncclSuccess
+    ctypedef enum ncclRedOp_t:
+        pass
+    ctypedef enum ncclDataType_t:
+        pass
+
+    const char* ncclGetErrorString(ncclResult_t result) nogil
+    ncclResult_t ncclGetVersion(int* version) nogil
+    ncclResult_t ncclCommGetAsyncError(ncclComm_t comm,
+                                       ncclResult_t *asyncError) nogil
+    ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId) nogil
+    ncclResult_t ncclCommInitRank(ncclComm_t* comm, int ndev,
+                                  ncclUniqueId commId, int rank) nogil
+    ncclResult_t ncclCommInitAll(ncclComm_t* comm, int ndev,
+                                 const int* devlist)
+    ncclResult_t ncclGroupStart() nogil
+    ncclResult_t ncclGroupEnd() nogil
+    void ncclCommDestroy(ncclComm_t comm) nogil
+    void ncclCommAbort(ncclComm_t comm) nogil
+    ncclResult_t ncclCommCuDevice(const ncclComm_t comm, int* device) nogil
+    ncclResult_t ncclCommUserRank(const ncclComm_t comm, int* rank) nogil
+    ncclResult_t ncclCommCount(const ncclComm_t comm, int* count) nogil
+    ncclResult_t _ncclAllReduce(const void* sendbuff, void* recvbuff,
+                                size_t count,
+                                ncclDataType_t datatype, ncclRedOp_t op,
+                                ncclComm_t comm, driver.Stream stream) nogil
+    ncclResult_t _ncclReduce(const void* sendbuff, void* recvbuf, size_t count,
+                             ncclDataType_t datatype, ncclRedOp_t op, int root,
+                             ncclComm_t comm, driver.Stream stream) nogil
+    ncclResult_t _ncclBroadcast(const void* sendbuff, void* recvbuff,
+                                size_t count, ncclDataType_t datatype,
+                                int root, ncclComm_t comm,
+                                driver.Stream stream) nogil
+    ncclResult_t _ncclBcast(void* buff, size_t count, ncclDataType_t datatype,
+                            int root, ncclComm_t comm,
+                            driver.Stream stream) nogil
+    ncclResult_t _ncclReduceScatter(const void* sendbuff,
+                                    void* recvbuff, size_t recvcount,
+                                    ncclDataType_t datatype, ncclRedOp_t op,
+                                    ncclComm_t comm,
+                                    driver.Stream stream) nogil
+    ncclResult_t _ncclAllGather(const void* sendbuff, void* recvbuff,
+                                size_t count, ncclDataType_t datatype,
+                                ncclComm_t comm, driver.Stream stream) nogil
+    ncclResult_t ncclSend(const void* sendbuff, size_t count,
+                          ncclDataType_t datatype, int peer, ncclComm_t comm,
+                          driver.Stream stream) nogil
+    ncclResult_t ncclRecv(void* recvbuff, size_t count,
+                          ncclDataType_t datatype, int peer, ncclComm_t comm,
+                          driver.Stream stream) nogil
+    # Build-time version
+    int NCCL_VERSION_CODE
+
+
+cdef dict ERROR1 = {
+    0: 'NCCL_ERROR_SUCCESS',
+    1: 'NCCL_ERROR_UNHANDLED_CUDA_ERROR',
+    2: 'NCCL_ERROR_SYSTEM_ERROR',
+    3: 'NCCL_ERROR_INTERNAL_ERROR',
+    4: 'NCCL_ERROR_INVALID_DEVICE_POINTER',
+    5: 'NCCL_ERROR_INVALID_RANK',
+    6: 'NCCL_ERROR_UNSUPPORTED_DEVICE_COUNT',
+    7: 'NCCL_ERROR_DEVICE_NOT_FOUND',
+    8: 'NCCL_ERROR_INVALID_DEVICE_INDEX',
+    9: 'NCCL_ERROR_LIB_WRAPPER_NOT_SET',
+    10: 'NCCL_ERROR_CUDA_MALLOC_FAILED',
+    11: 'NCCL_ERROR_RANK_MISMATCH',
+    12: 'NCCL_ERROR_INVALID_ARGUMENT',
+    13: 'NCCL_ERROR_INVALID_TYPE',
+    14: 'NCCL_ERROR_INVALID_OPERATION',
+}
+
+cdef dict ERROR2 = {
+    0: 'NCCL_ERROR_SUCCESS',
+    1: 'NCCL_ERROR_UNHANDLED_CUDA_ERROR',
+    2: 'NCCL_ERROR_SYSTEM_ERROR',
+    3: 'NCCL_ERROR_INTERNAL_ERROR',
+    4: 'NCCL_ERROR_INVALID_ARGUMENT',
+    5: 'NCCL_ERROR_INVALID_USAGE',
+}
+
+
+available = True
+
+
+class NcclError(RuntimeError):
+
+    def __init__(self, int status):
+        self.status = status
+        cdef const char* msg
+        with nogil:
+            msg = ncclGetErrorString(<ncclResult_t>status)
+        if NCCL_VERSION_CODE < 2000:
+            s = ERROR1[status]
+        else:
+            s = ERROR2[status]
+        super(NcclError, self).__init__(
+            '%s: %s' % (s, msg.decode()))
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(ncclResult_t status):
+    if status != ncclSuccess:
+        raise NcclError(status)
+
+
+def get_build_version():
+    return NCCL_VERSION_CODE
+
+
+def get_version():
+    """Returns the runtime version of NCCL.
+
+    This function will return 0 when built with NCCL version earlier than
+    2.3.4, which does not support ``ncclGetVersion`` API.
+    """
+    cdef int version
+    with nogil:
+        status = ncclGetVersion(&version)
+    check_status(status)
+    return version
+
+
+def get_unique_id():
+    cdef ncclUniqueId uniqueId
+    with nogil:
+        status = ncclGetUniqueId(&uniqueId)
+    check_status(status)
+    ret = tuple([<char>uniqueId.internal[i]
+                 for i in range(NCCL_UNIQUE_ID_BYTES)])
+    return ret
+
+
+def _bytesize(datatype):
+    bytesize = {NCCL_INT8: 1,
+                NCCL_UINT8: 1,
+                NCCL_INT32: 4,
+                NCCL_UINT32: 4,
+                NCCL_INT64: 8,
+                NCCL_UINT64: 8,
+                NCCL_FLOAT16: 2,
+                NCCL_FLOAT32: 4,
+                NCCL_FLOAT64: 8}
+    if datatype not in bytesize:
+        raise ValueError('Unknow datatype {}'.format(datatype))
+    return bytesize[datatype]
+
+
+cpdef groupStart():
+    """Start a group of NCCL calls. Must be paired with :func:`groupEnd()`.
+
+    .. note::
+        This method is useful when the ``NcclCommunicator`` instances are
+        created via :meth:`~.NcclCommunicator.initAll`. A typical usage pattern
+        is like this:
+
+        .. code-block:: python
+
+            comms = cupy.cuda.nccl.NcclCommunicator.initAll(n, dev_list)
+            # ... do some preparation work
+            cupy.cuda.nccl.groupStart()
+            for rank, comm in enumerate(comms):
+                # ... make some collective calls ...
+            cupy.cuda.nccl.groupEnd()
+
+        Other use cases include fusing several NCCL calls into one, and
+        point-to-point communications using :meth:`~.NcclCommunicator.send` and
+        :meth:`~.NcclCommunicator.recv` (with NCCL 2.7+).
+
+    .. seealso:: `ncclGroupStart`_, `Group Calls`_
+
+    .. _ncclGroupStart:
+        https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/group.html#ncclgroupstart
+
+    .. _Group calls:
+        https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/groups.html
+    """
+    if NCCL_VERSION_CODE < 2000:
+        raise RuntimeError('ncclGroupStart is not available in this version')
+    with nogil:
+        status = ncclGroupStart()
+    check_status(status)
+
+
+cpdef groupEnd():
+    """End a group of NCCL calls. Must be paired with :func:`groupStart()`.
+
+    .. note::
+        This method is useful when the ``NcclCommunicator`` instances are
+        created via :meth:`~.NcclCommunicator.initAll`. A typical usage pattern
+        is like this:
+
+        .. code-block:: python
+
+            comms = cupy.cuda.nccl.NcclCommunicator.initAll(n, dev_list)
+            # ... do some preparation work
+            cupy.cuda.nccl.groupStart()
+            for rank, comm in enumerate(comms):
+                # ... make some collective calls ...
+            cupy.cuda.nccl.groupEnd()
+
+        Other use cases include fusing several NCCL calls into one, and
+        point-to-point communications using :meth:`~.NcclCommunicator.send` and
+        :meth:`~.NcclCommunicator.recv` (with NCCL 2.7+).
+
+    .. seealso:: `ncclGroupEnd`_, `Group Calls`_
+
+    .. _ncclGroupEnd:
+        https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/group.html#ncclgroupend
+
+    .. _Group calls:
+        https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/groups.html
+    """
+    if NCCL_VERSION_CODE < 2000:
+        raise RuntimeError('ncclGroupEnd is not available in this version')
+    with nogil:
+        status = ncclGroupEnd()
+    check_status(status)
+
+
+cdef class NcclCommunicator:
+    """ Initialize an NCCL communicator for one device controlled by one
+    process.
+
+    Args:
+        ndev (int): Total number of GPUs to be used.
+        commId (tuple): The unique ID returned by :func:`get_unique_id`.
+        rank (int): The rank of the GPU managed by the current process.
+
+    Returns:
+        NcclCommunicator: An ``NcclCommunicator`` instance.
+
+    .. note::
+        This method is for creating an NCCL communicator in a multi-process
+        environment, typically managed by MPI or ``multiprocessing``. For
+        controlling multiple devices by one process, use :meth:`initAll`
+        instead.
+
+    .. seealso:: `ncclCommInitRank`_
+
+    .. _ncclCommInitRank:
+        https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/docs/api/comms.html#ncclcomminitrank
+    """  # noqa
+
+    cdef:
+        ncclComm_t _comm
+
+    def __cinit__(self):
+        self._comm = <ncclComm_t>0
+
+    def __init__(self, int ndev, tuple commId, int rank):
+        cdef ncclUniqueId _uniqueId
+        assert len(commId) == NCCL_UNIQUE_ID_BYTES
+        for i in range(NCCL_UNIQUE_ID_BYTES):
+            _uniqueId.internal[i] = commId[i]
+        with nogil:
+            status = ncclCommInitRank(&self._comm, ndev, _uniqueId, rank)
+        check_status(status)
+
+    def __dealloc__(self):
+        self.destroy()
+
+    @property
+    def comm(self):
+        return <intptr_t>self._comm
+
+    @staticmethod
+    def initAll(devices):
+        """ Initialize NCCL communicators for multiple devices in a single
+        process.
+
+        Args:
+            devices (int or list of int): The number of GPUs or a list of GPUs
+                to be used. For the former case, the first ``devices`` GPUs
+                will be used.
+
+        Returns:
+            list: A list of ``NcclCommunicator`` instances.
+
+        .. note::
+            This method is for creating a group of NCCL communicators, each
+            controlling one device, in a single process like this:
+
+            .. code-block:: python
+
+                from cupy.cuda import nccl
+                # Use 3 GPUs: #0, #2, and #3
+                comms = nccl.NcclCommunicator.initAll([0, 2, 3])
+                assert len(comms) == 3
+
+            In a multi-process setup, use the default initializer instead.
+
+        .. seealso:: `ncclCommInitAll`_
+
+        .. _ncclCommInitAll:
+            https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/docs/api/comms.html#ncclcomminitall
+        """  # noqa
+        cdef int i, ndev
+        cdef list comms = [], devlist = []
+        cdef NcclCommunicator comm
+
+        if isinstance(devices, list):
+            ndev = len(devices)
+            devlist = devices
+        elif isinstance(devices, int):
+            ndev = devices
+            for i in range(ndev):
+                devlist.append(i)
+        else:
+            raise ValueError("\"devices\" should be an int or a list of int.")
+
+        for i in range(ndev):
+            # Call to __new__ bypasses __init__ constructor
+            # these are just placeholders
+            comm = NcclCommunicator.__new__(NcclCommunicator)
+            comms.append(comm)
+        NcclCommunicator._initAll(comms, ndev, devlist)
+
+        return comms
+
+    @staticmethod
+    def _initAll(list comms, int ndev, list devlist=None):
+        # A helper function which does not return is favorable for subclassing
+        cdef vector.vector[int] devices
+        cdef vector.vector[ncclComm_t] ncclComms
+        cdef NcclCommunicator comm
+        cdef int* devices_ptr
+
+        if devlist is not None:
+            assert len(devlist) == ndev
+            for i in range(ndev):
+                devices.push_back(devlist[i])
+            devices_ptr = devices.data()
+        else:
+            devices_ptr = NULL
+        for i in range(ndev):
+            ncclComms.push_back(<ncclComm_t>0)
+        status = ncclCommInitAll(ncclComms.data(), ndev, devices_ptr)
+        check_status(status)
+        # overwrite the _comm attribute in existing NcclCommunicator instances
+        for i in range(ndev):
+            comm = comms[i]
+            comm._comm = ncclComms[i]
+
+    cpdef destroy(self):
+        if self._comm:
+            with nogil:
+                ncclCommDestroy(self._comm)
+            self._comm = <ncclComm_t>0
+
+    cpdef abort(self):
+        if NCCL_VERSION_CODE < 2400:
+            raise RuntimeError('ncclCommAbort is not available'
+                               ' in this version')
+        if self._comm:
+            with nogil:
+                ncclCommAbort(self._comm)
+            self._comm = <ncclComm_t>0
+
+    def device_id(self):
+        cdef int device_id
+        with nogil:
+            status = ncclCommCuDevice(self._comm, &device_id)
+        check_status(status)
+        return device_id
+
+    def rank_id(self):
+        cdef int rank_id
+        with nogil:
+            status = ncclCommUserRank(self._comm, &rank_id)
+        check_status(status)
+        return rank_id
+
+    def size(self):
+        cdef int ranks
+        with nogil:
+            status = ncclCommCount(self._comm, &ranks)
+        check_status(status)
+        return ranks
+
+    def allReduce(self, intptr_t sendbuf, intptr_t recvbuf,
+                  size_t count, int datatype, int op, intptr_t stream):
+        with nogil:
+            status = _ncclAllReduce(<void*>sendbuf, <void*>recvbuf,
+                                    count, <ncclDataType_t>datatype,
+                                    <ncclRedOp_t>op, self._comm,
+                                    <driver.Stream>stream)
+        check_status(status)
+
+    def reduce(self, intptr_t sendbuf, intptr_t recvbuf,
+               size_t count, int datatype, int op, int root, intptr_t stream):
+        with nogil:
+            status = _ncclReduce(<void*>sendbuf, <void*>recvbuf,
+                                 count, <ncclDataType_t>datatype,
+                                 <ncclRedOp_t>op, root, self._comm,
+                                 <driver.Stream>stream)
+        check_status(status)
+
+    def broadcast(self, intptr_t sendbuff, intptr_t recvbuff, int count,
+                  int datatype, int root, intptr_t stream):
+        if NCCL_VERSION_CODE < 2200:
+            # ncclBroadcast is not available in NCCL 2.1 or older.
+            if self.rank_id() == root and sendbuff != recvbuff:
+                runtime.memcpyAsync(recvbuff, sendbuff,
+                                    count * _bytesize(datatype),
+                                    runtime.memcpyDeviceToDevice, stream)
+            self.bcast(recvbuff, count, datatype, root, stream)
+            return
+        with nogil:
+            status = _ncclBroadcast(<const void*>sendbuff, <void*>recvbuff,
+                                    count, <ncclDataType_t>datatype, root,
+                                    self._comm, <driver.Stream>stream)
+        check_status(status)
+
+    def bcast(self, intptr_t buff, int count, int datatype,
+              int root, intptr_t stream):
+        with nogil:
+            status = _ncclBcast(<void*>buff, count,
+                                <ncclDataType_t>datatype, root,
+                                self._comm, <driver.Stream>stream)
+        check_status(status)
+
+    def reduceScatter(self, intptr_t sendbuf, intptr_t recvbuf,
+                      size_t recvcount, int datatype, int op, intptr_t stream):
+        with nogil:
+            status = _ncclReduceScatter(<void*>sendbuf, <void*>recvbuf,
+                                        recvcount, <ncclDataType_t>datatype,
+                                        <ncclRedOp_t>op, self._comm,
+                                        <driver.Stream>stream)
+        check_status(status)
+
+    def allGather(self, intptr_t sendbuf, intptr_t recvbuf, size_t count,
+                  int datatype, intptr_t stream):
+        with nogil:
+            status = _ncclAllGather(<void*>sendbuf, <void*>recvbuf,
+                                    count, <ncclDataType_t>datatype,
+                                    self._comm, <driver.Stream>stream)
+        check_status(status)
+
+    def send(self, intptr_t sendbuf, size_t count, int datatype, int peer,
+             intptr_t stream):
+        if NCCL_VERSION_CODE < 2700:
+            raise RuntimeError('ncclSend is not available in this version')
+        with nogil:
+            status = ncclSend(<void*>sendbuf, count, <ncclDataType_t>datatype,
+                              peer, self._comm, <driver.Stream>stream)
+        check_status(status)
+
+    def recv(self, intptr_t recvbuf, size_t count, int datatype, int peer,
+             intptr_t stream):
+        if NCCL_VERSION_CODE < 2700:
+            raise RuntimeError('ncclRecv is not available in this version')
+        with nogil:
+            status = ncclRecv(<void*>recvbuf, count, <ncclDataType_t>datatype,
+                              peer, self._comm, <driver.Stream>stream)
+        check_status(status)
+
+    def check_async_error(self):
+        if NCCL_VERSION_CODE < 2400:
+            raise RuntimeError('ncclCommGetAsyncError is not available'
+                               ' in this version')
+        cdef ncclResult_t asyncError = ncclSuccess
+        # Releasing GIL as the function *might* block in future and
+        # this won't be a hot code path. At least in NCCL 2.4 it does
+        # not block so far.
+        with nogil:
+            result = ncclCommGetAsyncError(self._comm, &asyncError)
+        check_status(asyncError)
+        check_status(result)
diff --git a/cupy_backends/cuda/libs/nvrtc.pxd b/cupy_backends/cuda/libs/nvrtc.pxd
new file mode 100644
index 0000000..1f4dc16
--- /dev/null
+++ b/cupy_backends/cuda/libs/nvrtc.pxd
@@ -0,0 +1,39 @@
+from libc.stdint cimport intptr_t
+
+
+###############################################################################
+# Types
+###############################################################################
+
+IF CUPY_USE_CUDA_PYTHON:
+    from cuda.cnvrtc cimport *
+    # Aliases for compatibillity with existing CuPy codebase.
+    # TODO(kmaehashi): Remove these aliases.
+    ctypedef nvrtcResult Result
+    ctypedef nvrtcProgram Program
+ELSE:
+    cdef extern from *:
+        ctypedef int Result 'nvrtcResult'
+        ctypedef void* Program 'nvrtcProgram'
+
+
+cpdef check_status(int status)
+
+cpdef tuple getVersion()
+cpdef tuple getSupportedArchs()
+
+
+###############################################################################
+# Program
+###############################################################################
+
+cpdef intptr_t createProgram(unicode src, unicode name, headers,
+                             include_names) except? 0
+cpdef destroyProgram(intptr_t prog)
+cpdef compileProgram(intptr_t prog, options)
+cpdef bytes getPTX(intptr_t prog)
+cpdef bytes getCUBIN(intptr_t prog)
+cpdef bytes getNVVM(intptr_t prog)
+cpdef unicode getProgramLog(intptr_t prog)
+cpdef addNameExpression(intptr_t prog, str name)
+cpdef str getLoweredName(intptr_t prog, str name)
diff --git a/cupy_backends/cuda/libs/nvrtc.pyx b/cupy_backends/cuda/libs/nvrtc.pyx
new file mode 100644
index 0000000..50a4af1
--- /dev/null
+++ b/cupy_backends/cuda/libs/nvrtc.pyx
@@ -0,0 +1,278 @@
+# distutils: language = c++
+
+"""Thin wrapper of NVRTC API.
+
+There are four differences compared to the original C API.
+
+1. Not all functions are ported.
+2. Errors are translated into NVRTCError exceptions.
+3. The 'nvrtc' prefix of each API is omitted and the next character is set to
+   lower case.
+4. The resulting values are returned directly instead of references.
+
+"""
+import sys as _sys  # no-cython-lint
+
+cimport cython  # NOQA
+from libcpp cimport vector
+
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda._softlink cimport SoftLink
+
+
+###############################################################################
+# Extern
+###############################################################################
+
+IF CUPY_USE_CUDA_PYTHON:
+    from cuda.cnvrtc cimport *
+ELSE:
+    cdef extern from '../../cupy_rtc.h' nogil:
+        const char *nvrtcGetErrorString(Result result)
+        int nvrtcVersion(int *major, int *minor)
+        int nvrtcCreateProgram(
+            Program* prog, const char* src, const char* name, int numHeaders,
+            const char** headers, const char** includeNames)
+        int nvrtcDestroyProgram(Program *prog)
+        int nvrtcCompileProgram(Program prog, int numOptions,
+                                const char** options)
+        int nvrtcGetPTXSize(Program prog, size_t *ptxSizeRet)
+        int nvrtcGetPTX(Program prog, char *ptx)
+        int nvrtcGetCUBINSize(Program prog, size_t *cubinSizeRet)
+        int nvrtcGetCUBIN(Program prog, char *cubin)
+        int nvrtcGetProgramLogSize(Program prog, size_t* logSizeRet)
+        int nvrtcGetProgramLog(Program prog, char* log)
+        int nvrtcAddNameExpression(Program, const char*)
+        int nvrtcGetLoweredName(Program, const char*, const char**)
+
+    ctypedef int (*f_type)(...) nogil  # NOQA
+    IF 11020 <= CUPY_CUDA_VERSION < 12000:
+        if _sys.platform == 'linux':
+            _libname = 'libnvrtc.so.11.2'
+        else:
+            _libname = 'nvrtc64_112_0.dll'
+    ELIF 12000 <= CUPY_CUDA_VERSION < 13000:
+        if _sys.platform == 'linux':
+            _libname = 'libnvrtc.so.12'
+        else:
+            _libname = 'nvrtc64_120_0.dll'
+    ELSE:
+        _libname = None
+
+    cdef SoftLink _lib = SoftLink(_libname, 'nvrtc')
+    # APIs added after CUDA 11.2+.
+    cdef f_type nvrtcGetNumSupportedArchs = <f_type>_lib.get('GetNumSupportedArchs')  # NOQA
+    cdef f_type nvrtcGetSupportedArchs = <f_type>_lib.get('GetSupportedArchs')  # NOQA
+    cdef f_type nvrtcGetNVVMSize = <f_type>_lib.get('GetNVVMSize')
+    cdef f_type nvrtcGetNVVM = <f_type>_lib.get('GetNVVM')
+
+
+###############################################################################
+# Error handling
+###############################################################################
+
+class NVRTCError(RuntimeError):
+
+    def __init__(self, status):
+        self.status = status
+        cdef bytes msg = nvrtcGetErrorString(<Result>status)
+        super(NVRTCError, self).__init__(
+            '{} ({})'.format(msg.decode(), status))
+
+    def __reduce__(self):
+        return (type(self), (self.status,))
+
+
+@cython.profile(False)
+cpdef inline check_status(int status):
+    if status != 0:
+        raise NVRTCError(status)
+
+
+cpdef tuple getVersion():
+    cdef int major, minor
+    with nogil:
+        status = nvrtcVersion(&major, &minor)
+    check_status(status)
+    return major, minor
+
+
+cpdef tuple getSupportedArchs():
+    cdef int status, num_archs
+    cdef vector.vector[int] archs
+    if runtime._is_hip_environment:
+        raise RuntimeError("HIP does not support getSupportedArchs")
+    if runtime.runtimeGetVersion() < 11020:
+        raise RuntimeError("getSupportedArchs is supported since CUDA 11.2")
+    with nogil:
+        status = nvrtcGetNumSupportedArchs(&num_archs)
+        if status == 0:
+            archs.resize(num_archs)
+            status = nvrtcGetSupportedArchs(archs.data())
+    check_status(status)
+    return tuple(archs)
+
+
+###############################################################################
+# Program
+###############################################################################
+
+cpdef intptr_t createProgram(unicode src, unicode name, headers,
+                             include_names) except? 0:
+    cdef Program prog
+    cdef bytes b_src = src.encode()
+    cdef const char* src_ptr = b_src
+    cdef bytes b_name = name.encode()
+    cdef const char* name_ptr
+    if len(name) > 0:
+        name_ptr = b_name
+    else:
+        name_ptr = NULL
+    cdef int num_headers = len(headers)
+    cdef vector.vector[const char*] header_vec
+    cdef vector.vector[const char*] include_name_vec
+    cdef const char** header_vec_ptr = NULL
+    cdef const char** include_name_vec_ptr = NULL
+    assert num_headers == len(include_names)
+    for i in headers:
+        header_vec.push_back(<const char*>i)
+    for i in include_names:
+        include_name_vec.push_back(<const char*>i)
+    if num_headers > 0:
+        header_vec_ptr = header_vec.data()
+        include_name_vec_ptr = include_name_vec.data()
+    with nogil:
+        status = nvrtcCreateProgram(
+            &prog, src_ptr, name_ptr, num_headers, header_vec_ptr,
+            include_name_vec_ptr)
+    check_status(status)
+    return <intptr_t>prog
+
+
+cpdef destroyProgram(intptr_t prog):
+    cdef Program p = <Program>prog
+    with nogil:
+        status = nvrtcDestroyProgram(&p)
+    check_status(status)
+
+
+cpdef compileProgram(intptr_t prog, options):
+    cdef int option_num = len(options)
+    cdef vector.vector[const char*] option_vec
+    cdef option_list = [opt.encode() for opt in options]
+    cdef const char** option_vec_ptr = NULL
+    for i in option_list:
+        option_vec.push_back(<const char*>i)
+    if option_num > 0:
+        option_vec_ptr = option_vec.data()
+    with nogil:
+        status = nvrtcCompileProgram(<Program>prog, option_num,
+                                     option_vec_ptr)
+    check_status(status)
+
+
+cpdef bytes getPTX(intptr_t prog):
+    cdef size_t ptxSizeRet
+    cdef vector.vector[char] ptx
+    cdef char* ptx_ptr = NULL
+    with nogil:
+        status = nvrtcGetPTXSize(<Program>prog, &ptxSizeRet)
+    check_status(status)
+    if ptxSizeRet == 0:
+        return b''
+    ptx.resize(ptxSizeRet)
+    ptx_ptr = ptx.data()
+    with nogil:
+        status = nvrtcGetPTX(<Program>prog, ptx_ptr)
+    check_status(status)
+
+    # Strip the trailing NULL.
+    return ptx_ptr[:ptxSizeRet-1]
+
+
+cpdef bytes getCUBIN(intptr_t prog):
+    cdef size_t cubinSizeRet = 0
+    cdef vector.vector[char] cubin
+    cdef char* cubin_ptr = NULL
+    if runtime._is_hip_environment:
+        raise RuntimeError("HIP does not support getCUBIN")
+    if runtime.runtimeGetVersion() < 11010:
+        raise RuntimeError("getCUBIN is supported since CUDA 11.1")
+    with nogil:
+        status = nvrtcGetCUBINSize(<Program>prog, &cubinSizeRet)
+    check_status(status)
+    if cubinSizeRet <= 1:
+        # On CUDA 11.1, cubinSizeRet=1 if -arch=compute_XX is used, but the
+        # spec says it should be 0 in this case...
+        raise RuntimeError('cubin is requested, but the real arch (sm_XX) is '
+                           'not provided')
+    cubin.resize(cubinSizeRet)
+    cubin_ptr = cubin.data()
+    with nogil:
+        status = nvrtcGetCUBIN(<Program>prog, cubin_ptr)
+    check_status(status)
+
+    # Strip the trailing NULL.
+    return cubin_ptr[:cubinSizeRet-1]
+
+
+cpdef bytes getNVVM(intptr_t prog):
+    if runtime._is_hip_environment:
+        raise RuntimeError("HIP does not support getNVVM")
+    if runtime.runtimeGetVersion() < 11040:
+        raise RuntimeError("getNVVM is supported since CUDA 11.4")
+
+    cdef size_t nvvmSizeRet = 0
+    cdef vector.vector[char] nvvm
+    cdef char* nvvm_ptr = NULL
+
+    with nogil:
+        status = nvrtcGetNVVMSize(<Program>prog, &nvvmSizeRet)
+    check_status(status)
+
+    nvvm.resize(nvvmSizeRet)
+    nvvm_ptr = nvvm.data()
+    with nogil:
+        status = nvrtcGetNVVM(<Program>prog, nvvm_ptr)
+    check_status(status)
+
+    # Strip the trailing NULL.
+    return nvvm_ptr[:nvvmSizeRet-1]
+
+
+cpdef unicode getProgramLog(intptr_t prog):
+    cdef size_t logSizeRet
+    cdef vector.vector[char] log
+    cdef char* log_ptr = NULL
+    with nogil:
+        status = nvrtcGetProgramLogSize(<Program>prog, &logSizeRet)
+    check_status(status)
+    if logSizeRet == 0:
+        return ''
+    log.resize(logSizeRet)
+    log_ptr = log.data()
+    with nogil:
+        status = nvrtcGetProgramLog(<Program>prog, log_ptr)
+    check_status(status)
+
+    # Strip the trailing NULL.
+    return log_ptr[:logSizeRet-1].decode('UTF-8')
+
+
+cpdef addNameExpression(intptr_t prog, str name):
+    cdef bytes b_name = name.encode()
+    cdef const char* c_name = b_name
+    with nogil:
+        status = nvrtcAddNameExpression(<Program>prog, c_name)
+    check_status(status)
+
+
+cpdef str getLoweredName(intptr_t prog, str name):
+    cdef bytes b_name = name.encode()
+    cdef const char* c_name = b_name
+    cdef const char* mangled_name
+    with nogil:
+        status = nvrtcGetLoweredName(<Program>prog, c_name, &mangled_name)
+    check_status(status)
+    b_name = mangled_name
+    return b_name.decode('UTF-8')
diff --git a/cupy_backends/cuda/libs/nvtx.pyx b/cupy_backends/cuda/libs/nvtx.pyx
new file mode 100644
index 0000000..8ecb9f7
--- /dev/null
+++ b/cupy_backends/cuda/libs/nvtx.pyx
@@ -0,0 +1,232 @@
+# distutils: language = c++
+
+"""
+Wrapper for NVIDIA Tools Extension Library (NVTX)
+
+"""
+from libc cimport string
+
+cdef extern from '../../cupy_tx.h' nogil:
+    cdef int NVTX_VERSION
+    cdef enum nvtxColorType_t:
+        NVTX_COLOR_UNKNOWN
+        NVTX_COLOR_ARGB
+    cdef enum nvtxMessageType_t:
+        NVTX_MESSAGE_UNKNOWN
+        NVTX_MESSAGE_TYPE_ASCII
+        NVTX_MESSAGE_TYPE_UNICODE
+    ctypedef   signed char         int8_t
+    ctypedef   signed short       int16_t
+    ctypedef   signed int         int32_t
+    ctypedef   signed long long   int64_t
+    ctypedef unsigned char        uint8_t
+    ctypedef unsigned short      uint16_t
+    ctypedef unsigned int        uint32_t
+    ctypedef unsigned long long  uint64_t
+    ctypedef          void        wchar_t
+    cdef union payload_t:
+        uint64_t  ullValue
+        int64_t    llValue
+        double      dValue
+    cdef union message_t:
+        char* ascii
+        wchar_t* unicode
+    cdef struct nvtxEventAttributes_v1:
+        uint16_t   version
+        uint16_t   size
+        uint32_t   category
+        int32_t    colorType
+        uint32_t   color
+        int32_t    payloadType
+        int32_t    reserved0
+        payload_t  payload
+        int32_t    messageType
+        message_t  message
+    ctypedef nvtxEventAttributes_v1 nvtxEventAttributes_t
+    ctypedef unsigned long long range_id_t
+    void nvtxMarkA(const char *message)
+    void nvtxMarkEx(const nvtxEventAttributes_t *eventAttrib)
+    int nvtxRangePushA(const char *message)
+    int nvtxRangePushEx(const nvtxEventAttributes_t *eventAttrib)
+    int nvtxRangePop()
+    range_id_t nvtxRangeStartEx(const nvtxEventAttributes_t *eventAttrib)
+    void nvtxRangeEnd(range_id_t)
+
+cdef int num_colors = 10
+cdef uint32_t colors[10]
+colors[0] = 0xFF00FF00
+colors[1] = 0xFF007FFF
+colors[2] = 0xFF7F00FF
+colors[3] = 0xFFFF0000
+colors[4] = 0xFF7FFF00
+colors[5] = 0xFF00FF7F
+colors[6] = 0xFF0000FF
+colors[7] = 0xFFFF007F
+colors[8] = 0xFFFF7F00
+colors[9] = 0xFF7F7F7F
+
+available = True
+
+
+cdef nvtxEventAttributes_t make_event_attributes(message, color):
+    cdef bytes b_message
+    cdef nvtxEventAttributes_t attrib
+
+    string.memset(&attrib, 0, sizeof(attrib))
+    attrib.version = NVTX_VERSION
+    attrib.size = sizeof(attrib)
+
+    if color is None:
+        attrib.colorType = NVTX_COLOR_UNKNOWN
+    else:
+        attrib.color = color
+        attrib.colorType = NVTX_COLOR_ARGB
+
+    if message is None:
+        attrib.messageType = NVTX_MESSAGE_UNKNOWN
+    else:
+        attrib.messageType = NVTX_MESSAGE_TYPE_ASCII
+        b_message = message.encode()
+        attrib.message.ascii = b_message
+
+    return attrib
+
+
+cpdef MarkC(message, uint32_t color=0):
+    """
+    Marks an instantaneous event (marker) in the application.
+
+    Markers are used to describe events at a specific time during execution of
+    the application.
+
+    Args:
+        message (str): Name of a marker.
+        color (uint32): Color code for a marker.
+    """
+    cdef bytes b_message = message.encode()
+    if NVTX_VERSION != 1 and NVTX_VERSION != 2:
+        nvtxMarkA(<const char*>b_message)
+        return
+
+    cdef nvtxEventAttributes_t attrib
+    string.memset(&attrib, 0, sizeof(attrib))
+    attrib.version = NVTX_VERSION
+    attrib.size = sizeof(attrib)
+    attrib.messageType = NVTX_MESSAGE_TYPE_ASCII
+    attrib.message.ascii = b_message
+    attrib.colorType = NVTX_COLOR_ARGB
+    attrib.color = color
+
+    nvtxMarkEx(&attrib)
+
+
+cpdef Mark(message, int id_color=-1):
+    """
+    Marks an instantaneous event (marker) in the application.
+
+    Markers are used to describe events at a specific time during execution of
+    the application.
+
+    Args:
+        message (str): Name of a marker.
+        id_color (int): ID of color for a marker.
+    """
+    cdef bytes b_message = message.encode()
+    if id_color < 0 or (NVTX_VERSION != 1 and NVTX_VERSION != 2):
+        nvtxMarkA(<const char*>b_message)
+        return
+
+    cdef uint32_t color = colors[id_color % num_colors]
+    MarkC(message, color)
+
+
+cpdef RangePushC(message, uint32_t color=0):
+    """
+    Starts a nested range.
+
+    Ranges are used to describe events over a time span during execution of the
+    application. This is particularly useful when profiling with Nsight Systems
+    to help connect user-specified ranges with CuPy's internal CUDA-kernels.
+    The duration of a range is defined by the corresponding pair of
+    ``RangePushC()`` to ``RangePop()`` calls, which can be nested.
+
+    .. code-block:: python
+
+        from cupy.cuda.nvtx import RangePushC, RangePop
+
+        RangePush("Nested Powers of A")
+        for i in range(N):
+            RangePushC("Iter {}: Double A".format(i))
+            A = 2*A
+            RangePop()
+        RangePop()
+
+    Args:
+        message (str): Name of a range.
+        color (uint32): ARGB color for a range.
+    """
+    cdef bytes b_message = message.encode()
+    if NVTX_VERSION != 1 and NVTX_VERSION != 2:
+        nvtxRangePushA(<const char*>b_message)
+        return
+
+    cdef nvtxEventAttributes_t attrib
+    string.memset(&attrib, 0, sizeof(attrib))
+    attrib.version = NVTX_VERSION
+    attrib.size = sizeof(attrib)
+    attrib.messageType = NVTX_MESSAGE_TYPE_ASCII
+    attrib.message.ascii = b_message
+    attrib.colorType = NVTX_COLOR_ARGB
+    attrib.color = color
+
+    nvtxRangePushEx(&attrib)
+
+
+cpdef RangePush(message, int id_color=-1):
+    """
+    Starts a nested range.
+
+    Ranges are used to describe events over a time span during execution of the
+    application. This is particularly useful when profiling with Nsight Systems
+    to help connect user-specified ranges with CuPy's internal CUDA-kernels.
+    The duration of a range is defined by the corresponding pair of
+    ``RangePush()`` to ``RangePop()`` calls, which can be nested.
+
+    .. code-block:: python
+
+        from cupy.cuda.nvtx import RangePush, RangePop
+
+        RangePush("Nested Powers of A")
+        for i in range(N):
+            RangePush("Iter {}: Double A".format(i))
+            A = 2*A
+            RangePop()
+        RangePop()
+
+    Args:
+        message (str): Name of a range.
+        id_color (int): ID of color for a range.
+    """
+    cdef bytes b_message = message.encode()
+    if id_color < 0 or (NVTX_VERSION != 1 and NVTX_VERSION != 2):
+        nvtxRangePushA(<const char*>b_message)
+        return
+
+    cdef uint32_t color = colors[id_color % num_colors]
+    RangePushC(message, color)
+
+
+cpdef RangePop():
+    """
+    Ends a nested range started by a ``RangePush*()`` call.
+    """
+    nvtxRangePop()
+
+
+cpdef unsigned long long RangeStart(message, color) except? 0:
+    cdef nvtxEventAttributes_t attrib = make_event_attributes(message, color)
+    return nvtxRangeStartEx(&attrib)
+
+
+cpdef RangeEnd(unsigned long long range_id):
+    nvtxRangeEnd(range_id)
diff --git a/cupy_backends/cuda/libs/profiler.pxd b/cupy_backends/cuda/libs/profiler.pxd
new file mode 100644
index 0000000..b1a012e
--- /dev/null
+++ b/cupy_backends/cuda/libs/profiler.pxd
@@ -0,0 +1,11 @@
+cdef extern from *:
+    ctypedef int OutputMode 'cudaOutputMode_t'
+
+
+cpdef enum:
+    cudaKeyValuePair = 0
+    cudaCSV = 1
+
+cpdef initialize(str config_file, str output_file, int output_mode)
+cpdef start()
+cpdef stop()
diff --git a/cupy_backends/cuda/libs/profiler.pyx b/cupy_backends/cuda/libs/profiler.pyx
new file mode 100644
index 0000000..db27ced
--- /dev/null
+++ b/cupy_backends/cuda/libs/profiler.pyx
@@ -0,0 +1,70 @@
+# distutils: language = c++
+
+"""Thin wrapper of cuda profiler."""
+from cupy_backends.cuda.api cimport runtime
+
+
+# TODO(kmaehashi): cudaProfilerInitialize is deprecated thus unsupported in
+# cudapython.
+cdef extern from '../../cupy_profiler.h' nogil:
+    ctypedef int cudaError_t
+
+    cudaError_t cudaProfilerInitialize(const char *configFile,
+                                       const char *outputFile,
+                                       int outputMode)
+    cudaError_t cudaProfilerStart()
+    cudaError_t cudaProfilerStop()
+
+
+cpdef initialize(str config_file,
+                 str output_file,
+                 int output_mode):
+    """Initialize the CUDA profiler.
+
+    This function initialize the CUDA profiler. See the CUDA document for
+    detail.
+
+    Args:
+        config_file (str): Name of the configuration file.
+        output_file (str): Name of the output file.
+        output_mode (int): ``cupy.cuda.profiler.cudaKeyValuePair`` or
+            ``cupy.cuda.profiler.cudaCSV``.
+
+    .. warning::
+       This API is marked as deprecated in CUDA 11, and has been removed
+       in CUDA 12.
+       This CuPy interface is subject to removal in future releases.
+    """
+    if 12000 <= runtime.runtimeGetVersion():
+        raise RuntimeError(
+            'cudaProfilerInitialize no longer available in CUDA 12+')
+    cdef bytes b_config_file = config_file.encode()
+    cdef bytes b_output_file = output_file.encode()
+    status = cudaProfilerInitialize(<const char*>b_config_file,
+                                    <const char*>b_output_file,
+                                    <OutputMode>output_mode)
+    runtime.check_status(status)
+
+
+cpdef start():
+    """Enable profiling.
+
+    A user can enable CUDA profiling. When an error occurs, it raises an
+    exception.
+
+    See the CUDA document for detail.
+    """
+    status = cudaProfilerStart()
+    runtime.check_status(status)
+
+
+cpdef stop():
+    """Disable profiling.
+
+    A user can disable CUDA profiling. When an error occurs, it raises an
+    exception.
+
+    See the CUDA document for detail.
+    """
+    status = cudaProfilerStop()
+    runtime.check_status(status)
diff --git a/cupy_backends/cuda/stream.pxd b/cupy_backends/cuda/stream.pxd
new file mode 100755
index 0000000..51e91fe
--- /dev/null
+++ b/cupy_backends/cuda/stream.pxd
@@ -0,0 +1,6 @@
+from libc.stdint cimport intptr_t
+
+cdef intptr_t get_current_stream_ptr()
+cdef set_current_stream_ptr(intptr_t ptr, int device_id=*)
+cpdef intptr_t get_default_stream_ptr()
+cdef bint is_ptds_enabled()
diff --git a/cupy_backends/cuda/stream.pyx b/cupy_backends/cuda/stream.pyx
new file mode 100755
index 0000000..ce83400
--- /dev/null
+++ b/cupy_backends/cuda/stream.pyx
@@ -0,0 +1,92 @@
+import os as _os
+import threading as _threading
+
+from cupy_backends.cuda.api cimport runtime
+
+
+cdef object _thread_local = _threading.local()
+
+
+cdef bint _ptds = bool(int(
+    _os.environ.get('CUPY_CUDA_PER_THREAD_DEFAULT_STREAM', '0')) != 0)
+
+
+cdef class _ThreadLocal:
+    cdef list current_stream  # list of intptr_t
+
+    def __init__(self):
+        cdef int i, num_devices = runtime.getDeviceCount()
+        self.current_stream = [0 for i in range(num_devices)]
+
+    @staticmethod
+    cdef _ThreadLocal get():
+        try:
+            tls = _thread_local.tls
+        except AttributeError:
+            tls = _thread_local.tls = _ThreadLocal()
+        return <_ThreadLocal>tls
+
+    cdef set_current_stream_ptr(self, intptr_t ptr, int device_id=-1):
+        if device_id == -1:
+            device_id = runtime.getDevice()
+        self.current_stream[device_id] = ptr
+
+    cdef intptr_t get_current_stream_ptr(self, int device_id=-1):
+        # Returns the stream previously set, otherwise returns
+        # nullptr or runtime.streamPerThread when
+        # CUPY_CUDA_PER_THREAD_DEFAULT_STREAM=1.
+        if device_id == -1:
+            device_id = runtime.getDevice()
+        cdef intptr_t curr_stream = self.current_stream[device_id]
+        if curr_stream == 0 and is_ptds_enabled():
+            return runtime.streamPerThread
+        return curr_stream
+
+
+cdef intptr_t get_current_stream_ptr():
+    """C API to get current CUDA stream pointer.
+
+    Returns:
+        intptr_t: The current CUDA stream pointer.
+    """
+    tls = _ThreadLocal.get()
+    return <intptr_t>tls.get_current_stream_ptr()
+
+
+cdef set_current_stream_ptr(intptr_t ptr, int device_id=-1):
+    """C API to set current CUDA stream pointer.
+
+    Args:
+        ptr (intptr_t): CUDA stream pointer.
+        device_id (int): device ID. Look up the current device if -1.
+
+    .. warning::
+
+        This method is intended to be called from `cupy.cuda.stream` module.
+        Do not call this method from somewhere else; this method only changes
+        the default stream for `cupy_backends.*`, so the stream used will be
+        inconsistent with the default one for `cupy.*`.
+
+    """
+    tls = _ThreadLocal.get()
+    tls.set_current_stream_ptr(ptr, device_id)
+
+
+# cpdef for unit testing
+cpdef intptr_t get_default_stream_ptr():
+    """Get the CUDA default stream pointer.
+
+    Returns:
+        intptr_t: CUDA stream pointer.
+    """
+    if is_ptds_enabled():
+        return runtime.streamPerThread
+    else:  # we don't return 0 here
+        return runtime.streamLegacy
+
+
+cdef bint is_ptds_enabled():
+    if runtime._is_hip_environment:
+        # HIP does not support PTDS, just ignore the env var
+        return False
+    return _ptds
diff --git a/cupy_backends/cupy_backend.h b/cupy_backends/cupy_backend.h
new file mode 100644
index 0000000..9a59b01
--- /dev/null
+++ b/cupy_backends/cupy_backend.h
@@ -0,0 +1,17 @@
+#ifndef INCLUDE_GUARD_CUPY_CUDA_H
+#define INCLUDE_GUARD_CUPY_CUDA_H
+
+#if CUPY_USE_HIP
+
+#include "hip/cupy_hip.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include "cuda/cupy_cuda.h"
+
+#else // #ifndef CUPY_NO_CUDA
+
+#include "stub/cupy_cuda.h"
+
+#endif // #ifndef CUPY_NO_CUDA
+#endif // #ifndef INCLUDE_GUARD_CUPY_CUDA_H
diff --git a/cupy_backends/cupy_backend_runtime.h b/cupy_backends/cupy_backend_runtime.h
new file mode 100644
index 0000000..4c375f5
--- /dev/null
+++ b/cupy_backends/cupy_backend_runtime.h
@@ -0,0 +1,17 @@
+#ifndef INCLUDE_GUARD_CUPY_CUDA_RUNTIME_H
+#define INCLUDE_GUARD_CUPY_CUDA_RUNTIME_H
+
+#if CUPY_USE_HIP
+
+#include "hip/cupy_hip_runtime.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include "cuda/cupy_cuda_runtime.h"
+
+#else // #ifndef CUPY_NO_CUDA
+
+#include "stub/cupy_cuda_runtime.h"
+
+#endif // #ifndef CUPY_NO_CUDA
+#endif // #ifndef INCLUDE_GUARD_CUPY_CUDA_RUNTIME_H
diff --git a/cupy_backends/cupy_blas.h b/cupy_backends/cupy_blas.h
new file mode 100644
index 0000000..a5334a8
--- /dev/null
+++ b/cupy_backends/cupy_blas.h
@@ -0,0 +1,17 @@
+#ifndef INCLUDE_GUARD_CUPY_CUBLAS_H
+#define INCLUDE_GUARD_CUPY_CUBLAS_H
+
+#if CUPY_USE_HIP
+
+#include "hip/cupy_hipblas.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include "cuda/cupy_cublas.h"
+
+#else // #ifndef CUPY_NO_CUDA
+
+#include "stub/cupy_cublas.h"
+
+#endif // #ifndef CUPY_NO_CUDA
+#endif // #ifndef INCLUDE_GUARD_CUPY_CUBLAS_H
diff --git a/cupy_backends/cupy_complex.h b/cupy_backends/cupy_complex.h
new file mode 100644
index 0000000..f52d386
--- /dev/null
+++ b/cupy_backends/cupy_complex.h
@@ -0,0 +1,17 @@
+#ifndef INCLUDE_GUARD_CUPY_COMPLEX_H
+#define INCLUDE_GUARD_CUPY_COMPLEX_H
+
+#ifdef CUPY_USE_HIP
+
+#include "hip/cupy_cuComplex.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include <cuComplex.h>
+
+#else // #if !defined(CUPY_NO_CUDA) || !defined(CUPY_USE_HIP)
+
+#include "stub/cupy_cuComplex.h"
+
+#endif // #ifndef CUPY_NO_CUDA
+#endif // #ifndef INCLUDE_GUARD_CUPY_COMPLEX_H
diff --git a/cupy_backends/cupy_cudnn.h b/cupy_backends/cupy_cudnn.h
new file mode 100644
index 0000000..a514f63
--- /dev/null
+++ b/cupy_backends/cupy_cudnn.h
@@ -0,0 +1,441 @@
+// This file is a stub header file of cudnn for Read the Docs.
+
+#ifndef INCLUDE_GUARD_CUPY_CUDNN_H
+#define INCLUDE_GUARD_CUPY_CUDNN_H
+
+#ifndef CUPY_NO_CUDA
+
+#include <cudnn.h>
+
+#elif defined(CUPY_NO_CUDA)
+
+#include "stub/cupy_cuda_common.h"
+#include "stub/cupy_cudnn.h"
+
+#else
+
+#include "hip/cupy_hip_common.h"
+#include "stub/cupy_cudnn.h"
+
+#endif // #ifdef CUPY_NO_CUDA
+
+
+///////////////////////////////////////////////////////////////////////////////
+// Definitions are for compatibility with cuDNN v5 and v6.
+///////////////////////////////////////////////////////////////////////////////
+
+extern "C" {
+
+#if defined(CUPY_NO_CUDA) || (CUDNN_VERSION < 6000)
+
+typedef enum {} cudnnRNNAlgo_t;
+typedef enum {} cudnnReduceTensorOp_t;
+typedef enum {} cudnnReduceTensorIndices_t;
+typedef enum {} cudnnIndicesType_t;
+
+typedef void* cudnnPersistentRNNPlan_t;
+typedef void* cudnnReduceTensorDescriptor_t;
+
+cudnnStatus_t cudnnCreatePersistentRNNPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetPersistentRNNPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyPersistentRNNPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetRNNDescriptor_v6(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+
+// Tensor reductions
+cudnnStatus_t cudnnCreateReduceTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetReduceTensorDescriptor(...){
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetReduceTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnDestroyReduceTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetReductionIndicesSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetReductionWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnReduceTensor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+#endif // #if defined(CUPY_NO_CUDA) || (CUDNN_VERSION < 6000)
+
+
+#if !defined(CUPY_NO_CUDA)
+// Some functions are renamed in cuDNN v5.
+// Following definitions are for compatibility with cuDNN v5 and higher.
+
+#define cudnnAddTensor_v3 cudnnAddTensor
+#define cudnnConvolutionBackwardData_v3 cudnnConvolutionBackwardData
+#define cudnnConvolutionBackwardFilter_v3 cudnnConvolutionBackwardFilter
+#define cudnnSetConvolutionNdDescriptor_v3 cudnnSetConvolutionNdDescriptor
+
+#endif // #if !defined(CUPY_NO_CUDA)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 6000)
+
+#define cudnnSetConvolution2dDescriptor_v4 cudnnSetConvolution2dDescriptor
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 6000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 6000)
+// Some functions are renamed in cuDNN v6.
+// Following definitions are for compatibility with cuDNN v6 and higher.
+
+#define cudnnSetFilter4dDescriptor_v4 cudnnSetFilter4dDescriptor
+#define cudnnSetFilterNdDescriptor_v4 cudnnSetFilterNdDescriptor
+#define cudnnGetFilterNdDescriptor_v4 cudnnGetFilterNdDescriptor
+#define cudnnSetPooling2dDescriptor_v4 cudnnSetPooling2dDescriptor
+#define cudnnSetPoolingNdDescriptor_v4 cudnnSetPoolingNdDescriptor
+#define cudnnActivationForward_v4 cudnnActivationForward
+#define cudnnActivationBackward_v4 cudnnActivationBackward
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 6000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+#define cudnnSetRNNDescriptor_v5 cudnnSetRNNDescriptor
+
+typedef enum {} cudnnMathType_t;
+typedef enum {} cudnnCTCLossAlgo_t;
+typedef void* cudnnCTCLossDescriptor_t;
+
+// CTC
+cudnnStatus_t cudnnCreateCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnDestroyCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnSetCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnGetCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnGetCTCLossWorkspaceSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnCTCLoss(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetConvolutionMathType(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionMathType(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetConvolutionGroupCount(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionGroupCount(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionForwardAlgorithm_v7(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardDataAlgorithm_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 7000)
+
+#define cudnnSetConvolution2dDescriptor_v5 cudnnSetConvolution2dDescriptor
+
+cudnnStatus_t cudnnSetConvolution2dDescriptor_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 7000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 6000)
+
+typedef enum {} cudnnDeterminism_t;
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 6000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+cudnnStatus_t cudnnFindConvolutionForwardAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnFindConvolutionBackwardFilterAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnFindConvolutionBackwardDataAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+typedef struct {
+  cudnnConvolutionFwdAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionFwdAlgoPerf_v7_t;
+
+typedef struct {
+  cudnnConvolutionBwdFilterAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionBwdFilterAlgoPerf_v7_t;
+
+typedef struct {
+  cudnnConvolutionBwdDataAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionBwdDataAlgoPerf_v7_t;
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 7000)
+
+#define cudnnFindConvolutionForwardAlgorithmEx_v7 cudnnFindConvolutionForwardAlgorithmEx
+#define cudnnFindConvolutionBackwardFilterAlgorithmEx_v7 cudnnFindConvolutionBackwardFilterAlgorithmEx
+#define cudnnFindConvolutionBackwardDataAlgorithmEx_v7 cudnnFindConvolutionBackwardDataAlgorithmEx
+
+#define cudnnConvolutionFwdAlgoPerf_v7_t cudnnConvolutionFwdAlgoPerf_t
+#define cudnnConvolutionBwdFilterAlgoPerf_v7_t cudnnConvolutionBwdFilterAlgoPerf_t
+#define cudnnConvolutionBwdDataAlgoPerf_v7_t cudnnConvolutionBwdDataAlgoPerf_t
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 7000)
+
+
+#if defined(CUPY_NO_CUDA) || (CUDNN_VERSION < 7200)
+
+typedef void* cudnnRNNDataDescriptor_t;
+
+typedef enum {} cudnnRNNDataLayout_t;
+typedef enum {} cudnnRNNPaddingMode_t;
+
+cudnnStatus_t cudnnSetRNNPaddingMode(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNPaddingMode(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateRNNDataDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyRNNDataDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetRNNDataDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNDataDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNForwardInferenceEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNForwardTrainingEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNBackwardDataEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNBackwardWeightsEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+#endif // defined(CUPY_NO_CUDA) || (CUDNN_VERSION < 7200)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 8000)
+// TODO: check function names when cuDNN 8 is released.
+
+#define cudnnGetConvolutionForwardAlgorithm_v6 cudnnGetConvolutionForwardAlgorithm
+#define cudnnGetConvolutionBackwardFilterAlgorithm_v6 cudnnGetConvolutionBackwardFilterAlgorithm
+#define cudnnGetConvolutionBackwardDataAlgorithm_v6 cudnnGetConvolutionBackwardDataAlgorithm
+
+#endif // #if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 8000)
+
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+typedef enum {} cudnnErrQueryMode_t;
+typedef struct cudnnRuntimeTag_t cudnnRuntimeTag_t;
+
+cudnnStatus_t cudnnQueryRuntimeError(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+#endif // !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7000)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7400)
+
+typedef enum {} cudnnBatchNormOps_t;
+
+cudnnStatus_t cudnnBatchNormalizationForwardTrainingEx(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnBatchNormalizationBackwardEx(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationBackwardExWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationTrainingExReserveSpaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+#endif // !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7400)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7600)
+
+// fused ops
+typedef void* cudnnFusedOpsConstParamPack_t;
+typedef void* cudnnFusedOpsVariantParamPack_t;
+typedef void* cudnnFusedOpsPlan_t;
+
+typedef enum {} cudnnFusedOps_t;
+typedef enum {} cudnnFusedOpsConstParamLabel_t;
+typedef enum {} cudnnFusedOpsPointerPlaceHolder_t;
+typedef enum {} cudnnFusedOpsVariantParamLabel_t;
+
+cudnnStatus_t cudnnCreateFusedOpsConstParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsConstParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetFusedOpsConstParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetFusedOpsConstParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateFusedOpsVariantParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsVariantParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetFusedOpsVariantParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetFusedOpsVariantParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnMakeFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFusedOpsExecute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+#endif // !defined(CUPY_NO_CUDA) && (CUDNN_VERSION < 7600)
+
+#if !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 8000)
+
+typedef enum {} cudnnConvolutionFwdPreference_t;
+typedef enum {} cudnnConvolutionBwdFilterPreference_t;
+typedef enum {} cudnnConvolutionBwdDataPreference_t;
+
+cudnnStatus_t cudnnGetConvolutionForwardAlgorithm_v6(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm_v6(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardDataAlgorithm_v6(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetRNNDescriptor_v5(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+#endif // !defined(CUPY_NO_CUDA) && (CUDNN_VERSION >= 8000)
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_CUPY_CUDNN_H
diff --git a/cupy_backends/cupy_cusparselt.h b/cupy_backends/cupy_cusparselt.h
new file mode 100644
index 0000000..76e8789
--- /dev/null
+++ b/cupy_backends/cupy_cusparselt.h
@@ -0,0 +1,18 @@
+#ifndef INCLUDE_GUARD_CUPY_CUSPARSELT_H
+#define INCLUDE_GUARD_CUPY_CUSPARSELT_H
+
+#ifdef CUPY_USE_HIP
+
+#include "stub/cupy_cusparselt.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include <cusparseLt.h>
+
+#else
+
+#include "stub/cupy_cusparselt.h"
+
+#endif // #ifndef CUPY_NO_CUDA
+
+#endif // #ifndef INCLUDE_GUARD_CUPY_CUSPARSELT_H
diff --git a/cupy_backends/cupy_cutensor.h b/cupy_backends/cupy_cutensor.h
new file mode 100644
index 0000000..ae7dbb2
--- /dev/null
+++ b/cupy_backends/cupy_cutensor.h
@@ -0,0 +1,20 @@
+#ifndef INCLUDE_GUARD_CUPY_CUTENSOR_H
+#define INCLUDE_GUARD_CUPY_CUTENSOR_H
+
+#ifdef CUPY_USE_HIP
+
+// Since ROCm/HIP does not have cuTENSOR, we simply include the stubs here
+// to avoid code dup.
+#include "stub/cupy_cutensor.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include "cuda/cupy_cutensor.h"
+
+#else
+
+#include "stub/cupy_cutensor.h"
+
+#endif // #ifndef CUPY_NO_CUDA
+
+#endif // #ifndef INCLUDE_GUARD_CUPY_CUTENSOR_H
diff --git a/cupy_backends/cupy_lapack.h b/cupy_backends/cupy_lapack.h
new file mode 100644
index 0000000..294c03b
--- /dev/null
+++ b/cupy_backends/cupy_lapack.h
@@ -0,0 +1,230 @@
+#ifndef INCLUDE_GUARD_CUPY_CUSOLVER_H
+#define INCLUDE_GUARD_CUPY_CUSOLVER_H
+
+#include <type_traits>
+
+#if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+
+#include "cuda/cupy_cusolver.h"
+
+#elif defined(CUPY_USE_HIP) // #if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+
+#include "hip/cupy_rocsolver.h"
+
+#else // #if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+
+#include "stub/cupy_cusolver.h"
+
+#endif // #if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+
+
+#if !defined(CUPY_USE_HIP)
+/*
+ * loop-based batched gesvd (only used on CUDA)
+ */
+template<typename T1, typename T2>
+using gesvd = cusolverStatus_t (*)(cusolverDnHandle_t, signed char, signed char, int, int, T1*, int, T2*, T1*, int, T1*, int, T1*, int, T2*, int*);
+
+template<typename T1, typename T2> struct gesvd_func { gesvd<T1, T2> ptr; };
+template<> struct gesvd_func<float, float> { gesvd<float, float> ptr = cusolverDnSgesvd; };
+template<> struct gesvd_func<double, double> { gesvd<double, double> ptr = cusolverDnDgesvd; };
+template<> struct gesvd_func<cuComplex, float> { gesvd<cuComplex, float> ptr = cusolverDnCgesvd; };
+template<> struct gesvd_func<cuDoubleComplex, double> { gesvd<cuDoubleComplex, double> ptr = cusolverDnZgesvd; };
+
+template<typename T>
+int gesvd_loop(
+        intptr_t handle, char jobu, char jobvt, int m, int n, intptr_t a_ptr,
+        intptr_t s_ptr, intptr_t u_ptr, intptr_t vt_ptr,
+        intptr_t w_ptr, int buffersize, intptr_t info_ptr,
+        int batch_size) {
+    /*
+     * Assumptions:
+     * 1. the stream is set prior to calling this function
+     * 2. the workspace is reused in the loop
+     */
+
+    cusolverStatus_t status;
+    int k = (m<n?m:n);
+    typedef typename std::conditional<(std::is_same<T, float>::value) || (std::is_same<T, cuComplex>::value), float,
+                                      /* double or cuDoubleComplex */ double>::type real_type;
+    T* A = reinterpret_cast<T*>(a_ptr);
+    real_type* S = reinterpret_cast<real_type*>(s_ptr);
+    T* U = reinterpret_cast<T*>(u_ptr);
+    T* VT = reinterpret_cast<T*>(vt_ptr);
+    T* Work = reinterpret_cast<T*>(w_ptr);
+    int* devInfo = reinterpret_cast<int*>(info_ptr);
+
+    // we can't use "if constexpr" to do a compile-time branch selection as it's C++17 only,
+    // so we use custom traits instead
+    gesvd<T, real_type> func = gesvd_func<T, real_type>().ptr;
+
+    for (int i=0; i<batch_size; i++) {
+        // setting rwork to NULL as we don't need it
+        status = func(
+            reinterpret_cast<cusolverDnHandle_t>(handle), jobu, jobvt, m, n, A, m,
+            S, U, m, VT, n, Work, buffersize, NULL, devInfo);
+        if (status != 0) break;
+        A += m * n;
+        S += k;
+        U += (jobu=='A' ? m*m : (jobu=='S' ? m*k : /* jobu=='O' or 'N' */ 0));
+        VT += (jobvt=='A' ? n*n : (jobvt=='S' ? n*k : /* jobvt=='O' or 'N' */ 0));
+        devInfo += 1;
+    }
+    return status;
+}
+
+
+/*
+ * loop-based batched geqrf (only used on CUDA)
+ */
+template<typename T>
+using geqrf = cusolverStatus_t (*)(cusolverDnHandle_t, int, int, T*, int, T*, T*, int, int*);
+
+template<typename T> struct geqrf_func { geqrf<T> ptr; };
+template<> struct geqrf_func<float> { geqrf<float> ptr = cusolverDnSgeqrf; };
+template<> struct geqrf_func<double> { geqrf<double> ptr = cusolverDnDgeqrf; };
+template<> struct geqrf_func<cuComplex> { geqrf<cuComplex> ptr = cusolverDnCgeqrf; };
+template<> struct geqrf_func<cuDoubleComplex> { geqrf<cuDoubleComplex> ptr = cusolverDnZgeqrf; };
+
+template<typename T>
+int geqrf_loop(
+        intptr_t handle, int m, int n, intptr_t a_ptr, int lda,
+        intptr_t tau_ptr, intptr_t w_ptr,
+        int buffersize, intptr_t info_ptr,
+        int batch_size) {
+    /*
+     * Assumptions:
+     * 1. the stream is set prior to calling this function
+     * 2. the workspace is reused in the loop
+     */
+
+    cusolverStatus_t status;
+    int k = (m<n?m:n);
+    T* A = reinterpret_cast<T*>(a_ptr);
+    T* Tau = reinterpret_cast<T*>(tau_ptr);
+    T* Work = reinterpret_cast<T*>(w_ptr);
+    int* devInfo = reinterpret_cast<int*>(info_ptr);
+
+    // we can't use "if constexpr" to do a compile-time branch selection as it's C++17 only,
+    // so we use custom traits instead
+    geqrf<T> func = geqrf_func<T>().ptr;
+
+    for (int i=0; i<batch_size; i++) {
+        status = func(reinterpret_cast<cusolverDnHandle_t>(handle),
+                      m, n, A, lda, Tau, Work, buffersize, devInfo);
+        if (status != 0) break;
+        A += m * n;
+        Tau += k;
+        devInfo += 1;
+    }
+    return status;
+}
+
+#else
+
+template<typename T>
+int gesvd_loop(
+        intptr_t handle, char jobu, char jobvt, int m, int n, intptr_t a_ptr,
+        intptr_t s_ptr, intptr_t u_ptr, intptr_t vt_ptr,
+        intptr_t w_ptr, int buffersize, intptr_t info_ptr,
+        int batch_size) {
+    // we need a dummy stub for HIP as it's not used
+    return 0;
+}
+
+
+/*
+ * batched geqrf (only used on HIP)
+ */
+template<typename T>
+using geqrf = cusolverStatus_t (*)(cusolverDnHandle_t, int, int, T* const[], int, T*, long int, int);
+
+template<typename T> struct geqrf_func { geqrf<T> ptr; };
+template<> struct geqrf_func<float> { geqrf<float> ptr = rocsolver_sgeqrf_batched; };
+template<> struct geqrf_func<double> { geqrf<double> ptr = rocsolver_dgeqrf_batched; };
+// we need the correct func pointer here, so can't cast!
+template<> struct geqrf_func<rocblas_float_complex> { geqrf<rocblas_float_complex> ptr = rocsolver_cgeqrf_batched; };
+template<> struct geqrf_func<rocblas_double_complex> { geqrf<rocblas_double_complex> ptr = rocsolver_zgeqrf_batched; };
+
+template<typename T>
+int geqrf_loop(
+        intptr_t handle, int m, int n, intptr_t a_ptr, int lda,
+        intptr_t tau_ptr, intptr_t w_ptr,
+        int buffersize, intptr_t info_ptr,
+        int batch_size) {
+    /*
+     * Assumptions:
+     * 1. the stream is set prior to calling this function
+     * 2. ignore w_ptr, buffersize, and info_ptr as rocSOLVER does not need them
+     */
+
+    cusolverStatus_t status;
+
+    // we can't use "if constexpr" to do a compile-time branch selection as it's C++17 only,
+    // so we use custom traits instead
+    typedef typename std::conditional<
+        std::is_floating_point<T>::value,
+        T,
+        typename std::conditional<std::is_same<T, cuComplex>::value,
+                                  rocblas_float_complex,
+                                  rocblas_double_complex>::type
+        >::type data_type;
+    geqrf<data_type> func = geqrf_func<data_type>().ptr;
+    data_type* const* A = reinterpret_cast<data_type* const*>(a_ptr);
+    data_type* Tau = reinterpret_cast<data_type*>(tau_ptr);
+    int k = (m<n)?m:n;
+
+    // use rocSOLVER's batched geqrf
+    status = func((cusolverDnHandle_t)handle, m, n, A, lda, Tau, k, batch_size);
+
+    return status;
+}
+#endif // #if !defined(CUPY_USE_HIP)
+
+
+/*
+ * loop-based batched orgqr (used on both CUDA & HIP)
+ */
+template<typename T>
+using orgqr = cusolverStatus_t (*)(cusolverDnHandle_t, int, int, int, T*, int, const T*, T*, int, int*);
+
+template<typename T> struct orgqr_func { orgqr<T> ptr; };
+template<> struct orgqr_func<float> { orgqr<float> ptr = cusolverDnSorgqr; };
+template<> struct orgqr_func<double> { orgqr<double> ptr = cusolverDnDorgqr; };
+template<> struct orgqr_func<cuComplex> { orgqr<cuComplex> ptr = cusolverDnCungqr; };
+template<> struct orgqr_func<cuDoubleComplex> { orgqr<cuDoubleComplex> ptr = cusolverDnZungqr; };
+
+template<typename T>
+int orgqr_loop(
+        intptr_t handle, int m, int n, int k, intptr_t a_ptr, int lda,
+        intptr_t tau_ptr, intptr_t w_ptr,
+        int buffersize, intptr_t info_ptr,
+        int batch_size, int origin_n) {
+    /*
+     * Assumptions:
+     * 1. the stream is set prior to calling this function
+     * 2. the workspace is reused in the loop
+     */
+
+    cusolverStatus_t status;
+    T* A = reinterpret_cast<T*>(a_ptr);
+    const T* Tau = reinterpret_cast<const T*>(tau_ptr);
+    T* Work = reinterpret_cast<T*>(w_ptr);
+    int* devInfo = reinterpret_cast<int*>(info_ptr);
+
+    // we can't use "if constexpr" to do a compile-time branch selection as it's C++17 only,
+    // so we use custom traits instead
+    orgqr<T> func = orgqr_func<T>().ptr;
+
+    for (int i=0; i<batch_size; i++) {
+        status = func(reinterpret_cast<cusolverDnHandle_t>(handle),
+                      m, n, k, A, lda, Tau, Work, buffersize, devInfo);
+        if (status != 0) break;
+        A += m * origin_n;
+        Tau += k;
+        devInfo += 1;
+    }
+
+    return status;
+}
+#endif // #ifndef INCLUDE_GUARD_CUPY_CUSOLVER_H
diff --git a/cupy_backends/cupy_nccl.h b/cupy_backends/cupy_nccl.h
new file mode 100644
index 0000000..03efb75
--- /dev/null
+++ b/cupy_backends/cupy_nccl.h
@@ -0,0 +1,169 @@
+#ifndef INCLUDE_GUARD_CUPY_NCCL_H
+#define INCLUDE_GUARD_CUPY_NCCL_H
+
+#if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+
+#include "cuda/cupy_nccl.h"
+
+#elif defined(CUPY_USE_HIP)
+
+#include "hip/cupy_rccl.h"
+
+#else // #ifndef CUPY_NO_CUDA
+
+#include "stub/cupy_nccl.h"
+
+#endif
+
+#ifndef NCCL_MAJOR
+#define NCCL_MAJOR 1
+#define NCCL_MINOR 0
+#define NCCL_PATCH 0
+#endif
+
+#ifndef NCCL_VERSION_CODE
+#define NCCL_VERSION_CODE (NCCL_MAJOR * 1000 + NCCL_MINOR * 100 + NCCL_PATCH)
+#endif
+
+
+#if (NCCL_VERSION_CODE >= 2000)
+
+ncclDataType_t _get_proper_datatype(ncclDataType_t datatype) {
+    return datatype;
+}
+
+#else // #if (NCCL_VERSION_CODE >= 2000)
+
+#define NCCL_CHAR_V1 ncclChar
+#define NCCL_INT_V1 ncclInt
+#define NCCL_HALF_V1 ncclHalf
+#define NCCL_FLOAT_V1 ncclFloat
+#define NCCL_DOUBLE_v1 ncclDouble
+#define NCCL_INT64_v1 ncclInt64
+#define NCCL_UINT64_v1 ncclUint64
+#define NCCL_INVALID_TYPE_V1 nccl_NUM_TYPES
+
+static const ncclDataType_t TYPE2TYPE_V1[] = {
+    NCCL_CHAR_V1,         // ncclInt8, ncclChar
+    NCCL_INVALID_TYPE_V1, // ncclUint8
+    NCCL_INT_V1,          // ncclInt32, ncclInt
+    NCCL_INVALID_TYPE_V1, // ncclUint32
+    NCCL_INT64_v1,        // ncclInt64
+    NCCL_UINT64_v1,       // ncclUint64
+    NCCL_HALF_V1,         // ncclFloat16, ncclHalf
+    NCCL_FLOAT_V1,        // ncclFloat32, ncclFloat
+    NCCL_DOUBLE_v1        // ncclFloat64, ncclDouble
+};
+
+ncclDataType_t _get_proper_datatype(ncclDataType_t datatype) {
+    return TYPE2TYPE_V1[datatype];
+}
+
+#ifndef CUPY_NO_CUDA
+ncclResult_t ncclGroupStart() {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclGroupEnd() {
+    return ncclSuccess;
+}
+#endif // #ifndef CUPY_NO_CUDA
+#endif // #if (NCCL_VERSION_CODE < 2000)
+
+#if (NCCL_VERSION_CODE < 2200)
+// New function in 2.2
+ncclResult_t ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count,
+			   ncclDataType_t datatype, int root, ncclComm_t comm,
+			   cudaStream_t stream) {
+    return ncclSuccess;
+}
+#endif // #if (NCCL_VERSION_CODE < 2200)
+
+#if (NCCL_VERSION_CODE < 2304)
+
+ncclResult_t ncclGetVersion(int *version) {
+    *version = 0;
+    return ncclSuccess;
+}
+
+#endif // #if (NCCL_VERSION_CODE < 2304)
+
+ncclResult_t _ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count,
+                            ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
+                            cudaStream_t stream) {
+    ncclDataType_t _datatype = _get_proper_datatype(datatype);
+    return ncclAllReduce(sendbuff, recvbuff, count, _datatype, op, comm, stream);
+}
+
+
+ncclResult_t _ncclReduce(const void* sendbuff, void* recvbuff, size_t count,
+                         ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm,
+                         cudaStream_t stream) {
+    ncclDataType_t _datatype = _get_proper_datatype(datatype);
+    return ncclReduce(sendbuff, recvbuff, count, _datatype, op, root, comm, stream);
+}
+
+
+ncclResult_t _ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count,
+			    ncclDataType_t datatype, int root, ncclComm_t comm,
+			    cudaStream_t stream) {
+    ncclDataType_t _datatype = _get_proper_datatype(datatype);
+    return ncclBroadcast(sendbuff, recvbuff, count, _datatype, root, comm,  stream);
+}
+
+
+ncclResult_t _ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root,
+                        ncclComm_t comm, cudaStream_t stream) {
+    ncclDataType_t _datatype = _get_proper_datatype(datatype);
+    return ncclBcast(buff, count, _datatype, root, comm,  stream);
+}
+
+
+ncclResult_t _ncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount,
+                                ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
+                                cudaStream_t stream) {
+    ncclDataType_t _datatype = _get_proper_datatype(datatype);
+    return ncclReduceScatter(sendbuff, recvbuff, recvcount, _datatype, op, comm, stream);
+}
+
+
+ncclResult_t _ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount,
+                            ncclDataType_t datatype, ncclComm_t comm,
+                            cudaStream_t stream) {
+    ncclDataType_t _datatype = _get_proper_datatype(datatype);
+#if (NCCL_VERSION_CODE >= 2000)
+    return ncclAllGather(sendbuff, recvbuff, sendcount, _datatype, comm, stream);
+#else
+    return ncclAllGather(sendbuff, sendcount, _datatype, recvbuff, comm, stream);
+#endif // #if (NCCL_VERSION_CODE < 2000)
+}
+
+#if (NCCL_VERSION_CODE < 2400)
+// New functions in 2.4
+#define UNUSED(x) ((void)x)
+
+ncclResult_t ncclCommGetAsyncError(ncclComm_t comm, ncclResult_t *asyncError) {
+  UNUSED(comm);
+  UNUSED(asyncError);
+  return ncclSuccess;
+}
+
+void ncclCommAbort(ncclComm_t comm) {
+  UNUSED(comm);
+}
+#endif
+
+#if (NCCL_VERSION_CODE < 2700)
+// New functions in 2.7
+ncclResult_t ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype,
+                      int peer, ncclComm_t comm, cudaStream_t stream) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype,
+                      int peer, ncclComm_t comm, cudaStream_t stream) {
+    return ncclSuccess;
+}
+#endif
+
+#endif // #ifndef INCLUDE_GUARD_CUPY_NCCL_H
diff --git a/cupy_backends/cupy_profiler.h b/cupy_backends/cupy_profiler.h
new file mode 100644
index 0000000..87170ed
--- /dev/null
+++ b/cupy_backends/cupy_profiler.h
@@ -0,0 +1,17 @@
+#ifndef INCLUDE_GUARD_CUPY_PROFILER_H
+#define INCLUDE_GUARD_CUPY_PROFILER_H
+
+#if CUPY_USE_HIP
+
+#include "hip/cupy_profiler.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include "cuda/cupy_cuda_profiler_api.h"
+
+#else // #ifndef CUPY_NO_CUDA
+
+#include "stub/cupy_profiler.h"
+
+#endif // #ifndef CUPY_NO_CUDA
+#endif // #ifndef INCLUDE_GUARD_CUPY_PROFILER_H
diff --git a/cupy_backends/cupy_rand.h b/cupy_backends/cupy_rand.h
new file mode 100644
index 0000000..aeaa5dc
--- /dev/null
+++ b/cupy_backends/cupy_rand.h
@@ -0,0 +1,17 @@
+#ifndef INCLUDE_GUARD_CUPY_CURAND_H
+#define INCLUDE_GUARD_CUPY_CURAND_H
+
+#if CUPY_USE_HIP
+
+#include "hip/cupy_hiprand.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include <curand.h>
+
+#else // #ifndef CUPY_NO_CUDA
+
+#include "stub/cupy_curand.h"
+
+#endif // #ifndef CUPY_NO_CUDA
+#endif // #ifndef INCLUDE_GUARD_CUPY_CURAND_H
diff --git a/cupy_backends/cupy_rtc.h b/cupy_backends/cupy_rtc.h
new file mode 100644
index 0000000..c751e66
--- /dev/null
+++ b/cupy_backends/cupy_rtc.h
@@ -0,0 +1,18 @@
+#ifndef INCLUDE_GUARD_CUPY_NVRTC_H
+#define INCLUDE_GUARD_CUPY_NVRTC_H
+
+#ifdef CUPY_USE_HIP
+
+#include "hip/cupy_hiprtc.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#include "cuda/cupy_nvrtc.h"
+
+#else
+
+#include "stub/cupy_nvrtc.h"
+
+#endif
+
+#endif // #ifndef INCLUDE_GUARD_CUPY_NVRTC_H
diff --git a/cupy_backends/cupy_sparse.h b/cupy_backends/cupy_sparse.h
new file mode 100644
index 0000000..1cfea11
--- /dev/null
+++ b/cupy_backends/cupy_sparse.h
@@ -0,0 +1,21 @@
+#ifndef INCLUDE_GUARD_CUPY_CUSPARSE_H
+#define INCLUDE_GUARD_CUPY_CUSPARSE_H
+
+#if !defined(CUPY_NO_CUDA) && !defined(CUPY_USE_HIP)
+
+#include "cuda/cupy_cusparse.h"
+
+#elif defined(CUPY_USE_HIP)
+
+#include "hip/cupy_hip_common.h"
+#include "hip/cupy_hipsparse.h"
+
+#else
+
+#include "stub/cupy_cuda_common.h"
+#include "stub/cupy_cusparse.h"
+
+#endif  // #if defined(CUPY_NO_CUDA) || defined(CUPY_USE_HIP)
+
+
+#endif  // INCLUDE_GUARD_CUPY_CUSPARSE_H
diff --git a/cupy_backends/cupy_tx.h b/cupy_backends/cupy_tx.h
new file mode 100644
index 0000000..171da19
--- /dev/null
+++ b/cupy_backends/cupy_tx.h
@@ -0,0 +1,37 @@
+#ifndef INCLUDE_GUARD_CUPY_TX_H
+#define INCLUDE_GUARD_CUPY_TX_H
+
+#if CUPY_USE_HIP
+
+#include "hip/cupy_roctx.h"
+#include "stub/cupy_nvtx.h"
+
+#elif !defined(CUPY_NO_CUDA)
+
+#define NVTX_EXPORT_API
+#include <nvtx3/nvToolsExt.h>
+
+#else  // defined(CUPY_NO_CUDA)
+
+#define NVTX_VERSION 1
+
+extern "C" {
+
+void nvtxMarkA(...) {
+}
+
+int nvtxRangePushA(...) {
+    return 0;
+}
+
+int nvtxRangePop() {
+    return 0;
+}
+
+}
+
+#include "stub/cupy_nvtx.h"
+
+#endif
+
+#endif // #ifndef INCLUDE_GUARD_CUPY_TX_H
diff --git a/cupy_backends/hip/cupy_cuComplex.h b/cupy_backends/hip/cupy_cuComplex.h
new file mode 100644
index 0000000..d0e3c6d
--- /dev/null
+++ b/cupy_backends/hip/cupy_cuComplex.h
@@ -0,0 +1,20 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_COMPLEX_H
+#define INCLUDE_GUARD_HIP_CUPY_COMPLEX_H
+
+extern "C" {
+
+///////////////////////////////////////////////////////////////////////////////
+// cuComplex.h
+///////////////////////////////////////////////////////////////////////////////
+
+struct cuComplex{
+    float x, y;
+};
+
+struct cuDoubleComplex{
+    double x, y;
+};
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_COMPLEX_H
diff --git a/cupy_backends/hip/cupy_hip.h b/cupy_backends/hip/cupy_hip.h
new file mode 100644
index 0000000..fcdf297
--- /dev/null
+++ b/cupy_backends/hip/cupy_hip.h
@@ -0,0 +1,138 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_CUDA_H
+#define INCLUDE_GUARD_HIP_CUPY_CUDA_H
+
+#include "cupy_hip_common.h"
+#include "cupy_cuComplex.h"
+
+extern "C" {
+
+// Error handling
+CUresult cuGetErrorName(CUresult hipError, const char** pStr) {
+    *pStr = hipGetErrorName(hipError);
+    return CUDA_SUCCESS;
+}
+
+CUresult cuGetErrorString(CUresult hipError, const char** pStr) {
+    *pStr = hipGetErrorString(hipError);
+    return CUDA_SUCCESS;
+}
+
+// Primary context management
+CUresult cuDevicePrimaryCtxRelease(CUdevice dev) {
+    return hipDevicePrimaryCtxRelease(dev);
+}
+
+// Context management
+CUresult cuCtxGetCurrent(CUcontext *ctx) {
+    // deprecated api
+    //return hipCtxGetCurrent(ctx);
+    return hipErrorUnknown;
+}
+
+CUresult cuCtxSetCurrent(CUcontext ctx) {
+    // deprecated api
+    //return hipCtxSetCurrent(ctx);
+    return hipErrorUnknown;
+}
+
+CUresult cuCtxCreate(CUcontext* pctx, unsigned int flags, CUdevice dev) {
+    // deprecated api
+    //return hipCtxCreate(pctx, flags, dev);
+    return hipErrorUnknown;
+}
+
+CUresult cuCtxDestroy(CUcontext ctx) {
+    // deprecated api
+    // return hipCtxDestroy(ctx);
+    return hipErrorUnknown;
+}
+
+CUresult cuCtxGetDevice(...) {
+    return hipErrorUnknown;
+}
+
+// Module load and kernel execution
+CUresult cuLinkCreate(...) {
+    return hipErrorUnknown;
+}
+
+CUresult cuLinkAddData(...) {
+    return hipErrorUnknown;
+}
+
+CUresult cuLinkAddFile(...) {
+    return hipErrorUnknown;
+}
+
+CUresult cuLinkComplete(...) {
+    return hipErrorUnknown;
+}
+
+CUresult cuLinkDestroy(...) {
+    return hipErrorUnknown;
+}
+
+CUresult cuModuleLoad(CUmodule *module, const char *fname) {
+    return hipModuleLoad(module, fname);
+}
+
+CUresult cuModuleLoadData(CUmodule *module, const void *image) {
+    return hipModuleLoadData(module, image);
+}
+
+CUresult cuModuleUnload(CUmodule module) {
+    return hipModuleUnload(module);
+}
+
+CUresult cuModuleGetFunction(CUfunction *function, CUmodule module,
+                             const char *kname) {
+    return hipModuleGetFunction(function, module, kname);
+}
+
+CUresult cuModuleGetGlobal(CUdeviceptr *dptr, size_t *bytes, CUmodule hmod,
+                           const char *name) {
+    return hipModuleGetGlobal(dptr, bytes, hmod, name);
+}
+
+CUresult cuLaunchKernel(CUfunction f, uint32_t gridDimX, uint32_t gridDimY,
+                        uint32_t gridDimZ, uint32_t blockDimX,
+                        uint32_t blockDimY, uint32_t blockDimZ,
+                        uint32_t sharedMemBytes, cudaStream_t hStream,
+                        void **kernelParams, void **extra) {
+    return hipModuleLaunchKernel(f, gridDimX, gridDimY, gridDimZ,
+                                 blockDimX, blockDimY, blockDimZ,
+                                 sharedMemBytes, hStream, kernelParams, extra);
+}
+
+CUresult cuLaunchCooperativeKernel(...) {
+    return hipErrorUnknown;
+}
+
+// Function attribute
+CUresult cuFuncGetAttribute(int* pi, CUfunction_attribute attrib, CUfunction hfunc) {
+    return hipFuncGetAttribute(pi, attrib, hfunc);
+}
+
+CUresult cuFuncSetAttribute(...) {
+    return hipErrorUnknown;
+}
+
+// Occupancy
+typedef size_t (*CUoccupancyB2DSize)(int);
+
+CUresult cuOccupancyMaxActiveBlocksPerMultiprocessor(...) {
+    return hipErrorUnknown;
+}
+
+CUresult cuOccupancyMaxPotentialBlockSize(...) {
+    return hipErrorUnknown;
+}
+
+// Stream
+CUresult cuStreamGetCtx(...) {
+    return hipErrorUnknown;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_CUDA_H
diff --git a/cupy_backends/hip/cupy_hip_common.h b/cupy_backends/hip/cupy_hip_common.h
new file mode 100644
index 0000000..37b96cd
--- /dev/null
+++ b/cupy_backends/hip/cupy_hip_common.h
@@ -0,0 +1,156 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_COMMON_H
+#define INCLUDE_GUARD_HIP_CUPY_COMMON_H
+
+#include <hip/hip_runtime_api.h>
+#include <hipblas.h>
+#include <rocsolver.h>
+
+#define CUDA_VERSION 0
+
+extern "C" {
+
+///////////////////////////////////////////////////////////////////////////////
+// cuda.h
+///////////////////////////////////////////////////////////////////////////////
+
+typedef int CUdevice;
+typedef hipError_t CUresult;
+const CUresult CUDA_SUCCESS = static_cast<CUresult>(0);
+enum CUjit_option {};
+enum CUjitInputType {};
+enum CUarray_format {};
+enum CUaddress_mode {};
+enum CUfilter_mode {};
+
+
+typedef hipDeviceptr_t CUdeviceptr;
+struct CUlinkState_st;
+
+
+typedef hipCtx_t CUcontext;
+typedef hipEvent_t cudaEvent_t;
+typedef hipFunction_t CUfunction;
+typedef hipFunction_attribute CUfunction_attribute;
+typedef hipModule_t CUmodule;
+typedef hipStream_t cudaStream_t;
+#if HIP_VERSION >= 40300000
+typedef hipGraph_t cudaGraph_t;
+typedef hipGraphNode_t cudaGraphNode_t;
+typedef hipGraphExec_t cudaGraphExec_t;
+#else
+typedef void* cudaGraph_t;
+typedef void* cudaGraphNode_t;
+typedef void* cudaGraphExec_t;
+#endif
+typedef struct CUlinkState_st* CUlinkState;
+typedef struct CUarray_st* CUarray;
+struct CUDA_ARRAY_DESCRIPTOR {
+    CUarray_format Format;
+    size_t Height;
+    unsigned int NumChannels;
+    size_t Width;
+};
+
+
+///////////////////////////////////////////////////////////////////////////////
+// cuda_runtime.h
+///////////////////////////////////////////////////////////////////////////////
+
+enum {
+    cudaDevAttrComputeCapabilityMajor
+        = hipDeviceAttributeComputeCapabilityMajor,
+    cudaDevAttrComputeCapabilityMinor
+        = hipDeviceAttributeComputeCapabilityMinor,
+};
+
+typedef hipError_t cudaError_t;
+const CUresult cudaSuccess = static_cast<CUresult>(0);
+const CUresult cudaErrorInvalidValue = hipErrorInvalidValue;
+const CUresult cudaErrorMemoryAllocation = hipErrorMemoryAllocation;
+const CUresult cudaErrorInvalidResourceHandle = hipErrorInvalidResourceHandle;
+const CUresult cudaErrorContextIsDestroyed = hipErrorUnknown;  // no counterpart in HIP
+const CUresult cudaErrorPeerAccessAlreadyEnabled = hipErrorPeerAccessAlreadyEnabled;
+typedef enum {} cudaDataType;
+typedef hipDeviceAttribute_t cudaDeviceAttr;
+typedef hipLimit_t cudaLimit;
+typedef hipMemoryAdvise cudaMemoryAdvise;
+typedef hipMemcpyKind cudaMemcpyKind;
+typedef hipDeviceProp_t cudaDeviceProp;
+typedef void* cudaMemPool_t;
+enum cudaMemPoolAttr {};
+
+typedef hipStreamCallback_t cudaStreamCallback_t;
+typedef void (*cudaHostFn_t)(void* userData);
+typedef hipPointerAttribute_t cudaPointerAttributes;
+
+typedef hipChannelFormatKind cudaChannelFormatKind;
+typedef hipTextureObject_t cudaTextureObject_t;
+typedef hipSurfaceObject_t cudaSurfaceObject_t;
+typedef hipResourceType cudaResourceType;
+typedef hipTextureAddressMode cudaTextureAddressMode;
+typedef hipTextureFilterMode cudaTextureFilterMode;
+typedef hipTextureReadMode cudaTextureReadMode;
+typedef hipResourceViewDesc cudaResourceViewDesc;
+typedef hipArray_t cudaArray_t;
+typedef hipExtent cudaExtent;
+typedef hipPos cudaPos;
+typedef hipPitchedPtr cudaPitchedPtr;
+typedef hipMipmappedArray_t cudaMipmappedArray_t;
+typedef hipMemcpy3DParms cudaMemcpy3DParms;
+typedef hipChannelFormatDesc cudaChannelFormatDesc;
+typedef hipResourceDesc cudaResourceDesc;
+typedef hipTextureDesc cudaTextureDesc;
+
+// IPC operations
+typedef hipIpcMemHandle_st cudaIpcMemHandle_t;
+typedef hipIpcEventHandle_st cudaIpcEventHandle_t;
+
+
+///////////////////////////////////////////////////////////////////////////////
+// blas & lapack (hipBLAS/rocBLAS & rocSOLVER)
+///////////////////////////////////////////////////////////////////////////////
+
+/* As of ROCm 3.5.0 (this may have started earlier) many rocSOLVER helper functions
+ * are deprecated and using their counterparts from rocBLAS is recommended. In
+ * particular, rocSOLVER simply uses rocBLAS's handle for its API calls. This means
+ * they are much more integrated than cuBLAS and cuSOLVER do, so it is better to
+ * put all of the relevant function in one place.
+ */
+
+// TODO(leofang): investigate if we should just remove the hipBLAS layer and use
+// rocBLAS directly, since we need to expose its handle anyway
+
+
+typedef hipblasHandle_t cublasHandle_t;
+
+typedef hipblasDiagType_t cublasDiagType_t;
+typedef hipblasFillMode_t cublasFillMode_t;
+typedef hipblasOperation_t cublasOperation_t;
+typedef hipblasPointerMode_t cublasPointerMode_t;
+typedef hipblasSideMode_t cublasSideMode_t;
+typedef enum {} cublasGemmAlgo_t;
+typedef enum {} cublasMath_t;
+typedef int cudaDataType_t;
+typedef hipblasStatus_t cublasStatus_t;
+
+// TODO(leofang): as of ROCm 3.5.0 this does not exist yet
+typedef enum {} cublasComputeType_t;
+
+typedef rocblas_status cusolverStatus_t;
+typedef rocblas_handle cusolverDnHandle_t;
+
+
+///////////////////////////////////////////////////////////////////////////////
+// library_types.h
+// (needed for supporting cusolver)
+///////////////////////////////////////////////////////////////////////////////
+
+typedef enum libraryPropertyType_t {
+    MAJOR_VERSION,
+    MINOR_VERSION,
+    PATCH_LEVEL
+} libraryPropertyType;
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_COMMON_H
diff --git a/cupy_backends/hip/cupy_hip_runtime.h b/cupy_backends/hip/cupy_hip_runtime.h
new file mode 100644
index 0000000..51a5432
--- /dev/null
+++ b/cupy_backends/hip/cupy_hip_runtime.h
@@ -0,0 +1,521 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_RUNTIME_H
+#define INCLUDE_GUARD_HIP_CUPY_RUNTIME_H
+
+#include <hip/hip_runtime_api.h>
+#include "cupy_hip_common.h"
+
+extern "C" {
+
+bool hip_environment = true;
+
+// Error handling
+const char* cudaGetErrorName(cudaError_t hipError) {
+    return hipGetErrorName(hipError);
+}
+
+const char* cudaGetErrorString(cudaError_t hipError) {
+    return hipGetErrorString(hipError);
+}
+
+cudaError_t cudaGetLastError() {
+    return hipGetLastError();
+}
+
+
+// Initialization
+cudaError_t cudaDriverGetVersion(int *driverVersion) {
+    return hipDriverGetVersion(driverVersion);
+}
+
+cudaError_t cudaRuntimeGetVersion(int *runtimeVersion) {
+    return hipRuntimeGetVersion(runtimeVersion);
+}
+
+
+// CUdevice operations
+cudaError_t cudaGetDevice(int *deviceId) {
+    return hipGetDevice(deviceId);
+}
+
+cudaError_t cudaDeviceGetAttribute(int* pi, cudaDeviceAttr attr,
+                                   int deviceId) {
+    return hipDeviceGetAttribute(pi, attr, deviceId);
+}
+
+cudaError_t cudaDeviceGetByPCIBusId(int *device, const char *pciBusId) {
+    return hipDeviceGetByPCIBusId(device, pciBusId);
+}
+
+cudaError_t cudaDeviceGetPCIBusId(char *pciBusId, int len, int device) {
+    return hipDeviceGetPCIBusId(pciBusId, len, device);
+}
+
+cudaError_t cudaGetDeviceCount(int *count) {
+    return hipGetDeviceCount(count);
+}
+
+cudaError_t cudaSetDevice(int deviceId) {
+    return hipSetDevice(deviceId);
+}
+
+cudaError_t cudaDeviceSynchronize() {
+    return hipDeviceSynchronize();
+}
+
+cudaError_t cudaDeviceCanAccessPeer(int* canAccessPeer, int deviceId,
+                                    int peerDeviceId) {
+    return hipDeviceCanAccessPeer(canAccessPeer, deviceId, peerDeviceId);
+}
+
+cudaError_t cudaDeviceEnablePeerAccess(int peerDeviceId, unsigned int flags) {
+    return hipDeviceEnablePeerAccess(peerDeviceId, flags);
+}
+
+cudaError_t cudaDeviceDisablePeerAccess(int peerDeviceId) {
+    return hipDeviceDisablePeerAccess(peerDeviceId);
+}
+
+cudaError_t cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) {
+    return hipDeviceGetLimit(pValue, limit);
+}
+
+cudaError_t cudaDeviceSetLimit(cudaLimit limit, size_t value) {
+    // see https://github.com/ROCm-Developer-Tools/HIP/issues/1632
+    return hipErrorUnknown;
+}
+
+// IPC operations
+cudaError_t cudaIpcCloseMemHandle(void* devPtr) {
+    return hipIpcCloseMemHandle(devPtr);
+}
+
+cudaError_t cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) {
+    return hipErrorUnknown;
+
+    // TODO(leofang): this is supported after ROCm-Developer-Tools/HIP#1996 is released;
+    // as of ROCm 3.5.0 it is still not supported
+    //return hipIpcGetEventHandle(handle, event);
+}
+
+cudaError_t cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) {
+    return hipIpcGetMemHandle(handle, devPtr);
+}
+
+cudaError_t cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) {
+    return hipErrorUnknown;
+
+    // TODO(leofang): this is supported after ROCm-Developer-Tools/HIP#1996 is released;
+    // as of ROCm 3.5.0 it is still not supported
+    //return hipIpcOpenEventHandle(event, handle);
+}
+
+cudaError_t cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) {
+    return hipIpcOpenMemHandle(devPtr, handle, flags);
+}
+
+// Memory management
+enum cudaMemAllocationType {};  // stub
+enum cudaMemAllocationHandleType {};  // stub
+enum cudaMemLocationType {};  // stub
+struct cudaMemLocation {  // stub
+    int id;
+    cudaMemLocationType type;
+};
+struct cudaMemPoolProps {  // stub
+    cudaMemAllocationType allocType;
+    cudaMemAllocationHandleType handleTypes;
+    struct cudaMemLocation location;
+    unsigned char reserved[64];
+    void* win32SecurityAttributes;
+};
+
+cudaError_t cudaMalloc(void** ptr, size_t size) {
+    return hipMalloc(ptr, size);
+}
+
+cudaError_t cudaMalloc3DArray(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMallocArray(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMallocAsync(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaHostAlloc(void** ptr, size_t size, unsigned int flags) {
+    return hipHostMalloc(ptr, size, flags);
+}
+
+cudaError_t cudaHostRegister(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaHostUnregister(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMallocManaged(void** ptr, size_t size, unsigned int flags) {
+#if HIP_VERSION >= 40300000
+    return hipMallocManaged(ptr, size, flags);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+int cudaFree(void* ptr) {
+    return hipFree(ptr);
+}
+
+cudaError_t cudaFreeArray(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaFreeHost(void* ptr) {
+    return hipHostFree(ptr);
+}
+
+cudaError_t cudaFreeAsync(...) {
+    return hipErrorUnknown;
+}
+
+int cudaMemGetInfo(size_t* free, size_t* total) {
+    return hipMemGetInfo(free, total);
+}
+
+cudaError_t cudaMemcpy(void* dst, const void* src, size_t sizeBytes,
+                       hipMemcpyKind kind) {
+    return hipMemcpy(dst, src, sizeBytes, kind);
+}
+
+cudaError_t cudaMemcpyAsync(void* dst, const void* src, size_t sizeBytes,
+                            cudaMemcpyKind kind, cudaStream_t stream) {
+    return hipMemcpyAsync(dst, src, sizeBytes, kind, stream);
+}
+
+cudaError_t cudaMemcpyPeer(void* dst, int dstDeviceId, const void* src,
+                           int srcDeviceId, size_t sizeBytes) {
+    return hipMemcpyPeer(dst, dstDeviceId, src, srcDeviceId, sizeBytes);
+}
+
+cudaError_t cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src,
+                                int srcDevice, size_t sizeBytes,
+                                cudaStream_t stream) {
+    return hipMemcpyPeerAsync(dst, dstDevice, src, srcDevice, sizeBytes,
+                              stream);
+}
+
+cudaError_t cudaMemcpy2D(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemcpy2DAsync(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemcpy2DFromArray(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemcpy2DFromArrayAsync(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemcpy2DToArray(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemcpy2DToArrayAsync(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemcpy3D(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemcpy3DAsync(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemset(void* dst, int value, size_t sizeBytes) {
+    return hipMemset(dst, value, sizeBytes);
+}
+
+cudaError_t cudaMemsetAsync(void* dst, int value, size_t sizeBytes,
+                            cudaStream_t stream) {
+    return hipMemsetAsync(dst, value, sizeBytes, stream);
+}
+
+cudaError_t cudaMemAdvise(const void *devPtr, size_t count,
+                          cudaMemoryAdvise advice, int device) {
+#if HIP_VERSION >= 40300000
+    return hipMemAdvise(devPtr, count, advice, device);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+cudaError_t cudaMemPrefetchAsync(const void *devPtr, size_t count,
+				 int dstDevice, cudaStream_t stream) {
+#if HIP_VERSION >= 40300000
+    return hipMemPrefetchAsync(devPtr, count, dstDevice, stream);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+cudaError_t cudaPointerGetAttributes(cudaPointerAttributes *attributes,
+                                     const void* ptr) {
+    cudaError_t status = hipPointerGetAttributes(attributes, ptr);
+    if (status == cudaSuccess) {
+        switch (attributes->memoryType) {
+            case 0 /* hipMemoryTypeHost */:
+                attributes->memoryType = (hipMemoryType)1; /* cudaMemoryTypeHost */
+                return status;
+            case 1 /* hipMemoryTypeDevice */:
+                attributes->memoryType = (hipMemoryType)2; /* cudaMemoryTypeDevice */
+                return status;
+            default:
+                /* we don't care the rest of possibilities */
+                return status;
+        }
+    } else {
+        return status;
+    }
+}
+
+cudaError_t cudaGetDeviceProperties(cudaDeviceProp *prop, int device) {
+    return hipGetDeviceProperties(prop, device);
+}
+
+cudaError_t cudaMallocFromPoolAsync(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemPoolCreate(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemPoolDestroy(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaDeviceGetDefaultMemPool(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaDeviceGetMemPool(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaDeviceSetMemPool(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemPoolTrimTo(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemPoolGetAttribute(...) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaMemPoolSetAttribute(...) {
+    return hipErrorUnknown;
+}
+
+
+// Stream and Event
+#if HIP_VERSION >= 40300000
+typedef hipStreamCaptureMode cudaStreamCaptureMode;
+typedef hipStreamCaptureStatus cudaStreamCaptureStatus;
+#else
+enum cudaStreamCaptureMode {};
+enum cudaStreamCaptureStatus {};
+#endif
+
+cudaError_t cudaStreamCreate(cudaStream_t *stream) {
+    return hipStreamCreate(stream);
+}
+
+cudaError_t cudaStreamCreateWithFlags(cudaStream_t *stream,
+                                      unsigned int flags) {
+    return hipStreamCreateWithFlags(stream, flags);
+}
+
+cudaError_t cudaStreamDestroy(cudaStream_t stream) {
+    return hipStreamDestroy(stream);
+}
+
+cudaError_t cudaStreamSynchronize(cudaStream_t stream) {
+    return hipStreamSynchronize(stream);
+}
+
+cudaError_t cudaStreamAddCallback(cudaStream_t stream,
+                                  cudaStreamCallback_t callback,
+                                  void *userData, unsigned int flags) {
+    return hipStreamAddCallback(stream, callback, userData, flags);
+}
+
+cudaError_t cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) {
+    return hipErrorUnknown;
+}
+
+cudaError_t cudaStreamQuery(cudaStream_t stream) {
+    return hipStreamQuery(stream);
+}
+
+cudaError_t cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event,
+                                unsigned int flags) {
+    return hipStreamWaitEvent(stream, event, flags);
+}
+
+cudaError_t cudaEventCreate(cudaEvent_t* event) {
+    return hipEventCreate(event);
+}
+
+cudaError_t cudaEventCreateWithFlags(cudaEvent_t* event, unsigned flags) {
+    return hipEventCreateWithFlags(event, flags);
+}
+
+cudaError_t cudaEventDestroy(cudaEvent_t event) {
+    return hipEventDestroy(event);
+}
+
+cudaError_t cudaEventElapsedTime(float *ms, cudaEvent_t start,
+                                 cudaEvent_t stop){
+    return hipEventElapsedTime(ms, start, stop);
+}
+
+cudaError_t cudaEventQuery(cudaEvent_t event) {
+    return hipEventQuery(event);
+}
+
+cudaError_t cudaEventRecord(cudaEvent_t event, cudaStream_t stream) {
+    return hipEventRecord(event, stream);
+}
+
+cudaError_t cudaEventSynchronize(cudaEvent_t event) {
+    return hipEventSynchronize(event);
+}
+
+cudaError_t cudaStreamBeginCapture(cudaStream_t stream,
+                                   cudaStreamCaptureMode mode) {
+#if HIP_VERSION >= 40300000
+    return hipStreamBeginCapture(stream, mode);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+cudaError_t cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) {
+#if HIP_VERSION >= 40300000
+    return hipStreamEndCapture(stream, pGraph);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+cudaError_t cudaStreamIsCapturing(cudaStream_t stream,
+                                  cudaStreamCaptureStatus* pCaptureStatus) {
+#if HIP_VERSION >= 50000000
+    return hipStreamIsCapturing(stream, pCaptureStatus);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+
+// Texture
+cudaError_t cudaCreateTextureObject(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDestroyTextureObject(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGetChannelDesc(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGetTextureObjectResourceDesc(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGetTextureObjectTextureDesc(...) {
+    return cudaSuccess;
+}
+
+cudaExtent make_cudaExtent(...) {
+    cudaExtent ex = {};
+    return ex;
+}
+
+cudaPitchedPtr make_cudaPitchedPtr(...) {
+    cudaPitchedPtr ptr = {};
+    return ptr;
+}
+
+cudaPos make_cudaPos(...) {
+    cudaPos pos = {};
+    return pos;
+}
+
+// Surface
+cudaError_t cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject,
+                                    const cudaResourceDesc* pResDesc) {
+    return hipCreateSurfaceObject(pSurfObject, pResDesc);
+}
+
+cudaError_t cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) {
+    return hipDestroySurfaceObject(surfObject);
+}
+
+// CUDA Graph
+cudaError_t cudaGraphInstantiate(
+	cudaGraphExec_t* pGraphExec,
+	cudaGraph_t graph,
+	cudaGraphNode_t* pErrorNode,
+	char* pLogBuffer,
+	size_t bufferSize) {
+#if HIP_VERSION >= 40300000
+    return hipGraphInstantiate(pGraphExec, graph, pErrorNode, pLogBuffer, bufferSize);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+cudaError_t cudaGraphDestroy(cudaGraph_t graph) {
+#if HIP_VERSION >= 40300000
+    return hipGraphDestroy(graph);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+cudaError_t cudaGraphExecDestroy(cudaGraphExec_t graphExec) {
+#if HIP_VERSION >= 40300000
+    return hipGraphExecDestroy(graphExec);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+cudaError_t cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) {
+#if HIP_VERSION >= 40300000
+    return hipGraphLaunch(graphExec, stream);
+#else
+    return hipErrorUnknown;
+#endif
+}
+
+cudaError_t cudaGraphUpload(...) {
+    return hipErrorUnknown;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_RUNTIME_H
diff --git a/cupy_backends/hip/cupy_hipblas.h b/cupy_backends/hip/cupy_hipblas.h
new file mode 100644
index 0000000..627bc86
--- /dev/null
+++ b/cupy_backends/hip/cupy_hipblas.h
@@ -0,0 +1,998 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_HIPBLAS_H
+#define INCLUDE_GUARD_HIP_CUPY_HIPBLAS_H
+
+#include "cupy_hip_common.h"
+#include <hipblas.h>
+#include <hip/hip_version.h>  // for HIP_VERSION
+#include <stdexcept>  // for gcc 10
+
+
+extern "C" {
+
+///////////////////////////////////////////////////////////////////////////////
+// blas & lapack (hipBLAS/rocBLAS & rocSOLVER)
+///////////////////////////////////////////////////////////////////////////////
+
+/* As of ROCm 3.5.0 (this may have started earlier) many rocSOLVER helper functions
+ * are deprecated and using their counterparts from rocBLAS is recommended. In
+ * particular, rocSOLVER simply uses rocBLAS's handle for its API calls. This means
+ * they are much more integrated than cuBLAS and cuSOLVER are, so it is better to
+ * put all of the relevant function in one place.
+ */
+
+// TODO(leofang): investigate if we should just remove the hipBLAS layer and use
+// rocBLAS directly, since we need to expose its handle anyway
+
+
+/* ---------- helpers ---------- */
+static hipblasOperation_t convert_hipblasOperation_t(cublasOperation_t op) {
+    return static_cast<hipblasOperation_t>(static_cast<int>(op) + 111);
+}
+
+static hipblasFillMode_t convert_hipblasFillMode_t(cublasFillMode_t mode) {
+    switch(static_cast<int>(mode)) {
+        case 0 /* CUBLAS_FILL_MODE_LOWER */: return HIPBLAS_FILL_MODE_LOWER;
+        case 1 /* CUBLAS_FILL_MODE_UPPER */: return HIPBLAS_FILL_MODE_UPPER;
+        default: throw std::runtime_error("unrecognized mode");
+    }
+}
+
+static hipblasDiagType_t convert_hipblasDiagType_t(cublasDiagType_t type) {
+    return static_cast<hipblasDiagType_t>(static_cast<int>(type) + 131);
+}
+
+static hipblasSideMode_t convert_hipblasSideMode_t(cublasSideMode_t mode) {
+    return static_cast<hipblasSideMode_t>(static_cast<int>(mode) + 141);
+}
+
+static hipblasDatatype_t convert_hipblasDatatype_t(cudaDataType_t type) {
+    switch(static_cast<int>(type)) {
+        case 0 /* CUDA_R_32F */: return HIPBLAS_R_32F;
+        case 1 /* CUDA_R_64F */: return HIPBLAS_R_64F;
+        case 2 /* CUDA_R_16F */: return HIPBLAS_R_16F;
+        case 3 /* CUDA_R_8I */ : return HIPBLAS_R_8I;
+        case 4 /* CUDA_C_32F */: return HIPBLAS_C_32F;
+        case 5 /* CUDA_C_64F */: return HIPBLAS_C_64F;
+        case 6 /* CUDA_C_16F */: return HIPBLAS_C_16F;
+        case 7 /* CUDA_C_8I */ : return HIPBLAS_C_8I;
+        case 8 /* CUDA_R_8U */ : return HIPBLAS_R_8U;
+        case 9 /* CUDA_C_8U */ : return HIPBLAS_C_8U;
+        default: throw std::runtime_error("unrecognized type");
+    }
+}
+
+
+// Context
+cublasStatus_t cublasCreate(cublasHandle_t* handle) {
+    return hipblasCreate(handle);
+}
+
+cublasStatus_t cublasDestroy(cublasHandle_t handle) {
+    return hipblasDestroy(handle);
+}
+
+cublasStatus_t cublasGetVersion(cublasHandle_t handle, int *version) {
+    // We use the rocBLAS version here because 1. it is the underlying workhorse,
+    // and 2. we might get rid of the hipBLAS layer at some point (see TODO above).
+    // ex: the rocBLAS version string is 2.22.0.2367-b2cceba in ROCm 3.5.0
+    *version = 10000 * ROCBLAS_VERSION_MAJOR + 100 * ROCBLAS_VERSION_MINOR + ROCBLAS_VERSION_PATCH;
+    return HIPBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSetPointerMode(cublasHandle_t handle, cublasPointerMode_t mode) {
+    return hipblasSetPointerMode(handle, mode);
+}
+
+cublasStatus_t cublasGetPointerMode(cublasHandle_t handle, cublasPointerMode_t *mode) {
+    return hipblasGetPointerMode(handle, mode);
+}
+
+// Stream
+cublasStatus_t cublasSetStream(cublasHandle_t handle, cudaStream_t streamId) {
+    return hipblasSetStream(handle, streamId);
+}
+
+cublasStatus_t cublasGetStream(cublasHandle_t handle, cudaStream_t *streamId) {
+    return hipblasGetStream(handle, streamId);
+}
+
+// Math Mode
+cublasStatus_t cublasSetMathMode(...) {
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+}
+
+cublasStatus_t cublasGetMathMode(...) {
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+}
+
+// BLAS Level 1
+cublasStatus_t cublasIsamax(cublasHandle_t handle, int n, const float *x, int incx, int *result) {
+    return hipblasIsamax(handle, n, x, incx, result);
+}
+
+cublasStatus_t cublasIdamax(cublasHandle_t handle, int n, const double *x, int incx, int *result) {
+    return hipblasIdamax(handle, n, x, incx, result);
+}
+
+cublasStatus_t cublasIcamax(cublasHandle_t handle, int n, const cuComplex *x, int incx, int *result) {
+    return hipblasIcamax(handle, n, reinterpret_cast<const hipblasComplex*>(x), incx, result);
+}
+
+cublasStatus_t cublasIzamax(cublasHandle_t handle, int n, const cuDoubleComplex *x, int incx, int *result) {
+    return hipblasIzamax(handle, n, reinterpret_cast<const hipblasDoubleComplex*>(x), incx, result);
+}
+
+cublasStatus_t cublasIsamin(cublasHandle_t handle, int n, float* x, int incx, int* result) {
+    return hipblasIsamin(handle, n, x, incx, result);
+}
+
+cublasStatus_t cublasIdamin(cublasHandle_t handle, int n, const double *x, int incx, int *result) {
+    return hipblasIdamin(handle, n, x, incx, result);
+}
+
+cublasStatus_t cublasIcamin(cublasHandle_t handle, int n, const cuComplex *x, int incx, int *result) {
+    return hipblasIcamin(handle, n, reinterpret_cast<const hipblasComplex*>(x), incx, result);
+}
+
+cublasStatus_t cublasIzamin(cublasHandle_t handle, int n, const cuDoubleComplex *x, int incx, int *result) {
+    return hipblasIzamin(handle, n, reinterpret_cast<const hipblasDoubleComplex*>(x), incx, result);
+}
+
+cublasStatus_t cublasSasum(cublasHandle_t handle, int n, float* x, int incx, float* result) {
+    return hipblasSasum(handle, n, x, incx, result);
+}
+
+cublasStatus_t cublasDasum(cublasHandle_t handle, int n, double* x, int incx, double* result) {
+    return hipblasDasum(handle, n, x, incx, result);
+}
+
+cublasStatus_t cublasScasum(cublasHandle_t handle, int n, cuComplex* x, int incx, float* result) {
+    return hipblasScasum(handle, n, reinterpret_cast<const hipblasComplex*>(x), incx, result);
+}
+
+cublasStatus_t cublasDzasum(cublasHandle_t handle, int n, cuDoubleComplex* x, int incx, double* result) {
+    return hipblasDzasum(handle, n, reinterpret_cast<const hipblasDoubleComplex*>(x), incx, result);
+}
+
+cublasStatus_t cublasSaxpy(cublasHandle_t handle, int n, float* alpha, float* x, int incx, float* y, int incy) {
+    return hipblasSaxpy(handle, n, alpha, x, incx, y, incy);
+}
+
+cublasStatus_t cublasDaxpy(cublasHandle_t handle, int n, double* alpha, double* x, int incx, double* y, int incy) {
+    return hipblasDaxpy(handle, n, alpha, x, incx, y, incy);
+}
+
+cublasStatus_t cublasCaxpy(cublasHandle_t handle, int n, cuComplex* alpha, cuComplex* x, int incx, cuComplex* y, int incy) {
+    return hipblasCaxpy(handle, n,
+                        reinterpret_cast<const hipblasComplex*>(alpha),
+                        reinterpret_cast<const hipblasComplex*>(x), incx,
+                        reinterpret_cast<hipblasComplex*>(y), incy);
+}
+
+cublasStatus_t cublasZaxpy(cublasHandle_t handle, int n, cuDoubleComplex* alpha, cuDoubleComplex* x, int incx, cuDoubleComplex* y, int incy) {
+    return hipblasZaxpy(handle, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+                        reinterpret_cast<const hipblasDoubleComplex*>(x), incx,
+                        reinterpret_cast<hipblasDoubleComplex*>(y), incy);
+}
+
+cublasStatus_t cublasSdot(cublasHandle_t handle, int n, float* x, int incx, float* y, int incy, float* result) {
+    return hipblasSdot(handle, n, x, incx, y, incy, result);
+}
+
+cublasStatus_t cublasDdot(cublasHandle_t handle, int n, double* x, int incx, double* y, int incy, double* result) {
+    return hipblasDdot(handle, n, x, incx, y, incy, result);
+}
+
+cublasStatus_t cublasCdotu(cublasHandle_t handle, int n, cuComplex* x, int incx, cuComplex* y,
+                           int incy, cuComplex* result) {
+    return hipblasCdotu(handle, n,
+                        reinterpret_cast<const hipblasComplex*>(x), incx,
+                        reinterpret_cast<const hipblasComplex*>(y), incy,
+                        reinterpret_cast<hipblasComplex*>(result));
+}
+
+cublasStatus_t cublasCdotc(cublasHandle_t handle, int n, cuComplex* x, int incx, cuComplex* y,
+                           int incy, cuComplex* result) {
+    return hipblasCdotc(handle, n,
+                        reinterpret_cast<const hipblasComplex*>(x), incx,
+                        reinterpret_cast<const hipblasComplex*>(y), incy,
+                        reinterpret_cast<hipblasComplex*>(result));
+}
+
+cublasStatus_t cublasZdotu(cublasHandle_t handle, int n, cuDoubleComplex* x, int incx, cuDoubleComplex* y,
+                           int incy, cuDoubleComplex* result) {
+    return hipblasZdotu(handle, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(x), incx,
+                        reinterpret_cast<const hipblasDoubleComplex*>(y), incy,
+                        reinterpret_cast<hipblasDoubleComplex*>(result));
+}
+
+cublasStatus_t cublasZdotc(cublasHandle_t handle, int n, cuDoubleComplex* x, int incx, cuDoubleComplex* y,
+                           int incy, cuDoubleComplex* result) {
+    return hipblasZdotc(handle, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(x), incx,
+                        reinterpret_cast<const hipblasDoubleComplex*>(y), incy,
+                        reinterpret_cast<hipblasDoubleComplex*>(result));
+}
+
+cublasStatus_t cublasSnrm2(cublasHandle_t handle, int n, float* x, int incx, float* result) {
+    return hipblasSnrm2(handle, n, x, incx, result);
+}
+
+cublasStatus_t cublasDnrm2(cublasHandle_t handle, int n, double* x, int incx, double* result) {
+    return hipblasDnrm2(handle, n, x, incx, result);
+}
+
+cublasStatus_t cublasScnrm2(cublasHandle_t handle, int n, cuComplex* x, int incx, float* result) {
+    return hipblasScnrm2(handle, n, reinterpret_cast<const hipblasComplex*>(x), incx, result);
+}
+
+cublasStatus_t cublasDznrm2(cublasHandle_t handle, int n, cuDoubleComplex* x, int incx, double* result) {
+    return hipblasDznrm2(handle, n, reinterpret_cast<const hipblasDoubleComplex*>(x), incx, result);
+}
+
+cublasStatus_t cublasSscal(cublasHandle_t handle, int n, float* alpha, float* x, int incx) {
+    return hipblasSscal(handle, n, alpha, x, incx);
+}
+
+cublasStatus_t cublasDscal(cublasHandle_t handle, int n, double* alpha, double* x, int incx) {
+    return hipblasDscal(handle, n, alpha, x, incx);
+}
+
+cublasStatus_t cublasCscal(cublasHandle_t handle, int n, cuComplex* alpha, cuComplex* x, int incx) {
+    return hipblasCscal(handle, n, reinterpret_cast<const hipblasComplex*>(alpha), reinterpret_cast<hipblasComplex*>(x), incx);
+}
+
+cublasStatus_t cublasCsscal(cublasHandle_t handle, int n, float* alpha, cuComplex* x, int incx) {
+    return hipblasCsscal(handle, n, alpha, reinterpret_cast<hipblasComplex*>(x), incx);
+}
+
+cublasStatus_t cublasZscal(cublasHandle_t handle, int n, cuDoubleComplex* alpha, cuDoubleComplex* x, int incx) {
+    return hipblasZscal(handle, n, reinterpret_cast<const hipblasDoubleComplex*>(alpha), reinterpret_cast<hipblasDoubleComplex*>(x), incx);
+}
+
+cublasStatus_t cublasZdscal(cublasHandle_t handle, int n, double* alpha, cuDoubleComplex* x, int incx) {
+    return hipblasZdscal(handle, n, alpha, reinterpret_cast<hipblasDoubleComplex*>(x), incx);
+}
+
+
+// BLAS Level 2
+cublasStatus_t cublasSgemv(cublasHandle_t handle, cublasOperation_t trans, int m, int n, float* alpha,
+                           float* A, int lda, float* x, int incx, float* beta,
+                           float* y, int incy) {
+    return hipblasSgemv(handle, convert_hipblasOperation_t(trans), m, n,
+                        alpha, A, lda, x, incx, beta, y, incy);
+}
+
+cublasStatus_t cublasDgemv(cublasHandle_t handle, cublasOperation_t trans, int m, int n, double* alpha,
+                           double* A, int lda, double* x, int incx, double* beta,
+                           double* y, int incy) {
+    return hipblasDgemv(handle, convert_hipblasOperation_t(trans), m, n,
+                        alpha, A, lda, x, incx, beta, y, incy);
+}
+
+cublasStatus_t cublasCgemv(cublasHandle_t handle, cublasOperation_t trans, int m, int n, cuComplex* alpha,
+                           cuComplex* A, int lda, cuComplex* x, int incx, cuComplex* beta,
+                           cuComplex* y, int incy) {
+    return hipblasCgemv(handle, convert_hipblasOperation_t(trans), m, n,
+                        reinterpret_cast<const hipblasComplex*>(alpha),
+                        reinterpret_cast<const hipblasComplex*>(A), lda,
+                        reinterpret_cast<const hipblasComplex*>(x), incx,
+                        reinterpret_cast<const hipblasComplex*>(beta),
+                        reinterpret_cast<hipblasComplex*>(y), incy);
+}
+
+cublasStatus_t cublasZgemv(cublasHandle_t handle, cublasOperation_t trans, int m, int n, cuDoubleComplex* alpha,
+                           cuDoubleComplex* A, int lda, cuDoubleComplex* x, int incx,
+                           cuDoubleComplex* beta, cuDoubleComplex* y, int incy) {
+    return hipblasZgemv(handle, convert_hipblasOperation_t(trans), m, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+                        reinterpret_cast<const hipblasDoubleComplex*>(A), lda,
+                        reinterpret_cast<const hipblasDoubleComplex*>(x), incx,
+                        reinterpret_cast<const hipblasDoubleComplex*>(beta),
+                        reinterpret_cast<hipblasDoubleComplex*>(y), incy);
+}
+
+cublasStatus_t cublasSger(cublasHandle_t handle, int m, int n, float* alpha, float* x, int incx,
+                          float* y, int incy, float* A, int lda) {
+    return hipblasSger(handle, m, n, alpha, x, incx, y, incy, A, lda);
+}
+
+cublasStatus_t cublasDger(cublasHandle_t handle, int m, int n, double* alpha, double* x,
+                          int incx, double* y, int incy, double* A, int lda) {
+    return hipblasDger(handle, m, n, alpha, x, incx, y, incy, A, lda);
+}
+
+cublasStatus_t cublasCgeru(cublasHandle_t handle, int m, int n, cuComplex* alpha, cuComplex* x,
+                           int incx, cuComplex* y, int incy, cuComplex* A, int lda) {
+    return hipblasCgeru(handle, m, n,
+                        reinterpret_cast<const hipblasComplex*>(alpha),
+                        reinterpret_cast<const hipblasComplex*>(x), incx,
+                        reinterpret_cast<const hipblasComplex*>(y), incy,
+                        reinterpret_cast<hipblasComplex*>(A), lda);
+}
+
+cublasStatus_t cublasCgerc(cublasHandle_t handle, int m, int n, cuComplex* alpha, cuComplex* x,
+                           int incx, cuComplex* y, int incy, cuComplex* A, int lda) {
+    return hipblasCgerc(handle, m, n,
+                        reinterpret_cast<const hipblasComplex*>(alpha),
+                        reinterpret_cast<const hipblasComplex*>(x), incx,
+                        reinterpret_cast<const hipblasComplex*>(y), incy,
+                        reinterpret_cast<hipblasComplex*>(A), lda);
+}
+
+cublasStatus_t cublasZgeru(cublasHandle_t handle, int m, int n, cuDoubleComplex* alpha,
+                           cuDoubleComplex* x, int incx, cuDoubleComplex* y, int incy,
+                           cuDoubleComplex* A, int lda) {
+    return hipblasZgeru(handle, m, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+                        reinterpret_cast<const hipblasDoubleComplex*>(x), incx,
+                        reinterpret_cast<const hipblasDoubleComplex*>(y), incy,
+                        reinterpret_cast<hipblasDoubleComplex*>(A), lda);
+}
+
+cublasStatus_t cublasZgerc(cublasHandle_t handle, int m, int n, cuDoubleComplex* alpha,
+                           cuDoubleComplex* x, int incx, cuDoubleComplex* y, int incy,
+                           cuDoubleComplex* A, int lda) {
+    return hipblasZgerc(handle, m, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+                        reinterpret_cast<const hipblasDoubleComplex*>(x), incx,
+                        reinterpret_cast<const hipblasDoubleComplex*>(y), incy,
+                        reinterpret_cast<hipblasDoubleComplex*>(A), lda);
+}
+
+cublasStatus_t cublasSsbmv(cublasHandle_t handle, cublasFillMode_t uplo, int n, int k,
+                           const float* alpha, const float* A, int lda,
+                           const float* x, int incx,
+                           const float* beta, float* y, int incy) {
+    return hipblasSsbmv(handle, convert_hipblasFillMode_t(uplo), n, k,
+                        alpha, A, lda, x, incx, beta, y, incy);
+}
+
+cublasStatus_t cublasDsbmv(cublasHandle_t handle, cublasFillMode_t uplo, int n, int k,
+                           const double* alpha, const double* A, int lda,
+                           const double* x, int incx,
+                           const double* beta, double* y, int incy) {
+    return hipblasDsbmv(handle, convert_hipblasFillMode_t(uplo), n, k,
+        alpha, A, lda, x, incx, beta, y, incy);
+}
+
+
+// BLAS Level 3
+cublasStatus_t cublasGemmStridedBatchedEx(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb,
+                                          int m, int n, int k, const void* alpha,
+                                          const void* A, cudaDataType Atype, int lda, long long int strideA,
+                                          const void* B, cudaDataType Btype, int ldb, long long int strideB,
+                                          const void* beta,
+                                          void* C, cudaDataType Ctype, int ldc, long long int strideC,
+                                          int batchCount, cudaDataType_t computeType, cublasGemmAlgo_t algo) {
+    if (algo != -1) { // must be CUBLAS_GEMM_DEFAULT
+        return HIPBLAS_STATUS_NOT_SUPPORTED;
+    }
+    return hipblasGemmStridedBatchedEx(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+                                       m, n, k, alpha,
+                                       A, convert_hipblasDatatype_t(Atype), lda, strideA,
+                                       B, convert_hipblasDatatype_t(Btype), ldb, strideB,
+                                       beta,
+                                       C, convert_hipblasDatatype_t(Ctype), ldc, strideC,
+                                       batchCount, convert_hipblasDatatype_t(computeType),
+                                       static_cast<hipblasGemmAlgo_t>(160));  // HIPBLAS_GEMM_DEFAULT
+}
+
+cublasStatus_t cublasSgemm(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb,
+                            int m, int n, int k, const float *alpha,
+                            const float *A, int lda,
+                            const float *B, int ldb,
+                            const float *beta,
+                            float *C, int ldc) {
+    return hipblasSgemm(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb), m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);
+}
+
+cublasStatus_t cublasDgemm(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb,
+                           int m, int n, int k, const double *alpha,
+                           const double *A, int lda,
+                           const double *B, int ldb,
+                           const double *beta, double *C, int ldc) {
+    return hipblasDgemm(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb), m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);
+}
+
+
+cublasStatus_t cublasCgemm(
+        cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k, const cuComplex *alpha,
+        const cuComplex *A, int lda,
+        const cuComplex *B, int ldb,
+        const cuComplex *beta, cuComplex *C, int ldc)
+{
+    return hipblasCgemm(
+        handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k,
+        reinterpret_cast<const hipblasComplex *>(alpha),
+        reinterpret_cast<const hipblasComplex *>(A), lda,
+        reinterpret_cast<const hipblasComplex *>(B), ldb,
+        reinterpret_cast<const hipblasComplex *>(beta),
+        reinterpret_cast<hipblasComplex *>(C), ldc);
+}
+
+cublasStatus_t cublasZgemm(
+        cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k, const cuDoubleComplex *alpha,
+        const cuDoubleComplex *A, int lda,
+        const cuDoubleComplex *B, int ldb,
+        const cuDoubleComplex *beta, cuDoubleComplex *C, int ldc)
+{
+    return hipblasZgemm(
+        handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k,
+        reinterpret_cast<const hipblasDoubleComplex *>(alpha),
+        reinterpret_cast<const hipblasDoubleComplex *>(A), lda,
+        reinterpret_cast<const hipblasDoubleComplex *>(B), ldb,
+        reinterpret_cast<const hipblasDoubleComplex *>(beta),
+        reinterpret_cast<hipblasDoubleComplex *>(C), ldc);
+}
+
+cublasStatus_t cublasSgemmBatched(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k,  const float *alpha,
+        const float *A[], int lda,
+        const float *B[], int ldb,
+        const float *beta,
+        float *C[], int ldc, int batchCount) {
+    return hipblasSgemmBatched(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb), m, n, k, alpha, A, lda, B, ldb, beta, C, ldc, batchCount);
+}
+
+cublasStatus_t cublasDgemmBatched(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k,  const double *alpha,
+        const double *A[], int lda,
+        const double *B[], int ldb,
+        const double *beta,
+        double *C[], int ldc, int batchCount) {
+    return hipblasDgemmBatched(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb), m, n, k, alpha, A, lda, B, ldb, beta, C, ldc, batchCount);
+}
+
+cublasStatus_t cublasCgemmBatched(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k,  const cuComplex *alpha,
+        const cuComplex *A[], int lda,
+        const cuComplex *B[], int ldb,
+        const cuComplex *beta,
+        cuComplex *C[], int ldc, int batchCount) {
+    return hipblasCgemmBatched(
+        handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k,
+        reinterpret_cast<const hipblasComplex*>(alpha),
+        reinterpret_cast<const hipblasComplex**>(A), lda,
+        reinterpret_cast<const hipblasComplex**>(B), ldb,
+        reinterpret_cast<const hipblasComplex*>(beta),
+        reinterpret_cast<hipblasComplex**>(C), ldc, batchCount);
+}
+
+cublasStatus_t cublasZgemmBatched(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k,  const cuDoubleComplex *alpha,
+        const cuDoubleComplex *A[], int lda,
+        const cuDoubleComplex *B[], int ldb,
+        const cuDoubleComplex *beta,
+        cuDoubleComplex *C[], int ldc, int batchCount) {
+    return hipblasZgemmBatched(
+        handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k,
+        reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+        reinterpret_cast<const hipblasDoubleComplex**>(A), lda,
+        reinterpret_cast<const hipblasDoubleComplex**>(B), ldb,
+        reinterpret_cast<const hipblasDoubleComplex*>(beta),
+        reinterpret_cast<hipblasDoubleComplex**>(C), ldc, batchCount);
+}
+
+cublasStatus_t cublasSgemmEx(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k,
+        const float *alpha, const void *A, cudaDataType_t Atype, int lda,
+        const void *B, cudaDataType_t Btype, int ldb, const float *beta,
+        void *C, cudaDataType_t Ctype, int ldc) {
+    if (Atype != 0 || Btype != 0 || Ctype != 0) {
+        return HIPBLAS_STATUS_NOT_SUPPORTED;
+    }
+    return hipblasSgemm(
+        handle,
+        convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k,
+        alpha, static_cast<const float*>(A), lda,
+        static_cast<const float*>(B), ldb, beta,
+        static_cast<float*>(C), ldc);
+}
+
+cublasStatus_t cublasGemmEx(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb,
+                            int m, int n, int k, const void *alpha,
+                            const void *A, cudaDataType_t Atype, int lda,
+                            const void *B, cudaDataType_t Btype, int ldb,
+                            const void *beta,
+                            void *C, cudaDataType_t Ctype, int ldc,
+                            cudaDataType_t computetype, cublasGemmAlgo_t algo) {
+    if (algo != -1) { // must be CUBLAS_GEMM_DEFAULT
+        return HIPBLAS_STATUS_NOT_SUPPORTED;
+    }
+    return hipblasGemmEx(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+                         m, n, k, alpha,
+                         A, convert_hipblasDatatype_t(Atype), lda,
+                         B, convert_hipblasDatatype_t(Btype), ldb,
+                         beta,
+                         C, convert_hipblasDatatype_t(Ctype), ldc,
+                         convert_hipblasDatatype_t(computetype),
+                         static_cast<hipblasGemmAlgo_t>(160));  // HIPBLAS_GEMM_DEFAULT
+}
+
+cublasStatus_t cublasGemmEx_v11(...) {
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+}
+cublasStatus_t cublasGemmStridedBatchedEx_v11(...) {
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+}
+
+cublasStatus_t cublasStrsm(cublasHandle_t handle, cublasSideMode_t size, cublasFillMode_t uplo, cublasOperation_t trans,
+                           cublasDiagType_t diag, int m, int n, const float* alpha,
+                           const float* A, int lda, float* B, int ldb) {
+    return hipblasStrsm(handle,
+                        convert_hipblasSideMode_t(size),
+                        convert_hipblasFillMode_t(uplo),
+                        convert_hipblasOperation_t(trans),
+                        convert_hipblasDiagType_t(diag),
+                        m, n, alpha, const_cast<float*>(A), lda, B, ldb);
+}
+
+cublasStatus_t cublasDtrsm(cublasHandle_t handle, cublasSideMode_t size, cublasFillMode_t uplo, cublasOperation_t trans,
+                           cublasDiagType_t diag, int m, int n, const double* alpha,
+                           const double* A, int lda, double* B, int ldb) {
+    return hipblasDtrsm(handle,
+                        convert_hipblasSideMode_t(size),
+                        convert_hipblasFillMode_t(uplo),
+                        convert_hipblasOperation_t(trans),
+                        convert_hipblasDiagType_t(diag),
+                        m, n, alpha, const_cast<double*>(A), lda, B, ldb);
+}
+
+cublasStatus_t cublasCtrsm(cublasHandle_t handle, cublasSideMode_t size, cublasFillMode_t uplo, cublasOperation_t trans,
+                           cublasDiagType_t diag, int m, int n, const cuComplex* alpha,
+                           const cuComplex* A, int lda, cuComplex* B, int ldb) {
+    return hipblasCtrsm(handle,
+                        convert_hipblasSideMode_t(size),
+                        convert_hipblasFillMode_t(uplo),
+                        convert_hipblasOperation_t(trans),
+                        convert_hipblasDiagType_t(diag),
+                        m, n,
+                        reinterpret_cast<const hipblasComplex*>(alpha),
+                        reinterpret_cast<hipblasComplex*>(const_cast<cuComplex*>(A)), lda,
+                        reinterpret_cast<hipblasComplex*>(B), ldb);
+}
+
+cublasStatus_t cublasZtrsm(cublasHandle_t handle, cublasSideMode_t size, cublasFillMode_t uplo, cublasOperation_t trans,
+                           cublasDiagType_t diag, int m, int n, const cuDoubleComplex* alpha,
+                           const cuDoubleComplex* A, int lda, cuDoubleComplex* B, int ldb) {
+    return hipblasZtrsm(handle,
+                        convert_hipblasSideMode_t(size),
+                        convert_hipblasFillMode_t(uplo),
+                        convert_hipblasOperation_t(trans),
+                        convert_hipblasDiagType_t(diag),
+                        m, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+                        reinterpret_cast<hipblasDoubleComplex*>(const_cast<cuDoubleComplex*>(A)), lda,
+                        reinterpret_cast<hipblasDoubleComplex*>(B), ldb);
+}
+
+cublasStatus_t cublasSsyrk(cublasHandle_t handle, cublasFillMode_t uplo, cublasOperation_t trans, int n, int k,
+                           const float* alpha, const float* A, int lda,
+                           const float* beta, float* C, int ldc) {
+    return hipblasSsyrk(handle, convert_hipblasFillMode_t(uplo), convert_hipblasOperation_t(trans), n, k,
+                        alpha, A, lda, beta, C, ldc);
+}
+
+cublasStatus_t cublasDsyrk(cublasHandle_t handle, cublasFillMode_t uplo, cublasOperation_t trans, int n, int k, 
+                           const double* alpha, const double* A, int lda,
+                           const double* beta, double* C, int ldc) {
+    return hipblasDsyrk(handle, convert_hipblasFillMode_t(uplo), convert_hipblasOperation_t(trans),  n, k, 
+                        alpha, A, lda, beta, C, ldc);
+}
+
+cublasStatus_t cublasCsyrk(cublasHandle_t handle, cublasFillMode_t uplo, cublasOperation_t trans, int n, int k,
+                           const cuComplex* alpha, const cuComplex* A,int lda,
+                           const cuComplex* beta, cuComplex* C, int ldc)
+{
+    return hipblasCsyrk(handle, convert_hipblasFillMode_t(uplo), convert_hipblasOperation_t(trans), n, k,
+                        reinterpret_cast<const hipblasComplex*>(alpha),
+                        reinterpret_cast<const hipblasComplex*>(A), lda,
+                        reinterpret_cast<const hipblasComplex*>(beta),
+                        reinterpret_cast<hipblasComplex*>(C), ldc);
+}
+
+cublasStatus_t cublasZsyrk(cublasHandle_t handle, cublasFillMode_t uplo, cublasOperation_t trans, int n, int k,
+                           const cuDoubleComplex* alpha, const cuDoubleComplex* A, int lda,
+                           const cuDoubleComplex* beta, cuDoubleComplex* C, int ldc)
+{
+    return hipblasZsyrk(handle, convert_hipblasFillMode_t(uplo), convert_hipblasOperation_t(trans), n, k,
+                        reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+                        reinterpret_cast<const hipblasDoubleComplex*>(A), lda,
+                        reinterpret_cast<const hipblasDoubleComplex*>(beta),
+                        reinterpret_cast<hipblasDoubleComplex*>(C), ldc);
+}
+
+// BLAS extension
+cublasStatus_t cublasSgeam(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, const float *alpha,
+        const float *A, int lda, const float *beta, const float *B, int ldb,
+        float *C, int ldc) {
+    return hipblasSgeam(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb), m, n, alpha, A, lda, beta, B, ldb, C, ldc);
+}
+
+cublasStatus_t cublasDgeam(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, const double *alpha,
+        const double *A, int lda, const double *beta, const double *B, int ldb,
+        double *C, int ldc) {
+    return hipblasDgeam(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb), m, n, alpha, A, lda, beta, B, ldb, C, ldc);
+}
+
+cublasStatus_t cublasCgeam(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, const cuComplex *alpha,
+        const cuComplex *A, int lda, const cuComplex *beta, const cuComplex *B, int ldb,
+        cuComplex *C, int ldc) {
+    #if HIP_VERSION < 307
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasCgeam(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb), m, n,
+                        reinterpret_cast<const hipblasComplex*>(alpha),
+                        reinterpret_cast<const hipblasComplex*>(A),
+                        lda,
+                        reinterpret_cast<const hipblasComplex*>(beta),
+                        reinterpret_cast<const hipblasComplex*>(B),
+                        ldb,
+                        reinterpret_cast<hipblasComplex*>(C),
+                        ldc);
+    #endif
+}
+
+cublasStatus_t cublasZgeam(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, const cuDoubleComplex *alpha,
+        const cuDoubleComplex *A, int lda, const cuDoubleComplex *beta, const cuDoubleComplex *B, int ldb,
+	    cuDoubleComplex *C, int ldc) {
+    #if HIP_VERSION < 307
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasZgeam(handle, convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb), m, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+                        reinterpret_cast<const hipblasDoubleComplex*>(A),
+                        lda,
+                        reinterpret_cast<const hipblasDoubleComplex*>(beta),
+                        reinterpret_cast<const hipblasDoubleComplex*>(B),
+                        ldb,
+                        reinterpret_cast<hipblasDoubleComplex*>(C),
+                        ldc);
+    #endif
+}
+
+cublasStatus_t cublasSdgmm(cublasHandle_t handle, cublasSideMode_t mode, int m, int n,
+                           const float *A, int lda,
+                           const float *x, int incx,
+                           float *C, int ldc) {
+    #if HIP_VERSION < 306
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasSdgmm(handle, convert_hipblasSideMode_t(mode), m, n, A, lda, x, incx, C, ldc);
+    #endif
+}
+
+cublasStatus_t cublasDdgmm(cublasHandle_t handle, cublasSideMode_t mode, int m, int n,
+                           const double *A, int lda,
+                           const double *x, int incx,
+                           double *C, int ldc) {
+    #if HIP_VERSION < 306
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasDdgmm(handle, convert_hipblasSideMode_t(mode), m, n, A, lda, x, incx, C, ldc);
+    #endif
+}
+
+cublasStatus_t cublasCdgmm(cublasHandle_t handle, cublasSideMode_t mode, int m, int n,
+                           const cuComplex *A, int lda,
+                           const cuComplex *x, int incx,
+                           cuComplex *C, int ldc) {
+    #if HIP_VERSION < 306
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasCdgmm(handle, convert_hipblasSideMode_t(mode), m, n,
+                        reinterpret_cast<const hipblasComplex*>(A), lda,
+                        reinterpret_cast<const hipblasComplex*>(x), incx,
+                        reinterpret_cast<hipblasComplex*>(C), ldc);
+    #endif
+}
+
+cublasStatus_t cublasZdgmm(cublasHandle_t handle, cublasSideMode_t mode, int m, int n,
+                           const cuDoubleComplex *A, int lda,
+                           const cuDoubleComplex *x, int incx,
+                           cuDoubleComplex *C, int ldc) {
+    #if HIP_VERSION < 306
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasZdgmm(handle, convert_hipblasSideMode_t(mode), m, n,
+                        reinterpret_cast<const hipblasDoubleComplex*>(A), lda,
+                        reinterpret_cast<const hipblasDoubleComplex*>(x), incx,
+                        reinterpret_cast<hipblasDoubleComplex*>(C), ldc);
+    #endif
+}
+
+cublasStatus_t cublasSgetriBatched(cublasHandle_t handle,
+                                   int n,
+                                   const float *const A[],
+                                   int lda,
+                                   const int *P,
+                                   float *const C[],
+                                   int ldc,
+                                   int *info,
+                                   int batchSize) {
+    #if HIP_VERSION < 308
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasSgetriBatched(handle, n, const_cast<float* const*>(A), lda, const_cast<int*>(P), C, ldc, info, batchSize);
+    #endif
+}
+
+cublasStatus_t cublasDgetriBatched(cublasHandle_t handle,
+                                   int n,
+                                   const double *const A[],
+                                   int lda,
+                                   const int *P,
+                                   double *const C[],
+                                   int ldc,
+                                   int *info,
+                                   int batchSize) {
+    #if HIP_VERSION < 308
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasDgetriBatched(handle, n, const_cast<double* const*>(A), lda, const_cast<int*>(P), C, ldc, info, batchSize);
+    #endif
+}
+
+cublasStatus_t cublasCgetriBatched(cublasHandle_t handle,
+                                   int n,
+                                   const cuComplex *const A[],
+                                   int lda,
+                                   const int *P,
+                                   cuComplex *const C[],
+                                   int ldc,
+                                   int *info,
+                                   int batchSize) {
+    #if HIP_VERSION < 308
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasCgetriBatched(handle, n,
+                                reinterpret_cast<hipblasComplex* const*>(const_cast<cuComplex* const*>(A)),
+                                lda, const_cast<int*>(P),
+                                reinterpret_cast<hipblasComplex* const*>(C),
+                                ldc, info, batchSize);
+    #endif
+}
+
+cublasStatus_t cublasZgetriBatched(cublasHandle_t handle,
+                                   int n,
+                                   const cuDoubleComplex *const A[],
+                                   int lda,
+                                   const int *P,
+                                   cuDoubleComplex *const C[],
+                                   int ldc,
+                                   int *info,
+                                   int batchSize) {
+    #if HIP_VERSION < 308
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+    #else
+    return hipblasZgetriBatched(handle, n,
+                                reinterpret_cast<hipblasDoubleComplex* const*>(const_cast<cuDoubleComplex* const*>(A)),
+                                lda, const_cast<int*>(P),
+                                reinterpret_cast<hipblasDoubleComplex* const*>(C),
+                                ldc, info, batchSize);
+    #endif
+}
+
+cublasStatus_t cublasSgemmStridedBatched(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k, const float *alpha,
+        const float *A, int lda, long long bsa,
+        const float *B, int ldb, long long bsb, const float *beta,
+        float *C, int ldc, long long bsc, int batchCount) {
+    return hipblasSgemmStridedBatched(
+        handle,
+        convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k, alpha,  A, lda, bsa, B, ldb, bsb, beta, C, ldc, bsc,
+        batchCount);
+}
+
+cublasStatus_t cublasDgemmStridedBatched(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k, const double *alpha,
+        const double *A, int lda, long long bsa,
+        const double *B, int ldb, long long bsb, const double *beta,
+        double *C, int ldc, long long bsc, int batchCount) {
+    return hipblasDgemmStridedBatched(
+        handle,
+        convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k, alpha,  A, lda, bsa, B, ldb, bsb, beta, C, ldc, bsc,
+        batchCount);
+}
+
+cublasStatus_t cublasCgemmStridedBatched(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k, const cuComplex *alpha,
+        const cuComplex *A, int lda, long long bsa,
+        const cuComplex *B, int ldb, long long bsb,
+        const cuComplex *beta,
+        cuComplex *C, int ldc, long long bsc, int batchCount) {
+    return hipblasCgemmStridedBatched(
+        handle,
+        convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k,
+        reinterpret_cast<const hipblasComplex *>(alpha),
+        reinterpret_cast<const hipblasComplex *>(A), lda, bsa,
+        reinterpret_cast<const hipblasComplex *>(B), ldb, bsb,
+        reinterpret_cast<const hipblasComplex *>(beta),
+        reinterpret_cast<hipblasComplex *>(C), ldc, bsc,
+        batchCount);
+}
+
+cublasStatus_t cublasZgemmStridedBatched(
+        cublasHandle_t handle,
+        cublasOperation_t transa, cublasOperation_t transb,
+        int m, int n, int k, const cuDoubleComplex *alpha,
+        const cuDoubleComplex *A, int lda, long long bsa,
+        const cuDoubleComplex *B, int ldb, long long bsb,
+        const cuDoubleComplex *beta,
+        cuDoubleComplex *C, int ldc, long long bsc, int batchCount) {
+    return hipblasZgemmStridedBatched(
+        handle,
+        convert_hipblasOperation_t(transa), convert_hipblasOperation_t(transb),
+        m, n, k,
+        reinterpret_cast<const hipblasDoubleComplex *>(alpha),
+        reinterpret_cast<const hipblasDoubleComplex *>(A), lda, bsa,
+        reinterpret_cast<const hipblasDoubleComplex *>(B), ldb, bsb,
+        reinterpret_cast<const hipblasDoubleComplex *>(beta),
+        reinterpret_cast<hipblasDoubleComplex *>(C), ldc, bsc,
+        batchCount);
+}
+
+cublasStatus_t cublasStrttp(...) {
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+}
+
+cublasStatus_t cublasDtrttp(...) {
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+}
+
+cublasStatus_t cublasStpttr(...) {
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+}
+
+cublasStatus_t cublasDtpttr(...) {
+    return HIPBLAS_STATUS_NOT_SUPPORTED;
+}
+
+cublasStatus_t cublasSgetrfBatched(cublasHandle_t handle, int n, float **Aarray, int lda,
+                                   int *PivotArray, int *infoArray, int batchSize) {
+    return hipblasSgetrfBatched(handle, n, Aarray, lda, PivotArray, infoArray, batchSize);
+}
+
+cublasStatus_t cublasDgetrfBatched(cublasHandle_t handle, int n, double **Aarray, int lda,
+                                   int *PivotArray, int *infoArray, int batchSize) {
+    return hipblasDgetrfBatched(handle, n, Aarray, lda, PivotArray, infoArray, batchSize);
+}
+
+cublasStatus_t cublasCgetrfBatched(cublasHandle_t handle, int n, cuComplex **Aarray, int lda,
+                                   int *PivotArray, int *infoArray, int batchSize) {
+    return hipblasCgetrfBatched(handle, n,
+                                reinterpret_cast<hipblasComplex** const>(Aarray), lda,
+                                PivotArray, infoArray, batchSize);
+}
+
+cublasStatus_t cublasZgetrfBatched(cublasHandle_t handle, int n, cuDoubleComplex **Aarray, int lda,
+                                   int *PivotArray, int *infoArray, int batchSize) {
+    return hipblasZgetrfBatched(handle, n,
+                                reinterpret_cast<hipblasDoubleComplex** const>(Aarray), lda,
+                                PivotArray, infoArray, batchSize);
+}
+
+cublasStatus_t cublasSgetrsBatched(cublasHandle_t handle,
+                                   cublasOperation_t trans,
+                                   int n,
+                                   int nrhs,
+                                   const float *const Aarray[],
+                                   int lda,
+                                   const int *devIpiv,
+                                   float *const Barray[],
+                                   int ldb,
+                                   int *info,
+                                   int batchSize) {
+    return hipblasSgetrsBatched(handle,
+                                convert_hipblasOperation_t(trans),
+                                n, nrhs,
+                                const_cast<float* const*>(Aarray), lda,
+                                devIpiv,
+                                Barray, ldb,
+                                info, batchSize);
+}
+
+cublasStatus_t cublasDgetrsBatched(cublasHandle_t handle,
+                                   cublasOperation_t trans,
+                                   int n,
+                                   int nrhs,
+                                   const double *const Aarray[],
+                                   int lda,
+                                   const int *devIpiv,
+                                   double *const Barray[],
+                                   int ldb,
+                                   int *info,
+                                   int batchSize) {
+    return hipblasDgetrsBatched(handle,
+                                convert_hipblasOperation_t(trans),
+                                n, nrhs,
+                                const_cast<double* const*>(Aarray), lda,
+                                devIpiv,
+                                Barray, ldb,
+                                info, batchSize);
+}
+
+cublasStatus_t cublasCgetrsBatched(cublasHandle_t handle,
+                                   cublasOperation_t trans,
+                                   int n,
+                                   int nrhs,
+                                   const cuComplex *const Aarray[],
+                                   int lda,
+                                   const int *devIpiv,
+                                   cuComplex *const Barray[],
+                                   int ldb,
+                                   int *info,
+                                   int batchSize) {
+    return hipblasCgetrsBatched(handle,
+                                convert_hipblasOperation_t(trans),
+                                n, nrhs,
+                                reinterpret_cast<hipblasComplex* const*>(const_cast<cuComplex* const*>(Aarray)), lda,
+                                devIpiv,
+                                reinterpret_cast<hipblasComplex* const*>(Barray), ldb,
+                                info, batchSize);
+}
+
+cublasStatus_t cublasZgetrsBatched(cublasHandle_t handle,
+                                   cublasOperation_t trans,
+                                   int n,
+                                   int nrhs,
+                                   const cuDoubleComplex** Aarray,
+                                   int lda,
+                                   const int *devIpiv,
+                                   cuDoubleComplex** Barray,
+                                   int ldb,
+                                   int *info,
+                                   int batchSize) {
+    return hipblasZgetrsBatched(handle,
+                                convert_hipblasOperation_t(trans),
+                                n, nrhs,
+                                reinterpret_cast<hipblasDoubleComplex* const*>(const_cast<cuDoubleComplex* const*>(Aarray)), lda,
+                                devIpiv,
+                                reinterpret_cast<hipblasDoubleComplex* const*>(Barray), ldb,
+                                info, batchSize);
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_HIPBLAS_H
diff --git a/cupy_backends/hip/cupy_hiprand.h b/cupy_backends/hip/cupy_hiprand.h
new file mode 100644
index 0000000..d3f7a6a
--- /dev/null
+++ b/cupy_backends/hip/cupy_hiprand.h
@@ -0,0 +1,104 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_HIPRAND_H
+#define INCLUDE_GUARD_HIP_CUPY_HIPRAND_H
+
+#include <hiprand/hiprand.h>
+#include "cupy_hip_common.h"
+
+extern "C" {
+
+typedef enum {} curandOrdering_t;
+typedef hiprandRngType curandRngType_t;
+typedef hiprandStatus_t curandStatus_t;
+
+typedef hiprandGenerator_t curandGenerator_t;
+
+curandRngType_t convert_hiprandRngType(curandRngType_t t) {
+    switch(static_cast<int>(t)) {
+    case 100: return HIPRAND_RNG_PSEUDO_DEFAULT;
+    case 101: return HIPRAND_RNG_PSEUDO_XORWOW;
+    case 121: return HIPRAND_RNG_PSEUDO_MRG32K3A;
+    case 141: return HIPRAND_RNG_PSEUDO_MTGP32;
+    case 142: return HIPRAND_RNG_PSEUDO_MT19937;
+    case 161: return HIPRAND_RNG_PSEUDO_PHILOX4_32_10;
+    case 200: return HIPRAND_RNG_QUASI_DEFAULT;
+    case 201: return HIPRAND_RNG_QUASI_SOBOL32;
+    case 202: return HIPRAND_RNG_QUASI_SCRAMBLED_SOBOL32;
+    case 203: return HIPRAND_RNG_QUASI_SOBOL64;
+    case 204: return HIPRAND_RNG_QUASI_SCRAMBLED_SOBOL64;
+    }
+    return HIPRAND_RNG_TEST;
+}
+
+// curandGenerator_t
+curandStatus_t curandCreateGenerator(curandGenerator_t *generator, curandRngType_t rng_type) {
+    rng_type = convert_hiprandRngType(rng_type);
+    return hiprandCreateGenerator(generator, rng_type);
+}
+
+curandStatus_t curandDestroyGenerator(curandGenerator_t generator) {
+    return hiprandDestroyGenerator(generator);
+}
+
+curandStatus_t curandGetVersion(int *version) {
+    return hiprandGetVersion(version);
+}
+
+
+// Stream
+curandStatus_t curandSetStream(curandGenerator_t generator, cudaStream_t stream) {
+    return hiprandSetStream(generator, stream);
+}
+
+curandStatus_t curandSetPseudoRandomGeneratorSeed(curandGenerator_t generator, unsigned long long seed) {
+    return hiprandSetPseudoRandomGeneratorSeed(generator, seed);
+}
+
+curandStatus_t curandSetGeneratorOffset(curandGenerator_t generator, unsigned long long offset) {
+    return hiprandSetGeneratorOffset(generator, offset);
+}
+
+curandStatus_t curandSetGeneratorOrdering(...) {
+    return HIPRAND_STATUS_NOT_IMPLEMENTED;
+}
+
+
+// Generation functions
+curandStatus_t curandGenerate(curandGenerator_t generator, unsigned int *output_data, size_t n) {
+    return hiprandGenerate(generator, output_data, n);
+}
+
+curandStatus_t curandGenerateLongLong(...) {
+    return HIPRAND_STATUS_NOT_IMPLEMENTED;
+}
+
+curandStatus_t curandGenerateUniform(curandGenerator_t generator, float *output_data, size_t n) {
+    return hiprandGenerateUniform(generator, output_data, n);
+}
+
+curandStatus_t curandGenerateUniformDouble(curandGenerator_t generator, double *output_data, size_t n) {
+    return hiprandGenerateUniformDouble(generator, output_data, n);
+}
+
+curandStatus_t curandGenerateNormal(curandGenerator_t generator, float *output_data, size_t n, float mean, float stddev) {
+    return hiprandGenerateNormal(generator, output_data, n, mean, stddev);
+}
+
+curandStatus_t curandGenerateNormalDouble(curandGenerator_t generator, double *output_data, size_t n, double mean, double stddev) {
+    return hiprandGenerateNormalDouble(generator, output_data, n, mean, stddev);
+}
+
+curandStatus_t curandGenerateLogNormal(curandGenerator_t generator, float *output_data, size_t n, float mean, float stddev) {
+    return hiprandGenerateLogNormal(generator, output_data, n, mean, stddev);
+}
+
+curandStatus_t curandGenerateLogNormalDouble(curandGenerator_t generator, double *output_data, size_t n, double mean, double stddev) {
+    return hiprandGenerateLogNormalDouble(generator, output_data, n, mean, stddev);
+}
+
+curandStatus_t curandGeneratePoisson(curandGenerator_t generator, unsigned int *output_data, size_t n, double lambda) {
+    return hiprandGeneratePoisson(generator, output_data, n, lambda);
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_HIPRAND_H
diff --git a/cupy_backends/hip/cupy_hiprtc.h b/cupy_backends/hip/cupy_hiprtc.h
new file mode 100644
index 0000000..4eadf2b
--- /dev/null
+++ b/cupy_backends/hip/cupy_hiprtc.h
@@ -0,0 +1,74 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_HIPRTC_H
+#define INCLUDE_GUARD_HIP_CUPY_HIPRTC_H
+
+#include <hip/hiprtc.h>
+
+extern "C" {
+
+typedef hiprtcResult nvrtcResult;
+const nvrtcResult NVRTC_SUCCESS = HIPRTC_SUCCESS;
+
+typedef hiprtcProgram nvrtcProgram;
+
+const char *nvrtcGetErrorString(nvrtcResult result) {
+    return hiprtcGetErrorString(result);
+}
+
+nvrtcResult nvrtcVersion(int* major, int* minor) {
+    return hiprtcVersion(major, minor);
+}
+
+nvrtcResult nvrtcCreateProgram(nvrtcProgram* prog, const char* src,
+                               const char* name, int numHeaders,
+                               const char** headers,
+                               const char** includeNames) {
+    return hiprtcCreateProgram(prog, src, name, numHeaders, headers, includeNames);
+}
+
+nvrtcResult nvrtcDestroyProgram(nvrtcProgram* prog) {
+    return hiprtcDestroyProgram(prog);
+}
+
+nvrtcResult nvrtcCompileProgram(nvrtcProgram prog, int numOptions,
+                                const char** options) {
+    return hiprtcCompileProgram(prog, numOptions, options);
+}
+
+nvrtcResult nvrtcGetPTXSize(nvrtcProgram prog, std::size_t* codeSizeRet) {
+    return hiprtcGetCodeSize(prog, codeSizeRet);
+}
+
+nvrtcResult nvrtcGetPTX(nvrtcProgram prog, char* code) {
+    return hiprtcGetCode(prog, code);
+}
+
+nvrtcResult nvrtcGetCUBINSize(...) {
+    return HIPRTC_ERROR_COMPILATION;
+}
+
+nvrtcResult nvrtcGetCUBIN(...) {
+    return HIPRTC_ERROR_COMPILATION;
+}
+
+nvrtcResult nvrtcGetProgramLogSize(nvrtcProgram prog, std::size_t* logSizeRet) {
+    return hiprtcGetProgramLogSize(prog, logSizeRet);
+}
+
+nvrtcResult nvrtcGetProgramLog(nvrtcProgram prog, char* log) {
+    return hiprtcGetProgramLog(prog, log);
+}
+
+nvrtcResult nvrtcAddNameExpression(nvrtcProgram prog,
+                                   const char* name_expression) {
+    return hiprtcAddNameExpression(prog, name_expression);
+}
+
+nvrtcResult nvrtcGetLoweredName(nvrtcProgram prog,
+                                const char* name_expression,
+                                const char** lowered_name ) {
+    return hiprtcGetLoweredName(prog, name_expression, lowered_name);
+}
+
+}
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_HIPRTC_H
diff --git a/cupy_backends/hip/cupy_hipsparse.h b/cupy_backends/hip/cupy_hipsparse.h
new file mode 100644
index 0000000..5b2e938
--- /dev/null
+++ b/cupy_backends/hip/cupy_hipsparse.h
@@ -0,0 +1,3881 @@
+
+
+#ifndef INCLUDE_GUARD_HIP_CUPY_HIPSPARSE_H
+#define INCLUDE_GUARD_HIP_CUPY_HIPSPARSE_H
+#include <hipsparse.h>
+#include <hip/hip_version.h>    // for HIP_VERSION
+#include <hip/library_types.h>  // for hipDataType
+#include <stdexcept>  // for gcc 10.0
+
+#if HIP_VERSION >= 402
+static hipDataType convert_hipDatatype(cudaDataType type) {
+    switch(static_cast<int>(type)) {
+        case 2 /* CUDA_R_16F */: return HIP_R_16F;
+        case 0 /* CUDA_R_32F */: return HIP_R_32F;
+        case 1 /* CUDA_R_64F */: return HIP_R_64F;
+        case 6 /* CUDA_C_16F */: return HIP_C_16F;
+        case 4 /* CUDA_C_32F */: return HIP_C_32F;
+        case 5 /* CUDA_C_64F */: return HIP_C_64F;
+        default: throw std::runtime_error("unrecognized type");
+    }
+}
+#endif
+
+
+#if HIP_VERSION < 401
+#define HIPSPARSE_STATUS_NOT_SUPPORTED (hipsparseStatus_t)10
+#endif
+
+
+extern "C" {
+
+typedef hipsparseIndexBase_t cusparseIndexBase_t;
+typedef hipsparseStatus_t cusparseStatus_t;
+
+typedef hipsparseHandle_t cusparseHandle_t;
+typedef hipsparseMatDescr_t cusparseMatDescr_t;
+#if HIP_VERSION < 308
+typedef void* bsric02Info_t;
+#endif
+
+#if HIP_VERSION < 309
+typedef void* bsrilu02Info_t;
+#endif
+
+
+typedef hipsparseMatrixType_t cusparseMatrixType_t;
+typedef hipsparseFillMode_t cusparseFillMode_t;
+typedef hipsparseDiagType_t cusparseDiagType_t;
+typedef hipsparseOperation_t cusparseOperation_t;
+typedef hipsparsePointerMode_t cusparsePointerMode_t;
+typedef hipsparseAction_t cusparseAction_t;
+typedef hipsparseDirection_t cusparseDirection_t;
+typedef enum {} cusparseAlgMode_t;
+typedef hipsparseSolvePolicy_t cusparseSolvePolicy_t;
+
+// Version
+cusparseStatus_t cusparseGetVersion(cusparseHandle_t handle,
+                                    int*             version) {
+  return hipsparseGetVersion(handle, version);
+}
+
+// Error handling
+const char* cusparseGetErrorName(...) {
+    // Unavailable in hipSparse; this should not be called
+    return "CUPY_HIPSPARSE_BINDING_UNEXPECTED_ERROR";
+}
+
+const char* cusparseGetErrorString(...) {
+    // Unavailable in hipSparse; this should not be called
+    return "unexpected error in CuPy hipSparse binding";
+}
+
+// cuSPARSE Helper Function
+cusparseStatus_t cusparseCreate(cusparseHandle_t* handle) {
+  return hipsparseCreate(handle);
+}
+
+cusparseStatus_t cusparseCreateMatDescr(cusparseMatDescr_t* descrA) {
+  return hipsparseCreateMatDescr(descrA);
+}
+
+cusparseStatus_t cusparseDestroy(cusparseHandle_t handle) {
+  return hipsparseDestroy(handle);
+}
+
+cusparseStatus_t cusparseDestroyMatDescr(cusparseMatDescr_t descrA) {
+  return hipsparseDestroyMatDescr(descrA);
+}
+
+cusparseStatus_t cusparseSetMatIndexBase(cusparseMatDescr_t  descrA,
+                                         cusparseIndexBase_t base) {
+  return hipsparseSetMatIndexBase(descrA, base);
+}
+
+cusparseStatus_t cusparseSetMatType(cusparseMatDescr_t   descrA,
+                                    cusparseMatrixType_t type) {
+  return hipsparseSetMatType(descrA, type);
+}
+
+cusparseStatus_t cusparseSetMatFillMode(cusparseMatDescr_t descrA,
+                                        cusparseFillMode_t fillMode) {
+  return hipsparseSetMatFillMode(descrA, fillMode);
+}
+
+cusparseStatus_t cusparseSetMatDiagType(cusparseMatDescr_t descrA,
+                                        cusparseDiagType_t diagType) {
+  return hipsparseSetMatDiagType(descrA, diagType);
+}
+
+cusparseStatus_t cusparseSetPointerMode(cusparseHandle_t      handle,
+                                        cusparsePointerMode_t mode) {
+  return hipsparseSetPointerMode(handle, mode);
+}
+
+// Stream
+cusparseStatus_t cusparseSetStream(cusparseHandle_t handle,
+                                   cudaStream_t     streamId) {
+  return hipsparseSetStream(handle, streamId);
+}
+
+cusparseStatus_t cusparseGetStream(cusparseHandle_t handle,
+                                   cudaStream_t*    streamId) {
+  return hipsparseGetStream(handle, streamId);
+}
+
+// cuSPARSE Level1 Function
+cusparseStatus_t cusparseSgthr(cusparseHandle_t    handle,
+                               int                 nnz,
+                               const float*        y,
+                               float*              xVal,
+                               const int*          xInd,
+                               cusparseIndexBase_t idxBase) {
+  return hipsparseSgthr(handle, nnz, y, xVal, xInd, idxBase);
+}
+
+cusparseStatus_t cusparseDgthr(cusparseHandle_t    handle,
+                               int                 nnz,
+                               const double*       y,
+                               double*             xVal,
+                               const int*          xInd,
+                               cusparseIndexBase_t idxBase) {
+  return hipsparseDgthr(handle, nnz, y, xVal, xInd, idxBase);
+}
+
+cusparseStatus_t cusparseCgthr(cusparseHandle_t    handle,
+                               int                 nnz,
+                               const cuComplex*    y,
+                               cuComplex*          xVal,
+                               const int*          xInd,
+                               cusparseIndexBase_t idxBase) {
+  return hipsparseCgthr(handle, nnz, reinterpret_cast<const hipComplex*>(y), reinterpret_cast<hipComplex*>(xVal), xInd, idxBase);
+}
+
+cusparseStatus_t cusparseZgthr(cusparseHandle_t       handle,
+                               int                    nnz,
+                               const cuDoubleComplex* y,
+                               cuDoubleComplex*       xVal,
+                               const int*             xInd,
+                               cusparseIndexBase_t    idxBase) {
+  return hipsparseZgthr(handle, nnz, reinterpret_cast<const hipDoubleComplex*>(y), reinterpret_cast<hipDoubleComplex*>(xVal), xInd, idxBase);
+}
+
+// cuSPARSE Level2 Function
+cusparseStatus_t cusparseScsrmv(cusparseHandle_t         handle,
+                                cusparseOperation_t      transA,
+                                int                      m,
+                                int                      n,
+                                int                      nnz,
+                                const float*             alpha,
+                                const cusparseMatDescr_t descrA,
+                                const float*             csrSortedValA,
+                                const int*               csrSortedRowPtrA,
+                                const int*               csrSortedColIndA,
+                                const float*             x,
+                                const float*             beta,
+                                float*                   y) {
+  return hipsparseScsrmv(handle, transA, m, n, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, x, beta, y);
+}
+
+cusparseStatus_t cusparseDcsrmv(cusparseHandle_t         handle,
+                                cusparseOperation_t      transA,
+                                int                      m,
+                                int                      n,
+                                int                      nnz,
+                                const double*            alpha,
+                                const cusparseMatDescr_t descrA,
+                                const double*            csrSortedValA,
+                                const int*               csrSortedRowPtrA,
+                                const int*               csrSortedColIndA,
+                                const double*            x,
+                                const double*            beta,
+                                double*                  y) {
+  return hipsparseDcsrmv(handle, transA, m, n, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, x, beta, y);
+}
+
+cusparseStatus_t cusparseCcsrmv(cusparseHandle_t         handle,
+                                cusparseOperation_t      transA,
+                                int                      m,
+                                int                      n,
+                                int                      nnz,
+                                const cuComplex*         alpha,
+                                const cusparseMatDescr_t descrA,
+                                const cuComplex*         csrSortedValA,
+                                const int*               csrSortedRowPtrA,
+                                const int*               csrSortedColIndA,
+                                const cuComplex*         x,
+                                const cuComplex*         beta,
+                                cuComplex*               y) {
+  return hipsparseCcsrmv(handle, transA, m, n, nnz, reinterpret_cast<const hipComplex*>(alpha), descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipComplex*>(x), reinterpret_cast<const hipComplex*>(beta), reinterpret_cast<hipComplex*>(y));
+}
+
+cusparseStatus_t cusparseZcsrmv(cusparseHandle_t         handle,
+                                cusparseOperation_t      transA,
+                                int                      m,
+                                int                      n,
+                                int                      nnz,
+                                const cuDoubleComplex*   alpha,
+                                const cusparseMatDescr_t descrA,
+                                const cuDoubleComplex*   csrSortedValA,
+                                const int*               csrSortedRowPtrA,
+                                const int*               csrSortedColIndA,
+                                const cuDoubleComplex*   x,
+                                const cuDoubleComplex*   beta,
+                                cuDoubleComplex*         y) {
+  return hipsparseZcsrmv(handle, transA, m, n, nnz, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipDoubleComplex*>(x), reinterpret_cast<const hipDoubleComplex*>(beta), reinterpret_cast<hipDoubleComplex*>(y));
+}
+
+cusparseStatus_t cusparseCsrmvEx_bufferSize(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCsrmvEx(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCreateCsrsv2Info(csrsv2Info_t* info) {
+  return hipsparseCreateCsrsv2Info(info);
+}
+
+cusparseStatus_t cusparseDestroyCsrsv2Info(csrsv2Info_t info) {
+  return hipsparseDestroyCsrsv2Info(info);
+}
+
+cusparseStatus_t cusparseScsrsv2_bufferSize(cusparseHandle_t         handle,
+                                            cusparseOperation_t      transA,
+                                            int                      m,
+                                            int                      nnz,
+                                            const cusparseMatDescr_t descrA,
+                                            float*                   csrSortedValA,
+                                            const int*               csrSortedRowPtrA,
+                                            const int*               csrSortedColIndA,
+                                            csrsv2Info_t             info,
+                                            int*                     pBufferSizeInBytes) {
+  return hipsparseScsrsv2_bufferSize(handle, transA, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseDcsrsv2_bufferSize(cusparseHandle_t         handle,
+                                            cusparseOperation_t      transA,
+                                            int                      m,
+                                            int                      nnz,
+                                            const cusparseMatDescr_t descrA,
+                                            double*                  csrSortedValA,
+                                            const int*               csrSortedRowPtrA,
+                                            const int*               csrSortedColIndA,
+                                            csrsv2Info_t             info,
+                                            int*                     pBufferSizeInBytes) {
+  return hipsparseDcsrsv2_bufferSize(handle, transA, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseCcsrsv2_bufferSize(cusparseHandle_t         handle,
+                                            cusparseOperation_t      transA,
+                                            int                      m,
+                                            int                      nnz,
+                                            const cusparseMatDescr_t descrA,
+                                            cuComplex*               csrSortedValA,
+                                            const int*               csrSortedRowPtrA,
+                                            const int*               csrSortedColIndA,
+                                            csrsv2Info_t             info,
+                                            int*                     pBufferSizeInBytes) {
+  return hipsparseCcsrsv2_bufferSize(handle, transA, m, nnz, descrA, reinterpret_cast<hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseZcsrsv2_bufferSize(cusparseHandle_t         handle,
+                                            cusparseOperation_t      transA,
+                                            int                      m,
+                                            int                      nnz,
+                                            const cusparseMatDescr_t descrA,
+                                            cuDoubleComplex*         csrSortedValA,
+                                            const int*               csrSortedRowPtrA,
+                                            const int*               csrSortedColIndA,
+                                            csrsv2Info_t             info,
+                                            int*                     pBufferSizeInBytes) {
+  return hipsparseZcsrsv2_bufferSize(handle, transA, m, nnz, descrA, reinterpret_cast<hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseScsrsv2_analysis(cusparseHandle_t         handle,
+                                          cusparseOperation_t      transA,
+                                          int                      m,
+                                          int                      nnz,
+                                          const cusparseMatDescr_t descrA,
+                                          const float*             csrSortedValA,
+                                          const int*               csrSortedRowPtrA,
+                                          const int*               csrSortedColIndA,
+                                          csrsv2Info_t             info,
+                                          cusparseSolvePolicy_t    policy,
+                                          void*                    pBuffer) {
+  return hipsparseScsrsv2_analysis(handle, transA, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseDcsrsv2_analysis(cusparseHandle_t         handle,
+                                          cusparseOperation_t      transA,
+                                          int                      m,
+                                          int                      nnz,
+                                          const cusparseMatDescr_t descrA,
+                                          const double*            csrSortedValA,
+                                          const int*               csrSortedRowPtrA,
+                                          const int*               csrSortedColIndA,
+                                          csrsv2Info_t             info,
+                                          cusparseSolvePolicy_t    policy,
+                                          void*                    pBuffer) {
+  return hipsparseDcsrsv2_analysis(handle, transA, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseCcsrsv2_analysis(cusparseHandle_t         handle,
+                                          cusparseOperation_t      transA,
+                                          int                      m,
+                                          int                      nnz,
+                                          const cusparseMatDescr_t descrA,
+                                          const cuComplex*         csrSortedValA,
+                                          const int*               csrSortedRowPtrA,
+                                          const int*               csrSortedColIndA,
+                                          csrsv2Info_t             info,
+                                          cusparseSolvePolicy_t    policy,
+                                          void*                    pBuffer) {
+  return hipsparseCcsrsv2_analysis(handle, transA, m, nnz, descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseZcsrsv2_analysis(cusparseHandle_t         handle,
+                                          cusparseOperation_t      transA,
+                                          int                      m,
+                                          int                      nnz,
+                                          const cusparseMatDescr_t descrA,
+                                          const cuDoubleComplex*   csrSortedValA,
+                                          const int*               csrSortedRowPtrA,
+                                          const int*               csrSortedColIndA,
+                                          csrsv2Info_t             info,
+                                          cusparseSolvePolicy_t    policy,
+                                          void*                    pBuffer) {
+  return hipsparseZcsrsv2_analysis(handle, transA, m, nnz, descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseScsrsv2_solve(cusparseHandle_t         handle,
+                                       cusparseOperation_t      transA,
+                                       int                      m,
+                                       int                      nnz,
+                                       const float*             alpha,
+                                       const cusparseMatDescr_t descrA,
+                                       const float*             csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       const int*               csrSortedColIndA,
+                                       csrsv2Info_t             info,
+                                       const float*             f,
+                                       float*                   x,
+                                       cusparseSolvePolicy_t    policy,
+                                       void*                    pBuffer) {
+  return hipsparseScsrsv2_solve(handle, transA, m, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, f, x, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseDcsrsv2_solve(cusparseHandle_t         handle,
+                                       cusparseOperation_t      transA,
+                                       int                      m,
+                                       int                      nnz,
+                                       const double*            alpha,
+                                       const cusparseMatDescr_t descrA,
+                                       const double*            csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       const int*               csrSortedColIndA,
+                                       csrsv2Info_t             info,
+                                       const double*            f,
+                                       double*                  x,
+                                       cusparseSolvePolicy_t    policy,
+                                       void*                    pBuffer) {
+  return hipsparseDcsrsv2_solve(handle, transA, m, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, f, x, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseCcsrsv2_solve(cusparseHandle_t         handle,
+                                       cusparseOperation_t      transA,
+                                       int                      m,
+                                       int                      nnz,
+                                       const cuComplex*         alpha,
+                                       const cusparseMatDescr_t descrA,
+                                       const cuComplex*         csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       const int*               csrSortedColIndA,
+                                       csrsv2Info_t             info,
+                                       const cuComplex*         f,
+                                       cuComplex*               x,
+                                       cusparseSolvePolicy_t    policy,
+                                       void*                    pBuffer) {
+  return hipsparseCcsrsv2_solve(handle, transA, m, nnz, reinterpret_cast<const hipComplex*>(alpha), descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, reinterpret_cast<const hipComplex*>(f), reinterpret_cast<hipComplex*>(x), policy, pBuffer);
+}
+
+cusparseStatus_t cusparseZcsrsv2_solve(cusparseHandle_t         handle,
+                                       cusparseOperation_t      transA,
+                                       int                      m,
+                                       int                      nnz,
+                                       const cuDoubleComplex*   alpha,
+                                       const cusparseMatDescr_t descrA,
+                                       const cuDoubleComplex*   csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       const int*               csrSortedColIndA,
+                                       csrsv2Info_t             info,
+                                       const cuDoubleComplex*   f,
+                                       cuDoubleComplex*         x,
+                                       cusparseSolvePolicy_t    policy,
+                                       void*                    pBuffer) {
+  return hipsparseZcsrsv2_solve(handle, transA, m, nnz, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, reinterpret_cast<const hipDoubleComplex*>(f), reinterpret_cast<hipDoubleComplex*>(x), policy, pBuffer);
+}
+
+cusparseStatus_t cusparseXcsrsv2_zeroPivot(cusparseHandle_t handle,
+                                           csrsv2Info_t     info,
+                                           int*             position) {
+  return hipsparseXcsrsv2_zeroPivot(handle, info, position);
+}
+
+// cuSPARSE Level3 Function
+cusparseStatus_t cusparseScsrmm(cusparseHandle_t         handle,
+                                cusparseOperation_t      transA,
+                                int                      m,
+                                int                      n,
+                                int                      k,
+                                int                      nnz,
+                                const float*             alpha,
+                                const cusparseMatDescr_t descrA,
+                                const float*             csrSortedValA,
+                                const int*               csrSortedRowPtrA,
+                                const int*               csrSortedColIndA,
+                                const float*             B,
+                                int                      ldb,
+                                const float*             beta,
+                                float*                   C,
+                                int                      ldc) {
+  return hipsparseScsrmm(handle, transA, m, n, k, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, beta, C, ldc);
+}
+
+cusparseStatus_t cusparseDcsrmm(cusparseHandle_t         handle,
+                                cusparseOperation_t      transA,
+                                int                      m,
+                                int                      n,
+                                int                      k,
+                                int                      nnz,
+                                const double*            alpha,
+                                const cusparseMatDescr_t descrA,
+                                const double*            csrSortedValA,
+                                const int*               csrSortedRowPtrA,
+                                const int*               csrSortedColIndA,
+                                const double*            B,
+                                int                      ldb,
+                                const double*            beta,
+                                double*                  C,
+                                int                      ldc) {
+  return hipsparseDcsrmm(handle, transA, m, n, k, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, beta, C, ldc);
+}
+
+cusparseStatus_t cusparseCcsrmm(cusparseHandle_t         handle,
+                                cusparseOperation_t      transA,
+                                int                      m,
+                                int                      n,
+                                int                      k,
+                                int                      nnz,
+                                const cuComplex*         alpha,
+                                const cusparseMatDescr_t descrA,
+                                const cuComplex*         csrSortedValA,
+                                const int*               csrSortedRowPtrA,
+                                const int*               csrSortedColIndA,
+                                const cuComplex*         B,
+                                int                      ldb,
+                                const cuComplex*         beta,
+                                cuComplex*               C,
+                                int                      ldc) {
+  return hipsparseCcsrmm(handle, transA, m, n, k, nnz, reinterpret_cast<const hipComplex*>(alpha), descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipComplex*>(B), ldb, reinterpret_cast<const hipComplex*>(beta), reinterpret_cast<hipComplex*>(C), ldc);
+}
+
+cusparseStatus_t cusparseZcsrmm(cusparseHandle_t         handle,
+                                cusparseOperation_t      transA,
+                                int                      m,
+                                int                      n,
+                                int                      k,
+                                int                      nnz,
+                                const cuDoubleComplex*   alpha,
+                                const cusparseMatDescr_t descrA,
+                                const cuDoubleComplex*   csrSortedValA,
+                                const int*               csrSortedRowPtrA,
+                                const int*               csrSortedColIndA,
+                                const cuDoubleComplex*   B,
+                                int                      ldb,
+                                const cuDoubleComplex*   beta,
+                                cuDoubleComplex*         C,
+                                int                      ldc) {
+  return hipsparseZcsrmm(handle, transA, m, n, k, nnz, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipDoubleComplex*>(B), ldb, reinterpret_cast<const hipDoubleComplex*>(beta), reinterpret_cast<hipDoubleComplex*>(C), ldc);
+}
+
+cusparseStatus_t cusparseScsrmm2(cusparseHandle_t         handle,
+                                 cusparseOperation_t      transA,
+                                 cusparseOperation_t      transB,
+                                 int                      m,
+                                 int                      n,
+                                 int                      k,
+                                 int                      nnz,
+                                 const float*             alpha,
+                                 const cusparseMatDescr_t descrA,
+                                 const float*             csrSortedValA,
+                                 const int*               csrSortedRowPtrA,
+                                 const int*               csrSortedColIndA,
+                                 const float*             B,
+                                 int                      ldb,
+                                 const float*             beta,
+                                 float*                   C,
+                                 int                      ldc) {
+  return hipsparseScsrmm2(handle, transA, transB, m, n, k, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, beta, C, ldc);
+}
+
+cusparseStatus_t cusparseDcsrmm2(cusparseHandle_t         handle,
+                                 cusparseOperation_t      transA,
+                                 cusparseOperation_t      transB,
+                                 int                      m,
+                                 int                      n,
+                                 int                      k,
+                                 int                      nnz,
+                                 const double*            alpha,
+                                 const cusparseMatDescr_t descrA,
+                                 const double* csrSortedValA,
+                                 const int*    csrSortedRowPtrA,
+                                 const int*    csrSortedColIndA,
+                                 const double* B,
+                                 int           ldb,
+                                 const double* beta,
+                                 double*       C,
+                                 int           ldc) {
+  return hipsparseDcsrmm2(handle, transA, transB, m, n, k, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, beta, C, ldc);
+}
+
+cusparseStatus_t cusparseCcsrmm2(cusparseHandle_t         handle,
+                                 cusparseOperation_t      transA,
+                                 cusparseOperation_t      transB,
+                                 int                      m,
+                                 int                      n,
+                                 int                      k,
+                                 int                      nnz,
+                                 const cuComplex*         alpha,
+                                 const cusparseMatDescr_t descrA,
+                                 const cuComplex* csrSortedValA,
+                                 const int*       csrSortedRowPtrA,
+                                 const int*       csrSortedColIndA,
+                                 const cuComplex* B,
+                                 int              ldb,
+                                 const cuComplex* beta,
+                                 cuComplex*       C,
+                                 int              ldc) {
+  return hipsparseCcsrmm2(handle, transA, transB, m, n, k, nnz, reinterpret_cast<const hipComplex*>(alpha), descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipComplex*>(B), ldb, reinterpret_cast<const hipComplex*>(beta), reinterpret_cast<hipComplex*>(C), ldc);
+}
+
+cusparseStatus_t cusparseZcsrmm2(cusparseHandle_t         handle,
+                                 cusparseOperation_t      transA,
+                                 cusparseOperation_t      transB,
+                                 int                      m,
+                                 int                      n,
+                                 int                      k,
+                                 int                      nnz,
+                                 const cuDoubleComplex*   alpha,
+                                 const cusparseMatDescr_t descrA,
+                                 const cuDoubleComplex*   csrSortedValA,
+                                 const int*               csrSortedRowPtrA,
+                                 const int*               csrSortedColIndA,
+                                 const cuDoubleComplex*   B,
+                                 int                      ldb,
+                                 const cuDoubleComplex*   beta,
+                                 cuDoubleComplex*         C,
+                                 int                      ldc) {
+  return hipsparseZcsrmm2(handle, transA, transB, m, n, k, nnz, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipDoubleComplex*>(B), ldb, reinterpret_cast<const hipDoubleComplex*>(beta), reinterpret_cast<hipDoubleComplex*>(C), ldc);
+}
+
+cusparseStatus_t cusparseCreateCsrsm2Info(csrsm2Info_t* info) {
+  return hipsparseCreateCsrsm2Info(info);
+}
+cusparseStatus_t cusparseDestroyCsrsm2Info(csrsm2Info_t info) {
+  return hipsparseDestroyCsrsm2Info(info);
+}
+
+cusparseStatus_t cusparseScsrsm2_bufferSizeExt(cusparseHandle_t         handle,
+                                               int                      algo,
+                                               cusparseOperation_t      transA,
+                                               cusparseOperation_t      transB,
+                                               int                      m,
+                                               int                      nrhs,
+                                               int                      nnz,
+                                               const float*             alpha,
+                                               const cusparseMatDescr_t descrA,
+                                               const float*             csrSortedValA,
+                                               const int*               csrSortedRowPtrA,
+                                               const int*               csrSortedColIndA,
+                                               const float*             B,
+                                               int                      ldb,
+                                               csrsm2Info_t             info,
+                                               cusparseSolvePolicy_t    policy,
+                                               size_t*                  pBufferSize) {
+  return hipsparseScsrsm2_bufferSizeExt(handle, algo, transA, transB, m, nrhs, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, info, policy, pBufferSize);
+}
+cusparseStatus_t cusparseDcsrsm2_bufferSizeExt(cusparseHandle_t         handle,
+                                               int                      algo,
+                                               cusparseOperation_t      transA,
+                                               cusparseOperation_t      transB,
+                                               int                      m,
+                                               int                      nrhs,
+                                               int                      nnz,
+                                               const double*            alpha,
+                                               const cusparseMatDescr_t descrA,
+                                               const double*            csrSortedValA,
+                                               const int*               csrSortedRowPtrA,
+                                               const int*               csrSortedColIndA,
+                                               const double*            B,
+                                               int                      ldb,
+                                               csrsm2Info_t             info,
+                                               cusparseSolvePolicy_t    policy,
+                                               size_t*                  pBufferSize) {
+  return hipsparseDcsrsm2_bufferSizeExt(handle, algo, transA, transB, m, nrhs, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, info, policy, pBufferSize);
+}
+cusparseStatus_t cusparseCcsrsm2_bufferSizeExt(cusparseHandle_t         handle,
+                                               int                      algo,
+                                               cusparseOperation_t      transA,
+                                               cusparseOperation_t      transB,
+                                               int                      m,
+                                               int                      nrhs,
+                                               int                      nnz,
+                                               const cuComplex*         alpha,
+                                               const cusparseMatDescr_t descrA,
+                                               const cuComplex*         csrSortedValA,
+                                               const int*               csrSortedRowPtrA,
+                                               const int*               csrSortedColIndA,
+                                               const cuComplex*         B,
+                                               int                      ldb,
+                                               csrsm2Info_t             info,
+                                               cusparseSolvePolicy_t    policy,
+                                               size_t*                  pBufferSize) {
+  return hipsparseCcsrsm2_bufferSizeExt(handle, algo, transA, transB, m, nrhs, nnz, reinterpret_cast<const hipComplex*>(alpha), descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipComplex*>(B), ldb, info, policy, pBufferSize);
+}
+cusparseStatus_t cusparseZcsrsm2_bufferSizeExt(cusparseHandle_t         handle,
+                                               int                      algo,
+                                               cusparseOperation_t      transA,
+                                               cusparseOperation_t      transB,
+                                               int                      m,
+                                               int                      nrhs,
+                                               int                      nnz,
+                                               const cuDoubleComplex*   alpha,
+                                               const cusparseMatDescr_t descrA,
+                                               const cuDoubleComplex*   csrSortedValA,
+                                               const int*               csrSortedRowPtrA,
+                                               const int*               csrSortedColIndA,
+                                               const cuDoubleComplex*   B,
+                                               int                      ldb,
+                                               csrsm2Info_t             info,
+                                               cusparseSolvePolicy_t    policy,
+                                               size_t*                  pBufferSize) {
+  return hipsparseZcsrsm2_bufferSizeExt(handle, algo, transA, transB, m, nrhs, nnz, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipDoubleComplex*>(B), ldb, info, policy, pBufferSize);
+}
+
+cusparseStatus_t cusparseScsrsm2_analysis(cusparseHandle_t         handle,
+                                          int                      algo,
+                                          cusparseOperation_t      transA,
+                                          cusparseOperation_t      transB,
+                                          int                      m,
+                                          int                      nrhs,
+                                          int                      nnz,
+                                          const float*             alpha,
+                                          const cusparseMatDescr_t descrA,
+                                          const float*             csrSortedValA,
+                                          const int*               csrSortedRowPtrA,
+                                          const int*               csrSortedColIndA,
+                                          const float*             B,
+                                          int                      ldb,
+                                          csrsm2Info_t             info,
+                                          cusparseSolvePolicy_t    policy,
+                                          void*                    pBuffer) {
+  return hipsparseScsrsm2_analysis(handle, algo, transA, transB, m, nrhs, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, info, policy, pBuffer);
+}
+cusparseStatus_t cusparseDcsrsm2_analysis(cusparseHandle_t         handle,
+                                          int                      algo,
+                                          cusparseOperation_t      transA,
+                                          cusparseOperation_t      transB,
+                                          int                      m,
+                                          int                      nrhs,
+                                          int                      nnz,
+                                          const double*            alpha,
+                                          const cusparseMatDescr_t descrA,
+                                          const double*            csrSortedValA,
+                                          const int*               csrSortedRowPtrA,
+                                          const int*               csrSortedColIndA,
+                                          const double*            B,
+                                          int                      ldb,
+                                          csrsm2Info_t             info,
+                                          cusparseSolvePolicy_t    policy,
+                                          void*                    pBuffer) {
+  return hipsparseDcsrsm2_analysis(handle, algo, transA, transB, m, nrhs, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, info, policy, pBuffer);
+}
+cusparseStatus_t cusparseCcsrsm2_analysis(cusparseHandle_t         handle,
+                                          int                      algo,
+                                          cusparseOperation_t      transA,
+                                          cusparseOperation_t      transB,
+                                          int                      m,
+                                          int                      nrhs,
+                                          int                      nnz,
+                                          const cuComplex*         alpha,
+                                          const cusparseMatDescr_t descrA,
+                                          const cuComplex*         csrSortedValA,
+                                          const int*               csrSortedRowPtrA,
+                                          const int*               csrSortedColIndA,
+                                          const cuComplex*         B,
+                                          int                      ldb,
+                                          csrsm2Info_t             info,
+                                          cusparseSolvePolicy_t    policy,
+                                          void*                    pBuffer) {
+  return hipsparseCcsrsm2_analysis(handle, algo, transA, transB, m, nrhs, nnz, reinterpret_cast<const hipComplex*>(alpha), descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipComplex*>(B), ldb, info, policy, pBuffer);
+}
+cusparseStatus_t cusparseZcsrsm2_analysis(cusparseHandle_t         handle,
+                                          int                      algo,
+                                          cusparseOperation_t      transA,
+                                          cusparseOperation_t      transB,
+                                          int                      m,
+                                          int                      nrhs,
+                                          int                      nnz,
+                                          const cuDoubleComplex*   alpha,
+                                          const cusparseMatDescr_t descrA,
+                                          const cuDoubleComplex*   csrSortedValA,
+                                          const int*               csrSortedRowPtrA,
+                                          const int*               csrSortedColIndA,
+                                          const cuDoubleComplex*   B,
+                                          int                      ldb,
+                                          csrsm2Info_t             info,
+                                          cusparseSolvePolicy_t    policy,
+                                          void*                    pBuffer) {
+  return hipsparseZcsrsm2_analysis(handle, algo, transA, transB, m, nrhs, nnz, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipDoubleComplex*>(B), ldb, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseScsrsm2_solve(cusparseHandle_t         handle,
+                                       int                      algo,
+                                       cusparseOperation_t      transA,
+                                       cusparseOperation_t      transB,
+                                       int                      m,
+                                       int                      nrhs,
+                                       int                      nnz,
+                                       const float*             alpha,
+                                       const cusparseMatDescr_t descrA,
+                                       const float*             csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       const int*               csrSortedColIndA,
+                                       float*                   B,
+                                       int                      ldb,
+                                       csrsm2Info_t             info,
+                                       cusparseSolvePolicy_t    policy,
+                                       void*                    pBuffer) {
+  return hipsparseScsrsm2_solve(handle, algo, transA, transB, m, nrhs, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, info, policy, pBuffer);
+}
+cusparseStatus_t cusparseDcsrsm2_solve(cusparseHandle_t         handle,
+                                       int                      algo,
+                                       cusparseOperation_t      transA,
+                                       cusparseOperation_t      transB,
+                                       int                      m,
+                                       int                      nrhs,
+                                       int                      nnz,
+                                       const double*            alpha,
+                                       const cusparseMatDescr_t descrA,
+                                       const double*            csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       const int*               csrSortedColIndA,
+                                       double*                  B,
+                                       int                      ldb,
+                                       csrsm2Info_t             info,
+                                       cusparseSolvePolicy_t    policy,
+                                       void*                    pBuffer) {
+  return hipsparseDcsrsm2_solve(handle, algo, transA, transB, m, nrhs, nnz, alpha, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, B, ldb, info, policy, pBuffer);
+}
+cusparseStatus_t cusparseCcsrsm2_solve(cusparseHandle_t         handle,
+                                       int                      algo,
+                                       cusparseOperation_t      transA,
+                                       cusparseOperation_t      transB,
+                                       int                      m,
+                                       int                      nrhs,
+                                       int                      nnz,
+                                       const cuComplex*         alpha,
+                                       const cusparseMatDescr_t descrA,
+                                       const cuComplex*         csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       const int*               csrSortedColIndA,
+                                       cuComplex*               B,
+                                       int                      ldb,
+                                       csrsm2Info_t             info,
+                                       cusparseSolvePolicy_t    policy,
+                                       void*                    pBuffer) {
+  return hipsparseCcsrsm2_solve(handle, algo, transA, transB, m, nrhs, nnz, reinterpret_cast<const hipComplex*>(alpha), descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<hipComplex*>(B), ldb, info, policy, pBuffer);
+}
+cusparseStatus_t cusparseZcsrsm2_solve(cusparseHandle_t         handle,
+                                       int                      algo,
+                                       cusparseOperation_t      transA,
+                                       cusparseOperation_t      transB,
+                                       int                      m,
+                                       int                      nrhs,
+                                       int                      nnz,
+                                       const cuDoubleComplex*   alpha,
+                                       const cusparseMatDescr_t descrA,
+                                       const cuDoubleComplex*   csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       const int*               csrSortedColIndA,
+                                       cuDoubleComplex*         B,
+                                       int                      ldb,
+                                       csrsm2Info_t             info,
+                                       cusparseSolvePolicy_t    policy,
+                                       void*                    pBuffer) {
+  return hipsparseZcsrsm2_solve(handle, algo, transA, transB, m, nrhs, nnz, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<hipDoubleComplex*>(B), ldb, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseXcsrsm2_zeroPivot(cusparseHandle_t handle,
+                                           csrsm2Info_t     info,
+                                           int* position) {
+  return hipsparseXcsrsm2_zeroPivot(handle, info, position);
+}
+
+// cuSPARSE Extra Function
+cusparseStatus_t cusparseXcsrgeamNnz(cusparseHandle_t         handle,
+                                     int                      m,
+                                     int                      n,
+                                     const cusparseMatDescr_t descrA,
+                                     int                      nnzA,
+                                     const int*               csrSortedRowPtrA,
+                                     const int*               csrSortedColIndA,
+                                     const cusparseMatDescr_t descrB,
+                                     int                      nnzB,
+                                     const int*               csrSortedRowPtrB,
+                                     const int*               csrSortedColIndB,
+                                     const cusparseMatDescr_t descrC,
+                                     int*                     csrSortedRowPtrC,
+                                     int*                     nnzTotalDevHostPtr) {
+  return hipsparseXcsrgeamNnz(handle, m, n, descrA, nnzA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedRowPtrC, nnzTotalDevHostPtr);
+}
+
+cusparseStatus_t cusparseScsrgeam(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      n,
+                                  const float*             alpha,
+                                  const cusparseMatDescr_t descrA,
+                                  int                      nnzA,
+                                  const float*             csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const float*             beta,
+                                  const cusparseMatDescr_t descrB,
+                                  int                      nnzB,
+                                  const float*             csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cusparseMatDescr_t descrC,
+                                  float*                   csrSortedValC,
+                                  int*                     csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC) {
+  return hipsparseScsrgeam(handle, m, n, alpha, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, beta, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC);
+}
+
+cusparseStatus_t cusparseDcsrgeam(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      n,
+                                  const double*            alpha,
+                                  const cusparseMatDescr_t descrA,
+                                  int                      nnzA,
+                                  const double*            csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const double*            beta,
+                                  const cusparseMatDescr_t descrB,
+                                  int                      nnzB,
+                                  const double*            csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cusparseMatDescr_t descrC,
+                                  double*                  csrSortedValC,
+                                  int*                     csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC) {
+  return hipsparseDcsrgeam(handle, m, n, alpha, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, beta, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC);
+}
+
+cusparseStatus_t cusparseCcsrgeam(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      n,
+                                  const cuComplex*         alpha,
+                                  const cusparseMatDescr_t descrA,
+                                  int                      nnzA,
+                                  const cuComplex*         csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const cuComplex*         beta,
+                                  const cusparseMatDescr_t descrB,
+                                  int                      nnzB,
+                                  const cuComplex*         csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cusparseMatDescr_t descrC,
+                                  cuComplex*               csrSortedValC,
+                                  int*                     csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC) {
+  return hipsparseCcsrgeam(handle, m, n, reinterpret_cast<const hipComplex*>(alpha), descrA, nnzA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipComplex*>(beta), descrB, nnzB, reinterpret_cast<const hipComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, descrC, reinterpret_cast<hipComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC);
+}
+
+cusparseStatus_t cusparseZcsrgeam(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      n,
+                                  const cuDoubleComplex*   alpha,
+                                  const cusparseMatDescr_t descrA,
+                                  int                      nnzA,
+                                  const cuDoubleComplex*   csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const cuDoubleComplex*   beta,
+                                  const cusparseMatDescr_t descrB,
+                                  int                      nnzB,
+                                  const cuDoubleComplex*   csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cusparseMatDescr_t descrC,
+                                  cuDoubleComplex*         csrSortedValC,
+                                  int*                     csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC) {
+  return hipsparseZcsrgeam(handle, m, n, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, nnzA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipDoubleComplex*>(beta), descrB, nnzB, reinterpret_cast<const hipDoubleComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, descrC, reinterpret_cast<hipDoubleComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC);
+}
+
+cusparseStatus_t cusparseScsrgeam2_bufferSizeExt(cusparseHandle_t         handle,
+                                                 int                      m,
+                                                 int                      n,
+                                                 const float*             alpha,
+                                                 const cusparseMatDescr_t descrA,
+                                                 int                      nnzA,
+                                                 const float*             csrSortedValA,
+                                                 const int*               csrSortedRowPtrA,
+                                                 const int*               csrSortedColIndA,
+                                                 const float*             beta,
+                                                 const cusparseMatDescr_t descrB,
+                                                 int                      nnzB,
+                                                 const float*             csrSortedValB,
+                                                 const int*               csrSortedRowPtrB,
+                                                 const int*               csrSortedColIndB,
+                                                 const cusparseMatDescr_t descrC,
+                                                 const float*             csrSortedValC,
+                                                 const int*               csrSortedRowPtrC,
+                                                 const int*               csrSortedColIndC,
+                                                 size_t*                  pBufferSizeInBytes) {
+  return hipsparseScsrgeam2_bufferSizeExt(handle, m, n, alpha, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, beta, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseDcsrgeam2_bufferSizeExt(cusparseHandle_t         handle,
+                                                 int                      m,
+                                                 int                      n,
+                                                 const double*            alpha,
+                                                 const cusparseMatDescr_t descrA,
+                                                 int                      nnzA,
+                                                 const double*            csrSortedValA,
+                                                 const int*               csrSortedRowPtrA,
+                                                 const int*               csrSortedColIndA,
+                                                 const double*            beta,
+                                                 const cusparseMatDescr_t descrB,
+                                                 int                      nnzB,
+                                                 const double*            csrSortedValB,
+                                                 const int*               csrSortedRowPtrB,
+                                                 const int*               csrSortedColIndB,
+                                                 const cusparseMatDescr_t descrC,
+                                                 const double*            csrSortedValC,
+                                                 const int*               csrSortedRowPtrC,
+                                                 const int*               csrSortedColIndC,
+                                                 size_t*                  pBufferSizeInBytes) {
+  return hipsparseDcsrgeam2_bufferSizeExt(handle, m, n, alpha, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, beta, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseCcsrgeam2_bufferSizeExt(cusparseHandle_t         handle,
+                                                 int                      m,
+                                                 int                      n,
+                                                 const cuComplex*         alpha,
+                                                 const cusparseMatDescr_t descrA,
+                                                 int                      nnzA,
+                                                 const cuComplex*         csrSortedValA,
+                                                 const int*               csrSortedRowPtrA,
+                                                 const int*               csrSortedColIndA,
+                                                 const cuComplex*         beta,
+                                                 const cusparseMatDescr_t descrB,
+                                                 int                      nnzB,
+                                                 const cuComplex*         csrSortedValB,
+                                                 const int*               csrSortedRowPtrB,
+                                                 const int*               csrSortedColIndB,
+                                                 const cusparseMatDescr_t descrC,
+                                                 const cuComplex*         csrSortedValC,
+                                                 const int*               csrSortedRowPtrC,
+                                                 const int*               csrSortedColIndC,
+                                                 size_t*                  pBufferSizeInBytes) {
+  return hipsparseCcsrgeam2_bufferSizeExt(handle, m, n, reinterpret_cast<const hipComplex*>(alpha), descrA, nnzA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipComplex*>(beta), descrB, nnzB, reinterpret_cast<const hipComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, descrC, reinterpret_cast<const hipComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseZcsrgeam2_bufferSizeExt(cusparseHandle_t         handle,
+                                                 int                      m,
+                                                 int                      n,
+                                                 const cuDoubleComplex*   alpha,
+                                                 const cusparseMatDescr_t descrA,
+                                                 int                      nnzA,
+                                                 const cuDoubleComplex*   csrSortedValA,
+                                                 const int*               csrSortedRowPtrA,
+                                                 const int*               csrSortedColIndA,
+                                                 const cuDoubleComplex*   beta,
+                                                 const cusparseMatDescr_t descrB,
+                                                 int                      nnzB,
+                                                 const cuDoubleComplex*   csrSortedValB,
+                                                 const int*               csrSortedRowPtrB,
+                                                 const int*               csrSortedColIndB,
+                                                 const cusparseMatDescr_t descrC,
+                                                 const cuDoubleComplex*   csrSortedValC,
+                                                 const int*               csrSortedRowPtrC,
+                                                 const int*               csrSortedColIndC,
+                                                 size_t*                  pBufferSizeInBytes) {
+  return hipsparseZcsrgeam2_bufferSizeExt(handle, m, n, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, nnzA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipDoubleComplex*>(beta), descrB, nnzB, reinterpret_cast<const hipDoubleComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, descrC, reinterpret_cast<const hipDoubleComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseXcsrgeam2Nnz(cusparseHandle_t         handle,
+                                      int                      m,
+                                      int                      n,
+                                      const cusparseMatDescr_t descrA,
+                                      int                      nnzA,
+                                      const int*               csrSortedRowPtrA,
+                                      const int*               csrSortedColIndA,
+                                      const cusparseMatDescr_t descrB,
+                                      int                      nnzB,
+                                      const int*               csrSortedRowPtrB,
+                                      const int*               csrSortedColIndB,
+                                      const cusparseMatDescr_t descrC,
+                                      int*                     csrSortedRowPtrC,
+                                      int*                     nnzTotalDevHostPtr,
+                                      void*                    workspace) {
+  return hipsparseXcsrgeam2Nnz(handle, m, n, descrA, nnzA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedRowPtrC, nnzTotalDevHostPtr, workspace);
+}
+
+cusparseStatus_t cusparseScsrgeam2(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      n,
+                                   const float*             alpha,
+                                   const cusparseMatDescr_t descrA,
+                                   int                      nnzA,
+                                   const float*             csrSortedValA,
+                                   const int*               csrSortedRowPtrA,
+                                   const int*               csrSortedColIndA,
+                                   const float*             beta,
+                                   const cusparseMatDescr_t descrB,
+                                   int                      nnzB,
+                                   const float*             csrSortedValB,
+                                   const int*               csrSortedRowPtrB,
+                                   const int*               csrSortedColIndB,
+                                   const cusparseMatDescr_t descrC,
+                                   float*                   csrSortedValC,
+                                   int*                     csrSortedRowPtrC,
+                                   int*                     csrSortedColIndC,
+                                   void*                    pBuffer) {
+  return hipsparseScsrgeam2(handle, m, n, alpha, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, beta, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC, pBuffer);
+}
+
+cusparseStatus_t cusparseDcsrgeam2(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      n,
+                                   const double*            alpha,
+                                   const cusparseMatDescr_t descrA,
+                                   int                      nnzA,
+                                   const double*            csrSortedValA,
+                                   const int*               csrSortedRowPtrA,
+                                   const int*               csrSortedColIndA,
+                                   const double*            beta,
+                                   const cusparseMatDescr_t descrB,
+                                   int                      nnzB,
+                                   const double*            csrSortedValB,
+                                   const int*               csrSortedRowPtrB,
+                                   const int*               csrSortedColIndB,
+                                   const cusparseMatDescr_t descrC,
+                                   double*                  csrSortedValC,
+                                   int*                     csrSortedRowPtrC,
+                                   int*                     csrSortedColIndC,
+                                   void*                    pBuffer) {
+  return hipsparseDcsrgeam2(handle, m, n, alpha, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, beta, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC, pBuffer);
+}
+
+cusparseStatus_t cusparseCcsrgeam2(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      n,
+                                   const cuComplex*         alpha,
+                                   const cusparseMatDescr_t descrA,
+                                   int                      nnzA,
+                                   const cuComplex*         csrSortedValA,
+                                   const int*               csrSortedRowPtrA,
+                                   const int*               csrSortedColIndA,
+                                   const cuComplex*         beta,
+                                   const cusparseMatDescr_t descrB,
+                                   int                      nnzB,
+                                   const cuComplex*         csrSortedValB,
+                                   const int*               csrSortedRowPtrB,
+                                   const int*               csrSortedColIndB,
+                                   const cusparseMatDescr_t descrC,
+                                   cuComplex*               csrSortedValC,
+                                   int*                     csrSortedRowPtrC,
+                                   int*                     csrSortedColIndC,
+                                   void*                    pBuffer) {
+  return hipsparseCcsrgeam2(handle, m, n, reinterpret_cast<const hipComplex*>(alpha), descrA, nnzA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipComplex*>(beta), descrB, nnzB, reinterpret_cast<const hipComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, descrC, reinterpret_cast<hipComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC, pBuffer);
+}
+
+cusparseStatus_t cusparseZcsrgeam2(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      n,
+                                   const cuDoubleComplex*   alpha,
+                                   const cusparseMatDescr_t descrA,
+                                   int                      nnzA,
+                                   const cuDoubleComplex*   csrSortedValA,
+                                   const int*               csrSortedRowPtrA,
+                                   const int*               csrSortedColIndA,
+                                   const cuDoubleComplex*   beta,
+                                   const cusparseMatDescr_t descrB,
+                                   int                      nnzB,
+                                   const cuDoubleComplex*   csrSortedValB,
+                                   const int*               csrSortedRowPtrB,
+                                   const int*               csrSortedColIndB,
+                                   const cusparseMatDescr_t descrC,
+                                   cuDoubleComplex*         csrSortedValC,
+                                   int*                     csrSortedRowPtrC,
+                                   int*                     csrSortedColIndC,
+                                   void*                    pBuffer) {
+  return hipsparseZcsrgeam2(handle, m, n, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, nnzA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<const hipDoubleComplex*>(beta), descrB, nnzB, reinterpret_cast<const hipDoubleComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, descrC, reinterpret_cast<hipDoubleComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC, pBuffer);
+}
+
+cusparseStatus_t cusparseXcsrgemmNnz(cusparseHandle_t         handle,
+                                     cusparseOperation_t      transA,
+                                     cusparseOperation_t      transB,
+                                     int                      m,
+                                     int                      n,
+                                     int                      k,
+                                     const cusparseMatDescr_t descrA,
+                                     const int                nnzA,
+                                     const int*               csrSortedRowPtrA,
+                                     const int*               csrSortedColIndA,
+                                     const cusparseMatDescr_t descrB,
+                                     const int                nnzB,
+                                     const int*               csrSortedRowPtrB,
+                                     const int*               csrSortedColIndB,
+                                     const cusparseMatDescr_t descrC,
+                                     int*                     csrSortedRowPtrC,
+                                     int*                     nnzTotalDevHostPtr) {
+  return hipsparseXcsrgemmNnz(handle, transA, transB, m, n, k, descrA, nnzA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedRowPtrC, nnzTotalDevHostPtr);
+}
+
+cusparseStatus_t cusparseScsrgemm(cusparseHandle_t         handle,
+                                  cusparseOperation_t      transA,
+                                  cusparseOperation_t      transB,
+                                  int                      m,
+                                  int                      n,
+                                  int                      k,
+                                  const cusparseMatDescr_t descrA,
+                                  const int                nnzA,
+                                  const float*             csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const cusparseMatDescr_t descrB,
+                                  const int                nnzB,
+                                  const float*             csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cusparseMatDescr_t descrC,
+                                  float*                   csrSortedValC,
+                                  const int*               csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC) {
+  return hipsparseScsrgemm(handle, transA, transB, m, n, k, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC);
+}
+
+cusparseStatus_t cusparseDcsrgemm(cusparseHandle_t         handle,
+                                  cusparseOperation_t      transA,
+                                  cusparseOperation_t      transB,
+                                  int                      m,
+                                  int                      n,
+                                  int                      k,
+                                  const cusparseMatDescr_t descrA,
+                                  int                      nnzA,
+                                  const double*            csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const cusparseMatDescr_t descrB,
+                                  int                      nnzB,
+                                  const double*            csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cusparseMatDescr_t descrC,
+                                  double*                  csrSortedValC,
+                                  const int*               csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC) {
+  return hipsparseDcsrgemm(handle, transA, transB, m, n, k, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC);
+}
+
+cusparseStatus_t cusparseCcsrgemm(cusparseHandle_t         handle,
+                                  cusparseOperation_t      transA,
+                                  cusparseOperation_t      transB,
+                                  int                      m,
+                                  int                      n,
+                                  int                      k,
+                                  const cusparseMatDescr_t descrA,
+                                  int                      nnzA,
+                                  const cuComplex*         csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const cusparseMatDescr_t descrB,
+                                  int                      nnzB,
+                                  const cuComplex*         csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cusparseMatDescr_t descrC,
+                                  cuComplex*               csrSortedValC,
+                                  const int*               csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC) {
+  return hipsparseCcsrgemm(handle, transA, transB, m, n, k, descrA, nnzA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, reinterpret_cast<const hipComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, descrC, reinterpret_cast<hipComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC);
+}
+
+cusparseStatus_t cusparseZcsrgemm(cusparseHandle_t         handle,
+                                  cusparseOperation_t      transA,
+                                  cusparseOperation_t      transB,
+                                  int                      m,
+                                  int                      n,
+                                  int                      k,
+                                  const cusparseMatDescr_t descrA,
+                                  int                      nnzA,
+                                  const cuDoubleComplex*   csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const cusparseMatDescr_t descrB,
+                                  int                      nnzB,
+                                  const cuDoubleComplex*   csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cusparseMatDescr_t descrC,
+                                  cuDoubleComplex*         csrSortedValC,
+                                  const int*               csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC) {
+  return hipsparseZcsrgemm(handle, transA, transB, m, n, k, descrA, nnzA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, reinterpret_cast<const hipDoubleComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, descrC, reinterpret_cast<hipDoubleComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC);
+}
+
+cusparseStatus_t cusparseCreateCsrgemm2Info(csrgemm2Info_t* info) {
+  return hipsparseCreateCsrgemm2Info(info);
+}
+
+cusparseStatus_t cusparseDestroyCsrgemm2Info(csrgemm2Info_t info) {
+  return hipsparseDestroyCsrgemm2Info(info);
+}
+
+cusparseStatus_t cusparseScsrgemm2_bufferSizeExt(cusparseHandle_t         handle,
+                                                 int                      m,
+                                                 int                      n,
+                                                 int                      k,
+                                                 const float*             alpha,
+                                                 const cusparseMatDescr_t descrA,
+                                                 int                      nnzA,
+                                                 const int*               csrSortedRowPtrA,
+                                                 const int*               csrSortedColIndA,
+                                                 const cusparseMatDescr_t descrB,
+                                                 int                      nnzB,
+                                                 const int*               csrSortedRowPtrB,
+                                                 const int*               csrSortedColIndB,
+                                                 const float*             beta,
+                                                 const cusparseMatDescr_t descrD,
+                                                 int                      nnzD,
+                                                 const int*               csrSortedRowPtrD,
+                                                 const int*               csrSortedColIndD,
+                                                 csrgemm2Info_t           info,
+                                                 size_t*                  pBufferSizeInBytes) {
+  return hipsparseScsrgemm2_bufferSizeExt(handle, m, n, k, alpha, descrA, nnzA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedRowPtrB, csrSortedColIndB, beta, descrD, nnzD, csrSortedRowPtrD, csrSortedColIndD, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseDcsrgemm2_bufferSizeExt(cusparseHandle_t         handle,
+                                                 int                      m,
+                                                 int                      n,
+                                                 int                      k,
+                                                 const double*            alpha,
+                                                 const cusparseMatDescr_t descrA,
+                                                 int                      nnzA,
+                                                 const int*               csrSortedRowPtrA,
+                                                 const int*               csrSortedColIndA,
+                                                 const cusparseMatDescr_t descrB,
+                                                 int                      nnzB,
+                                                 const int*               csrSortedRowPtrB,
+                                                 const int*               csrSortedColIndB,
+                                                 const double*            beta,
+                                                 const cusparseMatDescr_t descrD,
+                                                 int                      nnzD,
+                                                 const int*               csrSortedRowPtrD,
+                                                 const int*               csrSortedColIndD,
+                                                 csrgemm2Info_t           info,
+                                                 size_t*                  pBufferSizeInBytes) {
+  return hipsparseDcsrgemm2_bufferSizeExt(handle, m, n, k, alpha, descrA, nnzA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedRowPtrB, csrSortedColIndB, beta, descrD, nnzD, csrSortedRowPtrD, csrSortedColIndD, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseCcsrgemm2_bufferSizeExt(cusparseHandle_t         handle,
+                                                 int                      m,
+                                                 int                      n,
+                                                 int                      k,
+                                                 const cuComplex*         alpha,
+                                                 const cusparseMatDescr_t descrA,
+                                                 int                      nnzA,
+                                                 const int*               csrSortedRowPtrA,
+                                                 const int*               csrSortedColIndA,
+                                                 const cusparseMatDescr_t descrB,
+                                                 int                      nnzB,
+                                                 const int*               csrSortedRowPtrB,
+                                                 const int*               csrSortedColIndB,
+                                                 const cuComplex*         beta,
+                                                 const cusparseMatDescr_t descrD,
+                                                 int                      nnzD,
+                                                 const int*               csrSortedRowPtrD,
+                                                 const int*               csrSortedColIndD,
+                                                 csrgemm2Info_t           info,
+                                                 size_t*                  pBufferSizeInBytes) {
+  return hipsparseCcsrgemm2_bufferSizeExt(handle, m, n, k, reinterpret_cast<const hipComplex*>(alpha), descrA, nnzA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedRowPtrB, csrSortedColIndB, reinterpret_cast<const hipComplex*>(beta), descrD, nnzD, csrSortedRowPtrD, csrSortedColIndD, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseZcsrgemm2_bufferSizeExt(cusparseHandle_t         handle,
+                                                 int                      m,
+                                                 int                      n,
+                                                 int                      k,
+                                                 const cuDoubleComplex*   alpha,
+                                                 const cusparseMatDescr_t descrA,
+                                                 int                      nnzA,
+                                                 const int*               csrSortedRowPtrA,
+                                                 const int*               csrSortedColIndA,
+                                                 const cusparseMatDescr_t descrB,
+                                                 int                      nnzB,
+                                                 const int*               csrSortedRowPtrB,
+                                                 const int*               csrSortedColIndB,
+                                                 const cuDoubleComplex*   beta,
+                                                 const cusparseMatDescr_t descrD,
+                                                 int                      nnzD,
+                                                 const int*               csrSortedRowPtrD,
+                                                 const int*               csrSortedColIndD,
+                                                 csrgemm2Info_t           info,
+                                                 size_t*                  pBufferSizeInBytes) {
+  return hipsparseZcsrgemm2_bufferSizeExt(handle, m, n, k, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, nnzA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedRowPtrB, csrSortedColIndB, reinterpret_cast<const hipDoubleComplex*>(beta), descrD, nnzD, csrSortedRowPtrD, csrSortedColIndD, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseXcsrgemm2Nnz(cusparseHandle_t         handle,
+                                      int                      m,
+                                      int                      n,
+                                      int                      k,
+                                      const cusparseMatDescr_t descrA,
+                                      int                      nnzA,
+                                      const int*               csrSortedRowPtrA,
+                                      const int*               csrSortedColIndA,
+                                      const cusparseMatDescr_t descrB,
+                                      int                      nnzB,
+                                      const int*               csrSortedRowPtrB,
+                                      const int*               csrSortedColIndB,
+                                      const cusparseMatDescr_t descrD,
+                                      int                      nnzD,
+                                      const int*               csrSortedRowPtrD,
+                                      const int*               csrSortedColIndD,
+                                      const cusparseMatDescr_t descrC,
+                                      int*                     csrSortedRowPtrC,
+                                      int*                     nnzTotalDevHostPtr,
+                                      const csrgemm2Info_t     info,
+                                      void*                    pBuffer) {
+  return hipsparseXcsrgemm2Nnz(handle, m, n, k, descrA, nnzA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedRowPtrB, csrSortedColIndB, descrD, nnzD, csrSortedRowPtrD, csrSortedColIndD, descrC, csrSortedRowPtrC, nnzTotalDevHostPtr, info, pBuffer);
+}
+
+cusparseStatus_t cusparseScsrgemm2(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      n,
+                                   int                      k,
+                                   const float*             alpha,
+                                   const cusparseMatDescr_t descrA,
+                                   int                      nnzA,
+                                   const float*             csrSortedValA,
+                                   const int*               csrSortedRowPtrA,
+                                   const int*               csrSortedColIndA,
+                                   const cusparseMatDescr_t descrB,
+                                   int                      nnzB,
+                                   const float*             csrSortedValB,
+                                   const int*               csrSortedRowPtrB,
+                                   const int*               csrSortedColIndB,
+                                   const float*             beta,
+                                   const cusparseMatDescr_t descrD,
+                                   int                      nnzD,
+                                   const float*             csrSortedValD,
+                                   const int*               csrSortedRowPtrD,
+                                   const int*               csrSortedColIndD,
+                                   const cusparseMatDescr_t descrC,
+                                   float*                   csrSortedValC,
+                                   const int*               csrSortedRowPtrC,
+                                   int*                     csrSortedColIndC,
+                                   const csrgemm2Info_t     info,
+                                   void*                    pBuffer) {
+  return hipsparseScsrgemm2(handle, m, n, k, alpha, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, beta, descrD, nnzD, csrSortedValD, csrSortedRowPtrD, csrSortedColIndD, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC, info, pBuffer);
+}
+
+cusparseStatus_t cusparseDcsrgemm2(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      n,
+                                   int                      k,
+                                   const double*            alpha,
+                                   const cusparseMatDescr_t descrA,
+                                   int                      nnzA,
+                                   const double*            csrSortedValA,
+                                   const int*               csrSortedRowPtrA,
+                                   const int*               csrSortedColIndA,
+                                   const cusparseMatDescr_t descrB,
+                                   int                      nnzB,
+                                   const double*            csrSortedValB,
+                                   const int*               csrSortedRowPtrB,
+                                   const int*               csrSortedColIndB,
+                                   const double*            beta,
+                                   const cusparseMatDescr_t descrD,
+                                   int                      nnzD,
+                                   const double*            csrSortedValD,
+                                   const int*               csrSortedRowPtrD,
+                                   const int*               csrSortedColIndD,
+                                   const cusparseMatDescr_t descrC,
+                                   double*                  csrSortedValC,
+                                   const int*               csrSortedRowPtrC,
+                                   int*                     csrSortedColIndC,
+                                   const csrgemm2Info_t     info,
+                                   void*                    pBuffer) {
+  return hipsparseDcsrgemm2(handle, m, n, k, alpha, descrA, nnzA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, csrSortedValB, csrSortedRowPtrB, csrSortedColIndB, beta, descrD, nnzD, csrSortedValD, csrSortedRowPtrD, csrSortedColIndD, descrC, csrSortedValC, csrSortedRowPtrC, csrSortedColIndC, info, pBuffer);
+}
+
+cusparseStatus_t cusparseCcsrgemm2(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      n,
+                                  int                      k,
+                                  const cuComplex*         alpha,
+                                  const cusparseMatDescr_t descrA,
+                                  int                      nnzA,
+                                  const cuComplex*         csrSortedValA,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  const cusparseMatDescr_t descrB,
+                                  int                      nnzB,
+                                  const cuComplex*         csrSortedValB,
+                                  const int*               csrSortedRowPtrB,
+                                  const int*               csrSortedColIndB,
+                                  const cuComplex*         beta,
+                                  const cusparseMatDescr_t descrD,
+                                  int                      nnzD,
+                                  const cuComplex*         csrSortedValD,
+                                  const int*               csrSortedRowPtrD,
+                                  const int*               csrSortedColIndD,
+                                  const cusparseMatDescr_t descrC,
+                                  cuComplex*               csrSortedValC,
+                                  const int*               csrSortedRowPtrC,
+                                  int*                     csrSortedColIndC,
+                                  const csrgemm2Info_t     info,
+                                  void*                    pBuffer) {
+  return hipsparseCcsrgemm2(handle, m, n, k, reinterpret_cast<const hipComplex*>(alpha), descrA, nnzA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, reinterpret_cast<const hipComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, reinterpret_cast<const hipComplex*>(beta), descrD, nnzD, reinterpret_cast<const hipComplex*>(csrSortedValD), csrSortedRowPtrD, csrSortedColIndD, descrC, reinterpret_cast<hipComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC, info, pBuffer);
+}
+
+cusparseStatus_t cusparseZcsrgemm2(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      n,
+                                   int                      k,
+                                   const cuDoubleComplex*   alpha,
+                                   const cusparseMatDescr_t descrA,
+                                   int                      nnzA,
+                                   const cuDoubleComplex*   csrSortedValA,
+                                   const int*               csrSortedRowPtrA,
+                                   const int*               csrSortedColIndA,
+                                   const cusparseMatDescr_t descrB,
+                                   int                      nnzB,
+                                   const cuDoubleComplex*   csrSortedValB,
+                                   const int*               csrSortedRowPtrB,
+                                   const int*               csrSortedColIndB,
+                                   const cuDoubleComplex*   beta,
+                                   const cusparseMatDescr_t descrD,
+                                   int                      nnzD,
+                                   const cuDoubleComplex*   csrSortedValD,
+                                   const int*               csrSortedRowPtrD,
+                                   const int*               csrSortedColIndD,
+                                   const cusparseMatDescr_t descrC,
+                                   cuDoubleComplex*         csrSortedValC,
+                                   const int*               csrSortedRowPtrC,
+                                   int*                     csrSortedColIndC,
+                                   const csrgemm2Info_t     info,
+                                   void*                    pBuffer) {
+  return hipsparseZcsrgemm2(handle, m, n, k, reinterpret_cast<const hipDoubleComplex*>(alpha), descrA, nnzA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, descrB, nnzB, reinterpret_cast<const hipDoubleComplex*>(csrSortedValB), csrSortedRowPtrB, csrSortedColIndB, reinterpret_cast<const hipDoubleComplex*>(beta), descrD, nnzD, reinterpret_cast<const hipDoubleComplex*>(csrSortedValD), csrSortedRowPtrD, csrSortedColIndD, descrC, reinterpret_cast<hipDoubleComplex*>(csrSortedValC), csrSortedRowPtrC, csrSortedColIndC, info, pBuffer);
+}
+
+// cuSPARSE Format Convrsion
+cusparseStatus_t cusparseXcoo2csr(cusparseHandle_t    handle,
+                                  const int*          cooRowInd,
+                                  int                 nnz,
+                                  int                 m,
+                                  int*                csrSortedRowPtr,
+                                  cusparseIndexBase_t idxBase) {
+  return hipsparseXcoo2csr(handle, cooRowInd, nnz, m, csrSortedRowPtr, idxBase);
+}
+
+cusparseStatus_t cusparseScsc2dense(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const float*             cscSortedValA,
+                                    const int*               cscSortedRowIndA,
+                                    const int*               cscSortedColPtrA,
+                                    float*                   A,
+                                    int                      lda) {
+  return hipsparseScsc2dense(handle, m, n, descrA, cscSortedValA, cscSortedRowIndA, cscSortedColPtrA, A, lda);
+}
+
+cusparseStatus_t cusparseDcsc2dense(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const double*            cscSortedValA,
+                                    const int*               cscSortedRowIndA,
+                                    const int*               cscSortedColPtrA,
+                                    double*                  A,
+                                    int                      lda) {
+  return hipsparseDcsc2dense(handle, m, n, descrA, cscSortedValA, cscSortedRowIndA, cscSortedColPtrA, A, lda);
+}
+
+cusparseStatus_t cusparseCcsc2dense(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const cuComplex*         cscSortedValA,
+                                    const int*               cscSortedRowIndA,
+                                    const int*               cscSortedColPtrA,
+                                    cuComplex*               A,
+                                    int                      lda) {
+  return hipsparseCcsc2dense(handle, m, n, descrA, reinterpret_cast<const hipComplex*>(cscSortedValA), cscSortedRowIndA, cscSortedColPtrA, reinterpret_cast<hipComplex*>(A), lda);
+}
+
+cusparseStatus_t cusparseZcsc2dense(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const cuDoubleComplex*   cscSortedValA,
+                                    const int*               cscSortedRowIndA,
+                                    const int*               cscSortedColPtrA,
+                                    cuDoubleComplex*         A,
+                                    int                      lda) {
+  return hipsparseZcsc2dense(handle, m, n, descrA, reinterpret_cast<const hipDoubleComplex*>(cscSortedValA), cscSortedRowIndA, cscSortedColPtrA, reinterpret_cast<hipDoubleComplex*>(A), lda);
+}
+
+cusparseStatus_t cusparseXcsr2coo(cusparseHandle_t    handle,
+                                  const int*          csrSortedRowPtr,
+                                  int                 nnz,
+                                  int                 m,
+                                  int*                cooRowInd,
+                                  cusparseIndexBase_t idxBase) {
+  return hipsparseXcsr2coo(handle, csrSortedRowPtr, nnz, m, cooRowInd, idxBase);
+}
+
+cusparseStatus_t cusparseScsr2csc(cusparseHandle_t    handle,
+                                  int                 m,
+                                  int                 n,
+                                  int                 nnz,
+                                  const float*        csrSortedVal,
+                                  const int*          csrSortedRowPtr,
+                                  const int*          csrSortedColInd,
+                                  float*              cscSortedVal,
+                                  int*                cscSortedRowInd,
+                                  int*                cscSortedColPtr,
+                                  cusparseAction_t    copyValues,
+                                  cusparseIndexBase_t idxBase) {
+  return hipsparseScsr2csc(handle, m, n, nnz, csrSortedVal, csrSortedRowPtr, csrSortedColInd, cscSortedVal, cscSortedRowInd, cscSortedColPtr, copyValues, idxBase);
+}
+
+cusparseStatus_t cusparseDcsr2csc(cusparseHandle_t    handle,
+                                  int                 m,
+                                  int                 n,
+                                  int                 nnz,
+                                  const double*       csrSortedVal,
+                                  const int*          csrSortedRowPtr,
+                                  const int*          csrSortedColInd,
+                                  double*             cscSortedVal,
+                                  int*                cscSortedRowInd,
+                                  int*                cscSortedColPtr,
+                                  cusparseAction_t    copyValues,
+                                  cusparseIndexBase_t idxBase) {
+  return hipsparseDcsr2csc(handle, m, n, nnz, csrSortedVal, csrSortedRowPtr, csrSortedColInd, cscSortedVal, cscSortedRowInd, cscSortedColPtr, copyValues, idxBase);
+}
+
+cusparseStatus_t cusparseCcsr2csc(cusparseHandle_t    handle,
+                                  int                 m,
+                                  int                 n,
+                                  int                 nnz,
+                                  const cuComplex*    csrSortedVal,
+                                  const int*          csrSortedRowPtr,
+                                  const int*          csrSortedColInd,
+                                  cuComplex*          cscSortedVal,
+                                  int*                cscSortedRowInd,
+                                  int*                cscSortedColPtr,
+                                  cusparseAction_t    copyValues,
+                                  cusparseIndexBase_t idxBase) {
+  return hipsparseCcsr2csc(handle, m, n, nnz, reinterpret_cast<const hipComplex*>(csrSortedVal), csrSortedRowPtr, csrSortedColInd, reinterpret_cast<hipComplex*>(cscSortedVal), cscSortedRowInd, cscSortedColPtr, copyValues, idxBase);
+}
+
+cusparseStatus_t cusparseZcsr2csc(cusparseHandle_t       handle,
+                                  int                    m,
+                                  int                    n,
+                                  int                    nnz,
+                                  const cuDoubleComplex* csrSortedVal,
+                                  const int*             csrSortedRowPtr,
+                                  const int*             csrSortedColInd,
+                                  cuDoubleComplex*       cscSortedVal,
+                                  int*                   cscSortedRowInd,
+                                  int*                   cscSortedColPtr,
+                                  cusparseAction_t       copyValues,
+                                  cusparseIndexBase_t    idxBase) {
+  return hipsparseZcsr2csc(handle, m, n, nnz, reinterpret_cast<const hipDoubleComplex*>(csrSortedVal), csrSortedRowPtr, csrSortedColInd, reinterpret_cast<hipDoubleComplex*>(cscSortedVal), cscSortedRowInd, cscSortedColPtr, copyValues, idxBase);
+}
+
+
+cusparseStatus_t cusparseScsr2dense(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const float*             csrSortedValA,
+                                    const int*               csrSortedRowPtrA,
+                                    const int*               csrSortedColIndA,
+                                    float*                   A,
+                                    int                      lda) {
+  return hipsparseScsr2dense(handle, m, n, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, A, lda);
+}
+
+cusparseStatus_t cusparseDcsr2dense(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const double*            csrSortedValA,
+                                    const int*               csrSortedRowPtrA,
+                                    const int*               csrSortedColIndA,
+                                    double*                  A,
+                                    int                      lda) {
+  return hipsparseDcsr2dense(handle, m, n, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, A, lda);
+}
+
+cusparseStatus_t cusparseCcsr2dense(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const cuComplex*         csrSortedValA,
+                                    const int*               csrSortedRowPtrA,
+                                    const int*               csrSortedColIndA,
+                                    cuComplex*               A,
+                                    int                      lda) {
+  return hipsparseCcsr2dense(handle, m, n, descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<hipComplex*>(A), lda);
+}
+
+cusparseStatus_t cusparseZcsr2dense(cusparseHandle_t         handle,
+                                 int                      m,
+                                 int                      n,
+                                 const cusparseMatDescr_t descrA,
+                                 const cuDoubleComplex*   csrSortedValA,
+                                 const int*               csrSortedRowPtrA,
+                                 const int*               csrSortedColIndA,
+                                 cuDoubleComplex*         A,
+                                 int                      lda) {
+  return hipsparseZcsr2dense(handle, m, n, descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, reinterpret_cast<hipDoubleComplex*>(A), lda);
+}
+
+cusparseStatus_t cusparseSdense2csc(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const float*             A,
+                                    int                      lda,
+                                    const int*               nnzPerCol,
+                                    float*                   cscSortedValA,
+                                    int*                     cscSortedRowIndA,
+                                    int*                     cscSortedColPtrA) {
+  return hipsparseSdense2csc(handle, m, n, descrA, A, lda, nnzPerCol, cscSortedValA, cscSortedRowIndA, cscSortedColPtrA);
+}
+
+cusparseStatus_t cusparseDdense2csc(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const double*            A,
+                                    int                      lda,
+                                    const int*               nnzPerCol,
+                                    double*                  cscSortedValA,
+                                    int*                     cscSortedRowIndA,
+                                    int*                     cscSortedColPtrA) {
+  return hipsparseDdense2csc(handle, m, n, descrA, A, lda, nnzPerCol, cscSortedValA, cscSortedRowIndA, cscSortedColPtrA);
+}
+
+cusparseStatus_t cusparseCdense2csc(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const cuComplex*         A,
+                                    int                      lda,
+                                    const int*               nnzPerCol,
+                                    cuComplex*               cscSortedValA,
+                                    int*                     cscSortedRowIndA,
+                                    int*                     cscSortedColPtrA) {
+  return hipsparseCdense2csc(handle, m, n, descrA, reinterpret_cast<const hipComplex*>(A), lda, nnzPerCol, reinterpret_cast<hipComplex*>(cscSortedValA), cscSortedRowIndA, cscSortedColPtrA);
+}
+
+cusparseStatus_t cusparseZdense2csc(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const cuDoubleComplex*   A,
+                                    int                      lda,
+                                    const int*               nnzPerCol,
+                                    cuDoubleComplex*         cscSortedValA,
+                                    int*                     cscSortedRowIndA,
+                                    int*                     cscSortedColPtrA) {
+  return hipsparseZdense2csc(handle, m, n, descrA, reinterpret_cast<const hipDoubleComplex*>(A), lda, nnzPerCol, reinterpret_cast<hipDoubleComplex*>(cscSortedValA), cscSortedRowIndA, cscSortedColPtrA);
+}
+
+cusparseStatus_t cusparseSdense2csr(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const float*             A,
+                                    int                      lda,
+                                    const int*               nnzPerRow,
+                                    float*                   csrSortedValA,
+                                    int*                     csrSortedRowPtrA,
+                                    int*                     csrSortedColIndA) {
+  return hipsparseSdense2csr(handle, m, n, descrA, A, lda, nnzPerRow, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA);
+}
+
+cusparseStatus_t cusparseDdense2csr(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const double*            A,
+                                    int                      lda,
+                                    const int*               nnzPerRow,
+                                    double*                  csrSortedValA,
+                                    int*                     csrSortedRowPtrA,
+                                    int*                     csrSortedColIndA) {
+  return hipsparseDdense2csr(handle, m, n, descrA, A, lda, nnzPerRow, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA);
+}
+
+cusparseStatus_t cusparseCdense2csr(cusparseHandle_t           handle,
+                                      int                      m,
+                                      int                      n,
+                                      const cusparseMatDescr_t descrA,
+                                      const cuComplex*         A,
+                                      int                      lda,
+                                      const int*               nnzPerRow,
+                                      cuComplex*               csrSortedValA,
+                                      int*                     csrSortedRowPtrA,
+                                      int*                     csrSortedColIndA) {
+  return hipsparseCdense2csr(handle, m, n, descrA, reinterpret_cast<const hipComplex*>(A), lda, nnzPerRow, reinterpret_cast<hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA);
+}
+
+cusparseStatus_t cusparseZdense2csr(cusparseHandle_t         handle,
+                                    int                      m,
+                                    int                      n,
+                                    const cusparseMatDescr_t descrA,
+                                    const cuDoubleComplex*   A,
+                                    int                      lda,
+                                    const int*               nnzPerRow,
+                                    cuDoubleComplex*         csrSortedValA,
+                                    int*                     csrSortedRowPtrA,
+                                    int*                     csrSortedColIndA) {
+  return hipsparseZdense2csr(handle, m, n, descrA, reinterpret_cast<const hipDoubleComplex*>(A), lda, nnzPerRow, reinterpret_cast<hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA);
+}
+
+cusparseStatus_t cusparseSnnz(cusparseHandle_t         handle,
+                              cusparseDirection_t      dirA,
+                              int                      m,
+                              int                      n,
+                              const cusparseMatDescr_t descrA,
+                              const float*             A,
+                              int                      lda,
+                              int*                     nnzPerRowCol,
+                              int*                     nnzTotalDevHostPtr) {
+  return hipsparseSnnz(handle, dirA, m, n, descrA, A, lda, nnzPerRowCol, nnzTotalDevHostPtr);
+}
+
+cusparseStatus_t cusparseDnnz(cusparseHandle_t         handle,
+                              cusparseDirection_t      dirA,
+                              int                      m,
+                              int                      n,
+                              const cusparseMatDescr_t descrA,
+                              const double*            A,
+                              int                      lda,
+                              int*                     nnzPerRowCol,
+                              int*                     nnzTotalDevHostPtr) {
+  return hipsparseDnnz(handle, dirA, m, n, descrA, A, lda, nnzPerRowCol, nnzTotalDevHostPtr);
+}
+
+cusparseStatus_t cusparseCnnz(cusparseHandle_t         handle,
+                              cusparseDirection_t      dirA,
+                              int                      m,
+                              int                      n,
+                              const cusparseMatDescr_t descrA,
+                              const cuComplex*         A,
+                              int                      lda,
+                              int*                     nnzPerRowCol,
+                              int*                     nnzTotalDevHostPtr) {
+  return hipsparseCnnz(handle, dirA, m, n, descrA, reinterpret_cast<const hipComplex*>(A), lda, nnzPerRowCol, nnzTotalDevHostPtr);
+}
+
+cusparseStatus_t cusparseZnnz(cusparseHandle_t         handle,
+                              cusparseDirection_t      dirA,
+                              int                      m,
+                              int                      n,
+                              const cusparseMatDescr_t descrA,
+                              const cuDoubleComplex*   A,
+                              int                      lda,
+                              int*                     nnzPerRowCol,
+                              int*                     nnzTotalDevHostPtr) {
+  return hipsparseZnnz(handle, dirA, m, n, descrA, reinterpret_cast<const hipDoubleComplex*>(A), lda, nnzPerRowCol, nnzTotalDevHostPtr);
+}
+
+cusparseStatus_t cusparseCreateIdentityPermutation(cusparseHandle_t handle,
+                                                   int              n,
+                                                   int*             p) {
+  return hipsparseCreateIdentityPermutation(handle, n, p);
+}
+
+cusparseStatus_t cusparseXcoosort_bufferSizeExt(cusparseHandle_t handle,
+                                                int              m,
+                                                int              n,
+                                                int              nnz,
+                                                const int*       cooRowsA,
+                                                const int*       cooColsA,
+                                                size_t*          pBufferSizeInBytes) {
+  return hipsparseXcoosort_bufferSizeExt(handle, m, n, nnz, cooRowsA, cooColsA, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseXcoosortByRow(cusparseHandle_t handle,
+                                       int              m,
+                                       int              n,
+                                       int              nnz,
+                                       int*             cooRowsA,
+                                       int*             cooColsA,
+                                       int*             P,
+                                       void*            pBuffer) {
+  return hipsparseXcoosortByRow(handle, m, n, nnz, cooRowsA, cooColsA, P, pBuffer);
+}
+
+cusparseStatus_t cusparseXcoosortByColumn(cusparseHandle_t handle,
+                                          int              m,
+                                          int              n,
+                                          int              nnz,
+                                          int*             cooRowsA,
+                                          int*             cooColsA,
+                                          int*             P,
+                                          void*            pBuffer) {
+  return hipsparseXcoosortByColumn(handle, m, n, nnz, cooRowsA, cooColsA, P, pBuffer);
+}
+
+cusparseStatus_t cusparseXcsrsort_bufferSizeExt(cusparseHandle_t handle,
+                                                int              m,
+                                                int              n,
+                                                int              nnz,
+                                                const int*       csrRowPtrA,
+                                                const int*       csrColIndA,
+                                                size_t*          pBufferSizeInBytes) {
+  return hipsparseXcsrsort_bufferSizeExt(handle, m, n, nnz, csrRowPtrA, csrColIndA, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseXcsrsort(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      n,
+                                  int                      nnz,
+                                  const cusparseMatDescr_t descrA,
+                                  const int*               csrRowPtrA,
+                                  int*                     csrColIndA,
+                                  int*                     P,
+                                  void*                    pBuffer) {
+  return hipsparseXcsrsort(handle, m, n, nnz, descrA, csrRowPtrA, csrColIndA, P, pBuffer);
+}
+
+cusparseStatus_t cusparseXcscsort_bufferSizeExt(cusparseHandle_t handle,
+                                                int              m,
+                                                int              n,
+                                                int              nnz,
+                                                const int*       cscColPtrA,
+                                                const int*       cscRowIndA,
+                                                size_t*          pBufferSizeInBytes) {
+  return hipsparseXcscsort_bufferSizeExt(handle, m, n, nnz, cscColPtrA, cscRowIndA, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseXcscsort(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      n,
+                                  int                      nnz,
+                                  const cusparseMatDescr_t descrA,
+                                  const int*               cscColPtrA,
+                                  int*                     cscRowIndA,
+                                  int*                     P,
+                                  void*                    pBuffer) {
+  return hipsparseXcscsort(handle, m, n, nnz, descrA, cscColPtrA, cscRowIndA, P, pBuffer);
+}
+
+// cuSPARSE PRECONDITIONERS
+
+cusparseStatus_t cusparseCreateCsrilu02Info(csrilu02Info_t* info) {
+  return hipsparseCreateCsrilu02Info(info);
+}
+
+cusparseStatus_t cusparseDestroyCsrilu02Info(csrilu02Info_t info) {
+  return hipsparseDestroyCsrilu02Info(info);
+}
+
+cusparseStatus_t cusparseCreateBsrilu02Info(bsrilu02Info_t* info) {
+#if HIP_VERSION >= 309
+  return hipsparseCreateBsrilu02Info(info);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDestroyBsrilu02Info(bsrilu02Info_t info) {
+#if HIP_VERSION >= 309
+  return hipsparseDestroyBsrilu02Info(info);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCreateCsric02Info(csric02Info_t* info) {
+  return hipsparseCreateCsric02Info(info);
+}
+
+cusparseStatus_t cusparseDestroyCsric02Info(csric02Info_t info) {
+  return hipsparseDestroyCsric02Info(info);
+}
+
+cusparseStatus_t cusparseCreateBsric02Info(bsric02Info_t* info) {
+#if HIP_VERSION >= 308
+  return hipsparseCreateBsric02Info(info);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDestroyBsric02Info(bsric02Info_t info) {
+#if HIP_VERSION >= 308
+  return hipsparseDestroyBsric02Info(info);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseScsrilu02_numericBoost(cusparseHandle_t handle,
+                                                csrilu02Info_t   info,
+                                                int              enable_boost,
+                                                double*          tol,
+                                                float*           boost_val) {
+#if HIP_VERSION >= 400
+  return hipsparseScsrilu02_numericBoost(handle, info, enable_boost, tol, boost_val);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDcsrilu02_numericBoost(cusparseHandle_t handle,
+                                                csrilu02Info_t   info,
+                                                int              enable_boost,
+                                                double*          tol,
+                                                double*          boost_val) {
+#if HIP_VERSION >= 400
+  return hipsparseDcsrilu02_numericBoost(handle, info, enable_boost, tol, boost_val);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCcsrilu02_numericBoost(cusparseHandle_t handle,
+                                                csrilu02Info_t   info,
+                                                int              enable_boost,
+                                                double*          tol,
+                                                cuComplex*       boost_val) {
+#if HIP_VERSION >= 400
+  return hipsparseCcsrilu02_numericBoost(handle, info, enable_boost, tol, reinterpret_cast<hipComplex*>(boost_val));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseZcsrilu02_numericBoost(cusparseHandle_t handle,
+                                                csrilu02Info_t   info,
+                                                int              enable_boost,
+                                                double*          tol,
+                                                cuDoubleComplex* boost_val) {
+#if HIP_VERSION >= 400
+  return hipsparseZcsrilu02_numericBoost(handle, info, enable_boost, tol, reinterpret_cast<hipDoubleComplex*>(boost_val));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseXcsrilu02_zeroPivot(cusparseHandle_t handle,
+                                             csrilu02Info_t   info,
+                                             int*             position) {
+  return hipsparseXcsrilu02_zeroPivot(handle, info, position);
+}
+
+cusparseStatus_t cusparseScsrilu02_bufferSize(cusparseHandle_t         handle,
+                                              int                      m,
+                                              int                      nnz,
+                                              const cusparseMatDescr_t descrA,
+                                              float*                   csrSortedValA,
+                                              const int*               csrSortedRowPtrA,
+                                              const int*               csrSortedColIndA,
+                                              csrilu02Info_t           info,
+                                              int*                     pBufferSizeInBytes) {
+  return hipsparseScsrilu02_bufferSize(handle, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseDcsrilu02_bufferSize(cusparseHandle_t         handle,
+                                              int                      m,
+                                              int                      nnz,
+                                              const cusparseMatDescr_t descrA,
+                                              double*                  csrSortedValA,
+                                              const int*               csrSortedRowPtrA,
+                                              const int*               csrSortedColIndA,
+                                              csrilu02Info_t           info,
+                                              int*                     pBufferSizeInBytes) {
+  return hipsparseDcsrilu02_bufferSize(handle, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseCcsrilu02_bufferSize(cusparseHandle_t         handle,
+                                              int                      m,
+                                              int                      nnz,
+                                              const cusparseMatDescr_t descrA,
+                                              cuComplex*               csrSortedValA,
+                                              const int*               csrSortedRowPtrA,
+                                              const int*               csrSortedColIndA,
+                                              csrilu02Info_t           info,
+                                              int*                     pBufferSizeInBytes) {
+  return hipsparseCcsrilu02_bufferSize(handle, m, nnz, descrA, reinterpret_cast<hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseZcsrilu02_bufferSize(cusparseHandle_t         handle,
+                                              int                      m,
+                                              int                      nnz,
+                                              const cusparseMatDescr_t descrA,
+                                              cuDoubleComplex*         csrSortedValA,
+                                              const int*               csrSortedRowPtrA,
+                                              const int*               csrSortedColIndA,
+                                              csrilu02Info_t           info,
+                                              int*                     pBufferSizeInBytes) {
+  return hipsparseZcsrilu02_bufferSize(handle, m, nnz, descrA, reinterpret_cast<hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseScsrilu02_analysis(cusparseHandle_t         handle,
+                                            int                      m,
+                                            int                      nnz,
+                                            const cusparseMatDescr_t descrA,
+                                            const float*             csrSortedValA,
+                                            const int*               csrSortedRowPtrA,
+                                            const int*               csrSortedColIndA,
+                                            csrilu02Info_t           info,
+                                            cusparseSolvePolicy_t    policy,
+                                            void*                    pBuffer) {
+  return hipsparseScsrilu02_analysis(handle, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseDcsrilu02_analysis(cusparseHandle_t         handle,
+                                            int                      m,
+                                            int                      nnz,
+                                            const cusparseMatDescr_t descrA,
+                                            const double*            csrSortedValA,
+                                            const int*               csrSortedRowPtrA,
+                                            const int*               csrSortedColIndA,
+                                            csrilu02Info_t           info,
+                                            cusparseSolvePolicy_t    policy,
+                                            void*                    pBuffer) {
+  return hipsparseDcsrilu02_analysis(handle, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseCcsrilu02_analysis(cusparseHandle_t         handle,
+                                            int                      m,
+                                            int                      nnz,
+                                            const cusparseMatDescr_t descrA,
+                                            const cuComplex*         csrSortedValA,
+                                            const int*               csrSortedRowPtrA,
+                                            const int*               csrSortedColIndA,
+                                            csrilu02Info_t           info,
+                                            cusparseSolvePolicy_t    policy,
+                                            void*                    pBuffer) {
+  return hipsparseCcsrilu02_analysis(handle, m, nnz, descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseZcsrilu02_analysis(cusparseHandle_t         handle,
+                                            int                      m,
+                                            int                      nnz,
+                                            const cusparseMatDescr_t descrA,
+                                            const cuDoubleComplex*   csrSortedValA,
+                                            const int*               csrSortedRowPtrA,
+                                            const int*               csrSortedColIndA,
+                                            csrilu02Info_t           info,
+                                            cusparseSolvePolicy_t    policy,
+                                            void*                    pBuffer) {
+  return hipsparseZcsrilu02_analysis(handle, m, nnz, descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseScsrilu02(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      nnz,
+                                   const cusparseMatDescr_t descrA,
+                                   float*                   csrSortedValA_valM,
+                                   const int*            csrSortedRowPtrA,
+                                   const int*            csrSortedColIndA,
+                                   csrilu02Info_t        info,
+                                   cusparseSolvePolicy_t policy,
+                                   void*                 pBuffer) {
+  return hipsparseScsrilu02(handle, m, nnz, descrA, csrSortedValA_valM, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseDcsrilu02(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      nnz,
+                                   const cusparseMatDescr_t descrA,
+                                   double*                  csrSortedValA_valM,
+                                   const int*            csrSortedRowPtrA,
+                                   const int*            csrSortedColIndA,
+                                   csrilu02Info_t        info,
+                                   cusparseSolvePolicy_t policy,
+                                   void*                 pBuffer) {
+  return hipsparseDcsrilu02(handle, m, nnz, descrA, csrSortedValA_valM, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseCcsrilu02(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      nnz,
+                                   const cusparseMatDescr_t descrA,
+                                   cuComplex*               csrSortedValA_valM,
+                                   const int*            csrSortedRowPtrA,
+                                   const int*            csrSortedColIndA,
+                                   csrilu02Info_t        info,
+                                   cusparseSolvePolicy_t policy,
+                                   void*                 pBuffer) {
+  return hipsparseCcsrilu02(handle, m, nnz, descrA, reinterpret_cast<hipComplex*>(csrSortedValA_valM), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseZcsrilu02(cusparseHandle_t         handle,
+                                   int                      m,
+                                   int                      nnz,
+                                   const cusparseMatDescr_t descrA,
+                                   cuDoubleComplex*         csrSortedValA_valM,
+                                   const int*            csrSortedRowPtrA,
+                                   const int*            csrSortedColIndA,
+                                   csrilu02Info_t        info,
+                                   cusparseSolvePolicy_t policy,
+                                   void*                 pBuffer) {
+  return hipsparseZcsrilu02(handle, m, nnz, descrA, reinterpret_cast<hipDoubleComplex*>(csrSortedValA_valM), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseSbsrilu02_numericBoost(cusparseHandle_t handle,
+                                                bsrilu02Info_t   info,
+                                                int              enable_boost,
+                                                double*          tol,
+                                                float*           boost_val) {
+#if HIP_VERSION >= 309
+  return hipsparseSbsrilu02_numericBoost(handle, info, enable_boost, tol, boost_val);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDbsrilu02_numericBoost(cusparseHandle_t handle,
+                                                bsrilu02Info_t   info,
+                                                int              enable_boost,
+                                                double*          tol,
+                                                double*          boost_val) {
+#if HIP_VERSION >= 309
+  return hipsparseDbsrilu02_numericBoost(handle, info, enable_boost, tol, boost_val);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCbsrilu02_numericBoost(cusparseHandle_t handle,
+                                                bsrilu02Info_t   info,
+                                                int              enable_boost,
+                                                double*          tol,
+                                                cuComplex*       boost_val) {
+#if HIP_VERSION >= 309
+  return hipsparseCbsrilu02_numericBoost(handle, info, enable_boost, tol, reinterpret_cast<hipComplex*>(boost_val));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseZbsrilu02_numericBoost(cusparseHandle_t handle,
+                                                bsrilu02Info_t   info,
+                                                int              enable_boost,
+                                                double*          tol,
+                                                cuDoubleComplex* boost_val) {
+#if HIP_VERSION >= 309
+  return hipsparseZbsrilu02_numericBoost(handle, info, enable_boost, tol, reinterpret_cast<hipDoubleComplex*>(boost_val));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseXbsrilu02_zeroPivot(cusparseHandle_t handle,
+                                             bsrilu02Info_t   info,
+                                             int*             position) {
+#if HIP_VERSION >= 309
+  return hipsparseXbsrilu02_zeroPivot(handle, info, position);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSbsrilu02_bufferSize(cusparseHandle_t         handle,
+                                              cusparseDirection_t      dirA,
+                                              int                      mb,
+                                              int                      nnzb,
+                                              const cusparseMatDescr_t descrA,
+                                              float*                   bsrSortedVal,
+                                              const int*               bsrSortedRowPtr,
+                                              const int*               bsrSortedColInd,
+                                              int                      blockDim,
+                                              bsrilu02Info_t           info,
+                                              int*                     pBufferSizeInBytes) {
+#if HIP_VERSION >= 309
+  return hipsparseSbsrilu02_bufferSize(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, pBufferSizeInBytes);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDbsrilu02_bufferSize(cusparseHandle_t         handle,
+                                              cusparseDirection_t      dirA,
+                                              int                      mb,
+                                              int                      nnzb,
+                                              const cusparseMatDescr_t descrA,
+                                              double*                  bsrSortedVal,
+                                              const int*               bsrSortedRowPtr,
+                                              const int*               bsrSortedColInd,
+                                              int                      blockDim,
+                                              bsrilu02Info_t           info,
+                                              int*                     pBufferSizeInBytes) {
+#if HIP_VERSION >= 309
+  return hipsparseDbsrilu02_bufferSize(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, pBufferSizeInBytes);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCbsrilu02_bufferSize(cusparseHandle_t         handle,
+                                              cusparseDirection_t      dirA,
+                                              int                      mb,
+                                              int                      nnzb,
+                                              const cusparseMatDescr_t descrA,
+                                              cuComplex*               bsrSortedVal,
+                                              const int*               bsrSortedRowPtr,
+                                              const int*               bsrSortedColInd,
+                                              int                      blockDim,
+                                              bsrilu02Info_t           info,
+                                              int*                     pBufferSizeInBytes) {
+#if HIP_VERSION >= 309
+  return hipsparseCbsrilu02_bufferSize(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, pBufferSizeInBytes);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseZbsrilu02_bufferSize(cusparseHandle_t         handle,
+                                              cusparseDirection_t      dirA,
+                                              int                      mb,
+                                              int                      nnzb,
+                                              const cusparseMatDescr_t descrA,
+                                              cuDoubleComplex*         bsrSortedVal,
+                                              const int*               bsrSortedRowPtr,
+                                              const int*               bsrSortedColInd,
+                                              int                      blockDim,
+                                              bsrilu02Info_t           info,
+                                              int*                     pBufferSizeInBytes) {
+#if HIP_VERSION >= 309
+  return hipsparseZbsrilu02_bufferSize(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipDoubleComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, pBufferSizeInBytes);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSbsrilu02_analysis(cusparseHandle_t         handle,
+                                            cusparseDirection_t      dirA,
+                                            int                      mb,
+                                            int                      nnzb,
+                                            const cusparseMatDescr_t descrA,
+                                            float*                   bsrSortedVal,
+                                            const int*               bsrSortedRowPtr,
+                                            const int*               bsrSortedColInd,
+                                            int                      blockDim,
+                                            bsrilu02Info_t           info,
+                                            cusparseSolvePolicy_t    policy,
+                                            void*                    pBuffer) {
+#if HIP_VERSION >= 309
+  return hipsparseSbsrilu02_analysis(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDbsrilu02_analysis(cusparseHandle_t         handle,
+                                            cusparseDirection_t      dirA,
+                                            int                      mb,
+                                            int                      nnzb,
+                                            const cusparseMatDescr_t descrA,
+                                            double*                  bsrSortedVal,
+                                            const int*               bsrSortedRowPtr,
+                                            const int*               bsrSortedColInd,
+                                            int                      blockDim,
+                                            bsrilu02Info_t           info,
+                                            cusparseSolvePolicy_t    policy,
+                                            void*                    pBuffer) {
+#if HIP_VERSION >= 309
+  return hipsparseDbsrilu02_analysis(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCbsrilu02_analysis(cusparseHandle_t         handle,
+                                            cusparseDirection_t      dirA,
+                                            int                      mb,
+                                            int                      nnzb,
+                                            const cusparseMatDescr_t descrA,
+                                            cuComplex*               bsrSortedVal,
+                                            const int*               bsrSortedRowPtr,
+                                            const int*               bsrSortedColInd,
+                                            int                      blockDim,
+                                            bsrilu02Info_t           info,
+                                            cusparseSolvePolicy_t    policy,
+                                            void*                    pBuffer) {
+#if HIP_VERSION >= 309
+  return hipsparseCbsrilu02_analysis(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseZbsrilu02_analysis(cusparseHandle_t         handle,
+                                            cusparseDirection_t      dirA,
+                                            int                      mb,
+                                            int                      nnzb,
+                                            const cusparseMatDescr_t descrA,
+                                            cuDoubleComplex*         bsrSortedVal,
+                                            const int*               bsrSortedRowPtr,
+                                            const int*               bsrSortedColInd,
+                                            int                      blockDim,
+                                            bsrilu02Info_t           info,
+                                            cusparseSolvePolicy_t    policy,
+                                            void*                    pBuffer) {
+#if HIP_VERSION >= 309
+  return hipsparseZbsrilu02_analysis(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipDoubleComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSbsrilu02(cusparseHandle_t         handle,
+                                   cusparseDirection_t      dirA,
+                                   int                      mb,
+                                   int                      nnzb,
+                                   const cusparseMatDescr_t descrA,
+                                   float*                   bsrSortedVal,
+                                   const int*               bsrSortedRowPtr,
+                                   const int*               bsrSortedColInd,
+                                   int                      blockDim,
+                                   bsrilu02Info_t           info,
+                                   cusparseSolvePolicy_t    policy,
+                                   void*                    pBuffer) {
+#if HIP_VERSION >= 309
+  return hipsparseSbsrilu02(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDbsrilu02(cusparseHandle_t         handle,
+                                   cusparseDirection_t      dirA,
+                                   int                      mb,
+                                   int                      nnzb,
+                                   const cusparseMatDescr_t descrA,
+                                   double*                  bsrSortedVal,
+                                   const int*               bsrSortedRowPtr,
+                                   const int*               bsrSortedColInd,
+                                   int                      blockDim,
+                                   bsrilu02Info_t           info,
+                                   cusparseSolvePolicy_t    policy,
+                                   void*                    pBuffer) {
+#if HIP_VERSION >= 309
+  return hipsparseDbsrilu02(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCbsrilu02(cusparseHandle_t         handle,
+                                   cusparseDirection_t      dirA,
+                                   int                      mb,
+                                   int                      nnzb,
+                                   const cusparseMatDescr_t descrA,
+                                   cuComplex*               bsrSortedVal,
+                                   const int*               bsrSortedRowPtr,
+                                   const int*               bsrSortedColInd,
+                                   int                      blockDim,
+                                   bsrilu02Info_t           info,
+                                   cusparseSolvePolicy_t    policy,
+                                   void*                    pBuffer) {
+#if HIP_VERSION >= 309
+  return hipsparseCbsrilu02(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseZbsrilu02(cusparseHandle_t         handle,
+                                   cusparseDirection_t      dirA,
+                                   int                      mb,
+                                   int                      nnzb,
+                                   const cusparseMatDescr_t descrA,
+                                   cuDoubleComplex*         bsrSortedVal,
+                                   const int*               bsrSortedRowPtr,
+                                   const int*               bsrSortedColInd,
+                                   int                      blockDim,
+                                   bsrilu02Info_t           info,
+                                   cusparseSolvePolicy_t    policy,
+                                   void*                    pBuffer) {
+#if HIP_VERSION >= 309
+  return hipsparseZbsrilu02(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipDoubleComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseXcsric02_zeroPivot(cusparseHandle_t handle,
+                                            csric02Info_t    info,
+                                            int*             position) {
+  return hipsparseXcsric02_zeroPivot(handle, info, position);
+}
+
+cusparseStatus_t cusparseScsric02_bufferSize(cusparseHandle_t         handle,
+                                             int                      m,
+                                             int                      nnz,
+                                             const cusparseMatDescr_t descrA,
+                                             float*                   csrSortedValA,
+                                             const int*               csrSortedRowPtrA,
+                                             const int*               csrSortedColIndA,
+                                             csric02Info_t            info,
+                                             int*                     pBufferSizeInBytes) {
+  return hipsparseScsric02_bufferSize(handle, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseDcsric02_bufferSize(cusparseHandle_t         handle,
+                                             int                      m,
+                                             int                      nnz,
+                                             const cusparseMatDescr_t descrA,
+                                             double*                  csrSortedValA,
+                                             const int*               csrSortedRowPtrA,
+                                             const int*               csrSortedColIndA,
+                                             csric02Info_t            info,
+                                             int*                     pBufferSizeInBytes) {
+  return hipsparseDcsric02_bufferSize(handle, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseCcsric02_bufferSize(cusparseHandle_t         handle,
+                                             int                      m,
+                                             int                      nnz,
+                                             const cusparseMatDescr_t descrA,
+                                             cuComplex*               csrSortedValA,
+                                             const int*               csrSortedRowPtrA,
+                                             const int*               csrSortedColIndA,
+                                             csric02Info_t            info,
+                                             int*                     pBufferSizeInBytes) {
+  return hipsparseCcsric02_bufferSize(handle, m, nnz, descrA, reinterpret_cast<hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseZcsric02_bufferSize(cusparseHandle_t         handle,
+                                             int                      m,
+                                             int                      nnz,
+                                             const cusparseMatDescr_t descrA,
+                                             cuDoubleComplex*         csrSortedValA,
+                                             const int*               csrSortedRowPtrA,
+                                             const int*               csrSortedColIndA,
+                                             csric02Info_t            info,
+                                             int*                     pBufferSizeInBytes) {
+  return hipsparseZcsric02_bufferSize(handle, m, nnz, descrA, reinterpret_cast<hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, pBufferSizeInBytes);
+}
+
+cusparseStatus_t cusparseScsric02_analysis(cusparseHandle_t         handle,
+                                           int                      m,
+                                           int                      nnz,
+                                           const cusparseMatDescr_t descrA,
+                                           const float*             csrSortedValA,
+                                           const int*               csrSortedRowPtrA,
+                                           const int*               csrSortedColIndA,
+                                           csric02Info_t            info,
+                                           cusparseSolvePolicy_t    policy,
+                                           void*                    pBuffer) {
+  return hipsparseScsric02_analysis(handle, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseDcsric02_analysis(cusparseHandle_t         handle,
+                                           int                      m,
+                                           int                      nnz,
+                                           const cusparseMatDescr_t descrA,
+                                           const double*            csrSortedValA,
+                                           const int*               csrSortedRowPtrA,
+                                           const int*               csrSortedColIndA,
+                                           csric02Info_t            info,
+                                           cusparseSolvePolicy_t    policy,
+                                           void*                    pBuffer) {
+  return hipsparseDcsric02_analysis(handle, m, nnz, descrA, csrSortedValA, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseCcsric02_analysis(cusparseHandle_t         handle,
+                                           int                      m,
+                                           int                      nnz,
+                                           const cusparseMatDescr_t descrA,
+                                           const cuComplex*         csrSortedValA,
+                                           const int*               csrSortedRowPtrA,
+                                           const int*               csrSortedColIndA,
+                                           csric02Info_t            info,
+                                           cusparseSolvePolicy_t    policy,
+                                           void*                    pBuffer) {
+  return hipsparseCcsric02_analysis(handle, m, nnz, descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseZcsric02_analysis(cusparseHandle_t         handle,
+                                           int                      m,
+                                           int                      nnz,
+                                           const cusparseMatDescr_t descrA,
+                                           const cuDoubleComplex*   csrSortedValA,
+                                           const int*               csrSortedRowPtrA,
+                                           const int*               csrSortedColIndA,
+                                           csric02Info_t            info,
+                                           cusparseSolvePolicy_t    policy,
+                                           void*                    pBuffer) {
+  return hipsparseZcsric02_analysis(handle, m, nnz, descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseScsric02(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      nnz,
+                                  const cusparseMatDescr_t descrA,
+                                  float*                   csrSortedValA_valM,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  csric02Info_t            info,
+                                  cusparseSolvePolicy_t    policy,
+                                  void*                    pBuffer) {
+  return hipsparseScsric02(handle, m, nnz, descrA, csrSortedValA_valM, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseDcsric02(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      nnz,
+                                  const cusparseMatDescr_t descrA,
+                                  double*                  csrSortedValA_valM,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  csric02Info_t            info,
+                                  cusparseSolvePolicy_t    policy,
+                                  void*                    pBuffer) {
+  return hipsparseDcsric02(handle, m, nnz, descrA, csrSortedValA_valM, csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseCcsric02(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      nnz,
+                                  const cusparseMatDescr_t descrA,
+                                  cuComplex*               csrSortedValA_valM,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  csric02Info_t            info,
+                                  cusparseSolvePolicy_t    policy,
+                                  void*                    pBuffer) {
+  return hipsparseCcsric02(handle, m, nnz, descrA, reinterpret_cast<hipComplex*>(csrSortedValA_valM), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseZcsric02(cusparseHandle_t         handle,
+                                  int                      m,
+                                  int                      nnz,
+                                  const cusparseMatDescr_t descrA,
+                                  cuDoubleComplex*         csrSortedValA_valM,
+                                  const int*               csrSortedRowPtrA,
+                                  const int*               csrSortedColIndA,
+                                  csric02Info_t            info,
+                                  cusparseSolvePolicy_t    policy,
+                                  void*                    pBuffer) {
+  return hipsparseZcsric02(handle, m, nnz, descrA, reinterpret_cast<hipDoubleComplex*>(csrSortedValA_valM), csrSortedRowPtrA, csrSortedColIndA, info, policy, pBuffer);
+}
+
+cusparseStatus_t cusparseXbsric02_zeroPivot(cusparseHandle_t handle,
+                                            bsric02Info_t    info,
+                                            int*             position) {
+#if HIP_VERSION >= 308
+  return hipsparseXbsric02_zeroPivot(handle, info, position);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSbsric02_bufferSize(cusparseHandle_t         handle,
+                                             cusparseDirection_t      dirA,
+                                             int                      mb,
+                                             int                      nnzb,
+                                             const cusparseMatDescr_t descrA,
+                                             float*                   bsrSortedVal,
+                                             const int*               bsrSortedRowPtr,
+                                             const int*               bsrSortedColInd,
+                                             int                      blockDim,
+                                             bsric02Info_t            info,
+                                             int*                     pBufferSizeInBytes) {
+#if HIP_VERSION >= 308
+  return hipsparseSbsric02_bufferSize(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, pBufferSizeInBytes);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDbsric02_bufferSize(cusparseHandle_t         handle,
+                                             cusparseDirection_t      dirA,
+                                             int                      mb,
+                                             int                      nnzb,
+                                             const cusparseMatDescr_t descrA,
+                                             double*                  bsrSortedVal,
+                                             const int*               bsrSortedRowPtr,
+                                             const int*               bsrSortedColInd,
+                                             int                      blockDim,
+                                             bsric02Info_t            info,
+                                             int*                     pBufferSizeInBytes) {
+#if HIP_VERSION >= 308
+  return hipsparseDbsric02_bufferSize(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, pBufferSizeInBytes);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCbsric02_bufferSize(cusparseHandle_t         handle,
+                                             cusparseDirection_t      dirA,
+                                             int                      mb,
+                                             int                      nnzb,
+                                             const cusparseMatDescr_t descrA,
+                                             cuComplex*               bsrSortedVal,
+                                             const int*               bsrSortedRowPtr,
+                                             const int*               bsrSortedColInd,
+                                             int                      blockDim,
+                                             bsric02Info_t            info,
+                                             int*                     pBufferSizeInBytes) {
+#if HIP_VERSION >= 308
+  return hipsparseCbsric02_bufferSize(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, pBufferSizeInBytes);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseZbsric02_bufferSize(cusparseHandle_t         handle,
+                                             cusparseDirection_t      dirA,
+                                             int                      mb,
+                                             int                      nnzb,
+                                             const cusparseMatDescr_t descrA,
+                                             cuDoubleComplex*         bsrSortedVal,
+                                             const int*               bsrSortedRowPtr,
+                                             const int*               bsrSortedColInd,
+                                             int                      blockDim,
+                                             bsric02Info_t            info,
+                                             int*                     pBufferSizeInBytes) {
+#if HIP_VERSION >= 308
+  return hipsparseZbsric02_bufferSize(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipDoubleComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, pBufferSizeInBytes);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSbsric02_analysis(cusparseHandle_t         handle,
+                                           cusparseDirection_t      dirA,
+                                           int                      mb,
+                                           int                      nnzb,
+                                           const cusparseMatDescr_t descrA,
+                                           const float*             bsrSortedVal,
+                                           const int*               bsrSortedRowPtr,
+                                           const int*               bsrSortedColInd,
+                                           int                      blockDim,
+                                           bsric02Info_t            info,
+                                           cusparseSolvePolicy_t    policy,
+                                           void*                    pInputBuffer) {
+#if HIP_VERSION >= 308
+  return hipsparseSbsric02_analysis(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pInputBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDbsric02_analysis(cusparseHandle_t         handle,
+                                           cusparseDirection_t      dirA,
+                                           int                      mb,
+                                           int                      nnzb,
+                                           const cusparseMatDescr_t descrA,
+                                           const double*            bsrSortedVal,
+                                           const int*               bsrSortedRowPtr,
+                                           const int*               bsrSortedColInd,
+                                           int                      blockDim,
+                                           bsric02Info_t            info,
+                                           cusparseSolvePolicy_t    policy,
+                                           void*                    pInputBuffer) {
+#if HIP_VERSION >= 308
+  return hipsparseDbsric02_analysis(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pInputBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCbsric02_analysis(cusparseHandle_t         handle,
+                                           cusparseDirection_t      dirA,
+                                           int                      mb,
+                                           int                      nnzb,
+                                           const cusparseMatDescr_t descrA,
+                                           const cuComplex*         bsrSortedVal,
+                                           const int*               bsrSortedRowPtr,
+                                           const int*               bsrSortedColInd,
+                                           int                      blockDim,
+                                           bsric02Info_t            info,
+                                           cusparseSolvePolicy_t    policy,
+                                           void*                    pInputBuffer) {
+#if HIP_VERSION >= 308
+  return hipsparseCbsric02_analysis(handle, dirA, mb, nnzb, descrA, reinterpret_cast<const hipComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pInputBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseZbsric02_analysis(cusparseHandle_t         handle,
+                                           cusparseDirection_t      dirA,
+                                           int                      mb,
+                                           int                      nnzb,
+                                           const cusparseMatDescr_t descrA,
+                                           const cuDoubleComplex*   bsrSortedVal,
+                                           const int*               bsrSortedRowPtr,
+                                           const int*               bsrSortedColInd,
+                                           int                      blockDim,
+                                           bsric02Info_t            info,
+                                           cusparseSolvePolicy_t    policy,
+                                           void*                    pInputBuffer) {
+#if HIP_VERSION >= 308
+  return hipsparseZbsric02_analysis(handle, dirA, mb, nnzb, descrA, reinterpret_cast<const hipDoubleComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pInputBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSbsric02(cusparseHandle_t         handle,
+                                  cusparseDirection_t      dirA,
+                                  int                      mb,
+                                  int                      nnzb,
+                                  const cusparseMatDescr_t descrA,
+                                  float*                   bsrSortedVal,
+                                  const int*               bsrSortedRowPtr,
+                                  const int*               bsrSortedColInd,
+                                  int                      blockDim,
+                                  bsric02Info_t            info,
+                                  cusparseSolvePolicy_t    policy,
+                                  void*                    pBuffer) {
+#if HIP_VERSION >= 308
+  return hipsparseSbsric02(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDbsric02(cusparseHandle_t         handle,
+                                  cusparseDirection_t      dirA,
+                                  int                      mb,
+                                  int                      nnzb,
+                                  const cusparseMatDescr_t descrA,
+                                  double*                  bsrSortedVal,
+                                  const int*               bsrSortedRowPtr,
+                                  const int*               bsrSortedColInd,
+                                  int                      blockDim,
+                                  bsric02Info_t            info,
+                                  cusparseSolvePolicy_t    policy,
+                                  void*                    pBuffer) {
+#if HIP_VERSION >= 308
+  return hipsparseDbsric02(handle, dirA, mb, nnzb, descrA, bsrSortedVal, bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCbsric02(cusparseHandle_t         handle,
+                                  cusparseDirection_t      dirA,
+                                  int                      mb,
+                                  int                      nnzb,
+                                  const cusparseMatDescr_t descrA,
+                                  cuComplex*               bsrSortedVal,
+                                  const int*               bsrSortedRowPtr,
+                                  const int*
+                                       bsrSortedColInd,
+                                  int                      blockDim,
+                                  bsric02Info_t            info,
+                                  cusparseSolvePolicy_t    policy,
+                                  void*                    pBuffer) {
+#if HIP_VERSION >= 308
+  return hipsparseCbsric02(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseZbsric02(cusparseHandle_t         handle,
+                                  cusparseDirection_t      dirA,
+                                  int                      mb,
+                                  int                      nnzb,
+                                  const cusparseMatDescr_t descrA,
+                                  cuDoubleComplex*         bsrSortedVal,
+                                  const int*               bsrSortedRowPtr,
+                                  const int*               bsrSortedColInd,
+                                  int                      blockDim,
+                                  bsric02Info_t            info,
+                                  cusparseSolvePolicy_t    policy,
+                                  void*                    pBuffer) {
+#if HIP_VERSION >= 308
+  return hipsparseZbsric02(handle, dirA, mb, nnzb, descrA, reinterpret_cast<hipDoubleComplex*>(bsrSortedVal), bsrSortedRowPtr, bsrSortedColInd, blockDim, info, policy, pBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSgtsv2_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgtsv2_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgtsv2_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgtsv2_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgtsv2(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgtsv2(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgtsv2(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgtsv2(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgtsv2_nopivot_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgtsv2_nopivot_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgtsv2_nopivot_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgtsv2_nopivot_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgtsv2_nopivot(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgtsv2_nopivot(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgtsv2_nopivot(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgtsv2_nopivot(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgtsv2StridedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgtsv2StridedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgtsv2StridedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgtsv2StridedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgtsv2StridedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgtsv2StridedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgtsv2StridedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgtsv2StridedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgtsvInterleavedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgtsvInterleavedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgtsvInterleavedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgtsvInterleavedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgtsvInterleavedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgtsvInterleavedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgtsvInterleavedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgtsvInterleavedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgpsvInterleavedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgpsvInterleavedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgpsvInterleavedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgpsvInterleavedBatch_bufferSizeExt(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSgpsvInterleavedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDgpsvInterleavedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCgpsvInterleavedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseZgpsvInterleavedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+#define CUSPARSE_VERSION (hipsparseVersionMajor*100000+hipsparseVersionMinor*100+hipsparseVersionPatch)
+
+// cuSPARSE generic API
+#if HIP_VERSION >= 402
+typedef hipsparseSpVecDescr_t cusparseSpVecDescr_t;
+#else
+typedef void* cusparseSpVecDescr_t;
+#endif
+
+#if HIP_VERSION >= 402
+typedef hipsparseDnVecDescr_t cusparseDnVecDescr_t;
+#else
+typedef void* cusparseDnVecDescr_t;
+#endif
+
+#if HIP_VERSION >= 402
+typedef hipsparseSpMatDescr_t cusparseSpMatDescr_t;
+#else
+typedef void* cusparseSpMatDescr_t;
+#endif
+
+#if HIP_VERSION >= 402
+typedef hipsparseDnMatDescr_t cusparseDnMatDescr_t;
+#else
+typedef void* cusparseDnMatDescr_t;
+#endif
+
+
+#if HIP_VERSION >= 402
+typedef hipsparseIndexType_t cusparseIndexType_t;
+#else
+typedef enum {} cusparseIndexType_t;
+#endif
+
+#if HIP_VERSION >= 402
+typedef hipsparseFormat_t cusparseFormat_t;
+#else
+typedef enum {} cusparseFormat_t;
+#endif
+
+
+#if HIP_VERSION >= 402
+typedef enum {} cusparseOrder_t;
+static hipsparseOrder_t convert_hipsparseOrder_t(cusparseOrder_t type) {
+    switch(static_cast<int>(type)) {
+        case 1 /* CUSPARSE_ORDER_COL */: return HIPSPARSE_ORDER_COLUMN;
+        case 2 /* CUSPARSE_ORDER_ROW */: return HIPSPARSE_ORDER_ROW;
+        default: throw std::runtime_error("unrecognized type");
+    }
+}
+
+#else
+typedef enum {} cusparseOrder_t;
+#endif
+
+#if HIP_VERSION >= 402
+typedef hipsparseSpMVAlg_t cusparseSpMVAlg_t;
+#else
+typedef enum {} cusparseSpMVAlg_t;
+#endif
+
+#if HIP_VERSION >= 50000000
+typedef hipsparseSpMatAttribute_t cusparseSpMatAttribute_t;
+typedef hipsparseSpSMAlg_t cusparseSpSMAlg_t;
+typedef hipsparseSpSMDescr_t cusparseSpSMDescr_t;
+#else
+typedef enum {} cusparseSpMatAttribute_t;
+typedef enum {} cusparseSpSMAlg_t;
+typedef void * cusparseSpSMDescr_t;
+#endif
+
+#if HIP_VERSION >= 402
+typedef hipsparseSpMMAlg_t cusparseSpMMAlg_t;
+#else
+typedef enum {} cusparseSpMMAlg_t;
+#endif
+
+#if HIP_VERSION >= 402
+typedef hipsparseSparseToDenseAlg_t cusparseSparseToDenseAlg_t;
+#else
+typedef enum {} cusparseSparseToDenseAlg_t;
+#endif
+
+#if HIP_VERSION >= 402
+typedef hipsparseDenseToSparseAlg_t cusparseDenseToSparseAlg_t;
+#else
+typedef enum {} cusparseDenseToSparseAlg_t;
+#endif
+
+
+cusparseStatus_t cusparseCreateSpVec(cusparseSpVecDescr_t* spVecDescr,
+                                     int64_t               size,
+                                     int64_t               nnz,
+                                     void*                 indices,
+                                     void*                 values,
+                                     cusparseIndexType_t   idxType,
+                                     cusparseIndexBase_t   idxBase,
+                                     cudaDataType          valueType) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(valueType);
+  return hipsparseCreateSpVec(spVecDescr, size, nnz, indices, values, idxType, idxBase, blah);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDestroySpVec(cusparseSpVecDescr_t spVecDescr) {
+#if HIP_VERSION >= 402
+  return hipsparseDestroySpVec(spVecDescr);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpVecGet(cusparseSpVecDescr_t spVecDescr,
+                                  int64_t*             size,
+                                  int64_t*             nnz,
+                                  void**               indices,
+                                  void**               values,
+                                  cusparseIndexType_t* idxType,
+                                  cusparseIndexBase_t* idxBase,
+                                  cudaDataType*        valueType) {
+#if HIP_VERSION >= 402
+  return hipsparseSpVecGet(spVecDescr, size, nnz, indices, values, idxType, idxBase, reinterpret_cast<hipDataType*>(valueType));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpVecGetIndexBase(cusparseSpVecDescr_t spVecDescr,
+                                           cusparseIndexBase_t* idxBase) {
+#if HIP_VERSION >= 402
+  return hipsparseSpVecGetIndexBase(spVecDescr, idxBase);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpVecGetValues(cusparseSpVecDescr_t spVecDescr,
+                                        void**               values) {
+#if HIP_VERSION >= 402
+  return hipsparseSpVecGetValues(spVecDescr, values);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpVecSetValues(cusparseSpVecDescr_t spVecDescr,
+                                        void*                values) {
+#if HIP_VERSION >= 402
+  return hipsparseSpVecSetValues(spVecDescr, values);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCreateCoo(cusparseSpMatDescr_t* spMatDescr,
+                                   int64_t               rows,
+                                   int64_t               cols,
+                                   int64_t               nnz,
+                                   void*                 cooRowInd,
+                                   void*                 cooColInd,
+                                   void*                 cooValues,
+                                   cusparseIndexType_t   cooIdxType,
+                                   cusparseIndexBase_t   idxBase,
+                                   cudaDataType          valueType) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(valueType);
+  return hipsparseCreateCoo(spMatDescr, rows, cols, nnz, cooRowInd, cooColInd, cooValues, cooIdxType, idxBase, blah);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCreateCooAoS(cusparseSpMatDescr_t* spMatDescr,
+                                      int64_t               rows,
+                                      int64_t               cols,
+                                      int64_t               nnz,
+                                      void*                 cooInd,
+                                      void*                 cooValues,
+                                      cusparseIndexType_t   cooIdxType,
+                                      cusparseIndexBase_t   idxBase,
+                                      cudaDataType          valueType) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(valueType);
+  return hipsparseCreateCooAoS(spMatDescr, rows, cols, nnz, cooInd, cooValues, cooIdxType, idxBase, blah);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCreateCsr(cusparseSpMatDescr_t* spMatDescr,
+                                   int64_t               rows,
+                                   int64_t               cols,
+                                   int64_t               nnz,
+                                   void*                 csrRowOffsets,
+                                   void*                 csrColInd,
+                                   void*                 csrValues,
+                                   cusparseIndexType_t   csrRowOffsetsType,
+                                   cusparseIndexType_t   csrColIndType,
+                                   cusparseIndexBase_t   idxBase,
+                                   cudaDataType          valueType) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(valueType);
+  return hipsparseCreateCsr(spMatDescr, rows, cols, nnz, csrRowOffsets, csrColInd, csrValues, csrRowOffsetsType, csrColIndType, idxBase, blah);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCreateCsc(cusparseSpMatDescr_t* spMatDescr,
+                                   int64_t               rows,
+                                   int64_t               cols,
+                                   int64_t               nnz,
+                                   void*                 cscColOffsets,
+                                   void*                 cscRowInd,
+                                   void*                 cscValues,
+                                   cusparseIndexType_t   cscColOffsetsType,
+                                   cusparseIndexType_t   cscRowIndType,
+                                   cusparseIndexBase_t   idxBase,
+                                   cudaDataType          valueType) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(valueType);
+  return hipsparseCreateCsc(spMatDescr, rows, cols, nnz, cscColOffsets, cscRowInd, cscValues, cscColOffsetsType, cscRowIndType, idxBase, blah);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDestroySpMat(cusparseSpMatDescr_t spMatDescr) {
+#if HIP_VERSION >= 402
+  return hipsparseDestroySpMat(spMatDescr);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCooGet(cusparseSpMatDescr_t spMatDescr,
+                                int64_t*             rows,
+                                int64_t*             cols,
+                                int64_t*             nnz,
+                                void**               cooRowInd,  // COO row indices
+                                void**               cooColInd,  // COO column indices
+                                void**               cooValues,  // COO values
+                                cusparseIndexType_t* idxType,
+                                cusparseIndexBase_t* idxBase,
+                                cudaDataType*        valueType) {
+#if HIP_VERSION >= 402
+  return hipsparseCooGet(spMatDescr, rows, cols, nnz, cooRowInd, cooColInd, cooValues, idxType, idxBase, reinterpret_cast<hipDataType*>(valueType));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCooAoSGet(cusparseSpMatDescr_t spMatDescr,
+                                   int64_t*             rows,
+                                   int64_t*             cols,
+                                   int64_t*             nnz,
+                                   void**               cooInd,     // COO indices
+                                   void**               cooValues,  // COO values
+                                   cusparseIndexType_t* idxType,
+                                   cusparseIndexBase_t* idxBase,
+                                   cudaDataType*        valueType) {
+#if HIP_VERSION >= 402
+  return hipsparseCooAoSGet(spMatDescr, rows, cols, nnz, cooInd, cooValues, idxType, idxBase, reinterpret_cast<hipDataType*>(valueType));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCsrGet(cusparseSpMatDescr_t spMatDescr,
+                                int64_t*             rows,
+                                int64_t*             cols,
+                                int64_t*             nnz,
+                                void**               csrRowOffsets,
+                                void**               csrColInd,
+                                void**               csrValues,
+                                cusparseIndexType_t* csrRowOffsetsType,
+                                cusparseIndexType_t* csrColIndType,
+                                cusparseIndexBase_t* idxBase,
+                                cudaDataType*        valueType) {
+#if HIP_VERSION >= 402
+  return hipsparseCsrGet(spMatDescr, rows, cols, nnz, csrRowOffsets, csrColInd, csrValues, csrRowOffsetsType, csrColIndType, idxBase, reinterpret_cast<hipDataType*>(valueType));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCsrSetPointers(cusparseSpMatDescr_t spMatDescr,
+                                        void*                csrRowOffsets,
+                                        void*                csrColInd,
+                                        void*                csrValues) {
+#if HIP_VERSION >= 402
+  return hipsparseCsrSetPointers(spMatDescr, csrRowOffsets, csrColInd, csrValues);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMatGetSize(cusparseSpMatDescr_t spMatDescr,
+                                      int64_t*             rows,
+                                      int64_t*             cols,
+                                      int64_t*             nnz) {
+#if HIP_VERSION >= 402
+  return hipsparseSpMatGetSize(spMatDescr, rows, cols, nnz);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMatGetFormat(cusparseSpMatDescr_t spMatDescr,
+                                        cusparseFormat_t*    format) {
+#if HIP_VERSION >= 402
+  return hipsparseSpMatGetFormat(spMatDescr, format);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMatGetIndexBase(cusparseSpMatDescr_t spMatDescr,
+                                           cusparseIndexBase_t* idxBase) {
+#if HIP_VERSION >= 402
+  return hipsparseSpMatGetIndexBase(spMatDescr, idxBase);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMatGetValues(cusparseSpMatDescr_t spMatDescr,
+                                        void**               values) {
+#if HIP_VERSION >= 402
+  return hipsparseSpMatGetValues(spMatDescr, values);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMatSetValues(cusparseSpMatDescr_t spMatDescr,
+                                        void*                values) {
+#if HIP_VERSION >= 402
+  return hipsparseSpMatSetValues(spMatDescr, values);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMatGetStridedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSpMatSetStridedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSpMatSetAttribute(cusparseSpMatDescr_t     spMatDescr,
+                                           cusparseSpMatAttribute_t attribute,
+                                           void*                    data,
+                                           size_t                   dataSize) {
+#if HIP_VERSION >= 50000000
+  return hipsparseSpMatSetAttribute(spMatDescr, attribute, data, dataSize);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCreateDnVec(cusparseDnVecDescr_t* dnVecDescr,
+                                     int64_t               size,
+                                     void*                 values,
+                                     cudaDataType          valueType) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(valueType);
+  return hipsparseCreateDnVec(dnVecDescr, size, values, blah);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDestroyDnVec(cusparseDnVecDescr_t dnVecDescr) {
+#if HIP_VERSION >= 402
+  return hipsparseDestroyDnVec(dnVecDescr);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDnVecGet(cusparseDnVecDescr_t dnVecDescr,
+                                  int64_t*             size,
+                                  void**               values,
+                                  cudaDataType*        valueType) {
+#if HIP_VERSION >= 402
+  return hipsparseDnVecGet(dnVecDescr, size, values, reinterpret_cast<hipDataType*>(valueType));
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDnVecGetValues(cusparseDnVecDescr_t dnVecDescr,
+                                        void**               values) {
+#if HIP_VERSION >= 402
+  return hipsparseDnVecGetValues(dnVecDescr, values);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDnVecSetValues(cusparseDnVecDescr_t dnVecDescr,
+                                        void*                values) {
+#if HIP_VERSION >= 402
+  return hipsparseDnVecSetValues(dnVecDescr, values);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseCreateDnMat(cusparseDnMatDescr_t* dnMatDescr,
+                                     int64_t               rows,
+                                     int64_t               cols,
+                                     int64_t               ld,
+                                     void*                 values,
+                                     cudaDataType          valueType,
+                                     cusparseOrder_t       order) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(valueType);
+  hipsparseOrder_t blah2 = convert_hipsparseOrder_t(order);
+  return hipsparseCreateDnMat(dnMatDescr, rows, cols, ld, values, blah, blah2);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDestroyDnMat(cusparseDnMatDescr_t dnMatDescr) {
+#if HIP_VERSION >= 402
+  return hipsparseDestroyDnMat(dnMatDescr);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDnMatGet(cusparseDnMatDescr_t dnMatDescr,
+                                  int64_t*             rows,
+                                  int64_t*             cols,
+                                  int64_t*             ld,
+                                  void**               values,
+                                  cudaDataType*        type,
+                                  cusparseOrder_t*     order) {
+#if HIP_VERSION >= 402
+  hipsparseOrder_t blah2 = convert_hipsparseOrder_t(*order);
+  return hipsparseDnMatGet(dnMatDescr, rows, cols, ld, values, reinterpret_cast<hipDataType*>(type), &blah2);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDnMatGetValues(cusparseDnMatDescr_t dnMatDescr,
+                                        void**               values) {
+#if HIP_VERSION >= 402
+  return hipsparseDnMatGetValues(dnMatDescr, values);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDnMatSetValues(cusparseDnMatDescr_t dnMatDescr,
+                                        void*                values) {
+#if HIP_VERSION >= 402
+  return hipsparseDnMatSetValues(dnMatDescr, values);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDnMatGetStridedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseDnMatSetStridedBatch(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSpVV_bufferSize(cusparseHandle_t     handle,
+                                         cusparseOperation_t  opX,
+                                         cusparseSpVecDescr_t vecX,
+                                         cusparseDnVecDescr_t vecY,
+                                         const void*          result,
+                                         cudaDataType         computeType,
+                                         size_t*              bufferSize) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(computeType);
+  return hipsparseSpVV_bufferSize(handle, opX, vecX, vecY, const_cast<void*>(result), blah, bufferSize);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpVV(cusparseHandle_t     handle,
+                              cusparseOperation_t  opX,
+                              cusparseSpVecDescr_t vecX,
+                              cusparseDnVecDescr_t vecY,
+                              void*                result,
+                              cudaDataType         computeType,
+                              void*                externalBuffer) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(computeType);
+  return hipsparseSpVV(handle, opX, vecX, vecY, result, blah, externalBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMV_bufferSize(cusparseHandle_t    handle,
+                                         cusparseOperation_t opA,
+                                         const void*         alpha,
+                                         cusparseSpMatDescr_t matA,
+                                         cusparseDnVecDescr_t vecX,
+                                         const void*          beta,
+                                         cusparseDnVecDescr_t vecY,
+                                         cudaDataType         computeType,
+                                         cusparseSpMVAlg_t    alg,
+                                         size_t*              bufferSize) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(computeType);
+  return hipsparseSpMV_bufferSize(handle, opA, alpha, matA, vecX, beta, vecY, blah, alg, bufferSize);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMV(cusparseHandle_t     handle,
+                              cusparseOperation_t  opA,
+                              const void*          alpha,
+                              cusparseSpMatDescr_t matA,
+                              cusparseDnVecDescr_t vecX,
+                              const void*          beta,
+                              cusparseDnVecDescr_t vecY,
+                              cudaDataType         computeType,
+                              cusparseSpMVAlg_t    alg,
+                              void*                externalBuffer) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(computeType);
+  return hipsparseSpMV(handle, opA, alpha, matA, vecX, beta, vecY, blah, alg, externalBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpSM_createDescr(cusparseSpSMDescr_t* descr) {
+#if HIP_VERSION >= 50000000
+  return hipsparseSpSM_createDescr(descr);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpSM_destroyDescr(cusparseSpSMDescr_t descr) {
+#if HIP_VERSION >= 50000000
+  return hipsparseSpSM_destroyDescr(descr);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpSM_bufferSize(cusparseHandle_t     handle,
+                                         cusparseOperation_t  opA,
+                                         cusparseOperation_t  opB,
+                                         const void*          alpha,
+                                         cusparseSpMatDescr_t matA,
+                                         cusparseDnMatDescr_t matB,
+                                         cusparseDnMatDescr_t matC,
+                                         cudaDataType         computeType,
+                                         cusparseSpSMAlg_t    alg,
+                                         cusparseSpSMDescr_t  spsmDescr,
+                                         size_t*              bufferSize) {
+#if HIP_VERSION >= 50000000
+  hipDataType computeType1 = convert_hipDatatype(computeType);
+  return hipsparseSpSM_bufferSize(handle, opA, opB, alpha, matA, matB, matC, computeType1, alg, spsmDescr, bufferSize);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpSM_analysis(cusparseHandle_t     handle,
+                                       cusparseOperation_t  opA,
+                                       cusparseOperation_t  opB,
+                                       const void*          alpha,
+                                       cusparseSpMatDescr_t matA,
+                                       cusparseDnMatDescr_t matB,
+                                       cusparseDnMatDescr_t matC,
+                                       cudaDataType         computeType,
+                                       cusparseSpSMAlg_t    alg,
+                                       cusparseSpSMDescr_t  spsmDescr,
+                                       void*                externalBuffer) {
+#if HIP_VERSION >= 50000000
+  hipDataType computeType1 = convert_hipDatatype(computeType);
+  return hipsparseSpSM_analysis(handle, opA, opB, alpha, matA, matB, matC, computeType1, alg, spsmDescr, externalBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+// See cusparse.pyx for a comment
+cusparseStatus_t cusparseSpSM_solve(cusparseHandle_t     handle,
+                                    cusparseOperation_t  opA,
+                                    cusparseOperation_t  opB,
+                                    const void*          alpha,
+                                    cusparseSpMatDescr_t matA,
+                                    cusparseDnMatDescr_t matB,
+                                    cusparseDnMatDescr_t matC,
+                                    cudaDataType         computeType,
+                                    cusparseSpSMAlg_t    alg,
+                                    cusparseSpSMDescr_t  spsmDescr,
+                                    void*                externalBuffer) {
+#if HIP_VERSION >= 50000000
+  hipDataType computeType1 = convert_hipDatatype(computeType);
+  return hipsparseSpSM_solve(handle, opA, opB, alpha, matA, matB, matC, computeType1, alg, spsmDescr, externalBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMM_bufferSize(cusparseHandle_t     handle,
+                                         cusparseOperation_t  opA,
+                                         cusparseOperation_t  opB,
+                                         const void*          alpha,
+                                         cusparseSpMatDescr_t matA,
+                                         cusparseDnMatDescr_t matB,
+                                         const void*          beta,
+                                         cusparseDnMatDescr_t matC,
+                                         cudaDataType         computeType,
+                                         cusparseSpMMAlg_t    alg,
+                                         size_t*              bufferSize) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(computeType);
+  return hipsparseSpMM_bufferSize(handle, opA, opB, alpha, matA, matB, beta, matC, blah, alg, bufferSize);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSpMM(cusparseHandle_t     handle,
+                              cusparseOperation_t  opA,
+                              cusparseOperation_t  opB,
+                              const void*          alpha,
+                              cusparseSpMatDescr_t matA,
+                              cusparseDnMatDescr_t matB,
+                              const void*          beta,
+                              cusparseDnMatDescr_t matC,
+                              cudaDataType         computeType,
+                              cusparseSpMMAlg_t    alg,
+                              void*                externalBuffer) {
+#if HIP_VERSION >= 402
+  hipDataType blah = convert_hipDatatype(computeType);
+  return hipsparseSpMM(handle, opA, opB, alpha, matA, matB, beta, matC, blah, alg, externalBuffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseConstrainedGeMM_bufferSize(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseConstrainedGeMM(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSparseToDense_bufferSize(cusparseHandle_t           handle,
+                                                  cusparseSpMatDescr_t       matA,
+                                                  cusparseDnMatDescr_t       matB,
+                                                  cusparseSparseToDenseAlg_t alg,
+                                                  size_t*                    bufferSize) {
+#if HIP_VERSION >= 402
+  return hipsparseSparseToDense_bufferSize(handle, matA, matB, alg, bufferSize);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseSparseToDense(cusparseHandle_t           handle,
+                                       cusparseSpMatDescr_t       matA,
+                                       cusparseDnMatDescr_t       matB,
+                                       cusparseSparseToDenseAlg_t alg,
+                                       void*                      buffer) {
+#if HIP_VERSION >= 402
+  return hipsparseSparseToDense(handle, matA, matB, alg, buffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDenseToSparse_bufferSize(cusparseHandle_t           handle,
+                                                  cusparseDnMatDescr_t       matA,
+                                                  cusparseSpMatDescr_t       matB,
+                                                  cusparseDenseToSparseAlg_t alg,
+                                                  size_t*                    bufferSize) {
+#if HIP_VERSION >= 402
+  return hipsparseDenseToSparse_bufferSize(handle, matA, matB, alg, bufferSize);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDenseToSparse_analysis(cusparseHandle_t           handle,
+                                                cusparseDnMatDescr_t       matA,
+                                                cusparseSpMatDescr_t       matB,
+                                                cusparseDenseToSparseAlg_t alg,
+                                                void*                      buffer) {
+#if HIP_VERSION >= 402
+  return hipsparseDenseToSparse_analysis(handle, matA, matB, alg, buffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+cusparseStatus_t cusparseDenseToSparse_convert(cusparseHandle_t           handle,
+                                               cusparseDnMatDescr_t       matA,
+                                               cusparseSpMatDescr_t       matB,
+                                               cusparseDenseToSparseAlg_t alg,
+                                               void*                      buffer) {
+#if HIP_VERSION >= 402
+  return hipsparseDenseToSparse_convert(handle, matA, matB, alg, buffer);
+#else
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+#endif
+}
+
+typedef enum {} cusparseCsr2CscAlg_t;
+
+cusparseStatus_t cusparseCsr2cscEx2_bufferSize(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseCsr2cscEx2(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+typedef void* cusparseSpGEMMDescr_t;
+typedef enum {} cusparseSpGEMMAlg_t;
+
+cusparseStatus_t cusparseSpGEMM_createDescr(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSpGEMM_destroyDescr(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSpGEMM_workEstimation(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSpGEMM_compute(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseSpGEMM_copy(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+cusparseStatus_t cusparseGather(...) {
+  return HIPSPARSE_STATUS_NOT_SUPPORTED;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Definitions are for compatibility
+///////////////////////////////////////////////////////////////////////////////
+
+cusparseStatus_t cusparseSnnz_compress(cusparseHandle_t         handle,
+                                       int                      m,
+                                       const cusparseMatDescr_t descr,
+                                       const float*             csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       int*                     nnzPerRow,
+                                       int*                     nnzC,
+                                       float                    tol) {
+  return hipsparseSnnz_compress(handle, m, descr, csrSortedValA, csrSortedRowPtrA, nnzPerRow, nnzC, tol);
+}
+
+cusparseStatus_t cusparseDnnz_compress(cusparseHandle_t         handle,
+                                       int                      m,
+                                       const cusparseMatDescr_t descr,
+                                       const double*            csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       int*                     nnzPerRow,
+                                       int*                     nnzC,
+                                       double                   tol) {
+  return hipsparseDnnz_compress(handle, m, descr, csrSortedValA, csrSortedRowPtrA, nnzPerRow, nnzC, tol);
+}
+
+cusparseStatus_t cusparseCnnz_compress(cusparseHandle_t         handle,
+                                       int                      m,
+                                       const cusparseMatDescr_t descr,
+                                       const cuComplex*         csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       int*                     nnzPerRow,
+                                       int*                     nnzC,
+                                       cuComplex                tol) {
+  hipComplex blah;
+  blah.x=tol.x;
+  blah.y=tol.y;
+  return hipsparseCnnz_compress(handle, m, descr, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedRowPtrA, nnzPerRow, nnzC, blah);
+}
+
+cusparseStatus_t cusparseZnnz_compress(cusparseHandle_t         handle,
+                                       int                      m,
+                                       const cusparseMatDescr_t descr,
+                                       const cuDoubleComplex*   csrSortedValA,
+                                       const int*               csrSortedRowPtrA,
+                                       int*                     nnzPerRow,
+                                       int*                     nnzC,
+                                       cuDoubleComplex          tol) {
+  hipDoubleComplex blah;
+  blah.x=tol.x;
+  blah.y=tol.y;
+  return hipsparseZnnz_compress(handle, m, descr, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedRowPtrA, nnzPerRow, nnzC, blah);
+}
+
+cusparseStatus_t cusparseScsr2csr_compress(cusparseHandle_t         handle,
+                                           int                      m,
+                                           int                      n,
+                                           const cusparseMatDescr_t descrA,
+                                           const float*             csrSortedValA,
+                                           const int*               csrSortedColIndA,
+                                           const int*               csrSortedRowPtrA,
+                                           int                      nnzA,
+                                           const int*               nnzPerRow,
+                                           float*                   csrSortedValC,
+                                           int*                     csrSortedColIndC,
+                                           int*                     csrSortedRowPtrC,
+                                           float                    tol) {
+  return hipsparseScsr2csr_compress(handle, m, n, descrA, csrSortedValA, csrSortedColIndA, csrSortedRowPtrA, nnzA, nnzPerRow, csrSortedValC, csrSortedColIndC, csrSortedRowPtrC, tol);
+}
+
+cusparseStatus_t cusparseDcsr2csr_compress(cusparseHandle_t         handle,
+                                           int                      m,
+                                           int                      n,
+                                           const cusparseMatDescr_t descrA,
+                                           const double*            csrSortedValA,
+                                           const int*               csrSortedColIndA,
+                                           const int*               csrSortedRowPtrA,
+                                           int                      nnzA,
+                                           const int*               nnzPerRow,
+                                           double*                  csrSortedValC,
+                                           int*                     csrSortedColIndC,
+                                           int*                     csrSortedRowPtrC,
+                                           double                   tol) {
+  return hipsparseDcsr2csr_compress(handle, m, n, descrA, csrSortedValA, csrSortedColIndA, csrSortedRowPtrA, nnzA, nnzPerRow, csrSortedValC, csrSortedColIndC, csrSortedRowPtrC, tol);
+}
+
+cusparseStatus_t cusparseCcsr2csr_compress(cusparseHandle_t         handle,
+                                           int                      m,
+                                           int                      n,
+                                           const cusparseMatDescr_t descrA,
+                                           const cuComplex*         csrSortedValA,
+                                           const int*               csrSortedColIndA,
+                                           const int*               csrSortedRowPtrA,
+                                           int                      nnzA,
+                                           const int*               nnzPerRow,
+                                           cuComplex*               csrSortedValC,
+                                           int*                     csrSortedColIndC,
+                                           int*                     csrSortedRowPtrC,
+                                           cuComplex                tol) {
+  hipComplex blah;
+  blah.x=tol.x;
+  blah.y=tol.y;
+  return hipsparseCcsr2csr_compress(handle, m, n, descrA, reinterpret_cast<const hipComplex*>(csrSortedValA), csrSortedColIndA, csrSortedRowPtrA, nnzA, nnzPerRow, reinterpret_cast<hipComplex*>(csrSortedValC), csrSortedColIndC, csrSortedRowPtrC, blah);
+}
+
+cusparseStatus_t cusparseZcsr2csr_compress(cusparseHandle_t         handle,
+                                           int                      m,
+                                           int                      n,
+                                           const cusparseMatDescr_t descrA,
+                                           const cuDoubleComplex*   csrSortedValA,
+                                           const int*               csrSortedColIndA,
+                                           const int*               csrSortedRowPtrA,
+                                           int                      nnzA,
+                                           const int*               nnzPerRow,
+                                           cuDoubleComplex*         csrSortedValC,
+                                           int*                     csrSortedColIndC,
+                                           int*                     csrSortedRowPtrC,
+                                           cuDoubleComplex          tol) {
+  hipDoubleComplex blah;
+  blah.x=tol.x;
+  blah.y=tol.y;
+  return hipsparseZcsr2csr_compress(handle, m, n, descrA, reinterpret_cast<const hipDoubleComplex*>(csrSortedValA), csrSortedColIndA, csrSortedRowPtrA, nnzA, nnzPerRow, reinterpret_cast<hipDoubleComplex*>(csrSortedValC), csrSortedColIndC, csrSortedRowPtrC, blah);
+}
+
+}  // extern "C"
+
+
+#endif  // INCLUDE_GUARD_HIP_CUPY_HIPSPARSE_H
diff --git a/cupy_backends/hip/cupy_profiler.h b/cupy_backends/hip/cupy_profiler.h
new file mode 100644
index 0000000..cf48da8
--- /dev/null
+++ b/cupy_backends/hip/cupy_profiler.h
@@ -0,0 +1,24 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_PROFILER_H
+#define INCLUDE_GUARD_HIP_CUPY_PROFILER_H
+
+#include "cupy_hip_common.h"
+
+extern "C" {
+
+typedef enum {} cudaOutputMode_t;
+
+cudaError_t cudaProfilerInitialize(...) {
+  return cudaSuccess;
+}
+
+cudaError_t cudaProfilerStart() {
+  return hipProfilerStart();
+}
+
+cudaError_t cudaProfilerStop() {
+  return hipProfilerStop();
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_PROFILER_H
diff --git a/cupy_backends/hip/cupy_rccl.h b/cupy_backends/hip/cupy_rccl.h
new file mode 100644
index 0000000..94dde35
--- /dev/null
+++ b/cupy_backends/hip/cupy_rccl.h
@@ -0,0 +1,7 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_RCCL_H
+#define INCLUDE_GUARD_HIP_CUPY_RCCL_H
+
+#include <rccl.h>
+typedef hipStream_t cudaStream_t;
+
+#endif
diff --git a/cupy_backends/hip/cupy_rocsolver.h b/cupy_backends/hip/cupy_rocsolver.h
new file mode 100644
index 0000000..77039e9
--- /dev/null
+++ b/cupy_backends/hip/cupy_rocsolver.h
@@ -0,0 +1,2167 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_ROCSOLVER_H
+#define INCLUDE_GUARD_HIP_CUPY_ROCSOLVER_H
+
+#include "cupy_hip.h"
+#include "cupy_hipblas.h"
+#include <stdexcept>  // for gcc 10.0
+
+
+extern "C" {
+// TODO(leofang): perhaps these should be merged with the support of hipBLAS?
+static rocblas_fill convert_rocblas_fill(cublasFillMode_t mode) {
+    switch(static_cast<int>(mode)) {
+        case 0 /* CUBLAS_FILL_MODE_LOWER */: return rocblas_fill_lower;
+        case 1 /* CUBLAS_FILL_MODE_UPPER */: return rocblas_fill_upper;
+        default: throw std::runtime_error("unrecognized mode");
+    }
+}
+
+static rocblas_operation convert_rocblas_operation(cublasOperation_t op) {
+    return static_cast<rocblas_operation>(static_cast<int>(op) + 111);
+}
+
+static rocblas_side convert_rocblas_side(cublasSideMode_t mode) {
+    return static_cast<rocblas_side>(static_cast<int>(mode) + 141);
+}
+
+#if HIP_VERSION >= 309
+static rocblas_svect convert_rocblas_svect(signed char mode) {
+    switch(mode) {
+        case 'A': return rocblas_svect_all;
+        case 'S': return rocblas_svect_singular;
+        case 'O': return rocblas_svect_overwrite;
+        case 'N': return rocblas_svect_none;
+        default: throw std::runtime_error("unrecognized mode");
+    }
+}
+#endif
+
+
+// rocSOLVER
+/* ---------- helpers ---------- */
+
+cusolverStatus_t cusolverDnCreate(cusolverDnHandle_t *handle) {
+    return rocblas_create_handle(handle);
+}
+
+cusolverStatus_t cusolverDnDestroy(cusolverDnHandle_t handle) {
+    return rocblas_destroy_handle(handle);
+}
+
+cusolverStatus_t cusolverDnGetStream(cusolverDnHandle_t handle,
+                                     cudaStream_t *streamId) {
+    return rocblas_get_stream(handle, streamId);
+}
+
+cusolverStatus_t cusolverDnSetStream (cusolverDnHandle_t handle,
+                                      cudaStream_t streamId) {
+    return rocblas_set_stream(handle, streamId);
+}
+
+cusolverStatus_t cusolverGetProperty(libraryPropertyType type, int* val) {
+    switch(type) {
+        case MAJOR_VERSION: { *val = ROCSOLVER_VERSION_MAJOR; break; }
+        case MINOR_VERSION: { *val = ROCSOLVER_VERSION_MINOR; break; }
+        case PATCH_LEVEL:   { *val = ROCSOLVER_VERSION_PATCH; break; }
+        default: throw std::runtime_error("invalid type");
+    }
+    return rocblas_status_success;
+}
+
+
+typedef enum cusolverDnParams_t {};
+
+cusolverStatus_t cusolverDnCreateParams(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDestroyParams(...) {
+    return rocblas_status_not_implemented;
+}
+
+/* ---------- potrf ---------- */
+cusolverStatus_t cusolverDnSpotrf_bufferSize(cusolverDnHandle_t handle,
+                                             cublasFillMode_t uplo,
+                                             int n,
+                                             float *A,
+                                             int lda,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDpotrf_bufferSize(cusolverDnHandle_t handle,
+                                             cublasFillMode_t uplo,
+                                             int n,
+                                             double *A,
+                                             int lda,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCpotrf_bufferSize(cusolverDnHandle_t handle,
+                                             cublasFillMode_t uplo,
+                                             int n,
+                                             cuComplex *A,
+                                             int lda,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZpotrf_bufferSize(cusolverDnHandle_t handle,
+                                             cublasFillMode_t uplo,
+                                             int n,
+                                             cuDoubleComplex *A,
+                                             int lda,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSpotrf(cusolverDnHandle_t handle,
+                                  cublasFillMode_t uplo,
+                                  int n,
+                                  float *A,
+                                  int lda,
+                                  float *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    // ignore Workspace and Lwork as rocSOLVER does not need them
+    return rocsolver_spotrf(handle, convert_rocblas_fill(uplo),
+                            n, A, lda, devInfo);
+}
+
+cusolverStatus_t cusolverDnDpotrf(cusolverDnHandle_t handle,
+                                  cublasFillMode_t uplo,
+                                  int n,
+                                  double *A,
+                                  int lda,
+                                  double *Workspace,
+                                  int Lwork,
+                                  int *devInfo ) {
+    // ignore Workspace and Lwork as rocSOLVER does not need them
+    return rocsolver_dpotrf(handle, convert_rocblas_fill(uplo),
+                            n, A, lda, devInfo);
+}
+
+cusolverStatus_t cusolverDnCpotrf(cusolverDnHandle_t handle,
+                                  cublasFillMode_t uplo,
+                                  int n,
+                                  cuComplex *A,
+                                  int lda,
+                                  cuComplex *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore Workspace and Lwork as rocSOLVER does not need them
+    return rocsolver_cpotrf(handle, convert_rocblas_fill(uplo), n,
+                            reinterpret_cast<rocblas_float_complex*>(A), lda, devInfo);
+    #endif
+}
+
+cusolverStatus_t cusolverDnZpotrf(cusolverDnHandle_t handle,
+                                  cublasFillMode_t uplo,
+                                  int n,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  cuDoubleComplex *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore Workspace and Lwork as rocSOLVER does not need them
+    return rocsolver_zpotrf(handle, convert_rocblas_fill(uplo), n,
+                            reinterpret_cast<rocblas_double_complex*>(A), lda, devInfo);
+    #endif
+}
+
+cusolverStatus_t cusolverDnSpotrfBatched(cusolverDnHandle_t handle,
+                                         cublasFillMode_t uplo,
+                                         int n,
+                                         float *Aarray[],
+                                         int lda,
+                                         int *infoArray,
+                                         int batchSize) {
+    return rocsolver_spotrf_batched(handle, convert_rocblas_fill(uplo),
+                                    n, Aarray, lda, infoArray, batchSize);
+}
+
+cusolverStatus_t cusolverDnDpotrfBatched(cusolverDnHandle_t handle,
+                                         cublasFillMode_t uplo,
+                                         int n,
+                                         double *Aarray[],
+                                         int lda,
+                                         int *infoArray,
+                                         int batchSize) {
+    return rocsolver_dpotrf_batched(handle, convert_rocblas_fill(uplo),
+                                    n, Aarray, lda, infoArray, batchSize);
+}
+
+cusolverStatus_t cusolverDnCpotrfBatched(cusolverDnHandle_t handle,
+                                         cublasFillMode_t uplo,
+                                         int n,
+                                         cuComplex *Aarray[],
+                                         int lda,
+                                         int *infoArray,
+                                         int batchSize) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_cpotrf_batched(handle, convert_rocblas_fill(uplo), n,
+                                    reinterpret_cast<rocblas_float_complex* const*>(Aarray), lda,
+                                    infoArray, batchSize);
+    #endif
+}
+
+cusolverStatus_t cusolverDnZpotrfBatched(cusolverDnHandle_t handle,
+                                         cublasFillMode_t uplo,
+                                         int n,
+                                         cuDoubleComplex *Aarray[],
+                                         int lda,
+                                         int *infoArray,
+                                         int batchSize) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_zpotrf_batched(handle, convert_rocblas_fill(uplo), n,
+                                    reinterpret_cast<rocblas_double_complex* const*>(Aarray), lda,
+                                    infoArray, batchSize);
+    #endif
+}
+
+
+/* ---------- getrf ---------- */
+cusolverStatus_t cusolverDnSgetrf_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             float *A,
+                                             int lda,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDgetrf_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             double *A,
+                                             int lda,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCgetrf_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             cuComplex *A,
+                                             int lda,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZgetrf_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             cuDoubleComplex *A,
+                                             int lda,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSgetrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  float *A,
+                                  int lda,
+                                  float *Workspace,
+                                  int *devIpiv,
+                                  int *devInfo) {
+    // ignore Workspace as rocSOLVER does not need it
+    return rocsolver_sgetrf(handle, m, n, A, lda, devIpiv, devInfo);
+}
+
+cusolverStatus_t cusolverDnDgetrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  double *A,
+                                  int lda,
+                                  double *Workspace,
+                                  int *devIpiv,
+                                  int *devInfo) {
+    // ignore Workspace as rocSOLVER does not need it
+    return rocsolver_dgetrf(handle, m, n, A, lda, devIpiv, devInfo);
+}
+
+cusolverStatus_t cusolverDnCgetrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  cuComplex *A,
+                                  int lda,
+                                  cuComplex *Workspace,
+                                  int *devIpiv,
+                                  int *devInfo) {
+    // ignore Workspace as rocSOLVER does not need it
+    return rocsolver_cgetrf(handle, m, n,
+                            reinterpret_cast<rocblas_float_complex*>(A), lda,
+                            devIpiv, devInfo);
+}
+
+cusolverStatus_t cusolverDnZgetrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  cuDoubleComplex *Workspace,
+                                  int *devIpiv,
+                                  int *devInfo) {
+    // ignore Workspace as rocSOLVER does not need it
+    return rocsolver_zgetrf(handle, m, n,
+                            reinterpret_cast<rocblas_double_complex*>(A), lda,
+                            devIpiv, devInfo);
+}
+
+
+/* ---------- getrs ---------- */
+cusolverStatus_t cusolverDnSgetrs(cusolverDnHandle_t handle,
+                                  cublasOperation_t trans,
+                                  int n,
+                                  int nrhs,
+                                  const float *A,
+                                  int lda,
+                                  const int *devIpiv,
+                                  float *B,
+                                  int ldb,
+                                  int *devInfo) {
+    // ignore devInfo as rocSOLVER does not need it
+    return rocsolver_sgetrs(handle,
+                            convert_rocblas_operation(trans),
+                            n, nrhs, const_cast<float*>(A), lda, devIpiv, B, ldb);
+}
+
+cusolverStatus_t cusolverDnDgetrs(cusolverDnHandle_t handle,
+                                  cublasOperation_t trans,
+                                  int n,
+                                  int nrhs,
+                                  const double *A,
+                                  int lda,
+                                  const int *devIpiv,
+                                  double *B,
+                                  int ldb,
+                                  int *devInfo) {
+    // ignore devInfo as rocSOLVER does not need it
+    return rocsolver_dgetrs(handle,
+                            convert_rocblas_operation(trans),
+                            n, nrhs, const_cast<double*>(A), lda, devIpiv, B, ldb);
+}
+
+cusolverStatus_t cusolverDnCgetrs(cusolverDnHandle_t handle,
+                                  cublasOperation_t trans,
+                                  int n,
+                                  int nrhs,
+                                  const cuComplex *A,
+                                  int lda,
+                                  const int *devIpiv,
+                                  cuComplex *B,
+                                  int ldb,
+                                  int *devInfo) {
+    // ignore devInfo as rocSOLVER does not need it
+    return rocsolver_cgetrs(handle,
+                            convert_rocblas_operation(trans),
+                            n, nrhs,
+                            (rocblas_float_complex*)(A), lda,
+                            devIpiv,
+                            reinterpret_cast<rocblas_float_complex*>(B), ldb);
+}
+
+cusolverStatus_t cusolverDnZgetrs(cusolverDnHandle_t handle,
+                                  cublasOperation_t trans,
+                                  int n,
+                                  int nrhs,
+                                  const cuDoubleComplex *A,
+                                  int lda,
+                                  const int *devIpiv,
+                                  cuDoubleComplex *B,
+                                  int ldb,
+                                  int *devInfo) {
+    // ignore devInfo as rocSOLVER does not need it
+    return rocsolver_zgetrs(handle,
+                            convert_rocblas_operation(trans),
+                            n, nrhs,
+                            (rocblas_double_complex*)(A), lda,
+                            devIpiv,
+                            reinterpret_cast<rocblas_double_complex*>(B), ldb);
+}
+
+
+/* ---------- geqrf ---------- */
+cusolverStatus_t cusolverDnSgeqrf_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             float *A,
+                                             int lda,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDgeqrf_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             double *A,
+                                             int lda,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCgeqrf_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             cuComplex *A,
+                                             int lda,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZgeqrf_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             cuDoubleComplex *A,
+                                             int lda,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSgeqrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  float *A,
+                                  int lda,
+                                  float *TAU,
+                                  float *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    // ignore Workspace, Lwork and devInfo as rocSOLVER does not need them
+    return rocsolver_sgeqrf(handle, m, n, A, lda, TAU);
+}
+
+cusolverStatus_t cusolverDnDgeqrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  double *A,
+                                  int lda,
+                                  double *TAU,
+                                  double *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    // ignore Workspace, Lwork and devInfo as rocSOLVER does not need them
+    return rocsolver_dgeqrf(handle, m, n, A, lda, TAU);
+}
+
+cusolverStatus_t cusolverDnCgeqrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  cuComplex *A,
+                                  int lda,
+                                  cuComplex *TAU,
+                                  cuComplex *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    // ignore Workspace, Lwork and devInfo as rocSOLVER does not need them
+    return rocsolver_cgeqrf(handle, m, n,
+                            reinterpret_cast<rocblas_float_complex*>(A), lda,
+                            reinterpret_cast<rocblas_float_complex*>(TAU));
+}
+
+cusolverStatus_t cusolverDnZgeqrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  cuDoubleComplex *TAU,
+                                  cuDoubleComplex *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    // ignore Workspace, Lwork and devInfo as rocSOLVER does not need them
+    return rocsolver_zgeqrf(handle, m, n,
+                            reinterpret_cast<rocblas_double_complex*>(A), lda,
+                            reinterpret_cast<rocblas_double_complex*>(TAU));
+}
+
+
+/* ---------- orgqr ---------- */
+cusolverStatus_t cusolverDnSorgqr_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int k,
+                                             const float *A,
+                                             int lda,
+                                             const float *tau,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDorgqr_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int k,
+                                             const double *A,
+                                             int lda,
+                                             const double *tau,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSorgqr(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  float *A,
+                                  int lda,
+                                  const float *tau,
+                                  float *work,
+                                  int lwork,
+                                  int *info) {
+    // ignore work, lwork and info as rocSOLVER does not need them
+    return rocsolver_sorgqr(handle, m, n, k, A, lda, const_cast<float*>(tau));
+}
+
+cusolverStatus_t cusolverDnDorgqr(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  double *A,
+                                  int lda,
+                                  const double *tau,
+                                  double *work,
+                                  int lwork,
+                                  int *info) {
+    // ignore work, lwork and info as rocSOLVER does not need them
+    return rocsolver_dorgqr(handle, m, n, k, A, lda, const_cast<double*>(tau));
+}
+
+
+/* ---------- ungqr ---------- */
+cusolverStatus_t cusolverDnCungqr_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int k,
+                                             const cuComplex *A,
+                                             int lda,
+                                             const cuComplex *tau,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZungqr_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int k,
+                                             const cuDoubleComplex *A,
+                                             int lda,
+                                             const cuDoubleComplex *tau,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCungqr(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  cuComplex *A,
+                                  int lda,
+                                  const cuComplex *tau,
+                                  cuComplex *work,
+                                  int lwork,
+                                  int *info) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work, lwork and info as rocSOLVER does not need them
+    return rocsolver_cungqr(handle, m, n, k,
+                            reinterpret_cast<rocblas_float_complex*>(A), lda,
+                            reinterpret_cast<rocblas_float_complex*>(const_cast<cuComplex*>(tau)));
+    #endif
+}
+
+cusolverStatus_t cusolverDnZungqr(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  const cuDoubleComplex *tau,
+                                  cuDoubleComplex *work,
+                                  int lwork,
+                                  int *info) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work, lwork and info as rocSOLVER does not need them
+    return rocsolver_zungqr(handle, m, n, k,
+                            reinterpret_cast<rocblas_double_complex*>(A), lda,
+                            reinterpret_cast<rocblas_double_complex*>(const_cast<cuDoubleComplex*>(tau)));
+    #endif
+}
+
+
+/* ---------- ormqr ---------- */
+cusolverStatus_t cusolverDnSormqr_bufferSize(cusolverDnHandle_t handle,
+                                             cublasSideMode_t side,
+                                             cublasOperation_t trans,
+                                             int m,
+                                             int n,
+                                             int k,
+                                             const float *A,
+                                             int lda,
+                                             const float *tau,
+                                             const float *C,
+                                             int ldc,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDormqr_bufferSize(cusolverDnHandle_t handle,
+                                             cublasSideMode_t side,
+                                             cublasOperation_t trans,
+                                             int m,
+                                             int n,
+                                             int k,
+                                             const double *A,
+                                             int lda,
+                                             const double *tau,
+                                             const double *C,
+                                             int ldc,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSormqr(cusolverDnHandle_t handle,
+                                  cublasSideMode_t side,
+                                  cublasOperation_t trans,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  const float *A,
+                                  int lda,
+                                  const float *tau,
+                                  float *C,
+                                  int ldc,
+                                  float *work,
+                                  int lwork,
+                                  int *devInfo) {
+    // ignore work, lwork and devInfo as rocSOLVER does not need them
+    return rocsolver_sormqr(handle,
+                            convert_rocblas_side(side),
+                            convert_rocblas_operation(trans),
+                            m, n, k,
+                            const_cast<float*>(A), lda,
+                            const_cast<float*>(tau),
+                            C, ldc);
+}
+
+cusolverStatus_t cusolverDnDormqr(cusolverDnHandle_t handle,
+                                  cublasSideMode_t side,
+                                  cublasOperation_t trans,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  const double *A,
+                                  int lda,
+                                  const double *tau,
+                                  double *C,
+                                  int ldc,
+                                  double *work,
+                                  int lwork,
+                                  int *devInfo) {
+    // ignore work, lwork and devInfo as rocSOLVER does not need them
+    return rocsolver_dormqr(handle,
+                            convert_rocblas_side(side),
+                            convert_rocblas_operation(trans),
+                            m, n, k,
+                            const_cast<double*>(A), lda,
+                            const_cast<double*>(tau),
+                            C, ldc);
+}
+
+
+/* ---------- unmqr ---------- */
+cusolverStatus_t cusolverDnCunmqr_bufferSize(cusolverDnHandle_t handle,
+                                             cublasSideMode_t side,
+                                             cublasOperation_t trans,
+                                             int m,
+                                             int n,
+                                             int k,
+                                             const cuComplex *A,
+                                             int lda,
+                                             const cuComplex *tau,
+                                             const cuComplex *C,
+                                             int ldc,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZunmqr_bufferSize(cusolverDnHandle_t handle,
+                                             cublasSideMode_t side,
+                                             cublasOperation_t trans,
+                                             int m,
+                                             int n,
+                                             int k,
+                                             const cuDoubleComplex *A,
+                                             int lda,
+                                             const cuDoubleComplex *tau,
+                                             const cuDoubleComplex *C,
+                                             int ldc,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCunmqr(cusolverDnHandle_t handle,
+                                  cublasSideMode_t side,
+                                  cublasOperation_t trans,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  const cuComplex *A,
+                                  int lda,
+                                  const cuComplex *tau,
+                                  cuComplex *C,
+                                  int ldc,
+                                  cuComplex *work,
+                                  int lwork,
+                                  int *devInfo) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work, lwork and devInfo as rocSOLVER does not need them
+    return rocsolver_cunmqr(handle, convert_rocblas_side(side), convert_rocblas_operation(trans),
+                            m, n, k, reinterpret_cast<rocblas_float_complex*>(const_cast<cuComplex*>(A)),
+                            lda, reinterpret_cast<rocblas_float_complex*>(const_cast<cuComplex*>(tau)),
+                            reinterpret_cast<rocblas_float_complex*>(C), ldc);
+    #endif
+}
+
+cusolverStatus_t cusolverDnZunmqr(cusolverDnHandle_t handle,
+                                  cublasSideMode_t side,
+                                  cublasOperation_t trans,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  const cuDoubleComplex *A,
+                                  int lda,
+                                  const cuDoubleComplex *tau,
+                                  cuDoubleComplex *C,
+                                  int ldc,
+                                  cuDoubleComplex *work,
+                                  int lwork,
+                                  int *devInfo) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work, lwork and devInfo as rocSOLVER does not need them
+    return rocsolver_zunmqr(handle, convert_rocblas_side(side), convert_rocblas_operation(trans),
+                            m, n, k, reinterpret_cast<rocblas_double_complex*>(const_cast<cuDoubleComplex*>(A)),
+                            lda, reinterpret_cast<rocblas_double_complex*>(const_cast<cuDoubleComplex*>(tau)),
+                            reinterpret_cast<rocblas_double_complex*>(C), ldc);
+    #endif
+}
+
+
+/* ---------- gesvd ---------- */
+cusolverStatus_t cusolverDnSgesvd_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDgesvd_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCgesvd_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZgesvd_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int *lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSgesvd(cusolverDnHandle_t handle,
+                                  signed char jobu,
+                                  signed char jobvt,
+                                  int m,
+                                  int n,
+                                  float *A,
+                                  int lda,
+                                  float *S,
+                                  float *U,
+                                  int ldu,
+                                  float *VT,
+                                  int ldvt,
+                                  float *work,
+                                  int lwork,
+                                  float *rwork,
+                                  int *info) {
+    #if HIP_VERSION < 309
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work and lwork as rocSOLVER does not need them
+    return rocsolver_sgesvd(handle, convert_rocblas_svect(jobu), convert_rocblas_svect(jobvt),
+                            m, n, A, lda, S, U, ldu, VT, ldvt, rwork, rocblas_outofplace,  // always out-of-place
+                            info);
+    #endif
+}
+
+cusolverStatus_t cusolverDnDgesvd(cusolverDnHandle_t handle,
+                                  signed char jobu,
+                                  signed char jobvt,
+                                  int m,
+                                  int n,
+                                  double *A,
+                                  int lda,
+                                  double *S,
+                                  double *U,
+                                  int ldu,
+                                  double *VT,
+                                  int ldvt,
+                                  double *work,
+                                  int lwork,
+                                  double *rwork,
+                                  int *info) {
+    #if HIP_VERSION < 309
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work and lwork as rocSOLVER does not need them
+    return rocsolver_dgesvd(handle, convert_rocblas_svect(jobu), convert_rocblas_svect(jobvt),
+                            m, n, A, lda, S, U, ldu, VT, ldvt, rwork, rocblas_outofplace,  // always out-of-place
+                            info);
+    #endif
+}
+
+cusolverStatus_t cusolverDnCgesvd(cusolverDnHandle_t handle,
+                                  signed char jobu,
+                                  signed char jobvt,
+                                  int m,
+                                  int n,
+                                  cuComplex *A,
+                                  int lda,
+                                  float *S,
+                                  cuComplex *U,
+                                  int ldu,
+                                  cuComplex *VT,
+                                  int ldvt,
+                                  cuComplex *work,
+                                  int lwork,
+                                  float *rwork,
+                                  int *info) {
+    #if HIP_VERSION < 309
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work and lwork as rocSOLVER does not need them
+    return rocsolver_cgesvd(handle, convert_rocblas_svect(jobu), convert_rocblas_svect(jobvt),
+                            m, n, reinterpret_cast<rocblas_float_complex*>(A), lda,
+                            S, reinterpret_cast<rocblas_float_complex*>(U), ldu,
+                            reinterpret_cast<rocblas_float_complex*>(VT), ldvt, rwork,
+                            rocblas_outofplace,  // always out-of-place
+                            info);
+    #endif
+}
+
+cusolverStatus_t cusolverDnZgesvd(cusolverDnHandle_t handle,
+                                  signed char jobu,
+                                  signed char jobvt,
+                                  int m,
+                                  int n,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  double *S,
+                                  cuDoubleComplex *U,
+                                  int ldu,
+                                  cuDoubleComplex *VT,
+                                  int ldvt,
+                                  cuDoubleComplex *work,
+                                  int lwork,
+                                  double *rwork,
+                                  int *info) {
+    #if HIP_VERSION < 309
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work and lwork as rocSOLVER does not need them
+    return rocsolver_zgesvd(handle, convert_rocblas_svect(jobu), convert_rocblas_svect(jobvt),
+                            m, n, reinterpret_cast<rocblas_double_complex*>(A), lda,
+                            S, reinterpret_cast<rocblas_double_complex*>(U), ldu,
+                            reinterpret_cast<rocblas_double_complex*>(VT), ldvt, rwork,
+                            rocblas_outofplace,  // always out-of-place
+                            info);
+    #endif
+}
+
+
+/* ---------- batched gesvd ---------- */
+// Because rocSOLVER provides no counterpart for gesvdjBatched, we wrap its batched version directly.
+typedef enum {
+    CUSOLVER_EIG_MODE_NOVECTOR=0,
+    CUSOLVER_EIG_MODE_VECTOR=1
+} cusolverEigMode_t;
+typedef void* gesvdjInfo_t;
+
+cusolverStatus_t cusolverDnCreateGesvdjInfo(...) {
+    // should always success as rocSOLVER does not need it
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDestroyGesvdjInfo(...) {
+    // should always success as rocSOLVER does not need it
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSgesvdjBatched_bufferSize(
+        cusolverDnHandle_t handle,
+        cusolverEigMode_t jobz,
+        int m,                
+        int n,                
+        const float *A,    
+        int lda,           
+        const float *S, 
+        const float *U,   
+        int ldu, 
+        const float *V,
+        int ldv,  
+        int *lwork,
+        gesvdjInfo_t params,
+        int batchSize) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the bidiagonal matrix B associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = batchSize * (m<n?m:n);  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDgesvdjBatched_bufferSize(
+        cusolverDnHandle_t handle,
+        cusolverEigMode_t jobz,
+        int m,
+        int n,
+        const double *A, 
+        int lda,
+        const double *S,
+        const double *U,
+        int ldu,
+        const double *V,
+        int ldv,
+        int *lwork,
+        gesvdjInfo_t params,
+        int batchSize) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the bidiagonal matrix B associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = batchSize * (m<n?m:n);  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCgesvdjBatched_bufferSize(
+        cusolverDnHandle_t handle,
+        cusolverEigMode_t jobz,
+        int m,
+        int n,
+        const cuComplex *A,
+        int lda,
+        const float *S,
+        const cuComplex *U,
+        int ldu,
+        const cuComplex *V,
+        int ldv,
+        int *lwork,
+        gesvdjInfo_t params,
+        int batchSize) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the bidiagonal matrix B associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = batchSize * (m<n?m:n);  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZgesvdjBatched_bufferSize(
+        cusolverDnHandle_t handle,
+        cusolverEigMode_t jobz, 
+        int m, 
+        int n, 
+        const cuDoubleComplex *A,
+        int lda,
+        const double *S,
+        const cuDoubleComplex *U,
+        int ldu, 
+        const cuDoubleComplex *V,
+        int ldv,
+        int *lwork,
+        gesvdjInfo_t params,
+        int batchSize) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the bidiagonal matrix B associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = batchSize * (m<n?m:n);  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSgesvdjBatched(
+        cusolverDnHandle_t handle,
+        cusolverEigMode_t jobz, 
+        int m, 
+        int n, 
+        float *A, 
+        int lda, 
+        float *S, 
+        float *U,
+        int ldu,
+        float *V,
+        int ldv, 
+        float *work,
+        int lwork,
+        int *info,
+        gesvdjInfo_t params,
+        int batchSize) {
+    #if HIP_VERSION < 309
+    return rocblas_status_not_implemented;
+    #else
+    rocblas_svect leftv, rightv;
+    rocblas_stride stU, stV;
+    if (jobz == CUSOLVER_EIG_MODE_NOVECTOR) {
+        leftv = rocblas_svect_none;
+        rightv = rocblas_svect_none;
+        stU = ldu * (m<n?m:n);
+        stV = ldv * n;
+    } else {  // CUSOLVER_EIG_MODE_VECTOR
+        leftv = rocblas_svect_all;
+        rightv = rocblas_svect_all;
+        stU = ldu * m;
+        stV = ldv * n;
+    }
+    return rocsolver_sgesvd_batched(handle, leftv, rightv,
+                                    m, n, reinterpret_cast<float* const*>(A), lda,
+                                    S, m<n?m:n,
+                                    U, ldu, stU,
+                                    V, ldv, stV,
+                                    // since we can't pass in another array through the API, and work is unused,
+                                    // we use it to store the temporary E array, to be discarded after calculation
+                                    work, (m<n?m:n)-1,
+                                    rocblas_outofplace, // always out-of-place
+                                    info, batchSize);
+    #endif
+}
+
+cusolverStatus_t cusolverDnDgesvdjBatched(
+        cusolverDnHandle_t handle,
+        cusolverEigMode_t jobz,
+        int m,
+        int n,
+        double *A,
+        int lda,
+        double *S,
+        double *U,
+        int ldu,
+        double *V,
+        int ldv,
+        double *work,
+        int lwork,
+        int *info,
+        gesvdjInfo_t params,
+        int batchSize) { 
+    #if HIP_VERSION < 309
+    return rocblas_status_not_implemented;
+    #else
+    rocblas_svect leftv, rightv;
+    rocblas_stride stU, stV;
+    if (jobz == CUSOLVER_EIG_MODE_NOVECTOR) {
+        leftv = rocblas_svect_none;
+        rightv = rocblas_svect_none;
+        stU = ldu * (m<n?m:n);
+        stV = ldv * n;
+    } else {  // CUSOLVER_EIG_MODE_VECTOR
+        leftv = rocblas_svect_all;
+        rightv = rocblas_svect_all;
+        stU = ldu * m;
+        stV = ldv * n;
+    }
+    return rocsolver_dgesvd_batched(handle, leftv, rightv,
+                                    m, n, reinterpret_cast<double* const*>(A), lda,
+                                    S, m<n?m:n,
+                                    U, ldu, stU,
+                                    V, ldv, stV,
+                                    // since we can't pass in another array through the API, and work is unused,
+                                    // we use it to store the temporary E array, to be discarded after calculation
+                                    work, (m<n?m:n)-1,
+                                    rocblas_outofplace, // always out-of-place
+                                    info, batchSize);
+    #endif
+}
+
+cusolverStatus_t cusolverDnCgesvdjBatched(
+        cusolverDnHandle_t handle,
+        cusolverEigMode_t jobz,
+        int m,
+        int n,
+        cuComplex *A,
+        int lda,
+        float *S,
+        cuComplex *U,
+        int ldu,
+        cuComplex *V,
+        int ldv,
+        cuComplex *work,
+        int lwork,
+        int *info,
+        gesvdjInfo_t params,
+        int batchSize) {
+    #if HIP_VERSION < 309
+    return rocblas_status_not_implemented;
+    #else
+    rocblas_svect leftv, rightv;
+    rocblas_stride stU, stV;
+    if (jobz == CUSOLVER_EIG_MODE_NOVECTOR) {
+        leftv = rocblas_svect_none;
+        rightv = rocblas_svect_none;
+        stU = ldu * (m<n?m:n);
+        stV = ldv * n;
+    } else {  // CUSOLVER_EIG_MODE_VECTOR
+        leftv = rocblas_svect_all;
+        rightv = rocblas_svect_all;
+        stU = ldu * m;
+        stV = ldv * n;
+    }
+    return rocsolver_cgesvd_batched(handle, leftv, rightv,
+                                    m, n, reinterpret_cast<rocblas_float_complex* const*>(A), lda,
+                                    S, m<n?m:n,
+                                    reinterpret_cast<rocblas_float_complex*>(U), ldu, stU,
+                                    reinterpret_cast<rocblas_float_complex*>(V), ldv, stV,
+                                    // since we can't pass in another array through the API, and work is unused,
+                                    // we use it to store the temporary E array, to be discarded after calculation
+                                    reinterpret_cast<float*>(work), (m<n?m:n)-1,
+                                    rocblas_outofplace, // always out-of-place
+                                    info, batchSize);
+    #endif
+}
+
+cusolverStatus_t cusolverDnZgesvdjBatched(
+        cusolverDnHandle_t handle,
+        cusolverEigMode_t jobz,
+        int m,
+        int n,
+        cuDoubleComplex *A,
+        int lda,
+        double *S,
+        cuDoubleComplex *U,
+        int ldu,
+        cuDoubleComplex *V,
+        int ldv,
+        cuDoubleComplex *work,
+        int lwork,
+        int *info,
+        gesvdjInfo_t params,
+        int batchSize) {
+    #if HIP_VERSION < 309
+    return rocblas_status_not_implemented;
+    #else
+    rocblas_svect leftv, rightv;
+    rocblas_stride stU, stV;
+    if (jobz == CUSOLVER_EIG_MODE_NOVECTOR) {
+        leftv = rocblas_svect_none;
+        rightv = rocblas_svect_none;
+        stU = ldu * (m<n?m:n);
+        stV = ldv * n;
+    } else {  // CUSOLVER_EIG_MODE_VECTOR
+        leftv = rocblas_svect_all;
+        rightv = rocblas_svect_all;
+        stU = ldu * m;
+        stV = ldv * n;
+    }
+    return rocsolver_zgesvd_batched(handle, leftv, rightv,
+                                    m, n, reinterpret_cast<rocblas_double_complex* const*>(A), lda,
+                                    S, m<n?m:n,
+                                    reinterpret_cast<rocblas_double_complex*>(U), ldu, stU,
+                                    reinterpret_cast<rocblas_double_complex*>(V), ldv, stV,
+                                    // since we can't pass in another array through the API, and work is unused,
+                                    // we use it to store the temporary E array, to be discarded after calculation
+                                    reinterpret_cast<double*>(work), (m<n?m:n)-1,
+                                    rocblas_outofplace, // always out-of-place
+                                    info, batchSize);
+    #endif
+}
+
+
+/* ---------- gebrd ---------- */
+cusolverStatus_t cusolverDnSgebrd_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDgebrd_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCgebrd_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZgebrd_bufferSize(cusolverDnHandle_t handle,
+                                             int m,
+                                             int n,
+                                             int *Lwork) {
+    // this needs to return 0 because rocSolver does not rely on it
+    *Lwork = 0;
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSgebrd(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  float *A,
+                                  int lda,
+                                  float *D,
+                                  float *E,
+                                  float *TAUQ,
+                                  float *TAUP,
+                                  float *Work,
+                                  int Lwork,
+                                  int *devInfo) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work, lwork and devinfo as rocSOLVER does not need them
+    return rocsolver_sgebrd(handle, m, n, A, lda, D, E, TAUQ, TAUP);
+    #endif
+}
+
+cusolverStatus_t cusolverDnDgebrd(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  double *A,
+                                  int lda,
+                                  double *D,
+                                  double *E,
+                                  double *TAUQ,
+                                  double *TAUP,
+                                  double *Work,
+                                  int Lwork,
+                                  int *devInfo) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work, lwork and devinfo as rocSOLVER does not need them
+    return rocsolver_dgebrd(handle, m, n, A, lda, D, E, TAUQ, TAUP);
+    #endif
+}
+
+cusolverStatus_t cusolverDnCgebrd(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  cuComplex *A,
+                                  int lda,
+                                  float *D,
+                                  float *E,
+                                  cuComplex *TAUQ,
+                                  cuComplex *TAUP,
+                                  cuComplex *Work,
+                                  int Lwork,
+                                  int *devInfo) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work, lwork and devinfo as rocSOLVER does not need them
+    return rocsolver_cgebrd(handle, m, n, reinterpret_cast<rocblas_float_complex*>(A),
+                            lda, D, E, reinterpret_cast<rocblas_float_complex*>(TAUQ),
+                            reinterpret_cast<rocblas_float_complex*>(TAUP));
+    #endif
+}
+
+cusolverStatus_t cusolverDnZgebrd(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  double *D,
+                                  double *E,
+                                  cuDoubleComplex *TAUQ,
+                                  cuDoubleComplex *TAUP,
+                                  cuDoubleComplex *Work,
+                                  int Lwork,
+                                  int *devInfo) {
+    #if HIP_VERSION < 306
+    return rocblas_status_not_implemented;
+    #else
+    // ignore work, lwork and devinfo as rocSOLVER does not need them
+    return rocsolver_zgebrd(handle, m, n, reinterpret_cast<rocblas_double_complex*>(A),
+                            lda, D, E, reinterpret_cast<rocblas_double_complex*>(TAUQ),
+                            reinterpret_cast<rocblas_double_complex*>(TAUP));
+    #endif
+}
+
+
+/* ---------- syevj ---------- */
+typedef void* syevjInfo_t;
+
+#if HIP_VERSION >= 402
+static rocblas_evect convert_rocblas_evect(cusolverEigMode_t mode) {
+    switch(mode) {
+        // as of ROCm 4.2.0 rocblas_evect_tridiagonal is not supported
+        case 0 /* CUSOLVER_EIG_MODE_NOVECTOR */: return rocblas_evect_none;
+        case 1 /* CUSOLVER_EIG_MODE_VECTOR */  : return rocblas_evect_original;
+        default: throw std::runtime_error("unrecognized mode");
+    }
+}
+#endif
+
+cusolverStatus_t cusolverDnCreateSyevjInfo(syevjInfo_t *info) {
+    // TODO(leofang): set info to NULL? We don't use it anyway...
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDestroySyevjInfo(syevjInfo_t info) {
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSsyevj_bufferSize(cusolverDnHandle_t handle,
+                                             cusolverEigMode_t jobz,
+                                             cublasFillMode_t uplo,
+                                             int n,
+                                             const float *A,
+                                             int lda,
+                                             const float *W,
+                                             int *lwork,
+                                             syevjInfo_t params) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the tridiagonal matrix T associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = n;  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDsyevj_bufferSize(cusolverDnHandle_t handle,
+                                             cusolverEigMode_t jobz,
+                                             cublasFillMode_t uplo,
+                                             int n,
+                                             const double *A,
+                                             int lda,
+                                             const double *W,
+                                             int *lwork,
+                                             syevjInfo_t params) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the tridiagonal matrix T associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = n;  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCheevj_bufferSize(cusolverDnHandle_t handle,
+                                             cusolverEigMode_t jobz,
+                                             cublasFillMode_t uplo,
+                                             int n,
+                                             const cuComplex *A,
+                                             int lda,
+                                             const float *W,
+                                             int *lwork,
+                                             syevjInfo_t params) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the tridiagonal matrix T associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = n;  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZheevj_bufferSize(cusolverDnHandle_t handle,
+                                             cusolverEigMode_t jobz,
+                                             cublasFillMode_t uplo,
+                                             int n,
+                                             const cuDoubleComplex *A,
+                                             int lda,
+                                             const double *W,
+                                             int *lwork,
+                                             syevjInfo_t params) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the tridiagonal matrix T associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = n;  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSsyevj(cusolverDnHandle_t handle,
+                                  cusolverEigMode_t jobz,
+                                  cublasFillMode_t uplo,
+                                  int n,
+                                  float *A,
+                                  int lda,
+                                  float *W,
+                                  float *work,
+                                  int lwork,
+                                  int *info,
+                                  syevjInfo_t params) {
+    #if HIP_VERSION < 402
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_ssyev(handle, convert_rocblas_evect(jobz), convert_rocblas_fill(uplo),
+                           n, A, lda, W,
+                           // since we can't pass in another array through the API, and work is unused,
+                           // we use it to store the temporary E array, to be discarded after calculation
+                           work,
+                           info);
+    #endif
+}
+
+cusolverStatus_t cusolverDnDsyevj(cusolverDnHandle_t handle,
+                                  cusolverEigMode_t jobz,
+                                  cublasFillMode_t uplo,
+                                  int n,
+                                  double *A,
+                                  int lda,
+                                  double *W,
+                                  double *work,
+                                  int lwork,
+                                  int *info,
+                                  syevjInfo_t params) {
+    #if HIP_VERSION < 402
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_dsyev(handle, convert_rocblas_evect(jobz), convert_rocblas_fill(uplo),
+                           n, A, lda, W,
+                           // since we can't pass in another array through the API, and work is unused,
+                           // we use it to store the temporary E array, to be discarded after calculation
+                           work,
+                           info);
+    #endif
+}
+
+cusolverStatus_t cusolverDnCheevj(cusolverDnHandle_t handle,
+                                  cusolverEigMode_t jobz,
+                                  cublasFillMode_t uplo,
+                                  int n,
+                                  cuComplex *A,
+                                  int lda,
+                                  float *W,
+                                  cuComplex *work,
+                                  int lwork,
+                                  int *info,
+                                  syevjInfo_t params) {
+    #if HIP_VERSION < 402
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_cheev(handle, convert_rocblas_evect(jobz), convert_rocblas_fill(uplo),
+                           n, reinterpret_cast<rocblas_float_complex*>(A), lda, W,
+                           // since we can't pass in another array through the API, and work is unused,
+                           // we use it to store the temporary E array, to be discarded after calculation
+                           reinterpret_cast<float*>(work),
+                           info);
+    #endif
+}
+
+cusolverStatus_t cusolverDnZheevj(cusolverDnHandle_t handle,
+                                  cusolverEigMode_t jobz,
+                                  cublasFillMode_t uplo,
+                                  int n,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  double *W,
+                                  cuDoubleComplex *work,
+                                  int lwork,
+                                  int *info,
+                                  syevjInfo_t params) {
+    #if HIP_VERSION < 402
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_zheev(handle, convert_rocblas_evect(jobz), convert_rocblas_fill(uplo),
+                           n, reinterpret_cast<rocblas_double_complex*>(A), lda, W,
+                           // since we can't pass in another array through the API, and work is unused,
+                           // we use it to store the temporary E array, to be discarded after calculation
+                           reinterpret_cast<double*>(work),
+                           info);
+    #endif
+}
+
+/* ---------- batched syevj ---------- */
+cusolverStatus_t cusolverDnSsyevjBatched_bufferSize(cusolverDnHandle_t handle,
+                                                    cusolverEigMode_t jobz,
+                                                    cublasFillMode_t uplo,
+                                                    int n,
+                                                    const float *A,
+                                                    int lda,
+                                                    const float *W,
+                                                    int *lwork,
+                                                    syevjInfo_t params,
+                                                    int batchSize) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the tridiagonal matrix T associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = batchSize * n;  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnDsyevjBatched_bufferSize(cusolverDnHandle_t handle,
+                                                    cusolverEigMode_t jobz,
+                                                    cublasFillMode_t uplo,
+                                                    int n,
+                                                    const double *A,
+                                                    int lda,
+                                                    const double *W,
+                                                    int *lwork,
+                                                    syevjInfo_t params,
+                                                    int batchSize) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the tridiagonal matrix T associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = batchSize * n;  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnCheevjBatched_bufferSize(cusolverDnHandle_t handle,
+                                                    cusolverEigMode_t jobz,
+                                                    cublasFillMode_t uplo,
+                                                    int n,
+                                                    const cuComplex *A,
+                                                    int lda,
+                                                    const float *W,
+                                                    int *lwork,
+                                                    syevjInfo_t params,
+                                                    int batchSize) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the tridiagonal matrix T associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = batchSize * n;  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnZheevjBatched_bufferSize(cusolverDnHandle_t handle,
+                                                    cusolverEigMode_t jobz,
+                                                    cublasFillMode_t uplo,
+                                                    int n,
+                                                    const cuDoubleComplex *A,
+                                                    int lda,
+                                                    const double *W,
+                                                    int *lwork,
+                                                    syevjInfo_t params,
+                                                    int batchSize) {
+    // rocSOLVER does not need extra workspace, but it needs to allocate memory for storing
+    // the tridiagonal matrix T associated with A, which we don't need, so we use this workspace
+    // to store it
+    *lwork = batchSize * n;  // note: counts, not bytes!
+    return rocblas_status_success;
+}
+
+cusolverStatus_t cusolverDnSsyevjBatched(cusolverDnHandle_t handle,
+                                         cusolverEigMode_t jobz,
+                                         cublasFillMode_t uplo,
+                                         int n,
+                                         float *A,
+                                         int lda,
+                                         float *W,
+                                         float *work,
+                                         int lwork,
+                                         int *info,
+                                         syevjInfo_t params,
+                                         int batchSize) {
+    #if HIP_VERSION < 402
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_ssyev_batched(handle, convert_rocblas_evect(jobz), convert_rocblas_fill(uplo),
+                                   n, reinterpret_cast<float* const*>(A), lda, W, n,
+                                   // since we can't pass in another array through the API, and work is unused,
+                                   // we use it to store the temporary E array, to be discarded after calculation
+                                   work, n,
+                                   info, batchSize);
+    #endif
+}
+
+cusolverStatus_t cusolverDnDsyevjBatched(cusolverDnHandle_t handle,
+                                         cusolverEigMode_t jobz,
+                                         cublasFillMode_t uplo,
+                                         int n,
+                                         double *A,
+                                         int lda,
+                                         double *W,
+                                         double *work,
+                                         int lwork,
+                                         int *info,
+                                         syevjInfo_t params,
+                                         int batchSize) {
+    #if HIP_VERSION < 402
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_dsyev_batched(handle, convert_rocblas_evect(jobz), convert_rocblas_fill(uplo),
+                                   n, reinterpret_cast<double* const*>(A), lda, W, n,
+                                   // since we can't pass in another array through the API, and work is unused,
+                                   // we use it to store the temporary E array, to be discarded after calculation
+                                   work, n,
+                                   info, batchSize);
+    #endif
+}
+
+cusolverStatus_t cusolverDnCheevjBatched(cusolverDnHandle_t handle,
+                                         cusolverEigMode_t jobz,
+                                         cublasFillMode_t uplo,
+                                         int n,
+                                         cuComplex *A,
+                                         int lda,
+                                         float *W,
+                                         cuComplex *work,
+                                         int lwork,
+                                         int *info,
+                                         syevjInfo_t params,
+                                         int batchSize) {
+    #if HIP_VERSION < 402
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_cheev_batched(handle, convert_rocblas_evect(jobz), convert_rocblas_fill(uplo),
+                                   n, reinterpret_cast<rocblas_float_complex* const*>(A), lda, W, n,
+                                   // since we can't pass in another array through the API, and work is unused,
+                                   // we use it to store the temporary E array, to be discarded after calculation
+                                   reinterpret_cast<float*>(work), n,
+                                   info, batchSize);
+    #endif
+}
+
+cusolverStatus_t cusolverDnZheevjBatched(cusolverDnHandle_t handle,
+                                         cusolverEigMode_t jobz,
+                                         cublasFillMode_t uplo,
+                                         int n,
+                                         cuDoubleComplex *A,
+                                         int lda,
+                                         double *W,
+                                         cuDoubleComplex *work,
+                                         int lwork,
+                                         int *info,
+                                         syevjInfo_t params,
+                                         int batchSize) {
+    #if HIP_VERSION < 402
+    return rocblas_status_not_implemented;
+    #else
+    return rocsolver_zheev_batched(handle, convert_rocblas_evect(jobz), convert_rocblas_fill(uplo),
+                                   n, reinterpret_cast<rocblas_double_complex* const*>(A), lda, W, n,
+                                   // since we can't pass in another array through the API, and work is unused,
+                                   // we use it to store the temporary E array, to be discarded after calculation
+                                   reinterpret_cast<double*>(work), n,
+                                   info, batchSize);
+    #endif
+}
+
+
+/* all of the stubs below are unsupported functions; the supported ones are moved to above */
+
+typedef enum{} cusolverEigType_t;
+typedef void* cusolverSpHandle_t;
+typedef void* cusparseMatDescr_t;
+
+cusolverStatus_t cusolverSpGetStream(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpSetStream(...) {
+    return rocblas_status_not_implemented;
+}
+
+
+/* ---------- potrs ---------- */
+cusolverStatus_t cusolverDnSpotrs(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDpotrs(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCpotrs(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZpotrs(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnSpotrsBatched(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDpotrsBatched(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCpotrsBatched(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZpotrsBatched(...) {
+    return rocblas_status_not_implemented;
+}
+
+
+/* ---------- sytrf ---------- */
+cusolverStatus_t cusolverDnSsytrf_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDsytrf_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCsytrf_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZsytrf_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnSsytrf(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDsytrf(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCsytrf(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZsytrf(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXgesvdjSetTolerance(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXgesvdjSetMaxSweeps(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXgesvdjSetSortEig(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXgesvdjGetResidual(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXgesvdjGetSweeps(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnSgesvdj_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDgesvdj_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCgesvdj_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZgesvdj_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnSgesvdj(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDgesvdj(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCgesvdj(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZgesvdj(...) {
+    return rocblas_status_not_implemented;
+}
+
+
+cusolverStatus_t cusolverDnSgesvdaStridedBatched_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDgesvdaStridedBatched_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCgesvdaStridedBatched_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZgesvdaStridedBatched_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnSgesvdaStridedBatched(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDgesvdaStridedBatched(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCgesvdaStridedBatched(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZgesvdaStridedBatched(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZZgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZCgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZYgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZKgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCCgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCYgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCKgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDDgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDSgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDXgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDHgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSSgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSXgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSHgels_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZZgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZCgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZYgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZKgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCCgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCYgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCKgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDDgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDSgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDXgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDHgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSSgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSXgels(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSHgels(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnSsyevd_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDsyevd_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCheevd_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZheevd_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnSsyevd(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnDsyevd(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnCheevd(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZheevd(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXsyevjSetTolerance(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXsyevjSetMaxSweeps(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXsyevjSetSortEig(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXsyevjGetResidual(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXsyevjGetSweeps(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnZZgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZCgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZYgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZKgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCCgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCYgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCKgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDDgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDSgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDXgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDHgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSSgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSXgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSHgesv_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZZgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZCgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZYgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnZKgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCCgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCYgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnCKgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDDgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDSgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDXgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnDHgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSSgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSXgesv(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnSHgesv(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverDnXsyevd_bufferSize(...) {
+    return rocblas_status_not_implemented;
+}
+cusolverStatus_t cusolverDnXsyevd(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpCreate(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpDestroy(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpScsrlsvqr(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpDcsrlsvqr(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpCcsrlsvqr(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpZcsrlsvqr(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpScsrlsvchol(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpDcsrlsvchol(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpCcsrlsvchol(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpZcsrlsvchol(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpScsreigvsi(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpDcsreigvsi(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpCcsreigvsi(...) {
+    return rocblas_status_not_implemented;
+}
+
+cusolverStatus_t cusolverSpZcsreigvsi(...) {
+    return rocblas_status_not_implemented;
+}
+
+} // extern "C" 
+
+#endif // #ifdef INCLUDE_GUARD_HIP_CUPY_ROCSOLVER_H
diff --git a/cupy_backends/hip/cupy_roctx.h b/cupy_backends/hip/cupy_roctx.h
new file mode 100644
index 0000000..6a6e63a
--- /dev/null
+++ b/cupy_backends/hip/cupy_roctx.h
@@ -0,0 +1,31 @@
+#ifndef INCLUDE_GUARD_HIP_CUPY_ROCTX_H
+#define INCLUDE_GUARD_HIP_CUPY_ROCTX_H
+
+#ifndef CUPY_NO_NVTX
+#include <roctx.h>
+#endif // #ifndef CUPY_NO_NVTX
+
+// this is to ensure we use non-"Ex" APIs like roctxMarkA etc
+#define NVTX_VERSION (100 * ROCTX_VERSION_MAJOR + 10 * ROCTX_VERSION_MINOR)
+
+extern "C" {
+
+///////////////////////////////////////////////////////////////////////////////
+// roctx
+///////////////////////////////////////////////////////////////////////////////
+
+void nvtxMarkA(const char* message) {
+    roctxMarkA(message);
+}
+
+int nvtxRangePushA(const char* message) {
+    return roctxRangePushA(message);
+}
+
+int nvtxRangePop() {
+    return roctxRangePop();
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_HIP_CUPY_ROCTX_H
diff --git a/cupy_backends/stub/cupy_cuComplex.h b/cupy_backends/stub/cupy_cuComplex.h
new file mode 100644
index 0000000..30d1104
--- /dev/null
+++ b/cupy_backends/stub/cupy_cuComplex.h
@@ -0,0 +1,22 @@
+// This file is a stub header file of cuda for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_COMPLEX_H
+#define INCLUDE_GUARD_STUB_CUPY_COMPLEX_H
+
+extern "C" {
+
+///////////////////////////////////////////////////////////////////////////////
+// cuComplex.h
+///////////////////////////////////////////////////////////////////////////////
+
+struct cuComplex{
+    float x, y;
+};
+
+struct cuDoubleComplex{
+    double x, y;
+};
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_COMPLEX_H
diff --git a/cupy_backends/stub/cupy_cublas.h b/cupy_backends/stub/cupy_cublas.h
new file mode 100644
index 0000000..80367e2
--- /dev/null
+++ b/cupy_backends/stub/cupy_cublas.h
@@ -0,0 +1,430 @@
+// This file is a stub header file of cuda for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUBLAS_H
+#define INCLUDE_GUARD_STUB_CUPY_CUBLAS_H
+
+#include "cupy_cuda_common.h"
+
+extern "C" {
+
+// Context
+cublasStatus_t cublasCreate(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDestroy(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasGetVersion(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasGetPointerMode(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSetPointerMode(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+// Stream
+cublasStatus_t cublasSetStream(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasGetStream(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+// Math Mode
+cublasStatus_t cublasSetMathMode(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasGetMathMode(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+// BLAS Level 1
+cublasStatus_t cublasIsamax(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasIdamax(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasIcamax(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasIzamax(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasIsamin(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasIdamin(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasIcamin(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasIzamin(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSasum(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDasum(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasScasum(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDzasum(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSaxpy(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDaxpy(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCaxpy(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZaxpy(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSdot(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDdot(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCdotu(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCdotc(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZdotc(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZdotu(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSnrm2(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDnrm2(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasScnrm2(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDznrm2(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSscal(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDscal(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCscal(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCsscal(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZscal(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZdscal(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+// BLAS Level 2
+cublasStatus_t cublasSgemv(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDgemv(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+
+cublasStatus_t cublasCgemv(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZgemv(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSger(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDger(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+int cublasCgeru(...) {
+    return 0;
+}
+
+int cublasCgerc(...) {
+    return 0;
+}
+
+int cublasZgeru(...) {
+    return 0;
+}
+
+int cublasZgerc(...) {
+    return 0;
+}
+
+cublasStatus_t cublasSsbmv(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDsbmv(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+// BLAS Level 3
+cublasStatus_t cublasSgemm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDgemm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+
+cublasStatus_t cublasCgemm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZgemm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSgemmBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDgemmBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCgemmBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZgemmBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSgemmStridedBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDgemmStridedBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCgemmStridedBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZgemmStridedBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSgemmEx(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasGemmEx(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasStrsm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDtrsm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCtrsm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZtrsm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSsyrk(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDsyrk(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCsyrk(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZsyrk(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+typedef enum{} cublasComputeType_t;
+cublasStatus_t cublasGemmEx_v11(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasGemmStridedBatchedEx(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasGemmStridedBatchedEx_v11(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+// BLAS extension
+cublasStatus_t cublasSgeam(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDgeam(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCgeam(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZgeam(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSdgmm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDdgmm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCdgmm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZdgmm(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSgetrfBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDgetrfBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCgetrfBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZgetrfBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSgetrsBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDgetrsBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCgetrsBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZgetrsBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasSgetriBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDgetriBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasCgetriBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasZgetriBatched(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasStrttp(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDtrttp(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasStpttr(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+cublasStatus_t cublasDtpttr(...) {
+    return CUBLAS_STATUS_SUCCESS;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_CUBLAS_H
diff --git a/cupy_backends/stub/cupy_cuda.h b/cupy_backends/stub/cupy_cuda.h
new file mode 100644
index 0000000..98d6c13
--- /dev/null
+++ b/cupy_backends/stub/cupy_cuda.h
@@ -0,0 +1,121 @@
+// This file is a stub header file of cuda for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUDA_H
+#define INCLUDE_GUARD_STUB_CUPY_CUDA_H
+
+#include "cupy_cuda_common.h"
+
+extern "C" {
+
+// Error handling
+CUresult cuGetErrorName(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuGetErrorString(...) {
+    return CUDA_SUCCESS;
+}
+
+// Primary context management
+CUresult cuDevicePrimaryCtxRelease(...) {
+    return CUDA_SUCCESS;
+}
+
+// Context management
+CUresult cuCtxGetCurrent(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuCtxSetCurrent(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuCtxCreate(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuCtxDestroy(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuCtxGetDevice(...) {
+    return CUDA_SUCCESS;
+}
+
+// Module load and kernel execution
+CUresult cuLinkCreate (...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuLinkAddData(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuLinkAddFile(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuLinkComplete(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuLinkDestroy(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuModuleLoad(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuModuleLoadData(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuModuleUnload(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuModuleGetFunction(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuModuleGetGlobal(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuLaunchKernel(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuLaunchCooperativeKernel(...) {
+    return CUDA_SUCCESS;
+}
+
+// Function attribute
+CUresult cuFuncGetAttribute(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuFuncSetAttribute(...) {
+    return CUDA_SUCCESS;
+}
+
+// Occupancy
+typedef size_t (*CUoccupancyB2DSize)(int);
+
+CUresult cuOccupancyMaxActiveBlocksPerMultiprocessor(...) {
+    return CUDA_SUCCESS;
+}
+
+CUresult cuOccupancyMaxPotentialBlockSize(...) {
+    return CUDA_SUCCESS;
+}
+
+// Stream
+CUresult cuStreamGetCtx(...) {
+    return CUDA_SUCCESS;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_CUDA_H
diff --git a/cupy_backends/stub/cupy_cuda_common.h b/cupy_backends/stub/cupy_cuda_common.h
new file mode 100644
index 0000000..a127b57
--- /dev/null
+++ b/cupy_backends/stub/cupy_cuda_common.h
@@ -0,0 +1,235 @@
+// This file is a stub header file of cuda for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUDA_COMMON_H
+#define INCLUDE_GUARD_STUB_CUPY_CUDA_COMMON_H
+
+#define CUDA_VERSION 0
+
+extern "C" {
+
+///////////////////////////////////////////////////////////////////////////////
+// cuda.h
+///////////////////////////////////////////////////////////////////////////////
+
+typedef int CUdevice;
+typedef enum {
+    CUDA_SUCCESS = 0,
+} CUresult;
+enum CUjit_option {};
+enum CUjitInputType {};
+enum CUfunction_attribute {};;
+enum CUarray_format {};
+enum CUaddress_mode {};
+enum CUfilter_mode {};
+
+
+typedef void* CUdeviceptr;
+struct CUctx_st;
+struct CUevent_st;
+struct CUfunc_st;
+struct CUmod_st;
+struct CUstream_st;
+struct CUgraph_st;
+struct CUgraphExec_st;
+struct CUlinkState_st;
+
+
+typedef struct CUctx_st* CUcontext;
+typedef struct CUevent_st* cudaEvent_t;
+typedef struct CUfunc_st* CUfunction;
+typedef struct CUmod_st* CUmodule;
+typedef struct CUstream_st* cudaStream_t;
+typedef struct CUgraph_st* cudaGraph_t;
+typedef struct CUgraphExec_st* cudaGraphExec_t;
+typedef struct CUlinkState_st* CUlinkState;
+typedef struct CUarray_st* CUarray;
+struct CUDA_ARRAY_DESCRIPTOR {
+    CUarray_format Format;
+    size_t Height;
+    unsigned int NumChannels;
+    size_t Width;
+};
+
+
+///////////////////////////////////////////////////////////////////////////////
+// cuda_runtime.h
+///////////////////////////////////////////////////////////////////////////////
+
+enum {
+    cudaDevAttrComputeCapabilityMajor = 75,
+    cudaDevAttrComputeCapabilityMinor = 76,
+};
+
+typedef enum {
+    cudaSuccess = 0,
+    cudaErrorInvalidValue = 1,
+    cudaErrorMemoryAllocation = 2,
+    cudaErrorInvalidResourceHandle = 400,
+    cudaErrorContextIsDestroyed = 709,
+    cudaErrorPeerAccessAlreadyEnabled = 704,
+} cudaError_t;
+typedef enum {} cudaDataType;
+enum cudaDeviceAttr {};
+enum cudaLimit {};
+enum cudaMemoryAdvise {};
+enum cudaMemcpyKind {};
+
+
+typedef void (*cudaStreamCallback_t)(
+    cudaStream_t stream, cudaError_t status, void* userData);
+
+typedef void (*cudaHostFn_t)(void* userData);
+
+struct cudaPointerAttributes{
+    int device;
+    void* devicePointer;
+    void* hostPointer;
+};
+
+
+enum cudaChannelFormatKind {};
+typedef unsigned long long cudaTextureObject_t;
+typedef unsigned long long cudaSurfaceObject_t;
+enum cudaResourceType {};
+enum cudaTextureAddressMode {};
+enum cudaTextureFilterMode {};
+enum cudaTextureReadMode {};
+struct cudaResourceViewDesc;
+typedef void* cudaArray_t;
+struct cudaExtent {
+    size_t width, height, depth;
+};
+struct cudaPos {
+    size_t x, y, z;
+};
+struct cudaPitchedPtr {
+    size_t pitch;
+    void* ptr;
+    size_t xsize, ysize;
+};
+typedef void* cudaMipmappedArray_t;
+struct cudaMemcpy3DParms {
+    cudaArray_t srcArray;
+    struct cudaPos srcPos;
+    struct cudaPitchedPtr srcPtr;
+    cudaArray_t dstArray;
+    struct cudaPos dstPos;
+    struct cudaPitchedPtr dstPtr;
+    struct cudaExtent extent;
+    enum cudaMemcpyKind kind;
+};
+struct cudaChannelFormatDesc {
+    int x, y, z, w;
+    enum cudaChannelFormatKind f;
+};
+struct cudaResourceDesc {
+    enum cudaResourceType resType;
+
+    union {
+        struct {
+            cudaArray_t array;
+        } array;
+        struct {
+            cudaMipmappedArray_t mipmap;
+        } mipmap;
+        struct {
+            void *devPtr;
+            struct cudaChannelFormatDesc desc;
+            size_t sizeInBytes;
+        } linear;
+        struct {
+            void *devPtr;
+            struct cudaChannelFormatDesc desc;
+            size_t width;
+            size_t height;
+            size_t pitchInBytes;
+        } pitch2D;
+    } res;
+};
+struct cudaTextureDesc {
+    enum cudaTextureAddressMode addressMode[3];
+    enum cudaTextureFilterMode filterMode;
+    enum cudaTextureReadMode readMode;
+    int sRGB;
+    float borderColor[4];
+    int normalizedCoords;
+    unsigned int maxAnisotropy;
+    enum cudaTextureFilterMode mipmapFilterMode;
+    float mipmapLevelBias;
+    float minMipmapLevelClamp;
+    float maxMipmapLevelClamp;
+};
+
+// IPC operations
+typedef struct {
+    unsigned char reserved[64];
+} cudaIpcMemHandle_t;
+
+// IPC operations
+typedef struct {
+    unsigned char reserved[64];
+} cudaIpcEventHandle_t;
+
+typedef struct {
+     char name[256];
+     size_t totalGlobalMem;
+     size_t sharedMemPerBlock;
+     int regsPerBlock;
+     int warpSize;
+     int maxThreadsPerBlock;
+     int maxThreadsDim[3];
+     int maxGridSize[3];
+     int clockRate;
+     int memoryClockRate;
+     int memoryBusWidth;
+     size_t totalConstMem;
+     int major;
+     int minor;
+     int multiProcessorCount;
+     int l2CacheSize;
+     int maxThreadsPerMultiProcessor;
+     int computeMode;
+     int clockInstructionRate;
+     int concurrentKernels;
+     int pciBusID;
+     int pciDeviceID;
+     size_t maxSharedMemoryPerMultiProcessor;
+     int isMultiGpuBoard;
+     int canMapHostMemory;
+} cudaDeviceProp;
+
+typedef void* cudaMemPool_t;
+enum cudaMemPoolAttr {};
+
+
+///////////////////////////////////////////////////////////////////////////////
+// library_types.h
+///////////////////////////////////////////////////////////////////////////////
+
+typedef enum libraryPropertyType_t {
+	MAJOR_VERSION,
+	MINOR_VERSION,
+	PATCH_LEVEL
+} libraryPropertyType;
+
+
+///////////////////////////////////////////////////////////////////////////////
+// cublas_v2.h
+///////////////////////////////////////////////////////////////////////////////
+
+typedef void* cublasHandle_t;
+
+typedef enum {} cublasDiagType_t;
+typedef enum {} cublasFillMode_t;
+typedef enum {} cublasOperation_t;
+typedef enum {} cublasPointerMode_t;
+typedef enum {} cublasSideMode_t;
+typedef enum {} cublasGemmAlgo_t;
+typedef enum {} cublasMath_t;
+typedef enum {
+    CUBLAS_STATUS_SUCCESS=0,
+} cublasStatus_t;
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_CUDA_COMMON_H
diff --git a/cupy_backends/stub/cupy_cuda_runtime.h b/cupy_backends/stub/cupy_cuda_runtime.h
new file mode 100644
index 0000000..6cde831
--- /dev/null
+++ b/cupy_backends/stub/cupy_cuda_runtime.h
@@ -0,0 +1,427 @@
+// This file is a stub header file of cuda for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUDA_RUNTIME_H
+#define INCLUDE_GUARD_STUB_CUPY_CUDA_RUNTIME_H
+
+#include "cupy_cuda_common.h"
+
+extern "C" {
+
+bool hip_environment = false;
+
+// Error handling
+const char* cudaGetErrorName(...) {
+    return NULL;
+}
+
+const char* cudaGetErrorString(...) {
+    return NULL;
+}
+
+cudaError_t cudaGetLastError() {
+    return cudaSuccess;
+}
+
+
+// Initialization
+cudaError_t cudaDriverGetVersion(int* driverVersion) {
+    *driverVersion = 0;  // make the stub work safely without driver
+    return cudaSuccess;
+}
+
+cudaError_t cudaRuntimeGetVersion(...) {
+    return cudaSuccess;
+}
+
+
+// CUdevice operations
+cudaError_t cudaGetDevice(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceGetAttribute(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceGetByPCIBusId(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceGetPCIBusId(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGetDeviceCount(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaSetDevice(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceSynchronize() {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGetDeviceProperties(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceCanAccessPeer(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceEnablePeerAccess(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceDisablePeerAccess(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceGetLimit(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceSetLimit(...) {
+    return cudaSuccess;
+}
+
+// IPC operations
+cudaError_t cudaIpcCloseMemHandle(...){
+    return cudaSuccess;
+}
+
+cudaError_t cudaIpcGetEventHandle(...){
+    return cudaSuccess;
+}
+
+cudaError_t cudaIpcGetMemHandle(...){
+    return cudaSuccess;
+}
+
+cudaError_t cudaIpcOpenEventHandle(...){
+    return cudaSuccess;
+}
+
+cudaError_t cudaIpcOpenMemHandle(...){
+    return cudaSuccess;
+}
+
+// Memory management
+enum cudaMemAllocationType {};
+enum cudaMemAllocationHandleType {};
+enum cudaMemLocationType {};
+struct cudaMemLocation {
+    int id;
+};
+struct cudaMemPoolProps {
+    cudaMemAllocationType allocType;
+    cudaMemAllocationHandleType handleTypes;
+    struct cudaMemLocation location;
+    unsigned char reserved[64];
+    void* win32SecurityAttributes;
+};
+
+cudaError_t cudaMalloc(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMalloc3DArray(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMallocArray(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMallocAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaHostAlloc(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaHostRegister(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaHostUnregister(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMallocManaged(...) {
+    return cudaSuccess;
+}
+
+int cudaFree(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaFreeArray(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaFreeHost(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaFreeAsync(...) {
+    return cudaSuccess;
+}
+
+int cudaMemGetInfo(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpyAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpyPeer(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpyPeerAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy2D(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy2DAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy2DFromArray(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy2DFromArrayAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy2DToArray(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy2DToArrayAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy3D(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemcpy3DAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemset(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemsetAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemAdvise(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemPrefetchAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaPointerGetAttributes(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMallocFromPoolAsync(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemPoolCreate(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemPoolDestroy(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceGetDefaultMemPool(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceGetMemPool(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDeviceSetMemPool(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemPoolTrimTo(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemPoolGetAttribute(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaMemPoolSetAttribute(...) {
+    return cudaSuccess;
+}
+
+
+// Stream and Event
+enum cudaStreamCaptureMode {};
+enum cudaStreamCaptureStatus {};
+
+cudaError_t cudaStreamCreate(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamCreateWithFlags(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamDestroy(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamSynchronize(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamAddCallback(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaLaunchHostFunc(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamQuery(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamWaitEvent(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaEventCreate(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaEventCreateWithFlags(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaEventDestroy(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaEventElapsedTime(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaEventQuery(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaEventRecord(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaEventSynchronize(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamBeginCapture(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamEndCapture(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaStreamIsCapturing(...) {
+    return cudaSuccess;
+}
+
+// Texture
+cudaError_t cudaCreateTextureObject(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDestroyTextureObject(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGetChannelDesc(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGetTextureObjectResourceDesc(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGetTextureObjectTextureDesc(...) {
+    return cudaSuccess;
+}
+
+cudaExtent make_cudaExtent(...) {
+    struct cudaExtent ex = {0};
+    return ex;
+}
+
+cudaPitchedPtr make_cudaPitchedPtr(...) {
+    struct cudaPitchedPtr ptr = {0};
+    return ptr;
+}
+
+cudaPos make_cudaPos(...) {
+    struct cudaPos pos = {0};
+    return pos;
+}
+
+// Surface
+cudaError_t cudaCreateSurfaceObject(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaDestroySurfaceObject(...) {
+    return cudaSuccess;
+}
+
+// CUDA Graph
+cudaError_t cudaGraphInstantiate(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGraphDestroy(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGraphExecDestroy(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGraphLaunch(...) {
+    return cudaSuccess;
+}
+
+cudaError_t cudaGraphUpload(...) {
+    return cudaSuccess;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_CUDA_RUNTIME_H
diff --git a/cupy_backends/stub/cupy_cudnn.h b/cupy_backends/stub/cupy_cudnn.h
new file mode 100644
index 0000000..ad0d18b
--- /dev/null
+++ b/cupy_backends/stub/cupy_cudnn.h
@@ -0,0 +1,644 @@
+// This file is a stub header file of cudnn for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUDNN_H
+#define INCLUDE_GUARD_STUB_CUPY_CUDNN_H
+
+
+#define CUDNN_VERSION 0
+
+#define CUDNN_BN_MIN_EPSILON 0.0
+
+extern "C" {
+
+typedef enum {
+    CUDNN_STATUS_SUCCESS = 0,
+    CUDNN_STATUS_NOT_SUPPORTED = 9,
+} cudnnStatus_t;
+
+typedef enum {} cudnnActivationMode_t;
+typedef enum {} cudnnConvolutionFwdAlgo_t;
+typedef enum {} cudnnConvolutionFwdPreference_t;
+typedef enum {} cudnnConvolutionMode_t;
+typedef enum {} cudnnCTCLossAlgo_t;
+typedef enum {} cudnnDataType_t;
+typedef enum {} cudnnPoolingMode_t;
+typedef enum {} cudnnSoftmaxAlgorithm_t;
+typedef enum {} cudnnSoftmaxMode_t;
+typedef enum {} cudnnTensorFormat_t;
+typedef enum {} cudnnErrQueryMode_t;
+typedef struct cudnnRuntimeTag_t cudnnRuntimeTag_t;
+
+typedef void* cudnnConvolutionDescriptor_t;
+typedef void* cudnnCTCLossDescriptor_t;
+typedef void* cudnnFilterDescriptor_t;
+typedef void* cudnnHandle_t;
+typedef void* cudnnPoolingDescriptor_t;
+typedef void* cudnnTensorDescriptor_t;
+
+
+// Error handling
+const char* cudnnGetErrorString(...) {
+    return NULL;
+}
+
+
+// Version
+size_t cudnnGetVersion() {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+// Runtime error checking
+cudnnStatus_t cudnnQueryRuntimeError(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+// Initialization and CUDA cooperation
+cudnnStatus_t cudnnCreate(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnDestroy(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetStream(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetStream(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+
+// Tensor manipulation
+cudnnStatus_t cudnnCreateTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetTensor4dDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetTensor4dDescriptorEx(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetTensor4dDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetTensorNdDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetTensor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnScaleTensor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnDestroyTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+
+// Filter manipulation
+cudnnStatus_t cudnnCreateFilterDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnDestroyFilterDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+
+// Convolution
+cudnnStatus_t cudnnCreateConvolutionDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetConvolution2dDescriptor_v4(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnDestroyConvolutionDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetConvolutionForwardAlgorithm_v6(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetConvolutionForwardAlgorithm_v7(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetConvolutionForwardWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnConvolutionForward(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnConvolutionBackwardBias(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+// Pooling
+cudnnStatus_t cudnnCreatePoolingDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnDestroyPoolingDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnPoolingForward(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnPoolingBackward(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+
+// Activation
+cudnnStatus_t cudnnSoftmaxForward(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSoftmaxBackward(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+
+cudnnStatus_t cudnnAddTensor_v3(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFindConvolutionForwardAlgorithm(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnConvolutionBackwardFilter_v3(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardFilterWorkspaceSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFindConvolutionBackwardFilterAlgorithm(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm_v6(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnConvolutionBackwardData_v3(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFindConvolutionBackwardDataAlgorithm(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardDataAlgorithm_v6(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardDataAlgorithm_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionBackwardDataWorkspaceSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetConvolutionNdDescriptor_v3(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+
+typedef enum {} cudnnBatchNormMode_t;
+typedef enum {} cudnnBatchNormOps_t;
+typedef enum {} cudnnNanPropagation_t;
+
+
+typedef void* cudnnActivationDescriptor_t;
+
+
+cudnnStatus_t cudnnSetFilter4dDescriptor_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetFilterNdDescriptor_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetFilterNdDescriptor_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDeriveBNTensorDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnBatchNormalizationForwardTraining(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnBatchNormalizationForwardInference(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnBatchNormalizationBackward(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnBatchNormalizationForwardTrainingEx(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnBatchNormalizationBackwardEx(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationBackwardExWorkspaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetBatchNormalizationTrainingExReserveSpaceSize(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetPooling2dDescriptor_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetPoolingNdDescriptor_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateActivationDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetActivationDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyActivationDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnActivationForward_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnActivationBackward_v4(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+// CTC
+cudnnStatus_t cudnnCreateCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnDestroyCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnSetCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnGetCTCLossDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnGetCTCLossWorkspaceSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+cudnnStatus_t cudnnCTCLoss(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+typedef enum {} cudnnMathType_t;
+
+cudnnStatus_t cudnnSetConvolutionMathType(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionMathType(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetConvolutionGroupCount(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetConvolutionGroupCount(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+typedef enum {} cudnnConvolutionBwdDataAlgo_t;
+typedef enum {} cudnnConvolutionBwdDataPreference_t;
+typedef enum {} cudnnConvolutionBwdFilterAlgo_t;
+typedef enum {} cudnnConvolutionBwdFilterPreference_t;
+typedef enum {} cudnnDeterminism_t;
+
+typedef struct {
+  cudnnConvolutionFwdAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+} cudnnConvolutionFwdAlgoPerf_t;
+
+typedef struct {
+  cudnnConvolutionBwdFilterAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+} cudnnConvolutionBwdFilterAlgoPerf_t;
+
+typedef struct {
+  cudnnConvolutionBwdDataAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+} cudnnConvolutionBwdDataAlgoPerf_t;
+
+cudnnStatus_t cudnnFindConvolutionForwardAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFindConvolutionBackwardFilterAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFindConvolutionBackwardDataAlgorithmEx_v7(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+typedef struct {
+  cudnnConvolutionFwdAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionFwdAlgoPerf_v7_t;
+
+typedef struct {
+  cudnnConvolutionBwdFilterAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionBwdFilterAlgoPerf_v7_t;
+
+typedef struct {
+  cudnnConvolutionBwdDataAlgo_t algo;
+  cudnnStatus_t status;
+  float time;
+  size_t memory;
+  cudnnDeterminism_t determinism;
+  cudnnMathType_t mathType;
+  int reserved[3];
+} cudnnConvolutionBwdDataAlgoPerf_v7_t;
+
+
+// ***_v3 functions are not declared in cuDNN v2, v3 and v4.
+// Following definitions are for compatibility with cuDNN v5.
+
+typedef enum {} cudnnRNNMode_t;
+typedef enum {} cudnnDirectionMode_t;
+typedef enum {} cudnnOpTensorOp_t;
+typedef enum {} cudnnRNNInputMode_t;
+
+typedef void* cudnnDropoutDescriptor_t;
+typedef void* cudnnRNNDescriptor_t;
+typedef void* cudnnSpatialTransformerDescriptor_t;
+typedef void* cudnnSamplerType_t;
+typedef void* cudnnOpTensorDescriptor_t;
+
+
+cudnnStatus_t cudnnSetConvolution2dDescriptor_v5(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetFilterNdDescriptor_v5(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateDropoutDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyDropoutDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDropoutGetStatesSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDropoutGetReserveSpaceSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetDropoutDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDropoutForward(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDropoutBackward(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateRNNDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyRNNDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetRNNDescriptor_v5(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNWorkspaceSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNTrainingReserveSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNParamsSize(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNLinLayerMatrixParams(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetRNNLinLayerBiasParams(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNForwardInference(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNForwardTraining(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNBackwardData(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnRNNBackwardWeights(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFindConvolutionBackwardFilterAlgorithmEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFindConvolutionForwardAlgorithmEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFindConvolutionBackwardDataAlgorithmEx(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateSpatialTransformerDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroySpatialTransformerDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetSpatialTransformerNdDescriptor(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSpatialTfGridGeneratorForward(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSpatialTfGridGeneratorBackward(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSpatialTfSamplerForward(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSpatialTfSamplerBackward(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+
+// Tensor operations
+cudnnStatus_t cudnnCreateOpTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnSetOpTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnGetOpTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnDestroyOpTensorDescriptor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+cudnnStatus_t cudnnOpTensor(...) {
+    return CUDNN_STATUS_SUCCESS;
+}
+
+// fused ops
+typedef void* cudnnFusedOpsConstParamPack_t;
+typedef void* cudnnFusedOpsVariantParamPack_t;
+typedef void* cudnnFusedOpsPlan_t;
+
+typedef enum {} cudnnFusedOps_t;
+typedef enum {} cudnnFusedOpsConstParamLabel_t;
+typedef enum {} cudnnFusedOpsPointerPlaceHolder_t;
+typedef enum {} cudnnFusedOpsVariantParamLabel_t;
+
+cudnnStatus_t cudnnCreateFusedOpsConstParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsConstParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetFusedOpsConstParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetFusedOpsConstParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateFusedOpsVariantParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsVariantParamPack(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnSetFusedOpsVariantParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnGetFusedOpsVariantParamPackAttribute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnCreateFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnDestroyFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnMakeFusedOpsPlan(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+cudnnStatus_t cudnnFusedOpsExecute(...) {
+    return CUDNN_STATUS_NOT_SUPPORTED;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_CUDNN_H
diff --git a/cupy_backends/stub/cupy_curand.h b/cupy_backends/stub/cupy_curand.h
new file mode 100644
index 0000000..4b27b84
--- /dev/null
+++ b/cupy_backends/stub/cupy_curand.h
@@ -0,0 +1,94 @@
+// This file is a stub header file of cuda for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CURAND_H
+#define INCLUDE_GUARD_STUB_CUPY_CURAND_H
+
+#include "cupy_cuda_common.h"
+
+extern "C" {
+
+///////////////////////////////////////////////////////////////////////////////
+// curand.h
+///////////////////////////////////////////////////////////////////////////////
+
+typedef enum {} curandOrdering_t;
+typedef enum {} curandRngType_t;
+typedef enum {
+    CURAND_STATUS_SUCCESS = 0,
+} curandStatus_t;
+
+typedef void* curandGenerator_t;
+
+
+// curandGenerator_t
+curandStatus_t curandCreateGenerator(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandDestroyGenerator(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGetVersion(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+
+// Stream
+curandStatus_t curandSetStream(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandSetPseudoRandomGeneratorSeed(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandSetGeneratorOffset(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandSetGeneratorOrdering(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+
+// Generation functions
+curandStatus_t curandGenerate(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGenerateLongLong(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGenerateUniform(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGenerateUniformDouble(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGenerateNormal(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGenerateNormalDouble(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGenerateLogNormal(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGenerateLogNormalDouble(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+curandStatus_t curandGeneratePoisson(...) {
+    return CURAND_STATUS_SUCCESS;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_CURAND_H
diff --git a/cupy_backends/stub/cupy_cusolver.h b/cupy_backends/stub/cupy_cusolver.h
new file mode 100644
index 0000000..b3ff83f
--- /dev/null
+++ b/cupy_backends/stub/cupy_cusolver.h
@@ -0,0 +1,1000 @@
+// This file is a stub header file of cudnn for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUSOLVER_H
+#define INCLUDE_GUARD_STUB_CUPY_CUSOLVER_H
+
+#include "cupy_cuda_common.h"
+
+extern "C" {
+
+typedef enum {
+    CUSOLVER_STATUS_SUCCESS = 0,
+} cusolverStatus_t;
+
+typedef enum{} cusolverDnParams_t;
+typedef enum{} cusolverEigType_t;
+typedef enum{} cusolverEigMode_t;
+
+typedef void* cusolverDnHandle_t;
+typedef void* cusolverSpHandle_t;
+typedef void* cusparseMatDescr_t;
+typedef void* gesvdjInfo_t;
+typedef void* syevjInfo_t;
+
+cusolverStatus_t cusolverDnCreate(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDestroy(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnGetStream(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpGetStream(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSetStream(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpSetStream(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+
+cusolverStatus_t cusolverGetProperty(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSpotrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDpotrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCpotrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZpotrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSpotrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDpotrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCpotrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZpotrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSpotrfBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDpotrfBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCpotrfBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZpotrfBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSpotrs(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDpotrs(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCpotrs(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZpotrs(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSpotrsBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDpotrsBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCpotrsBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZpotrsBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgetrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgetrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgetrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgetrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgetrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgetrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgetrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgetrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgetrs(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgetrs(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgetrs(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgetrs(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgeqrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgeqrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgeqrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgeqrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+// The function signatures are explicitly spelled out because we need to
+// fetch the function pointers.
+cusolverStatus_t cusolverDnSgeqrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  float *A,
+                                  int lda,
+                                  float *TAU,
+                                  float *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgeqrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  double *A,
+                                  int lda,
+                                  double *TAU,
+                                  double *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgeqrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  cuComplex *A,
+                                  int lda,
+                                  cuComplex *TAU,
+                                  cuComplex *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgeqrf(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  cuDoubleComplex *TAU,
+                                  cuDoubleComplex *Workspace,
+                                  int Lwork,
+                                  int *devInfo) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSorgqr_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDorgqr_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCungqr_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZungqr_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+// The function signatures are explicitly spelled out because we need to
+// fetch the function pointers.
+cusolverStatus_t cusolverDnSorgqr(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  float *A,
+                                  int lda,
+                                  const float *tau,
+                                  float *work,
+                                  int lwork,
+                                  int *devInfo) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDorgqr(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  double *A,
+                                  int lda,
+                                  const double *tau,
+                                  double *work,
+                                  int lwork,
+                                  int *devInfo) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCungqr(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  cuComplex *A,
+                                  int lda,
+                                  const cuComplex *tau,
+                                  cuComplex *work,
+                                  int lwork,
+                                  int *devInfo) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZungqr(cusolverDnHandle_t handle,
+                                  int m,
+                                  int n,
+                                  int k,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  const cuDoubleComplex *tau,
+                                  cuDoubleComplex *work,
+                                  int lwork,
+                                  int *devInfo) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSormqr_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDormqr_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCunmqr_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZunmqr_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSormqr(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDormqr(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCunmqr(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZunmqr(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSsytrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDsytrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCsytrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZsytrf_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSsytrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDsytrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCsytrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZsytrf(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgebrd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgebrd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgebrd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgebrd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgebrd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgebrd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgebrd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgebrd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgesvd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+// The function signatures are explicitly spelled out because we need to
+// fetch the function pointers.
+cusolverStatus_t cusolverDnSgesvd(cusolverDnHandle_t handle,
+                                  signed char jobu,
+                                  signed char jobvt,
+                                  int m,
+                                  int n,
+                                  float *A,
+                                  int lda,
+                                  float *S,
+                                  float *U,
+                                  int ldu,
+                                  float *VT,
+                                  int ldvt,
+                                  float *work,
+                                  int lwork,
+                                  float *rwork,
+                                  int *info) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvd(cusolverDnHandle_t handle,
+                                  signed char jobu,
+                                  signed char jobvt,
+                                  int m,
+                                  int n,
+                                  double *A,
+                                  int lda,
+                                  double *S,
+                                  double *U,
+                                  int ldu,
+                                  double *VT,
+                                  int ldvt,
+                                  double *work,
+                                  int lwork,
+                                  double *rwork,
+                                  int *info) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvd(cusolverDnHandle_t handle,
+                                  signed char jobu,
+                                  signed char jobvt,
+                                  int m,
+                                  int n,
+                                  cuComplex *A,
+                                  int lda,
+                                  float *S,
+                                  cuComplex *U,
+                                  int ldu,
+                                  cuComplex *VT,
+                                  int ldvt,
+                                  cuComplex *work,
+                                  int lwork,
+                                  float *rwork,
+                                  int *info) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvd(cusolverDnHandle_t handle,
+                                  signed char jobu,
+                                  signed char jobvt,
+                                  int m,
+                                  int n,
+                                  cuDoubleComplex *A,
+                                  int lda,
+                                  double *S,
+                                  cuDoubleComplex *U,
+                                  int ldu,
+                                  cuDoubleComplex *VT,
+                                  int ldvt,
+                                  cuDoubleComplex *work,
+                                  int lwork,
+                                  double *rwork,
+                                  int *info) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCreateGesvdjInfo(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDestroyGesvdjInfo(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXgesvdjSetTolerance(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXgesvdjSetMaxSweeps(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXgesvdjSetSortEig(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXgesvdjGetResidual(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXgesvdjGetSweeps(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgesvdj_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvdj_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvdj_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvdj_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgesvdj(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvdj(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvdj(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvdj(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgesvdjBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvdjBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvdjBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvdjBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgesvdjBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvdjBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvdjBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvdjBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgesvdaStridedBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvdaStridedBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvdaStridedBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvdaStridedBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSgesvdaStridedBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDgesvdaStridedBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCgesvdaStridedBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZgesvdaStridedBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZZgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZCgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZYgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZKgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCCgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCYgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCKgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDDgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDSgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDXgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDHgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSSgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSXgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSHgels_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZZgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZCgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZYgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZKgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCCgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCYgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCKgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDDgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDSgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDXgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDHgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSSgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSXgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSHgels(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSsyevd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDsyevd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCheevd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZheevd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSsyevd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDsyevd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCheevd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZheevd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCreateSyevjInfo(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDestroySyevjInfo(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXsyevjSetTolerance(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXsyevjSetMaxSweeps(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXsyevjSetSortEig(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXsyevjGetResidual(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXsyevjGetSweeps(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSsyevj_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDsyevj_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCheevj_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZheevj_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSsyevj(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDsyevj(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCheevj(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZheevj(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSsyevjBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDsyevjBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCheevjBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZheevjBatched_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnSsyevjBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnDsyevjBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnCheevjBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnZheevjBatched(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+// Functions added in CUDA 10.2
+cusolverStatus_t cusolverDnZZgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZCgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZKgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCCgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCKgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDDgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDSgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDHgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSSgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSHgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZZgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZCgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZKgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCCgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCKgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDDgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDSgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDHgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSSgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSHgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+// Functions added in CUDA 11.0
+cusolverStatus_t cusolverDnCreateParams(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDestroyParams(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZYgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCYgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDXgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSXgesv_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnZYgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnCYgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnDXgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+cusolverStatus_t cusolverDnSXgesv(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpCreate(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpDestroy(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpScsrlsvqr(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpDcsrlsvqr(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpCcsrlsvqr(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpZcsrlsvqr(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpScsrlsvchol(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpDcsrlsvchol(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpCcsrlsvchol(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpZcsrlsvchol(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpScsreigvsi(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpDcsreigvsi(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpCcsreigvsi(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverSpZcsreigvsi(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+// Functions added in CUDA 11.1
+cusolverStatus_t cusolverDnXsyevd_bufferSize(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+cusolverStatus_t cusolverDnXsyevd(...) {
+    return CUSOLVER_STATUS_SUCCESS;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_CUSOLVER_H
diff --git a/cupy_backends/stub/cupy_cusparse.h b/cupy_backends/stub/cupy_cusparse.h
new file mode 100644
index 0000000..7982f5b
--- /dev/null
+++ b/cupy_backends/stub/cupy_cusparse.h
@@ -0,0 +1,1253 @@
+// This file is a stub header file of cusparse for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUSPARSE_H
+#define INCLUDE_GUARD_STUB_CUPY_CUSPARSE_H
+
+extern "C" {
+
+typedef enum {} cusparseIndexBase_t;
+typedef enum { CUSPARSE_STATUS_SUCCESS=0, } cusparseStatus_t;  // one-liner for easy parsing
+
+typedef void* cusparseHandle_t;
+typedef void* cusparseMatDescr_t;
+typedef void* csrsv2Info_t;
+typedef void* csrsm2Info_t;
+typedef void* csric02Info_t;
+typedef void* bsric02Info_t;
+typedef void* csrilu02Info_t;
+typedef void* bsrilu02Info_t;
+typedef void* csrgemm2Info_t;
+
+typedef enum {} cusparseMatrixType_t;
+typedef enum {} cusparseFillMode_t;
+typedef enum {} cusparseDiagType_t;
+typedef enum {} cusparseOperation_t;
+typedef enum {} cusparsePointerMode_t;
+typedef enum {} cusparseAction_t;
+typedef enum {} cusparseDirection_t;
+typedef enum {} cusparseAlgMode_t;
+typedef enum {} cusparseSolvePolicy_t;
+typedef enum {} cusparseSpMatAttribute_t;
+
+// Version
+cusparseStatus_t cusparseGetVersion(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// Error handling
+const char* cusparseGetErrorName(...) {
+    return NULL;
+}
+
+const char* cusparseGetErrorString(...) {
+    return NULL;
+}
+
+// cuSPARSE Helper Function
+cusparseStatus_t cusparseCreate(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateMatDescr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroy(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyMatDescr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSetMatIndexBase(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSetMatType(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSetMatFillMode(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSetMatDiagType(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSetPointerMode(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatSetAttribute(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// Stream
+cusparseStatus_t cusparseSetStream(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseGetStream(...) {
+   return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE Level1 Function
+cusparseStatus_t cusparseSgthr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgthr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgthr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgthr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE Level2 Function
+cusparseStatus_t cusparseScsrmv(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrmv(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrmv(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrmv(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsrmvEx_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsrmvEx(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCsrsv2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyCsrsv2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsv2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsv2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsv2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsv2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsv2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsv2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsv2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsv2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsv2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrsv2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrsv2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrsv2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrsv2_zeroPivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE Level3 Function
+cusparseStatus_t cusparseScsrmm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrmm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrmm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrmm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrmm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrmm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrmm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrmm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCsrsm2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseDestroyCsrsm2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseDcsrsm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseCcsrsm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseZcsrsm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsm2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseDcsrsm2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseCcsrsm2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseZcsrsm2_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrsm2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseDcsrsm2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseCcsrsm2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+cusparseStatus_t cusparseZcsrsm2_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrsm2_zeroPivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE Extra Function
+cusparseStatus_t cusparseXcsrgeamNnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgeam(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgeam(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgeam(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgeam(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgeam2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgeam2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgeam2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgeam2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrgeam2Nnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgeam2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgeam2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgeam2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgeam2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrgemmNnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgemm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgemm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgemm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgemm(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCsrgemm2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyCsrgemm2Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgemm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgemm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgemm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgemm2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrgemm2Nnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrgemm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrgemm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrgemm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrgemm2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE Format Convrsion
+cusparseStatus_t cusparseXcoo2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsc2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsc2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsc2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsc2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsr2coo(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsr2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsr2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsr2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsr2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+
+cusparseStatus_t cusparseScsr2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsr2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsr2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsr2dense(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSdense2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDdense2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCdense2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZdense2csc(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSdense2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDdense2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCdense2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZdense2csr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSnnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCnnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZnnz(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateIdentityPermutation(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcoosort_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcoosortByRow(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcoosortByColumn(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrsort_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrsort(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcscsort_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcscsort(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// cuSPARSE PRECONDITIONERS
+
+cusparseStatus_t cusparseCreateCsrilu02Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyCsrilu02Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateBsrilu02Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyBsrilu02Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCsric02Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyCsric02Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateBsric02Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyBsric02Info(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrilu02_numericBoost(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrilu02_numericBoost(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrilu02_numericBoost(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrilu02_numericBoost(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsrilu02_zeroPivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrilu02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrilu02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrilu02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrilu02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrilu02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrilu02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrilu02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrilu02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsrilu02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsrilu02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsrilu02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsrilu02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSbsrilu02_numericBoost(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDbsrilu02_numericBoost(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCbsrilu02_numericBoost(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZbsrilu02_numericBoost(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXbsrilu02_zeroPivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSbsrilu02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDbsrilu02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCbsrilu02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZbsrilu02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSbsrilu02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDbsrilu02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCbsrilu02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZbsrilu02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSbsrilu02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDbsrilu02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCbsrilu02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZbsrilu02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXcsric02_zeroPivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsric02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsric02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsric02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsric02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsric02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsric02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsric02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsric02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsric02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsric02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsric02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsric02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseXbsric02_zeroPivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSbsric02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDbsric02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCbsric02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZbsric02_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSbsric02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDbsric02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCbsric02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZbsric02_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSbsric02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDbsric02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCbsric02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZbsric02(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgtsv2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgtsv2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgtsv2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgtsv2_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgtsv2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgtsv2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgtsv2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgtsv2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgtsv2_nopivot_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgtsv2_nopivot_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgtsv2_nopivot_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgtsv2_nopivot_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgtsv2_nopivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgtsv2_nopivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgtsv2_nopivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgtsv2_nopivot(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgtsv2StridedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgtsv2StridedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgtsv2StridedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgtsv2StridedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgtsv2StridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgtsv2StridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgtsv2StridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgtsv2StridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgtsvInterleavedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgtsvInterleavedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgtsvInterleavedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgtsvInterleavedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgtsvInterleavedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgtsvInterleavedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgtsvInterleavedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgtsvInterleavedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgpsvInterleavedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgpsvInterleavedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgpsvInterleavedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgpsvInterleavedBatch_bufferSizeExt(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSgpsvInterleavedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDgpsvInterleavedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCgpsvInterleavedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZgpsvInterleavedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+#define CUSPARSE_VERSION -1
+
+// cuSPARSE generic API
+typedef void* cusparseSpVecDescr_t;
+typedef void* cusparseDnVecDescr_t;
+typedef void* cusparseSpMatDescr_t;
+typedef void* cusparseDnMatDescr_t;
+typedef void* cusparseSpSMDescr_t;
+
+typedef enum {} cusparseIndexType_t;
+typedef enum {} cusparseFormat_t;
+typedef enum {} cusparseOrder_t;
+typedef enum {} cusparseSpMVAlg_t;
+typedef enum {} cusparseSpSMAlg_t;
+typedef enum {} cusparseSpMMAlg_t;
+typedef enum {} cusparseSparseToDenseAlg_t;
+typedef enum {} cusparseDenseToSparseAlg_t;
+
+cusparseStatus_t cusparseCreateSpVec(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroySpVec(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVecGet(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVecGetIndexBase(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVecGetValues(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVecSetValues(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCoo(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCooAoS(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateCsr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroySpMat(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCooGet(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCooAoSGet(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsrGet(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsrSetPointers(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetFormat(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetIndexBase(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetValues(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatSetValues(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatGetStridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMatSetStridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateDnVec(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyDnVec(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnVecGet(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnVecGetValues(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnVecSetValues(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCreateDnMat(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDestroyDnMat(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatGet(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatGetValues(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatSetValues(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatGetStridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnMatSetStridedBatch(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVV_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpVV(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMV_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMV(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpSM_createDescr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpSM_destroyDescr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpSM_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpSM_analysis(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+// See cusparse.pyx for a comment
+cusparseStatus_t cusparseSpSM_solve(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMM_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpMM(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseConstrainedGeMM_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseConstrainedGeMM(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+typedef enum {} cusparseCsr2CscAlg_t;
+
+cusparseStatus_t cusparseCsr2cscEx2_bufferSize(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCsr2cscEx2(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+typedef void* cusparseSpGEMMDescr_t;
+typedef enum {} cusparseSpGEMMAlg_t;
+
+cusparseStatus_t cusparseSpGEMM_createDescr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpGEMM_destroyDescr(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpGEMM_workEstimation(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpGEMM_compute(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseSpGEMM_copy(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseGather(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Definitions are for compatibility
+///////////////////////////////////////////////////////////////////////////////
+
+cusparseStatus_t cusparseSnnz_compress(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDnnz_compress(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCnnz_compress(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZnnz_compress(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseScsr2csr_compress(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseDcsr2csr_compress(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseCcsr2csr_compress(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseZcsr2csr_compress(...) {
+  return CUSPARSE_STATUS_SUCCESS;
+}
+
+}  // extern "C"
+
+
+#endif  // INCLUDE_GUARD_STUB_CUPY_CUSPARSE_H
diff --git a/cupy_backends/stub/cupy_cusparselt.h b/cupy_backends/stub/cupy_cusparselt.h
new file mode 100644
index 0000000..01b7667
--- /dev/null
+++ b/cupy_backends/stub/cupy_cusparselt.h
@@ -0,0 +1,137 @@
+// Stub header file for cuSPARSELt
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUSPARSELT_H
+#define INCLUDE_GUARD_STUB_CUPY_CUSPARSELT_H
+
+#define CUSPARSELT_VERSION -1
+
+extern "C" {
+
+typedef enum {
+    CUSPARSE_STATUS_SUCCESS=0,
+} cusparseStatus_t;
+typedef enum {} cudaDataType;
+typedef enum {} cusparseOrder_t;
+typedef enum {} cusparseOperation_t;
+typedef enum {} cusparseLtSparsity_t;
+typedef enum {} cusparseLtMatDescAttribute_t;
+typedef enum {} cusparseComputeType;
+typedef enum {} cusparseLtMatmulDescAttribute_t;
+typedef enum {} cusparseLtMatmulAlg_t;
+typedef enum {} cusparseLtMatmulAlgAttribute_t;
+typedef enum {} cusparseLtPruneAlg_t;
+
+typedef void* cudaStream_t;
+typedef void* cusparseLtHandle_t;
+typedef void* cusparseLtMatDescriptor_t;
+typedef void* cusparseLtMatmulDescriptor_t;
+typedef void* cusparseLtMatmulAlgSelection_t;
+typedef void* cusparseLtMatmulPlan_t;
+
+cusparseStatus_t cusparseLtInit(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtDestroy(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtDenseDescriptorInit(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtStructuredDescriptorInit(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatDescriptorDestroy(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatDescSetAttribute(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatDescGetAttribute(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulDescriptorInit(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulDescSetAttribute(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulDescGetAttribute(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulAlgSelectionInit(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulAlgSetAttribute(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulAlgGetAttribute(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulGetWorkspace(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulPlanInit(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulPlanDestroy(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmul(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtMatmulSearch(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtSpMMAPrune(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtSpMMAPruneCheck(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtSpMMAPrune2(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtSpMMAPruneCheck2(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtSpMMACompressedSize(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtSpMMACompress(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtSpMMACompressedSize2(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+cusparseStatus_t cusparseLtSpMMACompress2(...) {
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+} // extern "C"
+
+#endif  // INCLUDE_GUARD_STUB_CUPY_CUSPARSE_H
diff --git a/cupy_backends/stub/cupy_cutensor.h b/cupy_backends/stub/cupy_cutensor.h
new file mode 100644
index 0000000..1a67638
--- /dev/null
+++ b/cupy_backends/stub/cupy_cutensor.h
@@ -0,0 +1,105 @@
+// Stub header file of cuTENSOR
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_CUTENSOR_H
+#define INCLUDE_GUARD_STUB_CUPY_CUTENSOR_H
+
+#define CUTENSOR_VERSION 0
+
+#include "cupy_cuda_common.h"
+
+extern "C" {
+
+    typedef enum {} cudaDataType_t;
+
+    typedef enum {
+	CUTENSOR_STATUS_SUCCESS = 0,
+    } cutensorStatus_t;
+
+    typedef enum {} cutensorAlgo_t;
+    typedef enum {} cutensorOperator_t;
+    typedef enum {} cutensorWorksizePreference_t;
+    typedef enum {} cutensorComputeType_t;
+
+    typedef void* cutensorHandle_t;
+    typedef void* cutensorTensorDescriptor_t;
+    typedef void* cutensorContractionDescriptor_t;
+    typedef void* cutensorContractionFind_t;
+    typedef void* cutensorContractionPlan_t;
+
+    cutensorStatus_t cutensorInit(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorInitTensorDescriptor(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorElementwiseTrinary(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorElementwiseBinary(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorPermutation(...) {
+    return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorInitContractionDescriptor(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorInitContractionFind(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorInitContractionPlan(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorContraction(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorContractionGetWorkspaceSize(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    // Deprecated in cuTENSOR 1.5.0
+    cutensorStatus_t cutensorContractionGetWorkspace(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorContractionMaxAlgos(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorReduction(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorReductionGetWorkspaceSize(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    // Deprecated in cuTENSOR 1.5.0
+    cutensorStatus_t cutensorReductionGetWorkspace(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    cutensorStatus_t cutensorGetAlignmentRequirement(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    size_t cutensorGetVersion(...) {
+	return CUTENSOR_STATUS_SUCCESS;
+    }
+
+    const char* cutensorGetErrorString(...) {
+	return NULL;
+    }
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_CUTENSOR_H
diff --git a/cupy_backends/stub/cupy_nccl.h b/cupy_backends/stub/cupy_nccl.h
new file mode 100644
index 0000000..68726cc
--- /dev/null
+++ b/cupy_backends/stub/cupy_nccl.h
@@ -0,0 +1,119 @@
+// This file is a stub header file of nccl for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_NCCL_H
+#define INCLUDE_GUARD_STUB_CUPY_NCCL_H
+
+#define NCCL_MAJOR 0
+#define NCCL_MINOR 0
+#define NCCL_PATCH 0
+
+extern "C" {
+
+typedef struct ncclComm* ncclComm_t;
+
+enum {
+    NCCL_UNIQUE_ID_BYTES = 128
+};
+typedef struct {
+    char internal[NCCL_UNIQUE_ID_BYTES];
+} ncclUniqueId;
+
+typedef enum {
+    ncclSuccess
+} ncclResult_t;
+
+typedef enum {} ncclRedOp_t;
+
+typedef enum {
+    ncclChar       = 0,
+    ncclInt        = 1,
+    ncclHalf       = 2,
+    ncclFloat      = 3,
+    ncclDouble     = 4,
+    ncclInt64      = 5,
+    ncclUint64     = 6,
+    nccl_NUM_TYPES = 7 } ncclDataType_t;
+
+const char* ncclGetErrorString(...) {
+    return "";
+}
+
+ncclResult_t  ncclCommGetAsyncError(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclGetUniqueId(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclCommInitRank(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclCommInitAll(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclGroupStart(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclGroupEnd(...) {
+    return ncclSuccess;
+}
+
+void ncclCommDestroy(...) {
+}
+
+void ncclCommAbort(...) {
+}
+
+ncclResult_t ncclCommCuDevice(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclCommUserRank(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclCommCount(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclAllReduce(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclReduce(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclBroadcast(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclBcast(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclReduceScatter(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclAllGather(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclSend(...) {
+    return ncclSuccess;
+}
+
+ncclResult_t ncclRecv(...) {
+    return ncclSuccess;
+}
+
+typedef struct CUstream_st *cudaStream_t;
+
+}  // extern "C"
+
+#endif  // INCLUDE_GUARD_STUB_CUPY_NCCL_H
diff --git a/cupy_backends/stub/cupy_nvrtc.h b/cupy_backends/stub/cupy_nvrtc.h
new file mode 100644
index 0000000..8dfa3f0
--- /dev/null
+++ b/cupy_backends/stub/cupy_nvrtc.h
@@ -0,0 +1,69 @@
+// This file is a stub header file of nvrtc for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_NVRTC_H
+#define INCLUDE_GUARD_STUB_CUPY_NVRTC_H
+
+
+extern "C" {
+
+typedef enum {
+    NVRTC_SUCCESS = 0,
+} nvrtcResult;
+
+typedef struct _nvrtcProgram *nvrtcProgram;
+
+const char *nvrtcGetErrorString(...) {
+    return NULL;
+}
+
+nvrtcResult nvrtcVersion(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcCreateProgram(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcDestroyProgram(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcCompileProgram(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcGetPTXSize(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcGetPTX(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcGetCUBINSize(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcGetCUBIN(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcGetProgramLogSize(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcGetProgramLog(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcAddNameExpression(...) {
+    return NVRTC_SUCCESS;
+}
+
+nvrtcResult nvrtcGetLoweredName(...) {
+    return NVRTC_SUCCESS;
+}
+
+}
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_NVRTC_H
diff --git a/cupy_backends/stub/cupy_nvtx.h b/cupy_backends/stub/cupy_nvtx.h
new file mode 100644
index 0000000..cf744e0
--- /dev/null
+++ b/cupy_backends/stub/cupy_nvtx.h
@@ -0,0 +1,65 @@
+// This file is a stub header file of cuda for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_NVTX_H
+#define INCLUDE_GUARD_STUB_CUPY_NVTX_H
+
+extern "C" {
+
+typedef enum nvtxColorType_t
+{
+    NVTX_COLOR_UNKNOWN  = 0,
+    NVTX_COLOR_ARGB     = 1
+} nvtxColorType_t;
+
+typedef enum nvtxMessageType_t
+{
+    NVTX_MESSAGE_UNKNOWN          = 0,
+    NVTX_MESSAGE_TYPE_ASCII       = 1,
+    NVTX_MESSAGE_TYPE_UNICODE     = 2,
+} nvtxMessageType_t;
+
+typedef union nvtxMessageValue_t
+{
+    const char* ascii;
+    const wchar_t* unicode;
+} nvtxMessageValue_t;
+
+typedef struct nvtxEventAttributes_v1
+{
+    uint16_t version;
+    uint16_t size;
+    uint32_t category;
+    int32_t colorType;
+    uint32_t color;
+    int32_t payloadType;
+    int32_t reserved0;
+    union payload_t
+    {
+        uint64_t ullValue;
+        int64_t llValue;
+        double dValue;
+    } payload;
+    int32_t messageType;
+    nvtxMessageValue_t message;
+} nvtxEventAttributes_v1;
+
+typedef nvtxEventAttributes_v1 nvtxEventAttributes_t;
+
+void nvtxMarkEx(...) {
+}
+
+int nvtxRangePushEx(...) {
+    return 0;
+}
+
+uint64_t nvtxRangeStartEx(...) {
+    return 0;
+}
+
+void nvtxRangeEnd(...) {
+}
+
+} // extern "C"
+
+
+#endif
diff --git a/cupy_backends/stub/cupy_profiler.h b/cupy_backends/stub/cupy_profiler.h
new file mode 100644
index 0000000..cdb92db
--- /dev/null
+++ b/cupy_backends/stub/cupy_profiler.h
@@ -0,0 +1,26 @@
+// This file is a stub header file of cuda for Read the Docs.
+
+#ifndef INCLUDE_GUARD_STUB_CUPY_PROFILER_H
+#define INCLUDE_GUARD_STUB_CUPY_PROFILER_H
+
+#include "cupy_cuda_common.h"
+
+extern "C" {
+
+typedef enum {} cudaOutputMode_t;
+
+cudaError_t cudaProfilerInitialize(...) {
+  return cudaSuccess;
+}
+
+cudaError_t cudaProfilerStart() {
+  return cudaSuccess;
+}
+
+cudaError_t cudaProfilerStop() {
+  return cudaSuccess;
+}
+
+} // extern "C"
+
+#endif // #ifndef INCLUDE_GUARD_STUB_CUPY_PROFILER_H
diff --git a/cupyx/__init__.py b/cupyx/__init__.py
new file mode 100644
index 0000000..3d86374
--- /dev/null
+++ b/cupyx/__init__.py
@@ -0,0 +1,25 @@
+# "NOQA" to suppress flake8 warning
+from cupyx._rsqrt import rsqrt  # NOQA
+from cupyx._runtime import get_runtime_info  # NOQA
+from cupyx._scatter import scatter_add  # NOQA
+from cupyx._scatter import scatter_max  # NOQA
+from cupyx._scatter import scatter_min  # NOQA
+
+from cupyx import linalg  # NOQA
+from cupyx import time  # NOQA
+from cupyx import scipy  # NOQA
+from cupyx import optimizing  # NOQA
+from cupyx import lapack  # NOQA
+
+from cupyx._ufunc_config import errstate  # NOQA
+from cupyx._ufunc_config import geterr  # NOQA
+from cupyx._ufunc_config import seterr  # NOQA
+from cupy._core.syncdetect import allow_synchronize  # NOQA
+from cupy._core.syncdetect import DeviceSynchronized  # NOQA
+
+from cupyx._pinned_array import empty_pinned  # NOQA
+from cupyx._pinned_array import empty_like_pinned  # NOQA
+from cupyx._pinned_array import zeros_pinned  # NOQA
+from cupyx._pinned_array import zeros_like_pinned  # NOQA
+
+from cupyx._gufunc import GeneralizedUFunc  # NOQA
diff --git a/cupyx/_gufunc.py b/cupyx/_gufunc.py
new file mode 100644
index 0000000..a38b361
--- /dev/null
+++ b/cupyx/_gufunc.py
@@ -0,0 +1,5 @@
+import cupy
+
+
+class GeneralizedUFunc(cupy._core._gufuncs._GUFunc):
+    __doc__ = cupy._core._gufuncs._GUFunc.__doc__
diff --git a/cupyx/_pinned_array.py b/cupyx/_pinned_array.py
new file mode 100644
index 0000000..f9b8c9c
--- /dev/null
+++ b/cupyx/_pinned_array.py
@@ -0,0 +1,141 @@
+import numpy
+
+from cupy import cuda
+from cupy._creation.basic import _new_like_order_and_strides
+from cupy._core import internal
+
+
+def _update_shape(a, shape):
+    if shape is None and a is not None:
+        shape = a.shape
+    elif isinstance(shape, int):
+        shape = (shape,)
+    else:
+        shape = tuple(shape)
+    return shape
+
+
+def empty_pinned(shape, dtype=float, order='C'):
+    """Returns a new, uninitialized NumPy array with the given shape
+    and dtype.
+
+    This is a convenience function which is just :func:`numpy.empty`,
+    except that the underlying memory is pinned/pagelocked.
+
+    Args:
+        shape (int or tuple of ints): Dimensionalities of the array.
+        dtype: Data type specifier.
+        order ({'C', 'F'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Returns:
+        numpy.ndarray: A new array with elements not initialized.
+
+    .. seealso:: :func:`numpy.empty`
+
+    """
+    shape = _update_shape(None, shape)
+    nbytes = internal.prod(shape) * numpy.dtype(dtype).itemsize
+    mem = cuda.alloc_pinned_memory(nbytes)
+    out = numpy.ndarray(shape, dtype=dtype, buffer=mem, order=order)
+    return out
+
+
+def empty_like_pinned(a, dtype=None, order='K', subok=None, shape=None):
+    """Returns a new, uninitialized NumPy array with the same shape and dtype
+    as those of the given array.
+
+    This is a convenience function which is just :func:`numpy.empty_like`,
+    except that the underlying memory is pinned/pagelocked.
+
+    This function currently does not support ``subok`` option.
+
+    Args:
+        a (numpy.ndarray or cupy.ndarray): Base array.
+        dtype: Data type specifier. The data type of ``a`` is used by default.
+        order ({'C', 'F', 'A', or 'K'}): Overrides the memory layout of the
+            result. ``'C'`` means C-order, ``'F'`` means F-order, ``'A'`` means
+            ``'F'`` if ``a`` is Fortran contiguous, ``'C'`` otherwise.
+            ``'K'`` means match the layout of ``a`` as closely as possible.
+        subok: Not supported yet, must be None.
+        shape (int or tuple of ints): Overrides the shape of the result. If
+            ``order='K'`` and the number of dimensions is unchanged, will try
+            to keep order, otherwise, ``order='C'`` is implied.
+
+    Returns:
+        numpy.ndarray: A new array with same shape and dtype of ``a`` with
+        elements not initialized.
+
+    .. seealso:: :func:`numpy.empty_like`
+
+    """
+    # We're kinda duplicating the code here because order='K' needs special
+    # treatment: strides need to be computed
+    if subok is not None:
+        raise TypeError('subok is not supported yet')
+    if dtype is None:
+        dtype = a.dtype
+    shape = _update_shape(a, shape)
+    order, strides, _ = _new_like_order_and_strides(
+        a, dtype, order, shape, get_memptr=False)
+    nbytes = internal.prod(shape) * numpy.dtype(dtype).itemsize
+    mem = cuda.alloc_pinned_memory(nbytes)
+    out = numpy.ndarray(shape, dtype=dtype, buffer=mem,
+                        strides=strides, order=order)
+    return out
+
+
+def zeros_pinned(shape, dtype=float, order='C'):
+    """Returns a new, zero-initialized NumPy array with the given shape
+    and dtype.
+
+    This is a convenience function which is just :func:`numpy.zeros`,
+    except that the underlying memory is pinned/pagelocked.
+
+    Args:
+        shape (int or tuple of ints): Dimensionalities of the array.
+        dtype: Data type specifier.
+        order ({'C', 'F'}): Row-major (C-style) or column-major
+            (Fortran-style) order.
+
+    Returns:
+        numpy.ndarray: An array filled with zeros.
+
+    .. seealso:: :func:`numpy.zeros`
+
+    """
+    out = empty_pinned(shape, dtype, order)
+    numpy.copyto(out, 0, casting='unsafe')
+    return out
+
+
+def zeros_like_pinned(a, dtype=None, order='K', subok=None, shape=None):
+    """Returns a new, zero-initialized NumPy array with the same shape and dtype
+    as those of the given array.
+
+    This is a convenience function which is just :func:`numpy.zeros_like`,
+    except that the underlying memory is pinned/pagelocked.
+
+    This function currently does not support ``subok`` option.
+
+    Args:
+        a (numpy.ndarray or cupy.ndarray): Base array.
+        dtype: Data type specifier. The dtype of ``a`` is used by default.
+        order ({'C', 'F', 'A', or 'K'}): Overrides the memory layout of the
+            result. ``'C'`` means C-order, ``'F'`` means F-order, ``'A'`` means
+            ``'F'`` if ``a`` is Fortran contiguous, ``'C'`` otherwise.
+            ``'K'`` means match the layout of ``a`` as closely as possible.
+        subok: Not supported yet, must be None.
+        shape (int or tuple of ints): Overrides the shape of the result. If
+            ``order='K'`` and the number of dimensions is unchanged, will try
+            to keep order, otherwise, ``order='C'`` is implied.
+
+    Returns:
+        numpy.ndarray: An array filled with zeros.
+
+    .. seealso:: :func:`numpy.zeros_like`
+
+    """  # NOQA
+    out = empty_like_pinned(a, dtype, order, subok, shape)
+    numpy.copyto(out, 0, casting='unsafe')
+    return out
diff --git a/cupyx/_rsqrt.py b/cupyx/_rsqrt.py
new file mode 100644
index 0000000..b5d958a
--- /dev/null
+++ b/cupyx/_rsqrt.py
@@ -0,0 +1,7 @@
+from cupy._core.core import create_ufunc
+
+rsqrt = create_ufunc(
+    'cupy_rsqrt',
+    ('e->e', 'f->f', 'd->d', 'F->F', 'D->D'),
+    'out0 = rsqrt(in0)',
+    doc='''Returns the reciprocal square root.''')
diff --git a/cupyx/_runtime.py b/cupyx/_runtime.py
new file mode 100644
index 0000000..cdc552d
--- /dev/null
+++ b/cupyx/_runtime.py
@@ -0,0 +1,320 @@
+import inspect
+import io
+import os
+import platform
+import warnings
+
+import numpy
+
+import cupy
+import cupy_backends
+
+
+is_hip = cupy_backends.cuda.api.runtime.is_hip
+
+
+def _eval_or_error(func, errors):
+    # Evaluates `func` and return the result.
+    # If an error specified by `errors` occured, it returns a string
+    # representing the error.
+    try:
+        return func()
+    except errors as e:
+        return repr(e)
+
+
+class _InstallInfo(object):
+
+    # TODO(niboshi): Add is_binary_distribution
+
+    def __init__(self):
+        cupy_package_root = self._get_cupy_package_root()
+        if cupy_package_root is not None:
+            data_root = os.path.join(cupy_package_root, '.data')
+            data_paths = {
+                'lib': _dir_or_none(os.path.join(data_root, 'lib')),
+                'include': _dir_or_none(os.path.join(data_root, 'include')),
+            }
+        else:
+            data_paths = {
+                'lib': None,
+                'include': None,
+            }
+
+        self.cupy_package_root = cupy_package_root
+        self.data_paths = data_paths
+
+    def get_data_path(self, data_type):
+        if data_type not in self.data_paths:
+            raise ValueError('Invalid data type: {}'.format(data_type))
+        return self.data_paths[data_type]
+
+    def _get_cupy_package_root(self):
+        try:
+            cupy_path = inspect.getfile(cupy)
+        except TypeError:
+            return None
+        return os.path.dirname(cupy_path)
+
+
+class _RuntimeInfo:
+
+    cupy_version = None
+    cuda_path = None
+
+    # CUDA Driver
+    cuda_build_version = None
+    cuda_driver_version = None
+
+    # CUDA Runtime
+    cuda_runtime_version = None
+
+    # CUDA Toolkit
+    cublas_version = None
+    cufft_version = None
+    curand_version = None
+    cusolver_version = None
+    cusparse_version = None
+    nvrtc_version = None
+    thrust_version = None
+
+    # Optional Libraries
+    cudnn_build_version = None
+    cudnn_version = None
+    nccl_build_version = None
+    nccl_runtime_version = None
+    cub_build_version = None
+    jitify_build_version = None
+    cutensor_version = None
+    cusparselt_version = None
+    cython_build_version = None
+    cython_version = None
+
+    numpy_version = None
+    scipy_version = None
+
+    def __init__(self, *, full=True):
+        self.cupy_version = cupy.__version__
+
+        if not is_hip:
+            self.cuda_path = cupy.cuda.get_cuda_path()
+        else:
+            self.cuda_path = cupy._environment.get_rocm_path()
+
+        if not is_hip:
+            self.nvcc_path = cupy._environment.get_nvcc_path()
+        else:
+            self.nvcc_path = cupy._environment.get_hipcc_path()
+
+        # CUDA Driver
+        self.cuda_build_version = str(cupy.cuda.driver.get_build_version())
+        if cupy.cuda.driver._is_cuda_python():
+            import cuda
+            self.cuda_build_version += f' (CUDA Python: {cuda.__version__})'
+        self.cuda_driver_version = _eval_or_error(
+            cupy.cuda.runtime.driverGetVersion,
+            cupy.cuda.runtime.CUDARuntimeError)
+
+        # CUDA Runtime
+        self.cuda_runtime_version = _eval_or_error(
+            cupy.cuda.runtime.runtimeGetVersion,
+            cupy.cuda.runtime.CUDARuntimeError)
+
+        # cuBLAS
+        self.cublas_version = '(available)'
+        if full:
+            self.cublas_version = _eval_or_error(
+                lambda: cupy.cuda.cublas.getVersion(
+                    cupy.cuda.device.get_cublas_handle()),
+                cupy.cuda.cublas.CUBLASError)
+
+        # cuFFT
+        self.cufft_version = _eval_or_error(
+            cupy.cuda.cufft.getVersion,
+            cupy.cuda.cufft.CuFFTError)
+
+        # cuRAND
+        self.curand_version = _eval_or_error(
+            cupy.cuda.curand.getVersion,
+            cupy.cuda.curand.CURANDError)
+
+        # cuSOLVER
+        self.cusolver_version = _eval_or_error(
+            cupy.cuda.cusolver._getVersion,
+            cupy.cuda.cusolver.CUSOLVERError)
+
+        # cuSPARSE
+        self.cusparse_version = '(available)'
+        if full:
+            self.cusparse_version = _eval_or_error(
+                lambda: cupy.cuda.cusparse.getVersion(
+                    cupy.cuda.device.get_cusparse_handle()),
+                cupy.cuda.cusparse.CuSparseError)
+
+        # NVRTC
+        self.nvrtc_version = _eval_or_error(
+            cupy.cuda.nvrtc.getVersion,
+            cupy.cuda.nvrtc.NVRTCError)
+
+        # Thrust
+        try:
+            import cupy.cuda.thrust as thrust
+            self.thrust_version = thrust.get_build_version()
+        except ImportError:
+            pass
+
+        # cuDNN
+        if cupy._environment._can_attempt_preload('cudnn'):
+            if full:
+                cupy._environment._preload_library('cudnn')
+            else:
+                self.cudnn_build_version = (
+                    '(not loaded; try `import cupy.cuda.cudnn` first)')
+                self.cudnn_version = self.cudnn_build_version
+        try:
+            import cupy_backends.cuda.libs.cudnn as cudnn
+            self.cudnn_build_version = cudnn.get_build_version()
+            self.cudnn_version = _eval_or_error(
+                cudnn.getVersion, cudnn.CuDNNError)
+        except ImportError:
+            pass
+
+        # NCCL
+        try:
+            import cupy_backends.cuda.libs.nccl as nccl
+            self.nccl_build_version = nccl.get_build_version()
+            nccl_runtime_version = nccl.get_version()
+            if nccl_runtime_version == 0:
+                nccl_runtime_version = '(unknown)'
+            self.nccl_runtime_version = nccl_runtime_version
+        except ImportError:
+            pass
+
+        # CUB
+        self.cub_build_version = cupy.cuda.cub.get_build_version()
+
+        try:
+            import cupy.cuda.jitify as jitify
+            self.jitify_build_version = jitify.get_build_version()
+        except ImportError:
+            pass
+
+        # cuTENSOR
+        try:
+            import cupy_backends.cuda.libs.cutensor as cutensor
+            self.cutensor_version = cutensor.get_version()
+        except ImportError:
+            pass
+
+        # cuSparseLT
+        try:
+            import cupy_backends.cuda.libs.cusparselt as cusparselt
+            self.cusparselt_version = cusparselt.get_build_version()
+        except ImportError:
+            pass
+
+        # Cython
+        self.cython_build_version = cupy._util.cython_build_ver
+        try:
+            import Cython
+            self.cython_version = Cython.__version__
+        except ImportError:
+            pass
+
+        # NumPy
+        self.numpy_version = numpy.version.full_version
+
+        # SciPy
+        try:
+            import scipy
+            self.scipy_version = scipy.version.full_version
+        except ImportError:
+            pass
+
+    def __str__(self):
+        records = [
+            ('OS',  platform.platform()),
+            ('Python Version', platform.python_version()),
+            ('CuPy Version', self.cupy_version),
+            ('CuPy Platform', 'NVIDIA CUDA' if not is_hip else 'AMD ROCm'),
+            ('NumPy Version', self.numpy_version),
+            ('SciPy Version', self.scipy_version),
+            ('Cython Build Version', self.cython_build_version),
+            ('Cython Runtime Version', self.cython_version),
+            ('CUDA Root', self.cuda_path),
+            ('hipcc PATH' if is_hip else 'nvcc PATH', self.nvcc_path),
+
+            ('CUDA Build Version', self.cuda_build_version),
+            ('CUDA Driver Version', self.cuda_driver_version),
+
+            ('CUDA Runtime Version', self.cuda_runtime_version),
+        ]
+
+        records += [
+            ('cuBLAS Version', self.cublas_version),
+            ('cuFFT Version', self.cufft_version),
+            ('cuRAND Version', self.curand_version),
+            ('cuSOLVER Version', self.cusolver_version),
+            ('cuSPARSE Version', self.cusparse_version),
+            ('NVRTC Version', self.nvrtc_version),
+            ('Thrust Version', self.thrust_version),
+            ('CUB Build Version', self.cub_build_version),
+            ('Jitify Build Version', self.jitify_build_version),
+        ]
+
+        records += [
+            ('cuDNN Build Version', self.cudnn_build_version),
+            ('cuDNN Version', self.cudnn_version),
+            ('NCCL Build Version', self.nccl_build_version),
+            ('NCCL Runtime Version', self.nccl_runtime_version),
+            ('cuTENSOR Version', self.cutensor_version),
+            ('cuSPARSELt Build Version', self.cusparselt_version),
+        ]
+
+        device_count = 0
+        try:
+            device_count = cupy.cuda.runtime.getDeviceCount()
+        except cupy.cuda.runtime.CUDARuntimeError as e:
+            if 'ErrorNoDevice' not in e.args[0]:
+                warnings.warn(f'Failed to detect number of GPUs: {e}')
+            # No GPU devices available.
+        for device_id in range(device_count):
+            with cupy.cuda.Device(device_id) as device:
+                props = cupy.cuda.runtime.getDeviceProperties(device_id)
+                name = ('Device {} Name'.format(device_id),
+                        props['name'].decode())
+                pci_bus = ('Device {} PCI Bus ID'.format(device_id),
+                           device.pci_bus_id)
+                if is_hip:
+                    try:
+                        arch = props['gcnArchName'].decode()
+                    except KeyError:  # ROCm < 3.6.0
+                        arch = 'gfx'+str(props['gcnArch'])
+                    arch = ('Device {} Arch'.format(device_id), arch)
+                else:
+                    arch = ('Device {} Compute Capability'.format(device_id),
+                            device.compute_capability)
+                records += [name, arch, pci_bus]
+
+        width = max([len(r[0]) for r in records]) + 2
+        fmt = '{:' + str(width) + '}: {}\n'
+        s = io.StringIO()
+        for k, v in records:
+            s.write(fmt.format(k, v))
+
+        return s.getvalue()
+
+
+def get_runtime_info(*, full=True):
+    return _RuntimeInfo(full=full)
+
+
+def get_install_info():
+    return _InstallInfo()
+
+
+def _dir_or_none(path):
+    """Returns None if path does not exist."""
+    if os.path.isdir(path):
+        return path
+    return None
diff --git a/cupyx/_scatter.py b/cupyx/_scatter.py
new file mode 100644
index 0000000..182d4cb
--- /dev/null
+++ b/cupyx/_scatter.py
@@ -0,0 +1,131 @@
+def scatter_add(a, slices, value):
+    """Adds given values to specified elements of an array.
+
+    It adds ``value`` to the specified elements of ``a``.
+    If all of the indices target different locations, the operation of
+    :func:`scatter_add` is equivalent to ``a[slices] = a[slices] + value``.
+    If there are multiple elements targeting the same location,
+    :func:`scatter_add` uses all of these values for addition. On the other
+    hand, ``a[slices] = a[slices] + value`` only adds the contribution from one
+    of the indices targeting the same location.
+
+    Note that just like an array indexing, negative indices are interpreted as
+    counting from the end of an array.
+
+    Also note that :func:`scatter_add` behaves identically
+    to :func:`numpy.add.at`.
+
+    Example
+    -------
+    >>> import cupy
+    >>> import cupyx
+    >>> a = cupy.zeros((6,), dtype=cupy.float32)
+    >>> i = cupy.array([1, 0, 1])
+    >>> v = cupy.array([1., 1., 1.])
+    >>> cupyx.scatter_add(a, i, v);
+    >>> a
+    array([1., 2., 0., 0., 0., 0.], dtype=float32)
+
+    Args:
+        a (ndarray): An array that gets added.
+        slices: It is integer, slices, ellipsis, numpy.newaxis,
+            integer array-like, boolean array-like or tuple of them.
+            It works for slices used for
+            :func:`cupy.ndarray.__getitem__` and
+            :func:`cupy.ndarray.__setitem__`.
+        v (array-like): Values to increment ``a`` at referenced locations.
+
+    .. note::
+        It only supports types that are supported by CUDA's atomicAdd when
+        an integer array is included in ``slices``.
+        The supported types are ``numpy.float32``, ``numpy.int32``,
+        ``numpy.uint32``, ``numpy.uint64`` and ``numpy.ulonglong``.
+
+    .. note::
+        :func:`scatter_add` does not raise an error when indices exceed size of
+        axes. Instead, it wraps indices.
+
+    .. seealso:: :meth:`numpy.ufunc.at`.
+
+    """
+    a.scatter_add(slices, value)
+
+
+def scatter_max(a, slices, value):
+    """Stores a maximum value of elements specified by indices to an array.
+
+    It stores the maximum value of elements in ``value`` array indexed by
+    ``slices`` to ``a``. If all of the indices target different locations,
+    the operation of :func:`scatter_max` is equivalent to
+    ``a[slices] = cupy.maximum(a[slices], value)``.
+    If there are multiple elements targeting the same location,
+    :func:`scatter_max` stores the maximum of all of these values to the given
+    index of ``a``, the initial element of ``a`` is also taken in account.
+
+    Note that just like an array indexing, negative indices are interpreted as
+    counting from the end of an array.
+
+    Also note that :func:`scatter_max` behaves identically
+    to :func:`numpy.maximum.at`.
+
+    Example
+    -------
+    >>> import numpy
+    >>> import cupy
+    >>> a = cupy.zeros((6,), dtype=numpy.float32)
+    >>> i = cupy.array([1, 0, 1, 2])
+    >>> v = cupy.array([1., 2., 3., -1.])
+    >>> cupyx.scatter_max(a, i, v);
+    >>> a
+    array([2., 3., 0., 0., 0., 0.], dtype=float32)
+
+    Args:
+        a (ndarray): An array to store the results.
+        slices: It is integer, slices, ellipsis, numpy.newaxis,
+            integer array-like, boolean array-like or tuple of them.
+            It works for slices used for
+            :func:`cupy.ndarray.__getitem__` and
+            :func:`cupy.ndarray.__setitem__`.
+        v (array-like): An array used for reference.
+    """
+    a.scatter_max(slices, value)
+
+
+def scatter_min(a, slices, value):
+    """Stores a minimum value of elements specified by indices to an array.
+
+    It stores the minimum value of elements in ``value`` array indexed by
+    ``slices`` to ``a``. If all of the indices target different locations,
+    the operation of :func:`scatter_min` is equivalent to
+    ``a[slices] = cupy.minimum(a[slices], value)``.
+    If there are multiple elements targeting the same location,
+    :func:`scatter_min` stores the minimum of all of these values to the given
+    index of ``a``, the initial element of ``a`` is also taken in account.
+
+    Note that just like an array indexing, negative indices are interpreted as
+    counting from the end of an array.
+
+    Also note that :func:`scatter_min` behaves identically
+    to :func:`numpy.minimum.at`.
+
+    Example
+    -------
+    >>> import numpy
+    >>> import cupy
+    >>> a = cupy.zeros((6,), dtype=numpy.float32)
+    >>> i = cupy.array([1, 0, 1, 2])
+    >>> v = cupy.array([1., 2., 3., -1.])
+    >>> cupyx.scatter_min(a, i, v);
+    >>> a
+    array([ 0.,  0., -1.,  0.,  0.,  0.], dtype=float32)
+
+    Args:
+        a (ndarray): An array to store the results.
+        slices: It is integer, slices, ellipsis, numpy.newaxis,
+            integer array-like, boolean array-like or tuple of them.
+            It works for slices used for
+            :func:`cupy.ndarray.__getitem__` and
+            :func:`cupy.ndarray.__setitem__`.
+        v (array-like): An array used for reference.
+    """
+    a.scatter_min(slices, value)
diff --git a/cupyx/_texture.py b/cupyx/_texture.py
new file mode 100644
index 0000000..179b22f
--- /dev/null
+++ b/cupyx/_texture.py
@@ -0,0 +1,197 @@
+import cupy
+
+from cupy import _core
+from cupy.cuda import texture
+from cupy.cuda import runtime
+
+
+_affine_transform_2d_array_kernel = _core.ElementwiseKernel(
+    'U texObj, raw float32 m, uint64 width', 'T transformed_image',
+    '''
+    float3 pixel = make_float3(
+        (float)(i / width),
+        (float)(i % width),
+        1.0f
+    );
+    float x = dot(pixel, make_float3(m[0],  m[1],  m[2])) + .5f;
+    float y = dot(pixel, make_float3(m[3],  m[4],  m[5])) + .5f;
+    transformed_image = tex2D<T>(texObj, y, x);
+    ''',
+    'cupyx_texture_affine_transformation_2d_array',
+    preamble='''
+    inline __host__ __device__ float dot(float3 a, float3 b)
+    {
+        return a.x * b.x + a.y * b.y + a.z * b.z;
+    }
+    ''')
+
+
+_affine_transform_3d_array_kernel = _core.ElementwiseKernel(
+    'U texObj, raw float32 m, uint64 height, uint64 width',
+    'T transformed_volume',
+    '''
+    float4 voxel = make_float4(
+        (float)(i / (width * height)),
+        (float)((i % (width * height)) / width),
+        (float)((i % (width * height)) % width),
+        1.0f
+    );
+    float x = dot(voxel, make_float4(m[0],  m[1],  m[2],  m[3])) + .5f;
+    float y = dot(voxel, make_float4(m[4],  m[5],  m[6],  m[7])) + .5f;
+    float z = dot(voxel, make_float4(m[8],  m[9],  m[10], m[11])) + .5f;
+    transformed_volume = tex3D<T>(texObj, z, y, x);
+    ''',
+    'cupyx_texture_affine_transformation_3d_array',
+    preamble='''
+    inline __host__ __device__ float dot(float4 a, float4 b)
+    {
+        return a.x * b.x + a.y * b.y + a.z * b.z + a.w * b.w;
+    }
+    ''')
+
+
+def _create_texture_object(data,
+                           address_mode: str,
+                           filter_mode: str,
+                           read_mode: str,
+                           border_color=0):
+
+    if cupy.issubdtype(data.dtype, cupy.unsignedinteger):
+        fmt_kind = runtime.cudaChannelFormatKindUnsigned
+    elif cupy.issubdtype(data.dtype, cupy.integer):
+        fmt_kind = runtime.cudaChannelFormatKindSigned
+    elif cupy.issubdtype(data.dtype, cupy.floating):
+        fmt_kind = runtime.cudaChannelFormatKindFloat
+    else:
+        raise ValueError(f'Unsupported data type {data.dtype}')
+
+    if address_mode == 'nearest':
+        address_mode = runtime.cudaAddressModeClamp
+    elif address_mode == 'constant':
+        address_mode = runtime.cudaAddressModeBorder
+    else:
+        raise ValueError(
+            f'Unsupported address mode {address_mode} '
+            '(supported: constant, nearest)')
+
+    if filter_mode == 'nearest':
+        filter_mode = runtime.cudaFilterModePoint
+    elif filter_mode == 'linear':
+        filter_mode = runtime.cudaFilterModeLinear
+    else:
+        raise ValueError(
+            f'Unsupported filter mode {filter_mode} '
+            f'(supported: nearest, linear)')
+
+    if read_mode == 'element_type':
+        read_mode = runtime.cudaReadModeElementType
+    elif read_mode == 'normalized_float':
+        read_mode = runtime.cudaReadModeNormalizedFloat
+    else:
+        raise ValueError(
+            f'Unsupported read mode {read_mode} '
+            '(supported: element_type, normalized_float)')
+
+    texture_fmt = texture.ChannelFormatDescriptor(
+        data.itemsize * 8, 0, 0, 0, fmt_kind)
+    # CUDAArray: last dimension is the fastest changing dimension
+    array = texture.CUDAarray(texture_fmt, *data.shape[::-1])
+    res_desc = texture.ResourceDescriptor(
+        runtime.cudaResourceTypeArray, cuArr=array)
+    # TODO(the-lay): each dimension can have a different addressing mode
+    # TODO(the-lay): border color/value can be defined for up to 4 channels
+    tex_desc = texture.TextureDescriptor(
+        (address_mode, ) * data.ndim, filter_mode, read_mode,
+        borderColors=(border_color, ))
+    tex_obj = texture.TextureObject(res_desc, tex_desc)
+    array.copy_from(data)
+
+    return tex_obj
+
+
+def affine_transformation(data,
+                          transformation_matrix,
+                          output_shape=None,
+                          output=None,
+                          interpolation: str = 'linear',
+                          mode: str = 'constant',
+                          border_value=0):
+    """
+    Apply an affine transformation.
+
+    The method uses texture memory and supports only 2D and 3D float32 arrays
+    without channel dimension.
+
+    Args:
+        data (cupy.ndarray): The input array or texture object.
+        transformation_matrix (cupy.ndarray): Affine transformation matrix.
+            Must be a homogeneous and have shape ``(ndim + 1, ndim + 1)``.
+        output_shape (tuple of ints): Shape of output. If not specified,
+            the input array shape is used. Default is None.
+        output (cupy.ndarray or ~cupy.dtype): The array in which to place the
+            output, or the dtype of the returned array. If not specified,
+            creates the output array with shape of ``output_shape``. Default is
+            None.
+        interpolation (str): Specifies interpolation mode: ``'linear'`` or
+            ``'nearest'``. Default is ``'linear'``.
+        mode (str): Specifies addressing mode for points outside of the array:
+            (`'constant'``, ``'nearest'``). Default is ``'constant'``.
+        border_value: Specifies value to be used for coordinates outside
+            of the array for ``'constant'`` mode. Default is 0.
+
+    Returns:
+        cupy.ndarray:
+            The transformed input.
+
+    .. seealso:: :func:`cupyx.scipy.ndimage.affine_transform`
+    """
+
+    ndim = data.ndim
+    if (ndim < 2) or (ndim > 3):
+        raise ValueError(
+            'Texture memory affine transformation is defined only for '
+            '2D and 3D arrays without channel dimension.')
+
+    dtype = data.dtype
+    if dtype != cupy.float32:
+        raise ValueError(f'Texture memory affine transformation is available '
+                         f'only for float32 data type (not {dtype})')
+
+    if interpolation not in ['linear', 'nearest']:
+        raise ValueError(
+            f'Unsupported interpolation {interpolation} '
+            f'(supported: linear, nearest)')
+
+    if transformation_matrix.shape != (ndim + 1, ndim + 1):
+        raise ValueError('Matrix must be have shape (ndim + 1, ndim + 1)')
+
+    texture_object = _create_texture_object(data,
+                                            address_mode=mode,
+                                            filter_mode=interpolation,
+                                            read_mode='element_type',
+                                            border_color=border_value)
+
+    if ndim == 2:
+        kernel = _affine_transform_2d_array_kernel
+    else:
+        kernel = _affine_transform_3d_array_kernel
+
+    if output_shape is None:
+        output_shape = data.shape
+
+    if output is None:
+        output = cupy.zeros(output_shape, dtype=dtype)
+    elif isinstance(output, (type, cupy.dtype)):
+        if output != cupy.float32:
+            raise ValueError(f'Texture memory affine transformation is '
+                             f'available only for float32 data type (not '
+                             f'{output})')
+        output = cupy.zeros(output_shape, dtype=output)
+    elif isinstance(output, cupy.ndarray):
+        if output.shape != output_shape:
+            raise ValueError('Output shapes do not match')
+    else:
+        raise ValueError('Output must be None, cupy.ndarray or cupy.dtype')
+
+    kernel(texture_object, transformation_matrix, *output_shape[1:], output)
+    return output
diff --git a/cupyx/_ufunc_config.py b/cupyx/_ufunc_config.py
new file mode 100644
index 0000000..33973a1
--- /dev/null
+++ b/cupyx/_ufunc_config.py
@@ -0,0 +1,123 @@
+import contextlib
+import threading
+
+
+_config = threading.local()
+
+
+def get_config_divide():
+    try:
+        value = _config.divide
+    except AttributeError:
+        value = _config.divide = None
+    return value
+
+
+def get_config_over():
+    try:
+        value = _config.over
+    except AttributeError:
+        value = _config.over = None
+    return value
+
+
+def get_config_under():
+    try:
+        value = _config.under
+    except AttributeError:
+        value = _config.under = None
+    return value
+
+
+def get_config_invalid():
+    try:
+        value = _config.invalid
+    except AttributeError:
+        value = _config.invalid = None
+    return value
+
+
+def get_config_linalg():
+    # In favor of performance, the `devInfo` input/output from cuSOLVER routine
+    # calls that is necessary to check the validity of the other outputs, are
+    # ignored, as D2H copy incurring device synchronizations would otherwise be
+    # required.
+    try:
+        value = _config.linalg
+    except AttributeError:
+        value = _config.linalg = 'ignore'
+    return value
+
+
+def get_config_fallback_mode():
+    try:
+        value = _config.fallback_mode
+    except AttributeError:
+        value = _config.fallback_mode = 'ignore'
+    return value
+
+
+@contextlib.contextmanager
+def errstate(*, divide=None, over=None, under=None,
+             invalid=None, linalg=None, fallback_mode=None):
+    """
+    TODO(hvy): Write docs.
+    """
+    old_state = seterr(
+        divide=divide, over=over, under=under,
+        invalid=invalid, linalg=linalg, fallback_mode=fallback_mode)
+    try:
+        yield  # Return `None` similar to `numpy.errstate`.
+    finally:
+        seterr(**old_state)
+
+
+def seterr(*, divide=None, over=None, under=None,
+           invalid=None, linalg=None, fallback_mode=None):
+    """
+    TODO(hvy): Write docs.
+    """
+    old_state = geterr()
+
+    if divide is not None:
+        raise NotImplementedError()
+    if over is not None:
+        raise NotImplementedError()
+    if under is not None:
+        raise NotImplementedError()
+    if invalid is not None:
+        raise NotImplementedError()
+    if linalg is not None:
+        if linalg in ('ignore', 'raise'):
+            _config.linalg = linalg
+        else:
+            raise NotImplementedError()
+    if fallback_mode is not None:
+        if fallback_mode in ['print', 'warn', 'ignore', 'raise']:
+            _config.fallback_mode = fallback_mode
+        elif fallback_mode in ['log', 'call']:
+            raise NotImplementedError
+        else:
+            raise ValueError(
+                '{} is not a valid dispatch type'.format(fallback_mode))
+
+    _config.divide = divide
+    _config.under = under
+    _config.over = over
+    _config.invalid = invalid
+
+    return old_state
+
+
+def geterr():
+    """
+    TODO(hvy): Write docs.
+    """
+    return dict(
+        divide=get_config_divide(),
+        over=get_config_over(),
+        under=get_config_under(),
+        invalid=get_config_invalid(),
+        linalg=get_config_linalg(),
+        fallback_mode=get_config_fallback_mode(),
+    )
diff --git a/cupyx/cudnn.pyx b/cupyx/cudnn.pyx
new file mode 100644
index 0000000..fcfb9e9
--- /dev/null
+++ b/cupyx/cudnn.pyx
@@ -0,0 +1,2613 @@
+from libcpp cimport vector
+from libc.stdint cimport int64_t
+
+import atexit as _atexit
+import threading as _threading
+import warnings as _warnings
+
+import numpy as _numpy
+
+from cupy import _core
+from cupy._core._carray cimport shape_t
+from cupy._core cimport _routines_manipulation as _manipulation
+from cupy._core cimport core
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport internal
+from cupy.cuda cimport device
+from cupy.cuda cimport memory as _memory
+from cupy_backends.cuda.libs cimport cudnn
+
+from cupy._core._ufuncs import elementwise_copy as _elementwise_copy
+from cupy import _util
+
+from cupy.cuda import cudnn as _py_cudnn
+
+
+cdef int _cudnn_version = -1
+cdef _thread_local = _threading.local()
+
+cdef vector.vector[size_t] _handles
+
+
+cdef inline int cudnn_version():
+    global _cudnn_version
+    if _cudnn_version == -1:
+        _cudnn_version = cudnn.getVersion()
+    return _cudnn_version
+
+
+cpdef size_t get_handle() except? 0:
+    cdef int dev
+    dev = device.get_device_id()
+    if <int>_handles.size() <= dev:
+        _handles.resize(dev + 1, 0)
+    ret = _handles[dev]
+    if ret != 0:
+        return ret
+    ret = cudnn.create()
+    _handles[dev] = ret
+    return ret
+
+
+@_atexit.register
+def reset_handles():
+    for handle in _handles:
+        if handle:
+            cudnn.destroy(handle)
+    _handles.clear()
+
+
+cpdef dict _get_nd_tensor_cache():
+    if not hasattr(_thread_local, 'cudnn_nd_tensor_cache'):
+        _thread_local.cudnn_nd_tensor_cache = {}
+    return _thread_local.cudnn_nd_tensor_cache
+
+
+cdef size_t _max_workspace_size = 8 * 1024 * 1024
+
+
+cpdef size_t get_max_workspace_size():
+    """Gets the workspace size for cuDNN.
+
+    Check "cuDNN Library User Guide" for detail.
+
+    Returns:
+        int: The workspace size for cuDNN.
+
+    """
+    return _max_workspace_size
+
+
+cpdef set_max_workspace_size(size):
+    """Sets the workspace size for cuDNN.
+
+    Check "cuDNN Library User Guide" for detail.
+
+    Args:
+        size: The workspace size for cuDNN.
+
+    """
+    global _max_workspace_size
+    _max_workspace_size = size
+
+
+cdef class Descriptor:
+
+    cdef public size_t value
+    cdef object destroy
+
+    def __init__(self, descriptor, destroyer):
+        self.value = descriptor
+        self.destroy = destroyer
+
+    def __dealloc__(self):
+        if self.value:
+            self.destroy(self.value)
+            self.value = 0
+
+
+cpdef int get_data_type(dtype) except? -1:
+    cdef char t = ord(dtype.char)
+    if t == b'f':
+        return cudnn.CUDNN_DATA_FLOAT
+    elif t == b'd':
+        return cudnn.CUDNN_DATA_DOUBLE
+    elif t == b'e':
+        return cudnn.CUDNN_DATA_HALF
+    else:
+        raise TypeError('Dtype {} is not supported in cuDNN'.format(dtype))
+
+
+cpdef int _get_byte_size(int data_type) except -1:
+    if data_type == cudnn.CUDNN_DATA_HALF:
+        return 2
+    elif data_type == cudnn.CUDNN_DATA_FLOAT:
+        return 4
+    elif data_type == cudnn.CUDNN_DATA_DOUBLE:
+        return 8
+    else:
+        raise TypeError('Invalid cuDNN data type: {}'.format(data_type))
+
+
+cpdef _create_tensor_nd_descriptor(
+        size_t desc, _ndarray_base arr, int data_type=-1):
+    cdef vector.vector[int] c_shape, c_strides
+    cdef Py_ssize_t itemsize, s
+    cdef int next_stride, i
+    if data_type == -1:  # `-1` is used instead of `None`
+        data_type = get_data_type(arr.dtype)
+    itemsize = arr.itemsize
+    for s in arr._strides:
+        c_strides.push_back(s // itemsize)
+    for s in arr._shape:
+        c_shape.push_back(s)
+    # Use "c-contiguous stride" with the next axis, if ambiguous
+    next_stride = 1
+    for i in reversed(range(c_shape.size())):
+        if c_shape[i] <= 1:
+            c_strides[i] = next_stride
+        else:
+            next_stride = c_shape[i] * c_strides[i]
+
+    cudnn.setTensorNdDescriptor(
+        desc, data_type, arr._shape.size(), <size_t>c_shape.data(),
+        <size_t>c_strides.data())
+
+
+cpdef _create_tensor_descriptor(size_t desc, _ndarray_base arr,
+                                int format=cudnn.CUDNN_TENSOR_NCHW):
+    if not arr._c_contiguous:
+        raise ValueError('cupyx.cudnn supports c-contiguous arrays only')
+    if arr._shape.size() == 4:
+        data_type = get_data_type(arr.dtype)
+        if format == cudnn.CUDNN_TENSOR_NCHW:
+            n, c, h, w = arr._shape
+        elif format == cudnn.CUDNN_TENSOR_NHWC:
+            n, h, w, c = arr._shape
+        else:
+            raise ValueError('unknown cudnnTensorFormat: {}'.format(format))
+        cudnn.setTensor4dDescriptor(desc, format, data_type, n, c, h, w)
+    else:
+        _create_tensor_nd_descriptor(desc, arr)
+
+
+cpdef _create_tensor_descriptor_as4darray(size_t desc,
+                                          _ndarray_base arr):
+    cdef Py_ssize_t dim1, dim2
+    assert arr._c_contiguous
+    data_type = get_data_type(arr.dtype)
+    dim1 = 1
+    if arr._shape.size() > 0:
+        dim1 = arr._shape[0]
+    dim2 = arr.size // dim1
+    cudnn.setTensor4dDescriptor(desc, cudnn.CUDNN_TENSOR_NCHW, data_type,
+                                dim1, dim2, 1, 1)
+
+
+cpdef _create_filter_descriptor(
+        size_t desc, _ndarray_base arr, int format=cudnn.CUDNN_TENSOR_NCHW):
+    cdef vector.vector[int] c_shape
+    cdef Py_ssize_t s, ndim = arr._shape.size()
+    data_type = get_data_type(arr.dtype)
+    if ndim == 4:
+        if format == cudnn.CUDNN_TENSOR_NCHW:
+            k, c, h, w = arr._shape
+        elif format == cudnn.CUDNN_TENSOR_NHWC:
+            k, h, w, c = arr._shape
+        else:
+            raise ValueError('unknown cudnnTensorFormat: {}'.format(format))
+        cudnn.setFilter4dDescriptor_v4(
+            desc, data_type, format, k, c, h, w)
+    else:
+        for s in arr._shape:
+            c_shape.push_back(s)
+        cudnn.setFilterNdDescriptor_v4(
+            desc, data_type, format, ndim, <size_t>c_shape.data())
+
+
+cpdef _create_convolution_descriptor(
+        size_t desc, tuple pad, tuple stride, tuple dilation, int groups,
+        object dtype, int mode, bint use_tensor_core):
+    cdef int d0, d1, p0, p1, s0, s1
+    cdef vector.vector[int] c_pad, c_stride, c_dilation
+    ndim = len(pad)
+    if ndim != len(stride):
+        raise ValueError('pad and stride must be of same length')
+
+    compute_type = get_data_type(dtype)
+    # TODO(takagi) Temporarily use computing precision of FP32 for
+    #     storing precision of FP16.
+    if compute_type == cudnn.CUDNN_DATA_HALF:
+        compute_type = cudnn.CUDNN_DATA_FLOAT
+
+    if ndim != 2:
+        c_pad = pad
+        c_stride = stride
+        if dilation is None:
+            c_dilation.assign(ndim, 1)
+        else:
+            c_dilation = dilation
+            if cudnn_version() < 6000:
+                for i in c_dilation:
+                    if i != 1:
+                        raise ValueError(
+                            'dilation must be one when cuDNN < 6.0')
+        cudnn.setConvolutionNdDescriptor_v3(
+            desc, ndim, <size_t>c_pad.data(), <size_t>c_stride.data(),
+            <size_t>c_dilation.data(), mode, compute_type)
+    else:
+        if dilation is None:
+            d0 = d1 = 1
+        else:
+            d0, d1 = dilation
+            if cudnn_version() < 6000 and (d0 != 1 or d1 != 1):
+                raise ValueError('dilation must be one when cuDNN < 6.0')
+        p0, p1 = pad
+        s0, s1 = stride
+        cudnn.setConvolution2dDescriptor_v5(
+            desc, p0, p1, s0, s1, d0, d1, mode, compute_type)
+    if cudnn_version() >= 7000:
+        if use_tensor_core:
+            math_type = cudnn.CUDNN_TENSOR_OP_MATH
+            cudnn.setConvolutionMathType(desc, math_type)
+        if groups > 1:
+            cudnn.setConvolutionGroupCount(desc, groups)
+    elif groups > 1:
+        raise ValueError('groups must be one when cuDNN < 7.0')
+
+
+cpdef _ndarray_base _ascontiguousarray_normalized_strides(_ndarray_base a):
+    cdef _ndarray_base newarray
+
+    if a._c_contiguous:
+        newarray = a.view()
+        newarray._set_contiguous_strides(newarray.itemsize, True)
+    else:
+        newarray = _core.ndarray(a.shape, a.dtype)
+        _elementwise_copy(a, newarray)
+    return newarray
+
+
+def create_tensor_descriptor(arr, format=cudnn.CUDNN_TENSOR_NCHW):
+    desc = Descriptor(cudnn.createTensorDescriptor(),
+                      _py_cudnn.destroyTensorDescriptor)
+    _create_tensor_descriptor(desc.value, arr, format)
+    return desc
+
+
+def create_uninitialized_tensor_descriptor():
+    """Create uninitialized tensor descriptor.
+
+    Create a cudnnCreateTensorDescriptor_t that is not yet initialized.
+    This is used by the batch normalization functions.
+    """
+    return Descriptor(cudnn.createTensorDescriptor(),
+                      _py_cudnn.destroyTensorDescriptor)
+
+
+def create_tensor_nd_descriptor(_ndarray_base arr):
+    cdef dict cache
+    if arr.size == 0:
+        return Descriptor(0, None)
+    if not arr.flags.c_contiguous:
+        raise ValueError('cupyx.cudnn supports c-contiguous arrays only')
+    data_type = get_data_type(arr.dtype)
+    key = (data_type, tuple(arr._shape))
+    cache = _get_nd_tensor_cache()
+    if key in cache:
+        return cache[key]
+
+    # numpy's stride is defined in bytes, but cudnn's stride is defined in
+    # size of element
+    desc = Descriptor(cudnn.createTensorDescriptor(),
+                      _py_cudnn.destroyTensorDescriptor)
+    _create_tensor_nd_descriptor(desc.value, arr, data_type)
+    cache[key] = desc
+    return desc
+
+
+def create_filter_descriptor(arr, format=cudnn.CUDNN_TENSOR_NCHW):
+    desc = Descriptor(cudnn.createFilterDescriptor(),
+                      _py_cudnn.destroyFilterDescriptor)
+    _create_filter_descriptor(desc.value, arr, format)
+    return desc
+
+
+def create_convolution_descriptor(pad, stride, dtype,
+                                  mode=cudnn.CUDNN_CROSS_CORRELATION,
+                                  dilation=None,
+                                  use_tensor_core=False,
+                                  groups=1):
+    desc = Descriptor(cudnn.createConvolutionDescriptor(),
+                      _py_cudnn.destroyConvolutionDescriptor)
+    _create_convolution_descriptor(
+        desc.value, pad, stride, dilation, groups,
+        dtype, mode, use_tensor_core)
+    return desc
+
+
+cdef _create_pooling_descriptor(
+        size_t desc, tuple ksize, tuple stride, tuple pad, int mode):
+    cdef vector.vector[int] c_ksize, c_pad, c_stride
+    cdef int ndim = len(ksize)
+    if ndim != len(stride) or ndim != len(pad):
+        raise ValueError('ksize, stride, and pad must be of same length')
+    if ndim == 2:
+        cudnn.setPooling2dDescriptor_v4(
+            desc, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, ksize[0],
+            ksize[1], pad[0], pad[1], stride[0], stride[1])
+    else:
+        c_ksize = ksize
+        c_pad = pad
+        c_stride = stride
+        cudnn.setPoolingNdDescriptor_v4(
+            desc, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, ndim,
+            <size_t>c_ksize.data(), <size_t>c_pad.data(),
+            <size_t>c_stride.data())
+
+    return desc
+
+
+def create_pooling_descriptor(ksize, stride, pad, int mode):
+    desc = Descriptor(cudnn.createPoolingDescriptor(),
+                      _py_cudnn.destroyPoolingDescriptor)
+    _create_pooling_descriptor(desc.value, ksize, stride, pad, mode)
+    return desc
+
+
+cdef Descriptor _create_rnn_data_descriptor():
+    return Descriptor(cudnn.createRNNDataDescriptor(),
+                      _py_cudnn.destroyRNNDataDescriptor)
+
+
+cdef Descriptor _make_unpacked_rnn_data_descriptor(_ndarray_base xs, lengths):
+    cdef Descriptor descriptor = _create_rnn_data_descriptor()
+    cdef int data_type = get_data_type(xs.dtype)
+    cdef Py_ssize_t max_length, batch, n_dim
+    max_length, batch, n_dim = xs.shape
+    cudnn.setRNNDataDescriptor(
+        descriptor.value, data_type,
+        cudnn.CUDNN_RNN_DATA_LAYOUT_SEQ_MAJOR_UNPACKED,
+        max_length, batch, n_dim,
+        lengths.ctypes.data, 0)
+    return descriptor
+
+
+def rnn_forward_inference_ex(
+        DropoutStates states, int direction_mode, int rnn_mode,
+        _ndarray_base hx, _ndarray_base cx, _ndarray_base w,
+        _ndarray_base xs, lengths):
+    hx = core._internal_ascontiguousarray(hx)
+    if cx is not None:
+        cx = core._internal_ascontiguousarray(cx)
+    w = core._internal_ascontiguousarray(w)
+    xs = core._internal_ascontiguousarray(xs)
+
+    cdef int length = xs._shape[0]
+    cdef int n_layers = _get_n_layers(direction_mode, hx)
+    cdef int n_units = hx._shape[2]
+
+    cdef _ndarray_base ys = _make_rnn_result_array(direction_mode, n_units, xs)
+    cdef _ndarray_base hy = _core.ndarray(hx.shape, hx.dtype)
+    if cx is None:
+        cx = _core.ndarray(0, dtype=xs.dtype)
+    cdef _ndarray_base cy = _core.ndarray(cx.shape, cx.dtype)
+
+    cdef size_t handle = get_handle()
+
+    cdef Descriptor rnn_desc = create_rnn_descriptor(
+        n_units, n_layers, states._desc,
+        cudnn.CUDNN_LINEAR_INPUT, direction_mode,
+        rnn_mode, get_data_type(xs.dtype))
+    cudnn.setRNNPaddingMode(
+        rnn_desc.value, cudnn.CUDNN_RNN_PADDED_IO_ENABLED)
+
+    cdef Descriptor x_data_desc = _make_unpacked_rnn_data_descriptor(
+        xs, lengths)
+    cdef Descriptor hx_desc = create_tensor_nd_descriptor(hx)
+    cdef Descriptor cx_desc = create_tensor_nd_descriptor(cx)
+    cdef Descriptor w_desc = create_filter_descriptor(w)
+    cdef Descriptor y_data_desc = _make_unpacked_rnn_data_descriptor(
+        ys, lengths)
+    cdef Descriptor hy_desc = create_tensor_nd_descriptor(hy)
+    cdef Descriptor cy_desc = create_tensor_nd_descriptor(cy)
+
+    cdef _DescriptorArray xs_descs = _make_tensor_descriptor_array_for_padded(
+        xs)
+    cdef _memory.MemoryPointer workspace = _make_rnn_workspace(
+        rnn_desc, length, xs_descs)
+
+    cudnn.RNNForwardInferenceEx(
+        handle, rnn_desc.value,
+        x_data_desc.value, xs.data.ptr,
+        hx_desc.value, hx.data.ptr,
+        cx_desc.value, cx.data.ptr,
+        w_desc.value, w.data.ptr,
+        y_data_desc.value, ys.data.ptr,
+        hy_desc.value, hy.data.ptr,
+        cy_desc.value, cy.data.ptr,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        workspace.ptr, workspace.mem.size)
+
+    return hy, cy, ys
+
+
+def rnn_forward_training_ex(
+        DropoutStates states, int direction_mode, int rnn_mode,
+        _ndarray_base hx, _ndarray_base cx, _ndarray_base w, _ndarray_base xs,
+        lengths):
+    hx = core._internal_ascontiguousarray(hx)
+    if cx is not None:
+        cx = core._internal_ascontiguousarray(cx)
+    w = core._internal_ascontiguousarray(w)
+    xs = core._internal_ascontiguousarray(xs)
+
+    cdef int length = xs._shape[0]
+    cdef int n_layers = _get_n_layers(direction_mode, hx)
+    cdef int n_units = hx._shape[2]
+
+    cdef size_t handle = get_handle()
+
+    cdef Descriptor rnn_desc = create_rnn_descriptor(
+        n_units, n_layers, states._desc,
+        cudnn.CUDNN_LINEAR_INPUT, direction_mode,
+        rnn_mode, get_data_type(xs.dtype))
+    cudnn.setRNNPaddingMode(
+        rnn_desc.value, cudnn.CUDNN_RNN_PADDED_IO_ENABLED)
+
+    cdef _ndarray_base ys = _make_rnn_result_array(direction_mode, n_units, xs)
+    cdef _ndarray_base hy = _core.ndarray(hx.shape, hx.dtype)
+    if cx is None:
+        cx = _core.ndarray(0, dtype=xs.dtype)
+    cdef _ndarray_base cy = _core.ndarray(cx.shape, cx.dtype)
+
+    cdef Descriptor x_data_desc = _make_unpacked_rnn_data_descriptor(
+        xs, lengths)
+    cdef Descriptor hx_desc = create_tensor_nd_descriptor(hx)
+    cdef Descriptor cx_desc = create_tensor_nd_descriptor(cx)
+    cdef Descriptor w_desc = create_filter_descriptor(w)
+    cdef Descriptor y_data_desc = _make_unpacked_rnn_data_descriptor(
+        ys, lengths)
+    cdef Descriptor hy_desc = create_tensor_nd_descriptor(hy)
+    cdef Descriptor cy_desc = create_tensor_nd_descriptor(cy)
+
+    cdef _DescriptorArray xs_descs = _make_tensor_descriptor_array_for_padded(
+        xs)
+    cdef _memory.MemoryPointer workspace = _make_rnn_workspace(
+        rnn_desc, length, xs_descs)
+    cdef _memory.MemoryPointer reserve_space = _make_rnn_reserve_space(
+        rnn_desc, length, xs_descs)
+
+    cudnn.RNNForwardTrainingEx(
+        handle, rnn_desc.value,
+        x_data_desc.value, xs.data.ptr,
+        hx_desc.value, hx.data.ptr,
+        cx_desc.value, cx.data.ptr,
+        w_desc.value, w.data.ptr,
+        y_data_desc.value, ys.data.ptr,
+        hy_desc.value, hy.data.ptr,
+        cy_desc.value, cy.data.ptr,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        workspace.ptr, workspace.mem.size,
+        reserve_space.ptr, reserve_space.mem.size)
+
+    return reserve_space, hy, cy, ys
+
+
+def rnn_backward_data_ex(
+        DropoutStates states, int direction_mode, int rnn_mode,
+        _ndarray_base hx, _ndarray_base cx, _ndarray_base w, _ndarray_base xs,
+        _ndarray_base ys, _memory.MemoryPointer reserve_space,
+        _ndarray_base dhy, _ndarray_base dcy, _ndarray_base dys,
+        lengths):
+    hx = core._internal_ascontiguousarray(hx)
+    if cx is not None:
+        cx = core._internal_ascontiguousarray(cx)
+    w = core._internal_ascontiguousarray(w)
+    xs = core._internal_ascontiguousarray(xs)
+    ys = core._internal_ascontiguousarray(ys)
+    dhy = core._internal_ascontiguousarray(dhy)
+    if dcy is not None:
+        dcy = core._internal_ascontiguousarray(dcy)
+    dys = core._internal_ascontiguousarray(dys)
+
+    cdef int length = xs._shape[0]
+    cdef int n_layers = _get_n_layers(direction_mode, hx)
+    cdef int n_units = hx._shape[2]
+
+    cdef size_t handle = get_handle()
+    cdef Descriptor rnn_desc = create_rnn_descriptor(
+        n_units, n_layers, states._desc,
+        cudnn.CUDNN_LINEAR_INPUT, direction_mode,
+        rnn_mode, get_data_type(xs.dtype))
+    cudnn.setRNNPaddingMode(
+        rnn_desc.value, cudnn.CUDNN_RNN_PADDED_IO_ENABLED)
+
+    cdef _ndarray_base dxs = _core.ndarray(xs.shape, xs.dtype)
+    cdef _ndarray_base dhx = _core.ndarray(hx.shape, hx.dtype)
+    if cx is None:
+        cx = dcy = _core.ndarray(0, dtype=xs.dtype)
+    cdef _ndarray_base dcx = _core.ndarray(cx.shape, cx.dtype)
+
+    cdef Descriptor y_data_desc = _make_unpacked_rnn_data_descriptor(
+        ys, lengths)
+    cdef Descriptor dy_data_desc = _make_unpacked_rnn_data_descriptor(
+        dys, lengths)
+    cdef Descriptor dhy_desc = create_tensor_nd_descriptor(dhy)
+    cdef Descriptor dcy_desc = create_tensor_nd_descriptor(dcy)
+    cdef Descriptor w_desc = create_filter_descriptor(w)
+    cdef Descriptor hx_desc = create_tensor_nd_descriptor(hx)
+    cdef Descriptor cx_desc = create_tensor_nd_descriptor(cx)
+    cdef Descriptor dx_data_desc = _make_unpacked_rnn_data_descriptor(
+        dxs, lengths)
+    cdef Descriptor dhx_desc = create_tensor_nd_descriptor(dhx)
+    cdef Descriptor dcx_desc = create_tensor_nd_descriptor(dcx)
+
+    cdef _DescriptorArray xs_descs = _make_tensor_descriptor_array_for_padded(
+        xs)
+    cdef _memory.MemoryPointer workspace = _make_rnn_workspace(
+        rnn_desc, length, xs_descs)
+
+    cudnn.RNNBackwardDataEx(
+        handle, rnn_desc.value,
+        y_data_desc.value, ys.data.ptr,
+        dy_data_desc.value, dys.data.ptr,
+        0, 0,
+        dhy_desc.value, dhy.data.ptr,
+        dcy_desc.value, dcy.data.ptr,
+        w_desc.value, w.data.ptr,
+        hx_desc.value, hx.data.ptr,
+        cx_desc.value, cx.data.ptr,
+        dx_data_desc.value, dxs.data.ptr,
+        dhx_desc.value, dhx.data.ptr,
+        dcx_desc.value, dcx.data.ptr,
+        0, 0,
+        workspace.ptr, workspace.mem.size,
+        reserve_space.ptr, reserve_space.mem.size)
+
+    return dhx, dcx, dxs
+
+
+def rnn_backward_weights_ex(
+        DropoutStates states, int direction_mode, int rnn_mode,
+        _ndarray_base xs, _ndarray_base hx, _ndarray_base ys,
+        _ndarray_base w,
+        _memory.MemoryPointer reserve_space, lengths):
+    xs = core._internal_ascontiguousarray(xs)
+    hx = core._internal_ascontiguousarray(hx)
+    ys = core._internal_ascontiguousarray(ys)
+    w = core._internal_ascontiguousarray(w)
+
+    cdef int length = xs._shape[0]
+    cdef int n_layers = _get_n_layers(direction_mode, hx)
+    cdef int n_units = hx._shape[2]
+
+    cdef size_t handle = get_handle()
+    cdef Descriptor rnn_desc = create_rnn_descriptor(
+        n_units, n_layers, states._desc,
+        cudnn.CUDNN_LINEAR_INPUT, direction_mode,
+        rnn_mode, get_data_type(xs.dtype))
+    cudnn.setRNNPaddingMode(
+        rnn_desc.value, cudnn.CUDNN_RNN_PADDED_IO_ENABLED)
+
+    cdef Descriptor x_data_desc = _make_unpacked_rnn_data_descriptor(
+        xs, lengths)
+    cdef Descriptor hx_desc = create_tensor_nd_descriptor(hx)
+    cdef Descriptor y_data_desc = _make_unpacked_rnn_data_descriptor(
+        ys, lengths)
+
+    cdef _DescriptorArray xs_descs = _make_tensor_descriptor_array_for_padded(
+        xs)
+    cdef _memory.MemoryPointer workspace = _make_rnn_workspace(
+        rnn_desc, length, xs_descs)
+
+    cdef _ndarray_base dw = _core.ndarray(w.shape, w.dtype)
+    dw.fill(0)
+    cdef Descriptor dw_desc = create_filter_descriptor(dw)
+
+    cudnn.RNNBackwardWeightsEx(
+        handle, rnn_desc.value,
+        x_data_desc.value, xs.data.ptr,
+        hx_desc.value, hx.data.ptr,
+        y_data_desc.value, ys.data.ptr,
+        workspace.ptr, workspace.mem.size,
+        dw_desc.value, dw.data.ptr,
+        reserve_space.ptr, reserve_space.mem.size)
+    return dw
+
+
+def create_activation_descriptor(mode, nan_prop_mode=cudnn.CUDNN_PROPAGATE_NAN,
+                                 coef=0.0):
+    desc = Descriptor(cudnn.createActivationDescriptor(),
+                      _py_cudnn.destroyActivationDescriptor)
+    cudnn.setActivationDescriptor(desc.value, mode, nan_prop_mode, coef)
+    return desc
+
+
+def activation_forward(_ndarray_base x, int mode, double coef=0.0):
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    cdef _ndarray_base y
+    if x.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+
+    x = core._internal_ascontiguousarray(x)
+    y = _core.ndarray(x._shape, x.dtype)
+
+    handle = get_handle()
+    desc = cudnn.createTensorDescriptor()
+    act_desc = cudnn.createActivationDescriptor()
+    try:
+        _create_tensor_descriptor_as4darray(desc, x)
+        cudnn.setActivationDescriptor(
+            act_desc, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, coef)
+        cudnn.activationForward_v4(
+            handle, act_desc, one, desc, x.data.ptr,
+            zero, desc, y.data.ptr)
+    finally:
+        cudnn.destroyActivationDescriptor(act_desc)
+        cudnn.destroyTensorDescriptor(desc)
+    return y
+
+
+def activation_backward(_ndarray_base x, _ndarray_base y, _ndarray_base gy,
+                        int mode, float coef=0.0):
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    cdef _ndarray_base gx
+    if x.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+
+    gx = _core.ndarray(x._shape, x.dtype)
+    x = core._internal_ascontiguousarray(x)
+    y = core._internal_ascontiguousarray(y)
+    gy = core._internal_ascontiguousarray(gy)
+
+    handle = get_handle()
+    desc = cudnn.createTensorDescriptor()
+    act_desc = cudnn.createActivationDescriptor()
+    try:
+        _create_tensor_descriptor_as4darray(desc, y)
+        cudnn.setActivationDescriptor(
+            act_desc, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, coef)
+        cudnn.activationBackward_v4(
+            handle, act_desc, one, desc, y.data.ptr,
+            desc, gy.data.ptr, desc, x.data.ptr,
+            zero, desc, gx.data.ptr)
+    finally:
+        cudnn.destroyActivationDescriptor(act_desc)
+        cudnn.destroyTensorDescriptor(desc)
+    return gx
+
+
+cdef int _create_tensor_descriptor_for_softmax(
+        size_t desc, _ndarray_base arr, int axis) except?-1:
+    cdef Py_ssize_t left, center, right
+    assert arr._c_contiguous
+    data_type = get_data_type(arr.dtype)
+    if axis < 0:
+        axis += arr._shape.size()
+    left = 1
+    for i in range(0, axis):
+        left *= arr._shape[i]
+    center = arr._shape[axis]
+    right = 1
+    for i in range(axis + 1, <int>arr._shape.size()):
+        right *= arr._shape[i]
+    cudnn.setTensor4dDescriptor(desc, cudnn.CUDNN_TENSOR_NCHW, data_type,
+                                left, center, right, 1)
+    if center == 1 and right == 1:
+        return cudnn.CUDNN_SOFTMAX_MODE_INSTANCE
+    else:
+        return cudnn.CUDNN_SOFTMAX_MODE_CHANNEL
+
+
+def softmax_forward(_ndarray_base x, int axis, int algorithm):
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    cdef _ndarray_base y
+    if x.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+
+    x = core._internal_ascontiguousarray(x)
+    y = _core.ndarray(x._shape, x.dtype)
+
+    handle = get_handle()
+    desc = cudnn.createTensorDescriptor()
+    try:
+        cudnn_mode = _create_tensor_descriptor_for_softmax(desc, x, axis)
+        cudnn.softmaxForward(
+            handle, algorithm, cudnn_mode,
+            one, desc, x.data.ptr, zero, desc, y.data.ptr)
+    finally:
+        cudnn.destroyTensorDescriptor(desc)
+    return y
+
+
+def softmax_backward(
+        _ndarray_base y, _ndarray_base gy, int axis, int algorithm):
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    cdef _ndarray_base gx
+    if y.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+
+    gx = _core.ndarray(y._shape, y.dtype)
+    y = core._internal_ascontiguousarray(y)
+    gy = core._internal_ascontiguousarray(gy)
+
+    handle = get_handle()
+    desc = cudnn.createTensorDescriptor()
+    try:
+        cudnn_mode = _create_tensor_descriptor_for_softmax(desc, y, axis)
+        cudnn.softmaxBackward(
+            handle, algorithm, cudnn_mode,
+            one, desc, y.data.ptr, desc, gy.data.ptr, zero, desc, gx.data.ptr)
+    finally:
+        cudnn.destroyTensorDescriptor(desc)
+    return gx
+
+
+def create_dropout_descriptor(
+        handle, dropout, states, state_size_in_bytes, seed):
+    desc = Descriptor(cudnn.createDropoutDescriptor(),
+                      _py_cudnn.destroyDropoutDescriptor)
+    cudnn.setDropoutDescriptor(desc.value, handle, dropout,
+                               states, state_size_in_bytes, seed)
+    return desc
+
+
+def set_dropout_descriptor(desc, handle, dropout):
+    # When the fourth argument is NULL, random state is not updated.
+    cudnn.setDropoutDescriptor(desc.value, handle, dropout, 0, 0, 0)
+
+
+def _create_ctc_loss_descriptor(data_type):
+    desc = Descriptor(cudnn.createCTCLossDescriptor(),
+                      _py_cudnn.destroyCTCLossDescriptor)
+    cudnn.setCTCLossDescriptor(desc.value, data_type)
+    return desc
+
+
+def ctc_loss(_ndarray_base probs, labels,
+             label_length, input_length, int algo):
+    batch_size = probs.shape[1]
+    labels_ptr = labels.ctypes.data
+    label_length_ptr = label_length.ctypes.data
+    input_length_ptr = input_length.ctypes.data
+    handle = get_handle()
+    data_type = get_data_type(probs.dtype)
+    ctc_desc = Descriptor(cudnn.createCTCLossDescriptor(),
+                          _py_cudnn.destroyCTCLossDescriptor)
+    cudnn.setCTCLossDescriptor(ctc_desc.value, data_type)
+
+    gradients = _core.ndarray(probs._shape, probs.dtype)
+    loss = _core.ndarray((batch_size, ), 'f')
+    probs_desc = create_tensor_descriptor(probs)
+    gradients_desc = create_tensor_descriptor(gradients)
+
+    work_size = cudnn.getCTCLossWorkspaceSize(
+        handle, probs_desc.value, gradients_desc.value,
+        labels_ptr, label_length_ptr,
+        input_length_ptr, algo, ctc_desc.value)
+    workspace = _core.ndarray((work_size,), 'b')
+
+    cudnn.CTCLoss(handle, probs_desc.value, probs.data.ptr,
+                  labels_ptr, label_length_ptr,
+                  input_length_ptr, loss.data.ptr, gradients_desc.value,
+                  gradients.data.ptr, algo, ctc_desc.value,
+                  workspace.data.ptr, work_size)
+    return loss, gradients
+
+
+def create_rnn_descriptor(hidden_size, num_layers, dropout_desc,
+                          input_mode, direction, mode, data_type, algo=None):
+    desc = Descriptor(cudnn.createRNNDescriptor(),
+                      _py_cudnn.destroyRNNDescriptor)
+    if cudnn_version() >= 6000:
+        _handle = get_handle()
+        if algo is None:
+            algo = cudnn.CUDNN_RNN_ALGO_STANDARD
+        cudnn.setRNNDescriptor_v6(
+            _handle, desc.value, hidden_size, num_layers, dropout_desc.value,
+            input_mode, direction, mode, algo, data_type)
+    else:
+        cudnn.setRNNDescriptor_v5(
+            desc.value, hidden_size, num_layers, dropout_desc.value,
+            input_mode, direction, mode, data_type)
+    return desc
+
+
+def get_rnn_lin_layer_matrix_params(
+        handle, rnn_desc, layer, x_desc, w_desc, _ndarray_base w,
+        lin_layer_id):
+    cdef size_t ptr = 0
+    mat_desc = cudnn.createFilterDescriptor()
+    try:
+        cudnn.getRNNLinLayerMatrixParams(
+            handle, rnn_desc.value, layer, x_desc.value, w_desc.value,
+            w.data.ptr, lin_layer_id, mat_desc, <size_t>&ptr)
+        data_type, _, _, dim = cudnn.getFilterNdDescriptor(mat_desc, 3)
+    finally:
+        cudnn.destroyFilterDescriptor(mat_desc)
+    byte_size = _get_byte_size(data_type)
+    offset = (ptr - w.data.ptr) // byte_size
+    size = internal.prod_sequence(dim)
+    mat = w[offset:offset + size]
+    return mat
+
+
+def get_rnn_lin_layer_bias_params(
+        handle, rnn_desc, layer, x_desc, w_desc, _ndarray_base w,
+        lin_layer_id):
+    cdef size_t ptr = 0
+    bias_desc = cudnn.createFilterDescriptor()
+    try:
+        cudnn.getRNNLinLayerBiasParams(
+            handle, rnn_desc.value, layer, x_desc.value, w_desc.value,
+            w.data.ptr, lin_layer_id, bias_desc, <size_t>&ptr)
+        data_type, _, _, dim = cudnn.getFilterNdDescriptor(bias_desc, 3)
+    finally:
+        cudnn.destroyFilterDescriptor(bias_desc)
+    byte_size = _get_byte_size(data_type)
+    offset = (ptr - w.data.ptr) // byte_size
+    size = internal.prod_sequence(dim)
+    bias = w[offset:offset + size]
+    return bias
+
+
+cdef class _DescriptorArray:
+
+    cdef:
+        vector.vector[size_t] _value
+        object _destroy
+
+    def __init__(self, destroyer):
+        self._destroy = destroyer
+
+    def __dealloc__(self):
+        for desc in self._value:
+            self._destroy(desc)
+
+    def append(self, desc):
+        self._value.push_back(desc)
+
+    @property
+    def data(self):
+        return <size_t>self._value.data()
+
+
+cdef _DescriptorArray _make_tensor_descriptor_array(xs, lengths):
+    """Make an array of pointers denoting pointers of tensor descriptors.
+
+    """
+    cdef _DescriptorArray descs = _DescriptorArray(
+        _py_cudnn.destroyTensorDescriptor)
+    cdef size_t desc
+    cdef int data_type = get_data_type(xs.dtype)
+    cdef vector.vector[int] c_shape, c_strides
+    cdef Py_ssize_t itemsize = xs.itemsize
+    cdef Py_ssize_t s
+    cdef int length
+
+    # RNN APIs assumes ndim == 3.
+    for s in xs._strides:
+        c_strides.push_back(s // itemsize)
+    for _ in range(3 - len(xs._strides)):
+        c_strides.push_back(1)
+    for s in xs._shape:
+        c_shape.push_back(s)
+    for _ in range(3 - len(xs._strides)):
+        c_shape.push_back(1)
+
+    for length in lengths:
+        c_shape[0] = length
+        desc = cudnn.createTensorDescriptor()
+        descs.append(desc)
+        cudnn.setTensorNdDescriptor(
+            desc, data_type, 3,
+            <size_t>c_shape.data(), <size_t>c_strides.data())
+
+    return descs
+
+
+cdef _DescriptorArray _make_tensor_descriptor_array_for_padded(xs):
+    assert xs.ndim == 3
+
+    cdef _DescriptorArray descs = _DescriptorArray(
+        _py_cudnn.destroyTensorDescriptor)
+    cdef size_t desc
+    cdef int data_type = get_data_type(xs.dtype)
+    cdef Py_ssize_t itemsize = xs.itemsize
+
+    # RNN APIs assumes ndim == 3.
+    cdef vector.vector[int] c_shape = [xs._shape[1], xs._shape[2], 1]
+    cdef vector.vector[int] c_strides = [
+        xs._strides[1] // itemsize, xs._strides[2] // itemsize, 1]
+
+    for _ in range(xs._shape[0]):
+        desc = cudnn.createTensorDescriptor()
+        descs.append(desc)
+        cudnn.setTensorNdDescriptor(
+            desc, data_type, 3,
+            <size_t>c_shape.data(), <size_t>c_strides.data())
+
+    return descs
+
+
+cdef _memory.MemoryPointer _make_rnn_workspace(
+        Descriptor rnn_desc, int length, _DescriptorArray descs):
+    cdef size_t handle = get_handle()
+    cdef size_t work_size = cudnn.getRNNWorkspaceSize(
+        handle, rnn_desc.value, length, descs.data)
+    return _memory.alloc(work_size)
+
+
+cdef _memory.MemoryPointer _make_rnn_reserve_space(
+        Descriptor rnn_desc, int length, _DescriptorArray descs):
+    cdef size_t handle = get_handle()
+    cdef size_t reserve_size = cudnn.getRNNTrainingReserveSize(
+        handle, rnn_desc.value, length, descs.data)
+    return _memory.alloc(reserve_size)
+
+
+cdef Py_ssize_t _get_n_layers(int direction_mode, _ndarray_base hx):
+    if direction_mode == cudnn.CUDNN_BIDIRECTIONAL:
+        return hx._shape[0] // 2
+    else:  # cudnn.CUDNN_UNIDIRECTIONAL
+        return hx._shape[0]
+
+
+cdef _ndarray_base _make_rnn_result_array(
+        int direction_mode, Py_ssize_t n_units, _ndarray_base xs):
+    cdef int output_units
+    if direction_mode == cudnn.CUDNN_BIDIRECTIONAL:
+        output_units = n_units * 2
+    else:  # cudnn.CUDNN_UNIDIRECTIONAL
+        output_units = n_units
+
+    shape = xs.shape[:-1] + (output_units,)
+    return _core.ndarray(shape, dtype=xs.dtype)
+
+
+def rnn_forward_inference(
+        DropoutStates states, int direction_mode, int rnn_mode,
+        _ndarray_base hx, _ndarray_base cx, _ndarray_base w, _ndarray_base xs,
+        lengths):
+    hx = core._internal_ascontiguousarray(hx)
+    if cx is not None:
+        cx = core._internal_ascontiguousarray(cx)
+    w = core._internal_ascontiguousarray(w)
+    xs = core._internal_ascontiguousarray(xs)
+
+    cdef int length = len(lengths)
+    cdef int n_layers = _get_n_layers(direction_mode, hx)
+    cdef int n_units = hx.shape[2]
+
+    cdef size_t handle = get_handle()
+
+    cdef Descriptor rnn_desc = create_rnn_descriptor(
+        n_units, n_layers, states._desc,
+        cudnn.CUDNN_LINEAR_INPUT, direction_mode,
+        rnn_mode, get_data_type(xs.dtype))
+
+    cdef _ndarray_base ys = _make_rnn_result_array(direction_mode, n_units, xs)
+    cdef _ndarray_base hy = _core.ndarray(hx.shape, hx.dtype)
+    if cx is None:
+        cx = _core.ndarray(0, dtype=xs.dtype)
+    cdef _ndarray_base cy = _core.ndarray(cx.shape, cx.dtype)
+
+    cdef _DescriptorArray xs_descs = _make_tensor_descriptor_array(xs, lengths)
+    cdef Descriptor hx_desc = create_tensor_nd_descriptor(hx)
+    cdef Descriptor cx_desc = create_tensor_nd_descriptor(cx)
+    cdef Descriptor w_desc = create_filter_descriptor(w)
+    cdef _DescriptorArray ys_descs = _make_tensor_descriptor_array(ys, lengths)
+    cdef Descriptor hy_desc = create_tensor_nd_descriptor(hy)
+    cdef Descriptor cy_desc = create_tensor_nd_descriptor(cy)
+
+    cdef _memory.MemoryPointer workspace = _make_rnn_workspace(
+        rnn_desc, length, xs_descs)
+
+    cudnn.RNNForwardInference(
+        handle, rnn_desc.value, length,
+        xs_descs.data, xs.data.ptr, hx_desc.value, hx.data.ptr,
+        cx_desc.value, cx.data.ptr, w_desc.value, w.data.ptr,
+        ys_descs.data, ys.data.ptr, hy_desc.value, hy.data.ptr,
+        cy_desc.value, cy.data.ptr, workspace.ptr, workspace.mem.size)
+
+    return hy, cy, ys
+
+
+def rnn_forward_training(
+        DropoutStates states, int direction_mode, int rnn_mode,
+        _ndarray_base hx, _ndarray_base cx, _ndarray_base w, _ndarray_base xs,
+        lengths):
+    hx = core._internal_ascontiguousarray(hx)
+    if cx is not None:
+        cx = core._internal_ascontiguousarray(cx)
+    w = core._internal_ascontiguousarray(w)
+    xs = core._internal_ascontiguousarray(xs)
+
+    cdef int length = len(lengths)
+    cdef int n_layers = _get_n_layers(direction_mode, hx)
+    cdef int n_units = hx.shape[2]
+
+    cdef size_t handle = get_handle()
+
+    cdef Descriptor rnn_desc = create_rnn_descriptor(
+        n_units, n_layers, states._desc,
+        cudnn.CUDNN_LINEAR_INPUT, direction_mode,
+        rnn_mode, get_data_type(xs.dtype))
+
+    cdef _ndarray_base ys = _make_rnn_result_array(direction_mode, n_units, xs)
+    cdef _ndarray_base hy = _core.ndarray(hx.shape, hx.dtype)
+    if cx is None:
+        cx = _core.ndarray(0, dtype=xs.dtype)
+    cdef _ndarray_base cy = _core.ndarray(cx.shape, cx.dtype)
+
+    cdef _DescriptorArray xs_descs = _make_tensor_descriptor_array(xs, lengths)
+    cdef Descriptor hx_desc = create_tensor_nd_descriptor(hx)
+    cdef Descriptor cx_desc = create_tensor_nd_descriptor(cx)
+    cdef Descriptor w_desc = create_filter_descriptor(w)
+    cdef _DescriptorArray ys_descs = _make_tensor_descriptor_array(ys, lengths)
+    cdef Descriptor hy_desc = create_tensor_nd_descriptor(hy)
+    cdef Descriptor cy_desc = create_tensor_nd_descriptor(cy)
+
+    cdef _memory.MemoryPointer workspace = _make_rnn_workspace(
+        rnn_desc, length, xs_descs)
+    cdef _memory.MemoryPointer reserve_space = _make_rnn_reserve_space(
+        rnn_desc, length, xs_descs)
+
+    cudnn.RNNForwardTraining(
+        handle, rnn_desc.value, length,
+        xs_descs.data, xs.data.ptr, hx_desc.value, hx.data.ptr,
+        cx_desc.value, cx.data.ptr, w_desc.value, w.data.ptr,
+        ys_descs.data, ys.data.ptr, hy_desc.value, hy.data.ptr,
+        cy_desc.value, cy.data.ptr, workspace.ptr, workspace.mem.size,
+        reserve_space.ptr, reserve_space.mem.size)
+
+    return reserve_space, hy, cy, ys
+
+
+def rnn_backward_data(
+        DropoutStates states, int direction_mode, int rnn_mode,
+        _ndarray_base hx, _ndarray_base cx, _ndarray_base w, _ndarray_base xs,
+        _ndarray_base ys, _memory.MemoryPointer reserve_space,
+        _ndarray_base dhy, _ndarray_base dcy, _ndarray_base dys,
+        lengths):
+    hx = core._internal_ascontiguousarray(hx)
+    if cx is not None:
+        cx = core._internal_ascontiguousarray(cx)
+    w = core._internal_ascontiguousarray(w)
+    xs = core._internal_ascontiguousarray(xs)
+    ys = _ascontiguousarray_normalized_strides(ys)
+    dhy = core._internal_ascontiguousarray(dhy)
+    if dcy is not None:
+        dcy = core._internal_ascontiguousarray(dcy)
+    dys = _ascontiguousarray_normalized_strides(dys)
+
+    cdef int length = len(lengths)
+    cdef int n_layers = _get_n_layers(direction_mode, hx)
+    cdef int n_units = hx.shape[2]
+
+    cdef size_t handle = get_handle()
+    cdef Descriptor rnn_desc = create_rnn_descriptor(
+        n_units, n_layers, states._desc,
+        cudnn.CUDNN_LINEAR_INPUT, direction_mode,
+        rnn_mode, get_data_type(xs.dtype))
+
+    cdef _ndarray_base dxs = _core.ndarray(xs.shape, xs.dtype)
+    cdef _ndarray_base dhx = _core.ndarray(hx.shape, hx.dtype)
+    if cx is None:
+        cx = dcy = _core.ndarray(0, dtype=xs.dtype)
+    cdef _ndarray_base dcx = _core.ndarray(cx.shape, cx.dtype)
+
+    cdef _DescriptorArray ys_descs = _make_tensor_descriptor_array(ys, lengths)
+    cdef _DescriptorArray dys_descs = _make_tensor_descriptor_array(
+        dys, lengths)
+    cdef Descriptor dhy_desc = create_tensor_nd_descriptor(dhy)
+    cdef Descriptor dcy_desc = create_tensor_nd_descriptor(dcy)
+    cdef Descriptor w_desc = create_filter_descriptor(w)
+    cdef Descriptor hx_desc = create_tensor_nd_descriptor(hx)
+    cdef Descriptor cx_desc = create_tensor_nd_descriptor(cx)
+    cdef _DescriptorArray dxs_descs = _make_tensor_descriptor_array(
+        dxs, lengths)
+    cdef Descriptor dhx_desc = create_tensor_nd_descriptor(dhx)
+    cdef Descriptor dcx_desc = create_tensor_nd_descriptor(dcx)
+
+    cdef _DescriptorArray xs_descs = _make_tensor_descriptor_array(xs, lengths)
+    cdef _memory.MemoryPointer workspace = _make_rnn_workspace(
+        rnn_desc, length, xs_descs)
+
+    cudnn.RNNBackwardData(
+        handle, rnn_desc.value, length,
+        ys_descs.data, ys.data.ptr,
+        dys_descs.data, dys.data.ptr, dhy_desc.value, dhy.data.ptr,
+        dcy_desc.value, dcy.data.ptr, w_desc.value, w.data.ptr,
+        hx_desc.value, hx.data.ptr, cx_desc.value, cx.data.ptr,
+        dxs_descs.data, dxs.data.ptr, dhx_desc.value, dhx.data.ptr,
+        dcx_desc.value, dcx.data.ptr, workspace.ptr, workspace.mem.size,
+        reserve_space.ptr, reserve_space.mem.size)
+
+    return dhx, dcx, dxs
+
+
+def rnn_backward_weights(
+        DropoutStates states, int direction_mode, int rnn_mode,
+        _ndarray_base xs, _ndarray_base hx, _ndarray_base ys,
+        _ndarray_base w,
+        _memory.MemoryPointer reserve_space, lengths):
+    xs = core._internal_ascontiguousarray(xs)
+    hx = core._internal_ascontiguousarray(hx)
+    ys = core._internal_ascontiguousarray(ys)
+    w = core._internal_ascontiguousarray(w)
+
+    cdef int length = len(lengths)
+    cdef int n_layers = _get_n_layers(direction_mode, hx)
+    cdef int n_units = hx.shape[2]
+
+    cdef size_t handle = get_handle()
+    cdef Descriptor rnn_desc = create_rnn_descriptor(
+        n_units, n_layers, states._desc,
+        cudnn.CUDNN_LINEAR_INPUT, direction_mode,
+        rnn_mode, get_data_type(xs.dtype))
+
+    cdef _DescriptorArray xs_descs = _make_tensor_descriptor_array(xs, lengths)
+    cdef Descriptor hx_desc = create_tensor_nd_descriptor(hx)
+    cdef _DescriptorArray ys_descs = _make_tensor_descriptor_array(ys, lengths)
+
+    cdef _memory.MemoryPointer workspace = _make_rnn_workspace(
+        rnn_desc, length, xs_descs)
+
+    cdef _ndarray_base dw = _core.ndarray(w.shape, w.dtype)
+    dw.fill(0)
+    cdef Descriptor dw_desc = create_filter_descriptor(dw)
+
+    cudnn.RNNBackwardWeights(
+        handle, rnn_desc.value, length,
+        xs_descs.data, xs.data.ptr,
+        hx_desc.value, hx.data.ptr, ys_descs.data, ys.data.ptr,
+        workspace.ptr, workspace.mem.size, dw_desc.value, dw.data.ptr,
+        reserve_space.ptr, reserve_space.mem.size)
+    return dw
+
+
+def create_dropout_states(handle):
+    _warnings.warn('create_dropout_states is deprecated.'
+                   'Please use DropoutStates class instead.',
+                   DeprecationWarning)
+    state_size = cudnn.dropoutGetStatesSize(handle)
+    return _core.ndarray((state_size,), 'b')
+
+
+def create_spatial_transformer_descriptor(sampler_type, dtype, nb_dims, dim_A):
+    desc = Descriptor(cudnn.createSpatialTransformerDescriptor(),
+                      _py_cudnn.destroySpatialTransformerDescriptor)
+    data_type = get_data_type(dtype)
+
+    cudnn.setSpatialTransformerDescriptor(
+        desc.value, sampler_type, data_type, nb_dims, dim_A)
+    return desc
+
+
+def add_tensor(handle, alpha, biasDesc, biasData, beta, srcDestDesc,
+               srcDestData):
+    cudnn.addTensor_v3(handle, alpha, biasDesc,
+                       biasData, beta, srcDestDesc, srcDestData)
+
+
+def create_op_tensor_descriptor(op_type, dtype):
+    desc = Descriptor(cudnn.createOpTensorDescriptor(),
+                      _py_cudnn.destroyOpTensorDescriptor)
+    data_type = get_data_type(dtype)
+
+    cudnn.setOpTensorDescriptor(desc.value, op_type, data_type,
+                                cudnn.CUDNN_NOT_PROPAGATE_NAN)
+    return desc
+
+
+def create_reduce_tensor_descriptor(reduce_type, dtype):
+    desc = Descriptor(cudnn.createReduceTensorDescriptor(),
+                      _py_cudnn.destroyReduceTensorDescriptor)
+    data_type = get_data_type(dtype)
+    if reduce_type in (cudnn.CUDNN_REDUCE_TENSOR_MIN,
+                       cudnn.CUDNN_REDUCE_TENSOR_MAX):
+        indices = cudnn.CUDNN_REDUCE_TENSOR_FLATTENED_INDICES
+    else:
+        indices = cudnn.CUDNN_REDUCE_TENSOR_NO_INDICES
+
+    cudnn.setReduceTensorDescriptor(desc.value, reduce_type, data_type,
+                                    cudnn.CUDNN_NOT_PROPAGATE_NAN,
+                                    indices,
+                                    cudnn.CUDNN_32BIT_INDICES)
+    return desc
+
+
+cpdef bint is_tensor_core_available(dtype) except *:
+    return (cudnn_version() >= 7000 and
+            (<str>dtype.char) == 'e' and
+            int(device.get_compute_capability()) == 70)
+
+
+cdef class DropoutStates:
+
+    cdef public:
+        # TODO(unno): Make these attributes private. This is for backward
+        # compatibility.
+        _memory.MemoryPointer _states
+        Descriptor _desc
+
+    def __init__(self, handle, seed):
+        cdef size_t cudnn_handle
+        if handle is None:
+            cudnn_handle = get_handle()
+        else:
+            cudnn_handle = handle
+        state_size = cudnn.dropoutGetStatesSize(cudnn_handle)
+        self._states = _memory.alloc(state_size)
+        self._desc = create_dropout_descriptor(
+            cudnn_handle, 0., self._states.ptr,
+            state_size, seed)
+
+    def set_dropout_ratio(self, dropout_ratio):
+        cudnn_handle = get_handle()
+        set_dropout_descriptor(self._desc, cudnn_handle, dropout_ratio)
+
+    def forward(self, handle, _ndarray_base x, dropout_ratio):
+        cdef _ndarray_base y, reserve_space
+        cdef size_t cudnn_handle
+        # This is for backward compatibility.
+        if handle is None:
+            cudnn_handle = get_handle()
+        else:
+            cudnn_handle = handle
+        set_dropout_descriptor(self._desc, cudnn_handle, dropout_ratio)
+
+        x = core._internal_ascontiguousarray(x)
+        y = _core.ndarray(x._shape, x.dtype)
+
+        x_desc = cudnn.createTensorDescriptor()
+        try:
+            _create_tensor_descriptor_as4darray(x_desc, x)
+            reserve_size = cudnn.getDropoutReserveSpaceSize(x_desc)
+            reserve_space = _core.ndarray((reserve_size,), 'b')
+
+            cudnn.dropoutForward(cudnn_handle, self._desc.value,
+                                 x_desc, x.data.ptr, x_desc, y.data.ptr,
+                                 reserve_space.data.ptr, reserve_size)
+        finally:
+            cudnn.destroyTensorDescriptor(x_desc)
+        return reserve_space, y
+
+    def backward(self, handle, _ndarray_base dy, dropout_ratio,
+                 _ndarray_base reserve_space):
+        cdef _ndarray_base dx
+        cdef size_t cudnn_handle
+        # This is for backward compatibility.
+        if handle is None:
+            cudnn_handle = get_handle()
+        else:
+            cudnn_handle = handle
+        set_dropout_descriptor(self._desc, cudnn_handle, dropout_ratio)
+
+        dy = core._internal_ascontiguousarray(dy)
+        dx = _core.ndarray(dy._shape, dy.dtype)
+
+        dy_desc = cudnn.createTensorDescriptor()
+        try:
+            _create_tensor_descriptor_as4darray(dy_desc, dy)
+            cudnn.dropoutBackward(cudnn_handle, self._desc.value,
+                                  dy_desc, dy.data.ptr,
+                                  dy_desc, dx.data.ptr,
+                                  reserve_space.data.ptr,
+                                  reserve_space.size)
+        finally:
+            cudnn.destroyTensorDescriptor(dy_desc)
+        return dx
+
+
+cdef class _Algorithm:
+    cdef:
+        int algo
+        int mathType
+        size_t memory
+
+    def __cinit__(self, int algo, size_t memory, int mathType=0):
+        self.algo = algo
+        self.memory = memory
+        self.mathType = mathType
+
+
+cdef dict _get_algorithm_fwd_cache = {}
+cdef dict _get_algorithm_bwd_filter_cache = {}
+cdef dict _get_algorithm_bwd_data_cache = {}
+cdef dict _algorithm_fwd_cache = {}
+cdef dict _algorithm_bwd_filter_cache = {}
+cdef dict _algorithm_bwd_data_cache = {}
+
+
+cpdef _warn_algorithm_fwd(
+        _ndarray_base x, _ndarray_base W, _ndarray_base y, tuple conv_param):
+    _warnings.warn(
+        'Tensor Core mode is set but the selected convolution forward '
+        'algorithm is not a Tensor Core enabled algorithm. '
+        'This might be due to lack of workspace memory. '
+        'x.shape:{}, W.shape:{}, y.shape:{}, pad:{}, stride:{}'
+        .format(x.shape, W.shape, y.shape, conv_param[0], conv_param[1]),
+        _util.PerformanceWarning)
+
+
+cpdef _Algorithm _find_algorithm_fwd(
+        _ndarray_base x, _ndarray_base W, _ndarray_base y, tuple conv_param,
+        size_t handle, size_t x_desc, size_t filter_desc, size_t conv_desc,
+        size_t y_desc, size_t max_workspace_size, bint use_tensor_core):
+    cdef cudnn.CuDNNAlgoPerf perf
+    key = (x.data.device.id, x.shape, W.shape, y.shape, conv_param,
+           max_workspace_size)
+    algo = _algorithm_fwd_cache.get(key, None)
+    if algo is not None:
+        return algo
+    workspace = _memory.alloc(max_workspace_size)
+    if cudnn_version() >= 7000:
+        perf = cudnn.findConvolutionForwardAlgorithmEx_v7(
+            handle, x_desc, x.data.ptr, filter_desc, W.data.ptr, conv_desc,
+            y_desc, y.data.ptr, 1, workspace.ptr, max_workspace_size)[0]
+        if use_tensor_core and perf.mathType != cudnn.CUDNN_TENSOR_OP_MATH:
+            _warn_algorithm_fwd(x, W, y, conv_param)
+    else:
+        perf = cudnn.findConvolutionForwardAlgorithmEx(
+            handle, x_desc, x.data.ptr, filter_desc, W.data.ptr, conv_desc,
+            y_desc, y.data.ptr, 1, workspace.ptr, max_workspace_size)[0]
+    if perf.status != cudnn.CUDNN_STATUS_SUCCESS:
+        raise RuntimeError('No available algorithm found.')
+    algo = _Algorithm(perf.algo, perf.memory, perf.mathType)
+    _algorithm_fwd_cache[key] = algo
+    return algo
+
+
+cpdef _Algorithm _get_algorithm_fwd(
+        _ndarray_base x, _ndarray_base W, _ndarray_base y, tuple conv_param,
+        size_t handle, size_t x_desc, size_t filter_desc, size_t conv_desc,
+        size_t y_desc, size_t max_workspace_size, bint use_tensor_core):
+    cdef cudnn.CuDNNAlgoPerf perf
+    key = (x.data.device.id, x.shape, W.shape, y.shape, conv_param,
+           max_workspace_size)
+    algo = _get_algorithm_fwd_cache.get(key, None)
+    if algo is not None:
+        return algo
+    cdef list ret
+    cdef bint skip
+    cdef int cudnn_ver = cudnn_version()
+    if (use_tensor_core and cudnn_ver >= 7000) or cudnn_ver >= 8000:
+        ret = cudnn.getConvolutionForwardAlgorithm_v7(
+            handle, x_desc, filter_desc, conv_desc, y_desc, 10)
+        skip = False
+        for perf in ret:
+            if perf.memory <= max_workspace_size:
+                break
+            skip = True
+        else:
+            raise RuntimeError('No conv fwd algo available with workspace size'
+                               ' less equal {}'.format(max_workspace_size))
+        if skip:
+            _warnings.warn(
+                'The best algo of conv fwd might not be selected due to '
+                'lack of workspace size ({})'.format(max_workspace_size),
+                _util.PerformanceWarning)
+        algo = perf.algo
+        workspace_size = perf.memory
+        math_type = perf.mathType
+        if use_tensor_core and math_type != cudnn.CUDNN_TENSOR_OP_MATH:
+            _warn_algorithm_fwd(x, W, y, conv_param)
+        algo = _Algorithm(perf.algo, perf.memory, perf.mathType)
+    else:
+        algo_no = cudnn.getConvolutionForwardAlgorithm_v6(
+            handle, x_desc, filter_desc, conv_desc, y_desc,
+            cudnn.CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT,
+            max_workspace_size)
+        workspace_size = cudnn.getConvolutionForwardWorkspaceSize(
+            handle, x_desc, filter_desc, conv_desc, y_desc, algo_no)
+        algo = _Algorithm(algo_no, workspace_size)
+    _get_algorithm_fwd_cache[key] = algo
+    return algo
+
+
+cpdef _warn_algorithm_bwd_filter(
+        _ndarray_base x, _ndarray_base dy, _ndarray_base dW, tuple conv_param):
+    _warnings.warn(
+        'Tensor Core mode is set but the selected convolution backward '
+        'filter algorithm is not a Tensor Core enabled algorithm. '
+        'This might be due to lack of workspace memory. '
+        'x.shape:{}, dy.shape:{}, dW.shape:{}, pad:{}, stride:{}'
+        .format(x.shape, dy.shape, dW.shape, conv_param[0], conv_param[1]),
+        _util.PerformanceWarning)
+
+
+cpdef _Algorithm _find_algorithm_bwd_filter(
+        _ndarray_base x, _ndarray_base dy, _ndarray_base dW, tuple conv_param,
+        size_t handle, size_t x_desc, size_t dy_desc, size_t conv_desc,
+        size_t filter_desc, size_t max_workspace_size, bint use_tensor_core,
+        bint deterministic):
+    cdef cudnn.CuDNNAlgoPerf perf
+    cdef _Algorithm algo
+    key = (x.data.device.id, x.shape, dW.shape, dy.shape, conv_param,
+           max_workspace_size)
+    algo = _algorithm_bwd_filter_cache.get(key, None)
+    if algo is not None:
+        return algo
+    workspace = _memory.alloc(max_workspace_size)
+    if cudnn_version() >= 7000:
+        if deterministic:
+            ret = cudnn.findConvolutionBackwardFilterAlgorithmEx_v7(
+                handle, x_desc, x.data.ptr, dy_desc, dy.data.ptr, conv_desc,
+                filter_desc, dW.data.ptr, 10, workspace.ptr,
+                max_workspace_size)
+            for perf in ret:
+                if perf.determinism:
+                    break
+            else:
+                raise RuntimeError(
+                    'No conv bwd filter algo available with workspace size '
+                    'less equal {}'.format(max_workspace_size))
+        else:
+            perf = cudnn.findConvolutionBackwardFilterAlgorithmEx_v7(
+                handle, x_desc, x.data.ptr, dy_desc, dy.data.ptr, conv_desc,
+                filter_desc, dW.data.ptr, 1, workspace.ptr,
+                max_workspace_size)[0]
+        if use_tensor_core and perf.mathType != cudnn.CUDNN_TENSOR_OP_MATH:
+            _warn_algorithm_bwd_filter(x, dy, dW, conv_param)
+    else:
+        perf = cudnn.findConvolutionBackwardFilterAlgorithmEx(
+            handle, x_desc, x.data.ptr, dy_desc, dy.data.ptr, conv_desc,
+            filter_desc, dW.data.ptr, 1, workspace.ptr, max_workspace_size)[0]
+    if perf.status != cudnn.CUDNN_STATUS_SUCCESS:
+        raise RuntimeError('No available algorithm found.')
+    algo = _Algorithm(perf.algo, perf.memory, perf.mathType)
+    _algorithm_bwd_filter_cache[key] = algo
+    return algo
+
+
+cpdef _Algorithm _get_algorithm_bwd_filter(
+        _ndarray_base x, _ndarray_base dy, _ndarray_base dW, tuple conv_param,
+        size_t handle, size_t x_desc, size_t gy_desc, size_t conv_desc,
+        size_t filter_desc, size_t max_workspace_size, bint use_tensor_core,
+        bint deterministic):
+    cdef cudnn.CuDNNAlgoPerf perf
+    key = (x.data.device.id, x.shape, dW.shape, dy.shape, conv_param,
+           max_workspace_size)
+    algo = _get_algorithm_bwd_filter_cache.get(key, None)
+    if algo is not None:
+        return algo
+    cdef list ret
+    cdef bint skip
+    if cudnn_version() >= 7000:
+        ret = cudnn.getConvolutionBackwardFilterAlgorithm_v7(
+            handle, x_desc, gy_desc, conv_desc, filter_desc, 10)
+        skip = False
+        for perf in ret:
+            if deterministic and not perf.determinism:
+                continue
+            if perf.memory <= max_workspace_size:
+                break
+            skip = True
+        else:
+            raise RuntimeError(
+                'No conv bwd filter algo available with workspace size less '
+                'equal {}'.format(max_workspace_size))
+        if use_tensor_core and skip:
+            _warnings.warn(
+                'The best algo of conv bwd filter might not not selected due '
+                'to lack of workspace size ({})'.format(max_workspace_size),
+                _util.PerformanceWarning)
+        algo = perf.algo
+        workspace_size = perf.memory
+        math_type = perf.mathType
+        if use_tensor_core and math_type != cudnn.CUDNN_TENSOR_OP_MATH:
+            _warn_algorithm_bwd_filter(x, dy, dW, conv_param)
+        algo = _Algorithm(perf.algo, perf.memory, perf.mathType)
+    else:
+        algo_no = cudnn.getConvolutionBackwardFilterAlgorithm_v6(
+            handle, x_desc, gy_desc, conv_desc, filter_desc,
+            cudnn.CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT,
+            max_workspace_size)
+        workspace_size = cudnn.getConvolutionBackwardFilterWorkspaceSize(
+            handle, x_desc, gy_desc, conv_desc, filter_desc, algo_no)
+        algo = _Algorithm(algo_no, workspace_size)
+    _get_algorithm_bwd_filter_cache[key] = algo
+    return algo
+
+
+cpdef _warn_algorithm_bwd_data(
+        _ndarray_base W, _ndarray_base x, _ndarray_base y, tuple conv_param):
+    _warnings.warn(
+        'Tensor Core mode is set but the selected convolution backward '
+        'data algorithm is not a Tensor Core enabled algorithm. '
+        'This might be due to lack of workspace memory. '
+        'W.shape:{}, x.shape:{}, y.shape:{}, pad:{}, stride:{}'
+        .format(W.shape, x.shape, y.shape, conv_param[0], conv_param[1]),
+        _util.PerformanceWarning)
+
+
+cpdef _Algorithm _find_algorithm_bwd_data(
+        _ndarray_base W, _ndarray_base x, _ndarray_base y, tuple conv_param,
+        size_t handle, size_t filter_desc, size_t x_desc, size_t conv_desc,
+        size_t y_desc, size_t max_workspace_size, bint use_tensor_core,
+        bint deterministic):
+    cdef _Algorithm algo
+    cdef cudnn.CuDNNAlgoPerf perf
+    key = (x.data.device.id, W.shape, x.shape, y.shape, conv_param,
+           max_workspace_size)
+    algo = _algorithm_bwd_data_cache.get(key, None)
+    if algo is not None:
+        return algo
+    workspace = _memory.alloc(max_workspace_size)
+    if cudnn_version() >= 7000:
+        if deterministic:
+            ret = cudnn.findConvolutionBackwardDataAlgorithmEx_v7(
+                handle, filter_desc, W.data.ptr, x_desc, x.data.ptr, conv_desc,
+                y_desc, y.data.ptr, 10, workspace.ptr, max_workspace_size)
+            for perf in ret:
+                if perf.determinism:
+                    break
+            else:
+                raise RuntimeError(
+                    'No conv bwd filter algo available with workspace size '
+                    'less equal {}'.format(max_workspace_size))
+        else:
+            perf = cudnn.findConvolutionBackwardDataAlgorithmEx_v7(
+                handle, filter_desc, W.data.ptr, x_desc, x.data.ptr, conv_desc,
+                y_desc, y.data.ptr, 1, workspace.ptr, max_workspace_size)[0]
+        if use_tensor_core and perf.mathType != cudnn.CUDNN_TENSOR_OP_MATH:
+            _warn_algorithm_bwd_data(W, x, y, conv_param)
+    else:
+        perf = cudnn.findConvolutionBackwardDataAlgorithmEx(
+            handle, filter_desc, W.data.ptr, x_desc, x.data.ptr, conv_desc,
+            y_desc, y.data.ptr, 1, workspace.ptr, max_workspace_size)[0]
+    if perf.status != cudnn.CUDNN_STATUS_SUCCESS:
+        raise RuntimeError('No available algorithm found.')
+    algo = _Algorithm(perf.algo, perf.memory, perf.mathType)
+    _algorithm_bwd_data_cache[key] = algo
+    return algo
+
+
+cpdef _Algorithm _get_algorithm_bwd_data(
+        _ndarray_base W, _ndarray_base x, _ndarray_base y, tuple conv_param,
+        size_t handle, size_t filter_desc, size_t x_desc, size_t conv_desc,
+        size_t y_desc, size_t max_workspace_size, bint use_tensor_core,
+        bint deterministic):
+    cdef cudnn.CuDNNAlgoPerf perf
+    key = (x.data.device.id, W.shape, x.shape, y.shape, conv_param,
+           max_workspace_size)
+    algo = _get_algorithm_bwd_data_cache.get(key, None)
+    if algo is not None:
+        return algo
+    cdef list ret
+    cdef bint skip
+    if cudnn_version() >= 7000:
+        ret = cudnn.getConvolutionBackwardDataAlgorithm_v7(
+            handle, filter_desc, x_desc, conv_desc, y_desc, 10)
+        skip = False
+        for perf in ret:
+            if deterministic and not perf.determinism:
+                continue
+            if perf.memory <= max_workspace_size:
+                break
+            skip = True
+        else:
+            raise RuntimeError(
+                'No conv bwd data algo available with workspace size less '
+                'equal {}'.format(max_workspace_size))
+        if use_tensor_core and skip:
+            _warnings.warn(
+                'The best algo of conv bwd data might not not selected due '
+                'to lack of workspace size ({})'.format(max_workspace_size),
+                _util.PerformanceWarning)
+        algo = perf.algo
+        workspace_size = perf.memory
+        math_type = perf.mathType
+        if use_tensor_core and math_type != cudnn.CUDNN_TENSOR_OP_MATH:
+            _warn_algorithm_bwd_data(W, x, y, conv_param)
+        algo = _Algorithm(perf.algo, perf.memory, perf.mathType)
+    else:
+        algo_no = cudnn.getConvolutionBackwardDataAlgorithm_v6(
+            handle, filter_desc, x_desc, conv_desc, y_desc,
+            cudnn.CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT,
+            max_workspace_size)
+        workspace_size = cudnn.getConvolutionBackwardDataWorkspaceSize(
+            handle, filter_desc, x_desc, conv_desc, y_desc, algo_no)
+        algo = _Algorithm(algo_no, workspace_size)
+    _get_algorithm_bwd_data_cache[key] = algo
+    return algo
+
+
+cpdef bint _should_use_tensor_core(
+        tensor_core_mode, object dtype) except *:
+    if tensor_core_mode == 'auto':
+        return is_tensor_core_available(dtype)
+    elif tensor_core_mode == 'always':
+        # TODO(oktua): more strict condition
+        return is_tensor_core_available(dtype)
+    elif tensor_core_mode == 'never':
+        return False
+    else:
+        raise ValueError(
+            'tensor_code_mode must be either of "always", "auto", or "never".')
+
+
+def _get_array_info(_ndarray_base arr):
+    if arr is None:
+        return 'None'
+    return 'shape={!r}, dtype={}, strides={!r}'.format(
+        arr.shape, arr.dtype.name, arr.strides)
+
+
+def convolution_forward(
+        _ndarray_base x, _ndarray_base W, _ndarray_base b, _ndarray_base y,
+        tuple pad, tuple stride, tuple dilation, int groups, *,
+        bint auto_tune, tensor_core,
+        int d_layout=cudnn.CUDNN_TENSOR_NCHW,
+        int w_layout=cudnn.CUDNN_TENSOR_NCHW):
+    cdef int dev_id = x.data.device.id
+    assert dev_id == W.data.device.id
+    assert dev_id == y.data.device.id
+
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    if x.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+
+    cdef bint use_tensor_core = _should_use_tensor_core(tensor_core, x.dtype)
+    cdef tuple conv_param = (pad, stride, x.dtype, use_tensor_core)
+
+    # cuDNN 7 supports dilation only in *_FWD_ALGO_IMPLICIT_GEMM, but
+    # it supports Tensor Cores only in *_FWD_ALGO_IMPLICIT_PRECOMP_GEMM.
+    if use_tensor_core:
+        for i in dilation:
+            if i > 1:
+                use_tensor_core = False
+                break
+
+    handle = get_handle()
+    x = core._internal_ascontiguousarray(x)
+    W = core._internal_ascontiguousarray(W)
+
+    # TODO(okuta) check performance
+    cdef size_t x_desc = cudnn.createTensorDescriptor()
+    cdef size_t y_desc = cudnn.createTensorDescriptor()
+    cdef size_t b_desc = cudnn.createTensorDescriptor()
+    cdef size_t filter_desc = cudnn.createFilterDescriptor()
+    cdef size_t conv_desc = cudnn.createConvolutionDescriptor()
+
+    cdef size_t max_workspace_size = get_max_workspace_size()
+    cdef shape_t b_shape
+    cdef _Algorithm perf
+    try:
+        _create_tensor_descriptor(x_desc, x, format=d_layout)
+        _create_tensor_descriptor(y_desc, y, format=d_layout)
+        _create_filter_descriptor(filter_desc, W, w_layout)
+        _create_convolution_descriptor(
+            conv_desc, pad, stride, dilation, groups, x.dtype,
+            cudnn.CUDNN_CROSS_CORRELATION, use_tensor_core)
+
+        if auto_tune:
+            perf = _find_algorithm_fwd(
+                x, W, y, conv_param, handle, x_desc, filter_desc,
+                conv_desc, y_desc, max_workspace_size, use_tensor_core)
+        else:
+            perf = _get_algorithm_fwd(
+                x, W, y, conv_param, handle, x_desc, filter_desc,
+                conv_desc, y_desc, max_workspace_size, use_tensor_core)
+
+        if cudnn_version() >= 7000:
+            cudnn.setConvolutionMathType(conv_desc, perf.mathType)
+
+        workspace = _memory.alloc(perf.memory)
+
+        try:
+            cudnn.convolutionForward(
+                handle, one, x_desc, x.data.ptr, filter_desc, W.data.ptr,
+                conv_desc, perf.algo, workspace.ptr, perf.memory, zero, y_desc,
+                y.data.ptr)
+        except _py_cudnn.CuDNNError as e:
+            infos = [
+                'func: cudnnConvolutionForward',
+                'x: {}'.format(_get_array_info(x)),
+                'W: {}'.format(_get_array_info(W)),
+                'b: {}'.format(_get_array_info(b)),
+                'y: {}'.format(_get_array_info(y)),
+                'pad={!r}, stride={!r}, dilation={!r}, groups={!r}'.format(
+                    pad, stride, dilation, groups),
+                'auto_tune={!r}, tensor_core={!r}'.format(
+                    auto_tune, tensor_core),
+                'd_layout={!r}, w_layout={!r}'.format(d_layout, w_layout),
+            ]
+            e.add_infos(infos)
+            raise
+
+        del workspace, x, W
+
+        if b is not None:
+            assert dev_id == b.data.device.id
+            b_shape.assign(y._shape.size(), 1)
+            b_shape[1] = -1
+            b = _manipulation._reshape(
+                core._internal_ascontiguousarray(b), b_shape)
+            _create_tensor_nd_descriptor(b_desc, b, -1)
+            cudnn.addTensor_v3(handle, one, b_desc,
+                               b.data.ptr, one, y_desc, y.data.ptr)
+    finally:
+        cudnn.destroyTensorDescriptor(x_desc)
+        cudnn.destroyTensorDescriptor(y_desc)
+        cudnn.destroyTensorDescriptor(b_desc)
+        cudnn.destroyFilterDescriptor(filter_desc)
+        cudnn.destroyConvolutionDescriptor(conv_desc)
+
+
+def convolution_backward_filter(
+        _ndarray_base x, _ndarray_base gy, _ndarray_base gW,
+        tuple pad, tuple stride, tuple dilation, int groups, *,
+        bint deterministic, bint auto_tune, tensor_core,
+        int d_layout=cudnn.CUDNN_TENSOR_NCHW,
+        int w_layout=cudnn.CUDNN_TENSOR_NCHW):
+    cdef int dev_id = x.data.device.id
+    assert dev_id == gy.data.device.id
+    assert dev_id == gW.data.device.id
+
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    if x.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+
+    # Disable use_tensor_core in deterministic mode because
+    # CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 does not use Tensor Core.
+    cdef bint use_tensor_core = (
+        not deterministic and _should_use_tensor_core(tensor_core, x.dtype))
+    cdef tuple conv_param = (
+        pad, stride, x.dtype, use_tensor_core, deterministic)
+
+    handle = get_handle()
+    x = core._internal_ascontiguousarray(x)
+    gy = core._internal_ascontiguousarray(gy)
+
+    # TODO(okuta) check performance
+    cdef size_t x_desc = cudnn.createTensorDescriptor()
+    cdef size_t gy_desc = cudnn.createTensorDescriptor()
+    cdef size_t filter_desc = cudnn.createFilterDescriptor()
+    cdef size_t conv_desc = cudnn.createConvolutionDescriptor()
+
+    cdef _Algorithm perf
+    cdef int algo
+    cdef size_t max_workspace_size = get_max_workspace_size()
+    cdef size_t workspace_size = 0
+    try:
+        _create_tensor_descriptor(x_desc, x, format=d_layout)
+        _create_tensor_descriptor(gy_desc, gy, format=d_layout)
+        _create_filter_descriptor(filter_desc, gW, w_layout)
+        _create_convolution_descriptor(
+            conv_desc, pad, stride, dilation, groups, x.dtype,
+            cudnn.CUDNN_CROSS_CORRELATION, use_tensor_core)
+
+        if deterministic and cudnn_version() < 7000:
+            # TODO(imanishi): Support Tensor Core in deterministic mode.
+            algo = cudnn.CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1
+            workspace_size = cudnn.getConvolutionBackwardFilterWorkspaceSize(
+                handle, x_desc, gy_desc, conv_desc, filter_desc, algo)
+            math_type = cudnn.CUDNN_DEFAULT_MATH
+            if workspace_size > max_workspace_size:
+                raise RuntimeError(
+                    'No conv bwd filter algo available with workspace size '
+                    'less equal {}'.format(max_workspace_size))
+        else:
+            if auto_tune and not deterministic:
+                perf = _find_algorithm_bwd_filter(
+                    x, gy, gW, conv_param, handle, x_desc, gy_desc, conv_desc,
+                    filter_desc, max_workspace_size, use_tensor_core,
+                    deterministic)
+            else:
+                perf = _get_algorithm_bwd_filter(
+                    x, gy, gW, conv_param, handle, x_desc, gy_desc, conv_desc,
+                    filter_desc, max_workspace_size, use_tensor_core,
+                    deterministic)
+            algo = perf.algo
+            workspace_size = perf.memory
+            math_type = perf.mathType
+
+        if cudnn_version() >= 7000:
+            cudnn.setConvolutionMathType(conv_desc, math_type)
+
+        workspace = _memory.alloc(workspace_size)
+
+        cudnn.convolutionBackwardFilter_v3(
+            handle, one, x_desc, x.data.ptr, gy_desc,
+            gy.data.ptr, conv_desc, algo, workspace.ptr,
+            workspace_size, zero, filter_desc, gW.data.ptr)
+    finally:
+        cudnn.destroyTensorDescriptor(x_desc)
+        cudnn.destroyTensorDescriptor(gy_desc)
+        cudnn.destroyFilterDescriptor(filter_desc)
+        cudnn.destroyConvolutionDescriptor(conv_desc)
+
+
+def convolution_backward_data(
+        _ndarray_base W, _ndarray_base x, _ndarray_base b, _ndarray_base y,
+        tuple pad, tuple stride, tuple dilation, int groups, *,
+        bint deterministic, bint auto_tune, tensor_core,
+        int d_layout=cudnn.CUDNN_TENSOR_NCHW,
+        int w_layout=cudnn.CUDNN_TENSOR_NCHW):
+    cdef int dev_id = W.data.device.id
+    assert dev_id == x.data.device.id
+    assert dev_id == y.data.device.id
+
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    if x.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+
+    # Disable use_tensor_core in deterministic mode because
+    # CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 does not use Tensor Core.
+    cdef bint use_tensor_core = (
+        not deterministic and _should_use_tensor_core(tensor_core, x.dtype))
+    cdef tuple conv_param = (
+        pad, stride, x.dtype, use_tensor_core, deterministic)
+
+    # cuDNN 7 supports dilation only in *_FWD_ALGO_IMPLICIT_GEMM, but
+    # it supports Tensor Cores only in *_FWD_ALGO_IMPLICIT_PRECOMP_GEMM.
+    if use_tensor_core:
+        for i in dilation:
+            if i > 1:
+                use_tensor_core = False
+                break
+
+    handle = get_handle()
+    x = core._internal_ascontiguousarray(x)
+    W = core._internal_ascontiguousarray(W)
+
+    # TODO(okuta) check performance
+    cdef size_t x_desc = cudnn.createTensorDescriptor()
+    cdef size_t y_desc = cudnn.createTensorDescriptor()
+    cdef size_t b_desc = cudnn.createTensorDescriptor()
+    cdef size_t filter_desc = cudnn.createFilterDescriptor()
+    cdef size_t conv_desc = cudnn.createConvolutionDescriptor()
+
+    cdef _Algorithm perf
+    cdef int algo
+    cdef size_t max_workspace_size = get_max_workspace_size()
+    cdef size_t workspace_size = 0
+    cdef shape_t b_shape
+    try:
+        _create_tensor_descriptor(x_desc, x, format=d_layout)
+        _create_tensor_descriptor(y_desc, y, format=d_layout)
+        _create_filter_descriptor(filter_desc, W, w_layout)
+        _create_convolution_descriptor(
+            conv_desc, pad, stride, dilation, groups, x.dtype,
+            cudnn.CUDNN_CROSS_CORRELATION, use_tensor_core)
+
+        if deterministic and cudnn_version() < 7000:
+            # TODO(imanishi): Support Tensor Core in deterministic mode.
+            algo = cudnn.CUDNN_CONVOLUTION_BWD_DATA_ALGO_1
+            workspace_size = cudnn.getConvolutionBackwardDataWorkspaceSize(
+                handle, filter_desc, x_desc, conv_desc, y_desc, algo)
+            math_type = cudnn.CUDNN_DEFAULT_MATH
+            if workspace_size > max_workspace_size:
+                raise RuntimeError(
+                    'No conv bwd data algo available with workspace size less '
+                    'equal {}'.format(max_workspace_size))
+        else:
+            if auto_tune and not deterministic:
+                perf = _find_algorithm_bwd_data(
+                    W, x, y, conv_param, handle, filter_desc, x_desc,
+                    conv_desc, y_desc, max_workspace_size, use_tensor_core,
+                    deterministic)
+            else:
+                perf = _get_algorithm_bwd_data(
+                    W, x, y, conv_param, handle, filter_desc, x_desc,
+                    conv_desc, y_desc, max_workspace_size, use_tensor_core,
+                    deterministic)
+            algo = perf.algo
+            workspace_size = perf.memory
+            math_type = perf.mathType
+
+        if cudnn_version() >= 7000:
+            cudnn.setConvolutionMathType(conv_desc, math_type)
+
+        workspace = _memory.alloc(workspace_size)
+
+        cudnn.convolutionBackwardData_v3(
+            handle, one, filter_desc, W.data.ptr, x_desc, x.data.ptr,
+            conv_desc, algo, workspace.ptr, workspace_size, zero, y_desc,
+            y.data.ptr)
+
+        del workspace, x, W
+
+        if b is not None:
+            assert dev_id == b.data.device.id
+            b_shape.assign(y._shape.size(), 1)
+            b_shape[1] = -1
+            b = _manipulation._reshape(
+                core._internal_ascontiguousarray(b), b_shape)
+            _create_tensor_nd_descriptor(b_desc, b, -1)
+            cudnn.addTensor_v3(handle, one, b_desc, b.data.ptr, one, y_desc,
+                               y.data.ptr)
+    finally:
+        cudnn.destroyTensorDescriptor(x_desc)
+        cudnn.destroyTensorDescriptor(y_desc)
+        cudnn.destroyTensorDescriptor(b_desc)
+        cudnn.destroyFilterDescriptor(filter_desc)
+        cudnn.destroyConvolutionDescriptor(conv_desc)
+
+
+def pooling_forward(
+        _ndarray_base x, _ndarray_base y,
+        tuple ksize, tuple stride, tuple pad, int mode):
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    if x.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+    x = core._internal_ascontiguousarray(x)
+    if not y._c_contiguous:
+        raise ValueError('pooling_forward supports c-contiguous y only')
+    handle = get_handle()
+    x_desc = cudnn.createTensorDescriptor()
+    y_desc = cudnn.createTensorDescriptor()
+    pool_desc = cudnn.createPoolingDescriptor()
+    try:
+        _create_tensor_nd_descriptor(x_desc, x)
+        _create_tensor_nd_descriptor(y_desc, y)
+        _create_pooling_descriptor(pool_desc, ksize, stride, pad, mode)
+        cudnn.poolingForward(
+            handle, pool_desc, one, x_desc,
+            x.data.ptr, zero, y_desc, y.data.ptr)
+    finally:
+        cudnn.destroyTensorDescriptor(x_desc)
+        cudnn.destroyTensorDescriptor(y_desc)
+        cudnn.destroyPoolingDescriptor(pool_desc)
+    return y
+
+
+def pooling_backward(
+        _ndarray_base x, _ndarray_base y, _ndarray_base gy,
+        tuple ksize, tuple stride, tuple pad, int mode):
+    cdef float float_zero = 0, float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero, one
+    cdef _ndarray_base gx
+    if x.dtype == 'd':
+        zero = <size_t>&double_zero
+        one = <size_t>&double_one
+    else:
+        zero = <size_t>&float_zero
+        one = <size_t>&float_one
+
+    gx = _core.ndarray(x._shape, x.dtype)
+    x = core._internal_ascontiguousarray(x)
+    y = core._internal_ascontiguousarray(y)
+    gy = core._internal_ascontiguousarray(gy)
+
+    handle = get_handle()
+    x_desc = cudnn.createTensorDescriptor()
+    y_desc = cudnn.createTensorDescriptor()
+    pool_desc = cudnn.createPoolingDescriptor()
+    try:
+        _create_tensor_nd_descriptor(x_desc, x)
+        _create_tensor_nd_descriptor(y_desc, y)
+        _create_pooling_descriptor(pool_desc, ksize, stride, pad, mode)
+        cudnn.poolingBackward(
+            handle, pool_desc,
+            one, y_desc, y.data.ptr, y_desc, gy.data.ptr,
+            x_desc, x.data.ptr, zero, x_desc, gx.data.ptr)
+    finally:
+        cudnn.destroyTensorDescriptor(x_desc)
+        cudnn.destroyTensorDescriptor(y_desc)
+        cudnn.destroyPoolingDescriptor(pool_desc)
+    return gx
+
+
+cdef _create_tensor_descriptor_for_bn(
+        size_t desc, _ndarray_base arr, bint is_for_conv2d,
+        int format=cudnn.CUDNN_TENSOR_NCHW):
+    assert arr._c_contiguous
+    if is_for_conv2d:
+        _create_tensor_descriptor(desc, arr, format)
+        return
+    data_type = get_data_type(arr.dtype)
+    cdef Py_ssize_t dim1, dim2
+    cdef int ndim = arr._shape.size()
+    dim2 = 1
+    if ndim > 0:
+        dim2 = arr._shape[ndim - 1]
+    dim1 = arr.size // dim2
+    cudnn.setTensor4dDescriptor(desc, cudnn.CUDNN_TENSOR_NCHW, data_type,
+                                dim1, dim2, 1, 1)
+
+
+cdef _get_dtype_of_tensor_descriptor(size_t desc):
+    cudnn_dtype, _, _, _, _, _, _, _, _ = cudnn.getTensor4dDescriptor(desc)
+    if cudnn_dtype == cudnn.CUDNN_DATA_DOUBLE:
+        return _numpy.dtype(_numpy.float64)
+    elif cudnn_dtype == cudnn.CUDNN_DATA_FLOAT:
+        return _numpy.dtype(_numpy.float32)
+    elif cudnn_dtype == cudnn.CUDNN_DATA_HALF:
+        return _numpy.dtype(_numpy.float16)
+    else:
+        raise RuntimeError('Unknown cudnn data type {} '.format(cudnn_dtype))
+
+
+def batch_normalization_forward_training(
+        _ndarray_base x, _ndarray_base gamma, _ndarray_base beta,
+        _ndarray_base running_mean, _ndarray_base running_var,
+        mean, inv_std, double eps, double decay,
+        bint is_for_conv2d, int cudnn_mode, bint debug,
+        int d_layout=cudnn.CUDNN_TENSOR_NCHW):
+
+    reserve_space, y, save_mean, save_inv_std = (
+        _batch_normalization_forward_training(
+            x, gamma, beta,
+            running_mean, running_var,
+            mean, inv_std,
+            eps, decay,
+            is_for_conv2d,
+            cudnn_mode,
+            debug,
+            d_layout))
+    if reserve_space is not None:
+        _warnings.warn(
+            'Could be faster by calling '
+            'batch_normalization_forward_training_ex() instead of '
+            'batch_normalization_forward_training().',
+            _util.PerformanceWarning)
+    if mean is None:
+        return y, save_mean, save_inv_std
+    else:
+        return y
+
+
+def batch_normalization_forward_training_ex(
+        _ndarray_base x, _ndarray_base gamma, _ndarray_base beta,
+        _ndarray_base running_mean, _ndarray_base running_var,
+        mean, inv_std, double eps, double decay,
+        bint is_for_conv2d, int cudnn_mode, bint debug,
+        int d_layout=cudnn.CUDNN_TENSOR_NCHW):
+
+    reserve_space, y, save_mean, save_inv_std = (
+        _batch_normalization_forward_training(
+            x, gamma, beta,
+            running_mean, running_var,
+            mean, inv_std,
+            eps, decay,
+            is_for_conv2d,
+            cudnn_mode,
+            debug,
+            d_layout))
+    if mean is None:
+        return reserve_space, y, save_mean, save_inv_std
+    else:
+        return reserve_space, y
+
+
+cdef _batch_normalization_forward_training(
+        _ndarray_base x, _ndarray_base gamma, _ndarray_base beta,
+        _ndarray_base running_mean, _ndarray_base running_var,
+        mean, inv_std, double eps, double decay,
+        bint is_for_conv2d, int cudnn_mode, bint debug,
+        int d_layout=cudnn.CUDNN_TENSOR_NCHW):
+
+    cdef _memory.MemoryPointer workspace = None
+    cdef _memory.MemoryPointer reserve_space = None
+
+    # Usually supply None to mean and inv_std, which are left for backward
+    # compatibility. See cupy#2060 and cupy#2070.
+    if (mean is None) != (inv_std is None):
+        raise ValueError('Both mean and inv_std must be None if one is.')
+
+    x = core._internal_ascontiguousarray(x)
+    dtype = x.dtype
+    y = _core.ndarray(x._shape, dtype)
+
+    cdef float float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero = <size_t>&double_zero, one
+    if x.dtype == 'd':
+        one = <size_t>&double_one
+    else:
+        one = <size_t>&float_one
+
+    handle = get_handle()
+    cdef size_t x_desc = cudnn.createTensorDescriptor()
+    cdef size_t derivedBnDesc = cudnn.createTensorDescriptor()
+    try:
+        _create_tensor_descriptor_for_bn(x_desc, x, is_for_conv2d,
+                                         format=d_layout)
+        cudnn.deriveBNTensorDescriptor(derivedBnDesc, x_desc, cudnn_mode)
+        dtype_param = _get_dtype_of_tensor_descriptor(derivedBnDesc)
+        if gamma.dtype != dtype_param:
+            gamma = gamma.astype(dtype_param)
+            beta = beta.astype(dtype_param)
+            running_mean_tmp = running_mean.astype(dtype_param)
+            running_var_tmp = running_var.astype(dtype_param)
+        else:
+            running_mean_tmp = running_mean
+            running_var_tmp = running_var
+            gamma = core._internal_ascontiguousarray(gamma)
+            beta = core._internal_ascontiguousarray(beta)
+        if mean is None:
+            save_mean = _core.ndarray(gamma.shape, dtype_param)
+            save_inv_std = _core.ndarray(gamma.shape, dtype_param)
+        else:
+            save_mean = mean
+            save_inv_std = inv_std
+
+        # Factor used in the moving average
+        factor = 1.0 - decay
+
+        # Note: cuDNN computes the mini-batch mean and variance
+        # internally. We can simply (optionally) pass
+        # it the running-average mean and variance arrays.
+        # Note: This API seems to set the inverse of the standard deviation
+        # (instead of variance) to resultSaveInvVariance argument. The
+        # current implementation of our BN depends on this behavior so that
+        # we can reduce the number of reduction kernels.
+
+        if cudnn_version() >= 7401:
+
+            bn_ops = cudnn.CUDNN_BATCHNORM_OPS_BN
+
+            if (
+                    cudnn_mode == cudnn.CUDNN_BATCHNORM_SPATIAL_PERSISTENT
+                    and x.dtype == _numpy.float16
+                    and d_layout == cudnn.CUDNN_TENSOR_NHWC
+                    and x.shape[3] % 4 == 0  # C mod 4 == 0
+            ):
+
+                # Faster NHWC kernel can be triggered by allocating extra
+                # spaces.
+                # https://docs.nvidia.com/deeplearning/sdk/cudnn-archived/cudnn_741/cudnn-developer-guide/index.html#cudnnBatchNormalizationForwardTrainingEx  # NOQA
+                workspace_size = (
+                    cudnn.getBatchNormalizationForwardTrainingExWorkspaceSize(
+                        handle,
+                        cudnn_mode,
+                        bn_ops,
+                        x_desc,  # x
+                        x_desc,  # z
+                        x_desc,  # y
+                        derivedBnDesc,
+                        0,  # activation desc
+                    ))
+                workspace = _memory.alloc(workspace_size)
+
+                reserve_space_size = (
+                    cudnn.getBatchNormalizationTrainingExReserveSpaceSize(
+                        handle,
+                        cudnn_mode,
+                        bn_ops,
+                        0,  # activation desc
+                        x_desc,
+                    ))
+                reserve_space = _memory.alloc(reserve_space_size)
+
+            cudnn.batchNormalizationForwardTrainingEx(
+                handle,
+                cudnn_mode,
+                bn_ops,
+                one,  # alpha
+                zero,  # beta
+                x_desc, x.data.ptr,  # x
+                x_desc, 0,  # z
+                x_desc, y.data.ptr,  # y
+                derivedBnDesc,
+                gamma.data.ptr,
+                beta.data.ptr,
+                factor,
+                running_mean_tmp.data.ptr,
+                running_var_tmp.data.ptr,
+                eps,
+                save_mean.data.ptr,
+                save_inv_std.data.ptr,
+                0,  # activation
+                0 if workspace is None else workspace.ptr,
+                0 if workspace is None else workspace.mem.size,
+                0 if reserve_space is None else reserve_space.ptr,
+                0 if reserve_space is None else reserve_space.mem.size,
+            )
+
+        else:  # cuDNN < 7401
+            cudnn.batchNormalizationForwardTraining(
+                handle, cudnn_mode, one, zero,
+                x_desc, x.data.ptr, x_desc, y.data.ptr,
+                derivedBnDesc, gamma.data.ptr,
+                beta.data.ptr, factor, running_mean_tmp.data.ptr,
+                running_var_tmp.data.ptr, eps,
+                save_mean.data.ptr, save_inv_std.data.ptr)
+
+        # Note: When the CUDNN_BATCHNORM_SPATIAL_PERSISTENT mode is used,
+        # there is a possibility of numerical overflow. You can use
+        # queryRuntimeError() to make sure whether the overflow actually
+        # occurred or not during the batch normalization.
+        if debug and cudnn_mode == cudnn.CUDNN_BATCHNORM_SPATIAL_PERSISTENT:
+            query_mode = cudnn.CUDNN_ERRQUERY_BLOCKING
+            rstatus = cudnn.queryRuntimeError(handle, query_mode)
+            if rstatus != cudnn.CUDNN_STATUS_SUCCESS:
+                _warnings.warn(
+                    'A numerical overflow might have happened in cuDNN'
+                    'batch normalization (status:{})'.format(rstatus))
+    finally:
+        cudnn.destroyTensorDescriptor(x_desc)
+        cudnn.destroyTensorDescriptor(derivedBnDesc)
+    if running_mean is not running_mean_tmp:
+        _core.elementwise_copy(running_mean_tmp, running_mean)
+        _core.elementwise_copy(running_var_tmp, running_var)
+    return reserve_space, y, save_mean, save_inv_std
+
+
+def batch_normalization_forward_inference(
+        _ndarray_base x, _ndarray_base gamma, _ndarray_base beta,
+        _ndarray_base mean, _ndarray_base var,
+        double eps, bint is_for_conv2d, int cudnn_mode,
+        int d_layout=cudnn.CUDNN_TENSOR_NCHW):
+    x = core._internal_ascontiguousarray(x)
+    dtype = x.dtype
+    y = _core.ndarray(x._shape, dtype)
+
+    cdef float float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero = <size_t>&double_zero, one
+    if x.dtype == 'd':
+        one = <size_t>&double_one
+    else:
+        one = <size_t>&float_one
+
+    handle = get_handle()
+    cdef size_t x_desc = cudnn.createTensorDescriptor()
+    cdef size_t derivedBnDesc = cudnn.createTensorDescriptor()
+    try:
+        _create_tensor_descriptor_for_bn(x_desc, x, is_for_conv2d,
+                                         format=d_layout)
+        cudnn.deriveBNTensorDescriptor(derivedBnDesc, x_desc, cudnn_mode)
+        dtype_param = _get_dtype_of_tensor_descriptor(derivedBnDesc)
+        if gamma.dtype != dtype_param:
+            gamma = gamma.astype(dtype_param)
+            beta = beta.astype(dtype_param)
+            mean = mean.astype(dtype_param)
+            var = var.astype(dtype_param)
+        else:
+            gamma = core._internal_ascontiguousarray(gamma)
+            beta = core._internal_ascontiguousarray(beta)
+
+        cudnn.batchNormalizationForwardInference(
+            handle, cudnn_mode, one, zero,
+            x_desc, x.data.ptr, x_desc, y.data.ptr,
+            derivedBnDesc, gamma.data.ptr, beta.data.ptr,
+            mean.data.ptr, var.data.ptr, eps)
+    finally:
+        cudnn.destroyTensorDescriptor(x_desc)
+        cudnn.destroyTensorDescriptor(derivedBnDesc)
+    return y
+
+
+def batch_normalization_backward(
+        _ndarray_base x, _ndarray_base gamma, _ndarray_base gy,
+        _ndarray_base mean, _ndarray_base inv_std,
+        double eps, bint is_for_conv2d, int cudnn_mode, bint debug,
+        int d_layout=cudnn.CUDNN_TENSOR_NCHW,
+        *,
+        _memory.MemoryPointer reserve_space=None,
+):
+    cdef _ndarray_base ggamma, gbeta
+    cdef bint need_cast
+    cdef _memory.MemoryPointer workspace = None
+
+    x = core._internal_ascontiguousarray(x)
+    gy = core._internal_ascontiguousarray(gy)
+    dtype = x.dtype
+    gx = _core.ndarray(x._shape, dtype)
+
+    cdef float float_one = 1
+    cdef double double_zero = 0, double_one = 1
+    cdef size_t zero = <size_t>&double_zero, one
+    if x.dtype == 'd':
+        one = <size_t>&double_one
+    else:
+        one = <size_t>&float_one
+
+    handle = get_handle()
+    cdef size_t x_desc = cudnn.createTensorDescriptor()
+    cdef size_t derivedBnDesc = cudnn.createTensorDescriptor()
+    try:
+        _create_tensor_descriptor_for_bn(x_desc, x, is_for_conv2d,
+                                         format=d_layout)
+        cudnn.deriveBNTensorDescriptor(derivedBnDesc, x_desc, cudnn_mode)
+        dtype_param = _get_dtype_of_tensor_descriptor(derivedBnDesc)
+        need_cast = gamma.dtype != dtype_param
+        if need_cast:
+            gamma = gamma.astype(dtype_param)
+        else:
+            gamma = core._internal_ascontiguousarray(gamma)
+        ggamma = _core.ndarray(gamma._shape, dtype_param)
+        gbeta = _core.ndarray(gamma._shape, dtype_param)
+
+        if cudnn_version() >= 7401:
+            bn_ops = cudnn.CUDNN_BATCHNORM_OPS_BN
+
+            workspace_size = (
+                cudnn.getBatchNormalizationBackwardExWorkspaceSize(
+                    handle,
+                    cudnn_mode,
+                    bn_ops,
+                    x_desc,
+                    x_desc,  # y
+                    x_desc,  # dy
+                    x_desc,  # dz
+                    x_desc,  # dx
+                    derivedBnDesc,
+                    0,  # activation desc
+                ))
+            workspace = _memory.alloc(workspace_size)
+
+            cudnn.batchNormalizationBackwardEx(
+                handle,
+                cudnn_mode,
+                bn_ops,
+                one, zero, one, zero,
+                x_desc, x.data.ptr,
+                x_desc, 0,  # y
+                x_desc, gy.data.ptr,
+                x_desc, 0,  # dz
+                x_desc, gx.data.ptr,
+                derivedBnDesc,
+                gamma.data.ptr,
+                0,  # beta
+                ggamma.data.ptr,
+                gbeta.data.ptr,
+                eps,
+                mean.data.ptr,
+                inv_std.data.ptr,
+                0,  # activation desc
+                workspace,
+                workspace_size,
+                0 if reserve_space is None else reserve_space.ptr,
+                0 if reserve_space is None else reserve_space.mem.size,
+            )
+
+        else:
+            # cuDNN < 7401
+            if reserve_space is not None:
+                raise ValueError(
+                    'reserve_space can only be passed in cuDNN >= 7401')
+            cudnn.batchNormalizationBackward(
+                handle, cudnn_mode, one, zero, one, zero,
+                x_desc, x.data.ptr,
+                x_desc, gy.data.ptr, x_desc, gx.data.ptr,
+                derivedBnDesc, gamma.data.ptr, ggamma.data.ptr, gbeta.data.ptr,
+                eps, mean.data.ptr, inv_std.data.ptr)
+
+        # Note: When the CUDNN_BATCHNORM_SPATIAL_PERSISTENT mode is used,
+        # there is a possibility of numerical overflow. You can use
+        # queryRuntimeError() to make sure whether the overflow actually
+        # occurred or not during the batch normalization.
+        if debug and cudnn_mode == cudnn.CUDNN_BATCHNORM_SPATIAL_PERSISTENT:
+            query_mode = cudnn.CUDNN_ERRQUERY_BLOCKING
+            rstatus = cudnn.queryRuntimeError(handle, query_mode)
+            if rstatus != cudnn.CUDNN_STATUS_SUCCESS:
+                _warnings.warn(
+                    'A numerical overflow might have happened in cuDNN'
+                    'batch normalization (status:{})'.format(rstatus))
+    finally:
+        cudnn.destroyTensorDescriptor(x_desc)
+        cudnn.destroyTensorDescriptor(derivedBnDesc)
+
+    if need_cast:
+        ggamma = ggamma.astype(dtype)
+        gbeta = gbeta.astype(dtype)
+    return gx, ggamma, gbeta
+
+
+def create_activation_descriptor(mode, relu_nan_opt=cudnn.CUDNN_PROPAGATE_NAN,
+                                 coef=0.0):
+    desc = Descriptor(cudnn.createActivationDescriptor(),
+                      _py_cudnn.destroyActivationDescriptor)
+    cudnn.setActivationDescriptor(desc.value, mode, relu_nan_opt, coef)
+    return desc
+
+
+def create_fused_ops_plan(ops):
+    plan = Descriptor(cudnn.createFusedOpsPlan(ops),
+                      _py_cudnn.destroyFusedOpsPlan)
+    return plan
+
+
+def create_fused_ops_const_param_pack(ops, list_attr_param):
+    const_pack = Descriptor(cudnn.createFusedOpsConstParamPack(ops),
+                            _py_cudnn.destroyFusedOpsConstParamPack)
+    for attr, param in list_attr_param:
+        set_fused_ops_const_param_pack_attribute(const_pack, attr, param)
+    return const_pack
+
+
+def make_fused_ops_plan(plan, const_pack):
+    handle = get_handle()
+    return cudnn.makeFusedOpsPlan(handle, plan.value, const_pack.value)
+
+
+def create_fused_ops_variant_param_pack(ops, list_attr_param):
+    var_pack = Descriptor(cudnn.createFusedOpsVariantParamPack(ops),
+                          _py_cudnn.destroyFusedOpsVariantParamPack)
+    for attr, param in list_attr_param:
+        set_fused_ops_variant_param_pack_attribute(var_pack, attr, param)
+    return var_pack
+
+
+def fused_ops_execute(plan, var_pack):
+    handle = get_handle()
+    cudnn.fusedOpsExecute(handle, plan.value, var_pack.value)
+
+
+cpdef set_fused_ops_const_param_pack_attribute(
+        Descriptor const_pack, int param_label, desc_or_scalar):
+    cdef int scalar
+    cdef Descriptor desc
+    if param_label in (cudnn.CUDNN_PARAM_XDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_MODE,
+                       cudnn.CUDNN_PARAM_BN_EQSCALE_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_EQBIAS_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_WDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_DWDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_YDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_DYDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_YSUM_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_YSQSUM_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_SCALE_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_BIAS_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_SAVED_MEAN_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_SAVED_INVSTD_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_RUNNING_MEAN_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_RUNNING_VAR_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_ZDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_Z_EQSCALE_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_Z_EQBIAS_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_ACTIVATION_BITMASK_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_DXDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_DZDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_DSCALE_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_DBIAS_PLACEHOLDER):
+        scalar = <int>desc_or_scalar
+        cudnn.setFusedOpsConstParamPackAttribute(const_pack.value, param_label,
+                                                 <size_t>&scalar)
+    else:
+        desc = <Descriptor>desc_or_scalar
+        cudnn.setFusedOpsConstParamPackAttribute(const_pack.value, param_label,
+                                                 <size_t>desc.value)
+
+
+cpdef get_fused_ops_const_param_pack_attribute(Descriptor const_pack,
+                                               int param_label):
+    cdef int param_int
+    cdef size_t param_desc
+    if param_label in (cudnn.CUDNN_PARAM_XDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_MODE,
+                       cudnn.CUDNN_PARAM_BN_EQSCALE_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_EQBIAS_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_WDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_DWDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_YDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_DYDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_YSUM_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_YSQSUM_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_SCALE_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_BIAS_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_SAVED_MEAN_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_SAVED_INVSTD_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_RUNNING_MEAN_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_RUNNING_VAR_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_ZDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_Z_EQSCALE_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_Z_EQBIAS_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_ACTIVATION_BITMASK_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_DXDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_DZDATA_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_DSCALE_PLACEHOLDER,
+                       cudnn.CUDNN_PARAM_BN_DBIAS_PLACEHOLDER):
+        is_null = cudnn.getFusedOpsConstParamPackAttribute(
+            const_pack.value, param_label, <size_t>&param_int)
+        return <size_t>param_int, is_null
+    else:
+        if param_label == cudnn.CUDNN_PARAM_ACTIVATION_DESC:
+            param_desc = cudnn.createActivationDescriptor()
+        elif param_label == cudnn.CUDNN_PARAM_CONV_DESC:
+            param_desc = cudnn.createConvolutionDescriptor()
+        elif param_label in (cudnn.CUDNN_PARAM_WDESC,
+                             cudnn.CUDNN_PARAM_DWDESC,):
+            param_desc = cudnn.createFilterDescriptor()
+        else:
+            param_desc = cudnn.createTensorDescriptor()
+        is_null = cudnn.getFusedOpsConstParamPackAttribute(
+            const_pack.value, param_label, param_desc)
+        return param_desc, is_null
+
+
+cpdef set_fused_ops_variant_param_pack_attribute(
+        Descriptor var_pack, int param_label, arr_or_scaler):
+    cdef size_t scalar_size_t
+    cdef int64_t scalar_int64_t
+    cdef double scalar_double
+    cdef size_t ptr
+    if param_label == cudnn.CUDNN_SCALAR_SIZE_T_WORKSPACE_SIZE_IN_BYTES:
+        scalar_size_t = <size_t>arr_or_scaler
+        cudnn.setFusedOpsVariantParamPackAttribute(var_pack.value, param_label,
+                                                   <size_t>&scalar_size_t)
+    elif param_label == cudnn.CUDNN_SCALAR_INT64_T_BN_ACCUMULATION_COUNT:
+        scalar_int64_t = <int64_t>arr_or_scaler
+        cudnn.setFusedOpsVariantParamPackAttribute(var_pack.value, param_label,
+                                                   <size_t>&scalar_int64_t)
+    elif param_label in (cudnn.CUDNN_SCALAR_DOUBLE_BN_EPSILON,
+                         cudnn.CUDNN_SCALAR_DOUBLE_BN_EXP_AVG_FACTOR):
+        scalar_double = <double>arr_or_scaler
+        cudnn.setFusedOpsVariantParamPackAttribute(var_pack.value, param_label,
+                                                   <size_t>&scalar_double)
+    else:
+        ptr = <size_t>arr_or_scaler.data.ptr
+        cudnn.setFusedOpsVariantParamPackAttribute(var_pack.value, param_label,
+                                                   ptr)
+
+
+cpdef get_fused_ops_variant_param_pack_attribute(size_t var_pack,
+                                                 int param_label):
+    cdef size_t scalar_size_t
+    cdef int64_t scalar_int64_t
+    cdef double scalar_double
+    cdef size_t ptr
+    if param_label == cudnn.CUDNN_SCALAR_SIZE_T_WORKSPACE_SIZE_IN_BYTES:
+        cudnn.getFusedOpsVariantParamPackAttribute(var_pack, param_label,
+                                                   <size_t>&scalar_size_t)
+        return <size_t>scalar_size_t
+    elif param_label == cudnn.CUDNN_SCALAR_INT64_T_BN_ACCUMULATION_COUNT:
+        cudnn.getFusedOpsVariantParamPackAttribute(var_pack, param_label,
+                                                   <size_t>&scalar_int64_t)
+        return <size_t>scalar_int64_t
+    elif param_label in (cudnn.CUDNN_SCALAR_DOUBLE_BN_EPSILON,
+                         cudnn.CUDNN_SCALAR_DOUBLE_BN_EXP_AVG_FACTOR):
+        cudnn.getFusedOpsVariantParamPackAttribute(var_pack, param_label,
+                                                   <size_t>&scalar_double)
+        return <size_t>scalar_double
+    else:
+        cudnn.getFusedOpsVariantParamPackAttribute(var_pack, param_label,
+                                                   <size_t>&ptr)
+        return <size_t>ptr
diff --git a/cupyx/cusolver.pyx b/cupyx/cusolver.pyx
new file mode 100644
index 0000000..9ff834d
--- /dev/null
+++ b/cupyx/cusolver.pyx
@@ -0,0 +1,984 @@
+# distutils: language = c++
+
+from libc.stdint cimport intptr_t
+
+import warnings as _warnings
+
+import numpy as _numpy
+
+from cupy_backends.cuda.api cimport runtime
+from cupy_backends.cuda.libs cimport cusolver
+# due to a Cython bug (cython/cython#4000) we cannot just cimport the module
+from cupy_backends.cuda.libs.cusolver cimport (  # noqa
+    sgesvd_bufferSize, dgesvd_bufferSize, cgesvd_bufferSize, zgesvd_bufferSize,
+    sgeqrf_bufferSize, dgeqrf_bufferSize, cgeqrf_bufferSize, zgeqrf_bufferSize,
+    sorgqr_bufferSize, dorgqr_bufferSize, cungqr_bufferSize, zungqr_bufferSize)
+
+from cupy.cuda cimport memory
+from cupy._core.core cimport _internal_ascontiguousarray
+from cupy._core.core cimport _ndarray_init, _ndarray_base
+
+import cupy as _cupy
+from cupy_backends.cuda.api import runtime as _runtime
+from cupy_backends.cuda.libs import cublas as _cublas
+from cupy_backends.cuda.libs import cusolver as _cusolver
+from cupy.cuda import device as _device
+from cupy._core import _routines_linalg as _linalg
+from cupy import _util
+
+import cupyx as _cupyx
+
+
+###############################################################################
+# Extern
+###############################################################################
+
+cdef extern from '../cupy_backends/cupy_complex.h':
+    ctypedef struct cuComplex 'cuComplex':
+        float x, y
+
+    ctypedef struct cuDoubleComplex 'cuDoubleComplex':
+        double x, y
+
+cdef extern from '../cupy_backends/cupy_lapack.h' nogil:
+    int gesvd_loop[T](
+        intptr_t handle, char jobu, char jobvt, int m, int n, intptr_t A,
+        intptr_t s_ptr, intptr_t u_ptr, intptr_t vt_ptr,
+        intptr_t w_ptr, int buffersize, intptr_t info_ptr,
+        int batch_size)
+    int geqrf_loop[T](
+        intptr_t handle, int m, int n, intptr_t a_ptr, int lda,
+        intptr_t tau_ptr, intptr_t w_ptr,
+        int buffersize, intptr_t info_ptr,
+        int batch_size)
+    int orgqr_loop[T](
+        intptr_t handle, int m, int n, int k, intptr_t a_ptr, int lda,
+        intptr_t tau_ptr, intptr_t w_ptr,
+        int buffersize, intptr_t info_ptr,
+        int batch_size, int origin_n)
+
+ctypedef int(*gesvd_ptr)(intptr_t, char, char, int, int, intptr_t,
+                         intptr_t, intptr_t, intptr_t,
+                         intptr_t, int, intptr_t, int) nogil
+ctypedef int(*geqrf_ptr)(intptr_t, int, int, intptr_t, int, intptr_t,
+                         intptr_t, int, intptr_t, int) nogil
+ctypedef int(*orgqr_ptr)(intptr_t, int, int, int, intptr_t, int, intptr_t,
+                         intptr_t, int, intptr_t, int, int) nogil
+
+
+_available_cuda_version = {
+    'gesvdj': (9000, None),
+    'gesvdjBatched': (9000, None),
+    'gesvda': (10010, None),
+    'potrfBatched': (9010, None),
+    'potrsBatched': (9010, None),
+    'syevj': (9000, None),
+    'gesv': (10020, None),
+    'gels': (11000, None),
+    'csrlsvqr': (9000, None),
+}
+
+_available_hip_version = {
+    # For APIs supported by CUDA but not yet by HIP, we still need them here
+    # so that our test suite can cover both platforms.
+    'gesvdj': (_numpy.inf, None),
+    'gesvdjBatched': (309, None),  # = rocsolver_<t>gesvd_batched
+    'gesvda': (_numpy.inf, None),
+    'potrfBatched': (306, None),
+    'potrsBatched': (_numpy.inf, None),
+    'syevj': (402, None),
+    'gesv': (_numpy.inf, None),
+    'gels': (_numpy.inf, None),
+    'csrlsvqr': (_numpy.inf, None),
+}
+
+_available_compute_capability = {
+    'gesv': 70,
+    'gels': 70,
+}
+
+
+@_util.memoize(for_each_device=True)
+def check_availability(name):
+    if not _runtime.is_hip:
+        available_version = _available_cuda_version
+    else:
+        available_version = _available_hip_version
+    version = cusolver._get_cuda_build_version()
+    if name not in available_version:
+        msg = 'No available version information specified for {}'.format(name)
+        raise ValueError(msg)
+    version_added, version_removed = available_version[name]
+    if version_added is not None and version < version_added:
+        return False
+    if version_removed is not None and version >= version_removed:
+        return False
+    # CUDA specific stuff
+    if not _runtime.is_hip and name in _available_compute_capability:
+        compute_capability = int(_device.get_compute_capability())
+        if compute_capability < _available_compute_capability[name]:
+            return False
+    return True
+
+
+def gesvdj(a, full_matrices=True, compute_uv=True, overwrite_a=False):
+    """Singular value decomposition using cusolverDn<t>gesvdj().
+
+    Factorizes the matrix ``a`` into two unitary matrices ``u`` and ``v`` and
+    a singular values vector ``s`` such that ``a == u @ diag(s) @ v*``.
+
+    Args:
+        a (cupy.ndarray): The input matrix with dimension ``(M, N)``.
+        full_matrices (bool): If True, it returns u and v with dimensions
+            ``(M, M)`` and ``(N, N)``. Otherwise, the dimensions of u and v
+            are respectively ``(M, K)`` and ``(K, N)``, where
+            ``K = min(M, N)``.
+        compute_uv (bool): If ``False``, it only returns singular values.
+        overwrite_a (bool): If ``True``, matrix ``a`` might be overwritten.
+
+    Returns:
+        tuple of :class:`cupy.ndarray`:
+            A tuple of ``(u, s, v)``.
+    """
+    if a.ndim == 3:
+        return _gesvdj_batched(a, full_matrices, compute_uv, overwrite_a)
+
+    if not check_availability('gesvdj'):
+        raise RuntimeError('gesvdj is not available.')
+
+    assert a.ndim == 2
+
+    if a.dtype == 'f':
+        helper = _cusolver.sgesvdj_bufferSize
+        solver = _cusolver.sgesvdj
+        s_dtype = 'f'
+    elif a.dtype == 'd':
+        helper = _cusolver.dgesvdj_bufferSize
+        solver = _cusolver.dgesvdj
+        s_dtype = 'd'
+    elif a.dtype == 'F':
+        helper = _cusolver.cgesvdj_bufferSize
+        solver = _cusolver.cgesvdj
+        s_dtype = 'f'
+    elif a.dtype == 'D':
+        helper = _cusolver.zgesvdj_bufferSize
+        solver = _cusolver.zgesvdj
+        s_dtype = 'd'
+    else:
+        raise TypeError
+
+    handle = _device.get_cusolver_handle()
+    m, n = a.shape
+    a = _cupy.array(a, order='F', copy=not overwrite_a)
+    lda = m
+    mn = min(m, n)
+    s = _cupy.empty(mn, dtype=s_dtype)
+    ldu = m
+    ldv = n
+    if compute_uv:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_VECTOR
+    else:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_NOVECTOR
+        full_matrices = False
+    if full_matrices:
+        econ = 0
+        u = _cupy.empty((ldu, m), dtype=a.dtype, order='F')
+        v = _cupy.empty((ldv, n), dtype=a.dtype, order='F')
+    else:
+        econ = 1
+        u = _cupy.empty((ldu, mn), dtype=a.dtype, order='F')
+        v = _cupy.empty((ldv, mn), dtype=a.dtype, order='F')
+    params = _cusolver.createGesvdjInfo()
+    lwork = helper(handle, jobz, econ, m, n, a.data.ptr, lda, s.data.ptr,
+                   u.data.ptr, ldu, v.data.ptr, ldv, params)
+    work = _cupy.empty(lwork, dtype=a.dtype)
+    info = _cupy.empty(1, dtype=_numpy.int32)
+    solver(handle, jobz, econ, m, n, a.data.ptr, lda, s.data.ptr,
+           u.data.ptr, ldu, v.data.ptr, ldv, work.data.ptr, lwork,
+           info.data.ptr, params)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        gesvdj, info)
+
+    _cusolver.destroyGesvdjInfo(params)
+    if compute_uv:
+        return u, s, v
+    else:
+        return s
+
+
+cpdef _gesvdj_batched(a, full_matrices, compute_uv, overwrite_a):
+    if not check_availability('gesvdjBatched'):
+        raise RuntimeError('gesvdj is not available.')
+
+    if a.dtype == 'f':
+        helper = _cusolver.sgesvdjBatched_bufferSize
+        solver = _cusolver.sgesvdjBatched
+        s_dtype = 'f'
+    elif a.dtype == 'd':
+        helper = _cusolver.dgesvdjBatched_bufferSize
+        solver = _cusolver.dgesvdjBatched
+        s_dtype = 'd'
+    elif a.dtype == 'F':
+        helper = _cusolver.cgesvdjBatched_bufferSize
+        solver = _cusolver.cgesvdjBatched
+        s_dtype = 'f'
+    elif a.dtype == 'D':
+        helper = _cusolver.zgesvdjBatched_bufferSize
+        solver = _cusolver.zgesvdjBatched
+        s_dtype = 'd'
+    else:
+        raise TypeError
+
+    handle = _device.get_cusolver_handle()
+    batch_size, m, n = a.shape
+    a = _cupy.array(a.swapaxes(-2, -1), order='C', copy=not overwrite_a)
+    if runtime._is_hip_environment:
+        # rocsolver_<t>gesvd_batched has a different signature...
+        ap = _linalg._mat_ptrs(a)
+    else:
+        ap = a
+    lda = m
+    mn = min(m, n)
+    s = _cupy.empty((batch_size, mn), dtype=s_dtype)
+    ldu = m
+    ldv = n
+    if compute_uv:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_VECTOR
+    else:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_NOVECTOR
+        # if not batched, `full_matrices = False` could speedup.
+
+    u = _cupy.empty((batch_size, m, ldu), dtype=a.dtype).swapaxes(-2, -1)
+    v = _cupy.empty((batch_size, n, ldv), dtype=a.dtype).swapaxes(-2, -1)
+    params = _cusolver.createGesvdjInfo()
+    lwork = helper(handle, jobz, m, n, ap.data.ptr, lda, s.data.ptr,
+                   u.data.ptr, ldu, v.data.ptr, ldv, params, batch_size)
+    work = _cupy.empty(lwork, dtype=a.dtype)
+    info = _cupy.empty(batch_size, dtype=_numpy.int32)
+    solver(handle, jobz, m, n, ap.data.ptr, lda, s.data.ptr,
+           u.data.ptr, ldu, v.data.ptr, ldv, work.data.ptr, lwork,
+           info.data.ptr, params, batch_size)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        gesvdj, info)
+
+    _cusolver.destroyGesvdjInfo(params)
+    if runtime._is_hip_environment:
+        v = v.swapaxes(-1, -2).conj()
+    if not full_matrices:
+        u = u[..., :mn]
+        v = v[..., :mn]
+    if compute_uv:
+        return u, s, v
+    else:
+        return s
+
+
+cpdef _gesvd_batched(a, a_dtype, full_matrices, compute_uv, overwrite_a):
+    """A loop-based gesvd wrapper to support batched SVD."""
+    # This function follows more closely with gesvd() in
+    # cupy/linalg/_decomposition.py.
+
+    # TODO(leofang): if cuSOLVER implements a batched version of gesvd, wrap
+    # it here and unify with rocSOLVER's counterpart (it's currently wrapped
+    # as gesvdj, not gesvd) here.
+    if runtime._is_hip_environment:
+        # we never arrive here, so just raise
+        raise RuntimeError("This function is disabled on HIP as "
+                           "it is not needed")
+
+    # TODO(leofang): try overlapping using a small stream pool?
+
+    cdef _ndarray_base x, s, u, vt, dev_info
+    cdef int n, m, k, batch_size, buffersize, status
+    cdef intptr_t a_ptr, s_ptr, u_ptr, vt_ptr, w_ptr, info_ptr
+    cdef str s_dtype
+    cdef char job_u, job_vt
+    cdef bint trans_flag
+    cdef gesvd_ptr gesvd
+
+    assert a.ndim > 2
+    assert not overwrite_a  # TODO(leofang): handle this?
+    batch_size, n, m = a.shape
+    s_dtype = a_dtype.lower()
+
+    # `a` must be copied because xgesvd destroys the matrix
+    if m >= n:
+        x = a.astype(a_dtype, order='C', copy=True)
+        trans_flag = False
+    else:
+        m, n = a.shape[-2:]
+        x = a.swapaxes(-2, -1).astype(a_dtype, order='C', copy=True)
+        trans_flag = True
+    a_ptr = x.data.ptr
+
+    k = n  # = min(m, n) where m >= n is ensured above
+    if compute_uv:
+        if full_matrices:
+            u = _ndarray_init(_cupy.ndarray, (batch_size, m, m), a_dtype, None)
+            vt = x[..., :n]
+            job_u = b'A'
+            job_vt = b'O'
+            u_ptr, vt_ptr = u.data.ptr, 0
+        else:
+            u = x
+            vt = _ndarray_init(
+                _cupy.ndarray, (batch_size, k, n), a_dtype, None)
+            job_u = b'O'
+            job_vt = b'S'
+            u_ptr, vt_ptr = 0, vt.data.ptr
+    else:
+        u_ptr, vt_ptr = 0, 0  # Use nullptr
+        job_u = b'N'
+        job_vt = b'N'
+    s = _ndarray_init(_cupy.ndarray, (batch_size, k), s_dtype, None)
+    s_ptr = s.data.ptr
+    cdef intptr_t handle = _device.get_cusolver_handle()
+    dev_info = _ndarray_init(_cupy.ndarray, (batch_size,), _numpy.int32, None)
+    info_ptr = dev_info.data.ptr
+
+    if a_dtype == 'f':
+        gesvd_bufferSize = sgesvd_bufferSize
+        gesvd = gesvd_loop[float]
+    elif a_dtype == 'd':
+        gesvd_bufferSize = dgesvd_bufferSize
+        gesvd = gesvd_loop[double]
+    elif a_dtype == 'F':
+        gesvd_bufferSize = cgesvd_bufferSize
+        gesvd = gesvd_loop[cuComplex]
+    elif a_dtype == 'D':
+        gesvd_bufferSize = zgesvd_bufferSize
+        gesvd = gesvd_loop[cuDoubleComplex]
+    else:
+        raise TypeError
+
+    # this wrapper also sets the stream for us
+    buffersize = gesvd_bufferSize(handle, m, n)
+    # we are on the same stream, so the workspace can be reused in the loop
+    workspace = memory.alloc(buffersize * x.dtype.itemsize)
+    w_ptr = workspace.ptr
+
+    # the loop starts here, with gil released to reduce overhead
+    with nogil:
+        status = gesvd(
+            handle, job_u, job_vt, m, n, a_ptr,
+            s_ptr, u_ptr, vt_ptr,
+            w_ptr, buffersize, info_ptr, batch_size)
+    if status != 0:
+        raise _cusolver.CUSOLVERError(status)
+
+    # check the full info array
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        'gesvd', dev_info)
+
+    # Note that the returned array may need to be transposed
+    # depending on the structure of an input
+    if compute_uv:
+        if trans_flag:
+            return u.swapaxes(-2, -1), s, vt.conj()
+        else:
+            return vt, s, u.swapaxes(-2, -1).conj()
+    else:
+        return s
+
+
+def gesvda(a, compute_uv=True):
+    """Singular value decomposition using cusolverDn<t>gesvdaStridedBatched().
+
+    Factorizes the matrix ``a`` into two unitary matrices ``u`` and ``v`` and
+    a singular values vector ``s`` such that ``a == u @ diag(s) @ v*``.
+
+    Args:
+        a (cupy.ndarray): The input matrix with dimension ``(.., M, N)``.
+        compute_uv (bool): If ``False``, it only returns singular values.
+
+    Returns:
+        tuple of :class:`cupy.ndarray`:
+            A tuple of ``(u, s, v)``.
+    """
+    if not check_availability('gesvda'):
+        raise RuntimeError('gesvda is not available.')
+
+    assert a.ndim >= 2
+    a_ndim = a.ndim
+    a_shape = a.shape
+    m, n = a_shape[-2:]
+    assert m >= n
+
+    if a.dtype == 'f':
+        helper = _cusolver.sgesvdaStridedBatched_bufferSize
+        solver = _cusolver.sgesvdaStridedBatched
+        s_dtype = 'f'
+    elif a.dtype == 'd':
+        helper = _cusolver.dgesvdaStridedBatched_bufferSize
+        solver = _cusolver.dgesvdaStridedBatched
+        s_dtype = 'd'
+    elif a.dtype == 'F':
+        helper = _cusolver.cgesvdaStridedBatched_bufferSize
+        solver = _cusolver.cgesvdaStridedBatched
+        s_dtype = 'f'
+    elif a.dtype == 'D':
+        helper = _cusolver.zgesvdaStridedBatched_bufferSize
+        solver = _cusolver.zgesvdaStridedBatched
+        s_dtype = 'd'
+    else:
+        raise TypeError
+
+    handle = _device.get_cusolver_handle()
+    if compute_uv:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_VECTOR
+    else:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_NOVECTOR
+    rank = min(m, n)
+    if a_ndim == 2:
+        batch_size = 1
+    else:
+        batch_size = _numpy.array(a_shape[:-2]).prod().item()
+    a = a.reshape((batch_size, m, n))
+    a = _cupy.ascontiguousarray(a.transpose(0, 2, 1))
+    lda = m
+    stride_a = lda * n
+    s = _cupy.empty((batch_size, rank), dtype=s_dtype)
+    stride_s = rank
+    ldu = m
+    ldv = n
+    u = _cupy.empty((batch_size, rank, ldu), dtype=a.dtype, order='C')
+    v = _cupy.empty((batch_size, rank, ldv), dtype=a.dtype, order='C')
+    stride_u = rank * ldu
+    stride_v = rank * ldv
+    lwork = helper(handle, jobz, rank, m, n, a.data.ptr, lda, stride_a,
+                   s.data.ptr, stride_s, u.data.ptr, ldu, stride_u,
+                   v.data.ptr, ldv, stride_v, batch_size)
+    work = _cupy.empty((lwork,), dtype=a.dtype)
+    info = _cupy.empty((batch_size,), dtype=_numpy.int32)
+    r_norm = _numpy.empty((batch_size,), dtype=_numpy.float64)
+    solver(handle, jobz, rank, m, n, a.data.ptr, lda, stride_a, s.data.ptr,
+           stride_s, u.data.ptr, ldu, stride_u, v.data.ptr, ldv, stride_v,
+           work.data.ptr, lwork, info.data.ptr, r_norm.ctypes.data, batch_size)
+
+    s = s.reshape(a_shape[:-2] + (s.shape[-1],))
+    if not compute_uv:
+        return s
+
+    u = u.transpose(0, 2, 1)
+    v = v.transpose(0, 2, 1)
+    u = u.reshape(a_shape[:-2] + (u.shape[-2:]))
+    v = v.reshape(a_shape[:-2] + (v.shape[-2:]))
+    return u, s, v
+
+
+def syevj(a, UPLO='L', with_eigen_vector=True):
+    """Eigenvalue decomposition of symmetric matrix using cusolverDn<t>syevj().
+
+    Computes eigenvalues ``w`` and (optionally) eigenvectors ``v`` of a complex
+    Hermitian or a real symmetric matrix.
+
+    Args:
+        a (cupy.ndarray): A symmetric 2-D square matrix ``(M, M)`` or a batch
+            of symmetric 2-D square matrices ``(..., M, M)``.
+        UPLO (str): Select from ``'L'`` or ``'U'``. It specifies which
+            part of ``a`` is used. ``'L'`` uses the lower triangular part of
+            ``a``, and ``'U'`` uses the upper triangular part of ``a``.
+        with_eigen_vector (bool): Indicates whether or not eigenvectors
+            are computed.
+
+    Returns:
+        tuple of :class:`~cupy.ndarray`:
+            Returns a tuple ``(w, v)``. ``w`` contains eigenvalues and
+            ``v`` contains eigenvectors. ``v[:, i]`` is an eigenvector
+            corresponding to an eigenvalue ``w[i]``. For batch input,
+            ``v[k, :, i]`` is an eigenvector corresponding to an eigenvalue
+            ``w[k, i]`` of ``a[k]``.
+    """
+    if not check_availability('syevj'):
+        raise RuntimeError('syevj is not available.')
+
+    if UPLO not in ('L', 'U'):
+        raise ValueError('UPLO argument must be \'L\' or \'U\'')
+
+    if a.ndim > 2:
+        return _syevj_batched(a, UPLO, with_eigen_vector)
+
+    assert a.ndim == 2
+
+    # reject_float16=False for backward compatibility
+    dtype, v_dtype = _cupy.linalg._util.linalg_common_type(
+        a, reject_float16=False)
+    real_dtype = dtype.char.lower()
+    w_dtype = v_dtype.char.lower()
+
+    # Note that cuSolver assumes fortran array
+    v = a.astype(dtype, order='F', copy=True)
+
+    m, lda = a.shape
+    w = _cupy.empty(m, real_dtype)
+    dev_info = _cupy.empty((1,), _cupy.int32)
+    handle = _device.Device().cusolver_handle
+
+    if with_eigen_vector:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_VECTOR
+    else:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_NOVECTOR
+
+    if UPLO == 'L':
+        uplo = _cublas.CUBLAS_FILL_MODE_LOWER
+    else:  # UPLO == 'U'
+        uplo = _cublas.CUBLAS_FILL_MODE_UPPER
+
+    if dtype == 'f':
+        buffer_size = _cusolver.ssyevj_bufferSize
+        syevj = _cusolver.ssyevj
+    elif dtype == 'd':
+        buffer_size = _cusolver.dsyevj_bufferSize
+        syevj = _cusolver.dsyevj
+    elif dtype == 'F':
+        buffer_size = _cusolver.cheevj_bufferSize
+        syevj = _cusolver.cheevj
+    elif dtype == 'D':
+        buffer_size = _cusolver.zheevj_bufferSize
+        syevj = _cusolver.zheevj
+    else:
+        raise RuntimeError('Only float and double and cuComplex and '
+                           + 'cuDoubleComplex are supported')
+
+    params = _cusolver.createSyevjInfo()
+    work_size = buffer_size(
+        handle, jobz, uplo, m, v.data.ptr, lda, w.data.ptr, params)
+    work = _cupy.empty(work_size, dtype)
+    syevj(
+        handle, jobz, uplo, m, v.data.ptr, lda,
+        w.data.ptr, work.data.ptr, work_size, dev_info.data.ptr, params)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        syevj, dev_info)
+
+    _cusolver.destroySyevjInfo(params)
+
+    w = w.astype(w_dtype, copy=False)
+    if not with_eigen_vector:
+        return w
+    v = v.astype(v_dtype, copy=False)
+    return w, v
+
+
+def _syevj_batched(a, UPLO, with_eigen_vector):
+    # reject_float16=False for backward compatibility
+    dtype, v_dtype = _cupy.linalg._util.linalg_common_type(
+        a, reject_float16=False)
+    real_dtype = dtype.char.lower()
+    w_dtype = v_dtype.char.lower()
+
+    *batch_shape, m, lda = a.shape
+    batch_size = _numpy.prod(batch_shape)
+    a = a.reshape(batch_size, m, lda)
+    v = _cupy.array(
+        a.swapaxes(-2, -1), order='C', copy=True, dtype=dtype)
+    if runtime._is_hip_environment:
+        # the batched syev/heev has a different signature...
+        vp = _linalg._mat_ptrs(v)
+    else:
+        vp = v
+
+    w = _cupy.empty((batch_size, m), real_dtype).swapaxes(-2, -1)
+    dev_info = _cupy.empty((batch_size,), _cupy.int32)
+    handle = _device.Device().cusolver_handle
+
+    if with_eigen_vector:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_VECTOR
+    else:
+        jobz = _cusolver.CUSOLVER_EIG_MODE_NOVECTOR
+
+    if UPLO == 'L':
+        uplo = _cublas.CUBLAS_FILL_MODE_LOWER
+    else:  # UPLO == 'U'
+        uplo = _cublas.CUBLAS_FILL_MODE_UPPER
+
+    if dtype == 'f':
+        buffer_size = _cusolver.ssyevjBatched_bufferSize
+        syevjBatched = _cusolver.ssyevjBatched
+    elif dtype == 'd':
+        buffer_size = _cusolver.dsyevjBatched_bufferSize
+        syevjBatched = _cusolver.dsyevjBatched
+    elif dtype == 'F':
+        buffer_size = _cusolver.cheevjBatched_bufferSize
+        syevjBatched = _cusolver.cheevjBatched
+    elif dtype == 'D':
+        buffer_size = _cusolver.zheevjBatched_bufferSize
+        syevjBatched = _cusolver.zheevjBatched
+    else:
+        raise RuntimeError('Only float and double and cuComplex and '
+                           + 'cuDoubleComplex are supported')
+
+    params = _cusolver.createSyevjInfo()
+    work_size = buffer_size(
+        handle, jobz, uplo, m, vp.data.ptr, lda,
+        w.data.ptr, params, batch_size)
+    work = _cupy.empty(work_size, dtype)
+    syevjBatched(
+        handle, jobz, uplo, m, vp.data.ptr, lda,
+        w.data.ptr, work.data.ptr, work_size, dev_info.data.ptr, params,
+        batch_size)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        syevjBatched, dev_info)
+
+    _cusolver.destroySyevjInfo(params)
+
+    w = w.astype(w_dtype, copy=False)
+    w = w.swapaxes(-2, -1).reshape(*batch_shape, m)
+    if not with_eigen_vector:
+        return w
+    v = v.astype(v_dtype, copy=False)
+    v = v.swapaxes(-2, -1).reshape(*batch_shape, m, m)
+    return w, v
+
+
+def gesv(a, b):
+    """Solve a linear matrix equation using cusolverDn<t1><t2>gesv().
+    Computes the solution to a system of linear equation ``ax = b``.
+    Args:
+        a (cupy.ndarray): The matrix with dimension ``(M, M)``.
+        b (cupy.ndarray): The matrix with dimension ``(M)`` or ``(M, K)``.
+    Returns:
+        cupy.ndarray:
+            The matrix with dimension ``(M)`` or ``(M, K)``.
+    """
+    if not check_availability('gesv'):
+        raise RuntimeError('gesv is not available.')
+
+    if a.ndim != 2:
+        raise ValueError('a.ndim must be 2 (actual:{})'.format(a.ndim))
+    if b.ndim not in (1, 2):
+        raise ValueError('b.ndim must be 1 or 2 (actual:{})'.format(b.ndim))
+    if a.shape[0] != a.shape[1]:
+        raise ValueError('a must be a square matrix.')
+    if a.shape[0] != b.shape[0]:
+        raise ValueError('shape mismatch (a:{}, b:{}).'.
+                         format(a.shape, b.shape))
+    if a.dtype != b.dtype:
+        raise ValueError('dtype mismatch (a:{}, b:{}).'.
+                         format(a.dtype, b.dtype))
+
+    if b.ndim == 2:
+        n, nrhs = b.shape
+    else:
+        n, nrhs = b.shape[0], 1
+
+    compute_type = _linalg.get_compute_type(a.dtype)
+    if a.dtype.char in 'fd':
+        if a.dtype.char == 'f':
+            t1 = t2 = 's'
+        else:
+            t1 = t2 = 'd'
+        if compute_type == _linalg.COMPUTE_TYPE_FP16:
+            t2 = 'h'
+        elif compute_type == _linalg.COMPUTE_TYPE_TF32:
+            t2 = 'x'
+        elif compute_type == _linalg.COMPUTE_TYPE_FP32:
+            t2 = 's'
+    elif a.dtype.char in 'FD':
+        if a.dtype.char == 'F':
+            t1 = t2 = 'c'
+        else:
+            t1 = t2 = 'z'
+        if compute_type == _linalg.COMPUTE_TYPE_FP16:
+            t2 = 'k'
+        elif compute_type == _linalg.COMPUTE_TYPE_TF32:
+            t2 = 'y'
+        elif compute_type == _linalg.COMPUTE_TYPE_FP32:
+            t2 = 'c'
+    else:
+        raise ValueError('unsupported dtype (actual:{})'.format(a.dtype))
+    solver_name = t1 + t2 + 'gesv'
+    solver = getattr(_cusolver, solver_name)
+    helper = getattr(_cusolver, solver_name + '_bufferSize')
+
+    a = a.copy(order='F')
+    b = b.copy(order='F')
+    x = _cupy.empty_like(b)
+    dipiv = _cupy.empty(n, dtype=_numpy.int32)
+    dinfo = _cupy.empty(1, dtype=_numpy.int32)
+    handle = _device.get_cusolver_handle()
+    lwork = helper(handle, n, nrhs, a.data.ptr, n, dipiv.data.ptr,
+                   b.data.ptr, n, x.data.ptr, n, 0)
+    dwork = _cupy.empty(lwork, dtype=_numpy.int8)
+    niters = solver(handle, n, nrhs, a.data.ptr, n, dipiv.data.ptr,
+                    b.data.ptr, n, x.data.ptr, n, dwork.data.ptr, lwork,
+                    dinfo.data.ptr)
+    if niters < 0:
+        raise RuntimeError('gesv has failed ({}).'.format(niters))
+    return x
+
+
+def gels(a, b):
+    """Compute least square solution using cusolverDn<t1><t2>gels().
+
+    Computes the least square solution to a system of ``ax = b``.
+
+    Args:
+        a (cupy.ndarray): The matrix with dimension ``(M, N)``.
+        b (cupy.ndarray): The matrix with dimension ``(M)`` or ``(M, K)``.
+
+    Returns:
+        cupy.ndarray:
+            The matrix with dimension ``(N)`` or ``(N, K)``.
+
+    """
+    if not check_availability('gels'):
+        raise RuntimeError('gels is not available.')
+
+    if a.ndim != 2:
+        raise ValueError('a.ndim must be 2 (actual:{})'.format(a.ndim))
+    if b.ndim == 1:
+        nrhs = 1
+    elif b.ndim == 2:
+        nrhs = b.shape[1]
+    else:
+        raise ValueError('b.ndim must be 1 or 2 (actual: {})'.format(b.ndim))
+    if a.shape[0] != b.shape[0]:
+        raise ValueError('shape mismatch (a:{}, b:{}).'.
+                         format(a.shape, b.shape))
+    if a.dtype != b.dtype:
+        raise ValueError('dtype mismatch (a:{}, b:{}).'.
+                         format(a.dtype, b.dtype))
+
+    m, n = a.shape
+    if m < n:
+        raise ValueError('m must be equal to or greater than n.')
+    max_mn = max(m, n)
+    b_ndim = b.ndim
+
+    compute_type = _linalg.get_compute_type(a.dtype)
+    if a.dtype.char in 'fd':
+        if a.dtype.char == 'f':
+            t1 = t2 = 's'
+        else:
+            t1 = t2 = 'd'
+        if compute_type == _linalg.COMPUTE_TYPE_FP16:
+            t2 = 'h'
+        elif compute_type == _linalg.COMPUTE_TYPE_TF32:
+            t2 = 'x'
+        elif compute_type == _linalg.COMPUTE_TYPE_FP32:
+            t2 = 's'
+    elif a.dtype.char in 'FD':
+        if a.dtype.char == 'F':
+            t1 = t2 = 'c'
+        else:
+            t1 = t2 = 'z'
+        if compute_type == _linalg.COMPUTE_TYPE_FP16:
+            t2 = 'k'
+        elif compute_type == _linalg.COMPUTE_TYPE_TF32:
+            t2 = 'y'
+        elif compute_type == _linalg.COMPUTE_TYPE_FP32:
+            t2 = 'c'
+    else:
+        raise ValueError('unsupported dtype (actual:{})'.format(a.dtype))
+    solver_name = t1 + t2 + 'gels'
+    solver = getattr(_cusolver, solver_name)
+    helper = getattr(_cusolver, solver_name + '_bufferSize')
+
+    a = a.copy(order='F')
+    org_nrhs = nrhs
+    if m > n and nrhs == 1:
+        # Note: this is workaround as there is bug in cusolverDn<T1><T2>gels()
+        # of CUDA 11.0/11.1 and it returns CUSOLVER_STATUS_IRS_NOT_SUPPORTED
+        # when m > n and nrhs == 1.
+        nrhs = 2
+        bb = b.reshape(m, 1)
+        b = _cupy.empty((max_mn, nrhs), dtype=a.dtype, order='F')
+        b[:m, :] = bb
+    else:
+        b = b.copy(order='F')
+    x = _cupy.empty((max_mn, nrhs), dtype=a.dtype, order='F')
+    dinfo = _cupy.empty(1, dtype=_numpy.int32)
+    handle = _device.get_cusolver_handle()
+    lwork = helper(handle, m, n, nrhs, a.data.ptr, m, b.data.ptr, m,
+                   x.data.ptr, max_mn, 0)
+    dwork = _cupy.empty(lwork, dtype=_numpy.int8)
+    niters = solver(handle, m, n, nrhs, a.data.ptr, m, b.data.ptr, m,
+                    x.data.ptr, max_mn, dwork.data.ptr, lwork, dinfo.data.ptr)
+    if niters < 0:
+        if niters <= -50:
+            _warnings.warn('gels reached maximum allowed iterations.')
+        else:
+            raise RuntimeError('gels has failed ({}).'.format(niters))
+    x = x[:n]
+    if org_nrhs != nrhs:
+        x = x[:, :org_nrhs]
+    if b_ndim == 1:
+        x = x.reshape(n)
+    return x
+
+
+def csrlsvqr(A, b, tol=0, reorder=1):
+    """Solves the linear system ``Ax = b`` using QR factorization.
+
+    Args:
+        A (cupyx.scipy.sparse.csr_matrix): Sparse matrix with dimension
+            ``(M, M)``.
+        b (cupy.ndarray): Dense vector with dimension ``(M,)``.
+        tol (float): Tolerance to decide if singular or not.
+        reorder (int): Reordering scheme to reduce zero fill-in.
+            1: symrcm is used.
+            2: symamd is used.
+            3: csrmetisnd is used.
+            else: no reordering.
+    """
+    if not check_availability('csrlsvqr'):
+        raise RuntimeError('csrlsvqr is not available.')
+
+    if not _cupyx.scipy.sparse.isspmatrix_csr(A):
+        raise ValueError('A must be CSR sparse matrix')
+    if not isinstance(b, _cupy.ndarray):
+        raise ValueError('b must be cupy.ndarray')
+    if b.ndim != 1:
+        raise ValueError('b.ndim must be 1 (actual: {})'.format(b.ndim))
+    if not (A.shape[0] == A.shape[1] == b.shape[0]):
+        raise ValueError('invalid shape')
+    if A.dtype != b.dtype:
+        raise TypeError('dtype mismatch')
+
+    dtype = A.dtype
+    if dtype.char == 'f':
+        t = 's'
+    elif dtype.char == 'd':
+        t = 'd'
+    elif dtype.char == 'F':
+        t = 'c'
+    elif dtype.char == 'D':
+        t = 'z'
+    else:
+        raise TypeError('Invalid dtype (actual: {})'.format(dtype))
+    solve = getattr(_cusolver, t + 'csrlsvqr')
+
+    tol = max(tol, 0)
+    m = A.shape[0]
+    b = _internal_ascontiguousarray(b)
+    x = _cupy.empty((m,), dtype=dtype)
+    singularity = _numpy.empty((1,), _numpy.int32)
+
+    handle = _device.get_cusolver_sp_handle()
+    solve(handle, m, A.nnz, A._descr.descriptor, A.data.data.ptr,
+          A.indptr.data.ptr, A.indices.data.ptr, b.data.ptr, tol, reorder,
+          x.data.ptr, singularity.ctypes.data)
+
+    if singularity[0] >= 0:
+        _warnings.warn('A is not positive definite or near singular under '
+                       'tolerance {} (singularity: {})'.
+                       format(tol, singularity))
+    return x
+
+
+cpdef _geqrf_orgqr_batched(a, mode):
+    '''Internal helper for batched QR solver. The input array ``a''
+    is of shape (batch_size, m, n)
+    '''
+    cdef intptr_t x_ptr, tau_ptr, w_ptr, info_ptr
+    cdef int m, n, mn, mc, batch_size, buffersize, orig_n
+
+    # support float32, float64, complex64, and complex128
+    dtype, out_dtype = _cupy.linalg._util.linalg_common_type(a)
+
+    batch_size, m, n = a.shape
+    mn = min(m, n)
+
+    x = a.swapaxes(-2, -1).astype(dtype, order='C', copy=True)
+    if runtime._is_hip_environment:
+        # rocsolver_<t>geqrf_batched has a different signature...
+        ap = _linalg._mat_ptrs(x)
+    else:
+        ap = x
+    x_ptr = ap.data.ptr
+
+    cdef intptr_t handle = _device.get_cusolver_handle()
+    dev_info = _ndarray_init(_cupy.ndarray, (batch_size,), _numpy.int32, None)
+    info_ptr = dev_info.data.ptr
+
+    cdef geqrf_ptr geqrf
+    if dtype == 'f':
+        geqrf_bufferSize = sgeqrf_bufferSize
+        geqrf = geqrf_loop[float]
+    elif dtype == 'd':
+        geqrf_bufferSize = dgeqrf_bufferSize
+        geqrf = geqrf_loop[double]
+    elif dtype == 'F':
+        geqrf_bufferSize = cgeqrf_bufferSize
+        geqrf = geqrf_loop[cuComplex]
+    elif dtype == 'D':
+        geqrf_bufferSize = zgeqrf_bufferSize
+        geqrf = geqrf_loop[cuDoubleComplex]
+    else:
+        msg = ('dtype must be float32, float64, complex64 or complex128'
+               ' (actual: {})'.format(a.dtype))
+        raise ValueError(msg)
+
+    # this wrapper also sets the stream for us
+    buffersize = geqrf_bufferSize(handle, m, n, x.data.ptr, n)
+    # we are on the same stream, so the workspace can be reused in the loop
+    workspace = memory.alloc(buffersize * a.dtype.itemsize)
+    w_ptr = workspace.ptr
+    tau = _cupy.empty((batch_size, mn), dtype=dtype)
+    tau_ptr = tau.data.ptr
+
+    # compute working space of geqrf and solve R
+    # the loop starts here, with gil released to reduce overhead
+    with nogil:
+        status = geqrf(handle, m, n, x_ptr, m, tau_ptr,
+                       w_ptr, buffersize, info_ptr, batch_size)
+    if status != 0:
+        raise _cusolver.CUSOLVERError(status)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        'geqrf', dev_info)
+
+    if mode == 'r':
+        r = x[..., :mn].swapaxes(-2, -1)
+        return _cupy.linalg._util._triu(r).astype(out_dtype, copy=False)
+
+    if mode == 'raw':
+        return (x.astype(out_dtype, copy=False),
+                tau.astype(out_dtype, copy=False))
+
+    if mode == 'complete' and m > n:
+        mc = m
+        orig_n = m
+        q = _cupy.empty((batch_size, m, m), dtype)
+    else:
+        mc = mn
+        orig_n = n
+        q = _cupy.empty((batch_size, n, m), dtype)
+    q[..., :n, :] = x
+    x_ptr = q.data.ptr
+
+    # compute working space of orgqr and solve Q
+    cdef orgqr_ptr orgqr = NULL
+    if dtype == 'f':
+        orgqr_bufferSize = sorgqr_bufferSize
+        orgqr = orgqr_loop[float]
+    elif dtype == 'd':
+        orgqr_bufferSize = dorgqr_bufferSize
+        orgqr = orgqr_loop[double]
+    elif dtype == 'F':
+        orgqr_bufferSize = cungqr_bufferSize
+        orgqr = orgqr_loop[cuComplex]
+    elif dtype == 'D':
+        orgqr_bufferSize = zungqr_bufferSize
+        orgqr = orgqr_loop[cuDoubleComplex]
+    else:
+        raise ValueError
+
+    # this wrapper also sets the stream for us
+    buffersize = orgqr_bufferSize(
+        handle, m, mc, mn, x_ptr, m, tau_ptr)
+    workspace = memory.alloc(buffersize * a.dtype.itemsize)
+    w_ptr = workspace.ptr
+
+    with nogil:
+        status = orgqr(
+            handle, m, mc, mn, x_ptr, m, tau_ptr, w_ptr,
+            buffersize, info_ptr, batch_size, orig_n)
+    if status != 0:
+        raise _cusolver.CUSOLVERError(status)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        'orgqr', dev_info)
+
+    q = q[..., :mc, :].swapaxes(-2, -1)
+    r = x[..., :mc].swapaxes(-2, -1)
+    return (q.astype(out_dtype, copy=False),
+            _cupy.linalg._util._triu(r).astype(out_dtype, copy=False))
diff --git a/cupyx/cusparse.py b/cupyx/cusparse.py
new file mode 100644
index 0000000..8727ab8
--- /dev/null
+++ b/cupyx/cusparse.py
@@ -0,0 +1,2092 @@
+import functools as _functools
+
+import numpy as _numpy
+import platform as _platform
+
+import cupy as _cupy
+from cupy_backends.cuda.api import driver as _driver
+from cupy_backends.cuda.api import runtime as _runtime
+from cupy_backends.cuda.libs import cusparse as _cusparse
+from cupy._core import _dtype
+from cupy.cuda import device as _device
+from cupy.cuda import stream as _stream
+from cupy import _util
+import cupyx.scipy.sparse
+
+
+class MatDescriptor(object):
+
+    def __init__(self, descriptor):
+        self.descriptor = descriptor
+
+    @classmethod
+    def create(cls):
+        descr = _cusparse.createMatDescr()
+        return MatDescriptor(descr)
+
+    def __reduce__(self):
+        return self.create, ()
+
+    def __del__(self, is_shutting_down=_util.is_shutting_down):
+        if is_shutting_down():
+            return
+        if self.descriptor:
+            _cusparse.destroyMatDescr(self.descriptor)
+            self.descriptor = None
+
+    def set_mat_type(self, typ):
+        _cusparse.setMatType(self.descriptor, typ)
+
+    def set_mat_index_base(self, base):
+        _cusparse.setMatIndexBase(self.descriptor, base)
+
+    def set_mat_fill_mode(self, fill_mode):
+        _cusparse.setMatFillMode(self.descriptor, fill_mode)
+
+    def set_mat_diag_type(self, diag_type):
+        _cusparse.setMatDiagType(self.descriptor, diag_type)
+
+
+def _cast_common_type(*xs):
+    dtypes = [x.dtype for x in xs if x is not None]
+    dtype = _functools.reduce(_numpy.promote_types, dtypes)
+    return [x.astype(dtype) if x is not None and x.dtype != dtype else x
+            for x in xs]
+
+
+def _transpose_flag(trans):
+    if trans:
+        return _cusparse.CUSPARSE_OPERATION_TRANSPOSE
+    else:
+        return _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+
+
+def _call_cusparse(name, dtype, *args):
+    if dtype == 'f':
+        prefix = 's'
+    elif dtype == 'd':
+        prefix = 'd'
+    elif dtype == 'F':
+        prefix = 'c'
+    elif dtype == 'D':
+        prefix = 'z'
+    else:
+        raise TypeError
+    f = getattr(_cusparse, prefix + name)
+    return f(*args)
+
+
+_available_cusparse_version = {
+    'csrmv': (8000, 11000),
+    'csrmvEx': (8000, 11000),  # TODO(anaruse): failure in cuSparse 11.0
+    'csrmm': (8000, 11000),
+    'csrmm2': (8000, 11000),
+    'csrgeam': (8000, 11000),
+    'csrgeam2': (9020, None),
+    'csrgemm': (8000, 11000),
+    'csrgemm2': (8000, 12000),
+    'gthr': (8000, 12000),
+
+    # Generic APIs are not available on CUDA 10.2 on Windows.
+    'spmv': ({'Linux': 10200, 'Windows': 11000}, None),
+    # accuracy bugs in cuSparse 10.3.0
+    'spmm': ({'Linux': 10301, 'Windows': 11000}, None),
+
+    'csr2dense': (8000, 12000),
+    'csc2dense': (8000, 12000),
+    'csrsort': (8000, None),
+    'cscsort': (8000, None),
+    'coosort': (8000, None),
+    'coo2csr': (8000, None),
+    'csr2coo': (8000, None),
+    'csr2csc': (8000, 11000),
+    'csc2csr': (8000, 11000),  # the entity is csr2csc
+    'csr2cscEx2': (10200, None),
+    'csc2csrEx2': (10200, None),  # the entity is csr2cscEx2
+    'dense2csc': (8000, None),
+    'dense2csr': (8000, None),
+    'csr2csr_compress': (8000, None),
+    'csrsm2': (9020, 12000),
+    'csrilu02': (8000, None),
+    'denseToSparse': (11300, None),
+    'sparseToDense': (11300, None),
+    'spgemm': (11100, None),
+    'spsm': (11600, None),  # CUDA 11.3.1
+}
+
+
+_available_hipsparse_version = {
+    # For APIs supported by CUDA but not yet by HIP, we still need them here
+    # so that our test suite can cover both platforms.
+    'csrmv': (305, None),
+    'csrmvEx': (_numpy.inf, None),
+    'csrmm': (305, None),
+    'csrmm2': (305, None),
+    'csrgeam': (305, None),
+    'csrgeam2': (305, None),
+    'csrgemm': (305, None),
+    'csrgemm2': (305, None),
+    'gthr': (305, None),
+    'spmv': (402, None),
+    'spmm': (402, None),
+    'csr2dense': (305, None),
+    'csc2dense': (305, None),
+    'csrsort': (305, None),
+    'cscsort': (305, None),
+    'coosort': (305, None),
+    'coo2csr': (305, None),
+    'csr2coo': (305, None),
+    'csr2csc': (305, None),
+    'csc2csr': (305, None),  # the entity is csr2csc
+    'csr2cscEx2': (_numpy.inf, None),
+    'csc2csrEx2': (_numpy.inf, None),  # the entity is csr2cscEx2
+    'dense2csc': (305, None),
+    'dense2csr': (305, None),
+    'csr2csr_compress': (305, None),
+    'csrsm2': (305, None),  # avaiable since 305 but seems buggy
+    'csrilu02': (305, None),
+    'denseToSparse': (402, None),
+    'sparseToDense': (402, None),
+    'spgemm': (_numpy.inf, None),
+    'spsm': (50000000, None),
+}
+
+
+def _get_avail_version_from_spec(x):
+    if isinstance(x, dict):
+        os_name = _platform.system()
+        if os_name not in x:
+            msg = 'No version information specified for the OS: {}'.format(
+                os_name)
+            raise ValueError(msg)
+        return x[os_name]
+    return x
+
+
+@_util.memoize()
+def check_availability(name):
+    if not _runtime.is_hip:
+        available_version = _available_cusparse_version
+        version = _cusparse.get_build_version()
+    else:
+        available_version = _available_hipsparse_version
+        version = _driver.get_build_version()  # = HIP_VERSION
+    if name not in available_version:
+        msg = 'No available version information specified for {}'.format(name)
+        raise ValueError(msg)
+    version_added, version_removed = available_version[name]
+    version_added = _get_avail_version_from_spec(version_added)
+    version_removed = _get_avail_version_from_spec(version_removed)
+    if version_added is not None and version < version_added:
+        return False
+    if version_removed is not None and version >= version_removed:
+        return False
+    return True
+
+
+def getVersion() -> int:
+    return _cusparse.getVersion(_device.get_cusparse_handle())
+
+
+def csrmv(a, x, y=None, alpha=1, beta=0, transa=False):
+    """Matrix-vector product for a CSR-matrix and a dense vector.
+
+    .. math::
+
+       y = \\alpha * o_a(A) x + \\beta y,
+
+    where :math:`o_a` is a transpose function when ``transa`` is ``True`` and
+    is an identity function otherwise.
+
+    Args:
+        a (cupyx.cusparse.csr_matrix): Matrix A.
+        x (cupy.ndarray): Vector x.
+        y (cupy.ndarray or None): Vector y. It must be F-contiguous.
+        alpha (float): Coefficient for x.
+        beta (float): Coefficient for y.
+        transa (bool): If ``True``, transpose of ``A`` is used.
+
+    Returns:
+        cupy.ndarray: Calculated ``y``.
+
+    """
+    if not check_availability('csrmv'):
+        raise RuntimeError('csrmv is not available.')
+
+    assert y is None or y.flags.f_contiguous
+
+    a_shape = a.shape if not transa else a.shape[::-1]
+    if a_shape[1] != len(x):
+        raise ValueError('dimension mismatch')
+
+    handle = _device.get_cusparse_handle()
+    m, n = a_shape
+    a, x, y = _cast_common_type(a, x, y)
+    dtype = a.dtype
+    if y is None:
+        y = _cupy.zeros(m, dtype)
+    alpha = _numpy.array(alpha, dtype).ctypes
+    beta = _numpy.array(beta, dtype).ctypes
+
+    _call_cusparse(
+        'csrmv', dtype,
+        handle, _transpose_flag(transa),
+        a.shape[0], a.shape[1], a.nnz, alpha.data, a._descr.descriptor,
+        a.data.data.ptr, a.indptr.data.ptr, a.indices.data.ptr,
+        x.data.ptr, beta.data, y.data.ptr)
+
+    return y
+
+
+def csrmvExIsAligned(a, x, y=None):
+    """Check if the pointers of arguments for csrmvEx are aligned or not
+
+    Args:
+        a (cupyx.cusparse.csr_matrix): Matrix A.
+        x (cupy.ndarray): Vector x.
+        y (cupy.ndarray or None): Vector y.
+
+        Check if a, x, y pointers are aligned by 128 bytes as
+        required by csrmvEx.
+
+    Returns:
+        bool:
+        ``True`` if all pointers are aligned.
+        ``False`` if otherwise.
+
+    """
+
+    if a.data.data.ptr % 128 != 0:
+        return False
+    if a.indptr.data.ptr % 128 != 0:
+        return False
+    if a.indices.data.ptr % 128 != 0:
+        return False
+    if x.data.ptr % 128 != 0:
+        return False
+    if y is not None and y.data.ptr % 128 != 0:
+        return False
+    return True
+
+
+def csrmvEx(a, x, y=None, alpha=1, beta=0, merge_path=True):
+    """Matrix-vector product for a CSR-matrix and a dense vector.
+
+    .. math::
+
+       y = \\alpha * A x + \\beta y,
+
+    Args:
+        a (cupyx.cusparse.csr_matrix): Matrix A.
+        x (cupy.ndarray): Vector x.
+        y (cupy.ndarray or None): Vector y. It must be F-contiguous.
+        alpha (float): Coefficient for x.
+        beta (float): Coefficient for y.
+        merge_path (bool): If ``True``, merge path algorithm is used.
+
+        All pointers must be aligned with 128 bytes.
+
+    Returns:
+        cupy.ndarray: Calculated ``y``.
+
+    """
+    if not check_availability('csrmvEx'):
+        raise RuntimeError('csrmvEx is not available.')
+
+    assert y is None or y.flags.f_contiguous
+
+    if a.shape[1] != len(x):
+        raise ValueError('dimension mismatch')
+
+    handle = _device.get_cusparse_handle()
+    m, n = a.shape
+
+    a, x, y = _cast_common_type(a, x, y)
+    dtype = a.dtype
+    if y is None:
+        y = _cupy.zeros(m, dtype)
+
+    datatype = _dtype.to_cuda_dtype(dtype)
+    algmode = _cusparse.CUSPARSE_ALG_MERGE_PATH if \
+        merge_path else _cusparse.CUSPARSE_ALG_NAIVE
+    transa_flag = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+
+    alpha = _numpy.array(alpha, dtype).ctypes
+    beta = _numpy.array(beta, dtype).ctypes
+
+    assert csrmvExIsAligned(a, x, y)
+
+    bufferSize = _cusparse.csrmvEx_bufferSize(
+        handle, algmode, transa_flag,
+        a.shape[0], a.shape[1], a.nnz, alpha.data, datatype,
+        a._descr.descriptor, a.data.data.ptr, datatype,
+        a.indptr.data.ptr, a.indices.data.ptr,
+        x.data.ptr, datatype, beta.data, datatype,
+        y.data.ptr, datatype, datatype)
+
+    buf = _cupy.empty(bufferSize, 'b')
+    assert buf.data.ptr % 128 == 0
+
+    _cusparse.csrmvEx(
+        handle, algmode, transa_flag,
+        a.shape[0], a.shape[1], a.nnz, alpha.data, datatype,
+        a._descr.descriptor, a.data.data.ptr, datatype,
+        a.indptr.data.ptr, a.indices.data.ptr,
+        x.data.ptr, datatype, beta.data, datatype,
+        y.data.ptr, datatype, datatype, buf.data.ptr)
+    return y
+
+
+def csrmm(a, b, c=None, alpha=1, beta=0, transa=False):
+    """Matrix-matrix product for a CSR-matrix and a dense matrix.
+
+    .. math::
+
+       C = \\alpha o_a(A) B + \\beta C,
+
+    where :math:`o_a` is a transpose function when ``transa`` is ``True`` and
+    is an identity function otherwise.
+
+    Args:
+        a (cupyx.scipy.sparse.csr): Sparse matrix A.
+        b (cupy.ndarray): Dense matrix B. It must be F-contiguous.
+        c (cupy.ndarray or None): Dense matrix C. It must be F-contiguous.
+        alpha (float): Coefficient for AB.
+        beta (float): Coefficient for C.
+        transa (bool): If ``True``, transpose of A is used.
+
+    Returns:
+        cupy.ndarray: Calculated C.
+
+    """
+    if not check_availability('csrmm'):
+        raise RuntimeError('csrmm is not available.')
+
+    assert a.ndim == b.ndim == 2
+    assert b.flags.f_contiguous
+    assert c is None or c.flags.f_contiguous
+
+    a_shape = a.shape if not transa else a.shape[::-1]
+    if a_shape[1] != b.shape[0]:
+        raise ValueError('dimension mismatch')
+
+    handle = _device.get_cusparse_handle()
+    m, k = a_shape
+    n = b.shape[1]
+
+    a, b, c = _cast_common_type(a, b, c)
+    if c is None:
+        c = _cupy.zeros((m, n), a.dtype, 'F')
+
+    ldb = k
+    ldc = m
+
+    alpha = _numpy.array(alpha, a.dtype).ctypes
+    beta = _numpy.array(beta, a.dtype).ctypes
+    _call_cusparse(
+        'csrmm', a.dtype,
+        handle, _transpose_flag(transa),
+        a.shape[0], n, a.shape[1], a.nnz,
+        alpha.data, a._descr.descriptor, a.data.data.ptr,
+        a.indptr.data.ptr, a.indices.data.ptr,
+        b.data.ptr, ldb, beta.data, c.data.ptr, ldc)
+    return c
+
+
+def csrmm2(a, b, c=None, alpha=1.0, beta=0.0, transa=False, transb=False):
+    """Matrix-matrix product for a CSR-matrix and a dense matrix.
+
+    .. math::
+
+       C = \\alpha o_a(A) o_b(B) + \\beta C,
+
+    where :math:`o_a` and :math:`o_b` are transpose functions when ``transa``
+    and ``tranb`` are ``True`` respectively. And they are identity functions
+    otherwise.
+    It is forbidden that both ``transa`` and ``transb`` are ``True`` in
+    cuSPARSE specification.
+
+    Args:
+        a (cupyx.scipy.sparse.csr): Sparse matrix A.
+        b (cupy.ndarray): Dense matrix B. It must be F-contiguous.
+        c (cupy.ndarray or None): Dense matrix C. It must be F-contiguous.
+        alpha (float): Coefficient for AB.
+        beta (float): Coefficient for C.
+        transa (bool): If ``True``, transpose of A is used.
+        transb (bool): If ``True``, transpose of B is used.
+
+    Returns:
+        cupy.ndarray: Calculated C.
+
+    """
+    if not check_availability('csrmm2'):
+        raise RuntimeError('csrmm2 is not available.')
+
+    assert a.ndim == b.ndim == 2
+    assert a.has_canonical_format
+    assert b.flags.f_contiguous
+    assert c is None or c.flags.f_contiguous
+    assert not (transa and transb)
+
+    a_shape = a.shape if not transa else a.shape[::-1]
+    b_shape = b.shape if not transb else b.shape[::-1]
+    if a_shape[1] != b_shape[0]:
+        raise ValueError('dimension mismatch')
+
+    handle = _device.get_cusparse_handle()
+    m, k = a_shape
+    n = b_shape[1]
+
+    a, b, c = _cast_common_type(a, b, c)
+    if c is None:
+        c = _cupy.zeros((m, n), a.dtype, 'F')
+
+    ldb = b.shape[0]
+    ldc = c.shape[0]
+    op_a = _transpose_flag(transa)
+    op_b = _transpose_flag(transb)
+    alpha = _numpy.array(alpha, a.dtype).ctypes
+    beta = _numpy.array(beta, a.dtype).ctypes
+    _call_cusparse(
+        'csrmm2', a.dtype,
+        handle, op_a, op_b, a.shape[0], n, a.shape[1], a.nnz,
+        alpha.data, a._descr.descriptor, a.data.data.ptr,
+        a.indptr.data.ptr, a.indices.data.ptr,
+        b.data.ptr, ldb, beta.data, c.data.ptr, ldc)
+    return c
+
+
+def csrgeam(a, b, alpha=1, beta=1):
+    """Matrix-matrix addition.
+
+    .. math::
+        C = \\alpha A + \\beta B
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
+        b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
+        alpha (float): Coefficient for A.
+        beta (float): Coefficient for B.
+
+    Returns:
+        cupyx.scipy.sparse.csr_matrix: Result matrix.
+
+    """
+    if not check_availability('csrgeam'):
+        raise RuntimeError('csrgeam is not available.')
+
+    if not isinstance(a, cupyx.scipy.sparse.csr_matrix):
+        raise TypeError('unsupported type (actual: {})'.format(type(a)))
+    if not isinstance(b, cupyx.scipy.sparse.csr_matrix):
+        raise TypeError('unsupported type (actual: {})'.format(type(b)))
+    assert a.has_canonical_format
+    assert b.has_canonical_format
+    if a.shape != b.shape:
+        raise ValueError('inconsistent shapes')
+
+    handle = _device.get_cusparse_handle()
+    m, n = a.shape
+    a, b = _cast_common_type(a, b)
+    nnz = _numpy.empty((), 'i')
+    _cusparse.setPointerMode(
+        handle, _cusparse.CUSPARSE_POINTER_MODE_HOST)
+
+    c_descr = MatDescriptor.create()
+    c_indptr = _cupy.empty(m + 1, 'i')
+
+    _cusparse.xcsrgeamNnz(
+        handle, m, n,
+        a._descr.descriptor, a.nnz, a.indptr.data.ptr, a.indices.data.ptr,
+        b._descr.descriptor, b.nnz, b.indptr.data.ptr, b.indices.data.ptr,
+        c_descr.descriptor, c_indptr.data.ptr, nnz.ctypes.data)
+
+    c_indices = _cupy.empty(int(nnz), 'i')
+    c_data = _cupy.empty(int(nnz), a.dtype)
+    alpha = _numpy.array(alpha, a.dtype).ctypes
+    beta = _numpy.array(beta, a.dtype).ctypes
+    _call_cusparse(
+        'csrgeam', a.dtype,
+        handle, m, n, alpha.data,
+        a._descr.descriptor, a.nnz, a.data.data.ptr,
+        a.indptr.data.ptr, a.indices.data.ptr, beta.data,
+        b._descr.descriptor, b.nnz, b.data.data.ptr,
+        b.indptr.data.ptr, b.indices.data.ptr,
+        c_descr.descriptor, c_data.data.ptr, c_indptr.data.ptr,
+        c_indices.data.ptr)
+
+    c = cupyx.scipy.sparse.csr_matrix(
+        (c_data, c_indices, c_indptr), shape=a.shape)
+    c._has_canonical_format = True
+    return c
+
+
+def csrgeam2(a, b, alpha=1, beta=1):
+    """Matrix-matrix addition.
+
+    .. math::
+        C = \\alpha A + \\beta B
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
+        b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
+        alpha (float): Coefficient for A.
+        beta (float): Coefficient for B.
+
+    Returns:
+        cupyx.scipy.sparse.csr_matrix: Result matrix.
+
+    """
+    if not check_availability('csrgeam2'):
+        raise RuntimeError('csrgeam2 is not available.')
+
+    if not isinstance(a, cupyx.scipy.sparse.csr_matrix):
+        raise TypeError('unsupported type (actual: {})'.format(type(a)))
+    if not isinstance(b, cupyx.scipy.sparse.csr_matrix):
+        raise TypeError('unsupported type (actual: {})'.format(type(b)))
+    assert a.has_canonical_format
+    assert b.has_canonical_format
+    if a.shape != b.shape:
+        raise ValueError('inconsistent shapes')
+
+    handle = _device.get_cusparse_handle()
+    m, n = a.shape
+    a, b = _cast_common_type(a, b)
+    nnz = _numpy.empty((), 'i')
+    _cusparse.setPointerMode(
+        handle, _cusparse.CUSPARSE_POINTER_MODE_HOST)
+
+    alpha = _numpy.array(alpha, a.dtype).ctypes
+    beta = _numpy.array(beta, a.dtype).ctypes
+    c_descr = MatDescriptor.create()
+    c_indptr = _cupy.empty(m + 1, 'i')
+
+    null_ptr = 0
+    buff_size = _call_cusparse(
+        'csrgeam2_bufferSizeExt', a.dtype,
+        handle, m, n, alpha.data, a._descr.descriptor, a.nnz, a.data.data.ptr,
+        a.indptr.data.ptr, a.indices.data.ptr, beta.data, b._descr.descriptor,
+        b.nnz, b.data.data.ptr, b.indptr.data.ptr, b.indices.data.ptr,
+        c_descr.descriptor, null_ptr, c_indptr.data.ptr, null_ptr)
+    buff = _cupy.empty(buff_size, _numpy.int8)
+    _cusparse.xcsrgeam2Nnz(
+        handle, m, n, a._descr.descriptor, a.nnz, a.indptr.data.ptr,
+        a.indices.data.ptr, b._descr.descriptor, b.nnz, b.indptr.data.ptr,
+        b.indices.data.ptr, c_descr.descriptor, c_indptr.data.ptr,
+        nnz.ctypes.data, buff.data.ptr)
+    c_indices = _cupy.empty(int(nnz), 'i')
+    c_data = _cupy.empty(int(nnz), a.dtype)
+    _call_cusparse(
+        'csrgeam2', a.dtype,
+        handle, m, n, alpha.data, a._descr.descriptor, a.nnz, a.data.data.ptr,
+        a.indptr.data.ptr, a.indices.data.ptr, beta.data, b._descr.descriptor,
+        b.nnz, b.data.data.ptr, b.indptr.data.ptr, b.indices.data.ptr,
+        c_descr.descriptor, c_data.data.ptr, c_indptr.data.ptr,
+        c_indices.data.ptr, buff.data.ptr)
+
+    c = cupyx.scipy.sparse.csr_matrix(
+        (c_data, c_indices, c_indptr), shape=a.shape)
+    c._has_canonical_format = True
+    return c
+
+
+def csrgemm(a, b, transa=False, transb=False):
+    """Matrix-matrix product for CSR-matrix.
+
+    math::
+       C = op(A) op(B),
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
+        b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
+        transa (bool): If ``True``, transpose of A is used.
+        transb (bool): If ``True``, transpose of B is used.
+
+    Returns:
+        cupyx.scipy.sparse.csr_matrix: Calculated C.
+
+    """
+    if not check_availability('csrgemm'):
+        raise RuntimeError('csrgemm is not available.')
+
+    assert a.ndim == b.ndim == 2
+    assert a.has_canonical_format
+    assert b.has_canonical_format
+    a_shape = a.shape if not transa else a.shape[::-1]
+    b_shape = b.shape if not transb else b.shape[::-1]
+    if a_shape[1] != b_shape[0]:
+        raise ValueError('dimension mismatch')
+
+    handle = _device.get_cusparse_handle()
+    m, k = a_shape
+    n = b_shape[1]
+
+    a, b = _cast_common_type(a, b)
+
+    if a.nnz == 0 or b.nnz == 0:
+        return cupyx.scipy.sparse.csr_matrix((m, n), dtype=a.dtype)
+
+    op_a = _transpose_flag(transa)
+    op_b = _transpose_flag(transb)
+
+    nnz = _numpy.empty((), 'i')
+    _cusparse.setPointerMode(
+        handle, _cusparse.CUSPARSE_POINTER_MODE_HOST)
+
+    c_descr = MatDescriptor.create()
+    c_indptr = _cupy.empty(m + 1, 'i')
+
+    _cusparse.xcsrgemmNnz(
+        handle, op_a, op_b, m, n, k, a._descr.descriptor, a.nnz,
+        a.indptr.data.ptr, a.indices.data.ptr, b._descr.descriptor, b.nnz,
+        b.indptr.data.ptr, b.indices.data.ptr, c_descr.descriptor,
+        c_indptr.data.ptr, nnz.ctypes.data)
+
+    c_indices = _cupy.empty(int(nnz), 'i')
+    c_data = _cupy.empty(int(nnz), a.dtype)
+    _call_cusparse(
+        'csrgemm', a.dtype,
+        handle, op_a, op_b, m, n, k, a._descr.descriptor, a.nnz,
+        a.data.data.ptr, a.indptr.data.ptr, a.indices.data.ptr,
+        b._descr.descriptor, b.nnz, b.data.data.ptr, b.indptr.data.ptr,
+        b.indices.data.ptr,
+        c_descr.descriptor, c_data.data.ptr, c_indptr.data.ptr,
+        c_indices.data.ptr)
+
+    c = cupyx.scipy.sparse.csr_matrix(
+        (c_data, c_indices, c_indptr), shape=(m, n))
+    c._has_canonical_format = True
+    return c
+
+
+def csrgemm2(a, b, d=None, alpha=1, beta=1):
+    """Matrix-matrix product for CSR-matrix.
+
+    math::
+       C = alpha * A * B + beta * D
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
+        b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
+        d (cupyx.scipy.sparse.csr_matrix or None): Sparse matrix D.
+        alpha (scalar): Coefficient
+        beta (scalar): Coefficient
+
+    Returns:
+        cupyx.scipy.sparse.csr_matrix
+
+    """
+    if not check_availability('csrgemm2'):
+        raise RuntimeError('csrgemm2 is not available.')
+
+    assert a.ndim == b.ndim == 2
+    if not isinstance(a, cupyx.scipy.sparse.csr_matrix):
+        raise TypeError('unsupported type (actual: {})'.format(type(a)))
+    if not isinstance(b, cupyx.scipy.sparse.csr_matrix):
+        raise TypeError('unsupported type (actual: {})'.format(type(b)))
+    assert a.has_canonical_format
+    assert b.has_canonical_format
+    if a.shape[1] != b.shape[0]:
+        raise ValueError('mismatched shape')
+    if d is not None:
+        assert d.ndim == 2
+        if not isinstance(d, cupyx.scipy.sparse.csr_matrix):
+            raise TypeError('unsupported type (actual: {})'.format(type(d)))
+        assert d.has_canonical_format
+        if a.shape[0] != d.shape[0] or b.shape[1] != d.shape[1]:
+            raise ValueError('mismatched shape')
+        if _runtime.is_hip and _driver.get_build_version() < 402:
+            raise RuntimeError('d != None is supported since ROCm 4.2.0')
+
+    handle = _device.get_cusparse_handle()
+    m, k = a.shape
+    _, n = b.shape
+
+    if d is None:
+        a, b = _cast_common_type(a, b)
+    else:
+        a, b, d = _cast_common_type(a, b, d)
+
+    info = _cusparse.createCsrgemm2Info()
+    alpha = _numpy.array(alpha, a.dtype).ctypes
+    null_ptr = 0
+    if d is None:
+        beta_data = null_ptr
+        d_descr = MatDescriptor.create()
+        d_nnz = 0
+        d_data = null_ptr
+        d_indptr = null_ptr
+        d_indices = null_ptr
+    else:
+        beta = _numpy.array(beta, a.dtype).ctypes
+        beta_data = beta.data
+        d_descr = d._descr
+        d_nnz = d.nnz
+        d_data = d.data.data.ptr
+        d_indptr = d.indptr.data.ptr
+        d_indices = d.indices.data.ptr
+
+    buff_size = _call_cusparse(
+        'csrgemm2_bufferSizeExt', a.dtype,
+        handle, m, n, k, alpha.data, a._descr.descriptor, a.nnz,
+        a.indptr.data.ptr, a.indices.data.ptr, b._descr.descriptor, b.nnz,
+        b.indptr.data.ptr, b.indices.data.ptr, beta_data, d_descr.descriptor,
+        d_nnz, d_indptr, d_indices, info)
+    buff = _cupy.empty(buff_size, _numpy.int8)
+
+    c_nnz = _numpy.empty((), 'i')
+    _cusparse.setPointerMode(handle, _cusparse.CUSPARSE_POINTER_MODE_HOST)
+
+    c_descr = MatDescriptor.create()
+    c_indptr = _cupy.empty(m + 1, 'i')
+    _cusparse.xcsrgemm2Nnz(
+        handle, m, n, k, a._descr.descriptor, a.nnz, a.indptr.data.ptr,
+        a.indices.data.ptr, b._descr.descriptor, b.nnz, b.indptr.data.ptr,
+        b.indices.data.ptr, d_descr.descriptor, d_nnz, d_indptr, d_indices,
+        c_descr.descriptor, c_indptr.data.ptr, c_nnz.ctypes.data, info,
+        buff.data.ptr)
+
+    c_indices = _cupy.empty(int(c_nnz), 'i')
+    c_data = _cupy.empty(int(c_nnz), a.dtype)
+    _call_cusparse(
+        'csrgemm2', a.dtype,
+        handle, m, n, k, alpha.data, a._descr.descriptor, a.nnz,
+        a.data.data.ptr, a.indptr.data.ptr, a.indices.data.ptr,
+        b._descr.descriptor, b.nnz, b.data.data.ptr, b.indptr.data.ptr,
+        b.indices.data.ptr, beta_data, d_descr.descriptor, d_nnz, d_data,
+        d_indptr, d_indices, c_descr.descriptor, c_data.data.ptr,
+        c_indptr.data.ptr, c_indices.data.ptr, info, buff.data.ptr)
+
+    c = cupyx.scipy.sparse.csr_matrix(
+        (c_data, c_indices, c_indptr), shape=(m, n))
+    c._has_canonical_format = True
+    _cusparse.destroyCsrgemm2Info(info)
+    return c
+
+
+def csr2dense(x, out=None):
+    """Converts CSR-matrix to a dense matrix.
+
+    Args:
+        x (cupyx.scipy.sparse.csr_matrix): A sparse matrix to convert.
+        out (cupy.ndarray or None): A dense metrix to store the result.
+            It must be F-contiguous.
+
+    Returns:
+        cupy.ndarray: Converted result.
+
+    """
+    if not check_availability('csr2dense'):
+        raise RuntimeError('csr2dense is not available.')
+
+    dtype = x.dtype
+    assert dtype.char in 'fdFD'
+    if out is None:
+        out = _cupy.empty(x.shape, dtype=dtype, order='F')
+    else:
+        assert out.flags.f_contiguous
+
+    handle = _device.get_cusparse_handle()
+    _call_cusparse(
+        'csr2dense', x.dtype,
+        handle, x.shape[0], x.shape[1], x._descr.descriptor,
+        x.data.data.ptr, x.indptr.data.ptr, x.indices.data.ptr,
+        out.data.ptr, x.shape[0])
+
+    return out
+
+
+def csc2dense(x, out=None):
+    """Converts CSC-matrix to a dense matrix.
+
+    Args:
+        x (cupyx.scipy.sparse.csc_matrix): A sparse matrix to convert.
+        out (cupy.ndarray or None): A dense metrix to store the result.
+            It must be F-contiguous.
+
+    Returns:
+        cupy.ndarray: Converted result.
+
+    """
+    if not check_availability('csc2dense'):
+        raise RuntimeError('csc2dense is not available.')
+
+    dtype = x.dtype
+    assert dtype.char in 'fdFD'
+    if out is None:
+        out = _cupy.empty(x.shape, dtype=dtype, order='F')
+    else:
+        assert out.flags.f_contiguous
+
+    handle = _device.get_cusparse_handle()
+    _call_cusparse(
+        'csc2dense', x.dtype,
+        handle, x.shape[0], x.shape[1], x._descr.descriptor,
+        x.data.data.ptr, x.indices.data.ptr, x.indptr.data.ptr,
+        out.data.ptr, x.shape[0])
+
+    return out
+
+
+def csrsort(x):
+    """Sorts indices of CSR-matrix in place.
+
+    Args:
+        x (cupyx.scipy.sparse.csr_matrix): A sparse matrix to sort.
+
+    """
+    if not check_availability('csrsort'):
+        raise RuntimeError('csrsort is not available.')
+
+    nnz = x.nnz
+    if nnz == 0:
+        return
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+
+    buffer_size = _cusparse.xcsrsort_bufferSizeExt(
+        handle, m, n, nnz, x.indptr.data.ptr,
+        x.indices.data.ptr)
+    buf = _cupy.empty(buffer_size, 'b')
+    P = _cupy.empty(nnz, 'i')
+    data_orig = x.data.copy()
+    _cusparse.createIdentityPermutation(handle, nnz, P.data.ptr)
+    _cusparse.xcsrsort(
+        handle, m, n, nnz, x._descr.descriptor, x.indptr.data.ptr,
+        x.indices.data.ptr, P.data.ptr, buf.data.ptr)
+
+    if check_availability('gthr'):
+        _call_cusparse(
+            'gthr', x.dtype,
+            handle, nnz, data_orig.data.ptr, x.data.data.ptr,
+            P.data.ptr, _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    else:
+        desc_x = SpVecDescriptor.create(P, x.data)
+        desc_y = DnVecDescriptor.create(data_orig)
+        _cusparse.gather(handle, desc_y.desc, desc_x.desc)
+
+
+def cscsort(x):
+    """Sorts indices of CSC-matrix in place.
+
+    Args:
+        x (cupyx.scipy.sparse.csc_matrix): A sparse matrix to sort.
+
+    """
+    if not check_availability('cscsort'):
+        raise RuntimeError('cscsort is not available.')
+
+    nnz = x.nnz
+    if nnz == 0:
+        return
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+
+    buffer_size = _cusparse.xcscsort_bufferSizeExt(
+        handle, m, n, nnz, x.indptr.data.ptr,
+        x.indices.data.ptr)
+    buf = _cupy.empty(buffer_size, 'b')
+    P = _cupy.empty(nnz, 'i')
+    data_orig = x.data.copy()
+    _cusparse.createIdentityPermutation(handle, nnz, P.data.ptr)
+    _cusparse.xcscsort(
+        handle, m, n, nnz, x._descr.descriptor, x.indptr.data.ptr,
+        x.indices.data.ptr, P.data.ptr, buf.data.ptr)
+
+    if check_availability('gthr'):
+        _call_cusparse(
+            'gthr', x.dtype,
+            handle, nnz, data_orig.data.ptr, x.data.data.ptr,
+            P.data.ptr, _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    else:
+        desc_x = SpVecDescriptor.create(P, x.data)
+        desc_y = DnVecDescriptor.create(data_orig)
+        _cusparse.gather(handle, desc_y.desc, desc_x.desc)
+
+
+def coosort(x, sort_by='r'):
+    """Sorts indices of COO-matrix in place.
+
+    Args:
+        x (cupyx.scipy.sparse.coo_matrix): A sparse matrix to sort.
+        sort_by (str): Sort the indices by row ('r', default) or column ('c').
+
+    """
+    if not check_availability('coosort'):
+        raise RuntimeError('coosort is not available.')
+
+    nnz = x.nnz
+    if nnz == 0:
+        return
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+
+    buffer_size = _cusparse.xcoosort_bufferSizeExt(
+        handle, m, n, nnz, x.row.data.ptr, x.col.data.ptr)
+    buf = _cupy.empty(buffer_size, 'b')
+    P = _cupy.empty(nnz, 'i')
+    data_orig = x.data.copy()
+    _cusparse.createIdentityPermutation(handle, nnz, P.data.ptr)
+    if sort_by == 'r':
+        _cusparse.xcoosortByRow(
+            handle, m, n, nnz, x.row.data.ptr, x.col.data.ptr,
+            P.data.ptr, buf.data.ptr)
+    elif sort_by == 'c':
+        _cusparse.xcoosortByColumn(
+            handle, m, n, nnz, x.row.data.ptr, x.col.data.ptr,
+            P.data.ptr, buf.data.ptr)
+    else:
+        raise ValueError("sort_by must be either 'r' or 'c'")
+
+    if check_availability('gthr'):
+        _call_cusparse(
+            'gthr', x.dtype,
+            handle, nnz, data_orig.data.ptr, x.data.data.ptr,
+            P.data.ptr, _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    else:
+        desc_x = SpVecDescriptor.create(P, x.data)
+        desc_y = DnVecDescriptor.create(data_orig)
+        _cusparse.gather(handle, desc_y.desc, desc_x.desc)
+
+    if sort_by == 'c':  # coo is sorted by row first
+        x._has_canonical_format = False
+
+
+def coo2csr(x):
+    handle = _device.get_cusparse_handle()
+    m = x.shape[0]
+    nnz = x.nnz
+    if nnz == 0:
+        indptr = _cupy.zeros(m + 1, 'i')
+    else:
+        indptr = _cupy.empty(m + 1, 'i')
+        _cusparse.xcoo2csr(
+            handle, x.row.data.ptr, nnz, m,
+            indptr.data.ptr, _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    return cupyx.scipy.sparse.csr_matrix(
+        (x.data, x.col, indptr), shape=x.shape)
+
+
+def coo2csc(x):
+    handle = _device.get_cusparse_handle()
+    n = x.shape[1]
+    nnz = x.nnz
+    if nnz == 0:
+        indptr = _cupy.zeros(n + 1, 'i')
+    else:
+        indptr = _cupy.empty(n + 1, 'i')
+        _cusparse.xcoo2csr(
+            handle, x.col.data.ptr, nnz, n,
+            indptr.data.ptr, _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    return cupyx.scipy.sparse.csc_matrix(
+        (x.data, x.row, indptr), shape=x.shape)
+
+
+def csr2coo(x, data, indices):
+    """Converts a CSR-matrix to COO format.
+
+    Args:
+        x (cupyx.scipy.sparse.csr_matrix): A matrix to be converted.
+        data (cupy.ndarray): A data array for converted data.
+        indices (cupy.ndarray): An index array for converted data.
+
+    Returns:
+        cupyx.scipy.sparse.coo_matrix: A converted matrix.
+
+    """
+    if not check_availability('csr2coo'):
+        raise RuntimeError('csr2coo is not available.')
+
+    handle = _device.get_cusparse_handle()
+    m = x.shape[0]
+    nnz = x.nnz
+    row = _cupy.empty(nnz, 'i')
+    _cusparse.xcsr2coo(
+        handle, x.indptr.data.ptr, nnz, m, row.data.ptr,
+        _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    # data and indices did not need to be copied already
+    return cupyx.scipy.sparse.coo_matrix(
+        (data, (row, indices)), shape=x.shape)
+
+
+def csr2csc(x):
+    if not check_availability('csr2csc'):
+        raise RuntimeError('csr2csc is not available.')
+
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+    nnz = x.nnz
+    data = _cupy.empty(nnz, x.dtype)
+    indices = _cupy.empty(nnz, 'i')
+    if nnz == 0:
+        indptr = _cupy.zeros(n + 1, 'i')
+    else:
+        indptr = _cupy.empty(n + 1, 'i')
+        _call_cusparse(
+            'csr2csc', x.dtype,
+            handle, m, n, nnz, x.data.data.ptr,
+            x.indptr.data.ptr, x.indices.data.ptr,
+            data.data.ptr, indices.data.ptr, indptr.data.ptr,
+            _cusparse.CUSPARSE_ACTION_NUMERIC,
+            _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    return cupyx.scipy.sparse.csc_matrix(
+        (data, indices, indptr), shape=x.shape)
+
+
+def csr2cscEx2(x):
+    if not check_availability('csr2cscEx2'):
+        raise RuntimeError('csr2cscEx2 is not available.')
+
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+    nnz = x.nnz
+    data = _cupy.empty(nnz, x.dtype)
+    indices = _cupy.empty(nnz, 'i')
+    if nnz == 0:
+        indptr = _cupy.zeros(n + 1, 'i')
+    else:
+        indptr = _cupy.empty(n + 1, 'i')
+        x_dtype = _dtype.to_cuda_dtype(x.dtype)
+        action = _cusparse.CUSPARSE_ACTION_NUMERIC
+        ibase = _cusparse.CUSPARSE_INDEX_BASE_ZERO
+        algo = _cusparse.CUSPARSE_CSR2CSC_ALG1
+        buffer_size = _cusparse.csr2cscEx2_bufferSize(
+            handle, m, n, nnz, x.data.data.ptr, x.indptr.data.ptr,
+            x.indices.data.ptr, data.data.ptr, indptr.data.ptr,
+            indices.data.ptr, x_dtype, action, ibase, algo)
+        buffer = _cupy.empty(buffer_size, _numpy.int8)
+        _cusparse.csr2cscEx2(
+            handle, m, n, nnz, x.data.data.ptr, x.indptr.data.ptr,
+            x.indices.data.ptr, data.data.ptr, indptr.data.ptr,
+            indices.data.ptr, x_dtype, action, ibase, algo, buffer.data.ptr)
+    return cupyx.scipy.sparse.csc_matrix(
+        (data, indices, indptr), shape=x.shape)
+
+
+def csc2coo(x, data, indices):
+    """Converts a CSC-matrix to COO format.
+
+    Args:
+        x (cupyx.scipy.sparse.csc_matrix): A matrix to be converted.
+        data (cupy.ndarray): A data array for converted data.
+        indices (cupy.ndarray): An index array for converted data.
+
+    Returns:
+        cupyx.scipy.sparse.coo_matrix: A converted matrix.
+
+    """
+    handle = _device.get_cusparse_handle()
+    n = x.shape[1]
+    nnz = x.nnz
+    col = _cupy.empty(nnz, 'i')
+    _cusparse.xcsr2coo(
+        handle, x.indptr.data.ptr, nnz, n, col.data.ptr,
+        _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    # data and indices did not need to be copied already
+    return cupyx.scipy.sparse.coo_matrix(
+        (data, (indices, col)), shape=x.shape)
+
+
+def csc2csr(x):
+    if not check_availability('csc2csr'):
+        raise RuntimeError('csr2csc is not available.')
+
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+    nnz = x.nnz
+    data = _cupy.empty(nnz, x.dtype)
+    indices = _cupy.empty(nnz, 'i')
+    if nnz == 0:
+        indptr = _cupy.zeros(m + 1, 'i')
+    else:
+        indptr = _cupy.empty(m + 1, 'i')
+        _call_cusparse(
+            'csr2csc', x.dtype,
+            handle, n, m, nnz, x.data.data.ptr,
+            x.indptr.data.ptr, x.indices.data.ptr,
+            data.data.ptr, indices.data.ptr, indptr.data.ptr,
+            _cusparse.CUSPARSE_ACTION_NUMERIC,
+            _cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    return cupyx.scipy.sparse.csr_matrix(
+        (data, indices, indptr), shape=x.shape)
+
+
+def csc2csrEx2(x):
+    if not check_availability('csc2csrEx2'):
+        raise RuntimeError('csc2csrEx2 is not available.')
+
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+    nnz = x.nnz
+    data = _cupy.empty(nnz, x.dtype)
+    indices = _cupy.empty(nnz, 'i')
+    if nnz == 0:
+        indptr = _cupy.zeros(m + 1, 'i')
+    else:
+        indptr = _cupy.empty(m + 1, 'i')
+        x_dtype = _dtype.to_cuda_dtype(x.dtype)
+        action = _cusparse.CUSPARSE_ACTION_NUMERIC
+        ibase = _cusparse.CUSPARSE_INDEX_BASE_ZERO
+        algo = _cusparse.CUSPARSE_CSR2CSC_ALG1
+        buffer_size = _cusparse.csr2cscEx2_bufferSize(
+            handle, n, m, nnz, x.data.data.ptr, x.indptr.data.ptr,
+            x.indices.data.ptr, data.data.ptr, indptr.data.ptr,
+            indices.data.ptr, x_dtype, action, ibase, algo)
+        buffer = _cupy.empty(buffer_size, _numpy.int8)
+        _cusparse.csr2cscEx2(
+            handle, n, m, nnz, x.data.data.ptr, x.indptr.data.ptr,
+            x.indices.data.ptr, data.data.ptr, indptr.data.ptr,
+            indices.data.ptr, x_dtype, action, ibase, algo, buffer.data.ptr)
+    return cupyx.scipy.sparse.csr_matrix(
+        (data, indices, indptr), shape=x.shape)
+
+
+def dense2csc(x):
+    """Converts a dense matrix in CSC format.
+
+    Args:
+        x (cupy.ndarray): A matrix to be converted.
+
+    Returns:
+        cupyx.scipy.sparse.csc_matrix: A converted matrix.
+
+    """
+    if not check_availability('dense2csc'):
+        raise RuntimeError('dense2csc is not available.')
+
+    assert x.ndim == 2
+    x = _cupy.asfortranarray(x)
+    nnz = _numpy.empty((), dtype='i')
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+
+    descr = MatDescriptor.create()
+    nnz_per_col = _cupy.empty(m, 'i')
+    _call_cusparse(
+        'nnz', x.dtype,
+        handle, _cusparse.CUSPARSE_DIRECTION_COLUMN, m, n, descr.descriptor,
+        x.data.ptr, m, nnz_per_col.data.ptr, nnz.ctypes.data)
+
+    nnz = int(nnz)
+    data = _cupy.empty(nnz, x.dtype)
+    indptr = _cupy.empty(n + 1, 'i')
+    indices = _cupy.empty(nnz, 'i')
+
+    _call_cusparse(
+        'dense2csc', x.dtype,
+        handle, m, n, descr.descriptor,
+        x.data.ptr, m, nnz_per_col.data.ptr,
+        data.data.ptr, indices.data.ptr, indptr.data.ptr)
+    # Note that a desciptor is recreated
+    csc = cupyx.scipy.sparse.csc_matrix((data, indices, indptr), shape=x.shape)
+    csc._has_canonical_format = True
+    return csc
+
+
+def dense2csr(x):
+    """Converts a dense matrix in CSR format.
+
+    Args:
+        x (cupy.ndarray): A matrix to be converted.
+
+    Returns:
+        cupyx.scipy.sparse.csr_matrix: A converted matrix.
+
+    """
+    if not check_availability('dense2csr'):
+        raise RuntimeError('dense2csr is not available.')
+
+    assert x.ndim == 2
+    x = _cupy.asfortranarray(x)
+    nnz = _numpy.empty((), dtype='i')
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+
+    descr = MatDescriptor.create()
+    nnz_per_row = _cupy.empty(m, 'i')
+    _call_cusparse(
+        'nnz', x.dtype,
+        handle, _cusparse.CUSPARSE_DIRECTION_ROW, m, n, descr.descriptor,
+        x.data.ptr, m, nnz_per_row.data.ptr, nnz.ctypes.data)
+
+    nnz = int(nnz)
+    if _runtime.is_hip:
+        if nnz == 0:
+            raise ValueError('hipSPARSE currently cannot handle '
+                             'sparse matrices with null ptrs')
+    data = _cupy.empty(nnz, x.dtype)
+    indptr = _cupy.empty(m + 1, 'i')
+    indices = _cupy.empty(nnz, 'i')
+
+    _call_cusparse(
+        'dense2csr', x.dtype,
+        handle, m, n, descr.descriptor,
+        x.data.ptr, m, nnz_per_row.data.ptr,
+        data.data.ptr, indptr.data.ptr, indices.data.ptr)
+    # Note that a desciptor is recreated
+    csr = cupyx.scipy.sparse.csr_matrix((data, indices, indptr), shape=x.shape)
+    csr._has_canonical_format = True
+    return csr
+
+
+def csr2csr_compress(x, tol):
+    if not check_availability('csr2csr_compress'):
+        raise RuntimeError('csr2csr_compress is not available.')
+
+    assert x.dtype.char in 'fdFD'
+
+    handle = _device.get_cusparse_handle()
+    m, n = x.shape
+
+    nnz_per_row = _cupy.empty(m, 'i')
+    nnz = _call_cusparse(
+        'nnz_compress', x.dtype,
+        handle, m, x._descr.descriptor,
+        x.data.data.ptr, x.indptr.data.ptr, nnz_per_row.data.ptr, tol)
+    data = _cupy.zeros(nnz, x.dtype)
+    indptr = _cupy.empty(m + 1, 'i')
+    indices = _cupy.zeros(nnz, 'i')
+    _call_cusparse(
+        'csr2csr_compress', x.dtype,
+        handle, m, n, x._descr.descriptor,
+        x.data.data.ptr, x.indices.data.ptr, x.indptr.data.ptr,
+        x.nnz, nnz_per_row.data.ptr, data.data.ptr, indices.data.ptr,
+        indptr.data.ptr, tol)
+
+    return cupyx.scipy.sparse.csr_matrix(
+        (data, indices, indptr), shape=x.shape)
+
+
+def _dtype_to_IndexType(dtype):
+    if dtype == 'uint16':
+        return _cusparse.CUSPARSE_INDEX_16U
+    elif dtype == 'int32':
+        return _cusparse.CUSPARSE_INDEX_32I
+    elif dtype == 'int64':
+        return _cusparse.CUSPARSE_INDEX_64I
+    else:
+        raise TypeError
+
+
+class BaseDescriptor(object):
+
+    def __init__(self, descriptor, get=None, destroyer=None):
+        self.desc = descriptor
+        self.get = get
+        self.destroy = destroyer
+
+    def __del__(self, is_shutting_down=_util.is_shutting_down):
+        if is_shutting_down():
+            return
+        if self.destroy is None:
+            self.desc = None
+        elif self.desc is not None:
+            self.destroy(self.desc)
+            self.desc = None
+
+    def __getattr__(self, name):
+        if self.get is not None:
+            return getattr(self.get(self.desc), name)
+        raise AttributeError
+
+
+class SpMatDescriptor(BaseDescriptor):
+
+    @classmethod
+    def create(cls, a):
+        assert cupyx.scipy.sparse.issparse(a)
+        rows, cols = a.shape
+        idx_base = _cusparse.CUSPARSE_INDEX_BASE_ZERO
+        cuda_dtype = _dtype.to_cuda_dtype(a.dtype)
+        if a.format == 'csr':
+            desc = _cusparse.createCsr(
+                rows, cols, a.nnz, a.indptr.data.ptr, a.indices.data.ptr,
+                a.data.data.ptr, _dtype_to_IndexType(a.indptr.dtype),
+                _dtype_to_IndexType(a.indices.dtype), idx_base, cuda_dtype)
+            get = _cusparse.csrGet
+        elif a.format == 'coo':
+            desc = _cusparse.createCoo(
+                rows, cols, a.nnz, a.row.data.ptr, a.col.data.ptr,
+                a.data.data.ptr, _dtype_to_IndexType(a.row.dtype),
+                idx_base, cuda_dtype)
+            get = _cusparse.cooGet
+        elif a.format == 'csc':
+            desc = _cusparse.createCsc(
+                rows, cols, a.nnz, a.indptr.data.ptr, a.indices.data.ptr,
+                a.data.data.ptr, _dtype_to_IndexType(a.indptr.dtype),
+                _dtype_to_IndexType(a.indices.dtype), idx_base, cuda_dtype)
+            get = None
+        else:
+            raise ValueError('csr, csc and coo format are supported '
+                             '(actual: {}).'.format(a.format))
+        destroy = _cusparse.destroySpMat
+        return SpMatDescriptor(desc, get, destroy)
+
+    def set_attribute(self, attribute, data):
+        _cusparse.spMatSetAttribute(self.desc, attribute, data)
+
+
+class SpVecDescriptor(BaseDescriptor):
+
+    @classmethod
+    def create(cls, idx, x):
+        nnz = x.size
+        cuda_dtype = _dtype.to_cuda_dtype(x.dtype)
+        desc = _cusparse.createSpVec(nnz, nnz, idx.data.ptr, x.data.ptr,
+                                     _dtype_to_IndexType(idx.dtype),
+                                     _cusparse.CUSPARSE_INDEX_BASE_ZERO,
+                                     cuda_dtype)
+        get = _cusparse.spVecGet
+        destroy = _cusparse.destroySpVec
+        return SpVecDescriptor(desc, get, destroy)
+
+
+class DnVecDescriptor(BaseDescriptor):
+
+    @classmethod
+    def create(cls, x):
+        cuda_dtype = _dtype.to_cuda_dtype(x.dtype)
+        desc = _cusparse.createDnVec(x.size, x.data.ptr, cuda_dtype)
+        get = _cusparse.dnVecGet
+        destroy = _cusparse.destroyDnVec
+        return DnVecDescriptor(desc, get, destroy)
+
+
+class DnMatDescriptor(BaseDescriptor):
+
+    @classmethod
+    def create(cls, a):
+        assert a.ndim == 2
+        assert a.flags.f_contiguous
+        rows, cols = a.shape
+        ld = rows
+        cuda_dtype = _dtype.to_cuda_dtype(a.dtype)
+        desc = _cusparse.createDnMat(rows, cols, ld, a.data.ptr, cuda_dtype,
+                                     _cusparse.CUSPARSE_ORDER_COL)
+        get = _cusparse.dnMatGet
+        destroy = _cusparse.destroyDnMat
+        return DnMatDescriptor(desc, get, destroy)
+
+
+def spmv(a, x, y=None, alpha=1, beta=0, transa=False):
+    """Multiplication of sparse matrix and dense vector.
+
+    .. math::
+
+        y = \\alpha * op(A) x + \\beta * y
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix, csc_matrix or coo_matrix):
+            Sparse matrix A
+        x (cupy.ndarray): Dense vector x
+        y (cupy.ndarray or None): Dense vector y
+        alpha (scalar): Coefficent
+        beta (scalar): Coefficent
+        transa (bool): If ``True``, op(A) = transpose of A.
+
+    Returns:
+        cupy.ndarray
+    """
+    if not check_availability('spmv'):
+        raise RuntimeError('spmv is not available.')
+
+    if isinstance(a, cupyx.scipy.sparse.csc_matrix):
+        aT = a.T
+        if not isinstance(aT, cupyx.scipy.sparse.csr_matrix):
+            msg = 'aT must be csr_matrix (actual: {})'.format(type(aT))
+            raise TypeError(msg)
+        a = aT
+        transa = not transa
+    if not isinstance(a, (cupyx.scipy.sparse.csr_matrix,
+                          cupyx.scipy.sparse.coo_matrix)):
+        raise TypeError('unsupported type (actual: {})'.format(type(a)))
+    a_shape = a.shape if not transa else a.shape[::-1]
+    if a_shape[1] != len(x):
+        raise ValueError('dimension mismatch')
+    assert a.has_canonical_format
+
+    m, n = a_shape
+    a, x, y = _cast_common_type(a, x, y)
+    if y is None:
+        y = _cupy.zeros(m, a.dtype)
+    elif len(y) != m:
+        raise ValueError('dimension mismatch')
+    if a.nnz == 0:
+        y.fill(0)
+        return y
+
+    desc_a = SpMatDescriptor.create(a)
+    desc_x = DnVecDescriptor.create(x)
+    desc_y = DnVecDescriptor.create(y)
+
+    handle = _device.get_cusparse_handle()
+    op_a = _transpose_flag(transa)
+    alpha = _numpy.array(alpha, a.dtype).ctypes
+    beta = _numpy.array(beta, a.dtype).ctypes
+    cuda_dtype = _dtype.to_cuda_dtype(a.dtype)
+    alg = _cusparse.CUSPARSE_MV_ALG_DEFAULT
+    buff_size = _cusparse.spMV_bufferSize(handle, op_a, alpha.data,
+                                          desc_a.desc, desc_x.desc, beta.data,
+                                          desc_y.desc, cuda_dtype, alg)
+    buff = _cupy.empty(buff_size, _cupy.int8)
+    _cusparse.spMV(handle, op_a, alpha.data, desc_a.desc, desc_x.desc,
+                   beta.data, desc_y.desc, cuda_dtype, alg, buff.data.ptr)
+
+    return y
+
+
+def spmm(a, b, c=None, alpha=1, beta=0, transa=False, transb=False):
+    """Multiplication of sparse matrix and dense matrix.
+
+    .. math::
+
+        C = \\alpha * op(A) op(B) + \\beta * C
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix, csc_matrix or coo_matrix):
+            Sparse matrix A
+        b (cupy.ndarray): Dense matrix B
+        c (cupy.ndarray or None): Dense matrix C
+        alpha (scalar): Coefficent
+        beta (scalar): Coefficent
+        transa (bool): If ``True``, op(A) = transpose of A.
+        transb (bool): If ``True``, op(B) = transpose of B.
+
+    Returns:
+        cupy.ndarray
+    """
+    if not check_availability('spmm'):
+        raise RuntimeError('spmm is not available.')
+
+    assert a.ndim == b.ndim == 2
+    assert b.flags.f_contiguous
+    assert c is None or c.flags.f_contiguous
+
+    if isinstance(a, cupyx.scipy.sparse.csc_matrix):
+        aT = a.T
+        if not isinstance(aT, cupyx.scipy.sparse.csr_matrix):
+            msg = 'aT must be csr_matrix (actual: {})'.format(type(aT))
+            raise TypeError(msg)
+        a = aT
+        transa = not transa
+    if not isinstance(a, (cupyx.scipy.sparse.csr_matrix,
+                          cupyx.scipy.sparse.coo_matrix)):
+        raise TypeError('unsupported type (actual: {})'.format(type(a)))
+    a_shape = a.shape if not transa else a.shape[::-1]
+    b_shape = b.shape if not transb else b.shape[::-1]
+    if a_shape[1] != b_shape[0]:
+        raise ValueError('dimension mismatch')
+    assert a.has_canonical_format
+
+    m, k = a_shape
+    _, n = b_shape
+    a, b, c = _cast_common_type(a, b, c)
+    if c is None:
+        c = _cupy.zeros((m, n), a.dtype, 'F')
+    elif c.shape[0] != m or c.shape[1] != n:
+        raise ValueError('dimension mismatch')
+    if a.nnz == 0:
+        c.fill(0)
+        return c
+
+    desc_a = SpMatDescriptor.create(a)
+    desc_b = DnMatDescriptor.create(b)
+    desc_c = DnMatDescriptor.create(c)
+
+    handle = _device.get_cusparse_handle()
+    op_a = _transpose_flag(transa)
+    op_b = _transpose_flag(transb)
+    alpha = _numpy.array(alpha, a.dtype).ctypes
+    beta = _numpy.array(beta, a.dtype).ctypes
+    cuda_dtype = _dtype.to_cuda_dtype(a.dtype)
+    alg = _cusparse.CUSPARSE_MM_ALG_DEFAULT
+    buff_size = _cusparse.spMM_bufferSize(handle, op_a, op_b, alpha.data,
+                                          desc_a.desc, desc_b.desc, beta.data,
+                                          desc_c.desc, cuda_dtype, alg)
+    buff = _cupy.empty(buff_size, _cupy.int8)
+    buff_size = _cusparse.spMM(handle, op_a, op_b, alpha.data, desc_a.desc,
+                               desc_b.desc, beta.data, desc_c.desc,
+                               cuda_dtype, alg, buff.data.ptr)
+
+    return c
+
+
+def csrsm2(a, b, alpha=1.0, lower=True, unit_diag=False, transa=False,
+           blocking=True, level_info=False):
+    """Solves a sparse triangular linear system op(a) * x = alpha * b.
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix or cupyx.scipy.sparse.csc_matrix):
+            Sparse matrix with dimension ``(M, M)``.
+        b (cupy.ndarray): Dense vector or matrix with dimension ``(M)`` or
+            ``(M, K)``.
+        alpha (float or complex): Coefficent.
+        lower (bool):
+            True: ``a`` is lower triangle matrix.
+            False: ``a`` is upper triangle matrix.
+        unit_diag (bool):
+            True: diagonal part of ``a`` has unit elements.
+            False: diagonal part of ``a`` has non-unit elements.
+        transa (bool or str): True, False, 'N', 'T' or 'H'.
+            'N' or False: op(a) == ``a``.
+            'T' or True: op(a) == ``a.T``.
+            'H': op(a) == ``a.conj().T``.
+        blocking (bool):
+            True: blocking algorithm is used.
+            False: non-blocking algorithm is used.
+        level_info (bool):
+            True: solves it with level infromation.
+            False: solves it without level information.
+
+    Note: ``b`` will be overwritten.
+    """
+    if not check_availability('csrsm2'):
+        raise RuntimeError('csrsm2 is not available.')
+
+    if not (cupyx.scipy.sparse.isspmatrix_csr(a) or
+            cupyx.scipy.sparse.isspmatrix_csc(a)):
+        raise ValueError('a must be CSR or CSC sparse matrix')
+    if not isinstance(b, _cupy.ndarray):
+        raise ValueError('b must be cupy.ndarray')
+    if b.ndim not in (1, 2):
+        raise ValueError('b.ndim must be 1 or 2')
+    if not (a.shape[0] == a.shape[1] == b.shape[0]):
+        raise ValueError('invalid shape')
+    if a.dtype != b.dtype:
+        raise TypeError('dtype mismatch')
+
+    if lower is True:
+        fill_mode = _cusparse.CUSPARSE_FILL_MODE_LOWER
+    elif lower is False:
+        fill_mode = _cusparse.CUSPARSE_FILL_MODE_UPPER
+    else:
+        raise ValueError('Unknown lower (actual: {})'.format(lower))
+
+    if unit_diag is False:
+        diag_type = _cusparse.CUSPARSE_DIAG_TYPE_NON_UNIT
+    elif unit_diag is True:
+        diag_type = _cusparse.CUSPARSE_DIAG_TYPE_UNIT
+    else:
+        raise ValueError('Unknown unit_diag (actual: {})'.format(unit_diag))
+
+    if blocking is False:
+        algo = 0
+    elif blocking is True:
+        algo = 1
+    else:
+        raise ValueError('Unknown blocking (actual: {})'.format(blocking))
+
+    if level_info is False:
+        policy = _cusparse.CUSPARSE_SOLVE_POLICY_NO_LEVEL
+    elif level_info is True:
+        policy = _cusparse.CUSPARSE_SOLVE_POLICY_USE_LEVEL
+    else:
+        raise ValueError('Unknown level_info (actual: {})'.format(level_info))
+
+    dtype = a.dtype
+    if dtype.char == 'f':
+        t = 's'
+    elif dtype.char == 'd':
+        t = 'd'
+    elif dtype.char == 'F':
+        t = 'c'
+    elif dtype.char == 'D':
+        t = 'z'
+    else:
+        raise TypeError('Invalid dtype (actual: {})'.format(dtype))
+    helper = getattr(_cusparse, t + 'csrsm2_bufferSizeExt')
+    analysis = getattr(_cusparse, t + 'csrsm2_analysis')
+    solve = getattr(_cusparse, t + 'csrsm2_solve')
+
+    if transa is False or transa == 'N':
+        transa = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+    elif transa is True or transa == 'T':
+        transa = _cusparse.CUSPARSE_OPERATION_TRANSPOSE
+    elif transa == 'H':
+        if dtype.char in 'fd':
+            transa = _cusparse.CUSPARSE_OPERATION_TRANSPOSE
+        else:
+            transa = _cusparse.CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE
+    else:
+        raise ValueError('Unknown transa (actual: {})'.format(transa))
+
+    if cupyx.scipy.sparse.isspmatrix_csc(a):
+        if transa == _cusparse.CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE:
+            raise ValueError('If matrix is CSC format and complex dtype,'
+                             'transa must not be \'H\'')
+        a = a.T
+        assert cupyx.scipy.sparse.isspmatrix_csr(a)
+        transa = 1 - transa
+        fill_mode = 1 - fill_mode
+
+    m = a.shape[0]
+    nrhs = 1 if b.ndim == 1 else b.shape[1]
+    if b._f_contiguous:
+        transb = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+        ldb = m
+    elif b._c_contiguous:
+        transb = _cusparse.CUSPARSE_OPERATION_TRANSPOSE
+        ldb = nrhs
+    else:
+        raise ValueError('b must be F-contiguous or C-contiguous.')
+
+    handle = _device.get_cusparse_handle()
+    alpha = _numpy.array(alpha, dtype=dtype)
+    a_desc = MatDescriptor.create()
+    a_desc.set_mat_type(_cusparse.CUSPARSE_MATRIX_TYPE_GENERAL)
+    a_desc.set_mat_index_base(_cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    a_desc.set_mat_fill_mode(fill_mode)
+    a_desc.set_mat_diag_type(diag_type)
+    info = _cusparse.createCsrsm2Info()
+    ws_size = helper(handle, algo, transa, transb, m, nrhs, a.nnz,
+                     alpha.ctypes.data, a_desc.descriptor, a.data.data.ptr,
+                     a.indptr.data.ptr, a.indices.data.ptr, b.data.ptr, ldb,
+                     info, policy)
+    ws = _cupy.empty((ws_size,), dtype=_numpy.int8)
+
+    analysis(handle, algo, transa, transb, m, nrhs, a.nnz, alpha.ctypes.data,
+             a_desc.descriptor, a.data.data.ptr, a.indptr.data.ptr,
+             a.indices.data.ptr, b.data.ptr, ldb, info, policy, ws.data.ptr)
+
+    solve(handle, algo, transa, transb, m, nrhs, a.nnz, alpha.ctypes.data,
+          a_desc.descriptor, a.data.data.ptr, a.indptr.data.ptr,
+          a.indices.data.ptr, b.data.ptr, ldb, info, policy, ws.data.ptr)
+
+    # without sync we'd get either segfault or cuda context error
+    _stream.get_current_stream().synchronize()
+    _cusparse.destroyCsrsm2Info(info)
+
+
+def csrilu02(a, level_info=False):
+    """Computes incomplete LU decomposition for a sparse square matrix.
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix):
+            Sparse matrix with dimension ``(M, M)``.
+        level_info (bool):
+            True: solves it with level infromation.
+            False: solves it without level information.
+
+    Note: ``a`` will be overwritten. This function does not support fill-in
+        (only ILU(0) is supported) nor pivoting.
+    """
+    if not check_availability('csrilu02'):
+        raise RuntimeError('csrilu02 is not available.')
+
+    if not cupyx.scipy.sparse.isspmatrix_csr(a):
+        raise TypeError('a must be CSR sparse matrix')
+    if a.shape[0] != a.shape[1]:
+        raise ValueError('invalid shape (a.shape: {})'.format(a.shape))
+
+    if level_info is False:
+        policy = _cusparse.CUSPARSE_SOLVE_POLICY_NO_LEVEL
+    elif level_info is True:
+        policy = _cusparse.CUSPARSE_SOLVE_POLICY_USE_LEVEL
+    else:
+        raise ValueError('Unknown level_info (actual: {})'.format(level_info))
+
+    dtype = a.dtype
+    if dtype.char == 'f':
+        t = 's'
+    elif dtype.char == 'd':
+        t = 'd'
+    elif dtype.char == 'F':
+        t = 'c'
+    elif dtype.char == 'D':
+        t = 'z'
+    else:
+        raise TypeError('Invalid dtype (actual: {})'.format(dtype))
+    helper = getattr(_cusparse, t + 'csrilu02_bufferSize')
+    analysis = getattr(_cusparse, t + 'csrilu02_analysis')
+    solve = getattr(_cusparse, t + 'csrilu02')
+    check = getattr(_cusparse, 'xcsrilu02_zeroPivot')
+
+    handle = _device.get_cusparse_handle()
+    m = a.shape[0]
+    nnz = a.nnz
+    desc = MatDescriptor.create()
+    desc.set_mat_type(_cusparse.CUSPARSE_MATRIX_TYPE_GENERAL)
+    desc.set_mat_index_base(_cusparse.CUSPARSE_INDEX_BASE_ZERO)
+    info = _cusparse.createCsrilu02Info()
+    ws_size = helper(handle, m, nnz, desc.descriptor, a.data.data.ptr,
+                     a.indptr.data.ptr, a.indices.data.ptr, info)
+    ws = _cupy.empty((ws_size,), dtype=_numpy.int8)
+    position = _numpy.empty((1,), dtype=_numpy.int32)
+
+    analysis(handle, m, nnz, desc.descriptor, a.data.data.ptr,
+             a.indptr.data.ptr, a.indices.data.ptr, info, policy, ws.data.ptr)
+    try:
+        check(handle, info, position.ctypes.data)
+    except Exception:
+        raise ValueError('a({0},{0}) is missing'.format(position[0]))
+
+    solve(handle, m, nnz, desc.descriptor, a.data.data.ptr,
+          a.indptr.data.ptr, a.indices.data.ptr, info, policy, ws.data.ptr)
+    try:
+        check(handle, info, position.ctypes.data)
+    except Exception:
+        raise ValueError('u({0},{0}) is zero'.format(position[0]))
+
+
+def denseToSparse(x, format='csr'):
+    """Converts a dense matrix into a CSR, CSC or COO format.
+
+    Args:
+        x (cupy.ndarray): A matrix to be converted.
+        format (str): Format of converted matrix. It must be either 'csr',
+            'csc' or 'coo'.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: A converted sparse matrix.
+
+    """
+    if not check_availability('denseToSparse'):
+        raise RuntimeError('denseToSparse is not available.')
+
+    assert x.ndim == 2
+    assert x.dtype.char in 'fdFD'
+    x = _cupy.asfortranarray(x)
+    desc_x = DnMatDescriptor.create(x)
+    if format == 'csr':
+        y = cupyx.scipy.sparse.csr_matrix(x.shape, dtype=x.dtype)
+    elif format == 'csc':
+        y = cupyx.scipy.sparse.csc_matrix(x.shape, dtype=x.dtype)
+    elif format == 'coo':
+        y = cupyx.scipy.sparse.coo_matrix(x.shape, dtype=x.dtype)
+    else:
+        raise TypeError('unsupported format (actual: {})'.format(format))
+    desc_y = SpMatDescriptor.create(y)
+    algo = _cusparse.CUSPARSE_DENSETOSPARSE_ALG_DEFAULT
+    handle = _device.get_cusparse_handle()
+    buff_size = _cusparse.denseToSparse_bufferSize(handle, desc_x.desc,
+                                                   desc_y.desc, algo)
+    buff = _cupy.empty(buff_size, _cupy.int8)
+    _cusparse.denseToSparse_analysis(handle, desc_x.desc,
+                                     desc_y.desc, algo, buff.data.ptr)
+    num_rows_tmp = _numpy.array(0, dtype='int64')
+    num_cols_tmp = _numpy.array(0, dtype='int64')
+    nnz = _numpy.array(0, dtype='int64')
+    _cusparse.spMatGetSize(desc_y.desc, num_rows_tmp.ctypes.data,
+                           num_cols_tmp.ctypes.data, nnz.ctypes.data)
+    nnz = int(nnz)
+    if _runtime.is_hip:
+        if nnz == 0:
+            raise ValueError('hipSPARSE currently cannot handle '
+                             'sparse matrices with null ptrs')
+    if format == 'csr':
+        indptr = y.indptr
+        indices = _cupy.empty(nnz, 'i')
+        data = _cupy.empty(nnz, x.dtype)
+        y = cupyx.scipy.sparse.csr_matrix((data, indices, indptr),
+                                          shape=x.shape)
+    elif format == 'csc':
+        indptr = y.indptr
+        indices = _cupy.empty(nnz, 'i')
+        data = _cupy.empty(nnz, x.dtype)
+        y = cupyx.scipy.sparse.csc_matrix((data, indices, indptr),
+                                          shape=x.shape)
+    elif format == 'coo':
+        row = _cupy.zeros(nnz, 'i')
+        col = _cupy.zeros(nnz, 'i')
+        # Note: I would like to use empty() here, but that might cause an
+        # exeption in the row/col number check when creating the coo_matrix,
+        # so I used zeros() instead.
+        data = _cupy.empty(nnz, x.dtype)
+        y = cupyx.scipy.sparse.coo_matrix((data, (row, col)), shape=x.shape)
+    desc_y = SpMatDescriptor.create(y)
+    _cusparse.denseToSparse_convert(handle, desc_x.desc,
+                                    desc_y.desc, algo, buff.data.ptr)
+    y._has_canonical_format = True
+    return y
+
+
+def sparseToDense(x, out=None):
+    """Converts sparse matrix to a dense matrix.
+
+    Args:
+        x (cupyx.scipy.sparse.spmatrix): A sparse matrix to convert.
+        out (cupy.ndarray or None): A dense metrix to store the result.
+            It must be F-contiguous.
+
+    Returns:
+        cupy.ndarray: A converted dense matrix.
+
+    """
+    if not check_availability('sparseToDense'):
+        raise RuntimeError('sparseToDense is not available.')
+
+    dtype = x.dtype
+    assert dtype.char in 'fdFD'
+    if out is None:
+        out = _cupy.zeros(x.shape, dtype=dtype, order='F')
+    else:
+        assert out.flags.f_contiguous
+        assert out.dtype == dtype
+
+    desc_x = SpMatDescriptor.create(x)
+    desc_out = DnMatDescriptor.create(out)
+    algo = _cusparse.CUSPARSE_SPARSETODENSE_ALG_DEFAULT
+    handle = _device.get_cusparse_handle()
+    buff_size = _cusparse.sparseToDense_bufferSize(handle, desc_x.desc,
+                                                   desc_out.desc, algo)
+    buff = _cupy.empty(buff_size, _cupy.int8)
+    if _runtime.is_hip:
+        if x.nnz == 0:
+            raise ValueError('hipSPARSE currently cannot handle '
+                             'sparse matrices with null ptrs')
+    _cusparse.sparseToDense(handle, desc_x.desc,
+                            desc_out.desc, algo, buff.data.ptr)
+
+    return out
+
+
+def spsm(a, b, alpha=1.0, lower=True, unit_diag=False, transa=False):
+    """Solves a sparse triangular linear system op(a) * x = alpha * op(b).
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix or cupyx.scipy.sparse.coo_matrix):
+            Sparse matrix with dimension ``(M, M)``.
+        b (cupy.ndarray): Dense matrix with dimension ``(M, K)``.
+        alpha (float or complex): Coefficient.
+        lower (bool):
+            True: ``a`` is lower triangle matrix.
+            False: ``a`` is upper triangle matrix.
+        unit_diag (bool):
+            True: diagonal part of ``a`` has unit elements.
+            False: diagonal part of ``a`` has non-unit elements.
+        transa (bool or str): True, False, 'N', 'T' or 'H'.
+            'N' or False: op(a) == ``a``.
+            'T' or True: op(a) == ``a.T``.
+            'H': op(a) == ``a.conj().T``.
+    """
+    if not check_availability('spsm'):
+        raise RuntimeError('spsm is not available.')
+
+    # Canonicalise transa
+    if transa is False:
+        transa = 'N'
+    elif transa is True:
+        transa = 'T'
+    elif transa not in 'NTH':
+        raise ValueError(f'Unknown transa (actual: {transa})')
+
+    # Check A's type and sparse format
+    if cupyx.scipy.sparse.isspmatrix_csr(a):
+        pass
+    elif cupyx.scipy.sparse.isspmatrix_csc(a):
+        if transa == 'N':
+            a = a.T
+            transa = 'T'
+        elif transa == 'T':
+            a = a.T
+            transa = 'N'
+        elif transa == 'H':
+            a = a.conj().T
+            transa = 'N'
+        lower = not lower
+    elif cupyx.scipy.sparse.isspmatrix_coo(a):
+        pass
+    else:
+        raise ValueError('a must be CSR, CSC or COO sparse matrix')
+    assert a.has_canonical_format
+
+    # Check B's ndim
+    if b.ndim == 1:
+        is_b_vector = True
+        b = b.reshape(-1, 1)
+    elif b.ndim == 2:
+        is_b_vector = False
+    else:
+        raise ValueError('b.ndim must be 1 or 2')
+
+    # Check shapes
+    if not (a.shape[0] == a.shape[1] == b.shape[0]):
+        raise ValueError('mismatched shape')
+
+    # Check dtypes
+    dtype = a.dtype
+    if dtype.char not in 'fdFD':
+        raise TypeError('Invalid dtype (actual: {})'.format(dtype))
+    if dtype != b.dtype:
+        raise TypeError('dtype mismatch')
+
+    # Prepare fill mode
+    if lower is True:
+        fill_mode = _cusparse.CUSPARSE_FILL_MODE_LOWER
+    elif lower is False:
+        fill_mode = _cusparse.CUSPARSE_FILL_MODE_UPPER
+    else:
+        raise ValueError('Unknown lower (actual: {})'.format(lower))
+
+    # Prepare diag type
+    if unit_diag is False:
+        diag_type = _cusparse.CUSPARSE_DIAG_TYPE_NON_UNIT
+    elif unit_diag is True:
+        diag_type = _cusparse.CUSPARSE_DIAG_TYPE_UNIT
+    else:
+        raise ValueError('Unknown unit_diag (actual: {})'.format(unit_diag))
+
+    # Prepare op_a
+    if transa == 'N':
+        op_a = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+    elif transa == 'T':
+        op_a = _cusparse.CUSPARSE_OPERATION_TRANSPOSE
+    else:  # transa == 'H'
+        if dtype.char in 'fd':
+            op_a = _cusparse.CUSPARSE_OPERATION_TRANSPOSE
+        else:
+            op_a = _cusparse.CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE
+
+    # Prepare op_b
+    if b._f_contiguous:
+        op_b = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+    elif b._c_contiguous:
+        if _cusparse.get_build_version() < 11701:  # eariler than CUDA 11.6
+            raise ValueError('b must be F-contiguous.')
+        b = b.T
+        op_b = _cusparse.CUSPARSE_OPERATION_TRANSPOSE
+    else:
+        raise ValueError('b must be F-contiguous or C-contiguous.')
+
+    # Allocate space for matrix C. Note that it is known cusparseSpSM requires
+    # the output matrix zero initialized.
+    m, _ = a.shape
+    if op_b == _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE:
+        _, n = b.shape
+    else:
+        n, _ = b.shape
+    c_shape = m, n
+    c = _cupy.zeros(c_shape, dtype=a.dtype, order='f')
+
+    # Prepare descriptors and other parameters
+    handle = _device.get_cusparse_handle()
+    mat_a = SpMatDescriptor.create(a)
+    mat_b = DnMatDescriptor.create(b)
+    mat_c = DnMatDescriptor.create(c)
+    spsm_descr = _cusparse.spSM_createDescr()
+    alpha = _numpy.array(alpha, dtype=c.dtype).ctypes
+    cuda_dtype = _dtype.to_cuda_dtype(c.dtype)
+    algo = _cusparse.CUSPARSE_SPSM_ALG_DEFAULT
+
+    try:
+        # Specify Lower|Upper fill mode
+        mat_a.set_attribute(_cusparse.CUSPARSE_SPMAT_FILL_MODE, fill_mode)
+
+        # Specify Unit|Non-Unit diagonal type
+        mat_a.set_attribute(_cusparse.CUSPARSE_SPMAT_DIAG_TYPE, diag_type)
+
+        # Allocate the workspace needed by the succeeding phases
+        buff_size = _cusparse.spSM_bufferSize(
+            handle, op_a, op_b, alpha.data, mat_a.desc, mat_b.desc,
+            mat_c.desc, cuda_dtype, algo, spsm_descr)
+        buff = _cupy.empty(buff_size, dtype=_cupy.int8)
+
+        # Perform the analysis phase
+        _cusparse.spSM_analysis(
+            handle, op_a, op_b, alpha.data, mat_a.desc, mat_b.desc,
+            mat_c.desc, cuda_dtype, algo, spsm_descr, buff.data.ptr)
+
+        # Executes the solve phase
+        _cusparse.spSM_solve(
+            handle, op_a, op_b, alpha.data, mat_a.desc, mat_b.desc,
+            mat_c.desc, cuda_dtype, algo, spsm_descr, buff.data.ptr)
+
+        # Reshape back if B was a vector
+        if is_b_vector:
+            c = c.reshape(-1)
+
+        return c
+
+    finally:
+        # Destroy matrix/vector descriptors
+        _cusparse.spSM_destroyDescr(spsm_descr)
+
+
+def spgemm(a, b, alpha=1):
+    """Matrix-matrix product for CSR-matrix.
+
+    math::
+       C = alpha * A * B
+
+    Args:
+        a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
+        b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
+        alpha (scalar): Coefficient
+
+    Returns:
+        cupyx.scipy.sparse.csr_matrix
+
+    """
+    if not check_availability('spgemm'):
+        raise RuntimeError('spgemm is not available.')
+
+    assert a.ndim == b.ndim == 2
+    if not isinstance(a, cupyx.scipy.sparse.csr_matrix):
+        raise TypeError('unsupported type (actual: {})'.format(type(a)))
+    if not isinstance(b, cupyx.scipy.sparse.csr_matrix):
+        raise TypeError('unsupported type (actual: {})'.format(type(b)))
+    assert a.has_canonical_format
+    assert b.has_canonical_format
+    if a.shape[1] != b.shape[0]:
+        raise ValueError('mismatched shape')
+
+    m, k = a.shape
+    _, n = b.shape
+    a, b = _cast_common_type(a, b)
+    c_shape = (m, n)
+    c = cupyx.scipy.sparse.csr_matrix((c_shape), dtype=a.dtype)
+
+    handle = _device.get_cusparse_handle()
+    mat_a = SpMatDescriptor.create(a)
+    mat_b = SpMatDescriptor.create(b)
+    mat_c = SpMatDescriptor.create(c)
+    spgemm_descr = _cusparse.spGEMM_createDescr()
+    op_a = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+    op_b = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+    alpha = _numpy.array(alpha, dtype=c.dtype).ctypes
+    beta = _numpy.array(0, dtype=c.dtype).ctypes
+    cuda_dtype = _dtype.to_cuda_dtype(c.dtype)
+    algo = _cusparse.CUSPARSE_SPGEMM_DEFAULT
+    null_ptr = 0
+
+    # Analyze the matrices A and B to understand the memory requirement
+    buff1_size = _cusparse.spGEMM_workEstimation(
+        handle, op_a, op_b, alpha.data, mat_a.desc, mat_b.desc, beta.data,
+        mat_c.desc, cuda_dtype, algo, spgemm_descr, 0, null_ptr)
+    buff1 = _cupy.empty(buff1_size, _cupy.int8)
+    _cusparse.spGEMM_workEstimation(
+        handle, op_a, op_b, alpha.data, mat_a.desc, mat_b.desc, beta.data,
+        mat_c.desc, cuda_dtype, algo, spgemm_descr, buff1_size, buff1.data.ptr)
+
+    # Compute the intermediate product of A and B
+    buff2_size = _cusparse.spGEMM_compute(
+        handle, op_a, op_b, alpha.data, mat_a.desc, mat_b.desc, beta.data,
+        mat_c.desc, cuda_dtype, algo, spgemm_descr, 0, null_ptr)
+    buff2 = _cupy.empty(buff2_size, _cupy.int8)
+    _cusparse.spGEMM_compute(
+        handle, op_a, op_b, alpha.data, mat_a.desc, mat_b.desc, beta.data,
+        mat_c.desc, cuda_dtype, algo, spgemm_descr, buff2_size, buff2.data.ptr)
+
+    # Prepare the arrays for matrix C
+    c_num_rows = _numpy.array(0, dtype='int64')
+    c_num_cols = _numpy.array(0, dtype='int64')
+    c_nnz = _numpy.array(0, dtype='int64')
+    _cusparse.spMatGetSize(mat_c.desc, c_num_rows.ctypes.data,
+                           c_num_cols.ctypes.data, c_nnz.ctypes.data)
+    assert c_shape[0] == int(c_num_rows)
+    assert c_shape[1] == int(c_num_cols)
+    c_nnz = int(c_nnz)
+    c_indptr = c.indptr
+    c_indices = _cupy.empty(c_nnz, 'i')
+    c_data = _cupy.empty(c_nnz, c.dtype)
+    _cusparse.csrSetPointers(mat_c.desc, c_indptr.data.ptr, c_indices.data.ptr,
+                             c_data.data.ptr)
+
+    # Copy the final product to the matrix C
+    _cusparse.spGEMM_copy(
+        handle, op_a, op_b, alpha.data, mat_a.desc, mat_b.desc, beta.data,
+        mat_c.desc, cuda_dtype, algo, spgemm_descr)
+    c = cupyx.scipy.sparse.csr_matrix((c_data, c_indices, c_indptr),
+                                      shape=c_shape)
+
+    _cusparse.spGEMM_destroyDescr(spgemm_descr)
+    return c
diff --git a/cupyx/cutensor.pyx b/cupyx/cutensor.pyx
new file mode 100644
index 0000000..3d7e985
--- /dev/null
+++ b/cupyx/cutensor.pyx
@@ -0,0 +1,919 @@
+import numpy as _numpy
+import warnings as _warnings
+
+import cupy as _cupy
+from cupy import _util
+from cupy.cuda import device as _device
+
+cimport cython
+from libcpp cimport vector
+from libc.stdint cimport intptr_t, uint32_t, uint64_t
+from cupy._core._carray cimport shape_t
+from cupy._core.core cimport _ndarray_base
+from cupy._core cimport internal
+from cupy_backends.cuda.libs.cutensor cimport Handle
+from cupy_backends.cuda.libs.cutensor cimport TensorDescriptor
+from cupy_backends.cuda.libs.cutensor cimport ContractionDescriptor
+from cupy_backends.cuda.libs.cutensor cimport ContractionFind
+from cupy_backends.cuda.libs.cutensor cimport ContractionPlan
+
+from cupy._core cimport core
+from cupy._core cimport _dtype
+from cupy._core cimport _routines_linalg as _linalg
+from cupy._core cimport _reduction
+from cupy.cuda cimport device
+from cupy_backends.cuda.libs cimport cutensor
+
+
+cdef dict _handles = {}
+cdef dict _tensor_descriptors = {}
+cdef dict _contraction_descriptors = {}
+cdef dict _contraction_finds = {}
+cdef dict _contraction_plans = {}
+cdef dict _modes = {}
+cdef dict _scalars = {}
+cdef dict _dict_contraction = {
+    'eee': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.R_MIN_32F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.R_MIN_32F},
+    'fff': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.R_MIN_32F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.R_MIN_32F,
+            _linalg.COMPUTE_TYPE_FP16: cutensor.R_MIN_16F},
+    'ddd': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.R_MIN_64F,
+            _linalg.COMPUTE_TYPE_FP64: cutensor.R_MIN_64F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.R_MIN_32F},
+    'FFF': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.C_MIN_32F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.C_MIN_32F},
+    'DDD': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.C_MIN_64F,
+            _linalg.COMPUTE_TYPE_FP64: cutensor.C_MIN_64F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.C_MIN_32F},
+    'dDD': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.C_MIN_64F,
+            _linalg.COMPUTE_TYPE_FP64: cutensor.C_MIN_64F},
+    'DdD': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.C_MIN_64F,
+            _linalg.COMPUTE_TYPE_FP64: cutensor.C_MIN_64F},
+}
+cdef dict _dict_contraction_v10200 = {
+    'eee': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.COMPUTE_32F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.COMPUTE_32F,
+            _linalg.COMPUTE_TYPE_FP16: cutensor.COMPUTE_16F},
+    'fff': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.COMPUTE_32F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.COMPUTE_32F,
+            _linalg.COMPUTE_TYPE_TF32: cutensor.COMPUTE_TF32,
+            _linalg.COMPUTE_TYPE_FP16: cutensor.COMPUTE_16F},
+    'ddd': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.COMPUTE_64F,
+            _linalg.COMPUTE_TYPE_FP64: cutensor.COMPUTE_64F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.COMPUTE_32F},
+    'FFF': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.COMPUTE_32F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.COMPUTE_32F,
+            _linalg.COMPUTE_TYPE_TF32: cutensor.COMPUTE_TF32,
+            _linalg.COMPUTE_TYPE_FP16: cutensor.COMPUTE_16F},
+    'DDD': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.COMPUTE_64F,
+            _linalg.COMPUTE_TYPE_FP64: cutensor.COMPUTE_64F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.COMPUTE_32F},
+    'dDD': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.COMPUTE_64F,
+            _linalg.COMPUTE_TYPE_FP64: cutensor.COMPUTE_64F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.COMPUTE_32F},
+    'DdD': {_linalg.COMPUTE_TYPE_DEFAULT: cutensor.COMPUTE_64F,
+            _linalg.COMPUTE_TYPE_FP64: cutensor.COMPUTE_64F,
+            _linalg.COMPUTE_TYPE_FP32: cutensor.COMPUTE_32F},
+}
+cdef dict _dict_compute_type = {
+    'e': cutensor.R_MIN_16F,
+    'f': cutensor.R_MIN_32F,
+    'd': cutensor.R_MIN_64F,
+    'F': cutensor.C_MIN_32F,
+    'D': cutensor.C_MIN_64F,
+}
+cdef dict _dict_compute_type_v10200 = {
+    'e': cutensor.COMPUTE_16F,
+    'f': cutensor.COMPUTE_32F,
+    'd': cutensor.COMPUTE_64F,
+    'F': cutensor.COMPUTE_32F,
+    'D': cutensor.COMPUTE_64F,
+}
+cdef dict _available_compute_capability = {
+    'contraction': 60,
+    'reduction': 60,
+    'elementwise': 60,
+}
+
+
+@_util.memoize(for_each_device=True)
+def check_availability(name):
+    if name in _available_compute_capability:
+        compute_capability = int(_device.get_compute_capability())
+        if compute_capability < _available_compute_capability[name]:
+            return False
+    return True
+
+
+cdef class Mode:
+
+    cdef:
+        object _array
+        readonly int ndim
+        readonly intptr_t data
+
+    def __init__(self, mode):
+        self._array = _numpy.array(mode, dtype=_numpy.int32)
+        assert self._array.ndim == 1
+        self.ndim = self._array.size
+        self.data = self._array.ctypes.data
+
+    def __repr__(self):
+        return 'mode(' + ', '.join([str(x) for x in self._array]) + ')'
+
+    def __eq__(self, other):
+        if not isinstance(other, Mode):
+            return False
+        return (self._array == (<Mode>other)._array).all()
+
+
+cdef class _Scalar:
+
+    cdef:
+        object _array
+        readonly intptr_t ptr
+
+    def __init__(self, value, dtype):
+        self._array = _numpy.asarray(value, dtype=dtype)
+        self.ptr = self._array.ctypes.data
+
+    def __repr__(self):
+        return (
+            'scalar(' + str(self._array.item()) +
+            ', dtype=' + str(self._array.dtype) + ')')
+
+
+cdef Handle _get_handle():
+    cdef Handle handle
+    cdef int dev = device.get_device_id()
+    if dev not in _handles:
+        handle = Handle()
+        cutensor.init(handle)
+        _handles[dev] = handle
+        return handle
+    return _handles[dev]
+
+
+cdef int _get_cutensor_compute_type(numpy_dtype) except -1:
+    if cutensor.get_version() >= 10200:
+        # version 1.2.0 or later
+        dict_compute_type = _dict_compute_type_v10200
+    else:
+        dict_compute_type = _dict_compute_type
+    key = _numpy.dtype(numpy_dtype).char
+    if key not in dict_compute_type:
+        raise TypeError('Dtype {} is not supported'.format(numpy_dtype))
+    return dict_compute_type[key]
+
+
+def create_mode(*mode):
+    """Create the tensor mode from the given integers or characters.
+
+    Args:
+        mode (tuple of int/str): A tuple that holds the labels of the modes
+            of tensor A (e.g., if A_{x,y,z}, mode_A = {'x','y','z'})
+    """
+    integer_mode = []
+    for x in mode:
+        if isinstance(x, int):
+            integer_mode.append(x)
+        elif isinstance(x, str):
+            integer_mode.append(ord(x))
+        else:
+            raise TypeError('Cannot create tensor mode: {}'.format(type(x)))
+    return _create_mode_with_cache(tuple(integer_mode))
+
+
+cdef inline Mode _auto_create_mode(_ndarray_base array, mode):
+    if not isinstance(mode, Mode):
+        mode = create_mode(*mode)
+    if array.ndim != mode.ndim:
+        raise ValueError(
+            'ndim mismatch: {} != {}'.format(array.ndim, mode.ndim))
+    return mode
+
+
+cdef inline _set_compute_dtype(array_dtype, compute_dtype=None):
+    if compute_dtype is None:
+        if array_dtype == _numpy.float16:
+            compute_dtype = _numpy.float32
+        else:
+            compute_dtype = array_dtype
+    return compute_dtype
+
+
+cdef inline _Scalar _create_scalar(scale, dtype):
+    cdef _Scalar scalar
+    key = (scale, dtype)
+    if key in _scalars:
+        scalar = _scalars[key]
+    else:
+        scalar = _Scalar(scale, dtype)
+        _scalars[key] = scalar
+    return scalar
+
+
+cdef inline Mode _create_mode_with_cache(axis_or_ndim):
+    cdef Mode mode
+    if axis_or_ndim in _modes:
+        mode = _modes[axis_or_ndim]
+    else:
+        if type(axis_or_ndim) is int:
+            mode = Mode(tuple(range(axis_or_ndim)))
+        else:
+            mode = Mode(axis_or_ndim)
+        _modes[axis_or_ndim] = mode
+    return mode
+
+
+cpdef TensorDescriptor create_tensor_descriptor(
+        _ndarray_base a, int uop=cutensor.OP_IDENTITY, Handle handle=None):
+    """Create a tensor descriptor
+
+    Args:
+        a (cupy.ndarray): tensor for which a descritpor are created.
+        uop (cutensorOperator_t): unary operator that will be applied to each
+            element of the corresponding tensor in a lazy fashion (i.e., the
+            algorithm uses this tensor as its operand only once). The
+            original data of this tensor remains unchanged.
+
+    Returns:
+        (Descriptor): A instance of class Descriptor which holds a pointer to
+        tensor descriptor and its destructor.
+    """
+    if handle is None:
+        handle = _get_handle()
+    key = (handle.ptr, a.dtype, tuple(a.shape), tuple(a.strides), uop)
+    if key in _tensor_descriptors:
+        desc = _tensor_descriptors[key]
+        return desc
+    num_modes = a.ndim
+    extent = _numpy.array(a.shape, dtype=_numpy.int64)
+    stride = _numpy.array(a.strides, dtype=_numpy.int64) // a.itemsize
+    cuda_dtype = _dtype.to_cuda_dtype(a.dtype, is_half_allowed=True)
+    desc = TensorDescriptor()
+    cutensor.initTensorDescriptor(
+        handle, desc, num_modes, extent.ctypes.data, stride.ctypes.data,
+        cuda_dtype, uop)
+    _tensor_descriptors[key] = desc
+    return desc
+
+
+def elementwise_trinary(
+        alpha, _ndarray_base A, TensorDescriptor desc_A, mode_A,
+        beta, _ndarray_base B, TensorDescriptor desc_B, mode_B,
+        gamma, _ndarray_base C, TensorDescriptor desc_C, mode_C,
+        _ndarray_base out=None,
+        op_AB=cutensor.OP_ADD, op_ABC=cutensor.OP_ADD, compute_dtype=None):
+    """Element-wise tensor operation for three input tensors
+
+    This function performs a element-wise tensor operation of the form:
+
+        D_{Pi^C(i_0,i_1,...,i_nc)} =
+            op_ABC(op_AB(alpha * uop_A(A_{Pi^A(i_0,i_1,...,i_na)}),
+                         beta  * uop_B(B_{Pi^B(i_0,i_1,...,i_nb)})),
+                         gamma * uop_C(C_{Pi^C(i_0,i_1,...,i_nc)}))
+
+    See cupy/cuda/cutensor.elementwiseTrinary() for details.
+
+    Args:
+        alpha (scalar): Scaling factor for tensor A.
+        A (cupy.ndarray): Input tensor.
+        desc_A (class Descriptor): A descriptor that holds the information
+            about the data type, modes, and strides of tensor A.
+        mode_A (cutensor.Mode): A mode object created by `create_mode`.
+        beta (scalar): Scaling factor for tensor B.
+        B (cupy.ndarray): Input tensor.
+        desc_B (class Descriptor): A descriptor that holds the information
+            about the data type, modes, and strides of tensor B.
+        mode_B (cutensor.Mode): A mode object created by `create_mode`.
+        gamma (scalar): Scaling factor for tensor C.
+        C (cupy.ndarray): Input tensor.
+        desc_C (class Descriptor): A descriptor that holds the information
+            about the data type, modes, and strides of tensor C.
+        mode_C (cutensor.Mode): A mode object created by `create_mode`.
+        out (cupy.ndarray): Output tensor.
+        op_AB (cutensorOperator_t): Element-wise binary operator.
+        op_ABC (cutensorOperator_t): Element-wise binary operator.
+        compute_dtype (numpy.dtype): Compute type for the intermediate
+            computation.
+
+    Returns:
+        out (cupy.ndarray): Output tensor.
+
+    Examples:
+        See examples/cutensor/elementwise_trinary.py
+    """
+    if not (A.dtype == B.dtype == C.dtype):
+        raise ValueError(
+            'dtype mismatch: ({}, {}, {})'.format(A.dtype, B.dtype, C.dtype))
+    if not (A._c_contiguous and B._c_contiguous and C._c_contiguous):
+        raise ValueError('The inputs should be contiguous arrays.')
+
+    if out is None:
+        out = core._ndarray_init(
+            _cupy.ndarray, C._shape, dtype=C.dtype, obj=None)
+    elif C.dtype != out.dtype:
+        raise ValueError('dtype mismatch: {} != {}'.format(C.dtype, out.dtype))
+    elif not internal.vector_equal(C._shape, out._shape):
+        raise ValueError('shape mismatch: {} != {}'.format(C.shape, out.shape))
+    elif not out._c_contiguous:
+        raise ValueError('`out` should be a contiguous array.')
+
+    if compute_dtype is None:
+        compute_dtype = A.dtype
+
+    return _elementwise_trinary_impl(
+        _get_handle(),
+        _create_scalar(alpha, compute_dtype),
+        A, desc_A, _auto_create_mode(A, mode_A),
+        _create_scalar(beta, compute_dtype),
+        B, desc_B, _auto_create_mode(B, mode_B),
+        _create_scalar(gamma, compute_dtype),
+        C, desc_C, _auto_create_mode(C, mode_C),
+        out, op_AB, op_ABC,
+        _dtype.to_cuda_dtype(compute_dtype, is_half_allowed=True))
+
+
+cdef inline _ndarray_base _elementwise_trinary_impl(
+        Handle handle,
+        _Scalar alpha, _ndarray_base A, TensorDescriptor desc_A, Mode mode_A,
+        _Scalar beta, _ndarray_base B, TensorDescriptor desc_B, Mode mode_B,
+        _Scalar gamma, _ndarray_base C, TensorDescriptor desc_C, Mode mode_C,
+        _ndarray_base out, int op_AB, int op_ABC, int compute_type):
+    cutensor.elementwiseTrinary(
+        handle,
+        alpha.ptr, A.data.ptr, desc_A, mode_A.data,
+        beta.ptr, B.data.ptr, desc_B, mode_B.data,
+        gamma.ptr, C.data.ptr, desc_C, mode_C.data,
+        out.data.ptr, desc_C, mode_C.data,
+        op_AB, op_ABC, compute_type)
+    return out
+
+
+def elementwise_binary(
+        alpha, _ndarray_base A, TensorDescriptor desc_A, mode_A,
+        gamma, _ndarray_base C, TensorDescriptor desc_C, mode_C,
+        _ndarray_base out=None,
+        op_AC=cutensor.OP_ADD, compute_dtype=None):
+    """Element-wise tensor operation for two input tensors
+
+    This function performs a element-wise tensor operation of the form:
+
+        D_{Pi^C(i_0,i_1,...,i_n)} =
+            op_AC(alpha * uop_A(A_{Pi^A(i_0,i_1,...,i_n)}),
+                  gamma * uop_C(C_{Pi^C(i_0,i_1,...,i_n)}))
+
+    See elementwise_trinary() for details.
+
+    Examples:
+        See examples/cutensor/elementwise_binary.py
+    """
+    if A.dtype != C.dtype:
+        raise ValueError('dtype mismatch: {} != {}'.format(A.dtype, C.dtype))
+    if not (A._c_contiguous and C._c_contiguous):
+        raise ValueError('The inputs should be contiguous arrays.')
+
+    if out is None:
+        out = core._ndarray_init(
+            _cupy.ndarray, C._shape, dtype=C.dtype, obj=None)
+    elif C.dtype != out.dtype:
+        raise ValueError('dtype mismatch: {} != {}'.format(C.dtype, out.dtype))
+    elif not internal.vector_equal(C._shape, out._shape):
+        raise ValueError('shape mismatch: {} != {}'.format(C.shape, out.shape))
+    elif not out._c_contiguous:
+        raise ValueError('`out` should be a contiguous array.')
+
+    if compute_dtype is None:
+        compute_dtype = A.dtype
+
+    return _elementwise_binary_impl(
+        _get_handle(),
+        _create_scalar(alpha, compute_dtype),
+        A, desc_A, _auto_create_mode(A, mode_A),
+        _create_scalar(gamma, compute_dtype),
+        C, desc_C, _auto_create_mode(C, mode_C),
+        out, op_AC, _dtype.to_cuda_dtype(compute_dtype, is_half_allowed=True))
+
+
+cdef inline _ndarray_base _elementwise_binary_impl(
+        Handle handle,
+        _Scalar alpha, _ndarray_base A, TensorDescriptor desc_A, Mode mode_A,
+        _Scalar gamma, _ndarray_base C, TensorDescriptor desc_C, Mode mode_C,
+        _ndarray_base out, int op_AC, int compute_type):
+    # stride and mode of `out` and `C` must be the same.
+    cutensor.elementwiseBinary(
+        handle,
+        alpha.ptr, A.data.ptr, desc_A, mode_A.data,
+        gamma.ptr, C.data.ptr, desc_C, mode_C.data,
+        out.data.ptr, desc_C, mode_C.data,
+        op_AC, compute_type)
+    return out
+
+
+cdef inline ContractionDescriptor _create_contraction_descriptor(
+        Handle handle,
+        _ndarray_base A, TensorDescriptor desc_A, Mode mode_A,
+        _ndarray_base B, TensorDescriptor desc_B, Mode mode_B,
+        _ndarray_base C, TensorDescriptor desc_C, Mode mode_C,
+        int cutensor_compute_type):
+    """Create a contraction descriptor"""
+    cdef uint32_t alignment_req_A = cutensor.getAlignmentRequirement(
+        handle, A.data.ptr, desc_A)
+    cdef uint32_t alignment_req_B = cutensor.getAlignmentRequirement(
+        handle, B.data.ptr, desc_B)
+    cdef uint32_t alignment_req_C = cutensor.getAlignmentRequirement(
+        handle, C.data.ptr, desc_C)
+    cdef ContractionDescriptor desc
+
+    # desc & mode ptrs are valid because we keep references to them internally
+    key = (handle.ptr, cutensor_compute_type,
+           desc_A.ptr, mode_A.data, alignment_req_A,
+           desc_B.ptr, mode_B.data, alignment_req_B,
+           desc_C.ptr, mode_C.data, alignment_req_C)
+    if key in _contraction_descriptors:
+        desc = _contraction_descriptors[key]
+        return desc
+
+    desc = ContractionDescriptor()
+    cutensor.initContractionDescriptor(
+        handle,
+        desc,
+        desc_A, mode_A.data, alignment_req_A,
+        desc_B, mode_B.data, alignment_req_B,
+        desc_C, mode_C.data, alignment_req_C,
+        desc_C, mode_C.data, alignment_req_C,
+        cutensor_compute_type)
+    _contraction_descriptors[key] = desc
+    return desc
+
+
+cdef inline ContractionFind _create_contraction_find(Handle handle, int algo):
+    """Create a contraction find"""
+    cdef ContractionFind find
+
+    key = (handle.ptr, algo)
+    if key in _contraction_finds:
+        find = _contraction_finds[key]
+    else:
+        find = ContractionFind()
+        cutensor.initContractionFind(handle, find, algo)
+        _contraction_finds[key] = find
+    return find
+
+
+cdef inline ContractionPlan _create_contraction_plan(
+        Handle handle,
+        ContractionDescriptor desc, ContractionFind find, uint64_t ws_size):
+    """Create a contraction plan"""
+    cdef ContractionPlan plan
+
+    key = (handle.ptr, desc.ptr, find.ptr, ws_size)
+    if key in _contraction_plans:
+        plan = _contraction_plans[key]
+    else:
+        plan = ContractionPlan()
+        cutensor.initContractionPlan(handle, plan, desc, find, ws_size)
+        _contraction_plans[key] = plan
+    return plan
+
+
+cdef _get_contraction_compute_type(a_dtype, b_dtype, out_dtype, compute_dtype):
+    key = a_dtype.char + b_dtype.char + out_dtype.char
+    if cutensor.get_version() >= 10200:
+        # version 1.2.0 or later
+        dict_contraction = _dict_contraction_v10200
+    else:
+        dict_contraction = _dict_contraction
+    if key not in dict_contraction:
+        raise ValueError('Un-supported dtype combinations: ({}, {}, {})'.
+                         format(a_dtype, b_dtype, out_dtype))
+    compute_capability = int(device.get_compute_capability())
+    if compute_capability < 70 and 'e' in key:
+        raise ValueError('FP16 dtype is only supported on GPU with compute '
+                         'capability 7.0 or higher.')
+    if compute_dtype is None:
+        compute_type = _linalg.get_compute_type(out_dtype)
+    else:
+        compute_dtype = _numpy.dtype(compute_dtype)
+        if compute_dtype.char == 'e':
+            compute_type = _linalg.COMPUTE_TYPE_FP16
+        elif compute_dtype.char in 'fF':
+            compute_type = _linalg.COMPUTE_TYPE_FP32
+        elif compute_dtype.char in 'dD':
+            compute_type = _linalg.COMPUTE_TYPE_FP64
+        else:
+            raise ValueError('Un-supported dtype: {}'.format(compute_dtype))
+    if compute_type in dict_contraction[key]:
+        cutensor_compute_type = dict_contraction[key][compute_type]
+        if not (compute_capability < 70 and
+                cutensor_compute_type in (cutensor.R_MIN_16F,
+                                          cutensor.C_MIN_16F,
+                                          cutensor.COMPUTE_16F)):
+            return cutensor_compute_type
+    _warnings.warn('Use of compute type ({}) for the dtype combination '
+                   '({}, {}, {}) is not supported in cuTENSOR contraction on '
+                   'GPU with compute capability ({}). Default compute type '
+                   'will be used instead.'.
+                   format(_linalg.compute_type_to_str(compute_type),
+                          a_dtype, b_dtype, out_dtype, compute_capability))
+    return dict_contraction[key][_linalg.COMPUTE_TYPE_DEFAULT]
+
+
+cdef _get_scalar_dtype(out_dtype):
+    if out_dtype == _numpy.float16:
+        return _numpy.float32
+    else:
+        return out_dtype
+
+
+def contraction(
+        alpha, _ndarray_base A, TensorDescriptor desc_A, mode_A,
+        _ndarray_base B, TensorDescriptor desc_B, mode_B,
+        beta, _ndarray_base C, TensorDescriptor desc_C, mode_C,
+        compute_dtype=None,
+        int algo=cutensor.ALGO_DEFAULT,
+        int ws_pref=cutensor.WORKSPACE_RECOMMENDED):
+    """General tensor contraction
+
+    This routine computes the tensor contraction:
+
+        C = alpha * uop_A(A) * uop_B(B) + beta * uop_C(C)
+
+    See cupy/cuda/cutensor.contraction for details.
+
+    Args:
+        alpha (scalar): Scaling factor for A * B.
+        A (cupy.ndarray): Input tensor.
+        desc_A (class Descriptor): A descriptor that holds the information
+            about the data type, modes, and strides of tensor A.
+        mode_A (cutensor.Mode): A mode object created by `create_mode`.
+        B (cupy.ndarray): Input tensor.
+        desc_B (class Descriptor): A descriptor that holds the information
+            about the data type, modes, and strides of tensor B.
+        mode_B (cutensor.Mode): A mode object created by `create_mode`.
+        beta (scalar): Scaling factor for C.
+        C (cupy.ndarray): Input/output tensor.
+        desc_C (class Descriptor): A descriptor that holds the information
+            about the data type, modes, and strides of tensor C.
+        mode_C (cutensor.Mode): A mode object created by `create_mode`.
+        compute_dtype (numpy.dtype): Compute type for the intermediate
+            computation.
+        algo (cutensorAlgo_t): Allows users to select a specific algorithm.
+            ALGO_DEFAULT lets the heuristic choose the algorithm.
+            Any value >= 0 selects a specific GEMM-like algorithm and
+            deactivates the heuristic. If a specified algorithm is not
+            supported, STATUS_NOT_SUPPORTED is returned.
+        ws_pref (cutensorWorksizePreference_t): User preference for the
+            workspace of cuTensor.
+
+    Returns:
+        out (cupy.ndarray): Output tensor.
+
+    Examples:
+        See examples/cutensor/contraction.py
+    """
+    if not (A._c_contiguous and B._c_contiguous and C._c_contiguous):
+        raise ValueError('The inputs should be contiguous arrays.')
+    compute_type = _get_contraction_compute_type(A.dtype, B.dtype, C.dtype,
+                                                 compute_dtype)
+    scalar_dtype = _get_scalar_dtype(C.dtype)
+
+    return _contraction_impl(
+        _get_handle(),
+        _create_scalar(alpha, scalar_dtype),
+        A, desc_A, _auto_create_mode(A, mode_A),
+        B, desc_B, _auto_create_mode(B, mode_B),
+        _create_scalar(beta, scalar_dtype),
+        C, desc_C, _auto_create_mode(C, mode_C),
+        compute_type, algo, ws_pref)
+
+
+cdef inline _ndarray_base _contraction_impl(
+        Handle handle,
+        _Scalar alpha, _ndarray_base A, TensorDescriptor desc_A, Mode mode_A,
+        _ndarray_base B, TensorDescriptor desc_B, Mode mode_B,
+        _Scalar beta, _ndarray_base C, TensorDescriptor desc_C, Mode mode_C,
+        int cutensor_compute_type, int algo, int ws_pref):
+    cdef ContractionDescriptor desc
+    cdef ContractionFind find
+    cdef ContractionPlan plan
+    cdef uint64_t ws_size
+    cdef _ndarray_base out, ws
+
+    out = C
+
+    desc = _create_contraction_descriptor(
+        handle,
+        A, desc_A, mode_A,
+        B, desc_B, mode_B,
+        C, desc_C, mode_C,
+        cutensor_compute_type)
+
+    find = _create_contraction_find(handle, algo)
+
+    # Allocate workspace
+    ws_size = cutensor.contractionGetWorkspace(handle, desc, find, ws_pref)
+    try:
+        ws = core._ndarray_init(
+            _cupy.ndarray, shape_t(1, ws_size), dtype=_numpy.int8, obj=None)
+    except Exception:
+        _warnings.warn('cuTENSOR: failed to allocate memory of workspace '
+                       'with preference ({}) and size ({}).'
+                       ''.format(ws_pref, ws_size))
+        ws_size = cutensor.contractionGetWorkspace(
+            handle, desc, find, cutensor.WORKSPACE_MIN)
+        ws = core._ndarray_init(
+            _cupy.ndarray, shape_t(1, ws_size), dtype=_numpy.int8, obj=None)
+
+    plan = _create_contraction_plan(handle, desc, find, ws_size)
+
+    cutensor.contraction(
+        handle, plan,
+        alpha.ptr, A.data.ptr, B.data.ptr,
+        beta.ptr, C.data.ptr, out.data.ptr,
+        ws.data.ptr, ws_size)
+    return out
+
+
+def contraction_max_algos():
+    """Returns the maximum number of algorithms for cutensor()
+
+    See cupy/cuda/cutensor.contractionMaxAlgos() for details.
+    """
+    return cutensor.contractionMaxAlgos()
+
+
+def reduction(
+        alpha, _ndarray_base A, TensorDescriptor desc_A, mode_A,
+        beta, _ndarray_base C, TensorDescriptor desc_C, mode_C,
+        int reduce_op=cutensor.OP_ADD, compute_dtype=None):
+    """Tensor reduction
+
+    This routine computes the tensor reduction:
+
+        C = alpha * reduce_op(uop_A(A)) + beta * uop_C(C))
+
+    See :func:`cupy.cuda.cutensor.reduction` for details.
+
+    Args:
+        alpha (scalar): Scaling factor for A.
+        A (cupy.ndarray): Input tensor.
+        desc_A (class Descriptor): A descriptor that holds the information
+            about the data type, modes, strides and unary operator (uop_A) of
+            tensor A.
+        mode_A (cutensor.Mode): A mode object created by `create_mode`.
+        beta (scalar): Scaling factor for C.
+        C (cupy.ndarray): Input/output tensor.
+        desc_C (class Descriptor): A descriptor that holds the information
+            about the data type, modes, strides and unary operator (uop_C) of
+            tensor C.
+        mode_C (cutensor.Mode): A mode object created by `create_mode`.
+        reduce_op (cutensorOperator_t): Binary operator used to reduce A.
+        compute_dtype (numpy.dtype): Compute type for the intermediate
+            computation.
+
+    Returns:
+        out (cupy.ndarray): Output tensor.
+
+    Examples:
+        See examples/cutensor/reduction.py
+    """
+    if A.dtype != C.dtype:
+        raise ValueError('dtype mismatch: {} != {}'.format(A.dtype, C.dtype))
+    if not (A._c_contiguous and C._c_contiguous):
+        raise ValueError('The inputs should be contiguous arrays.')
+
+    compute_dtype = _set_compute_dtype(A.dtype, compute_dtype)
+
+    return _reduction_impl(
+        _get_handle(),
+        _create_scalar(alpha, compute_dtype),
+        A, desc_A, _auto_create_mode(A, mode_A),
+        _create_scalar(beta, compute_dtype),
+        C, desc_C, _auto_create_mode(C, mode_C),
+        reduce_op, _get_cutensor_compute_type(compute_dtype)
+    )
+
+
+cdef inline _ndarray_base _reduction_impl(
+        Handle handle,
+        _Scalar alpha, _ndarray_base A, TensorDescriptor desc_A, Mode mode_A,
+        _Scalar beta, _ndarray_base C, TensorDescriptor desc_C, Mode mode_C,
+        int reduce_op, int cutensor_compute_type):
+    cdef uint64_t ws_size
+    cdef _ndarray_base ws, out
+
+    out = C
+    ws_size = cutensor.reductionGetWorkspace(
+        handle,
+        A.data.ptr, desc_A, mode_A.data,
+        C.data.ptr, desc_C, mode_C.data,
+        out.data.ptr, desc_C, mode_C.data,
+        reduce_op, cutensor_compute_type)
+    try:
+        ws = core._ndarray_init(
+            _cupy.ndarray, shape_t(1, ws_size), dtype=_numpy.int8, obj=None)
+    except _cupy.cuda.memory.OutOfMemoryError:
+        _warnings.warn('cuTENSOR: failed to allocate memory of workspace '
+                       '(size: {}).'.format(ws_size))
+        ws_size = 0
+        ws = core._ndarray_init(
+            _cupy.ndarray, shape_t(1, ws_size), dtype=_numpy.int8, obj=None)
+
+    cutensor.reduction(
+        handle,
+        alpha.ptr, A.data.ptr, desc_A, mode_A.data,
+        beta.ptr, C.data.ptr, desc_C, mode_C.data,
+        out.data.ptr, desc_C, mode_C.data,
+        reduce_op, cutensor_compute_type, ws.data.ptr, ws_size)
+    return out
+
+
+_cutensor_dtypes = [
+    # TODO(asi1024): Support float16
+    # _numpy.float16,
+    _numpy.float32,
+    _numpy.float64,
+    _numpy.complex64,
+    _numpy.complex128,
+]
+
+
+def _try_reduction_routine(
+        _ndarray_base x, axis, dtype, _ndarray_base out, keepdims, reduce_op,
+        alpha, beta):
+    cdef Handle handle
+    cdef _ndarray_base in_arg, out_arg
+    cdef shape_t out_shape
+    cdef tuple reduce_axis, out_axis
+    cdef TensorDescriptor desc_in, desc_out
+
+    if not check_availability('reduction'):
+        return None
+
+    if dtype is None:
+        dtype = x.dtype
+
+    if dtype not in _cutensor_dtypes:
+        return None
+    if dtype != x.dtype:
+        return None
+
+    if x.ndim == 0:
+        return None
+    if x.size == 0:
+        return None
+    if not x._c_contiguous:
+        # TODO(asi1024): Support also for F-contiguous array
+        return None
+
+    if x.size == 1 and cutensor.get_version() == 10400:
+        # WAR: element-1 reduction is buggy
+        return None
+
+    in_arg = x
+
+    reduce_axis, out_axis = _reduction._get_axis(axis, x.ndim)
+    if len(reduce_axis) == 0:
+        return None
+    out_shape = _reduction._get_out_shape(
+        x._shape, reduce_axis, out_axis, keepdims)
+    if out is None:
+        out = core._ndarray_init(
+            _cupy.ndarray, out_shape, dtype=dtype, obj=None)
+    elif not internal.vector_equal(out._shape, out_shape):
+        # TODO(asi1024): Support broadcast
+        return None
+    elif out.dtype != dtype:
+        return None
+    elif not out._c_contiguous:
+        # TODO(asi1024): Support also for F-contiguous array
+        return None
+
+    if keepdims:
+        out_arg = out.reshape(
+            _reduction._get_out_shape(x._shape, reduce_axis, out_axis, False))
+    else:
+        out_arg = out
+
+    # TODO(kmaeahshi): need to zero out when beta != 0
+
+    # TODO(asi1024): Remove temporary fix
+    in_arg._set_contiguous_strides(in_arg.itemsize, True)
+    out_arg._set_contiguous_strides(out_arg.itemsize, True)
+
+    handle = _get_handle()
+
+    desc_in = create_tensor_descriptor(in_arg, handle=handle)
+    desc_out = create_tensor_descriptor(out_arg, handle=handle)
+
+    compute_dtype = _set_compute_dtype(in_arg.dtype, dtype)
+
+    _reduction_impl(
+        handle,
+        _create_scalar(alpha, compute_dtype),
+        in_arg,
+        desc_in,
+        _create_mode_with_cache(in_arg._shape.size()),
+        _create_scalar(beta, compute_dtype),
+        out_arg,
+        desc_out,
+        _create_mode_with_cache(out_axis),
+        reduce_op, _get_cutensor_compute_type(compute_dtype))
+
+    return out
+
+
+@cython.profile(False)
+cdef inline bint _all_positive(const vector.vector[Py_ssize_t]& args):
+    # cuTENSOR requires each stride > 0.
+    for i in range(<Py_ssize_t>args.size()):
+        if args[i] <= 0:
+            return False
+    return True
+
+
+def _try_elementwise_binary_routine(
+        _ndarray_base a, _ndarray_base c, dtype, _ndarray_base out, op, alpha,
+        gamma):
+    cdef Handle handle
+
+    if not check_availability('elementwise'):
+        return None
+
+    if dtype is None:
+        dtype = a.dtype
+    if dtype not in _cutensor_dtypes:
+        return None
+
+    if not (a.dtype == c.dtype == dtype):
+        return None
+    if not internal.vector_equal(a._shape, c._shape):
+        return None
+    if a.size == 0:
+        return None
+    if not (_all_positive(a._strides) and _all_positive(c._strides)):
+        return None
+
+    compute_dtype = a.dtype
+
+    if compute_dtype.kind == 'c' and (
+            op == cutensor.OP_MAX or op == cutensor.OP_MIN):
+        return None
+
+    if out is None:
+        if c._c_contiguous:
+            pass
+        elif a._c_contiguous:
+            a, c = c, a
+            alpha, gamma = gamma, alpha
+        elif c._f_contiguous:
+            pass
+        elif a._f_contiguous:
+            a, c = c, a
+            alpha, gamma = gamma, alpha
+        else:
+            return None
+
+        # Determine a template object from which we initialize the output when
+        # inputs have subclass instances
+        def issubclass1(cls, classinfo):
+            return issubclass(cls, classinfo) and cls is not classinfo
+        subtype = _cupy.ndarray
+        template = None
+        a_type, c_type = type(a), type(c)
+        if issubclass1(a_type, _cupy.ndarray):
+            subtype = a_type
+            template = a
+        elif issubclass1(c_type, _cupy.ndarray):
+            subtype = c_type
+            template = c
+
+        out = core._create_ndarray_from_shape_strides(
+            subtype, c._shape, c._strides, compute_dtype, template)
+    elif out.dtype != compute_dtype:
+        return None
+    elif not internal.vector_equal(c._shape, out._shape):
+        return None
+    elif not internal.vector_equal(c._strides, out._strides):
+        return None
+    elif not _all_positive(out._strides):
+        return None
+
+    handle = _get_handle()
+
+    return _elementwise_binary_impl(
+        handle,
+        _create_scalar(alpha, compute_dtype),
+        a,
+        create_tensor_descriptor(a, handle=handle),
+        _create_mode_with_cache(a._shape.size()),
+        _create_scalar(gamma, compute_dtype),
+        c,
+        create_tensor_descriptor(c, handle=handle),
+        _create_mode_with_cache(c._shape.size()),
+        out,
+        op,
+        _dtype.to_cuda_dtype(compute_dtype, is_half_allowed=True))
diff --git a/cupyx/distributed/__init__.py b/cupyx/distributed/__init__.py
new file mode 100644
index 0000000..2d2ba76
--- /dev/null
+++ b/cupyx/distributed/__init__.py
@@ -0,0 +1,2 @@
+from cupyx.distributed._init import init_process_group  # NOQA
+from cupyx.distributed._nccl_comm import NCCLBackend  # NOQA
diff --git a/cupyx/distributed/_array.py b/cupyx/distributed/_array.py
new file mode 100644
index 0000000..38d3823
--- /dev/null
+++ b/cupyx/distributed/_array.py
@@ -0,0 +1,181 @@
+import cupy
+import numpy
+
+
+class _MultiDeviceDummyMemory(cupy.cuda.Memory):
+    pass
+
+
+class _MultiDeviceDummyPointer(cupy.cuda.MemoryPointer):
+    @property
+    def device(self):
+        # This override is needed to assign an invalid device id
+        # Since the array is not residing in a single device now
+        return cupy.cuda.device.Device(-1)
+
+
+class _DistributedArray(cupy.ndarray):
+    # Explicitly disable indexing, view, broadcast
+    def __new__(cls, shape, dtype, chunks, axis, tile_size, devices):
+        # Create a dummy memptr for the array data, we will also
+        # override its device methods
+        # memptr is needed to avoid actual device allocation on array init
+        # we need to generate other devices
+        mem = _MultiDeviceDummyMemory(0)
+        memptr = _MultiDeviceDummyPointer(mem, 0)
+        obj = super().__new__(cls, shape, dtype, memptr=memptr)
+        obj._chunks = chunks
+        obj._tile_size = tile_size
+        obj._mem = mem
+        obj._axis = axis
+        return obj
+
+    def __array_finalize__(self, obj):
+        if obj is None:
+            return
+        self._chunks = getattr(obj, 'chunks', None)
+        self._tile_size = getattr(obj, 'tile_size', None)
+        self._axis = getattr(obj, 'axis', None)
+        self._mem = getattr(obj, 'mem', None)
+
+    def _get_chunk(self, i):
+        return self._chunks[i]
+
+    def _prepare_args(self, dist_args, regular_args, device):
+        # Dist arrays must have chunks of compatible shapes, otherwise
+        # hard error.
+        # In case that they are of different, but broadcastable shapes
+        # Data movement may be needed
+        # Currently: Support only same shape chunks
+        args = []
+        c_shape = None
+        for (i, arg) in dist_args:
+            chunk = arg._get_chunk(device)
+            args.append((i, chunk))
+            if c_shape is None:
+                c_shape = chunk.shape
+            # TODO(ecastill) check if broadcastable, the array must have been
+            # split in the same axis?
+            if chunk.shape != c_shape:
+                raise RuntimeError(
+                    'Operating distributed arrays of different chunk sizes'
+                    ' together is not supported')
+
+        # Case of X.T and other data movement requiring cases not supported
+        # TODO(ecastill) add support for operands being non distributed arrays
+        # 1. Check if the regular arrays are on the specified device or
+        #    peer access is enabled
+        # 2. Check that their shape is compatible with the chunks
+        #    distributed arrays
+        # 3. Create views of this array and copy to the given device if needed
+        #    so that the chunks in the distributed operate with the right slice
+        if len(regular_args) > 0:
+            raise RuntimeError(
+                'Mix `cupy.ndarray` with distributed arrays is not currently'
+                'supported')
+
+        return args
+
+    def _get_execution_devices(self, dist_args):
+        devices = set()
+        for _, arg in dist_args:
+            # The key of chunks is the device id
+            for dev in arg._chunks:
+                devices.add(dev)
+        return devices
+
+    def _execute_kernel(self, kernel, args, kwargs):
+        distributed_arrays = []
+        regular_arrays = []
+        for i, arg in enumerate(args):
+            if isinstance(arg, _DistributedArray):
+                distributed_arrays.append((i, arg))
+            elif isinstance(arg, cupy.ndarray):
+                regular_arrays.append((i, arg))
+
+        # Do it for kwargs too
+        for k, arg in kwargs.items():
+            if isinstance(arg, _DistributedArray):
+                distributed_arrays.append((k, arg))
+            elif isinstance(arg, cupy.ndarray):
+                regular_arrays.append((k, arg))
+
+        args = list(args)
+        devices = self._get_execution_devices(distributed_arrays)
+        dev_outs = {}
+        dtype = None
+        for dev in devices:
+            array_args = self._prepare_args(
+                distributed_arrays, regular_arrays, dev)
+            for (i, arg) in array_args:
+                if isinstance(i, int):
+                    args[i] = arg
+                else:
+                    kwargs[i] = arg
+            with cupy.cuda.Device(dev):
+                out = kernel(*args, **kwargs)
+                dtype = out.dtype
+                dev_outs[dev] = out
+
+        for out in dev_outs.values():
+            if not isinstance(out, cupy.ndarray):
+                raise RuntimeError(
+                    'kernels returning other than single array not supported')
+
+        return _DistributedArray(
+            self.shape, dtype, dev_outs,
+            self._axis, self._tile_size, devices)
+
+    def __cupy_override_elementwise_kernel__(self, kernel, *args, **kwargs):
+        # This defines a protocol to be called from elementwise kernel
+        # to override some of the ops done there
+        outs = self._execute_kernel(kernel, args, kwargs)
+        return outs
+
+    def asnumpy(self):
+        # Coalesce it in a single array
+        chunks = [cupy.asnumpy(c) for c in self._chunks.values()]
+        return numpy.concatenate(chunks, axis=self._axis)
+
+
+def _split_tiles(array, nr_splits):
+    # TODO(ecastill) support arbitrary tiles shape
+    axis = nr_splits.argmax()
+    ind = nr_splits[axis]
+    if isinstance(array, cupy.ndarray):
+        arrs = [cupy.ascontiguousarray(a)
+                for a in cupy.array_split(array, int(ind), int(axis))]
+    else:
+        arrs = numpy.array_split(array, ind, axis)
+    return axis, arrs
+
+
+def array(array, devices, tile_shape):
+    """ Create an array that is splitted accross multiple devices in the
+    same host.
+    """
+    if not isinstance(array, (numpy.ndarray, cupy.ndarray)):
+        raise TypeError('`array` needs to be a numpy or cupy array')
+    # 1. Check that array_size / tile_shape == len(devices)
+    array_shape = numpy.array(array.shape, dtype=numpy.int32)
+    tile_shape = numpy.array(tile_shape, dtype=numpy.int32)
+    nr_splits = numpy.ceil(array_shape / tile_shape)
+    if nr_splits[nr_splits > 1].size > 1:
+        raise RuntimeError(
+            'Currently only sharding across a single axis is allowed')
+
+    assert len(tile_shape) == len(array_shape)
+
+    nr_chunks = numpy.prod(nr_splits)
+
+    if nr_chunks != len(devices):
+        raise RuntimeError(
+            'Array chunks must match the amount of devices')
+
+    axis, arrs = _split_tiles(array, nr_splits)
+    cp_chunks = {}
+    for arr, dev in zip(arrs, devices):
+        with cupy.cuda.Device(dev):
+            cp_chunks[dev] = cupy.array(arr)
+    return _DistributedArray(
+        array_shape, array.dtype, cp_chunks, axis, tile_shape, devices)
diff --git a/cupyx/distributed/_comm.py b/cupyx/distributed/_comm.py
new file mode 100644
index 0000000..92ae01d
--- /dev/null
+++ b/cupyx/distributed/_comm.py
@@ -0,0 +1,67 @@
+import abc
+
+from cupyx.distributed import _store
+
+
+class _Backend(abc.ABC):
+
+    def __init__(self, n_devices, rank,
+                 host=_store._DEFAULT_HOST, port=_store._DEFAULT_PORT):
+        self._n_devices = n_devices
+        self.rank = rank
+        self._store_proxy = _store.TCPStoreProxy(host, port)
+        if rank == 0:
+            self._store = _store.TCPStore(n_devices)
+
+    @abc.abstractmethod
+    def all_reduce(self, in_array, out_array, op='sum', stream=None):
+        pass
+
+    @abc.abstractmethod
+    def reduce(self, in_array, out_array, root=0, op='sum', stream=None):
+        pass
+
+    @abc.abstractmethod
+    def broadcast(self, in_out_array, root=0, stream=None):
+        pass
+
+    @abc.abstractmethod
+    def reduce_scatter(
+            self, in_array, out_array, count, op='sum', stream=None):
+        pass
+
+    @abc.abstractmethod
+    def all_gather(self, in_array, out_array, count, stream=None):
+        pass
+
+    @abc.abstractmethod
+    def send(self, array, peer, stream=None):
+        pass
+
+    @abc.abstractmethod
+    def recv(self, out_array, peer, stream=None):
+        pass
+
+    @abc.abstractmethod
+    def send_recv(self, in_array, out_array, peer, stream=None):
+        pass
+
+    @abc.abstractmethod
+    def scatter(self, in_array, out_array, root=0, stream=None):
+        pass
+
+    @abc.abstractmethod
+    def gather(self, in_array, out_array, root=0, stream=None):
+        pass
+
+    @abc.abstractmethod
+    def all_to_all(self, in_array, out_array, stream=None):
+        pass
+
+    @abc.abstractmethod
+    def barrier(self):
+        pass
+
+    def stop(self):
+        if self.rank == 0:
+            self._store.stop()
diff --git a/cupyx/distributed/_init.py b/cupyx/distributed/_init.py
new file mode 100644
index 0000000..81ab185
--- /dev/null
+++ b/cupyx/distributed/_init.py
@@ -0,0 +1,89 @@
+import os
+
+from cupy.cuda import nccl
+
+from cupyx.distributed import _store
+from cupyx.distributed._nccl_comm import NCCLBackend
+
+
+_backends = {'nccl': NCCLBackend}
+
+
+def init_process_group(
+        n_devices, rank, *, backend='nccl', host=None, port=None,
+        use_mpi=False):
+    """Start `cupyx.distributed` and obtain a communicator.
+
+    This call initializes the distributed environment, it needs to be
+    called for every process that is involved in the communications.
+
+    A single device per returned communication is only allowed. It is the user
+    responsibility of setting the appropiated gpu to be used before creating
+    and using the communicator.
+
+    Currently the user needs to specify each process rank and the total
+    number of processes, and start all the processes in different hosts
+    manually.
+
+    The process with rank 0 will spawn a TCP server using a
+    subprocess that listens in the port indicated by
+    the env var `CUPYX_DISTRIBUTED_PORT`, the rank 0 must be executed
+    in the host determined by the env var `CUPYX_DISTRIBUTED_HOST`.
+    In case their values are not specified, `'127.0.0.1'` and `13333` will be
+    used by default.
+
+    Note that this feature is expected to be used within a trusted cluster
+    environment.
+
+    Example:
+
+        >>> import cupy
+        >>> def process_0():
+        ...     import cupyx.distributed
+        ...     cupy.cuda.Device(0).use()
+        ...     comm = cupyx.distributed.init_process_group(2, 0)
+        ...     array = cupy.ones(1)
+        ...     comm.broadcast(array, 0)
+        ...
+        >>> def process_1():
+        ...     import cupyx.distributed
+        ...     cupy.cuda.Device(1).use()
+        ...     comm = cupyx.distributed.init_process_group(2, 1)
+        ...     array = cupy.zeros(1)
+        ...     comm.broadcast(array, 0)
+        ...     cupy.equal(array, cupy.ones(1))
+
+    Args:
+        n_devices (int): Total number of devices that will be used in the
+            distributed execution.
+        rank (int): Unique id of the GPU that the communicator is associated to
+            its value needs to be `0 <= rank < n_devices`.
+        backend (str): Backend to use for the communications. Optional,
+            defaults to `"nccl"`.
+        host (str): host address for the process rendezvous on initialization
+            defaults to `None`.
+        port (int): port for the process rendezvous on initialization
+            defaults to `None`.
+        use_mpi (bool): if ``False``, it avoids using MPI for synchronization
+            and uses the provided TCP server for exchanging CPU only
+            information.
+            defaults to `False`.
+    Returns:
+        Backend: object used to perform communications, adheres to the
+            :class:`~cupyx.distributed.Backend` specification:
+    """
+    if n_devices <= 0:
+        raise ValueError(f'Invalid number of devices {n_devices}')
+    if not (0 <= rank < n_devices):
+        raise ValueError(f'Invalid number of rank {rank} {n_devices}')
+    if backend not in _backends:
+        raise ValueError(f'{backend} is not supported')
+    if backend == 'nccl' and not nccl.available:
+        raise RuntimeError('NCCL is not available')
+    if host is None:
+        host = os.environ.get('CUPYX_DISTRIBUTED_HOST', _store._DEFAULT_HOST)
+    if port is None:
+        port = int(os.environ.get(
+            'CUPYX_DISTRIBUTED_PORT', _store._DEFAULT_PORT))
+
+    return _backends[backend](n_devices, rank, host, port, use_mpi)
diff --git a/cupyx/distributed/_klv_utils.py b/cupyx/distributed/_klv_utils.py
new file mode 100644
index 0000000..fc42b84
--- /dev/null
+++ b/cupyx/distributed/_klv_utils.py
@@ -0,0 +1,58 @@
+from ctypes import Structure, c_int, c_byte
+
+
+_VALUE_BUFFER_SIZE = 256
+
+
+class action_t(Structure):
+    _fields_ = [
+        ('action', c_int),
+        ('length', c_int),
+        ('value', c_byte * _VALUE_BUFFER_SIZE)]
+
+
+class result_action_t(Structure):
+    _fields_ = [
+        # If error value contains a message
+        ('status', c_int),  # 0 OK, 1 Error
+        ('length', c_int),
+        ('value', c_byte * _VALUE_BUFFER_SIZE)]
+
+
+def get_action_t(action, value):
+    l_v = len(value)
+    value = bytearray(value) + bytearray(
+        b'\x00' * (_VALUE_BUFFER_SIZE - len(value)))
+    value = (c_byte * _VALUE_BUFFER_SIZE).from_buffer(value)
+    assert (_VALUE_BUFFER_SIZE - len(value)) >= 0
+    return action_t(action, l_v, value)
+
+
+def get_result_action_t(status, value):
+    l_v = len(value)
+    value = bytearray(value) + bytearray(
+        b'\x00' * (_VALUE_BUFFER_SIZE - len(value)))
+    value = (c_byte * _VALUE_BUFFER_SIZE).from_buffer(value)
+    assert (_VALUE_BUFFER_SIZE - len(value)) >= 0
+    return result_action_t(status, l_v, value)
+
+
+def create_value_bytes(value):
+    if type(value) is bytes:
+        v = bytearray(b'b')
+        v = v + bytearray(value)
+    elif type(value) is int:
+        v = bytearray(b'i')
+        v = v + bytearray(value.to_bytes(8, byteorder='big'))
+    else:
+        raise ValueError(f'invalid type for self.value {value}')
+    return v
+
+
+def get_value_from_bytes(v):
+    if v[0:1] == b'i':
+        assert len(v[1:]) == 8
+        v = int.from_bytes(v[1:], 'big')
+    elif v[0:1] == b'b':
+        v = bytes(v[1:])
+    return v
diff --git a/cupyx/distributed/_nccl_comm.py b/cupyx/distributed/_nccl_comm.py
new file mode 100644
index 0000000..fb0df40
--- /dev/null
+++ b/cupyx/distributed/_nccl_comm.py
@@ -0,0 +1,840 @@
+import numpy
+import warnings
+
+import cupy
+from cupy.cuda import nccl
+from cupyx.distributed import _store
+from cupyx.distributed._comm import _Backend
+from cupyx.scipy import sparse
+
+
+try:
+    from mpi4py import MPI
+    _mpi_available = True
+except ImportError:
+    _mpi_available = False
+
+
+if nccl.available:
+    # types are not compliant with windows on long/int32 issue
+    # but nccl does not support windows so we don't care
+    _nccl_dtypes = {'b': nccl.NCCL_INT8,
+                    'B': nccl.NCCL_UINT8,
+                    'i': nccl.NCCL_INT32,
+                    'I': nccl.NCCL_UINT32,
+                    'l': nccl.NCCL_INT64,
+                    'L': nccl.NCCL_UINT64,
+                    'q': nccl.NCCL_INT64,
+                    'Q': nccl.NCCL_UINT64,
+                    'e': nccl.NCCL_FLOAT16,
+                    'f': nccl.NCCL_FLOAT32,
+                    'd': nccl.NCCL_FLOAT64,
+                    # Size of array will be doubled
+                    'F': nccl.NCCL_FLOAT32,
+                    'D': nccl.NCCL_FLOAT64}
+
+    _nccl_ops = {'sum': nccl.NCCL_SUM,
+                 'prod': nccl.NCCL_PROD,
+                 'max': nccl.NCCL_MAX,
+                 'min': nccl.NCCL_MIN}
+else:
+    _nccl_dtypes = {}
+
+    _nccl_ops = {}
+
+
+class NCCLBackend(_Backend):
+    """Interface that uses NVIDIA's NCCL to perform communications.
+
+    Args:
+        n_devices (int): Total number of devices that will be used in the
+            distributed execution.
+        rank (int): Unique id of the GPU that the communicator is associated to
+            its value needs to be `0 <= rank < n_devices`.
+        host (str, optional): host address for the process rendezvous on
+            initialization. Defaults to `"127.0.0.1"`.
+        port (int, optional): port used for the process rendezvous on
+            initialization. Defaults to `13333`.
+        use_mpi(bool, optional): switch between MPI and use the included TCP
+            server for initialization & synchronization. Defaults to `False`.
+    """
+
+    def __init__(self, n_devices, rank,
+                 host=_store._DEFAULT_HOST, port=_store._DEFAULT_PORT,
+                 use_mpi=False):
+        super().__init__(n_devices, rank, host, port)
+        self._use_mpi = _mpi_available and use_mpi
+        if self._use_mpi:
+            self._init_with_mpi(n_devices, rank)
+        else:
+            self._init_with_tcp_store(n_devices, rank, host, port)
+
+    def _init_with_mpi(self, n_devices, rank):
+        # MPI is used only for management purposes
+        # so the rank may be different than the one specified
+        self._mpi_comm = MPI.COMM_WORLD
+        self._mpi_rank = self._mpi_comm.Get_rank()
+        self._mpi_comm.Barrier()
+        nccl_id = None
+        if self._mpi_rank == 0:
+            nccl_id = nccl.get_unique_id()
+        nccl_id = self._mpi_comm.bcast(nccl_id, root=0)
+        # Initialize devices
+        self._comm = nccl.NcclCommunicator(n_devices, nccl_id, rank)
+
+    def _init_with_tcp_store(self, n_devices, rank, host, port):
+        nccl_id = None
+        if rank == 0:
+            self._store.run(host, port)
+            nccl_id = nccl.get_unique_id()
+            # get_unique_id return negative values due to cython issues
+            # with bytes && c strings. We shift them by 128 to
+            # make them positive and send them as bytes to the proxy store
+            shifted_nccl_id = bytes([b + 128 for b in nccl_id])
+            self._store_proxy['nccl_id'] = shifted_nccl_id
+            self._store_proxy.barrier()
+        else:
+            self._store_proxy.barrier()
+            nccl_id = self._store_proxy['nccl_id']
+            nccl_id = tuple([int(b) - 128 for b in nccl_id])
+        self._comm = nccl.NcclCommunicator(n_devices, nccl_id, rank)
+
+    def _check_contiguous(self, array):
+        if not array.flags.c_contiguous and not array.flags.f_contiguous:
+            raise RuntimeError(
+                'NCCL requires arrays to be either c- or f-contiguous')
+
+    def _get_nccl_dtype_and_count(self, array, count=None):
+        dtype = array.dtype.char
+        if dtype not in _nccl_dtypes:
+            raise TypeError(f'Unknown dtype {array.dtype} for NCCL')
+        nccl_dtype = _nccl_dtypes[dtype]
+        if count is None:
+            count = array.size
+        if dtype in 'FD':
+            return nccl_dtype, 2 * count
+        return nccl_dtype, count
+
+    def _get_stream(self, stream):
+        if stream is None:
+            stream = cupy.cuda.stream.get_current_stream()
+        return stream.ptr
+
+    def _get_op(self, op, dtype):
+        if op not in _nccl_ops:
+            raise RuntimeError(f'Unknown op {op} for NCCL')
+        if dtype in 'FD' and op != nccl.NCCL_SUM:
+            raise ValueError(
+                'Only nccl.SUM is supported for complex arrays')
+        return _nccl_ops[op]
+
+    def _dispatch_arg_type(self, function, args):
+        comm_class = _DenseNCCLCommunicator
+        if (
+            (isinstance(args[0], (list, tuple))
+             and sparse.issparse(args[0][0]))
+            or sparse.issparse(args[0])
+        ):
+            comm_class = _SparseNCCLCommunicator
+        getattr(comm_class, function)(self, *args)
+
+    def all_reduce(self, in_array, out_array, op='sum', stream=None):
+        """Performs an all reduce operation.
+
+        Args:
+            in_array (cupy.ndarray): array to be sent.
+            out_array (cupy.ndarray): array where the result with be stored.
+            op (str): reduction operation, can be one of
+                ('sum', 'prod', 'min' 'max'), arrays of complex type only
+                support `'sum'`. Defaults to `'sum'`.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type(
+            'all_reduce', (in_array, out_array, op, stream))
+
+    def reduce(self, in_array, out_array, root=0, op='sum', stream=None):
+        """Performs a reduce operation.
+
+        Args:
+            in_array (cupy.ndarray): array to be sent.
+            out_array (cupy.ndarray): array where the result with be stored.
+                will only be modified by the `root` process.
+            root (int, optional): rank of the process that will perform the
+                reduction. Defaults to `0`.
+            op (str): reduction operation, can be one of
+                ('sum', 'prod', 'min' 'max'), arrays of complex type only
+                support `'sum'`. Defaults to `'sum'`.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type(
+            'reduce', (in_array, out_array, root, op, stream))
+
+    def broadcast(self, in_out_array, root=0, stream=None):
+        """Performs a broadcast operation.
+
+        Args:
+            in_out_array (cupy.ndarray): array to be sent for `root` rank.
+                Other ranks will receive the broadcast data here.
+            root (int, optional): rank of the process that will send the
+                broadcast. Defaults to `0`.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        # in_out_array for rank !=0 will be used as output
+        self._dispatch_arg_type(
+            'broadcast', (in_out_array, root, stream))
+
+    def reduce_scatter(
+            self, in_array, out_array, count, op='sum', stream=None):
+        """Performs a reduce scatter operation.
+
+        Args:
+            in_array (cupy.ndarray): array to be sent.
+            out_array (cupy.ndarray): array where the result with be stored.
+            count (int): Number of elements to send to each rank.
+            op (str): reduction operation, can be one of
+                ('sum', 'prod', 'min' 'max'), arrays of complex type only
+                support `'sum'`. Defaults to `'sum'`.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type(
+            'reduce_scatter', (in_array, out_array, count, op, stream))
+
+    def all_gather(self, in_array, out_array, count, stream=None):
+        """Performs an all gather operation.
+
+        Args:
+            in_array (cupy.ndarray): array to be sent.
+            out_array (cupy.ndarray): array where the result with be stored.
+            count (int): Number of elements to send to each rank.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type(
+            'all_gather', (in_array, out_array, count, stream))
+
+    def send(self, array, peer, stream=None):
+        """Performs a send operation.
+
+        Args:
+            array (cupy.ndarray): array to be sent.
+            peer (int): rank of the process `array` will be sent to.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type('send', (array, peer, stream))
+
+    def recv(self, out_array, peer, stream=None):
+        """Performs a receive operation.
+
+        Args:
+            array (cupy.ndarray): array used to receive data.
+            peer (int): rank of the process `array` will be received from.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type('recv', (out_array, peer, stream))
+
+    def send_recv(self, in_array, out_array, peer, stream=None):
+        """Performs a send and receive operation.
+
+        Args:
+            in_array (cupy.ndarray): array to be sent.
+            out_array (cupy.ndarray): array used to receive data.
+            peer (int): rank of the process to send `in_array` and receive
+                `out_array`.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type(
+            'send_recv', (in_array, out_array, peer, stream))
+
+    def scatter(self, in_array, out_array, root=0, stream=None):
+        """Performs a scatter operation.
+
+        Args:
+            in_array (cupy.ndarray): array to be sent. Its shape must be
+                `(total_ranks, ...)`.
+            out_array (cupy.ndarray): array where the result with be stored.
+            root (int): rank that will send the `in_array` to other ranks.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type(
+            'scatter', (in_array, out_array, root, stream))
+
+    def gather(self, in_array, out_array, root=0, stream=None):
+        """Performs a gather operation.
+
+        Args:
+            in_array (cupy.ndarray): array to be sent.
+            out_array (cupy.ndarray): array where the result with be stored.
+                Its shape must be `(total_ranks, ...)`.
+            root (int): rank that will receive `in_array` from other ranks.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type(
+            'gather', (in_array, out_array, root, stream))
+
+    def all_to_all(self, in_array, out_array, stream=None):
+        """Performs an all to all operation.
+
+        Args:
+            in_array (cupy.ndarray): array to be sent. Its shape must be
+                `(total_ranks, ...)`.
+            out_array (cupy.ndarray): array where the result with be stored.
+                Its shape must be `(total_ranks, ...)`.
+            stream (cupy.cuda.Stream, optional): if supported, stream to
+                perform the communication.
+        """
+        self._dispatch_arg_type(
+            'all_to_all', (in_array, out_array, stream))
+
+    def barrier(self):
+        """Performs a barrier operation.
+
+        The barrier is done in the cpu and is a explicit synchronization
+        mechanism that halts the thread progression.
+        """
+        # implements a barrier CPU side
+        # TODO allow multiple barriers to be executed
+        if self._use_mpi:
+            self._mpi_comm.Barrier()
+        else:
+            self._store_proxy.barrier()
+
+
+class _DenseNCCLCommunicator:
+
+    @classmethod
+    def all_reduce(cls, comm, in_array, out_array, op='sum', stream=None):
+        comm._check_contiguous(in_array)
+        comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        dtype, count = comm._get_nccl_dtype_and_count(in_array)
+        op = comm._get_op(op, in_array.dtype.char)
+        comm._comm.allReduce(
+            in_array.data.ptr, out_array.data.ptr, count, dtype, op, stream)
+
+    @classmethod
+    def reduce(cls, comm, in_array, out_array, root=0, op='sum', stream=None):
+        comm._check_contiguous(in_array)
+        if comm.rank == root:
+            comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        dtype, count = comm._get_nccl_dtype_and_count(in_array)
+        op = comm._get_op(op, in_array.dtype.char)
+        comm._comm.reduce(
+            in_array.data.ptr, out_array.data.ptr,
+            count, dtype, op, root, stream)
+
+    @classmethod
+    def broadcast(cls, comm, in_out_array, root=0, stream=None):
+        comm._check_contiguous(in_out_array)
+        stream = comm._get_stream(stream)
+        dtype, count = comm._get_nccl_dtype_and_count(in_out_array)
+        comm._comm.broadcast(
+            in_out_array.data.ptr, in_out_array.data.ptr,
+            count, dtype, root, stream)
+
+    @classmethod
+    def reduce_scatter(
+            cls, comm, in_array, out_array, count, op='sum', stream=None):
+        comm._check_contiguous(in_array)
+        comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        dtype, count = comm._get_nccl_dtype_and_count(in_array, count)
+        op = comm._get_op(op, in_array.dtype.char)
+        comm._comm.reduceScatter(
+            in_array.data.ptr, out_array.data.ptr, count, dtype, op, stream)
+
+    @classmethod
+    def all_gather(cls, comm, in_array, out_array, count, stream=None):
+        comm._check_contiguous(in_array)
+        comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        dtype, count = comm._get_nccl_dtype_and_count(in_array, count)
+        comm._comm.allGather(
+            in_array.data.ptr, out_array.data.ptr, count, dtype, stream)
+
+    @classmethod
+    def send(cls, comm, array, peer, stream=None):
+        comm._check_contiguous(array)
+        stream = comm._get_stream(stream)
+        dtype, count = comm._get_nccl_dtype_and_count(array)
+        cls._send(comm, array, peer, dtype, count, stream)
+
+    @classmethod
+    def _send(cls, comm, array, peer, dtype, count, stream=None):
+        comm._comm.send(array.data.ptr, count, dtype, peer, stream)
+
+    @classmethod
+    def recv(cls, comm, out_array, peer, stream=None):
+        comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        dtype, count = comm._get_nccl_dtype_and_count(out_array)
+        cls._recv(comm, out_array, peer, dtype, count, stream)
+
+    @classmethod
+    def _recv(cls, comm, out_array, peer, dtype, count, stream=None):
+        comm._comm.recv(out_array.data.ptr, count, dtype, peer, stream)
+
+    @classmethod
+    def send_recv(cls, comm, in_array, out_array, peer, stream=None):
+        comm._check_contiguous(in_array)
+        comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        idtype, icount = comm._get_nccl_dtype_and_count(in_array)
+        odtype, ocount = comm._get_nccl_dtype_and_count(out_array)
+        nccl.groupStart()
+        cls._send(comm, in_array, peer, idtype, icount, stream)
+        cls._recv(comm, out_array, peer, odtype, ocount, stream)
+        nccl.groupEnd()
+
+    @classmethod
+    def scatter(cls, comm, in_array, out_array, root=0, stream=None):
+        if in_array.shape[0] != comm._n_devices:
+            raise RuntimeError(
+                f'scatter requires in_array to have {comm._n_devices}'
+                f'elements in its first dimension, found {in_array.shape}')
+        comm._check_contiguous(in_array)
+        comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        nccl.groupStart()
+        if root == comm.rank:
+            for i in range(comm._n_devices):
+                array = in_array[i]
+                idtype, icount = comm._get_nccl_dtype_and_count(array)
+                cls._send(comm, array, i, idtype, icount, stream)
+        dtype, count = comm._get_nccl_dtype_and_count(out_array)
+        cls._recv(comm, out_array, root, dtype, count, stream)
+        nccl.groupEnd()
+
+    @classmethod
+    def gather(cls, comm, in_array, out_array, root=0, stream=None):
+        # TODO(ecastill) out_array needs to have comm size in shape[0]
+        if out_array.shape[0] != comm._n_devices:
+            raise RuntimeError(
+                f'gather requires out_array to have {comm._n_devices}'
+                f'elements in its first dimension, found {out_array.shape}')
+        comm._check_contiguous(in_array)
+        comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        nccl.groupStart()
+        if root == comm.rank:
+            for i in range(comm._n_devices):
+                array = out_array[i]
+                odtype, ocount = comm._get_nccl_dtype_and_count(array)
+                cls._recv(comm, array, i, odtype, ocount, stream)
+        dtype, count = comm._get_nccl_dtype_and_count(in_array)
+        cls._send(comm, in_array, root, dtype, count, stream)
+        nccl.groupEnd()
+
+    @classmethod
+    def all_to_all(cls, comm, in_array, out_array, stream=None):
+        # TODO(ecastill) out_array needs to have comm size in shape[0]
+        if out_array.shape[0] != comm._n_devices:
+            raise RuntimeError(
+                f'all_to_all requires in_array to have {comm._n_devices}'
+                f'elements in its first dimension, found {in_array.shape}')
+        if out_array.shape[0] != comm._n_devices:
+            raise RuntimeError(
+                f'all_to_all requires out_array to have {comm._n_devices}'
+                f'elements in its first dimension, found {out_array.shape}')
+        comm._check_contiguous(in_array)
+        comm._check_contiguous(out_array)
+        stream = comm._get_stream(stream)
+        idtype, icount = comm._get_nccl_dtype_and_count(in_array[0])
+        odtype, ocount = comm._get_nccl_dtype_and_count(out_array[0])
+        # TODO check out dtypes are the same as in dtypes
+        nccl.groupStart()
+        for i in range(comm._n_devices):
+            cls._send(comm, in_array[i], i, idtype, icount, stream)
+            cls._recv(comm, out_array[i], i, odtype, ocount, stream)
+        nccl.groupEnd()
+
+
+def _make_sparse_empty(dtype, sparse_type):
+    data = cupy.empty(1, dtype)
+    a = cupy.empty(1, 'i')
+    b = cupy.empty(1, 'i')
+    if sparse_type == 'csr':
+        return sparse.csr_matrix((data, a, b), shape=(0, 0))
+    elif sparse_type == 'csc':
+        return sparse.csc_matrix((data, a, b), shape=(0, 0))
+    elif sparse_type == 'coo':
+        return sparse.coo_matrix((data, (a, b)), shape=(0, 0))
+    else:
+        raise TypeError(
+            'NCCL is not supported for this type of sparse matrix')
+
+
+def _get_sparse_type(matrix):
+    if sparse.isspmatrix_coo(matrix):
+        return 'coo'
+    elif sparse.isspmatrix_csr(matrix):
+        return 'csr'
+    elif sparse.isspmatrix_csc(matrix):
+        return 'csc'
+    else:
+        raise TypeError(
+            'NCCL is not supported for this type of sparse matrix')
+
+
+class _SparseNCCLCommunicator:
+
+    @classmethod
+    def _get_internal_arrays(cls, array):
+        if sparse.isspmatrix_coo(array):
+            array.sum_duplicates()  # set it to cannonical form
+            return (array.data, array.row, array.col)
+        elif sparse.isspmatrix_csr(array) or sparse.isspmatrix_csc(array):
+            return (array.data, array.indptr, array.indices)
+        raise TypeError('NCCL is not supported for this type of sparse matrix')
+
+    @classmethod
+    def _get_shape_and_sizes(cls, arrays, shape):
+        # We get the elements from the array and send them
+        # so that other process can create receiving arrays for it
+        # However, this exchange synchronizes the gpus
+        sizes_shape = shape + tuple((a.size for a in arrays))
+        return sizes_shape
+
+    @classmethod
+    def _exchange_shape_and_sizes(
+            cls, comm, peer, sizes_shape, method, stream):
+        if comm._use_mpi:
+            # Sends the metadata for the arrays using MPI
+            if method == 'send':
+                sizes_shape = numpy.array(sizes_shape, dtype='q')
+                comm._mpi_comm.Send(sizes_shape, dest=peer, tag=1)
+                return None
+            elif method == 'recv':
+                # Shape is a tuple of two elements, and a single scalar per
+                # each array (5)
+                sizes_shape = numpy.empty(5, dtype='q')
+                comm._mpi_comm.Recv(sizes_shape, source=peer, tag=1)
+                return sizes_shape
+            elif method == 'bcast':
+                if comm.rank == peer:
+                    sizes_shape = numpy.array(sizes_shape, dtype='q')
+                else:
+                    sizes_shape = numpy.empty(5, dtype='q')
+                comm._mpi_comm.Bcast(sizes_shape, root=peer)
+                return sizes_shape
+            elif method == 'gather':
+                sizes_shape = numpy.array(sizes_shape, dtype='q')
+                recv_buf = numpy.empty([comm._n_devices, 5], dtype='q')
+                comm._mpi_comm.Gather(sizes_shape, recv_buf, peer)
+                return recv_buf
+            elif method == 'alltoall':
+                sizes_shape = numpy.array(sizes_shape, dtype='q')
+                recv_buf = numpy.empty([comm._n_devices, 5], dtype='q')
+                comm._mpi_comm.Alltoall(sizes_shape, recv_buf)
+                return recv_buf
+            else:
+                raise RuntimeError('Unsupported method')
+        else:
+            warnings.warn(
+                'Using NCCL for transferring sparse arrays metadata. This'
+                ' will cause device synchronization and a huge performance'
+                ' degradation. Please install MPI and `mpi4py` in order to'
+                ' avoid this issue.'
+            )
+            if method == 'send':
+                sizes_shape = cupy.array(sizes_shape, dtype='q')
+                cls._send(
+                    comm, sizes_shape, peer, sizes_shape.dtype, 5, stream)
+                return None
+            elif method == 'recv':
+                # Shape is a tuple of two elements, and a single scalar per
+                # each array (5)
+                sizes_shape = cupy.empty(5, dtype='q')
+                cls._recv(
+                    comm, sizes_shape, peer, sizes_shape.dtype, 5, stream)
+                return cupy.asnumpy(sizes_shape)
+            elif method == 'bcast':
+                if comm.rank == peer:
+                    sizes_shape = cupy.array(sizes_shape, dtype='q')
+                else:
+                    sizes_shape = cupy.empty(5, dtype='q')
+                _DenseNCCLCommunicator.broadcast(
+                    comm, sizes_shape, root=peer, stream=stream)
+                return cupy.asnumpy(sizes_shape)
+            elif method == 'gather':
+                sizes_shape = cupy.array(sizes_shape, dtype='q')
+                recv_buf = cupy.empty((comm._n_devices, 5), dtype='q')
+                _DenseNCCLCommunicator.gather(
+                    comm, sizes_shape, recv_buf, root=peer, stream=stream)
+                return cupy.asnumpy(recv_buf)
+            elif method == 'alltoall':
+                sizes_shape = cupy.array(sizes_shape, dtype='q')
+                recv_buf = cupy.empty((comm._n_devices, 5), dtype='q')
+                _DenseNCCLCommunicator.all_to_all(
+                    comm, sizes_shape, recv_buf, stream=stream)
+                return cupy.asnumpy(recv_buf)
+            else:
+                raise RuntimeError('Unsupported method')
+
+    def _assign_arrays(matrix, arrays, shape):
+        if sparse.isspmatrix_coo(matrix):
+            matrix.data = arrays[0]
+            matrix.row = arrays[1]
+            matrix.col = arrays[2]
+            matrix._shape = tuple(shape)
+        elif sparse.isspmatrix_csr(matrix) or sparse.isspmatrix_csc(matrix):
+            matrix.data = arrays[0]
+            matrix.indptr = arrays[1]
+            matrix.indices = arrays[2]
+            matrix._shape = tuple(shape)
+        else:
+            raise TypeError(
+                'NCCL is not supported for this type of sparse matrix')
+
+    @classmethod
+    def all_reduce(cls, comm, in_array, out_array, op='sum', stream=None):
+        # TODO(ecastill) find a way to better determine the root, maybe random?
+        # super naive algorithm
+        root = 0
+        cls.reduce(comm, in_array, out_array, root, op, stream)
+        cls.broadcast(comm, out_array, root, stream)
+
+    @classmethod
+    def reduce(cls, comm, in_array, out_array, root=0, op='sum', stream=None):
+        arrays = cls._get_internal_arrays(in_array)
+        # All the matrices must share the same size
+        shape_and_sizes = cls._get_shape_and_sizes(arrays, in_array.shape)
+        shape_and_sizes = cls._exchange_shape_and_sizes(
+            comm, root, shape_and_sizes, 'gather', stream)
+        if comm.rank == root:
+            if _get_sparse_type(in_array) != _get_sparse_type(out_array):
+                raise ValueError(
+                    'in_array and out_array must be the same format')
+            result = in_array
+            partial = _make_sparse_empty(
+                in_array.dtype, _get_sparse_type(in_array))
+            # each device will send and array with a different size
+            for peer, ss in enumerate(shape_and_sizes):
+                shape = tuple(ss[0:2])
+                sizes = ss[2:]
+                arrays = [
+                    cupy.empty(s, dtype=a.dtype) for s, a in zip(sizes, arrays)
+                ]
+                if peer != root:
+                    nccl.groupStart()
+                    for a in arrays:
+                        cls._recv(comm, a, peer, a.dtype, a.size, stream)
+                    nccl.groupEnd()
+                    cls._assign_arrays(partial, arrays, shape)
+                    if op == 'sum':
+                        result = result + partial
+                    elif op == 'prod':
+                        result = result * partial
+                    else:
+                        raise ValueError(
+                            'Sparse matrix only supports sum/prod reduction')
+            # TODO, check output types
+            # If out_array is coo we need to convert result to coo before
+            # reasiging
+            cls._assign_arrays(
+                out_array, cls._get_internal_arrays(result), result.shape)
+        else:
+            nccl.groupStart()
+            for a in arrays:
+                cls._send(
+                    comm, a, root, a.dtype, a.size, stream)
+            nccl.groupEnd()
+
+    @classmethod
+    def broadcast(cls, comm, in_out_array, root=0, stream=None):
+        arrays = cls._get_internal_arrays(in_out_array)
+        if comm.rank == root:
+            shape_and_sizes = cls._get_shape_and_sizes(
+                arrays, in_out_array.shape)
+        else:
+            shape_and_sizes = ()
+
+        shape_and_sizes = cls._exchange_shape_and_sizes(
+            comm, root, shape_and_sizes, 'bcast', stream)
+        shape = tuple(shape_and_sizes[0:2])
+        sizes = shape_and_sizes[2:]
+        # Naive approach, we send each of the subarrays one by one
+        if comm.rank != root:
+            arrays = [
+                cupy.empty(s, dtype=a.dtype) for s, a in zip(sizes, arrays)]
+        # TODO(ecastill): measure if its faster to just contatenate
+        # the arrays in a single one and send it
+        nccl.groupStart()
+        for a in arrays:
+            _DenseNCCLCommunicator.broadcast(comm, a, root, stream)
+        nccl.groupEnd()
+        cls._assign_arrays(in_out_array, arrays, shape)
+
+    @classmethod
+    def reduce_scatter(
+            cls, comm, in_array, out_array, count, op='sum', stream=None):
+        # We need a LIST of sparse in_arrays and perform a reduction for each
+        # of the entries, then we will scatter that result
+        root = 0
+        reduce_out_arrays = []
+        if not isinstance(in_array, (list, tuple)):
+            raise ValueError(
+                'in_array must be a list or a tuple of sparse matrices')
+        for s_m in in_array:
+            partial_out_array = _make_sparse_empty(
+                s_m.dtype, _get_sparse_type(s_m))
+            cls.reduce(comm, s_m, partial_out_array, root, op, stream)
+            reduce_out_arrays.append(partial_out_array)
+        cls.scatter(comm, reduce_out_arrays, out_array, root, stream)
+
+    @classmethod
+    def all_gather(cls, comm, in_array, out_array, count, stream=None):
+        # OutArray is a list
+        # This is like gather follow by a broadcast
+        # TODO(ecastill), broadcast a single array and split it instead
+        # of doing a loop of broadcasts
+        # TODO(ecastill) find a way to better determine the root, maybe random?
+        # super naive algorithm
+        root = 0
+        gather_out_arrays = []
+        cls.gather(comm, in_array, gather_out_arrays, root, stream)
+        if comm.rank != root:
+            gather_out_arrays = [
+                _make_sparse_empty(in_array.dtype, _get_sparse_type(in_array))
+                for _ in range(comm._n_devices)
+            ]
+        for arr in gather_out_arrays:
+            cls.broadcast(comm, arr, root, stream)
+            out_array.append(arr)
+
+    @classmethod
+    def send(cls, comm, array, peer, stream=None):
+        arrays = cls._get_internal_arrays(array)
+        shape_and_sizes = cls._get_shape_and_sizes(arrays, array.shape)
+        cls._exchange_shape_and_sizes(
+            comm, peer, shape_and_sizes, 'send', stream)
+        # Naive approach, we send each of the subarrays one by one
+        nccl.groupStart()
+        for a in arrays:
+            cls._send(comm, a, peer, a.dtype, a.size, stream)
+        nccl.groupEnd()
+
+    @classmethod
+    def _send(cls, comm, array, peer, dtype, count, stream=None):
+        dtype = array.dtype.char
+        if dtype not in _nccl_dtypes:
+            raise TypeError(f'Unknown dtype {array.dtype} for NCCL')
+        dtype, count = comm._get_nccl_dtype_and_count(array)
+        stream = comm._get_stream(stream)
+        comm._comm.send(array.data.ptr, count, dtype, peer, stream)
+
+    @classmethod
+    def recv(cls, comm, out_array, peer, stream=None):
+        shape_and_sizes = cls._exchange_shape_and_sizes(
+            comm, peer, (), 'recv', stream)
+        # Change the array sizes in out_array to match the sent ones
+        # Receive the three arrays
+        # TODO(ecastill) dtype is not correct, it must match the internal
+        # sparse matrix arrays dtype
+        arrays = cls._get_internal_arrays(out_array)
+        shape = tuple(shape_and_sizes[0:2])
+        sizes = shape_and_sizes[2:]
+        # TODO(use the out_array datatypes)
+        arrs = [cupy.empty(s, dtype=a.dtype) for s, a in zip(sizes, arrays)]
+        nccl.groupStart()
+        for a in arrs:
+            cls._recv(comm, a, peer, a.dtype, a.size, stream)
+        nccl.groupEnd()
+        # Create a sparse matrix from the received arrays
+        cls._assign_arrays(out_array, arrs, shape)
+
+    @classmethod
+    def _recv(cls, comm, out_array, peer, dtype, count, stream=None):
+        dtype = dtype.char
+        if dtype not in _nccl_dtypes:
+            raise TypeError(f'Unknown dtype {out_array.dtype} for NCCL')
+        dtype, count = comm._get_nccl_dtype_and_count(out_array)
+        stream = comm._get_stream(stream)
+        comm._comm.recv(out_array.data.ptr, count, dtype, peer, stream)
+
+    @classmethod
+    def send_recv(cls, comm, in_array, out_array, peer, stream=None):
+        nccl.groupStart()
+        cls.send(comm, in_array, peer, stream)
+        cls.recv(comm, out_array, peer, stream)
+        nccl.groupEnd()
+
+    @classmethod
+    def scatter(cls, comm, in_array, out_array, root=0, stream=None):
+        # in_array is a list of sparse matrices
+        if comm.rank == root:
+            nccl.groupStart()
+            for peer, s_a in enumerate(in_array):
+                if peer != root:
+                    cls.send(comm, s_a, peer, stream)
+            nccl.groupEnd()
+            cls._assign_arrays(
+                out_array,
+                cls._get_internal_arrays(in_array[root]),
+                in_array[root].shape)
+        else:
+            cls.recv(comm, out_array, root, stream)
+
+    @classmethod
+    def gather(cls, comm, in_array, out_array, root=0, stream=None):
+        # out_array is a list of sparse matrices
+        if comm.rank == root:
+            for peer in range(comm._n_devices):
+                res = _make_sparse_empty(
+                    in_array.dtype, _get_sparse_type(in_array))
+                if peer != root:
+                    cls.recv(comm, res, peer, stream)
+                else:
+                    cls._assign_arrays(
+                        res,
+                        cls._get_internal_arrays(in_array),
+                        in_array.shape)
+                out_array.append(res)
+        else:
+            cls.send(comm, in_array, root, stream)
+
+    @classmethod
+    def all_to_all(cls, comm, in_array, out_array, stream=None):
+        # in_array & out_array is a list of sparse matrices
+        if len(in_array) != comm._n_devices:
+            raise RuntimeError(
+                f'all_to_all requires in_array to have {comm._n_devices}'
+                f'elements, found {len(in_array)}')
+
+        # Exchange metadata
+        shape_and_sizes = []
+        recv_shape_and_sizes = []
+        for i, a in enumerate(in_array):
+            arrays = cls._get_internal_arrays(a)
+            shape_and_sizes.append(cls._get_shape_and_sizes(arrays, a.shape))
+
+        recv_shape_and_sizes = cls._exchange_shape_and_sizes(
+            comm, i, shape_and_sizes, 'alltoall', stream)
+
+        # prepare the arrays to recv the data
+        for i in range(comm._n_devices):
+            shape = tuple(recv_shape_and_sizes[i][0:2])
+            sizes = recv_shape_and_sizes[i][2:]
+            s_arrays = cls._get_internal_arrays(in_array[i])
+            # TODO(use the out_array datatypes)
+            r_arrays = [
+                cupy.empty(s, dtype=a.dtype) for s, a in zip(sizes, s_arrays)]
+            nccl.groupStart()
+            for a in s_arrays:
+                cls._send(comm, a, i, a.dtype, a.size, stream)
+            for a in r_arrays:
+                cls._recv(comm, a, i, a.dtype, a.size, stream)
+            nccl.groupEnd()
+            out_array.append(_make_sparse_empty(
+                in_array[i].dtype,
+                _get_sparse_type(in_array[i])))
+            cls._assign_arrays(out_array[i], r_arrays, shape)
diff --git a/cupyx/distributed/_store.py b/cupyx/distributed/_store.py
new file mode 100644
index 0000000..ef5f7ae
--- /dev/null
+++ b/cupyx/distributed/_store.py
@@ -0,0 +1,153 @@
+import atexit
+from ctypes import sizeof
+import multiprocessing
+import threading
+import socket
+import time
+
+from cupyx.distributed import _klv_utils
+from cupyx.distributed import _store_actions
+
+
+_DEFAULT_HOST = '127.0.0.1'
+_DEFAULT_PORT = 13333
+
+_exit_mode = False
+
+
+@atexit.register
+def _exit():
+    global _exit_mode
+    _exit_mode = True
+
+
+class ExceptionAwareProcess(multiprocessing.Process):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._exception = None
+        self._parent_p, self._child_p = multiprocessing.Pipe()
+
+    def run(self):
+        try:
+            super().run()
+            self._child_p.send(None)
+        except Exception as e:
+            self._child_p.send(e)
+
+    def join(self):
+        super().join()
+        if self._parent_p.poll():
+            exception = self._parent_p.recv()
+            if exception is not None:
+                raise exception
+
+
+class TCPStore:
+    # This is only used for initialization of nccl so we don't care
+    # too much about peformance
+    def __init__(self, world_size):
+        self.storage = {}
+        self._process = None
+        self._world_size = world_size
+        self._run = multiprocessing.Value('b', 1)
+        # For implementing a barrier
+        self._lock = threading.Lock()
+        self._current_barrier = None
+
+    def __del__(self):
+        if not _exit_mode:
+            self.stop()
+
+    def _set_process(self, process):
+        self._process = process
+
+    def _process_request(self, c_socket):
+        with c_socket:
+            # Receive in KLV format
+            action_bytes = c_socket.recv(sizeof(_klv_utils.action_t))
+            if len(action_bytes) > 0:
+                action_m = _klv_utils.action_t.from_buffer_copy(action_bytes)
+                if action_m.length > 256:
+                    raise ValueError('Invalid length for message')
+                value = bytearray(action_m.value)[:action_m.length]
+                r = _store_actions.execute_action(action_m.action, value, self)
+                if r is not None:
+                    c_socket.sendall(r.klv())
+
+    def _server_loop(self, host, port):
+        # This is for minimum info exchange during initialization
+        # a single connection allows to implement locking mechanics easily
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            s.bind((host, port))
+            s.listen()
+            s.settimeout(0.5)
+            while self._run.value == 1:
+                try:
+                    c_socket, addr = s.accept()
+                except socket.timeout:
+                    continue
+
+                t = threading.Thread(
+                    target=self._process_request,
+                    args=(c_socket,), daemon=True)
+                t.start()
+
+    def run(self, host=_DEFAULT_HOST, port=_DEFAULT_PORT):
+        # Run the TCP store in a different process
+        p = ExceptionAwareProcess(
+            target=self._server_loop, args=(host, port))
+        p.start()
+        self._process = p
+
+    def stop(self):
+        if _exit_mode:
+            return  # Prevent shutdown errors
+        if self._process is not None:
+            with self._run.get_lock():
+                self._run.value = 0
+            if self._process.is_alive():
+                self._process.join()
+
+
+class TCPStoreProxy:
+
+    MAX_NUM_RETRIES = 50
+    DELAY_FOR_RETRY = 0.5
+
+    def __init__(self, host=_DEFAULT_HOST, port=_DEFAULT_PORT):
+        self.host = host
+        self.port = port
+
+    def _send_recv(self, action):
+        # Retry several times in case the rank 0 has not established the
+        # main store yet
+        for i in range(TCPStoreProxy.MAX_NUM_RETRIES):
+            try:
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                    # TODO retry connects
+                    s.connect((self.host, self.port))
+                    s.sendall(action.klv())
+                    result_bytes = s.recv(sizeof(
+                        _klv_utils.result_action_t))
+                    if len(result_bytes) > 0:
+                        result = _klv_utils.result_action_t.from_buffer_copy(
+                            result_bytes)
+                        value = bytearray(result.value)[:result.length]
+                        if result.status == 0:
+                            return action.decode_result(value)
+                        else:
+                            raise RuntimeError(value.decode('utf-8'))
+            except ConnectionRefusedError:
+                time.sleep(TCPStoreProxy.DELAY_FOR_RETRY)
+        raise RuntimeError('TCPStore is not available')
+
+    def __getitem__(self, key):
+        return self._send_recv(_store_actions.Get(key))
+
+    def __setitem__(self, key, value):
+        self._send_recv(_store_actions.Set(key, value))
+
+    def barrier(self):
+        # Barrier has special semantics
+        self._send_recv(_store_actions.Barrier())
diff --git a/cupyx/distributed/_store_actions.py b/cupyx/distributed/_store_actions.py
new file mode 100644
index 0000000..74fc69e
--- /dev/null
+++ b/cupyx/distributed/_store_actions.py
@@ -0,0 +1,177 @@
+import enum
+import threading
+
+from cupyx.distributed import _klv_utils
+
+
+class Actions(enum.IntEnum):
+    Set = 1
+    Get = 2
+    Barrier = 3
+
+
+class ActionError:
+    def __init__(self, exception):
+        self._exception = exception
+
+    def klv(self):
+        e = self._exception
+        return _klv_utils.get_result_action_t(1, str(e).encode('ascii'))
+
+    @staticmethod
+    def from_klv(klv):
+        raise RuntimeError(klv._exception.decode('utf-8'))
+
+    def decode_result(self, data):
+        ActionError.from_klv(data)
+
+
+def execute_action(action, value, store):
+    # receive the remaining amount of bytes that L field specifies
+    try:
+        if action == Actions.Set:
+            action_obj = Set.from_klv(value)
+        elif action == Actions.Get:
+            action_obj = Get.from_klv(value)
+        elif action == Actions.Barrier:
+            action_obj = Barrier.from_klv(value)
+        else:
+            raise ValueError(f'unknown action {action}')
+        return action_obj(store)
+    except Exception as e:
+        return ActionError(e)
+
+
+class Set:
+    class SetResult:
+        def klv(self):
+            v = bytearray(bytes(True))
+            action = _klv_utils.get_result_action_t(0, v)
+            return bytes(action)
+
+        @staticmethod
+        def from_klv(klv):
+            return True
+
+    def __init__(self, key, value):
+        self.key = key
+        self.value = value
+        if not isinstance(key, str):
+            raise ValueError('Invalid type for key, only str allowed')
+        if type(value) not in (bytes, bytearray, int):
+            raise ValueError(
+                'Invalid type for value, only int or bytes allowed')
+        # Check, value can only be integer or bytes
+
+    @staticmethod
+    def from_klv(value):
+        value = bytes(value)
+        for i, b in enumerate(value):
+            if b == 0:
+                k = value[:i].decode('utf-8')
+                value = value[i + 1:]
+                break
+        else:
+            raise ValueError('No separation character for key found')
+        v = _klv_utils.get_value_from_bytes(value)
+        return Set(k, v)
+
+    def klv(self):
+        v = bytearray(self.key.encode('ascii'))
+        v.append(0)  # marker to show where the value begins
+        v += _klv_utils.create_value_bytes(self.value)
+        action = _klv_utils.get_action_t(Actions.Set, v)
+        return bytes(action)
+
+    def __call__(self, store):
+        store.storage[self.key] = self.value
+        return Set.SetResult()
+
+    def decode_result(self, data):
+        return Set.SetResult.from_klv(data)
+
+
+class Get:
+    class GetResult:
+        def __init__(self, value):
+            self.value = value
+
+        def klv(self):
+            v = _klv_utils.create_value_bytes(self.value)
+            action = _klv_utils.get_result_action_t(0, v)
+            return bytes(action)
+
+        @staticmethod
+        def from_klv(value):
+            value = bytearray(value)
+            return _klv_utils.get_value_from_bytes(value)
+
+    def __init__(self, key):
+        self.key = key
+        if not isinstance(key, str):
+            raise ValueError('Invalid type for key, only str allowed')
+
+    @staticmethod
+    def from_klv(value):
+        k = value.decode('utf-8')
+        return Get(k)
+
+    def klv(self):
+        v = bytearray(self.key.encode('ascii'))
+        action = _klv_utils.get_action_t(Actions.Get, v)
+        return bytes(action)
+
+    def __call__(self, store):
+        return Get.GetResult(store.storage[self.key])
+
+    def decode_result(self, data):
+        return Get.GetResult.from_klv(data)
+
+
+class _BarrierImpl:
+    def __init__(self, world_size):
+        self._world_size = world_size
+        self._cvar = threading.Condition()
+
+    def __call__(self):
+        # Superlame implementation, should be improved
+        with self._cvar:
+            self._world_size -= 1
+            if self._world_size == 0:
+                self._cvar.notifyAll()
+            elif self._world_size > 0:
+                self._cvar.wait()
+
+
+class Barrier:
+    class BarrierResult:
+        def klv(self):
+            v = bytearray(bytes(True))
+            action = _klv_utils.get_result_action_t(0, v)
+            return bytes(action)
+
+        @staticmethod
+        def from_klv(klv):
+            # don't need to parse, barrier always sends True
+            return True
+
+    def klv(self):
+        action = _klv_utils.get_action_t(Actions.Barrier, bytes(0))
+        return bytes(action)
+
+    @staticmethod
+    def from_klv(klv):
+        return Barrier()
+
+    def __call__(self, store):
+        with store._lock:
+            if store._current_barrier is None:
+                store._current_barrier = _BarrierImpl(store._world_size)
+        store._current_barrier()
+        # Once the barrier has been completed, just clean it
+        with store._lock:
+            store._current_barrier = None
+        return Barrier.BarrierResult()
+
+    def decode_result(self, data):
+        return Barrier.BarrierResult.from_klv(data)
diff --git a/cupyx/fallback_mode/__init__.py b/cupyx/fallback_mode/__init__.py
new file mode 100644
index 0000000..ff9a64f
--- /dev/null
+++ b/cupyx/fallback_mode/__init__.py
@@ -0,0 +1,10 @@
+from cupy import _util
+
+# Attributes and Methods for fallback_mode
+# Auto-execute numpy method when corresponding cupy method is not found
+
+# "NOQA" to suppress flake8 warning
+from cupyx.fallback_mode.fallback import numpy  # NOQA
+
+
+_util.experimental('cupyx.fallback_mode.numpy')
diff --git a/cupyx/fallback_mode/fallback.py b/cupyx/fallback_mode/fallback.py
new file mode 100644
index 0000000..af87c8b
--- /dev/null
+++ b/cupyx/fallback_mode/fallback.py
@@ -0,0 +1,596 @@
+"""
+`fallback_mode` for cupy. Whenever a method is not yet implemented in CuPy,
+it will fallback to corresponding NumPy method.
+"""
+import types
+
+import numpy as np
+
+import cupy as cp
+
+
+from cupyx.fallback_mode import notification
+
+
+class _RecursiveAttr(object):
+    """
+    RecursiveAttr class to catch all attributes corresponding to numpy,
+    when user calls fallback_mode. numpy is an instance of this class.
+    """
+
+    def __init__(self, numpy_object, cupy_object, array=None):
+        """
+        _RecursiveAttr initializer.
+
+        Args:
+            numpy_object (method): NumPy method.
+            cupy_method (method): Corresponding CuPy method.
+            array (ndarray): Acts as flag to know if _RecursiveAttr object
+                is called from ``ndarray`` class. Also, acts as container for
+                modifying args in case it is called from ``ndarray``.
+                None otherwise.
+        """
+
+        self._numpy_object = numpy_object
+        self._cupy_object = cupy_object
+        self._fallback_array = array
+
+    def __instancecheck__(self, instance):
+        """
+        Enable support for isinstance(instance, _RecursiveAttr instance)
+        by redirecting it to appropriate isinstance method.
+        """
+
+        if self._cupy_object is not None:
+            return isinstance(instance, self._cupy_object)
+
+        return isinstance(instance, self._numpy_object)
+
+    def __getattr__(self, attr):
+        """
+        Catches attributes corresponding to numpy.
+
+        Runs recursively till attribute gets called.
+        Or numpy ScalarType is retrieved.
+
+        Args:
+            attr (str): Attribute of _RecursiveAttr class object.
+
+        Returns:
+            (_RecursiveAttr object, NumPy scalar):
+                Returns_RecursiveAttr object with new numpy_object,
+                cupy_object. OR
+                Returns objects in cupy which is an alias
+                of numpy object. OR
+                Returns wrapper objects, `ndarray`, `vectorize`.
+        """
+
+        numpy_object = getattr(self._numpy_object, attr)
+        cupy_object = getattr(self._cupy_object, attr, None)
+
+        if numpy_object is np.ndarray:
+            return ndarray
+
+        if numpy_object is np.vectorize:
+            return vectorize
+
+        if numpy_object is cupy_object:
+            return numpy_object
+
+        return _RecursiveAttr(numpy_object, cupy_object)
+
+    def __repr__(self):
+
+        if isinstance(self._numpy_object, types.ModuleType):
+            return "<numpy = module {}, cupy = module {}>".format(
+                self._numpy_object.__name__,
+                getattr(self._cupy_object, '__name__', None))
+
+        return "<numpy = {}, cupy = {}>".format(
+            self._numpy_object, self._cupy_object)
+
+    @property
+    def __doc__(self):
+        return self._numpy_object.__doc__
+
+    @staticmethod
+    def _is_cupy_compatible(arg):
+        """
+        Returns False if CuPy's functions never accept the arguments as
+        parameters due to the following reasons.
+        - The inputs include an object of a NumPy's specific class other than
+          `np.ndarray`.
+        - The inputs include a dtype which is not supported in CuPy.
+        """
+
+        if isinstance(arg, ndarray):
+            if not arg._supports_cupy:
+                return False
+
+        if isinstance(arg, (tuple, list)):
+            return all(_RecursiveAttr._is_cupy_compatible(i) for i in arg)
+
+        if isinstance(arg, dict):
+            bools = [_RecursiveAttr._is_cupy_compatible(arg[i]) for i in arg]
+            return all(bools)
+
+        return True
+
+    def __call__(self, *args, **kwargs):
+        """
+        Gets invoked when last attribute of _RecursiveAttr class gets called.
+        Calls _cupy_object if not None else call _numpy_object.
+
+        Args:
+            args (tuple): Arguments.
+            kwargs (dict): Keyword arguments.
+
+        Returns:
+            (res, ndarray): Returns of methods call_cupy or call_numpy
+        """
+
+        if not callable(self._numpy_object):
+            raise TypeError("'{}' object is not callable".format(
+                type(self._numpy_object).__name__))
+
+        # _RecursiveAttr gets called from ndarray
+        if self._fallback_array is not None:
+            args = ((self._fallback_array,) + args)
+
+        if self._cupy_object is not None and \
+           _RecursiveAttr._is_cupy_compatible((args, kwargs)):
+            try:
+                return _call_cupy(self._cupy_object, args, kwargs)
+            except Exception:
+                return _call_numpy(self._numpy_object, args, kwargs)
+
+        notification._dispatch_notification(self._numpy_object)
+        return _call_numpy(self._numpy_object, args, kwargs)
+
+
+numpy = _RecursiveAttr(np, cp)
+
+
+# -----------------------------------------------------------------------------
+# proxying of ndarray magic methods and wrappers
+# -----------------------------------------------------------------------------
+
+
+class ndarray(object):
+    """
+    Wrapper around cupy.ndarray
+    Supports cupy.ndarray.__init__ as well as,
+    gets initialized with a cupy ndarray.
+    """
+
+    __doc__ = np.ndarray.__doc__
+
+    def __new__(cls, *args, **kwargs):
+        """
+        If `_initial_array` and `_supports_cupy` are arguments,
+        initialize cls(ndarray).
+        Else get cupy.ndarray from provided arguments,
+        then initialize cls(ndarray).
+        """
+        _initial_array = kwargs.get('_initial_array', None)
+        if _initial_array is not None:
+            return object.__new__(cls)
+
+        cupy_ndarray_init = cp.ndarray(*args, **kwargs)
+        return cls(_initial_array=cupy_ndarray_init, _supports_cupy=True)
+
+    def __init__(self, *args, **kwargs):
+        """
+        Args:
+            _initial_array (None, cp.ndarray/np.ndarray(including variants)):
+                If _initial_array is None, object is not initialized.
+                Otherwise, _initial_array (ndarray) would be set to
+                _cupy_array and/or _numpy_array depending upon _supports_cupy.
+            _supports_cupy (bool): If _supports_cupy is True, _initial_array
+                is set as _cupy_array and _numpy_array.
+                Otherwise, _initial_array is set as only _numpy_array.
+
+        Attributes:
+            _cupy_array (None or cp.ndarray): ndarray fully compatible with
+                CuPy. This will be always set to a ndarray in GPU.
+            _numpy_array (None or np.ndarray(including variants)): ndarray not
+                supported by CuPy. Such as np.ndarray (where dtype is not in
+                '?bhilqBHILQefdFD') and it's variants. This will be always set
+                to a ndarray in CPU.
+            _supports_cupy (bool): If _supports_cupy is True, data of array
+                will contain in _cupy_array and _numpy_array.
+                Else only _numpy_array will have the data.
+        """
+
+        _supports_cupy = kwargs.pop('_supports_cupy', None)
+        _initial_array = kwargs.pop('_initial_array', None)
+        if _initial_array is None:
+            return
+
+        self._cupy_array = None
+        self._numpy_array = None
+        self.base = None
+        self._supports_cupy = _supports_cupy
+
+        assert isinstance(_initial_array, (cp.ndarray, np.ndarray))
+        if _supports_cupy:
+            if type(_initial_array) is cp.ndarray:
+                # _initial_array is in GPU memory
+                # called by _store_array_from_cupy
+                self._cupy_array = _initial_array
+                self._remember_numpy = False
+            else:
+                # _initial_array is in CPU memory
+                # called by _store_array_from_numpy
+                self._numpy_array = _initial_array
+                self._remember_numpy = True
+        else:
+            self._numpy_array = _initial_array
+
+    @classmethod
+    def _store_array_from_cupy(cls, array):
+        return cls(_initial_array=array, _supports_cupy=True)
+
+    @classmethod
+    def _store_array_from_numpy(cls, array):
+        if type(array) is np.ndarray and \
+           array.dtype.kind in '?bhilqBHILQefdFD':
+            return cls(_initial_array=array, _supports_cupy=True)
+
+        return cls(_initial_array=array, _supports_cupy=False)
+
+    @property
+    def dtype(self):
+        if self._supports_cupy and not self._remember_numpy:
+            return self._cupy_array.dtype
+        return self._numpy_array.dtype
+
+    def __getattr__(self, attr):
+        """
+        Catches attributes corresponding to ndarray.
+
+        Args:
+            attr (str): Attribute of ndarray class.
+
+        Returns:
+            (_RecursiveAttr object, self._array.attr):
+            Returns_RecursiveAttr object with numpy_object, cupy_object.
+            Returns self._array.attr if attr is not callable.
+        """
+
+        if self._supports_cupy:
+            cupy_object = getattr(cp.ndarray, attr, None)
+            numpy_object = getattr(np.ndarray, attr)
+        else:
+            cupy_object = None
+            numpy_object = getattr(self._numpy_array.__class__, attr)
+
+        if not callable(numpy_object):
+            if self._supports_cupy:
+                if self._remember_numpy:
+                    self._update_cupy_array()
+                return getattr(self._cupy_array, attr)
+            return getattr(self._numpy_array, attr)
+
+        return _RecursiveAttr(numpy_object, cupy_object, self)
+
+    def _get_cupy_array(self):
+        """
+        Returns _cupy_array (cupy.ndarray) of ndarray object. And marks
+        self(ndarray) and it's base (if exist) as numpy not up-to-date.
+        """
+        base = self.base
+        if base is not None:
+            base._remember_numpy = False
+        self._remember_numpy = False
+        return self._cupy_array
+
+    def _get_numpy_array(self):
+        """
+        Returns _numpy_array (ex: np.ndarray, numpy.ma.MaskedArray,
+        numpy.chararray etc.) of ndarray object. And marks self(ndarray)
+        and it's base (if exist) as numpy up-to-date.
+        """
+        base = self.base
+        if base is not None and base._supports_cupy:
+            base._remember_numpy = True
+        if self._supports_cupy:
+            self._remember_numpy = True
+        return self._numpy_array
+
+    def _update_numpy_array(self):
+        """
+        Updates _numpy_array from _cupy_array.
+        To be executed before calling numpy function.
+        """
+        base = self.base
+        _type = np.ndarray if self._supports_cupy \
+            else self._numpy_array.__class__
+
+        if self._supports_cupy:
+            # cupy-compatible
+            if base is None:
+                if not self._remember_numpy:
+                    if self._numpy_array is None:
+                        self._numpy_array = cp.asnumpy(self._cupy_array)
+                    else:
+                        self._cupy_array.get(out=self._numpy_array)
+            else:
+                if not base._remember_numpy:
+                    base._update_numpy_array()
+                    if self._numpy_array is None:
+                        self._numpy_array = base._numpy_array.view(type=_type)
+                        self._numpy_array.shape = self._cupy_array.shape
+                        self._numpy_array.strides = self._cupy_array.strides
+        else:
+            # not cupy-compatible
+            if base is not None:
+                assert base._supports_cupy
+                if not base._remember_numpy:
+                    base._update_numpy_array()
+
+    def _update_cupy_array(self):
+        """
+        Updates _cupy_array from _numpy_array.
+        To be executed before calling cupy function.
+        """
+        base = self.base
+
+        if base is None:
+            if self._remember_numpy:
+                if self._cupy_array is None:
+                    self._cupy_array = cp.array(self._numpy_array)
+                else:
+                    self._cupy_array[:] = self._numpy_array
+        else:
+            if base._remember_numpy:
+                base._update_cupy_array()
+
+
+def _create_magic_methods():
+    """
+    Set magic methods of cupy.ndarray as methods of fallback.ndarray.
+    """
+
+    # Decorator for ndarray magic methods
+    def make_method(name):
+        def method(self, *args, **kwargs):
+            CLASS = cp.ndarray if self._supports_cupy \
+                else self._numpy_array.__class__
+            _method = getattr(CLASS, name)
+            args = ((self,) + args)
+            if self._supports_cupy:
+                return _call_cupy(_method, args, kwargs)
+            return _call_numpy(_method, args, kwargs)
+        method.__doc__ = getattr(np.ndarray, name).__doc__
+        return method
+
+    for method in (
+        # Comparison operators:
+        '__eq__', '__ne__', '__lt__', '__gt__', '__le__', '__ge__',
+
+        # Unary operations:
+        '__neg__', '__pos__', '__abs__', '__invert__',
+
+        # Arithmetic:
+        '__add__', '__sub__', '__mul__', '__truediv__', '__floordiv__',
+        '__mod__', '__divmod__', '__pow__', '__lshift__', '__rshift__',
+        '__and__', '__or__', '__xor__',
+
+        # Arithmetic, in-place:
+        '__iadd__', '__isub__', '__imul__', '__itruediv__', '__ifloordiv__',
+        '__imod__', '__ipow__', '__ilshift__', '__irshift__',
+        '__iand__', '__ior__', '__ixor__',
+        '__matmul__',
+
+        # reflected-methods:
+        '__radd__', '__rsub__', '__rmul__', '__rtruediv__', '__rfloordiv__',
+        '__rmod__', '__rdivmod__', '__rpow__', '__rlshift__', '__rrshift__',
+        '__rand__', '__ror__', '__rxor__',
+        '__rmatmul__',
+
+        # For standard library functions:
+        '__copy__', '__deepcopy__', '__reduce__',
+
+        # Container customization:
+        '__iter__', '__len__', '__getitem__', '__setitem__',
+
+        # Conversion:
+        '__bool__', '__int__', '__float__', '__complex__',
+
+        # String representations:
+        '__repr__', '__str__'
+    ):
+        setattr(ndarray, method, make_method(method))
+
+
+_create_magic_methods()
+
+
+class vectorize(object):
+
+    __doc__ = np.vectorize.__doc__
+
+    def __init__(self, *args, **kwargs):
+        # NumPy will raise error if pyfunc is a cupy method
+        self.__dict__['_is_numpy_pyfunc'] = False
+        self.__dict__['_cupy_support'] = False
+        if isinstance(args[0], _RecursiveAttr):
+            self.__dict__['_is_numpy_pyfunc'] = True
+            if args[0]._cupy_object:
+                self.__dict__['_cupy_support'] = True
+            args = (args[0]._numpy_object,) + args[1:]
+        notification._dispatch_notification(np.vectorize)
+        self.__dict__['vec_obj'] = np.vectorize(*args, **kwargs)
+        self.__dict__['__doc__'] = self.__dict__['vec_obj'].__doc__
+
+    def __getattr__(self, attr):
+        return getattr(self.__dict__['vec_obj'], attr)
+
+    def __setattr__(self, name, value):
+        return setattr(self.vec_obj, name, value)
+
+    def __call__(self, *args, **kwargs):
+        if self._is_numpy_pyfunc:
+            notification._dispatch_notification(
+                self.vec_obj.pyfunc, self._cupy_support)
+        return _call_numpy(self.vec_obj, args, kwargs)
+
+
+# -----------------------------------------------------------------------------
+# Data Transfer methods
+# -----------------------------------------------------------------------------
+
+
+def _get_xp_args(ndarray_instance, to_xp, arg):
+    """
+    Converts ndarray_instance type object to target object using to_xp.
+
+    Args:
+        ndarray_instance (numpy.ndarray, cupy.ndarray or fallback.ndarray):
+            Objects of type `ndarray_instance` will be converted using `to_xp`.
+        to_xp (FunctionType): Method to convert ndarray_instance type objects.
+        arg (object): `ndarray_instance`, `tuple`, `list` and `dict` type
+            objects will be returned by either converting the object or it's
+            elements, if object is iterable. Objects of other types is
+            returned as it is.
+
+    Returns:
+        Return data structure will be same as before after converting ndarrays.
+    """
+
+    if isinstance(arg, ndarray_instance):
+        return to_xp(arg)
+
+    if isinstance(arg, tuple):
+        return tuple([_get_xp_args(ndarray_instance, to_xp, x) for x in arg])
+
+    if isinstance(arg, dict):
+        return {x_name: _get_xp_args(ndarray_instance, to_xp, x)
+                for x_name, x in arg.items()}
+
+    if isinstance(arg, list):
+        return [_get_xp_args(ndarray_instance, to_xp, x) for x in arg]
+
+    return arg
+
+
+def _convert_numpy_to_fallback(numpy_res):
+    return _get_xp_args(np.ndarray, ndarray._store_array_from_numpy, numpy_res)
+
+
+def _convert_fallback_to_numpy(args, kwargs):
+    return _get_xp_args(ndarray, ndarray._get_numpy_array, (args, kwargs))
+
+
+def _convert_fallback_to_cupy(args, kwargs):
+    return _get_xp_args(ndarray, ndarray._get_cupy_array, (args, kwargs))
+
+
+def _convert_cupy_to_fallback(cupy_res):
+    return _get_xp_args(cp.ndarray, ndarray._store_array_from_cupy, cupy_res)
+
+
+def _update_numpy_args(args, kwargs):
+    return _get_xp_args(ndarray, ndarray._update_numpy_array, (args, kwargs))
+
+
+def _update_cupy_args(args, kwargs):
+    return _get_xp_args(ndarray, ndarray._update_cupy_array, (args, kwargs))
+
+
+# -----------------------------------------------------------------------------
+# utils
+# -----------------------------------------------------------------------------
+
+
+def _call_cupy(func, args, kwargs):
+    """
+    Calls cupy function with *args and **kwargs and
+    does necessary data transfers.
+
+    Args:
+        func: A cupy function that needs to be called.
+        args (tuple): Arguments.
+        kwargs (dict): Keyword arguments.
+
+    Returns:
+        Result after calling func and performing data transfers.
+    """
+
+    _update_cupy_args(args, kwargs)
+    cupy_args, cupy_kwargs = _convert_fallback_to_cupy(args, kwargs)
+    cupy_res = func(*cupy_args, **cupy_kwargs)
+
+    # If existing argument is being returned
+    ext_res = _get_same_reference(
+        cupy_res, cupy_args, cupy_kwargs, args, kwargs)
+    if ext_res is not None:
+        return ext_res
+
+    if isinstance(cupy_res, cp.ndarray):
+        if cupy_res.base is None:
+            # Don't share memory
+            fallback_res = _convert_cupy_to_fallback(cupy_res)
+        else:
+            # Share memory with one of the arguments
+            base_arg = _get_same_reference(
+                cupy_res.base, cupy_args, cupy_kwargs, args, kwargs)
+            fallback_res = _convert_cupy_to_fallback(cupy_res)
+            fallback_res.base = base_arg
+        return fallback_res
+    return cupy_res
+
+
+def _call_numpy(func, args, kwargs):
+    """
+    Calls numpy function with *args and **kwargs and
+    does necessary data transfers.
+
+    Args:
+        func: A numpy function that needs to be called.
+        args (tuple): Arguments.
+        kwargs (dict): Keyword arguments.
+
+    Returns:
+        Result after calling func and performing data transfers.
+    """
+
+    _update_numpy_args(args, kwargs)
+    numpy_args, numpy_kwargs = _convert_fallback_to_numpy(args, kwargs)
+    numpy_res = func(*numpy_args, **numpy_kwargs)
+
+    # If existing argument is being returned
+    ext_res = _get_same_reference(
+        numpy_res, numpy_args, numpy_kwargs, args, kwargs)
+    if ext_res is not None:
+        return ext_res
+
+    if isinstance(numpy_res, np.ndarray):
+        if numpy_res.base is None:
+            # Don't share memory
+            fallback_res = _convert_numpy_to_fallback(numpy_res)
+        else:
+            # Share memory with one of the arguments
+            base_arg = _get_same_reference(
+                numpy_res.base, numpy_args, numpy_kwargs, args, kwargs)
+            fallback_res = _convert_numpy_to_fallback(numpy_res)
+            fallback_res.base = base_arg
+        return fallback_res
+    return numpy_res
+
+
+def _get_same_reference(res, args, kwargs, ret_args, ret_kwargs):
+    """
+    Returns object corresponding to res in (args, kwargs)
+    from (ret_args, ret_kwargs)
+    """
+    for i in range(len(args)):
+        if res is args[i]:
+            return ret_args[i]
+
+    for key in kwargs:
+        if res is kwargs[key]:
+            return ret_kwargs[key]
+
+    return
diff --git a/cupyx/fallback_mode/notification.py b/cupyx/fallback_mode/notification.py
new file mode 100644
index 0000000..9912a09
--- /dev/null
+++ b/cupyx/fallback_mode/notification.py
@@ -0,0 +1,75 @@
+"""
+Methods related to notifications.
+"""
+
+import warnings
+
+from cupyx import _ufunc_config
+
+
+def _init_warnings():
+    FallbackWarning = type('FallbackWarning', (Warning,), {})
+    warnings.simplefilter(action='always', category=FallbackWarning)
+    return FallbackWarning
+
+
+def _dispatch_notification(func, cupy_support=False):
+    """
+    Dispatch notifications using appropriate dispatch type.
+    """
+
+    dispatch_type = _ufunc_config.get_config_fallback_mode()
+
+    _module = getattr(func, '__module__', None)
+    _name = getattr(func, '__name__', None)
+
+    if not cupy_support:
+        if _name and _module:
+            msg = "'{}' method not in cupy, falling back to '{}.{}'".format(
+                _name, _module, _name)
+        elif _name:
+            msg = "'{}' method not in cupy, ".format(_name)
+            msg += "falling back to its numpy implementation"
+        else:
+            msg = "This method is not available in cupy, "
+            msg += "falling back to numpy"
+
+        if _name:
+            raise_msg = "'{}' method not found in cupy".format(_name)
+        else:
+            raise_msg = "This method is not available in cupy"
+    else:
+        if _name and _module:
+            msg = "'{}' method is available in cupy but ".format(_name)
+            msg += "cannot be used, falling back to '{}.{}'".format(
+                _module, _name)
+        elif _name:
+            msg = "'{}' method is available in cupy but ".format(_name)
+            msg += "cannot be used, falling back to its numpy implementation"
+        else:
+            msg = "This method is available in cupy, but cannot be used"
+            msg += "falling back to numpy"
+
+        if _name:
+            raise_msg = "'{}' method is available in cupy ".format(_name)
+            raise_msg += "but cannot be used"
+        else:
+            raise_msg = "This method is available in cupy but cannot be used"
+
+    if dispatch_type == 'print':
+        print("Warning: {}".format(msg))
+
+    elif dispatch_type == 'warn':
+        warnings.warn(msg, FallbackWarning, stacklevel=3)
+
+    elif dispatch_type == 'ignore':
+        pass
+
+    elif dispatch_type == 'raise':
+        raise AttributeError(raise_msg)
+
+    else:
+        assert False
+
+
+FallbackWarning = _init_warnings()
diff --git a/cupyx/jit/__init__.py b/cupyx/jit/__init__.py
new file mode 100644
index 0000000..6561cd6
--- /dev/null
+++ b/cupyx/jit/__init__.py
@@ -0,0 +1,36 @@
+from cupyx.jit._interface import rawkernel  # NOQA
+
+from cupyx.jit._interface import threadIdx  # NOQA
+from cupyx.jit._interface import blockDim  # NOQA
+from cupyx.jit._interface import blockIdx  # NOQA
+from cupyx.jit._interface import gridDim  # NOQA
+from cupyx.jit._interface import warpsize  # NOQA
+
+from cupyx.jit._builtin_funcs import range_ as range  # NOQA
+from cupyx.jit._builtin_funcs import syncthreads  # NOQA
+from cupyx.jit._builtin_funcs import syncwarp  # NOQA
+from cupyx.jit._builtin_funcs import shared_memory  # NOQA
+from cupyx.jit._builtin_funcs import atomic_add  # NOQA
+from cupyx.jit._builtin_funcs import atomic_sub  # NOQA
+from cupyx.jit._builtin_funcs import atomic_exch  # NOQA
+from cupyx.jit._builtin_funcs import atomic_min  # NOQA
+from cupyx.jit._builtin_funcs import atomic_max  # NOQA
+from cupyx.jit._builtin_funcs import atomic_inc  # NOQA
+from cupyx.jit._builtin_funcs import atomic_dec  # NOQA
+from cupyx.jit._builtin_funcs import atomic_cas  # NOQA
+from cupyx.jit._builtin_funcs import atomic_and  # NOQA
+from cupyx.jit._builtin_funcs import atomic_or  # NOQA
+from cupyx.jit._builtin_funcs import atomic_xor  # NOQA
+from cupyx.jit._builtin_funcs import grid  # NOQA
+from cupyx.jit._builtin_funcs import gridsize  # NOQA
+from cupyx.jit._builtin_funcs import laneid  # NOQA
+from cupyx.jit._builtin_funcs import shfl_sync  # NOQA
+from cupyx.jit._builtin_funcs import shfl_up_sync  # NOQA
+from cupyx.jit._builtin_funcs import shfl_down_sync  # NOQA
+from cupyx.jit._builtin_funcs import shfl_xor_sync  # NOQA
+
+from cupyx.jit import cg  # NOQA
+from cupyx.jit import cub  # NOQA
+from cupyx.jit import thrust  # NOQA
+
+_n_functions_upperlimit = 100
diff --git a/cupyx/jit/_builtin_funcs.py b/cupyx/jit/_builtin_funcs.py
new file mode 100644
index 0000000..af20387
--- /dev/null
+++ b/cupyx/jit/_builtin_funcs.py
@@ -0,0 +1,497 @@
+from typing import Any, Mapping
+import warnings
+
+import cupy
+
+from cupy_backends.cuda.api import runtime
+from cupy.cuda import device
+from cupyx.jit import _cuda_types
+from cupyx.jit import _cuda_typerules
+from cupyx.jit._internal_types import BuiltinFunc
+from cupyx.jit._internal_types import Data
+from cupyx.jit._internal_types import Constant
+from cupyx.jit._internal_types import Range
+from cupyx.jit import _compile
+
+from functools import reduce
+
+
+class RangeFunc(BuiltinFunc):
+
+    def __call__(self, *args, unroll=None):
+        """Range with loop unrolling support.
+
+        Args:
+            start (int):
+                Same as that of built-in :obj:`range`.
+            stop (int):
+                Same as that of built-in :obj:`range`.
+            step (int):
+                Same as that of built-in :obj:`range`.
+            unroll (int or bool or None):
+
+                - If `True`, add ``#pragma unroll`` directive before the
+                  loop.
+                - If `False`, add ``#pragma unroll(1)`` directive before
+                  the loop to disable unrolling.
+                - If an `int`, add ``#pragma unroll(n)`` directive before
+                  the loop, where the integer ``n`` means the number of
+                  iterations to unroll.
+                - If `None` (default), leave the control of loop unrolling
+                  to the compiler (no ``#pragma``).
+
+        .. seealso:: `#pragma unroll`_
+
+        .. _#pragma unroll:
+            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#pragma-unroll
+        """
+        super().__call__()
+
+    def call(self, env, *args, unroll=None):
+        if len(args) == 0:
+            raise TypeError('range expected at least 1 argument, got 0')
+        elif len(args) == 1:
+            start, stop, step = Constant(0), args[0], Constant(1)
+        elif len(args) == 2:
+            start, stop, step = args[0], args[1], Constant(1)
+        elif len(args) == 3:
+            start, stop, step = args
+        else:
+            raise TypeError(
+                f'range expected at most 3 argument, got {len(args)}')
+
+        if unroll is not None:
+            if not all(isinstance(x, Constant)
+                       for x in (start, stop, step, unroll)):
+                raise TypeError(
+                    'loop unrolling requires constant start, stop, step and '
+                    'unroll value')
+            unroll = unroll.obj
+            if not (isinstance(unroll, int) or isinstance(unroll, bool)):
+                raise TypeError(
+                    'unroll value expected to be of type int, '
+                    f'got {type(unroll).__name__}')
+            if unroll is False:
+                unroll = 1
+            if not (unroll is True or 0 < unroll < 1 << 31):
+                warnings.warn(
+                    'loop unrolling is ignored as the unroll value is '
+                    'non-positive or greater than INT_MAX')
+
+        if isinstance(step, Constant):
+            step_is_positive = step.obj >= 0
+        elif step.ctype.dtype.kind == 'u':
+            step_is_positive = True
+        else:
+            step_is_positive = None
+
+        stop = Data.init(stop, env)
+        start = Data.init(start, env)
+        step = Data.init(step, env)
+
+        if start.ctype.dtype.kind not in 'iu':
+            raise TypeError('range supports only for integer type.')
+        if stop.ctype.dtype.kind not in 'iu':
+            raise TypeError('range supports only for integer type.')
+        if step.ctype.dtype.kind not in 'iu':
+            raise TypeError('range supports only for integer type.')
+
+        if env.mode == 'numpy':
+            ctype = _cuda_types.Scalar(int)
+        elif env.mode == 'cuda':
+            ctype = stop.ctype
+        else:
+            assert False
+
+        return Range(start, stop, step, ctype, step_is_positive, unroll=unroll)
+
+
+class LenFunc(BuiltinFunc):
+
+    def call(self, env, *args, **kwds):
+        if len(args) != 1:
+            raise TypeError(f'len() expects only 1 argument, got {len(args)}')
+        if kwds:
+            raise TypeError('keyword arguments are not supported')
+        arg = args[0]
+        if not isinstance(arg.ctype, _cuda_types.CArray):
+            raise TypeError('len() supports only array type')
+        if not arg.ctype.ndim:
+            raise TypeError('len() of unsized array')
+        return Data(f'static_cast<long long>({arg.code}.shape()[0])',
+                    _cuda_types.Scalar('q'))
+
+
+class MinFunc(BuiltinFunc):
+
+    def call(self, env, *args, **kwds):
+        if len(args) < 2:
+            raise TypeError(
+                f'min() expects at least 2 arguments, got {len(args)}')
+        if kwds:
+            raise TypeError('keyword arguments are not supported')
+        return reduce(lambda a, b: _compile._call_ufunc(
+            cupy.minimum, (a, b), None, env), args)
+
+
+class MaxFunc(BuiltinFunc):
+
+    def call(self, env, *args, **kwds):
+        if len(args) < 2:
+            raise TypeError(
+                f'max() expects at least 2 arguments, got {len(args)}')
+        if kwds:
+            raise TypeError('keyword arguments are not supported')
+        return reduce(lambda a, b: _compile._call_ufunc(
+            cupy.maximum, (a, b), None, env), args)
+
+
+class SyncThreads(BuiltinFunc):
+
+    def __call__(self):
+        """Calls ``__syncthreads()``.
+
+        .. seealso:: `Synchronization functions`_
+
+        .. _Synchronization functions:
+            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#synchronization-functions
+        """
+        super().__call__()
+
+    def call_const(self, env):
+        return Data('__syncthreads()', _cuda_types.void)
+
+
+class SyncWarp(BuiltinFunc):
+
+    def __call__(self, *, mask=0xffffffff):
+        """Calls ``__syncwarp()``.
+
+        Args:
+            mask (int): Active threads in a warp. Default is 0xffffffff.
+
+        .. seealso:: `Synchronization functions`_
+
+        .. _Synchronization functions:
+            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#synchronization-functions
+        """
+        super().__call__()
+
+    def call(self, env, *, mask=None):
+        if runtime.is_hip:
+            if mask is not None:
+                warnings.warn(f'mask {mask} is ignored on HIP', RuntimeWarning)
+                mask = None
+
+        if mask:
+            if isinstance(mask, Constant):
+                if not (0x0 <= mask.obj <= 0xffffffff):
+                    raise ValueError('mask is out of range')
+            mask = _compile._astype_scalar(
+                mask, _cuda_types.int32, 'same_kind', env)
+            mask = Data.init(mask, env)
+            code = f'__syncwarp({mask.code})'
+        else:
+            code = '__syncwarp()'
+        return Data(code, _cuda_types.void)
+
+
+class SharedMemory(BuiltinFunc):
+
+    def __call__(self, dtype, size, alignment=None):
+        """Allocates shared memory and returns it as a 1-D array.
+
+        Args:
+            dtype (dtype):
+                The dtype of the returned array.
+            size (int or None):
+                If ``int`` type, the size of static shared memory.
+                If ``None``, declares the shared memory with extern specifier.
+            alignment (int or None): Enforce the alignment via __align__(N).
+        """
+        super().__call__()
+
+    def call_const(self, env, dtype, size, alignment=None):
+        name = env.get_fresh_variable_name(prefix='_smem')
+        ctype = _cuda_typerules.to_ctype(dtype)
+        var = Data(name, _cuda_types.SharedMem(ctype, size, alignment))
+        env.decls[name] = var
+        env.locals[name] = var
+        return Data(name, _cuda_types.Ptr(ctype))
+
+
+class AtomicOp(BuiltinFunc):
+
+    def __init__(self, op, dtypes):
+        self._op = op
+        self._name = 'atomic' + op
+        self._dtypes = dtypes
+        doc = f"""Calls the ``{self._name}`` function to operate atomically on
+        ``array[index]``. Please refer to `Atomic Functions`_ for detailed
+        explanation.
+
+        Args:
+            array: A :class:`cupy.ndarray` to index over.
+            index: A valid index such that the address to the corresponding
+                array element ``array[index]`` can be computed.
+            value: Represent the value to use for the specified operation. For
+                the case of :obj:`atomic_cas`, this is the value for
+                ``array[index]`` to compare with.
+            alt_value: Only used in :obj:`atomic_cas` to represent the value
+                to swap to.
+
+        .. seealso:: `Numba's corresponding atomic functions`_
+
+        .. _Atomic Functions:
+            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#atomic-functions
+
+        .. _Numba's corresponding atomic functions:
+            https://numba.readthedocs.io/en/stable/cuda-reference/kernel.html#synchronization-and-atomic-operations
+        """
+        self.__doc__ = doc
+
+    def __call__(self, array, index, value, alt_value=None):
+        super().__call__()
+
+    def call(self, env, array, index, value, value2=None):
+        name = self._name
+        op = self._op
+        array = Data.init(array, env)
+        if not isinstance(array.ctype, (_cuda_types.CArray, _cuda_types.Ptr)):
+            raise TypeError('The first argument must be of array type.')
+        target = _compile._indexing(array, index, env)
+        ctype = target.ctype
+        if ctype.dtype.name not in self._dtypes:
+            raise TypeError(f'`{name}` does not support {ctype.dtype} input.')
+        # On HIP, 'e' is not supported and we will never reach here
+        if (op == 'Add' and ctype.dtype.char == 'e'
+                and runtime.runtimeGetVersion() < 10000):
+            raise RuntimeError(
+                'float16 atomic operation is not supported before CUDA 10.0.')
+        value = _compile._astype_scalar(value, ctype, 'same_kind', env)
+        value = Data.init(value, env)
+        if op == 'CAS':
+            assert value2 is not None
+            # On HIP, 'H' is not supported and we will never reach here
+            if ctype.dtype.char == 'H':
+                if runtime.runtimeGetVersion() < 10010:
+                    raise RuntimeError(
+                        'uint16 atomic operation is not supported before '
+                        'CUDA 10.1')
+                if int(device.get_compute_capability()) < 70:
+                    raise RuntimeError(
+                        'uint16 atomic operation is not supported before '
+                        'sm_70')
+            value2 = _compile._astype_scalar(value2, ctype, 'same_kind', env)
+            value2 = Data.init(value2, env)
+            code = f'{name}(&{target.code}, {value.code}, {value2.code})'
+        else:
+            assert value2 is None
+            code = f'{name}(&{target.code}, {value.code})'
+        return Data(code, ctype)
+
+
+class GridFunc(BuiltinFunc):
+
+    def __init__(self, mode):
+        if mode == 'grid':
+            self._desc = 'Compute the thread index in the grid.'
+            self._eq = 'jit.threadIdx.x + jit.blockIdx.x * jit.blockDim.x'
+            self._link = 'numba.cuda.grid'
+            self._code = 'threadIdx.{n} + blockIdx.{n} * blockDim.{n}'
+        elif mode == 'gridsize':
+            self._desc = 'Compute the grid size.'
+            self._eq = 'jit.blockDim.x * jit.gridDim.x'
+            self._link = 'numba.cuda.gridsize'
+            self._code = 'blockDim.{n} * gridDim.{n}'
+        else:
+            raise ValueError('unsupported function')
+
+        doc = f"""        {self._desc}
+
+        Computation of the first integer is as follows::
+
+            {self._eq}
+
+        and for the other two integers the ``y`` and ``z`` attributes are used.
+
+        Args:
+            ndim (int): The dimension of the grid. Only 1, 2, or 3 is allowed.
+
+        Returns:
+            int or tuple:
+                If ``ndim`` is 1, an integer is returned, otherwise a tuple.
+
+        .. note::
+            This function follows the convention of Numba's
+            :func:`{self._link}`.
+        """
+        self.__doc__ = doc
+
+    def __call__(self, ndim):
+        super().__call__()
+
+    def call_const(self, env, ndim):
+        if not isinstance(ndim, int):
+            raise TypeError('ndim must be an integer')
+
+        # Numba convention: for 1D we return a single variable,
+        # otherwise a tuple
+        if ndim == 1:
+            return Data(self._code.format(n='x'), _cuda_types.uint32)
+        elif ndim == 2:
+            dims = ('x', 'y')
+        elif ndim == 3:
+            dims = ('x', 'y', 'z')
+        else:
+            raise ValueError('Only ndim=1,2,3 are supported')
+
+        elts_code = ', '.join(self._code.format(n=n) for n in dims)
+        ctype = _cuda_types.Tuple([_cuda_types.uint32]*ndim)
+        if ndim == 2:
+            return Data(f'thrust::make_pair({elts_code})', ctype)
+        else:
+            return Data(f'thrust::make_tuple({elts_code})', ctype)
+
+
+class WarpShuffleOp(BuiltinFunc):
+
+    def __init__(self, op, dtypes):
+        self._op = op
+        self._name = '__shfl_' + (op + '_' if op else '') + 'sync'
+        self._dtypes = dtypes
+        doc = f"""Calls the ``{self._name}`` function. Please refer to
+        `Warp Shuffle Functions`_ for detailed explanation.
+
+        .. _Warp Shuffle Functions:
+            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#warp-shuffle-functions
+        """
+        self.__doc__ = doc
+
+    def __call__(self, mask, var, val_id, *, width=32):
+        super().__call__()
+
+    def call(self, env, mask, var, val_id, *, width=None):
+        name = self._name
+
+        var = Data.init(var, env)
+        ctype = var.ctype
+        if ctype.dtype.name not in self._dtypes:
+            raise TypeError(f'`{name}` does not support {ctype.dtype} input.')
+
+        try:
+            mask = mask.obj
+        except Exception:
+            raise TypeError('mask must be an integer')
+        if runtime.is_hip:
+            warnings.warn(f'mask {mask} is ignored on HIP', RuntimeWarning)
+        elif not (0x0 <= mask <= 0xffffffff):
+            raise ValueError('mask is out of range')
+
+        # val_id refers to "delta" for shfl_{up, down}, "srcLane" for shfl, and
+        # "laneMask" for shfl_xor
+        if self._op in ('up', 'down'):
+            val_id_t = _cuda_types.uint32
+        else:
+            val_id_t = _cuda_types.int32
+        val_id = _compile._astype_scalar(val_id, val_id_t, 'same_kind', env)
+        val_id = Data.init(val_id, env)
+
+        if width:
+            if isinstance(width, Constant):
+                if width.obj not in (2, 4, 8, 16, 32):
+                    raise ValueError('width needs to be power of 2')
+        else:
+            width = Constant(64) if runtime.is_hip else Constant(32)
+        width = _compile._astype_scalar(
+            width, _cuda_types.int32, 'same_kind', env)
+        width = Data.init(width, env)
+
+        code = f'{name}({hex(mask)}, {var.code}, {val_id.code}'
+        code += f', {width.code})'
+        return Data(code, ctype)
+
+
+class LaneID(BuiltinFunc):
+    def __call__(self):
+        """Returns the lane ID of the calling thread, ranging in
+        ``[0, jit.warpsize)``.
+
+        .. note::
+            Unlike :obj:`numba.cuda.laneid`, this is a callable function
+            instead of a property.
+        """
+        super().__call__()
+
+    def _get_preamble(self):
+        preamble = '__device__ __forceinline__ unsigned int LaneId() {'
+        if not runtime.is_hip:
+            # see https://github.com/NVIDIA/cub/blob/main/cub/util_ptx.cuh#L419
+            preamble += """
+                unsigned int ret;
+                asm ("mov.u32 %0, %%laneid;" : "=r"(ret) );
+                return ret; }
+            """
+        else:
+            # defined in hip/hcc_detail/device_functions.h
+            preamble += """
+                return __lane_id(); }
+            """
+        return preamble
+
+    def call_const(self, env):
+        env.generated.add_code(self._get_preamble())
+        return Data('LaneId()', _cuda_types.uint32)
+
+
+builtin_functions_dict: Mapping[Any, BuiltinFunc] = {
+    range: RangeFunc(),
+    len: LenFunc(),
+    min: MinFunc(),
+    max: MaxFunc(),
+}
+
+range_ = RangeFunc()
+syncthreads = SyncThreads()
+syncwarp = SyncWarp()
+shared_memory = SharedMemory()
+grid = GridFunc('grid')
+gridsize = GridFunc('gridsize')
+laneid = LaneID()
+
+# atomic functions
+atomic_add = AtomicOp(
+    'Add',
+    ('int32', 'uint32', 'uint64', 'float32', 'float64')
+    + (() if runtime.is_hip else ('float16',)))
+atomic_sub = AtomicOp(
+    'Sub', ('int32', 'uint32'))
+atomic_exch = AtomicOp(
+    'Exch', ('int32', 'uint32', 'uint64', 'float32'))
+atomic_min = AtomicOp(
+    'Min', ('int32', 'uint32', 'uint64'))
+atomic_max = AtomicOp(
+    'Max', ('int32', 'uint32', 'uint64'))
+atomic_inc = AtomicOp(
+    'Inc', ('uint32',))
+atomic_dec = AtomicOp(
+    'Dec', ('uint32',))
+atomic_cas = AtomicOp(
+    'CAS',
+    ('int32', 'uint32', 'uint64')
+    + (() if runtime.is_hip else ('uint16',)))
+atomic_and = AtomicOp(
+    'And', ('int32', 'uint32', 'uint64'))
+atomic_or = AtomicOp(
+    'Or', ('int32', 'uint32', 'uint64'))
+atomic_xor = AtomicOp(
+    'Xor', ('int32', 'uint32', 'uint64'))
+
+# warp-shuffle functions
+_shfl_dtypes = (
+    ('int32', 'uint32', 'int64', 'float32', 'float64')
+    + (() if runtime.is_hip else ('uint64', 'float16')))
+shfl_sync = WarpShuffleOp('', _shfl_dtypes)
+shfl_up_sync = WarpShuffleOp('up', _shfl_dtypes)
+shfl_down_sync = WarpShuffleOp('down', _shfl_dtypes)
+shfl_xor_sync = WarpShuffleOp('xor', _shfl_dtypes)
diff --git a/cupyx/jit/_compile.py b/cupyx/jit/_compile.py
new file mode 100644
index 0000000..37b3e3b
--- /dev/null
+++ b/cupyx/jit/_compile.py
@@ -0,0 +1,1009 @@
+import ast
+import collections
+import inspect
+import linecache
+import numbers
+import re
+import sys
+from typing import Any, Dict, List, Optional, Sequence, Tuple, TypeVar, Union
+import warnings
+import types
+
+import numpy
+
+from cupy_backends.cuda.api import runtime
+from cupy._core._codeblock import CodeBlock, _CodeType
+from cupy._core import _kernel
+from cupy._core._dtype import _raise_if_invalid_cast
+from cupyx import jit
+from cupyx.jit import _cuda_types
+from cupyx.jit import _cuda_typerules
+from cupyx.jit import _internal_types
+from cupyx.jit._internal_types import Data
+from cupyx.jit._internal_types import Constant
+from cupyx.jit import _builtin_funcs
+from cupyx.jit import _interface
+
+
+_is_debug_mode = False
+
+_typeclasses = (bool, numpy.bool_, numbers.Number)
+
+if (3, 8) <= sys.version_info:
+    from typing import Literal
+    _CastingType = Optional[
+        Literal['no', 'equiv', 'safe', 'same_kind', 'unsafe']]
+else:
+    _CastingType = str
+
+Result = collections.namedtuple(
+    'Result',
+    [
+        'func_name',
+        'code',
+        'return_type',
+        'enable_cooperative_groups',
+        'backend'
+    ])
+
+
+class _JitCompileError(Exception):
+
+    def __init__(self, e, node):
+        self.error_type = type(e)
+        self.mes = str(e)
+        self.node = node
+
+    def reraise(self, pycode):
+        start = self.node.lineno
+        end = getattr(self.node, 'end_lineno', start)
+        pycode = '\n'.join([
+            (f'> {line}' if start <= i + 1 <= end else f'  {line}').rstrip()
+            for i, line in enumerate(pycode.split('\n'))])
+        raise self.error_type(self.mes + '\n\n' + pycode)
+
+
+def transpile_function_wrapper(func):
+    def new_func(node, *args, **kwargs):
+        try:
+            return func(node, *args, **kwargs)
+        except _JitCompileError:
+            raise
+        except Exception as e:
+            raise _JitCompileError(e, node)
+
+    return new_func
+
+
+def _parse_function_object(func):
+    """Returns the tuple of ``ast.FunctionDef`` object and the source string
+    for the given callable ``func``.
+
+    ``func`` can be a ``def`` function or a ``lambda`` expression.
+
+    The source is returned only for informational purposes (i.e., rendering
+    an exception message in case of an error).
+    """
+    if not callable(func):
+        raise ValueError('`func` must be a callable object.')
+
+    try:
+        # ``filename`` can be any of:
+        # - A "real" file path on the filesystem
+        # - "<stdin>" (within Python interpreter)
+        # - "<ipython-input-XXXXXXXX>" (within IPython interpreter)
+        filename = inspect.getsourcefile(func)
+    except TypeError:
+        # Built-in function or method, or inside Doctest
+        filename = None
+
+    if filename == '<stdin>':
+        raise RuntimeError(
+            f'JIT needs access to the Python source code for {func}'
+            ' but it cannot be retrieved within the Python interactive'
+            ' interpreter. Consider using IPython instead.')
+
+    if func.__name__ != '<lambda>':
+        lines, _ = inspect.getsourcelines(func)
+        num_indent = len(lines[0]) - len(lines[0].lstrip())
+        source = ''.join([
+            line.replace(' ' * num_indent, '', 1) for line in lines])
+        tree = ast.parse(source)
+        assert isinstance(tree, ast.Module)
+        assert len(tree.body) == 1
+        return tree.body[0], source
+
+    if filename is None:
+        # filename is needed for lambdas.
+        raise ValueError(
+            f'JIT needs access to Python source code for {func}'
+            ' but could not be located.\n'
+            '(hint: it is likely you passed a built-in function or method)')
+
+    # Extract the AST of the lambda from the AST of the whole source file
+    # that defines that lambda.
+    # This is needed because ``inspect.getsourcelines(lambda_expr)`` may
+    # return unparsable code snippet.
+
+    # Use ``linecache.getlines`` instead of directly opening a file to
+    # support notebook environments.
+    full_source = ''.join(linecache.getlines(filename))
+    source, start_line = inspect.getsourcelines(func)
+    end_line = start_line + len(source)
+    source = ''.join(source)
+
+    tree = ast.parse(full_source)
+
+    nodes = [node for node in ast.walk(tree)
+             if isinstance(node, ast.Lambda)
+             and start_line <= node.lineno < end_line]
+    if len(nodes) > 1:
+        # TODO(kmaehashi): can be improved by heuristics (e.g. number of args)
+        raise ValueError('Multiple callables are found near the'
+                         f' definition of {func}, and JIT could not'
+                         ' identify the source code for it.')
+    node = nodes[0]
+    return ast.FunctionDef(
+        name='_lambda_kernel', args=node.args,
+        body=[ast.Return(node.body)],
+        decorator_list=[], returns=None, type_comment=None,
+    ), source
+
+
+class Generated:
+
+    def __init__(self):
+        # list of str
+        self.codes: List[str] = []
+        # (function, in_types) => Optional(function_name, return_type)
+        self.device_function: \
+            Dict[Tuple[Any, Tuple[_cuda_types.TypeBase, ...]],
+                 Tuple[str, _cuda_types.TypeBase]] = {}
+        # whether to use cooperative launch
+        self.enable_cg = False
+        # whether to include cooperative_groups.h
+        self.include_cg = False
+        # whether to include cooperative_groups/memcpy_async.h
+        self.include_cg_memcpy_async = False
+        # whether to include cuda/barrier
+        self.include_cuda_barrier = False
+
+        # workaround for hipRTC: as of ROCm 4.1.0 hipRTC still does not
+        # recognize "-D", so we have to compile using hipcc...
+        self.backend = 'nvcc' if runtime.is_hip else 'nvrtc'
+
+    def add_code(self, code: str) -> None:
+        if code not in self.codes:
+            self.codes.append(code)
+            if len(self.codes) > jit._n_functions_upperlimit:
+                raise ValueError("Number of functions exceeds upper limit.")
+
+
+def transpile(func, attributes, mode, in_types, ret_type):
+    """Transpiles the target function.
+
+    Args:
+        func (function): Target function.
+        attributes (list of str): Attributes of the generated CUDA function.
+        mode ('numpy' or 'cuda'): The rule for typecast.
+        in_types (list of _cuda_types.TypeBase): Types of the arguments.
+        ret_type (_cuda_types.TypeBase or None): Type of the return value.
+    """
+    generated = Generated()
+    in_types = tuple(in_types)
+    name, return_type = _transpile_func_obj(
+        func, attributes, mode, in_types, ret_type, generated)
+    func_name, _ = generated.device_function[(func, in_types)]
+    code = '\n'.join(generated.codes)
+    backend = generated.backend
+    enable_cg = generated.enable_cg
+
+    if _is_debug_mode:
+        print(code)
+
+    return Result(
+        func_name=func_name, code=code, return_type=return_type,
+        enable_cooperative_groups=enable_cg, backend=backend)
+
+
+def _transpile_func_obj(func, attributes, mode, in_types, ret_type, generated):
+    if (func, in_types) in generated.device_function:
+        result = generated.device_function[(func, in_types)]
+        if result is None:
+            raise ValueError("Recursive function is not supported.")
+        return result
+
+    # Do sanity check first.
+    tree, source = _parse_function_object(func)
+
+    cvars = inspect.getclosurevars(func)
+    consts = dict(**cvars.globals, **cvars.nonlocals, **cvars.builtins)
+    attributes = ' '.join(attributes)
+    name = tree.name
+    if len(generated.device_function) > 0:
+        name += '_' + str(len(generated.device_function))
+    generated.device_function[(func, in_types)] = None
+
+    cuda_code, env = _transpile_function(
+        tree, name, attributes, mode, consts,
+        in_types, ret_type, generated, source=source)
+
+    generated.device_function[(func, in_types)] = (name, env.ret_type)
+    generated.add_code(cuda_code)
+    return name, env.ret_type
+
+
+def _indent(lines: List[str], spaces: str = '  ') -> List[str]:
+    return [spaces + line for line in lines]
+
+
+def is_constants(*values: _internal_types.Expr) -> bool:
+    assert all(isinstance(x, _internal_types.Expr) for x in values)
+    return all(isinstance(x, Constant) for x in values)
+
+
+class Environment:
+    """Environment of the scope
+
+    Attributes:
+        mode ('numpy' or 'cuda'): The rule for typecast.
+        consts (dict): The dictionary with keys as the variable names and
+            the values as the data that is determined at compile-time.
+        params (dict): The dictionary of function arguments with keys as
+            the variable names and the values as the Data.
+        locals (dict): The dictionary with keys as the variable names and the
+            values as the Data stored at the local scope of the function.
+        ret_type (_cuda_types.TypeBase):
+            The type of return value of the function.
+            If it is initialized to be ``None``, the return type must be
+            inferred until the end of transpilation of the function.
+        generated (Generated): Generated CUDA functions.
+    """
+
+    def __init__(
+            self,
+            mode: str,
+            consts: Dict[str, Constant],
+            params: Dict[str, Data],
+            ret_type: _cuda_types.TypeBase,
+            generated: Generated,
+    ):
+        self.mode = mode
+        self.consts = consts
+        self.params = params
+        self.locals: Dict[str, Data] = {}
+        self.decls: Dict[str, Data] = {}
+        self.ret_type = ret_type
+        self.generated = generated
+        self.count = 0
+
+    def __getitem__(self, key: str) -> Optional[Union[Constant, Data]]:
+        if key in self.locals:
+            return self.locals[key]
+        if key in self.params:
+            return self.params[key]
+        if key in self.consts:
+            return self.consts[key]
+        return None
+
+    def get_fresh_variable_name(
+            self, prefix: str = '', suffix: str = '') -> str:
+        self.count += 1
+        name = f'{prefix}{self.count}{suffix}'
+        if self[name] is None:
+            return name
+        else:
+            return self.get_fresh_variable_name(prefix, suffix)
+
+
+def _transpile_function(
+        func, name, attributes, mode, consts,
+        in_types, ret_type, generated, *, source):
+    """Transpile the function
+    Args:
+        func (ast.FunctionDef): Target function.
+        name (str): Function name.
+        attributes (str): The attributes of target function.
+        mode ('numpy' or 'cuda'): The rule for typecast.
+        consts (dict): The dictionary with keys as variable names and
+            values as concrete data object.
+        in_types (list of _cuda_types.TypeBase): The types of arguments.
+        ret_type (_cuda_types.TypeBase): The type of return value.
+
+    Returns:
+        code (str): The generated CUDA code.
+        env (Environment): More details of analysis result of the function,
+            which includes preambles, estimated return type and more.
+    """
+    try:
+        return _transpile_function_internal(
+            func, name, attributes, mode, consts,
+            in_types, ret_type, generated)
+    except _JitCompileError as e:
+        exc = e
+        if _is_debug_mode:
+            exc.reraise(source)
+
+    # Raises the error out of `except` block to clean stack trace.
+    exc.reraise(source)
+    assert False
+
+
+def _transpile_function_internal(
+        func, name, attributes, mode, consts, in_types, ret_type, generated):
+    consts = dict([(k, Constant(v)) for k, v, in consts.items()])
+
+    if not isinstance(func, ast.FunctionDef):
+        # TODO(asi1024): Support for `ast.ClassDef`.
+        raise NotImplementedError('Not supported: {}'.format(type(func)))
+    if len(func.decorator_list) > 0:
+        if sys.version_info >= (3, 9):
+            # Code path for Python versions that support `ast.unparse`.
+            for deco in func.decorator_list:
+                deco_code = ast.unparse(deco)
+                if not any(word in deco_code
+                           for word in ['rawkernel', 'vectorize']):
+                    warnings.warn(
+                        f'Decorator {deco_code} may not supported in JIT.',
+                        RuntimeWarning)
+    arguments = func.args
+    if arguments.vararg is not None:
+        raise NotImplementedError('`*args` is not supported currently.')
+    if len(arguments.kwonlyargs) > 0:  # same length with `kw_defaults`.
+        raise NotImplementedError(
+            'keyword only arguments are not supported currently .')
+    if arguments.kwarg is not None:
+        raise NotImplementedError('`**kwargs` is not supported currently.')
+    if len(arguments.defaults) > 0:
+        raise NotImplementedError(
+            'Default values are not supported currently.')
+
+    args = [arg.arg for arg in arguments.args]
+    if len(args) != len(in_types):
+        raise TypeError(
+            f'{name}() takes {len(args)} positional arguments '
+            f'but {len(in_types)} were given.')
+    params = dict([(x, Data(x, t)) for x, t in zip(args, in_types)])
+    env = Environment(mode, consts, params, ret_type, generated)
+    body = _transpile_stmts(func.body, True, env)
+    params_s = ', '.join([t.declvar(x, None) for x, t in zip(args, in_types)])
+    local_vars = [v.ctype.declvar(n, None) + ';' for n, v in env.decls.items()]
+
+    if env.ret_type is None:
+        env.ret_type = _cuda_types.void
+
+    head = f'{attributes} {env.ret_type} {name}({params_s})'
+    code = CodeBlock(head, local_vars + body)
+    return str(code), env
+
+
+def _eval_operand(
+        op: ast.AST,
+        args: Sequence[Union[Constant, Data]],
+        env: Environment,
+) -> Union[Constant, Data]:
+    if is_constants(*args):
+        pyfunc = _cuda_typerules.get_pyfunc(type(op))
+        return Constant(pyfunc(*[x.obj for x in args]))
+
+    if isinstance(op, ast.Add):
+        x, y = args
+        x = Data.init(x, env)
+        y = Data.init(y, env)
+        if hasattr(x.ctype, '_add'):
+            out = x.ctype._add(env, x, y)
+            if out is not NotImplemented:
+                return out
+        if hasattr(y.ctype, '_radd'):
+            out = y.ctype._radd(env, x, y)
+            if out is not NotImplemented:
+                return out
+
+    if isinstance(op, ast.Sub):
+        x, y = args
+        x = Data.init(x, env)
+        y = Data.init(y, env)
+        if hasattr(x.ctype, '_sub'):
+            out = x.ctype._sub(env, x, y)
+            if out is not NotImplemented:
+                return out
+        if hasattr(y.ctype, '_rsub'):
+            out = y.ctype._rsub(env, x, y)
+            if out is not NotImplemented:
+                return out
+
+    ufunc = _cuda_typerules.get_ufunc(env.mode, type(op))
+    return _call_ufunc(ufunc, args, None, env)
+
+
+def _call_ufunc(
+        ufunc: _kernel.ufunc,
+        args: Sequence[Union[Constant, Data]],
+        dtype: Optional[numpy.dtype],
+        env: Environment,
+) -> Data:
+    if len(args) != ufunc.nin:
+        raise ValueError('invalid number of arguments')
+
+    in_types = []
+    for x in args:
+        if isinstance(x, Constant):
+            t = _cuda_typerules.get_ctype_from_scalar(env.mode, x.obj).dtype
+        elif isinstance(x.ctype, _cuda_types.Scalar):
+            t = x.ctype.dtype
+        else:
+            raise TypeError(f'cupy.ufunc: {x.ctype} is unsupported')
+        in_types.append(t)
+
+    op = _cuda_typerules.guess_routine(ufunc, tuple(in_types), dtype, env.mode)
+
+    if op is None:
+        raise TypeError(
+            f'"{ufunc.name}" does not support for the input types: {in_types}')
+
+    if op.error_func is not None:
+        op.error_func()
+
+    if ufunc.nout == 1 and op.routine.startswith('out0 = '):
+        out_type = _cuda_types.Scalar(op.out_types[0])
+        expr = op.routine.replace('out0 = ', '')
+
+        in_params = []
+        for x, t in zip(args, op.in_types):
+            x = _astype_scalar(x, _cuda_types.Scalar(t), 'same_kind', env)
+            x = Data.init(x, env)
+            in_params.append(x)
+
+        can_use_inline_expansion = True
+        for i in range(ufunc.nin):
+            if len(list(re.finditer(r'in{}'.format(i), op.routine))) > 1:
+                can_use_inline_expansion = False
+            if f'in{i}_type' in op.routine:
+                can_use_inline_expansion = False
+
+        env.generated.add_code(ufunc._preamble)
+
+        if can_use_inline_expansion:
+            # Code pass for readable generated code
+            for i, x in enumerate(in_params):
+                expr = expr.replace(f'in{i}', x.code)
+            expr = '(' + expr.replace('out0_type', str(out_type)) + ')'
+        else:
+            template_typenames = ', '.join([
+                f'typename in{i}_type' for i in range(ufunc.nin)])
+            ufunc_name = f'{ufunc.name}_{str(numpy.dtype(op.out_types[0]))}'
+            params = ', '.join([f'in{i}_type in{i}' for i in range(ufunc.nin)])
+            ufunc_code = f"""template <{template_typenames}>
+__device__ {out_type} {ufunc_name}({params}) {{
+    typedef {out_type} out0_type;
+    return {expr};
+}}
+"""
+            env.generated.add_code(ufunc_code)
+            in_params_code = ', '.join([a.code for a in in_params])
+            expr = f'{ufunc_name}({in_params_code})'
+        return Data(expr, out_type)
+
+    raise NotImplementedError(f'ufunc `{ufunc.name}` is not supported.')
+
+
+def _transpile_stmts(
+        stmts: List[ast.stmt],
+        is_toplevel: bool,
+        env: Environment,
+) -> _CodeType:
+    codeblocks = []
+    for stmt in stmts:
+        codeblocks.extend(_transpile_stmt(stmt, is_toplevel, env))
+    return codeblocks
+
+
+@transpile_function_wrapper
+def _transpile_stmt(
+        stmt: ast.stmt,
+        is_toplevel: bool,
+        env: Environment,
+) -> _CodeType:
+    """Transpile the statement.
+
+    Returns (list of [CodeBlock or str]): The generated CUDA code.
+    """
+
+    if isinstance(stmt, ast.ClassDef):
+        raise NotImplementedError('class is not supported currently.')
+    if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
+        raise NotImplementedError(
+            'Nested functions are not supported currently.')
+    if isinstance(stmt, ast.Return):
+        value = _transpile_expr(stmt.value, env)
+        value = Data.init(value, env)
+        t = value.ctype
+        if env.ret_type is None:
+            env.ret_type = t
+        elif env.ret_type != t:
+            raise ValueError(
+                f'Failed to infer the return type: {env.ret_type} or {t}')
+        return [f'return {value.code};']
+    if isinstance(stmt, ast.Delete):
+        raise NotImplementedError('`del` is not supported currently.')
+
+    if isinstance(stmt, ast.Assign):
+        if len(stmt.targets) != 1:
+            raise NotImplementedError('Not implemented.')
+
+        value = _transpile_expr(stmt.value, env)
+        var = stmt.targets[0]
+
+        if is_constants(value) and isinstance(var, ast.Name):
+            name = var.id
+            if not isinstance(value.obj, _typeclasses):
+                if is_toplevel:
+                    if isinstance(env[name], Data):
+                        raise TypeError(f'Type mismatch of variable: `{name}`')
+                    env.consts[name] = value
+                    return []
+                else:
+                    raise TypeError(
+                        'Cannot assign constant value not at top-level.')
+
+        value = Data.init(value, env)
+        return _transpile_assign_stmt(var, env, value, is_toplevel)
+
+    if isinstance(stmt, ast.AugAssign):
+        value = _transpile_expr(stmt.value, env)
+        target = _transpile_expr(stmt.target, env)
+        if not isinstance(target, Data):
+            raise TypeError(f'Cannot augassign to {target.code}')
+        value = Data.init(value, env)
+        tmp = Data(env.get_fresh_variable_name('_tmp_'), target.ctype)
+        result = _eval_operand(stmt.op, (tmp, value), env)
+        assert isinstance(target, Data)
+        assert isinstance(result, Data)
+        assert isinstance(target.ctype, _cuda_types.Scalar)
+        assert isinstance(result.ctype, _cuda_types.Scalar)
+        _raise_if_invalid_cast(
+            result.ctype.dtype, target.ctype.dtype, 'same_kind')
+
+        return ['{ ' + target.ctype.declvar('&' + tmp.code, target) + '; ' +
+                target.ctype.assign(tmp, result) + '; }']
+
+    if isinstance(stmt, ast.For):
+        if len(stmt.orelse) > 0:
+            raise NotImplementedError('while-else is not supported.')
+        assert isinstance(stmt.target, ast.Name)
+        name = stmt.target.id
+        iters = _transpile_expr(stmt.iter, env)
+        loop_var = env[name]
+
+        if loop_var is None:
+            target = Data(stmt.target.id, iters.ctype)
+            env.locals[name] = target
+            env.decls[name] = target
+        elif isinstance(loop_var, Constant):
+            raise TypeError('loop counter must not be constant value')
+        elif loop_var.ctype != iters.ctype:
+            raise TypeError(
+                f'Data type mismatch of variable: `{name}`: '
+                f'{loop_var.ctype} != {iters.ctype}')
+
+        if not isinstance(iters, _internal_types.Range):
+            raise NotImplementedError(
+                'for-loop is supported only for range iterator.')
+
+        body = _transpile_stmts(stmt.body, False, env)
+
+        init_code = (f'{iters.ctype} '
+                     f'__it = {iters.start.code}, '
+                     f'__stop = {iters.stop.code}, '
+                     f'__step = {iters.step.code}')
+        cond = '__step >= 0 ? __it < __stop : __it > __stop'
+        if iters.step_is_positive is True:
+            cond = '__it < __stop'
+        elif iters.step_is_positive is False:
+            cond = '__it > __stop'
+
+        head = f'for ({init_code}; {cond}; __it += __step)'
+        code: _CodeType = [CodeBlock(head, [f'{name} = __it;'] + body)]
+
+        unroll = iters.unroll
+        if unroll is True:
+            code = ['#pragma unroll'] + code
+        elif unroll is not None:
+            code = [f'#pragma unroll({unroll})'] + code
+        return code
+
+    if isinstance(stmt, ast.AsyncFor):
+        raise ValueError('`async for` is not allowed.')
+    if isinstance(stmt, ast.While):
+        if len(stmt.orelse) > 0:
+            raise NotImplementedError('while-else is not supported.')
+        condition = _transpile_expr(stmt.test, env)
+        condition = _astype_scalar(condition, _cuda_types.bool_, 'unsafe', env)
+        condition = Data.init(condition, env)
+        body = _transpile_stmts(stmt.body, False, env)
+        head = f'while ({condition.code})'
+        return [CodeBlock(head, body)]
+    if isinstance(stmt, ast.If):
+        condition = _transpile_expr(stmt.test, env)
+        if is_constants(condition):
+            stmts = stmt.body if condition.obj else stmt.orelse
+            return _transpile_stmts(stmts, is_toplevel, env)
+        head = f'if ({condition.code})'
+        then_body = _transpile_stmts(stmt.body, False, env)
+        else_body = _transpile_stmts(stmt.orelse, False, env)
+        return [CodeBlock(head, then_body), CodeBlock('else', else_body)]
+    if isinstance(stmt, (ast.With, ast.AsyncWith)):
+        raise ValueError('Switching contexts are not allowed.')
+    if isinstance(stmt, (ast.Raise, ast.Try)):
+        raise ValueError('throw/catch are not allowed.')
+    if isinstance(stmt, ast.Assert):
+        value = _transpile_expr(stmt.test, env)
+        if is_constants(value):
+            assert value.obj
+            return [';']
+        else:
+            return ['assert(' + value + ');']
+    if isinstance(stmt, (ast.Import, ast.ImportFrom)):
+        raise ValueError('Cannot import modules from the target functions.')
+    if isinstance(stmt, (ast.Global, ast.Nonlocal)):
+        raise ValueError('Cannot use global/nonlocal in the target functions.')
+    if isinstance(stmt, ast.Expr):
+        value = _transpile_expr(stmt.value, env)
+        return [';'] if is_constants(value) else [value.code + ';']
+    if isinstance(stmt, ast.Pass):
+        return [';']
+    if isinstance(stmt, ast.Break):
+        raise NotImplementedError('Not implemented.')
+    if isinstance(stmt, ast.Continue):
+        raise NotImplementedError('Not implemented.')
+    assert False
+
+
+@transpile_function_wrapper
+def _transpile_expr(expr: ast.expr, env: Environment) -> _internal_types.Expr:
+    """Transpile the statement.
+
+    Returns (Data): The CUDA code and its type of the expression.
+    """
+    res = _transpile_expr_internal(expr, env)
+
+    if isinstance(res, Constant) and isinstance(res.obj, _internal_types.Expr):
+        return res.obj
+    else:
+        return res
+
+
+def _transpile_expr_internal(
+        expr: ast.expr,
+        env: Environment,
+) -> _internal_types.Expr:
+    if isinstance(expr, ast.BoolOp):
+        values = [_transpile_expr(e, env) for e in expr.values]
+        value = values[0]
+        for rhs in values[1:]:
+            value = _eval_operand(expr.op, (value, rhs), env)
+        return value
+    if isinstance(expr, ast.BinOp):
+        left = _transpile_expr(expr.left, env)
+        right = _transpile_expr(expr.right, env)
+        return _eval_operand(expr.op, (left, right), env)
+    if isinstance(expr, ast.UnaryOp):
+        value = _transpile_expr(expr.operand, env)
+        return _eval_operand(expr.op, (value,), env)
+    if isinstance(expr, ast.Lambda):
+        raise NotImplementedError('Not implemented.')
+    if isinstance(expr, ast.Compare):
+        values = [expr.left] + expr.comparators
+        if len(values) != 2:
+            raise NotImplementedError(
+                'Comparison of 3 or more values is not implemented.')
+        values = [_transpile_expr(e, env) for e in values]
+        return _eval_operand(expr.ops[0], values, env)
+    if isinstance(expr, ast.IfExp):
+        cond = _transpile_expr(expr.test, env)
+        x = _transpile_expr(expr.body, env)
+        y = _transpile_expr(expr.orelse, env)
+
+        if isinstance(expr, Constant):
+            return x if expr.obj else y
+        if cond.ctype.dtype.kind == 'c':
+            raise TypeError("Complex type value cannot be boolean condition.")
+        x, y = _infer_type(x, y, env), _infer_type(y, x, env)
+        if x.ctype.dtype != y.ctype.dtype:
+            raise TypeError(
+                'Type mismatch in conditional expression.: '
+                f'{x.ctype.dtype} != {y.ctype.dtype}')
+        cond = _astype_scalar(cond, _cuda_types.bool_, 'unsafe', env)
+        return Data(f'({cond.code} ? {x.code} : {y.code})', x.ctype)
+
+    if isinstance(expr, ast.Call):
+        func = _transpile_expr(expr.func, env)
+        args = [_transpile_expr(x, env) for x in expr.args]
+        kwargs: Dict[str, Union[Constant, Data]] = {}
+        for kw in expr.keywords:
+            assert kw.arg is not None
+            kwargs[kw.arg] = _transpile_expr(kw.value, env)
+
+        builtin_funcs = _builtin_funcs.builtin_functions_dict
+        if is_constants(func) and (func.obj in builtin_funcs):
+            func = builtin_funcs[func.obj]
+
+        if isinstance(func, _internal_types.BuiltinFunc):
+            return func.call(env, *args, **kwargs)
+
+        if not isinstance(func, Constant):
+            raise TypeError(f"'{func}' is not callable.")
+
+        func = func.obj
+
+        if isinstance(func, _interface._cuda_types.TypeBase):
+            return func._instantiate(env, *args, **kwargs)
+
+        if isinstance(func, _interface._JitRawKernel):
+            if not func._device:
+                raise TypeError(
+                    f'Calling __global__ function {func._func.__name__} '
+                    'from __global__ funcion is not allowed.')
+            args = [Data.init(x, env) for x in args]
+            in_types = tuple([x.ctype for x in args])
+            fname, return_type = _transpile_func_obj(
+                func._func, ['__device__'], env.mode,
+                in_types, None, env.generated)
+            in_params = ', '.join([x.code for x in args])
+            return Data(f'{fname}({in_params})', return_type)
+
+        if isinstance(func, _kernel.ufunc):
+            # ufunc call
+            dtype = kwargs.pop('dtype', Constant(None)).obj
+            if len(kwargs) > 0:
+                name = next(iter(kwargs))
+                raise TypeError(
+                    f"'{name}' is an invalid keyword to ufunc {func.name}")
+            return _call_ufunc(func, args, dtype, env)
+
+        if is_constants(*args, *kwargs.values()):
+            # compile-time function call
+            args = [x.obj for x in args]
+            kwargs = dict([(k, v.obj) for k, v in kwargs.items()])
+            return Constant(func(*args, **kwargs))
+
+        if inspect.isclass(func) and issubclass(func, _typeclasses):
+            # explicit typecast
+            if len(args) != 1:
+                raise TypeError(
+                    f'function takes {func} invalid number of argument')
+            ctype = _cuda_types.Scalar(func)
+            return _astype_scalar(args[0], ctype, 'unsafe', env)
+
+        raise TypeError(f"Invalid function call '{func.__name__}'.")
+
+    if isinstance(expr, ast.Constant):
+        return Constant(expr.value)
+    if isinstance(expr, ast.Subscript):
+        array = _transpile_expr(expr.value, env)
+        index = _transpile_expr(expr.slice, env)
+        return _indexing(array, index, env)
+    if isinstance(expr, ast.Name):
+        value = env[expr.id]
+        if value is None:
+            raise NameError(f'Unbound name: {expr.id}')
+        return value
+    if isinstance(expr, ast.Attribute):
+        value = _transpile_expr(expr.value, env)
+        if isinstance(value, Constant):
+            return Constant(getattr(value.obj, expr.attr))
+        if isinstance(value, Data) and hasattr(value.ctype, expr.attr):
+            attr = getattr(value.ctype, expr.attr, None)
+            if isinstance(attr, types.MethodType):
+                return attr(value)
+            if isinstance(attr, Data):
+                return attr
+        raise AttributeError(f'Unknown attribute: {expr.attr}')
+
+    if isinstance(expr, ast.Tuple):
+        elts = [_transpile_expr(x, env) for x in expr.elts]
+
+        if all([isinstance(x, Constant) for x in elts]):
+            return Constant(tuple([x.obj for x in elts]))
+
+        elts = [Data.init(x, env) for x in elts]
+        elts_code = ', '.join([x.code for x in elts])
+        if len(elts) == 2:
+            return Data(
+                f'thrust::make_pair({elts_code})',
+                _cuda_types.Tuple([x.ctype for x in elts]))
+        else:
+            return Data(
+                f'thrust::make_tuple({elts_code})',
+                _cuda_types.Tuple([x.ctype for x in elts]))
+
+    if isinstance(expr, ast.Index):
+        # Deprecated in Python 3.9
+        return _transpile_expr(expr.value, env)  # type: ignore
+
+    raise ValueError('Not supported: type {}'.format(type(expr)))
+
+
+def _emit_assign_stmt(
+        lvalue: Union[Constant, Data],
+        rvalue: Data,
+        env: Environment,
+) -> _CodeType:
+    if isinstance(lvalue, Constant):
+        raise TypeError('lvalue of assignment must not be constant value')
+
+    if (isinstance(lvalue.ctype, _cuda_types.Scalar)
+            and isinstance(rvalue.ctype, _cuda_types.Scalar)):
+        rvalue = _astype_scalar(rvalue, lvalue.ctype, 'same_kind', env)
+    elif lvalue.ctype != rvalue.ctype:
+        raise TypeError(
+            f'Data type mismatch of variable: `{lvalue.code}`: '
+            f'{lvalue.ctype} != {rvalue.ctype}')
+
+    return [lvalue.ctype.assign(lvalue, rvalue) + ';']
+
+
+def _transpile_assign_stmt(
+        target: ast.expr,
+        env: Environment,
+        value: Data,
+        is_toplevel: bool,
+        depth: int = 0,
+) -> _CodeType:
+    if isinstance(target, ast.Name):
+        name = target.id
+        lvalue = env[name]
+        if lvalue is None:
+            lvalue = Data(name, value.ctype)
+            env.locals[name] = lvalue
+            if is_toplevel and depth == 0:
+                return [value.ctype.declvar(name, value) + ';']
+            env.decls[name] = lvalue
+        return _emit_assign_stmt(lvalue, value, env)
+
+    if isinstance(target, ast.Subscript):
+        lvalue = Data.init(_transpile_expr(target, env), env)
+        return _emit_assign_stmt(lvalue, value, env)
+
+    if isinstance(target, ast.Tuple):
+        if not isinstance(value.ctype, _cuda_types.Tuple):
+            raise ValueError(f'{value.ctype} cannot be unpack')
+        size = len(target.elts)
+        if len(value.ctype.types) > size:
+            raise ValueError(f'too many values to unpack (expected {size})')
+        if len(value.ctype.types) < size:
+            raise ValueError(f'not enough values to unpack (expected {size})')
+        codes = [value.ctype.declvar(f'_temp{depth}', value) + ';']
+        for i in range(size):
+            code = f'thrust::get<{i}>(_temp{depth})'
+            ctype = value.ctype.types[i]
+            stmt = _transpile_assign_stmt(
+                target.elts[i], env, Data(code, ctype), is_toplevel, depth + 1)
+            codes.extend(stmt)
+        return [CodeBlock('', codes)]
+
+    assert False
+
+
+def _indexing(
+        array: _internal_types.Expr,
+        index: _internal_types.Expr,
+        env: Environment,
+) -> Union[Data, Constant]:
+    if isinstance(array, Constant):
+        if isinstance(index, Constant):
+            return Constant(array.obj[index.obj])
+        raise TypeError(
+            f'{type(array.obj)} is not subscriptable with non-constants.')
+
+    array = Data.init(array, env)
+
+    if isinstance(array.ctype, _cuda_types.Tuple):
+        if isinstance(index, Constant):
+            i = index.obj
+            t = array.ctype.types[i]
+            return Data(f'thrust::get<{i}>({array.code})', t)
+        raise TypeError('Tuple is not subscriptable with non-constants.')
+
+    if isinstance(array.ctype, _cuda_types.ArrayBase):
+        index = Data.init(index, env)
+        ndim = array.ctype._ndim
+        if isinstance(index.ctype, _cuda_types.Scalar):
+            index_dtype = index.ctype.dtype
+            if ndim == 0:
+                raise TypeError(
+                    'Scalar indexing is not supported for 0-dim array.')
+            if ndim > 1:
+                new_carray = _cuda_types.CArray(
+                    array.ctype.dtype,
+                    array.ctype._ndim - 1,
+                    array.ctype._c_contiguous,
+                    array.ctype._index_32_bits)
+                return Data(
+                    f'{array.code}._slicing({index.code})', new_carray)
+            if index_dtype.kind not in 'ui':
+                raise TypeError('Array indices must be integers.')
+            return Data(
+                f'{array.code}[{index.code}]', array.ctype.child_type)
+        if isinstance(index.ctype, _cuda_types.Tuple):
+            if ndim < len(index.ctype.types):
+                raise IndexError(
+                    f'The number of indices is beyond array dim: {ndim}')
+            for t in index.ctype.types:
+                if not isinstance(t, _cuda_types.Scalar):
+                    raise TypeError('Array indices must be scalar.')
+                if t.dtype.kind not in 'iu':
+                    raise TypeError('Array indices must be integer.')
+            if ndim > len(index.ctype.types):
+                new_carray = _cuda_types.CArray(
+                    array.ctype.dtype,
+                    array.ctype._ndim - len(index.ctype.types),
+                    array.ctype._c_contiguous,
+                    array.ctype._index_32_bits)
+                params = f'{index.code}, Dim<{len(index.ctype.types)}>()'
+                return Data(f'{array.code}._slicing({params})', new_carray)
+            if ndim == 0:
+                return Data(
+                    f'{array.code}[0]', array.ctype.child_type)
+            if ndim == 1:
+                return Data(
+                    f'{array.code}[thrust::get<0>({index.code})]',
+                    array.ctype.child_type)
+            return Data(
+                f'{array.code}._indexing({index.code})',
+                array.ctype.child_type)
+        if isinstance(index.ctype, _cuda_types.CArray):
+            raise TypeError('Advanced indexing is not supported.')
+        assert False  # Never reach.
+
+    raise TypeError(f'{array.code} is not subscriptable.')
+
+
+_T = TypeVar('_T', Constant, Data, Union[Constant, Data])
+
+
+def _astype_scalar(
+        x: _T,
+        ctype: _cuda_types.Scalar,
+        casting: _CastingType,
+        env: Environment,
+) -> _T:
+    if isinstance(x, Constant):
+        assert not isinstance(x, Data)
+        return Constant(ctype.dtype.type(x.obj))
+    # # TODO
+    # if not isinstance(x, Data):
+    #     raise TypeError(f'{x} is not scalar type.')
+    if not isinstance(x.ctype, _cuda_types.Scalar):
+        raise TypeError(f'{x.code} is not scalar type.')
+    from_t = x.ctype.dtype
+    to_t = ctype.dtype
+    if from_t == to_t:
+        return x
+    # Uses casting rules for scalar values.
+    # TODO(seberg): NEP 50/NumPy 2.0 hopefully makes `from_t.type(0)`
+    #               unnecessary allowing to use `_raise_if_invalid_cast()`.
+    if not numpy.can_cast(from_t.type(0), to_t, casting):
+        raise TypeError(
+            f"Cannot cast from '{from_t}' to {to_t} "
+            f"with casting rule {casting}.")
+    if from_t.kind == 'c' and to_t.kind != 'c':
+        if to_t.kind != 'b':
+            warnings.warn(
+                'Casting complex values to real discards the imaginary part',
+                numpy.ComplexWarning)
+        return Data(f'({ctype})({x.code}.real())', ctype)
+    return Data(f'({ctype})({x.code})', ctype)
+
+
+def _infer_type(
+        x: Union[Constant, Data],
+        hint: Union[Constant, Data],
+        env: Environment,
+) -> Data:
+    if not isinstance(x, Constant) or isinstance(x.obj, numpy.generic):
+        return Data.init(x, env)
+    hint = Data.init(hint, env)
+    assert isinstance(hint.ctype, _cuda_types.Scalar)
+    cast_x = _astype_scalar(x, hint.ctype, 'same_kind', env)
+    return Data.init(cast_x, env)
diff --git a/cupyx/jit/_cuda_typerules.py b/cupyx/jit/_cuda_typerules.py
new file mode 100644
index 0000000..d750b6a
--- /dev/null
+++ b/cupyx/jit/_cuda_typerules.py
@@ -0,0 +1,162 @@
+import ast
+from typing import Any, Callable, Mapping, Optional, Tuple, Type
+
+import numpy
+import numpy.typing as npt
+import operator
+
+import cupy
+from cupy._logic import ops
+from cupy._math import arithmetic
+from cupy._logic import comparison
+from cupy._binary import elementwise
+from cupy import _core
+
+from cupyx.jit import _cuda_types
+
+
+_numpy_scalar_invert = _core.create_ufunc(
+    'numpy_scalar_invert',
+    ('?->?', 'b->b', 'B->B', 'h->h', 'H->H', 'i->i', 'I->I',
+     'l->l', 'L->L', 'q->q', 'Q->Q'),
+    'out0 = ~in0',
+)
+
+
+_numpy_scalar_logical_not = _core.create_ufunc(
+    'numpy_scalar_logical_not',
+    ('?->?', 'b->?', 'B->?', 'h->?', 'H->?', 'i->?', 'I->?', 'l->?', 'L->?',
+     'q->?', 'Q->?', 'e->?', 'f->?', 'd->?',
+     ('F->?', 'out0 = !in0.real() && !in0.imag()'),
+     ('D->?', 'out0 = !in0.real() && !in0.imag()')),
+    'out0 = !in0',
+)
+
+
+_scalar_lt = _core.create_comparison('scalar_less', '<')
+_scalar_lte = _core.create_comparison('scalar_less', '<=')
+_scalar_gt = _core.create_comparison('scalar_less', '>')
+_scalar_gte = _core.create_comparison('scalar_less', '>=')
+
+
+_py_ops: Mapping[Type[ast.AST], Callable[..., Any]] = {
+    ast.And: lambda x, y: x and y,
+    ast.Or: lambda x, y: x or y,
+    ast.Add: operator.add,
+    ast.Sub: operator.sub,
+    ast.Mult: operator.mul,
+    ast.Pow: operator.pow,
+    ast.Div: operator.truediv,
+    ast.FloorDiv: operator.floordiv,
+    ast.Mod: operator.mod,
+    ast.LShift: operator.lshift,
+    ast.RShift: operator.rshift,
+    ast.BitOr: operator.or_,
+    ast.BitAnd: operator.and_,
+    ast.BitXor: operator.xor,
+    ast.Invert: operator.invert,
+    ast.Not: operator.not_,
+    ast.Eq: operator.eq,
+    ast.NotEq: operator.ne,
+    ast.Lt: operator.lt,
+    ast.LtE: operator.le,
+    ast.Gt: operator.gt,
+    ast.GtE: operator.ge,
+    ast.USub: operator.neg,
+}
+
+
+_numpy_ops: Mapping[Type[ast.AST], cupy.ufunc] = {
+    ast.And: ops.logical_and,
+    ast.Or: ops.logical_or,
+    ast.Add: arithmetic.add,
+    ast.Sub: arithmetic.subtract,
+    ast.Mult: arithmetic.multiply,
+    ast.Pow: arithmetic.power,
+    ast.Div: arithmetic.true_divide,
+    ast.FloorDiv: arithmetic.floor_divide,
+    ast.Mod: arithmetic.remainder,
+    ast.LShift: elementwise.left_shift,
+    ast.RShift: elementwise.right_shift,
+    ast.BitOr: elementwise.bitwise_or,
+    ast.BitAnd: elementwise.bitwise_and,
+    ast.BitXor: elementwise.bitwise_xor,
+    ast.Invert: _numpy_scalar_invert,
+    ast.Not: _numpy_scalar_logical_not,
+    ast.Eq: comparison.equal,
+    ast.NotEq: comparison.not_equal,
+    ast.Lt: _scalar_lt,
+    ast.LtE: _scalar_lte,
+    ast.Gt: _scalar_gt,
+    ast.GtE: _scalar_gte,
+    ast.USub: arithmetic.negative,
+}
+
+
+def get_pyfunc(op_type: Type[ast.AST]) -> Callable[..., Any]:
+    return _py_ops[op_type]
+
+
+def get_ufunc(mode: str, op_type: Type[ast.AST]) -> cupy.ufunc:
+    if mode == 'numpy':
+        return _numpy_ops[op_type]
+    if mode == 'cuda':
+        return _numpy_ops[op_type]
+    assert False
+
+
+def get_ctype_from_scalar(mode: str, x: Any) -> _cuda_types.Scalar:
+    if isinstance(x, numpy.generic):
+        return _cuda_types.Scalar(x.dtype)
+
+    if mode == 'numpy':
+        if isinstance(x, bool):
+            return _cuda_types.Scalar(numpy.bool_)
+        if isinstance(x, int):
+            # use plain int here for cross-platform portability
+            return _cuda_types.Scalar(int)
+        if isinstance(x, float):
+            return _cuda_types.Scalar(numpy.float64)
+        if isinstance(x, complex):
+            return _cuda_types.Scalar(numpy.complex128)
+
+    if mode == 'cuda':
+        if isinstance(x, bool):
+            return _cuda_types.Scalar(numpy.bool_)
+        if isinstance(x, int):
+            if -(1 << 31) <= x < (1 << 31):
+                return _cuda_types.Scalar(numpy.int32)
+            return _cuda_types.Scalar(numpy.int64)
+        if isinstance(x, float):
+            return _cuda_types.Scalar(numpy.float32)
+        if isinstance(x, complex):
+            return _cuda_types.Scalar(numpy.complex64)
+
+    raise NotImplementedError(f'{x} is not scalar object.')
+
+
+_typechars = '?bBhHiIlLqQefdFD'
+
+
+def _cuda_can_cast(from_dtype: npt.DTypeLike, to_dtype: npt.DTypeLike) -> bool:
+    from_dtype = numpy.dtype(from_dtype)
+    to_dtype = numpy.dtype(to_dtype)
+    return _typechars.find(from_dtype.char) <= _typechars.find(to_dtype.char)
+
+
+def guess_routine(
+        ufunc: cupy.ufunc,
+        in_types: Tuple[numpy.dtype, ...],
+        dtype: Optional[numpy.dtype],
+        mode: str,
+) -> cupy._core._kernel._Op:
+    if dtype is not None:
+        return ufunc._ops._guess_routine_from_dtype(dtype)
+    can_cast = numpy.can_cast if mode == 'numpy' else _cuda_can_cast
+    return ufunc._ops._guess_routine_from_in_types(tuple(in_types), can_cast)
+
+
+def to_ctype(t) -> _cuda_types.TypeBase:
+    if isinstance(t, _cuda_types.TypeBase):
+        return t
+    return _cuda_types.Scalar(numpy.dtype(t))
diff --git a/cupyx/jit/_cuda_types.py b/cupyx/jit/_cuda_types.py
new file mode 100644
index 0000000..cf00187
--- /dev/null
+++ b/cupyx/jit/_cuda_types.py
@@ -0,0 +1,334 @@
+from typing import Mapping, Optional, Sequence, Union, TYPE_CHECKING
+
+import numpy
+import numpy.typing as npt
+
+import cupy
+from cupy._core._scalar import get_typename
+
+if TYPE_CHECKING:
+    from cupyx.jit._internal_types import Data
+
+
+# Base class for cuda types.
+class TypeBase:
+
+    def __str__(self) -> str:
+        raise NotImplementedError
+
+    def declvar(self, x: str, init: Optional['Data']) -> str:
+        if init is None:
+            return f'{self} {x}'
+        return f'{self} {x} = {init.code}'
+
+    def assign(self, var: 'Data', value: 'Data') -> str:
+        return f'{var.code} = {value.code}'
+
+
+class Void(TypeBase):
+
+    def __init__(self) -> None:
+        pass
+
+    def __str__(self) -> str:
+        return 'void'
+
+
+class Unknown(TypeBase):
+
+    def __init__(self, *, label: Optional[str] = None) -> None:
+        self.label = label
+
+    def __str__(self) -> str:
+        raise TypeError('unknown type can be used only in ary of a function.')
+
+
+class Scalar(TypeBase):
+
+    def __init__(self, dtype: npt.DTypeLike) -> None:
+        self.dtype = numpy.dtype(dtype)
+
+    def __str__(self) -> str:
+        dtype = self.dtype
+        if dtype == numpy.float16:
+            # For the performance
+            dtype = numpy.dtype('float32')
+        return get_typename(dtype)
+
+    def __eq__(self, other: object) -> bool:
+        assert isinstance(other, TypeBase)
+        return isinstance(other, Scalar) and self.dtype == other.dtype
+
+    def __hash__(self) -> int:
+        return hash(self.dtype)
+
+
+class PtrDiff(Scalar):
+
+    def __init__(self) -> None:
+        super().__init__('q')
+
+    def __str__(self) -> str:
+        return 'ptrdiff_t'
+
+
+class ArrayBase(TypeBase):
+
+    def ndim(self, instance: 'Data'):
+        from cupyx.jit import _internal_types  # avoid circular import
+        return _internal_types.Constant(self._ndim)
+
+    def __init__(self, child_type: TypeBase, ndim: int) -> None:
+        assert isinstance(child_type, TypeBase)
+        self.child_type = child_type
+        self._ndim = ndim
+
+
+class PointerBase(ArrayBase):
+
+    def __init__(self, child_type: TypeBase) -> None:
+        super().__init__(child_type, 1)
+
+    @staticmethod
+    def _add(env, x: 'Data', y: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        if isinstance(y.ctype, Scalar) and y.ctype.dtype.kind in 'iu':
+            return _internal_types.Data(f'({x.code} + {y.code})', x.ctype)
+        return NotImplemented
+
+    @staticmethod
+    def _radd(env, x: 'Data', y: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        if isinstance(x.ctype, Scalar) and x.ctype.dtype.kind in 'iu':
+            return _internal_types.Data(f'({x.code} + {y.code})', y.ctype)
+        return NotImplemented
+
+    @staticmethod
+    def _sub(env, x: 'Data', y: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        if isinstance(y.ctype, Scalar) and y.ctype.dtype.kind in 'iu':
+            return _internal_types.Data(f'({x.code} - {y.code})', x.ctype)
+        if x.ctype == y.ctype:
+            return _internal_types.Data(f'({x.code} - {y.code})', PtrDiff())
+        return NotImplemented
+
+
+class CArray(ArrayBase):
+    from cupyx.jit import _internal_types  # avoid circular import
+
+    def __init__(
+            self,
+            dtype: npt.DTypeLike,
+            ndim: int,
+            is_c_contiguous: bool,
+            index_32_bits: bool,
+    ) -> None:
+        self.dtype = numpy.dtype(dtype)
+        self._ndim = ndim
+        self._c_contiguous = is_c_contiguous
+        self._index_32_bits = index_32_bits
+        super().__init__(Scalar(dtype), ndim)
+
+    @classmethod
+    def from_ndarray(cls, x: cupy.ndarray) -> 'CArray':
+        return CArray(x.dtype, x.ndim, x._c_contiguous, x._index_32_bits)
+
+    def size(self, instance: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        return _internal_types.Data(
+            f'static_cast<long long>({instance.code}.size())', Scalar('q'))
+
+    def shape(self, instance: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        if self._ndim > 10:
+            raise NotImplementedError(
+                'getting shape/strides for an array with ndim > 10 '
+                'is not supported yet')
+        return _internal_types.Data(
+            f'{instance.code}.get_shape()', Tuple([PtrDiff()] * self._ndim))
+
+    def strides(self, instance: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        if self._ndim > 10:
+            raise NotImplementedError(
+                'getting shape/strides for an array with ndim > 10 '
+                'is not supported yet')
+        return _internal_types.Data(
+            f'{instance.code}.get_strides()', Tuple([PtrDiff()] * self._ndim))
+
+    @_internal_types.wraps_class_method
+    def begin(self, env, instance: 'Data', *args) -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        if self._ndim != 1:
+            raise NotImplementedError(
+                'getting begin iterator for an array with ndim != 1 '
+                'is not supported yet')
+        method_name = 'begin_ptr' if self._c_contiguous else 'begin'
+        return _internal_types.Data(
+            f'{instance.code}.{method_name}()',
+            CArrayIterator(instance.ctype))  # type: ignore
+
+    @_internal_types.wraps_class_method
+    def end(self, env, instance: 'Data', *args) -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        if self._ndim != 1:
+            raise NotImplementedError(
+                'getting end iterator for an array with ndim != 1 '
+                'is not supported yet')
+        method_name = 'end_ptr' if self._c_contiguous else 'end'
+        return _internal_types.Data(
+            f'{instance.code}.{method_name}()',
+            CArrayIterator(instance.ctype))  # type: ignore
+
+    def __str__(self) -> str:
+        ctype = get_typename(self.dtype)
+        ndim = self._ndim
+        c_contiguous = get_cuda_code_from_constant(self._c_contiguous, bool_)
+        index_32_bits = get_cuda_code_from_constant(self._index_32_bits, bool_)
+        return f'CArray<{ctype}, {ndim}, {c_contiguous}, {index_32_bits}>'
+
+    def __eq__(self, other: object) -> bool:
+        return str(self) == str(other)
+
+    def __hash__(self) -> int:
+        return hash(str(self))
+
+
+class CArrayIterator(PointerBase):
+
+    def __init__(self, carray_type: CArray) -> None:
+        self._carray_type = carray_type
+        super().__init__(Scalar(carray_type.dtype))
+
+    def __str__(self) -> str:
+        return f'{str(self._carray_type)}::iterator'
+
+    def __eq__(self, other: object) -> bool:
+        assert isinstance(other, TypeBase)
+        return (
+            isinstance(other, CArrayIterator) and
+            self._carray_type == other._carray_type
+        )
+
+    def __hash__(self) -> int:
+        return hash(
+            (self.dtype, self.ndim, self._c_contiguous, self._index_32_bits))
+
+
+class SharedMem(ArrayBase):
+
+    def __init__(
+            self,
+            child_type: TypeBase,
+            size: Optional[int],
+            alignment: Optional[int] = None,
+    ) -> None:
+        if not (isinstance(size, int) or size is None):
+            raise 'size of shared_memory must be integer or `None`'
+        if not (isinstance(alignment, int) or alignment is None):
+            raise 'alignment must be integer or `None`'
+        self._size = size
+        self._alignment = alignment
+        super().__init__(child_type, 1)
+
+    def declvar(self, x: str, init: Optional['Data']) -> str:
+        assert init is None
+        if self._alignment is not None:
+            code = f'__align__({self._alignment})'
+        else:
+            code = ''
+        if self._size is None:
+            code = f'extern {code} __shared__ {self.child_type} {x}[]'
+        else:
+            code = f'{code} __shared__ {self.child_type} {x}[{self._size}]'
+        return code
+
+
+class Ptr(PointerBase):
+
+    def __init__(self, child_type: TypeBase) -> None:
+        super().__init__(child_type)
+
+    def __str__(self) -> str:
+        return f'{self.child_type}*'
+
+
+class Tuple(TypeBase):
+
+    def __init__(self, types: Sequence[TypeBase]) -> None:
+        self.types = types
+
+    def __str__(self) -> str:
+        types = ', '.join([str(t) for t in self.types])
+        if len(self.types) == 2:
+            return f'thrust::pair<{types}>'
+        else:
+            return f'thrust::tuple<{types}>'
+
+    def __eq__(self, other: object) -> bool:
+        assert isinstance(other, TypeBase)
+        return isinstance(other, Tuple) and self.types == other.types
+
+
+void: Void = Void()
+bool_: Scalar = Scalar(numpy.bool_)
+int32: Scalar = Scalar(numpy.int32)
+uint32: Scalar = Scalar(numpy.uint32)
+uint64: Scalar = Scalar(numpy.uint64)
+
+
+class Dim3(TypeBase):
+    """
+    An integer vector type based on uint3 that is used to specify dimensions.
+
+    Attributes:
+        x (uint32)
+        y (uint32)
+        z (uint32)
+    """
+
+    def x(self, instance: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        return _internal_types.Data(f'{instance.code}.x', uint32)
+
+    def y(self, instance: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        return _internal_types.Data(f'{instance.code}.y', uint32)
+
+    def z(self, instance: 'Data') -> 'Data':
+        from cupyx.jit import _internal_types  # avoid circular import
+        return _internal_types.Data(f'{instance.code}.z', uint32)
+
+    def __str__(self) -> str:
+        return 'dim3'
+
+
+dim3: Dim3 = Dim3()
+
+
+_suffix_literals_dict: Mapping[str, str] = {
+    'float64': '',
+    'float32': 'f',
+    'int64': 'll',
+    'int32': '',
+    'uint64': 'ull',
+    'uint32': 'u',
+    'bool': '',
+}
+
+
+def get_cuda_code_from_constant(
+        x: Union[bool, int, float, complex],
+        ctype: Scalar,
+) -> str:
+    dtype = ctype.dtype
+    suffix_literal = _suffix_literals_dict.get(dtype.name)
+    if suffix_literal is not None:
+        s = str(x).lower()
+        return f'{s}{suffix_literal}'
+    ctype_str = str(ctype)
+    if dtype.kind == 'c':
+        return f'{ctype_str}({x.real}, {x.imag})'
+    if ' ' in ctype_str:
+        return f'({ctype_str}){x}'
+    return f'{ctype_str}({x})'
diff --git a/cupyx/jit/_interface.py b/cupyx/jit/_interface.py
new file mode 100644
index 0000000..76b7544
--- /dev/null
+++ b/cupyx/jit/_interface.py
@@ -0,0 +1,190 @@
+import functools
+import warnings
+
+import numpy
+
+import cupy
+from cupy._core import core
+from cupyx.jit import _compile
+from cupyx.jit import _cuda_typerules
+from cupyx.jit import _cuda_types
+from cupyx.jit import _internal_types
+from cupyx.jit._cuda_types import Scalar
+
+
+class _CudaFunction:
+    """JIT cupy function object
+    """
+
+    def __init__(self, func, mode, device=False, inline=False):
+        self.attributes = []
+
+        if device:
+            self.attributes.append('__device__')
+        else:
+            self.attributes.append('__global__')
+
+        if inline:
+            self.attributes.append('inline')
+
+        self.name = getattr(func, 'name', func.__name__)
+        self.func = func
+        self.mode = mode
+
+    def __call__(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def _emit_code_from_types(self, in_types, ret_type=None):
+        return _compile.transpile(
+            self.func, self.attributes, self.mode, in_types, ret_type)
+
+
+class _JitRawKernel:
+    """JIT CUDA kernel object.
+
+    The decorator :func:``cupyx.jit.rawkernel`` converts the target function
+    to an object of this class. This class is not inteded to be instantiated
+    by users.
+    """
+
+    def __init__(self, func, mode, device):
+        self._func = func
+        self._mode = mode
+        self._device = device
+        self._cache = {}
+        self._cached_codes = {}
+
+    def __call__(
+            self, grid, block, args, shared_mem=0, stream=None):
+        """Calls the CUDA kernel.
+
+        The compilation will be deferred until the first function call.
+        CuPy's JIT compiler infers the types of arguments at the call
+        time, and will cache the compiled kernels for speeding up any
+        subsequent calls.
+
+        Args:
+            grid (tuple of int): Size of grid in blocks.
+            block (tuple of int): Dimensions of each thread block.
+            args (tuple):
+                Arguments of the kernel. The type of all elements must be
+                ``bool``, ``int``, ``float``, ``complex``, NumPy scalar or
+                ``cupy.ndarray``.
+            shared_mem (int):
+                Dynamic shared-memory size per thread block in bytes.
+            stream (cupy.cuda.Stream): CUDA stream.
+
+        .. seealso:: :ref:`jit_kernel_definition`
+        """
+        in_types = []
+        for x in args:
+            if isinstance(x, cupy.ndarray):
+                t = _cuda_types.CArray.from_ndarray(x)
+            elif numpy.isscalar(x):
+                t = _cuda_typerules.get_ctype_from_scalar(self._mode, x)
+            else:
+                raise TypeError(f'{type(x)} is not supported for RawKernel')
+            in_types.append(t)
+        in_types = tuple(in_types)
+        device_id = cupy.cuda.get_device_id()
+
+        kern, enable_cg = self._cache.get((in_types, device_id), (None, None))
+        if kern is None:
+            result = self._cached_codes.get(in_types)
+            if result is None:
+                result = _compile.transpile(
+                    self._func,
+                    ['extern "C"', '__global__'],
+                    self._mode,
+                    in_types,
+                    _cuda_types.void,
+                )
+                self._cached_codes[in_types] = result
+
+            fname = result.func_name
+            enable_cg = result.enable_cooperative_groups
+            options = ('-DCUPY_JIT_MODE', '--std=c++14',
+                       # WAR: for compiling any CCCL header
+                       '-DCUB_DISABLE_BF16_SUPPORT',)
+            backend = result.backend
+            if backend == 'nvcc':
+                options += ('-DCUPY_JIT_NVCC',)
+            module = core.compile_with_cache(
+                source=result.code,
+                options=options,
+                backend=backend)
+            kern = module.get_function(fname)
+            self._cache[(in_types, device_id)] = (kern, enable_cg)
+
+        new_args = []
+        for a, t in zip(args, in_types):
+            if isinstance(t, Scalar):
+                if t.dtype.char == 'e':
+                    a = numpy.float32(a)
+                else:
+                    a = t.dtype.type(a)
+            new_args.append(a)
+
+        kern(grid, block, tuple(new_args), shared_mem, stream, enable_cg)
+
+    def __getitem__(self, grid_and_block):
+        """Numba-style kernel call.
+
+        .. seealso:: :ref:`jit_kernel_definition`
+        """
+        grid, block = grid_and_block
+        if not isinstance(grid, tuple):
+            grid = (grid, 1, 1)
+        if not isinstance(block, tuple):
+            block = (block, 1, 1)
+        return lambda *args, **kwargs: self(grid, block, args, **kwargs)
+
+    @property
+    def cached_codes(self):
+        """Returns a dict that has input types as keys and codes values.
+
+        This proprety method is for debugging purpose.
+        The return value is not guaranteed to keep backward compatibility.
+        """
+        if len(self._cached_codes) == 0:
+            warnings.warn(
+                'No codes are cached because compilation is deferred until '
+                'the first function call.')
+        return dict([(k, v.code) for k, v in self._cached_codes.items()])
+
+    @property
+    def cached_code(self):
+        """Returns `next(iter(self.cached_codes.values()))`.
+
+        This proprety method is for debugging purpose.
+        The return value is not guaranteed to keep backward compatibility.
+        """
+        codes = self.cached_codes
+        if len(codes) > 1:
+            warnings.warn(
+                'The input types of the kernel could not be inferred. '
+                'Please use `.cached_codes` instead.')
+        return next(iter(codes.values()))
+
+
+def rawkernel(*, mode='cuda', device=False):
+    """A decorator compiles a Python function into CUDA kernel.
+    """
+    cupy._util.experimental('cupyx.jit.rawkernel')
+
+    def wrapper(func):
+        return functools.update_wrapper(
+            _JitRawKernel(func, mode, device), func)
+    return wrapper
+
+
+threadIdx = _internal_types.Data('threadIdx', _cuda_types.dim3)
+blockDim = _internal_types.Data('blockDim', _cuda_types.dim3)
+blockIdx = _internal_types.Data('blockIdx', _cuda_types.dim3)
+gridDim = _internal_types.Data('gridDim', _cuda_types.dim3)
+
+warpsize = _internal_types.Data('warpSize', _cuda_types.int32)
+warpsize.__doc__ = r"""Returns the number of threads in a warp.
+
+.. seealso:: :obj:`numba.cuda.warpsize`
+"""
diff --git a/cupyx/jit/_internal_types.py b/cupyx/jit/_internal_types.py
new file mode 100644
index 0000000..ac78154
--- /dev/null
+++ b/cupyx/jit/_internal_types.py
@@ -0,0 +1,130 @@
+import functools
+import itertools
+from typing import Any, NoReturn, Optional, Union, TYPE_CHECKING
+
+from cupyx.jit import _cuda_types
+from cupyx.jit import _cuda_typerules
+
+if TYPE_CHECKING:
+    from cupyx.jit._compile import Environment
+
+
+class Expr:
+
+    def __repr__(self) -> str:
+        raise NotImplementedError
+
+
+class Data(Expr):
+
+    def __init__(self, code: str, ctype: _cuda_types.TypeBase) -> None:
+        assert isinstance(code, str)
+        assert isinstance(ctype, _cuda_types.TypeBase)
+        self.code = code
+        self.ctype = ctype
+        if not isinstance(ctype, _cuda_types.Unknown):
+            try:
+                self.__doc__ = f'{str(ctype)} {code}\n{ctype.__doc__}'
+            except NotImplementedError:
+                self.__doc__ = f'{code}'
+
+    @property
+    def obj(self):
+        raise ValueError(f'Constant value is requried: {self.code}')
+
+    def __repr__(self) -> str:
+        return f'<Data code = "{self.code}", type = {self.ctype}>'
+
+    @classmethod
+    def init(cls, x: Expr, env) -> 'Data':
+        if isinstance(x, Data):
+            return x
+        if isinstance(x, Constant):
+            if isinstance(x.obj, tuple):
+                elts = [Data.init(Constant(e), env) for e in x.obj]
+                elts_code = ', '.join([e.code for e in elts])
+                if len(elts) == 2:
+                    return Data(
+                        f'thrust::make_pair({elts_code})',
+                        _cuda_types.Tuple([x.ctype for x in elts]))
+                return Data(
+                    f'thrust::make_tuple({elts_code})',
+                    _cuda_types.Tuple([x.ctype for x in elts]))
+            ctype = _cuda_typerules.get_ctype_from_scalar(env.mode, x.obj)
+            code = _cuda_types.get_cuda_code_from_constant(x.obj, ctype)
+            return Data(code, ctype)
+        raise TypeError(f"'{x}' cannot be interpreted as a cuda object.")
+
+
+class Constant(Expr):
+
+    def __init__(self, obj: Any) -> None:
+        self._obj = obj
+
+    @property
+    def obj(self) -> Any:
+        return self._obj
+
+    def __repr__(self) -> str:
+        return f'<Constant obj = "{self.obj}">'
+
+
+class Range(Expr):
+
+    def __init__(
+            self, start: Data, stop: Data, step: Data,
+            ctype: _cuda_types.Scalar,
+            step_is_positive: Optional[bool],
+            *,
+            unroll: Union[None, int, bool] = None,
+    ) -> None:
+        self.start = start
+        self.stop = stop
+        self.step = step
+        self.ctype = ctype
+        self.step_is_positive = step_is_positive  # True, False or None
+        self.unroll = unroll
+
+
+class BuiltinFunc(Expr):
+    # subclasses must implement:
+    # - either call or call_const
+    # - `__call__` with a correct signature, which calls the parent's __call__
+
+    def call(self, env: 'Environment', *args, **kwargs) -> Expr:
+        for x in itertools.chain(args, kwargs.values()):
+            if not isinstance(x, Constant):
+                raise TypeError('Arguments must be constants.')
+        args = tuple([x.obj for x in args])
+        kwargs = dict([(k, v.obj) for k, v in kwargs.items()])
+        return self.call_const(env, *args, **kwargs)
+
+    def call_const(self, env: 'Environment', *args: Any, **kwarg: Any) -> Expr:
+        raise NotImplementedError
+
+    def __init__(self) -> None:
+        self.__doc__ = type(self).__call__.__doc__
+
+    def __call__(self) -> NoReturn:
+        raise RuntimeError('Cannot call this function from Python layer.')
+
+    def __repr__(self) -> str:
+        return '<cupyx.jit function>'
+
+    @classmethod
+    def from_class_method(cls, method, ctype_self, instance):
+        class _Wrapper(BuiltinFunc):
+
+            def call(self, env, *args, **kwargs):
+                return method(ctype_self, env, instance, *args)
+
+        return _Wrapper()
+
+
+def wraps_class_method(method):
+
+    @functools.wraps(method)
+    def f(ctype_self: _cuda_types.TypeBase, instance: Data) -> BuiltinFunc:
+        return BuiltinFunc.from_class_method(method, ctype_self, instance)
+
+    return f
diff --git a/cupyx/jit/cg.py b/cupyx/jit/cg.py
new file mode 100644
index 0000000..24c6b57
--- /dev/null
+++ b/cupyx/jit/cg.py
@@ -0,0 +1,453 @@
+from cupy.cuda import runtime as _runtime
+from cupyx.jit import _compile
+from cupyx.jit import _cuda_types
+from cupyx.jit._internal_types import BuiltinFunc as _BuiltinFunc
+from cupyx.jit._internal_types import Constant as _Constant
+from cupyx.jit._internal_types import Data as _Data
+from cupyx.jit._internal_types import wraps_class_method as _wraps_class_method
+
+
+# public interface of this module
+__all__ = ['this_grid', 'this_thread_block',
+           'sync', 'wait', 'wait_prior', 'memcpy_async']
+
+
+_header_to_code = {
+    'cg': ("#include <cooperative_groups.h>\n"
+           "namespace cg = cooperative_groups;\n"),
+    'cg_memcpy_async': "#include <cooperative_groups/memcpy_async.h>",
+    'cuda_barrier': "#include <cuda/barrier>",
+}
+
+
+def _check_include(env, header):
+    flag = getattr(env.generated, f"include_{header}")
+    if flag is False:
+        # prepend the header
+        env.generated.codes.append(_header_to_code[header])
+        setattr(env.generated, f"include_{header}", True)
+
+
+class _ThreadGroup(_cuda_types.TypeBase):
+    """ Base class for all cooperative groups. """
+
+    child_type = None
+
+    def __init__(self):
+        raise NotImplementedError
+
+    def __str__(self):
+        return f'{self.child_type}'
+
+    def _sync(self, env, instance):
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.sync()', _cuda_types.void)
+
+
+class _GridGroup(_ThreadGroup):
+    """A handle to the current grid group. Must be created via :func:`this_grid`.
+
+    .. seealso:: `CUDA Grid Group API`_, :class:`numba.cuda.cg.GridGroup`
+
+    .. _CUDA Grid Group API:
+        https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#grid-group-cg
+    """  # NOQA
+
+    def __init__(self):
+        self.child_type = 'cg::grid_group'
+
+    @_wraps_class_method
+    def is_valid(self, env, instance):
+        """
+        is_valid()
+
+        Returns whether the grid_group can synchronize.
+        """
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.is_valid()', _cuda_types.bool_)
+
+    @_wraps_class_method
+    def sync(self, env, instance):
+        """
+        sync()
+
+        Synchronize the threads named in the group.
+
+        .. seealso:: :meth:`numba.cuda.cg.GridGroup.sync`
+        """
+        # when this methond is called, we need to use the cooperative
+        # launch API
+        env.generated.enable_cg = True
+        return super()._sync(env, instance)
+
+    @_wraps_class_method
+    def thread_rank(self, env, instance):
+        """
+        thread_rank()
+
+        Rank of the calling thread within ``[0, num_threads)``.
+        """
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.thread_rank()', _cuda_types.uint64)
+
+    @_wraps_class_method
+    def block_rank(self, env, instance):
+        """
+        block_rank()
+
+        Rank of the calling block within ``[0, num_blocks)``.
+        """
+        if _runtime.runtimeGetVersion() < 11060:
+            raise RuntimeError("block_rank() is supported on CUDA 11.6+")
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.block_rank()', _cuda_types.uint64)
+
+    @_wraps_class_method
+    def num_threads(self, env, instance):
+        """
+        num_threads()
+
+        Total number of threads in the group.
+        """
+        if _runtime.runtimeGetVersion() < 11060:
+            raise RuntimeError("num_threads() is supported on CUDA 11.6+")
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.num_threads()', _cuda_types.uint64)
+
+    @_wraps_class_method
+    def num_blocks(self, env, instance):
+        """
+        num_blocks()
+
+        Total number of blocks in the group.
+        """
+        if _runtime.runtimeGetVersion() < 11060:
+            raise RuntimeError("num_blocks() is supported on CUDA 11.6+")
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.num_blocks()', _cuda_types.uint64)
+
+    @_wraps_class_method
+    def dim_blocks(self, env, instance):
+        """
+        dim_blocks()
+
+        Dimensions of the launched grid in units of blocks.
+        """
+        if _runtime.runtimeGetVersion() < 11060:
+            raise RuntimeError("dim_blocks() is supported on CUDA 11.6+")
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.dim_blocks()', _cuda_types.dim3)
+
+    @_wraps_class_method
+    def block_index(self, env, instance):
+        """
+        block_index()
+
+        3-Dimensional index of the block within the launched grid.
+        """
+        if _runtime.runtimeGetVersion() < 11060:
+            raise RuntimeError("block_index() is supported on CUDA 11.6+")
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.block_index()', _cuda_types.dim3)
+
+    @_wraps_class_method
+    def size(self, env, instance):
+        """
+        size()
+
+        Total number of threads in the group.
+        """
+        # despite it is an alias of num_threads, we need it for earlier 11.x
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.size()', _cuda_types.uint64)
+
+    @_wraps_class_method
+    def group_dim(self, env, instance):
+        """
+        group_dim()
+
+        Dimensions of the launched grid in units of blocks.
+        """
+        # despite it is an alias of dim_blocks, we need it for earlier 11.x
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.group_dim()', _cuda_types.dim3)
+
+
+class _ThreadBlockGroup(_ThreadGroup):
+    """A handle to the current thread block group. Must be
+    created via :func:`this_thread_block`.
+
+    .. seealso:: `CUDA Thread Block Group API`_
+
+    .. _CUDA Thread Block Group API:
+        https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#thread-block-group-cg
+    """
+
+    def __init__(self):
+        self.child_type = 'cg::thread_block'
+
+    @_wraps_class_method
+    def sync(self, env, instance):
+        """
+        sync()
+
+        Synchronize the threads named in the group.
+        """
+        return super()._sync(env, instance)
+
+    @_wraps_class_method
+    def thread_rank(self, env, instance):
+        """
+        thread_rank()
+
+        Rank of the calling thread within ``[0, num_threads)``.
+        """
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.thread_rank()', _cuda_types.uint32)
+
+    @_wraps_class_method
+    def group_index(self, env, instance):
+        """
+        group_index()
+
+        3-Dimensional index of the block within the launched grid.
+        """
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.group_index()', _cuda_types.dim3)
+
+    @_wraps_class_method
+    def thread_index(self, env, instance):
+        """
+        thread_index()
+
+        3-Dimensional index of the thread within the launched block.
+        """
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.thread_index()', _cuda_types.dim3)
+
+    @_wraps_class_method
+    def dim_threads(self, env, instance):
+        """
+        dim_threads()
+
+        Dimensions of the launched block in units of threads.
+        """
+        if _runtime.runtimeGetVersion() < 11060:
+            raise RuntimeError("dim_threads() is supported on CUDA 11.6+")
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.dim_threads()', _cuda_types.dim3)
+
+    @_wraps_class_method
+    def num_threads(self, env, instance):
+        """
+        num_threads()
+
+        Total number of threads in the group.
+        """
+        if _runtime.runtimeGetVersion() < 11060:
+            raise RuntimeError("num_threads() is supported on CUDA 11.6+")
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.num_threads()', _cuda_types.uint32)
+
+    @_wraps_class_method
+    def size(self, env, instance):
+        """
+        size()
+
+        Total number of threads in the group.
+        """
+        # despite it is an alias of num_threads, we need it for earlier 11.x
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.size()', _cuda_types.uint32)
+
+    @_wraps_class_method
+    def group_dim(self, env, instance):
+        """
+        group_dim()
+
+        Dimensions of the launched block in units of threads.
+        """
+        # despite it is an alias of dim_threads, we need it for earlier 11.x
+        _check_include(env, 'cg')
+        return _Data(f'{instance.code}.group_dim()', _cuda_types.dim3)
+
+
+class _ThisCgGroup(_BuiltinFunc):
+
+    def __init__(self, group_type):
+        if group_type == "grid":
+            name = "grid group"
+            typename = "_GridGroup"
+        elif group_type == 'thread_block':
+            name = "thread block group"
+            typename = "_ThreadBlockGroup"
+        else:
+            raise NotImplementedError
+        self.group_type = group_type
+        self.__doc__ = f"""
+        Returns the current {name} (:class:`~cupyx.jit.cg.{typename}`).
+
+        .. seealso:: :class:`cupyx.jit.cg.{typename}`"""
+        if group_type == "grid":
+            self.__doc__ += ", :func:`numba.cuda.cg.this_grid`"
+
+    def __call__(self):
+        super().__call__()
+
+    def call_const(self, env):
+        if _runtime.is_hip:
+            raise RuntimeError('cooperative group is not supported on HIP')
+        if self.group_type == 'grid':
+            if _runtime.runtimeGetVersion() < 11000:
+                raise RuntimeError(
+                    "For pre-CUDA 11, the grid group has very limited "
+                    "functionality (only group.sync() works), and so we "
+                    "disable the grid group support to prepare the transition "
+                    "to support CUDA 11+ only.")
+            cg_type = _GridGroup()
+        elif self.group_type == 'thread_block':
+            cg_type = _ThreadBlockGroup()
+        return _Data(f'cg::this_{self.group_type}()', cg_type)
+
+
+class _Sync(_BuiltinFunc):
+
+    def __call__(self, group):
+        """Calls ``cg::sync()``.
+
+        Args:
+            group: a valid cooperative group
+
+        .. seealso:: `cg::sync`_
+
+        .. _cg::sync:
+            https://docs.nvidia.com/cuda/archive/11.6.0/cuda-c-programming-guide/index.html#collectives-cg-sync
+        """
+        super().__call__()
+
+    def call(self, env, group):
+        if _runtime.runtimeGetVersion() < 11000:
+            raise RuntimeError("not supported in CUDA < 11.0")
+        if not isinstance(group.ctype, _ThreadGroup):
+            raise ValueError("group must be a valid cooperative group")
+        _check_include(env, 'cg')
+        return _Data(f'cg::sync({group.code})', _cuda_types.void)
+
+
+class _MemcpySync(_BuiltinFunc):
+
+    def __call__(self, group, dst, dst_idx, src, src_idx, size, *,
+                 aligned_size=None):
+        """Calls ``cg::memcpy_sync()``.
+
+        Args:
+            group: a valid cooperative group
+            dst: the destination array that can be viewed as a 1D
+                C-contiguous array
+            dst_idx: the start index of the destination array element
+            src: the source array that can be viewed as a 1D C-contiguous
+                array
+            src_idx: the start index of the source array element
+            size (int): the number of bytes to be copied from
+                ``src[src_index]`` to ``dst[dst_idx]``
+            aligned_size (int): Use ``cuda::aligned_size_t<N>`` to guarantee
+                the compiler that ``src``/``dst`` are at least N-bytes aligned.
+                The behavior is undefined if the guarantee is not held.
+
+        .. seealso:: `cg::memcpy_sync`_
+
+        .. _cg::memcpy_sync:
+            https://docs.nvidia.com/cuda/archive/11.6.0/cuda-c-programming-guide/index.html#collectives-cg-memcpy-async
+        """
+        super().__call__()
+
+    def call(self, env, group, dst, dst_idx, src, src_idx, size, *,
+             aligned_size=None):
+        if _runtime.runtimeGetVersion() < 11010:
+            # the overloaded version of memcpy_async that we use does not yet
+            # exist in CUDA 11.0
+            raise RuntimeError("not supported in CUDA < 11.1")
+        _check_include(env, 'cg')
+        _check_include(env, 'cg_memcpy_async')
+
+        dst = _Data.init(dst, env)
+        src = _Data.init(src, env)
+        for arr in (dst, src):
+            if not isinstance(
+                    arr.ctype, (_cuda_types.CArray, _cuda_types.Ptr)):
+                raise TypeError('dst/src must be of array type.')
+        dst = _compile._indexing(dst, dst_idx, env)
+        src = _compile._indexing(src, src_idx, env)
+
+        size = _compile._astype_scalar(
+            # it's very unlikely that the size would exceed 2^32, so we just
+            # pick uint32 for simplicity
+            size, _cuda_types.uint32, 'same_kind', env)
+        size = _Data.init(size, env)
+        size_code = f'{size.code}'
+
+        if aligned_size:
+            if not isinstance(aligned_size, _Constant):
+                raise ValueError(
+                    'aligned_size must be a compile-time constant')
+            _check_include(env, 'cuda_barrier')
+            size_code = (f'cuda::aligned_size_t<{aligned_size.obj}>'
+                         f'({size_code})')
+        return _Data(f'cg::memcpy_async({group.code}, &({dst.code}), '
+                     f'&({src.code}), {size_code})', _cuda_types.void)
+
+
+class _Wait(_BuiltinFunc):
+
+    def __call__(self, group):
+        """Calls ``cg::wait()``.
+
+        Args:
+            group: a valid cooperative group
+
+        .. seealso: `cg::wait`_
+
+        .. _cg::wait:
+            https://docs.nvidia.com/cuda/archive/11.6.0/cuda-c-programming-guide/index.html#collectives-cg-wait
+        """
+        super().__call__()
+
+    def call(self, env, group):
+        if _runtime.runtimeGetVersion() < 11000:
+            raise RuntimeError("not supported in CUDA < 11.0")
+        _check_include(env, 'cg')
+        return _Data(f'cg::wait({group.code})', _cuda_types.void)
+
+
+class _WaitPrior(_BuiltinFunc):
+
+    def __call__(self, group):
+        """Calls ``cg::wait_prior<N>()``.
+
+        Args:
+            group: a valid cooperative group
+            step (int): wait for the first ``N`` steps to finish
+
+        .. seealso: `cg::wait_prior`_
+
+        .. _cg::wait_prior:
+            https://docs.nvidia.com/cuda/archive/11.6.0/cuda-c-programming-guide/index.html#collectives-cg-wait
+        """
+        super().__call__()
+
+    def call(self, env, group, step):
+        if _runtime.runtimeGetVersion() < 11000:
+            raise RuntimeError("not supported in CUDA < 11.0")
+        _check_include(env, 'cg')
+        if not isinstance(step, _Constant):
+            raise ValueError('step must be a compile-time constant')
+        return _Data(f'cg::wait_prior<{step.obj}>({group.code})',
+                     _cuda_types.void)
+
+
+this_grid = _ThisCgGroup('grid')
+this_thread_block = _ThisCgGroup('thread_block')
+sync = _Sync()
+wait = _Wait()
+wait_prior = _WaitPrior()
+memcpy_async = _MemcpySync()
diff --git a/cupyx/jit/cub.py b/cupyx/jit/cub.py
new file mode 100644
index 0000000..8f5d2ea
--- /dev/null
+++ b/cupyx/jit/cub.py
@@ -0,0 +1,115 @@
+from cupyx.jit import _cuda_types
+from cupyx.jit import _cuda_typerules
+from cupyx.jit import _internal_types
+from cupy_backends.cuda.api import runtime as _runtime
+
+
+class _ClassTemplate:
+
+    def __init__(self, class_type):
+        self._class_type = class_type
+        self.__doc__ = self._class_type.__doc__
+
+    def __getitem__(self, args):
+        if isinstance(args, tuple):
+            return self._class_type(*args)
+        else:
+            return self._class_type(args)
+
+
+def _include_cub(env):
+    if _runtime.is_hip:
+        env.generated.add_code('#include <hipcub/hipcub.hpp>')
+    elif _runtime.runtimeGetVersion() < 11000:
+        env.generated.add_code('#include <cupy/cub/cub/cub.cuh>')
+    else:
+        env.generated.add_code('#include <cub/cub.cuh>')
+    env.generated.backend = 'nvcc'
+
+
+def _get_cub_namespace():
+    return 'hipcub' if _runtime.is_hip else 'cub'
+
+
+class _TempStorageType(_cuda_types.TypeBase):
+
+    def __init__(self, parent_type):
+        assert isinstance(parent_type, _CubReduceBaseType)
+        self.parent_type = parent_type
+        super().__init__()
+
+    def __str__(self) -> str:
+        return f'typename {self.parent_type}::TempStorage'
+
+
+class _CubReduceBaseType(_cuda_types.TypeBase):
+
+    def _instantiate(self, env, temp_storage) -> _internal_types.Data:
+        _include_cub(env)
+        if temp_storage.ctype != self.TempStorage:
+            raise TypeError(
+                f'Invalid temp_storage type {temp_storage.ctype}. '
+                f'({self.TempStorage} is expected.)')
+        return _internal_types.Data(f'{self}({temp_storage.code})', self)
+
+    @_internal_types.wraps_class_method
+    def Sum(self, env, instance, input) -> _internal_types.Data:
+        if input.ctype != self.T:
+            raise TypeError(
+                f'Invalid input type {input.ctype}. ({self.T} is expected.)')
+        return _internal_types.Data(
+            f'{instance.code}.Sum({input.code})', input.ctype)
+
+    @_internal_types.wraps_class_method
+    def Reduce(self, env, instance, input, reduction_op):
+        if input.ctype != self.T:
+            raise TypeError(
+                f'Invalid input type {input.ctype}. ({self.T} is expected.)')
+        return _internal_types.Data(
+            f'{instance.code}.Reduce({input.code}, {reduction_op.code})',
+            input.ctype)
+
+
+class _WarpReduceType(_CubReduceBaseType):
+
+    def __init__(self, T) -> None:
+        self.T = _cuda_typerules.to_ctype(T)
+        self.TempStorage = _TempStorageType(self)
+        super().__init__()
+
+    def __str__(self) -> str:
+        namespace = _get_cub_namespace()
+        return f'{namespace}::WarpReduce<{self.T}>'
+
+
+class _BlockReduceType(_CubReduceBaseType):
+
+    def __init__(self, T, BLOCK_DIM_X: int) -> None:
+        self.T = _cuda_typerules.to_ctype(T)
+        self.BLOCK_DIM_X = BLOCK_DIM_X
+        self.TempStorage = _TempStorageType(self)
+        super().__init__()
+
+    def __str__(self) -> str:
+        namespace = _get_cub_namespace()
+        return f'{namespace}::BlockReduce<{self.T}, {self.BLOCK_DIM_X}>'
+
+
+WarpReduce = _ClassTemplate(_WarpReduceType)
+BlockReduce = _ClassTemplate(_BlockReduceType)
+
+
+class _CubFunctor(_internal_types.BuiltinFunc):
+
+    def __init__(self, name):
+        namespace = _get_cub_namespace()
+        self.fname = f'{namespace}::{name}()'
+
+    def call_const(self, env):
+        return _internal_types.Data(
+            self.fname, _cuda_types.Unknown(label='cub_functor'))
+
+
+Sum = _CubFunctor('Sum')
+Max = _CubFunctor('Max')
+Min = _CubFunctor('Min')
diff --git a/cupyx/jit/thrust.py b/cupyx/jit/thrust.py
new file mode 100644
index 0000000..be919e5
--- /dev/null
+++ b/cupyx/jit/thrust.py
@@ -0,0 +1,952 @@
+from cupyx.jit import _internal_types
+from cupyx.jit import _cuda_types
+from cupyx.jit._internal_types import Data as _Data
+
+
+def _wrap_thrust_func(headers):
+    def wrapper(func):
+        class FuncWrapper(_internal_types.BuiltinFunc):
+            def call(self, env, *args, **kwargs):
+                for header in headers:
+                    env.generated.add_code(f'#include <{header}>')
+                env.generated.add_code('#include <thrust/execution_policy.h>')
+                env.generated.add_code('#include <thrust/functional.h>')
+                env.generated.backend = 'nvcc'
+                data_args = [_Data.init(a, env) for a in args]
+                data_kwargs = {k: _Data.init(kwargs[k], env) for k in kwargs}
+                return func(env, *data_args, **data_kwargs)
+        return FuncWrapper()
+    return wrapper
+
+
+class _ExecPolicyType(_cuda_types.TypeBase):
+    pass
+
+
+def _assert_exec_policy_type(exec_policy: _Data):
+    if not isinstance(exec_policy.ctype, _ExecPolicyType):
+        raise TypeError(f'{exec_policy.code} must be execution policy type')
+
+
+def _assert_pointer_type(a: _Data) -> None:
+    # TODO(asi1024): Typecheck for EqualityComparable.
+    if not isinstance(a.ctype, _cuda_types.PointerBase):
+        raise TypeError(f'`{a.code}` must be of pointer type: `{a.ctype}`')
+
+
+def _assert_same_type(a: _Data, b: _Data) -> None:
+    if a.ctype != b.ctype:
+        raise TypeError(
+            f'`{a.code}` and `{b.code}` must be of the same type: '
+            f'`{a.ctype}` != `{b.ctype}`')
+
+
+def _assert_same_pointer_type(a: _Data, b: _Data) -> None:
+    # TODO(asi1024): Typecheck for EqualityComparable.
+    _assert_pointer_type(a)
+    _assert_pointer_type(b)
+    if a.ctype.child_type != b.ctype.child_type:
+        raise TypeError(
+            f'`{a.code}` and `{b.code}` must be of the same pointer type: '
+            f'`{a.ctype.child_type}` != `{b.type.child_type}`')
+
+
+def _assert_pointer_of(a: _Data, b: _Data) -> None:
+    _assert_pointer_type(a)
+    if a.ctype.child_type != b.ctype:
+        raise TypeError(
+            f'`*{a.code}` and `{b.code}` must be of the same type: '
+            f'`{a.ctype.child_type}` != `{b.ctype}`')
+
+
+host = _Data('thrust::host', _ExecPolicyType())
+device = _Data('thrust::device', _ExecPolicyType())
+seq = _Data('thrust::seq', _ExecPolicyType())
+
+
+@_wrap_thrust_func(['thrust/adjacent_difference.h'])
+def adjacent_difference(env, exec_policy, first, last, result, binary_op=None):
+    """Computes the differences of adjacent elements.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first, last)
+    _assert_same_pointer_type(first, result)
+    if binary_op is not None:
+        raise NotImplementedError('binary_op option is not supported')
+    args = [exec_policy, first, last, result]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::adjacent_difference({params})', result.ctype)
+
+
+# TODO(asi1024): Support all_of
+# TODO(asi1024): Support any_of
+
+
+@_wrap_thrust_func(['thrust/binary_search.h'])
+def binary_search(env, exec_policy, first, last, *args):
+    """Attempts to find the element value with binary search.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+
+    if 1 <= len(args) <= 2:
+        value = args[0]
+        comp = args[1] if len(args) == 2 else None
+        _assert_pointer_of(first, value)
+        result_ctype = _cuda_types.bool_
+    elif 3 <= len(args) <= 4:
+        value_first = args[0]
+        value_last = args[1]
+        result = args[2]
+        comp = args[3] if len(args) == 4 else None
+        _assert_same_pointer_type(first, value_first)
+        _assert_same_type(value_first, value_last)
+        result_ctype = result.ctype
+    else:
+        raise TypeError('Invalid number of inputs of thrust.binary_search')
+
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    args = [exec_policy, first, last, *args]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::binary_search({params})', result_ctype)
+
+
+@_wrap_thrust_func(['thrust/copy.h'])
+def copy(env, exec_policy, first, last, result):
+    """Copies the elements.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first, last)
+    _assert_same_pointer_type(first, result)
+    # TODO(asi1024): Typecheck for EqualityComparable.
+    args = [exec_policy, first, last, result]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::copy({params})', result.ctype)
+
+
+# TODO(asi1024): Add copy_if
+# TODO(asi1024): Add copy_n
+
+
+@_wrap_thrust_func(['thrust/count.h'])
+def count(env, exec_policy, first, last, value):
+    """Counts the number of elements in [first, last) that equals to ``value``.
+    """
+    _assert_exec_policy_type(exec_policy)
+    if not isinstance(first.ctype, _cuda_types.PointerBase):
+        raise TypeError('`first` must be of pointer type')
+    if first.ctype != last.ctype:
+        raise TypeError('`first` and `last` must be of the same type')
+    # TODO(asi1024): Typecheck for EqualityComparable.
+    args = [exec_policy, first, last, value]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::count({params})', _cuda_types.int32)
+
+
+# TODO(asi1024): Add count_if
+
+
+@_wrap_thrust_func(['thrust/equal.h'])
+def equal(env, exec_policy, first1, last1, first2, binary_pred=None):
+    """Returns true if the two ranges are identical.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first1, last1)
+    _assert_same_pointer_type(first1, first2)
+    if binary_pred is not None:
+        raise NotImplementedError('binary_pred option is not supported')
+    args = [exec_policy, first1, last1, first2]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::equal({params})', _cuda_types.bool_)
+
+
+@_wrap_thrust_func(['thrust/binary_search.h'])
+def equal_range(env, exec_policy, first, last, value, comp=None):
+    """Attempts to find the element value in an ordered range.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    args = [exec_policy, first, last, value]
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::equal_range({params})',
+        _cuda_types.Tuple([first.ctype, first.ctype]))
+
+
+@_wrap_thrust_func(['thrust/scan.h'])
+def exclusive_scan(
+        env, exec_policy, first, last, result, init=None, binary_op=None):
+    """Computes an exclusive prefix sum operation.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first, last)
+    _assert_same_pointer_type(first, result)
+    if binary_op is not None:
+        raise NotImplementedError('binary_op option is not supported')
+    args = [exec_policy, first, last, result]
+    if init is not None:
+        args.append(init)
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::exclusive_scan({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/scan.h'])
+def exclusive_scan_by_key(
+        env, exec_policy, first1, last1, first2, result,
+        init=None, binary_pred=None, binary_op=None):
+    """Computes an exclusive prefix sum operation by key.
+    _assert_exec_policy_type(exec_policy)
+    """
+    _assert_pointer_type(first1)
+    _assert_same_type(first1, last1)
+    _assert_same_pointer_type(first2, result)
+    if binary_pred is not None:
+        raise NotImplementedError('binary_pred option is not supported')
+    if binary_op is not None:
+        raise NotImplementedError('binary_op option is not supported')
+    args = [exec_policy, first1, last1, first2, result]
+    if init is not None:
+        args.append(init)
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::exclusive_scan_by_key({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/fill.h'])
+def fill(env, exec_policy, first, last, value):
+    """Assigns the value to every element in the range.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first, last)
+    # TODO(asi1024): Typecheck for EqualityComparable.
+    args = [exec_policy, first, last, value]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::fill({params})', _cuda_types.void)
+
+
+# TODO(asi1024): Add fill_n
+
+
+@_wrap_thrust_func(['thrust/find.h'])
+def find(env, exec_policy, first, last, value):
+    """Finds the first iterator whose value equals to ``value``.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    # TODO(asi1024): Typecheck for EqualityComparable.
+    args = [exec_policy, first, last, value]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::find({params})', first.ctype)
+
+
+# TODO(asi1024): Add find_if
+# TODO(asi1024): Add find_if_not
+# TODO(asi1024): Add for_each
+# TODO(asi1024): Add for_each_n
+
+
+@_wrap_thrust_func(['thrust/gather.h'])
+def gather(env, exec_policy, map_first, map_last, input_first, result):
+    """Copies elements from source into destination  according to a map.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(map_first)
+    _assert_same_type(map_first, map_last)
+    _assert_same_pointer_type(input_first, result)
+    args = [exec_policy, map_first, map_last, input_first, result]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::gather({params})', result.ctype)
+
+
+# TODO(asi1024): Add gather_if
+# TODO(asi1024): Add generate_n
+
+
+@_wrap_thrust_func(['thrust/scan.h'])
+def inclusive_scan(
+        env, exec_policy, first, last, result, binary_op=None):
+    """Computes an inclusive prefix sum operation.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first, last)
+    _assert_same_pointer_type(first, result)
+    if binary_op is not None:
+        raise NotImplementedError('binary_op option is not supported')
+    args = [exec_policy, first, last, result]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::inclusive_scan({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/scan.h'])
+def inclusive_scan_by_key(
+        env, exec_policy, first1, last1, first2, result,
+        binary_pred=None, binary_op=None):
+    """Computes an inclusive prefix sum operation by key.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first1)
+    _assert_same_type(first1, last1)
+    _assert_same_pointer_type(first2, result)
+    if binary_pred is not None:
+        raise NotImplementedError('binary_pred option is not supported')
+    if binary_op is not None:
+        raise NotImplementedError('binary_op option is not supported')
+    args = [exec_policy, first1, last1, first2, result]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::inclusive_scan_by_key({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/inner_product.h'])
+def inner_product(
+        env, exec_policy, first1, last1, first2, init,
+        binary_op1=None, binary_op2=None):
+    """Calculates an inner product of the ranges.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first1, last1)
+    _assert_same_pointer_type(first1, first2)
+    if binary_op1 is not None:
+        raise NotImplementedError('binary_op1 option is not supported')
+    if binary_op2 is not None:
+        raise NotImplementedError('binary_op2 option is not supported')
+    args = [exec_policy, first1, last1, first2, init]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::inner_product({params})', init.ctype)
+
+
+# TODO(asi1024): Add is_partitioned
+
+
+@_wrap_thrust_func(['thrust/sort.h'])
+def is_sorted(env, exec_policy, first, last, comp=None):
+    """Returns true if the range is sorted in ascending order.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    args = [exec_policy, first, last]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::is_sorted({params})', _cuda_types.bool_)
+
+
+@_wrap_thrust_func(['thrust/sort.h'])
+def is_sorted_until(env, exec_policy, first, last, comp=None):
+    """Returns the last iterator for which the range is sorted.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    args = [exec_policy, first, last]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::is_sorted_until({params})', first.ctype)
+
+
+@_wrap_thrust_func(['thrust/binary_search.h'])
+def lower_bound(env, exec_policy, first, last, *args):
+    """Attempts to find the element value with binary search.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+
+    if 1 <= len(args) <= 2:
+        value = args[0]
+        comp = args[1] if len(args) == 2 else None
+        _assert_pointer_of(first, value)
+        result_ctype = first.ctype
+    elif 3 <= len(args) <= 4:
+        value_first = args[0]
+        value_last = args[1]
+        result = args[2]
+        comp = args[3] if len(args) == 4 else None
+        _assert_same_pointer_type(first, value_first)
+        _assert_same_type(value_first, value_last)
+        result_ctype = result.ctype
+    else:
+        raise TypeError('Invalid number of inputs of thrust.lower_bound')
+
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    args = [exec_policy, first, last, *args]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::lower_bound({params})', result_ctype)
+
+
+class _ConstantIterator(_cuda_types.PointerBase):
+
+    def __str__(self) -> str:
+        value_type = self.child_type
+        return f'thrust::constant_iterator<{value_type}>'
+
+
+@_wrap_thrust_func(['thrust/iterator/constant_iterator.h'])
+def make_constant_iterator(env, x, i=None):
+    """Finds the first positions whose values differ.
+    """
+    if i is not None:
+        raise NotImplementedError('index_type is not supported')
+    args = [x]
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::make_constant_iterator({params})',
+        _ConstantIterator(x.ctype)
+    )
+
+
+class _CountingIterator(_cuda_types.PointerBase):
+
+    def __str__(self) -> str:
+        value_type = self.child_type
+        return f'thrust::counting_iterator<{value_type}>'
+
+
+@_wrap_thrust_func(['thrust/iterator/counting_iterator.h'])
+def make_counting_iterator(env, x, i=None):
+    """Finds the first positions whose values differ.
+    """
+    if i is not None:
+        raise NotImplementedError('index_type is not supported')
+    args = [x]
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::make_counting_iterator({params})',
+        _CountingIterator(x.ctype)
+    )
+
+
+# TODO(asi1024): Add make_discard_iterator
+# TODO(asi1024): Add make_index_sequence
+
+
+@_wrap_thrust_func(['thrust/mismatch.h'])
+def mismatch(env, exec_policy, first1, last1, first2, pred=None):
+    """Finds the first positions whose values differ.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first1, last1)
+    _assert_same_pointer_type(first1, first2)
+    # TODO(asi1024): Typecheck for EqualityComparable.
+    if pred is not None:
+        raise NotImplementedError('pred option is not supported')
+    args = [exec_policy, first1, last1, first2]
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::mismatch({params})',
+        _cuda_types.Tuple([first1.ctype, first2.ctype])
+    )
+
+
+# TODO(asi1024): Add none_of
+# TODO(asi1024): Add partition
+# TODO(asi1024): Add partition_copy
+# TODO(asi1024): Add partition_point
+
+
+@_wrap_thrust_func(['thrust/reduce.h'])
+def reduce(env, exec_policy, first, last, init=None, binary_op=None):
+    """Generalization of summation.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    args = [exec_policy, first, last]
+    if init is not None:
+        # TODO(asi1024): Typecheck for init.
+        args.append(init)
+        return_type = init.ctype
+    else:
+        return_type = first.ctype.child_type
+    if binary_op is not None:
+        raise NotImplementedError('binary_op option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::reduce({params})', return_type)
+
+
+@_wrap_thrust_func(['thrust/reduce.h'])
+def reduce_by_key(
+        env, exec_policy, keys_first, keys_last, values_first,
+        keys_output, values_output, binary_pred=None, binary_op=None):
+    """Generalization of reduce to key-value pairs.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(keys_first)
+    _assert_same_type(keys_first, keys_last)
+    _assert_pointer_type(values_first)
+    _assert_pointer_type(keys_output)
+    _assert_pointer_type(values_output)
+    args = [
+        exec_policy, keys_first, keys_last, values_first,
+        keys_output, values_output
+    ]
+    if binary_pred is not None:
+        raise NotImplementedError('binary_pred option is not supported')
+    if binary_op is not None:
+        raise NotImplementedError('binary_op option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::reduce_by_key({params})',
+        _cuda_types.Tuple([keys_output.ctype, values_output.ctype])
+    )
+
+
+@_wrap_thrust_func(['thrust/remove.h'])
+def remove(env, exec_policy, first, last, value):
+    """Removes from the range all elements that are equal to value.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    args = [exec_policy, first, last, value]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::remove({params})', first.ctype)
+
+
+@_wrap_thrust_func(['thrust/remove.h'])
+def remove_copy(env, exec_policy, first, last, result, value):
+    """Removes from the range all elements that are equal to value.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    _assert_pointer_type(result)
+    args = [exec_policy, first, last, result, value]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::remove_copy({params})', result.ctype)
+
+
+# TODO(asi1024): Add remove_copy_if
+# TODO(asi1024): Add remove_if
+
+
+@_wrap_thrust_func(['thrust/replace.h'])
+def replace(env, exec_policy, first, last, old_value, new_value):
+    """Replaces every element in the range equal to old_value with new_value.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    _assert_same_type(old_value, new_value)
+    args = [exec_policy, first, last, old_value, new_value]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::replace({params})', _cuda_types.void)
+
+
+@_wrap_thrust_func(['thrust/replace.h'])
+def replace_copy(env, exec_policy, first, last, result, old_value, new_value):
+    """Replaces every element in the range equal to old_value with new_value.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    _assert_pointer_type(result)
+    _assert_same_type(old_value, new_value)
+    args = [exec_policy, first, last, result, old_value, new_value]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::replace_copy({params})', result.ctype)
+
+
+# TODO(asi1024): Add replace_copy_if
+# TODO(asi1024): Add replace_if
+
+
+@_wrap_thrust_func(['thrust/reverse.h'])
+def reverse(env, exec_policy, first, last):
+    """Reverses a range.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    args = [exec_policy, first, last]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::reverse({params})', _cuda_types.void)
+
+
+@_wrap_thrust_func(['thrust/reverse.h'])
+def reverse_copy(env, exec_policy, first, last, result):
+    """Reverses a range.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    _assert_pointer_type(result)
+    args = [exec_policy, first, last, result]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::reverse_copy({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/scatter.h'])
+def scatter(env, exec_policy, first, last, map, result):
+    """Copies elements from source range into an output range according to map.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    _assert_pointer_type(map)
+    _assert_pointer_type(result)
+    args = [exec_policy, first, last, map, result]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::scatter({params})', _cuda_types.void)
+
+
+# TODO(asi1024): Add scatter_if
+
+
+@_wrap_thrust_func(['thrust/sequence.h'])
+def sequence(env, exec_policy, first, last, init=None, step=None):
+    """Fills the range with a sequence of numbers.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    args = [exec_policy, first, last]
+    if init is not None:
+        args.append(init)
+    if step is not None:
+        _assert_same_type(init, step)
+        args.append(step)
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::sequence({params})', _cuda_types.void)
+
+
+@_wrap_thrust_func(['thrust/set_operations.h'])
+def set_difference(
+        env, exec_policy, first1, last1, first2, last2, result, comp=None):
+    """Constructs a sorted range that is the set difference of sorted inputs.
+    """
+    # TODO(asi1024): Typecheck for Comparable.
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first1, last1)
+    _assert_same_type(first2, last2)
+    _assert_pointer_type(result)
+    args = [exec_policy, first1, last1, first2, last2, result]
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::set_difference({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/set_operations.h'])
+def set_difference_by_key(
+        env, exec_policy, keys_first1, keys_last1, keys_first2, keys_last2,
+        values_first1, values_first2, keys_result, values_result, comp=None):
+    """Constructs the key-value set difference of sorted inputs.
+    """
+    # TODO(asi1024): Typecheck for Comparable.
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(keys_first1, keys_last1)
+    _assert_same_type(keys_first2, keys_last2)
+    _assert_pointer_type(values_first1)
+    _assert_pointer_type(values_first2)
+    _assert_pointer_type(keys_result)
+    _assert_pointer_type(values_result)
+    args = [exec_policy, keys_first1, keys_last1, keys_first2, keys_last2,
+            values_first1, values_first2, keys_result, values_result]
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::set_difference_by_key({params})',
+        _cuda_types.Tuple([keys_result.ctype, values_result.ctype])
+    )
+
+
+@_wrap_thrust_func(['thrust/set_operations.h'])
+def set_intersection(
+        env, exec_policy, first1, last1, first2, last2, result, comp=None):
+    """Constructs a sorted range that is the set intersection of sorted inputs.
+    """
+    # TODO(asi1024): Typecheck for Comparable.
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first1, last1)
+    _assert_same_type(first2, last2)
+    _assert_pointer_type(result)
+    args = [exec_policy, first1, last1, first2, last2, result]
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::set_intersection({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/set_operations.h'])
+def set_intersection_by_key(
+        env, exec_policy, keys_first1, keys_last1, keys_first2, keys_last2,
+        values_first1, keys_result, values_result, comp=None):
+    """Constructs the key-value set intersection of sorted inputs.
+    """
+    # TODO(asi1024): Typecheck for Comparable.
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(keys_first1, keys_last1)
+    _assert_same_type(keys_first2, keys_last2)
+    _assert_pointer_type(values_first1)
+    _assert_pointer_type(keys_result)
+    _assert_pointer_type(values_result)
+    args = [exec_policy, keys_first1, keys_last1, keys_first2, keys_last2,
+            values_first1, keys_result, values_result]
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::set_intersection_by_key({params})',
+        _cuda_types.Tuple([keys_result.ctype, values_result.ctype])
+    )
+
+
+@_wrap_thrust_func(['thrust/set_operations.h'])
+def set_symmetric_difference(
+        env, exec_policy, first1, last1, first2, last2, result, comp=None):
+    """Constructs a sorted range that is the symmetric difference.
+    """
+    # TODO(asi1024): Typecheck for Comparable.
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first1, last1)
+    _assert_same_type(first2, last2)
+    _assert_pointer_type(result)
+    args = [exec_policy, first1, last1, first2, last2, result]
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::set_symmetric_difference({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/set_operations.h'])
+def set_symmetric_difference_by_key(
+        env, exec_policy, keys_first1, keys_last1, keys_first2, keys_last2,
+        values_first1, values_first2, keys_result, values_result, comp=None):
+    """Constructs the key-value symmetric difference of sorted inputs.
+    """
+    # TODO(asi1024): Typecheck for Comparable.
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(keys_first1, keys_last1)
+    _assert_same_type(keys_first2, keys_last2)
+    _assert_pointer_type(values_first1)
+    _assert_pointer_type(values_first2)
+    _assert_pointer_type(keys_result)
+    _assert_pointer_type(values_result)
+    args = [exec_policy, keys_first1, keys_last1, keys_first2, keys_last2,
+            values_first1, values_first2, keys_result, values_result]
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::set_symmetric_difference_by_key({params})',
+        _cuda_types.Tuple([keys_result.ctype, values_result.ctype])
+    )
+
+
+@_wrap_thrust_func(['thrust/set_operations.h'])
+def set_union(
+        env, exec_policy, first1, last1, first2, last2, result, comp=None):
+    """Constructs a sorted range that is the set union of sorted inputs.
+    """
+    # TODO(asi1024): Typecheck for Comparable.
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(first1, last1)
+    _assert_same_type(first2, last2)
+    _assert_pointer_type(result)
+    args = [exec_policy, first1, last1, first2, last2, result]
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::set_union({params})', result.ctype)
+
+
+@_wrap_thrust_func(['thrust/set_operations.h'])
+def set_union_by_key(
+        env, exec_policy, keys_first1, keys_last1, keys_first2, keys_last2,
+        values_first1, values_first2, keys_result, values_result, comp=None):
+    """Constructs the key-value union of sorted inputs.
+    """
+    # TODO(asi1024): Typecheck for Comparable.
+    _assert_exec_policy_type(exec_policy)
+    _assert_same_type(keys_first1, keys_last1)
+    _assert_same_type(keys_first2, keys_last2)
+    _assert_pointer_type(values_first1)
+    _assert_pointer_type(values_first2)
+    _assert_pointer_type(keys_result)
+    _assert_pointer_type(values_result)
+    args = [exec_policy, keys_first1, keys_last1, keys_first2, keys_last2,
+            values_first1, values_first2, keys_result, values_result]
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::set_union_by_key({params})',
+        _cuda_types.Tuple([keys_result.ctype, values_result.ctype])
+    )
+
+
+# TODO(asi1024): Add shuffle
+# TODO(asi1024): Add shuffle_copy
+
+
+@_wrap_thrust_func(['thrust/sort.h'])
+def sort(env, exec_policy, first, last, comp=None):
+    """Sorts the elements in [first, last) into ascending order.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    # TODO(asi1024): Typecheck for Comparable.
+    args = [exec_policy, first, last]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::sort({params})', _cuda_types.void)
+
+
+@_wrap_thrust_func(['thrust/sort.h'])
+def sort_by_key(
+        env, exec_policy, keys_first, keys_last, values_first, comp=None):
+    """Performs key-value sort.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(keys_first)
+    _assert_same_type(keys_first, keys_last)
+    _assert_pointer_type(values_first)
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    # TODO(asi1024): Typecheck for Comparable.
+    args = [exec_policy, keys_first, keys_last, values_first]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::sort_by_key({params})', _cuda_types.void)
+
+
+# TODO(asi1024): Add stable_partition
+# TODO(asi1024): Add stable_partition_copy
+
+
+@_wrap_thrust_func(['thrust/sort.h'])
+def stable_sort(env, exec_policy, first, last, comp=None):
+    """Sorts the elements in [first, last) into ascending order.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    # TODO(asi1024): Typecheck for Comparable.
+    args = [exec_policy, first, last]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::stable_sort({params})', _cuda_types.void)
+
+
+@_wrap_thrust_func(['thrust/sort.h'])
+def stable_sort_by_key(
+        env, exec_policy, keys_first, keys_last, values_first, comp=None):
+    """Performs key-value sort.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(keys_first)
+    _assert_same_type(keys_first, keys_last)
+    _assert_pointer_type(values_first)
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    # TODO(asi1024): Typecheck for Comparable.
+    args = [exec_policy, keys_first, keys_last, values_first]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::stable_sort_by_key({params})', _cuda_types.void)
+
+
+@_wrap_thrust_func(['thrust/swap.h'])
+def swap_ranges(env, exec_policy, first1, last1, first2):
+    """Swaps each of the elements in the range.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first1)
+    _assert_same_type(first1, last1)
+    _assert_pointer_type(first2)
+    # TODO(asi1024): Typecheck for Comparable.
+    args = [exec_policy, first1, last1, first2]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::swap_ranges({params})', first2.ctype)
+
+
+# TODO(asi1024): Add tabulate
+# TODO(asi1024): Add transform
+# TODO(asi1024): Add transform_exclusive_scan
+# TODO(asi1024): Add transform_if
+# TODO(asi1024): Add transform_inclusive_scan
+# TODO(asi1024): Add transform_reduce
+# TODO(asi1024): Add uninitialized_copy
+# TODO(asi1024): Add uninitialized_copy_n
+# TODO(asi1024): Add uninitialized_fill
+# TODO(asi1024): Add uninitialized_fill_n
+
+
+@_wrap_thrust_func(['thrust/unique.h'])
+def unique(env, exec_policy, first, last, binary_pred=None):
+    """Removes all but the first element of the group.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+    if binary_pred is not None:
+        raise NotImplementedError('binary_pred option is not supported')
+    args = [exec_policy, first, last]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::unique({params})', first.ctype)
+
+
+@_wrap_thrust_func(['thrust/unique.h'])
+def unique_by_key(
+        env, exec_policy, keys_first, keys_last, values_first,
+        binary_pred=None):
+    """Uniques key-value pairs.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(keys_first)
+    _assert_same_type(keys_first, keys_last)
+    _assert_pointer_type(values_first)
+    args = [exec_policy, keys_first, keys_last, values_first]
+    if binary_pred is not None:
+        raise NotImplementedError('binary_pred option is not supported')
+    params = ', '.join([a.code for a in args])
+    return _Data(
+        f'thrust::unique_by_key({params})',
+        _cuda_types.Tuple([keys_first.ctype, values_first.ctype]),
+    )
+
+
+# TODO(asi1024): Add unique_by_key_copy
+# TODO(asi1024): Add unique_by_key
+# TODO(asi1024): Add unique_count
+
+
+@_wrap_thrust_func(['thrust/binary_search.h'])
+def upper_bound(env, exec_policy, first, last, *args):
+    """Attempts to find the element value with binary search.
+    """
+    _assert_exec_policy_type(exec_policy)
+    _assert_pointer_type(first)
+    _assert_same_type(first, last)
+
+    if 1 <= len(args) <= 2:
+        value = args[0]
+        comp = args[1] if len(args) == 2 else None
+        _assert_pointer_of(first, value)
+        result_ctype = first.ctype
+    elif 3 <= len(args) <= 4:
+        value_first = args[0]
+        value_last = args[1]
+        result = args[2]
+        comp = args[3] if len(args) == 4 else None
+        _assert_same_pointer_type(first, value_first)
+        _assert_same_type(value_first, value_last)
+        result_ctype = result.ctype
+    else:
+        raise TypeError('Invalid number of inputs of thrust.upper_bound')
+
+    if comp is not None:
+        raise NotImplementedError('comp option is not supported')
+    args = [exec_policy, first, last, *args]
+    params = ', '.join([a.code for a in args])
+    return _Data(f'thrust::upper_bound({params})', result_ctype)
diff --git a/cupyx/lapack.py b/cupyx/lapack.py
new file mode 100644
index 0000000..d9df37e
--- /dev/null
+++ b/cupyx/lapack.py
@@ -0,0 +1,343 @@
+import numpy as _numpy
+
+import cupy as _cupy
+from cupy_backends.cuda.libs import cublas as _cublas
+from cupy_backends.cuda.libs import cusolver as _cusolver
+from cupy.cuda import device as _device
+import cupyx.cusolver
+
+
+def gesv(a, b):
+    """Solve a linear matrix equation using cusolverDn<t>getr[fs]().
+
+    Computes the solution to a system of linear equation ``ax = b``.
+
+    Args:
+        a (cupy.ndarray): The matrix with dimension ``(M, M)``.
+        b (cupy.ndarray): The matrix with dimension ``(M)`` or ``(M, K)``.
+
+    Returns:
+        cupy.ndarray:
+            The matrix with dimension ``(M)`` or ``(M, K)``.
+
+    Note: ``a`` and ``b`` will be overwritten.
+    """
+    if a.ndim != 2:
+        raise ValueError('a.ndim must be 2 (actual: {})'.format(a.ndim))
+    if b.ndim not in (1, 2):
+        raise ValueError('b.ndim must be 1 or 2 (actual: {})'.format(b.ndim))
+    if a.shape[0] != a.shape[1]:
+        raise ValueError('a must be a square matrix.')
+    if a.shape[0] != b.shape[0]:
+        raise ValueError('shape mismatch (a: {}, b: {}).'.
+                         format(a.shape, b.shape))
+    if a.dtype != b.dtype:
+        raise TypeError('dtype mismatch (a: {}, b: {})'.
+                        format(a.dtype, b.dtype))
+    dtype = a.dtype
+    if dtype == 'f':
+        t = 's'
+    elif dtype == 'd':
+        t = 'd'
+    elif dtype == 'F':
+        t = 'c'
+    elif dtype == 'D':
+        t = 'z'
+    else:
+        raise TypeError('unsupported dtype (actual:{})'.format(a.dtype))
+    helper = getattr(_cusolver, t + 'getrf_bufferSize')
+    getrf = getattr(_cusolver, t + 'getrf')
+    getrs = getattr(_cusolver, t + 'getrs')
+
+    n = b.shape[0]
+    nrhs = b.shape[1] if b.ndim == 2 else 1
+    if a._f_contiguous:
+        trans = _cublas.CUBLAS_OP_N
+    elif a._c_contiguous:
+        trans = _cublas.CUBLAS_OP_T
+    else:
+        raise ValueError('a must be F-contiguous or C-contiguous.')
+    if not b._f_contiguous:
+        raise ValueError('b must be F-contiguous.')
+
+    handle = _device.get_cusolver_handle()
+    dipiv = _cupy.empty(n, dtype=_numpy.int32)
+    dinfo = _cupy.empty(1, dtype=_numpy.int32)
+    lwork = helper(handle, n, n, a.data.ptr, n)
+    dwork = _cupy.empty(lwork, dtype=a.dtype)
+    # LU factrization (A = L * U)
+    getrf(handle, n, n, a.data.ptr, n, dwork.data.ptr, dipiv.data.ptr,
+          dinfo.data.ptr)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        getrf, dinfo)
+    # Solves Ax = b
+    getrs(handle, trans, n, nrhs, a.data.ptr, n,
+          dipiv.data.ptr, b.data.ptr, n, dinfo.data.ptr)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        getrs, dinfo)
+
+
+def gels(a, b):
+    """Solves over/well/under-determined linear systems.
+
+    Computes least-square solution to equation ``ax = b` by QR factorization
+    using cusolverDn<t>geqrf().
+
+    Args:
+        a (cupy.ndarray): The matrix with dimension ``(M, N)``.
+        b (cupy.ndarray): The matrix with dimension ``(M)`` or ``(M, K)``.
+
+    Returns:
+        cupy.ndarray:
+            The matrix with dimension ``(N)`` or ``(N, K)``.
+    """
+    if a.ndim != 2:
+        raise ValueError('a.ndim must be 2 (actual: {})'.format(a.ndim))
+    if b.ndim == 1:
+        nrhs = 1
+    elif b.ndim == 2:
+        nrhs = b.shape[1]
+    else:
+        raise ValueError('b.ndim must be 1 or 2 (actual: {})'.format(b.ndim))
+    if a.shape[0] != b.shape[0]:
+        raise ValueError('shape mismatch (a: {}, b: {}).'.
+                         format(a.shape, b.shape))
+    if a.dtype != b.dtype:
+        raise ValueError('dtype mismatch (a: {}, b: {}).'.
+                         format(a.dtype, b.dtype))
+
+    dtype = a.dtype
+    if dtype == 'f':
+        t = 's'
+    elif dtype == 'd':
+        t = 'd'
+    elif dtype == 'F':
+        t = 'c'
+    elif dtype == 'D':
+        t = 'z'
+    else:
+        raise ValueError('unsupported dtype (actual: {})'.format(dtype))
+
+    geqrf_helper = getattr(_cusolver, t + 'geqrf_bufferSize')
+    geqrf = getattr(_cusolver, t + 'geqrf')
+    trsm = getattr(_cublas, t + 'trsm')
+    if t in 'sd':
+        ormqr_helper = getattr(_cusolver, t + 'ormqr_bufferSize')
+        ormqr = getattr(_cusolver, t + 'ormqr')
+    else:
+        ormqr_helper = getattr(_cusolver, t + 'unmqr_bufferSize')
+        ormqr = getattr(_cusolver, t + 'unmqr')
+
+    no_trans = _cublas.CUBLAS_OP_N
+    if dtype.char in 'fd':
+        trans = _cublas.CUBLAS_OP_T
+    else:
+        trans = _cublas.CUBLAS_OP_C
+
+    m, n = a.shape
+    mn_min = min(m, n)
+    dev_info = _cupy.empty(1, dtype=_numpy.int32)
+    tau = _cupy.empty(mn_min, dtype=dtype)
+    cusolver_handle = _device.get_cusolver_handle()
+    cublas_handle = _device.get_cublas_handle()
+    one = _numpy.array(1.0, dtype=dtype)
+
+    if m >= n:  # over/well-determined systems
+        a = a.copy(order='F')
+        b = b.copy(order='F')
+
+        # geqrf (QR decomposition, A = Q * R)
+        ws_size = geqrf_helper(cusolver_handle, m, n, a.data.ptr, m)
+        workspace = _cupy.empty(ws_size, dtype=dtype)
+        geqrf(cusolver_handle, m, n, a.data.ptr, m, tau.data.ptr,
+              workspace.data.ptr, ws_size, dev_info.data.ptr)
+        _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+            geqrf, dev_info)
+
+        # ormqr (Computes Q^T * B)
+        ws_size = ormqr_helper(
+            cusolver_handle, _cublas.CUBLAS_SIDE_LEFT, trans, m, nrhs, mn_min,
+            a.data.ptr, m, tau.data.ptr, b.data.ptr, m)
+        workspace = _cupy.empty(ws_size, dtype=dtype)
+        ormqr(cusolver_handle, _cublas.CUBLAS_SIDE_LEFT, trans, m, nrhs,
+              mn_min, a.data.ptr, m, tau.data.ptr, b.data.ptr, m,
+              workspace.data.ptr, ws_size, dev_info.data.ptr)
+        _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+            ormqr, dev_info)
+
+        # trsm (Solves R * X = (Q^T * B))
+        trsm(cublas_handle, _cublas.CUBLAS_SIDE_LEFT,
+             _cublas.CUBLAS_FILL_MODE_UPPER, no_trans,
+             _cublas.CUBLAS_DIAG_NON_UNIT, mn_min, nrhs,
+             one.ctypes.data, a.data.ptr, m, b.data.ptr, m)
+
+        return b[:n]
+
+    else:  # under-determined systems
+        a = a.conj().T.copy(order='F')
+        bb = b
+        out_shape = (n,) if b.ndim == 1 else (n, nrhs)
+        b = _cupy.zeros(out_shape, dtype=dtype, order='F')
+        b[:m] = bb
+
+        # geqrf (QR decomposition, A^T = Q * R)
+        ws_size = geqrf_helper(cusolver_handle, n, m, a.data.ptr, n)
+        workspace = _cupy.empty(ws_size, dtype=dtype)
+        geqrf(cusolver_handle, n, m, a.data.ptr, n, tau.data.ptr,
+              workspace.data.ptr, ws_size, dev_info.data.ptr)
+        _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+            geqrf, dev_info)
+
+        # trsm (Solves R^T * Z = B)
+        trsm(cublas_handle, _cublas.CUBLAS_SIDE_LEFT,
+             _cublas.CUBLAS_FILL_MODE_UPPER, trans,
+             _cublas.CUBLAS_DIAG_NON_UNIT, m, nrhs,
+             one.ctypes.data, a.data.ptr, n, b.data.ptr, n)
+
+        # ormqr (Computes Q * Z)
+        ws_size = ormqr_helper(
+            cusolver_handle, _cublas.CUBLAS_SIDE_LEFT, no_trans, n, nrhs,
+            mn_min, a.data.ptr, n, tau.data.ptr, b.data.ptr, n)
+        workspace = _cupy.empty(ws_size, dtype=dtype)
+        ormqr(cusolver_handle, _cublas.CUBLAS_SIDE_LEFT, no_trans, n, nrhs,
+              mn_min, a.data.ptr, n, tau.data.ptr, b.data.ptr, n,
+              workspace.data.ptr, ws_size, dev_info.data.ptr)
+        _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+            ormqr, dev_info)
+
+        return b
+
+
+def _batched_posv(a, b):
+
+    if not cupyx.cusolver.check_availability('potrsBatched'):
+        raise RuntimeError('potrsBatched is not available')
+
+    dtype = _numpy.promote_types(a.dtype, b.dtype)
+    dtype = _numpy.promote_types(dtype, 'f')
+
+    if dtype == 'f':
+        potrfBatched = _cusolver.spotrfBatched
+        potrsBatched = _cusolver.spotrsBatched
+    elif dtype == 'd':
+        potrfBatched = _cusolver.dpotrfBatched
+        potrsBatched = _cusolver.dpotrsBatched
+    elif dtype == 'F':
+        potrfBatched = _cusolver.cpotrfBatched
+        potrsBatched = _cusolver.cpotrsBatched
+    elif dtype == 'D':
+        potrfBatched = _cusolver.zpotrfBatched
+        potrsBatched = _cusolver.zpotrsBatched
+    else:
+        msg = ('dtype must be float32, float64, complex64 or complex128'
+               ' (actual: {})'.format(a.dtype))
+        raise ValueError(msg)
+
+    a = a.astype(dtype, order='C', copy=True)
+    ap = _cupy._core._mat_ptrs(a)
+    lda, n = a.shape[-2:]
+    batch_size = int(_numpy.prod(a.shape[:-2]))
+
+    handle = _device.get_cusolver_handle()
+    uplo = _cublas.CUBLAS_FILL_MODE_LOWER
+    dev_info = _cupy.empty(batch_size, dtype=_numpy.int32)
+
+    # Cholesky factorization
+    potrfBatched(handle, uplo, n, ap.data.ptr, lda, dev_info.data.ptr,
+                 batch_size)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        potrfBatched, dev_info)
+
+    b_shape = b.shape
+    b = b.conj().reshape(batch_size, n, -1).astype(dtype, order='C', copy=True)
+    bp = _cupy._core._mat_ptrs(b)
+    ldb, nrhs = b.shape[-2:]
+    dev_info = _cupy.empty(1, dtype=_numpy.int32)
+
+    # NOTE: potrsBatched does not currently support nrhs > 1 (CUDA v10.2)
+    # Solve: A[i] * X[i] = B[i]
+    potrsBatched(handle, uplo, n, nrhs, ap.data.ptr, lda, bp.data.ptr, ldb,
+                 dev_info.data.ptr, batch_size)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        potrsBatched, dev_info)
+
+    # TODO: check if conj() is necessary when nrhs > 1
+    return b.conj().reshape(b_shape)
+
+
+def posv(a, b):
+    """Solve the linear equations A x = b via Cholesky factorization of A,
+    where A is a real symmetric or complex Hermitian positive-definite matrix.
+
+    If matrix ``A`` is not positive definite, Cholesky factorization fails
+    and it raises an error.
+
+    Note: For batch input, NRHS > 1 is not currently supported.
+
+    Args:
+        a (cupy.ndarray): Array of real symmetric or complex hermitian
+            matrices with dimension (..., N, N).
+        b (cupy.ndarray): right-hand side (..., N) or (..., N, NRHS).
+    Returns:
+        x (cupy.ndarray): The solution (shape matches b).
+    """
+
+    _util = _cupy.linalg._util
+    _util._assert_cupy_array(a, b)
+    _util._assert_stacked_2d(a)
+    _util._assert_stacked_square(a)
+
+    if a.ndim > 2:
+        return _batched_posv(a, b)
+
+    dtype = _numpy.promote_types(a.dtype, b.dtype)
+    dtype = _numpy.promote_types(dtype, 'f')
+
+    if dtype == 'f':
+        potrf = _cusolver.spotrf
+        potrf_bufferSize = _cusolver.spotrf_bufferSize
+        potrs = _cusolver.spotrs
+    elif dtype == 'd':
+        potrf = _cusolver.dpotrf
+        potrf_bufferSize = _cusolver.dpotrf_bufferSize
+        potrs = _cusolver.dpotrs
+    elif dtype == 'F':
+        potrf = _cusolver.cpotrf
+        potrf_bufferSize = _cusolver.cpotrf_bufferSize
+        potrs = _cusolver.cpotrs
+    elif dtype == 'D':
+        potrf = _cusolver.zpotrf
+        potrf_bufferSize = _cusolver.zpotrf_bufferSize
+        potrs = _cusolver.zpotrs
+    else:
+        msg = ('dtype must be float32, float64, complex64 or complex128'
+               ' (actual: {})'.format(a.dtype))
+        raise ValueError(msg)
+
+    a = a.astype(dtype, order='F', copy=True)
+    lda, n = a.shape
+
+    handle = _device.get_cusolver_handle()
+    uplo = _cublas.CUBLAS_FILL_MODE_LOWER
+    dev_info = _cupy.empty(1, dtype=_numpy.int32)
+
+    worksize = potrf_bufferSize(handle, uplo, n, a.data.ptr, lda)
+    workspace = _cupy.empty(worksize, dtype=dtype)
+
+    # Cholesky factorization
+    potrf(handle, uplo, n, a.data.ptr, lda, workspace.data.ptr,
+          worksize, dev_info.data.ptr)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        potrf, dev_info)
+
+    b_shape = b.shape
+    b = b.reshape(n, -1).astype(dtype, order='F', copy=True)
+    ldb, nrhs = b.shape
+
+    # Solve: A * X = B
+    potrs(handle, uplo, n, nrhs, a.data.ptr, lda, b.data.ptr, ldb,
+          dev_info.data.ptr)
+    _cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
+        potrs, dev_info)
+
+    return _cupy.ascontiguousarray(b.reshape(b_shape))
diff --git a/cupyx/linalg/__init__.py b/cupyx/linalg/__init__.py
new file mode 100644
index 0000000..1c58805
--- /dev/null
+++ b/cupyx/linalg/__init__.py
@@ -0,0 +1,3 @@
+# "NOQA" to suppress flake8 warning
+from cupyx.linalg import sparse  # NOQA
+from cupyx.linalg._solve import invh  # NOQA
diff --git a/cupyx/linalg/_solve.py b/cupyx/linalg/_solve.py
new file mode 100644
index 0000000..31e6b0e
--- /dev/null
+++ b/cupyx/linalg/_solve.py
@@ -0,0 +1,26 @@
+from cupy.linalg import _util
+from cupyx import lapack
+
+
+def invh(a):
+    """Compute the inverse of a Hermitian matrix.
+
+    This function computes a inverse of a real symmetric or complex hermitian
+    positive-definite matrix using Cholesky factorization. If matrix ``a`` is
+    not positive definite, Cholesky factorization fails and it raises an error.
+
+    Args:
+        a (cupy.ndarray): Real symmetric or complex hermitian maxtix.
+
+    Returns:
+        cupy.ndarray: The inverse of matrix ``a``.
+    """
+
+    _util._assert_cupy_array(a)
+    # TODO: Use `_assert_stacked_2d` instead, once cusolver supports nrhs > 1
+    # for potrsBatched
+    _util._assert_2d(a)
+    _util._assert_stacked_square(a)
+
+    b = _util.stacked_identity_like(a)
+    return lapack.posv(a, b)
diff --git a/cupyx/linalg/sparse/__init__.py b/cupyx/linalg/sparse/__init__.py
new file mode 100644
index 0000000..52c8b74
--- /dev/null
+++ b/cupyx/linalg/sparse/__init__.py
@@ -0,0 +1,3 @@
+
+# "NOQA" to suppress flake8 warning
+from cupyx.linalg.sparse._solve import lschol  # NOQA
diff --git a/cupyx/linalg/sparse/_solve.py b/cupyx/linalg/sparse/_solve.py
new file mode 100644
index 0000000..ca12ae0
--- /dev/null
+++ b/cupyx/linalg/sparse/_solve.py
@@ -0,0 +1,62 @@
+import numpy
+
+import cupy
+from cupy.cuda import cusolver
+from cupy.cuda import device
+from cupy.linalg import _util
+from cupyx.scipy import sparse
+
+
+def lschol(A, b):
+    """Solves linear system with cholesky decomposition.
+
+    Find the solution to a large, sparse, linear system of equations.
+    The function solves ``Ax = b``. Given two-dimensional matrix ``A`` is
+    decomposed into ``L * L^*``.
+
+    Args:
+        A (cupy.ndarray or cupyx.scipy.sparse.csr_matrix): The input matrix
+            with dimension ``(N, N)``. Must be positive-definite input matrix.
+            Only symmetric real matrix is supported currently.
+        b (cupy.ndarray): Right-hand side vector.
+
+    Returns:
+        ret (cupy.ndarray): The solution vector ``x``.
+
+    """
+
+    if not sparse.isspmatrix_csr(A):
+        A = sparse.csr_matrix(A)
+    # csr_matrix is 2d
+    _util._assert_stacked_square(A)
+    _util._assert_cupy_array(b)
+    m = A.shape[0]
+    if b.ndim != 1 or len(b) != m:
+        raise ValueError('b must be 1-d array whose size is same as A')
+
+    # Cast to float32 or float64
+    if A.dtype == 'f' or A.dtype == 'd':
+        dtype = A.dtype
+    else:
+        dtype = numpy.promote_types(A.dtype, 'f')
+
+    handle = device.get_cusolver_sp_handle()
+    nnz = A.nnz
+    tol = 1.0
+    reorder = 1
+    x = cupy.empty(m, dtype=dtype)
+    singularity = numpy.empty(1, numpy.int32)
+
+    if dtype == 'f':
+        csrlsvchol = cusolver.scsrlsvchol
+    else:
+        csrlsvchol = cusolver.dcsrlsvchol
+    csrlsvchol(
+        handle, m, nnz, A._descr.descriptor, A.data.data.ptr,
+        A.indptr.data.ptr, A.indices.data.ptr, b.data.ptr, tol, reorder,
+        x.data.ptr, singularity.ctypes.data)
+
+    # The return type of SciPy is always float64.
+    x = x.astype(numpy.float64)
+
+    return x
diff --git a/cupyx/optimizing/__init__.py b/cupyx/optimizing/__init__.py
new file mode 100644
index 0000000..bffcb15
--- /dev/null
+++ b/cupyx/optimizing/__init__.py
@@ -0,0 +1 @@
+from cupyx.optimizing._optimize import optimize  # NOQA
diff --git a/cupyx/optimizing/_optimize.py b/cupyx/optimizing/_optimize.py
new file mode 100644
index 0000000..1e63be7
--- /dev/null
+++ b/cupyx/optimizing/_optimize.py
@@ -0,0 +1,109 @@
+import contextlib
+import math
+import os
+import warnings
+
+
+try:
+    import optuna
+    _optuna_available = True
+except ImportError:
+    _optuna_available = False
+
+
+from cupy._core import _optimize_config
+from cupyx import profiler
+
+
+def _optimize(
+        optimize_config, target_func, suggest_func,
+        default_best, ignore_error=()):
+    assert isinstance(optimize_config, _optimize_config._OptimizationConfig)
+    assert callable(target_func)
+    assert callable(suggest_func)
+
+    def objective(trial):
+        args = suggest_func(trial)
+        max_total_time = optimize_config.max_total_time_per_trial
+        try:
+            perf = profiler.benchmark(
+                target_func, args, max_duration=max_total_time)
+            return perf.gpu_times.mean()
+        except Exception as e:
+            if isinstance(e, ignore_error):
+                return math.inf
+            else:
+                raise e
+
+    study = optuna.create_study()
+    study.enqueue_trial(default_best)
+    study.optimize(
+        objective,
+        n_trials=optimize_config.max_trials,
+        timeout=optimize_config.timeout)
+    return study.best_trial
+
+
+@contextlib.contextmanager
+def optimize(*, key=None, path=None, readonly=False, **config_dict):
+    """Context manager that optimizes kernel launch parameters.
+
+    In this context, CuPy's routines find the best kernel launch parameter
+    values (e.g., the number of threads and blocks). The found values are
+    cached and reused with keys as the shapes, strides and dtypes of the
+    given inputs arrays.
+
+    Args:
+        key (string or None): The cache key of optimizations.
+        path (string or None): The path to save optimization cache records.
+            When path is specified and exists, records will be loaded from
+            the path. When readonly option is set to ``False``, optimization
+            cache records will be saved to the path after the optimization.
+        readonly (bool): See the description of ``path`` option.
+        max_trials (int): The number of trials that defaults to 100.
+        timeout (float):
+            Stops study after the given number of seconds. Default is 1.
+        max_total_time_per_trial (float):
+            Repeats measuring the execution time of the routine for the
+            given number of seconds. Default is 0.1.
+
+    Examples
+    --------
+    >>> import cupy
+    >>> from cupyx import optimizing
+    >>>
+    >>> x = cupy.arange(100)
+    >>> with optimizing.optimize():
+    ...     cupy.sum(x)
+    ...
+    array(4950)
+
+    .. note::
+      Optuna (https://optuna.org) installation is required.
+      Currently it works for reduction operations only.
+    """
+    if not _optuna_available:
+        raise RuntimeError(
+            'Optuna is required to run optimization. '
+            'See https://optuna.org/ for the installation instructions.')
+
+    old_context = _optimize_config.get_current_context()
+    context = _optimize_config.get_new_context(key, _optimize, config_dict)
+    _optimize_config.set_current_context(context)
+
+    if path is not None:
+        if os.path.exists(path):
+            context.load(path)
+        elif readonly:
+            warnings.warn('''
+The specified path {} could not be found, and the readonly option is set.
+The optimization results will never be stored.
+'''.format(path))
+
+    try:
+        yield context
+        if path is not None and not readonly:
+            if context._is_dirty() or not os.path.exists(path):
+                context.save(path)
+    finally:
+        _optimize_config.set_current_context(old_context)
diff --git a/cupyx/profiler/__init__.py b/cupyx/profiler/__init__.py
new file mode 100644
index 0000000..e2b86e1
--- /dev/null
+++ b/cupyx/profiler/__init__.py
@@ -0,0 +1,31 @@
+import contextlib as _contextlib
+from cupy.cuda import profiler as _profiler
+from cupyx.profiler._time import benchmark  # NOQA
+from cupyx.profiler._time_range import time_range  # NOQA
+
+
+@_contextlib.contextmanager
+def profile():
+    """Enable CUDA profiling during with statement.
+
+    This function enables profiling on entering a with statement, and disables
+    profiling on leaving the statement.
+
+    >>> with cupyx.profiler.profile():
+    ...    # do something you want to measure
+    ...    pass
+
+    .. note::
+        When starting ``nvprof`` from the command line, manually setting
+        ``--profile-from-start off`` may be required for the desired behavior.
+        Likewise, when using ``nsys profile`` setting ``-c cudaProfilerApi``
+        may be required.
+
+    .. seealso:: :func:`cupy.cuda.profiler.start`,
+        :func:`cupy.cuda.profiler.stop`
+    """
+    _profiler.start()
+    try:
+        yield
+    finally:
+        _profiler.stop()
diff --git a/cupyx/profiler/_time.py b/cupyx/profiler/_time.py
new file mode 100644
index 0000000..dbd9981
--- /dev/null
+++ b/cupyx/profiler/_time.py
@@ -0,0 +1,227 @@
+import math as _math
+import time as _time
+
+import numpy as _numpy
+
+import cupy as _cupy
+from cupy_backends.cuda.api import runtime
+
+
+class _PerfCaseResult:
+    """ An obscure object encompassing timing results recorded by
+    :func:`~cupyx.profiler.benchmark`. Simple statistics can be obtained by
+    converting an instance of this class to a string.
+
+    .. warning::
+        This API is currently experimental and subject to change in future
+        releases.
+
+    """
+
+    def __init__(self, name, ts, devices):
+        assert ts.ndim == 2
+        assert ts.shape[0] == len(devices) + 1
+        assert ts.shape[1] > 0
+        self.name = name
+        self._ts = ts
+        self._devices = devices
+
+    def __repr__(self) -> str:
+        """ Returns a string representation of the object.
+
+        Returns:
+            str: A string representation of the object.
+        """
+        return self.to_str(show_gpu=True)
+
+    @property
+    def cpu_times(self) -> _numpy.ndarray:
+        """A :class:`numpy.ndarray` of shape ``(n_repeat,)``, holding times spent
+        on CPU in seconds.
+
+        These values are delta of the host-side performance counter
+        (:func:`time.perf_counter`) between each repeat step.
+        """  # NOQA
+        return self._ts[0]
+
+    @property
+    def gpu_times(self) -> _numpy.ndarray:
+        """A :class:`numpy.ndarray` of shape ``(len(devices), n_repeat)``,
+        holding times spent on GPU in seconds.
+
+        These values are measured using ``cudaEventElapsedTime`` with events
+        recoreded before/after each repeat step.
+        """
+        return self._ts[1:]
+
+    @staticmethod
+    def _to_str_per_item(device_name, t):
+        assert t.ndim == 1
+        assert t.size > 0
+        t_us = t * 1e6
+
+        s = '    {}: {:9.03f} us'.format(device_name, t_us.mean())
+        if t.size > 1:
+            s += '   +/- {:6.03f} (min: {:9.03f} / max: {:9.03f}) us'.format(
+                t_us.std(), t_us.min(), t_us.max())
+        return s
+
+    def to_str(self, show_gpu=False):
+        results = [self._to_str_per_item('CPU', self._ts[0])]
+        if show_gpu:
+            for i, d in enumerate(self._devices):
+                results.append(
+                    self._to_str_per_item('GPU-{}'.format(d),
+                                          self._ts[1 + i]))
+        return '{:<20s}:{}'.format(self.name, ' '.join(results))
+
+    def __str__(self):
+        return self.to_str(show_gpu=True)
+
+
+def benchmark(
+        func, args=(), kwargs={}, n_repeat=10000, *,
+        name=None, n_warmup=10, max_duration=_math.inf, devices=None):
+    """ Timing utility for measuring time spent by both CPU and GPU.
+
+    This function is a very convenient helper for setting up a timing test. The
+    GPU time is properly recorded by synchronizing internal streams. As a
+    result, to time a multi-GPU function all participating devices must be
+    passed as the ``devices`` argument so that this helper knows which devices
+    to record. A simple example is given as follows:
+
+    .. code-block:: py
+
+        import cupy as cp
+        from cupyx.profiler import benchmark
+
+        def f(a, b):
+            return 3 * cp.sin(-a) * b
+
+        a = 0.5 - cp.random.random((100,))
+        b = cp.random.random((100,))
+        print(benchmark(f, (a, b), n_repeat=1000))
+
+
+    Args:
+        func (callable): a callable object to be timed.
+        args (tuple): positional argumens to be passed to the callable.
+        kwargs (dict): keyword arguments to be passed to the callable.
+        n_repeat (int): number of times the callable is called. Increasing
+            this value would improve the collected statistics at the cost
+            of longer test time.
+        name (str): the function name to be reported. If not given, the
+            callable's ``__name__`` attribute is used.
+        n_warmup (int): number of times the callable is called. The warm-up
+            runs are not timed.
+        max_duration (float): the maximum time (in seconds) that the entire
+            test can use. If the taken time is longer than this limit, the test
+            is stopped and the statistics collected up to the breakpoint is
+            reported.
+        devices (tuple): a tuple of device IDs (int) that will be timed during
+            the timing test. If not given, the current device is used.
+
+    Returns:
+        :class:`~cupyx.profiler._time._PerfCaseResult`:
+            an object collecting all test results.
+
+    """
+
+    if name is None:
+        name = func.__name__
+
+    if devices is None:
+        devices = (_cupy.cuda.get_device_id(),)
+
+    if not callable(func):
+        raise ValueError('`func` should be a callable object.')
+    if not isinstance(args, tuple):
+        raise ValueError('`args` should be of tuple type.')
+    if not isinstance(kwargs, dict):
+        raise ValueError('`kwargs` should be of dict type.')
+    if not isinstance(n_repeat, int):
+        raise ValueError('`n_repeat` should be an integer.')
+    if not isinstance(name, str):
+        raise ValueError('`name` should be a string.')
+    if not isinstance(n_warmup, int):
+        raise ValueError('`n_warmup` should be an integer.')
+    if not _numpy.isreal(max_duration):
+        raise ValueError('`max_duration` should be given in seconds')
+    if not isinstance(devices, tuple):
+        raise ValueError('`devices` should be of tuple type')
+
+    return _repeat(
+        func, args, kwargs, n_repeat, name, n_warmup, max_duration, devices)
+
+
+def _repeat(
+        func, args, kwargs, n_repeat, name, n_warmup, max_duration, devices):
+
+    events_1 = []
+    events_2 = []
+
+    for i in devices:
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(i)
+            events_1.append(_cupy.cuda.stream.Event())
+            events_2.append(_cupy.cuda.stream.Event())
+        finally:
+            runtime.setDevice(prev_device)
+
+    for i in range(n_warmup):
+        func(*args, **kwargs)
+
+    for event, device in zip(events_1, devices):
+        prev_device = runtime.getDevice()
+        try:
+            runtime.setDevice(device)
+            event.record()
+        finally:
+            runtime.setDevice(prev_device)
+        event.synchronize()
+
+    cpu_times = []
+    gpu_times = [[] for i in events_1]
+    duration = 0
+    for i in range(n_repeat):
+        for event, device in zip(events_1, devices):
+            prev_device = runtime.getDevice()
+            try:
+                runtime.setDevice(device)
+                event.record()
+            finally:
+                runtime.setDevice(prev_device)
+
+        t1 = _time.perf_counter()
+
+        func(*args, **kwargs)
+
+        t2 = _time.perf_counter()
+        cpu_time = t2 - t1
+        cpu_times.append(cpu_time)
+
+        for event, device in zip(events_2, devices):
+            prev_device = runtime.getDevice()
+            try:
+                runtime.setDevice(device)
+                event.record()
+            finally:
+                runtime.setDevice(prev_device)
+        for event, device in zip(events_2, devices):
+            prev_device = runtime.getDevice()
+            try:
+                runtime.setDevice(device)
+                event.synchronize()
+            finally:
+                runtime.setDevice(prev_device)
+        for i, (ev1, ev2) in enumerate(zip(events_1, events_2)):
+            gpu_time = _cupy.cuda.get_elapsed_time(ev1, ev2) * 1e-3
+            gpu_times[i].append(gpu_time)
+
+        duration += _time.perf_counter() - t1
+        if duration > max_duration:
+            break
+
+    ts = _numpy.asarray([cpu_times] + gpu_times, dtype=_numpy.float64)
+    return _PerfCaseResult(name, ts, devices=devices)
diff --git a/cupyx/profiler/_time_range.py b/cupyx/profiler/_time_range.py
new file mode 100644
index 0000000..c89e077
--- /dev/null
+++ b/cupyx/profiler/_time_range.py
@@ -0,0 +1,89 @@
+import functools
+
+from cupy import cuda
+from cupy_backends.cuda.api import runtime
+
+
+# Note: We use an (old-fashioned) custom object instead of
+# @contextlib.contextmanager because for backward compatibility
+# when used as a decorator this object needs to fetch the target
+# function name.
+class time_range:
+    """Mark function calls with ranges using NVTX/rocTX. This object can be
+    used either as a decorator or a context manager.
+
+    When used as a decorator, the decorated function calls are marked as
+    ranges:
+
+    >>> from cupyx.profiler import time_range
+    >>> @time_range()
+    ... def function_to_profile():
+    ...     pass
+
+    When used as a context manager, it describes the enclosed block as a nested
+    range:
+
+    >>> from cupyx.profiler import time_range
+    >>> with time_range('some range in green', color_id=0):
+    ...    # do something you want to measure
+    ...    pass
+
+    The marked ranges are visible in the profiler (such as nvvp, nsys-ui, etc)
+    timeline.
+
+    Args:
+        message (str): Name of a range. When used as a decorator, the default
+            is ``func.__name__``.
+        color_id: range color ID
+        argb_color: range color in ARGB (e.g. 0xFF00FF00 for green)
+        sync (bool): If ``True``, waits for completion of all outstanding
+            processing on GPU before calling :func:`cupy.cuda.nvtx.RangePush()`
+            or :func:`cupy.cuda.nvtx.RangePop()`
+
+    .. seealso:: :func:`cupy.cuda.nvtx.RangePush`,
+        :func:`cupy.cuda.nvtx.RangePop`
+    """
+
+    def __init__(
+            self, message=None, color_id=None, argb_color=None, sync=False):
+        if not cuda.nvtx.available:
+            raise RuntimeError('nvtx is not installed')
+
+        if color_id is not None and argb_color is not None:
+            raise ValueError(
+                'Only either color_id or argb_color can be specified'
+            )
+        self.message = message
+        self.color_id = color_id if color_id is not None else -1
+        self.argb_color = argb_color
+        self.sync = sync
+
+    def __enter__(self):
+        if self.message is None:
+            raise ValueError(
+                'when used as a context manager, the message argument cannot '
+                'be None')
+        if self.sync:
+            runtime.deviceSynchronize()
+        if self.argb_color is not None:
+            cuda.nvtx.RangePushC(self.message, self.argb_color)
+        else:
+            cuda.nvtx.RangePush(self.message, self.color_id)
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.sync:
+            runtime.deviceSynchronize()
+        cuda.nvtx.RangePop()
+
+    def _recreate_cm(self, message):
+        if self.message is None:
+            self.message = message
+        return self
+
+    def __call__(self, func):
+        @functools.wraps(func)
+        def inner(*args, **kwargs):
+            with self._recreate_cm(func.__name__):
+                return func(*args, **kwargs)
+        return inner
diff --git a/cupyx/scipy/__init__.py b/cupyx/scipy/__init__.py
new file mode 100644
index 0000000..307914d
--- /dev/null
+++ b/cupyx/scipy/__init__.py
@@ -0,0 +1,35 @@
+import sys as _sys
+
+from cupy._core import ndarray as _ndarray
+from cupyx.scipy.sparse._base import spmatrix as _spmatrix
+
+
+try:
+    import scipy as _scipy
+    _scipy_available = True
+except ImportError:
+    _scipy_available = False
+
+
+_cupyx_scipy = _sys.modules[__name__]
+
+
+def get_array_module(*args):
+    """Returns the array module for arguments.
+
+    This function is used to implement CPU/GPU generic code. If at least one of
+    the arguments is a :class:`cupy.ndarray` object, the :mod:`cupyx.scipy`
+    module is returned.
+
+    Args:
+        args: Values to determine whether NumPy or CuPy should be used.
+
+    Returns:
+        module: :mod:`cupyx.scipy` or :mod:`scipy` is returned based on the
+        types of the arguments.
+
+    """
+    for arg in args:
+        if isinstance(arg, (_ndarray, _spmatrix)):
+            return _cupyx_scipy
+    return _scipy
diff --git a/cupyx/scipy/_lib/__init__.py b/cupyx/scipy/_lib/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cupyx/scipy/_lib/_util.py b/cupyx/scipy/_lib/_util.py
new file mode 100644
index 0000000..72a532a
--- /dev/null
+++ b/cupyx/scipy/_lib/_util.py
@@ -0,0 +1,73 @@
+import math
+
+import cupy
+
+
+def float_factorial(n):
+    """Compute the factorial and return as a float
+
+    Returns infinity when result is too large for a double
+    """
+    return float(math.factorial(n)) if n < 171 else cupy.inf
+
+
+def _asarray_validated(a, check_finite=True,
+                       sparse_ok=False, objects_ok=False, mask_ok=False,
+                       as_inexact=False):
+    """Helper function for SciPy argument validation.
+
+    Many CuPy linear algebra functions do support arbitrary array-like
+    input arguments. Examples of commonly unsupported inputs include
+    matrices containing inf/nan, sparse matrix representations, and
+    matrices with complicated elements.
+
+    Parameters
+    ----------
+    a : array-like
+        The array-like input
+    check_finite : bool, optional
+        By default True. To check whether the input matrices contain
+        only finite numbers. Disabling may give a performance gain,
+        but may result in problems (crashes, non-termination) if the
+        inputs do contain infinites or NaNs
+    sparse_ok : bool, optional
+        By default False. True if cupy sparse matrices are allowed
+    objects_ok : bool, optional
+        By default False. True if arrays with dype('O') are allowed
+    mask_ok : bool, optional
+        By default False. True if masked arrays are allowed.
+    as_inexact : bool, optional
+        By default False. True to convert the input array to a
+        cupy.inexact dtype
+
+    Returns
+    -------
+    ret : cupy.ndarray
+        The converted validated array
+
+    """
+
+    if not sparse_ok:
+        import cupyx.scipy.sparse
+        if cupyx.scipy.sparse.issparse(a):
+            msg = ('Sparse matrices are not supported by this function. '
+                   'Perhaps one of the cupyx.scipy.sparse.linalg functions '
+                   'would work instead.')
+            raise ValueError(msg)
+
+    # TODO: remove these comments when CuPy supports masked arrays
+    # Ref Issue: https://github.com/cupy/cupy/issues/2225
+    # if not mask_ok:
+    #     if cupy.ma.isMaskedArray(a):
+    #         raise ValueError('masked arrays are not supported')
+
+    # TODO: remove these comments when CuPy supports 'object' dtype
+    # if not objects_ok:
+    #    if a.dtype is cupy.dtype('O'):
+    #        raise ValueError('object arrays are not supported')
+
+    if as_inexact:
+        if not cupy.issubdtype(a, cupy.inexact):
+            a = a.astype(dtype=cupy.float_)
+
+    return a
diff --git a/cupyx/scipy/fft/__init__.py b/cupyx/scipy/fft/__init__.py
new file mode 100644
index 0000000..861d7d3
--- /dev/null
+++ b/cupyx/scipy/fft/__init__.py
@@ -0,0 +1,16 @@
+# flake8: NOQA
+from cupyx.scipy.fft._fft import (
+    fft, ifft, fft2, ifft2, fftn, ifftn,
+    rfft, irfft, rfft2, irfft2, rfftn, irfftn,
+    hfft, ihfft, hfft2, ihfft2, hfftn, ihfftn
+)
+from cupyx.scipy.fft._fft import (
+    __ua_domain__, __ua_convert__, __ua_function__)
+from cupyx.scipy.fft._fft import _scipy_150, _scipy_160
+from cupyx.scipy.fft._fftlog import fht, ifht
+from cupyx.scipy.fft._helper import next_fast_len  # NOQA
+from cupy.fft import fftshift, ifftshift, fftfreq, rfftfreq
+from cupyx.scipy.fftpack import get_fft_plan
+from cupyx.scipy.fft._realtransforms import (
+    dct, dctn, dst, dstn, idct, idctn, idst, idstn
+)
diff --git a/cupyx/scipy/fft/_fft.py b/cupyx/scipy/fft/_fft.py
new file mode 100644
index 0000000..cef6c47
--- /dev/null
+++ b/cupyx/scipy/fft/_fft.py
@@ -0,0 +1,677 @@
+from numbers import Number
+import warnings
+
+import numpy as np
+
+import cupy
+
+from cupy.cuda import cufft
+from cupy.fft._fft import (_fft, _default_fft_func, hfft as _hfft,
+                           ihfft as _ihfft, _size_last_transform_axis,
+                           _swap_direction)
+
+_scipy_150 = False
+_scipy_160 = False
+try:
+    import scipy
+    import scipy.fft as _scipy_fft
+except ImportError:
+    class _DummyModule:
+        def __getattr__(self, name):
+            return None
+
+    _scipy_fft = _DummyModule()
+else:
+    from numpy.lib import NumpyVersion as Version
+    _scipy_150 = Version(scipy.__version__) >= Version('1.5.0')
+    _scipy_160 = Version(scipy.__version__) >= Version('1.6.0')
+    del Version
+    del scipy
+
+# Backend support for scipy.fft
+
+__ua_domain__ = 'numpy.scipy.fft'
+_implemented: dict = {}
+
+
+def __ua_convert__(dispatchables, coerce):
+    if coerce:
+        try:
+            replaced = [
+                cupy.asarray(d.value) if d.coercible and d.type is np.ndarray
+                else d.value for d in dispatchables]
+        except TypeError:
+            return NotImplemented
+    else:
+        replaced = [d.value for d in dispatchables]
+
+    if not all(d.type is not np.ndarray or isinstance(r, cupy.ndarray)
+               for r, d in zip(replaced, dispatchables)):
+        return NotImplemented
+
+    return replaced
+
+
+def __ua_function__(method, args, kwargs):
+    fn = _implemented.get(method, None)
+    if fn is None:
+        return NotImplemented
+    if 'plan' in kwargs and not _scipy_150:
+        warnings.warn('The \'plan\' argument is supported in SciPy v1.5.0+')
+    return fn(*args, **kwargs)
+
+
+def _implements(scipy_func):
+    """Decorator adds function to the dictionary of implemented functions"""
+    def inner(func):
+        _implemented[scipy_func] = func
+        return func
+
+    return inner
+
+
+def _assequence(x):
+    """Convert scalars to a sequence, otherwise pass through ``x`` unchanged"""
+    if isinstance(x, Number):
+        return (x,)
+    return x
+
+
+@_implements(_scipy_fft.fft)
+def fft(x, n=None, axis=-1, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the one-dimensional FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.Plan1d` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axis``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, n, axis)
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fft.fft`
+    """
+    return _fft(x, (n,), (axis,), norm, cufft.CUFFT_FORWARD,
+                overwrite_x=overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.ifft)
+def ifft(x, n=None, axis=-1, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the one-dimensional inverse FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.Plan1d` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axis``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, n, axis)
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fft.ifft`
+    """
+    return _fft(x, (n,), (axis,), norm, cufft.CUFFT_INVERSE,
+                overwrite_x=overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.fft2)
+def fft2(x, s=None, axes=(-2, -1), norm=None, overwrite_x=False, *, plan=None):
+    """Compute the two-dimensional FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        s (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``s`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, s, axes)
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and
+            type will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fft.fft2`
+    """
+    return fftn(x, s, axes, norm, overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.ifft2)
+def ifft2(x, s=None, axes=(-2, -1), norm=None, overwrite_x=False, *,
+          plan=None):
+    """Compute the two-dimensional inverse FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        s (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``s`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, s, axes)
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and
+            type will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fft.ifft2`
+    """
+    return ifftn(x, s, axes, norm, overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.fftn)
+def fftn(x, s=None, axes=None, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the N-dimensional FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        s (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``s`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, s, axes)
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and
+            type will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fft.fftn`
+    """
+    s = _assequence(s)
+    axes = _assequence(axes)
+    func = _default_fft_func(x, s, axes)
+    return func(x, s, axes, norm, cufft.CUFFT_FORWARD, overwrite_x=overwrite_x,
+                plan=plan)
+
+
+@_implements(_scipy_fft.ifftn)
+def ifftn(x, s=None, axes=None, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the N-dimensional inverse FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        s (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``s`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, s, axes)
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and
+            type will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fft.ifftn`
+    """
+    s = _assequence(s)
+    axes = _assequence(axes)
+    func = _default_fft_func(x, s, axes)
+    return func(x, s, axes, norm, cufft.CUFFT_INVERSE, overwrite_x=overwrite_x,
+                plan=plan)
+
+
+@_implements(_scipy_fft.rfft)
+def rfft(x, n=None, axis=-1, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the one-dimensional FFT for real input.
+
+    The returned array contains the positive frequency components of the
+    corresponding :func:`fft`, up to and including the Nyquist frequency.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.Plan1d` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axis``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, n, axis,
+                                                        value_type='R2C')
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array.
+
+    .. seealso:: :func:`scipy.fft.rfft`
+
+    """
+    return _fft(x, (n,), (axis,), norm, cufft.CUFFT_FORWARD, 'R2C',
+                overwrite_x=overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.irfft)
+def irfft(x, n=None, axis=-1, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the one-dimensional inverse FFT for real input.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.Plan1d` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axis``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, n, axis,
+                                                        value_type='C2R')
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array.
+
+    .. seealso:: :func:`scipy.fft.irfft`
+    """
+    return _fft(x, (n,), (axis,), norm, cufft.CUFFT_INVERSE, 'C2R',
+                overwrite_x=overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.rfft2)
+def rfft2(x, s=None, axes=(-2, -1), norm=None, overwrite_x=False, *,
+          plan=None):
+    """Compute the two-dimensional FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape to use from the input. If ``s`` is not
+            given, the lengths of the input along the axes specified by
+            ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, s, axes,
+                                                        value_type='R2C')
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other. The length of the
+            last axis transformed will be ``s[-1]//2+1``.
+
+    .. seealso:: :func:`scipy.fft.rfft2`
+    """
+    return rfftn(x, s, axes, norm, overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.irfft2)
+def irfft2(x, s=None, axes=(-2, -1), norm=None, overwrite_x=False, *,
+           plan=None):
+    """Compute the two-dimensional inverse FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape of the output. If ``s`` is not given,
+            they are determined from the lengths of the input along the axes
+            specified by ``axes``.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, s, axes,
+                                                        value_type='C2R')
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other. If ``s`` is not
+            given, the length of final transformed axis of output will be
+            `2*(m-1)` where `m` is the length of the final transformed axis of
+            the input.
+
+    .. seealso:: :func:`scipy.fft.irfft2`
+    """
+    return irfftn(x, s, axes, norm, overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.rfftn)
+def rfftn(x, s=None, axes=None, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the N-dimensional FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape to use from the input. If ``s`` is not
+            given, the lengths of the input along the axes specified by
+            ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, s, axes,
+                                                        value_type='R2C')
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other. The length of the
+            last axis transformed will be ``s[-1]//2+1``.
+
+    .. seealso:: :func:`scipy.fft.rfftn`
+    """
+    s = _assequence(s)
+    axes = _assequence(axes)
+    func = _default_fft_func(x, s, axes, value_type='R2C')
+    return func(x, s, axes, norm, cufft.CUFFT_FORWARD, 'R2C',
+                overwrite_x=overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.irfftn)
+def irfftn(x, s=None, axes=None, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the N-dimensional inverse FFT for real input.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        s (None or tuple of ints): Shape of the output. If ``s`` is not given,
+            they are determined from the lengths of the input along the axes
+            specified by ``axes``.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, s, axes,
+                                                        value_type='C2R')
+
+            Note that ``plan`` is defaulted to ``None``, meaning CuPy will use
+            an auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``s`` and type
+            will convert to complex if the input is other. If ``s`` is not
+            given, the length of final transformed axis of output will be
+            ``2*(m-1)`` where `m` is the length of the final transformed axis
+            of the input.
+
+    .. seealso:: :func:`scipy.fft.irfftn`
+    """
+    s = _assequence(s)
+    axes = _assequence(axes)
+    if (10020 >= cupy.cuda.runtime.runtimeGetVersion() >= 10010
+            and int(cupy.cuda.device.get_compute_capability()) < 70
+            and _size_last_transform_axis(x.shape, s, axes) == 2):
+        warnings.warn('Output of irfftn might not be correct due to issue '
+                      'of cuFFT in CUDA 10.1/10.2 on Pascal or older GPUs.')
+    func = _default_fft_func(x, s, axes, value_type='C2R')
+    return func(x, s, axes, norm, cufft.CUFFT_INVERSE, 'C2R',
+                overwrite_x=overwrite_x, plan=plan)
+
+
+@_implements(_scipy_fft.hfft)
+def hfft(x, n=None, axis=-1, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the FFT of a signal that has Hermitian symmetry.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        n (None or int): Length of the transformed axis of the output. For
+            ``n`` output points, ``n//2+1`` input points are necessary. If
+            ``n`` is not given, it is determined from the length of the input
+            along the axis specified by ``axis``.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (None): This argument is currently not supported.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if the input is other. If ``n`` is not
+            given, the length of the transformed axis is ``2*(m-1)`` where `m`
+            is the length of the transformed axis of the input.
+
+    .. seealso:: :func:`scipy.fft.hfft`
+    """
+    # TODO(leofang): support R2C & C2R plans
+    if plan is not None:
+        raise NotImplementedError('hfft plan is currently not yet supported')
+    return _hfft(x, n, axis, norm)
+
+
+@_implements(_scipy_fft.ihfft)
+def ihfft(x, n=None, axis=-1, norm=None, overwrite_x=False, *, plan=None):
+    """Compute the FFT of a signal that has Hermitian symmetry.
+
+    Args:
+        a (cupy.ndarray): Array to be transform.
+        n (None or int): Number of points along transformation axis in the
+            input to use. If ``n`` is not given, the length of the input along
+            the axis specified by ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (None): This argument is currently not supported.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if the input is other. The length of the
+            transformed axis is ``n//2+1``.
+
+    .. seealso:: :func:`scipy.fft.ihfft`
+    """
+    # TODO(leofang): support R2C & C2R plans
+    if plan is not None:
+        raise NotImplementedError('ihfft plan is currently not yet supported')
+    return _ihfft(x, n, axis, norm)
+
+
+@_implements(_scipy_fft.hfft2)
+def hfft2(x, s=None, axes=(-2, -1), norm=None, overwrite_x=False, *,
+          plan=None):
+    """Compute the FFT of a two-dimensional signal that has Hermitian symmetry.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        s (None or tuple of ints): Shape of the real output.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+            (This argument is currently not supported)
+        plan (None): This argument is currently not supported.
+
+    Returns:
+        cupy.ndarray:
+            The real result of the 2-D Hermitian complex real FFT.
+
+    .. seealso:: :func:`scipy.fft.hfft2`
+    """
+    if plan is not None:
+        raise NotImplementedError('hfft2 plan is currently not yet supported')
+    return irfft2(x.conj(), s, axes, _swap_direction(norm))
+
+
+@_implements(_scipy_fft.ihfft2)
+def ihfft2(x, s=None, axes=(-2, -1), norm=None, overwrite_x=False, *,
+           plan=None):
+    """Compute the Inverse FFT of a two-dimensional signal that has Hermitian
+    symmetry.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        s (None or tuple of ints): Shape of the real output.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+            (This argument is currently not supported)
+        plan (None): This argument is currently not supported.
+
+    Returns:
+        cupy.ndarray:
+            The real result of the 2-D Hermitian inverse complex real FFT.
+
+    .. seealso:: :func:`scipy.fft.ihfft2`
+    """
+    if plan is not None:
+        raise NotImplementedError('ihfft2 plan is currently not yet supported')
+    return rfft2(x, s, axes, _swap_direction(norm)).conj()
+
+
+@_implements(_scipy_fft.hfftn)
+def hfftn(x, s=None, axes=None, norm=None, overwrite_x=False, *,
+          plan=None):
+    """Compute the FFT of a N-dimensional signal that has Hermitian symmetry.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        s (None or tuple of ints): Shape of the real output.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+            (This argument is currently not supported)
+        plan (None): This argument is currently not supported.
+
+    Returns:
+        cupy.ndarray:
+            The real result of the N-D Hermitian complex real FFT.
+
+    .. seealso:: :func:`scipy.fft.hfftn`
+    """
+    if plan is not None:
+        raise NotImplementedError('hfftn plan is currently not yet supported')
+    return irfftn(x.conj(), s, axes, _swap_direction(norm))
+
+
+@_implements(_scipy_fft.ihfftn)
+def ihfftn(x, s=None, axes=None, norm=None, overwrite_x=False, *,
+           plan=None):
+    """Compute the Inverse FFT of a N-dimensional signal that has Hermitian
+    symmetry.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        s (None or tuple of ints): Shape of the real output.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        norm (``"backward"``, ``"ortho"``, or ``"forward"``): Optional keyword
+            to specify the normalization mode. Default is ``None``, which is
+            an alias of ``"backward"``.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+            (This argument is currently not supported)
+        plan (None): This argument is currently not supported.
+
+    Returns:
+        cupy.ndarray:
+            The real result of the N-D Hermitian inverse complex real FFT.
+
+    .. seealso:: :func:`scipy.fft.ihfftn`
+    """
+    if plan is not None:
+        raise NotImplementedError('ihfftn plan is currently not yet supported')
+    return rfftn(x, s, axes, _swap_direction(norm)).conj()
diff --git a/cupyx/scipy/fft/_fftlog.py b/cupyx/scipy/fft/_fftlog.py
new file mode 100644
index 0000000..27e029a
--- /dev/null
+++ b/cupyx/scipy/fft/_fftlog.py
@@ -0,0 +1,225 @@
+'''Fast Hankel transforms using the FFTLog algorithm.
+The implementation closely follows the Fortran code of Hamilton (2000).
+'''
+
+import math
+from warnings import warn
+
+import cupy
+from cupyx.scipy.fft import _fft
+from cupyx.scipy.special import loggamma, poch
+
+try:
+    # fht only exists in SciPy >= 1.7
+    from scipy.fft import fht as _fht
+    _scipy_fft = _fft._scipy_fft
+    del _fht
+except ImportError:
+    class _DummyModule:
+        def __getattr__(self, name):
+            return None
+
+    _scipy_fft = _DummyModule()
+
+# Note scipy also defines fhtoffset but this only operates on scalars
+__all__ = ['fht', 'ifht']
+
+
+# constants
+LN_2 = math.log(2)
+
+
+@_fft._implements(_scipy_fft.fht)
+def fht(a, dln, mu, offset=0.0, bias=0.0):
+    """Compute the fast Hankel transform.
+
+    Computes the discrete Hankel transform of a logarithmically spaced periodic
+    sequence using the FFTLog algorithm [1]_, [2]_.
+
+    Parameters
+    ----------
+    a : cupy.ndarray (..., n)
+        Real periodic input array, uniformly logarithmically spaced.  For
+        multidimensional input, the transform is performed over the last axis.
+    dln : float
+        Uniform logarithmic spacing of the input array.
+    mu : float
+        Order of the Hankel transform, any positive or negative real number.
+    offset : float, optional
+        Offset of the uniform logarithmic spacing of the output array.
+    bias : float, optional
+        Exponent of power law bias, any positive or negative real number.
+
+    Returns
+    -------
+    A : cupy.ndarray (..., n)
+        The transformed output array, which is real, periodic, uniformly
+        logarithmically spaced, and of the same shape as the input array.
+
+    See Also
+    --------
+    :func:`scipy.special.fht`
+    :func:`scipy.special.fhtoffset` : Return an optimal offset for `fht`.
+
+    References
+    ----------
+    .. [1] Talman J. D., 1978, J. Comp. Phys., 29, 35
+    .. [2] Hamilton A. J. S., 2000, MNRAS, 312, 257 (astro-ph/9905191)
+
+    """
+
+    # size of transform
+    n = a.shape[-1]
+
+    # bias input array
+    if bias != 0:
+        # a_q(r) = a(r) (r/r_c)^{-q}
+        j_c = (n-1)/2
+        j = cupy.arange(n)
+        a = a * cupy.exp(-bias*(j - j_c)*dln)
+
+    # compute FHT coefficients
+    u = fhtcoeff(n, dln, mu, offset=offset, bias=bias)
+
+    # transform
+    A = _fhtq(a, u)
+
+    # bias output array
+    if bias != 0:
+        # A(k) = A_q(k) (k/k_c)^{-q} (k_c r_c)^{-q}
+        A *= cupy.exp(-bias*((j - j_c)*dln + offset))
+
+    return A
+
+
+@_fft._implements(_scipy_fft.ifht)
+def ifht(A, dln, mu, offset=0.0, bias=0.0):
+    """Compute the inverse fast Hankel transform.
+
+    Computes the discrete inverse Hankel transform of a logarithmically spaced
+    periodic sequence. This is the inverse operation to `fht`.
+
+    Parameters
+    ----------
+    A : cupy.ndarray (..., n)
+        Real periodic input array, uniformly logarithmically spaced.  For
+        multidimensional input, the transform is performed over the last axis.
+    dln : float
+        Uniform logarithmic spacing of the input array.
+    mu : float
+        Order of the Hankel transform, any positive or negative real number.
+    offset : float, optional
+        Offset of the uniform logarithmic spacing of the output array.
+    bias : float, optional
+        Exponent of power law bias, any positive or negative real number.
+
+    Returns
+    -------
+    a : cupy.ndarray (..., n)
+        The transformed output array, which is real, periodic, uniformly
+        logarithmically spaced, and of the same shape as the input array.
+
+    See Also
+    --------
+    :func:`scipy.special.ifht`
+    :func:`scipy.special.fhtoffset` : Return an optimal offset for `fht`.
+
+    """
+
+    # size of transform
+    n = A.shape[-1]
+
+    # bias input array
+    if bias != 0:
+        # A_q(k) = A(k) (k/k_c)^{q} (k_c r_c)^{q}
+        j_c = (n - 1) / 2
+        j = cupy.arange(n)
+        A = A * cupy.exp(bias * ((j - j_c) * dln + offset))
+
+    # compute FHT coefficients
+    u = fhtcoeff(n, dln, mu, offset=offset, bias=bias)
+
+    # transform
+    a = _fhtq(A, u, inverse=True)
+
+    # bias output array
+    if bias != 0:
+        # a(r) = a_q(r) (r/r_c)^{q}
+        a /= cupy.exp(-bias * (j - j_c) * dln)
+
+    return a
+
+
+def fhtcoeff(n, dln, mu, offset=0.0, bias=0.0):
+    '''Compute the coefficient array for a fast Hankel transform.
+    '''
+
+    lnkr, q = offset, bias
+
+    # Hankel transform coefficients
+    # u_m = (kr)^{-i 2m pi/(n dlnr)} U_mu(q + i 2m pi/(n dlnr))
+    # with U_mu(x) = 2^x Gamma((mu+1+x)/2)/Gamma((mu+1-x)/2)
+    xp = (mu + 1 + q)/2
+    xm = (mu + 1 - q)/2
+    y = cupy.linspace(0, math.pi * (n // 2) / (n * dln), n // 2 + 1)
+    u = cupy.empty(n // 2 + 1, dtype=complex)
+    v = cupy.empty(n // 2 + 1, dtype=complex)
+    u.imag[:] = y
+    u.real[:] = xm
+    loggamma(u, out=v)
+    u.real[:] = xp
+    loggamma(u, out=u)
+    y *= 2 * (LN_2 - lnkr)
+    u.real -= v.real
+    u.real += LN_2 * q
+    u.imag += v.imag
+    u.imag += y
+    cupy.exp(u, out=u)
+
+    # fix last coefficient to be real
+    u.imag[-1] = 0
+
+    # deal with special cases
+    if not cupy.isfinite(u[0]):
+        # write u_0 = 2^q Gamma(xp)/Gamma(xm) = 2^q poch(xm, xp-xm)
+        # poch() handles special cases for negative integers correctly
+        u[0] = 2**q * poch(xm, xp - xm)
+        # the coefficient may be inf or 0, meaning the transform or the
+        # inverse transform, respectively, is singular
+
+    return u
+
+
+def _fhtq(a, u, inverse=False):
+    '''Compute the biased fast Hankel transform.
+
+    This is the basic FFTLog routine.
+    '''
+
+    # size of transform
+    n = a.shape[-1]
+
+    # check for singular transform or singular inverse transform
+    if cupy.isinf(u[0]) and not inverse:
+        warn('singular transform; consider changing the bias')
+        # fix coefficient to obtain (potentially correct) transform anyway
+        u = u.copy()
+        u[0] = 0
+    elif u[0] == 0 and inverse:
+        warn('singular inverse transform; consider changing the bias')
+        # fix coefficient to obtain (potentially correct) inverse anyway
+        u = u.copy()
+        u[0] = cupy.inf
+
+    # biased fast Hankel transform via real FFT
+    A = _fft.rfft(a, axis=-1)
+    if not inverse:
+        # forward transform
+        A *= u
+    else:
+        # backward transform
+        A /= u.conj()
+    A = _fft.irfft(A, n, axis=-1)
+    A = A[..., ::-1]
+
+    return A
diff --git a/cupyx/scipy/fft/_helper.py b/cupyx/scipy/fft/_helper.py
new file mode 100644
index 0000000..f6d7b62
--- /dev/null
+++ b/cupyx/scipy/fft/_helper.py
@@ -0,0 +1,51 @@
+from typing import Dict, List, Tuple
+
+import math
+
+
+_next_fast_len_cache: Dict[Tuple[int, List[int]], int] = {}
+
+
+def _next_fast_len_impl(n, primes):
+    if len(primes) == 0:
+        return math.inf
+    result = _next_fast_len_cache.get((n, primes), None)
+    if result is None:
+        if n == 1:
+            result = 1
+        else:
+            p = primes[0]
+            result = min(
+                _next_fast_len_impl((n + p - 1) // p, primes) * p,
+                _next_fast_len_impl(n, primes[1:]))
+        _next_fast_len_cache[(n, primes)] = result
+    return result
+
+
+def next_fast_len(target, real=False):
+    """Find the next fast size to ``fft``.
+
+    Args:
+        target (int): The size of input array.
+        real (bool): ``True`` if the FFT involves real input or output.
+            This parameter is of no use, and only for compatibility to
+            SciPy's interface.
+
+    Returns:
+        int: The smallest fast length greater than or equal to the input value.
+
+    .. seealso:: :func:`scipy.fft.next_fast_len`
+
+    .. note::
+        It may return a different value to :func:`scipy.fft.next_fast_len`
+        as pocketfft's prime factors are different from cuFFT's factors.
+        For details, see the `cuFFT documentation`_.
+
+    .. _cuFFT documentation:
+        https://docs.nvidia.com/cuda/cufft/index.html#accuracy-and-performance
+    """
+    if target == 0:
+        return 0
+
+    primes = (2, 3, 5, 7)
+    return _next_fast_len_impl(target, primes)
diff --git a/cupyx/scipy/fft/_realtransforms.py b/cupyx/scipy/fft/_realtransforms.py
new file mode 100644
index 0000000..31b74de
--- /dev/null
+++ b/cupyx/scipy/fft/_realtransforms.py
@@ -0,0 +1,924 @@
+"""Real-to-real transforms
+
+cuFFT does not implement real-to-real FFTs. This module implements forward
+and inverse DCT-II and DCT-III transforms using FFTs.
+
+A length N DCT can be computed using a length N FFT and some additional
+multiplications and reordering of entries.
+
+The approach taken here is based on the work in [1]_, [2]_ and is discussed in
+the freely-available online resources [3]_, [4]_.
+
+The implementation here follows that approach with only minor modification to
+match the normalization conventions in SciPy.
+
+The modifications to turn a type II or III DCT to a DST were implemented as
+described in [5]_.
+
+.. [1] J. Makhoul, "A fast cosine transform in one and two dimensions," in
+    IEEE Transactions on Acoustics, Speech, and Signal Processing, vol. 28,
+    no. 1, pp. 27-34, February 1980.
+
+.. [2] M.J. Narasimha and A.M. Peterson, “On the computation of the discrete
+    cosine  transform,” IEEE Trans. Commun., vol. 26, no. 6, pp. 934–936, 1978.
+
+.. [3] http://fourier.eng.hmc.edu/e161/lectures/dct/node2.html
+
+.. [4] https://dsp.stackexchange.com/questions/2807/fast-cosine-transform-via-fft  # noqa
+
+.. [5] X. Shao, S. G. Johnson. Type-II/III DCT/DST algorithms with reduced
+    number of arithmetic operations, Signal Processing, Volume 88, Issue 6,
+    pp. 1553-1564, 2008.
+"""
+
+import math
+import numbers
+import operator
+
+import numpy
+
+import cupy
+from cupy import _core
+from cupy.fft._fft import _cook_shape
+from cupyx.scipy.fft import _fft
+
+
+__all__ = ['dct', 'dctn', 'dst', 'dstn', 'idct', 'idctn', 'idst', 'idstn']
+
+
+def _promote_dtype(x):
+    if x.dtype.kind in 'bui':
+        # use float64 instead of promote_types to match SciPy's behavior
+        float_dtype = cupy.float64
+    else:
+        float_dtype = cupy.promote_types(x.dtype, cupy.float32)
+    return x.astype(float_dtype, copy=False)
+
+
+def _get_dct_norm_factor(n, inorm, dct_type=2):
+    """Normalization factors for DCT/DST I-IV.
+
+    Parameters
+    ----------
+    n : int
+        Data size.
+    inorm : {'none', 'sqrt', 'full'}
+        When `inorm` is 'none', the scaling factor is 1.0 (unnormalized). When
+        `inorm` is 1, scaling by ``1/sqrt(d)`` as needed for an orthogonal
+        transform is used. When `inorm` is 2, normalization by ``1/d`` is
+        applied. The value of ``d`` depends on both `n` and the `dct_type`.
+    dct_type : {1, 2, 3, 4}
+        Which type of DCT or DST is being normalized?.
+
+    Returns
+    -------
+    fct : float
+        The normalization factor.
+    """
+    if inorm == 'none':
+        return 1
+    delta = -1 if dct_type == 1 else 0
+    d = 2 * (n + delta)
+    if inorm == 'full':
+        fct = 1 / d
+    elif inorm == 'sqrt':
+        fct = 1 / math.sqrt(d)
+    else:
+        raise ValueError('expected inorm = "none", "sqrt" or "full"')
+    return fct
+
+
+def _reshuffle_dct2(x, n, axis, dst=False):
+    """Reorder entries to allow computation of DCT/DST-II via FFT."""
+    sl_even = [slice(None)] * x.ndim
+    sl_even[axis] = slice(0, None, 2)
+    sl_even = tuple(sl_even)
+    sl_odd = [slice(None)] * x.ndim
+    if n % 2:
+        sl_odd[axis] = slice(-2, None, -2)
+        sl_odd = tuple(sl_odd)
+    else:
+        sl_odd[axis] = slice(None, None, -2)
+        sl_odd = tuple(sl_odd)
+    if dst:
+        x = cupy.concatenate((x[sl_even], -x[sl_odd]), axis=axis)
+    else:
+        x = cupy.concatenate((x[sl_even], x[sl_odd]), axis=axis)
+    return x
+
+
+_mult_factor_dct2 = _core.ElementwiseKernel(
+    in_params='R xr, int32 N, R norm_factor',
+    out_params='C y',
+    operation="""
+    C j(0., -1.);
+    y = (R)2.0 * norm_factor * exp(j * (R)(i * M_PI / (2 * N)));""",
+)
+
+
+def _exp_factor_dct2(x, n, axis, norm_factor, n_truncate=None):
+    """Twiddle & scaling factors for computation of DCT/DST-II via FFT."""
+    if n_truncate is None:
+        n_truncate = n
+    tmp = cupy.empty((n_truncate,), dtype=x.dtype)
+    _mult_factor_dct2(tmp.real, n, norm_factor, tmp)
+
+    if x.ndim == 1:
+        return tmp
+    tmp_shape = [1] * x.ndim
+    tmp_shape[axis] = n_truncate
+    tmp_shape = tuple(tmp_shape)
+    return tmp.reshape(tmp_shape)
+
+
+def _dct_or_dst_type2(
+    x, n=None, axis=-1, forward=True, norm=None, dst=False, overwrite_x=False
+):
+    """Forward DCT/DST-II (or inverse DCT/DST-III) along a single axis
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The data to transform.
+    n : int
+        The size of the transform. If None, ``x.shape[axis]`` is used.
+    axis : int
+        Axis along which the transform is applied.
+    forward : bool
+        Set true to indicate that this is a forward DCT-II as opposed to an
+        inverse DCT-III (The difference between the two is only in the
+        normalization factor).
+    norm : {None, 'ortho', 'forward', 'backward'}
+        The normalization convention to use.
+    dst : bool
+        If True, a discrete sine transform is computed rather than the discrete
+        cosine transform.
+    overwrite_x : bool
+        Indicates that it is okay to overwrite x. In practice, the current
+        implementation never performs the transform in-place.
+
+    Returns
+    -------
+    y: cupy.ndarray
+        The transformed array.
+    """
+    if axis < -x.ndim or axis >= x.ndim:
+        raise numpy.AxisError('axis out of range')
+    if axis < 0:
+        axis += x.ndim
+    if n is not None and n < 1:
+        raise ValueError(
+            f'invalid number of data points ({n}) specified'
+        )
+
+    x = _cook_shape(x, (n,), (axis,), 'R2R')
+    n = x.shape[axis]
+
+    x = _reshuffle_dct2(x, x.shape[axis], axis, dst)
+
+    if norm == 'ortho':
+        inorm = 'sqrt'
+    elif norm == 'forward':
+        inorm = 'full' if forward else 'none'
+    else:
+        inorm = 'none' if forward else 'full'
+    norm_factor = _get_dct_norm_factor(n, inorm=inorm, dct_type=2)
+
+    x = _fft.fft(x, n=n, axis=axis, overwrite_x=True)
+    tmp = _exp_factor_dct2(x, n, axis, norm_factor)
+
+    x *= tmp  # broadcasting
+    x = cupy.real(x)
+
+    if dst:
+        slrev = [slice(None)] * x.ndim
+        slrev[axis] = slice(None, None, -1)
+        x = x[tuple(slrev)]
+
+    if norm == 'ortho':
+        sl0 = [slice(None)] * x.ndim
+        sl0[axis] = slice(1)
+        x[tuple(sl0)] *= math.sqrt(2) * 0.5
+    return x
+
+
+def _reshuffle_dct3(y, n, axis, dst):
+    """Reorder entries to allow computation of DCT/DST-II via FFT."""
+    x = cupy.empty_like(y)
+    n_half = (n + 1) // 2
+
+    # Store first half of y in the even entries of the output
+    sl_even = [slice(None)] * y.ndim
+    sl_even[axis] = slice(0, None, 2)
+    sl_even = tuple(sl_even)
+
+    sl_half = [slice(None)] * y.ndim
+    sl_half[axis] = slice(0, n_half)
+    x[sl_even] = y[tuple(sl_half)]
+
+    # Store the second half of y in the odd entries of the output
+    sl_odd = [slice(None)] * y.ndim
+    sl_odd[axis] = slice(1, None, 2)
+    sl_odd = tuple(sl_odd)
+
+    sl_half[axis] = slice(-1, n_half - 1, -1)
+    if dst:
+        x[sl_odd] = -y[tuple(sl_half)]
+    else:
+        x[sl_odd] = y[tuple(sl_half)]
+    return x
+
+
+_mult_factor_dct3 = _core.ElementwiseKernel(
+    in_params='R xr, int32 N, R norm_factor',
+    out_params='C y',
+    operation="""
+    C j(0., 1.);
+    y = (R)(2 * N * norm_factor) * exp(j * (R)(i * M_PI / (2 * N)));""",
+)
+
+
+def _exp_factor_dct3(x, n, axis, dtype, norm_factor):
+    """Twiddle & scaling factors for computation of DCT/DST-III via FFT."""
+    tmp = cupy.empty((n,), dtype=dtype)
+    _mult_factor_dct3(tmp.real, n, norm_factor, tmp)
+    if x.ndim == 1:
+        return tmp
+    # prepare shape for broadcasting along non-transformed axes
+    tmp_shape = [1] * x.ndim
+    tmp_shape[axis] = n
+    tmp_shape = tuple(tmp_shape)
+    return tmp.reshape(tmp_shape)
+
+
+def _dct_or_dst_type3(
+    x, n=None, axis=-1, norm=None, forward=True, dst=False, overwrite_x=False
+):
+    """Forward DCT/DST-III (or inverse DCT/DST-II) along a single axis.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The data to transform.
+    n : int
+        The size of the transform. If None, ``x.shape[axis]`` is used.
+    axis : int
+        Axis along which the transform is applied.
+    forward : bool
+        Set true to indicate that this is a forward DCT-II as opposed to an
+        inverse DCT-III (The difference between the two is only in the
+        normalization factor).
+    norm : {None, 'ortho', 'forward', 'backward'}
+        The normalization convention to use.
+    dst : bool
+        If True, a discrete sine transform is computed rather than the discrete
+        cosine transform.
+    overwrite_x : bool
+        Indicates that it is okay to overwrite x. In practice, the current
+        implementation never performs the transform in-place.
+
+    Returns
+    -------
+    y: cupy.ndarray
+        The transformed array.
+
+    """
+    if axis < -x.ndim or axis >= x.ndim:
+        raise numpy.AxisError('axis out of range')
+    if axis < 0:
+        axis += x.ndim
+    if n is not None and n < 1:
+        raise ValueError(
+            f'invalid number of data points ({n}) specified'
+        )
+
+    x = _cook_shape(x, (n,), (axis,), 'R2R')
+    n = x.shape[axis]
+
+    # determine normalization factor
+    if norm == 'ortho':
+        sl0_scale = 0.5 * math.sqrt(2)
+        inorm = 'sqrt'
+    elif norm == 'forward':
+        sl0_scale = 0.5
+        inorm = 'full' if forward else 'none'
+    elif norm == 'backward' or norm is None:
+        sl0_scale = 0.5
+        inorm = 'none' if forward else 'full'
+    else:
+        raise ValueError(f'Invalid norm value "{norm}", should be "backward", '
+                         '"ortho" or "forward"')
+    norm_factor = _get_dct_norm_factor(n, inorm=inorm, dct_type=3)
+    dtype = cupy.promote_types(x, cupy.complex64)
+
+    sl0 = [slice(None)] * x.ndim
+    sl0[axis] = slice(1)
+
+    if dst:
+        if norm == 'ortho':
+            float_dtype = cupy.promote_types(x.dtype, cupy.float32)
+            if x.dtype != float_dtype:
+                x = x.astype(float_dtype)
+            elif not overwrite_x:
+                x = x.copy()
+            x[tuple(sl0)] *= math.sqrt(2)
+            sl0_scale = 0.5
+        slrev = [slice(None)] * x.ndim
+        slrev[axis] = slice(None, None, -1)
+        x = x[tuple(slrev)]
+
+    # scale by exponentials and normalization factor
+    tmp = _exp_factor_dct3(x, n, axis, dtype, norm_factor)
+    x = x * tmp  # broadcasting
+    x[tuple(sl0)] *= sl0_scale
+
+    # inverse fft
+    x = _fft.ifft(x, n=n, axis=axis, overwrite_x=True)
+    x = cupy.real(x)
+
+    # reorder entries
+    return _reshuffle_dct3(x, n, axis, dst)
+
+
+@_fft._implements(_fft._scipy_fft.dct)
+def dct(x, type=2, n=None, axis=-1, norm=None, overwrite_x=False):
+    """Return the Discrete Cosine Transform of an array, x.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The input array.
+    type : {1, 2, 3, 4}, optional
+        Type of the DCT (see Notes). Default type is 2. Currently CuPy only
+        supports types 2 and 3.
+    n : int, optional:
+        Length of the transform.  If ``n < x.shape[axis]``, `x` is
+        truncated. If ``n > x.shape[axis]``, `x` is zero-padded.
+        The default results in ``n = x.shape[axis]``.
+    axis : int, optional
+        Axis along which the dct is computed; the default is over the
+        last axis (i.e., ``axis=-1``).
+    norm : {"backward", "ortho", "forward"}, optional
+        Normalization mode (see Notes). Default is "backward".
+    overwrite_x : bool, optional
+        If True, the contents of `x` can be destroyed; the default is False.
+
+    Returns
+    -------
+    y : cupy.ndarray of real
+        The transformed input array.
+
+    See Also
+    --------
+    :func:`scipy.fft.dct`
+
+    Notes
+    -----
+    For a single dimension array ``x``, ``dct(x, norm='ortho')`` is equal
+    to MATLAB ``dct(x)``.
+
+    For ``norm="ortho"`` both the `dct` and `idct` are scaled by the same
+    overall factor in both directions. By default, the transform is also
+    orthogonalized which for types 1, 2 and 3 means the transform definition is
+    modified to give orthogonality of the DCT matrix (see below).
+
+    For ``norm="backward"``, there is no scaling on `dct` and the `idct` is
+    scaled by ``1/N`` where ``N`` is the "logical" size of the DCT. For
+    ``norm="forward"`` the ``1/N`` normalization is applied to the forward
+    `dct` instead and the `idct` is unnormalized.
+
+    CuPy currently only supports DCT types 2 and 3. 'The' DCT generally
+    refers to DCT type 2, and 'the' Inverse DCT generally refers to DCT
+    type 3 [1]_. See the :func:`scipy.fft.dct` documentation for a full
+    description of each type.
+
+    References
+    ----------
+    .. [1] Wikipedia, "Discrete cosine transform",
+           https://en.wikipedia.org/wiki/Discrete_cosine_transform
+
+    """
+    if x.dtype.kind == 'c':
+        # separable application on real and imaginary parts
+        out = dct(x.real, type, n, axis, norm, overwrite_x)
+        out = out + 1j * dct(x.imag, type, n, axis, norm, overwrite_x)
+        return out
+
+    x = _promote_dtype(x)
+
+    if type == 2:
+        return _dct_or_dst_type2(
+            x, n=n, axis=axis, norm=norm, forward=True, dst=False
+        )
+    elif type == 3:
+        return _dct_or_dst_type3(
+            x, n=n, axis=axis, norm=norm, forward=True, dst=False
+        )
+    elif type in [1, 4]:
+        raise NotImplementedError(
+            'Only DCT-II and DCT-III have been implemented.'
+        )
+    else:
+        raise ValueError('invalid DCT type')
+
+
+@_fft._implements(_fft._scipy_fft.dst)
+def dst(x, type=2, n=None, axis=-1, norm=None, overwrite_x=False):
+    """Return the Discrete Sine Transform of an array, x.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The input array.
+    type : {1, 2, 3, 4}, optional
+        Type of the DST (see Notes). Default type is 2.
+    n : int, optional
+        Length of the transform.  If ``n < x.shape[axis]``, `x` is
+        truncated.  If ``n > x.shape[axis]``, `x` is zero-padded. The
+        default results in ``n = x.shape[axis]``.
+    axis : int, optional
+        Axis along which the dst is computed; the default is over the
+        last axis (i.e., ``axis=-1``).
+    norm : {"backward", "ortho", "forward"}, optional
+        Normalization mode (see Notes). Default is "backward".
+    overwrite_x : bool, optional
+        If True, the contents of `x` can be destroyed; the default is False.
+
+    Returns
+    -------
+    dst : cupy.ndarray of real
+        The transformed input array.
+
+    See Also
+    --------
+    :func:`scipy.fft.dst`
+
+    Notes
+    -----
+
+    For ``norm="ortho"`` both the `dst` and `idst` are scaled by the same
+    overall factor in both directions. By default, the transform is also
+    orthogonalized which for types 2 and 3 means the transform definition is
+    modified to give orthogonality of the DST matrix (see below).
+
+    For ``norm="backward"``, there is no scaling on the `dst` and the `idst` is
+    scaled by ``1/N`` where ``N`` is the "logical" size of the DST.
+
+    See the :func:`scipy.fft.dst` documentation for a full description of each
+    type. CuPy currently only supports DST types 2 and 3.
+    """
+    if x.dtype.kind == 'c':
+        # separable application on real and imaginary parts
+        out = dst(x.real, type, n, axis, norm, overwrite_x)
+        out = out + 1j * dst(x.imag, type, n, axis, norm, overwrite_x)
+        return out
+
+    x = _promote_dtype(x)
+
+    if type == 2:
+        return _dct_or_dst_type2(
+            x, n=n, axis=axis, norm=norm, forward=True, dst=True
+        )
+    elif type == 3:
+        return _dct_or_dst_type3(
+            x, n=n, axis=axis, norm=norm, forward=True, dst=True
+        )
+    elif type in [1, 4]:
+        raise NotImplementedError(
+            'Only DST-II and DST-III have been implemented.'
+        )
+    else:
+        raise ValueError('invalid DST type')
+
+
+@_fft._implements(_fft._scipy_fft.idct)
+def idct(x, type=2, n=None, axis=-1, norm=None, overwrite_x=False):
+    """Return the Inverse Discrete Cosine Transform of an array, x.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The input array.
+    type : {1, 2, 3, 4}, optional
+        Type of the DCT (see Notes). Default type is 2.
+    n : int, optional
+        Length of the transform.  If ``n < x.shape[axis]``, `x` is
+        truncated.  If ``n > x.shape[axis]``, `x` is zero-padded. The
+        default results in ``n = x.shape[axis]``.
+    axis : int, optional
+        Axis along which the idct is computed; the default is over the
+        last axis (i.e., ``axis=-1``).
+    norm : {"backward", "ortho", "forward"}, optional
+        Normalization mode (see Notes). Default is "backward".
+    overwrite_x : bool, optional
+        If True, the contents of `x` can be destroyed; the default is False.
+
+    Returns
+    -------
+    idct : cupy.ndarray of real
+        The transformed input array.
+
+    See Also
+    --------
+    :func:`scipy.fft.idct`
+
+    Notes
+    -----
+    For a single dimension array `x`, ``idct(x, norm='ortho')`` is equal to
+    MATLAB ``idct(x)``.
+
+    For ``norm="ortho"`` both the `dct` and `idct` are scaled by the same
+    overall factor in both directions. By default, the transform is also
+    orthogonalized which for types 1, 2 and 3 means the transform definition is
+    modified to give orthogonality of the IDCT matrix (see `dct` for the full
+    definitions).
+
+    'The' IDCT is the IDCT-II, which is the same as the normalized DCT-III
+    [1]_. See the :func:`scipy.fft.dct` documentation for a full description of
+    each type. CuPy currently only supports DCT types 2 and 3.
+
+    References
+    ----------
+    .. [1] Wikipedia, "Discrete sine transform",
+           https://en.wikipedia.org/wiki/Discrete_sine_transform
+    """
+    if x.dtype.kind == 'c':
+        # separable application on real and imaginary parts
+        out = idct(x.real, type, n, axis, norm, overwrite_x)
+        out = out + 1j * idct(x.imag, type, n, axis, norm, overwrite_x)
+        return out
+
+    x = _promote_dtype(x)
+
+    if type == 2:
+        # DCT-III is the inverse of DCT-II
+        return _dct_or_dst_type3(x, n=n, axis=axis, norm=norm, forward=False)
+    elif type == 3:
+        # DCT-II is the inverse of DCT-III
+        return _dct_or_dst_type2(x, n=n, axis=axis, norm=norm, forward=False)
+    elif type in [1, 4]:
+        raise NotImplementedError(
+            'Only DCT-II and DCT-III have been implemented.'
+        )
+    else:
+        raise ValueError('invalid DCT type')
+
+
+@_fft._implements(_fft._scipy_fft.idst)
+def idst(x, type=2, n=None, axis=-1, norm=None, overwrite_x=False):
+    """Return the Inverse Discrete Sine Transform of an array, x.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The input array.
+    type : {1, 2, 3, 4}, optional
+        Type of the DST (see Notes). Default type is 2.
+    n : int, optional
+        Length of the transform. If ``n < x.shape[axis]``, `x` is
+        truncated.  If ``n > x.shape[axis]``, `x` is zero-padded. The
+        default results in ``n = x.shape[axis]``.
+    axis : int, optional
+        Axis along which the idst is computed; the default is over the
+        last axis (i.e., ``axis=-1``).
+    norm : {"backward", "ortho", "forward"}, optional
+        Normalization mode (see Notes). Default is "backward".
+    overwrite_x : bool, optional
+        If True, the contents of `x` can be destroyed; the default is False.
+
+    Returns
+    -------
+    idst : cupy.ndarray of real
+        The transformed input array.
+
+    See Also
+    --------
+    :func:`scipy.fft.idst`
+
+    Notes
+    -----
+    For full details of the DST types and normalization modes, as well as
+    references, see :func:`scipy.fft.dst`.
+    """
+    if x.dtype.kind == 'c':
+        # separable application on real and imaginary parts
+        out = idst(x.real, type, n, axis, norm, overwrite_x)
+        out = out + 1j * idst(x.imag, type, n, axis, norm, overwrite_x)
+        return out
+
+    x = _promote_dtype(x)
+
+    if type == 2:
+        # DCT-III is the inverse of DCT-II
+        return _dct_or_dst_type3(
+            x, n=n, axis=axis, norm=norm, forward=False, dst=True
+        )
+    elif type == 3:
+        # DCT-II is the inverse of DCT-III
+        return _dct_or_dst_type2(
+            x, n=n, axis=axis, norm=norm, forward=False, dst=True
+        )
+    elif type in [1, 4]:
+        raise NotImplementedError(
+            'Only DST-II and DST-III have been implemented.'
+        )
+    else:
+        raise ValueError('invalid DST type')
+
+
+def _iterable_of_int(x, name=None):
+    """Convert ``x`` to an iterable sequence of int."""
+    if isinstance(x, numbers.Number):
+        x = (x,)
+
+    try:
+        x = [operator.index(a) for a in x]
+    except TypeError as e:
+        name = name or 'value'
+        raise ValueError(
+            f'{name} must be a scalar or iterable of integers'
+        ) from e
+
+    return x
+
+
+def _init_nd_shape_and_axes(x, shape, axes):
+    """Handles shape and axes arguments for nd transforms."""
+    noshape = shape is None
+    noaxes = axes is None
+
+    if not noaxes:
+        axes = _iterable_of_int(axes, 'axes')
+        axes = [a + x.ndim if a < 0 else a for a in axes]
+
+        if any(a >= x.ndim or a < 0 for a in axes):
+            raise ValueError('axes exceeds dimensionality of input')
+        if len(set(axes)) != len(axes):
+            raise ValueError('all axes must be unique')
+
+    if not noshape:
+        shape = _iterable_of_int(shape, 'shape')
+        nshape = len(shape)
+        if axes and len(axes) != nshape:
+            raise ValueError(
+                'when given, axes and shape arguments'
+                ' have to be of the same length'
+            )
+        if noaxes:
+            if nshape > x.ndim:
+                raise ValueError('shape requires more axes than are present')
+            axes = range(x.ndim - len(shape), x.ndim)
+
+        shape = [x.shape[a] if s == -1 else s for s, a in zip(shape, axes)]
+    elif noaxes:
+        shape = list(x.shape)
+        axes = range(x.ndim)
+    else:
+        shape = [x.shape[a] for a in axes]
+
+    if any(s < 1 for s in shape):
+        raise ValueError(
+            f'invalid number of data points ({shape}) specified'
+        )
+
+    return shape, axes
+
+
+@_fft._implements(_fft._scipy_fft.dctn)
+def dctn(x, type=2, s=None, axes=None, norm=None, overwrite_x=False):
+    """Compute a multidimensional Discrete Cosine Transform.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The input array.
+    type : {1, 2, 3, 4}, optional
+        Type of the DCT (see Notes). Default type is 2.
+    s : int or array_like of ints or None, optional
+        The shape of the result. If both `s` and `axes` (see below) are None,
+        `s` is ``x.shape``; if `s` is None but `axes` is not None, then `s` is
+        ``numpy.take(x.shape, axes, axis=0)``.
+        If ``s[i] > x.shape[i]``, the ith dimension is padded with zeros.
+        If ``s[i] < x.shape[i]``, the ith dimension is truncated to length
+        ``s[i]``.
+        If any element of `s` is -1, the size of the corresponding dimension of
+        `x` is used.
+    axes : int or array_like of ints or None, optional
+        Axes over which the DCT is computed. If not given, the last ``len(s)``
+        axes are used, or all axes if `s` is also not specified.
+    norm : {"backward", "ortho", "forward"}, optional
+        Normalization mode (see Notes). Default is "backward".
+    overwrite_x : bool, optional
+        If True, the contents of `x` can be destroyed; the default is False.
+
+    Returns
+    -------
+    y : cupy.ndarray of real
+        The transformed input array.
+
+    See Also
+    --------
+    :func:`scipy.fft.dctn`
+
+    Notes
+    -----
+    For full details of the DCT types and normalization modes, as well as
+    references, see `dct`.
+    """
+    if x.dtype.kind == 'c':
+        # separable application on real and imaginary parts
+        out = dctn(x.real, type, s, axes, norm, overwrite_x)
+        out = out + 1j * dctn(x.imag, type, s, axes, norm, overwrite_x)
+        return out
+
+    shape, axes = _init_nd_shape_and_axes(x, s, axes)
+    x = _promote_dtype(x)
+
+    if len(axes) == 0:
+        return x
+
+    for n, axis in zip(shape, axes):
+        x = dct(
+            x, type=type, n=n, axis=axis, norm=norm, overwrite_x=overwrite_x
+        )
+    return x
+
+
+@_fft._implements(_fft._scipy_fft.idctn)
+def idctn(x, type=2, s=None, axes=None, norm=None, overwrite_x=False):
+    """Compute a multidimensional Discrete Cosine Transform.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The input array.
+    type : {1, 2, 3, 4}, optional
+        Type of the DCT (see Notes). Default type is 2.
+    s : int or array_like of ints or None, optional
+        The shape of the result. If both `s` and `axes` (see below) are None,
+        `s` is ``x.shape``; if `s` is None but `axes` is not None, then `s` is
+        ``numpy.take(x.shape, axes, axis=0)``.
+        If ``s[i] > x.shape[i]``, the ith dimension is padded with zeros.
+        If ``s[i] < x.shape[i]``, the ith dimension is truncated to length
+        ``s[i]``.
+        If any element of `s` is -1, the size of the corresponding dimension of
+        `x` is used.
+    axes : int or array_like of ints or None, optional
+        Axes over which the IDCT is computed. If not given, the last ``len(s)``
+        axes are used, or all axes if `s` is also not specified.
+    norm : {"backward", "ortho", "forward"}, optional
+        Normalization mode (see Notes). Default is "backward".
+    overwrite_x : bool, optional
+        If True, the contents of `x` can be destroyed; the default is False.
+
+    Returns
+    -------
+    y : cupy.ndarray of real
+        The transformed input array.
+
+    See Also
+    --------
+    :func:`scipy.fft.idctn`
+
+    Notes
+    -----
+    For full details of the IDCT types and normalization modes, as well as
+    references, see :func:`scipy.fft.idct`.
+    """
+    if x.dtype.kind == 'c':
+        # separable application on real and imaginary parts
+        out = idctn(x.real, type, s, axes, norm, overwrite_x)
+        out = out + 1j * idctn(x.imag, type, s, axes, norm, overwrite_x)
+        return out
+
+    shape, axes = _init_nd_shape_and_axes(x, s, axes)
+    x = _promote_dtype(x)
+
+    if len(axes) == 0:
+        return x
+
+    for n, axis in zip(shape, axes):
+        x = idct(
+            x, type=type, n=n, axis=axis, norm=norm, overwrite_x=overwrite_x
+        )
+    return x
+
+
+@_fft._implements(_fft._scipy_fft.dstn)
+def dstn(x, type=2, s=None, axes=None, norm=None, overwrite_x=False):
+    """Compute a multidimensional Discrete Sine Transform.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The input array.
+    type : {1, 2, 3, 4}, optional
+        Type of the DST (see Notes). Default type is 2.
+    s : int or array_like of ints or None, optional
+        The shape of the result. If both `s` and `axes` (see below) are None,
+        `s` is ``x.shape``; if `s` is None but `axes` is not None, then `s` is
+        ``numpy.take(x.shape, axes, axis=0)``.
+        If ``s[i] > x.shape[i]``, the ith dimension is padded with zeros.
+        If ``s[i] < x.shape[i]``, the ith dimension is truncated to length
+        ``s[i]``.
+        If any element of `s` is -1, the size of the corresponding dimension of
+        `x` is used.
+    axes : int or array_like of ints or None, optional
+        Axes over which the DST is computed. If not given, the last ``len(s)``
+        axes are used, or all axes if `s` is also not specified.
+    norm : {"backward", "ortho", "forward"}, optional
+        Normalization mode (see Notes). Default is "backward".
+    overwrite_x : bool, optional
+        If True, the contents of `x` can be destroyed; the default is False.
+
+    Returns
+    -------
+    y : cupy.ndarray of real
+        The transformed input array.
+
+    See Also
+    --------
+    :func:`scipy.fft.dstn`
+
+    Notes
+    -----
+    For full details of the DST types and normalization modes, as well as
+    references, see :func:`scipy.fft.dst`.
+    """
+    if x.dtype.kind == 'c':
+        # separable application on real and imaginary parts
+        out = dstn(x.real, type, s, axes, norm, overwrite_x)
+        out = out + 1j * dstn(x.imag, type, s, axes, norm, overwrite_x)
+        return out
+
+    shape, axes = _init_nd_shape_and_axes(x, s, axes)
+    x = _promote_dtype(x)
+
+    if len(axes) == 0:
+        return x
+
+    for n, axis in zip(shape, axes):
+        x = dst(
+            x, type=type, n=n, axis=axis, norm=norm, overwrite_x=overwrite_x
+        )
+    return x
+
+
+@_fft._implements(_fft._scipy_fft.idstn)
+def idstn(x, type=2, s=None, axes=None, norm=None, overwrite_x=False):
+    """Compute a multidimensional Discrete Sine Transform.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        The input array.
+    type : {1, 2, 3, 4}, optional
+        Type of the DST (see Notes). Default type is 2.
+    s : int or array_like of ints or None, optional
+        The shape of the result. If both `s` and `axes` (see below) are None,
+        `s` is ``x.shape``; if `s` is None but `axes` is not None, then `s` is
+        ``numpy.take(x.shape, axes, axis=0)``.
+        If ``s[i] > x.shape[i]``, the ith dimension is padded with zeros.
+        If ``s[i] < x.shape[i]``, the ith dimension is truncated to length
+        ``s[i]``.
+        If any element of `s` is -1, the size of the corresponding dimension of
+        `x` is used.
+    axes : int or array_like of ints or None, optional
+        Axes over which the IDST is computed. If not given, the last ``len(s)``
+        axes are used, or all axes if `s` is also not specified.
+    norm : {"backward", "ortho", "forward"}, optional
+        Normalization mode (see Notes). Default is "backward".
+    overwrite_x : bool, optional
+        If True, the contents of `x` can be destroyed; the default is False.
+
+    Returns
+    -------
+    y : cupy.ndarray of real
+        The transformed input array.
+
+    See Also
+    --------
+    :func:`scipy.fft.idstn`
+
+    Notes
+    -----
+    For full details of the IDST types and normalization modes, as well as
+    references, see :func:`scipy.fft.idst`.
+    """
+    if x.dtype.kind == 'c':
+        # separable application on real and imaginary parts
+        out = idstn(x.real, type, s, axes, norm, overwrite_x)
+        out = out + 1j * idstn(x.imag, type, s, axes, norm, overwrite_x)
+        return out
+
+    shape, axes = _init_nd_shape_and_axes(x, s, axes)
+    x = _promote_dtype(x)
+
+    if len(axes) == 0:
+        return x
+
+    for n, axis in zip(shape, axes):
+        x = idst(
+            x, type=type, n=n, axis=axis, norm=norm, overwrite_x=overwrite_x
+        )
+    return x
diff --git a/cupyx/scipy/fftpack/__init__.py b/cupyx/scipy/fftpack/__init__.py
new file mode 100644
index 0000000..c328ae2
--- /dev/null
+++ b/cupyx/scipy/fftpack/__init__.py
@@ -0,0 +1,9 @@
+from cupyx.scipy.fftpack._fft import fft  # NOQA
+from cupyx.scipy.fftpack._fft import fft2  # NOQA
+from cupyx.scipy.fftpack._fft import fftn  # NOQA
+from cupyx.scipy.fftpack._fft import ifft  # NOQA
+from cupyx.scipy.fftpack._fft import ifft2  # NOQA
+from cupyx.scipy.fftpack._fft import ifftn  # NOQA
+from cupyx.scipy.fftpack._fft import irfft  # NOQA
+from cupyx.scipy.fftpack._fft import rfft  # NOQA
+from cupyx.scipy.fftpack._fft import get_fft_plan  # NOQA
diff --git a/cupyx/scipy/fftpack/_fft.py b/cupyx/scipy/fftpack/_fft.py
new file mode 100644
index 0000000..bd656a4
--- /dev/null
+++ b/cupyx/scipy/fftpack/_fft.py
@@ -0,0 +1,493 @@
+from numpy import prod
+
+import cupy
+from cupy.cuda import cufft
+from cupy.fft import config
+from cupy.fft._fft import (_convert_fft_type, _default_fft_func, _fft,
+                           _get_cufft_plan_nd, _get_fftn_out_size,
+                           _output_dtype)
+from cupy.fft._cache import get_plan_cache
+
+
+def get_fft_plan(a, shape=None, axes=None, value_type='C2C'):
+    """ Generate a CUDA FFT plan for transforming up to three axes.
+
+    Args:
+        a (cupy.ndarray): Array to be transform, assumed to be either C- or
+            F- contiguous.
+        shape (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``shape`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (None or int or tuple of int):  The axes of the array to
+            transform. If `None`, it is assumed that all axes are transformed.
+
+            Currently, for performing N-D transform these must be a set of up
+            to three adjacent axes, and must include either the first or the
+            last axis of the array.
+        value_type (str): The FFT type to perform. Acceptable values are:
+
+            * 'C2C': complex-to-complex transform (default)
+            * 'R2C': real-to-complex transform
+            * 'C2R': complex-to-real transform
+
+    Returns:
+        a cuFFT plan for either 1D transform (``cupy.cuda.cufft.Plan1d``) or
+        N-D transform (``cupy.cuda.cufft.PlanNd``).
+
+    .. note::
+        The returned plan can not only be passed as one of the arguments of
+        the functions in ``cupyx.scipy.fftpack``, but also be used as a
+        context manager for both ``cupy.fft`` and ``cupyx.scipy.fftpack``
+        functions:
+
+        .. code-block:: python
+
+            x = cupy.random.random(16).reshape(4, 4).astype(complex)
+            plan = cupyx.scipy.fftpack.get_fft_plan(x)
+            with plan:
+                y = cupy.fft.fftn(x)
+                # alternatively:
+                y = cupyx.scipy.fftpack.fftn(x)  # no explicit plan is given!
+            # alternatively:
+            y = cupyx.scipy.fftpack.fftn(x, plan=plan)  # pass plan explicitly
+
+        In the first case, no cuFFT plan will be generated automatically,
+        even if ``cupy.fft.config.enable_nd_planning = True`` is set.
+
+    .. note::
+        If this function is called under the context of
+        :func:`~cupy.fft.config.set_cufft_callbacks`, the generated plan will
+        have callbacks enabled.
+
+    .. warning::
+        This API is a deviation from SciPy's, is currently experimental, and
+        may be changed in the future version.
+    """
+    # check input array
+    if a.flags.c_contiguous:
+        order = 'C'
+    elif a.flags.f_contiguous:
+        order = 'F'
+    else:
+        raise ValueError('Input array a must be contiguous')
+
+    if isinstance(shape, int):
+        shape = (shape,)
+    if isinstance(axes, int):
+        axes = (axes,)
+    if (shape is not None) and (axes is not None) and len(shape) != len(axes):
+        raise ValueError('Shape and axes have different lengths.')
+
+    # check axes
+    # n=1: 1d (need axis1D); n>1: Nd
+    if axes is None:
+        n = a.ndim if shape is None else len(shape)
+        axes = tuple(i for i in range(-n, 0))
+        if n == 1:
+            axis1D = 0
+    else:  # axes is a tuple
+        n = len(axes)
+        if n == 1:
+            axis1D = axes[0]
+            if axis1D >= a.ndim or axis1D < -a.ndim:
+                err = 'The chosen axis ({0}) exceeds the number of '\
+                      'dimensions of a ({1})'.format(axis1D, a.ndim)
+                raise ValueError(err)
+        elif n > 3:
+            raise ValueError('Only up to three axes is supported')
+
+    # Note that "shape" here refers to the shape along trasformed axes, not
+    # the shape of the output array, and we need to convert it to the latter.
+    # The result is as if "a=_cook_shape(a); return a.shape" is called.
+    # Because of this, we need to use (possibly unsorted) axes.
+    transformed_shape = shape
+    shape = list(a.shape)
+    if transformed_shape is not None:
+        for s, axis in zip(transformed_shape, axes):
+            if s is not None:
+                if axis == axes[-1] and value_type == 'C2R':
+                    s = s // 2 + 1
+                shape[axis] = s
+    shape = tuple(shape)
+
+    # check value_type
+    out_dtype = _output_dtype(a.dtype, value_type)
+    fft_type = _convert_fft_type(out_dtype, value_type)
+    # TODO(leofang): figure out if we really have to skip F-order?
+    if n > 1 and value_type != 'C2C' and a.flags.f_contiguous:
+        raise ValueError('C2R/R2C PlanNd for F-order arrays is not supported')
+
+    # generate plan
+    # (load from cache if it exists, otherwise create one but don't cache it)
+    if n > 1:  # ND transform
+        if cupy.cuda.runtime.is_hip and value_type == 'C2R':
+            raise RuntimeError("hipFFT's C2R PlanNd is buggy and unsupported")
+        out_size = _get_fftn_out_size(
+            shape, transformed_shape, axes[-1], value_type)
+        # _get_cufft_plan_nd interacts with plan cache and callback
+        plan = _get_cufft_plan_nd(
+            shape, fft_type, axes=axes, order=order, out_size=out_size,
+            to_cache=False)
+    else:  # 1D transform
+        # prepare plan arguments
+        if value_type != 'C2R':
+            out_size = shape[axis1D]
+        else:
+            out_size = _get_fftn_out_size(
+                shape, transformed_shape, axis1D, value_type)
+        batch = prod(shape) // shape[axis1D]
+        devices = None if not config.use_multi_gpus else config._devices
+
+        keys = (out_size, fft_type, batch, devices)
+        mgr = config.get_current_callback_manager()
+        if mgr is not None:
+            # to avoid a weird segfault, we generate and cache distinct plans
+            # for every possible (load_aux, store_aux) pairs; the plans are
+            # still generated from the same external Python module
+            load_aux = mgr.cb_load_aux_arr
+            store_aux = mgr.cb_store_aux_arr
+            keys += (mgr.cb_load, mgr.cb_store,
+                     0 if load_aux is None else load_aux.data.ptr,
+                     0 if store_aux is None else store_aux.data.ptr)
+        cache = get_plan_cache()
+        cached_plan = cache.get(keys)
+        if cached_plan is not None:
+            plan = cached_plan
+        elif mgr is None:
+            plan = cufft.Plan1d(out_size, fft_type, batch, devices=devices)
+        else:  # has callback
+            # TODO(leofang): support multi-GPU callback (devices is ignored)
+            if devices:
+                raise NotImplementedError('multi-GPU cuFFT callbacks are not '
+                                          'yet supported')
+            plan = mgr.create_plan(('Plan1d', keys[:-3]))
+            mgr.set_callbacks(plan)
+
+    return plan
+
+
+def fft(x, n=None, axis=-1, overwrite_x=False, plan=None):
+    """Compute the one-dimensional FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.Plan1d` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axis``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, axis)
+
+            Note that `plan` is defaulted to None, meaning CuPy will use an
+            auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if that of the input is another.
+
+    .. note::
+       The argument `plan` is currently experimental and the interface may be
+       changed in the future version.
+
+    .. seealso:: :func:`scipy.fftpack.fft`
+    """
+    return _fft(x, (n,), (axis,), None, cufft.CUFFT_FORWARD,
+                overwrite_x=overwrite_x, plan=plan)
+
+
+def ifft(x, n=None, axis=-1, overwrite_x=False, plan=None):
+    """Compute the one-dimensional inverse FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.Plan1d` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axis``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, axis)
+
+            Note that `plan` is defaulted to None, meaning CuPy will use an
+            auto-generated plan behind the scene.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``n`` and type
+            will convert to complex if that of the input is another.
+
+    .. note::
+       The argument `plan` is currently experimental and the interface may be
+       changed in the future version.
+
+    .. seealso:: :func:`scipy.fftpack.ifft`
+    """
+    return _fft(x, (n,), (axis,), None, cufft.CUFFT_INVERSE,
+                overwrite_x=overwrite_x, plan=plan)
+
+
+def fft2(x, shape=None, axes=(-2, -1), overwrite_x=False, plan=None):
+    """Compute the two-dimensional FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        shape (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``shape`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, axes)
+
+            Note that `plan` is defaulted to None, meaning CuPy will either
+            use an auto-generated plan behind the scene if cupy.fft.config.
+            enable_nd_planning = True, or use no cuFFT plan if it is set to
+            False.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``shape`` and
+            type will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fftpack.fft2`
+
+    .. note::
+       The argument `plan` is currently experimental and the interface may be
+       changed in the future version.
+    """
+    func = _default_fft_func(x, shape, axes, plan)
+    return func(x, shape, axes, None, cufft.CUFFT_FORWARD,
+                overwrite_x=overwrite_x, plan=plan)
+
+
+def ifft2(x, shape=None, axes=(-2, -1), overwrite_x=False, plan=None):
+    """Compute the two-dimensional inverse FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        shape (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``shape`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, axes)
+
+            Note that `plan` is defaulted to None, meaning CuPy will either
+            use an auto-generated plan behind the scene if cupy.fft.config.
+            enable_nd_planning = True, or use no cuFFT plan if it is set to
+            False.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``shape`` and
+            type will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fftpack.ifft2`
+
+    .. note::
+       The argument `plan` is currently experimental and the interface may be
+       changed in the future version.
+    """
+    func = _default_fft_func(x, shape, axes, plan)
+    return func(x, shape, axes, None, cufft.CUFFT_INVERSE,
+                overwrite_x=overwrite_x, plan=plan)
+
+
+def fftn(x, shape=None, axes=None, overwrite_x=False, plan=None):
+    """Compute the N-dimensional FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        shape (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``shape`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, axes)
+
+            Note that `plan` is defaulted to None, meaning CuPy will either
+            use an auto-generated plan behind the scene if cupy.fft.config.
+            enable_nd_planning = True, or use no cuFFT plan if it is set to
+            False.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``shape`` and
+            type will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fftpack.fftn`
+
+    .. note::
+       The argument `plan` is currently experimental and the interface may be
+       changed in the future version.
+    """
+    func = _default_fft_func(x, shape, axes, plan)
+    return func(x, shape, axes, None, cufft.CUFFT_FORWARD,
+                overwrite_x=overwrite_x, plan=plan)
+
+
+def ifftn(x, shape=None, axes=None, overwrite_x=False, plan=None):
+    """Compute the N-dimensional inverse FFT.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        shape (None or tuple of ints): Shape of the transformed axes of the
+            output. If ``shape`` is not given, the lengths of the input along
+            the axes specified by ``axes`` are used.
+        axes (tuple of ints): Axes over which to compute the FFT.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.PlanNd` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axes``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(x, axes)
+
+            Note that `plan` is defaulted to None, meaning CuPy will either
+            use an auto-generated plan behind the scene if cupy.fft.config.
+            enable_nd_planning = True, or use no cuFFT plan if it is set to
+            False.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array which shape is specified by ``shape`` and
+            type will convert to complex if that of the input is another.
+
+    .. seealso:: :func:`scipy.fftpack.ifftn`
+
+    .. note::
+       The argument `plan` is currently experimental and the interface may be
+       changed in the future version.
+    """
+    func = _default_fft_func(x, shape, axes, plan)
+    return func(x, shape, axes, None, cufft.CUFFT_INVERSE,
+                overwrite_x=overwrite_x, plan=plan)
+
+
+def rfft(x, n=None, axis=-1, overwrite_x=False, plan=None):
+    """Compute the one-dimensional FFT for real input.
+
+    The returned real array contains
+
+    .. code-block:: python
+
+        [y(0),Re(y(1)),Im(y(1)),...,Re(y(n/2))]  # if n is even
+        [y(0),Re(y(1)),Im(y(1)),...,Re(y(n/2)),Im(y(n/2))]  # if n is odd
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+        plan (:class:`cupy.cuda.cufft.Plan1d` or ``None``): a cuFFT plan for
+            transforming ``x`` over ``axis``, which can be obtained using::
+
+                plan = cupyx.scipy.fftpack.get_fft_plan(
+                    x, axes, value_type='R2C')
+
+            Note that `plan` is defaulted to None, meaning CuPy will either
+            use an auto-generated plan behind the scene if cupy.fft.config.
+            enable_nd_planning = True, or use no cuFFT plan if it is set to
+            False.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array.
+
+    .. seealso:: :func:`scipy.fftpack.rfft`
+
+    .. note::
+       The argument `plan` is currently experimental and the interface may be
+       changed in the future version.
+    """
+    if n is None:
+        n = x.shape[axis]
+
+    shape = list(x.shape)
+    shape[axis] = n
+    f = _fft(x, (n,), (axis,), None, cufft.CUFFT_FORWARD, 'R2C',
+             overwrite_x=overwrite_x, plan=plan)
+    z = cupy.empty(shape, f.real.dtype)
+
+    slice_z = [slice(None)] * x.ndim
+    slice_f = [slice(None)] * x.ndim
+
+    slice_z[axis] = slice(1)
+    slice_f[axis] = slice(1)
+    z[tuple(slice_z)] = f[tuple(slice_f)].real
+
+    slice_z[axis] = slice(1, None, 2)
+    slice_f[axis] = slice(1, None)
+    z[tuple(slice_z)] = f[tuple(slice_f)].real
+
+    slice_z[axis] = slice(2, None, 2)
+    slice_f[axis] = slice(1, n - f.shape[axis] + 1)
+    z[tuple(slice_z)] = f[tuple(slice_f)].imag
+
+    return z
+
+
+def irfft(x, n=None, axis=-1, overwrite_x=False):
+    """Compute the one-dimensional inverse FFT for real input.
+
+    Args:
+        x (cupy.ndarray): Array to be transformed.
+        n (None or int): Length of the transformed axis of the output. If ``n``
+            is not given, the length of the input along the axis specified by
+            ``axis`` is used.
+        axis (int): Axis over which to compute the FFT.
+        overwrite_x (bool): If True, the contents of ``x`` can be destroyed.
+
+    Returns:
+        cupy.ndarray:
+            The transformed array.
+
+    .. seealso:: :func:`scipy.fftpack.irfft`
+
+    .. note::
+       This function does not support a precomputed `plan`. If you need this
+       capability, please consider using :func:`cupy.fft.irfft` or :func:`
+       cupyx.scipy.fft.irfft`.
+    """
+    if n is None:
+        n = x.shape[axis]
+    m = min(n, x.shape[axis])
+
+    shape = list(x.shape)
+    shape[axis] = n // 2 + 1
+    if x.dtype in (cupy.float16, cupy.float32):
+        z = cupy.zeros(shape, dtype=cupy.complex64)
+    else:
+        z = cupy.zeros(shape, dtype=cupy.complex128)
+
+    slice_x = [slice(None)] * x.ndim
+    slice_z = [slice(None)] * x.ndim
+
+    slice_x[axis] = slice(1)
+    slice_z[axis] = slice(1)
+    z[tuple(slice_z)].real = x[tuple(slice_x)]
+
+    slice_x[axis] = slice(1, m, 2)
+    slice_z[axis] = slice(1, m // 2 + 1)
+    z[tuple(slice_z)].real = x[tuple(slice_x)]
+
+    slice_x[axis] = slice(2, m, 2)
+    slice_z[axis] = slice(1, (m + 1) // 2)
+    z[tuple(slice_z)].imag = x[tuple(slice_x)]
+
+    return _fft(z, (n,), (axis,), None, cufft.CUFFT_INVERSE, 'C2R',
+                overwrite_x=overwrite_x)
diff --git a/cupyx/scipy/interpolate/__init__.py b/cupyx/scipy/interpolate/__init__.py
new file mode 100644
index 0000000..a024aa7
--- /dev/null
+++ b/cupyx/scipy/interpolate/__init__.py
@@ -0,0 +1,21 @@
+# Univariate Interpolation
+from cupyx.scipy.interpolate._polyint import BarycentricInterpolator  # NOQA
+from cupyx.scipy.interpolate._polyint import KroghInterpolator  # NOQA
+from cupyx.scipy.interpolate._polyint import barycentric_interpolate  # NOQA
+from cupyx.scipy.interpolate._polyint import krogh_interpolate  # NOQA
+from cupyx.scipy.interpolate._interpolate import PPoly, BPoly  # NOQA
+from cupyx.scipy.interpolate._cubic import (  # NOQA
+    CubicHermiteSpline, PchipInterpolator, pchip_interpolate,  # NOQA
+    Akima1DInterpolator)  # NOQA
+
+# 1-D Splines
+from cupyx.scipy.interpolate._bspline import BSpline, splantider, splder  # NOQA
+from cupyx.scipy.interpolate._bspline2 import make_interp_spline  # NOQA
+
+# Radial basis functions
+from cupyx.scipy.interpolate._rbfinterp import RBFInterpolator  # NOQA
+from cupyx.scipy.interpolate._rgi import RegularGridInterpolator  # NOQA
+from cupyx.scipy.interpolate._rgi import interpn  # NOQA
+
+# Backward compatibility
+pchip = PchipInterpolator  # NOQA
diff --git a/cupyx/scipy/interpolate/_bspline.py b/cupyx/scipy/interpolate/_bspline.py
new file mode 100644
index 0000000..d037d59
--- /dev/null
+++ b/cupyx/scipy/interpolate/_bspline.py
@@ -0,0 +1,943 @@
+
+import operator
+
+import cupy
+from cupy._core import internal
+from cupy._core._scalar import get_typename
+
+from cupyx.scipy.sparse import csr_matrix
+
+import numpy as np
+
+TYPES = ['double', 'thrust::complex<double>']
+INT_TYPES = ['int', 'long long']
+
+INTERVAL_KERNEL = r'''
+#include <cupy/complex.cuh>
+extern "C" {
+__global__ void find_interval(
+        const double* t, const double* x, long long* out,
+        int k, int n, bool extrapolate, int total_x) {
+
+    int idx = blockDim.x * blockIdx.x + threadIdx.x;
+    if(idx >= total_x) {
+        return;
+    }
+
+    double xp = *&x[idx];
+    double tb = *&t[k];
+    double te = *&t[n];
+
+    if(isnan(xp)) {
+        out[idx] = -1;
+        return;
+    }
+
+    if((xp < tb || xp > te) && !extrapolate) {
+        out[idx] = -1;
+        return;
+    }
+
+    int left = k;
+    int right = n;
+    int mid;
+    bool found = false;
+
+    while(left < right && !found) {
+        mid = ((right + left) / 2);
+        if(xp > *&t[mid]) {
+            left = mid + 1;
+        } else if (xp < *&t[mid]) {
+            right = mid - 1;
+        } else {
+            found = true;
+        }
+    }
+
+    int default_value = left - 1 < k ? k : left - 1;
+    int result = found ? mid + 1 : default_value + 1;
+
+    while(xp >= *&t[result] && result != n) {
+        result++;
+    }
+
+    out[idx] = result - 1;
+}
+}
+'''
+
+INTERVAL_MODULE = cupy.RawModule(
+    code=INTERVAL_KERNEL, options=('-std=c++11',),)
+#    name_expressions=[f'find_interval<{type_name}>' for type_name in TYPES])
+
+
+D_BOOR_KERNEL = r'''
+#include <cupy/complex.cuh>
+#include <cupy/math_constants.h>
+#define COMPUTE_LINEAR 0x1
+
+template<typename T>
+__global__ void d_boor(
+        const double* t, const T* c, const int k, const int mu,
+        const double* x, const long long* intervals, T* out,
+        double* temp, int num_c, int mode, int num_x) {
+
+    int idx = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if(idx >= num_x) {
+        return;
+    }
+
+    double xp = *&x[idx];
+    long long interval = *&intervals[idx];
+
+    double* h = temp + idx * (2 * k + 1);
+    double* hh = h + k + 1;
+
+    int ind, j, n;
+    double xa, xb, w;
+
+    if(mode == COMPUTE_LINEAR && interval < 0) {
+        for(j = 0; j < num_c; j++) {
+            out[num_c * idx + j] = CUDART_NAN;
+        }
+        return;
+    }
+
+    /*
+     * Perform k-m "standard" deBoor iterations
+     * so that h contains the k+1 non-zero values of beta_{ell,k-m}(x)
+     * needed to calculate the remaining derivatives.
+     */
+    h[0] = 1.0;
+    for (j = 1; j <= k - mu; j++) {
+        for(int p = 0; p < j; p++) {
+            hh[p] = h[p];
+        }
+        h[0] = 0.0;
+        for (n = 1; n <= j; n++) {
+            ind = interval + n;
+            xb = t[ind];
+            xa = t[ind - j];
+            if (xb == xa) {
+                h[n] = 0.0;
+                continue;
+            }
+            w = hh[n - 1]/(xb - xa);
+            h[n - 1] += w*(xb - xp);
+            h[n] = w*(xp - xa);
+        }
+    }
+
+    /*
+     * Now do m "derivative" recursions
+     * to convert the values of beta into the mth derivative
+     */
+    for (j = k - mu + 1; j <= k; j++) {
+        for(int p = 0; p < j; p++) {
+            hh[p] = h[p];
+        }
+        h[0] = 0.0;
+        for (n = 1; n <= j; n++) {
+            ind = interval + n;
+            xb = t[ind];
+            xa = t[ind - j];
+            if (xb == xa) {
+                h[mu] = 0.0;
+                continue;
+            }
+            w = ((double) j) * hh[n - 1]/(xb - xa);
+            h[n - 1] -= w;
+            h[n] = w;
+        }
+    }
+
+    if(mode != COMPUTE_LINEAR) {
+        return;
+    }
+
+    // Compute linear combinations
+    for(j = 0; j < num_c; j++) {
+        out[num_c * idx + j] = 0;
+        for(n = 0; n < k + 1; n++) {
+            out[num_c * idx + j] = (
+                out[num_c * idx + j] +
+                c[(interval + n - k) * num_c + j] * ((T) h[n]));
+        }
+    }
+
+}
+'''
+
+D_BOOR_MODULE = cupy.RawModule(
+    code=D_BOOR_KERNEL, options=('-std=c++11',),
+    name_expressions=[f'd_boor<{type_name}>' for type_name in TYPES])
+
+
+DESIGN_MAT_KERNEL = r'''
+#include <cupy/complex.cuh>
+
+template<typename U>
+__global__ void compute_design_matrix(
+        const int k, const long long* intervals, double* bspline_basis,
+        double* data, U* indices, int num_intervals) {
+
+    int idx = blockDim.x * blockIdx.x + threadIdx.x;
+    if(idx >= num_intervals) {
+        return;
+    }
+
+    long long interval = *&intervals[idx];
+
+    double* work = bspline_basis + idx * (2 * k + 1);
+
+    for(int j = 0; j <= k; j++) {
+        int m = (k + 1) * idx + j;
+        data[m] = work[j];
+        indices[m] = (U) (interval - k + j);
+    }
+}
+'''
+
+DESIGN_MAT_MODULE = cupy.RawModule(
+    code=DESIGN_MAT_KERNEL, options=('-std=c++11',),
+    name_expressions=[f'compute_design_matrix<{itype}>'
+                      for itype in INT_TYPES])
+
+
+def _get_module_func(module, func_name, *template_args):
+    def _get_typename(dtype):
+        typename = get_typename(dtype)
+        if dtype.kind == 'c':
+            typename = 'thrust::' + typename
+        return typename
+    args_dtypes = [_get_typename(arg.dtype) for arg in template_args]
+    template = ', '.join(args_dtypes)
+    kernel_name = f'{func_name}<{template}>' if template_args else func_name
+    kernel = module.get_function(kernel_name)
+    return kernel
+
+
+def _get_dtype(dtype):
+    """Return np.complex128 for complex dtypes, np.float64 otherwise."""
+    if cupy.issubdtype(dtype, cupy.complexfloating):
+        return cupy.complex_
+    else:
+        return cupy.float_
+
+
+def _as_float_array(x, check_finite=False):
+    """Convert the input into a C contiguous float array.
+    NB: Upcasts half- and single-precision floats to double precision.
+    """
+    x = cupy.ascontiguousarray(x)
+    dtyp = _get_dtype(x.dtype)
+    x = x.astype(dtyp, copy=False)
+    if check_finite and not cupy.isfinite(x).all():
+        raise ValueError("Array must not contain infs or nans.")
+    return x
+
+
+def _evaluate_spline(t, c, k, xp, nu, extrapolate, out):
+    """
+    Evaluate a spline in the B-spline basis.
+
+    Parameters
+    ----------
+    t : ndarray, shape (n+k+1)
+        knots
+    c : ndarray, shape (n, m)
+        B-spline coefficients
+    xp : ndarray, shape (s,)
+        Points to evaluate the spline at.
+    nu : int
+        Order of derivative to evaluate.
+    extrapolate : int, optional
+        Whether to extrapolate to ouf-of-bounds points, or to return NaNs.
+    out : ndarray, shape (s, m)
+        Computed values of the spline at each of the input points.
+        This argument is modified in-place.
+    """
+    n = t.shape[0] - k - 1
+    intervals = cupy.empty_like(xp, dtype=cupy.int64)
+
+    # Compute intervals for each value
+    interval_kernel = _get_module_func(INTERVAL_MODULE, 'find_interval')
+    interval_kernel(((xp.shape[0] + 128 - 1) // 128,), (128,),
+                    (t, xp, intervals, k, n, extrapolate, xp.shape[0]))
+
+    # Compute interpolation
+    num_c = int(np.prod(c.shape[1:]))
+    temp = cupy.empty(xp.shape[0] * (2 * k + 1))
+    d_boor_kernel = _get_module_func(D_BOOR_MODULE, 'd_boor', c)
+    d_boor_kernel(((xp.shape[0] + 128 - 1) // 128,), (128,),
+                  (t, c, k, nu, xp, intervals, out, temp, num_c, 1,
+                   xp.shape[0]))
+
+
+def _make_design_matrix(x, t, k, extrapolate, indices):
+    """
+    Returns a design matrix in CSR format.
+    Note that only indices is passed, but not indptr because indptr is already
+    precomputed in the calling Python function design_matrix.
+
+    Parameters
+    ----------
+    x : array_like, shape (n,)
+        Points to evaluate the spline at.
+    t : array_like, shape (nt,)
+        Sorted 1D array of knots.
+    k : int
+        B-spline degree.
+    extrapolate : bool, optional
+        Whether to extrapolate to ouf-of-bounds points.
+    indices : ndarray, shape (n * (k + 1),)
+        Preallocated indices of the final CSR array.
+    Returns
+    -------
+    data
+        The data array of a CSR array of the b-spline design matrix.
+        In each row all the basis elements are evaluated at the certain point
+        (first row - x[0], ..., last row - x[-1]).
+
+    indices
+        The indices array of a CSR array of the b-spline design matrix.
+    """
+    n = t.shape[0] - k - 1
+    intervals = cupy.empty_like(x, dtype=cupy.int64)
+
+    # Compute intervals for each value
+    interval_kernel = _get_module_func(INTERVAL_MODULE, 'find_interval')
+    interval_kernel(((x.shape[0] + 128 - 1) // 128,), (128,),
+                    (t, x, intervals, k, n, extrapolate, x.shape[0]))
+
+    # Compute interpolation
+    bspline_basis = cupy.empty(x.shape[0] * (2 * k + 1))
+    d_boor_kernel = _get_module_func(D_BOOR_MODULE, 'd_boor', x)
+    d_boor_kernel(((x.shape[0] + 128 - 1) // 128,), (128,),
+                  (t, None, k, 0, x, intervals, None, bspline_basis, 0, 0,
+                   x.shape[0]))
+
+    data = cupy.zeros(x.shape[0] * (k + 1), dtype=cupy.float_)
+    design_mat_kernel = _get_module_func(
+        DESIGN_MAT_MODULE, 'compute_design_matrix', indices)
+    design_mat_kernel(((x.shape[0] + 128 - 1) // 128,), (128,),
+                      (k, intervals, bspline_basis, data, indices,
+                       x.shape[0]))
+
+    return data, indices
+
+
+def splder(tck, n=1):
+    """
+    Compute the spline representation of the derivative of a given spline
+
+    Parameters
+    ----------
+    tck : tuple of (t, c, k)
+        Spline whose derivative to compute
+    n : int, optional
+        Order of derivative to evaluate. Default: 1
+
+    Returns
+    -------
+    tck_der : tuple of (t2, c2, k2)
+        Spline of order k2=k-n representing the derivative
+        of the input spline.
+
+    Notes
+    -----
+    .. seealso:: :class:`scipy.interpolate.splder`
+
+    See Also
+    --------
+    splantider, splev, spalde
+    """
+    if n < 0:
+        return splantider(tck, -n)
+
+    t, c, k = tck
+
+    if n > k:
+        raise ValueError(("Order of derivative (n = %r) must be <= "
+                          "order of spline (k = %r)") % (n, tck[2]))
+
+    # Extra axes for the trailing dims of the `c` array:
+    sh = (slice(None),) + ((None,)*len(c.shape[1:]))
+
+    try:
+        for j in range(n):
+            # See e.g. Schumaker, Spline Functions: Basic Theory, Chapter 5
+
+            # Compute the denominator in the differentiation formula.
+            # (and append traling dims, if necessary)
+            dt = t[k+1:-1] - t[1:-k-1]
+            dt = dt[sh]
+            # Compute the new coefficients
+            c = (c[1:-1-k] - c[:-2-k]) * k / dt
+            # Pad coefficient array to same size as knots (FITPACK
+            # convention)
+            c = cupy.r_[c, np.zeros((k,) + c.shape[1:])]
+            # Adjust knots
+            t = t[1:-1]
+            k -= 1
+    except FloatingPointError as e:
+        raise ValueError(("The spline has internal repeated knots "
+                          "and is not differentiable %d times") % n) from e
+
+    return t, c, k
+
+
+def splantider(tck, n=1):
+    """
+    Compute the spline for the antiderivative (integral) of a given spline.
+
+    Parameters
+    ----------
+    tck : tuple of (t, c, k)
+        Spline whose antiderivative to compute
+    n : int, optional
+        Order of antiderivative to evaluate. Default: 1
+
+    Returns
+    -------
+    tck_ader : tuple of (t2, c2, k2)
+        Spline of order k2=k+n representing the antiderivative of the input
+        spline.
+
+    See Also
+    --------
+    splder, splev, spalde
+
+    Notes
+    -----
+    The `splder` function is the inverse operation of this function.
+    Namely, ``splder(splantider(tck))`` is identical to `tck`, modulo
+    rounding error.
+
+    .. seealso:: :class:`scipy.interpolate.splantider`
+    """
+    if n < 0:
+        return splder(tck, -n)
+
+    t, c, k = tck
+
+    # Extra axes for the trailing dims of the `c` array:
+    sh = (slice(None),) + (None,)*len(c.shape[1:])
+
+    for j in range(n):
+        # This is the inverse set of operations to splder.
+
+        # Compute the multiplier in the antiderivative formula.
+        dt = t[k+1:] - t[:-k-1]
+        dt = dt[sh]
+        # Compute the new coefficients
+        c = cupy.cumsum(c[:-k-1] * dt, axis=0) / (k + 1)
+        c = cupy.r_[cupy.zeros((1,) + c.shape[1:]),
+                    c, [c[-1]] * (k+2)]
+        # New knots
+        t = cupy.r_[t[0], t, t[-1]]
+        k += 1
+
+    return t, c, k
+
+
+class BSpline:
+    r"""Univariate spline in the B-spline basis.
+
+    .. math::
+        S(x) = \sum_{j=0}^{n-1} c_j  B_{j, k; t}(x)
+
+    where :math:`B_{j, k; t}` are B-spline basis functions of degree `k`
+    and knots `t`.
+
+    Parameters
+    ----------
+    t : ndarray, shape (n+k+1,)
+        knots
+    c : ndarray, shape (>=n, ...)
+        spline coefficients
+    k : int
+        B-spline degree
+    extrapolate : bool or 'periodic', optional
+        whether to extrapolate beyond the base interval, ``t[k] .. t[n]``,
+        or to return nans.
+        If True, extrapolates the first and last polynomial pieces of b-spline
+        functions active on the base interval.
+        If 'periodic', periodic extrapolation is used.
+        Default is True.
+    axis : int, optional
+        Interpolation axis. Default is zero.
+
+    Attributes
+    ----------
+    t : ndarray
+        knot vector
+    c : ndarray
+        spline coefficients
+    k : int
+        spline degree
+    extrapolate : bool
+        If True, extrapolates the first and last polynomial pieces of b-spline
+        functions active on the base interval.
+    axis : int
+        Interpolation axis.
+    tck : tuple
+        A read-only equivalent of ``(self.t, self.c, self.k)``
+
+    Notes
+    -----
+    B-spline basis elements are defined via
+
+    .. math::
+        B_{i, 0}(x) = 1, \textrm{if $t_i \le x < t_{i+1}$, otherwise $0$,}
+
+        B_{i, k}(x) = \frac{x - t_i}{t_{i+k} - t_i} B_{i, k-1}(x)
+                 + \frac{t_{i+k+1} - x}{t_{i+k+1} - t_{i+1}} B_{i+1, k-1}(x)
+
+    **Implementation details**
+
+    - At least ``k+1`` coefficients are required for a spline of degree `k`,
+      so that ``n >= k+1``. Additional coefficients, ``c[j]`` with
+      ``j > n``, are ignored.
+
+    - B-spline basis elements of degree `k` form a partition of unity on the
+      *base interval*, ``t[k] <= x <= t[n]``.
+
+    - Based on [1]_ and [2]_
+
+    .. seealso:: :class:`scipy.interpolate.BSpline`
+
+    References
+    ----------
+    .. [1] Tom Lyche and Knut Morken, Spline methods,
+        http://www.uio.no/studier/emner/matnat/ifi/INF-MAT5340/v05/undervisningsmateriale/
+    .. [2] Carl de Boor, A practical guide to splines, Springer, 2001.
+    """
+
+    def __init__(self, t, c, k, extrapolate=True, axis=0):
+        self.k = operator.index(k)
+        self.c = cupy.asarray(c)
+        self.t = cupy.ascontiguousarray(t, dtype=cupy.float64)
+
+        if extrapolate == 'periodic':
+            self.extrapolate = extrapolate
+        else:
+            self.extrapolate = bool(extrapolate)
+
+        n = self.t.shape[0] - self.k - 1
+
+        axis = internal._normalize_axis_index(axis, self.c.ndim)
+
+        # Note that the normalized axis is stored in the object.
+        self.axis = axis
+        if axis != 0:
+            # roll the interpolation axis to be the first one in self.c
+            # More specifically, the target shape for self.c is (n, ...),
+            # and axis !=0 means that we have c.shape (..., n, ...)
+            #                                               ^
+            #                                              axis
+            self.c = cupy.moveaxis(self.c, axis, 0)
+
+        if k < 0:
+            raise ValueError("Spline order cannot be negative.")
+        if self.t.ndim != 1:
+            raise ValueError("Knot vector must be one-dimensional.")
+        if n < self.k + 1:
+            raise ValueError("Need at least %d knots for degree %d" %
+                             (2*k + 2, k))
+        if (cupy.diff(self.t) < 0).any():
+            raise ValueError("Knots must be in a non-decreasing order.")
+        if len(cupy.unique(self.t[k:n+1])) < 2:
+            raise ValueError("Need at least two internal knots.")
+        if not cupy.isfinite(self.t).all():
+            raise ValueError("Knots should not have nans or infs.")
+        if self.c.ndim < 1:
+            raise ValueError("Coefficients must be at least 1-dimensional.")
+        if self.c.shape[0] < n:
+            raise ValueError(
+                "Knots, coefficients and degree are inconsistent.")
+
+        dt = _get_dtype(self.c.dtype)
+        self.c = cupy.ascontiguousarray(self.c, dtype=dt)
+
+    @classmethod
+    def construct_fast(cls, t, c, k, extrapolate=True, axis=0):
+        """Construct a spline without making checks.
+        Accepts same parameters as the regular constructor. Input arrays
+        `t` and `c` must of correct shape and dtype.
+        """
+        self = object.__new__(cls)
+        self.t, self.c, self.k = t, c, k
+        self.extrapolate = extrapolate
+        self.axis = axis
+        return self
+
+    @property
+    def tck(self):
+        """Equivalent to ``(self.t, self.c, self.k)`` (read-only).
+        """
+        return self.t, self.c, self.k
+
+    @classmethod
+    def basis_element(cls, t, extrapolate=True):
+        """Return a B-spline basis element ``B(x | t[0], ..., t[k+1])``.
+
+        Parameters
+        ----------
+        t : ndarray, shape (k+2,)
+            internal knots
+        extrapolate : bool or 'periodic', optional
+            whether to extrapolate beyond the base interval,
+            ``t[0] .. t[k+1]``, or to return nans.
+            If 'periodic', periodic extrapolation is used.
+            Default is True.
+
+        Returns
+        -------
+        basis_element : callable
+            A callable representing a B-spline basis element for the knot
+            vector `t`.
+
+        Notes
+        -----
+        The degree of the B-spline, `k`, is inferred from the length of `t` as
+        ``len(t)-2``. The knot vector is constructed by appending and
+        prepending ``k+1`` elements to internal knots `t`.
+
+        .. seealso:: :class:`scipy.interpolate.BSpline`
+        """
+        k = len(t) - 2
+        t = _as_float_array(t)
+        t = cupy.r_[(t[0]-1,) * k, t, (t[-1]+1,) * k]
+        c = cupy.zeros_like(t)
+        c[k] = 1.
+        return cls.construct_fast(t, c, k, extrapolate)
+
+    @classmethod
+    def design_matrix(cls, x, t, k, extrapolate=False):
+        """
+        Returns a design matrix as a CSR format sparse array.
+
+        Parameters
+        ----------
+        x : array_like, shape (n,)
+            Points to evaluate the spline at.
+        t : array_like, shape (nt,)
+            Sorted 1D array of knots.
+        k : int
+            B-spline degree.
+        extrapolate : bool or 'periodic', optional
+            Whether to extrapolate based on the first and last intervals
+            or raise an error. If 'periodic', periodic extrapolation is used.
+            Default is False.
+
+        Returns
+        -------
+        design_matrix : `csr_matrix` object
+            Sparse matrix in CSR format where each row contains all the basis
+            elements of the input row (first row = basis elements of x[0],
+            ..., last row = basis elements x[-1]).
+
+        Notes
+        -----
+        In each row of the design matrix all the basis elements are evaluated
+        at the certain point (first row - x[0], ..., last row - x[-1]).
+        `nt` is a length of the vector of knots: as far as there are
+        `nt - k - 1` basis elements, `nt` should be not less than `2 * k + 2`
+        to have at least `k + 1` basis element.
+
+        Out of bounds `x` raises a ValueError.
+
+        .. note::
+            This method returns a `csr_matrix` instance as CuPy still does not
+            have `csr_array`.
+
+        .. seealso:: :class:`scipy.interpolate.BSpline`
+        """
+        x = _as_float_array(x, True)
+        t = _as_float_array(t, True)
+
+        if extrapolate != 'periodic':
+            extrapolate = bool(extrapolate)
+
+        if k < 0:
+            raise ValueError("Spline order cannot be negative.")
+        if t.ndim != 1 or np.any(t[1:] < t[:-1]):
+            raise ValueError(f"Expect t to be a 1-D sorted array_like, but "
+                             f"got t={t}.")
+        # There are `nt - k - 1` basis elements in a BSpline built on the
+        # vector of knots with length `nt`, so to have at least `k + 1` basis
+        # elements we need to have at least `2 * k + 2` elements in the vector
+        # of knots.
+        if len(t) < 2 * k + 2:
+            raise ValueError(f"Length t is not enough for k={k}.")
+
+        if extrapolate == 'periodic':
+            # With periodic extrapolation we map x to the segment
+            # [t[k], t[n]].
+            n = t.size - k - 1
+            x = t[k] + (x - t[k]) % (t[n] - t[k])
+            extrapolate = False
+        elif not extrapolate and (
+            (min(x) < t[k]) or (max(x) > t[t.shape[0] - k - 1])
+        ):
+            # Checks from `find_interval` function
+            raise ValueError(f'Out of bounds w/ x = {x}.')
+
+        # Compute number of non-zeros of final CSR array in order to determine
+        # the dtype of indices and indptr of the CSR array.
+        n = x.shape[0]
+        nnz = n * (k + 1)
+        if nnz < cupy.iinfo(cupy.int32).max:
+            int_dtype = cupy.int32
+        else:
+            int_dtype = cupy.int64
+        # Preallocate indptr and indices
+        indices = cupy.empty(n * (k + 1), dtype=int_dtype)
+        indptr = cupy.arange(0, (n + 1) * (k + 1), k + 1, dtype=int_dtype)
+
+        # indptr is not passed to CUDA as it is already fully computed
+        data, indices = _make_design_matrix(
+            x, t, k, extrapolate, indices
+        )
+        return csr_matrix(
+            (data, indices, indptr),
+            shape=(x.shape[0], t.shape[0] - k - 1)
+        )
+
+    def __call__(self, x, nu=0, extrapolate=None):
+        """
+        Evaluate a spline function.
+
+        Parameters
+        ----------
+        x : array_like
+            points to evaluate the spline at.
+        nu : int, optional
+            derivative to evaluate (default is 0).
+        extrapolate : bool or 'periodic', optional
+            whether to extrapolate based on the first and last intervals
+            or return nans. If 'periodic', periodic extrapolation is used.
+            Default is `self.extrapolate`.
+
+        Returns
+        -------
+        y : array_like
+            Shape is determined by replacing the interpolation axis
+            in the coefficient array with the shape of `x`.
+        """
+        if extrapolate is None:
+            extrapolate = self.extrapolate
+
+        x = cupy.asarray(x)
+        x_shape, x_ndim = x.shape, x.ndim
+        x = cupy.ascontiguousarray(cupy.ravel(x), dtype=cupy.float_)
+
+        # With periodic extrapolation we map x to the segment
+        # [self.t[k], self.t[n]].
+        if extrapolate == 'periodic':
+            n = self.t.size - self.k - 1
+            x = self.t[self.k] + (x - self.t[self.k]) % (self.t[n] -
+                                                         self.t[self.k])
+            extrapolate = False
+
+        out = cupy.empty(
+            (len(x), int(np.prod(self.c.shape[1:]))), dtype=self.c.dtype)
+
+        self._evaluate(x, nu, extrapolate, out)
+        out = out.reshape(x_shape + self.c.shape[1:])
+        if self.axis != 0:
+            # transpose to move the calculated values to the interpolation axis
+            dim_order = list(range(out.ndim))
+            dim_order = (
+                dim_order[x_ndim:x_ndim+self.axis] +
+                dim_order[:x_ndim] +
+                dim_order[x_ndim+self.axis:])
+            out = out.transpose(dim_order)
+
+        return out
+
+    def _ensure_c_contiguous(self):
+        if not self.t.flags.c_contiguous:
+            self.t = self.t.copy()
+        if not self.c.flags.c_contiguous:
+            self.c = self.c.copy()
+
+    def _evaluate(self, xp, nu, extrapolate, out):
+        _evaluate_spline(self.t, self.c.reshape(self.c.shape[0], -1),
+                         self.k, xp, nu, extrapolate, out)
+
+    def derivative(self, nu=1):
+        """
+        Return a B-spline representing the derivative.
+
+        Parameters
+        ----------
+        nu : int, optional
+            Derivative order.
+            Default is 1.
+
+        Returns
+        -------
+        b : BSpline object
+            A new instance representing the derivative.
+
+        See Also
+        --------
+        splder, splantider
+        """
+        c = self.c
+        # pad the c array if needed
+        ct = len(self.t) - len(c)
+        if ct > 0:
+            c = cupy.r_[c, cupy.zeros((ct,) + c.shape[1:])]
+        tck = splder((self.t, c, self.k), nu)
+        return self.construct_fast(*tck, extrapolate=self.extrapolate,
+                                   axis=self.axis)
+
+    def antiderivative(self, nu=1):
+        """
+        Return a B-spline representing the antiderivative.
+
+        Parameters
+        ----------
+        nu : int, optional
+            Antiderivative order. Default is 1.
+
+        Returns
+        -------
+        b : BSpline object
+            A new instance representing the antiderivative.
+
+        Notes
+        -----
+        If antiderivative is computed and ``self.extrapolate='periodic'``,
+        it will be set to False for the returned instance. This is done because
+        the antiderivative is no longer periodic and its correct evaluation
+        outside of the initially given x interval is difficult.
+
+        See Also
+        --------
+        splder, splantider
+        """
+        c = self.c
+        # pad the c array if needed
+        ct = len(self.t) - len(c)
+        if ct > 0:
+            c = cupy.r_[c, cupy.zeros((ct,) + c.shape[1:])]
+        tck = splantider((self.t, c, self.k), nu)
+
+        if self.extrapolate == 'periodic':
+            extrapolate = False
+        else:
+            extrapolate = self.extrapolate
+
+        return self.construct_fast(*tck, extrapolate=extrapolate,
+                                   axis=self.axis)
+
+    def integrate(self, a, b, extrapolate=None):
+        """
+        Compute a definite integral of the spline.
+
+        Parameters
+        ----------
+        a : float
+            Lower limit of integration.
+        b : float
+            Upper limit of integration.
+        extrapolate : bool or 'periodic', optional
+            whether to extrapolate beyond the base interval,
+            ``t[k] .. t[-k-1]``, or take the spline to be zero outside of the
+            base interval. If 'periodic', periodic extrapolation is used.
+            If None (default), use `self.extrapolate`.
+
+        Returns
+        -------
+        I : array_like
+            Definite integral of the spline over the interval ``[a, b]``.
+        """
+        if extrapolate is None:
+            extrapolate = self.extrapolate
+
+        # Prepare self.t and self.c.
+        self._ensure_c_contiguous()
+
+        # Swap integration bounds if needed.
+        sign = 1
+        if b < a:
+            a, b = b, a
+            sign = -1
+        n = self.t.size - self.k - 1
+
+        if extrapolate != "periodic" and not extrapolate:
+            # Shrink the integration interval, if needed.
+            a = max(a, self.t[self.k].item())
+            b = min(b, self.t[n].item())
+
+            # if self.c.ndim == 1:
+            #     # Fast path: use FITPACK's routine
+            #     # (cf _fitpack_impl.splint).
+            #     integral = splint(a, b, self.tck)
+            #     return integral * sign
+
+        out = cupy.empty(
+            (2, int(np.prod(self.c.shape[1:]))), dtype=self.c.dtype)
+
+        # Compute the antiderivative.
+        c = self.c
+        ct = len(self.t) - len(c)
+        if ct > 0:
+            c = cupy.r_[c, cupy.zeros((ct,) + c.shape[1:])]
+        ta, ca, ka = splantider((self.t, c, self.k), 1)
+
+        if extrapolate == 'periodic':
+            # Split the integral into the part over period (can be several
+            # of them) and the remaining part.
+
+            ts, te = self.t[self.k], self.t[n]
+            period = te - ts
+            interval = b - a
+            n_periods, left = divmod(interval, period)
+
+            if n_periods > 0:
+                # Evaluate the difference of antiderivatives.
+                x = cupy.asarray([ts, te], dtype=cupy.float_)
+                _evaluate_spline(ta, ca.reshape(ca.shape[0], -1),
+                                 ka, x, 0, False, out)
+                integral = out[1] - out[0]
+                integral *= n_periods
+            else:
+                integral = cupy.zeros((1, int(np.prod(self.c.shape[1:]))),
+                                      dtype=self.c.dtype)
+
+            # Map a to [ts, te], b is always a + left.
+            a = ts + (a - ts) % period
+            b = a + left
+
+            # If b <= te then we need to integrate over [a, b], otherwise
+            # over [a, te] and from xs to what is remained.
+            if b <= te:
+                x = cupy.asarray([a, b], dtype=cupy.float_)
+                _evaluate_spline(ta, ca.reshape(ca.shape[0], -1),
+                                 ka, x, 0, False, out)
+                integral += out[1] - out[0]
+            else:
+                x = cupy.asarray([a, te], dtype=cupy.float_)
+                _evaluate_spline(ta, ca.reshape(ca.shape[0], -1),
+                                 ka, x, 0, False, out)
+                integral += out[1] - out[0]
+
+                x = cupy.asarray([ts, ts + b - te], dtype=cupy.float_)
+                _evaluate_spline(ta, ca.reshape(ca.shape[0], -1),
+                                 ka, x, 0, False, out)
+                integral += out[1] - out[0]
+        else:
+            # Evaluate the difference of antiderivatives.
+            x = cupy.asarray([a, b], dtype=cupy.float_)
+            _evaluate_spline(ta, ca.reshape(ca.shape[0], -1),
+                             ka, x, 0, extrapolate, out)
+            integral = out[1] - out[0]
+
+        integral *= sign
+        return integral.reshape(ca.shape[1:])
diff --git a/cupyx/scipy/interpolate/_bspline2.py b/cupyx/scipy/interpolate/_bspline2.py
new file mode 100644
index 0000000..5f501c0
--- /dev/null
+++ b/cupyx/scipy/interpolate/_bspline2.py
@@ -0,0 +1,564 @@
+import operator
+
+from numpy.core.multiarray import normalize_axis_index
+
+import cupy
+from cupyx.scipy import sparse
+from cupyx.scipy.sparse.linalg import spsolve
+
+from cupyx.scipy.interpolate._bspline import (
+    _get_module_func, INTERVAL_MODULE, D_BOOR_MODULE, BSpline)
+
+
+def _get_dtype(dtype):
+    """Return np.complex128 for complex dtypes, np.float64 otherwise."""
+    if cupy.issubdtype(dtype, cupy.complexfloating):
+        return cupy.complex_
+    else:
+        return cupy.float_
+
+
+def _as_float_array(x, check_finite=False):
+    """Convert the input into a C contiguous float array.
+
+    NB: Upcasts half- and single-precision floats to double precision.
+    """
+    x = cupy.asarray(x)
+    x = cupy.ascontiguousarray(x)
+    dtyp = _get_dtype(x.dtype)
+    x = x.astype(dtyp, copy=False)
+    if check_finite and not cupy.isfinite(x).all():
+        raise ValueError("Array must not contain infs or nans.")
+    return x
+
+
+# vendored from scipy/_lib/_util.py
+def prod(iterable):
+    """
+    Product of a sequence of numbers.
+    Faster than np.prod for short lists like array shapes, and does
+    not overflow if using Python integers.
+    """
+    product = 1
+    for x in iterable:
+        product *= x
+    return product
+
+
+#################################
+#  Interpolating spline helpers #
+#################################
+
+def _not_a_knot(x, k):
+    """Given data x, construct the knot vector w/ not-a-knot BC.
+    cf de Boor, XIII(12)."""
+    x = cupy.asarray(x)
+    if k % 2 != 1:
+        raise ValueError("Odd degree for now only. Got %s." % k)
+
+    m = (k - 1) // 2
+    t = x[m+1:-m-1]
+    t = cupy.r_[(x[0],)*(k+1), t, (x[-1],)*(k+1)]
+    return t
+
+
+def _augknt(x, k):
+    """Construct a knot vector appropriate for the order-k interpolation."""
+    return cupy.r_[(x[0],)*k, x, (x[-1],)*k]
+
+
+def _periodic_knots(x, k):
+    """Returns vector of nodes on a circle."""
+    xc = cupy.copy(x)
+    n = len(xc)
+    if k % 2 == 0:
+        dx = cupy.diff(xc)
+        xc[1: -1] -= dx[:-1] / 2
+    dx = cupy.diff(xc)
+    t = cupy.zeros(n + 2 * k)
+    t[k: -k] = xc
+    for i in range(0, k):
+        # filling first `k` elements in descending order
+        t[k - i - 1] = t[k - i] - dx[-(i % (n - 1)) - 1]
+        # filling last `k` elements in ascending order
+        t[-k + i] = t[-k + i - 1] + dx[i % (n - 1)]
+    return t
+
+
+def _convert_string_aliases(deriv, target_shape):
+    if isinstance(deriv, str):
+        if deriv == "clamped":
+            deriv = [(1, cupy.zeros(target_shape))]
+        elif deriv == "natural":
+            deriv = [(2, cupy.zeros(target_shape))]
+        else:
+            raise ValueError("Unknown boundary condition : %s" % deriv)
+    return deriv
+
+
+def _process_deriv_spec(deriv):
+    if deriv is not None:
+        try:
+            ords, vals = zip(*deriv)
+        except TypeError as e:
+            msg = ("Derivatives, `bc_type`, should be specified as a pair of "
+                   "iterables of pairs of (order, value).")
+            raise ValueError(msg) from e
+    else:
+        ords, vals = [], []
+    return cupy.atleast_1d(ords, vals)
+
+
+def make_interp_spline(x, y, k=3, t=None, bc_type=None, axis=0,
+                       check_finite=True):
+    """Compute the (coefficients of) interpolating B-spline.
+
+    Parameters
+    ----------
+    x : array_like, shape (n,)
+        Abscissas.
+    y : array_like, shape (n, ...)
+        Ordinates.
+    k : int, optional
+        B-spline degree. Default is cubic, ``k = 3``.
+    t : array_like, shape (nt + k + 1,), optional.
+        Knots.
+        The number of knots needs to agree with the number of data points and
+        the number of derivatives at the edges. Specifically, ``nt - n`` must
+        equal ``len(deriv_l) + len(deriv_r)``.
+    bc_type : 2-tuple or None
+        Boundary conditions.
+        Default is None, which means choosing the boundary conditions
+        automatically. Otherwise, it must be a length-two tuple where the first
+        element (``deriv_l``) sets the boundary conditions at ``x[0]`` and
+        the second element (``deriv_r``) sets the boundary conditions at
+        ``x[-1]``. Each of these must be an iterable of pairs
+        ``(order, value)`` which gives the values of derivatives of specified
+        orders at the given edge of the interpolation interval.
+        Alternatively, the following string aliases are recognized:
+
+        * ``"clamped"``: The first derivatives at the ends are zero. This is
+           equivalent to ``bc_type=([(1, 0.0)], [(1, 0.0)])``.
+        * ``"natural"``: The second derivatives at ends are zero. This is
+          equivalent to ``bc_type=([(2, 0.0)], [(2, 0.0)])``.
+        * ``"not-a-knot"`` (default): The first and second segments are the
+          same polynomial. This is equivalent to having ``bc_type=None``.
+        * ``"periodic"``: The values and the first ``k-1`` derivatives at the
+          ends are equivalent.
+
+    axis : int, optional
+        Interpolation axis. Default is 0.
+    check_finite : bool, optional
+        Whether to check that the input arrays contain only finite numbers.
+        Disabling may give a performance gain, but may result in problems
+        (crashes, non-termination) if the inputs do contain infinities or NaNs.
+        Default is True.
+
+    Returns
+    -------
+    b : a BSpline object of the degree ``k`` and with knots ``t``.
+
+    """
+    # convert string aliases for the boundary conditions
+    if bc_type is None or bc_type == 'not-a-knot' or bc_type == 'periodic':
+        deriv_l, deriv_r = None, None
+    elif isinstance(bc_type, str):
+        deriv_l, deriv_r = bc_type, bc_type
+    else:
+        try:
+            deriv_l, deriv_r = bc_type
+        except TypeError as e:
+            raise ValueError("Unknown boundary condition: %s" % bc_type) from e
+
+    y = cupy.asarray(y)
+
+    axis = normalize_axis_index(axis, y.ndim)
+
+    x = _as_float_array(x, check_finite)
+    y = _as_float_array(y, check_finite)
+
+    y = cupy.moveaxis(y, axis, 0)    # now internally interp axis is zero
+
+    # sanity check the input
+    if bc_type == 'periodic' and not cupy.allclose(y[0], y[-1], atol=1e-15):
+        raise ValueError("First and last points does not match while "
+                         "periodic case expected")
+    if x.size != y.shape[0]:
+        raise ValueError('Shapes of x {} and y {} are incompatible'
+                         .format(x.shape, y.shape))
+    if (x[1:] == x[:-1]).any():
+        raise ValueError("Expect x to not have duplicates")
+    if x.ndim != 1 or (x[1:] < x[:-1]).any():
+        raise ValueError("Expect x to be a 1D strictly increasing sequence.")
+
+    # special-case k=0 right away
+    if k == 0:
+        if any(_ is not None for _ in (t, deriv_l, deriv_r)):
+            raise ValueError("Too much info for k=0: t and bc_type can only "
+                             "be None.")
+        t = cupy.r_[x, x[-1]]
+        c = cupy.asarray(y)
+        c = cupy.ascontiguousarray(c, dtype=_get_dtype(c.dtype))
+        return BSpline.construct_fast(t, c, k, axis=axis)
+
+    # special-case k=1 (e.g., Lyche and Morken, Eq.(2.16))
+    if k == 1 and t is None:
+        if not (deriv_l is None and deriv_r is None):
+            raise ValueError(
+                "Too much info for k=1: bc_type can only be None.")
+        t = cupy.r_[x[0], x, x[-1]]
+        c = cupy.asarray(y)
+        c = cupy.ascontiguousarray(c, dtype=_get_dtype(c.dtype))
+        return BSpline.construct_fast(t, c, k, axis=axis)
+
+    k = operator.index(k)
+
+    if bc_type == 'periodic' and t is not None:
+        raise NotImplementedError("For periodic case t is constructed "
+                                  "automatically and can not be passed "
+                                  "manually")
+
+    # come up with a sensible knot vector, if needed
+    if t is None:
+        if deriv_l is None and deriv_r is None:
+            if bc_type == 'periodic':
+                t = _periodic_knots(x, k)
+            elif k == 2:
+                # OK, it's a bit ad hoc: Greville sites + omit
+                # 2nd and 2nd-to-last points, a la not-a-knot
+                t = (x[1:] + x[:-1]) / 2.
+                t = cupy.r_[(x[0],)*(k+1),
+                            t[1:-1],
+                            (x[-1],)*(k+1)]
+            else:
+                t = _not_a_knot(x, k)
+        else:
+            t = _augknt(x, k)
+
+    t = _as_float_array(t, check_finite)
+
+    if k < 0:
+        raise ValueError("Expect non-negative k.")
+    if t.ndim != 1 or (t[1:] < t[:-1]).any():
+        raise ValueError("Expect t to be a 1-D sorted array_like.")
+    if t.size < x.size + k + 1:
+        raise ValueError('Got %d knots, need at least %d.' %
+                         (t.size, x.size + k + 1))
+    if (x[0] < t[k]) or (x[-1] > t[-k]):
+        raise ValueError('Out of bounds w/ x = %s.' % x)
+
+    if bc_type == 'periodic':
+        return _make_periodic_spline(x, y, t, k, axis)
+
+    # Here : deriv_l, r = [(nu, value), ...]
+    deriv_l = _convert_string_aliases(deriv_l, y.shape[1:])
+    deriv_l_ords, deriv_l_vals = _process_deriv_spec(deriv_l)
+    nleft = deriv_l_ords.shape[0]
+
+    deriv_r = _convert_string_aliases(deriv_r, y.shape[1:])
+    deriv_r_ords, deriv_r_vals = _process_deriv_spec(deriv_r)
+    nright = deriv_r_ords.shape[0]
+
+    # have `n` conditions for `nt` coefficients; need nt-n derivatives
+    n = x.size
+    nt = t.size - k - 1
+
+    if nt - n != nleft + nright:
+        raise ValueError("The number of derivatives at boundaries does not "
+                         "match: expected %s, got %s + %s" %
+                         (nt-n, nleft, nright))
+
+    # bail out if the `y` array is zero-sized
+    if y.size == 0:
+        c = cupy.zeros((nt,) + y.shape[1:], dtype=float)
+        return BSpline.construct_fast(t, c, k, axis=axis)
+
+    # Consruct the colocation matrix of b-splines + boundary conditions.
+    # The coefficients of the interpolating B-spline function are the solution
+    # of the linear system `A @ c = rhs` where `A` is the colocation matrix
+    # (i.e., each row of A corresponds to a data point in the `x` array and
+    #  contains b-splines which are non-zero at this value of x)
+    # Each boundary condition is a fixed value of a certain derivative
+    # at the edge, so each derivative adds a row to `A`.
+    # The `rhs` is the array of data values, `y`, plus derivatives from
+    # boundary conditions, if any.
+    # The colocation matrix is banded (has at most k+1 diagonals). Since LAPACK
+    # linear algebra (?gbsv) is not available, we store it as a CSR array
+
+    # 1. Construct the colocation matrix itself.
+    matr = BSpline.design_matrix(x, t, k)
+
+    # 2. Boundary conditions: need to augment the design matrix with additional
+    # rows, one row per derivative at the left and right edges.
+    # The left-side boundary conditions go to the first rows of the matrix
+    # and the right-side boundary conditions go to the last rows.
+    # Will need a python loop for each derivative because in general they
+    # can be of any order, `m`.
+    # To compute the derivatives, will invoke the de Boor D kernel.
+    if nleft > 0 or nright > 0:
+        # Prepare the I/O arrays for the kernels. We only need the non-zero
+        # b-splines at x[0] and x[-1], but the kernel wants more arrays which
+        # we allocate and ignore (mode != 1)
+        temp = cupy.zeros((nt, ), dtype=float)
+        num_c = 1
+        dummy_c = cupy.empty((nt, num_c), dtype=float)
+        out = cupy.empty((1, 1), dtype=dummy_c.dtype)
+
+        d_boor_kernel = _get_module_func(D_BOOR_MODULE, 'd_boor', dummy_c)
+
+        # find the intervals for x[0] and x[-1]
+        intervals_bc = cupy.empty(2, dtype=cupy.int64)
+        interval_kernel = _get_module_func(INTERVAL_MODULE, 'find_interval')
+        interval_kernel((1,), (2,),
+                        (t, cupy.r_[x[0], x[-1]], intervals_bc, k, nt,
+                         False, 2))
+
+    # 3. B.C.s at x[0]
+    if nleft > 0:
+        x0 = cupy.array([x[0]], dtype=x.dtype)
+        rows = cupy.zeros((nleft, nt), dtype=float)
+
+        for j, m in enumerate(deriv_l_ords):
+            # place the derivatives of the order m at x[0] into `temp`
+            d_boor_kernel((1,), (1,),
+                          (t, dummy_c, k, int(m), x0, intervals_bc,
+                           out,     # ignore (mode !=1),
+                           temp,    # non-zero b-splines
+                           num_c,   # the 2nd dimension of `dummy_c`. Ignore.
+                           0,       # mode != 1 => do not touch dummy_c array
+                           1))      # the length of the `x0` array
+            left = intervals_bc[0]
+            rows[j, left-k:left+1] = temp[:k+1]
+
+        matr = sparse.vstack([sparse.csr_matrix(rows),   # A[:nleft, :]
+                              matr])
+
+    # 4. Repeat for B.C.s at x[-1]
+    if nright > 0:
+        intervals_bc[0] = intervals_bc[-1]   # use the intervals for x[-1]
+        x0 = cupy.array([x[-1]], dtype=x.dtype)
+        rows = cupy.zeros((nright, nt), dtype=float)
+
+        for j, m in enumerate(deriv_r_ords):
+            # place the derivatives of the order m at x[0] into `temp`
+            d_boor_kernel((1,), (1,),
+                          (t, dummy_c, k, int(m), x0, intervals_bc,
+                           out,     # ignore (mode !=1),
+                           temp,    # non-zero b-splines
+                           num_c,   # the 2nd dimension of `dummy_c`. Ignore.
+                           0,       # mode != 1 => do not touch dummy_c array
+                           1))      # the length of the `x0` array
+            left = intervals_bc[0]
+            rows[j, left-k:left+1] = temp[:k+1]
+
+        matr = sparse.vstack([matr,
+                              sparse.csr_matrix(rows)])  # A[nleft+len(x):, :]
+
+    # 5. Prepare the RHS: `y` values to interpolate (+ derivatives, if any)
+    extradim = prod(y.shape[1:])
+    rhs = cupy.empty((nt, extradim), dtype=y.dtype)
+    if nleft > 0:
+        rhs[:nleft] = deriv_l_vals.reshape(-1, extradim)
+    rhs[nleft:nt - nright] = y.reshape(-1, extradim)
+    if nright > 0:
+        rhs[nt - nright:] = deriv_r_vals.reshape(-1, extradim)
+
+    # 6. Finally, solve the linear system for the coefficients.
+    if cupy.issubdtype(rhs.dtype, cupy.complexfloating):
+        # avoid upcasting the l.h.s. to complex (that doubles the memory)
+        coef = (spsolve(matr, rhs.real) +
+                spsolve(matr, rhs.imag) * 1.j)
+    else:
+        coef = spsolve(matr, rhs)
+    coef = cupy.ascontiguousarray(coef.reshape((nt,) + y.shape[1:]))
+    return BSpline(t, coef, k)
+
+
+def _make_interp_spline_full_matrix(x, y, k, t, bc_type):
+    """ Construct the interpolating spline spl(x) = y with *full* linalg.
+
+        Only useful for testing, do not call directly!
+        This version is O(N**2) in memory and O(N**3) in flop count.
+    """
+    # convert string aliases for the boundary conditions
+    if bc_type is None or bc_type == 'not-a-knot':
+        deriv_l, deriv_r = None, None
+    elif isinstance(bc_type, str):
+        deriv_l, deriv_r = bc_type, bc_type
+    else:
+        try:
+            deriv_l, deriv_r = bc_type
+        except TypeError as e:
+            raise ValueError("Unknown boundary condition: %s" % bc_type) from e
+
+    # Here : deriv_l, r = [(nu, value), ...]
+    deriv_l = _convert_string_aliases(deriv_l, y.shape[1:])
+    deriv_l_ords, deriv_l_vals = _process_deriv_spec(deriv_l)
+    nleft = deriv_l_ords.shape[0]
+
+    deriv_r = _convert_string_aliases(deriv_r, y.shape[1:])
+    deriv_r_ords, deriv_r_vals = _process_deriv_spec(deriv_r)
+    nright = deriv_r_ords.shape[0]
+
+    # have `n` conditions for `nt` coefficients; need nt-n derivatives
+    n = x.size
+    nt = t.size - k - 1
+    # Here : deriv_l, r = [(nu, value), ...]
+    deriv_l = _convert_string_aliases(deriv_l, y.shape[1:])
+    deriv_l_ords, deriv_l_vals = _process_deriv_spec(deriv_l)
+    nleft = deriv_l_ords.shape[0]
+
+    deriv_r = _convert_string_aliases(deriv_r, y.shape[1:])
+    deriv_r_ords, deriv_r_vals = _process_deriv_spec(deriv_r)
+    nright = deriv_r_ords.shape[0]
+
+    # have `n` conditions for `nt` coefficients; need nt-n derivatives
+    n = x.size
+    nt = t.size - k - 1
+    assert nt - n == nleft + nright
+
+    # Consruct the colocation matrix of b-splines + boundary conditions.
+    # The coefficients of the interpolating B-spline function are the solution
+    # of the linear system `A @ c = rhs` where `A` is the colocation matrix
+    # (i.e., each row of A corresponds to a data point in the `x` array and
+    #  contains b-splines which are non-zero at this value of x)
+    # Each boundary condition is a fixed value of a certain derivative
+    # at the edge, so each derivative adds a row to `A`.
+    # The `rhs` is the array of data values, `y`, plus derivatives from
+    # boundary conditions, if any.
+
+    # 1. Compute intervals for each value
+    intervals = cupy.empty_like(x, dtype=cupy.int64)
+    interval_kernel = _get_module_func(INTERVAL_MODULE, 'find_interval')
+    interval_kernel(((x.shape[0] + 128 - 1) // 128,), (128,),
+                    (t, x, intervals, k, nt, False, x.shape[0]))
+
+    # 2. Compute non-zero b-spline basis elements for each value in `x`
+    # The way de_Boor_D kernel is written, it wants `c` and `out` arrays
+    # which we do not use (but need to provide to the kernel), and the
+    # `temp` array contains non-zero b-spline basis elements, which we do want.
+    dummy_c = cupy.empty((nt, 1), dtype=float)
+    out = cupy.empty(
+        (len(x), prod(dummy_c.shape[1:])), dtype=dummy_c.dtype)
+
+    num_c = prod(dummy_c.shape[1:])
+    temp = cupy.empty(x.shape[0] * (2 * k + 1))
+    d_boor_kernel = _get_module_func(D_BOOR_MODULE, 'd_boor', dummy_c)
+    d_boor_kernel(((x.shape[0] + 128 - 1) // 128,), (128,),
+                  (t, dummy_c, k, 0, x, intervals, out, temp, num_c, 0,
+                   x.shape[0]))
+
+    # 3. Construct the colocation matrix.
+    # For each value in `x`, the `temp` array contains 2k+1 entries : first
+    # k+1 elements are b-splines, followed by k entries used for work storage
+    # which we ignore.
+    # XXX: full matrices! Can / should use banded linear algebra instead.
+    A = cupy.zeros((nt, nt), dtype=float)
+    offset = nleft
+    for j in range(len(x)):
+        left = intervals[j]
+        A[j + offset, left-k:left+1] = temp[j*(2*k+1):j*(2*k+1)+k+1]
+
+    # 4. Handle boundary conditions: The colocation matrix is augmented with
+    # additional rows, one row per derivative at the left and right edges.
+    # We need a python loop for each derivative because in general they can be
+    # of any order, `m`.
+    # The left-side boundary conditions go to the first rows of the matrix
+    # and the right-side boundary conditions go to the last rows.
+    intervals_bc = cupy.empty(1, dtype=cupy.int64)
+    if nleft > 0:
+        intervals_bc[0] = intervals[0]
+        x0 = cupy.array([x[0]], dtype=x.dtype)
+        for j, m in enumerate(deriv_l_ords):
+            # place the derivatives of the order m at x[0] into `temp`
+            d_boor_kernel((1,), (1,),
+                          (t, dummy_c, k, int(m), x0, intervals_bc, out, temp,
+                           num_c, 0, 1))
+            left = intervals_bc[0]
+            A[j, left-k:left+1] = temp[:k+1]
+
+    # repeat for the b.c. at the right edge.
+    if nright > 0:
+        intervals_bc[0] = intervals[-1]
+        x0 = cupy.array([x[-1]], dtype=x.dtype)
+        for j, m in enumerate(deriv_r_ords):
+            # place the derivatives of the order m at x[0] into `temp`
+            d_boor_kernel((1,), (1,),
+                          (t, dummy_c, k, int(m), x0, intervals_bc, out, temp,
+                           num_c, 0, 1))
+            left = intervals_bc[0]
+            row = nleft + len(x) + j
+            A[row, left-k:left+1] = temp[:k+1]
+
+    # 5. Prepare the RHS: `y` values to interpolate (+ derivatives, if any)
+    extradim = prod(y.shape[1:])
+    rhs = cupy.empty((nt, extradim), dtype=y.dtype)
+    if nleft > 0:
+        rhs[:nleft] = deriv_l_vals.reshape(-1, extradim)
+    rhs[nleft:nt - nright] = y.reshape(-1, extradim)
+    if nright > 0:
+        rhs[nt - nright:] = deriv_r_vals.reshape(-1, extradim)
+
+    # 6. Finally, solve for the coefficients.
+    from cupy.linalg import solve
+    coef = solve(A, rhs)
+
+    coef = cupy.ascontiguousarray(coef.reshape((nt,) + y.shape[1:]))
+    return BSpline(t, coef, k)
+
+
+def _make_periodic_spline(x, y, t, k, axis):
+    n = x.size
+
+    # 1. Construct the colocation matrix.
+    matr = BSpline.design_matrix(x, t, k)
+
+    # 2. Boundary conditions: need to augment the design matrix with additional
+    # rows, one row per derivative at the left and right edges.
+    # The k-1 boundary conditions go to the first rows of the matrix
+    # To compute the derivatives, will invoke the de Boor D kernel.
+
+    # Prepare the I/O arrays for the kernels. We only need the non-zero
+    # b-splines at x[0] and x[-1], but the kernel wants more arrays which
+    # we allocate and ignore (mode != 1)
+    temp = cupy.zeros(2*(2*k+1), dtype=float)
+    num_c = 1
+    dummy_c = cupy.empty((t.size - k - 1, num_c), dtype=float)
+    out = cupy.empty((2, 1), dtype=dummy_c.dtype)
+
+    d_boor_kernel = _get_module_func(D_BOOR_MODULE, 'd_boor', dummy_c)
+
+    # find the intervals for x[0] and x[-1]
+    x0 = cupy.r_[x[0], x[-1]]
+    intervals_bc = cupy.array([k, n + k - 1], dtype=cupy.int64)   # match scipy
+
+    # 3. B.C.s
+    rows = cupy.zeros((k-1, n + k - 1), dtype=float)
+
+    for m in range(k-1):
+        # place the derivatives of the order m at x[0] into `temp`
+        d_boor_kernel((1,), (2,),
+                      (t, dummy_c, k, m+1, x0, intervals_bc,
+                       out,     # ignore (mode !=1),
+                       temp,    # non-zero b-splines
+                       num_c,   # the 2nd dimension of `dummy_c`. Ignore.
+                       0,       # mode != 1 => do not touch dummy_c array
+                       2))      # the length of the `x0` array
+        rows[m, :k+1] = temp[:k+1]
+        rows[m, -k:] -= temp[2*k + 1:(2*k + 1) + k+1][:-1]
+
+    matr_csr = sparse.vstack([sparse.csr_matrix(rows),   # A[:nleft, :]
+                              matr])
+
+    # r.h.s.
+    extradim = prod(y.shape[1:])
+    rhs = cupy.empty((n + k - 1, extradim), dtype=float)
+    rhs[:(k - 1), :] = 0
+    rhs[(k - 1):, :] = (y.reshape(n, 0) if y.size == 0 else
+                        y.reshape((-1, extradim)))
+
+    # solve for the coefficients
+    coef = spsolve(matr_csr, rhs)
+    coef = cupy.ascontiguousarray(coef.reshape((n + k - 1,) + y.shape[1:]))
+    return BSpline.construct_fast(t, coef, k,
+                                  extrapolate='periodic', axis=axis)
diff --git a/cupyx/scipy/interpolate/_cubic.py b/cupyx/scipy/interpolate/_cubic.py
new file mode 100644
index 0000000..e839a1a
--- /dev/null
+++ b/cupyx/scipy/interpolate/_cubic.py
@@ -0,0 +1,414 @@
+
+import cupy
+from cupyx.scipy.interpolate._interpolate import PPoly
+
+
+def _isscalar(x):
+    """Check whether x is if a scalar type, or 0-dim"""
+    return cupy.isscalar(x) or hasattr(x, 'shape') and x.shape == ()
+
+
+def prepare_input(x, y, axis, dydx=None):
+    """Prepare input for cubic spline interpolators.
+    All data are converted to numpy arrays and checked for correctness.
+    Axes equal to `axis` of arrays `y` and `dydx` are moved to be the 0th
+    axis. The value of `axis` is converted to lie in
+    [0, number of dimensions of `y`).
+    """
+
+    x, y = map(cupy.asarray, (x, y))
+    if cupy.issubdtype(x.dtype, cupy.complexfloating):
+        raise ValueError("`x` must contain real values.")
+    x = x.astype(float)
+
+    if cupy.issubdtype(y.dtype, cupy.complexfloating):
+        dtype = complex
+    else:
+        dtype = float
+
+    if dydx is not None:
+        dydx = cupy.asarray(dydx)
+        if y.shape != dydx.shape:
+            raise ValueError("The shapes of `y` and `dydx` must be identical.")
+        if cupy.issubdtype(dydx.dtype, cupy.complexfloating):
+            dtype = complex
+        dydx = dydx.astype(dtype, copy=False)
+
+    y = y.astype(dtype, copy=False)
+    axis = axis % y.ndim
+    if x.ndim != 1:
+        raise ValueError("`x` must be 1-dimensional.")
+    if x.shape[0] < 2:
+        raise ValueError("`x` must contain at least 2 elements.")
+    if x.shape[0] != y.shape[axis]:
+        raise ValueError("The length of `y` along `axis`={0} doesn't "
+                         "match the length of `x`".format(axis))
+
+    if not cupy.all(cupy.isfinite(x)):
+        raise ValueError("`x` must contain only finite values.")
+    if not cupy.all(cupy.isfinite(y)):
+        raise ValueError("`y` must contain only finite values.")
+
+    if dydx is not None and not cupy.all(cupy.isfinite(dydx)):
+        raise ValueError("`dydx` must contain only finite values.")
+
+    dx = cupy.diff(x)
+    if cupy.any(dx <= 0):
+        raise ValueError("`x` must be strictly increasing sequence.")
+
+    y = cupy.moveaxis(y, axis, 0)
+    if dydx is not None:
+        dydx = cupy.moveaxis(dydx, axis, 0)
+
+    return x, dx, y, axis, dydx
+
+
+class CubicHermiteSpline(PPoly):
+    """Piecewise-cubic interpolator matching values and first derivatives.
+
+    The result is represented as a `PPoly` instance. [1]_
+
+    Parameters
+    ----------
+    x : array_like, shape (n,)
+        1-D array containing values of the independent variable.
+        Values must be real, finite and in strictly increasing order.
+    y : array_like
+        Array containing values of the dependent variable. It can have
+        arbitrary number of dimensions, but the length along ``axis``
+        (see below) must match the length of ``x``. Values must be finite.
+    dydx : array_like
+        Array containing derivatives of the dependent variable. It can have
+        arbitrary number of dimensions, but the length along ``axis``
+        (see below) must match the length of ``x``. Values must be finite.
+    axis : int, optional
+        Axis along which `y` is assumed to be varying. Meaning that for
+        ``x[i]`` the corresponding values are ``cupy.take(y, i, axis=axis)``.
+        Default is 0.
+    extrapolate : {bool, 'periodic', None}, optional
+        If bool, determines whether to extrapolate to out-of-bounds points
+        based on first and last intervals, or to return NaNs. If 'periodic',
+        periodic extrapolation is used. If None (default), it is set to True.
+
+    Attributes
+    ----------
+    x : ndarray, shape (n,)
+        Breakpoints. The same ``x`` which was passed to the constructor.
+    c : ndarray, shape (4, n-1, ...)
+        Coefficients of the polynomials on each segment. The trailing
+        dimensions match the dimensions of `y`, excluding ``axis``.
+        For example, if `y` is 1-D, then ``c[k, i]`` is a coefficient for
+        ``(x-x[i])**(3-k)`` on the segment between ``x[i]`` and ``x[i+1]``.
+    axis : int
+        Interpolation axis. The same axis which was passed to the
+        constructor.
+
+    See Also
+    --------
+    Akima1DInterpolator : Akima 1D interpolator.
+    PchipInterpolator : PCHIP 1-D monotonic cubic interpolator.
+    PPoly : Piecewise polynomial in terms of coefficients and breakpoints
+
+    Notes
+    -----
+    If you want to create a higher-order spline matching higher-order
+    derivatives, use `BPoly.from_derivatives`.
+
+    References
+    ----------
+    .. [1] `Cubic Hermite spline
+            <https://en.wikipedia.org/wiki/Cubic_Hermite_spline>`_
+            on Wikipedia.
+    """
+
+    def __init__(self, x, y, dydx, axis=0, extrapolate=None):
+        if extrapolate is None:
+            extrapolate = True
+
+        x, dx, y, axis, dydx = prepare_input(x, y, axis, dydx)
+
+        dxr = dx.reshape([dx.shape[0]] + [1] * (y.ndim - 1))
+        slope = cupy.diff(y, axis=0) / dxr
+        t = (dydx[:-1] + dydx[1:] - 2 * slope) / dxr
+
+        c = cupy.empty((4, len(x) - 1) + y.shape[1:], dtype=t.dtype)
+        c[0] = t / dxr
+        c[1] = (slope - dydx[:-1]) / dxr - t
+        c[2] = dydx[:-1]
+        c[3] = y[:-1]
+
+        super().__init__(c, x, extrapolate=extrapolate)
+        self.axis = axis
+
+
+class PchipInterpolator(CubicHermiteSpline):
+    r"""PCHIP 1-D monotonic cubic interpolation.
+
+    ``x`` and ``y`` are arrays of values used to approximate some function f,
+    with ``y = f(x)``. The interpolant uses monotonic cubic splines
+    to find the value of new points. (PCHIP stands for Piecewise Cubic
+    Hermite Interpolating Polynomial).
+
+    Parameters
+    ----------
+    x : ndarray
+        A 1-D array of monotonically increasing real values. ``x`` cannot
+        include duplicate values (otherwise f is overspecified)
+    y : ndarray
+        A 1-D array of real values. ``y``'s length along the interpolation
+        axis must be equal to the length of ``x``. If N-D array, use ``axis``
+        parameter to select correct axis.
+    axis : int, optional
+        Axis in the y array corresponding to the x-coordinate values.
+    extrapolate : bool, optional
+        Whether to extrapolate to out-of-bounds points based on first
+        and last intervals, or to return NaNs.
+
+    See Also
+    --------
+    CubicHermiteSpline : Piecewise-cubic interpolator.
+    Akima1DInterpolator : Akima 1D interpolator.
+    PPoly : Piecewise polynomial in terms of coefficients and breakpoints.
+
+    Notes
+    -----
+    The interpolator preserves monotonicity in the interpolation data and does
+    not overshoot if the data is not smooth.
+
+    The first derivatives are guaranteed to be continuous, but the second
+    derivatives may jump at :math:`x_k`.
+
+    Determines the derivatives at the points :math:`x_k`, :math:`f'_k`,
+    by using PCHIP algorithm [1]_.
+
+    Let :math:`h_k = x_{k+1} - x_k`, and  :math:`d_k = (y_{k+1} - y_k) / h_k`
+    are the slopes at internal points :math:`x_k`.
+    If the signs of :math:`d_k` and :math:`d_{k-1}` are different or either of
+    them equals zero, then :math:`f'_k = 0`. Otherwise, it is given by the
+    weighted harmonic mean
+
+    .. math::
+
+        \frac{w_1 + w_2}{f'_k} = \frac{w_1}{d_{k-1}} + \frac{w_2}{d_k}
+
+    where :math:`w_1 = 2 h_k + h_{k-1}` and :math:`w_2 = h_k + 2 h_{k-1}`.
+
+    The end slopes are set using a one-sided scheme [2]_.
+
+
+    References
+    ----------
+    .. [1] F. N. Fritsch and J. Butland,
+           A method for constructing local
+           monotone piecewise cubic interpolants,
+           SIAM J. Sci. Comput., 5(2), 300-304 (1984).
+           `10.1137/0905021 <https://doi.org/10.1137/0905021>`_.
+    .. [2] see, e.g., C. Moler, Numerical Computing with Matlab, 2004.
+           `10.1137/1.9780898717952 <https://doi.org/10.1137/1.9780898717952>`_
+    """
+
+    def __init__(self, x, y, axis=0, extrapolate=None):
+        x, _, y, axis, _ = prepare_input(x, y, axis)
+        xp = x.reshape((x.shape[0],) + (1,)*(y.ndim-1))
+        dk = self._find_derivatives(xp, y)
+        super().__init__(x, y, dk, axis=0, extrapolate=extrapolate)
+        self.axis = axis
+
+    @staticmethod
+    def _edge_case(h0, h1, m0, m1):
+        # one-sided three-point estimate for the derivative
+        d = ((2 * h0 + h1) * m0 - h0 * m1) / (h0 + h1)
+
+        # try to preserve shape
+        mask = cupy.sign(d) != cupy.sign(m0)
+        mask2 = (cupy.sign(m0) != cupy.sign(m1)) & (
+            cupy.abs(d) > 3.*cupy.abs(m0))
+        mmm = (~mask) & mask2
+
+        d[mask] = 0.
+        d[mmm] = 3.*m0[mmm]
+
+        return d
+
+    @staticmethod
+    def _find_derivatives(x, y):
+        # Determine the derivatives at the points y_k, d_k, by using
+        #  PCHIP algorithm is:
+        # We choose the derivatives at the point x_k by
+        # Let m_k be the slope of the kth segment (between k and k+1)
+        # If m_k=0 or m_{k-1}=0 or sgn(m_k) != sgn(m_{k-1}) then d_k == 0
+        # else use weighted harmonic mean:
+        #   w_1 = 2h_k + h_{k-1}, w_2 = h_k + 2h_{k-1}
+        #   1/d_k = 1/(w_1 + w_2)*(w_1 / m_k + w_2 / m_{k-1})
+        #   where h_k is the spacing between x_k and x_{k+1}
+        y_shape = y.shape
+        if y.ndim == 1:
+            # So that _edge_case doesn't end up assigning to scalars
+            x = x[:, None]
+            y = y[:, None]
+
+        hk = x[1:] - x[:-1]
+        mk = (y[1:] - y[:-1]) / hk
+
+        if y.shape[0] == 2:
+            # edge case: only have two points, use linear interpolation
+            dk = cupy.zeros_like(y)
+            dk[0] = mk
+            dk[1] = mk
+            return dk.reshape(y_shape)
+
+        smk = cupy.sign(mk)
+        condition = (smk[1:] != smk[:-1]) | (mk[1:] == 0) | (mk[:-1] == 0)
+
+        w1 = 2*hk[1:] + hk[:-1]
+        w2 = hk[1:] + 2*hk[:-1]
+
+        # values where division by zero occurs will be excluded
+        # by 'condition' afterwards
+        whmean = (w1 / mk[:-1] + w2 / mk[1:]) / (w1 + w2)
+
+        dk = cupy.zeros_like(y)
+        dk[1:-1] = cupy.where(condition, 0.0, 1.0 / whmean)
+
+        # special case endpoints, as suggested in
+        # Cleve Moler, Numerical Computing with MATLAB, Chap 3.6 (pchiptx.m)
+        dk[0] = PchipInterpolator._edge_case(hk[0], hk[1], mk[0], mk[1])
+        dk[-1] = PchipInterpolator._edge_case(hk[-1], hk[-2], mk[-1], mk[-2])
+
+        return dk.reshape(y_shape)
+
+
+def pchip_interpolate(xi, yi, x, der=0, axis=0):
+    """
+    Convenience function for pchip interpolation.
+
+    xi and yi are arrays of values used to approximate some function f,
+    with ``yi = f(xi)``. The interpolant uses monotonic cubic splines
+    to find the value of new points x and the derivatives there.
+    See `scipy.interpolate.PchipInterpolator` for details.
+
+    Parameters
+    ----------
+    xi : array_like
+        A sorted list of x-coordinates, of length N.
+    yi : array_like
+        A 1-D array of real values. `yi`'s length along the interpolation
+        axis must be equal to the length of `xi`. If N-D array, use axis
+        parameter to select correct axis.
+    x : scalar or array_like
+        Of length M.
+    der : int or list, optional
+        Derivatives to extract. The 0th derivative can be included to
+        return the function value.
+    axis : int, optional
+        Axis in the yi array corresponding to the x-coordinate values.
+
+    See Also
+    --------
+    PchipInterpolator : PCHIP 1-D monotonic cubic interpolator.
+
+    Returns
+    -------
+    y : scalar or array_like
+        The result, of length R or length M or M by R.
+    """
+    P = PchipInterpolator(xi, yi, axis=axis)
+
+    if der == 0:
+        return P(x)
+    elif _isscalar(der):
+        return P.derivative(der)(x)
+    else:
+        return [P.derivative(nu)(x) for nu in der]
+
+
+class Akima1DInterpolator(CubicHermiteSpline):
+    """
+    Akima interpolator
+
+    Fit piecewise cubic polynomials, given vectors x and y. The interpolation
+    method by Akima uses a continuously differentiable sub-spline built from
+    piecewise cubic polynomials. The resultant curve passes through the given
+    data points and will appear smooth and natural [1]_.
+
+    Parameters
+    ----------
+    x : ndarray, shape (m, )
+        1-D array of monotonically increasing real values.
+    y : ndarray, shape (m, ...)
+        N-D array of real values. The length of ``y`` along the first axis
+        must be equal to the length of ``x``.
+    axis : int, optional
+        Specifies the axis of ``y`` along which to interpolate. Interpolation
+        defaults to the first axis of ``y``.
+
+    See Also
+    --------
+    CubicHermiteSpline : Piecewise-cubic interpolator.
+    PchipInterpolator : PCHIP 1-D monotonic cubic interpolator.
+    PPoly : Piecewise polynomial in terms of coefficients and breakpoints
+
+    Notes
+    -----
+    Use only for precise data, as the fitted curve passes through the given
+    points exactly. This routine is useful for plotting a pleasingly smooth
+    curve through a few given points for purposes of plotting.
+
+    References
+    ----------
+    .. [1] A new method of interpolation and smooth curve fitting based
+        on local procedures. Hiroshi Akima, J. ACM, October 1970, 17(4),
+        589-602.
+    """
+
+    def __init__(self, x, y, axis=0):
+        # Original implementation in MATLAB by N. Shamsundar (BSD licensed)
+        # https://www.mathworks.com/matlabcentral/fileexchange/1814-akima-interpolation  # noqa: E501
+        x, dx, y, axis, _ = prepare_input(x, y, axis)
+
+        # determine slopes between breakpoints
+        m = cupy.empty((x.size + 3, ) + y.shape[1:])
+        dx = dx[(slice(None), ) + (None, ) * (y.ndim - 1)]
+        m[2:-2] = cupy.diff(y, axis=0) / dx
+
+        # add two additional points on the left ...
+        m[1] = 2. * m[2] - m[3]
+        m[0] = 2. * m[1] - m[2]
+        # ... and on the right
+        m[-2] = 2. * m[-3] - m[-4]
+        m[-1] = 2. * m[-2] - m[-3]
+
+        # if m1 == m2 != m3 == m4, the slope at the breakpoint is not
+        # defined. This is the fill value:
+        t = .5 * (m[3:] + m[:-3])
+        # get the denominator of the slope t
+        dm = cupy.abs(cupy.diff(m, axis=0))
+        f1 = dm[2:]
+        f2 = dm[:-2]
+        f12 = f1 + f2
+        # These are the mask of where the slope at breakpoint is defined:
+        max_value = -cupy.inf if y.size == 0 else cupy.max(f12)
+        ind = cupy.nonzero(f12 > 1e-9 * max_value)
+        x_ind, y_ind = ind[0], ind[1:]
+        # Set the slope at breakpoint
+        t[ind] = (f1[ind] * m[(x_ind + 1,) + y_ind] +
+                  f2[ind] * m[(x_ind + 2,) + y_ind]) / f12[ind]
+
+        super().__init__(x, y, t, axis=0, extrapolate=False)
+        self.axis = axis
+
+    def extend(self, c, x, right=True):
+        raise NotImplementedError("Extending a 1-D Akima interpolator is not "
+                                  "yet implemented")
+
+    # These are inherited from PPoly, but they do not produce an Akima
+    # interpolator. Hence stub them out.
+    @classmethod
+    def from_spline(cls, tck, extrapolate=None):
+        raise NotImplementedError("This method does not make sense for "
+                                  "an Akima interpolator.")
+
+    @classmethod
+    def from_bernstein_basis(cls, bp, extrapolate=None):
+        raise NotImplementedError("This method does not make sense for "
+                                  "an Akima interpolator.")
diff --git a/cupyx/scipy/interpolate/_interpolate.py b/cupyx/scipy/interpolate/_interpolate.py
new file mode 100644
index 0000000..4fe5640
--- /dev/null
+++ b/cupyx/scipy/interpolate/_interpolate.py
@@ -0,0 +1,1763 @@
+
+import math
+
+import cupy
+from cupy._core import internal  # NOQA
+from cupy._core._scalar import get_typename  # NOQA
+from cupy_backends.cuda.api import runtime
+from cupyx.scipy import special as spec
+from cupyx.scipy.interpolate._bspline import BSpline, _get_dtype
+
+import numpy as np
+
+try:
+    from math import comb
+    _comb = comb
+except ImportError:
+    def _comb(n, k):
+        return math.factorial(n) // (math.factorial(n - k) * math.factorial(k))
+
+
+TYPES = ['double', 'thrust::complex<double>']
+INT_TYPES = ['int', 'long long']
+
+INTERVAL_KERNEL = r'''
+#include <cupy/complex.cuh>
+
+#define le_or_ge(x, y, r) ((r) ? ((x) < (y)) : ((x) > (y)))
+#define ge_or_le(x, y, r) ((r) ? ((x) > (y)) : ((x) < (y)))
+#define geq_or_leq(x, y, r) ((r) ? ((x) >= (y)) : ((x) <= (y)))
+
+extern "C" {
+__global__ void find_breakpoint_position(
+        const double* breakpoints, const double* x, long long* out,
+        bool extrapolate, int total_x, int total_breakpoints,
+        const bool* pasc) {
+
+    int idx = blockDim.x * blockIdx.x + threadIdx.x;
+    if(idx >= total_x) {
+        return;
+    }
+
+    double xp = *&x[idx];
+    double a = *&breakpoints[0];
+    double b = *&breakpoints[total_breakpoints - 1];
+    bool asc = pasc[0];
+
+    if(isnan(xp)) {
+        out[idx] = -1;
+        return;
+    }
+
+    if(le_or_ge(xp, a, asc) || ge_or_le(xp, b, asc)) {
+        if(!extrapolate) {
+            out[idx] = -1;
+        } else if(le_or_ge(xp, a, asc)) {
+            out[idx] = 0;
+        } else if(ge_or_le(xp, b, asc)) {
+            out[idx] = total_breakpoints - 2;
+        }
+        return;
+    } else if (xp == b) {
+        out[idx] = total_breakpoints - 2;
+        return;
+    }
+
+    int left = 0;
+    int right = total_breakpoints - 2;
+    int mid;
+
+    if(le_or_ge(xp, *&breakpoints[left + 1], asc)) {
+        right = left;
+    }
+
+    bool found = false;
+
+    while(left < right && !found) {
+        mid = ((right + left) / 2);
+        if(le_or_ge(xp, *&breakpoints[mid], asc)) {
+            right = mid;
+        } else if (geq_or_leq(xp, *&breakpoints[mid + 1], asc)) {
+            left = mid + 1;
+        } else {
+            found = true;
+            left = mid;
+        }
+    }
+
+    out[idx] = left;
+}
+}
+'''
+
+INTERVAL_MODULE = cupy.RawModule(
+    code=INTERVAL_KERNEL, options=('-std=c++11',),)
+
+if runtime.is_hip:
+    BASE_HEADERS = """#include <hip/hip_runtime.h>
+"""
+else:
+    BASE_HEADERS = """#include <cuda_runtime.h>
+#include <device_launch_parameters.h>
+"""
+
+PPOLY_KERNEL = BASE_HEADERS + r"""
+#include <cupy/complex.cuh>
+#include <cupy/math_constants.h>
+
+template<typename T>
+__device__ T eval_poly_1(
+        const double s, const T* coef, long long ci, int cj, int dx,
+        const long long* c_dims, const long long stride_0,
+        const long long stride_1) {
+    int kp, k;
+    T res, z;
+    double prefactor;
+
+    res = 0.0;
+    z = 1.0;
+
+    if(dx < 0) {
+        for(int i = 0; i < -dx; i++) {
+            z *= s;
+        }
+    }
+
+    int c_dim_0 = (int) *&c_dims[0];
+
+    for(kp = 0; kp < c_dim_0; kp++) {
+        if(dx == 0) {
+            prefactor = 1.0;
+        } else if(dx > 0) {
+            if(kp < dx) {
+                continue;
+            } else {
+                prefactor = 1.0;
+                for(k = kp; k > kp - dx; k--) {
+                    prefactor *= k;
+                }
+            }
+        } else {
+            prefactor = 1.0;
+            for(k = kp; k < kp - dx; k++) {
+                prefactor /= k + 1;
+            }
+        }
+
+        int off = stride_0 * (c_dim_0 - kp - 1) + stride_1 * ci + cj;
+        T cur_coef = *&coef[off];
+        res += cur_coef * z * ((T) prefactor);
+
+        if((kp < c_dim_0 - 1) && kp >= dx) {
+            z *= s;
+        }
+
+    }
+
+    return res;
+
+}
+
+template<typename T>
+__global__ void eval_ppoly(
+        const T* coef, const double* breakpoints, const double* x,
+        const long long* intervals, int dx, const long long* c_dims,
+        const long long* c_strides, int num_x, T* out) {
+
+    int idx = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if(idx >= num_x) {
+        return;
+    }
+
+    double xp = *&x[idx];
+    long long interval = *&intervals[idx];
+    double breakpoint = *&breakpoints[interval];
+
+    const int num_c = *&c_dims[2];
+    const long long stride_0 = *&c_strides[0];
+    const long long stride_1 = *&c_strides[1];
+
+    if(interval < 0) {
+        for(int j = 0; j < num_c; j++) {
+            out[num_c * idx + j] = CUDART_NAN;
+        }
+        return;
+    }
+
+    for(int j = 0; j < num_c; j++) {
+        T res = eval_poly_1<T>(
+            xp - breakpoint, coef, interval, ((long long) (j)), dx,
+            c_dims, stride_0, stride_1);
+        out[num_c * idx + j] = res;
+    }
+}
+
+template<typename T>
+__global__ void fix_continuity(
+        T* coef, const double* breakpoints, const int order,
+        const long long* c_dims, const long long* c_strides,
+        int num_breakpoints) {
+
+    const long long c_size0 = *&c_dims[0];
+    const long long c_size2 = *&c_dims[2];
+    const long long stride_0 = *&c_strides[0];
+    const long long stride_1 = *&c_strides[1];
+    const long long stride_2 = *&c_strides[2];
+
+    for(int idx = 1; idx < num_breakpoints - 1; idx++) {
+        const double breakpoint = *&breakpoints[idx];
+        const long long interval = idx - 1;
+        const double breakpoint_interval = *&breakpoints[interval];
+
+        for(int jp = 0; jp < c_size2; jp++) {
+            for(int dx = order; dx > -1; dx--) {
+                T res = eval_poly_1<T>(
+                    breakpoint - breakpoint_interval, coef,
+                    interval, jp, dx, c_dims, stride_0, stride_1);
+
+                for(int kp = 0; kp < dx; kp++) {
+                    res /= kp + 1;
+                }
+
+                const long long c_idx = (
+                    stride_0 * (c_size0 - dx - 1) + stride_1 * idx +
+                    stride_2 * jp);
+
+                coef[c_idx] = res;
+            }
+        }
+    }
+}
+
+template<typename T>
+__global__ void integrate(
+        const T* coef, const double* breakpoints,
+        const double* a_val, const double* b_val,
+        const long long* start, const long long* end,
+        const long long* c_dims, const long long* c_strides,
+        const bool* pasc, T* out) {
+
+    int idx = blockDim.x * blockIdx.x + threadIdx.x;
+    const long long c_dim2 = *&c_dims[2];
+
+    if(idx >= c_dim2) {
+        return;
+    }
+
+    const bool asc = pasc[0];
+    const long long start_interval = asc ? *&start[0] : *&end[0];
+    const long long end_interval = asc ? *&end[0] : *&start[0];
+    const double a = asc ? *&a_val[0] : *&b_val[0];
+    const double b = asc ? *&b_val[0] : *&a_val[0];
+
+    const long long stride_0 = *&c_strides[0];
+    const long long stride_1 = *&c_strides[1];
+
+    if(start_interval < 0 || end_interval < 0) {
+        out[idx] = CUDART_NAN;
+        return;
+    }
+
+    T vtot = 0;
+    T vb;
+    T va;
+    for(int interval = start_interval; interval <= end_interval; interval++) {
+        const double breakpoint = *&breakpoints[interval];
+        if(interval == end_interval) {
+            vb = eval_poly_1<T>(
+                b - breakpoint, coef, interval, idx, -1, c_dims,
+                stride_0, stride_1);
+        } else {
+            const double next_breakpoint = *&breakpoints[interval + 1];
+            vb = eval_poly_1<T>(
+                next_breakpoint - breakpoint, coef, interval,
+                idx, -1, c_dims, stride_0, stride_1);
+        }
+
+        if(interval == start_interval) {
+            va = eval_poly_1<T>(
+                a - breakpoint, coef, interval, idx, -1, c_dims,
+                stride_0, stride_1);
+        } else {
+            va = eval_poly_1<T>(
+                0, coef, interval, idx, -1, c_dims,
+                stride_0, stride_1);
+        }
+
+        vtot += (vb - va);
+    }
+
+    if(!asc) {
+        vtot = -vtot;
+    }
+
+    out[idx] = vtot;
+
+}
+"""
+
+PPOLY_MODULE = cupy.RawModule(
+    code=PPOLY_KERNEL, options=('-std=c++11',),
+    name_expressions=(
+        [f'eval_ppoly<{type_name}>' for type_name in TYPES] +
+        [f'fix_continuity<{type_name}>' for type_name in TYPES] +
+        [f'integrate<{type_name}>' for type_name in TYPES]))
+
+BPOLY_KERNEL = BASE_HEADERS + r"""
+#include <cupy/complex.cuh>
+#include <cupy/math_constants.h>
+
+template<typename T>
+__device__ T eval_bpoly1(
+        const double s, const T* coef, const long long ci, const long long cj,
+        const long long c_dims_0, const long long c_strides_0,
+        const long long c_strides_1) {
+
+    const long long k = c_dims_0 - 1;
+    const double s1 = 1 - s;
+    T res;
+
+    const long long i0 = 0 * c_strides_0 + ci * c_strides_1 + cj;
+    const long long i1 = 1 * c_strides_0 + ci * c_strides_1 + cj;
+    const long long i2 = 2 * c_strides_0 + ci * c_strides_1 + cj;
+    const long long i3 = 3 * c_strides_0 + ci * c_strides_1 + cj;
+
+    if(k == 0) {
+        res = coef[i0];
+    } else if(k == 1) {
+        res = coef[i0] * s1 + coef[i1] * s;
+    } else if(k == 2) {
+        res = coef[i0] * s1 * s1 + coef[i1] * 2.0 * s1 * s + coef[i2] * s * s;
+    } else if(k == 3) {
+        res = (coef[i0] * s1 * s1 * s1 + coef[i1] * 3.0 * s1 * s1 * s +
+               coef[i2] * 3.0 * s1 * s * s + coef[i3] * s * s * s);
+    } else {
+        T comb = 1;
+        res = 0;
+        for(int j = 0; j < k + 1; j++) {
+            const long long idx = j * c_strides_0 + ci * c_strides_1 + cj;
+            res += (comb * pow(s, ((double) j)) * pow(s1, ((double) k) - j) *
+                    coef[idx]);
+            comb *= 1.0 * (k - j) / (j + 1.0);
+        }
+    }
+
+    return res;
+}
+
+template<typename T>
+__device__ T eval_bpoly1_deriv(
+        const double s, const T* coef, const long long ci, const long long cj,
+        int dx, T* wrk, const long long c_dims_0, const long long c_strides_0,
+        const long long c_strides_1, const long long wrk_dims_0,
+        const long long wrk_strides_0, const long long wrk_strides_1) {
+
+    T res, term;
+    double comb, poch;
+
+    const long long k = c_dims_0 - 1;
+
+    if(dx == 0) {
+        res = eval_bpoly1<T>(s, coef, ci, cj, c_dims_0, c_strides_0,
+                             c_strides_1);
+    } else {
+        poch = 1.0;
+        for(int a = 0; a < dx; a++) {
+            poch *= k - a;
+        }
+
+        term = 0;
+        for(int a = 0; a < k - dx + 1; a++) {
+            term = 0;
+            comb = 1;
+            for(int j = 0; j < dx + 1; j++) {
+                const long long idx = (c_strides_0 * (j + a) +
+                                       c_strides_1 * ci + cj);
+                term += coef[idx] * pow(-1.0, ((double) (j + dx))) * comb;
+                comb *= 1.0 * (dx - j) / (j + 1);
+            }
+            wrk[a] = term * poch;
+        }
+
+        res = eval_bpoly1<T>(s, wrk, 0, 0, wrk_dims_0, wrk_strides_0,
+                             wrk_strides_1);
+    }
+    return res;
+}
+
+template<typename T>
+__global__ void eval_bpoly(
+        const T* coef, const double* breakpoints, const double* x,
+        const long long* intervals, int dx, T* wrk, const long long* c_dims,
+        const long long* c_strides, const long long* wrk_dims,
+        const long long* wrk_strides, int num_x, T* out) {
+
+    int idx = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if(idx >= num_x) {
+        return;
+    }
+
+    double xp = *&x[idx];
+    long long interval = *&intervals[idx];
+    const int num_c = *&c_dims[2];
+
+    const long long c_dims_0 = *&c_dims[0];
+    const long long c_strides_0 = *&c_strides[0];
+    const long long c_strides_1 = *&c_strides[1];
+
+    const long long wrk_dims_0 = *&wrk_dims[0];
+    const long long wrk_strides_0 = *&wrk_strides[0];
+    const long long wrk_strides_1 = *&wrk_strides[1];
+
+    if(interval < 0) {
+        for(int j = 0; j < num_c; j++) {
+            out[num_c * idx + j] = CUDART_NAN;
+        }
+        return;
+    }
+
+    const double ds = breakpoints[interval + 1] - breakpoints[interval];
+    const double ds_dx = pow(ds, ((double) dx));
+    T* off_wrk = wrk + idx * (c_dims_0 - dx);
+
+    for(int j = 0; j < num_c; j++) {
+        T res;
+        const double s = (xp - breakpoints[interval]) / ds;
+        if(dx == 0) {
+            res = eval_bpoly1<T>(
+                s, coef, interval, ((long long) (j)), c_dims_0, c_strides_0,
+                c_strides_1);
+        } else {
+            res = eval_bpoly1_deriv<T>(
+                s, coef, interval, ((long long) (j)), dx,
+                off_wrk, c_dims_0, c_strides_0, c_strides_1,
+                wrk_dims_0, wrk_strides_0, wrk_strides_1) / ds_dx;
+        }
+        out[num_c * idx + j] = res;
+    }
+
+}
+"""
+
+BPOLY_MODULE = cupy.RawModule(
+    code=BPOLY_KERNEL, options=('-std=c++11',),
+    name_expressions=(
+        [f'eval_bpoly<{type_name}>' for type_name in TYPES]))
+
+
+def _get_module_func(module, func_name, *template_args):
+    def _get_typename(dtype):
+        typename = get_typename(dtype)
+        if dtype.kind == 'c':
+            typename = 'thrust::' + typename
+        return typename
+    args_dtypes = [_get_typename(arg.dtype) for arg in template_args]
+    template = ', '.join(args_dtypes)
+    kernel_name = f'{func_name}<{template}>' if template_args else func_name
+    kernel = module.get_function(kernel_name)
+    return kernel
+
+
+def _ppoly_evaluate(c, x, xp, dx, extrapolate, out):
+    """
+    Evaluate a piecewise polynomial.
+
+    Parameters
+    ----------
+    c : ndarray, shape (k, m, n)
+        Coefficients local polynomials of order `k-1` in `m` intervals.
+        There are `n` polynomials in each interval.
+        Coefficient of highest order-term comes first.
+    x : ndarray, shape (m+1,)
+        Breakpoints of polynomials.
+    xp : ndarray, shape (r,)
+        Points to evaluate the piecewise polynomial at.
+    dx : int
+        Order of derivative to evaluate.  The derivative is evaluated
+        piecewise and may have discontinuities.
+    extrapolate : bool
+        Whether to extrapolate to out-of-bounds points based on first
+        and last intervals, or to return NaNs.
+    out : ndarray, shape (r, n)
+        Value of each polynomial at each of the input points.
+        This argument is modified in-place.
+    """
+    # Determine if the breakpoints are in ascending order or descending one
+    ascending = x[-1] >= x[0]
+
+    intervals = cupy.empty(xp.shape, dtype=cupy.int64)
+    interval_kernel = INTERVAL_MODULE.get_function('find_breakpoint_position')
+    interval_kernel(((xp.shape[0] + 128 - 1) // 128,), (128,),
+                    (x, xp, intervals, extrapolate, xp.shape[0], x.shape[0],
+                     ascending))
+
+    # Compute coefficient displacement stride (in elements)
+    c_shape = cupy.asarray(c.shape, dtype=cupy.int64)
+    c_strides = cupy.asarray(c.strides, dtype=cupy.int64) // c.itemsize
+
+    ppoly_kernel = _get_module_func(PPOLY_MODULE, 'eval_ppoly', c)
+    ppoly_kernel(((xp.shape[0] + 128 - 1) // 128,), (128,),
+                 (c, x, xp, intervals, dx, c_shape, c_strides,
+                  xp.shape[0], out))
+
+
+def _fix_continuity(c, x, order):
+    """
+    Make a piecewise polynomial continuously differentiable to given order.
+
+    Parameters
+    ----------
+    c : ndarray, shape (k, m, n)
+        Coefficients local polynomials of order `k-1` in `m` intervals.
+        There are `n` polynomials in each interval.
+        Coefficient of highest order-term comes first.
+
+        Coefficients c[-order-1:] are modified in-place.
+    x : ndarray, shape (m+1,)
+        Breakpoints of polynomials
+    order : int
+        Order up to which enforce piecewise differentiability.
+    """
+    # Compute coefficient displacement stride (in elements)
+    c_shape = cupy.asarray(c.shape, dtype=cupy.int64)
+    c_strides = cupy.asarray(c.strides, dtype=cupy.int64) // c.itemsize
+
+    continuity_kernel = _get_module_func(PPOLY_MODULE, 'fix_continuity', c)
+    continuity_kernel((1,), (1,),
+                      (c, x, order, c_shape, c_strides, x.shape[0]))
+
+
+def _integrate(c, x, a, b, extrapolate, out):
+    """
+    Compute integral over a piecewise polynomial.
+
+    Parameters
+    ----------
+    c : ndarray, shape (k, m, n)
+        Coefficients local polynomials of order `k-1` in `m` intervals.
+    x : ndarray, shape (m+1,)
+        Breakpoints of polynomials
+    a : double
+        Start point of integration.
+    b : double
+        End point of integration.
+    extrapolate : bool, optional
+        Whether to extrapolate to out-of-bounds points based on first
+        and last intervals, or to return NaNs.
+    out : ndarray, shape (n,)
+        Integral of the piecewise polynomial, assuming the polynomial
+        is zero outside the range (x[0], x[-1]).
+        This argument is modified in-place.
+    """
+    # Determine if the breakpoints are in ascending order or descending one
+    ascending = x[-1] >= x[0]
+
+    a = cupy.asarray([a], dtype=cupy.float64)
+    b = cupy.asarray([b], dtype=cupy.float64)
+
+    start_interval = cupy.empty(a.shape, dtype=cupy.int64)
+    end_interval = cupy.empty(b.shape, dtype=cupy.int64)
+
+    interval_kernel = INTERVAL_MODULE.get_function('find_breakpoint_position')
+    interval_kernel(((a.shape[0] + 128 - 1) // 128,), (128,),
+                    (x, a, start_interval, extrapolate, a.shape[0], x.shape[0],
+                     ascending))
+    interval_kernel(((b.shape[0] + 128 - 1) // 128,), (128,),
+                    (x, b, end_interval, extrapolate, b.shape[0], x.shape[0],
+                     ascending))
+
+    # Compute coefficient displacement stride (in elements)
+    c_shape = cupy.asarray(c.shape, dtype=cupy.int64)
+    c_strides = cupy.asarray(c.strides, dtype=cupy.int64) // c.itemsize
+
+    int_kernel = _get_module_func(PPOLY_MODULE, 'integrate', c)
+    int_kernel(((c.shape[2] + 128 - 1) // 128,), (128,),
+               (c, x, a, b, start_interval, end_interval, c_shape, c_strides,
+                ascending, out))
+
+
+def _bpoly_evaluate(c, x, xp, dx, extrapolate, out):
+    """
+    Evaluate a Bernstein polynomial.
+
+    Parameters
+    ----------
+    c : ndarray, shape (k, m, n)
+        Coefficients local polynomials of order `k-1` in `m` intervals.
+        There are `n` polynomials in each interval.
+        Coefficient of highest order-term comes first.
+    x : ndarray, shape (m+1,)
+        Breakpoints of polynomials.
+    xp : ndarray, shape (r,)
+        Points to evaluate the piecewise polynomial at.
+    dx : int
+        Order of derivative to evaluate.  The derivative is evaluated
+        piecewise and may have discontinuities.
+    extrapolate : bool
+        Whether to extrapolate to out-of-bounds points based on first
+        and last intervals, or to return NaNs.
+    out : ndarray, shape (r, n)
+        Value of each polynomial at each of the input points.
+        This argument is modified in-place.
+    """
+    # Determine if the breakpoints are in ascending order or descending one
+    ascending = x[-1] >= x[0]
+
+    intervals = cupy.empty(xp.shape, dtype=cupy.int64)
+    interval_kernel = INTERVAL_MODULE.get_function('find_breakpoint_position')
+    interval_kernel(((xp.shape[0] + 128 - 1) // 128,), (128,),
+                    (x, xp, intervals, extrapolate, xp.shape[0], x.shape[0],
+                     ascending))
+
+    # Compute coefficient displacement stride (in elements)
+    c_shape = cupy.asarray(c.shape, dtype=cupy.int64)
+    c_strides = cupy.asarray(c.strides, dtype=cupy.int64) // c.itemsize
+
+    wrk = cupy.empty((xp.shape[0] * (c.shape[0] - dx), 1, 1),
+                     dtype=_get_dtype(c))
+    wrk_shape = cupy.asarray([c.shape[0] - dx, 1, 1], dtype=cupy.int64)
+    wrk_strides = cupy.asarray(wrk.strides, dtype=cupy.int64) // wrk.itemsize
+
+    bpoly_kernel = _get_module_func(BPOLY_MODULE, 'eval_bpoly', c)
+    bpoly_kernel(((xp.shape[0] + 128 - 1) // 128,), (128,),
+                 (c, x, xp, intervals, dx, wrk, c_shape, c_strides, wrk_shape,
+                  wrk_strides, xp.shape[0], out))
+
+
+class _PPolyBase:
+    """Base class for piecewise polynomials."""
+    __slots__ = ('c', 'x', 'extrapolate', 'axis')
+
+    def __init__(self, c, x, extrapolate=None, axis=0):
+        self.c = cupy.asarray(c)
+        self.x = cupy.ascontiguousarray(x, dtype=cupy.float64)
+
+        if extrapolate is None:
+            extrapolate = True
+        elif extrapolate != 'periodic':
+            extrapolate = bool(extrapolate)
+        self.extrapolate = extrapolate
+
+        if self.c.ndim < 2:
+            raise ValueError("Coefficients array must be at least "
+                             "2-dimensional.")
+
+        if not (0 <= axis < self.c.ndim - 1):
+            raise ValueError("axis=%s must be between 0 and %s" %
+                             (axis, self.c.ndim-1))
+
+        self.axis = axis
+        if axis != 0:
+            # move the interpolation axis to be the first one in self.c
+            # More specifically, the target shape for self.c is (k, m, ...),
+            # and axis !=0 means that we have c.shape (..., k, m, ...)
+            #                                               ^
+            #                                              axis
+            # So we roll two of them.
+            self.c = cupy.moveaxis(self.c, axis+1, 0)
+            self.c = cupy.moveaxis(self.c, axis+1, 0)
+
+        if self.x.ndim != 1:
+            raise ValueError("x must be 1-dimensional")
+        if self.x.size < 2:
+            raise ValueError("at least 2 breakpoints are needed")
+        if self.c.ndim < 2:
+            raise ValueError("c must have at least 2 dimensions")
+        if self.c.shape[0] == 0:
+            raise ValueError("polynomial must be at least of order 0")
+        if self.c.shape[1] != self.x.size-1:
+            raise ValueError("number of coefficients != len(x)-1")
+        dx = cupy.diff(self.x)
+        if not (cupy.all(dx >= 0) or cupy.all(dx <= 0)):
+            raise ValueError("`x` must be strictly increasing or decreasing.")
+
+        dtype = self._get_dtype(self.c.dtype)
+        self.c = cupy.ascontiguousarray(self.c, dtype=dtype)
+
+    def _get_dtype(self, dtype):
+        if (cupy.issubdtype(dtype, cupy.complexfloating)
+                or cupy.issubdtype(self.c.dtype, cupy.complexfloating)):
+            return cupy.complex_
+        else:
+            return cupy.float_
+
+    @classmethod
+    def construct_fast(cls, c, x, extrapolate=None, axis=0):
+        """
+        Construct the piecewise polynomial without making checks.
+        Takes the same parameters as the constructor. Input arguments
+        ``c`` and ``x`` must be arrays of the correct shape and type. The
+        ``c`` array can only be of dtypes float and complex, and ``x``
+        array must have dtype float.
+        """
+        self = object.__new__(cls)
+        self.c = c
+        self.x = x
+        self.axis = axis
+        if extrapolate is None:
+            extrapolate = True
+        self.extrapolate = extrapolate
+        return self
+
+    def _ensure_c_contiguous(self):
+        """
+        c and x may be modified by the user. The Cython code expects
+        that they are C contiguous.
+        """
+        if not self.x.flags.c_contiguous:
+            self.x = self.x.copy()
+        if not self.c.flags.c_contiguous:
+            self.c = self.c.copy()
+
+    def extend(self, c, x):
+        """
+        Add additional breakpoints and coefficients to the polynomial.
+
+        Parameters
+        ----------
+        c : ndarray, size (k, m, ...)
+            Additional coefficients for polynomials in intervals. Note that
+            the first additional interval will be formed using one of the
+            ``self.x`` end points.
+        x : ndarray, size (m,)
+            Additional breakpoints. Must be sorted in the same order as
+            ``self.x`` and either to the right or to the left of the current
+            breakpoints.
+        """
+
+        c = cupy.asarray(c)
+        x = cupy.asarray(x)
+
+        if c.ndim < 2:
+            raise ValueError("invalid dimensions for c")
+        if x.ndim != 1:
+            raise ValueError("invalid dimensions for x")
+        if x.shape[0] != c.shape[1]:
+            raise ValueError("Shapes of x {} and c {} are incompatible"
+                             .format(x.shape, c.shape))
+        if c.shape[2:] != self.c.shape[2:] or c.ndim != self.c.ndim:
+            raise ValueError("Shapes of c {} and self.c {} are incompatible"
+                             .format(c.shape, self.c.shape))
+
+        if c.size == 0:
+            return
+
+        dx = cupy.diff(x)
+        if not (cupy.all(dx >= 0) or cupy.all(dx <= 0)):
+            raise ValueError("`x` is not sorted.")
+
+        if self.x[-1] >= self.x[0]:
+            if not x[-1] >= x[0]:
+                raise ValueError("`x` is in the different order "
+                                 "than `self.x`.")
+
+            if x[0] >= self.x[-1]:
+                action = 'append'
+            elif x[-1] <= self.x[0]:
+                action = 'prepend'
+            else:
+                raise ValueError("`x` is neither on the left or on the right "
+                                 "from `self.x`.")
+        else:
+            if not x[-1] <= x[0]:
+                raise ValueError("`x` is in the different order "
+                                 "than `self.x`.")
+
+            if x[0] <= self.x[-1]:
+                action = 'append'
+            elif x[-1] >= self.x[0]:
+                action = 'prepend'
+            else:
+                raise ValueError("`x` is neither on the left or on the right "
+                                 "from `self.x`.")
+
+        dtype = self._get_dtype(c.dtype)
+
+        k2 = max(c.shape[0], self.c.shape[0])
+        c2 = cupy.zeros(
+            (k2, self.c.shape[1] + c.shape[1]) + self.c.shape[2:],
+            dtype=dtype)
+
+        if action == 'append':
+            c2[k2 - self.c.shape[0]:, :self.c.shape[1]] = self.c
+            c2[k2 - c.shape[0]:, self.c.shape[1]:] = c
+            self.x = cupy.r_[self.x, x]
+        elif action == 'prepend':
+            c2[k2 - self.c.shape[0]:, :c.shape[1]] = c
+            c2[k2 - c.shape[0]:, c.shape[1]:] = self.c
+            self.x = cupy.r_[x, self.x]
+
+        self.c = c2
+
+    def __call__(self, x, nu=0, extrapolate=None):
+        """
+        Evaluate the piecewise polynomial or its derivative.
+
+        Parameters
+        ----------
+        x : array_like
+            Points to evaluate the interpolant at.
+        nu : int, optional
+            Order of derivative to evaluate. Must be non-negative.
+        extrapolate : {bool, 'periodic', None}, optional
+            If bool, determines whether to extrapolate to out-of-bounds points
+            based on first and last intervals, or to return NaNs.
+            If 'periodic', periodic extrapolation is used.
+            If None (default), use `self.extrapolate`.
+
+        Returns
+        -------
+        y : array_like
+            Interpolated values. Shape is determined by replacing
+            the interpolation axis in the original array with the shape of x.
+
+        Notes
+        -----
+        Derivatives are evaluated piecewise for each polynomial
+        segment, even if the polynomial is not differentiable at the
+        breakpoints. The polynomial intervals are considered half-open,
+        ``[a, b)``, except for the last interval which is closed
+        ``[a, b]``.
+        """
+        if extrapolate is None:
+            extrapolate = self.extrapolate
+        x = cupy.asarray(x)
+        x_shape, x_ndim = x.shape, x.ndim
+        x = cupy.ascontiguousarray(x.ravel(), dtype=cupy.float_)
+
+        # With periodic extrapolation we map x to the segment
+        # [self.x[0], self.x[-1]].
+        if extrapolate == 'periodic':
+            x = self.x[0] + (x - self.x[0]) % (self.x[-1] - self.x[0])
+            extrapolate = False
+
+        out = cupy.empty((len(x), int(np.prod(self.c.shape[2:]))),
+                         dtype=self.c.dtype)
+        self._ensure_c_contiguous()
+        self._evaluate(x, nu, extrapolate, out)
+        out = out.reshape(x_shape + self.c.shape[2:])
+        if self.axis != 0:
+            # transpose to move the calculated values to the interpolation axis
+            dims = list(range(out.ndim))
+            dims = (dims[x_ndim:x_ndim + self.axis] + dims[:x_ndim] +
+                    dims[x_ndim + self.axis:])
+            out = out.transpose(dims)
+        return out
+
+
+class PPoly(_PPolyBase):
+    """
+    Piecewise polynomial in terms of coefficients and breakpoints
+    The polynomial between ``x[i]`` and ``x[i + 1]`` is written in the
+    local power basis::
+
+        S = sum(c[m, i] * (xp - x[i]) ** (k - m) for m in range(k + 1))
+
+    where ``k`` is the degree of the polynomial.
+
+    Parameters
+    ----------
+    c : ndarray, shape (k, m, ...)
+        Polynomial coefficients, order `k` and `m` intervals.
+    x : ndarray, shape (m+1,)
+        Polynomial breakpoints. Must be sorted in either increasing or
+        decreasing order.
+    extrapolate : bool or 'periodic', optional
+        If bool, determines whether to extrapolate to out-of-bounds points
+        based on first and last intervals, or to return NaNs. If 'periodic',
+        periodic extrapolation is used. Default is True.
+    axis : int, optional
+        Interpolation axis. Default is zero.
+
+    Attributes
+    ----------
+    x : ndarray
+        Breakpoints.
+    c : ndarray
+        Coefficients of the polynomials. They are reshaped
+        to a 3-D array with the last dimension representing
+        the trailing dimensions of the original coefficient array.
+    axis : int
+        Interpolation axis.
+
+    See also
+    --------
+    BPoly : piecewise polynomials in the Bernstein basis
+
+    Notes
+    -----
+    High-order polynomials in the power basis can be numerically
+    unstable. Precision problems can start to appear for orders
+    larger than 20-30.
+
+    .. seealso:: :class:`scipy.interpolate.BSpline`
+    """
+
+    def _evaluate(self, x, nu, extrapolate, out):
+        _ppoly_evaluate(self.c.reshape(self.c.shape[0], self.c.shape[1], -1),
+                        self.x, x, nu, bool(extrapolate), out)
+
+    def derivative(self, nu=1):
+        """
+        Construct a new piecewise polynomial representing the derivative.
+
+        Parameters
+        ----------
+        nu : int, optional
+            Order of derivative to evaluate. Default is 1, i.e., compute the
+            first derivative. If negative, the antiderivative is returned.
+
+        Returns
+        -------
+        pp : PPoly
+            Piecewise polynomial of order k2 = k - n representing the
+            derivative of this polynomial.
+
+        Notes
+        -----
+        Derivatives are evaluated piecewise for each polynomial
+        segment, even if the polynomial is not differentiable at the
+        breakpoints. The polynomial intervals are considered half-open,
+        ``[a, b)``, except for the last interval which is closed
+        ``[a, b]``.
+        """
+        if nu < 0:
+            return self.antiderivative(-nu)
+
+        # reduce order
+        if nu == 0:
+            c2 = self.c.copy()
+        else:
+            c2 = self.c[:-nu, :].copy()
+
+        if c2.shape[0] == 0:
+            # derivative of order 0 is zero
+            c2 = cupy.zeros((1,) + c2.shape[1:], dtype=c2.dtype)
+
+        # multiply by the correct rising factorials
+        factor = spec.poch(cupy.arange(c2.shape[0], 0, -1), nu)
+        c2 *= factor[(slice(None),) + (None,)*(c2.ndim-1)]
+
+        # construct a compatible polynomial
+        return self.construct_fast(c2, self.x, self.extrapolate, self.axis)
+
+    def antiderivative(self, nu=1):
+        """
+        Construct a new piecewise polynomial representing the antiderivative.
+        Antiderivative is also the indefinite integral of the function,
+        and derivative is its inverse operation.
+
+        Parameters
+        ----------
+        nu : int, optional
+            Order of antiderivative to evaluate. Default is 1, i.e., compute
+            the first integral. If negative, the derivative is returned.
+
+        Returns
+        -------
+        pp : PPoly
+            Piecewise polynomial of order k2 = k + n representing
+            the antiderivative of this polynomial.
+
+        Notes
+        -----
+        The antiderivative returned by this function is continuous and
+        continuously differentiable to order n-1, up to floating point
+        rounding error.
+
+        If antiderivative is computed and ``self.extrapolate='periodic'``,
+        it will be set to False for the returned instance. This is done because
+        the antiderivative is no longer periodic and its correct evaluation
+        outside of the initially given x interval is difficult.
+        """
+        if nu <= 0:
+            return self.derivative(-nu)
+
+        c = cupy.zeros(
+            (self.c.shape[0] + nu, self.c.shape[1]) + self.c.shape[2:],
+            dtype=self.c.dtype)
+        c[:-nu] = self.c
+
+        # divide by the correct rising factorials
+        factor = spec.poch(cupy.arange(self.c.shape[0], 0, -1), nu)
+        c[:-nu] /= factor[(slice(None),) + (None,) * (c.ndim-1)]
+
+        # fix continuity of added degrees of freedom
+        self._ensure_c_contiguous()
+        _fix_continuity(c.reshape(c.shape[0], c.shape[1], -1),
+                        self.x, nu - 1)
+
+        if self.extrapolate == 'periodic':
+            extrapolate = False
+        else:
+            extrapolate = self.extrapolate
+
+        # construct a compatible polynomial
+        return self.construct_fast(c, self.x, extrapolate, self.axis)
+
+    def integrate(self, a, b, extrapolate=None):
+        """
+        Compute a definite integral over a piecewise polynomial.
+
+        Parameters
+        ----------
+        a : float
+            Lower integration bound
+        b : float
+            Upper integration bound
+        extrapolate : {bool, 'periodic', None}, optional
+            If bool, determines whether to extrapolate to out-of-bounds points
+            based on first and last intervals, or to return NaNs.
+            If 'periodic', periodic extrapolation is used.
+            If None (default), use `self.extrapolate`.
+
+        Returns
+        -------
+        ig : array_like
+            Definite integral of the piecewise polynomial over [a, b]
+        """
+        if extrapolate is None:
+            extrapolate = self.extrapolate
+
+        # Swap integration bounds if needed
+        sign = 1
+        if b < a:
+            a, b = b, a
+            sign = -1
+
+        range_int = cupy.empty(
+            (int(np.prod(self.c.shape[2:])),), dtype=self.c.dtype)
+        self._ensure_c_contiguous()
+
+        # Compute the integral.
+        if extrapolate == 'periodic':
+            # Split the integral into the part over period (can be several
+            # of them) and the remaining part.
+
+            xs, xe = self.x[0], self.x[-1]
+            period = xe - xs
+            interval = b - a
+            n_periods, left = divmod(interval, period)
+
+            if n_periods > 0:
+                _integrate(
+                    self.c.reshape(self.c.shape[0], self.c.shape[1], -1),
+                    self.x, xs, xe, False, out=range_int)
+                range_int *= n_periods
+            else:
+                range_int.fill(0)
+
+            # Map a to [xs, xe], b is always a + left.
+            a = xs + (a - xs) % period
+            b = a + left
+
+            # If b <= xe then we need to integrate over [a, b], otherwise
+            # over [a, xe] and from xs to what is remained.
+            remainder_int = cupy.empty_like(range_int)
+            if b <= xe:
+                _integrate(
+                    self.c.reshape(self.c.shape[0], self.c.shape[1], -1),
+                    self.x, a, b, False, out=remainder_int)
+                range_int += remainder_int
+            else:
+                _integrate(
+                    self.c.reshape(self.c.shape[0], self.c.shape[1], -1),
+                    self.x, a, xe, False, out=remainder_int)
+                range_int += remainder_int
+
+                _integrate(
+                    self.c.reshape(self.c.shape[0], self.c.shape[1], -1),
+                    self.x, xs, xs + left + a - xe, False, out=remainder_int)
+                range_int += remainder_int
+        else:
+            _integrate(
+                self.c.reshape(self.c.shape[0], self.c.shape[1], -1),
+                self.x, a, b, bool(extrapolate), out=range_int)
+
+        # Return
+        range_int *= sign
+        return range_int.reshape(self.c.shape[2:])
+
+    def solve(self, y=0., discontinuity=True, extrapolate=None):
+        """
+        Find real solutions of the equation ``pp(x) == y``.
+
+        Parameters
+        ----------
+        y : float, optional
+            Right-hand side. Default is zero.
+        discontinuity : bool, optional
+            Whether to report sign changes across discontinuities at
+            breakpoints as roots.
+        extrapolate : {bool, 'periodic', None}, optional
+            If bool, determines whether to return roots from the polynomial
+            extrapolated based on first and last intervals, 'periodic' works
+            the same as False. If None (default), use `self.extrapolate`.
+
+        Returns
+        -------
+        roots : ndarray
+            Roots of the polynomial(s).
+            If the PPoly object describes multiple polynomials, the
+            return value is an object array whose each element is an
+            ndarray containing the roots.
+
+        Notes
+        -----
+        This routine works only on real-valued polynomials.
+        If the piecewise polynomial contains sections that are
+        identically zero, the root list will contain the start point
+        of the corresponding interval, followed by a ``nan`` value.
+        If the polynomial is discontinuous across a breakpoint, and
+        there is a sign change across the breakpoint, this is reported
+        if the `discont` parameter is True.
+
+        At the moment, there is not an actual implementation.
+        """
+        raise NotImplementedError(
+            'At the moment there is not a GPU implementation for solve')
+
+    def roots(self, discontinuity=True, extrapolate=None):
+        """
+        Find real roots of the piecewise polynomial.
+
+        Parameters
+        ----------
+        discontinuity : bool, optional
+            Whether to report sign changes across discontinuities at
+            breakpoints as roots.
+        extrapolate : {bool, 'periodic', None}, optional
+            If bool, determines whether to return roots from the polynomial
+            extrapolated based on first and last intervals, 'periodic' works
+            the same as False. If None (default), use `self.extrapolate`.
+
+        Returns
+        -------
+        roots : ndarray
+            Roots of the polynomial(s).
+            If the PPoly object describes multiple polynomials, the
+            return value is an object array whose each element is an
+            ndarray containing the roots.
+
+        See Also
+        --------
+        PPoly.solve
+        """
+        return self.solve(0, discontinuity, extrapolate)
+
+    @classmethod
+    def from_spline(cls, tck, extrapolate=None):
+        """
+        Construct a piecewise polynomial from a spline
+
+        Parameters
+        ----------
+        tck
+            A spline, as a (knots, coefficients, degree) tuple or
+            a BSpline object.
+        extrapolate : bool or 'periodic', optional
+            If bool, determines whether to extrapolate to out-of-bounds points
+            based on first and last intervals, or to return NaNs.
+            If 'periodic', periodic extrapolation is used. Default is True.
+        """
+        if isinstance(tck, BSpline):
+            t, c, k = tck.tck
+            if extrapolate is None:
+                extrapolate = tck.extrapolate
+        else:
+            t, c, k = tck
+
+        spl = BSpline(t, c, k, extrapolate=extrapolate)
+        cvals = cupy.empty((k + 1, len(t) - 1), dtype=c.dtype)
+        for m in range(k, -1, -1):
+            y = spl(t[:-1], nu=m)
+            cvals[k - m, :] = y / spec.gamma(m + 1)
+
+        return cls.construct_fast(cvals, t, extrapolate)
+
+    @classmethod
+    def from_bernstein_basis(cls, bp, extrapolate=None):
+        """
+        Construct a piecewise polynomial in the power basis
+        from a polynomial in Bernstein basis.
+
+        Parameters
+        ----------
+        bp : BPoly
+            A Bernstein basis polynomial, as created by BPoly
+        extrapolate : bool or 'periodic', optional
+            If bool, determines whether to extrapolate to out-of-bounds points
+            based on first and last intervals, or to return NaNs.
+            If 'periodic', periodic extrapolation is used. Default is True.
+        """
+        if not isinstance(bp, BPoly):
+            raise TypeError(".from_bernstein_basis only accepts BPoly "
+                            "instances. Got %s instead." % type(bp))
+
+        dx = cupy.diff(bp.x)
+        k = bp.c.shape[0] - 1  # polynomial order
+
+        rest = (None,)*(bp.c.ndim-2)
+
+        c = cupy.zeros_like(bp.c)
+        for a in range(k+1):
+            factor = (-1)**a * _comb(k, a) * bp.c[a]
+            for s in range(a, k+1):
+                val = _comb(k-a, s-a) * (-1)**s
+                c[k-s] += factor * val / dx[(slice(None),)+rest]**s
+
+        if extrapolate is None:
+            extrapolate = bp.extrapolate
+
+        return cls.construct_fast(c, bp.x, extrapolate, bp.axis)
+
+
+class BPoly(_PPolyBase):
+    """
+    Piecewise polynomial in terms of coefficients and breakpoints.
+
+    The polynomial between ``x[i]`` and ``x[i + 1]`` is written in the
+
+    Bernstein polynomial basis::
+
+        S = sum(c[a, i] * b(a, k; x) for a in range(k+1)),
+
+    where ``k`` is the degree of the polynomial, and::
+
+        b(a, k; x) = binom(k, a) * t**a * (1 - t)**(k - a),
+
+    with ``t = (x - x[i]) / (x[i+1] - x[i])`` and ``binom`` is the binomial
+    coefficient.
+
+    Parameters
+    ----------
+    c : ndarray, shape (k, m, ...)
+        Polynomial coefficients, order `k` and `m` intervals
+    x : ndarray, shape (m+1,)
+        Polynomial breakpoints. Must be sorted in either increasing or
+        decreasing order.
+    extrapolate : bool, optional
+        If bool, determines whether to extrapolate to out-of-bounds points
+        based on first and last intervals, or to return NaNs. If 'periodic',
+        periodic extrapolation is used. Default is True.
+    axis : int, optional
+        Interpolation axis. Default is zero.
+
+    Attributes
+    ----------
+    x : ndarray
+        Breakpoints.
+    c : ndarray
+        Coefficients of the polynomials. They are reshaped
+        to a 3-D array with the last dimension representing
+        the trailing dimensions of the original coefficient array.
+    axis : int
+        Interpolation axis.
+
+    See also
+    --------
+    PPoly : piecewise polynomials in the power basis
+
+    Notes
+    -----
+    Properties of Bernstein polynomials are well documented in the literature,
+    see for example [1]_ [2]_ [3]_.
+
+    References
+    ----------
+    .. [1] https://en.wikipedia.org/wiki/Bernstein_polynomial
+    .. [2] Kenneth I. Joy, Bernstein polynomials,
+       http://www.idav.ucdavis.edu/education/CAGDNotes/Bernstein-Polynomials.pdf
+    .. [3] E. H. Doha, A. H. Bhrawy, and M. A. Saker, Boundary Value Problems,
+           vol 2011, article ID 829546,
+           `10.1155/2011/829543 <https://doi.org/10.1155/2011/829543>`_.
+
+    Examples
+    --------
+    >>> from cupyx.scipy.interpolate import BPoly
+    >>> x = [0, 1]
+    >>> c = [[1], [2], [3]]
+    >>> bp = BPoly(c, x)
+
+    This creates a 2nd order polynomial
+
+    .. math::
+
+        B(x) = 1 \\times b_{0, 2}(x) + 2 \\times b_{1, 2}(x) +
+               3 \\times b_{2, 2}(x) \\\\
+             = 1 \\times (1-x)^2 + 2 \\times 2 x (1 - x) + 3 \\times x^2
+    """
+
+    def _evaluate(self, x, nu, extrapolate, out):
+        # check derivative order
+        if nu < 0:
+            raise NotImplementedError(
+                "Cannot do antiderivatives in the B-basis yet.")
+
+        _bpoly_evaluate(
+            self.c.reshape(self.c.shape[0], self.c.shape[1], -1),
+            self.x, x, nu, bool(extrapolate), out)
+
+    def derivative(self, nu=1):
+        """
+        Construct a new piecewise polynomial representing the derivative.
+
+        Parameters
+        ----------
+        nu : int, optional
+            Order of derivative to evaluate. Default is 1, i.e., compute the
+            first derivative. If negative, the antiderivative is returned.
+
+        Returns
+        -------
+        bp : BPoly
+            Piecewise polynomial of order k - nu representing the derivative of
+            this polynomial.
+        """
+        if nu < 0:
+            return self.antiderivative(-nu)
+
+        if nu > 1:
+            bp = self
+            for k in range(nu):
+                bp = bp.derivative()
+            return bp
+
+        # reduce order
+        if nu == 0:
+            c2 = self.c.copy()
+        else:
+            # For a polynomial
+            #    B(x) = \sum_{a=0}^{k} c_a b_{a, k}(x),
+            # we use the fact that
+            #   b'_{a, k} = k ( b_{a-1, k-1} - b_{a, k-1} ),
+            # which leads to
+            #   B'(x) = \sum_{a=0}^{k-1} (c_{a+1} - c_a) b_{a, k-1}
+            #
+            # finally, for an interval [y, y + dy] with dy != 1,
+            # we need to correct for an extra power of dy
+
+            rest = (None,) * (self.c.ndim-2)
+
+            k = self.c.shape[0] - 1
+            dx = cupy.diff(self.x)[(None, slice(None))+rest]
+            c2 = k * cupy.diff(self.c, axis=0) / dx
+
+        if c2.shape[0] == 0:
+            # derivative of order 0 is zero
+            c2 = cupy.zeros((1,) + c2.shape[1:], dtype=c2.dtype)
+
+        # construct a compatible polynomial
+        return self.construct_fast(c2, self.x, self.extrapolate, self.axis)
+
+    def antiderivative(self, nu=1):
+        """
+        Construct a new piecewise polynomial representing the antiderivative.
+
+        Parameters
+        ----------
+        nu : int, optional
+            Order of antiderivative to evaluate. Default is 1, i.e., compute
+            the first integral. If negative, the derivative is returned.
+
+        Returns
+        -------
+        bp : BPoly
+            Piecewise polynomial of order k + nu representing the
+            antiderivative of this polynomial.
+
+        Notes
+        -----
+        If antiderivative is computed and ``self.extrapolate='periodic'``,
+        it will be set to False for the returned instance. This is done because
+        the antiderivative is no longer periodic and its correct evaluation
+        outside of the initially given x interval is difficult.
+        """
+        if nu <= 0:
+            return self.derivative(-nu)
+
+        if nu > 1:
+            bp = self
+            for k in range(nu):
+                bp = bp.antiderivative()
+            return bp
+
+        # Construct the indefinite integrals on individual intervals
+        c, x = self.c, self.x
+        k = c.shape[0]
+        c2 = cupy.zeros((k+1,) + c.shape[1:], dtype=c.dtype)
+
+        c2[1:, ...] = cupy.cumsum(c, axis=0) / k
+        delta = x[1:] - x[:-1]
+        c2 *= delta[(None, slice(None)) + (None,)*(c.ndim-2)]
+
+        # Now fix continuity: on the very first interval, take the integration
+        # constant to be zero; on an interval [x_j, x_{j+1}) with j>0,
+        # the integration constant is then equal to the jump of the `bp`
+        # at x_j.
+        # The latter is given by the coefficient of B_{n+1, n+1}
+        # *on the previous interval* (other B. polynomials are zero at the
+        # breakpoint). Finally, use the fact that BPs form a partition of
+        # unity.
+        c2[:, 1:] += cupy.cumsum(c2[k, :], axis=0)[:-1]
+
+        if self.extrapolate == 'periodic':
+            extrapolate = False
+        else:
+            extrapolate = self.extrapolate
+
+        return self.construct_fast(c2, x, extrapolate, axis=self.axis)
+
+    def integrate(self, a, b, extrapolate=None):
+        """
+        Compute a definite integral over a piecewise polynomial.
+
+        Parameters
+        ----------
+        a : float
+            Lower integration bound
+        b : float
+            Upper integration bound
+        extrapolate : {bool, 'periodic', None}, optional
+            Whether to extrapolate to out-of-bounds points based on first
+            and last intervals, or to return NaNs. If 'periodic', periodic
+            extrapolation is used. If None (default), use `self.extrapolate`.
+
+        Returns
+        -------
+        array_like
+            Definite integral of the piecewise polynomial over [a, b]
+        """
+        # XXX: can probably use instead the fact that
+        # \int_0^{1} B_{j, n}(x) \dx = 1/(n+1)
+        ib = self.antiderivative()
+        if extrapolate is None:
+            extrapolate = self.extrapolate
+
+        # ib.extrapolate shouldn't be 'periodic', it is converted to
+        # False for 'periodic. in antiderivative() call.
+        if extrapolate != 'periodic':
+            ib.extrapolate = extrapolate
+
+        if extrapolate == 'periodic':
+            # Split the integral into the part over period (can be several
+            # of them) and the remaining part.
+
+            # For simplicity and clarity convert to a <= b case.
+            if a <= b:
+                sign = 1
+            else:
+                a, b = b, a
+                sign = -1
+
+            xs, xe = self.x[0], self.x[-1]
+            period = xe - xs
+            interval = b - a
+            n_periods, left = divmod(interval, period)
+            res = n_periods * (ib(xe) - ib(xs))
+
+            # Map a and b to [xs, xe].
+            a = xs + (a - xs) % period
+            b = a + left
+
+            # If b <= xe then we need to integrate over [a, b], otherwise
+            # over [a, xe] and from xs to what is remained.
+            if b <= xe:
+                res += ib(b) - ib(a)
+            else:
+                res += ib(xe) - ib(a) + ib(xs + left + a - xe) - ib(xs)
+
+            return sign * res
+        else:
+            return ib(b) - ib(a)
+
+    def extend(self, c, x):
+        k = max(self.c.shape[0], c.shape[0])
+        self.c = self._raise_degree(self.c, k - self.c.shape[0])
+        c = self._raise_degree(c, k - c.shape[0])
+        return _PPolyBase.extend(self, c, x)
+    extend.__doc__ = _PPolyBase.extend.__doc__
+
+    @staticmethod
+    def _raise_degree(c, d):
+        r"""
+        Raise a degree of a polynomial in the Bernstein basis.
+
+        Given the coefficients of a polynomial degree `k`, return (the
+        coefficients of) the equivalent polynomial of degree `k+d`.
+
+        Parameters
+        ----------
+        c : array_like
+            coefficient array, 1-D
+        d : integer
+
+        Returns
+        -------
+        array
+            coefficient array, 1-D array of length `c.shape[0] + d`
+
+        Notes
+        -----
+        This uses the fact that a Bernstein polynomial `b_{a, k}` can be
+        identically represented as a linear combination of polynomials of
+        a higher degree `k+d`:
+
+            .. math:: b_{a, k} = comb(k, a) \sum_{j=0}^{d} b_{a+j, k+d} \
+                                 comb(d, j) / comb(k+d, a+j)
+        """
+        if d == 0:
+            return c
+
+        k = c.shape[0] - 1
+        out = cupy.zeros((c.shape[0] + d,) + c.shape[1:], dtype=c.dtype)
+
+        for a in range(c.shape[0]):
+            f = c[a] * _comb(k, a)
+            for j in range(d + 1):
+                out[a + j] += f * _comb(d, j) / _comb(k + d, a + j)
+        return out
+
+    @classmethod
+    def from_power_basis(cls, pp, extrapolate=None):
+        """
+        Construct a piecewise polynomial in Bernstein basis
+        from a power basis polynomial.
+
+        Parameters
+        ----------
+        pp : PPoly
+            A piecewise polynomial in the power basis
+        extrapolate : bool or 'periodic', optional
+            If bool, determines whether to extrapolate to out-of-bounds points
+            based on first and last intervals, or to return NaNs.
+            If 'periodic', periodic extrapolation is used. Default is True.
+        """
+        if not isinstance(pp, PPoly):
+            raise TypeError(".from_power_basis only accepts PPoly instances. "
+                            "Got %s instead." % type(pp))
+
+        dx = cupy.diff(pp.x)
+        k = pp.c.shape[0] - 1   # polynomial order
+
+        rest = (None,)*(pp.c.ndim-2)
+
+        c = cupy.zeros_like(pp.c)
+        for a in range(k+1):
+            factor = pp.c[a] / _comb(k, k-a) * dx[(slice(None),)+rest]**(k-a)
+            for j in range(k-a, k+1):
+                c[j] += factor * _comb(j, k-a)
+
+        if extrapolate is None:
+            extrapolate = pp.extrapolate
+
+        return cls.construct_fast(c, pp.x, extrapolate, pp.axis)
+
+    @classmethod
+    def from_derivatives(cls, xi, yi, orders=None, extrapolate=None):
+        """
+        Construct a piecewise polynomial in the Bernstein basis,
+        compatible with the specified values and derivatives at breakpoints.
+
+        Parameters
+        ----------
+        xi : array_like
+            sorted 1-D array of x-coordinates
+        yi : array_like or list of array_likes
+            ``yi[i][j]`` is the ``j`` th derivative known at ``xi[i]``
+        orders : None or int or array_like of ints. Default: None.
+            Specifies the degree of local polynomials. If not None, some
+            derivatives are ignored.
+        extrapolate : bool or 'periodic', optional
+            If bool, determines whether to extrapolate to out-of-bounds points
+            based on first and last intervals, or to return NaNs.
+            If 'periodic', periodic extrapolation is used. Default is True.
+
+        Notes
+        -----
+        If ``k`` derivatives are specified at a breakpoint ``x``, the
+        constructed polynomial is exactly ``k`` times continuously
+        differentiable at ``x``, unless the ``order`` is provided explicitly.
+        In the latter case, the smoothness of the polynomial at
+        the breakpoint is controlled by the ``order``.
+
+        Deduces the number of derivatives to match at each end
+        from ``order`` and the number of derivatives available. If
+        possible it uses the same number of derivatives from
+        each end; if the number is odd it tries to take the
+        extra one from y2. In any case if not enough derivatives
+        are available at one end or another it draws enough to
+        make up the total from the other end.
+
+        If the order is too high and not enough derivatives are available,
+        an exception is raised.
+
+        Examples
+        --------
+        >>> from cupyx.scipy.interpolate import BPoly
+        >>> BPoly.from_derivatives([0, 1], [[1, 2], [3, 4]])
+
+        Creates a polynomial `f(x)` of degree 3, defined on `[0, 1]`
+        such that `f(0) = 1, df/dx(0) = 2, f(1) = 3, df/dx(1) = 4`
+
+        >>> BPoly.from_derivatives([0, 1, 2], [[0, 1], [0], [2]])
+
+        Creates a piecewise polynomial `f(x)`, such that
+        `f(0) = f(1) = 0`, `f(2) = 2`, and `df/dx(0) = 1`.
+        Based on the number of derivatives provided, the order of the
+        local polynomials is 2 on `[0, 1]` and 1 on `[1, 2]`.
+        Notice that no restriction is imposed on the derivatives at
+        ``x = 1`` and ``x = 2``.
+
+        Indeed, the explicit form of the polynomial is::
+
+            f(x) = | x * (1 - x),  0 <= x < 1
+                   | 2 * (x - 1),  1 <= x <= 2
+
+        So that f'(1-0) = -1 and f'(1+0) = 2
+        """
+        xi = cupy.asarray(xi)
+        if len(xi) != len(yi):
+            raise ValueError("xi and yi need to have the same length")
+        if cupy.any(xi[1:] - xi[:1] <= 0):
+            raise ValueError("x coordinates are not in increasing order")
+
+        # number of intervals
+        m = len(xi) - 1
+
+        # global poly order is k-1, local orders are <=k and can vary
+        try:
+            k = max(len(yi[i]) + len(yi[i+1]) for i in range(m))
+        except TypeError as e:
+            raise ValueError(
+                "Using a 1-D array for y? Please .reshape(-1, 1)."
+            ) from e
+
+        if orders is None:
+            orders = [None] * m
+        else:
+            if isinstance(orders, (int, cupy.integer)):
+                orders = [orders] * m
+            k = max(k, max(orders))
+
+            if any(o <= 0 for o in orders):
+                raise ValueError("Orders must be positive.")
+
+        c = []
+        for i in range(m):
+            y1, y2 = yi[i], yi[i+1]
+            if orders[i] is None:
+                n1, n2 = len(y1), len(y2)
+            else:
+                n = orders[i]+1
+                n1 = min(n//2, len(y1))
+                n2 = min(n - n1, len(y2))
+                n1 = min(n - n2, len(y2))
+                if n1+n2 != n:
+                    mesg = ("Point %g has %d derivatives, point %g"
+                            " has %d derivatives, but order %d requested" % (
+                                xi[i], len(y1), xi[i+1], len(y2), orders[i]))
+                    raise ValueError(mesg)
+
+                if not (n1 <= len(y1) and n2 <= len(y2)):
+                    raise ValueError("`order` input incompatible with"
+                                     " length y1 or y2.")
+
+            b = BPoly._construct_from_derivatives(xi[i], xi[i+1],
+                                                  y1[:n1], y2[:n2])
+            if len(b) < k:
+                b = BPoly._raise_degree(b, k - len(b))
+            c.append(b)
+
+        c = cupy.asarray(c)
+        return cls(c.swapaxes(0, 1), xi, extrapolate)
+
+    @staticmethod
+    def _construct_from_derivatives(xa, xb, ya, yb):
+        r"""
+        Compute the coefficients of a polynomial in the Bernstein basis
+        given the values and derivatives at the edges.
+
+        Return the coefficients of a polynomial in the Bernstein basis
+        defined on ``[xa, xb]`` and having the values and derivatives at the
+        endpoints `xa` and `xb` as specified by `ya`` and `yb`.
+
+        The polynomial constructed is of the minimal possible degree, i.e.,
+        if the lengths of `ya` and `yb` are `na` and `nb`, the degree
+        of the polynomial is ``na + nb - 1``.
+
+        Parameters
+        ----------
+        xa : float
+            Left-hand end point of the interval
+        xb : float
+            Right-hand end point of the interval
+        ya : array_like
+            Derivatives at `xa`. `ya[0]` is the value of the function, and
+            `ya[i]` for ``i > 0`` is the value of the ``i``th derivative.
+        yb : array_like
+            Derivatives at `xb`.
+
+        Returns
+        -------
+        array
+            coefficient array of a polynomial having specified derivatives
+
+        Notes
+        -----
+        This uses several facts from life of Bernstein basis functions.
+        First of all,
+
+            .. math:: b'_{a, n} = n (b_{a-1, n-1} - b_{a, n-1})
+
+        If B(x) is a linear combination of the form
+
+            .. math:: B(x) = \sum_{a=0}^{n} c_a b_{a, n},
+
+        then :math: B'(x) = n \sum_{a=0}^{n-1} (c_{a+1} - c_{a}) b_{a, n-1}.
+        Iterating the latter one, one finds for the q-th derivative
+
+            .. math:: B^{q}(x) = n!/(n-q)! \sum_{a=0}^{n-q} Q_a b_{a, n-q},
+
+        with
+
+            .. math:: Q_a = \sum_{j=0}^{q} (-)^{j+q} comb(q, j) c_{j+a}
+
+        This way, only `a=0` contributes to :math: `B^{q}(x = xa)`, and
+        `c_q` are found one by one by iterating `q = 0, ..., na`.
+
+        At ``x = xb`` it's the same with ``a = n - q``.
+        """
+        ya, yb = cupy.asarray(ya), cupy.asarray(yb)
+        if ya.shape[1:] != yb.shape[1:]:
+            raise ValueError('Shapes of ya {} and yb {} are incompatible'
+                             .format(ya.shape, yb.shape))
+
+        dta, dtb = ya.dtype, yb.dtype
+        if (cupy.issubdtype(dta, cupy.complexfloating) or
+                cupy.issubdtype(dtb, cupy.complexfloating)):
+            dt = cupy.complex_
+        else:
+            dt = cupy.float_
+
+        na, nb = len(ya), len(yb)
+        n = na + nb
+
+        c = cupy.empty((na+nb,) + ya.shape[1:], dtype=dt)
+
+        # compute coefficients of a polynomial degree na+nb-1
+        # walk left-to-right
+        for q in range(0, na):
+            c[q] = ya[q] / spec.poch(n - q, q) * (xb - xa)**q
+            for j in range(0, q):
+                c[q] -= (-1)**(j+q) * _comb(q, j) * c[j]
+
+        # now walk right-to-left
+        for q in range(0, nb):
+            c[-q-1] = yb[q] / spec.poch(n - q, q) * (-1)**q * (xb - xa)**q
+            for j in range(0, q):
+                c[-q-1] -= (-1)**(j+1) * _comb(q, j+1) * c[-q+j]
+
+        return c
diff --git a/cupyx/scipy/interpolate/_polyint.py b/cupyx/scipy/interpolate/_polyint.py
new file mode 100644
index 0000000..5b6754d
--- /dev/null
+++ b/cupyx/scipy/interpolate/_polyint.py
@@ -0,0 +1,527 @@
+import cupy
+from cupyx.scipy._lib._util import _asarray_validated, float_factorial
+
+
+def _isscalar(x):
+    """Check whether x is if a scalar type, or 0-dim"""
+    return cupy.isscalar(x) or hasattr(x, 'shape') and x.shape == ()
+
+
+class _Interpolator1D:
+    """Common features in univariate interpolation.
+
+    Deal with input data type and interpolation axis rolling. The
+    actual interpolator can assume the y-data is of shape (n, r) where
+    `n` is the number of x-points, and `r` the number of variables,
+    and use self.dtype as the y-data type.
+
+    Attributes
+    ----------
+    _y_axis : Axis along which the interpolation goes in the
+        original array
+    _y_extra_shape : Additional shape of the input arrays, excluding
+        the interpolation axis
+    dtype : Dtype of the y-data arrays. It can be set via _set_dtype,
+        which forces it to be float or complex
+
+    Methods
+    -------
+    __call__
+    _prepare_x
+    _finish_y
+    _reshape_y
+    _reshape_yi
+    _set_yi
+    _set_dtype
+    _evaluate
+
+    """
+
+    def __init__(self, xi=None, yi=None, axis=None):
+        self._y_axis = axis
+        self._y_extra_shape = None
+        self.dtype = None
+        if yi is not None:
+            self._set_yi(yi, xi=xi, axis=axis)
+
+    def __call__(self, x):
+        """Evaluate the interpolant
+
+        Parameters
+        ----------
+        x : cupy.ndarray
+            The points to evaluate the interpolant
+
+        Returns
+        -------
+        y : cupy.ndarray
+            Interpolated values. Shape is determined by replacing
+            the interpolation axis in the original array with the shape of x
+
+        Notes
+        -----
+        Input values `x` must be convertible to `float` values like `int`
+        or `float`.
+
+        """
+        x, x_shape = self._prepare_x(x)
+        y = self._evaluate(x)
+        return self._finish_y(y, x_shape)
+
+    def _evaluate(self, x):
+        """
+        Actually evaluate the value of the interpolator
+        """
+        raise NotImplementedError()
+
+    def _prepare_x(self, x):
+        """
+        Reshape input array to 1-D
+        """
+        x = _asarray_validated(x, check_finite=False, as_inexact=True)
+        x_shape = x.shape
+        return x.ravel(), x_shape
+
+    def _finish_y(self, y, x_shape):
+        """
+        Reshape interpolated y back to an N-D array similar to initial y
+        """
+        y = y.reshape(x_shape + self._y_extra_shape)
+        if self._y_axis != 0 and x_shape != ():
+            nx = len(x_shape)
+            ny = len(self._y_extra_shape)
+            s = (list(range(nx, nx + self._y_axis))
+                 + list(range(nx)) + list(range(nx + self._y_axis, nx + ny)))
+            y = y.transpose(s)
+        return y
+
+    def _reshape_yi(self, yi, check=False):
+        """
+        Reshape the updated yi to a 1-D array
+        """
+        yi = cupy.moveaxis(yi, self._y_axis, 0)
+        if check and yi.shape[1:] != self._y_extra_shape:
+            ok_shape = "%r + (N,) + %r" % (self._y_extra_shape[-self._y_axis:],
+                                           self._y_extra_shape[:-self._y_axis])
+            raise ValueError("Data must be of shape %s" % ok_shape)
+        return yi.reshape((yi.shape[0], -1))
+
+    def _set_yi(self, yi, xi=None, axis=None):
+        if axis is None:
+            axis = self._y_axis
+        if axis is None:
+            raise ValueError("no interpolation axis specified")
+
+        shape = yi.shape
+        if shape == ():
+            shape = (1,)
+        if xi is not None and shape[axis] != len(xi):
+            raise ValueError("x and y arrays must be equal in length along "
+                             "interpolation axis.")
+
+        self._y_axis = (axis % yi.ndim)
+        self._y_extra_shape = yi.shape[:self._y_axis]+yi.shape[self._y_axis+1:]
+        self.dtype = None
+        self._set_dtype(yi.dtype)
+
+    def _set_dtype(self, dtype, union=False):
+        if cupy.issubdtype(dtype, cupy.complexfloating) \
+                or cupy.issubdtype(self.dtype, cupy.complexfloating):
+            self.dtype = cupy.complex_
+        else:
+            if not union or self.dtype != cupy.complex_:
+                self.dtype = cupy.float_
+
+
+class _Interpolator1DWithDerivatives(_Interpolator1D):
+
+    def derivatives(self, x, der=None):
+        """Evaluate many derivatives of the polynomial at the point x.
+
+        The function produce an array of all derivative values at
+        the point x.
+
+        Parameters
+        ----------
+        x : cupy.ndarray
+            Point or points at which to evaluate the derivatives
+        der : int or None, optional
+            How many derivatives to extract; None for all potentially
+            nonzero derivatives (that is a number equal to the number
+            of points). This number includes the function value as 0th
+            derivative
+
+        Returns
+        -------
+        d : cupy.ndarray
+            Array with derivatives; d[j] contains the jth derivative.
+            Shape of d[j] is determined by replacing the interpolation
+            axis in the original array with the shape of x
+
+        """
+        x, x_shape = self._prepare_x(x)
+        y = self._evaluate_derivatives(x, der)
+
+        y = y.reshape((y.shape[0],) + x_shape + self._y_extra_shape)
+        if self._y_axis != 0 and x_shape != ():
+            nx = len(x_shape)
+            ny = len(self._y_extra_shape)
+            s = ([0] + list(range(nx+1, nx + self._y_axis+1))
+                 + list(range(1, nx+1)) +
+                 list(range(nx+1+self._y_axis, nx+ny+1)))
+            y = y.transpose(s)
+        return y
+
+    def derivative(self, x, der=1):
+        """Evaluate one derivative of the polynomial at the point x
+
+        Parameters
+        ----------
+        x : cupy.ndarray
+            Point or points at which to evaluate the derivatives
+        der : integer, optional
+            Which derivative to extract. This number includes the
+            function value as 0th derivative
+
+        Returns
+        -------
+        d : cupy.ndarray
+            Derivative interpolated at the x-points. Shape of d is
+            determined by replacing the interpolation axis in the
+            original array with the shape of x
+
+        Notes
+        -----
+        This is computed by evaluating all derivatives up to the desired
+        one (using self.derivatives()) and then discarding the rest.
+
+        """
+        x, x_shape = self._prepare_x(x)
+        y = self._evaluate_derivatives(x, der+1)
+        return self._finish_y(y[der], x_shape)
+
+
+class BarycentricInterpolator(_Interpolator1D):
+    """The interpolating polynomial for a set of points.
+
+    Constructs a polynomial that passes through a given set of points.
+    Allows evaluation of the polynomial, efficient changing of the y
+    values to be interpolated, and updating by adding more x values.
+    For reasons of numerical stability, this function does not compute
+    the coefficients of the polynomial.
+    The value `yi` need to be provided before the function is
+    evaluated, but none of the preprocessing depends on them,
+    so rapid updates are possible.
+
+    Parameters
+    ----------
+    xi : cupy.ndarray
+        1-D array of x-coordinates of the points the polynomial should
+        pass through
+    yi : cupy.ndarray, optional
+        The y-coordinates of the points the polynomial should pass through.
+        If None, the y values will be supplied later via the `set_y` method
+    axis : int, optional
+        Axis in the yi array corresponding to the x-coordinate values
+
+    See Also
+    --------
+    scipy.interpolate.BarycentricInterpolator
+
+    """
+
+    def __init__(self, xi, yi=None, axis=0):
+        _Interpolator1D.__init__(self, xi, yi, axis)
+
+        self.xi = xi.astype(cupy.float_)
+        self.set_yi(yi)
+        self.n = len(self.xi)
+
+        self._inv_capacity = 4.0 / (cupy.max(self.xi) - cupy.min(self.xi))
+        permute = cupy.random.permutation(self.n)
+        inv_permute = cupy.zeros(self.n, dtype=cupy.int32)
+        inv_permute[permute] = cupy.arange(self.n)
+
+        self.wi = cupy.zeros(self.n)
+        for i in range(self.n):
+            dist = self._inv_capacity * (self.xi[i] - self.xi[permute])
+            dist[inv_permute[i]] = 1.0
+            self.wi[i] = 1.0 / cupy.prod(dist)
+
+    def set_yi(self, yi, axis=None):
+        """Update the y values to be interpolated.
+
+        The barycentric interpolation algorithm requires the calculation
+        of weights, but these depend only on the xi. The yi can be changed
+        at any time.
+
+        Parameters
+        ----------
+        yi : cupy.ndarray
+            The y-coordinates of the points the polynomial should pass
+            through. If None, the y values will be supplied later.
+        axis : int, optional
+            Axis in the yi array corresponding to the x-coordinate values
+
+        """
+
+        if yi is None:
+            self.yi = None
+            return
+        self._set_yi(yi, xi=self.xi, axis=axis)
+        self.yi = self._reshape_yi(yi)
+        self.n, self.r = self.yi.shape
+
+    def add_xi(self, xi, yi=None):
+        """Add more x values to the set to be interpolated.
+
+        The barycentric interpolation algorithm allows easy updating
+        by adding more points for the polynomial to pass through.
+
+        Parameters
+        ----------
+        xi : cupy.ndarray
+            The x-coordinates of the points that the polynomial should
+            pass through
+        yi : cupy.ndarray, optional
+            The y-coordinates of the points the polynomial should pass
+            through. Should have shape ``(xi.size, R)``; if R > 1 then
+            the polynomial is vector-valued
+            If `yi` is not given, the y values will be supplied later.
+            `yi` should be given if and only if the interpolator has y
+            values specified
+
+        """
+
+        if yi is not None:
+            if self.yi is None:
+                raise ValueError("No previous yi value to update!")
+            yi = self._reshape_yi(yi, check=True)
+            self.yi = cupy.vstack((self.yi, yi))
+        else:
+            if self.yi is not None:
+                raise ValueError("No update to yi provided!")
+        old_n = self.n
+        self.xi = cupy.concatenate((self.xi, xi))
+        self.n = len(self.xi)
+        self.wi **= -1
+        old_wi = self.wi
+        self.wi = cupy.zeros(self.n)
+        self.wi[:old_n] = old_wi
+        for j in range(old_n, self.n):
+            self.wi[:j] *= self._inv_capacity * (self.xi[j] - self.xi[:j])
+            self.wi[j] = cupy.prod(
+                self._inv_capacity * (self.xi[:j] - self.xi[j])
+            )
+        self.wi **= -1
+
+    def __call__(self, x):
+        """Evaluate the interpolating polynomial at the points x.
+
+        Parameters
+        ----------
+        x : cupy.ndarray
+            Points to evaluate the interpolant at
+
+        Returns
+        -------
+        y : cupy.ndarray
+            Interpolated values. Shape is determined by replacing the
+            interpolation axis in the original array with the shape of x
+
+        Notes
+        -----
+        Currently the code computes an outer product between x and the
+        weights, that is, it constructs an intermediate array of size
+        N by len(x), where N is the degree of the polynomial.
+
+        """
+
+        return super().__call__(x)
+
+    def _evaluate(self, x):
+        if x.size == 0:
+            p = cupy.zeros((0, self.r), dtype=self.dtype)
+        else:
+            c = x[..., cupy.newaxis] - self.xi
+            z = c == 0
+            c[z] = 1
+            c = self.wi / c
+            p = cupy.dot(c, self.yi) / cupy.sum(c, axis=-1)[..., cupy.newaxis]
+            r = cupy.nonzero(z)
+            if len(r) == 1:  # evaluation at a scalar
+                if len(r[0]) > 0:  # equals one of the points
+                    p = self.yi[r[0][0]]
+            else:
+                p[r[:-1]] = self.yi[r[-1]]
+        return p
+
+
+def barycentric_interpolate(xi, yi, x, axis=0):
+    """Convenience function for polynomial interpolation.
+
+    Constructs a polynomial that passes through a given
+    set of points, then evaluates the polynomial. For
+    reasons of numerical stability, this function does
+    not compute the coefficients of the polynomial.
+
+    Parameters
+    ----------
+    xi : cupy.ndarray
+        1-D array of coordinates of the points the polynomial
+        should pass through
+    yi : cupy.ndarray
+        y-coordinates of the points the polynomial should pass
+        through
+    x : scalar or cupy.ndarray
+        Points to evaluate the interpolator at
+    axis : int, optional
+        Axis in the yi array corresponding to the x-coordinate
+        values
+
+    Returns
+    -------
+    y : scalar or cupy.ndarray
+        Interpolated values. Shape is determined by replacing
+        the interpolation axis in the original array with the
+        shape x
+
+    See Also
+    --------
+    scipy.interpolate.barycentric_interpolate
+
+    """
+
+    return BarycentricInterpolator(xi, yi, axis=axis)(x)
+
+
+class KroghInterpolator(_Interpolator1DWithDerivatives):
+    """Interpolating polynomial for a set of points.
+
+    The polynomial passes through all the pairs (xi,yi). One may
+    additionally specify a number of derivatives at each point xi;
+    this is done by repeating the value xi and specifying the
+    derivatives as successive yi values
+    Allows evaluation of the polynomial and all its derivatives.
+    For reasons of numerical stability, this function does not compute
+    the coefficients of the polynomial, although they can be obtained
+    by evaluating all the derivatives.
+
+    Parameters
+    ----------
+    xi : cupy.ndarray, length N
+        x-coordinate, must be sorted in increasing order
+    yi : cupy.ndarray
+        y-coordinate, when a xi occurs two or more times in a row,
+        the corresponding yi's represent derivative values
+    axis : int, optional
+        Axis in the yi array corresponding to the x-coordinate values.
+
+    """
+
+    def __init__(self, xi, yi, axis=0):
+        _Interpolator1DWithDerivatives.__init__(self, xi, yi, axis)
+
+        self.xi = xi.astype(cupy.float_)
+        self.yi = self._reshape_yi(yi)
+        self.n, self.r = self.yi.shape
+
+        c = cupy.zeros((self.n+1, self.r), dtype=self.dtype)
+        c[0] = self.yi[0]
+        Vk = cupy.zeros((self.n, self.r), dtype=self.dtype)
+        for k in range(1, self.n):
+            s = 0
+            while s <= k and xi[k-s] == xi[k]:
+                s += 1
+            s -= 1
+            Vk[0] = self.yi[k]/float_factorial(s)
+            for i in range(k-s):
+                if xi[i] == xi[k]:
+                    raise ValueError("Elements if `xi` can't be equal.")
+                if s == 0:
+                    Vk[i+1] = (c[i]-Vk[i])/(xi[i]-xi[k])
+                else:
+                    Vk[i+1] = (Vk[i+1]-Vk[i])/(xi[i]-xi[k])
+            c[k] = Vk[k-s]
+        self.c = c
+
+    def _evaluate(self, x):
+        pi = 1
+        p = cupy.zeros((len(x), self.r), dtype=self.dtype)
+        p += self.c[0, cupy.newaxis, :]
+        for k in range(1, self.n):
+            w = x - self.xi[k-1]
+            pi = w*pi
+            p += pi[:, cupy.newaxis] * self.c[k]
+        return p
+
+    def _evaluate_derivatives(self, x, der=None):
+        n = self.n
+        r = self.r
+
+        if der is None:
+            der = self.n
+        pi = cupy.zeros((n, len(x)))
+        w = cupy.zeros((n, len(x)))
+        pi[0] = 1
+        p = cupy.zeros((len(x), self.r), dtype=self.dtype)
+        p += self.c[0, cupy.newaxis, :]
+
+        for k in range(1, n):
+            w[k-1] = x - self.xi[k-1]
+            pi[k] = w[k-1] * pi[k-1]
+            p += pi[k, :, cupy.newaxis] * self.c[k]
+
+        cn = cupy.zeros((max(der, n+1), len(x), r), dtype=self.dtype)
+        cn[:n+1, :, :] += self.c[:n+1, cupy.newaxis, :]
+        cn[0] = p
+        for k in range(1, n):
+            for i in range(1, n-k+1):
+                pi[i] = w[k+i-1]*pi[i-1] + pi[i]
+                cn[k] = cn[k] + pi[i, :, cupy.newaxis]*cn[k+i]
+            cn[k] *= float_factorial(k)
+
+        cn[n, :, :] = 0
+        return cn[:der]
+
+
+def krogh_interpolate(xi, yi, x, der=0, axis=0):
+    """Convenience function for polynomial interpolation
+
+    Parameters
+    ----------
+    xi : cupy.ndarray
+        x-coordinate
+    yi : cupy.ndarray
+        y-coordinates, of shape ``(xi.size, R)``. Interpreted as
+        vectors of length R, or scalars if R=1
+    x : cupy.ndarray
+        Point or points at which to evaluate the derivatives
+    der : int or list, optional
+        How many derivatives to extract; None for all potentially
+        nonzero derivatives (that is a number equal to the number
+        of points), or a list of derivatives to extract. This number
+        includes the function value as 0th derivative
+    axis : int, optional
+        Axis in the yi array corresponding to the x-coordinate values
+
+    Returns
+    -------
+    d : cupy.ndarray
+        If the interpolator's values are R-D then the
+        returned array will be the number of derivatives by N by R.
+        If `x` is a scalar, the middle dimension will be dropped; if
+        the `yi` are scalars then the last dimension will be dropped
+
+    See Also
+    --------
+    scipy.interpolate.krogh_interpolate
+
+    """
+    P = KroghInterpolator(xi, yi, axis=axis)
+    if der == 0:
+        return P(x)
+    elif _isscalar(der):
+        return P.derivative(x, der=der)
+    else:
+        return P.derivatives(x, der=cupy.amax(der)+1)[der]
diff --git a/cupyx/scipy/interpolate/_rbfinterp.py b/cupyx/scipy/interpolate/_rbfinterp.py
new file mode 100644
index 0000000..f461fb5
--- /dev/null
+++ b/cupyx/scipy/interpolate/_rbfinterp.py
@@ -0,0 +1,790 @@
+"""Module for RBF interpolation."""
+import math
+import warnings
+from itertools import combinations_with_replacement
+
+import cupy as cp
+
+
+# Define the kernel functions.
+
+kernel_definitions = """
+static __device__ double linear(double r)
+{
+     return -r;
+}
+
+static __device__ float linear_f(float r)
+{
+    return -r;
+}
+
+
+static __device__ double cubic(double r)
+{
+     return r*r*r;
+}
+
+static __device__ float cubic_f(float r)
+{
+    return r*r*r;
+}
+
+
+static __device__ double thin_plate_spline(double r)
+{
+    if (r == 0.0) {
+        return 0.0;
+    }
+    else {
+        return r*r*log(r);
+    }
+}
+
+static __device__ float thin_plate_spline_f(float r)
+{
+    if (r == 0.0) {
+        return 0.0;
+    }
+    else {
+        return r*r*log(r);
+    }
+}
+
+
+static __device__ double multiquadric(double r)
+{
+    return -sqrt(r*r + 1);
+}
+
+static __device__ float multiquadric_f(float r)
+{
+    return -sqrt(r*r + 1);
+}
+
+
+static __device__ double inverse_multiquadric(double r)
+{
+    return 1.0 / sqrt(r*r + 1);
+}
+
+static __device__ float inverse_multiquadric_f(float r)
+{
+    return 1.0 / sqrt(r*r + 1);
+}
+
+
+static __device__ double inverse_quadratic(double r)
+{
+    return 1.0 / (r*r + 1);
+}
+
+static __device__ float inverse_quadrtic_f(float r)
+{
+    return 1.0 / (r*r + 1);
+}
+
+
+static __device__ double gaussian(double r)
+{
+    return exp(-r*r);
+}
+
+static __device__ float gaussian_f(float r)
+{
+    return exp(-r*r);
+}
+
+
+static __device__ double quintic(double r)
+{
+    double r2 = r*r;
+    return -r2*r2*r;
+}
+
+static __device__ float qunitic_f(float r)
+{
+    float r2 = r*r;
+    return -r2*r2*r;
+}
+
+"""
+
+linear = cp._core.create_ufunc(
+    'cupyx_scipy_interpolate_linear',
+    (('f->f', 'out0 = linear_f(in0)'),
+     'd->d'),
+    'out0 = linear(in0)',
+    preamble=kernel_definitions,
+    doc="""Linear kernel function.
+
+    ``-r``
+    """,
+)
+
+cubic = cp._core.create_ufunc(
+    'cupyx_scipy_interpolate_cubic',
+    (('f->f', 'out0 = cubic_f(in0)'),
+     'd->d'),
+    'out0 = cubic(in0)',
+    preamble=kernel_definitions,
+    doc="""Cubic kernel function.
+
+    ``r**3``
+    """,
+)
+
+thin_plate_spline = cp._core.create_ufunc(
+    'cupyx_scipy_interpolate_thin_plate_spline',
+    (('f->f', 'out0 = thin_plate_spline_f(in0)'),
+     'd->d'),
+    'out0 = thin_plate_spline(in0)',
+    preamble=kernel_definitions,
+    doc="""Thin-plate spline kernel function.
+
+    ``r**2 * log(r) if r != 0 else 0``
+    """,
+)
+
+
+multiquadric = cp._core.create_ufunc(
+    'cupyx_scipy_interpolate_multiquadric',
+    (('f->f', 'out0 = multiquadric_f(in0)'),
+     'd->d'),
+    'out0 = multiquadric(in0)',
+    preamble=kernel_definitions,
+    doc="""Multiquadric kernel function.
+
+    ``-sqrt(r**2 + 1)``
+    """,
+)
+
+
+inverse_multiquadric = cp._core.create_ufunc(
+    'cupyx_scipy_interpolate_inverse_multiquadric',
+    (('f->f', 'out0 = inverse_multiquadric_f(in0)'),
+     'd->d'),
+    'out0 = inverse_multiquadric(in0)',
+    preamble=kernel_definitions,
+    doc="""Inverse multiquadric kernel function.
+
+    ``1 / sqrt(r**2 + 1)``
+    """,
+)
+
+
+inverse_quadratic = cp._core.create_ufunc(
+    'cupyx_scipy_interpolate_inverse_quadratic',
+    (('f->f', 'out0 = inverse_quadratic_f(in0)'),
+     'd->d'),
+    'out0 = inverse_quadratic(in0)',
+    preamble=kernel_definitions,
+    doc="""Inverse quadratic kernel function.
+
+    ``1 / (r**2 + 1)``
+    """,
+)
+
+
+gaussian = cp._core.create_ufunc(
+    'cupyx_scipy_interpolate_gaussian',
+    (('f->f', 'out0 = gaussian_f(in0)'),
+     'd->d'),
+    'out0 = gaussian(in0)',
+    preamble=kernel_definitions,
+    doc="""Gaussian kernel function.
+
+    ``exp(-r**2)``
+    """,
+)
+
+
+quintic = cp._core.create_ufunc(
+    'cupyx_scipy_interpolate_quintic',
+    (('f->f', 'out0 = quintic_f(in0)'),
+     'd->d'),
+    'out0 = quintic(in0)',
+    preamble=kernel_definitions,
+    doc="""Quintic kernel function.
+
+    ``-r**5``
+    """,
+)
+
+
+NAME_TO_FUNC = {
+    "linear": linear,
+    "thin_plate_spline": thin_plate_spline,
+    "cubic": cubic,
+    "quintic": quintic,
+    "multiquadric": multiquadric,
+    "inverse_multiquadric": inverse_multiquadric,
+    "inverse_quadratic": inverse_quadratic,
+    "gaussian": gaussian
+}
+
+
+def kernel_matrix(x, kernel_func, out):
+    """Evaluate RBFs, with centers at `x`, at `x`."""
+    delta = x[None, :, :] - x[:, None, :]
+    out[...] = kernel_func(cp.linalg.norm(delta, axis=-1))
+# The above is equivalent to the original semi-scalar version:
+#        for j in range(i+1):
+#            out[i, j] = kernel_func(cp.linalg.norm(x[i] - x[j]))
+#            out[j, i] = out[i, j]
+
+
+def polynomial_matrix(x, powers, out):
+    """Evaluate monomials, with exponents from `powers`, at `x`."""
+    pwr = x[:, None, :] ** powers[None, :, :]
+    cp.prod(pwr, axis=-1, out=out)
+# The above is equivalent to the following loop
+#    for i in range(x.shape[0]):
+#        for j in range(powers.shape[0]):
+#            out[i, j] = cp.prod(x[i]**powers[j])
+
+
+def _build_system(y, d, smoothing, kernel, epsilon, powers):
+    """Build the system used to solve for the RBF interpolant coefficients.
+
+    Parameters
+    ----------
+    y : (P, N) float ndarray
+        Data point coordinates.
+    d : (P, S) float ndarray
+        Data values at `y`.
+    smoothing : (P,) float ndarray
+        Smoothing parameter for each data point.
+    kernel : str
+        Name of the RBF.
+    epsilon : float
+        Shape parameter.
+    powers : (R, N) int ndarray
+        The exponents for each monomial in the polynomial.
+
+    Returns
+    -------
+    lhs : (P + R, P + R) float ndarray
+        Left-hand side matrix.
+    rhs : (P + R, S) float ndarray
+        Right-hand side matrix.
+    shift : (N,) float ndarray
+        Domain shift used to create the polynomial matrix.
+    scale : (N,) float ndarray
+        Domain scaling used to create the polynomial matrix.
+
+    """
+    p = d.shape[0]
+    s = d.shape[1]
+    r = powers.shape[0]
+    kernel_func = NAME_TO_FUNC[kernel]
+
+    # Shift and scale the polynomial domain to be between -1 and 1
+    mins = cp.min(y, axis=0)
+    maxs = cp.max(y, axis=0)
+    shift = (maxs + mins)/2
+    scale = (maxs - mins)/2
+    # The scale may be zero if there is a single point or all the points have
+    # the same value for some dimension. Avoid division by zero by replacing
+    # zeros with ones.
+    scale[scale == 0.0] = 1.0
+
+    yeps = y * epsilon
+    yhat = (y - shift)/scale
+
+    # Transpose to make the array fortran contiguous. This is required for
+    # dgesv to not make a copy of lhs.
+    lhs = cp.empty((p + r, p + r), dtype=float).T
+    kernel_matrix(yeps, kernel_func, lhs[:p, :p])
+    polynomial_matrix(yhat, powers, lhs[:p, p:])
+    lhs[p:, :p] = lhs[:p, p:].T
+    lhs[p:, p:] = 0.0
+    for i in range(p):
+        lhs[i, i] += smoothing[i]
+
+    # Transpose to make the array fortran contiguous.
+    rhs = cp.empty((s, p + r), dtype=float).T
+    rhs[:p] = d
+    rhs[p:] = 0.0
+
+    return lhs, rhs, shift, scale
+
+
+def _build_evaluation_coefficients(x, y, kernel, epsilon, powers,
+                                   shift, scale):
+    """Construct the coefficients needed to evaluate
+    the RBF.
+
+    Parameters
+    ----------
+    x : (Q, N) float ndarray
+        Evaluation point coordinates.
+    y : (P, N) float ndarray
+        Data point coordinates.
+    kernel : str
+        Name of the RBF.
+    epsilon : float
+        Shape parameter.
+    powers : (R, N) int ndarray
+        The exponents for each monomial in the polynomial.
+    shift : (N,) float ndarray
+        Shifts the polynomial domain for numerical stability.
+    scale : (N,) float ndarray
+        Scales the polynomial domain for numerical stability.
+
+    Returns
+    -------
+    (Q, P + R) float ndarray
+
+    """
+    q = x.shape[0]
+    p = y.shape[0]
+    r = powers.shape[0]
+    kernel_func = NAME_TO_FUNC[kernel]
+
+    yeps = y*epsilon
+    xeps = x*epsilon
+    xhat = (x - shift)/scale
+
+    vec = cp.empty((q, p + r), dtype=float)
+
+    # Evaluate RBFs, with centers at `y`, at the point `x`.
+    delta = xeps[:, None, :] - yeps[None, :, :]
+    vec[:, :p] = kernel_func(cp.linalg.norm(delta, axis=-1))
+
+    # Evaluate monomials, with exponents from `powers`, at the point `x`.
+    pwr = xhat[:, None, :]**powers[None, :, :]
+    vec[:, p:] = cp.prod(pwr, axis=-1)
+#    for i in range(q):
+#        polynomial_vector(xhat[i], powers, vec[i, p:])
+
+    return vec
+
+
+###############################################################################
+
+# These RBFs are implemented.
+_AVAILABLE = {
+    "linear",
+    "thin_plate_spline",
+    "cubic",
+    "quintic",
+    "multiquadric",
+    "inverse_multiquadric",
+    "inverse_quadratic",
+    "gaussian"
+}
+
+
+# The shape parameter does not need to be specified when using these RBFs.
+_SCALE_INVARIANT = {"linear", "thin_plate_spline", "cubic", "quintic"}
+
+
+# For RBFs that are conditionally positive definite of order m, the interpolant
+# should include polynomial terms with degree >= m - 1. Define the minimum
+# degrees here. These values are from Chapter 8 of Fasshauer's "Meshfree
+# Approximation Methods with MATLAB". The RBFs that are not in this dictionary
+# are positive definite and do not need polynomial terms.
+_NAME_TO_MIN_DEGREE = {
+    "multiquadric": 0,
+    "linear": 0,
+    "thin_plate_spline": 1,
+    "cubic": 1,
+    "quintic": 2
+}
+
+
+try:
+    _comb = math.comb
+except AttributeError:
+    # Naive combination for Python 3.7
+    def _comb(n, k):
+        return math.factorial(n) // (math.factorial(n - k) * math.factorial(k))
+
+
+def _monomial_powers(ndim, degree):
+    """Return the powers for each monomial in a polynomial.
+
+    Parameters
+    ----------
+    ndim : int
+        Number of variables in the polynomial.
+    degree : int
+        Degree of the polynomial.
+
+    Returns
+    -------
+    (nmonos, ndim) int ndarray
+        Array where each row contains the powers for each variable in a
+        monomial.
+
+    """
+    nmonos = _comb(degree + ndim, ndim)
+    out = cp.zeros((nmonos, ndim), dtype=int)
+    count = 0
+    for deg in range(degree + 1):
+        for mono in combinations_with_replacement(range(ndim), deg):
+            # `mono` is a tuple of variables in the current monomial with
+            # multiplicity indicating power (e.g., (0, 1, 1) represents x*y**2)
+            for var in mono:
+                out[count, var] += 1
+
+            count += 1
+
+    return out
+
+
+def _build_and_solve_system(y, d, smoothing, kernel, epsilon, powers):
+    """Build and solve the RBF interpolation system of equations.
+
+    Parameters
+    ----------
+    y : (P, N) float ndarray
+        Data point coordinates.
+    d : (P, S) float ndarray
+        Data values at `y`.
+    smoothing : (P,) float ndarray
+        Smoothing parameter for each data point.
+    kernel : str
+        Name of the RBF.
+    epsilon : float
+        Shape parameter.
+    powers : (R, N) int ndarray
+        The exponents for each monomial in the polynomial.
+
+    Returns
+    -------
+    coeffs : (P + R, S) float ndarray
+        Coefficients for each RBF and monomial.
+    shift : (N,) float ndarray
+        Domain shift used to create the polynomial matrix.
+    scale : (N,) float ndarray
+        Domain scaling used to create the polynomial matrix.
+
+    """
+    lhs, rhs, shift, scale = _build_system(
+        y, d, smoothing, kernel, epsilon, powers
+    )
+    coeffs = cp.linalg.solve(lhs, rhs)
+    return shift, scale, coeffs
+
+
+class RBFInterpolator:
+    """Radial basis function (RBF) interpolation in N dimensions.
+
+    Parameters
+    ----------
+    y : (P, N) array_like
+        Data point coordinates.
+    d : (P, ...) array_like
+        Data values at `y`.
+    neighbors : int, optional
+        If specified, the value of the interpolant at each evaluation point
+        will be computed using only this many nearest data points. All the data
+        points are used by default.
+    smoothing : float or (P,) array_like, optional
+        Smoothing parameter. The interpolant perfectly fits the data when this
+        is set to 0. For large values, the interpolant approaches a least
+        squares fit of a polynomial with the specified degree. Default is 0.
+    kernel : str, optional
+        Type of RBF. This should be one of
+
+            - 'linear'               : ``-r``
+            - 'thin_plate_spline'    : ``r**2 * log(r)``
+            - 'cubic'                : ``r**3``
+            - 'quintic'              : ``-r**5``
+            - 'multiquadric'         : ``-sqrt(1 + r**2)``
+            - 'inverse_multiquadric' : ``1/sqrt(1 + r**2)``
+            - 'inverse_quadratic'    : ``1/(1 + r**2)``
+            - 'gaussian'             : ``exp(-r**2)``
+
+        Default is 'thin_plate_spline'.
+    epsilon : float, optional
+        Shape parameter that scales the input to the RBF. If `kernel` is
+        'linear', 'thin_plate_spline', 'cubic', or 'quintic', this defaults to
+        1 and can be ignored because it has the same effect as scaling the
+        smoothing parameter. Otherwise, this must be specified.
+    degree : int, optional
+        Degree of the added polynomial. For some RBFs the interpolant may not
+        be well-posed if the polynomial degree is too small. Those RBFs and
+        their corresponding minimum degrees are
+
+            - 'multiquadric'      : 0
+            - 'linear'            : 0
+            - 'thin_plate_spline' : 1
+            - 'cubic'             : 1
+            - 'quintic'           : 2
+
+        The default value is the minimum degree for `kernel` or 0 if there is
+        no minimum degree. Set this to -1 for no added polynomial.
+
+    Notes
+    -----
+    An RBF is a scalar valued function in N-dimensional space whose value at
+    :math:`x` can be expressed in terms of :math:`r=||x - c||`, where :math:`c`
+    is the center of the RBF.
+
+    An RBF interpolant for the vector of data values :math:`d`, which are from
+    locations :math:`y`, is a linear combination of RBFs centered at :math:`y`
+    plus a polynomial with a specified degree. The RBF interpolant is written
+    as
+
+    .. math::
+        f(x) = K(x, y) a + P(x) b,
+
+    where :math:`K(x, y)` is a matrix of RBFs with centers at :math:`y`
+    evaluated at the points :math:`x`, and :math:`P(x)` is a matrix of
+    monomials, which span polynomials with the specified degree, evaluated at
+    :math:`x`. The coefficients :math:`a` and :math:`b` are the solution to the
+    linear equations
+
+    .. math::
+        (K(y, y) + \\lambda I) a + P(y) b = d
+
+    and
+
+    .. math::
+        P(y)^T a = 0,
+
+    where :math:`\\lambda` is a non-negative smoothing parameter that controls
+    how well we want to fit the data. The data are fit exactly when the
+    smoothing parameter is 0.
+
+    The above system is uniquely solvable if the following requirements are
+    met:
+
+        - :math:`P(y)` must have full column rank. :math:`P(y)` always has full
+          column rank when `degree` is -1 or 0. When `degree` is 1,
+          :math:`P(y)` has full column rank if the data point locations are not
+          all collinear (N=2), coplanar (N=3), etc.
+        - If `kernel` is 'multiquadric', 'linear', 'thin_plate_spline',
+          'cubic', or 'quintic', then `degree` must not be lower than the
+          minimum value listed above.
+        - If `smoothing` is 0, then each data point location must be distinct.
+
+    When using an RBF that is not scale invariant ('multiquadric',
+    'inverse_multiquadric', 'inverse_quadratic', or 'gaussian'), an appropriate
+    shape parameter must be chosen (e.g., through cross validation). Smaller
+    values for the shape parameter correspond to wider RBFs. The problem can
+    become ill-conditioned or singular when the shape parameter is too small.
+
+    The memory required to solve for the RBF interpolation coefficients
+    increases quadratically with the number of data points, which can become
+    impractical when interpolating more than about a thousand data points.
+    To overcome memory limitations for large interpolation problems, the
+    `neighbors` argument can be specified to compute an RBF interpolant for
+    each evaluation point using only the nearest data points.
+
+    See Also
+    --------
+    scipy.interpolate.RBFInterpolator
+
+    """
+
+    def __init__(self, y, d,
+                 neighbors=None,
+                 smoothing=0.0,
+                 kernel="thin_plate_spline",
+                 epsilon=None,
+                 degree=None):
+        y = cp.asarray(y, dtype=float, order="C")
+        if y.ndim != 2:
+            raise ValueError("`y` must be a 2-dimensional array.")
+
+        ny, ndim = y.shape
+
+        d_dtype = complex if cp.iscomplexobj(d) else float
+        d = cp.asarray(d, dtype=d_dtype, order="C")
+        if d.shape[0] != ny:
+            raise ValueError(
+                f"Expected the first axis of `d` to have length {ny}."
+            )
+
+        d_shape = d.shape[1:]
+        d = d.reshape((ny, -1))
+        # If `d` is complex, convert it to a float array with twice as many
+        # columns. Otherwise, the LHS matrix would need to be converted to
+        # complex and take up 2x more memory than necessary.
+        d = d.view(float)
+
+        isscalar = cp.isscalar(smoothing) or smoothing.shape == ()
+        if isscalar:
+            smoothing = cp.full(ny, smoothing, dtype=float)
+        else:
+            smoothing = cp.asarray(smoothing, dtype=float, order="C")
+            if smoothing.shape != (ny,):
+                raise ValueError(
+                    "Expected `smoothing` to be a scalar or have shape "
+                    f"({ny},)."
+                )
+
+        kernel = kernel.lower()
+        if kernel not in _AVAILABLE:
+            raise ValueError(f"`kernel` must be one of {_AVAILABLE}.")
+
+        if epsilon is None:
+            if kernel in _SCALE_INVARIANT:
+                epsilon = 1.0
+            else:
+                raise ValueError(
+                    "`epsilon` must be specified if `kernel` is not one of "
+                    f"{_SCALE_INVARIANT}."
+                )
+        else:
+            epsilon = float(epsilon)
+
+        min_degree = _NAME_TO_MIN_DEGREE.get(kernel, -1)
+        if degree is None:
+            degree = max(min_degree, 0)
+        else:
+            degree = int(degree)
+            if degree < -1:
+                raise ValueError("`degree` must be at least -1.")
+            elif degree < min_degree:
+                warnings.warn(
+                    f"`degree` should not be below {min_degree} when `kernel` "
+                    f"is '{kernel}'. The interpolant may not be uniquely "
+                    "solvable, and the smoothing parameter may have an "
+                    "unintuitive effect.",
+                    UserWarning
+                )
+
+        if neighbors is None:
+            nobs = ny
+        else:
+            raise NotImplementedError("neighbors is not implemented yet")
+            # Make sure the number of nearest neighbors used for interpolation
+            # does not exceed the number of observations.
+            neighbors = int(min(neighbors, ny))
+            nobs = neighbors
+
+        powers = _monomial_powers(ndim, degree)
+        # The polynomial matrix must have full column rank in order for the
+        # interpolant to be well-posed, which is not possible if there are
+        # fewer observations than monomials.
+        if powers.shape[0] > nobs:
+            raise ValueError(
+                f"At least {powers.shape[0]} data points are required when "
+                f"`degree` is {degree} and the number of dimensions is {ndim}."
+            )
+
+        if neighbors is None:
+            shift, scale, coeffs = _build_and_solve_system(
+                y, d, smoothing, kernel, epsilon, powers
+            )
+
+            # Make these attributes private since they do not always exist.
+            self._shift = shift
+            self._scale = scale
+            self._coeffs = coeffs
+
+        else:
+            raise NotImplementedError
+            # self._tree = KDTree(y)
+
+        self.y = y
+        self.d = d
+        self.d_shape = d_shape
+        self.d_dtype = d_dtype
+        self.neighbors = neighbors
+        self.smoothing = smoothing
+        self.kernel = kernel
+        self.epsilon = epsilon
+        self.powers = powers
+
+    def _chunk_evaluator(self, x, y, shift, scale, coeffs,
+                         memory_budget=1000000):
+        """
+        Evaluate the interpolation.
+
+        Parameters
+        ----------
+        x : (Q, N) float ndarray
+            array of points on which to evaluate
+        y: (P, N) float ndarray
+            array of points on which we know function values
+        shift: (N, ) ndarray
+            Domain shift used to create the polynomial matrix.
+        scale : (N,) float ndarray
+            Domain scaling used to create the polynomial matrix.
+        coeffs: (P+R, S) float ndarray
+            Coefficients in front of basis functions
+
+        Returns
+        -------
+        (Q, S) float ndarray
+        Interpolated array
+        """
+        nx, ndim = x.shape
+        nnei = len(y)
+
+        # in each chunk we consume the same space we already occupy
+        chunksize = memory_budget // ((self.powers.shape[0] + nnei)) + 1
+        if chunksize <= nx:
+            out = cp.empty((nx, self.d.shape[1]), dtype=float)
+            for i in range(0, nx, chunksize):
+                vec = _build_evaluation_coefficients(
+                    x[i:i + chunksize, :],
+                    y,
+                    self.kernel,
+                    self.epsilon,
+                    self.powers,
+                    shift,
+                    scale)
+                out[i:i + chunksize, :] = cp.dot(vec, coeffs)
+        else:
+            vec = _build_evaluation_coefficients(
+                x,
+                y,
+                self.kernel,
+                self.epsilon,
+                self.powers,
+                shift,
+                scale)
+            out = cp.dot(vec, coeffs)
+
+        return out
+
+    def __call__(self, x):
+        """Evaluate the interpolant at `x`.
+
+        Parameters
+        ----------
+        x : (Q, N) array_like
+            Evaluation point coordinates.
+
+        Returns
+        -------
+        (Q, ...) ndarray
+            Values of the interpolant at `x`.
+
+        """
+        x = cp.asarray(x, dtype=float, order="C")
+        if x.ndim != 2:
+            raise ValueError("`x` must be a 2-dimensional array.")
+
+        nx, ndim = x.shape
+        if ndim != self.y.shape[1]:
+            raise ValueError("Expected the second axis of `x` to have length "
+                             f"{self.y.shape[1]}.")
+
+        # Our memory budget for storing RBF coefficients is
+        # based on how many floats in memory we already occupy
+        # If this number is below 1e6 we just use 1e6
+        # This memory budget is used to decide how we chunk
+        # the inputs
+        memory_budget = max(x.size + self.y.size + self.d.size, 1000000)
+
+        if self.neighbors is None:
+            out = self._chunk_evaluator(
+                x,
+                self.y,
+                self._shift,
+                self._scale,
+                self._coeffs, memory_budget=memory_budget)
+        else:
+            raise NotImplementedError    # XXX: needs KDTree
+
+        out = out.view(self.d_dtype)
+        out = out.reshape((nx, ) + self.d_shape)
+        return out
diff --git a/cupyx/scipy/interpolate/_rgi.py b/cupyx/scipy/interpolate/_rgi.py
new file mode 100644
index 0000000..35d210e
--- /dev/null
+++ b/cupyx/scipy/interpolate/_rgi.py
@@ -0,0 +1,622 @@
+__all__ = ['RegularGridInterpolator', 'interpn']
+
+import itertools
+import cupy as cp
+from cupyx.scipy.interpolate._bspline2 import make_interp_spline
+from cupyx.scipy.interpolate._cubic import PchipInterpolator
+
+
+def _ndim_coords_from_arrays(points, ndim=None):
+    """
+    Convert a tuple of coordinate arrays to a (..., ndim)-shaped array.
+    """
+    if isinstance(points, tuple) and len(points) == 1:
+        # handle argument tuple
+        points = points[0]
+    if isinstance(points, tuple):
+        p = cp.broadcast_arrays(*points)
+        n = len(p)
+        for j in range(1, n):
+            if p[j].shape != p[0].shape:
+                raise ValueError(
+                    "coordinate arrays do not have the same shape")
+        points = cp.empty(p[0].shape + (len(points),), dtype=float)
+        for j, item in enumerate(p):
+            points[..., j] = item
+    else:
+        points = cp.asanyarray(points)
+        if points.ndim == 1:
+            if ndim is None:
+                points = points.reshape(-1, 1)
+            else:
+                points = points.reshape(-1, ndim)
+    return points
+
+
+def _check_points(points):
+    descending_dimensions = []
+    grid = []
+    for i, p in enumerate(points):
+        # early make points float
+        # see https://github.com/scipy/scipy/pull/17230
+        p = cp.asarray(p, dtype=float)
+        if not cp.all(p[1:] > p[:-1]):
+            if cp.all(p[1:] < p[:-1]):
+                # input is descending, so make it ascending
+                descending_dimensions.append(i)
+                p = cp.flip(p)
+                p = cp.ascontiguousarray(p)
+            else:
+                raise ValueError(
+                    "The points in dimension %d must be strictly "
+                    "ascending or descending" % i)
+        grid.append(p)
+    return tuple(grid), tuple(descending_dimensions)
+
+
+def _check_dimensionality(points, values):
+    if len(points) > values.ndim:
+        raise ValueError("There are %d point arrays, but values has %d "
+                         "dimensions" % (len(points), values.ndim))
+    for i, p in enumerate(points):
+        if not cp.asarray(p).ndim == 1:
+            raise ValueError("The points in dimension %d must be "
+                             "1-dimensional" % i)
+        if not values.shape[i] == len(p):
+            raise ValueError("There are %d points and %d values in "
+                             "dimension %d" % (len(p), values.shape[i], i))
+
+
+class RegularGridInterpolator:
+    """
+    Interpolation on a regular or rectilinear grid in arbitrary dimensions.
+
+    The data must be defined on a rectilinear grid; that is, a rectangular
+    grid with even or uneven spacing. Linear and nearest-neighbor
+    interpolations are supported. After setting up the interpolator object,
+    the interpolation method may be chosen at each evaluation.
+
+    Parameters
+    ----------
+    points : tuple of ndarray of float, with shapes (m1, ), ..., (mn, )
+        The points defining the regular grid in n dimensions. The points in
+        each dimension (i.e. every elements of the points tuple) must be
+        strictly ascending or descending.
+
+    values : ndarray, shape (m1, ..., mn, ...)
+        The data on the regular grid in n dimensions. Complex data can be
+        acceptable.
+
+    method : str, optional
+        The method of interpolation to perform. Supported are "linear",
+        "nearest", "slinear", "cubic", "quintic" and "pchip".
+        This parameter will become the default for the object's
+        ``__call__`` method. Default is "linear".
+
+    bounds_error : bool, optional
+        If True, when interpolated values are requested outside of the
+        domain of the input data, a ValueError is raised.
+        If False, then `fill_value` is used.
+        Default is True.
+
+    fill_value : float or None, optional
+        The value to use for points outside of the interpolation domain.
+        If None, values outside the domain are extrapolated.
+        Default is ``cp.nan``.
+
+    Notes
+    -----
+    Contrary to scipy's `LinearNDInterpolator` and `NearestNDInterpolator`,
+    this class avoids expensive triangulation of the input data by taking
+    advantage of the regular grid structure.
+
+    In other words, this class assumes that the data is defined on a
+    *rectilinear* grid.
+
+    If the input data is such that dimensions have incommensurate
+    units and differ by many orders of magnitude, the interpolant may have
+    numerical artifacts. Consider rescaling the data before interpolating.
+
+    Examples
+    --------
+    **Evaluate a function on the points of a 3-D grid**
+
+    As a first example, we evaluate a simple example function on the points of
+    a 3-D grid:
+
+    >>> from cupyx.scipy.interpolate import RegularGridInterpolator
+    >>> import cupy as cp
+    >>> def f(x, y, z):
+    ...     return 2 * x**3 + 3 * y**2 - z
+    >>> x = cp.linspace(1, 4, 11)
+    >>> y = cp.linspace(4, 7, 22)
+    >>> z = cp.linspace(7, 9, 33)
+    >>> xg, yg ,zg = cp.meshgrid(x, y, z, indexing='ij', sparse=True)
+    >>> data = f(xg, yg, zg)
+
+    ``data`` is now a 3-D array with ``data[i, j, k] = f(x[i], y[j], z[k])``.
+    Next, define an interpolating function from this data:
+
+    >>> interp = RegularGridInterpolator((x, y, z), data)
+
+    Evaluate the interpolating function at the two points
+    ``(x,y,z) = (2.1, 6.2, 8.3)`` and ``(3.3, 5.2, 7.1)``:
+
+    >>> pts = cp.array([[2.1, 6.2, 8.3],
+    ...                 [3.3, 5.2, 7.1]])
+    >>> interp(pts)
+    array([ 125.80469388,  146.30069388])
+
+    which is indeed a close approximation to
+
+    >>> f(2.1, 6.2, 8.3), f(3.3, 5.2, 7.1)
+    (125.54200000000002, 145.894)
+
+    **Interpolate and extrapolate a 2D dataset**
+
+    As a second example, we interpolate and extrapolate a 2D data set:
+
+    >>> x, y = cp.array([-2, 0, 4]), cp.array([-2, 0, 2, 5])
+    >>> def ff(x, y):
+    ...     return x**2 + y**2
+
+    >>> xg, yg = cp.meshgrid(x, y, indexing='ij')
+    >>> data = ff(xg, yg)
+    >>> interp = RegularGridInterpolator((x, y), data,
+    ...                                  bounds_error=False, fill_value=None)
+
+    >>> import matplotlib.pyplot as plt
+    >>> fig = plt.figure()
+    >>> ax = fig.add_subplot(projection='3d')
+    >>> ax.scatter(xg.ravel().get(), yg.ravel().get(), data.ravel().get(),
+    ...            s=60, c='k', label='data')
+
+    Evaluate and plot the interpolator on a finer grid
+
+    >>> xx = cp.linspace(-4, 9, 31)
+    >>> yy = cp.linspace(-4, 9, 31)
+    >>> X, Y = cp.meshgrid(xx, yy, indexing='ij')
+
+    >>> # interpolator
+    >>> ax.plot_wireframe(X.get(), Y.get(), interp((X, Y)).get(),
+                          rstride=3, cstride=3, alpha=0.4, color='m',
+                          label='linear interp')
+
+    >>> # ground truth
+    >>> ax.plot_wireframe(X.get(), Y.get(), ff(X, Y).get(),
+                          rstride=3, cstride=3,
+    ...                   alpha=0.4, label='ground truth')
+    >>> plt.legend()
+    >>> plt.show()
+
+    See Also
+    --------
+    interpn : a convenience function which wraps `RegularGridInterpolator`
+
+    scipy.ndimage.map_coordinates : interpolation on grids with equal spacing
+                                    (suitable for e.g., N-D image resampling)
+
+    References
+    ----------
+    [1] Python package *regulargrid* by Johannes Buchner, see
+        https://pypi.python.org/pypi/regulargrid/
+    [2] Wikipedia, "Trilinear interpolation",
+        https://en.wikipedia.org/wiki/Trilinear_interpolation
+    [3] Weiser, Alan, and Sergio E. Zarantonello. "A note on piecewise
+        linear and multilinear table interpolation in many dimensions."
+        MATH. COMPUT. 50.181 (1988): 189-196.
+        https://www.ams.org/journals/mcom/1988-50-181/S0025-5718-1988-0917826-0/S0025-5718-1988-0917826-0.pdf
+    """
+    # this class is based on code originally programmed by Johannes Buchner,
+    # see https://github.com/JohannesBuchner/regulargrid
+
+    _SPLINE_DEGREE_MAP = {"slinear": 1, "cubic": 3, "quintic": 5, 'pchip': 3}
+    _SPLINE_METHODS = list(_SPLINE_DEGREE_MAP.keys())
+    _ALL_METHODS = ["linear", "nearest"] + _SPLINE_METHODS
+
+    def __init__(self, points, values, method="linear", bounds_error=True,
+                 fill_value=cp.nan):
+        if method not in self._ALL_METHODS:
+            raise ValueError("Method '%s' is not defined" % method)
+        elif method in self._SPLINE_METHODS:
+            self._validate_grid_dimensions(points, method)
+
+        self.method = method
+        self.bounds_error = bounds_error
+        self.grid, self._descending_dimensions = _check_points(points)
+        self.values = self._check_values(values)
+        self._check_dimensionality(self.grid, self.values)
+        self.fill_value = self._check_fill_value(self.values, fill_value)
+        if self._descending_dimensions:
+            self.values = cp.flip(values, axis=self._descending_dimensions)
+
+    def _check_dimensionality(self, grid, values):
+        _check_dimensionality(grid, values)
+
+    def _validate_grid_dimensions(self, points, method):
+        k = self._SPLINE_DEGREE_MAP[method]
+        for i, point in enumerate(points):
+            ndim = len(cp.atleast_1d(point))
+            if ndim <= k:
+                raise ValueError(f"There are {ndim} points in dimension {i},"
+                                 f" but method {method} requires at least "
+                                 f" {k+1} points per dimension.")
+
+    def _check_points(self, points):
+        return _check_points(points)
+
+    def _check_values(self, values):
+        if not cp.issubdtype(values.dtype, cp.inexact):
+            values = values.astype(float)
+
+        return values
+
+    def _check_fill_value(self, values, fill_value):
+        if fill_value is not None:
+            fill_value_dtype = cp.asarray(fill_value).dtype
+            if (hasattr(values, 'dtype') and
+                not cp.can_cast(fill_value_dtype, values.dtype,
+                                casting='same_kind')):
+                raise ValueError("fill_value must be either 'None' or "
+                                 "of a type compatible with values")
+        return fill_value
+
+    def __call__(self, xi, method=None):
+        """
+        Interpolation at coordinates.
+
+        Parameters
+        ----------
+        xi : cupy.ndarray of shape (..., ndim)
+            The coordinates to evaluate the interpolator at.
+
+        method : str, optional
+            The method of interpolation to perform. Supported are "linear" and
+            "nearest".  Default is the method chosen when the interpolator was
+            created.
+
+        Returns
+        -------
+        values_x : cupy.ndarray, shape xi.shape[:-1] + values.shape[ndim:]
+            Interpolated values at `xi`. See notes for behaviour when
+            ``xi.ndim == 1``.
+
+        Notes
+        -----
+        In the case that ``xi.ndim == 1`` a new axis is inserted into
+        the 0 position of the returned array, values_x, so its shape is
+        instead ``(1,) + values.shape[ndim:]``.
+
+        Examples
+        --------
+        Here we define a nearest-neighbor interpolator of a simple function
+
+        >>> import cupy as cp
+        >>> x, y = cp.array([0, 1, 2]), cp.array([1, 3, 7])
+        >>> def f(x, y):
+        ...     return x**2 + y**2
+        >>> data = f(*cp.meshgrid(x, y, indexing='ij', sparse=True))
+        >>> from cupyx.scipy.interpolate import RegularGridInterpolator
+        >>> interp = RegularGridInterpolator((x, y), data, method='nearest')
+
+        By construction, the interpolator uses the nearest-neighbor
+        interpolation
+
+        >>> interp([[1.5, 1.3], [0.3, 4.5]])
+        array([2., 9.])
+
+        We can however evaluate the linear interpolant by overriding the
+        `method` parameter
+
+        >>> interp([[1.5, 1.3], [0.3, 4.5]], method='linear')
+        array([ 4.7, 24.3])
+        """
+        is_method_changed = self.method != method
+        method = self.method if method is None else method
+        if method not in self._ALL_METHODS:
+            raise ValueError("Method '%s' is not defined" % method)
+
+        xi, xi_shape, ndim, nans, out_of_bounds = self._prepare_xi(xi)
+
+        if method == "linear":
+            indices, norm_distances = self._find_indices(xi.T)
+            result = self._evaluate_linear(indices, norm_distances)
+        elif method == "nearest":
+            indices, norm_distances = self._find_indices(xi.T)
+            result = self._evaluate_nearest(indices, norm_distances)
+        elif method in self._SPLINE_METHODS:
+            if is_method_changed:
+                self._validate_grid_dimensions(self.grid, method)
+            result = self._evaluate_spline(xi, method)
+
+        if not self.bounds_error and self.fill_value is not None:
+            result[out_of_bounds] = self.fill_value
+
+        if nans.ndim < result.ndim:
+            nans = nans[..., None]
+        result = cp.where(nans, cp.nan, result)
+        return result.reshape(xi_shape[:-1] + self.values.shape[ndim:])
+
+    def _prepare_xi(self, xi):
+        ndim = len(self.grid)
+        xi = _ndim_coords_from_arrays(xi, ndim=ndim)
+        if xi.shape[-1] != len(self.grid):
+            raise ValueError("The requested sample points xi have dimension "
+                             f"{xi.shape[-1]} but this "
+                             f"RegularGridInterpolator has dimension {ndim}")
+
+        xi_shape = xi.shape
+        xi = xi.reshape(-1, xi_shape[-1])
+        xi = cp.asarray(xi, dtype=float)
+
+        # find nans in input
+        is_nans = cp.isnan(xi).T
+        nans = is_nans[0].copy()
+        for is_nan in is_nans[1:]:
+            cp.logical_or(nans, is_nan, nans)
+
+        if self.bounds_error:
+            for i, p in enumerate(xi.T):
+                if not cp.logical_and(cp.all(self.grid[i][0] <= p),
+                                      cp.all(p <= self.grid[i][-1])):
+                    raise ValueError("One of the requested xi is out of bounds"
+                                     " in dimension %d" % i)
+            out_of_bounds = None
+        else:
+            out_of_bounds = self._find_out_of_bounds(xi.T)
+
+        return xi, xi_shape, ndim, nans, out_of_bounds
+
+    def _evaluate_linear(self, indices, norm_distances):
+        # slice for broadcasting over trailing dimensions in self.values
+        vslice = (slice(None),) + (None,)*(self.values.ndim - len(indices))
+
+        # Compute shifting up front before zipping everything together
+        shift_norm_distances = [1 - yi for yi in norm_distances]
+        shift_indices = [i + 1 for i in indices]
+
+        # The formula for linear interpolation in 2d takes the form:
+        # values = self.values[(i0, i1)] * (1 - y0) * (1 - y1) + \
+        #          self.values[(i0, i1 + 1)] * (1 - y0) * y1 + \
+        #          self.values[(i0 + 1, i1)] * y0 * (1 - y1) + \
+        #          self.values[(i0 + 1, i1 + 1)] * y0 * y1
+        # We pair i with 1 - yi (zipped1) and i + 1 with yi (zipped2)
+        zipped1 = zip(indices, shift_norm_distances)
+        zipped2 = zip(shift_indices, norm_distances)
+
+        # Take all products of zipped1 and zipped2 and iterate over them
+        # to get the terms in the above formula. This corresponds to iterating
+        # over the vertices of a hypercube.
+        hypercube = itertools.product(*zip(zipped1, zipped2))
+        value = cp.array([0.])
+        for h in hypercube:
+            edge_indices, weights = zip(*h)
+            term = cp.asarray(self.values[edge_indices])
+            for w in weights:
+                term *= w[vslice]
+            value = value + term   # cannot use += because broadcasting
+        return value
+
+    def _evaluate_nearest(self, indices, norm_distances):
+        idx_res = [cp.where(yi <= .5, i, i + 1)
+                   for i, yi in zip(indices, norm_distances)]
+        return self.values[tuple(idx_res)]
+
+    def _evaluate_spline(self, xi, method):
+        # ensure xi is 2D list of points to evaluate (`m` is the number of
+        # points and `n` is the number of interpolation dimensions,
+        # ``n == len(self.grid)``.)
+        if xi.ndim == 1:
+            xi = xi.reshape((1, xi.size))
+        m, n = xi.shape
+
+        # Reorder the axes: n-dimensional process iterates over the
+        # interpolation axes from the last axis downwards: E.g. for a 4D grid
+        # the order of axes is 3, 2, 1, 0. Each 1D interpolation works along
+        # the 0th axis of its argument array (for 1D routine it's its ``y``
+        # array). Thus permute the interpolation axes of `values` *and keep
+        # trailing dimensions trailing*.
+        axes = tuple(range(self.values.ndim))
+        axx = axes[:n][::-1] + axes[n:]
+        values = self.values.transpose(axx)
+
+        if method == 'pchip':
+            _eval_func = self._do_pchip
+        else:
+            _eval_func = self._do_spline_fit
+        k = self._SPLINE_DEGREE_MAP[method]
+
+        # Non-stationary procedure: difficult to vectorize this part entirely
+        # into numpy-level operations. Unfortunately this requires explicit
+        # looping over each point in xi.
+
+        # can at least vectorize the first pass across all points in the
+        # last variable of xi.
+        last_dim = n - 1
+        first_values = _eval_func(self.grid[last_dim],
+                                  values,
+                                  xi[:, last_dim],
+                                  k)
+
+        # the rest of the dimensions have to be on a per point-in-xi basis
+        shape = (m, *self.values.shape[n:])
+        result = cp.empty(shape, dtype=self.values.dtype)
+        for j in range(m):
+            # Main process: Apply 1D interpolate in each dimension
+            # sequentially, starting with the last dimension.
+            # These are then "folded" into the next dimension in-place.
+            folded_values = first_values[j, ...]
+            for i in range(last_dim-1, -1, -1):
+                # Interpolate for each 1D from the last dimensions.
+                # This collapses each 1D sequence into a scalar.
+                folded_values = _eval_func(self.grid[i],
+                                           folded_values,
+                                           xi[j, i],
+                                           k)
+            result[j, ...] = folded_values
+
+        return result
+
+    @staticmethod
+    def _do_spline_fit(x, y, pt, k):
+        local_interp = make_interp_spline(x, y, k=k, axis=0)
+        values = local_interp(pt)
+        return values
+
+    @staticmethod
+    def _do_pchip(x, y, pt, k):
+        local_interp = PchipInterpolator(x, y, axis=0)
+        values = local_interp(pt)
+        return values
+
+    def _find_indices(self, xi):
+        # find relevant edges between which xi are situated
+        indices = []
+        # compute distance to lower edge in unity units
+        norm_distances = []
+        # iterate through dimensions
+        for x, grid in zip(xi, self.grid):
+            i = cp.searchsorted(grid, x) - 1
+            cp.clip(i, 0, grid.size - 2, i)
+            indices.append(i)
+
+            # compute norm_distances, incl length-1 grids,
+            # where `grid[i+1] == grid[i]`
+            denom = grid[i + 1] - grid[i]
+            norm_dist = cp.where(denom != 0, (x - grid[i]) / denom, 0)
+            norm_distances.append(norm_dist)
+
+        return indices, norm_distances
+
+    def _find_out_of_bounds(self, xi):
+        # check for out of bounds xi
+        out_of_bounds = cp.zeros((xi.shape[1]), dtype=bool)
+        # iterate through dimensions
+        for x, grid in zip(xi, self.grid):
+            out_of_bounds += x < grid[0]
+            out_of_bounds += x > grid[-1]
+        return out_of_bounds
+
+
+def interpn(points, values, xi, method="linear", bounds_error=True,
+            fill_value=cp.nan):
+    """
+    Multidimensional interpolation on regular or rectilinear grids.
+
+    Strictly speaking, not all regular grids are supported - this function
+    works on *rectilinear* grids, that is, a rectangular grid with even or
+    uneven spacing.
+
+    Parameters
+    ----------
+    points : tuple of cupy.ndarray of float, with shapes (m1, ), ..., (mn, )
+        The points defining the regular grid in n dimensions. The points in
+        each dimension (i.e. every elements of the points tuple) must be
+        strictly ascending or descending.
+
+    values : cupy.ndarray of shape (m1, ..., mn, ...)
+        The data on the regular grid in n dimensions. Complex data can be
+        acceptable.
+
+    xi : cupy.ndarray of shape (..., ndim)
+        The coordinates to sample the gridded data at
+
+    method : str, optional
+        The method of interpolation to perform. Supported are "linear",
+        "nearest", "slinear", "cubic", "quintic" and "pchip".
+
+    bounds_error : bool, optional
+        If True, when interpolated values are requested outside of the
+        domain of the input data, a ValueError is raised.
+        If False, then `fill_value` is used.
+
+    fill_value : number, optional
+        If provided, the value to use for points outside of the
+        interpolation domain. If None, values outside
+        the domain are extrapolated.
+
+    Returns
+    -------
+    values_x : ndarray, shape xi.shape[:-1] + values.shape[ndim:]
+        Interpolated values at `xi`. See notes for behaviour when
+        ``xi.ndim == 1``.
+
+    Notes
+    -----
+
+    In the case that ``xi.ndim == 1`` a new axis is inserted into
+    the 0 position of the returned array, values_x, so its shape is
+    instead ``(1,) + values.shape[ndim:]``.
+
+    If the input data is such that input dimensions have incommensurate
+    units and differ by many orders of magnitude, the interpolant may have
+    numerical artifacts. Consider rescaling the data before interpolation.
+
+    Examples
+    --------
+    Evaluate a simple example function on the points of a regular 3-D grid:
+
+    >>> import cupy as cp
+    >>> from cupyx.scipy.interpolate import interpn
+    >>> def value_func_3d(x, y, z):
+    ...     return 2 * x + 3 * y - z
+    >>> x = cp.linspace(0, 4, 5)
+    >>> y = cp.linspace(0, 5, 6)
+    >>> z = cp.linspace(0, 6, 7)
+    >>> points = (x, y, z)
+    >>> values = value_func_3d(*cp.meshgrid(*points, indexing='ij'))
+
+    Evaluate the interpolating function at a point
+
+    >>> point = cp.array([2.21, 3.12, 1.15])
+    >>> print(interpn(points, values, point))
+    [12.63]
+
+    See Also
+    --------
+    RegularGridInterpolator : interpolation on a regular or rectilinear grid
+                              in arbitrary dimensions (`interpn` wraps this
+                              class).
+
+    cupyx.scipy.ndimage.map_coordinates : interpolation on grids with equal
+                                          spacing (suitable for e.g., N-D image
+                                          resampling)
+    """
+    # sanity check 'method' kwarg
+    if method not in ["linear", "nearest", "slinear", "cubic",
+                      "quintic", "pchip"]:
+        raise ValueError(
+            "interpn only understands the methods 'linear', 'nearest', "
+            "'slinear', 'cubic', 'quintic' and 'pchip'. "
+            "You provided {method}.")
+
+    ndim = values.ndim
+
+    # sanity check consistency of input dimensions
+    if len(points) > ndim:
+        raise ValueError("There are %d point arrays, but values has %d "
+                         "dimensions" % (len(points), ndim))
+
+    grid, descending_dimensions = _check_points(points)
+    _check_dimensionality(grid, values)
+
+    # sanity check requested xi
+    xi = _ndim_coords_from_arrays(xi, ndim=len(grid))
+    if xi.shape[-1] != len(grid):
+        raise ValueError("The requested sample points xi have dimension "
+                         "%d, but this RegularGridInterpolator has "
+                         "dimension %d" % (xi.shape[-1], len(grid)))
+
+    if bounds_error:
+        for i, p in enumerate(xi.T):
+            if not cp.logical_and(cp.all(grid[i][0] <= p),
+                                  cp.all(p <= grid[i][-1])):
+                raise ValueError("One of the requested xi is out of bounds "
+                                 "in dimension %d" % i)
+
+    # perform interpolation
+    if method in ["linear", "nearest", "slinear", "cubic", "quintic", "pchip"]:
+        interp = RegularGridInterpolator(points, values, method=method,
+                                         bounds_error=bounds_error,
+                                         fill_value=fill_value)
+        return interp(xi)
diff --git a/cupyx/scipy/linalg/__init__.py b/cupyx/scipy/linalg/__init__.py
new file mode 100644
index 0000000..dc65885
--- /dev/null
+++ b/cupyx/scipy/linalg/__init__.py
@@ -0,0 +1,16 @@
+# flake8: NOQA
+from cupyx.scipy.linalg._special_matrices import (
+    tri, tril, triu, toeplitz, circulant, hankel,
+    hadamard, leslie, kron, block_diag, companion,
+    helmert, hilbert, dft,
+    fiedler, fiedler_companion, convolution_matrix
+)
+from cupyx.scipy.linalg._solve_triangular import solve_triangular  # NOQA
+from cupyx.scipy.linalg._decomp_lu import lu, lu_factor, lu_solve  # NOQA
+
+# uarray backend support (NEP 31)
+# The uarray feature for scipy.linalg is experimental.
+# The interface can change in the future.
+from cupyx.scipy.linalg._uarray import __ua_convert__  # NOQA
+from cupyx.scipy.linalg._uarray import __ua_domain__  # NOQA
+from cupyx.scipy.linalg._uarray import __ua_function__  # NOQA
diff --git a/cupyx/scipy/linalg/_decomp_lu.py b/cupyx/scipy/linalg/_decomp_lu.py
new file mode 100644
index 0000000..6b98363
--- /dev/null
+++ b/cupyx/scipy/linalg/_decomp_lu.py
@@ -0,0 +1,354 @@
+from warnings import warn
+
+import numpy
+
+import cupy
+from cupy.cuda import cublas
+from cupy.cuda import cusolver
+from cupy.cuda import device
+from cupy.cuda import runtime
+from cupy.linalg import _util
+from cupyx.scipy.linalg import _uarray
+
+
+@_uarray.implements('lu_factor')
+def lu_factor(a, overwrite_a=False, check_finite=True):
+    """LU decomposition.
+
+    Decompose a given two-dimensional square matrix into ``P * L * U``,
+    where ``P`` is a permutation matrix,  ``L`` lower-triangular with
+    unit diagonal elements, and ``U`` upper-triangular matrix.
+
+    Args:
+        a (cupy.ndarray): The input matrix with dimension ``(M, N)``
+        overwrite_a (bool): Allow overwriting data in ``a`` (may enhance
+            performance)
+        check_finite (bool): Whether to check that the input matrices contain
+            only finite numbers. Disabling may give a performance gain, but may
+            result in problems (crashes, non-termination) if the inputs do
+            contain infinities or NaNs.
+
+    Returns:
+        tuple:
+            ``(lu, piv)`` where ``lu`` is a :class:`cupy.ndarray`
+            storing ``U`` in its upper triangle, and ``L`` without
+            unit diagonal elements in its lower triangle, and ``piv`` is
+            a :class:`cupy.ndarray` storing pivot indices representing
+            permutation matrix ``P``. For ``0 <= i < min(M,N)``, row
+            ``i`` of the matrix was interchanged with row ``piv[i]``
+
+    .. seealso:: :func:`scipy.linalg.lu_factor`
+    """
+    return _lu_factor(a, overwrite_a, check_finite)
+
+
+@_uarray.implements('lu')
+def lu(a, permute_l=False, overwrite_a=False, check_finite=True):
+    """LU decomposition.
+
+    Decomposes a given two-dimensional matrix into ``P @ L @ U``, where ``P``
+    is a permutation matrix, ``L`` is a lower triangular or trapezoidal matrix
+    with unit diagonal, and ``U`` is a upper triangular or trapezoidal matrix.
+
+    Args:
+        a (cupy.ndarray): The input matrix with dimension ``(M, N)``.
+        permute_l (bool): If ``True``, perform the multiplication ``P @ L``.
+        overwrite_a (bool): Allow overwriting data in ``a`` (may enhance
+            performance)
+        check_finite (bool): Whether to check that the input matrices contain
+            only finite numbers. Disabling may give a performance gain, but may
+            result in problems (crashes, non-termination) if the inputs do
+            contain infinities or NaNs.
+
+    Returns:
+        tuple:
+            ``(P, L, U)`` if ``permute_l == False``, otherwise ``(PL, U)``.
+            ``P`` is a :class:`cupy.ndarray` storing permutation matrix with
+            dimension ``(M, M)``. ``L`` is a :class:`cupy.ndarray` storing
+            lower triangular or trapezoidal matrix with unit diagonal with
+            dimension ``(M, K)`` where ``K = min(M, N)``. ``U`` is a
+            :class:`cupy.ndarray` storing upper triangular or trapezoidal
+            matrix with dimension ``(K, N)``. ``PL`` is a :class:`cupy.ndarray`
+            storing permuted ``L`` matrix with dimension ``(M, K)``.
+
+    .. seealso:: :func:`scipy.linalg.lu`
+    """
+    lu, piv = _lu_factor(a, overwrite_a, check_finite)
+
+    m, n = lu.shape
+    k = min(m, n)
+    L, U = _cupy_split_lu(lu)
+
+    if permute_l:
+        _cupy_laswp(L, 0, k-1, piv, -1)
+        return (L, U)
+    else:
+        r_dtype = numpy.float32 if lu.dtype.char in 'fF' else numpy.float64
+        P = cupy.diag(cupy.ones((m,), dtype=r_dtype))
+        _cupy_laswp(P, 0, k-1, piv, -1)
+        return (P, L, U)
+
+
+def _lu_factor(a, overwrite_a=False, check_finite=True):
+    a = cupy.asarray(a)
+    _util._assert_2d(a)
+
+    dtype = a.dtype
+
+    if dtype.char == 'f':
+        getrf = cusolver.sgetrf
+        getrf_bufferSize = cusolver.sgetrf_bufferSize
+    elif dtype.char == 'd':
+        getrf = cusolver.dgetrf
+        getrf_bufferSize = cusolver.dgetrf_bufferSize
+    elif dtype.char == 'F':
+        getrf = cusolver.cgetrf
+        getrf_bufferSize = cusolver.cgetrf_bufferSize
+    elif dtype.char == 'D':
+        getrf = cusolver.zgetrf
+        getrf_bufferSize = cusolver.zgetrf_bufferSize
+    else:
+        msg = 'Only float32, float64, complex64 and complex128 are supported.'
+        raise NotImplementedError(msg)
+
+    a = a.astype(dtype, order='F', copy=(not overwrite_a))
+
+    if check_finite:
+        if a.dtype.kind == 'f' and not cupy.isfinite(a).all():
+            raise ValueError(
+                'array must not contain infs or NaNs')
+
+    cusolver_handle = device.get_cusolver_handle()
+    dev_info = cupy.empty(1, dtype=numpy.int32)
+
+    m, n = a.shape
+
+    ipiv = cupy.empty((min(m, n),), dtype=numpy.intc)
+
+    buffersize = getrf_bufferSize(cusolver_handle, m, n, a.data.ptr, m)
+    workspace = cupy.empty(buffersize, dtype=dtype)
+
+    # LU factorization
+    getrf(cusolver_handle, m, n, a.data.ptr, m, workspace.data.ptr,
+          ipiv.data.ptr, dev_info.data.ptr)
+
+    if not runtime.is_hip and dev_info[0] < 0:
+        # rocSOLVER does not inform us this info
+        raise ValueError('illegal value in %d-th argument of '
+                         'internal getrf (lu_factor)' % -dev_info[0])
+    elif dev_info[0] > 0:
+        warn('Diagonal number %d is exactly zero. Singular matrix.'
+             % dev_info[0], RuntimeWarning, stacklevel=2)
+
+    # cuSolver uses 1-origin while SciPy uses 0-origin
+    ipiv -= 1
+
+    return (a, ipiv)
+
+
+def _cupy_split_lu(LU, order='C'):
+    assert LU._f_contiguous
+    m, n = LU.shape
+    k = min(m, n)
+    order = 'F' if order == 'F' else 'C'
+    L = cupy.empty((m, k), order=order, dtype=LU.dtype)
+    U = cupy.empty((k, n), order=order, dtype=LU.dtype)
+    size = m * n
+    _kernel_cupy_split_lu(LU, m, n, k, L._c_contiguous, L, U, size=size)
+    return (L, U)
+
+
+_device_get_index = '''
+__device__ inline int get_index(int row, int col, int num_rows, int num_cols,
+                                bool c_contiguous)
+{
+    if (c_contiguous) {
+        return col + num_cols * row;
+    } else {
+        return row + num_rows * col;
+    }
+}
+'''
+
+_kernel_cupy_split_lu = cupy.ElementwiseKernel(
+    'raw T LU, int32 M, int32 N, int32 K, bool C_CONTIGUOUS',
+    'raw T L, raw T U',
+    '''
+    // LU: shape: (M, N)
+    // L: shape: (M, K)
+    // U: shape: (K, N)
+    const T* ptr_LU = &(LU[0]);
+    T* ptr_L = &(L[0]);
+    T* ptr_U = &(U[0]);
+    int row, col;
+    if (C_CONTIGUOUS) {
+        row = i / N;
+        col = i % N;
+    } else {
+        row = i % M;
+        col = i / M;
+    }
+    T lu_val = ptr_LU[get_index(row, col, M, N, false)];
+    T l_val, u_val;
+    if (row > col) {
+        l_val = lu_val;
+        u_val = static_cast<T>(0);
+    } else if (row == col) {
+        l_val = static_cast<T>(1);
+        u_val = lu_val;
+    } else {
+        l_val = static_cast<T>(0);
+        u_val = lu_val;
+    }
+    if (col < K) {
+        ptr_L[get_index(row, col, M, K, C_CONTIGUOUS)] = l_val;
+    }
+    if (row < K) {
+        ptr_U[get_index(row, col, K, N, C_CONTIGUOUS)] = u_val;
+    }
+    ''',
+    'cupyx_scipy_linalg_split_lu', preamble=_device_get_index
+)
+
+
+def _cupy_laswp(A, k1, k2, ipiv, incx):
+    m, n = A.shape
+    k = ipiv.shape[0]
+    assert 0 <= k1 and k1 <= k2 and k2 < k
+    assert A._c_contiguous or A._f_contiguous
+    _kernel_cupy_laswp(m, n, k1, k2, ipiv, incx, A._c_contiguous, A, size=n)
+
+
+_kernel_cupy_laswp = cupy.ElementwiseKernel(
+    'int32 M, int32 N, int32 K1, int32 K2, raw I IPIV, int32 INCX, '
+    'bool C_CONTIGUOUS',
+    'raw T A',
+    '''
+    // IPIV: 0-based pivot indices. shape: (K,)  (*) K > K2
+    // A: shape: (M, N)
+    T* ptr_A = &(A[0]);
+    if (K1 > K2) return;
+    int row_start, row_end, row_inc;
+    if (INCX > 0) {
+        row_start = K1; row_end = K2; row_inc = 1;
+    } else if (INCX < 0) {
+        row_start = K2; row_end = K1; row_inc = -1;
+    } else {
+        return;
+    }
+    int col = i;
+    int row1 = row_start;
+    while (1) {
+        int row2 = IPIV[row1];
+        if (row1 != row2) {
+            int idx1 = get_index(row1, col, M, N, C_CONTIGUOUS);
+            int idx2 = get_index(row2, col, M, N, C_CONTIGUOUS);
+            T tmp       = ptr_A[idx1];
+            ptr_A[idx1] = ptr_A[idx2];
+            ptr_A[idx2] = tmp;
+        }
+        if (row1 == row_end) break;
+        row1 += row_inc;
+    }
+    ''',
+    'cupyx_scipy_linalg_laswp', preamble=_device_get_index
+)
+
+
+@_uarray.implements('lu_solve')
+def lu_solve(lu_and_piv, b, trans=0, overwrite_b=False, check_finite=True):
+    """Solve an equation system, ``a * x = b``, given the LU factorization of ``a``
+
+    Args:
+        lu_and_piv (tuple): LU factorization of matrix ``a`` (``(M, M)``)
+            together with pivot indices.
+        b (cupy.ndarray): The matrix with dimension ``(M,)`` or
+            ``(M, N)``.
+        trans ({0, 1, 2}): Type of system to solve:
+
+            ========  =========
+            trans     system
+            ========  =========
+            0         a x  = b
+            1         a^T x = b
+            2         a^H x = b
+            ========  =========
+        overwrite_b (bool): Allow overwriting data in b (may enhance
+            performance)
+        check_finite (bool): Whether to check that the input matrices contain
+            only finite numbers. Disabling may give a performance gain, but may
+            result in problems (crashes, non-termination) if the inputs do
+            contain infinities or NaNs.
+
+    Returns:
+        cupy.ndarray:
+            The matrix with dimension ``(M,)`` or ``(M, N)``.
+
+    .. seealso:: :func:`scipy.linalg.lu_solve`
+    """  # NOQA
+
+    (lu, ipiv) = lu_and_piv
+
+    _util._assert_cupy_array(lu)
+    _util._assert_2d(lu)
+    _util._assert_stacked_square(lu)
+
+    m = lu.shape[0]
+    if m != b.shape[0]:
+        raise ValueError('incompatible dimensions.')
+
+    dtype = lu.dtype
+    if dtype.char == 'f':
+        getrs = cusolver.sgetrs
+    elif dtype.char == 'd':
+        getrs = cusolver.dgetrs
+    elif dtype.char == 'F':
+        getrs = cusolver.cgetrs
+    elif dtype.char == 'D':
+        getrs = cusolver.zgetrs
+    else:
+        msg = 'Only float32, float64, complex64 and complex128 are supported.'
+        raise NotImplementedError(msg)
+
+    if trans == 0:
+        trans = cublas.CUBLAS_OP_N
+    elif trans == 1:
+        trans = cublas.CUBLAS_OP_T
+    elif trans == 2:
+        trans = cublas.CUBLAS_OP_C
+    else:
+        raise ValueError('unknown trans')
+
+    lu = lu.astype(dtype, order='F', copy=False)
+    ipiv = ipiv.astype(ipiv.dtype, order='F', copy=True)
+    # cuSolver uses 1-origin while SciPy uses 0-origin
+    ipiv += 1
+    b = b.astype(dtype, order='F', copy=(not overwrite_b))
+
+    if check_finite:
+        if lu.dtype.kind == 'f' and not cupy.isfinite(lu).all():
+            raise ValueError(
+                'array must not contain infs or NaNs.\n'
+                'Note that when a singular matrix is given, unlike '
+                'scipy.linalg.lu_factor, cupyx.scipy.linalg.lu_factor '
+                'returns an array containing NaN.')
+        if b.dtype.kind == 'f' and not cupy.isfinite(b).all():
+            raise ValueError(
+                'array must not contain infs or NaNs')
+
+    n = 1 if b.ndim == 1 else b.shape[1]
+    cusolver_handle = device.get_cusolver_handle()
+    dev_info = cupy.empty(1, dtype=numpy.int32)
+
+    # solve for the inverse
+    getrs(cusolver_handle,
+          trans,
+          m, n, lu.data.ptr, m, ipiv.data.ptr, b.data.ptr,
+          m, dev_info.data.ptr)
+
+    if not runtime.is_hip and dev_info[0] < 0:
+        # rocSOLVER does not inform us this info
+        raise ValueError('illegal value in %d-th argument of '
+                         'internal getrs (lu_solve)' % -dev_info[0])
+
+    return b
diff --git a/cupyx/scipy/linalg/_solve_triangular.py b/cupyx/scipy/linalg/_solve_triangular.py
new file mode 100644
index 0000000..cdb28fb
--- /dev/null
+++ b/cupyx/scipy/linalg/_solve_triangular.py
@@ -0,0 +1,101 @@
+import numpy
+
+import cupy
+from cupy.cuda import cublas
+from cupy.cuda import device
+from cupy.linalg import _util
+from cupyx.scipy.linalg import _uarray
+
+
+@_uarray.implements('solve_triangular')
+def solve_triangular(a, b, trans=0, lower=False, unit_diagonal=False,
+                     overwrite_b=False, check_finite=False):
+    """Solve the equation a x = b for x, assuming a is a triangular matrix.
+
+    Args:
+        a (cupy.ndarray): The matrix with dimension ``(M, M)``.
+        b (cupy.ndarray): The matrix with dimension ``(M,)`` or
+            ``(M, N)``.
+        lower (bool): Use only data contained in the lower triangle of ``a``.
+            Default is to use upper triangle.
+        trans (0, 1, 2, 'N', 'T' or 'C'): Type of system to solve:
+
+            - *'0'* or *'N'* -- :math:`a x  = b`
+            - *'1'* or *'T'* -- :math:`a^T x = b`
+            - *'2'* or *'C'* -- :math:`a^H x = b`
+
+        unit_diagonal (bool): If ``True``, diagonal elements of ``a`` are
+            assumed to be 1 and will not be referenced.
+        overwrite_b (bool): Allow overwriting data in b (may enhance
+            performance)
+        check_finite (bool): Whether to check that the input matrices contain
+            only finite numbers. Disabling may give a performance gain, but may
+            result in problems (crashes, non-termination) if the inputs do
+            contain infinities or NaNs.
+
+    Returns:
+        cupy.ndarray:
+            The matrix with dimension ``(M,)`` or ``(M, N)``.
+
+    .. seealso:: :func:`scipy.linalg.solve_triangular`
+    """
+
+    _util._assert_cupy_array(a, b)
+
+    if len(a.shape) != 2 or a.shape[0] != a.shape[1]:
+        raise ValueError('expected square matrix')
+    if len(a) != len(b):
+        raise ValueError('incompatible dimensions')
+
+    # Cast to float32 or float64
+    if a.dtype.char in 'fd':
+        dtype = a.dtype
+    else:
+        dtype = numpy.promote_types(a.dtype.char, 'f')
+
+    a = cupy.array(a, dtype=dtype, order='F', copy=False)
+    b = cupy.array(b, dtype=dtype, order='F', copy=(not overwrite_b))
+
+    if check_finite:
+        if a.dtype.kind == 'f' and not cupy.isfinite(a).all():
+            raise ValueError(
+                'array must not contain infs or NaNs')
+        if b.dtype.kind == 'f' and not cupy.isfinite(b).all():
+            raise ValueError(
+                'array must not contain infs or NaNs')
+
+    m, n = (b.size, 1) if b.ndim == 1 else b.shape
+    cublas_handle = device.get_cublas_handle()
+
+    if dtype == 'f':
+        trsm = cublas.strsm
+    elif dtype == 'd':
+        trsm = cublas.dtrsm
+    elif dtype == 'F':
+        trsm = cublas.ctrsm
+    else:  # dtype == 'D'
+        trsm = cublas.ztrsm
+    one = numpy.array(1, dtype=dtype)
+
+    if lower:
+        uplo = cublas.CUBLAS_FILL_MODE_LOWER
+    else:
+        uplo = cublas.CUBLAS_FILL_MODE_UPPER
+
+    if trans == 'N':
+        trans = cublas.CUBLAS_OP_N
+    elif trans == 'T':
+        trans = cublas.CUBLAS_OP_T
+    elif trans == 'C':
+        trans = cublas.CUBLAS_OP_C
+
+    if unit_diagonal:
+        diag = cublas.CUBLAS_DIAG_UNIT
+    else:
+        diag = cublas.CUBLAS_DIAG_NON_UNIT
+
+    trsm(
+        cublas_handle, cublas.CUBLAS_SIDE_LEFT, uplo,
+        trans, diag,
+        m, n, one.ctypes.data, a.data.ptr, m, b.data.ptr, m)
+    return b
diff --git a/cupyx/scipy/linalg/_special_matrices.py b/cupyx/scipy/linalg/_special_matrices.py
new file mode 100644
index 0000000..8d9ca36
--- /dev/null
+++ b/cupyx/scipy/linalg/_special_matrices.py
@@ -0,0 +1,570 @@
+import math
+import cupy
+from cupy import _core
+from cupyx.scipy.linalg import _uarray
+
+
+@_uarray.implements('tri')
+def tri(N, M=None, k=0, dtype=None):
+    """ Construct (``N``, ``M``) matrix filled with ones at and below the
+    ``k``-th diagonal. The matrix has ``A[i,j] == 1`` for ``i <= j + k``.
+
+    Args:
+        N (int): The size of the first dimension of the matrix.
+        M (int, optional): The size of the second dimension of the matrix. If
+            ``M`` is None, ``M = N`` is assumed.
+        k (int, optional):  Number of subdiagonal below which matrix is filled
+            with ones. ``k = 0`` is the main diagonal, ``k < 0`` subdiagonal
+            and ``k > 0`` superdiagonal.
+        dtype (dtype, optional): Data type of the matrix.
+
+    Returns:
+        cupy.ndarray: Tri matrix.
+
+    .. seealso:: :func:`scipy.linalg.tri`
+    """
+    if M is None:
+        M = N
+    elif isinstance(M, str):
+        # handle legacy interface
+        M, dtype = N, M
+    # TODO: no outer method
+    # m = cupy.greater_equal.outer(cupy.arange(k, N+k), cupy.arange(M))
+    # return m if dtype is None else m.astype(dtype, copy=False)
+    return cupy.tri(N, M, k, bool if dtype is None else dtype)
+
+
+@_uarray.implements('tril')
+def tril(m, k=0):
+    """Make a copy of a matrix with elements above the ``k``-th diagonal
+    zeroed.
+
+    Args:
+        m (cupy.ndarray): Matrix whose elements to return
+        k (int, optional): Diagonal above which to zero elements.
+            ``k == 0`` is the main diagonal, ``k < 0`` subdiagonal and
+            ``k > 0`` superdiagonal.
+
+    Returns:
+        (cupy.ndarray): Return is the same shape and type as ``m``.
+
+    .. seealso:: :func:`scipy.linalg.tril`
+    """
+    # this is ~2x faster than cupy.tril for a 500x500 float64 matrix
+    t = tri(m.shape[0], m.shape[1], k=k, dtype=m.dtype.char)
+    t *= m
+    return t
+
+
+@_uarray.implements('triu')
+def triu(m, k=0):
+    """Make a copy of a matrix with elements below the ``k``-th diagonal
+    zeroed.
+
+    Args:
+        m (cupy.ndarray): Matrix whose elements to return
+        k (int, optional): Diagonal above which to zero elements.
+            ``k == 0`` is the main diagonal, ``k < 0`` subdiagonal and
+            ``k > 0`` superdiagonal.
+
+    Returns:
+        (cupy.ndarray): Return matrix with zeroed elements below the kth
+        diagonal and has same shape and type as ``m``.
+
+    .. seealso:: :func:`scipy.linalg.triu`
+    """
+    # this is ~25% faster than cupy.tril for a 500x500 float64 matrix
+    t = tri(m.shape[0], m.shape[1], k - 1, m.dtype.char)
+    cupy.subtract(1, t, out=t)
+    t *= m
+    return t
+
+
+@_uarray.implements('toeplitz')
+def toeplitz(c, r=None):
+    """Construct a Toeplitz matrix.
+
+    The Toeplitz matrix has constant diagonals, with ``c`` as its first column
+    and ``r`` as its first row. If ``r`` is not given, ``r == conjugate(c)`` is
+    assumed.
+
+    Args:
+        c (cupy.ndarray): First column of the matrix. Whatever the actual shape
+            of ``c``, it will be converted to a 1-D array.
+        r (cupy.ndarray, optional): First row of the matrix. If None,
+            ``r = conjugate(c)`` is assumed; in this case, if ``c[0]`` is real,
+            the result is a Hermitian matrix. r[0] is ignored; the first row of
+            the returned matrix is ``[c[0], r[1:]]``. Whatever the actual shape
+            of ``r``, it will be converted to a 1-D array.
+
+    Returns:
+        cupy.ndarray: The Toeplitz matrix. Dtype is the same as
+        ``(c[0] + r[0]).dtype``.
+
+    .. seealso:: :func:`cupyx.scipy.linalg.circulant`
+    .. seealso:: :func:`cupyx.scipy.linalg.hankel`
+    .. seealso:: :func:`cupyx.scipy.linalg.solve_toeplitz`
+    .. seealso:: :func:`cupyx.scipy.linalg.fiedler`
+    .. seealso:: :func:`scipy.linalg.toeplitz`
+    """
+    c = c.ravel()
+    r = c.conjugate() if r is None else r.ravel()
+    return _create_toeplitz_matrix(c[::-1], r[1:])
+
+
+@_uarray.implements('circulant')
+def circulant(c):
+    """Construct a circulant matrix.
+
+    Args:
+        c (cupy.ndarray): 1-D array, the first column of the matrix.
+
+    Returns:
+        cupy.ndarray: A circulant matrix whose first column is ``c``.
+
+    .. seealso:: :func:`cupyx.scipy.linalg.toeplitz`
+    .. seealso:: :func:`cupyx.scipy.linalg.hankel`
+    .. seealso:: :func:`cupyx.scipy.linalg.solve_circulant`
+    .. seealso:: :func:`cupyx.scipy.linalg.fiedler`
+    .. seealso:: :func:`scipy.linalg.circulant`
+    """
+    c = c.ravel()
+    return _create_toeplitz_matrix(c[::-1], c[:0:-1])
+
+
+@_uarray.implements('hankel')
+def hankel(c, r=None):
+    """Construct a Hankel matrix.
+
+    The Hankel matrix has constant anti-diagonals, with ``c`` as its first
+    column and ``r`` as its last row. If ``r`` is not given, then
+    ``r = zeros_like(c)`` is assumed.
+
+    Args:
+        c (cupy.ndarray): First column of the matrix. Whatever the actual shape
+            of ``c``, it will be converted to a 1-D array.
+        r (cupy.ndarray, optionnal): Last row of the matrix. If None,
+            ``r = zeros_like(c)`` is assumed. ``r[0]`` is ignored; the last row
+            of the returned matrix is ``[c[-1], r[1:]]``. Whatever the actual
+            shape of ``r``, it will be converted to a 1-D array.
+
+    Returns:
+        cupy.ndarray: The Hankel matrix. Dtype is the same as
+        ``(c[0] + r[0]).dtype``.
+
+    .. seealso:: :func:`cupyx.scipy.linalg.toeplitz`
+    .. seealso:: :func:`cupyx.scipy.linalg.circulant`
+    .. seealso:: :func:`scipy.linalg.hankel`
+    """
+    c = c.ravel()
+    r = cupy.zeros_like(c) if r is None else r.ravel()
+    return _create_toeplitz_matrix(c, r[1:], True)
+
+
+def _create_toeplitz_matrix(c, r, hankel=False):
+    vals = cupy.concatenate((c, r))
+    n = vals.strides[0]
+    return cupy.lib.stride_tricks.as_strided(
+        vals if hankel else vals[c.size-1:],
+        shape=(c.size, r.size+1),
+        strides=(n if hankel else -n, n)).copy()
+
+
+@_uarray.implements('hadamard')
+def hadamard(n, dtype=int):
+    """Construct an Hadamard matrix.
+
+    Constructs an n-by-n Hadamard matrix, using Sylvester's construction. ``n``
+    must be a power of 2.
+
+    Args:
+        n (int): The order of the matrix. ``n`` must be a power of 2.
+        dtype (dtype, optional): The data type of the array to be constructed.
+
+    Returns:
+        (cupy.ndarray): The Hadamard matrix.
+
+    .. seealso:: :func:`scipy.linalg.hadamard`
+    """
+    lg2 = 0 if n < 1 else (int(n).bit_length() - 1)
+    if 2 ** lg2 != n:
+        raise ValueError('n must be an positive a power of 2 integer')
+    H = cupy.empty((n, n), dtype)
+    return _hadamard_kernel(H, H)
+
+
+_hadamard_kernel = _core.ElementwiseKernel(
+    'T in', 'T out',
+    'out = (__popc(_ind.get()[0] & _ind.get()[1]) & 1) ? -1 : 1;',
+    'cupyx_scipy_linalg_hadamard', reduce_dims=False)
+
+
+@_uarray.implements('leslie')
+def leslie(f, s):
+    """Create a Leslie matrix.
+
+    Given the length n array of fecundity coefficients ``f`` and the length n-1
+    array of survival coefficients ``s``, return the associated Leslie matrix.
+
+    Args:
+        f (cupy.ndarray): The "fecundity" coefficients.
+        s (cupy.ndarray): The "survival" coefficients, has to be 1-D.  The
+            length of ``s`` must be one less than the length of ``f``, and it
+            must be at least 1.
+
+    Returns:
+        cupy.ndarray: The array is zero except for the first row, which is
+        ``f``, and the first sub-diagonal, which is ``s``. The data-type of
+        the array will be the data-type of ``f[0]+s[0]``.
+
+    .. seealso:: :func:`scipy.linalg.leslie`
+    """
+    if f.ndim != 1:
+        raise ValueError('Incorrect shape for f. f must be 1D')
+    if s.ndim != 1:
+        raise ValueError('Incorrect shape for s. s must be 1D')
+    n = f.size
+    if n != s.size + 1:
+        raise ValueError('Length of s must be one less than length of f')
+    if s.size == 0:
+        raise ValueError('The length of s must be at least 1.')
+    a = cupy.zeros((n, n), dtype=cupy.result_type(f, s))
+    a[0] = f
+    cupy.fill_diagonal(a[1:], s)
+    return a
+
+
+@_uarray.implements('kron')
+def kron(a, b):
+    """Kronecker product.
+
+    The result is the block matrix::
+        a[0,0]*b    a[0,1]*b  ... a[0,-1]*b
+        a[1,0]*b    a[1,1]*b  ... a[1,-1]*b
+        ...
+        a[-1,0]*b   a[-1,1]*b ... a[-1,-1]*b
+
+    Args:
+        a (cupy.ndarray): Input array
+        b (cupy.ndarray): Input array
+
+    Returns:
+        cupy.ndarray: Kronecker product of ``a`` and ``b``.
+
+    .. seealso:: :func:`scipy.linalg.kron`
+    """
+    o = cupy.outer(a, b)
+    o = o.reshape(a.shape + b.shape)
+    return cupy.concatenate(cupy.concatenate(o, axis=1), axis=1)
+
+
+@_uarray.implements('block_diag')
+def block_diag(*arrs):
+    """Create a block diagonal matrix from provided arrays.
+
+    Given the inputs ``A``, ``B``, and ``C``, the output will have these
+    arrays arranged on the diagonal::
+
+        [A, 0, 0]
+        [0, B, 0]
+        [0, 0, C]
+
+    Args:
+        A, B, C, ... (cupy.ndarray): Input arrays. A 1-D array of length ``n``
+            is treated as a 2-D array with shape ``(1,n)``.
+
+    Returns:
+        (cupy.ndarray): Array with ``A``, ``B``, ``C``, ... on the diagonal.
+        Output has the same dtype as ``A``.
+
+    .. seealso:: :func:`scipy.linalg.block_diag`
+    """
+    if not arrs:
+        return cupy.empty((1, 0))
+
+    # Convert to 2D and check
+    if len(arrs) == 1:
+        arrs = (cupy.atleast_2d(*arrs),)
+    else:
+        arrs = cupy.atleast_2d(*arrs)
+    if any(a.ndim != 2 for a in arrs):
+        bad = [k for k in range(len(arrs)) if arrs[k].ndim != 2]
+        raise ValueError('arguments in the following positions have dimension '
+                         'greater than 2: {}'.format(bad))
+
+    shapes = tuple(a.shape for a in arrs)
+    shape = tuple(sum(x) for x in zip(*shapes))
+    out = cupy.zeros(shape, dtype=cupy.result_type(*arrs))
+    r, c = 0, 0
+    for arr in arrs:
+        rr, cc = arr.shape
+        out[r:r + rr, c:c + cc] = arr
+        r += rr
+        c += cc
+    return out
+
+
+@_uarray.implements('companion')
+def companion(a):
+    """Create a companion matrix.
+
+    Create the companion matrix associated with the polynomial whose
+    coefficients are given in ``a``.
+
+    Args:
+        a (cupy.ndarray): 1-D array of polynomial coefficients. The length of
+            ``a`` must be at least two, and ``a[0]`` must not be zero.
+
+    Returns:
+        (cupy.ndarray): The first row of the output is ``-a[1:]/a[0]``, and the
+        first sub-diagonal is all ones. The data-type of the array is the
+        same as the data-type of ``-a[1:]/a[0]``.
+
+    .. seealso:: :func:`cupyx.scipy.linalg.fiedler_companion`
+    .. seealso:: :func:`scipy.linalg.companion`
+    """
+    n = a.size
+    if a.ndim != 1:
+        raise ValueError('`a` must be one-dimensional.')
+    if n < 2:
+        raise ValueError('The length of `a` must be at least 2.')
+    # Following check requires device-to-host synchronization so will we not
+    # raise an error this situation
+    # if a[0] == 0:
+    #     raise ValueError('The first coefficient in `a` must not be zero.')
+    first_row = -a[1:] / a[0]
+    c = cupy.zeros((n - 1, n - 1), dtype=first_row.dtype)
+    c[0] = first_row
+    cupy.fill_diagonal(c[1:], 1)
+    return c
+
+
+@_uarray.implements('helmert')
+def helmert(n, full=False):
+    """Create an Helmert matrix of order ``n``.
+
+    This has applications in statistics, compositional or simplicial analysis,
+    and in Aitchison geometry.
+
+    Args:
+        n (int): The size of the array to create.
+        full (bool, optional): If True the (n, n) ndarray will be returned.
+            Otherwise, the default, the submatrix that does not include the
+            first row will be returned.
+
+    Returns:
+        cupy.ndarray: The Helmert matrix. The shape is (n, n) or (n-1, n)
+        depending on the ``full`` argument.
+
+    .. seealso:: :func:`scipy.linalg.helmert`
+    """
+    d = cupy.arange(n)
+    H = cupy.tri(n, n, -1)
+    H.diagonal()[:] -= d
+    d *= cupy.arange(1, n+1)
+    H[0] = 1
+    d[0] = n
+    H /= cupy.sqrt(d)[:, None]
+    return H if full else H[1:]
+
+
+@_uarray.implements('hilbert')
+def hilbert(n):
+    """Create a Hilbert matrix of order ``n``.
+
+    Returns the ``n`` by ``n`` array with entries ``h[i,j] = 1 / (i + j + 1)``.
+
+    Args:
+        n (int): The size of the array to create.
+
+    Returns:
+        cupy.ndarray: The Hilbert matrix.
+
+    .. seealso:: :func:`scipy.linalg.hilbert`
+    """
+    values = cupy.arange(1, 2*n, dtype=cupy.float64)
+    cupy.reciprocal(values, values)
+    return hankel(values[:n], r=values[n-1:])
+
+
+# TODO: invhilbert(n, exact=False)
+# TODO: pascal(n, kind='symmetric', exact=True)
+# TODO: invpascal(n, kind='symmetric', exact=True)
+
+
+@_uarray.implements('dft')
+def dft(n, scale=None):
+    """Discrete Fourier transform matrix.
+
+    Create the matrix that computes the discrete Fourier transform of a
+    sequence. The nth primitive root of unity used to generate the matrix is
+    exp(-2*pi*i/n), where i = sqrt(-1).
+
+    Args:
+        n (int): Size the matrix to create.
+        scale (str, optional): Must be None, 'sqrtn', or 'n'.
+            If ``scale`` is 'sqrtn', the matrix is divided by ``sqrt(n)``.
+            If ``scale`` is 'n', the matrix is divided by ``n``.
+            If ``scale`` is None (default), the matrix is not normalized, and
+            the return value is simply the Vandermonde matrix of the roots of
+            unity.
+
+    Returns:
+        (cupy.ndarray): The DFT matrix.
+
+    Notes:
+        When ``scale`` is None, multiplying a vector by the matrix returned by
+        ``dft`` is mathematically equivalent to (but much less efficient than)
+        the calculation performed by ``scipy.fft.fft``.
+
+    .. seealso:: :func:`scipy.linalg.dft`
+    """
+    if scale not in (None, 'sqrtn', 'n'):
+        raise ValueError('scale must be None, \'sqrtn\', or \'n\'; '
+                         '%r is not valid.' % (scale,))
+    r = cupy.arange(n, dtype='complex128')
+    r *= -2j*cupy.pi/n
+    omegas = cupy.exp(r, out=r)[:, None]
+    m = omegas ** cupy.arange(n)
+    if scale is not None:
+        m *= (1/math.sqrt(n)) if scale == 'sqrtn' else (1/n)
+    return m
+
+
+@_uarray.implements('fiedler')
+def fiedler(a):
+    """Returns a symmetric Fiedler matrix
+
+    Given an sequence of numbers ``a``, Fiedler matrices have the structure
+    ``F[i, j] = np.abs(a[i] - a[j])``, and hence zero diagonals and nonnegative
+    entries. A Fiedler matrix has a dominant positive eigenvalue and other
+    eigenvalues are negative. Although not valid generally, for certain inputs,
+    the inverse and the determinant can be derived explicitly.
+
+    Args:
+        a (cupy.ndarray): coefficient array
+
+    Returns:
+        cupy.ndarray: the symmetric Fiedler matrix
+
+    .. seealso:: :func:`cupyx.scipy.linalg.circulant`
+    .. seealso:: :func:`cupyx.scipy.linalg.toeplitz`
+    .. seealso:: :func:`scipy.linalg.fiedler`
+    """
+    if a.ndim != 1:
+        raise ValueError('Input `a` must be a 1D array.')
+    if a.size == 0:
+        return cupy.zeros(0)
+    if a.size == 1:
+        return cupy.zeros((1, 1))
+    a = a[:, None] - a
+    return cupy.abs(a, out=a)
+
+
+@_uarray.implements('fiedler_companion')
+def fiedler_companion(a):
+    """Returns a Fiedler companion matrix
+
+    Given a polynomial coefficient array ``a``, this function forms a
+    pentadiagonal matrix with a special structure whose eigenvalues coincides
+    with the roots of ``a``.
+
+    Args:
+        a (cupy.ndarray): 1-D array of polynomial coefficients in descending
+            order with a nonzero leading coefficient. For ``N < 2``, an empty
+            array is returned.
+
+    Returns:
+        cupy.ndarray: Resulting companion matrix
+
+    Notes:
+        Similar to ``companion`` the leading coefficient should be nonzero. In
+        the case the leading coefficient is not 1, other coefficients are
+        rescaled before the array generation. To avoid numerical issues, it is
+        best to provide a monic polynomial.
+
+    .. seealso:: :func:`cupyx.scipy.linalg.companion`
+    .. seealso:: :func:`scipy.linalg.fiedler_companion`
+    """
+    if a.ndim != 1:
+        raise ValueError('Input `a` must be a 1-D array.')
+    if a.size < 2:
+        return cupy.zeros((0,), a.dtype)
+    if a.size == 2:
+        return (-a[1]/a[0])[None, None]
+    # Following check requires device-to-host synchronization so will we not
+    # raise an error this situation
+    # if a[0] == 0.:
+    #     raise ValueError('Leading coefficient is zero.')
+    a = a/a[0]
+    n = a.size - 1
+    c = cupy.zeros((n, n), dtype=a.dtype)
+    # subdiagonals
+    cupy.fill_diagonal(c[3::2, 1::2], 1)
+    cupy.fill_diagonal(c[2::2, 1::2], -a[3::2])
+    # superdiagonals
+    cupy.fill_diagonal(c[::2, 2::2], 1)
+    cupy.fill_diagonal(c[::2, 1::2], -a[2::2])
+    c[0, 0] = -a[1]
+    c[1, 0] = 1
+    return c
+
+
+@_uarray.implements('convolution_matrix')
+def convolution_matrix(a, n, mode='full'):
+    """Construct a convolution matrix.
+
+    Constructs the Toeplitz matrix representing one-dimensional convolution.
+
+    Args:
+        a (cupy.ndarray): The 1-D array to convolve.
+        n (int): The number of columns in the resulting matrix. It gives the
+            length of the input to be convolved with ``a``. This is analogous
+            to the length of ``v`` in ``numpy.convolve(a, v)``.
+        mode (str): This must be one of (``'full'``, ``'valid'``, ``'same'``).
+            This is analogous to ``mode`` in ``numpy.convolve(v, a, mode)``.
+
+    Returns:
+        cupy.ndarray: The convolution matrix whose row count k depends on
+        ``mode``:
+
+        =========== =========================
+        ``mode``    k
+        =========== =========================
+        ``'full'``  m + n - 1
+        ``'same'``  max(m, n)
+        ``'valid'`` max(m, n) - min(m, n) + 1
+        =========== =========================
+
+    .. seealso:: :func:`cupyx.scipy.linalg.toeplitz`
+    .. seealso:: :func:`scipy.linalg.convolution_matrix`
+    """
+    if n <= 0:
+        raise ValueError('n must be a positive integer.')
+    if a.ndim != 1:
+        raise ValueError('convolution_matrix expects a one-dimensional '
+                         'array as input')
+    if a.size == 0:
+        raise ValueError('len(a) must be at least 1.')
+    if mode not in ('full', 'valid', 'same'):
+        raise ValueError(
+            '`mode` argument must be one of (\'full\', \'valid\', \'same\')')
+
+    # create zero padded versions of the array
+    az = cupy.pad(a, (0, n-1), 'constant')
+    raz = cupy.pad(a[::-1], (0, n-1), 'constant')
+    if mode == 'same':
+        trim = min(n, a.size) - 1
+        tb = trim//2
+        te = trim - tb
+        col0 = az[tb:az.size-te]
+        row0 = raz[-n-tb:raz.size-tb]
+    elif mode == 'valid':
+        tb = min(n, a.size) - 1
+        te = tb
+        col0 = az[tb:az.size-te]
+        row0 = raz[-n-tb:raz.size-tb]
+    else:  # 'full'
+        col0 = az
+        row0 = raz[-n:]
+    return toeplitz(col0, row0)
diff --git a/cupyx/scipy/linalg/_uarray.py b/cupyx/scipy/linalg/_uarray.py
new file mode 100644
index 0000000..aa5395e
--- /dev/null
+++ b/cupyx/scipy/linalg/_uarray.py
@@ -0,0 +1,75 @@
+import numpy as np
+import cupy
+import cupy.linalg as _cp_linalg
+
+
+try:
+    import scipy.linalg as _scipy_linalg
+except ImportError:
+    _scipy_linalg = None
+
+
+# Backend support for scipy.linalg
+
+__ua_domain__ = 'numpy.scipy.linalg'
+_implemented = {}  # type: ignore
+_notfound = []  # for test
+
+
+def __ua_convert__(dispatchables, coerce):
+    if coerce:
+        try:
+            replaced = [
+                cupy.asarray(d.value) if d.coercible and d.type is np.ndarray
+                else d.value for d in dispatchables]
+        except TypeError:
+            return NotImplemented
+    else:
+        replaced = [d.value for d in dispatchables]
+
+    if not all(d.type is not np.ndarray or isinstance(r, cupy.ndarray)
+               for r, d in zip(replaced, dispatchables)):
+        return NotImplemented
+
+    return replaced
+
+
+def __ua_function__(method, args, kwargs):
+    fn = _implemented.get(method, None)
+    if fn is None:
+        return NotImplemented
+    return fn(*args, **kwargs)
+
+
+def implements(scipy_func_name):
+    """Decorator adds function to the dictionary of implemented functions"""
+    def inner(func):
+        scipy_func = (
+            _scipy_linalg and getattr(_scipy_linalg, scipy_func_name, None))
+        if scipy_func:
+            _implemented[scipy_func] = func
+        else:
+            _notfound.append(scipy_func_name)
+        return func
+
+    return inner
+
+
+# cupy linalg functions
+
+_cp_linalg_functions = [
+    # cupy.linalg._eigenvalue
+    'eigh', 'eigvalsh',
+    # cupy.linalg._decomposition
+    'cholesky', 'qr', 'svd',
+    # cupy.linalg._norms
+    'norm', 'det',
+    # cupy.linalg._solve
+    'solve', 'lstsq', 'inv', 'pinv'
+]
+
+if _scipy_linalg:
+    for func_name in _cp_linalg_functions:
+        cp_func = getattr(_cp_linalg, func_name)
+        scipy_func = getattr(_scipy_linalg, func_name)
+        _implemented[scipy_func] = cp_func
diff --git a/cupyx/scipy/ndimage/__init__.py b/cupyx/scipy/ndimage/__init__.py
new file mode 100644
index 0000000..7329ddd
--- /dev/null
+++ b/cupyx/scipy/ndimage/__init__.py
@@ -0,0 +1,71 @@
+from cupyx.scipy.ndimage._filters import correlate  # NOQA
+from cupyx.scipy.ndimage._filters import convolve  # NOQA
+from cupyx.scipy.ndimage._filters import correlate1d  # NOQA
+from cupyx.scipy.ndimage._filters import convolve1d  # NOQA
+from cupyx.scipy.ndimage._filters import uniform_filter1d  # NOQA
+from cupyx.scipy.ndimage._filters import uniform_filter  # NOQA
+from cupyx.scipy.ndimage._filters import gaussian_filter1d  # NOQA
+from cupyx.scipy.ndimage._filters import gaussian_filter  # NOQA
+from cupyx.scipy.ndimage._filters import prewitt  # NOQA
+from cupyx.scipy.ndimage._filters import sobel  # NOQA
+from cupyx.scipy.ndimage._filters import generic_laplace  # NOQA
+from cupyx.scipy.ndimage._filters import laplace  # NOQA
+from cupyx.scipy.ndimage._filters import gaussian_laplace  # NOQA
+from cupyx.scipy.ndimage._filters import generic_gradient_magnitude  # NOQA
+from cupyx.scipy.ndimage._filters import gaussian_gradient_magnitude  # NOQA
+from cupyx.scipy.ndimage._filters import minimum_filter  # NOQA
+from cupyx.scipy.ndimage._filters import maximum_filter  # NOQA
+from cupyx.scipy.ndimage._filters import minimum_filter1d  # NOQA
+from cupyx.scipy.ndimage._filters import maximum_filter1d  # NOQA
+from cupyx.scipy.ndimage._filters import median_filter  # NOQA
+from cupyx.scipy.ndimage._filters import rank_filter  # NOQA
+from cupyx.scipy.ndimage._filters import percentile_filter  # NOQA
+from cupyx.scipy.ndimage._filters import generic_filter  # NOQA
+from cupyx.scipy.ndimage._filters import generic_filter1d  # NOQA
+
+from cupyx.scipy.ndimage._fourier import fourier_ellipsoid  # NOQA
+from cupyx.scipy.ndimage._fourier import fourier_gaussian  # NOQA
+from cupyx.scipy.ndimage._fourier import fourier_shift  # NOQA
+from cupyx.scipy.ndimage._fourier import fourier_uniform  # NOQA
+
+from cupyx.scipy.ndimage._interpolation import affine_transform  # NOQA
+from cupyx.scipy.ndimage._interpolation import map_coordinates  # NOQA
+from cupyx.scipy.ndimage._interpolation import rotate  # NOQA
+from cupyx.scipy.ndimage._interpolation import shift  # NOQA
+from cupyx.scipy.ndimage._interpolation import spline_filter  # NOQA
+from cupyx.scipy.ndimage._interpolation import spline_filter1d  # NOQA
+from cupyx.scipy.ndimage._interpolation import zoom  # NOQA
+
+from cupyx.scipy.ndimage._measurements import label  # NOQA
+from cupyx.scipy.ndimage._measurements import sum  # NOQA
+from cupyx.scipy.ndimage._measurements import sum_labels  # NOQA
+from cupyx.scipy.ndimage._measurements import mean  # NOQA
+from cupyx.scipy.ndimage._measurements import variance  # NOQA
+from cupyx.scipy.ndimage._measurements import standard_deviation  # NOQA
+from cupyx.scipy.ndimage._measurements import minimum  # NOQA
+from cupyx.scipy.ndimage._measurements import maximum  # NOQA
+from cupyx.scipy.ndimage._measurements import minimum_position  # NOQA
+from cupyx.scipy.ndimage._measurements import maximum_position  # NOQA
+from cupyx.scipy.ndimage._measurements import median  # NOQA
+from cupyx.scipy.ndimage._measurements import extrema  # NOQA
+from cupyx.scipy.ndimage._measurements import center_of_mass  # NOQA
+from cupyx.scipy.ndimage._measurements import histogram  # NOQA
+from cupyx.scipy.ndimage._measurements import labeled_comprehension  # NOQA
+
+from cupyx.scipy.ndimage._morphology import generate_binary_structure  # NOQA
+from cupyx.scipy.ndimage._morphology import iterate_structure  # NOQA
+from cupyx.scipy.ndimage._morphology import binary_erosion  # NOQA
+from cupyx.scipy.ndimage._morphology import binary_dilation  # NOQA
+from cupyx.scipy.ndimage._morphology import binary_opening  # NOQA
+from cupyx.scipy.ndimage._morphology import binary_closing  # NOQA
+from cupyx.scipy.ndimage._morphology import binary_hit_or_miss  # NOQA
+from cupyx.scipy.ndimage._morphology import binary_fill_holes  # NOQA
+from cupyx.scipy.ndimage._morphology import binary_propagation  # NOQA
+from cupyx.scipy.ndimage._morphology import grey_erosion  # NOQA
+from cupyx.scipy.ndimage._morphology import grey_dilation  # NOQA
+from cupyx.scipy.ndimage._morphology import grey_closing  # NOQA
+from cupyx.scipy.ndimage._morphology import grey_opening  # NOQA
+from cupyx.scipy.ndimage._morphology import morphological_gradient  # NOQA
+from cupyx.scipy.ndimage._morphology import morphological_laplace  # NOQA
+from cupyx.scipy.ndimage._morphology import white_tophat  # NOQA
+from cupyx.scipy.ndimage._morphology import black_tophat  # NOQA
diff --git a/cupyx/scipy/ndimage/_filters.py b/cupyx/scipy/ndimage/_filters.py
new file mode 100644
index 0000000..7467edc
--- /dev/null
+++ b/cupyx/scipy/ndimage/_filters.py
@@ -0,0 +1,1255 @@
+import numpy
+
+import cupy
+
+from cupy import _core
+from cupy._core import internal
+from cupyx.scipy.ndimage import _util
+from cupyx.scipy.ndimage import _filters_core
+from cupyx.scipy.ndimage import _filters_generic
+
+
+def correlate(input, weights, output=None, mode='reflect', cval=0.0, origin=0):
+    """Multi-dimensional correlate.
+
+    The array is correlated with the given kernel.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        weights (cupy.ndarray): Array of weights, same number of dimensions as
+            input
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of correlate.
+
+    .. seealso:: :func:`scipy.ndimage.correlate`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    return _correlate_or_convolve(input, weights, output, mode, cval, origin)
+
+
+def convolve(input, weights, output=None, mode='reflect', cval=0.0, origin=0):
+    """Multi-dimensional convolution.
+
+    The array is convolved with the given kernel.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        weights (cupy.ndarray): Array of weights, same number of dimensions as
+            input
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of convolution.
+
+    .. seealso:: :func:`scipy.ndimage.convolve`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    return _correlate_or_convolve(input, weights, output, mode, cval, origin,
+                                  True)
+
+
+def correlate1d(input, weights, axis=-1, output=None, mode="reflect", cval=0.0,
+                origin=0):
+    """One-dimensional correlate.
+
+    The array is correlated with the given kernel.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        weights (cupy.ndarray): One-dimensional array of weights
+        axis (int): The axis of input along which to calculate. Default is -1.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int): The origin parameter controls the placement of the
+            filter, relative to the center of the current element of the
+            input. Default is ``0``.
+
+    Returns:
+        cupy.ndarray: The result of the 1D correlation.
+
+    .. seealso:: :func:`scipy.ndimage.correlate1d`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    weights, origins = _filters_core._convert_1d_args(input.ndim, weights,
+                                                      origin, axis)
+    return _correlate_or_convolve(input, weights, output, mode, cval, origins)
+
+
+def convolve1d(input, weights, axis=-1, output=None, mode="reflect", cval=0.0,
+               origin=0):
+    """One-dimensional convolution.
+
+    The array is convolved with the given kernel.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        weights (cupy.ndarray): One-dimensional array of weights
+        axis (int): The axis of input along which to calculate. Default is -1.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int): The origin parameter controls the placement of the
+            filter, relative to the center of the current element of the
+            input. Default is ``0``.
+    Returns:
+        cupy.ndarray: The result of the 1D convolution.
+
+    .. seealso:: :func:`scipy.ndimage.convolve1d`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    weights, origins = _filters_core._convert_1d_args(input.ndim, weights,
+                                                      origin, axis)
+    return _correlate_or_convolve(input, weights, output, mode, cval, origins,
+                                  True)
+
+
+def _correlate_or_convolve(input, weights, output, mode, cval, origin,
+                           convolution=False):
+    origins, int_type = _filters_core._check_nd_args(input, weights,
+                                                     mode, origin)
+    if weights.size == 0:
+        return cupy.zeros_like(input)
+
+    _util._check_cval(mode, cval, _util._is_integer_output(output, input))
+
+    if convolution:
+        weights = weights[tuple([slice(None, None, -1)] * weights.ndim)]
+        origins = list(origins)
+        for i, wsize in enumerate(weights.shape):
+            origins[i] = -origins[i]
+            if wsize % 2 == 0:
+                origins[i] -= 1
+        origins = tuple(origins)
+    elif weights.dtype.kind == "c":
+        # numpy.correlate conjugates weights rather than input.
+        weights = weights.conj()
+    weights_dtype = _util._get_weights_dtype(input, weights)
+    offsets = _filters_core._origins_to_offsets(origins, weights.shape)
+    kernel = _get_correlate_kernel(mode, weights.shape, int_type,
+                                   offsets, cval)
+    output = _filters_core._call_kernel(kernel, input, weights, output,
+                                        weights_dtype=weights_dtype)
+    return output
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_correlate_kernel(mode, w_shape, int_type, offsets, cval):
+    return _filters_core._generate_nd_kernel(
+        'correlate',
+        'W sum = (W)0;',
+        'sum += cast<W>({value}) * wval;',
+        'y = cast<Y>(sum);',
+        mode, w_shape, int_type, offsets, cval, ctype='W')
+
+
+def _run_1d_correlates(input, params, get_weights, output, mode, cval,
+                       origin=0):
+    """
+    Enhanced version of _run_1d_filters that uses correlate1d as the filter
+    function. The params are a list of values to pass to the get_weights
+    callable given. If duplicate param values are found, the weights are
+    reused from the first invocation of get_weights. The get_weights callable
+    must return a 1D array of weights to give to correlate1d.
+    """
+    wghts = {}
+    for param in params:
+        if param not in wghts:
+            wghts[param] = get_weights(param)
+    wghts = [wghts[param] for param in params]
+    return _filters_core._run_1d_filters(
+        [None if w is None else correlate1d for w in wghts],
+        input, wghts, output, mode, cval, origin)
+
+
+def uniform_filter1d(input, size, axis=-1, output=None, mode="reflect",
+                     cval=0.0, origin=0):
+    """One-dimensional uniform filter along the given axis.
+
+    The lines of the array along the given axis are filtered with a uniform
+    filter of the given size.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (int): Length of the uniform filter.
+        axis (int): The axis of input along which to calculate. Default is -1.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int): The origin parameter controls the placement of the
+            filter, relative to the center of the current element of the
+            input. Default is ``0``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.uniform_filter1d`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    weights_dtype = _util._init_weights_dtype(input)
+    weights = cupy.full(size, 1 / size, dtype=weights_dtype)
+    return correlate1d(input, weights, axis, output, mode, cval, origin)
+
+
+def uniform_filter(input, size=3, output=None, mode="reflect", cval=0.0,
+                   origin=0):
+    """Multi-dimensional uniform filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (int or sequence of int): Lengths of the uniform filter for each
+            dimension. A single value applies to all axes.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int or sequence of int): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of ``0`` is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.uniform_filter`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    sizes = _util._fix_sequence_arg(size, input.ndim, 'size', int)
+    weights_dtype = _util._init_weights_dtype(input)
+
+    def get(size, dtype=weights_dtype):
+        return None if size <= 1 else cupy.full(size, 1 / size, dtype=dtype)
+
+    return _run_1d_correlates(input, sizes, get, output, mode, cval, origin)
+
+
+def gaussian_filter1d(input, sigma, axis=-1, order=0, output=None,
+                      mode="reflect", cval=0.0, truncate=4.0):
+    """One-dimensional Gaussian filter along the given axis.
+
+    The lines of the array along the given axis are filtered with a Gaussian
+    filter of the given standard deviation.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        sigma (scalar): Standard deviation for Gaussian kernel.
+        axis (int): The axis of input along which to calculate. Default is -1.
+        order (int): An order of ``0``, the default, corresponds to convolution
+            with a Gaussian kernel. A positive order corresponds to convolution
+            with that derivative of a Gaussian.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        truncate (float): Truncate the filter at this many standard deviations.
+            Default is ``4.0``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.gaussian_filter1d`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    radius = int(float(truncate) * float(sigma) + 0.5)
+    weights_dtype = _util._init_weights_dtype(input)
+    weights = _gaussian_kernel1d(
+        sigma, int(order), radius, dtype=weights_dtype
+    )
+    return correlate1d(input, weights, axis, output, mode, cval)
+
+
+def gaussian_filter(input, sigma, order=0, output=None, mode="reflect",
+                    cval=0.0, truncate=4.0):
+    """Multi-dimensional Gaussian filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        sigma (scalar or sequence of scalar): Standard deviations for each axis
+            of Gaussian kernel. A single value applies to all axes.
+        order (int or sequence of scalar): An order of ``0``, the default,
+            corresponds to convolution with a Gaussian kernel. A positive order
+            corresponds to convolution with that derivative of a Gaussian. A
+            single value applies to all axes.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        truncate (float): Truncate the filter at this many standard deviations.
+            Default is ``4.0``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.gaussian_filter`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    sigmas = _util._fix_sequence_arg(sigma, input.ndim, 'sigma', float)
+    orders = _util._fix_sequence_arg(order, input.ndim, 'order', int)
+    truncate = float(truncate)
+    weights_dtype = _util._init_weights_dtype(input)
+
+    def get(param):
+        sigma, order = param
+        radius = int(truncate * float(sigma) + 0.5)
+        if radius <= 0:
+            return None
+        return _gaussian_kernel1d(sigma, order, radius, dtype=weights_dtype)
+
+    return _run_1d_correlates(input, list(zip(sigmas, orders)), get, output,
+                              mode, cval, 0)
+
+
+def _gaussian_kernel1d(sigma, order, radius, dtype=cupy.float64):
+    """
+    Computes a 1-D Gaussian correlation kernel.
+    """
+    if order < 0:
+        raise ValueError('order must be non-negative')
+    sigma2 = sigma * sigma
+    x = numpy.arange(-radius, radius+1)
+    phi_x = numpy.exp(-0.5 / sigma2 * x ** 2)
+    phi_x /= phi_x.sum()
+
+    if order == 0:
+        return cupy.asarray(phi_x)
+
+    # f(x) = q(x) * phi(x) = q(x) * exp(p(x))
+    # f'(x) = (q'(x) + q(x) * p'(x)) * phi(x)
+    # p'(x) = -1 / sigma ** 2
+    # Implement q'(x) + q(x) * p'(x) as a matrix operator and apply to the
+    # coefficients of q(x)
+    exponent_range = numpy.arange(order + 1)
+    q = numpy.zeros(order + 1)
+    q[0] = 1
+    D = numpy.diag(exponent_range[1:], 1)  # D @ q(x) = q'(x)
+    P = numpy.diag(numpy.ones(order)/-sigma2, -1)  # P @ q(x) = q(x) * p'(x)
+    Q_deriv = D + P
+    for _ in range(order):
+        q = Q_deriv.dot(q)
+    q = (x[:, None] ** exponent_range).dot(q)
+    return cupy.asarray((q * phi_x)[::-1], dtype=dtype)
+
+
+def prewitt(input, axis=-1, output=None, mode="reflect", cval=0.0):
+    """Compute a Prewitt filter along the given axis.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        axis (int): The axis of input along which to calculate. Default is -1.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.prewitt`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    weights_dtype = _util._init_weights_dtype(input)
+    weights = cupy.ones(3, dtype=weights_dtype)
+    return _prewitt_or_sobel(input, axis, output, mode, cval, weights)
+
+
+def sobel(input, axis=-1, output=None, mode="reflect", cval=0.0):
+    """Compute a Sobel filter along the given axis.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        axis (int): The axis of input along which to calculate. Default is -1.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.sobel`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    weights_dtype = _util._init_weights_dtype(input)
+    return _prewitt_or_sobel(input, axis, output, mode, cval,
+                             cupy.array([1, 2, 1], dtype=weights_dtype))
+
+
+def _prewitt_or_sobel(input, axis, output, mode, cval, weights):
+    axis = internal._normalize_axis_index(axis, input.ndim)
+
+    def get(is_diff):
+        return cupy.array([-1, 0, 1], dtype=weights.dtype) if is_diff else weights  # noqa
+
+    return _run_1d_correlates(input, [a == axis for a in range(input.ndim)],
+                              get, output, mode, cval)
+
+
+def generic_laplace(input, derivative2, output=None, mode="reflect",
+                    cval=0.0, extra_arguments=(), extra_keywords=None):
+    """Multi-dimensional Laplace filter using a provided second derivative
+    function.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        derivative2 (callable): Function or other callable with the following
+            signature that is called once per axis::
+
+                derivative2(input, axis, output, mode, cval,
+                            *extra_arguments, **extra_keywords)
+
+            where ``input`` and ``output`` are ``cupy.ndarray``, ``axis`` is an
+            ``int`` from ``0`` to the number of dimensions, and ``mode``,
+            ``cval``, ``extra_arguments``, ``extra_keywords`` are the values
+            given to this function.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        extra_arguments (sequence, optional):
+            Sequence of extra positional arguments to pass to ``derivative2``.
+        extra_keywords (dict, optional):
+            dict of extra keyword arguments to pass ``derivative2``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.generic_laplace`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    if extra_keywords is None:
+        extra_keywords = {}
+    ndim = input.ndim
+    modes = _util._fix_sequence_arg(mode, ndim, 'mode',
+                                    _util._check_mode)
+    output = _util._get_output(output, input)
+    if ndim == 0:
+        _core.elementwise_copy(input, output)
+        return output
+    derivative2(input, 0, output, modes[0], cval,
+                *extra_arguments, **extra_keywords)
+    if ndim > 1:
+        tmp = _util._get_output(output.dtype, input)
+        for i in range(1, ndim):
+            derivative2(input, i, tmp, modes[i], cval,
+                        *extra_arguments, **extra_keywords)
+            output += tmp
+    return output
+
+
+def laplace(input, output=None, mode="reflect", cval=0.0):
+    """Multi-dimensional Laplace filter based on approximate second
+    derivatives.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.laplace`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    weights_dtype = _util._init_weights_dtype(input)
+    weights = cupy.array([1, -2, 1], dtype=weights_dtype)
+
+    def derivative2(input, axis, output, mode, cval):
+        return correlate1d(input, weights, axis, output, mode, cval)
+
+    return generic_laplace(input, derivative2, output, mode, cval)
+
+
+def gaussian_laplace(input, sigma, output=None, mode="reflect",
+                     cval=0.0, **kwargs):
+    """Multi-dimensional Laplace filter using Gaussian second derivatives.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        sigma (scalar or sequence of scalar): Standard deviations for each axis
+            of Gaussian kernel. A single value applies to all axes.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        kwargs (dict, optional):
+            dict of extra keyword arguments to pass ``gaussian_filter()``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.gaussian_laplace`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    def derivative2(input, axis, output, mode, cval):
+        order = [0] * input.ndim
+        order[axis] = 2
+        return gaussian_filter(input, sigma, order, output, mode, cval,
+                               **kwargs)
+    return generic_laplace(input, derivative2, output, mode, cval)
+
+
+def generic_gradient_magnitude(input, derivative, output=None,
+                               mode="reflect", cval=0.0,
+                               extra_arguments=(), extra_keywords=None):
+    """Multi-dimensional gradient magnitude filter using a provided derivative
+    function.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        derivative (callable): Function or other callable with the following
+            signature that is called once per axis::
+
+                derivative(input, axis, output, mode, cval,
+                           *extra_arguments, **extra_keywords)
+
+            where ``input`` and ``output`` are ``cupy.ndarray``, ``axis`` is an
+            ``int`` from ``0`` to the number of dimensions, and ``mode``,
+            ``cval``, ``extra_arguments``, ``extra_keywords`` are the values
+            given to this function.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        extra_arguments (sequence, optional):
+            Sequence of extra positional arguments to pass to ``derivative2``.
+        extra_keywords (dict, optional):
+            dict of extra keyword arguments to pass ``derivative2``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.generic_gradient_magnitude`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    if extra_keywords is None:
+        extra_keywords = {}
+    ndim = input.ndim
+    modes = _util._fix_sequence_arg(mode, ndim, 'mode',
+                                    _util._check_mode)
+    output = _util._get_output(output, input)
+    if ndim == 0:
+        _core.elementwise_copy(input, output)
+        return output
+    derivative(input, 0, output, modes[0], cval,
+               *extra_arguments, **extra_keywords)
+    output *= output
+    if ndim > 1:
+        tmp = _util._get_output(output.dtype, input)
+        for i in range(1, ndim):
+            derivative(input, i, tmp, modes[i], cval,
+                       *extra_arguments, **extra_keywords)
+            tmp *= tmp
+            output += tmp
+    return cupy.sqrt(output, output, casting='unsafe')
+
+
+def gaussian_gradient_magnitude(input, sigma, output=None, mode="reflect",
+                                cval=0.0, **kwargs):
+    """Multi-dimensional gradient magnitude using Gaussian derivatives.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        sigma (scalar or sequence of scalar): Standard deviations for each axis
+            of Gaussian kernel. A single value applies to all axes.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        kwargs (dict, optional):
+            dict of extra keyword arguments to pass ``gaussian_filter()``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.gaussian_gradient_magnitude`
+
+    .. note::
+        When the output data type is integral (or when no output is provided
+        and input is integral) the results may not perfectly match the results
+        from SciPy due to floating-point rounding of intermediate results.
+    """
+    def derivative(input, axis, output, mode, cval):
+        order = [0] * input.ndim
+        order[axis] = 1
+        return gaussian_filter(input, sigma, order, output, mode, cval,
+                               **kwargs)
+    return generic_gradient_magnitude(input, derivative, output, mode, cval)
+
+
+def minimum_filter(input, size=None, footprint=None, output=None,
+                   mode="reflect", cval=0.0, origin=0):
+    """Multi-dimensional minimum filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (int or sequence of int): One of ``size`` or ``footprint`` must be
+            provided. If ``footprint`` is given, ``size`` is ignored. Otherwise
+            ``footprint = cupy.ones(size)`` with ``size`` automatically made to
+            match the number of dimensions in ``input``.
+        footprint (cupy.ndarray): a boolean array which specifies which of the
+            elements within this shape will get passed to the filter function.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int or sequence of int): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.minimum_filter`
+    """
+    return _min_or_max_filter(input, size, footprint, None, output, mode,
+                              cval, origin, 'min')
+
+
+def maximum_filter(input, size=None, footprint=None, output=None,
+                   mode="reflect", cval=0.0, origin=0):
+    """Multi-dimensional maximum filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (int or sequence of int): One of ``size`` or ``footprint`` must be
+            provided. If ``footprint`` is given, ``size`` is ignored. Otherwise
+            ``footprint = cupy.ones(size)`` with ``size`` automatically made to
+            match the number of dimensions in ``input``.
+        footprint (cupy.ndarray): a boolean array which specifies which of the
+            elements within this shape will get passed to the filter function.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int or sequence of int): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.maximum_filter`
+    """
+    return _min_or_max_filter(input, size, footprint, None, output, mode,
+                              cval, origin, 'max')
+
+
+def _min_or_max_filter(input, size, ftprnt, structure, output, mode, cval,
+                       origin, func):
+    # structure is used by morphology.grey_erosion() and grey_dilation()
+    # and not by the regular min/max filters
+
+    sizes, ftprnt, structure = _filters_core._check_size_footprint_structure(
+        input.ndim, size, ftprnt, structure)
+    if cval is cupy.nan:
+        raise NotImplementedError("NaN cval is unsupported")
+
+    if sizes is not None:
+        # Seperable filter, run as a series of 1D filters
+        fltr = minimum_filter1d if func == 'min' else maximum_filter1d
+        return _filters_core._run_1d_filters(
+            [fltr if size > 1 else None for size in sizes],
+            input, sizes, output, mode, cval, origin)
+
+    origins, int_type = _filters_core._check_nd_args(input, ftprnt,
+                                                     mode, origin, 'footprint')
+    if structure is not None and structure.ndim != input.ndim:
+        raise RuntimeError('structure array has incorrect shape')
+
+    if ftprnt.size == 0:
+        return cupy.zeros_like(input)
+    offsets = _filters_core._origins_to_offsets(origins, ftprnt.shape)
+    kernel = _get_min_or_max_kernel(mode, ftprnt.shape, func,
+                                    offsets, float(cval), int_type,
+                                    has_structure=structure is not None,
+                                    has_central_value=bool(ftprnt[offsets]))
+    return _filters_core._call_kernel(kernel, input, ftprnt, output,
+                                      structure, weights_dtype=bool)
+
+
+def minimum_filter1d(input, size, axis=-1, output=None, mode="reflect",
+                     cval=0.0, origin=0):
+    """Compute the minimum filter along a single axis.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (int): Length of the minimum filter.
+        axis (int): The axis of input along which to calculate. Default is -1.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int): The origin parameter controls the placement of the
+            filter, relative to the center of the current element of the
+            input. Default is ``0``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.minimum_filter1d`
+    """
+    return _min_or_max_1d(input, size, axis, output, mode, cval, origin, 'min')
+
+
+def maximum_filter1d(input, size, axis=-1, output=None, mode="reflect",
+                     cval=0.0, origin=0):
+    """Compute the maximum filter along a single axis.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (int): Length of the maximum filter.
+        axis (int): The axis of input along which to calculate. Default is -1.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int): The origin parameter controls the placement of the
+            filter, relative to the center of the current element of the
+            input. Default is ``0``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.maximum_filter1d`
+    """
+    return _min_or_max_1d(input, size, axis, output, mode, cval, origin, 'max')
+
+
+def _min_or_max_1d(input, size, axis=-1, output=None, mode="reflect", cval=0.0,
+                   origin=0, func='min'):
+    ftprnt = cupy.ones(size, dtype=bool)
+    ftprnt, origin = _filters_core._convert_1d_args(input.ndim, ftprnt,
+                                                    origin, axis)
+    origins, int_type = _filters_core._check_nd_args(input, ftprnt,
+                                                     mode, origin, 'footprint')
+    offsets = _filters_core._origins_to_offsets(origins, ftprnt.shape)
+    kernel = _get_min_or_max_kernel(mode, ftprnt.shape, func, offsets,
+                                    float(cval), int_type, has_weights=False)
+    return _filters_core._call_kernel(kernel, input, None, output,
+                                      weights_dtype=bool)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_min_or_max_kernel(mode, w_shape, func, offsets, cval, int_type,
+                           has_weights=True, has_structure=False,
+                           has_central_value=True):
+    # When there are no 'weights' (the footprint, for the 1D variants) then
+    # we need to make sure intermediate results are stored as doubles for
+    # consistent results with scipy.
+    ctype = 'X' if has_weights else 'double'
+    value = '{value}'
+    if not has_weights:
+        value = 'cast<double>({})'.format(value)
+
+    # Having a non-flat structure biases the values
+    if has_structure:
+        value += ('-' if func == 'min' else '+') + 'cast<X>(sval)'
+
+    if has_central_value:
+        pre = '{} value = x[i];'
+        found = 'value = {func}({value}, value);'
+    else:
+        # If the central pixel is not included in the footprint we cannot
+        # assume `x[i]` is not below the min or above the max and thus cannot
+        # seed with that value. Instead we keep track of having set `value`.
+        pre = '{} value; bool set = false;'
+        found = 'value = set ? {func}({value}, value) : {value}; set=true;'
+
+    return _filters_core._generate_nd_kernel(
+        func, pre.format(ctype),
+        found.format(func=func, value=value), 'y = cast<Y>(value);',
+        mode, w_shape, int_type, offsets, cval, ctype=ctype,
+        has_weights=has_weights, has_structure=has_structure)
+
+
+def rank_filter(input, rank, size=None, footprint=None, output=None,
+                mode="reflect", cval=0.0, origin=0):
+    """Multi-dimensional rank filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        rank (int): The rank of the element to get. Can be negative to count
+            from the largest value, e.g. ``-1`` indicates the largest value.
+        size (int or sequence of int): One of ``size`` or ``footprint`` must be
+            provided. If ``footprint`` is given, ``size`` is ignored. Otherwise
+            ``footprint = cupy.ones(size)`` with ``size`` automatically made to
+            match the number of dimensions in ``input``.
+        footprint (cupy.ndarray): a boolean array which specifies which of the
+            elements within this shape will get passed to the filter function.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int or sequence of int): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.rank_filter`
+    """
+    rank = int(rank)
+    return _rank_filter(input, lambda fs: rank+fs if rank < 0 else rank,
+                        size, footprint, output, mode, cval, origin)
+
+
+def median_filter(input, size=None, footprint=None, output=None,
+                  mode="reflect", cval=0.0, origin=0):
+    """Multi-dimensional median filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (int or sequence of int): One of ``size`` or ``footprint`` must be
+            provided. If ``footprint`` is given, ``size`` is ignored. Otherwise
+            ``footprint = cupy.ones(size)`` with ``size`` automatically made to
+            match the number of dimensions in ``input``.
+        footprint (cupy.ndarray): a boolean array which specifies which of the
+            elements within this shape will get passed to the filter function.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int or sequence of int): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.median_filter`
+    """
+    return _rank_filter(input, lambda fs: fs//2,
+                        size, footprint, output, mode, cval, origin)
+
+
+def percentile_filter(input, percentile, size=None, footprint=None,
+                      output=None, mode="reflect", cval=0.0, origin=0):
+    """Multi-dimensional percentile filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        percentile (scalar): The percentile of the element to get (from ``0``
+            to ``100``). Can be negative, thus ``-20`` equals ``80``.
+        size (int or sequence of int): One of ``size`` or ``footprint`` must be
+            provided. If ``footprint`` is given, ``size`` is ignored. Otherwise
+            ``footprint = cupy.ones(size)`` with ``size`` automatically made to
+            match the number of dimensions in ``input``.
+        footprint (cupy.ndarray): a boolean array which specifies which of the
+            elements within this shape will get passed to the filter function.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int or sequence of int): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. seealso:: :func:`scipy.ndimage.percentile_filter`
+    """
+    percentile = float(percentile)
+    if percentile < 0.0:
+        percentile += 100.0
+    if percentile < 0.0 or percentile > 100.0:
+        raise RuntimeError('invalid percentile')
+    if percentile == 100.0:
+        def get_rank(fs):
+            return fs - 1
+    else:
+        def get_rank(fs):
+            return int(float(fs) * percentile / 100.0)
+    return _rank_filter(input, get_rank,
+                        size, footprint, output, mode, cval, origin)
+
+
+def _rank_filter(input, get_rank, size=None, footprint=None, output=None,
+                 mode="reflect", cval=0.0, origin=0):
+    _, footprint, _ = _filters_core._check_size_footprint_structure(
+        input.ndim, size, footprint, None, force_footprint=True)
+    if cval is cupy.nan:
+        raise NotImplementedError("NaN cval is unsupported")
+    origins, int_type = _filters_core._check_nd_args(input, footprint,
+                                                     mode, origin, 'footprint')
+    if footprint.size == 0:
+        return cupy.zeros_like(input)
+    filter_size = int(footprint.sum())
+    rank = get_rank(filter_size)
+    if rank < 0 or rank >= filter_size:
+        raise RuntimeError('rank not within filter footprint size')
+    if rank == 0:
+        return _min_or_max_filter(input, None, footprint, None, output, mode,
+                                  cval, origins, 'min')
+    if rank == filter_size - 1:
+        return _min_or_max_filter(input, None, footprint, None, output, mode,
+                                  cval, origins, 'max')
+    offsets = _filters_core._origins_to_offsets(origins, footprint.shape)
+    kernel = _get_rank_kernel(filter_size, rank, mode, footprint.shape,
+                              offsets, float(cval), int_type)
+    return _filters_core._call_kernel(kernel, input, footprint, output,
+                                      weights_dtype=bool)
+
+
+__SHELL_SORT = '''
+__device__ void sort(X *array, int size) {{
+    int gap = {gap};
+    while (gap > 1) {{
+        gap /= 3;
+        for (int i = gap; i < size; ++i) {{
+            X value = array[i];
+            int j = i - gap;
+            while (j >= 0 && value < array[j]) {{
+                array[j + gap] = array[j];
+                j -= gap;
+            }}
+            array[j + gap] = value;
+        }}
+    }}
+}}'''
+
+
+@cupy._util.memoize()
+def _get_shell_gap(filter_size):
+    gap = 1
+    while gap < filter_size:
+        gap = 3*gap+1
+    return gap
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_rank_kernel(filter_size, rank, mode, w_shape, offsets, cval,
+                     int_type):
+    s_rank = min(rank, filter_size - rank - 1)
+    # The threshold was set based on the measurements on a V100
+    # TODO(leofang, anaruse): Use Optuna to automatically tune the threshold,
+    # as it may vary depending on the GPU in use, compiler version, dtype,
+    # filter size, etc.
+    if s_rank <= 80:
+        # When s_rank is small and register usage is low, this partial
+        # selection sort approach is faster than general sorting approach
+        # using shell sort.
+        if s_rank == rank:
+            comp_op = '<'
+        else:
+            comp_op = '>'
+        array_size = s_rank + 2
+        found_post = '''
+            if (iv > {rank} + 1) {{{{
+                int target_iv = 0;
+                X target_val = values[0];
+                for (int jv = 1; jv <= {rank} + 1; jv++) {{{{
+                    if (target_val {comp_op} values[jv]) {{{{
+                        target_val = values[jv];
+                        target_iv = jv;
+                    }}}}
+                }}}}
+                if (target_iv <= {rank}) {{{{
+                    values[target_iv] = values[{rank} + 1];
+                }}}}
+                iv = {rank} + 1;
+            }}}}'''.format(rank=s_rank, comp_op=comp_op)
+        post = '''
+            X target_val = values[0];
+            for (int jv = 1; jv <= {rank}; jv++) {{
+                if (target_val {comp_op} values[jv]) {{
+                    target_val = values[jv];
+                }}
+            }}
+            y=cast<Y>(target_val);'''.format(rank=s_rank, comp_op=comp_op)
+        sorter = ''
+    else:
+        array_size = filter_size
+        found_post = ''
+        post = 'sort(values,{});\ny=cast<Y>(values[{}]);'.format(
+            filter_size, rank)
+        sorter = __SHELL_SORT.format(gap=_get_shell_gap(filter_size))
+
+    return _filters_core._generate_nd_kernel(
+        'rank_{}_{}'.format(filter_size, rank),
+        'int iv = 0;\nX values[{}];'.format(array_size),
+        'values[iv++] = {value};' + found_post, post,
+        mode, w_shape, int_type, offsets, cval, preamble=sorter)
+
+
+def generic_filter(input, function, size=None, footprint=None,
+                   output=None, mode="reflect", cval=0.0, origin=0):
+    """Compute a multi-dimensional filter using the provided raw kernel or
+    reduction kernel.
+
+    Unlike the scipy.ndimage function, this does not support the
+    ``extra_arguments`` or ``extra_keywordsdict`` arguments and has significant
+    restrictions on the ``function`` provided.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        function (cupy.ReductionKernel or cupy.RawKernel):
+            The kernel or function to apply to each region.
+        size (int or sequence of int): One of ``size`` or ``footprint`` must be
+            provided. If ``footprint`` is given, ``size`` is ignored. Otherwise
+            ``footprint = cupy.ones(size)`` with ``size`` automatically made to
+            match the number of dimensions in ``input``.
+        footprint (cupy.ndarray): a boolean array which specifies which of the
+            elements within this shape will get passed to the filter function.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. note::
+        If the `function` is a :class:`cupy.RawKernel` then it must be for a
+        function that has the following signature. Unlike most functions, this
+        should not utilize `blockDim`/`blockIdx`/`threadIdx`::
+
+            __global__ void func(double *buffer, int filter_size,
+                                 double *return_value)
+
+        If the `function` is a :class:`cupy.ReductionKernel` then it must be
+        for a kernel that takes 1 array input and produces 1 'scalar' output.
+
+    .. seealso:: :func:`scipy.ndimage.generic_filter`
+    """
+    _, footprint, _ = _filters_core._check_size_footprint_structure(
+        input.ndim, size, footprint, None, 2, True)
+    filter_size = int(footprint.sum())
+    origins, int_type = _filters_core._check_nd_args(input, footprint,
+                                                     mode, origin, 'footprint')
+    in_dtype = input.dtype
+    sub = _filters_generic._get_sub_kernel(function)
+    if footprint.size == 0:
+        return cupy.zeros_like(input)
+    output = _util._get_output(output, input)
+    offsets = _filters_core._origins_to_offsets(origins, footprint.shape)
+    args = (filter_size, mode, footprint.shape, offsets, float(cval), int_type)
+    if isinstance(sub, cupy.RawKernel):
+        kernel = _filters_generic._get_generic_filter_raw(sub, *args)
+    elif isinstance(sub, cupy.ReductionKernel):
+        kernel = _filters_generic._get_generic_filter_red(
+            sub, in_dtype, output.dtype, *args)
+    return _filters_core._call_kernel(kernel, input, footprint, output,
+                                      weights_dtype=bool)
+
+
+def generic_filter1d(input, function, filter_size, axis=-1, output=None,
+                     mode="reflect", cval=0.0, origin=0):
+    """Compute a 1D filter along the given axis using the provided raw kernel.
+
+    Unlike the scipy.ndimage function, this does not support the
+    ``extra_arguments`` or ``extra_keywordsdict`` arguments and has significant
+    restrictions on the ``function`` provided.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        function (cupy.RawKernel): The kernel to apply along each axis.
+        filter_size (int): Length of the filter.
+        axis (int): The axis of input along which to calculate. Default is -1.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output. Default is is same dtype as the input.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``'constant'``. Default is ``0.0``.
+        origin (int): The origin parameter controls the placement of the
+            filter, relative to the center of the current element of the
+            input. Default is ``0``.
+
+    Returns:
+        cupy.ndarray: The result of the filtering.
+
+    .. note::
+        The provided function (as a RawKernel) must have the following
+        signature. Unlike most functions, this should not utilize
+        `blockDim`/`blockIdx`/`threadIdx`::
+
+            __global__ void func(double *input_line, ptrdiff_t input_length,
+                                 double *output_line, ptrdiff_t output_length)
+
+    .. seealso:: :func:`scipy.ndimage.generic_filter1d`
+    """
+    # This filter is very different than all other filters (including
+    # generic_filter and all 1d filters) and it has a customized solution.
+    # It is also likely fairly terrible, but only so much can be done when
+    # matching the scipy interface of having the sub-kernel work on entire
+    # lines of data.
+    if input.dtype.kind == 'c':
+        raise TypeError('Complex type not supported')
+    if not isinstance(function, cupy.RawKernel):
+        raise TypeError('bad function type')
+    if filter_size < 1:
+        raise RuntimeError('invalid filter size')
+    axis = internal._normalize_axis_index(axis, input.ndim)
+    origin = _util._check_origin(origin, filter_size)
+    _util._check_mode(mode)
+    output = _util._get_output(output, input)
+    in_ctype = cupy._core._scalar.get_typename(input.dtype)
+    out_ctype = cupy._core._scalar.get_typename(output.dtype)
+    int_type = _util._get_inttype(input)
+    n_lines = input.size // input.shape[axis]
+    kernel = _filters_generic._get_generic_filter1d(
+        function, input.shape[axis], n_lines, filter_size,
+        origin, mode, float(cval), in_ctype, out_ctype, int_type)
+    data = cupy.array(
+        (axis, input.ndim) + input.shape + input.strides + output.strides,
+        dtype=cupy.int32 if int_type == 'int' else cupy.int64)
+    kernel(((n_lines+128-1) // 128,), (128,), (input, output, data))
+    return output
diff --git a/cupyx/scipy/ndimage/_filters_core.py b/cupyx/scipy/ndimage/_filters_core.py
new file mode 100644
index 0000000..3f09d22
--- /dev/null
+++ b/cupyx/scipy/ndimage/_filters_core.py
@@ -0,0 +1,309 @@
+import warnings
+
+import numpy
+import cupy
+
+from cupy_backends.cuda.api import runtime
+from cupy import _core
+from cupy._core import internal
+from cupyx.scipy.ndimage import _util
+
+
+def _origins_to_offsets(origins, w_shape):
+    return tuple(x//2+o for x, o in zip(w_shape, origins))
+
+
+def _check_size_footprint_structure(ndim, size, footprint, structure,
+                                    stacklevel=3, force_footprint=False):
+    if structure is None and footprint is None:
+        if size is None:
+            raise RuntimeError("no footprint or filter size provided")
+        sizes = _util._fix_sequence_arg(size, ndim, 'size', int)
+        if force_footprint:
+            return None, cupy.ones(sizes, bool), None
+        return sizes, None, None
+    if size is not None:
+        warnings.warn("ignoring size because {} is set".format(
+            'structure' if footprint is None else 'footprint'),
+            UserWarning, stacklevel=stacklevel+1)
+
+    if footprint is not None:
+        footprint = cupy.array(footprint, bool, True, 'C')
+        if not footprint.any():
+            raise ValueError("all-zero footprint is not supported")
+
+    if structure is None:
+        if not force_footprint and footprint.all():
+            if footprint.ndim != ndim:
+                raise RuntimeError("size must have length equal to input rank")
+            return footprint.shape, None, None
+        return None, footprint, None
+
+    structure = cupy.ascontiguousarray(structure)
+    if footprint is None:
+        footprint = cupy.ones(structure.shape, bool)
+    return None, footprint, structure
+
+
+def _convert_1d_args(ndim, weights, origin, axis):
+    if weights.ndim != 1 or weights.size < 1:
+        raise RuntimeError('incorrect filter size')
+    axis = internal._normalize_axis_index(axis, ndim)
+    w_shape = [1]*ndim
+    w_shape[axis] = weights.size
+    weights = weights.reshape(w_shape)
+    origins = [0]*ndim
+    origins[axis] = _util._check_origin(origin, weights.size)
+    return weights, tuple(origins)
+
+
+def _check_nd_args(input, weights, mode, origin, wghts_name='filter weights'):
+    _util._check_mode(mode)
+    # Weights must always be less than 2 GiB
+    if weights.nbytes >= (1 << 31):
+        raise RuntimeError('weights must be 2 GiB or less, use FFTs instead')
+    weight_dims = [x for x in weights.shape if x != 0]
+    if len(weight_dims) != input.ndim:
+        raise RuntimeError('{} array has incorrect shape'.format(wghts_name))
+    origins = _util._fix_sequence_arg(origin, len(weight_dims), 'origin', int)
+    for origin, width in zip(origins, weight_dims):
+        _util._check_origin(origin, width)
+    return tuple(origins), _util._get_inttype(input)
+
+
+def _run_1d_filters(filters, input, args, output, mode, cval, origin=0):
+    """
+    Runs a series of 1D filters forming an nd filter. The filters must be a
+    list of callables that take input, arg, axis, output, mode, cval, origin.
+    The args is a list of values that are passed for the arg value to the
+    filter. Individual filters can be None causing that axis to be skipped.
+    """
+    output = _util._get_output(output, input)
+    modes = _util._fix_sequence_arg(mode, input.ndim, 'mode',
+                                    _util._check_mode)
+    # for filters, "wrap" is a synonym for "grid-wrap".
+    modes = ['grid-wrap' if m == 'wrap' else m for m in modes]
+    origins = _util._fix_sequence_arg(origin, input.ndim, 'origin', int)
+    n_filters = sum(filter is not None for filter in filters)
+    if n_filters == 0:
+        _core.elementwise_copy(input, output)
+        return output
+    # We can't operate in-place efficiently, so use a 2-buffer system
+    temp = _util._get_output(output.dtype, input) if n_filters > 1 else None
+    iterator = zip(filters, args, modes, origins)
+    # skip any axes where the filter is None
+    for axis, (fltr, arg, mode, origin) in enumerate(iterator):
+        if fltr is not None:
+            break
+    # To avoid need for any additional copies, we have to start with a
+    # different output array depending on whether the total number of filters
+    # is odd or even.
+    if n_filters % 2 == 0:
+        fltr(input, arg, axis, temp, mode, cval, origin)
+        input = temp
+    else:
+        fltr(input, arg, axis, output, mode, cval, origin)
+        input, output = output, temp
+    for axis, (fltr, arg, mode, origin) in enumerate(iterator, start=axis + 1):
+        if fltr is None:
+            continue
+        fltr(input, arg, axis, output, mode, cval, origin)
+        input, output = output, input
+    return input
+
+
+def _call_kernel(kernel, input, weights, output, structure=None,
+                 weights_dtype=numpy.float64, structure_dtype=numpy.float64):
+    """
+    Calls a constructed ElementwiseKernel. The kernel must take an input image,
+    an optional array of weights, an optional array for the structure, and an
+    output array.
+
+    weights and structure can be given as None (structure defaults to None) in
+    which case they are not passed to the kernel at all. If the output is given
+    as None then it will be allocated in this function.
+
+    This function deals with making sure that the weights and structure are
+    contiguous and float64 (or bool for weights that are footprints)*, that the
+    output is allocated and appriopately shaped. This also deals with the
+    situation that the input and output arrays overlap in memory.
+
+    * weights is always cast to float64 or bool in order to get an output
+    compatible with SciPy, though float32 might be sufficient when input dtype
+    is low precision. If weights_dtype is passed as weights.dtype then no
+    dtype conversion will occur. The input and output are never converted.
+    """
+    args = [input]
+    complex_output = input.dtype.kind == 'c'
+    if weights is not None:
+        weights = cupy.ascontiguousarray(weights, weights_dtype)
+        complex_output = complex_output or weights.dtype.kind == 'c'
+        args.append(weights)
+    if structure is not None:
+        structure = cupy.ascontiguousarray(structure, structure_dtype)
+        args.append(structure)
+    output = _util._get_output(output, input, None, complex_output)
+    needs_temp = cupy.shares_memory(output, input, 'MAY_SHARE_BOUNDS')
+    if needs_temp:
+        output, temp = _util._get_output(output.dtype, input), output
+    args.append(output)
+    kernel(*args)
+    if needs_temp:
+        _core.elementwise_copy(temp, output)
+        output = temp
+    return output
+
+
+if runtime.is_hip:
+    includes = r'''
+// workaround for HIP: line begins with #include
+#include <cupy/math_constants.h>\n
+'''
+else:
+    includes = r'''
+#include <type_traits>  // let Jitify handle this
+#include <cupy/math_constants.h>
+
+template<> struct std::is_floating_point<float16> : std::true_type {};
+template<> struct std::is_signed<float16> : std::true_type {};
+template<class T> struct std::is_signed<complex<T>> : std::is_signed<T> {};
+'''
+
+
+_CAST_FUNCTION = """
+// Implements a casting function to make it compatible with scipy
+// Use like cast<to_type>(value)
+template <class B, class A>
+__device__ __forceinline__
+typename std::enable_if<(!std::is_floating_point<A>::value
+                         || std::is_signed<B>::value), B>::type
+cast(A a) { return (B)a; }
+
+template <class B, class A>
+__device__ __forceinline__
+typename std::enable_if<(std::is_floating_point<A>::value
+                         && (!std::is_signed<B>::value)), B>::type
+cast(A a) { return (a >= 0) ? (B)a : -(B)(-a); }
+
+template <class T>
+__device__ __forceinline__ bool nonzero(T x) { return x != static_cast<T>(0); }
+"""
+
+
+def _generate_nd_kernel(name, pre, found, post, mode, w_shape, int_type,
+                        offsets, cval, ctype='X', preamble='', options=(),
+                        has_weights=True, has_structure=False, has_mask=False,
+                        binary_morphology=False, all_weights_nonzero=False):
+    # Currently this code uses CArray for weights but avoids using CArray for
+    # the input data and instead does the indexing itself since it is faster.
+    # If CArray becomes faster than follow the comments that start with
+    # CArray: to switch over to using CArray for the input data as well.
+
+    ndim = len(w_shape)
+    in_params = 'raw X x'
+    if has_weights:
+        in_params += ', raw W w'
+    if has_structure:
+        in_params += ', raw S s'
+    if has_mask:
+        in_params += ', raw M mask'
+    out_params = 'Y y'
+
+    # for filters, "wrap" is a synonym for "grid-wrap"
+    mode = 'grid-wrap' if mode == 'wrap' else mode
+
+    # CArray: remove xstride_{j}=... from string
+    size = ('%s xsize_{j}=x.shape()[{j}], ysize_{j} = _raw_y.shape()[{j}]'
+            ', xstride_{j}=x.strides()[{j}];' % int_type)
+    sizes = [size.format(j=j) for j in range(ndim)]
+    inds = _util._generate_indices_ops(ndim, int_type, offsets)
+    # CArray: remove expr entirely
+    expr = ' + '.join(['ix_{}'.format(j) for j in range(ndim)])
+
+    ws_init = ws_pre = ws_post = ''
+    if has_weights or has_structure:
+        ws_init = 'int iws = 0;'
+        if has_structure:
+            ws_pre = 'S sval = s[iws];\n'
+        if has_weights:
+            ws_pre += 'W wval = w[iws];\n'
+            if not all_weights_nonzero:
+                ws_pre += 'if (nonzero(wval))'
+        ws_post = 'iws++;'
+
+    loops = []
+    for j in range(ndim):
+        if w_shape[j] == 1:
+            # CArray: string becomes 'inds[{j}] = ind_{j};', remove (int_)type
+            loops.append('{{ {type} ix_{j} = ind_{j} * xstride_{j};'.
+                         format(j=j, type=int_type))
+        else:
+            boundary = _util._generate_boundary_condition_ops(
+                mode, 'ix_{}'.format(j), 'xsize_{}'.format(j), int_type)
+            # CArray: last line of string becomes inds[{j}] = ix_{j};
+            loops.append('''
+    for (int iw_{j} = 0; iw_{j} < {wsize}; iw_{j}++)
+    {{
+        {type} ix_{j} = ind_{j} + iw_{j};
+        {boundary}
+        ix_{j} *= xstride_{j};
+        '''.format(j=j, wsize=w_shape[j], boundary=boundary, type=int_type))
+
+    # CArray: string becomes 'x[inds]', no format call needed
+    value = '(*(X*)&data[{expr}])'.format(expr=expr)
+    if mode == 'constant':
+        cond = ' || '.join(['(ix_{} < 0)'.format(j) for j in range(ndim)])
+
+    if cval is numpy.nan:
+        cval = 'CUDART_NAN'
+    elif cval == numpy.inf:
+        cval = 'CUDART_INF'
+    elif cval == -numpy.inf:
+        cval = '-CUDART_INF'
+
+    if binary_morphology:
+        found = found.format(cond=cond, value=value)
+    else:
+        if mode == 'constant':
+            value = '(({cond}) ? cast<{ctype}>({cval}) : {value})'.format(
+                cond=cond, ctype=ctype, cval=cval, value=value)
+        found = found.format(value=value)
+
+    # CArray: replace comment and next line in string with
+    #   {type} inds[{ndim}] = {{0}};
+    # and add ndim=ndim, type=int_type to format call
+    operation = '''
+    {sizes}
+    {inds}
+    // don't use a CArray for indexing (faster to deal with indexing ourselves)
+    const unsigned char* data = (const unsigned char*)&x[0];
+    {ws_init}
+    {pre}
+    {loops}
+        // inner-most loop
+        {ws_pre} {{
+            {found}
+        }}
+        {ws_post}
+    {end_loops}
+    {post}
+    '''.format(sizes='\n'.join(sizes), inds=inds, pre=pre, post=post,
+               ws_init=ws_init, ws_pre=ws_pre, ws_post=ws_post,
+               loops='\n'.join(loops), found=found, end_loops='}'*ndim)
+
+    mode_str = mode.replace('-', '_')  # avoid potential hyphen in kernel name
+    name = 'cupyx_scipy_ndimage_{}_{}d_{}_w{}'.format(
+        name, ndim, mode_str, '_'.join(['{}'.format(x) for x in w_shape]))
+    if all_weights_nonzero:
+        name += '_all_nonzero'
+    if int_type == 'ptrdiff_t':
+        name += '_i64'
+    if has_structure:
+        name += '_with_structure'
+    if has_mask:
+        name += '_with_mask'
+    preamble = includes + _CAST_FUNCTION + preamble
+    options += ('--std=c++11', '-DCUPY_USE_JITIFY')
+    return cupy.ElementwiseKernel(in_params, out_params, operation, name,
+                                  reduce_dims=False, preamble=preamble,
+                                  options=options)
diff --git a/cupyx/scipy/ndimage/_filters_generic.py b/cupyx/scipy/ndimage/_filters_generic.py
new file mode 100644
index 0000000..63c663a
--- /dev/null
+++ b/cupyx/scipy/ndimage/_filters_generic.py
@@ -0,0 +1,268 @@
+import cupy
+from cupy_backends.cuda.api import runtime
+from cupy import _util
+from cupyx.scipy.ndimage import _filters_core
+
+
+def _get_sub_kernel(f):
+    """
+    Takes the "function" given to generic_filter and returns the "sub-kernel"
+    that will be called, one of RawKernel or ReductionKernel.
+
+    This supports:
+     * cupy.RawKernel
+       no checks are possible
+     * cupy.ReductionKernel
+       checks that there is a single input and output
+    """
+    if isinstance(f, cupy.RawKernel):
+        # We will assume that it has the correct API
+        return f
+    elif isinstance(f, cupy.ReductionKernel):
+        if f.nin != 1 or f.nout != 1:
+            raise TypeError('ReductionKernel must have 1 input and output')
+        return f
+    elif isinstance(f, cupy.ElementwiseKernel):
+        # special error message for ElementwiseKernels
+        raise TypeError('only ReductionKernel allowed (not ElementwiseKernel)')
+    else:
+        raise TypeError('bad function type')
+
+
+@_util.memoize(for_each_device=True)
+def _get_generic_filter_red(rk, in_dtype, out_dtype, filter_size, mode,
+                            wshape, offsets, cval, int_type):
+    """Generic filter implementation based on a reduction kernel."""
+    # Get the temporary output c type
+    in_param, out_param = rk.in_params[0], rk.out_params[0]
+    out_ctype = out_param.ctype
+    if out_param.dtype is None:  # resolve template
+        out_ctype = cupy._core._scalar.get_typename(
+            in_dtype if out_param.ctype == in_param.ctype else out_dtype)
+
+    # Get code chunks
+    setup = '''
+    int iv = 0;
+    X values[{size}];
+    CArray<X, 1, true, true> sub_in(values, {{{size}}});
+    {out_ctype} val_out;
+    CArray<{out_ctype}, 1, true, true> sub_out(&val_out, {{1}});
+    '''.format(size=filter_size, out_ctype=out_ctype)
+
+    sub_call = '''reduction_kernel::{}(sub_in, sub_out);
+    y = cast<Y>(val_out);'''.format(rk.name)
+
+    sub_kernel = _reduction_kernel_code(rk, filter_size, out_dtype, in_dtype)
+
+    # Get the final kernel
+    return _filters_core._generate_nd_kernel(
+        'generic_{}_{}'.format(filter_size, rk.name),
+        setup, 'values[iv++] = {value};', sub_call,
+        mode, wshape, int_type, offsets, cval, preamble=sub_kernel,
+        options=getattr(rk, 'options', ()))
+
+
+def _reduction_kernel_code(rk, filter_size, out_dtype, in_dtype):
+    # NOTE: differences from the code generated for real reduction kernels:
+    #  * input is always 1D and always less than 2^31 elements
+    #  * output is always 1D with a single element
+    #  * never across threads (no _block_stride, _sdata, _sdata_raw, _REDUCE,
+    #       _tid, _J, _i, _i_base, _j_offset, _J_offset, _j_stride, _J_stride)
+    # Also, the code is moved into a namespace so that clashes are minimized
+    # between the typedefs for the "template" variables.
+
+    # figure out the types
+    types = {}
+    in_param, out_param = rk.in_params[0], rk.out_params[0]
+    in_ctype = _get_type_info(in_param, in_dtype, types)
+    out_ctype = _get_type_info(out_param, out_dtype, types)
+    types = '\n'.join('typedef {} {};'.format(typ, name)
+                      for name, typ in types.items())
+
+    return '''namespace reduction_kernel {{
+{type_preamble}
+{preamble}
+__device__
+void {name}({in_const} CArray<{in_ctype}, 1, true, true>& _raw_{in_name},
+            CArray<{out_ctype}, 1, true, true>& _raw_{out_name}) {{
+    // these are just provided so if they are available for the RK
+    CIndexer<1> _in_ind({{{size}}});
+    CIndexer<0> _out_ind;
+
+    #define REDUCE(a, b) ({reduce_expr})
+    #define POST_MAP(a) ({post_map_expr})
+    typedef {reduce_type} _type_reduce;
+    _type_reduce _s = _type_reduce({identity});
+    for (int _j = 0; _j < {size}; ++_j) {{
+        _in_ind.set(_j);
+        {in_const} {in_ctype}& {in_name} = _raw_{in_name}[_j];
+        _type_reduce _a = static_cast<_type_reduce>({pre_map_expr});
+        _s = REDUCE(_s, _a);
+    }}
+    _out_ind.set(0);
+    {out_ctype} &{out_name} = _raw_{out_name}[0];
+    POST_MAP(_s);
+    #undef REDUCE
+    #undef POST_MAP
+}}
+}}'''.format(
+        name=rk.name, type_preamble=types, preamble=rk.preamble,
+        in_const='const' if in_param.is_const else '',
+        in_ctype=in_ctype, in_name=in_param.name,
+        out_ctype=out_ctype, out_name=out_param.name,
+
+        pre_map_expr=rk.map_expr,
+        identity='' if rk.identity is None else rk.identity,
+        size=filter_size,
+        reduce_type=rk.reduce_type, reduce_expr=rk.reduce_expr,
+        post_map_expr=rk.post_map_expr,
+    )
+
+
+def _get_type_info(param, dtype, types):
+    if param.dtype is not None:
+        return param.ctype
+    # Template type -> map to actual output type
+    ctype = cupy._core._scalar.get_typename(dtype)
+    types.setdefault(param.ctype, ctype)
+    return ctype
+
+
+@_util.memoize(for_each_device=True)
+def _get_generic_filter_raw(rk, filter_size, mode, wshape, offsets, cval,
+                            int_type):
+    """Generic filter implementation based on a raw kernel."""
+    setup = '''
+    int iv = 0;
+    double values[{}];
+    double val_out;'''.format(filter_size)
+
+    sub_call = '''raw_kernel::{}(values, {}, &val_out);
+    y = cast<Y>(val_out);'''.format(rk.name, filter_size)
+
+    return _filters_core._generate_nd_kernel(
+        'generic_{}_{}'.format(filter_size, rk.name),
+        setup, 'values[iv++] = cast<double>({value});', sub_call,
+        mode, wshape, int_type, offsets, cval,
+        preamble='namespace raw_kernel {{\n{}\n}}'.format(
+            # Users can test RawKernel independently, but when passed to here
+            # it must be used as a device function here. In fact, RawKernel
+            # wouldn't compile if code only contains device functions, so this
+            # is necessary.
+            rk.code.replace('__global__', '__device__')),
+        options=rk.options)
+
+
+@_util.memoize(for_each_device=True)
+def _get_generic_filter1d(rk, length, n_lines, filter_size, origin, mode, cval,
+                          in_ctype, out_ctype, int_type):
+    """
+    The generic 1d filter is different than all other filters and thus is the
+    only filter that doesn't use _generate_nd_kernel() and has a completely
+    custom raw kernel.
+    """
+    in_length = length + filter_size - 1
+    start = filter_size // 2 + origin
+    end = start + length
+
+    if mode == 'constant':
+        boundary, boundary_early = '', '''
+        for (idx_t j = 0; j < {start}; ++j) {{ input_line[j] = {cval}; }}
+        for (idx_t j = {end}; j<{in_length}; ++j) {{ input_line[j] = {cval}; }}
+        '''.format(start=start, end=end, in_length=in_length, cval=cval)
+    else:
+        if length == 1:
+            a = b = 'j_ = 0;'
+        elif mode == 'reflect':
+            j = ('j_ = ({j}) % ({length} * 2);\n'
+                 'j_ = min(j_, 2 * {length} - 1 - j_);')
+            a = j.format(j='-1 - j_', length=length)
+            b = j.format(j='j_', length=length)
+        elif mode == 'mirror':
+            j = ('j_ = 1 + (({j}) - 1) % (({length} - 1) * 2);\n'
+                 'j_ = min(j_, 2 * {length} - 2 - j_);')
+            a = j.format(j='-j_', length=length)
+            b = j.format(j='j_', length=length)
+        elif mode == 'nearest':
+            a, b = 'j_ = 0;', 'j_ = {length}-1;'.format(length=length)
+        elif mode == 'wrap':
+            a = 'j_ = j_ % {length} + {length};'.format(length=length)
+            b = 'j_ = j_ % {length};'.format(length=length)
+        loop = '''for (idx_t j = {{}}; j < {{}}; ++j) {{{{
+            idx_t j_ = j - {start};
+            {{}}
+            input_line[j] = input_line[j_ + {start}];
+        }}}}'''.format(start=start)
+        boundary_early = ''
+        boundary = (loop.format(0, start, a) + '\n' +
+                    loop.format(end, in_length, b))
+
+    name = 'generic1d_{}_{}_{}'.format(length, filter_size, rk.name)
+    code = '''#include "cupy/carray.cuh"
+#include "cupy/complex.cuh"
+{include_type_traits}  // let Jitify handle this
+
+namespace raw_kernel {{\n{rk_code}\n}}
+
+{CAST}
+
+typedef unsigned char byte;
+typedef {in_ctype} X;
+typedef {out_ctype} Y;
+typedef {int_type} idx_t;
+
+__device__ idx_t offset(idx_t i, idx_t axis, idx_t ndim,
+                        const idx_t* shape, const idx_t* strides) {{
+    idx_t index = 0;
+    for (idx_t a = ndim; --a > 0; ) {{
+        if (a == axis) {{ continue; }}
+        index += (i % shape[a]) * strides[a];
+        i /= shape[a];
+    }}
+    return index + strides[0] * i;
+}}
+
+extern "C" __global__
+void {name}(const byte* input, byte* output, const idx_t* x) {{
+    const idx_t axis = x[0], ndim = x[1],
+        *shape = x+2, *in_strides = x+2+ndim, *out_strides = x+2+2*ndim;
+
+    const idx_t in_elem_stride = in_strides[axis];
+    const idx_t out_elem_stride = out_strides[axis];
+
+    double input_line[{in_length}];
+    double output_line[{length}];
+    {boundary_early}
+
+    for (idx_t i = ((idx_t)blockIdx.x) * blockDim.x + threadIdx.x;
+            i < {n_lines};
+            i += ((idx_t)blockDim.x) * gridDim.x) {{
+        // Copy line from input (with boundary filling)
+        const byte* input_ = input + offset(i, axis, ndim, shape, in_strides);
+        for (idx_t j = 0; j < {length}; ++j) {{
+            input_line[j+{start}] = (double)*(X*)(input_+j*in_elem_stride);
+        }}
+        {boundary}
+
+        raw_kernel::{rk_name}(input_line, {in_length}, output_line, {length});
+
+        // Copy line to output
+        byte* output_ = output + offset(i, axis, ndim, shape, out_strides);
+        for (idx_t j = 0; j < {length}; ++j) {{
+            *(Y*)(output_+j*out_elem_stride) = cast<Y>(output_line[j]);
+        }}
+    }}
+}}'''.format(n_lines=n_lines, length=length, in_length=in_length, start=start,
+             in_ctype=in_ctype, out_ctype=out_ctype, int_type=int_type,
+             boundary_early=boundary_early, boundary=boundary,
+             name=name, rk_name=rk.name,
+             # Users can test RawKernel independently, but when passed to here
+             # it must be used as a device function here. In fact, RawKernel
+             # wouldn't compile if code only contains device functions, so this
+             # is necessary.
+             rk_code=rk.code.replace('__global__', '__device__'),
+             include_type_traits=(
+                 '' if runtime.is_hip else '#include <type_traits>\n'),
+             CAST=_filters_core._CAST_FUNCTION)
+    return cupy.RawKernel(code, name, ('--std=c++11',) + rk.options,
+                          jitify=True)
diff --git a/cupyx/scipy/ndimage/_fourier.py b/cupyx/scipy/ndimage/_fourier.py
new file mode 100644
index 0000000..a0ee3a5
--- /dev/null
+++ b/cupyx/scipy/ndimage/_fourier.py
@@ -0,0 +1,253 @@
+import numpy
+
+import cupy
+from cupy import _core
+from cupy._core import internal
+from cupyx.scipy.ndimage import _util
+from cupyx.scipy import special
+
+
+def _get_output_fourier(output, input, complex_only=False):
+    types = [cupy.complex64, cupy.complex128]
+    if not complex_only:
+        types += [cupy.float32, cupy.float64]
+
+    if output is None:
+        if input.dtype in types:
+            output = cupy.empty(input.shape, dtype=input.dtype)
+        else:
+            output = cupy.empty(input.shape, dtype=types[-1])
+    elif type(output) is type:
+        if output not in types:
+            raise RuntimeError('output type not supported')
+        output = cupy.empty(input.shape, dtype=output)
+    elif output.shape != input.shape:
+        raise RuntimeError('output shape not correct')
+    return output
+
+
+def _reshape_nd(arr, ndim, axis):
+    """Promote a 1d array to ndim with non-singleton size along axis."""
+    nd_shape = (1,) * axis + (arr.size,) + (1,) * (ndim - axis - 1)
+    return arr.reshape(nd_shape)
+
+
+def fourier_gaussian(input, sigma, n=-1, axis=-1, output=None):
+    """Multidimensional Gaussian shift filter.
+
+    The array is multiplied with the Fourier transform of a (separable)
+    Gaussian kernel.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        sigma (float or sequence of float):  The sigma of the Gaussian kernel.
+            If a float, `sigma` is the same for all axes. If a sequence,
+            `sigma` has to contain one value for each axis.
+        n (int, optional):  If `n` is negative (default), then the input is
+            assumed to be the result of a complex fft. If `n` is larger than or
+            equal to zero, the input is assumed to be the result of a real fft,
+            and `n` gives the length of the array before transformation along
+            the real transform direction.
+        axis (int, optional): The axis of the real transform (only used when
+            ``n > -1``).
+        output (cupy.ndarray, optional):
+            If given, the result of shifting the input is placed in this array.
+
+    Returns:
+        output (cupy.ndarray): The filtered output.
+    """
+    ndim = input.ndim
+    output = _get_output_fourier(output, input)
+    axis = internal._normalize_axis_index(axis, ndim)
+    sigmas = _util._fix_sequence_arg(sigma, ndim, 'sigma')
+
+    _core.elementwise_copy(input, output)
+    for ax, (sigmak, ax_size) in enumerate(zip(sigmas, output.shape)):
+
+        # compute the frequency grid in Hz
+        if ax == axis and n > 0:
+            arr = cupy.arange(ax_size, dtype=output.real.dtype)
+            arr /= n
+        else:
+            arr = cupy.fft.fftfreq(ax_size)
+        arr = arr.astype(output.real.dtype, copy=False)
+
+        # compute the Gaussian weights
+        arr *= arr
+        scale = sigmak * sigmak / -2
+        arr *= (4 * numpy.pi * numpy.pi) * scale
+        cupy.exp(arr, out=arr)
+
+        # reshape for broadcasting
+        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
+        output *= arr
+
+    return output
+
+
+def fourier_uniform(input, size, n=-1, axis=-1, output=None):
+    """Multidimensional uniform shift filter.
+
+    The array is multiplied with the Fourier transform of a box of given size.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (float or sequence of float):  The sigma of the box used for
+            filtering. If a float, `size` is the same for all axes. If a
+            sequence, `size` has to contain one value for each axis.
+        n (int, optional):  If `n` is negative (default), then the input is
+            assumed to be the result of a complex fft. If `n` is larger than or
+            equal to zero, the input is assumed to be the result of a real fft,
+            and `n` gives the length of the array before transformation along
+            the real transform direction.
+        axis (int, optional): The axis of the real transform (only used when
+            ``n > -1``).
+        output (cupy.ndarray, optional):
+            If given, the result of shifting the input is placed in this array.
+
+    Returns:
+        output (cupy.ndarray): The filtered output.
+    """
+    ndim = input.ndim
+    output = _get_output_fourier(output, input)
+    axis = internal._normalize_axis_index(axis, ndim)
+    sizes = _util._fix_sequence_arg(size, ndim, 'size')
+
+    _core.elementwise_copy(input, output)
+    for ax, (size, ax_size) in enumerate(zip(sizes, output.shape)):
+
+        # compute the frequency grid in Hz
+        if ax == axis and n > 0:
+            arr = cupy.arange(ax_size, dtype=output.real.dtype)
+            arr /= n
+        else:
+            arr = cupy.fft.fftfreq(ax_size)
+        arr = arr.astype(output.real.dtype, copy=False)
+
+        # compute the uniform filter weights
+        arr *= size
+        cupy.sinc(arr, out=arr)
+
+        # reshape for broadcasting
+        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
+        output *= arr
+
+    return output
+
+
+def fourier_shift(input, shift, n=-1, axis=-1, output=None):
+    """Multidimensional Fourier shift filter.
+
+    The array is multiplied with the Fourier transform of a shift operation.
+
+    Args:
+        input (cupy.ndarray): The input array. This should be in the Fourier
+            domain.
+        shift (float or sequence of float):  The size of shift. If a float,
+            `shift` is the same for all axes. If a sequence, `shift` has to
+            contain one value for each axis.
+        n (int, optional):  If `n` is negative (default), then the input is
+            assumed to be the result of a complex fft. If `n` is larger than or
+            equal to zero, the input is assumed to be the result of a real fft,
+            and `n` gives the length of the array before transformation along
+            the real transform direction.
+        axis (int, optional): The axis of the real transform (only used when
+            ``n > -1``).
+        output (cupy.ndarray, optional):
+            If given, the result of shifting the input is placed in this array.
+
+    Returns:
+        output (cupy.ndarray): The shifted output (in the Fourier domain).
+    """
+    ndim = input.ndim
+    output = _get_output_fourier(output, input, complex_only=True)
+    axis = internal._normalize_axis_index(axis, ndim)
+    shifts = _util._fix_sequence_arg(shift, ndim, 'shift')
+
+    _core.elementwise_copy(input, output)
+    for ax, (shiftk, ax_size) in enumerate(zip(shifts, output.shape)):
+        if shiftk == 0:
+            continue
+        if ax == axis and n > 0:
+            # cp.fft.rfftfreq(ax_size) * (-2j * numpy.pi * shiftk *  ax_size/n)
+            arr = cupy.arange(ax_size, dtype=output.dtype)
+            arr *= -2j * numpy.pi * shiftk / n
+        else:
+            arr = cupy.fft.fftfreq(ax_size)
+            arr = arr * (-2j * numpy.pi * shiftk)
+        cupy.exp(arr, out=arr)
+
+        # reshape for broadcasting
+        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
+        output *= arr
+
+    return output
+
+
+def fourier_ellipsoid(input, size, n=-1, axis=-1, output=None):
+    """Multidimensional ellipsoid Fourier filter.
+
+    The array is multiplied with the fourier transform of a ellipsoid of
+    given sizes.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (float or sequence of float):  The size of the box used for
+            filtering. If a float, `size` is the same for all axes. If a
+            sequence, `size` has to contain one value for each axis.
+        n (int, optional):  If `n` is negative (default), then the input is
+            assumed to be the result of a complex fft. If `n` is larger than or
+            equal to zero, the input is assumed to be the result of a real fft,
+            and `n` gives the length of the array before transformation along
+            the real transform direction.
+        axis (int, optional): The axis of the real transform (only used when
+            ``n > -1``).
+        output (cupy.ndarray, optional):
+            If given, the result of shifting the input is placed in this array.
+
+    Returns:
+        output (cupy.ndarray): The filtered output.
+    """
+    ndim = input.ndim
+    if ndim == 1:
+        return fourier_uniform(input, size, n, axis, output)
+
+    if ndim > 3:
+        # Note: SciPy currently does not do any filtering on >=4d inputs, but
+        #       does not warn about this!
+        raise NotImplementedError('Only 1d, 2d and 3d inputs are supported')
+    output = _get_output_fourier(output, input)
+    axis = internal._normalize_axis_index(axis, ndim)
+    sizes = _util._fix_sequence_arg(size, ndim, 'size')
+
+    _core.elementwise_copy(input, output)
+
+    # compute the distance from the origin for all samples in Fourier space
+    distance = 0
+    for ax, (size, ax_size) in enumerate(zip(sizes, output.shape)):
+        # compute the frequency grid in Hz
+        if ax == axis and n > 0:
+            arr = cupy.arange(ax_size, dtype=output.real.dtype)
+            arr *= numpy.pi * size / n
+        else:
+            arr = cupy.fft.fftfreq(ax_size)
+            arr *= numpy.pi * size
+        arr = arr.astype(output.real.dtype, copy=False)
+        arr *= arr
+        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
+        distance = distance + arr
+    cupy.sqrt(distance, out=distance)
+
+    if ndim == 2:
+        special.j1(distance, out=output)
+        output *= 2
+        output /= distance
+    elif ndim == 3:
+        cupy.sin(distance, out=output)
+        output -= distance * cupy.cos(distance)
+        output *= 3
+        output /= distance ** 3
+    output[(0,) * ndim] = 1.0  # avoid NaN in corner at frequency=0 location
+    output *= input
+
+    return output
diff --git a/cupyx/scipy/ndimage/_interp_kernels.py b/cupyx/scipy/ndimage/_interp_kernels.py
new file mode 100644
index 0000000..1020ea5
--- /dev/null
+++ b/cupyx/scipy/ndimage/_interp_kernels.py
@@ -0,0 +1,598 @@
+import numpy
+
+import cupy
+import cupy._core.internal
+
+from cupyx.scipy.ndimage import _spline_prefilter_core
+from cupyx.scipy.ndimage import _spline_kernel_weights
+from cupyx.scipy.ndimage import _util
+
+math_constants_preamble = r'''
+// workaround for HIP: line begins with #include
+#include <cupy/math_constants.h>
+'''
+
+spline_weights_inline = _spline_kernel_weights.spline_weights_inline
+
+
+def _get_coord_map(ndim, nprepad=0):
+    """Extract target coordinate from coords array (for map_coordinates).
+
+    Notes
+    -----
+    Assumes the following variables have been initialized on the device::
+
+        coords (ndarray): array of shape (ncoords, ndim) containing the target
+            coordinates.
+        c_j: variables to hold the target coordinates
+
+    computes::
+
+        c_j = coords[i + j * ncoords];
+
+    ncoords is determined by the size of the output array, y.
+    y will be indexed by the CIndexer, _ind.
+    Thus ncoords = _ind.size();
+
+    """
+    ops = []
+    ops.append('ptrdiff_t ncoords = _ind.size();')
+    pre = f" + (W){nprepad}" if nprepad > 0 else ''
+    for j in range(ndim):
+        ops.append(f'''
+    W c_{j} = coords[i + {j} * ncoords]{pre};''')
+    return ops
+
+
+def _get_coord_zoom_and_shift(ndim, nprepad=0):
+    """Compute target coordinate based on a shift followed by a zoom.
+
+    This version zooms from the center of the edge pixels.
+
+    Notes
+    -----
+    Assumes the following variables have been initialized on the device::
+
+        in_coord[ndim]: array containing the source coordinate
+        zoom[ndim]: array containing the zoom for each axis
+        shift[ndim]: array containing the zoom for each axis
+
+    computes::
+
+        c_j = zoom[j] * (in_coord[j] - shift[j])
+
+    """
+    ops = []
+    pre = f" + (W){nprepad}" if nprepad > 0 else ''
+    for j in range(ndim):
+        ops.append(f'''
+    W c_{j} = zoom[{j}] * ((W)in_coord[{j}] - shift[{j}]){pre};''')
+    return ops
+
+
+def _get_coord_zoom_and_shift_grid(ndim, nprepad=0):
+    """Compute target coordinate based on a shift followed by a zoom.
+
+    This version zooms from the outer edges of the grid pixels.
+
+    Notes
+    -----
+    Assumes the following variables have been initialized on the device::
+
+        in_coord[ndim]: array containing the source coordinate
+        zoom[ndim]: array containing the zoom for each axis
+        shift[ndim]: array containing the zoom for each axis
+
+    computes::
+
+        c_j = zoom[j] * (in_coord[j] - shift[j] + 0.5) - 0.5
+
+    """
+    ops = []
+    pre = f" + (W){nprepad}" if nprepad > 0 else ''
+    for j in range(ndim):
+        ops.append(f'''
+    W c_{j} = zoom[{j}] * ((W)in_coord[{j}] - shift[j] + 0.5) - 0.5{pre};''')
+    return ops
+
+
+def _get_coord_zoom(ndim, nprepad=0):
+    """Compute target coordinate based on a zoom.
+
+    This version zooms from the center of the edge pixels.
+
+    Notes
+    -----
+    Assumes the following variables have been initialized on the device::
+
+        in_coord[ndim]: array containing the source coordinate
+        zoom[ndim]: array containing the zoom for each axis
+
+    computes::
+
+        c_j = zoom[j] * in_coord[j]
+
+    """
+    ops = []
+    pre = f" + (W){nprepad}" if nprepad > 0 else ''
+    for j in range(ndim):
+        ops.append(f'''
+    W c_{j} = zoom[{j}] * (W)in_coord[{j}]{pre};''')
+    return ops
+
+
+def _get_coord_zoom_grid(ndim, nprepad=0):
+    """Compute target coordinate based on a zoom (grid_mode=True version).
+
+    This version zooms from the outer edges of the grid pixels.
+
+    Notes
+    -----
+    Assumes the following variables have been initialized on the device::
+
+        in_coord[ndim]: array containing the source coordinate
+        zoom[ndim]: array containing the zoom for each axis
+
+    computes::
+
+        c_j = zoom[j] * (in_coord[j] + 0.5) - 0.5
+
+    """
+    ops = []
+    pre = f" + (W){nprepad}" if nprepad > 0 else ''
+    for j in range(ndim):
+        ops.append(f'''
+    W c_{j} = zoom[{j}] * ((W)in_coord[{j}] + 0.5) - 0.5{pre};''')
+    return ops
+
+
+def _get_coord_shift(ndim, nprepad=0):
+    """Compute target coordinate based on a shift.
+
+    Notes
+    -----
+    Assumes the following variables have been initialized on the device::
+
+        in_coord[ndim]: array containing the source coordinate
+        shift[ndim]: array containing the zoom for each axis
+
+    computes::
+
+        c_j = in_coord[j] - shift[j]
+
+    """
+    ops = []
+    pre = f" + (W){nprepad}" if nprepad > 0 else ''
+    for j in range(ndim):
+        ops.append(f'''
+    W c_{j} = (W)in_coord[{j}] - shift[{j}]{pre};''')
+    return ops
+
+
+def _get_coord_affine(ndim, nprepad=0):
+    """Compute target coordinate based on a homogeneous transformation matrix.
+
+    The homogeneous matrix has shape (ndim, ndim + 1). It corresponds to
+    affine matrix where the last row of the affine is assumed to be:
+    ``[0] * ndim + [1]``.
+
+    Notes
+    -----
+    Assumes the following variables have been initialized on the device::
+
+        mat(array): array containing the (ndim, ndim + 1) transform matrix.
+        in_coords(array): coordinates of the input
+
+    For example, in 2D:
+
+        c_0 = mat[0] * in_coords[0] + mat[1] * in_coords[1] + aff[2];
+        c_1 = mat[3] * in_coords[0] + mat[4] * in_coords[1] + aff[5];
+
+    """
+    ops = []
+    pre = f" + (W){nprepad}" if nprepad > 0 else ''
+    ncol = ndim + 1
+    for j in range(ndim):
+        ops.append(f'''
+            W c_{j} = (W)0.0;''')
+        for k in range(ndim):
+            ops.append(f'''
+            c_{j} += mat[{ncol * j + k}] * (W)in_coord[{k}];''')
+        ops.append(f'''
+            c_{j} += mat[{ncol * j + ndim}]{pre};''')
+    return ops
+
+
+def _unravel_loop_index(shape, uint_t='unsigned int'):
+    """
+    declare a multi-index array in_coord and unravel the 1D index, i into it.
+    This code assumes that the array is a C-ordered array.
+    """
+    ndim = len(shape)
+    code = [f'''
+        {uint_t} in_coord[{ndim}];
+        {uint_t} s, t, idx = i;''']
+    for j in range(ndim - 1, 0, -1):
+        code.append(f'''
+        s = {shape[j]};
+        t = idx / s;
+        in_coord[{j}] = idx - t * s;
+        idx = t;''')
+    code.append('''
+        in_coord[0] = idx;''')
+    return '\n'.join(code)
+
+
+def _generate_interp_custom(coord_func, ndim, large_int, yshape, mode, cval,
+                            order, name='', integer_output=False, nprepad=0,
+                            omit_in_coord=False):
+    """
+    Args:
+        coord_func (function): generates code to do the coordinate
+            transformation. See for example, `_get_coord_shift`.
+        ndim (int): The number of dimensions.
+        large_int (bool): If true use Py_ssize_t instead of int for indexing.
+        yshape (tuple): Shape of the output array.
+        mode (str): Signal extension mode to use at the array boundaries
+        cval (float): constant value used when `mode == 'constant'`.
+        name (str): base name for the interpolation kernel
+        integer_output (bool): boolean indicating whether the output has an
+            integer type.
+        nprepad (int): integer indicating the amount of prepadding at the
+            boundaries.
+
+    Returns:
+        operation (str): code body for the ElementwiseKernel
+        name (str): name for the ElementwiseKernel
+    """
+
+    ops = []
+    internal_dtype = 'double' if integer_output else 'Y'
+    ops.append(f'{internal_dtype} out = 0.0;')
+
+    if large_int:
+        uint_t = 'size_t'
+        int_t = 'ptrdiff_t'
+    else:
+        uint_t = 'unsigned int'
+        int_t = 'int'
+
+    # determine strides for x along each axis
+    for j in range(ndim):
+        ops.append(f'const {int_t} xsize_{j} = x.shape()[{j}];')
+    ops.append(f'const {uint_t} sx_{ndim - 1} = 1;')
+    for j in range(ndim - 1, 0, -1):
+        ops.append(f'const {uint_t} sx_{j - 1} = sx_{j} * xsize_{j};')
+
+    if not omit_in_coord:
+        # create in_coords array to store the unraveled indices
+        ops.append(_unravel_loop_index(yshape, uint_t))
+
+    # compute the transformed (target) coordinates, c_j
+    ops = ops + coord_func(ndim, nprepad)
+
+    if cval is numpy.nan:
+        cval = '(Y)CUDART_NAN'
+    elif cval == numpy.inf:
+        cval = '(Y)CUDART_INF'
+    elif cval == -numpy.inf:
+        cval = '(Y)(-CUDART_INF)'
+    else:
+        cval = f'({internal_dtype}){cval}'
+
+    if mode == 'constant':
+        # use cval if coordinate is outside the bounds of x
+        _cond = ' || '.join(
+            [f'(c_{j} < 0) || (c_{j} > xsize_{j} - 1)' for j in range(ndim)])
+        ops.append(f'''
+        if ({_cond})
+        {{
+            out = {cval};
+        }}
+        else
+        {{''')
+
+    if order == 0:
+        if mode == 'wrap':
+            ops.append('double dcoord;')  # mode 'wrap' requires this to work
+        for j in range(ndim):
+            # determine nearest neighbor
+            if mode == 'wrap':
+                ops.append(f'''
+                dcoord = c_{j};''')
+            else:
+                ops.append(f'''
+                {int_t} cf_{j} = ({int_t})floor((double)c_{j} + 0.5);''')
+
+            # handle boundary
+            if mode != 'constant':
+                if mode == 'wrap':
+                    ixvar = 'dcoord'
+                    float_ix = True
+                else:
+                    ixvar = f'cf_{j}'
+                    float_ix = False
+                ops.append(
+                    _util._generate_boundary_condition_ops(
+                        mode, ixvar, f'xsize_{j}', int_t, float_ix))
+                if mode == 'wrap':
+                    ops.append(f'''
+                {int_t} cf_{j} = ({int_t})floor(dcoord + 0.5);''')
+
+            # sum over ic_j will give the raveled coordinate in the input
+            ops.append(f'''
+            {int_t} ic_{j} = cf_{j} * sx_{j};''')
+        _coord_idx = ' + '.join([f'ic_{j}' for j in range(ndim)])
+        if mode == 'grid-constant':
+            _cond = ' || '.join([f'(ic_{j} < 0)' for j in range(ndim)])
+            ops.append(f'''
+            if ({_cond}) {{
+                out = {cval};
+            }} else {{
+                out = ({internal_dtype})x[{_coord_idx}];
+            }}''')
+        else:
+            ops.append(f'''
+            out = ({internal_dtype})x[{_coord_idx}];''')
+
+    elif order == 1:
+        for j in range(ndim):
+            # get coordinates for linear interpolation along axis j
+            ops.append(f'''
+            {int_t} cf_{j} = ({int_t})floor((double)c_{j});
+            {int_t} cc_{j} = cf_{j} + 1;
+            {int_t} n_{j} = (c_{j} == cf_{j}) ? 1 : 2;  // points needed
+            ''')
+
+            if mode == 'wrap':
+                ops.append(f'''
+                double dcoordf = c_{j};
+                double dcoordc = c_{j} + 1;''')
+            else:
+                # handle boundaries for extension modes.
+                ops.append(f'''
+                {int_t} cf_bounded_{j} = cf_{j};
+                {int_t} cc_bounded_{j} = cc_{j};''')
+
+            if mode != 'constant':
+                if mode == 'wrap':
+                    ixvar = 'dcoordf'
+                    float_ix = True
+                else:
+                    ixvar = f'cf_bounded_{j}'
+                    float_ix = False
+                ops.append(
+                    _util._generate_boundary_condition_ops(
+                        mode, ixvar, f'xsize_{j}', int_t, float_ix))
+
+                ixvar = 'dcoordc' if mode == 'wrap' else f'cc_bounded_{j}'
+                ops.append(
+                    _util._generate_boundary_condition_ops(
+                        mode, ixvar, f'xsize_{j}', int_t, float_ix))
+                if mode == 'wrap':
+                    ops.append(
+                        f'''
+                    {int_t} cf_bounded_{j} = ({int_t})floor(dcoordf);;
+                    {int_t} cc_bounded_{j} = ({int_t})floor(dcoordf + 1);;
+                    '''
+                    )
+
+            ops.append(f'''
+            for (int s_{j} = 0; s_{j} < n_{j}; s_{j}++)
+                {{
+                    W w_{j};
+                    {int_t} ic_{j};
+                    if (s_{j} == 0)
+                    {{
+                        w_{j} = (W)cc_{j} - c_{j};
+                        ic_{j} = cf_bounded_{j} * sx_{j};
+                    }} else
+                    {{
+                        w_{j} = c_{j} - (W)cf_{j};
+                        ic_{j} = cc_bounded_{j} * sx_{j};
+                    }}''')
+    elif order > 1:
+        if mode == 'grid-constant':
+            spline_mode = 'constant'
+        elif mode == 'nearest':
+            spline_mode = 'nearest'
+        else:
+            spline_mode = _spline_prefilter_core._get_spline_mode(mode)
+
+        # wx, wy are temporary variables used during spline weight computation
+        ops.append(f'''
+            W wx, wy;
+            {int_t} start;''')
+        for j in range(ndim):
+            # determine weights along the current axis
+            ops.append(f'''
+            W weights_{j}[{order + 1}];''')
+            ops.append(spline_weights_inline[order].format(j=j, order=order))
+
+            # get starting coordinate for spline interpolation along axis j
+            if mode in ['wrap']:
+                ops.append(f'double dcoord = c_{j};')
+                coord_var = 'dcoord'
+                ops.append(
+                    _util._generate_boundary_condition_ops(
+                        mode, coord_var, f'xsize_{j}', int_t, True))
+            else:
+                coord_var = f'(double)c_{j}'
+
+            if order & 1:
+                op_str = '''
+                start = ({int_t})floor({coord_var}) - {order_2};'''
+            else:
+                op_str = '''
+                start = ({int_t})floor({coord_var} + 0.5) - {order_2};'''
+            ops.append(
+                op_str.format(
+                    int_t=int_t, coord_var=coord_var, order_2=order // 2
+                ))
+
+            # set of coordinate values within spline footprint along axis j
+            ops.append(f'''{int_t} ci_{j}[{order + 1}];''')
+            for k in range(order + 1):
+                ixvar = f'ci_{j}[{k}]'
+                ops.append(f'''
+                {ixvar} = start + {k};''')
+                ops.append(
+                    _util._generate_boundary_condition_ops(
+                        spline_mode, ixvar, f'xsize_{j}', int_t))
+
+            # loop over the order + 1 values in the spline filter
+            ops.append(f'''
+            W w_{j};
+            {int_t} ic_{j};
+            for (int k_{j} = 0; k_{j} <= {order}; k_{j}++)
+                {{
+                    w_{j} = weights_{j}[k_{j}];
+                    ic_{j} = ci_{j}[k_{j}] * sx_{j};
+            ''')
+
+    if order > 0:
+
+        _weight = ' * '.join([f'w_{j}' for j in range(ndim)])
+        _coord_idx = ' + '.join([f'ic_{j}' for j in range(ndim)])
+        if mode == 'grid-constant' or (order > 1 and mode == 'constant'):
+            _cond = ' || '.join([f'(ic_{j} < 0)' for j in range(ndim)])
+            ops.append(f'''
+            if ({_cond}) {{
+                out += {cval} * ({internal_dtype})({_weight});
+            }} else {{
+                {internal_dtype} val = ({internal_dtype})x[{_coord_idx}];
+                out += val * ({internal_dtype})({_weight});
+            }}''')
+        else:
+            ops.append(f'''
+            {internal_dtype} val = ({internal_dtype})x[{_coord_idx}];
+            out += val * ({internal_dtype})({_weight});''')
+
+        ops.append('}' * ndim)
+
+    if mode == 'constant':
+        ops.append('}')
+
+    if integer_output:
+        ops.append('y = (Y)rint((double)out);')
+    else:
+        ops.append('y = (Y)out;')
+    operation = '\n'.join(ops)
+
+    mode_str = mode.replace('-', '_')  # avoid hyphen in kernel name
+    name = 'cupyx_scipy_ndimage_interpolate_{}_order{}_{}_{}d_y{}'.format(
+        name, order, mode_str, ndim, '_'.join([f'{j}' for j in yshape]),
+    )
+    if uint_t == 'size_t':
+        name += '_i64'
+    return operation, name
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_map_kernel(ndim, large_int, yshape, mode, cval=0.0, order=1,
+                    integer_output=False, nprepad=0):
+    in_params = 'raw X x, raw W coords'
+    out_params = 'Y y'
+    operation, name = _generate_interp_custom(
+        coord_func=_get_coord_map,
+        ndim=ndim,
+        large_int=large_int,
+        yshape=yshape,
+        mode=mode,
+        cval=cval,
+        order=order,
+        name='map',
+        integer_output=integer_output,
+        nprepad=nprepad,
+        omit_in_coord=True,  # input image coordinates are not needed
+    )
+    return cupy.ElementwiseKernel(in_params, out_params, operation, name,
+                                  preamble=math_constants_preamble)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_shift_kernel(ndim, large_int, yshape, mode, cval=0.0, order=1,
+                      integer_output=False, nprepad=0):
+    in_params = 'raw X x, raw W shift'
+    out_params = 'Y y'
+    operation, name = _generate_interp_custom(
+        coord_func=_get_coord_shift,
+        ndim=ndim,
+        large_int=large_int,
+        yshape=yshape,
+        mode=mode,
+        cval=cval,
+        order=order,
+        name='shift',
+        integer_output=integer_output,
+        nprepad=nprepad,
+    )
+    return cupy.ElementwiseKernel(in_params, out_params, operation, name,
+                                  preamble=math_constants_preamble)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_zoom_shift_kernel(ndim, large_int, yshape, mode, cval=0.0, order=1,
+                           integer_output=False, grid_mode=False, nprepad=0):
+    in_params = 'raw X x, raw W shift, raw W zoom'
+    out_params = 'Y y'
+    if grid_mode:
+        zoom_shift_func = _get_coord_zoom_and_shift_grid
+    else:
+        zoom_shift_func = _get_coord_zoom_and_shift
+    operation, name = _generate_interp_custom(
+        coord_func=zoom_shift_func,
+        ndim=ndim,
+        large_int=large_int,
+        yshape=yshape,
+        mode=mode,
+        cval=cval,
+        order=order,
+        name="zoom_shift_grid" if grid_mode else "zoom_shift",
+        integer_output=integer_output,
+        nprepad=nprepad,
+    )
+    return cupy.ElementwiseKernel(in_params, out_params, operation, name,
+                                  preamble=math_constants_preamble)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_zoom_kernel(ndim, large_int, yshape, mode, cval=0.0, order=1,
+                     integer_output=False, grid_mode=False, nprepad=0):
+    in_params = 'raw X x, raw W zoom'
+    out_params = 'Y y'
+    operation, name = _generate_interp_custom(
+        coord_func=_get_coord_zoom_grid if grid_mode else _get_coord_zoom,
+        ndim=ndim,
+        large_int=large_int,
+        yshape=yshape,
+        mode=mode,
+        cval=cval,
+        order=order,
+        name="zoom_grid" if grid_mode else "zoom",
+        integer_output=integer_output,
+        nprepad=nprepad,
+    )
+    return cupy.ElementwiseKernel(in_params, out_params, operation, name,
+                                  preamble=math_constants_preamble)
+
+
+@cupy._util.memoize(for_each_device=True)
+def _get_affine_kernel(ndim, large_int, yshape, mode, cval=0.0, order=1,
+                       integer_output=False, nprepad=0):
+    in_params = 'raw X x, raw W mat'
+    out_params = 'Y y'
+    operation, name = _generate_interp_custom(
+        coord_func=_get_coord_affine,
+        ndim=ndim,
+        large_int=large_int,
+        yshape=yshape,
+        mode=mode,
+        cval=cval,
+        order=order,
+        name='affine',
+        integer_output=integer_output,
+        nprepad=nprepad,
+    )
+    return cupy.ElementwiseKernel(in_params, out_params, operation, name,
+                                  preamble=math_constants_preamble)
diff --git a/cupyx/scipy/ndimage/_interpolation.py b/cupyx/scipy/ndimage/_interpolation.py
new file mode 100644
index 0000000..7d10476
--- /dev/null
+++ b/cupyx/scipy/ndimage/_interpolation.py
@@ -0,0 +1,779 @@
+import math
+import warnings
+
+import cupy
+import numpy
+
+from cupy import _core
+from cupy._core import internal
+from cupy.cuda import runtime
+from cupyx import _texture
+from cupyx.scipy.ndimage import _util
+from cupyx.scipy.ndimage import _interp_kernels
+from cupyx.scipy.ndimage import _spline_prefilter_core
+
+_prod = cupy._core.internal.prod
+
+
+def _check_parameter(func_name, order, mode):
+    if order is None:
+        warnings.warn(f'Currently the default order of {func_name} is 1. In a '
+                      'future release this may change to 3 to match '
+                      'scipy.ndimage ')
+    elif order < 0 or 5 < order:
+        raise ValueError('spline order is not supported')
+
+    if mode not in ('constant', 'grid-constant', 'nearest', 'mirror',
+                    'reflect', 'grid-mirror', 'wrap', 'grid-wrap', 'opencv',
+                    '_opencv_edge'):
+        raise ValueError('boundary mode ({}) is not supported'.format(mode))
+
+
+def _get_spline_output(input, output):
+    """Create workspace array, temp, and the final dtype for the output.
+
+    Differs from SciPy by not always forcing the internal floating point dtype
+    to be double precision.
+    """
+    complex_data = input.dtype.kind == 'c'
+    if complex_data:
+        min_float_dtype = cupy.complex64
+    else:
+        min_float_dtype = cupy.float32
+    if isinstance(output, cupy.ndarray):
+        if complex_data and output.dtype.kind != 'c':
+            raise ValueError(
+                'output must have complex dtype for complex inputs'
+            )
+        float_dtype = cupy.promote_types(output.dtype, min_float_dtype)
+        output_dtype = output.dtype
+    else:
+        if output is None:
+            output = output_dtype = input.dtype
+        else:
+            output_dtype = cupy.dtype(output)
+        float_dtype = cupy.promote_types(output, min_float_dtype)
+
+    if (isinstance(output, cupy.ndarray)
+            and output.dtype == float_dtype == output_dtype
+            and output.flags.c_contiguous):
+        if output is not input:
+            _core.elementwise_copy(input, output)
+        temp = output
+    else:
+        temp = input.astype(float_dtype, copy=False)
+        temp = cupy.ascontiguousarray(temp)
+        if cupy.shares_memory(temp, input, 'MAY_SHARE_BOUNDS'):
+            temp = temp.copy()
+    return temp, float_dtype, output_dtype
+
+
+def spline_filter1d(input, order=3, axis=-1, output=cupy.float64,
+                    mode='mirror'):
+    """
+    Calculate a 1-D spline filter along the given axis.
+
+    The lines of the array along the given axis are filtered by a
+    spline filter. The order of the spline must be >= 2 and <= 5.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        order (int): The order of the spline interpolation, default is 3. Must
+            be in the range 0-5.
+        axis (int): The axis along which the spline filter is applied. Default
+            is the last axis.
+        output (cupy.ndarray or dtype, optional): The array in which to place
+            the output, or the dtype of the returned array. Default is
+            ``numpy.float64``.
+        mode (str): Points outside the boundaries of the input are filled
+            according to the given mode (``'constant'``, ``'nearest'``,
+            ``'mirror'``, ``'reflect'``, ``'wrap'``, ``'grid-mirror'``,
+            ``'grid-wrap'``, ``'grid-constant'`` or ``'opencv'``).
+
+    Returns:
+        cupy.ndarray: The result of prefiltering the input.
+
+    .. seealso:: :func:`scipy.spline_filter1d`
+    """
+    if order < 0 or order > 5:
+        raise RuntimeError('spline order not supported')
+    x = input
+    ndim = x.ndim
+    axis = internal._normalize_axis_index(axis, ndim)
+
+    # order 0, 1 don't require reshaping as no CUDA kernel will be called
+    # scalar or size 1 arrays also don't need to be filtered
+    run_kernel = not (order < 2 or x.ndim == 0 or x.shape[axis] == 1)
+    if not run_kernel:
+        output = _util._get_output(output, input)
+        _core.elementwise_copy(x, output)
+        return output
+
+    temp, data_dtype, output_dtype = _get_spline_output(x, output)
+    data_type = cupy._core._scalar.get_typename(temp.dtype)
+    pole_type = cupy._core._scalar.get_typename(temp.real.dtype)
+
+    index_type = _util._get_inttype(input)
+    index_dtype = cupy.int32 if index_type == 'int' else cupy.int64
+
+    n_samples = x.shape[axis]
+    n_signals = x.size // n_samples
+    info = cupy.array((n_signals, n_samples) + x.shape, dtype=index_dtype)
+
+    # empirical choice of block size that seemed to work well
+    block_size = max(2 ** math.ceil(numpy.log2(n_samples / 32)), 8)
+    kern = _spline_prefilter_core.get_raw_spline1d_kernel(
+        axis,
+        ndim,
+        mode,
+        order=order,
+        index_type=index_type,
+        data_type=data_type,
+        pole_type=pole_type,
+        block_size=block_size,
+    )
+
+    # Due to recursive nature, a given line of data must be processed by a
+    # single thread. n_signals lines will be processed in total.
+    block = (block_size,)
+    grid = ((n_signals + block[0] - 1) // block[0],)
+
+    # apply prefilter gain
+    poles = _spline_prefilter_core.get_poles(order=order)
+    temp *= _spline_prefilter_core.get_gain(poles)
+
+    # apply caual + anti-causal IIR spline filters
+    kern(grid, block, (temp, info))
+
+    if isinstance(output, cupy.ndarray) and temp is not output:
+        # copy kernel output into the user-provided output array
+        _core.elementwise_copy(temp, output)
+        return output
+    return temp.astype(output_dtype, copy=False)
+
+
+def spline_filter(input, order=3, output=cupy.float64, mode='mirror'):
+    """Multidimensional spline filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        order (int): The order of the spline interpolation, default is 3. Must
+            be in the range 0-5.
+        output (cupy.ndarray or dtype, optional): The array in which to place
+            the output, or the dtype of the returned array. Default is
+            ``numpy.float64``.
+        mode (str): Points outside the boundaries of the input are filled
+            according to the given mode (``'constant'``, ``'nearest'``,
+            ``'mirror'``, ``'reflect'``, ``'wrap'``, ``'grid-mirror'``,
+            ``'grid-wrap'``, ``'grid-constant'`` or ``'opencv'``).
+
+    Returns:
+        cupy.ndarray: The result of prefiltering the input.
+
+    .. seealso:: :func:`scipy.spline_filter1d`
+    """
+    if order < 2 or order > 5:
+        raise RuntimeError('spline order not supported')
+
+    x = input
+    temp, data_dtype, output_dtype = _get_spline_output(x, output)
+    if order not in [0, 1] and input.ndim > 0:
+        for axis in range(x.ndim):
+            spline_filter1d(x, order, axis, output=temp, mode=mode)
+            x = temp
+    if isinstance(output, cupy.ndarray):
+        _core.elementwise_copy(temp, output)
+    else:
+        output = temp
+    if output.dtype != output_dtype:
+        output = output.astype(output_dtype)
+    return output
+
+
+def _check_coordinates(coordinates, order, allow_float32=True):
+    if coordinates.dtype.kind == 'f':
+        if allow_float32:
+            coord_dtype = cupy.promote_types(coordinates.dtype, cupy.float32)
+        else:
+            coord_dtype = cupy.promote_types(coordinates.dtype, cupy.float64)
+        coordinates = coordinates.astype(coord_dtype, copy=False)
+    elif coordinates.dtype.kind in 'iu':
+        if order > 1:
+            # order > 1 (spline) kernels require floating-point coordinates
+            if allow_float32:
+                coord_dtype = cupy.promote_types(
+                    coordinates.dtype, cupy.float32
+                )
+            else:
+                coord_dtype = cupy.promote_types(
+                    coordinates.dtype, cupy.float64
+                )
+            coordinates = coordinates.astype(coord_dtype)
+    else:
+        raise ValueError('coordinates should have floating point dtype')
+    if not coordinates.flags.c_contiguous:
+        coordinates = cupy.ascontiguousarray(coordinates)
+    return coordinates
+
+
+def _prepad_for_spline_filter(input, mode, cval):
+    if mode in ['nearest', 'grid-constant']:
+        # these modes need padding to get accurate boundary values
+        npad = 12  # empirical factor chosen by SciPy
+        if mode == 'grid-constant':
+            kwargs = dict(mode='constant', constant_values=cval)
+        else:
+            kwargs = dict(mode='edge')
+        padded = cupy.pad(input, npad, **kwargs)
+    else:
+        npad = 0
+        padded = input
+    return padded, npad
+
+
+def _filter_input(image, prefilter, mode, cval, order):
+    """Perform spline prefiltering when needed.
+
+    Spline orders > 1 need a prefiltering stage to preserve resolution.
+
+    For boundary modes without analytical spline boundary conditions, some
+    prepadding of the input with cupy.pad is used to maintain accuracy.
+    ``npad`` is an integer corresponding to the amount of padding at each edge
+    of the array.
+    """
+    if not prefilter or order < 2:
+        return (cupy.ascontiguousarray(image), 0)
+    padded, npad = _prepad_for_spline_filter(image, mode, cval)
+    float_dtype = cupy.promote_types(image.dtype, cupy.float32)
+    filtered = spline_filter(padded, order, output=float_dtype, mode=mode)
+    return cupy.ascontiguousarray(filtered), npad
+
+
+def map_coordinates(input, coordinates, output=None, order=3,
+                    mode='constant', cval=0.0, prefilter=True):
+    """Map the input array to new coordinates by interpolation.
+
+    The array of coordinates is used to find, for each point in the output, the
+    corresponding coordinates in the input. The value of the input at those
+    coordinates is determined by spline interpolation of the requested order.
+
+    The shape of the output is derived from that of the coordinate array by
+    dropping the first axis. The values of the array along the first axis are
+    the coordinates in the input array at which the output value is found.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        coordinates (array_like): The coordinates at which ``input`` is
+            evaluated.
+        output (cupy.ndarray or ~cupy.dtype): The array in which to place the
+            output, or the dtype of the returned array.
+        order (int): The order of the spline interpolation, default is 3. Must
+            be in the range 0-5.
+        mode (str): Points outside the boundaries of the input are filled
+            according to the given mode (``'constant'``, ``'nearest'``,
+            ``'mirror'``, ``'reflect'``, ``'wrap'``, ``'grid-mirror'``,
+            ``'grid-wrap'``, ``'grid-constant'`` or ``'opencv'``).
+        cval (scalar): Value used for points outside the boundaries of
+            the input if ``mode='constant'`` or ``mode='opencv'``. Default is
+            0.0
+        prefilter (bool): Determines if the input array is prefiltered with
+            ``spline_filter`` before interpolation. The default is True, which
+            will create a temporary ``float64`` array of filtered values if
+            ``order > 1``. If setting this to False, the output will be
+            slightly blurred if ``order > 1``, unless the input is prefiltered,
+            i.e. it is the result of calling ``spline_filter`` on the original
+            input.
+
+    Returns:
+        cupy.ndarray:
+            The result of transforming the input. The shape of the output is
+            derived from that of ``coordinates`` by dropping the first axis.
+
+    .. seealso:: :func:`scipy.ndimage.map_coordinates`
+    """
+
+    _check_parameter('map_coordinates', order, mode)
+
+    if mode == 'opencv' or mode == '_opencv_edge':
+        input = cupy.pad(input, [(1, 1)] * input.ndim, 'constant',
+                         constant_values=cval)
+        coordinates = cupy.add(coordinates, 1)
+        mode = 'constant'
+
+    ret = _util._get_output(output, input, coordinates.shape[1:])
+    integer_output = ret.dtype.kind in 'iu'
+    _util._check_cval(mode, cval, integer_output)
+
+    if input.dtype.kind in 'iu':
+        input = input.astype(cupy.float32)
+    coordinates = _check_coordinates(coordinates, order)
+    filtered, nprepad = _filter_input(input, prefilter, mode, cval, order)
+    large_int = max(_prod(input.shape), coordinates.shape[0]) > 1 << 31
+    kern = _interp_kernels._get_map_kernel(
+        input.ndim, large_int, yshape=coordinates.shape, mode=mode, cval=cval,
+        order=order, integer_output=integer_output, nprepad=nprepad)
+    kern(filtered, coordinates, ret)
+    return ret
+
+
+def affine_transform(input, matrix, offset=0.0, output_shape=None, output=None,
+                     order=3, mode='constant', cval=0.0, prefilter=True, *,
+                     texture_memory=False):
+    """Apply an affine transformation.
+
+    Given an output image pixel index vector ``o``, the pixel value is
+    determined from the input image at position
+    ``cupy.dot(matrix, o) + offset``.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        matrix (cupy.ndarray): The inverse coordinate transformation matrix,
+            mapping output coordinates to input coordinates. If ``ndim`` is the
+            number of dimensions of ``input``, the given matrix must have one
+            of the following shapes:
+
+                - ``(ndim, ndim)``: the linear transformation matrix for each
+                  output coordinate.
+                - ``(ndim,)``: assume that the 2D transformation matrix is
+                  diagonal, with the diagonal specified by the given value.
+                - ``(ndim + 1, ndim + 1)``: assume that the transformation is
+                  specified using homogeneous coordinates. In this case, any
+                  value passed to ``offset`` is ignored.
+                - ``(ndim, ndim + 1)``: as above, but the bottom row of a
+                  homogeneous transformation matrix is always
+                  ``[0, 0, ..., 1]``, and may be omitted.
+
+        offset (float or sequence): The offset into the array where the
+            transform is applied. If a float, ``offset`` is the same for each
+            axis. If a sequence, ``offset`` should contain one value for each
+            axis.
+        output_shape (tuple of ints): Shape tuple.
+        output (cupy.ndarray or ~cupy.dtype): The array in which to place the
+            output, or the dtype of the returned array.
+        order (int): The order of the spline interpolation, default is 3. Must
+            be in the range 0-5.
+        mode (str): Points outside the boundaries of the input are filled
+            according to the given mode (``'constant'``, ``'nearest'``,
+            ``'mirror'``, ``'reflect'``, ``'wrap'``, ``'grid-mirror'``,
+            ``'grid-wrap'``, ``'grid-constant'`` or ``'opencv'``).
+        cval (scalar): Value used for points outside the boundaries of
+            the input if ``mode='constant'`` or ``mode='opencv'``. Default is
+            0.0
+        prefilter (bool): Determines if the input array is prefiltered with
+            ``spline_filter`` before interpolation. The default is True, which
+            will create a temporary ``float64`` array of filtered values if
+            ``order > 1``. If setting this to False, the output will be
+            slightly blurred if ``order > 1``, unless the input is prefiltered,
+            i.e. it is the result of calling ``spline_filter`` on the original
+            input.
+        texture_memory (bool): If True, uses GPU texture memory. Supports only:
+
+            - 2D and 3D float32 arrays as input
+            - ``(ndim + 1, ndim + 1)`` homogeneous float32 transformation
+                matrix
+            - ``mode='constant'`` and ``mode='nearest'``
+            - ``order=0`` (nearest neighbor) and ``order=1`` (linear
+                interpolation)
+            - NVIDIA CUDA GPUs
+
+    Returns:
+        cupy.ndarray or None:
+            The transformed input. If ``output`` is given as a parameter,
+            ``None`` is returned.
+
+    .. seealso:: :func:`scipy.ndimage.affine_transform`
+    """
+
+    if texture_memory:
+        if runtime.is_hip:
+            raise RuntimeError(
+                'HIP currently does not support texture acceleration')
+        tm_interp = 'linear' if order > 0 else 'nearest'
+        return _texture.affine_transformation(data=input,
+                                              transformation_matrix=matrix,
+                                              output_shape=output_shape,
+                                              output=output,
+                                              interpolation=tm_interp,
+                                              mode=mode,
+                                              border_value=cval)
+
+    _check_parameter('affine_transform', order, mode)
+
+    offset = _util._fix_sequence_arg(offset, input.ndim, 'offset', float)
+
+    if matrix.ndim not in [1, 2] or matrix.shape[0] < 1:
+        raise RuntimeError('no proper affine matrix provided')
+    if matrix.ndim == 2:
+        if matrix.shape[0] == matrix.shape[1] - 1:
+            offset = matrix[:, -1]
+            matrix = matrix[:, :-1]
+        elif matrix.shape[0] == input.ndim + 1:
+            offset = matrix[:-1, -1]
+            matrix = matrix[:-1, :-1]
+        if matrix.shape != (input.ndim, input.ndim):
+            raise RuntimeError('improper affine shape')
+
+    if mode == 'opencv':
+        m = cupy.zeros((input.ndim + 1, input.ndim + 1))
+        m[:-1, :-1] = matrix
+        m[:-1, -1] = offset
+        m[-1, -1] = 1
+        m = cupy.linalg.inv(m)
+        m[:2] = cupy.roll(m[:2], 1, axis=0)
+        m[:2, :2] = cupy.roll(m[:2, :2], 1, axis=1)
+        matrix = m[:-1, :-1]
+        offset = m[:-1, -1]
+
+    if output_shape is None:
+        output_shape = input.shape
+
+    if mode == 'opencv' or mode == '_opencv_edge':
+        if matrix.ndim == 1:
+            matrix = cupy.diag(matrix)
+        coordinates = cupy.indices(output_shape, dtype=cupy.float64)
+        coordinates = cupy.dot(matrix, coordinates.reshape((input.ndim, -1)))
+        coordinates += cupy.expand_dims(cupy.asarray(offset), -1)
+        ret = _util._get_output(output, input, shape=output_shape)
+        ret[:] = map_coordinates(input, coordinates, ret.dtype, order, mode,
+                                 cval, prefilter).reshape(output_shape)
+        return ret
+
+    matrix = matrix.astype(cupy.float64, copy=False)
+    ndim = input.ndim
+    output = _util._get_output(output, input, shape=output_shape)
+    if input.dtype.kind in 'iu':
+        input = input.astype(cupy.float32)
+    filtered, nprepad = _filter_input(input, prefilter, mode, cval, order)
+
+    integer_output = output.dtype.kind in 'iu'
+    _util._check_cval(mode, cval, integer_output)
+    large_int = max(_prod(input.shape), _prod(output_shape)) > 1 << 31
+    if matrix.ndim == 1:
+        offset = cupy.asarray(offset, dtype=cupy.float64)
+        offset = -offset / matrix
+        kern = _interp_kernels._get_zoom_shift_kernel(
+            ndim, large_int, output_shape, mode, cval=cval, order=order,
+            integer_output=integer_output, nprepad=nprepad)
+        kern(filtered, offset, matrix, output)
+    else:
+        kern = _interp_kernels._get_affine_kernel(
+            ndim, large_int, output_shape, mode, cval=cval, order=order,
+            integer_output=integer_output, nprepad=nprepad)
+        m = cupy.zeros((ndim, ndim + 1), dtype=cupy.float64)
+        m[:, :-1] = matrix
+        m[:, -1] = cupy.asarray(offset, dtype=cupy.float64)
+        kern(filtered, m, output)
+    return output
+
+
+def _minmax(coor, minc, maxc):
+    if coor[0] < minc[0]:
+        minc[0] = coor[0]
+    if coor[0] > maxc[0]:
+        maxc[0] = coor[0]
+    if coor[1] < minc[1]:
+        minc[1] = coor[1]
+    if coor[1] > maxc[1]:
+        maxc[1] = coor[1]
+    return minc, maxc
+
+
+def rotate(input, angle, axes=(1, 0), reshape=True, output=None, order=3,
+           mode='constant', cval=0.0, prefilter=True):
+    """Rotate an array.
+
+    The array is rotated in the plane defined by the two axes given by the
+    ``axes`` parameter using spline interpolation of the requested order.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        angle (float): The rotation angle in degrees.
+        axes (tuple of 2 ints): The two axes that define the plane of rotation.
+            Default is the first two axes.
+        reshape (bool): If ``reshape`` is True, the output shape is adapted so
+            that the input array is contained completely in the output. Default
+            is True.
+        output (cupy.ndarray or ~cupy.dtype): The array in which to place the
+            output, or the dtype of the returned array.
+        order (int): The order of the spline interpolation, default is 3. Must
+            be in the range 0-5.
+        mode (str): Points outside the boundaries of the input are filled
+            according to the given mode (``'constant'``, ``'nearest'``,
+            ``'mirror'``, ``'reflect'``, ``'wrap'``, ``'grid-mirror'``,
+            ``'grid-wrap'``, ``'grid-constant'`` or ``'opencv'``).
+        cval (scalar): Value used for points outside the boundaries of
+            the input if ``mode='constant'`` or ``mode='opencv'``. Default is
+            0.0
+        prefilter (bool): Determines if the input array is prefiltered with
+            ``spline_filter`` before interpolation. The default is True, which
+            will create a temporary ``float64`` array of filtered values if
+            ``order > 1``. If setting this to False, the output will be
+            slightly blurred if ``order > 1``, unless the input is prefiltered,
+            i.e. it is the result of calling ``spline_filter`` on the original
+            input.
+
+    Returns:
+        cupy.ndarray or None:
+            The rotated input.
+
+    .. seealso:: :func:`scipy.ndimage.rotate`
+    """
+
+    _check_parameter('rotate', order, mode)
+
+    if mode == 'opencv':
+        mode = '_opencv_edge'
+
+    input_arr = input
+    axes = list(axes)
+    if axes[0] < 0:
+        axes[0] += input_arr.ndim
+    if axes[1] < 0:
+        axes[1] += input_arr.ndim
+    if axes[0] > axes[1]:
+        axes = [axes[1], axes[0]]
+    if axes[0] < 0 or input_arr.ndim <= axes[1]:
+        raise ValueError('invalid rotation plane specified')
+
+    ndim = input_arr.ndim
+    rad = numpy.deg2rad(angle)
+    sin = math.sin(rad)
+    cos = math.cos(rad)
+
+    # determine offsets and output shape as in scipy.ndimage.rotate
+    rot_matrix = numpy.array([[cos, sin],
+                              [-sin, cos]])
+
+    img_shape = numpy.asarray(input_arr.shape)
+    in_plane_shape = img_shape[axes]
+    if reshape:
+        # Compute transformed input bounds
+        iy, ix = in_plane_shape
+        out_bounds = rot_matrix @ [[0, 0, iy, iy],
+                                   [0, ix, 0, ix]]
+        # Compute the shape of the transformed input plane
+        out_plane_shape = (out_bounds.ptp(axis=1) + 0.5).astype(cupy.int64)
+    else:
+        out_plane_shape = img_shape[axes]
+
+    out_center = rot_matrix @ ((out_plane_shape - 1) / 2)
+    in_center = (in_plane_shape - 1) / 2
+
+    output_shape = img_shape
+    output_shape[axes] = out_plane_shape
+    output_shape = tuple(output_shape)
+
+    matrix = numpy.identity(ndim)
+    matrix[axes[0], axes[0]] = cos
+    matrix[axes[0], axes[1]] = sin
+    matrix[axes[1], axes[0]] = -sin
+    matrix[axes[1], axes[1]] = cos
+
+    offset = numpy.zeros(ndim, dtype=cupy.float64)
+    offset[axes] = in_center - out_center
+
+    matrix = cupy.asarray(matrix)
+    offset = cupy.asarray(offset)
+
+    return affine_transform(input, matrix, offset, output_shape, output, order,
+                            mode, cval, prefilter)
+
+
+def shift(input, shift, output=None, order=3, mode='constant', cval=0.0,
+          prefilter=True):
+    """Shift an array.
+
+    The array is shifted using spline interpolation of the requested order.
+    Points outside the boundaries of the input are filled according to the
+    given mode.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        shift (float or sequence): The shift along the axes. If a float,
+            ``shift`` is the same for each axis. If a sequence, ``shift``
+            should contain one value for each axis.
+        output (cupy.ndarray or ~cupy.dtype): The array in which to place the
+            output, or the dtype of the returned array.
+        order (int): The order of the spline interpolation, default is 3. Must
+            be in the range 0-5.
+        mode (str): Points outside the boundaries of the input are filled
+            according to the given mode (``'constant'``, ``'nearest'``,
+            ``'mirror'``, ``'reflect'``, ``'wrap'``, ``'grid-mirror'``,
+            ``'grid-wrap'``, ``'grid-constant'`` or ``'opencv'``).
+        cval (scalar): Value used for points outside the boundaries of
+            the input if ``mode='constant'`` or ``mode='opencv'``. Default is
+            0.0
+        prefilter (bool): Determines if the input array is prefiltered with
+            ``spline_filter`` before interpolation. The default is True, which
+            will create a temporary ``float64`` array of filtered values if
+            ``order > 1``. If setting this to False, the output will be
+            slightly blurred if ``order > 1``, unless the input is prefiltered,
+            i.e. it is the result of calling ``spline_filter`` on the original
+            input.
+
+    Returns:
+        cupy.ndarray or None:
+            The shifted input.
+
+    .. seealso:: :func:`scipy.ndimage.shift`
+    """
+
+    _check_parameter('shift', order, mode)
+
+    shift = _util._fix_sequence_arg(shift, input.ndim, 'shift', float)
+
+    if mode == 'opencv':
+        mode = '_opencv_edge'
+
+        output = affine_transform(
+            input,
+            cupy.ones(input.ndim, input.dtype),
+            cupy.negative(cupy.asarray(shift)),
+            None,
+            output,
+            order,
+            mode,
+            cval,
+            prefilter,
+        )
+    else:
+        output = _util._get_output(output, input)
+        if input.dtype.kind in 'iu':
+            input = input.astype(cupy.float32)
+        filtered, nprepad = _filter_input(input, prefilter, mode, cval, order)
+        integer_output = output.dtype.kind in 'iu'
+        _util._check_cval(mode, cval, integer_output)
+        large_int = _prod(input.shape) > 1 << 31
+        kern = _interp_kernels._get_shift_kernel(
+            input.ndim, large_int, input.shape, mode, cval=cval, order=order,
+            integer_output=integer_output, nprepad=nprepad)
+        shift = cupy.asarray(shift, dtype=cupy.float64, order='C')
+        if shift.ndim != 1:
+            raise ValueError('shift must be 1d')
+        if shift.size != filtered.ndim:
+            raise ValueError('len(shift) must equal input.ndim')
+        kern(filtered, shift, output)
+    return output
+
+
+def zoom(input, zoom, output=None, order=3, mode='constant', cval=0.0,
+         prefilter=True, *, grid_mode=False):
+    """Zoom an array.
+
+    The array is zoomed using spline interpolation of the requested order.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        zoom (float or sequence): The zoom factor along the axes. If a float,
+            ``zoom`` is the same for each axis. If a sequence, ``zoom`` should
+            contain one value for each axis.
+        output (cupy.ndarray or ~cupy.dtype): The array in which to place the
+            output, or the dtype of the returned array.
+        order (int): The order of the spline interpolation, default is 3. Must
+            be in the range 0-5.
+        mode (str): Points outside the boundaries of the input are filled
+            according to the given mode (``'constant'``, ``'nearest'``,
+            ``'mirror'``, ``'reflect'``, ``'wrap'``, ``'grid-mirror'``,
+            ``'grid-wrap'``, ``'grid-constant'`` or ``'opencv'``).
+        cval (scalar): Value used for points outside the boundaries of
+            the input if ``mode='constant'`` or ``mode='opencv'``. Default is
+            0.0
+        prefilter (bool): Determines if the input array is prefiltered with
+            ``spline_filter`` before interpolation. The default is True, which
+            will create a temporary ``float64`` array of filtered values if
+            ``order > 1``. If setting this to False, the output will be
+            slightly blurred if ``order > 1``, unless the input is prefiltered,
+            i.e. it is the result of calling ``spline_filter`` on the original
+            input.
+        grid_mode (bool, optional): If False, the distance from the pixel
+            centers is zoomed. Otherwise, the distance including the full pixel
+            extent is used. For example, a 1d signal of length 5 is considered
+            to have length 4 when ``grid_mode`` is False, but length 5 when
+            ``grid_mode`` is True. See the following visual illustration:
+
+            .. code-block:: text
+
+                    | pixel 1 | pixel 2 | pixel 3 | pixel 4 | pixel 5 |
+                         |<-------------------------------------->|
+                                            vs.
+                    |<----------------------------------------------->|
+
+            The starting point of the arrow in the diagram above corresponds to
+            coordinate location 0 in each mode.
+
+    Returns:
+        cupy.ndarray or None:
+            The zoomed input.
+
+    .. seealso:: :func:`scipy.ndimage.zoom`
+    """
+
+    _check_parameter('zoom', order, mode)
+
+    zoom = _util._fix_sequence_arg(zoom, input.ndim, 'zoom', float)
+
+    output_shape = []
+    for s, z in zip(input.shape, zoom):
+        output_shape.append(int(round(s * z)))
+    output_shape = tuple(output_shape)
+
+    if mode == 'opencv':
+        zoom = []
+        offset = []
+        for in_size, out_size in zip(input.shape, output_shape):
+            if out_size > 0:
+                zoom.append(float(in_size) / out_size)
+                offset.append((zoom[-1] - 1) / 2.0)
+            else:
+                zoom.append(0)
+                offset.append(0)
+        mode = 'nearest'
+
+        output = affine_transform(
+            input,
+            cupy.asarray(zoom),
+            offset,
+            output_shape,
+            output,
+            order,
+            mode,
+            cval,
+            prefilter,
+        )
+    else:
+        if grid_mode:
+
+            # warn about modes that may have surprising behavior
+            suggest_mode = None
+            if mode == 'constant':
+                suggest_mode = 'grid-constant'
+            elif mode == 'wrap':
+                suggest_mode = 'grid-wrap'
+            if suggest_mode is not None:
+                warnings.warn(
+                    f'It is recommended to use mode = {suggest_mode} instead '
+                    f'of {mode} when grid_mode is True.')
+
+        zoom = []
+        for in_size, out_size in zip(input.shape, output_shape):
+            if grid_mode and out_size > 0:
+                zoom.append(in_size / out_size)
+            elif out_size > 1:
+                zoom.append((in_size - 1) / (out_size - 1))
+            else:
+                zoom.append(0)
+
+        output = _util._get_output(output, input, shape=output_shape)
+        if input.dtype.kind in 'iu':
+            input = input.astype(cupy.float32)
+        filtered, nprepad = _filter_input(input, prefilter, mode, cval, order)
+        integer_output = output.dtype.kind in 'iu'
+        _util._check_cval(mode, cval, integer_output)
+        large_int = max(_prod(input.shape), _prod(output_shape)) > 1 << 31
+        kern = _interp_kernels._get_zoom_kernel(
+            input.ndim, large_int, output_shape, mode, order=order,
+            integer_output=integer_output, grid_mode=grid_mode,
+            nprepad=nprepad)
+        zoom = cupy.asarray(zoom, dtype=cupy.float64)
+        kern(filtered, zoom, output)
+    return output
diff --git a/cupyx/scipy/ndimage/_measurements.py b/cupyx/scipy/ndimage/_measurements.py
new file mode 100644
index 0000000..9ec1d4f
--- /dev/null
+++ b/cupyx/scipy/ndimage/_measurements.py
@@ -0,0 +1,1239 @@
+import warnings
+
+import numpy
+
+import cupy
+from cupy import _core
+from cupy import _util
+
+
+def label(input, structure=None, output=None):
+    """Labels features in an array.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        structure (array_like or None): A structuring element that defines
+            feature connections. ```structure``` must be centersymmetric. If
+            None, structure is automatically generated with a squared
+            connectivity equal to one.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+    Returns:
+        label (cupy.ndarray): An integer array where each unique feature in
+        ```input``` has a unique label in the array.
+
+        num_features (int): Number of features found.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`scipy.ndimage.label`
+    """
+    if not isinstance(input, cupy.ndarray):
+        raise TypeError('input must be cupy.ndarray')
+    if input.dtype.char in 'FD':
+        raise TypeError('Complex type not supported')
+    if structure is None:
+        structure = _generate_binary_structure(input.ndim, 1)
+    elif isinstance(structure, cupy.ndarray):
+        structure = cupy.asnumpy(structure)
+    structure = numpy.array(structure, dtype=bool)
+    if structure.ndim != input.ndim:
+        raise RuntimeError('structure and input must have equal rank')
+    for i in structure.shape:
+        if i != 3:
+            raise ValueError('structure dimensions must be equal to 3')
+
+    if isinstance(output, cupy.ndarray):
+        if output.shape != input.shape:
+            raise ValueError("output shape not correct")
+        caller_provided_output = True
+    else:
+        caller_provided_output = False
+        if output is None:
+            output = cupy.empty(input.shape, numpy.int32)
+        else:
+            output = cupy.empty(input.shape, output)
+
+    if input.size == 0:
+        # empty
+        maxlabel = 0
+    elif input.ndim == 0:
+        # 0-dim array
+        maxlabel = 0 if input.item() == 0 else 1
+        output.fill(maxlabel)
+    else:
+        if output.dtype != numpy.int32:
+            y = cupy.empty(input.shape, numpy.int32)
+        else:
+            y = output
+        maxlabel = _label(input, structure, y)
+        if output.dtype != numpy.int32:
+            _core.elementwise_copy(y, output)
+
+    if caller_provided_output:
+        return maxlabel
+    else:
+        return output, maxlabel
+
+
+def _generate_binary_structure(rank, connectivity):
+    if connectivity < 1:
+        connectivity = 1
+    if rank < 1:
+        return numpy.array(True, dtype=bool)
+    output = numpy.fabs(numpy.indices([3] * rank) - 1)
+    output = numpy.add.reduce(output, 0)
+    return output <= connectivity
+
+
+def _label(x, structure, y):
+    elems = numpy.where(structure != 0)
+    vecs = [elems[dm] - 1 for dm in range(x.ndim)]
+    offset = vecs[0]
+    for dm in range(1, x.ndim):
+        offset = offset * 3 + vecs[dm]
+    indxs = numpy.where(offset < 0)[0]
+    dirs = [[vecs[dm][dr] for dm in range(x.ndim)] for dr in indxs]
+    dirs = cupy.array(dirs, dtype=numpy.int32)
+    ndirs = indxs.shape[0]
+    y_shape = cupy.array(y.shape, dtype=numpy.int32)
+    count = cupy.zeros(2, dtype=numpy.int32)
+    _kernel_init()(x, y)
+    _kernel_connect()(y_shape, dirs, ndirs, x.ndim, y, size=y.size)
+    _kernel_count()(y, count, size=y.size)
+    maxlabel = int(count[0])
+    labels = cupy.empty(maxlabel, dtype=numpy.int32)
+    _kernel_labels()(y, count, labels, size=y.size)
+    _kernel_finalize()(maxlabel, cupy.sort(labels), y, size=y.size)
+    return maxlabel
+
+
+def _kernel_init():
+    return _core.ElementwiseKernel(
+        'X x', 'Y y', 'if (x == 0) { y = -1; } else { y = i; }',
+        'cupyx_scipy_ndimage_label_init')
+
+
+def _kernel_connect():
+    return _core.ElementwiseKernel(
+        'raw int32 shape, raw int32 dirs, int32 ndirs, int32 ndim',
+        'raw Y y',
+        '''
+        if (y[i] < 0) continue;
+        for (int dr = 0; dr < ndirs; dr++) {
+            int j = i;
+            int rest = j;
+            int stride = 1;
+            int k = 0;
+            for (int dm = ndim-1; dm >= 0; dm--) {
+                int pos = rest % shape[dm] + dirs[dm + dr * ndim];
+                if (pos < 0 || pos >= shape[dm]) {
+                    k = -1;
+                    break;
+                }
+                k += pos * stride;
+                rest /= shape[dm];
+                stride *= shape[dm];
+            }
+            if (k < 0) continue;
+            if (y[k] < 0) continue;
+            while (1) {
+                while (j != y[j]) { j = y[j]; }
+                while (k != y[k]) { k = y[k]; }
+                if (j == k) break;
+                if (j < k) {
+                    int old = atomicCAS( &y[k], k, j );
+                    if (old == k) break;
+                    k = old;
+                }
+                else {
+                    int old = atomicCAS( &y[j], j, k );
+                    if (old == j) break;
+                    j = old;
+                }
+            }
+        }
+        ''',
+        'cupyx_scipy_ndimage_label_connect')
+
+
+def _kernel_count():
+    return _core.ElementwiseKernel(
+        '', 'raw Y y, raw int32 count',
+        '''
+        if (y[i] < 0) continue;
+        int j = i;
+        while (j != y[j]) { j = y[j]; }
+        if (j != i) y[i] = j;
+        else atomicAdd(&count[0], 1);
+        ''',
+        'cupyx_scipy_ndimage_label_count')
+
+
+def _kernel_labels():
+    return _core.ElementwiseKernel(
+        '', 'raw Y y, raw int32 count, raw int32 labels',
+        '''
+        if (y[i] != i) continue;
+        int j = atomicAdd(&count[1], 1);
+        labels[j] = i;
+        ''',
+        'cupyx_scipy_ndimage_label_labels')
+
+
+def _kernel_finalize():
+    return _core.ElementwiseKernel(
+        'int32 maxlabel', 'raw int32 labels, raw Y y',
+        '''
+        if (y[i] < 0) {
+            y[i] = 0;
+            continue;
+        }
+        int yi = y[i];
+        int j_min = 0;
+        int j_max = maxlabel - 1;
+        int j = (j_min + j_max) / 2;
+        while (j_min < j_max) {
+            if (yi == labels[j]) break;
+            if (yi < labels[j]) j_max = j - 1;
+            else j_min = j + 1;
+            j = (j_min + j_max) / 2;
+        }
+        y[i] = j + 1;
+        ''',
+        'cupyx_scipy_ndimage_label_finalize')
+
+
+_ndimage_variance_kernel = _core.ElementwiseKernel(
+    'T input, R labels, raw X index, uint64 size, raw float64 mean',
+    'raw float64 out',
+    """
+    for (ptrdiff_t j = 0; j < size; j++) {
+      if (labels == index[j]) {
+        atomicAdd(&out[j], (input - mean[j]) * (input - mean[j]));
+        break;
+      }
+    }
+    """,
+    'cupyx_scipy_ndimage_variance')
+
+
+_ndimage_sum_kernel = _core.ElementwiseKernel(
+    'T input, R labels, raw X index, uint64 size',
+    'raw float64 out',
+    """
+    for (ptrdiff_t j = 0; j < size; j++) {
+      if (labels == index[j]) {
+        atomicAdd(&out[j], input);
+        break;
+      }
+    }
+    """,
+    'cupyx_scipy_ndimage_sum')
+
+
+def _ndimage_sum_kernel_2(input, labels, index, sum_val, batch_size=4):
+    for i in range(0, index.size, batch_size):
+        matched = labels == index[i:i + batch_size].reshape(
+            (-1,) + (1,) * input.ndim)
+        sum_axes = tuple(range(1, 1 + input.ndim))
+        sum_val[i:i + batch_size] = cupy.where(matched, input, 0).sum(
+            axis=sum_axes)
+    return sum_val
+
+
+_ndimage_mean_kernel = _core.ElementwiseKernel(
+    'T input, R labels, raw X index, uint64 size',
+    'raw float64 out, raw uint64 count',
+    """
+    for (ptrdiff_t j = 0; j < size; j++) {
+      if (labels == index[j]) {
+        atomicAdd(&out[j], input);
+        atomicAdd(&count[j], 1);
+        break;
+      }
+    }
+    """,
+    'cupyx_scipy_ndimage_mean')
+
+
+def _ndimage_mean_kernel_2(input, labels, index, batch_size=4,
+                           return_count=False):
+    sum_val = cupy.empty_like(index, dtype=cupy.float64)
+    count = cupy.empty_like(index, dtype=cupy.uint64)
+    for i in range(0, index.size, batch_size):
+        matched = labels == index[i:i + batch_size].reshape(
+            (-1,) + (1,) * input.ndim)
+        mean_axes = tuple(range(1, 1 + input.ndim))
+        count[i:i + batch_size] = matched.sum(axis=mean_axes)
+        sum_val[i:i + batch_size] = cupy.where(matched, input, 0).sum(
+            axis=mean_axes)
+    if return_count:
+        return sum_val / count, count
+    return sum_val / count
+
+
+def _mean_driver(input, labels, index, return_count=False, use_kern=False):
+    if use_kern:
+        return _ndimage_mean_kernel_2(input, labels, index,
+                                      return_count=return_count)
+
+    out = cupy.zeros_like(index, cupy.float64)
+    count = cupy.zeros_like(index, dtype=cupy.uint64)
+    sum, count = _ndimage_mean_kernel(input,
+                                      labels, index, index.size, out, count)
+    if return_count:
+        return sum / count, count
+    return sum / count
+
+
+def variance(input, labels=None, index=None):
+    """Calculates the variance of the values of an n-D image array, optionally
+    at specified sub-regions.
+
+    Args:
+        input (cupy.ndarray): Nd-image data to process.
+        labels (cupy.ndarray or None): Labels defining sub-regions in `input`.
+            If not None, must be same shape as `input`.
+        index (cupy.ndarray or None): `labels` to include in output. If None
+            (default), all values where `labels` is non-zero are used.
+
+    Returns:
+        cupy.ndarray: Values of variance, for each sub-region if
+        `labels` and `index` are specified.
+
+    .. seealso:: :func:`scipy.ndimage.variance`
+    """
+    if not isinstance(input, cupy.ndarray):
+        raise TypeError('input must be cupy.ndarray')
+
+    if input.dtype in (cupy.complex64, cupy.complex128):
+        raise TypeError("cupyx.scipy.ndimage.variance doesn't support %{}"
+                        "".format(input.dtype.type))
+
+    use_kern = False
+    # There are constraints on types because of atomicAdd() in CUDA.
+    if input.dtype not in [cupy.int32, cupy.float16, cupy.float32,
+                           cupy.float64, cupy.uint32, cupy.uint64,
+                           cupy.ulonglong]:
+        warnings.warn(
+            'Using the slower implementation because the provided '
+            f'type {input.dtype} is not supported by cupyx.scipy.ndimage.sum. '
+            'Consider using an array of type int32, float16, '
+            'float32, float64, uint32, uint64 as data types '
+            'for the fast implementation', _util.PerformanceWarning)
+        use_kern = True
+
+    def calc_var_with_intermediate_float(input):
+        vals_c = input - input.mean()
+        count = vals_c.size
+        # Does not use `ndarray.mean()` here to return the same results as
+        # SciPy does, especially in case `input`'s dtype is float16.
+        return cupy.square(vals_c).sum() / cupy.asanyarray(count).astype(float)
+
+    if labels is None:
+        return calc_var_with_intermediate_float(input)
+
+    if not isinstance(labels, cupy.ndarray):
+        raise TypeError('label must be cupy.ndarray')
+
+    input, labels = cupy.broadcast_arrays(input, labels)
+
+    if index is None:
+        return calc_var_with_intermediate_float(input[labels > 0])
+
+    if cupy.isscalar(index):
+        return calc_var_with_intermediate_float(input[labels == index])
+
+    if not isinstance(index, cupy.ndarray):
+        if not isinstance(index, int):
+            raise TypeError('index must be cupy.ndarray or a scalar int')
+        else:
+            return (input[labels == index]).var().astype(cupy.float64,
+                                                         copy=False)
+
+    mean_val, count = _mean_driver(input, labels, index, True, use_kern)
+    if use_kern:
+        new_axis = (..., *(cupy.newaxis for _ in range(input.ndim)))
+        return cupy.where(labels[None, ...] == index[new_axis],
+                          cupy.square(input - mean_val[new_axis]),
+                          0).sum(tuple(range(1, input.ndim + 1))) / count
+    out = cupy.zeros_like(index, dtype=cupy.float64)
+    return _ndimage_variance_kernel(input, labels, index, index.size, mean_val,
+                                    out) / count
+
+
+def sum_labels(input, labels=None, index=None):
+    """Calculates the sum of the values of an n-D image array, optionally
+       at specified sub-regions.
+
+    Args:
+        input (cupy.ndarray): Nd-image data to process.
+        labels (cupy.ndarray or None): Labels defining sub-regions in `input`.
+            If not None, must be same shape as `input`.
+        index (cupy.ndarray or None): `labels` to include in output. If None
+            (default), all values where `labels` is non-zero are used.
+
+    Returns:
+       sum (cupy.ndarray): sum of values, for each sub-region if
+       `labels` and `index` are specified.
+
+    .. seealso:: :func:`scipy.ndimage.sum_labels`
+    """
+    if not isinstance(input, cupy.ndarray):
+        raise TypeError('input must be cupy.ndarray')
+
+    if input.dtype in (cupy.complex64, cupy.complex128):
+        raise TypeError("cupyx.scipy.ndimage.sum does not support %{}".format(
+            input.dtype.type))
+
+    use_kern = False
+    # There is constraints on types because of atomicAdd() in CUDA.
+    if input.dtype not in [cupy.int32, cupy.float16, cupy.float32,
+                           cupy.float64, cupy.uint32, cupy.uint64,
+                           cupy.ulonglong]:
+        warnings.warn(
+            'Using the slower implmentation as '
+            'cupyx.scipy.ndimage.sum supports int32, float16, '
+            'float32, float64, uint32, uint64 as data types'
+            'for the fast implmentation', _util.PerformanceWarning)
+        use_kern = True
+
+    if labels is None:
+        return input.sum()
+
+    if not isinstance(labels, cupy.ndarray):
+        raise TypeError('label must be cupy.ndarray')
+
+    input, labels = cupy.broadcast_arrays(input, labels)
+
+    if index is None:
+        return input[labels != 0].sum()
+
+    if not isinstance(index, cupy.ndarray):
+        if not isinstance(index, int):
+            raise TypeError('index must be cupy.ndarray or a scalar int')
+        else:
+            return (input[labels == index]).sum()
+
+    if index.size == 0:
+        return cupy.array([], dtype=cupy.int64)
+
+    out = cupy.zeros_like(index, dtype=cupy.float64)
+
+    # The following parameters for sum where determined using a Tesla P100.
+    if (input.size >= 262144 and index.size <= 4) or use_kern:
+        return _ndimage_sum_kernel_2(input, labels, index, out)
+    return _ndimage_sum_kernel(input, labels, index, index.size, out)
+
+
+def sum(input, labels=None, index=None):
+    """Calculates the sum of the values of an n-D image array, optionally
+       at specified sub-regions.
+
+    Args:
+        input (cupy.ndarray): Nd-image data to process.
+        labels (cupy.ndarray or None): Labels defining sub-regions in `input`.
+            If not None, must be same shape as `input`.
+        index (cupy.ndarray or None): `labels` to include in output. If None
+            (default), all values where `labels` is non-zero are used.
+
+    Returns:
+       sum (cupy.ndarray): sum of values, for each sub-region if
+       `labels` and `index` are specified.
+
+    Notes:
+        This is an alias for `cupyx.scipy.ndimage.sum_labels` kept for
+        backwards compatibility reasons. For new code please prefer
+        `sum_labels`.
+
+    .. seealso:: :func:`scipy.ndimage.sum`
+    """
+    return sum_labels(input, labels, index)
+
+
+def mean(input, labels=None, index=None):
+    """Calculates the mean of the values of an n-D image array, optionally
+       at specified sub-regions.
+
+    Args:
+        input (cupy.ndarray): Nd-image data to process.
+        labels (cupy.ndarray or None): Labels defining sub-regions in `input`.
+            If not None, must be same shape as `input`.
+        index (cupy.ndarray or None): `labels` to include in output. If None
+            (default), all values where `labels` is non-zero are used.
+
+    Returns:
+        mean (cupy.ndarray): mean of values, for each sub-region if
+        `labels` and `index` are specified.
+
+
+    .. seealso:: :func:`scipy.ndimage.mean`
+    """
+    if not isinstance(input, cupy.ndarray):
+        raise TypeError('input must be cupy.ndarray')
+
+    if input.dtype in (cupy.complex64, cupy.complex128):
+        raise TypeError("cupyx.scipy.ndimage.mean does not support %{}".format(
+            input.dtype.type))
+
+    use_kern = False
+    # There is constraints on types because of atomicAdd() in CUDA.
+    if input.dtype not in [cupy.int32, cupy.float16, cupy.float32,
+                           cupy.float64, cupy.uint32, cupy.uint64,
+                           cupy.ulonglong]:
+        warnings.warn(
+            'Using the slower implmentation as '
+            'cupyx.scipy.ndimage.mean supports int32, float16, '
+            'float32, float64, uint32, uint64 as data types '
+            'for the fast implmentation', _util.PerformanceWarning)
+        use_kern = True
+
+    def calc_mean_with_intermediate_float(input):
+        sum = input.sum()
+        count = input.size
+        # Does not use `ndarray.mean()` here to return the same results as
+        # SciPy does, especially in case `input`'s dtype is float16.
+        return sum / cupy.asanyarray(count).astype(float)
+
+    if labels is None:
+        return calc_mean_with_intermediate_float(input)
+
+    if not isinstance(labels, cupy.ndarray):
+        raise TypeError('label must be cupy.ndarray')
+
+    input, labels = cupy.broadcast_arrays(input, labels)
+
+    if index is None:
+        return calc_mean_with_intermediate_float(input[labels > 0])
+
+    if cupy.isscalar(index):
+        return calc_mean_with_intermediate_float(input[labels == index])
+
+    if not isinstance(index, cupy.ndarray):
+        if not isinstance(index, int):
+            raise TypeError('index must be cupy.ndarray or a scalar int')
+        else:
+            return (input[labels == index]).mean(dtype=cupy.float64)
+
+    return _mean_driver(input, labels, index, use_kern=use_kern)
+
+
+def standard_deviation(input, labels=None, index=None):
+    """Calculates the standard deviation of the values of an n-D image array,
+    optionally at specified sub-regions.
+
+    Args:
+        input (cupy.ndarray): Nd-image data to process.
+        labels (cupy.ndarray or None): Labels defining sub-regions in `input`.
+            If not None, must be same shape as `input`.
+        index (cupy.ndarray or None): `labels` to include in output. If None
+            (default), all values where `labels` is non-zero are used.
+
+    Returns:
+        standard_deviation (cupy.ndarray): standard deviation of values, for
+        each sub-region if `labels` and `index` are specified.
+
+    .. seealso:: :func:`scipy.ndimage.standard_deviation`
+    """
+    return cupy.sqrt(variance(input, labels, index))
+
+
+def _safely_castable_to_int(dt):
+    """Test whether the NumPy data type `dt` can be safely cast to an int."""
+    int_size = cupy.dtype(int).itemsize
+    safe = (
+        cupy.issubdtype(dt, cupy.signedinteger) and dt.itemsize <= int_size
+    ) or (cupy.issubdtype(dt, cupy.unsignedinteger) and dt.itemsize < int_size)
+    return safe
+
+
+def _get_values(arrays, func):
+    """Concatenated result of applying func to a list of arrays.
+
+    func should be cupy.min, cupy.max or cupy.median
+    """
+    dtype = arrays[0].dtype
+    return cupy.concatenate(
+        [
+            func(a, keepdims=True)
+            if a.size != 0 else cupy.asarray([0], dtype=dtype)
+            for a in arrays
+        ]
+    )
+
+
+def _get_positions(arrays, position_arrays, arg_func):
+    """Concatenated positions from applying arg_func to arrays.
+
+    arg_func should be cupy.argmin or cupy.argmax
+    """
+    return cupy.concatenate(
+        [
+            pos[arg_func(a, keepdims=True)]
+            if a.size != 0 else cupy.asarray([0], dtype=int)
+            for pos, a in zip(position_arrays, arrays)
+        ]
+    )
+
+
+def _select_via_looping(input, labels, idxs, positions, find_min,
+                        find_min_positions, find_max, find_max_positions,
+                        find_median):
+    """Internal helper routine for _select.
+
+    With relatively few labels it is faster to call this function rather than
+    using the implementation based on cupy.lexsort.
+    """
+    find_positions = find_min_positions or find_max_positions
+
+    # extract labeled regions into separate arrays
+    arrays = []
+    position_arrays = []
+    for i in idxs:
+        label_idx = labels == i
+        arrays.append(input[label_idx])
+        if find_positions:
+            position_arrays.append(positions[label_idx])
+
+    result = []
+    # the order below matches the order expected by cupy.ndimage.extrema
+    if find_min:
+        result += [_get_values(arrays, cupy.min)]
+    if find_min_positions:
+        result += [_get_positions(arrays, position_arrays, cupy.argmin)]
+    if find_max:
+        result += [_get_values(arrays, cupy.max)]
+    if find_max_positions:
+        result += [_get_positions(arrays, position_arrays, cupy.argmax)]
+    if find_median:
+        result += [_get_values(arrays, cupy.median)]
+    return result
+
+
+def _select(input, labels=None, index=None, find_min=False, find_max=False,
+            find_min_positions=False, find_max_positions=False,
+            find_median=False):
+    """Return one or more of: min, max, min position, max position, median.
+
+    If neither `labels` or `index` is provided, these are the global values
+    in `input`. If `index` is None, but `labels` is provided, a global value
+    across all non-zero labels is given. When both `labels` and `index` are
+    provided, lists of values are provided for each labeled region specified
+    in `index`. See further details in :func:`cupyx.scipy.ndimage.minimum`,
+    etc.
+
+    Used by minimum, maximum, minimum_position, maximum_position, extrema.
+    """
+    find_positions = find_min_positions or find_max_positions
+    positions = None
+    if find_positions:
+        positions = cupy.arange(input.size).reshape(input.shape)
+
+    def single_group(vals, positions):
+        result = []
+        if find_min:
+            result += [vals.min()]
+        if find_min_positions:
+            result += [positions[vals == vals.min()][0]]
+        if find_max:
+            result += [vals.max()]
+        if find_max_positions:
+            result += [positions[vals == vals.max()][0]]
+        if find_median:
+            result += [cupy.median(vals)]
+        return result
+
+    if labels is None:
+        return single_group(input, positions)
+
+    # ensure input and labels match sizes
+    input, labels = cupy.broadcast_arrays(input, labels)
+
+    if index is None:
+        mask = labels > 0
+        masked_positions = None
+        if find_positions:
+            masked_positions = positions[mask]
+        return single_group(input[mask], masked_positions)
+
+    if cupy.isscalar(index):
+        mask = labels == index
+        masked_positions = None
+        if find_positions:
+            masked_positions = positions[mask]
+        return single_group(input[mask], masked_positions)
+
+    index = cupy.asarray(index)
+
+    safe_int = _safely_castable_to_int(labels.dtype)
+    min_label = labels.min()
+    max_label = labels.max()
+
+    # Remap labels to unique integers if necessary, or if the largest label is
+    # larger than the number of values.
+    if (not safe_int or min_label < 0 or max_label > labels.size):
+        # Remap labels, and indexes
+        unique_labels, labels = cupy.unique(labels, return_inverse=True)
+        idxs = cupy.searchsorted(unique_labels, index)
+
+        # Make all of idxs valid
+        idxs[idxs >= unique_labels.size] = 0
+        found = unique_labels[idxs] == index
+    else:
+        # Labels are an integer type, and there aren't too many
+        idxs = cupy.asanyarray(index, int).copy()
+        found = (idxs >= 0) & (idxs <= max_label)
+
+    idxs[~found] = max_label + 1
+
+    input = input.ravel()
+    labels = labels.ravel()
+    if find_positions:
+        positions = positions.ravel()
+
+    using_cub = _core._accelerator.ACCELERATOR_CUB in \
+        cupy._core.get_routine_accelerators()
+
+    if using_cub:
+        # Cutoff values below were determined empirically for relatively large
+        # input arrays.
+        if find_positions or find_median:
+            n_label_cutoff = 15
+        else:
+            n_label_cutoff = 30
+    else:
+        n_label_cutoff = 0
+
+    if n_label_cutoff and len(idxs) <= n_label_cutoff:
+        return _select_via_looping(
+            input, labels, idxs, positions, find_min, find_min_positions,
+            find_max, find_max_positions, find_median
+        )
+
+    order = cupy.lexsort(cupy.stack((input.ravel(), labels.ravel())))
+    input = input[order]
+    labels = labels[order]
+    if find_positions:
+        positions = positions[order]
+
+    # Determine indices corresponding to the min or max value for each label
+    label_change_index = cupy.searchsorted(labels,
+                                           cupy.arange(1, max_label + 2))
+    if find_min or find_min_positions or find_median:
+        # index corresponding to the minimum value at each label
+        min_index = label_change_index[:-1]
+    if find_max or find_max_positions or find_median:
+        # index corresponding to the maximum value at each label
+        max_index = label_change_index[1:] - 1
+
+    result = []
+    # the order below matches the order expected by cupy.ndimage.extrema
+    if find_min:
+        mins = cupy.zeros(int(labels.max()) + 2, input.dtype)
+        mins[labels[min_index]] = input[min_index]
+        result += [mins[idxs]]
+    if find_min_positions:
+        minpos = cupy.zeros(labels.max().item() + 2, int)
+        minpos[labels[min_index]] = positions[min_index]
+        result += [minpos[idxs]]
+    if find_max:
+        maxs = cupy.zeros(int(labels.max()) + 2, input.dtype)
+        maxs[labels[max_index]] = input[max_index]
+        result += [maxs[idxs]]
+    if find_max_positions:
+        maxpos = cupy.zeros(labels.max().item() + 2, int)
+        maxpos[labels[max_index]] = positions[max_index]
+        result += [maxpos[idxs]]
+    if find_median:
+        locs = cupy.arange(len(labels))
+        lo = cupy.zeros(int(labels.max()) + 2, int)
+        lo[labels[min_index]] = locs[min_index]
+        hi = cupy.zeros(int(labels.max()) + 2, int)
+        hi[labels[max_index]] = locs[max_index]
+        lo = lo[idxs]
+        hi = hi[idxs]
+        # lo is an index to the lowest value in input for each label,
+        # hi is an index to the largest value.
+        # move them to be either the same ((hi - lo) % 2 == 0) or next
+        # to each other ((hi - lo) % 2 == 1), then average.
+        step = (hi - lo) // 2
+        lo += step
+        hi -= step
+        if input.dtype.kind in 'iub':
+            # fix for https://github.com/scipy/scipy/issues/12836
+            result += [(input[lo].astype(float) + input[hi].astype(float)) /
+                       2.0]
+        else:
+            result += [(input[lo] + input[hi]) / 2.0]
+
+    return result
+
+
+def minimum(input, labels=None, index=None):
+    """Calculate the minimum of the values of an array over labeled regions.
+
+    Args:
+        input (cupy.ndarray):
+            Array of values. For each region specified by `labels`, the
+            minimal values of `input` over the region is computed.
+        labels (cupy.ndarray, optional): An array of integers marking different
+            regions over which the minimum value of `input` is to be computed.
+            `labels` must have the same shape as `input`. If `labels` is not
+            specified, the minimum over the whole array is returned.
+        index (array_like, optional): A list of region labels that are taken
+            into account for computing the minima. If `index` is None, the
+            minimum over all elements where `labels` is non-zero is returned.
+
+    Returns:
+        cupy.ndarray: Array of minima of `input` over the regions
+        determined by `labels` and whose index is in `index`. If `index` or
+        `labels` are not specified, a 0-dimensional cupy.ndarray is
+        returned: the minimal value of `input` if `labels` is None,
+        and the minimal value of elements where `labels` is greater than
+        zero if `index` is None.
+
+    .. seealso:: :func:`scipy.ndimage.minimum`
+    """
+    return _select(input, labels, index, find_min=True)[0]
+
+
+def maximum(input, labels=None, index=None):
+    """Calculate the maximum of the values of an array over labeled regions.
+
+    Args:
+        input (cupy.ndarray):
+            Array of values. For each region specified by `labels`, the
+            maximal values of `input` over the region is computed.
+        labels (cupy.ndarray, optional): An array of integers marking different
+            regions over which the maximum value of `input` is to be computed.
+            `labels` must have the same shape as `input`. If `labels` is not
+            specified, the maximum over the whole array is returned.
+        index (array_like, optional): A list of region labels that are taken
+            into account for computing the maxima. If `index` is None, the
+            maximum over all elements where `labels` is non-zero is returned.
+
+    Returns:
+        cupy.ndarray: Array of maxima of `input` over the regions
+        determaxed by `labels` and whose index is in `index`. If `index` or
+        `labels` are not specified, a 0-dimensional cupy.ndarray is
+        returned: the maximal value of `input` if `labels` is None,
+        and the maximal value of elements where `labels` is greater than
+        zero if `index` is None.
+
+    .. seealso:: :func:`scipy.ndimage.maximum`
+    """
+    return _select(input, labels, index, find_max=True)[0]
+
+
+def median(input, labels=None, index=None):
+    """Calculate the median of the values of an array over labeled regions.
+
+    Args:
+        input (cupy.ndarray):
+            Array of values. For each region specified by `labels`, the
+            median values of `input` over the region is computed.
+        labels (cupy.ndarray, optional): An array of integers marking different
+            regions over which the median value of `input` is to be computed.
+            `labels` must have the same shape as `input`. If `labels` is not
+            specified, the median over the whole array is returned.
+        index (array_like, optional): A list of region labels that are taken
+            into account for computing the medians. If `index` is None, the
+            median over all elements where `labels` is non-zero is returned.
+
+    Returns:
+        cupy.ndarray: Array of medians of `input` over the regions
+        determined by `labels` and whose index is in `index`. If `index` or
+        `labels` are not specified, a 0-dimensional cupy.ndarray is
+        returned: the median value of `input` if `labels` is None,
+        and the median value of elements where `labels` is greater than
+        zero if `index` is None.
+
+    .. seealso:: :func:`scipy.ndimage.median`
+    """
+    return _select(input, labels, index, find_median=True)[0]
+
+
+def minimum_position(input, labels=None, index=None):
+    """Find the positions of the minimums of the values of an array at labels.
+
+    For each region specified by `labels`, the position of the minimum
+    value of `input` within the region is returned.
+
+    Args:
+        input (cupy.ndarray):
+            Array of values. For each region specified by `labels`, the
+            minimal values of `input` over the region is computed.
+        labels (cupy.ndarray, optional): An array of integers marking different
+            regions over which the position of the minimum value of `input` is
+            to be computed. `labels` must have the same shape as `input`. If
+            `labels` is not specified, the location of the first minimum over
+            the whole array is returned.
+
+            The `labels` argument only works when `index` is specified.
+        index (array_like, optional): A list of region labels that are taken
+            into account for finding the location of the minima. If `index` is
+            None, the ``first`` minimum over all elements where `labels` is
+            non-zero is returned.
+
+            The `index` argument only works when `labels` is specified.
+
+    Returns:
+        Tuple of ints or list of tuples of ints that specify the location of
+        minima of `input` over the regions determined by `labels` and  whose
+        index is in `index`.
+
+        If `index` or `labels` are not specified, a tuple of ints is returned
+        specifying the location of the first minimal value of `input`.
+
+    .. note::
+        When `input` has multiple identical minima within a labeled region,
+        the coordinates returned are not guaranteed to match those returned by
+        SciPy.
+
+    .. seealso:: :func:`scipy.ndimage.minimum_position`
+    """
+    dims = numpy.asarray(input.shape)
+    # see numpy.unravel_index to understand this line.
+    dim_prod = numpy.cumprod([1] + list(dims[:0:-1]))[::-1]
+
+    result = _select(input, labels, index, find_min_positions=True)[0]
+
+    # have to transfer result back to the CPU to return index tuples
+    if result.ndim == 0:
+        result = int(result)  # synchronize
+    else:
+        result = cupy.asnumpy(result)  # synchronize
+
+    if cupy.isscalar(result):
+        return tuple((result // dim_prod) % dims)
+
+    return [tuple(v) for v in (result.reshape(-1, 1) // dim_prod) % dims]
+
+
+def maximum_position(input, labels=None, index=None):
+    """Find the positions of the maximums of the values of an array at labels.
+
+    For each region specified by `labels`, the position of the maximum
+    value of `input` within the region is returned.
+
+    Args:
+        input (cupy.ndarray):
+            Array of values. For each region specified by `labels`, the
+            maximal values of `input` over the region is computed.
+        labels (cupy.ndarray, optional): An array of integers marking different
+            regions over which the position of the maximum value of `input` is
+            to be computed. `labels` must have the same shape as `input`. If
+            `labels` is not specified, the location of the first maximum over
+            the whole array is returned.
+
+            The `labels` argument only works when `index` is specified.
+        index (array_like, optional): A list of region labels that are taken
+            into account for finding the location of the maxima. If `index` is
+            None, the ``first`` maximum over all elements where `labels` is
+            non-zero is returned.
+
+            The `index` argument only works when `labels` is specified.
+
+    Returns:
+        Tuple of ints or list of tuples of ints that specify the location of
+        maxima of `input` over the regions determaxed by `labels` and  whose
+        index is in `index`.
+
+        If `index` or `labels` are not specified, a tuple of ints is returned
+        specifying the location of the first maximal value of `input`.
+
+    .. note::
+        When `input` has multiple identical maxima within a labeled region,
+        the coordinates returned are not guaranteed to match those returned by
+        SciPy.
+
+    .. seealso:: :func:`scipy.ndimage.maximum_position`
+    """
+    dims = numpy.asarray(input.shape)
+    # see numpy.unravel_index to understand this line.
+    dim_prod = numpy.cumprod([1] + list(dims[:0:-1]))[::-1]
+
+    result = _select(input, labels, index, find_max_positions=True)[0]
+
+    # have to transfer result back to the CPU to return index tuples
+    if result.ndim == 0:
+        result = int(result)
+    else:
+        result = cupy.asnumpy(result)
+
+    if cupy.isscalar(result):
+        return tuple((result // dim_prod) % dims)
+
+    return [tuple(v) for v in (result.reshape(-1, 1) // dim_prod) % dims]
+
+
+def extrema(input, labels=None, index=None):
+    """Calculate the minimums and maximums of the values of an array at labels,
+    along with their positions.
+
+    Args:
+        input (cupy.ndarray): N-D image data to process.
+        labels (cupy.ndarray, optional): Labels of features in input. If not
+            None, must be same shape as `input`.
+        index (int or sequence of ints, optional): Labels to include in output.
+            If None (default), all values where non-zero `labels` are used.
+
+    Returns:
+        A tuple that contains the following values.
+
+        **minimums (cupy.ndarray)**: Values of minimums in each feature.
+
+        **maximums (cupy.ndarray)**: Values of maximums in each feature.
+
+        **min_positions (tuple or list of tuples)**: Each tuple gives the N-D
+        coordinates of the corresponding minimum.
+
+        **max_positions (tuple or list of tuples)**: Each tuple gives the N-D
+        coordinates of the corresponding maximum.
+
+    .. seealso:: :func:`scipy.ndimage.extrema`
+    """
+    dims = numpy.array(input.shape)
+    # see numpy.unravel_index to understand this line.
+    dim_prod = numpy.cumprod([1] + list(dims[:0:-1]))[::-1]
+
+    minimums, min_positions, maximums, max_positions = _select(
+        input,
+        labels,
+        index,
+        find_min=True,
+        find_max=True,
+        find_min_positions=True,
+        find_max_positions=True,
+    )
+
+    if min_positions.ndim == 0:
+        # scalar output case
+        min_positions = min_positions.item()
+        max_positions = max_positions.item()
+        return (
+            minimums,
+            maximums,
+            tuple((min_positions // dim_prod) % dims),
+            tuple((max_positions // dim_prod) % dims),
+        )
+
+    # convert indexes to tuples on the host
+    min_positions = cupy.asnumpy(min_positions)
+    max_positions = cupy.asnumpy(max_positions)
+    min_positions = [
+        tuple(v) for v in (min_positions.reshape(-1, 1) // dim_prod) % dims
+    ]
+    max_positions = [
+        tuple(v) for v in (max_positions.reshape(-1, 1) // dim_prod) % dims
+    ]
+
+    return minimums, maximums, min_positions, max_positions
+
+
+def center_of_mass(input, labels=None, index=None):
+    """
+    Calculate the center of mass of the values of an array at labels.
+
+    Args:
+        input (cupy.ndarray): Data from which to calculate center-of-mass. The
+            masses can either be positive or negative.
+        labels (cupy.ndarray, optional): Labels for objects in `input`, as
+            enerated by `ndimage.label`. Only used with `index`. Dimensions
+            must be the same as `input`.
+        index (int or sequence of ints, optional): Labels for which to
+            calculate centers-of-mass. If not specified, all labels greater
+            than zero are used. Only used with `labels`.
+
+    Returns:
+        tuple or list of tuples: Coordinates of centers-of-mass.
+
+    .. seealso:: :func:`scipy.ndimage.center_of_mass`
+    """
+    normalizer = sum(input, labels, index)
+    grids = cupy.ogrid[[slice(0, i) for i in input.shape]]
+
+    results = [
+        sum(input * grids[dir].astype(float), labels, index) / normalizer
+        for dir in range(input.ndim)
+    ]
+
+    # have to transfer 0-dim array back to CPU?
+    # may want to modify to avoid this
+    is_0dim_array = (
+        isinstance(results[0], cupy.ndarray) and results[0].ndim == 0
+    )
+    if is_0dim_array:
+        # tuple of 0-dimensional cupy arrays
+        return tuple(res for res in results)
+    # list of cupy coordinate arrays
+    return [v for v in cupy.stack(results, axis=-1)]
+
+
+def labeled_comprehension(
+    input, labels, index, func, out_dtype, default, pass_positions=False
+):
+    """Array resulting from applying ``func`` to each labeled region.
+
+    Roughly equivalent to [func(input[labels == i]) for i in index].
+
+    Sequentially applies an arbitrary function (that works on array_like input)
+    to subsets of an N-D image array specified by `labels` and `index`.
+    The option exists to provide the function with positional parameters as the
+    second argument.
+
+    Args:
+        input (cupy.ndarray): Data from which to select `labels` to process.
+        labels (cupy.ndarray or None):  Labels to objects in `input`. If not
+            None, array must be same shape as `input`. If None, `func` is
+            applied to raveled `input`.
+        index (int, sequence of ints or None): Subset of `labels` to which to
+            apply `func`. If a scalar, a single value is returned. If None,
+            `func` is applied to all non-zero values of `labels`.
+        func (callable): Python function to apply to `labels` from `input`.
+        out_dtype (dtype): Dtype to use for `result`.
+        default (int, float or None): Default return value when a element of
+            `index` does not exist in `labels`.
+        pass_positions (bool, optional): If True, pass linear indices to `func`
+            as a second argument.
+
+    Returns:
+        cupy.ndarray: Result of applying `func` to each of `labels` to `input`
+        in `index`.
+
+    .. seealso:: :func:`scipy.ndimage.labeled_comprehension`
+    """
+    as_scalar = cupy.isscalar(index)
+    input = cupy.asarray(input)
+
+    if pass_positions:
+        positions = cupy.arange(input.size).reshape(input.shape)
+
+    if labels is None:
+        if index is not None:
+            raise ValueError('index without defined labels')
+        if not pass_positions:
+            return func(input.ravel())
+        else:
+            return func(input.ravel(), positions.ravel())
+
+    try:
+        input, labels = cupy.broadcast_arrays(input, labels)
+    except ValueError:
+        raise ValueError(
+            'input and labels must have the same shape '
+            '(excepting dimensions with width 1)'
+        )
+
+    if index is None:
+        if not pass_positions:
+            return func(input[labels > 0])
+        else:
+            return func(input[labels > 0], positions[labels > 0])
+
+    index = cupy.atleast_1d(index)
+    if cupy.any(index.astype(labels.dtype).astype(index.dtype) != index):
+        raise ValueError(
+            'Cannot convert index values from <%s> to <%s> '
+            '(labels.dtype) without loss of precision'
+            % (index.dtype, labels.dtype)
+        )
+
+    index = index.astype(labels.dtype)
+
+    # optimization: find min/max in index, and select those parts of labels,
+    #               input, and positions
+    lo = index.min()
+    hi = index.max()
+    mask = (labels >= lo) & (labels <= hi)
+
+    # this also ravels the arrays
+    labels = labels[mask]
+    input = input[mask]
+    if pass_positions:
+        positions = positions[mask]
+
+    # sort everything by labels
+    label_order = labels.argsort()
+    labels = labels[label_order]
+    input = input[label_order]
+    if pass_positions:
+        positions = positions[label_order]
+
+    index_order = index.argsort()
+    sorted_index = index[index_order]
+
+    def do_map(inputs, output):
+        """labels must be sorted"""
+        nidx = sorted_index.size
+
+        # Find boundaries for each stretch of constant labels
+        # This could be faster, but we already paid N log N to sort labels.
+        lo = cupy.searchsorted(labels, sorted_index, side='left')
+        hi = cupy.searchsorted(labels, sorted_index, side='right')
+
+        for i, low, high in zip(range(nidx), lo, hi):
+            if low == high:
+                continue
+            output[i] = func(*[inp[low:high] for inp in inputs])
+
+    if out_dtype == object:
+        temp = {i: default for i in range(index.size)}
+    else:
+        temp = cupy.empty(index.shape, out_dtype)
+        if default is None and temp.dtype.kind in 'fc':
+            default = numpy.nan  # match NumPy floating-point None behavior
+        temp[:] = default
+
+    if not pass_positions:
+        do_map([input], temp)
+    else:
+        do_map([input, positions], temp)
+
+    if out_dtype == object:
+        # use a list of arrays since object arrays are not supported
+        index_order = cupy.asnumpy(index_order)
+        output = [temp[i] for i in index_order.argsort()]
+    else:
+        output = cupy.zeros(index.shape, out_dtype)
+        output[cupy.asnumpy(index_order)] = temp
+    if as_scalar:
+        output = output[0]
+    return output
+
+
+def histogram(input, min, max, bins, labels=None, index=None):
+    """Calculate the histogram of the values of an array, optionally at labels.
+
+    Histogram calculates the frequency of values in an array within bins
+    determined by `min`, `max`, and `bins`. The `labels` and `index`
+    keywords can limit the scope of the histogram to specified sub-regions
+    within the array.
+
+    Args:
+        input (cupy.ndarray): Data for which to calculate histogram.
+        min (int): Minimum values of range of histogram bins.
+        max (int): Maximum values of range of histogram bins.
+        bins (int): Number of bins.
+        labels (cupy.ndarray, optional): Labels for objects in `input`. If not
+            None, must be same shape as `input`.
+        index (int or sequence of ints, optional): Label or labels for which to
+            calculate histogram. If None, all values where label is greater
+            than zero are used.
+
+    Returns:
+        cupy.ndarray: Histogram counts.
+
+    .. seealso:: :func:`scipy.ndimage.histogram`
+    """
+    _bins = cupy.linspace(min, max, bins + 1)
+
+    def _hist(vals):
+        return cupy.histogram(vals, _bins)[0]
+
+    return labeled_comprehension(
+        input, labels, index, _hist, object, None, pass_positions=False
+    )
diff --git a/cupyx/scipy/ndimage/_morphology.py b/cupyx/scipy/ndimage/_morphology.py
new file mode 100644
index 0000000..733d904
--- /dev/null
+++ b/cupyx/scipy/ndimage/_morphology.py
@@ -0,0 +1,1017 @@
+import operator
+import warnings
+
+import numpy
+
+import cupy
+from cupy import _core
+
+from cupyx.scipy.ndimage import _filters_core
+from cupyx.scipy.ndimage import _util
+from cupyx.scipy.ndimage import _filters
+
+
+@cupy.memoize(for_each_device=True)
+def _get_binary_erosion_kernel(
+    w_shape, int_type, offsets, center_is_true, border_value, invert, masked,
+    all_weights_nonzero
+):
+    if invert:
+        border_value = int(not border_value)
+        true_val = 0
+        false_val = 1
+    else:
+        true_val = 1
+        false_val = 0
+
+    if masked:
+        pre = """
+            bool mv = (bool)mask[i];
+            bool _in = (bool)x[i];
+            if (!mv) {{
+                y = cast<Y>(_in);
+                return;
+            }} else if ({center_is_true} && _in == {false_val}) {{
+                y = cast<Y>(_in);
+                return;
+            }}""".format(center_is_true=int(center_is_true),
+                         false_val=false_val)
+    else:
+        pre = """
+            bool _in = (bool)x[i];
+            if ({center_is_true} && _in == {false_val}) {{
+                y = cast<Y>(_in);
+                return;
+            }}""".format(center_is_true=int(center_is_true),
+                         false_val=false_val)
+    pre = pre + """
+            y = cast<Y>({true_val});""".format(true_val=true_val)
+
+    # {{{{ required because format is called again within _generate_nd_kernel
+    found = """
+        if ({{cond}}) {{{{
+            if (!{border_value}) {{{{
+                y = cast<Y>({false_val});
+                return;
+            }}}}
+        }}}} else {{{{
+            bool nn = {{value}} ? {true_val} : {false_val};
+            if (!nn) {{{{
+                y = cast<Y>({false_val});
+                return;
+            }}}}
+        }}}}""".format(true_val=int(true_val),
+                       false_val=int(false_val),
+                       border_value=int(border_value),)
+
+    name = 'binary_erosion'
+    if false_val:
+        name += '_invert'
+    return _filters_core._generate_nd_kernel(
+        name,
+        pre,
+        found,
+        '',
+        'constant', w_shape, int_type, offsets, 0, ctype='Y', has_weights=True,
+        has_structure=False, has_mask=masked, binary_morphology=True,
+        all_weights_nonzero=all_weights_nonzero)
+
+
+def _center_is_true(structure, origin):
+    coor = tuple([oo + ss // 2 for ss, oo in zip(structure.shape, origin)])
+    return bool(structure[coor])  # device synchronization
+
+
+def iterate_structure(structure, iterations, origin=None):
+    """Iterate a structure by dilating it with itself.
+
+    Args:
+        structure(array_like): Structuring element (an array of bools,
+            for example), to be dilated with itself.
+        iterations(int): The number of dilations performed on the structure
+            with itself.
+        origin(int or tuple of int, optional): If origin is None, only the
+            iterated structure is returned. If not, a tuple of the iterated
+            structure and the modified origin is returned.
+
+    Returns:
+        cupy.ndarray: A new structuring element obtained by dilating
+        ``structure`` (``iterations`` - 1) times with itself.
+
+    .. seealso:: :func:`scipy.ndimage.iterate_structure`
+    """
+    if iterations < 2:
+        return structure.copy()
+    ni = iterations - 1
+    shape = [ii + ni * (ii - 1) for ii in structure.shape]
+    pos = [ni * (structure.shape[ii] // 2) for ii in range(len(shape))]
+    slc = tuple(
+        slice(pos[ii], pos[ii] + structure.shape[ii], None)
+        for ii in range(len(shape))
+    )
+    out = cupy.zeros(shape, bool)
+    out[slc] = structure != 0
+    out = binary_dilation(out, structure, iterations=ni)
+    if origin is None:
+        return out
+    else:
+        origin = _util._fix_sequence_arg(origin, structure.ndim, 'origin', int)
+        origin = [iterations * o for o in origin]
+        return out, origin
+
+
+def generate_binary_structure(rank, connectivity):
+    """Generate a binary structure for binary morphological operations.
+
+    Args:
+        rank(int): Number of dimensions of the array to which the structuring
+            element will be applied, as returned by ``np.ndim``.
+        connectivity(int): ``connectivity`` determines which elements of the
+            output array belong to the structure, i.e., are considered as
+            neighbors of the central element. Elements up to a squared distance
+            of ``connectivity`` from the center are considered neighbors.
+            ``connectivity`` may range from 1 (no diagonal elements are
+            neighbors) to ``rank`` (all elements are neighbors).
+
+    Returns:
+        cupy.ndarray: Structuring element which may be used for binary
+        morphological operations, with ``rank`` dimensions and all
+        dimensions equal to 3.
+
+    .. seealso:: :func:`scipy.ndimage.generate_binary_structure`
+    """
+    if connectivity < 1:
+        connectivity = 1
+    if rank < 1:
+        return cupy.asarray(True, dtype=bool)
+    output = numpy.fabs(numpy.indices([3] * rank) - 1)
+    output = numpy.add.reduce(output, 0)
+    output = output <= connectivity
+    return cupy.asarray(output)
+
+
+def _binary_erosion(input, structure, iterations, mask, output, border_value,
+                    origin, invert, brute_force=True):
+    try:
+        iterations = operator.index(iterations)
+    except TypeError:
+        raise TypeError('iterations parameter should be an integer')
+
+    if input.dtype.kind == 'c':
+        raise TypeError('Complex type not supported')
+    if structure is None:
+        structure = generate_binary_structure(input.ndim, 1)
+        all_weights_nonzero = input.ndim == 1
+        center_is_true = True
+        default_structure = True
+    else:
+        structure = structure.astype(dtype=bool, copy=False)
+        # transfer to CPU for use in determining if it is fully dense
+        # structure_cpu = cupy.asnumpy(structure)
+        default_structure = False
+    if structure.ndim != input.ndim:
+        raise RuntimeError('structure and input must have same dimensionality')
+    if not structure.flags.c_contiguous:
+        structure = cupy.ascontiguousarray(structure)
+    if structure.size < 1:
+        raise RuntimeError('structure must not be empty')
+
+    if mask is not None:
+        if mask.shape != input.shape:
+            raise RuntimeError('mask and input must have equal sizes')
+        if not mask.flags.c_contiguous:
+            mask = cupy.ascontiguousarray(mask)
+        masked = True
+    else:
+        masked = False
+    origin = _util._fix_sequence_arg(origin, input.ndim, 'origin', int)
+
+    if isinstance(output, cupy.ndarray):
+        if output.dtype.kind == 'c':
+            raise TypeError('Complex output type not supported')
+    else:
+        output = bool
+    output = _util._get_output(output, input)
+    temp_needed = cupy.shares_memory(output, input, 'MAY_SHARE_BOUNDS')
+    if temp_needed:
+        # input and output arrays cannot share memory
+        temp = output
+        output = _util._get_output(output.dtype, input)
+    if structure.ndim == 0:
+        # kernel doesn't handle ndim=0, so special case it here
+        if float(structure):
+            output[...] = cupy.asarray(input, dtype=bool)
+        else:
+            output[...] = ~cupy.asarray(input, dtype=bool)
+        return output
+    origin = tuple(origin)
+    int_type = _util._get_inttype(input)
+    offsets = _filters_core._origins_to_offsets(origin, structure.shape)
+    if not default_structure:
+        # synchronize required to determine if all weights are non-zero
+        nnz = int(cupy.count_nonzero(structure))
+        all_weights_nonzero = nnz == structure.size
+        if all_weights_nonzero:
+            center_is_true = True
+        else:
+            center_is_true = _center_is_true(structure, origin)
+
+    erode_kernel = _get_binary_erosion_kernel(
+        structure.shape, int_type, offsets, center_is_true, border_value,
+        invert, masked, all_weights_nonzero,
+    )
+
+    if iterations == 1:
+        if masked:
+            output = erode_kernel(input, structure, mask, output)
+        else:
+            output = erode_kernel(input, structure, output)
+    elif center_is_true and not brute_force:
+        raise NotImplementedError(
+            'only brute_force iteration has been implemented'
+        )
+    else:
+        if cupy.shares_memory(output, input, 'MAY_SHARE_BOUNDS'):
+            raise ValueError('output and input may not overlap in memory')
+        tmp_in = cupy.empty_like(input, dtype=output.dtype)
+        tmp_out = output
+        if iterations >= 1 and not iterations & 1:
+            tmp_in, tmp_out = tmp_out, tmp_in
+        if masked:
+            tmp_out = erode_kernel(input, structure, mask, tmp_out)
+        else:
+            tmp_out = erode_kernel(input, structure, tmp_out)
+        # TODO: kernel doesn't return the changed status, so determine it here
+        changed = not (input == tmp_out).all()  # synchronize!
+        ii = 1
+        while ii < iterations or ((iterations < 1) and changed):
+            tmp_in, tmp_out = tmp_out, tmp_in
+            if masked:
+                tmp_out = erode_kernel(tmp_in, structure, mask, tmp_out)
+            else:
+                tmp_out = erode_kernel(tmp_in, structure, tmp_out)
+            changed = not (tmp_in == tmp_out).all()
+            ii += 1
+            if not changed and (not ii & 1):  # synchronize!
+                # can exit early if nothing changed
+                # (only do this after even number of tmp_in/out swaps)
+                break
+        output = tmp_out
+    if temp_needed:
+        _core.elementwise_copy(output, temp)
+        output = temp
+    return output
+
+
+def binary_erosion(input, structure=None, iterations=1, mask=None, output=None,
+                   border_value=0, origin=0, brute_force=False):
+    """Multidimensional binary erosion with a given structuring element.
+
+    Binary erosion is a mathematical morphology operation used for image
+    processing.
+
+    Args:
+        input(cupy.ndarray): The input binary array_like to be eroded.
+            Non-zero (True) elements form the subset to be eroded.
+        structure(cupy.ndarray, optional): The structuring element used for the
+            erosion. Non-zero elements are considered True. If no structuring
+            element is provided an element is generated with a square
+            connectivity equal to one. (Default value = None).
+        iterations(int, optional): The erosion is repeated ``iterations`` times
+            (one, by default). If iterations is less than 1, the erosion is
+            repeated until the result does not change anymore. Only an integer
+            of iterations is accepted.
+        mask(cupy.ndarray or None, optional): If a mask is given, only those
+            elements with a True value at the corresponding mask element are
+            modified at each iteration. (Default value = None)
+        output(cupy.ndarray, optional): Array of the same shape as input, into
+            which the output is placed. By default, a new array is created.
+        border_value(int (cast to 0 or 1), optional): Value at the
+            border in the output array. (Default value = 0)
+        origin(int or tuple of ints, optional): Placement of the filter, by
+            default 0.
+        brute_force(boolean, optional): Memory condition: if False, only the
+            pixels whose value was changed in the last iteration are tracked as
+            candidates to be updated (eroded) in the current iteration; if
+            True all pixels are considered as candidates for erosion,
+            regardless of what happened in the previous iteration.
+
+    Returns:
+        cupy.ndarray: The result of binary erosion.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`scipy.ndimage.binary_erosion`
+    """
+    return _binary_erosion(input, structure, iterations, mask, output,
+                           border_value, origin, 0, brute_force)
+
+
+def binary_dilation(input, structure=None, iterations=1, mask=None,
+                    output=None, border_value=0, origin=0, brute_force=False):
+    """Multidimensional binary dilation with the given structuring element.
+
+    Args:
+        input(cupy.ndarray): The input binary array_like to be dilated.
+            Non-zero (True) elements form the subset to be dilated.
+        structure(cupy.ndarray, optional): The structuring element used for the
+            dilation. Non-zero elements are considered True. If no structuring
+            element is provided an element is generated with a square
+            connectivity equal to one. (Default value = None).
+        iterations(int, optional): The dilation is repeated ``iterations``
+            times (one, by default). If iterations is less than 1, the dilation
+            is repeated until the result does not change anymore. Only an
+            integer of iterations is accepted.
+        mask(cupy.ndarray or None, optional): If a mask is given, only those
+            elements with a True value at the corresponding mask element are
+            modified at each iteration. (Default value = None)
+        output(cupy.ndarray, optional): Array of the same shape as input, into
+            which the output is placed. By default, a new array is created.
+        border_value(int (cast to 0 or 1), optional): Value at the
+            border in the output array. (Default value = 0)
+        origin(int or tuple of ints, optional): Placement of the filter, by
+            default 0.
+        brute_force(boolean, optional): Memory condition: if False, only the
+            pixels whose value was changed in the last iteration are tracked as
+            candidates to be updated (dilated) in the current iteration; if
+            True all pixels are considered as candidates for dilation,
+            regardless of what happened in the previous iteration.
+
+    Returns:
+        cupy.ndarray: The result of binary dilation.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`scipy.ndimage.binary_dilation`
+    """
+    if structure is None:
+        structure = generate_binary_structure(input.ndim, 1)
+    origin = _util._fix_sequence_arg(origin, input.ndim, 'origin', int)
+    structure = structure[tuple([slice(None, None, -1)] * structure.ndim)]
+    for ii in range(len(origin)):
+        origin[ii] = -origin[ii]
+        if not structure.shape[ii] & 1:
+            origin[ii] -= 1
+    return _binary_erosion(input, structure, iterations, mask, output,
+                           border_value, origin, 1, brute_force)
+
+
+def binary_opening(input, structure=None, iterations=1, output=None, origin=0,
+                   mask=None, border_value=0, brute_force=False):
+    """
+    Multidimensional binary opening with the given structuring element.
+
+    The *opening* of an input image by a structuring element is the
+    *dilation* of the *erosion* of the image by the structuring element.
+
+    Args:
+        input(cupy.ndarray): The input binary array to be opened.
+            Non-zero (True) elements form the subset to be opened.
+        structure(cupy.ndarray, optional): The structuring element used for the
+            opening. Non-zero elements are considered True. If no structuring
+            element is provided an element is generated with a square
+            connectivity equal to one. (Default value = None).
+        iterations(int, optional): The opening is repeated ``iterations`` times
+            (one, by default). If iterations is less than 1, the opening is
+            repeated until the result does not change anymore. Only an integer
+            of iterations is accepted.
+        output(cupy.ndarray, optional): Array of the same shape as input, into
+            which the output is placed. By default, a new array is created.
+        origin(int or tuple of ints, optional): Placement of the filter, by
+            default 0.
+        mask(cupy.ndarray or None, optional): If a mask is given, only those
+            elements with a True value at the corresponding mask element are
+            modified at each iteration. (Default value = None)
+        border_value(int (cast to 0 or 1), optional): Value at the
+            border in the output array. (Default value = 0)
+        brute_force(boolean, optional): Memory condition: if False, only the
+            pixels whose value was changed in the last iteration are tracked as
+            candidates to be updated (dilated) in the current iteration; if
+            True all pixels are considered as candidates for opening,
+            regardless of what happened in the previous iteration.
+
+    Returns:
+        cupy.ndarray: The result of binary opening.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`scipy.ndimage.binary_opening`
+    """
+    if structure is None:
+        rank = input.ndim
+        structure = generate_binary_structure(rank, 1)
+    tmp = binary_erosion(input, structure, iterations, mask, None,
+                         border_value, origin, brute_force)
+    return binary_dilation(tmp, structure, iterations, mask, output,
+                           border_value, origin, brute_force)
+
+
+def binary_closing(input, structure=None, iterations=1, output=None, origin=0,
+                   mask=None, border_value=0, brute_force=False):
+    """
+    Multidimensional binary closing with the given structuring element.
+
+    The *closing* of an input image by a structuring element is the
+    *erosion* of the *dilation* of the image by the structuring element.
+
+    Args:
+        input(cupy.ndarray): The input binary array to be closed.
+            Non-zero (True) elements form the subset to be closed.
+        structure(cupy.ndarray, optional): The structuring element used for the
+            closing. Non-zero elements are considered True. If no structuring
+            element is provided an element is generated with a square
+            connectivity equal to one. (Default value = None).
+        iterations(int, optional): The closing is repeated ``iterations`` times
+            (one, by default). If iterations is less than 1, the closing is
+            repeated until the result does not change anymore. Only an integer
+            of iterations is accepted.
+        output(cupy.ndarray, optional): Array of the same shape as input, into
+            which the output is placed. By default, a new array is created.
+        origin(int or tuple of ints, optional): Placement of the filter, by
+            default 0.
+        mask(cupy.ndarray or None, optional): If a mask is given, only those
+            elements with a True value at the corresponding mask element are
+            modified at each iteration. (Default value = None)
+        border_value(int (cast to 0 or 1), optional): Value at the
+            border in the output array. (Default value = 0)
+        brute_force(boolean, optional): Memory condition: if False, only the
+            pixels whose value was changed in the last iteration are tracked as
+            candidates to be updated (dilated) in the current iteration; if
+            True all pixels are considered as candidates for closing,
+            regardless of what happened in the previous iteration.
+
+    Returns:
+        cupy.ndarray: The result of binary closing.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`scipy.ndimage.binary_closing`
+    """
+    if structure is None:
+        rank = input.ndim
+        structure = generate_binary_structure(rank, 1)
+    tmp = binary_dilation(input, structure, iterations, mask, None,
+                          border_value, origin, brute_force)
+    return binary_erosion(tmp, structure, iterations, mask, output,
+                          border_value, origin, brute_force)
+
+
+def binary_hit_or_miss(input, structure1=None, structure2=None, output=None,
+                       origin1=0, origin2=None):
+    """
+    Multidimensional binary hit-or-miss transform.
+
+    The hit-or-miss transform finds the locations of a given pattern
+    inside the input image.
+
+    Args:
+        input (cupy.ndarray): Binary image where a pattern is to be detected.
+        structure1 (cupy.ndarray, optional): Part of the structuring element to
+            be fitted to the foreground (non-zero elements) of ``input``. If no
+            value is provided, a structure of square connectivity 1 is chosen.
+        structure2 (cupy.ndarray, optional): Second part of the structuring
+            element that has to miss completely the foreground. If no value is
+            provided, the complementary of ``structure1`` is taken.
+        output (cupy.ndarray, dtype or None, optional): Array of the same shape
+            as input, into which the output is placed. By default, a new array
+            is created.
+        origin1 (int or tuple of ints, optional): Placement of the first part
+            of the structuring element ``structure1``, by default 0 for a
+            centered structure.
+        origin2 (int or tuple of ints or None, optional): Placement of the
+            second part of the structuring element ``structure2``, by default 0
+            for a centered structure. If a value is provided for ``origin1``
+            and not for ``origin2``, then ``origin2`` is set to ``origin1``.
+
+    Returns:
+        cupy.ndarray: Hit-or-miss transform of ``input`` with the given
+        structuring element (``structure1``, ``structure2``).
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`scipy.ndimage.binary_hit_or_miss`
+    """
+    if structure1 is None:
+        structure1 = generate_binary_structure(input.ndim, 1)
+    if structure2 is None:
+        structure2 = cupy.logical_not(structure1)
+    origin1 = _util._fix_sequence_arg(origin1, input.ndim, 'origin1', int)
+    if origin2 is None:
+        origin2 = origin1
+    else:
+        origin2 = _util._fix_sequence_arg(origin2, input.ndim, 'origin2', int)
+
+    tmp1 = _binary_erosion(input, structure1, 1, None, None, 0, origin1, 0,
+                           False)
+    inplace = isinstance(output, cupy.ndarray)
+    result = _binary_erosion(input, structure2, 1, None, output, 0, origin2, 1,
+                             False)
+    if inplace:
+        cupy.logical_not(output, output)
+        cupy.logical_and(tmp1, output, output)
+    else:
+        cupy.logical_not(result, result)
+        return cupy.logical_and(tmp1, result)
+
+
+def binary_propagation(input, structure=None, mask=None, output=None,
+                       border_value=0, origin=0):
+    """
+    Multidimensional binary propagation with the given structuring element.
+
+    Args:
+        input (cupy.ndarray): Binary image to be propagated inside ``mask``.
+        structure (cupy.ndarray, optional): Structuring element used in the
+            successive dilations. The output may depend on the structuring
+            element, especially if ``mask`` has several connex components. If
+            no structuring element is provided, an element is generated with a
+            squared connectivity equal to one.
+        mask (cupy.ndarray, optional): Binary mask defining the region into
+            which ``input`` is allowed to propagate.
+        output (cupy.ndarray, optional): Array of the same shape as input, into
+            which the output is placed. By default, a new array is created.
+        border_value (int, optional): Value at the border in the output array.
+            The value is cast to 0 or 1.
+        origin (int or tuple of ints, optional): Placement of the filter.
+
+    Returns:
+        cupy.ndarray : Binary propagation of ``input`` inside ``mask``.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`scipy.ndimage.binary_propagation`
+    """
+    return binary_dilation(input, structure, -1, mask, output, border_value,
+                           origin, brute_force=True)
+
+
+def binary_fill_holes(input, structure=None, output=None, origin=0):
+    """Fill the holes in binary objects.
+
+    Args:
+        input (cupy.ndarray): N-D binary array with holes to be filled.
+        structure (cupy.ndarray, optional):  Structuring element used in the
+            computation; large-size elements make computations faster but may
+            miss holes separated from the background by thin regions. The
+            default element (with a square connectivity equal to one) yields
+            the intuitive result where all holes in the input have been filled.
+        output (cupy.ndarray, dtype or None, optional): Array of the same shape
+            as input, into which the output is placed. By default, a new array
+            is created.
+        origin (int, tuple of ints, optional): Position of the structuring
+            element.
+
+    Returns:
+        cupy.ndarray: Transformation of the initial image ``input`` where holes
+        have been filled.
+
+    .. warning::
+
+        This function may synchronize the device.
+
+    .. seealso:: :func:`scipy.ndimage.binary_fill_holes`
+    """
+    mask = cupy.logical_not(input)
+    tmp = cupy.zeros(mask.shape, bool)
+    inplace = isinstance(output, cupy.ndarray)
+    # TODO (grlee77): set brute_force=False below once implemented
+    if inplace:
+        binary_dilation(tmp, structure, -1, mask, output, 1, origin,
+                        brute_force=True)
+        cupy.logical_not(output, output)
+    else:
+        output = binary_dilation(tmp, structure, -1, mask, None, 1, origin,
+                                 brute_force=True)
+        cupy.logical_not(output, output)
+        return output
+
+
+def grey_erosion(input, size=None, footprint=None, structure=None, output=None,
+                 mode='reflect', cval=0.0, origin=0):
+    """Calculates a greyscale erosion.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (tuple of ints): Shape of a flat and full structuring element used
+            for the greyscale erosion. Optional if ``footprint`` or
+            ``structure`` is provided.
+        footprint (array of ints): Positions of non-infinite elements of a flat
+            structuring element used for greyscale erosion. Non-zero values
+            give the set of neighbors of the center over which minimum is
+            chosen.
+        structure (array of ints): Structuring element used for the greyscale
+            erosion. ``structure`` may be a non-flat structuring element.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of greyscale erosion.
+
+    .. seealso:: :func:`scipy.ndimage.grey_erosion`
+    """
+
+    if size is None and footprint is None and structure is None:
+        raise ValueError('size, footprint or structure must be specified')
+
+    return _filters._min_or_max_filter(input, size, footprint, structure,
+                                       output, mode, cval, origin, 'min')
+
+
+def grey_dilation(input, size=None, footprint=None, structure=None,
+                  output=None, mode='reflect', cval=0.0, origin=0):
+    """Calculates a greyscale dilation.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (tuple of ints): Shape of a flat and full structuring element used
+            for the greyscale dilation. Optional if ``footprint`` or
+            ``structure`` is provided.
+        footprint (array of ints): Positions of non-infinite elements of a flat
+            structuring element used for greyscale dilation. Non-zero values
+            give the set of neighbors of the center over which maximum is
+            chosen.
+        structure (array of ints): Structuring element used for the greyscale
+            dilation. ``structure`` may be a non-flat structuring element.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of greyscale dilation.
+
+    .. seealso:: :func:`scipy.ndimage.grey_dilation`
+    """
+
+    if size is None and footprint is None and structure is None:
+        raise ValueError('size, footprint or structure must be specified')
+    if structure is not None:
+        structure = cupy.array(structure)
+        structure = structure[tuple([slice(None, None, -1)] * structure.ndim)]
+    if footprint is not None:
+        footprint = cupy.array(footprint)
+        footprint = footprint[tuple([slice(None, None, -1)] * footprint.ndim)]
+
+    origin = _util._fix_sequence_arg(origin, input.ndim, 'origin', int)
+    for i in range(len(origin)):
+        origin[i] = -origin[i]
+        if footprint is not None:
+            sz = footprint.shape[i]
+        elif structure is not None:
+            sz = structure.shape[i]
+        elif numpy.isscalar(size):
+            sz = size
+        else:
+            sz = size[i]
+        if sz % 2 == 0:
+            origin[i] -= 1
+
+    return _filters._min_or_max_filter(input, size, footprint, structure,
+                                       output, mode, cval, origin, 'max')
+
+
+def grey_closing(input, size=None, footprint=None, structure=None,
+                 output=None, mode='reflect', cval=0.0, origin=0):
+    """Calculates a multi-dimensional greyscale closing.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (tuple of ints): Shape of a flat and full structuring element used
+            for the greyscale closing. Optional if ``footprint`` or
+            ``structure`` is provided.
+        footprint (array of ints): Positions of non-infinite elements of a flat
+            structuring element used for greyscale closing. Non-zero values
+            give the set of neighbors of the center over which closing is
+            chosen.
+        structure (array of ints): Structuring element used for the greyscale
+            closing. ``structure`` may be a non-flat structuring element.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of greyscale closing.
+
+    .. seealso:: :func:`scipy.ndimage.grey_closing`
+    """
+    if (size is not None) and (footprint is not None):
+        warnings.warn('ignoring size because footprint is set', UserWarning,
+                      stacklevel=2)
+    tmp = grey_dilation(input, size, footprint, structure, None, mode, cval,
+                        origin)
+    return grey_erosion(tmp, size, footprint, structure, output, mode, cval,
+                        origin)
+
+
+def grey_opening(input, size=None, footprint=None, structure=None,
+                 output=None, mode='reflect', cval=0.0, origin=0):
+    """Calculates a multi-dimensional greyscale opening.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (tuple of ints): Shape of a flat and full structuring element used
+            for the greyscale opening. Optional if ``footprint`` or
+            ``structure`` is provided.
+        footprint (array of ints): Positions of non-infinite elements of a flat
+            structuring element used for greyscale opening. Non-zero values
+            give the set of neighbors of the center over which opening is
+            chosen.
+        structure (array of ints): Structuring element used for the greyscale
+            opening. ``structure`` may be a non-flat structuring element.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The result of greyscale opening.
+
+    .. seealso:: :func:`scipy.ndimage.grey_opening`
+    """
+    if (size is not None) and (footprint is not None):
+        warnings.warn('ignoring size because footprint is set', UserWarning,
+                      stacklevel=2)
+    tmp = grey_erosion(input, size, footprint, structure, None, mode, cval,
+                       origin)
+    return grey_dilation(tmp, size, footprint, structure, output, mode, cval,
+                         origin)
+
+
+def morphological_gradient(
+    input,
+    size=None,
+    footprint=None,
+    structure=None,
+    output=None,
+    mode='reflect',
+    cval=0.0,
+    origin=0,
+):
+    """
+    Multidimensional morphological gradient.
+
+    The morphological gradient is calculated as the difference between a
+    dilation and an erosion of the input with a given structuring element.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (tuple of ints): Shape of a flat and full structuring element used
+            for the morphological gradient. Optional if ``footprint`` or
+            ``structure`` is provided.
+        footprint (array of ints): Positions of non-infinite elements of a flat
+            structuring element used for morphological gradient. Non-zero
+            values give the set of neighbors of the center over which opening
+            is chosen.
+        structure (array of ints): Structuring element used for the
+            morphological gradient. ``structure`` may be a non-flat
+            structuring element.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The morphological gradient of the input.
+
+    .. seealso:: :func:`scipy.ndimage.morphological_gradient`
+    """
+    tmp = grey_dilation(
+        input, size, footprint, structure, None, mode, cval, origin
+    )
+    if isinstance(output, cupy.ndarray):
+        grey_erosion(
+            input, size, footprint, structure, output, mode, cval, origin
+        )
+        return cupy.subtract(tmp, output, output)
+    else:
+        return tmp - grey_erosion(
+            input, size, footprint, structure, None, mode, cval, origin
+        )
+
+
+def morphological_laplace(
+    input,
+    size=None,
+    footprint=None,
+    structure=None,
+    output=None,
+    mode='reflect',
+    cval=0.0,
+    origin=0,
+):
+    """
+    Multidimensional morphological laplace.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (tuple of ints): Shape of a flat and full structuring element used
+            for the morphological laplace. Optional if ``footprint`` or
+            ``structure`` is provided.
+        footprint (array of ints): Positions of non-infinite elements of a flat
+            structuring element used for morphological laplace. Non-zero
+            values give the set of neighbors of the center over which opening
+            is chosen.
+        structure (array of ints): Structuring element used for the
+            morphological laplace. ``structure`` may be a non-flat
+            structuring element.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: The morphological laplace of the input.
+
+    .. seealso:: :func:`scipy.ndimage.morphological_laplace`
+    """
+    tmp1 = grey_dilation(
+        input, size, footprint, structure, None, mode, cval, origin
+    )
+    if isinstance(output, cupy.ndarray):
+        grey_erosion(
+            input, size, footprint, structure, output, mode, cval, origin
+        )
+        cupy.add(tmp1, output, output)
+        cupy.subtract(output, input, output)
+        return cupy.subtract(output, input, output)
+    else:
+        tmp2 = grey_erosion(
+            input, size, footprint, structure, None, mode, cval, origin
+        )
+        cupy.add(tmp1, tmp2, tmp2)
+        cupy.subtract(tmp2, input, tmp2)
+        cupy.subtract(tmp2, input, tmp2)
+        return tmp2
+
+
+def white_tophat(
+    input,
+    size=None,
+    footprint=None,
+    structure=None,
+    output=None,
+    mode='reflect',
+    cval=0.0,
+    origin=0,
+):
+    """
+    Multidimensional white tophat filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (tuple of ints): Shape of a flat and full structuring element used
+            for the white tophat. Optional if ``footprint`` or ``structure`` is
+            provided.
+        footprint (array of ints): Positions of non-infinite elements of a flat
+            structuring element used for the white tophat. Non-zero values
+            give the set of neighbors of the center over which opening is
+            chosen.
+        structure (array of ints): Structuring element used for the white
+            tophat. ``structure`` may be a non-flat structuring element.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarray: Result of the filter of ``input`` with ``structure``.
+
+    .. seealso:: :func:`scipy.ndimage.white_tophat`
+    """
+    if (size is not None) and (footprint is not None):
+        warnings.warn(
+            'ignoring size because footprint is set', UserWarning, stacklevel=2
+        )
+    tmp = grey_erosion(
+        input, size, footprint, structure, None, mode, cval, origin
+    )
+    tmp = grey_dilation(
+        tmp, size, footprint, structure, output, mode, cval, origin
+    )
+    if input.dtype == numpy.bool_ and tmp.dtype == numpy.bool_:
+        cupy.bitwise_xor(input, tmp, out=tmp)
+    else:
+        cupy.subtract(input, tmp, out=tmp)
+    return tmp
+
+
+def black_tophat(
+    input,
+    size=None,
+    footprint=None,
+    structure=None,
+    output=None,
+    mode='reflect',
+    cval=0.0,
+    origin=0,
+):
+    """
+    Multidimensional black tophat filter.
+
+    Args:
+        input (cupy.ndarray): The input array.
+        size (tuple of ints): Shape of a flat and full structuring element used
+            for the black tophat. Optional if ``footprint`` or ``structure`` is
+            provided.
+        footprint (array of ints): Positions of non-infinite elements of a flat
+            structuring element used for the black tophat. Non-zero values
+            give the set of neighbors of the center over which opening is
+            chosen.
+        structure (array of ints): Structuring element used for the black
+            tophat. ``structure`` may be a non-flat structuring element.
+        output (cupy.ndarray, dtype or None): The array in which to place the
+            output.
+        mode (str): The array borders are handled according to the given mode
+            (``'reflect'``, ``'constant'``, ``'nearest'``, ``'mirror'``,
+            ``'wrap'``). Default is ``'reflect'``.
+        cval (scalar): Value to fill past edges of input if mode is
+            ``constant``. Default is ``0.0``.
+        origin (scalar or tuple of scalar): The origin parameter controls the
+            placement of the filter, relative to the center of the current
+            element of the input. Default of 0 is equivalent to
+            ``(0,)*input.ndim``.
+
+    Returns:
+        cupy.ndarry : Result of the filter of ``input`` with ``structure``.
+
+    .. seealso:: :func:`scipy.ndimage.black_tophat`
+    """
+    if (size is not None) and (footprint is not None):
+        warnings.warn(
+            'ignoring size because footprint is set', UserWarning, stacklevel=2
+        )
+    tmp = grey_dilation(
+        input, size, footprint, structure, None, mode, cval, origin
+    )
+    tmp = grey_erosion(
+        tmp, size, footprint, structure, output, mode, cval, origin
+    )
+    if input.dtype == numpy.bool_ and tmp.dtype == numpy.bool_:
+        cupy.bitwise_xor(tmp, input, out=tmp)
+    else:
+        cupy.subtract(tmp, input, out=tmp)
+    return tmp
diff --git a/cupyx/scipy/ndimage/_spline_kernel_weights.py b/cupyx/scipy/ndimage/_spline_kernel_weights.py
new file mode 100644
index 0000000..b2fc844
--- /dev/null
+++ b/cupyx/scipy/ndimage/_spline_kernel_weights.py
@@ -0,0 +1,73 @@
+"""Determination of spline kernel weights (adapted from SciPy)
+
+See more verbose comments for each case there:
+https://github.com/scipy/scipy/blob/eba29d69846ab1299976ff4af71c106188397ccc/scipy/ndimage/src/ni_splines.c#L7   # NOQA
+
+``spline_weights_inline`` is a dict where the key is the spline order and the
+value is the spline weight initialization code.
+"""
+
+spline_weights_inline = {}
+
+# Note: This order = 1 case is currently unused (order = 1 has a different code
+# path in _interp_kernels.py). I think that existing code is a bit more
+# efficient.
+spline_weights_inline[1] = '''
+wx = c_{j} - floor({order} & 1 ? c_{j} : c_{j} + 0.5);
+weights_{j}[0] = 1.0 - wx;
+weights_{j}[1] = wx;
+'''
+
+spline_weights_inline[2] = '''
+wx = c_{j} - floor({order} & 1 ? c_{j} : c_{j} + 0.5);
+weights_{j}[1] = 0.75 - wx * wx;
+wy = 0.5 - wx;
+weights_{j}[0] = 0.5 * wy * wy;
+weights_{j}[2] = 1.0 - weights_{j}[0] - weights_{j}[1];
+'''
+
+spline_weights_inline[3] = '''
+wx = c_{j} - floor({order} & 1 ? c_{j} : c_{j} + 0.5);
+wy = 1.0 - wx;
+weights_{j}[1] = (wx * wx * (wx - 2.0) * 3.0 + 4.0) / 6.0;
+weights_{j}[2] = (wy * wy * (wy - 2.0) * 3.0 + 4.0) / 6.0;
+weights_{j}[0] = wy * wy * wy / 6.0;
+weights_{j}[3] = 1.0 - weights_{j}[0] - weights_{j}[1] - weights_{j}[2];
+'''
+
+spline_weights_inline[4] = '''
+wx = c_{j} - floor({order} & 1 ? c_{j} : c_{j} + 0.5);
+wy = wx * wx;
+weights_{j}[2] = wy * (wy * 0.25 - 0.625) + 115.0 / 192.0;
+wy = 1.0 + wx;
+weights_{j}[1] = wy * (wy * (wy * (5.0 - wy) / 6.0 - 1.25) + 5.0 / 24.0) +
+             55.0 / 96.0;
+wy = 1.0 - wx;
+weights_{j}[3] = wy * (wy * (wy * (5.0 - wy) / 6.0 - 1.25) + 5.0 / 24.0) +
+             55.0 / 96.0;
+wy = 0.5 - wx;
+wy = wy * wy;
+weights_{j}[0] = wy * wy / 24.0;
+weights_{j}[4] = 1.0 - weights_{j}[0] - weights_{j}[1]
+                     - weights_{j}[2] - weights_{j}[3];
+'''
+
+spline_weights_inline[5] = '''
+wx = c_{j} - floor({order} & 1 ? c_{j} : c_{j} + 0.5);
+wy = wx * wx;
+weights_{j}[2] = wy * (wy * (0.25 - wx / 12.0) - 0.5) + 0.55;
+wy = 1.0 - wx;
+wy = wy * wy;
+weights_{j}[3] = wy * (wy * (0.25 - (1.0 - wx) / 12.0) - 0.5) + 0.55;
+wy = wx + 1.0;
+weights_{j}[1] = wy * (wy * (wy * (wy * (wy / 24.0 - 0.375) + 1.25) - 1.75)
+                   + 0.625) + 0.425;
+wy = 2.0 - wx;
+weights_{j}[4] = wy * (wy * (wy * (wy * (wy / 24.0 - 0.375) + 1.25) - 1.75)
+                  + 0.625) + 0.425;
+wy = 1.0 - wx;
+wy = wy * wy;
+weights_{j}[0] = (1.0 - wx) * wy * wy / 120.0;
+weights_{j}[5] = 1.0 - weights_{j}[0] - weights_{j}[1] - weights_{j}[2]
+                     - weights_{j}[3] - weights_{j}[4];
+'''
diff --git a/cupyx/scipy/ndimage/_spline_prefilter_core.py b/cupyx/scipy/ndimage/_spline_prefilter_core.py
new file mode 100644
index 0000000..29403b0
--- /dev/null
+++ b/cupyx/scipy/ndimage/_spline_prefilter_core.py
@@ -0,0 +1,261 @@
+"""
+Spline poles and boundary handling implemented as in SciPy
+
+https://github.com/scipy/scipy/blob/master/scipy/ndimage/src/ni_splines.c
+"""
+import functools
+import math
+import operator
+import textwrap
+
+import cupy
+
+
+def get_poles(order):
+    if order == 2:
+        # sqrt(8.0) - 3.0
+        return (-0.171572875253809902396622551580603843,)
+    elif order == 3:
+        # sqrt(3.0) - 2.0
+        return (-0.267949192431122706472553658494127633,)
+    elif order == 4:
+        # sqrt(664.0 - sqrt(438976.0)) + sqrt(304.0) - 19.0
+        # sqrt(664.0 + sqrt(438976.0)) - sqrt(304.0) - 19.0
+        return (-0.361341225900220177092212841325675255,
+                -0.013725429297339121360331226939128204)
+    elif order == 5:
+        # sqrt(67.5 - sqrt(4436.25)) + sqrt(26.25) - 6.5
+        # sqrt(67.5 + sqrt(4436.25)) - sqrt(26.25) - 6.5
+        return (-0.430575347099973791851434783493520110,
+                -0.043096288203264653822712376822550182)
+    else:
+        raise ValueError('only order 2-5 supported')
+
+
+def get_gain(poles):
+    return functools.reduce(operator.mul,
+                            [(1.0 - z) * (1.0 - 1.0 / z) for z in poles])
+
+
+def _causal_init_code(mode):
+    """Code for causal initialization step of IIR filtering.
+
+    c is a 1d array of length n and z is a filter pole
+    """
+    code = f'''
+        // causal init for mode={mode}'''
+    if mode == 'mirror':
+        code += '''
+        z_i = z;
+        z_n_1 = pow(z, (P)(n - 1));
+
+        c[0] = c[0] + z_n_1 * c[(n - 1) * element_stride];
+        for (i = 1; i < min(n - 1, static_cast<idx_t>({n_boundary})); ++i) {{
+            c[0] += z_i * (c[i * element_stride] +
+                           z_n_1 * c[(n - 1 - i) * element_stride]);
+            z_i *= z;
+        }}
+        c[0] /= 1 - z_n_1 * z_n_1;'''
+    elif mode == 'grid-wrap':
+        code += '''
+        z_i = z;
+
+        for (i = 1; i < min(n, static_cast<idx_t>({n_boundary})); ++i) {{
+            c[0] += z_i * c[(n - i) * element_stride];
+            z_i *= z;
+        }}
+        c[0] /= 1 - z_i; /* z_i = pow(z, n) */'''
+    elif mode == 'reflect':
+        code += '''
+        z_i = z;
+        z_n = pow(z, (P)n);
+        c0 = c[0];
+
+        c[0] = c[0] + z_n * c[(n - 1) * element_stride];
+        for (i = 1; i < min(n, static_cast<idx_t>({n_boundary})); ++i) {{
+            c[0] += z_i * (c[i * element_stride] +
+                           z_n * c[(n - 1 - i) * element_stride]);
+            z_i *= z;
+        }}
+        c[0] *= z / (1 - z_n * z_n);
+        c[0] += c0;'''
+    else:
+        raise ValueError('invalid mode: {}'.format(mode))
+    return code
+
+
+def _anticausal_init_code(mode):
+    """Code for the anti-causal initialization step of IIR filtering.
+
+    c is a 1d array of length n and z is a filter pole
+    """
+    code = f'''
+        // anti-causal init for mode={mode}'''
+    if mode == 'mirror':
+        code += '''
+        c[(n - 1) * element_stride] = (
+            z * c[(n - 2) * element_stride] +
+            c[(n - 1) * element_stride]) * z / (z * z - 1);'''
+    elif mode == 'grid-wrap':
+        code += '''
+        z_i = z;
+
+        for (i = 0; i < min(n - 1, static_cast<idx_t>({n_boundary})); ++i) {{
+            c[(n - 1) * element_stride] += z_i * c[i * element_stride];
+            z_i *= z;
+        }}
+        c[(n - 1) * element_stride] *= z / (z_i - 1); /* z_i = pow(z, n) */'''
+    elif mode == 'reflect':
+        code += '''
+        c[(n - 1) * element_stride] *= z / (z - 1);'''
+    else:
+        raise ValueError('invalid mode: {}'.format(mode))
+    return code
+
+
+def _get_spline_mode(mode):
+    """spline boundary mode for interpolation with order >= 2."""
+    if mode in ['mirror', 'reflect', 'grid-wrap']:
+        # exact analytic boundary conditions exist for these modes.
+        return mode
+    elif mode == 'grid-mirror':
+        # grid-mirror is a synonym for 'reflect'
+        return 'reflect'
+    # No exact analytical spline boundary condition implemented. Reflect gives
+    # lower error than using mirror or wrap for mode 'nearest'. Otherwise, a
+    # mirror spline boundary condition is used.
+    return 'reflect' if mode == 'nearest' else 'mirror'
+
+
+def _get_spline1d_code(mode, poles, n_boundary):
+    """Generates the code required for IIR filtering of a single 1d signal.
+
+    Prefiltering is done by causal filtering followed by anti-causal filtering.
+    Multiple boundary conditions have been implemented.
+    """
+    code = ['''
+    __device__ void spline_prefilter1d(
+        T* __restrict__ c, idx_t signal_length, idx_t element_stride)
+    {{''']
+
+    # variables common to all boundary modes
+    code.append('''
+        idx_t i, n = signal_length;
+        P z, z_i;''')
+
+    # retrieve the spline boundary extension mode to use
+    mode = _get_spline_mode(mode)
+
+    if mode == 'mirror':
+        # variables specific to mirror boundary mode
+        code.append('''
+        P z_n_1;''')
+    elif mode == 'reflect':
+        # variables specific to reflect boundary mode
+        code.append('''
+        P z_n;
+        T c0;''')
+
+    for pole in poles:
+
+        code.append(f'''
+        // select the current pole
+        z = {pole};''')
+
+        # initialize and apply the causal filter
+        code.append(_causal_init_code(mode))
+        code.append('''
+        // apply the causal filter for the current pole
+        for (i = 1; i < n; ++i) {{
+            c[i * element_stride] += z * c[(i - 1) * element_stride];
+        }}''')
+        code.append('''
+        #ifdef __HIP_DEVICE_COMPILE__
+        __syncthreads();
+        #endif
+        ''')
+        # initialize and apply the anti-causal filter
+        code.append(_anticausal_init_code(mode))
+        code.append('''
+        // apply the anti-causal filter for the current pole
+        for (i = n - 2; i >= 0; --i) {{
+            c[i * element_stride] = z * (c[(i + 1) * element_stride] -
+                                         c[i * element_stride]);
+        }}''')
+
+    code += ['''
+    }}''']
+    return textwrap.dedent('\n'.join(code)).format(n_boundary=n_boundary)
+
+
+_FILTER_GENERAL = '''
+#include "cupy/carray.cuh"
+#include "cupy/complex.cuh"
+typedef {data_type} T;
+typedef {pole_type} P;
+typedef {index_type} idx_t;
+template <typename T>
+__device__ T* row(
+        T* ptr, idx_t i, idx_t axis, idx_t ndim, const idx_t* shape) {{
+    idx_t index = 0, stride = 1;
+    for (idx_t a = ndim - 1; a > 0; --a) {{
+        if (a != axis) {{
+            index += (i % shape[a]) * stride;
+            i /= shape[a];
+        }}
+        stride *= shape[a];
+    }}
+    return ptr + index + stride * i;
+}}
+'''
+
+
+_batch_spline1d_strided_template = """
+extern "C" __global__
+__launch_bounds__({block_size})
+void {kernel_name}(T* __restrict__ y, const idx_t* __restrict__ info) {{
+    const idx_t n_signals = info[0], n_samples = info[1],
+        * __restrict__ shape = info+2;
+    idx_t y_elem_stride = 1;
+    for (int a = {ndim} - 1; a > {axis}; --a) {{ y_elem_stride *= shape[a]; }}
+    idx_t unraveled_idx = blockDim.x * blockIdx.x + threadIdx.x;
+    idx_t batch_idx = unraveled_idx;
+    if (batch_idx < n_signals)
+    {{
+        T* __restrict__ y_i = row(y, batch_idx, {axis}, {ndim}, shape);
+        spline_prefilter1d(y_i, n_samples, y_elem_stride);
+    }}
+}}
+"""
+
+
+@cupy.memoize(for_each_device=True)
+def get_raw_spline1d_kernel(axis, ndim, mode, order, index_type='int',
+                            data_type='double', pole_type='double',
+                            block_size=128):
+    """Generate a kernel for applying a spline prefilter along a given axis."""
+    poles = get_poles(order)
+
+    # determine number of samples for the boundary approximation
+    # (SciPy uses n_boundary = n_samples but this is excessive)
+    largest_pole = max([abs(p) for p in poles])
+    # tol < 1e-7 fails test cases comparing to SciPy at atol = rtol = 1e-5
+    tol = 1e-10 if pole_type == 'float' else 1e-18
+    n_boundary = math.ceil(math.log(tol, largest_pole))
+
+    # headers and general utility function for extracting rows of data
+    code = _FILTER_GENERAL.format(index_type=index_type,
+                                  data_type=data_type,
+                                  pole_type=pole_type)
+
+    # generate source for a 1d function for a given boundary mode and poles
+    code += _get_spline1d_code(mode, poles, n_boundary)
+
+    # generate code handling batch operation of the 1d filter
+    mode_str = mode.replace('-', '_')  # cannot have '-' in kernel name
+    kernel_name = (f'cupyx_scipy_ndimage_spline_filter_{ndim}d_ord{order}_'
+                   f'axis{axis}_{mode_str}')
+    code += _batch_spline1d_strided_template.format(ndim=ndim, axis=axis,
+                                                    block_size=block_size,
+                                                    kernel_name=kernel_name)
+    return cupy.RawKernel(code, kernel_name)
diff --git a/cupyx/scipy/ndimage/_util.py b/cupyx/scipy/ndimage/_util.py
new file mode 100644
index 0000000..aa7a557
--- /dev/null
+++ b/cupyx/scipy/ndimage/_util.py
@@ -0,0 +1,162 @@
+import warnings
+
+import numpy
+
+import cupy
+
+
+def _is_integer_output(output, input):
+    if output is None:
+        return input.dtype.kind in 'iu'
+    elif isinstance(output, cupy.ndarray):
+        return output.dtype.kind in 'iu'
+    return cupy.dtype(output).kind in 'iu'
+
+
+def _check_cval(mode, cval, integer_output):
+    if mode == 'constant' and integer_output and not cupy.isfinite(cval):
+        raise NotImplementedError("Non-finite cval is not supported for "
+                                  "outputs with integer dtype.")
+
+
+def _init_weights_dtype(input):
+    """Initialize filter weights based on the input array.
+
+    This helper is only used during initialization of some internal filters
+    like prewitt and sobel to avoid costly double-precision computation.
+    """
+    if input.dtype.kind == "c":
+        return cupy.promote_types(input.real.dtype, cupy.complex64)
+    return cupy.promote_types(input.real.dtype, cupy.float32)
+
+
+def _get_weights_dtype(input, weights):
+    if weights.dtype.kind == "c" or input.dtype.kind == "c":
+        return cupy.promote_types(input.real.dtype, cupy.complex64)
+    elif weights.dtype.kind in 'iub':
+        # convert integer dtype weights to double as in SciPy
+        return cupy.float64
+    return cupy.promote_types(input.real.dtype, cupy.float32)
+
+
+def _get_output(output, input, shape=None, complex_output=False):
+    shape = input.shape if shape is None else shape
+    if output is None:
+        if complex_output:
+            _dtype = cupy.promote_types(input.dtype, cupy.complex64)
+        else:
+            _dtype = input.dtype
+        output = cupy.empty(shape, dtype=_dtype)
+    elif isinstance(output, (type, cupy.dtype)):
+        if complex_output and cupy.dtype(output).kind != 'c':
+            warnings.warn("promoting specified output dtype to complex")
+            output = cupy.promote_types(output, cupy.complex64)
+        output = cupy.empty(shape, dtype=output)
+    elif isinstance(output, str):
+        output = numpy.sctypeDict[output]
+        if complex_output and cupy.dtype(output).kind != 'c':
+            raise RuntimeError("output must have complex dtype")
+        output = cupy.empty(shape, dtype=output)
+    elif output.shape != shape:
+        raise RuntimeError("output shape not correct")
+    elif complex_output and output.dtype.kind != 'c':
+        raise RuntimeError("output must have complex dtype")
+    return output
+
+
+def _fix_sequence_arg(arg, ndim, name, conv=lambda x: x):
+    if isinstance(arg, str):
+        return [conv(arg)] * ndim
+    try:
+        arg = iter(arg)
+    except TypeError:
+        return [conv(arg)] * ndim
+    lst = [conv(x) for x in arg]
+    if len(lst) != ndim:
+        msg = "{} must have length equal to input rank".format(name)
+        raise RuntimeError(msg)
+    return lst
+
+
+def _check_origin(origin, width):
+    origin = int(origin)
+    if (width // 2 + origin < 0) or (width // 2 + origin >= width):
+        raise ValueError('invalid origin')
+    return origin
+
+
+def _check_mode(mode):
+    if mode not in ('reflect', 'constant', 'nearest', 'mirror', 'wrap',
+                    'grid-mirror', 'grid-wrap', 'grid-reflect'):
+        msg = f'boundary mode not supported (actual: {mode})'
+        raise RuntimeError(msg)
+    return mode
+
+
+def _get_inttype(input):
+    # The integer type to use for indices in the input array
+    # The indices actually use byte positions and we can't just use
+    # input.nbytes since that won't tell us the number of bytes between the
+    # first and last elements when the array is non-contiguous
+    nbytes = sum((x-1)*abs(stride) for x, stride in
+                 zip(input.shape, input.strides)) + input.dtype.itemsize
+    return 'int' if nbytes < (1 << 31) else 'ptrdiff_t'
+
+
+def _generate_boundary_condition_ops(mode, ix, xsize, int_t="int",
+                                     float_ix=False):
+    min_func = "fmin" if float_ix else "min"
+    max_func = "fmax" if float_ix else "max"
+    if mode in ['reflect', 'grid-mirror']:
+        ops = '''
+        if ({ix} < 0) {{
+            {ix} = - 1 -{ix};
+        }}
+        {ix} %= {xsize} * 2;
+        {ix} = {min}({ix}, 2 * {xsize} - 1 - {ix});'''.format(
+            ix=ix, xsize=xsize, min=min_func)
+    elif mode == 'mirror':
+        ops = '''
+        if ({xsize} == 1) {{
+            {ix} = 0;
+        }} else {{
+            if ({ix} < 0) {{
+                {ix} = -{ix};
+            }}
+            {ix} = 1 + ({ix} - 1) % (({xsize} - 1) * 2);
+            {ix} = {min}({ix}, 2 * {xsize} - 2 - {ix});
+        }}'''.format(ix=ix, xsize=xsize, min=min_func)
+    elif mode == 'nearest':
+        ops = '''
+        {ix} = {min}({max}(({T}){ix}, ({T})0), ({T})({xsize} - 1));'''.format(
+            ix=ix, xsize=xsize, min=min_func, max=max_func,
+            # force using 64-bit signed integer for ptrdiff_t,
+            # see cupy/cupy#6048
+            T=('int' if int_t == 'int' else 'long long'))
+    elif mode == 'grid-wrap':
+        ops = '''
+        {ix} %= {xsize};
+        while ({ix} < 0) {{
+            {ix} += {xsize};
+        }}'''.format(ix=ix, xsize=xsize)
+    elif mode == 'wrap':
+        ops = '''
+        if ({ix} < 0) {{
+            {ix} += ({sz} - 1) * (({int_t})(-{ix} / ({sz} - 1)) + 1);
+        }} else if ({ix} > ({sz} - 1)) {{
+            {ix} -= ({sz} - 1) * ({int_t})({ix} / ({sz} - 1));
+        }};'''.format(ix=ix, sz=xsize, int_t=int_t)
+    elif mode in ['constant', 'grid-constant']:
+        ops = '''
+        if (({ix} < 0) || {ix} >= {xsize}) {{
+            {ix} = -1;
+        }}'''.format(ix=ix, xsize=xsize)
+    return ops
+
+
+def _generate_indices_ops(ndim, int_type, offsets):
+    code = '{type} ind_{j} = _i % ysize_{j} - {offset}; _i /= ysize_{j};'
+    body = [code.format(type=int_type, j=j, offset=offsets[j])
+            for j in range(ndim-1, 0, -1)]
+    return '{type} _i = i;\n{body}\n{type} ind_0 = _i - {offset};'.format(
+        type=int_type, body='\n'.join(body), offset=offsets[0])
diff --git a/cupyx/scipy/signal/__init__.py b/cupyx/scipy/signal/__init__.py
new file mode 100644
index 0000000..9e57be7
--- /dev/null
+++ b/cupyx/scipy/signal/__init__.py
@@ -0,0 +1,13 @@
+from cupyx.scipy.signal._signaltools import convolve  # NOQA
+from cupyx.scipy.signal._signaltools import correlate  # NOQA
+from cupyx.scipy.signal._signaltools import fftconvolve  # NOQA
+from cupyx.scipy.signal._signaltools import choose_conv_method  # NOQA
+from cupyx.scipy.signal._signaltools import oaconvolve  # NOQA
+from cupyx.scipy.signal._signaltools import convolve2d  # NOQA
+from cupyx.scipy.signal._signaltools import correlate2d  # NOQA
+from cupyx.scipy.signal._signaltools import wiener  # NOQA
+from cupyx.scipy.signal._signaltools import order_filter  # NOQA
+from cupyx.scipy.signal._signaltools import medfilt  # NOQA
+from cupyx.scipy.signal._signaltools import medfilt2d  # NOQA
+
+from cupyx.scipy.signal._bsplines import sepfir2d  # NOQA
diff --git a/cupyx/scipy/signal/_bsplines.py b/cupyx/scipy/signal/_bsplines.py
new file mode 100644
index 0000000..4093841
--- /dev/null
+++ b/cupyx/scipy/signal/_bsplines.py
@@ -0,0 +1,39 @@
+import cupy
+import cupyx.scipy.ndimage
+
+
+def sepfir2d(input, hrow, hcol):
+    """Convolve with a 2-D separable FIR filter.
+
+    Convolve the rank-2 input array with the separable filter defined by the
+    rank-1 arrays hrow, and hcol. Mirror symmetric boundary conditions are
+    assumed. This function can be used to find an image given its B-spline
+    representation.
+
+    The arguments `hrow` and `hcol` must be 1-dimensional and of off length.
+
+    Args:
+        input (cupy.ndarray): The input signal
+        hrow (cupy.ndarray): Row direction filter
+        hcol (cupy.ndarray): Column direction filter
+
+    Returns:
+        cupy.ndarray: The filtered signal
+
+    .. seealso:: :func:`scipy.signal.sepfir2d`
+    """
+    if any(x.ndim != 1 or x.size % 2 == 0 for x in (hrow, hcol)):
+        raise ValueError('hrow and hcol must be 1 dimensional and odd length')
+    dtype = input.dtype
+    if dtype.kind == 'c':
+        dtype = cupy.complex64 if dtype == cupy.complex64 else cupy.complex128
+    elif dtype == cupy.float32 or dtype.itemsize <= 2:
+        dtype = cupy.float32
+    else:
+        dtype = cupy.float64
+    input = input.astype(dtype, copy=False)
+    hrow = hrow.astype(dtype, copy=False)
+    hcol = hcol.astype(dtype, copy=False)
+    filters = (hcol[::-1].conj(), hrow[::-1].conj())
+    return cupyx.scipy.ndimage._filters._run_1d_correlates(
+        input, (0, 1), lambda i: filters[i], None, 'reflect', 0)
diff --git a/cupyx/scipy/signal/_signaltools.py b/cupyx/scipy/signal/_signaltools.py
new file mode 100644
index 0000000..b0833d6
--- /dev/null
+++ b/cupyx/scipy/signal/_signaltools.py
@@ -0,0 +1,577 @@
+import warnings
+
+import cupy
+from cupy._core import internal
+
+from cupyx.scipy.ndimage import _util
+from cupyx.scipy.ndimage import _filters
+from cupyx.scipy.signal import _signaltools_core as _st_core
+
+
+def convolve(in1, in2, mode='full', method='auto'):
+    """Convolve two N-dimensional arrays.
+
+    Convolve ``in1`` and ``in2``, with the output size determined by the
+    ``mode`` argument.
+
+    Args:
+        in1 (cupy.ndarray): First input.
+        in2 (cupy.ndarray): Second input. Should have the same number of
+            dimensions as `in1`.
+        mode (str): Indicates the size of the output:
+
+            - ``'full'``: output is the full discrete linear convolution \
+                (default)
+            - ``'valid'``: output consists only of those elements that do \
+                not rely on the zero-padding. Either ``in1`` or ``in2`` must \
+                be at least as large as the other in every dimension.
+            - ``'same'``: - output is the same size as ``in1``, centered with \
+                respect to the ``'full'`` output
+
+        method (str): Indicates which method to use for the computations:
+
+            - ``'direct'``: The convolution is determined directly from sums, \
+                the definition of convolution
+            - ``'fft'``: The Fourier Transform is used to perform the \
+                convolution by calling ``fftconvolve``.
+            - ``'auto'``: Automatically choose direct of FFT based on an \
+                estimate of which is faster for the arguments (default).
+
+    Returns:
+        cupy.ndarray: the result of convolution.
+
+    .. seealso:: :func:`cupyx.scipy.signal.choose_conv_method`
+    .. seealso:: :func:`cupyx.scipy.signal.correlation`
+    .. seealso:: :func:`cupyx.scipy.signal.fftconvolve`
+    .. seealso:: :func:`cupyx.scipy.signal.oaconvolve`
+    .. seealso:: :func:`cupyx.scipy.ndimage.convolve`
+    .. seealso:: :func:`scipy.signal.convolve`
+    .. note::
+        By default, ``convolve`` and ``correlate`` use ``method='auto'``, which
+        calls ``choose_conv_method`` to choose the fastest method using
+        pre-computed values. CuPy may not choose the same method to compute
+        the convolution as SciPy does given the same inputs.
+    """
+    return _correlate(in1, in2, mode, method, True)
+
+
+def correlate(in1, in2, mode='full', method='auto'):
+    """Cross-correlate two N-dimensional arrays.
+
+    Cross-correlate ``in1`` and ``in2``, with the output size determined by the
+    ``mode`` argument.
+
+    Args:
+        in1 (cupy.ndarray): First input.
+        in2 (cupy.ndarray): Second input. Should have the same number of
+            dimensions as ``in1``.
+        mode (str): Indicates the size of the output:
+
+            - ``'full'``: output is the full discrete linear convolution \
+                (default)
+            - ``'valid'``: output consists only of those elements that do \
+                not rely on the zero-padding. Either ``in1`` or ``in2`` must \
+                be at least as large as the other in every dimension.
+            - ``'same'``: - output is the same size as ``in1``, centered with \
+                respect to the ``'full'`` output
+
+        method (str): Indicates which method to use for the computations:
+
+            - ``'direct'``: The convolution is determined directly from sums, \
+                the definition of convolution
+            - ``'fft'``: The Fourier Transform is used to perform the \
+                convolution by calling ``fftconvolve``.
+            - ``'auto'``: Automatically choose direct of FFT based on an \
+                estimate of which is faster for the arguments (default).
+
+    Returns:
+        cupy.ndarray: the result of correlation.
+
+    .. seealso:: :func:`cupyx.scipy.signal.choose_conv_method`
+    .. seealso:: :func:`cupyx.scipy.signal.convolve`
+    .. seealso:: :func:`cupyx.scipy.signal.fftconvolve`
+    .. seealso:: :func:`cupyx.scipy.signal.oaconvolve`
+    .. seealso:: :func:`cupyx.scipy.ndimage.correlation`
+    .. seealso:: :func:`scipy.signal.correlation`
+    .. note::
+        By default, ``convolve`` and ``correlate`` use ``method='auto'``, which
+        calls ``choose_conv_method`` to choose the fastest method using
+        pre-computed values. CuPy may not choose the same method to compute
+        the convolution as SciPy does given the same inputs.
+    """
+    return _correlate(in1, in2, mode, method, False)
+
+
+def _correlate(in1, in2, mode='full', method='auto', convolution=False):
+    quick_out = _st_core._check_conv_inputs(in1, in2, mode, convolution)
+    if quick_out is not None:
+        return quick_out
+    if method not in ('auto', 'direct', 'fft'):
+        raise ValueError('acceptable methods are "auto", "direct", or "fft"')
+
+    if method == 'auto':
+        method = choose_conv_method(in1, in2, mode=mode)
+
+    if method == 'direct':
+        return _st_core._direct_correlate(in1, in2, mode, in1.dtype,
+                                          convolution)
+
+    # if method == 'fft':
+    if not convolution:
+        in2 = _st_core._reverse(in2).conj()
+    inputs_swapped = _st_core._inputs_swap_needed(mode, in1.shape, in2.shape)
+    if inputs_swapped:
+        in1, in2 = in2, in1
+    out = fftconvolve(in1, in2, mode)
+    result_type = cupy.result_type(in1, in2)
+    if result_type.kind in 'ui':
+        out = out.round()
+    out = out.astype(result_type, copy=False)
+    return out
+
+
+def fftconvolve(in1, in2, mode='full', axes=None):
+    """Convolve two N-dimensional arrays using FFT.
+
+    Convolve ``in1`` and ``in2`` using the fast Fourier transform method, with
+    the output size determined by the ``mode`` argument.
+
+    This is generally much faster than the ``'direct'`` method of ``convolve``
+    for large arrays, but can be slower when only a few output values are
+    needed, and can only output float arrays (int or object array inputs will
+    be cast to float).
+
+    Args:
+        in1 (cupy.ndarray): First input.
+        in2 (cupy.ndarray): Second input. Should have the same number of
+            dimensions as ``in1``.
+        mode (str): Indicates the size of the output:
+
+            - ``'full'``: output is the full discrete linear \
+                          cross-correlation (default)
+            - ``'valid'``: output consists only of those elements that do \
+                           not rely on the zero-padding. Either ``in1`` or \
+                           ``in2`` must be at least as large as the other in \
+                           every dimension.
+            - ``'same'``: output is the same size as ``in1``, centered \
+                          with respect to the 'full' output
+
+        axes (scalar or tuple of scalar or None): Axes over which to compute
+            the convolution. The default is over all axes.
+
+    Returns:
+        cupy.ndarray: the result of convolution
+
+    .. seealso:: :func:`cupyx.scipy.signal.choose_conv_method`
+    .. seealso:: :func:`cupyx.scipy.signal.correlation`
+    .. seealso:: :func:`cupyx.scipy.signal.convolve`
+    .. seealso:: :func:`cupyx.scipy.signal.oaconvolve`
+    .. seealso:: :func:`cupyx.scipy.ndimage.convolve`
+    .. seealso:: :func:`scipy.signal.correlation`
+    """
+    out = _st_core._check_conv_inputs(in1, in2, mode)
+    if out is not None:
+        return out
+    in1, in2, axes = _st_core._init_freq_conv_axes(in1, in2, mode, axes, False)
+    shape = [max(x1, x2) if a not in axes else x1 + x2 - 1
+             for a, (x1, x2) in enumerate(zip(in1.shape, in2.shape))]
+    out = _st_core._freq_domain_conv(in1, in2, axes, shape, calc_fast_len=True)
+    return _st_core._apply_conv_mode(out, in1.shape, in2.shape, mode, axes)
+
+
+def choose_conv_method(in1, in2, mode='full'):
+    """Find the fastest convolution/correlation method.
+
+    Args:
+        in1 (cupy.ndarray): first input.
+        in2 (cupy.ndarray): second input.
+        mode (str, optional): ``'valid'``, ``'same'``, ``'full'``.
+
+    Returns:
+        str: A string indicating which convolution method is fastest,
+        either ``'direct'`` or ``'fft'``.
+
+    .. warning::
+        This function currently doesn't support measure option,
+        nor multidimensional inputs. It does not guarantee
+        the compatibility of the return value to SciPy's one.
+
+    .. seealso:: :func:`scipy.signal.choose_conv_method`
+
+    """
+    return cupy._math.misc._choose_conv_method(in1, in2, mode)
+
+
+def oaconvolve(in1, in2, mode="full", axes=None):
+    """Convolve two N-dimensional arrays using the overlap-add method.
+
+    Convolve ``in1`` and ``in2`` using the overlap-add method, with the output
+    size determined by the ``mode`` argument. This is generally faster than
+    ``convolve`` for large arrays, and generally faster than ``fftconvolve``
+    when one array is much larger than the other, but can be slower when only a
+    few output values are needed or when the arrays are very similar in shape,
+    and can only output float arrays (int or object array inputs will be cast
+    to float).
+
+    Args:
+        in1 (cupy.ndarray): First input.
+        in2 (cupy.ndarray): Second input. Should have the same number of
+            dimensions as ``in1``.
+        mode (str): Indicates the size of the output:
+
+            - ``'full'``: output is the full discrete linear \
+                          cross-correlation (default)
+            - ``'valid'``: output consists only of those elements that do \
+                           not rely on the zero-padding. Either ``in1`` or \
+                           ``in2`` must be at least as large as the other in \
+                           every dimension.
+            - ``'same'``: output is the same size as ``in1``, centered \
+                          with respect to the ``'full'`` output
+
+        axes (scalar or tuple of scalar or None): Axes over which to compute
+            the convolution. The default is over all axes.
+
+    Returns:
+        cupy.ndarray: the result of convolution
+
+    .. seealso:: :func:`cupyx.scipy.signal.convolve`
+    .. seealso:: :func:`cupyx.scipy.signal.fftconvolve`
+    .. seealso:: :func:`cupyx.scipy.ndimage.convolve`
+    .. seealso:: :func:`scipy.signal.oaconvolve`
+    """
+    out = _st_core._check_conv_inputs(in1, in2, mode)
+    if out is not None:
+        return out
+    if in1.shape == in2.shape:  # Equivalent to fftconvolve
+        return fftconvolve(in1, in2, mode=mode, axes=axes)
+
+    in1, in2, axes = _st_core._init_freq_conv_axes(in1, in2, mode, axes,
+                                                   sorted_axes=True)
+    s1, s2 = in1.shape, in2.shape
+    if not axes:
+        return _st_core._apply_conv_mode(in1*in2, s1, s2, mode, axes)
+
+    # Calculate the block sizes for the output, steps, first and second inputs.
+    # It is simpler to calculate them all together than doing them in separate
+    # loops due to all the special cases that need to be handled.
+    optimal_sizes = (_st_core._calc_oa_lens(s1[i], s2[i]) if i in axes else
+                     (-1, -1, s1[i], s2[i]) for i in range(in1.ndim))
+    block_size, overlaps, in1_step, in2_step = zip(*optimal_sizes)
+
+    # Fall back to fftconvolve if there is only one block in every dimension
+    if in1_step == s1 and in2_step == s2:
+        return fftconvolve(in1, in2, mode=mode, axes=axes)
+
+    # Pad and reshape the inputs for overlapping and adding
+    shape_final = [s1[i]+s2[i]-1 if i in axes else None
+                   for i in range(in1.ndim)]
+    in1, in2 = _st_core._oa_reshape_inputs(in1, in2, axes, shape_final,
+                                           block_size, overlaps,
+                                           in1_step, in2_step)
+
+    # Reshape the overlap-add parts to input block sizes
+    split_axes = [iax+i for i, iax in enumerate(axes)]
+    fft_axes = [iax+1 for iax in split_axes]
+
+    # Do the convolution
+    fft_shape = [block_size[i] for i in axes]
+    ret = _st_core._freq_domain_conv(in1, in2, fft_axes, fft_shape,
+                                     calc_fast_len=False)
+
+    # Do the overlap-add
+    for ax, ax_fft, ax_split in zip(axes, fft_axes, split_axes):
+        overlap = overlaps[ax]
+        if overlap is None:
+            continue
+
+        ret, overpart = cupy.split(ret, [-overlap], ax_fft)
+        overpart = cupy.split(overpart, [-1], ax_split)[0]
+
+        ret_overpart = cupy.split(ret, [overlap], ax_fft)[0]
+        ret_overpart = cupy.split(ret_overpart, [1], ax_split)[1]
+        ret_overpart += overpart
+
+    # Reshape back to the correct dimensionality
+    shape_ret = [ret.shape[i] if i not in fft_axes else
+                 ret.shape[i]*ret.shape[i-1]
+                 for i in range(ret.ndim) if i not in split_axes]
+    ret = ret.reshape(*shape_ret)
+
+    # Slice to the correct size
+    ret = ret[tuple([slice(islice) for islice in shape_final])]
+
+    return _st_core._apply_conv_mode(ret, s1, s2, mode, axes)
+
+
+def convolve2d(in1, in2, mode='full', boundary='fill', fillvalue=0):
+    """Convolve two 2-dimensional arrays.
+
+    Convolve ``in1`` and ``in2`` with output size determined by ``mode``, and
+    boundary conditions determined by ``boundary`` and ``fillvalue``.
+
+    Args:
+        in1 (cupy.ndarray): First input.
+        in2 (cupy.ndarray): Second input. Should have the same number of
+            dimensions as ``in1``.
+        mode (str): Indicates the size of the output:
+
+            - ``'full'``: output is the full discrete linear convolution \
+                (default)
+            - ``'valid'``: output consists only of those elements that do \
+                not rely on the zero-padding. Either ``in1`` or ``in2`` must \
+                be at least as large as the other in every dimension.
+            - ``'same'``: - output is the same size as ``in1``, centered with \
+                respect to the ``'full'`` output
+
+        boundary (str): Indicates how to handle boundaries:
+
+            - ``fill``: pad input arrays with fillvalue (default)
+            - ``wrap``: circular boundary conditions
+            - ``symm``: symmetrical boundary conditions
+
+        fillvalue (scalar): Value to fill pad input arrays with. Default is 0.
+
+    Returns:
+        cupy.ndarray: A 2-dimensional array containing a subset of the discrete
+        linear convolution of ``in1`` with ``in2``.
+
+    .. seealso:: :func:`cupyx.scipy.signal.convolve`
+    .. seealso:: :func:`cupyx.scipy.signal.fftconvolve`
+    .. seealso:: :func:`cupyx.scipy.signal.oaconvolve`
+    .. seealso:: :func:`cupyx.scipy.signal.correlate2d`
+    .. seealso:: :func:`cupyx.scipy.ndimage.convolve`
+    .. seealso:: :func:`scipy.signal.convolve2d`
+    """
+    return _correlate2d(in1, in2, mode, boundary, fillvalue, True)
+
+
+def correlate2d(in1, in2, mode='full', boundary='fill', fillvalue=0):
+    """Cross-correlate two 2-dimensional arrays.
+
+    Cross correlate ``in1`` and ``in2`` with output size determined by
+    ``mode``, and boundary conditions determined by ``boundary`` and
+    ``fillvalue``.
+
+    Args:
+        in1 (cupy.ndarray): First input.
+        in2 (cupy.ndarray): Second input. Should have the same number of
+            dimensions as ``in1``.
+        mode (str): Indicates the size of the output:
+
+            - ``'full'``: output is the full discrete linear convolution \
+                (default)
+            - ``'valid'``: output consists only of those elements that do \
+                not rely on the zero-padding. Either ``in1`` or ``in2`` must \
+                be at least as large as the other in every dimension.
+            - ``'same'``: - output is the same size as ``in1``, centered with \
+                respect to the ``'full'`` output
+
+        boundary (str): Indicates how to handle boundaries:
+
+            - ``fill``: pad input arrays with fillvalue (default)
+            - ``wrap``: circular boundary conditions
+            - ``symm``: symmetrical boundary conditions
+
+        fillvalue (scalar): Value to fill pad input arrays with. Default is 0.
+
+    Returns:
+        cupy.ndarray: A 2-dimensional array containing a subset of the discrete
+        linear cross-correlation of ``in1`` with ``in2``.
+
+    Note:
+        When using ``"same"`` mode with even-length inputs, the outputs of
+        ``correlate`` and ``correlate2d`` differ: There is a 1-index offset
+        between them.
+
+    .. seealso:: :func:`cupyx.scipy.signal.correlate`
+    .. seealso:: :func:`cupyx.scipy.signal.convolve2d`
+    .. seealso:: :func:`cupyx.scipy.ndimage.correlate`
+    .. seealso:: :func:`scipy.signal.correlate2d`
+    """
+    return _correlate2d(in1, in2, mode, boundary, fillvalue, False)
+
+
+def _correlate2d(in1, in2, mode, boundary, fillvalue, convolution=False):
+    if not (in1.ndim == in2.ndim == 2):
+        raise ValueError('{} inputs must both be 2-D arrays'.format(
+            'convolve2d' if convolution else 'correlate2d'))
+    _boundaries = {
+        'fill': 'constant', 'pad': 'constant',
+        'wrap': 'wrap', 'circular': 'wrap',
+        'symm': 'reflect', 'symmetric': 'reflect',
+    }
+    boundary = _boundaries.get(boundary)
+    if boundary is None:
+        raise ValueError('Acceptable boundary flags are "fill" (or "pad"), '
+                         '"circular" (or "wrap"), and '
+                         '"symmetric" (or "symm").')
+    quick_out = _st_core._check_conv_inputs(in1, in2, mode, convolution)
+    if quick_out is not None:
+        return quick_out
+    return _st_core._direct_correlate(in1, in2, mode, in1.dtype, convolution,
+                                      boundary, fillvalue, not convolution)
+
+
+def wiener(im, mysize=None, noise=None):
+    """Perform a Wiener filter on an N-dimensional array.
+
+    Apply a Wiener filter to the N-dimensional array `im`.
+
+    Args:
+        im (cupy.ndarray): An N-dimensional array.
+        mysize (int or cupy.ndarray, optional): A scalar or an N-length list
+            giving the size of the Wiener filter window in each dimension.
+            Elements of mysize should be odd. If mysize is a scalar, then this
+            scalar is used as the size in each dimension.
+        noise (float, optional): The noise-power to use. If None, then noise is
+            estimated as the average of the local variance of the input.
+
+    Returns:
+        cupy.ndarray: Wiener filtered result with the same shape as `im`.
+
+    .. seealso:: :func:`scipy.signal.wiener`
+    """
+    if mysize is None:
+        mysize = 3
+    mysize = _util._fix_sequence_arg(mysize, im.ndim, 'mysize', int)
+    im = im.astype(cupy.complex128 if im.dtype.kind == 'c' else cupy.float64,
+                   copy=False)
+
+    # Estimate the local mean
+    local_mean = _filters.uniform_filter(im, mysize, mode='constant')
+
+    # Estimate the local variance
+    local_var = _filters.uniform_filter(im*im, mysize, mode='constant')
+    local_var -= local_mean*local_mean
+
+    # Estimate the noise power if needed.
+    if noise is None:
+        noise = local_var.mean()
+
+    # Perform the filtering
+    res = im - local_mean
+    res *= 1 - noise / local_var
+    res += local_mean
+    return cupy.where(local_var < noise, local_mean, res)
+
+
+def order_filter(a, domain, rank):
+    """Perform an order filter on an N-D array.
+
+    Perform an order filter on the array in. The domain argument acts as a mask
+    centered over each pixel. The non-zero elements of domain are used to
+    select elements surrounding each input pixel which are placed in a list.
+    The list is sorted, and the output for that pixel is the element
+    corresponding to rank in the sorted list.
+
+    Args:
+        a (cupy.ndarray): The N-dimensional input array.
+        domain (cupy.ndarray): A mask array with the same number of dimensions
+            as `a`. Each dimension should have an odd number of elements.
+        rank (int): A non-negative integer which selects the element from the
+            sorted list (0 corresponds to the smallest element).
+
+    Returns:
+        cupy.ndarray: The results of the order filter in an array with the same
+        shape as `a`.
+
+    .. seealso:: :func:`cupyx.scipy.ndimage.rank_filter`
+    .. seealso:: :func:`scipy.signal.order_filter`
+    """
+    if a.dtype.kind in 'bc' or a.dtype == cupy.float16:
+        # scipy doesn't support these types
+        raise ValueError("data type not supported")
+    if any(x % 2 != 1 for x in domain.shape):
+        raise ValueError("Each dimension of domain argument "
+                         " should have an odd number of elements.")
+    return _filters.rank_filter(a, rank, footprint=domain, mode='constant')
+
+
+def medfilt(volume, kernel_size=None):
+    """Perform a median filter on an N-dimensional array.
+
+    Apply a median filter to the input array using a local window-size
+    given by `kernel_size`. The array will automatically be zero-padded.
+
+    Args:
+        volume (cupy.ndarray): An N-dimensional input array.
+        kernel_size (int or list of ints): Gives the size of the median filter
+            window in each dimension. Elements of `kernel_size` should be odd.
+            If `kernel_size` is a scalar, then this scalar is used as the size
+            in each dimension. Default size is 3 for each dimension.
+
+    Returns:
+        cupy.ndarray: An array the same size as input containing the median
+        filtered result.
+
+    .. seealso:: :func:`cupyx.scipy.ndimage.median_filter`
+    .. seealso:: :func:`scipy.signal.medfilt`
+    """
+    if volume.dtype.kind == 'c':
+        # scipy doesn't support complex
+        raise ValueError("complex types not supported")
+    if volume.dtype.char == 'e':
+        # scipy doesn't support float16
+        raise ValueError("float16 type not supported")
+    if volume.dtype.kind == 'b':
+        # scipy doesn't support bool
+        raise ValueError("bool type not supported")
+    kernel_size = _get_kernel_size(kernel_size, volume.ndim)
+    if any(k > s for k, s in zip(kernel_size, volume.shape)):
+        warnings.warn('kernel_size exceeds volume extent: '
+                      'volume will be zero-padded')
+
+    size = internal.prod(kernel_size)
+    return _filters.rank_filter(volume, size // 2, size=kernel_size,
+                                mode='constant')
+
+
+def medfilt2d(input, kernel_size=3):
+    """Median filter a 2-dimensional array.
+
+    Apply a median filter to the `input` array using a local window-size given
+    by `kernel_size` (must be odd). The array is zero-padded automatically.
+
+    Args:
+        input (cupy.ndarray): A 2-dimensional input array.
+        kernel_size (int of list of ints of length 2): Gives the size of the
+            median filter window in each dimension. Elements of `kernel_size`
+            should be odd. If `kernel_size` is a scalar, then this scalar is
+            used as the size in each dimension. Default is a kernel of size
+            (3, 3).
+
+    Returns:
+        cupy.ndarray: An array the same size as input containing the median
+        filtered result.
+
+    See also
+    --------
+    .. seealso:: :func:`cupyx.scipy.ndimage.median_filter`
+    .. seealso:: :func:`cupyx.scipy.signal.medfilt`
+    .. seealso:: :func:`scipy.signal.medfilt2d`
+    """
+    if input.dtype.kind == 'c':
+        # scipy doesn't support complex
+        raise ValueError("complex types not supported")
+    if input.dtype.char == 'e':
+        # scipy doesn't support float16
+        raise ValueError("float16 type not supported")
+    if input.dtype.kind == 'b':
+        # scipy doesn't support bool
+        raise ValueError("bool type not supported")
+    if input.ndim != 2:
+        raise ValueError('input must be 2d')
+    kernel_size = _get_kernel_size(kernel_size, input.ndim)
+    order = kernel_size[0] * kernel_size[1] // 2
+    return _filters.rank_filter(
+        input, order, size=kernel_size, mode='constant')
+
+
+def _get_kernel_size(kernel_size, ndim):
+    if kernel_size is None:
+        kernel_size = (3,) * ndim
+    kernel_size = _util._fix_sequence_arg(kernel_size, ndim,
+                                          'kernel_size', int)
+    if any((k % 2) != 1 for k in kernel_size):
+        raise ValueError("Each element of kernel_size should be odd")
+    return kernel_size
diff --git a/cupyx/scipy/signal/_signaltools_core.py b/cupyx/scipy/signal/_signaltools_core.py
new file mode 100644
index 0000000..8fce637
--- /dev/null
+++ b/cupyx/scipy/signal/_signaltools_core.py
@@ -0,0 +1,302 @@
+import math
+
+import cupy
+from cupy._core import internal
+from cupyx.scipy import fft
+from cupyx.scipy.ndimage import _filters
+from cupyx.scipy.ndimage import _util
+
+
+def _check_conv_inputs(in1, in2, mode, convolution=True):
+    if in1.ndim == in2.ndim == 0:
+        return in1 * (in2 if convolution else in2.conj())
+    if in1.ndim != in2.ndim:
+        raise ValueError('in1 and in2 should have the same dimensionality')
+    if in1.size == 0 or in2.size == 0:
+        return cupy.array([], dtype=in1.dtype)
+    if mode not in ('full', 'same', 'valid'):
+        raise ValueError('acceptable modes are "valid", "same", or "full"')
+    return None
+
+
+def _direct_correlate(in1, in2, mode='full', output=float, convolution=False,
+                      boundary='constant', fillvalue=0.0, shift=False):
+    if in1.ndim != 1 and (in1.dtype.kind == 'b' or
+                          (in1.dtype.kind == 'f' and in1.dtype.itemsize < 4)):
+        raise ValueError('unsupported type in SciPy')
+
+    # Swaps inputs so smaller one is in2:
+    # NOTE: when mode != 'valid' we can only swap with a constant-0 boundary
+    swapped_inputs = False
+    orig_in1_shape = in1.shape
+    if _inputs_swap_needed(mode, in1.shape, in2.shape) or (
+            in2.size > in1.size and boundary == 'constant' and fillvalue == 0):
+        in1, in2 = in2, in1
+        swapped_inputs = True
+
+    # Due to several optimizations, the second array can only be 2 GiB
+    if in2.nbytes >= (1 << 31):
+        raise RuntimeError('smaller array must be 2 GiB or less, '
+                           'use method="fft" instead')
+
+    # At this point, in1.size > in2.size
+    # (except some cases when boundary != 'constant' or fillvalue != 0)
+    # Figure out the output shape and the origin of the kernel
+    if mode == 'full':
+        out_shape = tuple(x1+x2-1 for x1, x2 in zip(in1.shape, in2.shape))
+        offsets = tuple(x-1 for x in in2.shape)
+    elif mode == 'valid':
+        out_shape = tuple(x1-x2+1 for x1, x2 in zip(in1.shape, in2.shape))
+        offsets = (0,) * in1.ndim
+    else:  # mode == 'same':
+        # In correlate2d: When using "same" mode with even-length inputs, the
+        # outputs of correlate and correlate2d differ: There is a 1-index
+        # offset between them.
+        # This is dealt with by using "shift" parameter.
+        out_shape = orig_in1_shape
+        if orig_in1_shape == in1.shape:
+            offsets = tuple((x-shift)//2 for x in in2.shape)
+        else:
+            offsets = tuple((2*x2-x1-(not convolution)+shift)//2
+                            for x1, x2 in zip(in1.shape, in2.shape))
+
+    # Check the output
+    # In SciPy, the output dtype is determined by inputs' dtypes
+    out_dtype = cupy.promote_types(in1, in2)
+    if not isinstance(output, cupy.ndarray):
+        if not cupy.can_cast(output, out_dtype):
+            raise ValueError('not available for this type')
+        output = cupy.empty(out_shape, out_dtype)
+    elif output.shape != out_shape:
+        raise ValueError('out has wrong shape')
+    elif output.dtype != out_dtype:
+        raise ValueError('out has wrong dtype')
+
+    # Check input dtypes
+    # Internally, the kernel accumulates in in2's type, so if in2 has lower
+    # precision (can_cast = True!) we hit overflow easier
+    # TODO(leofang): this is a band-aid fix for cupy/cupy#6047
+    if cupy.can_cast(in2, in1):
+        in2 = in2.astype(out_dtype)  # make a copy while upcasting
+
+    # Get and run the CuPy kernel
+    int_type = _util._get_inttype(in1)
+    kernel = _filters._get_correlate_kernel(
+        boundary, in2.shape, int_type, offsets, fillvalue)
+    in2 = _reverse(in2) if convolution else in2.conj()
+    if not swapped_inputs or convolution:
+        kernel(in1, in2, output)
+    elif output.dtype.kind != 'c':
+        # Avoids one array copy
+        kernel(in1, in2, _reverse(output))
+    else:
+        kernel(in1, in2, output)
+        output = cupy.ascontiguousarray(_reverse(output))
+        if swapped_inputs and (mode != 'valid' or not shift):
+            cupy.conjugate(output, out=output)
+    return output
+
+
+def _reverse(x):
+    # Reverse array `x` in all dimensions
+    return x[(slice(None, None, -1),) * x.ndim]
+
+
+def _inputs_swap_needed(mode, shape1, shape2, axes=None):
+    # See scipy's documentation in scipy.signal._signaltools
+    if mode != 'valid' or not shape1:
+        return False
+    if axes is None:
+        axes = tuple(range(len(shape1)))
+    not_ok1 = any(shape1[i] < shape2[i] for i in axes)
+    not_ok2 = any(shape1[i] > shape2[i] for i in axes)
+    if not_ok1 and not_ok2:
+        raise ValueError('For "valid" mode, one must be at least '
+                         'as large as the other in every dimension')
+    return not_ok1
+
+
+def _init_freq_conv_axes(in1, in2, mode, axes, sorted_axes=False):
+    # See scipy's documentation in scipy.signal._signaltools
+    s1, s2 = in1.shape, in2.shape
+    axes = _init_nd_and_axes(in1, axes)
+    # Length-1 axes can rely on broadcasting rules, no fft needed
+    axes = [ax for ax in axes if s1[ax] != 1 and s2[ax] != 1]
+    if sorted_axes:
+        axes.sort()
+
+    # Check that unused axes are either 1 (broadcast) or the same length
+    for ax, (dim1, dim2) in enumerate(zip(s1, s2)):
+        if ax not in axes and dim1 != dim2 and dim1 != 1 and dim2 != 1:
+            raise ValueError('incompatible shapes for in1 and in2:'
+                             ' {} and {}'.format(s1, s2))
+
+    # Check that input sizes are compatible with 'valid' mode.
+    if _inputs_swap_needed(mode, s1, s2, axes=axes):
+        # Convolution is commutative
+        in1, in2 = in2, in1
+
+    return in1, in2, tuple(axes)
+
+
+def _init_nd_and_axes(x, axes):
+    # See documentation in scipy.fft._helper._init_nd_shape_and_axes
+    # except shape argument is always None and doesn't return new shape
+    axes = internal._normalize_axis_indices(axes, x.ndim, sort_axes=False)
+    if not len(axes):
+        raise ValueError('when provided, axes cannot be empty')
+    if any(x.shape[ax] < 1 for ax in axes):
+        raise ValueError('invalid number of data points specified')
+    return axes
+
+
+def _freq_domain_conv(in1, in2, axes, shape, calc_fast_len=False):
+    # See scipy's documentation in scipy.signal._signaltools
+    if not axes:
+        # rfftn/irfftn require an axis or more.
+        return in1 * in2
+
+    real = (in1.dtype.kind != 'c' and in2.dtype.kind != 'c')
+    fshape = ([fft.next_fast_len(shape[a], real) for a in axes]
+              if calc_fast_len else shape)
+    fftn, ifftn = (fft.rfftn, fft.irfftn) if real else (fft.fftn, fft.ifftn)
+
+    # Perform the convolution
+    sp1 = fftn(in1, fshape, axes=axes)
+    sp2 = fftn(in2, fshape, axes=axes)
+    out = ifftn(sp1 * sp2, fshape, axes=axes)
+
+    return out[tuple(slice(x) for x in shape)] if calc_fast_len else out
+
+
+def _apply_conv_mode(full, s1, s2, mode, axes):
+    # See scipy's documentation in scipy.signal._signaltools
+    if mode == 'full':
+        return cupy.ascontiguousarray(full)
+    if mode == 'valid':
+        s1 = [full.shape[a] if a not in axes else s1[a] - s2[a] + 1
+              for a in range(full.ndim)]
+    starts = [(cur-new)//2 for cur, new in zip(full.shape, s1)]
+    slices = tuple(slice(start, start+length)
+                   for start, length in zip(starts, s1))
+    return cupy.ascontiguousarray(full[slices])
+
+
+__EXP_N1 = 0.36787944117144232159553  # exp(-1)
+
+
+def _optimal_oa_block_size(overlap):
+    """
+    Computes the optimal block size for the OA method given the overlap size.
+
+    Computed as ``ceil(-overlap*W(-1/(2*e*overlap)))`` where ``W(z)`` is the
+    Lambert W function solved as per ``scipy.special.lambertw(z, -1)`` with a
+    fixed 4 iterations.
+
+    Returned size should still be given to ``cupyx.scipy.fft.next_fast_len()``.
+    """
+
+    # This function is 10x faster in Cython (but only 1.7us in Python). Can be
+    # easily moved to Cython by:
+    #  * adding `DEF` before `__EXP_N1`
+    #  * changing `import math` to `from libc cimport math`
+    #  * adding `@cython.cdivision(True)` before the function
+    #  * adding `Py_ssize_t` as the type for the `overlap` argument
+    #  * adding a cast `<Py_ssize_t>` or `int(...)` to the return value
+    #  * adding the following type declarations:
+    #      cdef double z, w, ew, wew, wewz
+    #      cdef int i
+
+    # Compute W(-1/(2*e*overlap))
+    z = -__EXP_N1/(2*overlap)  # value to compute for
+    w = -1 - math.log(2*overlap)  # initial guess
+    for i in range(4):
+        ew = math.exp(w)
+        wew = w*ew
+        wewz = wew - z
+        w -= wewz/(wew + ew - (w + 2)*wewz/(2*w + 2))
+    return math.ceil(-overlap*w)
+
+
+def _calc_oa_lens(s1, s2):
+    # See scipy's documentation in scipy.signal._signaltools
+
+    # Set up the arguments for the conventional FFT approach.
+    fallback = (s1+s2-1, None, s1, s2)
+
+    # Use conventional FFT convolve if sizes are same.
+    if s1 == s2 or s1 == 1 or s2 == 1:
+        return fallback
+
+    # Make s1 the larger size
+    swapped = s2 > s1
+    if swapped:
+        s1, s2 = s2, s1
+
+    # There cannot be a useful block size if s2 is more than half of s1.
+    if s2 >= s1//2:
+        return fallback
+
+    # Compute the optimal block size from the overlap
+    overlap = s2-1
+    block_size = fft.next_fast_len(_optimal_oa_block_size(overlap))
+
+    # Use conventional FFT convolve if there is only going to be one block.
+    if block_size >= s1:
+        return fallback
+
+    # Get step size for each of the blocks
+    in1_step, in2_step = block_size-s2+1, s2
+    if swapped:
+        in1_step, in2_step = in2_step, in1_step
+
+    return block_size, overlap, in1_step, in2_step
+
+
+def _oa_reshape_inputs(in1, in2, axes, shape_final,
+                       block_size, overlaps, in1_step, in2_step):
+    # Figure out the number of steps and padding.
+    # This would get too complicated in a list comprehension.
+    nsteps1 = []
+    nsteps2 = []
+    pad_size1 = []
+    pad_size2 = []
+    for i in range(in1.ndim):
+        if i not in axes:
+            pad_size1 += [(0, 0)]
+            pad_size2 += [(0, 0)]
+            continue
+
+        curnstep1, curpad1, curnstep2, curpad2 = 1, 0, 1, 0
+
+        if in1.shape[i] > in1_step[i]:
+            curnstep1 = math.ceil((in1.shape[i]+1)/in1_step[i])
+            if (block_size[i] - overlaps[i])*curnstep1 < shape_final[i]:
+                curnstep1 += 1
+            curpad1 = curnstep1*in1_step[i] - in1.shape[i]
+        if in2.shape[i] > in2_step[i]:
+            curnstep2 = math.ceil((in2.shape[i]+1)/in2_step[i])
+            if (block_size[i] - overlaps[i])*curnstep2 < shape_final[i]:
+                curnstep2 += 1
+            curpad2 = curnstep2*in2_step[i] - in2.shape[i]
+
+        nsteps1 += [curnstep1]
+        nsteps2 += [curnstep2]
+        pad_size1 += [(0, curpad1)]
+        pad_size2 += [(0, curpad2)]
+
+    # Pad array to a size that can be reshaped to desired shape if necessary
+    if not all(curpad == (0, 0) for curpad in pad_size1):
+        in1 = cupy.pad(in1, pad_size1, mode='constant', constant_values=0)
+    if not all(curpad == (0, 0) for curpad in pad_size2):
+        in2 = cupy.pad(in2, pad_size2, mode='constant', constant_values=0)
+
+    # We need to put each new dimension before the corresponding dimension
+    # being reshaped in order to get the data in the right layout at the end.
+    reshape_size1 = list(in1_step)
+    reshape_size2 = list(in2_step)
+    for i, iax in enumerate(axes):
+        reshape_size1.insert(iax+i, nsteps1[i])
+        reshape_size2.insert(iax+i, nsteps2[i])
+
+    return in1.reshape(*reshape_size1), in2.reshape(*reshape_size2)
diff --git a/cupyx/scipy/sparse/__init__.py b/cupyx/scipy/sparse/__init__.py
new file mode 100644
index 0000000..c779ca7
--- /dev/null
+++ b/cupyx/scipy/sparse/__init__.py
@@ -0,0 +1,44 @@
+from cupyx.scipy.sparse._base import issparse  # NOQA
+from cupyx.scipy.sparse._base import isspmatrix  # NOQA
+from cupyx.scipy.sparse._base import spmatrix  # NOQA
+from cupyx.scipy.sparse._base import SparseWarning  # NOQA
+from cupyx.scipy.sparse._base import SparseEfficiencyWarning  # NOQA
+from cupyx.scipy.sparse._coo import coo_matrix  # NOQA
+from cupyx.scipy.sparse._coo import isspmatrix_coo  # NOQA
+from cupyx.scipy.sparse._csc import csc_matrix  # NOQA
+from cupyx.scipy.sparse._csc import isspmatrix_csc  # NOQA
+from cupyx.scipy.sparse._csr import csr_matrix  # NOQA
+from cupyx.scipy.sparse._csr import isspmatrix_csr  # NOQA
+from cupyx.scipy.sparse._dia import dia_matrix  # NOQA
+from cupyx.scipy.sparse._dia import isspmatrix_dia  # NOQA
+
+from cupyx.scipy.sparse._construct import eye  # NOQA
+from cupyx.scipy.sparse._construct import identity  # NOQA
+from cupyx.scipy.sparse._construct import rand  # NOQA
+from cupyx.scipy.sparse._construct import random  # NOQA
+from cupyx.scipy.sparse._construct import spdiags  # NOQA
+from cupyx.scipy.sparse._construct import diags  # NOQA
+
+from cupyx.scipy.sparse._construct import bmat  # NOQA
+from cupyx.scipy.sparse._construct import hstack  # NOQA
+from cupyx.scipy.sparse._construct import vstack  # NOQA
+
+# TODO(unno): implement bsr_matrix
+# TODO(unno): implement dok_matrix
+# TODO(unno): implement lil_matrix
+
+from cupyx.scipy.sparse._construct import kron  # NOQA
+from cupyx.scipy.sparse._construct import kronsum  # NOQA
+# TODO(unno): implement diags
+# TODO(unno): implement block_diag
+
+from cupyx.scipy.sparse._extract import find  # NOQA
+from cupyx.scipy.sparse._extract import tril  # NOQA
+from cupyx.scipy.sparse._extract import triu  # NOQA
+
+# TODO(unno): implement save_npz
+# TODO(unno): implement load_npz
+
+# TODO(unno): implement isspmatrix_bsr(x)
+# TODO(unno): implement isspmatrix_lil(x)
+# TODO(unno): implement isspmatrix_dok(x)
diff --git a/cupyx/scipy/sparse/_base.py b/cupyx/scipy/sparse/_base.py
new file mode 100644
index 0000000..0a9e1c0
--- /dev/null
+++ b/cupyx/scipy/sparse/_base.py
@@ -0,0 +1,582 @@
+import numpy
+
+import cupy
+from cupy import _core
+from cupyx.scipy.sparse import _util
+from cupyx.scipy.sparse import _sputils
+
+
+try:
+    import scipy.sparse as _sparse
+    SparseWarning = _sparse.SparseWarning
+    SparseEfficiencyWarning = _sparse.SparseEfficiencyWarning
+except ImportError:
+    class SparseWarning(Warning):  # type: ignore
+        pass
+
+    class SparseEfficiencyWarning(SparseWarning):  # type: ignore
+        pass
+
+
+class spmatrix(object):
+
+    """Base class of all sparse matrixes.
+
+    See :class:`scipy.sparse.spmatrix`
+    """
+
+    __array_priority__ = 101
+
+    def __init__(self, maxprint=50):
+        if self.__class__ == spmatrix:
+            raise ValueError(
+                'This class is not intended to be instantiated directly.')
+        self.maxprint = maxprint
+
+    @property
+    def device(self):
+        """CUDA device on which this array resides."""
+        raise NotImplementedError
+
+    def get(self, stream=None):
+        """Return a copy of the array on host memory.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+
+        Returns:
+            scipy.sparse.spmatrix: An array on host memory.
+
+        """
+        raise NotImplementedError
+
+    def __len__(self):
+        raise TypeError('sparse matrix length is ambiguous; '
+                        'use getnnz() or shape[0]')
+
+    def __str__(self):
+        # TODO(unno): Do not use get method which is only available when scipy
+        # is installed.
+        return str(self.get())
+
+    def __iter__(self):
+        for r in range(self.shape[0]):
+            yield self[r, :]
+
+    def __bool__(self):
+        if self.shape == (1, 1):
+            return self.nnz != 0
+        else:
+            raise ValueError('The truth value of an array with more than one '
+                             'element is ambiguous. Use a.any() or a.all().')
+
+    __nonzero__ = __bool__
+
+    def __eq__(self, other):
+        return self.tocsr().__eq__(other)
+
+    def __ne__(self, other):
+        return self.tocsr().__ne__(other)
+
+    def __lt__(self, other):
+        return self.tocsr().__lt__(other)
+
+    def __gt__(self, other):
+        return self.tocsr().__gt__(other)
+
+    def __le__(self, other):
+        return self.tocsr().__le__(other)
+
+    def __ge__(self, other):
+        return self.tocsr().__ge__(other)
+
+    def __abs__(self):
+        return self.tocsr().__abs__()
+
+    def __add__(self, other):
+        return self.tocsr().__add__(other)
+
+    def __radd__(self, other):
+        return self.tocsr().__radd__(other)
+
+    def __sub__(self, other):
+        return self.tocsr().__sub__(other)
+
+    def __rsub__(self, other):
+        return self.tocsr().__rsub__(other)
+
+    def __mul__(self, other):
+        return self.tocsr().__mul__(other)
+
+    def __rmul__(self, other):
+        if cupy.isscalar(other) or isdense(other) and other.ndim == 0:
+            return self * other
+        else:
+            try:
+                tr = other.T
+            except AttributeError:
+                return NotImplemented
+            return (self.T * tr).T
+
+    # matmul (@) operator
+    def __matmul__(self, other):
+        if _util.isscalarlike(other):
+            raise ValueError('Scalar operands are not allowed, '
+                             'use \'*\' instead')
+        return self.__mul__(other)
+
+    def __rmatmul__(self, other):
+        if _util.isscalarlike(other):
+            raise ValueError('Scalar operands are not allowed, '
+                             'use \'*\' instead')
+        return self.__rmul__(other)
+
+    def __div__(self, other):
+        return self.tocsr().__div__(other)
+
+    def __rdiv__(self, other):
+        return self.tocsr().__rdiv__(other)
+
+    def __truediv__(self, other):
+        return self.tocsr().__truediv__(other)
+
+    def __rtruediv__(self, other):
+        return self.tocsr().__rdtrueiv__(other)
+
+    def __neg__(self):
+        return -self.tocsr()
+
+    def __iadd__(self, other):
+        return NotImplemented
+
+    def __isub__(self, other):
+        return NotImplemented
+
+    def __imul__(self, other):
+        return NotImplemented
+
+    def __idiv__(self, other):
+        return self.__itruediv__(other)
+
+    def __itruediv__(self, other):
+        return NotImplemented
+
+    def __pow__(self, other):
+        """Calculates n-th power of the matrix.
+
+        This method calculates n-th power of a given matrix. The matrix must
+        be a squared matrix, and a given exponent must be an integer.
+
+        Args:
+            other (int): Exponent.
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix: A sparse matrix representing n-th
+            power of this matrix.
+
+        """
+        m, n = self.shape
+        if m != n:
+            raise TypeError('matrix is not square')
+
+        if _util.isintlike(other):
+            other = int(other)
+            if other < 0:
+                raise ValueError('exponent must be >= 0')
+
+            if other == 0:
+                import cupyx.scipy.sparse
+                return cupyx.scipy.sparse.identity(
+                    m, dtype=self.dtype, format='csr')
+            elif other == 1:
+                return self.copy()
+            else:
+                tmp = self.__pow__(other // 2)
+                if other % 2:
+                    return self * tmp * tmp
+                else:
+                    return tmp * tmp
+        elif _util.isscalarlike(other):
+            raise ValueError('exponent must be an integer')
+        else:
+            return NotImplemented
+
+    @property
+    def A(self):
+        """Dense ndarray representation of this matrix.
+
+        This property is equivalent to
+        :meth:`~cupyx.scipy.sparse.spmatrix.toarray` method.
+
+        """
+        return self.toarray()
+
+    @property
+    def T(self):
+        return self.transpose()
+
+    @property
+    def H(self):
+        return self.getH()
+
+    @property
+    def ndim(self):
+        return 2
+
+    @property
+    def size(self):
+        return self.getnnz()
+
+    @property
+    def nnz(self):
+        return self.getnnz()
+
+    @property
+    def shape(self):
+        return self.get_shape()
+
+    @shape.setter
+    def shape(self, value):
+        self.set_shape(value)
+
+    def asformat(self, format):
+        """Return this matrix in a given sparse format.
+
+        Args:
+            format (str or None): Format you need.
+        """
+        if format is None or format == self.format:
+            return self
+        else:
+            return getattr(self, 'to' + format)()
+
+    def asfptype(self):
+        """Upcasts matrix to a floating point format.
+
+        When the matrix has floating point type, the method returns itself.
+        Otherwise it makes a copy with floating point type and the same format.
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix: A matrix with float type.
+
+        """
+        if self.dtype.kind == 'f':
+            return self
+        else:
+            typ = numpy.promote_types(self.dtype, 'f')
+            return self.astype(typ)
+
+    def astype(self, t):
+        """Casts the array to given data type.
+
+        Args:
+            t: Type specifier.
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix:
+                A copy of the array with the given type and the same format.
+
+        """
+        return self.tocsr().astype(t).asformat(self.format)
+
+    def conj(self, copy=True):
+        """Element-wise complex conjugation.
+
+        If the matrix is of non-complex data type and `copy` is False,
+        this method does nothing and the data is not copied.
+
+        Args:
+            copy (bool):
+                If True, the result is guaranteed to not share data with self.
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix : The element-wise complex conjugate.
+
+        """
+        if self.dtype.kind == 'c':
+            return self.tocsr(copy=copy).conj(copy=False)
+        elif copy:
+            return self.copy()
+        else:
+            return self
+
+    def conjugate(self, copy=True):
+        return self.conj(copy=copy)
+
+    conjugate.__doc__ = conj.__doc__
+
+    def copy(self):
+        """Returns a copy of this matrix.
+
+        No data/indices will be shared between the returned value and current
+        matrix.
+        """
+        return self.__class__(self, copy=True)
+
+    def count_nonzero(self):
+        """Number of non-zero entries, equivalent to"""
+        raise NotImplementedError
+
+    def diagonal(self, k=0):
+        """Returns the k-th diagonal of the matrix.
+
+        Args:
+            k (int, optional): Which diagonal to get, corresponding to elements
+            a[i, i+k]. Default: 0 (the main diagonal).
+
+        Returns:
+            cupy.ndarray : The k-th diagonal.
+        """
+        return self.tocsr().diagonal(k=k)
+
+    def dot(self, other):
+        """Ordinary dot product"""
+        if numpy.isscalar(other):
+            return self * other
+        else:
+            return self @ other
+
+    def getH(self):
+        return self.transpose().conj()
+
+    def get_shape(self):
+        raise NotImplementedError
+
+    # TODO(unno): Implement getcol
+
+    def getformat(self):
+        return self.format
+
+    def getmaxprint(self):
+        return self.maxprint
+
+    def getnnz(self, axis=None):
+        """Number of stored values, including explicit zeros."""
+        raise NotImplementedError
+
+    # TODO(unno): Implement getrow
+
+    def maximum(self, other):
+        return self.tocsr().maximum(other)
+
+    def mean(self, axis=None, dtype=None, out=None):
+        """
+        Compute the arithmetic mean along the specified axis.
+
+        Returns the average of the matrix elements. The average is taken
+        over all elements in the matrix by default, otherwise over the
+        specified axis. `float64` intermediate and return values are used
+        for integer inputs.
+
+        Args:
+            axis {-2, -1, 0, 1, None}: optional
+                Axis along which the mean is computed. The default is to
+                compute the mean of all elements in the matrix
+                (i.e., `axis` = `None`).
+            dtype (dtype): optional
+                Type to use in computing the mean. For integer inputs, the
+                default is `float64`; for floating point inputs, it is the same
+                as the input dtype.
+            out (cupy.ndarray): optional
+                Alternative output matrix in which to place the result. It must
+                have the same shape as the expected output, but the type of the
+                output values will be cast if necessary.
+
+        Returns:
+            m (cupy.ndarray) : Output array of means
+
+        .. seealso::
+            :meth:`scipy.sparse.spmatrix.mean`
+
+        """
+        def _is_integral(dtype):
+            return (cupy.issubdtype(dtype, cupy.integer) or
+                    cupy.issubdtype(dtype, cupy.bool_))
+
+        _sputils.validateaxis(axis)
+
+        res_dtype = self.dtype.type
+        integral = _is_integral(self.dtype)
+
+        # output dtype
+        if dtype is None:
+            if integral:
+                res_dtype = cupy.float64
+        else:
+            res_dtype = cupy.dtype(dtype).type
+
+        # intermediate dtype for summation
+        inter_dtype = cupy.float64 if integral else res_dtype
+        inter_self = self.astype(inter_dtype)
+
+        if axis is None:
+            return (inter_self / cupy.array(
+                self.shape[0] * self.shape[1]))\
+                .sum(dtype=res_dtype, out=out)
+
+        if axis < 0:
+            axis += 2
+
+        # axis = 0 or 1 now
+        if axis == 0:
+            return (inter_self * (1.0 / self.shape[0])).sum(
+                axis=0, dtype=res_dtype, out=out)
+        else:
+            return (inter_self * (1.0 / self.shape[1])).sum(
+                axis=1, dtype=res_dtype, out=out)
+
+    def minimum(self, other):
+        return self.tocsr().minimum(other)
+
+    def multiply(self, other):
+        """Point-wise multiplication by another matrix"""
+        return self.tocsr().multiply(other)
+
+    # TODO(unno): Implement nonzero
+
+    def power(self, n, dtype=None):
+        return self.tocsr().power(n, dtype=dtype)
+
+    def reshape(self, *shape, order='C'):
+        """Gives a new shape to a sparse matrix without changing its data.
+
+        Args:
+            shape (tuple):
+                The new shape should be compatible with the original shape.
+            order: {'C', 'F'} (optional)
+                Read the elements using this index order. 'C' means to read and
+                write the elements using C-like index order. 'F' means to read
+                and write the elements using Fortran-like index order. Default:
+                C.
+
+        Returns:
+            cupyx.scipy.sparse.coo_matrix: sparse matrix
+
+        """
+        shape = _sputils.check_shape(shape, self.shape)
+
+        if shape == self.shape:
+            return self
+
+        return self.tocoo().reshape(shape, order=order)
+
+    def set_shape(self, shape):
+        self.reshape(shape)
+
+    def setdiag(self, values, k=0):
+        """Set diagonal or off-diagonal elements of the array.
+
+        Args:
+            values (cupy.ndarray): New values of the diagonal elements.
+                Values may have any length. If the diagonal is longer than
+                values, then the remaining diagonal entries will not be set.
+                If values is longer than the diagonal, then the remaining
+                values are ignored. If a scalar value is given, all of the
+                diagonal is set to it.
+            k (int, optional): Which diagonal to set, corresponding to elements
+                a[i, i+k]. Default: 0 (the main diagonal).
+        """
+        raise NotImplementedError
+
+    def sum(self, axis=None, dtype=None, out=None):
+        """Sums the matrix elements over a given axis.
+
+        Args:
+            axis (int or ``None``): Axis along which the sum is comuted.
+                If it is ``None``, it computes the sum of all the elements.
+                Select from ``{None, 0, 1, -2, -1}``.
+            dtype: The type of returned matrix. If it is not specified, type
+                of the array is used.
+            out (cupy.ndarray): Output matrix.
+
+        Returns:
+            cupy.ndarray: Summed array.
+
+        .. seealso::
+           :meth:`scipy.sparse.spmatrix.sum`
+
+        """
+        _sputils.validateaxis(axis)
+
+        # This implementation uses multiplication, though it is not efficient
+        # for some matrix types. These should override this function.
+
+        m, n = self.shape
+
+        if axis is None:
+            return self.dot(cupy.ones(n, dtype=self.dtype)).sum(
+                dtype=dtype, out=out)
+
+        if axis < 0:
+            axis += 2
+
+        if axis == 0:
+            ret = self.T.dot(cupy.ones(m, dtype=self.dtype)).reshape(1, n)
+        else:  # axis == 1
+            ret = self.dot(cupy.ones(n, dtype=self.dtype)).reshape(m, 1)
+
+        if out is not None:
+            if out.shape != ret.shape:
+                raise ValueError('dimensions do not match')
+            _core.elementwise_copy(ret, out)
+            return out
+        elif dtype is not None:
+            return ret.astype(dtype, copy=False)
+        else:
+            return ret
+
+    def toarray(self, order=None, out=None):
+        """Return a dense ndarray representation of this matrix."""
+        return self.tocsr().toarray(order=order, out=out)
+
+    def tobsr(self, blocksize=None, copy=False):
+        """Convert this matrix to Block Sparse Row format."""
+        return self.tocsr(copy=copy).tobsr(copy=False)
+
+    def tocoo(self, copy=False):
+        """Convert this matrix to COOrdinate format."""
+        return self.tocsr(copy=copy).tocoo(copy=False)
+
+    def tocsc(self, copy=False):
+        """Convert this matrix to Compressed Sparse Column format."""
+        return self.tocsr(copy=copy).tocsc(copy=False)
+
+    def tocsr(self, copy=False):
+        """Convert this matrix to Compressed Sparse Row format."""
+        raise NotImplementedError
+
+    def todense(self, order=None, out=None):
+        """Return a dense matrix representation of this matrix."""
+        return self.toarray(order=order, out=out)
+
+    def todia(self, copy=False):
+        """Convert this matrix to sparse DIAgonal format."""
+        return self.tocsr(copy=copy).todia(copy=False)
+
+    def todok(self, copy=False):
+        """Convert this matrix to Dictionary Of Keys format."""
+        return self.tocsr(copy=copy).todok(copy=False)
+
+    def tolil(self, copy=False):
+        """Convert this matrix to LInked List format."""
+        return self.tocsr(copy=copy).tolil(copy=False)
+
+    def transpose(self, axes=None, copy=False):
+        """Reverses the dimensions of the sparse matrix."""
+        return self.tocsr(copy=copy).transpose(axes=axes, copy=False)
+
+
+def issparse(x):
+    """Checks if a given matrix is a sparse matrix.
+
+    Returns:
+        bool: Returns if ``x`` is :class:`cupyx.scipy.sparse.spmatrix` that is
+        a base class of all sparse matrix classes.
+
+    """
+    return isinstance(x, spmatrix)
+
+
+isdense = _util.isdense
+isspmatrix = issparse
diff --git a/cupyx/scipy/sparse/_compressed.py b/cupyx/scipy/sparse/_compressed.py
new file mode 100644
index 0000000..402a9c3
--- /dev/null
+++ b/cupyx/scipy/sparse/_compressed.py
@@ -0,0 +1,861 @@
+import string
+import warnings
+
+import numpy
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+import cupy
+import cupyx
+
+from cupy import _core
+from cupy._core import _scalar
+from cupy._creation import basic
+from cupyx import cusparse
+from cupyx.scipy.sparse import _base
+from cupyx.scipy.sparse import _coo
+from cupyx.scipy.sparse import _data as sparse_data
+from cupyx.scipy.sparse import _sputils
+from cupyx.scipy.sparse import _util
+
+from cupyx.scipy.sparse import _index
+
+
+class _compressed_sparse_matrix(sparse_data._data_matrix,
+                                sparse_data._minmax_mixin,
+                                _index.IndexMixin):
+
+    _max_min_reduction_code = r'''
+        extern "C" __global__
+        void ${func}(double* data, int* x, int* y, int length,
+                           double* z) {
+            // Get the index of the block
+            int tid = blockIdx.x * blockDim.x + threadIdx.x;
+
+            // Calculate the block length
+            int block_length = y[tid] - x[tid];
+
+            // Select initial value based on the block density
+            double running_value = 0;
+            if (${cond}){
+                running_value = data[x[tid]];
+            } else {
+                running_value = 0;
+            }
+
+            // Iterate over the block and update
+            for (int entry = x[tid]; entry < y[tid]; entry++){
+                if (data[entry] != data[entry]){
+                    // Check for NaN
+                    running_value = nan("");
+                    break;
+                } else {
+                    // Check for a value update
+                    if (data[entry] ${op} running_value){
+                        running_value = data[entry];
+                    }
+                }
+            }
+
+            // Store in the return function
+            z[tid] = running_value;
+        }'''
+
+    _max_reduction_kern = _core.RawKernel(
+        string.Template(_max_min_reduction_code).substitute(
+            func='max_reduction', op='>', cond='block_length == length'),
+        'max_reduction')
+
+    _max_nonzero_reduction_kern = _core.RawKernel(
+        string.Template(_max_min_reduction_code).substitute(
+            func='max_nonzero_reduction', op='>', cond='block_length > 0'),
+        'max_nonzero_reduction')
+
+    _min_reduction_kern = _core.RawKernel(
+        string.Template(_max_min_reduction_code).substitute(
+            func='min_reduction', op='<', cond='block_length == length'),
+        'min_reduction')
+
+    _min_nonzero_reduction_kern = _core.RawKernel(
+        string.Template(_max_min_reduction_code).substitute(
+            func='min_nonzero_reduction', op='<', cond='block_length > 0'),
+        'min_nonzero_reduction')
+
+    # For _max_arg_reduction_mod and _min_arg_reduction_mod below, we pick
+    # the right template specialization according to input dtypes at runtime.
+    # The distinction in int types (T2) is important for portability in OS.
+
+    _argmax_argmin_code = r'''
+        template<typename T1, typename T2> __global__ void
+        ${func}_arg_reduction(T1* data, int* indices, int* x, int* y,
+                              int length, T2* z) {
+            // Get the index of the block
+            int tid = blockIdx.x * blockDim.x + threadIdx.x;
+
+            // Calculate the block length
+            int block_length = y[tid] - x[tid];
+
+            // Select initial value based on the block density
+            int data_index = 0;
+            double data_value = 0;
+
+            if (block_length == length){
+                // Block is dense. Fill the first value
+                data_value = data[x[tid]];
+                data_index = indices[x[tid]];
+            } else if (block_length > 0)  {
+                // Block has at least one zero. Assign first occurrence as the
+                // starting reference
+                data_value = 0;
+                for (data_index = 0; data_index < length; data_index++){
+                    if (data_index != indices[x[tid] + data_index] ||
+                        x[tid] + data_index >= y[tid]){
+                        break;
+                    }
+                }
+            } else {
+                // Zero valued array
+                data_value = 0;
+                data_index = 0;
+            }
+
+            // Iterate over the section of the sparse matrix
+            for (int entry = x[tid]; entry < y[tid]; entry++){
+                if (data[entry] != data[entry]){
+                    // Check for NaN
+                    data_value = nan("");
+                    data_index = 0;
+                    break;
+                } else {
+                    // Check for a value update
+                    if (data[entry] ${op} data_value){
+                        data_index = indices[entry];
+                        data_value = data[entry];
+                    }
+                }
+            }
+
+            // Store in the return function
+            z[tid] = data_index;
+        }'''
+
+    _max_arg_reduction_mod = _core.RawModule(
+        code=string.Template(_argmax_argmin_code).substitute(
+            func='max', op='>'),
+        options=('-std=c++11',),
+        name_expressions=['max_arg_reduction<float, int>',
+                          'max_arg_reduction<float, long long>',
+                          'max_arg_reduction<double, int>',
+                          'max_arg_reduction<double, long long>'])
+
+    _min_arg_reduction_mod = _core.RawModule(
+        code=string.Template(_argmax_argmin_code).substitute(
+            func='min', op='<'),
+        options=('-std=c++11',),
+        name_expressions=['min_arg_reduction<float, int>',
+                          'min_arg_reduction<float, long long>',
+                          'min_arg_reduction<double, int>',
+                          'min_arg_reduction<double, long long>'])
+
+    # TODO(leofang): rewrite a more load-balanced approach than this naive one?
+    _has_sorted_indices_kern = _core.ElementwiseKernel(
+        'raw T indptr, raw T indices',
+        'bool diff',
+        '''
+        bool diff_out = true;
+        for (T jj = indptr[i]; jj < indptr[i+1] - 1; jj++) {
+            if (indices[jj] > indices[jj+1]){
+                diff_out = false;
+            }
+        }
+        diff = diff_out;
+        ''', 'cupyx_scipy_sparse_has_sorted_indices')
+
+    # TODO(leofang): rewrite a more load-balanced approach than this naive one?
+    _has_canonical_format_kern = _core.ElementwiseKernel(
+        'raw T indptr, raw T indices',
+        'bool diff',
+        '''
+        bool diff_out = true;
+        if (indptr[i] > indptr[i+1]) {
+            diff = false;
+            return;
+        }
+        for (T jj = indptr[i]; jj < indptr[i+1] - 1; jj++) {
+            if (indices[jj] >= indices[jj+1]) {
+                diff_out = false;
+            }
+        }
+        diff = diff_out;
+        ''', 'cupyx_scipy_sparse_has_canonical_format')
+
+    def __init__(self, arg1, shape=None, dtype=None, copy=False):
+        if shape is not None:
+            if not _util.isshape(shape):
+                raise ValueError('invalid shape (must be a 2-tuple of int)')
+            shape = int(shape[0]), int(shape[1])
+
+        if _base.issparse(arg1):
+            x = arg1.asformat(self.format)
+            data = x.data
+            indices = x.indices
+            indptr = x.indptr
+
+            if arg1.format != self.format:
+                # When formats are differnent, all arrays are already copied
+                copy = False
+
+            if shape is None:
+                shape = arg1.shape
+
+        elif _util.isshape(arg1):
+            m, n = arg1
+            m, n = int(m), int(n)
+            data = basic.zeros(0, dtype if dtype else 'd')
+            indices = basic.zeros(0, 'i')
+            indptr = basic.zeros(self._swap(m, n)[0] + 1, dtype='i')
+            # shape and copy argument is ignored
+            shape = (m, n)
+            copy = False
+
+        elif scipy_available and scipy.sparse.issparse(arg1):
+            # Convert scipy.sparse to cupyx.scipy.sparse
+            x = arg1.asformat(self.format)
+            data = cupy.array(x.data)
+            indices = cupy.array(x.indices, dtype='i')
+            indptr = cupy.array(x.indptr, dtype='i')
+            copy = False
+
+            if shape is None:
+                shape = arg1.shape
+
+        elif isinstance(arg1, tuple) and len(arg1) == 2:
+            # Note: This implementation is not efficeint, as it first
+            # constructs a sparse matrix with coo format, then converts it to
+            # compressed format.
+            sp_coo = _coo.coo_matrix(arg1, shape=shape, dtype=dtype, copy=copy)
+            sp_compressed = sp_coo.asformat(self.format)
+            data = sp_compressed.data
+            indices = sp_compressed.indices
+            indptr = sp_compressed.indptr
+
+        elif isinstance(arg1, tuple) and len(arg1) == 3:
+            data, indices, indptr = arg1
+            if not (_base.isdense(data) and data.ndim == 1 and
+                    _base.isdense(indices) and indices.ndim == 1 and
+                    _base.isdense(indptr) and indptr.ndim == 1):
+                raise ValueError(
+                    'data, indices, and indptr should be 1-D')
+
+            if len(data) != len(indices):
+                raise ValueError('indices and data should have the same size')
+
+        elif _base.isdense(arg1):
+            if arg1.ndim > 2:
+                raise TypeError('expected dimension <= 2 array or matrix')
+            elif arg1.ndim == 1:
+                arg1 = arg1[None]
+            elif arg1.ndim == 0:
+                arg1 = arg1[None, None]
+            data, indices, indptr = self._convert_dense(arg1)
+            copy = False
+            if shape is None:
+                shape = arg1.shape
+
+        else:
+            raise ValueError(
+                'Unsupported initializer format')
+
+        if dtype is None:
+            dtype = data.dtype
+        else:
+            dtype = numpy.dtype(dtype)
+
+        if dtype.char not in '?fdFD':
+            raise ValueError(
+                'Only bool, float32, float64, complex64 and complex128 '
+                'are supported')
+
+        data = data.astype(dtype, copy=copy)
+        sparse_data._data_matrix.__init__(self, data)
+
+        self.indices = indices.astype('i', copy=copy)
+        self.indptr = indptr.astype('i', copy=copy)
+
+        if shape is None:
+            shape = self._swap(len(indptr) - 1, int(indices.max()) + 1)
+
+        major, minor = self._swap(*shape)
+        if len(indptr) != major + 1:
+            raise ValueError('index pointer size (%d) should be (%d)'
+                             % (len(indptr), major + 1))
+
+        self._descr = cusparse.MatDescriptor.create()
+        self._shape = shape
+
+    def _with_data(self, data, copy=True):
+        if copy:
+            return self.__class__(
+                (data, self.indices.copy(), self.indptr.copy()),
+                shape=self.shape,
+                dtype=data.dtype)
+        else:
+            return self.__class__(
+                (data, self.indices, self.indptr),
+                shape=self.shape,
+                dtype=data.dtype)
+
+    def _convert_dense(self, x):
+        raise NotImplementedError
+
+    def _swap(self, x, y):
+        raise NotImplementedError
+
+    def _add_sparse(self, other, alpha, beta):
+        raise NotImplementedError
+
+    def _add(self, other, lhs_negative, rhs_negative):
+        if cupy.isscalar(other):
+            if other == 0:
+                if lhs_negative:
+                    return -self
+                else:
+                    return self.copy()
+            else:
+                raise NotImplementedError(
+                    'adding a nonzero scalar to a sparse matrix is not '
+                    'supported')
+        elif _base.isspmatrix(other):
+            alpha = -1 if lhs_negative else 1
+            beta = -1 if rhs_negative else 1
+            return self._add_sparse(other, alpha, beta)
+        elif _base.isdense(other):
+            if lhs_negative:
+                if rhs_negative:
+                    return -self.todense() - other
+                else:
+                    return other - self.todense()
+            else:
+                if rhs_negative:
+                    return self.todense() - other
+                else:
+                    return self.todense() + other
+        else:
+            return NotImplemented
+
+    def __add__(self, other):
+        return self._add(other, False, False)
+
+    def __radd__(self, other):
+        return self._add(other, False, False)
+
+    def __sub__(self, other):
+        return self._add(other, False, True)
+
+    def __rsub__(self, other):
+        return self._add(other, True, False)
+
+    def _get_intXint(self, row, col):
+        major, minor = self._swap(row, col)
+        data, indices, _ = _index._get_csr_submatrix_major_axis(
+            self.data, self.indices, self.indptr, major, major + 1)
+        dtype = data.dtype
+        res = cupy.zeros((), dtype=dtype)
+        if dtype.kind == 'c':
+            _index._compress_getitem_complex_kern(
+                data.real, data.imag, indices, minor, res.real, res.imag)
+        else:
+            _index._compress_getitem_kern(data, indices, minor, res)
+        return res
+
+    def _get_sliceXslice(self, row, col):
+        major, minor = self._swap(row, col)
+        copy = major.step in (1, None)
+        return self._major_slice(major)._minor_slice(minor, copy=copy)
+
+    def _get_arrayXarray(self, row, col, not_found_val=0):
+        # inner indexing
+        idx_dtype = self.indices.dtype
+        M, N = self._swap(*self.shape)
+        major, minor = self._swap(row, col)
+        major = major.astype(idx_dtype, copy=False)
+        minor = minor.astype(idx_dtype, copy=False)
+
+        val = _index._csr_sample_values(
+            M, N, self.indptr, self.indices, self.data,
+            major.ravel(), minor.ravel(),
+            not_found_val)
+
+        if major.ndim == 1:
+            # Scipy returns `matrix` here
+            return cupy.expand_dims(val, 0)
+        return self.__class__(val.reshape(major.shape))
+
+    def _get_columnXarray(self, row, col):
+        # outer indexing
+        major, minor = self._swap(row, col)
+        return self._major_index_fancy(major)._minor_index_fancy(minor)
+
+    def _major_index_fancy(self, idx):
+        """Index along the major axis where idx is an array of ints.
+        """
+        _, N = self._swap(*self.shape)
+        M = idx.size
+        new_shape = self._swap(M, N)
+        if self.nnz == 0 or M == 0:
+            return self.__class__(new_shape, dtype=self.dtype)
+
+        return self.__class__(
+            _index._csr_row_index(self.data, self.indices, self.indptr, idx),
+            shape=new_shape, copy=False)
+
+    def _minor_index_fancy(self, idx):
+        """Index along the minor axis where idx is an array of ints.
+        """
+        M, _ = self._swap(*self.shape)
+        N = idx.size
+        new_shape = self._swap(M, N)
+        if self.nnz == 0 or N == 0:
+            return self.__class__(new_shape, dtype=self.dtype)
+
+        if idx.size * M < self.nnz:
+            # TODO (asi1024): Implement faster algorithm.
+            pass
+
+        return self._tocsx()._major_index_fancy(idx)._tocsx()
+
+    def _major_slice(self, idx, copy=False):
+        """Index along the major axis where idx is a slice object.
+        """
+        M, N = self._swap(*self.shape)
+        start, stop, step = idx.indices(M)
+
+        if start == 0 and stop == M and step == 1:
+            return self.copy() if copy else self
+
+        M = len(range(start, stop, step))
+        new_shape = self._swap(M, N)
+
+        if step == 1:
+            if M == 0 or self.nnz == 0:
+                return self.__class__(new_shape, dtype=self.dtype)
+            return self.__class__(
+                _index._get_csr_submatrix_major_axis(
+                    self.data, self.indices, self.indptr, start, stop),
+                shape=new_shape, copy=copy)
+        rows = cupy.arange(start, stop, step, dtype=self.indptr.dtype)
+        return self._major_index_fancy(rows)
+
+    def _minor_slice(self, idx, copy=False):
+        """Index along the minor axis where idx is a slice object.
+        """
+        M, N = self._swap(*self.shape)
+        start, stop, step = idx.indices(N)
+
+        if start == 0 and stop == N and step == 1:
+            return self.copy() if copy else self
+
+        N = len(range(start, stop, step))
+        new_shape = self._swap(M, N)
+
+        if N == 0 or self.nnz == 0:
+            return self.__class__(new_shape, dtype=self.dtype)
+        if step == 1:
+            return self.__class__(
+                _index._get_csr_submatrix_minor_axis(
+                    self.data, self.indices, self.indptr, start, stop),
+                shape=new_shape, copy=False)
+        cols = cupy.arange(start, stop, step, dtype=self.indices.dtype)
+        return self._minor_index_fancy(cols)
+
+    def _set_intXint(self, row, col, x):
+        i, j = self._swap(row, col)
+        self._set_many(i, j, x)
+
+    def _set_arrayXarray(self, row, col, x):
+        i, j = self._swap(row, col)
+        self._set_many(i, j, x)
+
+    def _set_arrayXarray_sparse(self, row, col, x):
+        # clear entries that will be overwritten
+        self._zero_many(*self._swap(row, col))
+
+        M, N = row.shape  # matches col.shape
+        broadcast_row = M != 1 and x.shape[0] == 1
+        broadcast_col = N != 1 and x.shape[1] == 1
+        r, c = x.row, x.col
+        x = cupy.asarray(x.data, dtype=self.dtype)
+        if broadcast_row:
+            r = cupy.repeat(cupy.arange(M), r.size)
+            c = cupy.tile(c, M)
+            x = cupy.tile(x, M)
+        if broadcast_col:
+            r = cupy.repeat(r, N)
+            c = cupy.tile(cupy.arange(N), c.size)
+            x = cupy.repeat(x, N)
+        # only assign entries in the new sparsity structure
+        i, j = self._swap(row[r, c], col[r, c])
+        self._set_many(i, j, x)
+
+    def _prepare_indices(self, i, j):
+        M, N = self._swap(*self.shape)
+
+        def check_bounds(indices, bound):
+            idx = indices.max()
+            if idx >= bound:
+                raise IndexError('index (%d) out of range (>= %d)' %
+                                 (idx, bound))
+            idx = indices.min()
+            if idx < -bound:
+                raise IndexError('index (%d) out of range (< -%d)' %
+                                 (idx, bound))
+
+        i = cupy.array(i, dtype=self.indptr.dtype,
+                       copy=True, ndmin=1).ravel()
+        j = cupy.array(j, dtype=self.indices.dtype,
+                       copy=True, ndmin=1).ravel()
+        check_bounds(i, M)
+        check_bounds(j, N)
+        return i, j, M, N
+
+    def _set_many(self, i, j, x):
+        """Sets value at each (i, j) to x
+        Here (i,j) index major and minor respectively, and must not contain
+        duplicate entries.
+        """
+        i, j, M, N = self._prepare_indices(i, j)
+        x = cupy.array(x, dtype=self.dtype, copy=True, ndmin=1).ravel()
+
+        new_sp = cupyx.scipy.sparse.csr_matrix(
+            (cupy.arange(self.nnz, dtype=cupy.float32),
+             self.indices, self.indptr), shape=(M, N))
+
+        offsets = new_sp._get_arrayXarray(
+            i, j, not_found_val=-1).astype(cupy.int32).ravel()
+
+        if -1 not in offsets:
+            # only affects existing non-zero cells
+            self.data[offsets] = x
+            return
+
+        else:
+            warnings.warn('Changing the sparsity structure of a '
+                          '{}_matrix is expensive.'.format(self.format),
+                          _base.SparseEfficiencyWarning)
+            # replace where possible
+            mask = offsets > -1
+            self.data[offsets[mask]] = x[mask]
+            # only insertions remain
+            mask = ~mask
+            i = i[mask]
+            i[i < 0] += M
+            j = j[mask]
+            j[j < 0] += N
+            self._insert_many(i, j, x[mask])
+
+    def _zero_many(self, i, j):
+        """Sets value at each (i, j) to zero, preserving sparsity structure.
+        Here (i,j) index major and minor respectively.
+        """
+        i, j, M, N = self._prepare_indices(i, j)
+
+        new_sp = cupyx.scipy.sparse.csr_matrix(
+            (cupy.arange(self.nnz, dtype=cupy.float32),
+             self.indices, self.indptr), shape=(M, N))
+
+        offsets = new_sp._get_arrayXarray(
+            i, j, not_found_val=-1).astype(cupy.int32).ravel()
+
+        # only assign zeros to the existing sparsity structure
+        self.data[offsets[offsets > -1]] = 0
+
+    def _perform_insert(self, indices_inserts, data_inserts,
+                        rows, row_counts, idx_dtype):
+        """Insert new elements into current sparse matrix in sorted order"""
+        indptr_diff = cupy.diff(self.indptr)
+        indptr_diff[rows] += row_counts
+
+        new_indptr = cupy.empty(self.indptr.shape, dtype=idx_dtype)
+        new_indptr[0] = idx_dtype(0)
+        new_indptr[1:] = indptr_diff
+
+        # Build output arrays
+        cupy.cumsum(new_indptr, out=new_indptr)
+        out_nnz = int(new_indptr[-1])
+
+        new_indices = cupy.empty(out_nnz, dtype=idx_dtype)
+        new_data = cupy.empty(out_nnz, dtype=self.data.dtype)
+
+        # Build an indexed indptr that contains the offsets for each
+        # row but only for in i, j, and x.
+        new_indptr_lookup = cupy.zeros(new_indptr.size, dtype=idx_dtype)
+        new_indptr_lookup[1:][rows] = row_counts
+        cupy.cumsum(new_indptr_lookup, out=new_indptr_lookup)
+
+        _index._insert_many_populate_arrays(
+            indices_inserts, data_inserts, new_indptr_lookup,
+            self.indptr, self.indices, self.data, new_indptr, new_indices,
+            new_data, size=self.indptr.size-1)
+
+        self.indptr = new_indptr
+        self.indices = new_indices
+        self.data = new_data
+
+    def _insert_many(self, i, j, x):
+        """Inserts new nonzero at each (i, j) with value x
+        Here (i,j) index major and minor respectively.
+        i, j and x must be non-empty, 1d arrays.
+        Inserts each major group (e.g. all entries per row) at a time.
+        Maintains has_sorted_indices property.
+        Modifies i, j, x in place.
+        """
+
+        order = cupy.argsort(i)  # stable for duplicates
+        i = i.take(order)
+        j = j.take(order)
+        x = x.take(order)
+
+        # Update index data type
+
+        idx_dtype = _sputils.get_index_dtype(
+            (self.indices, self.indptr), maxval=(
+                self.nnz + x.size))
+
+        self.indptr = self.indptr.astype(idx_dtype)
+        self.indices = self.indices.astype(idx_dtype)
+        self.data = self.data.astype(self.dtype)
+
+        indptr_inserts, indices_inserts, data_inserts = \
+            _index._select_last_indices(i, j, x, idx_dtype)
+
+        rows, ui_indptr = cupy.unique(indptr_inserts, return_index=True)
+
+        to_add = cupy.empty(ui_indptr.size+1, ui_indptr.dtype)
+        to_add[-1] = j.size
+        to_add[:-1] = ui_indptr
+        ui_indptr = to_add
+
+        # Compute the counts for each row in the insertion array
+        row_counts = cupy.zeros(ui_indptr.size-1, dtype=idx_dtype)
+        cupy.add.at(row_counts, cupy.searchsorted(rows, indptr_inserts), 1)
+
+        self._perform_insert(indices_inserts, data_inserts,
+                             rows, row_counts, idx_dtype)
+
+    def __get_has_canonical_format(self):
+        """Determine whether the matrix has sorted indices and no duplicates.
+
+        Returns
+            bool: ``True`` if the above applies, otherwise ``False``.
+
+        .. note::
+            :attr:`has_canonical_format` implies :attr:`has_sorted_indices`, so
+            if the latter flag is ``False``, so will the former be; if the
+            former is found ``True``, the latter flag is also set.
+
+        .. warning::
+            Getting this property might synchronize the device.
+
+        """
+        # Modified from the SciPy counterpart.
+
+        # In CuPy the implemented conversions do not exactly match those of
+        # SciPy's, so it's hard to put this exactly as where it is in SciPy,
+        # but this should do the job.
+        if self.data.size == 0:
+            self._has_canonical_format = True
+        # check to see if result was cached
+        elif not getattr(self, '_has_sorted_indices', True):
+            # not sorted => not canonical
+            self._has_canonical_format = False
+        elif not hasattr(self, '_has_canonical_format'):
+            is_canonical = self._has_canonical_format_kern(
+                self.indptr, self.indices, size=self.indptr.size-1)
+            self._has_canonical_format = bool(is_canonical.all())
+        return self._has_canonical_format
+
+    def __set_has_canonical_format(self, val):
+        """Taken from SciPy as is."""
+        self._has_canonical_format = bool(val)
+        if val:
+            self.has_sorted_indices = True
+
+    has_canonical_format = property(fget=__get_has_canonical_format,
+                                    fset=__set_has_canonical_format)
+
+    def __get_sorted(self):
+        """Determine whether the matrix has sorted indices.
+
+        Returns
+            bool:
+                ``True`` if the indices of the matrix are in sorted order,
+                otherwise ``False``.
+
+        .. warning::
+            Getting this property might synchronize the device.
+
+        """
+        # Modified from the SciPy counterpart.
+
+        # In CuPy the implemented conversions do not exactly match those of
+        # SciPy's, so it's hard to put this exactly as where it is in SciPy,
+        # but this should do the job.
+        if self.data.size == 0:
+            self._has_sorted_indices = True
+        # check to see if result was cached
+        elif not hasattr(self, '_has_sorted_indices'):
+            is_sorted = self._has_sorted_indices_kern(
+                self.indptr, self.indices, size=self.indptr.size-1)
+            self._has_sorted_indices = bool(is_sorted.all())
+        return self._has_sorted_indices
+
+    def __set_sorted(self, val):
+        self._has_sorted_indices = bool(val)
+
+    has_sorted_indices = property(fget=__get_sorted, fset=__set_sorted)
+
+    def get_shape(self):
+        """Returns the shape of the matrix.
+
+        Returns:
+            tuple: Shape of the matrix.
+
+        """
+        return self._shape
+
+    def getnnz(self, axis=None):
+        """Returns the number of stored values, including explicit zeros.
+
+        Args:
+            axis: Not supported yet.
+
+        Returns:
+            int: The number of stored values.
+
+        """
+        if axis is None:
+            return self.data.size
+        else:
+            raise ValueError
+
+    def sorted_indices(self):
+        """Return a copy of this matrix with sorted indices
+
+        .. warning::
+            Calling this function might synchronize the device.
+        """
+        # Taken from SciPy as is.
+        A = self.copy()
+        A.sort_indices()
+        return A
+
+    def sort_indices(self):
+        # Unlike in SciPy, here this is implemented in child classes because
+        # each child needs to call its own sort function from cuSPARSE
+        raise NotImplementedError
+
+    def sum_duplicates(self):
+        """Eliminate duplicate matrix entries by adding them together.
+
+        .. note::
+            This is an *in place* operation.
+
+        .. warning::
+            Calling this function might synchronize the device.
+
+        .. seealso::
+           :meth:`scipy.sparse.csr_matrix.sum_duplicates`,
+           :meth:`scipy.sparse.csc_matrix.sum_duplicates`
+        """
+        if self.has_canonical_format:
+            return
+        # TODO(leofang): add a kernel for compressed sparse matrices without
+        # converting to coo
+        coo = self.tocoo()
+        coo.sum_duplicates()
+        self.__init__(coo.asformat(self.format))
+        self.has_canonical_format = True
+
+    #####################
+    # Reduce operations #
+    #####################
+
+    def _minor_reduce(self, ufunc, axis, nonzero):
+        """Reduce nonzeros with a ufunc over the minor axis when non-empty
+
+        Can be applied to a function of self.data by supplying data parameter.
+        Warning: this does not call sum_duplicates()
+
+        Args:
+            ufunc (object): Function handle giving the operation to be
+                conducted.
+            axis (int): Matrix over which the reduction should be
+                conducted.
+
+        Returns:
+            (cupy.ndarray): Reduce result for nonzeros in each
+            major_index.
+
+        """
+        out_shape = self.shape[1 - axis]
+        # Call to the appropriate kernel function
+        out = cupy.zeros(out_shape).astype(cupy.float64)
+        if nonzero:
+            kerns = {cupy.amax: self._max_nonzero_reduction_kern,
+                     cupy.amin: self._min_nonzero_reduction_kern}
+        else:
+            kerns = {cupy.amax: self._max_reduction_kern,
+                     cupy.amin: self._min_reduction_kern}
+
+        kerns[ufunc]((out_shape,), (1,),
+                     (self.data.astype(cupy.float64),
+                      self.indptr[:len(self.indptr) - 1],
+                      self.indptr[1:], cupy.int64(self.shape[axis]),
+                      out))
+
+        return out
+
+    def _arg_minor_reduce(self, ufunc, axis):
+        """Reduce nonzeros with a ufunc over the minor axis when non-empty
+
+        Can be applied to a function of self.data by supplying data parameter.
+        Warning: this does not call sum_duplicates()
+
+        Args:
+            ufunc (object): Function handle giving the operation to be
+                conducted.
+            axis (int): Maxtrix over which the reduction should be conducted
+
+        Returns:
+            (cupy.ndarray): Reduce result for nonzeros in each
+            major_index
+
+        """
+
+        # Call to the appropriate kernel function
+        # Create the vector to hold output
+        # Note: it's important to set "int" here, following what SciPy
+        # does, as the outcome dtype is platform dependent
+        out_shape = self.shape[1 - axis]
+        out = cupy.zeros(out_shape, dtype=int)
+
+        # Perform the calculation
+        ker_name = '_arg_reduction<{}, {}>'.format(
+            _scalar.get_typename(self.data.dtype),
+            _scalar.get_typename(out.dtype))
+
+        if ufunc == cupy.argmax:
+            ker = self._max_arg_reduction_mod.get_function('max' + ker_name)
+        elif ufunc == cupy.argmin:
+            ker = self._min_arg_reduction_mod.get_function('min' + ker_name)
+
+        ker((out_shape,), (1,),
+            (self.data, self.indices,
+             self.indptr[:len(self.indptr) - 1],
+             self.indptr[1:], cupy.int64(self.shape[axis]),
+             out))
+
+        return out
diff --git a/cupyx/scipy/sparse/_construct.py b/cupyx/scipy/sparse/_construct.py
new file mode 100644
index 0000000..3aa3d87
--- /dev/null
+++ b/cupyx/scipy/sparse/_construct.py
@@ -0,0 +1,582 @@
+import numpy
+import cupy
+from cupyx.scipy.sparse import _coo
+from cupyx.scipy.sparse import _csc
+from cupyx.scipy.sparse import _csr
+from cupyx.scipy.sparse import _dia
+from cupyx.scipy.sparse import _sputils
+
+
+def eye(m, n=None, k=0, dtype='d', format=None):
+    """Creates a sparse matrix with ones on diagonal.
+
+    Args:
+        m (int): Number of rows.
+        n (int or None): Number of columns. If it is ``None``,
+            it makes a square matrix.
+        k (int): Diagonal to place ones on.
+        dtype: Type of a matrix to create.
+        format (str or None): Format of the result, e.g. ``format="csr"``.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: Created sparse matrix.
+
+    .. seealso:: :func:`scipy.sparse.eye`
+
+    """
+    if n is None:
+        n = m
+    m, n = int(m), int(n)
+
+    if m == n and k == 0:
+        if format in ['csr', 'csc']:
+            indptr = cupy.arange(n + 1, dtype='i')
+            indices = cupy.arange(n, dtype='i')
+            data = cupy.ones(n, dtype=dtype)
+            if format == 'csr':
+                cls = _csr.csr_matrix
+            else:
+                cls = _csc.csc_matrix
+            return cls((data, indices, indptr), (n, n))
+
+        elif format == 'coo':
+            row = cupy.arange(n, dtype='i')
+            col = cupy.arange(n, dtype='i')
+            data = cupy.ones(n, dtype=dtype)
+            return _coo.coo_matrix((data, (row, col)), (n, n))
+
+    diags = cupy.ones((1, max(0, min(m + k, n))), dtype=dtype)
+    return spdiags(diags, k, m, n).asformat(format)
+
+
+def identity(n, dtype='d', format=None):
+    """Creates an identity matrix in sparse format.
+
+    .. note::
+       Currently it only supports csr, csc and coo formats.
+
+    Args:
+        n (int): Number of rows and columns.
+        dtype: Type of a matrix to create.
+        format (str or None): Format of the result, e.g. ``format="csr"``.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: Created identity matrix.
+
+    .. seealso:: :func:`scipy.sparse.identity`
+
+    """
+    return eye(n, n, dtype=dtype, format=format)
+
+
+def spdiags(data, diags, m, n, format=None):
+    """Creates a sparse matrix from diagonals.
+
+    Args:
+        data (cupy.ndarray): Matrix diagonals stored row-wise.
+        diags (cupy.ndarray): Diagonals to set.
+        m (int): Number of rows.
+        n (int): Number of cols.
+        format (str or None): Sparse format, e.g. ``format="csr"``.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: Created sparse matrix.
+
+    .. seealso:: :func:`scipy.sparse.spdiags`
+
+    """
+    return _dia.dia_matrix((data, diags), shape=(m, n)).asformat(format)
+
+
+def _compressed_sparse_stack(blocks, axis):
+    """Fast path for stacking CSR/CSC matrices
+    (i) vstack for CSR, (ii) hstack for CSC.
+    """
+    other_axis = 1 if axis == 0 else 0
+    data = cupy.concatenate([b.data for b in blocks])
+    constant_dim = blocks[0].shape[other_axis]
+    idx_dtype = _sputils.get_index_dtype(arrays=[b.indptr for b in blocks],
+                                         maxval=max(data.size, constant_dim))
+    indices = cupy.empty(data.size, dtype=idx_dtype)
+    indptr = cupy.empty(sum(b.shape[axis]
+                            for b in blocks) + 1, dtype=idx_dtype)
+    last_indptr = idx_dtype(0)
+    sum_dim = 0
+    sum_indices = 0
+    for b in blocks:
+        if b.shape[other_axis] != constant_dim:
+            raise ValueError(
+                'incompatible dimensions for axis %d' % other_axis)
+        indices[sum_indices:sum_indices+b.indices.size] = b.indices
+        sum_indices += b.indices.size
+        idxs = slice(sum_dim, sum_dim + b.shape[axis])
+        indptr[idxs] = b.indptr[:-1]
+        indptr[idxs] += last_indptr
+        sum_dim += b.shape[axis]
+        last_indptr += b.indptr[-1]
+    indptr[-1] = last_indptr
+    if axis == 0:
+        return _csr.csr_matrix((data, indices, indptr),
+                               shape=(sum_dim, constant_dim))
+    else:
+        return _csc.csc_matrix((data, indices, indptr),
+                               shape=(constant_dim, sum_dim))
+
+
+def hstack(blocks, format=None, dtype=None):
+    """Stacks sparse matrices horizontally (column wise)
+
+    Args:
+        blocks (sequence of cupyx.scipy.sparse.spmatrix):
+            sparse matrices to stack
+
+        format (str):
+            sparse format of the result (e.g. "csr")
+            by default an appropriate sparse matrix format is returned.
+            This choice is subject to change.
+        dtype (dtype, optional):
+            The data-type of the output matrix.  If not given, the dtype is
+            determined from that of ``blocks``.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: the stacked sparse matrix
+
+    .. seealso:: :func:`scipy.sparse.hstack`
+
+    Examples:
+        >>> from cupy import array
+        >>> from cupyx.scipy.sparse import csr_matrix, hstack
+        >>> A = csr_matrix(array([[1., 2.], [3., 4.]]))
+        >>> B = csr_matrix(array([[5.], [6.]]))
+        >>> hstack([A, B]).toarray()
+        array([[1., 2., 5.],
+               [3., 4., 6.]])
+    """
+    return bmat([blocks], format=format, dtype=dtype)
+
+
+def vstack(blocks, format=None, dtype=None):
+    """Stacks sparse matrices vertically (row wise)
+
+    Args:
+        blocks (sequence of cupyx.scipy.sparse.spmatrix)
+            sparse matrices to stack
+        format (str, optional):
+            sparse format of the result (e.g. "csr")
+            by default an appropriate sparse matrix format is returned.
+            This choice is subject to change.
+        dtype (dtype, optional):
+            The data-type of the output matrix.  If not given, the dtype is
+            determined from that of `blocks`.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: the stacked sparse matrix
+
+    .. seealso:: :func:`scipy.sparse.vstack`
+
+    Examples:
+        >>> from cupy import array
+        >>> from cupyx.scipy.sparse import csr_matrix, vstack
+        >>> A = csr_matrix(array([[1., 2.], [3., 4.]]))
+        >>> B = csr_matrix(array([[5., 6.]]))
+        >>> vstack([A, B]).toarray()
+        array([[1., 2.],
+               [3., 4.],
+               [5., 6.]])
+    """
+    return bmat([[b] for b in blocks], format=format, dtype=dtype)
+
+
+def bmat(blocks, format=None, dtype=None):
+    """Builds a sparse matrix from sparse sub-blocks
+
+    Args:
+        blocks (array_like):
+            Grid of sparse matrices with compatible shapes.
+            An entry of None implies an all-zero matrix.
+        format ({'bsr', 'coo', 'csc', 'csr', 'dia', 'dok', 'lil'}, optional):
+            The sparse format of the result (e.g. "csr").  By default an
+            appropriate sparse matrix format is returned.
+            This choice is subject to change.
+        dtype (dtype, optional):
+            The data-type of the output matrix.  If not given, the dtype is
+            determined from that of `blocks`.
+    Returns:
+        bmat (sparse matrix)
+
+    .. seealso:: :func:`scipy.sparse.bmat`
+
+    Examples:
+        >>> from cupy import array
+        >>> from cupyx.scipy.sparse import csr_matrix, bmat
+        >>> A = csr_matrix(array([[1., 2.], [3., 4.]]))
+        >>> B = csr_matrix(array([[5.], [6.]]))
+        >>> C = csr_matrix(array([[7.]]))
+        >>> bmat([[A, B], [None, C]]).toarray()
+        array([[1., 2., 5.],
+               [3., 4., 6.],
+               [0., 0., 7.]])
+        >>> bmat([[A, None], [None, C]]).toarray()
+        array([[1., 2., 0.],
+               [3., 4., 0.],
+               [0., 0., 7.]])
+
+    """
+
+    # We assume here that blocks will be 2-D so we need to look, at most,
+    # 2 layers deep for the shape
+    # TODO(Corey J. Nolet): Check this assumption and raise ValueError
+
+    # NOTE: We can't follow scipy exactly here
+    # since we don't have an `object` datatype
+    M = len(blocks)
+    N = len(blocks[0])
+
+    blocks_flat = []
+    for m in range(M):
+        for n in range(N):
+            if blocks[m][n] is not None:
+                blocks_flat.append(blocks[m][n])
+
+    if len(blocks_flat) == 0:
+        return _coo.coo_matrix((0, 0), dtype=dtype)
+
+    # check for fast path cases
+    if (N == 1 and format in (None, 'csr') and
+            all(isinstance(b, _csr.csr_matrix)
+                for b in blocks_flat)):
+        A = _compressed_sparse_stack(blocks_flat, 0)
+        if dtype is not None:
+            A = A.astype(dtype)
+        return A
+    elif (M == 1 and format in (None, 'csc')
+          and all(isinstance(b, _csc.csc_matrix) for b in blocks_flat)):
+        A = _compressed_sparse_stack(blocks_flat, 1)
+        if dtype is not None:
+            A = A.astype(dtype)
+        return A
+
+    block_mask = numpy.zeros((M, N), dtype=bool)
+    brow_lengths = numpy.zeros(M+1, dtype=numpy.int64)
+    bcol_lengths = numpy.zeros(N+1, dtype=numpy.int64)
+
+    # convert everything to COO format
+    for i in range(M):
+        for j in range(N):
+            if blocks[i][j] is not None:
+                A = _coo.coo_matrix(blocks[i][j])
+                blocks[i][j] = A
+                block_mask[i][j] = True
+
+                if brow_lengths[i+1] == 0:
+                    brow_lengths[i+1] = A.shape[0]
+                elif brow_lengths[i+1] != A.shape[0]:
+                    msg = ('blocks[{i},:] has incompatible row dimensions. '
+                           'Got blocks[{i},{j}].shape[0] == {got}, '
+                           'expected {exp}.'.format(i=i, j=j,
+                                                    exp=brow_lengths[i+1],
+                                                    got=A.shape[0]))
+                    raise ValueError(msg)
+
+                if bcol_lengths[j+1] == 0:
+                    bcol_lengths[j+1] = A.shape[1]
+                elif bcol_lengths[j+1] != A.shape[1]:
+                    msg = ('blocks[:,{j}] has incompatible row dimensions. '
+                           'Got blocks[{i},{j}].shape[1] == {got}, '
+                           'expected {exp}.'.format(i=i, j=j,
+                                                    exp=bcol_lengths[j+1],
+                                                    got=A.shape[1]))
+                    raise ValueError(msg)
+
+    nnz = sum(block.nnz for block in blocks_flat)
+    if dtype is None:
+        all_dtypes = [blk.dtype for blk in blocks_flat]
+        dtype = _sputils.upcast(*all_dtypes) if all_dtypes else None
+
+    row_offsets = numpy.cumsum(brow_lengths)
+    col_offsets = numpy.cumsum(bcol_lengths)
+
+    shape = (row_offsets[-1], col_offsets[-1])
+
+    data = cupy.empty(nnz, dtype=dtype)
+    idx_dtype = _sputils.get_index_dtype(maxval=max(shape))
+    row = cupy.empty(nnz, dtype=idx_dtype)
+    col = cupy.empty(nnz, dtype=idx_dtype)
+
+    nnz = 0
+    ii, jj = numpy.nonzero(block_mask)
+    for i, j in zip(ii, jj):
+        B = blocks[int(i)][int(j)]
+        idx = slice(nnz, nnz + B.nnz)
+        data[idx] = B.data
+        row[idx] = B.row + row_offsets[i]
+        col[idx] = B.col + col_offsets[j]
+        nnz += B.nnz
+
+    return _coo.coo_matrix((data, (row, col)), shape=shape).asformat(format)
+
+
+def random(m, n, density=0.01, format='coo', dtype=None,
+           random_state=None, data_rvs=None):
+    """Generates a random sparse matrix.
+
+    This function generates a random sparse matrix. First it selects non-zero
+    elements with given density ``density`` from ``(m, n)`` elements.
+    So the number of non-zero elements ``k`` is ``k = m * n * density``.
+    Value of each element is selected with ``data_rvs`` function.
+
+    Args:
+        m (int): Number of rows.
+        n (int): Number of cols.
+        density (float): Ratio of non-zero entries.
+        format (str): Matrix format.
+        dtype (~cupy.dtype): Type of the returned matrix values.
+        random_state (cupy.random.RandomState or int):
+            State of random number generator.
+            If an integer is given, the method makes a new state for random
+            number generator and uses it.
+            If it is not given, the default state is used.
+            This state is used to generate random indexes for nonzero entries.
+        data_rvs (callable): A function to generate data for a random matrix.
+            If it is not given, `random_state.rand` is used.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: Generated matrix.
+
+    .. seealso:: :func:`scipy.sparse.random`
+
+    """
+    if density < 0 or density > 1:
+        raise ValueError('density expected to be 0 <= density <= 1')
+    dtype = cupy.dtype(dtype)
+    if dtype.char not in 'fd':
+        raise NotImplementedError('type %s not supported' % dtype)
+
+    mn = m * n
+
+    k = int(density * m * n)
+
+    if random_state is None:
+        random_state = cupy.random
+    elif isinstance(random_state, (int, cupy.integer)):
+        random_state = cupy.random.RandomState(random_state)
+
+    if data_rvs is None:
+        data_rvs = random_state.rand
+
+    ind = random_state.choice(mn, size=k, replace=False)
+    j = ind // m
+    i = ind - j * m
+    vals = data_rvs(k).astype(dtype)
+    return _coo.coo_matrix(
+        (vals, (i, j)), shape=(m, n)).asformat(format)
+
+
+def rand(m, n, density=0.01, format='coo', dtype=None, random_state=None):
+    """Generates a random sparse matrix.
+
+    See :func:`cupyx.scipy.sparse.random` for detail.
+
+    Args:
+        m (int): Number of rows.
+        n (int): Number of cols.
+        density (float): Ratio of non-zero entries.
+        format (str): Matrix format.
+        dtype (~cupy.dtype): Type of the returned matrix values.
+        random_state (cupy.random.RandomState or int):
+            State of random number generator.
+            If an integer is given, the method makes a new state for random
+            number generator and uses it.
+            If it is not given, the default state is used.
+            This state is used to generate random indexes for nonzero entries.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: Generated matrix.
+
+    .. seealso:: :func:`scipy.sparse.rand`
+    .. seealso:: :func:`cupyx.scipy.sparse.random`
+
+    """
+    return random(m, n, density, format, dtype, random_state)
+
+
+def diags(diagonals, offsets=0, shape=None, format=None, dtype=None):
+    """Construct a sparse matrix from diagonals.
+
+    Args:
+        diagonals (sequence of array_like):
+            Sequence of arrays containing the matrix diagonals, corresponding
+            to `offsets`.
+        offsets (sequence of int or an int):
+            Diagonals to set:
+                - k = 0  the main diagonal (default)
+                - k > 0  the k-th upper diagonal
+                - k < 0  the k-th lower diagonal
+        shape (tuple of int):
+            Shape of the result. If omitted, a square matrix large enough
+            to contain the diagonals is returned.
+        format ({"dia", "csr", "csc", "lil", ...}):
+            Matrix format of the result.  By default (format=None) an
+            appropriate sparse matrix format is returned.  This choice is
+            subject to change.
+        dtype (dtype): Data type of the matrix.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix: Generated matrix.
+
+    Notes:
+        This function differs from `spdiags` in the way it handles
+        off-diagonals.
+
+        The result from `diags` is the sparse equivalent of::
+
+            cupy.diag(diagonals[0], offsets[0])
+            + ...
+            + cupy.diag(diagonals[k], offsets[k])
+
+        Repeated diagonal offsets are disallowed.
+    """
+    # if offsets is not a sequence, assume that there's only one diagonal
+    if _sputils.isscalarlike(offsets):
+        # now check that there's actually only one diagonal
+        if len(diagonals) == 0 or _sputils.isscalarlike(diagonals[0]):
+            diagonals = [cupy.atleast_1d(diagonals)]
+        else:
+            raise ValueError('Different number of diagonals and offsets.')
+    else:
+        diagonals = list(map(cupy.atleast_1d, diagonals))
+
+    if isinstance(offsets, cupy.ndarray):
+        offsets = offsets.get()
+    offsets = numpy.atleast_1d(offsets)
+
+    # Basic check
+    if len(diagonals) != len(offsets):
+        raise ValueError('Different number of diagonals and offsets.')
+
+    # Determine shape, if omitted
+    if shape is None:
+        m = len(diagonals[0]) + abs(int(offsets[0]))
+        shape = (m, m)
+
+    # Determine data type, if omitted
+    if dtype is None:
+        dtype = cupy.common_type(*diagonals)
+
+    # Construct data array
+    m, n = shape
+
+    M = max([min(m + offset, n - offset) + max(0, offset)
+             for offset in offsets])
+    M = max(0, M)
+    data_arr = cupy.zeros((len(offsets), M), dtype=dtype)
+
+    K = min(m, n)
+
+    for j, diagonal in enumerate(diagonals):
+        offset = offsets[j]
+        k = max(0, offset)
+        length = min(m + offset, n - offset, K)
+        if length < 0:
+            raise ValueError(
+                'Offset %d (index %d) out of bounds' % (offset, j))
+        try:
+            data_arr[j, k:k+length] = diagonal[..., :length]
+        except ValueError:
+            if len(diagonal) != length and len(diagonal) != 1:
+                raise ValueError(
+                    'Diagonal length (index %d: %d at offset %d) does not '
+                    'agree with matrix size (%d, %d).' % (
+                        j, len(diagonal), offset, m, n))
+            raise
+
+    return _dia.dia_matrix((data_arr, offsets), shape=(m, n)).asformat(format)
+
+
+def kron(A, B, format=None):
+    """Kronecker product of sparse matrices A and B.
+
+    Args:
+        A (cupyx.scipy.sparse.spmatrix): a sparse matrix.
+        B (cupyx.scipy.sparse.spmatrix): a sparse matrix.
+        format (str): the format of the returned sparse matrix.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix:
+            Generated sparse matrix with the specified ``format``.
+
+    .. seealso:: :func:`scipy.sparse.kron`
+
+    """
+    # TODO(leofang): support BSR format when it's added to CuPy
+    # TODO(leofang): investigate if possible to optimize performance by
+    #                starting with CSR instead of COO matrices
+
+    A = _coo.coo_matrix(A)
+    B = _coo.coo_matrix(B)
+    out_shape = (A.shape[0] * B.shape[0], A.shape[1] * B.shape[1])
+
+    if A.nnz == 0 or B.nnz == 0:
+        # kronecker product is the zero matrix
+        return _coo.coo_matrix(out_shape).asformat(format)
+
+    if max(out_shape[0], out_shape[1]) > cupy.iinfo('int32').max:
+        dtype = cupy.int64
+    else:
+        dtype = cupy.int32
+
+    # expand entries of A into blocks
+    row = A.row.astype(dtype, copy=True) * B.shape[0]
+    row = row.repeat(B.nnz)
+    col = A.col.astype(dtype, copy=True) * B.shape[1]
+    col = col.repeat(B.nnz)
+    data = A.data.repeat(B.nnz)  # data's dtype follows that of A in SciPy
+
+    # increment block indices
+    row, col = row.reshape(-1, B.nnz), col.reshape(-1, B.nnz)
+    row += B.row
+    col += B.col
+    row, col = row.ravel(), col.ravel()
+
+    # compute block entries
+    data = data.reshape(-1, B.nnz) * B.data
+    data = data.ravel()
+
+    return _coo.coo_matrix(
+        (data, (row, col)), shape=out_shape).asformat(format)
+
+
+def kronsum(A, B, format=None):
+    """Kronecker sum of sparse matrices A and B.
+
+    Kronecker sum is the sum of two Kronecker products
+    ``kron(I_n, A) + kron(B, I_m)``, where ``I_n`` and ``I_m`` are identity
+    matrices.
+
+    Args:
+        A (cupyx.scipy.sparse.spmatrix): a sparse matrix.
+        B (cupyx.scipy.sparse.spmatrix): a sparse matrix.
+        format (str): the format of the returned sparse matrix.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix:
+            Generated sparse matrix with the specified ``format``.
+
+    .. seealso:: :func:`scipy.sparse.kronsum`
+
+    """
+    A = _coo.coo_matrix(A)
+    B = _coo.coo_matrix(B)
+
+    if A.shape[0] != A.shape[1]:
+        raise ValueError('A is not square matrix')
+
+    if B.shape[0] != B.shape[1]:
+        raise ValueError('B is not square matrix')
+
+    dtype = _sputils.upcast(A.dtype, B.dtype)
+
+    L = kron(eye(B.shape[0], dtype=dtype), A, format=format)
+    R = kron(B, eye(A.shape[0], dtype=dtype), format=format)
+
+    return (L + R).asformat(format)
diff --git a/cupyx/scipy/sparse/_coo.py b/cupyx/scipy/sparse/_coo.py
new file mode 100644
index 0000000..0cc2f79
--- /dev/null
+++ b/cupyx/scipy/sparse/_coo.py
@@ -0,0 +1,553 @@
+import numpy
+try:
+    import scipy.sparse
+    _scipy_available = True
+except ImportError:
+    _scipy_available = False
+
+import cupy
+from cupy import _core
+from cupyx import cusparse
+from cupyx.scipy.sparse import _base
+from cupyx.scipy.sparse import _csc
+from cupyx.scipy.sparse import _csr
+from cupyx.scipy.sparse import _data as sparse_data
+from cupyx.scipy.sparse import _util
+from cupyx.scipy.sparse import _sputils
+
+
+class coo_matrix(sparse_data._data_matrix):
+
+    """COOrdinate format sparse matrix.
+
+    This can be instantiated in several ways.
+
+    ``coo_matrix(D)``
+        ``D`` is a rank-2 :class:`cupy.ndarray`.
+
+    ``coo_matrix(S)``
+        ``S`` is another sparse matrix. It is equivalent to ``S.tocoo()``.
+
+    ``coo_matrix((M, N), [dtype])``
+        It constructs an empty matrix whose shape is ``(M, N)``. Default dtype
+        is float64.
+
+    ``coo_matrix((data, (row, col)))``
+        All ``data``, ``row`` and ``col`` are one-dimenaional
+        :class:`cupy.ndarray`.
+
+    Args:
+        arg1: Arguments for the initializer.
+        shape (tuple): Shape of a matrix. Its length must be two.
+        dtype: Data type. It must be an argument of :class:`numpy.dtype`.
+        copy (bool): If ``True``, copies of given data are always used.
+
+    .. seealso::
+       :class:`scipy.sparse.coo_matrix`
+
+    """
+
+    format = 'coo'
+
+    _sum_duplicates_diff = _core.ElementwiseKernel(
+        'raw T row, raw T col',
+        'T diff',
+        '''
+        T diff_out = 1;
+        if (i == 0 || row[i - 1] == row[i] && col[i - 1] == col[i]) {
+          diff_out = 0;
+        }
+        diff = diff_out;
+        ''', 'cupyx_scipy_sparse_coo_sum_duplicates_diff')
+
+    def __init__(self, arg1, shape=None, dtype=None, copy=False):
+        if shape is not None and len(shape) != 2:
+            raise ValueError(
+                'Only two-dimensional sparse arrays are supported.')
+
+        if _base.issparse(arg1):
+            x = arg1.asformat(self.format)
+            data = x.data
+            row = x.row
+            col = x.col
+
+            if arg1.format != self.format:
+                # When formats are differnent, all arrays are already copied
+                copy = False
+
+            if shape is None:
+                shape = arg1.shape
+
+            self.has_canonical_format = x.has_canonical_format
+
+        elif _util.isshape(arg1):
+            m, n = arg1
+            m, n = int(m), int(n)
+            data = cupy.zeros(0, dtype if dtype else 'd')
+            row = cupy.zeros(0, dtype='i')
+            col = cupy.zeros(0, dtype='i')
+            # shape and copy argument is ignored
+            shape = (m, n)
+            copy = False
+
+            self.has_canonical_format = True
+
+        elif _scipy_available and scipy.sparse.issparse(arg1):
+            # Convert scipy.sparse to cupyx.scipy.sparse
+            x = arg1.tocoo()
+            data = cupy.array(x.data)
+            row = cupy.array(x.row, dtype='i')
+            col = cupy.array(x.col, dtype='i')
+            copy = False
+            if shape is None:
+                shape = arg1.shape
+
+            self.has_canonical_format = x.has_canonical_format
+
+        elif isinstance(arg1, tuple) and len(arg1) == 2:
+            try:
+                data, (row, col) = arg1
+            except (TypeError, ValueError):
+                raise TypeError('invalid input format')
+
+            if not (_base.isdense(data) and data.ndim == 1 and
+                    _base.isdense(row) and row.ndim == 1 and
+                    _base.isdense(col) and col.ndim == 1):
+                raise ValueError('row, column, and data arrays must be 1-D')
+            if not (len(data) == len(row) == len(col)):
+                raise ValueError(
+                    'row, column, and data array must all be the same length')
+
+            self.has_canonical_format = False
+
+        elif _base.isdense(arg1):
+            if arg1.ndim > 2:
+                raise TypeError('expected dimension <= 2 array or matrix')
+            dense = cupy.atleast_2d(arg1)
+            row, col = dense.nonzero()
+            data = dense[row, col]
+            shape = dense.shape
+
+            self.has_canonical_format = True
+
+        else:
+            raise TypeError('invalid input format')
+
+        if dtype is None:
+            dtype = data.dtype
+        else:
+            dtype = numpy.dtype(dtype)
+
+        if dtype != 'f' and dtype != 'd' and dtype != 'F' and dtype != 'D':
+            raise ValueError(
+                'Only float32, float64, complex64 and complex128'
+                ' are supported')
+
+        data = data.astype(dtype, copy=copy)
+        row = row.astype('i', copy=copy)
+        col = col.astype('i', copy=copy)
+
+        if shape is None:
+            if len(row) == 0 or len(col) == 0:
+                raise ValueError(
+                    'cannot infer dimensions from zero sized index arrays')
+            shape = (int(row.max()) + 1, int(col.max()) + 1)
+
+        if len(data) > 0:
+            if row.max() >= shape[0]:
+                raise ValueError('row index exceeds matrix dimensions')
+            if col.max() >= shape[1]:
+                raise ValueError('column index exceeds matrix dimensions')
+            if row.min() < 0:
+                raise ValueError('negative row index found')
+            if col.min() < 0:
+                raise ValueError('negative column index found')
+
+        sparse_data._data_matrix.__init__(self, data)
+        self.row = row
+        self.col = col
+        if not _util.isshape(shape):
+            raise ValueError('invalid shape (must be a 2-tuple of int)')
+        self._shape = int(shape[0]), int(shape[1])
+
+    def _with_data(self, data, copy=True):
+        """Returns a matrix with the same sparsity structure as self,
+        but with different data.  By default the index arrays
+        (i.e. .row and .col) are copied.
+        """
+        if copy:
+            return coo_matrix(
+                (data, (self.row.copy(), self.col.copy())),
+                shape=self.shape, dtype=data.dtype)
+        else:
+            return coo_matrix(
+                (data, (self.row, self.col)), shape=self.shape,
+                dtype=data.dtype)
+
+    def diagonal(self, k=0):
+        """Returns the k-th diagonal of the matrix.
+
+        Args:
+            k (int, optional): Which diagonal to get, corresponding to elements
+            a[i, i+k]. Default: 0 (the main diagonal).
+
+        Returns:
+            cupy.ndarray : The k-th diagonal.
+        """
+        rows, cols = self.shape
+        if k <= -rows or k >= cols:
+            return cupy.empty(0, dtype=self.data.dtype)
+        diag = cupy.zeros(min(rows + min(k, 0), cols - max(k, 0)),
+                          dtype=self.dtype)
+        diag_mask = (self.row + k) == self.col
+
+        if self.has_canonical_format:
+            row = self.row[diag_mask]
+            data = self.data[diag_mask]
+        else:
+            diag_coo = coo_matrix((self.data[diag_mask],
+                                   (self.row[diag_mask], self.col[diag_mask])),
+                                  shape=self.shape)
+            diag_coo.sum_duplicates()
+            row = diag_coo.row
+            data = diag_coo.data
+        diag[row + min(k, 0)] = data
+
+        return diag
+
+    def setdiag(self, values, k=0):
+        """Set diagonal or off-diagonal elements of the array.
+
+        Args:
+            values (ndarray): New values of the diagonal elements. Values may
+                have any length. If the diagonal is longer than values, then
+                the remaining diagonal entries will not be set. If values are
+                longer than the diagonal, then the remaining values are
+                ignored. If a scalar value is given, all of the diagonal is set
+                to it.
+            k (int, optional): Which off-diagonal to set, corresponding to
+                elements a[i,i+k]. Default: 0 (the main diagonal).
+
+        """
+        M, N = self.shape
+        if (k > 0 and k >= N) or (k < 0 and -k >= M):
+            raise ValueError("k exceeds matrix dimensions")
+        if values.ndim and not len(values):
+            return
+        idx_dtype = self.row.dtype
+
+        # Determine which triples to keep and where to put the new ones.
+        full_keep = self.col - self.row != k
+        if k < 0:
+            max_index = min(M + k, N)
+            if values.ndim:
+                max_index = min(max_index, len(values))
+            keep = cupy.logical_or(full_keep, self.col >= max_index)
+            new_row = cupy.arange(-k, -k + max_index, dtype=idx_dtype)
+            new_col = cupy.arange(max_index, dtype=idx_dtype)
+        else:
+            max_index = min(M, N - k)
+            if values.ndim:
+                max_index = min(max_index, len(values))
+            keep = cupy.logical_or(full_keep, self.row >= max_index)
+            new_row = cupy.arange(max_index, dtype=idx_dtype)
+            new_col = cupy.arange(k, k + max_index, dtype=idx_dtype)
+
+        # Define the array of data consisting of the entries to be added.
+        if values.ndim:
+            new_data = values[:max_index]
+        else:
+            new_data = cupy.full(max_index, values, dtype=self.dtype)
+
+        # Update the internal structure.
+        self.row = cupy.concatenate((self.row[keep], new_row))
+        self.col = cupy.concatenate((self.col[keep], new_col))
+        self.data = cupy.concatenate((self.data[keep], new_data))
+        self.has_canonical_format = False
+
+    def eliminate_zeros(self):
+        """Removes zero entories in place."""
+        ind = self.data != 0
+        self.data = self.data[ind]
+        self.row = self.row[ind]
+        self.col = self.col[ind]
+
+    def get_shape(self):
+        """Returns the shape of the matrix.
+
+        Returns:
+            tuple: Shape of the matrix.
+        """
+        return self._shape
+
+    def getnnz(self, axis=None):
+        """Returns the number of stored values, including explicit zeros."""
+        if axis is None:
+            return self.data.size
+        else:
+            raise ValueError
+
+    def get(self, stream=None):
+        """Returns a copy of the array on host memory.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+
+        Returns:
+            scipy.sparse.coo_matrix: Copy of the array on host memory.
+
+        """
+        if not _scipy_available:
+            raise RuntimeError('scipy is not available')
+
+        data = self.data.get(stream)
+        row = self.row.get(stream)
+        col = self.col.get(stream)
+        return scipy.sparse.coo_matrix(
+            (data, (row, col)), shape=self.shape)
+
+    def reshape(self, *shape, order='C'):
+        """Gives a new shape to a sparse matrix without changing its data.
+
+        Args:
+            shape (tuple):
+                The new shape should be compatible with the original shape.
+            order: {'C', 'F'} (optional)
+                Read the elements using this index order. 'C' means to read and
+                write the elements using C-like index order. 'F' means to read
+                and write the elements using Fortran-like index order. Default:
+                C.
+
+        Returns:
+            cupyx.scipy.sparse.coo_matrix: sparse matrix
+
+        """
+
+        shape = _sputils.check_shape(shape, self.shape)
+
+        if shape == self.shape:
+            return self
+
+        nrows, ncols = self.shape
+
+        if order == 'C':  # C to represent matrix in row major format
+            dtype = _sputils.get_index_dtype(
+                maxval=(ncols * max(0, nrows - 1) + max(0, ncols - 1)))
+            flat_indices = cupy.multiply(ncols, self.row,
+                                         dtype=dtype) + self.col
+            new_row, new_col = divmod(flat_indices, shape[1])
+        elif order == 'F':
+            dtype = _sputils.get_index_dtype(
+                maxval=(ncols * max(0, nrows - 1) + max(0, ncols - 1)))
+            flat_indices = cupy.multiply(ncols, self.row,
+                                         dtype=dtype) + self.row
+            new_col, new_row = divmod(flat_indices, shape[0])
+        else:
+            raise ValueError("'order' must be 'C' or 'F'")
+
+        new_data = self.data
+
+        return coo_matrix((new_data, (new_row, new_col)), shape=shape,
+                          copy=False)
+
+    def sum_duplicates(self):
+        """Eliminate duplicate matrix entries by adding them together.
+
+        .. warning::
+            When sorting the indices, CuPy follows the convention of cuSPARSE,
+            which is different from that of SciPy. Therefore, the order of the
+            output indices may differ:
+
+            .. code-block:: python
+
+                >>> #     1 0 0
+                >>> # A = 1 1 0
+                >>> #     1 1 1
+                >>> data = cupy.array([1, 1, 1, 1, 1, 1], 'f')
+                >>> row = cupy.array([0, 1, 1, 2, 2, 2], 'i')
+                >>> col = cupy.array([0, 0, 1, 0, 1, 2], 'i')
+                >>> A = cupyx.scipy.sparse.coo_matrix((data, (row, col)),
+                ...                                   shape=(3, 3))
+                >>> a = A.get()
+                >>> A.sum_duplicates()
+                >>> a.sum_duplicates()  # a is scipy.sparse.coo_matrix
+                >>> A.row
+                array([0, 1, 1, 2, 2, 2], dtype=int32)
+                >>> a.row
+                array([0, 1, 2, 1, 2, 2], dtype=int32)
+                >>> A.col
+                array([0, 0, 1, 0, 1, 2], dtype=int32)
+                >>> a.col
+                array([0, 0, 0, 1, 1, 2], dtype=int32)
+
+        .. warning::
+            Calling this function might synchronize the device.
+
+        .. seealso::
+           :meth:`scipy.sparse.coo_matrix.sum_duplicates`
+
+        """
+        if self.has_canonical_format:
+            return
+        # Note: The sorting order below follows the cuSPARSE convention (first
+        # row then col, so-called row-major) and differs from that of SciPy, as
+        # the cuSPARSE functions such as cusparseSpMV() assume this sorting
+        # order.
+        # See https://docs.nvidia.com/cuda/cusparse/index.html#coo-format
+        keys = cupy.stack([self.col, self.row])
+        order = cupy.lexsort(keys)
+        src_data = self.data[order]
+        src_row = self.row[order]
+        src_col = self.col[order]
+        diff = self._sum_duplicates_diff(src_row, src_col, size=self.row.size)
+
+        if diff[1:].all():
+            # All elements have different indices.
+            data = src_data
+            row = src_row
+            col = src_col
+        else:
+            # TODO(leofang): move the kernels outside this method
+            index = cupy.cumsum(diff, dtype='i')
+            size = int(index[-1]) + 1
+            data = cupy.zeros(size, dtype=self.data.dtype)
+            row = cupy.empty(size, dtype='i')
+            col = cupy.empty(size, dtype='i')
+            if self.data.dtype.kind == 'f':
+                cupy.ElementwiseKernel(
+                    'T src_data, int32 src_row, int32 src_col, int32 index',
+                    'raw T data, raw int32 row, raw int32 col',
+                    '''
+                    atomicAdd(&data[index], src_data);
+                    row[index] = src_row;
+                    col[index] = src_col;
+                    ''',
+                    'cupyx_scipy_sparse_coo_sum_duplicates_assign'
+                )(src_data, src_row, src_col, index, data, row, col)
+            elif self.data.dtype.kind == 'c':
+                cupy.ElementwiseKernel(
+                    'T src_real, T src_imag, int32 src_row, int32 src_col, '
+                    'int32 index',
+                    'raw T real, raw T imag, raw int32 row, raw int32 col',
+                    '''
+                    atomicAdd(&real[index], src_real);
+                    atomicAdd(&imag[index], src_imag);
+                    row[index] = src_row;
+                    col[index] = src_col;
+                    ''',
+                    'cupyx_scipy_sparse_coo_sum_duplicates_assign_complex'
+                )(src_data.real, src_data.imag, src_row, src_col, index,
+                  data.real, data.imag, row, col)
+
+        self.data = data
+        self.row = row
+        self.col = col
+        self.has_canonical_format = True
+
+    def toarray(self, order=None, out=None):
+        """Returns a dense matrix representing the same value.
+
+        Args:
+            order (str): Not supported.
+            out: Not supported.
+
+        Returns:
+            cupy.ndarray: Dense array representing the same value.
+
+        .. seealso:: :meth:`scipy.sparse.coo_matrix.toarray`
+
+        """
+        return self.tocsr().toarray(order=order, out=out)
+
+    def tocoo(self, copy=False):
+        """Converts the matrix to COOdinate format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible.
+
+        Returns:
+            cupyx.scipy.sparse.coo_matrix: Converted matrix.
+
+        """
+        if copy:
+            return self.copy()
+        else:
+            return self
+
+    def tocsc(self, copy=False):
+        """Converts the matrix to Compressed Sparse Column format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible. Actually this option is ignored because all
+                arrays in a matrix cannot be shared in coo to csc conversion.
+
+        Returns:
+            cupyx.scipy.sparse.csc_matrix: Converted matrix.
+
+        """
+        if self.nnz == 0:
+            return _csc.csc_matrix(self.shape, dtype=self.dtype)
+        # copy is silently ignored (in line with SciPy) because both
+        # sum_duplicates and coosort change the underlying data
+        x = self.copy()
+        x.sum_duplicates()
+        cusparse.coosort(x, 'c')
+        x = cusparse.coo2csc(x)
+        x.has_canonical_format = True
+        return x
+
+    def tocsr(self, copy=False):
+        """Converts the matrix to Compressed Sparse Row format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible. Actually this option is ignored because all
+                arrays in a matrix cannot be shared in coo to csr conversion.
+
+        Returns:
+            cupyx.scipy.sparse.csr_matrix: Converted matrix.
+
+        """
+        if self.nnz == 0:
+            return _csr.csr_matrix(self.shape, dtype=self.dtype)
+        # copy is silently ignored (in line with SciPy) because both
+        # sum_duplicates and coosort change the underlying data
+        x = self.copy()
+        x.sum_duplicates()
+        cusparse.coosort(x, 'r')
+        x = cusparse.coo2csr(x)
+        x.has_canonical_format = True
+        return x
+
+    def transpose(self, axes=None, copy=False):
+        """Returns a transpose matrix.
+
+        Args:
+            axes: This option is not supported.
+            copy (bool): If ``True``, a returned matrix shares no data.
+                Otherwise, it shared data arrays as much as possible.
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix: Transpose matrix.
+
+        """
+        if axes is not None:
+            raise ValueError(
+                'Sparse matrices do not support an \'axes\' parameter because '
+                'swapping dimensions is the only logical permutation.')
+        shape = self.shape[1], self.shape[0]
+        return coo_matrix(
+            (self.data, (self.col, self.row)), shape=shape, copy=copy)
+
+
+def isspmatrix_coo(x):
+    """Checks if a given matrix is of COO format.
+
+    Returns:
+        bool: Returns if ``x`` is :class:`cupyx.scipy.sparse.coo_matrix`.
+
+    """
+    return isinstance(x, coo_matrix)
diff --git a/cupyx/scipy/sparse/_csc.py b/cupyx/scipy/sparse/_csc.py
new file mode 100644
index 0000000..df0e9a5
--- /dev/null
+++ b/cupyx/scipy/sparse/_csc.py
@@ -0,0 +1,400 @@
+try:
+    import scipy.sparse
+    _scipy_available = True
+except ImportError:
+    _scipy_available = False
+
+import cupy
+from cupyx import cusparse
+from cupy_backends.cuda.api import driver
+from cupy_backends.cuda.api import runtime
+import cupyx.scipy.sparse
+from cupyx.scipy.sparse import _base
+from cupyx.scipy.sparse import _compressed
+
+
+class csc_matrix(_compressed._compressed_sparse_matrix):
+
+    """Compressed Sparse Column matrix.
+
+    This can be instantiated in several ways.
+
+    ``csc_matrix(D)``
+        ``D`` is a rank-2 :class:`cupy.ndarray`.
+    ``csc_matrix(S)``
+        ``S`` is another sparse matrix. It is equivalent to ``S.tocsc()``.
+    ``csc_matrix((M, N), [dtype])``
+        It constructs an empty matrix whose shape is ``(M, N)``. Default dtype
+        is float64.
+    ``csc_matrix((data, (row, col)))``
+        All ``data``, ``row`` and ``col`` are one-dimenaional
+        :class:`cupy.ndarray`.
+    ``csc_matrix((data, indices, indptr))``
+        All ``data``, ``indices`` and ``indptr`` are one-dimenaional
+        :class:`cupy.ndarray`.
+
+    Args:
+        arg1: Arguments for the initializer.
+        shape (tuple): Shape of a matrix. Its length must be two.
+        dtype: Data type. It must be an argument of :class:`numpy.dtype`.
+        copy (bool): If ``True``, copies of given arrays are always used.
+
+    .. seealso::
+        :class:`scipy.sparse.csc_matrix`
+
+    """
+
+    format = 'csc'
+
+    def get(self, stream=None):
+        """Returns a copy of the array on host memory.
+
+        .. warning::
+           You need to install SciPy to use this method.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+
+        Returns:
+            scipy.sparse.csc_matrix: Copy of the array on host memory.
+
+        """
+        if not _scipy_available:
+            raise RuntimeError('scipy is not available')
+        data = self.data.get(stream)
+        indices = self.indices.get(stream)
+        indptr = self.indptr.get(stream)
+        return scipy.sparse.csc_matrix(
+            (data, indices, indptr), shape=self._shape)
+
+    def _convert_dense(self, x):
+        if cusparse.check_availability('denseToSparse'):
+            m = cusparse.denseToSparse(x, format='csc')
+        else:
+            m = cusparse.dense2csc(x)
+        return m.data, m.indices, m.indptr
+
+    def _swap(self, x, y):
+        return (y, x)
+
+    def __mul__(self, other):
+        if cupy.isscalar(other):
+            self.sum_duplicates()
+            return self._with_data(self.data * other)
+        elif cupyx.scipy.sparse.isspmatrix_csr(other):
+            self.sum_duplicates()
+            other.sum_duplicates()
+            if cusparse.check_availability('spgemm'):
+                a = self.tocsr()
+                a.sum_duplicates()
+                return cusparse.spgemm(a, other)
+            elif cusparse.check_availability('csrgemm') and not runtime.is_hip:
+                # trans=True is still buggy as of ROCm 4.2.0
+                a = self.T
+                return cusparse.csrgemm(a, other, transa=True)
+            elif cusparse.check_availability('csrgemm2'):
+                a = self.tocsr()
+                a.sum_duplicates()
+                return cusparse.csrgemm2(a, other)
+            else:
+                raise NotImplementedError
+        elif isspmatrix_csc(other):
+            self.sum_duplicates()
+            other.sum_duplicates()
+            if cusparse.check_availability('csrgemm') and not runtime.is_hip:
+                # trans=True is still buggy as of ROCm 4.2.0
+                a = self.T
+                b = other.T
+                return cusparse.csrgemm(a, b, transa=True, transb=True)
+            elif cusparse.check_availability('csrgemm2'):
+                a = self.tocsr()
+                b = other.tocsr()
+                a.sum_duplicates()
+                b.sum_duplicates()
+                return cusparse.csrgemm2(a, b)
+            elif cusparse.check_availability('spgemm'):
+                a = self.tocsr()
+                b = other.tocsr()
+                a.sum_duplicates()
+                b.sum_duplicates()
+                return cusparse.spgemm(a, b)
+            else:
+                raise NotImplementedError
+        elif cupyx.scipy.sparse.isspmatrix(other):
+            return self * other.tocsr()
+        elif _base.isdense(other):
+            if other.ndim == 0:
+                self.sum_duplicates()
+                return self._with_data(self.data * other)
+            elif other.ndim == 1:
+                self.sum_duplicates()
+                if (
+                    cusparse.check_availability('csrmv')
+                    and (
+                        not runtime.is_hip
+                        or driver.get_build_version() >= 5_00_00000
+                    )
+                ):
+                    # trans=True is buggy as of ROCm 4.2.0
+                    csrmv = cusparse.csrmv
+                elif (cusparse.check_availability('spmv')
+                        and not runtime.is_hip):
+                    # trans=True is buggy as of ROCm 4.2.0
+                    # (I got HIPSPARSE_STATUS_INTERNAL_ERROR...)
+                    csrmv = cusparse.spmv
+                else:
+                    raise NotImplementedError
+                return csrmv(self.T, cupy.asfortranarray(other), transa=True)
+            elif other.ndim == 2:
+                self.sum_duplicates()
+                if (
+                    cusparse.check_availability('csrmm2')
+                    and (
+                        not runtime.is_hip
+                        or driver.get_build_version() >= 5_00_00000
+                    )
+                ):
+                    # trans=True is buggy as of ROCm 4.2.0
+                    csrmm = cusparse.csrmm2
+                elif cusparse.check_availability('spmm'):
+                    csrmm = cusparse.spmm
+                else:
+                    raise NotImplementedError
+                return csrmm(self.T, cupy.asfortranarray(other), transa=True)
+            else:
+                raise ValueError('could not interpret dimensions')
+        else:
+            return NotImplemented
+
+    # TODO(unno): Implement check_format
+    # TODO(unno): Implement diagonal
+
+    def eliminate_zeros(self):
+        """Removes zero entories in place."""
+        t = self.T
+        t.eliminate_zeros()
+        compress = t.T
+        self.data = compress.data
+        self.indices = compress.indices
+        self.indptr = compress.indptr
+
+    # TODO(unno): Implement maximum
+    # TODO(unno): Implement minimum
+    # TODO(unno): Implement multiply
+    # TODO(unno): Implement prune
+
+    def sort_indices(self):
+        """Sorts the indices of this matrix *in place*.
+
+        .. warning::
+            Calling this function might synchronize the device.
+
+        """
+        if not self.has_sorted_indices:
+            cusparse.cscsort(self)
+            self.has_sorted_indices = True
+
+    def toarray(self, order=None, out=None):
+        """Returns a dense matrix representing the same value.
+
+        Args:
+            order ({'C', 'F', None}): Whether to store data in C (row-major)
+                order or F (column-major) order. Default is C-order.
+            out: Not supported.
+
+        Returns:
+            cupy.ndarray: Dense array representing the same matrix.
+
+        .. seealso:: :meth:`scipy.sparse.csc_matrix.toarray`
+
+        """
+        if order is None:
+            order = 'C'
+        order = order.upper()
+        if self.nnz == 0:
+            return cupy.zeros(shape=self.shape, dtype=self.dtype, order=order)
+
+        x = self.copy()
+        x.has_canonical_format = False  # need to enforce sum_duplicates
+        x.sum_duplicates()
+        if (cusparse.check_availability('sparseToDense')
+                and (not runtime.is_hip or x.nnz > 0)):
+            # On HIP, nnz=0 is problematic as of ROCm 4.2.0
+            y = cusparse.sparseToDense(x)
+            if order == 'F':
+                return y
+            elif order == 'C':
+                return cupy.ascontiguousarray(y)
+            else:
+                raise ValueError('order not understood')
+        else:
+            # csc2dense and csr2dense returns F-contiguous array.
+            if order == 'C':
+                # To return C-contiguous array, it uses transpose.
+                return cusparse.csr2dense(x.T).T
+            elif order == 'F':
+                return cusparse.csc2dense(x)
+            else:
+                raise ValueError('order not understood')
+
+    def _add_sparse(self, other, alpha, beta):
+        self.sum_duplicates()
+        other = other.tocsc().T
+        other.sum_duplicates()
+        if cusparse.check_availability('csrgeam2'):
+            csrgeam = cusparse.csrgeam2
+        elif cusparse.check_availability('csrgeam'):
+            csrgeam = cusparse.csrgeam
+        else:
+            raise NotImplementedError
+        return csrgeam(self.T, other, alpha, beta).T
+
+    # TODO(unno): Implement tobsr
+
+    def tocoo(self, copy=False):
+        """Converts the matrix to COOdinate format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible.
+
+        Returns:
+            cupyx.scipy.sparse.coo_matrix: Converted matrix.
+
+        """
+        if copy:
+            data = self.data.copy()
+            indices = self.indices.copy()
+        else:
+            data = self.data
+            indices = self.indices
+
+        return cusparse.csc2coo(self, data, indices)
+
+    def tocsc(self, copy=None):
+        """Converts the matrix to Compressed Sparse Column format.
+
+        Args:
+            copy (bool): If ``False``, the method returns itself.
+                Otherwise it makes a copy of the matrix.
+
+        Returns:
+            cupyx.scipy.sparse.csc_matrix: Converted matrix.
+
+        """
+        if copy:
+            return self.copy()
+        else:
+            return self
+
+    def tocsr(self, copy=False):
+        """Converts the matrix to Compressed Sparse Row format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible. Actually this option is ignored because all
+                arrays in a matrix cannot be shared in csr to csc conversion.
+
+        Returns:
+            cupyx.scipy.sparse.csr_matrix: Converted matrix.
+
+        """
+        # copy is ignored
+        if cusparse.check_availability('csc2csr'):
+            csc2csr = cusparse.csc2csr
+        elif cusparse.check_availability('csc2csrEx2'):
+            csc2csr = cusparse.csc2csrEx2
+        else:
+            raise NotImplementedError
+        # don't touch has_sorted_indices, as cuSPARSE made no guarantee
+        return csc2csr(self)
+
+    def _tocsx(self):
+        """Inverts the format.
+        """
+        return self.tocsr()
+
+    # TODO(unno): Implement todia
+    # TODO(unno): Implement todok
+    # TODO(unno): Implement tolil
+
+    def transpose(self, axes=None, copy=False):
+        """Returns a transpose matrix.
+
+        Args:
+            axes: This option is not supported.
+            copy (bool): If ``True``, a returned matrix shares no data.
+                Otherwise, it shared data arrays as much as possible.
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix: Transpose matrix.
+
+        """
+        if axes is not None:
+            raise ValueError(
+                'Sparse matrices do not support an \'axes\' parameter because '
+                'swapping dimensions is the only logical permutation.')
+
+        shape = self.shape[1], self.shape[0]
+        trans = cupyx.scipy.sparse.csr_matrix(
+            (self.data, self.indices, self.indptr), shape=shape, copy=copy)
+        trans.has_canonical_format = self.has_canonical_format
+        return trans
+
+    def getrow(self, i):
+        """Returns a copy of row i of the matrix, as a (1 x n)
+        CSR matrix (row vector).
+
+        Args:
+            i (integer): Row
+
+        Returns:
+            cupyx.scipy.sparse.csc_matrix: Sparse matrix with single row
+        """
+        return self._minor_slice(slice(i, i + 1), copy=True).tocsr()
+
+    def getcol(self, i):
+        """Returns a copy of column i of the matrix, as a (m x 1)
+        CSC matrix (column vector).
+
+        Args:
+            i (integer): Column
+
+        Returns:
+            cupyx.scipy.sparse.csc_matrix: Sparse matrix with single column
+        """
+        return self._major_slice(slice(i, i + 1), copy=True)
+
+    def _get_intXarray(self, row, col):
+        row = slice(row, row + 1)
+        return self._major_index_fancy(col)._minor_slice(row)
+
+    def _get_intXslice(self, row, col):
+        row = slice(row, row + 1)
+        copy = col.step in (1, None)
+        return self._major_slice(col)._minor_slice(row, copy=copy)
+
+    def _get_sliceXint(self, row, col):
+        col = slice(col, col + 1)
+        return self._major_slice(col)._minor_slice(row, copy=True)
+
+    def _get_sliceXarray(self, row, col):
+        return self._major_index_fancy(col)._minor_slice(row)
+
+    def _get_arrayXint(self, row, col):
+        col = slice(col, col + 1)
+        return self._major_slice(col)._minor_index_fancy(row)
+
+    def _get_arrayXslice(self, row, col):
+        return self._major_slice(col)._minor_index_fancy(row)
+
+
+def isspmatrix_csc(x):
+    """Checks if a given matrix is of CSC format.
+
+    Returns:
+        bool: Returns if ``x`` is :class:`cupyx.scipy.sparse.csc_matrix`.
+
+    """
+    return isinstance(x, csc_matrix)
diff --git a/cupyx/scipy/sparse/_csr.py b/cupyx/scipy/sparse/_csr.py
new file mode 100644
index 0000000..102b495
--- /dev/null
+++ b/cupyx/scipy/sparse/_csr.py
@@ -0,0 +1,1211 @@
+import operator
+import warnings
+
+import numpy
+
+try:
+    import scipy.sparse
+    _scipy_available = True
+except ImportError:
+    _scipy_available = False
+
+import cupy
+from cupy._core import _accelerator
+from cupy.cuda import cub
+from cupy.cuda import runtime
+from cupyx import cusparse
+from cupyx.scipy.sparse import _base
+from cupyx.scipy.sparse import _compressed
+from cupyx.scipy.sparse import _csc
+from cupyx.scipy.sparse import SparseEfficiencyWarning
+from cupyx.scipy.sparse import _util
+
+
+class csr_matrix(_compressed._compressed_sparse_matrix):
+
+    """Compressed Sparse Row matrix.
+
+    This can be instantiated in several ways.
+
+    ``csr_matrix(D)``
+        ``D`` is a rank-2 :class:`cupy.ndarray`.
+    ``csr_matrix(S)``
+        ``S`` is another sparse matrix. It is equivalent to ``S.tocsr()``.
+    ``csr_matrix((M, N), [dtype])``
+        It constructs an empty matrix whose shape is ``(M, N)``. Default dtype
+        is float64.
+    ``csr_matrix((data, (row, col)))``
+        All ``data``, ``row`` and ``col`` are one-dimenaional
+        :class:`cupy.ndarray`.
+    ``csr_matrix((data, indices, indptr))``
+        All ``data``, ``indices`` and ``indptr`` are one-dimenaional
+        :class:`cupy.ndarray`.
+
+    Args:
+        arg1: Arguments for the initializer.
+        shape (tuple): Shape of a matrix. Its length must be two.
+        dtype: Data type. It must be an argument of :class:`numpy.dtype`.
+        copy (bool): If ``True``, copies of given arrays are always used.
+
+    .. seealso::
+        :class:`scipy.sparse.csr_matrix`
+
+    """
+
+    format = 'csr'
+
+    def get(self, stream=None):
+        """Returns a copy of the array on host memory.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+
+        Returns:
+            scipy.sparse.csr_matrix: Copy of the array on host memory.
+
+        """
+        if not _scipy_available:
+            raise RuntimeError('scipy is not available')
+        data = self.data.get(stream)
+        indices = self.indices.get(stream)
+        indptr = self.indptr.get(stream)
+        return scipy.sparse.csr_matrix(
+            (data, indices, indptr), shape=self._shape)
+
+    def _convert_dense(self, x):
+        m = dense2csr(x)
+        return m.data, m.indices, m.indptr
+
+    def _swap(self, x, y):
+        return (x, y)
+
+    def _add_sparse(self, other, alpha, beta):
+        self.sum_duplicates()
+        other = other.tocsr()
+        other.sum_duplicates()
+        if cusparse.check_availability('csrgeam2'):
+            csrgeam = cusparse.csrgeam2
+        elif cusparse.check_availability('csrgeam'):
+            csrgeam = cusparse.csrgeam
+        else:
+            raise NotImplementedError
+        return csrgeam(self, other, alpha, beta)
+
+    def _comparison(self, other, op, op_name):
+        if _util.isscalarlike(other):
+            data = cupy.asarray(other, dtype=self.dtype).reshape(1)
+            if numpy.isnan(data[0]):
+                if op_name == '_ne_':
+                    return csr_matrix(cupy.ones(self.shape, dtype=numpy.bool_))
+                else:
+                    return csr_matrix(self.shape, dtype=numpy.bool_)
+            indices = cupy.zeros((1,), dtype=numpy.int32)
+            indptr = cupy.arange(2, dtype=numpy.int32)
+            other = csr_matrix((data, indices, indptr), shape=(1, 1))
+            return binopt_csr(self, other, op_name)
+        elif _util.isdense(other):
+            return op(self.todense(), other)
+        elif isspmatrix_csr(other):
+            self.sum_duplicates()
+            other.sum_duplicates()
+            if op_name in ('_ne_', '_lt_', '_gt_'):
+                return binopt_csr(self, other, op_name)
+
+            warnings.warn(
+                "Comparing sparse matrices using ==, <=, and >= is "
+                "inefficient, try using !=, <, or > instead.",
+                SparseEfficiencyWarning)
+            if op_name == '_eq_':
+                opposite_op_name = '_ne_'
+            elif op_name == '_le_':
+                opposite_op_name = '_gt_'
+            elif op_name == '_ge_':
+                opposite_op_name = '_lt_'
+            res = binopt_csr(self, other, opposite_op_name)
+            out = cupy.logical_not(res.toarray())
+            return csr_matrix(out)
+        raise NotImplementedError
+
+    def __eq__(self, other):
+        return self._comparison(other, operator.eq, '_eq_')
+
+    def __ne__(self, other):
+        return self._comparison(other, operator.ne, '_ne_')
+
+    def __lt__(self, other):
+        return self._comparison(other, operator.lt, '_lt_')
+
+    def __gt__(self, other):
+        return self._comparison(other, operator.gt, '_gt_')
+
+    def __le__(self, other):
+        return self._comparison(other, operator.le, '_le_')
+
+    def __ge__(self, other):
+        return self._comparison(other, operator.ge, '_ge_')
+
+    def __mul__(self, other):
+        if cupy.isscalar(other):
+            self.sum_duplicates()
+            return self._with_data(self.data * other)
+        elif isspmatrix_csr(other):
+            self.sum_duplicates()
+            other.sum_duplicates()
+            if cusparse.check_availability('spgemm'):
+                return cusparse.spgemm(self, other)
+            elif cusparse.check_availability('csrgemm2'):
+                return cusparse.csrgemm2(self, other)
+            elif cusparse.check_availability('csrgemm'):
+                return cusparse.csrgemm(self, other)
+            else:
+                raise NotImplementedError
+        elif _csc.isspmatrix_csc(other):
+            self.sum_duplicates()
+            other.sum_duplicates()
+            if cusparse.check_availability('csrgemm') and not runtime.is_hip:
+                # trans=True is still buggy as of ROCm 4.2.0
+                return cusparse.csrgemm(self, other.T, transb=True)
+            elif cusparse.check_availability('spgemm'):
+                b = other.tocsr()
+                b.sum_duplicates()
+                return cusparse.spgemm(self, b)
+            elif cusparse.check_availability('csrgemm2'):
+                b = other.tocsr()
+                b.sum_duplicates()
+                return cusparse.csrgemm2(self, b)
+            else:
+                raise NotImplementedError
+        elif _base.isspmatrix(other):
+            return self * other.tocsr()
+        elif _base.isdense(other):
+            if other.ndim == 0:
+                self.sum_duplicates()
+                return self._with_data(self.data * other)
+            elif other.ndim == 1:
+                self.sum_duplicates()
+                other = cupy.asfortranarray(other)
+                # need extra padding to ensure not stepping on the CUB bug,
+                # see cupy/cupy#3679 for discussion
+                is_cub_safe = (self.indptr.data.mem.size
+                               > self.indptr.size * self.indptr.dtype.itemsize)
+                # CUB spmv is buggy since CUDA 11.0, see
+                # https://github.com/cupy/cupy/issues/3822#issuecomment-782607637
+                is_cub_safe &= (cub._get_cuda_build_version() < 11000)
+                for accelerator in _accelerator.get_routine_accelerators():
+                    if (accelerator == _accelerator.ACCELERATOR_CUB
+                            and not runtime.is_hip
+                            and is_cub_safe and other.flags.c_contiguous):
+                        return cub.device_csrmv(
+                            self.shape[0], self.shape[1], self.nnz,
+                            self.data, self.indptr, self.indices, other)
+                if (cusparse.check_availability('csrmvEx') and self.nnz > 0 and
+                        cusparse.csrmvExIsAligned(self, other)):
+                    # csrmvEx does not work if nnz == 0
+                    csrmv = cusparse.csrmvEx
+                elif cusparse.check_availability('csrmv'):
+                    csrmv = cusparse.csrmv
+                elif cusparse.check_availability('spmv'):
+                    csrmv = cusparse.spmv
+                else:
+                    raise NotImplementedError
+                return csrmv(self, other)
+            elif other.ndim == 2:
+                self.sum_duplicates()
+                if cusparse.check_availability('csrmm2'):
+                    csrmm = cusparse.csrmm2
+                elif cusparse.check_availability('spmm'):
+                    csrmm = cusparse.spmm
+                else:
+                    raise NotImplementedError
+                return csrmm(self, cupy.asfortranarray(other))
+            else:
+                raise ValueError('could not interpret dimensions')
+        else:
+            return NotImplemented
+
+    def __div__(self, other):
+        raise NotImplementedError
+
+    def __rdiv__(self, other):
+        raise NotImplementedError
+
+    def __truediv__(self, other):
+        """Point-wise division by another matrix, vector or scalar"""
+        if _util.isscalarlike(other):
+            dtype = self.dtype
+            if dtype == numpy.float32:
+                # Note: This is a work-around to make the output dtype the same
+                # as SciPy. It might be SciPy version dependent.
+                dtype = numpy.float64
+            dtype = cupy.result_type(dtype, other)
+            d = cupy.reciprocal(other, dtype=dtype)
+            return multiply_by_scalar(self, d)
+        elif _util.isdense(other):
+            other = cupy.atleast_2d(other)
+            check_shape_for_pointwise_op(self.shape, other.shape)
+            return self.todense() / other
+        elif _base.isspmatrix(other):
+            # Note: If broadcasting is needed, an exception is raised here for
+            # compatibility with SciPy, as SciPy does not support broadcasting
+            # in the "sparse / sparse" case.
+            check_shape_for_pointwise_op(self.shape, other.shape,
+                                         allow_broadcasting=False)
+            dtype = numpy.promote_types(self.dtype, other.dtype)
+            if dtype.char not in 'FD':
+                dtype = numpy.promote_types(numpy.float64, dtype)
+            # Note: The following implementation converts two sparse matrices
+            # into dense matrices and then performs a point-wise division,
+            # which can use lots of memory.
+            self_dense = self.todense().astype(dtype, copy=False)
+            return self_dense / other.todense()
+        raise NotImplementedError
+
+    def __rtruediv__(self, other):
+        raise NotImplementedError
+
+    # TODO(unno): Implement check_format
+
+    def diagonal(self, k=0):
+        rows, cols = self.shape
+        ylen = min(rows + min(k, 0), cols - max(k, 0))
+        if ylen <= 0:
+            return cupy.empty(0, dtype=self.dtype)
+        self.sum_duplicates()
+        y = cupy.empty(ylen, dtype=self.dtype)
+        _cupy_csr_diagonal()(k, rows, cols, self.data, self.indptr,
+                             self.indices, y)
+        return y
+
+    def eliminate_zeros(self):
+        """Removes zero entories in place."""
+        compress = cusparse.csr2csr_compress(self, 0)
+        self.data = compress.data
+        self.indices = compress.indices
+        self.indptr = compress.indptr
+
+    def _maximum_minimum(self, other, cupy_op, op_name, dense_check):
+        if _util.isscalarlike(other):
+            other = cupy.asarray(other, dtype=self.dtype)
+            if dense_check(other):
+                dtype = self.dtype
+                # Note: This is a work-around to make the output dtype the same
+                # as SciPy. It might be SciPy version dependent.
+                if dtype == numpy.float32:
+                    dtype = numpy.float64
+                elif dtype == numpy.complex64:
+                    dtype = numpy.complex128
+                dtype = cupy.result_type(dtype, other)
+                other = other.astype(dtype, copy=False)
+                # Note: The computation steps below are different from SciPy.
+                new_array = cupy_op(self.todense(), other)
+                return csr_matrix(new_array)
+            else:
+                self.sum_duplicates()
+                new_data = cupy_op(self.data, other)
+                return csr_matrix((new_data, self.indices, self.indptr),
+                                  shape=self.shape, dtype=self.dtype)
+        elif _util.isdense(other):
+            self.sum_duplicates()
+            other = cupy.atleast_2d(other)
+            return cupy_op(self.todense(), other)
+        elif isspmatrix_csr(other):
+            self.sum_duplicates()
+            other.sum_duplicates()
+            return binopt_csr(self, other, op_name)
+        raise NotImplementedError
+
+    def maximum(self, other):
+        return self._maximum_minimum(other, cupy.maximum, '_maximum_',
+                                     lambda x: x > 0)
+
+    def minimum(self, other):
+        return self._maximum_minimum(other, cupy.minimum, '_minimum_',
+                                     lambda x: x < 0)
+
+    def multiply(self, other):
+        """Point-wise multiplication by another matrix, vector or scalar"""
+        if cupy.isscalar(other):
+            return multiply_by_scalar(self, other)
+        elif _util.isdense(other):
+            self.sum_duplicates()
+            other = cupy.atleast_2d(other)
+            return multiply_by_dense(self, other)
+        elif isspmatrix_csr(other):
+            self.sum_duplicates()
+            other.sum_duplicates()
+            return multiply_by_csr(self, other)
+        else:
+            msg = 'expected scalar, dense matrix/vector or csr matrix'
+            raise TypeError(msg)
+
+    # TODO(unno): Implement prune
+
+    def setdiag(self, values, k=0):
+        """Set diagonal or off-diagonal elements of the array."""
+        rows, cols = self.shape
+        row_st, col_st = max(0, -k), max(0, k)
+        x_len = min(rows - row_st, cols - col_st)
+        if x_len <= 0:
+            raise ValueError('k exceeds matrix dimensions')
+        values = values.astype(self.dtype)
+        if values.ndim == 0:
+            # broadcast
+            x_data = cupy.full((x_len,), values, dtype=self.dtype)
+        else:
+            x_len = min(x_len, values.size)
+            x_data = values[:x_len]
+        x_indices = cupy.arange(col_st, col_st + x_len, dtype='i')
+        x_indptr = cupy.zeros((rows + 1,), dtype='i')
+        x_indptr[row_st:row_st+x_len+1] = cupy.arange(x_len+1, dtype='i')
+        x_indptr[row_st+x_len+1:] = x_len
+        x_data -= self.diagonal(k=k)[:x_len]
+        y = self + csr_matrix((x_data, x_indices, x_indptr), shape=self.shape)
+        self.data = y.data
+        self.indices = y.indices
+        self.indptr = y.indptr
+
+    def sort_indices(self):
+        """Sorts the indices of this matrix *in place*.
+
+        .. warning::
+            Calling this function might synchronize the device.
+
+        """
+        if not self.has_sorted_indices:
+            cusparse.csrsort(self)
+            self.has_sorted_indices = True
+
+    def toarray(self, order=None, out=None):
+        """Returns a dense matrix representing the same value.
+
+        Args:
+            order ({'C', 'F', None}): Whether to store data in C (row-major)
+                order or F (column-major) order. Default is C-order.
+            out: Not supported.
+
+        Returns:
+            cupy.ndarray: Dense array representing the same matrix.
+
+        .. seealso:: :meth:`scipy.sparse.csr_matrix.toarray`
+
+        """
+        order = 'C' if order is None else order.upper()
+        if self.nnz == 0:
+            return cupy.zeros(shape=self.shape, dtype=self.dtype, order=order)
+
+        if self.dtype.char not in 'fdFD':
+            return csr2dense(self, order)
+
+        x = self.copy()
+        x.has_canonical_format = False  # need to enforce sum_duplicates
+        x.sum_duplicates()
+        if (cusparse.check_availability('sparseToDense')
+                and (not runtime.is_hip or (x.nnz > 0))):
+            # On HIP, nnz=0 is problematic as of ROCm 4.2.0
+            y = cusparse.sparseToDense(x)
+            if order == 'F':
+                return y
+            elif order == 'C':
+                return cupy.ascontiguousarray(y)
+            else:
+                raise ValueError('order not understood')
+        else:
+            # csr2dense returns F-contiguous array.
+            if order == 'C':
+                # To return C-contiguous array, it uses transpose.
+                return cusparse.csc2dense(x.T).T
+            elif order == 'F':
+                return cusparse.csr2dense(x)
+            else:
+                raise ValueError('order not understood')
+
+    def tobsr(self, blocksize=None, copy=False):
+        # TODO(unno): Implement tobsr
+        raise NotImplementedError
+
+    def tocoo(self, copy=False):
+        """Converts the matrix to COOdinate format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible.
+
+        Returns:
+            cupyx.scipy.sparse.coo_matrix: Converted matrix.
+
+        """
+        if copy:
+            data = self.data.copy()
+            indices = self.indices.copy()
+        else:
+            data = self.data
+            indices = self.indices
+
+        return cusparse.csr2coo(self, data, indices)
+
+    def tocsc(self, copy=False):
+        """Converts the matrix to Compressed Sparse Column format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible. Actually this option is ignored because all
+                arrays in a matrix cannot be shared in csr to csc conversion.
+
+        Returns:
+            cupyx.scipy.sparse.csc_matrix: Converted matrix.
+
+        """
+        # copy is ignored
+        if cusparse.check_availability('csr2csc'):
+            csr2csc = cusparse.csr2csc
+        elif cusparse.check_availability('csr2cscEx2'):
+            csr2csc = cusparse.csr2cscEx2
+        else:
+            raise NotImplementedError
+        # don't touch has_sorted_indices, as cuSPARSE made no guarantee
+        return csr2csc(self)
+
+    def tocsr(self, copy=False):
+        """Converts the matrix to Compressed Sparse Row format.
+
+        Args:
+            copy (bool): If ``False``, the method returns itself.
+                Otherwise it makes a copy of the matrix.
+
+        Returns:
+            cupyx.scipy.sparse.csr_matrix: Converted matrix.
+
+        """
+        if copy:
+            return self.copy()
+        else:
+            return self
+
+    def _tocsx(self):
+        """Inverts the format.
+        """
+        return self.tocsc()
+
+    def todia(self, copy=False):
+        # TODO(unno): Implement todia
+        raise NotImplementedError
+
+    def todok(self, copy=False):
+        # TODO(unno): Implement todok
+        raise NotImplementedError
+
+    def tolil(self, copy=False):
+        # TODO(unno): Implement tolil
+        raise NotImplementedError
+
+    def transpose(self, axes=None, copy=False):
+        """Returns a transpose matrix.
+
+        Args:
+            axes: This option is not supported.
+            copy (bool): If ``True``, a returned matrix shares no data.
+                Otherwise, it shared data arrays as much as possible.
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix: Transpose matrix.
+
+        """
+        if axes is not None:
+            raise ValueError(
+                'Sparse matrices do not support an \'axes\' parameter because '
+                'swapping dimensions is the only logical permutation.')
+
+        shape = self.shape[1], self.shape[0]
+        trans = _csc.csc_matrix(
+            (self.data, self.indices, self.indptr), shape=shape, copy=copy)
+        trans.has_canonical_format = self.has_canonical_format
+        return trans
+
+    def getrow(self, i):
+        """Returns a copy of row i of the matrix, as a (1 x n)
+        CSR matrix (row vector).
+
+        Args:
+            i (integer): Row
+
+        Returns:
+            cupyx.scipy.sparse.csr_matrix: Sparse matrix with single row
+        """
+        return self._major_slice(slice(i, i + 1), copy=True)
+
+    def getcol(self, i):
+        """Returns a copy of column i of the matrix, as a (m x 1)
+        CSR matrix (column vector).
+
+        Args:
+            i (integer): Column
+
+        Returns:
+            cupyx.scipy.sparse.csr_matrix: Sparse matrix with single column
+        """
+        return self._minor_slice(slice(i, i + 1), copy=True)
+
+    def _get_intXarray(self, row, col):
+        row = slice(row, row + 1)
+        return self._major_slice(row)._minor_index_fancy(col)
+
+    def _get_intXslice(self, row, col):
+        row = slice(row, row + 1)
+        return self._major_slice(row)._minor_slice(col, copy=True)
+
+    def _get_sliceXint(self, row, col):
+        col = slice(col, col + 1)
+        copy = row.step in (1, None)
+        return self._major_slice(row)._minor_slice(col, copy=copy)
+
+    def _get_sliceXarray(self, row, col):
+        return self._major_slice(row)._minor_index_fancy(col)
+
+    def _get_arrayXint(self, row, col):
+        col = slice(col, col + 1)
+        return self._major_index_fancy(row)._minor_slice(col)
+
+    def _get_arrayXslice(self, row, col):
+        if col.step not in (1, None):
+            start, stop, step = col.indices(self.shape[1])
+            cols = cupy.arange(start, stop, step, self.indices.dtype)
+            return self._get_arrayXarray(row, cols)
+        return self._major_index_fancy(row)._minor_slice(col)
+
+
+def isspmatrix_csr(x):
+    """Checks if a given matrix is of CSR format.
+
+    Returns:
+        bool: Returns if ``x`` is :class:`cupyx.scipy.sparse.csr_matrix`.
+
+    """
+    return isinstance(x, csr_matrix)
+
+
+def check_shape_for_pointwise_op(a_shape, b_shape, allow_broadcasting=True):
+    if allow_broadcasting:
+        a_m, a_n = a_shape
+        b_m, b_n = b_shape
+        if not (a_m == b_m or a_m == 1 or b_m == 1):
+            raise ValueError('inconsistent shape')
+        if not (a_n == b_n or a_n == 1 or b_n == 1):
+            raise ValueError('inconsistent shape')
+    else:
+        if a_shape != b_shape:
+            raise ValueError('inconsistent shape')
+
+
+def multiply_by_scalar(sp, a):
+    data = sp.data * a
+    indices = sp.indices.copy()
+    indptr = sp.indptr.copy()
+    return csr_matrix((data, indices, indptr), shape=sp.shape)
+
+
+def multiply_by_dense(sp, dn):
+    check_shape_for_pointwise_op(sp.shape, dn.shape)
+    sp_m, sp_n = sp.shape
+    dn_m, dn_n = dn.shape
+    m, n = max(sp_m, dn_m), max(sp_n, dn_n)
+    nnz = sp.nnz * (m // sp_m) * (n // sp_n)
+    dtype = numpy.promote_types(sp.dtype, dn.dtype)
+    data = cupy.empty(nnz, dtype=dtype)
+    indices = cupy.empty(nnz, dtype=sp.indices.dtype)
+    if m > sp_m:
+        if n > sp_n:
+            indptr = cupy.arange(0, nnz+1, n, dtype=sp.indptr.dtype)
+        else:
+            indptr = cupy.arange(0, nnz+1, sp.nnz, dtype=sp.indptr.dtype)
+    else:
+        indptr = sp.indptr.copy()
+        if n > sp_n:
+            indptr *= n
+
+    # out = sp * dn
+    cupy_multiply_by_dense()(sp.data, sp.indptr, sp.indices, sp_m, sp_n,
+                             dn, dn_m, dn_n, indptr, m, n, data, indices)
+
+    return csr_matrix((data, indices, indptr), shape=(m, n))
+
+
+_GET_ROW_ID_ = '''
+__device__ inline int get_row_id(int i, int min, int max, const int *indptr) {
+    int row = (min + max) / 2;
+    while (min < max) {
+        if (i < indptr[row]) {
+            max = row - 1;
+        } else if (i >= indptr[row + 1]) {
+            min = row + 1;
+        } else {
+            break;
+        }
+        row = (min + max) / 2;
+    }
+    return row;
+}
+'''
+
+_FIND_INDEX_HOLDING_COL_IN_ROW_ = '''
+__device__ inline int find_index_holding_col_in_row(
+        int row, int col, const int *indptr, const int *indices) {
+    int j_min = indptr[row];
+    int j_max = indptr[row+1] - 1;
+    while (j_min <= j_max) {
+        int j = (j_min + j_max) / 2;
+        int j_col = indices[j];
+        if (j_col == col) {
+            return j;
+        } else if (j_col < col) {
+            j_min = j + 1;
+        } else {
+            j_max = j - 1;
+        }
+    }
+    return -1;
+}
+'''
+
+
+@cupy._util.memoize(for_each_device=True)
+def cupy_multiply_by_dense():
+    return cupy.ElementwiseKernel(
+        '''
+        raw S SP_DATA, raw I SP_INDPTR, raw I SP_INDICES,
+        int32 SP_M, int32 SP_N,
+        raw D DN_DATA, int32 DN_M, int32 DN_N,
+        raw I OUT_INDPTR, int32 OUT_M, int32 OUT_N
+        ''',
+        'O OUT_DATA, I OUT_INDICES',
+        '''
+        int i_out = i;
+        int m_out = get_row_id(i_out, 0, OUT_M - 1, &(OUT_INDPTR[0]));
+        int i_sp = i_out;
+        if (OUT_M > SP_M && SP_M == 1) {
+            i_sp -= OUT_INDPTR[m_out];
+        }
+        if (OUT_N > SP_N && SP_N == 1) {
+            i_sp /= OUT_N;
+        }
+        int n_out = SP_INDICES[i_sp];
+        if (OUT_N > SP_N && SP_N == 1) {
+            n_out = i_out - OUT_INDPTR[m_out];
+        }
+        int m_dn = m_out;
+        if (OUT_M > DN_M && DN_M == 1) {
+            m_dn = 0;
+        }
+        int n_dn = n_out;
+        if (OUT_N > DN_N && DN_N == 1) {
+            n_dn = 0;
+        }
+        OUT_DATA = (O)(SP_DATA[i_sp] * DN_DATA[n_dn + (DN_N * m_dn)]);
+        OUT_INDICES = n_out;
+        ''',
+        'cupyx_scipy_sparse_csr_multiply_by_dense',
+        preamble=_GET_ROW_ID_
+    )
+
+
+def multiply_by_csr(a, b):
+    check_shape_for_pointwise_op(a.shape, b.shape)
+    a_m, a_n = a.shape
+    b_m, b_n = b.shape
+    m, n = max(a_m, b_m), max(a_n, b_n)
+    a_nnz = a.nnz * (m // a_m) * (n // a_n)
+    b_nnz = b.nnz * (m // b_m) * (n // b_n)
+    if a_nnz > b_nnz:
+        return multiply_by_csr(b, a)
+    c_nnz = a_nnz
+    dtype = numpy.promote_types(a.dtype, b.dtype)
+    c_data = cupy.empty(c_nnz, dtype=dtype)
+    c_indices = cupy.empty(c_nnz, dtype=a.indices.dtype)
+    if m > a_m:
+        if n > a_n:
+            c_indptr = cupy.arange(0, c_nnz+1, n, dtype=a.indptr.dtype)
+        else:
+            c_indptr = cupy.arange(0, c_nnz+1, a.nnz, dtype=a.indptr.dtype)
+    else:
+        c_indptr = a.indptr.copy()
+        if n > a_n:
+            c_indptr *= n
+    flags = cupy.zeros(c_nnz+1, dtype=a.indices.dtype)
+    nnz_each_row = cupy.zeros(m+1, dtype=a.indptr.dtype)
+
+    # compute c = a * b where necessary and get sparsity pattern of matrix d
+    cupy_multiply_by_csr_step1()(
+        a.data, a.indptr, a.indices, a_m, a_n,
+        b.data, b.indptr, b.indices, b_m, b_n,
+        c_indptr, m, n, c_data, c_indices, flags, nnz_each_row)
+
+    flags = cupy.cumsum(flags, dtype=a.indptr.dtype)
+    d_indptr = cupy.cumsum(nnz_each_row, dtype=a.indptr.dtype)
+    d_nnz = int(d_indptr[-1])
+    d_data = cupy.empty(d_nnz, dtype=dtype)
+    d_indices = cupy.empty(d_nnz, dtype=a.indices.dtype)
+
+    # remove zero elements in matric c
+    cupy_multiply_by_csr_step2()(c_data, c_indices, flags, d_data, d_indices)
+
+    return csr_matrix((d_data, d_indices, d_indptr), shape=(m, n))
+
+
+@cupy._util.memoize(for_each_device=True)
+def cupy_multiply_by_csr_step1():
+    return cupy.ElementwiseKernel(
+        '''
+        raw A A_DATA, raw I A_INDPTR, raw I A_INDICES, int32 A_M, int32 A_N,
+        raw B B_DATA, raw I B_INDPTR, raw I B_INDICES, int32 B_M, int32 B_N,
+        raw I C_INDPTR, int32 C_M, int32 C_N
+        ''',
+        'C C_DATA, I C_INDICES, raw I FLAGS, raw I NNZ_EACH_ROW',
+        '''
+        int i_c = i;
+        int m_c = get_row_id(i_c, 0, C_M - 1, &(C_INDPTR[0]));
+
+        int i_a = i;
+        if (C_M > A_M && A_M == 1) {
+            i_a -= C_INDPTR[m_c];
+        }
+        if (C_N > A_N && A_N == 1) {
+            i_a /= C_N;
+        }
+        int n_c = A_INDICES[i_a];
+        if (C_N > A_N && A_N == 1) {
+            n_c = i % C_N;
+        }
+        int m_b = m_c;
+        if (C_M > B_M && B_M == 1) {
+            m_b = 0;
+        }
+        int n_b = n_c;
+        if (C_N > B_N && B_N == 1) {
+            n_b = 0;
+        }
+        int i_b = find_index_holding_col_in_row(m_b, n_b,
+            &(B_INDPTR[0]), &(B_INDICES[0]));
+        if (i_b >= 0) {
+            atomicAdd(&(NNZ_EACH_ROW[m_c+1]), 1);
+            FLAGS[i+1] = 1;
+            C_DATA = (C)(A_DATA[i_a] * B_DATA[i_b]);
+            C_INDICES = n_c;
+        }
+        ''',
+        'cupyx_scipy_sparse_csr_multiply_by_csr_step1',
+        preamble=_GET_ROW_ID_ + _FIND_INDEX_HOLDING_COL_IN_ROW_
+    )
+
+
+@cupy._util.memoize(for_each_device=True)
+def cupy_multiply_by_csr_step2():
+    return cupy.ElementwiseKernel(
+        'T C_DATA, I C_INDICES, raw I FLAGS',
+        'raw D D_DATA, raw I D_INDICES',
+        '''
+        int j = FLAGS[i];
+        if (j < FLAGS[i+1]) {
+            D_DATA[j] = (D)(C_DATA);
+            D_INDICES[j] = C_INDICES;
+        }
+        ''',
+        'cupyx_scipy_sparse_csr_multiply_by_csr_step2'
+    )
+
+
+_BINOPT_MAX_ = '''
+__device__ inline O binopt(T in1, T in2) {
+    return max(in1, in2);
+}
+'''
+_BINOPT_MIN_ = '''
+__device__ inline O binopt(T in1, T in2) {
+    return min(in1, in2);
+}
+'''
+_BINOPT_EQ_ = '''
+__device__ inline O binopt(T in1, T in2) {
+    return (in1 == in2);
+}
+'''
+_BINOPT_NE_ = '''
+__device__ inline O binopt(T in1, T in2) {
+    return (in1 != in2);
+}
+'''
+_BINOPT_LT_ = '''
+__device__ inline O binopt(T in1, T in2) {
+    return (in1 < in2);
+}
+'''
+_BINOPT_GT_ = '''
+__device__ inline O binopt(T in1, T in2) {
+    return (in1 > in2);
+}
+'''
+_BINOPT_LE_ = '''
+__device__ inline O binopt(T in1, T in2) {
+    return (in1 <= in2);
+}
+'''
+_BINOPT_GE_ = '''
+__device__ inline O binopt(T in1, T in2) {
+    return (in1 >= in2);
+}
+'''
+
+
+def binopt_csr(a, b, op_name):
+    check_shape_for_pointwise_op(a.shape, b.shape)
+    a_m, a_n = a.shape
+    b_m, b_n = b.shape
+    m, n = max(a_m, b_m), max(a_n, b_n)
+    a_nnz = a.nnz * (m // a_m) * (n // a_n)
+    b_nnz = b.nnz * (m // b_m) * (n // b_n)
+
+    a_info = cupy.zeros(a_nnz + 1, dtype=a.indices.dtype)
+    b_info = cupy.zeros(b_nnz + 1, dtype=b.indices.dtype)
+    a_valid = cupy.zeros(a_nnz, dtype=numpy.int8)
+    b_valid = cupy.zeros(b_nnz, dtype=numpy.int8)
+    c_indptr = cupy.zeros(m + 1, dtype=a.indptr.dtype)
+    in_dtype = numpy.promote_types(a.dtype, b.dtype)
+    a_data = a.data.astype(in_dtype, copy=False)
+    b_data = b.data.astype(in_dtype, copy=False)
+    funcs = _GET_ROW_ID_
+    if op_name == '_maximum_':
+        funcs += _BINOPT_MAX_
+        out_dtype = in_dtype
+    elif op_name == '_minimum_':
+        funcs += _BINOPT_MIN_
+        out_dtype = in_dtype
+    elif op_name == '_eq_':
+        funcs += _BINOPT_EQ_
+        out_dtype = numpy.bool_
+    elif op_name == '_ne_':
+        funcs += _BINOPT_NE_
+        out_dtype = numpy.bool_
+    elif op_name == '_lt_':
+        funcs += _BINOPT_LT_
+        out_dtype = numpy.bool_
+    elif op_name == '_gt_':
+        funcs += _BINOPT_GT_
+        out_dtype = numpy.bool_
+    elif op_name == '_le_':
+        funcs += _BINOPT_LE_
+        out_dtype = numpy.bool_
+    elif op_name == '_ge_':
+        funcs += _BINOPT_GE_
+        out_dtype = numpy.bool_
+    else:
+        raise ValueError('invalid op_name: {}'.format(op_name))
+    a_tmp_data = cupy.empty(a_nnz, dtype=out_dtype)
+    b_tmp_data = cupy.empty(b_nnz, dtype=out_dtype)
+    a_tmp_indices = cupy.empty(a_nnz, dtype=a.indices.dtype)
+    b_tmp_indices = cupy.empty(b_nnz, dtype=b.indices.dtype)
+    _size = a_nnz + b_nnz
+    cupy_binopt_csr_step1(op_name, preamble=funcs)(
+        m, n,
+        a.indptr, a.indices, a_data, a_m, a_n, a.nnz, a_nnz,
+        b.indptr, b.indices, b_data, b_m, b_n, b.nnz, b_nnz,
+        a_info, a_valid, a_tmp_indices, a_tmp_data,
+        b_info, b_valid, b_tmp_indices, b_tmp_data,
+        c_indptr, size=_size)
+    a_info = cupy.cumsum(a_info, dtype=a_info.dtype)
+    b_info = cupy.cumsum(b_info, dtype=b_info.dtype)
+    c_indptr = cupy.cumsum(c_indptr, dtype=c_indptr.dtype)
+    c_nnz = int(c_indptr[-1])
+    c_indices = cupy.empty(c_nnz, dtype=a.indices.dtype)
+    c_data = cupy.empty(c_nnz, dtype=out_dtype)
+    cupy_binopt_csr_step2(op_name)(
+        a_info, a_valid, a_tmp_indices, a_tmp_data, a_nnz,
+        b_info, b_valid, b_tmp_indices, b_tmp_data, b_nnz,
+        c_indices, c_data, size=_size)
+    return csr_matrix((c_data, c_indices, c_indptr), shape=(m, n))
+
+
+@cupy._util.memoize(for_each_device=True)
+def cupy_binopt_csr_step1(op_name, preamble=''):
+    name = 'cupyx_scipy_sparse_csr_binopt_' + op_name + 'step1'
+    return cupy.ElementwiseKernel(
+        '''
+        int32 M, int32 N,
+        raw I A_INDPTR, raw I A_INDICES, raw T A_DATA,
+        int32 A_M, int32 A_N, int32 A_NNZ_ACT, int32 A_NNZ,
+        raw I B_INDPTR, raw I B_INDICES, raw T B_DATA,
+        int32 B_M, int32 B_N, int32 B_NNZ_ACT, int32 B_NNZ
+        ''',
+        '''
+        raw I A_INFO, raw B A_VALID, raw I A_TMP_INDICES, raw O A_TMP_DATA,
+        raw I B_INFO, raw B B_VALID, raw I B_TMP_INDICES, raw O B_TMP_DATA,
+        raw I C_INFO
+        ''',
+        '''
+        if (i >= A_NNZ + B_NNZ) return;
+
+        const int *MY_INDPTR, *MY_INDICES;  int *MY_INFO;  const T *MY_DATA;
+        const int *OP_INDPTR, *OP_INDICES;  int *OP_INFO;  const T *OP_DATA;
+        int MY_M, MY_N, MY_NNZ_ACT, MY_NNZ;
+        int OP_M, OP_N, OP_NNZ_ACT, OP_NNZ;
+        signed char *MY_VALID;  I *MY_TMP_INDICES;  O *MY_TMP_DATA;
+
+        int my_j;
+        if (i < A_NNZ) {
+            // in charge of one of non-zero element of sparse matrix A
+            my_j = i;
+            MY_INDPTR  = &(A_INDPTR[0]);   OP_INDPTR  = &(B_INDPTR[0]);
+            MY_INDICES = &(A_INDICES[0]);  OP_INDICES = &(B_INDICES[0]);
+            MY_INFO    = &(A_INFO[0]);     OP_INFO    = &(B_INFO[0]);
+            MY_DATA    = &(A_DATA[0]);     OP_DATA    = &(B_DATA[0]);
+            MY_M       = A_M;              OP_M       = B_M;
+            MY_N       = A_N;              OP_N       = B_N;
+            MY_NNZ_ACT = A_NNZ_ACT;        OP_NNZ_ACT = B_NNZ_ACT;
+            MY_NNZ     = A_NNZ;            OP_NNZ     = B_NNZ;
+            MY_VALID   = &(A_VALID[0]);
+            MY_TMP_DATA= &(A_TMP_DATA[0]);
+            MY_TMP_INDICES = &(A_TMP_INDICES[0]);
+        } else {
+            // in charge of one of non-zero element of sparse matrix B
+            my_j = i - A_NNZ;
+            MY_INDPTR  = &(B_INDPTR[0]);   OP_INDPTR  = &(A_INDPTR[0]);
+            MY_INDICES = &(B_INDICES[0]);  OP_INDICES = &(A_INDICES[0]);
+            MY_INFO    = &(B_INFO[0]);     OP_INFO    = &(A_INFO[0]);
+            MY_DATA    = &(B_DATA[0]);     OP_DATA    = &(A_DATA[0]);
+            MY_M       = B_M;              OP_M       = A_M;
+            MY_N       = B_N;              OP_N       = A_N;
+            MY_NNZ_ACT = B_NNZ_ACT;        OP_NNZ_ACT = A_NNZ_ACT;
+            MY_NNZ     = B_NNZ;            OP_NNZ     = A_NNZ;
+            MY_VALID   = &(B_VALID[0]);
+            MY_TMP_DATA= &(B_TMP_DATA[0]);
+            MY_TMP_INDICES = &(B_TMP_INDICES[0]);
+        }
+        int _min, _max, _mid;
+
+        // get column location
+        int my_col;
+        int my_j_act = my_j;
+        if (MY_M == 1 && MY_M < M) {
+            if (MY_N == 1 && MY_N < N) my_j_act = 0;
+            else                       my_j_act = my_j % MY_NNZ_ACT;
+        } else {
+            if (MY_N == 1 && MY_N < N) my_j_act = my_j / N;
+        }
+        my_col = MY_INDICES[my_j_act];
+        if (MY_N == 1 && MY_N < N) {
+            my_col = my_j % N;
+        }
+
+        // get row location
+        int my_row = get_row_id(my_j_act, 0, MY_M - 1, &(MY_INDPTR[0]));
+        if (MY_M == 1 && MY_M < M) {
+            if (MY_N == 1 && MY_N < N) my_row = my_j / N;
+            else                       my_row = my_j / MY_NNZ_ACT;
+        }
+
+        int op_row = my_row;
+        int op_row_act = op_row;
+        if (OP_M == 1 && OP_M < M) {
+            op_row_act = 0;
+        }
+
+        int op_col = 0;
+        _min = OP_INDPTR[op_row_act];
+        _max = OP_INDPTR[op_row_act + 1] - 1;
+        int op_j_act = _min;
+        bool op_nz = false;
+        if (_min <= _max) {
+            if (OP_N == 1 && OP_N < N) {
+                op_col = my_col;
+                op_nz = true;
+            }
+            else {
+                _mid = (_min + _max) / 2;
+                op_col = OP_INDICES[_mid];
+                while (_min < _max) {
+                    if (op_col < my_col) {
+                        _min = _mid + 1;
+                    } else if (op_col > my_col) {
+                        _max = _mid;
+                    } else {
+                        break;
+                    }
+                    _mid = (_min + _max) / 2;
+                    op_col = OP_INDICES[_mid];
+                }
+                op_j_act = _mid;
+                if (op_col == my_col) {
+                    op_nz = true;
+                } else if (op_col < my_col) {
+                    op_col = N;
+                    op_j_act += 1;
+                }
+            }
+        }
+
+        int op_j = op_j_act;
+        if (OP_M == 1 && OP_M < M) {
+            if (OP_N == 1 && OP_N < N) {
+                op_j = (op_col + N * op_row) * OP_NNZ_ACT;
+            } else {
+                op_j = op_j_act + OP_NNZ_ACT * op_row;
+            }
+        } else {
+            if (OP_N == 1 && OP_N < N) {
+                op_j = op_col + N * op_j_act;
+            }
+        }
+
+        if (i < A_NNZ || !op_nz) {
+            T my_data = MY_DATA[my_j_act];
+            T op_data = 0;
+            if (op_nz) op_data = OP_DATA[op_j_act];
+            O out;
+            if (i < A_NNZ) out = binopt(my_data, op_data);
+            else           out = binopt(op_data, my_data);
+            if (out != static_cast<O>(0)) {
+                MY_VALID[my_j] = 1;
+                MY_TMP_DATA[my_j] = out;
+                MY_TMP_INDICES[my_j] = my_col;
+                atomicAdd( &(C_INFO[my_row + 1]), 1 );
+                atomicAdd( &(MY_INFO[my_j + 1]), 1 );
+                atomicAdd( &(OP_INFO[op_j]), 1 );
+            }
+        }
+        ''',
+        name, preamble=preamble,
+    )
+
+
+@cupy._util.memoize(for_each_device=True)
+def cupy_binopt_csr_step2(op_name):
+    name = 'cupyx_scipy_sparse_csr_binopt' + op_name + 'step2'
+    return cupy.ElementwiseKernel(
+        '''
+        raw I A_INFO, raw B A_VALID, raw I A_TMP_INDICES, raw O A_TMP_DATA,
+        int32 A_NNZ,
+        raw I B_INFO, raw B B_VALID, raw I B_TMP_INDICES, raw O B_TMP_DATA,
+        int32 B_NNZ
+        ''',
+        'raw I C_INDICES, raw O C_DATA',
+        '''
+        if (i < A_NNZ) {
+            int j = i;
+            if (A_VALID[j]) {
+                C_INDICES[A_INFO[j]] = A_TMP_INDICES[j];
+                C_DATA[A_INFO[j]]    = A_TMP_DATA[j];
+            }
+        } else if (i < A_NNZ + B_NNZ) {
+            int j = i - A_NNZ;
+            if (B_VALID[j]) {
+                C_INDICES[B_INFO[j]] = B_TMP_INDICES[j];
+                C_DATA[B_INFO[j]]    = B_TMP_DATA[j];
+            }
+        }
+        ''',
+        name,
+    )
+
+
+def csr2dense(a, order):
+    out = cupy.zeros(a.shape, dtype=a.dtype, order=order)
+    m, n = a.shape
+    kern = _cupy_csr2dense(a.dtype)
+    kern(m, n, a.indptr, a.indices, a.data, (order == 'C'), out)
+    return out
+
+
+@cupy._util.memoize(for_each_device=True)
+def _cupy_csr2dense(dtype):
+    if dtype == '?':
+        op = "if (DATA) OUT[index] = true;"
+    else:
+        op = "atomicAdd(&OUT[index], DATA);"
+
+    return cupy.ElementwiseKernel(
+        'int32 M, int32 N, raw I INDPTR, I INDICES, T DATA, bool C_ORDER',
+        'raw T OUT',
+        '''
+        int row = get_row_id(i, 0, M - 1, &(INDPTR[0]));
+        int col = INDICES;
+        int index = C_ORDER ? col + N * row : row + M * col;
+        ''' + op,
+        'cupyx_scipy_sparse_csr2dense',
+        preamble=_GET_ROW_ID_
+    )
+
+
+def dense2csr(a):
+    if a.dtype.char in 'fdFD':
+        if cusparse.check_availability('denseToSparse'):
+            return cusparse.denseToSparse(a, format='csr')
+        else:
+            return cusparse.dense2csr(a)
+    m, n = a.shape
+    a = cupy.ascontiguousarray(a)
+    indptr = cupy.zeros(m + 1, dtype=numpy.int32)
+    info = cupy.zeros(m * n + 1, dtype=numpy.int32)
+    cupy_dense2csr_step1()(m, n, a, indptr, info)
+    indptr = cupy.cumsum(indptr, dtype=numpy.int32)
+    info = cupy.cumsum(info, dtype=numpy.int32)
+    nnz = int(indptr[-1])
+    indices = cupy.empty(nnz, dtype=numpy.int32)
+    data = cupy.empty(nnz, dtype=a.dtype)
+    cupy_dense2csr_step2()(m, n, a, info, indices, data)
+    return csr_matrix((data, indices, indptr), shape=(m, n))
+
+
+@cupy._util.memoize(for_each_device=True)
+def cupy_dense2csr_step1():
+    return cupy.ElementwiseKernel(
+        'int32 M, int32 N, T A',
+        'raw I INDPTR, raw I INFO',
+        '''
+        int row = i / N;
+        int col = i % N;
+        if (A != static_cast<T>(0)) {
+            atomicAdd( &(INDPTR[row + 1]), 1 );
+            INFO[i + 1] = 1;
+        }
+        ''',
+        'cupyx_scipy_sparse_dense2csr_step1')
+
+
+@cupy._util.memoize(for_each_device=True)
+def cupy_dense2csr_step2():
+    return cupy.ElementwiseKernel(
+        'int32 M, int32 N, T A, raw I INFO',
+        'raw I INDICES, raw T DATA',
+        '''
+        int row = i / N;
+        int col = i % N;
+        if (A != static_cast<T>(0)) {
+            int idx = INFO[i];
+            INDICES[idx] = col;
+            DATA[idx] = A;
+        }
+        ''',
+        'cupyx_scipy_sparse_dense2csr_step2')
+
+
+@cupy._util.memoize(for_each_device=True)
+def _cupy_csr_diagonal():
+    return cupy.ElementwiseKernel(
+        'int32 k, int32 rows, int32 cols, '
+        'raw T data, raw I indptr, raw I indices',
+        'T y',
+        '''
+        int row = i;
+        int col = i;
+        if (k < 0) row -= k;
+        if (k > 0) col += k;
+        if (row >= rows || col >= cols) return;
+        int j = find_index_holding_col_in_row(row, col,
+            &(indptr[0]), &(indices[0]));
+        if (j >= 0) {
+            y = data[j];
+        } else {
+            y = static_cast<T>(0);
+        }
+        ''',
+        'cupyx_scipy_sparse_csr_diagonal',
+        preamble=_FIND_INDEX_HOLDING_COL_IN_ROW_
+    )
diff --git a/cupyx/scipy/sparse/_data.py b/cupyx/scipy/sparse/_data.py
new file mode 100644
index 0000000..1615aea
--- /dev/null
+++ b/cupyx/scipy/sparse/_data.py
@@ -0,0 +1,398 @@
+import cupy
+import numpy as np
+from cupy._core import internal
+from cupy import _util
+from cupyx.scipy.sparse import _base
+from cupyx.scipy.sparse import _coo
+from cupyx.scipy.sparse import _sputils
+
+
+_ufuncs = [
+    'arcsin', 'arcsinh', 'arctan', 'arctanh', 'ceil', 'deg2rad', 'expm1',
+    'floor', 'log1p', 'rad2deg', 'rint', 'sign', 'sin', 'sinh', 'sqrt', 'tan',
+    'tanh', 'trunc',
+]
+
+
+class _data_matrix(_base.spmatrix):
+
+    def __init__(self, data):
+        self.data = data
+
+    @property
+    def dtype(self):
+        """Data type of the matrix."""
+        return self.data.dtype
+
+    def _with_data(self, data, copy=True):
+        raise NotImplementedError
+
+    def __abs__(self):
+        """Elementwise abosulte."""
+        return self._with_data(abs(self.data))
+
+    def __neg__(self):
+        """Elementwise negative."""
+        return self._with_data(-self.data)
+
+    def astype(self, t):
+        """Casts the array to given data type.
+
+        Args:
+            dtype: Type specifier.
+
+        Returns:
+            A copy of the array with a given type.
+
+        """
+        return self._with_data(self.data.astype(t))
+
+    def conj(self, copy=True):
+        if cupy.issubdtype(self.dtype, cupy.complexfloating):
+            return self._with_data(self.data.conj(), copy=copy)
+        elif copy:
+            return self.copy()
+        else:
+            return self
+
+    conj.__doc__ = _base.spmatrix.conj.__doc__
+
+    def copy(self):
+        return self._with_data(self.data.copy(), copy=True)
+
+    copy.__doc__ = _base.spmatrix.copy.__doc__
+
+    def count_nonzero(self):
+        """Returns number of non-zero entries.
+
+        .. note::
+           This method counts the actual number of non-zero entories, which
+           does not include explicit zero entries.
+           Instead ``nnz`` returns the number of entries including explicit
+           zeros.
+
+        Returns:
+            Number of non-zero entries.
+
+        """
+        return cupy.count_nonzero(self.data)
+
+    def mean(self, axis=None, dtype=None, out=None):
+        """Compute the arithmetic mean along the specified axis.
+
+        Args:
+            axis (int or ``None``): Axis along which the sum is computed.
+                If it is ``None``, it computes the average of all the elements.
+                Select from ``{None, 0, 1, -2, -1}``.
+
+        Returns:
+            cupy.ndarray: Summed array.
+
+        .. seealso::
+           :meth:`scipy.sparse.spmatrix.mean`
+
+        """
+        _sputils.validateaxis(axis)
+        nRow, nCol = self.shape
+        data = self.data.copy()
+
+        if axis is None:
+            n = nRow * nCol
+        elif axis in (0, -2):
+            n = nRow
+        else:
+            n = nCol
+
+        return self._with_data(data / n).sum(axis, dtype, out)
+
+    def power(self, n, dtype=None):
+        """Elementwise power function.
+
+        Args:
+            n: Exponent.
+            dtype: Type specifier.
+
+        """
+        if dtype is None:
+            data = self.data.copy()
+        else:
+            data = self.data.astype(dtype, copy=True)
+        data **= n
+        return self._with_data(data)
+
+
+def _find_missing_index(ind, n):
+    positions = cupy.arange(ind.size)
+    diff = ind != positions
+    return cupy.where(
+        diff.any(),
+        diff.argmax(),
+        cupy.asarray(ind.size if ind.size < n else -1))
+
+
+def _non_zero_cmp(mat, am, zero, m):
+    size = np.prod(mat.shape)
+    if size == mat.nnz:
+        return am
+    else:
+        ind = mat.row * mat.shape[1] + mat.col
+        zero_ind = _find_missing_index(ind, size)
+        return cupy.where(
+            m == zero,
+            cupy.minimum(zero_ind, am),
+            zero_ind)
+
+
+class _minmax_mixin(object):
+    """Mixin for min and max methods.
+    These are not implemented for dia_matrix, hence the separate class.
+
+    """
+
+    def _min_or_max_axis(self, axis, min_or_max, explicit):
+        N = self.shape[axis]
+        if N == 0:
+            raise ValueError("zero-size array to reduction operation")
+        M = self.shape[1 - axis]
+
+        mat = self.tocsc() if axis == 0 else self.tocsr()
+        mat.sum_duplicates()
+
+        # Do the reduction
+        value = mat._minor_reduce(min_or_max, axis, explicit)
+        major_index = cupy.arange(M)
+
+        mask = value != 0
+        major_index = cupy.compress(mask, major_index)
+        value = cupy.compress(mask, value)
+
+        if axis == 0:
+            return _coo.coo_matrix(
+                (value, (cupy.zeros(len(value)), major_index)),
+                dtype=self.dtype, shape=(1, M))
+        else:
+            return _coo.coo_matrix(
+                (value, (major_index, cupy.zeros(len(value)))),
+                dtype=self.dtype, shape=(M, 1))
+
+    def _min_or_max(self, axis, out, min_or_max, explicit):
+        if out is not None:
+            raise ValueError(("Sparse matrices do not support "
+                              "an 'out' parameter."))
+
+        _sputils.validateaxis(axis)
+
+        if axis is None:
+            if 0 in self.shape:
+                raise ValueError("zero-size array to reduction operation")
+
+            zero = cupy.zeros((), dtype=self.dtype)
+            if self.nnz == 0:
+                return zero
+            self.sum_duplicates()
+            m = min_or_max(self.data)
+            if explicit:
+                return m
+            if self.nnz != internal.prod(self.shape):
+                if min_or_max is cupy.min:
+                    m = cupy.minimum(zero, m)
+                elif min_or_max is cupy.max:
+                    m = cupy.maximum(zero, m)
+                else:
+                    assert False
+            return m
+
+        if axis < 0:
+            axis += 2
+
+        return self._min_or_max_axis(axis, min_or_max, explicit)
+
+    def _arg_min_or_max_axis(self, axis, op):
+        if self.shape[axis] == 0:
+            raise ValueError("Can't apply the operation along a zero-sized "
+                             "dimension.")
+
+        mat = self.tocsc() if axis == 0 else self.tocsr()
+        mat.sum_duplicates()
+
+        # Do the reduction
+        value = mat._arg_minor_reduce(op, axis)
+
+        if axis == 0:
+            return value[None, :]
+        else:
+            return value[:, None]
+
+    def _arg_min_or_max(self, axis, out, op, compare):
+        if out is not None:
+            raise ValueError("Sparse matrices do not support "
+                             "an 'out' parameter.")
+
+        _sputils.validateaxis(axis)
+
+        if axis is None:
+            if 0 in self.shape:
+                raise ValueError("Can't apply the operation to "
+                                 "an empty matrix.")
+
+            if self.nnz == 0:
+                return 0
+            else:
+                zero = cupy.asarray(self.dtype.type(0))
+                mat = self.tocoo()
+
+                mat.sum_duplicates()
+
+                am = op(mat.data)
+                m = mat.data[am]
+
+                return cupy.where(
+                    compare(m, zero), mat.row[am] * mat.shape[1] + mat.col[am],
+                    _non_zero_cmp(mat, am, zero, m))
+
+        if axis < 0:
+            axis += 2
+
+        return self._arg_min_or_max_axis(axis, op)
+
+    def max(self, axis=None, out=None, *, explicit=False):
+        """Returns the maximum of the matrix or maximum along an axis.
+
+        Args:
+            axis (int): {-2, -1, 0, 1, ``None``} (optional)
+                Axis along which the sum is computed. The default is to
+                compute the maximum over all the matrix elements, returning
+                a scalar (i.e. ``axis`` = ``None``).
+            out (None): (optional)
+                This argument is in the signature *solely* for NumPy
+                compatibility reasons. Do not pass in anything except
+                for the default value, as this argument is not used.
+            explicit (bool): Return the maximum value explicitly specified and
+                ignore all implicit zero entries. If the dimension has no
+                explicit values, a zero is then returned to indicate that it is
+                the only implicit value. This parameter is experimental and may
+                change in the future.
+
+        Returns:
+            (cupy.ndarray or float): Maximum of ``a``. If ``axis`` is
+            ``None``, the result is a scalar value. If ``axis`` is given,
+            the result is an array of dimension ``a.ndim - 1``. This
+            differs from numpy for computational efficiency.
+
+        .. seealso:: min : The minimum value of a sparse matrix along a given
+          axis.
+        .. seealso:: numpy.matrix.max : NumPy's implementation of ``max`` for
+          matrices
+
+        """
+        if explicit:
+            api_name = 'explicit of cupyx.scipy.sparse.{}.max'.format(
+                self.__class__.__name__)
+            _util.experimental(api_name)
+        return self._min_or_max(axis, out, cupy.max, explicit)
+
+    def min(self, axis=None, out=None, *, explicit=False):
+        """Returns the minimum of the matrix or maximum along an axis.
+
+        Args:
+            axis (int): {-2, -1, 0, 1, ``None``} (optional)
+                Axis along which the sum is computed. The default is to
+                compute the minimum over all the matrix elements, returning
+                a scalar (i.e. ``axis`` = ``None``).
+            out (None): (optional)
+                This argument is in the signature *solely* for NumPy
+                compatibility reasons. Do not pass in anything except for
+                the default value, as this argument is not used.
+            explicit (bool): Return the minimum value explicitly specified and
+                ignore all implicit zero entries. If the dimension has no
+                explicit values, a zero is then returned to indicate that it is
+                the only implicit value. This parameter is experimental and may
+                change in the future.
+
+        Returns:
+            (cupy.ndarray or float): Minimum of ``a``. If ``axis`` is
+            None, the result is a scalar value. If ``axis`` is given, the
+            result is an array of dimension ``a.ndim - 1``. This differs
+            from numpy for computational efficiency.
+
+        .. seealso:: max : The maximum value of a sparse matrix along a given
+          axis.
+        .. seealso:: numpy.matrix.min : NumPy's implementation of 'min' for
+          matrices
+
+        """
+        if explicit:
+            api_name = 'explicit of cupyx.scipy.sparse.{}.min'.format(
+                self.__class__.__name__)
+            _util.experimental(api_name)
+        return self._min_or_max(axis, out, cupy.min, explicit)
+
+    def argmax(self, axis=None, out=None):
+        """Returns indices of maximum elements along an axis.
+
+        Implicit zero elements are taken into account. If there are several
+        maximum values, the index of the first occurrence is returned. If
+        ``NaN`` values occur in the matrix, the output defaults to a zero entry
+        for the row/column in which the NaN occurs.
+
+        Args:
+            axis (int): {-2, -1, 0, 1, ``None``} (optional)
+                Axis along which the argmax is computed. If ``None`` (default),
+                index of the maximum element in the flatten data is returned.
+            out (None): (optional)
+                This argument is in the signature *solely* for NumPy
+                compatibility reasons. Do not pass in anything except for
+                the default value, as this argument is not used.
+
+        Returns:
+            (cupy.narray or int): Indices of maximum elements. If array,
+            its size along ``axis`` is 1.
+
+        """
+        return self._arg_min_or_max(axis, out, cupy.argmax, cupy.greater)
+
+    def argmin(self, axis=None, out=None):
+        """
+        Returns indices of minimum elements along an axis.
+
+        Implicit zero elements are taken into account. If there are several
+        minimum values, the index of the first occurrence is returned. If
+        ``NaN`` values occur in the matrix, the output defaults to a zero entry
+        for the row/column in which the NaN occurs.
+
+        Args:
+            axis (int): {-2, -1, 0, 1, ``None``} (optional)
+                Axis along which the argmin is computed. If ``None`` (default),
+                index of the minimum element in the flatten data is returned.
+            out (None): (optional)
+                This argument is in the signature *solely* for NumPy
+                compatibility reasons. Do not pass in anything except for
+                the default value, as this argument is not used.
+
+        Returns:
+            (cupy.narray or int): Indices of minimum elements. If matrix,
+            its size along ``axis`` is 1.
+
+        """
+        return self._arg_min_or_max(axis, out, cupy.argmin, cupy.less)
+
+
+def _install_ufunc(func_name):
+
+    def f(self):
+        ufunc = getattr(cupy, func_name)
+        result = ufunc(self.data)
+        return self._with_data(result)
+
+    f.__doc__ = 'Elementwise %s.' % func_name
+    f.__name__ = func_name
+
+    setattr(_data_matrix, func_name, f)
+
+
+def _install_ufuncs():
+    for func_name in _ufuncs:
+        _install_ufunc(func_name)
+
+
+_install_ufuncs()
diff --git a/cupyx/scipy/sparse/_dia.py b/cupyx/scipy/sparse/_dia.py
new file mode 100644
index 0000000..b1a23be
--- /dev/null
+++ b/cupyx/scipy/sparse/_dia.py
@@ -0,0 +1,219 @@
+try:
+    import scipy.sparse
+    _scipy_available = True
+except ImportError:
+    _scipy_available = False
+
+import cupy
+from cupy import _core
+from cupyx.scipy.sparse import _csc
+from cupyx.scipy.sparse import _data
+from cupyx.scipy.sparse import _util
+
+
+# TODO(leofang): The current implementation is CSC-based, which is troublesome
+# on ROCm/HIP. We should convert it to CSR-based for portability.
+class dia_matrix(_data._data_matrix):
+
+    """Sparse matrix with DIAgonal storage.
+
+    Now it has only one initializer format below:
+
+    ``dia_matrix((data, offsets))``
+
+    Args:
+        arg1: Arguments for the initializer.
+        shape (tuple): Shape of a matrix. Its length must be two.
+        dtype: Data type. It must be an argument of :class:`numpy.dtype`.
+        copy (bool): If ``True``, copies of given arrays are always used.
+
+    .. seealso::
+       :class:`scipy.sparse.dia_matrix`
+
+    """
+
+    format = 'dia'
+
+    def __init__(self, arg1, shape=None, dtype=None, copy=False):
+        if _scipy_available and scipy.sparse.issparse(arg1):
+            x = arg1.todia()
+            data = x.data
+            offsets = x.offsets
+            shape = x.shape
+            dtype = x.dtype
+            copy = False
+        elif isinstance(arg1, tuple):
+            data, offsets = arg1
+            if shape is None:
+                raise ValueError('expected a shape argument')
+
+        else:
+            raise ValueError(
+                'unrecognized form for dia_matrix constructor')
+
+        data = cupy.array(data, dtype=dtype, copy=copy)
+        data = cupy.atleast_2d(data)
+        offsets = cupy.array(offsets, dtype='i', copy=copy)
+        offsets = cupy.atleast_1d(offsets)
+
+        if offsets.ndim != 1:
+            raise ValueError('offsets array must have rank 1')
+
+        if data.ndim != 2:
+            raise ValueError('data array must have rank 2')
+
+        if data.shape[0] != len(offsets):
+            raise ValueError(
+                'number of diagonals (%d) does not match the number of '
+                'offsets (%d)'
+                % (data.shape[0], len(offsets)))
+
+        sorted_offsets = cupy.sort(offsets)
+        if (sorted_offsets[:-1] == sorted_offsets[1:]).any():
+            raise ValueError('offset array contains duplicate values')
+
+        self.data = data
+        self.offsets = offsets
+        if not _util.isshape(shape):
+            raise ValueError('invalid shape (must be a 2-tuple of int)')
+        self._shape = int(shape[0]), int(shape[1])
+
+    def _with_data(self, data, copy=True):
+        """Returns a matrix with the same sparsity structure as self,
+        but with different data.  By default the structure arrays are copied.
+        """
+        if copy:
+            return dia_matrix((data, self.offsets.copy()), shape=self.shape)
+        else:
+            return dia_matrix((data, self.offsets), shape=self.shape)
+
+    def get(self, stream=None):
+        """Returns a copy of the array on host memory.
+
+        Args:
+            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
+                copy runs asynchronously. Otherwise, the copy is synchronous.
+
+        Returns:
+            scipy.sparse.dia_matrix: Copy of the array on host memory.
+
+        """
+        if not _scipy_available:
+            raise RuntimeError('scipy is not available')
+        data = self.data.get(stream)
+        offsets = self.offsets.get(stream)
+        return scipy.sparse.dia_matrix((data, offsets), shape=self._shape)
+
+    def get_shape(self):
+        """Returns the shape of the matrix.
+
+        Returns:
+            tuple: Shape of the matrix.
+        """
+        return self._shape
+
+    def getnnz(self, axis=None):
+        """Returns the number of stored values, including explicit zeros.
+
+        Args:
+            axis: Not supported yet.
+
+        Returns:
+            int: The number of stored values.
+
+        """
+        if axis is not None:
+            raise NotImplementedError(
+                'getnnz over an axis is not implemented for DIA format')
+
+        m, n = self.shape
+        nnz = _core.ReductionKernel(
+            'int32 offsets, int32 m, int32 n', 'int32 nnz',
+            'offsets > 0 ? min(m, n - offsets) : min(m + offsets, n)',
+            'a + b', 'nnz = a', '0', 'dia_nnz')(self.offsets, m, n)
+        return int(nnz)
+
+    def toarray(self, order=None, out=None):
+        """Returns a dense matrix representing the same value."""
+        return self.tocsc().toarray(order=order, out=out)
+
+    def tocsc(self, copy=False):
+        """Converts the matrix to Compressed Sparse Column format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible. Actually this option is ignored because all
+                arrays in a matrix cannot be shared in dia to csc conversion.
+
+        Returns:
+            cupyx.scipy.sparse.csc_matrix: Converted matrix.
+
+        """
+        if self.data.size == 0:
+            return _csc.csc_matrix(self.shape, dtype=self.dtype)
+
+        num_rows, num_cols = self.shape
+        num_offsets, offset_len = self.data.shape
+
+        row, mask = _core.ElementwiseKernel(
+            'int32 offset_len, int32 offsets, int32 num_rows, '
+            'int32 num_cols, T data',
+            'int32 row, bool mask',
+            '''
+            int offset_inds = i % offset_len;
+            row = offset_inds - offsets;
+            mask = (row >= 0 && row < num_rows && offset_inds < num_cols
+                    && data != T(0));
+            ''',
+            'cupyx_scipy_sparse_dia_tocsc')(offset_len, self.offsets[:, None],
+                                            num_rows, num_cols, self.data)
+        indptr = cupy.zeros(num_cols + 1, dtype='i')
+        indptr[1: offset_len + 1] = cupy.cumsum(mask.sum(axis=0))
+        indptr[offset_len + 1:] = indptr[offset_len]
+        indices = row.T[mask.T].astype('i', copy=False)
+        data = self.data.T[mask.T]
+        return _csc.csc_matrix(
+            (data, indices, indptr), shape=self.shape, dtype=self.dtype)
+
+    def tocsr(self, copy=False):
+        """Converts the matrix to Compressed Sparse Row format.
+
+        Args:
+            copy (bool): If ``False``, it shares data arrays as much as
+                possible. Actually this option is ignored because all
+                arrays in a matrix cannot be shared in dia to csr conversion.
+
+        Returns:
+            cupyx.scipy.sparse.csc_matrix: Converted matrix.
+
+        """
+        return self.tocsc().tocsr()
+
+    def diagonal(self, k=0):
+        """Returns the k-th diagonal of the matrix.
+
+        Args:
+            k (int, optional): Which diagonal to get, corresponding to elements
+            a[i, i+k]. Default: 0 (the main diagonal).
+
+        Returns:
+            cupy.ndarray : The k-th diagonal.
+        """
+        rows, cols = self.shape
+        if k <= -rows or k >= cols:
+            return cupy.empty(0, dtype=self.data.dtype)
+        idx, = cupy.nonzero(self.offsets == k)
+        first_col, last_col = max(0, k), min(rows + k, cols)
+        if idx.size == 0:
+            return cupy.zeros(last_col - first_col, dtype=self.data.dtype)
+        return self.data[idx[0], first_col:last_col]
+
+
+def isspmatrix_dia(x):
+    """Checks if a given matrix is of DIA format.
+
+    Returns:
+        bool: Returns if ``x`` is :class:`cupyx.scipy.sparse.dia_matrix`.
+
+    """
+    return isinstance(x, dia_matrix)
diff --git a/cupyx/scipy/sparse/_extract.py b/cupyx/scipy/sparse/_extract.py
new file mode 100644
index 0000000..1bd5e52
--- /dev/null
+++ b/cupyx/scipy/sparse/_extract.py
@@ -0,0 +1,81 @@
+import cupy
+import cupyx
+
+from cupyx.scipy import sparse
+
+
+def find(A):
+    """Returns the indices and values of the nonzero elements of a matrix
+
+    Args:
+        A (cupy.ndarray or cupyx.scipy.sparse.spmatrix): Matrix whose nonzero
+            elements are desired.
+
+    Returns:
+        tuple of cupy.ndarray:
+            It returns (``I``, ``J``, ``V``). ``I``, ``J``, and ``V`` contain
+            respectively the row indices, column indices, and values of the
+            nonzero matrix entries.
+
+    .. seealso:: :func:`scipy.sparse.find`
+    """
+    _check_A_type(A)
+    A = sparse.coo_matrix(A, copy=True)
+    A.sum_duplicates()
+    nz_mask = A.data != 0
+    return A.row[nz_mask], A.col[nz_mask], A.data[nz_mask]
+
+
+def tril(A, k=0, format=None):
+    """Returns the lower triangular portion of a matrix in sparse format
+
+    Args:
+        A (cupy.ndarray or cupyx.scipy.sparse.spmatrix): Matrix whose lower
+            triangular portion is desired.
+        k (integer): The top-most diagonal of the lower triangle.
+        format (string): Sparse format of the result, e.g. 'csr', 'csc', etc.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix:
+            Lower triangular portion of A in sparse format.
+
+    .. seealso:: :func:`scipy.sparse.tril`
+    """
+    _check_A_type(A)
+    A = sparse.coo_matrix(A, copy=False)
+    mask = A.row + k >= A.col
+    return _masked_coo(A, mask).asformat(format)
+
+
+def triu(A, k=0, format=None):
+    """Returns the upper triangular portion of a matrix in sparse format
+
+    Args:
+        A (cupy.ndarray or cupyx.scipy.sparse.spmatrix): Matrix whose upper
+            triangular portion is desired.
+        k (integer): The bottom-most diagonal of the upper triangle.
+        format (string): Sparse format of the result, e.g. 'csr', 'csc', etc.
+
+    Returns:
+        cupyx.scipy.sparse.spmatrix:
+            Upper triangular portion of A in sparse format.
+
+    .. seealso:: :func:`scipy.sparse.triu`
+    """
+    _check_A_type(A)
+    A = sparse.coo_matrix(A, copy=False)
+    mask = A.row + k <= A.col
+    return _masked_coo(A, mask).asformat(format)
+
+
+def _check_A_type(A):
+    if not (isinstance(A, cupy.ndarray) or cupyx.scipy.sparse.isspmatrix(A)):
+        msg = 'A must be cupy.ndarray or cupyx.scipy.sparse.spmatrix'
+        raise TypeError(msg)
+
+
+def _masked_coo(A, mask):
+    row = A.row[mask]
+    col = A.col[mask]
+    data = A.data[mask]
+    return sparse.coo_matrix((data, (row, col)), shape=A.shape, dtype=A.dtype)
diff --git a/cupyx/scipy/sparse/_index.py b/cupyx/scipy/sparse/_index.py
new file mode 100644
index 0000000..75bf4f7
--- /dev/null
+++ b/cupyx/scipy/sparse/_index.py
@@ -0,0 +1,702 @@
+"""Indexing mixin for sparse matrix classes.
+"""
+
+import cupy
+from cupy import _core
+
+from cupyx.scipy.sparse._base import isspmatrix
+from cupyx.scipy.sparse._base import spmatrix
+
+from cupy_backends.cuda.libs import cusparse
+from cupy.cuda import device
+from cupy.cuda import runtime
+
+import numpy
+
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+_int_scalar_types = (int, numpy.integer, numpy.int_)
+_bool_scalar_types = (bool, numpy.bool_)
+
+
+_compress_getitem_kern = _core.ElementwiseKernel(
+    'T d, S ind, int32 minor', 'raw T answer',
+    'if (ind == minor) atomicAdd(&answer[0], d);',
+    'cupyx_scipy_sparse_compress_getitem')
+
+
+_compress_getitem_complex_kern = _core.ElementwiseKernel(
+    'T real, T imag, S ind, int32 minor',
+    'raw T answer_real, raw T answer_imag',
+    '''
+    if (ind == minor) {
+    atomicAdd(&answer_real[0], real);
+    atomicAdd(&answer_imag[0], imag);
+    }
+    ''',
+    'cupyx_scipy_sparse_compress_getitem_complex')
+
+
+def _get_csr_submatrix_major_axis(Ax, Aj, Ap, start, stop):
+    """Return a submatrix of the input sparse matrix by slicing major axis.
+
+    Args:
+        Ax (cupy.ndarray): data array from input sparse matrix
+        Aj (cupy.ndarray): indices array from input sparse matrix
+        Ap (cupy.ndarray): indptr array from input sparse matrix
+        start (int): starting index of major axis
+        stop (int): ending index of major axis
+
+    Returns:
+        Bx (cupy.ndarray): data array of output sparse matrix
+        Bj (cupy.ndarray): indices array of output sparse matrix
+        Bp (cupy.ndarray): indptr array of output sparse matrix
+
+    """
+    Ap = Ap[start:stop + 1]
+    start_offset, stop_offset = int(Ap[0]), int(Ap[-1])
+    Bp = Ap - start_offset
+    Bj = Aj[start_offset:stop_offset]
+    Bx = Ax[start_offset:stop_offset]
+
+    return Bx, Bj, Bp
+
+
+def _get_csr_submatrix_minor_axis(Ax, Aj, Ap, start, stop):
+    """Return a submatrix of the input sparse matrix by slicing minor axis.
+
+    Args:
+        Ax (cupy.ndarray): data array from input sparse matrix
+        Aj (cupy.ndarray): indices array from input sparse matrix
+        Ap (cupy.ndarray): indptr array from input sparse matrix
+        start (int): starting index of minor axis
+        stop (int): ending index of minor axis
+
+    Returns:
+        Bx (cupy.ndarray): data array of output sparse matrix
+        Bj (cupy.ndarray): indices array of output sparse matrix
+        Bp (cupy.ndarray): indptr array of output sparse matrix
+
+    """
+    mask = (start <= Aj) & (Aj < stop)
+    mask_sum = cupy.empty(Aj.size + 1, dtype=Aj.dtype)
+    mask_sum[0] = 0
+    mask_sum[1:] = mask
+    cupy.cumsum(mask_sum, out=mask_sum)
+    Bp = mask_sum[Ap]
+    Bj = Aj[mask] - start
+    Bx = Ax[mask]
+
+    return Bx, Bj, Bp
+
+
+_csr_row_index_ker = _core.ElementwiseKernel(
+    'int32 out_rows, raw I rows, '
+    'raw int32 Ap, raw int32 Aj, raw T Ax, raw int32 Bp',
+    'int32 Bj, T Bx',
+    '''
+    const I row = rows[out_rows];
+
+    // Look up starting offset
+    const I starting_output_offset = Bp[out_rows];
+    const I output_offset = i - starting_output_offset;
+    const I starting_input_offset = Ap[row];
+
+    Bj = Aj[starting_input_offset + output_offset];
+    Bx = Ax[starting_input_offset + output_offset];
+''', 'cupyx_scipy_sparse_csr_row_index_ker')
+
+
+def _csr_row_index(Ax, Aj, Ap, rows):
+    """Populate indices and data arrays from the given row index
+    Args:
+        Ax (cupy.ndarray): data array from input sparse matrix
+        Aj (cupy.ndarray): indices array from input sparse matrix
+        Ap (cupy.ndarray): indptr array from input sparse matrix
+        rows (cupy.ndarray): index array of rows to populate
+    Returns:
+        Bx (cupy.ndarray): data array of output sparse matrix
+        Bj (cupy.ndarray): indices array of output sparse matrix
+        Bp (cupy.ndarray): indptr array for output sparse matrix
+    """
+    row_nnz = cupy.diff(Ap)
+    Bp = cupy.empty(rows.size + 1, dtype=Ap.dtype)
+    Bp[0] = 0
+    cupy.cumsum(row_nnz[rows], out=Bp[1:])
+    nnz = int(Bp[-1])
+
+    out_rows = _csr_indptr_to_coo_rows(nnz, Bp)
+
+    Bj, Bx = _csr_row_index_ker(out_rows, rows, Ap, Aj, Ax, Bp)
+    return Bx, Bj, Bp
+
+
+def _csr_indptr_to_coo_rows(nnz, Bp):
+    out_rows = cupy.empty(nnz, dtype=numpy.int32)
+
+    # Build a COO row array from output CSR indptr.
+    # Calling backend cusparse API directly to avoid
+    # constructing a whole COO object.
+    handle = device.get_cusparse_handle()
+    if runtime.is_hip and nnz == 0:
+        raise ValueError('hipSPARSE currently cannot handle '
+                         'sparse matrices with null ptrs')
+    cusparse.xcsr2coo(
+        handle, Bp.data.ptr, nnz, Bp.size-1, out_rows.data.ptr,
+        cusparse.CUSPARSE_INDEX_BASE_ZERO)
+
+    return out_rows
+
+
+def _select_last_indices(i, j, x, idx_dtype):
+    """Find the unique indices for each row and keep only the last"""
+    i = cupy.asarray(i, dtype=idx_dtype)
+    j = cupy.asarray(j, dtype=idx_dtype)
+
+    stacked = cupy.stack([j, i])
+    order = cupy.lexsort(stacked).astype(idx_dtype)
+
+    indptr_inserts = i[order]
+    indices_inserts = j[order]
+    data_inserts = x[order]
+
+    mask = cupy.ones(indptr_inserts.size, dtype='bool')
+    _unique_mask_kern(indptr_inserts, indices_inserts, order, mask,
+                      size=indptr_inserts.size-1)
+
+    return indptr_inserts[mask], indices_inserts[mask], data_inserts[mask]
+
+
+_insert_many_populate_arrays = _core.ElementwiseKernel(
+    '''raw I insert_indices, raw T insert_values, raw I insertion_indptr,
+        raw I Ap, raw I Aj, raw T Ax, raw I Bp''',
+    'raw I Bj, raw T Bx', '''
+
+        const I input_row_start = Ap[i];
+        const I input_row_end = Ap[i+1];
+        const I input_count = input_row_end - input_row_start;
+
+        const I insert_row_start = insertion_indptr[i];
+        const I insert_row_end = insertion_indptr[i+1];
+        const I insert_count = insert_row_end - insert_row_start;
+
+        I input_offset = 0;
+        I insert_offset = 0;
+
+        I output_n = Bp[i];
+
+        I cur_existing_index = -1;
+        T cur_existing_value = -1;
+
+        I cur_insert_index = -1;
+        T cur_insert_value = -1;
+
+        if(input_offset < input_count) {
+            cur_existing_index = Aj[input_row_start+input_offset];
+            cur_existing_value = Ax[input_row_start+input_offset];
+        }
+
+        if(insert_offset < insert_count) {
+            cur_insert_index = insert_indices[insert_row_start+insert_offset];
+            cur_insert_value = insert_values[insert_row_start+insert_offset];
+        }
+
+
+        for(I jj = 0; jj < input_count + insert_count; jj++) {
+
+            // if we have both available, use the lowest one.
+            if(input_offset < input_count &&
+               insert_offset < insert_count) {
+
+                if(cur_existing_index < cur_insert_index) {
+                    Bj[output_n] = cur_existing_index;
+                    Bx[output_n] = cur_existing_value;
+
+                    ++input_offset;
+
+                    if(input_offset < input_count) {
+                        cur_existing_index = Aj[input_row_start+input_offset];
+                        cur_existing_value = Ax[input_row_start+input_offset];
+                    }
+
+
+                } else {
+                    Bj[output_n] = cur_insert_index;
+                    Bx[output_n] = cur_insert_value;
+
+                    ++insert_offset;
+                    if(insert_offset < insert_count) {
+                        cur_insert_index =
+                            insert_indices[insert_row_start+insert_offset];
+                        cur_insert_value =
+                            insert_values[insert_row_start+insert_offset];
+                    }
+                }
+
+            } else if(input_offset < input_count) {
+                Bj[output_n] = cur_existing_index;
+                Bx[output_n] = cur_existing_value;
+
+                ++input_offset;
+                if(input_offset < input_count) {
+                    cur_existing_index = Aj[input_row_start+input_offset];
+                    cur_existing_value = Ax[input_row_start+input_offset];
+                }
+
+            } else {
+                    Bj[output_n] = cur_insert_index;
+                    Bx[output_n] = cur_insert_value;
+
+                    ++insert_offset;
+                    if(insert_offset < insert_count) {
+                        cur_insert_index =
+                            insert_indices[insert_row_start+insert_offset];
+                        cur_insert_value =
+                            insert_values[insert_row_start+insert_offset];
+                    }
+            }
+
+            output_n++;
+        }
+    ''', 'cupyx_scipy_sparse_csr_copy_existing_indices_kern', no_return=True)
+
+
+# Create a filter mask based on the lowest value of order
+_unique_mask_kern = _core.ElementwiseKernel(
+    '''raw I rows, raw I cols, raw I order''',
+    '''raw bool mask''',
+    """
+    I cur_row = rows[i];
+    I next_row = rows[i+1];
+
+    I cur_col = cols[i];
+    I next_col = cols[i+1];
+
+    I cur_order = order[i];
+    I next_order = order[i+1];
+
+    if(cur_row == next_row && cur_col == next_col) {
+        if(cur_order < next_order)
+            mask[i] = false;
+        else
+            mask[i+1] = false;
+    }
+    """,
+    'cupyx_scipy_sparse_unique_mask_kern',
+    no_return=True
+)
+
+
+def _csr_sample_values(n_row, n_col,
+                       Ap, Aj, Ax,
+                       Bi, Bj, not_found_val=0):
+    """Populate data array for a set of rows and columns
+    Args
+        n_row : total number of rows in input array
+        n_col : total number of columns in input array
+        Ap : indptr array for input sparse matrix
+        Aj : indices array for input sparse matrix
+        Ax : data array for input sparse matrix
+        Bi : array of rows to extract from input sparse matrix
+        Bj : array of columns to extract from input sparse matrix
+    Returns
+        Bx : data array for output sparse matrix
+    """
+
+    Bi[Bi < 0] += n_row
+    Bj[Bj < 0] += n_col
+
+    return _csr_sample_values_kern(n_row, n_col,
+                                   Ap, Aj, Ax,
+                                   Bi, Bj,
+                                   not_found_val,
+                                   size=Bi.size)
+
+
+_csr_sample_values_kern = _core.ElementwiseKernel(
+    '''I n_row, I n_col, raw I Ap, raw I Aj, raw T Ax,
+    raw I Bi, raw I Bj, I not_found_val''',
+    'raw T Bx', '''
+    const I j = Bi[i]; // sample row
+    const I k = Bj[i]; // sample column
+    const I row_start = Ap[j];
+    const I row_end   = Ap[j+1];
+    T x = 0;
+    bool val_found = false;
+    for(I jj = row_start; jj < row_end; jj++) {
+        if (Aj[jj] == k) {
+            x += Ax[jj];
+            val_found = true;
+        }
+    }
+    Bx[i] = val_found ? x : not_found_val;
+''', 'cupyx_scipy_sparse_csr_sample_values_kern')
+
+
+class IndexMixin(object):
+    """
+    This class provides common dispatching and validation logic for indexing.
+    """
+
+    def __getitem__(self, key):
+
+        # For testing- Scipy >= 1.4.0 is needed to guarantee
+        # results match.
+        if scipy_available and numpy.lib.NumpyVersion(
+                scipy.__version__) < '1.4.0':
+            raise NotImplementedError(
+                "Sparse __getitem__() requires Scipy >= 1.4.0")
+
+        row, col = self._parse_indices(key)
+
+        # Dispatch to specialized methods.
+        if isinstance(row, _int_scalar_types):
+            if isinstance(col, _int_scalar_types):
+                return self._get_intXint(row, col)
+            elif isinstance(col, slice):
+                return self._get_intXslice(row, col)
+            elif col.ndim == 1:
+                return self._get_intXarray(row, col)
+            raise IndexError('index results in >2 dimensions')
+        elif isinstance(row, slice):
+            if isinstance(col, _int_scalar_types):
+                return self._get_sliceXint(row, col)
+            elif isinstance(col, slice):
+                if row == slice(None) and row == col:
+                    return self.copy()
+                return self._get_sliceXslice(row, col)
+            elif col.ndim == 1:
+                return self._get_sliceXarray(row, col)
+            raise IndexError('index results in >2 dimensions')
+        elif row.ndim == 1:
+            if isinstance(col, _int_scalar_types):
+                return self._get_arrayXint(row, col)
+            elif isinstance(col, slice):
+                return self._get_arrayXslice(row, col)
+        else:  # row.ndim == 2
+            if isinstance(col, _int_scalar_types):
+                return self._get_arrayXint(row, col)
+            elif isinstance(col, slice):
+                raise IndexError('index results in >2 dimensions')
+            elif row.shape[1] == 1 and (col.ndim == 1 or col.shape[0] == 1):
+                # special case for outer indexing
+                return self._get_columnXarray(row[:, 0], col.ravel())
+
+        # The only remaining case is inner (fancy) indexing
+        row, col = cupy.broadcast_arrays(row, col)
+        if row.shape != col.shape:
+            raise IndexError('number of row and column indices differ')
+        if row.size == 0:
+            return self.__class__(cupy.atleast_2d(row).shape, dtype=self.dtype)
+        return self._get_arrayXarray(row, col)
+
+    def __setitem__(self, key, x):
+        row, col = self._parse_indices(key)
+
+        if isinstance(row, _int_scalar_types) and\
+                isinstance(col, _int_scalar_types):
+            x = cupy.asarray(x, dtype=self.dtype)
+            if x.size != 1:
+                raise ValueError('Trying to assign a sequence to an item')
+            self._set_intXint(row, col, x.flat[0])
+            return
+
+        if isinstance(row, slice):
+            row = cupy.arange(*row.indices(self.shape[0]))[:, None]
+        else:
+            row = cupy.atleast_1d(row)
+
+        if isinstance(col, slice):
+            col = cupy.arange(*col.indices(self.shape[1]))[None, :]
+            if row.ndim == 1:
+                row = row[:, None]
+        else:
+            col = cupy.atleast_1d(col)
+
+        i, j = cupy.broadcast_arrays(row, col)
+        if i.shape != j.shape:
+            raise IndexError('number of row and column indices differ')
+
+        if isspmatrix(x):
+            if i.ndim == 1:
+                # Inner indexing, so treat them like row vectors.
+                i = i[None]
+                j = j[None]
+            broadcast_row = x.shape[0] == 1 and i.shape[0] != 1
+            broadcast_col = x.shape[1] == 1 and i.shape[1] != 1
+            if not ((broadcast_row or x.shape[0] == i.shape[0]) and
+                    (broadcast_col or x.shape[1] == i.shape[1])):
+                raise ValueError('shape mismatch in assignment')
+            if x.size == 0:
+                return
+            x = x.tocoo(copy=True)
+            x.sum_duplicates()
+            self._set_arrayXarray_sparse(i, j, x)
+        else:
+            # Make x and i into the same shape
+            x = cupy.asarray(x, dtype=self.dtype)
+            x, _ = cupy.broadcast_arrays(x, i)
+            if x.size == 0:
+                return
+            x = x.reshape(i.shape)
+            self._set_arrayXarray(i, j, x)
+
+    def _is_scalar(self, index):
+        if isinstance(index, (cupy.ndarray, numpy.ndarray)) and \
+                index.ndim == 0 and index.size == 1:
+            return True
+        return False
+
+    def _parse_indices(self, key):
+        M, N = self.shape
+        row, col = _unpack_index(key)
+
+        if self._is_scalar(row):
+            row = row.item()
+        if self._is_scalar(col):
+            col = col.item()
+
+        # Scipy calls sputils.isintlike() rather than
+        # isinstance(x, _int_scalar_types). Comparing directly to int
+        # here to minimize the impact of nested exception catching
+
+        if isinstance(row, _int_scalar_types):
+            row = _normalize_index(row, M, 'row')
+        elif not isinstance(row, slice):
+            row = self._asindices(row, M)
+
+        if isinstance(col, _int_scalar_types):
+            col = _normalize_index(col, N, 'column')
+        elif not isinstance(col, slice):
+            col = self._asindices(col, N)
+
+        return row, col
+
+    def _asindices(self, idx, length):
+        """Convert `idx` to a valid index for an axis with a given length.
+        Subclasses that need special validation can override this method.
+
+        idx is assumed to be at least a 1-dimensional array-like, but can
+        have no more than 2 dimensions.
+        """
+        try:
+            x = cupy.asarray(idx, dtype=self.indices.dtype)
+        except (ValueError, TypeError, MemoryError):
+            raise IndexError('invalid index')
+
+        if x.ndim not in (1, 2):
+            raise IndexError('Index dimension must be <= 2')
+
+        return x % length
+
+    def getrow(self, i):
+        """Return a copy of row i of the matrix, as a (1 x n) row vector.
+
+        Args:
+            i (integer): Row
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix: Sparse matrix with single row
+        """
+        M, N = self.shape
+        i = _normalize_index(i, M, 'index')
+        return self._get_intXslice(i, slice(None))
+
+    def getcol(self, i):
+        """Return a copy of column i of the matrix, as a (m x 1) column vector.
+
+        Args:
+            i (integer): Column
+
+        Returns:
+            cupyx.scipy.sparse.spmatrix: Sparse matrix with single column
+        """
+        M, N = self.shape
+        i = _normalize_index(i, N, 'index')
+        return self._get_sliceXint(slice(None), i)
+
+    def _get_intXint(self, row, col):
+        raise NotImplementedError()
+
+    def _get_intXarray(self, row, col):
+        raise NotImplementedError()
+
+    def _get_intXslice(self, row, col):
+        raise NotImplementedError()
+
+    def _get_sliceXint(self, row, col):
+        raise NotImplementedError()
+
+    def _get_sliceXslice(self, row, col):
+        raise NotImplementedError()
+
+    def _get_sliceXarray(self, row, col):
+        raise NotImplementedError()
+
+    def _get_arrayXint(self, row, col):
+        raise NotImplementedError()
+
+    def _get_arrayXslice(self, row, col):
+        raise NotImplementedError()
+
+    def _get_columnXarray(self, row, col):
+        raise NotImplementedError()
+
+    def _get_arrayXarray(self, row, col):
+        raise NotImplementedError()
+
+    def _set_intXint(self, row, col, x):
+        raise NotImplementedError()
+
+    def _set_arrayXarray(self, row, col, x):
+        raise NotImplementedError()
+
+    def _set_arrayXarray_sparse(self, row, col, x):
+        # Fall back to densifying x
+        x = cupy.asarray(x.toarray(), dtype=self.dtype)
+        x, _ = cupy.broadcast_arrays(x, row)
+        self._set_arrayXarray(row, col, x)
+
+
+def _try_is_scipy_spmatrix(index):
+    if scipy_available:
+        return isinstance(index, scipy.sparse.spmatrix)
+    return False
+
+
+def _unpack_index(index):
+    """ Parse index. Always return a tuple of the form (row, col).
+    Valid type for row/col is integer, slice, or array of integers.
+
+    Returns:
+          resulting row & col indices : single integer, slice, or
+          array of integers. If row & column indices are supplied
+          explicitly, they are used as the major/minor indices.
+          If only one index is supplied, the minor index is
+          assumed to be all (e.g., [maj, :]).
+    """
+    # First, check if indexing with single boolean matrix.
+    if ((isinstance(index, (spmatrix, cupy.ndarray,
+                            numpy.ndarray))
+         or _try_is_scipy_spmatrix(index))
+            and index.ndim == 2 and index.dtype.kind == 'b'):
+        return index.nonzero()
+
+    # Parse any ellipses.
+    index = _eliminate_ellipsis(index)
+
+    # Next, parse the tuple or object
+    if isinstance(index, tuple):
+        if len(index) == 2:
+            row, col = index
+        elif len(index) == 1:
+            row, col = index[0], slice(None)
+        else:
+            raise IndexError('invalid number of indices')
+    else:
+        idx = _compatible_boolean_index(index)
+        if idx is None:
+            row, col = index, slice(None)
+        elif idx.ndim < 2:
+            return _boolean_index_to_array(idx), slice(None)
+        elif idx.ndim == 2:
+            return idx.nonzero()
+    # Next, check for validity and transform the index as needed.
+    if isspmatrix(row) or isspmatrix(col):
+        # Supporting sparse boolean indexing with both row and col does
+        # not work because spmatrix.ndim is always 2.
+        raise IndexError(
+            'Indexing with sparse matrices is not supported '
+            'except boolean indexing where matrix and index '
+            'are equal shapes.')
+    bool_row = _compatible_boolean_index(row)
+    bool_col = _compatible_boolean_index(col)
+    if bool_row is not None:
+        row = _boolean_index_to_array(bool_row)
+    if bool_col is not None:
+        col = _boolean_index_to_array(bool_col)
+    return row, col
+
+
+def _eliminate_ellipsis(index):
+    """Process indices with Ellipsis. Returns modified index."""
+    if index is Ellipsis:
+        return (slice(None), slice(None))
+
+    if not isinstance(index, tuple):
+        return index
+
+    # Find first ellipsis.
+    for j, v in enumerate(index):
+        if v is Ellipsis:
+            first_ellipsis = j
+            break
+    else:
+        return index
+
+    # Try to expand it using shortcuts for common cases
+    if len(index) == 1:
+        return (slice(None), slice(None))
+    if len(index) == 2:
+        if first_ellipsis == 0:
+            if index[1] is Ellipsis:
+                return (slice(None), slice(None))
+            return (slice(None), index[1])
+        return (index[0], slice(None))
+
+    # Expand it using a general-purpose algorithm
+    tail = []
+    for v in index[first_ellipsis+1:]:
+        if v is not Ellipsis:
+            tail.append(v)
+    nd = first_ellipsis + len(tail)
+    nslice = max(0, 2 - nd)
+    return index[:first_ellipsis] + (slice(None),) * nslice + tuple(tail,)
+
+
+def _normalize_index(x, dim, name):
+    if x < -dim or x >= dim:
+        raise IndexError('{} ({}) out of range'.format(name, x))
+    if x < 0:
+        x += dim
+    return x
+
+
+def _first_element_bool(idx, max_dim=2):
+    """Returns True if first element of the incompatible
+    array type is boolean.
+    """
+    if max_dim < 1:
+        return None
+    try:
+        first = idx[0] if len(idx) > 0 else None
+    except TypeError:
+        return None
+    if isinstance(first, _bool_scalar_types):
+        return True
+    return _first_element_bool(first, max_dim-1)
+
+
+def _compatible_boolean_index(idx):
+    """Returns a boolean index array that can be converted to
+    integer array. Returns None if no such array exists.
+    """
+    # presence of attribute `ndim` indicates a compatible array type.
+    if hasattr(idx, 'ndim'):
+        if idx.dtype.kind == 'b':
+            return idx
+    # non-ndarray bool collection should be converted to ndarray
+    elif _first_element_bool(idx):
+        return cupy.asarray(idx, dtype='bool')
+    return None
+
+
+def _boolean_index_to_array(idx):
+    if idx.ndim > 1:
+        raise IndexError('invalid index shape')
+    idx = cupy.array(idx, dtype=idx.dtype)
+    return cupy.where(idx)[0]
diff --git a/cupyx/scipy/sparse/_sputils.py b/cupyx/scipy/sparse/_sputils.py
new file mode 100644
index 0000000..b00d0ab
--- /dev/null
+++ b/cupyx/scipy/sparse/_sputils.py
@@ -0,0 +1,169 @@
+import cupy
+import operator
+import numpy
+
+from cupy._core._dtype import get_dtype
+
+supported_dtypes = [get_dtype(x) for x in
+                    ('single', 'double', 'csingle', 'cdouble')]
+
+_upcast_memo: dict = {}
+
+
+def isdense(x):
+    return isinstance(x, cupy.ndarray)
+
+
+def isscalarlike(x):
+    """Is x either a scalar, an array scalar, or a 0-dim array?"""
+    return cupy.isscalar(x) or (isdense(x) and x.ndim == 0)
+
+
+def get_index_dtype(arrays=(), maxval=None, check_contents=False):
+    """Based on input (integer) arrays ``a``, determines a suitable index data
+    type that can hold the data in the arrays.
+
+    Args:
+        arrays (tuple of array_like):
+            Input arrays whose types/contents to check
+        maxval (float, optional):
+            Maximum value needed
+        check_contents (bool, optional):
+            Whether to check the values in the arrays and not just their types.
+            Default: False (check only the types)
+
+    Returns:
+        dtype: Suitable index data type (int32 or int64)
+    """
+
+    int32min = cupy.iinfo(cupy.int32).min
+    int32max = cupy.iinfo(cupy.int32).max
+
+    dtype = cupy.int32
+    if maxval is not None:
+        if maxval > int32max:
+            dtype = cupy.int64
+
+    if isinstance(arrays, cupy.ndarray):
+        arrays = (arrays,)
+
+    for arr in arrays:
+        arr = cupy.asarray(arr)
+        if not cupy.can_cast(arr.dtype, cupy.int32):
+            if check_contents:
+                if arr.size == 0:
+                    # a bigger type not needed
+                    continue
+                elif cupy.issubdtype(arr.dtype, cupy.integer):
+                    maxval = arr.max()
+                    minval = arr.min()
+                    if minval >= int32min and maxval <= int32max:
+                        # a bigger type not needed
+                        continue
+
+            dtype = cupy.int64
+            break
+
+    return dtype
+
+
+def validateaxis(axis):
+    if axis is not None:
+        axis_type = type(axis)
+
+        # In NumPy, you can pass in tuples for 'axis', but they are
+        # not very useful for sparse matrices given their limited
+        # dimensions, so let's make it explicit that they are not
+        # allowed to be passed in
+        if axis_type == tuple:
+            raise TypeError(("Tuples are not accepted for the 'axis' "
+                             "parameter. Please pass in one of the "
+                             "following: {-2, -1, 0, 1, None}."))
+
+        # If not a tuple, check that the provided axis is actually
+        # an integer and raise a TypeError similar to NumPy's
+        if not cupy.issubdtype(cupy.dtype(axis_type), cupy.integer):
+            raise TypeError("axis must be an integer, not {name}"
+                            .format(name=axis_type.__name__))
+
+        if not (-2 <= axis <= 1):
+            raise ValueError("axis out of range")
+
+
+def upcast(*args):
+    """Returns the nearest supported sparse dtype for the
+    combination of one or more types.
+
+    upcast(t0, t1, ..., tn) -> T  where T is a supported dtype
+
+    Examples:
+        >>> upcast('int32')
+        <type 'numpy.int32'>
+        >>> upcast('int32','float32')
+        <type 'numpy.float64'>
+        >>> upcast('bool',float)
+        <type 'numpy.complex128'>
+    """
+
+    t = _upcast_memo.get(args)
+    if t is not None:
+        return t
+
+    upcast = numpy.result_type(*args)
+
+    for t in supported_dtypes:
+        if cupy.can_cast(upcast, t):
+            _upcast_memo[args] = t
+            return t
+
+    raise TypeError('no supported conversion for types: %r' % (args,))
+
+
+def check_shape(args, current_shape=None):
+    """Check validity of the shape"""
+
+    if len(args) == 0:
+        raise TypeError("function missing 1 required positional argument: "
+                        "'shape'")
+
+    elif len(args) == 1:
+        try:
+            shape_iter = iter(args[0])
+        except TypeError:
+            new_shape = (operator.index(args[0]), )
+        else:
+            new_shape = tuple(operator.index(arg) for arg in shape_iter)
+    else:
+        new_shape = tuple(operator.index(arg) for arg in args)
+
+    if current_shape is None:
+        if len(new_shape) != 2:
+            raise ValueError('shape must be a 2-tuple of positive integers')
+        elif new_shape[0] < 0 or new_shape[1] < 0:
+            raise ValueError("'shape' elements cannot be negative")
+
+    else:
+        current_size = numpy.prod(current_shape)
+
+        negative_indexes = [i for i, x in enumerate(new_shape) if x < 0]
+        if len(negative_indexes) == 0:
+            new_size = numpy.prod(new_shape)
+            if new_size != current_size:
+                raise ValueError('cannot reshape array of size {} into shape'
+                                 '{}'.format(current_size, new_shape))
+        elif len(negative_indexes) == 1:
+            skip = negative_indexes[0]
+            specified = numpy.prod(new_shape[0:skip] + new_shape[skip+1:])
+            unspecified, remainder = divmod(current_size, specified)
+            if remainder != 0:
+                err_shape = tuple('newshape'if x < 0 else x for x in new_shape)
+                raise ValueError('cannot reshape array of size {} into shape'
+                                 '{}'.format(current_size, err_shape))
+            new_shape = new_shape[0:skip] + (unspecified,) + new_shape[skip+1:]
+        else:
+            raise ValueError('can only specify one unknown dimension')
+
+    if len(new_shape) != 2:
+        raise ValueError('matrix shape must be two-dimensional')
+
+    return new_shape
diff --git a/cupyx/scipy/sparse/_util.py b/cupyx/scipy/sparse/_util.py
new file mode 100644
index 0000000..99f60e5
--- /dev/null
+++ b/cupyx/scipy/sparse/_util.py
@@ -0,0 +1,26 @@
+import cupy
+from cupy._core import core
+
+
+def isdense(x):
+    return isinstance(x, core.ndarray)
+
+
+def isintlike(x):
+    try:
+        return bool(int(x) == x)
+    except (TypeError, ValueError):
+        return False
+
+
+def isscalarlike(x):
+    return cupy.isscalar(x) or (isdense(x) and x.ndim == 0)
+
+
+def isshape(x):
+    if not isinstance(x, tuple) or len(x) != 2:
+        return False
+    m, n = x
+    if isinstance(n, tuple):
+        return False
+    return isintlike(m) and isintlike(n)
diff --git a/cupyx/scipy/sparse/csgraph/__init__.py b/cupyx/scipy/sparse/csgraph/__init__.py
new file mode 100644
index 0000000..8557ca7
--- /dev/null
+++ b/cupyx/scipy/sparse/csgraph/__init__.py
@@ -0,0 +1,4 @@
+# Functions from the following SciPy document
+# https://docs.scipy.org/doc/scipy/reference/sparse.csgraph.html
+
+from cupyx.scipy.sparse.csgraph._traversal import connected_components  # NOQA
diff --git a/cupyx/scipy/sparse/csgraph/_traversal.py b/cupyx/scipy/sparse/csgraph/_traversal.py
new file mode 100644
index 0000000..a3b92d1
--- /dev/null
+++ b/cupyx/scipy/sparse/csgraph/_traversal.py
@@ -0,0 +1,119 @@
+import cupy
+import cupyx.scipy.sparse
+try:
+    import pylibcugraph
+    pylibcugraph_available = True
+except ModuleNotFoundError:
+    pylibcugraph_available = False
+
+
+def connected_components(csgraph, directed=True, connection='weak',
+                         return_labels=True):
+    """Analyzes the connected components of a sparse graph
+
+    Args:
+        csgraph (cupy.ndarray of cupyx.scipy.sparse.csr_matrix): The adjacency
+            matrix representing connectivity among nodes.
+        directed (bool): If ``True``, it operates on a directed graph. If
+            ``False``, it operates on an undirected graph.
+        connection (str): ``'weak'`` or ``'strong'``. For directed graphs, the
+            type of connection to use. Nodes i and j are "strongly" connected
+            only when a path exists both from i to j and from j to i.
+            If ``directed`` is ``False``, this argument is ignored.
+        return_labels (bool): If ``True``, it returns the labels for each of
+            the connected components.
+
+    Returns:
+        tuple of int and cupy.ndarray, or int:
+            If ``return_labels`` == ``True``, returns a tuple ``(n, labels)``,
+            where ``n`` is the number of connected components and ``labels`` is
+            labels of each connected components. Otherwise, returns ``n``.
+
+    .. seealso:: :func:`scipy.sparse.csgraph.connected_components`
+    """
+    if not pylibcugraph_available:
+        raise RuntimeError('pylibcugraph is not available')
+
+    connection = connection.lower()
+    if connection not in ('weak', 'strong'):
+        raise ValueError("connection must be 'weak' or 'strong'")
+
+    if not directed:
+        connection = 'weak'
+
+    if csgraph.ndim != 2:
+        raise ValueError('graph should have two dimensions')
+
+    if not cupyx.scipy.sparse.isspmatrix_csr(csgraph):
+        csgraph = cupyx.scipy.sparse.csr_matrix(csgraph)
+    m, m1 = csgraph.shape
+    if m != m1:
+        raise ValueError('graph should be a square array')
+    if csgraph.nnz == 0:
+        return m, cupy.arange(m, dtype=csgraph.indices.dtype)
+
+    if connection == 'strong':
+        labels = cupy.empty(m, dtype=csgraph.indices.dtype)
+        pylibcugraph.strongly_connected_components(
+            offsets=csgraph.indptr, indices=csgraph.indices, weights=None,
+            num_verts=m, num_edges=csgraph.nnz, labels=labels)
+    else:
+        csgraph += csgraph.T
+        if not cupyx.scipy.sparse.isspmatrix_csr(csgraph):
+            csgraph = cupyx.scipy.sparse.csr_matrix(csgraph)
+        _, labels = pylibcugraph.weakly_connected_components(
+            resource_handle=None,
+            graph=None,
+            indices=csgraph.indices,
+            offsets=csgraph.indptr,
+            weights=None,
+            labels=None,
+            do_expensive_check=False,
+        )
+
+    count = cupy.zeros((1,), dtype=csgraph.indices.dtype)
+    root_labels = cupy.empty((m,), dtype=csgraph.indices.dtype)
+    _cupy_count_components(labels, count, root_labels, size=m)
+    n = int(count[0])
+    if not return_labels:
+        return n
+    _cupy_adjust_labels(n, cupy.sort(root_labels[:n]), labels)
+    return n, labels
+
+
+_cupy_count_components = cupy.ElementwiseKernel(
+    '',
+    'raw I labels, raw int32 count, raw int32 root_labels',
+    '''
+    int j = i;
+    while (j != labels[j]) { j = labels[j]; }
+    if (j != i) {
+        labels[i] = j;
+    } else {
+        int k = atomicAdd(&count[0], 1);
+        root_labels[k] = i;
+    }
+    ''',
+    '_cupy_count_components')
+
+
+_cupy_adjust_labels = cupy.ElementwiseKernel(
+    'int32 n_root_labels, raw I root_labels',
+    'I labels',
+    '''
+    int cur_label = labels;
+    int j_min = 0;
+    int j_max = n_root_labels - 1;
+    int j = (j_min + j_max) / 2;
+    while (j_min < j_max) {
+        if (cur_label == root_labels[j]) break;
+        if (cur_label < root_labels[j]) {
+            j_max = j - 1;
+        } else {
+            j_min = j + 1;
+        }
+        j = (j_min + j_max) / 2;
+    }
+    labels = j;
+    ''',
+    '_cupy_adjust_labels')
diff --git a/cupyx/scipy/sparse/linalg/__init__.py b/cupyx/scipy/sparse/linalg/__init__.py
new file mode 100644
index 0000000..d682600
--- /dev/null
+++ b/cupyx/scipy/sparse/linalg/__init__.py
@@ -0,0 +1,22 @@
+# Functions from the following SciPy document
+# https://docs.scipy.org/doc/scipy/reference/sparse.linalg.html
+
+# "NOQA" to suppress flake8 warning
+from cupyx.scipy.sparse.linalg._norm import norm  # NOQA
+from cupyx.scipy.sparse.linalg._solve import spsolve  # NOQA
+from cupyx.scipy.sparse.linalg._solve import spsolve_triangular  # NOQA
+from cupyx.scipy.sparse.linalg._solve import factorized  # NOQA
+from cupyx.scipy.sparse.linalg._solve import lsqr  # NOQA
+from cupyx.scipy.sparse.linalg._solve import lsmr  # NOQA
+from cupyx.scipy.sparse.linalg._solve import splu  # NOQA
+from cupyx.scipy.sparse.linalg._solve import spilu  # NOQA
+from cupyx.scipy.sparse.linalg._solve import SuperLU  # NOQA
+from cupyx.scipy.sparse.linalg._solve import minres  # NOQA
+from cupyx.scipy.sparse.linalg._eigen import eigsh  # NOQA
+from cupyx.scipy.sparse.linalg._eigen import svds  # NOQA
+from cupyx.scipy.sparse.linalg._iterative import cg  # NOQA
+from cupyx.scipy.sparse.linalg._iterative import gmres  # NOQA
+from cupyx.scipy.sparse.linalg._iterative import cgs  # NOQA
+from cupyx.scipy.sparse.linalg._interface import LinearOperator  # NOQA
+from cupyx.scipy.sparse.linalg._interface import aslinearoperator  # NOQA
+from cupyx.scipy.sparse.linalg._lobpcg import lobpcg  # NOQA
diff --git a/cupyx/scipy/sparse/linalg/_eigen.py b/cupyx/scipy/sparse/linalg/_eigen.py
new file mode 100644
index 0000000..270a1a5
--- /dev/null
+++ b/cupyx/scipy/sparse/linalg/_eigen.py
@@ -0,0 +1,422 @@
+import numpy
+import cupy
+
+from cupy import cublas
+from cupyx import cusparse
+from cupy._core import _dtype
+from cupy.cuda import device
+from cupy_backends.cuda.libs import cublas as _cublas
+from cupy_backends.cuda.libs import cusparse as _cusparse
+from cupyx.scipy.sparse import _csr
+from cupyx.scipy.sparse.linalg import _interface
+
+
+def eigsh(a, k=6, *, which='LM', ncv=None, maxiter=None, tol=0,
+          return_eigenvectors=True):
+    """
+    Find ``k`` eigenvalues and eigenvectors of the real symmetric square
+    matrix or complex Hermitian matrix ``A``.
+
+    Solves ``Ax = wx``, the standard eigenvalue problem for ``w`` eigenvalues
+    with corresponding eigenvectors ``x``.
+
+    Args:
+        a (ndarray, spmatrix or LinearOperator): A symmetric square matrix with
+            dimension ``(n, n)``. ``a`` must :class:`cupy.ndarray`,
+            :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        k (int): The number of eigenvalues and eigenvectors to compute. Must be
+            ``1 <= k < n``.
+        which (str): 'LM' or 'LA'. 'LM': finds ``k`` largest (in magnitude)
+            eigenvalues. 'LA': finds ``k`` largest (algebraic) eigenvalues.
+            'SA': finds ``k`` smallest (algebraic) eigenvalues.
+
+        ncv (int): The number of Lanczos vectors generated. Must be
+            ``k + 1 < ncv < n``. If ``None``, default value is used.
+        maxiter (int): Maximum number of Lanczos update iterations.
+            If ``None``, default value is used.
+        tol (float): Tolerance for residuals ``||Ax - wx||``. If ``0``, machine
+            precision is used.
+        return_eigenvectors (bool): If ``True``, returns eigenvectors in
+            addition to eigenvalues.
+
+    Returns:
+        tuple:
+            If ``return_eigenvectors is True``, it returns ``w`` and ``x``
+            where ``w`` is eigenvalues and ``x`` is eigenvectors. Otherwise,
+            it returns only ``w``.
+
+    .. seealso:: :func:`scipy.sparse.linalg.eigsh`
+
+    .. note::
+        This function uses the thick-restart Lanczos methods
+        (https://sdm.lbl.gov/~kewu/ps/trlan.html).
+
+    """
+    n = a.shape[0]
+    if a.ndim != 2 or a.shape[0] != a.shape[1]:
+        raise ValueError('expected square matrix (shape: {})'.format(a.shape))
+    if a.dtype.char not in 'fdFD':
+        raise TypeError('unsupprted dtype (actual: {})'.format(a.dtype))
+    if k <= 0:
+        raise ValueError('k must be greater than 0 (actual: {})'.format(k))
+    if k >= n:
+        raise ValueError('k must be smaller than n (actual: {})'.format(k))
+    if which not in ('LM', 'LA', 'SA'):
+        raise ValueError('which must be \'LM\',\'LA\'or\'SA\' (actual: {})'
+                         ''.format(which))
+    if ncv is None:
+        ncv = min(max(2 * k, k + 32), n - 1)
+    else:
+        ncv = min(max(ncv, k + 2), n - 1)
+    if maxiter is None:
+        maxiter = 10 * n
+    if tol == 0:
+        tol = numpy.finfo(a.dtype).eps
+
+    alpha = cupy.zeros((ncv,), dtype=a.dtype)
+    beta = cupy.zeros((ncv,), dtype=a.dtype.char.lower())
+    V = cupy.empty((ncv, n), dtype=a.dtype)
+
+    # Set initial vector
+    u = cupy.random.random((n,)).astype(a.dtype)
+    V[0] = u / cublas.nrm2(u)
+
+    # Choose Lanczos implementation, unconditionally use 'fast' for now
+    upadte_impl = 'fast'
+    if upadte_impl == 'fast':
+        lanczos = _lanczos_fast(a, n, ncv)
+    else:
+        lanczos = _lanczos_asis
+
+    # Lanczos iteration
+    lanczos(a, V, u, alpha, beta, 0, ncv)
+
+    iter = ncv
+    w, s = _eigsh_solve_ritz(alpha, beta, None, k, which)
+    x = V.T @ s
+
+    # Compute residual
+    beta_k = beta[-1] * s[-1, :]
+    res = cublas.nrm2(beta_k)
+
+    uu = cupy.empty((k,), dtype=a.dtype)
+
+    while res > tol and iter < maxiter:
+        # Setup for thick-restart
+        beta[:k] = 0
+        alpha[:k] = w
+        V[:k] = x.T
+
+        # u -= u.T @ V[:k].conj().T @ V[:k]
+        cublas.gemv(_cublas.CUBLAS_OP_C, 1, V[:k].T, u, 0, uu)
+        cublas.gemv(_cublas.CUBLAS_OP_N, -1, V[:k].T, uu, 1, u)
+        V[k] = u / cublas.nrm2(u)
+
+        u[...] = a @ V[k]
+        cublas.dotc(V[k], u, out=alpha[k])
+        u -= alpha[k] * V[k]
+        u -= V[:k].T @ beta_k
+        cublas.nrm2(u, out=beta[k])
+        V[k+1] = u / beta[k]
+
+        # Lanczos iteration
+        lanczos(a, V, u, alpha, beta, k + 1, ncv)
+
+        iter += ncv - k
+        w, s = _eigsh_solve_ritz(alpha, beta, beta_k, k, which)
+        x = V.T @ s
+
+        # Compute residual
+        beta_k = beta[-1] * s[-1, :]
+        res = cublas.nrm2(beta_k)
+
+    if return_eigenvectors:
+        idx = cupy.argsort(w)
+        return w[idx], x[:, idx]
+    else:
+        return cupy.sort(w)
+
+
+def _lanczos_asis(a, V, u, alpha, beta, i_start, i_end):
+    for i in range(i_start, i_end):
+        u[...] = a @ V[i]
+        cublas.dotc(V[i], u, out=alpha[i])
+        u -= u.T @ V[:i+1].conj().T @ V[:i+1]
+        cublas.nrm2(u, out=beta[i])
+        if i >= i_end - 1:
+            break
+        V[i+1] = u / beta[i]
+
+
+def _lanczos_fast(A, n, ncv):
+    cublas_handle = device.get_cublas_handle()
+    cublas_pointer_mode = _cublas.getPointerMode(cublas_handle)
+    if A.dtype.char == 'f':
+        dotc = _cublas.sdot
+        nrm2 = _cublas.snrm2
+        gemv = _cublas.sgemv
+        axpy = _cublas.saxpy
+    elif A.dtype.char == 'd':
+        dotc = _cublas.ddot
+        nrm2 = _cublas.dnrm2
+        gemv = _cublas.dgemv
+        axpy = _cublas.daxpy
+    elif A.dtype.char == 'F':
+        dotc = _cublas.cdotc
+        nrm2 = _cublas.scnrm2
+        gemv = _cublas.cgemv
+        axpy = _cublas.caxpy
+    elif A.dtype.char == 'D':
+        dotc = _cublas.zdotc
+        nrm2 = _cublas.dznrm2
+        gemv = _cublas.zgemv
+        axpy = _cublas.zaxpy
+    else:
+        raise TypeError('invalid dtype ({})'.format(A.dtype))
+
+    cusparse_handle = None
+    if _csr.isspmatrix_csr(A) and cusparse.check_availability('spmv'):
+        cusparse_handle = device.get_cusparse_handle()
+        spmv_op_a = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+        spmv_alpha = numpy.array(1.0, A.dtype)
+        spmv_beta = numpy.array(0.0, A.dtype)
+        spmv_cuda_dtype = _dtype.to_cuda_dtype(A.dtype)
+        spmv_alg = _cusparse.CUSPARSE_MV_ALG_DEFAULT
+
+    v = cupy.empty((n,), dtype=A.dtype)
+    uu = cupy.empty((ncv,), dtype=A.dtype)
+    vv = cupy.empty((n,), dtype=A.dtype)
+    b = cupy.empty((), dtype=A.dtype)
+    one = numpy.array(1.0, dtype=A.dtype)
+    zero = numpy.array(0.0, dtype=A.dtype)
+    mone = numpy.array(-1.0, dtype=A.dtype)
+
+    outer_A = A
+
+    def aux(A, V, u, alpha, beta, i_start, i_end):
+        assert A is outer_A
+
+        # Get ready for spmv if enabled
+        if cusparse_handle is not None:
+            # Note: I would like to reuse descriptors and working buffer
+            # on the next update, but I gave it up because it sometimes
+            # caused illegal memory access error.
+            spmv_desc_A = cusparse.SpMatDescriptor.create(A)
+            spmv_desc_v = cusparse.DnVecDescriptor.create(v)
+            spmv_desc_u = cusparse.DnVecDescriptor.create(u)
+            buff_size = _cusparse.spMV_bufferSize(
+                cusparse_handle, spmv_op_a, spmv_alpha.ctypes.data,
+                spmv_desc_A.desc, spmv_desc_v.desc, spmv_beta.ctypes.data,
+                spmv_desc_u.desc, spmv_cuda_dtype, spmv_alg)
+            spmv_buff = cupy.empty(buff_size, cupy.int8)
+
+        v[...] = V[i_start]
+        for i in range(i_start, i_end):
+            # Matrix-vector multiplication
+            if cusparse_handle is None:
+                u[...] = A @ v
+            else:
+                _cusparse.spMV(
+                    cusparse_handle, spmv_op_a, spmv_alpha.ctypes.data,
+                    spmv_desc_A.desc, spmv_desc_v.desc,
+                    spmv_beta.ctypes.data, spmv_desc_u.desc,
+                    spmv_cuda_dtype, spmv_alg, spmv_buff.data.ptr)
+
+            # Call dotc: alpha[i] = v.conj().T @ u
+            _cublas.setPointerMode(
+                cublas_handle, _cublas.CUBLAS_POINTER_MODE_DEVICE)
+            try:
+                dotc(cublas_handle, n, v.data.ptr, 1, u.data.ptr, 1,
+                     alpha.data.ptr + i * alpha.itemsize)
+            finally:
+                _cublas.setPointerMode(cublas_handle, cublas_pointer_mode)
+
+            # Orthogonalize: u = u - alpha[i] * v - beta[i - 1] * V[i - 1]
+            vv.fill(0)
+            b[...] = beta[i - 1]    # cast from real to complex
+            _cublas.setPointerMode(
+                cublas_handle, _cublas.CUBLAS_POINTER_MODE_DEVICE)
+            try:
+                axpy(cublas_handle, n,
+                     alpha.data.ptr + i * alpha.itemsize,
+                     v.data.ptr, 1, vv.data.ptr, 1)
+                axpy(cublas_handle, n,
+                     b.data.ptr,
+                     V[i - 1].data.ptr, 1, vv.data.ptr, 1)
+            finally:
+                _cublas.setPointerMode(cublas_handle, cublas_pointer_mode)
+            axpy(cublas_handle, n,
+                 mone.ctypes.data,
+                 vv.data.ptr, 1, u.data.ptr, 1)
+
+            # Reorthogonalize: u -= V @ (V.conj().T @ u)
+            gemv(cublas_handle, _cublas.CUBLAS_OP_C,
+                 n, i + 1,
+                 one.ctypes.data, V.data.ptr, n,
+                 u.data.ptr, 1,
+                 zero.ctypes.data, uu.data.ptr, 1)
+            gemv(cublas_handle, _cublas.CUBLAS_OP_N,
+                 n, i + 1,
+                 mone.ctypes.data, V.data.ptr, n,
+                 uu.data.ptr, 1,
+                 one.ctypes.data, u.data.ptr, 1)
+            alpha[i] += uu[i]
+
+            # Call nrm2
+            _cublas.setPointerMode(
+                cublas_handle, _cublas.CUBLAS_POINTER_MODE_DEVICE)
+            try:
+                nrm2(cublas_handle, n, u.data.ptr, 1,
+                     beta.data.ptr + i * beta.itemsize)
+            finally:
+                _cublas.setPointerMode(cublas_handle, cublas_pointer_mode)
+
+            # Break here as the normalization below touches V[i+1]
+            if i >= i_end - 1:
+                break
+
+            # Normalize
+            _kernel_normalize(u, beta, i, n, v, V)
+
+    return aux
+
+
+_kernel_normalize = cupy.ElementwiseKernel(
+    'T u, raw S beta, int32 j, int32 n', 'T v, raw T V',
+    'v = u / beta[j]; V[i + (j+1) * n] = v;', 'cupy_eigsh_normalize'
+)
+
+
+def _eigsh_solve_ritz(alpha, beta, beta_k, k, which):
+    # Note: This is done on the CPU, because there is an issue in
+    # cupy.linalg.eigh with CUDA 9.2, which can return NaNs. It will has little
+    # impact on performance, since the matrix size processed here is not large.
+    alpha = cupy.asnumpy(alpha)
+    beta = cupy.asnumpy(beta)
+    t = numpy.diag(alpha)
+    t = t + numpy.diag(beta[:-1], k=1)
+    t = t + numpy.diag(beta[:-1], k=-1)
+    if beta_k is not None:
+        beta_k = cupy.asnumpy(beta_k)
+        t[k, :k] = beta_k
+        t[:k, k] = beta_k
+    w, s = numpy.linalg.eigh(t)
+
+    # Pick-up k ritz-values and ritz-vectors
+    if which == 'LA':
+        idx = numpy.argsort(w)
+        wk = w[idx[-k:]]
+        sk = s[:, idx[-k:]]
+    elif which == 'LM':
+        idx = numpy.argsort(numpy.absolute(w))
+        wk = w[idx[-k:]]
+        sk = s[:, idx[-k:]]
+
+    elif which == 'SA':
+        idx = numpy.argsort(w)
+        wk = w[idx[:k]]
+        sk = s[:, idx[:k]]
+    # elif which == 'SM':  #dysfunctional
+    #   idx = cupy.argsort(abs(w))
+    #   wk = w[idx[:k]]
+    #   sk = s[:,idx[:k]]
+    return cupy.array(wk), cupy.array(sk)
+
+
+def svds(a, k=6, *, ncv=None, tol=0, which='LM', maxiter=None,
+         return_singular_vectors=True):
+    """Finds the largest ``k`` singular values/vectors for a sparse matrix.
+
+    Args:
+        a (ndarray, spmatrix or LinearOperator): A real or complex array with
+            dimension ``(m, n)``. ``a`` must :class:`cupy.ndarray`,
+            :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        k (int): The number of singular values/vectors to compute. Must be
+            ``1 <= k < min(m, n)``.
+        ncv (int): The number of Lanczos vectors generated. Must be
+            ``k + 1 < ncv < min(m, n)``. If ``None``, default value is used.
+        tol (float): Tolerance for singular values. If ``0``, machine precision
+            is used.
+        which (str): Only 'LM' is supported. 'LM': finds ``k`` largest singular
+            values.
+        maxiter (int): Maximum number of Lanczos update iterations.
+            If ``None``, default value is used.
+        return_singular_vectors (bool): If ``True``, returns singular vectors
+            in addition to singular values.
+
+    Returns:
+        tuple:
+            If ``return_singular_vectors`` is ``True``, it returns ``u``, ``s``
+            and ``vt`` where ``u`` is left singular vectors, ``s`` is singular
+            values and ``vt`` is right singular vectors. Otherwise, it returns
+            only ``s``.
+
+    .. seealso:: :func:`scipy.sparse.linalg.svds`
+
+    .. note::
+        This is a naive implementation using cupyx.scipy.sparse.linalg.eigsh as
+        an eigensolver on ``a.H @ a`` or ``a @ a.H``.
+
+    """
+    if a.ndim != 2:
+        raise ValueError('expected 2D (shape: {})'.format(a.shape))
+    if a.dtype.char not in 'fdFD':
+        raise TypeError('unsupprted dtype (actual: {})'.format(a.dtype))
+    m, n = a.shape
+    if k <= 0:
+        raise ValueError('k must be greater than 0 (actual: {})'.format(k))
+    if k >= min(m, n):
+        raise ValueError('k must be smaller than min(m, n) (actual: {})'
+                         ''.format(k))
+
+    a = _interface.aslinearoperator(a)
+    if m >= n:
+        aH, a = a.H, a
+    else:
+        aH, a = a, a.H
+
+    if return_singular_vectors:
+        w, x = eigsh(aH @ a, k=k, which=which, ncv=ncv, maxiter=maxiter,
+                     tol=tol, return_eigenvectors=True)
+    else:
+        w = eigsh(aH @ a, k=k, which=which, ncv=ncv, maxiter=maxiter, tol=tol,
+                  return_eigenvectors=False)
+
+    w = cupy.maximum(w, 0)
+    t = w.dtype.char.lower()
+    factor = {'f': 1e3, 'd': 1e6}
+    cond = factor[t] * numpy.finfo(t).eps
+    cutoff = cond * cupy.max(w)
+    above_cutoff = (w > cutoff)
+    n_large = above_cutoff.sum().item()
+    s = cupy.zeros_like(w)
+    s[:n_large] = cupy.sqrt(w[above_cutoff])
+    if not return_singular_vectors:
+        return s
+
+    x = x[:, above_cutoff]
+    if m >= n:
+        v = x
+        u = a @ v / s[:n_large]
+    else:
+        u = x
+        v = a @ u / s[:n_large]
+    u = _augmented_orthnormal_cols(u, k - n_large)
+    v = _augmented_orthnormal_cols(v, k - n_large)
+
+    return u, s, v.conj().T
+
+
+def _augmented_orthnormal_cols(x, n_aug):
+    if n_aug <= 0:
+        return x
+    m, n = x.shape
+    y = cupy.empty((m, n + n_aug), dtype=x.dtype)
+    y[:, :n] = x
+    for i in range(n, n + n_aug):
+        v = cupy.random.random((m, )).astype(x.dtype)
+        v -= v @ y[:, :i].conj() @ y[:, :i].T
+        y[:, i] = v / cupy.linalg.norm(v)
+    return y
diff --git a/cupyx/scipy/sparse/linalg/_interface.py b/cupyx/scipy/sparse/linalg/_interface.py
new file mode 100644
index 0000000..7ee35e0
--- /dev/null
+++ b/cupyx/scipy/sparse/linalg/_interface.py
@@ -0,0 +1,578 @@
+import warnings
+
+import cupy
+
+from cupyx.scipy import sparse
+from cupyx.scipy.sparse import _util
+
+
+class LinearOperator(object):
+    """LinearOperator(shape, matvec, rmatvec=None, matmat=None, dtype=None, \
+rmatmat=None)
+
+    Common interface for performing matrix vector products
+
+    To construct a concrete LinearOperator, either pass appropriate callables
+    to the constructor of this class, or subclass it.
+
+    Args:
+        shape (tuple):  Matrix dimensions ``(M, N)``.
+        matvec (callable f(v)):  Returns returns ``A * v``.
+        rmatvec (callable f(v)):  Returns ``A^H * v``, where ``A^H`` is the
+                                  conjugate transpose of ``A``.
+        matmat (callable f(V)):  Returns ``A * V``, where ``V`` is a dense
+                                 matrix with dimensions ``(N, K)``.
+        dtype (dtype):  Data type of the matrix.
+        rmatmat (callable f(V)):  Returns ``A^H * V``, where ``V`` is a dense
+                                  matrix with dimensions ``(M, K)``.
+
+    .. seealso:: :class:`scipy.sparse.linalg.LinearOperator`
+    """
+
+    ndim = 2
+
+    def __new__(cls, *args, **kwargs):
+        if cls is LinearOperator:
+            # Operate as _CustomLinearOperator factory.
+            return super(LinearOperator, cls).__new__(_CustomLinearOperator)
+        else:
+            obj = super(LinearOperator, cls).__new__(cls)
+
+            if (type(obj)._matvec == LinearOperator._matvec
+                    and type(obj)._matmat == LinearOperator._matmat):
+                warnings.warn('LinearOperator subclass should implement'
+                              ' at least one of _matvec and _matmat.',
+                              category=RuntimeWarning, stacklevel=2)
+
+            return obj
+
+    def __init__(self, dtype, shape):
+        """Initialize this :class:`LinearOperator`
+        """
+        if dtype is not None:
+            dtype = cupy.dtype(dtype)
+
+        shape = tuple(shape)
+        if not _util.isshape(shape):
+            raise ValueError('invalid shape %r (must be 2-d)' % (shape,))
+
+        self.dtype = dtype
+        self.shape = shape
+
+    def _init_dtype(self):
+        """Called from subclasses at the end of the `__init__` routine.
+        """
+        if self.dtype is None:
+            v = cupy.zeros(self.shape[-1])
+            self.dtype = self.matvec(v).dtype
+
+    def _matmat(self, X):
+        """Default matrix-matrix multiplication handler.
+        """
+
+        return cupy.hstack([self.matvec(col.reshape(-1, 1)) for col in X.T])
+
+    def _matvec(self, x):
+        """Default matrix-vector multiplication handler.
+        """
+        return self.matmat(x.reshape(-1, 1))
+
+    def matvec(self, x):
+        """Matrix-vector multiplication.
+        """
+
+        M, N = self.shape
+
+        if x.shape != (N,) and x.shape != (N, 1):
+            raise ValueError('dimension mismatch')
+
+        y = self._matvec(x)
+
+        if x.ndim == 1:
+            y = y.reshape(M)
+        elif x.ndim == 2:
+            y = y.reshape(M, 1)
+        else:
+            raise ValueError('invalid shape returned by user-defined matvec()')
+
+        return y
+
+    def rmatvec(self, x):
+        """Adjoint matrix-vector multiplication.
+        """
+
+        M, N = self.shape
+
+        if x.shape != (M,) and x.shape != (M, 1):
+            raise ValueError('dimension mismatch')
+
+        y = self._rmatvec(x)
+
+        if x.ndim == 1:
+            y = y.reshape(N)
+        elif x.ndim == 2:
+            y = y.reshape(N, 1)
+        else:
+            raise ValueError(
+                'invalid shape returned by user-defined rmatvec()')
+
+        return y
+
+    def _rmatvec(self, x):
+        """Default implementation of _rmatvec; defers to adjoint.
+        """
+        if type(self)._adjoint == LinearOperator._adjoint:
+            # _adjoint not overridden, prevent infinite recursion
+            raise NotImplementedError
+        else:
+            return self.H.matvec(x)
+
+    def matmat(self, X):
+        """Matrix-matrix multiplication.
+        """
+
+        if X.ndim != 2:
+            raise ValueError('expected 2-d ndarray or matrix, not %d-d'
+                             % X.ndim)
+
+        if X.shape[0] != self.shape[1]:
+            raise ValueError('dimension mismatch: %r, %r'
+                             % (self.shape, X.shape))
+
+        Y = self._matmat(X)
+
+        return Y
+
+    def rmatmat(self, X):
+        """Adjoint matrix-matrix multiplication.
+        """
+
+        if X.ndim != 2:
+            raise ValueError('expected 2-d ndarray or matrix, not %d-d'
+                             % X.ndim)
+
+        if X.shape[0] != self.shape[0]:
+            raise ValueError('dimension mismatch: %r, %r'
+                             % (self.shape, X.shape))
+
+        Y = self._rmatmat(X)
+        return Y
+
+    def _rmatmat(self, X):
+        """Default implementation of _rmatmat defers to rmatvec or adjoint."""
+        if type(self)._adjoint == LinearOperator._adjoint:
+            return cupy.hstack([self.rmatvec(col.reshape(-1, 1))
+                                for col in X.T])
+        else:
+            return self.H.matmat(X)
+
+    def __call__(self, x):
+        return self*x
+
+    def __mul__(self, x):
+        return self.dot(x)
+
+    def dot(self, x):
+        """Matrix-matrix or matrix-vector multiplication.
+        """
+        if isinstance(x, LinearOperator):
+            return _ProductLinearOperator(self, x)
+        elif cupy.isscalar(x):
+            return _ScaledLinearOperator(self, x)
+        else:
+            if x.ndim == 1 or x.ndim == 2 and x.shape[1] == 1:
+                return self.matvec(x)
+            elif x.ndim == 2:
+                return self.matmat(x)
+            else:
+                raise ValueError('expected 1-d or 2-d array, got %r'
+                                 % x)
+
+    def __matmul__(self, other):
+        if cupy.isscalar(other):
+            raise ValueError('Scalar operands are not allowed, '
+                             'use \'*\' instead')
+        return self.__mul__(other)
+
+    def __rmatmul__(self, other):
+        if cupy.isscalar(other):
+            raise ValueError('Scalar operands are not allowed, '
+                             'use \'*\' instead')
+        return self.__rmul__(other)
+
+    def __rmul__(self, x):
+        if cupy.isscalar(x):
+            return _ScaledLinearOperator(self, x)
+        else:
+            return NotImplemented
+
+    def __pow__(self, p):
+        if cupy.isscalar(p):
+            return _PowerLinearOperator(self, p)
+        else:
+            return NotImplemented
+
+    def __add__(self, x):
+        if isinstance(x, LinearOperator):
+            return _SumLinearOperator(self, x)
+        else:
+            return NotImplemented
+
+    def __neg__(self):
+        return _ScaledLinearOperator(self, -1)
+
+    def __sub__(self, x):
+        return self.__add__(-x)
+
+    def __repr__(self):
+        M, N = self.shape
+        if self.dtype is None:
+            dt = 'unspecified dtype'
+        else:
+            dt = 'dtype=' + str(self.dtype)
+
+        return '<%dx%d %s with %s>' % (M, N, self.__class__.__name__, dt)
+
+    def adjoint(self):
+        """Hermitian adjoint.
+        """
+        return self._adjoint()
+
+    H = property(adjoint)
+
+    def transpose(self):
+        """Transpose this linear operator.
+        """
+        return self._transpose()
+
+    T = property(transpose)
+
+    def _adjoint(self):
+        """Default implementation of _adjoint; defers to rmatvec."""
+        return _AdjointLinearOperator(self)
+
+    def _transpose(self):
+        """ Default implementation of _transpose; defers to rmatvec + conj"""
+        return _TransposedLinearOperator(self)
+
+
+class _CustomLinearOperator(LinearOperator):
+    """Linear operator defined in terms of user-specified operations."""
+
+    def __init__(self, shape, matvec, rmatvec=None, matmat=None,
+                 dtype=None, rmatmat=None):
+        super(_CustomLinearOperator, self).__init__(dtype, shape)
+
+        self.args = ()
+
+        self.__matvec_impl = matvec
+        self.__rmatvec_impl = rmatvec
+        self.__rmatmat_impl = rmatmat
+        self.__matmat_impl = matmat
+
+        self._init_dtype()
+
+    def _matmat(self, X):
+        if self.__matmat_impl is not None:
+            return self.__matmat_impl(X)
+        else:
+            return super(_CustomLinearOperator, self)._matmat(X)
+
+    def _matvec(self, x):
+        return self.__matvec_impl(x)
+
+    def _rmatvec(self, x):
+        func = self.__rmatvec_impl
+        if func is None:
+            raise NotImplementedError('rmatvec is not defined')
+        return self.__rmatvec_impl(x)
+
+    def _rmatmat(self, X):
+        if self.__rmatmat_impl is not None:
+            return self.__rmatmat_impl(X)
+        else:
+            return super(_CustomLinearOperator, self)._rmatmat(X)
+
+    def _adjoint(self):
+        return _CustomLinearOperator(shape=(self.shape[1], self.shape[0]),
+                                     matvec=self.__rmatvec_impl,
+                                     rmatvec=self.__matvec_impl,
+                                     matmat=self.__rmatmat_impl,
+                                     rmatmat=self.__matmat_impl,
+                                     dtype=self.dtype)
+
+
+class _AdjointLinearOperator(LinearOperator):
+    """Adjoint of arbitrary Linear Operator"""
+
+    def __init__(self, A):
+        shape = (A.shape[1], A.shape[0])
+        super(_AdjointLinearOperator, self).__init__(
+            dtype=A.dtype, shape=shape)
+        self.A = A
+        self.args = (A,)
+
+    def _matvec(self, x):
+        return self.A._rmatvec(x)
+
+    def _rmatvec(self, x):
+        return self.A._matvec(x)
+
+    def _matmat(self, x):
+        return self.A._rmatmat(x)
+
+    def _rmatmat(self, x):
+        return self.A._matmat(x)
+
+
+class _TransposedLinearOperator(LinearOperator):
+    """Transposition of arbitrary Linear Operator"""
+
+    def __init__(self, A):
+        shape = (A.shape[1], A.shape[0])
+        super(_TransposedLinearOperator, self).__init__(
+            dtype=A.dtype, shape=shape)
+        self.A = A
+        self.args = (A,)
+
+    def _matvec(self, x):
+        # NB. cupy.conj works also on sparse matrices
+        return cupy.conj(self.A._rmatvec(cupy.conj(x)))
+
+    def _rmatvec(self, x):
+        return cupy.conj(self.A._matvec(cupy.conj(x)))
+
+    def _matmat(self, x):
+        # NB. cupy.conj works also on sparse matrices
+        return cupy.conj(self.A._rmatmat(cupy.conj(x)))
+
+    def _rmatmat(self, x):
+        return cupy.conj(self.A._matmat(cupy.conj(x)))
+
+
+def _get_dtype(operators, dtypes=None):
+    if dtypes is None:
+        dtypes = []
+    for obj in operators:
+        if obj is not None and hasattr(obj, 'dtype'):
+            dtypes.append(obj.dtype)
+    return cupy.result_type(*dtypes)
+
+
+class _SumLinearOperator(LinearOperator):
+    def __init__(self, A, B):
+        if not isinstance(A, LinearOperator) or \
+                not isinstance(B, LinearOperator):
+            raise ValueError('both operands have to be a LinearOperator')
+        if A.shape != B.shape:
+            raise ValueError('cannot add %r and %r: shape mismatch'
+                             % (A, B))
+        self.args = (A, B)
+        super(_SumLinearOperator, self).__init__(_get_dtype([A, B]), A.shape)
+
+    def _matvec(self, x):
+        return self.args[0].matvec(x) + self.args[1].matvec(x)
+
+    def _rmatvec(self, x):
+        return self.args[0].rmatvec(x) + self.args[1].rmatvec(x)
+
+    def _rmatmat(self, x):
+        return self.args[0].rmatmat(x) + self.args[1].rmatmat(x)
+
+    def _matmat(self, x):
+        return self.args[0].matmat(x) + self.args[1].matmat(x)
+
+    def _adjoint(self):
+        A, B = self.args
+        return A.H + B.H
+
+
+class _ProductLinearOperator(LinearOperator):
+    def __init__(self, A, B):
+        if not isinstance(A, LinearOperator) or \
+                not isinstance(B, LinearOperator):
+            raise ValueError('both operands have to be a LinearOperator')
+        if A.shape[1] != B.shape[0]:
+            raise ValueError('cannot multiply %r and %r: shape mismatch'
+                             % (A, B))
+        super(_ProductLinearOperator, self).__init__(_get_dtype([A, B]),
+                                                     (A.shape[0], B.shape[1]))
+        self.args = (A, B)
+
+    def _matvec(self, x):
+        return self.args[0].matvec(self.args[1].matvec(x))
+
+    def _rmatvec(self, x):
+        return self.args[1].rmatvec(self.args[0].rmatvec(x))
+
+    def _rmatmat(self, x):
+        return self.args[1].rmatmat(self.args[0].rmatmat(x))
+
+    def _matmat(self, x):
+        return self.args[0].matmat(self.args[1].matmat(x))
+
+    def _adjoint(self):
+        A, B = self.args
+        return B.H * A.H
+
+
+class _ScaledLinearOperator(LinearOperator):
+    def __init__(self, A, alpha):
+        if not isinstance(A, LinearOperator):
+            raise ValueError('LinearOperator expected as A')
+        if not cupy.isscalar(alpha):
+            raise ValueError('scalar expected as alpha')
+        dtype = _get_dtype([A], [type(alpha)])
+        super(_ScaledLinearOperator, self).__init__(dtype, A.shape)
+        self.args = (A, alpha)
+
+    def _matvec(self, x):
+        return self.args[1] * self.args[0].matvec(x)
+
+    def _rmatvec(self, x):
+        return cupy.conj(self.args[1]) * self.args[0].rmatvec(x)
+
+    def _rmatmat(self, x):
+        return cupy.conj(self.args[1]) * self.args[0].rmatmat(x)
+
+    def _matmat(self, x):
+        return self.args[1] * self.args[0].matmat(x)
+
+    def _adjoint(self):
+        A, alpha = self.args
+        return A.H * cupy.conj(alpha)
+
+
+class _PowerLinearOperator(LinearOperator):
+    def __init__(self, A, p):
+        if not isinstance(A, LinearOperator):
+            raise ValueError('LinearOperator expected as A')
+        if A.shape[0] != A.shape[1]:
+            raise ValueError('square LinearOperator expected, got %r' % A)
+        if not _util.isintlike(p) or p < 0:
+            raise ValueError('non-negative integer expected as p')
+
+        super(_PowerLinearOperator, self).__init__(_get_dtype([A]), A.shape)
+        self.args = (A, p)
+
+    def _power(self, fun, x):
+        res = cupy.array(x, copy=True)
+        for i in range(self.args[1]):
+            res = fun(res)
+        return res
+
+    def _matvec(self, x):
+        return self._power(self.args[0].matvec, x)
+
+    def _rmatvec(self, x):
+        return self._power(self.args[0].rmatvec, x)
+
+    def _rmatmat(self, x):
+        return self._power(self.args[0].rmatmat, x)
+
+    def _matmat(self, x):
+        return self._power(self.args[0].matmat, x)
+
+    def _adjoint(self):
+        A, p = self.args
+        return A.H ** p
+
+
+class MatrixLinearOperator(LinearOperator):
+    def __init__(self, A):
+        super(MatrixLinearOperator, self).__init__(A.dtype, A.shape)
+        self.A = A
+        self.__adj = None
+        self.args = (A,)
+
+    def _matmat(self, X):
+        return self.A.dot(X)
+
+    def _adjoint(self):
+        if self.__adj is None:
+            self.__adj = _AdjointMatrixOperator(self)
+        return self.__adj
+
+
+class _AdjointMatrixOperator(MatrixLinearOperator):
+    def __init__(self, adjoint):
+        self.A = adjoint.A.T.conj()
+        self.__adjoint = adjoint
+        self.args = (adjoint,)
+        self.shape = adjoint.shape[1], adjoint.shape[0]
+
+    @property
+    def dtype(self):
+        return self.__adjoint.dtype
+
+    def _adjoint(self):
+        return self.__adjoint
+
+
+class IdentityOperator(LinearOperator):
+    def __init__(self, shape, dtype=None):
+        super(IdentityOperator, self).__init__(dtype, shape)
+
+    def _matvec(self, x):
+        return x
+
+    def _rmatvec(self, x):
+        return x
+
+    def _rmatmat(self, x):
+        return x
+
+    def _matmat(self, x):
+        return x
+
+    def _adjoint(self):
+        return self
+
+
+def aslinearoperator(A):
+    """Return `A` as a LinearOperator.
+
+    Args:
+        A (array-like):
+            The input array to be converted to a `LinearOperator` object.
+            It may be any of the following types:
+
+               * :class:`cupy.ndarray`
+               * sparse matrix (e.g. ``csr_matrix``, ``coo_matrix``, etc.)
+               * :class:`cupyx.scipy.sparse.linalg.LinearOperator`
+               * object with ``.shape`` and ``.matvec`` attributes
+
+    Returns:
+        cupyx.scipy.sparse.linalg.LinearOperator: `LinearOperator` object
+
+    .. seealso:: :func:`scipy.sparse.aslinearoperator``
+    """
+    if isinstance(A, LinearOperator):
+        return A
+
+    elif isinstance(A, cupy.ndarray):
+        if A.ndim > 2:
+            raise ValueError('array must have ndim <= 2')
+        A = cupy.atleast_2d(A)
+        return MatrixLinearOperator(A)
+
+    elif sparse.isspmatrix(A):
+        return MatrixLinearOperator(A)
+
+    else:
+        if hasattr(A, 'shape') and hasattr(A, 'matvec'):
+            rmatvec = None
+            rmatmat = None
+            dtype = None
+
+            if hasattr(A, 'rmatvec'):
+                rmatvec = A.rmatvec
+            if hasattr(A, 'rmatmat'):
+                rmatmat = A.rmatmat
+            if hasattr(A, 'dtype'):
+                dtype = A.dtype
+            return LinearOperator(A.shape, A.matvec, rmatvec=rmatvec,
+                                  rmatmat=rmatmat, dtype=dtype)
+
+        else:
+            raise TypeError('type not understood')
diff --git a/cupyx/scipy/sparse/linalg/_iterative.py b/cupyx/scipy/sparse/linalg/_iterative.py
new file mode 100644
index 0000000..4296906
--- /dev/null
+++ b/cupyx/scipy/sparse/linalg/_iterative.py
@@ -0,0 +1,408 @@
+import numpy
+
+import cupy
+from cupy import cublas
+from cupyx import cusparse
+from cupy._core import _dtype
+from cupy.cuda import device
+from cupy_backends.cuda.libs import cusparse as _cusparse
+from cupy_backends.cuda.libs import cublas as _cublas
+from cupyx.scipy.sparse import _csr
+from cupyx.scipy.sparse.linalg import _interface
+
+
+def cg(A, b, x0=None, tol=1e-5, maxiter=None, M=None, callback=None,
+       atol=None):
+    """Uses Conjugate Gradient iteration to solve ``Ax = b``.
+
+    Args:
+        A (ndarray, spmatrix or LinearOperator): The real or complex matrix of
+            the linear system with shape ``(n, n)``. ``A`` must be a hermitian,
+            positive definitive matrix with type of :class:`cupy.ndarray`,
+            :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        b (cupy.ndarray): Right hand side of the linear system with shape
+            ``(n,)`` or ``(n, 1)``.
+        x0 (cupy.ndarray): Starting guess for the solution.
+        tol (float): Tolerance for convergence.
+        maxiter (int): Maximum number of iterations.
+        M (ndarray, spmatrix or LinearOperator): Preconditioner for ``A``.
+            The preconditioner should approximate the inverse of ``A``.
+            ``M`` must be :class:`cupy.ndarray`,
+            :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        callback (function): User-specified function to call after each
+            iteration. It is called as ``callback(xk)``, where ``xk`` is the
+            current solution vector.
+        atol (float): Tolerance for convergence.
+
+    Returns:
+        tuple:
+            It returns ``x`` (cupy.ndarray) and ``info`` (int) where ``x`` is
+            the converged solution and ``info`` provides convergence
+            information.
+
+    .. seealso:: :func:`scipy.sparse.linalg.cg`
+    """
+    A, M, x, b = _make_system(A, M, x0, b)
+    matvec = A.matvec
+    psolve = M.matvec
+
+    n = A.shape[0]
+    if maxiter is None:
+        maxiter = n * 10
+    if n == 0:
+        return cupy.empty_like(b), 0
+    b_norm = cupy.linalg.norm(b)
+    if b_norm == 0:
+        return b, 0
+    if atol is None:
+        atol = tol * float(b_norm)
+    else:
+        atol = max(float(atol), tol * float(b_norm))
+
+    r = b - matvec(x)
+    iters = 0
+    rho = 0
+    while iters < maxiter:
+        z = psolve(r)
+        rho1 = rho
+        rho = cublas.dotc(r, z)
+        if iters == 0:
+            p = z
+        else:
+            beta = rho / rho1
+            p = z + beta * p
+        q = matvec(p)
+        alpha = rho / cublas.dotc(p, q)
+        x = x + alpha * p
+        r = r - alpha * q
+        iters += 1
+        if callback is not None:
+            callback(x)
+        resid = cublas.nrm2(r)
+        if resid <= atol:
+            break
+
+    info = 0
+    if iters == maxiter and not (resid <= atol):
+        info = iters
+
+    return x, info
+
+
+def gmres(A, b, x0=None, tol=1e-5, restart=None, maxiter=None, M=None,
+          callback=None, atol=None, callback_type=None):
+    """Uses Generalized Minimal RESidual iteration to solve ``Ax = b``.
+
+    Args:
+        A (ndarray, spmatrix or LinearOperator): The real or complex
+            matrix of the linear system with shape ``(n, n)``. ``A`` must be
+            :class:`cupy.ndarray`, :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        b (cupy.ndarray): Right hand side of the linear system with shape
+            ``(n,)`` or ``(n, 1)``.
+        x0 (cupy.ndarray): Starting guess for the solution.
+        tol (float): Tolerance for convergence.
+        restart (int): Number of iterations between restarts. Larger values
+            increase iteration cost, but may be necessary for convergence.
+        maxiter (int): Maximum number of iterations.
+        M (ndarray, spmatrix or LinearOperator): Preconditioner for ``A``.
+            The preconditioner should approximate the inverse of ``A``.
+            ``M`` must be :class:`cupy.ndarray`,
+            :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        callback (function): User-specified function to call on every restart.
+            It is called as ``callback(arg)``, where ``arg`` is selected by
+            ``callback_type``.
+        callback_type (str): 'x' or 'pr_norm'. If 'x', the current solution
+            vector is used as an argument of callback function. if 'pr_norm',
+            relative (preconditioned) residual norm is used as an arugment.
+        atol (float): Tolerance for convergence.
+
+    Returns:
+        tuple:
+            It returns ``x`` (cupy.ndarray) and ``info`` (int) where ``x`` is
+            the converged solution and ``info`` provides convergence
+            information.
+
+    Reference:
+        M. Wang, H. Klie, M. Parashar and H. Sudan, "Solving Sparse Linear
+        Systems on NVIDIA Tesla GPUs", ICCS 2009 (2009).
+
+    .. seealso:: :func:`scipy.sparse.linalg.gmres`
+    """
+    A, M, x, b = _make_system(A, M, x0, b)
+    matvec = A.matvec
+    psolve = M.matvec
+
+    n = A.shape[0]
+    if n == 0:
+        return cupy.empty_like(b), 0
+    b_norm = cupy.linalg.norm(b)
+    if b_norm == 0:
+        return b, 0
+    if atol is None:
+        atol = tol * float(b_norm)
+    else:
+        atol = max(float(atol), tol * float(b_norm))
+    if maxiter is None:
+        maxiter = n * 10
+    if restart is None:
+        restart = 20
+    restart = min(restart, n)
+    if callback_type is None:
+        callback_type = 'pr_norm'
+    if callback_type not in ('x', 'pr_norm'):
+        raise ValueError('Unknown callback_type: {}'.format(callback_type))
+    if callback is None:
+        callback_type = None
+
+    V = cupy.empty((n, restart), dtype=A.dtype, order='F')
+    H = cupy.zeros((restart+1, restart), dtype=A.dtype, order='F')
+    e = numpy.zeros((restart+1,), dtype=A.dtype)
+
+    compute_hu = _make_compute_hu(V)
+
+    iters = 0
+    while True:
+        mx = psolve(x)
+        r = b - matvec(mx)
+        r_norm = cublas.nrm2(r)
+        if callback_type == 'x':
+            callback(mx)
+        elif callback_type == 'pr_norm' and iters > 0:
+            callback(r_norm / b_norm)
+        if r_norm <= atol or iters >= maxiter:
+            break
+        v = r / r_norm
+        V[:, 0] = v
+        e[0] = r_norm
+
+        # Arnoldi iteration
+        for j in range(restart):
+            z = psolve(v)
+            u = matvec(z)
+            H[:j+1, j], u = compute_hu(u, j)
+            cublas.nrm2(u, out=H[j+1, j])
+            if j+1 < restart:
+                v = u / H[j+1, j]
+                V[:, j+1] = v
+
+        # Note: The least-square solution to equation Hy = e is computed on CPU
+        # because it is faster if tha matrix size is small.
+        ret = numpy.linalg.lstsq(cupy.asnumpy(H), e)
+        y = cupy.array(ret[0])
+        x += V @ y
+        iters += restart
+
+    info = 0
+    if iters == maxiter and not (r_norm <= atol):
+        info = iters
+    return mx, info
+
+
+def cgs(A, b, x0=None, tol=1e-5, maxiter=None, M=None, callback=None,
+        atol=None):
+    """Use Conjugate Gradient Squared iteration to solve ``Ax = b``.
+
+    Args:
+        A (ndarray, spmatrix or LinearOperator): The real or complex matrix of
+            the linear system with shape ``(n, n)``.
+        b (cupy.ndarray): Right hand side of the linear system with shape
+            ``(n,)`` or ``(n, 1)``.
+        x0 (cupy.ndarray): Starting guess for the solution.
+        tol (float): Tolerance for convergence.
+        maxiter (int): Maximum number of iterations.
+        M (ndarray, spmatrix or LinearOperator): Preconditioner for ``A``.
+            The preconditioner should approximate the inverse of ``A``.
+            ``M`` must be :class:`cupy.ndarray`,
+            :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        callback (function): User-specified function to call after each
+            iteration. It is called as ``callback(xk)``, where ``xk`` is the
+            current solution vector.
+        atol (float): Tolerance for convergence.
+
+    Returns:
+        tuple:
+            It returns ``x`` (cupy.ndarray) and ``info`` (int) where ``x`` is
+            the converged solution and ``info`` provides convergence
+            information.
+
+    .. seealso:: :func:`scipy.sparse.linalg.cgs`
+    """
+    A, M, x, b = _make_system(A, M, x0, b)
+
+    matvec = A.matvec
+    psolve = M.matvec
+
+    n = A.shape[0]
+    if n == 0:
+        return cupy.empty_like(b), 0
+    b_norm = cupy.linalg.norm(b)
+    if b_norm == 0:
+        return b, 0
+    if atol is None:
+        atol = tol * float(b_norm)
+    else:
+        atol = max(float(atol), tol * float(b_norm))
+    if maxiter is None:
+        maxiter = n * 5
+
+    r0 = b - matvec(x)
+
+    rho = cupy.dot(r0, r0)
+
+    # initialise vectors
+    r = r0.copy()
+    u = r0
+    p = r0.copy()
+
+    iters = 0
+    while True:
+        y = psolve(p)
+        v = matvec(y)
+        sigma = cupy.dot(r0, v)
+        alpha = rho / sigma
+        q = u - alpha * v
+
+        z = psolve(u + q)
+        x += alpha * z
+        Az = matvec(z)
+        r -= alpha * Az
+
+        # Update residual norm and check convergence
+        r_norm = cupy.linalg.norm(r)
+
+        iters += 1
+        if callback is not None:
+            callback(x)
+
+        if r_norm <= atol or iters >= maxiter:
+            break
+
+        rho_new = cupy.dot(r0, r)
+        beta = rho_new / rho
+        rho = rho_new
+        u = r + beta * q
+        p *= beta
+        p += q
+        p *= beta
+        p += u
+
+    info = 0
+    if iters == maxiter and not (r_norm < atol):
+        info = iters
+
+    return x, info
+
+
+def _make_system(A, M, x0, b):
+    """Make a linear system Ax = b
+
+    Args:
+        A (cupy.ndarray or cupyx.scipy.sparse.spmatrix or
+            cupyx.scipy.sparse.LinearOperator): sparse or dense matrix.
+        M (cupy.ndarray or cupyx.scipy.sparse.spmatrix or
+            cupyx.scipy.sparse.LinearOperator): preconditioner.
+        x0 (cupy.ndarray): initial guess to iterative method.
+        b (cupy.ndarray): right hand side.
+
+    Returns:
+        tuple:
+            It returns (A, M, x, b).
+            A (LinaerOperator): matrix of linear system
+            M (LinearOperator): preconditioner
+            x (cupy.ndarray): initial guess
+            b (cupy.ndarray): right hand side.
+    """
+    fast_matvec = _make_fast_matvec(A)
+    A = _interface.aslinearoperator(A)
+    if fast_matvec is not None:
+        A = _interface.LinearOperator(A.shape, matvec=fast_matvec,
+                                      rmatvec=A.rmatvec, dtype=A.dtype)
+    if A.shape[0] != A.shape[1]:
+        raise ValueError('expected square matrix (shape: {})'.format(A.shape))
+    if A.dtype.char not in 'fdFD':
+        raise TypeError('unsupprted dtype (actual: {})'.format(A.dtype))
+    n = A.shape[0]
+    if not (b.shape == (n,) or b.shape == (n, 1)):
+        raise ValueError('b has incompatible dimensions')
+    b = b.astype(A.dtype).ravel()
+    if x0 is None:
+        x = cupy.zeros((n,), dtype=A.dtype)
+    else:
+        if not (x0.shape == (n,) or x0.shape == (n, 1)):
+            raise ValueError('x0 has incompatible dimensions')
+        x = x0.astype(A.dtype).ravel()
+    if M is None:
+        M = _interface.IdentityOperator(shape=A.shape, dtype=A.dtype)
+    else:
+        fast_matvec = _make_fast_matvec(M)
+        M = _interface.aslinearoperator(M)
+        if fast_matvec is not None:
+            M = _interface.LinearOperator(M.shape, matvec=fast_matvec,
+                                          rmatvec=M.rmatvec, dtype=M.dtype)
+        if A.shape != M.shape:
+            raise ValueError('matrix and preconditioner have different shapes')
+    return A, M, x, b
+
+
+def _make_fast_matvec(A):
+    matvec = None
+    if _csr.isspmatrix_csr(A) and cusparse.check_availability('spmv'):
+        handle = device.get_cusparse_handle()
+        op_a = _cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+        alpha = numpy.array(1.0, A.dtype)
+        beta = numpy.array(0.0, A.dtype)
+        cuda_dtype = _dtype.to_cuda_dtype(A.dtype)
+        alg = _cusparse.CUSPARSE_MV_ALG_DEFAULT
+        x = cupy.empty((A.shape[0],), dtype=A.dtype)
+        y = cupy.empty((A.shape[0],), dtype=A.dtype)
+        desc_A = cusparse.SpMatDescriptor.create(A)
+        desc_x = cusparse.DnVecDescriptor.create(x)
+        desc_y = cusparse.DnVecDescriptor.create(y)
+        buff_size = _cusparse.spMV_bufferSize(
+            handle, op_a, alpha.ctypes.data, desc_A.desc, desc_x.desc,
+            beta.ctypes.data, desc_y.desc, cuda_dtype, alg)
+        buff = cupy.empty(buff_size, cupy.int8)
+        del x, desc_x, y, desc_y
+
+        def matvec(x):
+            y = cupy.empty_like(x)
+            desc_x = cusparse.DnVecDescriptor.create(x)
+            desc_y = cusparse.DnVecDescriptor.create(y)
+            _cusparse.spMV(
+                handle, op_a, alpha.ctypes.data, desc_A.desc, desc_x.desc,
+                beta.ctypes.data, desc_y.desc, cuda_dtype, alg, buff.data.ptr)
+            return y
+
+    return matvec
+
+
+def _make_compute_hu(V):
+    handle = device.get_cublas_handle()
+    if V.dtype.char == 'f':
+        gemv = _cublas.sgemv
+    elif V.dtype.char == 'd':
+        gemv = _cublas.dgemv
+    elif V.dtype.char == 'F':
+        gemv = _cublas.cgemv
+    elif V.dtype.char == 'D':
+        gemv = _cublas.zgemv
+    n = V.shape[0]
+    one = numpy.array(1.0, V.dtype)
+    zero = numpy.array(0.0, V.dtype)
+    mone = numpy.array(-1.0, V.dtype)
+
+    def compute_hu(u, j):
+        # h = V[:, :j+1].conj().T @ u
+        # u -= V[:, :j+1] @ h
+        h = cupy.empty((j+1,), dtype=V.dtype)
+        gemv(handle, _cublas.CUBLAS_OP_C, n, j+1, one.ctypes.data, V.data.ptr,
+             n, u.data.ptr, 1, zero.ctypes.data, h.data.ptr, 1)
+        gemv(handle, _cublas.CUBLAS_OP_N, n, j+1, mone.ctypes.data, V.data.ptr,
+             n, h.data.ptr, 1, one.ctypes.data, u.data.ptr, 1)
+        return h, u
+    return compute_hu
diff --git a/cupyx/scipy/sparse/linalg/_lobpcg.py b/cupyx/scipy/sparse/linalg/_lobpcg.py
new file mode 100644
index 0000000..0795c2a
--- /dev/null
+++ b/cupyx/scipy/sparse/linalg/_lobpcg.py
@@ -0,0 +1,674 @@
+import warnings
+
+import numpy
+import cupy
+import cupy.linalg as linalg
+# waiting implementation of the following modules in PR #4172
+# from cupyx.scipy.linalg import (cho_factor, cho_solve)
+from cupyx.scipy.sparse import linalg as splinalg
+
+
+def _cholesky(B):
+    """
+    Wrapper around `cupy.linalg.cholesky` that raises LinAlgError if there are
+    NaNs in the output
+    """
+    R = cupy.linalg.cholesky(B)
+    if cupy.any(cupy.isnan(R)):
+        raise numpy.linalg.LinAlgError
+    return R
+
+
+# TODO: This helper function can be replaced after cupy.block is supported
+def _bmat(list_obj):
+    """
+    Helper function to create a block matrix in cupy from a list
+    of smaller 2D dense arrays
+    """
+    n_rows = len(list_obj)
+    n_cols = len(list_obj[0])
+    final_shape = [0, 0]
+    # calculating expected size of output
+    for i in range(n_rows):
+        final_shape[0] += list_obj[i][0].shape[0]
+    for j in range(n_cols):
+        final_shape[1] += list_obj[0][j].shape[1]
+    # obtaining result's datatype
+    dtype = cupy.result_type(*[arr.dtype for
+                               list_iter in list_obj for arr in list_iter])
+    # checking order
+    F_order = all(arr.flags['F_CONTIGUOUS'] for list_iter
+                  in list_obj for arr in list_iter)
+    C_order = all(arr.flags['C_CONTIGUOUS'] for list_iter
+                  in list_obj for arr in list_iter)
+    order = 'F' if F_order and not C_order else 'C'
+    result = cupy.empty(tuple(final_shape), dtype=dtype, order=order)
+
+    start_idx_row = 0
+    start_idx_col = 0
+    end_idx_row = 0
+    end_idx_col = 0
+    for i in range(n_rows):
+        end_idx_row = start_idx_row + list_obj[i][0].shape[0]
+        start_idx_col = 0
+        for j in range(n_cols):
+            end_idx_col = start_idx_col + list_obj[i][j].shape[1]
+            result[start_idx_row:end_idx_row,
+                   start_idx_col: end_idx_col] = list_obj[i][j]
+            start_idx_col = end_idx_col
+        start_idx_row = end_idx_row
+    return result
+
+
+def _report_nonhermitian(M, name):
+    """
+    Report if `M` is not a hermitian matrix given its type.
+    """
+
+    md = M - M.T.conj()
+
+    nmd = linalg.norm(md, 1)
+    tol = 10 * cupy.finfo(M.dtype).eps
+    tol *= max(1, float(linalg.norm(M, 1)))
+    if nmd > tol:
+        warnings.warn(
+            f'Matrix {name} of the type {M.dtype} is not Hermitian: '
+            f'condition: {nmd} < {tol} fails.',
+            UserWarning, stacklevel=4)
+
+
+def _as2d(ar):
+    """
+    If the input array is 2D return it, if it is 1D, append a dimension,
+    making it a column vector.
+    """
+    if ar.ndim == 2:
+        return ar
+    else:  # Assume 1!
+        aux = cupy.array(ar, copy=False)
+        aux.shape = (ar.shape[0], 1)
+        return aux
+
+
+def _makeOperator(operatorInput, expectedShape):
+    """Takes a dense numpy array or a sparse matrix or
+    a function and makes an operator performing matrix * blockvector
+    products.
+    """
+    if operatorInput is None:
+        return None
+    else:
+        operator = splinalg.aslinearoperator(operatorInput)
+
+    if operator.shape != expectedShape:
+        raise ValueError('operator has invalid shape')
+
+    return operator
+
+
+def _applyConstraints(blockVectorV, YBY, blockVectorBY, blockVectorY):
+    """Changes blockVectorV in place."""
+    YBV = cupy.dot(blockVectorBY.T.conj(), blockVectorV)
+    # awaiting the implementation of cho_solve in PR #4172
+    # tmp = cho_solve(factYBY, YBV)
+    tmp = linalg.solve(YBY, YBV)
+    blockVectorV -= cupy.dot(blockVectorY, tmp)
+
+
+def _b_orthonormalize(B, blockVectorV, blockVectorBV=None, retInvR=False):
+    """B-orthonormalize the given block vector using Cholesky."""
+    normalization = blockVectorV.max(
+        axis=0) + cupy.finfo(blockVectorV.dtype).eps
+    blockVectorV = blockVectorV / normalization
+    if blockVectorBV is None:
+        if B is not None:
+            blockVectorBV = B(blockVectorV)
+        else:
+            blockVectorBV = blockVectorV
+    else:
+        blockVectorBV = blockVectorBV / normalization
+    VBV = cupy.matmul(blockVectorV.T.conj(), blockVectorBV)
+    try:
+        # VBV is a Cholesky factor
+        VBV = _cholesky(VBV)
+        VBV = linalg.inv(VBV.T)
+        blockVectorV = cupy.matmul(blockVectorV, VBV)
+        if B is not None:
+            blockVectorBV = cupy.matmul(blockVectorBV, VBV)
+        else:
+            blockVectorBV = None
+    except numpy.linalg.LinAlgError:
+        # LinAlg Error: cholesky transformation might fail in rare cases
+        # raise ValueError("cholesky has failed")
+        blockVectorV = None
+        blockVectorBV = None
+        VBV = None
+
+    if retInvR:
+        return blockVectorV, blockVectorBV, VBV, normalization
+    else:
+        return blockVectorV, blockVectorBV
+
+
+def _get_indx(_lambda, num, largest):
+    """Get `num` indices into `_lambda` depending on `largest` option."""
+    ii = cupy.argsort(_lambda)
+    if largest:
+        ii = ii[:-num - 1:-1]
+    else:
+        ii = ii[:num]
+    return ii
+
+
+# TODO: This helper function can be replaced after cupy.eigh
+#       supports generalized eigen value problems.
+def _eigh(A, B=None):
+    """
+    Helper function for converting a generalized eigenvalue problem
+    A(X) = lambda(B(X)) to standard eigen value problem using cholesky
+    transformation
+    """
+    if B is None:  # use cupy's eigh in standard case
+        vals, vecs = linalg.eigh(A)
+        return vals, vecs
+    R = _cholesky(B)
+    RTi = linalg.inv(R)
+    Ri = linalg.inv(R.T)
+    F = cupy.matmul(RTi, cupy.matmul(A, Ri))
+    vals, vecs = linalg.eigh(F)
+    eigVec = cupy.matmul(Ri, vecs)
+    return vals, eigVec
+
+
+def lobpcg(A, X,
+           B=None, M=None, Y=None,
+           tol=None, maxiter=None,
+           largest=True, verbosityLevel=0,
+           retLambdaHistory=False, retResidualNormsHistory=False):
+    """Locally Optimal Block Preconditioned Conjugate Gradient Method (LOBPCG)
+
+    LOBPCG is a preconditioned eigensolver for large symmetric positive
+    definite (SPD) generalized eigenproblems.
+
+    Args:
+        A (array-like): The symmetric linear operator of the problem,
+            usually a sparse matrix. Can be of the following types
+            - cupy.ndarray
+            - cupyx.scipy.sparse.csr_matrix
+            - cupy.scipy.sparse.linalg.LinearOperator
+        X (cupy.ndarray): Initial approximation to the ``k``
+            eigenvectors (non-sparse). If `A` has ``shape=(n,n)``
+            then `X` should have shape ``shape=(n,k)``.
+        B (array-like): The right hand side operator in a generalized
+            eigenproblem. By default, ``B = Identity``.
+            Can be of following types:
+            - cupy.ndarray
+            - cupyx.scipy.sparse.csr_matrix
+            - cupy.scipy.sparse.linalg.LinearOperator
+        M (array-like): Preconditioner to `A`; by default ``M = Identity``.
+            `M` should approximate the inverse of `A`.
+            Can be of the following types:
+            - cupy.ndarray
+            - cupyx.scipy.sparse.csr_matrix
+            - cupy.scipy.sparse.linalg.LinearOperator
+        Y (cupy.ndarray):
+            `n-by-sizeY` matrix of constraints (non-sparse), `sizeY < n`
+            The iterations will be performed in the B-orthogonal complement
+            of the column-space of Y. Y must be full rank.
+        tol (float):
+            Solver tolerance (stopping criterion).
+            The default is ``tol=n*sqrt(eps)``.
+        maxiter (int):
+            Maximum number of iterations.  The default is ``maxiter = 20``.
+        largest (bool):
+            When True, solve for the largest eigenvalues,
+            otherwise the smallest.
+        verbosityLevel (int):
+            Controls solver output.  The default is ``verbosityLevel=0``.
+        retLambdaHistory (bool):
+            Whether to return eigenvalue history.  Default is False.
+        retResidualNormsHistory (bool):
+            Whether to return history of residual norms.  Default is False.
+
+    Returns:
+        tuple:
+            - `w` (cupy.ndarray): Array of ``k`` eigenvalues
+            - `v` (cupy.ndarray) An array of ``k`` eigenvectors.
+              `v` has the same shape as `X`.
+            - `lambdas` (list of cupy.ndarray): The eigenvalue history,
+              if `retLambdaHistory` is True.
+            - `rnorms` (list of cupy.ndarray): The history of residual norms,
+              if `retResidualNormsHistory` is True.
+
+    .. seealso:: :func:`scipy.sparse.linalg.lobpcg`
+
+    .. note::
+        If both ``retLambdaHistory`` and ``retResidualNormsHistory`` are `True`
+        the return tuple has the following format
+        ``(lambda, V, lambda history, residual norms history)``.
+    """
+    blockVectorX = X
+    blockVectorY = Y
+    residualTolerance = tol
+
+    if maxiter is None:
+        maxiter = 20
+
+    if blockVectorY is not None:
+        sizeY = blockVectorY.shape[1]
+    else:
+        sizeY = 0
+
+    if len(blockVectorX.shape) != 2:
+        raise ValueError('expected rank-2 array for argument X')
+
+    n, sizeX = blockVectorX.shape
+
+    if verbosityLevel:
+        aux = "Solving "
+        if B is None:
+            aux += "standard"
+        else:
+            aux += "generalized"
+        aux += " eigenvalue problem with"
+        if M is None:
+            aux += "out"
+        aux += " preconditioning\n\n"
+        aux += "matrix size %d\n" % n
+        aux += "block size %d\n\n" % sizeX
+        if blockVectorY is None:
+            aux += "No constraints\n\n"
+        else:
+            if sizeY > 1:
+                aux += "%d constraints\n\n" % sizeY
+            else:
+                aux += "%d constraint\n\n" % sizeY
+        print(aux)
+
+    A = _makeOperator(A, (n, n))
+    B = _makeOperator(B, (n, n))
+    M = _makeOperator(M, (n, n))
+
+    if (n - sizeY) < (5 * sizeX):
+        # The problem size is small compared to the block size.
+        # Using dense general eigensolver instead of LOBPCG.
+        sizeX = min(sizeX, n)
+
+        if blockVectorY is not None:
+            raise NotImplementedError('The dense eigensolver '
+                                      'does not support constraints.')
+
+        A_dense = A(cupy.eye(n, dtype=A.dtype))
+        B_dense = None if B is None else B(cupy.eye(n, dtype=B.dtype))
+
+        # call numerically unstable general eigen solver
+        vals, vecs = _eigh(A_dense, B_dense)
+        if largest:
+            # Reverse order to be compatible with eigs() in 'LM' mode.
+            vals = vals[::-1]
+            vecs = vecs[:, ::-1]
+
+        vals = vals[:sizeX]
+        vecs = vecs[:, :sizeX]
+
+        return vals, vecs
+
+    if (residualTolerance is None) or (residualTolerance <= 0.0):
+        residualTolerance = cupy.sqrt(1e-15) * n
+
+    # Apply constraints to X.
+    if blockVectorY is not None:
+
+        if B is not None:
+            blockVectorBY = B(blockVectorY)
+        else:
+            blockVectorBY = blockVectorY
+
+        # gramYBY is a dense array.
+        gramYBY = cupy.dot(blockVectorY.T.conj(), blockVectorBY)
+
+        # awaiting implementation of cho_factor in PR #4172
+        # try:
+        #    gramYBY is a Cholesky factor from now on...
+        #    gramYBY = cho_factor(gramYBY)
+        # except numpy.linalg.LinAlgError:
+        #    raise ValueError("cannot handle linearly dependent constraints")
+
+        _applyConstraints(blockVectorX, gramYBY, blockVectorBY, blockVectorY)
+
+    # B-orthonormalize X.
+    blockVectorX, blockVectorBX = _b_orthonormalize(B, blockVectorX)
+
+    # Compute the initial Ritz vectors: solve the eigenproblem.
+    blockVectorAX = A(blockVectorX)
+    gramXAX = cupy.dot(blockVectorX.T.conj(), blockVectorAX)
+
+    _lambda, eigBlockVector = _eigh(gramXAX)
+    ii = _get_indx(_lambda, sizeX, largest)
+    _lambda = _lambda[ii]
+
+    eigBlockVector = cupy.asarray(eigBlockVector[:, ii])
+    blockVectorX = cupy.dot(blockVectorX, eigBlockVector)
+    blockVectorAX = cupy.dot(blockVectorAX, eigBlockVector)
+    if B is not None:
+        blockVectorBX = cupy.dot(blockVectorBX, eigBlockVector)
+
+    # Active index set.
+    activeMask = cupy.ones((sizeX,), dtype=bool)
+
+    lambdaHistory = [_lambda]
+    residualNormsHistory = []
+
+    previousBlockSize = sizeX
+    ident = cupy.eye(sizeX, dtype=A.dtype)
+    ident0 = cupy.eye(sizeX, dtype=A.dtype)
+
+    ##
+    # Main iteration loop.
+
+    blockVectorP = None  # set during iteration
+    blockVectorAP = None
+    blockVectorBP = None
+
+    iterationNumber = -1
+    restart = True
+    explicitGramFlag = False
+    while iterationNumber < maxiter:
+        iterationNumber += 1
+        if verbosityLevel > 0:
+            print('-' * 50)
+            print('iteration %d' % iterationNumber)
+
+        if B is not None:
+            aux = blockVectorBX * _lambda[cupy.newaxis, :]
+        else:
+            aux = blockVectorX * _lambda[cupy.newaxis, :]
+
+        blockVectorR = blockVectorAX - aux
+
+        aux = cupy.sum(blockVectorR.conj() * blockVectorR, 0)
+        residualNorms = cupy.sqrt(aux)
+
+        residualNormsHistory.append(residualNorms)
+
+        ii = cupy.where(residualNorms > residualTolerance, True, False)
+        activeMask = activeMask & ii
+        if verbosityLevel > 2:
+            print(activeMask)
+
+        currentBlockSize = int(activeMask.sum())
+        if currentBlockSize != previousBlockSize:
+            previousBlockSize = currentBlockSize
+            ident = cupy.eye(currentBlockSize, dtype=A.dtype)
+
+        if currentBlockSize == 0:
+            break
+
+        if verbosityLevel > 0:
+            print(f'current block size: {currentBlockSize}')
+            print(f'eigenvalue(s):\n{_lambda}')
+            print(f'residual norm(s):\n{residualNorms}')
+        if verbosityLevel > 10:
+            print(eigBlockVector)
+
+        activeBlockVectorR = _as2d(blockVectorR[:, activeMask])
+
+        if iterationNumber > 0:
+            activeBlockVectorP = _as2d(blockVectorP[:, activeMask])
+            activeBlockVectorAP = _as2d(blockVectorAP[:, activeMask])
+            if B is not None:
+                activeBlockVectorBP = _as2d(blockVectorBP[:, activeMask])
+
+        if M is not None:
+            # Apply preconditioner T to the active residuals.
+            activeBlockVectorR = M(activeBlockVectorR)
+
+        # Apply constraints to the preconditioned residuals.
+        if blockVectorY is not None:
+            _applyConstraints(activeBlockVectorR,
+                              gramYBY, blockVectorBY, blockVectorY)
+
+        # B-orthogonalize the preconditioned residuals to X.
+        if B is not None:
+            activeBlockVectorR = activeBlockVectorR\
+                - cupy.matmul(blockVectorX,
+                              cupy
+                              .matmul(blockVectorBX.T.conj(),
+                                      activeBlockVectorR))
+        else:
+            activeBlockVectorR = activeBlockVectorR - \
+                cupy.matmul(blockVectorX,
+                            cupy.matmul(blockVectorX.T.conj(),
+                                        activeBlockVectorR))
+
+        ##
+        # B-orthonormalize the preconditioned residuals.
+        aux = _b_orthonormalize(B, activeBlockVectorR)
+        activeBlockVectorR, activeBlockVectorBR = aux
+
+        activeBlockVectorAR = A(activeBlockVectorR)
+
+        if iterationNumber > 0:
+            if B is not None:
+                aux = _b_orthonormalize(B, activeBlockVectorP,
+                                        activeBlockVectorBP, retInvR=True)
+                activeBlockVectorP, activeBlockVectorBP, invR, normal = aux
+            else:
+                aux = _b_orthonormalize(B, activeBlockVectorP, retInvR=True)
+                activeBlockVectorP, _, invR, normal = aux
+            # Function _b_orthonormalize returns None if Cholesky fails
+            if activeBlockVectorP is not None:
+                activeBlockVectorAP = activeBlockVectorAP / normal
+                activeBlockVectorAP = cupy.dot(activeBlockVectorAP, invR)
+                restart = False
+            else:
+                restart = True
+
+        ##
+        # Perform the Rayleigh Ritz Procedure:
+        # Compute symmetric Gram matrices:
+
+        if activeBlockVectorAR.dtype == 'float32':
+            myeps = 1
+        elif activeBlockVectorR.dtype == 'float32':
+            myeps = 1e-4
+        else:
+            myeps = 1e-8
+
+        if residualNorms.max() > myeps and not explicitGramFlag:
+            explicitGramFlag = False
+        else:
+            # Once explicitGramFlag, forever explicitGramFlag.
+            explicitGramFlag = True
+
+        # Shared memory assingments to simplify the code
+        if B is None:
+            blockVectorBX = blockVectorX
+            activeBlockVectorBR = activeBlockVectorR
+            if not restart:
+                activeBlockVectorBP = activeBlockVectorP
+
+        # Common submatrices:
+        gramXAR = cupy.dot(blockVectorX.T.conj(), activeBlockVectorAR)
+        gramRAR = cupy.dot(activeBlockVectorR.T.conj(), activeBlockVectorAR)
+
+        if explicitGramFlag:
+            gramRAR = (gramRAR + gramRAR.T.conj()) / 2
+            gramXAX = cupy.dot(blockVectorX.T.conj(), blockVectorAX)
+            gramXAX = (gramXAX + gramXAX.T.conj()) / 2
+            gramXBX = cupy.dot(blockVectorX.T.conj(), blockVectorBX)
+            gramRBR = cupy.dot(activeBlockVectorR.T.conj(),
+                               activeBlockVectorBR)
+            gramXBR = cupy.dot(blockVectorX.T.conj(), activeBlockVectorBR)
+        else:
+            gramXAX = cupy.diag(_lambda)
+            gramXBX = ident0
+            gramRBR = ident
+            gramXBR = cupy.zeros((int(sizeX), int(currentBlockSize)),
+                                 dtype=A.dtype)
+
+        def _handle_gramA_gramB_verbosity(gramA, gramB):
+            if verbosityLevel > 0:
+                _report_nonhermitian(gramA, 'gramA')
+                _report_nonhermitian(gramB, 'gramB')
+            if verbosityLevel > 10:
+                # Note: not documented, but leave it in here for now
+                numpy.savetxt('gramA.txt', cupy.asnumpy(gramA))
+                numpy.savetxt('gramB.txt', cupy.asnumpy(gramB))
+
+        if not restart:
+            gramXAP = cupy.dot(blockVectorX.T.conj(), activeBlockVectorAP)
+            gramRAP = cupy.dot(activeBlockVectorR.T.conj(),
+                               activeBlockVectorAP)
+            gramPAP = cupy.dot(activeBlockVectorP.T.conj(),
+                               activeBlockVectorAP)
+            gramXBP = cupy.dot(blockVectorX.T.conj(), activeBlockVectorBP)
+            gramRBP = cupy.dot(activeBlockVectorR.T.conj(),
+                               activeBlockVectorBP)
+            if explicitGramFlag:
+                gramPAP = (gramPAP + gramPAP.T.conj()) / 2
+                gramPBP = cupy.dot(activeBlockVectorP.T.conj(),
+                                   activeBlockVectorBP)
+            else:
+                gramPBP = ident
+
+            gramA = _bmat([[gramXAX, gramXAR, gramXAP],
+                           [gramXAR.T.conj(), gramRAR, gramRAP],
+                           [gramXAP.T.conj(), gramRAP.T.conj(), gramPAP]])
+            gramB = _bmat([[gramXBX, gramXBR, gramXBP],
+                           [gramXBR.T.conj(), gramRBR, gramRBP],
+                           [gramXBP.T.conj(), gramRBP.T.conj(), gramPBP]])
+
+            _handle_gramA_gramB_verbosity(gramA, gramB)
+
+            try:
+                _lambda, eigBlockVector = _eigh(gramA, gramB)
+            except numpy.linalg.LinAlgError:
+                # try again after dropping the direction vectors P from RR
+                restart = True
+
+        if restart:
+            gramA = _bmat([[gramXAX, gramXAR],
+                           [gramXAR.T.conj(), gramRAR]])
+            gramB = _bmat([[gramXBX, gramXBR],
+                           [gramXBR.T.conj(), gramRBR]])
+
+            _handle_gramA_gramB_verbosity(gramA, gramB)
+
+            try:
+                _lambda, eigBlockVector = _eigh(gramA, gramB)
+            except numpy.linalg.LinAlgError:
+                raise ValueError('eigh has failed in lobpcg iterations')
+
+        ii = _get_indx(_lambda, sizeX, largest)
+        if verbosityLevel > 10:
+            print(ii)
+            print(_lambda)
+
+        _lambda = _lambda[ii]
+        eigBlockVector = eigBlockVector[:, ii]
+
+        lambdaHistory.append(_lambda)
+
+        if verbosityLevel > 10:
+            print('lambda:', _lambda)
+
+        if verbosityLevel > 10:
+            print(eigBlockVector)
+
+        # Compute Ritz vectors.
+        if B is not None:
+            if not restart:
+                eigBlockVectorX = eigBlockVector[:sizeX]
+                eigBlockVectorR = eigBlockVector[sizeX:sizeX +
+                                                 currentBlockSize]
+                eigBlockVectorP = eigBlockVector[sizeX + currentBlockSize:]
+
+                pp = cupy.dot(activeBlockVectorR, eigBlockVectorR)
+                pp += cupy.dot(activeBlockVectorP, eigBlockVectorP)
+
+                app = cupy.dot(activeBlockVectorAR, eigBlockVectorR)
+                app += cupy.dot(activeBlockVectorAP, eigBlockVectorP)
+
+                bpp = cupy.dot(activeBlockVectorBR, eigBlockVectorR)
+                bpp += cupy.dot(activeBlockVectorBP, eigBlockVectorP)
+            else:
+                eigBlockVectorX = eigBlockVector[:sizeX]
+                eigBlockVectorR = eigBlockVector[sizeX:]
+
+                pp = cupy.dot(activeBlockVectorR, eigBlockVectorR)
+                app = cupy.dot(activeBlockVectorAR, eigBlockVectorR)
+                bpp = cupy.dot(activeBlockVectorBR, eigBlockVectorR)
+
+            if verbosityLevel > 10:
+                print(pp)
+                print(app)
+                print(bpp)
+
+            blockVectorX = cupy.dot(blockVectorX, eigBlockVectorX) + pp
+            blockVectorAX = cupy.dot(blockVectorAX, eigBlockVectorX) + app
+            blockVectorBX = cupy.dot(blockVectorBX, eigBlockVectorX) + bpp
+
+            blockVectorP, blockVectorAP, blockVectorBP = pp, app, bpp
+
+        else:
+            if not restart:
+                eigBlockVectorX = eigBlockVector[:sizeX]
+                eigBlockVectorR = eigBlockVector[sizeX:sizeX +
+                                                 currentBlockSize]
+                eigBlockVectorP = eigBlockVector[sizeX + currentBlockSize:]
+
+                pp = cupy.dot(activeBlockVectorR, eigBlockVectorR)
+                pp += cupy.dot(activeBlockVectorP, eigBlockVectorP)
+
+                app = cupy.dot(activeBlockVectorAR, eigBlockVectorR)
+                app += cupy.dot(activeBlockVectorAP, eigBlockVectorP)
+            else:
+                eigBlockVectorX = eigBlockVector[:sizeX]
+                eigBlockVectorR = eigBlockVector[sizeX:]
+
+                pp = cupy.dot(activeBlockVectorR, eigBlockVectorR)
+                app = cupy.dot(activeBlockVectorAR, eigBlockVectorR)
+
+            if verbosityLevel > 10:
+                print(pp)
+                print(app)
+
+            blockVectorX = cupy.dot(blockVectorX, eigBlockVectorX) + pp
+            blockVectorAX = cupy.dot(blockVectorAX, eigBlockVectorX) + app
+
+            blockVectorP, blockVectorAP = pp, app
+
+    if B is not None:
+        aux = blockVectorBX * _lambda[cupy.newaxis, :]
+
+    else:
+        aux = blockVectorX * _lambda[cupy.newaxis, :]
+
+    blockVectorR = blockVectorAX - aux
+
+    aux = cupy.sum(blockVectorR.conj() * blockVectorR, 0)
+    residualNorms = cupy.sqrt(aux)
+
+    # Future work:
+    # Generalized eigen value solver like `scipy.linalg.eigh`
+    # that takes in `B` matrix as input
+    # `cupy.linalg.cholesky` is more unstable than `scipy.linalg.cholesky`
+    # Making sure eigenvectors "exactly" satisfy the blockVectorY constrains?
+    # Making sure eigenvecotrs are "exactly" othonormalized by final "exact" RR
+    # Computing the actual true residuals
+
+    if verbosityLevel > 0:
+        print(f'Final eigenvalue(s):\n{_lambda}')
+        print(f'Final residual norm(s):\n{residualNorms}')
+
+    if retLambdaHistory:
+        if retResidualNormsHistory:
+            return _lambda, blockVectorX, lambdaHistory, residualNormsHistory
+        else:
+            return _lambda, blockVectorX, lambdaHistory
+    else:
+        if retResidualNormsHistory:
+            return _lambda, blockVectorX, residualNormsHistory
+        else:
+            return _lambda, blockVectorX
diff --git a/cupyx/scipy/sparse/linalg/_norm.py b/cupyx/scipy/sparse/linalg/_norm.py
new file mode 100644
index 0000000..82e33a6
--- /dev/null
+++ b/cupyx/scipy/sparse/linalg/_norm.py
@@ -0,0 +1,111 @@
+import numpy
+
+import cupy
+import cupyx.scipy.sparse
+
+
+def _sparse_frobenius_norm(x):
+    if cupy.issubdtype(x.dtype, cupy.complexfloating):
+        sqnorm = abs(x).power(2).sum()
+    else:
+        sqnorm = x.power(2).sum()
+    return cupy.sqrt(sqnorm)
+
+
+def norm(x, ord=None, axis=None):
+    """Norm of a cupy.scipy.spmatrix
+
+    This function is able to return one of seven different sparse matrix norms,
+    depending on the value of the ``ord`` parameter.
+
+    Args:
+        x (sparse matrix) : Input sparse matrix.
+        ord (non-zero int, inf, -inf, 'fro', optional) : Order of the norm (see
+            table under ``Notes``). inf means numpy's `inf` object.
+        axis : (int, 2-tuple of ints, None, optional): If `axis` is an
+            integer, it specifies the axis of `x` along which to
+            compute the vector norms.  If `axis` is a 2-tuple, it specifies the
+            axes that hold 2-D matrices, and the matrix norms of these matrices
+            are computed.  If `axis` is None then either a vector norm
+            (when `x` is 1-D) or a matrix norm (when `x` is 2-D) is returned.
+    Returns:
+        ndarray : 0-D or 1-D array or norm(s).
+
+    .. seealso:: :func:`scipy.sparse.linalg.norm`
+    """
+
+    if not cupyx.scipy.sparse.issparse(x):
+        raise TypeError(("input is not sparse. use cupy.linalg.norm"))
+
+    # Check the default case first and handle it immediately.
+    if axis is None and ord in (None, 'fro', 'f'):
+        return _sparse_frobenius_norm(x)
+
+    # Some norms require functions that are not implemented for all types.
+    x = x.tocsr()
+
+    if axis is None:
+        axis = (0, 1)
+    elif not isinstance(axis, tuple):
+        msg = "'axis' must be None, an integer or a tuple of integers"
+        try:
+            int_axis = int(axis)
+        except TypeError:
+            raise TypeError(msg)
+        if axis != int_axis:
+            raise TypeError(msg)
+        axis = (int_axis,)
+
+    nd = 2
+    if len(axis) == 2:
+        row_axis, col_axis = axis
+        if not (-nd <= row_axis < nd and -nd <= col_axis < nd):
+            raise ValueError('Invalid axis %r for an array with shape %r' %
+                             (axis, x.shape))
+        if row_axis % nd == col_axis % nd:
+            raise ValueError('Duplicate axes given.')
+        if ord == 2:
+            raise NotImplementedError
+            # return _multi_svd_norm(x, row_axis, col_axis, amax)
+        elif ord == -2:
+            raise NotImplementedError
+            # return _multi_svd_norm(x, row_axis, col_axis, amin)
+        elif ord == 1:
+            return abs(x).sum(axis=row_axis).max()
+        elif ord == numpy.inf:
+            return abs(x).sum(axis=col_axis).max()
+        elif ord == -1:
+            return abs(x).sum(axis=row_axis).min()
+        elif ord == -numpy.inf:
+            return abs(x).sum(axis=col_axis).min()
+        elif ord in (None, 'f', 'fro'):
+            # The axis order does not matter for this norm.
+            return _sparse_frobenius_norm(x)
+        else:
+            raise ValueError("Invalid norm order for matrices.")
+    elif len(axis) == 1:
+        a, = axis
+        if not (-nd <= a < nd):
+            raise ValueError('Invalid axis %r for an array with shape %r' %
+                             (axis, x.shape))
+        if ord == numpy.inf:
+            return abs(x).max(axis=a).A.ravel()
+        elif ord == -numpy.inf:
+            return abs(x).min(axis=a).A.ravel()
+        elif ord == 0:
+            # Zero norm
+            return (x != 0).astype(numpy.float32).sum(axis=a).ravel().astype(
+                numpy.int_)
+        elif ord == 1:
+            # special case for speedup
+            return abs(x).sum(axis=a).ravel()
+        elif ord in (2, None):
+            return cupy.sqrt(abs(x).power(2).sum(axis=a)).ravel()
+        else:
+            try:
+                ord + 1
+            except TypeError:
+                raise ValueError('Invalid norm order for vectors.')
+            return cupy.power(abs(x).power(ord).sum(axis=a), 1 / ord).ravel()
+    else:
+        raise ValueError("Improper number of dimensions to norm.")
diff --git a/cupyx/scipy/sparse/linalg/_solve.py b/cupyx/scipy/sparse/linalg/_solve.py
new file mode 100644
index 0000000..cac873a
--- /dev/null
+++ b/cupyx/scipy/sparse/linalg/_solve.py
@@ -0,0 +1,1028 @@
+import numpy
+
+import cupy
+import cupyx.cusolver
+from cupy import cublas
+from cupyx import cusparse
+from cupy.cuda import cusolver
+from cupy.cuda import device
+from cupy.cuda import runtime
+from cupy.linalg import _util
+from cupy_backends.cuda.libs import cusparse as _cusparse
+from cupyx.scipy import sparse
+from cupyx.scipy.sparse.linalg import _interface
+from cupyx.scipy.sparse.linalg._iterative import _make_system
+
+import warnings
+try:
+    import scipy.sparse
+    import scipy.sparse.linalg
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+
+def lsqr(A, b):
+    """Solves linear system with QR decomposition.
+
+    Find the solution to a large, sparse, linear system of equations.
+    The function solves ``Ax = b``. Given two-dimensional matrix ``A`` is
+    decomposed into ``Q * R``.
+
+    Args:
+        A (cupy.ndarray or cupyx.scipy.sparse.csr_matrix): The input matrix
+            with dimension ``(N, N)``
+        b (cupy.ndarray): Right-hand side vector.
+
+    Returns:
+        tuple:
+            Its length must be ten. It has same type elements
+            as SciPy. Only the first element, the solution vector ``x``, is
+            available and other elements are expressed as ``None`` because
+            the implementation of cuSOLVER is different from the one of SciPy.
+            You can easily calculate the fourth element by ``norm(b - Ax)``
+            and the ninth element by ``norm(x)``.
+
+    .. seealso:: :func:`scipy.sparse.linalg.lsqr`
+    """
+    if runtime.is_hip:
+        raise RuntimeError('HIP does not support lsqr')
+    if not sparse.isspmatrix_csr(A):
+        A = sparse.csr_matrix(A)
+    # csr_matrix is 2d
+    _util._assert_stacked_square(A)
+    _util._assert_cupy_array(b)
+    m = A.shape[0]
+    if b.ndim != 1 or len(b) != m:
+        raise ValueError('b must be 1-d array whose size is same as A')
+
+    # Cast to float32 or float64
+    if A.dtype == 'f' or A.dtype == 'd':
+        dtype = A.dtype
+    else:
+        dtype = numpy.promote_types(A.dtype, 'f')
+
+    handle = device.get_cusolver_sp_handle()
+    nnz = A.nnz
+    tol = 1.0
+    reorder = 1
+    x = cupy.empty(m, dtype=dtype)
+    singularity = numpy.empty(1, numpy.int32)
+
+    if dtype == 'f':
+        csrlsvqr = cusolver.scsrlsvqr
+    else:
+        csrlsvqr = cusolver.dcsrlsvqr
+    csrlsvqr(
+        handle, m, nnz, A._descr.descriptor, A.data.data.ptr,
+        A.indptr.data.ptr, A.indices.data.ptr, b.data.ptr, tol, reorder,
+        x.data.ptr, singularity.ctypes.data)
+
+    # The return type of SciPy is always float64. Therefore, x must be casted.
+    x = x.astype(numpy.float64)
+    ret = (x, None, None, None, None, None, None, None, None, None)
+    return ret
+
+
+def lsmr(A, b, x0=None, damp=0.0, atol=1e-6, btol=1e-6, conlim=1e8,
+         maxiter=None):
+    """Iterative solver for least-squares problems.
+
+    lsmr solves the system of linear equations ``Ax = b``. If the system
+    is inconsistent, it solves the least-squares problem ``min ||b - Ax||_2``.
+    A is a rectangular matrix of dimension m-by-n, where all cases are
+    allowed: m = n, m > n, or m < n. B is a vector of length m.
+    The matrix A may be dense or sparse (usually sparse).
+
+    Args:
+        A (ndarray, spmatrix or LinearOperator): The real or complex
+            matrix of the linear system. ``A`` must be
+            :class:`cupy.ndarray`, :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        b (cupy.ndarray): Right hand side of the linear system with shape
+            ``(m,)`` or ``(m, 1)``.
+        x0 (cupy.ndarray): Starting guess for the solution. If None zeros are
+            used.
+        damp (float): Damping factor for regularized least-squares.
+            `lsmr` solves the regularized least-squares problem
+            ::
+
+                min ||(b) - (  A   )x||
+                    ||(0)   (damp*I) ||_2
+
+            where damp is a scalar. If damp is None or 0, the system
+            is solved without regularization.
+        atol, btol (float):
+            Stopping tolerances. `lsmr` continues iterations until a
+            certain backward error estimate is smaller than some quantity
+            depending on atol and btol.
+        conlim (float): `lsmr` terminates if an estimate of ``cond(A)`` i.e.
+            condition number of matrix exceeds `conlim`. If `conlim` is None,
+            the default value is 1e+8.
+        maxiter (int): Maximum number of iterations.
+
+    Returns:
+        tuple:
+            - `x` (ndarray): Least-square solution returned.
+            - `istop` (int): istop gives the reason for stopping::
+
+                    0 means x=0 is a solution.
+
+                    1 means x is an approximate solution to A*x = B,
+                    according to atol and btol.
+
+                    2 means x approximately solves the least-squares problem
+                    according to atol.
+
+                    3 means COND(A) seems to be greater than CONLIM.
+
+                    4 is the same as 1 with atol = btol = eps (machine
+                    precision)
+
+                    5 is the same as 2 with atol = eps.
+
+                    6 is the same as 3 with CONLIM = 1/eps.
+
+                    7 means ITN reached maxiter before the other stopping
+                    conditions were satisfied.
+
+            - `itn` (int): Number of iterations used.
+            - `normr` (float): ``norm(b-Ax)``
+            - `normar` (float): ``norm(A^T (b - Ax))``
+            - `norma` (float): ``norm(A)``
+            - `conda` (float): Condition number of A.
+            - `normx` (float): ``norm(x)``
+
+    .. seealso:: :func:`scipy.sparse.linalg.lsmr`
+
+    References:
+        D. C.-L. Fong and M. A. Saunders, "LSMR: An iterative algorithm for
+        sparse least-squares problems", SIAM J. Sci. Comput.,
+        vol. 33, pp. 2950-2971, 2011.
+    """
+    A = _interface.aslinearoperator(A)
+    b = b.squeeze()
+    matvec = A.matvec
+    rmatvec = A.rmatvec
+    m, n = A.shape
+    minDim = min([m, n])
+
+    if maxiter is None:
+        maxiter = minDim * 5
+
+    u = b.copy()
+    normb = cublas.nrm2(b)
+    beta = normb.copy()
+    normb = normb.get().item()
+    if x0 is None:
+        x = cupy.zeros((n,), dtype=A.dtype)
+    else:
+        if not (x0.shape == (n,) or x0.shape == (n, 1)):
+            raise ValueError('x0 has incompatible dimensions')
+        x = x0.astype(A.dtype).ravel()
+        u -= matvec(x)
+        beta = cublas.nrm2(u)
+
+    beta_cpu = beta.get().item()
+
+    v = cupy.zeros(n)
+    alpha = cupy.zeros((), dtype=beta.dtype)
+    alpha_cpu = 0
+
+    if beta_cpu > 0:
+        u /= beta
+        v = rmatvec(u)
+        alpha = cublas.nrm2(v)
+        alpha_cpu = alpha.get().item()
+
+    if alpha_cpu > 0:
+        v /= alpha
+
+    # Initialize variables for 1st iteration.
+
+    itn = 0
+    zetabar = alpha_cpu * beta_cpu
+    alphabar = alpha_cpu
+    rho = 1
+    rhobar = 1
+    cbar = 1
+    sbar = 0
+
+    h = v.copy()
+    hbar = cupy.zeros(n)
+    # x = cupy.zeros(n)
+
+    # Initialize variables for estimation of ||r||.
+
+    betadd = beta_cpu
+    betad = 0
+    rhodold = 1
+    tautildeold = 0
+    thetatilde = 0
+    zeta = 0
+    d = 0
+
+    # Initialize variables for estimation of ||A|| and cond(A)
+
+    normA2 = alpha_cpu * alpha_cpu
+    maxrbar = 0
+    minrbar = 1e+100
+    normA = alpha_cpu
+    condA = 1
+    normx = 0
+
+    # Items for use in stopping rules.
+    istop = 0
+    ctol = 0
+    if conlim > 0:
+        ctol = 1 / conlim
+    normr = beta_cpu
+
+    # Golub-Kahan process terminates when either alpha or beta is zero.
+    # Reverse the order here from the original matlab code because
+    # there was an error on return when arnorm==0
+    normar = alpha_cpu * beta_cpu
+    if normar == 0:
+        return x, istop, itn, normr, normar, normA, condA, normx
+
+    # Main iteration loop.
+    while itn < maxiter:
+        itn = itn + 1
+
+        # Perform the next step of the bidiagonalization to obtain the
+        # next  beta, u, alpha, v.  These satisfy the relations
+        #         beta*u  =  a*v   -  alpha*u,
+        #        alpha*v  =  A'*u  -  beta*v.
+
+        u *= -alpha
+        u += matvec(v)
+        beta = cublas.nrm2(u)  # norm(u)
+        beta_cpu = beta.get().item()
+
+        if beta_cpu > 0:
+            u /= beta
+            v *= -beta
+            v += rmatvec(u)
+            alpha = cublas.nrm2(v)  # norm(v)
+            alpha_cpu = alpha.get().item()
+            if alpha_cpu > 0:
+                v /= alpha
+
+        # At this point, beta = beta_{k+1}, alpha = alpha_{k+1}.
+
+        # Construct rotation Qhat_{k,2k+1}.
+
+        chat, shat, alphahat = _symOrtho(alphabar, damp)
+
+        # Use a plane rotation (Q_i) to turn B_i to R_i
+
+        rhoold = rho
+        c, s, rho = _symOrtho(alphahat, beta_cpu)
+        thetanew = s * alpha_cpu
+        alphabar = c * alpha_cpu
+
+        # Use a plane rotation (Qbar_i) to turn R_i^T to R_i^bar
+
+        rhobarold = rhobar
+        zetaold = zeta
+        thetabar = sbar * rho
+        rhotemp = cbar * rho
+        cbar, sbar, rhobar = _symOrtho(cbar * rho, thetanew)
+        zeta = cbar * zetabar
+        zetabar = - sbar * zetabar
+
+        # Update h, h_hat, x.
+
+        # hbar = h - (thetabar * rho / (rhoold * rhobarold)) * hbar
+        hbar *= -(thetabar * rho / (rhoold * rhobarold))
+        hbar += h
+        x += (zeta / (rho * rhobar)) * hbar
+        # h = v - (thetanew / rho) * h
+        h *= -(thetanew / rho)
+        h += v
+
+        # Estimate of ||r||.
+
+        # Apply rotation Qhat_{k,2k+1}.
+        betaacute = chat * betadd
+        betacheck = -shat * betadd
+
+        # Apply rotation Q_{k,k+1}.
+        betahat = c * betaacute
+        betadd = -s * betaacute
+
+        # Apply rotation Qtilde_{k-1}.
+        # betad = betad_{k-1} here.
+
+        thetatildeold = thetatilde
+        ctildeold, stildeold, rhotildeold = _symOrtho(rhodold, thetabar)
+        thetatilde = stildeold * rhobar
+        rhodold = ctildeold * rhobar
+        betad = - stildeold * betad + ctildeold * betahat
+
+        # betad   = betad_k here.
+        # rhodold = rhod_k  here.
+
+        tautildeold = (zetaold - thetatildeold * tautildeold) / rhotildeold
+        taud = (zeta - thetatilde * tautildeold) / rhodold
+        d = d + betacheck * betacheck
+        normr = numpy.sqrt(d + (betad - taud)**2 + betadd * betadd)
+
+        # Estimate ||A||.
+        normA2 = normA2 + beta_cpu * beta_cpu
+        normA = numpy.sqrt(normA2)
+        normA2 = normA2 + alpha_cpu * alpha_cpu
+
+        # Estimate cond(A).
+        maxrbar = max(maxrbar, rhobarold)
+        if itn > 1:
+            minrbar = min(minrbar, rhobarold)
+        condA = max(maxrbar, rhotemp) / min(minrbar, rhotemp)
+
+        # Test for convergence.
+
+        # Compute norms for convergence testing.
+        normar = abs(zetabar)
+        normx = cublas.nrm2(x)
+        normx = normx.get().item()
+
+        # Now use these norms to estimate certain other quantities,
+        # some of which will be small near a solution.
+
+        test1 = normr / normb
+        if (normA * normr) != 0:
+            test2 = normar / (normA * normr)
+        else:
+            test2 = numpy.inf
+        test3 = 1 / condA
+        t1 = test1 / (1 + normA*normx/normb)
+        rtol = btol + atol*normA*normx/normb
+
+        # The following tests guard against extremely small values of
+        # atol, btol or ctol.  (The user may have set any or all of
+        # the parameters atol, btol, conlim  to 0.)
+        # The effect is equivalent to the normAl tests using
+        # atol = eps,  btol = eps,  conlim = 1/eps.
+
+        if itn >= maxiter:
+            istop = 7
+        if 1 + test3 <= 1:
+            istop = 6
+        if 1 + test2 <= 1:
+            istop = 5
+        if 1 + t1 <= 1:
+            istop = 4
+
+        # Allow for tolerances set by the user.
+
+        if test3 <= ctol:
+            istop = 3
+        if test2 <= atol:
+            istop = 2
+        if test1 <= rtol:
+            istop = 1
+
+        if istop > 0:
+            break
+
+    # The return type of SciPy is always float64. Therefore, x must be casted.
+    x = x.astype(numpy.float64)
+
+    return x, istop, itn, normr, normar, normA, condA, normx
+
+
+def _should_use_spsm(b):
+    if not runtime.is_hip:
+        # Starting with CUDA 12.0, we use cusparseSpSM
+        return _cusparse.get_build_version() >= 12000
+    else:
+        # Keep using hipsparse<t>csrsm2 for ROCm before it is dropped
+        return False
+
+
+def spsolve_triangular(A, b, lower=True, overwrite_A=False, overwrite_b=False,
+                       unit_diagonal=False):
+    """Solves a sparse triangular system ``A x = b``.
+
+    Args:
+        A (cupyx.scipy.sparse.spmatrix):
+            Sparse matrix with dimension ``(M, M)``.
+        b (cupy.ndarray):
+            Dense vector or matrix with dimension ``(M)`` or ``(M, K)``.
+        lower (bool):
+            Whether ``A`` is a lower or upper trinagular matrix.
+            If True, it is lower triangular, otherwise, upper triangular.
+        overwrite_A (bool):
+            (not supported)
+        overwrite_b (bool):
+            Allows overwriting data in ``b``.
+        unit_diagonal (bool):
+            If True, diagonal elements of ``A`` are assumed to be 1 and will
+            not be referenced.
+
+    Returns:
+        cupy.ndarray:
+            Solution to the system ``A x = b``. The shape is the same as ``b``.
+    """
+    if not (cusparse.check_availability('spsm') or
+            cusparse.check_availability('csrsm2')):
+        raise NotImplementedError
+
+    if not sparse.isspmatrix(A):
+        raise TypeError('A must be cupyx.scipy.sparse.spmatrix')
+    if not isinstance(b, cupy.ndarray):
+        raise TypeError('b must be cupy.ndarray')
+    if A.shape[0] != A.shape[1]:
+        raise ValueError(f'A must be a square matrix (A.shape: {A.shape})')
+    if b.ndim not in [1, 2]:
+        raise ValueError(f'b must be 1D or 2D array (b.shape: {b.shape})')
+    if A.shape[0] != b.shape[0]:
+        raise ValueError('The size of dimensions of A must be equal to the '
+                         'size of the first dimension of b '
+                         f'(A.shape: {A.shape}, b.shape: {b.shape})')
+    if A.dtype.char not in 'fdFD':
+        raise TypeError(f'unsupported dtype (actual: {A.dtype})')
+
+    if cusparse.check_availability('spsm') and _should_use_spsm(b):
+        if not (sparse.isspmatrix_csr(A) or
+                sparse.isspmatrix_csc(A) or
+                sparse.isspmatrix_coo(A)):
+            warnings.warn('CSR, CSC or COO format is required. Converting to '
+                          'CSR format.', sparse.SparseEfficiencyWarning)
+            A = A.tocsr()
+        A.sum_duplicates()
+        x = cusparse.spsm(A, b, lower=lower, unit_diag=unit_diagonal)
+    elif cusparse.check_availability('csrsm2'):
+        if not (sparse.isspmatrix_csr(A) or sparse.isspmatrix_csc(A)):
+            warnings.warn('CSR or CSC format is required. Converting to CSR '
+                          'format.', sparse.SparseEfficiencyWarning)
+            A = A.tocsr()
+        A.sum_duplicates()
+
+        if (overwrite_b and A.dtype == b.dtype and
+                (b._c_contiguous or b._f_contiguous)):
+            x = b
+        else:
+            x = b.astype(A.dtype, copy=True)
+
+        cusparse.csrsm2(A, x, lower=lower, unit_diag=unit_diagonal)
+    else:
+        assert False
+
+    if x.dtype.char in 'fF':
+        # Note: This is for compatibility with SciPy.
+        dtype = numpy.promote_types(x.dtype, 'float64')
+        x = x.astype(dtype)
+    return x
+
+
+def spsolve(A, b):
+    """Solves a sparse linear system ``A x = b``
+
+    Args:
+        A (cupyx.scipy.sparse.spmatrix):
+            Sparse matrix with dimension ``(M, M)``.
+        b (cupy.ndarray):
+            Dense vector or matrix with dimension ``(M)`` or ``(M, N)``.
+
+    Returns:
+        cupy.ndarray:
+            Solution to the system ``A x = b``.
+    """
+    if not cupyx.cusolver.check_availability('csrlsvqr'):
+        raise NotImplementedError
+    if not sparse.isspmatrix(A):
+        raise TypeError('A must be cupyx.scipy.sparse.spmatrix')
+    if not isinstance(b, cupy.ndarray):
+        raise TypeError('b must be cupy.ndarray')
+    if A.shape[0] != A.shape[1]:
+        raise ValueError('A must be a square matrix (A.shape: {})'.
+                         format(A.shape))
+    if not (b.ndim == 1 or b.ndim == 2):
+        raise ValueError('Invalid b.shape (b.shape: {})'.format(b.shape))
+    if A.shape[0] != b.shape[0]:
+        raise ValueError('matrix dimension mismatch (A.shape: {}, b.shape: {})'
+                         .format(A.shape, b.shape))
+
+    if not sparse.isspmatrix_csr(A):
+        warnings.warn('CSR format is required. Converting to CSR format.',
+                      sparse.SparseEfficiencyWarning)
+        A = A.tocsr()
+    A.sum_duplicates()
+    b = b.astype(A.dtype, copy=False)
+
+    if b.ndim > 1:
+        res = cupy.empty_like(b)
+        for j in range(res.shape[1]):
+            res[:, j] = cupyx.cusolver.csrlsvqr(A, b[:, j])
+        res = cupy.asarray(res, order='F')
+        return res
+    else:
+        return cupyx.cusolver.csrlsvqr(A, b)
+
+
+class SuperLU():
+
+    def __init__(self, obj):
+        """LU factorization of a sparse matrix.
+
+        Args:
+            obj (scipy.sparse.linalg.SuperLU): LU factorization of a sparse
+                matrix, computed by `scipy.sparse.linalg.splu`, etc.
+        """
+        if not scipy_available:
+            raise RuntimeError('scipy is not available')
+        if not isinstance(obj, scipy.sparse.linalg.SuperLU):
+            raise TypeError('obj must be scipy.sparse.linalg.SuperLU')
+
+        self.shape = obj.shape
+        self.nnz = obj.nnz
+        self.perm_r = cupy.array(obj.perm_r)
+        self.perm_c = cupy.array(obj.perm_c)
+        self.L = sparse.csr_matrix(obj.L.tocsr())
+        self.U = sparse.csr_matrix(obj.U.tocsr())
+
+        self._perm_r_rev = cupy.argsort(self.perm_r)
+        self._perm_c_rev = cupy.argsort(self.perm_c)
+
+    def solve(self, rhs, trans='N'):
+        """Solves linear system of equations with one or several right-hand sides.
+
+        Args:
+            rhs (cupy.ndarray): Right-hand side(s) of equation with dimension
+                ``(M)`` or ``(M, K)``.
+            trans (str): 'N', 'T' or 'H'.
+                'N': Solves ``A * x = rhs``.
+                'T': Solves ``A.T * x = rhs``.
+                'H': Solves ``A.conj().T * x = rhs``.
+
+        Returns:
+            cupy.ndarray:
+                Solution vector(s)
+        """  # NOQA
+        if not isinstance(rhs, cupy.ndarray):
+            raise TypeError('ojb must be cupy.ndarray')
+        if rhs.ndim not in (1, 2):
+            raise ValueError('rhs.ndim must be 1 or 2 (actual: {})'.
+                             format(rhs.ndim))
+        if rhs.shape[0] != self.shape[0]:
+            raise ValueError('shape mismatch (self.shape: {}, rhs.shape: {})'
+                             .format(self.shape, rhs.shape))
+        if trans not in ('N', 'T', 'H'):
+            raise ValueError('trans must be \'N\', \'T\', or \'H\'')
+
+        if cusparse.check_availability('spsm') and _should_use_spsm(rhs):
+            def spsm(A, B, lower, transa):
+                return cusparse.spsm(A, B, lower=lower, transa=transa)
+            sm = spsm
+        elif cusparse.check_availability('csrsm2'):
+            def csrsm2(A, B, lower, transa):
+                cusparse.csrsm2(A, B, lower=lower, transa=transa)
+                return B
+            sm = csrsm2
+        else:
+            raise NotImplementedError
+
+        x = rhs.astype(self.L.dtype)
+        if trans == 'N':
+            if self.perm_r is not None:
+                if x.ndim == 2 and x._f_contiguous:
+                    x = x.T[:, self._perm_r_rev].T  # want to keep f-order
+                else:
+                    x = x[self._perm_r_rev]
+            x = sm(self.L, x, lower=True, transa=trans)
+            x = sm(self.U, x, lower=False, transa=trans)
+            if self.perm_c is not None:
+                x = x[self.perm_c]
+        else:
+            if self.perm_c is not None:
+                if x.ndim == 2 and x._f_contiguous:
+                    x = x.T[:, self._perm_c_rev].T  # want to keep f-order
+                else:
+                    x = x[self._perm_c_rev]
+            x = sm(self.U, x, lower=False, transa=trans)
+            x = sm(self.L, x, lower=True, transa=trans)
+            if self.perm_r is not None:
+                x = x[self.perm_r]
+
+        if not x._f_contiguous:
+            # For compatibility with SciPy
+            x = x.copy(order='F')
+        return x
+
+
+class CusparseLU(SuperLU):
+
+    def __init__(self, a):
+        """Incomplete LU factorization of a sparse matrix.
+
+        Args:
+            a (cupyx.scipy.sparse.csr_matrix): Incomplete LU factorization of a
+                sparse matrix, computed by `cusparse.csrilu02`.
+        """
+        if not scipy_available:
+            raise RuntimeError('scipy is not available')
+        if not sparse.isspmatrix_csr(a):
+            raise TypeError('a must be cupyx.scipy.sparse.csr_matrix')
+
+        self.shape = a.shape
+        self.nnz = a.nnz
+        self.perm_r = None
+        self.perm_c = None
+        # TODO(anaruse): Computes tril and triu on GPU
+        a = a.get()
+        al = scipy.sparse.tril(a)
+        al.setdiag(1.0)
+        au = scipy.sparse.triu(a)
+        self.L = sparse.csr_matrix(al.tocsr())
+        self.U = sparse.csr_matrix(au.tocsr())
+
+
+def factorized(A):
+    """Return a function for solving a sparse linear system, with A pre-factorized.
+
+    Args:
+        A (cupyx.scipy.sparse.spmatrix): Sparse matrix to factorize.
+
+    Returns:
+        callable: a function to solve the linear system of equations given in
+        ``A``.
+
+    Note:
+        This function computes LU decomposition of a sparse matrix on the CPU
+        using `scipy.sparse.linalg.splu`. Therefore, LU decomposition is not
+        accelerated on the GPU. On the other hand, the computation of solving
+        linear equations using the method returned by this function is
+        performed on the GPU.
+
+    .. seealso:: :func:`scipy.sparse.linalg.factorized`
+    """  # NOQA
+    return splu(A).solve
+
+
+def splu(A, permc_spec=None, diag_pivot_thresh=None, relax=None,
+         panel_size=None, options={}):
+    """Computes the LU decomposition of a sparse square matrix.
+
+    Args:
+        A (cupyx.scipy.sparse.spmatrix): Sparse matrix to factorize.
+        permc_spec (str): (For further augments, see
+            :func:`scipy.sparse.linalg.splu`)
+        diag_pivot_thresh (float):
+        relax (int):
+        panel_size (int):
+        options (dict):
+
+    Returns:
+        cupyx.scipy.sparse.linalg.SuperLU:
+            Object which has a ``solve`` method.
+
+    Note:
+        This function LU-decomposes a sparse matrix on the CPU using
+        `scipy.sparse.linalg.splu`. Therefore, LU decomposition is not
+        accelerated on the GPU. On the other hand, the computation of solving
+        linear equations using the ``solve`` method, which this function
+        returns, is performed on the GPU.
+
+    .. seealso:: :func:`scipy.sparse.linalg.splu`
+    """
+    if not scipy_available:
+        raise RuntimeError('scipy is not available')
+    if not sparse.isspmatrix(A):
+        raise TypeError('A must be cupyx.scipy.sparse.spmatrix')
+    if A.shape[0] != A.shape[1]:
+        raise ValueError('A must be a square matrix (A.shape: {})'
+                         .format(A.shape))
+    if A.dtype.char not in 'fdFD':
+        raise TypeError('Invalid dtype (actual: {})'.format(A.dtype))
+
+    a = A.get().tocsc()
+    a_inv = scipy.sparse.linalg.splu(
+        a, permc_spec=permc_spec, diag_pivot_thresh=diag_pivot_thresh,
+        relax=relax, panel_size=panel_size, options=options)
+    return SuperLU(a_inv)
+
+
+def spilu(A, drop_tol=None, fill_factor=None, drop_rule=None,
+          permc_spec=None, diag_pivot_thresh=None, relax=None,
+          panel_size=None, options={}):
+    """Computes the incomplete LU decomposition of a sparse square matrix.
+
+    Args:
+        A (cupyx.scipy.sparse.spmatrix): Sparse matrix to factorize.
+        drop_tol (float): (For further augments, see
+            :func:`scipy.sparse.linalg.spilu`)
+        fill_factor (float):
+        drop_rule (str):
+        permc_spec (str):
+        diag_pivot_thresh (float):
+        relax (int):
+        panel_size (int):
+        options (dict):
+
+    Returns:
+        cupyx.scipy.sparse.linalg.SuperLU:
+            Object which has a ``solve`` method.
+
+    Note:
+        This function computes incomplete LU decomposition of a sparse matrix
+        on the CPU using `scipy.sparse.linalg.spilu` (unless you set
+        ``fill_factor`` to ``1``). Therefore, incomplete LU decomposition is
+        not accelerated on the GPU. On the other hand, the computation of
+        solving linear equations using the ``solve`` method, which this
+        function returns, is performed on the GPU.
+
+        If you set ``fill_factor`` to ``1``, this function computes incomplete
+        LU decomposition on the GPU, but without fill-in or pivoting.
+
+    .. seealso:: :func:`scipy.sparse.linalg.spilu`
+    """
+    if not scipy_available:
+        raise RuntimeError('scipy is not available')
+    if not sparse.isspmatrix(A):
+        raise TypeError('A must be cupyx.scipy.sparse.spmatrix')
+    if A.shape[0] != A.shape[1]:
+        raise ValueError('A must be a square matrix (A.shape: {})'
+                         .format(A.shape))
+    if A.dtype.char not in 'fdFD':
+        raise TypeError('Invalid dtype (actual: {})'.format(A.dtype))
+
+    if fill_factor == 1:
+        # Computes ILU(0) on the GPU using cuSparse functions
+        if not sparse.isspmatrix_csr(A):
+            a = A.tocsr()
+        else:
+            a = A.copy()
+        cusparse.csrilu02(a)
+        return CusparseLU(a)
+
+    a = A.get().tocsc()
+    a_inv = scipy.sparse.linalg.spilu(
+        a, fill_factor=fill_factor, drop_tol=drop_tol, drop_rule=drop_rule,
+        permc_spec=permc_spec, diag_pivot_thresh=diag_pivot_thresh,
+        relax=relax, panel_size=panel_size, options=options)
+    return SuperLU(a_inv)
+
+
+def _symOrtho(a, b):
+    """
+    A stable implementation of Givens rotation according to
+    S.-C. Choi, "Iterative Methods for Singular Linear Equations
+      and Least-Squares Problems", Dissertation,
+      http://www.stanford.edu/group/SOL/dissertations/sou-cheng-choi-thesis.pdf
+    """
+    if b == 0:
+        return numpy.sign(a), 0, abs(a)
+    elif a == 0:
+        return 0, numpy.sign(b), abs(b)
+    elif abs(b) > abs(a):
+        tau = a / b
+        s = numpy.sign(b) / numpy.sqrt(1+tau*tau)
+        c = s * tau
+        r = b / s
+    else:
+        tau = b / a
+        c = numpy.sign(a) / numpy.sqrt(1+tau*tau)
+        s = c * tau
+        r = a / c
+    return c, s, r
+
+
+def minres(A, b, x0=None, shift=0.0, tol=1e-5, maxiter=None,
+           M=None, callback=None, check=False):
+    """Uses MINimum RESidual iteration to solve  ``Ax = b``.
+
+    Args:
+        A (ndarray, spmatrix or LinearOperator): The real or complex matrix of
+            the linear system with shape ``(n, n)``.
+        b (cupy.ndarray): Right hand side of the linear system with shape
+            ``(n,)`` or ``(n, 1)``.
+        x0 (cupy.ndarray): Starting guess for the solution.
+        shift (int or float): If shift != 0 then the method solves
+            ``(A - shift*I)x = b``
+        tol (float): Tolerance for convergence.
+        maxiter (int): Maximum number of iterations.
+        M (ndarray, spmatrix or LinearOperator): Preconditioner for ``A``.
+            The preconditioner should approximate the inverse of ``A``.
+            ``M`` must be :class:`cupy.ndarray`,
+            :class:`cupyx.scipy.sparse.spmatrix` or
+            :class:`cupyx.scipy.sparse.linalg.LinearOperator`.
+        callback (function): User-specified function to call after each
+            iteration. It is called as ``callback(xk)``, where ``xk`` is the
+            current solution vector.
+
+    Returns:
+        tuple:
+            It returns ``x`` (cupy.ndarray) and ``info`` (int) where ``x`` is
+            the converged solution and ``info`` provides convergence
+            information.
+
+    .. seealso:: :func:`scipy.sparse.linalg.minres`
+    """
+    A, M, x, b = _make_system(A, M, x0, b)
+
+    matvec = A.matvec
+    psolve = M.matvec
+
+    n = b.shape[0]
+
+    if maxiter is None:
+        maxiter = n * 5
+
+    istop = 0
+    itn = 0
+    Anorm = 0
+    Acond = 0
+    rnorm = 0
+    ynorm = 0
+
+    xtype = x.dtype
+
+    eps = cupy.finfo(xtype).eps
+
+    Ax = matvec(x)
+    r1 = b - Ax
+    y = psolve(r1)
+
+    beta1 = cupy.inner(r1, y)
+
+    if beta1 < 0:
+        raise ValueError('indefinite preconditioner')
+    elif beta1 == 0:
+        return x, 0
+
+    beta1 = cupy.sqrt(beta1)
+    beta1 = beta1.get().item()
+
+    if check:
+        # see if A is symmetric
+        if not _check_symmetric(A, Ax, x, eps):
+            raise ValueError('non-symmetric matrix')
+
+        # see if M is symmetric
+        if not _check_symmetric(M, y, r1, eps):
+            raise ValueError('non-symmetric preconditioner')
+
+    oldb = 0
+    beta = beta1
+    dbar = 0
+    epsln = 0
+    qrnorm = beta1
+    phibar = beta1
+    rhs1 = beta1
+    rhs2 = 0
+    tnorm2 = 0
+    gmax = 0
+    gmin = cupy.finfo(xtype).max
+    cs = -1
+    sn = 0
+    w = cupy.zeros(n, dtype=xtype)
+    w2 = cupy.zeros(n, dtype=xtype)
+    r2 = r1
+
+    while itn < maxiter:
+
+        itn += 1
+        s = 1.0 / beta
+        v = s * y
+
+        y = matvec(v)
+        y -= shift * v
+
+        if itn >= 2:
+            y -= (beta / oldb) * r1
+
+        alpha = cupy.inner(v, y)
+        alpha = alpha.get().item()
+        y -= (alpha / beta) * r2
+        r1 = r2
+        r2 = y
+        y = psolve(r2)
+        oldb = beta
+        beta = cupy.inner(r2, y)
+        beta = beta.get().item()
+        beta = numpy.sqrt(beta)
+        if beta < 0:
+            raise ValueError('non-symmetric matrix')
+
+        tnorm2 += alpha ** 2 + oldb ** 2 + beta ** 2
+
+        if itn == 1:
+            if beta / beta1 <= 10 * eps:
+                istop = -1
+
+        # Apply previous rotation Qk-1 to get
+        #   [deltak epslnk+1] = [cs  sn][dbark    0   ]
+        #   [gbar k dbar k+1]   [sn -cs][alfak betak+1].
+
+        oldeps = epsln
+        delta = cs * dbar + sn * alpha  # delta1 = 0         deltak
+        gbar = sn * dbar - cs * alpha  # gbar 1 = alfa1     gbar k
+        epsln = sn * beta  # epsln2 = 0         epslnk+1
+        dbar = - cs * beta  # dbar 2 = beta2     dbar k+1
+        root = numpy.linalg.norm([gbar, dbar])
+
+        # Compute the next plane rotation Qk
+
+        gamma = numpy.linalg.norm([gbar, beta])  # gammak
+        gamma = max(gamma, eps)
+        cs = gbar / gamma  # ck
+        sn = beta / gamma  # sk
+        phi = cs * phibar  # phik
+        phibar = sn * phibar  # phibark+1
+
+        # Update  x.
+
+        denom = 1.0 / gamma
+        w1 = w2
+        w2 = w
+        w = (v - oldeps * w1 - delta * w2) * denom
+        x += phi * w
+
+        # Go round again.
+
+        gmax = max(gmax, gamma)
+        gmin = min(gmin, gamma)
+        z = rhs1 / gamma
+        rhs1 = rhs2 - delta * z
+        rhs2 = - epsln * z
+
+        # Estimate various norms and test for convergence.
+
+        Anorm = numpy.sqrt(tnorm2)
+        ynorm = cupy.linalg.norm(x)
+        ynorm = ynorm.get().item()
+        epsa = Anorm * eps
+        epsx = Anorm * ynorm * eps
+        diag = gbar
+
+        if diag == 0:
+            diag = epsa
+
+        qrnorm = phibar
+        rnorm = qrnorm
+        if ynorm == 0 or Anorm == 0:
+            test1 = numpy.inf
+        else:
+            test1 = rnorm / (Anorm * ynorm)  # ||r||  / (||A|| ||x||)
+        if Anorm == 0:
+            test2 = numpy.inf
+        else:
+            test2 = root / Anorm  # ||Ar|| / (||A|| ||r||)
+
+        # Estimate  cond(A).
+        # In this version we look at the diagonals of  R  in the
+        # factorization of the lower Hessenberg matrix,  Q * H = R,
+        # where H is the tridiagonal matrix from Lanczos with one
+        # extra row, beta(k+1) e_k^T.
+
+        Acond = gmax / gmin
+
+        # See if any of the stopping criteria are satisfied.
+        # In rare cases, istop is already -1 from above (Abar = const*I).
+
+        if istop == 0:
+            t1 = 1 + test1  # These tests work if tol < eps
+            t2 = 1 + test2
+            if t2 <= 1:
+                istop = 2
+            if t1 <= 1:
+                istop = 1
+
+            if itn >= maxiter:
+                istop = 6
+            if Acond >= 0.1 / eps:
+                istop = 4
+            if epsx >= beta1:
+                istop = 3
+            # epsr = Anorm * ynorm * tol
+            # if rnorm <= epsx   : istop = 2
+            # if rnorm <= epsr   : istop = 1
+            if test2 <= tol:
+                istop = 2
+            if test1 <= tol:
+                istop = 1
+
+        if callback is not None:
+            callback(x)
+
+        if istop != 0:
+            break
+
+    if istop == 6:
+        info = maxiter
+    else:
+        info = 0
+
+    return x, info
+
+
+def _check_symmetric(op1, op2, vec, eps):
+    r2 = op1 * op2
+    s = cupy.inner(op2, op2)
+    t = cupy.inner(vec, r2)
+    z = abs(s - t)
+    epsa = (s + eps) * eps ** (1.0 / 3.0)
+    if z > epsa:
+        return False
+    return True
diff --git a/cupyx/scipy/spatial/__init__.py b/cupyx/scipy/spatial/__init__.py
new file mode 100644
index 0000000..5e65839
--- /dev/null
+++ b/cupyx/scipy/spatial/__init__.py
@@ -0,0 +1 @@
+from cupyx.scipy.spatial.distance import distance_matrix   # NOQA
diff --git a/cupyx/scipy/spatial/distance.py b/cupyx/scipy/spatial/distance.py
new file mode 100644
index 0000000..50c477c
--- /dev/null
+++ b/cupyx/scipy/spatial/distance.py
@@ -0,0 +1,579 @@
+import cupy
+try:
+    from pylibraft.distance import pairwise_distance
+    pylibraft_available = True
+except ModuleNotFoundError:
+    pylibraft_available = False
+
+
+def _convert_to_type(X, out_type):
+    return cupy.ascontiguousarray(X, dtype=out_type)
+
+
+def _validate_pdist_input(X, m, n, metric_info, **kwargs):
+    # get supported types
+    types = metric_info.types
+    # choose best type
+    typ = types[types.index(X.dtype)] if X.dtype in types else types[0]
+    # validate data
+    X = _convert_to_type(X, out_type=typ)
+
+    # validate kwargs
+    _validate_kwargs = metric_info.validator
+    if _validate_kwargs:
+        kwargs = _validate_kwargs(X, m, n, **kwargs)
+    return X, typ, kwargs
+
+
+class MetricInfo:
+
+    def __init__(self, canonical_name=None, aka=None,
+                 validator=None, types=None):
+        self.canonical_name_ = canonical_name
+        self.aka_ = aka
+        self.validator_ = validator
+        self.types_ = types
+
+
+_METRIC_INFOS = [
+    MetricInfo(
+        canonical_name="canberra",
+        aka={'canberra'}
+    ),
+    MetricInfo(
+        canonical_name="chebyshev",
+        aka={"chebychev", "chebyshev", "cheby", "cheb", "ch"}
+    ),
+    MetricInfo(
+        canonical_name="cityblock",
+        aka={"cityblock", "cblock", "cb", "c"}
+    ),
+    MetricInfo(
+        canonical_name="correlation",
+        aka={"correlation", "co"}
+    ),
+    MetricInfo(
+        canonical_name="cosine",
+        aka={"cosine", "cos"}
+    ),
+    MetricInfo(
+        canonical_name="hamming",
+        aka={"matching", "hamming", "hamm", "ha", "h"},
+        types=["double", "bool"]
+    ),
+    MetricInfo(
+        canonical_name="euclidean",
+        aka={"euclidean", "euclid", "eu", "e"},
+    ),
+    MetricInfo(
+        canonical_name="jensenshannon",
+        aka={"jensenshannon", "js"}
+    ),
+    MetricInfo(
+        canonical_name="minkowski",
+        aka={"minkowski", "mi", "m", "pnorm"}
+    ),
+    MetricInfo(
+        canonical_name="russellrao",
+        aka={"russellrao"},
+        types=["bool"]
+    ),
+    MetricInfo(
+        canonical_name="sqeuclidean",
+        aka={"sqeuclidean", "sqe", "sqeuclid"}
+    ),
+    MetricInfo(
+        canonical_name="hellinger",
+        aka={"hellinger"}
+    ),
+    MetricInfo(
+        canonical_name="kl_divergence",
+        aka={"kl_divergence", "kl_div", "kld"}
+    )
+
+
+]
+
+_METRICS = {info.canonical_name_: info for info in _METRIC_INFOS}
+_METRIC_ALIAS = dict((alias, info)
+                     for info in _METRIC_INFOS
+                     for alias in info.aka_)
+
+_METRICS_NAMES = list(_METRICS.keys())
+
+
+def minkowski(u, v, p):
+    """Compute the Minkowski distance between two 1-D arrays.
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+        p (float): The order of the norm of the difference
+            :math:`{\\|u-v\\|}_p`. Note that for :math:`0 < p < 1`,
+            the triangle inequality only holds with an additional
+            multiplicative factor, i.e. it is only a quasi-metric.
+
+    Returns:
+        minkowski (double): The Minkowski distance between vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "minkowski", p)
+
+    return output_arr[0]
+
+
+def canberra(u, v):
+    """Compute the Canberra distance between two 1-D arrays.
+
+    The Canberra distance is defined as
+
+    .. math::
+        d(u, v) = \\sum_{i} \\frac{| u_i - v_i |}{|u_i| + |v_i|}
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        canberra (double): The Canberra distance between vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "canberra")
+
+    return output_arr[0]
+
+
+def chebyshev(u, v):
+    """Compute the Chebyshev distance between two 1-D arrays.
+
+    The Chebyshev distance is defined as
+
+    .. math::
+        d(u, v) = \\max_{i} |u_i - v_i|
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        chebyshev (double): The Chebyshev distance between vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "chebyshev")
+
+    return output_arr[0]
+
+
+def cityblock(u, v):
+    """Compute the City Block (Manhattan) distance between two 1-D arrays.
+
+    The City Block distance is defined as
+
+    .. math::
+        d(u, v) = \\sum_{i} |u_i - v_i|
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        cityblock (double): The City Block distance between
+        vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "cityblock")
+
+    return output_arr[0]
+
+
+def correlation(u, v):
+    """Compute the correlation distance between two 1-D arrays.
+
+    The correlation distance is defined as
+
+    .. math::
+        d(u, v) = 1 - \\frac{(u - \\bar{u}) \\cdot (v - \\bar{v})}{
+        \\|(u - \\bar{u})\\|_2 \\|(v - \\bar{v})\\|_2}
+
+    where :math:`\\bar{u}` is the mean of the elements of :math:`u` and
+    :math:`x \\cdot y` is the dot product.
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        correlation (double): The correlation distance between
+        vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "correlation")
+
+    return output_arr[0]
+
+
+def cosine(u, v):
+    """Compute the Cosine distance between two 1-D arrays.
+
+    The Cosine distance is defined as
+
+    .. math::
+        d(u, v) = 1 - \\frac{u \\cdot v}{\\|u\\|_2 \\|v\\|_2}
+
+    where :math:`x \\cdot y` is the dot product.
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        cosine (double): The Cosine distance between vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "cosine")
+
+    return output_arr[0]
+
+
+def hamming(u, v):
+    """Compute the Hamming distance between two 1-D arrays.
+
+    The Hamming distance is defined as the proportion of elements
+    in both `u` and `v` that are not in the exact same position:
+
+    .. math::
+        d(u, v) = \\frac{1}{n} \\sum_{k=0}^n u_i \\neq v_i
+
+    where :math:`x \\neq y` is one if :math:`x` is different from :math:`y`
+    and zero otherwise.
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        hamming (double): The Hamming distance between vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "hamming")
+
+    return output_arr[0]
+
+
+def euclidean(u, v):
+    """Compute the Euclidean distance between two 1-D arrays.
+
+    The Euclidean distance is defined as
+
+    .. math::
+        d(u, v) = \\left(\\sum_{i} (u_i - v_i)^2\\right)^{\\sfrac{1}{2}}
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        euclidean (double): The Euclidean distance between vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "euclidean")
+
+    return output_arr[0]
+
+
+def jensenshannon(u, v):
+    """Compute the Jensen-Shannon distance between two 1-D arrays.
+
+    The Jensen-Shannon distance is defined as
+
+    .. math::
+        d(u, v) = \\sqrt{\\frac{KL(u \\| m) + KL(v \\| m)}{2}}
+
+    where :math:`KL` is the Kullback-Leibler divergence and :math:`m` is the
+    pointwise mean of `u` and `v`.
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        jensenshannon (double): The Jensen-Shannon distance between
+        vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "jensenshannon")
+
+    return output_arr[0]
+
+
+def russellrao(u, v):
+    """Compute the Russell-Rao distance between two 1-D arrays.
+
+    The Russell-Rao distance is defined as the proportion of elements
+    in both `u` and `v` that are in the exact same position:
+
+    .. math::
+        d(u, v) = \\frac{1}{n} \\sum_{k=0}^n u_i = v_i
+
+    where :math:`x = y` is one if :math:`x` is different from :math:`y`
+    and zero otherwise.
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        hamming (double): The Hamming distance between vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "russellrao")
+
+    return output_arr[0]
+
+
+def sqeuclidean(u, v):
+    """Compute the squared Euclidean distance between two 1-D arrays.
+
+    The squared Euclidean distance is defined as
+
+    .. math::
+        d(u, v) = \\sum_{i} (u_i - v_i)^2
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        sqeuclidean (double): The squared Euclidean distance between
+        vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed.')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "sqeuclidean")
+
+    return output_arr[0]
+
+
+def hellinger(u, v):
+    """Compute the Hellinger distance between two 1-D arrays.
+
+    The Hellinger distance is defined as
+
+    .. math::
+        d(u, v) = \\frac{1}{\\sqrt{2}} \\sqrt{
+            \\sum_{i} (\\sqrt{u_i} - \\sqrt{v_i})^2}
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        hellinger (double): The Hellinger distance between
+        vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "hellinger")
+
+    return output_arr[0]
+
+
+def kl_divergence(u, v):
+    """Compute the Kullback-Leibler divergence between two 1-D arrays.
+
+    The Kullback-Leibler divergence is defined as
+
+    .. math::
+        KL(U \\| V) = \\sum_{i} U_i \\log{\\left(\\frac{U_i}{V_i}\\right)}
+
+    Args:
+        u (array_like): Input array of size (N,)
+        v (array_like): Input array of size (N,)
+
+    Returns:
+        kl_divergence (double): The Kullback-Leibler divergence between
+        vectors `u` and `v`.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    u = cupy.asarray(u)
+    v = cupy.asarray(v)
+    output_arr = cupy.zeros((1,), dtype=u.dtype)
+    pairwise_distance(u, v, output_arr, "kl_divergence")
+
+    return output_arr[0]
+
+
+def cdist(XA, XB, metric='euclidean', out=None, **kwargs):
+    """Compute distance between each pair of the two collections of inputs.
+
+    Args:
+        XA (array_like): An :math:`m_A` by :math:`n` array of :math:`m_A`
+            original observations in an :math:`n`-dimensional space.
+            Inputs are converted to float type.
+        XB (array_like): An :math:`m_B` by :math:`n` array of :math:`m_B`
+            original observations in an :math:`n`-dimensional space.
+            Inputs are converted to float type.
+        metric (str, optional): The distance metric to use.
+            The distance function can be 'canberra', 'chebyshev',
+            'cityblock', 'correlation', 'cosine', 'euclidean', 'hamming',
+            'hellinger', 'jensenshannon', 'kl_divergence', 'matching',
+            'minkowski', 'russellrao', 'sqeuclidean'.
+        out (cupy.ndarray, optional): The output array. If not None, the
+            distance matrix Y is stored in this array.
+        **kwargs (dict, optional): Extra arguments to `metric`: refer to each
+            metric documentation for a list of all possible arguments.
+            Some possible arguments:
+            p (float): The p-norm to apply for Minkowski, weighted and
+            unweighted. Default: 2.0
+
+    Returns:
+        Y (cupy.ndarray): A :math:`m_A` by :math:`m_B` distance matrix is
+            returned. For each :math:`i` and :math:`j`, the metric
+            ``dist(u=XA[i], v=XB[j])`` is computed and stored in the
+            :math:`ij` th entry.
+    """
+    if not pylibraft_available:
+        raise RuntimeError('pylibraft is not installed')
+    XA = cupy.asarray(XA, dtype='float32')
+    XB = cupy.asarray(XB, dtype='float32')
+
+    s = XA.shape
+    sB = XB.shape
+
+    if len(s) != 2:
+        raise ValueError('XA must be a 2-dimensional array.')
+    if len(sB) != 2:
+        raise ValueError('XB must be a 2-dimensional array.')
+    if s[1] != sB[1]:
+        raise ValueError('XA and XB must have the same number of columns '
+                         '(i.e. feature dimension.)')
+
+    mA = s[0]
+    mB = sB[0]
+
+    p = kwargs["p"] if "p" in kwargs else 2.0
+
+    if out is not None:
+        if out.dtype != 'float32':
+            out = out.astype('float32', copy=False)
+        if out.shape != (mA, mB):
+            cupy.resize(out, (mA, mB))
+        out[:] = 0.0
+
+    if isinstance(metric, str):
+        mstr = metric.lower()
+        metric_info = _METRIC_ALIAS.get(mstr, None)
+        if metric_info is not None:
+            output_arr = out if out is not None else cupy.zeros((mA, mB),
+                                                                dtype=XA.dtype)
+            pairwise_distance(XA, XB, output_arr, metric, p=p)
+            return output_arr
+        else:
+            raise ValueError('Unknown Distance Metric: %s' % mstr)
+    else:
+        raise TypeError('2nd argument metric must be a string identifier')
+
+
+def pdist(X, metric='euclidean', *, out=None, **kwargs):
+    """Compute distance between observations in n-dimensional space.
+
+    Args:
+        X (array_like): An :math:`m` by :math:`n` array of :math:`m`
+            original observations in an :math:`n`-dimensional space.
+            Inputs are converted to float type.
+        metric (str, optional): The distance metric to use.
+            The distance function can be 'canberra', 'chebyshev',
+            'cityblock', 'correlation', 'cosine', 'euclidean', 'hamming',
+            'hellinger', 'jensenshannon', 'kl_divergence', 'matching',
+            'minkowski', 'russellrao', 'sqeuclidean'.
+        out (cupy.ndarray, optional): The output array. If not None, the
+            distance matrix Y is stored in this array.
+        **kwargs (dict, optional): Extra arguments to `metric`: refer to each
+            metric documentation for a list of all possible arguments.
+            Some possible arguments:
+            p (float): The p-norm to apply for Minkowski, weighted and
+            unweighted. Default: 2.0
+
+    Returns:
+        Y (cupy.ndarray):
+            A :math:`m` by :math:`m` distance matrix is
+            returned. For each :math:`i` and :math:`j`, the metric
+            ``dist(u=X[i], v=X[j])`` is computed and stored in the
+            :math:`ij` th entry.
+    """
+    all_dist = cdist(X, X, metric=metric, out=out, **kwargs)
+    up_idx = cupy.triu_indices_from(all_dist, 1)
+    return all_dist[up_idx]
+
+
+def distance_matrix(x, y, p=2.0):
+    """Compute the distance matrix.
+
+    Returns the matrix of all pair-wise distances.
+
+    Args:
+        x (array_like): Matrix of M vectors in K dimensions.
+        y (array_like): Matrix of N vectors in K dimensions.
+        p (float): Which Minkowski p-norm to use (1 <= p <= infinity).
+            Default=2.0
+    Returns:
+        result (cupy.ndarray): Matrix containing the distance from every
+            vector in `x` to every vector in `y`, (size M, N).
+    """
+    x = cupy.asarray(x)
+    m, k = x.shape
+    y = cupy.asarray(y)
+    n, kk = y.shape
+
+    if k != kk:
+        raise ValueError("x contains %d-dimensional vectors but y "
+                         "contains %d-dimensional vectors" % (k, kk))
+
+    return cdist(x, y, metric="minkowski", p=p)
diff --git a/cupyx/scipy/special/__init__.py b/cupyx/scipy/special/__init__.py
new file mode 100644
index 0000000..6ed741d
--- /dev/null
+++ b/cupyx/scipy/special/__init__.py
@@ -0,0 +1,109 @@
+# Bessel Functions
+from cupyx.scipy.special._bessel import i0  # NOQA
+from cupyx.scipy.special._bessel import i0e  # NOQA
+from cupyx.scipy.special._bessel import i1  # NOQA
+from cupyx.scipy.special._bessel import i1e  # NOQA
+from cupyx.scipy.special._bessel import j0  # NOQA
+from cupyx.scipy.special._bessel import j1  # NOQA
+from cupyx.scipy.special._bessel import k0  # NOQA
+from cupyx.scipy.special._bessel import k0e  # NOQA
+from cupyx.scipy.special._bessel import k1  # NOQA
+from cupyx.scipy.special._bessel import k1e  # NOQA
+from cupyx.scipy.special._bessel import y0  # NOQA
+from cupyx.scipy.special._bessel import y1  # NOQA
+from cupyx.scipy.special._bessel import yn  # NOQA
+
+# Raw statistical functions
+from cupyx.scipy.special._stats_distributions import bdtr  # NOQA
+from cupyx.scipy.special._stats_distributions import bdtrc  # NOQA
+from cupyx.scipy.special._stats_distributions import bdtri  # NOQA
+from cupyx.scipy.special._stats_distributions import btdtr  # NOQA
+from cupyx.scipy.special._stats_distributions import btdtri  # NOQA
+from cupyx.scipy.special._stats_distributions import fdtr  # NOQA
+from cupyx.scipy.special._stats_distributions import fdtrc  # NOQA
+from cupyx.scipy.special._stats_distributions import fdtri  # NOQA
+from cupyx.scipy.special._stats_distributions import gdtr  # NOQA
+from cupyx.scipy.special._stats_distributions import gdtrc  # NOQA
+from cupyx.scipy.special._stats_distributions import nbdtr  # NOQA
+from cupyx.scipy.special._stats_distributions import nbdtrc  # NOQA
+from cupyx.scipy.special._stats_distributions import nbdtri  # NOQA
+from cupyx.scipy.special._stats_distributions import pdtr  # NOQA
+from cupyx.scipy.special._stats_distributions import pdtrc  # NOQA
+from cupyx.scipy.special._stats_distributions import pdtri  # NOQA
+from cupyx.scipy.special._stats_distributions import chdtr  # NOQA
+from cupyx.scipy.special._stats_distributions import chdtrc  # NOQA
+from cupyx.scipy.special._stats_distributions import chdtri  # NOQA
+from cupyx.scipy.special._stats_distributions import ndtr  # NOQA
+from cupyx.scipy.special._stats_distributions import log_ndtr  # NOQA
+from cupyx.scipy.special._stats_distributions import ndtri  # NOQA
+from cupyx.scipy.special._statistics import logit  # NOQA
+from cupyx.scipy.special._statistics import expit  # NOQA
+from cupyx.scipy.special._statistics import log_expit  # NOQA
+from cupyx.scipy.special._statistics import boxcox  # NOQA
+from cupyx.scipy.special._statistics import boxcox1p  # NOQA
+from cupyx.scipy.special._statistics import inv_boxcox  # NOQA
+from cupyx.scipy.special._statistics import inv_boxcox1p  # NOQA
+
+# Information Theory functions
+from cupyx.scipy.special._convex_analysis import entr  # NOQA
+from cupyx.scipy.special._convex_analysis import huber  # NOQA
+from cupyx.scipy.special._convex_analysis import kl_div  # NOQA
+from cupyx.scipy.special._convex_analysis import pseudo_huber  # NOQA
+from cupyx.scipy.special._convex_analysis import rel_entr  # NOQA
+
+# Gamma and related functions
+from cupyx.scipy.special._gamma import gamma  # NOQA
+from cupyx.scipy.special._gammaln import gammaln  # NOQA
+from cupyx.scipy.special._loggamma import loggamma  # NOQA
+from cupyx.scipy.special._gammasgn import gammasgn  # NOQA
+from cupyx.scipy.special._gammainc import gammainc  # NOQA
+from cupyx.scipy.special._gammainc import gammaincinv  # NOQA
+from cupyx.scipy.special._gammainc import gammaincc  # NOQA
+from cupyx.scipy.special._gammainc import gammainccinv  # NOQA
+from cupyx.scipy.special._beta import beta  # NOQA
+from cupyx.scipy.special._beta import betaln  # NOQA
+from cupyx.scipy.special._beta import betainc  # NOQA
+from cupyx.scipy.special._beta import betaincinv  # NOQA
+from cupyx.scipy.special._digamma import digamma as psi  # NOQA
+from cupyx.scipy.special._gamma import rgamma  # NOQA
+from cupyx.scipy.special._polygamma import polygamma  # NOQA
+from cupyx.scipy.special._gammaln import multigammaln  # NOQA
+from cupyx.scipy.special._digamma import digamma  # NOQA
+from cupyx.scipy.special._poch import poch  # NOQA
+
+# Error function and Fresnel integrals
+from cupyx.scipy.special._erf import erf  # NOQA
+from cupyx.scipy.special._erf import erfc  # NOQA
+from cupyx.scipy.special._erf import erfcx  # NOQA
+from cupyx.scipy.special._erf import erfinv  # NOQA
+from cupyx.scipy.special._erf import erfcinv  # NOQA
+
+# Legendre functions
+from cupyx.scipy.special._lpmv import lpmv  # NOQA
+from cupyx.scipy.special._sph_harm import sph_harm  # NOQA
+
+# Other special functions
+from cupyx.scipy.special._exp1 import exp1  # NOQA
+from cupyx.scipy.special._expi import expi  # NOQA
+from cupyx.scipy.special._expn import expn  # NOQA
+from cupyx.scipy.special._softmax import softmax  # NOQA
+from cupyx.scipy.special._logsoftmax import log_softmax  # NOQA
+from cupyx.scipy.special._zeta import zeta  # NOQA
+
+# Convenience functions
+from cupyx.scipy.special._basic import cbrt  # NOQA
+from cupyx.scipy.special._basic import exp10  # NOQA
+from cupyx.scipy.special._basic import exp2  # NOQA
+from cupyx.scipy.special._basic import radian  # NOQA
+from cupyx.scipy.special._basic import cosdg  # NOQA
+from cupyx.scipy.special._basic import sindg  # NOQA
+from cupyx.scipy.special._basic import tandg  # NOQA
+from cupyx.scipy.special._basic import cotdg  # NOQA
+from cupyx.scipy.special._basic import log1p  # NOQA
+from cupyx.scipy.special._basic import expm1  # NOQA
+from cupyx.scipy.special._basic import cosm1  # NOQA
+from cupy._math.rounding import round_ as round  # NOQA
+from cupyx.scipy.special._xlogy import xlogy  # NOQA
+from cupyx.scipy.special._xlogy import xlog1py  # NOQA
+from cupyx.scipy.special._logsumexp import logsumexp  # NOQA
+from cupy._math.special import sinc  # NOQA
diff --git a/cupyx/scipy/special/_basic.py b/cupyx/scipy/special/_basic.py
new file mode 100644
index 0000000..1024f8a
--- /dev/null
+++ b/cupyx/scipy/special/_basic.py
@@ -0,0 +1,287 @@
+"""basic special functions
+
+
+cotdg and tandg implementations are adapted from the following SciPy code:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/tandg.c
+
+radian is from
+
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/sindg.c
+
+cosm1 is from
+
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/unity.c
+
+polevl is from
+
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/polevl.h
+
+
+Cephes Math Library Release 2.0:  April, 1987
+Copyright 1984, 1987 by Stephen L. Moshier
+Direct inquiries to 30 Frost Street, Cambridge, MA 02140
+"""
+
+from cupy import _core
+
+
+# Note: cast complex<single> to complex<double> or tests fail tolerance
+log1p = _core.create_ufunc(
+    'cupyx_scipy_log1p',
+    (('f->f', 'out0 = log1pf(in0)'),
+     ('F->F', 'out0 = out0_type(log1p((complex<double>)in0))'),
+     'd->d', 'D->D'),
+    'out0 = log1p(in0);',
+    doc="""Elementwise function for scipy.special.log1p
+
+    Calculates log(1 + x) for use when `x` is near zero.
+
+    Notes
+    -----
+    This implementation currently does not support complex-valued `x`.
+
+    .. seealso:: :meth:`scipy.special.log1p`
+
+    """,
+)
+
+
+cbrt = _core.create_ufunc(
+    'cupyx_scipy_special_cbrt',
+    (('f->f', 'out0 = cbrtf(in0)'), 'd->d'),
+    'out0 = cbrt(in0)',
+    doc='''Cube root.
+
+    .. seealso:: :meth:`scipy.special.cbrt`
+
+    ''')
+
+
+exp2 = _core.create_ufunc(
+    'cupyx_scipy_special_exp2',
+    (('f->f', 'out0 = exp2f(in0)'), 'd->d'),
+    'out0 = exp2(in0)',
+    doc='''Computes ``2**x``.
+
+    .. seealso:: :meth:`scipy.special.exp2`
+
+    ''')
+
+
+exp10 = _core.create_ufunc(
+    'cupyx_scipy_special_exp10',
+    (('f->f', 'out0 = exp10f(in0)'), 'd->d'),
+    'out0 = exp10(in0)',
+    doc='''Computes ``10**x``.
+
+    .. seealso:: :meth:`scipy.special.exp10`
+
+    ''')
+
+
+expm1 = _core.create_ufunc(
+    'cupyx_scipy_special_expm1',
+    (('f->f', 'out0 = expm1f(in0)'), 'd->d', 'F->F', 'D->D'),
+    'out0 = expm1(in0)',
+    doc='''Computes ``exp(x) - 1``.
+
+    .. seealso:: :meth:`scipy.special.expm1`
+
+    ''')
+
+cosm1_implementation = """
+//Define from npy_math.h
+//https://github.com/numpy/numpy/blob/main/numpy/core/include/numpy/npy_math.h
+#define NPY_PI_4      0.785398163397448309615660845819875721  /* pi/4 */
+__constant__ double coscof[] = {
+    4.7377507964246204691685E-14,
+    -1.1470284843425359765671E-11,
+    2.0876754287081521758361E-9,
+    -2.7557319214999787979814E-7,
+    2.4801587301570552304991E-5,
+    -1.3888888888888872993737E-3,
+    4.1666666666666666609054E-2,
+};
+
+__device__ static double polevl(double x, const double coef[], int N)
+{
+    double ans;
+    int i;
+    const double *p;
+
+    p = coef;
+    ans = *p++;
+    i = N;
+
+    do
+        ans = ans * x + *p++;
+    while (--i);
+
+    return (ans);
+}
+
+__device__ static double cosm1(double x)
+{
+    double xx;
+
+    if ((x < -NPY_PI_4) || (x > NPY_PI_4))
+        return (cos(x) - 1.0);
+    xx = x * x;
+    xx = -0.5 * xx + xx * xx * polevl(xx, coscof, 6);
+    return xx;
+}
+"""
+
+cosm1 = _core.create_ufunc(
+    'cupyx_scipy_special_cosm1', ('f->f', 'd->d'),
+    'out0 = cosm1(in0)',
+    preamble=cosm1_implementation,
+    doc='''Computes ``cos(x) - 1``.
+
+    .. seealso:: :meth:`scipy.special.cosm1`
+
+    ''')
+
+pi180_preamble = """
+    __constant__ double PI180 = 1.74532925199432957692E-2;  // pi/180
+"""
+
+cosdg = _core.create_ufunc(
+    'cupyx_scipy_special_cosdg',
+    (('f->f', 'out0 = cosf(PI180 * in0)'), 'd->d'),
+    'out0 = cos(PI180 * in0)',
+    preamble=pi180_preamble,
+    doc='''Cosine of x with x in degrees.
+
+    .. seealso:: :meth:`scipy.special.cosdg`
+
+    ''')
+
+
+sindg = _core.create_ufunc(
+    'cupyx_scipy_special_sindg',
+    (('f->f', 'out0 = sinf(PI180 * in0)'), 'd->d'),
+    'out0 = sin(PI180 * in0)',
+    preamble=pi180_preamble,
+    doc='''Sine of x with x in degrees.
+
+    .. seealso:: :meth:`scipy.special.sindg`
+
+    ''')
+
+
+tancot_implementation = pi180_preamble + """
+
+
+// include for CUDART_INF
+#include <cupy/math_constants.h>
+
+__constant__ double  lossth = 1.0e14;
+
+__device__ static double tancot(double, int);
+
+__device__ static double tandg(double x)
+{
+    return tancot(x, 0);
+}
+
+
+__device__ static double cotdg(double x)
+{
+    return tancot(x, 1);
+}
+
+
+__device__ static double tancot(double xx, int cotflg)
+{
+    double x;
+    int sign;
+
+    /* make argument positive but save the sign */
+    if (xx < 0) {
+        x = -xx;
+        sign = -1;
+    }
+    else {
+        x = xx;
+        sign = 1;
+    }
+
+    if (x > lossth) {
+        // sf_error("tandg", SF_ERROR_NO_RESULT, NULL);
+        return 0.0;
+    }
+
+    /* modulo 180 */
+    x = x - 180.0 * floor(x / 180.0);
+    if (cotflg) {
+        if (x <= 90.0) {
+            x = 90.0 - x;
+        } else {
+            x = x - 90.0;
+            sign *= -1;
+        }
+    } else {
+        if (x > 90.0) {
+            x = 180.0 - x;
+            sign *= -1;
+        }
+    }
+    if (x == 0.0) {
+        return 0.0;
+    }
+    else if (x == 45.0) {
+        return sign * 1.0;
+    }
+    else if (x == 90.0) {
+        // sf_error((cotflg ? "cotdg" : "tandg"), SF_ERROR_SINGULAR, NULL);
+        return CUDART_INF;
+    }
+    /* x is now transformed into [0, 90) */
+    return sign * tan(x * PI180);
+}
+"""
+
+tandg = _core.create_ufunc(
+    'cupyx_scipy_special_tandg', ('f->f', 'd->d'),
+    'out0 = tandg(in0)',
+    preamble=tancot_implementation,
+    doc='''Tangent of x with x in degrees.
+
+    .. seealso:: :meth:`scipy.special.tandg`
+
+    ''')
+
+
+cotdg = _core.create_ufunc(
+    'cupyx_scipy_special_cotdg', ('f->f', 'd->d'),
+    'out0 = cotdg(in0)',
+    preamble=tancot_implementation,
+    doc='''Cotangent of x with x in degrees.
+
+    .. seealso:: :meth:`scipy.special.cotdg`
+
+    ''')
+
+radian_implementation = """
+/* 1 arc second, in radians*/
+__constant__ double P64800 =
+    4.848136811095359935899141023579479759563533023727e-6;
+
+template <typename T>
+__device__ T radian(T d, T m, T s)
+{
+    return (((d * 60.0 + m) * 60.0 + s) * P64800);
+}
+"""
+
+radian = _core.create_ufunc(
+    'cupyx_scipy_special_radian', ('fff->f', 'ddd->d'),
+    'out0 = radian(in0, in1, in2)',
+    preamble=radian_implementation,
+    doc='''Degrees, minutes, seconds to radians:
+
+    .. seealso:: :meth:`scipy.special.radian`
+
+    ''')
diff --git a/cupyx/scipy/special/_bessel.py b/cupyx/scipy/special/_bessel.py
new file mode 100644
index 0000000..b5eb1f9
--- /dev/null
+++ b/cupyx/scipy/special/_bessel.py
@@ -0,0 +1,462 @@
+# This source code contains SciPy's code.
+# https://github.com/scipy/scipy/blob/master/scipy/special/cephes/k0.c
+# https://github.com/scipy/scipy/blob/master/scipy/special/cephes/k1.c
+#
+#
+# Cephes Math Library Release 2.8:  June, 2000
+# Copyright 1984, 1987, 1992, 2000 by Stephen L. Moshier
+
+from cupy import _core
+from cupyx.scipy.special._gamma import chbevl_implementation
+
+j0 = _core.create_ufunc(
+    'cupyx_scipy_special_j0', ('f->f', 'd->d'),
+    'out0 = j0(in0)',
+    doc='''Bessel function of the first kind of order 0.
+
+    .. seealso:: :meth:`scipy.special.j0`
+
+    ''')
+
+
+j1 = _core.create_ufunc(
+    'cupyx_scipy_special_j1', ('f->f', 'd->d'),
+    'out0 = j1(in0)',
+    doc='''Bessel function of the first kind of order 1.
+
+    .. seealso:: :meth:`scipy.special.j1`
+
+    ''')
+
+
+y0 = _core.create_ufunc(
+    'cupyx_scipy_special_y0', ('f->f', 'd->d'),
+    'out0 = y0(in0)',
+    doc='''Bessel function of the second kind of order 0.
+
+    .. seealso:: :meth:`scipy.special.y0`
+
+    ''')
+
+
+y1 = _core.create_ufunc(
+    'cupyx_scipy_special_y1', ('f->f', 'd->d'),
+    'out0 = y1(in0)',
+    doc='''Bessel function of the second kind of order 1.
+
+    .. seealso:: :meth:`scipy.special.y1`
+
+    ''')
+
+
+# Note: oddly, unlike for y0 or y1, SciPy always returns double for yn
+# dd->d because SciPy will accept 2.0
+yn = _core.create_ufunc(
+    'cupyx_scipy_special_yn', ('id->d', 'dd->d'),
+    'out0 = yn((int)in0, in1)',
+    doc='''Bessel function of the second kind of order n.
+
+    Args:
+        n (cupy.ndarray): order (integer)
+        x (cupy.ndarray): argument (float)
+
+    Returns:
+        cupy.ndarray: The result.
+
+    Notes
+    -----
+    Unlike SciPy, no warning will be raised on unsafe casting of `order` to
+    32-bit integer.
+
+    .. seealso:: :meth:`scipy.special.yn`
+    ''')
+
+
+i0 = _core.create_ufunc(
+    'cupyx_scipy_special_i0', ('f->f', 'd->d'),
+    'out0 = cyl_bessel_i0(in0)',
+    doc='''Modified Bessel function of order 0.
+
+    .. seealso:: :meth:`scipy.special.i0`
+
+    ''')
+
+
+i0e = _core.create_ufunc(
+    'cupyx_scipy_special_i0e', ('f->f', 'd->d'),
+    'out0 = exp(-abs(in0)) * cyl_bessel_i0(in0)',
+    doc='''Exponentially scaled modified Bessel function of order 0.
+
+    .. seealso:: :meth:`scipy.special.i0e`
+
+    ''')
+
+
+i1 = _core.create_ufunc(
+    'cupyx_scipy_special_i1', ('f->f', 'd->d'),
+    'out0 = cyl_bessel_i1(in0)',
+    doc='''Modified Bessel function of order 1.
+
+    .. seealso:: :meth:`scipy.special.i1`
+
+    ''')
+
+
+i1e = _core.create_ufunc(
+    'cupyx_scipy_special_i1e', ('f->f', 'd->d'),
+    'out0 = exp(-abs(in0)) * cyl_bessel_i1(in0)',
+    doc='''Exponentially scaled modified Bessel function of order 1.
+
+    .. seealso:: :meth:`scipy.special.i1e`
+
+    ''')
+
+
+k_preamble = chbevl_implementation + """
+#include <cupy/math_constants.h>
+
+/* Chebyshev coefficients for K0(x) + log(x/2) I0(x)
+ * in the interval [0,2].  The odd order coefficients are all
+ * zero; only the even order coefficients are listed.
+ *
+ * lim(x->0){ K0(x) + log(x/2) I0(x) } = -EUL.
+ */
+__device__ double k0_A[] = {
+    1.37446543561352307156E-16,
+    4.25981614279661018399E-14,
+    1.03496952576338420167E-11,
+    1.90451637722020886025E-9,
+    2.53479107902614945675E-7,
+    2.28621210311945178607E-5,
+    1.26461541144692592338E-3,
+    3.59799365153615016266E-2,
+    3.44289899924628486886E-1,
+    -5.35327393233902768720E-1
+};
+__device__ float k0_AF[] = {
+    1.37446543561352307156E-16,
+    4.25981614279661018399E-14,
+    1.03496952576338420167E-11,
+    1.90451637722020886025E-9,
+    2.53479107902614945675E-7,
+    2.28621210311945178607E-5,
+    1.26461541144692592338E-3,
+    3.59799365153615016266E-2,
+    3.44289899924628486886E-1,
+    -5.35327393233902768720E-1
+};
+
+
+/* Chebyshev coefficients for exp(x) sqrt(x) K0(x)
+ * in the inverted interval [2,infinity].
+ *
+ * lim(x->inf){ exp(x) sqrt(x) K0(x) } = sqrt(pi/2).
+ */
+__device__ double k0_B[] = {
+    5.30043377268626276149E-18,
+    -1.64758043015242134646E-17,
+    5.21039150503902756861E-17,
+    -1.67823109680541210385E-16,
+    5.51205597852431940784E-16,
+    -1.84859337734377901440E-15,
+    6.34007647740507060557E-15,
+    -2.22751332699166985548E-14,
+    8.03289077536357521100E-14,
+    -2.98009692317273043925E-13,
+    1.14034058820847496303E-12,
+    -4.51459788337394416547E-12,
+    1.85594911495471785253E-11,
+    -7.95748924447710747776E-11,
+    3.57739728140030116597E-10,
+    -1.69753450938905987466E-9,
+    8.57403401741422608519E-9,
+    -4.66048989768794782956E-8,
+    2.76681363944501510342E-7,
+    -1.83175552271911948767E-6,
+    1.39498137188764993662E-5,
+    -1.28495495816278026384E-4,
+    1.56988388573005337491E-3,
+    -3.14481013119645005427E-2,
+    2.44030308206595545468E0
+};
+__device__ float k0_BF[] = {
+    5.30043377268626276149E-18,
+    -1.64758043015242134646E-17,
+    5.21039150503902756861E-17,
+    -1.67823109680541210385E-16,
+    5.51205597852431940784E-16,
+    -1.84859337734377901440E-15,
+    6.34007647740507060557E-15,
+    -2.22751332699166985548E-14,
+    8.03289077536357521100E-14,
+    -2.98009692317273043925E-13,
+    1.14034058820847496303E-12,
+    -4.51459788337394416547E-12,
+    1.85594911495471785253E-11,
+    -7.95748924447710747776E-11,
+    3.57739728140030116597E-10,
+    -1.69753450938905987466E-9,
+    8.57403401741422608519E-9,
+    -4.66048989768794782956E-8,
+    2.76681363944501510342E-7,
+    -1.83175552271911948767E-6,
+    1.39498137188764993662E-5,
+    -1.28495495816278026384E-4,
+    1.56988388573005337491E-3,
+    -3.14481013119645005427E-2,
+    2.44030308206595545468E0
+};
+
+
+/* Chebyshev coefficients for x(K1(x) - log(x/2) I1(x))
+ * in the interval [0,2].
+ *
+ * lim(x->0){ x(K1(x) - log(x/2) I1(x)) } = 1.
+ */
+__device__ static double k1_A[] = {
+    -7.02386347938628759343E-18,
+    -2.42744985051936593393E-15,
+    -6.66690169419932900609E-13,
+    -1.41148839263352776110E-10,
+    -2.21338763073472585583E-8,
+    -2.43340614156596823496E-6,
+    -1.73028895751305206302E-4,
+    -6.97572385963986435018E-3,
+    -1.22611180822657148235E-1,
+    -3.53155960776544875667E-1,
+    1.52530022733894777053E0
+};
+__device__ static float k1_AF[] = {
+    -7.02386347938628759343E-18,
+    -2.42744985051936593393E-15,
+    -6.66690169419932900609E-13,
+    -1.41148839263352776110E-10,
+    -2.21338763073472585583E-8,
+    -2.43340614156596823496E-6,
+    -1.73028895751305206302E-4,
+    -6.97572385963986435018E-3,
+    -1.22611180822657148235E-1,
+    -3.53155960776544875667E-1,
+    1.52530022733894777053E0
+};
+
+
+/* Chebyshev coefficients for exp(x) sqrt(x) K1(x)
+ * in the interval [2,infinity].
+ *
+ * lim(x->inf){ exp(x) sqrt(x) K1(x) } = sqrt(pi/2).
+ */
+__device__ static double k1_B[] = {
+    -5.75674448366501715755E-18,
+    1.79405087314755922667E-17,
+    -5.68946255844285935196E-17,
+    1.83809354436663880070E-16,
+    -6.05704724837331885336E-16,
+    2.03870316562433424052E-15,
+    -7.01983709041831346144E-15,
+    2.47715442448130437068E-14,
+    -8.97670518232499435011E-14,
+    3.34841966607842919884E-13,
+    -1.28917396095102890680E-12,
+    5.13963967348173025100E-12,
+    -2.12996783842756842877E-11,
+    9.21831518760500529508E-11,
+    -4.19035475934189648750E-10,
+    2.01504975519703286596E-9,
+    -1.03457624656780970260E-8,
+    5.74108412545004946722E-8,
+    -3.50196060308781257119E-7,
+    2.40648494783721712015E-6,
+    -1.93619797416608296024E-5,
+    1.95215518471351631108E-4,
+    -2.85781685962277938680E-3,
+    1.03923736576817238437E-1,
+    2.72062619048444266945E0
+};
+__device__ static float k1_BF[] = {
+    -5.75674448366501715755E-18,
+    1.79405087314755922667E-17,
+    -5.68946255844285935196E-17,
+    1.83809354436663880070E-16,
+    -6.05704724837331885336E-16,
+    2.03870316562433424052E-15,
+    -7.01983709041831346144E-15,
+    2.47715442448130437068E-14,
+    -8.97670518232499435011E-14,
+    3.34841966607842919884E-13,
+    -1.28917396095102890680E-12,
+    5.13963967348173025100E-12,
+    -2.12996783842756842877E-11,
+    9.21831518760500529508E-11,
+    -4.19035475934189648750E-10,
+    2.01504975519703286596E-9,
+    -1.03457624656780970260E-8,
+    5.74108412545004946722E-8,
+    -3.50196060308781257119E-7,
+    2.40648494783721712015E-6,
+    -1.93619797416608296024E-5,
+    1.95215518471351631108E-4,
+    -2.85781685962277938680E-3,
+    1.03923736576817238437E-1,
+    2.72062619048444266945E0
+};
+
+__device__ double k0(double x){
+    if (x == 0) {
+        return CUDART_INF;
+    }
+
+    if (x < 0) {
+        return CUDART_NAN;
+    }
+
+    double y, z;
+
+    if (x <= 2.0) {
+        y = x * x - 2.0;
+        y = chbevl(y, k0_A, 10) - log(0.5 * x) * cyl_bessel_i0(x);
+        return y;
+    }
+
+    z = 8.0 / x - 2.0;
+    y = chbevl(z, k0_B, 25) / sqrt(x);
+    return y * exp(-x);
+}
+
+__device__ float k0f(float x){
+    if (x == 0) {
+        return CUDART_INF;
+    }
+
+    if (x < 0) {
+        return CUDART_NAN;
+    }
+
+    float y, z;
+
+    if (x <= 2.0) {
+        y = x * x - 2.0;
+        y = chbevl(y, k0_AF, 10) - logf(0.5 * x) * cyl_bessel_i0f(x);
+        return y;
+    }
+
+    z = 8.0 / x - 2.0;
+    y = chbevl(z, k0_BF, 25) / sqrtf(x);
+    return y * expf(-x);
+}
+
+__device__ double k1(double x){
+    if (x == 0) {
+        return CUDART_INF;
+    }
+
+    if (x < 0) {
+        return CUDART_NAN;
+    }
+
+    double y;
+
+    if (x <= 2.0) {
+        y = x * x - 2.0;
+        y = log(0.5 * x) * cyl_bessel_i1(x) + chbevl(y, k1_A, 11) / x;
+        return y;
+    }
+
+    return (exp(-x) * chbevl(8.0 / x - 2.0, k1_B, 25) / sqrt(x));
+}
+
+__device__ float k1f(float x){
+    if (x == 0) {
+        return CUDART_INF;
+    }
+
+    if (x < 0) {
+        return CUDART_NAN;
+    }
+
+    float y;
+    float i1;
+
+    if (x <= 2.0) {
+        y = x * x - 2.0;
+        i1 = cyl_bessel_i1f(x);
+        y = logf(0.5 * x) * i1 + chbevl(y, k1_AF, 11) / x;
+        return y;
+    }
+
+    float z = 8.0 / x - 2.0;
+    return (expf(-x) * chbevl(z, k1_BF, 25) / sqrtf(x));
+}
+"""
+
+
+k0 = _core.create_ufunc(
+    'cupyx_scipy_special_k0',
+    (('f->f', 'out0 = k0f(in0)'), 'd->d'),
+    'out0 = k0(in0)',
+    preamble=k_preamble,
+    doc='''Modified Bessel function of the second kind of order 0.
+
+    Args:
+        x (cupy.ndarray): argument (float)
+
+    Returns:
+        cupy.ndarray: Value of the modified Bessel function K of order 0 at x.
+
+    .. seealso:: :meth:`scipy.special.k0`
+
+    ''')
+
+
+k0e = _core.create_ufunc(
+    'cupyx_scipy_special_k0e',
+    (('f->f', 'out0 = expf(in0) * k0f(in0)'), 'd->d'),
+    'out0 = exp(in0) * k0(in0)',
+    preamble=k_preamble,
+    doc='''Exponentially scaled modified Bessel function K of order 0
+
+    Args:
+        x (cupy.ndarray): argument (float)
+
+    Returns:
+        cupy.ndarray: Value at x.
+
+    .. seealso:: :meth:`scipy.special.k0e`
+
+    ''')
+
+
+k1 = _core.create_ufunc(
+    'cupyx_scipy_special_k1',
+    (('f->f', 'out0 = k1f(in0)'), 'd->d'),
+    'out0 = k1(in0)',
+    preamble=k_preamble,
+    doc='''Modified Bessel function of the second kind of order 1.
+
+    Args:
+        x (cupy.ndarray): argument (float)
+
+    Returns:
+        cupy.ndarray: Value of the modified Bessel function K of order 1 at x.
+
+    .. seealso:: :meth:`scipy.special.k1`
+
+    ''')
+
+
+k1e = _core.create_ufunc(
+    'cupyx_scipy_special_k1e',
+    (('f->f', 'out0 = expf(in0) * k1f(in0)'), 'd->d'),
+    'out0 = exp(in0) * k1(in0)',
+    preamble=k_preamble,
+    doc='''Exponentially scaled modified Bessel function K of order 1
+
+    Args:
+        x (cupy.ndarray): argument (float)
+
+    Returns:
+        cupy.ndarray: Value at x.
+
+    .. seealso:: :meth:`scipy.special.k1e`
+
+    ''')
diff --git a/cupyx/scipy/special/_beta.py b/cupyx/scipy/special/_beta.py
new file mode 100644
index 0000000..fb1568c
--- /dev/null
+++ b/cupyx/scipy/special/_beta.py
@@ -0,0 +1,1091 @@
+"""Beta and log(abs(beta)) functions.
+
+Also the incomplete beta function and its inverse.
+
+The source code here is an adaptation with minimal changes from the following
+files in SciPy's bundled Cephes library:
+
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/beta.c
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/incbet.c
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/incbi.c
+
+Cephes Math Library, Release 2.3:  March, 1995
+Copyright 1984, 1995 by Stephen L. Moshier
+"""
+
+from cupy import _core
+from cupyx.scipy.special._digamma import polevl_definition
+from cupyx.scipy.special._gamma import gamma_definition
+from cupyx.scipy.special._gammainc import p1evl_definition
+
+
+beta_preamble = """
+// include for CUDART_INF, CUDART_NAN
+#include <cupy/math_constants.h>
+
+// defines from /scipy/special/cephes/const.c
+// The ones chosen here are the IEEE variants from that file
+__constant__ double MAXLOG = 7.09782712893383996732E2;
+__constant__ double MINLOG = -7.08396418532264106224E2;
+__constant__ double MACHEP = 1.11022302462515654042E-16;  // 2**-53
+
+// defines from npy_math.h
+#define NPY_PI        3.141592653589793238462643383279502884  /* pi */
+
+#define MAXGAM 171.624376956302725
+#define ASYMP_FACTOR 1e6
+"""
+
+
+lgam_sgn_definition = """
+
+
+/* A[]: Stirling's formula expansion of log Gamma
+ * B[], C[]: log Gamma function between 2 and 3
+ */
+__constant__ double A[] = {
+    8.11614167470508450300E-4,
+    -5.95061904284301438324E-4,
+    7.93650340457716943945E-4,
+    -2.77777777730099687205E-3,
+    8.33333333333331927722E-2
+};
+
+__constant__ double B[] = {
+    -1.37825152569120859100E3,
+    -3.88016315134637840924E4,
+    -3.31612992738871184744E5,
+    -1.16237097492762307383E6,
+    -1.72173700820839662146E6,
+    -8.53555664245765465627E5
+};
+
+__constant__ double C[] = {
+    /* 1.00000000000000000000E0, */
+    -3.51815701436523470549E2,
+    -1.70642106651881159223E4,
+    -2.20528590553854454839E5,
+    -1.13933444367982507207E6,
+    -2.53252307177582951285E6,
+    -2.01889141433532773231E6
+};
+
+/* log( sqrt( 2*pi ) ) */
+__constant__ double LS2PI = 0.91893853320467274178;
+
+__constant__ double LOGPI = 1.14472988584940017414;
+
+#define MAXLGM 2.556348e305
+
+
+__noinline__ __device__ static double lgam_sgn(double x, int *sign)
+{
+    double p, q, u, w, z;
+    int i;
+
+    *sign = 1;
+
+    if (isinf(x)) {
+        return x;
+    }
+
+    if (x < -34.0) {
+        q = -x;
+        w = lgam_sgn(q, sign);
+        p = floor(q);
+        if (p == q) {
+            return CUDART_INF;
+        }
+        i = p;
+        if ((i & 1) == 0)
+            *sign = -1;
+        else
+            *sign = 1;
+        z = q - p;
+        if (z > 0.5) {
+            p += 1.0;
+            z = p - q;
+        }
+        z = q * sin(NPY_PI * z);
+        if (z == 0.0){
+            return CUDART_INF;
+        }
+        /*     z = log(NPY_PI) - log( z ) - w; */
+        z = LOGPI - log(z) - w;
+        return z;
+    }
+
+    if (x < 13.0) {
+        z = 1.0;
+        p = 0.0;
+        u = x;
+        while (u >= 3.0) {
+            p -= 1.0;
+            u = x + p;
+            z *= u;
+        }
+        while (u < 2.0) {
+            if (u == 0.0) {
+                return CUDART_INF;
+            }
+            z /= u;
+            p += 1.0;
+            u = x + p;
+        }
+        if (z < 0.0) {
+            *sign = -1;
+            z = -z;
+        }
+        else {
+            *sign = 1;
+        }
+        if (u == 2.0) {
+            return log(z);
+        }
+        p -= 2.0;
+        x = x + p;
+        p = x * polevl<5>(x, B) / p1evl<6>(x, C);
+        return (log(z) + p);
+    }
+
+    if (x > MAXLGM) {
+        return (*sign * CUDART_INF);
+    }
+
+    q = (x - 0.5) * log(x) - x + LS2PI;
+    if (x > 1.0e8) {
+        return q;
+    }
+
+    p = 1.0 / (x * x);
+    if (x >= 1000.0) {
+        q += ((7.9365079365079365079365e-4 * p
+               - 2.7777777777777777777778e-3) * p
+              + 0.0833333333333333333333) / x;
+    } else {
+        q += polevl<4>(p, A) / x;
+    }
+    return q;
+}
+
+
+__device__ static double lgam(double x)
+{
+    int sign;
+    return lgam_sgn(x, &sign);
+}
+
+"""
+
+lbeta_symp_definition = """
+/*
+ * Asymptotic expansion for  ln(|B(a, b)|) for a > ASYMP_FACTOR*max(|b|, 1).
+ */
+__noinline__ __device__ double lbeta_asymp(double a, double b, int *sgn)
+{
+    double r = lgam_sgn(b, sgn);
+    r -= b * log(a);
+
+    r += b*(1-b)/(2*a);
+    r += b*(1-b)*(1-2*b)/(12*a*a);
+    r += - b*b*(1-b)*(1-b)/(12*a*a*a);
+
+    return r;
+}
+"""
+
+
+beta_definition = """
+
+__noinline__ __device__ double beta(double, double);
+
+
+/*
+ * Special case for a negative integer argument
+ */
+
+__noinline__ __device__ static double beta_negint(int a, double b)
+{
+    int sgn;
+    if (b == (int)b && 1 - a - b > 0) {
+        sgn = ((int)b % 2 == 0) ? 1 : -1;
+        return sgn * beta(1 - a - b, b);
+    }
+    else {
+        return CUDART_INF;
+    }
+}
+
+__noinline__ __device__ double beta(double a, double b)
+{
+    double y;
+    int sign = 1;
+
+    if (a <= 0.0) {
+        if (a == floor(a)) {
+            if (a == (int)a) {
+                return beta_negint((int)a, b);
+            }
+            else {
+                return CUDART_INF;
+            }
+        }
+    }
+
+    if (b <= 0.0) {
+        if (b == floor(b)) {
+            if (b == (int)b) {
+                return beta_negint((int)b, a);
+            }
+            else {
+                return CUDART_INF;
+            }
+        }
+    }
+
+    if (fabs(a) < fabs(b)) {
+        y = a; a = b; b = y;
+    }
+
+    if (fabs(a) > ASYMP_FACTOR * fabs(b) && a > ASYMP_FACTOR) {
+        /* Avoid loss of precision in lgam(a + b) - lgam(a) */
+        y = lbeta_asymp(a, b, &sign);
+        return sign * exp(y);
+    }
+
+    y = a + b;
+    if (fabs(y) > MAXGAM || fabs(a) > MAXGAM || fabs(b) > MAXGAM) {
+        int sgngam;
+        y = lgam_sgn(y, &sgngam);
+        sign *= sgngam;     /* keep track of the sign */
+        y = lgam_sgn(b, &sgngam) - y;
+        sign *= sgngam;
+        y = lgam_sgn(a, &sgngam) + y;
+        sign *= sgngam;
+        if (y > MAXLOG) {
+            return sign * CUDART_INF;
+        }
+        return sign * exp(y);
+    }
+
+    y = Gamma(y);
+    a = Gamma(a);
+    b = Gamma(b);
+    if (y == 0.0) {
+        return sign * CUDART_INF;
+    }
+
+    if (fabs(fabs(a) - fabs(y)) > fabs(fabs(b) - fabs(y))) {
+        y = b / y;
+        y *= a;
+    }
+    else {
+        y = a / y;
+        y *= b;
+    }
+
+    return (y);
+}
+"""
+
+
+lbeta_definition = """
+
+__noinline__ __device__ double lbeta(double, double);
+
+
+/*
+ * Special case for a negative integer argument
+ */
+
+__noinline__ __device__ static double lbeta_negint(int a, double b)
+{
+    double r;
+    if (b == (int)b && 1 - a - b > 0) {
+        r = lbeta(1 - a - b, b);
+        return r;
+    }
+    else {
+        return CUDART_INF;
+    }
+}
+
+// Natural log of |beta|
+
+__noinline__ __device__ double lbeta(double a, double b)
+{
+    double y;
+    int sign;
+
+    sign = 1;
+
+    if (a <= 0.0) {
+        if (a == floor(a)) {
+            if (a == (int)a) {
+                return lbeta_negint((int)a, b);
+            }
+            else {
+                return CUDART_INF;
+            }
+        }
+    }
+
+    if (b <= 0.0) {
+        if (b == floor(b)) {
+            if (b == (int)b) {
+                return lbeta_negint((int)b, a);
+            }
+            else {
+                return CUDART_INF;
+            }
+        }
+    }
+
+    if (fabs(a) < fabs(b)) {
+        y = a; a = b; b = y;
+    }
+
+    if (fabs(a) > ASYMP_FACTOR * fabs(b) && a > ASYMP_FACTOR) {
+        /* Avoid loss of precision in lgam(a + b) - lgam(a) */
+        y = lbeta_asymp(a, b, &sign);
+        return y;
+    }
+
+    y = a + b;
+    if (fabs(y) > MAXGAM || fabs(a) > MAXGAM || fabs(b) > MAXGAM) {
+        int sgngam;
+        y = lgam_sgn(y, &sgngam);
+        sign *= sgngam;     /* keep track of the sign */
+        y = lgam_sgn(b, &sgngam) - y;
+        sign *= sgngam;
+        y = lgam_sgn(a, &sgngam) + y;
+        sign *= sgngam;
+        return (y);
+    }
+
+    y = Gamma(y);
+    a = Gamma(a);
+    b = Gamma(b);
+    if (y == 0.0) {
+        return (sign * CUDART_INF);
+    }
+
+    if (fabs(fabs(a) - fabs(y)) > fabs(fabs(b) - fabs(y))) {
+        y = b / y;
+        y *= a;
+    }
+    else {
+        y = a / y;
+        y *= b;
+    }
+
+    if (y < 0) {
+        y = -y;
+    }
+    return log(y);
+}
+
+"""
+
+
+beta = _core.create_ufunc(
+    "cupyx_scipy_beta",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(beta(in0, in1));",
+    preamble=(
+        beta_preamble +
+        gamma_definition +
+        polevl_definition +
+        p1evl_definition +
+        lgam_sgn_definition +
+        lbeta_symp_definition +
+        beta_definition
+    ),
+    doc="""Beta function.
+
+    Parameters
+    ----------
+    a, b : cupy.ndarray
+        Real-valued arguments
+    out : cupy.ndarray, optional
+        Optional output array for the function result
+
+    Returns
+    -------
+    scalar or ndarray
+        Value of the beta function
+
+    See Also
+    --------
+    :func:`scipy.special.beta`
+
+    """,
+)
+
+
+betaln = _core.create_ufunc(
+    "cupyx_scipy_betaln",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(lbeta(in0, in1));",
+    preamble=(
+        beta_preamble +
+        gamma_definition +
+        polevl_definition +
+        p1evl_definition +
+        lgam_sgn_definition +
+        lbeta_symp_definition +
+        lbeta_definition
+    ),
+    doc="""Natural logarithm of absolute value of beta function.
+
+    Computes ``ln(abs(beta(a, b)))``.
+
+    Parameters
+    ----------
+    a, b : cupy.ndarray
+        Real-valued arguments
+    out : cupy.ndarray, optional
+        Optional output array for the function result
+
+    Returns
+    -------
+    scalar or ndarray
+        Value of the natural log of the magnitude of beta.
+
+    See Also
+    --------
+    :func:`scipy.special.betaln`
+
+    """,
+)
+
+
+incbet_definition = """
+
+__noinline__ __device__ static double incbd(double, double, double);
+__noinline__ __device__ static double incbcf(double, double, double);
+__noinline__ __device__ static double pseries(double, double, double);
+
+__constant__ double big = 4.503599627370496e15;
+__constant__ double biginv = 2.22044604925031308085e-16;
+
+__noinline__ __device__ double incbet(double aa, double bb, double xx)
+{
+    double a, b, t, x, xc, w, y;
+    int flag;
+
+    if (aa <= 0.0 || bb <= 0.0)
+    {
+        return CUDART_NAN;
+    }
+
+    if ((xx <= 0.0) || (xx >= 1.0)) {
+        if (xx == 0.0) {
+            return 0.0;
+        }
+        if (xx == 1.0) {
+            return 1.0;
+        }
+        return CUDART_NAN;
+    }
+
+    flag = 0;
+    if ((bb * xx) <= 1.0 && xx <= 0.95) {
+        t = pseries(aa, bb, xx);
+        goto done;
+    }
+
+    w = 1.0 - xx;
+
+    /* Reverse a and b if x is greater than the mean. */
+    if (xx > (aa / (aa + bb))) {
+        flag = 1;
+        a = bb;
+        b = aa;
+        xc = xx;
+        x = w;
+    } else {
+        a = aa;
+        b = bb;
+        xc = w;
+        x = xx;
+    }
+
+    if (flag == 1 && (b * x) <= 1.0 && x <= 0.95) {
+        t = pseries(a, b, x);
+        goto done;
+    }
+
+    /* Choose expansion for better convergence. */
+    y = x * (a + b - 2.0) - (a - 1.0);
+    if (y < 0.0) {
+        w = incbcf(a, b, x);
+    } else {
+        w = incbd(a, b, x) / xc;
+    }
+
+    /* Multiply w by the factor
+     * a      b   _             _     _
+     * x  (1-x)   | (a+b) / ( a | (a) | (b) ) .   */
+
+    y = a * log(x);
+    t = b * log(xc);
+    if ((a + b) < MAXGAM && fabs(y) < MAXLOG && fabs(t) < MAXLOG) {
+        t = pow(xc, b);
+        t *= pow(x, a);
+        t /= a;
+        t *= w;
+        t *= 1.0 / beta(a, b);
+        goto done;
+    }
+    /* Resort to logarithms.  */
+    y += t - lbeta(a,b);
+    y += log(w / a);
+    if (y < MINLOG) {
+        t = 0.0;
+    } else {
+        t = exp(y);
+    }
+
+  done:
+
+    if (flag == 1) {
+        if (t <= MACHEP) {
+            t = 1.0 - MACHEP;
+        } else {
+            t = 1.0 - t;
+        }
+    }
+    return t;
+}
+
+
+/* Continued fraction expansion #1
+ * for incomplete beta integral
+ */
+
+__noinline__ __device__ static double incbcf(double a, double b, double x)
+{
+    double xk, pk, pkm1, pkm2, qk, qkm1, qkm2;
+    double k1, k2, k3, k4, k5, k6, k7, k8;
+    double r, t, ans, thresh;
+    int n;
+
+    k1 = a;
+    k2 = a + b;
+    k3 = a;
+    k4 = a + 1.0;
+    k5 = 1.0;
+    k6 = b - 1.0;
+    k7 = k4;
+    k8 = a + 2.0;
+
+    pkm2 = 0.0;
+    qkm2 = 1.0;
+    pkm1 = 1.0;
+    qkm1 = 1.0;
+    ans = 1.0;
+    r = 1.0;
+    n = 0;
+    thresh = 3.0 * MACHEP;
+    do {
+
+        xk = -(x * k1 * k2) / (k3 * k4);
+        pk = pkm1 + pkm2 * xk;
+        qk = qkm1 + qkm2 * xk;
+        pkm2 = pkm1;
+        pkm1 = pk;
+        qkm2 = qkm1;
+        qkm1 = qk;
+
+        xk = (x * k5 * k6) / (k7 * k8);
+        pk = pkm1 + pkm2 * xk;
+        qk = qkm1 + qkm2 * xk;
+        pkm2 = pkm1;
+        pkm1 = pk;
+        qkm2 = qkm1;
+        qkm1 = qk;
+
+        if (qk != 0)
+            r = pk / qk;
+        if (r != 0) {
+            t = fabs((ans - r) / r);
+            ans = r;
+        } else {
+            t = 1.0;
+        }
+
+        if (t < thresh) {
+            return ans;
+        }
+
+        k1 += 1.0;
+        k2 += 1.0;
+        k3 += 2.0;
+        k4 += 2.0;
+        k5 += 1.0;
+        k6 -= 1.0;
+        k7 += 2.0;
+        k8 += 2.0;
+
+        if ((fabs(qk) + fabs(pk)) > big) {
+            pkm2 *= biginv;
+            pkm1 *= biginv;
+            qkm2 *= biginv;
+            qkm1 *= biginv;
+        }
+        if ((fabs(qk) < biginv) || (fabs(pk) < biginv)) {
+            pkm2 *= big;
+            pkm1 *= big;
+            qkm2 *= big;
+            qkm1 *= big;
+        }
+    }
+    while (++n < 300);
+
+    return ans;
+}
+
+
+
+/* Continued fraction expansion #2
+ * for incomplete beta integral
+ */
+
+__noinline__ __device__ static double incbd(double a, double b, double x)
+{
+    double xk, pk, pkm1, pkm2, qk, qkm1, qkm2;
+    double k1, k2, k3, k4, k5, k6, k7, k8;
+    double r, t, ans, z, thresh;
+    int n;
+
+    k1 = a;
+    k2 = b - 1.0;
+    k3 = a;
+    k4 = a + 1.0;
+    k5 = 1.0;
+    k6 = a + b;
+    k7 = a + 1.0;;
+    k8 = a + 2.0;
+
+    pkm2 = 0.0;
+    qkm2 = 1.0;
+    pkm1 = 1.0;
+    qkm1 = 1.0;
+    z = x / (1.0 - x);
+    ans = 1.0;
+    r = 1.0;
+    n = 0;
+    thresh = 3.0 * MACHEP;
+    do {
+
+        xk = -(z * k1 * k2) / (k3 * k4);
+        pk = pkm1 + pkm2 * xk;
+        qk = qkm1 + qkm2 * xk;
+        pkm2 = pkm1;
+        pkm1 = pk;
+        qkm2 = qkm1;
+        qkm1 = qk;
+
+        xk = (z * k5 * k6) / (k7 * k8);
+        pk = pkm1 + pkm2 * xk;
+        qk = qkm1 + qkm2 * xk;
+        pkm2 = pkm1;
+        pkm1 = pk;
+        qkm2 = qkm1;
+        qkm1 = qk;
+
+        if (qk != 0){
+            r = pk / qk;
+        }
+        if (r != 0) {
+            t = fabs((ans - r) / r);
+            ans = r;
+        } else {
+            t = 1.0;
+        }
+
+        if (t < thresh) {
+            return ans;
+        }
+
+        k1 += 1.0;
+        k2 -= 1.0;
+        k3 += 2.0;
+        k4 += 2.0;
+        k5 += 1.0;
+        k6 += 1.0;
+        k7 += 2.0;
+        k8 += 2.0;
+
+        if ((fabs(qk) + fabs(pk)) > big) {
+            pkm2 *= biginv;
+            pkm1 *= biginv;
+            qkm2 *= biginv;
+            qkm1 *= biginv;
+        }
+        if ((fabs(qk) < biginv) || (fabs(pk) < biginv)) {
+            pkm2 *= big;
+            pkm1 *= big;
+            qkm2 *= big;
+            qkm1 *= big;
+        }
+    }
+    while (++n < 300);
+
+    return ans;
+}
+
+
+/* Power series for incomplete beta integral.
+ * Use when b*x is small and x not too close to 1.  */
+
+__noinline__ __device__ static double pseries(double a, double b, double x)
+{
+    double s, t, u, v, n, t1, z, ai;
+
+    ai = 1.0 / a;
+    u = (1.0 - b) * x;
+    v = u / (a + 1.0);
+    t1 = v;
+    t = u;
+    n = 2.0;
+    s = 0.0;
+    z = MACHEP * ai;
+    while (fabs(v) > z) {
+        u = (n - b) * x / n;
+        t *= u;
+        v = t / (a + n);
+        s += v;
+        n += 1.0;
+    }
+    s += t1;
+    s += ai;
+
+    u = a * log(x);
+    if ((a + b) < MAXGAM && fabs(u) < MAXLOG) {
+        t = 1.0 / beta(a, b);
+    s = s * t * pow(x, a);
+    } else {
+        t = -lbeta(a,b) + u + log(s);
+        if (t < MINLOG) {
+            s = 0.0;
+        } else {
+            s = exp(t);
+        }
+    }
+    return (s);
+}
+
+
+"""
+
+incbet_preamble = (
+    beta_preamble +
+    gamma_definition +
+    polevl_definition +
+    p1evl_definition +
+    lgam_sgn_definition +
+    lbeta_symp_definition +
+    beta_definition +
+    lbeta_definition +
+    incbet_definition
+)
+
+
+betainc = _core.create_ufunc(
+    "cupyx_scipy_betainc",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(incbet(in0, in1, in2));",
+    preamble=incbet_preamble,
+    doc="""Incomplete beta function.
+
+    Parameters
+    ----------
+    a, b : cupy.ndarray
+        Positive, real-valued parameters
+    x : cupy.ndarray
+        Real-valued such that 0 <= x <= 1, the upper limit of integration.
+    out : ndarray, optional
+        Optional output array for the function result
+
+    Returns
+    -------
+    scalar or ndarray
+        Value of the incomplete beta function
+
+    See Also
+    --------
+    :func:`scipy.special.betainc`
+
+    """,
+)
+
+
+incbi_definition = """
+
+__noinline__ __device__ double incbi(double aa, double bb, double yy0)
+{
+    double a, b, y0, d, y, x, x0, x1, lgm, yp, di, dithresh, yl, yh, xt;
+    int i, rflg, dir, nflg;
+
+    i = 0;
+    if (yy0 <= 0) {
+        return 0.0;
+    }
+    if (yy0 >= 1.0) {
+        return 1.0;
+    }
+    x0 = 0.0;
+    yl = 0.0;
+    x1 = 1.0;
+    yh = 1.0;
+    nflg = 0;
+
+    if (aa <= 1.0 || bb <= 1.0) {
+        dithresh = 1.0e-6;
+        rflg = 0;
+        a = aa;
+        b = bb;
+        y0 = yy0;
+        x = a / (a + b);
+        y = incbet(a, b, x);
+        goto ihalve;
+    }
+    else {
+        dithresh = 1.0e-4;
+    }
+    /* approximation to inverse function */
+
+    // normcdfinv is the CUDA Math API equivalent of cephes ndtri
+    yp = -normcdfinv(yy0);
+
+    if (yy0 > 0.5) {
+        rflg = 1;
+        a = bb;
+        b = aa;
+        y0 = 1.0 - yy0;
+        yp = -yp;
+    }
+    else {
+        rflg = 0;
+        a = aa;
+        b = bb;
+        y0 = yy0;
+    }
+
+    lgm = (yp * yp - 3.0) / 6.0;
+    x = 2.0 / (1.0 / (2.0 * a - 1.0) + 1.0 / (2.0 * b - 1.0));
+    d = yp * sqrt(x + lgm) / x
+        - (1.0 / (2.0 * b - 1.0) - 1.0 / (2.0 * a - 1.0))
+        * (lgm + 5.0 / 6.0 - 2.0 / (3.0 * x));
+    d = 2.0 * d;
+    if (d < MINLOG) {
+        x = 1.0;
+        goto under;
+    }
+    x = a / (a + b * exp(d));
+    y = incbet(a, b, x);
+    yp = (y - y0) / y0;
+    if (fabs(yp) < 0.2) {
+        goto newt;
+    }
+
+    /* Resort to interval halving if not close enough. */
+ihalve:
+
+    dir = 0;
+    di = 0.5;
+    for (i = 0; i < 100; i++) {
+        if (i != 0) {
+            x = x0 + di * (x1 - x0);
+            if (x == 1.0) {
+                x = 1.0 - MACHEP;
+            }
+            if (x == 0.0) {
+                di = 0.5;
+                x = x0 + di * (x1 - x0);
+                if (x == 0.0)
+                    goto under;
+            }
+            y = incbet(a, b, x);
+            yp = (x1 - x0) / (x1 + x0);
+            if (fabs(yp) < dithresh) {
+                goto newt;
+            }
+            yp = (y - y0) / y0;
+            if (fabs(yp) < dithresh) {
+                goto newt;
+            }
+        }
+        if (y < y0) {
+            x0 = x;
+            yl = y;
+            if (dir < 0) {
+                dir = 0;
+                di = 0.5;
+            } else if (dir > 3) {
+                di = 1.0 - (1.0 - di) * (1.0 - di);
+            } else if (dir > 1) {
+                di = 0.5 * di + 0.5;
+            } else {
+                di = (y0 - y) / (yh - yl);
+            }
+            dir += 1;
+            if (x0 > 0.75) {
+                if (rflg == 1) {
+                    rflg = 0;
+                    a = aa;
+                    b = bb;
+                    y0 = yy0;
+                } else {
+                    rflg = 1;
+                    a = bb;
+                    b = aa;
+                    y0 = 1.0 - yy0;
+                }
+                x = 1.0 - x;
+                y = incbet(a, b, x);
+                x0 = 0.0;
+                yl = 0.0;
+                x1 = 1.0;
+                yh = 1.0;
+                goto ihalve;
+            }
+        }
+        else {
+            x1 = x;
+            if (rflg == 1 && x1 < MACHEP) {
+                x = 0.0;
+                goto done;
+            }
+            yh = y;
+            if (dir > 0) {
+                dir = 0;
+                di = 0.5;
+            } else if (dir < -3) {
+                di = di * di;
+            } else if (dir < -1) {
+                di = 0.5 * di;
+            } else {
+                di = (y - y0) / (yh - yl);
+            }
+            dir -= 1;
+        }
+    }
+    if (x0 >= 1.0) {
+        x = 1.0 - MACHEP;
+        goto done;
+    }
+    if (x <= 0.0) {
+under:
+        x = 0.0;
+        goto done;
+    }
+
+newt:
+
+    if (nflg) {
+        goto done;
+    }
+    nflg = 1;
+    lgm = lgam(a + b) - lgam(a) - lgam(b);
+
+    for (i = 0; i < 8; i++) {
+        /* Compute the function at this point. */
+        if (i != 0) {
+            y = incbet(a, b, x);
+        }
+        if (y < yl) {
+            x = x0;
+            y = yl;
+        } else if (y > yh) {
+            x = x1;
+            y = yh;
+        } else if (y < y0) {
+            x0 = x;
+            yl = y;
+        } else {
+            x1 = x;
+            yh = y;
+        }
+        if (x == 1.0 || x == 0.0)
+            break;
+        /* Compute the derivative of the function at this point. */
+        d = (a - 1.0) * log(x) + (b - 1.0) * log(1.0 - x) + lgm;
+        if (d < MINLOG) {
+            goto done;
+        }
+        if (d > MAXLOG) {
+            break;
+        }
+        d = exp(d);
+        /* Compute the step to the next approximation of x. */
+        d = (y - y0) / d;
+        xt = x - d;
+        if (xt <= x0) {
+            y = (x - x0) / (x1 - x0);
+            xt = x0 + 0.5 * y * (x - x0);
+            if (xt <= 0.0) {
+                break;
+            }
+        }
+        if (xt >= x1) {
+            y = (x1 - x) / (x1 - x0);
+            xt = x1 - 0.5 * y * (x1 - x);
+            if (xt >= 1.0) {
+                break;
+            }
+        }
+        x = xt;
+        if (fabs(d / x) < 128.0 * MACHEP) {
+            goto done;
+        }
+    }
+    /* Did not converge.  */
+    dithresh = 256.0 * MACHEP;
+    goto ihalve;
+
+done:
+
+    if (rflg) {
+        if (x <= MACHEP)
+            x = 1.0 - MACHEP;
+        else
+            x = 1.0 - x;
+    }
+    return x;
+}
+
+"""
+
+incbi_preamble = incbet_preamble + incbi_definition
+
+
+betaincinv = _core.create_ufunc(
+    "cupyx_scipy_betaincinv",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(incbi(in0, in1, in2));",
+    preamble=incbi_preamble,
+    doc="""Inverse of the incomplete beta function.
+
+    Parameters
+    ----------
+    a, b : cupy.ndarray
+        Positive, real-valued parameters
+    y : cupy.ndarray
+        Real-valued input.
+    out : ndarray, optional
+        Optional output array for the function result
+
+    Returns
+    -------
+    scalar or ndarray
+        Value of the inverse of the incomplete beta function
+
+    See Also
+    --------
+    :func:`scipy.special.betaincinv`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_complexstuff.py b/cupyx/scipy/special/_complexstuff.py
new file mode 100644
index 0000000..41512fd
--- /dev/null
+++ b/cupyx/scipy/special/_complexstuff.py
@@ -0,0 +1,54 @@
+"""complex-valued functions adapted from SciPy's cython code:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/_complexstuff.pxd
+
+Notes:
+- instead of zabs, use thrust::abs
+- instead of zarg, use thrust::arg
+- instead of zcos, use thrust::cos
+- instead of zexp, use thrust::exp
+- instead of zisfinite, use isfinite defined in _core/include/cupy/complex.cuh
+- instead of zisinf, use isinf defined in _core/include/cupy/complex.cuh
+- instead of zisnan, use isnan defined in _core/include/cupy/complex.cuh
+- instead of zpack, use complex<double>(real, imag)
+- instead of zpow, use thrust::pow
+- instead of zreal, use z.real()
+- instead of zsin, use thrust::sin
+- instead of zsqrt, use thrust::sqrt
+
+"""
+
+
+zlog1_definition = """
+
+/* Compute log, paying special attention to accuracy around 1. We
+ * implement this ourselves because some systems (most notably the
+ * Travis CI machines) are weak in this regime.
+ */
+
+#define TOL_ZLOG1 2.220446092504131e-16
+
+
+__device__ complex<double> zlog1(complex<double> z)
+{
+    complex<double> coeff = -1.0;
+    complex<double> res = 0.0;
+
+    if (abs(z - 1.0) > 0.1) {
+        return log(z);  // complex log via Thrust
+    }
+    z = z - 1.0;
+    if (z == 0.0) {
+        return 0;
+    }
+    for (int n=1; n<17; n++)
+    {
+        coeff *= -z;
+        res += coeff / complex<double>(n, 0);
+        if (abs(res/coeff) < TOL_ZLOG1) {
+            break;
+        }
+    }
+    return res;
+}
+"""
diff --git a/cupyx/scipy/special/_convex_analysis.py b/cupyx/scipy/special/_convex_analysis.py
new file mode 100644
index 0000000..5e60c30
--- /dev/null
+++ b/cupyx/scipy/special/_convex_analysis.py
@@ -0,0 +1,120 @@
+from cupy import _core
+
+
+_float_preamble = '''
+#include <cupy/math_constants.h>
+
+double __device__ entr(double x) {
+    if (isnan(x)) {
+        return CUDART_NAN;
+    } else if (x > 0){
+        return -x * log(x);
+    } else if (x == 0){
+        return 0;
+    } else {
+        return -CUDART_INF;
+    }
+}
+
+double __device__ kl_div(double x, double y) {
+    if (isnan(x) || isnan(y)) {
+        return CUDART_NAN;
+    } else if (x > 0 && y > 0) {
+        return x * log(x / y) - x + y;
+    } else if (x == 0 && y >= 0) {
+        return y;
+    } else {
+        return CUDART_INF;
+    }
+}
+
+double __device__ rel_entr(double x, double y) {
+    if (isnan(x) || isnan(y)) {
+        return CUDART_NAN;
+    } else if (x > 0 && y > 0) {
+        return x * log(x / y);
+    } else if (x == 0 && y >= 0) {
+        return 0;
+    } else {
+        return CUDART_INF;
+    }
+}
+
+double __device__ huber(double delta, double r) {
+    if (delta < 0) {
+        return CUDART_INF;
+    } else if (abs(r) <= delta) {
+        return 0.5 * r * r;
+    } else {
+        return delta * (abs(r) - 0.5 * delta);
+    }
+}
+
+double __device__ pseudo_huber(double delta, double r) {
+    if (delta < 0) {
+        return CUDART_INF;
+    } else if (delta == 0 || r == 0) {
+        return 0;
+    } else {
+        double u = delta;
+        double v = r / delta;
+        return u * u * (sqrt(1 + v * v) - 1);
+    }
+}
+
+'''
+
+
+entr = _core.create_ufunc(
+    'cupyx_scipy_special_entr', ('f->f', 'd->d'),
+    'out0 = out0_type(entr(in0));',
+    preamble=_float_preamble,
+    doc='''Elementwise function for computing entropy.
+
+    .. seealso:: :meth:`scipy.special.entr`
+
+    ''')
+
+
+kl_div = _core.create_ufunc(
+    'cupyx_scipy_special_kl_div', ('ff->f', 'dd->d'),
+    'out0 = out0_type(kl_div(in0, in1));',
+    preamble=_float_preamble,
+    doc='''Elementwise function for computing Kullback-Leibler divergence.
+
+    .. seealso:: :meth:`scipy.special.kl_div`
+
+    ''')
+
+
+rel_entr = _core.create_ufunc(
+    'cupyx_scipy_special_rel_entr', ('ff->f', 'dd->d'),
+    'out0 = out0_type(rel_entr(in0, in1));',
+    preamble=_float_preamble,
+    doc='''Elementwise function for computing relative entropy.
+
+    .. seealso:: :meth:`scipy.special.rel_entr`
+
+    ''')
+
+
+huber = _core.create_ufunc(
+    'cupyx_scipy_special_huber', ('ff->f', 'dd->d'),
+    'out0 = out0_type(huber(in0, in1));',
+    preamble=_float_preamble,
+    doc='''Elementwise function for computing the Huber loss.
+
+    .. seealso:: :meth:`scipy.special.huber`
+
+    ''')
+
+
+pseudo_huber = _core.create_ufunc(
+    'cupyx_scipy_special_pseudo_huber', ('ff->f', 'dd->d'),
+    'out0 = out0_type(pseudo_huber(in0, in1));',
+    preamble=_float_preamble,
+    doc='''Elementwise function for computing the Pseudo-Huber loss.
+
+    .. seealso:: :meth:`scipy.special.pseudo_huber`
+
+    ''')
diff --git a/cupyx/scipy/special/_digamma.py b/cupyx/scipy/special/_digamma.py
new file mode 100644
index 0000000..f98a43f
--- /dev/null
+++ b/cupyx/scipy/special/_digamma.py
@@ -0,0 +1,191 @@
+# This source code contains SciPy's code.
+# https://github.com/scipy/scipy/blob/master/scipy/special/cephes/psi.c
+#
+#
+# Cephes Math Library Release 2.8:  June, 2000
+# Copyright 1984, 1987, 1992, 2000 by Stephen L. Moshier
+#
+#
+# Code for the rational approximation on [1, 2] is:
+#
+# (C) Copyright John Maddock 2006.
+# Use, modification and distribution are subject to the
+# Boost Software License, Version 1.0. (See accompanying file
+# LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+from cupy import _core
+
+
+polevl_definition = '''
+template<int N> static __device__ double polevl(double x, double coef[])
+{
+    double ans;
+    double *p;
+
+    p = coef;
+    ans = *p++;
+
+    for (int i = 0; i < N; ++i){
+        ans = ans * x + *p++;
+    }
+
+    return ans;
+}
+'''
+
+
+psi_definition = '''
+__constant__ double A[] = {
+    8.33333333333333333333E-2,
+    -2.10927960927960927961E-2,
+    7.57575757575757575758E-3,
+    -4.16666666666666666667E-3,
+    3.96825396825396825397E-3,
+    -8.33333333333333333333E-3,
+    8.33333333333333333333E-2
+};
+
+__constant__ double PI = 3.141592653589793;
+__constant__ double EULER = 0.5772156649015329;
+
+__constant__ float Y = 0.99558162689208984f;
+
+__constant__ double root1 = 1569415565.0 / 1073741824.0;
+__constant__ double root2 = (381566830.0 / 1073741824.0) / 1073741824.0;
+__constant__ double root3 = 0.9016312093258695918615325266959189453125e-19;
+
+__constant__ double P[] = {
+    -0.0020713321167745952,
+    -0.045251321448739056,
+    -0.28919126444774784,
+    -0.65031853770896507,
+    -0.32555031186804491,
+    0.25479851061131551
+};
+__constant__ double Q[] = {
+    -0.55789841321675513e-6,
+    0.0021284987017821144,
+    0.054151797245674225,
+    0.43593529692665969,
+    1.4606242909763515,
+    2.0767117023730469,
+    1.0
+};
+
+static __device__ double digamma_imp_1_2(double x)
+{
+    /*
+     * Rational approximation on [1, 2] taken from Boost.
+     *
+     * Now for the approximation, we use the form:
+     *
+     * digamma(x) = (x - root) * (Y + R(x-1))
+     *
+     * Where root is the location of the positive root of digamma,
+     * Y is a constant, and R is optimised for low absolute error
+     * compared to Y.
+     *
+     * Maximum Deviation Found:               1.466e-18
+     * At double precision, max error found:  2.452e-17
+     */
+    double r, g;
+    g = x - root1 - root2 - root3;
+    r = polevl<5>(x - 1.0, P) / polevl<6>(x - 1.0, Q);
+
+    return g * Y + g * r;
+}
+
+
+static __device__ double psi_asy(double x)
+{
+    double y, z;
+
+    if (x < 1.0e17) {
+        z = 1.0 / (x * x);
+        y = z * polevl<6>(z, A);
+    }
+    else {
+        y = 0.0;
+    }
+
+    return log(x) - (0.5 / x) - y;
+}
+
+
+double __device__ psi(double x)
+{
+    double y = 0.0;
+    double q, r;
+    int i, n;
+
+    if (isnan(x)) {
+        return x;
+    }
+    else if (isinf(x)){
+        if(x > 0){
+            return x;
+        }else{
+            return nan("");
+        }
+    }
+    else if (x == 0) {
+        return -1.0/0.0;
+    }
+    else if (x < 0.0) {
+        /* argument reduction before evaluating tan(pi * x) */
+        r = modf(x, &q);
+        if (r == 0.0) {
+            return nan("");
+        }
+        y = -PI / tan(PI * r);
+        x = 1.0 - x;
+    }
+
+    /* check for positive integer up to 10 */
+    if ((x <= 10.0) && (x == floor(x))) {
+        n = (int)x;
+        for (i = 1; i < n; i++) {
+            y += 1.0 / i;
+        }
+        y -= EULER;
+        return y;
+    }
+
+    /* use the recurrence relation to move x into [1, 2] */
+    if (x < 1.0) {
+        y -= 1.0 / x;
+        x += 1.0;
+    }
+    else if (x < 10.0) {
+        while (x > 2.0) {
+            x -= 1.0;
+            y += 1.0 / x;
+        }
+    }
+    if ((1.0 <= x) && (x <= 2.0)) {
+        y += digamma_imp_1_2(x);
+        return y;
+    }
+
+    /* x is large, use the asymptotic series */
+    y += psi_asy(x);
+    return y;
+}
+'''
+
+
+digamma = _core.create_ufunc(
+    'cupyx_scipy_special_digamma', ('f->f', 'd->d'),
+    'out0 = psi(in0)',
+    preamble=polevl_definition+psi_definition,
+    doc="""The digamma function.
+
+    Args:
+        x (cupy.ndarray): The input of digamma function.
+
+    Returns:
+        cupy.ndarray: Computed value of digamma function.
+
+    .. seealso:: :data:`scipy.special.digamma`
+
+    """)
diff --git a/cupyx/scipy/special/_erf.py b/cupyx/scipy/special/_erf.py
new file mode 100644
index 0000000..7eea14a
--- /dev/null
+++ b/cupyx/scipy/special/_erf.py
@@ -0,0 +1,59 @@
+from cupy import _core
+
+
+erf = _core.create_ufunc(
+    'cupyx_scipy_special_erf', ('f->f', 'd->d'),
+    'out0 = erf(in0)',
+    doc='''Error function.
+
+    .. seealso:: :meth:`scipy.special.erf`
+
+    ''')
+
+
+erfc = _core.create_ufunc(
+    'cupyx_scipy_special_erfc', ('f->f', 'd->d'),
+    'out0 = erfc(in0)',
+    doc='''Complementary error function.
+
+    .. seealso:: :meth:`scipy.special.erfc`
+
+    ''')
+
+
+erfcx = _core.create_ufunc(
+    'cupyx_scipy_special_erfcx', ('f->f', 'd->d'),
+    'out0 = erfcx(in0)',
+    doc='''Scaled complementary error function.
+
+    .. seealso:: :meth:`scipy.special.erfcx`
+
+    ''')
+
+
+erfinv = _core.create_ufunc(
+    'cupyx_scipy_special_erfinv', ('f->f', 'd->d'),
+    'out0 = erfinv(in0);',
+    doc='''Inverse function of error function.
+
+    .. seealso:: :meth:`scipy.special.erfinv`
+
+    .. note::
+        The behavior close to (and outside) the domain follows that of
+        SciPy v1.4.0+.
+
+    ''')
+
+
+erfcinv = _core.create_ufunc(
+    'cupyx_scipy_special_erfcinv', ('f->f', 'd->d'),
+    'out0 = erfcinv(in0);',
+    doc='''Inverse function of complementary error function.
+
+    .. seealso:: :meth:`scipy.special.erfcinv`
+
+    .. note::
+        The behavior close to (and outside) the domain follows that of
+        SciPy v1.4.0+.
+
+    ''')
diff --git a/cupyx/scipy/special/_exp1.py b/cupyx/scipy/special/_exp1.py
new file mode 100644
index 0000000..ef550ba
--- /dev/null
+++ b/cupyx/scipy/special/_exp1.py
@@ -0,0 +1,62 @@
+# This source code contains SciPy's code.
+# https://github.com/scipy/scipy/blob/main/scipy/special/specfun/specfun.f
+
+from cupy import _core  # NOQA
+
+
+math_constants_and_eul = """
+#include <cupy/math_constants.h>
+__constant__ double EUL = 0.5772156649015328;
+"""
+
+exp1_definition = """
+template <typename T>
+static __device__ T exp1(T x) {
+    if (x == 0) {
+        return CUDART_INF;
+    } else if (x <= 1) {
+        T e1 = 1.0;
+        T R = 1.0;
+
+        for (int k = 1; k <= 25; k++) {
+            int den = (k + 1) * (k + 1);
+            R = -R * k * x / den;
+            e1 += R;
+        }
+        return -EUL - log(x) + x * e1;
+    }
+
+    int M = 20 + 80.0 / x;
+    T t0 = 0;
+    for (int k = M; k != 0; k--) {
+        t0 = k / (1.0 + k / (x + t0));
+    }
+    T t = 1.0 / (x + t0);
+    return exp(-x) * t;
+}
+"""
+
+
+exp1 = _core.create_ufunc(
+    'cupyx_scipy_special_exp1',
+    ('f->f', 'd->d'),
+    'out0 = exp1(in0)',
+    preamble=math_constants_and_eul + exp1_definition,
+    doc="""Exponential integral E1.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        Real argument
+
+    Returns
+    -------
+    y : scalar or cupy.ndarray
+        Values of the exponential integral E1
+
+    See Also
+    --------
+    :func:`scipy.special.exp1`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_expi.py b/cupyx/scipy/special/_expi.py
new file mode 100644
index 0000000..02c018c
--- /dev/null
+++ b/cupyx/scipy/special/_expi.py
@@ -0,0 +1,60 @@
+# This source code contains SciPy's code.
+# https://github.com/scipy/scipy/blob/main/scipy/special/specfun/specfun.f
+
+from cupy import _core  # NOQA
+from cupyx.scipy.special._exp1 import exp1_definition
+from cupyx.scipy.special._exp1 import math_constants_and_eul
+
+
+expi_definition = """
+template < typename T >
+__device__ double expi(T x) {
+    T ei = 1;
+    T r = 1;
+
+    if (x == 0) {
+        return -CUDART_INF;
+    } else if (x < 0) {
+        return -exp1(-x);
+    } else if (x <= 40.0) {
+        for (int k = 1; k <= 100; k++) {
+            int den = (k + 1) * (k + 1);
+            r = r * k * x / den;
+            ei += r;
+        }
+
+        return EUL + x * ei + log(x);
+    }
+
+    for (int k = 1; k <= 40; k++) {
+        r = r * k / x;
+        ei += r;
+    }
+
+    return exp(x) / x * ei;
+}
+"""
+
+expi = _core.create_ufunc(
+    'cupyx_scipy_special_expi',
+    ('f->f', 'd->d'),
+    'out0 = expi(in0)',
+    preamble=math_constants_and_eul + exp1_definition + expi_definition,
+    doc="""Exponential integral Ei.
+
+    Parameters
+    ----------
+    x : cupy.ndarray
+        Real argument
+
+    Returns
+    -------
+    y : scalar or cupy.ndarray
+        Values of exponential integral
+
+    See Also
+    --------
+    :func:`scipy.special.expi`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_expn.py b/cupyx/scipy/special/_expn.py
new file mode 100644
index 0000000..f30605d
--- /dev/null
+++ b/cupyx/scipy/special/_expn.py
@@ -0,0 +1,359 @@
+# This source code contains SciPy's code.
+# https://github.com/scipy/scipy/blob/main/scipy/special/cephes/expn.c
+#
+#
+# Cephes Math Library Release 1.1:  March, 1985
+# Copyright 1985 by Stephen L. Moshier
+# Direct inquiries to 30 Frost Street, Cambridge, MA 02140
+
+from cupy import _core
+
+from cupyx.scipy.special._gamma import gamma_definition
+
+
+polevl_definition = '''
+
+__device__ double polevl(double x, double coef[], int N)
+{
+    double ans;
+    double *p;
+    p = coef;
+    ans = *p++;
+    for (int i = 0; i < N; ++i){
+        ans = ans * x + *p++;
+    }
+    return ans;
+}
+
+'''
+
+
+expn_large_n_definition = '''
+
+__constant__ double EUL = 0.57721566490153286060;
+__constant__ double BIG = 1.44115188075855872E+17;
+__constant__ double MACHEP = 1.11022302462515654042E-16;
+__constant__ double MAXLOG = 7.08396418532264106224E2;
+
+#define nA 13
+
+__constant__ double _A0[] = {
+    1.00000000000000000
+};
+
+__constant__ double _A1[] = {
+    1.00000000000000000
+};
+
+__constant__ double _A2[] = {
+    -2.00000000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A3[] = {
+    6.00000000000000000,
+    -8.00000000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A4[] = {
+    -24.0000000000000000,
+    58.0000000000000000,
+    -22.0000000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A5[] = {
+    120.000000000000000,
+    -444.000000000000000,
+    328.000000000000000,
+    -52.0000000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A6[] = {
+    -720.000000000000000,
+    3708.00000000000000,
+    -4400.00000000000000,
+    1452.00000000000000,
+    -114.000000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A7[] = {
+    5040.00000000000000,
+    -33984.0000000000000,
+    58140.0000000000000,
+    -32120.0000000000000,
+    5610.00000000000000,
+    -240.000000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A8[] = {
+    -40320.0000000000000,
+    341136.000000000000,
+    -785304.000000000000,
+    644020.000000000000,
+    -195800.000000000000,
+    19950.0000000000000,
+    -494.000000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A9[] = {
+    362880.000000000000,
+    -3733920.00000000000,
+    11026296.0000000000,
+    -12440064.0000000000,
+    5765500.00000000000,
+    -1062500.00000000000,
+    67260.0000000000000,
+    -1004.00000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A10[] = {
+    -3628800.00000000000,
+    44339040.0000000000,
+    -162186912.000000000,
+    238904904.000000000,
+    -155357384.000000000,
+    44765000.0000000000,
+    -5326160.00000000000,
+    218848.000000000000,
+    -2026.00000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A11[] = {
+    39916800.0000000000,
+    -568356480.000000000,
+    2507481216.00000000,
+    -4642163952.00000000,
+    4002695088.00000000,
+    -1648384304.00000000,
+    314369720.000000000,
+    -25243904.0000000000,
+    695038.000000000000,
+    -4072.00000000000000,
+    1.00000000000000000
+};
+
+__constant__ double _A12[] = {
+    -479001600.000000000,
+    7827719040.00000000,
+    -40788301824.0000000,
+    92199790224.0000000,
+    -101180433024.000000,
+    56041398784.0000000,
+    -15548960784.0000000,
+    2051482776.00000000,
+    -114876376.000000000,
+    2170626.00000000000,
+    -8166.00000000000000,
+    1.00000000000000000
+};
+
+__constant__ double *A[] = {
+    _A0, _A1, _A2,
+    _A3, _A4, _A5,
+    _A6, _A7, _A8,
+    _A9, _A10, _A11,
+    _A12
+};
+
+__constant__ int Adegs[] = {
+    0, 0, 1,
+    2, 3, 4,
+    5, 6, 7,
+    8, 9, 10,
+    11
+};
+
+/* Asymptotic expansion for large n, DLMF 8.20(ii) */
+__device__ double expn_large_n(int n, double x)
+{
+    int k;
+    double p = n;
+    double lambda = x/p;
+    double multiplier = 1/p/(lambda + 1)/(lambda + 1);
+    double fac = 1;
+    double res = 1; /* A[0] = 1 */
+    double expfac, term;
+
+    expfac = exp(-lambda*p)/(lambda + 1)/p;
+    if (expfac == 0) {
+        return 0;
+    }
+
+    /* Do the k = 1 term outside the loop since A[1] = 1 */
+    fac *= multiplier;
+    res += fac;
+
+    for (k = 2; k < nA; k++) {
+        fac *= multiplier;
+        term = fac*polevl(lambda, A[k], Adegs[k]);
+        res += term;
+        if (fabs(term) < MACHEP*fabs(res)) {
+            break;
+        }
+    }
+
+    return expfac*res;
+}
+
+'''
+
+
+expn_definition = (
+    polevl_definition
+    + gamma_definition
+    + expn_large_n_definition
+    + '''
+
+// include for CUDART_NAN, CUDART_INF
+#include <cupy/math_constants.h>
+
+__device__ double expn(int n, double x)
+{
+    double ans, r, t, yk, xk;
+    double pk, pkm1, pkm2, qk, qkm1, qkm2;
+    double psi, z;
+    int i, k;
+    double big = BIG;
+
+    if (isnan(x)) {
+        return CUDART_NAN;
+    } else if (n < 0 || x < 0) {
+        return CUDART_NAN;
+    }
+
+    if (x > MAXLOG) {
+        return (0.0);
+    }
+
+    if (x == 0.0) {
+        if (n < 2) {
+            return CUDART_INF;
+        } else {
+            return (1.0 / (n - 1.0));
+        }
+    }
+
+    if (n == 0) {
+        return (exp(-x) / x);
+    }
+
+    /* Asymptotic expansion for large n, DLMF 8.20(ii) */
+    if (n > 50) {
+        ans = expn_large_n(n, x);
+        return (ans);
+    }
+
+    /* Continued fraction, DLMF 8.19.17 */
+    if (x > 1.0) {
+        k = 1;
+        pkm2 = 1.0;
+        qkm2 = x;
+        pkm1 = 1.0;
+        qkm1 = x + n;
+        ans = pkm1 / qkm1;
+
+        do {
+            k += 1;
+            if (k & 1) {
+                yk = 1.0;
+                xk = n + (k - 1) / 2;
+            } else {
+                yk = x;
+                xk = k / 2;
+            }
+            pk = pkm1 * yk + pkm2 * xk;
+            qk = qkm1 * yk + qkm2 * xk;
+            if (qk != 0) {
+                r = pk / qk;
+                t = fabs((ans - r) / r);
+                ans = r;
+            } else {
+                t = 1.0;
+            }
+            pkm2 = pkm1;
+            pkm1 = pk;
+            qkm2 = qkm1;
+            qkm1 = qk;
+            if (fabs(pk) > big) {
+                pkm2 /= big;
+                pkm1 /= big;
+                qkm2 /= big;
+                qkm1 /= big;
+            }
+        } while (t > MACHEP);
+
+        ans *= exp(-x);
+        return ans;
+    }
+
+    /* Power series expansion, DLMF 8.19.8 */
+    psi = -EUL - log(x);
+    for (i = 1; i < n; i++) {
+        psi = psi + 1.0 / i;
+    }
+
+    z = -x;
+    xk = 0.0;
+    yk = 1.0;
+    pk = 1.0 - n;
+    if (n == 1) {
+        ans = 0.0;
+    } else {
+        ans = 1.0 / pk;
+    }
+    do {
+        xk += 1.0;
+        yk *= z / xk;
+        pk += 1.0;
+        if (pk != 0.0) {
+            ans += yk / pk;
+        }
+        if (ans != 0.0)
+            t = fabs(yk / ans);
+        else
+            t = 1.0;
+    } while (t > MACHEP);
+    k = xk;
+    t = n;
+    r = n - 1;
+    ans = (pow(z, r) * psi / Gamma(t)) - ans;
+    return (ans);
+}
+
+'''
+)
+
+
+expn = _core.create_ufunc(
+    'cupyx_scipy_special_expn',
+    ('ff->f', 'dd->d'),
+    'out0 = expn(in0, in1)',
+    preamble=expn_definition,
+    doc="""Generalized exponential integral En.
+
+    Parameters
+    ----------
+    n : cupy.ndarray
+        Non-negative integers
+    x : cupy.ndarray
+        Real argument
+
+    Returns
+    -------
+    y : scalar or cupy.ndarray
+        Values of the generalized exponential integral
+
+    See Also
+    --------
+    :func:`scipy.special.expn`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_gamma.py b/cupyx/scipy/special/_gamma.py
new file mode 100644
index 0000000..73a9cae
--- /dev/null
+++ b/cupyx/scipy/special/_gamma.py
@@ -0,0 +1,220 @@
+from cupy import _core
+from cupyx.scipy.special._loggamma import loggamma_definition
+
+
+_gamma_body = """
+
+    if (isinf(in0) && in0 < 0) {
+        out0 = -1.0 / 0.0;
+    } else if (in0 < 0. && in0 == floor(in0)) {
+        out0 = 1.0 / 0.0;
+    } else {
+        out0 = tgamma(in0);
+    }
+"""
+
+# Also define a standalone Gamma device function for internal use in other code
+# like beta, betaln, etc.
+gamma_definition = f"""
+
+__noinline__ __device__ double Gamma(double in0)
+{{
+    double out0;
+    {_gamma_body}
+    return out0;
+}}
+"""
+
+cgamma_definition = loggamma_definition + """
+
+// Compute Gamma(z) using loggamma.
+
+__device__ complex<double> cgamma(complex<double> z)
+{
+    if ((z.real() <= 0.0) && (z == floor(z.real()))){
+        // Poles
+        return complex<double>(CUDART_NAN, CUDART_NAN);
+    }
+    return exp(loggamma(z));
+}
+"""
+
+
+gamma = _core.create_ufunc(
+    'cupyx_scipy_gamma',
+    (
+        'f->f',
+        'd->d',
+        ('F->F', 'out0 = out0_type(cgamma(in0))'),
+        ('D->D', 'out0 = cgamma(in0)')
+    ),
+    _gamma_body,
+    preamble=cgamma_definition,
+    doc="""Gamma function.
+
+    Args:
+        z (cupy.ndarray): The input of gamma function.
+
+    Returns:
+        cupy.ndarray: Computed value of gamma function.
+
+    .. seealso:: :func:`scipy.special.gamma`
+
+    """)
+
+# Kernel fusion involves preambles concatenating so
+# if there are several kernels that depend on the same cpp function,
+# compiler throws an error because of duplicates
+# ifndef allows to fixes it as compiler throw all duplicates away
+chbevl_implementation = """
+#ifndef chbevl_defined
+#define chbevl_defined
+template<typename T>
+__device__ T chbevl(T x, T array[], int n)
+{
+    T b0, b1, b2, *p;
+    int i;
+
+    p = array;
+    b0 = *p++;
+    b1 = 0.0;
+    i = n - 1;
+
+    do {
+        b2 = b1;
+        b1 = b0;
+        b0 = x * b1 - b2 + *p++;
+    }
+    while (--i);
+
+    return (0.5 * (b0 - b2));
+}
+#endif
+
+"""
+
+
+rgamma_implementation = chbevl_implementation + """
+
+#include <cupy/math_constants.h>
+
+#define MAXLOG 7.09782712893383996732E2
+#define NPY_PI 3.141592653589793238462643383279502884
+
+
+/* Chebyshev coefficients for reciprocal Gamma function
+ * in interval 0 to 1.  Function is 1/(x Gamma(x)) - 1
+ */
+
+__device__ double R[] = {
+    3.13173458231230000000E-17,
+    -6.70718606477908000000E-16,
+    2.20039078172259550000E-15,
+    2.47691630348254132600E-13,
+    -6.60074100411295197440E-12,
+    5.13850186324226978840E-11,
+    1.08965386454418662084E-9,
+    -3.33964630686836942556E-8,
+    2.68975996440595483619E-7,
+    2.96001177518801696639E-6,
+    -8.04814124978471142852E-5,
+    4.16609138709688864714E-4,
+    5.06579864028608725080E-3,
+    -6.41925436109158228810E-2,
+    -4.98558728684003594785E-3,
+    1.27546015610523951063E-1
+};
+
+
+/*
+ *     Reciprocal Gamma function
+ */
+__device__ double rgamma(double x)
+{
+    double w, y, z;
+    int sign;
+
+    if (x > 34.84425627277176174) {
+        return exp(-lgamma(x));
+    }
+    if (x < -34.034) {
+        w = -x;
+        z = sinpi(w);
+        if (z == 0.0) {
+            return 0.0;
+        }
+        if (z < 0.0) {
+            sign = 1;
+            z = -z;
+        } else {
+            sign = -1;
+        }
+        y = log(w * z) - log(NPY_PI) + lgamma(w);
+        if (y < -MAXLOG) {
+            return sign * 0.0;
+        }
+        if (y > MAXLOG) {
+            return sign * CUDART_INF;
+        }
+        return sign * exp(y);
+    }
+    z = 1.0;
+    w = x;
+    while (w > 1.0) {       /* Downward recurrence */
+        w -= 1.0;
+        z *= w;
+    }
+    while (w < 0.0) {       /* Upward recurrence */
+        z /= w;
+        w += 1.0;
+    }
+    if (w == 0.0) {     /* Nonpositive integer */
+        return 0.0;
+    }
+    if (w == 1.0) {      /* Other integer */
+        return 1.0 / z;
+    }
+    y = w * (1.0 + chbevl(4.0 * w - 2.0, R, 16)) / z;
+    return y;
+}
+
+"""
+
+
+crgamma_implementation = loggamma_definition + """
+
+// Compute 1/Gamma(z) using loggamma
+
+__device__ complex<double> crgamma(complex<double> z)
+{
+
+    if ((z.real() <= 0) && (z == floor(z.real()))){
+        // Zeros at 0, -1, -2, ...
+        return complex<double>(0.0, 0.0);
+    }
+    return exp(-loggamma(z));  // complex exp via Thrust
+}
+"""
+
+
+rgamma = _core.create_ufunc(
+    'cupyx_scipy_rgamma',
+    (
+        'f->f',
+        'd->d',
+        ('F->F', 'out0 = out0_type(crgamma(in0))'),
+        ('D->D', 'out0 = crgamma(in0)')
+    ),
+    'out0 = out0_type(rgamma(in0))',
+    preamble=rgamma_implementation + crgamma_implementation,
+    doc="""Reciprocal gamma function.
+
+    Args:
+        z (cupy.ndarray): The input to the rgamma function.
+
+    Returns:
+        cupy.ndarray: Computed value of the rgamma function.
+
+    .. seealso:: :func:`scipy.special.rgamma`
+
+    """)
diff --git a/cupyx/scipy/special/_gammainc.py b/cupyx/scipy/special/_gammainc.py
new file mode 100644
index 0000000..f52b773
--- /dev/null
+++ b/cupyx/scipy/special/_gammainc.py
@@ -0,0 +1,1144 @@
+"""
+The source code here is an adaptation with minimal changes from the following
+files in SciPy's bundled Cephes library:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/const.c
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/igam.c
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/igam.h
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/igami.c
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/lanczos.c
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/lanczos.h
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/mconf.h
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/polevl.h
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/unity.c
+
+Cephes Math Library Release 2.0:  April, 1987
+Copyright 1984, 1987 by Stephen L. Moshier
+Direct inquiries to 30 Frost Street, Cambridge, MA 02140
+"""
+
+from cupy import _core
+
+from cupyx.scipy.special._digamma import polevl_definition
+from cupyx.scipy.special._gamma import gamma_definition
+from cupyx.scipy.special._zeta import zeta_definition
+
+_zeta_c = zeta_definition
+# comment out duplicate MACHEP definition
+_zeta_c = _zeta_c.replace(
+    "__constant__ double MACHEP", "// __constant__ double MACHEP"
+)
+
+
+# TODO: finish
+
+# See the following on error states:
+#    docs.scipy.org/doc/scipy/reference/generated/scipy.special.errstate.html
+#    scipy/special/sf_error.h
+
+# TODO: which value for MACHEP, MAXLOG from const.c to use?
+
+_misc_preamble = """
+
+// include for CUDART_NAN, CUDART_INF
+#include <cupy/math_constants.h>
+
+// defines from /scipy/special/cephes/const.c
+__constant__ double MACHEP = 1.11022302462515654042E-16;
+__constant__ double MAXLOG = 7.09782712893383996732E2;
+
+// defines from numpy/core/include/numpy/npy_math.h
+#define NPY_EULER     0.577215664901532860606512090082402431  /* Euler constant */
+#define NPY_PI        3.141592653589793238462643383279502884  /* pi */
+#define NPY_SQRT2     1.414213562373095048801688724209698079  /* sqrt(2) */
+#define NPY_SQRT1_2   0.707106781186547524400844362104849039  /* 1/sqrt(2) */
+
+// from /scipy/special/cephes/mconf.h
+#ifdef MAXITER
+#undef MAXITER
+#endif
+
+#define MAXITER 2000
+#define IGAM 1
+#define IGAMC 0
+#define SMALL 20
+#define LARGE 200
+#define SMALLRATIO 0.3
+#define LARGERATIO 4.5
+
+__constant__ double big = 4.503599627370496e15;
+__constant__ double biginv = 2.22044604925031308085e-16;
+
+"""
+
+
+# Note: p1evl -> followed the template style used for polevl in digamma.py
+p1evl_definition = """
+template<int N> static __device__ double p1evl(double x, const double coef[])
+{
+    double ans;
+    const double *p;
+    int i;
+
+    p = coef;
+    ans = x + *p++;
+
+    for (i = 0; i < N - 1; ++i){
+        ans = ans * x + *p++;
+    }
+
+    return ans;
+}
+"""
+
+lgam_definition = """
+// define lgam, using CUDA's lgamma as done for cupyx.scipy.special.gammaln
+__device__ double lgam(double x)
+{
+    if (isinf(x) && x < 0) {
+        return -1.0 / 0.0;
+    } else {
+        return lgamma(x);
+    }
+}
+"""
+
+# lgam1p from unity.c
+_unity_c_partial = (
+    _zeta_c
+    + lgam_definition
+    + polevl_definition
+    + p1evl_definition
+    + """
+// part of cephes/unity.c
+
+
+/* log(1 + x) - x */
+#ifdef __HIP_DEVICE_COMPILE__
+// For some reason ROCm 4.3 crashes with the noinline in this function
+__device__ double log1pmx(double x)
+#else
+__noinline__ __device__ double log1pmx(double x)
+#endif
+{
+    if (fabs(x) < 0.5) {
+        int n;
+        double xfac = x;
+        double term;
+        double res = 0;
+
+        for(n = 2; n < MAXITER; n++) {
+            xfac *= -x;
+            term = xfac / n;
+            res += term;
+            if (fabs(term) < MACHEP * fabs(res)) {
+                break;
+            }
+        }
+        return res;
+    }
+    else {
+        return log1p(x) - x;
+    }
+}
+
+
+/* Compute lgam(x + 1) around x = 0 using its Taylor series. */
+__noinline__ __device__ double lgam1p_taylor(double x)
+{
+    int n;
+    double xfac, coeff, res;
+
+    if (x == 0) {
+        return 0;
+    }
+    res = -NPY_EULER * x;
+    xfac = -x;
+    for (n = 2; n < 42; n++) {
+        xfac *= -x;
+        coeff = zeta(n, 1) * xfac / n;
+        res += coeff;
+        if (fabs(coeff) < MACHEP * fabs(res)) {
+                break;
+        }
+    }
+
+    return res;
+}
+
+
+/* Compute lgam(x + 1). */
+__noinline__ __device__ double lgam1p(double x)
+{
+    if (fabs(x) <= 0.5) {
+        return lgam1p_taylor(x);
+    } else if (fabs(x - 1) < 0.5) {
+        return log(x) + lgam1p_taylor(x - 1);
+    } else {
+        return lgam(x + 1);
+    }
+}
+
+
+"""
+)
+
+ratevl_definition = """
+// from scipy/special/cephes/polevl.h
+
+/* Evaluate a rational function. See [1]. */
+
+static __device__ double ratevl(double x, const double num[], int M,
+                                const double denom[], int N)
+{
+    int i, dir;
+    double y, num_ans, denom_ans;
+    double absx = fabs(x);
+    const double *p;
+
+    if (absx > 1) {
+        /* Evaluate as a polynomial in 1/x. */
+        dir = -1;
+        p = num + M;
+        y = 1 / x;
+    } else {
+        dir = 1;
+        p = num;
+        y = x;
+    }
+
+    /* Evaluate the numerator */
+    num_ans = *p;
+    p += dir;
+    for (i = 1; i <= M; i++) {
+        num_ans = num_ans * y + *p;
+        p += dir;
+    }
+
+    /* Evaluate the denominator */
+    if (absx > 1) {
+        p = denom + N;
+    } else {
+        p = denom;
+    }
+
+    denom_ans = *p;
+    p += dir;
+    for (i = 1; i <= N; i++) {
+        denom_ans = denom_ans * y + *p;
+        p += dir;
+    }
+
+    if (absx > 1) {
+        i = N - M;
+        return pow(x, (double)i) * num_ans / denom_ans;
+    } else {
+        return num_ans / denom_ans;
+    }
+}
+"""
+
+_lanczos_h = """
+// from scipy/special/cephes/lanczos.h
+
+__constant__ double lanczos_num[13] = {
+    2.506628274631000270164908177133837338626,
+    210.8242777515793458725097339207133627117,
+    8071.672002365816210638002902272250613822,
+    186056.2653952234950402949897160456992822,
+    2876370.628935372441225409051620849613599,
+    31426415.58540019438061423162831820536287,
+    248874557.8620541565114603864132294232163,
+    1439720407.311721673663223072794912393972,
+    6039542586.35202800506429164430729792107,
+    17921034426.03720969991975575445893111267,
+    35711959237.35566804944018545154716670596,
+    42919803642.64909876895789904700198885093,
+    23531376880.41075968857200767445163675473
+};
+
+__constant__ double lanczos_denom[13] = {
+    1,
+    66,
+    1925,
+    32670,
+    357423,
+    2637558,
+    13339535,
+    45995730,
+    105258076,
+    150917976,
+    120543840,
+    39916800,
+    0
+};
+
+__constant__ double lanczos_sum_expg_scaled_num[13] = {
+    0.006061842346248906525783753964555936883222,
+    0.5098416655656676188125178644804694509993,
+    19.51992788247617482847860966235652136208,
+    449.9445569063168119446858607650988409623,
+    6955.999602515376140356310115515198987526,
+    75999.29304014542649875303443598909137092,
+    601859.6171681098786670226533699352302507,
+    3481712.15498064590882071018964774556468,
+    14605578.08768506808414169982791359218571,
+    43338889.32467613834773723740590533316085,
+    86363131.28813859145546927288977868422342,
+    103794043.1163445451906271053616070238554,
+    56906521.91347156388090791033559122686859
+};
+
+__constant__ double lanczos_sum_expg_scaled_denom[13] = {
+    1,
+    66,
+    1925,
+    32670,
+    357423,
+    2637558,
+    13339535,
+    45995730,
+    105258076,
+    150917976,
+    120543840,
+    39916800,
+    0
+};
+
+__constant__ double lanczos_sum_near_1_d[12] = {
+    0.3394643171893132535170101292240837927725e-9,
+    -0.2499505151487868335680273909354071938387e-8,
+    0.8690926181038057039526127422002498960172e-8,
+    -0.1933117898880828348692541394841204288047e-7,
+    0.3075580174791348492737947340039992829546e-7,
+    -0.2752907702903126466004207345038327818713e-7,
+    -0.1515973019871092388943437623825208095123e-5,
+    0.004785200610085071473880915854204301886437,
+    -0.1993758927614728757314233026257810172008,
+    1.483082862367253753040442933770164111678,
+    -3.327150580651624233553677113928873034916,
+    2.208709979316623790862569924861841433016
+};
+
+__constant__ double lanczos_sum_near_2_d[12] = {
+    0.1009141566987569892221439918230042368112e-8,
+    -0.7430396708998719707642735577238449585822e-8,
+    0.2583592566524439230844378948704262291927e-7,
+    -0.5746670642147041587497159649318454348117e-7,
+    0.9142922068165324132060550591210267992072e-7,
+    -0.8183698410724358930823737982119474130069e-7,
+    -0.4506604409707170077136555010018549819192e-5,
+    0.01422519127192419234315002746252160965831,
+    -0.5926941084905061794445733628891024027949,
+    4.408830289125943377923077727900630927902,
+    -9.8907772644920670589288081640128194231,
+    6.565936202082889535528455955485877361223
+};
+
+__constant__ double lanczos_g = 6.024680040776729583740234375;
+"""
+
+_lanczos_preamble = (
+    ratevl_definition
+    + _lanczos_h
+    + """
+
+// from scipy/special/cephes/lanczos.c
+
+__device__ double lanczos_sum(double x)
+{
+    return ratevl(x, lanczos_num,
+          sizeof(lanczos_num) / sizeof(lanczos_num[0]) - 1,
+          lanczos_denom,
+          sizeof(lanczos_denom) / sizeof(lanczos_denom[0]) - 1);
+}
+
+
+__device__ double lanczos_sum_expg_scaled(double x)
+{
+    return ratevl(x, lanczos_sum_expg_scaled_num,
+          sizeof(lanczos_sum_expg_scaled_num)
+              / sizeof(lanczos_sum_expg_scaled_num[0]) - 1,
+          lanczos_sum_expg_scaled_denom,
+          sizeof(lanczos_sum_expg_scaled_denom)
+              / sizeof(lanczos_sum_expg_scaled_denom[0]) - 1);
+}
+
+
+__device__ double lanczos_sum_near_1(double dx)
+{
+    double result = 0;
+    unsigned k;
+
+    for (k = 1;
+         k <= sizeof(lanczos_sum_near_1_d) / sizeof(lanczos_sum_near_1_d[0]);
+         ++k) {
+    result += (-lanczos_sum_near_1_d[k-1]*dx)/(k*dx + k*k);
+    }
+    return result;
+}
+
+
+__device__ double lanczos_sum_near_2(double dx)
+{
+    double result = 0;
+    double x = dx + 2;
+    unsigned k;
+
+    for(k = 1;
+        k <= sizeof(lanczos_sum_near_2_d) / sizeof(lanczos_sum_near_2_d[0]);
+        ++k) {
+    result += (-lanczos_sum_near_2_d[k-1]*dx)/(x + k*x + k*k - 1);
+    }
+    return result;
+}
+
+"""
+)
+
+# add predefined array from igam.h
+_igam_h = """
+
+// from scipy/special/cephes/igam.h
+
+#define d_K 25
+#define d_N 25
+
+__device__ static const double d[d_K][d_N] =
+{{-3.3333333333333333e-1, 8.3333333333333333e-2, -1.4814814814814815e-2, 1.1574074074074074e-3, 3.527336860670194e-4, -1.7875514403292181e-4, 3.9192631785224378e-5, -2.1854485106799922e-6, -1.85406221071516e-6, 8.296711340953086e-7, -1.7665952736826079e-7, 6.7078535434014986e-9, 1.0261809784240308e-8, -4.3820360184533532e-9, 9.1476995822367902e-10, -2.551419399494625e-11, -5.8307721325504251e-11, 2.4361948020667416e-11, -5.0276692801141756e-12, 1.1004392031956135e-13, 3.3717632624009854e-13, -1.3923887224181621e-13, 2.8534893807047443e-14, -5.1391118342425726e-16, -1.9752288294349443e-15},
+{-1.8518518518518519e-3, -3.4722222222222222e-3, 2.6455026455026455e-3, -9.9022633744855967e-4, 2.0576131687242798e-4, -4.0187757201646091e-7, -1.8098550334489978e-5, 7.6491609160811101e-6, -1.6120900894563446e-6, 4.6471278028074343e-9, 1.378633446915721e-7, -5.752545603517705e-8, 1.1951628599778147e-8, -1.7543241719747648e-11, -1.0091543710600413e-9, 4.1627929918425826e-10, -8.5639070264929806e-11, 6.0672151016047586e-14, 7.1624989648114854e-12, -2.9331866437714371e-12, 5.9966963656836887e-13, -2.1671786527323314e-16, -4.9783399723692616e-14, 2.0291628823713425e-14, -4.13125571381061e-15},
+{4.1335978835978836e-3, -2.6813271604938272e-3, 7.7160493827160494e-4, 2.0093878600823045e-6, -1.0736653226365161e-4, 5.2923448829120125e-5, -1.2760635188618728e-5, 3.4235787340961381e-8, 1.3721957309062933e-6, -6.298992138380055e-7, 1.4280614206064242e-7, -2.0477098421990866e-10, -1.4092529910867521e-8, 6.228974084922022e-9, -1.3670488396617113e-9, 9.4283561590146782e-13, 1.2872252400089318e-10, -5.5645956134363321e-11, 1.1975935546366981e-11, -4.1689782251838635e-15, -1.0940640427884594e-12, 4.6622399463901357e-13, -9.905105763906906e-14, 1.8931876768373515e-17, 8.8592218725911273e-15},
+{6.4943415637860082e-4, 2.2947209362139918e-4, -4.6918949439525571e-4, 2.6772063206283885e-4, -7.5618016718839764e-5, -2.3965051138672967e-7, 1.1082654115347302e-5, -5.6749528269915966e-6, 1.4230900732435884e-6, -2.7861080291528142e-11, -1.6958404091930277e-7, 8.0994649053880824e-8, -1.9111168485973654e-8, 2.3928620439808118e-12, 2.0620131815488798e-9, -9.4604966618551322e-10, 2.1541049775774908e-10, -1.388823336813903e-14, -2.1894761681963939e-11, 9.7909989511716851e-12, -2.1782191880180962e-12, 6.2088195734079014e-17, 2.126978363279737e-13, -9.3446887915174333e-14, 2.0453671226782849e-14},
+{-8.618882909167117e-4, 7.8403922172006663e-4, -2.9907248030319018e-4, -1.4638452578843418e-6, 6.6414982154651222e-5, -3.9683650471794347e-5, 1.1375726970678419e-5, 2.5074972262375328e-10, -1.6954149536558306e-6, 8.9075075322053097e-7, -2.2929348340008049e-7, 2.956794137544049e-11, 2.8865829742708784e-8, -1.4189739437803219e-8, 3.4463580499464897e-9, -2.3024517174528067e-13, -3.9409233028046405e-10, 1.8602338968504502e-10, -4.356323005056618e-11, 1.2786001016296231e-15, 4.6792750266579195e-12, -2.1492464706134829e-12, 4.9088156148096522e-13, -6.3385914848915603e-18, -5.0453320690800944e-14},
+{-3.3679855336635815e-4, -6.9728137583658578e-5, 2.7727532449593921e-4, -1.9932570516188848e-4, 6.7977804779372078e-5, 1.419062920643967e-7, -1.3594048189768693e-5, 8.0184702563342015e-6, -2.2914811765080952e-6, -3.252473551298454e-10, 3.4652846491085265e-7, -1.8447187191171343e-7, 4.8240967037894181e-8, -1.7989466721743515e-14, -6.3061945000135234e-9, 3.1624176287745679e-9, -7.8409242536974293e-10, 5.1926791652540407e-15, 9.3589442423067836e-11, -4.5134262161632782e-11, 1.0799129993116827e-11, -3.661886712685252e-17, -1.210902069055155e-12, 5.6807435849905643e-13, -1.3249659916340829e-13},
+{5.3130793646399222e-4, -5.9216643735369388e-4, 2.7087820967180448e-4, 7.9023532326603279e-7, -8.1539693675619688e-5, 5.6116827531062497e-5, -1.8329116582843376e-5, -3.0796134506033048e-9, 3.4651553688036091e-6, -2.0291327396058604e-6, 5.7887928631490037e-7, 2.338630673826657e-13, -8.8286007463304835e-8, 4.7435958880408128e-8, -1.2545415020710382e-8, 8.6496488580102925e-14, 1.6846058979264063e-9, -8.5754928235775947e-10, 2.1598224929232125e-10, -7.6132305204761539e-16, -2.6639822008536144e-11, 1.3065700536611057e-11, -3.1799163902367977e-12, 4.7109761213674315e-18, 3.6902800842763467e-13},
+{3.4436760689237767e-4, 5.1717909082605922e-5, -3.3493161081142236e-4, 2.812695154763237e-4, -1.0976582244684731e-4, -1.2741009095484485e-7, 2.7744451511563644e-5, -1.8263488805711333e-5, 5.7876949497350524e-6, 4.9387589339362704e-10, -1.0595367014026043e-6, 6.1667143761104075e-7, -1.7562973359060462e-7, -1.2974473287015439e-12, 2.695423606288966e-8, -1.4578352908731271e-8, 3.887645959386175e-9, -3.8810022510194121e-17, -5.3279941738772867e-10, 2.7437977643314845e-10, -6.9957960920705679e-11, 2.5899863874868481e-17, 8.8566890996696381e-12, -4.403168815871311e-12, 1.0865561947091654e-12},
+{-6.5262391859530942e-4, 8.3949872067208728e-4, -4.3829709854172101e-4, -6.969091458420552e-7, 1.6644846642067548e-4, -1.2783517679769219e-4, 4.6299532636913043e-5, 4.5579098679227077e-9, -1.0595271125805195e-5, 6.7833429048651666e-6, -2.1075476666258804e-6, -1.7213731432817145e-11, 3.7735877416110979e-7, -2.1867506700122867e-7, 6.2202288040189269e-8, 6.5977038267330006e-16, -9.5903864974256858e-9, 5.2132144922808078e-9, -1.3991589583935709e-9, 5.382058999060575e-16, 1.9484714275467745e-10, -1.0127287556389682e-10, 2.6077347197254926e-11, -5.0904186999932993e-18, -3.3721464474854592e-12},
+{-5.9676129019274625e-4, -7.2048954160200106e-5, 6.7823088376673284e-4, -6.4014752602627585e-4, 2.7750107634328704e-4, 1.8197008380465151e-7, -8.4795071170685032e-5, 6.105192082501531e-5, -2.1073920183404862e-5, -8.8585890141255994e-10, 4.5284535953805377e-6, -2.8427815022504408e-6, 8.7082341778646412e-7, 3.6886101871706965e-12, -1.5344695190702061e-7, 8.862466778790695e-8, -2.5184812301826817e-8, -1.0225912098215092e-14, 3.8969470758154777e-9, -2.1267304792235635e-9, 5.7370135528051385e-10, -1.887749850169741e-19, -8.0931538694657866e-11, 4.2382723283449199e-11, -1.1002224534207726e-11},
+{1.3324454494800656e-3, -1.9144384985654775e-3, 1.1089369134596637e-3, 9.932404122642299e-7, -5.0874501293093199e-4, 4.2735056665392884e-4, -1.6858853767910799e-4, -8.1301893922784998e-9, 4.5284402370562147e-5, -3.127053674781734e-5, 1.044986828530338e-5, 4.8435226265680926e-11, -2.1482565873456258e-6, 1.329369701097492e-6, -4.0295693092101029e-7, -1.7567877666323291e-13, 7.0145043163668257e-8, -4.040787734999483e-8, 1.1474026743371963e-8, 3.9642746853563325e-18, -1.7804938269892714e-9, 9.7480262548731646e-10, -2.6405338676507616e-10, 5.794875163403742e-18, 3.7647749553543836e-11},
+{1.579727660730835e-3, 1.6251626278391582e-4, -2.0633421035543276e-3, 2.1389686185689098e-3, -1.0108559391263003e-3, -3.9912705529919201e-7, 3.6235025084764691e-4, -2.8143901463712154e-4, 1.0449513336495887e-4, 2.1211418491830297e-9, -2.5779417251947842e-5, 1.7281818956040463e-5, -5.6413773872904282e-6, -1.1024320105776174e-11, 1.1223224418895175e-6, -6.8693396379526735e-7, 2.0653236975414887e-7, 4.6714772409838506e-14, -3.5609886164949055e-8, 2.0470855345905963e-8, -5.8091738633283358e-9, -1.332821287582869e-16, 9.0354604391335133e-10, -4.9598782517330834e-10, 1.3481607129399749e-10},
+{-4.0725121195140166e-3, 6.4033628338080698e-3, -4.0410161081676618e-3, -2.183732802866233e-6, 2.1740441801254639e-3, -1.9700440518418892e-3, 8.3595469747962458e-4, 1.9445447567109655e-8, -2.5779387120421696e-4, 1.9009987368139304e-4, -6.7696499937438965e-5, -1.4440629666426572e-10, 1.5712512518742269e-5, -1.0304008744776893e-5, 3.304517767401387e-6, 7.9829760242325709e-13, -6.4097794149313004e-7, 3.8894624761300056e-7, -1.1618347644948869e-7, -2.816808630596451e-15, 1.9878012911297093e-8, -1.1407719956357511e-8, 3.2355857064185555e-9, 4.1759468293455945e-20, -5.0423112718105824e-10},
+{-5.9475779383993003e-3, -5.4016476789260452e-4, 8.7910413550767898e-3, -9.8576315587856125e-3, 5.0134695031021538e-3, 1.2807521786221875e-6, -2.0626019342754683e-3, 1.7109128573523058e-3, -6.7695312714133799e-4, -6.9011545676562133e-9, 1.8855128143995902e-4, -1.3395215663491969e-4, 4.6263183033528039e-5, 4.0034230613321351e-11, -1.0255652921494033e-5, 6.612086372797651e-6, -2.0913022027253008e-6, -2.0951775649603837e-13, 3.9756029041993247e-7, -2.3956211978815887e-7, 7.1182883382145864e-8, 8.925574873053455e-16, -1.2101547235064676e-8, 6.9350618248334386e-9, -1.9661464453856102e-9},
+{1.7402027787522711e-2, -2.9527880945699121e-2, 2.0045875571402799e-2, 7.0289515966903407e-6, -1.2375421071343148e-2, 1.1976293444235254e-2, -5.4156038466518525e-3, -6.3290893396418616e-8, 1.8855118129005065e-3, -1.473473274825001e-3, 5.5515810097708387e-4, 5.2406834412550662e-10, -1.4357913535784836e-4, 9.9181293224943297e-5, -3.3460834749478311e-5, -3.5755837291098993e-12, 7.1560851960630076e-6, -4.5516802628155526e-6, 1.4236576649271475e-6, 1.8803149082089664e-14, -2.6623403898929211e-7, 1.5950642189595716e-7, -4.7187514673841102e-8, -6.5107872958755177e-17, 7.9795091026746235e-9},
+{3.0249124160905891e-2, 2.4817436002649977e-3, -4.9939134373457022e-2, 5.9915643009307869e-2, -3.2483207601623391e-2, -5.7212968652103441e-6, 1.5085251778569354e-2, -1.3261324005088445e-2, 5.5515262632426148e-3, 3.0263182257030016e-8, -1.7229548406756723e-3, 1.2893570099929637e-3, -4.6845138348319876e-4, -1.830259937893045e-10, 1.1449739014822654e-4, -7.7378565221244477e-5, 2.5625836246985201e-5, 1.0766165333192814e-12, -5.3246809282422621e-6, 3.349634863064464e-6, -1.0381253128684018e-6, -5.608909920621128e-15, 1.9150821930676591e-7, -1.1418365800203486e-7, 3.3654425209171788e-8},
+{-9.9051020880159045e-2, 1.7954011706123486e-1, -1.2989606383463778e-1, -3.1478872752284357e-5, 9.0510635276848131e-2, -9.2828824411184397e-2, 4.4412112839877808e-2, 2.7779236316835888e-7, -1.7229543805449697e-2, 1.4182925050891573e-2, -5.6214161633747336e-3, -2.39598509186381e-9, 1.6029634366079908e-3, -1.1606784674435773e-3, 4.1001337768153873e-4, 1.8365800754090661e-11, -9.5844256563655903e-5, 6.3643062337764708e-5, -2.076250624489065e-5, -1.1806020912804483e-13, 4.2131808239120649e-6, -2.6262241337012467e-6, 8.0770620494930662e-7, 6.0125912123632725e-16, -1.4729737374018841e-7},
+{-1.9994542198219728e-1, -1.5056113040026424e-2, 3.6470239469348489e-1, -4.6435192311733545e-1, 2.6640934719197893e-1, 3.4038266027147191e-5, -1.3784338709329624e-1, 1.276467178337056e-1, -5.6213828755200985e-2, -1.753150885483011e-7, 1.9235592956768113e-2, -1.5088821281095315e-2, 5.7401854451350123e-3, 1.0622382710310225e-9, -1.5335082692563998e-3, 1.0819320643228214e-3, -3.7372510193945659e-4, -6.6170909729031985e-12, 8.4263617380909628e-5, -5.5150706827483479e-5, 1.7769536448348069e-5, 3.8827923210205533e-14, -3.53513697488768e-6, 2.1865832130045269e-6, -6.6812849447625594e-7},
+{7.2438608504029431e-1, -1.3918010932653375, 1.0654143352413968, 1.876173868950258e-4, -8.2705501176152696e-1, 8.9352433347828414e-1, -4.4971003995291339e-1, -1.6107401567546652e-6, 1.9235590165271091e-1, -1.6597702160042609e-1, 6.8882222681814333e-2, 1.3910091724608687e-8, -2.146911561508663e-2, 1.6228980898865892e-2, -5.9796016172584256e-3, -1.1287469112826745e-10, 1.5167451119784857e-3, -1.0478634293553899e-3, 3.5539072889126421e-4, 8.1704322111801517e-13, -7.7773013442452395e-5, 5.0291413897007722e-5, -1.6035083867000518e-5, 1.2469354315487605e-14, 3.1369106244517615e-6},
+{1.6668949727276811, 1.165462765994632e-1, -3.3288393225018906, 4.4692325482864037, -2.6977693045875807, -2.600667859891061e-4, 1.5389017615694539, -1.4937962361134612, 6.8881964633233148e-1, 1.3077482004552385e-6, -2.5762963325596288e-1, 2.1097676102125449e-1, -8.3714408359219882e-2, -7.7920428881354753e-9, 2.4267923064833599e-2, -1.7813678334552311e-2, 6.3970330388900056e-3, 4.9430807090480523e-11, -1.5554602758465635e-3, 1.0561196919903214e-3, -3.5277184460472902e-4, 9.3002334645022459e-14, 7.5285855026557172e-5, -4.8186515569156351e-5, 1.5227271505597605e-5},
+{-6.6188298861372935, 1.3397985455142589e+1, -1.0789350606845146e+1, -1.4352254537875018e-3, 9.2333694596189809, -1.0456552819547769e+1, 5.5105526029033471, 1.2024439690716742e-5, -2.5762961164755816, 2.3207442745387179, -1.0045728797216284, -1.0207833290021914e-7, 3.3975092171169466e-1, -2.6720517450757468e-1, 1.0235252851562706e-1, 8.4329730484871625e-10, -2.7998284958442595e-2, 2.0066274144976813e-2, -7.0554368915086242e-3, 1.9402238183698188e-12, 1.6562888105449611e-3, -1.1082898580743683e-3, 3.654545161310169e-4, -5.1290032026971794e-11, -7.6340103696869031e-5},
+{-1.7112706061976095e+1, -1.1208044642899116, 3.7131966511885444e+1, -5.2298271025348962e+1, 3.3058589696624618e+1, 2.4791298976200222e-3, -2.061089403411526e+1, 2.088672775145582e+1, -1.0045703956517752e+1, -1.2238783449063012e-5, 4.0770134274221141, -3.473667358470195, 1.4329352617312006, 7.1359914411879712e-8, -4.4797257159115612e-1, 3.4112666080644461e-1, -1.2699786326594923e-1, -2.8953677269081528e-10, 3.3125776278259863e-2, -2.3274087021036101e-2, 8.0399993503648882e-3, -1.177805216235265e-9, -1.8321624891071668e-3, 1.2108282933588665e-3, -3.9479941246822517e-4},
+{7.389033153567425e+1, -1.5680141270402273e+2, 1.322177542759164e+2, 1.3692876877324546e-2, -1.2366496885920151e+2, 1.4620689391062729e+2, -8.0365587724865346e+1, -1.1259851148881298e-4, 4.0770132196179938e+1, -3.8210340013273034e+1, 1.719522294277362e+1, 9.3519707955168356e-7, -6.2716159907747034, 5.1168999071852637, -2.0319658112299095, -4.9507215582761543e-9, 5.9626397294332597e-1, -4.4220765337238094e-1, 1.6079998700166273e-1, -2.4733786203223402e-8, -4.0307574759979762e-2, 2.7849050747097869e-2, -9.4751858992054221e-3, 6.419922235909132e-6, 2.1250180774699461e-3},
+{2.1216837098382522e+2, 1.3107863022633868e+1, -4.9698285932871748e+2, 7.3121595266969204e+2, -4.8213821720890847e+2, -2.8817248692894889e-2, 3.2616720302947102e+2, -3.4389340280087117e+2, 1.7195193870816232e+2, 1.4038077378096158e-4, -7.52594195897599e+1, 6.651969984520934e+1, -2.8447519748152462e+1, -7.613702615875391e-7, 9.5402237105304373, -7.5175301113311376, 2.8943997568871961, -4.6612194999538201e-7, -8.0615149598794088e-1, 5.8483006570631029e-1, -2.0845408972964956e-1, 1.4765818959305817e-4, 5.1000433863753019e-2, -3.3066252141883665e-2, 1.5109265210467774e-2},
+{-9.8959643098322368e+2, 2.1925555360905233e+3, -1.9283586782723356e+3, -1.5925738122215253e-1, 1.9569985945919857e+3, -2.4072514765081556e+3, 1.3756149959336496e+3, 1.2920735237496668e-3, -7.525941715948055e+2, 7.3171668742208716e+2, -3.4137023466220065e+2, -9.9857390260608043e-6, 1.3356313181291573e+2, -1.1276295161252794e+2, 4.6310396098204458e+1, -7.9237387133614756e-6, -1.4510726927018646e+1, 1.1111771248100563e+1, -4.1690817945270892, 3.1008219800117808e-3, 1.1220095449981468, -7.6052379926149916e-1, 3.6262236505085254e-1, 2.216867741940747e-1, 4.8683443692930507e-1}};
+
+"""
+
+# add functions from igam.c
+_igam_preamble = (
+    _misc_preamble
+    + _unity_c_partial
+    + _lanczos_preamble
+    + _igam_h
+    + """
+
+// from scipy/special/cephes/igam.c
+
+/* Compute igam/igamc using DLMF 8.12.3/8.12.4. */
+__noinline__ __device__ double  asymptotic_series(double a, double x, int func)
+{
+    int k, n, sgn;
+    int maxpow = 0;
+    double lambda = x / a;
+    double sigma = (x - a) / a;
+    double eta, res, ck, ckterm, term, absterm;
+    double absoldterm = CUDART_INF;
+    double etapow[d_N] = {1};
+    double sum = 0;
+    double afac = 1;
+
+    if (func == IGAM) {
+        sgn = -1;
+    } else {
+        sgn = 1;
+    }
+
+    if (lambda > 1) {
+        eta = sqrt(-2 * log1pmx(sigma));
+    } else if (lambda < 1) {
+        eta = -sqrt(-2 * log1pmx(sigma));
+    } else {
+        eta = 0;
+    }
+    res = 0.5 * erfc(sgn * eta * sqrt(a / 2));
+
+    for (k = 0; k < d_K; k++) {
+        ck = d[k][0];
+        for (n = 1; n < d_N; n++) {
+            if (n > maxpow) {
+                etapow[n] = eta * etapow[n-1];
+                maxpow += 1;
+            }
+            ckterm = d[k][n]*etapow[n];
+            ck += ckterm;
+            if (fabs(ckterm) < MACHEP * fabs(ck)) {
+                break;
+            }
+        }
+        term = ck * afac;
+        absterm = fabs(term);
+        if (absterm > absoldterm) {
+            break;
+        }
+        sum += term;
+        if (absterm < MACHEP * fabs(sum)) {
+            break;
+        }
+        absoldterm = absterm;
+        afac /= a;
+    }
+    res += sgn * exp(-0.5 * a * eta * eta) * sum / sqrt(2 * NPY_PI * a);
+
+    return res;
+}
+
+
+/* Compute igamc using DLMF 8.7.3. This is related to the series in
+ * igam_series but extra care is taken to avoid cancellation.
+ */
+__noinline__ __device__ double  igamc_series(double a, double x)
+{
+    int n;
+    double fac = 1;
+    double sum = 0;
+    double term, logx;
+
+    for (n = 1; n < MAXITER; n++) {
+        fac *= -x / n;
+        term = fac / (a + n);
+        sum += term;
+        if (fabs(term) <= MACHEP * fabs(sum)) {
+            break;
+        }
+    }
+
+    logx = log(x);
+    term = -expm1(a * logx - lgam1p(a));
+    return term - exp(a * logx - lgam(a)) * sum;
+}
+
+
+/* Compute
+ *
+ * x^a * exp(-x) / gamma(a)
+ *
+ * corrected from (15) and (16) in [2] by replacing exp(x - a) with
+ * exp(a - x).
+ */
+__noinline__ __device__ double igam_fac(double a, double x)
+{
+    double ax, fac, res, num;
+
+    if (fabs(a - x) > 0.4 * fabs(a)) {
+        ax = a * log(x) - x - lgam(a);
+        if (ax < -MAXLOG) {
+            // sf_error("igam", SF_ERROR_UNDERFLOW, NULL);
+            return 0.0;
+        }
+        return exp(ax);
+    }
+
+    fac = a + lanczos_g - 0.5;
+    res = sqrt(fac / exp(1.)) / lanczos_sum_expg_scaled(a);
+
+    if ((a < 200) && (x < 200)) {
+        res *= exp(a - x) * pow(x / fac, a);
+    } else {
+        num = x - a - lanczos_g + 0.5;
+        res *= exp(a * log1pmx(num / fac) + x * (0.5 - lanczos_g) / fac);
+    }
+
+    return res;
+}
+
+
+/* Compute igamc using DLMF 8.9.2. */
+__noinline__ __device__ double  igamc_continued_fraction(double a, double x)
+{
+    int i;
+    double ans, ax, c, yc, r, t, y, z;
+    double pk, pkm1, pkm2, qk, qkm1, qkm2;
+
+    ax = igam_fac(a, x);
+    if (ax == 0.0) {
+        return 0.0;
+    }
+
+    /* continued fraction */
+    y = 1.0 - a;
+    z = x + y + 1.0;
+    c = 0.0;
+    pkm2 = 1.0;
+    qkm2 = x;
+    pkm1 = x + 1.0;
+    qkm1 = z * x;
+    ans = pkm1 / qkm1;
+
+    for (i = 0; i < MAXITER; i++) {
+        c += 1.0;
+        y += 1.0;
+        z += 2.0;
+        yc = y * c;
+        pk = pkm1 * z - pkm2 * yc;
+        qk = qkm1 * z - qkm2 * yc;
+        if (qk != 0) {
+            r = pk / qk;
+            t = fabs((ans - r) / r);
+            ans = r;
+        }
+        else
+            t = 1.0;
+        pkm2 = pkm1;
+        pkm1 = pk;
+        qkm2 = qkm1;
+        qkm1 = qk;
+        if (fabs(pk) > big) {
+            pkm2 *= biginv;
+            pkm1 *= biginv;
+            qkm2 *= biginv;
+            qkm1 *= biginv;
+        }
+        if (t <= MACHEP) {
+            break;
+        }
+    }
+
+    return (ans * ax);
+}
+
+
+/* Compute igam using DLMF 8.11.4. */
+__noinline__ __device__ double  igam_series(double a, double x)
+{
+    int i;
+    double ans, ax, c, r;
+
+    ax = igam_fac(a, x);
+    if (ax == 0.0) {
+        return 0.0;
+    }
+
+    /* power series */
+    r = a;
+    c = 1.0;
+    ans = 1.0;
+
+    for (i = 0; i < MAXITER; i++) {
+        r += 1.0;
+        c *= x / r;
+        ans += c;
+        if (c <= MACHEP * ans) {
+            break;
+        }
+    }
+
+    return (ans * ax / a);
+}
+
+__noinline__ __device__ double igamc(double a, double x)
+{
+    double absxma_a;
+
+    if (x < 0 || a < 0) {
+        // sf_error("gammaincc", SF_ERROR_DOMAIN, NULL);
+        return CUDART_NAN;
+    } else if (a == 0) {
+        if (x > 0) {
+            return 0;
+        } else {
+            return CUDART_NAN;
+        }
+    } else if (x == 0) {
+        return 1;
+    } else if (isinf(a)) {
+        if (isinf(x)) {
+            return CUDART_NAN;
+        }
+        return 1;
+    } else if (isinf(x)) {
+        return 0;
+    }
+
+    /* Asymptotic regime where a ~ x; see [2]. */
+    absxma_a = fabs(x - a) / a;
+    if ((a > SMALL) && (a < LARGE) && (absxma_a < SMALLRATIO)) {
+        return asymptotic_series(a, x, IGAMC);
+    } else if ((a > LARGE) && (absxma_a < LARGERATIO / sqrt(a))) {
+        return asymptotic_series(a, x, IGAMC);
+    }
+
+    /* Everywhere else; see [2]. */
+    if (x > 1.1) {
+        if (x < a) {
+            return 1.0 - igam_series(a, x);
+        } else {
+            return igamc_continued_fraction(a, x);
+        }
+    } else if (x <= 0.5) {
+        if (-0.4 / log(x) < a) {
+            return 1.0 - igam_series(a, x);
+        } else {
+            return igamc_series(a, x);
+        }
+    } else {
+        if (x * 1.1 < a) {
+            return 1.0 - igam_series(a, x);
+        } else {
+            return igamc_series(a, x);
+        }
+    }
+}
+
+
+__noinline__ __device__ double igam(double a, double x)
+{
+    double absxma_a;
+
+    if (x < 0 || a < 0) {
+        // sf_error("gammainc", SF_ERROR_DOMAIN, NULL);
+        return CUDART_NAN;
+    } else if (a == 0) {
+        if (x > 0) {
+            return 1;
+        } else {
+            return CUDART_NAN;
+        }
+    } else if (x == 0) {
+        /* Zero integration limit */
+        return 0;
+    } else if (isinf(a)) {
+        if (isinf(x)) {
+            return CUDART_NAN;
+        }
+        return 0;
+    } else if (isinf(x)) {
+        return 1;
+    }
+
+    /* Asymptotic regime where a ~ x; see [2]. */
+    absxma_a = fabs(x - a) / a;
+    if ((a > SMALL) && (a < LARGE) && (absxma_a < SMALLRATIO)) {
+        return asymptotic_series(a, x, IGAM);
+    } else if ((a > LARGE) && (absxma_a < LARGERATIO / sqrt(a))) {
+        return asymptotic_series(a, x, IGAM);
+    }
+
+    if ((x > 1.0) && (x > a)) {
+        return (1.0 - igamc(a, x));
+    }
+
+    return igam_series(a, x);
+}
+
+
+"""
+)
+
+
+_igami_preamble = (
+    _igam_preamble
+    + gamma_definition
+    + """
+
+// from scipy/special/cephes/igami.c
+
+__noinline__ __device__ double igamci(double a, double q);
+__noinline__ __device__ double igami(double a, double q);
+
+__noinline__ __device__ double find_inverse_s(double p, double q)
+{
+    /*
+     * Computation of the Incomplete Gamma Function Ratios and their Inverse
+     * ARMIDO R. DIDONATO and ALFRED H. MORRIS, JR.
+     * ACM Transactions on Mathematical Software, Vol. 12, No. 4,
+     * December 1986, Pages 377-393.
+     *
+     * See equation 32.
+     */
+    double s, t;
+    double a[4] = {0.213623493715853, 4.28342155967104,
+           11.6616720288968, 3.31125922108741};
+    double b[5] = {0.3611708101884203e-1, 1.27364489782223,
+           6.40691597760039, 6.61053765625462, 1};
+
+    if (p < 0.5) {
+        t = sqrt(-2 * log(p));
+    }
+    else {
+        t = sqrt(-2 * log(q));
+    }
+    s = t - polevl<3>(t, a) / polevl<4>(t, b);
+    if(p < 0.5)
+        s = -s;
+    return s;
+}
+
+
+__noinline__ __device__ double didonato_SN(double a, double x, unsigned N, double tolerance)
+{
+    /*
+     * Computation of the Incomplete Gamma Function Ratios and their Inverse
+     * ARMIDO R. DIDONATO and ALFRED H. MORRIS, JR.
+     * ACM Transactions on Mathematical Software, Vol. 12, No. 4,
+     * December 1986, Pages 377-393.
+     *
+     * See equation 34.
+     */
+    double sum = 1.0;
+
+    if (N >= 1) {
+    unsigned i;
+        double partial = x / (a + 1);
+
+        sum += partial;
+        for(i = 2; i <= N; ++i) {
+            partial *= x / (a + i);
+            sum += partial;
+            if(partial < tolerance) {
+                break;
+            }
+        }
+    }
+    return sum;
+}
+
+
+__noinline__ __device__ double find_inverse_gamma(double a, double p, double q)
+{
+    /*
+     * In order to understand what's going on here, you will
+     * need to refer to:
+     *
+     * Computation of the Incomplete Gamma Function Ratios and their Inverse
+     * ARMIDO R. DIDONATO and ALFRED H. MORRIS, JR.
+     * ACM Transactions on Mathematical Software, Vol. 12, No. 4,
+     * December 1986, Pages 377-393.
+     */
+    double result;
+
+    if (a == 1) {
+        if (q > 0.9) {
+            result = -log1p(-p);
+        }
+        else {
+            result = -log(q);
+        }
+    }
+    else if (a < 1) {
+        double g = Gamma(a);
+        double b = q * g;
+
+        if ((b > 0.6) || ((b >= 0.45) && (a >= 0.3))) {
+            /* DiDonato & Morris Eq 21:
+             *
+             * There is a slight variation from DiDonato and Morris here:
+             * the first form given here is unstable when p is close to 1,
+             * making it impossible to compute the inverse of Q(a,x) for small
+             * q. Fortunately the second form works perfectly well in this case.
+             */
+            double u;
+            if((b * q > 1e-8) && (q > 1e-5)) {
+                u = pow(p * g * a, 1 / a);
+            }
+            else {
+                u = exp((-q / a) - NPY_EULER);
+            }
+            result = u / (1 - (u / (a + 1)));
+        }
+        else if ((a < 0.3) && (b >= 0.35)) {
+            /* DiDonato & Morris Eq 22: */
+            double t = exp(-NPY_EULER - b);
+            double u = t * exp(t);
+            result = t * exp(u);
+        }
+        else if ((b > 0.15) || (a >= 0.3)) {
+            /* DiDonato & Morris Eq 23: */
+            double y = -log(b);
+            double u = y - (1 - a) * log(y);
+            result = y - (1 - a) * log(u) - log(1 + (1 - a) / (1 + u));
+        }
+        else if (b > 0.1) {
+            /* DiDonato & Morris Eq 24: */
+            double y = -log(b);
+            double u = y - (1 - a) * log(y);
+            result = y - (1 - a) * log(u)
+                - log((u * u + 2 * (3 - a) * u + (2 - a) * (3 - a))
+                      / (u * u + (5 - a) * u + 2));
+        }
+        else {
+            /* DiDonato & Morris Eq 25: */
+            double y = -log(b);
+            double c1 = (a - 1) * log(y);
+            double c1_2 = c1 * c1;
+            double c1_3 = c1_2 * c1;
+            double c1_4 = c1_2 * c1_2;
+            double a_2 = a * a;
+            double a_3 = a_2 * a;
+
+            double c2 = (a - 1) * (1 + c1);
+            double c3 = (a - 1) * (-(c1_2 / 2)
+                                   + (a - 2) * c1
+                                   + (3 * a - 5) / 2);
+            double c4 = (a - 1) * ((c1_3 / 3) - (3 * a - 5) * c1_2 / 2
+                                   + (a_2 - 6 * a + 7) * c1
+                                   + (11 * a_2 - 46 * a + 47) / 6);
+            double c5 = (a - 1)
+                        * (-(c1_4 / 4)
+                        + (11 * a - 17) * c1_3 / 6
+                        + (-3 * a_2 + 13 * a -13) * c1_2
+                        + (2 * a_3 - 25 * a_2 + 72 * a - 61) * c1 / 2
+                        + (25 * a_3 - 195 * a_2 + 477 * a - 379) / 12);
+
+            double y_2 = y * y;
+            double y_3 = y_2 * y;
+            double y_4 = y_2 * y_2;
+            result = y + c1 + (c2 / y) + (c3 / y_2) + (c4 / y_3) + (c5 / y_4);
+        }
+    }
+    else {
+        /* DiDonato and Morris Eq 31: */
+        double s = find_inverse_s(p, q);
+
+        double s_2 = s * s;
+        double s_3 = s_2 * s;
+        double s_4 = s_2 * s_2;
+        double s_5 = s_4 * s;
+        double ra = sqrt(a);
+
+        double w = a + s * ra + (s_2 - 1) / 3;
+        w += (s_3 - 7 * s) / (36 * ra);
+        w -= (3 * s_4 + 7 * s_2 - 16) / (810 * a);
+        w += (9 * s_5 + 256 * s_3 - 433 * s) / (38880 * a * ra);
+
+        if ((a >= 500) && (fabs(1 - w / a) < 1e-6)) {
+            result = w;
+        }
+        else if (p > 0.5) {
+            if (w < 3 * a) {
+                result = w;
+            }
+            else {
+                double D = fmax(2, a * (a - 1));
+                double lg = lgam(a);
+                double lb = log(q) + lg;
+                if (lb < -D * 2.3) {
+                    /* DiDonato and Morris Eq 25: */
+                    double y = -lb;
+                    double c1 = (a - 1) * log(y);
+                    double c1_2 = c1 * c1;
+                    double c1_3 = c1_2 * c1;
+                    double c1_4 = c1_2 * c1_2;
+                    double a_2 = a * a;
+                    double a_3 = a_2 * a;
+
+                    double c2 = (a - 1) * (1 + c1);
+                    double c3 = (a - 1) * (-(c1_2 / 2)
+                               + (a - 2) * c1
+                               + (3 * a - 5) / 2);
+                    double c4 = (a - 1) * ((c1_3 / 3)
+                               - (3 * a - 5) * c1_2 / 2
+                               + (a_2 - 6 * a + 7) * c1
+                               + (11 * a_2 - 46 * a + 47) / 6);
+                    double c5 = (a - 1) * (-(c1_4 / 4)
+                               + (11 * a - 17) * c1_3 / 6
+                               + (-3 * a_2 + 13 * a -13) * c1_2
+                               + (2 * a_3 - 25 * a_2 + 72 * a - 61) * c1 / 2
+                               + (25 * a_3 - 195 * a_2 + 477 * a - 379) / 12);
+
+                    double y_2 = y * y;
+                    double y_3 = y_2 * y;
+                    double y_4 = y_2 * y_2;
+                    result = y + c1 + (c2 / y) + (c3 / y_2) + (c4 / y_3)
+                             + (c5 / y_4);
+                }
+                else {
+                    /* DiDonato and Morris Eq 33: */
+                    double u = -lb + (a - 1) * log(w)
+                               - log(1 + (1 - a) / (1 + w));
+                    result = -lb + (a - 1) * log(u)
+                             - log(1 + (1 - a) / (1 + u));
+                }
+            }
+        }
+        else {
+            double z = w;
+            double ap1 = a + 1;
+            double ap2 = a + 2;
+            if (w < 0.15 * ap1) {
+                /* DiDonato and Morris Eq 35: */
+                double v = log(p) + lgam(ap1);
+                z = exp((v + w) / a);
+                s = log1p(z / ap1 * (1 + z / ap2));
+                z = exp((v + z - s) / a);
+                s = log1p(z / ap1 * (1 + z / ap2));
+                z = exp((v + z - s) / a);
+                s = log1p(z / ap1 * (1 + z / ap2 * (1 + z / (a + 3))));
+                z = exp((v + z - s) / a);
+            }
+
+            if ((z <= 0.01 * ap1) || (z > 0.7 * ap1)) {
+                result = z;
+            }
+            else {
+                /* DiDonato and Morris Eq 36: */
+                double ls = log(didonato_SN(a, z, 100, 1e-4));
+                double v = log(p) + lgam(ap1);
+                z = exp((v + z - ls) / a);
+                result = z * (1 - (a * log(z) - z - v + ls) / (a - z));
+            }
+        }
+    }
+    return result;
+}
+
+
+__noinline__ __device__ double igamci(double a, double q)
+{
+    int i;
+    double x, fac, f_fp, fpp_fp;
+
+    if (isnan(a) || isnan(q)) {
+        return CUDART_NAN;
+    }
+    else if ((a < 0.0) || (q < 0.0) || (q > 1.0)) {
+        // sf_error("gammainccinv", SF_ERROR_DOMAIN, NULL);
+    }
+    else if (q == 0.0) {
+        return CUDART_INF;
+    }
+    else if (q == 1.0) {
+        return 0.0;
+    }
+    else if (q > 0.9) {
+        return igami(a, 1 - q);
+    }
+
+    x = find_inverse_gamma(a, 1 - q, q);
+    for (i = 0; i < 3; i++) {
+        fac = igam_fac(a, x);
+        if (fac == 0.0) {
+            return x;
+        }
+        f_fp = (igamc(a, x) - q) * x / (-fac);
+        fpp_fp = -1.0 + (a - 1) / x;
+        if (isinf(fpp_fp)) {
+            x = x - f_fp;
+        }
+        else {
+            x = x - f_fp / (1.0 - 0.5 * f_fp * fpp_fp);
+        }
+    }
+
+    return x;
+}
+
+
+__noinline__ __device__ double igami(double a, double p)
+{
+    int i;
+    double x, fac, f_fp, fpp_fp;
+
+    if (isnan(a) || isnan(p)) {
+        return CUDART_NAN;
+    }
+    else if ((a < 0) || (p < 0) || (p > 1)) {
+        // sf_error("gammaincinv", SF_ERROR_DOMAIN, NULL);
+    }
+    else if (p == 0.0) {
+        return 0.0;
+    }
+    else if (p == 1.0) {
+        return CUDART_INF;
+    }
+    else if (p > 0.9) {
+        return igamci(a, 1 - p);
+    }
+    x = find_inverse_gamma(a, p, 1 - p);
+    /* Halley's method */
+    for (i = 0; i < 3; i++) {
+        fac = igam_fac(a, x);
+        if (fac == 0.0) {
+            return x;
+        }
+        f_fp = (igam(a, x) - p) * x / fac;
+        // The ratio of the first and second derivatives simplifies
+        fpp_fp = -1.0 + (a - 1) / x;
+        if (isinf(fpp_fp)) {
+            // Resort to Newton's method in the case of overflow
+            x = x - f_fp;
+        }
+        else {
+            x = x - f_fp / (1.0 - 0.5 * f_fp * fpp_fp);
+        }
+    }
+
+    return x;
+}
+
+"""
+)
+
+
+gammaincc = _core.create_ufunc(
+    "cupyx_scipy_gammaincc",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(igamc(in0, in1));",
+    preamble=_igam_preamble,
+    doc="""Elementwise function for scipy.special.gammaincc
+
+    Regularized upper incomplete gamma function.
+
+    .. seealso:: :meth:`scipy.special.gammaincc`
+    """,
+)
+
+
+gammainc = _core.create_ufunc(
+    "cupyx_scipy_gammainc",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(igam(in0, in1));",
+    preamble=_igam_preamble,
+    doc="""Elementwise function for scipy.special.gammainc
+
+    Regularized lower incomplete gamma function.
+
+    .. seealso:: :meth:`scipy.special.gammainc`
+    """,
+)
+
+
+gammainccinv = _core.create_ufunc(
+    "cupyx_scipy_gammainccinv",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(igamci(in0, in1));",
+    preamble=_igami_preamble,
+    doc="""Elementwise function for scipy.special.gammainccinv
+
+    Inverse to gammaincc.
+
+    .. seealso:: :meth:`scipy.special.gammainccinv`
+    """,
+)
+
+
+gammaincinv = _core.create_ufunc(
+    "cupyx_scipy_gammaincinv",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(igami(in0, in1));",
+    preamble=_igami_preamble,
+    doc="""Elementwise function for scipy.special.gammaincinv
+
+    Inverse to gammainc.
+
+    .. seealso:: :meth:`scipy.special.gammaincinv`
+    """,
+)
diff --git a/cupyx/scipy/special/_gammaln.py b/cupyx/scipy/special/_gammaln.py
new file mode 100644
index 0000000..da53815
--- /dev/null
+++ b/cupyx/scipy/special/_gammaln.py
@@ -0,0 +1,65 @@
+import math
+
+import cupy
+from cupy import _core
+
+
+gammaln = _core.create_ufunc(
+    'cupyx_scipy_special_gammaln', ('f->f', 'd->d'),
+    '''
+    if (isinf(in0) && in0 < 0) {
+        out0 = -1.0 / 0.0;
+    } else {
+        out0 = lgamma(in0);
+    }
+    ''',
+    doc="""Logarithm of the absolute value of the Gamma function.
+
+    Args:
+        x (cupy.ndarray): Values on the real line at which to compute
+        ``gammaln``.
+
+    Returns:
+        cupy.ndarray: Values of ``gammaln`` at x.
+
+    .. seealso:: :data:`scipy.special.gammaln`
+
+    """)
+
+
+def multigammaln(a, d):
+    r"""Returns the log of multivariate gamma, also sometimes called the
+    generalized gamma.
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        The multivariate gamma is computed for each item of `a`.
+    d : int
+        The dimension of the space of integration.
+
+    Returns
+    -------
+    res : ndarray
+        The values of the log multivariate gamma at the given points `a`.
+
+    See Also
+    --------
+    :func:`scipy.special.multigammaln`
+
+    """
+    if not cupy.isscalar(d) or (math.floor(d) != d):
+        raise ValueError("d should be a positive integer (dimension)")
+    if cupy.isscalar(a):
+        a = cupy.asarray(a, dtype=float)
+    if int(cupy.any(a <= 0.5 * (d - 1))):
+        raise ValueError("condition a > 0.5 * (d-1) not met")
+    res = (d * (d - 1) * 0.25) * math.log(math.pi)
+    gam0 = gammaln(a)
+    if a.dtype.kind != 'f':
+        # make sure all integer dtypes do the summation with float64
+        gam0 = gam0.astype(cupy.float64)
+    res = res + gam0
+    for j in range(2, d + 1):
+        res += gammaln(a - (j - 1.0) / 2)
+    return res
diff --git a/cupyx/scipy/special/_gammasgn.py b/cupyx/scipy/special/_gammasgn.py
new file mode 100644
index 0000000..3f24e07
--- /dev/null
+++ b/cupyx/scipy/special/_gammasgn.py
@@ -0,0 +1,51 @@
+"""
+The source code here is an adaptation with minimal changes from the following
+file in SciPy's bundled Cephes library:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/gammasgn.c
+
+Cephes Math Library Release 2.0:  April, 1987
+Copyright 1984, 1987 by Stephen L. Moshier
+Direct inquiries to 30 Frost Street, Cambridge, MA 02140
+"""
+
+from cupy import _core
+
+
+gammasgn_definition = """
+__device__ double gammasgn(double x)
+{
+    double fx;
+
+    if (isnan(x)) {
+      return x;
+    }
+    if (x > 0) {
+        return 1.0;
+    }
+    else {
+        fx = floor(x);
+        if (x - fx == 0.0) {
+            return 0.0;
+        }
+        else if ((int)fx % 2) {
+            return -1.0;
+        }
+        else {
+            return 1.0;
+        }
+    }
+}
+"""
+
+gammasgn = _core.create_ufunc(
+    "cupyx_scipy_gammasgn",
+    ("f->f", "d->d"),
+    "out0 = out0_type(gammasgn(in0));",
+    preamble=gammasgn_definition,
+    doc="""Elementwise function for scipy.special.gammasgn
+
+    .. seealso:: :meth:`scipy.special.gammasgn`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_loggamma.py b/cupyx/scipy/special/_loggamma.py
new file mode 100644
index 0000000..0a659c3
--- /dev/null
+++ b/cupyx/scipy/special/_loggamma.py
@@ -0,0 +1,229 @@
+"""
+The source code here is an adaptation with minimal changes from the following
+SciPy files:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/_complexstuff.pxd
+https://github.com/scipy/scipy/blob/master/scipy/special/_evalpoly.pxd
+https://github.com/scipy/scipy/blob/master/scipy/special/_loggamma.pxd
+
+"""
+
+from cupy import _core
+from cupyx.scipy.special._complexstuff import zlog1_definition
+from cupyx.scipy.special._trig import csinpi_definition
+
+
+ceval_poly_definition = """
+
+/* Evaluate polynomials.
+ *
+ *
+ * All of the coefficients are stored in reverse order, i.e. if the
+ * polynomial is
+ *
+ *     u_n x^n + u_{n - 1} x^{n - 1} + ... + u_0,
+ *
+ * then coeffs[0] = u_n, coeffs[1] = u_{n - 1}, ..., coeffs[n] = u_0.
+ *
+ * References
+ * ----------
+ * [1] Knuth, 'The Art of Computer Programming, Volume II'
+ *
+ */
+
+/* Evaluate a polynomial with real coefficients at a complex point.
+ * Uses equation (3) in section 4.6.4 of [1]. Note that it is more
+ * efficient than Horner's method.
+ */
+__device__ complex<double> cevalpoly(double *coeffs, int degree,
+                                     complex<double> z)
+{
+    int j;
+    double a = coeffs[0];
+    double b = coeffs[1];
+    double r = 2*z.real();
+    double s = z.real()*z.real() + z.imag()*z.imag();
+    double tmp;
+    for (j=2; j<=degree; j++)
+    {
+        tmp = b;
+        b = fma(-s, a, coeffs[j]);
+        a = fma(r, a, tmp);
+    }
+    return z*a + b;
+}
+"""
+
+
+loggamma_real_definition = """
+
+#include <cupy/math_constants.h>
+
+__device__ double loggamma_real(double x)
+{
+    if (x < 0.0) {
+        return CUDART_NAN;
+    } else {
+        return lgamma(x);
+    }
+}
+
+"""
+
+
+loggamma_definition = (
+    ceval_poly_definition
+    + zlog1_definition
+    + csinpi_definition
+    + loggamma_real_definition
+    + """
+
+#include <cupy/math_constants.h>
+
+#define TWOPI 6.2831853071795864769252842  // 2*pi
+#define LOGPI 1.1447298858494001741434262  // log(pi)
+#define HLOG2PI 0.918938533204672742  // log(2*pi)/2
+#define SMALLX 7.0
+#define SMALLY 7.0
+#define TAYLOR_RADIUS 0.2
+
+__device__ complex<double> loggamma_recurrence(complex<double>);
+__device__ complex<double> loggamma_stirling(complex<double>);
+__device__ complex<double> loggamma_taylor(complex<double>);
+
+
+// Compute the principal branch of log-Gamma
+
+__noinline__ __device__ complex<double> loggamma(complex<double> z)
+{
+    double tmp;
+    if (isnan(z)) {
+        return complex<double>(CUDART_NAN, CUDART_NAN);
+    } else if ((z.real() <= 0) && (z == floor(z.real()))) {
+        return complex<double>(CUDART_NAN, CUDART_NAN);
+    } else if ((z.real() > SMALLX) || (fabs(z.imag()) > SMALLY)) {
+        return loggamma_stirling(z);
+    } else if (abs(z - 1.0) <= TAYLOR_RADIUS) {
+        return loggamma_taylor(z);
+    } else if (abs(z - 2.0) <= TAYLOR_RADIUS) {
+        // Recurrence relation and the Taylor series around 1
+        return zlog1(z - 1.0) + loggamma_taylor(z - 1.0);
+    } else if (z.real() < 0.1) {
+        // Reflection formula; see Proposition 3.1 in [1]
+        tmp = copysign(TWOPI, z.imag())*floor(0.5*z.real() + 0.25);
+        complex<double> ctemp(LOGPI, tmp);
+        return ctemp - log(csinpi(z)) - loggamma(1.0 - z);
+    } else if (signbit(z.imag()) == 0.0) {
+        // z.imag() >= 0 but is not -0.0
+        return loggamma_recurrence(z);
+    } else {
+        return conj(loggamma_recurrence(conj(z)));
+    }
+}
+
+
+/* Backward recurrence relation.
+ *
+ * See Proposition 2.2 in [1] and the Julia implementation [2].
+ */
+__noinline__ __device__ complex<double> loggamma_recurrence(complex<double> z)
+{
+    int signflips = 0;
+    int sb = 0;
+    int nsb;
+    complex<double> shiftprod = z;
+    z += 1.0;
+    while(z.real() <= SMALLX) {
+        shiftprod *= z;
+        nsb = signbit(shiftprod.imag());
+        if (nsb != 0 and sb == 0) {
+            signflips += 1;
+        }
+        sb = nsb;
+        z += 1.0;
+    }
+    complex<double> ctemp(0.0, -signflips*TWOPI);
+    return loggamma_stirling(z) - log(shiftprod) + ctemp;
+}
+
+
+/* Stirling series for log-Gamma.
+ *
+ * The coefficients are B[2*n]/(2*n*(2*n - 1)) where B[2*n] is the
+ * (2*n)th Bernoulli number. See (1.1) in [1].
+ */
+__device__ complex<double> loggamma_stirling(complex<double> z)
+{
+    double coeffs[] = {
+        -2.955065359477124183e-2, 6.4102564102564102564e-3,
+        -1.9175269175269175269e-3, 8.4175084175084175084e-4,
+        -5.952380952380952381e-4, 7.9365079365079365079e-4,
+        -2.7777777777777777778e-3, 8.3333333333333333333e-2
+    };
+    complex<double> rz = 1.0/z;
+    complex<double> rzz = rz/z;
+    return (z - 0.5)*log(z) - z + HLOG2PI + rz*cevalpoly(coeffs, 7, rzz);
+}
+
+
+/* Taylor series for log-Gamma around z = 1.
+ *
+ * It is
+ *
+ * loggamma(z + 1) = -gamma*z + zeta(2)*z**2/2 - zeta(3)*z**3/3 ...
+ *
+ * where gamma is the Euler-Mascheroni constant.
+ */
+__device__ complex<double> loggamma_taylor(complex<double> z)
+{
+    double coeffs[] = {
+        -4.3478266053040259361e-2, 4.5454556293204669442e-2,
+        -4.7619070330142227991e-2, 5.000004769810169364e-2,
+        -5.2631679379616660734e-2, 5.5555767627403611102e-2,
+        -5.8823978658684582339e-2, 6.2500955141213040742e-2,
+        -6.6668705882420468033e-2, 7.1432946295361336059e-2,
+        -7.6932516411352191473e-2, 8.3353840546109004025e-2,
+        -9.0954017145829042233e-2, 1.0009945751278180853e-1,
+        -1.1133426586956469049e-1, 1.2550966952474304242e-1,
+        -1.4404989676884611812e-1, 1.6955717699740818995e-1,
+        -2.0738555102867398527e-1, 2.7058080842778454788e-1,
+        -4.0068563438653142847e-1, 8.2246703342411321824e-1,
+        -5.7721566490153286061e-1
+    };
+
+    z = z - 1.0;
+    return z*cevalpoly(coeffs, 22, z);
+}
+
+""")
+
+
+loggamma = _core.create_ufunc(
+    'cupyx_scipy_loggamma',
+    (
+        ('f->f', 'out0 = out0_type(loggamma_real(in0))'),
+        ('d->d', 'out0 = loggamma_real(in0)'),
+        'F->F',
+        'D->D'
+    ),
+    'out0 = out0_type(loggamma(in0))',
+    preamble=loggamma_definition,
+    doc="""Principal branch of the logarithm of the gamma function.
+
+    Parameters
+    ----------
+    z : cupy.ndarray
+        Values in the complex plain at which to compute loggamma
+    out : cupy.ndarray, optional
+        Output array for computed values of loggamma
+
+    Returns
+    -------
+    cupy.ndarray
+        Values of loggamma at z.
+
+    See Also
+    --------
+    :func:`scipy.special.loggamma`
+    """,
+)
diff --git a/cupyx/scipy/special/_logsoftmax.py b/cupyx/scipy/special/_logsoftmax.py
new file mode 100644
index 0000000..da582b8
--- /dev/null
+++ b/cupyx/scipy/special/_logsoftmax.py
@@ -0,0 +1,53 @@
+import cupy as cp
+
+
+_log_softmax_kernel = cp._core.ReductionKernel(
+    'T x1',
+    'T y',
+    'exp(x1)',
+    'a + b',
+    'y = log(a)',
+    '0',
+    name='log_softmax'
+)
+
+
+def log_softmax(x, axis=None):
+    """Compute logarithm of softmax function
+
+    Parameters
+    ----------
+    x : array-like
+        Input array
+    axis : int or tuple of ints, optional
+        Axis to compute values along. Default is None and softmax
+        will be  computed over the entire array `x`
+
+    Returns
+    -------
+    s : cupy.ndarry
+        An array with the same shape as `x`. Exponential of the
+        result will sum to 1 along the specified axis. If `x` is a
+        scalar, a scalar is returned
+
+    """
+
+    x_max = cp.amax(x, axis=axis, keepdims=True)
+
+    if x_max.ndim > 0:
+        x_max[~cp.isfinite(x_max)] = 0
+    elif not cp.isfinite(x_max):
+        x_max = 0
+
+    tmp = x - x_max
+
+    if tmp.dtype.kind in 'iu':
+        for out_dtype in [cp.float16, cp.float32, cp.float64]:
+            if cp.can_cast(tmp.dtype, out_dtype):
+                tmp = tmp.astype(out_dtype)
+                break
+
+    out = _log_softmax_kernel(tmp, axis=axis, keepdims=True)
+
+    out = tmp - out
+    return out
diff --git a/cupyx/scipy/special/_logsumexp.py b/cupyx/scipy/special/_logsumexp.py
new file mode 100644
index 0000000..005e0f5
--- /dev/null
+++ b/cupyx/scipy/special/_logsumexp.py
@@ -0,0 +1,75 @@
+import cupy as cp
+
+
+def logsumexp(a, axis=None, b=None, keepdims=False, return_sign=False):
+    """Compute the log of the sum of exponentials of input elements.
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        Input array
+    axis : None or int or tuple of ints, optional
+        Axis or axes over which the sum is taken. By default
+        `axis` is None, and all elements are summed
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced
+        are left in the result as dimensions with size one. With
+        this option, the result will broadcast correctly
+        against the original array
+    b : cupy.ndarray, optional
+        Scaling factor for exp(`a`) must be of the same shape as `a` or
+        broadcastable to `a`. These values may be negative in order to
+        implement subtraction
+    return_sign : bool, optional
+        If this is set to True, the result will be a pair containing sign
+        information; if False, results that are negative will be returned
+        as NaN. Default is False
+
+    Returns
+    -------
+    res : cupy.ndarray
+        The result, ``cp.log(cp.sum(cp.exp(a)))`` calculated
+        in a numerically more stable way. If `b` is given then
+        ``cp.log(cp.sum(b*cp.exp(a)))`` is returned
+    sgn : cupy.ndarray
+        If return_sign is True, this will be an array of floating-point
+        numbers matching res and +1, 0, or -1 depending on the sign of
+        the result. If False, only onw result is returned
+
+    See Also
+    --------
+    scipy.special.logsumexp
+
+    """
+    if b is not None:
+        a, b = cp.broadcast_arrays(a, b)
+        if cp.any(b == 0):
+            a = a + 0.  # promote to at least float
+            a[b == 0] = -cp.inf
+
+    a_max = cp.max(a, axis=axis, keepdims=True)
+
+    if a_max.ndim > 0:
+        a_max[~cp.isfinite(a_max)] = 0
+    elif not cp.isfinite(a_max):
+        a_max = 0
+
+    if b is not None:
+        tmp = b * cp.exp(a - a_max)
+    else:
+        tmp = cp.exp(a - a_max)
+
+    s = cp.sum(tmp, axis=axis, keepdims=keepdims)
+    if return_sign:
+        sgn = cp.sign(s)
+        s *= sgn
+    out = cp.log(s)
+
+    if not keepdims:
+        a_max = cp.squeeze(a_max, axis=axis)
+    out += a_max
+
+    if return_sign:
+        return out, sgn
+    else:
+        return out
diff --git a/cupyx/scipy/special/_lpmv.py b/cupyx/scipy/special/_lpmv.py
new file mode 100644
index 0000000..e2a767d
--- /dev/null
+++ b/cupyx/scipy/special/_lpmv.py
@@ -0,0 +1,417 @@
+"""
+The source code here is an adaptation with minimal changes from the following
+files in SciPy:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/specfun_wrappers.c
+
+Code for psi_spec, gamma2, lpmv0, lpmv was manually translated to C++ from
+SciPy's Fortran-77 code located in:
+https://github.com/scipy/scipy/blob/master/scipy/special/specfun/specfun.f
+
+The fortran code in scipy originated in the following book.
+
+    "Computation of Special Functions", 1996, John Wiley & Sons, Inc.
+
+    Shanjie Zhang and Jianming Jin
+
+    Copyrighted but permission granted to use code in programs.
+"""
+
+from cupy import _core
+
+# May want to use CUDART_PI instead of redefining here.
+# The two seem to differ in the last didit, though.
+#     #define CUDART_PI               3.1415926535897931e+0
+
+nan_inf = """
+// include for CUDART_NAN, CUDART_INF
+#include <cupy/math_constants.h>
+
+"""
+
+
+lpmv_definition = """
+
+// include for CUDART_NAN, CUDART_INF
+#include <cupy/math_constants.h>
+
+// CUDA C++ translation of code from:
+// https://github.com/scipy/scipy/blob/master/scipy/special/specfun/specfun.f
+
+/*
+ *  Purpose: Compute Psi function
+ *   Input :  x  --- Argument of psi(x)
+ */
+__device__ double psi_spec(double x)
+{
+    double xa = fabs(x);
+    double PI = 3.1415926535897930;
+    double EL = 0.5772156649015329;  // Euler-Mascheroni constant
+    double s = 0.;
+    double ps;
+    int n = 0;
+    if ((x - (int)x == 0.) && (x <= 0.))
+    {
+        return CUDART_INF;
+    } else if (xa - (int)xa == 0.) {
+        n = (int) xa;
+        for (int k=1; k <= n - 1; k++)
+        {
+           s += 1.0/k;
+        }
+        ps = -EL + s;
+    } else if ((xa - 0.5) - (int)(xa - 0.5) == 0.) {
+        n = (int)(xa - 0.5);
+        for (int k=1; k <= n; k++)
+        {
+           s += 1.0/(2.0*k - 1.0);
+        }
+        ps = -EL + 2.0 * s - 1.386294361119891;
+    } else {
+        if (xa < 10.0)
+        {
+            n = 10 - (int)(xa);
+            for (int k=1; k < n; k++)
+            {
+               s += 1.0/(xa + k);
+            }
+            xa += (double)n;
+        }
+        double x2 = 1.0 / (xa*xa);
+        double a1 = -.8333333333333e-01;
+        double a2 = .83333333333333333e-02;
+        double a3 = -.39682539682539683e-02;
+        double a4 = .41666666666666667e-02;
+        double a5 = -.75757575757575758e-02;
+        double a6 = .21092796092796093e-01;
+        double a7 = -.83333333333333333e-01;
+        double a8 = .44325980392156860;
+        ps = log(xa) - 0.5/xa + x2*(((((((a8*x2 + a7)*x2 +
+             a6)*x2 + a5)*x2 + a4)*x2 + a3)*x2 + a2)*x2 + a1);
+        ps -= s;
+    }
+    if (x < 0.) {
+        ps = ps - PI*cos(PI*x)/sin(PI*x) - 1.0/x;
+    }
+    return ps;
+}
+
+
+/*
+ *       Purpose: Compute gamma function Gamma(x)
+ *       Input :  x  --- Argument of Gamma(x)
+ *                      ( x is not equal to 0,-1,-2,...)
+ *       Output:  GA --- Gamma(x)
+ */
+__device__ double gamma2(double x)
+{
+    double ga;
+    double G[26] = {1.0, 0.5772156649015329,
+                   -0.6558780715202538, -0.420026350340952e-1,
+                    0.1665386113822915,-.421977345555443e-1,
+                    -.96219715278770e-2, .72189432466630e-2,
+                    -.11651675918591e-2, -.2152416741149e-3,
+                     .1280502823882e-3, -.201348547807e-4,
+                    -.12504934821e-5, .11330272320e-5,
+                    -.2056338417e-6, .61160950e-8,
+                     .50020075e-8, -.11812746e-8,
+                     .1043427e-9, .77823e-11,
+                    -.36968e-11, .51e-12,
+                    -.206e-13, -.54e-14, .14e-14, .1e-15};
+    double PI = 3.141592653589793;
+    int k;
+    if ((x - (int)x) == 0.)
+    {
+        if (x > 0.)
+        {
+            ga = 1.0;
+            for (k=2; k < x; k++)
+            {
+                ga *= k;
+            }
+        } else
+        {
+            ga = CUDART_INF;
+        }
+    }
+    else
+    {
+        double r = 1.0;
+        double z = fabs(x);
+        if (z > 1.0){
+            int m = (int)z;
+            for (k=1; k<=m; k++)
+            {
+                r *= (z - k);
+            }
+            z -= m;
+        } else {
+            z = x;
+        }
+        double gr = G[24];
+        for (k=25; k>=1; k--)
+        {
+            gr = gr*z + G[k];
+        }
+        ga = 1.0 / (gr * z);
+        if (fabs(x) > 1.0)
+        {
+            ga *= r;
+            if (x < 0.0)
+            {
+                ga = -PI / (x*ga*sin(PI*x));
+            }
+        }
+    }
+    return ga;
+}
+
+
+/*
+ *     Purpose: Compute the associated Legendre function
+ *              Pmv(x) with an integer order and an arbitrary
+ *              nonnegative degree v
+ *     Input :  x   --- Argument of Pm(x)  ( -1 <= x <= 1 )
+ *              m   --- Order of Pmv(x)
+ *              v   --- Degree of Pmv(x)
+ */
+__device__ double lpmv0(double v, double m, double x)
+{
+    double pmv, r;
+    int j, k;
+
+    double PI = 3.141592653589793;
+    double EL = .5772156649015329;  // Euler-Mascheroni constant
+    double EPS = 1.0e-14;
+    int nv = (int)v;
+    double v0 = v - nv;
+    if ((x == -1.0) && (v != nv))
+    {
+        if (m == 0)
+            return -CUDART_INF;
+        else
+            return CUDART_INF;
+    }
+    double c0 = 1.0;
+    if (m != 0)
+    {
+        double rg = v*(v + m);
+        for (j=1; j < m; j++)
+        {
+            rg *= (v*v - j*j);
+        }
+        double xq = sqrt(1.0 - x*x);
+
+        double r0 = 1.0;
+        for (j=1; j <= m; j++)
+        {
+            r0 *= 0.5*xq/j;
+        }
+        c0 = r0*rg;
+    }
+    if (v0 == 0.)
+    {
+        // DLMF 14.3.4, 14.7.17, 15.2.4
+        pmv = 1.0;
+        r = 1.0;
+        for (k=1; k <= nv - m; k++)
+        {
+            r *= 0.5*(-nv + m + k - 1.0)*(nv + m + k)/(k*(k + m))*(1.0 + x);
+            pmv += r;
+        }
+        pmv *= c0;
+        if ((nv % 2) == 1)
+        {
+            pmv = -pmv;
+        }
+    }
+    else
+    {
+        if (x >= -0.35)
+        {
+            // DLMF 14.3.4, 15.2.1
+            pmv = 1.0;
+            r = 1.0;
+            for (k = 1; k <= 100; k++)
+            {
+                 r *= 0.5*(-v + m + k - 1.0)*(v + m + k)/(k*(m + k))*(1.0 - x);
+                 pmv += r;
+                 if ((k > 12) & (fabs(r/pmv) < EPS)){
+                    break;
+                 }
+
+            }
+            pmv *= c0;
+            if (((int)m % 2) == 1)
+            {
+                pmv = -pmv;
+            }
+        }
+        else
+        {
+            // DLMF 14.3.5, 15.8.10
+            double vs = sin(v * PI) / PI;
+            double pv0 = 0.0;
+            double r2;
+            if (m != 0)
+            {
+                double qr = sqrt((1.0 - x)/(1.0 + x));
+                r2 = 1.0;
+                for (j = 1; j <= m; j++)
+                {
+                    r2 *= qr*j;
+                }
+                double s0 = 1.0;
+                double r1 = 1.0;
+                for (k = 1; k < m; k++)
+                {
+                    r1=0.5*r1*(-v + k - 1)*(v + k)/(k*(k - m))*(1.0 + x);
+                    s0 += r1;
+                }
+                pv0 = -vs*r2/m*s0;
+            }
+            double psv = psi_spec(v);
+            double pa = 2.0*(psv + EL) + PI/tan(PI*v) + 1.0/v;
+            double s1 = 0.0;
+            for (j = 1; j <= m; j++)
+            {
+                s1 += (j*j + v*v)/(j*(j*j - v*v));
+            }
+            pmv = pa + s1 - 1.0/(m - v) + log(0.5*(1.0 + x));
+            r = 1.0;
+            for (k = 1; j <= 100; k++)
+            {
+                r *= 0.5*(-v + m + k-1.0)*(v + m + k)/(k*(k + m))*(1.0 + x);
+                double s = 0.0;
+                for (j = 1; j <= m; j++)
+                {
+                    double kjsq = (k + j) * (k + j);
+                    s += (kjsq + v*v)/((k + j)*(kjsq - v*v));
+                }
+                double s2 = 0.0;
+                for (j = 1; j <= k; j++)
+                {
+                    s2 += 1.0/(j*(j*j - v*v));
+                }
+                double pss = pa + s + 2.0*v*v*s2 - 1.0/(m + k - v)
+                             + log(0.5*(1.0 + x));
+                r2 = pss*r;
+                pmv += r2;
+                if (fabs(r2/pmv) < EPS)
+                {
+                    break;
+                }
+            }
+            pmv = pv0 + pmv*vs*c0;
+        }
+    }
+    return pmv;
+}
+
+
+/*       Purpose: Compute the associated Legendre function
+ *                Pmv(x) with an integer order and an arbitrary
+ *                degree v, using recursion for large degrees
+ *       Input :  x   --- Argument of Pm(x)  ( -1 <= x <= 1 )
+ *                m   --- Order of Pmv(x)
+ *                v   --- Degree of Pmv(x)
+ *       Output:  PMV --- Pmv(x)
+ */
+__device__ double lpmv(double v, int m, double x)
+{
+    double pmv;
+    int j;
+
+    if ((x == -1.0) && (v != (int)v))
+    {
+        if (m == 0)
+        {
+            return -CUDART_INF;
+        } else
+        {
+            return CUDART_INF;
+        }
+    }
+
+    double vx = v;
+    double mx = m;
+    // DLMF 14.9.5
+    if (v < 0)
+    {
+        vx = -vx - 1;
+    }
+    int neg_m = 0;
+    if (m < 0)
+    {
+        if (((vx + m + 1) > 0) || (vx != (int)vx))
+        {
+            neg_m = 1;
+            mx = -m;
+        }
+        else
+        {
+            // We don't handle cases where DLMF 14.9.3 doesn't help
+            return CUDART_NAN;
+        }
+    }
+
+    int nv = (int)vx;
+    double v0 = vx - nv;
+    if ((nv > 2) && (nv > mx))
+    {
+        // Up-recursion on degree, AMS 8.5.3 / DLMF 14.10.3
+        double p0 = lpmv0(v0 + mx, mx, x);
+        double p1 = lpmv0(v0 + mx + 1, mx, x);
+        pmv = p1;
+        for (j = mx + 2; j <= nv; j++)
+        {
+          pmv = ((2*(v0 + j) - 1)*x*p1 - (v0 + j - 1 + mx)*p0) / (v0 + j - mx);
+          p0 = p1;
+          p1 = pmv;
+        }
+    }
+    else
+    {
+        pmv = lpmv0(vx, mx, x);
+    }
+
+    if ((neg_m != 0) && (fabs(pmv) < 1.0e300))
+    {
+        double g1 = gamma2(vx - mx + 1);
+        double g2 = gamma2(vx + mx + 1);
+        pmv = pmv * g1/g2 * pow(-1, mx);
+    }
+    return pmv;
+}
+
+
+// pmv_wrap as in
+// https://github.com/scipy/scipy/blob/master/scipy/special/specfun_wrappers.c
+
+__device__  double pmv_wrap(double m, double v, double x){
+  int int_m;
+  double out;
+
+  if (m != floor(m))
+  {
+      return CUDART_NAN;
+  }
+  int_m = (int) m;
+  out = lpmv(v, int_m, x);
+  // should raise an overflow warning here on INF
+  return out;
+}
+
+"""
+
+lpmv = _core.create_ufunc(
+    "cupyx_scipy_lpmv",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(pmv_wrap(in0, in1, in2));",
+    preamble=lpmv_definition,
+    doc="""Associated Legendre function of integer order and real degree.
+
+    .. seealso:: :meth:`scipy.special.lpmv`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_poch.py b/cupyx/scipy/special/_poch.py
new file mode 100644
index 0000000..ef88b8f
--- /dev/null
+++ b/cupyx/scipy/special/_poch.py
@@ -0,0 +1,119 @@
+"""
+The source code here is an adaptation with minimal changes from the following
+files in SciPy's bundled Cephes library:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/cephes/poch.c
+
+Cephes Math Library Release 2.0:  April, 1987
+Copyright 1984, 1987 by Stephen L. Moshier
+Direct inquiries to 30 Frost Street, Cambridge, MA 02140
+"""
+
+from cupy import _core
+
+from cupyx.scipy.special._gammasgn import gammasgn_definition
+from cupyx.scipy.special._gammainc import lgam_definition
+
+
+poch_definition = (
+    lgam_definition
+    + gammasgn_definition
+    + """
+/*
+ * Pochhammer symbol (a)_m = gamma(a + m) / gamma(a)
+ */
+
+// include for CUDART_NAN, CUDART_INF
+#include <cupy/math_constants.h>
+
+
+__device__ double is_nonpos_int(double x)
+{
+    return x <= 0 && x == ceil(x) && fabs(x) < 1e13;
+}
+
+__device__ double poch(double a, double m)
+{
+    double r;
+
+    r = 1.0;
+
+    /*
+     * 1. Reduce magnitude of `m` to |m| < 1 by using recurrence relations.
+     *
+     * This may end up in over/underflow, but then the function itself either
+     * diverges or goes to zero. In case the remainder goes to the opposite
+     * direction, we end up returning 0*INF = NAN, which is OK.
+     */
+
+    /* Recurse down */
+    while (m >= 1.0) {
+        if (a + m == 1) {
+            break;
+        }
+        m -= 1.0;
+        r *= (a + m);
+        if (!isfinite(r) || r == 0) {
+            break;
+        }
+    }
+
+    /* Recurse up */
+    while (m <= -1.0) {
+        if (a + m == 0) {
+            break;
+        }
+        r /= (a + m);
+        m += 1.0;
+        if (!isfinite(r) || r == 0) {
+            break;
+        }
+    }
+
+    /*
+     * 2. Evaluate function with reduced `m`
+     *
+     * Now either `m` is not big, or the `r` product has over/underflown.
+     * If so, the function itself does similarly.
+     */
+
+    if (m == 0) {
+        /* Easy case */
+        return r;
+    }
+    else if (a > 1e4 && fabs(m) <= 1) {
+        /* Avoid loss of precision */
+        return r * pow(a, m) * (
+            1
+            + m*(m-1)/(2*a)
+            + m*(m-1)*(m-2)*(3*m-1)/(24*a*a)
+            + m*m*(m-1)*(m-1)*(m-2)*(m-3)/(48*a*a*a)
+            );
+    }
+
+    /* Check for infinity */
+    if (is_nonpos_int(a + m) && !is_nonpos_int(a) && a + m != m) {
+        return CUDART_INF;
+    }
+
+    /* Check for zero */
+    if (!is_nonpos_int(a + m) && is_nonpos_int(a)) {
+        return 0;
+    }
+
+    return r * exp(lgam(a + m) - lgam(a)) * gammasgn(a + m) * gammasgn(a);
+}
+"""
+)
+
+poch = _core.create_ufunc(
+    "cupyx_scipy_poch",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(poch(in0, in1));",
+    preamble=poch_definition,
+    doc="""Elementwise function for scipy.special.poch (Pochhammer symbol)
+
+    .. seealso:: :meth:`scipy.special.poch`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_polygamma.py b/cupyx/scipy/special/_polygamma.py
new file mode 100644
index 0000000..44451a8
--- /dev/null
+++ b/cupyx/scipy/special/_polygamma.py
@@ -0,0 +1,22 @@
+import cupy
+from cupyx.scipy.special import _digamma
+from cupyx.scipy.special import _gamma
+from cupyx.scipy.special import _zeta
+
+
+def polygamma(n, x):
+    """Polygamma function n.
+
+    Args:
+        n (cupy.ndarray): The order of the derivative of `psi`.
+        x (cupy.ndarray): Where to evaluate the polygamma function.
+
+    Returns:
+        cupy.ndarray: The result.
+
+    .. seealso:: :data:`scipy.special.polygamma`
+
+    """
+    n, x = cupy.broadcast_arrays(n, x)
+    fac2 = (-1.0)**(n+1) * _gamma.gamma(n+1.0) * _zeta.zeta(n+1.0, x)
+    return cupy.where(n == 0, _digamma.digamma(x), fac2)
diff --git a/cupyx/scipy/special/_softmax.py b/cupyx/scipy/special/_softmax.py
new file mode 100644
index 0000000..2ecae0c
--- /dev/null
+++ b/cupyx/scipy/special/_softmax.py
@@ -0,0 +1,28 @@
+import cupy
+
+
+def softmax(x, axis=None):
+    """Softmax function.
+
+    The softmax function transforms each element of a
+    collection by computing the exponential of each element
+    divided by the sum of the exponentials of all the elements.
+
+    Parameters
+    ----------
+    x : array-like
+        The input array
+    axis : int or tuple of ints, optional
+        Axis to compute values along. Default is None
+
+    Returns
+    -------
+    s : cupy.ndarray
+        Returns an array with same shape as input. The result
+        will sum to 1 along the provided axis
+
+    """
+
+    x_max = cupy.amax(x, axis=axis, keepdims=True)
+    exp_x_shifted = cupy.exp(x - x_max)
+    return exp_x_shifted / cupy.sum(exp_x_shifted, axis=axis, keepdims=True)
diff --git a/cupyx/scipy/special/_sph_harm.py b/cupyx/scipy/special/_sph_harm.py
new file mode 100644
index 0000000..4af131b
--- /dev/null
+++ b/cupyx/scipy/special/_sph_harm.py
@@ -0,0 +1,84 @@
+"""
+The source code here is an adaptation with minimal changes from the following
+SciPy Cython file:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/sph_harm.pxd
+"""
+
+from cupy import _core
+
+from cupyx.scipy.special._poch import poch_definition
+from cupyx.scipy.special._lpmv import lpmv_definition
+
+sph_harmonic_definition = (
+    poch_definition
+    + lpmv_definition
+    + """
+
+#include <cupy/complex.cuh>
+
+// include for CUDART_NAN, CUDART_INF
+#include <cupy/math_constants.h>
+
+#define NPY_PI        3.141592653589793238462643383279502884  /* pi */
+
+// from scipy/special/sph_harm.pxd
+__device__ complex<double> sph_harmonic(int m, int n, double theta, double phi)
+{
+    double x, prefactor;
+    complex<double> val;
+    int mp;
+
+    x = cos(phi);
+    if (abs(m) > n)
+    {
+        // sf_error.error("sph_harm", sf_error.ARG,
+        //                "m should not be greater than n")
+        return CUDART_NAN;
+    }
+    if (n < 0)
+    {
+        // sf_error.error("sph_harm", sf_error.ARG, "n should not be negative")
+        return CUDART_NAN;
+    }
+    if (m < 0)
+    {
+        mp = -m;
+        prefactor = poch(n + mp + 1, -2 * mp);
+        if ((mp % 2) == 1)
+        {
+            prefactor = -prefactor;
+        }
+    }
+    else
+    {
+        mp = m;
+    }
+    val = pmv_wrap(mp, n, x);
+    if (m < 0)
+    {
+        val *= prefactor;
+    }
+    val *= sqrt((2*n + 1) / 4.0 / NPY_PI);
+    val *= sqrt(poch(n + m + 1, -2 * m));
+
+    complex<double> exponent(0, m * theta);
+    val *= exp(exponent);
+
+    return val;
+}
+"""
+)
+
+
+sph_harm = _core.create_ufunc(
+    "cupyx_scipy_lpmv",
+    ("iiff->F", "iidd->D", "llff->F", "lldd->D"),
+    "out0 = out0_type(sph_harmonic(in0, in1, in2, in3));",
+    preamble=sph_harmonic_definition,
+    doc="""Spherical Harmonic.
+
+    .. seealso:: :meth:`scipy.special.sph_harm`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_statistics.py b/cupyx/scipy/special/_statistics.py
new file mode 100644
index 0000000..345ad3d
--- /dev/null
+++ b/cupyx/scipy/special/_statistics.py
@@ -0,0 +1,230 @@
+from cupy import _core
+
+
+logit_definition = """
+template <typename T>
+static __device__ T logit(T x) {
+    x /= 1 - x;
+    return log(x);
+}
+
+"""
+
+
+logit = _core.create_ufunc(
+    'cupy_logit',
+    ('e->f', 'f->f', 'd->d'),
+    'out0 = logit(in0)',
+    preamble=logit_definition,
+    doc='''Logit function.
+
+    Args:
+        x (cupy.ndarray): input data
+
+    Returns:
+        cupy.ndarray: values of logit(x)
+
+    .. seealso:: :data:`scipy.special.logit`
+
+    ''')
+
+
+expit_definition = """
+template <typename T>
+static __device__ T expit(T x) {
+    return 1 / (1 + exp(-x));
+}
+
+"""
+
+
+expit = _core.create_ufunc(
+    'cupy_expit',
+    ('e->f', 'f->f', 'd->d'),
+    'out0 = expit(in0)',
+    preamble=expit_definition,
+    doc='''Logistic sigmoid function (expit).
+
+    Args:
+        x (cupy.ndarray): input data (must be real)
+
+    Returns:
+        cupy.ndarray: values of expit(x)
+
+    .. seealso:: :data:`scipy.special.expit`
+
+    .. note::
+        expit is the inverse of logit.
+
+    ''')
+
+
+# log_expit implemented based on log1p as in SciPy's scipy/special/_logit.h
+
+log_expit_definition = """
+template <typename T>
+static __device__ T log_expit(T x)
+{
+    if (x < 0.0) {
+        return x - log1p(exp(x));
+    } else {
+        return -log1p(exp(-x));
+    }
+}
+
+"""
+
+
+log_expit = _core.create_ufunc(
+    'cupy_log_expit',
+    ('e->f', 'f->f', 'd->d'),
+    'out0 = log_expit(in0)',
+    preamble=log_expit_definition,
+    doc='''Logarithm of the logistic sigmoid function.
+
+    Args:
+        x (cupy.ndarray): input data (must be real)
+
+    Returns:
+        cupy.ndarray: values of log(expit(x))
+
+    .. seealso:: :data:`scipy.special.log_expit`
+
+    .. note::
+        The function is mathematically equivalent to ``log(expit(x))``, but
+        is formulated to avoid loss of precision for inputs with large
+        (positive or negative) magnitude.
+    ''')
+
+
+boxcox_definition = """
+static __device__ double boxcox(double x, double lmbda) {
+    // if lmbda << 1 and log(x) < 1.0, the lmbda*log(x) product can lose
+    // precision, furthermore, expm1(x) == x for x < eps.
+    // For doubles, the range of log is -744.44 to +709.78, with eps being
+    // the smallest value produced.  This range means that we will have
+    // abs(lmbda)*log(x) < eps whenever abs(lmbda) <= eps/-log(min double)
+    // which is ~2.98e-19.
+    if (fabs(lmbda) < 1e-19) {
+        return log(x);
+    } else {
+        return expm1(lmbda * log(x)) / lmbda;
+    }
+}
+
+"""
+
+
+boxcox = _core.create_ufunc(
+    'cupy_boxcox',
+    ('ee->f', 'ff->f', 'dd->d'),
+    'out0 = out0_type(boxcox(in0, in1))',
+    preamble=boxcox_definition,
+    doc='''Compute the Box-Cox transformation.
+
+    Args:
+        x (cupy.ndarray): input data (must be real)
+
+    Returns:
+        cupy.ndarray: values of boxcox(x)
+
+    .. seealso:: :data:`scipy.special.boxcox`
+
+    ''')
+
+
+boxcox1p_definition = """
+static __device__ double boxcox1p(double x, double lmbda) {
+    // The argument given above in boxcox applies here with the modification
+    // that the smallest value produced by log1p is the minimum representable
+    // value, rather than eps.  The second condition here prevents underflow
+    // when log1p(x) is < eps.
+    double lgx = log1p(x);
+    if ((fabs(lmbda) < 1e-19)
+        || ((fabs(lgx) < 1e-289) && (fabs(lmbda) < 1e273))) {
+        return lgx;
+    } else {
+        return expm1(lmbda * lgx) / lmbda;
+    }
+}
+
+"""
+
+
+boxcox1p = _core.create_ufunc(
+    'cupy_boxcox1p',
+    ('ee->f', 'ff->f', 'dd->d'),
+    'out0 = out0_type(boxcox1p(in0, in1))',
+    preamble=boxcox1p_definition,
+    doc='''Compute the Box-Cox transformation op 1 + `x`.
+
+    Args:
+        x (cupy.ndarray): input data (must be real)
+
+    Returns:
+        cupy.ndarray: values of boxcox1p(x)
+
+    .. seealso:: :data:`scipy.special.boxcox1p`
+
+    ''')
+
+
+inv_boxcox_definition = """
+static __device__ double inv_boxcox(double x, double lmbda) {
+    if (lmbda == 0.0) {
+        return exp(x);
+    } else {
+        return exp(log1p(lmbda * x) / lmbda);
+    }
+}
+
+"""
+
+
+inv_boxcox = _core.create_ufunc(
+    'cupy_inv_boxcox',
+    ('ee->f', 'ff->f', 'dd->d'),
+    'out0 = out0_type(inv_boxcox(in0, in1))',
+    preamble=inv_boxcox_definition,
+    doc='''Compute the Box-Cox transformation.
+
+    Args:
+        x (cupy.ndarray): input data (must be real)
+
+    Returns:
+        cupy.ndarray: values of inv_boxcox(x)
+
+    .. seealso:: :data:`scipy.special.inv_boxcox`
+
+    ''')
+
+
+inv_boxcox1p_definition = """
+static __device__ double inv_boxcox1p(double x, double lmbda) {
+    if (lmbda == 0.0) {
+        return expm1(x);
+    } else if (fabs(lmbda * x) < 1e-154) {
+        return x;
+    } else {
+        return expm1(log1p(lmbda * x) / lmbda);
+    }
+}
+
+"""
+
+
+inv_boxcox1p = _core.create_ufunc(
+    'cupy_inv_boxcox1p',
+    ('ee->f', 'ff->f', 'dd->d'),
+    'out0 = out0_type(inv_boxcox1p(in0, in1))',
+    preamble=inv_boxcox1p_definition,
+    doc='''Compute the Box-Cox transformation op 1 + `x`.
+
+    Args:
+        x (cupy.ndarray): input data (must be real)
+
+    Returns:
+        cupy.ndarray: values of inv_boxcox1p(x)
+
+    .. seealso:: :data:`scipy.special.inv_boxcox1p`
+''')
diff --git a/cupyx/scipy/special/_stats_distributions.py b/cupyx/scipy/special/_stats_distributions.py
new file mode 100644
index 0000000..8804166
--- /dev/null
+++ b/cupyx/scipy/special/_stats_distributions.py
@@ -0,0 +1,1125 @@
+"""Statistical distribution functions (Beta, Binomial, Poisson, etc.)
+
+The source code here is an adaptation with minimal changes from the following
+files in SciPy's bundled Cephes library:
+
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/bdtr.c
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/chdtr.c
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/fdtr.c
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/gdtr.c
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/nbdtr.c
+https://github.com/scipy/scipy/blob/main/scipy/special/cephes/pdtr.c
+
+Cephes Math Library, Release 2.3:  March, 1995
+Copyright 1984, 1995 by Stephen L. Moshier
+"""
+
+from cupy import _core
+from cupyx.scipy.special._beta import incbet_preamble, incbi_preamble
+from cupyx.scipy.special._gammainc import _igam_preamble, _igami_preamble
+
+
+# Normal distribution functions
+
+ndtr = _core.create_ufunc(
+    'cupyx_scipy_special_ndtr',
+    (('f->f', 'out0 = normcdff(in0)'), 'd->d'),
+    'out0 = normcdf(in0)',
+    doc='''Cumulative distribution function of normal distribution.
+
+    .. seealso:: :data:`scipy.special.ndtr`
+
+    ''')
+
+
+log_ndtr_definition = """
+
+#define NPY_SQRT1_2   0.707106781186547524400844362104849039  /* 1/sqrt(2) */
+
+static __device__ double log_ndtr(double x)
+{
+    double t = x * NPY_SQRT1_2;
+    if (x < -1.0) {
+        return log(erfcx(-t) / 2) - t * t;
+    } else {
+        return log1p(-erfc(t) / 2);
+    }
+}
+
+static __device__ float log_ndtrf(float x)
+{
+    float t = x * NPY_SQRT1_2;
+    if (x < -1.0) {
+        return logf(erfcxf(-t) / 2) - t * t;
+    } else {
+        return log1pf(-erfcf(t) / 2);
+    }
+}
+
+"""
+
+
+log_ndtr = _core.create_ufunc(
+    'cupyx_scipy_special_log_ndtr',
+    (('f->f', 'out0 = log_ndtrf(in0)'), 'd->d'),
+    'out0 = log_ndtr(in0)',
+    preamble=log_ndtr_definition,
+    doc="""Logarithm of Gaussian cumulative distribution function.
+
+    Returns the log of the area under the standard Gaussian propability
+    density function.
+
+    Parameters
+    ----------
+    x : array-like
+        The input array
+
+    Returns
+    -------
+    y : cupy.ndarray
+        The value of the log of the normal cumulative distribution
+        function evaluated at x
+
+    See Also
+    --------
+    :func:`scipy.special.log_ndtr`
+
+    """,
+)
+
+
+ndtri = _core.create_ufunc(
+    'cupyx_scipy_special_ndtri',
+    (('f->f', 'out0 = normcdfinvf(in0)'), 'd->d'),
+    'out0 = normcdfinv(in0)',
+    doc='''Inverse of the cumulative distribution function of the standard
+           normal distribution.
+
+    .. seealso:: :data:`scipy.special.ndtri`
+''')
+
+
+# Binomial distribution functions
+
+bdtr_definition = """
+
+__device__ double bdtr(double k, int n, double p)
+{
+    double dk, dn;
+    double fk = floor(k);
+
+    if (isnan(p) || isnan(k)) {
+        return CUDART_NAN;
+    }
+
+    if (p < 0.0 || p > 1.0 || fk < 0 || n < fk) {
+        return CUDART_NAN;
+    }
+
+    if (fk == n) {
+        return 1.0;
+    }
+
+    dn = n - fk;
+    if (fk == 0) {
+        dk = pow(1.0 - p, dn);
+    } else {
+        dk = fk + 1.;
+        dk = incbet(dn, dk, 1.0 - p);
+    }
+    return dk;
+}
+
+
+__device__ double bdtr_unsafe(double k, double n, double p)
+{
+    if (isnan(n) || isinf(n)) {
+        return CUDART_NAN;
+    } else {
+        return bdtr(k, (int)n, p);
+    }
+}
+
+"""
+
+
+bdtrc_definition = """
+
+__device__ double bdtrc(double k, int n, double p)
+{
+    double dk, dn;
+    double fk = floor(k);
+
+    if (isnan(p) || isnan(k)) {
+        return CUDART_NAN;
+    }
+
+    if (p < 0.0 || p > 1.0 || n < fk) {
+        return CUDART_NAN;
+    }
+
+    if (fk < 0) {
+        return 1.0;
+    }
+
+    if (fk == n) {
+        return 0.0;
+    }
+
+    dn = n - fk;
+    if (k == 0) {
+        if (p < .01) {
+            dk = -expm1(dn * log1p(-p));
+        } else {
+            dk = 1.0 - pow(1.0 - p, dn);
+        }
+    } else {
+        dk = fk + 1;
+        dk = incbet(dk, dn, p);
+    }
+    return dk;
+}
+
+__device__ double bdtrc_unsafe(double k, double n, double p)
+{
+    if (isnan(n) || isinf(n)) {
+        return CUDART_NAN;
+    } else {
+        return bdtrc(k, (int)n, p);
+    }
+}
+
+"""
+
+
+bdtri_definition = """
+
+__device__ double bdtri(double k, int n, double y)
+{
+    double p, dn, dk;
+    double fk = floor(k);
+
+    if (isnan(k)) {
+        return CUDART_NAN;
+    }
+
+    if (y < 0.0 || y > 1.0 || fk < 0.0 || n <= fk) {
+        return CUDART_NAN;
+    }
+
+    dn = n - fk;
+
+    if (fk == n) {
+        return 1.0;
+    }
+
+    if (fk == 0) {
+        if (y > 0.8) {
+            p = -expm1(log1p(y - 1.0) / dn);
+        } else {
+            p = 1.0 - pow(y, 1.0 / dn);
+        }
+    } else {
+        dk = fk + 1;
+        p = incbet(dn, dk, 0.5);
+        if (p > 0.5) {
+            p = incbi(dk, dn, 1.0 - y);
+        } else {
+            p = 1.0 - incbi(dn, dk, y);
+        }
+    }
+    return p;
+}
+
+__device__ double bdtri_unsafe(double k, double n, double p)
+{
+    if (isnan(n) || isinf(n)) {
+        return CUDART_NAN;
+    } else {
+        return bdtri(k, (int)n, p);
+    }
+}
+
+"""
+
+
+# Note: bdtr ddd->d and fff-> are deprecated as of SciPy 1.7
+bdtr = _core.create_ufunc(
+    "cupyx_scipy_bdtr",
+    (
+        ('fff->f', 'out0 = out0_type(bdtr_unsafe(in0, in1, in2));'),
+        'dld->d',
+        ('ddd->d', 'out0 = bdtr_unsafe(in0, in1, in2);'),
+    ),
+    "out0 = bdtr(in0, (int)in1, in2);",
+    preamble=incbet_preamble + bdtr_definition,
+    doc="""Binomial distribution cumulative distribution function.
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        Number of successes (float), rounded down to the nearest integer.
+    n : cupy.ndarray
+        Number of events (int).
+    p : cupy.ndarray
+        Probability of success in a single event (float).
+
+    Returns
+    -------
+    y : cupy.ndarray
+        Probability of floor(k) or fewer successes in n independent events with
+        success probabilities of p.
+
+    See Also
+    --------
+    :func:`scipy.special.bdtr`
+
+    """,
+)
+
+
+# Note: bdtrc ddd->d and fff->f are deprecated as of SciPy 1.7
+bdtrc = _core.create_ufunc(
+    "cupyx_scipy_bdtrc",
+    (
+        ('fff->f', 'out0 = out0_type(bdtrc_unsafe(in0, in1, in2));'),
+        'dld->d',
+        ('ddd->d', 'out0 = bdtrc_unsafe(in0, in1, in2);'),
+    ),
+    "out0 = out0_type(bdtrc(in0, in1, in2));",
+    preamble=incbet_preamble + bdtrc_definition,
+    doc="""Binomial distribution survival function.
+
+    Returns the complemented binomial distribution function (the integral of
+    the density from x to infinity).
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        Number of successes (float), rounded down to the nearest integer.
+    n : cupy.ndarray
+        Number of events (int).
+    p : cupy.ndarray
+        Probability of success in a single event (float).
+
+    Returns
+    -------
+    y : cupy.ndarray
+        Probability of floor(k) + 1 or more successes in n independent events
+        with success probabilities of p.
+
+    See Also
+    --------
+    :func:`scipy.special.bdtrc`
+
+    """,
+)
+
+
+# Note: bdtri ddd->d and fff->f are deprecated as of SciPy 1.7
+bdtri = _core.create_ufunc(
+    "cupyx_scipy_bdtri",
+    (
+        ('fff->f', 'out0 = out0_type(bdtri_unsafe(in0, in1, in2));'),
+        'dld->d',
+        ('ddd->d', 'out0 = bdtri_unsafe(in0, in1, in2);'),
+    ),
+    "out0 = out0_type(bdtri(in0, in1, in2));",
+    preamble=incbi_preamble + bdtri_definition,
+    doc="""Inverse function to `bdtr` with respect to `p`.
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        Number of successes (float), rounded down to the nearest integer.
+    n : cupy.ndarray
+        Number of events (int).
+    y : cupy.ndarray
+        Cumulative probability (probability of k or fewer successes in n
+        events).
+
+    Returns
+    -------
+    p : cupy.ndarray
+        The event probability such that bdtr(floor(k), n, p) = y.
+
+    See Also
+    --------
+    :func:`scipy.special.bdtri`
+
+    """,
+)
+
+
+# Beta distribution functions
+
+btdtr = _core.create_ufunc(
+    "cupyx_scipy_btdtr",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(incbet(in0, in1, in2));",
+    preamble=incbet_preamble,
+    doc="""Cumulative distribution function of the beta distribution.
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        Shape parameter (a > 0).
+    b : cupy.ndarray
+        Shape parameter (b > 0).
+    x : cupy.ndarray
+        Upper limit of integration, in [0, 1].
+
+    Returns
+    -------
+    I : cupy.ndarray
+        Cumulative distribution function of the beta distribution with
+        parameters a and b at x.
+
+    See Also
+    --------
+    :func:`scipy.special.btdtr`
+
+    """,
+)
+
+
+btdtri = _core.create_ufunc(
+    "cupyx_scipy_btdtri",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(incbi(in0, in1, in2));",
+    preamble=incbi_preamble,
+    doc="""The p-th quantile of the beta distribution.
+
+    This function is the inverse of the beta cumulative distribution function,
+    `btdtr`, returning the value of `x` for which ``btdtr(a, b, x) = p``.
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        Shape parameter (a > 0).
+    b : cupy.ndarray
+        Shape parameter (b > 0).
+    p : cupy.ndarray
+        Cumulative probability, in [0, 1].
+
+    Returns
+    -------
+    x : cupy.ndarray
+        The quantile corresponding to p.
+
+    See Also
+    --------
+    :func:`scipy.special.btdtri`
+
+    """,
+)
+
+
+# Chi square distribution functions
+
+chdtrc_definition = """
+
+__device__ double chdtrc(double df, double x)
+{
+
+    if (x < 0.0) {
+        return 1.0;     /* modified by T. Oliphant */
+    }
+    return igamc(df / 2.0, x / 2.0);
+}
+"""
+
+
+chdtr_definition = """
+
+__device__ double chdtr(double df, double x)
+{
+    if (x < 0.0) {   /* || (df < 1.0) ) */
+        return CUDART_NAN;
+    }
+    return igam(df / 2.0, x / 2.0);
+}
+"""
+
+
+chdtri_definition = """
+__device__ double chdtri(double df, double y)
+{
+    double x;
+
+    if ((y < 0.0) || (y > 1.0)) {   /* || (df < 1.0) ) */
+        return CUDART_NAN;
+    }
+
+    x = igamci(0.5 * df, y);
+    return 2.0 * x;
+}
+"""
+
+
+chdtrc = _core.create_ufunc(
+    "cupyx_scipy_chdtrc",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(chdtrc(in0, in1));",
+    preamble=_igam_preamble + chdtrc_definition,
+    doc="""Chi square survival function.
+
+    Returns the complemented chi-squared distribution function (the integral of
+    the density from x to infinity).
+
+    Parameters
+    ----------
+    v : cupy.ndarray
+        Degrees of freedom.
+    x : cupy.ndarray
+        Upper bound of the integral (nonnegative float).
+
+    Returns
+    -------
+    y : cupy.ndarray
+        The complemented chi-squared distribution function with parameter df at
+        x.
+
+    See Also
+    --------
+    :func:`scipy.special.chdtrc`
+
+    """,
+)
+
+chdtri = _core.create_ufunc(
+    "cupyx_scipy_chdtri",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(chdtri(in0, in1));",
+    preamble=_igami_preamble + chdtri_definition,
+    doc="""Inverse to `chdtrc` with respect to `x`.
+
+    Parameters
+    ----------
+    v : cupy.ndarray
+        Degrees of freedom.
+    p : cupy.ndarray
+        Probability.
+    p : cupy.ndarray, optional
+        Optional output array for the function results.
+
+    Returns
+    -------
+    x : cupy.ndarray
+        Value so that the probability a Chi square random variable with `v`
+        degrees of freedom is greater than `x` equals `p`.
+
+    See Also
+    --------
+    :func:`scipy.special.chdtri`
+
+    """,
+)
+
+
+chdtr = _core.create_ufunc(
+    "cupyx_scipy_chdtr",
+    ("ff->f", "dd->d"),
+    "out0 = out0_type(chdtr(in0, in1));",
+    preamble=_igam_preamble + chdtr_definition,
+    doc="""Chi-square cumulative distribution function.
+
+    Parameters
+    ----------
+    v : cupy.ndarray
+        Degrees of freedom.
+    x : cupy.ndarray
+        Upper bound of the integral (nonnegative float).
+
+    Returns
+    -------
+    y : cupy.ndarray
+        The CDF of the chi-squared distribution with parameter df at x.
+
+    See Also
+    --------
+    :func:`scipy.special.chdtr`
+
+    """,
+)
+
+
+# F distribution functions
+
+fdtrc_definition = """
+
+__device__ double fdtrc(double a, double b, double x)
+{
+    double w;
+
+    if ((a <= 0.0) || (b <= 0.0) || (x < 0.0)) {
+        // sf_error("fdtrc", SF_ERROR_DOMAIN, NULL);
+        return CUDART_NAN;
+    }
+    w = b / (b + a * x);
+    return incbet(0.5 * b, 0.5 * a, w);
+}
+"""
+
+
+fdtr_definition = """
+
+__device__ double fdtr(double a, double b, double x)
+{
+    double w;
+
+    if ((a <= 0.0) || (b <= 0.0) || (x < 0.0)) {
+        // sf_error("fdtr", SF_ERROR_DOMAIN, NULL);
+        return CUDART_NAN;
+    }
+    w = a * x;
+    w = w / (b + w);
+    return incbet(0.5 * a, 0.5 * b, w);
+}
+"""
+
+
+fdtri_definition = """
+__device__ double fdtri(double a, double b, double y)
+{
+    double w, x;
+
+    if ((a <= 0.0) || (b <= 0.0) || (y <= 0.0) || (y > 1.0)) {
+        // sf_error("fdtri", SF_ERROR_DOMAIN, NULL);
+        return CUDART_NAN;
+    }
+    y = 1.0 - y;
+    /* Compute probability for x = 0.5.  */
+    w = incbet(0.5 * b, 0.5 * a, 0.5);
+    /* If that is greater than y, then the solution w < .5.
+     * Otherwise, solve at 1-y to remove cancellation in (b - b*w).  */
+    if (w > y || y < 0.001) {
+        w = incbi(0.5 * b, 0.5 * a, y);
+        x = (b - b * w) / (a * w);
+    }
+    else {
+        w = incbi(0.5 * a, 0.5 * b, 1.0 - y);
+        x = b * w / (a * (1.0 - w));
+    }
+    return x;
+}
+"""
+
+
+fdtrc = _core.create_ufunc(
+    "cupyx_scipy_fdtrc",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(fdtrc(in0, in1, in2));",
+    preamble=incbi_preamble + fdtrc_definition,
+    doc="""F survival function.
+
+    Returns the complemented F-distribution function (the integral of the
+    density from x to infinity).
+
+    Parameters
+    ----------
+    dfn : cupy.ndarray
+        First parameter (positive float).
+    dfd : cupy.ndarray
+        Second parameter (positive float).
+    x : cupy.ndarray
+        Argument (nonnegative float).
+
+    Returns
+    -------
+    y : cupy.ndarray
+        The complemented F-distribution function with parameters dfn and dfd at
+        x.
+
+    .. seealso:: :meth:`scipy.special.fdtrc`
+
+    """,
+)
+
+fdtri = _core.create_ufunc(
+    "cupyx_scipy_fdtri",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(fdtri(in0, in1, in2));",
+    preamble=incbi_preamble + fdtri_definition,
+    doc="""The p-th quantile of the F-distribution.
+
+    This function is the inverse of the F-distribution CDF, `fdtr`, returning
+    the `x` such that `fdtr(dfn, dfd, x)` = `p`.
+
+    Parameters
+    ----------
+    dfn : cupy.ndarray
+        First parameter (positive float).
+    dfd : cupy.ndarray
+        Second parameter (positive float).
+    p : cupy.ndarray
+        Cumulative probability, in [0, 1].
+
+    Returns
+    -------
+    y : cupy.ndarray
+        The quantile corresponding to p.
+
+    .. seealso:: :meth:`scipy.special.fdtri`
+
+    """,
+)
+
+
+fdtr = _core.create_ufunc(
+    "cupyx_scipy_fdtr",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(fdtr(in0, in1, in2));",
+    preamble=incbi_preamble + fdtr_definition,
+    doc="""F cumulative distribution function.
+
+
+    Parameters
+    ----------
+    dfn : cupy.ndarray
+        First parameter (positive float).
+    dfd : cupy.ndarray
+        Second parameter (positive float).
+    x : cupy.ndarray
+        Argument (nonnegative float).
+
+    Returns
+    -------
+    y : cupy.ndarray
+        The CDF of the F-distribution with parameters dfn and dfd at x.
+
+    .. seealso:: :meth:`scipy.special.fdtr`
+
+    """,
+)
+
+
+# Gamma distribution functions
+
+gdtr_definition = """
+
+__device__ double gdtr(double a, double b, double x)
+{
+
+    if (x < 0.0) {
+        return CUDART_NAN;
+    }
+    return igam(b, a * x);
+}
+"""
+
+
+gdtrc_definition = """
+
+__device__ double gdtrc(double a, double b, double x)
+{
+    if (x < 0.0) {
+        return CUDART_NAN;
+    }
+    return (igamc(b, a * x));
+}
+
+"""
+
+
+gdtr = _core.create_ufunc(
+    "cupyx_scipy_gdtr",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(gdtr(in0, in1, in2));",
+    preamble=_igam_preamble + gdtr_definition,
+    doc="""Gamma distribution cumulative distribution function.
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        The rate parameter of the gamma distribution, sometimes denoted
+        beta (float). It is also the reciprocal of the scale parameter theta.
+    b : cupy.ndarray
+        The shape parameter of the gamma distribution, sometimes denoted
+        alpha (float).
+    x : cupy.ndarray
+        The quantile (upper limit of integration; float).
+
+    Returns
+    -------
+    F : cupy.ndarray
+        The CDF of the gamma distribution with parameters `a` and `b` evaluated
+        at `x`.
+
+    See Also
+    --------
+    :func:`scipy.special.gdtr`
+
+    """,
+)
+
+
+gdtrc = _core.create_ufunc(
+    "cupyx_scipy_gdtrc",
+    ("fff->f", "ddd->d"),
+    "out0 = out0_type(gdtrc(in0, in1, in2));",
+    preamble=_igam_preamble + gdtrc_definition,
+    doc="""Gamma distribution survival function.
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        The rate parameter of the gamma distribution, sometimes denoted
+        beta (float). It is also the reciprocal of the scale parameter theta.
+    b : cupy.ndarray
+        The shape parameter of the gamma distribution, sometimes denoted
+        alpha (float).
+    x : cupy.ndarray
+        The quantile (lower limit of integration; float).
+
+    Returns
+    -------
+    I : cupy.ndarray
+        The survival function of the gamma distribution with parameters `a` and
+        `b` at `x`.
+
+    See Also
+    --------
+    :func:`scipy.special.gdtrc`
+
+    """,
+)
+
+
+# Negative Binomial distribution functions
+
+nbdtr_definition = """
+
+__device__ double nbdtr(int k, int n, double p)
+{
+    double dk, dn;
+
+    if (((p < 0.0) || (p > 1.0)) || (k < 0))
+    {
+        return CUDART_NAN;
+    }
+    dk = k + 1;
+    dn = n;
+    return (incbet(dn, dk, p));
+}
+
+__device__ double nbdtr_unsafe(double k, double n, double p)
+{
+    if (isnan(k) || isnan(n))
+    {
+        return CUDART_NAN;
+    }
+    return nbdtr((int)k, (int)n, p);
+}
+
+"""
+
+
+nbdtrc_definition = """
+
+__device__ double nbdtrc(int k, int n, double p)
+{
+    double dk, dn;
+
+    if (((p < 0.0) || (p > 1.0)) || k < 0)
+    {
+        return CUDART_NAN;
+    }
+
+    dk = k + 1;
+    dn = n;
+    return (incbet(dk, dn, 1.0 - p));
+}
+
+__device__ double nbdtrc_unsafe(double k, double n, double p)
+{
+    if (isnan(k) || isnan(n))
+    {
+        return CUDART_NAN;
+    }
+    return nbdtrc((int)k, (int)n, p);
+}
+
+"""
+
+
+nbdtri_definition = """
+
+__device__ double nbdtri(int k, int n, double y)
+{
+    double dk, dn, w;
+
+    if (((y < 0.0) || (y > 1.0)) || (k < 0)) {
+        return CUDART_NAN;
+    }
+    dk = k + 1;
+    dn = n;
+    w = incbi(dn, dk, y);
+    return (w);
+}
+
+__device__ double nbdtri_unsafe(double k, double n, double y)
+{
+    if (isnan(k) || isnan(n))
+    {
+        return CUDART_NAN;
+    }
+    return nbdtri((int)k, (int)n, y);
+}
+
+"""
+
+# Note: as in scipy we have a safe iid->d version and unsafe ddd->d one
+nbdtr = _core.create_ufunc(
+    "cupyx_scipy_nbdtr",
+    (
+        'lld->d',
+        ('fff->f', 'out0 = out0_type(nbdtr_unsafe(in0, in1, in2));'),
+        ('ddd->d', 'out0 = nbdtr_unsafe(in0, in1, in2);'),
+    ),
+    "out0 = nbdtr(in0, in1, in2);",
+    preamble=incbet_preamble + nbdtr_definition,
+    doc="""Negative binomial distribution cumulative distribution function.
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        The maximum number of allowed failures (nonnegative int).
+    n : cupy.ndarray
+        The target number of successes (positive int).
+    p : cupy.ndarray
+        Probability of success in a single event (float).
+
+    Returns
+    -------
+    F : cupy.ndarray
+        The probability of `k` or fewer failures before `n` successes in a
+        sequence of events with individual success probability `p`.
+
+    See Also
+    --------
+    :func:`scipy.special.nbdtr`
+
+    """,
+)
+
+
+nbdtrc = _core.create_ufunc(
+    "cupyx_scipy_nbdtrc",
+    (
+        'lld->d',
+        ('fff->f', 'out0 = out0_type(nbdtrc_unsafe(in0, in1, in2));'),
+        ('ddd->d', 'out0 = nbdtrc_unsafe(in0, in1, in2);'),
+    ),
+    "out0 = nbdtrc(in0, in1, in2);",
+    preamble=incbet_preamble + nbdtrc_definition,
+    doc="""Negative binomial distribution survival function.
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        The maximum number of allowed failures (nonnegative int).
+    n : cupy.ndarray
+        The target number of successes (positive int).
+    p : cupy.ndarray
+        Probability of success in a single event (float).
+
+    Returns
+    -------
+    F : cupy.ndarray
+        The probability of ``k + 1`` or more failures before `n` successes in a
+        sequence of events with individual success probability `p`.
+
+    See Also
+    --------
+    :func:`scipy.special.nbdtrc`
+
+    """,
+)
+
+nbdtri = _core.create_ufunc(
+    "cupyx_scipy_nbdtri",
+    (
+        'lld->d',
+        ('fff->f', 'out0 = out0_type(nbdtri_unsafe(in0, in1, in2));'),
+        ('ddd->d', 'out0 = nbdtri_unsafe(in0, in1, in2);'),
+    ),
+    "out0 = nbdtri(in0, in1, in2);",
+    preamble=incbi_preamble + nbdtri_definition,
+    doc="""Inverse function to `nbdtr` with respect to `p`.
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        The maximum number of allowed failures (nonnegative int).
+    n : cupy.ndarray
+        The target number of successes (positive int).
+    y : cupy.ndarray
+        The probability of `k` or fewer failures before `n` successes (float).
+
+    Returns
+    -------
+    p : cupy.ndarray
+        Probability of success in a single event (float) such that
+        ``nbdtr(k, n, p) = y``.
+
+    See Also
+    --------
+    :func:`scipy.special.nbdtri`
+
+    """,
+)
+
+
+# Poisson distribution functions
+
+pdtr_definition = """
+
+__device__ double pdtr(double k, double m)
+{
+    double v;
+
+    if ((k < 0) || (m < 0)) {
+        return CUDART_NAN;
+    }
+    if (m == 0.0) {
+        return 1.0;
+    }
+    v = floor(k) + 1;
+    return igamc(v, m);
+}
+
+"""
+
+
+pdtrc_definition = """
+
+__device__ double pdtrc(double k, double m)
+{
+    double v;
+
+    if ((k < 0.0) || (m < 0.0)) {
+        return CUDART_NAN;
+    }
+    if (m == 0.0) {
+        return 0.0;
+    }
+    v = floor(k) + 1;
+    return igam(v, m);
+}
+
+"""
+
+
+pdtri_definition = """
+
+__device__ double pdtri(int k, double y)
+{
+    double v;
+
+    if ((k < 0) || (y < 0.0) || (y >= 1.0)) {
+        return CUDART_NAN;
+    }
+    v = k + 1;
+    return igamci(v, y);
+}
+
+__device__ double pdtri_unsafe(double k, double y)
+{
+    if (isnan(k)) {
+        return CUDART_NAN;
+    } else {
+        return pdtri((int)k, y);
+    }
+}
+
+"""
+
+
+pdtr = _core.create_ufunc(
+    "cupyx_scipy_pdtr",
+    ('ff->f', 'dd->d'),
+    "out0 = out0_type(pdtr(in0, in1));",
+    preamble=_igam_preamble + pdtr_definition,
+    doc="""Poisson cumulative distribution function.
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        Nonnegative real argument.
+    m : cupy.ndarray
+        Nonnegative real shape parameter.
+
+    Returns
+    -------
+    y : cupy.ndarray
+        Values of the Poisson cumulative distribution function.
+
+    See Also
+    --------
+    :func:`scipy.special.pdtr`
+
+    """,
+)
+
+
+pdtrc = _core.create_ufunc(
+    "cupyx_scipy_pdtrc",
+    ('ff->f', 'dd->d'),
+    "out0 = out0_type(pdtrc(in0, in1));",
+    preamble=_igam_preamble + pdtrc_definition,
+    doc="""Binomial distribution survival function.
+
+    Returns the complemented binomial distribution function (the integral of
+    the density from x to infinity).
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        Nonnegative real argument.
+    m : cupy.ndarray
+        Nonnegative real shape parameter.
+
+    Returns
+    -------
+    y : cupy.ndarray
+        The sum of the terms from k+1 to infinity of the Poisson
+        distribution.
+
+    See Also
+    --------
+    :func:`scipy.special.pdtrc`
+
+    """,
+)
+
+
+pdtri = _core.create_ufunc(
+    "cupyx_scipy_pdtri",
+    # Note order of entries here is important to match SciPy behavior
+    ('ld->d',
+     ('ff->f', 'out0 = out0_type(pdtri_unsafe(in0, in1));'),
+     ('dd->d', 'out0 = pdtri_unsafe(in0, in1);')),
+    "out0 = pdtri((int)in0, in1);",
+    preamble=_igami_preamble + pdtri_definition,
+    doc="""Inverse function to `pdtr` with respect to `m`.
+
+    Parameters
+    ----------
+    k : cupy.ndarray
+        Nonnegative real argument.
+    y : cupy.ndarray
+        Cumulative probability.
+
+    Returns
+    -------
+    m : cupy.ndarray
+        The Poisson variable `m` such that the sum from 0 to `k` of the Poisson
+        density is equal to the given probability `y`.
+
+    See Also
+    --------
+    :func:`scipy.special.pdtri`
+
+    """,
+)
diff --git a/cupyx/scipy/special/_trig.py b/cupyx/scipy/special/_trig.py
new file mode 100644
index 0000000..87fb4bb
--- /dev/null
+++ b/cupyx/scipy/special/_trig.py
@@ -0,0 +1,97 @@
+"""complex-valued trig functions adapted from SciPy's cython code:
+
+https://github.com/scipy/scipy/blob/master/scipy/special/_trig.pxd
+
+Note: The CUDA Math library defines the real-valued cospi, sinpi
+"""
+
+csinpi_definition = """
+
+#include <cupy/math_constants.h>
+
+// Compute sin(pi*z) for complex arguments
+
+__device__ complex<double> csinpi(complex<double> z)
+{
+    double x = z.real();
+    double piy = M_PI*z.imag();
+    double abspiy = abs(piy);
+    double sinpix = sinpi(x);
+    double cospix = cospi(x);
+    double exphpiy, coshfac, sinhfac;
+
+    if (abspiy < 700) {
+        return complex<double>(sinpix*cosh(piy), cospix*sinh(piy));
+    }
+
+    /* Have to be careful--sinh/cosh could overflow while cos/sin are
+     * small. At this large of values
+     *
+     * cosh(y) ~ exp(y)/2
+     * sinh(y) ~ sgn(y)*exp(y)/2
+     *
+     * so we can compute exp(y/2), scale by the right factor of sin/cos
+     * and then multiply by exp(y/2) to avoid overflow.
+     */
+    exphpiy = exp(abspiy/2.0);
+    if (exphpiy == CUDART_INF) {
+        if (sinpix == 0.0) {
+            // Preserve the sign of zero
+            coshfac = copysign(0.0, sinpix);
+        } else {
+            coshfac = copysign(CUDART_INF, sinpix);
+        }
+        if (cospix == 0.0) {
+            sinhfac = copysign(0.0, cospix);
+        } else {
+            sinhfac = copysign(CUDART_INF, cospix);
+        }
+        return complex<double>(coshfac, sinhfac);
+    }
+    coshfac = 0.5*sinpix*exphpiy;
+    sinhfac = 0.5*cospix*exphpiy;
+    return complex<double>(coshfac*exphpiy, sinhfac*exphpiy);
+}
+"""
+
+
+ccospi_definition = """
+
+#include <cupy/math_constants.h>
+
+// Compute cos(pi*z) for complex arguments
+
+__device__ complex<double> csinpi(complex<double> z)
+{
+    double x = z.real();
+    double piy = M_PI*z.imag();
+    double abspiy = abs(piy);
+    double sinpix = sinpi(x);
+    double cospix = cospi(x);
+    double exphpiy, coshfac, sinhfac;
+
+    if (abspiy < 700) {
+        return complex<double>(cospix*cosh(piy), -sinpix*sinh(piy));
+    }
+
+    // See csinpi(z) for an idea of what's going on here
+    exphpiy = exp(abspiy/2.0);
+    if (exphpiy == CUDART_INF) {
+        if (sinpix == 0.0) {
+            // Preserve the sign of zero
+            coshfac = copysign(0.0, cospix);
+        } else {
+            coshfac = copysign(CUDART_INF, cospix);
+        }
+        if (cospix == 0.0) {
+            sinhfac = copysign(0.0, sinpix);
+        } else {
+            sinhfac = copysign(CUDART_INF, sinpix);
+        }
+        return complex<double>(coshfac, sinhfac);
+    }
+    coshfac = 0.5*cospix*exphpiy;
+    sinhfac = 0.5*sinpix*exphpiy;
+    return complex<double>(coshfac*exphpiy, sinhfac*exphpiy);
+}
+"""
diff --git a/cupyx/scipy/special/_xlogy.py b/cupyx/scipy/special/_xlogy.py
new file mode 100644
index 0000000..54c2f1b
--- /dev/null
+++ b/cupyx/scipy/special/_xlogy.py
@@ -0,0 +1,68 @@
+from cupy import _core
+
+
+# Note: complex-valued isnan, log and log1p are all defined in
+#       cupy/_core/include/cupy/complex.cuh
+xlogy_definition = """
+
+template <typename T>
+static __device__ T xlogy(T x, T y) {
+    if ((x == (T)0.0) && !isnan(y)) {
+        return (T)0.0;
+    } else {
+        return x * log(y);
+    }
+}
+
+"""
+
+
+# Note: SciPy only defines dd->d and DD->D
+xlogy = _core.create_ufunc(
+    'cupy_xlogy',
+    ('ee->f', 'ff->f', 'dd->d', 'FF->F', 'DD->D'),
+    'out0 = out0_type(xlogy(in0, in1));',
+    preamble=xlogy_definition,
+    doc='''Compute ``x*log(y)`` so that the result is 0 if ``x = 0``.
+
+    Args:
+        x (cupy.ndarray): input data
+
+    Returns:
+        cupy.ndarray: values of ``x * log(y)``
+
+    .. seealso:: :data:`scipy.special.xlogy`
+
+    ''')
+
+
+xlog1py_definition = """
+
+template <typename T>
+static __device__ T xlog1py(T x, T y) {
+    if ((x == (T)0.0) && ~isnan(y)) {
+        return (T)0.0;
+    } else {
+        return x * log1p(y);
+    }
+}
+
+"""
+
+# Note: SciPy only defines dd->d and DD->D
+xlog1py = _core.create_ufunc(
+    'cupy_xlog1py',
+    ('ee->f', 'ff->f', 'dd->d', 'FF->F', 'DD->D'),
+    'out0 = out0_type(xlog1py(in0, in1));',
+    preamble=xlog1py_definition,
+    doc='''Compute ``x*log1p(y)`` so that the result is 0 if ``x = 0``.
+
+    Args:
+        x (cupy.ndarray): input data
+
+    Returns:
+        cupy.ndarray: values of ``x * log1p(y)``
+
+    .. seealso:: :data:`scipy.special.xlog1py`
+
+    ''')
diff --git a/cupyx/scipy/special/_zeta.py b/cupyx/scipy/special/_zeta.py
new file mode 100644
index 0000000..caff3b4
--- /dev/null
+++ b/cupyx/scipy/special/_zeta.py
@@ -0,0 +1,129 @@
+# This source code contains SciPy's code.
+# https://github.com/scipy/scipy/blob/master/scipy/special/cephes/zeta.c
+#
+#
+# Cephes Math Library Release 2.0:  April, 1987
+# Copyright 1984, 1987 by Stephen L. Moshier
+# Direct inquiries to 30 Frost Street, Cambridge, MA 02140
+
+# TODO(YoshikawaMasashi): float implementation of zeta function
+
+from cupy import _core
+
+
+zeta_definition = '''
+/* Expansion coefficients
+ * for Euler-Maclaurin summation formula
+ * (2k)! / B2k
+ * where B2k are Bernoulli numbers
+ */
+__constant__ double A[] = {
+    12.0,
+    -720.0,
+    30240.0,
+    -1209600.0,
+    47900160.0,
+    -1.8924375803183791606e9,	/*1.307674368e12/691 */
+    7.47242496e10,
+    -2.950130727918164224e12,	/*1.067062284288e16/3617 */
+    1.1646782814350067249e14,	/*5.109094217170944e18/43867 */
+    -4.5979787224074726105e15,	/*8.028576626982912e20/174611 */
+    1.8152105401943546773e17,	/*1.5511210043330985984e23/854513 */
+    -7.1661652561756670113e18	/*1.6938241367317436694528e27/236364091 */
+};
+
+__constant__ double MACHEP = 1.11022302462515654042E-16;
+
+/* 30 Nov 86 -- error in third coefficient fixed */
+
+
+double __device__ zeta(double x, double q)
+{
+    int i;
+    double a, b, k, s, t, w;
+
+    if (x == 1.0){
+        return 1.0/0.0;
+    }
+
+    if (x < 1.0) {
+        return nan("");
+    }
+
+    if (q <= 0.0) {
+        if (q == floor(q)) {
+            return 1.0/0.0;
+        }
+        if (x != floor(x)){
+            return nan("");	/* because q^-x not defined */
+        }
+    }
+
+    /* Asymptotic expansion
+     * http://dlmf.nist.gov/25.11#E43
+     */
+    if (q > 1e8) {
+        return (1/(x - 1) + 1/(2*q)) * pow(q, 1 - x);
+    }
+
+    /* Euler-Maclaurin summation formula */
+
+    /* Permit negative q but continue sum until n+q > +9 .
+     * This case should be handled by a reflection formula.
+     * If q<0 and x is an integer, there is a relation to
+     * the polyGamma function.
+     */
+    s = pow(q, -x);
+    a = q;
+    i = 0;
+    b = 0.0;
+    while ((i < 9) || (a <= 9.0)) {
+        i += 1;
+        a += 1.0;
+        b = pow(a, -x);
+        s += b;
+        if (fabs(b / s) < MACHEP){
+            return s;
+        }
+    }
+
+    w = a;
+    s += b * w / (x - 1.0);
+    s -= 0.5 * b;
+    a = 1.0;
+    k = 0.0;
+    for (i = 0; i < 12; i++) {
+        a *= x + k;
+        b /= w;
+        t = a * b / A[i];
+        s = s + t;
+        t = fabs(t / s);
+        if (t < MACHEP){
+            return s;
+        }
+        k += 1.0;
+        a *= x + k;
+        b /= w;
+        k += 1.0;
+    }
+    return s;
+}
+'''
+
+
+zeta = _core.create_ufunc(
+    'cupyx_scipy_special_zeta', ('ff->f', 'dd->d'),
+    'out0 = zeta(in0, in1)',
+    preamble=zeta_definition,
+    doc="""Hurwitz zeta function.
+
+    Args:
+        x (cupy.ndarray): Input data, must be real.
+        q (cupy.ndarray): Input data, must be real.
+
+    Returns:
+        cupy.ndarray: Values of zeta(x, q).
+
+    .. seealso:: :data:`scipy.special.zeta`
+
+    """)
diff --git a/cupyx/scipy/stats/__init__.py b/cupyx/scipy/stats/__init__.py
new file mode 100644
index 0000000..14b1d17
--- /dev/null
+++ b/cupyx/scipy/stats/__init__.py
@@ -0,0 +1,11 @@
+# Summary statistics
+
+from cupyx.scipy.stats._distributions import entropy  # NOQA
+from cupyx.scipy.stats._stats import trim_mean  # NOQA
+
+
+# Other statistical functionality
+
+from cupyx.scipy.stats._morestats import boxcox_llf  # NOQA
+from cupyx.scipy.stats._stats_py import zmap  # NOQA
+from cupyx.scipy.stats._stats_py import zscore  # NOQA
diff --git a/cupyx/scipy/stats/_distributions.py b/cupyx/scipy/stats/_distributions.py
new file mode 100644
index 0000000..089a86c
--- /dev/null
+++ b/cupyx/scipy/stats/_distributions.py
@@ -0,0 +1,59 @@
+import math
+
+import cupy
+from cupyx.scipy import special
+
+
+def _normalize(x, axis):
+    """Normalize, preserving floating point precision of x."""
+    x_sum = x.sum(axis=axis, keepdims=True)
+    if x.dtype.kind == 'f':
+        x /= x_sum
+    else:
+        x = x / x_sum
+    return x
+
+
+def entropy(pk, qk=None, base=None, axis=0):
+    """Calculate the entropy of a distribution for given probability values.
+
+    If only probabilities ``pk`` are given, the entropy is calculated as
+    ``S = -sum(pk * log(pk), axis=axis)``.
+
+    If ``qk`` is not None, then compute the Kullback-Leibler divergence
+    ``S = sum(pk * log(pk / qk), axis=axis)``.
+
+    This routine will normalize ``pk`` and ``qk`` if they don't sum to 1.
+
+    Args:
+        pk (ndarray): Defines the (discrete) distribution. ``pk[i]`` is the
+            (possibly unnormalized) probability of event ``i``.
+        qk (ndarray, optional): Sequence against which the relative entropy is
+            computed. Should be in the same format as ``pk``.
+        base (float, optional): The logarithmic base to use, defaults to ``e``
+            (natural logarithm).
+        axis (int, optional): The axis along which the entropy is calculated.
+            Default is 0.
+
+    Returns:
+        S (cupy.ndarray): The calculated entropy.
+
+    """
+    if pk.dtype.kind == 'c' or qk is not None and qk.dtype.kind == 'c':
+        raise TypeError("complex dtype not supported")
+
+    float_type = cupy.float32 if pk.dtype.char in 'ef' else cupy.float64
+    pk = pk.astype(float_type, copy=False)
+    pk = _normalize(pk, axis)
+    if qk is None:
+        vec = special.entr(pk)
+    else:
+        if qk.shape != pk.shape:
+            raise ValueError("qk and pk must have same shape.")
+        qk = qk.astype(float_type, copy=False)
+        qk = _normalize(qk, axis)
+        vec = special.rel_entr(pk, qk)
+    s = cupy.sum(vec, axis=axis)
+    if base is not None:
+        s /= math.log(base)
+    return s
diff --git a/cupyx/scipy/stats/_morestats.py b/cupyx/scipy/stats/_morestats.py
new file mode 100644
index 0000000..88be820
--- /dev/null
+++ b/cupyx/scipy/stats/_morestats.py
@@ -0,0 +1,47 @@
+import cupy
+
+
+def boxcox_llf(lmb, data):
+    """The boxcox log-likelihood function.
+
+    Parameters
+    ----------
+    lmb : scalar
+        Parameter for Box-Cox transformation
+    data : array-like
+        Data to calculate Box-Cox log-likelihood for. If
+        `data` is multi-dimensional, the log-likelihood
+        is calculated along the first axis
+
+    Returns
+    -------
+    llf : float or cupy.ndarray
+        Box-Cox log-likelihood of `data` given `lmb`. A float
+        for 1-D `data`, an array otherwise
+
+    See Also
+    --------
+    scipy.stats.boxcox_llf
+
+    """
+
+    if data.ndim == 1 and data.dtype == cupy.float16:
+        data = data.astype(cupy.float64)
+    if data.ndim == 1 and data.dtype == cupy.float32:
+        data = data.astype(cupy.float64)
+    if data.ndim == 1 and data.dtype == cupy.complex64:
+        data = data.astype(cupy.complex128)
+
+    N = data.shape[0]
+    if N == 0:
+        return cupy.array(cupy.nan)
+
+    logdata = cupy.log(data)
+
+    # Compute the variance of the transformed data
+    if lmb == 0:
+        variance = cupy.var(logdata, axis=0)
+    else:
+        variance = cupy.var(data**lmb / lmb, axis=0)
+
+    return (lmb - 1) * cupy.sum(logdata, axis=0) - N/2 * cupy.log(variance)
diff --git a/cupyx/scipy/stats/_stats.py b/cupyx/scipy/stats/_stats.py
new file mode 100644
index 0000000..e8b30ec
--- /dev/null
+++ b/cupyx/scipy/stats/_stats.py
@@ -0,0 +1,77 @@
+"""
+A collection of basic statistical functions for Python.
+
+References
+----------
+.. [CRCProbStat2000] Zwillinger, D. and Kokoska, S. (2000). CRC Standard
+   Probability and Statistics Tables and Formulae. Chapman & Hall: New
+   York. 2000.
+"""
+import cupy as cp
+
+
+def trim_mean(a, proportiontocut, axis=0):
+    """Return mean of array after trimming distribution from both tails.
+
+    If `proportiontocut` = 0.1, slices off 'leftmost' and 'rightmost' 10% of
+    scores. The input is sorted before slicing. Slices off less if proportion
+    results in a non-integer slice index (i.e., conservatively slices off
+    `proportiontocut` ).
+
+    Parameters
+    ----------
+    a : cupy.ndarray
+        Input array.
+    proportiontocut : float
+        Fraction to cut off of both tails of the distribution.
+    axis : int or None, optional
+        Axis along which the trimmed means are computed. Default is 0.
+        If None, compute over the whole array `a`.
+
+    Returns
+    -------
+    trim_mean : ndarray
+        Mean of trimmed array.
+
+    See Also
+    --------
+    trimboth
+    tmean : Compute the trimmed mean ignoring values outside given `limits`.
+
+    Examples
+    --------
+    >>> import cupy as cp
+    >>> from cupyx.scipy import stats
+    >>> x = cp.arange(20)
+    >>> stats.trim_mean(x, 0.1)
+    array(9.5)
+    >>> x2 = x.reshape(5, 4)
+    >>> x2
+    array([[ 0,  1,  2,  3],
+           [ 4,  5,  6,  7],
+           [ 8,  9, 10, 11],
+           [12, 13, 14, 15],
+           [16, 17, 18, 19]])
+    >>> stats.trim_mean(x2, 0.25)
+    array([ 8.,  9., 10., 11.])
+    >>> stats.trim_mean(x2, 0.25, axis=1)
+    array([ 1.5,  5.5,  9.5, 13.5, 17.5])
+    """
+    if a.size == 0:
+        return cp.nan
+
+    if axis is None:
+        a = a.ravel()
+        axis = 0
+
+    nobs = a.shape[axis]
+    lowercut = int(proportiontocut * nobs)
+    uppercut = nobs - lowercut
+    if (lowercut > uppercut):
+        raise ValueError("Proportion too big.")
+
+    atmp = cp.partition(a, (lowercut, uppercut - 1), axis)
+
+    sl = [slice(None)] * atmp.ndim
+    sl[axis] = slice(lowercut, uppercut)
+    return cp.mean(atmp[tuple(sl)], axis=axis)
diff --git a/cupyx/scipy/stats/_stats_py.py b/cupyx/scipy/stats/_stats_py.py
new file mode 100644
index 0000000..2525404
--- /dev/null
+++ b/cupyx/scipy/stats/_stats_py.py
@@ -0,0 +1,138 @@
+import cupy
+
+
+def _first(arr, axis):
+    """Return arr[..., 0:1, ...] where 0:1 is in the `axis` position
+
+    """
+
+    return cupy.take_along_axis(arr, cupy.array(0, ndmin=arr.ndim), axis)
+
+
+def _isconst(x):
+    """Check if all values in x are the same.  nans are ignored.
+    x must be a 1d array. The return value is a 1d array
+    with length 1, so it can be used in cupy.apply_along_axis.
+
+    """
+
+    y = x[~cupy.isnan(x)]
+    if y.size == 0:
+        return cupy.array([True])
+    else:
+        return (y[0] == y).all(keepdims=True)
+
+
+def zscore(a, axis=0, ddof=0, nan_policy='propagate'):
+    """Compute the z-score.
+
+    Compute the z-score of each value in the sample, relative to
+    the sample mean and standard deviation.
+
+    Parameters
+    ----------
+    a : array-like
+        An array like object containing the sample data
+    axis : int or None, optional
+        Axis along which to operate. Default is 0. If None,
+        compute over the whole arrsy `a`
+    ddof : int, optional
+        Degrees of freedom correction in the calculation of the
+        standard deviation. Default is 0
+    nan_policy : {'propagate', 'raise', 'omit'}, optional
+        Defines how to handle when input contains nan. 'propagate'
+        returns nan, 'raise' throws an error, 'omit' performs
+        the calculations ignoring nan values. Default is
+        'propagate'. Note that when the value is 'omit',
+        nans in the input also propagate to the output,
+        but they do not affect the z-scores computed
+        for the non-nan values
+
+    Returns
+    -------
+    zscore : array-like
+        The z-scores, standardized by mean and standard deviation of
+        input array `a`
+
+    """
+
+    return zmap(a, a, axis=axis, ddof=ddof, nan_policy=nan_policy)
+
+
+def zmap(scores, compare, axis=0, ddof=0, nan_policy='propagate'):
+    """Calculate the relative z-scores.
+
+    Return an array of z-scores, i.e., scores that are standardized
+    to zero mean and unit variance, where mean and variance are
+    calculated from the comparison array.
+
+    Parameters
+    ----------
+    scores : array-like
+        The input for which z-scores are calculated
+    compare : array-like
+        The input from which the mean and standard deviation of
+        the normalization are taken; assumed to have the same
+        dimension as `scores`
+    axis : int or None, optional
+        Axis over which mean and variance of `compare` are calculated.
+        Default is 0. If None, compute over the whole array `scores`
+    ddof : int, optional
+        Degrees of freedom correction in the calculation of the
+        standard deviation. Default is 0
+    nan_policy : {'propagate', 'raise', 'omit'}, optional
+        Defines how to handle the occurrence of nans in `compare`.
+        'propagate' returns nan, 'raise' raises an exception, 'omit'
+        performs the calculations ignoring nan values. Default is
+        'propagate'. Note that when the value is 'omit', nans in `scores`
+        also propagate to the output, but they do not affect the z-scores
+        computed for the non-nan values
+
+    Returns
+    -------
+    zscore : array-like
+        Z-scores, in the same shape as `scores`
+
+    """
+
+    policies = ['propagate', 'raise', 'omit']
+
+    if nan_policy not in policies:
+        raise ValueError("nan_policy must be one of {%s}" %
+                         ', '.join("'%s'" % s for s in policies))
+
+    a = compare
+
+    if a.size == 0:
+        return cupy.empty(a.shape)
+
+    if nan_policy == 'raise':
+        contains_nan = cupy.isnan(cupy.sum(a))
+
+        if contains_nan:  # synchronize!
+            raise ValueError("The input contains nan values")
+
+    if nan_policy == 'omit':
+        if axis is None:
+            mn = cupy.nanmean(a.ravel())
+            std = cupy.nanstd(a.ravel(), ddof=ddof)
+            isconst = _isconst(a.ravel())
+        else:
+            mn = cupy.nanmean(a, axis=axis, keepdims=True)
+            std = cupy.nanstd(a, axis=axis, keepdims=True, ddof=ddof)
+            isconst = (_first(a, axis) == a).all(axis=axis, keepdims=True)
+    else:
+        mn = a.mean(axis=axis, keepdims=True)
+        std = a.std(axis=axis, ddof=ddof, keepdims=True)
+        if axis is None:
+            isconst = (a.ravel()[0] == a).all()
+        else:
+            isconst = (_first(a, axis) == a).all(axis=axis, keepdims=True)
+
+    # Set std deviations that are 0 to 1 to avoid division by 0.
+    std[isconst] = 1.0
+    z = (scores - mn) / std
+
+    # Set the outputs associated with a constant input to nan.
+    z[cupy.broadcast_to(isconst, z.shape)] = cupy.nan
+    return z
diff --git a/cupyx/time.py b/cupyx/time.py
new file mode 100644
index 0000000..e06fe49
--- /dev/null
+++ b/cupyx/time.py
@@ -0,0 +1,90 @@
+import math as _math
+import warnings as _warnings
+
+import numpy as _numpy
+
+import cupy as _cupy
+from cupyx.profiler._time import _repeat, _PerfCaseResult  # for tests  # NOQA
+
+
+# TODO(leofang): remove this function in CuPy v11
+def repeat(
+        func, args=(), kwargs={}, n_repeat=10000, *,
+        name=None, n_warmup=10, max_duration=_math.inf, devices=None):
+    """ Timing utility for measuring time spent by both CPU and GPU.
+
+    This function is a very convenient helper for setting up a timing test. The
+    GPU time is properly recorded by synchronizing internal streams. As a
+    result, to time a multi-GPU function all participating devices must be
+    passed as the ``devices`` argument so that this helper knows which devices
+    to record. A simple example is given as follows:
+
+    .. code-block:: py
+
+        import cupy as cp
+        from cupyx.time import repeat
+
+        def f(a, b):
+            return 3 * cp.sin(-a) * b
+
+        a = 0.5 - cp.random.random((100,))
+        b = cp.random.random((100,))
+        print(repeat(f, (a, b), n_repeat=1000))
+
+
+    Args:
+        func (callable): a callable object to be timed.
+        args (tuple): positional argumens to be passed to the callable.
+        kwargs (dict): keyword arguments to be passed to the callable.
+        n_repeat (int): number of times the callable is called. Increasing
+            this value would improve the collected statistics at the cost
+            of longer test time.
+        name (str): the function name to be reported. If not given, the
+            callable's ``__name__`` attribute is used.
+        n_warmup (int): number of times the callable is called. The warm-up
+            runs are not timed.
+        max_duration (float): the maximum time (in seconds) that the entire
+            test can use. If the taken time is longer than this limit, the test
+            is stopped and the statistics collected up to the breakpoint is
+            reported.
+        devices (tuple): a tuple of device IDs (int) that will be timed during
+            the timing test. If not given, the current device is used.
+
+    Returns:
+        :class:`~cupyx.profiler._time._PerfCaseResult`:
+            an object collecting all test results.
+
+    .. warning::
+        This API is moved to :func:`cupyx.profiler.benchmark` since CuPy v10.
+        Access through ``cupyx.time`` is deprecated.
+    """
+
+    _warnings.warn(
+        'cupyx.time.repeat has been moved to cupyx.profiler.benchmark since '
+        'CuPy v10. Access through cupyx.time is deprecated and will be '
+        'removed in the future.')
+    if name is None:
+        name = func.__name__
+
+    if devices is None:
+        devices = (_cupy.cuda.get_device_id(),)
+
+    if not callable(func):
+        raise ValueError('`func` should be a callable object.')
+    if not isinstance(args, tuple):
+        raise ValueError('`args` should be of tuple type.')
+    if not isinstance(kwargs, dict):
+        raise ValueError('`kwargs` should be of dict type.')
+    if not isinstance(n_repeat, int):
+        raise ValueError('`n_repeat` should be an integer.')
+    if not isinstance(name, str):
+        raise ValueError('`name` should be a string.')
+    if not isinstance(n_warmup, int):
+        raise ValueError('`n_warmup` should be an integer.')
+    if not _numpy.isreal(max_duration):
+        raise ValueError('`max_duration` should be given in seconds')
+    if not isinstance(devices, tuple):
+        raise ValueError('`devices` should be of tuple type')
+
+    return _repeat(
+        func, args, kwargs, n_repeat, name, n_warmup, max_duration, devices)
diff --git a/cupyx/tools/__init__.py b/cupyx/tools/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cupyx/tools/_hipsparse_stub_mapper.py b/cupyx/tools/_hipsparse_stub_mapper.py
new file mode 100644
index 0000000..bc05d70
--- /dev/null
+++ b/cupyx/tools/_hipsparse_stub_mapper.py
@@ -0,0 +1,402 @@
+import urllib.request
+import sys
+from typing import Tuple
+
+
+# Take cupy_backends/stub/cupy_cusparse.h and generate
+# cupy_backends/hip/cupy_hipsparse.h, with all return values replaced by an
+# error if not supprted. This script mainly focuses on getting the CUDA ->
+# HIP API mapping done correctly; structs, enums, etc, are handled
+# automatically to the maximal extent.
+#
+# The stub functions, such as this,
+#
+# cusparseStatus_t cusparseDestroyMatDescr(...) {
+#   return HIPSPARSE_STATUS_NOT_SUPPORTED;
+# }
+#
+# are mapped to their HIP counterparts, like this
+#
+# cusparseStatus_t cusparseDestroyMatDescr(cusparseMatDescr_t descrA) {
+#   return hipsparseDestroyMatDescr(descrA);
+# }
+
+
+# some cuSPARSE APIs are removed in recent CUDA 11.x, so we also need to
+# look up older CUDA to fetch the API signatures
+cusparse_h = '/usr/local/cuda-{0}/include/cusparse.h'
+cu_versions = ('11.3', '11.0', '10.2')
+
+
+hipsparse_url = ('https://raw.githubusercontent.com/ROCmSoftwarePlatform/'
+                 'hipSPARSE/rocm-{0}/library/include/hipsparse.h')
+hip_versions = ("3.5.0", "3.7.0", "3.8.0", "3.9.0", "4.0.0", "4.2.0")
+
+
+# typedefs
+typedefs: Tuple[str, ...]
+typedefs = ('cusparseIndexBase_t', 'cusparseStatus_t', 'cusparseHandle_t',
+            'cusparseMatDescr_t', 'csrsv2Info_t', 'csrsm2Info_t',
+            'csric02Info_t', 'bsric02Info_t', 'csrilu02Info_t',
+            'bsrilu02Info_t', 'csrgemm2Info_t',
+            'cusparseMatrixType_t', 'cusparseFillMode_t', 'cusparseDiagType_t',
+            'cusparsePointerMode_t', 'cusparseAction_t', 'cusparseDirection_t',
+            'cusparseSolvePolicy_t', 'cusparseOperation_t')
+
+
+# typedefs for generic API
+typedefs += ('cusparseSpVecDescr_t', 'cusparseDnVecDescr_t',
+             'cusparseSpMatDescr_t', 'cusparseDnMatDescr_t',
+             'cusparseIndexType_t', 'cusparseFormat_t', 'cusparseOrder_t',
+             'cusparseSpMVAlg_t', 'cusparseSpMMAlg_t',
+             'cusparseSparseToDenseAlg_t', 'cusparseDenseToSparseAlg_t',
+             'cusparseCsr2CscAlg_t',)
+
+
+# helpers
+cudaDataType_converter = r"""
+#if HIP_VERSION >= 402
+static hipDataType convert_hipDatatype(cudaDataType type) {
+    switch(static_cast<int>(type)) {
+        case 2 /* CUDA_R_16F */: return HIP_R_16F;
+        case 0 /* CUDA_R_32F */: return HIP_R_32F;
+        case 1 /* CUDA_R_64F */: return HIP_R_64F;
+        case 6 /* CUDA_C_16F */: return HIP_C_16F;
+        case 4 /* CUDA_C_32F */: return HIP_C_32F;
+        case 5 /* CUDA_C_64F */: return HIP_C_64F;
+        default: throw std::runtime_error("unrecognized type");
+    }
+}
+#endif
+"""
+
+cusparseOrder_converter = r"""
+#if HIP_VERSION >= 402
+typedef enum {} cusparseOrder_t;
+static hipsparseOrder_t convert_hipsparseOrder_t(cusparseOrder_t type) {
+    switch(static_cast<int>(type)) {
+        case 1 /* CUSPARSE_ORDER_COL */: return HIPSPARSE_ORDER_COLUMN;
+        case 2 /* CUSPARSE_ORDER_ROW */: return HIPSPARSE_ORDER_ROW;
+        default: throw std::runtime_error("unrecognized type");
+    }
+}
+"""
+
+default_return_code = r"""
+#if HIP_VERSION < 401
+#define HIPSPARSE_STATUS_NOT_SUPPORTED (hipsparseStatus_t)10
+#endif
+"""
+
+
+# keep track of typedefs that are already handled (as we move from older
+# to newer HIP version)
+processed_typedefs = set()
+
+
+def get_idx_to_func(cu_h, cu_func):
+    cu_sig = cu_h.find(cu_func)
+    # 1. function names are always followed immediately by a "("
+    # 2. we need a loop here to find the exact match
+    while True:
+        if cu_sig == -1:
+            break
+        elif cu_h[cu_sig+len(cu_func)] != "(":
+            cu_sig = cu_h.find(cu_func, cu_sig+1)
+        else:
+            break  # match
+    return cu_sig
+
+
+def get_hip_ver_num(hip_version):
+    # "3.5.0" -> 305
+    hip_version = hip_version.split('.')
+    return int(hip_version[0]) * 100 + int(hip_version[1])
+
+
+def merge_bad_broken_lines(cu_sig):
+    # each line ends with an argument, which is followed by a "," or ")"
+    # except for cusparseCbsric02, cusparseCooGet, ...
+    cu_sig_processed = []
+    skip_line = None
+    for line, s in enumerate(cu_sig):
+        if line != skip_line:
+            if s.endswith(',') or s.endswith(')'):
+                cu_sig_processed.append(s)
+            else:
+                break_idx = s.find(',')
+                if break_idx == -1:
+                    break_idx = s.find(')')
+                if break_idx == -1:
+                    cu_sig_processed.append(s + cu_sig[line+1])
+                    skip_line = line+1
+                else:
+                    # argument could be followed by an inline comment
+                    cu_sig_processed.append(s[:break_idx+1])
+    return cu_sig_processed
+
+
+def process_func_args(s, hip_sig, decl, hip_func):
+    # TODO(leofang): prettier print? note that we currently rely on "hip_sig"
+    # being a one-liner...
+    # TODO(leofang): I am being silly here; these can probably be handled more
+    # elegantly using regex...
+    if 'const cuComplex*' in s:
+        s = s.split()
+        arg = '(' + s[-1][:-1] + ')' + s[-1][-1]
+        cast = 'reinterpret_cast<const hipComplex*>'
+    elif 'const cuDoubleComplex*' in s:
+        s = s.split()
+        arg = '(' + s[-1][:-1] + ')' + s[-1][-1]
+        cast = 'reinterpret_cast<const hipDoubleComplex*>'
+    elif 'cuComplex*' in s:
+        s = s.split()
+        arg = '(' + s[-1][:-1] + ')' + s[-1][-1]
+        cast = 'reinterpret_cast<hipComplex*>'
+    elif 'cuDoubleComplex*' in s:
+        s = s.split()
+        arg = '(' + s[-1][:-1] + ')' + s[-1][-1]
+        cast = 'reinterpret_cast<hipDoubleComplex*>'
+    elif 'cuComplex' in s:
+        s = s.split()
+        decl += '  hipComplex blah;\n'
+        decl += f'  blah.x={s[-1][:-1]}.x;\n  blah.y={s[-1][:-1]}.y;\n'
+        arg = 'blah' + s[-1][-1]
+        cast = ''
+    elif 'cuDoubleComplex' in s:
+        s = s.split()
+        decl += '  hipDoubleComplex blah;\n'
+        decl += f'  blah.x={s[-1][:-1]}.x;\n  blah.y={s[-1][:-1]}.y;\n'
+        arg = 'blah' + s[-1][-1]
+        cast = ''
+    elif 'cudaDataType*' in s:
+        s = s.split()
+        arg = '(' + s[-1][:-1] + ')' + s[-1][-1]
+        cast = 'reinterpret_cast<hipDataType*>'
+    elif 'cudaDataType' in s:
+        s = s.split()
+        decl += '  hipDataType blah = convert_hipDatatype('
+        decl += s[-1][:-1] + ');\n'
+        arg = 'blah' + s[-1][-1]
+        cast = ''
+    elif 'cusparseOrder_t*' in s:
+        s = s.split()
+        decl += '  hipsparseOrder_t blah2 = '
+        decl += 'convert_hipsparseOrder_t(*' + s[-1][:-1] + ');\n'
+        arg = '&blah2' + s[-1][-1]
+        cast = ''
+    elif 'cusparseOrder_t' in s:
+        s = s.split()
+        decl += '  hipsparseOrder_t blah2 = '
+        decl += 'convert_hipsparseOrder_t(' + s[-1][:-1] + ');\n'
+        arg = 'blah2' + s[-1][-1]
+        cast = ''
+    elif ('const void*' in s
+            and hip_func == 'hipsparseSpVV_bufferSize'):
+        # work around HIP's bad typing...
+        s = s.split()
+        arg = '(' + s[-1][:-1] + ')' + s[-1][-1]
+        cast = 'const_cast<void*>'
+    else:
+        s = s.split()
+        arg = s[-1]
+        cast = ''
+    hip_sig += (cast + arg + ' ')
+    return hip_sig, decl
+
+
+def main(hip_h, cu_h, stubs, hip_version, init):
+    hip_version = get_hip_ver_num(hip_version)
+
+    # output HIP stub
+    hip_stub_h = []
+
+    for i, line in enumerate(stubs):
+        if i == 3 and not init:
+            hip_stub_h.append(line)
+            if hip_version == 305:
+                # insert the include after the include guard
+                hip_stub_h.append('#include <hipsparse.h>')
+                hip_stub_h.append(
+                    '#include <hip/hip_version.h>    // for HIP_VERSION')
+                hip_stub_h.append(
+                    '#include <hip/library_types.h>  // for hipDataType')
+                hip_stub_h.append(cudaDataType_converter)
+                hip_stub_h.append(default_return_code)
+
+        elif line.startswith('typedef'):
+            old_line = ''
+            typedef_found = False
+            typedef_needed = True
+            for t in typedefs:
+                if t in line and t not in processed_typedefs:
+                    hip_t = 'hip' + t[2:] if t.startswith('cu') else t
+                    if hip_t in hip_h:
+                        old_line = line
+                        if t != hip_t:
+                            old_line = line
+                            line = 'typedef ' + hip_t + ' ' + t + ';'
+                        else:
+                            if hip_version == 305:
+                                line = None
+                            typedef_needed = False
+                        typedef_found = True
+                    else:
+                        # new API not supported yet, use typedef from stub
+                        pass
+                    break
+            else:
+                t = None
+
+            if line is not None:
+                # hack...
+                if t == 'cusparseOrder_t' and hip_version == 402:
+                    hip_stub_h.append(cusparseOrder_converter)
+
+                elif typedef_found and hip_version > 305:
+                    if typedef_needed:
+                        hip_stub_h.append(f'#if HIP_VERSION >= {hip_version}')
+                    else:
+                        hip_stub_h.append(f'#if HIP_VERSION < {hip_version}')
+
+                # hack...
+                if not (t == 'cusparseOrder_t' and hip_version == 402):
+                    hip_stub_h.append(line)
+
+                if typedef_found and hip_version > 305:
+
+                    if typedef_needed:
+                        hip_stub_h.append('#else')
+                        hip_stub_h.append(old_line)
+                    hip_stub_h.append('#endif\n')
+
+            if t is not None and typedef_found:
+                processed_typedefs.add(t)
+
+        elif '...' in line:
+            # ex: line = "cusparseStatus_t cusparseDestroyMatDescr(...) {"
+            sig = line.split()
+            try:
+                assert len(sig) == 3
+            except AssertionError:
+                print(f"sig is {sig}")
+                raise
+
+            # strip the prefix "cu" and the args "(...)", and assume HIP has
+            # the same function
+            cu_func = sig[1]
+            cu_func = cu_func[:cu_func.find('(')]
+            hip_func = 'hip' + cu_func[2:]
+
+            # find the full signature from cuSPARSE header
+            cu_sig = get_idx_to_func(cu_h, cu_func)
+            # check if HIP has the corresponding function
+            hip_sig = get_idx_to_func(hip_h, hip_func)
+            if cu_sig == -1 and hip_sig == -1:
+                assert False
+            elif cu_sig == -1 and hip_sig != -1:
+                print(cu_func, "not found in cuSPARSE, maybe removed?",
+                      file=sys.stderr)
+                can_map = False
+            elif cu_sig != -1 and hip_sig == -1:
+                print(hip_func, "not found in hipSPARSE, maybe not supported?",
+                      file=sys.stderr)
+                can_map = False
+            else:
+                end_idx = cu_h[cu_sig:].find(')')
+                assert end_idx != -1
+                cu_sig = cu_h[cu_sig:cu_sig+end_idx+1]
+
+                # pretty print
+                cu_sig = cu_sig.split('\n')
+                new_cu_sig = cu_sig[0] + '\n'
+                for s in cu_sig[1:]:
+                    new_cu_sig += (' ' * (len(sig[0]) + 1)) + s + '\n'
+                cu_sig = new_cu_sig[:-1]
+
+                sig[1] = cu_sig
+                can_map = True
+            hip_stub_h.append(' '.join(sig))
+
+            # now we have the full signature, map the return to HIP's function;
+            # note that the "return" line is in the next two lines
+            line = stubs[i+1]
+            if 'return' not in line:
+                line = stubs[i+2]
+                assert 'return' in line
+            if can_map:
+                cu_sig = cu_sig.split('\n')
+                cu_sig = merge_bad_broken_lines(cu_sig)
+
+                if hip_version != 305:
+                    hip_stub_h.append(f"#if HIP_VERSION >= {hip_version}")
+                hip_sig = '  return ' + hip_func + '('
+                decl = ''
+                for s in cu_sig:
+                    hip_sig, decl = process_func_args(
+                        s, hip_sig, decl, hip_func)
+                hip_sig = hip_sig[:-1] + ';'
+                hip_stub_h.append(decl+hip_sig)
+                if hip_version != 305:
+                    hip_stub_h.append("#else")
+                    hip_stub_h.append(
+                        '  return HIPSPARSE_STATUS_NOT_SUPPORTED;')
+                    hip_stub_h.append("#endif")
+            else:
+                hip_stub_h.append(
+                    (line[:line.find('return')+6]
+                     + ' HIPSPARSE_STATUS_NOT_SUPPORTED;'))
+
+        elif 'return' in line:
+            if 'CUSPARSE_STATUS' in line:
+                # don't do anything, as we handle the return when
+                # parsing "(...)"
+                pass
+            elif 'HIPSPARSE_STATUS_NOT_SUPPORTED' in line:
+                if '#else' in stubs[i-1]:
+                    # just copy from the stub
+                    hip_stub_h.append(line)
+            else:
+                # just copy from the stub
+                hip_stub_h.append(line)
+
+        else:
+            # just copy from the stub
+            hip_stub_h.append(line)
+
+    return ('\n'.join(hip_stub_h)) + '\n'
+
+
+if __name__ == '__main__':
+    with open('cupy_backends/stub/cupy_cusparse.h', 'r') as f:
+        stubs = f.read()
+
+    init = False
+    for cu_ver in cu_versions:
+        with open(cusparse_h.format(cu_ver), 'r') as f:
+            cu_h = f.read()
+
+        x = 0
+        for hip_ver in hip_versions:
+            stubs_splitted = stubs.splitlines()
+
+            req = urllib.request.urlopen(hipsparse_url.format(hip_ver))
+            with req as f:
+                hip_h = f.read().decode()
+
+            stubs = main(hip_h, cu_h, stubs_splitted, hip_ver, init)
+            init = True
+
+    # more hacks...
+    stubs = stubs.replace(
+        '#define CUSPARSE_VERSION -1',
+        ('#define CUSPARSE_VERSION '
+         '(hipsparseVersionMajor*100000+hipsparseVersionMinor*100'
+         '+hipsparseVersionPatch)'))
+    stubs = stubs.replace(
+        'INCLUDE_GUARD_STUB_CUPY_CUSPARSE_H',
+        'INCLUDE_GUARD_HIP_CUPY_HIPSPARSE_H')
+    stubs = stubs[stubs.find('\n'):]
+
+    with open('cupy_backends/hip/cupy_hipsparse.h', 'w') as f:
+        f.write(stubs)
diff --git a/cupyx/tools/install_library.py b/cupyx/tools/install_library.py
new file mode 100755
index 0000000..7d3e63e
--- /dev/null
+++ b/cupyx/tools/install_library.py
@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+
+"""
+CUDA Library Installer
+
+Installs the latest CUDA library supported by CuPy.
+"""
+
+# This script will also be used as a standalone script when building wheels.
+# Keep the script runnable without CuPy dependency.
+
+import argparse
+import json
+import os
+import platform
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib.request
+
+
+_cudnn_records = []
+_cutensor_records = []
+_nccl_records = []
+library_records = {}
+
+
+def _make_cudnn_url(platform, filename):
+    # https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.8.1.3_cuda12-archive.tar.xz
+    return (
+        'https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/' +
+        f'{platform}/{filename}')
+
+
+def __make_cudnn_record(
+        cuda_version, public_version, filename_linux, filename_windows):
+    major_version = public_version.split('.')[0]
+    # Dependency order is documented at:
+    # https://docs.nvidia.com/deeplearning/cudnn/api/index.html
+    suffix_list = ['', '_ops_infer', '_ops_train',
+                   '_cnn_infer', '_cnn_train',
+                   '_adv_infer', '_adv_train']
+    return {
+        'cuda': cuda_version,
+        'cudnn': public_version,
+        'assets': {
+            'Linux': {
+                'url': _make_cudnn_url('linux-x86_64', filename_linux),
+                'filenames': [f'libcudnn{suffix}.so.{public_version}'
+                              for suffix in suffix_list]
+            },
+            'Windows': {
+                'url': _make_cudnn_url('windows-x86_64', filename_windows),
+                'filenames': [f'cudnn{suffix}64_{major_version}.dll'
+                              for suffix in suffix_list]
+            },
+        }
+    }
+
+
+def _make_cudnn_record(cuda_version):
+    cuda_major = int(cuda_version.split('.')[0])
+    assert cuda_major in (11, 12)
+    return __make_cudnn_record(
+        cuda_version, '8.8.1',
+        f'cudnn-linux-x86_64-8.8.1.3_cuda{cuda_major}-archive.tar.xz',
+        f'cudnn-windows-x86_64-8.8.1.3_cuda{cuda_major}-archive.zip')
+
+
+# Latest cuDNN versions: https://developer.nvidia.com/rdp/cudnn-download
+_cudnn_records.append(_make_cudnn_record('12.x'))
+_cudnn_records.append(_make_cudnn_record('11.x'))  # CUDA 11.2+
+_cudnn_records.append(_make_cudnn_record('11.1'))
+_cudnn_records.append(_make_cudnn_record('11.0'))
+_cudnn_records.append(__make_cudnn_record(
+    '10.2', '8.7.0',
+    'cudnn-linux-x86_64-8.7.0.84_cuda10-archive.tar.xz',
+    'cudnn-windows-x86_64-8.7.0.84_cuda10-archive.zip'))
+library_records['cudnn'] = _cudnn_records
+
+
+def _make_cutensor_url(platform, filename):
+    # https://developer.download.nvidia.com/compute/cutensor/redist/libcutensor/linux-x86_64/libcutensor-linux-x86_64-1.5.0.3-archive.tar.xz
+    return (
+        'https://developer.download.nvidia.com/compute/cutensor/' +
+        f'redist/libcutensor/{platform}-x86_64/{filename}')
+
+
+def __make_cutensor_record(
+        cuda_version, public_version, filename_linux, filename_windows):
+    return {
+        'cuda': cuda_version,
+        'cutensor': public_version,
+        'assets': {
+            'Linux': {
+                'url': _make_cutensor_url('linux', filename_linux),
+                'filenames': ['libcutensor.so.{}'.format(public_version)],
+            },
+            'Windows': {
+                'url': _make_cutensor_url('windows', filename_windows),
+                'filenames': ['cutensor.dll'],
+            },
+        }
+    }
+
+
+def _make_cutensor_record(cuda_version):
+    return __make_cutensor_record(
+        cuda_version, '1.6.2',
+        'libcutensor-linux-x86_64-1.6.2.3-archive.tar.xz',
+        'libcutensor-windows-x86_64-1.6.2.3-archive.zip')
+
+
+_cutensor_records.append(_make_cutensor_record('12.x'))
+_cutensor_records.append(_make_cutensor_record('11.x'))  # CUDA 11.2+
+_cutensor_records.append(_make_cutensor_record('11.1'))
+_cutensor_records.append(_make_cutensor_record('11.0'))
+_cutensor_records.append(_make_cutensor_record('10.2'))
+library_records['cutensor'] = _cutensor_records
+
+
+def _make_nccl_url(public_version, filename):
+    # https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.4-1+cuda11.2_x86_64.txz
+    return (
+        'https://developer.download.nvidia.com/compute/redist/nccl/' +
+        'v{}/{}'.format(public_version, filename))
+
+
+def _make_nccl_record(
+        cuda_version, full_version, public_version, filename_linux):
+    return {
+        'cuda': cuda_version,
+        'nccl': full_version,
+        'assets': {
+            'Linux': {
+                'url': _make_nccl_url(public_version, filename_linux),
+                'filenames': ['libnccl.so.{}'.format(full_version)],
+            },
+        },
+    }
+
+
+# https://docs.nvidia.com/deeplearning/nccl/release-notes/overview.html
+_nccl_records.append(_make_nccl_record(
+    '12.x', '2.16.2', '2.16.2',
+    'nccl_2.16.2-1+cuda12.0_x86_64.txz'))
+_nccl_records.append(_make_nccl_record(
+    '11.x', '2.16.2', '2.16.2',  # CUDA 11.2+
+    'nccl_2.16.2-1+cuda11.8_x86_64.txz'))
+_nccl_records.append(_make_nccl_record(
+    '11.1', '2.8.4', '2.8',
+    'nccl_2.8.4-1+cuda11.1_x86_64.txz'))
+_nccl_records.append(_make_nccl_record(
+    '11.0', '2.16.2', '2.16.2',
+    'nccl_2.16.2-1+cuda11.0_x86_64.txz'))
+_nccl_records.append(_make_nccl_record(
+    '10.2', '2.15.5', '2.15.5',
+    'nccl_2.15.5-1+cuda10.2_x86_64.txz'))
+library_records['nccl'] = _nccl_records
+
+
+def _unpack_archive(filename, extract_dir):
+    try:
+        shutil.unpack_archive(filename, extract_dir)
+    except shutil.ReadError:
+        print('The archive format is not supported in your Python '
+              'environment. Falling back to "tar" command...')
+        try:
+            os.makedirs(extract_dir, exist_ok=True)
+            subprocess.run(
+                ['tar', 'xf', filename, '-C', extract_dir], check=True)
+        except subprocess.CalledProcessError:
+            msg = 'Failed to extract the archive using "tar" command.'
+            raise RuntimeError(msg)
+
+
+def install_lib(cuda, prefix, library):
+    if platform.uname().machine.lower() not in ('x86_64', 'amd64'):
+        raise RuntimeError('''
+Currently this tool only supports x86_64 architecture.''')
+    record = None
+    lib_records = library_records
+    for record in lib_records[library]:
+        if record['cuda'] == cuda:
+            break
+    else:
+        raise RuntimeError('''
+The CUDA version specified is not supported.
+Should be one of {}.'''.format(str([x['cuda'] for x in lib_records[library]])))
+    if prefix is None:
+        prefix = os.path.expanduser('~/.cupy/cuda_lib')
+    destination = calculate_destination(prefix, cuda, library, record[library])
+
+    if os.path.exists(destination):
+        raise RuntimeError('''
+The destination directory {} already exists.
+Remove the directory first if you want to reinstall.'''.format(destination))
+
+    target_platform = platform.system()
+    asset = record['assets'].get(target_platform, None)
+    if asset is None:
+        raise RuntimeError('''
+The current platform ({}) is not supported.'''.format(target_platform))
+
+    if library == 'cudnn':
+        print('By downloading and using cuDNN, you accept the terms and'
+              ' conditions of the NVIDIA cuDNN Software License Agreement:')
+        print('  https://docs.nvidia.com/deeplearning/cudnn/sla/index.html')
+        print()
+    elif library == 'cutensor':
+        print('By downloading and using cuTENSOR, you accept the terms and'
+              ' conditions of the NVIDIA cuTENSOR Software License Agreement:')
+        print('  https://docs.nvidia.com/cuda/cutensor/license.html')
+        print()
+    elif library == 'nccl':
+        pass  # BSD
+    else:
+        assert False
+
+    print('Installing {} {} for CUDA {} to: {}'.format(
+        library, record[library], record['cuda'], destination))
+
+    url = asset['url']
+    print('Downloading {}...'.format(url))
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with open(os.path.join(tmpdir, os.path.basename(url)), 'wb') as f:
+            with urllib.request.urlopen(url) as response:
+                f.write(response.read())
+        print('Extracting...')
+        outdir = os.path.join(tmpdir, 'extract')
+        _unpack_archive(f.name, outdir)
+
+        subdir = os.listdir(outdir)
+        assert len(subdir) == 1
+        dir_name = subdir[0]
+
+        print('Installing...')
+        if library == 'cudnn':
+            libdirs = ['bin', 'lib'] if sys.platform == 'win32' else ['lib']
+            for item in libdirs + ['include', 'LICENSE']:
+                shutil.move(
+                    os.path.join(outdir, dir_name, item),
+                    os.path.join(destination, item))
+        elif library == 'cutensor':
+            if cuda.startswith('11.') and cuda != '11.0':
+                cuda = '11'
+            elif cuda.startswith('12.'):
+                cuda = '12'
+            license = 'LICENSE'
+            shutil.move(
+                os.path.join(outdir, dir_name, 'include'),
+                os.path.join(destination, 'include'))
+            shutil.move(
+                os.path.join(outdir, dir_name, 'lib', cuda),
+                os.path.join(destination, 'lib'))
+            shutil.move(
+                os.path.join(outdir, dir_name, license), destination)
+        elif library == 'nccl':
+            shutil.move(os.path.join(outdir, dir_name), destination)
+        else:
+            assert False
+        print('Cleaning up...')
+    print('Done!')
+
+
+def calculate_destination(prefix, cuda, lib, lib_ver):
+    """Calculates the installation directory.
+
+    ~/.cupy/cuda_lib/{cuda_version}/{library_name}/{library_version}
+    """
+    return os.path.join(prefix, cuda, lib, lib_ver)
+
+
+def main(args):
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--library',
+                        choices=['cudnn', 'cutensor', 'nccl'],
+                        required=True,
+                        help='Library to install')
+    parser.add_argument('--cuda', type=str, required=True,
+                        help='CUDA version')
+    parser.add_argument('--prefix', type=str, default=None,
+                        help='Install destination')
+    parser.add_argument('--action', choices=['install', 'dump'],
+                        default='install',
+                        help='Action to perform')
+    params = parser.parse_args(args)
+
+    if params.prefix is not None:
+        params.prefix = os.path.abspath(params.prefix)
+
+    if params.action == 'install':
+        install_lib(params.cuda, params.prefix, params.library)
+    elif params.action == 'dump':
+        print(json.dumps(library_records[params.library], indent=4))
+    else:
+        assert False
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/docker/python3/Dockerfile b/docker/python3/Dockerfile
new file mode 100644
index 0000000..acaa524
--- /dev/null
+++ b/docker/python3/Dockerfile
@@ -0,0 +1,12 @@
+FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
+
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+    python3-dev \
+    python3-pip \
+    python3-wheel \
+    python3-setuptools && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+
+RUN pip3 install --no-cache-dir -U install setuptools pip
+RUN pip3 install --no-cache-dir "cupy-cuda11x[all]==12.3.0"
diff --git a/docker/rocm/Dockerfile b/docker/rocm/Dockerfile
new file mode 100644
index 0000000..c420456
--- /dev/null
+++ b/docker/rocm/Dockerfile
@@ -0,0 +1,22 @@
+FROM rocm/rocm-terminal:5.0.1
+LABEL maintainer="CuPy Team"
+
+USER root
+RUN curl -qL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+    hipblas hipsparse rocsparse rocrand rocthrust rocsolver rocfft hipfft hipcub rocprim rccl && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends python3.8 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1
+
+RUN python3 -m pip install --no-cache-dir -U install setuptools pip
+RUN python3 -m pip install --no-cache-dir "cupy-rocm-5-0[all]==12.3.0"
+
+USER rocm-user
+
+ENV LD_LIBRARY_PATH=/opt/rocm/lib
+RUN python3 -c "import cupy; cupy.show_config()"
diff --git a/docker/rocm/README.md b/docker/rocm/README.md
new file mode 100644
index 0000000..97391a1
--- /dev/null
+++ b/docker/rocm/README.md
@@ -0,0 +1,5 @@
+# CuPy ROCm Docker Image
+
+```
+docker run -it --device=/dev/kfd --device=/dev/dri --group-add video cupy/cupy-rocm
+```
diff --git a/docs/LICENSE_THIRD_PARTY b/docs/LICENSE_THIRD_PARTY
new file mode 100644
index 0000000..31c8480
--- /dev/null
+++ b/docs/LICENSE_THIRD_PARTY
@@ -0,0 +1,70 @@
+######################################################################
+# The CuPy is designed based on NumPy's API.
+# CuPy's source code and documents contain the original NumPy ones.
+######################################################################
+Copyright (c) 2005-2016, NumPy Developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+       copyright notice, this list of conditions and the following
+       disclaimer in the documentation and/or other materials provided
+       with the distribution.
+
+    * Neither the name of the NumPy Developers nor the names of any
+       contributors may be used to endorse or promote products derived
+       from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+######################################################################
+
+######################################################################
+# The CuPy is designed based on SciPy's API.
+# CuPy's source code and documents contain the original SciPy ones.
+######################################################################
+Copyright (c) 2001, 2002 Enthought, Inc.
+All rights reserved.
+
+Copyright (c) 2003-2016 SciPy Developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  a. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  b. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  c. Neither the name of Enthought nor the names of the SciPy Developers
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..ae1e47a
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,197 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS   ?=
+SPHINXBUILD  ?= sphinx-build
+PAPER        ?=
+BUILDDIR     ?= build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(word 1, $(SPHINXBUILD)) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(word 1, $(SPHINXBUILD))' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  applehelp  to make an Apple Help Book"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+
+clean:
+	rm -rf $(BUILDDIR)/*
+	find source -type d -name generated -print0 | xargs -0 rm -rf
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Chainer.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Chainer.qhc"
+
+applehelp:
+	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+	@echo
+	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+	@echo "N.B. You won't be able to view it unless you put it in" \
+	      "~/Library/Documentation/Help or install it in your application" \
+	      "bundle."
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/Chainer"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Chainer"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+
+spelling:
+	$(SPHINXBUILD) -b spelling $(ALLSPHINXOPTS) $(BUILDDIR)/spelling
diff --git a/docs/image/cupy_logo_1000px.png b/docs/image/cupy_logo_1000px.png
new file mode 100644
index 0000000000000000000000000000000000000000..fcde17744e099b9e816b000cac492a4549a9975d
GIT binary patch
literal 56999
zcmdqIg;$i_7dCtkDag<v9a7RrgEUA<NrQBQba$wfBHi8HDXp}mbc1wv=R1Qw&-44%
z_x=U%&0?{b#SCYkv-h>HeeHdM733tq$OOm$0Dz?=#oht{>?Qz!ik~1rUXeQst%LmX
z(OgtiK}uAV%+}t<#QeQ60MJHwMes|u3lVkdD#?h`p*kzr^(dq;1ch*0;DjiUk0F=v
z#=pCUmnDeA#*_G7v$WK0F#f$$Lk&?)82uKob8vX;7=if^hppM+YC9L^azWy90AwiY
zCysNVnl)JEMj^2#Y2mb&J<&%O7Goh}>x+-d#`f%<<cINW{(Qpl;NbIBhr@=Ij0|(=
zWqWlnS5O|8)rS)SyXY9Bn9F3>{_Llig7(@+m!S^7jQvbs54>?t<(6i^<x}6%Q+>&?
z{9-x2a4VYGh+mb*#%_X+W1@k^N1w8+jKLvgoh4Ge1CugHK|vHBXDVp4Fv@^j_6CXc
zA$%vUCnvlJ<@Hy1<qToxyCt8)My(<ip5IjBm_nPKb^@<`51EFG`35g4R@mizm)LA`
z>h8u7SWcwen@c*s`nFgtEjN_B@Mv2@WjQqJ$=#I}P87)in$mu8t@++-rJD}yl+=Wc
zB9e(}=UDrG5Y?-eoZb;Q*x%pXTVyM__XhBJ>_2qrnSO>G3xa`+gc$Gu{Yr1ni-Np@
zWGkuR002zSq2D0a96@Kui-?X=vf_wq@c6LcClhH8BLF}KNQnt6yUy&+yLn=t-3cB}
zdYE7dv5ZZ)FImAyGU$kbFeGtnkmJF_)Sv^^&w?IgQcdLglnUA5a22(k6C|mWN1u&g
z-!ePH2BmkvN=jmvxdRsTa4<a18+SL)DG|g!CiC^~T)AGl){8o?I}GeLZJZ<wA(B!2
zKl)hU>tmoRnyOcOzfqoVA-U^RuG4?6Dn1V|`ETxR9d*JpkV7pu@RbuA3t;f^FDfg#
z*dnn1rou6Z-a)s7fDUfs?oc}yF>-bH{AChLf&qD0g=i=&{M;;??<jVtRKZT7RB~d5
zD!&niPiS4vGD_T!h*OuDH44QbPX~|^w8^)6)%`&BahPm0TwTQp7W3&64A5OSKQ5(H
zTIr%|>;@wo%MJZpCqyP(5QZ0C5d+Ul3sYGi4oY0)x+lv(2i<NJ_=&Qgv&UebyN2#_
z3jtsD3AjJ+*cGOs0Kn~|&;2p;47x)Itc6wp6gWNSeHCdNsW4{mXb$Ky#88HV^ZPVb
z(_Af{oJElZ5N2)^Qs4@evB+A7>dXA7{v}SR?g0I}u^4;?m`rQ4hkIt_Ra4|<DI7a}
zqGVVhc@D>|rkVxsV(x!-OZ(?o1-<15f@EL`izZvm&l^J+n|UGL6~!_v2=Y8j$5M>5
zkK6c0Y+#1zdxt7@ytHkx!m84eDEiNOK1&g*+y)5QIKA;#@rflo95fn$+N7w~kxEM;
zqSKa?CDfqw!!hVx9Et8Jw&hpo$2MS|r(<ErT{U%U1FF1~-(bci9BzZ#|5<!FVoDGI
zI81)wE0yS(1JYYXBb!9U1B^QGy!vOJLzbRd>dU_=1_#u;tcH}m7wv+b+6$*lqbw9Z
zNqaE2nfT{;Gnv-DDvGZKFctv*kfnbz2}=#rvN+Ow{m<Gh6F3_--Ja~*Dn!zV#WW=h
zBtk!(B5#FnQ_NA)4Rnx(T|dBs$JR{Fm~m-0K?F672ghSLcr*O#mcz&%y4&=NEtQ@{
zdo`-As&%MkJq1%A<ir-Y+WPl*Kq$77k*z2JU-VtHYj^qoEQ<#WeLLSgpM9sAK|mL9
zIYKTauwXpXf~fOmzwgmH3WQCuv7GGCcSkKV=aj%1s+9eEBe6X!c=%rG|2dXOR`;tu
z^yUF!sg~;%tQ``)$9>0ub)?v~A6fZKs?sAWCsmfe5TO+5NdITwACJ>vW+TEJh~=Y|
z^r59$IxGLPZj6gE1{exnf|CjNFeiB96d^>O=bhJr`6#j%m$en=P}_n)yoxj$&=B;p
zS|!VG_`t;Q&EumDnoQ2w@$Jfxas?N_{heH@6rF=f>V#po^Iv23dX8!8uN&OcO3CB2
zR8~-gi(#yRUuK>$q6*3!7KdUo3`tZI?oV%npQA*?0jIV-Yx|$*=J^MY;rxsF|9a{e
z!g|Ts`RC#|b&fG`3^zuo$|pjhp@W-Jy8WTUONLm3*&fEDg(sbYDUmv#UPVVJu*Yyn
z{5|KdBB?&NXd+L*FTueoDqLh0025&Axnsbd5>2A}t_{;Vi;REi4%U~CIC9Rtf|cB=
z+6MmL4fFb-5=(?%Q<d9Ffcf;37MvYm07@R0vvOn>3&`unvohJL6TT}j7Ax^Ke1R>c
z?`(G?4Zo(qh!as6Ub5y<VVbpuG*&+GcmJF`Ap3hG4o4-|`4xv_&bJHn>`B}1QnsgA
zk&b6z-lfch^bY^7{Js)>jU^~~5h<_?wj3P(7N_ye7%i9o^-Oi27oP?j)GP=jH8g*-
z>>|d)V4n*hw^Qg4SqfN&0h-(M&ieXP&i9MB&DCdnysoyD^au!n<dLs|)2Qjn%0J)3
zu3=Wn$lq0(ic3rD2XP4f-K0df^40Boa*hdr`xge1>Z|zuBiWt(GgXD~_ukH)dpvaZ
zr6FESWFG+jnguGXeUjfdyOI}CEAm9t6y8MNKYA^$v;v%<1`TqpH(H$Hl19@uuReLW
z#X>^{;**)rqwyqfQ$%oi)+y2Fl5r1iJX{LW4xY{1lKUE>Dstu!{4zAUKKyYf(eN-A
znQfVV-GQN_Yayn95Hmp6(5d?VC3rHnT2ZF%<PFpzpdn&K#<k=H*H%UZIW+jve#yhD
z#<#=_nu64Y*1BQ$28CK<U-?*`-hboX{slgEbr<;smQZ=1T6j-u;%VrB1DBK}KZE;C
z@bVf8fNovr&~>s3<vCZnu0#hld$=@Fw$TPsUbQNq$Uq{1r=*zi|C(0JvgkXjPhA&3
zNL+6ehJT`3;6W}y{8A@k(CTFBd4%|i?eXNFr7O{>Uq)Vt&3>8$C9j85ss$0mWxK`h
zAy2{$YVGXdX)sq<<P*}_Q4ka%b7?*dUR)pv+c`CKU)a>X|7-B75D(SDQ}GJiK-X3t
zqJpI>e&_M=6#goY|8?#Qzu#MU0Cx+P%!beRaGurDzS!pOSlPcGLquEF;ZZ`DYTGYG
zt{#tjAhVOd!qM+(^VPRd3vS~D%agWVea%-%9%f;`lVww&E6?A9WD*M-^P?6T6(c{T
zk)jZDH>!mf#7<4<{i(i8{$!$l^gp<{5q%vjbIx@OJ?kH14&t0dTh%EM7l8VHI@w*A
z8&z-?-?IAFk$95$nLAJ`8e(c-9em+kYXP+_v$j)*Wt#<xJfTkv7=2HOFmoT|Whbfd
zpQm}{J^vcJa-m5PzZJf%jPWiF_rN2PG3jqO@eKRNb*Jjbmx8&sYwG!AjZ5$wRu>}Z
zTP!Yu)W{(Vr2^g(z2)R6-0nTZE;H?;hW<MXepI_WRHts^A+Kx-EYKJ)WiRY=(e;;*
zd|la$MhUZ*!EcRFK22^ssw9@`Woxy(^?n+<e~|3k>-Tc`^90hV8U$E@tW1HjJ0%IC
z9j??L&6xpsZFs0JQlmtKU${@D%?19qMg>igqZOEbNPdLp+JMLOnItz}(Ksh36NFXR
z2p{>D{(76FZfyDN<3Be2s&-v;R2*=TcE^c~Taj;Sc>XmeRiPRo1SKyzRH$2`gej$6
z_1_bfgd->}YQ}F%LY1m~K_R~y^CoC3oNRvOCJXR!e)FktE(!wwuv(l@_lCrweL7#6
zXGX)qry`K&ayaFM3b;hQ+Py!FZ&|Og#)8f_F8OU*=Jn*9O7PKNs~PLlOfVF<d$dxx
z)vYvOm!1VCb6XC0oyEx$H&)J@X6Zr`LOHboF;FL3c+&bkob~j7oX&9-k`MLCt(1=l
zGwq)PP9p9&&oFqYkC}<>iVNCRRfF-5VIV1R{OWm}`3;FH>o?~AmTC~b|HAz>PE|4l
z(HEjO<eRcYf10d@T!8(eOq|0)zvOGbg@99~F&5M`2%AHcmOq?F+f@3BG$OvfRx15w
zJIGM>^F;~ncPrGw!6E_vjB~X2<NqF|Sh`BXh6%7k?0|}Y-0@?4;e>Zt58~*pqDSdU
z+e&_?uY5)r|L5o#<kPA1zas$x&XSQgQ?-}4Evx($&rwxlSh$JX?7QK%GJSs1OA!tH
z7j2tNLk@h(;c)4^O-gj6Aa$nUJpTYY3kjlMn0=)#zX^aO;<qUng|wbf<o)}f0)v^<
z;jjnoVWqGDFNBl_7W1$t3yLxJL9ro>#<?07=VOZWt((PgYggTA1qL>Nt-oktRpkVV
zw*5g5j&8+&oADFL^#Et-g4nfZ50Dnm>E~2nB5$nziB&g_*@s$J*{ikItPE~KNe>id
z;%45TD6EL3K{xsnArKmk8jW{&FXKlz<65T+GX>Xa@~mN6;r;eFqH&#$a?_BMag^3_
zGe(FHFI6-0oz`oSqq3>OV5kQ_OOJxpFz$^|)SYMn#66$3=%6iWgMPjj{}vP-g3lmB
z_-*;F^sN97=-?G--@gBo#A==SkvLk_e+y51`1uj5|9Th!FeO<Z5&#|I!r25f+RQQ;
zj+)PEALL|X2Q=7Et6ShJ7A>kHi${t^5@8ZbnP+*?PgVZA1V;4~Y|E(zR3Tj=SG9LI
zITUXR#;zw1j$hSfHO3CK->(YPylup7yMoZd((uhCzLn2{djA<;-6ME3Hq+3oJMXsk
zbetjd!vGYcF*+;(Ii#xWanKZkWI>2FwM8_cu$R@$T1!eX2NYSLSxI}uUu9(lrCkTS
z6fY<Jqt4{J;8|ZP&hO(yzcqPV@vREj&NerN6;BRF5BH`6;aVc-`=E7;6%);pEG}7V
z%Fxj~`g^}j6o32Ki4mR6s|jc#ZO<XXEg{8HsatePHJ2<dY#_%(?EJ)i`TG;{X;<~f
z<?d<wzxaVsBM~kj<J-?Ur9$kBEL4}Tho$!(p`7P=%jSyhPn-5b?_i~IZc+3iUqYdN
z44sW_u~7QaSA8(=d7zO2t#oFh=_u<uZnA->uw~0xP&7!h)F9eI^5^-_(wBW$`AAtp
zw^Kdo{5`lQul#g?PB&l>b~rDl@m{2VxV*Ib_FwkR64A)?4R6~5#uw$POAsE^H0o*P
z8mCz2y`3MU`%z{F&n6jDQM<;=7fP%eHOgGsfvgl%Epm%@k?R_@pGyM&_!E|5Qexf2
z>=4e6Me#L$#U}i>Zfupg&VA6l-TopJLpWHe!cZJs7t~ew1DtM3^A1-xe`J1zL7uE<
z;@>JoCb7|E&JIJk-Rc=8qBsMbH7ye`%vX$A=*m97()fwv-;$uO_asbSp+Xl%k9T#O
zJG8G13NDHgFV*41#xO=8`vfz$5f4Xi?*J<mT+H+L@U-9VH>{Gb!<V**#bj|c7>B1W
z-N5=%4So~Yj(V!*F=Y5i%@rl|!o~J}(8+xLGNUo7EBkz~9HS`H8mIsUEzDeAvhin_
z;mQ==^lHb=$!uFr=2Hxk$MuGNNk>g_{$@k-5A#==vgmxPPB&~hiW!n-^1@m0sSb%p
zfPRAx;-R6$J6iah?$ZfHXpn~0{N=v|N;ue*NolD~2PoigbC8&^ME50OoZIt$_FCjR
z-?*&eV`EyfikdEUz>y8W1mQdh)vmfBPon>Qzvv%U)w=WQvGf(CL=AZgA{s>voZWvr
z8D5Z@lUBakm^O!LgMj@sIm#dflCEAw{lmTs2L1$3bbXhO9i*)7aI~Fu&2k|n?)}|9
z&n}5ZDS*_&As0L>ZtJDlbRhMo$g1JM`<(7!CSabFPmCzZ;QJJ#BMkVI52SF}yVyF_
z9L3W`@_5!0hU~h1o)|yJTQWe*R#iSSlam@j#6J?>QFnv-@>nic5`%xK<!_k<EGs-*
z5I1QX572KbSN5l5n0R*)Rp-9kYk{Z>Yg;H9d7BfTkzxK^t!d~_jamD{X}Gc6JnqI>
z<kSm*ZRh7k1Tc)&;4x{c)*I5GT=a8>6p(zJ*nULt@-rSJI{#EWnVg@4)#0?i7~_{S
zG;U>Sz8>4=iWP}i4JR6VR!jMYP`lbM20wMZ6yN2Y*sv{OK!-3D>M*z8Jr$(J%$V5c
z8%jHMz+to<tdUCNJ3C2Th>re&u7&U?$K`zx8<I|WI`vT6L&sn~cU~^|O}=ET6xOy>
zwDNnxe0vAQLx<8%=#t*lGlt9$AbgaiXwh49$@i2wq30!Ld#nyJP16efw!`9%)|&nY
z0=2k*9at8#7;X~vq$3Oz{6r+sy071->}c^%QKt)gXc%a;F`!^I&;#WTi5#={iSiU?
zQ$Z?<Tm;lsd`Gm-RcclqVqKeSHg(jD5EAUyk*7jA{$edJWQXll3(ZvIUuXgW29-)(
zbVi+iA@iDq`nt#W{GS>LAH4ho?LLwpzx0(1G%`1bIKf8pbTb&%uI5XA8=<M_>N`Kw
zw@a_s-%wm)RSzU%lTJ3W?CxW<3~5$H%{lcpE2uRRw27m>m?7}v=VjEtDtd4H7s*&y
z_?&`)f&q(*t0;fIGE{XxJ*6cq$!qv@9~bFV?kbNyjuv`r0?eX9>!1!{I{{yTJ<m-p
zks6LHO}^{j-)n;N0{tsV+Imr69H->U1kMR$UJAbV`t#w<gnZo%oIi~bT!789P{2js
zSk+te#OfQ>LSXu3iITy;g2&bHAK)&N@L?UFfA4xN4#;hH(D#KXxtvQ{?5&!+6=5M3
zF&2SNoh6-`#v8+Q+;_w~Z>C;8<}#2w7fwpbF})B8kI}~>cP(Sp@45tnc+hxGR?tGx
z`yyJ|oiLB4z9j#J6CC`3aicKfL;A}n!)!lTOXw=eAfP`uu}@<Y<C083>kk`?{GNO9
z&?unWg}6)_YKLQW!x`_UH9gNucU22HwVuyy^}qQZWQ`Pe@1mG2Z<8b6J}cx9orH0n
zB%j_fCv0HIqui?A_x=$=4y$|=AF=@MthWUZf`pV0dXQXgEcd)o`%!$#47Vk1tswl~
z`M&9~tb#qwLU+WMLeG{Ml56hny?Pd(;u$N#%-^3THWh#?_8{OSv=ZPUQF{5aJmQI7
zoM@$6WIpbI&wXBVqeId_(8ZN<#9q7}j;pP3n}@jNMO5li5_4M`v65m4JP)WWP>}V{
zk=#jsKpfX4sZY4~>fe<r_&R{tPk3}!Mt_UWsft=oO}bdGKVzm#C8h-WNEA8#bby4P
zv}$a7Vvm#myGtwZ302DaBj^>|qM)^crAxI|Q7-afrs(k=X1cAn)Uw8H!lS~7u(@d1
zw!5kXd|Td8*M4BtQ{C?T+7~1O=RS{dzc(FV{*X2+ZLMj-$lr1a|771;;VaGSnCQXX
zL7I{$>~B48-;!$)MXu5+^)FwQz2VN_;H!A$mp)x}zX5RaQ7#I;REcGRs&76Pe?sQ^
zw?9!2v+ZHFQZvf;N{3=gd8fAm2J6{h#sWBs!^UW5jY-&*x%n63V3*2J3!D3)rE|kc
z@C~8-Zyl}o%)Sqh;>MKVEMu(%teS_ar9oxk3fgs)Y+;^i+!&7H!u*3Oh^VK$eW8Ju
zao$X9>e!|{dEI>h?|jCl)$Z2XcE>(YS8UI3WG@OSEJr^fE(}}0#0qrUh`JcOTt<7Z
z_mb#1#2>&rn5BPEwwm9@Fi$lmC8(tYO_6p@mmrj54EsDAu6rc*KGsm1HlOS#G^Y7o
z<hgc)s{|q~J(N+f3F5<mH$0Qg=f|o7zxgx*f@a?*MS(6gqjhGRx$;A*gWy^j&W|Vk
zON-_6Ii=)E{4#=DgEV%ff?Fg(gHF=!Dt0~r-KI6pf^PTOX+>rNycfcT{JvqJ=%P$)
z|CbdtO@CPLx;)AsdmmZ=e!eqS5bEMdalA@I4C%xHFT^zyP-5~lD?alp+>C@2|HyT5
z$19c#;drl&*!F_o+vbyn<?A^p9h^U-RQ<FX<C+G;(#tBPIv4)?i2v6N@fuvy1@hqy
zs-(k{HVNS8BjYY5e=$qn_!Ck&1PD_7#SQquiqcT~gztMmi_Npf!lmn<_*e1)R=%kz
zm6+j0n$s`>#b~cly@uRxPr78?{5n{XUvzyx*bJ=7Q91e$)#wnGvnIcZPt_d@yzGF8
zZLq7yrynyptkAOs>DriQS*%E58}R*QVWtgW*D%J<)|9$Si`V4;rVpFh^mDZ~4EY&l
z!+gqUQ^sj@>#Jw%rpLj;v-fbmMPTO(>2jNK<JnwqV&By(tOJFJD^!c#s?im%g!wC@
zK`s6xqnZz8Alsi}fqd__8|rJyXc-o`sWKbfH>C4*oKMk^{(owf@L#j<I*sHtvxYGw
zu*BlYJ<JkRF4M~0hgm#Eh!w2*fa{8o{@V}KgMDCEJlOc<@IsY8rT)kZE4!nv7Z^7e
zdqEW{2*IHzW5dD}i7}jom!YwUTfra3V>vGGF(`g7hc0+Iu9o2899Ry>J?7T1@NcL>
zr!Kd>)<6y*tpceUj@JLFA09QY4xky!U(<3Vf#vKDceRn>nLT*XgkJ-P)2&y|i?Gej
zYK~yPBtGIn=me;AKlr)<awU4(`BzFntNZxKYW_d6jc@8AH@P6&I7c-{^G}{4Odsk1
zY~IaVZwd&$cv-r<Y^6M6P%`HC)j@ynZ&47UNJrin9aRf;NetLi+o!`-N+vJ{HTmh-
z^mVUigP(jL;krm(eJWCWWWu*QgBy`%lTxud-39)(6X_GwFmnc6^JOQx<oASsxy%gX
zdx0)EQOt%ARA8;hhZ?TV{z{VPH?8yNw$9p|(DI$H5zppn6FvX;$!!dBw5IsEi~YW*
zMU?$&dXFt5178998ccr><6R5gy+T5akIS=HW<BtfB`MCWi>e*1Q85qhD*^EL9cH4d
zD%J0K>IR4R*%}|S&|g+rNMQQJ0LL~<Kgdao;pXg(W)pKxj$baUsdJqQ$BTBQrsO!|
zKU`y>6Pz4lUl3Gmb@o-+x}-<*SkiRu`FCmCJr?=fXKhk@uJJ?u<defz6H&75#SSu<
zKFBdRW=T?8Gkx92FFxsIfo}?+<(k>&ltFE_{raHv-<&AHA1H8PxUlguQx;Ef{bFzr
z8e4M~b(>g^;<uYC5!&<ulXDwU2-|Z*&sB~`-auUv1;Xt%cgI?R`WmpaN?EOr22WMn
zqSRGAfsh>{(RS!UhSzhMlCnUJjqB#tPfHmr=JmQS6beJs#Io7*&&n3p+^i)m&&^|~
zBwp_oZ<Xa@m_8CBUv7ZL<t?X5`eZH@w&45-asQvn5rNy(<ibTPGy{T<jjnEwLKMQL
z_He?4EWbzMjUcS;J3^dUp8*9k^#qmrlB^TVDE=3^883*jNR}VyUY<n#dLQp(O}3>k
z<TLN5fyxPI)i90gMp&5CIl(o`taWd0={$p8FP$7hOm9&S3Wa4s|6O<P@wY3)sI$$S
z*Wr;tI{|F&h4+m-&GRPViDfH(6i#4Inaq}U$BY!QtL+Xoe~p$luQC>`xL*%?lYO%>
zYOWv_Dnq4z)ma=;e47wT`Y3{m@#nSS?AcZm!@O7Rq<%J4+?E@(eSKVD*s7D(Ooa<6
z>+E*K!^XHC-f1_lh|?|;k9@2v>KUGR;~A&6oRS|RQax~UM6B=a?tj3fllhnD;9;et
zyoC#@BbGk+m#qb^8ct-tK-Zr8-kC>N-(vsTd-NJ2z=Z`PPly$Zwr86$R>;5|6iD-k
z$#lk6x{ukI51>2sr7dmmY<uzgmsaU-PM5Fk!itU=f|98@sk5OZ`1S_k$1C6nFfiyW
z$>z<bti@Kzfwj1H9kuVb`L6q2G8|W4=gs|KAN!wVaX>MrdW<HT3)u)GKH}vJ2v8Le
zN5sYNN?Bs_W@3d@yoO6HVLP7mA>MTx^3&t@B_&?>xB*2AAJvM~J!7eDWA7Vz%{afX
zOy;0%&x|zDF=c_`U#OMN>WjjSZYDvrpW&OhKKvzOsAaUC+;n_=<FkGfOgGA1*GS>z
z_udkXAwI4pZ4|u_(PJ&x_8@uEnq3Z^KT-?G9S@z2{%n%YwO;$F;!#qG#+8r7kfE4X
z)ZoX00~dpoJr>dP;g>~+g1U$vQqMtNNMJO<T3A}x>UW*A<KfYJ{Hz95YfQzKz26Bu
zIyhGC>wbGI$2gtw2P`;<|K9HcBL;X2u0%Ts`|W;#)BQZjrQ2jDEq1m*JOq2ly^6i?
zHQdY2a)anEf{-9&Rw?O%SejDMurTvoEKR;U?8Svi;ij|kedF$fMD!sfqE$i(aZ8U;
zm06u@O(yoBEw3H>@#LFO<9>uJ;)NQu?<aYCyR4;Ga7H}dWmNtNsOBSd{yU?~Uh=h$
zi2nn8%M?_!<3S|PM+5QpH>e)v;AiQ4%Kf{Rg9>1;Vd1Odmt)U#QqLSUW&@4F<EIM_
z=nkXgY*<2Kw6gZZs%-=Ia;HZhJ5iiX-6W$CR*K3S*=`}G>^Nm2$I?wQKsI-_`DA_}
z4FtJba`@EifAT4Sn+mOJD}@^;XI8e5#JLV5A2nE%;S_>&Q~96Do5iTFRuP3qNgDQA
zelXA2=fRq!rKhO>GYdcy#xkakB|uelp<;wf<Xl-1c-QN7Zh0rfI?ia(Eet8CPe7+g
zfhhSss}MZDeS~=rDX%>Ol@6Y4H%KNXFG-l@hxkWauzqpJ@5T9pVwq71Ux*0xz80-t
zmxWP(dUoV}uOJj;AvzQ)CO^FsJ=`y1f-8{aJMx?3mqo@0I;1a2r=v(fr)v76X4qWT
z13!IslV(q(F(rzAS^-Q#?b%kpo3hnvX3lf)v*KFk>-E3)84j_$nS|(Gwvt#r^pIG|
z<X~&uh;j8P2$A4pMQbFZ{Qy;}DsuYScNS84E!Ob<y2AnoN_>oa1^H(2blQVrM8ZQ~
z?bNSFw2Qx~dwTf67U}&!xYt_<*j*$&u-KGbOa+nYfFN~jVs9W*mbc`I-XS5@BG30K
z2<!1#Xh@jHasWwLZa7Z-C%%3DBPV+41Uy1WgR3t|R7!(_-z;C59+j8ndSsL8hT(a(
zvWhn5Yw;U(ag+Pd#j`NC`y<oBDZ-^_9kcKBrQ6qixnH$4X6s*cr@2N2-L}85YOeX4
z+L9Gc7|4RH!ifmmJb(pUJuGFe$=UUFuPo-vbPhC(M}`jZIu+qcd~I`2n5AsFH=jzv
zkzx{LJWHl~A<P*E<_kD3ljHrjCeQWAxTGAL>)wo%6Bt#psd}rwHE;6B(~?WIhHe+7
z6ImX=Y;l;o6hi~Yh#g1+H`QVbmE@e}q9A6b1{f3!DN_kv(pGs9XjWilY0iA=3F$l7
zgq1#SccwK#I$D3KXH1(8K7!PARsd_J{q<;)=J8sZ`eYiq{)88D8xFPjjd`cg&6h6x
zIm61bt+FR7FvoQv{ZvYc5BiYIg6Dfb&r0LJonoH43?G_C=pJMF8T7?Y64yfS+0j;3
zY|UlO*8u$UBLZI~)T1GYELiv-xvpuv2-%3HOra+CC=Yo@3sD}L5+E877Bta9GA&_;
zk-Yo6jLA<@NIv-(H$z2DvvnJ0tpxmh9K<2XEEY@-^e^M)CZk(+rCJeB7hp;ARE^dJ
z9Tg6ji*?Mj;IUpVd&xd0`G?hkP&Jr|+MMGW0^Xc4+-2wR^=R{ZKN-7+RsJb<YBf6{
z>#V)EKJi9uQiWZYLIKjRWkRrBuDe&T(}J|q@cxvfigjqiE5#j#g#(7p6t7}WV0?$@
z5N7h~jOGvHcgLoY-=M`+f<YArChIovfO;(J@Z5C<kjHUfB|^<fI&#pav)LTrCzBU?
z%n|DPSbK}<h+n=Lx$s)poR^STa92DM&{@zl`E|{z=yc<LYNJVkj`=wt2I;Hd5ACtf
zKd0E`f5|Y&LPK?X0(9LrT`(E49BeBUCbFSof;%^w$;{Ox(~8nIp0cR0T2qoB6X1P!
zNM>9tqUyOabUqalabUE@#i+gBq+@nPi>;6H?PSDM&}OdER%IlnHiI6BN96&hN`1b_
zu^l*UuewDlZRxVR!};#e+2w}LqeiGo#$5zvD%Sm^9Ju)VMX8<?ln5x+frkxEas=`)
z1^8<XmgY)EPwJ{K{n}q0RJ@%m8jWMZ>anRaVREZKa&UB(hP1B)JJP&yQN9IOSEaru
zFxjiu*%a;HP1RZ+@HOyL8id73h``Wg4~(lvcx-u3$P^Q72RyR051><v?sjkV5SBm=
zpQ%TQcFYRqlqFf&$`BHyoh9>9)f`)I;?EI;crCXodHIZzkH4Os#sTPeH%Njaw8u&S
zf6L0jV$d!=kHu(8n3@mq67@Yc9sQ4dYg8YD{B?O5+SIuwJYMFdyC|4C9;(-Yo7tzA
z%l`Y0!b$(gz);m^9~VX5W`};aj^zANr|`m4i~ISd$QzTr`1n*^Yd8Cc=JD?~jTczr
zhsXN<%g!igj^c*4r~pAolN@T=*Y8doPGR3GBlS7@k5fo=Z$;$VJnG1jshj!U#EQRl
zP@}OzXxaZHkUJz*r*<&^i?nS`&b14Ok#JDe)=q#7Z_H}%{osHgpL*5cGqT(UFkuDV
zHUfGex?BKI2o@i{0wcvCgay04!HK&334)RlFVZX-<JWdp&#3YQ%w}X}EKFlG;LCyx
zr1Qkwk4q52T*Tiesb3Rnp%ECQL@f2eAFAb5(<lxMKBbhizV(DsvPSobhc{3O;*)?u
zy!cby<J8SDhigB)sGId@N^spR5+c8=!zbIX`cp;XXS#ImTm<Xp+&pipA|llyqkGc`
zp&om^@GQH?$jF!D%C)`PC7@3fQ4!l};3w(g#7$^w3@@I)2QLfF1NeO2>t}5NSk!Ue
zA(HvYbXStZwUtWqx0glf=KQCfgn44hY6q?uA+CY!C3gXgspeUqIoYk_bt7G43m}>s
z5^8Td?_|sMuipaw4d(c4P=%i>-44h^OCI%+l|Ljt^`+}RXqJNu<I6@>#^Ew@^_2I8
zB65Ba;wFF|#lL0!NdO%ep8loX&KIho+B+)&BM8gQgO}V(FMg#Qksw&J1rm0aW+F3}
zIJ$Fhk7ZqBi))&z*Y^`Vauqd#^{pZvsdkQMB6hzCT7F>fYkwlTO6Ax^kH~1r^&ow5
zX^k8u)MW1JUS-_R1b3Kq(A$16!rvmuxfs23!Sr#-Ij}m50OuRS^A<B=ANR*~^+DFf
zV(Yx!o_Xm+L$l)Zy+!4Yz$0aLrr=v|^R+S|6~JVD@MS8591`2Dr~s0xzh$7?ayKBy
zJ9Dfs)lSd$l%9}lwYcHhll6-l01*pE=^W0ANd=syeir7wil)p7*&dbS3B&H;K1>vr
zKv)Uf!4cFXS0PUO{X%D<J|mrM`u4J81}C$AgnVyPhL%YRGSzUj@z`T&@bvxaZ^YK{
z!xh)RLylowP_@73LAy~PsGRyNDj;EA-m1H+!pYLa`~Awsf%?s@q!&8-UdHS`{cWb5
zs7|m;CLQwc@NejFhkIX1Uww}Z2tEMFhSIpT#B=#pcgs~c3fz2WRqZFK9)|PFh>eO+
zF+1HXddO%IK%28&frZHCn<ISu^tJ4QoZA+<frhV*HdPfJToX;FneHFQ;}j=RhksQ&
zUlITtE^^qXPGt@rAqi2u>Q=?CJ<dd)LW}gp)1N|)&fB8^b&aP>Wo^sBeqT};L(dt_
zJnXM;a^_lprO@0V<X-wCCQvwLKyL}W0%{Mg5Eo(X`@*Q}y2sXF4&OT{#dR7#2U$om
zU|VB!fP&y{RlBw2SD!!67tPb&pPjDnbbJ1iUEypcRZVmM2k-r(rlc5$I_wufViMn=
z<2%ZG-a>ewY_x{9TkdHhfBjA9=}ZRGo?eer7w1l|Jg`KxGeaAPKL<)zqrbRO@BDI`
zH0<eR=d-M<ysFyG&eUnOMY0GHc{T4prYerO;H1Yn8WjEyZ5N*yvo;^-uw7(*<wXXt
zp;+h}9taTlKt<!k8O8RXFQ!Be+Dfky_mAd(q&<q3+s7bYpkpx6F@v#zKqw8B;nxTE
z6h2DHw3L^k%9XR6q}K+WU;NWJy}$IPI#!81Opq9lu~gj*#waxRlgk$ASb4mGi2I1L
zo~9wiYD+YGp1c2aC@XRpD2$q}X8Y3aY_xV?)lu4&3Y98|GiW}r<fg1>y-FcDXjDaY
zpNcwd^D~|#!pw5HZ33XA03yDOcJwE(%ULmsbe~=mkL}F)xm>-3#78S!eanDzfxDNn
zs_<x}!msHl7~k6Q5wlukYSqCWW(#B#!iDELfdM3%G#^zT*xus{SXGxD?<{d{y~{6!
zM;}K(%olAxJRJyaAGHEoNwvPe$}5>JfXr}BZ=57nfjfnsy(D|~GIqgfy7D3sX0B~K
zjuu6b=p|qCV^^p+_ZBkC6%)+_OZkgDF5fu`?-5m=r3dEtO&8QXEib>5Ia$)#__|sf
z`CK&jN`w&p*SJZKKtVs<+*5$Hs*FdJ<5XbA`DI<R#j`k7IUjc=CEI$FxoLuzldzW+
zqKOkX5OzBSY2qhQw`kSVZAs4#uL18QD$;Dg@{518!rUAr59rv5b2F<@CkYS(LPtT@
z^Mx^8UJ6u6f^;X^51NiYZRa9_y$>EfEDfO+e6T5DzW>t-YJpjL6}YO8CdMQi#yr!X
zMhQi1gw1)8H_qFTVaAe_=bxZwKLc1EALW|YdB@i*rQxt0`~2DkbvuGg>O><23(>?;
z-JtSg?h;ElJLWLn@948mKaNtbGc|UfNj~QAOTJGGyd8@OIB&_@k_yiWGhM5g#!gB+
zN@?fjM?MVwBg#N$?JoQ{I++36Tg6=yDR+Mx`xEdt(18J<MD!MMQB&9Rf6EIJ0Z05Z
z(L}-BzdF0Ns(;MV%s%hYVy0QyMKap9(>}xgB29OINEdIlfg62|ljB-HjL*sjM8#nQ
z=Jujjogk2!g_bzo+~M&Y`@wNgit^!&4DEHNFsO8~Q32(PLMI_|lljbXz&xA15_Hy#
zk>@p}Z>y7E;?zET0{>Eyd_%+u-4|<?7H<nu;kkvTL8T1EW|!_h;5BbHK?`)06A?VA
z`)6_@@YODc-rhdxMg14HMks?~6*|BjOz{}7J3GrbLq{<SjC_?p`Hc<yNX(hKeu1m3
zPt!sa`ZdZ4Cay`iJx0VcXBK3-_*Ekkd{%@(=49rm=@*|o6lmL$6x`a`ckE{hFB}#$
zZCZQWgH=T56@qutr$;|zcHuxYB)=}O&tVvUFI6dIo^1OI+wJVmXTCjswDbo_dD?p_
zpTwRQ>qBXajk&qGy0qB57~tos6n1(fs2E$HYrQz=ZU@!J%TR9*7GNK|_U*q3gocD%
zdv1>v)VwE$#^#*|N_i>IEQ=_it)r-K>x0BKVe?=Lqlxo2yUJ=!x^Vw=VuE0nx!`oc
zQ=IzdJFUz)4OYWm|1ukQGDriA20osMTHJluV;M4NIQpt{ZNL;=P*YXI-e_zv@!o>K
zE+AAL#|ieC{~%%P-8UI+JBXm4M9tHGM1zVI0VKaPPyRF81{p|sKNpL4xK10spZvK}
zPF9B2J4jF-TaKZR7pF}R_%O`UD$3wn1v)j^^2{R!YJn+A3N2-)P-Bf*b%sYfw7y&Z
zq*Zal0Lx#`QZQU*Q($|%HY{_PTCIXVn4itXI2rVkR|CLh@VkG+N8<2zgqbV!CfJbb
zxChdQf9k{m5(;qCs~CsPSLmb`EtkSc@_*9e1f7U3nKq|63#rRyY+awhBO1R=xlWGE
zCKNl<mE&o}xvu;2aaorG9v3pELS=&us-Gd(cv29=1!>6f+>0sFAguC>cVra`!rM~$
zK8ux3az$%90aD@1e<D}iAZ)+gIOGSI45g}%7!k^M+M1Vj01n{%a}NpdfDwUaHG2=V
znZBwE2DyLspX7WusdaXNqX9qK&KPF*8N#c4T5foNIMUy06Oayy?#QKn+g*Vn3M?@`
z<*wgMoT?EH=YWZqUi=vXkXP4!<cwV2i|aP4;DI$Z*L3qS=(yG!(e<~*fm>~;ldYWm
z=<_Y47Of;3$aJj$T&E53Db{tPW7t7hC@Hk#t)h!{6#^L*Uj1<!W0lzc^0mMXs!wkI
z2b*m~&?OE*?B1)dR3oJVm=Qkp?@A?d<<nYHI)$w~nFLXncw@!#@?j52frF5qB4C2V
zWzKHra^@+kkRZ&js<$K`d&VaA^W4fD>UsY3Stox#p>|VRnv6-flI^n;H_a0OEzO`>
zN01_bJ$h<ykdqcRbjo>OOqAC^)Vx2XvRXx-Sg1W~f2es@Z0&YHeuWxQK<fN^gtID;
za_Y5PSO=uFuCTJW9^*<=xQySO?6CO6_9=4YfV^BjBLYhod7jHZopTEbFk7RIP?Bu%
z<UZZbwg-yU0K<crgdLk8QQJEpOPzq&TF_Sl0=4U`3N#R1cY+D{1l(I|Ot|ABvI;#g
z!S|Xt=2W1RJiDd~pR6@u!Y3G<e)Z#Ge@AcPtf>BA`T?fR?WF8LTTQ7BRw(VOP4piQ
z0|B%7gJxv^gQUw+lD0;e=;;vqCqFwTjwhPvTQ<M3@{2UPcjRYOWb#T92INS<aw7(M
zh>Ju-K~n!OPo=Mmmce?tN-*z)sIzIXIhyTRTn6e_+}Vqj1l!2n^8HKNM<wv$I4mXb
zgcV>QbA9j9y&6{#<;y>cuH}0fnVMdpeG)+f;(JCm9mfHaj;=0&=*UP+13PqxWV|EZ
z&hoOVvXZl$HVAZ0j*`5=ue$Bt46`(cyC-zi2oZ|~13Ux_O=StpKewFD3+D(ZMQDW2
zUA{0m&)!ivtB`x2u$YJ=NE6Pkl8y%RBvXG=;ah(6G{R3Rqqx*-<DroC<_U>2(RF#-
zGp92FZBeGWM`c@_T@~lU@DHRBF<VQ>pg=?}rfSa*q2*D!$nEe^!!Uilj%vjg*}ozq
zY+0d0K#R9t5$d;grQ5a1y`Cfnp%$f5AByskWv`2qjgXn&2=}-E+7EKH;)aN`TyveM
zgoG>5Xa8sEE*i*7WxxnS-%R&18KmipZY9f$mD>yo&r0EMTtLOYTc}9AAHhI-Ua90m
z-nNJ~k2yB$>0i4!UGNaF{8O&QUbd*e->3YlwkZ;c)x>GwW`Ks3PZOd&2?Dl$1m{-@
z{G{P0zgXDpj0jCL2GQTKGA`6L*on4xxYQJZ-mQj@t|)!n&De@#e5kf-_xHU(hIK_{
z)aoi@D=xUU3+qc7Vq7Z!g+fR7!aOz8)U_tJmc?eF$?)P}@j(qJU<fTggZB-NH7+q$
zgdW!30Hgw89nF(p?9GH4gEC?&Pk=hE!X2{k_D}70kncGtJ>chQ*;~Nuo;G|DJ)%Wg
zfav>=dPIN#QkO)Ln&I{84Z&RuDuoQ<KaqZCSa#_*J!;5XzW0C^qQ6C=J37LPRx107
zA3J(-DcmJPT*3W~4Y3o1n99n#LMBh@L@S10?CJQmno?d`TVEH4&-q^1MWCh{W~`mJ
zO~@poP10BdBa*nL+sMiew1Q9!%|AA40xOJt*IbWBVjcr9)iLZ~;lhJ_zf+hgKvE8`
zN^{0<o%<&?*I=(XyQ^x$eYi0f1pwjI_%q0Lmuw9m#vfTOW6D}-UwT_|A1>}7eL^J)
z$Zd@V2kA)b9x$aqJmFyr<l7qqqa)AvT5Js@R3_0r(@cD&w=IE=O}bfw2t1B`(7ilr
z8&JDMtP81!UVxD)$=wl$&hIH|??yW_w=ti<1h-nA4+~$UZ{rH-I7#Au{X{O4RQEg+
zcdq;)JvVpV4PoQ!FW9ybjVhB+BSmy4b68MuIm72NLP+n=rm&xm7JcbL;&MY5KI@#p
zWr;Y^D&SNTNJ^~n|FFQJX2#8Lw@3K$EZp+b23%9!8UkioKiIXe@M3=oQM8u5c#=`+
zS9$f^k$z;L8P3g?zyE5kC*-zF$%iG??rOsEG;zq?mU++7eJpo5ewN>fY?VN-c0FCe
zRt*?3&sQ;iF5TYHA;;3jxl1)Li}6blZi5LB2Y3zNj%ouf+Y}Fb2TPSZ#<S-S0|{&!
zMzsc+Ona|@&*~^}wm~=K2g_LyEQZ$`xxk)Ft20@Dm?@b4lHfIfUzh;~T8ex6Z8QlR
ze4H)5AVmi%VtLh2IP@Nh-u3w<%*Vm95RcT7yt%l{<yeeXTOLsAWJlCywU!RMxGOqZ
zA=OPh%;HtsQ~~aYBR^<cB#I`LF$+M(Gs0i2(*v15D#rm*K$AM0k*+dlVQjfk;`02^
zKvn`jof!!Kh1$4)kdD&ENAb3xFa{8SRe;yiRZG3y6tbB_1GisB6yBqQ(LTwmjXMA+
z0*AU<p|3r@R~8rf#+FEm%pWIjpu5l@m{16&@Hni0W<WyNJO$^iQ(tT=SzX%Iry|q@
zsu|(5Y_EEvCi8Dmh_X6D#O6T4I`JXt1lWk1AG8V3sfGu>z_%NJn&@&b9N@hnl8mLM
zWsAPo5dkTgA#voVzqb~Ww)F<;t2m6j7hih+Ue((9%=|9KF~@4t-f_8z7{)Wl-=6j-
z8Eu^v<KCxKo4YQh`ZzUXi?z^eS3qWVlS>e8$qhpzvyfRU_6%R<lCQ5BjUd*p8MjTn
z+I}oE=pC2CdOwXu<=7zTx(~ST#(X8Y2)l7$XQu9b`KH?TQ`+<L7o42VEODAfzH))+
zY#R+xR*l{tfIQ;Nv09ZkL!`i#fqW(5YqTcBrD7H-McE;Ry<QuQK4*~7-0}+y50`!E
zl72eZ%q!A0gr+dyGDcLIC%@K91yV0Eb6G#$*DI%9R$*@<J%D@Q=xkX%zg#=;)<%T3
zqbi7f%4e>BT$G%3Lc>Q%QUZl}mdgJ^Ifp1S$3Q0GqvHwWzDx4=n3=iTs}m13$OFTj
z6Vo_@rpM}gh4M2@$JyEW4J~#e{L)6r)Bqxo$=FwD82r`)eltKHrT6Dg_Y^AE%h{mm
zD$|Rn2p>!{^v$kw%v_&LQlE4L^>DD64tgeb_5&%X<jxb>Ppy7&0Pdvk%pz+z9HgA%
zJZM?@jH3%bj`DTNK>DvVT;4&cDaUO2tFX^>BN7%*?;oI0Qg~tQ0}6)=Om@_r&gCh-
zV%hmr)C6k<yED7Jya1xJ7#Kc;&NUU<iGCG4qQ<xibkJ-Xl$NIzFe|8;!#ogGxUCIE
zRX|LCLuNx?-Wy=e`uCPgons)F9(V@sgGC;!^OM+@JhY=FBHpL=;`H4rnHxX`9>UmA
z>ZH8hh_&12d1<+>c#rlcusw7}QX*{rHvW|<&VPT=`alJ$Cj^((Rd><H8i(o?ycEhi
zK-S!NLy%XC`+Mgic^oxiqsfr}%v-I`@^p8sQ{fF#(jih;{LlWpq}aINK{*Yi9u?p`
zU;j&f^1b7zn2V36`H!$|?F=yTZ3lVhT}uh?DXRQYoGxNuu;8bk$lS(WcEVEQNBrJ1
zo?b^yz84qBb34&bwEe~)?L+;QfB-dAi?|GZ_3d9%@WS}YGUC|`MwVAPyVGWFFdU2x
z6_86}nWs~J|0pldV3t1G+?uOMipDK$X;getv*hgnUvHVIx8#{Vbx~xds2fV=IT>OY
zsAaR9D1HiI#TVp*8_z4cDvK2v#0`8&YS*lRdUzpJf`InAu}`ta&7GLWzH*B=*{1Rh
zS{Z0Wm-P*h*`pTvbv9nU8aSc&+rypn5I=W<x5%P^?N_J00w%Ug{VGH%(r+k+o3UH+
zT;QjF7^V}k<`mnnBQ&yd_n|P4cfVe>4KN;=2vxMPhQ2;BgEVtws)E$)Wt!xXS*FQ(
zrU*xdsRm%@AU93BU&9?C`#PW<U62E$AX+u?f`5kMGdt0Cp;J<2eJT#6`jSN073lRD
zL3|<U+aDA@^6U=Eq}?T}SzdRAF24d?Y7Td9K(35^?!i>kOv2YxwN^KFr)BU3=81y6
z{r$JTIg%*D?bk$t4{aanqcrP3PsbV?S5h|8FjYCjLC9wU;`iqOSx9%Ig#ZU-6v&eR
z*Id!9i4`*58i(Hik2o~p{d%}Iu3IBm*7EfNYhnJo^e8kqMSXc+Ma8tq`+}?5Q&-)u
zA`k2Fub)H6dD;~F)8;426drRZdd;Ht!`;|bu6}|Q=KhUPL=Ga*0S;i)VZW2&#2W|&
zl2vwBK<bIKY?@0<wpRl5{2BR|k{=u(EqhJK1mhr*Yt+eYaFqk>!ZL&#tn>uUXb1EV
z(Bi_x!*u3wH;@cfy*tZz&^n!qCn3AOUcY_De!5&OCMibE#%&|0p`lS0(lgKaZ970D
zgI>d5OyB2gtJSkEbzq!vu9G{~;9|`HgD7D^aQ|l3i_ax(qP^5Bw`jd$MDtkVw0M;=
zsDhx4HHGkETAsael-;+4i+Fj6X~vY~M4t97A<eFg46ROrQY5#wGKsCUZxWuQCm(cR
zw@Vh|#=jLzg2js7CMjVgY$OaBLgs@EArFCspc(_DN>5qBoP^X`e7{dQ_2yvG3kDz;
zzr3F43CXx{+mzJl8Z~zmW?1KU1e+Dp84wK7$J7!xWw!+>Fi*BBvub?oW5FP!m?TlZ
zE}~j&_F3;w5NGH-$HBrn={`O_CIr26jn-`@YSUGR*h`w->h^GVu~<~dnBaTv56NB0
zue`2L_ZWKj1Nk>4@=qruw&kl`+zCyw4+`ljk!4hDv3Old#wY13k-50MpP8xNW4GZi
zV#HHQnC58?)ERf+4LQ7Jt`<2yR&`5Cjh2Jl_~CqMCMP6{R*Ds$*+5xlB=Tk@pz+nv
zUO5;aR?p>GjzicJX^`7H@dHo?y~!pG<gBqq_{ACB)7hY=rV4HyLtt+YA_6T#e$LGK
zy4Z-Jrxk8~1~)IeH*=zB*h~iM&_yQV9m*kM04lQ9zty-3x8KF;-N;BuAq7jy1eBv1
zZ(;C=zIoH11~iTOmq~kh-E}g2-w!@W$MQDN_W9v4v4Ys#N)VO8tEDA)V)K+R=k|Q5
zW?CuDZ^<+;Om?-age{i6*;@}fx#OjA1A{G>z`Udt<;D1JjW{1cXFR}(D<%EMc$N3&
zm_B_G0oUqvvzoBNm#@SnN`G&%@W2@xSh~f~SX-VLkALm@Gjcub7O+jG(h=e(JmAsM
zPRE3mkc^=lJw&f{VXUEw?aO>&Yq;~zEC3)zvgZPBe-;xHQxlLgrSD`6Oa)}=E}V+>
zRdOG|5tmvW5uY9weAb0^z05xd6t6THOimcU-1Uasr3zx%QGLCbW<ShlrDeKmN;izp
zb>);1&Q9zCZOHUo4CXTyUe|5zN+HSOiMr^gSnUjfxQ%P?XqDb9W{w+rLs}e=A=sQJ
zU0M3ym}r7fCtknGy*~TU3aRNA6|!qak9J|XYKKHv2r~<6$@l3>c&YinZ^+3U=0G|u
z4{6P8VPS>54-XsP*fgg6WyC8~6(io)|0uJTq%QrmD(BBD1qe4v;e9rOfr<PW1r{a)
z71oZ-G#ErGlCv??s6^4$)`VywNzLu4b!rR8N|~$+v<gPzQ%6uau=UPK4c?qB=WG}a
zCOD*d>?fSeoiJp!Ejvz?IdW{N41(ISE}3Z78BvOX-|P3>7?VgCCm&S632{GTtP~td
zNY;qjR11<XZo0HTd|lKOSB%U#rHst~jkFmQ*3rKq^-Bs~`{fg<l5%ZeiGp@j^2Gq(
zDx$>FU3+yJ%;sWUKFXxl@{!OsYiIGm>!HlKE|_<Ga{b(Wx~zipXF0jy$a*%>b6?rK
zH2amle{WgO;YmY0vdJ_NVXd5C*TjQUn;uQM@U@j^_E>=Z@fn!-z#?zh@?&ycu3Rg5
zV~TNbXU}|)!XvTB`F^!Qo1*#qY!vFGRSKrsRdQlO!*>fUZ5tdF@Wf2v>mB+S@OEkA
zPM&jc!;BrpfQq6$-ZFm3$-ZgTP=HanedwUGWzHsrh*v(CUG^pdJ4p<a*io~wsUXWd
zGkRf33?Z0s^^(2Q=AK>Txs{Cl)fbthr(B*-0Mcn<L@aR~&V=eXfVV^!A^T8t9=jz;
z)XXa~^RhPCWAaPieWPnTmym?f#dxU3Xz+A!sp1m2aM2*r#%z}qbjzN;l>3b&|9v?%
zj5<0oFE3qTYE?_TPtJ+B&%AkB=fI6w^bLb=E{WwLk}bXOWc(qu`6&YXUh8=g=guHg
z`SF%zU_t<jtg$SgE_pVK60)xt^Ce39=ujZ<2PRvI?{C%_>1(wZpVIKOo?KMLwKhH%
zma3<fo}>2G<1`;!dnnhg-6ZHH`8ERPJ-uY~PL7N`F2>e|$p2I2c#x{MjJ#SjYBYNl
za^lN^Px#4JU%Y+)0n5!_Ab#u6tB+4vI7{QiivOY%AC$JCs(6R=A{u^4NHE_y4UKQB
zzV%8bCGRF9;Q~GKm5!BRVL5z1-_w!p+L!1Y-zf1t`?e8^`0K50X2d@O@4rOKuvGbP
zFO3=V@Svu3zi|T=HqWQ$0Zz6oI5K*J6}d{ScQ2ejTbc+bT7Kf6zp7J+KEatS$y4Pt
zDkvfx{}cn(KvUyXPkLi_<dI>{j8UG7e*(1Nru@88tf%ez+@NQP5V5eN@~Y>P@LP;?
z-{|<MS^`Cz2xbFB3<z3?0$(P7ofae~6QxD#Jyhv^_rK$vh>3V>k13^m^z|#JAi7A0
z^e?c$YL^vy)SD7D*28vdWb2UbomqD0`Dn|<gvJsukQg`Cl&Bu-!sciBtIiLebe7e$
zxm)+}Iy&jhIBS``OzIn&Fpuw3hQUTU!pcZcDh~J}z_(Ftvd9J}yO9&tmn=i&ev}hy
zZy&+en+QJ6Hc`Yi2tVPzS{HhFT8-K@?#<GiRp;C29j*7Xj@<&<(mjG!G`1eIS3t^B
zvke$W0Z_rA0c{%@^+$3H;RaL5e%Su_wblSPH9kQt?Xm?)vC!d-mD$#lAiMb|1R6+s
zrSFaq5O2=+8Udj%Uvp=3PAPtR03xhfq!i-R5n?!`TUYr1hpM*@3*w9Zgoo~K=}x7)
zyFrjpq@}yNLrNOyPHB*mZjdhN?(Xg#zQ5gf_vLRM9`4M&=X~ni;Z7T&g}M`_7b?z{
zheb%u9pMXj;h(RiT_=ZUg(8Nq8rM>LmQXdfA%O4o<{~Ndb8nXXJe;L^w(s*7@nkH1
z=pr>Wq6EsP+fzCGS+Wbyx(GrhI$-e|MJ|eEW`j~H_xwH0c&wgRughUAKC^BB>j>d3
z@bu1K*Ep9rwXZT!1f2{r_2M{Q`xR<@(<?i7#ZYMtBdqKrB7U(%GC&4%Y;CIA!P0uV
zLV#glvVHlb*}uupQ&hC+aP{Z)3?#Y_DashHzpv&OTkoOFlNPu*|Mm(?t&-6~Rnh2*
z5zln&6bVd0^a+xkaWvEx-BTHGkzg^C;j!H=GdMoEF_%`cbBs^ex&7+b`_hz{`z}^H
zGi}(f9PRs6-`8GFh|67NMcP;B)~i;7v%&m}bXP4zOP#la?wOflV#)pZ0<)G*_eYVE
zBQl*)Fa2f{Cdu}R2*6cx@KIIIDrYli2JlHbtG!{?I1Lv%!*3WkJUCZ3yk>nTz2G&Y
z`CATye{6nhkt@j^hh|X~k%9RJ34iV*d>;Dri)~DAS$|na-(KV4@MwS3v*g`x8kW1n
zy*UE?kpquwsCMv950N}to;4Ek9$Hl=4C8iee+f1_gPlN+1^RG`a${KXNx0T%87;2k
zo~_>k-4B8db#<nspE@a-<QPVpIe!hWqD<M)jQc)__`CgCRCRI0_u%w5@BTBnboq?a
zpwoKJ7&$N3!L9N;Kg;0ajtlltY`4hPBcp?NJodLZ+%e#Sv3B+t7C+!xzr69}zr4{x
zBiw2eD>$NVsqETz>An55S7tN(pkf@@RR6p4LEAX+K|Ou!s!r?*65kS7LhRcHHX?IY
z4ULUk>mnrk$JZBZ&%Lue@(d|*`!!}ve}qUd?5*w!($|MNiiC?$5RAKBb4+q5Y7NQ2
z=wNbrRkP!3$7yGp%4C^BdT@Q%@WhmNfTUS}E7dv4H4ra!Ke{J4%=`tah(!-%P(H18
z<8;l~Z`b=(K0i5W$|r2&wGZ=i+$eYA+~dx2nPiggkVfGG%IKn1ecZnX65JMc?R6Z&
z7vDQqJTZcho!!^A)%vbv_cyH?r<Q4jk5Vi7o$bwlGbw}<qMwtW63dg8tec%;p7ZZ^
z!%>M@h>_h9$TRnZ;XG)p6z#*zEQdJlY&Z_n1GCE<?5{n%bdqGTGN|G$*#2tc)a&1H
z0mB`*032;fyZU5K#2=*|cjx%3dtqt@gol-8x)*yk!8nXv#&Obj{a=HlM4%U4Dw)yN
zuPgc04L&VH=b&C|4$C_Vp^aDOn)02O4L+1fc5=e(S8xK$MR_&LKqiD6G1hXK`CyMh
zw|oBbhC=<<^!g_bQ`yd+6|_hdb9Wj_QF+ouSFWuw_D>}q&Z_Ep*E1y#W!pbdG&GF^
zua~(j7W3#97>C^wuuIClltvJjTp%gAPH=%ri2=pXDSYK}>tW<S(!=c*han0aC?a1I
zw7c<{`<<%)#vt=t^%UJ{gulOt02eDk!HYae{AI;SmPbY_ksq~9gk%9<6<gY29_+Lc
zTrJi?=e?-{?7X#}z5&WA+IXQy@_Kk_Z7_a1z0;M{iQCJVQGZXMJ<7hiYuw#32pF`R
z&t-KchbeusYA!gb{E0GSBq8f#kIr0brM6mA)M)Xn?&&^uOqmc=6K21>&b5r(f-p{q
zxTy%+E@TCLn#w*=u$b_>MCg%+)N4}_v4F*qL4+O{4MC&z2TTB`9?v3?9BEDAB;Mk+
z!4X@4*#C6KM$`7pecu?ozF_S``rfzRFX~y|<~|<%H6LR%ZpQ|t4vAPueqeI0Ih@8M
zKSC@~XdXhqLGY+Z^PB6wNBF3v<snIDQg?oS&djV;Yc@ufI2)W=xbJ>ak3_!Y&?g(!
z^b3pl1`vUdg<Ml3)@h>;y3>}!MHfp))z_x8#+E$C#>3;5gAsgpelfBaNBTqA=uUfg
zg>!TA^}`Cf{i^9qyv2{0Ew0|_&q36Aq8x7iqkm7f-GlWj_ImAZ7<Ki?+HZMSH<>zI
zWl44Jukyo}5^?_}z+x}eM&MF&wDI9~)c%_DVi){VDC+H-*?Y}49-HQQFKpFVU45(^
za8C%i@(<pYF7?91=b;eIiCOD9@owEqyCnC_9hwAfO&hk4y><fempI1kt*!|VJqWuV
zit$!s+pa%lQ~rKD=qoetc_PZCWV6SVDaE-?ET(AP{8ClKOIlLapbOu1tdiTetUVud
z{e4+glN#^Hzx!f)2y4lvArsqjvPBD1WUBVyBiD;f!=ozjO44(og}Sy7!@Xv5i7#`D
zIZ%yrbwl|tYk2r|e4+qKBr1UkSR)4#(X(!aA@&?$bW;C>=si-?acr{q%3is7r>O?N
z&uxq!qaRX)waXdZnf0;YYh?SATTtae;P-ePfuzhCCf-;zaFpTS_G5MgUADm8|B+J0
ztc~lIiQ*_UVE&pHVv%6)0`l7gXO!SgZBywnZqfVI4JJkzM?P_tQBA9cOfB^+XJ_NQ
z>zl#vWg(^S$}6cpc9BR<=tA5vBhL~1f$i91(dpkeVQW6|-HsU&71df6PPx~dxJ=$m
z#o*XL>vS&i#?S=LX6UraG|DS9*|&2A?atf;gvS@v<di95WsnC9aIOmElV+gCYj&%C
zCMN0%%>^?A{b5quCi3eyVq2aj-Hb={NS0+9srF;tf%nZkVyE~0v2~~LW4`rh!*!sb
zu^piTpUYzFH2JJ7a}?gKRz8KkGFnoE3>;n%;LhIq@aN?8QPc#dZ;-u_uw%z2c1n4)
zfv0k$;g8eyDz0Mv+~DrNe<0qi;m<32%>!$^`wu))_H$vr+@eb+69r&=mN|}wWKJ55
zjnsqSQ_P7+jGwT?P>oWyp4!Y;+sh<J2O=TI7FRw$9Tf)zL=5Yg*kw4>VXyRr011j%
zWFihleytyAkb9x1mF5c!5=`#p26Ph>JC%}>lBe=8f{5h7QN}_qQ$4?z{@w{7<@mAW
z&8CTTEo{Z@&)yC3@08spw`!sK&7gZOEXVzV!G27bY(av^zX_z;NU65>a$kvl!nu#n
zT2KVf_09FAj```wy*&){9rIz$9{VNu=3e)b<qfrK2_-l_rCvtD-{49Qy58O;BVm~$
zf?h3gkQjg1J-OcwBslfbL*c=$UuseH0ilAB?<rvs;-B;wR5WL2(YPb?e7)v`onS8Q
z-V8j*YuSE3`iV+CMyd{*TzZG+Dje+#1Fo{gvQ&WqH<Y%Al1P4VPFL<>=oFS!h-&cm
zvb!Wa*a9^1=I-s!G=JSmg_KHbQ96CJeUAhM6KRunTc@-?w^j@HC&@HV0(^o*m)~)Y
z!xAR<la3jDS`dKLpv}V`wnglK3)QioLJNNB3nB@2k!~i!;hWfKHSFqh6=!L4Z*9G`
zM0CU;2FZ{9sfnn~pz8z6slKX9jPj!WLelwr!N`P_PbCM_4mXAfAo9fL<6+6EEfFfE
z7H&O30Fv45TTRtlQ}fVUq1Vdj+>R6_uJ%fHYd)k2H`?F<!_AZPTJyhl0?g$T%BoF0
z>XJcB-Z*pmi8)LunO8%OoR6M~37lQ(SH;8(%P&IopzQ?;p=98zSGR9heP0tnAqH6r
zugH+t`ejnDXxC3b6pgX_Hl+{(e;%um{qo!9ptbPH*f~ATb|H&PQZ{|6V|p^lnY-0{
zhnC7%7eS9?hrxWakm1GPkHF}$-ZDC#o1z?wg*Ak+!0N_AUk7pPk%w}KGQVI~Ps9_=
z!9DIS!VSkRRdcViCwL^gZy0qHbLe(`^v*$gSuRR*aH_$jh1BHLXe`F+H==SehmIwz
zeedP6_9Z#%<RR{FLf9T}FiEjl3-ulLLgvl`e`p?EB<UnpOqE>4hDtQp?lHrR(CPQY
zLdU}PY>1y}PF7kUcLNoZ#X=*1C#oJ3lK6qzTs>zl-qY@43$q*vY0_L4YecS^6ZyW-
zuN#~@husVGn@qj=f&@f&C*@-OE)-1!{e1PtM;2NeZh}>EPU$CEGp7y6$*FU4I=SO`
zpVNN|4l|RmD4*tr7V$=pDLX3FemyY3%Wu>O0==kIt7;IWXM*T?q_jkH2;AFIgRqBb
zVC!#K)l4V2g-iLB1=F_-GB1m4G|bfg9|Xt?1k}o-=w=Ir&eyYWC%!%leI}B50()oW
z;ppGIK#HHcKVGA+>?kNAVtiGm_owwY`C*X*c@kn1c&cT(p0MQe8RZqVwU=JLd7Jcv
zV*CVNit--ga2Pdy;9K$WiQTsQW(zBi6$Fq-^=T>y6J(=56PP&_rCgu2k?5s6bjXS5
zL8GT0ankD~%<ps^yki-Rs{SarRL~{CPT_nCU&~U=w43x+JxTcSg5GCHqWS4S^CYWu
ziJ)?jv~wTllDkB(*ELfpIa#GSsMxY<f~ZHMD$5N7JwJn(7jDentpMxxG+K~rQo!!U
z-hMx}A9tDiibrO<;X2i>>pOD|Ku>Cbl4C)BrRk@#9{;c&>*-)qk!UjMdrJT+9<$BP
z6#=r5_Ba;?c>!CHkzlo<Q1sc?K{8q^e=!w&0{G{%Ki!j~Ojdd$@JOdp$88!&sdB}L
zpQHKQ&h$I8Cx(On03yHUEd>CLtJ}}uQ=n-{J*5#$qlEl0j$ObtH{7J0@G<v`_YC*h
z<<{?(RoK}&>}hyNL60Ds)Uc{fJsF((&xYEh^ZYWmG>KdJkMeI+n|GAGTVmnQlu$IG
zVFS;n%lD({0Zv79RXoIR`#T_LCO)(DpeDN1OG&ZRs&++pI_7#7+y<5i2~}C%g{}}T
zm{S+od;1=N0w}W_4i#SZ3v|A{0UWcBfKK$cM`nWH2!(TT5=<t@!#Kf_My^zfW~i&F
z64+~3e^MyScfEb1z@4y)1H}2eJp=*{C2$*1#Z{ejU+0%RLkc2O)jI7@N<*~dZt^2C
zFfd4=#VvmPF|IuJ$`o`b!mu~-xVNw2m%>6_EZNvxDaGZIa!duxUT;scyIDJY8U06A
zo`LqT;ft`A6>{&?J=%33BF8>rH3-;=1=;E&(}z>4&-Fbp?&$uuK0P*1Tb<dYJsG3d
zo!7TEoNNmaL(wlYIC~F{?_?$CUJ%ek+io_Me}XJJc9CC3lDKpJCm=P<O~rBY`JFbW
z(;a<7;NJYNM(B^zAe!l@(QhBNk4Z?j!e@+Jo#1I@H%k|Y{3%0)8AiplNJx^G_k~U)
zri?C}q_lgTB)Z5gd#Js(mIFuRSM1>;#1e7XIf&$tsWcRX(K_y1r%nx&8HO^Z@HW{!
ze`*7)-b}FZupU-RzxQQd!qIvHZITzt$y%lq{oQL1a|=dT`ZBh#ylD!g_ZcKwd$s=`
zCG3BL_L>vkUWojwOGlR!SF_>n&imfWsdQh1zSfCL2^AIfJWlip-_f4yOs-x_s56ZO
zZNIQ&(n>ESlzimjtfzlAW&$uBnRJccub|)AbAu}aH*OG=;K{eFkc;(=jSVS4iFox=
zo?K#31h=iwT;BPppVGnO1J0FQF_w_tpW*&fS^9xyhNI0x@ynZvQ+&Jhjj4K<yG_?|
zhuywP4oMXVuc<HFH$7&`_pd@4!wLXhH_I)i+p*pC?abnLO-_?%7w5o-B5(^1Atam;
zV=u^xyvIn)?JhD-5dA<|^n0R-?$3s_Ue7YB{?}Ro)#1BKkP&10_HHoNdv(*q%Z2R3
zPRRO+DKB4(^_{;UKK`trfmD|M2i=zUG<|*W-XS>Y?KMdV|ECIRm*wr_<#4g2D7a>K
z+w$d`01U7tPeX%)aS%)BhBYoV^F-Bwh{f5rxa8RAifEr~O%&G`wtf@r23filcX1_l
zJ`-H+D^wOX(!e|aYTqd5C$)vJoEkS^_`TmM86_Bki+PFye8=B*VYB26DSfqW`bNs0
zC7rDYB*Q<qZ9HG+=H&iX89^{R;ziX<8{o#8a6}|lmLQz9(Nw<G#MZ7pe@|gV>EU(I
z-+Ql6*IBqhZ2t6j58}%U+?p+*5f=dE5c|JZv+8}jtOj^MiiMPA<Kf`cL4K~>Z2ldL
z9uJ%Q<SnF^gPL)x92^`>3hp~O*<77j;LZTQS-^kYQHShzH>UlvW$rwv{)yS%`#vg)
zlN|iDUJ;eA&I9w6ifhtM^l|LSPW-R4s#|xru`fJU;9-FH6&+VBtM9?WLvj1|<nzp`
zq$cRE=!KZ4m%sYb4Ry9~>|UyDe?rq143Kh>t<%si{tDM67@I|Q2OES3Q3hu~!m9I4
zLm=-6cbC8=!Z94w5V|vWN`98MytN9d2ZJ523`OC|c_oo7`Q-!nJ!J22bVDY;bo8`3
zh|eSg_VO^TvzSL%m*)}<ycG)+9t8Peq7oeJVvDbYSU-1~11z;7EMF(JKNGUlDB;w9
z0`dt>VR~Vq3uNR!jHM+~UG&`zInq?z_IN0AL9Rr`&GQC{rZHrVAh(F5x)#(-!);M`
zTtw)NOP!=HWHZ6tn~%2pRdZ=2!b85=I0x1Z$p=pfMWn%1m1cdfN`mJHJc7dTAdIMc
zy)4>#-2((icn56kN2`l?XLpKb1=(FM>X|{-|5hHf5jP@%53}=LMWd%raiJ%Z6IcCW
zp@gt=@9D($zSO@^1>Mob1tubTFz*fPWlqy(uva_|J6)8lII&iAx$xqfN-D6DFU4x7
zKBk>4gunW7*2IONw&RKGvpKWdn$8XMd<`rouxa+#)Fg`!r$e~u?I<?VIQWlebb3Wx
zl1?P(5oH&XW*mJu&9&G1@cT9|fDo3N*s}E9G~@GjI#I~jx55-)1$lF}7Q!vkF9c!<
zq7R#fZ`sB^YWV-b4|8wo$bj<kiFwhd!oLZY|5GO9zI>UV2TrCISFE^RHE&;{8xbE+
z6Ext-4dG&*#OwZ9i$h;zg82Q18_uA=P*z+fgJuJ5;?9%Y?TGPqVJKs0khAPI@4-4A
z+AzoXv6VFlYnJt7Fx|i5d+|aI*nHAftl|`Z105_Ez+J9ECfm2J7w@Z&c1EtAsgCdA
zeSf5eM4GFpd(SXGNs5Xcs@c5~Qfs1jBTE~%a>n%8zBTbP^G`WrLDt-Urhsn01h-`>
z+k{sK`=pJ%Zi21-id6%T=E78lFQZK9p}j@J(b3VSntFy&otCY-y0f*3$&Rp^`ty<%
zSJK~U-TzuyCS?%*?&bXps>rGQ$rvIM(rYE&>Z+22BwuPK%-W2<9h#TWY6krCm6vzE
z@>tHd@i?*vN<Kb}a`5Wb7Y)ma&9UG~o1-Rr`#GpIh#n)zsc!=~jY*u$#R$m<3g`#p
z&my`0CnnAfDysqFK0LE0Tp3T&00hiL=E>7n<3MLtiUd*>IBpG26C4QVf|qcEZl#oF
z-_@cpmbkP|6=$cm$_w6WYp@1NwFi7v&$2e2)D5tw*dw)$$^ZgNLHT3y=}K`G((k!s
zgj93pYmxsry#C(0j~-A9SsVDpb2@*#w2a>NVezojq^7rN`NT2V9TKAekb#NWxV#^Q
z1Wax!1^1YQDKtIb+JuZpK7Lx587g}c)SjY1ad|88&}4ypX)t;N#2==f!d|h^kYIGr
zygQ+ERI-18at-rVx&}e#Ls(83R7qnzP#s%7cP01%J0Ogy&f#D?mHi7|(kI%(+10O=
zBxw<1M?B84)kRh`E_6nI=>m)KXGmPJ<4uBYg1!gcB1VFz-My4LrjmF6MEd-Bv1UEd
z30*12Ih9-e!~DgRSDvkI=3m0W<&v^Ya5yDXNkYk{=BmRz{#%^NsAO^cz?VC93i-cD
z;N8L!<uR^l$+9YUGud*TiEirUeI~;$ZbWZ-Y^=S8iAkYFomOy3NnJ?^SL5<YAmD;I
zAr9c?QcAAHY8$M~x42)nZ-+C#o%jJ2oUS`C{YmKe%62K(sx}It@5saD6P`N^huWBQ
z6B7?CVQe>SDFHdD2bIhq8=8X;OrqW6n^?<`%zp9cwEw?qn4kdfw>AW$WRV|PE!&FE
zv}=2+_cN2J)&s`w^7up(GGWLqFD@#IHSOuDiQKZ3LXVGGo9bcPji+_EGyh;X`k+=8
zTe=P7I-8*Uu4sF*;TR6fZQm9T_&u}r&z5G64XSeZup$f7Ja>lCVA*O_FMG^)$&Q`^
z*<|G>S?VYU%mi0<98AQB-;Q20mJ%U4_A8lWT&>$)e|`-aOZl^sSM;fL+JlWfOIZSs
z<?<dF6%PrPQ|Nc-@aTUGjUsjVZ;QYNstSD%{b00X1$-G6U{lcXeY2v9x+hfZ?a$kT
z&_QFK)bNw1yZilfD{D<O_V)I+@P$WfL(tqX@j%P2swi4gVcx{BkD`Q#j6>RHiHhZr
zNYv)9qR}@o%q@DBdzxo*;~jhs@d@*Y<TNAZki!`H+Ac<Ol0-C2k~iX5VRtDNA}iMq
z#*w-fb?r+wU^t_x9GtI9Ee9#A#0N4xcw~sq#wON7!*<_`rvc~Yl7pHeR&!;_tA@yD
z-}VkrvHw+GUT3`HB!mRY$CRM^XatK(X!grfWDY?J3%;2n(iK_s>9AeV^@x8d#4|Vj
zD--Tz6?iadbJZkOXjm?x0jy+qCuW@UZlOP^g@O$KogD%JvsA9;{QNCB`mk(L@tqt%
zDx31;pZ%Upr2vVDShBC{bD3V1e&-9zO#>hT>-Xp5O@wZxD}BHce;!OFc(GTHC5+!=
z<QousJ59T~T|e?)^vLaWR3jW*kv`hhM!UDP5fNk6jD7sPk4Y@pxKTJUB2R)M5o-gE
zmb_otQi8aflzGB+Z!PqDy~nqg7r0xCl)iuZ7+hsumh67czfm0x==(Gs4ylis7Kit}
zLdKxOd~)Mpoh|ozSYX7}zP)`0VzSzbbAGElqa^3{8oknzMq4x!ijH-GP#}&I<=~4#
z!s6JsbHFXlk0!D@BGe^jV3EUiGVwO-Dobe5sy-If_p~YG40hB2EI>O{N$*-{0h`DJ
zL#)<&DDnO20FE9-goYaQ|H%Tx%E00@Twk9?Z2hWD8QED4sSzjd@e~Uc29og%r@jWE
zTLz)?=SjNZ;Nav5l8lXsS?zG8jW9+!5l_PDWaSF*8?Gu=(f-^aW^CGEf|5P-Z4NJa
z-DJ*3Ttp~y*OmQ{W*^ml0(G9fST@c9OJN{&nXWeh>qkD=X~kDxd|PZ1KkL3S_@(;^
z=aP2zm|`al!c2uNf|;YYSD5KD^bFvcdI-UyAwBjD{!RYm4Z)UU>F<5TUVFV-pY>t(
zc+g!*)BnG8tm^B5_PwT+JG0uhw)!+j>9kg1Ny#VmX|!dV+Xg^af9DZmSW=o~I_H^M
zPjSk~j_1F#^pLdeZ68O7ho_0SR*IgM0x(6>th)<j1_R%N=8mGJ_yj@bjSA|WiK4t1
z#L(x0vc~R--Ot7BZp+3}v3Tqy&09j62?9V7WSjM7&lTz2q3Z|tC-$1$FAztks}L@n
zws0O#7u#Jof!3Vxh&Mm8P8NDZ46qe<52ZQU7y=k?v_3Qiil*gmLx$epf;8hei5Io2
ze-`UZmbrOlwRrCp;m0a{5QpKm+y5wPEqylj%etUie)h!3eT=@3a+2xfw(LBckzI1|
zv!rCFwB$iEZ3I_l!mMUq0VqjnME65nyMm9HvF1u6cIf>N>gKPyHCJM9fDGCfN<Ss4
zkbs(+T2nIc3}lZ=uxCRbZ}|a@cgI@!0VEVc?8QJ9{<=XA;AUv`<z#+Ulc#PTK*l0g
zV$CTm6<V@y8ta%hn#_FHY$Ml!-Oi47Z*{dwU)wA=(=efjugCgqM58_BcpSqPr}b+{
zOiSN>1%}Mx$pwL~%%T?dVgl(ucDuLvX|GBMY1pm|G(tWb%~dWuv0`gXTqX(XDD~!?
z-@Wl}^M*ekK;xwWL>9L}n1aMWnZ{x9a$Zq?U!M=c+85v_FzUdAdY=oLbUFxY*iZx0
zjpp$$YBv9YDj6|DW<MhGpk}o8O_II8OWnhzLFnIS3BWxK8DwLA2M1b!Qq8#gfg~kj
zc@s=rK@UT;ss4%}JIRsdSwIt%xD)RFwHxI6=3XB#(<F`XgseC7$ndswwa~7MvY6Z{
z&wmZK$dqFkdk>M9Pb&52Kj!ihpn16r{n2a;dg2LGS#Px>EMCw*8@Qh`W6>Ef`N`)R
z$gyvZfhm&H?XO&~NP9fN6&^1SRx>Tk5O*rr+Fb}jG^Tbw=T8F_nxOt&ld|5CgBN>D
zEUZuq)x$57M-yNC9N8G(;GJ`u#?HnTrHM`bGn9s<M!4wCeOMqMG-`)dcj~y`wy86S
zVPp~1%_J!HX_qnm6yeW3^nAk5b6deusbo@tQoRrJ4`X%LJ`d4u{<kr0eM7?-LUjsZ
z2Lq*UG^iH(=NnHc8FgVpq0m>D7w5@u?Ivy<z@-?3azNy(IM2Ewh6pZ?$(Q_nf8c;3
zZ!rj1?S+uLu-nq=v^_$XDSk*H;c>^?N3mh{EFxzUy7mY)rBg0nu7Djbxug}YOZ!|6
zIfwT?n)W|WqXQ}NiDUOunLR4&d<Tyyn0gR@;_U_FD|Pq#&SO_<yzMU!htglmXDr#+
z*^f&~UcRVL`<9k8NvkAOPOQ_m!FH7IonevM|2XlIxXN#5F&<3d0j7|thM1+INuD|W
zhlpQs>OuX*7JF(HZnN5-EI<Vo#jX2?hi1yZ;+d+EzpY+cI^p@_pV`p=QA@2IEbQ44
ztdcp@6rNa6u|iAK<#p4IEJz~89ZA09VUhAt8=)s%4tMLpOE|c%JdOlUM>|=x9N~)l
z2=#1{46Tvg{={Un`uEolGfvB(xua;^E5p0DDpxR8JJ!?vGA=a(u#<k6Q3eI1a)kvX
zKHMKNcHv|4&Wu@YE~+1mZf8s1{Ud^H9LJ#RG4nza!%B8#q32(la$!=mJqyc5!`9yS
zxcE5WK8$M3^|fsASSTcg7CrV=!Yj9blNSaE?J~@qTyx?RMiQW*nNq{2gA-)8{oJmr
zC-gs1f=Y158Q_K|f8hKUo=bPhsrZN|<`_nNIl{gA1S|4$B8V=C2cpAP+{y)=_(RBd
ziOlQ}fR&J*kFRF7WgVVfeUV$CIZOoOlG0?&x&1dHAZuJd0E`O!HoI3I*yfFY#vu)b
zTTVrr8h_N+lfIMsca-cXgK&PIu(GlUZSpTt$Ao`CFE*=f|9Uh*uZ=Ec121`FgVaQ*
zR)6=a_F}TTPfu2BlzSDynktP^%KQ^0SOZ($h#_g!h)+Q&eZt>;jZ){JX$wV7b?4Y=
z3vW5~Y1d_&2Eq;{3J>@WF(3uL#+C<T+J%vDuq7j<z`glDV(i^ND^UyI(feEu&S`*<
zNgfbGg&rD~%owbi_}gDyj@SHcZpu+4Gy68#anK<iLV;sEM%PW=3SQt}SOISmV^Hjf
zWZhSYHOl5qeTQDfd9c^z!rx(FIY&l~<`47rSvK*QEF^1lZN}~7qS!)-l>U05YNXIK
zMqwr!-Y?)Fk|Q_9z`ZOAA~0}cv`?HbvLb4>8B{+VUY>ykV9cGrmcA=<Wvil)1G8};
zG@|yaS<+L9Ink<VTH#Bi+iHmSfWHf4H&^T#{J`v~+eYbqSk$`z&!i@0$;MT6ci;}1
zqA5LvS)|BVM0m&RI$5ERHc$_f#j();#P>4s{Z|+8s_y@ISJfps#&cN8sZNuVxdc>o
zszRZpP!=<i2%X7cKhz(n4vIW@r~8X-O0a)glXESSVqw(`{>736ABz)(G~4a{I=24F
zyPmL6pV91teAUH9|BRbBX;;_64~MqGh9ifZFi&Ispym#-G#=<e%kr2G`q7)HI^rXx
z$V~(yJe<E8RvDb2%aDX3c(?h2P7|&iE;EC!-vR}@%fO8Eh!Jgq#<b}&zV<co&FD8V
zp69qaR7oOb2yHn9E-dp+rAGmqU#rS3WFvK5HpPzDYP=v9>Q+}5tof>A^~N&!9K#=#
z`*ztbp(XmPUgskTxnVD2EK<2pa&`}o<Vg}sTmRb(2AlDtt24-e>3f}~S}}Nxb0+#X
zhYaejHL9=yiCl?0+-}*%RY=SgtHpL8jyj9Leey5XRVG<b*FS;yAM6ya=`%M$?RIZC
zW?}i58xkKv(mtv(*=JysI@%Z3DgOhZW?Whx9XKpuNb|3qlH~EPUKBTJGHhX{pg<5+
z1ARpw1rq^EZYL5@E;+4Cm9(SIV*lCU2m9GaZgtirW<3+ypSxTQ*fY?oRVai$gVv9?
z-X5raS7sk4Qtm1l581VF^KIu505I3rY}h=FyV>gk>;@N}ZW7WQL0+O#SX`L5L@X}q
zUOqM*c3RYBmY8c*W!*tHEM`bZMEHw~Vxo}<m{I4JqELQ5c*j%C&QPK<jxFh~@Zs*i
z&bhZC4=4g!K#MtB%0WbUcq)p(3L?+|+=;?lZ$Kot)JIf#&|m_?DAn^m_#6(+YUAQm
z^f!^4>k-5z$%;a29y&yWDoZ}&-=LfybXoe`zIa-IOkML~q-<47tLA;Nz%43^#_bT$
zXzAocT)bcLtVU4xuH*P6d?)u{1B?q%`Pm%RVTu)sWM0%HE=v6lDfcD|4BxtFnE}0v
zNPxw3!(&WvOrfSO&HELWg$uFp*$zyMlpajKa)a4PYJe$y>&a%i=2Ln2z!RolJld2A
zGscQZq0Ty+au%DhTtF_}F1iZYPXY{`zr(a7H2Q>$;gielk43O?=4rWf&M#fH=2KGc
ztt*zu9BmIxO-*J40&gNzT-*8=CkfmnCz~4npM1a_s$T=>Fi|`NqzHe)-bPZKazP?*
z`%#F5CFaE#L<d#H)l*m#gnk%=UMZzUO7TF<PD*N6;~9*eB@g2b{3q(F%3c2E+%N?s
z1j6}Uxy7kZDuT3eYfQJg^|xRyBJgI^Xp*+0jMrwp1q2#sr921qc)kw5>;2~w8C7Is
zWBA2Jo~)h6|5p9tX#H`Y<BF!8C#Sm6t~=z#P`laio0u2rUr~dd{=T0(g{}WFwk4PA
zzDkYh(h{*NW_&t}TNV5~I=Q-E4NnsJuAkZei=tR&4h>FXw?z|-e7tK9k9K*OYj%85
z|0;Yml60@!izA6JN}m}-rkS26sY083oW^DS(4jX9P5_Ym9sZzDpm4BWaN6rl9otPq
zCF>K}jPG0r+pkef9^D`)fNvxfP{G3GNk_kQpDZ`~B7ymgDd-M1KsfB*AI%DFUaQxa
z=Z26uE8%hFke^WMqM){qRdgrx{x*$QIu%T_FjIkwS86Cp=-J1nd^mZgnhb=s@KeU~
z^!U%2k0y*33{TFqk3wmW#qR$;vs}*?6Z@y1zIQk}Px>}|%<X?Y!VWo8kg(NSXK8+2
zBO<-ltY(n5!%qJSYxlh(iE!ELHie}I?Q|Obe~4T4>XxrhQ)vOHGQ&*a;y=NFK~5G+
zxTrh~JV^Mb&tuKMPpAQ`s~djjgMaJ_G@lUJqiJ*hDM<<hr-Wd}p7>c0FD-jL;o{Q?
z60r*}lw0Nh!G!~nr&Oc*e{!bL8H?#Ck+Q7p0ELMULr>u>0(r%8<<s#puiaXQHK2r8
zJuf(6=W*f-Wyqr%Y|Yfsn_)8gJ*V!^4fF74taSr5xFWbub64y{<k931#$YVL{ly{B
z?98K;l^b$yUNdlNe7YrrpjB4SWCq)X>irN(%d{nCl(Hg?HG`jEQ$kGk)U%oaKXJ#F
zP@7&+s^Ur$f*6WKPD4@S3#wFGCrR`F$(Pdaycu7l3-3Q13Pi}yrd|E`@nci{|5NtL
zOG*~i)t4>VQiOo2pP?tPcI<q=DX+NX+Gnk}geROIq*sI)LMvN!Pgets;*r-Vpjrif
zE!JBv>Ncu}+591$!g$kmjr6NPH!BjzoSJYdK+j!umL>1`Xj1>S6_C?!pF9y`FyOX1
zC#)d%JWWQWK2Fwc1yFsu{Oq(Ped1cXhP4*8<}A%0OiA-(tlfAl==<-&wdb2>e;C1+
z@PS*~pqHI}{Bcc_EavPWC#i(??4%45G-mDME*JB7J1-PK(Us4I`sdb`Sk4If=z3&l
z(W4O7h{S74ev?@{+m!)}l}{3sXVYco^z}@>kDLzf4>aP5sa4<(g=X71KE!pnd7+p;
z)mTcZ?>sy=A5#U9y~yPuEDKIwt!Q$bboTnHFOm?-R+62aeM+B6?={tWk<U;J@XMv^
z8$_IV|2~(>;#}^(hd`x<Etvv_HW<lIBF3QXupW4!kTIT9Q>26Ej&3I$JZuzTGDOux
z*`H<Wn!p1=M9!gWkRf_3t_y8vBRsV8zFn9dp!#&Ej!eWj-H9G-K6Q7|65)!g@k$D`
zP;<2;eNJVM49%1@yGM&Y?>XC4?3(b|gHSaImgJ=TahZ}32|kd>_SeLMI%V3VotIPu
z--E`x1RJ4xj=Wb5tfd&WL`JQTHUIRk^O!emUiY@@S@n&sCK_M^>iyk93eTffq?FNA
z{l;i>>wrEe9@fto!l_wqq#tIMM;3iesz}?Un}=b^#zeXw`GkaCPGcklCH~Q%P2^#b
z_aKa8q#4as8DzcxT&3ThkPR{xl-}OnN(6EeI5C%JZJHoAF?jpzteHW#jG|q2mFun^
zGbp)=zm1STY6A2qoVXpMILeVSRy(CN{2p%=e~9T!23F@TFLhflnlWD%UTTaYO61zU
zukengYq{?w5Z%NQGdR0q(S0@=u{to6ozT|4G(B+N8W@v$bP)!1jD#JXxnk{`%@YDi
zPy_T2W~S|5x3rB;L$I^`WOwv!%@|S;ehrmokW6E=_G>yNXc1x^CpOK40f>eB{fc8S
z_yEoY2@ehqHZ54xtt5hW*`y*yG8EZJWx8}Sl)b%u+EXz7r$Q>=Pp8PxLbJ28^Xmm(
znMNs02R>9}W#2l_m7yYLD@%>E9JvKOaAx!C_2zU{STY>6fP=y$0WtMjU--3ppG7&>
zBVlE*1vi=T33BHLRQRLr)_4hc*=2&!`prnTQCyX02=)rN@flxF=GDy?v7+A@IAV_x
z3uyI%0c3iyZ!rMaq!3yD)oHEI&*AOKMiy=FvZuP3{IaN%ww=$HZEmO+!&GJi`=X)Y
zi2ikOOX@G6v%aKjXA*{rwZBE<Ny(u)@C40PQT6d4P)!n8VxS`XPZgWZjQo4=H@)PS
zm5*_E2Lo!G%J26>a#KiAK!u{T#G<66g!@;CM^Q;h^skbK-z6oUApgiJG=J6K`B`0E
zO$yfl5%Z0#pl9EX=jMKhCY~Oz_0}AXHky1-5}$pHZG0}@{jo!=&FsH2`XLA%{K`#H
z-u{O1P~!Ey|45Vb;YPB%Xz0jIZ(pB=<YFsJLfGT~sB*Cv+O!@mZHk>Hqp^ixA?*c`
z2Cu$H6>0s%SOYHa3ffWE!h#L7nOi9Txw&j%;sx&7Xa^_u%m+n=F+oC+%HdYo^^hj@
z>#*qeftp~jvUO2Q?f6JrkwbFM<{~u#Z;$Tf#EK1kaB!TvQf3Q1?BBp(zs?dB(JPc6
zjCsYa=5*;Sqr_Nt_jWdM2R}52`41g3j@Z8Dn?hwXT&mhr*ZqSDJdoyo3ViCJMOO)5
zJXV5<N--JTw<Ee(c5PmsXXzW@1w>R5`kQ~65)2F{g>pf+F5z#uG)Fo$z4}k>KNT${
zBI94B;`hl+$i9`;zcJ)Ps3Zb+p8(pB@uNRCH#htf1_=L5y`HaUg{Aw$yk$I|GhA2h
zC2N=A#Oe3M69VVGoJh+CbmJXJ{q*pShu;U+p;0I96r!(32`z8sef3}&XY$T$YvBRo
zBF~Ar3&PGW!#1p2yXS?0rlP(+_9L5-2z2D2g0h6Su<s-DfbUN#UzY6Sei+xk3hhU;
zK9@vy!KOc{%hy%>tn_N??|1rwM#*EXbe@83T2iHGfk{l;emSMHs5xV*5R`kAB|Lu(
zF|n?V=1~}GYHCUf`bB+M6OF*=w;e}_AErC8o#B?sKj%*NJI68Hm+8DV&)qUnn{HTz
zvTWo0`)=c=_N}h*78VwM1625CUWDjU+<3A<zT+c1l`XnD_8Z@2_GPJ>Vmk?_p}q$r
z;rY-8(ZSf+hAP@JGBP$-{`$4I+=f334!4okc;4Cw7VTQ*o(tg*_UPn&E83!1+m$V7
zY#=aSD%ZN$AW8Xh*iXFcD7?>172fhG%wRvb$=6o(i*EKa+=ghwK#?NUnQ#WN(B{}B
z3ZNTr2*uKZw#}N(F~Zws=&^RJ3vxvXgWm8c5%gvCVYp^&MXMk{XV)@ZHA3qvneuJG
z)q8$0d$c0l;_%|lgd4$2Nx4}zJghKnQHQ9mZmF(reR1yvj&Av<oPOKUgZM+#A(OHB
zwDIqgGK$1bgH|~$)SnrV=qySez9tZvg;C}c|3F7`^_*v&lCexGG>;xS{@Y=&H=d(+
zV8wvr1X}!hU`v9<q^GC9g)&2|{!%u!%XnPMdJLwnir@u(#um&31i2Fee8Av!xSyp*
zp`M|lgQV_vgM7N6LbC-$a6l#Egy5H_j@1K~-IPKyOvk^758nWRiCEjVK$`VCH3MS?
zzi~%$$JsUKf6_b0%RJqs=4pYK5vL3lK9SgilF|&-exp)yR%=eaUC#}DEROXUg(z5s
zR1K*+w_7!$yuG%X>eoK%>J8EOtcI1jbf=iH@hM4=RS+@=H33?!KF=P?F#)_tu)kXj
zc1{p?obch${Iq-x1{3IMA^wcg1ybhDHdxL7{$wKwi|W$+Oy(EJmrBB1KpVTagG3KG
zAVNV)+b=B<;mncoAt)LD<b28Z^_g`KULOzf5hx3r@zlqM5uDjd3Cn!!O!>TtLSHcT
zDU$64+}`~{sfcTvXRA~v9*7}!>0Rexsj~K&bxT|qF#(TmSk4l<x548RD+UWMBGAWM
z7X+q2@{+P)mRkqMmFO9HaIL%SHLf2XYyx`UYFd&&)1scx+4c(fSw{T?_dE<=Z0#F3
z>!IP{^zxDe2X%Fwk`kX1y6{&uFBHG86ZQ-%q(!>Rs#BdZ5Til_Y({^PUIL~qGT_Ok
zMK%`?92y~M8Ch9b8(Cs1nc0Wd?4Ym&i`-3RswIiyu|@w~lw<oFWmh}xtK#{Yidjhk
z8o{1S&90}Q6+4WcR*oSiQSKaJfbAspH#QQBV^aHD(!3JeAZKm(fpRh(;RDn)QPh~y
zjW@|(m8BGtqouRIL<7~j1vFsrs_~Z#BUFU=UU#+lAIRC#PX1EiT^y)-Fh`xc22PGF
zOe_3B13s@;v^~Dt#oxQ<*T1potiafEX-<PUqRI^cn4n|A#vz@9;u~~6%nrSXfj|oy
z*d>thAtXYTUNerHG3Yg<BH1rQCmd+ex%6nAc0ghyyZ<Pi#;R(CL92UO!z77kzmX45
z1%)B0vUB+P+hh>>r*6*g3kzDGc71rTZzu*YG6_{hm32(yA?$LZ3QbeJ2IGsna4qqv
zTdN*$XqE!(U~~^MNq*FxW$Lq4$lIu&fSRWyCnLntYWz;&$<Fb+>vqhpx41JH9c+l2
z+JU^fV>#{ggA@QKpm?ry&iDkaF5tqTAn&We_H{>cQ2&vTo`(p49$tKp8<7&~fWlF4
z3l7<s_>%MUKUx8Jgl1lmAVz+au@Q!)I^8#O^pce9HKT9(r8XWJGfkth!XIX|rlPI=
z1lwjSJCCJ-K;E^1+GsgjkplDu&smkDZ&-rC8_Y`0*S;)2MlYJ}Xemq!OM}S3WQML>
zPuK#u8u$o;>?-hsFI8m49H)tEO$K>VFQ{v~QRGi(E;#D8w|MbGj?56`^pF&^=)&mf
zdPHE#M&96hcx}piY9%O3#J8$e2;WH)>zfHz9OUp`=ot!G%;S>G%kn9B>5}Gi@OpQM
z9ula>M^<w2xMdWY*XWn6IZw8D`-w?-U^}@9WDdQJRMBPJHje<`l<~K2PoR0dsiAOc
z=3AgcP&&NS;xN9usex$E_4}L^bMD;j5)Wb%IwHo{o-wnq;)k&eQ>sobJas&%y>f*8
z($4)(p`7^2eRk>~H*eMA_ak~sZv$pq-w6F0qRYA_#NnfOMxQZ>PGs&bBeF=!AY5A|
z@CRC+BXaVh9z)vgj_YPoq4e{g=9t0%{`NVzzP?@o5M~)(474d;4gLFZVgkZRS<y5<
ziSQWwA>6M2)^QOv>Jp#7M-g7J>S7Z0CqPMp6864TK5j~r?c;yR+=t=bp&)c|&jNW)
z?p;6zxq3d;OjwXRC?FGrSr71yrQ%PqT?hFWMAIyB)Msr53uEwCq|37s|3-R}AJ{}R
zk!9Qs(~~h+s%<c%Dha@<lGcqf79sw!4DQUKY#GLmqC17bnQFtG+m)SYjE0`P?yN;R
z29x6`-jLk9{@&heXNx9X`i@}%LRd_a0Hs@gJ)6%c9BpXYA7&$7#56Q!dWQu@!NTx#
zHKi>59G$ckI##h_MZxWosUbtrJ<qpzE$PRm7wTxUMb>6Zh2u0prVrz{OkU56BVvoH
z9F`&;N!3pK;e*YH2)bJjj+5A2f)b2EH1*GgQO<BX;otnIVyvc6W&N|@twLT9q>DW`
zGZ?^a-Rl(^cJD5thTYc)qH=xfcTM+=tX0n$xv8#D?K>u_5JD4xp>m{3%XFtdc%kR0
z*BKxF0CIE!%Cf}|qm>iTg+%o=Sg1A60~9#N?{t@1+#Nf9Q*v?Lz*4p@iAbMZ`dxo7
zH2==GOiN+DB2+aGv&K(+>@&BtWEHz{=sP$x<O`ocbF!C4v~+?lENoJ>(mXjic^*S^
z5(g25Zb%B`<n8B%ha>9vN6E{{*+}e(R?jl&w%iedrzsAk5%2ooEtL*1K4S0A8_v+>
z!T)g|$ZM9(VG7!7@9ao)J#iu$ET0$Jk^;mZy*h%h;|CtmgW!iF6Ah&){e7mKH@uj6
z(hVkUS=VW0n-zo+D=ytfu7MSfzv#kNUsRu&eGS0t@SamrD`*jyb`;aNV3dr(s(=hC
z2>4AfmF%hR{1`C0ZJ(KN+4bpW!&^ZjvAE^(O~f{z1E}2B;!oL?L%0zZ$~t9hwN~a?
z52}37m&tiZ%M|{6O3bduXj)~&5(Cb#0)DUGOOimZliH(Rb(qMaSrGg~mT=)MlqK1o
zsP!SQTM?{%Ru;ZpGO^2Pp!0>ek%l2w0tvIEh_K)OGzD!$o0clytSzo`4F!Ei`_jpi
z^(gzg8keZZAB*q_IiKRWSyIgxUHdZ1&8kvLD{HG5&3igWI^cOzk!EUy<>R2ZQ^cIC
z1o)1)Qxuf%kiaom&39&AuMT_}<u77lVqMOm+NlEB-Bi}>7(ZERa60BuLpHOQERfxF
zC+A1Cz+s?jY`u0ycMo4-;Vgbv8@;>i5Q{;+WSlDj-azU=e{fKh)Q{hZ<5?X{RPp9;
z(9zQmpgQZ2=a%?8c#zn`=zMyh`Sn@a$aTtt5}~P}>P8k_&ch8x&t1v~k7xUxrp+Vy
zA7ACq1b^781saA1jB1$YQNJT~^v(9>U}?;3m~@GlYwW>elz=L>7xIi)y$Ss5-LIxY
zI|HiMlgiDFjh7u}BO@aan<9bQVNp}Y$r!Xg*YFZ$xY5a_Ekp~vJ`DR;Z=qGQq#@^$
zp>5R+rJ}hO_nM(?2XBM{{WaSr-y}}!$ScWH;UIK`aZ<9Y454>3ufccs6JnuLu<Na9
z(E7RQy>X%(yRWcor{r7_=l5RZ#`T6fA1ND7!y+SfxMPfnSz)|E9VDcu>MAGP-O=#N
z38XmvK4{X7$6ra63Eu3VzqtROEWqfJIDIO1;XLr**)7p?=a}@*Sj<d!#4BEQLc`^r
z?2-v8Y<}@10^v?MZ^i`irr4I=<%lfU0mBeIoh974zha|5EdYB>cAS*x7c=&7KQXE_
z*$Sv_2(!<n8XH~SxR{_0w+f-^&TboNPu13d9US39l4ab`wpOkCrO7Plz6t+a4C@@&
zo5<7ids#C`;^5|9B`=t>wLGcJ5f&O;^yXj^d|u(1vmy3ldwahzFnl-TT;vkb+U`N<
zDe%v|n%g3Lg3-SMDnwGIXXR)ihOmYlYmh0eA)CF$eIV9WrJs|FOMSUT*eN(R+)S*9
z^fK|zc73D;>Kn|%Hl2O%(|<1+;qRlceOqkah#h<G&DYpwsG0w|7%ScuxI;bt6?ozT
z$*zv>I9<3Q)#dpI+K!BuDRyLzmgvpR&8%xY(2^W8*g=4Ow>q<ibuhZ--0d;Hs|jS%
z=6a^wYXHJg(b1#}&qiKvK@13;DMUF~X7GBX^K87Ly<u&JpkxN>`387$5aOyZH_gpU
zw?lz@H;pkUl$hzZ0(5FMMf6D6WU0REG^{|MWk(ebdEPaCob9oNC?hm;h)NIR&l6T^
z!!{I1{ml9-kO%+t7Q5J*DulRO#SohO*OI@Z?n_=RU`ye5jzM%mue;OqR>oZJ5PJGl
zS$>{M{>nt@b8a4?eI@_%yWrQ3kI9uQ1MG-&o0hMr`g9Ip1&g`52<j6MtJ7~CJ>u_!
zvg<BJNcTFDM<eUvN5{%EuqG{Ii(T4|8)UYTbmV4#Iwu8tC`Zwh`{K?>+Uu4>qwYyD
zn!Ah34ZbCuq(*5Se4Ho@hTuZPjtNj#P*7m58s}X<pa`5{d)#843VXA1nXEu`5E|C7
zLktsB-P*9RS0>hcX8Rl#7WO4zye3#%MO8J<`G@HtE#O16VXISszOm>}vl&Xa_gv<L
zN~`DynlLu_l6kT`LY(wkxPIOtDduP{^cJZY!~M)w!gDX^`2UV&RpD9U!p>JtzdxkE
zh?P{Kz{S7-J$I6-UrtT~gqKRF@tka89wK&Ez%x!%i;xNnRp7(_Bf$LE-X0h`A`%cV
z;hNlknI(xfjcB(6sdRMTdf+S!wmn5^5#1nXM%#wZ<rOcu&n3r;?^qkJXTE+@Q&$}M
z|KawN%lj469X^u}Fm@s~^ssotEhw$*{v?Iw3#(x0PsaRv5~ycnK<hL5A$d+jqIHyd
z=lh+&oGBA4GJ06_y(>=uC)W@=5cEUS2jV&h<EZ6@D05@<R9P<($$KU?V1wT^K7TcN
zc@{~qm%zS)r^_&P<2){|J}m4ZkdoEgaLede_ReXQ*y%v10;kf+@F~UU*1v;t$c9b(
z#yc!t6ppN{r@uaXP7|dcQJYK<et{VWmB4=CVOM?J?4fk#D-`ysoHR%)sza!odoTaS
z+5VR)4>xmgkB_`@j#kKyFR&hsD$Ul5Ie#!nqWc4)L>NkeC&-QhaEh}ZhH*ow$kAFy
zXPG_cF5J|}h^VsUfC>!2HJ6m2f%=E0%jwE9v0sM76}n$Azy^_y?PUuJ$XE{Ytbg_t
z8eEi_o5R41=E-}o#6myit93i*Rao2FzM}aJdCpHGWi*C8hQH_A9h&hk$wwmh$url2
z@{4t#k%#4{l=|_9Fb6I{fhsqPbp5{RT}M9p<5xj+<uQMF3E_J9i2F~uOYqu<#;l%F
zLda6OcHyNmliF&{oW%kS=s%kIaQ?J6Fu8NYtZaRn8W4uiPfkBsJ1r70BQ6?Iv{LUw
zN4NKtlsvRpQOX}a*-5knBdK9HQ?zwpGDs1aBiDwDW3*BIV)=?D;!O}da~w7SS#+I}
zk|G@BKN*NIakGZCI5#(E6~qr?c-l8h{Q-)7jBEH4L`(m$FG7V=qjofuo3?EcSK9mf
z)>c84sanUBdoPM6fS6ud>7>O6`t=t7$@6fgoOJNGXYCuP{P+sGovj%Ht?CuvCz~_S
zH%I_=aDJ-5Njxt<pDH5-2`~yO9-@78XaQMaGziHNi647nfz0>x^cce}zb_?KY8iL%
zayE@Eo6-j`W#K@coQ*C9xRK>6Vma=I<1lhDY5%Ee4r?J<z4U}F@p{w|ef=k3fyl$=
zQ;s|;%P{8N3RCAwunk*_J;MY#Is8a#dI8PbLU1F7P`%!U1)gVo061i$Je+RB<^GC5
z#u?E4Xr%eO5*)_)kYx{Ba~j_5Wg}5E)ZNRUB=AB2a4@?&xjIPN;99zow1QVNyTM@D
zwKn{%rj=Q5A$TTDc?~s7HF;|}@dIjv`7NmS&C9W&_)AeywNm_{W^fps^x+o^pcdbL
zpjf0)dab57MT4$_=%+>@2mWR;u)%*SX{%@R?{9g}n$db!@G1g1F4UbReiI?8J_gW?
zZiwV(0gNG#8y&QDfBf==*J!oP8;etM2#6UO4l_7vdrw1CsEl((EuVOF(lX`&dECc-
z=>W`&g;I-!?&f}^*9_fzXYzu>S2lpnTS^^wdQQd{@Ien-EKS9xVG)%`PVjwD6_8U9
zPm`-IpY-9`B^$AR701ai73a&(cm_;)#}(`M73G|Riuha%>kV*1i|iJwb}sjaec>qx
z$dwl9#bSCXt!>sJk3(SIuywNsxzz9i94v&xARs=&cR+<fhqA1Rz6kXAt~te}jBc@j
zy3RMAlRx|4tsF@Hmi$dfkB5;c%M3pfQb)ne7}S1ep+brSrr(<b+N4iCckH4pzM6)A
zVzNU+vCumVKGFUl1gzBSOxSs=`p*rE-l%?5Kv>4BB1}xz@Oh`48=7*jxZ7>6?Bz}8
zNH=(k4>wwLUeo03>Zj?27BNUztS&>sQfdNmibN6okMOLq+cxqdrQ0atGDiw=<n5$T
z{&R8KiAEvD-xdBUn4D3GwCKyC)Gz%nroO`w%eZ_0_Sl;c*-`f1*<?jXLiWnu+516e
zM#_5Z%w+GqQYs!RBAaCIz4?9A`@Y{_e?Xr5{@kB)u5(@2Ifru``)*`7rzvq7XOTs;
zxi*4^yK3U8PR2G}Ys3BCI=wYV!})!0T8TJEclVz~zD`<WzT1t<(23ROJ6_8ad8jPd
z*ScY*H85-R3RLOl0-Mbw3wMCl+yaJv%zSQn9Q*y-HzhH*l|gIpaq#UUpnkKZa@YwQ
zoE3}$Nz44~S@IK{&8E_kyKQRSrz(zRq5zV1l(~U0Qvp;jEa_5HimZ<lWrX58(%pqw
z0X}51v_aK~Q)+KNtWUgIp`)NR+sSIF0b@Sp)T4`sRsfH&>NEGu_`Z7E2fKlA`E2#3
zNx@8C9=nD_CSnP`+MdfennZW*kAql!uXPTyXWacuIvx)P=a)Psx>Yw?OA)?rK3P0H
zHXdO#Wn^?t0B!KlPABSXd1T06yf{JU%1@aR^S=`kCyeT+QJ8t?%cIPwLOa23OV?Y(
zrQbdG5iRZ5X5i#<s1vjzTN>WJZT)f1&e}R6R(8U^jt{SEZkpHeS6e1kq(O7w=MpbZ
z&tp6NI{VX=2<5#~3^(b*;Zs%1{z&5AN7D4pTw$H&!4a8<r+(Yt?LJ3d!YnAdIiXBU
z9=0J$g7@(9q3rS`GRiJdw-qAUnK%)Ot4ZPD;@!LoB^vU^I6@lHk#!$UrMR;7<GkN;
zKrP{>46CGls9uktyYlVeiEc+dK_!;RN=iD~yUBBp+|k2APvjsg2GZVpbRA!^0?t-L
zLFgMpVX5JGO!V|il|}EBEk(EG=lT(MJh(C>1q6pCWL1e*vfm6Hfh8+Uf;i7}-I1u6
zUbH%5$18KbDeue9hDOePGpj%p9#+9F9JtQ$xU&wS>%_tNZG+u28aq*x?AofM)s2ro
z0-17V^G5}0wmy{ja*BFA(e!wL@kdQb_{CTPmy@Euz=@&0w?j;8cSg)Bg~H`SF((OX
z>F9_A+sS|*yhVRzt~yr>v##W#Pb5MR?Gh=x)KD|jLXaIAQOCbuheRTU%{1%3u=>>`
z8wYsTjIB@8na{de1}G%<RCB0IO`=nTRr&*O&a9Q>r+3U`(Ky!`44m<AmJg%AKVeO;
zo<MYG{97c1ii#{EGs1{F)6>&DcndU8b621f@z7T+u<0f`LNsz^KV?~$SvKS&lx>qL
zG*eZ=@hS-w7(?zL#Zx1?`SyS9bSQ7keuvbq0XM*W@4exc#nPfu6lNy<hnTYU`m}-v
zjB19B-*}?0YdGw5(+kh<v9qs_86SwM7D+RIf@x_#BXM53Ye-dUmiuZ<-S~a9Otf}j
zJ~QR&l+mInI5D(rvB6f#XsVp*!Nfzt#b?ch{5!*>Z50Q!;t`B+il81>+nyO~_Hsj#
znGO`j^>c)e$*qp$`*Qb&<~w8LU*FDmcXRuYaDMR)NYPSqI6x|Vr#jpsclUg>Ot1DM
zbAdSB*R03@Drt6NhiVQxF=B@SB+{PB`fBHjg^f*AceqBJp~qrL96Igm>mv`t<QJmZ
z^11L2UF-iL;E>xQiUm~wZuoG1WkM=`f}*m;?`*ZO{P7|-e%G&Rztg>K-fHegzxk^;
z=CVEw45nnglw5k7)i#6cf-&&;p3|%S9PVT;w)BW+&z{xX3ELHeUM~zCVp#|`(f4~^
z1#UBUq+=a)nhE#xo9+l}H9Es2nyYh;RPXDDhIm7)ByV0}#~l>4J|}*e;UC)kDX!^*
zzL$R8Vfm4V1K*psi+D+D;ls%p=2HpSZi(H)WGSchM{8A{7i7SfW>UYaL_gqcXtg%6
z5A{>#_-~5!NN~&H_#SPrzLMpCglw5vc+le3DbVNr#2O=edk&X>fbe<}f6nmLXj>i8
z5SOLn{hTo~tD1%TRodx@T$xGe4@3Ho(t+2rg`Flmq753)pQoVIwh-3&w9Xta{*;Lz
z)~}}wdyjGwKhhns;17*)8?`hr|713*v)3I4v2Odtl5hT<*lf%Wzb+nvuw*x^Pm#*^
z;SaDl(J5KTKYPcjgWBnOU+x2PBvP72xai>bK(ydTA!U8}OG_T9?1%?RHWN9kIR)ZR
zQc!Yc)`ut*)KtgkN4)cC-wih}!G0wL$0uFCE(yVe<85Y>+!e0#Qo`JMo;ja-Ykbhm
z>&UvK@AH!}Ra#aaty|6}OOhen4~^_nl#GsiTFbviEORiN-GM(kJw4S0Vo}w=vg?Uo
zTQi_r*s=zWAB+E<T6N3hwHxICwNAtEymN_%2sLMr^c;OGk~N^lZ{5AJalqY>8)N5w
zD2DLp_w(Sv<0N6c1!m|<|D&1QE~~7~mIq-n3IN2-gdGw;NELS&yLgP=*b=F^;dPz7
ziFhj!aqPtWX~WwVU38y1w5*w`?qOW<6i<NIEXEQO`pk0XFPjoy;ipxbRK7H^=_pK=
z0V2%>8#&*?j>KBeP2ZBt<t_Ez-%308t5|P7?JX!c?Z5sq^`XJ3)L>S2?g3V*G3|&9
zvAB4%@$=^*1aR5-)5_AKqM`>16-KX~m{iZV3O_^@b5FA1jJ)hv*mVO|Cn@m!fuBiK
zTPrGd9*1}FcFj9aZ8?k3z7(P67Nw3=vkZufXo0pASUv`D{l@(>Y9~=%U?a>vQiq~C
zEcA^%#F;eyit$x2CvC#olT4bXGGkMUZVz;$j%|jVN$B|x4im_eGHh4B2uB8s*JI<?
zIkBE)3Z5jGn0r=ElFy~dFj!rz+>ZU}StxC><n`N_ES71$d|H0%{62J^5~-fgt&G<E
zD@nN<sq)A;b$wpu)P)(Rk*&Ija!h9yU50yIcygF}OjNi`P=m(~Ij*nz;6#d5LypGE
zB-@&B$P>wLu}3=X8qH!)f4E=zu#h6!op_{rtCk9P4rE>h*9pQJ!V%<pN$x{em*-#4
z{n}O~v&u_~idN%xsVf>9PVR+wxr7d{8vJn<2@nK>U!@*U#}e8LO^QKRSaXt)FlqzV
z!#K;!q=98$ArB%rqFm#H1C=@BKCY~knVlfK7pi&Wz@owlf6dOumSB?$D!a?K5$VuT
zddKEVtd3*6E~4Z>nTYwiNyl}DoV^#+X%Smu8kPe=q@^-CoVJ2?Yfbf&F;%V%IgW*u
zj>aCfy^jVnF`d_VQmUO2+y|*x9}6z2F$l#a;>?$1TR5cbNzea`|F8(I|G*2K=4*uP
zgC``D<<M+#w<+;k$MwwK=$gAnvBjW`j{VghMmPp#&-zw>nTtA8_wj;&vwPOL^!?+b
zt<22K6}*bx#UH3aa53y<UX+&59X%IIM~kH-ja-FV0wL?P;Z<t8)TL((J+BR!9aAmK
z-G5g$etEa^SU+SC4+o;><lMBh98L7@=6&tk%tI7^brwP>=_RA6__YuE2&FUC=Ab!2
z4L^CJr9N0sfvla1*Yyo`hyB%W$8@Fm-bn$~G#y>t(<nDhnU&+yicy|ued1KSN2GLC
zmX=pbav#5rphYtJVSaq&gz0aFo2viJKAN?=h5!Arc$QT|+qjvR2Ta;GbeXPE%VLaB
zJ;3Foosgdfxssu9``546ciY4rwoiVVB+&G88y#1)eiR^=t7je&FFoF9`|>;`q8`oi
z%2dsL{HQK!;sbkMisskr5BpvI%P4+7A;RJBHbMBjIeU1PTu(kUJ4E#z(qN&r;m;ks
ze?|olgnr#77_I9Q=o7MZd~H}L@jP9v@HK)Q_RV~$T2Pg0lw5XLz<hd=)jo~Q?=~Jr
zqF*=m8Zm)r-P%_3b{Q@j>>6hd^P%5utdPPx9f6P2MJgRdf;+#20B{L=ZbH$`O&nz;
zxsE#{ltnzmopP!J0MqmjE%f=7?U2-hX|Vrc^i>gYj<0&&_&&%IuvuibRKAhPQMlHm
znOA~eL&54&A6}}->Yvs@mNs!dF>!;_FNo6FuNRbd(W9ZLcUY{PZ9tqo5*pD~tUE28
z5_7Go+fU&UlUee2)*Hx_Qn<K!S-kW|w}R{OuiB76LH9=Au~-XteJI;|_ap6j&)fCy
zj{`G<zN0=VrkdE^Agiz^_^m&iZcn?lzmrC;Xq)OUccAb*LYPG3b%9N}{ZvYg1><LA
zdk3^8aesh6X9;YNsY~e$fmvpjJel;l#mmBw{(Q=P%+MF*ndHe_$WM^YT=@10AwMcs
zwXZ&~A1K*iN@tj<IRB4m5P&V9Sb+9$IeO1fWck;a{evb7V<G~Q5s1b$ITEa3p66`r
z?AHT4c-)W`jNq7;Kg&5tRdcUvQrqbI8941Y3h|9j0fEsGHzzA1TiI%7ygz6M(M_=2
zZ0q7mL9ADMuL0sFhn*yo>K?=$MV&1_xx@D7$$9-9H%n@mXJhNf4b?!4&D?oo7ZrYk
zeN{u>+Z_jJ(X=IzBlyts7xzV_lgn(Lr&%>{#dz_bB->wnHd<suu^E6-_#y)=54Wt4
zB~RTBjzL<?iXg?o!Qn$Rwg<hVCs-zRQurl5y}VxwE835A5uw)k$k5;AFh1sg!IVgQ
zA1qx+978p&cxB~Chq=>WN&!S}$__ke?1LL!tcfJ%jwdvL>UFEbbr-Gm*-7YyASBgz
zeRaP5t3B}gv1Qxs4o~797s<CP7&|y{DcjVg(vlMU*oEd}dIHaD6P^ZP1i7qS4Dw=z
zKH#C~A0e`^i(94ZQv9mbL=<a3G^fldx`Rkcub!?k*_N|il0`VubQe#r8tVxQ{`M)o
zb;LHNMeksj)z^NcGeT2V?vw0z3<__u`|Te;RF(eT!Se#Iof6|Kl^h3DS9YZ1b9Gxn
zt#7It(YMU|xi+@MhP;N`@Y?73g`G$1rUPXB2)LiDQp4Mbn@1&@efmAIlL!^*=6q(e
z95O<m#$iMMz~EHT6Hoos{nI1`9JRvgXhdMhpy;k8gRSY3HpC6$yJ_6s2FPu)EDdS5
z&XN%Ww)XaZgpegNg;$i3F?YcWRfzS^=EcQ@YdaSAMX=8l7!k058q{$L72FjV^tlb^
zCwdu~1=Y5^3VRK~)rpj2Mp5QeA+g96Rn_4O&^9Lq9nQ$Sm3MbPe9>D4IyP6KeoSWq
zc;x(54*omc^Dc*djgxYkj&;Z=OT@*jd94>0%`^|(?LgScE)l^lFWgmKE-3Tqm|%N`
zQ+56DJ-lJP`(}EjIU<C23TLMVoQ~T6JWH-s9FtA<L1yzhm8lX+hCeYWdC^%=ByWYi
zk2gj^Vjv!|@FG}XJHhr?*h71y6&rF`jBB$fXKf}x2BN&-{nld}8|4a(`<zGLi*$v)
z5&Yt_)!1f>!rD5DIt=_)^zxdY<f?wQ2emwM0~ac=Nn#zH^y%9IY@I!rr&Y@~bDQ51
zNr69y5&GDNkD(uho}a(^6+(-6*<{I`p+EEmu}4TC)5X}a{Nsn;YESP$g8~j5P@09V
zF2vVvqUG{rxaJ{bqN~j;TV2*Ul&cu{@+*y}o3i?Y$3W^%o0p0DdVWBj)@LckNr7hx
z8ayfA-+zOczv6o#pt=qt0$VJI_w*oH8+0jQJVts>My&75{QRyq^hDNJ<mIs6{ZVeR
z{h5UZ*|{U;dlLPPBN>MrvW!Kn{+=;p8EnM2JI3hbH!j(%<j1$_@@rM41*)1uvRmPd
z`|-@SHFHHY-MP&S`PpTSSQPx<LNd(Ey_lyroBYNMGh3lK?s9*w=8p#o3?IK_w}>?8
zIEw@)PH~__+ieCkO%hHKkB?~U9_#zoP7=%+$NBMQ8*z2Zxjm#_ZzlP!2u-9k8V=WZ
zKmo%7lc}kx9$&Wp?0O~Od%XkA(KI8749QQe8Hk@fPdZEN!*J$bt9qHN4DV#oQ<t}+
zp|mgy-o-LD_!QEl9=3fNCDMMy3%`D#vCg%)CBNzEbKG^0xr46Y;0u`-leXK(SNZFX
z)P~Ty{ci39cCg;n{rUW9c()skZ0^yc55;XT3Sk1(ZARf-ef>BeosrAR)~Ytz>}E`t
znyqX{c`M&Siv4aB7BwG6;^|Dy#jQlkKBuJ%ypAwQ-?ovu8m`pl05BF@6LI?Ko_;-5
z_<m#{VTVPMh5z^Eebt>)KOI3KAuiB+e7$w(lXe!hPYhQc8X^Rw3GY*9(F`kkQaKf}
z%s6}_W~^X(5S6E%0W%B6T#rcm(Q6X-BfsZpz+4|QF^y~>knUl?CmG4mHQU)89g8)`
zond7+95PrbO{`T%RqFP+3Q^kj;duJ32;qA~@A#3iJpPIG@$hBqnT+@5Q;fCW;?EZP
z?`K;3@E8&r5t=+xo`~E@OHcjL<!(wb8Z<QO2fy<0qFehpU@xT4;QBfc1GE$tN32{A
z3v7x(YJksJKJ$|dA6RZKuB_ZO1(m@uNKXR_XI(`%C>cV>LDYO-m!L+>R+WIFs*h7+
zyC&r3iRiJqz(%k{mw-jo^p5J~E~Y`0?AEF)U4}E6C^hnL0?iU^Legr7Q~QvCB;0VI
z7BOM&9mbyevENwZm>zD-{3|b}6?j<P%|D%C50yU9+M=W33K3J(7e@7RnG3noDvw3{
zbk!`q`|;thZBLGd&1`Gmum$1|nkh5fgi@m%1=EridA3RmO+qF2b0&ekQjLcqeQy7>
z?a%&c891NUfKD$La+o;C8(2Q~%EshmVWH4ied}h}{s`c_GIj4Z*`Gg4*L_-BF#euU
zHcptS0oL4v$ZY)<FW$mjq=8e{*{2z8qvmcS!z9$5fp@-4j?ponN^ji`#>EmCbpNH3
z+SG=_Ym;#?K6>qRJe>DR-lN{fmy*ZJNLD|1M_SE&b*nDwtdBS-%^`v8T|f6+L(t4g
z7cWV-Q)I#(-k2`a!bei!aT^Kb%<3^`!|y|}Q8!ozf%)avWv%86(Uty9m9apg@u5{W
z%W%dq+SmT3yfeU}gq%u8AjUU#y_zuWOf)oxzNZ!}5^HerFY~%`5vwxfh&kt}v{kV%
z!KFkm4~BD#R>72W@4L6+xp-XM7gfz@ePoExWQbmMR@~bCk~YzWB+Oh%6x!goz1o||
z9~azugx2uRogl2Kali(>?=^RsI5)D0)xm+iyqz3+vc}6F^ZOKqbHR!JSGe_)aH;UD
z=!BQHQ?n)t9>L{$_0sDFW$CWtvG><6p+U!yHmCBJeHb?C^AuZa$OG&B-&#Wl%mu1h
zgi8s&K}61`p4q;F6vA><uoexCu`r)8&em>Gj^b#axK=kPVu^$~X9?t|IvIs&CQzIZ
z5N8Y^hl!MZ6KJleXg+esMhHVEet@A)Q|t-(zUmh64pE)hsD)V9E)|_R29-;&Db9tb
z!1?D^!<z?2v*5wYrI0pFHx`?>8b~Bsp?6-s9PKj~J9NF{_vlvtPEnu0b+*}}y5am8
z6UDpVfp-dxK?2&cF8<&I!F6K72C9n<`LPc?;Y@gKtk#^$>wPK7SEnI3RRMlMzPAvg
zOW@WfD~!H7XP4+3|7n4XZ4j56K2o6HDj)F$_pQl{bYs7frrxUJ<=#qC%80Eg0qleo
zKIUc^So(fmiJKs6z;lX(jGR2{Me0(GB>!)6n8<yC_~-5J>!ZZ)^uY3sP`SKliOQ`_
zu4ENLG?**d$1*}Dwe%B3Z?eCSCxgD#etlKqe||Jgw>)+PJ-=Uz$+C|jUfX?07E?<O
zrN{qb+z&iuUkl7JTb#6;A`Gsyeq+2{MWM3^Xg=Nog7U%VzPCb<-2JH@GWXiOf;3Df
zbF+@P;KOg-FY&}3VrP#tGU^>8F>q^ilJ7v=>sH60m#JyryzKB+Ozm=bBQ_#psU~r3
z^!{%Xu4MMs>S}(SvBUky6+dVgae5&)MUrG;I6g9RaG>PH(r8dtB27PtV$#CE)H=U^
zTJ_NIHVB%t7E7EsTI1ZnJK(VA%JlbI;!uQXpqkjE8=qI2QnbfUP<7M0_>XIBdB+Ke
zKyFNX4Yzk~$=I)`7YZI*CJ@XM=~BKojpE%#ril85Mch_+C70f!Unl4EUoXHd;$>(@
zGUF)Y1po&0b6NK1-G@;>R=CRz2?oy?kMqNQJPlMsFTfI2LaVU^L(*;X47KGu)m8&-
z-K%a_BR4FGXa+YrEU2}qtQ}?|$UobO9An@`)pimT6qMk9N~;_so&Ef-3jx#~*IG!5
z^yk{z{2MJ<WJF3QXj}sNq+EV~zD}ILegEv)a$UYVH$dCMu$_A^cHoufilNKsWbBd;
z36TtQd3R}-@X~0foR==kif39mNuQOJw$$|-X^GOvsr)ACZ5<9-rR=lFGl}6`VDgRe
zO>1x2x~jPAHy5y<#G5g34@wR@#<4B9L$-NJ$fdk~;p~K9P({ci4^1A&mMF&MS5!Es
z7uYNlYtRxehEET3n=EOXPY=Iq$@CV4XjuvGGw6L7JafdAKA1`2HSe~D)IvYn>6@D`
z*zIZlJ(*9=02Mz-vXCI4m+1Fw&*B8kn4&GutAbP+@Lk~JJ!4v08|lM+TyaYf*K;>T
zO%GqP#h}wz!+2(v<A8UC6OR9t1%te>gpcA_&2TFgBi`#+NS)lq=008v$&9J%>-lk#
z(~Ay8Un`i1Jfi!BV+`3cP*mS{&F=taVp=H80dO<ip{~znmF31D5y8YIL_woRp7@(|
za9&8i7+I#vpUeBLG^}}rg~o&?(XvMp_oU4K9>oXxev}Kcj?W_PXEzG;X&wgI?G<Ry
z$f*9Uxn(H^XWB1=^52Jxcgna?4yMt1E~%<?61ZYCuKi4Eym8^}gpcVKBA=s7+4QDn
zm=WjLQI0O%DPxVtcTgGgSb1EH;+EsI<%)|-+WeX}_G0~I;}68?EQEwP*TF_5+P(Le
zXc!lRJ|l%?-%l-LHP(O4c{+=mTA<7RpC5-;Ez(ZefeZCV?iVM4RkAAhu}}~JEDphQ
zb(`o)^p&~2L;23d!!NznZvF(n-7ZD87Wp{qb(+gxC){Mn{4{NBU<jPVZWu6$2=(rI
z!f9OP)+hEQMVPDVJ>d7rV99F*KPkpNuDF*n@jh{cimo$vx}4`8p~FxxNmHk-mINtu
zr#`&l!~_+Z@J(s4kH>J9Bzxp=Gq#5I^bW~QEHzDeb;=3~Qmi?3iuCZ65Ky0&E2$oJ
z7ko6m(Q$t1BsG{6T;q3m@8Xht-~eiutN>(JZP!y~50E7`q=z?<IrZ@YKWA0um!S~H
zO)3kT0|gxKuyqxtA<r=R$FK)NrOU)TWpNGB^;#pxJ%HmUaU;<PW5fTBv@^>cM1B6@
ziPZYt9MNbZ%f3{1Y&do3<2|76qQ(bX=<Y`-Esp#DOL8w>zf>KrQQ_>>i+E!-|3QX!
zO%;-QM@b_Cqk%3|gaY{77pV&^^}s5R2eIWIhk3eG&sqx7*BY03k+0vmop6}VBse4(
zPJv8D2$d$T>lbi&4h#vL492bVkgTQQ&vt&m8mi&D=j9>A6`@>ZH>EP6QJ-@i_ox(W
zqJuI18W`-D&_X)5+03c-le)XR;}i$?m5S=_=O~^fEVI9u0i1J%M!=_=H!7o)LqUf*
z=LWrtU<pTXqc|s8=#qqcS>tx7hM7{o(W5m5KiPvrce`GvQCWI|#0BeXO!L;>hSpy2
zuO{Mzk&At+H?dy<k9<=44Y_qgR)Dlz&%coD?>Ra+iJpq>8g58|qIG0@YzJ<q5$uI>
ztuG(Em(;c|+2h%$Jfk&cNT)aSgS5)&b3;8jDe13>Z~1Q?HMbf$2{^TM&3{AXF}!<j
zTW`O**m(CJpd}i*d<f};zg}?h`d&@TmGx@y^U9<5V|x_}O7>wH=lES-3VOMLHSs2S
z?LYPdP0o@<+HwlknM0m0kYbs%J)RXS)8Xo@+iSWu`Q&uAaiTr><Lm=gC6&c8)L;-F
z(C9G{^`LdHA})*P$^oAFH2*;)A!ayk?pxpUnUa!{g7`gpN@2&TT1ZWEN!ua-tyOb)
zI&UOb?qwG4%^l|v*+Xgg6vz)5kX)VLhL@wJkltSTp8DiM431+HM$9$%kfGIsjN+Ki
z7<GXru<p=`to*K({HjzCFYEo^#LwRtZniOc=$}7Z*X9wXDPuR9X7{YecYWu@4<a7Q
zQs$gh<zy;(`C3rGiVu+otBF}hQh{PM8WJqdl7+}w?#-=9(5jCqHMS$XGnJQq+1FXy
ziYE8wqo(A(@#l<*iQ)b>ko*@yGe!#!4-aRAnWE*fwLv#urFK8f=U;C2rakQsi@rV^
z3cxKglv3@O28g16CG2V3oths&7N8xyyMFfJo`oD1%FOMqVShQg=T*BTOr>ZYJmtFY
zV<rQxtlc1)vBOa1>0hZrW-qe_Knxdv3T2rjehv=qSmrr{Mu=h*KZ_LmRcH`vj@Hr$
z%R~>fmZSf}5)`^MK=O9t)UEo%SQKS$PNgwQ87A1{jNBYibU+_8&K7+A?I<FuAnQh@
z*=oA`Q#7W-ZWEX=&e$a;ArrvhW=*rWzUjW6FF&YJ^hnoRY=b%k+|fjx#3!r(tZmQK
zG;D&)#F?jHASh}?_X1^Pg79toD%uVY%L6a(l#huqIIaV!kSfKyhVAHJK{7LH&N&Ok
z8h!Aw94B(h2mE@SOt0UOYNv=>f8)`seP+l441>y2*20tSh!F?@ms3Wlh+@w3T*E!k
zmOb!*lWenpkU*4`)c)?z=+FXhW<aFN3Efm+xoT9~YliM!bktm7A-TvhW;5Xf_c14c
z`%JQ5rretfC+Ul2&LtD~uqP(flfpLv68NZ6V<DNV)AoqKW~EZ+ymPeK(pxmzb}^36
zV^G6u1TcfE9e`#P(6FKdB!?^@CTRiB>K)?czxgNHnE(cc5P(%hL@tuj_KZU-9=S)C
z>112u_VcXJ5JDIupzvc~S1N6^i(1pn{z(ERPI;u9hm%puN9HBh95NR4;-~s84}=aU
zM<~bce{L8vS`fNibRQBy+gx;K-xS<3$O~{boz^zD<0)_;XA-Gee1m?b)R<h}ywwyb
zAt9l|?v^D9s@CJLc(e&bL%(FG*|6sDAuyDJir4{c0zCu;U?61j_nOoDx$vTh+g6V!
zB_$`fGyZgeVYxxp`g+GztrPx?5Fv!0t&Vi@O*~l5sQTf~_SitI<Sn^hsKac-HHx$A
z?7+s)GIkjeMFfs{X|iu~_r%4H{8X~xLO^wE3m0iP{6_msdfaNq2UpRw$n4e=@Ye<1
z!pR|>Ch@Y<oAOx$%XF;AF6!#)cVulO5<03%e)^-GOHWPCGUjYTM)e$az~C_H<Kv?j
zo{G^2<SZGvMr8U|<SEovwtiL;78W+WH+=u^g9Nt_n#A;3o^ONoUiEpRYCJ&%&<##`
z@Y6zt!cRCQkPbnkjX(Wk84|xZt-K>_q0?;aQ2yauYr9X6w@A})dV1AT;D_H0EJjUG
zsNv-uRV#gIO#EG{ZPM%Bn`*NFZ2f6&$juUGt)dp>*+eB?3cP;wH)YC;#6xSu9#q9O
zHNQ(=b2Y7$gZob$<VVSGOtSBd$zKmtNe0{dIk#1Zjb<BM7Q!DsSj~}t{X<k#w1Cf5
zl6VpJ4b>T&Uwvf(vD*^i`ZuqU=8$x#K$$WuA2x9rZ4mib2ls}wHGJd1k#Mc2R(S_S
zdbjb_LD!NFU(zT3RgncYXMsFvuGt-;wiuaH?#Wh8c`b@ko6Y_kl)1!0>ZLA*Rk>69
zv``JF;Uuxq-i`oV<58(psVBtyWoj}6%R3<&3+V4mcN1P~l4*av`oOc#D(6;hlj;Dz
zrz^P<Z0`T5&Q?nXkT(qt%_+ne87uZ!n^{Wt?^mC<RZC5Ge0mO#n8BzBK}@2Y;CEpu
zYej;^)F1$xCycTp^!nO<NRU~!6EBCq9(@il^#4caVZ3>e-fa^d(~3!IjY8M>;utUH
zQ1vuyq34_4;SG$o(n;pL=PdCyxp(@<dwJI^G3wZe^-4;0jio+t^+Mub&aO29XV1LH
z3=hf%Y!Eg#{|gLXG*<EQogyVAeGy9~%!MgU%dgIugNyJM1vF+|RI|5v$QbtDACLD|
z@W$@i3UOG@sK_x9Z&q{kqE&Ena+X#X7A`X)@X3QDy-53f2_URJ**BsDe;yx924Ftu
zvCHZ$V>48et9?Y;@HIEhVOk?d5@4}dVmInV<vp>(lGzXq930k@qo|3mFF#JR-+7*z
z-qdAWpFGx$(bVv$6LTvf2%(vg%Jn;EhVqdG#+*ggwzKpn87!AcU?(16$Z4TCKJbIa
zktJUZ)}J^8dJGq+-g+T_6UATM1?r?Gl=(K5C@8?}6Mac`BhDB5Eg?bI%OOeo@{;m_
zGOi4T?5O!7E_+Ags@-==4^)K>@m|neQn`h+PB@%s)N@qj_aTdF>`EKy*&fySqZyH~
zBcTrowDwNg+S<cPje^t!;WQnmk3`&vIwrC5`RoAqZ0R^8Q0l-U3&iDE^f&k&{YVdn
z9!pCxc#)DSUNCXB+utu3{Jx;Fdq=5N#gL@pdlb}Jz+lD2)~fh=U-J2?j+4?aR;SVb
zu5In3>6q}BmG{tErP@q9e7{W@<~|oN)&lCAVQJav0m7GhCJ*y?fY!QAzfo2qH_SKY
zE1p>v|I&#_W1Pq;k@J<ourYIvHpH1^QHv;8rL#S7gMdMzNqe|v+!V#pYN`C7X#1IV
zGFXzp1pCJm$e=ax2oMLOv*mwf%Alo%&n==Jr`1QVj`XAN#E_=eA9t|KP_zCSK}w{J
zx4bmbi@O>#(yZ$>Hae$<T71(A*$XYV4v|ayN>Ap*U=BO8-Xc;$aCF<l+ITX+d#Wt_
z@|PmsG+EMSu(e!T0y$0x3`_`mp;ZW)gV9Z@7=9j(gyB<({)uG!7k^=f6!(@Xge~|y
zr?cNY@>N?O-~U>NvT?HIkXa_)rDgWl385VSxg_wou8Y3|*_j6Yz&Kfa%;S{$bn~ja
zKdMPM=*e6&u=T7*t_(nYVH_;xQ<GLe2B3sLd|x-c31(N)upW1VJ*V7gZ(I}N#2^92
zjZ8JUKHwX0D8hUC+gYS7|BVPHzsiBfzq2D=`SgbL-L6uHk)bx`4&k$kaJZm;Oev~b
z$c-oAk9HrzCjO8b?(zsUCe`xVrhEvy{dv{TS($RK8R&5rTDkMCkac-Upip5?YXtT~
z)z0HKsmU|fyf-%lF`O62z5XXq^9f=@%GMnr31|iaS-($i<j3^%<ph}W6Z<?EPzFyO
ze)r*M>BQNj#`ThD{evyw<m438DUKKV36C=|vrQTthTPWUz1oNM8rv+^UV0)5Ix(fy
zbG)7E(Ot75NnaNCl+_8u`eM8bN{4gi)^)I?kll&4E-uk`DCeq1z1Z)h{2E<biGW2|
zXUaAX=#Cu{FV18Fj~2XaeUUcKXPg`~8l3=dSO&mVL$leTe<5A)&KJ3NY)Z5@h0sCD
zsw0sHVId8g&wGo=LU*3o>B<sETvRFbk3mQ~;)PH4HsU9D9(ULJ)?i=LHh(uZY9^K5
zoLk@C*$4o65a_dYveO&o`1trvv3ciaU19h%Y=E~A0rouuD3_GaCPVxY`rkv5%f`k2
zt!myE-UMd7{t}C}#Ip&jZOz$|gb10+N!H>NuCm<f?Zn~l`%G?4r)n-<X+(coLd`AR
zCSD1Zq2445%(v3dBf`U2mc2Vbl7oVFMS_9HU;Jlg#(QP(Zh1}AzemGzyBk!)v0I~e
zdkLb5)$O7k&by0=pp-DZXooJ6@BKP9vs*k%Qo383wltU0IDpAggGC5PwEpP7wJCa^
zHnKy^1>0mWRWdE1$G;y*^@l(O56mPQh73xgB;*cGNYurLSb@~>H|W9)m3YLGemdO!
zeZNSG-)Y>m<l5_lq78sSio)aQ-LbEth@O-~j+8c$shuNtY9#Az)r?*|ocvKsBRXsk
zp^{0+^a+e9;#6Dqi4u0m<6t)&f=rM-Wr*!VVLNgBtSTJw<v-jYxbtr~CVJMlEj~GE
zp1mb>P|b<`Y33z~f6EefA|eyM5t}hRP@4Mg>zv9huhRvNzCAU;rF7uI(bGN0vH8<2
z-Et@8`WN{4KV$tqWjkc<tx9ll@&qaXGF7m5-dSYJO@z7_^o>*lH-XsGE!E`SL;~sj
zqv>}D#Hp3HEzaM^O@FRLdQ8f{G_otTcHunh4H+i}4{vT#I|$w1gNw1R%^(epv@LB~
z(4{Ac!U<a{tIuyE(;(#otS|Reh1d7!3=XE3z7+dF*$y@inm}=UiZSb7QvTNC^ISJ|
zXwy>_z}?^$N!t96GM40D2H0Xp(2`buq<C`;_=2wDUm9b;f)J3J*0;RFj~GAx7W*<p
za@m2V*5I{&ZL9j@!#A8|W%BZ=y21xvK-lQy5KcCg91V<X<xAbmnZ^GuuIhVFj|UHb
zL@y~Xs^P37-1nNQ<Csx@g3Wfja;$+lsB(qHc%nWq;ST=bA_y%6bK+WO#QahT$OZD-
zw_|}SA<hsL<-!0=bT;$*bj{t7qtO-mu0hJ(&y()|hfBj^7z}+jcayVP61fU(YzI&%
zvTBn3o<Aaa1hNF-c(F1!G*lWQ{u#6IaO2cmzu}F_m6L8Z|2tvrApMP>TVoew?(MPm
zRQ_uxS#AL*r=;*QNm+<w%^XuW<upu^|8rNT477=y0(ZSmh!`^+@Lww+Qd3)IDgb;b
zj^7~w8E-N+EIil<2Oiin62`><4uF5|NXIM3MF8+@>HYNIkw!?9Kb7mfrM^D0xb+|v
zgAS6wF7&ed`gEY0MLj2~V&CD&BNfeLlpbw=t+ad8#EnO0;hCJ`wMBnam7#-*T^jt2
z@Wagcmk7~I8h^{Lf;*{_u%C&aws!BpK}v%>cwO1~>ka&Nzck0B`0eh_0j>!sxZZLJ
zxb)oJ64U)iJuf1LyTGJ<rd2H4i>&~Y`-Ac3{hLSa$;^3wFZ8EK>_=pH+z$lvrOXR_
zZ_QLZ7Ia9gCHr0{bxynde68Py;Jxmc$g^}oiMPfV$a6zobJ1FYdqZjTK6-PmKU|6e
z9~lZz`g_!}r)6K_cYKfU#1HhIZlvddNXf_+?NX1P-a?)db%1u4zxGdJjov4tj$pqP
zOKEPdxvi9}IMw$%SfVf>A5%W!?*RAICn<tU=QFhmF#g{tHw1r~5u#Tl1PB>A%bWYz
zSLTwX=;!6{COfnD-5?F;d~*9>tDee&VC4daLHlC;ON0m}d<gR!igFv+y9`qc#@^0r
zHcZ}X>a8Y+CfLO{!J~db9xy|zCKr-G(S=7ShkH$gKy@fy_#GY)XQzqtBuFum%5OS2
zm~+aBhiYPDW4}WOa(!Wh!K3l_`d`0(&74-?jR5qb?&rq;MnzMKM}99n&-8sm3njFU
z&OuQL{*JI5`qu91f@G{WYuVmLwBzQnNrK!fFMaPzO>kSTAIr5{RMBoa&>7Nwtz_I<
z?L$t1GzWSmt~wR|7}Z#7Ok_&<GU+BIs>qGYxpk}`URnKd1v&^X+?L*jYwh6+IrMN~
z{X`+%yJu$U;za)MT?KLBG2djK$3nLg3RXcyBI5gv`_z9_y)d|L@2s$@cv|Y-oY&A7
z-{2-KD%L-M<3wcJX>z^7s;>S=Uoljb0u}FldTJGI;3@E;f-UdHwz}GZLr1ER6AGB8
zX4zYn3OEbni3-z)Urjb#8V$^0LDB^LVA>b~u?4@YnwNq&=dHZHsuFpB48q%N49K82
zzDG(?l4`*Dv9<3nSEG>`@i*XMF>_U67~b8}ZhA>VG~P<(bA0FOK9$s+Si5g!T&-!+
zp0wu43W4$&j7@<ohLeGW$=UZ}(&MvTE<r@S!tY>UnCL{7%8ekAeUK4o(ZK|>P7gyc
z2{TpYyi*IlEZwR{9gN*S1j(9&ijq>2kd9q~2&)Bij#R#B#~93AuUG83TyhpEgLVMN
zz_b4l)MfSG-aYvXCFVcNTMm-tfSfJZ?4x$4a}WQ$17)}h7^(;m>HsLQ%V8;#v&)}X
zB)XIZJSmC|%wLazm#G4n&Hex<qO{E!FqCfwg?`J%7PN_B@T;3I;R3H$H*y&h7G$70
z%=amnqilg#!(0yhmpVGh?;_MRKH#cs?Z0<ZP$fD3AJ7U`uF)yEe(0e(-hOMp90Tly
za&eOS+Q7=MxqX%)HQ7X*AEJ*cL4gWW80AFxj6HzP6uM`mdPG-ZNDE3+T=fR4+8!Xa
z0;-B^+JJ9N#glUV=<w2cp*x0-b{mQ(wM3@VMP?GrF<44UaFmeLO13zy)raD^Plte9
ztkgL0$-$#lmT42%N!1_zgIGjCTr5mKdhO^cbduH3SH8Lg91dshp4ZFa=<4b4qPlJd
z_{<Vc#q;Hxir-4HMIuU9M_nWOxYfK`#zi)6JEkU~x*61OJ;P)B<>0d;48=Cnj)St@
zR9Cx%XaLAs<Iur3(cOEyHedn*9Yg5bw{NjKO?LFO&Nzr2sFi&ho{?aoQA2G2EFdqi
zFy{0Bagp>laTYF>7HI2KSdD^=dT1@#v_Hxj&LID~0xr9@dbDACTy_-Ilu&v&a|0sQ
zdLF3t)nq;tX&2!=e(7Z8Tq$ZwB*IuHVBQ@)w7R-FOP{lU(|D9hK%b0IyssNUrwWxH
zdx=lSSJ|ZELU~0$z?fxfo2^YY^l<aEzPj80KIc!^aZFm-EdRW_3<*-c($6b$7;LI4
zO<*Ye1a_X{Z>s(JJ)e4GizSsDlYoo-5Snr7U20d5?BzEwm(Ej@<{kBsAL#$~?YXR+
zI_%^=u|L2qkOF?Px7>&d#J?Ydb2v&UDkgR<{?xJ5uDta~1;QcYTU`Y_hw$S2e?|lF
zE_71<W1#K8ME!LcL$jDk&JZe~f9!W~^roWHQX;APlLTkt*dQ>3k@7k+m>6z|dZ`rJ
zK`+>z{`^aGylpn6pd3jxoo=)lF0h^i)>uHGmc8+}t9P8Hq-H(VQcwt^k`7}#B8o=~
z{~tz(L%UGebbLJzIU^H{@D6;wTWtx;9*S6=+;~>=p5^I`{*2`#r1altRX`|PUJaKu
zi~BNu(J}H~w1uo#@6BoF9-qBP4DJd-C?@!d@fVY}wz7j6FfRYVAXYHeL40RyPZK%9
znIx80E(&FRNa-$se6%R!v=#%dKMuCK{kr0xdT#!jb2c3>ik4<x+njOKO;U!Xs18S$
zO&f!dN(p~}Gd}j?*RPWaz=uVFSdMxUCh8w2<Ru=C8U8PVqJi=TE4#I&A63Eq+UfMy
zgwgz|;<KS47xbksn1>p1@Nj0?8{$m|dcpp*)-ujfL9-p`nY>UGneCZX8Zf-b;{f_Y
z{su&~DJuN%f1x`H`#^#qT3I+{`1dT}b5+2)iV{WbXlk28nD3cWg>}y@^jGlBRgEQZ
zQpK**k6N;57QHI;;M(;gWu+B$`#{<OCKaudP!rfT*6Rb=K>O$U7f@km0IRMAm~maX
zI`V8!(u*6sFYdh;|2Yx{rPSJZv>;T4(PRF@j(c!!T&K;oN7X`d9+jHwo{*@k3Fe<3
z5a@*`d+aou!k&j*MWgS#!qg~_L@f@$wXjS?bFMF2A3FRF{(><9MhHJy)%JHz{kj&h
zDFr~9`~stg{cjU5+N358ARD2*lyNmVOd!0kLOj4g+?(S4QTDLDd*K1Kh{k_{hg^0X
zj|Wv8S^=eNdQ*R(l?fj_rV^z;TSFYBA_5U;SI*`8)mWohPz!vVyDL;jwqV|^C3NQ8
z<4UFRFz<<hfnTgm>UG`|*K;Aqsgp-}CaFt)(CSBuv4M?%AFThSprUYGAmf{MSES-A
z8?Pzg{SMvKy3cnV0C5)BNOGtP{D*=-JHxUHue@3SAku1ZBon>ngHme1Z1B)@(AKyM
zEf-CP`GcRX+;+uY2QkkKL)vq;B0(JCgV~0ungnM~gMK8nJbT?y)hBlTFK4Hg+4{CT
zEnmD4xVJtBbjlK&RLw{hw(cAKyO2gc;ux2|92Ek%pB4*ES<^sK2695ZNv05>vs1IO
zW-Teg?yK|t?}w4UO1hsp3S2zn4w@kY3U%(DhYivpx>9|{mO`z|*JfLbfa71W`d;Pz
zW4dg&vikCd`2{T{1^f+eB5Q+Y>>d(Qyw%E$15)@HA%PL3fV%_z(y46S#6t_TamyV~
z;;OBOa&Ev)ksPUuN?cQ+9Y~q!PCVg(Qta$@>=4@DGT@O}q8~;!nKVCfEsrZZJQ)b{
z#H+;ql3C&%|GO;40sxq0t(B&hpoe5aOuu}CEdgddQ+qwXsutF|I$hImw0T7rC~s(f
zl3sFWiL;<=5fYL5@^ehJj#fQy4kTbX>+f*35y$bW#V2{$>t}ta8G>r@(s8}55*`^|
za?z;YwpUiZVPrF}PSD2aE@XV@Oum$>VdUu0PifQ|t|1473+A5OUmf74RKbPdHi8z$
z_<ZDeyF5A=4ikYWCo-Ql^?8?TvMmg6i(qm?5VgT0Ihkj;b3K4`@10B<b-;w*?sDyL
zOnC5q|FzN<o)tI6uc@{_J7<m`$Y^MgEkb+krWAUvuGRW?_h$ebeZ4*;=?2O>q8s1M
zVZEBTmfYUgW`nZb`8x+*M92T-xBwp^5^UdxY5mP+KVsM4H>SDI*vK=`Y;ys9{z@o`
z)2pz=&5-hu(~_{Q=Q1dhO$X`TRAsx6x;th4mE46{M$*NosHm=KSz8W%sLV3tl(W1r
zZXdcCLBI$vPVtV0RbO(Z2|ERlL12Qba^#A22r8b5FkI*_nGfFtGx{882jVmD{m$h2
zU+BSL5Lss0coKcSA(-!U(z{Mls2~fI_|#}q&}Q0$8alSoBKW6!T3h4#g-7lAGTKyH
zaL<JAH>7<uDIXtS87hzVeH7~3vBXOsM}`Gd^~d5w+9{;tWi1sXh$2=LURixcc#{*?
zDvU+~3wv8?MD4-9<QPm3GlZbWfHTK4(PI##NEsb3Hkduo@u|Z<Gy(Q?<LZuK-#C;V
zvO52TE6k3*c!y0g_v7*6-hR{*%<m=KOkOvE!%V2?&rE-kh5Q!_Q&Uq6V#C5W{^j-)
zHLDSlGIF?NdW-{2Wn&7DQFppPy(WB?M*JUGjFvc<htmedC_R7ksw)=VCas{vEkkq!
z6){J3o~+UE-AL2JccWm4r2CTMRXbC4TLfWqJt#*rv9qNU>+a=P-S`;gdu(iMKv;kJ
zp_zu5x_|LyK7X~c-JdL5YwKS$s}`W0LjJ!M0eDs=(Hq+Z>XW3rt}HwL+89k*79_<6
z3QjbPYKV8kyp6fJo2GAiro~H(u*Q)bOYEJS>h@`MUm+tp02D;5`o{r&9KCC4=LVop
z0kp8KpF3`>t-WLjMY)Pqym$Pc%HuEa=?RKE>PK_7y`{dZIyvJ~jB&mwab`1%?tE5x
zM$bEHS)M}T9(2q-eiB)xTaB`z(&E!anQ(;TTmaBGywCD=K>#}k2Q{Wi0kB9^0)K*p
zzwoViWfioh;U7PQzP<ZTOaj0!kJ@Z1C3tu$MN{*vT4p>PWn4InqgM=w48sgmdP!b-
zrY@|qovpXREQSN<PG{$x{ZQeC@0Sh)1qA$PSy!3@B%Y?HZlpMjt8eY!AUmi`>jMFm
zu;WIer27971nzg{0cU?~)~NKp$L$gB@^o15wmf32OhzG3DKTr^7^|7>D4eI)RcBIs
z&c3kwOS3nkcpPy$TVG}f;vB~$acc6Ix4M428}xRZ0zsQnj_AK|`iAB4;ZS8qeG~t&
zO*Xn#vv`Ue?*P|kS^#sm4>A_0(+otzi?X){st*|)lY|jo`ZNBToKr8M8<~$IxU6WE
z?s5Z45&Tww9-Vz^y)(xPTw4`%GJd(T?hV3>jOMS!@BA+=%C5<i-2M66I$Gfu>3UQE
zM?_7-rn$3?M~v^Ijk@7J!T{LZ_q!9c->RAzi|Q+^1~Ts`ZB1Ny={DQ`{Ftc(#9KZV
z(iVbCxf>qI7}Yt5?-<p!_3OgwTjr2|W#`RD#tJeNn4|igC;bdd=iurazkw!n)or3C
zE$yzBC`Nhe4=nu@0uE!S)^~!Z@yjX0$2|G>#S<sqAp&QuhSTSTusgUICcYub$PL|w
zH2pm*I8^fIoEaxtTft<&7ynv<E{eM5y$0u<#Ytl0|MQpArGMq&D6lPkdXq$Ro1tvQ
zg5Eh$?0IS%(Ol<*;$tss+ZemC*p~yiP{r``dhX_lMSAN2b0YrP=&sKB8wWZiT=RXm
z9~gwWTdVZGp}`BlYVyLJi~r+#&2&eL0a>^3jQ~_-t!+=9JTYO+;R{{c$pDVPLXd!d
zcsc;h*x_Jh$ck7av%uyQnNCPg-f<5yiJejN1>P0ogyr=Amgh3=6rTmVsHCGi^N+|Z
z%1<?#s{+r%#2;ql^9nxZT{5g1etyga)#!L#@z!>`XLL`zi*VY!Jl`Del#<5*^}rY^
zMX9put)Z^UMH3#8tcbUQCwmWMGqFI)jzj`!!C@A3_PqBZ-~&Pg160WBs=`w)h}ZRW
zRmi>3;IFA(E}<9c4eg=$(abmC@tU<rtx04WtwR5}TYxx&qh5?5Pj8!K6vNp*elKG6
ztp;8%E|2Qgf(<q}i3y!*&1$FW@V375;<{(aDjFPMdMgmVUw0lTQRiY8o4aO_#4B%Y
zq6`bzonsxzHPoj;9xyM#i(^XS)}=$AS@k#5!bNWCL_=%yqQ^J#BIq+g72{vlO%lwA
zC?FX^2Yr7?6s+A`L-khi)qmfxnz+(OlRu{AE>d{f3iR%41%Nu)*@&Fb*$lwy=!>Mn
z>R1S1FC^vV&pTo}=9X@-gDVRj$3!&=(I?c4QQ);<x?>d50)O!k3M#09iTS@SOi-ik
z2;ERuay73J`k*QLpy!70eLxXnz?D2RKK=ku?E5HwOJHN#t#j`iA-Kyu<-HQkRp0Wy
zVP+J`gN%ND2L9imXyq5nM05<dEPyp*7#;tK9cnBuF#KQfUbdm<Hlt3DvA_5+lVkwk
z^B<~Mjf@U_Yzfr)GyB$`)|hOJX@CG!*$~`vg{Ylq4rUpDQ$xX>CJ5zAqv4wHb9>Q=
z4R1oYCxdJ{LLbp+G4es9^kJld(X(f6RRuPHUi*u7VL=_CO=XHn5K`Psj-DdD5NIUu
zS?0aJ?#Je4UmX5Plo$N21dte<+LD$;j8QUWd>7>j^KXNHzZZZ5*vf+uR~mTb$6sAQ
zXV_Z6w9i$JAaSA#>H%cTCV+J7pQuhcE=P~MJ=We)_<bMRT&JPQKumwvpP><U^53*G
zKJP71O@p8?Q)|%YdtXGHY6d6!i$I@Yh%Pg&imB<mi0W`ZW6lQ(=A3<EBgFvH_owrq
z{qK5Yz#}oI9+RV&@~v1y*Zhen{3!%vI&3u9yRiLnB{}!Ky#Le?L0r)g#V9Oan`vdD
zX-1Y3_icebMSEY7OZ&LC(Xl6VW7^Z{k2?Kge<-frp>s~+v}#k=L=o@9L~K~lKZhYT
zC0wB!5O9_79!(6S86<s9`cBCSXfg@C?%!gBVVcdJmFm}HoJWj{zk+2HeFE%h!9bW-
z^f_eU9~~6+tg3hYP^kBv+JE0!wuq*kficSPPaMR7-uaKiaz&hBZR7(^Yu#D3-=}<t
zN?y<&JYnWFg8{G0w-W;rkEQ-+brgzWtLrcasXzg?88<`jFFVnWZal8I#G4E`K!?M|
z2x$fh{))I+Q~Z;-kRkbv=#E<+o*9PLm_>JgXF@e<`u4+@PTX8vl3xY-V4-DJ6907-
zgPf8rNm+vqxmn;b=uzq8RS9yV`>(lajtQxQ$kGPlA0ed2ZfE?xh|8V-8<oBk=&J1s
z9!`{NW@!9e&%ZAO1y~;GCmnV_F&K`!`#BElanjuQtPt?wzsy2GD{XA_j+L$doSEA&
zM@TU&;DnLQ>KbR4+ED-3=5&+)70~b!e}w3EW*bfF)0@de6%_8fNi@(Na#_8&^aXv!
zOQ4Flk91S_$}(yk0BPn@3-GO~gQtxAzX5sn30Vp#$G(TyqZqFkUg;+#&MHPMGo%ZP
zJ(6uPj<8mpb>MEl*sDkJlSfPjdAl2vUwB~=ihQWays8D$+Z^6uX8bS6x&Ul5NjrGg
zqQyA!x&?$($7j6dZi}f*T#v_BgAn7JH90ilnx(agW3zVU9_p})lukRdkTKqUlc~vO
zqEzC*#0VqMoej0zSW^g#IB{|83p0J#e8s~z(?FS9jyPqmwCf6Ykr1#;t!tx`Dfs~r
zyBQjbC2Y5!{C+(<B(P~8ia|a4{r^4|9@sBid;F!6lx>sZb98t=wTHjsp^FotEv_aF
zb%wO(M6n7$pd;5#FD6&87p@n9+qCfcV?{Y9ss4G%hb%rmzycYrnc}bysGo<Gl{a0o
z!vck08uFJ(hQ=`os8e7aXtFk~fc``Px5_moC8fvlooC<Bo|L?a;U;je18w+eWKFax
zW4^w5|6ws02GK)p^qcA!twMc;^t~uMK9~wDW+x8S^6K#c(M0I6-MiKHjh_}yC+xQM
zFlk_!mjGd(=lrHQK;g()gnAj)z+xbP(t@6y7?H}^3q<53aAgc1R(=A2)YIL4kw}9U
zz8WtzEYN<7mOf(oGSF-+9}}26;hN7@tu7nu<EVtoQV?H@&CF*Hymw!MxsU<a9rmG)
z+t-s|BZSyq!vEcTmUyG%mmcopvam7plf6<0`}X~cG|$)>AHk3|(Z_XpC|(=@tL1{M
zfChi=LVG*i7-Q#QZ<SGBPtSx<LhWy0RhI(udB*)i>VRX~MowUGW!2xXI<ben%w{4i
z0Id?Ftm%h+WB{s3()kB(;Pd~U`Cpwv7g?h-B57IvQYYle1L|cf>sGV^^d=WC{_6c?
zPQ!g8Yf|H@k3&1&v+AJHLj^L5I{pAw*jg3j=c}UfNTj#f>(LW^`zgSg8|M|XTyfL(
z?iw!(a>^?co)g2+rsVqf?>%X51B{O|r>BK?kD+{Q{n9HUyz$X{QNop`s|NA$|3NuM
zQourkM5qoHTh{ho%*$0Td_Qt#9dM4}4SG9YMu_neRu?^SJvK9qq3AyCNqwEse2e3`
z$U8}qN~*8?1o&?&7a|jV#N8(~m|5`%*>!1fO+)Bz6*{J=oqfuvxf8N;d&6zw#YIjz
zW$NRKzL2>IdN+IzrS;#xe^qK?SG2NYi(wSrgN><m!0ruhT^^(B>E2?XKX7~QFU9ZU
zvb8lMalA2<aXRh0^+0R&<Co@P#<X$Y&R}#MwKSp4Zjo50m;KWt<lR!ts|)558fmVM
zxfsZUvB}BisRuf18e^hXUcR8RX+?NzejOrkQqO<tdf$I07nfjwpp^v%K@d!o>;F}E
z<$p<L-@md?wrXXQCaF2Ow2WA(x#cpJ<KO~nIcAAgCQX_kqM;#RI_5`9EzBLIM$&T4
z6wDMYt^BwEg}LB@3mGneDeky@uk}}apP!%CdCvRXbI(1m^W5`3`taMsQwzr{;CiK|
z{1TM~He}0|L9<8C(~H}ZzveGGL?cM{ydD|r*_$_a(lYzS)FR^wM_4MQMQEn9)G!bp
z*ShO$1D~ZiTew!UG+snDbIzHZd!FP_^X5&i)`+!qe#|$M-@9?c23uRhO>tV=YmRGP
zCdKImqwuQWuWg*h(Y2O`wyXLSv{A)n9gi&j4V`M(r3P_<+dhW!^Ij2Kj7nB7!%BJ>
zECo}#`wN8A8+M*WsTR9O1eo38b8|ImP_yM@>8(~x{G9+1W#6aG=~c{MCM=PI0XjMA
z^TALCqolj<(h;sp7yC3{qG{p~C@*|T9xDM{97qhRhApP)b~t5~G-qE#aH@aDI||L_
zZibj;@`EQ52COoJPCG-a&9k^Pi?DGgO6k;dK17i-s{GS<@`rWR9bcvGQ8%)ibl)~_
zv7#N+kCv}&$HdTRO3{OcX9|6x>8-QeGd><39>Y}GO^K{3mpI3#DOIr0l!-8BUNT!e
zkc0CLM59SWV@oHUz4Hq*^_D%NF<>!QZ?$0`9Se1JT5biZIS*cXwQoAYy8DZ-V}Zi?
zY{XT4)vv=J?{2(EL2&VFe~UOd4%=e^8;(KR%(mp{;S}|dyEgz86fHHhb+gCNYn(L5
zsMWD0!;6U_F2$(l?LYtn`h!bef?}eSYPs9~b0C!kV1ae|_^>Xd_3%hC^u@!z9N|m3
zYHxdD-1gY*T~qcc))Mx+?3EKKN^*r@P=s(K?eh1r(V_NR!np60d+(f$8*p+56YrBP
zmz$_K8wmFh$r^kE?|4jB(AuMH76@=fMfns{#qWi;F?<+4hi;?RenNZKow=^)1yY%k
z)z)0~y;1e!ZnJQ`l;&@CPnpcQDqMA+UGjL9e!{;ojkkPHB@lBdGx?bkJco4^#Rrva
ze^9>eF^_K=YyWSu+15=cj3NNCk(HrAIwH<Z@wpS{)E!76VIJHjIJUo`Y3}=!!!Vb9
zo018UMzWQS;b2G6D8>WX_Z2x-D))xs?ab3#4N{Ez^Uyd>j|jAXn0~;Xng7*h;3n#T
z$h))c5*26AeEpTx&c<3p#f#_9<AM%0d1x{g$>@Q)6|%YCPmEtJc!~AD`64|Jt1eg3
zGxPdsFmCAbSzOnX$&2M9<<kxmt-1!XLz^(>%dXwXBVgE_qK<ClqSZYwQBMKCm#{bf
zefwF{k63S;HUdGHrTfC3*x2sm<mB5A29OZy)({!|kVTWD-z+#60NO)O5PKv=4QUA-
zv&tFme<X(J$!u>+d4It>Q*x&|uKz1H%K94iN<luV1ln<w0267Q_%i}qSyVI;Yp0#+
z$nJRWQhWR4I|ks{80$V^xblcD(v?ig39O=`B0uB6hlpwHQsvxY93{)l=?6aR*vgWg
zzKeCJo7LBoA6zr7|2x}n&7^Nf9B<;zxBQ6!S^S`fi{KyZBnqPL%PXvi?d<G~pp?o+
zZ*-gKd?E2G(9>UnA$dm%kEN}}TsRhXpANAqbXT+ge3at~@Tet(eY)>0Z}~`mTscL-
zUzz(Sjk@MF>}+v{vUOM;rmr`$!3FU@qOlx*uxUPGPGyEG5LC_do=P5{6U3|N-*&l6
zRw(PVP8Mb`xIkZ9TN;I0tZyUS2(ziCpKzmukK0W)7jK<6=UY*Rw{b;KY5l^>EAOI@
z=ioXt8Klem>`P+p014|E7;WY`2?FiXo=`0+Ern{|eBt0>QR&x|lx}wNry=Y$9LpwP
zUxk-oHd;fV7kF^UMWUK|4#G5sF+-x2B_m1qUgo(1D%Th7YR(C!_qLP}H8<ZyaDP8e
z<Jx$MiEA@s?PR@VXhH_xB2ji{^2cg6;xa?SbzWY1%7MFvm0!Jf?ku(!!dy-<Ct@}=
zaz53_t#|pKhGlP$y>QH=y)7Vee6J;5+UF@=5wz8gtAMipggNAgdP@GDK`rmg(;VMq
z%)a2hrU-)R18$W2Yk!0o!ru<<EF5Tv-BFe@)1+xKpPYBZ!t9Cn;TLV`E#@MUt9XSd
zsEzRab9(GioRy1<i;d73j9~?*h7L?olfO)*%D??bf$Du;-q>rcHnwUs_T8C^$6MTt
zjQQ_T^^5g~wo0d7*eF-bRU;aEWk4m!eO8%%O0(rim;D~SI0YbTqecv)?ACG;eRB01
z9=YV)uE@(4DDTBG=tANCsF-)HG3KJkEDcswWo6}>bai=AMt0(z0ihrea0TXk5O9TN
zK2o=^Hw6gV>dqTlK;rIC*rem7OWj?%grPHrS}W&PS#AdPsS##vEs4Et(#iC%!=iPg
zg=N!2lC$t4W{_5YKe+CR{Pkv*!z`h&TMkmKSrbmR&FnL4PBpj^uBi&d+S)d`8j!1*
zrjM#DV3=L9>81X$&xZ9Kbs<jId8SRZ)9^&FUMSlIFR7PIN!?n?7fZApdwj$zmPtZ?
z%3jS2&pdhAs5uNZ?5DH{f8J1;GuDw;riiXOq*gIfWubnWd>J0tWE^=^TCcD?7^9q)
z<;6m5G|P1UBi7Evh0Q?Su5p;%0T#(aOe9dCJ)v2A@}EndMm2^>LPMRD=45-f(DVE3
zi-}4g$nH4ybXz`i-Sr~FkR{$Poo>Wkos?WGADb;-PC5r&y#w8I;dObYKTG(w<vU~b
z+!wyNFK;sZ3!VGV)$V@sN$sS`G+)(*1X8#hOkKsK%~WHT6Z|NrD5p5p#W<)+(^TD&
z1cB(m&phIfXGD*MYag<n`5N)(oz-QT`Rm@WSQQYMN}Efd4`{LL+W1ex=<HjWI8HWm
zCUR~L59U<NpL4tLWhG+gWG?N|43oYecEG4L%L~<%q%j%X2@C}sM3`UlbV@!_l>S)c
zbjv$!Ab=d+t4LevM0WgcP4eP38JesG*lx%*#Rz`kR6llLFSH&oLW!aK_8VLZ&HFeH
zm;U)xx5TBO9eZ+K7;2$W$z(!JDjtz4aaVTZIgAT`{l)x+K=`>+5R9^(>BOnNwYzbq
z4CMoG8+U?*dP7WX1K6Nz-n?HlO@=<6*eQ#R#-U$HbLR&Z!#Io#uhy~lf_Z5VW8!-G
z{16CohFSaq=<fFw6v4xpd>pzy(?5;?t?zbF2XLaIl8%!1fUEbMab2>6DqsE6W<uP@
z;2I9W$6kZ>bI2A?u0c*znZ&r_0S+37GU!|EnAVm|!-veVMufcT>ct~#u|UrE)Vysc
z4zzTAAv^E}5(f`kH7}19%|FXdg1H9;Nj$2(rK;1SU(bUmX1Z^CM@L6hHvb~0FDE6`
zmZ*~Q0o%8C^Si*!oOb~M0q9Ywl!QO_U_X#*J2j+ap=VF18=gEK&E$W(fIB&f4`Ig!
zLt%@Nt5Dif84f)v9>pVvOPzVzupr7SBKPbhLN5h#Z^$9A-Y>kj3d7}X2a*mwQ8@ev
z-h@o<>NlCVeyyq*&HB_UD=&dOzQ#_A2sK8b5@YQ^P86TnfqXAoiYu7k^^OD1d*W5K
z+$S+Wz=qACR~in2<GVEc)}qG#`MPm>Ai42!vbUcDIrK@HD-vnd+g3N&iZv7Y1fbDU
zT5T!d{m7*kEhU)4Y`XaI&A?6ZZNFK1=mo6@8nA!;{|)-E#DV-}SQ4Kj`5b||Zpv4w
zX|KU4oHj^zkm>J4%KOZ_gw)FvuBdmtvpVYO*t#NHM_>PT-P1%`5`OC@>jM+%er<H3
zyieuqrm6lE?-4)BoMX=olpNGP%fklmyX|AWEn7Fh6?(=84M9dl{wf5%l&+1ONu9g*
z(dM_u`we{}i*VA8ZN;7Dtr_-ik30Cyee0jO(iW_!AsGF;ds&j;knV#)kvr-?SE60L
z@#tE;xK^h-p6;FIT@9yD>o0E|&ETO{rKx1XG)q{aON6;2PD%Oh;b5Lw;W_sewMaG$
zb|6P1zs_`PG!c8Seh)ZRNx;W9Y)FaKmVW1mXI>q5+FPBjE5Ts8`^Zg^pZ~R&#T*JJ
zZLYD9QVG3nTi8#0z~Pq$FYvqc!*3C<W0WRqa|GXq#3r*Qy8C@if(g4fiy(s43A=@~
z$0sX_GyOr06zke}=D4Q)p2YQB{H<l*?@j*e$;IOt8;Vl47Xa{$8VHiy%%EN-(RRlV
hjn=jL|M2Ls*AxV`4F}O_B_Zqj!tCMaO6{&h{~r`ePtO1V

literal 0
HcmV?d00001

diff --git a/docs/make.bat b/docs/make.bat
new file mode 100644
index 0000000..04cc1a0
--- /dev/null
+++ b/docs/make.bat
@@ -0,0 +1,263 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source
+set I18NSPHINXOPTS=%SPHINXOPTS% source
+if NOT "%PAPER%" == "" (
+	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+	:help
+	echo.Please use `make ^<target^>` where ^<target^> is one of
+	echo.  html       to make standalone HTML files
+	echo.  dirhtml    to make HTML files named index.html in directories
+	echo.  singlehtml to make a single large HTML file
+	echo.  pickle     to make pickle files
+	echo.  json       to make JSON files
+	echo.  htmlhelp   to make HTML files and a HTML help project
+	echo.  qthelp     to make HTML files and a qthelp project
+	echo.  devhelp    to make HTML files and a Devhelp project
+	echo.  epub       to make an epub
+	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+	echo.  text       to make text files
+	echo.  man        to make manual pages
+	echo.  texinfo    to make Texinfo files
+	echo.  gettext    to make PO message catalogs
+	echo.  changes    to make an overview over all changed/added/deprecated items
+	echo.  xml        to make Docutils-native XML files
+	echo.  pseudoxml  to make pseudoxml-XML files for display purposes
+	echo.  linkcheck  to check all external links for integrity
+	echo.  doctest    to run all doctests embedded in the documentation if enabled
+	echo.  coverage   to run coverage check of the documentation if enabled
+	goto end
+)
+
+if "%1" == "clean" (
+	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+	del /q /s %BUILDDIR%\*
+	goto end
+)
+
+
+REM Check if sphinx-build is available and fallback to Python version if any
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 goto sphinx_python
+goto sphinx_ok
+
+:sphinx_python
+
+set SPHINXBUILD=python -m sphinx.__init__
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+:sphinx_ok
+
+
+if "%1" == "html" (
+	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+	goto end
+)
+
+if "%1" == "dirhtml" (
+	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+	goto end
+)
+
+if "%1" == "singlehtml" (
+	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+	goto end
+)
+
+if "%1" == "pickle" (
+	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the pickle files.
+	goto end
+)
+
+if "%1" == "json" (
+	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the JSON files.
+	goto end
+)
+
+if "%1" == "htmlhelp" (
+	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+	goto end
+)
+
+if "%1" == "qthelp" (
+	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Chainer.qhcp
+	echo.To view the help file:
+	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Chainer.ghc
+	goto end
+)
+
+if "%1" == "devhelp" (
+	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished.
+	goto end
+)
+
+if "%1" == "epub" (
+	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The epub file is in %BUILDDIR%/epub.
+	goto end
+)
+
+if "%1" == "latex" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "latexpdf" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	cd %BUILDDIR%/latex
+	make all-pdf
+	cd %~dp0
+	echo.
+	echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "latexpdfja" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	cd %BUILDDIR%/latex
+	make all-pdf-ja
+	cd %~dp0
+	echo.
+	echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "text" (
+	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The text files are in %BUILDDIR%/text.
+	goto end
+)
+
+if "%1" == "man" (
+	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The manual pages are in %BUILDDIR%/man.
+	goto end
+)
+
+if "%1" == "texinfo" (
+	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+	goto end
+)
+
+if "%1" == "gettext" (
+	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+	goto end
+)
+
+if "%1" == "changes" (
+	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.The overview file is in %BUILDDIR%/changes.
+	goto end
+)
+
+if "%1" == "linkcheck" (
+	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+	goto end
+)
+
+if "%1" == "doctest" (
+	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+	goto end
+)
+
+if "%1" == "coverage" (
+	%SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Testing of coverage in the sources finished, look at the ^
+results in %BUILDDIR%/coverage/python.txt.
+	goto end
+)
+
+if "%1" == "xml" (
+	%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The XML files are in %BUILDDIR%/xml.
+	goto end
+)
+
+if "%1" == "pseudoxml" (
+	%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
+	goto end
+)
+
+:end
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 0000000..43649dc
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,8 @@
+sphinx==5.3.0
+# TODO(kmaehashi): Bump pydata-sphinx-theme when https://github.com/pydata/pydata-sphinx-theme/issues/1161 closed
+pydata_sphinx_theme==0.11.0
+sphinx-copybutton==0.5.1
+
+# These requirements must match with the Baseline API version documented in the Installation Guide.
+numpy==1.24.*
+scipy==1.9.*
diff --git a/docs/source/.gitignore b/docs/source/.gitignore
new file mode 100644
index 0000000..97b6f2b
--- /dev/null
+++ b/docs/source/.gitignore
@@ -0,0 +1 @@
+comparison_table.rst.inc
diff --git a/docs/source/_comparison_generator.py b/docs/source/_comparison_generator.py
new file mode 100644
index 0000000..0059c01
--- /dev/null
+++ b/docs/source/_comparison_generator.py
@@ -0,0 +1,297 @@
+import importlib
+import inspect
+
+import numpy
+
+
+_footnotes = {}
+
+
+def _get_functions(obj, exclude=None):
+    return set([
+        n for n, target in [(n, getattr(obj, n)) for n in dir(obj)]
+        if (
+            # not in exclude list
+            (exclude is None or n not in exclude)
+            # not module:
+            and not inspect.ismodule(target)
+            # not constant:
+            and not isinstance(target, (int, float, bool, str, numpy.bool_))
+            # not exceptions or warning classes:
+            and (not inspect.isclass(target) or
+                 not issubclass(target, (BaseException,)))
+            # not private/special method:
+            and not n.startswith('_')
+            )
+    ])
+
+
+def _import(mod, klass):
+    obj = importlib.import_module(mod)
+    if klass:
+        obj = getattr(obj, klass)
+        return obj, ':obj:`{}.{}.{{}}`'.format(mod, klass)
+    else:
+        # ufunc is not a function
+        return obj, ':obj:`{}.{{}}`'.format(mod)
+
+
+def _generate_comparison_rst(
+        base_mod, cupy_mod, base_type, klass, exclude_mod, exclude,
+        footnotes=None):
+    base_obj, base_fmt = _import(base_mod, klass)
+    base_funcs = _get_functions(base_obj, exclude)
+    cp_obj, cp_fmt = _import(cupy_mod, klass)
+    cp_funcs = _get_functions(cp_obj)
+
+    if exclude_mod:
+        exclude_obj, _ = _import(exclude_mod, klass)
+        exclude_funcs = _get_functions(exclude_obj)
+        base_funcs -= exclude_funcs
+        cp_funcs -= exclude_funcs
+
+    buf = []
+    buf += [
+        '.. csv-table::',
+        '   :header: {}, CuPy'.format(base_type),
+        '',
+    ]
+    for f in sorted(base_funcs):
+        footnote_id = None
+        if footnotes is not None and f in footnotes:
+            footnote_id = _footnotes.setdefault(
+                footnotes[f], f'f{len(_footnotes)}')
+
+        base_cell = base_fmt.format(f)
+        cp_cell = r'\-'
+        if f in cp_funcs:
+            cp_cell = cp_fmt.format(f)
+            if getattr(base_obj, f) is getattr(cp_obj, f):
+                cp_cell = '{} (*alias of* {})'.format(cp_cell, base_cell)
+        if footnote_id is not None:
+            cp_cell += f' [#{footnote_id}]_'
+        line = '   {}, {}'.format(base_cell, cp_cell)
+        buf.append(line)
+
+    buf += [
+        '',
+        '.. Summary:',
+        '   Number of NumPy functions: {}'.format(len(base_funcs)),
+        '   Number of functions covered by CuPy: {}'.format(
+            len(cp_funcs & base_funcs)),
+        '   CuPy specific functions:',
+    ] + [
+        '   - {}'.format(f) for f in sorted(cp_funcs - base_funcs)
+    ]
+    return buf
+
+
+def _section(
+        header, base_mod, cupy_mod,
+        base_type='NumPy', klass=None, exclude_mod=None, exclude=None,
+        footnotes=None, header_char='~'):
+    return [
+        header,
+        header_char * len(header),
+        '',
+    ] + _generate_comparison_rst(
+        base_mod, cupy_mod, base_type, klass, exclude_mod, exclude, footnotes
+    ) + [
+        '',
+    ]
+
+
+_deprecated = 'Not supported as it has been deprecated in NumPy.'
+_np_matrix = (
+    'Use of :class:`numpy.matrix` is discouraged in NumPy and thus'
+    ' we have no plan to add it to CuPy.')
+_np_poly1d = (
+    'Use of :class:`numpy.poly1d` is discouraged in NumPy and thus'
+    ' we have stopped adding functions with the interface.')
+_dtype_na = (
+    '`object` and string dtypes are not supported in GPU and thus'
+    ' left unimplemented in CuPy.')
+_dtype_time = (
+    '`datetime64` and `timedelta64` dtypes are currently unsupported.')
+_dtype_rec = (
+    'Structured arrays and record arrays are currently unsupported.')
+_float_error_state = (
+    'Floating point error handling depends on CPU-specific features'
+    ' which is not available in GPU.')
+_byte_order = 'Not supported as GPUs only support little-endian byte-encoding.'
+
+
+def generate():
+    buf = []
+
+    buf += [
+        'NumPy / CuPy APIs',
+        '-----------------',
+        '',
+    ]
+    buf += _section(
+        'Module-Level',
+        'numpy', 'cupy', exclude=[
+            'add_docstring',
+            'add_newdoc',
+            'add_newdoc_ufunc',
+            '_add_newdoc_ufunc',
+            'fastCopyAndTranspose',
+            'kernel_version',
+            'test',
+            'Tester',
+        ], footnotes={
+            'Datetime64': _deprecated,  # NumPy 1.20
+            'Uint64': _deprecated,  # NumPy 1.20
+            'mafromtxt': _deprecated,  # NumPy 1.17
+            'alen': _deprecated,  # NumPy 1.18
+            'asscalar': _deprecated,  # NumPy 1.16
+            'loads': _deprecated,  # NumPy 1.15
+            'ndfromtxt': _deprecated,  # NumPy 1.17
+            'set_numeric_ops': _deprecated,  # NumPy 1.16
+
+            'asmatrix': _np_matrix,
+            'bmat': _np_matrix,
+            'mat': _np_matrix,
+            'matrix': _np_matrix,
+
+            'poly': _np_poly1d,
+            'polyder': _np_poly1d,
+            'polydiv': _np_poly1d,
+            'polyint': _np_poly1d,
+
+            'Bytes0': _dtype_na,  # also deprecated in NumPy 1.20
+            'bytes0': _dtype_na,
+            'bytes_': _dtype_na,
+            'character': _dtype_na,
+            'chararray': _dtype_na,
+            'compare_chararrays': _dtype_na,
+            'flexible': _dtype_na,
+            'object0': _dtype_na,
+            'object_': _dtype_na,
+            'Str0': _dtype_na,  # also deprecated in NumPy 1.20
+            'str0': _dtype_na,
+            'str_': _dtype_na,
+            'string_': _dtype_na,
+            'unicode_': _dtype_na,
+            'void': _dtype_na,
+            'void0': _dtype_na,
+
+            'busday_count': _dtype_time,
+            'busday_offset': _dtype_time,
+            'busdaycalendar': _dtype_time,
+            'is_busday': _dtype_time,
+            'isnat': _dtype_time,
+            'datetime64': _dtype_time,
+            'datetime_as_string': _dtype_time,
+            'datetime_data': _dtype_time,
+            'timedelta64': _dtype_time,
+
+            'fromregex': _dtype_rec,
+            'recarray': _dtype_rec,
+            'recfromcsv': _dtype_rec,
+            'recfromtxt': _dtype_rec,
+            'record': _dtype_rec,
+
+            'errstate': _float_error_state,
+            'geterr': _float_error_state,
+            'geterrcall': _float_error_state,
+            'geterrobj': _float_error_state,
+            'seterr': _float_error_state,
+            'seterrcall': _float_error_state,
+            'seterrobj': _float_error_state,
+        })
+    buf += _section(
+        'Multi-Dimensional Array',
+        'numpy', 'cupy', klass='ndarray', footnotes={
+            'tostring': _deprecated,
+
+            'byteswap': _byte_order,
+            'newbyteorder': _byte_order,
+        })
+    buf += _section(
+        'Linear Algebra',
+        'numpy.linalg', 'cupy.linalg', exclude=['test'])
+    buf += _section(
+        'Discrete Fourier Transform',
+        'numpy.fft', 'cupy.fft', exclude=['test'])
+    buf += _section(
+        'Random Sampling',
+        'numpy.random', 'cupy.random', exclude=['test'])
+    buf += _section(
+        'Polynomials',
+        'numpy.polynomial', 'cupy.polynomial', exclude=['test'])
+    buf += _section(
+        'Power Series',
+        'numpy.polynomial.polynomial', 'cupy.polynomial.polynomial',
+        header_char='"')
+    buf += _section(
+        'Polyutils',
+        'numpy.polynomial.polyutils', 'cupy.polynomial.polyutils',
+        header_char='"')
+
+    buf += [
+        'SciPy / CuPy APIs',
+        '-----------------',
+        '',
+    ]
+    buf += _section(
+        'Discrete Fourier Transform',
+        'scipy.fft', 'cupyx.scipy.fft', 'SciPy', exclude=['test'])
+    buf += _section(
+        'Legacy Discrete Fourier Transform',
+        'scipy.fftpack', 'cupyx.scipy.fftpack', 'SciPy', exclude=['test'])
+    buf += _section(
+        'Interpolation',
+        'scipy.interpolate', 'cupyx.scipy.interpolate', 'SciPy',
+        exclude=['test'])
+    buf += _section(
+        'Advanced Linear Algebra',
+        'scipy.linalg', 'cupyx.scipy.linalg', 'SciPy',
+        exclude_mod='numpy.linalg', exclude=['test'])
+    buf += _section(
+        'Multidimensional Image Processing',
+        'scipy.ndimage', 'cupyx.scipy.ndimage', 'SciPy', exclude=['test'])
+    buf += _section(
+        'Signal processing',
+        'scipy.signal', 'cupyx.scipy.signal', 'SciPy', exclude=['test'])
+    buf += _section(
+        'Sparse Matrices',
+        'scipy.sparse', 'cupyx.scipy.sparse', 'SciPy', exclude=['test'])
+    buf += _section(
+        'Sparse Linear Algebra',
+        'scipy.sparse.linalg', 'cupyx.scipy.sparse.linalg', 'SciPy',
+        exclude=['test'])
+    buf += _section(
+        'Compressed sparse graph routines',
+        'scipy.sparse.csgraph', 'cupyx.scipy.sparse.csgraph', 'SciPy',
+        exclude=['test'])
+    buf += _section(
+        'Special Functions',
+        'scipy.special', 'cupyx.scipy.special', 'SciPy',
+        exclude=['test'])
+    buf += _section(
+        'Statistical Functions',
+        'scipy.stats', 'cupyx.scipy.stats', 'SciPy',
+        exclude=['test'])
+
+    # numpy.array_api is not ready yet...
+    #    buf += [
+    #        'NumPy / CuPy Array APIs',
+    #        '-----------------------',
+    #        '',
+    #    ]
+    #    buf += _section(
+    #        'Python array API compliance',
+    #        'numpy.array_api', 'cupy.array_api', 'NumPy')
+
+    buf += [
+        '',
+        '.. rubric:: Footnotes',
+        '',
+    ] + [
+        f'.. [#{footnote_id}] {footnote}'
+        for footnote, footnote_id in _footnotes.items()
+    ]
+
+    return '\n'.join(buf)
diff --git a/docs/source/_static/favicon.ico b/docs/source/_static/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..0c81c0bc3b02f9f339993546159fe108d8816317
GIT binary patch
literal 203401
zcmXVX1yJ1F`}N}PEG~<?ySuv-XmKg-6nA%bw^F1y#kDvThoXxuTHIYe-rxVrOeXsz
zGdIc2WA~i1000yK0r1}j1pom;W1#>ZAHT)L|IenE1OOgC&IJYie;W}BKrsacU}yh7
zyMhJ)P*{WpfI$D>#smN^pP>N+1pjA8Apro{s4xJ`kMsYHZw(6oP~gD;qSaO9(2$9c
zKW>GlATO=?-_`$iO!$ujD&*V8ut5L?X$ftw)suWrFQCj)Utp83lB1(fD0LW-{Gp~a
zEJblNeHg8bIIczvND@=po?Av8I;I4~s3$J{MN%Usj4~uh{yZO3Qr@Zc%rB6dqN}H{
z4oxNG{$5x#*4gTQ;^br^+igD&tIc`4KRmW~Q0Ux#eYF%pFE#v^WsqZneN7KK=NE&N
z^dQkZ%z<EY?LkCs>R4eADbvAkv{&>I_IhEP^U|NYXS-Tke}be4D&s^?7;4R>P~IbQ
zc2kU7q41_k7bj`(qpKYT@ZscY@jvlL*PO#?R!EI=c;4->E^XOVGr6r7&{+*e@j7I%
zv7IiuYUs?Qw*XiT+MG3XveTP*dXN8r1fymtQQF4EPx)tbz9s#Eg3+19Ao({pzl~l+
z*KSKI|0$cXJD7O|zW5LPjr_S@b%lB{(&vXAjXjN{ZHebEILvgXIgY;4cy?pr#@9T;
zP`-Q9$-5B53PnVsxoa8Z8TDHQd2Q+~BkBrZG&;W&i?UYgZ0hg$@|%UC4(q%E^O~w(
zjr&_VH+D)ktv_QMgXtZhKAvwIQV6LTTGJo&LvU@#qUnKZE_G`qfnK&5?G6f6r-Z`j
zLW0^N;Vt5%eT2VY#vihE$wQD~6PPRILdTsQd8anI;s*|}Vo%w`ZFf4;6~?Vbe}|~b
z*5qW(mez4X6AMbqV<Kb!b0cSj9;(MeLpxQ|Rl+kw8F53{aWNYfV@-v2iOIm(*fr|e
z&RLJFGBC%twV$j|xg>VUxO;Zi=xD<Q;IkTlFDKdqpL1B<Ft3Ok;!VC!xGVzagGrx=
z2+8`eI@j;>D#cc`%QKvO8FwVYlUVddsXAB5T;)p?YrcA!Qi)u_F=r{%-625(TNRO^
zQJ#XH)m=c7ghApv!(s+1a<iXw9}4%_S71#Dg>?6A3&*vtMR6zp#!H68f#fVHhZhqC
zvy<Glf}Jnf2Fd{cO<97KN4EolnGW=hbx`K2c8&#h;f?SVP_@La7kkE~^a+_-&|+B{
zmhq`bCkW@?3XU`7!U~`_i}l^@){vK;z9WadXe8-KhI$27WHEG}<ok8RkXdkBH7N{-
z3Otj8$&J)3+YldvLxbVVd|H|6aep`pejrK{zT<uLKd4oCE7Z>*QT%yC|A&BD=ir;f
zpGC#RUaa&E0#01sxQz6cxZnnId|1qhbh)Z)tjauoFVzk4(P{Q*2}-%oiW2tr+6y^6
zWh^bRfyiaQ;QlXW@`<z$_$6&h{9?!}{Jr93jMYFv(5|!x#n8(_?<Lya_sO{oM!NzG
z;>P+VJJ>u{)20Hzf0$3<UT8lb;-If~f!;{h-xV}fbF^2@Iv>iNKLb8o>$1+&qj`|r
zT(&rssUkQeRH+26*io%XEnu{v4dY@+;>no~dJj)4@H6-IJJy?i0FZ8YdY!{LeXHE~
z2V>@PJ{OkL-5(kPbA9PLWyFy$t0A2az8XsU|2ah4L-_-*+iWD=eUV80_T>~-I>&5M
z=fmtJ%o@MqlMxIMsb6Cng-Q^W=Nw8rZ7sSt%j~f^_ni%k|CnbGjVg23gd>>TqV*J+
z)ay&!9Is~_fkl)T%I#WB>|K7GrvrAEmP`Bpy~4m{TvB5YG?toDa%{F*dnD^p4fl8K
zgMzil2&7wQC+gPP+-oKMvtId2FDItz^to(k)GRaov29`6MRZY$7J)?m*I-y^{zx~S
zcn5Ni70ojs2){Q(3F>3shk6rSc1lXHvHLWF=}15CGWg)x;Rh90gSM8fs_9@hIo2CY
zq&hnaBzcPXPgRX<{=DOJ;2K`Lloauu_C@uXkEb>kJQewz{x&epEVqHaa^#3(-mUhJ
zR_e58{qP;IqXBXd3O?&wbDTa&_t$lqq<`S@Wz2=&p`oBJ$O@Y*uk2g#&x6JC+!l`c
zC1(32b|0TPlb#z}NKsv1)g@c7DObVMvc6V9Y7gokjZUa*;d0%BiQmD_>A9F!*RLTD
zmrmzwkN;9gNff`i*<aR3bsIUHqaA9gfC-7OP@#jW9QKeeR~$R{M<m?h<v(?_tNFha
zb)8qH$gpArOaE@TMqan>ONOfS8<!<AuvZ4!y24@TSG`7X<bjcJ1iv}wv#1BS{Lhm<
z?x1IP6CMW7Um1o$1TPpHYWAHsOcmmi0{KR8A?bGoW|N(QMEAZpW#Gs(&%hk{r^}m&
zVL7RqlZz$orv|5+=g}--R*KmehFAU?a=%=1X82!nzC<y@mkvxTG8=&5tA(hhC{xn~
z($rLrXJ?8IB!mPE4D{7_EL=HiuteiI<H?{})3@CUD)0zmy^Mboh??IeK<6y5T_`hI
z)Lld6P%C0{_-*(i@PCtT@Pa%?fm|WiBM%B?$+|Sh$gj<9a0~5QCMB3vM;CsJHQRgr
z)eZ6fZG;Q$9cFvjkl>}%HMstZkYiO;Q&<M0WnQP263!Ql6YXFYVtB3KUD{<=&1#T_
zvhi}{Tqh?ijwIAhf64=*at>X~>_Z}qVFPK*QVb`=JK~+kbh5g2?PWfwAMg~3l(WpD
z!LPIKRWA})M@JLaY)CnWA{U<b2jTedO~=OU;Tei3>c92MkF7xKP*a+7cnDWsq~*)%
zwKYN*Hg<8c0n0L76$lMt(q+S)yeJvo|9ic_5QWqPveUmPe$(aLja^8qw(dck7`wLK
znrBcchq4$ZY|R1~VOxI~`bJHeLAGPD(m_|v*0Hb$gqQ4pgNZovzo9l1O8>lcGGRLP
z>3C3s+|f8Dc*9d1CsS;SGb-^jScY)Vr(!(Ay5{2^`CkZn^zB|#1FkZVe)*;9a+_y0
zfUpj%8Zm@g0!k5gC|4a*UBytcBBst*t!&S?2=Lc`Os-BV>f7V@M*J6O5IoBo@SWm>
zfC*1xEGx*q>jx>|Xq=*i)kJa*c2pt-4mLFgHltAk9Ufbcb@69nL&W96y}_8HZ3^}U
zCFW>C{kNBjIM0W2{`%s@DY{ggigG-4t*)^+3^r%Io-zF?q>4Bb9L1CV!}Ux2_kM*-
z);By$8rrE-R~Q-leTrFhmC9o7%<C|TKXVx|y2BPw$ix5YN+@OsEpdUJw{06&aowY`
zWg`g@Emoul@)V`U?(lO#hi>kJ&w1s@WEnnHt!`-TY+W-|e>>d&OWVc}O@IM-wrkc}
zZO1VibDw`{BLl#YA0l+*6G9>_;wNs81#HJ>A9a6#_CS>{!p>}mZ!B;uBk6O!EHlr^
zvLlN@Csui3F_5C+;*KVIP;Vdh_G^&X6>t@h`@%vBcc@-9HI#rZC(j8D%soejg&ukZ
zL8BdpN+;%|#)cDd6D-Pf(fBleQz!w`5vQOkvUKfDv#%10h!o4(wEl=S#=ci65&1#t
z)72)6?(rhuZ)yLhb-D6YHZYHWjJUh!@W2?~Fc#P$G-72Wj=uO%yD%(0lX`3Yn}bu+
zxOKncIse~<<fKmQ(kY_@h2aZ!CiGT`0Y~6lkH1ue_*C$TtwN3Xm}bd>dJ%l*X8>${
zL_kpTP8Y(idZL{;K?-IOWAPkvBep4|-ZX7l9BBItOR*f}d2w$*g)7yJbXdPb$U({n
z9;<jHBKo{MuqihbBU+8`f%~hE;}Dacj8{B_Kb|?J?9D$dD*|{^kx5oW*qr=Y#STEZ
z_v`J$s^Fy{&%adajJ$$Ru=?37(!FZ5>hybvE&3UBdCJPffhGA-lb~5KC)hXnX5|l7
zBRgkD`&X~+#_cMKG`V&80mUME4}2@+6aYJ0dLQl^EAFeYE%0%6KMqF5$xZVnjU7Mw
zz(a;ZRn58aYP$2K(s(Ds_z>yTNk=sK2=_n~2DE3j_pK{eT|NSh+)8-?j3oc4#1^xa
zb@`pF#MVVxg~bl4;(?%OR6U=3nP9*+Li@z0;>1Hn5`(_dMIF@d#h+fWNYA2Taye3?
z*PlTE!VYR`Z@%l_#}Alj!LZuYA*F%N{qHnGqNYsP2vg&m&QYP5d<E9jaEZc|*Y0Qg
zyY1mqOX+{U7*1RtoVyXv#dQ@+<?<Rjm**Q9BM&yK`cYd#$u~s-*@CT^GRMe+|NgS%
zn2E>CfUfciItF}6(2Is1|I;Y+#om5F>5@>qXWOy03^&RC=U0J5gvd7LC(F1}_-6J*
zUu=<Q_V9=jK%+NyxPs?+h*FWW*;1(-9`jHHPO_s+B+yWkU@PqU6UUa!*_ePqUuF`H
z(5$I+Ub`e3^NAwS6rXmn$`futilku8><_XZ_#$V}2P<7zpA8ioJXqi8SmF9)R)&eu
zP;?l##0iL#W04GZ*sm%T!gErX@8*gZzX*@gwpv<|Xq(9JU|e})-p>HFrPb=42tq7e
zRaBg41!pOOjKo=yyJs-_a^Ejc0v&5CyDDXOOF%i3)Ffj=e0i`b%J1d-spHdKnPvM9
zlQsUQQs9{J^n2%~wJH7B*4C&;^j%&K84?7Z`D})6308c60(gkIcV0%27=srSeiC(1
zo7!25XrI7-Y!k%)=2uG2XAJsf7<?!)J!YT+PTkmfYA@}l#VxzICM&1))mzeP^Wk6C
z4SoU>pWh98Yh%aUJ#-GN#}0=A&VZ|r;q<Qf#ihQk+2$2WeheUt^CLnTv&|{%;QoN`
zQjztW@#4Re9_a<CJ1p2Zos)cLGySRwR5v4|s|Zwww8CQygi?R&XsWaNt5F4juAT@q
zM}HVO!jSDEp$lC%G{#zhB28|%Kk6mAGcucS(6e{iXA`7WAnpR7+F(oB<{tSY4gZ1L
zDpBUrryd?shV56s<Y^xp15XyTlGBpXwUmoWF<h083qfw~^!*2*DKe9!fPNd`2?H0l
ze{3|@A%3CS&wRhq{l}#7*DB&U-UcVrcS%R$2!%X{2ZL<NYFZx1NSYT4u1L1RRSRHz
zMm5Z!ox_g4LxC<~C?5)-FWzRg)~;zSRQ?cpsL{=9if$pQypCW>ikcSNoE$K&d=i<(
z0&I7>lmgsXV7v)NYXZ=zkRLzomq1|3@wS55&eSIQDTzR=-4X;LCKC;YrJRt13=(Wy
z{7eF16F|Dic|5h~Q<BkmB#1McEwtl&ixD5uI4dCnC)9Yti8l<Ie$^dzYl1KYifecw
zXPI8ys9+k6dT}0QHaXaXGoJ|7T*$wGQM+s^{QMFW+qYXzC;`%M%!6pm#+BX$YmSaB
zK>M|40!+y9u7U+mXtDVv*eO73?0NVSU9xBaLCd7GxjW|!ryJXlI?cBTdHi=qOI<}j
z-u(bnSKJ!;5n0;!N~1j`ztO%P3tUETYZgsR>#u{~eT3-mu8vU*(K8a-ip>aw7J0#l
zP7}x0%ciIOO5$1IY5%BYB8RFGF|B}?KTR!8BKTB6oevqoAlqpADuA1euVfIHD+d!m
z4S5EJ7CZ|B)x|ghjXxx^u#iZj*@73^wYkNQMXifzKpCa@Dbt)9Y4eLO;+CMTH1}g$
z@)tK&%WoCw#6<%!L|$yX+}w$0;Ydyk?|+?;wIFR8K}+}u7bWmDk#$96?FJLxD$*`F
z@dU#bGWQAZ4b*;ace-mG9{&>itpyS}tdCM*tsmoDwfBArzJ2zbQ=oEQHS0^T^(TrZ
zd;aTY4Qs}~7NpEMCL$WNimQISj0kRU`?mlBTT6w1XC}!5ahhAJMpXav;}n~@HgZuV
z{Y}`zwrcqW3xg2=Y`{Y85_ew=^-Hyp4n9gXq;n!%7DK5dUq&77Uf{k`(p}!@h5xTS
zDdXU`I_N8-Zk4Y%n@J<$IO-{2sbMbKqgb{l#VTQ#4MjIS5g<Q*yHuQbIVd!sr56p{
zAQj==jn}PrgFVJ^kjF-O5G`hL_$t!_;~Jjtj=4s6^!&i^-c{tXj=dR^jdZiePuLew
za9{^g`&A@_sSHI7^%Jbdzcs>hB93#3$(yv{ilvo~=wcMNi;&1fl0x3ZyamT>Q0llW
zS*=aiG|Vb_AuG+>k3$95Jhej&XLTE_H%r0f7Zec#xu)6@=lCZ1QYJz)1h&oA(D=p2
zZqnzu>Sl(gYt31f`GTs}<63K(eaO0_VRj(9NAr*t<}_@mlt*YTKEm{{6B=5|E#XG2
zYBeFf$wh)7#PM|!n$bnT2E}g3G{Ry76IeFh?Gfu2L@O+}y^%^%#ObMpR__|qA#SW%
z4>*z9Rcqv-H#W1Nj8n4V5-6S%+axzg!}#biV_hY{iEBoQ(NHe(5lK+1G7T}pn4k8j
zyhLBH_z-`;4ltuyANmjDqCE|bG!rTP*TF_hQq`LacPD}vljrIHbh<8^OJ9!ZiDL^n
z2o@`iS6j`9Z>5j7kL}*wpoe)BNif~W4kzGT+sS{E5JjuW<DU{Z*hYvEA@v-q@0xxI
zYZ)3=!`1#7j~@Q=<=A<ERBKS9>oWYPv)+72lYtBY?}8ZWu<ylXmMp<Z>7NL0-3eL+
z;*B?uU+EKL+$iw+pG^MHV`Des0gd=x*9!?2G5M{O<N-F2x>g_3n3a=-NC_Z@gH4ig
z=o80#18DDYUgYcbu_rlvX2A$uO<1Xl7zLNeSe!|y<}CbBF*9Rm4v7`|pR*ZoJZBGn
z;ozWmC9_+J+Rv(+az_W6cKn?Bee~A_Z^FVPkOd)Fd>$CNvd&Nog@z=L`!Yk-GmTq{
zG+-|NwLMP@)@alei_l<jV6pGaeBs?~q<J`g6_11q=AsY5=~`ZLk1|8-9`VuItGO~h
zgv|F)N}jnQSQm5o_YO8F$<~`VT9KJc^(F~7B-+yYmHhW;B)FO7tSa)nb*xY#eE+jU
z=s{nbZB1ePxzPs{;CxvK7Ko-dcwr*o;(OBe6VCCI<xUxyIo1(0)=;_PWCZGQwn7p4
zprQy#C1d<b`dGt(A4UVi`>pMvEKDOaL|r*YMza_mf~i}cnfJ)@^5~Y-<%}nNAmqv-
zYe|gN2=J9yH&cJaBZs47M_fi^1Devs(G?$9aY^LQmBOv2GCQMt3GPdN!T}K|Zx}rS
zBnMS!aM(O#su7VUtbqo^-_)#8R^lHg`C1+Q4~ysU*1}`o2R|?#`pmd|ob*g}QlWsC
zApEmKj=6FZ6v=N}4@_1K>R1D-^xS9PcfQ2VIx_O9#vS@7KqFDS?6nNmM{h^K9@U47
zBU}EAHg+F-K}fLUG2Nn%sW{bBHSEEp8-!hX)Ixysg1pMIGa^@A9|ry{_1lqEcu^27
zX%Nz_jPnFjti<TR!hcGc%!b<iM;leQkdzw}ih}<Y`hT5pDfeIDWtASk80IPSeWgdD
z{RNmoxAcjI$Qk;+_V@5yrUS|9z_;kHA@9PqpOWH%q%Bw}@!u#+2XSw6CT5Qs67eid
zhed@bTqNQy&!0EUpJf~pdxM7!4T>q@A9D<-s;mdBX>&LjYuO6)wqxOYms50^)x29L
z<SjJLktL*jom0GWxu;YfRhhqboLH>VHB}CAT<y$Z<CtS}q_y<18U%f;xHP5+MLHbT
z9=v9{>$(_U$FzFfM9Y4L771C-seaW267SDuV(~W^`qfqFUH<3;i#>xmB`?Q&0MX(^
zOeQ15>wk>--AgiDFSh|T#aR?R&?WySwX9v1S{$NQp6@?P4xhx<;spYTM`SSHvIN)y
zEqxu)c}DvkAaLNM!&cyMcc$|5-a4lib1zPFg%K_41Nxg}PyaO}o}+0{=^Nf>MX9No
zFs)_CoueJ;b+z4#`v=S@SQIN1Vnx^T#%;=F{rB$*I-3a{N;~WVcIcz^0lv1(boQmM
zI_5H$7$c)jo_9@`)dJJq?SYvL$luNtnrl`6K8EayYO&=-wbp{*vdGW$cEYo-+J95!
z$m8ERj=cR+PrBXM4(Y7UMlA8>OvbuGx<-XufW3K~7fAK>%CD_U?6$r!^vkzqGr7o;
zTxni1FXcAoV>cO{e@pVbspOY1TSas{Gms~Tz@B5FJtHVr4ON#92;+?2%t^WzTMR+L
zFmAV`Uh8de0MwPPo7L?{iZ?e}hzBdry;f3UHdN{U$5&P)v7~ZPQIN4@{8NHi-_@N`
zo;hs1-R*1@0a2$|D6Z~bE>*P&7Z?53;TI3G4zY<W4u@50YQXiL{8vuAmKXz3z|l;2
zshA2auE?V~u^H^Ih3b!?y@{{h7o`5658?d~=I%Rc4`oIr*r<=1mSF;~?K+G5Gy<gI
zzB0azwjH!O9Qk#Bz($u&|DmXDRMGxaK8{_aXc)5uVmFQA`U6DSQ{BrIH)rtQ_}RPP
z8HhmM%@ukszr?*a^K??~`yzmHspqrXeY`)hz5V1S#i?{wv{301?fQc+&)4$s*0`V2
zd7{P9$hHl`4yRNL;MeX~=|`0821cbLmwx)YYz2it80z)N)(U-Xvy0V>v2Z~RJE(?=
zMBdbs0)uEdnq_CYb8DB`UGU0jhUtYaiLpw`C#%i>d*I(+eOxc=K#tOoWC9@0KY|K*
zC_*r3`JSN01@gLW-nKp1eUR9ffAc9VFGk&<+@AXJ{C<yr&6-3gpEzr*QJW40VY4`K
zo%mJq#pbOACeo@^=>_ydNN^<{ml}N~`QPrpTZml`wNnfhRnJ3wZ=84uy{OEP<K&>H
zRgsj*dYTOOfs5!J3(Lho1~}6HWSe8c{D-tkEV*(4CS>YgJnJ}#=M=(T|ImA%`kISM
zd~uMlM{+a*mfuKtoM0YJ;rz4;MIkgv=h3TKrjK_kEUp?^LT)oB6}!kIvnS4EvXycA
z^WOXkc44UtpV9J84y{q5&HH^kig}APMyzpoXt;JMfbSAD_o88KtP}=r9*BEGkP-0N
z%l>TA%v1H3I0t$jfshxDGO$W@lY$mohAlDF8Z1TT-4xpD5aV48wyOW8G5nvuz5(M%
zR(f8mK7oi2aSZ|O5JcZycGRqk;_Yx%l09gF={reEj=Wm&?cL8W{%QA<Hbb>&OpKtL
zd%QD`tYvv&IuDJV<Z-hgUcG%7D;f8y*Vh=eGW@zu!tVMa>`{q{2e%lzl@)p06d?*Y
zV!@MW@{7W|tg9X()@VHXGks`@)lv_E?CY0{uH<QHF;{m_jwHtKX7NH<4{pkdxw$BY
zZEZFG5f1N+!>XN5TjQ}H3~UB;(^MV-1rX){+RL2^B^tS(86U`{XybyB(c{2gbv{O?
z*yrXDylC1Vzfiuur$Lj^&C{Pni~olruou+rdw<sLCoofBSHkPvuX|&LK${Z-3aM4@
zmr$CMwKA7Uyk1s)bGtR(*p#{qa~we8PsiX_!+UHX6sFN_aG2q}i@N&-tK*zbo#%2E
zP1@<F@gIEr74@SOK#Ubm;t%<Db|eNhf`1}XrxT#8qTZG?KdsVgQ$~*0`2~whHG~0y
zOdEN3t&gLDz(?Zu63V2h?eM_y2a<Yo!FDvhu8Y-WJRrt|TQ6NG_imIULB(i%BqzIO
z_IxPpy1?ZAcN*a_QJweDa|&KTpGl$_?iKwXS|Bx)aOy2G(c-(`2O6VPUngX7m80y2
z=w|SHBd;8@8B`I|5@V#@oDvX<a`7aT9%*BRK2b*4LrbWZ(Soe<mHen^EB*tI%eOvY
zte2nB-303#Z^YD=DIBF*8*WDIjN$~oUY6Sy%KR68zO=|hqvbI%K)S8R;~!VXxyw9&
z_J#{^I|<W9mO9%8T*Yp{J5q|UpXhNYHB4eA8muYh0T!K+w(O~M1n$&qhYGgrYy^Ji
ze!adZqi#Y6hEtVkc54JQE~DccQZSq<-)|h^&lx)Xsi>~+kIL@m2y(pzGpK`_kI3Cv
zdDuduP@kom#x9A=Av7h3d2QsuF%yQy+SSXPS{qzK86QZwh^;zjFsx}ob3s#B^6U&_
z&YT)ERI<v~vjJ{jk6DftR)ZHf16mq<R!0g)S%?+~#+EF<<6*;0Fv*c`WPRF*z*8`W
zH)5WDqNW$^=D8dw3D@#cx0*#meTK7RJ6On3*VlAF^2r-Pfc{Vo0~F2;R{eo3V<@aT
zT@D4b@uJD8lTY@a@7810MPa!&r)6TprXb%AdC7<bFp0Lo2%MB+nLsqi0K$X(u6;F%
z``7#2^-i~frehISR?Rj|RZcdoPw&@cqN|HkKj0YQs)vG8X~9$o^kd8!PEqS@v|5C8
z8M<p`k^{w-xD1+8C6K*MD<fe}JNc97`%<YSk*yDwfXpf-q@ZfyH;E8khsw*y-Uc?L
zB#r)jq_o4?MW5r@3$;d%P-)`~%(H~eZvLGnv#}Oi!ce%NZju9io3w)r1zGVYLhJ&-
znA*ppHUtOZ=$6;ZY~ZvXwlQh9&>)oD^g9{afGm2msuOLhQeOAT7$!$nr)Ce{n`>`g
zj=x5&RkR*New)By8SQXXat=yJLMA#ENFpAkH-Si&oV@DQK%~dASlYK7KI>H7-dn+0
z2z3G5@UAf=P_{nY8@c>jgK5WeJMLLnV;CG7TppImUk3LF3xZ-fnZr;!rV)qe3s1!m
zRW1%n=A(N{xfr=pbV0)v$M431*}Bw$(H0&hAd7;MA6I5^;0TOgiKx*xZwbzWp@z0&
zIPoTjLILr%spLKwm9dn?UOfkY5Mq4T__`KmuhO)(^UkHfX~uyD_3*-r-k%7JqPzDm
z=qT8?eu2mp@k?WR2`)h>nT$<Q<`ZqnOsYQ%=PUbk9bM7O#J2h}am5o;Nld~B<Z=3_
zTgM2+xn?zBgOfP5iJ`AF2G)LDV-l~G3|;M&tuvO5FlV>1T8f;Nz<*kn5G6%2H&jQ5
zI-#)8zzZN?_du4>ObCT*hNq`LY>X^6lly)nJ%m3$gj#KFEjO|2g52s^x1)xbaIIn<
zDJM4)_;jztYd$=c&%t4mW1VV-!0%HzZ^1ZIpq&j({b#)deQOB*2}TgC0DmQoF&B(r
zBkc}SzAHg?&P#YG8str^1&VcmBiAzY|3@LbhqetwJGs}uVqq8^yW7L%*?yq_X~GGp
zf5S%VL;B%NG*h9<=$uD=(kGd~3Dg^f+7E53u!Gz{q1&y8kq-OKuR5K+%+(q`n%<q9
zu62Hc+L%4lvNg}J8%;?crAl7+)9NHb2RE<@Aj&IydgB3^K3LZ`u+}^55JM)in#~JR
z^k6c1G;CU&hkTwFt$#lnvqk=Th}fI+T%0T~WS5uGwB%^T5H?xqDcz6+2ZKwb6zG<3
z^o8z^OBMZQR9&!^HVx+eveK*(QgEOsqnS=$%S#puXcMbu8NH<2UR4FtfA<?6^nHq?
zOc4^TUywn9GhxN=HBLN!nnvr=-cgx$4X@{^fmG=N#Mc<q32+5RLX1U<K3x=XUnLAm
z_P}w{MSK*_r%|}C>1{t$WQ03gZl{_PGmZb&bo~eaw>^+{x!e0r)}{i8Pumg4?XDzF
z(b{z8jcVqa)GJFUUM2D3i2iomMxd3koaX<@O{8h@tw28f`HyuM4S{9p^7|5hpR!wy
zj8h*1%qP+Fguw)=UwTmVcgeYdBUEpJ_rc6Lt7wNa_F^V-QWo$KYI3P7;R(cr*vhtT
zxjUP90Q99;a~f)`E93^Qb}M3OPs__W+O*Q$f=ps}&mIVWSFcSR)NhytP!p~Mgt?$G
z)#rM2A;ZQXeSf<nOW;E|wI;mAvNn&xN1y2}y-msuG63)SiWm9m`vI&g@?b*7sA0T<
zCYL30f5u^|B#(cZWbM6gOQijdLt;|a5e1@VPKCK5g5zP*$VRb{Ef_$fYflCy)9>m`
z16(hWTd1)3yGaqLb9@L%k)N0@!Lb+;i8<dB?z2<8de?Aod3?3%6TY=9L`vCr9^zbX
zrhBkG-~U&PKExVlSv#$#<x}ohbA|j3_?7-ew^lGwq?F2CB3tId@<Ub|HrlXWJpoD)
zBkyF_VSa7wV6JkY<Aa%!Kq48!lhU6CSJFBzs<8o-=dqeFjm+b6oOVf~eU$imY_-tc
zp}4{RiW9lY_;H43Q6G(H8FBu<0!X+feTZ4@{p=<ohY7G7R1tN-QjLuh!ONd#-EbB&
zqwapUtMHh&h%36e@$A`LDA&u`6!$Sa0;XY>_{W*Wsj@7C5K^v#Fjg!=4(tA=7KV}(
zsSextp*aaCFkkrR3x6{}eggz*wrTTVh)LEjBKIr8Lt#l|jh2JD1bw(G3}9dcZ{7;c
z2ZHC#nUPhwr{0S*X#p78%m}0g1B$A>1JvfS0*hH93+cw&Q-A=?h*V<|vLnd((Eo%G
zC17k%GySHl&-B!zt)tTB@Y9#YbBWu7vo0YaP~lw~xf~^QV$H-pt{43GaDu`OB>F+&
z%w35irqYl=TT&JoD-R<h7Wmod(O)BTxeuZ)5>^~f@MV0pK0xbXH03@4aFRIGu{A(A
zc$~g3euES3YIy%3_ds%as0e3iv41C8MTqV#aVK-9!>PCP*iXA<KH1mhs-9OO0paKx
zr@ikhE_85gBiUP@`t~HIKh<4RT(r@(^OjsLa_(sV=ptj?wiFP#KTw(?GN!o)hPr**
zVRm%5Y1|Bu2lUgJarPn9fT|=$uns(~R<DluJX`OOGyLVmI`MHf1>~11x1}O_%AV-W
zQ6=+&x%3$^kgs;weY;<#ozC$aBrmI$K)x>ljyTgsF`*J6zAZ6}5vP(aL@!q-eZv)S
z%8B5d6?)PX7@D6s1NDZB3E;+j<%c$-Hbg>rth84kSFhza-f=W3s+d3|@UJ;Ndq{89
zcq>?A^2|r`QzAaoDcHt3b$B%Ui>|#VtoYZU>te7s0xp_Vsiu>|WT<o>>#AMV=<i+-
znfZ_gVA;P50GwQo>h+a;H&4?ujcI+gA-FJp;qR9A^hqQc<#+_*yRMm<@F@_O3rUNS
zi`9S+yy=_L@@qdN3gjjp;*iBGb5PUZQV9ZkD$1%_P`>so1|kI3J@|=00cL+(ZP!~F
zBIh1Xkr~1<^2NUq@;p}?9=oQJm-C=FLDi;JC`&>|F19)cte|0RLONu4p2Os41GmHH
z$UIA*NO=*7@Y#7%M4~&aa#IxJ?iO@9C@d`ja=lcm7raYt?0)%Zk<jBhn=7lq7IjP(
zJ7eFLk&FmB+x5T)$EY5MLb099&kySH7OU9O@NG*(m}CE38Fcy9&{U#3XoXu~vJJW0
z{s`?fB*+&!QS2swh4HL{Jv@iblY$7lo-pGkRM#F27Lqn)s!q`G=R9$0{bl$C+*c$G
zylbA<%F|-Vz*i#Jg;Ffw=qT9;?PDW}t2s{k=LwGqPK7vK-XuN`d+oG8ONS-DvcGO_
zue%FX5!8i`skvY?FC`oN1JP<G6+W9#ksE673E2;}01eT?vw2Yjchq<z^!OlYh0^aB
zI_&bD#mz7U(v4(T$7HVolAEqIFKG;C)n~gU6BqmILIu%}BzZMy(j17WLL~QN1%0@W
zoCrZo;!>b;YtQB_VVxpE-$$hH2xIm*u|sAYYERz6iAF}iVkq~Dpm^2dfn<wfZIN(N
z;>LaS7Z;Fij~Xj4a@A`XI$cElv<sxq|Gu2xWx#2fpV55#Bpze*fK!frNEC0K>Fs6G
znFF6yF?^+|)$w!{xXbtzl@~mkd_LZxR}r?Wp1RFF9C4|n)}?{9%@q^#ntT}1sO&;e
zD#S>{c?<u!Et%(mMW^&2JB-kbGk_CCub;$)Y?wK|fcX83O|0SDf2+X&{8U;60UK<u
zaC?W|I{X<!PaOrAf0byIaD%uU;Riv`?TH!UDw?A(j7bqVX^yHtPmosOn@h7Ce4>;p
z7H-#+S!J(65K`1c1^xy|y2jUk0bbIgS=D^vzo0A|J_sM~uP>(Oce&O_;qMd1i<XWV
zNXwEAY!{?I+2JM8+x#gDcQ<}ea9tvJAA^-4WmbRK2YX{aCB3p!NFOd0V9?eU-sT<M
zBxnqiEkor3pQZqjxWpRG+%3;Ly964AL0)HJ^OiOIr#@PXyJX`3A<#y%OLoJgY=haB
z3z%x^KQ~QyO?DSHOeZp{gu{)V6%FZn`rix=bb2mtmGn=pVwIpCGP#y~4+cfKD6C>=
zC~O4U4ClzN=Y#Vo>eLAb)muQpmcNEk>GSA-{J)(8-~}RV27emI+)K0hc(G8cot1u_
z`Bs$_*?dGjS1Bz~W1{Wq91D&j|GaGH9}yE$;acK6j+Pfa7`?v&491bkhXbRtSv5?n
zM@maI@eQHy#o^#r`rH!k9aaZSB}}caVtQxY+HYy4E&rb7xD@N;Ol_w(wA`b1jQtmn
zX*@$jteO2&nBf?y^e%>1{YBW3RyX<3XwAT@CyceHVmaRxF9e<*5xJCqfxKsz^OFlI
zfhsK1rD*0DmTGc%WNp>lnj_s(df?9VM>xLiP53=@Zjh294r#^d^h#D?C6spfiWAqJ
zOY)f;UcwSbU;HUW!@$H7Oi-?M<M5+VrDkxSt4~oYTSfK(HGR>d_5e$gNko#q_>>k?
z#xFUo-0KtR#nd_mq4Qk(l1Oin5XC2Z5&7+H#dsMez>3mlv{LZ0*h=@`%moOszMT#_
zZ0gWia{FBE84;P&<x_WLc{oRy3%}msWL;9;S85@q>C0YU1>#8}4dDqZ3|D<2IiT|p
z6~J=RhX0NMfui&x)jA2C#TbliH}DQj6DkgEdw^M3;Y{D~fpzzakV{S^)bFxhhkk-T
zqHdq`Cr_LYot=*BN5lE&AX_>RW1QaUJP=m#Ku}cdoi8&Gt)&1pK3_aXLn}+gc-7C-
z<xJQ7o+>=3Q2=gM4KQa>&n+dQ^X)Oo<d-Y3P3~FzYW7V-`f>wrgmw=u8F@J3Zu&bu
z$mWl85uu|n?7}v-U|*tmwRqzpr7pCGeF^K+a$7~*Lu&8l9&RhwEH)Rm=Tz#T%+<;u
zAN41%BoiX_uNp!j&Rc$<J4;50P$)zuG(=jeF`5z4%c>`9fY6ATyzjI2M)Q;v``H!R
zjrk9mdzA~f_NxH4J*z{H%gm^_ysly=V|CflqY~g85o_oWx8CJ&Fp4Y;eX|UmcTW+a
zw_aPyqbMzjSF?at))RLlEs;l_Xy=SG<)8II<G9CtpZ*9Ojb_Sr{);RvUD%o6=IzL#
zPv$*MVyefKxiSJVBWq6Q;FU5Zo7hvxIW4Q;rDWzQW;yj#>}Al3M08^;4n$Pxg?}F_
z>NvTL{s&uyN?!M8_~Dv6@?F^(H0>YO`t-tJ1)e>vppYJ0DctJs&35uKk{8ykQK&t*
zq__xlyDoDLBkgqXo`kp0y&#9y7NvGc(R^!Wu|Z`%R&&+QYS?GmiT<b}mN${IZzX!(
z##s2RiKV0cvmdAn>OjtDdBs3AR=W>^%@;WIh6LIYy1O=)wX=lBc9I0;vcdfCEFBNz
z(j?hW=I4LtAp!}}nX{u5xh*w#*oy39EmqL`-V1ZKCk%%LtJa0?A+^oR;y6cYE5VX@
zt6^2v^56M&zjb0yc6F6%J^*2{!>~8w(J6BteiJV<-p^b{#2e;)$2n_oD<O{x+@B`z
zfItfOYQoptIkd!CF((Po#&HjJe!*U{{H;W~DPoWXAu$DuqbnEyx?C^;5Tyo*jhdSz
z!>_=Gc{dj0Epd1}%;@Cms>0$Clx7`b<t&cHnmA%cf#g3LGsSfQNFpB6aB6V`NsnsD
zq1kq7YeV;D4Tulp^ay_txXIP47*aK`N{L(FD7LRRH*T+Vx=G0lb3^s2-c?(%rP=;8
z@#0+YlSln_NS#TJzHXwSm$w*olNmZovxZXAKxsFHmmdClBOekWOgGG~&~tsO%PC19
zjqWVlGun!oi$zNMZv9a<JU+_CJ2s`N^7?u?QW9l{x`Yaz+9-bIO*Xk;-tLm1JWvp_
zc;GX6Q`A1?H0|NY#a&W-YE)-vp%JNc?T;B0xVajRq@dWz+ZvB;TuHxLW-=I+r5nj~
zsK<RrC8=Qgqb%Gj0~4==d?(5gw{V}6<86N8UmMqZv7dTvHrwdZZx^c0c2*!ih`bo?
zLS#SzaHoaBS44C`4+we4Cb_7t4wM*_n4)}^y8_O~euI$3^-wjGsB`_|j9eb8v=y*M
zY^376pa}~ZZQ*YZ){Te9Es`iR4~n@F3nlAgdH2i(vsmk~iT(CwOclDQ_KVwopmSWR
zoAVx}``xfIYkWv?XL2)enIG`%;UeT2=J_*p3*nUX)R#Kn0#zj_{H1Rov0D|<J1?-6
zQn2rcqv9hgW3wdb-|YDmNkwk^p*O1|9rpr^q1!J8+{Upcb#r5;j5bZQx}4)W4|=v9
z#>-edR(2A!07r}Y+?Bd@#_#u~a&#(BeEC1g2zhzBrUhVAwyR|+owtDP2sr2-_Gb|>
z5i~c9g2L)c(i*Xyj~*9)SWbA1ni0y~E36Q+Z8vqk^4(JoQ7M73mR;;LPJnDHUyXYh
zDkCF8Y}E$Q$QzA}SifxjaL;)JU^wf2rZKPhgIFL-*u?Yn<prSotIzoBVs(xF$I>2;
zV!?wR6&GU^mb)S6wh<mv3Jp+2)c|wwkgbgQuWL9UX?iXWKhDVBieA4<%~cN^?m2|T
zos>CKWw3=zH&&fJiB`RAH@qE*RJKp!$4<d`J`fVY%8rZRK~exSGm~Vy>u^_=S#1C(
z_$|_iI$K!z6@NAbS(>DMT;`IDS9y-&*)1xh!MwC|XD9G^*d?{~aZorU#>lZqM0HA2
z<)zx7l!k6EJ`*n`9Q2+p6)yVC{j)}c5Y3l9otq3)L=u8P`$!=hg6DjyIIf3+?x5^g
z3lq!2ZI{yYuH|JsmN%81cPEWEl{Oh^=DaTr_wr$epL)Kc^P^~!(-7v(FTju6IuBRf
zGgysxBb{JSk?ZD;B2uG0k}-iU%#26Q=;cp}b)9O19d=F5JL2c<%icbE=$GlcMGSZ1
z0&aG<@~6k4{fClm(oS?L(1%7n>uP_nP=EFIB^GySK+}sqMwCc?gq=gf2qYu0-dcPD
zk{zGyWAzS`{?)dp@4(S~E)3VPV9izl#F_QV(sAu^#79O}-K3TOMqtbG9pE{)RyPR4
zm0&9Ooz7|AOVvSVCO-ZrV~Q&LUCqE{dZ+vvwgfW%nBI1<_-}z@?%=jk)MHIAL6c4_
z3v-FZ7g!<Q6VZ9(YX$J#2T|La>zTgn!tE80>}BRFL@G9$Ok2!ULA^O58;l4FQ}pV(
zo_e~?4~V1z-TkeYcqrqab;}k<4Z{R6*#ePpFf{LLg5_UkMbF+*4nlmia<1aHEz!8O
zt0!BMR1yDibqgr_GCuKc>31Y6k~P~iEXKB_xAc9D7m|gV*+@ORBH?UppI3?lg>bi+
zI6N@)RA-cTp>L~DhIEbM|4=>TLXxui^X#c*kAy`om>b>QCz><AMg<K+IGV*6)J4<O
zf)*@Ze34d*EL8v@z!%^=I(=XeBjpW?YMtY})`@=fHR=mRhK)au!ot~)^Q9+|7LGa7
z+TbGvVqkhN14@@PYRz_@P+?y8P-ppV)BOw)r-E!3L-OX6-S!>U*1zig*EM4~e1;gn
znn9m>7vI94jYteXaS`xOrw@{k?V?hea_W4Gx<7^_pq;LmFVF6re8Wb;{%i$A#`_ht
z<YFQ>@3ga2omW=cF_w!Q5ko4s27Nci!<^loGufZ|b3=s#vtP!QbRB%Yr2FuSqjS=B
zxN*s`+!kw*mnA^Qw>420m&#t%{R<6_>xaAV<FvxNs)otlWCwwB$$4@tf{rGw5k(lY
z&ykRM*@s9Cg547CDPVZULf#thS_hDp{m?PAvQ-FP#;3p<t+}5^r54y#N;AD|fQQ-r
z2I~)|G*V$uwTZ8Ati}fM{_!<?7h-uj$~oU?_G%VLZzW{cL0ue7$`({0UghHD0F*@E
z&B8~a>5BqU5Y8-m4{IWhsfIfJrEo%3^SLW#4$FxUCWLerXi<sp81OY_cYeKTYj0?O
zV>uVSl2l3_3_%03R8t;YmaXPFESQiT*yiS=EJPonJM$)W4{2pN<urcCxcl{{k&1*4
z5W$-SICv)qKR_-J#DYW^;+(bVv~z`@dLC%egGln#yTWr(w4BLt_;iTI8v3eB2QHZL
zuX>i<sd|{#gXf+N(Msdq?U}-lxE!2>T=?E9J<cNN(ea@TOiA<>=XEc!9=~2Uo-;LK
zjy3tu<flx2kF6vVem8}5OCWVI{G=xZV7OK!aIvzw*yz5yu4g);d~m--CFq24{Q?P^
z2)jltv(5k1nXMn1>q{`B?w2FILqw|ymKE;yK0!<TaK;#Mk@e$p_aiS9s-ud6bEVBq
zz9QX&@qhyIOE?~U5D&{;cQ1+!W?Em#yydy;{O&+V1sZIpk6;%?`-R+8{aT_U>uFzz
zDTmrr!98AxdfxRD@ZZ)w-nyLe|0Xj9n1zP%&2v(AhvI0EhFwmXuDk!3GUYr=wZNKD
zSQ_uFjK$g8{A8Ke0UH~@b#FQ}K|%J_O~<pR6g`sGFOHN+KuGl+g5v(xUY<@EGh!$N
zjbw%jr-xNaJ1XLu?B(*%NRt08%F{c?_YsPZ#6=6|eyHnZZRa+Nqeh>OI_N5)LrN@6
z*g@MUiWTO!P3^^8B_Tbmv-v`#eBSajz8nHB8(_SXsm=>Lh5%cY8-XMyNjEYpo3|KX
z@0q=Rh3PAC%qBz>k>QI!YW(Ac2|%~!)lpK0>Q%W`E>;pHZ*SpA8itOL^T@?5-nKTj
zhfPt(hCjbDk^T%bh}s_oDA-Lhh>1hYb}gFrEHljRRb9`RkG%IK+EutjTMF*@V$KM6
zN`v4@LW=UBhO<@Y4;b>LVQ!<Ks3xNg_+g@Y)cpJNpY-Zg4t>5t5*vGK-!vYB^1TH{
zf=j8#r&MEb^UHl5#DkG{!w7`Xim-ao97pH-#S>QP!hL!VwZM9Jgk?omc{%^wtYdxD
zt45rRQY{akmTV+`v4piD69W>AY`#g20=Ki0DkH|$0^WOFIY`Hfp{s^Iv8ErMKV<L!
zkT2iIrd~A{C=D8M1fBMQpzBK}18mktvwrvb?{Jjr|21AANV7;(Bh?V+svW}!@-V~Q
zJo5<xfiJzLNJWr4Zk9jj|JtrAf-|3EJ)M%MpYjVa4Y0{!d8-&0wK0ySo=m<&_={{x
z`nz-GVDuTRKR=ky)}bzT`~xD8H&uKCw<_4-oLn13*^z>~%9M72wyVJDEl}n}+tTMy
zVw+pOXo8X<LCiv=&gux?(^akVyEDVp=I86>&Xowkh~g+vwE@&@=of9rSuz+w>C@-H
z%j?ZXzcvbnmtWehGiNyxhzJeC{(iQyiWNs1+t*S8x>ZL36U(CtDVRH@GpXS(#VV%0
zo8P<YmSff`XRLoh37#y&$6L{$jryJ0ctpv%rc<sk&R0e`WN^9YPV9Kr#{To*@~~fP
zK|%s8+1iy6<Fy-!CelTv8f2WGy<*1a801{=2%_?Hv1u$bDdGOTQfWA5#`h#a?A<3A
z9lXgvSaC)3Y}Om^X12iByS^UQ!lF5AHvIPb(CS1&3~Uyo0FHgyzk)~hcBPjRUoUU@
z8(DiY@hQ$Au#5>bgY5c2f~pmLo#j#IueA;`D@^D^<ij%=L~uD_0AKVy#)se+jJET}
z9m;#}W9u{u2<$PsUu0Ohp3`I<gXvGJ(nR-kpwvY*q;8t9ys*MxPhKX<9X4dmsOVl$
zwif@@SlstwMeE!%%(?WRblCr3%%<S_Hzj(>__(SD#!NJY|I9imHEtD?3y9YpGI5cv
zj7t`0b^jLzZ5;s#zDitFp>)h>q$0aGK)tqpScR<roxWdmV&C*_P<+Zg>bPFAnfl5q
zyoSZ6jXd=LOlK=V{L>lT&_<@ijT=U!M*FrKPaLoD##}(8XZ#Hp#sUrV+&MUw{~&4*
z-yD1{j6YvT@paqN&H29h(_hcN&bsrJ`BVPE)AjTPCTIx6<Mhecw4*`5!AK<vE{7vI
z6){`JY8EZ5*GrAK1tQnjVCruUG7)#zW?|GhsbV;f`5`yUXDKBE`e`&w%GB;uw~Msu
zQo7$65phO-2_%B4L9gN1|J>B#aJ+i^@oApTZnDFkXQZ9txWy0rX0ok&Cwv-ZJUyoB
zVkHrcnv7*(v>n7X8qAayH#cj2VEI{#r6Re^*@-}E>!CpB%@^SmVaBO<qA|3E2CjA}
zFOe5K^fO62tF4ciTWI;SqG+z_*{M%VkFV~QmK>{RbKwZ|yP!D9ZTfEBmv@Qoag%lN
za$xMTQUXR-#@bZ#FPetm5?*SzN5x`u7PIU1!{V|8cZe8jcN$w(KTk_>!eRztJ(mw0
z4v5&36Ha5d%yQFZ+j}@8??jifIeqcCvxJW4RI+HUMDp)|`t27BtWOU1Y<AeiiU#}X
znoj=oeH|J}8^mB*PMV5d)?io3-?690XHfb<NET2(n-T)D(g77ne2LFmwQvPq%WYfB
z-#`r|PtqI&W;!WyRs#XHjl|yIqODmRKRN7AE!y+`#eO>{NK<+?*ggbJw6=Szyk#tR
z<Oh@8Yu^l&#np($>cl@)RCK}OI>#@Hdp-hvilF2g0ECC_PzDgKDX0M(W#lA9P9v{4
z8}eJr7Y)~N%y&1`7@-ap{vSj>9&=F?%L1xAXI!^MAtZxxF1ItON>BA;y?WzdfynJq
zN~Iq-BDRv$e=q2ck?8&+{Wyx}YKz_fHHfP(bR_+f+AD@k_d&~da?pa8BZ0=uBqbc{
zB6wW5;dbY^Z=<}{prP^ib2+eUek=|GXCN+ZTogBqDlTf~K~Ylt^Gf#aPU5wfAWysM
z#+a}3To4m}?+yuF#kX7T!mW{cLP7uBr}UlkR_zu(vFc$k!uKn#-+%=gd~1d;rf{;O
zrv?BQ9k3K|CNR|U9C1pe>!(48m2q^;XTG{$#<nZPV1H1u+mERet)ML)@V~^tT+r3|
z6J~hogS{UVTvBjluS>2z9cOUK<acCvFMmWig^V3qzGS)!ynNCFF93#ZX#x1{%I!}}
z+B#5n@Hv$PLsvG+b3yNK95syf%iy)wPUP(X6r#3v*@lA;xQ1{DccCk$O)L4A6<KAG
zPaw!&EL+Q6KdATB*Sd}0B96=KfuEVcrSNa7vo$XdE2FW!8S}a5wI4Daspq$>QLz`0
zsr1DwRqjKug%7EFJ$zunupwbBDR6#_{68TekDa307NP5PazQGEo5EdsP%Y-xwkNj$
z&Y+x`;3nXH%Ytr;wYMK<2LH2|ph+1mzngpZ74rOFvMB+|Ws#^mSGcpc!r3U1WvTNK
zRerIEKuOrQtV!<@l=9h(f;2HV5|*{pd<#e{hlkhA;6r+EjmZew@*<Tb^VvfavGJ8c
z>c}MqSX^qV=^Doqmvbzjf+xh4zv`8-)v}8DMGOsQ+}D&F`?T^cFX8BLwB7*I0kJP6
zVB)dD?b7(o19$3we{6*{!f#=D{U<bmNaPkG{AHV^PFqh-P%Ji~P+58?Os>4p34Yba
z8qr$}{badGmO@=a^9M9Ks+HIP3dT1&TxRX<;Y&LUXXJGoK3j<F)ZS;Wxe(sUN$)7+
z60=$aJ1+@Qx<cy0#czO{hg4*PEfQc5#)Ua5dkU}wW%TQmuNRS*e6<BeTt4`2hi6Uc
znJ3+Ne7#)rxlg;yiO9PAGgGK|*9hKOVSUi?!xqOY19jkFgUc~2b;Cg&L{RY8tfoVq
z$WN7r`tpIw2`xx)YL}YT;20HKq5fjY$thX9J>Ttss208QlsONzTQ>C+b7xRYnB!z@
zfJdj>LuPdlk9g-I0S0Bom7bTnlUmc=?eO0EF%OYdx4!2WqYM-*t;OU-(&KB6a9{2;
z+rU(VGNo-3o58zN*?=Fk@nk<I*C8M3AMJfpT6GvL`=JW5OQ(&uE+m)R7MZ~tn3XKG
zO1KGgMhTE;>NEgTk+CzF>p>HsU4l0vkxkbzDp=&5${2!HZWFSLPzclZ=j#J(9S8>D
z_NQnhF{47W4<!;US3e3I!VjYV2XjD-zfYkj4<&2}25iW{iAYrJo+}EsxycQ9kc;m1
zhsDRdMTJT4%UHT_#Tm_hs-vRDPK)jqzvkB!>^XD3`z|I1wDHO~8HJe`x-*PCs?$ca
zjaWbkIe0GJiG~~QjTRRbyY(38@=QslN`ZpTUW-UH923XrvXlhbeAau;k#e^9K_10M
z2U?<x$&UIqe!_1@;mn@Bb!)e7AX_Rkv{20aB$sl4Jbd^JYs6qK&=`*i^T`9Wbg*!h
ziU;R$owJ-c9z+!);K~9%Whd3s_sR3TC}&%<#fy9|rUMF2l7ZTUkqx+srbGi?asu~*
z^b-AqJBTOc8*(GW9*$lTKcTdn``<&kQ2-($3gi7nxauzp^;;7b?}{XojPKed<OBBm
zijAv()C5>!lD46urD!<OV;%GW1MuL|hE3WxGMdF8wv$M8IB9-m__8wKL5kQrRo7=8
zLQLg(zCvOoi>eKcoFs_6jh@&LC_%qv5&5eW=R|i$;D*o9WG+$_W51EruLN#z9yzqK
zeT&G4ZM{8DF$^y@%zFk6JKBJ6Zb{(*<=J%mOUbpOKUcs5pO03H3~`DM&-P#mU!%{H
zV|GWajPLW?x1Ie`;<9eFK)gl|F&CUkjB&mO1TlvLASMYN;XtTdM<u|C2O5ZHd5<j6
zR)}W2rR0kg*^oV6&8W&%ey$M2aePoJZz3NHIpKrI#ScToX}mF$Rr6Ey7!g(W13Acw
zg8_-^b+xdrdN26JHqYNe!HF*4j3-wxk9vj{%FE*Ok7lq=TOl6?F^!+_MvTdf9vZ*!
z*Y7-g&W^)9egYBUtfFoO?Q_zsr-8PdN47pj78)s^1L)AgY8Mr|a}L8P#0^jMkoheE
z<YE}Ip4BXoA1!}}&MXw408h$X83nOfZG00N{q~LH-rw4<%V1!wE{(qE#A)YJ@iq?a
zZ7#}CWuB74TS~b0w9I3;`~f)3Ud{pjgrNgIVjo_>SsKw9@-ExM%>@wrocZMS5%A>N
z7X7yPly>vE_$B3LppBd`?Ug=vqp)0>sep#cc`mdafgqU3HGL2$s2;8U;$!lPTEt8?
z;5ZQ{w<5UE?=3v&EcYBp7WKqB0&pWpYN8{S6DMANdD*hR_Rzo7F7HnMvCnjnPbl7;
z2GN0cagLisI)PYb#{OXs-e0A7=(b}--P)TLbY|_N?VE&DdD<SIG_*;J%1a!NhQ7mP
z&eQO-i`cQEw0VoNzWOgo<yw4|MOPJROU4U<GyMu+!+O!qCG40_zE1(cWbc{AePSbx
zmbi0VKF77ppxBAgB7P~%I<|FDm*+tluN$RwK$QW`G`STg;q1W%_>;qB;4*@J;QoQ}
zK{`QxhBc4RObeJO%mn+jz*~OK-Ju~|p%R7~pW{U(u0<wcAPsZGX*|h6RIh*z4NIku
zv7><R20);>wc)?Vg4?N#l_=O68eCgTE_LyHEQ{}1%8HExL?f|HSU^@3YqT#RHnz2v
z(DT=C7?+w@yNeYe!sP)m0V#(mFYyszl`(YgcZ6UZ<i5WNJqmb@E-H4XSOoHl`kxtr
zg8PLKE?~mH-&1rJQ3U)jMb_n-7=+${IO0ga{!D1=mg}DR?HkX^NHla=1uT|%d<_f~
z>*xTYB5m;lWuVAV)*)=Qv~%{$$noRbv{}Etw41}lpQZdL0Bxrp>k|mt^IDv83Y2k5
zRc%0A0X8HaV{ZQkT^#<Vkd0w`UFu_J)qv+XPd-g9KY=6<Fgr56@5wfkcjnBfQ5!b=
zbL~;J>ZD1JKMroG>dAs0fE^8+=V+=?i`5As2n^Vx#@$jI`1{}Yt<Mof;e$rR=PLKm
z5}&6(YadyAwC!?E|LX1mW1Vgdqp?v9xnu>9kF!p3)Ie#j*dz)%oahjFCP2Y?!$d9?
zxNO6Uj`Y*_<B1*Ju}{cGpq@BOYpRoW*1?19mx))GYF!vhR`ywCEE~iGxeANuNg8lO
z{=h-tTP$<{=817M0lrr@iPOOM&NI^2dpVW(85!CbHw;{_+MFr`V5rrGlN={0qA3Dz
ztTnr1#)w*F94a~J0-K|ufM*f{fmGh%k4Jp`os=j7228jg-9rI+ct3KE*6b$ya6gW4
zL2SdFEUm95$d8RJ3=jM@W_xe7#e9MyUlHBej3%4Q{v?Yt;-IL89RD^N8lF>|W5W*m
zenrLZltZivU6=v``DBr`rw7;giQ|vtv774Le|{rbL?AI|_m8@#aX5LCCJ8ZVQfZ6r
zqO4Hn6@Z9{$@}}HfT*N=#&Ez3Cra=oP7r{y=^_#ZQTM*0pWm*D?ms6ojlv|8mM^Pw
zmU9?~b)W{`WMwUohw;Z9x0CKKzqD-4KNpW2kvPq}cRK`8@+skt<N{X-YV9G~P?Nlb
zgSeqXdR70Y<oN|9K7A)3v>jZ*iT#a0Sx45T;j0I`*So8Sb(*7ZPz$d#tyjknv2cvc
z#UJ7X)C$F!3U5cWfGm<k1IqzGkCG7tXvpN(@WdVa#jRW=2kWW(@xn&L$?GH&iSA4r
zvhgM7@*X~CJ2!3|$$61)@^!g7&wTE3ug@+#xtpUv08TOh!6Cvy>`+IslP9&F&Uk9N
z4G=qM%vsMS_~A}c;5xo0n=YuzMdhS09e-MTT)rqT%T%h$7ch33j};@#GvEMZ=x7ua
zq&~Cow7UPNe&yg>{(s*KxRJkQ7!B~^#=yhW5uFIQx=U1*CkVwn+8SkwUrgR_Mhm>1
zI4Y(edb%^6`>=~nVhMHJ-j+|vI(+hGKQydljx5}?DsP5=-FbUY3}m4$J4iZdbRF7=
z9MYMcC0Ry>T!k_AQuJ#HHOhWelZ$_bGT-r)yY+*${wR1+9|3$zWkT+I6Hw9B%K635
zZ`UArt0z~Y0D3gceMS;a1KeJ0OujfKSLrXajFIfX5)nRif1jHF;EH0toGkK@;;J~x
z6Cm2)OWK29V?-^*iNBcQeq}=iq|%CN2;9aj6%EDD?0s_Ffj(FFn6|>z5m}>j{@*@D
z66vJ9lnQ8w{Kt#R|Nj&J%_-0RH7W{*)a96v*bWUX{QHFSILW5MoHjVQhW2zQW-cPT
zK0pv2SlTwH0^a1dA0kF`K(6TZC692O0TbTAP7Yb0e2FJXT-`AgjrE+k@UcJ{DSG7A
z_nb=ucboY^D&fY3%7?I#MR1D<j&qu@zTGg=jB^uxWdr9H!WSlqT`0(e47@o?<?oVE
z*vCCvGwH@zvJ+}){WO1hzRg4?%JH-WF_MN{+&Ym=+%T^mi4%#`@A<{VU)~RNDcSj<
zV8y-N2PAV0uj%RHak*Z8>FFmwXD;89H@o+?7h@Xy_WAX(&qlu!zXZEzDe94eJE!FR
zd~SJ`dR!u6(qud^qO5-Yre%+!r{5VADl!7F;f3ry5=j`}jgORmWXXN%B`V?}p2QmW
zZerz_H}9BwyZDQg`N1Qy?X;G?mjXBF&gZHJ*QF5pkFVoRKE{Ejq8j|&tDxer8a<$(
zSydb;q>P)K-otp#VVV1lZ!PP+y2^YnPO?}0h;ri1m*4;L((TWMU+dE3`H$2F;i&{_
zbtw$Ua|9upa{}GBOL4=AdRO0~pO=-@VaZBfMPf<qh6lOW27MA3=h;rHyw0!Z$Bj-|
ztEgF8E!mDkFLy>y2DX7eA+ck+5{rrf{pr@&u#tSCIePN(Jb8^O1X1by462jEisTU9
zr5YkDk9OiU+0H(TWAa&1=u6Kh6EGJIeoyie%>f-9m~8x<56I<N|7Xb~mx9nwfI(cS
zx&wc_@edzPIw#1jb(L%+j_QqY2b8dP06l8JNkGGCE(|P{$|pCc>tF62D4{sziyQLi
zr|}_x>V4}F&S{pOe1;)d2=%MLV|Zfv$OljK_;))|Cf?@l!&%e{DsHGqZ6-l@kuv>f
z-sL6v!X%9WbPzop%V<NAyp9?h9z7!=@YkaSX3W^RGbRS~SM4=<01u>?$`*2|Nl%ii
zuR`L)`-`eL(4lL7S5bSGgawyxO2#P-4J}`x5vTDP>CO|%4SDX9Jk~j%5*tCR6(hxJ
z7&9UIt2G1u$|GHrXUgnQ68X&3QGI%V97P-U0vEC38Qxhu!BxUH2Vy1{Q<abZ;Z1G@
zMNc!4g}{;CSSFE*cyc(C86>By8X~q~QwD+@DQ3zC;AGF2zFs!%_6ZO%a$0<^2T{a_
z@}hDX4Lt{F-}O5xP=%|@Oof}CjLrZ0ll~S;eEUwEh*Pf-j3;jWqN*na-0jruZ`D4g
zj?H$(Z6Y4>Zh`{R@!&@WX0D~W?j`-w++Lh>d~QF^Y;rk85D<bFX23}r-+;U!&ZCNX
zVgs+z$MY1yavS=fPpC>Lrz)r5#*es1Ou~~Oa%&$2Y{*)JFp!I}YC{wxN(C<j?l?RX
z5D0`i2>D;8_uGR?;aBLu2PL5&1)?YC2X!D&uApAp6J+Ae)|!48Nu%M?9Y}J~HK|i1
z%bl{Rtt1>*!--blO*+Y##*Czfx=AjY|2mDFbw~#Bh*(7f#F&#&yA$&NN3_7uT^(aQ
zK_*!uxf>momB%=NgS$8j&90Dz4|SJxL`9E#jUxcao0%#-D8Cddo`oZo9t)<FanvQB
z>W>8S5b){N=S3#pD>q<_A$rzomqXxROPN&wygr!Oy1gd^ceCE_e;446o%&A=7Y$HJ
zpYM+w0x71Jl6WNpg}{mF+<MXpq+b6VV)y}NcVF?UGe!oZb4D3n7oIpyot^mevd(|~
z!4a3I&g^{=L<0_>C{w6O6)}0?BV=PHcThM`vO~S>zTXo06g)9+=|fX7m4UtD!=DLE
zb}q=UwA~PqX_LSaK2aYAj^pS2!Z?P`bYY9PjZ3WOt`Lboci~<=hbDLu8W~CwsU+2Q
zqM>36KY|V`qEbtAsM!3|5ZF*mafO@Y{$JUCea-%NwnL$B`Vx9P(Z9WtP-3Y$zBc~!
z=BVufvgk*JsODVYD4AdE#eofbXz^yYJ0;Am#8M>D&{7+tCmqY6A4p~yFRybG3&lq5
zC%`Bl5*=X7mFOi2|0BKCY-8fWm{^eEjFdG&<ca#iALJzY26MYck&OY*g{yGjL>J)u
zUviW_|NpmK5``jA>@xuaT9$%}0a6YDxo8(_VZ?(QXCI;mX?VrkYi-K*sBYN&;2G6x
z;)T7WjqVq{ZFNd%SJ6djQC24iB0Z<>>+>)WiUU96vl#de>sR%$;ly(DJgm~*#8YT;
z6X?mujQ+<ucvFXne$y~<ifw0m;?H>8@z7lsF9{R5M~=Mo(wBeY6spMZsju{&3Y;NK
zd=DrD5fDC%Y8=I~c?)`+s8vptQak)nO5U3e2c~TSR3?C?U8ayh4(D=4^t`+A_m;8X
zf4BKD*+zGqtEg0e{sVM`qV*q$oji;|N$5}!-uA+rj=COy+e-cK<<>+mPE_kk3@9+}
z3Nx&qo0I20%N*lZxjNt|hegeSlW^lAY42SwO!VZju&Dg485+wog$j+pjD{fR)N=C3
z<7B^YIU^pT@&1prMOvy@i_vgczs9i)>!`c#8@%p+9O5-ACsEhsVT72#E}9}8eX?eT
zdV}to_t2}~JsnCiI_z(4#)&{PMHUtNCo@$RuiO=cGady4YK|KQbZDvGq+`c=`3>x7
zn9dBrN*0DwYjEI>{`6*yG$#MT7%Gd(WL-N6@)LqGJF`$W6ZhCU@~HSwUI3ZS^KvXY
zu>f~56Gv=PH1Y40)@ycA+LSp3AR>Qi!#>*)q5<6*j^qMHG8$f_8XC<u9z;dM7}TY-
z<@_(i&HsE**Yv~5lWZJ3R?cBBrnIKK!wwqaesaUhFD^^?<Hv1=XS|l&t`+E-X)6OB
z03d_ZVWUxzT%Y#>08SB9+RgC0DPF;3V8=RU9u25Kqd6nA9rA8xE2fyb>QAWm@=Eu8
zjgDBp0I%t89p#ed?~#lN&n0Kjup_I*m3B35GsT$!O@IM=S`8}UL+0Q#Y2#C()@0#F
zA5xSJ;wU<JF%TnZq`mq|p$eg{D=L4d__SrwragfP3;La_sZIsz4)~5M{6z4@x$Gta
zGak+eYo6r`a>tG9*m2{=U%z48l5JjHW}~A(t`T!_;KxtArAy@~(R@7RGG=m6pEngg
zS1DEm{&;(en`5g$G739(MBYr?af37?0tL4#mjE3%&b$IR2sjl)7Wv52KKRj=(DO-D
z0qj@LFd8T8(+A}CP^YLb6-@HD#~J0Ez<JK1r!A004GMrzQKwK1lpj$E2od9CU-0P}
zRMj~F@{|ml8UoK)T73V%!Yj)fvqvXh>-GqUY_UPz4cd0u+jbnN$$3!`s7nwLfDJ#g
zKp5G^9!$<_tk8~8@Z4w7^VZ&5+O;TqlsyID)hQeMJP6`Lkw$$$;UEXb<zvPjpP8mY
zpKNJM`Ct5}DOUqBFd*NXDJ!BQ^|=weNe&Gy9^xL(iH#^HykB1Q<%!?FNied*)ZM)`
zpg!(6Okc8-1kr8fV6xHhdLXw*iqf`-{C-sIC`p^~Aq8Ux2N@VhJ>Ch8NVI12$R(NJ
zQ0Z2|@P#vS)rj7<#y4y|!L5b<bB1*vL3KPaf3zGu3Yu@R5%W&*iRTtgeIA72j`Pej
zJcK9d{k|p%BWZiD;UJs-W`(b@bNuZ&n6Y9R{0su;ExoF&yMlg;|CC<=BxX!seMT7W
zJUH}6^6=)n%y+3p06TGuZ5izrfBLt>apTq{J|8m`H~gr@4J@3b9+y1hNh6o+_fn83
zm>+%e=4eyWf=W?NehKz0Bs$!(sv*fkB%LH5BkG0YgyM$fbbC~MxqjjY4s(*gGv{cE
zKPi`&(ST<3Or4BT)R*bGoB2w%6Q9V@1TmOqfFO)Opt1-+1%?8ffINAc2SA%3ny7O@
zOqQ?7%P=-O`kB>LOZ#>Gac13@_*B;^z`LTcw1fCaHbDa}iD*242J#r+0X|$M2*@Fg
zo#4pUZ+G59JKZV!Fm;Q^LBa{3UQiTv;m?(vOZ1VK>6W^eL-^AUv#_&En@8T5G11X$
z*Oqpz%MN8G5k`|H^XCs40(tS*r)2d4>XJl(XA4s3&5eg1#D*Z3twldCYXe)c91cK3
zDLfJeh71+g(YWvYy}mkX)iBMfm~n}`bCFyF;KUzz!g+GUDBwGCP#bom1_pfYX^Mj7
zmOasBec~@sP$V80w$<kbH5q-nq4N#V2@TYD<=e#dGjI(RM;~8Hpsk~NAjfCr_ybkk
z+HB5z?6X#V?NO_$+k>pDoCm5kArC#a>8aRI(HR17c@I#A`u88iqiEvMxIzJH_*_0A
zk4kfv(}kwA?q82Y1R)oWa9j-J-j{aMLeAjhu)WxEkbnQDl-Q)oGq;B07JgZRJPLaC
z<Qi^lxjY9&yn~$R2Yi6pmOi^$;-M~^(2Y4C-5{aE&3^D-Wbc&^WN*RZ)ly`6wZzOO
zPU4djg9#mbURX*Sf@$gTC{~<UD$XMXJDe&Yh`L-mu5p4S>?a>1k7~3%Cy?x<!qq;q
z4cYX{+AVU(rm5Xd3Xw$76f+L&6ejH{Jb6T-X9XMO2jX3zjiQNHLDrEuS|=Fm8x@dH
z=l>PGCi8%?K;fx&zazijKgX<NJ09v@9e7x&q4WXqp8Wy)AYci5Spi(89&3Oonv)Bt
zT$THQB>4kbz(zu8rI@LYhTY>FV8Bt~Z7dW}wVDY7wXjEiq6@NDR&(oaB9IiC)NX=~
zYshNZ6v#yC?QhFA&T?w#6)vKXl9DX1iAR+6H}}Zo{i({`fs~m6B4Xr}`}(|%5c@?i
zyO87&f$6><&``0hn}P!!POtKkbh2b68h{mL{!z7_^c8u-RA)c2LOxFI6Rls~wXDbQ
zzhPvZsZaN53ZxT3db>4vlScio??Rx*&8PBzJ-4i)1El>+LeP;p;91hSM5V8;lZFq~
zLl0rVjbodxAP_ifcS4U2dCW*9?xAwOtsxzKrf9Pcw(PR8ew1aG(JuSZTD&ytZc<P%
zk2rw%4<)FqDT;rd;+3d3CUjppxsOm94xP>h%$(1BnGlX)ty`aD1X)X+IIyCtf3&Fd
zt*(XgOi7rVKt)>X;f)VB#$WL0sQB5QeW!h1Wi3E&WR09Dy7K`dc5Kx8gl~U+SIDec
ziHR{WAin2QOt@2zrt%~5#VXp1XFS`I%wDcBNlTIGa#rpV5hE^$jq(H9)0H29WNJ_e
z$YnbLARoj*M-Uh2$7axKs%u#Xy5m+e-Y|M}`}XnizpJigf*Fr(54(RmxW8fT>c0%^
zbM=qqeF%IbTJa7rM9#+s93_+kKy{3y0v`O#agg0blw&K1m*r~RXKsx+Mgvy)deDRB
zWQSD1ozsL@e~=s`ZV{KUAh1WY1N69sx4?h{OW0}zPW1ksfWYC_{+6TO|C>V9*^T6n
z#`A#~C09P%j^T0{hP(&p&IPO(W2;BENeFW#L@F}_W5m&^3;Hev(UHl{OMn|ME2*!q
zN96{WU+DA)j~*#~G&Hn=Yh2_ySr31Y9)B9Ff7mgPjfTC5GXr1V{n@tPU#G|>Q|Y}2
zcv|MOL~Xz#F<m4-7eyL6l&NjdP>}lWML#cVE8voeSce1aksegWgYz9eL?MKLYXNc%
z34{#@6Sb+0Wl>0B<~w!w&2_BAjMw&$s84O$J^E0+njv>Pu3idmaiYfi_M8Aod!jkF
zs0%inq(78!`!g_<DkYW^fj~$Lfmm_GRTWc_a_qzu`3xV5DK^MuB4O<sV<elbPs0m+
z*&)k{TK`Uo7AR3@^k_KJJ7ePl=}&$u#*-tGRypK-Vy;#NIKTzb8)CDg9Py)m{f2SV
z6J^XSh<zc7HL)<e!nTuL2LT55k^=I6=CU1>YRVq%HPAfd^U7$59UV2Z9L?uUnsoGN
zuU_Dtel5=80a~~^PJ9l8(^Ytbn8v%Z2hfyIz5~KBQV~!%N(~?o9c_RQ=!jK6iC%n$
z%{z-jgmSpTVideNSa~TSgc4S>F76yhAK4Bk8rB-$VMD=Ir3NNUID;k>PV8O+aDw{E
zUkvLm*{h+U;&2`#2X8W7e?m+pSJs~*<qJgep5sRuCHjZ~=*He3wJCntuUm@lbV^+T
zsB!Tv|E^U)cer4N+e3hX47>y4#2}np{nmyTb}WsbzZvvWM&$k#O60R65kI~|4t6p+
z-9W)gUiwtSa@oWB9UN-)O7qo!U~h3`qbW(f)6wG){Y881c;iKTx0PI{4j10)fPy=W
z@PPl=BT<&xtI(@$BbO_)CbC+bQT9AHUG9>Lv<D3B#boASUYde80@348QJIntS}9uF
zdt*d?t)nTAvU|w&*7-HRw2jU1k8D=a*Lf;pBvyjCyrdrv3^*I#TU7fyWp{kZ3RnR&
zYh6;YA;!OyJX6b7mc278*si2#Pq3eb)F$y!69p*T{cd`p`Pr-3GF0|0MZb?*qXnSg
z>>z4j;`}2zl1RaK??U0tRJj17c3r%syoz<~*r=#AYf4+}+s~{$i78#K0=4KUT7rlX
ztLXv^kP6=e^>_>c+{GC#qEjmrWZKr^knfpx8@hw@pbQg!YOMHC{3!4BY(+i7_$F`*
zmGGlh*ieGWr)v0o@<}GSRu^noFx6~}iUWPs3D{7vRQw(V1&2@JSrm%l(*Ink|9zC4
zk)Qz9d@EK|PL6HwKG_~D&VR5oKhW^D90$TqPn?J`SEHii<I340^EOcG6GQknug~wX
z5p-jmhxMC*bXxOW=LvXmnrh3vQPEJ`%GU|eg%h0_{i_CkmTl#vDR!;CdXF%|sf#aK
zmsS4rw~oALYL`BfamPY4c92gRfu#B8<3m0TK01P(JZxR>E;56a9TeOUxn1xjlFXPy
zQpuyz)?5TZ<li+MJ3pa)TN?uf$e>^PHE%szKtVExd@7;6^C4HoV6KcG(Rob$r>$+Q
zBSM;ckI=qu^)%PWApG;DIIy6vJgBJrowoumK!;4<jfN-IArI1tt?XKn!9glBP@Key
zH`kvZg9SYWQ(gu%RDE%xsIq1$`m>ZMiUxGZoflE5fwTYHH0L6_F3n~yZWu9-c`B-5
ze9CX1El}FL(Q?~E_vgU#J1+!&F$QMOsnY+6dqF--Loo)#t7tq8R3wRwKp;5?pbiy?
z03KCV5}=%5s`e;9cr2y~)v@KxU@;GK#Hq<pB4ka)LtNlq;njCDK{%+|!v_N%*sHvV
z0ToNgYILZmzQYkX(3xJKaI101ExAU@G9~#OtO#s3I+B9}{ooGR_(;x;AIDT7#55pH
zek(^qY^CTM|0}P=okppD1QD~R4eC7uL^W-oG8GsquF1NSPvFB@D!&qeD9XiOUQlu1
zys?rXZjk=KH(bP<YWqLb)(bbh5vL+wdS%&P&@>|>Cr#?x7kEhVlwB}$P<+W*^2p>_
zs)aYDfXi(BVkH_5Y|X-oo^)#}<A0L>JXFcP^DEBLj;#+pB){fa4A~xH27`&77)?5<
z$k(<a|ECe~)(J~R+P0B_8>ufok#W7iQyD)i=ZWnt#ugltse!va4r{ym?nWUA`v+MF
zY~X!=QTaP%1)N*t8uP%38_M|{yr_(IKyUV-60qwV8j~*<xoskyqoUJWgD_#nZN$$g
zD1d*_eacVt$Zt{HcM4GOShA7ocyo04BNX6EqwI9@L?qc5V@F2i$FKb7eZaHEZ5`I7
zB8YT4%8DSL7R?nE#0Tu51t9Uo3RFko91un}HGl#<L^R00gv+6Z>0$(KjPOk-fNQJ^
zc@h&h2&{3P04#XaUVx42$l5(|<2uf2P4M9ge0)lM*%+w%=EHW*O;)?(D_|ME%xf2n
z87rnQnis0r6;_t^YrM_enz&7edUhmwJSpg(&IxfZ_wHG$eXk5v7N9e<a#pri#4+lf
z=rQZ|m$uk{Kc%h!MCQKMruQ3wuU1<m15N43#(HDP=L(HK^TSFyj=Lrn{k*K2a!Ur#
z46NkF+vFC`aiVv!Xn}^#TJ=Qs%U>=#dHWt1-hRg855EhllJ}%q6P1<v>ukp<nh{8P
za~s(lAah9rJjg<;cD$(AKcDKf(0>o!+`u`pCP#RIoatv|I-lb7&K8ZZ<G+3yPBcV|
z8AYXj%XLvQflENk@}vS@)EL@7;5E;!&Q{Cj51i5BlI61KlN*&kL)kfQC<+RB)2s+o
zocaff%HJs~;F5$j3c$Vyzym|Sr%A<&Yo|x^4a>RLzZ(zJj+76_V#Gpb_xt|*Bg-VW
z?v((k-5`iS(x>kei6T{QnB*h-(_1{?xR;IEVz~|x`+A#B3BG?`A)mD|F*v3gh)!r!
z0D5yyt^>kFRk0dqNGkEbbs3KYteCI@{-QUlKu(uSWTrAow8k!8_ZUYm-Z+C=5r!96
zDqo^HzW9f~g^g>Rt$mGbta#QsfQk<Hs?Vd~B){T~!r)`?cToVHy5s+;krn5SP_jtp
z@)OT+l=eKA@B&ZD0@)Er5E-KD<ISi-Dt_+{M~4ob`SGH%rL};#-IRRq2EeiB4KyO~
z@1M#UjuLV<PR^kcWAJdk<xEnBkvo2TyLPKrm$uk{AEmAU)c)$VUA+TAci8m0(hzJh
zxr?;5AY#M@@yva1pmGY!XK&zv0bA9KqGDw)XSYB@YEC3+oEy@Z2DmZeqeF%oT7T0H
z#G&c0HeB@a?Khi6zC86rpZXyEg_GJK8fvTf8R#TTg!cG`FtYLb@H#3w-25IbD)!Iq
z3Rlsgq$Z)K5tfdp$!9;_2ilN92KTf(fHxj|zh(hCRP5$?fAH=lCVdaUeCZ?lQoxY`
zl|$owr)!=&?tdt<UMK4qx1GQUR?qsBT6l2%>G?&`+{N5EE8r3wXlomj&S^5AsD+Lj
z$kb)%>B*q#LAZ0B4WEBg=&riA5x_0}+VbBH#c$Vu4qeuEDpG-<r}D)jR*}2BllC?h
zxddawqX))sFDVxH<kqcSYl5~#drOQ3<q`EwXKUbuXe|~2ccZZb$dQNf1^hUT3Gl=V
zE66L#Ui(aX2t_`x-{yXi+GMiYPa#Qe!4xtH4IRPNd!QqOBekw@4HJgQiTI<(yZS~1
z3NrK!1OkU&FH{7o`(+AE?^XY#>AmDTaYMm+rWPvosN8QXcCNF3?=U%7=E@%G*J7ba
z65A$E#f~?oiN^|yiDCuwPU;_pd!;<_#<`|5t?(vy@Ml~QktC=2v4ARaJ?^oqqE02$
z_~VJ&$_u5g02C8>a@xVZVF=RXCb}WHU)Z(gAai7a*wB4|2*-r|Llr#^@LyC^tnB3M
zcEf?~{Bv|5nV;Jq<0|fWo!cPyQCS-WCr`c{yL|9pf6T8D(PP@0J{wR7m)Q(Kv!-i1
zt#jz(woIvFd~4KDf^br4=Jvw#42`cA75lq1`;vje4~+_IJ!wOICOy1goR$l@V2zMJ
ziI%)SryV*pa-ZG&AEo{{Rto#F5f3cn{&bxNR3^87aP7vyagHZXX!rFFf5pBmz)Q@=
zAN|5=1Y$sT{TWJWo6_GyiRGmDPe$^|p5KE<`BZ*9W-8zElo(b}4--LH?w-RXa`F2j
z1}j#yn$?O*mA$<Z|GBkw90DDNypBkmkUp!>_HiW?3mVyHHzTp(cS0-7m47OSN~BQx
z=iV~exf|Q*ts+sWEUR%^9;J!AAH)};vC;%Lf`l(#SlBEN5(pUhR#XKxh}WgR;Vcb+
zAAAmwF3%BE(Hk?Ts8+QW0o)+F))zSN=5noeNK^vD7hyt&GwfxoWaA8c4~c^CEHtr8
z<=8*miq4|8RM9ZN4+l6cI&p2lAUerO<eWQ*iLUr~z0YB8;JsipIxOh>SkUgEujD_^
zA}J&piEiE(vL_K_jeSInCz-Prl^LYVJs4u|iSCe4;ijeSj-kvb0L4U3pK`Tt9TZtZ
z+>lS9MLv3Pk=IE$8PTwMcE-~kv5|>*>t#S9O)ZOx{gXMnZ>6LoSqLNh;}ljaZ_B%$
z>WxL-V5+wLMDv#}fAtR<m?Pep+NF0cI{c|By5nFxktl}VBAX1{UjK=jfEv3LkmyCz
zqGGpSvv;C`e3v|e&<~r#b!=Q*aF+T=+(V|3Ll$0}0@2|_sWari+VijCz!TH@6$BHF
zb5LB1)-~4IP1kqTyBN~UL3Jnh5J1uwKj0u6wRy{;mDQb+Lbt@!sp3RB-o!}?>ZCBS
zMRpZyr(c#jF+@w(#`7>k#0y<QxPc(`qX|efEP=(FF#lc!yYMZmMQ`1SV(o*910^%J
z@Px$V!T1o0df^-<(lDN0OFq6_KKUgM%(OT&4l5bhFCV5EzR1+I;z3TxExF_6Ga^Pr
z%B6V89q0iMaxnnc&`1X!5)aY|L@#OKVUQ#FmKK1HP~1IV#KZ~e_Kw3sFgF@LNgg_k
zHO8W1L{!;>6Dw5kK*5gctrc1T^_{s@`6{F=MdI8_;kTZGC)%pHRAnPqo_dp9I^dah
znTFIRt?H+oWG_{h9><OoUGvwA>T&0}1RSVHdpZh`_lgkPSgL(Fv#yNbZei1s<yh8o
z1!Zh-^ooS$caAbt);i@)A|_26)TakJUXVkmi9__~0?l*>*~39Toje<N+)*b~Kp@a7
zZxj{#=Lc|shRxOnJu2t>Jc}1ssI<DfNRYLattiKRUwL+E-ap@ABK@cK=u;mzs?wb+
zSjeThyy4@A8(YM{`R>?BMfSdoc1x?>&cRAzM7h{Z9;7Y$_64E_`#5{%2=zEh%|4M_
zCV;%p)6q~+bhqm~vRhZcg>|r^M?Zf9wW!JiW70c)Q7_;2Ou>M#)qazly_!uTM}E!q
zFHAr#TKM>)x|h8b@Sme(&y0eG$gPJrb{r$?i+MzHeesvtBC*0z0Ah=5q-!xW2nUkn
zxK)6HaBopm{`V``qR9WQ0@B$GiHalJhdlDprOd(|Z;TfOz8Eo|o<=$q$vUx|Op?%@
z=}b5Wsd{N9KG-;My{|YZBIKg1x^k<$8z09PJjf;SFyMtAZ@`NTjsd;oqkIZ<R{GM<
zm`Gbz^Pta@=;aF1gB~RuNp^5=f=NYRNzZ;9T&=u@Os*01?NqjiA*}o%ljU+ZVaB`I
zN;dUp4Cf;G*q$AN{g(6hifdYaDN*<pG$a=Y!vXL9z!mWx+PJL*$;Id_AL8H$!OdSL
z1t&>M-bO(ps$~_GFMF=oKS#-)B7wk}qhKYQ%;5_~s8GbUF9vHuTDAQkhKL+-84;Tw
z^;mrUauEsfKPmH_IP$TnQ~NxN8|iql8H-$khur7sEAGZFV&2O{as$6dwO{DnfA>dP
za(@r<;ZPGg^COP?uak`l_mnHLo%+1}aX9(V#8xZY{qyZssZv~AuU_EMD{rxqi<z^S
zX-FP9bl~pFSI7p|f3A{`j@(uiZ&Oz;X%`3Hcr89FkBO6_+7rvfJ0d}@5<V=`Ceq;3
z@#K(7ZolFI<aS#iN)$nfKy~!Rj~qf?+2Y7{&Jh#e_~DT&cYj-uCTse>Z&qb&moG>~
zMfXt@PHZ>>LyO9n9T(GIqeP)`Ah0F3MdHS#$2zkE3pty{(wWty>w-yVAa!3&!HqnE
zUc3)*poqehRfQWmb;yDoRBVv;6&e!5mBR$!P5P;2cv2bl#B=0u1@F__Fi?>*=N7Yp
z9|^yzl0^gV%{U~d%1tuB+Dasdn&MjucX8wcZlq%X0y)DAz&!CXpBQE`LZ))o$IJ~R
z=885n_Tyfq2bo;QQY!`{70HP>gAE;2NyQBXw@R;~k&6~M8Wj!AYYw1+@hDCNl6ClE
zN58in^=U#u?>A65P5OmqR1x>m&3_+@<W{yX>JK<kJD=b`oe|}oTQ|Y#(Cp*zrz+Q9
z&SsqwOWm0Rn6Ev6Q?_O{jk(%03UBi8{LmAD9j9;caYA=Wrpc1;#DW9cm4RF(kV^wP
z(wn)YZ5xmq5xn2;Q}CFWTsJKYVn2%h{8v&omkY{4SDrL!(j<s%tnNHM4>RWop$&Ps
z(TAK6C&BE&`?Km;(4n=x@E0>l{eFtq16FLPZzjqfoM*}SHew9j_!>@xz1(Zr!=>GJ
zP^;fepPnN?>1|o2Z;z2>B1ay@ii0b-HGZ9AWO4Za*gNm=sEYRQf99NR>AjE;LQm)-
zT~R=K2dUBo6crUkR8*Qs6;Tlt6hQ>(C@MvYQWc~~?;TQrkX|;)wsYqFV>bl#@%LCz
z0`K#^F4$~xX3q4x-?Q`#8Va_!1)gGmT>^F@(Vf9u#7N#F11W_KZPg6Q6N*;(7?~6j
zvH1i#cJ!ED|8VzjmVk{YxpT1KgX^i@0eBzMD=Z6aaK7~0_UTZzU&K}VJ^2MOTqkSB
zP|T>{mF6jbyG%^C9@6vA7Y*e|9KA^7a?c0(gkTz-org(G;Zwif80A<xjc!0L!I&bh
zl7|n*qwixSlI-K-h+seZ#h1`Y=iapW%6O%=d^PVyy`H{8ESCjzrX?EzZvsdIDqz9|
zd?fnmGejm|$X3dD|9s}r1sT2$1J#L&?TUlbxZ-{WY$Q~^gcB9-id`{cM2+2y#D$C~
zK*5C)uon#%x^W@JXW6e=+5h`ID6#^p1fVUROgGV(^-q=Ikep4k`~fV)i|g_8sYN(3
zD?Y&+9aGKSp3-*<gLtz9tm4f5oK*bCefks&C`aujZwk95)vF^z<(szCh09;~2^(Hy
zycvlL1DG~?%HJ(fHO1qh;)UVRhde}CA|6TL1a=%}-sXs?PFH*g(^vBe`6EU&Yj!84
zcrL;nzyP=rA3teQ&z>N=$zh@*UgAAc$fS^-tRu3<BKG0M<z=7XB9nq@KX{bD|KtXB
zV#}LH1}8Y#Q4&cgh8<6c4V>T^#HrF_cdmH0<fw?^)=s>y*Vn*zD5499WFFBk1IO4e
z-f7ny6Q#KRSr2r)P{N<|6#Jv&S1kGK*wK#<kq0<Iddp6t33ePQoe1C|Y7ZU4bqYwE
zeFPl=?EROzU(C=LaA8QQM{}%X^xYBK!}k$=@Q*%S*Vlf{R@X2{<ngq4^!59QC4j(D
zlTlC*$|X<vlItT-#P|vXiu0)Zdsa;hsCZu)iX9uS3ts_FtjAB0j}>{comfexT=?T&
znZ<r_IR)Mng|a#bwEp4~v03g=OkzF#=*tP|k|~_v9xOm0`S^g?CF}Dc$W`J^N||@l
znGU=hJcn#D$&Wrk4Bljv3nYp{9PxWF5KUTqHv+g$V8xHIVS&;iI4~iCvH&M|87XG5
z|4IBZ#WGq8Kt*|dhHS(n&P;7CV#r6)(^$tW>bE$KMg}>H%oHMUMOO5T`rRi1D}cOG
z3l{;H1{4xT7!g~~iLV(!6LSUGheugIEru)jkp2849>%iBm*?(@7AV^3-e?%s!8Tlk
z46aReTsYDW)0lkn@fz`9)1GU-ys-}M7~C-oKoP2ZF=1%0bs*k0>{7x&`@;3Q-VO*9
z8g*K}Chp-B{tJg=LP0GR?kRSwY`QnA{v8;9Y(X7%bD)Eu5EYrb*|F2uOs@eKFTVQf
z7hl}&(fTdo-kw;hcXtH$$iuP~Qd~3kpg!=fs|*u{{7eCH7~cENow~=nDF4DAk$~L?
zwh0x}yBlREw(1jJY%BKjh|<@xj>}|IX4OWZ5ZZm<Df4e16vdpq7Y-Z;hfswdIP%Dg
z<-3P&)LNdc(!73^N9@ym2Z`mpBHF$G779*+28Du$Whv^b8~@z?DN;1MN2nbF7v$GL
z;Dak;1_~<9q=(2M4bz41P>95KDxP4XsC=p=xfEc(P=!iFQu}Hlm&h0I<v;=n@-gjM
znvl;az!zVB22PPrIlx6Aen1%#hy<o1=?`>5=ZWx@l);Y%wX?~<furJFvWX-qt_v2d
z7%G=W!GWdhWOolDQZOLU{9eI;K&qo|j9kgGq!(+7bQF}_o#Ycr-c!8^Bb=aRS6sXJ
zmdjwck1!fPk%|)&muE(xV?ehQt>e4JIj)$Yv15i~4N1m~_Q)<=q$5rk7=}Wp;Ahy3
z58utpM8}4s>rzU}jNTgDEZDeV!;bx<ItZ3>|A@Cqpg;Mxhvay%3Ma-FqndoQ=G`0X
zpv2>joviq)ljrpAgP^nCR%RmQGT{>EKn#-$WQ{?cNW_5ogAHg1@F^l-e?O5>k*`54
zcD!HRF9+}d-%YEhT$7tbB{=osxEag0-|o@*?}$4)A+gtksKknw@Izw(&#+IGVhShd
zG`<YJWKcNDf(B^HSx>Rwmw+u8Fjzn30sIi1y7Cj5sM|s?kcO?!Q{)kX+&CK@6%kqz
z)}4N%CysUR;ZJp%4IWotmTl=Yq~+aHt4s~o*O8N{AlKr%;4%g>&{}`OAKx>-N8*IX
zp01<X$D*L4WFJJsi8EzCCd?FGDo=U5;Oz74#|N*ok7MT)^4ttc@gV6*+lUkm<*u~h
z^0-u)ZOs;iEEUPLCYV&t0tO-wKoyn}37TfQuGQr=wZEgE8soK$oy2f3pc<<v!-nt!
z*g1rM%n|}fB0#HzkNBAvH$Tef@-eo$wo#X8zN?u^Sx#YI*#jdQB5phKt_q_p78C?V
zNcfCEqVVs#2_yNKcdZZ~(2r1K8~F^MaeY!E>Ex05atA;MQIyE)?va4Oy^?b<p>Lkf
zWbWmGM_*$SKXB-(h9I51^3yO-3j30S2o!u8F87owxf-)Shnr0s5}fn#ArRB(xAl#N
z?hKoLKw0NH#U#AxFE7Y%5F_R{tFqRo<Y<6@4<+Fs{#%GUIq|Vx?NG%~8dHeI3bBFO
zY8Vg6#w3nTMk5<B?hUtURGsW8_SYAtf;W0vZ~O@*wbgXMi0N<{>aj#v5T`#F`|a|z
zC4b_J5brgyVXtAB>BGGQVn?MRjeG}lnyak-pdaZt$sXA3SBLzs<v0Z+68Yn+G@v1-
z4%Ilva=y(SfR(EB7<GvP3dxyVwuFq#FNw1O$m2SpAC(1S**fB@wm()sZ=Y<d6EiQU
zv%V-knQ#KpmY#Au3D$0LBg0{l@?AlKb2ADmdQL4={83Wfx1G5X7m<O2ey%+h9ORyD
zjKE3h^J|FYJh2JyVIzrysh{x(F>*+bNv2arB)i@bpOepbKrjWEfCT*U2Cpg675%0X
zQ@HJlmN&=y0&6(N`H*&$A&cy&P{JsWclj<{CIV66AO>==mGc5p35sn`7%qr@nmkP8
zhu(w0i4t%Mff1!33p2imXDiV_yegM2Ud_Gotmq9a<0HKKG$sQB1s`q2g#!LX5^qV(
z)cI%O@T?6Jh1g;LhioGo`o7LnIPp1~$zFOvzzg`Wi`+To@kK}N^{gib;%-6lKG8hh
z5@|TdUl>O+_2nDSHlZ#VVwrU=QdH$~{ox>YqAaO}z>2Im&{O{Iix$ukkadm&<e)uP
zo~B|CZN4_pL2u6wpuDa$5vO7Jye6*I$8LACe;;?vVvTP<@l>x`m^jNE1P-|ZC;2{Y
zL~qQ*y&s4m0L$QI2$Y-1{vTCD>Yis6G|V{GR>w=M#k<Fwc#%ewz2oV<W6rFYe!EBE
zuRuzG_)(Ku_F9Aw7pTG(@+pTmdZ`y!j~V;I<2W!PT7LJ7QFD)s0SR0ugv60147BI1
z=}x&z+{@PiZABY7n)GigU?B&`{Y9$(lA-{<nX`8wonPYM63#L8w1L`>V&U}VeNHx*
zT)0@i8W7{vNnU!O3gH~&%!>^vKtV~3!;eBD&-5mkEDF!LFcHSJgmzRC&rz6?#B@$_
zEn5-a$*R0z&zI9#%r+p66taP6a*!Yfvld^_^F=b!^rPq_*Ap7#q$XF`6;%_3GnB1x
z5HlK%%5Cwb8fwK62r`i68&XIC&X_j=i42*JiUHxb#Lem|pZw|Tv<*m<D{l~r3&(&r
zsK#<iZI$Ah9LvK>w7f17d3{tAnMi`iS3)B3QkQsU_imB=idSk5-sG;^MSYz34E~vI
zsAy>~u!AdXjU7r?{Aj!+4{uEP+`s6CmdA}dT@!Hgd<rx@iVR#_o%D+AO9EFW6o?ai
zNuD-}9A;C$QzNV-67Z>?JAqK>CQtdh#cA^9sG35I=c{5Qo2XuE*})MK4px)~`p6M@
zDfi0^^*st-c%fOdl9*EXb|1jE9C3vc>-MUQHy5c%8U^@qMy5t?5KXyXbXiyi6Mp!$
zatChwxg$}c%cm(IpJPuRA(m>qIQJ*t)G`ap<--0mk8XMM#oOKczcFrc6DL0M2#8HI
z(Kq1DXjU?a0une!LA3|TrwSQMCs0TTYRx7$3Tk*$m{OAqv}9k)wZLiW9xp4CNuzSJ
z4@5%>xV9!2A8hEpqe_Sd;McI>-U}A&<Rfxs%Zpql|H-cYFIk$gbwyMQYrM(kfUt3X
z4(V^4?s}A1?se`n^uwD}z90>6!f-2(RiGYr3V<$h9IM#Jt70SX`E4MRe7qwjlgiH=
zEwhqX!U(BYhe8UuQsr$_K##kQijKLg9V1Tc5#14JC;{&RAk_VcKTXpA3<^CbgBLC=
zIFIfn4S#(4YzBN-cVY~R7$iUN>429UBR=`y6?_py3~KDwwB4wuq%eAa6~D|FCjeh^
zQ9s#9M=p@_f;S84O6h_+<dMbCbyG2rN9AQ*@kc|ixYMH_x?A!}6#D{6Br^XpL1>ts
z??e@1nD^;1y6AIUeSt+vKSR`0`CKHA)OTl4fCR76zp3ZWM)CUa!hrwwPbftm-TF4-
zI??=mwI<0p`5NyL6PwmqbE@Q}6uwh8=I^3v)d>@N_XfR}@YQGG&1%v_CkjwdDO>F|
zTo|w~3&4hoTH_T@vH#nPR{##o6kaZr`)SI?7UwW>4gU+Zn8{N8OE@!T?20d%{E-%x
zY{Z356ulaPJfs;&0HwN`$^$?RhI6IaTr#La<f8A9I59@AKoEfVLZLN-<g0xj<f^cU
zMa~@@lf$_1c5ft6m|k4Qow3%64{8ks{aizGsYY<y3hFc^ob?x$KGyjD3xy%TYNR;I
z$E4CgP6!^18GjZ>Rl|$RR46wGK_OvPI-=qvx5`Eg_(A0lP*73JT*FR2_K<2AP*J?5
z<IeS7a#f^8H1zMcVWU8P@n|A0;zhY@1KG!MnGm&-#)OD=i>)Z=_%<7KBi!)rK(PcY
zP(W)%S$y%I_=G4ghKe61U1Of?E^XR#vI6z!+}Q;fNuM8%juTgK1_0u&i54i9<OK>M
zWhuT?pu(d*oFR;a@4ARSq84Qg3*>gWkt=U(#zhi^uf}1(g;2))W?b)NxV1gfo%{+r
z-k)D0g$U%c791#~<VEEMHmP}U6Bs{Ui1FiZcY}*@r>mxZgOex6ja$AP<ok>e?J3a9
zsxJL~z{pCfo|sAj1zdgMDqzQcb&IE1(T2?bRdvO=zd~>kwEZ5@NxV(<OGoh)gW&EV
z{6#pEK31;S1LCssDhCi$Wwjg+jMYCD<F^eVkO<;loq#t2OJ)sx2q1X%Gi>B3G1|F<
z4RR_ABj)0wG)l`)`6b~m=_H|8H7QXr6*MH1Ov0=WYtQM&XbYRxoIGQS2iH>GVmmo#
zI7#kb3O_DVam!}RxKLKD!h|4eniUl@u7Qn_IB?y!0*ylKrGLPH0$v|cvI&3jd^u5(
zEt?_n744?JjX#2Wd_SQsCh^48p47yLQX^_(z=U%86Mi|@`G5M&B@0Ca%I;QatKV^*
z0`f-=mu9(KHW(b@8Y@;~f^v(Owo0_EHsBx^=lDS==ooE9`6P<YnLEZy{C7~aPYRIe
z5ZMP8r$~IaDsikqukn?<C_m(+Yam<1QZnD%iv=U5#qLXuicf^1aV?4Gg8VOkkx19;
z=xCTT2T*_$?abp?F_0S+$TiZ*srV(886{`k?ymek?gXjqZy_#zvQN(o2$88jK_rq-
zL{nuD@PgbZ4Fm4SNF;_2E25xa_gmm8R_yGdp`hC}k_jiTcYR8eiumRrg*+`kM>(B5
zHfUwoUE~M-E5uiwSkUtv3PU*~o<*k?2`o4GF@@_4>$ZguPLlXzG&{*hUX7+4_Y&15
zmlV>`Ry+j=RMV2%-H$&E;R$C{0qf+ixmQMWXw1W6srXEKiFNWhUglM$ag(6R)17%*
zVBCihy|tf9nHWxK9P^K2CIDBPIQ}!^bEodNn<ZdCe~>6Lxje-vUZf9sZyc3Vm`y2H
zIt5gqY}+#wVxnNd0wi|y@F|}1cZ&_UxF24aP}1TF!ia0|W)2d{_;oR2DJcXSzm{VO
z<?Tsk3b@AAmlq<@(ZzC4`MbqOzzqpFG2peiBIQwO-+wS?&^Vbgnlkv)sFY4BIefX$
zfr>_MOY?0Y{lDCczX<fxZ0?z`K8=|`MA}pCE5?#vbEgQtP>Mow$Ub|80t(1H7fk_K
zw7=}dW~}VF)(jV+l+Jp{m0%pDHL~Alc+~l-x|B;uDiQ#EzyR3}s6;j?C?ifZ&00JD
z_R){KW8wWDICFY*{>lnS9_M{g7L|A8EZGa>R7xp57N=oH!8Ua-prARgc#2tWRA9n^
z>C{uipb`GlReZ1z1=?0+D5LIP!e4~bC&zlPtb|T0k}zWA92V-h^5jt6UtvXkSyr21
zm~%B+{>)slSE)}bPAnZ)0|LVh!e0_O{QnFUFl2S7EN6&V+ku1PG!K9@(^2?U$>%FJ
zi{-D)*wOH83*-IXXA_%ehAT6!Iq>BSJ4Vz(MN)X!oj9j~xkms1fB;EEK~$)i+}r+Q
zYf1`=BsGV7CY2w!N&sa?ydYoaYibXU)P(Yd^aYM;vvtFUI2`z59vhC1Kq(CKl=r?-
z&z+5;epo5MfgX06Y#it_CefM$jIRH<d<r|Gon5I*DAgz5gEtQB%LV~A-@Q?E{@ywJ
z<3GdQes|o23rCCdT*JZ6iJMRviSYI%h{Jq(O+mq*N-vD25HmhUzy4M4Z(SlCcb#?I
z@eBDQQBg4`ltm*5ubq<+I0#<!5Ld9Gj{TWD5^(mdO&A%_sv7r+0H;7$zm8mM)&`5{
zPlxs~uHmkYdcxyN@F5WS)^@z`A?E8P1X7CRL-Tl-gYwIizSzWgR6-1Goy*jz(Q^@)
z2)=JH@%Z9fXOGK2b5pW60ccbv1pVCx+jn=0zI}qhohp(1WyF`BIIWk4NuHx6591(3
z43rZs=S6EdP>g!}3ls|QdFEx@zRW5D!L_La<zkx3cSbD~@9>Dy0!}X-o3z~ex0`dv
zAwGQ4;qIRx7)GK9MKVbiDiZ+}9a-)gVO*xt78NgCDCLR{{*o>M7e{$G2|?X3nRM|1
zncZyEl)X?2Cvk=-j4#i}pX(T>1Q39Vc>O+6O~#AlH@9?qpiZo<ncgqh>T^-gX!<dd
zNU|3V1at~Z7lp;$EzV%Y60jM8A$2=}_)s{gHhG+4`uad7i-F<+!vIFg<8r_o7jclr
zrB|PDOIk|*o5mgPd_ko6nq6Nc`FkED1|!$|Y~d=w6kIZ4V;|Q`&!vEH9{Ri!2D}l?
z!tYvN35g5E^(wwNF^@TESVd=%@YZqHJR0a9$V)6>j`*lqRf4!e>gR9Z#E4S*_mii~
z8*4<v<s9txNy(ZFC<LUe#X<z;OG`0eCFjgDTp}O+%o##SCg8k^nW|JvjNpQpL7DVr
zc*&v4`}wDMno4@Gyuvb05Dpk|kOjDiWedm?vK+evLMr}D@qLR{avNvDC42FuOl%b*
zxq_v#5f}c5s$ocsXz}w=(aA6S7J__Cr2<h9Skqr13ysSIvv`OEe0R)as5(@i20W#=
zS0}@W5lvRF-udrMSV=vQ|8c}!p18J`8E<h9;nb&qa##oq=|~Po37*#r3+J&ne;r@)
zC~UI;6WIj*^nmQ9zepP5%nPqqTe1Ew+PZ%c@eL=A>s1%G3QdoBg+d~z@>N|5$-z~(
zyT@0(*rZZYAra_u99MB-d#VBt<ImMk26Bk|@C)>&fE@&{n}z`=#`~XjUt?_TLn!{_
z4jbb)!?IkTw4hr1*Q#%{=YW45uLLIan@14Y_|87+jt}>n>3Q4T5-)#qie-uG%uW(W
zMCl`_Mlb~j8?#RgrncWes>!c7F{>9EP7M9qxHInEH2QAb`R7ql4+_YGRhQV#b-o><
z^9+p%&#H(om+4mX5NGfuX{jG}T*zt_Zj^H4PB)Ph!vrX~!KfGsdi=0(kwb$oV)c5W
z6e@^3>11D@OirPZ{!R)ubkLr6vl59b+}Ayy+khWs$vgEXiR9yR@@*P(oc$LbLZuSV
zU4M=O@s0Q(yDy_;Sy|cDm}jU?Jm8OmT)+!~8F-L><O4CxL^prMM>HiUXeospBcxO=
z2BOI>@1PV$jB)pIodj~?PvgKJYlSiRqM^r>rVtl|K8k`w@mdf3F-FdX3)|sLf)SkU
z+X#OI){|$spQ>^U-g3D-#TE))e6d-xwQK)CeSZ^oI_Le@JKk0tQM*wfvv^*v!B_sw
z2+n1!C4n4_BbVYplK9@IgyYBRW8WdDC})H3;>eu=rT<6xtK2^^q{n(-B<rLD<RIBm
zE=ih(P6R$L#sUtMqzRtKiwJ+obp|sdFkC)FO==RpIfJW2;0t7N2(@83jDV=!_{Q@t
zRCM2^q~gGP>i)dJwuhBgZTf7Qd#bi&IVh7vIr-Vr&&a`m-?VKw(UGoiZ@kS7?N7F{
zq@h@o;Q|bCjR_?d<IG2K5kq8|YgCl$NX=V7Q+CQPU(3UVhI4DdjoSs?xbq3W8~Tlt
z!HgfyZXeQzva}j|UOdBGI+-5fM<$VYv^G9i5ihjDhz?haG$BQVzvS+dBK%$us<>ee
zFd++<;Y6SpN|JGq+vPhsj64cL8jBe+m1Gb-gf5n^Q1HQvthO_`L=}RY&So#QiE0z+
z=<UkZ#yz@}>s+JLo5!d^1_f(clS&2RP8f+1pAen?5ixRt94Qtsl#YakuEmLq*s5zu
z#EXo!X08*&g<+NWlFjfzSLRX{ZS|MH1v0lKVZ_A6%_amHq%Hr1RIYPn@?<0?^7^mB
zhz)sPZR|J^F=6g&k^cnYFM<&R=6iz4#z3_hvp9-3v8BqgiIZXn1ws*r#LM_dDgPB3
zbGK}~zl<t*Q?no03-T>TrVb|7s6&azL2sEZr<tPpNmP*=9NkAJj#<3&AT#?P0rCIw
zsKvcEF}_zYf*xFzjgjJKxlk`BCd<|$$@r-p%{9?W4+POven{g-wqn7E_~>2~1bRS#
zr<#(Y5s){Is{r!I{j|SQPOgz7hjiDPD$B)FoD&n|R_e@|g+KXdLoWSl$K-ozaTCIv
zGf~*_#lO!xF_VKwX+a13wDGF;wEb!CcyWQ&V$6Ffs2EX(Wc}*a$~#>GE^dF(9R}Xm
zkSppDKsa@u9D$j9F6Q*%dx8k4@fww|a&751ND8r6Kj|rdw}i<RCzw)@DE7|y;2>he
zRDHABPn$WrnuyjuS0#vQn0c-LNOBQeoi!baz~)z+dESlNw@-v8x1j+Q-SHz%1csbN
zWZ{D;<tGBj#E`Ju&Cr}HKm~HoF2bAUq@U|bE=MSxG>DBjv845vZ8;($b5@BpAk+2J
z3>AwB0#fBi90Jx8LT!+9I7+PXoR~{GmVhiAn6QVR#)>4f%y6RdCZ+uM*wMLIaTvaY
z;Zv~+b`-R-p8_t75g`b`8L%Eu(9QLMf9t;3|AgN<Aeu2?UKl1OV<F_(_3{IbQPWx<
zUs6fIJfd~eSJq6tp)vdiruTmUAWoauzvs6g2C|)&Km`KX4qTus?*hrRV>6uIHn!nP
z*WY{Bos77UiKBWA2IZW%tWN}4Q<hWu_6@<x10<f`ih(z2wYQdhL)I5Z#XDk(P-M`{
zFW`d(%cPwss2Ge7c&aN2*M~|}`#kc<LG*lpKuS@1O*|C{;Zns0T;Mv&_pQ-zBFnz<
zf7T=bCq}$>j-eIJxYRGBW<hjKmtVri4V~pDI1fb_sYnwleU?Wc7W~_qJuSFfaI;Nc
zV342Vj{&dgy)cWGTo^No59CZt&Ubl=dkKGN4LRiE{M>zww~QX<DSx;4N)$6R1~e?G
zPooimdBAaMvx1|WeiUzUPU$S1@?oCW&yIYIJe=fDv?0+E#uc9Ow{D*lX<m#dNUae%
zDvn$qtQ4SLpNKDZltcp{a9zkEnCh6$ZpDNz*7NU>g-V6XNxUN-pn8fzkX$2%=X^{#
z(Oh>pU#Gg<D0<`0q@e+&xPlMx6Hycb8N9{@<1vhM<N?3?s4Zr5I!{NCN@Zt0^QGix
zpb6&~+HpAL_=&^`Cy@voQDxlT_yJ*nKsQg}=6s8H4vKvxR3wW2IF)g6Vd^2ViVd6@
z-&)4=3awpJWsDd@EetRAYBG1tByKt7`5y!DKl~w7sWNF&pFSY3%X->Yz>F1V;B#0w
zNt2DEFyaNq$vChg`!uJJFqAW0<f|N%i!EoxhlGmW@BD-jUo1~Q`m5^yZ3+11;2(+i
z;pUi8>E3ECIb^XWtpcSGqz>Ik9)6f+Kj*m%@XUX$uc!x3LP%e^f>GR$^T~{+rQ+^!
zOwWF%^3m{P+7nf}<4Zn4%g$h<5dT_7@3L!UZ@v{zLD<G%B8-@qW@(d@CF0bqY_+f6
zMRo^T<KkrBOI)OY<k?FoKtb~_${2V1CE#0weB5i>2NzECqeiZ9n4{gD7-&L+bAmx)
z2wz(2uujgR|FrFxagg)eCvL`w5x=SN*E`1j{U<<2%YP4vH>UKJ?(bLX;YBuq7YeXr
zKs$4e00Oa}>53qNgBL$w0bAfIv|$>tSw>W`l)3g`&dEeR0vZy{X&?tTo3~W7ldl7-
zWwbb9dX%sEfTX~t*r~~xh^2Uu%DLD?Oe7OrVJ>-qSNYEn*pcORY#7i>e}x%utY1D%
zs(1zGw2`!vJ-Fbi20Xx{p_MsG7U$+qz=4XZW3Ssik+&Gd84*=<WMLbwQwqa~A%UOd
z@8Z<Q-Lp-;i+LKvTe6`Thj^i1(>GUV-#U@ue+<C?@P`mzdE(`s+fmS1CdQ%TLjhM?
zyo3WYHNWWz=oo$KVZ@61eJ5&D4(GsSd`BVvSA8ku9O<Dy5{y5N`8Le>;@hzWaC<ya
zic;x1@xuD%Eau?P1Ft>HP##5iZN<n;PJESzh7CuXPM%W#&1?Oae-{|Ih>GLg`g$hk
zNq?<#=To)5wtwV!wbWw&V|uHmmnlaA^%l25$B7<&%2WPslz>4=pMjAm0tW|klGo*o
zwF3>aWPfF}_}2B6@RJ!Y*X06Wn=}N0hBU7BlrIUIqQ8Ti$+h&vo9N_xv7BV8;vAAh
zJ1$e0Gm>mvGz)T(h#%>nyoZjJyzUAT4TH54;KG@555a_By)cSPxDaPdILXIyVjY6(
zB%khw#sjoY8jeZ;$I}~#Gjfq~-g-$!i`IIWyhu%~7;z_kb^<1@69P0=u8Ph|IH`JR
zS?G6!nOx>d*iy<;k3(g);R}RUYC{CqFvYb;!A@>m4_tU7%O1vs347Ra1QI>KiGmAZ
zs{VgS=?d6XGV#Us>i1;J_w}XkC(~ApQeKdcl8*sBeg$b{llR3-*vN(Ii5|*zx9pQ*
z=g$I6$n@F>(s2%6Chw&l2iI0nW)sCbAhYEH=>;QxZ1(8d?tkl@!2cM4TZXFdOy2T{
z0@$%jC&(8V!>1re=<mo>U>iF*2B%Mt9lK)1-+kM^kNDRo#rFChZ08D11&hIQ*z%aS
z@x3W%ILUkdwHqs`C}+A#Hr`8H$OD|@@uyphx-@52&PdWo;Ok1)h#-U(%LZdd!K==C
zk7e%P^Nx37Lgq%|q&`>MPeO2=xSd1rrYyM)UO@6A%7!0+LI~Q=pW7?N60iduVgC*b
zi3Gk<;q=M#!)*g+j(jzv`x%b&Rrp-LC^^61NemPcIQwz8iNtT0r~KU`0lP_{4}KW&
zIah~rs5l>)PH8UVIMAG3Vl5$-8sy2>$bRbqBx$&YwZ_e={ZR(2<U?RUcTPtKoY$|B
zOA^-S8qrC7PRQ3i`AH5!09VC02FH9uIr$x5HVGo1Tq-17#l?QoE-e&J@xE-H)`eNJ
ztf*msN=|1YuLDV>Vg+)D2H+&aIS7pBW6GK*v4HJ_2fs@hT5~k2Ik_ZIru=&dvhb}^
z8(+#`ijTxX8ft|jSaD#F?tzLCEqI;#=1Z@_8ws=i9*PN{3k`ejAPUIB{9J%2$87oR
z!kKa#x)=piU>t#MyAq7Rci|)05$It(Jmv46ed1;oeegj+oL@!(c{uuBqPgf#$iC4;
zs!u8{F^E^~&uEKry)eId%-Zk&`V0TZ0Nf(P*PD>mD;rh6py$yaO%4z*N*5^aD3wHO
z5W(yuAD`1Xal`e)<U`M%Uwm=9Tm1+675~P!z1{*+6n~Nc3jugVJ&7NiN&dbPD*0$t
zi>^VsISyE&$hG94qRv=CAtSjqY7;@ck8aQA9xiiF<6&H&6d8-($BBkLw&=D0)b*jo
zl9(CLw~Zi;WG-~_=Xpla$od8!@&OkoSnwtQzp?I1asK))DfGO}3xR??D;9-7vW7ey
z`k?Otz5c5AdUdS(Q$bzD=uFjStFvF3LKYT+CoFNN4mKR})N`jxz(sIJy-NgwK>T)2
zro3oC>Ap{K1Sh7%PzvQNehQv~nOH*RPR17>91mMjxY<|Y-!&N(xopXszWuUKV8DPZ
zT#N&WAu|R8P7KK#F%eGTxtB<%0Mm(PR3m`PCoYhL4>=d&sKPS&S<);5)P-!#yN_Jg
z2GJk*QmM!eprYK#8bIS3(Le__vLEE@jF*em{-O%uqOX59!pR~yJdPx;5fj^uid;Zc
z+(#j$NQ-x223(b=lSej6Y+sCc<BVA0&UWG-;!c>O4*S=}^p_~!?;0SkPsNFooX5&=
znR<kN+t2ZVtYm){Ok2d)=B2}$aS9tLZzm!!fa$!a{N1uoG><@};vx^*o^#aY7(ex^
z&T2Z)=7NJGqA@mn4c+u_3?{74J=d(+S6}`47yXX`xJ9VK#Fu;h3^ETt61;*Kgb#<h
z%$6TwqSA))qA^!U&u+kAd^nHg^r*38SA1OZGsIQ#nKZiR=P0s_3>0h7F_VW+&l%V-
zU|;M`^(&3P=Ddp~U>zgE(G@E}D6gH?1}bIP^6s0;IIW8Oi7w&+v0HrBQ)1-;m*%WT
zB5?(_^OV2!oPBYamOrK!H_NH4B<0<fG@%Jf$d4i(1AZIJ<A)KoZq!Y;LBL<%FIErv
z@WqTDrc2*pp%OLvCY2r?bV`3?_v7shH7Ywwd3EQytS#%mQ=42akvDV{`Y*he@1|%0
zw|BWe0_Td-IK@y}Jvo^RatRxA29m=GOl7Z8nP9|XUlc6J`~R6c6aG_jSPgJG%evQN
z_FTO1$1AlP8abFRzl@W70?+M2p^z(Q4-!Kqa?f7jJbu(mtcH_a#H1#OY-J4(=MIrg
z^#Syhvq+#iQGkYnY(P?g2I4)^nFjI{(-G>*oTI^WP1CiW0Uu>Hb9@ds2o<Hhzo80K
z<s(n7MBvSZ2OO9&VT|302@SQ>CN!KVK}jfRNK1QuKVYvUqFAMGLc)dGSTPgaZwESF
z81@7(p3(AS^CP^*3!?qok@#c8^6a}F<H~=Y(K{JMx-M_>$p2;<Ndyz{xQz@VIG!;c
zA8{Xnc#W9eBz4WcKc7kQ&L5!LjQF|}*Y&!LAde)zKyn>0)=VUt432FWfeoFk&!3_U
zeTf+NsQjL@<T<a<36%>N&a5BXYQ^qb@7;OJ`uEzz3O&<+2sxGS)P>9wKZvCZpTk82
zzJ0SjkNuu%w%7(NF(R@JxF|$@qCMNiha`TnM*op=oF`p*PIjm2OOFtQM(~uAsOTuh
zWKa3O&)FBP^n5^1`BLsgVsABERK!5=u`Aq*#_9G)$OdA*bo0`RW&AB3PNzqsU?d*{
zhJ72TKomKB+E$ZM8(c465ckih@TqfA=pIgSnB4hqk%EK5j;px4_Ug^+Ma3XLBLsic
z=`F<&PRr?IK9;Z3M7-jhObkhw>%2-9F7lTQN5hUYF5*wT%gtygNTh2b5(Q`8Via8X
zq>jgqj(5^xe6Yg>0}2J?p3UJj0bD;9#}zW=PnY-8NemL*QVvicKc;bZHX&lMOmYU1
zEN6>kz)U_az@H>kP|k~eya@UtIf92Q4n~U7JQ6qof0;~vSX~a0N?Ob`s&I<1azR|d
zpX~DMD1{gPvB~&zjQpj&2^7h4=vP091KJ2>vS=<#D+i?iGZ)A}#W*7X4T)+ve7h&~
z7Q=NzuVSMBW&c8~*a+{NKz$D4I{P|D#SHAB%kh^-$(Y{@NfM@xx4Yf98o#)y0DwRO
z3HWd+{ltDEULHg;qqt^sa74}~5aWn$O}np|^~W!px|<G_+k$x4<R>553NlUqPAdhX
zHp9e>k>&6upPXqIF(Sa$mPGQgye!fl)K4jaA6i34DbMIT_)+OA-jW}ooJt(?{mMoq
zC!i&+#l)Vy@<14gq$?1J1*P(QPU3~%<{z+Qz!3A<AFR&mS!ur5u_F>1V&@{6ecKRD
z0o8WBA%Erz+1KJD;(1prc<TZx4l;(>5P(jRZQ!rkfQ!r70M}O?NFa*L-U~z%Jzu@o
z>5}<Mik3ZR33+&7ed5JGS^r;$z=!PBg9)UWT<}<*#;;V&bf#vsth%SvMr~=u^8|1|
z=ayc^L?Etu%{}9IvzPsyjN*Q%fV3?_P=@H29uXhQDKcXAJ+d|L(omkDsyHa-w0j0C
z;QH)GcP7E`BW@?N{CBY9g_5=#6NTuBZ82j<T+9P1BTfyWEZI2D`EnH#i3xM?$3cT@
z$$ZXXu{tYU_7NSlt+ruem$WLAfEW@<0t&E_3H;0u%)rkgno}kdPs$aP37Cj40o03J
zgoYp?_Cf4iA-0kcjbyBq+u(~ou8PgDV8K*+0|p!r`U);2x_=iGoG7Mg9&i6*NNgAr
z-oT0=v~PKOv6l_I24G-_oN4UL*PIihCRav9r@~9_>va|3?<sxf`$`m{$OM;75ZZb{
zE+)jFHwa`A_SDy@$dec=r;)>HOv@W$$Bb6{yk{)$7T<tk0Z3v%afV>QjL)bxB1+a0
z8Sm_nL1L4vEU$=8;zOcPUdU~-Y0azuou2%U0r-okP+|P|{{6w2sTv(u0ShK{|Htt~
z60z<SDt6=p#aW&6(GZ#aNx>V(6Nlsg8B3d0CzJ!qN?8@;V7ZVQh||xF^<5co`;~UF
z&yKYX=nh^s(|r2@U>(<ac<4m}QMoevaonoKEq}6FJ4NX)bR@3*uM>&`N1rHks$e_!
z6fu<Hv5<0{;yhdCzKN5o<aX`(w-uQx3Yz{XIh5z(V?R@c%`{zbibSF@mB~Rt;p&#k
zNL1_<Q~u%ME#~YcZgq1mQ1D@roJA*kyxg}#SdA=uZQGC0p9k*J$K2PDs?^1@xGfSh
zj&kEX<^L&PiGLTu^S=V<*wN2Fhr(G>`h7_$;;`+#rkF(+y(0_ti!xm~(!Lx{(r`t8
zi;)uqo_?9L)S^^kEYa*DF|C(cSzjtP<TujJDs5yphTuy*5&?rpnrc88J|HV9pUP5V
zp=c*(3SYm2l#<_K2%kk&yf|A%Ary@=m7I9v!}Tgv$it3SDFc;U3d;Bc3f>W85r7uZ
z)|~_26#5^W<o=9Wy4V@ki3N549BOhem-_^9n*Btb>qR&&ehyqiAqHyCZAvZ<%-#KN
z_f+0u{4#n1AOwkF?s1~oEoTfr$SK;3h6RsOp9I=AX~b><NMGF6P5MfY^_0I`q6LZ>
z8Uvtazez4;^e5ip2c~o3i!2@%NxTWNqHG{r!HX@Lez?~3`xX9=0r-oEKRD^#o}*BB
zSsH~MolGPF&5YQ|;oO>1=zzmF!Si_WL2bf_cG*r1Ol0)WWew@H*cmRH8|En^KvvWO
z<VZOC_Ly_aKe&DUi?dH0)hiUpz=h`3N!*7{yH$;`5P;V9Wx#!ty&HdC(kNPae&nI9
zO(li`DhxJpgh=+M2jQd#Uge)4fR&h*<lsai8!q=)#EYE`|0pyR^b3oyP=%8%(uwAK
z+Fu^QE~41dZzEAi$}dQFTc_2&|I#oQbM_hn>3oPnTx5){QFzIgFQ0q+K)*frueP6-
zyF#veua+CSRw9`!yysR$$AK~A8_ztHyd3g3pDaQPb-Vv%FTjBh`n7Lypb>P@i~?vU
zFA;);s1pxyi8S)hS0#feLQYrYym$y(c6kvd56Dp<8}N)62;|WNe;|xf*a3wAd_ZK#
zASQyGAyyJ+K1z4OXcqh!4QNebR5TiwDO0`;S!835ABRdga^m)5r;vaOb1+bVD6L}v
z93gMHjTBy)xL3RXpB;u<GL8~3VnJJ*k01!cU~evP3TL{412dWBUnT-Gq05fqje=T#
zqi6h!<IL}D+<YCiMAyPmv0)znsq&nR7xkxX*E7W-nWw+YY$~&$-(I}Qr26b5fR4#;
zh^PGBa|egpoz@2h6JD1qV#SDGpEv0xno?zV59L9I$%in=S9r-d8P^LhGztA`)o&~M
z9|Q0g5vNbg>2(0q@^ZYZHt2Q{Vi^9;F#?dJj;Id&rwnh!60iva+M({`6NI6Ul{k)b
z==3I<vXCfz^m*b{J(T>jXUB~BH=_mO-kjX8XB0wwqt_Otk(A{;vR*}wQ;L`k=Wrn~
zhWy>v-0Tb{QGp0D(LQ>ZnfS^|BUUnuXz_*3h=X%HP=5zm1d+eg9U`SyxZo-O$0gu2
z3^?c2*G4mqPESOrZ?l(7wHFrFaA?U$OxSSryy|)S-&Ua(bM_`2=s6?N$s_xzu0eHt
zPU`EI-)DK(S<iF|I<9WeD)057n<+rU@y^#cP*A--^pyYS4*Xx<TKA@V<_eksI);pT
z*zv}YY(T?_<H7(eq+>p_1ruHbogRb>16k)?B-4azR|d!yqPjSgJyQ6~sWQO%E#DJM
zWgwJmlmfsD6}R>C8ZeGZ{0JBsPJN&)LkR-zCz^iIkCROz8$0`uOC-th?Xh6RP}zz?
zA;z+gW5EYk_%ncp>Tg7%A=I&y>@)CYO_95u3J!8V8caMRkRv{1Js(or=EHR=k#ye`
z>T&^n>8BXb&|-@u;JZUpbTa{0(NV481R}ZqjF~h(XT#^o7)5`%Ub%;SSw(z0>M<-_
z!fX6TfD56X@Ra}a0z}C~@qW*z5Zk`C6mXFXefFUdhVw!u;h0Is2gMX-ZcqM*5%=A9
zGd(HEi@!u7Zq|hRdiMq~OVp9ExUaztDAgVtk(~Jc5GvrRcE$7f|E2bG|IUR!rVVe)
zv&6A{&<x=VbiRI--dv**-lyk}9k}Af8xr0>j`-b^eR?znSxYyofN08-QXPAoJS>Dx
zsCLt}oVNnS60ktva-P9NF8bIXlvP?O{q?sreVqEL(o+s)18-BW^ZkU8h4a%S6a*rq
zNW=2`60i#er+otFDbK;4M<~S(?pgB{g*docC6rRc68rsoNEDRle@pWJJ>0CS)X<Wa
zaGh#o_rDr8%D0i;VaLda&(vOQOEyfC)rc0GU+;iMG_j-Vqo5#_zj8sf^yF@kNL!Bk
zXw1HZK!fW|FcXQ_<vQ3fbMahNAPsRQ5pNU>7m_LDDv^n?{49D<EA5Ir%L>Zn3GtF_
z#z%4=SNNF_z<?JD(1>huKsuC}>=a$ZY#B>=ueoS)E|x$egV;*Vi1nP{9x_U&VWSW#
zL{N(Jc=<+C0foKu5=o>wrBnKnLl9-7UckkEau>UAo-{U_!QFMX-RuPGLOM?axj4tw
zme)iO&rjPVUXkl%71@OEct{NCxtmNZD07bCKwu6ml7R0Pn|N_v2@3|aE1PLf7+(E<
z;Crkb-!_qxVifmzFQtr}hIK|;EaYH&BJfv+nY$-ipr|(m6u?*25xD5~xDPoPn4h(e
zniTK@mJ$7$c3)Gx4(^hU|BI+%o4BFpRTSBtAH|F47^y*E_r)Z0f$XJw-OfG6_y0>Z
zzqA4CxNnI{nKytqI??0#N2thZ5)0aq!%{k8I{oq3<13on;P`Gis#P05zDEymZB&~m
zkw6w^2&ybPaENQ`U&EVB)P@6p)Ban7;)BUZAzE%JqDkZWQxEYRU1Z#{rAA$AYnTPj
zm(;P^V^60bFyeY!L!u(RqCDk`#;3?R&YfIH9v)8$HjcC!MoUsDofgU}Y7zU4i;5Ir
z{cw$k2zLFa#`jN;hxCkxAQ;=>_wXl<gg$!t<WRd7f9k~+SE~$h`1=MjjX7k_)G*<P
zeQ@qgm$xYBj#u6O)Y<M7s?O_f+o2`#N%AP9^tl0;aiE^7ii=Eg&PR}e5BW*=;Dx{^
zbuc+%IlXe4lOr#%$<>(<#*+br;7<lnii_L}_jPG9uvRyWKkeU<eq@<<Hmf<sg^;d9
za{+7AY5ee|RJlAdh{9I6ItnJN6<4_9=gXW&;)19=e6V5)NkBtI_sc^Aq;VugQd<Bn
z#JpBwn;3-u^Bv_7`3`3bo+FnSa%#QC6*MX=J47KGhAJC8<9YYlfOP~ScR5F};rnzS
zQC;+-%Mz)+t~AxlbH+7AdqR2T#R4=Ge5aH~MMXD-ddlA|@$!Y9<lA0&p&<@0BZHqg
z`N#&!(wsV{7om&#6o!LU{)Fk%R0^=+>=5R0F1}mv%kx;FDn6)$O<zkB@gO@!H|BG>
z2%9rSzRvq{9qNnbru){`MEf5D@Fx-f^~A?}eF>DsM?9g1@+f6whxzAmP>5|n0Q|zA
z_rHzr&9*H8S}YQsun889$;<dn&SR)ZqK<ZrT0BS$>gi!)$F68_>#vcmymxZFo(oWv
z2Kr#J5}le{#JlAb^6^LcW{vwgT<_ig@bDHN)H)I+DVEBp?CY1vF{(2C2k_@5_8Kmc
zMt?4jAB8_E;<azw1Y9ZBloY8<ui=Ah+E@m2nS-xRB1bG?x>p~5k`vkRRT_eHtZlyY
zlq#9t(>h>BLESW*NQ^u^@MN7;u@{{FDc{%mx=an%6OqG+A&_#98E{Y=r?nelm`OXA
z&qXxs31v~p!I3(Ix}qzoIp?{=b(!Y4%6|Ess0;X`5(A`pn5gr_paI93Oa<d=`iif3
zKj;PW<kvh<>SOBRi?Kp1shF_G`J)j)P^DR@WFX7erjP=hG1CBv40!_u1;uZ;dtQ{&
zo;fQ1zZn1bo>39t(x?gOZ-lF($d#W=8KOPS8qr3dhK1o++M2jV1`!MT<3dM-&-0Wn
zdA<^V4aNJ!h6_Ep84CI2^;^z4YLc}hl1q#x(|nLtJWTz!ESL$!`P^yGWk5xYZpmEl
ze+_QgJ_m{E+b`IPg<gX`BZ4SSrEQ=*I$4#zCYY<7Tl$1MZ$jNv@FWCx<L-&75eW3c
ztz5^&mEj|mVM;kA_mczqBl-i{J0P-IN%{Y;FSh-2h%20^_I843&M4Uq2*N>n#Umu)
zL+obVt!j(C=Py=&vu(?c(W+62Br=Ab(O+d1`hsf8YP!(|<jZ`=R>Ya$vDa3ty>-z7
zaq}m2>KO`eHyyKPMH;-^ZsOa$Th9CxK_MZ}4gQzo5^T7&BNkK~SPy<iMe+#fGnp+I
z2|aT`?xhCXef>pmu2K8F>$q^@c+7&laVNWmp8FFi<YMg*Misn>{joA8Dp8}(2GY37
zflXhaqhSsC6ex-MuwQyXyN~H?ygmzOHRUlpx4XfSay6YsYsH!kV!T}|Kunk3)B|EM
z;0>4%WCD>KM}T^YGBO&pPskAw-ro?9g`dK@(-jNxv1Kt+z`?jC*su{DcMlF+m?{rJ
zArx!5d^D_BO7+12hyaht#L)j=S|4`Ld&~r&qUW9D9xAi{ndkVFPsGcMwqVhlYUh9&
zid9K@Vi~zu(B2#GmYBUxd+NDcoa1bsHEY2@9@^m%<g=UOedb{$mU~a-(Fh+d27g8Z
zrKvP;H8#M}_3PW+@LP;uYE%^<6G8aw<8p$S!@w6m6EDdZxoCTrN<_1w(QFh>BNz9h
z08q*oMNix<z7oZZt{pqhYgZ^w0-|44=AqDFYp&~%IBg6<DS1a;2}SrlZO<Rs0l}vG
z4o!Xlu(M0_eBulaf)O7lA`vK}#ov5Q@%{2fz;NvcyvQcvQ9%nH<ns3q$|f>V{0QO-
zZI}Y52al_?{E5H*!b)FGoY-p<sJ#$;k2^pwKWYn+n91AR8WRqjRf;&v+ha3xXN)uB
zzdnj+;;1+1X^vrLbLv=1i}utBcX3U=$+`J&BPb-d{VO=JqwN@lg`bceuXCQ)X_+^l
z3M^#RYrWAaOVE^M6eWr&*(lZ&Dd>n)0}<HpYIc#TMB_Rc`^3=34GSlLnk>V#5YR>|
zA?mA918Kn(4pu9`f)T&?Ex1S|pvs$=k?2)cVE~-5V=$v)2y29bj?;Iw=ODe=BvO*>
zrNuU2GYYmV!-*yjrGGBoR3>HHXWYXv?1s^NN+lY;@j5D*<h+`N5s6C)_tbOuNWew5
z?I3gtH_m1N3Yh~>vw&PKU;h9L2AY@uk`%njTsi}Rz`McD8=gpR-03EP;(oYLP@Fw5
z1BIh%3Y)%7_Ita7ysBOl`?!}z@@0;8eU%gn@SXc2It8d9MJAHFMN?F~TfI?`M{5v>
z3%?$3P(VEj&vv`hy&(TJl>X+?npqctW|+ihmP+DF3J4z)je<arE4q}T=s@|ahos3<
zx&R%JeibReNa4NDlTR9<KSqibA_Hq`g8V?Vm63QyeACoYqZH1xwC&rEAOG_&EOom1
z;u=XPGLEwdBv;ABr|mof5EKmg5Cwr$M*huy{Z$kb9Xk?dr5Djj;&9Ls-r^%UeXEJ>
zJT0bYKS45?l>a#m6M<wjD2IwKtbJGR;|UsX-7Gd>W~bi?oaCU6YXC?Lv88_Vo{|Z1
z<Ff!K@+Xsgi!0dY3?PuRIGlxt`zJ*iUxM1RVkc-H0EehSP}5;lAQ$t(s~E|})cq&C
zk?8gA#e_gF+t2NO8C1sY1ZXOX$glE$w4o%xbmRJD9f`reBbgZRyXPH}@#f(jTUm_}
zk=jE}C7y2kKM>ow!ui-9n1F(s5uP6X@-p3$)yF>r>FJBWj=)&49*5D13S3D#r5Lnj
z0y$XlOYO=}SU7d;c}nBNr`Z{|AAE2fPd!Bv@SUm|DPrVo?#z#BA5>ge>W`(g2o~or
zrtk`Fc=P1G5nnW^vHF?QyIS9uki2|u{yr?&FgI)C*0ULdJoOZ5ib^t_bobeSg3x?W
zFrejJa+|)ACSeDB4i#b~pTJ{Zkjiz`GrD+*AY}6KJ9WYTYlv?-F{xK;1exq67>Sh(
z0`IMeAdjp~cEFAdP5j3iP&7jpEXd90sn0R?4w%b%9wznLTU-|p6N%61<74y{mu}bw
z|9Jm~Ogzy$6v*N%1JrLg#LrS&xBw$wr1ULP72a-1saOIwprL7_h$08aq#qfClXDXn
zG8`8zv<mVneOXog5iX)r-nv4xVmslNbwaQc-uM!Ec#-?<e0Meh{|UFLKYt!YVR$K6
zNE`7rO*nwxjE7}+8c^RfTs%z@NBkR<=@Q&RYnuK@8>QM~_D2m1=)?x`<d|+GW5qTm
z7abQ)wa97mZV88iB64m-VBYi^PNE4K_#-L?4yEiO3XO~y1DRYWZgC(5*fBns_N(y5
zI~_M)#sxwPCkxQtunSe3mf4d(mJXg`h2Dn>@|ft*`3G{yA>`Ai5xDRPIR=!>yb^y6
zR?pn}5{eO5LJhi8o7}z?d7Hgd-t`TA=tmvHRo1YP&2NlGl7sfD3k4S_pZ{iD|1C&B
z=#QbGW3c9+;=+}lfeSNA(q>emP%pMe!$|&Ff5H%ipDV<Ok&~CVW5q`F)x9D@UCWDE
zr?xhD$45J5>mJ!9E?#K~$tMoJ&gQ#8_4N-!&cr2hkOZ?sE(LN)f!5~$JN$!$-}Cw#
zhm(y$WQ&TNpe*$#SD~lqkGIcy9+2y3Lw@{>iFJDI{NtQ^{7(}n_RIv*I3(W$%s6Dj
z3VHYufc?QTp0^j_cCVA7jc3P&imFZ_i4ZI=_hKZ~h#wO0T1YwDy0d_TD^xkYj7{X>
zysV2{v0k48q+-R<wmSle>kcCPMbNP$QU~E8pNK6Fa)bbCf|y4l(KTZT#7MxL`xC#)
z+^Fs@w|~op)4j~2T?08F9uym<>j>=RE$V|nzsdW#Xk71f5?FN0AUM&nwVsNT1X3n{
zL|K*+;PW+CxkiU17Zs^U?HA%O;Dh&>f?xIiPczUZ48{9I08&jQ6a_hQfDD)Qg}J{)
zT%tMWK^g2Y2fwoVL9FC*b=*OBf*o5uPx&Ik|Bpo-{Y&t@kq{{Z7lznlEa7KD-pimW
zDP&cu%LzuY&=tkEVi1<8-7w-pe7@sXnMZ%UPkw*Dh~IoK^2`?~9B~u$f~g2Jj7b-;
zVMN*EjU5N3#bFpRQt*05QZN$K?*wLYsNDJr6-dUvX(_Q<d7j|*tzUSqNky-HgR`!O
zpRUL5H@`?)uwR!iu3C5Q<E_W!xC7Rcmi0Cnc&*yX15XIK)E+AnUHg=8{YR#BIk}Us
z%6S_WScVbm_wxTV0KbExRKJE7)b$QuAP*gLz*FLV+Hw8cU;;6s-#_^l-4^%XaC5YP
z5lvf%1+Y(ainB6O?3*+}wh=SrNbRs3$u(X_J=0)p>I&=cM+?+g{f4!BGjMiO=BU+x
zx@1u}G=v<?<j;=s9H~cd_dNU#iX{{W5;bEI8U>i2Y^HZ$k(#lpv3Q-DS~_qMh48><
zL>Fpe+*Fr9yiw{8^pr2@w$Gb2FE*5QtK=G%bFO<c@gWaV(H_Dk%HtfJfIwoHR0bOX
z$gkezS?;CqiOCJlm&<eZ&d97)uJi@h(o(zeqAFgWXW~S`(W2c=YVf-O#rs4>2kkpt
zSn-=w6E7Nz<cVJp!3+9Q?K!T|TlDNu4ILH#`KxeY!Wb6kDSx*}z{Sx5b|mteH&-!m
zWk3y{Kqq?hKJv(;z*w6Tl)`_)+vJjmYh+`$K|v`JC~mg~@K&Pe#8xq_IgS@Om>ye>
zje9YqPsT|BfsxJ8NF(DjA1pYqwpo2s$zQ^Vg48<VzyLW7$j6TJ>QNLVUY91|g{0uz
zdsHTxm@_M|aS-2gF7j}qCI%2A6cW<vM4heKi&yh3hre6-4$b6B$C!eOay!^A=1#*7
zj)B&yWW&6|>7N_tQM1E`?JqkkcpnlQl_$6yG8z{OdC`IR;Db;3BS<8&@-<ADFjbmQ
z0R<FP9E&&Jc-y`ui!8DRExVgG|1Cw8k0u@O@hys+CYLKS(MiLA;i2k80KThdVM0at
zKKL&;tJr&4MMKSxA`UC5&#uQTzGTya+C<1WxeUZa`p7R4r~Jk~`o%ZDUApn%6QAzY
z8Zcr)SjG`RC|+OILPuh1|FGw=lD_ftJfD78&?x-&B3)P}dvsgK6f88iPmsL`74^(D
z#U2`Q&pR;~u;Ba5x;rKzh>if>LC1#AGpSsmF}r5JMMcWt9XOprzNW(Z3=B9?8%_1Z
z3nUUqKH0Me@fJ_WwJ()#<5MlyainlwnQwy+>Yvo8MJVOCzAP6T0#WUvX9VsR!dJXc
z1OjuugL?SmJ$#t!sGcLj=6xcj8#n0_L7l2SD4rd<f~)wD{_a>nM_SH!%HJ*FVMUq_
zZ>)g*xDQsYVtu4L1*DLB{Bsf+Mz((x^*KTOY$x92;MLb1CU^H-M_r7f6w7&}V3}(n
zlhNEeLZtyy*=jA|C)(l@UzJ!+<C^m}CJONiZc7dYI8RRl{4gAUi<<b6abhIrh{or_
z0FH=K6egS09;SO}n*B=RqkFz(tMgOkedA^M71$=^?B^;Jewb5=bKp}wtcm;)&L7?O
zx!7cRR+*)KL(QP-9ONw9OU<M<_fWg+gBUSlto##Z%$O^Oqobpje+D~t>@i=uzlVk(
z5D4FU-3)*-(Ea@v{T7G??{k%_Tz&kDJAWeoA;f<&seaExpciUql&3&hqP#3Cypv8I
zI@*YkUu=tiA2%m9aiCx8K@6uz8Q?9OQ6MgF{@hqu&a-a^S837M*bz=Q9D8x)Nr*c;
zv01OG2+@c~=!BF>K*f7&i&~_!Wm5zy8hYr4zuvTe8Aa|-g_sB)r;|u!`N2e0zJew$
zgZPF+OaM9)&Hk`PY$1sXpJijkj92@Uo-1fdCfv6kNb;{nlY+v@7LOB(1LLU)R40S7
zEvs;maD2W#h=zi}r#Qo#L_v3T!h{jS_@T0&tSFjJt=?VK`L?i@UM8fvPgnVV%j=xR
z7whMql(CU>Jf-iJ#5u(bjSohY>))bLn#h46v}G^Ojg8nR=F!=B4{7pCwoG*3<Or8X
zT9If-QIu)2q%*>Q9EyjC7ownU-p3qV#6H@LuK028Y5{4~BQM%YJX;A}^eH|lxSBn8
zo6o!djv|Ks*BY%NRlb4=y|6Pj9GEiPQ8Dt$SyWV<$Ct2;nYi9;O$e=Mc)kEByOIQi
z3Q0tmlR=^-f(w>R{}tq9E=m}`r<LrXO?IXi4x$1SEY6<GLC}WVms1rI@A-KjJJk!O
zoj#+u;2(;lHd`aQ5<vtJvEf|gA{Q%lrvVLU5LcIU(n*hd00#~nWsbT(!*bn(-#^h!
zxDAc}^7se@!qE{w{P62lfmBjS{bpv#p48ibDrYBs-1}M3vt)!U1g_(cPvlkcBnP<i
z{bmBOW2oN$UsC=5Hxzi<;Efru9?QgsOXNQi%w=MUJ0XbXDEH${9<{}K3V4z=vNAA*
zHf#s-@TJPDpHUhoxl;#Xxao?YI{^}bG5aB|6HK|b;7ulhd$!5y)DSOPYBHK_RMnb_
zHkjpD=lj$kgHv(OQb<*<EZc>X0`h7f^wd;RQl%CX9F1TKR_>MGiqW!T-`Wh<Lew@c
zv&c{e>0@T5l7qzYT$pDpicp+aLg2z+JwzCVlzFafaGdYETK~_2n-x?_ciJJOmT!)H
zd+;k5NFeY1M(%g}boP|LTjJ%5!v+-;lvA0MA)UxRd+?_fCl9wLOSGl8e?Lyj4>&Tr
z3f|<9+IJKN<V}XO+r8iaE^fYG-c8aovEf))2RjbVKHG^l)TGclo>VSVt5zd2@W;4x
z6E5<gYEiuWzr!H@I!2@Do4~`!ICmgW(Q|6!gD>XP(xhTTzg!<9L72~cjSm;l&-Mi>
za_#KX9OgXL6M}eCoJ6Ee3vXB{9+@|H-m9JW>krD%kgqD;BZk3GyOws7R?J=e#?2Lb
zu^ZcX{V0~f`M@1ExyF>C9x`60W=LPcsf<$kDi=7<g^I1*D)?$=-JezJV!?u?Y%uxc
zlOHt^6%{o|M?*t1r@GhUzcAk_D(aC!%20+ftuA84iZ!+Row|_!F~n7$r1XkF5lM0;
z&CrP=gV1j0&@hm(tQ!gv-FW17xAQ+hdXi#VFns@{JU|P420V|0R8FQ;6FupJ10TZa
zBR7(d126LZYqAF)F?1a$sCYkc<VIa?l+^W6+`a(p*tXZ9Be`7e*n$tJg<^dR2j^)0
z=x0P@A~b9yeW{{Md^b}LWt?u)HBp!ObbfF*c6K8d`{9EfC8nCE-rFw$s|XC*JfbmB
zdSG3$xy%oXYm-MBC!#l@Q;lYuJ0LJ(i7z75zn-_^ei)G$Q>^5XkKdrk&}V(U^u-(V
zyY;!Zr9E67Ef>p6)F1aE21;QY(*;n`lpUV(cTcoH(f#rcvT^J<K?*yt^>52HhT)Zz
zB$tV<OpVSK<K#jnE^12<**Kd0bi12<i*a*a2|%xcnJYLZ=kkTzF6X^5i+J&+ctGn)
zb&k`h^+yzPikyX?ATXdt9Q2gGH6|nXoJi0k5lFPW&Zzj3k!r?<i>SoWm{3T+I28+a
zqRxJYiL02;Zp1_wR}$KZA>spqt}PT9>M+`5uTL8J^EG+IF-{$6?m-{0mCv~!D-?W`
z^$N-0Z4MbPo4~~_Kin&hcr6g;l|#(843=l5)W6P36i<on%6Lk<ZRPjcILS#)_V4bN
zq~jjN7hinKzvEW5M|5x#bpIsx-<6MV_XPik8Q0L>TF-Kpv;39zGD#-Mq>_8U-!{Zq
zCy8FOfi#*ErFIqVWK&jsRsiTY`fv1**^)*eZ*Gs9pDZAo((`|mGdPI${P%Pun>_18
z(S$a94JceErm2~1Vkon=0vZZFz9)WBTgir7>F1A&{MXjWnH=GsH#>@lkgTv=q%5_l
z^Hnwi6Xqt9$RLc1Pxg>|*~xc{A5r4z#*-kcAh7lM2`4AuqpP^kQT)4k-f=sx57j6*
zx0q^J$!y`^I)3;hK1MdVlzC(}F8s;+yey!Aq3wUEt2n_F==h%coGQS*eHN5U2wkT6
z9^Tfnbd>;y^xe+=yvg=i>roJ74w&Y-k*|2n-6$#R{?Fo<?RTfe&Eo3PX$b6c$k0&I
zKpf3|9EI*&uQ?cBc4J){fe{zxYEwLA?>=7%i4&zDig;Y8&-c~p;U&h*ds}^97{o__
zRbJDNJ*HEdP$J(CK|!E-Xa1jK?asHapcmx20~6Ct1OjUE$7qC*cPW)j5>U@35JVwm
z&%A>PN%Ey7)DhF=)GOa84V3T2vCQ;EQ~Ql%d*SKk##fy3eQ=DmKFM{kh2=Up3$FI|
z>NJCkAMGs7D&;xvlOc1M;@4lqi6NW|TZ%sc_?Kx;Yg*Ad?o+qqU3D!sY}l$DbKAtf
z@9IC&f29A&si*VK<(<nrr*xKk<zBh>(^+mOx`ENRd&2%SbaeDwE7@d|-SKI+X5xoM
z9N`E@{+9{x2I7C7nA>w8Fo-4MB#=u1p_TXIg%h96Zz6D_#C&zT+xcH`v!v!gXqj9l
zg@WOqv4#e;`tT(l!^%FuffcVN`|(GR_vLe_=xCw4JmpLN0Vk1&%pv5EL1^m_2_=A2
z`z<8m&((TeC`BQ08$Lh)=8)2Wj=VaT9C8U9xJRC*EXm)`$4mp+RwRf@Y!~0Xx)CpQ
z{3dKh!HH&A`5(sh-+`TUvgRDr7rCBvWj-a92MuKn?F^?V&qrKX{1XNWaP^w)dHT=U
zs0Ho-Df5n%yhmlW^#8c_`7$G&KO}9dH!Svb*F#Y{V~C{2qVF)_g?Om-9l{j;Q527t
z035Ow7BVP&D~YP|3f^M_`CQcIlJgQ#6cF07I)3=#yYL<iI5Cx4<SBpqHAS})0x+YY
zr*EVjk;DwTE~m@;<^Agh3SGRWyrO<d6Pn2;<GSOEjr?c2VZezY<03kN7%sh!1C{(U
zmoSrp|EV2h5{d21Tzv3FBz!62L^Q!^^DB3M(87~e)Akl0US_CV<9x(+MJ@%$)A>C(
z2d;d_0*=D*<!e9CuPe=hov}&EHs5CA10gVnbfzp7h%WtrTeV)Uid)59@dvk!yV54|
z$Rn?O1Wuedqn~z5=D{g$CpxcWx8#0fISh0Ey+<XYi6;81=LsW>uyTz!&w0*oo=7ga
z<d$##8>hOGiC?Mycf`5Jsbf7iLy)(`uG&?*dgpTTw;VN|o#xl&b#TscJZLrmp|s}k
zGfVL&2lc}<9yQhY+uhDT3<)4zWC9G$A{Z-qbHe2^`r!w9hKLu}$WtGcU&ty$4-m*D
za^Cp@9Ub{E+%NyvP<(Kk@Wyv@P0CP@xXvHT{;b7RaK9KsAUj^4iGq%C{Bt){zC~NY
zgbTx|a@e^<OrKqNQI5Tbqvaa9Fv#mm@c=vc?!7ngA`izCM{ktBX;k?q_l*&Wv|XkG
z1?2X4n`49!v~CtEAzZ4u7)c5--{+yCp+r6PCu=Wmzv6KRD@88mW$_F{(s8KrP?Pu^
z$C;c{71xH<){fRZPCDgrEqf7x39ahJyDVCu*vntVfv`p3OAy{89#@~$`!VCADSBt|
z0)T8R9%lG}bR3+d@I&{F>e8=h{7Z`O$e)3mS=0gDxdsquSx;g>!S`xeqA81c)l#uV
z1n|W4&t-F(u-iVK?UcpervVoEB!?=@Mnj;bKOwS}Bt~Q&NVM)9DNQzveSP{90fo!5
z-r*G3%H~z$5PULi>Ph2i=wRzm#`&qw808t|eg=mM6nwFk?tm}8_?27bCcKrakVz()
zmFE#iAc3(H-A-{)9$$oDr<<VpeB~z0V)`!$`yYTnAWpqZ3^ByC$mIYBIB@k7B8en&
zz-e5#aDCR-quSqpyC>~W!w*0FUVcu_kTc|rw?9y)s#Ddeu=>5#@2!6Ce+&RHe*F0H
z5I<Did-4T9VkOsm8M|m99{<{a6NSK5-8~QAIo`l_6l|Bjmj@`%;TGPa8an<L8q3eb
zP>!iik*(JeX3fZ6F;R{*jAtLax%T=)WMC(6RC7RrQp5n192>Ck%z+DULPvPfhc(Qb
zVIzd$bPit2QqCeaK1miXsCVTiWv{z*5j32be{=!Y$nL)zL01e356XSmxDuW(>T#Nw
zg@G7wA=)_HcRl=Ly}H?_!G`5f6!)<Q|JK)Va4$cl!8dr}KYR~<7zv%x0Qftruy`FS
zNK`ovh0>U&N9%7<9m|+gU6<9;?46x6BWn8R>tD1|uwW!_VK7c4_Q*6(`MX5|E@o&X
zCP-_B55YJGCdfy57;XI}v6eihDUC?uE#4Yg89R{_PKX5ry73K9`I3-V!Y!eiFrpPc
zg^m+f#*?_PLGo@2iAJ373;5xC@&&Gu&BaqY@xsK_a~U)dwdis6RpV02AzJ3<KD(rT
zRi%<PXXLjZ&(jBy6P;ai9K&!VPS`s}wt&po%UxIpL2s0cmrdYYqb>8qJATn(sMv}g
zS&jk<DJWmoO_-}TcdMeSg}S}mE9DVD00Ct_aeoiByPxyPa1(ZM=I;*1e}#&Q`cb~}
zit>u`%F_e&E&3LHi{V)gafm}puDILPvybQLJLmid;ChVP*{|h#LJ1}GSWUXng)WG-
zeb@F~+ZXx&Jp)kX(}~@BJdKba5ult$5=8`#MvFL)k+AjK&@rL}etNsx`3F%<NHTz|
z1!eX1l##n19GwGuUCkPXV>eD?+qN6qw(T^wZ8o-T?bvFZ#<sQNoP6iHf5KX8*37&Q
zY-%+<q8v0uIY**7^Q7{r8FanPS@N{G{BkLp8urd@e#Uai-b@K)dlVtb@21P{UABSZ
zQj5x+3UPHsSZX0r{+1FH$hA2zvB`>%rXH%a2?)BW{Oa0Q$>yGkx)by6?jZc43`?5D
zUbb*21516TAsrJYwblBG%Ij70!WmS)anl*fdTg%%vXkvq_epQj&U2P9AIL}WXeY(c
z@~_7W3Kje*ywcO_)*Y$X7<M%@sy?<~6hVLy-}+S~Q?kQv+3PDK-&6-o6mYp6hzZ>?
zIAVpm{e=}6_7HoU(whT7Z9{!&Ux`4X+2T_8O_na?2INpo1<@w>2xIlQjXN|d<Q_S(
z0v{U@yK;H^3o>r1z~7dn*lu@96L60W0p}I>gH&Hr`B+5~^l8!|ISM<k={DnL6!;)^
zhBET#a#^o5wmXDjoPZUZbQU97^MdSffPv?0bV#Bl12E&%!uJsV%!t{=1M~K=*Z3)X
zHfa&AsB>8@VB!_46}1DGH#hYXTTW}<_}vGnccw1uE^{Q8);YLtjpQ2Mw8V3wKYPg0
zE-0e!=&w!k*&OnzAWV|@Btcc2FhHQ*R$+L_BkvTYa%;Bvv7Q|a3LbVc)%!Kz4>^3!
z5W_S8@QNw(h$<uC9-n=mnvLVP$<un9TC<3Pf4>aw_RLZ;;PAk^V;$g0@kK|_e}b-g
z6tZ=zaWwCXFJ(Ri(Dv7&ISI-mT)vP}1@3Y0f;WaX-`}N7LhOs(*ffmcuGDW~BbTq@
zZ{8H3p7tO)XRG=E05LxZWts0IDN(L#67ALQA0;~i-N>G}C9wqe(x2NSDB_zVIz5V%
z-Tc72=a}2x64tBeY4le?(f8gf@x2ddou#fjYBe8;?>UY~q2#U)o{x&|p9>Q17aIk2
z^Ss@r$-rN`S`I{&xo)(1okHxI9lQIvDfYIjp}4b&Je8o?$XL9h%HC^|17bnAiz10I
z>z+=Z6BV9n73!5^yGLM)6NGQtrX5KX-x)XdZt2Y}5wH?Y8ESX;?o7Ci3|<iDDQjRa
z1)8qlTi2uum-W20x>?MzPSF#X<%c09f2WrDjL)He=#Ali;i22-1|`bMKlr1Pnpx$b
zPAGO)pf{&>6gRlY6MIr&dA|dvqIzNM7S>3Wn2%m}Vqv{sS2U@K1aHA!$|3Z*f&@cF
zvuk7^I?!`*!$O&A4dnB)J+hstlb5W*%3x==oUCqYhv)Y)jJvNk(`pUZ(|AJ>>W7ch
z_DMpzXdLp{@wVQ~Bs@17rKFTYv2f=jktcRA@V~=5$TVO`<GA+rygG7*X`qtDajaal
z+hHf-ne9^>HJaBc0J81mQ!`SWw)^CveP5r|TWs(WgwB-nRxi0-+Z8RVGj6R&mnVqn
zS$aFL3+p~zxS!#~$S^%%{Sat;?U7xA@PaNr7tgD=%*HBNa&{ao)VT${KINk|8Smym
zU!gd^DMtdr%5mTU-CZO~(J4%JpBSIt1WXCo<QVU7>xUBzT6^Hhd$`|2uB-oSVuIEZ
z9h{94yJ=KSD6kTCJVnG1&wEvbXu^|#v4SRlFH|oA#fdCWb|PVg2O-BswOc;|Q|bcm
z1_d-k(daX|oC-!QAx>~D%HA2C;h_vM>IsjW>WxJ>g{0t4lTH&x^Up6(!}9w`34#uM
z)I6cHCPgZlU6bIytHP&{0v>Y#CoG@`$us}1la-u|>)s`<?-|kpe!GVFvdh22_#dje
z0sngIX0rAo=;i<xZ@{7Jm2w(C!i`3xPv{&}$;5j(`47J;f3$n>zZi~mL+a-vZRIfD
z^|Z%m_(1~FZQ5VWUEXiQH8}2PM=a50`fzC0UXD~td7cE;Cu3l13x9lSIV5Y?b?t|=
z1s4s~`yRbfc^G-VizzThIsUiHg%kTE{an5Myug^)9UMAV`8TM9ct}qa?P{L-l7f7O
z+Hk`=rCHA@DcX-oPmfHBI5p0g6b`;Jj7@c5EYOrEL^8&w^2b$L>2koWQu$AHip4PY
zq0Woz@-Tagc*uBis36D5y+=U#k5&#(MpTec#$z%km}afGtEqrVJ2zlyt@etIP(ZNh
zm(n7-f)l$jty+cjQls1VP_FFGp0gmDv!mx9?Qmz#t}$%~tnM=y=kAlu*;RAnK+PWo
z)aF8rnm_EONZzY1Mo5ulNvn=&I==y1zEvc`o1c3pTLLF1C)WZ)N^#B)!*&oTq6ubk
z>#&YXtoGx*Irk$Axq%&<)1P%rhJB8oT6>X{fY3H?eYDc(tHQ1mM4PkPimuYD>G&_P
za@L1`XIki+(cTBrcNIkHD*Q9UpfvX;%gzWe@+D8nj}MMK`_t5h!#{L%A_Lcy42Us_
zeyOkKJ|XUqDP549^uXLz#21_me%GM6wu!X6RHZILDqd6`k>M?T`>K6QC)`%x%_w>1
z2AgA-pCh*Ptc@T;&Jh2C!jR&c%({y*Hae1rj+;~`0^=A+X%uzvxOfwPLKypf+Qg|~
zY?A{5P>iE%f6%|a7eN58h4$KgkZ>^D8~1&!6GSj}EWo(l^M}&WWiiA(P5%PF40FV&
z_2D;{9bLYCwvzZ3NbgL((ftXozbF)y03R58BDsm_>9%{`Y1EW5b(GOIT$CH&T#nd?
z-e)gO9EOJ@7Z`@ajrPnA4KaE5Z^`8G3+%QW!lKD*UVJ!r;+J1|fseO~6(*viicavD
z+#w%5t3!dA<7e4`!zVtMQ<wdQ$<i>`!!2i9!iCUAszQZRPZ+kswd}j*s~O67)W`(J
z4s8B&tqTZa2SX!5a|%wclz14T=PpP-8O#@}J(FyUHV^xRXgJIE3BIwOLe`aJo+#P5
z?4whis%+mKn}!K5xAn+)=(Ewo-=p4;gocNmc9t{T%C|046N2#}hhgqKD#=KH_~(Qp
z%6!yw%fIYoCPG?8OVEYqY@U-kQ>Y0=Kwj9QTs&%Ed~$}`O`3-S)pw1uA<*zJPSuB@
zTU^jU?i+AF?nd4O+wJAwV%@vK!dic0E>*}s%DfFZGV3B(#8)$(PkvVkd!eqe^=vLY
zl<f|LPIl&*CP{sAEL>hd>+XPIR;E%!Q5E7qd{wGS@dN$g@$r4n?c1y<n1lB5j-MT1
z{ZT&e#ACmVlwD_i@(~85)b~%wei$!yS4eilE^L+1iC_uXB8I<2?T0QMFm%6wE@ZNp
ztk-z-9&E6vHS9g#`o0Aj#d-BDfe>yLe#f1q8_cWvsn+I{k2~ECC7EY18{0mhUZr4=
z_y0RiuP1gqdYSx4Da{`=RElMdAT-}QfK#(sArI0_;^f|xmp2OMFNq>Ij6|E>im)km
zG&_KK)2iuwym-s|v#oUg=4OO;&mGew!DD1<_-2ZiPYPz?IkD}P)s8!S%AbS2!~T_o
zCmEIf<WZ?9RbW|Ink=DbO`jPBiQ9NAuaMb0(V)YB=!=q}#Na2(1$3&p#>D$YX*sK|
z9aZ)`b#5d-q2B!}PZng@7(Vl>H_^1Eebe96-wJETnt*`@ZGM6flb4DMR2)zDoC}e3
z7s9szG!VsV%RPD&7(X2d!KUlss}xR?vU`G7$0k>$JA7;JRyP5ZK)00H>!k!EY)s?$
zj*D(;9bULk+Q+Z+NE$c`_?rjenzCWPI=A~OuFEyYpeiaxd~?v%?r6l5D`Dk2FpJlp
z$Aw6Sw1X#1R)*VhudU}}ckV)32QhZ~{AwERL$uZZ$N9q4<wXF!dVFn!<(bS;+QbQI
zkjH0TKndy@J*sEo?E;}v8q6x23PYM}HzZx|5W#{=#VS;Xr0yYZokDLn;LUqV=V~@V
zfv;n$%rpDI@7+T6s+QwQl+*V(Srk=@wrI2Uc5Gs?l%u8hz8{{k+Zvr5g60$_&!EL&
z#i?!f*W2~#gnH{yx1<@3isyr88w7vF06yxK1_B1gQrb>#8v$V#cF%7ImJnhDVJ#g8
zyPlRuzpVX71$XAF@JL#3d5~Kw*gf=w=$yaOH4#FNX7{cTVAu1s38MM&zRHNm$<!V$
z9Ha-DYbOh$&|8Hl6Q|cBx!Ouq2V^-l^)?f!xxm1GlaDDZctxMpmJ|j<<5GZqz-u1r
zOpj)|{;z)cZ8$Xk;ZHf(XTgFDp*6*Z4@D`zK$V$OrW}ZQn5lCb@m*5!*iYn0Bn=Aw
z;Mb~XhD|Dp9r=q@7L8;%(3Nl>F+Eob|3~iB6f8O7Nche+9`hks)NxYlGbpwTs!|}|
zp##53aY5jqEZC_B&|1Oso%?org{vdZJryA)g~ZgwqA!N{Y*gn#of0*0C;}4xY-<U^
zkfiZ?K@>H3cR;Q71_c=AH$_fh`dC(SE3tRBNbpSh)A3`3yueKsz4l#{!-A~q%}w3*
z-O@uiI*sz*WhcjIxDTE#VbXr)u|F;+t1@aNXkbo|yZa=P*6isDgWFxrOGi&yk$Xiv
zSoh-&^g@LgDs{A=E9JjP&rOxjyy%})+LH_rtB8drzb5|#VT0FJ393kPY+-&-i;#Ji
z#U+G-9-rE1c8I>*=ec~x79B{=Y2i<2#&|F2WY>}Bxo?n~u${mx7g#AILUk~fJ~!c4
ztIQW_LN0%vLU}^QI2|oaDD_1qcusip4T<nAo{_deEEA1f`s&rTW*Mg6bCrl+yI1Xr
zgtfzVZS%AZ)d={n+noqJ@3?T};w<Ki-{b4<p{6I1ZlXE^WjO=2&S3r7(u9hxuJfr5
z?U98cVrQIZCsPbr#)wGk8`jH=KIGe0{EG|>-gK0;0{5P<nGCs1i(%y~=O@l)f)niL
zIxY2@8R`KeuQ=y6@2x?Cn+;L4Wjbs6V1wxEj$ar7plBB1>fbHOp-qh5#i4vc{R!?F
zR0B=|$z;5D%;V8Kk!_J?pps}EBp6Od^Q+U7gjC%2q)rYB7h6@tH}yVpwP4Jewu&@h
zkr(N5oz+9pG!VW#5I+bXc5G+K-c?X<2m^UFJVD&<^gYFD$uReI98e_Lz=J2EpdA~W
zJ`A%Z?{MNi7~Qcn`FkIS!$99k)bC)X$Io!7uZ&1DyD*t$*vJ+(k|;|u#TV|F?nx)^
zH<^h7G5H_g^|Jm=b5FWB;pH+Y+>=Vxq#G8|Fjq(n`KtCk=;Cp>W@r`fy;IuT(Z;27
z_-@9AOA(*kDmE&$XS~4DmP=U=V@8lAPsj-VG9FZ``@0am-76;#?pnjqq1pamDKap&
zUqDU>j?YcFPMAY@vxXNb4_X`L@B#ku%G%yR$WZ)&0O?`k%l_fhJ5cEr4vdSh_WW`@
z_REFizhc#C6GRmuqPP>p`Ih}5dE9;5Z}Ha+hXVFI;4$#Y%Mf_Xn-@pNhXYB4jfaG{
zUgQsmZv@+7$+{y1c(YvhxU!j2Q%M4w7Z?0e2U%BBT%Io;=*JtY^_AY%l&~q$<;R$v
zVeokGew;XZd`VM!i`k|WscTq!#S;ZecVlG768b%-YhR>=$aIxU@SlZ3D+j`<YWn_@
z!>Tju$kN|B0+=4Yxo=b+-4*$fO8rE6PNFe_LY3U1gilc<SB3R$Yye1X7hNkYY(lsc
zSO@~?qiQ=`Z1-tstX~P*?7HO`+eQwB3VGr3JEn6C3?I9$!hQ9-Hm5%Zsk9{k&wIST
z+p=!rJL5HtBSTvVYd;(>!@o3c1tZ&wBLyG%5KK`bmTYoQMktsct4_-`Z*5#~Ks*=N
zYoPiZ4-w35A%1vdzEPR3FSjf%i!4s&Gn584A!e_>dPOt(wQm+_#du7H1oPv=oEiU&
zCsH%{M86(nTb%cyMdxF){lO5US=Oaee|A&|f%5hFSnyNTwqjceo4D8hH=)E|(JWHG
z;{CqA&X}L&&;Ium=IF)1KZ2T7Xuvfl<3Y@QK#s0h(l$^eQvq&iwr&Udj|+W!lI18f
z+>w4M9pniTuV13;@pZqQ6o;bZXBKG2Q25@NO~*sxxvAN8CY=T2lY>Bi6u!yielAPf
zt57Gm6Jj480(##RoxBsy?ZgU}(m%6p2|V_bzc*a!L{3t6s^~tGn3<h^Rn*Q#Lw@EI
z>U7z*w`3)-Vlq$P`5~|1LG{KWS^lVzaf+l4O8%b=QP`WmeC8V6lEsL)6`sSFpDy#;
z-W$DoA%E1aF-EyT!sBx=0Gj_0T~MI~JCZO*Agc<QD~v`6^=_pi`yh|wj|uO%_lKe7
zR9U1Wi_;NfCczM4_ZDBNqV|jfa_cyl5e@y_D(*wq{dQDFsqaDkgKZI>81HEu5iZnf
zXikyU3e@v$q2FfcHHS$iNgmAxe>V_4W6__A)s(Boxy0-rc6K1mw9~agsCfpCa@!~C
z{9@dkL>o!l`$XV6q9F4w>mq-3qvq=a=S$$ut$X=Xm*=xBM<rN!ti|!e0T}XO^nG@S
zW4RjQ<rj3Rde&%j^iCf2wmsfv{O(PaMr!}>1?zDZ=?{TjXrs0d`u71&u+L1;Bxm%q
zDdSN;<qbG%(6RaBbHQ<UAl19B%l|UZUBd1HPm-KQ3AVv)cP-Qg^Yb(EJsfF*=8*z0
zxBnxgd>D3)K-708N`U=J*mzL!OJ?UNxc6ka{8x4WD^Hv*k<AFME_o>El0ZUYl<*r?
zHjwtmBCUr;&FctmkN0;a1l@?$db_vApM)Wk5@M*!d<WBhk&8rHtjO!^D9P@dSHtQz
z!vspc;V76dLAy1o+U+3tUVepnRKnY>(~&9n2j!1ICZB@+9YLw*Xv1Pl4*G+=%Iyqi
zuH78pfw3N2#8`BN#`K3N7iU4O6t7c^>{aTGoVzM7UH`it&UDxOL%tEn8Dm`_Q%7S}
zH7uPOIWc?gh&X6zr-{GXnYs*#7((MD>z;z~Abc1BMJ`VWAHPO!3PlSAZMm35gR8=C
z=UdXpP(~)hflPf<IBJsjP|gvx666*0VZ(~Md|=1H{I7kCHJqyagt(5u5d)OZBx4%d
zc}K0=ktm$S2uB9A_b2|-j%ChN72xb=<63`b>8msUhmvWLmf0<NP;@*Sd7z>*k3&B$
zDK-*zxtP`6A0)-j@}F+*o)6Ajxrxredi;tbRww5}=n}NNZFePkz3r2!$ZLg3z-Vo@
zpS207w@Onld#rTgQVSA;VEb`?H!9!mRNr;AYjx6Z&y_3D(l%a~%S{sr&#%yNXbJ`a
zS|~qNY2OuOlAkp>G6?L<4mLv=Nga0|Zf%v`nL36>#{NLzHUDT{v$J3*Ab{Z-)&XJl
zFtOQrb9?2{m9Rc@AA6K*#Pjku+7tsW|72;zO?ryw8~+$CYB4_V(*o}1)~7ee{&gI$
z)A5AEXH@u2+4dFGmz2j0c;=z`#q!c2&Puy^*1*r!dh*KVtO6VCn=GeqHN1Tu8s$uI
zzcV14@tDw_qHXfhnO75z4l*t1Yb#!`8Z_d|Q`r$nEu@2OuM0bLS3QG^q$tFZI{LBm
z^yh;yl}gB}Q4v5m;E9;TWk}&j7=}*h8FMAN=Q84O(L<ZIS&yBDHI$yGS+*icci<!_
zXp;O!J>1Z@R!FSRmcg1vba!+8Zz)In-oP>;C`gH;l!YQFk7?2~M8$ebovs)eZ_{8A
z!tWITO@X&s>G~e+K%Ev4?KeS@Gw{!0q9A1AALA1`S|o;w9Yz9KeJvpMPV$USFBUYg
z;uoO;z5k-do_XA6oQVus$@3rZ9EmR0xkJHGfj@tDV~=kru}0f7zbyR#+l887%Sue<
zzR}6^AxO;dtk`r0OY@mKw&b({1>Lt1jsmwlNv4Gsw!cOL`@P%Va$kbbZHxE3JOn7-
znH+HB5KmRBjVZD%8GQU<F~yrtl)sF(KjETO#Jr34ntB7m1K{Ex=ngTlYOcAz>?Ru<
zjxUdieE+?OJJTN!$~ue*)Q@T&Bg1yZPcPNicg#;5b$KqAHD-{qNrwVLK}oGoz3j7W
zLy50E?w<|YiqYIRFNf6ZQ*;ioQ}<hMq3qny>=!C<LoU^a%itkMNC#@DEqnfbY=N_P
z3}Do<=?ZI2D7*QFjPe4;FLN^lV+x&|_J&?&wB2bo(s{;@5AW|h>)eMa8l-GrD^qmP
z{+6~Kzwk;tG9pKWkhNJAnaQ~e&YE@Ye|oq64hLoU?ECOG30>dZ5JbSo$KRlcNyi+&
zK5)P5YJ)E4UBo{_)ZFy>eZMsQ<<0)}8*G1%Lhfu6SCb@9RRhe|s^S=mQ8-D?aee3e
zW8Gdio?oz(BmvQw?8vZ^`bRB@JiG7u5R0zX5hU;D(-OJwoyzEmJzR~L2Y4m7<r~NQ
zvqL)Kl%mmuWK_y6KpL!c6R1b{VgBcnnoG%v0^Kpp`oOM>j9_5I3b|pG>n;ETlJ`k^
z{qsmCihQRq4-962tGGimf<fQcFVaKZ^=HYj1z%vQ(VNfsK$yp=kt;;p6iHih7WvK>
zyuAf0M!gAz*R=7SXTa&pEQ&m%8^kO{lH-&eko2=?<wG!*1eS{s1K&c?`+_W4fM27u
zko5>K6V^DgTS(3HYjNm!AuQ#)GC;@XdtN+`0(ryx`9nar><UnO;5{Y26xrW(D!>+l
z%>QX{HoGl$i;tqNyISeKqthrw)y7>j_~?-c)@3twNt6D^J6omIckCVrWpL?l%L3g6
zZK2xX8Z@h)rPQGp{UQ4`MRo)iv35gE1C?@lzmaN>cTDjN57hzt*Jfpd<}LhpTd}@(
zQr3-x&yLkh#db@_7d!g30p|CJC9NSj=Yp?6{wdYh%)l`Vgs0Y-70Mm?J2^Be%398C
zTc_8rlNX#IV#&BgQ;&a!f9p}>%NkFHXP42wA}Xr$gKGQc$F^<;iF%}(aVpR1W!K#4
zdmW6w)XFNl2oZ8c`bNwxu%WQUl3qI)ptfD<@Ub8C;h!UU=qs?8C$S5X6xAo}W2x<r
z@5V+_`NdQC0Myrg@V*QiD$U>XYQ`V0K3<$e`j(U&X7E58Of3!(2-@w{>u;xK#~6=~
zlZyM?LRvzOs9G?+OwISIJQ<b-oT|!MPW%o+84Rv3D#)gVi>Jof)+w2zx_gDj{HNI#
zoATRJ&%vKsgHV`NDL=OuIcFdao|z~CDqdxBTYpuo(iRtv2Fiq=&SMv=MeZt5=O9iZ
zZL;SfjZw(_#Zbg+inHXFBqM-Rwj4&)J8`-`B7nyqC`Usxk@~dCg!bTT0Y^P5u)`+5
zykrmz{=?$&FYbO{o!_hb&VD8=H0a6dB)=>+oBR}>oEFYp>0Os?VwPojiy3`17Pl{M
z!79w3DkBPHvrSI6^o4_DEw#*Xj+i4k(;}DdO~ZJ!%WNGshc7mJ^M1H7ffokYi@k6W
zY7~5irAB|GYIF7~9sVnd16&KoIG1c4ff;&X1?($zdL*AdxiLblYk}1XFD$X`uhA{f
zQ3*9W|F?e@=JT}3)>H-L4{@re`~_6Vpkfdh=opGZJd#RE5a0F0FcHcSfg$?eg@HML
zdC3w&9%OmnK1^oGuowk-q<lVJJKQtI{~cerJ)PO;Ddv%MomKIwjh*9H7(?^gj*^Ci
z-yjpbyis9zu3urJYAt(?XO8_MLi+A~owjUsD3E>!r$N)tT3PHo?9ecu&W75(vC61p
zW1H+%oymI5YRtQ!d`taP4s+ZT+|)dI8Y6dvct1+`wYiNfj%7r7jw*^QjLb}JtUOfw
zw(SojxN&jD82%fp-ms`22s^Yd<#S@QI`?<e-?tBx3$_z75<_pHAwhJ&dw*#CphUV(
z&KYMI<c25$sUba!%Sd}=jeFGK>0!|bAhQKJf;F;AX>!EAe9Jn<x<_9Kbld+aZW}CN
zEI#OX4lOa8v#SvQ2S$phzDVb6fABZL<YZK=>m0{<!x|SLQBCF?y&v+tX}_`ATNig7
zdp0L%DCXv$BwX3;+cQ$)kQB;Vxkm1Sli2+*>ZDBfhzx7&L4U;Q<TZ-qg>zYvuew3x
zRBTQh=9SK}vyATGWAl~M-+G!BI`T-ddP-L+H;B`#NrK})0${o<`Yo<gLo3x{Y&Rdx
zc~ns&o$t6Fk(?*#IPzjX)EraN&^U^Z>t@;L=R;+Em=EG9Ik6aMjq2%?&Grg#B>P{<
zw_(5E^mvQIoF3D9ExOjr9^kZjj=4rVW>}X3@&2kvC53@%!I;m4#>M$rrOyl5dMIH;
zg8Ci1`_%!#Z->wfJ}2q^gyXsBZ$grg(lH=;jx|*lHbzpSE7LgjYMOajEpS91<auM^
zV(L0_dnQb%mfp1z4gU>rcDDUYHs*^KsIA{EgPd-d#Y4ECIJFl1qjz!s9|T8X?!p~N
zBZNp_;HCs|QBsDyG^5z|5I_6lP*#o>2%U)JmyBp|I6^G2eIf&;&WDK*cb3l5=th0K
zz{7{b?>HK5k71_=dI%2_<1s1A;&AHg(RUd9hw=LjoY5n9Sa~<~Y_$M5?x?qT_fdCV
z^6pO?70e9+JNqo0ZahTt)qTG}m0QybsJx}1yJ3hrUW8;<>!FXl2L|0)*iJPYh@v}9
z4rmA#{0l%{I?!2?M5c_o%{a}RMmcVAa3F6D>Iouu<Mx#%pHpC}IY?o#xO%3dkxzdW
z)AKLQZjF(WIKuyAlbv?4mbb@PHG`gaBrmx{aZ8ApE%vxGOatf7asty>9<4cMJCBTX
zBF)WlCUHb`u#HtP79LdJ`M4k$6N%mUa}mai6Ir$G%$t*KNQ-b6#gRQx!dnjBcAH@v
z@8JJe>0{d6(8Wnh@n;I;tlEDIH@&(|sq>zD`Q8n;X5Jh_8&Qty{Qg9xV>>$|jW|@e
zb;r@~672E^tu%G+yauOQ9hPyjI)YItd1n@{Qf)z?)qjHI!R(m{7yP`AKn%6NfB)9L
zmln!#l=9@T|42pnhTzD}J<vCiL)4?t2_D{@2n2rahfP(bMoH<`7wM-07pyUy8-!ON
zmg~EbHtsFH-b;-%qn3_eWeeyaM${ZCFd-olt;@}Kt;x?l8;`woJP1wl=?=<*nZj>J
zT|5npLMdlj5Mp0kzq@Svm$5{0V4OJ;4t)Z(6)dWwo{P9;A;15~^VrWlqVvsglyK<L
zU*L0df3ZIUfc~28n@(iKCe_x%Bjk5-AIUkYd(xl(YzttNV9B}Vsm#K+hGN1z@0<L2
zr6G2r;3Jj1EoD7BOeZnm3`Y<{5&{%R8D&?IOWgT)Z~5q0<aB5Dl$fz@{LBcu+Q}$>
zSOxLwNiwThkltX9U3ZXdxt>Pm$YA~X3L29{gaAIJj@fTeTLrqUrh34+JoVXZZ8|(Q
z_CTZ*a|zQ!QLTG47&LYbBEb~E<o^zu@WDi(tBpS2>)eucU)lT(%vVAjF~1XBJG3>;
zg|;eAY_)0fkr$xM<JG1L29+cZO>q;DzTZ7f-o*Da%wRH--k4(L8)T)utM~&&bQJw!
z>^FQ*qh0j0q<gfG081(!v+n!fB}>j`g)A;I3idZIJ8fO(eZyJRNnm<@M5Q-{4T4a$
zyYqe^I2=p@_TMAm7!68ne4CEV3rYgtHA8@rp#M)T5?@J)ByrMAhUGoj{gGH)Ae+VK
zm-yo98~}c=UMVK8Jpy+hOVamTi_GQZQHhJAI>TSBRnJ?T_4s`|{(y*?OnT!czc1?A
ztouE9MTjnK=|uAunSpFajkikr+{(_@;*!mg7*r(7j_j7Qh2kO!?No>#*wlOoGcswQ
z2t&#V-5m`Y=;uRk`ix#ix`NmezcPsxr3Xx^8%Jt4di}%tT<CsLGgjG8ZFQ&`w`sMf
zJe;J9K$sMUSz=&0h<Nmus}Bh&X$;yJWc`3o$8SS&XDxTeGN3<g$MX6i)#I6If)d_>
zK;)3%tnC%wJrljEjLc^jz(OsQx0_gQ`+=XvtNNjWcj<iA$({H@P?{*dc8c_G%ACXD
zejkdnJ&<%)x+y(HLFg9z-{vlK$UWa!-Dk8VZEFO*vCP`9wfVHxQ0a<CYb9XqeG+Tp
zKf!Nj2eg0+Y$oEOqS6l`{4>R}B`FNV3=1hY_vc*4KbCB4%?F|-4culbrI{~Tsf<xI
z1$H^#Y!0&Kei0$}<5Mic8-N*yy~pMWzmrMao1CIzfrtzS1FgRz#ttT8+V@t9!359?
zha~JgB9H66;|HSm&?(4^D+|se*|{0D`|h<rIRKZE$0>YqNV%5Hhx<cjx}|*UVRj>N
zv?Fq{S5({J?&)~0qt$)%WCU*t<8^;e{C2=%tz^0KRRUI*f8T&S@L4okt@@`2K<)&`
zU0jlmFh#cq)`&5kXOgKa#*lj8eLIrOYkS{Vp!krLpd|}<JV_;Ry{vE_bBVT4ZD4;y
z_G6;4eJAhUJM)a@p5hb2dt3?-cd;U>Jh;*nslb&FVkv3l1yRz&<XZPbkof}X!6M_5
z)7^Otj`#*ngZhvy%D@B;bX2B_41f80GZ~`fz|?liNzt^7SXiYltkB={KaAjB|9%@5
zb5hdn)>mf+vp-Co=Cch|W{)l_vGVr$Ud$L;RQ12ky{UD$5iO>@#4rk|vs%<;<X3D>
zU)j)0nL;(*3rCWhL!DEKvK`FKP%ZU{WxC2{SIhnMmGY=1QYrQZTqT-+CCnT8sK{ba
zdo@)WFRtmU$T!J_kFOlq+_fB@=ZNLV7Rh}>=!f^^LA;Q@=8xt8l2e}urh5$2db9#U
z>y@ab;Ng{xD>wVxhK<X>Pm(Ew{&@|}lB|JcY&(O!tO(~K`SSf=6CC571ABXVynNeq
zCyI{5-XMZ>03>TTmO*q;dmO4LQ;TK>dNP{~?LRNk$)vMu3r!lLcaO<fi$-XM#nu<M
z*cR}tf0!GUfSlU#Y?NS4FNl{yR2zxl2o4CbEgLa7qj^@~>I?rYMO72N!gM)kY@ya2
z4(DUUETHT6xW*qOr*J_S3nqlEnJhF42%2uP;zNQ@<rLN=WdgBB=^8c@uV2>BM8sn9
zK5O1(L_Ge$zzvs_;SbKyd>K5UB_m;a|D+G%wSnV{C_RxPN&f3;NbKam4u{%zRb=o7
zt^{f2H<2JHM2NZ)LV$q~2N2kA)E7Y966lPhb6aFfyFSs{=P|+x2ue?4I@cXDqdAz$
zU8^@*S5-&JZ<0G;>5yeuuvUurg%0N{&&*Mp;s3pH{C>Q~ei|W8m#~NUNHl^XOW@Xm
z>Yqe1RT37q*)iclL>FdD?<0C@ZwI0o#(e6Tsx^V9OBliP?iC~#mwO!}?6vHEGO5J&
zoI-xYy$J^0o?4m40PbRzH#_<=>z}$GvgG-V{>1T<`*&O;hwN!9n)>@noq3Q~jM%Lc
z{C(`YnTIHnMtnC2cGnp+7QZqnr*C=Q%!Hv4v@L`ZILcwgGN#Tn)E7h`#<7Xoj7Olo
z9fx7b2iuPE<k@My#VU~SLbVt#E%wEyP{Glb5D#UXv`8PI2c$QfkywuXk0S<g=JMYq
zqcD;}BBLMHcqNfVM`t7#BmqwWgeo0sF0i!$4R<Yc@v~Q+j@IVNi}JVJ-}BYn<iR0z
z8WbJ>OiTL7=uMW$AY-g9cMKs+2h^<gOpOp|2EdZzF3A%US|w9FTF0)CvajnXwEUNp
znf`F69p$ZR2|99Mn8bNaOg?N4pj#4^U2<F}yijxKxt;OQ2ySk#Cr`33%xLl-9dPFA
z_Y$7%k!Lq^;!mv=7vLrJ6Jmu#_xX@o1LrhhJ|cj_z|1QB`!j9w2Mgp<Ru9#>lp{|N
z0t>9MHM2RVITM;)5g0wX``Sj9ZfxFAgfM;46*GLGDQu>qmrgIQ`rIlX`u`7dzJM%A
za_t4vMsx5_vdXYmgMKx1>Pwv+tyo6S^>(bTgv5#t%?Pn@<qJ1vxerXMtU3U)DOxcp
zDFqKZKl^qdz48wVJ(N=zYqk}OTqY4utqBM?7Sl<<*{r)mmGg8~p4tb%a{krkqvf+A
zcnE}92;t^g9>atNpxmQyI6IDX7=9-*D3w})Lh|t+1e>B6{U8t$vRF8uyg^v=&tMV#
z6X_V3#sd*8>VsV^-UIEd+CtFrjWF{K@^m9m9jeWi_pQ&#Yg1sE@^sgv!7_3;SFg(|
zXPdr66I|9`??!#UI+W8yndG}?M)TViqfZ`JrpOXmEJ;Tb!$A4Xp0@{}Aa~?Rtg?=G
z28>_xqA=C$pnp_>(B=+>&M4y!HOaZf-!nL}u$kLRQn_Vh;ZYc^{d|F(Z76Ie(^Jo<
z-C;B!h2f&7RIC`Vcim%B`!N&Ra;WC7humIWXX%i5m;Ai9X0N2yiVXduCURI-cGPi?
z$>fv+fe_IUt1B~VW6!<KLd+j*)t6e({5=An2`Lcv(qq|z&zkErcu<N&FEOHuI%OBm
zbEV7c36Ntd2%dcehk52nGv2PdY#Ejf2+O27*h4RSuC&w=Sx#Wz>rGTM#n2jVi<3@W
zFID_G@ZT{4q)?ms`nK1N777Htz-mJw6Cz4u_6kvuh)sg#2}Vqjv~(^?y_tnE#nVhb
zlk<yOY)t2cx74It`#ZHGDk1N*<P-3Mf4I)v6-P2wgybAbt9v@9Bz0DERJErN+jA<d
z(b?LQ?_2oI{5kk?Q|=?-ma=?7G;R4gkWqO>B3(&7t(7c{C8fZRSsEPKSvWIeEu|DK
z=Y*}kc+23r4*1m?e#fuXkhghZ4*V$*_toY#ml)@o@^T{Y-YE!xom!nBR-^2``$e1}
zI45LH^y(7YX>yDfANK8Qxz@12Rc3)Pg;PHmuNZR7T(oykii}$1I-wCT`C;d{U{FN7
zAn7i>#|=sRo8r%a;0RG~LeXlhUfO>Bg{_BN`<x^lHoII%LWogw96Zn1o=0L#O;_)R
zyr(ohj|93sI*qbk^Dfxi%F+lte46w2M$v@9OyJas9!TkTn7zDgiBhkk+nD3TIfhda
z)p#f^vK&qH(ZirTN#yj_1p8eY3T#CYwJeKvnV7D)p#g3vB)nfgyVEe{eP`eamL%gE
zI5?`+WehauIA>)%*EIX(a@iQ1=$V<Rjy+479=+fY;z3RiCgz2rV}nUao^Ix5FKLrc
z12h8}y=ECSmi7Ha^jj6(rUy$))~D3#`4#WU=b-VS+^4jOL|1k%;a3=!#3O(sk0z*h
zz0`IuR-tkY6y+l&vhQ4p)mFgQ($X@IBOi*roY^(YM;Jh4;OMLLfcYWz6`F+({`a$S
z|13N2Y-K#6Vi9n;I;sEEN?il<@{h4xTCEbQRnh<Y@3(TJ_P46MYkC^B+*s_rtub2G
z-nzAvc2Zi(pYrhYT`#U5sv{DLr6`J(9>^}ky3CjdLfndouGe*{Z=Uw$z6L`|v307L
z!<L~~sYk@^f6iPhjI9YFKMS#tzWl85YL`9}F-4d;Qd7^q#5g6ekNB5&|07~aPJ>f#
z^I;BK5e1dS&nKNhh}%hw7yQ|TqTe6>jY_IRjUZ$0#%sjNY|N9fIfrX1OT#79HA$aN
z0o?eeOF5g7Tj2+vLKp<n0#X8)UKfLQ%4W{A=*?f)>A|WUX{urBh)gwCim|2~%@L?1
z3k0z}dF@zEN8*G@df)b6bR`ZK#US<>2G8KM6&XhaiV#E3{YAi&3Sq8`^%CUdG~^oZ
z+tc!6W}B;de3+xCXNU;mfD~zm-Toi;U#$@1?9~{xJZ`xvU{4-n0(|zS08B7UWq14{
zMDpjVX*FFf@v3L_>N3c-?X{S|s%IZhlt9rN<q>?gKKfgHzT4r$X*6e8!uzxCM-n78
zQ)I=c)4iN6zjhQ*gYF1iD@g?jI9-WFP=&wL>LSMVz<i=mNskw^YXgVrme|~tO2aaj
z`8)lJdvqd3&~rwrSk=gpe?3INNZDV<(Tsqq?PLC}{hs4@)ymaolqpjRR>6)!vJqmj
z|4v#6k)jkl)>O@1`B+)KK5D8WEf^F3Y~?zQk->5&Tm3%envC;`0F<v4vzDA<?TppR
z+#f~aw5dL-7{dvdy_IYr-U5m6hrc=$2}uCx22qUT&2x1o_~(uxbGtB@5Nc>39yu$X
zo5O{wkABf=>|;SrG$O{gj`EH{Xl4S+_v*`HFE}Yz)vr3$)oZ{Pe50+c9M#^Ia9}2@
z(SWOYQKjcxciyYtE!+3@+9bk*VXq@%v|bzMbd4%lXdg@%0?g)ysN4-{D!{v>THcB~
z^v1$jK>Xd&Zt)Kbdj8O;`Q1W@v;e7=tRI*ZEH*N`4(=QJ<oy<9p#XM|UqA5_S#mfL
zAJzm(ZAo=LvWye%W^T}s5f(m;d!zm*s$LAh7$Q8T&V&SG&PesY6}LO#hvY-M)&V&|
z`XR@JWw-~X<Vc?ykbEdwsWF^dlOb(7)5R33tRmtqzq5EJs{QIAb!>_-ku_g=gevj?
z{i-VcZ+NoL&+Vw|@-g;V>tYlU0HR(Kvfpv%cXY379-|~hXvX-;a8tKJTPgcX+<-CI
zn2+%f;U=A(c<l2O!PKmR4@wDd%H1|Wp*%u#oN<dR<XPrs`mJ1Fw5M;!s4#ko$34p#
z2pvJrCj4!miSmpOBMpYVs8Z)#ti2WKa}whVi+=_juFZ>cnj5#?W1R)^;(vRD1$p^Y
zmBU4FOqPMYN~Kt9J1}%mU$~HMNKD=Ns`A+?s@TCR^(djQP92n^21Z)2?T<gN-te4{
zJyP`lIUw-s#*Z31Yz3GYCAbC>9cs%Rd-as1CSH;K_9*@KbcT^V>@}1?0>LqYH3kb&
z!GF7&^7zcg7vUYAtZ-tDFon@)6}~nwOY!>$SIyJW`K3dkN<Q00z4ly(OB*|=>-nbL
zD&JpvzoEeT9|5FEB8zKoX;Qz1^_e%vPAGnqqGPORz5{uX)=YBoMmRQy*fl3HNmw^T
z3SUp$VFzChrp`Cw5Kvl}w15l&=k%;dllP3+LSi0^i@k%>IYPSj_%lfb+{4N5n5F@E
z!#onV)9pQ}`fT!vqyg=oUI4<7L>TH?eBbZ)zY}(4R0-`f_?WH_Bx+4vU%*ISwJ7x%
z+BfGk<R`aJG5dmM-IO$%*Ctt?J9T8Y_x2K`hRb<r+*pHND{XUNOAcgDD3@G|eqB_j
zsL5Aj2hUdb06x(74)vyICkhlrFg^<AuGMOUd84DJ3Yl5+@A=!F98N2SXKKqo;Dz2*
zl9nA4HaxY2{$TKB#1*wO{pQR4iGwlLdHKbQ05rAS4rW<myU_KpdxM-Hplv1Ahs-C(
z^=_x!&$AU+w*?T@9H#}f$2U_`3YG44S4I%(^7=CAapBN)8h*S3LLdtX-Mjs#wv@+k
zDj&M1!nhg`h>I2+zq;DuOgZRMs3_caQcL<uPhD((*H@#tOzouwQ|J5)ysj%M(uuEl
zBcD-*U^j1h{TMJe+ZX7OMcGpstdO&_&YNGl#-yNu<vD^{$eei?0;Wjods#?c%s3B~
zaSc;D`p!P`uK+DAa_zsldTAip7@^?6%!-~SFKBMa`39diHCgYzghtoJmAQn6u~8A_
zhHo%cs?R%&<Y5HmUytn9w`0qed<k%lq5w&wXtZl5Liry<^IkIN^fka|EJf1sMsBM<
z0d>n-F`#iV9|3*#FyWd7&}RdD23(vw^WPAnZp83qW^y56x|5ThBur3cg(FW_*K4oS
z`*Q~K{jtbPYcc|3dlCuK9cGm<2iq1$41`}ZV@gdxVfX0rQCs9O#|X*quQX1EW09Uk
z@fTEphq9}c!&y`Opw)I{G9KFY>ry{Br>?iCQ<f>;#1&`jLi#%2tU*GXC{d3_DsuF5
zNHe#Hqy@5fv~K|JReOw9l+d7|<CR7k#C!9=FIL!`u*k)p+?3LmY~O?8>X;FIWEm|I
zfCHu|A5Q0W59(;U#rq^K8bK;xRFwZt{tF@&)%Bb^I26KVUt8ab_>T>#Ao}5$LVuql
zLIzW_t5+8vpof@QSQ}nG5n4v{l1Is*C0<|>yF;>j#{DqQM9_N`IK@2-vSSM3%(FYI
z%ZWxIt3#49<p1<X{}*FHdSA8LJdIWA+NEO106S3aWnpfP5t@2>WQ{yZC>V~jYG3j|
zogbdm+Eb-P#<e`(K~EiFeo4pLcea!lm8?Z0<fS>)aR9dWCe4`2K4h%!aOLiN-RvdJ
z;bj?*5-|dN4}?mGyS$$BwILY)qOTyy{j#w8_~T0vL8d_CTX#F;G9ju&G1Wc&4gGVr
zb0O`5p43<qd}i%`e6*_&BuX5h3~C<^{W}jJ$)9y|H07#J>nEZ`6Ph7?mv7$<Tkkky
zE90JS^Do2Vywjf1{D#kOIyuoeWd!3u8H{T6B%)jtr?y0Y;<oN)d-qRb07WRDt`>Xz
zhwYB;nJC}09L@Mw_{G)1SLl9B^K9M^)kAJ`k(;wffn61$ls(89aEXRwzkl$NeQ!@{
zAjNqL6q#4guHF$6g7{d-tFyo|9CGf40(grCMU>|N8{OLdC|*dXs&3dwlw=9BS{0~u
zi^Blz><)Ew|H=ecs*H2pgDDG07;#kHYRsSWOixZUyE!3WRxg#7Cvh2HE*^>P#{AwM
z3h$|}sqMco;EtogyKk&ZoV~dDF^BlyC|Mil%J!von+ggU)|uoPs2B}Judf-SO!6OM
zQP78w_Vdv+B?sSi^r}xZX|5eNC{>DX4970=MEg*1#m|x)fU^Wk$?E4e0X)?fCRKwI
z71RdXlzg-CxPM5Ou5rX1)4s&K*A})23YM0wZCavVsE&In$Q!$gkbZD(IIio)E;pAx
zW?DDK+LEpZJd65}2;6Q|k?q&@s#7F1wWWmk=<jp1b@-BJ+@+NH#Bl1J{7Mg5@G|RZ
z)aRyGhhO^DqKHT{fKZtRLg*AJqYE~|lGXsl=rtFu?{|+5I&(dyW$-S6=$sI2rmd0~
zrHYUkHzab55v<H7iSqegqT1_&fGE{dq#Vi%o!%~NWg7>mE@X$%05WzNSmEIq%out&
z%e0rW)`dXAr^%VyrM61z{BCL5AO()`*WXP{(9+iNeU9R2jl!LE$fA;a#>rg9?}ft8
z@6aV@c&mC+ur=ep+)^g?v0v}V$N-?>=5<ts98p)(_wC$6sxvj2yHR%HOfd|P(X@lQ
z8KUZ){#oymhbvpo|KOXud2{T-NjwcH>R@G9d}f;UD|95r;CLtr=oVgmIr6fYfb~x_
zI47@mRw`Rbv^*gwVX0l2OFWd%Z^(MEVjWj7O464eNoR{Pz$f8U0Y4wZ6oo`IqDBoQ
zROizP=PpFZ-J*3E09!fVnf82NF*pV($$(ysAa9vbdv}ybVT|lgT$I|?|Cp4GDB*4t
z_5f`qXcKjbvDDyE`eQjif;Th)6*D^Nl#?z=$OWT7wzh$Sg={G%;JR@vTm-55O8uif
z7IY=mUVnDP-@yULeY(m1UHDBpZ~TG`IleU*A38l5bj(wJz(3seol>=3eGx(bKumwb
zIeGgzcAJT2ymfayN|=g!d9GH)&7BXNho{r7Q8|?|NByH)O^4Z7gOcjF<-Q^+-+v_s
z@K93c8y1Y1rmBA_<Ck`P=;`U#&<2BMM1p`$8Zuaw`!RtI=A4a3IXb6+A$<3)Wd(94
z{kTDbsirDik*?Ys6drkL+_g%et7#^W@EX;9K<==co+3`HARxB0F=Fn~!>jgb-@Zu<
zxR`#{PzSq;q|D_76cYci@+eLttziDX#BS1AaW8+zH3hQpGqJI;xk#j1kTeEg6}_DZ
z+_8t7PfPg|cGabe6Av(JAdbS5mO@20b$4>-+*rs*z~XNCryjyju#(Xr>A{x5W@qzz
zT?Xk2J?A9!(52cE!)%D+(ZZI5XUIUVyuX(lB<p-%nUFX%IgGG;M>Oet(P58ghX@|W
ztVH1R`{J85xLCc&0WjM%N&T09h01-ZHqsEZ6YGS5m^K*6Kr^o}bDoY`GwgpRA~k4A
z-ds_xHK2;PveTjOEO)il8xMI=*-Y$Es;N@CY!90ZcGj@f(ix6cSPWShqu3>+ltPp;
z;e%7SwuQ&4I|BUGe<Q%*;;8dM=Mv>w-1=|oQ50u1SM@>7z~ARB)GG!g>}DmxY$OZ(
zo-#KV2z&ch;+#4Kq8A}hYH+yu@^<9)Ff*mQK52ASt5&N4rsiqNP(P<WxnbkJo!GTr
z3;Ofl+=VFt=nT>raRn7oEi`#Y$z+|Iy&r;s_DjgISj=nbR6$<eEhHXg_)e$i@fChE
z2&%33{Wd8>8^7bNH})va&h0m{W?V~My3El_sKmS-&;SD(&sW*~PH`@Hl*IfN)wbWz
zNl-6!ODL(ymP-1P&#l<zXCLL}mB0Y&HAF#`gwUTW>ZCoI_JSsGZ;naKnhaFp@3W8J
zv+bhl<E<-UT+@*;+%)cG#QNY?)IZ#!Jx*BuT5lFE`P5{L#=nonSF~Y`ONzg?(?Au8
zywKG+ce=QyL8wydEB{H%VXS(AN)Kg#*ZV2Z;e<5~ty?J&?MhY}FnblT0B8#4ejHwA
z0uz6eSY_I4ynrrk3Ljr~)~1)AD39?w#tZU5ib$7Z#O)PBt!T3VNa&DA<*(oL@8v(s
z)9E|nXu!?io!H!Q2{>=HBZ>V`7V9cp8IQ-fMhtb7cK8=P<!d5p$ooldMkTp66KLC)
zyw|M&EAP^}B<spXwVU@F2QRO)QXo>CMF@%WU8?`u*+5H8KAm+mEShJJoB3804N)a;
z5d7E~jf%;g;-@VZ*c<oNef$0)I|Qngfdq@WavJBcR_9&zBL=t7mbu04$H8D+PX%SX
z1nz!}u@of4{3R8~e8dNlCYcwY?oE$cj5jLT|DRTFLFdyPA7<gRw13Yk=(kBORtHkD
zTHIowyz52U)cW1lBN0&5yg>$bP$$vL_M>?@Q7q6nrATwV>Wt{=Gf6jch;PD%k0ibz
z>l<7uhx_>~jKIoXp6G)Y1>qBi!=SI*HxP+hpaO<b)V-R#LpreHIQK;x<w+$JmLwv7
z!RicnpPowdv4)o##buvm7yXUQn%X}7e`r@Nwwi`DQD>lwK;a!Rg=f;+%~RO*x%-T|
z?|EWu{yj|UtbQ~qWl60d2dIbe)>q?52$71__Xw(P|0rV_0RoS?3rS(RXodYZ4)cDy
zKG-<W{uq7Lnh@=$I)qnr={YlqO^vHE=sxc}q}!-KLyu_5r2+jbhe-)u-@dyS7L6+~
zfOZA?gVwxN{5=E%-0MJWtTD%h2f{pQHn$MZqTWFuG*m{jZ!5E{MOQVD?|Cic2P2zA
zeR59<NF0j9F5EqnOR|lFTnWif<5JgrP~y9FFoP&gZYrE1j>Lu&G6el-42^2<MnF_r
z_{qe+&=H^^S@!Pi{mxfAI;d@{44(F7avyV}zUX&GFbU7h#=h2DUM8w`=*j~blD-Eb
z5AkxO!Z2^0R?P8e2na|3h?J<XYQ5%G!jkaMJsZD`2UPnM;;;JIzmCFW3?i2nl)DPY
zPW>^UPPMo#R6$Pn(j-o!ARhYzFC#q-;7>^f@t*pugM`27EIo^EcENI0)^-Ny5toA+
z-;2Y~vTlkP(8}z{kZ|k;djiASP28GZrA=qrL<2%-PfjMcAK6CI7voFx9$UAqZLcwI
zsz^-fzl9}}r`K9XV|q@*zgGyv0)U)G;JB+GaZ~|wQI62&VLy0bd*B(`HP6eRIMZp$
zT1iz>CVXCgiJg=ks3g5s`KCf_+UJn0#`3a+F~?ZhnL&_Z%~e%P)ALysC;i2gcMUn8
zRu8$TRBgRz{up60$zVo^86Pn<VX;(csaR8ucqZM8j}Ro<D~re@@e7JzK+z-V49KRo
zn8*}D`|}q<LaAurQ@Z87pEFKW4tEJiQ4j@6rJ^1?c-=|T?2i1I-Dr=4v(}0`!RADv
zAZ_{$S=?C+{uwVSThH@N`~-z08Otdn`xk8u;>g7JZeV=Dfb?pr^ziZgj;ThH@Pa0Y
zD#)e)V#BHbDTeXvaK^LMtStLnN*HxTfwJ%K5Z-T3`1MCX=?k)&>teUq_t7;ODd66?
z5HEPBSmBs^r+8In;cxOPO7ZLFK%A%pBxeLNKZS2-EU73`IYVV*>|fTN1C%f)kH0WV
zp|l5??r7j(B2JXx2^Y@~Ue@~!1;l7x6Z;ud_0Ba^lMeG>2L-61RkGyxlC>W>`)AW}
zvt3RXng3(y9Jni48zme&6Hjd06Wf|t6Wg|J+qP}nP9~n%c6NC8`PS|F2kF(F-t|^J
zMO}c1wXoG?AZUo!7oEdUHk@sBG(!KEG!6g6KYeggFKtORHQ%H`!B(Hl-~+~bZNPon
zAx|RWr-Dy31>@f*gNYr>EG|c>m(JQ9^u<R?y(Pg^1fF3Vgsnv=&JA8$Jse7rB2rMR
zGe)UHD)O?PF^;Xn$*3n|goSRey-xR>6HdNPPOEjP020JIqMz%wRixO;YKR&A(fADS
z<M3Gan+hvDe%Vnyn74Fy%QZEVnrJM%QNh_ukRj6^@{~D`0@m5UuA!2zLZwrh{0gXc
zZxYF)A0&@T<Hd3SUW;1AsC)>|;99Q{j0l$FL7;4s1<REy;U2|Sa29*&+j1E{Mlyt-
z1d_^AE8zELr_w?YL21~T0Wv`PkT#)iYfO?xF+%!#OmG$W1>IuKB0f5f<*b_8v4C8X
z$*4org}EQFDu=P&Q4M1<Y(@ju|7WwQ!OUoGR4h?-O>c?=Lu7~}7U<?U<*fGsiFbUD
z9ojT$lO@n^umXFOAN@X!@5V0^^0|;p2&8BIB+-R%VrveriM)aEZBX7pG0ur|+`6v_
zgLxnINS>F-AQsd>cq<J{EdH}*?UVW3*fZ7D6<*!jgdo>c7;dKx<m9BC2ihURUYuXA
zsfHwJ?e+sFl80F#HV<l>E`I$&JX{bOwG`H%f+Sl3Nu6!is9a<MQ#1Hai2LW8*fCDn
z?}H@&Zw?kN*8mio9d5;Jigv=Spa8aY+;HKcsvP)V2rzv!NbY`R5ZTfAxUJs!OlUDT
zX4@50>axD3MC2wea1-~@CLjb$7e-`|n@9!NiX?DuN&Yv8h&;tHB;n}VX@1x2x{Uqs
z(*_JN#`d>TiOlgze%k_n_^{30lJ%f^d|{>I3G*i2gE=@fcR3P8s`nM9#XL~#S`jkC
zV8JhE{={y)wRLsc(Ny{6KN&8pYi6PO-7_x!NUrIaVRs?wYVj04@Q|(lp#s5In@sXr
z)~6Qir70ygug;(V%{ac_*iL8bm2pI823W(w@Ll1KD@X=v`J#^c=3$PT_M|`DEv4Ak
zI7O*rhnE<*Cpp8N?lEmdu%>)f@vOo`o{=akg?wi*T!xY-ELspQj1odr=q36ReqH}D
zeos0T#|yI8CqHiCn3)|q3rwnyJf=9-USL&ySw6aGr!LtXWH13eQ(x-;3byp{EV4h^
zI%clS#>z9HN_dPdQj}r81>VzsbDz=9MR1}}?xSX1_&oA^5^6M?FBFy~bo@2<)BSOb
z{3~Oc(He|d_B}_qD>O;I)ojn}T~e6Dw>lU<fbRxHqU%z6By)nR!)VmEn-_6Sc<Xw7
z?Y1A`*`b%Q*v8rV7BB4^h`=dNZHfLw$ZRjBM+Pk16{G$2%xzDKE!a<>OUqht5W?ag
zl?1sK`Hd_>bl+IkJ<0aDp6A<ig*oTow5Or0Y_S0JfWifbj)v|e5hRxk#(Z*<#VM2K
z$>XD`S^Q14jkTnif8Rx~<=m*lIcBifq>@NF#v7POkss{5fK!SX4Ta*`mWZQ<P_pyu
zh2ztbdnOw)!E-u1+W>eHcbN$zCfn^v?I}j;^<l3h#KSi~>6OL5ofqrzJYp=T4actM
z4&5FL3BrG`Nm_CJ<Pq<N7Z;c}C&2GK@W7ckZFjpXK`tq`KcnxEo=$_8=ni4%iqb8E
z2NiPfet`T{s}C@tfLbgt7GQ&Loig1TA6qxk0}PL@G`6kEg!OFo4vdh5@KXec8fGE*
zS(3hGg$`^<te4u2QKj6(q+tZ~ISWetJt?EAduh7BFyQz5v%bi9qlW4x%e0g}+WfjI
zs=T1WCWlK~H%@yGmnnT$d7{n_!hwI}KcKq~6gK)EXo@=&iMV)lG!{VMZ8L<y*~7Bw
z0V%i#`B``k9_9!lTqT(KFf}nA_uu#}io!TMSmNUN$63Y9ys0@W!>7TG`G2yeWQdyD
zcYG%sna9@Rsh#Chw6bmAb8V`Ser$RJA$(va3_7Lbj+%w(J@M-GyT<`Jka|zghdd(I
zApEv*rwz^~n%a$EYi{<o#_g>RwRQ5Cd)3tU?T0@|HidDvw#CJ}%1InrB583$g^zG}
z!o7z>b0=<%NR{`Tin8|{>;IT?0^OVQ*U6$%%b&_Y9}&SN$7w>(i#218kKK0BOl3Ap
zT~Ap{$eofrq=8N1N9Si-G1cKRs3bFU(zoIA$ZaK?y~SYf2$a9;bl;6VtEmqDzN?xf
zV;H~iYCx9UYtaHkEt$V2<QZ$9GYIX<8-_@J&3f}*KEz<xEY0s_)S`<zJ~b(jwQw$&
z^_@<_)^1nFLi1QW>q=->^k)d?)bEBCh=YzHY|$p^mv&nZ;WwOX+A@a?IHZM}A<A^a
zIsHT%*_i1c&hZQ>b0PZV`_oUw%Vbz|s6|~EXiYw<lTQ9`#rLn@CmbM_pc--G?nhF&
z^kUaPchb9$Gmwh*MBh&@TSd*;oz@XI`$uUlCi_gg)9+~rTeO*)#T(w->kVj5#~wSi
z;Ey+iBX(blbdq7SUIo;D6*1@eygec#S$f_wD;{ywrDSiz{49CXln<Kp`#N{E?rCPK
zBv^RAeC{j2e~RYApa*zc=NSS8W|+4|`AkguB7PmS<t4GId%&*iHCXioL6j((C@DPo
zUyFe!pPb%e-L!_}19HsR?VA7#_`BVs)Ucbx=n=;gu_u(5l0-B(oa&%XmKYJ&<W6<`
zELh-yH(v7a88g}9ti21a&e+7bYFsU$CO7v*Y&2NRBlfII9}&2^GJspuJ9U(Phi5M{
zji%LC6+GLN&hLj$OHu5ME;LT`AunHI3>FQ0a7UT@QI!gp6Q_x^|8s>a<3Lpao2tt_
z(0z|Jo`<?0g9r(yP1-jadO{;>`uN**)Oo{Sxj~<_WfvxT*vaq@Ny=%cS&CF4PVH+6
zrG6=l#lf4wNU(-0+0<W8?T3=TdScZVWHt{)KZF|3Dv(TIT`j-%5fiDx_1@Jc*>>OB
zhaOmlpklU0w}=yI>59&w;g39Y_v$y<w(KbZ%Svo)wFqznP{n$1PR(=1)z?rw+@%`$
zq_yU>;!Z(wcwh_on2{<IakMzkI8}|!o*U4K4?eOY`V#GsyHU8u4L9fDPn8>+SKJC2
z758YnT%WW?HzralL!w_OVv++PyzpO@8}@U0EcBfQ1(^-3&AQ&j*$aPWl%N}@9>19%
zX6@wyBsc5GPOAvH9eX&`f7|rcDg^sPlbTq!kw~{}dZ#pwaKtD(Y~%MTU~m7cE!gnk
z5;bYm@)1V)K>HLESdf2=@)_OR*40yOg2&2NdBVV-b*{0bf^E-hqxpAlX|yionwX=#
z8bNIsnGW`2Fz?gWHZJ^^%|^4u5)>Y+g96u;j|^3eltzwUwc2cG_kkX&hhxbGG*2Cs
zDi_tw>#^0!7fs@3_7@&M;+ymlS*Lek;^Pdh&5Sirx%{7RW1twS4$dWzO$`$ef0aQv
zI#hoo@YUcRDlLQ$BW)f;!0SGQae@^z{yUMfsh5Qsg2p`jnvpF5$Ac@y%$eVNWh7QM
z@s8cu#m3U3x?TQw5o|7HG807Kqac>E0>LSKb$ju^jf<u?8VKz1wwcBoW9dNv08-lf
zXOhi$APspYZQN1$UTMy-8m`xWY)O3)0z+aehVh*oLg^hC4&h6en0;CTDDP(lQK)4g
z?SI+eCX)0qb4VLZ)`sOyPwOzz)OyPR{;+>oKe4&gy$7v-KCJxREKVE+6g&V!?69-;
z1Jw&)8GJ2?zyIMkIAUda0LEQ9a;QE6E>4SE=}QtbU;o{7@>y`QewXMA27we&388eC
zx=$v0lkwNE1m(4~*@XF%-N6!mXvkla3*^bgb*m0k6_*f^F|c@QJ)l=tSs$(Fst11#
zx|TR@w`=0h?^LS>Kj(N)%5;4Kol6ee!pNECY6`!2fpJ1zw#FD6>#TXLid)Q{I<oK2
z5R8Dh>-_V&O|%iDeds%p+-J6YP{c_K>5%QpDrt=Ozn<%&zY^q3Ra7d!OQ58cGZ%6B
zRqrlwPb#AAr*st^u6D?1VgvyuD5)?q!6RG3cxYq84H6j}*^mdQ|A~z~E<bX6d94Dg
zwNklJJ9i0^EMY*wb1!6d<40qFH4*NxcO98=9QY(A`6W_VM0tFX&h?!|z0sI&Z$68r
z&DJ^7x<1Z-x_vxb)v@W<mAkj<?MuwvqA!&YX32ewA6d^j6(UkQ@5$@8EhMVv1k9r#
zs=n@iZmJNXM>-{zLy-T)&~1dE-jdBN!dSkT;04ivKQ4Ccdl_8It9t}4+o!jS`Lx{+
zX2p7={)NDVuqf;^qTQ;4)8zi5lpKRxZ-@VNj=}_hwC0uH=I}+?){t9bH7Hid{se#6
z7h-eh^Nu0Ic2ZCu^C95>(+}*xcjz_jOC`)mt<@iZsgU6~6%ml>OE5Y-1>C#?k9AI*
zw-T^q?Gz9gwI!zizF1Zi9v-oF&$P+DWAy3=ly}sl5GrfGVZMi}7)InSgYmdlm&#J|
z!%FhSLO9zpY*X?V@ZFTg8Bj#HA7_Q*o<X$4PVbASDR;-SJ`lX_hmafRnuiF$D$}Bl
zI#e@Wlg4FF6ko94j>*=MZSBanoA-e%GW^@ESzakjJ0<t*;ZsCeKK-i#v^MO1sF_GF
zjowrG-BWU^q$hEXGRHEX9dBe=+0};lUxRLgcY09(pJng4a`0YrUsc_<Y01pR<kA7V
zavB$tf69s83W1$^1s*g@S>Hq{6|*h@q{Pz+e$gr^ZhX0SIua7{8>LvuCj161LzZDH
zP!jCj+G&J`wdLaMl?AxMhy|ev^?8jz!SPgKvDa-KKi$*=C`T%9CsOnxm-d@EV}>+q
z@6FB&Gg+IOC(kOsW!-lJ#ztp>IFKR6-Wigg4KS(l+WZcH;j*2aE*6mg&x0y3xCl?f
zGsX|{?mtDePkaSO9;NwkiAeE6^+IKjeq3M(k_3V)YHuC<?gleuqvN1x3X;Cm=#oD?
zAPW(yT@H;(>^KSe;gF$0%_|`6ORl$zmB<-+uMY5BK4hA3iM@5RefH&WHQmRHD1|%R
z4Lx;D0mD0-Kl1)yxW8RLwS<^r|LFb%A$TmFjXxl33X|Dc71bX&@53w;)7krS8ozMg
zZEc`qjGB8K#EfE<`MDaK@z*h_v^^vgL|nV25L;bf?Uew1wz!b`6pI;fDJr30>InbC
zksz(A;v+h&yM8^${f;!J*Bi|k3qJB_#Q%p~2@mRCC$-5NAod<djH{<0j%onX5(E^z
z(0ZGkEPnRvUhC^u<t62Z=AQ8kq-ph97J&(T@3<5>Q$6kj59Z@40*0u&BkaG$zmX(F
zvL6#15rnk3_XJqOL{OkM<|4Stc0jfvTc2XZF+95>v~xsnBs0|}Prj+(jDIF^ZnP2J
zfwWe#W$zkcB<T1_{9`u`4_R+>^xI`y`$!FsXDwL|CJW2!wpI458W7O?V@u(ad;WJx
zKjaWMP@UO+zqjBG{&($8d)UP_N~{;b)j1?H&T{u=561sP%=m25A8@=CIU4ThYaq=V
zAU2t_Xi~m{{vMN1&;4n=R$t;L%(fuPs8!{yX^j2J2UHJb+WCuadl}|y1X_{ATprSZ
z2c;}bT5*(%Av>l}ZPG8#n{VCVr0Ibd-VQlp+u2Bv=yb4I&P)NzW<iPqNlM5#n=BK(
z-a$`CO5)@rEU}`@xZIXq!E29#qm$q$h~n2=jC*)}G8^+P?o=>;vtG$C{7-{~GXyAZ
zCF>)og)M^l+{wkfy|mOw)J7=2K89KyhnSc07WbBI^9Z>dtTlLb?*{oVd|>9g<pKXC
z;CpIfDN-;q$`KrwWl>gd{m<rzTsQ2n#xowjB=5QAynoqEbyCIRYCSR7s$#mk^%5;=
ze^h&~{&8saa0QimM=p4A3@-J#lD~f;D=4s40<|v(if|EFBKN-1dnPQ>A-Q8%P4&~V
z2b-L4<GrGuyDi(7hzrS=hr7}|kw;9@yB6D{VsIowrboAnZ{zvE)_y3g0hBd)Sa0Ss
zobWrH!y}1fs#ln403yvp5#M+Ow`%jJ3U%0>L-1HeWI1|+@f}M3x6k9FA`Mq~^fO>w
zq8EdahO%+kxk1btMN7ZJ1*oTjn>;7`oC|FUWg6iBd_@n}=Edw}L`Q8^0V7?<Uo7t@
zz@KjkgcXHC5w#z)Hy^oW(S{Izklgo8>IfNG-pxiLpH9U(q-ztaTx-!h#B>udx7xbw
z+;(z(D5Br_!;1_h;&kgl-19ux#CF9A|AC1J_=FlfM4lBpWF;@qFVq*5usg9ip?-z6
zFjpy2EfNHwuu@Oi7((vxn$go(S<Xe_aJlY%2Pl<{#(15PBi~3>IfDYA*zX=c@#Jld
z%y}SGUq>P&Ow3u7$2RKq-LzscnYUi`n?%<%Hu<;sw;E|(;JOaSqEiq0xIz#)hSg<2
z11L;V-;KBPGd@}MN9=iZM;(9E<<z1Ni&OT8z!f)_2lR&*C{Q<Ix~WOY4r)J9y@w0R
zuE*k@tmW8coJyiEG}I|z``!NlK#=Bt19+geq;z^!tEvTz%q6I8pai63<}cFi_gt4K
z<j2eQr1oZIVZiDR&hSugln<QO*XH{p8Sn+AZbi0|F?|m&D!nh%nI|Lc`i)Blt-GX9
z-rx?F&Hj;N2Y=_udv~s^0a4heXpzXZ{6#RNr|OTiZz)?&x?5-^)N&S-dmOz5Z1`9B
z9s<!1tfr)G{lT!xvZN;OdaMH(aA~=Z#b!0=BUmcdIVGyU_2y(lmw~*8$h48TeTh4x
z%dE(T&>hT=-rX0-xR30xQE(YU%Br}|PJNa{pVx^Qtjj#}?MuXf6JxE0yRpT!h%tT}
zR3brvQ}g5L!{=SxMO>b@4m}Sjj^WamiYd@q|8kS6iq#3e#J)TKJsl?_wm|!WU*VJO
zO6{S9x~EuG%j1rKgN?O}2q$MUmK_<g#MrNOW)fK8Itvt1O)}qB?+LE+MR#Fl+k-d*
zMX}-_fYSK+AvKGoxTNf%R!26Wdyq0m9dbWy_C041hZ&45W}IN35!<>CkqH!{=KS3c
zn`yafbX*>FS7WqKo%eBzkI<t>mkw+~xW`-yN@m=+nmuijp8~sJ98?R<Z2A%qDbvSy
zyj0E6E<F)E)G}r7cyYRhS=c+RlLIa%iRTcWRp@O7Q0&SSbzuzu?wu3XMgFTN8Kuww
z4x!4DRE-11!4e=H=R6!Zg6R>ouv(GLuyvCwyJcZV{Q10CrjEF_f8}=A{-o1r#XLj~
z<8&@8ydYTRtMw8D1UF~Zz*I=M!lNSnfd5J7K;AJSXIm?2!r+tvA1j-%KHJax2o+(;
z(>Xo}wRScDT&9^6YtV^p><NP}fqCO3fd8rFd*zsR9{BP?Qi->fi@s&Qu}f|1bf=`;
zkmKk`A&Nbk`SbVG^_dsQ^pTAXP3NofR^t~`v0=%d(=7k~!=wB1wm#_W-#*;>VT%&&
z<8u4_jEr;k8MNfNW!zdBPVc1q=F^?T81qbq+lwe)#~Xl`d#cvWm59+aE85V=8vs78
zeXszR-Wir!b@A#exr{*LyI?0UD?SjL5LKu5vk7~0aWzWQ^N(MnI@m&Npmtw<PULcl
z(!F~}+^=YtISDr8Eg`XUoRJcqV|KUxaQI_f@kr$yL|VoUoJE>*BT&Xkg?dHIqbog%
zwviZ~p{l>sa$=&5ku>$))~xHYzm%T|o8ugcCou;n(lLs#M*dW*^kG^gY;Sw*E|j~x
z>t?W|VyYd~ck9m@0i{eP`Pl>L#Z$`K^-b^~kR)?717f8614uErrZi`ThXcaFQMme@
zKgXf(Cla3L9V5~mis-=P!Kg(iid%~pkf-V?((Q+C&Gb`EUwnzl!4g~ptxgZMzA`?m
z5ky7Mp6-_V#$AOScc}FCf*f?ZK;|9-9>G6L@eIy2Znd1f8=B1rc3>}){vrPn@AelR
z>iK7(2#5HYBwV~NC)B5Eo^8BaB1U!~w*so3(7x;DJt1&?7y});%?F4Px8{8e<Q?~}
z1sP#h>VCq7A<OiOyUhLWB4Yj1?+?1yh=vP=l{3aZif-}RJ;UR^7-b{KTnny{!BtIL
zj}d{PC=`NdTC4WEjphQo=SiGYe@b$bBu%yOW<m>atXrjTWndYF0zMjxiKn{jNN8jP
ze2p7w9X<6`fQV5EqcDkOlE@{j`_1$J`tOU*K{A!#bGd!9CwF8y(d)0AgoJ>eGRl{L
zm&vli?I3&|)KQ9f*>v{h^)dLMRGuZxsA)!mQgWe+r=X}dP+O$i?-9<AC``6n?G8jQ
z6HWJ2)WvtD2!lEI)s%qqmqTKIRPZ!|LCCp-Cx^;745g;6%Nxc@*J2cY^xflH{^0`=
zKFRm+TmlTIIjFYZ9Lb9~g$biS0{sPl?(6n3a$9Ij%}yB-`}|3aWOboyCt`BT5W`Wo
z$8AXvOdO$0O2JGYI1ci<8%4V4XDb=+<NwiB6gPG|<OVSKw{I3%@&g4jBV_#sl6Ssi
z{d|dzbfN;<-n7{tI|9KyiZk)f98H`#`VVJ5YXENl4~KZ^v<K4hxE-WE13~>}E-QLY
zG}v{*c4&EM9H>hy>#18qjsTgwDX*)0r@vgSe_Q(yM$++E1XUxfZm<4V1i7XKKK5CY
zrvm3d>EcmcOm;Rh(8+7(w(~M~bpcLcuJAW_n%%@J^~!;XJ$36UwqGrc#|+bF7pXu>
zsN@FI^jE|u+B+p?*LdBb`aSIs>PUafPAyheG>PuI-}R0h+Fjeul@`W<V{s`<MNOYH
z3~{jF8m56>uusBv(D<;?CY7S1(I}lZ%)Q%Pwre_@riS&ZC8#Z4e9(_oGB^%Y=+ScT
z<yBYCOp{a&Rb)wtJO%}1Wl<TmM5_0(&&s|*8bzd8OxzrnGB;QG%})`hG{Q;lQ|;7D
z&bk?trw+-6dE9m*k#=`u^f!;mZAb4v_)@k>4rAa$T6()NV!!<OeqYzOCVd95po>>~
zB~-mC8V^fZ!)(RA>5JGq`_MzAP=S)whx}c^Oz;n_Pzs`(n>wS>kdKzEpX-ObUDM3N
z?JkoDIvoQAxvTgH1%K{#Cd-&Bts9DRe#}$t*yGV5!El5dax$CaU6B8tG<i#?KN4Ft
zq&~XB1gR(8^sWN-I#Or<D`nvQR!hB60V@ngYDxVgQ&Z9_)>p2#%y>*9?9N#34WFp{
zScv&US@1&e=`0WU&h$H1$RO2IU7(N&A>Kl^g-I(DV_lGCqxF$wcAhhxk+^^SyCgMF
zj#{Cv27t|Op_;9BVanbqXIdkKF1HN6aZ~EW(t#@9>FS*YfnwVIpc|<d8s<T{5#=on
z-iQN_^ARq(k9o8G($7yvNWc+(ChJQ_xI8J<=($GV7?%k!U^E&-G}@<@XyO?vM+ndI
zO)ep7Q@bo-{@Ry8HlxnTQu>5?$;Cn?04W5W$p9(L;G~X%o*_rXq>D+3VSk^hFfEOR
ztnVr{B90Aq5E0F<NO`4qE<rGkT?2DupUL2@1I6qBsq+=IVnFus#mYGX{lr4{$F7mn
zLRjmsS{rhu(jxMAjxN^zjATRQY_nqRVrnAf6hl9O;}s#1nx07-6$U%Y?u<2ewvG8M
zVo2s=1NQB9ete0Sah?`R+i>A6Lw=c=pnG|)xP#5n*VYA~6DuqVv+0y^SN$z!`VCx=
z6YHn~_mr?R>(EZ?WN^_rKL&}oLqWm$?}FQR0s6-H61fn48swU-895_TtZ6ly1t%(O
zL$jHa>qzB7h?Z<|>V8O<?=bRjedI2{h@?gAoD~6$$WHF$`3*c+GWA)|stU$90c)~&
z;gMWQW2XbXYhzN*RUiWWf70!#RC@PqRN*^}1D9rXY7G5DxxT#j^v!fMmPeI~C4s?i
z@%ZvkMocY=z`-=`{x-2A_${hcas@EK%{R(RMp#>@Il7TFF`}@U!gWOMOK`3({fHEZ
z2raLZkk9%S`p*fo&#|bJ#D_ps#X>cq3inC^OSd0W`*<Z4*u5iRb*aXOfA((Tnlacf
z(->y91R`N$P#F(IBrZ0W%^=F$dSpPg&&;LBgxW1XA2^0EG0hvm9Eph&1bM%(+>p}F
z3In&i&*VDdF4(~6%-f$bMm!8FZttw_5%1tGcqh@Kc-ZaeC~^!n{%Hk}A=!>KdQzA1
zn}_9suvt?lx-Yh)cQEwY=_3v0_0uqCNwZ6o2i2WF(%Cq|fje<*okdp`)bJF=%}bn;
z+72&e{d){8=}CLm&5U9TaSQsZVO7m8PKAa@W!9M4gHceq%r$k?VYW?%ij6WYt=7om
zi;90HDe>YIc^iuFtj&I(>*mZN2sEpo5=l-QYVJlbcMdR;<Ax_YB8mL?m3{n%bdDm-
z^VCh0ta;Ob*gG2!?ULT>Uhxt&f|no~3iiGPxTtOAgdx1++ZA2}0~isd%{BG^ghita
ztUgV=_e$U4wns~nM4(eooAEyo8!8OvhYNAdiM|f2(kRM_`}OW(UTUMyDL`Z1RsWN(
zqkEWAJ-vUJn@Wr&Y_*Ft6dcpH38z*2z$8TL1LXq_6uf)tiBxTsO*TC)-%sttLljN{
zgA@zxLe}pX{yz+>4jQv53aI3wA(SME?d+j{Cs26G`#|%oHelcnGnXFxz#|IL>Zhs3
zs|FCeP8WybO*{5w`w^|95SbYcgQu}WLOQ0M+VC-+-_vw?_jH6{)aZ~S4Da7>DBZ<m
zOV0>JnTWkj{M#_oPT|w=YKC^5%<p@LM#T)by=m`T8h&cV1}&?i)EE!qwN7Y)AiDxk
zmE`JiTw&xhK~iCvDzr-1r8RmH6JUV_Nb2U3RbePMCu~UCkj*ysaQW<Pa~NZ;i@6GI
zFs=i&H%_(cnonY!i%0+fT@Or`M=#J@pQ^x7clc7>QGxi#K(+L-p<L!X&_)l4v#h}i
zdy0^5{Hjfm4rhDhQ~ld91$#ugV4Y5KDK}S@JeUJ#4?LtNd=2%ymOc8g(x{`M8qT?{
zW)b_>I@Q=%O;@!V|KIG-s!i{rEINMyzZIr2qu2W8mCt{@wuAS|r*r|`5EI+W(orCe
zlb3OB>VYLAbGg{mb?#;RZW+dEIjX3Z0j8fMvoTK39@EZV#+k`WIrl?jOdbf8{h)O1
zmqzvZl}DUv=pJ}!6yAtF=@5Kh3lp1HPv6ZN2s0T)LDW|kd8$(YdDdRIGC}Xq8R-tw
z?+;ry3YvVx*sb?2Sto;=3C7Mj;nJUTpxvPc%5!#_)_6=X2WXW-3~0}Uwu?U8d;<kl
z8N@6fQuMCOa2SeIJL430W=p)KfckzB8Mg67|J40R<jxGmNH23ku~}8y<Mx2T!T=uJ
zrk+>-dM$@-AV4Q6@^27mqw1}U)mz66v9@jN!q~L0ZsBUY#ia?5ovMZ)b2GQ@N4fDk
zB@culG1n85FvL^oSi=*D^(qw8kLv#!(7)L|v#DQ?dkN+B<(_g|w%*b;EAu-H7Ok}A
zJ;m6kyO{<xWa+Q&Z!yqlyL?MQUIFEb-K;A$h~5d4l=F5U_BuXYukPo^?ZQb(KW<f%
za_zr=xG)GT3t3>r%kB{7Zlg<53=t3+&bfR6%toWy$L)=0ZSGpkP@j3KCMkW+m^or*
z3myRPl8v3^-#v_0%IHP6GSQ89ak&+2<~cTzDCKL47Rb-f=t;w5%n@K77bi7&ZC|8w
z8)!p(%Af+Shs;=XHIhSyaW2%sm|mQMfVIhc=$ahoPlel`JK+nhD~(kjD%oW#6>SnO
zZ=Z&)@=M_Q6j~XPhxqpksG`)i3T00OAN2_Y@r2`ubr(4_Y7wO`L-4r&FsJuLek2gK
zQ+ytk5edvcN5%c{jQHD#QhUQr1dDb|sAVu5jevF3K2=m2?7KBC%X}39t;^$g$iXYc
zmqz6}A=(ShB{jTQwoQ<Nv))Aqf=Y{ns&0ptDEc8VIp?q*c~bYwEXP&V)AluBa=C1z
zx<K0$4P_+jK$<QTWzXDyKAWpAyRx6s46WH{80Vd*ZGo%ZGTowzOPFv3evNvlAvCAK
zF*3^}>^}qm`N{OOrFb-!dhjiL->uA`EwIo}UGIH=KQTpV=EgyYpDV%_=J`Xc>B4op
zwy286uVuA7m}P*Fi{T=dnNc3s&ndBqJ2IR+P3^+5Dk<@_&Q^5=oAFU07$8eZzEF2M
zOz>Pkb|3gx*s7qVf24O2D92ScMFn(pNH-JHcooz~Ol*pHVCI{bEKi0%&HMKUwb_KG
zW2#xR`)^*#+^H@dlSaDxRFW<7J`;k6bWGkbjB0(c^~gvA7)N!QZ<6igNq%{Y6}`3@
z=_dOz&KRtlDCtaQqu4iDi(*&NrT|21|5M9_mBK2%r<D#nUKgm{!wD49uUGKE3*Y2<
z57dcemdE&_fW*{@(3M{ksIyHuY9vzEDRw-+o*%gKjMd7SY_-!E^jE25@bf`zx1Otj
z#-L(>4)@tOk10>tN&PJFk-EFIY(4nB@N^)`3jA&|o4oCP1&a}%F8kR@&DWE`iF%jD
zK5`XJ$92x`-4UGr46ZNX*?r!gr3~h~7!h|3DfcN)C=G^zLTgwtHkRE=kaAtfw)0bV
zBuEgUCXD+>2P6JfHf=gwy8&Hy6p6|Jrwsw+Ers0+)hy~jn*{jXSX}YMNrJm_$C@^o
z)cg`xnPw3r!7BeeiX;RM7>BlNUlk^^cG2X59sK#pgBAytFNchq4*Dtb8P;rVv}?vg
zBE0<ryvc!%PUJDL3?YFYFu^A_a1H5`%r8wCVry;$)DVbQCf`i%-(yBGd!0DmH?(Uo
zpe6B49%~s{d;U<G_x5=&%3vP=Z^lg%@nVbwr<^`$#!-1u+g+d}wnGKPswm#HDlY$Q
z5xZ)xGge<$bxk2P7lQxee{oS;*VX&u%xmeT^k$^+YU$bHJz8Kw{iy0*^=1xZV9x7z
z#Vx#zQfAOTCAtAUMqMGa^o`mIjSPYsgurca(YDajS&OXHHHkQiISV#?*^Pfu`l85?
zhX8Q(bMvF4o7`?fYxSQQD7*$p)nqkIQ|0v1HV;zk)_QQc=8%tH-_nQ?TrKZIn+ydr
z?MwIg*~7NT2n|z98(tw>eB1@Z%0?XIeAOE8oazC#53LqUN|#5YJ3nOoUypHumx+!D
zS@srh=gpwqe9+PG!X6@%MrzkeDtSXx>%Pv$$1-k|cM`qWUxQs#DN+uD{#aA<%l3<;
zH2)h&c(6A3ZHEIkz%N{?{-)QaI5H(k+je+FeEN2r{LeTVKLYIU);-u1Py1{dGPxb9
zm!-JZ+$2#S{*sv%BDlqH@u8vyC;yUwn|S|><Wd?`4X?|0K_@&3L@G(t5EA}-zlV*m
z)|<u6Og|StT{gsyh#zG9U5A8Mj=!A8gsP}9US~ZTbF3*h@!ggw^ev31##0mMo&NKa
zIIBJ;K3Mvukz9ke^*PZ0Rl}$rU_{>XUv4k`Q_pMQLnn1G$s(yPJE2+eRPjlvKW@&d
z1a<g=`>*V0b%F())f|25<~ljf&T?9$fM|v{$DEJ#{Y}{xjSw;`4{eYlFp+mCPt*zA
ziQthWJR|j)pVrIj`{ftIAS@aVWAPm6E!9h==h@XB=?FuY&jtzkb%=Mud9t&d@G4n8
z%x(C+!!Cpkj+aGFQ#ggD$Al4jUU+Pyn%Pdp5E+H_M8zi^H`z<$_Y!4o1oCCW#NOoF
z4^1SI$nfl+E=GQ~ZA!`QOXl%c@Q72=P5b(pYR*%79+~=1n-V$CgqcMt&j@cE0=r*a
zd)~j%pP<ELIT&(BAE|=7z{j0cTPh*wb=Wl3_1qqi^iYTnU-4&{5S$v~85H?cUKXqu
zlO^xuM>>Ze<-3EFN-tAvB6`wT%uaZ&x%P#|4<tbjcGt&CThvD9LkIL?B1xO)qVn03
z<cw@+B;TaGM@Eu93bn^BHNkEv3s<B{Sjo!IH*z52VQ@2+fR7(7B9v$UeyYDIdqZKe
z#AL&`hx!(tD2pf383XW0J@ax9NNQ4TErL<0U;<EB&;ypcXXFaK#9kt=Q-HRVH*y7u
zYH?t60y7I#GK7|0Y$JPG)e(`m`pU{P*SVJ={}c9Ah4nn<Un<6zayJZOz7Cy=R{0wh
zh;I_<C;TPh_1+|k3fIQ79o2Wv><7X1?)Ai*7;n2nFYxNJ|0GzAb}`JYanv|zvXbM;
z7Y31yamCYiv(?MWU3Uvtm%e`p)gff-UKE$O#>3_kU1tGBT0r&!o`Lq593`_24~2mB
zJ4+>UXS<Io51k1L2RtT#EA5siVTtQo8yr^0940ddzLCm4MKBWY3kH$2SY!D=bJ~I{
zh;Nt)&mOo`wyW!@PhsK4_y<Lh4lAiy?5)Pn_A-gO$clc=KVxuM2}xctFbF~VEGXuj
z4XXnVsryd*GgIXAco<0J4lx{}qo0t+!%g!e=O>Z;qJs~oZf_{NB!|-{>%5C|t1!5m
zd#7GweT3*eZGVv*nDyER?D3?N9MAt-PUeLu!v7~@!FCxGX(*xJ<-5GVZzWs}9g@5E
zm<C5?gP;4{`l7G6XU+JX#4)@1tAj%g+Oq2fZ?$dOJWY!zVH{pAMYMgPxVyWce-|tW
z9%1%+mXH4w|0&i}f~gcoIi?~Xl?~%h9n5WET=Fo_Ghd10wXYn?=}(4pucSZsW`L%T
zj-o6&a+wJwaPVX@fu^9w5xcd*<GX3fNPBW%b3aWk=g2|~d_j=SKRg5lp7f{iupWw^
z*aswv4atURbHKQ3!WGHpz`0NrbPd6y0L-_}{6JHX8MJ(`HBHuJDG}0S(7k>LQVM;q
z!EVn@{Lrqk=zV+`FD`A$rlze~M-GRsi&~zxnv3R&$$u&0mr*|)|FICbeS9d=Lp82A
zPO_|Zv9Auoc{l;ld1_v!BsUdhh+n}!56n|OD{uQmP}oL1WZ=ZNOu==NrK!ZC>B#&7
zjF?IF^qhCEIp%wY>G=ZCZR&kh-(lr@Q5TY9zX;G670<U}$r{1Gg&7U|qkAS9V<}6y
zGP@p(<Ea5%#LT&;Qjl=Fou448T(bQ7)I#WXP2lR*?U)32{x5E|c^FJY!Pr0q9L{T_
z2K87y;9P3RD+0H6ct#NEu@-&JNr!S>56@saTpD6uB!N*2RD7#DJF#Q^?#wf9ZZ$dD
z`a+6}$Q+O&{8eW4XLMux$QyAQgMzeUq~j0qH}H<?4~IL57@&|EicSNd?x6ueAD|>3
zq(C)wImIE}S9a6X1-g_1MUkWmmVa|mE*qw&s(=vi<ru20O#uAM`mHsGblGEGD>R!e
z<cfI>J}ItB=nz8sZG9XZ6LNfy1=-mL;l}T!e5VtShBF%?DD{%=B;5GB%XL%e!Xqm6
z;sSQ29TGo#@{5YfD7Y&>I<l!;3YR04)-(oDQx6IBlDF87I>3mND7=xsSZcEi2`19H
zh;(=5>lk1FD2S%RN#bk-jCCRhvsVdD*<dxN;pq*<iIG-Qj$}kqe9LlQ6Y7r0B<C3m
z)bJPEUPjP;tQM+mZGw828+t#$hng%v?|t;Cbzp3>rHSc8KNnFkFV`YDA4oXK_2;~{
zzXOQz*{QSx)+QDjp~$hBs`DlDL16iVxTb{EL}y22X&UzsC91+)Vr5)-m%#B6=hk}2
z9Fx53S`wZ{94EgWqUs*wOh9c4%qeZd*GXlI6N7)X7Eex_RbRHxeX@pAXwU!Fa1X_C
zZ^g>+8-gU1nW$3st{V0>-{fOa#n(XFcG6U350m5a$((##HOWk@Ct-|oEX<|~YxRD;
z-<t|M@s<-`i>OC4p!C<dG{ls&JEQLrLRt)ZBvSb09&ApB6E!N?ph5RDN(cdSXvBUO
zYm5`Bxd{KtKMs%{M_gzwx)5Ht&ez#h^sqY{#DfIog#DlJaAJ5<RMGn@ohI27c_th@
zX1Sn&ap!llet&~DmD1E-b<~J*TXETY2xdei|GT=nyzCB_BW9mq3<d*lrQ9b)_a({2
z4#yz9_JBJj8CFhk`a}tjrd9sg^>Ch{mv%$&9o3naWAv86dc#Rm?pX&#Z!x(|yED*z
z4)#M*Xq~?Wnf}Zo|18!h_M;yIn;M}<`PwJ<4i|cVb3*Vx<{MeK6e8dC8j6Jn4h#T?
z8rE?-wU^BpYrZ{krFa8oTn&dtS4{Bhi5lGhxj#aK#b0u=bl;AIa;`>4Vrg#n5Rcdw
z_17Z3hMtsPDWduList=<ch~p%#Ph3Twtop)0LY+n!?IScW;d8BO?I~-kwLuSW7fUJ
z>Q`g=n;f}C#{cc|n0r4$kR0<9#;^9`FT>qF{d?@j9=q1}Sn8?VLtTBVvDZL?8z?h0
z_@9K9F`vYI;;&4+)^la0KIHx#%8lRhyGfrn45PyT4~J?2FcH&KR7)0TXYjBQ17IAf
zC(m7<`9(i0vsYVbsIwh8`PUBKZVc1p1&#15b9c2(UHksy$j*cVQs&S$sO6^9QwC++
zVHq`gT>%?MR{y=#;<k#wrcIEKILVO`1_?iARe29pQ9C@YK+?u#&hzc3-4A@NHBw1A
zyOhQ%Sq4s5$Dv;tz7yLbOvAYNdm?S9B9P>%3R`3GdhFQ}8y(T{=Q$Y?#ccH1EE-Si
zJ|#<uI4B8~K&@L{Vol?!I&))_#xqcguX1QAnKW&=;{~43hF}gr`?ExA-9E73Q7?>u
zrDb=MVK_2FVaAStnc^ky&RNB0R6OI6VZ~gz?Wlkg3#M~UHMsp263!`~uR6eLLzydi
zdKMN5@0?9V`<MyUZ39+7gQtI@3XQzC|2r`Mi$GZq99W8d6~s`D_>{>2ArRQ}tDScW
z$)$zh%X~5aSHu-O@}C4XQtD|8REaL%>u-j7!|It0j0Q*jN%V=vrtTbhflY0HPNH)f
zjP4W6NkHu>x){@WcNxow%eyE_H&SL4xnYFHy-@ToZQNSKm)`1_zv5O!O<o&>0>PT;
zA=o-uwQ{SaQ5?y2!V+PumJB2xza`1;pTWohb)Nq;a}={t%!V`YzI8vrzX2e;#Mzw8
zo{xhe3fE5&%3ZqTssb|Q>5wW3d3Os5>g^Lv&Yr6ntfs@(v<w>YkOEF@evXy5T?4WG
zo_CtAMu*FJl%m*?1p#4Y75t?V>NpZ9Sf9^<nWw}y6w{}ff$$Gp4BY3mQl!yKOVX&K
zyaXe$$YGFqt6KTGEfO(5=kaRlkmWII8T*9YdXc@92l<XScbWm1+oEvs%lEVr0mvt4
z1Z~0A5E9+~@U;qNyk*o<8)Y07GdRC`Tm#^<$(~AieoZPpZl7!1k=5uc%f1>^H7qy~
zmOYd=g(vlgrZkHXJCcYTW^cz(G595vy+-Y|vo_7((G?p#Z+xeZ-W}ex8Pd14Rz*9Y
zz~g0n{oz)reraxSy<BTB8jIDZw+0GfKeV0L*1z(X+gOOe9h!iNu33*MP*9$Mx%rg-
z6u_#%YUd=x-l_)IHU@Wz^}SFcu$b*qulPmbm(&+xt+)!QssL*Mc0wz~oEHG3PPMPv
zj3Ap|Ec^JDm)FdznbRiDN_90);iXZ3?V)&*KT4ls#OqA~yZc0#QzMQJcO^8_bs269
ztYRiW3yZDUrzVB^;8v(iTz|d8<Rmp`N{rNTR{&}k_8{0}xeGo@uzoCfYR?MAk>Uy~
zWcfD>{!WW2S1>;wp(ZLOgX2XMOn>O3b7{QKVuTY5Y)%Z@;#W0s-@ZSu7jrrV?-U*<
zHZP5V!P3}Ee^QlrU&}n2%xwwm_K@GONOFJw%(4@D0fxFn-N3ZxK6qQaFkEV@jK^|6
zAuGkSdhckOKu7B{T|)FeCQ*cpJ9}<0&vkq9xr@X$FFT<;RA&(4Jkvq!_XDA0LgKJ4
zgNW83Clehq_z0D~r9j_3_!CrjG&GJwUo{ydzGmUojwc1MwAVWB<}-L8y&?bac~0x#
zkHzM^_jZF4)VJy9jndK0gba-)(ra~<`B<!u8>e5%vdZ1Z-Db)t$#G)hhq}5M{)L7c
zZyxJEt?RV+BVXfp?yHODI`Bmj`bDtFTQDG~z(OIyNwBM2XPE&k59m&S6W)RnVofPW
z{aL=}uY61Rj*qNk^i325gQZhe!>x;19Zl=W?@dEg7qg0-QK0z(GfTGIacjXTyDN)!
z;9>*+hB14`U9>8H_Cr{uy-0bQY>b|?@*kxVXQnxMXWN3`u(oOag<F7nK$LPLe!ybc
z8+305jC(tCV!v~Rpew0isUQX>j~W(ZQHwjS){kY!eq;HU1()*{mC>6=AoOw@IJ$E}
zY2xDEt(^^%8=6rzLEB}b3xw0cY7?&D2gJ%bagXs<n1dE%qs*vvj|aH`5bpR8MZx*{
z=;)+Tz7A0}u{O;y9#@Jt)Yh#m&fFv2h6TelqbQH0d|#-p-4*la?H475>Yfvd+Fr6G
zh?9)uN#ld%ZcZgyxZjeOyoU3h$!r=dBHh)q#asPw?Ik$cIj_HqS@*eatVE7N(#7Ex
z+<VqzZO9f6l{`c+=TJRcEby)Al@>);ZS*3BR8UAm(J69_ihW+7xqoVZ^g$!6+4ahh
zkcVp??R(O7<od#&M#I;#*X>6LCOb)gJ+b_Kml7gEenymj%3;&0@U+WVIVr)8%+Iqb
zh^)(=tJ*J}lT*juVz5y_{d%HOo#kLqIP#UBtoo`&awzpz-4S9eOIw->coq5}@@0A}
zf7Vq`hU4EEJLd|J!7x&A#Pc~7$zyp(7h+&l(ZPI7m^4-|FEQl(2tp5VtE#Q+Q`{CB
zufY!1Dd;?Cy7<lw^ZUr0a0k|axJ{KiuH90UO-u(4ipQklk~;Ek%@AJedFhZyQ$|Ne
zP+He~YYZQoXJV%u<)Ng(p%7LDlrpnh@Ut)IvL;L$HLvg9<|6;Qv(e=wMLIv}`U#Yb
z2bXUb+Z{oKyxy7iD<=BoHxu#%u$8+ZaB$wHBB}VkhPg6c7EsdVzk*>|!`u$VNU;$`
z>AiG6;}0O|KVDDv)*2E>G3f&x?R?Fzx#Q5=;_i2Tbqb$1t%|R8iK?Ra3_EWiX3t<$
zyMMDl;Zi&n9^lX1n2;>Qo9oku5^~sU9alnLU^cTqv!;vUd2-2n-wxIc;Ed5#&zQeB
z?|R=Xf<)MB3L$VGD|%fHDgU%oJUOwjFf8qEL8s3nUifwGX4h3$lYKYRFAu^KpnMNB
zXV&&e`+hpK<A2XEjL2alp%-&QYNe>{sb1Uh+0SJ|$ohPhz?hy(L^A#Btc6)STABo$
zY5R!X%Fie`0Ls`}8&hgdpvuvER{efHjCeB(*jDw#0`+@?ct-wV=|ga&M0s7voH$uQ
zSV+|X4KhS$EkvjGge9#5AsAU$G67j^e)CC{=5vvJ#YrYc{gcKukbAk{cY(l)%h$Cf
z9sw)f!Fo3HZ?Spc>DXsPo!jyIIWQMjRa6o$9Jz+GVQJISiu02|&)wVLH!bQHGK?4h
zC7}KUMSBlUJzjbriK}!}r%pL_$_dDDF;)Pc6`5RYTK45y7I}jZ4buLn7a%>!F=uJ8
z50RK3Ei@!=5Y3O<DGL3A2~DdVgyPF^0?#gyDQV;K(I_r)rut{u-d%;1FMVFwqC%Ft
zc>ARKUa^PsyzosKC(;QJhFriFXwC&KU%W*${Z>UpF*Rst)%Ndj2yCZ2hL*^?id(m%
z8Ye(@lmeU7LSl51e%Z`tYG;^rnB{TQqWd+d=f*pLw&eEX!^UxxIC)~eSbxO%4H8Up
z_&zY9FnP%5n-_q4Gf}7WBLeCXgO8$kPg-QSAuAW!Zu1iII2hxZhkYCmC0<gMM#<wd
zPTEA*WB1!P`%_bF4Xu`X(Br^LqA&+@Wv7UyYPpmWib@LbxJvU6urHAeq)^%V(&i=1
zjhiLPL_O?xbI$3fHf0}ivP^^8-9p*<M<+S3%s%G4ihYpy-tmo1bx%)07OF<B_gjjh
z$vNNsl`i?y8Gnx&$4&erYH0sfzHS`+-k+BUh(wC{l&Y8=>q(rl9*;fs-#o!MSgx%y
z<ZZo4!8e9p2SwrHbhJO*)n+gJj(DDH_AC8l%>W>%$oUhOq|AIIEv;|&L5D6{rzj!x
zPno2ITK=<K&}PG&Sx60IK+++|^kV1W+Pi)K#_<%|&RHlP--BR9K*E+n&#D!3f+B4Q
z$^^Ymk28^Txj$V13v?D;BueCp(N<LKrs6+gQiu)O_kmgewO4j4gpA6)2e?g$`)Kys
zH=Zw#e3&ERBI7{|qM!hR*&#2P(aG(7nOv*WDKi>K>NRtw<CAW+SjCTw!Fa<5dXGC_
z1uu7RW0vkk?h<9)kVMK5+BS=azMEsrFtaleA}}fFOpc94Fx%#0lOj{%2aiH9lf-Cc
zFgjq@00V}A9QTBhsJuC^{d<%tvR%#dh5SV!F{~aX^nmrYKU_uhx5|~*4iqUeeOaZG
z`M=^6A$rcR#F+w7a!@VAX#ezd39ZO~_wt=^%>m@9j%2x%XL<GHER~Y}SgMzFKFg@I
z-mz>Zj}~;)B}TuA-LVQhJUk{DR%p<qek~|g_;Vy1k>akmFVv#W(IrI=ytYOwT_N2$
z7JuLu?;jp9>+X~5PM*}g;XlhUkbYzY3)PH)0<BJ#u+fV976sN0Q^%P@ZMMR$e24#t
z-yBe?kI>7$_qip1tG4Gf8Rh^7F4wGf-FN&x(QY1TZX6tCK&S}J8-q_;FYODeFXh(~
z$RTp_1y6(HY84R<c4hcZ|G)7!RxQEch=y#t-fU;K$ypCve6&$vOuQKawm<zI?r^-4
z5<8B8>Y5Ty^B`xVKZTVyBG2f|D(66fIk)|Do2=)=)WVxFV;(~ROUuZo$2;bk6MFCa
z2NZAfLE615W`~qg>;6d-=?SS?+!1%Ra6N(80eJPuA9((P`edx$@)BXi{2J}j)*8HR
z)be*Te(}K=eU?Gg5h+zxK=UPb`UPG5UT}O_pYt3eQJCQHrEk1*-wE*Ke}hFYfO53d
zYKY8;iZr0RDexNrX)Hg4dM76|A;#diU<V0g_Z{Fu0y&ZD({k<PE)_C5t)+Zd7D&U$
zA_%t_dUqJ*jh)vzsx7VE7~w>L;r;A1wmm9NN5dG4G9|Hy)D_lm<eoIDzI(LNMZ0y8
zcze2g-&h43L7Fy44DGv)8->=@^pO?sI?-xRyF<S!qJFkGs0>Zj+OI>0Ex{^CND!je
zU<I5(ju0_RA_N}Aq;mZEN7cmx%spk{{9?DJ8B~N?ZHUYB5sDF5Oh6su;p%z^^aBye
zy|TY}&D>{iw|H_cSN@X*p(0nIqv(-DpO5Q-0HWZJfe1SZd|eF)W4}+B8I*mF21r=<
zg>=ThZIlV3`=Rn;g(CM>Kp63$UGLmjcr=`I8+AbMJ^iovu-9S`<`70~wZ-2!W|D6#
zI_M^)*1B(*ytRJx)*6UUBidHTx&s4A6fPIfhbN?!lc$>SsYXXkEhebf!4A?0Y#b?R
zTvE%>`op+OfdS!kKecgn+X+sIzYBl)Jo_=e)CU9#^4U6`t)oY45RU=h@+?;Hr&NFi
zNw8L3U&FRbO5HP_#iE>7H31nSif#|WLy)07L;hIDVh)Iw3c0JKg2;XfKYH4!$wtS2
zCe9_aI<#cexTfW8G)>&6G(%XU^%*kBO=`wWPU%D0a!&1&s<pKaiow3KRgnf*7{#*i
zIVDOftJyU_#s_g#+P;wn^SB1TZKfCJ=YPH7n&JSZyH%L|+^m0;v$G%ozc&X!Ek^tA
zPJVLy*C)A%<+Q>YpF1?WZ}N)+q2QRBab#UQTguO7+H6(yn+XZt0j7>C%N!YEuxtga
zftvPgkhnzg`r>nb&|{7Fga17o$J?GygDTHo1O?^nt4}^|v(;~9OF&k8nsiZa&yuxN
z>iQkwWPLiGMc{5YW+lh0wh;LssLSBcb*H91zo5&e5Xi>htA?2)_c-O2v+^1tLKa2y
z=F7LstWwh@k9D%)m$X+Lf%-bd>JBR~seJnke^`JL5>_o}IM7(pS)@<5)gN}M>bj7k
zcPlqZS$T`6s;wkN3bJFmLzE*G=btSjnJp`aLve{ETX0DJL($A<<Ag;g9YHSgHK)_Y
z7c`l}LrmSI>E^wfWKH>Rx%<V4UPYgsyU;ol$0%)5|NhaWnm|iE$*B(yweY5Mw;{50
zj^eBjzaPzD)hFxv<$=;jl}ev&CE=7_quc+HbPnv1bz2agU}D?0F|j7LZ6_1knb@{%
z+qP}nwsCvD+wVU({dDiMSJkSj0YH%Q^-%0RdPOB{U2meUap>5WT8hQ0-`8MOdH0vy
zDd^ZO>zL5MZQw5Cki+?VlZ|ux%Cr663c*PTS04O9{Eg(VLJt;{LX2PKk!dS;aWq@K
zcWFVKyKS|EddD%<`s$Q(4Sg+u(a$n(@-M?5_7pN*jGXqp>=y&E$p$SUbxL^|TS8Yk
z^u<MAzUXct-=oCgpcyP0J)vCZ*N4LEU8+@us4UUle&p*1D9gGbgV^10)22dY<OHp&
zU%-ArBvNI9@>KFm`{*)?@ze!i-yHhhEvJk{XPwYdxY%sgvqUlcIB3M;MMWjVE_+eJ
zzjlTWyTyq3j=?PZ{oom~{cl8uEk3~_ei`7&MXM3{o5cGCCY)`XwEmSLC%O6~1`|;X
zG3sjhwK15$DgI@aef}!^LH(qj#hei4HC9da_4mkxRKI&!!A-Z>#lZ;JQ>8TLo(rN?
za=)GT_7uf<bR!!eX+)_g-#)chQ3d@J<@OGDJbX~yNSZvq$(dR3T(_YVp)m-Z`=Pcy
zqwR>kNjiMp{w{J>v7laBKl3ku*U#R*!a=RlqIDf`a-8l@UFbql76@gT03)Z^zQg~G
z;T6LNrB<m>t2GdIcDcXZ<pU!K-aojV)unaC4&M;|(Rf1uZ`L58&F^)Kc_<o9nD`@t
zTIQSha%?+SU)1jlyty^P&P`H2*6gDBS6@*2OWmH1C5Bx!^Oud4(v$`I-rK=3QZffq
zL82uo*@#Mz!vaAXaEc(6R13=1=_t5@uy7<^ryh{4boMCY+(l!f&|lg=hZmm%(7`+h
zWKeb&i8?UDbm%yr$lh8Ua2o{_M;h0JKQwbU63YaE!MGU)D}?g#^t~saS`pTJwtzy2
zqQ;dyC`UV^l8Iipc}G7+1&ki<0rh)LWi?|j=-8zs^w04U%3E8QMd;UmlJ80B$w3mO
zm`0D&OUnY&CC8-g<2OKqa{tk)7kGsn<o1t~F*l`-{-1QVgcpW`Ev?qmS6@5~fDy@X
zh0;`MxtB2!aw>!dTV?tZ?JzR~e1!L=&sSxMoc>$^op#St<R_06SJocsEi<avE@v|&
zMS67IO6+}EDXf-5rC^lm)-5nHlK9%H{2hY$g@q1ii%2Q7O;}h$!CZ+2hApA903oOg
zUVW;tn<MDaLT}5y$V5(1pLVv)f(!2Bbwm6Dn25m+1QC@%!@D1~?3WMqy3^ci)hA-c
zZzlXw-1+Dl+>;?7gPx_N&ZnUcOB@j=!xJ{an}Gn$Zjbd62Qa2gAZ@$;8Zx;nL`bDe
zrJ|X4QwxoU2I(zuo4?kY_qW+&>EEA2|GG7TV6mIl=^bMHaV*<qN#9oOq*5ue+(Z~5
z%B#x5Kj}?eT<|frT=zdKGsBb$_ONMRv#wuttRvX*?RfV*on5Z7T|Xa}b-h2GTCf~h
zPRtQ7&aeQO1l^Ca0uuZ{_JnWqToDuMDKSG+anjO`l~6|9gON>7%~pA;-i4A}zxm)s
zDDA^C?gZ*2w2a(XnMeQ}#Ad}3-)4gmB=@AUi*gx3potf(1gUQvQvB*z4Zbebd{IO=
z=bD_c6VBo~|5H@=BDvsOZl!kOzK)a4`O&9!OO&lJ-pvGbu<zXN8emWF*dy+fLG`8d
zi=+xqu_j(%XF%4tY$=&6SLF65ejL~xtULrPrk16Ek?Tu*cC;e2C>ZSrl-t?JNF21`
zZUJvwf1h5=ww&O^4?)p446e7Sp#bnAbM^3#1II*rGDB<rowlbkm&O*#CLpu)j_K{U
zxu(`0`Y^es)yzU|nPeJfmlQys<liVzso?T4W&1>}!2Z-)D@9WAV}U*!Krh3m)!{tC
zEoLysgnsL6EXHwZc608MoJ9$$z;tjHJP*9SO<*VL6I#F`WI3`5dHu@l<H`B#az6ih
znbl&GpO+S26+XDRoIh4jthts~_(B_~s<M9SptF1+l@gF~pP8d3;#UqJ^%MpkVz}*O
z6Lr7UGU`RmlZLm6o4QXZ1>*e~YV3Q>_j4U_&cu(W18)R=P_@MmqpCTds7Bs$*L7G3
zxRt*PKYTw7b&@&A$diGuIj$^Y*uogxS})&oJhmV%n%EnqKMFB@`25+`(qu)S6WGlh
ze%ba=<XQNkC+hEZab5>EjrE>6-%iuq8t{FoZt#}Q$Krf?@=iXoc|i2fxRAZL(u$Kl
z_B`_TozS1x{Gtfn4ha*X#&*itj@+|0sbYDUr?f~QesoC<51c2v#lp>v(jL69x}LY6
zw7+p%iV8$MkLE=IZA<V>)al=jY^Z@T%@t#+HD@VpB}vSN1~OX3T!5CC!4)ONsf0=u
z3P`4=kiTgFuqrAV>gL8I;J<U4wyZkW-38}j3JApodt7R-4YmgfIs%(-z|v+TOnP{q
zYOkY$@BsUx#31<yRMY;!!jH^3c~IRjT7?iEqtle);u1-;g;Gq$^yNa*{K|<Y`8_72
zl<<%<#V3jnl;PIqh7F8sDL=O<z;YdyQ1&=vXFfcJBvJaEFPbDC=2^8Ouje#4)Mb+5
z2^GQ@h#a;k0N##1!q0DCz`PP0%@UkWf7kfes=P<`{tU6(2X>-j^4pKN7rXizeKPCe
z_5j<sj`qAp<tOP6u#Ft5Wr5CG!vQbZzH1?qJ}fqpJWE<AL&EKcS)tfq=s~tojy|$=
zgQd91#7KTu<^MaEWUcZ1OrvbDyl@4AWER?r_Nj{YhQtwgUV-?|Dfe6x&+wo6QFXOc
z;C#EyIPJ0cM`?5~f>77D_#R?4?68CqDRQ#iNIU&|m2qcqlZ^;JIJQ}@0ssM88zcD&
zPgOQ~7(h8cep1{>2|LLTR`8N?SY_4`)qAtGj!r6@$ZMgX4;qG#(A9-s*t_&?7fL46
zX}A4OZ?!sVN`=lnl5;4*>~jxzgg6J%XX!H!SoEo>)i>-P^2^Nxfcuhp%eC(U1~ZM3
z#mK$ySyi4)wDMCRaX{x#8Q@1O*@rcr_wsJMYkjdRhuYU-5eW4EdQBzrDW9U6oSSx2
zUO66VUVL+W6=H;AX{T21rj=pj<YKk(Z|q&$K`70@!%moH>bS_K2P~gX+)|~H4wbxl
zv{bs=Bvq??epVD?d2h#uA)YvieR{RF5-UG;C*5$}>eBjXMU}wigVCP^;&Hq0QSW9l
ze#QMds_fT8X5~o0-X0#O&0`tjvl&6SXG~_S)g5cCqNRltrV43p5th35bmb39-O=BP
zimLTxLyIeYH-jFVLEAsyu1(Ir==yB4s@6|-Mq^X@tbPdg9MjGv#Q<EE$%r4f2Sp~7
zOhB&&iNubu#|Ci|lhu3So41e_0Z#LouHT%(Vd%Y?N^{g&P8&m*x?c-@*w?H$1!V?4
zBAh{qzF%L<E-Na9gDHZY)L4_fqAS5NvqLuJAW>JC080=PDZi?g-DXqbvupHoo<ZXe
z+TYLw!q>UaGae22m3;vDl(YAn90r47U|Rw6OGPS1XHL?6R{a+Wtgv81jkJDxA8Rgi
zj4uqRFyJrG`R(#yT`$8B00`ABVpPIdkZ~BpD~)OZM@|ya#NZHRXxD?tXPTFEH`?5+
zd?SN)DpnZ0QrQKtjV&P6^{%oSBK=k>X$}1Gn-Pu_k!|{}-0|aBEjiNLe7M1ItyZ$O
zEN!>Ft<9!4KKidt=2RY1=)xvyejG{}gqu_hRDH)oV%&=CqXNuK=z16-Cy&77ej%~B
zw!xP2eGI}VS4ThBQfm~Imji0PPri!a(0sNUV?&1D#&h0h@HaAHC%J0?`amQwO5Cw{
zF$B*lEX=Uv2N=o$#&EwV0YpI$y?3s@0JF8%$ndB3AUT5_T8->qm8D*?Zk(+&wpk&r
zaXBg?M!kr#U+x=zo6*x?_;t>Hw4N*>ba}w2u&nBeT_ZI%`Ju;TUaH<`Hse$eczoJV
z$bXHz-lgvtR6S?RIjgityrvj7VleKCIjoOA9KGMDk9~k*mZ|r{(F@b*p~X5Ld<!>k
z<Ghq@0@k|REQ;esx+3gcKZAMh?z7N9fpeb~Weop1+XHs5-(O>h0tU4cP4qhRFPu&X
zF0QWr5;=Ug??=6_X;W==&wC+>lQ2F?qt(R(5=4a%?}a#P9VG(FL9z+M5XXwC$)|cG
zCF+qIBqjw@1*6IW`%!nH-E|N@1pHBQ@4BnJT)f?_&9Jc21{G5++pv^ti&<jBe5y98
z-WCr&BhIgrZf7AM-=`&*V>QIJ@t?zqPE?yIP5T`FDyLX1q~jb?h{`_6H~yT$rVt{8
z($+2+;~Yknznq6d5Jr*DQBd+XherYI*V!5wfwsz2#h>>$1RO(5BO-JXxbr^%uQ-~`
z7fL7BS@H2WeGLEjIR9_AD|OqCkL`+22abJ$I)do}`TW31-mwW6-T?AMe%x*~f9))z
z^+j53Huh+7zyWvv9kF!%0PF9@ar34(x^{!k#~92Vu!hpZ$uqj8x9b$A>+t~DE;?df
z_D4)~Y(cIa^$WSyO4NAJB!-yKJ-!&vM(fsh$%P22{@W|5e;{TGzm1Gy>EnG4iP=H=
zQ5dvyfo6svavVoivH!$tGcaVhh1yzMBC`A@oJ!bQ0|V*Eg6Q)+!S>uA(%lAIu^^6>
zp)^#=f*AWJQRSoNR(0G1^tbYDU8+)&c6GioJ4`H7z#AKa#UlIGRDKxtWXT|4{)8f`
z)iDlBx*CmJ)OVoqr~umdTYHf^ni#WY*lD;;XXFZqBj$8{#}^zgIGCbu`$7Akz5Q@=
z!Aq-A8&PlC*<8yEByni(2}c?9WC)Z69^G4&g+!)V9?M^mHZlHSl@MnSA((D5pZBrU
zdtUW2`rx#zD1cY$Q2xi@pWO<wP-ZrO4o2NA@K%0WUDORVboq}O^SLtdax&xRUq@R1
zz8guz4NrmP8xjQUk}-@P5cCS;?>qt(q-Fep09c3Fujtp>RHOzODk5u=WpTnUWn@7P
ziF`iZnDRe_;l3e;jLf?Sqq839UynQRR1Ri)2k7TMxz*e@oj-8$5f-<<AYFcOcT<G@
z_?2eZ<Ky+o^>H)wY^>S}CEHYaPU<;;F2%4|)R5782l+SFa+uqZlx%{)%Z2;s6j3b{
zv>n!aF7)a%-#hC(xiWj=G`af482^0Q$<*H8LQB_8tn}-%%WLn`P0Loo2XL$F;-fnr
z;jow4=B1j?-gU9#*Fr!)QcqAcTqY1L-{@ZfRIR8UQTHtvZ~}oM*xIX#q-22Oxhh%Z
zLL71i3uRXWI_uuMcD1?<Q1Vwy2M_IAwdn9Y#TyW<gwmIA6O6Y0L}7iWRxfg9Mehph
z91K!Vst;1BY;qFA&Jn<dCYniVuC1kom>Hy{h?q!+NowsOmDGsqfpe_@<#kZdkNGp>
zdAk)KW+u6~Cjfm0bKPdc_jIFB&QTf5-?%R-iT(P4W(W0bVvS)dO5FxA_`zch)>FzU
zE7FggYKjzSOL$$}oFb%5RP}_0+DfExNk^A72Bo1RMGEib$jps!-ijWit7Ut*1rGs}
z!PzK@J~Ki2vw}5dA2P4x*m5>09An)@Q8<c;<k{$AWfrPa2Ftl(J}5jHBA35;F(FY<
z-L0VULm*;szh6498phw?*p?91fsKjg7D{#tdeaZrZA;n<WMQphH1VUyX@{aS#74E3
z|K**x_Lz?npl!1y%a?72s1Rudgxj_%iia30p!WWmnYL@pFZvxt{*ek3hOXmkwOSQM
zmAN`?fL4!@5DEgmKYwHjzDu96iR&E_dj?3#OaSdRD4`&M3Ki%eZ6Cj?dr$klOY9ji
zZLa$-OML3quMBRq0Y=e>Sl0)wD9}t_Pr89j1G%>|*@OOBr{>@!E?+XrkNMCVEaQG9
zROUnU1)mEr;rJ@Np|ZdtGWa2$M{%+h)0Yvy*?*Kr9HNihgRDfzr44QM2ctH5Oq5Ux
z<$XiA>Hx;HT{b_Ry?C3KZ{(0vlkk;Xhg8z*LnIlIB%6!PawSjrdpI(foPNQb=ns+K
zD-fj%!bG8Y_KtsKkA}5RkJvKTTW*X5(kUwM9=AuWY&AcrPH%H@I|fBOqt4?F=8~R=
zH^VT6f1xe6Gk0uchH*-EN6~~I_0~C?l{WjEf9E6}>xCj;gXrt%t4zIU!S4DH_4r~d
zIgJbf!Z6S3q_y+4RwdKWt{BPu2Q?`NT2{Oy69#9l`G*-R-(U~TGYLS#vQklw2TFME
z-1a<-KzoYH&s3~W@K~Dm0qW0PDvP-N3Iu3c5FuZA-6!<jasCGc91udJ^5ROs>t6p*
zTXayKR_6JRCTCZE`6kqiRb<=ob`=Xi!SUw$kOWom>{ljH*$b^qmHE-7rYcH2BR;N4
zgWXn?52RusrhDNxgm3?3VKpi%zKh=-0li`bd_`UzwbeR9-Jfs+589lugZM>*Armul
zt;aX!=q~+)$O5qX{pr(=L)un1QB7P!ZRy!)=7=8X{I&uPHL;na|I?yaq9NXc+N>)I
z47?n(6}1Gf;q|%|bCNRWZkw8(;Ogb2bG~R(nXYo`cBX6fhb1s_5E`Bvy;3>g7kZ(}
zCpQ203FfNoQ~amM1=_z1isk+hK!}dhsFyLZwK8S$%dd^r=wA){^puI-(Dn9v?M-4F
z-@6pR3uD3@!?dqLlflD27kE7&5lWT1fWVSyjUG!AW{)}OJL1>a4+x@Tk*Uax^v%o<
z_~@{VH^7TZaDbE{E_KXA4tyC@Gp1lVqz_Lz1Rq6WX%Jisf{$w#owy@9tca$us>Wz4
z-S`2(5<DmdUCOR+`%>V|xt;j@T_fH+)Fa?7k8{E0n)!Txji@X((5@^5XthyJLc#o;
z!FBs_kuvL{R0L|KwuMjA9n*iPIt29M02d8%EOrD{yhR(1iOW+|pWhjO!)MO(Hu05u
zLvVSc-^#DH+U@Y7o0Pgl>MajSZ9<a^Cj7@9arYY=$%Kmj0(I9QDuwvS2dcZQM%T@I
zc`a>FlDjxtG?fMBpC)M~_-P1W-y;0r7l*VAhzIdM01E*1aPCzA>k|f}LByxd>cR5I
z6ri>Ua{$s#Amim3I%+CFve9%QOWp%4pbAzSyA7?cELa)798N!oK`;eKz$NesYBSM;
z<+QbF|9kh^mTaR|7cn*gM8}BH>9a~M!63tCeZp=+#!>65K{;m1e=}^rCv@^<Px_Q?
zIt<?q^_<=cnX6%JjecHL@&W3NRs;MTIXYOEKYUK4QVSu(`g_Ssq1%^?N*>(CF^*Bb
z)S48S!M-a)A@&?|>wORJvxdE^D+pG6tsPc~i;*9ui+kpVOJ1YahVPhx_7cC!{Lgp9
zhr#>EJ<0)@D(|z9V2%en3s7n!5+2N66t8*62=v$8nn%;*ZTwm{rZZ!OX06SwFmbN<
zs8$iybi&S0;99njf927=XLZFFlpc3kDRf2Qr?k|RqkOWYSu&5k!{C5b4a$0@7lVNC
zrzik$Q7A)tH;gYBo?*}Oyl|8Gnfj)!psDAk01xXl7Zvk<?QsGn2+@<sMdV6)xU&&j
zz!$=s;0b)r<eZ6F{cqOzNh9Ww5GGr8Y|cz%XdDXLoE{(qHzB;(;Zh7c?2D0#6)|HG
zscD|w7WEr^kQ;hmr=-yWX`y-ZbJ&z98B%9t&2s8fSN#f4+VR6xrvpyX_g|n`WXlRC
ze=vqE6Pq8Q-|$u6<N~t>zM?3`n->J&&DdrUzK|HS;0if8!OC{?4*Qc!Ev<B+VSOF#
zkiOoHEpcv-3d-`QKI#2#a&H2Dyt`tZX+b9UzbZ#)*8T>a_am!UvG+a008Wn!USAP~
z{hdP$lg)LtFGc@Bpp0l`d(RBG-BhMnVVRGXSMRw|DTe)=#_NkbPmwHQM@snR?elIz
zo&mF@p2%qZ`s?sBt4;(4&3{lYTQ<?E!^x(Q>6m=@an%M{mE1$n_+AFD5+gqSJpt@!
zuUH~4`c09e{t~a6WIr>WW0+&gm+BqbTq}}Kj*WJ4&F*;<ezm+97JPpw*_@YXTS>;%
zOz}{rRq<-#>gp3Y)7|-OwUm5dJE9xY8@VIhgYDTG4rk{F_;1wIzS9H<N%b*>n13}0
zc){`cE0r0_om^$tK-!DXd%;R8CaKd0FdTwY6fWY2_!OU65}6<BsI#IEh=0E(uFPEj
z{P$``0CUFxJ2MZR_<l3Q%e^5*UWMXG{(x)ga|1E0&+TE-(DQ9^J+W^|^Un9xb>(n0
z!RtK4`y?-q5{M>r%aAmdq@dJr5-X&@?fz%6$i0Aq7HNzrU7G}Dl_;V%LZ4xX>D_)R
z!$G&3uj@IrwA3q!fte*+$0;q<qxI^%ZTfMOPnWH=g_?Ures47E4u69tckQ-ad@=vk
zWFNThfr1&feXEQWGu(DxpSIVDyc`YVrP(G_0NO1|s{dzH|M*98B(5%wvP`TbK~pnU
z`Ipo|5;ESIT}b`9C1)C*WtM{M=3sI|nHx{<pAGnM#TL1nY{?i@4K-fSxK@21^~I(5
zpmyZ41-?2yzNIa)?*f}TiK{JL(C*8m?Mzjwm<7=g>!e||0U8!z!j`7+0MII2+5onn
zP(6dtUwp(FvA=lEudif<Yl=-rz_L3OSziRJ!^sV{m0sPh)yic$+eygDs*)nV8oCiJ
z2C!;5mQ!gb&`h9X?pWVYH;xIwIZj+fL1m^f_kb8W{JIaSvt97~A<5WF!2!uy>5H<i
zskcg+iz<zQz+V320Z*6f%{JS30(j7FGzW|fZ0YHflQ|!W-FM)5p=4MAtMD7@S2QkY
zpr-%u=~CV<v-$ZL{wDmnhe5qhzb(DgI%IIWurR%YceR^kD~J+!GnqHSf<v72_p<q4
zHIphQaVyRjElqfLpsNU+F6pv(357BNN&;;XYXaG^Cg#@<=`9a4lWo`>ved*yksgxd
z7!7@;9Sm%hrXdl!sT;SY-9a0(Pe!dI#SLFr-3QrKcWSePt!1UgUe#_aZxnRaM;O~U
z6b(3M`6vmTj8`6K7^(byQf%?uJSE?rM^xLHxGE4;!?@l%;^p(nUnG@hGETJHd8Uy=
zGE7h2v|QI5k2^s1p<9v&LowIm%Q75Cym#u2WlnH5_NlvkilKK(YAn*ADwC&!oeamJ
z0zk(@k?-NhVq~cYy=L4K{oEI+Q|?RkUxmRW&b-z~9D>-wKii)b^Jm+-CuE<HR()i@
zRkaqxGY43_PiSBNZd4{6@@m9A!f6vf9t~@T4`!|TwYR3X--DxA?r<E$_>)5T{&R{$
z5JnmTrLPPWeK-p!<qNu{Cc9w<5MHmX-EE8`3(uCQ%ypR_>+qQ|J<=yu%_+a-NR@8c
zR!4JUIx(Fc1nRad$9HC3JdzwZPhDoLW8(1q<M;6iP9}(X3|0Y{ukD@TW7rQf70`!5
z`oH{XXsczng!p&*HG(KURFj2iIZqY=ND_bg2Mub2x9K8MrPszh6kL)b$QptumJaO?
z|JFTIK+I-@x7OGeNlmC{*JOWQc)zg7qfES<EY6nat16z!9RX1sGVr>{DoZmu!sb)T
zr@YMh%{OZcgT1CicF`YPgGprVJonv}Hx0I?R01!;2FUFu*^C&}pP_maCqgQ4M2VRr
zmYqNSk)Q)Bvg_geR!~>lT2;U(h;?zGDy=YGJ)dM3AyDu4BJep~kTHwuZDbdf0fu2J
zTNKE*)y_78q*m>9K+W{Petp5m$@@qhX0c%UfHUzjqXP!3%}lrUmSxRJvkLd%r$Y#`
z+?cu)z^I=gv@+nC58SPcQ>@!>_qx#>$0@ho;3h(IjXLCD=Tf1m+W(F->K#}$=pAfE
zmF@dtD|DO5ZEwkbZ{?%R@Mul;nC>XSlaDSDt*ctLc);om>b2*o@OrFBJg-rqVb7*d
znY1)f)uF1rGS5{iAVNK_O0;9SAT_p;M!#dkzqfl53WfbPrCVPaYP#sdT$%QIav4ps
zQ&lXaY3_|Vc7jiGi{vaXKPzi1^N6<!0{|7I{N=K>3WHetj}>m`=uN}n{ghn*KthPZ
zC?joa_%Y7}fAT663mNl#7PP3dQrzipA+vA9`)v^)BDuym7tcv^lduepwW+620;jwi
zNM|h6u9MB=O}_DjQD+<~lLd0`94f67NAoDsZaF1QPb>VGC6lrrx#fy}B-o#THC1u@
zhw_t`y+|2X^G?NPHG_yUTB&LVCwih6po3v)-l?V-?bNs8mM-KHpB!e)_U_6o2L6_5
zdi`L_dz|B0wkiL~jRCqxQhqJjARO=XV78c|-?rXjG`Cu?6%Kz$Gg%W^!mqstk@CsG
z+cFA|ZKO>--ll`F#llbmTCNi@?yc|O-!E!_)r~morb@mIsx7SC+kLJ^&C1)Nhm-vD
zN{4?pLTvMFvg)&395NV`D5ZRCQri<vi->+U#eV^~<icS!6tPsoUnbhu_s<XB@p=Ky
z7vqam4)gN0vL=(sxkE<|?J3Wd@k$#=#{yp50;_TfMC&(%XodylOu{g+Ok%BJC#~fh
z^;(+#^bn$%lJEh?b2V|BTn}M+%5DJ^)Zj&>;9}t(DYGh-@<nr}r^Pq%&jK%`mbCI#
z&I6C8nn2o34Na&ZbkfEBr}tY-42<{9J&xC}5La`8ZZEITk2mz#A+^*or6J;4m;5Y8
z!i!hB>l?W1jk}_>qM}kWeO7rK3;YBp$VQ4>Y8=F$ggJ2m1`D(*bW&X(nMBflEnjuh
z2_WE6g81Qs-~IgZ2|XBA5v(Hl!gj5B7B1p)3c3?a86eVKDO%OtxP@JF%#Xx>#^z(<
zC$06+!*A5>xL%EZpOhy=v9MNFj`aA3M+6?G4$|-#_{7Vm@|&vSWh0k^MbnA0l<j%)
zz^<dZRp~y1&bZ$FGMy=iGPAYF?%xg9^`PHvFYdGVDDrEI4l{q`sg4&R66KTVCxCSc
z)UA(Mw>vD5i^95SwNQO+h@U<lroC1wv^Vho&`eh?A|g!Ct(AqVRJjeS?#XWbEiEPG
z(DK3i?!FqM*c{p;rbF(xce0LTk?EbtxPf5}ldZeyISv52oVHw$*%wl(X4y%}8Nk%(
zsm29>2@gv>5bT`a`u~b({qRb_&wO)=N+``GdOjocK#8f~5XRe06jkct5+C`{g4)Qw
z_K>XO$dW91yb%dl(#HgqE{OM`*$vb~9;e#d`UVCj{!kJISx-wM?YwzF9hFikm#*yw
zL=c}Ta`67SK-KN;hQemGyykt5*(V$pj6$rQ%3yI=novhT#z4k!0IEPo6#s|Z%wr8w
zVhPF-!cRSAXpS;|NGb9UWY=wEWaRt&<V8moZE5cZLv3NzS>=b^Ki_}8PxdTupOWPP
zf3lS=vegUavl08&I-qH{z%1=#xuay;NFRm6gvNTqL1tUg@fHMYM!y6XldHIKC*k(|
zg3B<<1e!O9{J!lq)dH2Fb822+k^LeizEc~n=4%8yhj4KFY#$Lb(b`9#G|KS4$+8C~
zaCq95NY3oRo-I%rYV=(S4DxEOCrA><Tv?}~wPMZhKVZ1S_<V7vMy#q5cG!}`^rq_4
zrZe0mv?@fak3h-U?Q#;;_Q1!*=eBam2U!5+1OpnfIH*JI^_Q;kY;MdJZCAM-CwN)a
zJS^M*erL*&FF?i^u_w=IJE?9iIwdo&rd~+FwaDWti!RPWnF{AZ;L;BV@=#z~_iy%L
z&rnFwl!NWk7!g8Sc#~!YQPmpw{^X&`9%|V>g3v>}{WyyqMAf?HCNH_^yxZxmzf1@~
zasH)(s`OJCXd?R`J+7kwO5ZOOs9%6abZF1<wQu(yqUZT!z)|xJlqLe3`<<q30sLOw
zzr1~NZD1hT>XNo6K3k+RHjYgKK5vt@7fLFEy3ge`vSew((v<XW-(mWhlOO=QXKf8?
zog4PHRE@(GvMjjH7Z9qB;QLhx-DcDu7!)7_iqLo4NGnzV{1u^~Tw8n72Dv~KIfF+2
zW4e0aGx5&a)#=01lR-Wa3@I%9&c>4ee0KonTGaNa*?sVcl{4=3E#Az3fxue9|E!@1
zm@ptJz|;J<C2jZoYX3xADKqet$%eVtb#t2D9c0(N@%W?nt?f=vtV`vGt#h%Oqj=I_
zqj%<Gt19T)S7#iI%ZW<uMASQ?Bu%Yp+c|4nq;))Q5m<SzIm5HMpQAlc|0ZLU9ZW}+
z+Goo@6<KG=%1LuG2|pq|5`Z|8s_VYK*7T#+wrSE?4rd7*Y!$N1^V^b>g@#d0BAJG<
z8uVbjUiBAObinhMa6B+QJXE#ia1_rG`J<mZe`k#H=D4^`hcy75%ZLCX&TgH9kGcJ~
zX#)R<Y|T)g^}8t5-yU~7uUQ}W=$6Rj66kZ49GVVIM>(gm-f!>5Vt)#rr|q@#8v;ne
z<p{4O^8r!vul?mJA;sw)nE-H2_|tb4!s6O~7_*~)O)};YJU9kn-l=E#M($^1ezy%&
z)o(RK-9dRB4{jYn{bn7V2i;Y#A7oAjaTKc<@p^3if}|Km>}$gv=<$bY%}p6Ta|&t`
zX87f*`AC>vQ`(6!hNsk-y?E{@U)^SY&Z0G5G+N0nLB~5k6*q=>A;|~*!fh2%%yL7@
zA`a6n54stN{iUaZGWIZH`sTIz`x!Usy~<VIpyHYy0ovGA6ARG9z!TTvi`|m@89b-*
z7!^!X#6{r;sI7IaZ6qn3#3U8B<!g$=_=r{Hp4#Wm`V_cP?3Mk=H^?Orf@E8+jHWD!
z62aP7Ky$@gOw!rOq^_0MdEMM<Wu2a05MYXwCGHA#0;pc+&7ynGzDR($+PSUE-6}eC
z!zvm#4GdCIaem};n$$fl;0zSV>=5))L+q1}vPXe3=mey-tvLHcG>=|S)2&iSDZmi;
z_@uB25lV$K5>G%d11Hg_*BOn*-FNRS;cB{AZu%lg;v02UUDBPe7yUuVEf~-2dvw?v
zXl3#pc8wU8alR)*9a;*lMAi)Z*^O?D=Q9_^P`Ba}+d*Am{l=-_grIuf&c~Mv7n|F(
z_!{bdpr3xj>1?TYuX3dWqd$2V2(*%MJre`<BD&lNYeWmjqh>RQyXaG9rS!uWr@P)P
zwmxjVuTJw5l;rPN2W4t=$?sRn^HK@mujr3Fuo$72Rf`9^U;V>4LxVC>ruX(!3D=t-
zzAmS~{n2`nh^5^XK@VK!4;-o=-#g(YBL043g!1EO3bVD%US5dL>vKwp;sr(T8ou<{
zoX=vCg#n#>p8@czqi(Nzf4c?3zu|XFCplJ5r#iHBYF7&&?@~|w$F7WcpM|T&l`gp~
zC}4j=fgZO*l2zc@67j}G_>|E-0uMz0yl*M6viuRs#SYzNb2TrgqTHxry;bf{>QTJx
z*N_(GcNe1qI=p$JMxhF3WJC4SV%O$yh>ydDHRM9vmuKJ&Ab={~q5#iJoMs<$+Eq?)
zQH}+LbmRLNUr1?x(Uk@7lH&jXXkR+QZ9oS(swT?wz77KnOhg5<f(q7pWb0k@)Ius}
z{e#Kp_ZK5TeMD3>_T1_yO>rj%D%jT}K3~wOZ_AQEYVK&o1@Sk}-uBUlq~@3JOWPCJ
z{e+d8=ya-y7v0C#9XK$D^D7GukGCq)A13KGlZd^S90Rl1tEEwa=Y84YNmu-JFcIMX
z#&tF`(aiU?V5I((ERttRBio+s)U9Nett%<8GAiRdM1lB_$r`42iQNMT48MBi-!sp!
zlCbogFE?<*;~SMSVPb|S-V|e-yKKZ9q!3ZYk)Sn~FE?{2^}ec;FnE^bTohwbi1c*f
zk-h&VOD}=&b9a4-Xhj9QKC#kLn?a`TX1juh<45#D!abK3HMXliws23N8d_$Xjy<UJ
zmZYrv+<owLhx+~*t5bSg<zE(#uGtQvdu4^C%W1t3VVf>mG|LC;+m@EE;eP?)5Bz`S
z@GC6b6xOmS_EITMr&UkpR4Epom*>J%8rJINUl0h2w58fpoR7Qt9=y7r-#fR#GYjG!
zkuX|Cs$w-C|GLSxp-MN=T>U-PdU@<J(!0<ihPqoiMFRejZO*&sS(f;6%x;tLfZhai
z97<+E)_As@{$nwFb%8Kgri|7_MR!2<4fe=e2YFE5o~x0{ZgKQ*l<>1rsGhBIp{!QF
zjLxCdrlWdjUd-k7#l!N&?yk3WX_j=c&t5az`L?!Y$bcBb!IJDzi8yb|&P?}_?<?Pw
zwd&STO$JG2lihWEC}K#&FrP66WCG~O$46rVpo&l}Cw4EJktW+bq(NMfRR-=9$#aO0
zmAfcJm4CH*s;H4;S6?%CW1qevWqn|F94{8qIq8;~@frePzG{I|>7`nk71*r4QhlDT
zEd#xJ%2J*)><VOHV1Q$ZE%TLm7v`6o`MC(iQo<hzzC31!?@gaQ1U<-eaUTRQW;TnW
zbb`0r0`@6B?Gs;90APq*IMhxL^16K0ep`38J)SK=_p@8#u)-NiC<|Rj?dhjrlZkTh
z(p?Sn+PLzVC6-DZ-hu<7)e1%<J38_5j)kYgvz}FL6zt{DfM3#kTsJPPUt;s$F&if|
zA@zK-2HTK3AD@koZXPfeLx0zGuSL*vV5suxhbq{pw)=#4-wO*02BoumG9+=Ecn)a%
z<=Y{O?g4AR9o}&b#_XvEK_ivE@!0*uJJb{O-&g(^6?94ql3XsHY&c33h$cP2x0@nK
zHfJvn5=Z+xAEr>{+!&quhb%30k}M>=YxLQGf033vTX|U{fO3&``770r#)TgI?=B1_
zO69j>fj3K%wVs}lClQ`s;bVmS5QQPiFloVzV~%A)vr;`uWc*LlL^Oczr5jH;AP@`>
zd5`n>M@$gwjV9^!9FG`akqL8BIhD8lxE=QXW*~f0PbXjf)@$E#?!0`1o;G0<!6J@~
zO))P^D=Wf6c8r0W<LBXxsS^uM#;z`)y6O%N4RvFcT5@r#1P*KkMlHE{w)^yuze6v8
zK@T7!CVM<hhc>a&6#hh9X>!erF!@dOZLDz&6BXz8)<H`~CUQb|-XOVs{1q+Vg4hB{
z=AFrpx5>WvN&PqEny|k6Y|2Z6`wLh6#o^4mS}%`5U1Q_k9Y`+4vXJTPAgX1#+2V<+
zeyaV}Nx`}JI1}n(gwZ^Q50{AX$a>P)|JFvOul)<PDjndfbLuw|yx7vDIhd#t=(ykb
z@%ldQ5a!W?ej9*wsYRD$(S0@7PcS!WwJ~I+cIlnIMd4?axOiACB5)g(>_F|<%`w#}
z@8wnB=uigWObve<vy0i&^!G#gY_&+Y`&V<cSQG%Tts|?gr>){%#W>402}xzi6|O;6
zKJ*}gpLFq3Ov$|Y$>)deubblk0vyJUr^2Tq0enh6Q^ZodVp(9)LA)15eY<z@VQ4<k
zJYq_C_sBkg>^b6kX<_<_#4=gK(QTYu_%500^m@0J|4a}r1EM^LL0nH^ohWoN@m@@P
zd`tHU#2Frn%`y#x_?nY5r52bA)>Bfz0m~dwljkZJJXLa0-Vv3G+Okz)9pa)DNJFB3
z>7+nmBn&{PJFr`-jdDuWE}SUOXrkGt@@6L|MGxGE2AuSrxTK}G!B-8%JeU04#oP1O
z`fgxr<L0MWY}&W6?ms&dbgh#&VlsXrm9g_Gd3ApZV(KpNC~Q4X^&=s<*EMBq8`jvq
zccWgE9RK6*6?8HPWHy;++70E;%|{ZUx!R14>Z*5vYSoFFEZt*?DM=+j38TAaXC@c@
zW9dS%LN?XauJ#i+eg8J>`w{PTS~qNE8iPOc%7e}$k~Y703G?agD=1DuEXyI!52@(p
z%F5>I&f`35cis`ve8r&i{FnGzPlWPYeQ4W+w6iO;=G{K@{6{C@DXP=!(oj{gBLsmX
z7t~47OUQyN{Kl&D3H<$W#0WmGm(9me%bDJ3)Mp=h47Y<=m$J--aC#Rv?O_Fxl`{VW
zGOhKN5p~kdel>Pp5D-J*7`(jT6GF=H&O=LcAFLd(7v4XQOd<6sP1P0kbeRTcz5c_8
z<bIro`mb)L_?W5-3a_t|;05bDHR**|1@CbG41|~1kS<>2#Szz8(&m{O#Y>SNYD<To
zB&B<5SoBab5?R89ID4<N_xZ=e(B4rsZ~Bk@XZ{HLFXpYX(W6XWk6)zO+2gQ*0=Tzg
zig}Ci9hU*)*VNtBA6<KR8;64Ga6Q{Nc9i?N8K!$?bvAR4pJjVHWKId%S%XSLLse{J
z4rxEu7CC%|@Et}V#^>0#pPG)t1<*C=2w$u+#)P66DCvA)sz~u#!u)B>II@^2EDyg`
z2k7Wd8HGS1L&v^DkUxav3Mw~765x5>dElSbuo*kNJU5*K?9*nd_%v4Uw(3gazO(|*
zi{wi1t7h*XJ$byZHJkkZp*O{HoaR9M6edc16S{|Y^HVTa4l|e}0Z~)EvqDFO-a#vl
zhM(qQh}VF!xv32SE#vbztkjXVLt20b5cmD$Wu7=OA6rbCQ1jhFl76W}#)10IC5f~d
zE3p^Gn@Sy~;tnHw11fvqJ;mnP&nEYKMrpy{2=P~{%)A+A?uXAE$QZm&1(rHXoq5lm
za5y@xPCz2`@1Z~$)q%1QWyEs1j*;+=kfh^w_m%^%4W)$m?gHv7VF>S~FM_NH|62uV
z!N8P+e{|c2$3-&z+`;$zOFLwf`HWOWR!r`6{%ixhDud^{ntcW{WxMx^SVpO@RN@2z
z1>fajSu4}7<Tv`;P><PrKw5cBn6FT!HaSoRUH`%eQ!X^=S5#Ya{{{c!Eg=M7%i#E+
zuENiPrmymw9Rl0zbndFW^S5KaEuPPME|8T$7@X$jU6R%V)*D)TFY&*0^c8jN+S;sd
zCC27CSS4Vb)y#lnPx;=tmYb`Y+W7HH*dD9N1<AMj(}a56!^?NP6JVrT=8)4`U))N1
zwZ%=mpA!k+Vds^jITKVSDd(Lc4HMiT*i~g`2uEwk5uWQFG_)sUc>R6ndjP`Gm3uC@
z99&hL`Af3z{4AOh;X)-b<f?+}d<vn3%vO2_i~V(UR940d*dn@ur|xM4pL^G4c|WfA
z#K@6}GnXeeOH7ueB+xupW986?+vQMtu(Jjof)py%qKZLFWD+TGLW^*+n1Q6V5yg#(
z4#60zhYzMnvr)S+#_DZWvs>J!EH_JE=1$44RCs3u{D<vSsBrE`dsBL2pvSx}UDahE
z7_c3Ztt*Pt$}q0QnBq^qIiDXo$J8IMVV=OqHcRX5{D+8iPS7%Q-DMJ_QXA3=6}0aX
zq|{mT0UyFZgc-l-y=x2x_xW0hWVsO&vfGwE6Swu?$~{?HTw*Lwh7is>=HgiZhcFHv
zxv{qenwph9Em?=ANjQP!gF2#QLjlxR->n<RsjNfVJfq+vQl}98sHaRCEYN)Fl%rt^
zXAWEmh-sSSw6E!>5W%$>N!3e0)*gqL4uK!>rV*g}bMMQK_aPTPkO#8k(3l?-F`!)I
z+Pt&;5p12hN!7C2=c@Ys{0pI#!e0I$j}OCcb_+}uW3$cae5n#UrG0Yq$Xc#{fq|t$
z%h3AB#QJMFizql5@8^s!i0FoM+I+?l_L)a2o-+QT=wt5znf2b7siziW0_BfyhKp5Z
z{l1M>1YV=N7_ZoaKmDTes7giDaY-J$hRpEsp9`I!aT2ZcMf*ek^5^pB576j6V{gmg
z*uz~`Lg-KDE&cv(!9iz&N&5@<2;x=b@l9u-D}bGy)xYf^*g)np^RghlD<-(0zn;oq
zE6UMKUk&6jR_b*KQ#F*rDhu%C#mkUu1k4_9SQJT}*G)uYEidEfUZB~;ygzPr=e5`F
z$o}9~?>lw;<!ZL4ZhrEVr_p-{U%D-ns^xGi9(Yo%Xu`OeA><Sa{t<lx5uFmcYW9!p
zK~Oq)ExZ<-;s==>^m3g0WME;UnBTzsY%{QqAaa1Q3Jb&rUi0v!Z{dCNlrASe)EZ_o
zlNcRn*uQnl*}Ye@RC1`<Mm7`*c7RO9v*o}e)BBxyKti5urLsz;X`HQ9KJS<&<A1`I
zza8Oqzh@(-0hpVA2qHd@LN^{A3dEc0xOZQlQ=FBmZ}_v~FJqc-u5m@8n~ipuPTTus
zkG|*;VWDTmgf5+KHL-Xdcf<~%XIzdHVixCTnQ}TOR<6fe7~l+raD8U)>N@pUb{HyD
zeo4!@6GJQgTEn<<TY0e@)pSp@s#&$@BLW`H?OR7>oOyZi5JcH(5>4)1b&D!SqQG&L
z<Et)0jBIsJANO`!GM@)8iNZbZ<{TMPGaJ`!RcGwk3v*H5_?T5aeQWEgs_sTbcP4Hh
zsIYNc9FMflEr9+K_&JEK>&YgCI=0s`iWYd6+?Nv4{5n(TlLo{BV(#aq#8-y@lSgoR
zhMAyA)G~Sn?|V8EV6>*osi+z-L~maSRKZ>pX+(77+mwAiPojj<o9R<fm{fp3r_(wO
z-2)2iQfzOKY*({L{7Y4uM4>F}C{J;krh!?7(C3}?s~jIgDhE#1Fs=|uIb}^Sh_?bh
zno<;wHA<Q@6dc5}L)<bD#<hE}I^H59WnB|djI+z-NFERUO0hdd5l2%?SKy;@+FJXe
zd<t~eE1v1ysi$bVE*;i2WzdUfj90!>JVWE#z!2v}-mqd(dE$?Crt=}KY+xubl|9D#
z{WI8;RQLnC?pK+J%7@jI1ZtKsIGzkmWJB8L#ph;6f)t9G|NA{J<~MH0VLB3B?ZvJj
z>CyX^_R^5IsU7@BhrWS98V~8w_l!iX5#J=s683rIW66kN>@ZRE=O^3Oz3}p;wu{Rq
z>=W8Q)TI0tvl5v`QF#XLXFDpLT*bY+p`Rh)@wm&oV>gnrE=fk2+k=+#k#YZhv^V-Y
zKPXPIJOIyWuL_sX6)&ZN9O7m-1#V9Fw0tBNFTAK|2c%%C1%6+ZRu<}~_bhC^Er{3<
ziQy<GRXP5ma<SL0^roQbx6TcE1hI|hO$Ups5(m}U<4c(fgwpyZGGre2rEX;iIJTFA
zNS<7`yGJA+=J27vFYf+>=pS<hyH)BhR3<c69q=c;W2y4@!yFU(uhW0e#5ok2IHKlI
zf~FwB*3AE$FZkD-*g0=~$PT@+xI%D{nEi4Z?wmgmcm6-dK)vPJqqqPVGW>Ib>kC=V
z3&-K~lcRHV6m)N^ut3v=)8__JFs-sYFz`shzaK+<^+8e!VtNI(Fz?%BJfpq7#0pUW
zE#%Tw-f19A@hGL$gnJ4w5PA1}Oq#sD(|7gx%V^75NnJ2X)}EF>xae*45Cmm-<cEwu
zPqdmt%4oz&Y64S6gLF^O8QJ104>=D=^H!Wf9%Bp_3g_mG4@6Cqr9Uxl?Pc-ER(l_`
zo|wD+hFd)ldKm>no_S`9KA!6b{5F{TNhztEt0{ncqCZBWmpy5d6wAxQGL-M05v#-3
z0-=}B1|5r-GoxV&Af^8RGs+Hg<K8>k!Pg!1MlkPBuuGY-S^Y{dnhCKPe7wG4g`wjK
zQ>1G>AhR-o_gJYnK)t@_z4mvP;wi$TzJ;0~6&?NFOUiYPIUbekI$%&)K0@}pIdtR9
zsEw6C)!=4^!S{pM=t$m%+7uw`RYE~!Jv^IcADhX(Uf93c*I(I(`Tk9wJsreow>J=)
z(|qRthF<q7_jd^5!;Bwcs21H;f0*O$SJw+d&2sSwYW?)z&u~BAP=Qm&D7|5F6QP^A
z`O1yNJ2a68JiD8f77b<+>Mn<4>P{_h#K8LFfl!C_>3_0YQ+okC=hBedak9PKpf<Wo
zx<LToX14K5-Zn38lrR{lbFeI!ejPlVz`$|fBxLqCj@^JhOEpwEsWt(ZCAGclDPqQ|
zQT*p7d0Vb^3HH@>b@BPBJEsu22!JFa;UlEni$VH+d?Gq)z@*WkmuK{ubh_B+^scJ6
zjgYky(%%@|8OXncDG#bOp`V-(1Cgj}<%i&P4ksOV?7b4qG9wk+iIV7Ic^zzy&31J+
z@3-B>I-7pq$7@b`D(xeb{i7Ul)HjgKW_q^QR!ATJ$_VC)+=^^dt<1d&<w3@G<`gpT
zfgv8Qtmc%h`Qu5?l$qkhln*XAZ?lH>F4l(@O}O(d#YPYF2mvvx1jZ55dtJsM4gIzn
zgjpNfCV}KzP$@Kx_rG1GLzb)q@vIL!Rzx8LAoK9GyrzUEgdx-(fU6XS)GC(04cz$;
z9pimX;vi24Q3-WcX-4+opm&A8tGk!G(9ruM80PnQcgXQxBw<;id7*||8b*2K;?S9Q
z70$;nuPf#hJh%$a`zc#Lf8fiMyy<O?_T~u6j=c5$CVu%3x{qwLh)@uhH47iy=~<hW
zsG^cY`r4qh4ah{~hal#9dM})JDw!0HZ@L|J8wOZmt9@+|Akzua)mBdi!j(}kePmvW
z4Gx_yV4KvO^Svt#9|8VjNxZ6SjW%6}2BW=Ra;gU2S)y;)b@;pCt4pDs75~b7$y7b@
zT4=(_FwvJJn^&}z$$I?=)gM2g$;5^Dm8_kcJtbI|EYi_9atk!Q%5u0TLy?c;nDb^b
zDlHkAkQ#Z+Jr<t!@Jt|u46)`hXvMv{I!TW3^(kA%&QIGtubUC*E|Z?Gl&o66r^ezM
zFi&LxL&9?>!574!580*WcPFN;r|WSKk72A^Etn7$Oe#TR?rkY<3@&T^A0QaNV~RTg
z{55;e(%}TW5gD`74@kS~Z*_S1IC%<zz-WvAw1;i62a!4YG}Zsh%`X004U6x`fri=-
zS;dGi&M;yEh$G$;9G1;$zp0Kk+`c*9Kj!f{B1@`A#U_pfUGh1bkjvzy4hz$g-on;q
zyybW6z%kom>J7iN(~BiOj_!x|_RMcn-td%a?Pe->b$%5pnZ~+6^|g0sCOS@hxK`aA
zN|vN+<@2mKKH+em+5$cGh;Br5X{0&r830_O26rDkLm!7lLP#ObS~#nsl3jlug%R)7
zh71$o35Fg<6tc6xCS8293YUg>eS3X(ANm~Ojn+-45cz%X$h>MXYYQ;q;~~n<mzczP
zx<7{tEX>#9K9-1fu47ZA*HLQXD~g^{;67F)$xT&201TLF&wRF#8|0Tc2*2EP?gv~A
zxEb{T132tpLI<IvkO@Si!7Wq|)eG%Gp7X)rI{M$3bWqZ0Qdf)57|zzq)g`Dc%Gzi*
zKH2nyBn-?(m0gjXWdah|o1)!4`VlW#$d(SGy6)0kANS(p??=1i=zkVL$ZyNwpMRR@
zpV?Wu(<l9wnu584?iSbN-nK8)ZKiM9-S{LB|8?kk9CKEE`Q5krG2p>3sp!Bh9t}1s
z^2i_62&B_xB9V**1{Rb>i2YH+u=>YzqgVE0KvtO-t*9NfbdR&sdg15PcHyv1+uq@G
zT(%-#!Y#fo%|5rBBIxf8bdp%1Ia%xzF;Pysb2{MONBxJC6UJG?FpoP)JR&$DoDlXO
zobu#nLUNLe3XKnLy0r!5$Dd7|HM|5>&l@s#@AZ{JKKSWmPy@T5y9?3YzcCP=e?r|(
z^?HwcIA7T5CvZD>x9464HDRYQJM_Q6?z2S<OG|3Z#R-57iu}q$EWrTCls^A<?0YlJ
zWmt;+WR<hac$M{M^Mu3EXx}UxZe#^85#<1n`TfdE|MnxTrT<+PYr*EH*M^K?*XVe^
z%#ZZDHQTuLMOAj3&<tm~w{f}`KH-g=TbHTle#|ZNxizmw2olk3eRF37Yor$Q4OSeQ
z9m~m9_L+q4e7?5~+y(E*zWq8od8b3RTxjIA!$+Q2+=Y(|1&4-fUX>C<t>sHv3&RX+
zTy1xZ+h16J8+iYPKs27yUOAMA2~#!UQTwre@Fi!Ssj3V@u^2_8_1Yh~*TO^>jZeQX
zhxly2H$=ExQ{<jVyNbsx%8=BgEmp{A5)8W$1=J$YCtd}sB=gzG290liwJq?Qo?B+Z
z?wLPhkI{eh(W9P;KYcI156q)>3xK})+6ebbxT=GCC=lz*aq|diJ)}ba3o=cy%ahx)
z5_s?#EcoM;YV;Pg09%T!(91ds0P>;uP4Sabs47AW^HmfR{O_scX?ZQB9fjhLv72NB
zHkS|W`zs1PFd8Uy>RpMMffAZ~u6?vRr;=DJUj83&&Is`vaY9YC&PH@Vzjo8A$q;<_
z2q(WSGM%sR!11_KR&xvD2TD&4xP!kdn}R^WUF5q=3X3F51f?G$h3oeu+ji(1b{s}Y
zT|PB7<y{tV;T!*4nFGlCs3L7oy3m+&Dt0Wqhwl6zd*=ZkRh9PP2~|bK3JPctkx*37
zRZ&og*eIC^J)j~i_6in65J(82cM%j_iiKuDY={UXR0|>&>>x!{EEEChFyHgPb5C+|
zGc%b9>@K_C4!?)f&pqdT-&5|ncP86PzP{@GyC1Gtwb&K+&z<w+n48*_+fcUi=&cp2
zH0inj>{GAq-Ko_6Z{A;U?)**HCe@ozVcHMPyG*b9M(Irr=8oL*Li$PNFWTMi;2&i=
ztbCtGEWIwMR`>XU>+4iH>D%+)Xn)(AnV*e4_E5vV&Ra3<etKf&$o^&07oGc2$Gx?J
zS3Wyp)#!SkAE>K8+&QmUuLrN*aZZ<au3ewD?aR%N)L(OA)ly|Y>vC=O$p@Duu72#)
zXRF`;YSy7r)ypJ2xxP#PYCS*C-uGzV1E0*T)@tFO&(FQSTj{xDhQ7ME`Aa_kf<enF
zCuTqM{;OXM*>!pKx7MAtaa4An%M&Us`>J<`6<aslQ@`%it;cP8>g26=U;1;+2QST@
z{obk;&)sv$cMngf?&~+R+|Zt%RBH14OP2<o4ld|6#Mfo#^?hr7)vxr(H($89>ip5i
zY=82uE4JR1*=}R$?q&Y|>7n&)pZ)%UD$_@{E|<3b!{jyBb^olx=czkRD*I>iLwa)5
z`rI1pR=rc=%GHanTHWub(Y~vimt9r7?}d*%S7yoiL)({o^OZSCJGS;4^m3J%L*^WJ
zP3PaAF1cdTi=}<HH2dM32kxso`<i{@c3$x7O`SG>_x8PC&%9vEXHTtf)PfhN)^{1a
zY<RW5Hoa47%Jn<Wp781FS5s#l*ZTf$gH~=?)2RC3je#9Y`&Rs@!>to`{eE`o0oPSJ
zb=i!I?mTen-$suAq{i5L*SypHfm<4PZ4)?o$jM{RFPHlG*4@)zZ1r5ZMmPVqa&X3$
zoo`n!al<Pw&S*Py>6i7lXPy4?gmd5ZmA?0(jN+9pzvQO#7u?!?=<iqkbkDJ;znJ>y
z$Zq;~=B~T-ulc9vPrG>RjN?0h+iyXuwKKMs_-lI4)>j?Z^W9r!KGpB_kJ{b-O6GLu
zvti<k7eBdv>+mPCTa10~`1FMJW0NZO{@X>544e4E0}EIEIA?s{d)|6|c$YmrTGx4Q
z<GLO*Dy&=h$nhoTtvxRN;wQQ+^VdJ?@%P7lwDs)4D?7fsy#4yc8?W1Pcil3@pBu5J
zYOxD9Ui)CfI}?_i+9IvN-_Bif`lKt{be=h|;n<6(_kJO(O!`lqKAbwD{fe>&a?1XE
z;a`3CCHL->-E8-&tGcDmZhUFR{y{yP&7bna^L1)0|9;|S6Cc~~de4#>_q<Z!<z<}?
ze81<NKfBafGxhsNu36Q)#qR6Yq^%yKzp%6Z>*~uBsxMw!;lXJg>-<vs*315!@Yb6h
z%k~;pd{wpN)aNGloRqfqy=TuqSb5AnZ!Jq6bLh#Ns(k%XpiQr1Zrtq)wybv1gI63n
z{crW38T!qLXZLRGzPU<fo!kB#+q=Tb4plz*Vp@}n+dPzU%)Sysmt3~xrn^6XbK0ai
zU4Q(p)uer^Qm<UzsN5UdhIN=b=cg&P8vZoxz+=;rM-5%R=7yQS+|s!Eb6J_krLC`7
z>Aw5*uX6e|hXx!t;iQUZ@^5eHYj)1vQumF~!<W9d>-Or`oOvky$sWu{BO5*Ta+6y7
zhcBE|eB-*xX-TvC_53NZdEHiH5~mLQv0DFE8m!&0FtDk2+WxZ3kJ<g!!QUp_`1_AP
z*UO&yWTOVZZajZdjpK6uTG?S|@{&u>u0C_eAHk-ltp4`QCmwm~lAAjJJaBf?&u_kY
zP}SR)cBnmW*K>Ey8C~(+Kif6b&*{u)(rZytuMHJyUHEwQ58jyl#KxV)<|o(oy?yfq
zY0J;qGj?5CX2-relRlr+IK5cs{Y!TZ-JG`iWk>8ypZ~ac^8x9nt)4OQgk868{I1qF
zjaCmyK7CpH4WBpou|m&QyT7gb!R;sfqvj3wFKV*?(3{&wjCpqT_Zc-yv{*m2)IX}0
z`Q_4~eV*Ex)TCSLhh?vrU-gO$?;mu`x*yjLy>8;B(ckRvyS~b%-Ca&9zyG;k?(*Mz
zV^YbBp8fXY2E9us{&{lZ?IkaowD;q>Bb)wlR@TWUbeY}lk*jCVZ<BQYf;;(FdRNls
z)bA!`eQ@3Ap;gMa*fh8HkZ!Ns+Tp?1nx4teb8Sc-UwZQK!NyqwQzx!|EwTBe+viqD
z`1qF=Q%5bX)BC-i^}7x@^zDL#8v<*3Uh)0xjbDA<ea~Afn|-$YmI{?K-wXCRdETFi
zgQm=XZQ^b7-|v3f*9ZER?Dc;4Ufo|j@!+qi!6zTscl*IN`dra_#*MF(dwJ>Ei4T5Q
z{L2}EZm(SSa^+%)RqLPqR;&7NpSr5|Zw<O7UYj;@=Gh(Q&$;up<8Q}u6N07pt^WAJ
zdtU5Z!^fw|b%ASVHtLjF9e*-pK!UF_|MN+9{y*aX-I#fvuYG1UNb<Goo8(LDSKUx1
zt$)%jfmThj{C@vhm;=M06I>4EpcweVk&=?)#7AmyxH2X#Qkf$iZ}rvm*>T(UOG+T^
zp|q@|jV<r0-_4&A*h1TaF8gr*fE-AHL2y4bh8l1R6o*{7glo5(u6$*oq@CaE4_E$h
z<E*}y!hb20u;SX{-v+_9>9t1F%XzfP^xt+fiC^WW?Zy2SR>DhgDMYjZH|<#5aWET(
z!|jj^=Rz5XSpIPOZ}n69z70NwbKLm)xa%S4kd?%m(Rd(jvHNZOZxa7>gtVv37%=^a
z;6COl{}<t(L2@Ythtt&pW!*j+d%vFWCbzGhaL<L0-LTvL>VLKBa@k{yfsM4q^xL?b
zi0cm9!=!yjTotGcr$GsD^yzVH6PDeWI*t1qaTl7r1v*lM2>NCEE%!qB%IfQ@`1@Mn
zQ2%8IeQQki2mI5+v{7vx3QNMI9U!hVjE65_77PZ(T@4jL@+b|jgXy>JO13`P(UJJ1
zv9i!VQ3<j=ef^p6!|<mkKA4tO{hidb>+8~%4443O;Vak*^I-_s?Kb{t#NQVtZ4Ys;
zff?hH$-yxB77=$M_*%4Rao8fDU#9>43-mEG-W9*dleU)rKhV0{#Vr{FGiZzSI0|~e
zgK$4IfahQ-tbsq_5c~oyVJ#S$7*~1fgKQuXR)a}1?hfKqw=-@IR{&Q3@5Fz|(@)v^
zE0Au*Z^8d4EWvhW`U5Fbtu{(u>ce`Fo!$#$p)WiRneZeugh^ofWV@PI)wg2M3>4Pd
z@iAx)+z(n`k97a5(5JOlKh^i&pb1<Af51MFJ-iHzfutXscS`QVTDY6`*nN<H1@w30
zHSV{8^eOx84L5<-M{l`Lh|}EY^v`v*CpZbJf%X(lz?nM_SO0CZLiXRTUv=$oCOZ8J
z+5Im3cY(%7X?Ptbz*xw&lQums9L5;<l=gW0v5PpZV;jPxZ6>ZMd>AH8GHC-<Kyvt}
z?t|!y8UM%Q*Sf#g?cYFLd;cRF$OOqnwqaNhXwfv%dav>SGI(X9a38n;I)iNdRnR`^
zcbEe2f!2sOU>Mv6SHtOWq;3CL`tuYF0quLH!U<qxBexuU1g761H!Ep#%RB3L_w0vF
zf4`-z8u!{Cn>gcsK%6s=<ElThXYG?~!bLCwOkU&4HZ*?}gTpd?t8c~dKkoKxF>XgV
z3+%qvBdmSEi;%zn*h||!wc7ierAj~J;dNL6KY{i@>c7@S(+A`JMcj4ZYu2opJ=Tr?
zD$-Qm<TvhG;xsmjYy+OYbs)UM?bmkPt32u2|1AgYe|v$(hb@B_xF_t$Uf_P(v=p=_
zlI}8GTH)RSZ^P&C8%U4pkM^_q`eE&lPatg}n0^~qYv}EeuUtIF&}KlcZm*^<KY98y
zn6T0PDfnM=(+}ci!RKyRGI$prXD{$es|POY%^3LAYOngZ8MJQI0L|f=&uW3zwTIvl
zC<9Jk!njH|{kC1*W0Y3@NFUl|`jG3Iylz@W-2D6ckX}C{USnPNfKPy}>p6rqPbh9B
zJOm;6^{uJ7PVKVWY5d*P)(EH{DR3=R1<eW0_{OacYe70Ub;<n$>Ofc<piQRF8rvtE
zHO|xjGf4Xa?Ec9w`_`J@1+?DV>Dmu;1I>|LA+LN^(l@C$UfYj0X|1>0D*qxl0epc#
zz-e=8YO0Y(0`X6Q=E#kpc~N^R)pY};LNCymdI^-SeD{J=pXrxe+GNv$@OU^z@#bUY
z(R^?BPyY8o<6>HvG^J|{_#xYqzMuVYU&g?9w9DIn&0`JKUSU2aj4S<X|L-mL32_g?
zmoN_|faLNhw15)eYudD_snYJd&M(%$X3*O0q$#8E<2Hotpni$(K>B?fG}n3czn{4K
z;8rkYZ1)X+YBP-kZ$A_sqP7b2F=5=(L2JLaoMh4y)_c+vzXmRae09<;^{+B4v)b4*
z6z?1<`r72(^J$G=<mQuYcL&Y&>Qf*1%JrK*YJVvGYuuYN0+XhD!ez~GX;dF=N=MSw
zyRZs2!$(jKoc78!W5JCZi0kdY!c$;cgfv~p+y7H(+e?twF5Rp2t9~N*(y~G_xG}S;
zu><vaH)wAsy_<5H_a|84xA3RK9?+P26HH#YzrcfSdtIRQ?s-@MlEDDb_<am=*KPc{
za&Z0iaQ_6a45Vk-fLSx`^cBQwP05u3ZFm;6R@rUaL0Dtj>9c1H)FNK}viJTft9jkX
z<|F(M!>^#U&mjZMb#~(K4Rf#o?T1IxCcB+~5!Sjt68;Xupfg+r=fDXr8aqG1PO#_v
zzJ#@Ru<Mjx_gz{`?f$<(JM6w2|4`z?^<Q!Kg2uMVYurJ^JrA<?udQ^g5s$+f(7n(I
z2+1!iX-mty>OIOlxS4i&+q;%H?FaX}Y3k>C(40OB8iMRZWWh6_F*FDyvxDHR^J(Jj
z{<oto-g^}3|7LLdZ``(hYbtytNOwk;rtfm!gL<$O?0EULK1epYSCE`?=St3p$IvFt
z*WR|O{nBN)`$g$e^SbPPGkgVeVFJkh>%+q^7-T0JU&{L-oC%H$Xh(Ti0j9kN@T-5t
z!0CS*o@==NM!3?Uboo7)GIBM}RZeT&R<P6L*M2w^9)mw2FFh-1d1{**`qL(F{}p~4
zq_c2o6N!H{LYnH+Jg9YI7K{VQQZlIurvK{O0INOE;y(^dpN(6SIE`bqdpf+}`W2Q<
zKLz7pILw8-{)PN1t2H*KKw4Jyzgl;_<WWYZ^yBTP_Q=w6xVi2d;`@LZA9h*op@xGU
zcaZQ7m;|N3^i4LcIcW~a9!$J(&mhk3(>lUEU^l3rGodfYre#Z7^CS}^gN^t#ANB{y
z$;2D?P}}|&KI-p!>#hEq+6?nI4@~hlOIZwSU^~Blq4BK#*nN>-{q7BtgEvj`>JE~h
zH*FJfwZQaGYu!jVA55HaParNdSLL-^eai&x{Xc+5;CE1eG?z#=8{F`A+@7Gh;SYEe
zwt?xp+*ecINzV!_8nq^{Xyk{1>=82qvj;sHc)oXrKkL38{(Cy4_*<m5=luLl<gpw!
zB9p(^H>h7TVG?-zsIbP3#!tAkF2vJEXKW|X2FvC{Yd&$3K>##=+wGNK_V6G`uim-f
zIpSUc<<mOyqU%>!<6}{7I5TNU^Vc4FG_Y{w=0MJfU_KU(JcK;f29}JvIk0fV(}DTJ
zrUqUgFeLCykH`JJ?&|6Pd%J=D)-C$^n>K5XOg@I+k;^`-zuHS>g1t5xS-eO5@wCI}
zIoCD&O*gF;Zs?vZzYJCrUl;QFl9&E1@v?!ZLH*U7YUHQ2LHEqTw9IPXwtS&OSNgst
zU%&I@u%Gy$FeNvAAu?JxaxZfE4ZcJ!y^xDP@M^!Tz{rPs`XBszAI8vdXAGsJyo*e3
zg)fndR|Z?@e<Rb^a4v0>OiF;}JJWu-d*D7e33LWzr^&DWc7VG<YpI>C@$hI`X41;m
z;~GCgUp~z3TVB779ZV)|Bpis4=X>HGfwjm%a@mPYeuR%0L$5K0hBAgO4h-&cCu69u
zzirDg^!YX95gJ3yQig}?w~3>j#wOb1H#*eVmra*+)4Su^{gYq!yVt>!Vbaih(o=1R
zTv9KPJ$w>vKNz8Jvx)BwJ0qn3O1%31QG|5lF=7&oKrUO5$!b`Lt-Qz>8p;^z<L`5K
zJJZ*2E^RmcuY~^_w=Xkr%UWr5@c-tfsqgEcE4%}bfmi>B+77HeIIw7ROSk>%<0yCu
zmclyM?TY&w7+bL2zlhI(w<4q~@7vG^3d%2JD;ok!M_(JR-*z1B_ciWaO>)rMu6x9@
z6=zWz|LdT>Yu_V#$acfp=V^UVnoErtsqbbEpk2FQ2dKXvz{?;x{2eZV&hRw60W0BW
z*bN#BPr_udWn%kRkk$=kBX+#;tDd{S?rH%)ZT7Xz>uYVo7bq?aw-Ki~VIbIZ#R9^k
zAk_EFq@3ojcY6$b+!~=>+lYS%Wc#aOK0FVa1A9StxCL&2H{om0x}kZ}^ldHv2SI5j
z&UOzGKL|=dfj*lE+8nN*xpB&8<*9;SYy22E2u9zs2c3^>VXptG<<!oa>(@tU*IwfL
z!;A>&>f;O05oFurArpFl?0hu52yI~^XpDrLTa@QS@QH9A+UxYK-2lh#n-D%L6j#HE
zH)Tvd*KLQZy^ra;amO;>J=1!;zdL=G9oxFF{m+snyAIc8<@*XAh7DlKNtYT!ipzl}
za6cHiWZ)kQ8bhl<a;peV-vc=Xr+-hMZXo<K$iAcX{c_SZ=V=U>e!FhcinaqTtj4t-
z<GSC2Y~Q<{seOGxbAsuc?P}~l0@39657NhhWTN`Fz_+jfw12o8PK8+eej$DN%I(iA
z+*07E-rX}A``0~<_B&bd3E2HV&~{LrVS(&n>0DQBQy+Um2FPAD=gA&Bz#6doZ2a4a
zSDAMsq$}U+Ae~2&-|wVd3vS<|+dX}#2-#L&e#Ni(&-B^2m5I}upgzxmv93QPgRGh_
zq`udmxn`usuG%TvR)5yQGEkfAgVs8Y<9_fA$WA|lUtl;)1|tuz`#x#iKz;P4Df}07
zhNyj)VrjcL38v)hN2q_gZ!vk~o(yJ>um0-(M|m{o5l(uq`I|i+XUwa=;o7Bfpgv0u
zlFJ|P6}$)P?*M27XF(UxoVE~t08^*N(}NJnc2uuyt~faTH*;4wmv*aPO<`}S5BY3a
z``ogiJjX)bx|V12+UqZ5zFXCD`W+9^_l;aPT;H|+O82itNLQJ;a5^~ccHLueE5Q}e
z5v~T!iJ!nv;I;n=#Fc?U`)=u9YuOJFvf(^;9dT7b{cix?eVy`unAZ2a`&#{@rPjGG
zxo)_=YknFF+S|w<F3%Ul*8;WsVz>j;*D7#4*fJ?e_#`+Ft^w_9w2tT+XMmG8IXfeq
z%Jap!eJh4L5$yhJUAr4}pRE3CJ^TjjJou9~wH<y%y}+U;T5(;?YqD3_ruH>v&Yy)}
zw!Ig^^<8>YKh*AO@G&T#bo(QG0MnokR0gMiVccT5aoOSLyL_D|;Py>!b1-8-`<Vey
z0z!FsCV*di9jpII``Qk^cxYh3(8*j^>(r|-1~mRO&edMI*TWpp-gFPxGT1@*7EqsL
z`)`AtZu@7uc_Q^cH{RRNNMUPTD@WgSwxj;(obLp%<s<*~AiD_3pnuItsq+Soq)nQ0
z>~<=we!T_dAOTK+>Y#nB_A9bijrVS#^kZQ>*!9?c$u|kqN6iN}LOnPOj%&GKSZ?15
z`}~Uv?sFo`Ah(}G=-WcjTIlp$&%xU9p1T{?{Qn`y?q@fDyH_8^{7$ZCx6}AH6W?5I
zi9%`8WV4#9dx2MurmhX7H2~FhGQ12*)0(gba$o{H1~tIvUl^PHOD3kTxYeM#3G=>J
zuE<OrZeQm5Pg=Zu@5A){7p`U6XuF#0@6}#YZ4u^U!XCFOVcClMDZ4oYrasLh?LhTN
zUZcUJdEGBbbNe5={@V^X&z3>1-?Uk->3_TQDpgxeY}0|he;%P7YWrZ&9G$Q4c74Y0
z=8^tSfXbk~k>-Pe@H!yJ5eY384pLnY!#?o#TVd(K+3)$eu76?d{df9@ojdROx~^%T
z?MenMpYP!FFCBLV*J=yBVG3w|);z8GdMZS{*Rty*uOp)fZVBQwHz<#;_a}twyT;((
zA-6938b`{X)~|-o|G_`FMxI@BO^sjm_guIV%0q5DY=5M7Sb1bKnlF#B;#8OB&S|g-
zBx73!T4SXLwZpk?r0X2Xc;{M+^3S>E`mQV8dHa!1E^1F+nc>p&gP=2vcVP_3_HTe%
za5iY3XaskH)*a;=4#&a2tnZQJXxe~V3bH`^`X}HXxDJxwWYBoG`tJDK5VsX<-O9fg
zzJsg4Tc$K|G0iiP+I*zqP1{;6dP;5V0&2J27p)1u!kcgw*nQEQw+`(5#xMKs1F|Q(
zjK@FnNOdLddOYoFNm%C)rv0|7v3on1IZb^y56pho&TIV7kbYFycR0bM=8UxUa5-&V
z3#RS1D?O%zZTn>jzY-z8%Fc$<!M;YW-@kBZxF-Ee<5CxnGVM8sc76nQpKbpH(#m-H
zqBXxMJOtxm0qFjIJLvva^TkPU4QOAZxud{1@P|3m`&S9U>c7^%M?m(P4XeSd*O~{l
zXK-Ytwi&<2J(h4qs0-CV^V>Ms16une6WPlE(0Z?H`u?r`&u{<q%NY+Eua!XkNCS;=
zr_X-ReQ9cTzP*H3E<Gr>A8b1^{`JIbyiq2;z1}~=HbU(PKThqmXxFy$;lCL)4orV-
zS9aANTzUQb`yZ`ev8Ltq$Jyij6ru0h|33=3Ye=m1AMN?v{%ek#0n)M8_74%KePQmt
z@@Vf<Jnf@jvgeXe8MNQK4MxE{_!+XHJUDhAPu+IjU~s!Hz0ZzdF!+csp<!af;GtnY
zU$!qhcyN%<*CQi6xS!l*Gdhxu58(rHlWTk2hQXbRzdSLx^BCOHTZ21~S479)rb<5F
z@xF$^PdPs*n^QYDt2xP(U)KqDUE-P?pRf3~%H^lvCY<Cef9pEjyL@FDO~&2iE7K3x
z_pz^RLRZ{gzLE(P=TkD_8^<l4Fw1d^B~%W%naPfukd+Z~2Rn{0;na}p>+ZTi*FDrX
z<T=zY<OY{IZo;7tGUQ1(xZQD!?Yk^T;S#$paKguKJX1km#Z9L;ZryFCId1K3>i#-k
zN4W{5eUs!C^Brs`F_iEH(>GNmJ)wBR1K0WzbBg(bJ16=s>q^3odcNA*(7^uQzK&Zz
zD2|-h?bzA17<$Q`G~HJc_n@y`@#VPtea(wKjJwO%NzuWLzMQg?aQ7r6`+kvIY@RO>
z_m>jC<jCE-_h3d{Tt2?qj&lgL=yM1)qZQ{jBKVPryx)mX;J@T!&T4Q^Z#S^oMZ9NQ
zk9T0cgJm!Yx<LTWhf~4%jsULnexz~Dcdq_R#Pa{58<y@%;n(x3S<O3MT>+hb0Omb7
zy%(nU%{Rhwm;riEwjEpo<)I|xvxjiu{})3259wsC?z5cVVR@La-p$>THlW(ym;)Y1
ze@6e2T)iu-cLcwNiEs+kg(jePXHN&`9b()9wxRE9>H8{2>J#o7s{cq{<4CpjpIuL`
zZu^Cv%XqKHcS>^4=*im-wn6AQRaR0TzI&^2y#*ra-W#uXYW3Z^TVXfoo$4=OHar3M
z!u5~@CqXgDXA_63C$~P;dm==fkN(@&DNFZy|5WcGpzF~4+OmUo_|y94KBK#e^bHWw
zeaQW&Ro6?;LWcUDj91Um!g_bM8vGccob2Lf&>EueUG#xApldm{;I-3u!fqYM;Hn*O
zfWF^z2Q&hGpQ9`kush;%?J%C}9gTTIx~}ePKgfA!zYAgay|g5!xvhufen(H^KMB0=
zW62J7u?Ae{Zx*-@*+kQ~H%;q6B2105+ywF$hr<%7<`2~YJ-}|;F2dToXiuZIr9&EA
z3YFkEGbhOXPyOimGrBXh!msz}=R+CA-<Xj*=9Ne9FU$se+~@hTk~D_2-`9AxWf{${
z@7lBm?F$dYQEmfy&x6DI4JJ$fwW%xIcY5r$t9{#GH7tND;jrhRf0ymF)tsKE`z!FT
zb#<$C{YooM`LrgChdg~3^ncs-@e8Z){UPo3ZMhZZf06uEK=+S@<+aoAA^&0T)l>(R
z0)3y)uD>w<c=9K}ze|NhzV1zV-@BG>_5R&*k8TH9bA#SpdmQ$A@_23GKwAH5ZP^F5
zM<(%%WzDnILEoh;Oy+j_JLEqW%p6|8ZQeX|uoX>E)*PUHsP0?sYee&F9;mGAAF1Dg
zi`LE~nT8(C*gp;b8)5YNDRG+bm9GP6UGVC=U^uJ#)IjUT=OB}B<B+3dd>80mK>NLD
zGPlz<llM|&uQ6B`E`U>^1Qf6Z#kFkNvcP;{{nm}<ptWFMBbs0LvYl10_)qz`I+b3J
zfpHP^s`oVgp#9$Z2zd(XKlGi(8k_};j6;r+@f^^eUi<uLGWVvXlV9@J+@X8V&)_u}
z2pypwB*O8W9XN9}{(QC}y}5D4a3?`94qhj3Y50#&KYBFdUhf<~2GQp9XGuR7Rzo;l
zhmY%@bbm_I)T@xm&N$>K-Dz)~9!DAJ=|eaHnuGR|w(c~SZ-=$87$$?xsI)#^4rRcZ
zyXCsHz}*XWJ<<I7J&lXpvPZQISnnar25y8u3(>3I*VlXd;lFPh*(T=lTXdR-yTRM=
zHEe~#_5{7i6Ym+)UzE8P%E6ZqO;;vO^S12b@8GNl)*4WrIL#p@Z!B&PQ_s<C06Nb>
zmqOpO@Sfj<8_Rk}{}#9gc0ss2-Z*`qxlcXc6F&BVI#<zi0nO_dLrajXo`<Ed4t9Xq
zb7}6c2(p=2<Zkjj;r@=-6LFTCO@3$pc*h-gnDH-naGdpMZqfOnQ^(P21L#G%*Zb!y
zz^k`N;hE$u51#KEM=U>qycPJ4XC0pY)j}pYFcUIC_xP8>*`V=S88i>y1pPsFkp!JV
zXTwIng}ItvG;XhhU2&Atno}1Xy;*L3{JPIC?7AjhHk@qAALaUQfqtT``CdENME+XP
z3%qIG@Vn%x2pO=T)qR(1EN5GC)HweWbl)@|o`$Z_3~Gby;dm$uihBT_ggGD`OJDoJ
z$lbVFSK7jf@KKm_lP{dxkNjT!A4lAKan>_L_mM~QeIPo~_?LdZ0<Z3(h4q~8Z?HK^
zxz9<f0UhBlhXK`+IS+m~202Qn(yi=5bAa@C83aJ<#<@_%^{3)?g9)$*)`G^o&X9^h
zUkIn?NO2#NUwajw)(~^1OIT-;OF(UkW$*F{`A&nQ)&|gtuWhDd10(TA)3-NmF!^=<
zr|%+r(-i&*8iIbCMDs-`Pgc?h<RSh24w2;Vjo(9F?dLjx_TiGf&IL7JjE4sy1!}`N
zP!+BKl_?Ign=Rn2H&S>vc{SHKbAUMua?cdX5MK$jCzU<)gvlVgSPwhF)T{NXDLC~U
zjcWjU(fAiv!wv|iOK;pf@@dXcIsG<|H(lpKx|gU8UwhKOZqvO^RkZb%B_|`VXzn=j
zYrOvup`7Lm+0^Io8ffouGsqsa=X?jE)$dJv8oeh_#!;KxNVxg5M(J7&;5HZry5<9L
z)aw7nfzIy?N{{-!`4W$=yn0pm3pgK)-BicF5lmj~BXovP85UXbCSB*j?N|fuKrUXH
zM+<AtmHsEjS#ApXrAM75YOfLP8g`oIm$RV2*`OK6N-G3CKRChUG3Aanm+zGtUG@m0
zFQYfPKf|@)Yn^4D0Vb6oZ2yMCQ-sfi>0p=5^JgW!#Ch=f$YlfAGLPmzNM4O+o#|*^
zjh5F=({sn_ps~3wj&cXcdlQ7!vDSnpO_~%?rCBS&l`D|>D2cGnfG$F3`fZIoUFG?A
z65k1ot=MiG;&m?Y7HD12-rx{~t4DF0+NRg6!+G#?ap*%jeH9WR)-xSr2fCk-9^;)m
zrctH>G=dtSJz+@*YZKb%AI*FHaOM`$eeM~M-ramNjNWv9lMcnf(Yxxh?4|<olR#_1
zU9blt={^+SKdB36K`oGFJY#ti`7Z$NA&qWhakWp>{hM^y6DC0ptO4D7YHkTvkLIVw
z@CEz^pTQfTwf%Ou8gw6WB6=@|Tfi>D$^2-EOY3Lma7U*#(OEsX4ZLIbAH+HDe&FV7
zzI%SCXTkd2ipF67K3_B&c(3^_S4(!<uZ5F+q&SUb$=%*(N6KsBw~@a-*!s~v{w&a0
z(~Zy<ro(dh33Lyj@?Ah@5~eQO{fqb?;eB`!`apZA1G?|XC#$0=&}Kv>d;ZpXy#QVX
z*}*z6y3_rg&iZo4z3GSbdwoii{=@C}Og&zA3xA{O64ro0$jO#zVg3c=uLMiuEH{w+
zw*JHUrL&Wu8e9YSKubsk-Gdg^Hl=Hi`2%Qw<6OhY>u7R~K3%;%hb#ST0__2GUaPR)
z>C>F<=)Mil3e2ZX?*7k7)7)@8gj+kjdd~~@PkMxBg7uJ7ykl8owmZb450hsl`A>&@
zI!R7WcJ}wt_HAxCl{a<8;y$Wt9gXh+8hzqw@7^D@cQbl<2!ASk1NDL5;R)>zjPB(s
z-sL1*y)U5O=IaWko><&>THM;;6y!7j7K6sk4$v5iCgXmj*9NW0(el}88o&DF)4$y(
z+pm3`$?LlD&Qt2oi7Ko2OOBQg`f>E;zQ?7#n)exA4sn&?5_^8P+i&W?Ee#_;y6z1Z
z!xlGOnEtiL=o@I+glB>Qt;KrAqy3rY1Fgr>@yqZDYyx}!)t*6X_e`*LT$ukE@+W}Z
zov420b!mXR9SXaKNtYe#z9A-ii`s}Il!&~|n3dZM!tK>`c6XKX%0`6!QM`-Z6?qu;
zf^_{MoCP`?EKK)GpAKj8_sK5D2F^uJIj{i6z}=v;+ghMBt)<t&U7)@Hbhr?-*U~w^
zk$GXR&X5uzW}R5;!U>dH2!&n4P9GW1wQM;bX}@)^CjDugehZQGr2BVA&vrY^-|BMH
z+T!Y+U{i<Yk?SFrHQ;^no#W<R+2-*J^nS3$x-C16cb(lV2kBe;bDhU%9Lg?Eh2ueX
zBD>H%*(~@JeuceY*C)T$m5vaz{?&hLEzmP2Z=KP?I;W@*@AYh%9|?c^{O{{)j25;}
z|CBuCAlC-k%O849q&V#h)`8KzamNvN0sIms&E#1_n)V8M=DZaSaUPt;SwIKm6zM+P
z$XNQ*J(ui4XX|Hzqf6H<i7UIP2~FSu7z^*gH=y}@29$!BMXS6nnsYXRX-{G9e)6_;
z%f@`YBb^Z$J9@-D5vGFCMI={qy7quhxt4?TofDi#{KpaMSWUdrHMfVWTYG~5Tn>5m
zlU02>=fPV4*Tx}R*@A4w(Tz1mjjrtYW5_34`5S0WJ{I!jGvP?GchhAzuR>vZx6@xB
ze{qOZ^*=oxSxb-Fs|^D0*-kh;DQ*JjT*ugs+(_?*n>r4X=N8a;u$}{%c)2<g)U(2B
z;I*Hh(=uvaz*)e=IAuGN{9(tkp4&&V3F!i~57hcm!VTxM2d(?24k^gWBb(E+0b9q>
z{OieERpnLZVSn^v8a6Rhh`gn{WY__=PNMnOl2`Bf8vP%Ge{2lpXOgEftg`aT=45+%
zHlSw&1+Uvb=|0wg=E!MZ9J2kDyt)r`^b><9oGfmBjma!{4f=uh7ImQ_9BaMvVe4Lg
zrE6V}rgJ+@^VIF&*ki1143`hN@i!84_2K9xTt07{Q!X?wuL~>taQX<%6L-7U)I87=
zykokc9z%bJP<sIFAx*lT59$n2&jwAL*PYwEL!)z%(-(2bR&v%kP*L=+GIv|o(%wVQ
z0=|cN@H9LGDWG-#3^;}{ta;;Vcnowusl4C97Kk)2M4YF*x{4I$tQ6hnuER;lQERzR
zd-B3Q$_LWnB=E{TT6hNM`zm)6#4~q1K>qsh7c7Lcp*uvgpRM%qa@K$$$k84@h4~ke
z|3vUzbImn{iNZ-&UKi~-HbS&(Xg>M_zJ|GQ6=>bK6skaZs04LD^Ox*#GAx51VK@8?
zx@U>0ecnu-F@GU)Tn+(<X$~;?aGQg<Uyr7DrTqbZ&@~1?VfpR!cgTMxj0c@3r-9BD
z>^uei{gW~{3u=Izc0e?p*lEg}EZK@!KW=`l{WHL>uQ2}v@+ZPdknOw=Q=t!NuYU=g
z1+ojxXX?+bP#xlGqxy1V==bj(JtIHeyXClYoP_J_&kE~v8_gXdJ(kD)%q?F~{(1V*
zUhN*xJfrtL8bCbbU*}47;8?f{HbAuNnY87t?yq|ea>{|i^kAnCjaRlNKkm&CY2LQ4
zVf^dJe;$kj6K7nrhO8!SK0F0opefXV;!v2n@}(m;qdUEyHO-Radi=WAa?<k^3KIzB
zbI$RHT6Gw?dtK$J1RBQ+K+pFVgEwEKu<q+M|Hu1Ga5rZGO_&3^A}1r$SX@1qFDKb*
z&mYT2d0o!N)tVp6bxfW+$bUU(ZZ~n!-2LR!HMEb2c@98+M)xK04*~6?ZJCGmXXGpF
zW7U<6KiXJ+iS&x_4Cq>a!WHnC8@8_>&aZpGbkP3fWOxt4<qIE|mGn%Yc{9l=o^zHx
z<WG%9wvv~dU;DZ75KVvHw0Fp_IsW4~%k?0CO!`M=uI`<=TldYo!^lzNQ*&%#>i4CL
z%>BI<=_S%wUP_wo`Fg<tFgogwUuVYJ6PUP2?sMeP+CK(lYj;7Uyy4<kw{d<iBA|Iw
z?+ELD;CCqOzHkD0ib0{WFd3BBr5SFlV_CMa1+>oh1+Q&H3oj%8Nf6Zrke3<va&-<D
zN$#6T(^;yM-}Ebz%iL<}4L6p*r=GvTEwB@8-F!m$Y?$qdxAVxawL<yQ-F%~Qb>0)}
z`mZ{kYB%KEQq3B*)>&>fs0y_~YpB)*-E-x@8qj&}L6FQhfbvG_et~%FeCSllEr)2j
z^QQGC|K+ePj&eG?z7As5zjQ0Tc;#$|`;Z=+{`2fgdo%6POg+EjZwlAIFIK!s*M3mx
zcZNwbdA=k~_x+tf_kD{07WACwc{gnGMshWuRpdUfE@uJdk<**-AxwsD5P&+MJ)(3i
z`KQ6-pz&WH9NFG*!wtS`uPyXXt~z<$GVzY(4=8sQ%mc4&L<^5;+_-Tp`mc;Emc<~q
z_sMe-_}cdk{WeztL$@N74@$#pAU$YZxdYAz`^?U(qo+x)4&lbKY^D+DUSSiMvRXrK
zgjTTMiZ|)D`#I^VzaxCbbK&Y?&H}WL(!Ka9(AqE@?gZ_{G)L%+U+cN8*8+Y^9Cq4e
zl-mr^bZ4h&EH{Gt<0zx@-#{gZxL2kBr^72?%O{dwbG628Y;!=~Gd#&zI>`df+4CUM
zH59)Zv^UWDZs*fp@@CN3|H_l5eA-XSmcyl)dVV538S?C>f6_g?8<vcmc7iSUaQ?mI
zkzHsGuLU)s6I=<2aJ+kNXY1SRKZbm>Azb~D;zp2P*ZVn+atFwJBRJQ$ZPECVgVxPM
zVB}MnJIcJPW9o?Hw(XyL7LZ8&T01Jjq!_NHeZBOoGo*cPe%VhCko_!h<GgLv+@QH-
zWQ25MKNphzwUuXXi_Q%-PBhO(l7G1Pp5(s>eh1mkT38H|pgS}Ntsz=pN`q|&tqFew
zKSC^a_a*ss|2Q4OwIx#AH1d~#uxdgEI&X_d{~BAmM-N+{H_|yz%&L(<e%(9t1?kqS
zw{T(2NtNJJH~&+(<>3W49IlMwx`5UPvoA3DWIsJY_LB`JUhY=*t94lew9kqp$8ho5
zFI0fHB9zk_vKhXDIWP_$2JIbmK2r{~w@|nPWWg)&32cHrV6J-ze@l?vs9qCam@E5A
zg0OP{eb>E~=0sZ_(fkL<+Zw#8E)Z58i%aoufoOHxX}^+J^L!@E0ok0^kHYL{K6zv}
z+N;?0MYo@<q)g6&E<;XR<B;uc@@hYDM;v9OuTMey2id|D=mVOk>w(Vu5<zpl_V4$?
z7?=mLPubfSP?+BB^t;I)N&lsglh#dJR?+<5llLMhpl>_EJV$WX0O>>P@|O^;Zf~0A
z`$VV$+Bfb3Zys-W8+k4Vl{IN^;6D>|wh(EqThaQ#OC|Sr<CN`K@>d6qGh61-{QJnO
zu`HdNd>Y?7U_C5>N$@D>K1BP?L@397r1qRQKtGV}EQNKTvZ}+>XS;6@@2o$zj^sxk
z;%3meHu8$({z0Aq6nIzDuGi>Z`9Syi^Pncgavp47dp32Ipj>0neWRTg&98N)4rsnv
z4pl++vl}9n+r#e%H)amdSg~bSnEyNS>)D)iQCMC({YCOu2JO-9c(32AA*;xzbw>73
z3jPM?f$j^>g)2bzlnF0`){tL8<$r?<!B&&wM;_9V)<li#NU~A9WOr{n>lsmj%82m%
zUa#t1<kovXZUfLBEZTKU+E(&8>kMwM7|WQl(s?Bq0rj966mCCRN#p#1lz7Lo_So$~
zX9Mzk^%p6;k-RlvbR1<=*IcLo^Fj9l+C$C&%?H{;TnVy^M9>=27`lMwkOFG}azX#j
z-%`*$w~?RM)tos5N`~@Bp8HKYZdsTPrmk%K6=AUzXVSdxKJq*W=RgWv2Ajc~FH%^x
ziq30jZL1G!Vl4Yz+dk*({ov=3RV2NIi=R&ZbKu)J%iTl%YhhO$<+hOb640I@Tw8V$
zuQg;bOn|PSJxWz50nWQ-X6zytjensr5@~*v9;HtwuPN(wO**dZWGKjPjE=VBmo6T)
z;!K*?eS$pYLHm-AL3Z;tc=JUH%YJSKwd0*Q%IWta()>;1emCqV<XRJ+fN(mA6sNJQ
z`CDT+Ql4<}y~rQWUUe_|wMUR0<eLN1`a92QrPtG7MHpFW{?<J1=)PUn`Chd<VUwPg
zRZICa*Y5$Zeg+b!y<@bwcoyl}tI2*;UN(6>+y=HidF!^rW5|0790$*Wofh4nReegJ
zP2-BlDc(JHQO2_FHR?hJgwu1RxDUuLyLm6pa!<RzQ<zVO$j8wwuJ*^ae7+@oac+8q
z_rF5vxo17i2}hc%KP64J^aj}1G5(K;*Iw;Du<hq5!qs3sn7onPndH&jp*>(5(0*0(
zL8ScN_znC<Nfm$Nls5j96s@Zon;PGR&C&0YM|Sf*MABQh_;m8i24oB2(!<56Em}A3
zi=&L{340!lY>fW3SK8*v>Nnh`d1>L#``hLJUcc7q)oyvOTov96TF*O!cPy_Zt}aMl
zUi(p<IuD)<;p#PUACRtl#d}~sybQ9pIPN=>UTytUlQRO##+)B`XYjfHzW1EvZ`nM_
ze?!w`f8(aN`<pf!fnB_fEOj2Y8MMEaY<2&21w7`4!|5$t+=t{Z%2*ylIjuRe&2aUG
zi&I<MLB2KMri|R@^<|Ki&QGMLJIy&S{(SR=3AD@1{XHkykM?TbGZ&*5+tqx09;^cs
zzXiYMjY}ciUhN?99iRn78p}VBrt+#s^{H;12iJlTZrEHelKXyIR?Tw*3rBV3?{EGA
z>jDc$ED0<c`Fvo>s9u3(quT}+kE$DZsc%((*T0{`KCupV!MA^!b@ew(d5Ulfvb_L)
zgmCf?7xx!=^=v2Jv8=tO_MFooT)mOvrjWll7&SU>o4j|uHFmO~1UU0TzKmADq~o3p
zuRtWZ+VRq_#_|L=zvg-E7Zj)QYv;B7GfArqtHR`wzEeT-$68N%7s8F;FHgK(xBRQq
zGS0muu;__a=zSa5e~Z}o4-&7xSG+o~aO48)Vq9R!=z9Y1KXEnws)0$5AIlqI#h529
zgIky@GMOu;V-<z1A;Zu|b=U}z^yZD%y{7It<Gt^cZB~PPa{&E!bd7rg9B0Cox(j$i
zd6J6Z4hFA0BZW1VZ-Hj88-{`Qd(v;DGK$xE@P#nklTUM%>_>CTa!;DdEr7Ei+T64`
zEu*SGuz1WB=yrVs-5VPS=j!h@zsFpmzuS8qbHsqa($O8^D%Ozm{8{&%?Qh+}tRc5y
z7sK5(<SX>B4OW2m4ljUL&(XqD$zRm_&V2fpa*<I!nmnODY8(0E<}Z%>B1F@jH|;g@
zODB(k?(MW+_vQ%~?n=I7*lFdL{ZxS0tvI{vpTuil(=$T)p0vK_-HYDzx9C?uIQ<qD
zr?sLfTn1~bI{w1H0lRpg`Qb&@klxN3vSieytRYqW_jRm{E!1JIXv+F=3jTW7h0bcf
zgY6LMJlEKS#<J!Ook5$pSloBWe=I~+ZVV~b_q=sZ7inF#bz%I!lfMq={#<&|Ud^OM
zbLW#!_OisyuXBkrpfLN<dZzoBKftd45PSV0&EM=%F_1IzElY2O>Dx}<L%F-*JXjt>
z9a=;5chA3P4VjBwJi$ECBK3z!#c|68X7oMV-?{xc>?tnCE^fvq`ePHXu!gKaUq6HQ
zKIC@dn!p~2Mfb|H87_qwv>Z80n)_QV$W%J`!XsN-&R+l5<kftwd%B<FD7Tfo+T(SH
z9MInWAt=oDJ|U0hn>WE~OVYH~;{#`)ho@rDo2_%>*Waoi4A{?zH;Y)ue#+ei)nPGw
zO?%#84H=9*bb<zfw+1G0A5xh$q=tJR(iyuLf?do;hl4=p!7CtE{VSi&4<jo##uVgf
z)(p+VmTV*Xe<PpPew8Z=iy@YBCQmQ&R|4%9TfsIkX^~v*F`IyW-?=btzy*oy^&ds%
z(t*)wJnl5goe57DvApbO091gtBh;Za<VSZ6nNna4>3(~K6kquS|8<T3=5NxJ@3p3k
zW3I?Xw`(BU8lrQG&|kNYQP<(JAycFOWc)iIk`5HF=eKvY%lnRy%4!b~t&BHKYd|T`
zJg_c~vg62G6@G-T+3R1xUOx@lYA%jP=O*tW%2kFQP*nSwLfJDQUi&d?h~^4EoWgu@
z7ki5q*h%fc_{S3c9n;Rm9_lh*v}3N|yKX7du!|4Tx$ZHvhUhs#>u@@aC$3XwWuyNS
zk!!TEe28?N|CRt#M(#zR{eV}`;ljFK&|Y8VZUWUAE?>B~Z^)Yn{n+oZM;~(~a^C<(
zpYgcgP_7Q#2s?^cM-Jr@p(hm8ex9eS>}M>f{RcpM(a*7o*K^m9Wvn4{1}6muK9J~d
z(=v&DMFZ%JP23QVt|R4jey=5@lW1dkDzcZ2IJ#F|uKyTZ&$=8|rw7T`AGH6|ep7pg
zA0V8r!^LTjnRe6t#YzSi45^5WvLPOwo4mhL?s~WiHWsmtuPIj>IzUnFXC7rMK`ir+
z){u?N4Ii@KoQ7RwIQJnRj5#%uPUDG3u11&I(`$YA%6}no+J8CuZX9@lQ@!&yy+d(#
zkXG1OevUllU@WWwt^X6id#y-e>QB1orh9!p#>NTAL-%SLzgmB1!RPQh>;t1~ue*!1
zTOkQPjgrq>&JG*<X+-&-i&)2(lsg}8fuh=v`g}z6FJ+<!Q5w9@cC^>m9Pj4I`(4kx
zIA>j6M>vx1=94D-=>ZzYvXu^y3We?c-8z$AX#H$+_G%+D>GxFV1#Lifa4u8^jmc)v
z1;)X9ps}hsRl3iDv*2B@bsDQ*<FGSGcdLq6hwSG%xE6jXVjZ7RPWKz}4D+aYt>5dq
z&DD?Ot~0yF;M3oFZp;0m_oFn>8KTB_IQ@S~`~`3`*!#;@2`56h^I&g1^V<%pa~6C3
z$LO=xNTV<9qjaA811y9I@Cf+fLXfUC$I71!$AZpzmcj;*K22SPxw=+gP@VJQC}-De
z<{#Oq?9xt))i0Zo?f9B39O_j^k;B$|VO7wH#<KSM+;8PR7Y<uDdE9xyxtH^f<xQkt
z3yt7su=T1m*~JJ?yq!n>FWK*%A6PUxp1D%{vCuv$XT)pxo24Qz+>4<WNQXM_xe7YL
zV0az0hWrM5!K;tS#Fd8$VbZ*L!-Wr0t{)^qb`k5?Njc3w>Wfc(I+7n`ZN{?hwKCAX
z#<ip0NY}A`=kW~kegQ^*I^*pK=R>41uJhol;V&@dZ1)%L^BS?&zlQ!6b*?mP?M>7n
z-D=JL2(*S|Kzq0XWDn=S1<)AogHbRWlvg&R`6rxiV~Lwg*)u_IEcv~8WIvjJii1xa
z=?^+EV_AFjGA3Nubxc0o;UN7TfZm`re;(L6wfzf7(^=5cFnM+|?r&k-OCMi>S01s1
zf1+GNxEkaSS4L+z+Jmiy#V`rFK?-QDC<~<+Yb9{6haMmuE`?avkax(ddzRiHJ%(#X
zVR35L@o+?S-zKA$&(JO_SO3Ni3RAt4j_yo;jb*LzPlNXQlfl-v*KeNf=seg?JCK%H
ztyk)M{m-Vq*+rC>bgD6bJ``oICOt{V6`%n;4G)0UUhTckg7e@i_&W>%ts!54y@tF`
zxGJ=RqVDzIrR*6H(tS`r>Ua2`(099Xb?4}?D7D@^)Olx6<xhr3K+l8w!a=b08qPm}
zJhdV3zB4oF*_Kl}(O=%%DB4)j9#Q8qhhsn5SGEPsKeqksA*?lIIZTHPXb)F_?7?0`
z?jl?Rq&w{ywSLDt{#Q~)bvU|D$r)J`6%{Chyq(_%Y}40ye^hI9Xujut(xQCQ2GsDi
z;XCELb5xwVbPo|}P50Wsr{vWd@G>av?Y6_}R%EZQy;8igGIQuNlsyxsfQgI8mHj*p
ziI5#b9o{u$GG%TL<P6uEaSEIZnltJ`8<1^22TNf+$S%UQPwR{39$(9xp}s)&ky-PR
zv6-*s)&o6XYhdEz)xT%m(Ad|RfmhFw!g~KB0BP_(YyNqx`M1(%-7AHYNnvr?uT_FU
zAiWfpKV14hD5rT}`@V4Lg~jQ*55d`>wSNt04%dG24rm0jhqFL)#+A?!bRY5tXul)7
zNCRiw`;XALZ#(c@Tlac~yBwq&y#wT=#iM`o_deXR6>v|3SrAFz;o@H)f4Q{2RjZ==
zru2C;6ei<v>6$Ag*SlbM5$lk?q&MlJsP^+DWy`^f5U!nS+fSf1WCmoy%}^iIe%XX5
z4vzlg(Mhzt$l2&$da}>$KO<aT>9PDysIKApt$5Ahv!N+;1fB7P({;GGuiK@cQ@-`s
z>(y7SrPDyN(LJBm+i)`U#%ZpU{A53W#8}2##|p|x{`W$x*Y(!%7G*Ve41{>>M|rgl
ztcBchk3X6^<4QvxM)!JVqkDW?7n+A#m^`smZ1UpjxzH3)ea}EuSnG!EYlQRfW3T^U
z>fEe~^tCNWhgvT+H)`Lb{q{XD1~i9$4O?J87}@I?u*%kkZ>@Nf9*et<a#w&KiaP&%
zLfIrxdy8s6uTb`6@TD#s9*Y`c%7ZS9?i29qe#?7SC*4eRf5W%XI<5Lj;vWQNy#Ih-
z`kw&d^qoI0Gih{-m+v{<-Rqk%AYCj0?U8ywE4T!zz&TI{n!){`x%M_V74%Fb7F~Pu
zY^7WaxENL!u?{o;1Snrr`}vTv>Yq>eKOSGG9%9L0pW&jD8$tKo`SlPAOLsc|b<X=E
z)os2X?!I$=FZp*{^~nx8f%bIaY|OsytTv;r()(Kj=x?;S@&M`EKx6w|&^}5!*IZGW
zeQ^S=#>{2V4l+RN>I&Ed(Z=FF(zRF9e6tw5wh>EMZPUK~Quv{Wb;#!Hfuno+6iNJX
z#XGvsyugh8hUnx+uytVjb-&J;f8>2&TRjiR%AHH}{LcGMrs~uFFA?nXVEa1qYu~B;
zp5{vJhhmYlH;;6D8uS6##Zc%7+D~i$ln6E9dbk@#!)#auzk%%HSvUixfVciw!h0$A
zAZX3{pon$IezbmtbdP%^KKA#2kaZ2vy|z~m-mr963JNr#UEX(1WlxvEYVej<SZ6sp
z(-{&WeI4iemj@O<kxHM7I#+&7*=i7Ot=&ny=HsO>8G3-dhR7~1fU804hU{e_tcM*C
zZ7%ZK#7OeXezb0T(_#s0{&~coJzR5`qyGYeFJgrAcL2~MI?#Fkdl7Wu)t{dEN_Wnj
zVJ_V^@9(}V{v3$*eGJv9bHE0m^I-cLTe;6`iSFyt-=fZyYbkp<#Ix2OB)`VlcQ7A1
zf+KI-5^w@k2HAqrv?fWXT0<h;hrCQ$IT#GF=sH{;%}J%fX^WBfk#Lc_?&b6hZ;VF=
zwm!o7^&Xfr&zp8QVa0{MmpwCv`WBK$_k@eAynE9!&+8J%9#)n9epE!6Y46_x4%dEE
zo_C3IZNQPg6PMpEP9{$Ggg3wg@DyZ&&S@$`XV4lDPPc`{y-K-Lz`3@O^^tP5PPfZ+
zbf&YNaC(jur{{5H!D))Mf7E^B=ostTPQEKa@ArqUlbJNU`K*U4u-AXBi1N~UeLrYz
zFRJ})rfhQvr*EG>XBhFrx%Ud6PTN)PI5^&)<ILID;O_;KLHozGupJ6pLza-IGN?Z1
zdPhp{rXK6r{8i{6(w;w@4e0E?5xDvftp$X&*Dq{OZ`UWgs|CF4ku;<A*c-~T*B?cH
z3mX@a(zU<Cew?uvDNnR`*-w{huZ?nKj^DV);6Dk9TX81MaZ~1#H`~#DzWChy3!QtA
z;|V81J?H@aKx>575Z#BEdEy)V7lT{Zk-N_~x|iE7Blq2%6VQR?V_P@T{Ca1rCb;^~
zt>^JL>QMcwpfcRscHo6o-0x-Yqu<eV6e;Z`%AE<i-;b21u=wGWEd{m?HI}soOGnco
z9om8J<toCle(uF^3y_-g1aTvMDZGajF5QijO=%6O0V&W88US-ms7_mFN7~;i>pW*4
zhYlWvXgW7(zmV@bz<xr%6=Ti+7ZI<$y~$gcyPv#iEoR)wIy>qD`Yajmg2MFVO<zWt
zbKns;9Q)B2w{=yFu;%UqU|-YTE6yYBX?O_ypnjYV?AbIQ6nq}rZ1F%RUe9dE7nLKe
zZ*P6lqe~s!!gRNjJb$;!<W}d#>HT2c9~P!}ll}+a&%7>W#S3MTgT}7*eLA!01Vi8r
z_!2h5ez0|5{A-EVSZoQ~BIGf7Be_~@6Tz#Cj>PGHFH$*gyln4x_zE6K%^Af#TS5Kj
z3eDqj_L|yPxOkHPX+cYO-gmo=p5z{YK8$tUFE}=1H^@3mkbT7RUdL{p<=!309<IG}
zHhc^^W6^kP1DC<MPzCCM=Ees>W9?m#UFiN$c`pau&wKR|OZYp={S6%5>rCDCUx)iM
z#8QVh&*zk@8LB6C{oC^VSNg4e;O*#6=d-rnjsIEVkAnibS6kcV-T$9Mo*yFAVd~<0
z+0_O$f2*g~&$q+KSo7<jum*DAS&;6VLM_l-Q2{E0*3afJ4L*P$A=WiSV^wzLygT8z
zAv?GbzwFDa*I2@`nMSHBC8xkX(s?IW`-P+FV|0pJ1=fbqou1p4gRr_ceR0w=YiNEb
z<k?Qr)K<?nFUt3_HNXA_AHvm=v4Qt5wN{@9nlmK-J3wb|Z^0^%u2tUXa1Z{w!|dH;
zli5~2lRp-BC*?Ya*8RM;rOb2Ylw<w#Bkco2`+zptV~r^P<!(r~j_xMnH{(?9N~j)6
zukLHfvw}z;lLt2y^t{-v$M_ev9avl6%kF`!BCY9#$y)nx?U{Y4*~6UYpz=qc6!E8n
z#&iH4h6&IelxCMZf$;NC*!9DuYtG6<_nMFFI^}N+b3yZ17tmQz5@^2}F27U0*@EHD
zJwPORAIW%2cX}?j$JL#l1vOM!yUftvdx~~V>-&D1E5bcX_=dCH+I%njCi)%EzRH&O
zLdsSGM@HeEfnr;v<*wsSK9gT=$9@m^{EJ6><E4jwAU)dGi`D-e<&Fn$z1mxR>#0+7
zkJbt8d1gQs+yYlZWsn^f*ZrUPSA5#$eP>_q{d@x2>+4ytdw&qyd!N?3e%>?1UwNi;
zJ>S(xrf=$dta7#UeM7m6z>%?^eR4b>)(BtY#kwb_e8Ka(PQ?EO_Vr@*zehRkNkVn!
zx#tkK3}UHA=Ybj%>)?GD2AUt@>EdDKZI}05t{&)JBpCh2TCvHG+W^d&;$Q6bJM%18
zd*Ux37I}E{{7Shdl7~oTd>`(Idk>>8?IkvW_nNVUwb$34)YN6%Q;93;Ikwit2OvT9
z{;NKg?sT@Vv;3(0y>RVfU+>Nr)sLrqHUx>wNS%|ZIa_OBEHd@x(YU<bK64Lu?Rev~
zMl>0Gi}ZCZj0StJxR<cE9p3Pd<Vgl!(`8Q@y~{lYo&<0HSi>VIo4YrOw_k^o-{{=9
zMX4(-^qm3OL08TTCbpWCR*rLh?R{eHznxF(vDRg+&yI|X(xz}_H2zyH8*A%LGS^;F
zXL!0V)II0ZFdx2$KSA@f+P4kbfn8qnlE$p|miE1d-JinzGbwwrsW)7^kM=lYJDlyE
z+IH|or?VfEJPPY~r1Z&@&D~cW>AnXR4$D1fH!`<f_YD46;xvC<4%(+_ZSKtTy<)ie
z^iNneAv=lGpJ?$PP_7C%b!xxzpZbvvv>9`aFJ<NQlJrOSUH#!zSP5FoW0_wy?sE54
zZC-lmUnUN%H7WCR_Xl1bM+(=cz3U;Gjw7Z0Ksni0sBWGm{-=L(`(pYOdXGl?BJBfp
zmS^VIdHBDBt)RV+Wb`Fy9(LrxS^B?S9D&6{XiLOrHX64dK_vY~i{DJS7N)Oq|64xI
zv183+!tP(Le}N-U&94>UZ_o;~UR(vCxjD}Jyu*>5u33=KJm9T2BbZ_Go<bR|+tG9!
zDQ!39?hCxD_ew(7_Fm_|G3?xbM|f^-<Y3%?RsYTPaJ3KAdVl;)OCpVTp6iEfS9;U^
z-vn3;n#Z?;*(XNQyW*8+Fldc*^#8Bw>(P20*@bhTw-C9z<6ds<8MAW7M__@@a1Q_e
z-<E-qKiAU!{SjCLi$M1+TBmeBa6X&{x%$HIw9CKPamx@s8|uN$kO?zjIs60~>$W{i
zA$%hIPxrPmEWdw2NH(sFiU$_vy$5aNCO5Rca6h*s{<Sl7R-C)nOQSyB+u3@OU;EYH
zK<7Pgz;L(&u7-0#dX(+t*=fkH`&OOfRfFr`K6o1b0cSxd|Ga<y{>BpfQRB^(Pw1X_
zG~vncFlhf+6SSW!Fdm6><l1xz=loIj525FOxpVxLl#6%&uCpnP?dRZR@U>Z{Yx9mM
zDD1_3?<e;><4DhCM~(6hvmut;q+i{0+hbDWQTLIb!Yt4_dMh*p?Zr=k-0>LhIZBJA
zd3!GMYJ9E$`x>$OwSR98j;<qhJ2cPa*#qYX>~D|kIu_~L|7r0?mj2K?6!#O}1F`5s
z<8cElgBKtfc)z2doSpcP&dT9`8)B*7n@4N<y;fcSX`(*r+N<w2TJqGm()jYqJ(jS}
zn)IBV_XP6mUvVzS;XVVg)aT9fzy13Www~1d=PNo7(>~lQ_gKQ~D5rh6qi4M*Vm>Ag
zw;1#SbLJFFy>^}{l>OiS-3azDT|Li7o`!Sre+ag`WA*DCv9(oKZp-XA{2gEi#9FVN
zFPn0eAk^Of?cZ}{TvSi4ED!Iu8niEbFpxdQ*imkCS}bBep0~C%B;Q7`bsDSxYs%FI
z_Zs=vmfQ>e&l9r*@?`0J*L6%0^&<U^fKsMDy8({=LpnQ;bnOjd)wi8b_Z)hkFZXOZ
zpfif2?44e_-$%P&vUFJFyqon9{(WG}JXZh9lsn0(i#olZ7K_Yyb-o{}GlcOlAHIie
za1f$hOYfE60l9NaB){Kz2k}4u8J<^$|BSFD!#4Q;glIa5l(v*|l}%mzUP6JUTI1i*
zpL-5-EM+RfrO*!2LHjyAi~kXHKWVP1Glmji^j{1A5YU<4HK09<_Ji72nmqYktv^xD
zqY5<o|3QQ+E4?$-6Bfa0kR22@9*thskXQP1WSz1o`dO;>nW^(4=qFT%^d~(^pUH3=
zbOq;p4_DVY1KtDE2F(q!7un5JkR7#$x}bgHasCDQzYp^NK(#S-G4xE`IfIqlbw4h7
zNp3enUziRbf%a?O@Ahb~-wJZaSxgrVlS$*!pZEQ*-1_XgU4I$eiC}ae$(8-ej%Cl;
z@C@jz=u`;Pg8!dU(toSPk-i(_=(RZVDht}TT?2Q*aCj5$=Xqi7Iq#8{U<!LXJtIiX
zyKhiiGT;D2(tEV{$&@=Dj&xK1tLoFcUXoEEl;li3cdh+r3Bc2aTM65Hf@r$-rfI!6
z)71CRYVUt@z0KLzh34q|&({7~eGa{gkZXsH(Vg!9yn2l#`~~ICH@erH^1si~EdjsY
zLCKqg>fqP8Pb|9j=J|<oTDN@7vj_iEe#__RmgIk4>#TSCWCPldE(EV$V+reBunyqb
zQo;Mu|2*mZ_qRAtU(S03FBVbv`ziN0{L}X+|NR>I*Y?oWzs`~SLoDMwTppb->U_x=
z`~TV={O@XqBLP=u+OKKPu5*WY_x!UdQx5*8`@>Phzp1RNw-UHIcc=mlp)=?V_%-+x
zHo<Q2j{VPws{xMgn=SqK{jSHqNqqnDmo>VJ;A&ma*uDtbg3c&kfv@0aNCjVKz7w4K
zZ~h)vP@jXo#B9go{<qKPJCtrk?DYAH?+*rhxKW$`Kz%2s2S;uVMST7N;e_;HM#E5{
z=bhvAU@*s*?PU0SWzz8{hr)-hP7Z~0d`|uYeKQp9lRnwkQTY%0oN)2|ZkX2SlU%lw
zuc6XuP3SYqS6gAV#`%m2rEl{2iV?2tYv(Jk^lfgkudye5moG72dR1Q;h4+~JZg>+_
zIiHH#$|`-6uTv;oe(n1TAMmXUg%c+}uJFzTr~f4oqJqJV3C%<4zT#^Y{-BuCqT)n0
zRCrWL$<J5(fG^aa$;Ue76Yks=N>4s@veFaM=Q!;@lv6j^cM=gaMEyIM@j!;sec79w
z{_OSjcEbmp{%rR-zsym|m%h^p@AOqs?(*Au9PpK<{2t$F3YVQU@*`i#;Gwm9d}k<J
zCYaHO@Ik`5n6E7A^c4^8ukDJbB<du*z4p#fdP$`h3vNtw)l)h+nKlPEl|STjKE<4H
z`lsbxbruV*Q~Ip3Za5+MDfzNHm2v+zp&L#vdoWZ!p`$Jw%qVI8cHtrP?Ca<&c_<^4
zp}5jJ`bwLyuM**gG|^3Wa@F<~Ka}J6LjMsym|=wzGfMBZg1*Fz6Sn(8_UJ^H&nS~?
zeu^lQTwcw{{gg>g`X)Dkr}VniS$4SO<h7Oag`ZoHY=?_~)+fUb7yI0FHxyg!kNn|;
zU~YGv3<bl%91VutPcWEVIyZoaa7iDgYCgf>x-!NuHyE6JN`}vOjEN(?8e#KE2nIjx
zVugzlUYo4c%KFb&is6@??0B93N(O_!rmxEjGvMwBPR<LT8a%Ys4krZ<J|)94pVZ*q
z^1)Fi<nyf!ZchwW&I@M;ca}L68Xk%)o*vv(#*&blSK2iwrI!c>XO-M9U+8meFj%?d
zE)(+kDh3ZF7vGo{E|0j2IiuB=kU=!_kFU7w7E_j;cMJwI5+*C?EB-k4IEnlT>zr`0
zkt=f&8s>#d`qI;B9_pFwOD4y0KHpx=7s;hU`O6{u&4hDGhU!0`!J^OPGNJHYzG4qM
z;qu$Kc#iK=U#XERiCC9}K_;a8Ci_Zeq^Hy0RKmr298C5dD{Z14=9v<~pL2YtIN?3n
z81xCW+gH^I|H?E$zcI7Bs3sWP72L;U5j@n1D<D2Tgz0y~j9C&NALN<JEQlHqAD=_(
z=yx#qbF!+@r(xghU~u=O3}0O*NY->_(LNcW>5jNv<7rp%oY3^IJW0O<XZgxH)hHND
zZkSE&2OBy8u8`ql*d+dv?uHq{)aKSs92)KIPNq%{hVV0va@_nIeF?Lr9P`=aOSsL4
za?FRG?(u~tYbOd>?qOmv$-#BLV}J3LD!#)+<oHhAyXSb{S$W~Q!L8I_!jm!(So6H_
zfkPcJJRgI>{p`pN20OXpbEB|gSA3y}19k_ALh4kS_rd%+=7o?ElWG32)j<*t?veEc
zt7Cd~pc5pRg$_$NoR9SzPv*B?LT*-4rMA7#f5e}fGRN=tuY&2&6Rv}Da4M7lpBKA5
z-n4LGJAXX>gan`6=41Jt0{wj|#mWB+zlr~SYWwRZbIqW|Px$-75_kprLkFk~PG9Uc
zxN+ri+e1S*2Tp(lD3ISSTbSQ%tJ+%zMnJL~r{CT_K>H80?t1BXe@ft2u4VV(dBUqb
zaT5s_1E)<M_jJN5U=M7751=2EhQgW|Hr;Kf+OFS->;w8uK>bbd_K^2?K>F64iw!N{
zI(GY45SHxixb=kV!qrd%P6pe)q$fN77Qz=pk#s;C)ppSd^!sRP|K~8-^(!1i2OFE;
za&-@Xihmi`S_|7@8;pfrVA^yDe>zCEzr!am3myRJr5>o9$uGAT98UZ7w|qBP?bYAr
z(%=8sV#U4D=FyAVaGfV%0*nT={dSPed;vehUU(D6f!#*gUM)BV-u0xZuO~rR8;KT4
z`y_k)zU&&St&;gbnD53d#(gd=vqny!MN@O_-CR%doeK}bYcLs}fkAK+NJn-b9wsa~
z*9Pfh45+@3;WCKU)Uau^%a@kv^g)0BJY*}OcAr7`5%>zm!Vdm6&$bq~HuwkQSmV#M
zOY(jS{&eGHOPfG*Y>pde@+y2Ps1F8>jWeLE;^O(xCdpptH{osp{hbH>9e@i!@rlqE
z_*)L4zP0Ij;S8?x4VX5`mAx$W#4RMO&k&GqJ_D5-3NOI`xCQFNiQx1n*7nmL({{O?
z;YD~GZUB>ZDt_;8pMDx>-Q-!Ww*@47wf{%Z7_j@agYZ?Lc1gykKoT^Ct)RRb>(W<m
zh`Iev`?7M|sNdcH6*PYJw^K~p&&K~UjEB6xzmZjaPRqOMr!Ws`?EV8@fY(7|Kz6Kl
z9|W~;Bp5p}ZYkp4a??$JR}wzW<h6C6zb9j(G^>pl;$H=_d;LCmCs4Y6Yi>Fy{&`sK
z`t|#v<JnJ4@dr{q;(C&u)}c>fG*pGl;YP@UbSML+?Z&->IMa5ydq5v2-L#);*=_Z;
z>s`|hYmYF^ZI>Ch`mJ=eeFz+K(>}(14fMOI`a3v(Adj`F?XG)~>)CD4oPLREgXNxy
zUvo`sxCqLD>?j+4fxkfYNSDrDB24?w;<~nNn0^c+-n3n==1Kj&XWqIS@(&<x5QOxQ
zRegV}u9rT`Jf!*4?n4G)BV*&1CC-fP&G^5A@o=B(N^hET2SdJb%=Oex{T&j?`8~+}
zt#SNHYXI9pZ8!2&JMFbhzgw%{FxObf>uW~M1^(1#^SGwne%ZgqsUt$UE<JGVd>X%+
zbG3%-1daa{&<vb1s(|asM*6$g-G+NP7}=Y?N#@!&Xpa8@-iNn9ZIunF-J@U%<h4Jm
z`hw;+UDJ~5Np{mgebG8v7fid2TZcHWu4I2-f!(iZgiC<af3Ei^sBPxDnfRUdx6jP|
z8zJ`-HxWK`^J&j84m9V?1?g@VC~h;TuGOq%qu5UjU>;h*^}dJ2@H%L0%FZtY%_${a
zv~C;#)85bVYaK9Qxj#V-aQe-4K60;T+OP0QVCz70{HvaPQwcu@&w%+mrP9e#Q2(Tl
zAK6cQ$bRAtPaD1^to7j!_yV-H4Tkpc1WX3S&4*r~F>Lp%3t`iK{obnPMbrML@E-%F
z{TdJ2C!2ka$v2O<PGIlfG$u6vJP9j-J!#U`7PnnBj&UqIHgepHU$(v3ic^~U^a;q;
zm&4oeFgyaX!6~3V4Fku9xJE5l?_4YNcfQNvH*J^uFsNPH`{?h!sQrJyU`Pks_VyCK
z4Q7JrXRz(#7fi*5z5vsHx$l7X4|d##glBo;Rudi%n?du^63}{B9n`)OAY0IWJ_+o$
zw;{Y0)RsxG1~k`5N1uY`eB1usA*}ws3%Tv*Zzs<S+}p0ZzkTaj{?x!ye@e<4f3uX&
zanVbP_6Md9+wqSBt$8MHAO5GoJGL{3Gi|+^>v`w?D~MZXwL@d~B~V{A&#Haj!(`X5
zIcWlH1nEj+o8NFv`lZ$6_D=*BjnuQPm4SDMyc>8Wb86tpZo~cEZ}<Cey>WuSO^ez7
z0Q-t&DI0+`BW0yWS957=Hkdxi)jW0@*YLH?`|Vg?yMDRr?>)}-^W>=YN^zQNG-s;)
zYNOh^2*$uRP`t+V_q6@5*5hs%OS@M=-m|)e_!o_w0n_+>(jNi~hQA+}J?M?Vi+x@Q
z414f4|3jVc_urECvcF~X1@z-f`t%7rNxu$Q{gRGaavh@s*UkN#ST#xe8H^n)!Cwbf
zSaC{w1#X9*toY@$eQKK#^<Ly}A}-0-_E0|-689YBe;0Y-$PM&q7<@~=z6s15mJ@h=
zz$<|XJxBX9?@jgJ*YRm=@HK3334K{jpT;|jfagEjp}uHb$X?V&?FlXg#SaD3Cym_!
zkO4*~O5;AJ#!ad34d}r;-apVzy$f&J`X&CS!24WX-@_UM-+@<GQ|Z&|pl56=19OMW
z3B1y88aCL)({!_@xw@#{fwpXdR4{Q3o%V(PuFYip3qW?i4nqCPs=2uN+r362L;c>*
z^YDy|Z0R<52Yvz5hkf`b!E(rJd&obJxM`r@49H7cIC5=Z>F5UD_R|K_PPrAq4+&QM
ziTGvjZ-JR7wPw8uYX7PhuXY(r+jm;m)B8yJu1q(W33`sL_Ub(q_3csU1K+^!uou+U
zPv9R`*tCBuajhU<^PRr|o!9<Sw8id=?Qcw)#;92jw1$nt=2y3Tp~Gm}F4>xE$yIwM
zfWBknjhBA(U923CoSy~tX(Z_VySHE+$i8HQs;jH{KCtOPsbh-S?W;h027-4j<37jv
zo$HH9TL&R~k(*g<V(X`yb*AocHlSz3Z-d=-`B#G4y2nbBovekPFc;?hUvcLGr&GD_
z@ij9vZpBW0sJ2_wq)6r3-pYtAZQgh8GYCV{PAbIC?HKQ1T&tl?I6gIVDwmQ_gfM1C
zF5^0*<WkHKigM}ZwlU}XTkm>a%WF)|$LE|gd_VtZJ@0z{|NnDY&vm^spggLsl(#+*
z4wf)K_+tF8NLX_<jinu2@#b^I%?2|DsE<FNklvDVpF9GcGeC9z0c3&t8GYx%-JayT
z%5)0+1nzw4y$JcLw&RXh*-Q6LaE0%#zd@irXCzzzm96@AciUJ0xd$|V&}X!th~mWY
z56_QUMgQ%$r?ONV-UIUQA0WFT0~Wz*P+o(5l)q|2tHABM!t0?n*uLxUMVTMh-%*Ok
zC-sS@U8vuz4J!MZkOStk48r@9(jLf;TIPF?Hm~w=+dRniR50bM^4I&sdGG{Ogr?9L
zQbBD@<vtlqzhv}%L!9oleV26qxXNGi9rH}e&$I9($p6>D_%6AXb~7?&;efaBU1e@;
zNuGc=VP8<ZKAQ&R;1N(h>Odt>yAZlI-}TWmycqrtCHk&>xct8ZzjnI(%p|P-QvMHc
z#moP_v2&hDpv{j%x5_RDcER_cXL2jJ^Q7=Xs0>YDFPsC_iJtj+&<p~8I^nYJICr_b
z!;X*gGu`o1&!_UEI8h(`bA(Y5C*uFptFdx790PsFK+o!I@IXUQ|Dk?k9efA5P#bE(
z1_;iZ#<bf(b#VuTLvUW)E?*IL`Pqi>dT8K^lmGtt#Hff12~%28zOQdWf3Ocrh^y+d
z8$$SIxEtitt)TW0oKAIB5A;lAfa*s1(ewWjGypp<S2*puNbr}j{Hz4^Y~RxYf78HU
z2csf7B~EHO*SmVcO7sQ$pf(^M{AE`1--K10p99MKA<%QPAN~TNFaa)sTlW#JTSBS+
zD=W6om-As?UyiSDbs2bbxu|6+Ugl)#-}G$$0<u*IkAmNKr29wTsn%!C@A3^Jd<!VA
z?ZCA87YPUV1HTg24NBDOw(&<{`Wh=~Yy+$P(H&|N6`%&RfUfX31Z)I{rP~#!^tXZf
zAQP{=d;uyC(+^AP8B)F2_x+KNrG0+Q--CSF2idt7?gF=OejhLYjvSS_#;t77d|(hn
z!#$vWDG5?Rb2^pN4f}qrd>P%2j6og%jWudtGvIls0{%Ss{}$IyuHEfhVddlMJpbSM
z-W>ZSja6?EUmoPY+MCKe*!Pvhsc$lQ49JeocSo;Pem4^DvZs1i-xKVY<~gP>Rofa4
z8kheK=H9Oe-wy$wgTpuCV`*s}J9-vh2KCqS%{+U`PhBv6={}83jUWotw$;A$+&>2I
zfbtg)jiEXyO=&LN&}Y)^p!Okq>f7UBFf;}8d1+E*zXMEtDE~i$57f`zkdLJ$mhS5z
zScfZ(+PvDn@!iO5;%b`vg46$RUh_MG`5hY?tNfH?%kfKn&2%t68>#2vLFXR3d`i;s
z{~71_rheTEDexhvPyGR6AyBW^<7e>^-=9xz=MAwuEA&(&WPufOsunWSvZ_}@7NJGg
z*~rR%S;mne>0Gxap8GDX<xO7hyExXehVZ80vND#{68Q-4O7hR@fb^BMtTf)B%SQ4)
zESpzl!+oUZO~z13uKBMQmaLl`d5a^bO7dgg(tOr`X<ew|51iM_PW+sfB>f@1&4=7L
z1Bw3pMM#Wh_CWH+Tu~&lsw0~d&voT~mUSwdWZNyPAQBH+S=N`e%NI$W$gG-e@p*Cn
z<1xNu<o30TtTo6>**$8FL|*VkMo1Q0zLa9chxkHl8Jc3tix++i)phalvt?{qeD-Ff
z-qWfLkq~%K%gGZ_fA~|<oUhZ^dmG0OXsSKqU7$0U@z5TgfV-hQ*yj%}S&6XD=IrxD
z6JMGGokj=NCYo#B2b%k9&61VqyZ>v>O>};vy<VL?=sZN{C54a<I`eV+(1z<Q$ON4a
zm4njEyXU*wr&$53U?%8YoL^&}X|;rNk6+OvAJ)NPC;S7lJ3J5Vpf=P5ol)stozpf0
z+rMi$fgSk}3TDlAoUnY*`toGL;D=Jt@h&Wf*`T}~0F^@q$PexBZ-rX$I>^2l1J`3;
zK1_3LY9DO`=)7e)e~&B28{b}MB04|T8M@kp&P;^rU?muPJqhb9E(V5!&V9n6#O$Nr
zUH;nl(Ej{z(E4d4?Bnd|T<mi#S5XJ%>}w19_JeF510Q@0UxCgg`$7Ud0&0&X<{#Z|
z+uDnp4@00l+zO-NA{6s3%0l$$9BUt_J$?oz&*KP(K~=a17Jxqc8Po;?_R;0GX~Jet
z=XXAT48kE3w9lh`=kMVi-UFD;xq)o!SsVs^U>Zz-ieTG8D(~jKj8%jy=rjGGM}&cF
znCpIo)z(BBsO}X$084GgMEn@{VvE((!C7=EZ)>2u^n&vKS9lGcg(%Q@YXRsSup!vK
zp-VQCp^p>RbtGuN>m+DDPW#ZZuXC7@P^fF(<=DnKymYD^`~t0vUSxgH`KWwT+fX}D
zd(iWm4930}T`E)AFnc<ZpTXy_AJhkE&uR_42Sp%%)^LvQqYiWql?}b&CZijvXH@ou
zo(c6=3U7capuBE(bg7-U1jUIcu=9yr3m?HYP~EE??B+RG89$)OQtTf^m-4SO$TuMq
z{t5j+zNyUTgY4>zR()4nke#<ddC~b%T~J&NP`&>I`Ya3Ha>AF8?}Pj~Pk+3V{<y$7
zzmjhEv#)l!5eh-&GZA#Ysy3*0AzM#E0jQnpx-!fK?Qw2~@8A<y4CmlycmwnqzC|35
zd8z%|o}_r%g0~1iKIQyLXHdnQMIFONE|@xyUB&5)P3=JWR9!#h+Upza*K61J2ZR@b
z+M?`z32F=ScQtzl7dcDIVUIYEGqJti#c5l-8UHcKJF(v^&%j=*Jv}?E^~A^g3p;!9
z=Qq&V>^7&Y)Mk63M|pNLkZYCs85jUgKJ9uR&-HH5-ukM<36HFZTAsRzeNz8CGbg=>
zv#gbH8h+(>l<Q3EU)}|ybG$Q$toDrVJJ-{<`*=^+giW59sBb)8?;Ot4ID3H0J~SY#
z^F@`h`n`8R{%nAB_{F|Xdmt-%$*|$r);{1K&|Fb-M(uUTpEIxmezg6|Wgj{>eHq~`
zoQoEDmyXZ(zLzq=3<BshKInWW8E$jpVvuV0I-8QzKI{)MANKhST}RP>5Y|E(%m=L<
zvOx3j&q3?Iv!K1nZQzs@>!Wp39(MJCvR@Yd48|8pwGZ_{+Q0sp^Oz6g{;LaX;Pg|D
zu8T<ROK6Wmx=(}F$V*@=<bc+c+Jnmltpy686}asv+#S^BDuDUSOLz&W&d<hAf9efv
zXOLFqo)0Q>={4(b>E8(ceERojmvXIg_#RXaTH9K}Wv4aa!=OI4xf9<Oc?ecv`$R(e
zV{6!>R6Q$?S{KX(t&vxQ$~_bMgZw${pmNat@<(Mg9PBkH(rsUDTkTBrgKD7qR=#&8
zPG~$MYQ-pgciOste={2$mq5B@uQfD;7SP-w9+?iyKy{#XwVn5%x>rA>{zcDreW(h4
z`(qmxaBlk=?d>zt%I0#I427U_{~ex%CqeB%Wuv`@j$qqDhB^dn2icds`OpSTxs2u6
zJQDNKi+XO<hR=ia%Z~Qb9{}0#Kr_&OhpG2Z3D<=YFcEq}gO0Ae2R(Zs*j1TJ`uhZ5
z-3Mde?KX=tKTcXxf6`Y5Oumd%dCSgXSP1oDB$)cq{zG6LX>13TfyuYUyCGKe!dEQ*
z#`E2z)BeXW7zBa(BW!FNc{6cUp&Yb=Z$V{t4hDnjz`oaQzoc;dw02hKVbv{f)>Xep
zt^qop2DPyoV9G*et^Q8W-uuu6R0dB$0#tzOvK{P?NrzNf@C?{>p*EKZ72$SJeVe-f
zhVVmR$|WF!bzjf5qvsanJTNxY)*3sXsjT&UX&g|z>Z=~8F3Lg(T#u;|?@?M%d;dEO
zgz2FDYLhSN4~Ob-4~zjlgW3bjfIFc?-PaSTwEqMC!M%D`RZi*~UIn|qyB=Gm-Gd%W
zf8RyYVD5LW)fRlP9)1GlUC)Ajzq!{fFBFx(>zox{_B^{7e9<o)!uUdC_#v+OLf_~a
zR=Lm_R;93VR%l2iwi3#Lwh^=)atiXH0Ok_ki?q4yie!^E(y}(UL(lrK?8+;{Gb>&^
z6<Mt~Gvs7(wy#xj4y@_ls`yAzvi2#egY*UVDcDE(qaRlvsWCf*y@}SbFFd-Cxz)GK
zs}{gZ&<h>}d#>-udQM!G4jnq!@xiuSx-}OW49Tqf^LSryFYAp%q}N(%GiZ&S2M<FH
z(A>W>=pL;H?}V#txb$};d<Hgw=9qh;pKi-{H0*cjKO*gRI1IC4FK7<m9Cm`{wAyPZ
zO@AMBYCN3@AG2P_i|f}+YxEys7kmQJrFDwt=+gNNXpbTp+QJq3(PR2(?N9WEOt>9-
z!taSG5A0(-kWV_TG1kK+P#(HMAJ99W3t&2Ejb`g7t<h^_SK{`;qtFVp?mSLCY>ZBh
z%w*ju8y~<pP`Nw-(iIMx=PNH0!PcKR%AVuifsQcHyml3+Y*ImUY^`+`QbxPu2Q}83
zQvPed=21sSb)?p_qhUO3fq8HX$mhpF^RzmkYw4B$@>}z&BFKb)Cut8W?kSbM%KR0j
zcgRG_Z`Hf@6%NARK>Ex&;Az6T)*R&v(0p9$RJEae*h3z2Vtcpzgfdck?GX)uzMwUs
zbZbqfwVBo+(;$R&i$QCx=AgP&+xZ%PhqvLFLk4wlB4J3QuRI+)9AQm!#<iwVTT;1b
zy*&qHM|Gt4o;1IT0{Pw(PJqty#TM8LT7xduJU4Mt^R3=#uYB%F>9fVtr(1@nOZ;};
zo7}^*k&VrZunW6~xz;n-mvqX5bc^X=+P!q2hdHo^_2Rz7agA0-FB!VSn>$(WYko)n
znB`qL={xVDv^?*-BQm_N57_J((tDYwSLgYjq__jv)f&*~meibF<y8~rf#%4w;bhX7
z`wzx^+<zwXwLJ80f{)-3{6gFYIE78Eh1Rh?+vQz8@u2sE)TyL(>#xFfH7Bk;awBN(
zZGG(f$>?Q$=bz7iiqslF>xoQQ>Vy@a3#Yv~Z>(pXqqbu7M-qO)xmKBJeN{-goQ|6n
zJ%coBK=X&?pz&Pe*ls8U&E>R)%7T5c2<``?yB4~(gVxFF6I9>dQlDEArncSc&6=pP
zJ_XYKIcR*I1<%2Upmmhy5obZ`mB4%vJ+h&Es2%Qvy0E~0Pis=Mxlt?A@Fo2SX;c>5
zL4M1}XwZCM6#Nb9faVUZAuu0Am-MT@e-r%ebX3G3{@#2MbN_tm?MLpH&zfg<1kER<
z`)!b|W6pICWEarf&l37^^h;KTP~Ok7l2Yr{j9Kuq-b+-uXuhJgik-%hn$LEJNze&K
zf^2E-*9}V2Z`*phk7acpa=Ycpdq?x8d!Yeno>(0;CkTmJF;(d`KTQUe$q3NgeJE(&
z7SJC&MpE8Z^x{EoU+>_4wL8tZ-+&4b1*&iPJOOmS=4yd*57rxe?WEED@=3Z5gM1Ey
zo}lMTbr1)UPz!ARO~&>JR(I7k_Zpv6HV?tWARkTMq)WPwf}Xn+u=QW1tb_>eHF~Zk
z%X7~ZQ2pn^U0~nqR`*-TTI&Z6ocJvP$T0nK1?FFEEFWJ$uM(2#|D7r%s0}@!2y~X!
zo>`TT`+eL$9@)M{xHYIuS?dfZXP2ys)}>07t&6`^vosH@0<|G94=cmPHGZ7-tp1};
zQd&e3-v!y|jLEa;j|V~}(74tS5<ug!t<6cRcG?888BccbeV(nn>#s3gW5yns0;%v%
z&{)_4SeFL+F{SSSSul<>iGrw3?RH~LW4GouSs-06!HaMo*v|&{OWq3VclE5Np!ZPh
z^M6_C<Y6KAYTvF7B*Hh~1*N+OYCvNsg7<AX=DwVSp^f%9^TXlX8wS0>jDr&hYh2Zs
zodV^-8I$aBZyVp`Er{;YCYNu5Y3>*gLqU1p0xBEXo)7WR5^`Y&XxwL<i`W_W%3r@?
z%-YXhD0{N@KB&HDu$eI5bLsdT_uU5JAU*3q>GMdtBWY~IEinuF?_zImuXmnrt@o{0
z^F66A<aqk^*yib$n2%n~nM@fb5{`rn*uc0~7(1``7xZQ7D_6sA#<&e|fcCtV@$~@v
zkeWxBGHmLEUqv2`%Sc+vc&Kq_6R1B_AFjS6&k4`=uAiduMf0~zm<6%y$9>73*cXf~
zjIr4Ek6G^`hk@3K8ecTtGz8<jY$lSnJMRqci&{1AOZ=U~=P$!F7ztm)K=42dumtHu
z`A#EhhEQIAVh?mEX*G7K@38MdJ_gb?1P+4Mgzof~`u|QNYFV8J-=_FG;Wg+F>WAd_
zQfLnHTYZF`-aL-}>%i|5ew=IF-xaFD3ostkpR3=$BJGtr?9bGnY5i>aJ@u_Kp%9ut
nAbruFIMe3*TVzA<Pc{*g70G8lNFB|&eW+#a=8XL8ILrEPg}vnr

literal 0
HcmV?d00001

diff --git a/docs/source/_templates/autosummary/class.rst b/docs/source/_templates/autosummary/class.rst
new file mode 100644
index 0000000..33c3448
--- /dev/null
+++ b/docs/source/_templates/autosummary/class.rst
@@ -0,0 +1,53 @@
+{{ fullname }}
+{{ underline }}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+
+   ..
+      Methods
+
+{% block methods %}
+
+   .. rubric:: Methods
+
+   ..
+      Special methods
+
+{% for item in ('__call__', '__enter__', '__exit__', '__getitem__', '__setitem__', '__len__', '__next__', '__iter__', '__copy__') %}
+{% if item in all_methods or item in all_attributes %}
+   .. automethod:: {{ item }}
+{% endif %}
+{%- endfor %}
+
+   ..
+      Ordinary methods
+
+{% for item in methods %}
+{% if item not in ('__init__',) %}
+   .. automethod:: {{ item }}
+{% endif %}
+{%- endfor %}
+
+   ..
+      Special methods
+
+{% for item in ('__eq__', '__ne__', '__lt__', '__le__', '__gt__', '__ge__', '__nonzero__', '__bool__') %}
+{% if item in all_methods %}
+   .. automethod:: {{ item }}
+{% endif %}
+{%- endfor %}
+{% endblock %}
+
+   ..
+      Atributes
+
+{% block attributes %} {% if attributes %}
+
+   .. rubric:: Attributes
+
+{% for item in attributes %}
+   .. autoattribute:: {{ item }}
+{%- endfor %}
+{% endif %} {% endblock %}
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 0000000..6d9ad20
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,518 @@
+# -*- coding: utf-8 -*-
+#
+# CuPy documentation build configuration file, created by
+# sphinx-quickstart on Sun May 10 12:22:10 2015.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import importlib
+import inspect
+import os
+import sys
+
+import cupy
+
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+import _comparison_generator
+
+
+__version__ = cupy.__version__
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+rtd_version = os.environ.get('READTHEDOCS_VERSION')
+if rtd_version == 'latest':
+    tag = 'main'
+else:
+    tag = 'v{}'.format(__version__)
+extlinks = {
+    'blob': ('https://github.com/cupy/cupy/blob/{}/%s'.format(tag), '%s'),
+    'tree': ('https://github.com/cupy/cupy/tree/{}/%s'.format(tag), '%s'),
+}
+
+
+# Generate comparison table.
+with open('reference/comparison_table.rst.inc', 'w') as f:
+    f.write(_comparison_generator.generate())
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc',
+              'sphinx.ext.autosummary',
+              'sphinx.ext.doctest',
+              'sphinx.ext.extlinks',
+              'sphinx.ext.intersphinx',
+              'sphinx.ext.mathjax',
+              'sphinx.ext.napoleon',
+              'sphinx.ext.linkcode',
+              'sphinx_copybutton']
+
+try:
+    import sphinxcontrib.spelling  # noqa
+    extensions.append('sphinxcontrib.spelling')
+except ImportError:
+    pass
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'CuPy'
+copyright = u'2015, Preferred Networks, Inc. and Preferred Infrastructure, Inc.'
+author = u'Preferred Networks, Inc. and Preferred Infrastructure, Inc.'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = __version__
+# The full version, including alpha/beta/rc tags.
+release = __version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'en'
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+# Suppress a warning that multiple targets are found for a cross-reference.
+# See #3250
+suppress_warnings = ['ref.python']
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+# Napoleon settings
+napoleon_use_ivar = True
+napoleon_include_special_with_doc = True
+
+# -- Copybutton settings --------------------------------------------------
+
+# Only copy lines starting with the input prompts,
+# valid prompt styles: [
+#     Python Repl + continuation (e.g., '>>> ', '... '),
+#     Bash (e.g., '$ '),
+#     ipython and qtconsole + continuation (e.g., 'In [29]: ', '  ...: '),
+#     jupyter-console + continuation (e.g., 'In [29]: ', '     ...: ')
+# ]
+# regex taken from https://sphinx-copybutton.readthedocs.io/en/latest/#using-regexp-prompt-identifiers
+copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: "
+copybutton_prompt_is_regexp = True
+
+# Continue copying lines as long as they end with this character
+copybutton_line_continuation_character = "\\"
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'pydata_sphinx_theme'
+
+html_logo = '../image/cupy_logo_1000px.png'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+# https://pydata-sphinx-theme.readthedocs.io/en/latest/user_guide/configuring.html
+html_theme_options = {
+    "icon_links": [
+        {
+            "name": "GitHub",
+            "url": "https://github.com/cupy/cupy",
+            "icon": "fab fa-github-square",
+        },
+        {
+            "name": "Twitter",
+            "url": "https://twitter.com/CuPy_Team",
+            "icon": "fab fa-twitter-square",
+        },
+    ],
+}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+html_favicon = '_static/favicon.ico'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+html_show_sourcelink = False
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
+#html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# Now only 'ja' uses this config value
+#html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'CuPydoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+
+    # Latex figure (float) alignment
+    #'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'CuPy.tex', u'CuPy Documentation',
+     u'Preferred Networks, inc. and Preferred Infrastructure, inc.', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'cupy', u'CuPy Documentation',
+     [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'CuPy', u'CuPy Documentation',
+     author, 'CuPy', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False
+
+autosummary_generate = True
+
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3/', None),
+    'numpy': ('https://numpy.org/doc/stable/', None),
+    'scipy': ('https://docs.scipy.org/doc/scipy/', None),
+    'numba': ('https://numba.readthedocs.io/en/stable', None),
+    'cuquantum': ('https://docs.nvidia.com/cuda/cuquantum/latest', None),
+    # blocked by data-apis/array-api#428
+    #'array-api': ('https://data-apis.org/array-api/2021.12/', None),
+}
+
+doctest_global_setup = '''
+import numpy as np
+import cupy # TODO(okuta) : Remove this line
+import cupyx
+import cupy as cp
+np.random.seed(0)
+'''
+
+spelling_lang = 'en_US'
+spelling_word_list_filename = 'spelling_wordlist.txt'
+
+
+def _import_object_from_name(module_name, fullname):
+    obj = sys.modules.get(module_name)
+    if obj is None:
+        return None
+    for comp in fullname.split('.'):
+        obj = getattr(obj, comp)
+    return obj
+
+
+# note: cupy_backends is excluded as it is undocumented
+_top_modules = ['cupy', 'cupyx']
+_source_root = None
+
+
+def _find_source_root(source_abs_path):
+    # Note that READTHEDOCS* environment variable cannot be used, because they
+    # are not set under the CI environment.
+    global _source_root
+    if _source_root is not None:
+        return _source_root
+
+    dirname = os.path.dirname(source_abs_path)
+    while True:
+        parent = os.path.dirname(dirname)
+        if os.path.basename(dirname) in _top_modules:
+            _source_root = parent
+            return _source_root
+        if len(parent) == len(dirname):
+            raise RuntimeError(
+                'Couldn\'t parse root directory from '
+                'source file: {}'.format(source_abs_path))
+        dirname = parent
+
+
+def _get_source_relative_path(source_abs_path):
+    return os.path.relpath(source_abs_path, _find_source_root(source_abs_path))
+
+
+def linkcode_resolve(domain, info):
+    if domain != 'py' or not info['module']:
+        return None
+
+    # Import the object from module path
+    obj = _import_object_from_name(info['module'], info['fullname'])
+
+    # If it's not defined in the internal module, return None.
+    mod = inspect.getmodule(obj)
+    if mod is None:
+        return None
+    if not mod.__name__.split('.')[0] in _top_modules:
+        return None
+
+    # If it's wrapped (e.g., by `contextlib.contextmanager`), unwrap it
+    obj = inspect.unwrap(obj)
+
+    # Get the source file name and line number at which obj is defined.
+    try:
+        filename = inspect.getsourcefile(obj)
+    except TypeError:
+        # obj is not a module, class, function, ..etc.
+        return None
+
+    def get_pyx_file(obj):
+        filename = inspect.getfile(obj)
+        for ext in importlib.machinery.EXTENSION_SUFFIXES:
+            if filename.endswith(ext):
+                filename = filename[:-len(ext)] + '.pyx'
+                return filename
+        else:
+            return None
+
+    # `cupy.ndarray` (aka. `cupy._core.core.ndarray`) has `__module__`
+    # attribute overwritten and `inspect.getsourcefile` doesn't work on it,
+    # so use `cupy._core.core`'s source location instead
+    if obj is cupy.ndarray:
+        filename = get_pyx_file(cupy._core.core)
+        if filename is None:
+            return None
+        linenum = None
+    # `inspect.getsourcefile` returns None for C-extension objects
+    elif filename is None:
+        filename = get_pyx_file(obj)
+        if filename is None:
+            return None
+        linenum = None
+    else:
+        # Get the source line number
+        _, linenum = inspect.getsourcelines(obj)
+        assert isinstance(linenum, int)
+
+    filename = os.path.realpath(filename)
+    relpath = _get_source_relative_path(filename)
+
+    fragment = '' if linenum is None else f'#L{linenum}'
+    return f'https://github.com/cupy/cupy/blob/{tag}/{relpath}{fragment}'
+
+
+# Python Array API methods have type hints, which do not render
+# nicely by default. This option moves the type hints to the
+# doc content so as to make the function signatures shorter and
+# look nicer.
+autodoc_typehints = 'description'
+
+
+def remove_array_api_module_docstring(app, what, name, obj, options, lines):
+    # We don't want to take the docstring in cupyx.array_api because:
+    #   1. It's not how we document module-level stuff
+    #   2. The docstring is taken from numpy.array_api, which requires rewriting
+    # Here we remove the docstring and will add our own description in array_api.rst
+    if what == "module" and 'array_api' in name:
+        del lines[:]
+
+def fix_jit_callable_signature(
+        app, what, name, obj, options, signature, return_annotation):
+    if 'cupyx.jit' in name and callable(obj) and signature is None:
+        return (f'{inspect.signature(obj)}', None)
+
+def fix_ndarray_signature(
+        app, what, name, obj, options, signature, return_annotation):
+    # Replace `_ndarray_base` with `ndarray` for signatures and return types
+    # on docs.
+    if signature is not None:
+        signature = signature.replace('_ndarray_base', 'ndarray')
+    if return_annotation == '_ndarray_base':
+        return_annotation = 'ndarray'
+    return (signature, return_annotation)
+
+def setup(app):
+    app.connect("autodoc-process-docstring", remove_array_api_module_docstring)
+    app.connect("autodoc-process-signature", fix_jit_callable_signature)
+    app.connect("autodoc-process-signature", fix_ndarray_signature)
diff --git a/docs/source/contribution.rst b/docs/source/contribution.rst
new file mode 100644
index 0000000..7cf7f25
--- /dev/null
+++ b/docs/source/contribution.rst
@@ -0,0 +1,420 @@
+.. _contrib:
+
+Contribution Guide
+==================
+
+This is a guide for all contributions to CuPy.
+The development of CuPy is running on `the official repository at GitHub <https://github.com/cupy/cupy>`_.
+Anyone that wants to register an issue or to send a pull request should read through this document.
+
+
+Classification of Contributions
+-------------------------------
+
+There are several ways to contribute to CuPy community:
+
+1. Registering an issue
+2. Sending a pull request (PR)
+3. Sending a question to `CuPy's Gitter channel <https://gitter.im/cupy/community>`_, `CuPy User Group <https://groups.google.com/forum/#!forum/cupy>`_, or `StackOverflow <https://stackoverflow.com/questions/tagged/cupy>`_
+4. Open-sourcing an external example
+5. Writing a post about CuPy
+
+This document mainly focuses on 1 and 2, though other contributions are also appreciated.
+
+
+Development Cycle
+-----------------
+
+This section explains the development process of CuPy.
+Before contributing to CuPy, it is strongly recommended to understand the development cycle.
+
+Versioning
+~~~~~~~~~~
+
+The versioning of CuPy follows `PEP 440 <https://www.python.org/dev/peps/pep-0440/>`_ and a part of `Semantic versioning <https://semver.org/>`_.
+The version number consists of three or four parts: ``X.Y.Zw`` where ``X`` denotes the **major version**, ``Y`` denotes the **minor version**, ``Z`` denotes the **revision number**, and the optional ``w`` denotes the prelease suffix.
+While the major, minor, and revision numbers follow the rule of semantic versioning, the pre-release suffix follows PEP 440 so that the version string is much friendly with Python eco-system.
+
+**Note that a major update basically does not contain compatibility-breaking changes from the last release candidate (RC).**
+This is not a strict rule, though; if there is a critical API bug that we have to fix for the major version, we may add breaking changes to the major version up.
+
+As for the backward compatibility, see :doc:`user_guide/compatibility`.
+
+
+.. _contrib-release-cycle:
+
+Release Cycle
+~~~~~~~~~~~~~
+
+The first one is the track of **stable versions**, which is a series of revision updates for the latest major version.
+The second one is the track of **development versions**, which is a series of pre-releases for the upcoming major version.
+
+Consider that ``X.0.0`` is the latest major version and ``Y.0.0``, ``Z.0.0`` are the succeeding major versions.
+Then, the timeline of the updates is depicted by the following table.
+
+========== =========== =========== ============
+   Date       ver X       ver Y       ver Z
+========== =========== =========== ============
+  0 weeks    X.0.0rc1    --         --
+  4 weeks    X.0.0       Y.0.0a1    --
+  8 weeks    X.1.0*      Y.0.0b1    --
+ 12 weeks    X.2.0*      Y.0.0rc1   --
+ 16 weeks    --          Y.0.0      Z.0.0a1
+========== =========== =========== ============
+
+(* These might be revision releases)
+
+The dates shown in the left-most column are relative to the release of ``X.0.0rc1``.
+In particular, each revision/minor release is made four weeks after the previous one of the same major version, and the pre-release of the upcoming major version is made at the same time.
+Whether these releases are revision or minor is determined based on the contents of each update.
+
+Note that there are only three stable releases for the versions ``X.x.x``.
+During the parallel development of ``Y.0.0`` and ``Z.0.0a1``, the version ``Y`` is treated as an **almost-stable version** and ``Z`` is treated as a development version.
+
+If there is a critical bug found in ``X.x.x`` after stopping the development of version ``X``, we may release a hot-fix for this version at any time.
+
+We create a milestone for each upcoming release at GitHub.
+The GitHub milestone is basically used for collecting the issues and PRs resolved in the release.
+
+.. _contrib-git-branches:
+
+Git Branches
+~~~~~~~~~~~~
+
+The ``main`` branch is used to develop pre-release versions.
+It means that **alpha, beta, and RC updates are developed at the** ``main`` **branch**.
+This branch contains the most up-to-date source tree that includes features newly added after the latest major version.
+
+The stable version is developed at the individual branch named as ``vN`` where "N" reflects the version number (we call it a *versioned branch*).
+For example, v1.0.0, v1.0.1, and v1.0.2 will be developed at the ``v1`` branch.
+
+**Notes for contributors:**
+When you send a pull request, you basically have to send it to the ``main`` branch.
+If the change can also be applied to the stable version, a core team member will apply the same change to the stable version so that the change is also included in the next revision update.
+
+If the change is only applicable to the stable version and not to the ``main`` branch, please send it to the versioned branch.
+We basically only accept changes to the latest versioned branch (where the stable version is developed) unless the fix is critical.
+
+If you want to make a new feature of the ``main`` branch available in the current stable version, please send a *backport PR* to the stable version (the latest ``vN`` branch).
+See the next section for details.
+
+*Note: a change that can be applied to both branches should be sent to the* ``main`` *branch.*
+*Each release of the stable version is also merged to the development version so that the change is also reflected to the next major version.*
+
+Feature Backport PRs
+~~~~~~~~~~~~~~~~~~~~
+
+We basically do not backport any new features of the development version to the stable versions.
+If you desire to include the feature to the current stable version and you can work on the backport work, we welcome such a contribution.
+In such a case, you have to send a backport PR to the latest ``vN`` branch.
+**Note that we do not accept any feature backport PRs to older versions because we are not running quality assurance workflows (e.g. CI) for older versions so that we cannot ensure that the PR is correctly ported.**
+
+There are some rules on sending a backport PR.
+
+- Start the PR title from the prefix **[backport]**.
+- Clarify the original PR number in the PR description (something like "This is a backport of #XXXX").
+- (optional) Write to the PR description the motivation of backporting the feature to the stable version.
+
+Please follow these rules when you create a feature backport PR.
+
+Note: PRs that do not include any changes/additions to APIs (e.g. bug fixes, documentation improvements) are usually backported by core dev members.
+It is also appreciated to make such a backport PR by any contributors, though, so that the overall development proceeds more smoothly!
+
+Issues and Pull Requests
+------------------------
+
+In this section, we explain how to send pull requests (PRs).
+
+How to Send a Pull Request
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you can write code to fix an issue, we encourage to send a PR.
+
+First of all, before starting to write any code, do not forget to confirm the following points.
+
+- Read through the :ref:`coding-guide` and :ref:`testing-guide`.
+- Check the appropriate branch that you should send the PR following :ref:`contrib-git-branches`.
+  If you do not have any idea about selecting a branch, please choose the ``main`` branch.
+
+In particular, **check the branch before writing any code.**
+The current source tree of the chosen branch is the starting point of your change.
+
+After writing your code **(including unit tests and hopefully documentations!)**, send a PR on GitHub.
+You have to write a precise explanation of **what** and **how** you fix;
+it is the first documentation of your code that developers read, which is a very important part of your PR.
+
+Once you send a PR, it is automatically tested on ``GitHub Actions``.
+After the automatic test passes, core developers will start reviewing your code.
+Note that this automatic PR test only includes CPU tests.
+
+.. note::
+
+   We are also running continuous integration with GPU tests for the ``main`` branch and the versioned branch of the latest major version.
+   Since this service is currently running on our internal server, we do not use it for automatic PR tests to keep the server secure.
+
+If you are planning to add a new feature or modify existing APIs, **it is recommended to open an issue and discuss the design first.**
+The design discussion needs lower cost for the core developers than code review.
+Following the consequences of the discussions, you can send a PR that is smoothly reviewed in a shorter time.
+
+Even if your code is not complete, you can send a pull request as a *work-in-progress PR* by putting the ``[WIP]`` prefix to the PR title.
+If you write a precise explanation about the PR, core developers and other contributors can join the discussion about how to proceed the PR.
+WIP PR is also useful to have discussions based on a concrete code.
+
+
+.. _coding-guide:
+
+Coding Guidelines
+-----------------
+
+.. note::
+
+   Coding guidelines are updated at v5.0.
+   Those who have contributed to older versions should read the guidelines again.
+
+We use `PEP8 <https://www.python.org/dev/peps/pep-0008/>`_ and a part of `OpenStack Style Guidelines <https://docs.openstack.org/developer/hacking/>`_ related to general coding style as our basic style guidelines.
+
+You can use ``autopep8`` and ``flake8`` commands to check your code.
+
+In order to avoid confusion from using different tool versions, we pin the versions of those tools.
+Install them with the following command (from within the top directory of CuPy repository)::
+
+  $ pip install -e '.[stylecheck]'
+
+And check your code with::
+
+  $ autopep8 path/to/your/code.py
+  $ flake8 path/to/your/code.py
+
+To check Cython code, use ``.flake8.cython`` configuration file::
+
+  $ flake8 --config=.flake8.cython path/to/your/cython/code.pyx
+
+The ``autopep8`` supports automatically correct Python code to conform to the PEP 8 style guide::
+
+  $ autopep8 --in-place path/to/your/code.py
+
+The ``flake8`` command lets you know the part of your code not obeying our style guidelines.
+Before sending a pull request, be sure to check that your code passes the ``flake8`` checking.
+
+Note that ``flake8`` command is not perfect.
+It does not check some of the style guidelines.
+Here is a (not-complete) list of the rules that ``flake8`` cannot check.
+
+* Relative imports are prohibited. [H304]
+* Importing non-module symbols is prohibited.
+* Import statements must be organized into three parts: standard libraries, third-party libraries, and internal imports. [H306]
+
+In addition, we restrict the usage of *shortcut symbols* in our code base.
+They are symbols imported by packages and sub-packages of ``cupy``.
+For example, ``cupy.cuda.Device`` is a shortcut of ``cupy.cuda.device.Device``.
+**It is not allowed to use such shortcuts in the ``cupy`` library implementation**.
+Note that you can still use them in :tree:`tests` and :tree:`examples` directories.
+
+Once you send a pull request, your coding style is automatically checked by `GitHub Actions`.
+The reviewing process starts after the check passes.
+
+The CuPy is designed based on NumPy's API design. CuPy's source code and documents contain the original NumPy ones.
+Please note the followings when writing the document.
+
+* In order to identify overlapping parts, it is preferable to add some remarks
+  that this document is just copied or altered from the original one. It is
+  also preferable to briefly explain the specification of the function in a
+  short paragraph, and refer to the corresponding function in NumPy so that
+  users can read the detailed document. However, it is possible to include a
+  complete copy of the document with such a remark if users cannot summarize
+  in such a way.
+* If a function in CuPy only implements a limited amount of features in the
+  original one, users should explicitly describe only what is implemented in
+  the document.
+
+For changes that modify or add new Cython files, please make sure the pointer types follow these guidelines (`#1913 <https://github.com/cupy/cupy/issues/1913>`_).
+
+* Pointers should be ``void*`` if only used within Cython, or ``intptr_t`` if exposed to the Python space.
+* Memory sizes should be ``size_t``.
+* Memory offsets should be ``ptrdiff_t``.
+
+.. note::
+
+     We are incrementally enforcing the above rules, so some existing code may not follow the above guidelines, but please ensure all new contributions do.
+
+.. _testing-guide:
+
+Unit Testing
+------------
+
+Testing is one of the most important part of your code.
+You must write test cases and verify your implementation by following our testing guide.
+
+Note that we are using pytest and mock package for testing, so install them before writing your code::
+
+  $ pip install pytest mock
+
+How to Run Tests
+~~~~~~~~~~~~~~~~
+
+In order to run unit tests at the repository root, you first have to build Cython files in place by running the following command::
+
+  $ pip install -e .
+
+.. note::
+
+  When you modify ``*.pxd`` files, before running ``pip install -e .``, you must clean ``*.cpp`` and ``*.so`` files once with the following command, because Cython does not automatically rebuild those files nicely::
+
+    $ git clean -fdx
+
+Once Cython modules are built, you can run unit tests by running the following command at the repository root::
+
+  $ python -m pytest
+
+CUDA must be installed to run unit tests.
+
+Some GPU tests require cuDNN to run.
+In order to skip unit tests that require cuDNN, specify ``-m='not cudnn'`` option::
+
+  $ python -m pytest path/to/your/test.py -m='not cudnn'
+
+Some GPU tests involve multiple GPUs.
+If you want to run GPU tests with insufficient number of GPUs, specify the number of available GPUs to ``CUPY_TEST_GPU_LIMIT``.
+For example, if you have only one GPU, launch ``pytest`` by the following command to skip multi-GPU tests::
+
+  $ export CUPY_TEST_GPU_LIMIT=1
+  $ python -m pytest path/to/gpu/test.py
+
+Following this naming convention, you can run all the tests by running the following command at the repository root::
+
+  $ python -m pytest
+
+Or you can also specify a root directory to search test scripts from::
+
+  $ python -m pytest tests/cupy_tests     # to just run tests of CuPy
+  $ python -m pytest tests/install_tests  # to just run tests of installation modules
+
+If you modify the code related to existing unit tests, you must run appropriate commands.
+
+Test File and Directory Naming Conventions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Tests are put into the :tree:`tests/cupy_tests` directory.
+In order to enable test runner to find test scripts correctly, we are using special naming convention for the test subdirectories and the test scripts.
+
+* The name of each subdirectory of ``tests`` must end with the ``_tests`` suffix.
+* The name of each test script must start with the ``test_`` prefix.
+
+When we write a test for a module, we use the appropriate path and file name for the test script whose correspondence to the tested module is clear.
+For example, if you want to write a test for a module ``cupy.x.y.z``, the test script must be located at ``tests/cupy_tests/x_tests/y_tests/test_z.py``.
+
+How to Write Tests
+~~~~~~~~~~~~~~~~~~
+
+There are many examples of unit tests under the :tree:`tests` directory, so reading some of them is a good and recommended way to learn how to write tests for CuPy.
+They simply use the :mod:`unittest` package of the standard library, while some tests are using utilities from :mod:`cupy.testing`.
+
+In addition to the :ref:`coding-guide` mentioned above, the following rules are applied to the test code:
+
+* All test classes must inherit from :class:`unittest.TestCase`.
+* Use :mod:`unittest` features to write tests, except for the following cases:
+
+    * Use ``assert`` statement instead of ``self.assert*`` methods (e.g., write ``assert x == 1`` instead of ``self.assertEqual(x, 1)``).
+    * Use ``with pytest.raises(...):`` instead of ``with self.assertRaises(...):``.
+
+.. note::
+
+   We are incrementally applying the above style.
+   Some existing tests may be using the old style (``self.assertRaises``, etc.), but all newly written tests should follow the above style.
+
+In order to write tests for multiple GPUs, use ``cupy.testing.multi_gpu()`` decorators instead::
+
+  import unittest
+  from cupy import testing
+
+  class TestMyFunc(unittest.TestCase):
+      ...
+
+      @testing.multi_gpu(2)  # specify the number of required GPUs here
+      def test_my_two_gpu_func(self):
+          ...
+
+If your test requires too much time, add ``cupy.testing.slow`` decorator.
+The test functions decorated by ``slow`` are skipped if ``-m='not slow'`` is given::
+
+  import unittest
+  from cupy import testing
+
+  class TestMyFunc(unittest.TestCase):
+      ...
+
+      @testing.slow
+      def test_my_slow_func(self):
+          ...
+
+Once you send a pull request, GitHub Actions automatically checks if your code meets our coding guidelines described above.
+Since GitHub Actions does not support CUDA, we cannot run unit tests automatically.
+The reviewing process starts after the automatic check passes.
+Note that reviewers will test your code without the option to check CUDA-related code.
+
+.. note::
+   Some of numerically unstable tests might cause errors irrelevant to your changes.
+   In such a case, we ignore the failures and go on to the review process, so do not worry about it!
+
+
+Documentation
+-------------
+
+When adding a new feature to the framework, you also need to document it in the reference.
+
+.. note::
+
+   If you are unsure about how to fix the documentation, you can submit a pull request without doing so.
+   Reviewers will help you fix the documentation appropriately.
+
+The documentation source is stored under `docs directory <https://github.com/cupy/cupy/tree/main/docs>`_ and written in `reStructuredText <http://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html>`_ format.
+
+To build the documentation, you need to install `Sphinx <http://www.sphinx-doc.org/>`_::
+
+  $ pip install -r docs/requirements.txt
+
+Then you can build the documentation in HTML format locally::
+
+  $ cd docs
+  $ make html
+
+HTML files are generated under ``build/html`` directory.
+Open ``index.html`` with the browser and see if it is rendered as expected.
+
+.. note::
+
+   Docstrings (documentation comments in the source code) are collected from the installed CuPy module.
+   If you modified docstrings, make sure to install the module (e.g., using `pip install -e .`) before building the documentation.
+
+
+Tips for Developers
+-------------------
+
+Here are some tips for developers hacking CuPy source code.
+
+Install as Editable
+~~~~~~~~~~~~~~~~~~~
+
+During the development we recommend using ``pip`` with ``-e`` option to install as editable mode::
+
+  $ pip install -e .
+
+Please note that even with ``-e``, you will have to rerun ``pip install -e .`` to regenerate C++ sources using Cython if you modified Cython source files (e.g., ``*.pyx`` files).
+
+Use ccache
+~~~~~~~~~~
+
+``NVCC`` environment variable can be specified at the build time to use the custom command instead of ``nvcc`` .
+You can speed up the rebuild using `ccache <https://ccache.dev/>`_ (v3.4 or later) by::
+
+  $ export NVCC='ccache nvcc'
+
+Limit Architecture
+~~~~~~~~~~~~~~~~~~
+
+Use ``CUPY_NVCC_GENERATE_CODE`` environment variable to reduce the build time by limiting the target CUDA architectures.
+For example, if you only run your CuPy build with NVIDIA P100 and V100, you can use::
+
+  $ export CUPY_NVCC_GENERATE_CODE=arch=compute_60,code=sm_60;arch=compute_70,code=sm_70
+
+See :doc:`reference/environment` for the description.
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 0000000..84088d6
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,26 @@
+============================================================
+CuPy -- NumPy & SciPy for GPU
+============================================================
+
+.. module:: cupy
+
+.. toctree::
+   :maxdepth: 2
+
+   overview
+   install
+   user_guide/index
+   reference/index
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Development
+
+   contribution
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Misc Notes
+
+   upgrade
+   license
diff --git a/docs/source/install.rst b/docs/source/install.rst
new file mode 100644
index 0000000..5589b99
--- /dev/null
+++ b/docs/source/install.rst
@@ -0,0 +1,476 @@
+Installation
+============
+
+Requirements
+------------
+
+* `NVIDIA CUDA GPU <https://developer.nvidia.com/cuda-gpus>`_ with the Compute Capability 3.0 or larger.
+
+* `CUDA Toolkit <https://developer.nvidia.com/cuda-toolkit>`_: v10.2 / v11.0 / v11.1 / v11.2 / v11.3 / v11.4 / v11.5 / v11.6 / v11.7 / v11.8 / v12.0 / v12.1 / v12.2
+
+    * If you have multiple versions of CUDA Toolkit installed, CuPy will automatically choose one of the CUDA installations.
+      See :ref:`install_cuda` for details.
+
+    * This requirement is optional if you install CuPy from ``conda-forge``. However, you still need to have a compatible
+      driver installed for your GPU. See :ref:`install_cupy_from_conda_forge` for details.
+
+* `Python <https://python.org/>`_: v3.8 / v3.9 / v3.10 / v3.11 / v3.12
+
+.. note::
+
+   Currently, CuPy is tested against  `Ubuntu <https://www.ubuntu.com/>`_ 18.04 LTS / 20.04 LTS (x86_64), `CentOS <https://www.centos.org/>`_ 7 / 8 (x86_64) and Windows Server 2016 (x86_64).
+
+Python Dependencies
+~~~~~~~~~~~~~~~~~~~
+
+NumPy/SciPy-compatible API in CuPy v12 is based on NumPy 1.24 and SciPy 1.9, and has been tested against the following versions:
+
+* `NumPy <https://numpy.org/>`_: v1.21 / v1.22 / v1.23 / v1.24 / v1.25 / v1.26
+
+* `SciPy <https://scipy.org/>`_ (*optional*): v1.7 / v1.8 / v1.9 / v1.10 / v1.11
+
+    * Required only when coping sparse matrices from GPU to CPU (see :doc:`../reference/scipy_sparse`.)
+
+* `Optuna <https://optuna.org/>`_ (*optional*): v3.x
+
+    * Required only when using :ref:`kernel_param_opt`.
+
+.. note::
+
+   SciPy and Optuna are optional dependencies and will not be installed automatically.
+
+.. note::
+
+   Before installing CuPy, we recommend you to upgrade ``setuptools`` and ``pip``::
+
+    $ python -m pip install -U setuptools pip
+
+Additional CUDA Libraries
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Part of the CUDA features in CuPy will be activated only when the corresponding libraries are installed.
+
+* `cuTENSOR <https://developer.nvidia.com/cutensor>`_: v1.4 / v1.5 / v1.6 / v1.7
+
+    * The library to accelerate tensor operations. See :doc:`../reference/environment` for the details.
+
+* `NCCL <https://developer.nvidia.com/nccl>`_: v2.8 / v2.9 / v2.10 / v2.11 / v2.12 / v2.13 / v2.14 / v2.15 / v2.16 / v2.17
+
+    * The library to perform collective multi-GPU / multi-node computations.
+
+* `cuDNN <https://developer.nvidia.com/cudnn>`_: v7.6 / v8.0 / v8.1 / v8.2 / v8.3 / v8.4 / v8.5 / v8.6 / v8.7 / v8.8
+
+    * The library to accelerate deep neural network computations.
+
+* `cuSPARSELt <https://docs.nvidia.com/cuda/cusparselt/>`_: v0.2.0
+
+    * The library to accelerate sparse matrix-matrix multiplication.
+
+
+Installing CuPy
+---------------
+
+Installing CuPy from PyPI
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Wheels (precompiled binary packages) are available for Linux and Windows.
+Package names are different depending on your CUDA Toolkit version.
+
+.. list-table::
+   :header-rows: 1
+
+   * - CUDA
+     - Command
+   * - **v10.2** (x86_64 / aarch64)
+     - ``pip install cupy-cuda102``
+   * - **v11.0** (x86_64)
+     - ``pip install cupy-cuda110``
+   * - **v11.1** (x86_64)
+     - ``pip install cupy-cuda111``
+   * - **v11.2 ~ 11.8** (x86_64 / aarch64)
+     - ``pip install cupy-cuda11x``
+   * - **v12.x** (x86_64 / aarch64)
+     - ``pip install cupy-cuda12x``
+
+.. note::
+
+   To enable features provided by additional CUDA libraries (cuTENSOR / NCCL / cuDNN), you need to install them manually.
+   If you installed CuPy via wheels, you can use the installer command below to setup these libraries in case you don't have a previous installation::
+
+    $ python -m cupyx.tools.install_library --cuda 11.x --library cutensor
+
+.. note::
+
+   Append ``--pre -U -f https://pip.cupy.dev/pre`` options to install pre-releases (e.g., ``pip install cupy-cuda11x --pre -U -f https://pip.cupy.dev/pre``).
+
+
+When using wheels, please be careful not to install multiple CuPy packages at the same time.
+Any of these packages and ``cupy`` package (source installation) conflict with each other.
+Please make sure that only one CuPy package (``cupy`` or ``cupy-cudaXX`` where XX is a CUDA version) is installed::
+
+  $ pip freeze | grep cupy
+
+
+.. _install_cupy_from_conda_forge:
+
+Installing CuPy from Conda-Forge
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Conda/Anaconda is a cross-platform package management solution widely used in scientific computing and other fields.
+The above ``pip install`` instruction is compatible with ``conda`` environments. Alternatively, for both Linux (x86_64,
+ppc64le, aarch64-sbsa) and
+Windows once the CUDA driver is correctly set up, you can also install CuPy from the ``conda-forge`` channel::
+
+    $ conda install -c conda-forge cupy
+
+and ``conda`` will install a pre-built CuPy binary package for you, along with the CUDA runtime libraries
+(``cudatoolkit``). It is not necessary to install CUDA Toolkit in advance.
+
+Conda has a built-in mechanism to determine and install the latest version of ``cudatoolkit`` supported by your driver.
+However, if for any reason you need to force-install a particular CUDA version (say 11.8), you can do::
+
+    $ conda install -c conda-forge cupy cuda-version=11.8
+
+.. note::
+
+    cuDNN, cuTENSOR, and NCCL are available on ``conda-forge`` as optional dependencies. The following command can install them all at once::
+
+        $ conda install -c conda-forge cupy cudnn cutensor nccl
+
+    Each of them can also be installed separately as needed.
+
+.. note::
+
+    If you encounter any problem with CuPy installed from ``conda-forge``, please feel free to report to `cupy-feedstock
+    <https://github.com/conda-forge/cupy-feedstock/issues>`_, and we will help investigate if it is just a packaging
+    issue in ``conda-forge``'s recipe or a real issue in CuPy.
+
+.. note::
+
+    If you did not install CUDA Toolkit by yourself, the ``nvcc`` compiler might not be available, as
+    the ``cudatoolkit`` package from ``conda-forge`` does not include the ``nvcc`` compiler toolchain. If you would like to use
+    it from a local CUDA installation, you need to make sure the version of CUDA Toolkit matches that of ``cudatoolkit`` to
+    avoid surprises.
+
+
+.. _install_cupy_from_source:
+
+Installing CuPy from Source
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Use of wheel packages is recommended whenever possible.
+However, if wheels cannot meet your requirements (e.g., you are running non-Linux environment or want to use a version of CUDA / cuDNN / NCCL not supported by wheels), you can also build CuPy from source.
+
+.. note::
+
+   CuPy source build requires ``g++-6`` or later.
+   For Ubuntu 18.04, run ``apt-get install g++``.
+   For Ubuntu 16.04, CentOS 6 or 7, follow the instructions :ref:`here <install_gcc6>`.
+
+.. note::
+
+   When installing CuPy from source, features provided by additional CUDA libraries will be disabled if these libraries are not available at the build time.
+   See :ref:`install_cudnn` for the instructions.
+
+.. note::
+
+   If you upgrade or downgrade the version of CUDA Toolkit, cuDNN, NCCL or cuTENSOR, you may need to reinstall CuPy.
+   See :ref:`install_reinstall` for details.
+
+You can install the latest stable release version of the `CuPy source package <https://pypi.python.org/pypi/cupy>`_ via ``pip``.
+
+::
+
+  $ pip install cupy
+
+If you want to install the latest development version of CuPy from a cloned Git repository::
+
+  $ git clone --recursive https://github.com/cupy/cupy.git
+  $ cd cupy
+  $ pip install .
+
+.. note::
+
+   Cython 0.29.22 or later is required to build CuPy from source.
+   It will be automatically installed during the build process if not available.
+
+
+Uninstalling CuPy
+-----------------
+
+Use ``pip`` to uninstall CuPy::
+
+  $ pip uninstall cupy
+
+.. note::
+
+   If you are using a wheel, ``cupy`` shall be replaced with ``cupy-cudaXX`` (where XX is a CUDA version number).
+
+.. note::
+
+   If CuPy is installed via ``conda``, please do ``conda uninstall cupy`` instead.
+
+
+Upgrading CuPy
+---------------
+
+Just use ``pip install`` with ``-U`` option::
+
+  $ pip install -U cupy
+
+.. note::
+
+   If you are using a wheel, ``cupy`` shall be replaced with ``cupy-cudaXX`` (where XX is a CUDA version number).
+
+
+.. _install_reinstall:
+
+
+Reinstalling CuPy
+-----------------
+
+To reinstall CuPy, please uninstall CuPy and then install it.
+When reinstalling CuPy, we recommend using ``--no-cache-dir`` option as ``pip`` caches the previously built binaries::
+
+  $ pip uninstall cupy
+  $ pip install cupy --no-cache-dir
+
+.. note::
+
+   If you are using a wheel, ``cupy`` shall be replaced with ``cupy-cudaXX`` (where XX is a CUDA version number).
+
+
+Using CuPy inside Docker
+------------------------
+
+We are providing the `official Docker images <https://hub.docker.com/r/cupy/cupy/>`_.
+Use `NVIDIA Container Toolkit <https://github.com/NVIDIA/nvidia-docker>`_ to run CuPy image with GPU.
+You can login to the environment with bash, and run the Python interpreter::
+
+  $ docker run --gpus all -it cupy/cupy /bin/bash
+
+Or run the interpreter directly::
+
+  $ docker run --gpus all -it cupy/cupy /usr/bin/python3
+
+
+FAQ
+---
+
+.. _install_error:
+
+``pip`` fails to install CuPy
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Please make sure that you are using the latest ``setuptools`` and ``pip``::
+
+  $ pip install -U setuptools pip
+
+Use ``-vvvv`` option with ``pip`` command.
+This will display all logs of installation::
+
+  $ pip install cupy -vvvv
+
+If you are using ``sudo`` to install CuPy, note that ``sudo`` command does not propagate environment variables.
+If you need to pass environment variable (e.g., ``CUDA_PATH``), you need to specify them inside ``sudo`` like this::
+
+  $ sudo CUDA_PATH=/opt/nvidia/cuda pip install cupy
+
+If you are using certain versions of conda, it may fail to build CuPy with error ``g++: error: unrecognized command line option ‘-R’``.
+This is due to a bug in conda (see `conda/conda#6030 <https://github.com/conda/conda/issues/6030>`_ for details).
+If you encounter this problem, please upgrade your conda.
+
+.. _install_cudnn:
+
+Installing cuDNN and NCCL
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We recommend installing cuDNN and NCCL using binary packages (i.e., using ``apt`` or ``yum``) provided by NVIDIA.
+
+If you want to install tar-gz version of cuDNN and NCCL, we recommend installing it under the ``CUDA_PATH`` directory.
+For example, if you are using Ubuntu, copy ``*.h`` files to ``include`` directory and ``*.so*`` files to ``lib64`` directory::
+
+  $ cp /path/to/cudnn.h $CUDA_PATH/include
+  $ cp /path/to/libcudnn.so* $CUDA_PATH/lib64
+
+The destination directories depend on your environment.
+
+If you want to use cuDNN or NCCL installed in another directory, please use ``CFLAGS``, ``LDFLAGS`` and ``LD_LIBRARY_PATH`` environment variables before installing CuPy::
+
+  $ export CFLAGS=-I/path/to/cudnn/include
+  $ export LDFLAGS=-L/path/to/cudnn/lib
+  $ export LD_LIBRARY_PATH=/path/to/cudnn/lib:$LD_LIBRARY_PATH
+
+.. _install_cuda:
+
+Working with Custom CUDA Installation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you have installed CUDA on the non-default directory or multiple CUDA versions on the same host, you may need to manually specify the CUDA installation directory to be used by CuPy.
+
+CuPy uses the first CUDA installation directory found by the following order.
+
+#. ``CUDA_PATH`` environment variable.
+#. The parent directory of ``nvcc`` command. CuPy looks for ``nvcc`` command from ``PATH`` environment variable.
+#. ``/usr/local/cuda``
+
+For example, you can build CuPy using non-default CUDA directory by ``CUDA_PATH`` environment variable::
+
+  $ CUDA_PATH=/opt/nvidia/cuda pip install cupy
+
+.. note::
+
+   CUDA installation discovery is also performed at runtime using the rule above.
+   Depending on your system configuration, you may also need to set ``LD_LIBRARY_PATH`` environment variable to ``$CUDA_PATH/lib64`` at runtime.
+
+CuPy always raises ``cupy.cuda.compiler.CompileException``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If CuPy raises a ``CompileException`` for almost everything, it is possible that CuPy cannot detect CUDA installed on your system correctly.
+The followings are error messages commonly observed in such cases.
+
+* ``nvrtc: error: failed to load builtins``
+* ``catastrophic error: cannot open source file "cuda_fp16.h"``
+* ``error: cannot overload functions distinguished by return type alone``
+* ``error: identifier "__half_raw" is undefined``
+
+Please try setting ``LD_LIBRARY_PATH`` and ``CUDA_PATH`` environment variable.
+For example, if you have CUDA installed at ``/usr/local/cuda-9.2``::
+
+  $ export CUDA_PATH=/usr/local/cuda-9.2
+  $ export LD_LIBRARY_PATH=$CUDA_PATH/lib64:$LD_LIBRARY_PATH
+
+Also see :ref:`install_cuda`.
+
+.. _install_gcc6:
+
+Build fails on Ubuntu 16.04, CentOS 6 or 7
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to build CuPy from source on systems with legacy GCC (g++-5 or earlier), you need to manually set up g++-6 or later and configure ``NVCC`` environment variable.
+
+On Ubuntu 16.04::
+
+  $ sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+  $ sudo apt update
+  $ sudo apt install g++-6
+  $ export NVCC="nvcc --compiler-bindir gcc-6"
+
+On CentOS 6 / 7::
+
+  $ sudo yum install centos-release-scl
+  $ sudo yum install devtoolset-7-gcc-c++
+  $ source /opt/rh/devtoolset-7/enable
+  $ export NVCC="nvcc --compiler-bindir gcc"
+
+
+Using CuPy on AMD GPU (experimental)
+====================================
+
+CuPy has an experimental support for AMD GPU (ROCm).
+
+Requirements
+------------
+
+* `AMD GPU supported by ROCm <https://github.com/RadeonOpenCompute/ROCm#Hardware-and-Software-Support>`_
+
+* `ROCm <https://rocmdocs.amd.com/en/latest/index.html>`_: v4.3 / v5.0
+    * See the `ROCm Installation Guide <https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html>`_ for details.
+
+The following ROCm libraries are required:
+
+::
+
+  $ sudo apt install hipblas hipsparse rocsparse rocrand rocthrust rocsolver rocfft hipcub rocprim rccl
+
+Environment Variables
+---------------------
+
+When building or running CuPy for ROCm, the following environment variables are effective.
+
+* ``ROCM_HOME``: directory containing the ROCm software (e.g., ``/opt/rocm``).
+
+Docker
+------
+
+You can try running CuPy for ROCm using Docker.
+
+::
+
+  $ docker run -it --device=/dev/kfd --device=/dev/dri --group-add video cupy/cupy-rocm
+
+.. _install_hip:
+
+Installing Binary Packages
+--------------------------
+
+Wheels (precompiled binary packages) are available for Linux (x86_64).
+Package names are different depending on your ROCm version.
+
+.. list-table::
+   :header-rows: 1
+
+   * - ROCm
+     - Command
+   * - v4.3
+     - ``$ pip install cupy-rocm-4-3``
+   * - v5.0
+     - ``$ pip install cupy-rocm-5-0``
+
+Building CuPy for ROCm From Source
+----------------------------------
+
+To build CuPy from source, set the ``CUPY_INSTALL_USE_HIP``, ``ROCM_HOME``, and ``HCC_AMDGPU_TARGET`` environment variables.
+(``HCC_AMDGPU_TARGET`` is the ISA name supported by your GPU.
+Run ``rocminfo`` and use the value displayed in ``Name:`` line (e.g., ``gfx900``).
+You can specify a comma-separated list of ISAs if you have multiple GPUs of different architectures.)
+
+::
+
+  $ export CUPY_INSTALL_USE_HIP=1
+  $ export ROCM_HOME=/opt/rocm
+  $ export HCC_AMDGPU_TARGET=gfx906
+  $ pip install cupy
+
+.. note::
+
+  If you don't specify the ``HCC_AMDGPU_TARGET`` environment variable, CuPy will be built for the GPU architectures available on the build host.
+  This behavior is specific to ROCm builds; when building CuPy for NVIDIA CUDA, the build result is not affected by the host configuration.
+
+Limitations
+-----------
+
+The following features are not available due to the limitation of ROCm or because that they are specific to CUDA:
+
+* CUDA Array Interface
+* cuTENSOR
+* Handling extremely large arrays whose size is around 32-bit boundary (HIP is known to fail with sizes `2**32-1024`)
+* Atomic addition in FP16 (``cupy.ndarray.scatter_add`` and ``cupyx.scatter_add``)
+* Multi-GPU FFT and FFT callback
+* Some random number generation algorithms
+* Several options in RawKernel/RawModule APIs: Jitify, dynamic parallelism
+* Per-thread default stream
+
+The following features are not yet supported:
+
+* Sparse matrices (``cupyx.scipy.sparse``)
+* cuDNN (hipDNN)
+* Hermitian/symmetric eigenvalue solver (``cupy.linalg.eigh``)
+* Polynomial roots (uses Hermitian/symmetric eigenvalue solver)
+* Splines in ``cupyx.scipy.interpolate`` (``make_interp_spline``, spline modes of ``RegularGridInterpolator``/``interpn``), as they depend on sparse matrices.
+
+The following features may not work in edge cases (e.g., some combinations of dtype):
+
+.. note::
+   We are investigating the root causes of the issues. They are not necessarily
+   CuPy's issues, but ROCm may have some potential bugs.
+
+* ``cupy.ndarray.__getitem__`` (`#4653 <https://github.com/cupy/cupy/pull/4653>`_)
+* ``cupy.ix_`` (`#4654 <https://github.com/cupy/cupy/pull/4654>`_)
+* Some polynomial routines (`#4758 <https://github.com/cupy/cupy/pull/4758>`_, `#4759 <https://github.com/cupy/cupy/pull/4759>`_)
+* ``cupy.broadcast`` (`#4662 <https://github.com/cupy/cupy/pull/4662>`_)
+* ``cupy.convolve`` (`#4668 <https://github.com/cupy/cupy/pull/4668>`_)
+* ``cupy.correlate`` (`#4781 <https://github.com/cupy/cupy/pull/4781>`_)
+* Some random sampling routines (``cupy.random``, `#4770 <https://github.com/cupy/cupy/pull/4770>`_)
+* ``cupy.linalg.einsum``
+* ``cupyx.scipy.ndimage`` and ``cupyx.scipy.signal`` (`#4878 <https://github.com/cupy/cupy/pull/4878>`_, `#4879 <https://github.com/cupy/cupy/pull/4879>`_, `#4880 <https://github.com/cupy/cupy/pull/4880>`_)
diff --git a/docs/source/license.rst b/docs/source/license.rst
new file mode 100644
index 0000000..a9c564f
--- /dev/null
+++ b/docs/source/license.rst
@@ -0,0 +1,107 @@
+License
+=======
+
+Copyright (c) 2015 Preferred Infrastructure, Inc.
+
+Copyright (c) 2015 Preferred Networks, Inc.
+
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
+NumPy
+-----
+
+The CuPy is designed based on NumPy's API.
+CuPy's source code and documents contain the original NumPy ones.
+
+
+Copyright (c) 2005-2016, NumPy Developers.
+
+All rights reserved.
+
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following
+  disclaimer in the documentation and/or other materials provided
+  with the distribution.
+* Neither the name of the NumPy Developers nor the names of any
+  contributors may be used to endorse or promote products derived
+  from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+SciPy
+-----
+
+The CuPy is designed based on SciPy's API.
+CuPy's source code and documents contain the original SciPy ones.
+
+
+Copyright (c) 2001, 2002 Enthought, Inc.
+
+All rights reserved.
+
+
+Copyright (c) 2003-2016 SciPy Developers.
+
+All rights reserved.
+
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+a. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+b. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+c. Neither the name of Enthought nor the names of the SciPy Developers
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/source/overview.rst b/docs/source/overview.rst
new file mode 100644
index 0000000..f344eee
--- /dev/null
+++ b/docs/source/overview.rst
@@ -0,0 +1,68 @@
+.. _overview:
+
+Overview
+========
+
+`CuPy <https://github.com/cupy/cupy>`__ is a NumPy/SciPy-compatible array library for GPU-accelerated computing with Python.
+CuPy acts as a drop-in replacement to run existing NumPy/SciPy code on `NVIDIA CUDA <https://developer.nvidia.com/cuda-toolkit>`__ or `AMD ROCm <https://www.amd.com/en/graphics/servers-solutions-rocm>`__ platforms.
+
+CuPy provides a ``ndarray``, sparse matrices, and the associated routines for GPU devices, all having the same API as NumPy and SciPy:
+
+* **N-dimensional array** (``ndarray``): :doc:`cupy.ndarray <reference/ndarray>`
+
+  * Data types (dtypes): boolean (``bool_``), integer (``int8``, ``int16``, ``int32``, ``int64``, ``uint8``, ``uint16``, ``uint32``, ``uint64``), float (``float16``, ``float32``, ``float64``), and complex (``complex64``, ``complex128``)
+  * Supports the semantics identical to :class:`numpy.ndarray`, including basic / advanced indexing and broadcasting
+
+* **Sparse matrices**: :doc:`cupyx.scipy.sparse <reference/scipy_sparse>`
+
+  * 2-D sparse matrix: ``csr_matrix``, ``coo_matrix``, ``csc_matrix``, and ``dia_matrix``
+
+* **NumPy Routines**
+
+  * :doc:`Module-level Functions <reference/routines>` (``cupy.*``)
+  * :doc:`Linear Algebra Functions <reference/linalg>` (``cupy.linalg.*``)
+  * :doc:`Fast Fourier Transform <reference/fft>` (``cupy.fft.*``)
+  * :doc:`Random Number Generator <reference/random>` (``cupy.random.*``)
+
+* **SciPy Routines**
+
+  * :doc:`Discrete Fourier Transforms <reference/scipy_fft>` (``cupyx.scipy.fft.*`` and ``cupyx.scipy.fftpack.*``)
+  * :doc:`Advanced Linear Algebra <reference/scipy_linalg>` (``cupyx.scipy.linalg.*``)
+  * :doc:`Multidimensional Image Processing <reference/scipy_ndimage>` (``cupyx.scipy.ndimage.*``)
+  * :doc:`Sparse Matrices <reference/scipy_sparse>` (``cupyx.scipy.sparse.*``)
+  * :doc:`Sparse Linear Algebra <reference/scipy_sparse_linalg>` (``cupyx.scipy.sparse.linalg.*``)
+  * :doc:`Special Functions <reference/scipy_special>` (``cupyx.scipy.special.*``)
+  * :doc:`Signal Processing <reference/scipy_signal>` (``cupyx.scipy.signal.*``)
+  * :doc:`Statistical Functions <reference/scipy_stats>` (``cupyx.scipy.stats.*``)
+
+Routines are backed by CUDA libraries (cuBLAS, cuFFT, cuSPARSE, cuSOLVER, cuRAND), Thrust, CUB, and cuTENSOR to provide the best performance.
+
+It is also possible to easily implement :doc:`custom CUDA kernels <user_guide/kernel>` that work with ``ndarray`` using:
+
+* **Kernel Templates**: Quickly define element-wise and reduction operation as a single CUDA kernel
+* **Raw Kernel**: Import existing CUDA C/C++ code
+* **Just-in-time Transpiler (JIT)**: Generate CUDA kernel from Python source code
+* **Kernel Fusion**: Fuse multiple CuPy operations into a single CUDA kernel
+
+CuPy can run in multi-GPU or cluster environments. The distributed communication package (:mod:`cupyx.distributed`) provides collective and peer-to-peer primitives for ``ndarray``, backed by NCCL.
+
+For users who need more fine-grain control for performance, accessing :doc:`low-level CUDA features <user_guide/cuda_api>` are available:
+
+* **Stream and Event**: CUDA stream and per-thread default stream are supported by all APIs
+* **Memory Pool**: Customizable memory allocator with a built-in memory pool
+* **Profiler**: Supports profiling code using CUDA Profiler and NVTX
+* **Host API Binding**: Directly call CUDA libraries, such as NCCL, cuDNN, cuTENSOR, and cuSPARSELt APIs from Python
+
+CuPy implements standard APIs for data exchange and interoperability, such as `DLPack <https://github.com/dmlc/dlpack>`__, `CUDA Array Interface <https://numba.readthedocs.io/en/stable/cuda/cuda_array_interface.html>`__, ``__array_ufunc__`` (`NEP 13 <https://numpy.org/neps/nep-0013-ufunc-overrides.html>`__), ``__array_function__`` (`NEP 18 <https://numpy.org/neps/nep-0018-array-function-protocol.html>`__), and `Array API Standard <https://data-apis.org/array-api/latest/>`__.
+Thanks to these protocols, CuPy easily :doc:`integrates <user_guide/interoperability>` with NumPy, PyTorch, TensorFlow, MPI4Py, and any other libraries supporting the standard.
+
+Under AMD ROCm environment, CuPy automatically translates all CUDA API calls to ROCm HIP (hipBLAS, hipFFT, hipSPARSE, hipRAND, hipCUB, hipThrust, RCCL, etc.), allowing code written using CuPy to run on both NVIDIA and AMD GPU without any modification.
+
+Project Goal
+------------
+
+The goal of the CuPy project is to provide Python users GPU acceleration capabilities, without the in-depth knowledge of underlying GPU technologies.
+The CuPy team focuses on providing:
+
+* A complete NumPy and SciPy API coverage to become a full drop-in replacement, as well as advanced CUDA features to maximize the performance.
+* Mature and quality library as a fundamental package for all projects needing acceleration, from a lab environment to a large-scale cluster.
diff --git a/docs/source/reference/_deprecated.rst b/docs/source/reference/_deprecated.rst
new file mode 100644
index 0000000..f004f47
--- /dev/null
+++ b/docs/source/reference/_deprecated.rst
@@ -0,0 +1,42 @@
+:orphan:
+
+.. This page is to generate documentation for deprecated APIs removed from the
+   public table of contents.
+
+DLPack helper
+-------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.fromDlpack
+
+Time range
+----------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.prof.TimeRangeDecorator
+   cupy.prof.time_range
+
+Timing helper
+-------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.time.repeat
+
+Device synchronization detection
+--------------------------------
+
+.. warning::
+
+   These APIs are deprecated in CuPy v10 and will be removed in future releases.
+
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.allow_synchronize
+   cupyx.DeviceSynchronized
diff --git a/docs/source/reference/_private.rst b/docs/source/reference/_private.rst
new file mode 100644
index 0000000..654c9d9
--- /dev/null
+++ b/docs/source/reference/_private.rst
@@ -0,0 +1,31 @@
+:orphan:
+
+.. This page is to generate documentation for private classes exposed to users,
+   i.e., users cannot instantiate it by themselves but may use it's properties
+   or methods via returned values from CuPy methods.
+   These classes must be referred in public APIs returning their instances.
+
+Benchmark Data
+--------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.profiler._time._PerfCaseResult
+
+cuFFT Plan Cache
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.fft._cache.PlanCache
+
+JIT Cooperative Groups
+----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.jit.cg._ThreadBlockGroup
+   cupyx.jit.cg._GridGroup
diff --git a/docs/source/reference/array_api.rst b/docs/source/reference/array_api.rst
new file mode 100644
index 0000000..ec5d137
--- /dev/null
+++ b/docs/source/reference/array_api.rst
@@ -0,0 +1,26 @@
+Python Array API Support
+========================
+
+The `Python array API standard <https://data-apis.org/array-api/2021.12/>`_ aims to provide a coherent set of
+APIs for array and tensor libraries developed by the community to build upon. This solves the API fragmentation
+issue across the community by offering concrete function signatures, semantics and scopes of coverage, enabling
+writing backend-agnostic codes for better portability.
+
+CuPy provides **experimental** support based on NumPy's `NEP-47 <https://numpy.org/neps/nep-0047-array-api-standard.html>`_,
+which is in turn based on the v2021 standard. All of the functionalities can be accessed
+through the :mod:`cupy.array_api` namespace.
+
+NumPy's `Array API Standard Compatibility <https://numpy.org/devdocs/reference/array_api.html>`_ is an excellent starting
+point to understand better the differences between the APIs under the main namespace and the :mod:`~cupy.array_api` namespace.
+Keep in mind, however, that the key difference between NumPy and CuPy is that we are a GPU-only library, therefore CuPy users should be aware
+of potential `device management <https://data-apis.org/array-api/latest/design_topics/device_support.html>`_ issues.
+Same as in regular CuPy codes, the GPU-to-use can be specified via the :class:`~cupy.cuda.Device` objects, see
+:ref:`device_management`. GPU-related semantics (e.g. streams, asynchronicity, etc) are also respected.
+Finally, remember there are already :doc:`differences between NumPy and CuPy <../user_guide/difference>`,
+although some of which are rectified in the standard.
+
+.. toctree::
+   :maxdepth: 2
+
+   array_api_functions
+   array_api_array
diff --git a/docs/source/reference/array_api_array.rst b/docs/source/reference/array_api_array.rst
new file mode 100644
index 0000000..aad722a
--- /dev/null
+++ b/docs/source/reference/array_api_array.rst
@@ -0,0 +1,18 @@
+Array API Compliant Object
+==========================
+
+:class:`~cupy.array_api._array_object.Array` is a wrapper class built upon :class:`cupy.ndarray`
+to enforce strict compliance with the array API standard. See the
+`documentation <https://data-apis.org/array-api/latest/API_specification/array_object.html>`_
+for detail.
+
+This object should not be constructed directly. Rather, use one of the
+`creation functions <https://data-apis.org/array-api/latest/API_specification/creation_functions.html>`_,
+such as :func:`cupy.array_api.asarray`.
+
+.. currentmodule:: cupy.array_api._array_object
+
+.. autosummary::
+   :toctree: generated/
+
+   Array
diff --git a/docs/source/reference/array_api_functions.rst b/docs/source/reference/array_api_functions.rst
new file mode 100644
index 0000000..7186be0
--- /dev/null
+++ b/docs/source/reference/array_api_functions.rst
@@ -0,0 +1,8 @@
+Array API Functions
+===================
+
+This section is a full list of implemented APIs. For the detailed documentation, see the
+`array API specification <https://data-apis.org/array-api/latest/API_specification/index.html>`_.
+
+.. automodule:: cupy.array_api
+   :members:
diff --git a/docs/source/reference/binary.rst b/docs/source/reference/binary.rst
new file mode 100644
index 0000000..6655d8d
--- /dev/null
+++ b/docs/source/reference/binary.rst
@@ -0,0 +1,38 @@
+Binary operations
+=================
+
+.. Hint:: `NumPy API Reference: Binary operations <https://numpy.org/doc/stable/reference/routines.bitwise.html>`_
+
+.. currentmodule:: cupy
+
+Elementwise bit operations
+--------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   bitwise_and
+   bitwise_or
+   bitwise_xor
+   invert
+   left_shift
+   right_shift
+
+
+Bit packing
+-----------
+
+.. autosummary::
+   :toctree: generated/
+
+   packbits
+   unpackbits
+
+
+Output formatting
+-----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   binary_repr
diff --git a/docs/source/reference/comparison.rst b/docs/source/reference/comparison.rst
new file mode 100644
index 0000000..3e4b49a
--- /dev/null
+++ b/docs/source/reference/comparison.rst
@@ -0,0 +1,9 @@
+Comparison Table
+================
+
+Here is a list of NumPy / SciPy APIs and its corresponding CuPy implementations.
+
+``-`` in CuPy column denotes that CuPy implementation is not provided yet.
+We welcome contributions for these functions.
+
+.. include:: comparison_table.rst.inc
diff --git a/docs/source/reference/creation.rst b/docs/source/reference/creation.rst
new file mode 100644
index 0000000..f5a9ff5
--- /dev/null
+++ b/docs/source/reference/creation.rst
@@ -0,0 +1,70 @@
+Array creation routines
+=======================
+
+.. Hint:: `NumPy API Reference: Array creation routines <https://numpy.org/doc/stable/reference/routines.array-creation.html>`_
+
+.. currentmodule:: cupy
+
+Ones and zeros
+--------------
+
+.. autosummary::
+   :toctree: generated/
+
+   empty
+   empty_like
+   eye
+   identity
+   ones
+   ones_like
+   zeros
+   zeros_like
+   full
+   full_like
+
+
+From existing data
+------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   array
+   asarray
+   asanyarray
+   ascontiguousarray
+   copy
+   frombuffer
+   fromfile
+   fromfunction
+   fromiter
+   fromstring
+   loadtxt
+
+
+Numerical ranges
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   arange
+   linspace
+   logspace
+   meshgrid
+   mgrid
+   ogrid
+
+
+Building matrices
+-----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   diag
+   diagflat
+   tri
+   tril
+   triu
+   vander
diff --git a/docs/source/reference/cuda.rst b/docs/source/reference/cuda.rst
new file mode 100644
index 0000000..33a64be
--- /dev/null
+++ b/docs/source/reference/cuda.rst
@@ -0,0 +1,209 @@
+Low-level CUDA support
+======================
+
+.. _device_management:
+
+Device management
+-----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.cuda.Device
+
+
+Memory management
+-----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.get_default_memory_pool
+   cupy.get_default_pinned_memory_pool
+   cupy.cuda.Memory
+   cupy.cuda.MemoryAsync
+   cupy.cuda.ManagedMemory
+   cupy.cuda.UnownedMemory
+   cupy.cuda.PinnedMemory
+   cupy.cuda.MemoryPointer
+   cupy.cuda.PinnedMemoryPointer
+   cupy.cuda.malloc_managed
+   cupy.cuda.malloc_async
+   cupy.cuda.alloc
+   cupy.cuda.alloc_pinned_memory
+   cupy.cuda.get_allocator
+   cupy.cuda.set_allocator
+   cupy.cuda.using_allocator
+   cupy.cuda.set_pinned_memory_allocator
+   cupy.cuda.MemoryPool
+   cupy.cuda.MemoryAsyncPool
+   cupy.cuda.PinnedMemoryPool
+   cupy.cuda.PythonFunctionAllocator
+   cupy.cuda.CFunctionAllocator
+
+
+Memory hook
+-----------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.cuda.MemoryHook
+   cupy.cuda.memory_hooks.DebugPrintHook
+   cupy.cuda.memory_hooks.LineProfileHook
+
+
+.. _stream_event_api:
+
+Streams and events
+------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.cuda.Stream
+   cupy.cuda.ExternalStream
+   cupy.cuda.get_current_stream
+   cupy.cuda.Event
+   cupy.cuda.get_elapsed_time
+
+
+.. _graph_api:
+
+Graphs
+------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.cuda.Graph
+
+
+Texture and surface memory
+--------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.cuda.texture.ChannelFormatDescriptor
+   cupy.cuda.texture.CUDAarray
+   cupy.cuda.texture.ResourceDescriptor
+   cupy.cuda.texture.TextureDescriptor
+   cupy.cuda.texture.TextureObject
+   cupy.cuda.texture.SurfaceObject
+
+
+Profiler
+--------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.cuda.profile
+   cupy.cuda.profiler.initialize
+   cupy.cuda.profiler.start
+   cupy.cuda.profiler.stop
+   cupy.cuda.nvtx.Mark
+   cupy.cuda.nvtx.MarkC
+   cupy.cuda.nvtx.RangePush
+   cupy.cuda.nvtx.RangePushC
+   cupy.cuda.nvtx.RangePop
+
+
+NCCL
+----
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.cuda.nccl.NcclCommunicator
+   cupy.cuda.nccl.get_build_version
+   cupy.cuda.nccl.get_version
+   cupy.cuda.nccl.get_unique_id
+   cupy.cuda.nccl.groupStart
+   cupy.cuda.nccl.groupEnd
+
+
+.. _runtime_api:
+
+Runtime API
+-----------
+
+CuPy wraps CUDA Runtime APIs to provide the native CUDA operations.
+Please check the `CUDA Runtime API documentation <https://docs.nvidia.com/cuda/cuda-runtime-api/index.html>`_
+to use these functions.
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.cuda.runtime.driverGetVersion
+   cupy.cuda.runtime.runtimeGetVersion
+   cupy.cuda.runtime.getDevice
+   cupy.cuda.runtime.getDeviceProperties
+   cupy.cuda.runtime.deviceGetAttribute
+   cupy.cuda.runtime.deviceGetByPCIBusId
+   cupy.cuda.runtime.deviceGetPCIBusId
+   cupy.cuda.runtime.deviceGetDefaultMemPool
+   cupy.cuda.runtime.deviceGetMemPool
+   cupy.cuda.runtime.deviceSetMemPool
+   cupy.cuda.runtime.memPoolCreate
+   cupy.cuda.runtime.memPoolDestroy
+   cupy.cuda.runtime.memPoolTrimTo
+   cupy.cuda.runtime.getDeviceCount
+   cupy.cuda.runtime.setDevice
+   cupy.cuda.runtime.deviceSynchronize
+   cupy.cuda.runtime.deviceCanAccessPeer
+   cupy.cuda.runtime.deviceEnablePeerAccess
+   cupy.cuda.runtime.deviceGetLimit
+   cupy.cuda.runtime.deviceSetLimit
+   cupy.cuda.runtime.malloc
+   cupy.cuda.runtime.mallocManaged
+   cupy.cuda.runtime.malloc3DArray
+   cupy.cuda.runtime.mallocArray
+   cupy.cuda.runtime.mallocAsync
+   cupy.cuda.runtime.mallocFromPoolAsync
+   cupy.cuda.runtime.hostAlloc
+   cupy.cuda.runtime.hostRegister
+   cupy.cuda.runtime.hostUnregister
+   cupy.cuda.runtime.free
+   cupy.cuda.runtime.freeHost
+   cupy.cuda.runtime.freeArray
+   cupy.cuda.runtime.freeAsync
+   cupy.cuda.runtime.memGetInfo
+   cupy.cuda.runtime.memcpy
+   cupy.cuda.runtime.memcpyAsync
+   cupy.cuda.runtime.memcpyPeer
+   cupy.cuda.runtime.memcpyPeerAsync
+   cupy.cuda.runtime.memcpy2D
+   cupy.cuda.runtime.memcpy2DAsync
+   cupy.cuda.runtime.memcpy2DFromArray
+   cupy.cuda.runtime.memcpy2DFromArrayAsync
+   cupy.cuda.runtime.memcpy2DToArray
+   cupy.cuda.runtime.memcpy2DToArrayAsync
+   cupy.cuda.runtime.memcpy3D
+   cupy.cuda.runtime.memcpy3DAsync
+   cupy.cuda.runtime.memset
+   cupy.cuda.runtime.memsetAsync
+   cupy.cuda.runtime.memPrefetchAsync
+   cupy.cuda.runtime.memAdvise
+   cupy.cuda.runtime.pointerGetAttributes
+   cupy.cuda.runtime.streamCreate
+   cupy.cuda.runtime.streamCreateWithFlags
+   cupy.cuda.runtime.streamDestroy
+   cupy.cuda.runtime.streamSynchronize
+   cupy.cuda.runtime.streamAddCallback
+   cupy.cuda.runtime.streamQuery
+   cupy.cuda.runtime.streamWaitEvent
+   cupy.cuda.runtime.launchHostFunc
+   cupy.cuda.runtime.eventCreate
+   cupy.cuda.runtime.eventCreateWithFlags
+   cupy.cuda.runtime.eventDestroy
+   cupy.cuda.runtime.eventElapsedTime
+   cupy.cuda.runtime.eventQuery
+   cupy.cuda.runtime.eventRecord
+   cupy.cuda.runtime.eventSynchronize
+   cupy.cuda.runtime.ipcGetMemHandle
+   cupy.cuda.runtime.ipcOpenMemHandle
+   cupy.cuda.runtime.ipcCloseMemHandle
+   cupy.cuda.runtime.ipcGetEventHandle
+   cupy.cuda.runtime.ipcOpenEventHandle
diff --git a/docs/source/reference/distributed.rst b/docs/source/reference/distributed.rst
new file mode 100644
index 0000000..6daefce
--- /dev/null
+++ b/docs/source/reference/distributed.rst
@@ -0,0 +1,15 @@
+----------------
+Distributed
+----------------
+
+The following pages describe the APIs used to easily perform communication
+between different processes in CuPy.
+
+
+.. module:: cupyx.distributed
+
+.. autosummary::
+   :toctree: generated/
+
+   init_process_group
+   NCCLBackend
diff --git a/docs/source/reference/dtype.rst b/docs/source/reference/dtype.rst
new file mode 100644
index 0000000..a5c35f7
--- /dev/null
+++ b/docs/source/reference/dtype.rst
@@ -0,0 +1,61 @@
+Data type routines
+==================
+
+.. Hint:: `NumPy API Reference: Data type routines <https://numpy.org/doc/stable/reference/routines.dtype.html>`_
+
+.. currentmodule:: cupy
+
+.. autosummary::
+   :toctree: generated/
+
+   can_cast
+   min_scalar_type
+   result_type
+   common_type
+
+.. csv-table::
+   :align: left
+
+   ``promote_types`` (alias of :func:`numpy.promote_types`)
+   ``obj2sctype`` (alias of :func:`numpy.obj2sctype`)
+
+Creating data types
+-------------------
+
+.. csv-table::
+   :align: left
+
+   ``dtype`` (alias of :class:`numpy.dtype`)
+   ``format_parser`` (alias of :class:`numpy.format_parser`)
+
+Data type information
+---------------------
+
+.. csv-table::
+   :align: left
+
+   ``finfo`` (alias of :class:`numpy.finfo`)
+   ``iinfo`` (alias of :class:`numpy.iinfo`)
+   ``MachAr`` (alias of :class:`numpy.MachAr`)
+
+Data type testing
+-----------------
+
+.. csv-table::
+   :align: left
+
+   ``issctype`` (alias of :func:`numpy.issctype`)
+   ``issubdtype`` (alias of :func:`numpy.issubdtype`)
+   ``issubsctype`` (alias of :func:`numpy.issubsctype`)
+   ``issubclass_`` (alias of :func:`numpy.issubclass_`)
+   ``find_common_type`` (alias of :func:`numpy.find_common_type`)
+
+Miscellaneous
+-------------
+
+.. csv-table::
+   :align: left
+
+   ``typename`` (alias of :func:`numpy.typename`)
+   ``sctype2char`` (alias of :func:`numpy.sctype2char`)
+   ``mintypecode`` (alias of :func:`numpy.mintypecode`)
diff --git a/docs/source/reference/environment.rst b/docs/source/reference/environment.rst
new file mode 100644
index 0000000..82661c2
--- /dev/null
+++ b/docs/source/reference/environment.rst
@@ -0,0 +1,187 @@
+.. _environment:
+
+Environment variables
+=====================
+
+For runtime
+-----------
+
+Here are the environment variables that CuPy uses at runtime.
+
+.. envvar:: CUDA_PATH
+
+  Path to the directory containing CUDA.
+  The parent of the directory containing ``nvcc`` is used as default.
+  When ``nvcc`` is not found, ``/usr/local/cuda`` is used.
+  See :ref:`install_cuda` for details.
+
+.. envvar:: CUPY_CACHE_DIR
+
+  Default: ``${HOME}/.cupy/kernel_cache``
+
+  Path to the directory to store kernel cache.
+  See :doc:`../user_guide/performance` for details.
+
+.. envvar:: CUPY_CACHE_SAVE_CUDA_SOURCE
+
+  Default: ``0``
+
+  If set to ``1``, CUDA source file will be saved along with compiled binary in the cache directory for debug purpose.
+  Note: the source file will not be saved if the compiled binary is already stored in the cache.
+
+.. envvar:: CUPY_CACHE_IN_MEMORY
+
+  Default: ``0``
+
+  If set to ``1``, :envvar:`CUPY_CACHE_DIR` and :envvar:`CUPY_CACHE_SAVE_CUDA_SOURCE` will be ignored, and the cache is in memory.
+  This environment variable allows reducing disk I/O, but is ignoed when ``nvcc`` is set to be the compiler backend.
+
+.. envvar:: CUPY_DUMP_CUDA_SOURCE_ON_ERROR
+
+  Default: ``0``
+
+  If set to ``1``, when CUDA kernel compilation fails,
+  CuPy dumps CUDA kernel code to standard error.
+
+.. envvar:: CUPY_CUDA_COMPILE_WITH_DEBUG
+
+  Default: ``0``
+
+  If set to ``1``, CUDA kernel will be compiled with debug information (``--device-debug`` and ``--generate-line-info``).
+
+.. envvar:: CUPY_GPU_MEMORY_LIMIT
+
+  Default: ``0`` (unlimited)
+
+  The amount of memory that can be allocated for each device.
+  The value can be specified in absolute bytes or fraction (e.g., ``"90%"``) of the total memory of each GPU.
+  See :doc:`../user_guide/memory` for details.
+
+.. envvar:: CUPY_SEED
+
+  Set the seed for random number generators.
+
+.. envvar:: CUPY_EXPERIMENTAL_SLICE_COPY
+
+  Default: ``0``
+  
+  If set to ``1``, the following syntax is enabled::
+
+    cupy_ndarray[:] = numpy_ndarray
+
+.. envvar:: CUPY_ACCELERATORS
+
+  Default: ``"cub"`` (In ROCm HIP environment, the default value is ``""``. i.e., no accelerators are used.)
+
+  A comma-separated string of backend names (``cub``, ``cutensor``, or ``cutensornet``) which indicates the acceleration backends used in CuPy operations and its priority (in descending order).
+  By default, all accelerators are disabled on HIP and only CUB is enabled on CUDA.
+
+.. envvar:: CUPY_TF32
+
+  Default: ``0``
+
+  If set to ``1``, it allows CUDA libraries to use Tensor Cores TF32 compute for 32-bit floating point compute.
+
+.. envvar:: CUPY_CUDA_ARRAY_INTERFACE_SYNC
+
+  Default: ``1``
+
+  This controls CuPy's behavior as a Consumer.
+  If set to ``0``, a stream synchronization will *not* be performed when a device array provided by an external library that implements the CUDA Array Interface is being consumed by CuPy.
+  For more detail, see the `Synchronization`_ requirement in the CUDA Array Interface v3 documentation.
+
+.. envvar:: CUPY_CUDA_ARRAY_INTERFACE_EXPORT_VERSION
+
+  Default: ``3``
+
+  This controls CuPy's behavior as a Producer.
+  If set to ``2``, the CuPy stream on which the data is being operated will not be exported and thus the Consumer (another library) will not perform any stream synchronization.
+  For more detail, see the `Synchronization`_ requirement in the CUDA Array Interface v3 documentation.
+
+.. envvar:: CUPY_DLPACK_EXPORT_VERSION
+
+  Default: ``0.6``
+
+  This controls CuPy's DLPack support. Currently, setting a value smaller than 0.6 would disguise managed memory as normal device memory, which enables data exchanges with libraries that have not updated their DLPack support, whereas starting 0.6 CUDA managed memory can be correctly recognized as a valid device type.
+
+.. envvar:: NVCC
+
+  Default: ``nvcc``
+
+  Define the compiler to use when compiling CUDA source.
+  Note that most CuPy kernels are built with NVRTC; this environment variable is only effective for :class:`~cupy.RawKernel`/:class:`~cupy.RawModule` with the ``nvcc`` backend or when using ``cub`` as the accelerator.
+
+.. envvar:: CUPY_CUDA_PER_THREAD_DEFAULT_STREAM
+
+  Default: ``0``
+
+  If set to ``1``, CuPy will use the CUDA per-thread default stream, effectively causing each host thread to automatically execute in its own stream, unless the CUDA default (``null``) stream or a user-created stream is specified.
+  If set to ``0`` (default), the CUDA default (``null``) stream is used, unless the per-thread default stream (``ptds``) or a user-created stream is specified.
+
+.. envvar:: CUPY_COMPILE_WITH_PTX
+
+  Default: ``0``
+
+  By default, CuPy directly compiles kernels into SASS (CUBIN) to support `CUDA Enhanced Compatibility <https://docs.nvidia.com/deploy/cuda-compatibility/>`_
+  If set to ``1``, CuPy instead compiles kernels into PTX and lets CUDA Driver assemble SASS from PTX.
+  This option is only effective for CUDA 11.1 or later; CuPy always compiles into PTX on earlier CUDA versions. Also, this option only applies when NVRTC is selected as the compilation backend. NVCC backend always compiles into SASS (CUBIN).
+
+CUDA Toolkit Environment Variables
+  In addition to the environment variables listed above, as in any CUDA programs, all of the CUDA environment variables listed in the `CUDA Toolkit Documentation`_ will also be honored.
+
+.. note::
+
+  When :envvar:`CUPY_ACCELERATORS` or :envvar:`NVCC` environment variables are set, g++-6 or later is required as the runtime host compiler.
+  Please refer to :ref:`install_cupy_from_source` for the details on how to install g++.
+
+.. _CUDA Toolkit Documentation: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars
+
+.. _Synchronization: https://numba.readthedocs.io/en/latest/cuda/cuda_array_interface.html#synchronization
+
+
+For installation
+----------------
+
+These environment variables are used during installation (building CuPy from source).
+
+.. envvar:: CUTENSOR_PATH
+
+  Path to the cuTENSOR root directory that contains ``lib`` and ``include`` directories. (experimental)
+
+.. envvar:: CUPY_INSTALL_USE_HIP
+
+  Default: ``0``
+
+  If set to ``1``, CuPy is built for AMD ROCm Platform (experimental).
+  For building the ROCm support, see :ref:`install_hip` for further detail.
+
+.. envvar:: CUPY_USE_CUDA_PYTHON
+
+  Default: ``0``
+
+  If set to ``1``, CuPy is built using `CUDA Python <https://github.com/NVIDIA/cuda-python>`_.
+
+.. envvar:: CUPY_NVCC_GENERATE_CODE
+
+  Build CuPy for a particular CUDA architecture. For example::
+
+    CUPY_NVCC_GENERATE_CODE="arch=compute_60,code=sm_60"
+
+  For specifying multiple archs, concatenate the ``arch=...`` strings with semicolons (``;``).
+  If ``current`` is specified, then it will automatically detect the currently installed GPU architectures in build time.
+  When this is not set, the default is to support all architectures.
+
+.. envvar:: CUPY_NUM_BUILD_JOBS
+
+  Default: ``4``
+
+  To enable or disable parallel build, sets the number of processes used to build the extensions in parallel.
+
+
+.. envvar:: CUPY_NUM_NVCC_THREADS
+
+  Default: ``2``
+
+  To enable or disable nvcc parallel compilation, sets the number of threads used to compile files using nvcc.
+
+Additionally, the environment variables :envvar:`CUDA_PATH` and :envvar:`NVCC` are also respected at build time.
diff --git a/docs/source/reference/ext.rst b/docs/source/reference/ext.rst
new file mode 100644
index 0000000..3c83928
--- /dev/null
+++ b/docs/source/reference/ext.rst
@@ -0,0 +1,51 @@
+CuPy-specific functions
+=======================
+
+CuPy-specific functions are placed under ``cupyx`` namespace.
+
+.. TODO(kmaehashi): use module:: cupyx
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.rsqrt
+   cupyx.scatter_add
+   cupyx.scatter_max
+   cupyx.scatter_min
+   cupyx.empty_pinned
+   cupyx.empty_like_pinned
+   cupyx.zeros_pinned
+   cupyx.zeros_like_pinned
+
+Profiling utilities
+-------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.profiler.benchmark
+   cupyx.profiler.time_range
+   cupyx.profiler.profile
+
+DLPack utilities
+----------------
+
+Below are helper functions for creating a :class:`cupy.ndarray` from either a DLPack tensor
+or any object supporting the DLPack data exchange protocol.
+For further detail see :ref:`dlpack`.
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.from_dlpack
+
+
+.. _kernel_param_opt:
+
+Automatic Kernel Parameters Optimizations (:mod:`cupyx.optimizing`)
+-------------------------------------------------------------------
+
+.. module:: cupyx.optimizing
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.optimizing.optimize
diff --git a/docs/source/reference/fft.rst b/docs/source/reference/fft.rst
new file mode 100644
index 0000000..ed9a47c
--- /dev/null
+++ b/docs/source/reference/fft.rst
@@ -0,0 +1,97 @@
+.. module:: cupy.fft
+
+Discrete Fourier Transform (:mod:`cupy.fft`)
+============================================
+
+.. Hint:: `NumPy API Reference: Discrete Fourier Transform (numpy.fft) <https://numpy.org/doc/stable/reference/routines.fft.html>`_
+
+.. seealso:: :doc:`scipy_fft`, :doc:`../user_guide/fft`
+
+Standard FFTs
+-------------
+
+.. autosummary::
+   :toctree: generated/
+
+   fft
+   ifft
+   fft2
+   ifft2
+   fftn
+   ifftn
+
+
+Real FFTs
+---------
+
+.. autosummary::
+   :toctree: generated/
+
+   rfft
+   irfft
+   rfft2
+   irfft2
+   rfftn
+   irfftn
+
+
+Hermitian FFTs
+--------------
+
+.. autosummary::
+   :toctree: generated/
+
+   hfft
+   ihfft
+
+
+Helper routines
+---------------
+
+.. autosummary::
+   :toctree: generated/
+
+   fftfreq
+   rfftfreq
+   fftshift
+   ifftshift
+
+CuPy-specific APIs
+------------------
+
+See the description below for details.
+
+.. autosummary::
+   :toctree: generated/
+
+   config.set_cufft_callbacks
+   config.set_cufft_gpus
+   config.get_plan_cache
+   config.show_plan_cache_info
+
+
+Normalization
+-------------
+The default normalization (``norm`` is ``"backward"`` or ``None``) has the direct transforms unscaled and the inverse transforms scaled by :math:`1/n`.
+If the keyword argument ``norm`` is ``"forward"``, it is the exact opposite of ``"backward"``:
+the direct transforms are scaled by :math:`1/n` and the inverse transforms are unscaled.
+Finally, if the keyword argument ``norm`` is ``"ortho"``, both transforms are scaled by :math:`1/\sqrt{n}`.
+
+Code compatibility features
+---------------------------
+FFT functions of NumPy always return numpy.ndarray which type is ``numpy.complex128`` or ``numpy.float64``.
+CuPy functions do not follow the behavior, they will return ``numpy.complex64`` or ``numpy.float32`` if the type of the input is ``numpy.float16``, ``numpy.float32``, or ``numpy.complex64``.
+
+Internally, ``cupy.fft`` always generates a *cuFFT plan* (see the `cuFFT documentation`_ for detail) corresponding to the desired transform. When possible, an n-dimensional plan will be used, as opposed to applying separate 1D plans for each axis to be transformed. Using n-dimensional planning can provide better performance for multidimensional transforms, but requires more GPU memory than separable 1D planning. The user can disable n-dimensional planning by setting ``cupy.fft.config.enable_nd_planning = False``. This ability to adjust the planning type is a deviation from the NumPy API, which does not use precomputed FFT plans.
+
+Moreover, the automatic plan generation can be suppressed by using an existing plan returned by :func:`cupyx.scipy.fftpack.get_fft_plan` as a context manager. This is again a deviation from NumPy.
+
+Finally, when using the high-level NumPy-like FFT APIs as listed above, internally the cuFFT plans are cached for possible reuse. The plan cache can be retrieved by :func:`~cupy.fft.config.get_plan_cache`, and its current status can be queried by :func:`~cupy.fft.config.show_plan_cache_info`. For finer control of the plan cache, see :class:`~cupy.fft._cache.PlanCache`.
+
+
+Multi-GPU FFT
+-------------
+:mod:`cupy.fft` can use multiple GPUs. To enable (disable) this feature, set :data:`cupy.fft.config.use_multi_gpus` to ``True`` (``False``). Next, to set the number of GPUs or the participating GPU IDs, use the function :func:`cupy.fft.config.set_cufft_gpus`. All of the limitations listed in the `cuFFT documentation`_ apply here. In particular, using more than one GPU does not guarantee better performance.
+
+
+.. _cuFFT documentation: https://docs.nvidia.com/cuda/cufft/index.html
diff --git a/docs/source/reference/fftpack.rst b/docs/source/reference/fftpack.rst
new file mode 100644
index 0000000..4333fcd
--- /dev/null
+++ b/docs/source/reference/fftpack.rst
@@ -0,0 +1,3 @@
+:orphan:
+
+This document has been moved to :doc:`scipy_fftpack`.
diff --git a/docs/source/reference/functional.rst b/docs/source/reference/functional.rst
new file mode 100644
index 0000000..c252d01
--- /dev/null
+++ b/docs/source/reference/functional.rst
@@ -0,0 +1,18 @@
+Functional programming
+======================
+
+.. Hint:: `NumPy API Reference: Functional programming <https://numpy.org/doc/stable/reference/routines.functional.html>`_
+
+.. currentmodule:: cupy
+
+.. note::
+
+   :class:`cupy.vectorize` applies JIT compiler to the given Python function.
+   See :ref:`jit_kernel_definition` for details.
+
+.. autosummary::
+   :toctree: generated/
+
+   apply_along_axis
+   vectorize
+   piecewise
diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst
new file mode 100644
index 0000000..85d224d
--- /dev/null
+++ b/docs/source/reference/index.rst
@@ -0,0 +1,33 @@
+.. _cupy_reference:
+
+*************
+API Reference
+*************
+
+* :ref:`genindex`
+* :ref:`modindex`
+
+----
+
+.. currentmodule:: cupy
+
+..
+  For NumPy/SciPy-compatible APIs (ndarray, ufunc, routines, scipy),
+  omit the module name prefix in the API list, following the convension in
+  NumPy/SciPy documentation.
+  For CuPy-specific APIs, use fully-qualified names.
+
+.. toctree::
+   :maxdepth: 2
+
+   ndarray
+   ufunc
+   routines
+   scipy
+   ext
+   cuda
+   kernel
+   distributed
+   environment
+   comparison
+   array_api
diff --git a/docs/source/reference/indexing.rst b/docs/source/reference/indexing.rst
new file mode 100644
index 0000000..db4038a
--- /dev/null
+++ b/docs/source/reference/indexing.rst
@@ -0,0 +1,65 @@
+Indexing routines
+=================
+
+.. Hint:: `NumPy API Reference: Indexing routines <https://numpy.org/doc/stable/reference/routines.indexing.html>`_
+
+.. currentmodule:: cupy
+
+Generating index arrays
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   c_
+   r_
+   nonzero
+   where
+   indices
+   mask_indices
+   tril_indices
+   tril_indices_from
+   triu_indices
+   triu_indices_from
+   ix_
+   ravel_multi_index
+   unravel_index
+   diag_indices
+   diag_indices_from
+
+
+Indexing-like operations
+------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   take
+   take_along_axis
+   choose
+   compress
+   diag
+   diagonal
+   select
+   lib.stride_tricks.as_strided
+
+
+Inserting data into arrays
+--------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   place
+   put
+   putmask
+   fill_diagonal
+
+
+Iterating over arrays
+---------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   flatiter
diff --git a/docs/source/reference/io.rst b/docs/source/reference/io.rst
new file mode 100644
index 0000000..f742193
--- /dev/null
+++ b/docs/source/reference/io.rst
@@ -0,0 +1,51 @@
+Input and output
+================
+
+.. Hint:: `NumPy API Reference: Input and output <https://numpy.org/doc/stable/reference/routines.io.html>`_
+
+.. currentmodule:: cupy
+
+
+NumPy binary files (NPY, NPZ)
+-----------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   load
+   save
+   savez
+   savez_compressed
+
+Text files
+-----------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   loadtxt
+   savetxt
+   genfromtxt
+   fromstring
+
+String formatting
+-----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   array2string
+   array_repr
+   array_str
+   format_float_positional
+   format_float_scientific
+
+
+Base-n representations
+----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   binary_repr
+   base_repr
diff --git a/docs/source/reference/kernel.rst b/docs/source/reference/kernel.rst
new file mode 100644
index 0000000..42083e0
--- /dev/null
+++ b/docs/source/reference/kernel.rst
@@ -0,0 +1,68 @@
+Custom kernels
+==============
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.ElementwiseKernel
+   cupy.ReductionKernel
+   cupy.RawKernel
+   cupy.RawModule
+   cupy.fuse
+
+
+JIT kernel definition
+---------------------
+
+Supported Python built-in functions include: :obj:`range`, :func:`len`, :func:`max`, :func:`min`.
+
+.. note:: If loop unrolling is needed, use :func:`cupyx.jit.range` instead of the built-in :obj:`range`.
+
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.jit.rawkernel
+   cupyx.jit.threadIdx
+   cupyx.jit.blockDim
+   cupyx.jit.blockIdx
+   cupyx.jit.gridDim
+   cupyx.jit.grid
+   cupyx.jit.gridsize
+   cupyx.jit.laneid
+   cupyx.jit.warpsize
+   cupyx.jit.range
+   cupyx.jit.syncthreads
+   cupyx.jit.syncwarp
+   cupyx.jit.shfl_sync
+   cupyx.jit.shfl_up_sync
+   cupyx.jit.shfl_down_sync
+   cupyx.jit.shfl_xor_sync
+   cupyx.jit.shared_memory
+   cupyx.jit.atomic_add
+   cupyx.jit.atomic_sub
+   cupyx.jit.atomic_exch
+   cupyx.jit.atomic_min
+   cupyx.jit.atomic_max
+   cupyx.jit.atomic_inc
+   cupyx.jit.atomic_dec
+   cupyx.jit.atomic_cas
+   cupyx.jit.atomic_and
+   cupyx.jit.atomic_or
+   cupyx.jit.atomic_xor
+   cupyx.jit.cg.this_grid
+   cupyx.jit.cg.this_thread_block
+   cupyx.jit.cg.sync
+   cupyx.jit.cg.memcpy_async
+   cupyx.jit.cg.wait
+   cupyx.jit.cg.wait_prior
+   cupyx.jit._interface._JitRawKernel
+
+
+Kernel binary memoization
+-------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.memoize
+   cupy.clear_memo
diff --git a/docs/source/reference/linalg.rst b/docs/source/reference/linalg.rst
new file mode 100644
index 0000000..da5134c
--- /dev/null
+++ b/docs/source/reference/linalg.rst
@@ -0,0 +1,71 @@
+.. module:: cupy.linalg
+
+Linear algebra (:mod:`cupy.linalg`)
+===================================
+
+.. Hint:: `NumPy API Reference: Linear algebra (numpy.linalg) <https://numpy.org/doc/stable/reference/routines.linalg.html>`_
+
+.. seealso:: :doc:`scipy_linalg`
+
+.. currentmodule:: cupy
+
+Matrix and vector products
+--------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   dot
+   vdot
+   inner
+   outer
+   matmul
+   tensordot
+   einsum
+   linalg.matrix_power
+   kron
+
+Decompositions
+--------------
+
+.. autosummary::
+   :toctree: generated/
+
+   linalg.cholesky
+   linalg.qr
+   linalg.svd
+
+Matrix eigenvalues
+------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   linalg.eigh
+   linalg.eigvalsh
+
+Norms and other numbers
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   linalg.norm
+   linalg.det
+   linalg.matrix_rank
+   linalg.slogdet
+   trace
+
+
+Solving equations and inverting matrices
+----------------------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   linalg.solve
+   linalg.tensorsolve
+   linalg.lstsq
+   linalg.inv
+   linalg.pinv
+   linalg.tensorinv
diff --git a/docs/source/reference/logic.rst b/docs/source/reference/logic.rst
new file mode 100644
index 0000000..9276bfa
--- /dev/null
+++ b/docs/source/reference/logic.rst
@@ -0,0 +1,73 @@
+Logic functions
+===============
+
+.. Hint:: `NumPy API Reference: Logic functions <https://numpy.org/doc/stable/reference/routines.logic.html>`_
+
+.. currentmodule:: cupy
+
+Truth value testing
+-------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   all
+   any
+   union1d
+
+
+Array contents
+--------------
+
+.. autosummary::
+   :toctree: generated/
+
+   isfinite
+   isinf
+   isnan
+   isneginf
+   isposinf
+
+
+Array type testing
+------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   iscomplex
+   iscomplexobj
+   isfortran
+   isreal
+   isrealobj
+   isscalar
+
+
+Logic operations
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   logical_and
+   logical_or
+   logical_not
+   logical_xor
+
+
+Comparison
+----------
+
+.. autosummary::
+   :toctree: generated/
+
+   allclose
+   isclose
+   array_equal
+   array_equiv
+   greater
+   greater_equal
+   less
+   less_equal
+   equal
+   not_equal
diff --git a/docs/source/reference/manipulation.rst b/docs/source/reference/manipulation.rst
new file mode 100644
index 0000000..f436d7c
--- /dev/null
+++ b/docs/source/reference/manipulation.rst
@@ -0,0 +1,134 @@
+Array manipulation routines
+===========================
+
+.. Hint:: `NumPy API Reference: Array manipulation routines <https://numpy.org/doc/stable/reference/routines.array-manipulation.html>`_
+
+.. currentmodule:: cupy
+
+Basic operations
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   copyto
+   shape
+
+
+Changing array shape
+--------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   reshape
+   ravel
+
+.. seealso:: :attr:`cupy.ndarray.flat` and :func:`cupy.ndarray.flatten`
+
+Transpose-like operations
+-------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   moveaxis
+   rollaxis
+   swapaxes
+   transpose
+
+.. seealso:: :attr:`cupy.ndarray.T`
+
+Changing number of dimensions
+-----------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   atleast_1d
+   atleast_2d
+   atleast_3d
+   broadcast
+   broadcast_to
+   broadcast_arrays
+   expand_dims
+   squeeze
+
+
+Changing kind of array
+----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   asarray
+   asanyarray
+   asfarray
+   asfortranarray
+   ascontiguousarray
+   asarray_chkfinite
+   require
+
+
+Joining arrays
+--------------
+
+.. autosummary::
+   :toctree: generated/
+
+   concatenate
+   stack
+   vstack
+   hstack
+   dstack
+   column_stack
+   row_stack
+
+
+Splitting arrays
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   split
+   array_split
+   dsplit
+   hsplit
+   vsplit
+
+
+Tiling arrays
+-------------
+
+.. autosummary::
+   :toctree: generated/
+
+   tile
+   repeat
+
+
+Adding and removing elements
+----------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   append
+   resize
+   unique
+   trim_zeros
+
+
+Rearranging elements
+--------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   flip
+   fliplr
+   flipud
+   reshape
+   roll
+   rot90
diff --git a/docs/source/reference/math.rst b/docs/source/reference/math.rst
new file mode 100644
index 0000000..8c9c2d4
--- /dev/null
+++ b/docs/source/reference/math.rst
@@ -0,0 +1,187 @@
+Mathematical functions
+======================
+
+.. Hint:: `NumPy API Reference: Mathematical functions <https://numpy.org/doc/stable/reference/routines.math.html>`_
+
+.. currentmodule:: cupy
+
+Trigonometric functions
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   sin
+   cos
+   tan
+   arcsin
+   arccos
+   arctan
+   hypot
+   arctan2
+   degrees
+   radians
+   unwrap
+   deg2rad
+   rad2deg
+
+
+Hyperbolic functions
+--------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   sinh
+   cosh
+   tanh
+   arcsinh
+   arccosh
+   arctanh
+
+
+Rounding
+--------
+
+.. autosummary::
+   :toctree: generated/
+
+   around
+   round_
+   rint
+   fix
+   floor
+   ceil
+   trunc
+
+
+Sums, products, differences
+---------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   prod
+   sum
+   nanprod
+   nansum
+   cumprod
+   cumsum
+   nancumprod
+   nancumsum
+   diff
+   gradient
+   ediff1d
+   cross
+   trapz
+
+
+Exponents and logarithms
+------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   exp
+   expm1
+   exp2
+   log
+   log10
+   log2
+   log1p
+   logaddexp
+   logaddexp2
+
+
+Other special functions
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   i0
+   sinc
+
+
+Floating point routines
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   signbit
+   copysign
+   frexp
+   ldexp
+   nextafter
+
+
+Rational routines
+-----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   lcm
+   gcd
+
+
+Arithmetic operations
+---------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   add
+   reciprocal
+   positive
+   negative
+   multiply
+   divide
+   power
+   subtract
+   true_divide
+   floor_divide
+   float_power
+   fmod
+   mod
+   modf
+   remainder
+   divmod
+
+
+Handling complex numbers
+------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   angle
+   real
+   imag
+   conj
+   conjugate
+
+
+Miscellaneous
+-------------
+
+.. autosummary::
+   :toctree: generated/
+
+   convolve
+   clip
+   sqrt
+   cbrt
+   square
+   absolute
+   fabs
+   sign
+   maximum
+   minimum
+   fmax
+   fmin
+   nan_to_num
+   heaviside
+   real_if_close
+   interp
diff --git a/docs/source/reference/misc.rst b/docs/source/reference/misc.rst
new file mode 100644
index 0000000..2fca1ab
--- /dev/null
+++ b/docs/source/reference/misc.rst
@@ -0,0 +1,32 @@
+Miscellaneous routines
+======================
+
+.. Hint:: `NumPy API Reference: Miscellaneous routines <https://numpy.org/doc/stable/reference/routines.other.html>`_
+
+.. currentmodule:: cupy
+
+Memory ranges
+-------------
+
+.. autosummary::
+   :toctree: generated/
+
+   byte_bounds
+   shares_memory
+   may_share_memory
+
+Utility
+-------------
+
+.. autosummary::
+   :toctree: generated/
+
+   show_config
+
+Matlab-like Functions
+---------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   who
diff --git a/docs/source/reference/ndarray.rst b/docs/source/reference/ndarray.rst
new file mode 100644
index 0000000..955591e
--- /dev/null
+++ b/docs/source/reference/ndarray.rst
@@ -0,0 +1,56 @@
+The N-dimensional array (:class:`ndarray <cupy.ndarray>`)
+=========================================================
+
+:class:`cupy.ndarray` is the CuPy counterpart of NumPy :class:`numpy.ndarray`.
+It provides an intuitive interface for a fixed-size multidimensional array which resides
+in a CUDA device.
+
+For the basic concept of ``ndarray``\s, please refer to the `NumPy documentation <https://numpy.org/doc/stable/reference/arrays.ndarray.html>`_.
+
+
+.. TODO(kmaehashi): use currentmodule:: cupy
+.. autosummary::
+   :toctree: generated/
+
+   cupy.ndarray
+
+
+Conversion to/from NumPy arrays
+-------------------------------
+
+:class:`cupy.ndarray` and :class:`numpy.ndarray` are not implicitly convertible to each other.
+That means, NumPy functions cannot take :class:`cupy.ndarray`\s as inputs, and vice versa.
+
+- To convert :class:`numpy.ndarray` to :class:`cupy.ndarray`, use :func:`cupy.array` or :func:`cupy.asarray`.
+- To convert :class:`cupy.ndarray` to :class:`numpy.ndarray`, use :func:`cupy.asnumpy` or :meth:`cupy.ndarray.get`.
+
+Note that converting between :class:`cupy.ndarray` and :class:`numpy.ndarray` incurs data transfer between
+the host (CPU) device and the GPU device, which is costly in terms of performance.
+
+
+.. TODO(kmaehashi): use currentmodule:: cupy
+.. autosummary::
+   :toctree: generated/
+
+   cupy.array
+   cupy.asarray
+   cupy.asnumpy
+
+
+Code compatibility features
+---------------------------
+
+:class:`cupy.ndarray` is designed to be interchangeable with :class:`numpy.ndarray` in terms of code compatibility as much as possible.
+But occasionally, you will need to know whether the arrays you're handling are :class:`cupy.ndarray` or :class:`numpy.ndarray`.
+One example is when invoking module-level functions such as :func:`cupy.sum` or :func:`numpy.sum`.
+In such situations, :func:`cupy.get_array_module` can be used.
+
+.. autosummary::
+   :toctree: generated/
+
+   cupy.get_array_module
+
+.. autosummary::
+   :toctree: generated/
+
+   cupyx.scipy.get_array_module
diff --git a/docs/source/reference/ndimage.rst b/docs/source/reference/ndimage.rst
new file mode 100644
index 0000000..eacb684
--- /dev/null
+++ b/docs/source/reference/ndimage.rst
@@ -0,0 +1,3 @@
+:orphan:
+
+This document has been moved to :doc:`scipy_ndimage`.
diff --git a/docs/source/reference/pad.rst b/docs/source/reference/pad.rst
new file mode 100644
index 0000000..c0b16a4
--- /dev/null
+++ b/docs/source/reference/pad.rst
@@ -0,0 +1,10 @@
+Padding arrays
+==============
+
+.. Hint:: `NumPy API Reference: Padding arrays <https://numpy.org/doc/stable/reference/routines.padding.html>`_
+
+.. currentmodule:: cupy
+.. autosummary::
+   :toctree: generated/
+
+   pad
diff --git a/docs/source/reference/polynomials.rst b/docs/source/reference/polynomials.rst
new file mode 100644
index 0000000..5376b39
--- /dev/null
+++ b/docs/source/reference/polynomials.rst
@@ -0,0 +1,75 @@
+Polynomials
+===========
+
+.. Hint:: `NumPy API Reference: Polynomials <https://numpy.org/doc/stable/reference/routines.polynomials.html>`_
+
+Power Series (:mod:`cupy.polynomial.polynomial`)
+------------------------------------------------
+
+.. Hint:: `NumPy API Reference: Power Series (numpy.polynomial.polynomial) <https://numpy.org/doc/stable/reference/routines.polynomials.polynomial.html>`_
+
+Misc Functions
+~~~~~~~~~~~~~~
+
+.. module:: cupy.polynomial.polynomial
+.. autosummary::
+   :toctree: generated/
+
+   polyvander
+   polycompanion
+
+
+Polyutils
+---------
+
+.. Hint:: `NumPy API Reference: Polyutils <https://numpy.org/doc/stable/reference/routines.polynomials.polyutils.html>`_
+
+Functions
+~~~~~~~~~
+
+.. module:: cupy.polynomial.polyutils
+.. autosummary::
+   :toctree: generated/
+
+   as_series
+   trimseq
+   trimcoef
+
+
+Poly1d
+------
+
+.. Hint:: `NumPy API Reference: Poly1d <https://numpy.org/doc/stable/reference/routines.polynomials.poly1d.html>`_
+
+.. currentmodule:: cupy
+
+Basics
+~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   poly1d
+   cupy.poly
+   polyval
+   roots
+
+
+Fitting
+~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   polyfit
+
+
+Arithmetic
+~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   polyadd
+   polysub
+   polymul
diff --git a/docs/source/reference/random.rst b/docs/source/reference/random.rst
new file mode 100644
index 0000000..ae0f915
--- /dev/null
+++ b/docs/source/reference/random.rst
@@ -0,0 +1,127 @@
+.. module:: cupy.random
+
+Random sampling (:mod:`cupy.random`)
+====================================
+
+Differences between :mod:`cupy.random` and :mod:`numpy.random`:
+
+* Most functions under :mod:`cupy.random` support the ``dtype`` option, which do not exist in the corresponding NumPy APIs.
+  This option enables generation of float32 values directly without any space overhead.
+* :func:`cupy.random.default_rng` uses XORWOW bit generator by default.
+* Random states cannot be serialized. See the description below for details.
+* CuPy does not guarantee that the same number generator is used across major versions.
+  This means that numbers generated by :mod:`cupy.random` by new major version may not be the same as the previous one, even if the same seed and distribution are used.
+
+.. currentmodule:: cupy.random
+
+New Random Generator API
+------------------------
+
+.. Hint:: `NumPy API Reference: Random sampling (numpy.random) <https://numpy.org/doc/stable/reference/random/>`_
+
+Random Generator
+~~~~~~~~~~~~~~~~
+
+.. Hint:: `NumPy API Reference: Random Generator <https://numpy.org/doc/stable/reference/random/generator.html>`_
+
+.. autosummary::
+   :toctree: generated/
+
+   default_rng
+   Generator
+
+Bit Generators
+~~~~~~~~~~~~~~
+
+.. Hint:: `NumPy API Reference: Bit Generators <https://numpy.org/doc/stable/reference/random/bit_generators/index.html>`_
+
+.. autosummary::
+   :toctree: generated/
+
+   BitGenerator
+
+CuPy provides the following bit generator implementations:
+
+.. autosummary::
+   :toctree: generated/
+
+   XORWOW
+   MRG32k3a
+   Philox4x3210
+
+Legacy Random Generation
+------------------------
+
+.. Hint::
+   * `NumPy API Reference: Legacy Random Generation <https://numpy.org/doc/stable/reference/random/legacy.html>`_
+   * `NumPy 1.16 Reference <https://numpy.org/doc/1.16/reference/routines.random.html>`_
+
+.. autosummary::
+   :toctree: generated/
+
+   RandomState
+
+Functions in :mod:`cupy.random`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   beta
+   binomial
+   bytes
+   chisquare
+   choice
+   dirichlet
+   exponential
+   f
+   gamma
+   geometric
+   gumbel
+   hypergeometric
+   laplace
+   logistic
+   lognormal
+   logseries
+   multinomial
+   multivariate_normal
+   negative_binomial
+   noncentral_chisquare
+   noncentral_f
+   normal
+   pareto
+   permutation
+   poisson
+   power
+   rand
+   randint
+   randn
+   random
+   random_integers
+   random_sample
+   ranf
+   rayleigh
+   sample
+   seed
+   shuffle
+   standard_cauchy
+   standard_exponential
+   standard_gamma
+   standard_normal
+   standard_t
+   triangular
+   uniform
+   vonmises
+   wald
+   weibull
+   zipf
+
+CuPy does not provide ``cupy.random.get_state`` nor ``cupy.random.set_state`` at this time.
+Use the following CuPy-specific APIs instead.
+Note that these functions use :class:`cupy.random.RandomState` instance to represent the internal state, which cannot be serialized.
+
+.. autosummary::
+   :toctree: generated/
+
+   get_random_state
+   set_random_state
diff --git a/docs/source/reference/routines.rst b/docs/source/reference/routines.rst
new file mode 100644
index 0000000..b283d74
--- /dev/null
+++ b/docs/source/reference/routines.rst
@@ -0,0 +1,31 @@
+----------------
+Routines (NumPy)
+----------------
+
+The following pages describe NumPy-compatible routines.
+These functions cover a subset of
+`NumPy routines <https://numpy.org/doc/stable/reference/routines.html>`_.
+
+.. toctree::
+   :maxdepth: 2
+
+   creation
+   manipulation
+   binary
+   dtype
+   fft
+   functional
+   indexing
+   io
+   linalg
+   logic
+   math
+   misc
+   pad
+   polynomials
+   random
+   set
+   sorting
+   statistics
+   testing
+   window
diff --git a/docs/source/reference/scipy.rst b/docs/source/reference/scipy.rst
new file mode 100644
index 0000000..ec56678
--- /dev/null
+++ b/docs/source/reference/scipy.rst
@@ -0,0 +1,27 @@
+----------------
+Routines (SciPy)
+----------------
+
+The following pages describe SciPy-compatible routines.
+These functions cover a subset of
+`SciPy routines <https://docs.scipy.org/doc/scipy/reference/#api-reference>`_.
+
+
+.. module:: cupyx.scipy
+
+.. toctree::
+   :maxdepth: 2
+
+   scipy_fft
+   scipy_fftpack
+   scipy_interpolate
+   scipy_linalg
+   scipy_ndimage
+   scipy_signal
+   scipy_sparse
+   scipy_sparse_linalg
+   scipy_sparse_csgraph
+   scipy_spatial
+   scipy_spatial_distance
+   scipy_special
+   scipy_stats
diff --git a/docs/source/reference/scipy_fft.rst b/docs/source/reference/scipy_fft.rst
new file mode 100644
index 0000000..ff18427
--- /dev/null
+++ b/docs/source/reference/scipy_fft.rst
@@ -0,0 +1,84 @@
+.. module:: cupyx.scipy.fft
+
+Discrete Fourier transforms (:mod:`cupyx.scipy.fft`)
+====================================================
+
+.. Hint:: `SciPy API Reference: Discrete Fourier transforms (scipy.fft) <https://docs.scipy.org/doc/scipy/reference/fft.html>`_
+
+.. seealso:: :doc:`../user_guide/fft`
+
+Fast Fourier Transforms (FFTs)
+------------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   fft
+   ifft
+   fft2
+   ifft2
+   fftn
+   ifftn
+   rfft
+   irfft
+   rfft2
+   irfft2
+   rfftn
+   irfftn
+   hfft
+   ihfft
+   hfft2
+   ihfft2
+   hfftn
+   ihfftn
+
+Discrete Cosine and Sine Transforms (DST and DCT)
+-------------------------------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   dct
+   idct
+   dctn
+   idctn
+   dst
+   idst
+   dstn
+   idstn
+
+Fast Hankel Transforms
+----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   fht
+   ifht
+
+Helper functions
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   fftshift
+   ifftshift
+   fftfreq
+   rfftfreq
+   next_fast_len
+
+
+Code compatibility features
+---------------------------
+1. As with other FFT modules in CuPy, FFT functions in this module can take advantage of an existing cuFFT plan (returned by :func:`~cupyx.scipy.fftpack.get_fft_plan`) to accelerate the computation. The plan can be either passed in explicitly via the keyword-only ``plan`` argument or used as a context manager. One exception to this are the DCT and DST transforms, which do not currently support a plan argument.
+
+2. The boolean switch ``cupy.fft.config.enable_nd_planning`` also affects the FFT functions in this module, see :doc:`./fft`. This switch is neglected when planning manually using :func:`~cupyx.scipy.fftpack.get_fft_plan`.
+
+3. Like in ``scipy.fft``, all FFT functions in this module have an optional argument ``overwrite_x`` (default is ``False``), which has the same semantics as in ``scipy.fft``: when it is set to ``True``, the input array ``x`` *can* (not *will*) be overwritten arbitrarily. For this reason, when an in-place FFT is desired, the user should always reassign the input in the following manner: ``x = cupyx.scipy.fftpack.fft(x, ..., overwrite_x=True, ...)``.
+
+4. The ``cupyx.scipy.fft`` module can also be used as a backend for ``scipy.fft`` e.g. by installing with ``scipy.fft.set_backend(cupyx.scipy.fft)``. This can allow ``scipy.fft`` to work with both ``numpy`` and ``cupy`` arrays. For more information, see :ref:`scipy_fft_backend`.
+
+5. The boolean switch :data:`cupy.fft.config.use_multi_gpus` also affects the FFT functions in this module, see :doc:`./fft`. Moreover, this switch is *honored* when planning manually using :func:`~cupyx.scipy.fftpack.get_fft_plan`.
+
+6. Both type II and III DCT and DST transforms are implemented. Type I and IV transforms are currently unavailable.
diff --git a/docs/source/reference/scipy_fftpack.rst b/docs/source/reference/scipy_fftpack.rst
new file mode 100644
index 0000000..4c23c57
--- /dev/null
+++ b/docs/source/reference/scipy_fftpack.rst
@@ -0,0 +1,38 @@
+.. module:: cupyx.scipy.fftpack
+
+Legacy discrete fourier transforms (:mod:`cupyx.scipy.fftpack`)
+===============================================================
+
+.. note::
+
+   As of SciPy version 1.4.0, :mod:`scipy.fft` is recommended over
+   :mod:`scipy.fftpack`. Consider using :mod:`cupyx.scipy.fft` instead.
+
+.. Hint:: `SciPy API Reference: Legacy discrete Fourier transforms (scipy.fftpack) <https://docs.scipy.org/doc/scipy/reference/fftpack.html>`_
+
+Fast Fourier Transforms (FFTs)
+------------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   fft
+   ifft
+   fft2
+   ifft2
+   fftn
+   ifftn
+   rfft
+   irfft
+   get_fft_plan
+
+
+Code compatibility features
+---------------------------
+1. As with other FFT modules in CuPy, FFT functions in this module can take advantage of an existing cuFFT plan (returned by :func:`~cupyx.scipy.fftpack.get_fft_plan`) to accelarate the computation. The plan can be either passed in explicitly via the ``plan`` argument or used as a context manager. The argument ``plan`` is currently experimental and the interface may be changed in the future version. The :func:`~cupyx.scipy.fftpack.get_fft_plan` function has no counterpart in ``scipy.fftpack``.
+
+2. The boolean switch :data:`cupy.fft.config.enable_nd_planning` also affects the FFT functions in this module, see :doc:`./fft`. This switch is neglected when planning manually using :func:`~cupyx.scipy.fftpack.get_fft_plan`.
+
+3. Like in ``scipy.fftpack``, all FFT functions in this module have an optional argument ``overwrite_x`` (default is ``False``), which has the same semantics as in ``scipy.fftpack``: when it is set to ``True``, the input array ``x`` *can* (not *will*) be overwritten arbitrarily. For this reason, when an in-place FFT is desired, the user should always reassign the input in the following manner: ``x = cupyx.scipy.fftpack.fft(x, ..., overwrite_x=True, ...)``.
+
+4. The boolean switch :data:`cupy.fft.config.use_multi_gpus` also affects the FFT functions in this module, see :doc:`./fft`. Moreover, this switch is *honored* when planning manually using :func:`~cupyx.scipy.fftpack.get_fft_plan`.
diff --git a/docs/source/reference/scipy_interpolate.rst b/docs/source/reference/scipy_interpolate.rst
new file mode 100644
index 0000000..6d69dc6
--- /dev/null
+++ b/docs/source/reference/scipy_interpolate.rst
@@ -0,0 +1,47 @@
+.. module:: cupyx.scipy.interpolate
+
+Interpolation (:mod:`cupyx.scipy.interpolate`)
+==============================================
+
+.. Hint:: `SciPy API Reference: Interpolation functions (scipy.interpolate) <https://docs.scipy.org/doc/scipy/reference/interpolate.html>`_
+
+Univariate interpolation
+------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   BarycentricInterpolator
+   KroghInterpolator
+   barycentric_interpolate
+   krogh_interpolate
+   pchip_interpolate
+   CubicHermiteSpline
+   PchipInterpolator
+   Akima1DInterpolator
+   PPoly
+   BPoly
+
+1-D Splines
+-----------
+
+.. autosummary::
+   :toctree: generated/
+
+   BSpline
+   make_interp_spline
+
+   splder
+   splantider
+
+
+Multivariate interpolation
+--------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   RBFInterpolator
+   interpn
+   RegularGridInterpolator
+
diff --git a/docs/source/reference/scipy_linalg.rst b/docs/source/reference/scipy_linalg.rst
new file mode 100644
index 0000000..a87cc19
--- /dev/null
+++ b/docs/source/reference/scipy_linalg.rst
@@ -0,0 +1,50 @@
+.. module:: cupyx.scipy.linalg
+
+Linear algebra (:mod:`cupyx.scipy.linalg`)
+==========================================
+
+.. Hint:: `SciPy API Reference: Linear algebra (scipy.linalg) <https://docs.scipy.org/doc/scipy/reference/linalg.html>`_
+
+Basics
+------
+
+.. autosummary::
+   :toctree: generated/
+
+   solve_triangular
+   tril
+   triu
+
+
+Decompositions
+--------------
+
+.. autosummary::
+   :toctree: generated/
+
+   lu
+   lu_factor
+   lu_solve
+
+
+Special Matrices
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   block_diag
+   circulant
+   companion
+   convolution_matrix
+   dft
+   fiedler
+   fiedler_companion
+   hadamard
+   hankel
+   helmert
+   hilbert
+   kron
+   leslie
+   toeplitz
+   tri
diff --git a/docs/source/reference/scipy_ndimage.rst b/docs/source/reference/scipy_ndimage.rst
new file mode 100644
index 0000000..c312864
--- /dev/null
+++ b/docs/source/reference/scipy_ndimage.rst
@@ -0,0 +1,139 @@
+.. module:: cupyx.scipy.ndimage
+
+Multidimensional image processing (:mod:`cupyx.scipy.ndimage`)
+==============================================================
+
+.. Hint:: `SciPy API Reference: Multidimensional image processing (scipy.ndimage) <https://docs.scipy.org/doc/scipy/reference/ndimage.html>`_
+
+
+Filters
+-------
+
+.. autosummary::
+   :toctree: generated/
+
+   convolve
+   convolve1d
+   correlate
+   correlate1d
+   gaussian_filter
+   gaussian_filter1d
+   gaussian_gradient_magnitude
+   gaussian_laplace
+   generic_filter
+   generic_filter1d
+   generic_gradient_magnitude
+   generic_laplace
+   laplace
+   maximum_filter
+   maximum_filter1d
+   median_filter
+   minimum_filter
+   minimum_filter1d
+   percentile_filter
+   prewitt
+   rank_filter
+   sobel
+   uniform_filter
+   uniform_filter1d
+
+
+Fourier filters
+---------------
+
+.. autosummary::
+   :toctree: generated/
+
+   fourier_ellipsoid
+   fourier_gaussian
+   fourier_shift
+   fourier_uniform
+
+
+Interpolation
+-------------
+
+.. autosummary::
+   :toctree: generated/
+
+   affine_transform
+   map_coordinates
+   rotate
+   shift
+   spline_filter
+   spline_filter1d
+   zoom
+
+
+Measurements
+------------
+
+.. autosummary::
+   :toctree: generated/
+
+   center_of_mass
+   extrema
+   histogram
+   label
+   labeled_comprehension
+   maximum
+   maximum_position
+   mean
+   median
+   minimum
+   minimum_position
+   standard_deviation
+   sum_labels
+   variance
+
+
+Morphology
+----------
+
+.. autosummary::
+   :toctree: generated/
+
+   binary_closing
+   binary_dilation
+   binary_erosion
+   binary_fill_holes
+   binary_hit_or_miss
+   binary_opening
+   binary_propagation
+   black_tophat
+   generate_binary_structure
+   grey_closing
+   grey_dilation
+   grey_erosion
+   grey_opening
+   iterate_structure
+   morphological_gradient
+   morphological_laplace
+   white_tophat
+
+
+OpenCV mode
+-----------
+:mod:`cupyx.scipy.ndimage` supports additional mode, ``opencv``.
+If it is given, the function performs like `cv2.warpAffine <https://docs.opencv.org/master/da/d54/group__imgproc__transform.html#ga0203d9ee5fcd28d40dbc4a1ea4451983>`_ or `cv2.resize <https://docs.opencv.org/master/da/d54/group__imgproc__transform.html#ga47a974309e9102f5f08231edc7e7529d>`_. Example:
+
+
+.. code:: python
+
+   import cupyx.scipy.ndimage
+   import cupy as cp
+   import cv2
+
+   im = cv2.imread('TODO') # pls fill in your image path
+
+   trans_mat = cp.eye(4)
+   trans_mat[0][0] = trans_mat[1][1] = 0.5
+
+   smaller_shape = (im.shape[0] // 2, im.shape[1] // 2, 3)
+   smaller = cp.zeros(smaller_shape) # preallocate memory for resized image
+
+   cupyx.scipy.ndimage.affine_transform(im, trans_mat, output_shape=smaller_shape,
+                                        output=smaller, mode='opencv')
+
+   cv2.imwrite('smaller.jpg', cp.asnumpy(smaller)) # smaller image saved locally
+
diff --git a/docs/source/reference/scipy_signal.rst b/docs/source/reference/scipy_signal.rst
new file mode 100644
index 0000000..091c686
--- /dev/null
+++ b/docs/source/reference/scipy_signal.rst
@@ -0,0 +1,33 @@
+.. module:: cupyx.scipy.signal
+
+Signal processing (:mod:`cupyx.scipy.signal`)
+=============================================
+
+.. Hint:: `SciPy API Reference: Signal processing (scipy.signal) <https://docs.scipy.org/doc/scipy/reference/signal.html>`_
+
+Convolution
+-----------
+
+.. autosummary::
+   :toctree: generated/
+
+   convolve
+   correlate
+   fftconvolve
+   oaconvolve
+   convolve2d
+   correlate2d
+   sepfir2d
+   choose_conv_method
+
+
+Filtering
+---------
+
+.. autosummary::
+   :toctree: generated/
+
+   order_filter
+   medfilt
+   medfilt2d
+   wiener
diff --git a/docs/source/reference/scipy_sparse.rst b/docs/source/reference/scipy_sparse.rst
new file mode 100644
index 0000000..637ab20
--- /dev/null
+++ b/docs/source/reference/scipy_sparse.rst
@@ -0,0 +1,102 @@
+.. module:: cupyx.scipy.sparse
+
+Sparse matrices (:mod:`cupyx.scipy.sparse`)
+===========================================
+
+.. Hint:: `SciPy API Reference: Sparse matrices (scipy.sparse) <https://docs.scipy.org/doc/scipy/reference/sparse.html>`_
+
+CuPy supports sparse matrices using `cuSPARSE <https://developer.nvidia.com/cusparse>`_.
+These matrices have the same interfaces of `SciPy's sparse matrices <https://docs.scipy.org/doc/scipy/reference/sparse.html>`_.
+
+Conversion to/from SciPy sparse matrices
+----------------------------------------
+
+``cupyx.scipy.sparse.*_matrix`` and ``scipy.sparse.*_matrix`` are not implicitly convertible to each other.
+That means, SciPy functions cannot take ``cupyx.scipy.sparse.*_matrix`` objects as inputs, and vice versa.
+
+- To convert SciPy sparse matrices to CuPy, pass it to the constructor of each CuPy sparse matrix class.
+- To convert CuPy sparse matrices to SciPy, use :func:`get <cupyx.scipy.sparse.spmatrix.get>` method of each CuPy sparse matrix class.
+
+Note that converting between CuPy and SciPy incurs data transfer between
+the host (CPU) device and the GPU device, which is costly in terms of performance.
+
+Conversion to/from CuPy ndarrays
+--------------------------------
+
+- To convert CuPy ndarray to CuPy sparse matrices, pass it to the constructor of each CuPy sparse matrix class.
+- To convert CuPy sparse matrices to CuPy ndarray, use ``toarray`` of each CuPy sparse matrix instance (e.g., :func:`cupyx.scipy.sparse.csr_matrix.toarray`).
+
+Converting between CuPy ndarray and CuPy sparse matrices does not incur data transfer; it is copied inside the GPU device.
+
+Contents
+--------
+
+Sparse matrix classes
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   coo_matrix
+   csc_matrix
+   csr_matrix
+   dia_matrix
+   spmatrix
+
+
+Functions
+~~~~~~~~~
+
+Building sparse matrices:
+
+.. autosummary::
+   :toctree: generated/
+
+   eye
+   identity
+   kron
+   kronsum
+   diags
+   spdiags
+   tril
+   triu
+   bmat
+   hstack
+   vstack
+   rand
+   random
+
+
+Sparse matrix tools:
+
+.. autosummary::
+   :toctree: generated/
+
+   find
+
+Identifying sparse matrices:
+
+.. autosummary::
+   :toctree: generated/
+
+   issparse
+   isspmatrix
+   isspmatrix_csc
+   isspmatrix_csr
+   isspmatrix_coo
+   isspmatrix_dia
+
+
+Submodules
+~~~~~~~~~~
+
+.. autosummary::
+
+   csgraph - Compressed sparse graph routines
+   linalg - Sparse linear algebra routines
+
+Exceptions
+~~~~~~~~~~
+
+* :class:`scipy.sparse.SparseEfficiencyWarning`
+* :class:`scipy.sparse.SparseWarning`
diff --git a/docs/source/reference/scipy_sparse_csgraph.rst b/docs/source/reference/scipy_sparse_csgraph.rst
new file mode 100644
index 0000000..58ff352
--- /dev/null
+++ b/docs/source/reference/scipy_sparse_csgraph.rst
@@ -0,0 +1,22 @@
+.. module:: cupyx.scipy.sparse.csgraph
+
+Compressed sparse graph routines (:mod:`cupyx.scipy.sparse.csgraph`)
+====================================================================
+
+.. note::
+
+   The ``csgraph`` module uses ``pylibcugraph`` as a backend.
+   You need to install `pylibcugraph package <https://anaconda.org/rapidsai/pylibcugraph>` from ``rapidsai`` Conda channel to use features listed on this page.
+
+.. note::
+   Currently, the ``csgraph`` module is not supported on AMD ROCm platforms.
+
+.. Hint:: `SciPy API Reference: Compressed sparse graph routines (scipy.sparse.csgraph) <https://docs.scipy.org/doc/scipy/reference/sparse.csgraph.html>`_
+
+Contents
+---------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   connected_components
diff --git a/docs/source/reference/scipy_sparse_linalg.rst b/docs/source/reference/scipy_sparse_linalg.rst
new file mode 100644
index 0000000..2b4eaf7
--- /dev/null
+++ b/docs/source/reference/scipy_sparse_linalg.rst
@@ -0,0 +1,83 @@
+.. module:: cupyx.scipy.sparse.linalg
+
+Sparse linear algebra (:mod:`cupyx.scipy.sparse.linalg`)
+========================================================
+
+.. Hint:: `SciPy API Reference: Sparse linear algebra (scipy.sparse.linalg) <https://docs.scipy.org/doc/scipy/reference/sparse.linalg.html>`_
+
+Abstract linear operators
+-------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   LinearOperator
+   aslinearoperator
+
+
+Matrix norms
+------------
+
+.. autosummary::
+   :toctree: generated/
+
+   norm
+
+
+Solving linear problems
+-----------------------
+
+Direct methods for linear equation systems:
+
+.. autosummary::
+   :toctree: generated/
+
+   spsolve
+   spsolve_triangular
+   factorized
+
+Iterative methods for linear equation systems:
+
+.. autosummary::
+   :toctree: generated/
+
+   cg
+   gmres
+   cgs
+   minres
+
+Iterative methods for least-squares problems:
+
+.. autosummary::
+   :toctree: generated/
+
+   lsqr
+   lsmr
+
+
+Matrix factorizations
+---------------------
+
+Eigenvalue problems:
+
+.. autosummary::
+   :toctree: generated/
+
+   eigsh
+   lobpcg
+
+Singular values problems:
+
+.. autosummary::
+   :toctree: generated/
+
+   svds
+
+Complete or incomplete LU factorizations:
+
+.. autosummary::
+   :toctree: generated/
+
+   splu
+   spilu
+   SuperLU
diff --git a/docs/source/reference/scipy_spatial.rst b/docs/source/reference/scipy_spatial.rst
new file mode 100644
index 0000000..33e5aae
--- /dev/null
+++ b/docs/source/reference/scipy_spatial.rst
@@ -0,0 +1,22 @@
+.. module:: cupyx.scipy.spatial
+
+Spatial algorithms and data structures  (:mod:`cupyx.scipy.spatial`)
+====================================================================
+
+.. Hint:: `SciPy API Reference: Spatial (scipy.spatial) <https://docs.scipy.org/doc/scipy/reference/spatial.html>`_
+
+.. note::
+
+   The ``spatial`` module uses ``pylibraft`` as a backend.
+   You need to install `pylibraft package <https://anaconda.org/rapidsai/pylibraft>` from ``rapidsai`` Conda channel to use features listed on this page.
+
+.. note::
+   Currently, the ``spatial`` module is not supported on AMD ROCm platforms.
+
+Functions
+---------
+
+.. autosummary::
+   :toctree: generated/
+
+    distance_matrix
diff --git a/docs/source/reference/scipy_spatial_distance.rst b/docs/source/reference/scipy_spatial_distance.rst
new file mode 100644
index 0000000..aa7b696
--- /dev/null
+++ b/docs/source/reference/scipy_spatial_distance.rst
@@ -0,0 +1,51 @@
+.. module:: cupyx.scipy.spatial.distance
+
+Distance computations (:mod:`cupyx.scipy.spatial.distance`)
+===========================================================
+
+.. note::
+
+   The ``distance`` module uses ``pylibraft`` as a backend.
+   You need to install `pylibraft package <https://anaconda.org/rapidsai/pylibraft>` from ``rapidsai`` Conda channel to use features listed on this page.
+
+.. note::
+   Currently, the ``distance`` module is not supported on AMD ROCm platforms.
+
+.. Hint:: `SciPy API Reference: Spatial distance routines (scipy.spatial.distance) <https://docs.scipy.org/doc/scipy/reference/spatial.distance.html>`_
+
+
+Distance matrix computations
+----------------------------
+
+Distance matrix computation from a collection of raw observation vectors stored in a rectangular array.
+
+.. autosummary::
+   :toctree: generated/
+
+   pdist
+   cdist
+   distance_matrix
+
+
+Distance functions
+------------------
+
+Distance functions between two numeric vectors `u` and `v`. Computing distances over a large collection of vectors is inefficient for these functions. Use `cdist` for this purpose.
+
+.. autosummary::
+   :toctree: generated/
+
+   minkowski
+   canberra
+   chebyshev
+   cityblock
+   correlation
+   cosine
+   hamming
+   euclidean
+   jensenshannon
+   russellrao
+   sqeuclidean
+   hellinger
+   kl_divergence
+
diff --git a/docs/source/reference/scipy_special.rst b/docs/source/reference/scipy_special.rst
new file mode 100644
index 0000000..6d0e1dd
--- /dev/null
+++ b/docs/source/reference/scipy_special.rst
@@ -0,0 +1,164 @@
+.. module:: cupyx.scipy.special
+
+Special functions (:mod:`cupyx.scipy.special`)
+===============================================
+
+.. Hint:: `SciPy API Reference: Special functions (scipy.special) <https://docs.scipy.org/doc/scipy/reference/special.html>`_
+
+Bessel functions
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   j0
+   j1
+   k0
+   k0e
+   k1
+   k1e
+   y0
+   y1
+   yn
+   i0
+   i0e
+   i1
+   i1e
+
+
+Raw statistical functions
+-------------------------
+
+.. seealso:: :mod:`cupyx.scipy.stats`
+
+.. autosummary::
+   :toctree: generated/
+
+   bdtr
+   bdtrc
+   bdtri
+   btdtr
+   btdtri
+   fdtr
+   fdtrc
+   fdtri
+   gdtr
+   gdtrc
+   nbdtr
+   nbdtrc
+   nbdtri
+   pdtr
+   pdtrc
+   pdtri
+   chdtr
+   chdtrc
+   chdtri
+   ndtr
+   log_ndtr
+   ndtri
+   logit
+   expit
+   log_expit
+   boxcox
+   boxcox1p
+   inv_boxcox
+   inv_boxcox1p
+
+
+Information Theory functions
+----------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   entr
+   rel_entr
+   kl_div
+   huber
+   pseudo_huber
+
+
+Gamma and related functions
+---------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   gamma
+   gammaln
+   loggamma
+   gammainc
+   gammaincinv
+   gammaincc
+   gammainccinv
+   beta
+   betaln
+   betainc
+   betaincinv
+   psi
+   rgamma
+   polygamma
+   multigammaln
+   digamma
+   poch
+
+
+Error function and Fresnel integrals
+------------------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   erf
+   erfc
+   erfcx
+   erfinv
+   erfcinv
+
+
+Legendre functions
+---------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   lpmv
+   sph_harm
+
+
+Other special functions
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   exp1
+   expi
+   expn
+   softmax
+   log_softmax
+   zeta
+
+
+Convenience functions
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   cbrt
+   exp10
+   exp2
+   radian
+   cosdg
+   sindg
+   tandg
+   cotdg
+   log1p
+   expm1
+   cosm1
+   round
+   xlogy
+   xlog1py
+   logsumexp
+   sinc
diff --git a/docs/source/reference/scipy_stats.rst b/docs/source/reference/scipy_stats.rst
new file mode 100644
index 0000000..a90d3db
--- /dev/null
+++ b/docs/source/reference/scipy_stats.rst
@@ -0,0 +1,28 @@
+.. module:: cupyx.scipy.stats
+
+Statistical functions (:mod:`cupyx.scipy.stats`)
+================================================
+
+.. Hint:: `SciPy API Reference: Statistical functions (scipy.stats) <https://docs.scipy.org/doc/scipy/reference/stats.html>`_
+
+
+Summary statistics
+------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   trim_mean
+   entropy
+
+
+Other statistical functionality
+-------------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   boxcox_llf
+   zmap
+   zscore
+
diff --git a/docs/source/reference/set.rst b/docs/source/reference/set.rst
new file mode 100644
index 0000000..23ee0e1
--- /dev/null
+++ b/docs/source/reference/set.rst
@@ -0,0 +1,27 @@
+Set routines
+============
+
+.. Hint:: `NumPy API Reference: Set routines <https://numpy.org/doc/stable/reference/routines.set.html>`_
+
+.. currentmodule:: cupy
+
+Making proper sets
+------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   unique
+
+
+Boolean operations
+------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   in1d
+   intersect1d
+   isin
+   setdiff1d
+   setxor1d
diff --git a/docs/source/reference/signal.rst b/docs/source/reference/signal.rst
new file mode 100644
index 0000000..f484230
--- /dev/null
+++ b/docs/source/reference/signal.rst
@@ -0,0 +1,3 @@
+:orphan:
+
+This document has been moved to :doc:`scipy_signal`.
diff --git a/docs/source/reference/sorting.rst b/docs/source/reference/sorting.rst
new file mode 100644
index 0000000..be5c313
--- /dev/null
+++ b/docs/source/reference/sorting.rst
@@ -0,0 +1,47 @@
+Sorting, searching, and counting
+================================
+
+.. Hint:: `NumPy API Reference: Sorting, searching, and counting <https://numpy.org/doc/stable/reference/routines.sort.html>`_
+
+.. currentmodule:: cupy
+
+Sorting
+-------
+
+.. autosummary::
+   :toctree: generated/
+
+   sort
+   lexsort
+   argsort
+   msort
+   sort_complex
+   partition
+   argpartition
+
+.. seealso:: :func:`cupy.ndarray.sort`
+
+Searching
+---------
+
+.. autosummary::
+   :toctree: generated/
+
+   argmax
+   nanargmax
+   argmin
+   nanargmin
+   argwhere
+   nonzero
+   flatnonzero
+   where
+   searchsorted
+   extract
+
+Counting
+--------
+
+.. autosummary::
+   :toctree: generated/
+
+   count_nonzero
diff --git a/docs/source/reference/sparse.rst b/docs/source/reference/sparse.rst
new file mode 100644
index 0000000..d701314
--- /dev/null
+++ b/docs/source/reference/sparse.rst
@@ -0,0 +1,3 @@
+:orphan:
+
+This document has been moved to :doc:`scipy_sparse`.
diff --git a/docs/source/reference/special.rst b/docs/source/reference/special.rst
new file mode 100644
index 0000000..cd5bce9
--- /dev/null
+++ b/docs/source/reference/special.rst
@@ -0,0 +1,4 @@
+:orphan:
+
+
+This document has been moved to :doc:`scipy_special`.
diff --git a/docs/source/reference/statistics.rst b/docs/source/reference/statistics.rst
new file mode 100644
index 0000000..ad7637d
--- /dev/null
+++ b/docs/source/reference/statistics.rst
@@ -0,0 +1,61 @@
+Statistics
+==========
+
+.. Hint:: `NumPy API Reference: Statistics <https://numpy.org/doc/stable/reference/routines.statistics.html>`_
+
+.. currentmodule:: cupy
+
+Order statistics
+----------------
+
+.. autosummary::
+   :toctree: generated/
+
+   amin
+   amax
+   nanmin
+   nanmax
+   ptp
+   percentile
+   quantile
+
+
+Averages and variances
+----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   median
+   average
+   mean
+   std
+   var
+   nanmedian
+   nanmean
+   nanstd
+   nanvar
+
+
+Correlations
+------------
+
+.. autosummary::
+   :toctree: generated/
+
+   corrcoef
+   correlate
+   cov
+
+
+Histograms
+----------
+
+.. autosummary::
+   :toctree: generated/
+
+   histogram
+   histogram2d
+   histogramdd
+   bincount
+   digitize
diff --git a/docs/source/reference/stats.rst b/docs/source/reference/stats.rst
new file mode 100644
index 0000000..7ba615b
--- /dev/null
+++ b/docs/source/reference/stats.rst
@@ -0,0 +1,3 @@
+:orphan:
+
+This document has been moved to :doc:`scipy_stats`.
diff --git a/docs/source/reference/testing.rst b/docs/source/reference/testing.rst
new file mode 100644
index 0000000..f86c4bb
--- /dev/null
+++ b/docs/source/reference/testing.rst
@@ -0,0 +1,87 @@
+.. module:: cupy.testing
+
+Test support (:mod:`cupy.testing`)
+==================================
+
+.. Hint:: `NumPy API Reference: Test support (numpy.testing) <https://numpy.org/doc/stable/reference/routines.testing.html>`_
+
+Asserts
+-------
+
+.. Hint:: These APIs can accept both :class:`numpy.ndarray` and :class:`cupy.ndarray`.
+
+.. autosummary::
+   :toctree: generated/
+
+   assert_array_almost_equal
+   assert_allclose
+   assert_array_almost_equal_nulp
+   assert_array_max_ulp
+   assert_array_equal
+   assert_array_less
+
+CuPy-specific APIs
+------------------
+
+Asserts
+~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   assert_array_list_equal
+
+
+NumPy-CuPy Consistency Check
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following decorators are for testing consistency
+between CuPy's functions and corresponding NumPy's ones.
+
+.. autosummary::
+   :toctree: generated/
+
+   numpy_cupy_allclose
+   numpy_cupy_array_almost_equal
+   numpy_cupy_array_almost_equal_nulp
+   numpy_cupy_array_max_ulp
+   numpy_cupy_array_equal
+   numpy_cupy_array_list_equal
+   numpy_cupy_array_less
+
+
+Parameterized dtype Test
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following decorators offer the standard way for
+parameterized test with respect to single or the
+combination of dtype(s).
+
+.. autosummary::
+   :toctree: generated/
+
+   for_dtypes
+   for_all_dtypes
+   for_float_dtypes
+   for_signed_dtypes
+   for_unsigned_dtypes
+   for_int_dtypes
+   for_complex_dtypes
+   for_dtypes_combination
+   for_all_dtypes_combination
+   for_signed_dtypes_combination
+   for_unsigned_dtypes_combination
+   for_int_dtypes_combination
+
+
+Parameterized order Test
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following decorators offer the standard way to parameterize tests with
+orders.
+
+.. autosummary::
+   :toctree: generated/
+
+   for_orders
+   for_CF_orders
diff --git a/docs/source/reference/ufunc.rst b/docs/source/reference/ufunc.rst
new file mode 100644
index 0000000..011fbb4
--- /dev/null
+++ b/docs/source/reference/ufunc.rst
@@ -0,0 +1,186 @@
+Universal functions (:class:`cupy.ufunc`)
+=========================================
+
+.. Hint:: `NumPy API Reference: Universal functions (numpy.ufunc) <https://numpy.org/doc/stable/reference/ufuncs.html>`_
+
+.. currentmodule:: cupy
+
+CuPy provides universal functions (a.k.a. ufuncs) to support various elementwise operations.
+CuPy's ufunc supports following features of NumPy's one:
+
+- Broadcasting
+- Output type determination
+- Casting rules
+
+
+ufunc
+-----
+
+.. autosummary::
+   :toctree: generated/
+
+   ufunc
+
+Methods
+~~~~~~~
+
+These methods are only available for selected ufuncs.
+
+* :meth:`ufunc.reduce <cupy.ufunc.reduce>`: :func:`~cupy.add`, :func:`~cupy.multiply`
+* :meth:`ufunc.accumulate <cupy.ufunc.accumulate>`: :func:`~cupy.add`, :func:`~cupy.multiply`
+* :meth:`ufunc.reduceat <ufunc.reduceat>`: :func:`~cupy.add`
+* :meth:`ufunc.outer <cupy.ufunc.outer>`: All ufuncs
+* :meth:`ufunc.at <cupy.ufunc.at>`: :func:`~cupy.add`, :func:`~cupy.subtract`, :func:`~cupy.maximum`, :func:`~cupy.minimum`, :func:`~cupy.bitwise_and`, :func:`~cupy.bitwise_or`, :func:`~cupy.bitwise_xor`
+
+.. hint::
+
+   In case you need support for other ufuncs, submit a feature request along with your use-case in `the tracker issue <https://github.com/cupy/cupy/issues/7082>`_.
+
+
+Available ufuncs
+----------------
+
+Math operations
+~~~~~~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   add
+   subtract
+   multiply
+   matmul
+   divide
+   logaddexp
+   logaddexp2
+   true_divide
+   floor_divide
+   negative
+   positive
+   power
+   float_power
+   remainder
+   mod
+   fmod
+   divmod
+   absolute
+   fabs
+   rint
+   sign
+   heaviside
+   conj
+   conjugate
+   exp
+   exp2
+   log
+   log2
+   log10
+   expm1
+   log1p
+   sqrt
+   square
+   cbrt
+   reciprocal
+   gcd
+   lcm
+
+
+Trigonometric functions
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   sin
+   cos
+   tan
+   arcsin
+   arccos
+   arctan
+   arctan2
+   hypot
+   sinh
+   cosh
+   tanh
+   arcsinh
+   arccosh
+   arctanh
+   degrees
+   radians
+   deg2rad
+   rad2deg
+
+
+Bit-twiddling functions
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   bitwise_and
+   bitwise_or
+   bitwise_xor
+   invert
+   left_shift
+   right_shift
+
+
+Comparison functions
+~~~~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   greater
+   greater_equal
+   less
+   less_equal
+   not_equal
+   equal
+   logical_and
+   logical_or
+   logical_xor
+   logical_not
+   maximum
+   minimum
+   fmax
+   fmin
+
+
+Floating functions
+~~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   isfinite
+   isinf
+   isnan
+   fabs
+   signbit
+   copysign
+   nextafter
+   modf
+   ldexp
+   frexp
+   fmod
+   floor
+   ceil
+   trunc
+
+
+Generalized Universal Functions
+-------------------------------
+
+.. currentmodule:: cupyx
+
+In addition to regular ufuncs, CuPy also provides a wrapper class to convert
+regular cupy functions into Generalized Universal Functions as in NumPy `<https://numpy.org/doc/stable/reference/c-api/generalized-ufuncs.html>`_.
+This allows to automatically use keyword arguments such as ``axes``, ``order``, ``dtype``
+without needing to explicitly implement them in the wrapped function.
+
+
+.. autosummary::
+   :toctree: generated/
+
+   GeneralizedUFunc
diff --git a/docs/source/reference/window.rst b/docs/source/reference/window.rst
new file mode 100644
index 0000000..dcfc81f
--- /dev/null
+++ b/docs/source/reference/window.rst
@@ -0,0 +1,18 @@
+Window functions
+================
+
+.. Hint:: `NumPy API Reference: Window functions <https://numpy.org/doc/stable/reference/routines.window.html>`_
+
+.. currentmodule:: cupy
+
+Various windows
+---------------
+
+.. autosummary::
+   :toctree: generated/
+
+   bartlett
+   blackman
+   hamming
+   hanning
+   kaiser
diff --git a/docs/source/spelling_wordlist.txt b/docs/source/spelling_wordlist.txt
new file mode 100644
index 0000000..7922494
--- /dev/null
+++ b/docs/source/spelling_wordlist.txt
@@ -0,0 +1,173 @@
+acyclic
+affine
+allocator
+arccosine
+backend
+backprop
+backpropagation
+Bernoulli
+bilinear
+bool
+boolean
+broadcast
+broadcasted
+Caffe
+Chainer
+Cifer
+channelwise
+compatibilities
+composability
+composable
+compositional
+connectionist
+concat
+contrastive
+convolutional
+covariance
+CPU
+CTC
+cuBLAS
+CUDA
+cuDNN
+CuPy
+cuRAND
+customizable
+cutorch
+dataset
+datasets
+de
+deallocated
+deallocation
+deconvolution
+deconvolutional
+deserialization
+deserialize
+deserialized
+deserializes
+deserializer
+deserializers
+deserializing
+differentiable
+dimensionalities
+dimensionality
+dimentional
+dtype
+dtypes
+elementwise
+embeddings
+evaluator
+facto
+FFT
+finalizer
+functionalities
+gaussian
+GPU
+grey
+greyscale
+gzip
+hdf
+huber
+hyperparameter
+hyperparameters
+hypoteneous
+implementers
+indices
+infeasible
+initializer
+initializers
+instantiation
+iterable
+iteratively
+libhdf
+learnable
+Linux
+maxout
+memoization
+memoize
+memoized
+memoizes
+memoizing
+minibatch
+mlpconv
+mnist
+multi
+multiclass
+namespace
+NaN
+normalizer
+npz
+NumPy
+online
+optimizer
+optimizers
+outliers
+Overfeat
+parallelization
+parallelizations
+parallelize
+parallelized
+parallelizes
+parallelizing
+parameterize
+parameterized
+perceptron
+Perf
+permutate
+permutates
+pluggable
+postprocess
+pre
+preprocessing
+profiler
+profilers
+proto
+PRs
+quantization
+reimplement
+reinstall
+resized
+runtime
+serializable
+serializer
+serializers
+sharding
+siamese
+sigmoid
+softmax
+stateful
+subdirectories
+subdirectory
+summarization
+th
+Theano
+treebank
+tuple
+tuples
+ufunc
+ufuncs
+unigram
+uninstall
+unnormalize
+unnormalized
+unpooling
+unregisters
+unuseful
+variadic
+variational
+vecLib
+vectorized
+versioning
+workspace
+
+Bengio
+Cho
+Courville
+farley
+Goodfellow
+Jurgen
+Mirza
+Nesterov
+Schmidhuber
+Socher
+Warde
+Zeiler
diff --git a/docs/source/upgrade.rst b/docs/source/upgrade.rst
new file mode 100644
index 0000000..2bcfe4b
--- /dev/null
+++ b/docs/source/upgrade.rst
@@ -0,0 +1,709 @@
+=============
+Upgrade Guide
+=============
+
+This page covers changes introduced in each major version that users should know when migrating from older releases.
+Please see also the :ref:`compatibility_matrix` for supported environments of each major version.
+
+
+CuPy v13
+========
+
+Modernized CCCL support and requirement
+---------------------------------------
+
+NVIDIA's CUDA C++ Core Libraries (CCCL) is the new home for the inter-dependent C++ libraries Thrust, CUB, and libcu++ that are shipped
+with CUDA Toolkit 11.0+. To better serve our users with the latest CCCL features, improvements, and bug fixes, starting CuPy v13
+we bundle CCCL in the source and binary (pip/conda) releases of CuPy. The same version of CCCL is used at both build-time (for building
+CuPy) and run-time (for JIT-compiling kernels). This ensures uniform behavior, avoids surprises, and allows dual CUDA support as promised
+by CCCL (currently CUDA 11 & 12), but this change leads to the following consequences distinct from the past releases:
+
+* after the upgrade, the very first time of executing certain CuPy features may take longer than usual;
+* the CCCL from any local CUDA installation is now ignored on purpose, either at build- or run- time;
+* adventurous users who want to experiment with local CCCL changes need to update the CCCL submodule and build CuPy from source;
+
+As a result of this movement, CuPy now follows the same compiler requirement as CCCL (and, in turn, CUDA Toolkit) and requires C++11 as
+the lowest C++ standard. CCCL expects to move to C++17 in the near future.
+
+Requirement Changes
+-------------------
+
+The following versions are no longer supported in CuPy v13.
+
+* CUDA 11.1 or earlier
+* cuDNN 8.7 or earlier
+* cuTENSOR 1.x
+    * Support for cuTENSOR 2.0 is added starting with CuPy v13, and support for cuTENSOR 1.x will be dropped.
+      This is because there are significant API changes from cuTENSOR 1.x to 2.0, and from the maintenance perspective, it is not practical to support both cuTENSOR 1.x and 2.0 APIs simultaneously.
+* Python 3.8 or earlier
+* NumPy 1.21 or earlier
+* Ubuntu 18.04
+
+NumPy/SciPy Baseline API Update
+-------------------------------
+
+Baseline API has been bumped from NumPy 1.24 and SciPy 1.9 to NumPy 1.26 and SciPy 1.11.
+CuPy v13 will follow the upstream products' specifications of these baseline versions.
+
+Change in :func:`cupy.asnumpy`/:meth:`cupy.ndarray.get` Behavior
+----------------------------------------------------------------
+
+When transferring a CuPy array from GPU to CPU (as a NumPy array), previously the transfer could be nonblocking and not properly ordered when a non-default stream is in use,
+leading to potential data race if the resulting array is modified on host immediately after the copy starts. In CuPy v13, the default
+behavior is changed to be always blocking, with a new optional argument ``blocking`` added to allow the previous nonblocking behavior
+if set to ``False``, in which case users are responsible for ensuring proper stream order.
+
+Change in :meth:`cupy.array`/:meth:`cupy.asarray`/:meth:`cupy.asanyarray` Behavior
+----------------------------------------------------------------------------------
+
+When transferring a NumPy array from CPU to GPU, previously the transfer was always blocking even if the source array is backed by pinned memory.
+In CuPy v13, the default behavior is changed to be asynchronous if the source array is allocated as pinned to improve the performance.
+
+A new optional argument ``blocking`` has been added to allow the previous blocking behavior if set to ``True``.
+You might want to set this option in case there is a possibility of overwriting the source array on CPU before the transfer completes.
+
+Removal of ``cupy-wheel`` package
+---------------------------------
+
+The ``cupy-wheel`` package, which aimed to serve as a "meta" package that chooses and installs the right CuPy binary packages for the users' environment, has been removed in CuPy v13.
+This is because the recent Pip no longer allows changing requirements dynamically.
+See `#7628 <https://github.com/cupy/cupy/issues/7628>`_ for the details.
+
+API Changes
+-----------
+
+* An *internal and undocumented* API :func:`cupy.cuda.compile_with_cache`, which was marked deprecated in CuPy v10, has been removed.
+  We encourage downstream libraries and users to migrate to use public APIs, such as :class:`~cupy.RawModule` (added in CuPy v7) or :class:`~cupy.RawKernel` (added in CuPy v5).
+  See :doc:`./user_guide/kernel` for their tutorials.
+
+
+CUDA Runtime API is now statically linked
+-----------------------------------------
+
+CuPy is now shipped with CUDA Runtime statically linked.
+Due to this, :func:`cupy.cuda.runtime.runtimeGetVersion` always returns the version of CUDA Runtime that CuPy is built with, regardless of the version of CUDA Runtime installed locally.
+If you need to retrieve the version of CUDA Runtime shared library installed locally, use :func:`cupy.cuda.get_local_runtime_version` instead.
+
+Update of Docker Images
+-----------------------
+
+CuPy official Docker images (see :doc:`install` for details) are now updated to use CUDA 12.2.
+
+
+CuPy v12
+========
+
+Change in :class:`cupy.cuda.Device` Behavior
+--------------------------------------------
+
+The CUDA current device (set via :meth:`cupy.cuda.Device.use()` or ``cudaSetDevice()``) will be reactivated when exiting a device context manager.
+This reverts the :ref:`change introduced in CuPy v10 <change in CuPy Device behavior>`, making the behavior identical to the one in CuPy v9 or earlier.
+
+This decision was made for better interoperability with other libraries that might mutate the current CUDA device.
+Suppose the following code:
+
+.. code-block:: py
+
+   def do_preprocess_cupy():
+       with cupy.cuda.Device(2):
+           # ...
+           pass
+
+   torch.cuda.set_device(1)
+   do_preprocess_cupy()
+   print(torch.cuda.get_device())  # -> ???
+
+In CuPy v10 and v11, the code prints ``0``, which can be surprising for users.
+In CuPy v12, the code now prints ``1``, making it easy for both users and library developers to maintain the current device where multiple devices are involved.
+
+Deprecation of ``cupy.ndarray.scatter_{add,max,min}``
+-----------------------------------------------------
+
+These APIs have been marked as deprecated as ``cupy.{add,maximum,minimum}.at`` ufunc methods have been implemented, which behave as equivalent and NumPy-compatible.
+
+Requirement Changes
+-------------------
+
+The following versions are no longer supported in CuPy v12.
+
+* Python 3.7 or earlier
+* NumPy 1.20 or earlier
+* SciPy 1.6 or earlier
+
+Baseline API Update
+-------------------
+
+Baseline API has been bumped from NumPy 1.23 and SciPy 1.8 to NumPy 1.24 and SciPy 1.9.
+CuPy v12 will follow the upstream products' specifications of these baseline versions.
+
+Update of Docker Images
+-----------------------
+
+CuPy official Docker images (see :doc:`install` for details) are now updated to use CUDA 11.8.
+
+
+CuPy v11
+========
+
+Unified Binary Package for CUDA 11.2+
+-------------------------------------
+
+CuPy v11 provides a unified binary package named ``cupy-cuda11x`` that supports all CUDA 11.2+ releases.
+This replaces per-CUDA version binary packages (``cupy-cuda112`` ~ ``cupy-cuda117``).
+
+Note that CUDA 11.1 or earlier still requires per-CUDA version binary packages.
+``cupy-cuda102``, ``cupy-cuda110``, and ``cupy-cuda111`` will be provided for CUDA 10.2, 11.0, and 11.1, respectively.
+
+Requirement Changes
+-------------------
+
+The following versions are no longer supported in CuPy v11.
+
+* ROCm 4.2 or earlier
+* NumPy 1.19 or earlier
+* SciPy 1.5 or earlier
+
+CUB Enabled by Default
+----------------------
+
+CuPy v11 accelerates the computation with CUB by default.
+In case needed, you can turn it off by setting :envvar:`CUPY_ACCELERATORS` environment variable to ``""``.
+
+Baseline API Update
+-------------------
+
+Baseline API has been bumped from NumPy 1.21 and SciPy 1.7 to NumPy 1.23 and SciPy 1.8.
+CuPy v11 will follow the upstream products' specifications of these baseline versions.
+
+Update of Docker Images
+-----------------------
+
+CuPy official Docker images (see :doc:`install` for details) are now updated to use CUDA 11.7 and ROCm 5.0.
+
+
+CuPy v10
+========
+
+Dropping CUDA 9.2 / 10.0 / 10.1 Support
+---------------------------------------
+
+CUDA 10.1 or earlier is no longer supported.
+Use CUDA 10.2 or later.
+
+Dropping NCCL v2.4 / v2.6 / v2.7 Support
+----------------------------------------
+
+NCCL v2.4, v2.6, and v2.7 are no longer supported.
+
+Dropping Python 3.6 Support
+---------------------------
+
+Python 3.6 is no longer supported.
+
+Dropping NumPy 1.17 Support
+---------------------------
+
+NumPy 1.17 is no longer supported.
+
+.. _change in CuPy Device behavior:
+
+Change in :class:`cupy.cuda.Device` Behavior
+--------------------------------------------
+
+Current device set via :meth:`~cupy.cuda.Device.use` will not be honored by the ``with Device`` block
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. note::
+   This change has been reverted in CuPy v12. See **CuPy v12** section above for details.
+
+The current device set via :meth:`cupy.cuda.Device.use()` will not be reactivated when exiting a device context manager. An existing code mixing ``with device:`` block and ``device.use()`` may get different results between CuPy v10 and v9.
+
+.. code-block:: py
+
+   cupy.cuda.Device(1).use()
+   with cupy.cuda.Device(0):
+       pass
+   cupy.cuda.Device()  # -> CuPy v10 returns device 0 instead of device 1
+
+This decision was made to serve CuPy *users* better, but it could lead to surprises to downstream *developers* depending on CuPy,
+as essentially CuPy's :class:`~cupy.cuda.Device` context manager no longer respects the CUDA ``cudaSetDevice()`` API. Mixing
+device management functionalities (especially using context manager) from different libraries is highly discouraged.
+
+For downstream libraries that still wish to respect the ``cudaGetDevice()``/``cudaSetDevice()`` APIs, you should avoid managing
+current devices using the ``with Device`` context manager, and instead calling these APIs explicitly, see for example
+`cupy/cupy#5963 <https://github.com/cupy/cupy/pull/5963>`_.
+
+Changes in :class:`cupy.cuda.Stream` Behavior
+---------------------------------------------
+
+Stream is now managed per-device
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Previoulys, it was users' responsibility to keep the current stream consistent with the current CUDA device.
+For example, the following code raises an error in CuPy v9 or earlier:
+
+.. code-block:: py
+
+   import cupy
+
+   with cupy.cuda.Device(0):
+       # Create a stream on device 0.
+       s0 = cupy.cuda.Stream()
+
+   with cupy.cuda.Device(1):
+       with s0:
+           # Try to use the stream on device 1
+           cupy.arange(10)  # -> CUDA_ERROR_INVALID_HANDLE: invalid resource handle
+
+CuPy v10 manages the current stream per-device, thus eliminating the need of switching the stream every time the active device is changed.
+When using CuPy v10, the above example behaves differently because whenever a stream is created, it is automatically associated with the current device and will be ignored when switching devices. 
+In early versions, trying to use `s0` in device 1 raises an error because `s0` is associated with device 0. However, in v10, this `s0` is ignored and the default stream for device 1 will be used instead.
+
+Current stream set via ``use()`` will not be restored when exiting ``with`` block
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Samely as the change of :class:`cupy.cuda.Device` above, the current stream set via :func:`cupy.cuda.Stream.use` will not be reactivated when exiting a stream context manager.
+An existing code mixing ``with stream:`` block and ``stream.use()`` may get different results between CuPy v10 and v9.
+
+.. code-block:: py
+
+   s1 = cupy.cuda.Stream()
+   s2 = cupy.cuda.Stream()
+   s3 = cupy.cuda.Stream()
+   with s1:
+       s2.use()
+       with s3:
+           pass
+       cupy.cuda.get_current_stream()  # -> CuPy v10 returns `s1` instead of `s2`.
+
+Streams can now be shared between threads
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The same :class:`cupy.cuda.Stream` instance can now safely be shared between multiple threads.
+
+To achieve this, CuPy v10 will not destroy the stream (``cudaStreamDestroy``) if the stream is the current stream of any thread.
+
+Big-Endian Arrays Automatically Converted to Little-Endian
+----------------------------------------------------------
+
+:func:`cupy.array`, :func:`cupy.asarray` and its variants now always transfer the data to GPU in little-endian byte order.
+
+Previously CuPy was copying the given :class:`numpy.ndarray` to GPU as-is, regardless of the endianness.
+In CuPy v10, big-endian arrays are converted to little-endian before the transfer, which is the native byte order on GPUs.
+This change eliminates the need to manually change the array endianness before creating the CuPy array.
+
+Baseline API Update
+-------------------
+
+Baseline API has been bumped from NumPy 1.20 and SciPy 1.6 to NumPy 1.21 and SciPy 1.7.
+CuPy v10 will follow the upstream products' specifications of these baseline versions.
+
+API Changes
+-----------
+
+* Device synchronize detection APIs (:func:`cupyx.allow_synchronize` and :class:`cupyx.DeviceSynchronized`), introduced as an experimental feature in CuPy v8, have been marked as deprecated because it is impossible to detect synchronizations reliably.
+
+* An *internal* API :func:`cupy.cuda.compile_with_cache` has been marked as deprecated as there are better alternatives (see :class:`~cupy.RawModule` added since CuPy v7 and :class:`~cupy.RawKernel` since v5). While it has a longstanding history, this API has never been meant to be public. We encourage downstream libraries and users to migrate to the aforementioned public APIs. See :doc:`./user_guide/kernel` for their tutorials.
+
+* The DLPack routine :func:`cupy.fromDlpack` is deprecated in favor of :func:`cupy.from_dlpack`, which addresses potential data race issues.
+
+* A new module :mod:`cupyx.profiler` is added to host all profiling related APIs in CuPy. Accordingly, the following APIs are relocated to this module as follows. The old routines are deprecated.
+
+    * :func:`cupy.prof.TimeRangeDecorator` -> :func:`cupyx.profiler.time_range`
+    * :func:`cupy.prof.time_range` -> :func:`cupyx.profiler.time_range`
+    * :func:`cupy.cuda.profile` -> :func:`cupyx.profiler.profile`
+    * :func:`cupyx.time.repeat` -> :func:`cupyx.profiler.benchmark`
+
+* :func:`cupy.ndarray.__pos__` now returns a copy (samely as :func:`cupy.positive`) instead of returning ``self``.
+
+Note that deprecated APIs may be removed in the future CuPy releases.
+
+Update of Docker Images
+-----------------------
+
+CuPy official Docker images (see :doc:`install` for details) are now updated to use CUDA 11.4 and ROCm 4.3.
+
+CuPy v9
+=======
+
+Dropping Support of CUDA 9.0
+----------------------------
+
+CUDA 9.0 is no longer supported.
+Use CUDA 9.2 or later.
+
+Dropping Support of cuDNN v7.5 and NCCL v2.3
+--------------------------------------------
+
+cuDNN v7.5 (or earlier) and NCCL v2.3 (or earlier) are no longer supported.
+
+Dropping Support of NumPy 1.16 and SciPy 1.3
+--------------------------------------------
+
+NumPy 1.16 and SciPy 1.3 are no longer supported.
+
+Dropping Support of Python 3.5
+------------------------------
+
+Python 3.5 is no longer supported in CuPy v9.
+
+NCCL and cuDNN No Longer Included in Wheels
+-------------------------------------------
+
+NCCL and cuDNN shared libraires are no longer included in wheels (see `#4850 <https://github.com/cupy/cupy/issues/4850>`_ for discussions). 
+You can manually install them after installing wheel if you don't have a previous installation; see :doc:`install` for details.
+
+cuTENSOR Enabled in Wheels
+--------------------------
+
+cuTENSOR can now be used when installing CuPy via wheels.
+
+``cupy.cuda.{nccl,cudnn}`` Modules Needs Explicit Import
+--------------------------------------------------------
+
+Previously ``cupy.cuda.nccl`` and ``cupy.cuda.cudnn`` modules were automatically imported.
+Since CuPy v9, these modules need to be explicitly imported (i.e., ``import cupy.cuda.nccl`` / ``import cupy.cuda.cudnn``.)
+
+Baseline API Update
+-------------------
+
+Baseline API has been bumped from NumPy 1.19 and SciPy 1.5 to NumPy 1.20 and SciPy 1.6.
+CuPy v9 will follow the upstream products' specifications of these baseline versions.
+
+Following NumPy 1.20, aliases for the Python scalar types (``cupy.bool``, ``cupy.int``, ``cupy.float``, and ``cupy.complex``) are now deprecated.
+``cupy.bool_``, ``cupy.int_``, ``cupy.float_`` and ``cupy.complex_`` should be used instead when required.
+
+Update of Docker Images
+-----------------------
+
+CuPy official Docker images (see :doc:`install` for details) are now updated to use CUDA 11.2 and Python 3.8.
+
+
+CuPy v8
+=======
+
+Dropping Support of CUDA 8.0 and 9.1
+------------------------------------
+
+CUDA 8.0 and 9.1 are no longer supported.
+Use CUDA 9.0, 9.2, 10.0, or later.
+
+Dropping Support of NumPy 1.15 and SciPy 1.2
+--------------------------------------------
+
+NumPy 1.15 (or earlier) and SciPy 1.2 (or earlier) are no longer supported.
+
+Update of Docker Images
+-----------------------
+
+* CuPy official Docker images (see :doc:`install` for details) are now updated to use CUDA 10.2 and Python 3.6.
+* SciPy and Optuna are now pre-installed.
+
+CUB Support and Compiler Requirement
+------------------------------------
+
+CUB module is now built by default.
+You can enable the use of CUB by setting ``CUPY_ACCELERATORS="cub"`` (see :envvar:`CUPY_ACCELERATORS` for details).
+
+Due to this change, g++-6 or later is required when building CuPy from the source.
+See :doc:`install` for details.
+
+The following environment variables are no longer effective:
+
+* ``CUB_DISABLED``: Use :envvar:`CUPY_ACCELERATORS` as aforementioned.
+* ``CUB_PATH``: No longer required as CuPy uses either the CUB source bundled with CUDA (only when using CUDA 11.0 or later) or the one in the CuPy distribution.
+
+API Changes
+-----------
+
+* ``cupy.scatter_add``, which was deprecated in CuPy v4, has been removed. Use :func:`cupyx.scatter_add` instead.
+* ``cupy.sparse`` module has been deprecated and will be removed in future releases. Use :mod:`cupyx.scipy.sparse` instead.
+* ``dtype`` argument of :func:`cupy.ndarray.min` and :func:`cupy.ndarray.max` has been removed to align with the NumPy specification.
+* :func:`cupy.allclose` now returns the result as 0-dim GPU array instead of Python bool to avoid device synchronization.
+* :class:`cupy.RawModule` now delays the compilation to the time of the first call to align the behavior with :class:`cupy.RawKernel`.
+* ``cupy.cuda.*_enabled`` flags (``nccl_enabled``, ``nvtx_enabled``, etc.) has been deprecated. Use ``cupy.cuda.*.available`` flag (``cupy.cuda.nccl.available``, ``cupy.cuda.nvtx.available``, etc.) instead.
+* ``CHAINER_SEED`` environment variable is no longer effective. Use ``CUPY_SEED`` instead.
+
+
+CuPy v7
+=======
+
+Dropping Support of Python 2.7 and 3.4
+--------------------------------------
+
+Starting from CuPy v7, Python 2.7 and 3.4 are no longer supported as it reaches its end-of-life (EOL) in January 2020 (2.7) and March 2019 (3.4).
+Python 3.5.1 is the minimum Python version supported by CuPy v7.
+Please upgrade the Python version if you are using affected versions of Python to any later versions listed under :doc:`install`.
+
+
+CuPy v6
+=======
+
+Binary Packages Ignore ``LD_LIBRARY_PATH``
+------------------------------------------
+
+Prior to CuPy v6, ``LD_LIBRARY_PATH`` environment variable can be used to override cuDNN / NCCL libraries bundled in the binary distribution (also known as wheels).
+In CuPy v6, ``LD_LIBRARY_PATH`` will be ignored during discovery of cuDNN / NCCL; CuPy binary distributions always use libraries that comes with the package to avoid errors caused by unexpected override.
+
+
+CuPy v5
+=======
+
+``cupyx.scipy`` Namespace
+-------------------------
+
+:mod:`cupyx.scipy` namespace has been introduced to provide CUDA-enabled SciPy functions.
+:mod:`cupy.sparse` module has been renamed to :mod:`cupyx.scipy.sparse`; :mod:`cupy.sparse` will be kept as an alias for backward compatibility.
+
+Dropped Support for CUDA 7.0 / 7.5
+----------------------------------
+
+CuPy v5 no longer supports CUDA 7.0 / 7.5.
+
+Update of Docker Images
+-----------------------
+
+CuPy official Docker images (see :doc:`install` for details) are now updated to use CUDA 9.2 and cuDNN 7.
+
+To use these images, you may need to upgrade the NVIDIA driver on your host.
+See `Requirements of nvidia-docker <https://github.com/NVIDIA/nvidia-docker/wiki/CUDA#requirements>`_ for details.
+
+
+CuPy v4
+=======
+
+.. note::
+
+   The version number has been bumped from v2 to v4 to align with the versioning of Chainer.
+   Therefore, CuPy v3 does not exist.
+
+Default Memory Pool
+-------------------
+
+Prior to CuPy v4, memory pool was only enabled by default when CuPy is used with Chainer.
+In CuPy v4, memory pool is now enabled by default, even when you use CuPy without Chainer.
+The memory pool significantly improves the performance by mitigating the overhead of memory allocation and CPU/GPU synchronization.
+
+.. attention::
+
+   When you monitor GPU memory usage (e.g., using ``nvidia-smi``), you may notice that GPU memory not being freed even after the array instance become out of scope.
+   This is expected behavior, as the default memory pool "caches" the allocated memory blocks.
+
+To access the default memory pool instance, use :func:`get_default_memory_pool` and :func:`get_default_pinned_memory_pool`.
+You can access the statistics and free all unused memory blocks "cached" in the memory pool.
+
+.. code-block:: py
+
+   import cupy
+   a = cupy.ndarray(100, dtype=cupy.float32)
+   mempool = cupy.get_default_memory_pool()
+
+   # For performance, the size of actual allocation may become larger than the requested array size.
+   print(mempool.used_bytes())   # 512
+   print(mempool.total_bytes())  # 512
+
+   # Even if the array goes out of scope, its memory block is kept in the pool.
+   a = None
+   print(mempool.used_bytes())   # 0
+   print(mempool.total_bytes())  # 512
+
+   # You can clear the memory block by calling `free_all_blocks`.
+   mempool.free_all_blocks()
+   print(mempool.used_bytes())   # 0
+   print(mempool.total_bytes())  # 0
+
+You can even disable the default memory pool by the code below.
+Be sure to do this before any other CuPy operations.
+
+.. code-block:: py
+
+   import cupy
+   cupy.cuda.set_allocator(None)
+   cupy.cuda.set_pinned_memory_allocator(None)
+
+Compute Capability
+------------------
+
+CuPy v4 now requires NVIDIA GPU with Compute Capability 3.0 or larger.
+See the `List of CUDA GPUs <https://developer.nvidia.com/cuda-gpus>`_ to check if your GPU supports Compute Capability 3.0.
+
+
+CUDA Stream
+-----------
+
+As CUDA Stream is fully supported in CuPy v4, ``cupy.cuda.RandomState.set_stream``, the function to change the stream used by the random number generator, has been removed.
+Please use :func:`cupy.cuda.Stream.use` instead.
+
+See the discussion in `#306 <https://github.com/cupy/cupy/pull/306>`_ for more details.
+
+``cupyx`` Namespace
+-------------------
+
+``cupyx`` namespace has been introduced to provide features specific to CuPy (i.e., features not provided in NumPy) while avoiding collision in future.
+See :doc:`reference/ext` for the list of such functions.
+
+For this rule, :func:`cupy.scatter_add` has been moved to :func:`cupyx.scatter_add`.
+:func:`cupy.scatter_add` is still available as an alias, but it is encouraged to use :func:`cupyx.scatter_add` instead.
+
+Update of Docker Images
+-----------------------
+
+CuPy official Docker images (see :doc:`install` for details) are now updated to use CUDA 8.0 and cuDNN 6.0.
+This change was introduced because CUDA 7.5 does not support NVIDIA Pascal GPUs.
+
+To use these images, you may need to upgrade the NVIDIA driver on your host.
+See `Requirements of nvidia-docker <https://github.com/NVIDIA/nvidia-docker/wiki/CUDA#requirements>`_ for details.
+
+CuPy v2
+=======
+
+Changed Behavior of count_nonzero Function
+------------------------------------------
+
+For performance reasons, :func:`cupy.count_nonzero` has been changed to return zero-dimensional :class:`ndarray` instead of `int` when `axis=None`.
+See the discussion in `#154 <https://github.com/cupy/cupy/pull/154>`_ for more details.
+
+
+.. _compatibility_matrix:
+
+Compatibility Matrix
+====================
+
+.. list-table::
+   :header-rows: 1
+
+   * - CuPy
+     - CC [1]_
+     - CUDA
+     - ROCm
+     - cuTENSOR
+     - NCCL
+     - cuDNN
+     - Python
+     - NumPy
+     - SciPy
+     - Baseline API Spec.
+     - Docs
+   * - v13
+     - 3.5~
+     - 11.2~
+     - 4.3~
+     - 2.0~
+     - 2.16~
+     - 8.8~
+     - 3.9~
+     - 1.22~
+     - 1.7~
+     - NumPy 1.26 & SciPy 1.11
+     - `latest <https://docs.cupy.dev/en/latest/install.html>`__
+   * - v12
+     - 3.0~9.0
+     - 10.2~12.x
+     - 4.3 & 5.0
+     - 1.4~1.7
+     - 2.8~2.17
+     - 7.6~8.8
+     - 3.8~3.12
+     - 1.21~1.26
+     - 1.7~1.11
+     - NumPy 1.24 & SciPy 1.9
+     - `stable <https://docs.cupy.dev/en/stable/install.html>`__
+   * - v11
+     - 3.0~9.0
+     - 10.2~12.0
+     - 4.3 & 5.0
+     - 1.4~1.6
+     - 2.8~2.16
+     - 7.6~8.7
+     - 3.7~3.11
+     - 1.20~1.24
+     - 1.6~1.9
+     - NumPy 1.23 & SciPy 1.8
+     - `v11.6.0 <https://docs.cupy.dev/en/v11.6.0/install.html>`__
+   * - v10
+     - 3.0~8.x
+     - 10.2~11.7
+     - 4.0 & 4.2 & 4.3 & 5.0
+     - 1.3~1.5
+     - 2.8~2.11
+     - 7.6~8.4
+     - 3.7~3.10
+     - 1.18~1.22
+     - 1.4~1.8
+     - NumPy 1.21 & SciPy 1.7
+     - `v10.6.0 <https://docs.cupy.dev/en/v10.6.0/install.html>`__
+   * - v9
+     - 3.0~8.x
+     - 9.2~11.5
+     - 3.5~4.3
+     - 1.2~1.3
+     - 2.4 & 2.6~2.11
+     - 7.6~8.2
+     - 3.6~3.9
+     - 1.17~1.21
+     - 1.4~1.7
+     - NumPy 1.20 & SciPy 1.6
+     - `v9.6.0 <https://docs.cupy.dev/en/v9.6.0/install.html>`__
+   * - v8
+     - 3.0~8.x
+     - 9.0 & 9.2~11.2
+     - 3.x [2]_
+     - 1.2
+     - 2.0~2.8
+     - 7.0~8.1
+     - 3.5~3.9
+     - 1.16~1.20
+     - 1.3~1.6
+     - NumPy 1.19 & SciPy 1.5
+     - `v8.6.0 <https://docs.cupy.dev/en/v8.6.0/install.html>`__
+   * - v7
+     - 3.0~8.x
+     - 8.0~11.0
+     - 2.x [2]_
+     - 1.0
+     - 1.3~2.7
+     - 5.0~8.0
+     - 3.5~3.8
+     - 1.9~1.19
+     - (not specified)
+     - (not specified)
+     - `v7.8.0 <https://docs.cupy.dev/en/v7.8.0/install.html>`__
+   * - v6
+     - 3.0~7.x
+     - 8.0~10.1
+     - n/a
+     - n/a
+     - 1.3~2.4
+     - 5.0~7.5
+     - 2.7 & 3.4~3.8
+     - 1.9~1.17
+     - (not specified)
+     - (not specified)
+     - `v6.7.0 <https://docs.cupy.dev/en/v6.7.0/install.html>`__
+   * - v5
+     - 3.0~7.x
+     - 8.0~10.1
+     - n/a
+     - n/a
+     - 1.3~2.4
+     - 5.0~7.5
+     - 2.7 & 3.4~3.7
+     - 1.9~1.16
+     - (not specified)
+     - (not specified)
+     - `v5.4.0 <https://docs.cupy.dev/en/v5.4.0/install.html>`__
+   * - v4
+     - 3.0~7.x
+     - 7.0~9.2
+     - n/a
+     - n/a
+     - 1.3~2.2
+     - 4.0~7.1
+     - 2.7 & 3.4~3.6
+     - 1.9~1.14
+     - (not specified)
+     - (not specified)
+     - `v4.5.0 <https://docs.cupy.dev/en/v4.5.0/install.html>`__
+
+.. [1] CUDA Compute Capability
+.. [2] Highly experimental support with limited features.
diff --git a/docs/source/user_guide/basic.rst b/docs/source/user_guide/basic.rst
new file mode 100644
index 0000000..d0288f2
--- /dev/null
+++ b/docs/source/user_guide/basic.rst
@@ -0,0 +1,222 @@
+Basics of CuPy
+==============
+
+.. currentmodule:: cupy
+
+In this section, you will learn about the following things:
+
+* Basics of :class:`cupy.ndarray`
+* The concept of *current device*
+* host-device and device-device array transfer
+
+
+Basics of cupy.ndarray
+----------------------
+
+CuPy is a GPU array backend that implements a subset of NumPy interface.
+In the following code, ``cp`` is an abbreviation of ``cupy``, following the standard convention of abbreviating ``numpy`` as ``np``:
+
+.. doctest::
+
+   >>> import numpy as np
+   >>> import cupy as cp
+
+The :class:`cupy.ndarray` class is at the core of ``CuPy`` and is a replacement class for ``NumPy``'s :class:`numpy.ndarray`.
+
+.. doctest::
+
+   >>> x_gpu = cp.array([1, 2, 3])
+
+``x_gpu`` above is an instance of :class:`cupy.ndarray`.
+As one can see, CuPy's syntax here is identical to that of NumPy.
+The main difference between :class:`cupy.ndarray` and :class:`numpy.ndarray` is that
+the CuPy arrays are allocated on the *current device*, which we will talk about later.
+
+Most of the array manipulations are also done in the way similar to NumPy.
+Take the Euclidean norm (a.k.a L2 norm), for example.
+NumPy has :func:`numpy.linalg.norm` function that calculates it on CPU.
+
+.. doctest::
+
+   >>> x_cpu = np.array([1, 2, 3])
+   >>> l2_cpu = np.linalg.norm(x_cpu)
+
+Using CuPy, we can perform the same calculations on GPU in a similar way:
+
+.. doctest::
+
+   >>> x_gpu = cp.array([1, 2, 3])
+   >>> l2_gpu = cp.linalg.norm(x_gpu)
+
+CuPy implements many functions on :class:`cupy.ndarray` objects.
+See the :ref:`reference <cupy_reference>` for the supported subset of NumPy API.
+Knowledge of NumPy will help you utilize most of the CuPy features.
+We, therefore, recommend you familiarize yourself with the `NumPy documentation <https://numpy.org/doc/stable/index.html>`_.
+
+
+Current Device
+--------------
+
+CuPy has a concept of a *current device*, which is the default GPU device on which
+the allocation, manipulation, calculation, etc., of arrays take place.
+Suppose ID of the current device is 0.
+In such a case, the following code would create an array ``x_on_gpu0`` on GPU 0.
+
+.. doctest::
+
+   >>> x_on_gpu0 = cp.array([1, 2, 3, 4, 5])
+
+To switch to another GPU device, use the :class:`~cupy.cuda.Device` context manager:
+
+.. doctest::
+
+   >>> with cp.cuda.Device(1):
+   ...    x_on_gpu1 = cp.array([1, 2, 3, 4, 5])
+   >>> x_on_gpu0 = cp.array([1, 2, 3, 4, 5])
+
+All CuPy operations (except for multi-GPU features and device-to-device copy) are performed on the currently active device.
+
+In general, CuPy functions expect that the array is on the same device as the current one.
+Passing an array stored on a non-current device may work depending on the hardware configuration but is generally discouraged as it may not be performant.
+
+.. note::
+  If the array's device and the current device mismatch, CuPy functions try to establish `peer-to-peer memory access <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#peer-to-peer-memory-access>`_ (P2P) between them so that the current device can directly read the array from another device.
+  Note that P2P is available only when the topology permits it.
+  If P2P is unavailable, such an attempt will fail with ``ValueError``.
+
+``cupy.ndarray.device`` attribute indicates the device on which the array is allocated.
+
+.. doctest::
+
+   >>> with cp.cuda.Device(1):
+   ...    x = cp.array([1, 2, 3, 4, 5])
+   >>> x.device
+   <CUDA Device 1>
+
+.. note::
+
+   When only one device is available, explicit device switching is not needed.
+
+
+.. _current_stream:
+
+Current Stream
+--------------
+
+Associated with the concept of current devices are *current streams*, which help avoid explicitly passing streams
+in every single operation so as to keep the APIs pythonic and user-friendly. In CuPy, all CUDA operations
+such as data transfer (see the :ref:`data-transfer-basics` section) and kernel launches are enqueued onto the current stream,
+and the queued tasks on the same stream will be executed in serial (but *asynchronously* with respect to the host).
+
+The default current stream in CuPy is CUDA's null stream (i.e., stream 0). It is also known as the *legacy*
+default stream, which is unique per device. However, it is possible to change the current stream using the
+:class:`cupy.cuda.Stream` API, please see :doc:`cuda_api` for example. The current stream in CuPy can be
+retrieved using :func:`cupy.cuda.get_current_stream`.
+
+It is worth noting that CuPy's current stream is managed on a *per thread, per device* basis, meaning that on different
+Python threads or different devices the current stream (if not the null stream) can be different.
+
+.. _data-transfer-basics:
+
+Data Transfer
+-------------
+
+Move arrays to a device
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:func:`cupy.asarray` can be used to move a :class:`numpy.ndarray`, a list, or any object
+that can be passed to :func:`numpy.array` to the current device:
+
+.. doctest::
+
+   >>> x_cpu = np.array([1, 2, 3])
+   >>> x_gpu = cp.asarray(x_cpu)  # move the data to the current device.
+
+:func:`cupy.asarray` can accept :class:`cupy.ndarray`, which means we can
+transfer the array between devices with this function.
+
+.. doctest::
+
+   >>> with cp.cuda.Device(0):
+   ...     x_gpu_0 = cp.ndarray([1, 2, 3])  # create an array in GPU 0
+   >>> with cp.cuda.Device(1):
+   ...     x_gpu_1 = cp.asarray(x_gpu_0)  # move the array to GPU 1
+
+.. note::
+
+   :func:`cupy.asarray` does not copy the input array if possible.
+   So, if you put an array of the current device, it returns the input object itself.
+
+   If we do copy the array in this situation, you can use :func:`cupy.array` with `copy=True`.
+   Actually :func:`cupy.asarray` is equivalent to `cupy.array(arr, dtype, copy=False)`.
+
+Move array from a device to the host
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Moving a device array to the host can be done by :func:`cupy.asnumpy` as follows:
+
+.. doctest::
+
+   >>> x_gpu = cp.array([1, 2, 3])  # create an array in the current device
+   >>> x_cpu = cp.asnumpy(x_gpu)  # move the array to the host.
+
+We can also use :meth:`cupy.ndarray.get()`:
+
+.. doctest::
+
+   >>> x_cpu = x_gpu.get()
+
+
+Memory management
+-----------------
+
+Check :doc:`./memory` for a detailed description of how memory is managed in CuPy
+using memory pools.
+
+
+How to write CPU/GPU agnostic code
+----------------------------------
+
+CuPy's compatibility with NumPy makes it possible to write CPU/GPU agnostic code.
+For this purpose, CuPy implements the :func:`cupy.get_array_module` function that
+returns a reference to :mod:`cupy` if any of its arguments resides on a GPU
+and :mod:`numpy` otherwise.
+Here is an example of a CPU/GPU agnostic function that computes ``log1p``:
+
+.. doctest::
+
+   >>> # Stable implementation of log(1 + exp(x))
+   >>> def softplus(x):
+   ...     xp = cp.get_array_module(x)  # 'xp' is a standard usage in the community
+   ...     print("Using:", xp.__name__)
+   ...     return xp.maximum(0, x) + xp.log1p(xp.exp(-abs(x)))
+
+When you need to manipulate CPU and GPU arrays, an explicit data
+transfer may be required to move them to the same location -- either CPU or GPU.
+For this purpose, CuPy implements two sister methods called :func:`cupy.asnumpy`  and
+:func:`cupy.asarray`. Here is an example that demonstrates the use of both methods:
+
+.. doctest::
+
+   >>> x_cpu = np.array([1, 2, 3])
+   >>> y_cpu = np.array([4, 5, 6])
+   >>> x_cpu + y_cpu
+   array([5, 7, 9])
+   >>> x_gpu = cp.asarray(x_cpu)
+   >>> x_gpu + y_cpu
+   Traceback (most recent call last):
+   ...
+   TypeError: Unsupported type <class 'numpy.ndarray'>
+   >>> cp.asnumpy(x_gpu) + y_cpu
+   array([5, 7, 9])
+   >>> cp.asnumpy(x_gpu) + cp.asnumpy(y_cpu)
+   array([5, 7, 9])
+   >>> x_gpu + cp.asarray(y_cpu)
+   array([5, 7, 9])
+   >>> cp.asarray(x_gpu) + cp.asarray(y_cpu)
+   array([5, 7, 9])
+
+The :func:`cupy.asnumpy` method returns a NumPy array (array on the host),
+whereas :func:`cupy.asarray` method returns a CuPy array (array on the current device).
+Both methods can accept arbitrary input, meaning that they can be applied to any data that
+is located on either the host or device and can be converted to an array.
diff --git a/docs/source/user_guide/compatibility.rst b/docs/source/user_guide/compatibility.rst
new file mode 100644
index 0000000..d2c3bb2
--- /dev/null
+++ b/docs/source/user_guide/compatibility.rst
@@ -0,0 +1,150 @@
+API Compatibility Policy
+========================
+
+This document expresses the design policy on compatibilities of CuPy APIs.
+Development team should obey this policy on deciding to add, extend, and change APIs and their behaviors.
+
+This document is written for both users and developers.
+Users can decide the level of dependencies on CuPy’s implementations in their codes based on this document.
+Developers should read through this document before creating pull requests that contain changes on the interface.
+Note that this document may contain ambiguities on the level of supported compatibilities.
+
+
+Versioning and Backward Compatibilities
+---------------------------------------
+
+The updates of CuPy are classified into three levels: major, minor, and revision.
+These types have distinct levels of backward compatibilities.
+
+- **Major update** contains disruptive changes that break the backward compatibility.
+- **Minor update** contains additions and extensions to the APIs that keep the backward compatibility supported.
+- **Revision update** contains improvements on the API implementations without changing any API specifications.
+
+Note that we do not support full backward compatibility, which is almost infeasible for Python-based APIs, since there is no way to completely hide the implementation details.
+
+
+Processes to Break Backward Compatibilities
+-------------------------------------------
+
+Deprecation, Dropping, and Its Preparation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Any APIs may be *deprecated* at some minor updates.
+In such a case, the deprecation note is added to the API documentation, and the API implementation is changed to fire a deprecation warning (if possible).
+There should be another way to reimplement the same functionality previously written using the deprecated APIs.
+
+Any APIs may be marked as *to be dropped in the future*.
+In such a case, the dropping is stated in the documentation with the major version number on which the API is planned to be dropped, and the API implementation is changed to fire a future warning (if possible).
+
+The actual dropping should be done through the following steps:
+
+- Make the API deprecated.
+  At this point, users should not use the deprecated API in their new application codes.
+- After that, mark the API as *to be dropped in the future*.
+  It must be done in the minor update different from that of the deprecation.
+- At the major version announced in the above update, drop the API.
+
+Consequently, it takes at least two minor versions to drop any APIs after the first deprecation.
+
+API Changes and Its Preparation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Any APIs may be marked as *to be changed in the future* for changes without backward compatibility.
+In such a case, the change is stated in the documentation with the version number on which the API is planned to be changed, and the API implementation is changed to fire the future warning on the certain usages.
+
+The actual change should be done in the following steps:
+
+- Announce that the API will be changed in the future.
+  At this point, the actual version of change need not be accurate.
+- After the announcement, mark the API as *to be changed in the future* with version number of planned changes.
+  At this point, users should not use the marked API in their new application codes.
+- At the major update announced in the above update, change the API.
+
+
+Supported Backward Compatibility
+--------------------------------
+
+This section defines backward compatibilities that minor updates must maintain.
+
+Documented Interface
+~~~~~~~~~~~~~~~~~~~~
+
+CuPy has an official API documentation.
+Many applications can be written based on the documented features.
+We support backward compatibilities of documented features.
+In other words, codes only based on the documented features run correctly with minor-/revision- updated versions.
+
+Developers are encouraged to use apparent names for objects of implementation details.
+For example, attributes outside of the documented APIs should have one or more underscores at the prefix of their names.
+
+.. _undocumented_behavior:
+
+Undocumented behaviors
+~~~~~~~~~~~~~~~~~~~~~~
+
+Behaviors of CuPy implementation not stated in the documentation are undefined.
+Undocumented behaviors are not guaranteed to be stable between different minor/revision versions.
+
+Minor update may contain changes to undocumented behaviors.
+For example, suppose an API X is added at the minor update.
+In the previous version, attempts to use X cause AttributeError.
+This behavior is not stated in the documentation, so this is undefined.
+Thus, adding the API X in minor version is permissible.
+
+Revision update may also contain changes to undefined behaviors.
+Typical example is a bug fix.
+Another example is an improvement on implementation, which may change the internal object structures not shown in the documentation.
+As a consequence, **even revision updates do not support compatibility of pickling, unless the full layout of pickled objects is clearly documented.**
+
+Documentation Error
+~~~~~~~~~~~~~~~~~~~
+
+Compatibility is basically determined based on the documentation, though it sometimes contains errors.
+It may make the APIs confusing to assume the documentation always stronger than the implementations.
+We therefore may fix the documentation errors in any updates that may break the compatibility in regard to the documentation.
+
+.. note::
+   Developers MUST NOT fix the documentation and implementation of the same functionality at the same time in revision updates as "bug fix".
+   Such a change completely breaks the backward compatibility.
+   If you want to fix the bugs in both sides, first fix the documentation to fit it into the implementation, and start the API changing procedure described above.
+
+Object Attributes and Properties
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Object attributes and properties are sometimes replaced by each other at minor updates.
+It does not break the user codes, except for the codes depending on how the attributes and properties are implemented.
+
+Functions and Methods
+~~~~~~~~~~~~~~~~~~~~~
+
+Methods may be replaced by callable attributes keeping the compatibility of parameters and return values in minor updates.
+It does not break the user codes, except for the codes depending on how the methods and callable attributes are implemented.
+
+Exceptions and Warnings
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The specifications of raising exceptions are considered as a part of standard backward compatibilities.
+No exception is raised in the future versions with correct usages that the documentation allows, unless the API changing process is completed.
+
+On the other hand, warnings may be added at any minor updates for any APIs.
+It means minor updates do not keep backward compatibility of warnings.
+
+
+Installation Compatibility
+--------------------------
+
+The installation process is another concern of compatibilities.
+We support environmental compatibilities in the following ways.
+
+- Any changes of dependent libraries that force modifications on the existing environments must be done in major updates.
+  Such changes include following cases:
+
+  - dropping supported versions of dependent libraries (e.g. dropping cuDNN v2)
+  - adding new mandatory dependencies (e.g. adding h5py to setup_requires)
+
+- Supporting optional packages/libraries may be done in minor updates (e.g. supporting h5py in optional features).
+
+.. note::
+   The installation compatibility does not guarantee that all the features of CuPy correctly run on supported environments.
+   It may contain bugs that only occurs in certain environments.
+   Such bugs should be fixed in some updates.
diff --git a/docs/source/user_guide/cuda_api.rst b/docs/source/user_guide/cuda_api.rst
new file mode 100644
index 0000000..7d00e41
--- /dev/null
+++ b/docs/source/user_guide/cuda_api.rst
@@ -0,0 +1,89 @@
+Accessing CUDA Functionalities
+==============================
+
+.. _cuda_stream_event:
+
+Streams and Events
+------------------
+
+In this section we discuss basic usages for CUDA streams and events. For the API reference please see
+:ref:`stream_event_api`. For their roles in the CUDA programming model, please refer to `CUDA Programming Guide`_.
+
+CuPy provides high-level Python APIs :class:`~cupy.cuda.Stream` and :class:`~cupy.cuda.Event` for creating
+streams and events, respectively. Data copies and kernel launches are enqueued onto the :ref:`current_stream`,
+which can be queried via :func:`~cupy.cuda.get_current_stream` and changed either by setting up a context
+manager:
+
+.. doctest::
+
+    >>> import numpy as np
+    >>>
+    >>> a_np = np.arange(10)
+    >>> s = cp.cuda.Stream()
+    >>> with s:
+    ...     a_cp = cp.asarray(a_np)  # H2D transfer on stream s
+    ...     b_cp = cp.sum(a_cp)      # kernel launched on stream s
+    ...     assert s == cp.cuda.get_current_stream()
+    ...
+    >>> # fall back to the previous stream in use (here the default stream)
+    >>> # when going out of the scope of s
+
+or by using the :meth:`~cupy.cuda.Stream.use` method:
+
+.. doctest::
+
+    >>> s = cp.cuda.Stream()
+    >>> s.use()  # any subsequent operations are done on steam s  # doctest: +ELLIPSIS
+    <Stream ... (device ...)>
+    >>> b_np = cp.asnumpy(b_cp)
+    >>> assert s == cp.cuda.get_current_stream()
+    >>> cp.cuda.Stream.null.use()  # fall back to the default (null) stream
+    <Stream 0 (device -1)>
+    >>> assert cp.cuda.Stream.null == cp.cuda.get_current_stream()
+
+Events can be created either manually or through the :meth:`~cupy.cuda.Stream.record` method.
+:class:`~cupy.cuda.Event` objects can be used for timing GPU activities (via :func:`~cupy.cuda.get_elapsed_time`)
+or setting up inter-stream dependencies:
+
+.. doctest::
+
+    >>> e1 = cp.cuda.Event()
+    >>> e1.record()
+    >>> a_cp = b_cp * a_cp + 8
+    >>> e2 = cp.cuda.get_current_stream().record()
+    >>>
+    >>> # set up a stream order
+    >>> s2 = cp.cuda.Stream()
+    >>> s2.wait_event(e2)
+    >>> with s2:
+    ...     # the a_cp is guaranteed updated when this copy (on s2) starts
+    ...     a_np = cp.asnumpy(a_cp)
+    >>>
+    >>> # timing
+    >>> e2.synchronize()
+    >>> t = cp.cuda.get_elapsed_time(e1, e2)  # only include the compute time, not the copy time
+
+Just like the :class:`~cupy.cuda.Device` objects, :class:`~cupy.cuda.Stream` and :class:`~cupy.cuda.Event`
+objects can also be used for synchronization.
+
+.. note::
+
+    In CuPy, the :class:`~cupy.cuda.Stream` objects are managed on the per thread, per device basis.
+
+.. note::
+
+    On NVIDIA GPUs, there are two stream singleton objects :obj:`~cupy.cuda.Stream.null` and
+    :obj:`~cupy.cuda.Stream.ptds`, referred to as the *legacy* default stream and the *per-thread* default
+    stream, respectively. CuPy uses the former as default when no user-defined stream is in use. To
+    change this behavior, set the environment variable ``CUPY_CUDA_PER_THREAD_DEFAULT_STREAM`` to 1,
+    see :ref:`environment`. This is not applicable to AMD GPUs.
+
+.. _CUDA Programming Guide: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html
+
+To interoperate with streams created in other Python libraries, CuPy provides the :class:`~cupy.cuda.ExternalStream`
+API to wrap an existing stream pointer (given as a Python ``int``). See :doc:`interoperability` for details.
+
+CUDA Driver and Runtime API
+---------------------------
+
+Under construction. Please see :ref:`runtime_api` for the API reference.
diff --git a/docs/source/user_guide/difference.rst b/docs/source/user_guide/difference.rst
new file mode 100644
index 0000000..2f523ff
--- /dev/null
+++ b/docs/source/user_guide/difference.rst
@@ -0,0 +1,194 @@
+Differences between CuPy and NumPy
+==================================
+
+The interface of CuPy is designed to obey that of NumPy.
+However, there are some differences.
+
+
+Cast behavior from float to integer
+-----------------------------------
+
+Some casting behaviors from float to integer are not defined in C++ specification.
+The casting from a negative float to unsigned integer and infinity to integer is one of such examples.
+The behavior of NumPy depends on your CPU architecture.
+This is the result on an Intel CPU:
+
+  >>> np.array([-1], dtype=np.float32).astype(np.uint32)
+  array([4294967295], dtype=uint32)
+  >>> cupy.array([-1], dtype=np.float32).astype(np.uint32)
+  array([0], dtype=uint32)
+
+  >>> np.array([float('inf')], dtype=np.float32).astype(np.int32)
+  array([-2147483648], dtype=int32)
+  >>> cupy.array([float('inf')], dtype=np.float32).astype(np.int32)
+  array([2147483647], dtype=int32)
+
+
+Random methods support dtype argument
+-------------------------------------
+
+NumPy's random value generator does not support a `dtype` argument and instead always returns a ``float64`` value.
+We support the option in CuPy because cuRAND, which is used in CuPy, supports both ``float32`` and ``float64``.
+
+
+  >>> np.random.randn(dtype=np.float32)
+  Traceback (most recent call last):
+    File "<stdin>", line 1, in <module>
+  TypeError: randn() got an unexpected keyword argument 'dtype'
+  >>> cupy.random.randn(dtype=np.float32)    # doctest: +SKIP
+  array(0.10689262300729752, dtype=float32)
+
+
+Out-of-bounds indices
+---------------------
+CuPy handles out-of-bounds indices differently by default from NumPy when
+using integer array indexing.
+NumPy handles them by raising an error, but CuPy wraps around them.
+
+  >>> x = np.array([0, 1, 2])
+  >>> x[[1, 3]] = 10
+  Traceback (most recent call last):
+    File "<stdin>", line 1, in <module>
+  IndexError: index 3 is out of bounds for axis 1 with size 3
+  >>> x = cupy.array([0, 1, 2])
+  >>> x[[1, 3]] = 10
+  >>> x
+  array([10, 10,  2])
+
+
+Duplicate values in indices
+---------------------------
+CuPy's ``__setitem__`` behaves differently from NumPy when integer arrays
+reference the same location multiple times.
+In that case, the value that is actually stored is undefined.
+Here is an example of CuPy.
+
+  >>> a = cupy.zeros((2,))
+  >>> i = cupy.arange(10000) % 2
+  >>> v = cupy.arange(10000).astype(np.float32)
+  >>> a[i] = v
+  >>> a  # doctest: +SKIP
+  array([ 9150.,  9151.])
+
+NumPy stores the value corresponding to the
+last element among elements referencing duplicate locations.
+
+  >>> a_cpu = np.zeros((2,))
+  >>> i_cpu = np.arange(10000) % 2
+  >>> v_cpu = np.arange(10000).astype(np.float32)
+  >>> a_cpu[i_cpu] = v_cpu
+  >>> a_cpu
+  array([9998., 9999.])
+
+
+Zero-dimensional array
+-----------------------------------------------
+
+Reduction methods
+~~~~~~~~~~~~~~~~~
+
+NumPy's reduction functions (e.g. :func:`numpy.sum`) return scalar values (e.g. :class:`numpy.float32`).
+However CuPy counterparts return zero-dimensional :class:`cupy.ndarray` s.
+That is because CuPy scalar values (e.g. :class:`cupy.float32`) are aliases of NumPy scalar values and are allocated in CPU memory.
+If these types were returned, it would be required to synchronize between GPU and CPU.
+If you want to use scalar values, cast the returned arrays explicitly.
+
+  >>> type(np.sum(np.arange(3))) == np.int64
+  True
+  >>> type(cupy.sum(cupy.arange(3))) == cupy.ndarray
+  True
+
+
+Type promotion
+~~~~~~~~~~~~~~
+
+CuPy automatically promotes dtypes of :class:`cupy.ndarray` s in a function with two or more operands, the result dtype is determined by the dtypes of the inputs.
+This is different from NumPy's rule on type promotion, when operands contain zero-dimensional arrays.
+Zero-dimensional :class:`numpy.ndarray` s are treated as if they were scalar values if they appear in operands of NumPy's function,
+This may affect the dtype of its output, depending on the values of the "scalar" inputs.
+
+  >>> (np.array(3, dtype=np.int32) * np.array([1., 2.], dtype=np.float32)).dtype
+  dtype('float32')
+  >>> (np.array(300000, dtype=np.int32) * np.array([1., 2.], dtype=np.float32)).dtype
+  dtype('float64')
+  >>> (cupy.array(3, dtype=np.int32) * cupy.array([1., 2.], dtype=np.float32)).dtype
+  dtype('float64')
+
+
+Matrix type (:class:`numpy.matrix`)
+-----------------------------------
+
+SciPy returns :class:`numpy.matrix` (a subclass of :class:`numpy.ndarray`) when dense matrices are computed from sparse matrices (e.g., ``coo_matrix + ndarray``). However, CuPy returns :class:`cupy.ndarray` for such operations.
+
+There is no plan to provide :class:`numpy.matrix` equivalent in CuPy.
+This is because the use of :class:`numpy.matrix` is no longer recommended since NumPy 1.15.
+
+
+Data types
+----------
+
+Data type of CuPy arrays cannot be non-numeric like strings or objects.
+See :ref:`overview` for details.
+
+
+Universal Functions only work with CuPy array or scalar
+-------------------------------------------------------
+
+Unlike NumPy, Universal Functions in CuPy only work with CuPy array or scalar.
+They do not accept other objects (e.g., lists or :class:`numpy.ndarray`).
+
+  >>> np.power([np.arange(5)], 2)
+  array([[ 0,  1,  4,  9, 16]])
+
+  >>> cupy.power([cupy.arange(5)], 2)
+  Traceback (most recent call last):
+    File "<stdin>", line 1, in <module>
+  TypeError: Unsupported type <class 'list'>
+
+
+Random seed arrays are hashed to scalars
+----------------------------------------
+
+Like Numpy, CuPy's RandomState objects accept seeds either as numbers or as
+full numpy arrays.
+
+  >>> seed = np.array([1, 2, 3, 4, 5])
+  >>> rs = cupy.random.RandomState(seed=seed)
+
+However, unlike Numpy, array seeds will be hashed down to a single number and
+so may not communicate as much entropy to the underlying random number
+generator.
+
+
+NaN (not-a-number) handling
+---------------------------
+
+By default CuPy's reduction functions (e.g., :func:`cupy.sum`) handle NaNs in complex numbers differently from NumPy's
+counterparts:
+
+  >>> a = [0.5 + 3.7j, complex(0.7, np.nan), complex(np.nan, -3.9), complex(np.nan, np.nan)]
+  >>>
+  >>> a_np = np.asarray(a)
+  >>> print(a_np.max(), a_np.min())
+  (0.7+nanj) (0.7+nanj)
+  >>>
+  >>> a_cp = cp.asarray(a_np)
+  >>> print(a_cp.max(), a_cp.min())
+  (nan-3.9j) (nan-3.9j)
+
+The reason is that internally the reduction is performed in a strided fashion, thus it does not ensure a proper
+comparison order and cannot follow NumPy's rule to always propagate the first-encountered NaN.
+Note that this difference does not apply when CUB is enabled (which is the default for CuPy v11 or later.)
+
+Contiguity / Strides
+--------------------
+
+To provide the best performance, the contiguity of a resulting ndarray is not guaranteed to match with that of NumPy's output.
+
+  >>> a = np.array([[1, 2], [3, 4]], order='F')
+  >>> print((a + a).flags.f_contiguous)
+  True
+
+  >>> a = cp.array([[1, 2], [3, 4]], order='F')
+  >>> print((a + a).flags.f_contiguous)
+  False
diff --git a/docs/source/user_guide/fft.rst b/docs/source/user_guide/fft.rst
new file mode 100644
index 0000000..06d2e4d
--- /dev/null
+++ b/docs/source/user_guide/fft.rst
@@ -0,0 +1,315 @@
+Fast Fourier Transform with CuPy
+================================
+
+CuPy covers the full Fast Fourier Transform (FFT) functionalities provided in NumPy (:mod:`cupy.fft`) and a
+subset in SciPy (:mod:`cupyx.scipy.fft`). In addition to those high-level APIs that can be used
+as is, CuPy provides additional features to
+
+1. access advanced routines that `cuFFT`_ offers for NVIDIA GPUs,
+2. control better the performance and behavior of the FFT routines.
+
+Some of these features are *experimental* (subject to change, deprecation, or removal, see :doc:`./compatibility`)
+or may be absent in `hipFFT`_/`rocFFT`_ targeting AMD GPUs.
+
+.. _cuFFT: https://docs.nvidia.com/cuda/cufft/index.html
+.. _hipFFT: https://hipfft.readthedocs.io/en/latest/
+.. _rocFFT: https://rocfft.readthedocs.io/en/latest/
+
+
+.. _scipy_fft_backend:
+
+SciPy FFT backend
+-----------------
+
+Since SciPy v1.4 a backend mechanism is provided so that users can register different FFT backends and use SciPy's API to perform the actual transform
+with the target backend, such as CuPy's :mod:`cupyx.scipy.fft` module. For a one-time only usage, a context manager :func:`scipy.fft.set_backend` can be used:
+
+.. code-block:: python
+
+    import cupy as cp
+    import cupyx.scipy.fft as cufft
+    import scipy.fft
+
+    a = cp.random.random(100).astype(cp.complex64)
+    with scipy.fft.set_backend(cufft):
+        b = scipy.fft.fft(a)  # equivalent to cufft.fft(a)
+
+However, such usage can be tedious. Alternatively, users can register a backend through :func:`scipy.fft.register_backend` or :func:`scipy.fft.set_global_backend`
+to avoid using context managers:
+
+.. code-block:: python
+
+    import cupy as cp
+    import cupyx.scipy.fft as cufft
+    import scipy.fft
+    scipy.fft.set_global_backend(cufft)
+
+    a = cp.random.random(100).astype(cp.complex64)
+    b = scipy.fft.fft(a)  # equivalent to cufft.fft(a)
+
+.. note::
+
+    Please refer to `SciPy FFT documentation`_ for further information.
+
+.. note::
+    To use the backend together with an explicit ``plan`` argument requires SciPy version 1.5.0 or higher.
+    See below for how to create FFT plans.
+
+.. _SciPy FFT documentation: https://docs.scipy.org/doc/scipy/reference/fft.html#backend-control
+
+
+User-managed FFT plans
+----------------------
+
+For performance reasons, users may wish to create, reuse, and manage the FFT plans themselves. CuPy provides a high-level *experimental* API :func:`~cupyx.scipy.fftpack.get_fft_plan` for this need. Users specify the transform to be performed as they would with most of the high-level FFT APIs, and a plan will be generated based on the input.
+
+.. code-block:: python
+
+    import cupy as cp
+    from cupyx.scipy.fft import get_fft_plan
+
+    a = cp.random.random((4, 64, 64)).astype(cp.complex64)
+    plan = get_fft_plan(a, axes=(1, 2), value_type='C2C')  # for batched, C2C, 2D transform
+
+The returned plan can be used either explicitly as an argument with the :mod:`cupyx.scipy.fft` APIs:
+
+.. code-block:: python
+
+    import cupyx.scipy.fft
+
+    # the rest of the arguments must match those used when generating the plan
+    out = cupyx.scipy.fft.fft2(a, axes=(1, 2), plan=plan)
+
+or as a context manager for the :mod:`cupy.fft` APIs:
+
+.. code-block:: python
+
+    with plan:
+        # the arguments must match those used when generating the plan
+        out = cp.fft.fft2(a, axes=(1, 2))
+
+
+.. _fft_plan_cache:
+
+FFT plan cache
+--------------
+
+However, there are occasions when users may *not* want to manage the FFT plans by themselves. Moreover, plans could also be reused internally in CuPy's routines, to which user-managed plans would not be applicable. Therefore, starting CuPy v8 we provide a built-in plan cache, enabled by default. The plan cache is done on a *per device, per thread* basis, and can be retrieved by the :func:`~cupy.fft.config.get_plan_cache` API.
+
+.. code-block:: python
+
+    >>> import cupy as cp
+    >>>
+    >>> cache = cp.fft.config.get_plan_cache()
+    >>> cache.show_info()
+    ------------------- cuFFT plan cache (device 0) -------------------
+    cache enabled? True
+    current / max size   : 0 / 16 (counts)
+    current / max memsize: 0 / (unlimited) (bytes)
+    hits / misses: 0 / 0 (counts)
+    
+    cached plans (most recently used first):
+    
+    >>> # perform a transform, which would generate a plan and cache it
+    >>> a = cp.random.random((4, 64, 64))
+    >>> out = cp.fft.fftn(a, axes=(1, 2))
+    >>> cache.show_info()  # hit = 0
+    ------------------- cuFFT plan cache (device 0) -------------------
+    cache enabled? True
+    current / max size   : 1 / 16 (counts)
+    current / max memsize: 262144 / (unlimited) (bytes)
+    hits / misses: 0 / 1 (counts)
+    
+    cached plans (most recently used first):
+    key: ((64, 64), (64, 64), 1, 4096, (64, 64), 1, 4096, 105, 4, 'C', 2, None), plan type: PlanNd, memory usage: 262144
+    
+    >>> # perform the same transform again, the plan is looked up from cache and reused
+    >>> out = cp.fft.fftn(a, axes=(1, 2))
+    >>> cache.show_info()  # hit = 1
+    ------------------- cuFFT plan cache (device 0) -------------------
+    cache enabled? True
+    current / max size   : 1 / 16 (counts)
+    current / max memsize: 262144 / (unlimited) (bytes)
+    hits / misses: 1 / 1 (counts)
+    
+    cached plans (most recently used first):
+    key: ((64, 64), (64, 64), 1, 4096, (64, 64), 1, 4096, 105, 4, 'C', 2, None), plan type: PlanNd, memory usage: 262144
+    
+    >>> # clear the cache
+    >>> cache.clear()
+    >>> cp.fft.config.show_plan_cache_info()  # = cache.show_info(), for all devices
+    =============== cuFFT plan cache info (all devices) ===============
+    ------------------- cuFFT plan cache (device 0) -------------------
+    cache enabled? True
+    current / max size   : 0 / 16 (counts)
+    current / max memsize: 0 / (unlimited) (bytes)
+    hits / misses: 0 / 0 (counts)
+    
+    cached plans (most recently used first):
+    
+
+The returned :class:`~cupy.fft._cache.PlanCache` object has other methods for finer control, such as setting the cache size (either by counts or by memory usage). If the size is set to 0, the cache is disabled. Please refer to its documentation for more detail.
+
+.. note::
+
+    As shown above each FFT plan has an associated working area allocated. If an out-of-memory error happens, one may want to inspect, clear, or limit the plan cache.
+
+.. note::
+
+    The plans returned by :func:`~cupyx.scipy.fftpack.get_fft_plan` are not cached.
+
+
+FFT callbacks
+-------------
+
+`cuFFT`_ provides FFT callbacks for merging pre- and/or post- processing kernels with the FFT routines so as to reduce the access to global memory.
+This capability is supported *experimentally* by CuPy. Users need to supply custom load and/or store kernels as strings, and set up a context manager
+via :func:`~cupy.fft.config.set_cufft_callbacks`. Note that the load (store) kernel pointer has to be named as ``d_loadCallbackPtr`` (``d_storeCallbackPtr``).
+
+.. code-block:: python
+
+    import cupy as cp
+
+    # a load callback that overwrites the input array to 1
+    code = r'''
+    __device__ cufftComplex CB_ConvertInputC(
+        void *dataIn,
+        size_t offset,
+        void *callerInfo,
+        void *sharedPtr)
+    {
+        cufftComplex x;
+        x.x = 1.;
+        x.y = 0.;
+        return x;
+    }
+    __device__ cufftCallbackLoadC d_loadCallbackPtr = CB_ConvertInputC;
+    '''
+
+    a = cp.random.random((64, 128, 128)).astype(cp.complex64)
+
+    # this fftn call uses callback
+    with cp.fft.config.set_cufft_callbacks(cb_load=code):
+        b = cp.fft.fftn(a, axes=(1,2))
+
+    # this does not use
+    c = cp.fft.fftn(cp.ones(shape=a.shape, dtype=cp.complex64), axes=(1,2))
+
+    # result agrees
+    assert cp.allclose(b, c)
+
+    # "static" plans are also cached, but are distinct from their no-callback counterparts
+    cp.fft.config.get_plan_cache().show_info()
+
+
+.. note::
+
+    Internally, this feature requires recompiling a Python module *for each distinct pair* of load and store kernels. Therefore, the first invocation will be very slow, and this cost is amortized if the callbacks can be reused in the subsequent calculations. The compiled modules are cached on disk, with a default position ``$HOME/.cupy/callback_cache`` that can be changed by the environment variable ``CUPY_CACHE_DIR``.
+
+
+Multi-GPU FFT
+-------------
+
+CuPy currently provides two kinds of *experimental* support for multi-GPU FFT.
+
+.. warning::
+
+    Using multiple GPUs to perform FFT is not guaranteed to be more performant. The rule of thumb is if the transform fits in 1 GPU, you should avoid using multiple.
+
+The first kind of support is with the high-level :func:`~cupy.fft.fft` and :func:`~cupy.fft.ifft` APIs, which requires the input array to reside on one of the participating GPUs. The multi-GPU calculation is done under the hood, and by the end of the calculation the result again resides on the device where it started. Currently only 1D complex-to-complex (C2C) transform is supported; complex-to-real (C2R) or real-to-complex (R2C) transforms (such as :func:`~cupy.fft.rfft` and friends) are not. The transform can be either batched (batch size > 1) or not (batch size = 1).
+
+.. code-block:: python
+
+    import cupy as cp
+
+    cp.fft.config.use_multi_gpus = True
+    cp.fft.config.set_cufft_gpus([0, 1])  # use GPU 0 & 1
+
+    shape = (64, 64)  # batch size = 64
+    dtype = cp.complex64
+    a = cp.random.random(shape).astype(dtype)  # reside on GPU 0
+
+    b = cp.fft.fft(a)  # computed on GPU 0 & 1, reside on GPU 0
+
+If you need to perform 2D/3D transforms (ex: :func:`~cupy.fft.fftn`) instead of 1D (ex: :func:`~cupy.fft.fft`), it would likely still work, but in this particular use case it loops over the transformed axes under the hood (which is exactly what is done in NumPy too), which could lead to suboptimal performance.
+
+The second kind of usage is to use the low-level, *private* CuPy APIs. You need to construct a :class:`~cupy.cuda.cufft.Plan1d` object and use it as if you are programming in C/C++ with `cuFFT`_. Using this approach, your input array can reside on the host as a :class:`numpy.ndarray` so that its size can be much larger than what a single GPU can accommodate, which is one of the main reasons to run multi-GPU FFT.
+
+.. code-block:: python
+
+    import numpy as np
+    import cupy as cp
+
+    # no need to touch cp.fft.config, as we are using low-level API
+
+    shape = (64, 64)
+    dtype = np.complex64
+    a = np.random.random(shape).astype(dtype)  # reside on CPU
+
+    if len(shape) == 1:
+        batch = 1
+        nx = shape[0]
+    elif len(shape) == 2:
+        batch = shape[0]
+        nx = shape[1]
+
+    # compute via cuFFT
+    cufft_type = cp.cuda.cufft.CUFFT_C2C  # single-precision c2c
+    plan = cp.cuda.cufft.Plan1d(nx, cufft_type, batch, devices=[0,1])
+    out_cp = np.empty_like(a)  # output on CPU
+    plan.fft(a, out_cp, cufft.CUFFT_FORWARD)
+
+    out_np = numpy.fft.fft(a)  # use NumPy's fft
+    # np.fft.fft alway returns np.complex128
+    if dtype is numpy.complex64:
+        out_np = out_np.astype(dtype)
+
+    # check result
+    assert np.allclose(out_cp, out_np, rtol=1e-4, atol=1e-7)
+
+For this use case, please consult the `cuFFT`_ documentation on multi-GPU transform for further detail.
+
+.. note::
+
+    The multi-GPU plans are cached if auto-generated via the high-level APIs, but not if manually generated via the low-level APIs.
+
+
+Half-precision FFT
+------------------
+
+`cuFFT`_ provides ``cufftXtMakePlanMany`` and ``cufftXtExec`` routines to support a wide range of FFT needs, including 64-bit indexing and half-precision FFT. CuPy provides an *experimental* support for this capability via the new (though *private*) :class:`~cupy.cuda.cufft.XtPlanNd` API. For half-precision FFT, on supported hardware it can be twice as fast than its single-precision counterpart. NumPy does not yet provide the necessary infrastructure for half-precision complex numbers (i.e., ``numpy.complex32``), though, so the steps for this feature is currently a bit more involved than common cases.
+
+.. code-block:: python
+
+    import cupy as cp
+    import numpy as np
+
+
+    shape = (1024, 256, 256)  # input array shape
+    idtype = odtype = edtype = 'E'  # = numpy.complex32 in the future
+
+    # store the input/output arrays as fp16 arrays twice as long, as complex32 is not yet available
+    a = cp.random.random((shape[0], shape[1], 2*shape[2])).astype(cp.float16)
+    out = cp.empty_like(a)
+
+    # FFT with cuFFT
+    plan = cp.cuda.cufft.XtPlanNd(shape[1:],
+                                  shape[1:], 1, shape[1]*shape[2], idtype,
+                                  shape[1:], 1, shape[1]*shape[2], odtype,
+                                  shape[0], edtype,
+                                  order='C', last_axis=-1, last_size=None)
+
+    plan.fft(a, out, cp.cuda.cufft.CUFFT_FORWARD)
+
+    # FFT with NumPy
+    a_np = cp.asnumpy(a).astype(np.float32)  # upcast
+    a_np = a_np.view(np.complex64)
+    out_np = np.fft.fftn(a_np, axes=(-2,-1))
+    out_np = np.ascontiguousarray(out_np).astype(np.complex64)  # downcast
+    out_np = out_np.view(np.float32)
+    out_np = out_np.astype(np.float16)
+
+    # don't worry about accruacy for now, as we probably lost a lot during casting
+    print('ok' if cp.mean(cp.abs(out - cp.asarray(out_np))) < 0.1 else 'not ok')
+
+The 64-bit indexing support for all high-level FFT APIs is planned for a future CuPy release.
diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst
new file mode 100644
index 0000000..854c0c3
--- /dev/null
+++ b/docs/source/user_guide/index.rst
@@ -0,0 +1,17 @@
+User Guide
+==========
+
+This user guide provides an overview of CuPy and explains its important features; details are found in :ref:`CuPy API Reference <cupy_reference>`.
+
+.. toctree::
+   :maxdepth: 1
+
+   basic
+   kernel
+   cuda_api
+   fft
+   memory
+   performance
+   interoperability
+   difference
+   compatibility
diff --git a/docs/source/user_guide/interoperability.rst b/docs/source/user_guide/interoperability.rst
new file mode 100644
index 0000000..bb870f0
--- /dev/null
+++ b/docs/source/user_guide/interoperability.rst
@@ -0,0 +1,462 @@
+Interoperability
+================
+
+CuPy can be used in conjunction with other libraries.
+
+
+NumPy
+-----
+
+:class:`cupy.ndarray` implements ``__array_ufunc__`` interface (see `NEP 13 — A Mechanism for Overriding Ufuncs <http://www.numpy.org/neps/nep-0013-ufunc-overrides.html>`_ for details).
+This enables NumPy ufuncs to be directly operated on CuPy arrays.
+``__array_ufunc__`` feature requires NumPy 1.13 or later.
+
+.. code:: python
+
+    import cupy
+    import numpy
+
+    arr = cupy.random.randn(1, 2, 3, 4).astype(cupy.float32)
+    result = numpy.sum(arr)
+    print(type(result))  # => <class 'cupy._core.core.ndarray'>
+
+:class:`cupy.ndarray` also implements ``__array_function__`` interface (see `NEP 18 — A dispatch mechanism for NumPy’s high level array functions <http://www.numpy.org/neps/nep-0018-array-function-protocol.html>`_ for details).
+This enables code using NumPy to be directly operated on CuPy arrays.
+``__array_function__`` feature requires NumPy 1.16 or later; As of NumPy 1.17, ``__array_function__`` is enabled by default.
+
+
+Numba
+-----
+
+`Numba <https://numba.pydata.org/>`_ is a Python JIT compiler with NumPy support.
+
+:class:`cupy.ndarray` implements ``__cuda_array_interface__``, which is the CUDA array interchange interface compatible with Numba v0.39.0 or later (see `CUDA Array Interface <https://numba.readthedocs.io/en/stable/cuda/cuda_array_interface.html>`_ for details).
+It means you can pass CuPy arrays to kernels JITed with Numba.
+The following is a simple example code borrowed from `numba/numba#2860 <https://github.com/numba/numba/pull/2860>`_:
+
+.. code:: python
+
+	import cupy
+	from numba import cuda
+
+	@cuda.jit
+	def add(x, y, out):
+		start = cuda.grid(1)
+		stride = cuda.gridsize(1)
+		for i in range(start, x.shape[0], stride):
+			out[i] = x[i] + y[i]
+
+	a = cupy.arange(10)
+	b = a * 2
+	out = cupy.zeros_like(a)
+
+	print(out)  # => [0 0 0 0 0 0 0 0 0 0]
+
+	add[1, 32](a, b, out)
+
+	print(out)  # => [ 0  3  6  9 12 15 18 21 24 27]
+
+In addition, :func:`cupy.asarray` supports zero-copy conversion from Numba CUDA array to CuPy array.
+
+.. code:: python
+
+    import numpy
+    import numba
+    import cupy
+
+    x = numpy.arange(10)  # type: numpy.ndarray
+    x_numba = numba.cuda.to_device(x)  # type: numba.cuda.cudadrv.devicearray.DeviceNDArray
+    x_cupy = cupy.asarray(x_numba)  # type: cupy.ndarray
+
+.. warning::
+
+    ``__cuda_array_interface__`` specifies that the object lifetime must be managed by the user, so it is an undefined behavior if the
+    exported object is destroyed while still in use by the consumer library.
+
+.. note::
+
+    CuPy uses two environment variables controlling the exchange behavior: :envvar:`CUPY_CUDA_ARRAY_INTERFACE_SYNC` and :envvar:`CUPY_CUDA_ARRAY_INTERFACE_EXPORT_VERSION`.
+
+
+mpi4py
+------
+
+`MPI for Python (mpi4py) <https://mpi4py.readthedocs.io/en/latest/>`_ is a Python wrapper for the Message Passing Interface (MPI) libraries.
+
+MPI is the most widely used standard for high-performance inter-process communications. Recently several MPI vendors, including MPICH, Open MPI and MVAPICH, have extended their support beyond the MPI-3.1 standard to enable "CUDA-awareness"; that is, passing CUDA device pointers directly to MPI calls to avoid explicit data movement between the host and the device.
+
+With the ``__cuda_array_interface__`` (as mentioned above) and ``DLPack`` data exchange protocols (see :ref:`dlpack` below) implemented in CuPy, mpi4py now provides (experimental) support for passing CuPy arrays to MPI calls, provided that mpi4py is built against a CUDA-aware MPI implementation. The following is a simple example code borrowed from `mpi4py Tutorial <https://mpi4py.readthedocs.io/en/latest/tutorial.html>`_:
+
+.. code:: python
+
+    # To run this script with N MPI processes, do
+    # mpiexec -n N python this_script.py
+
+    import cupy
+    from mpi4py import MPI
+
+    comm = MPI.COMM_WORLD
+    size = comm.Get_size()
+
+    # Allreduce
+    sendbuf = cupy.arange(10, dtype='i')
+    recvbuf = cupy.empty_like(sendbuf)
+    comm.Allreduce(sendbuf, recvbuf)
+    assert cupy.allclose(recvbuf, sendbuf*size)
+
+This new feature is added since mpi4py 3.1.0. See the `mpi4py website <https://mpi4py.readthedocs.io/en/latest/>`_ for more information.
+
+
+PyTorch
+-------
+
+`PyTorch <https://pytorch.org/>`_ is a machine learning framefork that provides high-performance, differentiable tensor operations.
+
+PyTorch also supports ``__cuda_array_interface__``, so zero-copy data exchange between CuPy and PyTorch can be achieved at no cost.
+The only caveat is PyTorch by default creates CPU tensors, which do not have the ``__cuda_array_interface__`` property defined, and
+users need to ensure the tensor is already on GPU before exchanging.
+
+.. code:: python
+
+    >>> import cupy as cp
+    >>> import torch
+    >>>
+    >>> # convert a torch tensor to a cupy array
+    >>> a = torch.rand((4, 4), device='cuda')
+    >>> b = cp.asarray(a)
+    >>> b *= b
+    >>> b
+    array([[0.8215962 , 0.82399917, 0.65607935, 0.30354425],
+           [0.422695  , 0.8367199 , 0.00208597, 0.18545236],
+           [0.00226746, 0.46201342, 0.6833052 , 0.47549972],
+           [0.5208748 , 0.6059282 , 0.1909013 , 0.5148635 ]], dtype=float32)
+    >>> a
+    tensor([[0.8216, 0.8240, 0.6561, 0.3035],
+            [0.4227, 0.8367, 0.0021, 0.1855],
+            [0.0023, 0.4620, 0.6833, 0.4755],
+            [0.5209, 0.6059, 0.1909, 0.5149]], device='cuda:0')
+    >>> # check the underlying memory pointer is the same
+    >>> assert a.__cuda_array_interface__['data'][0] == b.__cuda_array_interface__['data'][0]
+    >>>
+    >>> # convert a cupy array to a torch tensor
+    >>> a = cp.arange(10)
+    >>> b = torch.as_tensor(a, device='cuda')
+    >>> b += 3
+    >>> b
+    tensor([ 3,  4,  5,  6,  7,  8,  9, 10, 11, 12], device='cuda:0')
+    >>> a
+    array([ 3,  4,  5,  6,  7,  8,  9, 10, 11, 12])
+    >>> assert a.__cuda_array_interface__['data'][0] == b.__cuda_array_interface__['data'][0]
+
+PyTorch also supports zero-copy data exchange through ``DLPack`` (see :ref:`dlpack` below):
+
+.. code:: python
+
+	import cupy
+	import torch
+
+	from torch.utils.dlpack import to_dlpack
+	from torch.utils.dlpack import from_dlpack
+
+	# Create a PyTorch tensor.
+	tx1 = torch.randn(1, 2, 3, 4).cuda()
+
+	# Convert it into a DLPack tensor.
+	dx = to_dlpack(tx1)
+
+	# Convert it into a CuPy array.
+	cx = cupy.from_dlpack(dx)
+
+	# Convert it back to a PyTorch tensor.
+	tx2 = from_dlpack(cx.toDlpack())
+
+`pytorch-pfn-extras <https://github.com/pfnet/pytorch-pfn-extras/>`_ library provides additional integration features with PyTorch, including memory pool sharing and stream sharing:
+
+.. code:: python
+
+   >>> import cupy
+   >>> import torch
+   >>> import pytorch_pfn_extras as ppe
+   >>>
+   >>> # Perform CuPy memory allocation using the PyTorch memory pool.
+   >>> ppe.cuda.use_torch_mempool_in_cupy()
+   >>> torch.cuda.memory_allocated()
+   0
+   >>> arr = cupy.arange(10)
+   >>> torch.cuda.memory_allocated()
+   512
+   >>>
+   >>> # Change the default stream in PyTorch and CuPy:
+   >>> stream = torch.cuda.Stream()
+   >>> with ppe.cuda.stream(stream):
+   ...     ...
+
+
+Using custom kernels in PyTorch
+*******************************
+
+With the DLPack protocol, it becomes very simple to implement functions in PyTorch using CuPy user-defined kernels. Below is the example of a PyTorch autograd function
+that computes the forward and backward pass of the logarithm using :class:`cupy.RawKernel` s.
+
+.. code:: python
+
+    import cupy
+    import torch
+    
+    
+    cupy_custom_kernel_fwd = cupy.RawKernel(
+        r"""
+    extern "C" __global__
+    void cupy_custom_kernel_fwd(const float* x, float* y, int size) {
+        int tid = blockDim.x * blockIdx.x + threadIdx.x;
+        if (tid < size)
+            y[tid] = log(x[tid]);
+    }
+    """,
+        "cupy_custom_kernel_fwd",
+    )
+    
+    
+    cupy_custom_kernel_bwd = cupy.RawKernel(
+        r"""
+    extern "C" __global__
+    void cupy_custom_kernel_bwd(const float* x, float* gy, float* gx, int size) {
+        int tid = blockDim.x * blockIdx.x + threadIdx.x;
+        if (tid < size)
+            gx[tid] = gy[tid] / x[tid];
+    }
+    """,
+        "cupy_custom_kernel_bwd",
+    )
+    
+    
+    class CuPyLog(torch.autograd.Function):
+        @staticmethod
+        def forward(ctx, x):
+            ctx.input = x
+            # Enforce contiguous arrays to simplify RawKernel indexing.
+            cupy_x = cupy.ascontiguousarray(cupy.from_dlpack(x.detach()))
+            cupy_y = cupy.empty(cupy_x.shape, dtype=cupy_x.dtype)
+            x_size = cupy_x.size
+            bs = 128
+            cupy_custom_kernel_fwd(
+                (bs,), ((x_size + bs - 1) // bs,), (cupy_x, cupy_y, x_size)
+            )
+            # the ownership of the device memory backing cupy_y is implicitly
+            # transferred to torch_y, so this operation is safe even after
+            # going out of scope of this function.
+            torch_y = torch.from_dlpack(cupy_y)
+            return torch_y
+    
+        @staticmethod
+        def backward(ctx, grad_y):
+            # Enforce contiguous arrays to simplify RawKernel indexing.
+            cupy_input = cupy.from_dlpack(ctx.input.detach()).ravel()
+            cupy_grad_y = cupy.from_dlpack(grad_y.detach()).ravel()
+            cupy_grad_x = cupy.zeros(cupy_grad_y.shape, dtype=cupy_grad_y.dtype)
+            gy_size = cupy_grad_y.size
+            bs = 128
+            cupy_custom_kernel_bwd(
+                (bs,),
+                ((gy_size + bs - 1) // bs,),
+                (cupy_input, cupy_grad_y, cupy_grad_x, gy_size),
+            )
+            # the ownership of the device memory backing cupy_grad_x is implicitly
+            # transferred to torch_y, so this operation is safe even after
+            # going out of scope of this function.
+            torch_grad_x = torch.from_dlpack(cupy_grad_x)
+            return torch_grad_x
+
+.. note::
+
+   Directly feeding a ``torch.Tensor`` to :func:`cupy.from_dlpack` is only supported in the (new) DLPack data exchange protocol added in CuPy v10+ and PyTorch 1.10+.
+   For earlier versions, you will need to wrap the ``Tensor`` with ``torch.utils.dlpack.to_dlpack()`` as shown in the above examples.
+
+RMM
+---
+
+`RMM (RAPIDS Memory Manager) <https://docs.rapids.ai/api/rmm/stable/index.html>`_ provides highly configurable memory allocators.
+
+RMM provides an interface to allow CuPy to allocate memory from the RMM memory pool instead of from CuPy's own pool. It can be set up
+as simple as:
+
+.. code:: python
+
+    import cupy
+    import rmm
+    cupy.cuda.set_allocator(rmm.rmm_cupy_allocator)
+
+Sometimes, a more performant allocator may be desirable. RMM provides an option to switch the allocator:
+
+.. code:: python
+
+    import cupy
+    import rmm
+    rmm.reinitialize(pool_allocator=True)  # can also set init pool size etc here
+    cupy.cuda.set_allocator(rmm.rmm_cupy_allocator)
+
+For more information on CuPy's memory management, see :doc:`./memory`.
+
+
+.. _dlpack:
+
+DLPack
+------
+
+`DLPack <https://github.com/dmlc/dlpack>`__ is a specification of tensor structure to share tensors among frameworks.
+
+CuPy supports importing from and exporting to DLPack data structure (:func:`cupy.from_dlpack` and :func:`cupy.ndarray.toDlpack`).
+
+Here is a simple example:
+
+.. code:: python
+
+	import cupy
+
+	# Create a CuPy array.
+	cx1 = cupy.random.randn(1, 2, 3, 4).astype(cupy.float32)
+
+	# Convert it into a DLPack tensor.
+	dx = cx1.toDlpack()
+
+	# Convert it back to a CuPy array.
+	cx2 = cupy.from_dlpack(dx)
+
+`TensorFlow <https://www.tensorflow.org>`_ also supports DLpack, so zero-copy data exchange between CuPy and TensorFlow through
+DLPack is possible:
+
+.. code:: python
+
+    >>> import tensorflow as tf
+    >>> import cupy as cp
+    >>>
+    >>> # convert a TF tensor to a cupy array
+    >>> with tf.device('/GPU:0'):
+    ...     a = tf.random.uniform((10,))
+    ...
+    >>> a
+    <tf.Tensor: shape=(10,), dtype=float32, numpy=
+    array([0.9672388 , 0.57568085, 0.53163004, 0.6536236 , 0.20479882,
+           0.84908986, 0.5852566 , 0.30355775, 0.1733712 , 0.9177849 ],
+          dtype=float32)>
+    >>> a.device
+    '/job:localhost/replica:0/task:0/device:GPU:0'
+    >>> cap = tf.experimental.dlpack.to_dlpack(a)
+    >>> b = cp.from_dlpack(cap)
+    >>> b *= 3
+    >>> b
+    array([1.4949363 , 0.60699713, 1.3276931 , 1.5781245 , 1.1914308 ,
+           2.3180873 , 1.9560868 , 1.3932796 , 1.9299742 , 2.5352407 ],
+          dtype=float32)
+    >>> a
+    <tf.Tensor: shape=(10,), dtype=float32, numpy=
+    array([1.4949363 , 0.60699713, 1.3276931 , 1.5781245 , 1.1914308 ,
+           2.3180873 , 1.9560868 , 1.3932796 , 1.9299742 , 2.5352407 ],
+          dtype=float32)>
+    >>>
+    >>> # convert a cupy array to a TF tensor
+    >>> a = cp.arange(10)
+    >>> cap = a.toDlpack()
+    >>> b = tf.experimental.dlpack.from_dlpack(cap)
+    >>> b.device
+    '/job:localhost/replica:0/task:0/device:GPU:0'
+    >>> b
+    <tf.Tensor: shape=(10,), dtype=int64, numpy=array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])>
+    >>> a
+    array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+
+Be aware that in TensorFlow all tensors are immutable, so in the latter case any changes in ``b`` cannot be reflected in the CuPy array ``a``.
+
+Note that as of DLPack v0.5 for correctness the above approach (implicitly) requires users to ensure that such conversion (both importing and exporting a CuPy array) must happen on the same CUDA/HIP stream. If in doubt, the current CuPy stream in use can be fetched by, for example, calling :func:`cupy.cuda.get_current_stream`. Please consult the other framework's documentation for how to access and control the streams.
+
+DLPack data exchange protocol
+*****************************
+
+To obviate user-managed streams and DLPack tensor objects, the `DLPack data exchange protocol <https://data-apis.org/array-api/latest/design_topics/data_interchange.html>`_ provides a mechanism to shift the responsibility from users to libraries. Any compliant objects (such as :class:`cupy.ndarray`) must implement a pair of methods ``__dlpack__`` and ``__dlpack_device__``. The function :func:`cupy.from_dlpack` accepts such object and returns a :class:`cupy.ndarray` that is safely accessible on CuPy's current stream. Likewise, :class:`cupy.ndarray` can be exported via any compliant library's ``from_dlpack()`` function.
+
+.. note::
+
+    CuPy uses :envvar:`CUPY_DLPACK_EXPORT_VERSION` to control how to handle tensors backed by CUDA managed memory.
+
+
+Device Memory Pointers
+----------------------
+
+Import
+******
+
+CuPy provides :class:`~cupy.cuda.UnownedMemory` API that allows interoperating with GPU device memory allocated in other libraries.
+
+.. code:: python
+
+   # Create a memory chunk from raw pointer and its size.
+   mem = cupy.cuda.UnownedMemory(140359025819648, 1024, owner=None)
+
+   # Wrap it as a MemoryPointer.
+   memptr = cupy.cuda.MemoryPointer(mem, offset=0)
+
+   # Create an ndarray view backed by the memory pointer.
+   arr = cupy.ndarray((16, 16), dtype=cupy.float32, memptr=memptr)
+   assert arr.nbytes <= arr.data.mem.size
+
+Be aware that you are responsible for specifying a correct shape, dtype, strides, and order such that it fits in the chunk when creating an :class:`~cupy.ndarray` view.
+
+The :class:`~cupy.cuda.UnownedMemory` API does not manage the lifetime of the memory allocation.
+You must ensure that the pointer is alive while in use by CuPy.
+In case the pointer lifetime is managed by a Python object, you can pass it to the ``owner`` argument of the :class:`~cupy.cuda.UnownedMemory` to keep the reference to the object.
+
+Export
+******
+
+You can pass memory pointers allocated in CuPy to other libraries.
+
+.. code:: python
+
+   arr = cupy.arange(10)
+   print(arr.data.ptr, arr.nbytes)  # => (140359025819648, 80)
+
+The memory allocated by CuPy will be freed when the :class:`~cupy.ndarray` (``arr``) gets destructed.
+You must keep :class:`~cupy.ndarray` instance alive while the pointer is in use by other libraries.
+
+
+CUDA Stream Pointers
+--------------------
+
+Import
+******
+
+CuPy provides :class:`~cupy.cuda.ExternalStream` API that allows interoperating with CUDA streams created in other libraries.
+
+.. code:: python
+
+   import torch
+
+   # Create a stream on PyTorch.
+   s = torch.cuda.Stream()
+
+   # Switch the current stream in PyTorch.
+   with torch.cuda.stream(s):
+       # Switch the current stream in CuPy, using the pointer of the stream created in PyTorch.
+       with cupy.cuda.ExternalStream(s.cuda_stream):
+           # This block runs on the same CUDA stream.
+           torch.arange(10, device='cuda')
+           cupy.arange(10)
+
+The :class:`~cupy.cuda.ExternalStream` API does not manage the lifetime of the stream.
+You must ensure that the stream pointer is alive while in use by CuPy.
+
+You also need to make sure that the :class:`~cupy.cuda.ExternalStream` object is used on the device where the stream was created.
+CuPy can validate that for you if you pass ``device_id`` argument when creating :class:`~cupy.cuda.ExternalStream`.
+
+Export
+******
+
+You can pass streams created in CuPy to other libraries.
+
+.. code:: python
+
+   s = cupy.cuda.Stream()
+   print(s.ptr, s.device_id)  # => (93997451352336, 0)
+
+The CUDA stream will be destroyed when the :class:`~cupy.cuda.Stream` (``s``) gets destructed.
+You must keep the :class:`~cupy.cuda.Stream` instance alive while the pointer is in use by other libraries.
diff --git a/docs/source/user_guide/kernel.rst b/docs/source/user_guide/kernel.rst
new file mode 100644
index 0000000..b077886
--- /dev/null
+++ b/docs/source/user_guide/kernel.rst
@@ -0,0 +1,581 @@
+.. _udkernel:
+
+User-Defined Kernels
+====================
+
+CuPy provides easy ways to define three types of CUDA kernels: elementwise kernels, reduction kernels and raw kernels.
+In this documentation, we describe how to define and call each kernels.
+
+
+Basics of elementwise kernels
+-----------------------------
+
+An elementwise kernel can be defined by the :class:`~cupy.ElementwiseKernel` class.
+The instance of this class defines a CUDA kernel which can be invoked by the ``__call__`` method of this instance.
+
+A definition of an elementwise kernel consists of four parts: an input argument list, an output argument list, a loop body code, and the kernel name.
+For example, a kernel that computes a squared difference :math:`f(x, y) = (x - y)^2` is defined as follows:
+
+.. doctest::
+
+   >>> squared_diff = cp.ElementwiseKernel(
+   ...    'float32 x, float32 y',
+   ...    'float32 z',
+   ...    'z = (x - y) * (x - y)',
+   ...    'squared_diff')
+
+The argument lists consist of comma-separated argument definitions.
+Each argument definition consists of a *type specifier* and an *argument name*.
+Names of NumPy data types can be used as type specifiers.
+
+.. note::
+   ``n``, ``i``, and names starting with an underscore ``_`` are reserved for the internal use.
+
+The above kernel can be called on either scalars or arrays with broadcasting:
+
+.. doctest::
+
+   >>> x = cp.arange(10, dtype=np.float32).reshape(2, 5)
+   >>> y = cp.arange(5, dtype=np.float32)
+   >>> squared_diff(x, y)
+   array([[ 0.,  0.,  0.,  0.,  0.],
+          [25., 25., 25., 25., 25.]], dtype=float32)
+   >>> squared_diff(x, 5)
+   array([[25., 16.,  9.,  4.,  1.],
+          [ 0.,  1.,  4.,  9., 16.]], dtype=float32)
+
+Output arguments can be explicitly specified (next to the input arguments):
+
+.. doctest::
+
+   >>> z = cp.empty((2, 5), dtype=np.float32)
+   >>> squared_diff(x, y, z)
+   array([[ 0.,  0.,  0.,  0.,  0.],
+          [25., 25., 25., 25., 25.]], dtype=float32)
+
+
+Type-generic kernels
+--------------------
+
+If a type specifier is one character, then it is treated as a **type placeholder**.
+It can be used to define a type-generic kernels.
+For example, the above ``squared_diff`` kernel can be made type-generic as follows:
+
+.. doctest::
+
+   >>> squared_diff_generic = cp.ElementwiseKernel(
+   ...     'T x, T y',
+   ...     'T z',
+   ...     'z = (x - y) * (x - y)',
+   ...     'squared_diff_generic')
+
+Type placeholders of a same character in the kernel definition indicate the same type.
+The actual type of these placeholders is determined by the actual argument type.
+The ElementwiseKernel class first checks the output arguments and then the input arguments to determine the actual type.
+If no output arguments are given on the kernel invocation, then only the input arguments are used to determine the type.
+
+The type placeholder can be used in the loop body code:
+
+.. doctest::
+
+   >>> squared_diff_generic = cp.ElementwiseKernel(
+   ...     'T x, T y',
+   ...     'T z',
+   ...     '''
+   ...         T diff = x - y;
+   ...         z = diff * diff;
+   ...     ''',
+   ...     'squared_diff_generic')
+
+More than one type placeholder can be used in a kernel definition.
+For example, the above kernel can be further made generic over multiple arguments:
+
+.. doctest::
+
+   >>> squared_diff_super_generic = cp.ElementwiseKernel(
+   ...     'X x, Y y',
+   ...     'Z z',
+   ...     'z = (x - y) * (x - y)',
+   ...     'squared_diff_super_generic')
+
+Note that this kernel requires the output argument explicitly specified, because the type ``Z`` cannot be automatically determined from the input arguments.
+
+
+Raw argument specifiers
+-----------------------
+
+The ElementwiseKernel class does the indexing with broadcasting automatically, which is useful to define most elementwise computations.
+On the other hand, we sometimes want to write a kernel with manual indexing for some arguments.
+We can tell the ElementwiseKernel class to use manual indexing by adding the ``raw`` keyword preceding the type specifier.
+
+We can use the special variable ``i`` and method ``_ind.size()`` for the manual indexing.
+``i`` indicates the index within the loop.
+``_ind.size()`` indicates total number of elements to apply the elementwise operation.
+Note that it represents the size **after** broadcast operation.
+
+For example, a kernel that adds two vectors with reversing one of them can be written as follows:
+
+.. doctest::
+
+   >>> add_reverse = cp.ElementwiseKernel(
+   ...     'T x, raw T y', 'T z',
+   ...     'z = x + y[_ind.size() - i - 1]',
+   ...     'add_reverse')
+
+(Note that this is an artificial example and you can write such operation just by ``z = x + y[::-1]`` without defining a new kernel).
+A raw argument can be used like an array.
+The indexing operator ``y[_ind.size() - i - 1]`` involves an indexing computation on ``y``, so ``y`` can be arbitrarily shaped and strode.
+
+Note that raw arguments are not involved in the broadcasting.
+If you want to mark all arguments as ``raw``, you must specify the ``size`` argument on invocation, which defines the value of ``_ind.size()``.
+
+
+Texture memory
+--------------
+Texture objects (:class:`~cupy.cuda.texture.TextureObject`) can be passed to :class:`~cupy.ElementwiseKernel` with their type marked by a unique type placeholder distinct from any other types used in the same kernel, as its actual datatype is determined when populating the texture memory. The texture coordinates can be computed in the kernel by the per-thread loop index ``i``.
+
+
+Reduction kernels
+-----------------
+
+Reduction kernels can be defined by the :class:`~cupy.ReductionKernel` class.
+We can use it by defining four parts of the kernel code:
+
+1. Identity value: This value is used for the initial value of reduction.
+2. Mapping expression: It is used for the pre-processing of each element to be reduced.
+3. Reduction expression: It is an operator to reduce the multiple mapped values.
+   The special variables ``a`` and ``b`` are used for its operands.
+4. Post mapping expression: It is used to transform the resulting reduced values.
+   The special variable ``a`` is used as its input.
+   Output should be written to the output parameter.
+
+ReductionKernel class automatically inserts other code fragments that are required for an efficient and flexible reduction implementation.
+
+For example, L2 norm along specified axes can be written as follows:
+
+.. doctest::
+
+   >>> l2norm_kernel = cp.ReductionKernel(
+   ...     'T x',  # input params
+   ...     'T y',  # output params
+   ...     'x * x',  # map
+   ...     'a + b',  # reduce
+   ...     'y = sqrt(a)',  # post-reduction map
+   ...     '0',  # identity value
+   ...     'l2norm'  # kernel name
+   ... )
+   >>> x = cp.arange(10, dtype=np.float32).reshape(2, 5)
+   >>> l2norm_kernel(x, axis=1)
+   array([ 5.477226 , 15.9687195], dtype=float32)
+
+.. note::
+   ``raw`` specifier is restricted for usages that the axes to be reduced are put at the head of the shape.
+   It means, if you want to use ``raw`` specifier for at least one argument, the ``axis`` argument must be ``0`` or a contiguous increasing sequence of integers starting from ``0``, like ``(0, 1)``, ``(0, 1, 2)``, etc.
+
+.. note::
+   Texture memory is not yet supported in :class:`~cupy.ReductionKernel`.
+
+
+Raw kernels
+-----------
+
+Raw kernels can be defined by the :class:`~cupy.RawKernel` class.
+By using raw kernels, you can define kernels from raw CUDA source.
+
+:class:`~cupy.RawKernel` object allows you to call the kernel with CUDA's ``cuLaunchKernel`` interface.
+In other words, you have control over grid size, block size, shared memory size and stream.
+
+.. doctest::
+
+   >>> add_kernel = cp.RawKernel(r'''
+   ... extern "C" __global__
+   ... void my_add(const float* x1, const float* x2, float* y) {
+   ...     int tid = blockDim.x * blockIdx.x + threadIdx.x;
+   ...     y[tid] = x1[tid] + x2[tid];
+   ... }
+   ... ''', 'my_add')
+   >>> x1 = cp.arange(25, dtype=cp.float32).reshape(5, 5)
+   >>> x2 = cp.arange(25, dtype=cp.float32).reshape(5, 5)
+   >>> y = cp.zeros((5, 5), dtype=cp.float32)
+   >>> add_kernel((5,), (5,), (x1, x2, y))  # grid, block and arguments
+   >>> y
+   array([[ 0.,  2.,  4.,  6.,  8.],
+          [10., 12., 14., 16., 18.],
+          [20., 22., 24., 26., 28.],
+          [30., 32., 34., 36., 38.],
+          [40., 42., 44., 46., 48.]], dtype=float32)
+
+Raw kernels operating on complex-valued arrays can be created as well:
+
+.. doctest::
+
+   >>> complex_kernel = cp.RawKernel(r'''
+   ... #include <cupy/complex.cuh>
+   ... extern "C" __global__
+   ... void my_func(const complex<float>* x1, const complex<float>* x2,
+   ...              complex<float>* y, float a) {
+   ...     int tid = blockDim.x * blockIdx.x + threadIdx.x;
+   ...     y[tid] = x1[tid] + a * x2[tid];
+   ... }
+   ... ''', 'my_func')
+   >>> x1 = cupy.arange(25, dtype=cupy.complex64).reshape(5, 5)
+   >>> x2 = 1j*cupy.arange(25, dtype=cupy.complex64).reshape(5, 5)
+   >>> y = cupy.zeros((5, 5), dtype=cupy.complex64)
+   >>> complex_kernel((5,), (5,), (x1, x2, y, cupy.float32(2.0)))  # grid, block and arguments
+   >>> y
+   array([[ 0. +0.j,  1. +2.j,  2. +4.j,  3. +6.j,  4. +8.j],
+          [ 5.+10.j,  6.+12.j,  7.+14.j,  8.+16.j,  9.+18.j],
+          [10.+20.j, 11.+22.j, 12.+24.j, 13.+26.j, 14.+28.j],
+          [15.+30.j, 16.+32.j, 17.+34.j, 18.+36.j, 19.+38.j],
+          [20.+40.j, 21.+42.j, 22.+44.j, 23.+46.j, 24.+48.j]],
+         dtype=complex64)
+
+Note that while we encourage the usage of ``complex<T>`` types for complex numbers (available by including ``<cupy/complex.cuh>`` as shown above), for CUDA codes already written using functions from ``cuComplex.h`` there is no need to make the conversion yourself: just set the option ``translate_cucomplex=True`` when creating a :class:`~cupy.RawKernel` instance.
+
+The CUDA kernel attributes can be retrieved by either accessing the :attr:`~cupy.RawKernel.attributes` dictionary,
+or by accessing the :class:`~cupy.RawKernel` object's attributes directly; the latter can also be used to set certain
+attributes:
+
+.. doctest::
+
+   >>> add_kernel = cp.RawKernel(r'''
+   ... extern "C" __global__
+   ... void my_add(const float* x1, const float* x2, float* y) {
+   ...     int tid = blockDim.x * blockIdx.x + threadIdx.x;
+   ...     y[tid] = x1[tid] + x2[tid];
+   ... }
+   ... ''', 'my_add')
+   >>> add_kernel.attributes  # doctest: +SKIP
+   {'max_threads_per_block': 1024, 'shared_size_bytes': 0, 'const_size_bytes': 0, 'local_size_bytes': 0, 'num_regs': 10, 'ptx_version': 70, 'binary_version': 70, 'cache_mode_ca': 0, 'max_dynamic_shared_size_bytes': 49152, 'preferred_shared_memory_carveout': -1}
+   >>> add_kernel.max_dynamic_shared_size_bytes  # doctest: +SKIP
+   49152
+   >>> add_kernel.max_dynamic_shared_size_bytes = 50000  # set a new value for the attribute  # doctest: +SKIP
+   >>> add_kernel.max_dynamic_shared_size_bytes  # doctest: +SKIP
+   50000
+
+Dynamical parallelism is supported by :class:`~cupy.RawKernel`. You just need to provide the linking flag (such as ``-dc``) to :class:`~cupy.RawKernel`'s ``options`` argument. The static CUDA device runtime library (``cudadevrt``) is automatically discovered by CuPy. For further detail, see `CUDA Toolkit's documentation`_.
+
+.. _CUDA Toolkit's documentation: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#compiling-and-linking
+
+Accessing texture (surface) memory in :class:`~cupy.RawKernel` is supported via CUDA Runtime's Texture (Surface) Object API, see the documentation for :class:`~cupy.cuda.texture.TextureObject` (:class:`~cupy.cuda.texture.SurfaceObject`) as well as CUDA C Programming Guide. For using the Texture Reference API, which is marked as deprecated as of CUDA Toolkit 10.1, see the introduction to :class:`~cupy.RawModule` below.
+
+If your kernel relies on the C++ std library headers such as ``<type_traits>``, it is likely you will encounter compilation errors. In this case, try enabling CuPy's `Jitify <https://github.com/NVIDIA/jitify>`_ support by setting ``jitify=True`` when creating the :class:`~cupy.RawKernel` instance. It provides basic C++ std support to remedy common errors.
+
+.. note::
+    The kernel does not have return values.
+    You need to pass both input arrays and output arrays as arguments.
+
+.. note::
+    When using ``printf()`` in your CUDA kernel, you may need to synchronize the stream to see the output.
+    You can use ``cupy.cuda.Stream.null.synchronize()`` if you are using the default stream.
+
+.. note::
+    In all of the examples above, we declare the kernels in an ``extern "C"`` block,
+    indicating that the C linkage is used. This is to ensure the kernel names are not
+    mangled so that they can be retrived by name.
+
+Kernel arguments
+----------------
+Python primitive types and NumPy scalars are passed to the kernel by value.
+Array arguments (pointer arguments) have to be passed as CuPy ndarrays.
+No validation is performed by CuPy for arguments passed to the kernel, including types and number of arguments.
+
+Especially note that when passing a CuPy :class:`~cupy.ndarray`, its ``dtype`` should match with the type of the argument declared in the function signature of the CUDA source code (unless you are casting arrays intentionally). 
+
+As an example, ``cupy.float32`` and ``cupy.uint64`` arrays must be passed to the argument typed as ``float*`` and ``unsigned long long*``, respectively. CuPy does not directly support arrays of non-primitive types such as ``float3``, but nothing prevents you from casting a ``float*`` or ``void*`` to a ``float3*`` in a kernel.
+
+Python primitive types, ``int``, ``float``, ``complex`` and ``bool`` map to ``long long``, ``double``, ``cuDoubleComplex`` and ``bool``, respectively.
+
+NumPy scalars (``numpy.generic``) and NumPy arrays (``numpy.ndarray``) **of size one** 
+are passed to the kernel by value.
+This means that you can pass by value any base NumPy types such as ``numpy.int8`` or ``numpy.float64``, provided the kernel arguments match in size. You can refer to this table to match CuPy/NumPy dtype and CUDA types:
+
++-----------------+-----------------------------------------------+------------------+
+| CuPy/NumPy type | Corresponding kernel types                    | itemsize (bytes) |
++=================+===============================================+==================+
+| bool            | bool                                          | 1                |
++-----------------+-----------------------------------------------+------------------+
+| int8            | char, signed char                             | 1                |
++-----------------+-----------------------------------------------+------------------+
+| int16           | short, signed short                           | 2                |
++-----------------+-----------------------------------------------+------------------+
+| int32           | int, signed int                               | 4                |
++-----------------+-----------------------------------------------+------------------+
+| int64           | long long, signed long long                   | 8                |
++-----------------+-----------------------------------------------+------------------+
+| uint8           | unsigned char                                 | 1                |
++-----------------+-----------------------------------------------+------------------+
+| uint16          | unsigned short                                | 2                |
++-----------------+-----------------------------------------------+------------------+
+| uint32          | unsigned int                                  | 4                |
++-----------------+-----------------------------------------------+------------------+
+| uint64          | unsigned long long                            | 8                |
++-----------------+-----------------------------------------------+------------------+
+| float16         | half                                          | 2                |
++-----------------+-----------------------------------------------+------------------+
+| float32         | float                                         | 4                |
++-----------------+-----------------------------------------------+------------------+
+| float64         | double                                        | 8                |
++-----------------+-----------------------------------------------+------------------+
+| complex64       | float2, cuFloatComplex, complex<float>        | 8                |
++-----------------+-----------------------------------------------+------------------+
+| complex128      | double2, cuDoubleComplex, complex<double>     | 16               |
++-----------------+-----------------------------------------------+------------------+
+
+The CUDA standard guarantees that the size of fundamental types on the host and device always match.
+The itemsize of ``size_t``, ``ptrdiff_t``, ``intptr_t``, ``uintptr_t``, 
+``long``, ``signed long`` and ``unsigned long`` are however platform dependent. 
+To pass any CUDA vector builtins such as ``float3`` or any other user defined structure 
+as kernel arguments (provided it matches the device-side kernel parameter type), see :ref:`custom_user_structs` below.
+
+.. _custom_user_structs:
+
+Custom user types
+-----------------
+
+It is possible to use custom types (composite types such as structures and structures of structures)
+as kernel arguments by defining a custom NumPy dtype.
+When doing this, it is your responsibility to match host and device structure memory layout.
+The CUDA standard guarantees that the size of fundamental types on the host and device always match.
+It may however impose device alignment requirements on composite types.
+This means that for composite types the struct member offsets may be different from what you might expect.
+
+When a kernel argument is passed by value, the CUDA driver will copy exactly ``sizeof(param_type)`` bytes starting from the beginning of the NumPy object data pointer, where ``param_type`` is the parameter type in your kernel. 
+You have to match ``param_type``'s memory layout (ex: size, alignment and struct padding/packing) 
+by defining a corresponding `NumPy dtype <https://numpy.org/doc/stable/reference/arrays.dtypes.html>`_.
+
+For builtin CUDA vector types such as ``int2`` and ``double4`` and other packed structures with 
+named members you can directly define such NumPy dtypes as the following:
+
+.. doctest::
+
+    >>> import numpy as np
+    >>> names = ['x', 'y', 'z']
+    >>> types = [np.float32]*3
+    >>> float3 = np.dtype({'names': names, 'formats': types})
+    >>> arg = np.random.rand(3).astype(np.float32).view(float3)
+    >>> print(arg)  # doctest: +SKIP
+    [(0.9940819, 0.62873816, 0.8953669)]
+    >>> arg['x'] = 42.0
+    >>> print(arg)  # doctest: +SKIP
+    [(42., 0.62873816, 0.8953669)]
+
+Here ``arg`` can be used directly as a kernel argument.
+When there is no need to name fields you may prefer this syntax to define packed structures such as 
+vectors or matrices:
+
+.. doctest::
+
+    >>> import numpy as np
+    >>> float5x5 = np.dtype({'names': ['dummy'], 'formats': [(np.float32,(5,5))]}) 
+    >>> arg = np.random.rand(25).astype(np.float32).view(float5x5)
+    >>> print(arg.itemsize)
+    100
+
+Here ``arg`` represents a 100-byte scalar (i.e. a NumPy array of size 1)
+that can be passed by value to any kernel.
+Kernel parameters are passed by value in a dedicated 4kB memory bank which has its own cache with broadcast.
+Upper bound for total kernel parameters size is thus 4kB
+(see `this link <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#function-parameters>`_).
+It may be important to note that this dedicated memory bank is not shared with the device ``__constant__`` memory space.
+
+For now, CuPy offers no helper routines to create user defined composite types. 
+Such composite types can however be built recursively using NumPy dtype `offsets` and `itemsize` capabilities,
+see `cupy/examples/custum_struct <https://github.com/cupy/cupy/tree/main/examples/custom_struct>`_ for examples of advanced usage.
+
+.. warning::
+    You cannot directly pass static arrays as kernel arguments with the ``type arg[N]`` syntax where N is a compile time constant. The signature of ``__global__ void kernel(float arg[5])`` is seen as ``__global__ void kernel(float* arg)`` by the compiler. If you want to pass five floats to the kernel by value you need to define a custom structure ``struct float5 { float val[5]; };`` and modify the kernel signature to ``__global__ void kernel(float5 arg)``.
+
+
+Raw modules
+-----------
+
+For dealing a large raw CUDA source or loading an existing CUDA binary, the :class:`~cupy.RawModule` class can be more handy. It can be initialized either by a CUDA source code, or by a path to the CUDA binary. It accepts most of the arguments as in :class:`~cupy.RawKernel`. The needed kernels can then be retrieved by calling the :meth:`~cupy.RawModule.get_function` method, which returns a :class:`~cupy.RawKernel` instance that can be invoked as discussed above.
+
+.. doctest::
+
+    >>> loaded_from_source = r'''
+    ... extern "C"{
+    ...
+    ... __global__ void test_sum(const float* x1, const float* x2, float* y, \
+    ...                          unsigned int N)
+    ... {
+    ...     unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    ...     if (tid < N)
+    ...     {
+    ...         y[tid] = x1[tid] + x2[tid];
+    ...     }
+    ... }
+    ...
+    ... __global__ void test_multiply(const float* x1, const float* x2, float* y, \
+    ...                               unsigned int N)
+    ... {
+    ...     unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    ...     if (tid < N)
+    ...     {
+    ...         y[tid] = x1[tid] * x2[tid];
+    ...     }
+    ... }
+    ...
+    ... }'''
+    >>> module = cp.RawModule(code=loaded_from_source)
+    >>> ker_sum = module.get_function('test_sum')
+    >>> ker_times = module.get_function('test_multiply')
+    >>> N = 10
+    >>> x1 = cp.arange(N**2, dtype=cp.float32).reshape(N, N)
+    >>> x2 = cp.ones((N, N), dtype=cp.float32)
+    >>> y = cp.zeros((N, N), dtype=cp.float32)
+    >>> ker_sum((N,), (N,), (x1, x2, y, N**2))   # y = x1 + x2
+    >>> assert cp.allclose(y, x1 + x2)
+    >>> ker_times((N,), (N,), (x1, x2, y, N**2)) # y = x1 * x2
+    >>> assert cp.allclose(y, x1 * x2)
+
+The instruction above for using complex numbers in :class:`~cupy.RawKernel` also applies to :class:`~cupy.RawModule`.
+
+For CUDA kernels that need to access global symbols, such as constant memory, the :meth:`~cupy.RawModule.get_global` method can be used, see its documentation for further detail.
+
+Note that the deprecated API :meth:`cupy.RawModule.get_texref` has been removed since CuPy vX.X due to the removal of texture reference support from CUDA.
+
+To support C++ template kernels, :class:`~cupy.RawModule` additionally provide a ``name_expressions`` argument. A list of template specializations should be provided, so that the corresponding kernels can be generated and retrieved by type:
+
+.. doctest::
+
+    >>> code = r'''
+    ... template<typename T>
+    ... __global__ void fx3(T* arr, int N) {
+    ...     unsigned int tid = blockIdx.x * blockDim.x + threadIdx.x;
+    ...     if (tid < N) {
+    ...         arr[tid] = arr[tid] * 3;
+    ...     }
+    ... }
+    ... '''
+    >>>
+    >>> name_exp = ['fx3<float>', 'fx3<double>']
+    >>> mod = cp.RawModule(code=code, options=('-std=c++11',),
+    ...     name_expressions=name_exp)
+    >>> ker_float = mod.get_function(name_exp[0])  # compilation happens here
+    >>> N=10
+    >>> a = cp.arange(N, dtype=cp.float32)
+    >>> ker_float((1,), (N,), (a, N))
+    >>> a
+    array([ 0.,  3.,  6.,  9., 12., 15., 18., 21., 24., 27.], dtype=float32)
+    >>> ker_double = mod.get_function(name_exp[1])
+    >>> a = cp.arange(N, dtype=cp.float64)
+    >>> ker_double((1,), (N,), (a, N))
+    >>> a
+    array([ 0.,  3.,  6.,  9., 12., 15., 18., 21., 24., 27.])
+
+.. note::
+
+    The name expressions used to both initialize a :class:`~cupy.RawModule` instance and retrieve the kernels are
+    the original (*un-mangled*) kernel names with all template parameters unambiguously specified. The name mangling
+    and demangling are handled under the hood so that users do not need to worry about it.
+
+.. _kernel_fusion:
+
+Kernel fusion
+--------------------
+
+:func:`cupy.fuse` is a decorator that fuses functions.  This decorator can be used to define an elementwise or reduction kernel more easily than :class:`~cupy.ElementwiseKernel` or :class:`~cupy.ReductionKernel`.
+
+By using this decorator, we can define the ``squared_diff`` kernel as follows:
+
+.. doctest::
+
+   >>> @cp.fuse()
+   ... def squared_diff(x, y):
+   ...     return (x - y) * (x - y)
+
+The above kernel can be called on either scalars, NumPy arrays or CuPy arrays likes the original function.
+
+.. doctest::
+
+   >>> x_cp = cp.arange(10)
+   >>> y_cp = cp.arange(10)[::-1]
+   >>> squared_diff(x_cp, y_cp)
+   array([81, 49, 25,  9,  1,  1,  9, 25, 49, 81])
+   >>> x_np = np.arange(10)
+   >>> y_np = np.arange(10)[::-1]
+   >>> squared_diff(x_np, y_np)
+   array([81, 49, 25,  9,  1,  1,  9, 25, 49, 81])
+
+At the first function call, the fused function analyzes the original function based on the abstracted information of arguments (e.g. their dtypes and ndims) and creates and caches an actual CUDA kernel.  From the second function call with the same input types, the fused function calls the previously cached kernel, so it is highly recommended to reuse the same decorated functions instead of decorating local functions that are defined multiple times.
+
+:func:`cupy.fuse` also supports simple reduction kernel.
+
+.. doctest::
+
+   >>> @cp.fuse()
+   ... def sum_of_products(x, y):
+   ...     return cp.sum(x * y, axis = -1)
+
+You can specify the kernel name by using the ``kernel_name`` keyword argument as follows:
+
+.. doctest::
+
+   >>> @cp.fuse(kernel_name='squared_diff')
+   ... def squared_diff(x, y):
+   ...     return (x - y) * (x - y)
+
+.. note::
+   Currently, :func:`cupy.fuse` can fuse only simple elementwise and reduction operations.  Most other routines (e.g. :func:`cupy.matmul`, :func:`cupy.reshape`) are not supported.
+
+.. _jit_kernel_definition:
+
+JIT kernel definition
+---------------------
+
+The :class:`cupyx.jit.rawkernel` decorator can create raw CUDA kernels from Python functions.
+
+In this section, a Python function wrapped with the decorator is called a *target function*.
+
+A target function consists of elementary scalar operations, and users have to manage how to parallelize them. CuPy's array operations which automatically parallelize operations (e.g., :func:`~cupy.add`, :func:`~cupy.sum`) are not supported. If a custom kernel based on such array functions is desired, please refer to the :ref:`kernel_fusion` section.
+
+Basic Usage
+^^^^^^^^^^^
+
+Here is a short example for how to write a :class:`cupyx.jit.rawkernel` to copy the values from ``x`` to ``y`` using a grid-stride loop:
+
+.. doctest::
+
+   >>> from cupyx import jit
+   >>>
+   >>> @jit.rawkernel()
+   ... def elementwise_copy(x, y, size):
+   ...     tid = jit.blockIdx.x * jit.blockDim.x + jit.threadIdx.x
+   ...     ntid = jit.gridDim.x * jit.blockDim.x
+   ...     for i in range(tid, size, ntid):
+   ...         y[i] = x[i]
+
+   >>> size = cupy.uint32(2 ** 22)
+   >>> x = cupy.random.normal(size=(size,), dtype=cupy.float32)
+   >>> y = cupy.empty((size,), dtype=cupy.float32)
+
+   >>> elementwise_copy((128,), (1024,), (x, y, size))  # RawKernel style
+   >>> assert (x == y).all()
+
+   >>> elementwise_copy[128, 1024](x, y, size)  #  Numba style
+   >>> assert (x == y).all()
+
+Both styles to launch the kernel, as shown above, are supported. The first two entries are the grid and block sizes, respectively. ``grid`` ( RawKernel style ``(128,)`` or Numba style ``[128]``) is the sizes of the grid, i.e., the numbers of blocks in each dimension; ``block`` (``(1024,)`` or ``[1024]``) is the dimensions of each thread block, please refer to :class:`cupyx.jit._interface._JitRawKernel` for details. Launching a CUDA kernel on a GPU with pre-determined grid/block sizes requires basic understanding in the `CUDA Programming Model`_.
+
+.. _CUDA Programming Model: https://developer.nvidia.com/blog/cuda-refresher-cuda-programming-model/
+
+The compilation will be deferred until the first function call. CuPy's JIT compiler infers the types of arguments at the call time, and will cache the compiled kernels for speeding up any subsequent calls.
+
+See :doc:`../reference/kernel` for a full list of API.
+
+Basic Design
+^^^^^^^^^^^^
+
+CuPy's JIT compiler generates CUDA code via Python AST. We decided not to use Python bytecode to analyze the target function to avoid perforamance degradation. The CUDA source code generated from the Python bytecode will not effectively optimized by CUDA compiler, because for-loops and other control statements of the target function are fully transformed to jump instruction when converting the target function to bytecode.
+
+Typing rule
+^^^^^^^^^^^
+
+The types of local variables are inferred at the first assignment in the function. The first assignment must be done at the top-level of the function; in other words, it must *not* be in ``if``/``else`` bodies or ``for``-loops.
+
+Limitations
+^^^^^^^^^^^
+
+JIT does not work inside Python's interactive interpreter (REPL) as the compiler needs to get the source code of the target function.
diff --git a/docs/source/user_guide/memory.rst b/docs/source/user_guide/memory.rst
new file mode 100644
index 0000000..639d8ff
--- /dev/null
+++ b/docs/source/user_guide/memory.rst
@@ -0,0 +1,200 @@
+Memory Management
+=================
+
+CuPy uses *memory pool* for memory allocations by default.
+The memory pool significantly improves the performance by mitigating the overhead of memory allocation and CPU/GPU synchronization.
+
+There are two different memory pools in CuPy:
+
+* Device memory pool (GPU device memory), which is used for GPU memory allocations.
+* Pinned memory pool (non-swappable CPU memory), which is used during CPU-to-GPU data transfer.
+
+.. attention::
+
+   When you monitor the memory usage (e.g., using ``nvidia-smi`` for GPU memory or ``ps`` for CPU memory), you may notice that memory not being freed even after the array instance become out of scope.
+   This is an expected behavior, as the default memory pool "caches" the allocated memory blocks.
+
+See :doc:`../reference/cuda` for the details of memory management APIs.
+
+For using pinned memory more conveniently, we also provide a few high-level APIs in the ``cupyx`` namespace,
+including :func:`cupyx.empty_pinned`, :func:`cupyx.empty_like_pinned`, :func:`cupyx.zeros_pinned`, and
+:func:`cupyx.zeros_like_pinned`. They return NumPy arrays backed by pinned memory. If CuPy's pinned memory pool
+is in use, the pinned memory is allocated from the pool.
+
+.. note::
+
+    CuPy v8 and above provides a :ref:`FFT plan cache <fft_plan_cache>` that could use a portion of device memory if FFT and related functions are used.
+    The memory taken can be released by shrinking or disabling the cache.
+
+
+Memory Pool Operations
+----------------------
+
+The memory pool instance provides statistics about memory allocation.
+To access the default memory pool instance, use :func:`cupy.get_default_memory_pool` and :func:`cupy.get_default_pinned_memory_pool`.
+You can also free all unused memory blocks hold in the memory pool.
+See the example code below for details:
+
+.. code-block:: py
+
+   import cupy
+   import numpy
+
+   mempool = cupy.get_default_memory_pool()
+   pinned_mempool = cupy.get_default_pinned_memory_pool()
+
+   # Create an array on CPU.
+   # NumPy allocates 400 bytes in CPU (not managed by CuPy memory pool).
+   a_cpu = numpy.ndarray(100, dtype=numpy.float32)
+   print(a_cpu.nbytes)                      # 400
+
+   # You can access statistics of these memory pools.
+   print(mempool.used_bytes())              # 0
+   print(mempool.total_bytes())             # 0
+   print(pinned_mempool.n_free_blocks())    # 0
+
+   # Transfer the array from CPU to GPU.
+   # This allocates 400 bytes from the device memory pool, and another 400
+   # bytes from the pinned memory pool.  The allocated pinned memory will be
+   # released just after the transfer is complete.  Note that the actual
+   # allocation size may be rounded to larger value than the requested size
+   # for performance.
+   a = cupy.array(a_cpu)
+   print(a.nbytes)                          # 400
+   print(mempool.used_bytes())              # 512
+   print(mempool.total_bytes())             # 512
+   print(pinned_mempool.n_free_blocks())    # 1
+
+   # When the array goes out of scope, the allocated device memory is released
+   # and kept in the pool for future reuse.
+   a = None  # (or `del a`)
+   print(mempool.used_bytes())              # 0
+   print(mempool.total_bytes())             # 512
+   print(pinned_mempool.n_free_blocks())    # 1
+
+   # You can clear the memory pool by calling `free_all_blocks`.
+   mempool.free_all_blocks()
+   pinned_mempool.free_all_blocks()
+   print(mempool.used_bytes())              # 0
+   print(mempool.total_bytes())             # 0
+   print(pinned_mempool.n_free_blocks())    # 0
+
+See :class:`cupy.cuda.MemoryPool` and :class:`cupy.cuda.PinnedMemoryPool` for details.
+
+Limiting GPU Memory Usage
+-------------------------
+
+You can hard-limit the amount of GPU memory that can be allocated by using ``CUPY_GPU_MEMORY_LIMIT`` environment variable (see :doc:`../reference/environment` for details).
+
+.. code-block:: py
+
+   # Set the hard-limit to 1 GiB:
+   #   $ export CUPY_GPU_MEMORY_LIMIT="1073741824"
+
+   # You can also specify the limit in fraction of the total amount of memory
+   # on the GPU. If you have a GPU with 2 GiB memory, the following is
+   # equivalent to the above configuration.
+   #   $ export CUPY_GPU_MEMORY_LIMIT="50%"
+
+   import cupy
+   print(cupy.get_default_memory_pool().get_limit())  # 1073741824
+
+You can also set the limit (or override the value specified via the environment variable) using :meth:`cupy.cuda.MemoryPool.set_limit`.
+In this way, you can use a different limit for each GPU device.
+
+.. code-block:: py
+
+   import cupy
+
+   mempool = cupy.get_default_memory_pool()
+
+   with cupy.cuda.Device(0):
+       mempool.set_limit(size=1024**3)  # 1 GiB
+
+   with cupy.cuda.Device(1):
+       mempool.set_limit(size=2*1024**3)  # 2 GiB
+
+.. note::
+
+   CUDA allocates some GPU memory outside of the memory pool (such as CUDA context, library handles, etc.).
+   Depending on the usage, such memory may take one to few hundred MiB.
+   That will not be counted in the limit.
+
+Changing Memory Pool
+--------------------
+
+You can use your own memory allocator instead of the default memory pool by passing the memory allocation function to :func:`cupy.cuda.set_allocator` / :func:`cupy.cuda.set_pinned_memory_allocator`.
+The memory allocator function should take 1 argument (the requested size in bytes) and return :class:`cupy.cuda.MemoryPointer` / :class:`cupy.cuda.PinnedMemoryPointer`.
+
+CuPy provides two such allocators for using managed memory and stream ordered memory on GPU,
+see :func:`cupy.cuda.malloc_managed` and :func:`cupy.cuda.malloc_async`, respectively, for details.
+To enable a memory pool backed by managed memory, you can construct a new :class:`~cupy.cuda.MemoryPool` instance with its allocator
+set to :func:`~cupy.cuda.malloc_managed` as follows
+
+.. code-block:: py
+
+    import cupy
+
+    # Use managed memory
+    cupy.cuda.set_allocator(cupy.cuda.MemoryPool(cupy.cuda.malloc_managed).malloc)
+
+Note that if you pass :func:`~cupy.cuda.malloc_managed` directly to :func:`~cupy.cuda.set_allocator` without constructing
+a :class:`~cupy.cuda.MemoryPool` instance, when the memory is freed it will be released back to the system immediately,
+which may or may not be desired.
+
+Stream Ordered Memory Allocator is a new feature added since CUDA 11.2. CuPy provides an *experimental* interface to it.
+Similar to CuPy's memory pool, Stream Ordered Memory Allocator also allocates/deallocates memory *asynchronously* from/to
+a memory pool in a stream-ordered fashion. The key difference is that it is a built-in feature implemented in the CUDA
+driver by NVIDIA, so other CUDA applications in the same processs can easily allocate memory from the same pool.
+
+To enable a memory pool that manages stream ordered memory, you can construct a new :class:`~cupy.cuda.MemoryAsyncPool`
+instance:
+
+.. code-block:: py
+
+    import cupy
+
+    # Use asynchronous stream ordered memory
+    cupy.cuda.set_allocator(cupy.cuda.MemoryAsyncPool().malloc)
+
+    # Create a custom stream
+    s = cupy.cuda.Stream()
+
+    # This would allocate memory asynchronously on stream s
+    with s:
+        a = cupy.empty((100,), dtype=cupy.float64)
+
+Note that in this case we do not use the :class:`~cupy.cuda.MemoryPool` class. The :class:`~cupy.cuda.MemoryAsyncPool` takes
+a different input argument from that of :class:`~cupy.cuda.MemoryPool` to indicate which pool to use.
+Please refer to :class:`~cupy.cuda.MemoryAsyncPool`'s documentation for further detail.
+
+Note that if you pass :func:`~cupy.cuda.malloc_async` directly to :func:`~cupy.cuda.set_allocator` without constructing
+a :class:`~cupy.cuda.MemoryAsyncPool` instance, the device's *current* memory pool will be used.
+
+When using stream ordered memory, it is important that you maintain a correct stream semantics yourselves using, for example,
+the :class:`~cupy.cuda.Stream` and :class:`~cupy.cuda.Event` APIs (see :ref:`cuda_stream_event` for details); CuPy does not
+attempt to act smartly for you. Upon deallocation, the memory is freed asynchronously either on the stream it was
+allocated (first attempt), or on any current CuPy stream (second attempt). It is permitted that the stream on which the
+memory was allocated gets destroyed before all memory allocated on it is freed.
+
+In addition, applications/libraries internally use ``cudaMalloc`` (CUDA's default, synchronous allocator) could have unexpected
+interplay with Stream Ordered Memory Allocator. Specifically, memory freed to the memory pool might not be immediately visible
+to ``cudaMalloc``, leading to potential out-of-memory errors. In this case, you can either call :meth:`~cupy.cuda.MemoryAsyncPool.free_all_blocks()`
+or just manually perform a (event/stream/device) synchronization, and retry.
+
+Currently the :class:`~cupy.cuda.MemoryAsyncPool` interface is *experimental*. In particular, while its API is largely identical
+to that of :class:`~cupy.cuda.MemoryPool`, several of the pool's methods require a sufficiently new driver (and of course, a
+supported hardware, CUDA version, and platform) due to CUDA's limitation.
+
+You can even disable the default memory pool by the code below.
+Be sure to do this before any other CuPy operations.
+
+.. code-block:: py
+
+   import cupy
+
+   # Disable memory pool for device memory (GPU)
+   cupy.cuda.set_allocator(None)
+
+   # Disable memory pool for pinned memory (CPU).
+   cupy.cuda.set_pinned_memory_allocator(None)
diff --git a/docs/source/user_guide/performance.rst b/docs/source/user_guide/performance.rst
new file mode 100644
index 0000000..d9b36f4
--- /dev/null
+++ b/docs/source/user_guide/performance.rst
@@ -0,0 +1,128 @@
+Performance Best Practices
+==========================
+
+Here we gather a few tricks and advices for improving CuPy's performance.
+
+Benchmarking
+------------
+
+It is utterly important to first identify the performance bottleneck before making any attempt to optimize
+your code. To help set up a baseline benchmark, CuPy provides a useful utility :func:`cupyx.profiler.benchmark`
+for timing the elapsed time of a Python function on both CPU and GPU:
+
+.. doctest::
+
+    >>> from cupyx.profiler import benchmark
+    >>> 
+    >>> def my_func(a):
+    ...     return cp.sqrt(cp.sum(a**2, axis=-1))
+    ... 
+    >>> a = cp.random.random((256, 1024))
+    >>> print(benchmark(my_func, (a,), n_repeat=20))  # doctest: +SKIP
+    my_func             :    CPU:   44.407 us   +/- 2.428 (min:   42.516 / max:   53.098) us     GPU-0:  181.565 us   +/- 1.853 (min:  180.288 / max:  188.608) us
+
+Because GPU executions run asynchronously with respect to CPU executions, a common pitfall in GPU programming is to mistakenly
+measure the elapsed time using CPU timing utilities (such as :py:func:`time.perf_counter` from the Python Standard Library
+or the ``%timeit`` magic from IPython), which have no knowledge in the GPU runtime. :func:`cupyx.profiler.benchmark` addresses
+this by setting up CUDA events on the :ref:`current_stream` right before and after the function to be measured and
+synchronizing over the end event (see :ref:`cuda_stream_event` for detail). Below we sketch what is done internally in :func:`cupyx.profiler.benchmark`:
+
+.. doctest::
+
+    >>> import time
+    >>> start_gpu = cp.cuda.Event()
+    >>> end_gpu = cp.cuda.Event()
+    >>>
+    >>> start_gpu.record()
+    >>> start_cpu = time.perf_counter()
+    >>> out = my_func(a)
+    >>> end_cpu = time.perf_counter()
+    >>> end_gpu.record()
+    >>> end_gpu.synchronize()
+    >>> t_gpu = cp.cuda.get_elapsed_time(start_gpu, end_gpu)
+    >>> t_cpu = end_cpu - start_cpu
+
+Additionally, :func:`cupyx.profiler.benchmark` runs a few warm-up runs to reduce timing fluctuation and exclude the overhead in first invocations.
+
+
+One-Time Overheads
+~~~~~~~~~~~~~~~~~~
+
+Be aware of these overheads when benchmarking CuPy code.
+
+Context Initialization
+......................
+
+It may take several seconds when calling a CuPy function for the first time in a process.
+This is because the CUDA driver creates a CUDA context during the first CUDA API call in CUDA applications.
+
+Kernel Compilation
+..................
+
+CuPy uses on-the-fly kernel synthesis. When a kernel call is required, it compiles a kernel code optimized for the dimensions and dtypes of the given arguments, sends them to the GPU device, and executes the kernel.
+
+CuPy caches the kernel code sent to GPU device within the process, which reduces the kernel compilation time on further calls.
+
+The compiled code is also cached in the directory ``${HOME}/.cupy/kernel_cache`` (the path can be overwritten by setting the :envvar:`CUPY_CACHE_DIR` environment variable).
+This allows reusing the compiled kernel binary across the process.
+
+
+In-depth profiling
+------------------
+
+Under construction. To mark with NVTX/rocTX ranges, you can use the :func:`cupyx.profiler.time_range` API. To start/stop the profiler, you can use the :func:`cupyx.profiler.profile` API.
+
+
+Use CUB/cuTENSOR backends for reduction and other routines
+----------------------------------------------------------
+
+For reduction operations (such as :func:`~cupy.sum`, :func:`~cupy.prod`, :func:`~cupy.amin`, :func:`~cupy.amax`, :func:`~cupy.argmin`, :func:`~cupy.argmax`) and many more routines built upon them, CuPy ships with our own implementations so that things just work out of the box. However, there are dedicated efforts to further accelerate these routines, such as `CUB <https://github.com/NVIDIA/cub>`_ and `cuTENSOR <https://developer.nvidia.com/cutensor>`_.
+
+In order to support more performant backends wherever applicable, starting v8 CuPy introduces an environment variable :envvar:`CUPY_ACCELERATORS` to allow users to specify the desired backends (and in what order they are tried). For example, consider summing over a 256-cubic array:
+
+.. doctest::
+
+    >>> from cupyx.profiler import benchmark
+    >>> a = cp.random.random((256, 256, 256), dtype=cp.float32)
+    >>> print(benchmark(a.sum, (), n_repeat=100))  # doctest: +SKIP
+    sum                 :    CPU:   12.101 us   +/- 0.694 (min:   11.081 / max:   17.649) us     GPU-0:10174.898 us   +/-180.551 (min:10084.576 / max:10595.936) us
+
+We can see that it takes about 10 ms to run (on this GPU). However, if we launch the Python session using ``CUPY_ACCELERATORS=cub python``, we get a ~100x speedup for free (only ~0.1 ms):
+
+.. doctest::
+
+    >>> print(benchmark(a.sum, (), n_repeat=100))  # doctest: +SKIP
+    sum                 :    CPU:   20.569 us   +/- 5.418 (min:   13.400 / max:   28.439) us     GPU-0:  114.740 us   +/- 4.130 (min:  108.832 / max:  122.752) us
+
+CUB is a backend shipped together with CuPy.
+It also accelerates other routines, such as inclusive scans (ex: :func:`~cupy.cumsum`), histograms,
+sparse matrix-vector multiplications (not applicable in CUDA 11), and :class:`~cupy.ReductionKernel`.
+cuTENSOR offers optimized performance for binary elementwise ufuncs, reduction and tensor contraction.
+If cuTENSOR is installed, setting ``CUPY_ACCELERATORS=cub,cutensor``, for example, would try CUB first and fall back to cuTENSOR if CUB does not provide the needed support. In the case that both backends are not applicable, it falls back to CuPy's default implementation.
+
+Note that while in general the accelerated reductions are faster, there could be exceptions
+depending on the data layout. In particular, the CUB reduction only supports reduction over
+contiguous axes.
+In any case, we recommend to perform some benchmarks to determine whether CUB/cuTENSOR offers
+better performance or not.
+
+.. note::
+   CuPy v11 and above uses CUB by default. To turn it off, you need to explicitly specify the environment variable ``CUPY_ACCELERATORS=""``.
+
+
+Overlapping work using streams
+------------------------------
+
+Under construction.
+
+
+Use JIT compiler
+----------------
+
+Under construction. For now please refer to :ref:`jit_kernel_definition` for a quick introduction.
+
+
+Prefer float32 over float64
+---------------------------
+
+Under construction.
diff --git a/examples/cg/cg.py b/examples/cg/cg.py
new file mode 100644
index 0000000..d27baa2
--- /dev/null
+++ b/examples/cg/cg.py
@@ -0,0 +1,84 @@
+import argparse
+import contextlib
+import time
+
+import numpy as np
+
+import cupy
+
+
+@contextlib.contextmanager
+def timer(message):
+    cupy.cuda.Stream.null.synchronize()
+    start = time.time()
+    yield
+    cupy.cuda.Stream.null.synchronize()
+    end = time.time()
+    print('%s:  %f sec' % (message, end - start))
+
+
+def fit(A, b, tol, max_iter):
+    # Note that this function works even tensors 'A' and 'b' are NumPy or CuPy
+    # arrays.
+    xp = cupy.get_array_module(A)
+    x = xp.zeros_like(b, dtype=np.float64)
+    r0 = b - xp.dot(A, x)
+    p = r0
+    for i in range(max_iter):
+        a = xp.inner(r0, r0) / xp.inner(p, xp.dot(A, p))
+        x += a * p
+        r1 = r0 - a * xp.dot(A, p)
+        if xp.linalg.norm(r1) < tol:
+            return x
+        b = xp.inner(r1, r1) / xp.inner(r0, r0)
+        p = r1 + b * p
+        r0 = r1
+    print('Failed to converge. Increase max-iter or tol.')
+    return x
+
+
+def run(gpu_id, tol, max_iter):
+    """CuPy Conjugate gradient example
+
+    Solve simultaneous linear equations, Ax = b.
+    'A' and 'x' are created randomly and 'b' is computed by 'Ax' at first.
+    Then, 'x' is computed from 'A' and 'b' in two ways, namely with CPU and
+    GPU. To evaluate the accuracy of computation, the Euclidean distances
+    between the answer 'x' and the reconstructed 'x' are computed.
+
+    """
+    for repeat in range(3):
+        print('Trial: %d' % repeat)
+        # Create the large symmetric matrix 'A'.
+        N = 2000
+        A = np.random.randint(-50, 50, size=(N, N))
+        A = (A @ A.T).astype(np.float64)
+        x_ans = np.random.randint(-50, 50, size=N).astype(np.float64)
+        b = np.dot(A, x_ans)
+
+        print('Running CPU...')
+        with timer(' CPU '):
+            x_cpu = fit(A, b, tol, max_iter)
+        print(np.linalg.norm(x_cpu - x_ans))
+
+        with cupy.cuda.Device(gpu_id):
+            A_gpu = cupy.asarray(A)
+            b_gpu = cupy.asarray(b)
+            print('Running GPU...')
+            with timer(' GPU '):
+                x_gpu = fit(A_gpu, b_gpu, tol, max_iter)
+            print(np.linalg.norm(cupy.asnumpy(x_gpu) - x_ans))
+
+        print()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu-id', '-g', default=0, type=int,
+                        help='ID of GPU.')
+    parser.add_argument('--tol', '-t', default=0.1, type=float,
+                        help='tolerance to stop iteration')
+    parser.add_argument('--max-iter', '-m', default=5000, type=int,
+                        help='number of iterations')
+    args = parser.parse_args()
+    run(args.gpu_id, args.tol, args.max_iter)
diff --git a/examples/cusparselt/matmul.py b/examples/cusparselt/matmul.py
new file mode 100644
index 0000000..5394477
--- /dev/null
+++ b/examples/cusparselt/matmul.py
@@ -0,0 +1,99 @@
+#
+# Example of matrix multiply using cuSPARSELt
+#
+# (*) https://docs.nvidia.com/cuda/cusparselt/getting_started.html#code-example
+#
+import cupy
+import numpy
+
+from cupy.cuda import runtime
+from cupy_backends.cuda.libs.cusparselt import Handle, MatDescriptor, MatmulDescriptor, MatmulAlgSelection, MatmulPlan  # NOQA
+from cupy_backends.cuda.libs import cusparselt, cusparse
+
+dtype = 'float16'
+m, n, k = 1024, 1024, 1024
+A = cupy.random.random((m, k)).astype(dtype)
+B = cupy.ones((k, n), dtype=dtype)
+C = cupy.zeros((m, n), dtype=dtype)
+
+#
+# initializes cusparselt handle and data structures
+#
+handle = Handle()
+matA = MatDescriptor()
+matB = MatDescriptor()
+matC = MatDescriptor()
+matmul = MatmulDescriptor()
+alg_sel = MatmulAlgSelection()
+plan = MatmulPlan()
+cusparselt.init(handle)
+
+#
+# initializes matrix descriptors
+#
+alignment = 128
+order = cusparse.CUSPARSE_ORDER_ROW
+cuda_dtype = runtime.CUDA_R_16F
+cusparselt.structuredDescriptorInit(handle, matA, A.shape[0], A.shape[1],
+                                    A.shape[1], alignment, cuda_dtype, order,
+                                    cusparselt.CUSPARSELT_SPARSITY_50_PERCENT)
+cusparselt.denseDescriptorInit(handle, matB, B.shape[0], B.shape[1],
+                               B.shape[1], alignment, cuda_dtype, order)
+cusparselt.denseDescriptorInit(handle, matC, C.shape[0], C.shape[1],
+                               C.shape[1], alignment, cuda_dtype, order)
+
+#
+# initializes matmul, algorithm selection and plan
+#
+opA = cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+opB = cusparse.CUSPARSE_OPERATION_NON_TRANSPOSE
+compute_type = cusparselt.CUSPARSE_COMPUTE_16F
+cusparselt.matmulDescriptorInit(handle, matmul, opA, opB, matA, matB, matC,
+                                matC, compute_type)
+cusparselt.matmulAlgSelectionInit(handle, alg_sel, matmul,
+                                  cusparselt.CUSPARSELT_MATMUL_ALG_DEFAULT)
+alg = numpy.array(0, dtype='int32')
+cusparselt.matmulAlgSetAttribute(handle, alg_sel,
+                                 cusparselt.CUSPARSELT_MATMUL_ALG_CONFIG_ID,
+                                 alg.ctypes.data, 4)
+workspace_size = cusparselt.matmulGetWorkspace(handle, alg_sel)
+workspace = cupy.empty(workspace_size, dtype='int8')
+cusparselt.matmulPlanInit(handle, plan, matmul, alg_sel, workspace_size)
+
+#
+# prunes the matrix A in-place and checks the correstness
+#
+print('Before pruning, A[0]:\n{}'.format(A[0]))
+cusparselt.spMMAPrune(handle, matmul, A.data.ptr, A.data.ptr,
+                      cusparselt.CUSPARSELT_PRUNE_SPMMA_TILE)
+print('After pruning, A[0]:\n{}'.format(A[0]))
+is_valid = numpy.array(-1, dtype='int32')
+cusparselt.spMMAPruneCheck(handle, matmul, A.data.ptr, is_valid.ctypes.data)
+
+#
+# compresses the matrix A
+#
+compressed_size = cusparselt.spMMACompressedSize(handle, plan)
+A_compressed = cupy.zeros(compressed_size, dtype='uint8')
+cusparselt.spMMACompress(handle, plan, A.data.ptr, A_compressed.data.ptr)
+
+#
+# matmul: C = A @ B
+#
+alpha = numpy.array(1.0, dtype='float32')
+beta = numpy.array(0.0, dtype='float32')
+cusparselt.matmul(handle, plan, alpha.ctypes.data, A_compressed.data.ptr,
+                  B.data.ptr, beta.ctypes.data, C.data.ptr, C.data.ptr,
+                  workspace.data.ptr)
+
+print('A.sum(axis=1): {}'.format(A.sum(axis=1)))
+print('C[:, 0]: {}'.format(C[:, 0]))
+
+#
+# destroys plan and handle
+#
+cusparselt.matDescriptorDestroy(matA)
+cusparselt.matDescriptorDestroy(matB)
+cusparselt.matDescriptorDestroy(matC)
+cusparselt.matmulPlanDestroy(plan)
+cusparselt.destroy(handle)
diff --git a/examples/custom_struct/README.md b/examples/custom_struct/README.md
new file mode 100644
index 0000000..de81450
--- /dev/null
+++ b/examples/custom_struct/README.md
@@ -0,0 +1,11 @@
+# Custom user structure examples
+
+This folder contains examples of custom user structures in `cupy.RawKernel` (see [https://docs.cupy.dev/en/stable/tutorial/kernel.html](https://docs.cupy.dev/en/stable/tutorial/kernel.html) for corresponding documentation).
+
+This folder provides three scripts ranked by increasing complexity:
+
+1. `builtins_vectors.py` shows how to use CUDA builtin vectors such as `float4` both as scalar parameter (pass by value from host) and array parameter in RawKernels.
+2. `packed_matrix.py` demonstrates how to create and use templated packed structures in RawModules.
+3. `complex_struct.py` illustrates the possibility to recursively build complex NumPy dtypes matching device structure memory layout.
+
+All examples can be run as simple python scripts: `python3.x example_name.py`.
diff --git a/examples/custom_struct/builtin_vectors.py b/examples/custom_struct/builtin_vectors.py
new file mode 100644
index 0000000..667d6e7
--- /dev/null
+++ b/examples/custom_struct/builtin_vectors.py
@@ -0,0 +1,48 @@
+import sys
+import numpy
+import cupy
+
+code = '''
+__device__ double3 operator+(const double3& lhs, const double3& rhs) {
+    return make_double3(lhs.x + rhs.x,
+                        lhs.y + rhs.y,
+                        lhs.z + rhs.z);
+}
+
+extern "C" __global__ void sum_kernel(const double3* lhs,
+                                            double3  rhs,
+                                            double3* out) {
+  int i = threadIdx.x;
+  out[i] = lhs[i] + rhs;
+}
+'''
+
+double3 = numpy.dtype(
+    {
+        'names': ['x', 'y', 'z'],
+        'formats': [numpy.float64]*3
+    }
+)
+
+
+def main():
+    N = 8
+
+    # The kernel computes out = lhs+rhs where lhs and rhs are double3 vectors.
+    # lhs is an array of N such vectors and rhs is double3 kernel parameter.
+
+    lhs = cupy.random.rand(3*N, dtype=numpy.float64).reshape(N, 3)
+    rhs = numpy.random.rand(3).astype(numpy.float64)
+    out = cupy.empty_like(lhs)
+
+    kernel = cupy.RawKernel(code, 'sum_kernel')
+    args = (lhs, rhs.view(double3), out)
+    kernel((1,), (N,), args)
+
+    expected = lhs + cupy.asarray(rhs[None, :])
+    cupy.testing.assert_array_equal(expected, out)
+    print("Kernel output matches expected value.")
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/examples/custom_struct/complex_struct.py b/examples/custom_struct/complex_struct.py
new file mode 100644
index 0000000..cc0dd26
--- /dev/null
+++ b/examples/custom_struct/complex_struct.py
@@ -0,0 +1,133 @@
+import sys
+import numpy
+import cupy
+
+struct_definition = '''
+struct complex_struct {
+    int4 a;
+    char b;
+    double c[2];
+    short1 d;
+    unsigned long long int e[3];
+};
+'''
+
+struct_layout_code = '''
+{struct_definition}
+
+extern "C" __global__ void get_struct_layout(
+                                unsigned long long *itemsize,
+                                unsigned long long *sizes,
+                                unsigned long long *offsets) {{
+    const complex_struct* ptr = nullptr;
+
+    itemsize[0] = sizeof(complex_struct);
+
+    sizes[0] = sizeof(ptr->a);
+    sizes[1] = sizeof(ptr->b);
+    sizes[2] = sizeof(ptr->c);
+    sizes[3] = sizeof(ptr->d);
+    sizes[4] = sizeof(ptr->e);
+
+    offsets[0] = (unsigned long long)&ptr->a;
+    offsets[1] = (unsigned long long)&ptr->b;
+    offsets[2] = (unsigned long long)&ptr->c;
+    offsets[3] = (unsigned long long)&ptr->d;
+    offsets[4] = (unsigned long long)&ptr->e;
+}}
+'''.format(struct_definition=struct_definition)
+
+
+kernel_code = '''
+{struct_definition}
+
+extern "C" __global__ void test_kernel(const complex_struct s,
+                                       double* out) {{
+    int i = threadIdx.x;
+    double sum = 0.0;
+    sum += s.a.x + s.a.y + s.a.z + s.a.w;
+    sum += s.b;
+    sum += s.c[0] + s.c[1];
+    sum += s.d.x;
+    sum += s.e[0] + s.e[1] + s.e[2];
+    out[i] = i * sum;
+}}
+'''.format(struct_definition=struct_definition)
+
+
+def make_packed(basetype, N, itemsize):
+    # A small utility function to make packed structs
+    # Can represent simple packed vectors such as float4 or double[3].
+    assert 0 < N <= 4, N
+    names = list('xyzw')[:N]
+    formats = [basetype]*N
+    return numpy.dtype(dict(names=names,
+                            formats=formats,
+                            itemsize=itemsize))
+
+
+def main():
+    # This program demonstrate how to build a hostside
+    # representation of device structure 'complex_struct'
+    # defined in variable 'struct_definition' that can be
+    # used as a RawKernel argument.
+
+    # First step is to determine structure memory layout
+    #  itemsize -> overall struct size
+    #  sizes    -> individual struct member sizes, determined with sizeof
+    #  offsets  -> individual struct member offsets, determined with offsetof
+    # Results (in terms of bytes) are copied to host after kernel launch.
+    # Note that 'complex_struct' has 5 members named a, b, c, d and e.
+    itemsize = cupy.ndarray(shape=(1,), dtype=numpy.uint64)
+    sizes = cupy.ndarray(shape=(5,), dtype=numpy.uint64)
+    offsets = cupy.ndarray(shape=(5,), dtype=numpy.uint64)
+
+    kernel = cupy.RawKernel(
+        struct_layout_code, 'get_struct_layout', options=('--std=c++11',))
+    kernel((1,), (1,), (itemsize, sizes, offsets))
+
+    (itemsize, sizes, offsets) = map(cupy.asnumpy, (itemsize, sizes, offsets))
+    print("Overall structure itemsize: {} bytes".format(itemsize.item()))
+    print("Structure members itemsize: {}".format(sizes))
+    print("Structure members offsets: {}".format(offsets))
+
+    # Second step: build a numpy dtype for each struct member
+    atype = make_packed(numpy.int32,   4, sizes[0])
+    btype = make_packed(numpy.int8,    1, sizes[1])
+    ctype = make_packed(numpy.float64, 2, sizes[2])
+    dtype = make_packed(numpy.int16,   1, sizes[3])
+    etype = make_packed(numpy.uint64,  3, sizes[4])
+
+    # Third step: create the complex struct representation with
+    #  the right offsets
+    names = list('abcde')
+    formats = [atype, btype, ctype, dtype, etype]
+    complex_struct = numpy.dtype(dict(names=names,
+                                      formats=formats,
+                                      offsets=offsets,
+                                      itemsize=itemsize.item()))
+
+    # Build a complex_struct kernel argument
+    s = numpy.empty(shape=(1,), dtype=complex_struct)
+    s['a'] = numpy.arange(0, 4).astype(numpy.int32).view(atype)
+    s['b'] = numpy.arange(4, 5).astype(numpy.int8).view(btype)
+    s['c'] = numpy.arange(5, 7).astype(numpy.float64).view(ctype)
+    s['d'] = numpy.arange(7, 8).astype(numpy.int16).view(dtype)
+    s['e'] = numpy.arange(8, 11).astype(numpy.uint64).view(etype)
+    print("Complex structure value:\n  {}".format(s))
+
+    # Setup test kernel
+    N = 8
+    out = cupy.empty(shape=(N,), dtype=numpy.float64)
+    kernel = cupy.RawKernel(kernel_code, 'test_kernel')
+    kernel((1,), (N,), (s, out))
+
+    # the sum of all members of our complex struct instance is 55.0
+    expected = cupy.arange(N) * 55.0
+
+    cupy.testing.assert_array_almost_equal(expected, out)
+    print("Kernel output matches expected value.")
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/examples/custom_struct/packed_matrix.py b/examples/custom_struct/packed_matrix.py
new file mode 100644
index 0000000..85fe7ca
--- /dev/null
+++ b/examples/custom_struct/packed_matrix.py
@@ -0,0 +1,91 @@
+import sys
+import numpy
+import cupy
+
+code = '''
+template<typename T>
+struct Matrix {
+    T value[4][4];
+
+    __device__ T& operator() (int i, int j) {
+        return this->value[i][j];
+    }
+
+    __device__ const T& operator() (int i, int j) const {
+        return this->value[i][j];
+    }
+};
+
+template<typename T>
+__device__ Matrix<T> operator+ (const Matrix<T>& lhs, const Matrix<T>& rhs) {
+    Matrix<T> res;
+    for (int i = 0; i<4; i++) {
+        for (int j = 0; j<4; j++) {
+            res(i,j) = lhs(i,j) + rhs(i,j);
+        }
+    }
+    return res;
+}
+
+template<typename T>
+__device__ Matrix<T> operator* (const Matrix<T>& lhs, const Matrix<T>& rhs) {
+    Matrix<T> res;
+    for (int i = 0; i<4; i++) {
+        for (int j = 0; j<4; j++) {
+            res(i,j) = T(0);
+            for (int k = 0; k<4; k++) {
+                res(i,j) += lhs(i,k) * rhs(k,j);
+            }
+        }
+    }
+    return res;
+}
+
+template<typename T>
+__global__ void kernel(const Matrix<T>* A,
+                       const Matrix<T>* B,
+                       const Matrix<T>  C,
+                             Matrix<T>* out) {
+  int i = threadIdx.x;
+  out[i] = A[i] * B[i] + C;
+}
+'''
+
+
+def main():
+    N = 8
+    module = cupy.RawModule(code=code, options=('-std=c++11',),
+                            name_expressions=('kernel<float>',
+                                              'kernel<double>'))
+
+    # The kernel computes out = A*B+C where A, B and C are 4x4 matrices.
+    # A and B are arrays of N such matrices and C is a matrix kernel parameter.
+
+    for (ctype, dtype) in zip(('float', 'double'),
+                              (numpy.float32, numpy.float64)):
+
+        A = cupy.random.rand(16*N, dtype=dtype).reshape(N, 4, 4)
+        B = cupy.random.rand(16*N, dtype=dtype).reshape(N, 4, 4)
+        C = numpy.random.rand(16).astype(dtype).reshape(4, 4)
+        out = cupy.empty_like(A)
+
+        Matrix = numpy.dtype(
+            {
+                'names': ['value'],
+                'formats': [(dtype, (4, 4))]
+            }
+        )
+
+        kernel = module.get_function('kernel<{}>'.format(ctype))
+        args = (A, B, C.ravel().view(Matrix), out)
+        kernel((1,), (N,), args)
+
+        expected = cupy.matmul(A, B) + cupy.asarray(C[None, :, :])
+
+        cupy.testing.assert_array_almost_equal(expected, out)
+        print("Kernel output matches expected value for "
+              "type '{}'.".format(ctype))
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/examples/cutensor/contraction.py b/examples/cutensor/contraction.py
new file mode 100644
index 0000000..63a1414
--- /dev/null
+++ b/examples/cutensor/contraction.py
@@ -0,0 +1,45 @@
+#
+# C_{m,u,n,v} = alpha * A_{m,h,k,n} * B_{u,k,v,h} + beta * C_{m,u,n,v}
+#
+import numpy
+import cupy
+from cupyx import cutensor
+import cupyx.time
+
+
+dtype = numpy.float32
+
+mode_a = ('m', 'h', 'k', 'n')
+mode_b = ('u', 'k', 'v', 'h')
+mode_c = ('m', 'u', 'n', 'v')
+
+extent = {'m': 96, 'n': 96, 'u': 96, 'v': 64, 'h': 64, 'k': 64}
+
+a = cupy.random.random([extent[i] for i in mode_a])
+b = cupy.random.random([extent[i] for i in mode_b])
+c = cupy.random.random([extent[i] for i in mode_c])
+a = a.astype(dtype)
+b = b.astype(dtype)
+c = c.astype(dtype)
+
+desc_a = cutensor.create_tensor_descriptor(a)
+desc_b = cutensor.create_tensor_descriptor(b)
+desc_c = cutensor.create_tensor_descriptor(c)
+
+mode_a = cutensor.create_mode(*mode_a)
+mode_b = cutensor.create_mode(*mode_b)
+mode_c = cutensor.create_mode(*mode_c)
+alpha = 1.1
+beta = 1.0
+
+perf = cupyx.time.repeat(
+    cutensor.contraction,
+    (alpha, a, desc_a, mode_a, b, desc_b, mode_b, beta, c, desc_c, mode_c),
+    n_warmup=1, n_repeat=5)
+
+total_flops = 2 * numpy.prod(numpy.array(list(extent.values())))
+elapsed = perf.gpu_times.mean()
+
+print('dtype: {}'.format(numpy.dtype(dtype).name))
+print(perf)
+print('GFLOPS: {}'.format(total_flops / elapsed / 1e9))
diff --git a/examples/cutensor/elementwise_binary.py b/examples/cutensor/elementwise_binary.py
new file mode 100644
index 0000000..b4d3a9c
--- /dev/null
+++ b/examples/cutensor/elementwise_binary.py
@@ -0,0 +1,46 @@
+#
+# D_{x,y,z} = alpha * A_{z,y,x} + gamma * C_{x,y,z}
+#
+import numpy
+import cupy
+from cupyx import cutensor
+import cupyx.time
+
+
+dtype = numpy.float32
+
+mode_a = ('z', 'y', 'x')
+mode_c = ('x', 'y', 'z')
+
+extent = {'x': 400, 'y': 200, 'z': 300}
+
+a = cupy.random.random([extent[i] for i in mode_a])
+c = cupy.random.random([extent[i] for i in mode_c])
+a = a.astype(dtype)
+c = c.astype(dtype)
+
+desc_a = cutensor.create_tensor_descriptor(a)
+desc_c = cutensor.create_tensor_descriptor(c)
+
+mode_a = cutensor.create_mode(*mode_a)
+mode_c = cutensor.create_mode(*mode_c)
+alpha = 1.1
+gamma = 1.3
+
+perf = cupyx.time.repeat(
+    cutensor.elementwise_binary,
+    (alpha, a, desc_a, mode_a, gamma, c, desc_c, mode_c),
+    n_warmup=1, n_repeat=5)
+
+itemsize = numpy.dtype(dtype).itemsize
+transfer_byte = a.size * itemsize
+if alpha != 0.0:
+    transfer_byte += a.size * itemsize
+if gamma != 0.0:
+    transfer_byte += c.size * itemsize
+elapsed = perf.gpu_times.mean()
+gbs = transfer_byte / elapsed / 1e9
+
+print('dtype: {}'.format(numpy.dtype(dtype).name))
+print(perf)
+print('effective memory bandwidth (GB/s): {}'.format(gbs))
diff --git a/examples/cutensor/elementwise_trinary.py b/examples/cutensor/elementwise_trinary.py
new file mode 100644
index 0000000..63aacb1
--- /dev/null
+++ b/examples/cutensor/elementwise_trinary.py
@@ -0,0 +1,56 @@
+#
+# D_{x,y,z} = alpha * A_{z,y,x} + beta * B_{y,z,x} + gamma * C_{x,y,z}
+#
+import numpy
+import cupy
+from cupyx import cutensor
+import cupyx.time
+
+
+dtype = numpy.float32
+
+mode_a = ('z', 'y', 'x')
+mode_b = ('y', 'z', 'x')
+mode_c = ('x', 'y', 'z')
+
+extent = {'x': 400, 'y': 200, 'z': 300}
+
+a = cupy.random.random([extent[i] for i in mode_a])
+b = cupy.random.random([extent[i] for i in mode_b])
+c = cupy.random.random([extent[i] for i in mode_c])
+a = a.astype(dtype)
+b = b.astype(dtype)
+c = c.astype(dtype)
+
+desc_a = cutensor.create_tensor_descriptor(a)
+desc_b = cutensor.create_tensor_descriptor(b)
+desc_c = cutensor.create_tensor_descriptor(c)
+
+mode_a = cutensor.create_mode(*mode_a)
+mode_b = cutensor.create_mode(*mode_b)
+mode_c = cutensor.create_mode(*mode_c)
+alpha = 1.1
+beta = 1.2
+gamma = 1.3
+
+perf = cupyx.time.repeat(
+    cutensor.elementwise_trinary,
+    (alpha, a, desc_a, mode_a,
+     beta, b, desc_b, mode_b,
+     gamma, c, desc_c, mode_c),
+    n_warmup=1, n_repeat=5)
+
+itemsize = numpy.dtype(dtype).itemsize
+transfer_byte = a.size * itemsize
+if alpha != 0.0:
+    transfer_byte += a.size * itemsize
+if beta != 0.0:
+    transfer_byte += b.size * itemsize
+if gamma != 0.0:
+    transfer_byte += c.size * itemsize
+elapsed = perf.gpu_times.mean()
+gbs = transfer_byte / elapsed / 1e9
+
+print('dtype: {}'.format(numpy.dtype(dtype).name))
+print(perf)
+print('effective memory bandwidth (GB/s): {}'.format(gbs))
diff --git a/examples/cutensor/reduction.py b/examples/cutensor/reduction.py
new file mode 100644
index 0000000..a8384bf
--- /dev/null
+++ b/examples/cutensor/reduction.py
@@ -0,0 +1,43 @@
+#
+# C_{m,v} = alpha * A_{m,h,k,v} + beta * C_{m,v}
+#
+import numpy
+import cupy
+from cupyx import cutensor
+import cupyx.time
+
+
+dtype = numpy.float32
+
+mode_a = ('m', 'h', 'k', 'v')
+mode_c = ('m', 'v')
+
+extent = {'m': 196, 'h': 256, 'k': 64, 'v': 64}
+
+a = cupy.random.random([extent[i] for i in mode_a])
+c = cupy.random.random([extent[i] for i in mode_c])
+a = a.astype(dtype)
+c = c.astype(dtype)
+
+desc_a = cutensor.create_tensor_descriptor(a)
+desc_c = cutensor.create_tensor_descriptor(c)
+
+mode_a = cutensor.create_mode(*mode_a)
+mode_c = cutensor.create_mode(*mode_c)
+alpha = 1.0
+beta = 0.1
+
+perf = cupyx.time.repeat(
+    cutensor.reduction,
+    (alpha, a, desc_a, mode_a, beta, c, desc_c, mode_c),
+    n_warmup=1, n_repeat=5)
+
+transfer_byte = a.size * a.itemsize + c.size * c.itemsize
+if beta != 0.0:
+    transfer_byte += c.size * c.itemsize
+elapsed = perf.gpu_times.mean()
+gbs = transfer_byte / elapsed / 1e9
+
+print('dtype: {}'.format(numpy.dtype(dtype).name))
+print(perf)
+print('effective memory bandwidth (GB/s): {}'.format(gbs))
diff --git a/examples/finance/black_scholes.py b/examples/finance/black_scholes.py
new file mode 100644
index 0000000..d9863ff
--- /dev/null
+++ b/examples/finance/black_scholes.py
@@ -0,0 +1,144 @@
+import argparse
+import contextlib
+import time
+
+import cupy
+import numpy
+
+# This sample computes call and put prices for European options with
+# Black-Scholes equation. It was based on a sample of the financial package
+# in CUDA toolkit. For details, please see the corresponding whitepaper.
+#
+# The following code shows that CuPy enables us to write algorithms for GPUs
+# without significantly modifying the existing NumPy's code.
+# It also briefly describes how to create your own kernel with CuPy.
+# If you want to speed up the existing code, please define the kernel
+# with cupy.ElementwiseKernel.
+
+
+# Naive implementation of the pricing of options with NumPy and CuPy.
+def black_scholes(xp, s, x, t, r, v):
+    sqrt_t = xp.sqrt(t)
+    d1 = (xp.log(s / x) + (r + v * v / 2) * t) / (v * sqrt_t)
+    d2 = d1 - v * sqrt_t
+
+    def get_cumulative_normal_distribution(x):
+        A1 = 0.31938153
+        A2 = -0.356563782
+        A3 = 1.781477937
+        A4 = -1.821255978
+        A5 = 1.330274429
+        RSQRT2PI = 0.39894228040143267793994605993438
+        W = 0.2316419
+
+        k = 1 / (1 + W * xp.abs(x))
+        cnd = RSQRT2PI * xp.exp(-x * x / 2) * \
+            (k * (A1 + k * (A2 + k * (A3 + k * (A4 + k * A5)))))
+        cnd = xp.where(x > 0, 1 - cnd, cnd)
+        return cnd
+
+    cnd_d1 = get_cumulative_normal_distribution(d1)
+    cnd_d2 = get_cumulative_normal_distribution(d2)
+
+    exp_rt = xp.exp(- r * t)
+    call = s * cnd_d1 - x * exp_rt * cnd_d2
+    put = x * exp_rt * (1 - cnd_d2) - s * (1 - cnd_d1)
+    return call, put
+
+
+# An example of calling the kernel via cupy.ElementwiseKernel.
+# When executing __call__ method of the instance, it automatically compiles
+# the code depending on the types of the given arrays, and calls the kernel.
+# Other functions used inside the kernel can be defined by 'preamble' option.
+black_scholes_kernel = cupy.ElementwiseKernel(
+    'T s, T x, T t, T r, T v',  # Inputs
+    'T call, T put',  # Outputs
+    '''
+    const T sqrt_t = sqrt(t);
+    const T d1 = (log(s / x) + (r + v * v / 2) * t) / (v * sqrt_t);
+    const T d2 = d1 - v * sqrt_t;
+
+    const T cnd_d1 = get_cumulative_normal_distribution(d1);
+    const T cnd_d2 = get_cumulative_normal_distribution(d2);
+
+    const T exp_rt = exp(- r * t);
+    call = s * cnd_d1 - x * exp_rt * cnd_d2;
+    put = x * exp_rt * (1 - cnd_d2) - s * (1 - cnd_d1);
+    ''',
+    'black_scholes_kernel',
+    preamble='''
+    __device__
+    inline T get_cumulative_normal_distribution(T x) {
+        const T A1 = 0.31938153;
+        const T A2 = -0.356563782;
+        const T A3 = 1.781477937;
+        const T A4 = -1.821255978;
+        const T A5 = 1.330274429;
+        const T RSQRT2PI = 0.39894228040143267793994605993438;
+        const T W = 0.2316419;
+
+        const T k = 1 / (1 + W * abs(x));
+        T cnd = RSQRT2PI * exp(- x * x / 2) *
+            (k * (A1 + k * (A2 + k * (A3 + k * (A4 + k * A5)))));
+        if (x > 0) {
+            cnd = 1 - cnd;
+        }
+        return cnd;
+    }
+    ''',
+)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu-id', '-g', default=0, type=int, help='GPU ID')
+    parser.add_argument('--n-options', '-n', default=10000000, type=int)
+    args = parser.parse_args()
+
+    cupy.cuda.Device(args.gpu_id).use()
+
+    def rand_range(m, M):
+        samples = cupy.random.rand(args.n_options)
+        return (m + (M - m) * samples).astype(numpy.float64)
+
+    print('initializing...')
+    stock_price_gpu = rand_range(5, 30)
+    option_strike_gpu = rand_range(1, 100)
+    option_years_gpu = rand_range(0.25, 10)
+
+    stock_price_cpu = stock_price_gpu.get()
+    option_strike_cpu = option_strike_gpu.get()
+    option_years_cpu = option_years_gpu.get()
+
+    @contextlib.contextmanager
+    def timer(message):
+        cupy.cuda.Stream.null.synchronize()
+        start = time.time()
+        yield
+        cupy.cuda.Stream.null.synchronize()
+        end = time.time()
+        print('%s:\t%f sec' % (message, end - start))
+
+    print('start computation')
+    risk_free = 0.02
+    volatility = 0.3
+    with timer(' CPU (NumPy, Naive implementation)'):
+        call_cpu, put_cpu = black_scholes(
+            numpy, stock_price_cpu, option_strike_cpu, option_years_cpu,
+            risk_free, volatility)
+
+    with timer(' GPU (CuPy, Naive implementation)'):
+        call_gpu1, put_gpu1 = black_scholes(
+            cupy, stock_price_gpu, option_strike_gpu, option_years_gpu,
+            risk_free, volatility)
+
+    with timer(' GPU (CuPy, Elementwise kernel)'):
+        call_gpu2, put_gpu2 = black_scholes_kernel(
+            stock_price_gpu, option_strike_gpu, option_years_gpu,
+            risk_free, volatility)
+
+    # Check whether all elements in gpu arrays are equal to those of cpus
+    cupy.testing.assert_allclose(call_cpu, call_gpu1)
+    cupy.testing.assert_allclose(call_cpu, call_gpu2)
+    cupy.testing.assert_allclose(put_cpu, put_gpu1)
+    cupy.testing.assert_allclose(put_cpu, put_gpu2)
diff --git a/examples/finance/monte_carlo.py b/examples/finance/monte_carlo.py
new file mode 100644
index 0000000..5f26238
--- /dev/null
+++ b/examples/finance/monte_carlo.py
@@ -0,0 +1,161 @@
+import argparse
+import contextlib
+import sys
+import time
+
+import cupy
+import numpy
+
+from black_scholes import black_scholes_kernel
+
+# This sample computes call prices for European options with
+# Monte-Carlo simulation. It was based on a sample of the financial package
+# in CUDA toolkit. For details, please see the corresponding whitepaper.
+#
+# The present price of an option can also be represented as a discounted
+# expectation of the option price under a risk-neutral measure.
+# Since it is assumed that the stock price follows a lognormal distribution,
+# the call price for European option can be evaluated by approximating the
+# risk-neutral expectation at the time of exercise with the Monte-Carlo method.
+# Note that as current ElementwiseKernel does not support 'curand'
+# due to nvrtc, this sample manually implements a pseudorandom function.
+
+
+monte_carlo_kernel = cupy.ElementwiseKernel(
+    'T s, T x, T t, T r, T v, int32 n_samples, int32 seed', 'T call',
+    '''
+    // We can use special variables i and _ind to get the index of the thread.
+    // In this case, we used an index as a seed of random sequence.
+    uint64_t rand_state[2];
+    init_state(rand_state, i, seed);
+
+    T call_sum = 0;
+    const T v_by_sqrt_t = v * sqrt(t);
+    const T mu_by_t = (r - v * v / 2) * t;
+
+    // compute the price of the call option with Monte Carlo method
+    for (int i = 0; i < n_samples; ++i) {
+        const T p = sample_normal(rand_state);
+        call_sum += get_call_value(s, x, p, mu_by_t, v_by_sqrt_t);
+    }
+    // convert the future value of the call option to the present value
+    const T discount_factor = exp(- r * t);
+    call = discount_factor * call_sum / n_samples;
+    ''',
+    preamble='''
+    #ifndef __HIPCC__
+    typedef unsigned long long uint64_t;
+    #endif
+
+    __device__
+    inline T get_call_value(T s, T x, T p, T mu_by_t, T v_by_sqrt_t) {
+        const T call_value = s * exp(mu_by_t + v_by_sqrt_t * p) - x;
+        return (call_value > 0) ? call_value : 0;
+    }
+
+    // Initialize state
+    __device__ inline void init_state(uint64_t* a, int i, int seed) {
+        a[0] = i + 1;
+        a[1] = 0x5c721fd808f616b6 + seed;
+    }
+
+    __device__ inline uint64_t xorshift128plus(uint64_t* x) {
+        uint64_t s1 = x[0];
+        uint64_t s0 = x[1];
+        x[0] = s0;
+        s1 = s1 ^ (s1 << 23);
+        s1 = s1 ^ (s1 >> 17);
+        s1 = s1 ^ s0;
+        s1 = s1 ^ (s0 >> 26);
+        x[1] = s1;
+        return s0 + s1;
+    }
+
+    // Draw a sample from an uniform distribution in a range of [0, 1]
+    __device__ inline T sample_uniform(uint64_t* state) {
+        const uint64_t x = xorshift128plus(state);
+        // 18446744073709551615 = 2^64 - 1
+        return T(x) / T(18446744073709551615);
+    }
+
+    // Draw a sample from a normal distribution with N(0, 1)
+    __device__ inline T sample_normal(uint64_t* state) {
+        T x = sample_uniform(state);
+        T s = T(-1.4142135623730950488016887242097);  // = -sqrt(2)
+        if (x > 0.5) {
+            x = 1 - x;
+            s = -s;
+        }
+        T p = x + T(0.5);
+        return s * erfcinv(2 * p);
+    }
+    ''',
+)
+
+
+def compute_option_prices(
+        stock_price, option_strike, option_years, risk_free, volatility,
+        n_threads_per_option, n_samples_per_thread, seed=0):
+
+    n_options = len(stock_price)
+    call_prices = cupy.empty(
+        (n_options, n_threads_per_option), dtype=numpy.float64)
+    # Because of the broadcasting rule, in this case this kernel
+    # launches n_options * n_threads_per_options threads
+    # each of which corresponds to the element of 'call_prices'.
+    monte_carlo_kernel(
+        stock_price[:, None], option_strike[:, None], option_years[:, None],
+        risk_free, volatility, n_samples_per_thread, seed, call_prices)
+    return call_prices.mean(axis=1)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu-id', '-g', default=0, type=int, help='GPU ID')
+    parser.add_argument('--n-options', default=1000, type=int)
+    parser.add_argument('--n-samples-per-thread', default=1000, type=int)
+    parser.add_argument('--n-threads-per-option', default=100000, type=int)
+    args = parser.parse_args()
+
+    cupy.cuda.Device(args.gpu_id).use()
+
+    def rand_range(m, M):
+        samples = cupy.random.rand(args.n_options)
+        return (m + (M - m) * samples).astype(numpy.float64)
+
+    print('initializing...')
+    stock_price = rand_range(5, 30)
+    option_strike = rand_range(1, 100)
+    option_years = rand_range(0.25, 10)
+    risk_free = 0.02
+    volatility = 0.3
+
+    @contextlib.contextmanager
+    def timer(message):
+        cupy.cuda.Stream.null.synchronize()
+        start = time.time()
+        yield
+        cupy.cuda.Stream.null.synchronize()
+        end = time.time()
+        print('%s:\t%f sec' % (message, end - start))
+
+    print('start computation')
+    print('    # of options: {}'.format(args.n_options))
+    print('    # of samples per option: {}'.format(
+        args.n_samples_per_thread * args.n_threads_per_option))
+    with timer('GPU (CuPy, Monte Carlo method)'):
+        call_mc = compute_option_prices(
+            stock_price, option_strike, option_years, risk_free, volatility,
+            args.n_threads_per_option, args.n_samples_per_thread)
+
+    # Compute the error between the value of the exact solution
+    # and that of the Monte-Carlo simulation
+    call_bs, _ = black_scholes_kernel(
+        stock_price, option_strike, option_years, risk_free, volatility)
+    error = cupy.std(call_mc - call_bs)
+    print('Error: %f' % error)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/examples/finance/monte_carlo_multigpu.py b/examples/finance/monte_carlo_multigpu.py
new file mode 100644
index 0000000..508b3ca
--- /dev/null
+++ b/examples/finance/monte_carlo_multigpu.py
@@ -0,0 +1,93 @@
+import argparse
+import contextlib
+import sys
+import time
+
+import cupy
+import numpy
+
+from black_scholes import black_scholes_kernel
+from monte_carlo import monte_carlo_kernel
+
+# CuPy also implements a feature to call kernels in different GPUs.
+# Through this sample, we will explain how to allocate arrays
+# in different devices, and call kernels in parallel.
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpus', type=int, nargs='*',
+                        default=[0], help='GPU IDs')
+    parser.add_argument('--n-options', default=1000, type=int)
+    parser.add_argument('--n-samples-per-thread', default=1000, type=int)
+    parser.add_argument('--n-threads-per-option', default=10000, type=int)
+    args = parser.parse_args()
+
+    if len(args.gpus) == 0:
+        print('At least one GPU is required.')
+        sys.exit(1)
+
+    def rand_range(m, M):
+        samples = numpy.random.rand(args.n_options)
+        return (m + (M - m) * samples).astype(numpy.float64)
+
+    print('initializing...')
+    stock_price_cpu = rand_range(5, 30)
+    option_strike_cpu = rand_range(1, 100)
+    option_years_cpu = rand_range(0.25, 10)
+    risk_free = 0.02
+    volatility = 0.3
+
+    stock_price_gpus = []
+    option_strike_gpus = []
+    option_years_gpus = []
+    call_prices_gpus = []
+    print('start computation')
+    print('    # of gpus: {}'.format(len(args.gpus)))
+    print('    # of options: {}'.format(args.n_options))
+    print('    # of samples per option: {}'.format(
+        len(args.gpus) * args.n_samples_per_thread * args.n_threads_per_option)
+    )
+    # Allocate arrays in different devices
+    for gpu_id in args.gpus:
+        with cupy.cuda.Device(gpu_id):
+            stock_price_gpus.append(cupy.array(stock_price_cpu))
+            option_strike_gpus.append(cupy.array(option_strike_cpu))
+            option_years_gpus.append(cupy.array(option_years_cpu))
+            call_prices_gpus.append(cupy.empty(
+                (args.n_options, args.n_threads_per_option),
+                dtype=numpy.float64))
+
+    @contextlib.contextmanager
+    def timer(message):
+        cupy.cuda.Stream.null.synchronize()
+        start = time.time()
+        yield
+        cupy.cuda.Stream.null.synchronize()
+        end = time.time()
+        print('%s:\t%f sec' % (message, end - start))
+
+    with timer('GPU (CuPy, Monte Carlo method)'):
+        for i, gpu_id in enumerate(args.gpus):
+            # Performs Monte-Carlo simulations in parallel
+            with cupy.cuda.Device(gpu_id):
+                monte_carlo_kernel(
+                    stock_price_gpus[i][:, None],
+                    option_strike_gpus[i][:, None],
+                    option_years_gpus[i][:, None],
+                    risk_free, volatility, args.n_samples_per_thread, i,
+                    call_prices_gpus[i])
+
+    # Transfer the result from the GPUs
+    call_prices = [c.get() for c in call_prices_gpus]
+    call_mc = numpy.concatenate(call_prices).reshape(
+        len(args.gpus), args.n_options, args.n_threads_per_option)
+    call_mc = call_mc.mean(axis=(0, 2))
+    # Compute the error between the value of the exact solution
+    # and that of the Monte-Carlo simulation
+    with cupy.cuda.Device(args.gpus[0]):
+        call_bs = black_scholes_kernel(
+            stock_price_gpus[0], option_strike_gpus[0], option_years_gpus[0],
+            risk_free, volatility)[0].get()
+    error = cupy.std(call_mc - call_bs)
+    print('Error: %f' % error)
diff --git a/examples/gemm/README.md b/examples/gemm/README.md
new file mode 100644
index 0000000..838812f
--- /dev/null
+++ b/examples/gemm/README.md
@@ -0,0 +1,37 @@
+# SGEMM example
+
+This example contains implementation of single-precision general matrix-multiplication  (SGEMM).
+The implementation is based on the one in [MAGMA](http://icl.cs.utk.edu/magma/).
+
+
+### How to demo
+The demo contains a script that calculates matrix multiplication of A (m x k) and B (k x n).
+The demo can be run by the following command.
+
+```
+python sgemm.py [--gpu GPU_ID] [--m m] [--n n] [--k k]
+```
+
+
+### What this demo contains
+
+In this example, we work on a SGEMM kernel that requires a complete interface to `cuLaunchKernel` (e.g. grid size and size of shared memory), which is not provided by `cupy.ElementwiseKernel`.
+CuPy arrays work regardless of the underlying memory layouts thanks to `ndarray` abstraction.
+As is the case for this example, `ndarray` abstraction does not need to be used if the underlying memory layouts of arrays match the ones expected by a kernel.
+The SGEMM kernel expects input and output arrays to be in Fortran contiguous memory layout, and this layout is enforced by `cupy.asfortranarray`.
+
+#### How to dynamically compile and launch a kernel function written in CUDA C
+
+For compilation, `cupy.RawKernel` class is used to compile a CUDA code written in `sgemm.cu`.
+The class takes a text of code and name of the kernel as an constructor argument.
+The instance is a callable; the CUDA code will be compiled and then invoked when it is called.
+The compiled code is cached, and it avoids the compilation process after the first time.
+Also, the CUDA code can be modified at Python level because it is simply a text.
+In this example, C macros that determine a distribution of data to threads are specified at runtime.
+Note that `"extern C"` needs to be put on top of the kernel that is called.
+
+#### How to supply grid size, block size and shared memory size on launching a kernel function
+
+`cupy.RawKernel` object allows you to call the kernel with CUDA's `cuLaunchKernel` interface.
+In other words, you have control over grid size, block size, shared memory size and stream.
+At this level of interface, it becomes straightforward to replace host `.cu` that calls CUDA kernels with Python code.
diff --git a/examples/gemm/sgemm.cu b/examples/gemm/sgemm.cu
new file mode 100644
index 0000000..a34cf84
--- /dev/null
+++ b/examples/gemm/sgemm.cu
@@ -0,0 +1,200 @@
+/*
+Original works by:
+--------------------------------------------------------
+MAGMA
+Copyright (c) 2017 The University of Tennessee. All rights reserved.
+Licensed under modified BSD license
+*/
+
+
+// These parameters will be determined by utils.read_code
+//#define DIM_X  ${DIM_X}
+//#define DIM_Y  ${DIM_Y}
+//#define BLK_M  ${BLK_M}
+//#define BLK_N  ${BLK_N}
+//#define BLK_K  ${BLK_K}
+//#define DIM_XA  ${DIM_XA}
+//#define DIM_YA  ${DIM_YA}
+//#define DIM_XB  ${DIM_XB}
+//#define DIM_YB  ${DIM_YB}
+//#define THR_N  ${THR_N}
+//#define THR_M  ${THR_M}
+
+#define fetch(arr, col, m, n, bound) arr[min(n*col + m, bound)]
+
+
+extern "C" __global__
+void sgemm(
+        int M, int N, int K,
+        const float* A,
+        const float* B,
+        float * C)
+{
+    int idx = threadIdx.x;
+    int idy = threadIdx.y;
+
+    int idt = DIM_X * idy + idx;
+
+    int idxA = idt % DIM_XA;
+    int idyA = idt / DIM_XA;
+
+    int idxB = idt % DIM_XB;
+    int idyB = idt / DIM_XB;
+
+    int blx = blockIdx.x;
+    int bly = blockIdx.y;
+
+    __shared__ float sA[BLK_K][BLK_M + 1];
+    __shared__ float sB[BLK_N][BLK_K + 1];
+
+    // registers for the innermost loop
+    float rC[THR_N][THR_M];
+    float rA[THR_M];
+    float rB[THR_N];
+
+    float ra[BLK_K / DIM_YA][BLK_M / DIM_XA];
+    float rb[BLK_N / DIM_YB][BLK_K / DIM_XB];
+
+    const float* offs_dA = A + blx * BLK_M       + idyA * M + idxA;
+    int boundA = (M * (K - 1) + M) - (blx * BLK_M + idyA * M + idxA) - 1;
+    const float* offs_dB = B + bly * BLK_N * K + idyB * K + idxB;
+    int boundB = (K * (N - 1) + K) - (bly * BLK_N * K + idyB * K + idxB) - 1;
+
+    int m, n, k, kk;
+    
+    #pragma unroll
+    for (n = 0; n < THR_N; n++) {
+        #pragma unroll
+        for (m = 0 ; m < THR_M; m++) {
+            rC[n][m] = 0;
+        }
+    }
+
+    // blockwise transpose to transpose load
+    #pragma unroll
+    for (n = 0; n < BLK_K; n += DIM_YA) {
+        #pragma unroll
+        for (m = 0; m < BLK_M; m += DIM_XA) {
+            sA[n + idyA][m + idxA] = fetch(offs_dA, M, m, n, boundA);
+        }
+    }
+    // blockwise transpose to transpose load
+    #pragma unroll
+    for (n = 0; n < BLK_N; n += DIM_YB) {
+        #pragma unroll
+        for (m = 0; m < BLK_K; m += DIM_XB) {
+            sB[n + idyB][m + idxB] = fetch(offs_dB, K, m, n, boundB);
+        }
+    }
+    __syncthreads();
+
+    for (kk = 0; kk < K - BLK_K; kk += BLK_K)
+    {
+        offs_dA += BLK_K * M;
+        boundA -= BLK_K * M;
+        offs_dB += BLK_K;
+        boundB -= BLK_K;
+        
+        #pragma unroll
+        for (n = 0; n < BLK_K / DIM_YA; n++) {
+            #pragma unroll
+            for (m = 0; m < BLK_M / DIM_XA; m++) {
+                ra[n][m] = fetch(offs_dA, M, m * DIM_XA, n * DIM_YA, boundA);
+            }
+        }
+
+        #pragma unroll
+        for (n = 0; n < BLK_N / DIM_YB; n++) {
+            #pragma unroll
+            for (m = 0; m < BLK_K / DIM_XB; m++) {
+                rb[n][m] = fetch(offs_dB, K, m * DIM_XB, n * DIM_YB, boundB);
+            }
+        }
+
+        // multiply
+        #pragma unroll
+        for (k = 0; k < BLK_K; k++)
+        {
+            #pragma unroll
+            for (m = 0; m < THR_M; m++) {
+                rA[m] = sA[k][m * DIM_X + idx];
+            }
+            
+            #pragma unroll
+            for (n = 0; n < THR_N; n++) {
+                rB[n] = sB[n * DIM_Y + idy][k];
+            }
+
+            #pragma unroll
+            for (n = 0; n < THR_N; n++) {
+                #pragma unroll
+                for (m = 0; m < THR_M; m++) {
+                    rC[n][m] += rA[m] * rB[n];
+                }
+            }
+        }
+        __syncthreads();
+
+        // store A regs->smem
+        #pragma unroll
+        for (n = 0; n < BLK_K / DIM_YA; n++)
+        {
+            #pragma unroll
+            for (m = 0; m < BLK_M / DIM_XA; m++)
+            {
+                sA[n * DIM_YA + idyA][m * DIM_XA + idxA] = ra[n][m];
+            }
+        }
+
+        #pragma unroll
+        for (n = 0; n < BLK_N / DIM_YB; n++)
+        {
+            #pragma unroll
+            for (m = 0; m < BLK_K / DIM_XB; m++)
+            {
+                sB[n * DIM_YB + idyB][m * DIM_XB + idxB] = rb[n][m];
+            }
+        }
+        __syncthreads();
+    }
+
+    // Multiply last full (BLK_K) or partial block of columns of A and
+    // rows of B.
+    // It's okay that m,n exceed matrix bounds as all work is in registers
+    // or shared memory, and out-of-bounds rC[n][m] will not be saved later.
+
+    kk = K - kk;
+    #pragma unroll
+    for (k = 0; k < kk; k++)
+    {
+        #pragma unroll
+        for (m = 0; m < THR_M; m++) {
+            rA[m] = sA[k][m * DIM_X + idx];
+        }
+
+        #pragma unroll
+        for (n = 0; n < THR_N; n++) {
+            rB[n] = sB[n * DIM_Y + idy][k];
+        }
+        
+        #pragma unroll
+        for (n = 0; n < THR_N; n++) {
+            #pragma unroll
+            for (m = 0; m < THR_M; m++) {
+                rC[n][m] += rA[m] * rB[n];
+            }
+        }
+    }
+
+    #pragma unroll
+    for (n = 0; n < THR_N; n++) {
+        int coord_dCn = bly * BLK_N + n * DIM_Y + idy;
+        #pragma unroll
+        for (m = 0; m < THR_M; m++) {
+            int coord_dCm = blx * BLK_M + m * DIM_X + idx;
+            if (coord_dCm < M && coord_dCn < N) {
+                C[coord_dCn * M + coord_dCm] = rC[n][m];
+            }
+        }
+    }
+}
diff --git a/examples/gemm/sgemm.py b/examples/gemm/sgemm.py
new file mode 100644
index 0000000..c6da765
--- /dev/null
+++ b/examples/gemm/sgemm.py
@@ -0,0 +1,89 @@
+import argparse
+import math
+import os
+
+import cupy as cp
+import numpy as np
+
+from utils import benchmark
+from utils import read_code
+
+
+sgemm_file = os.path.join(os.path.dirname(__file__), 'sgemm.cu')
+
+
+def sgemm(A, B,
+          dim_x=16, dim_y=16, blk_m=64, blk_n=64, blk_k=4,
+          dim_xa=64, dim_ya=4, dim_xb=4, dim_yb=64):
+    assert A.dtype == cp.float32
+    assert B.dtype == cp.float32
+    assert (dim_x * dim_y == dim_xa * dim_ya == dim_xb * dim_yb)
+
+    m, k = A.shape
+    k, n = B.shape
+
+    # Inputs matrices need to be in Fortran order.
+    A = cp.asfortranarray(A)
+    B = cp.asfortranarray(B)
+
+    C = cp.empty((m, n), dtype=cp.float32, order='F')
+
+    config = {'DIM_X': dim_x, 'DIM_Y': dim_y,
+              'BLK_M': blk_m, 'BLK_N': blk_n, 'BLK_K': blk_k,
+              'DIM_XA': dim_xa, 'DIM_YA': dim_ya,
+              'DIM_XB': dim_xb, 'DIM_YB': dim_yb,
+              'THR_M': blk_m // dim_x, 'THR_N': blk_n // dim_y}
+    code = read_code(sgemm_file, params=config)
+    kern = cp.RawKernel(code, 'sgemm')
+
+    grid = (int(math.ceil(m / blk_m)), int(math.ceil(n / blk_n)), 1)
+    block = (dim_x, dim_y, 1)
+    args = (m, n, k, A, B, C)
+    shared_mem = blk_k * (blk_m + 1) * 4 + blk_n * (blk_k + 1) * 4
+    kern(grid, block, args=args, shared_mem=shared_mem)
+    return C
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='SGEMM kernel call from CuPy')
+    parser.add_argument('--gpu', '-g', default=0, type=int,
+                        help='ID of GPU.')
+    parser.add_argument(
+        '--m', type=int, default=np.random.randint(1000, 1500))
+    parser.add_argument(
+        '--n', type=int, default=np.random.randint(1000, 1500))
+    parser.add_argument(
+        '--k', type=int, default=np.random.randint(500, 3000))
+    args = parser.parse_args()
+
+    print('m={} n={} k={}'.format(args.m, args.n, args.k))
+    print('start benchmarking')
+    print('')
+
+    with cp.cuda.Device(args.gpu):
+        A = cp.random.uniform(
+            low=-1., high=1., size=(args.m, args.k)).astype(cp.float32)
+        B = cp.random.uniform(
+            low=-1., high=1., size=(args.k, args.n)).astype(cp.float32)
+
+        # check correctness
+        cp.testing.assert_array_almost_equal(
+            sgemm(A, B), cp.dot(A, B), decimal=3)
+
+        # dry run
+        for _ in range(3):
+            sgemm(A, B)
+        kernel_times = benchmark(sgemm, (A, B), n_run=5)
+
+        for _ in range(3):
+            cp.dot(A, B)
+        cublas_times = benchmark(cp.dot, (A, B), n_run=5)
+
+    print('=============================Result===============================')
+    print('hand written kernel time {} ms'.format(np.mean(kernel_times)))
+    print('cuBLAS              time {} ms'.format(np.mean(cublas_times)))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/examples/gemm/utils.py b/examples/gemm/utils.py
new file mode 100644
index 0000000..3fc7201
--- /dev/null
+++ b/examples/gemm/utils.py
@@ -0,0 +1,22 @@
+import cupy as cp
+
+
+def read_code(code_filename, params):
+    with open(code_filename, 'r') as f:
+        code = f.read()
+    for k, v in params.items():
+        code = '#define ' + k + ' ' + str(v) + '\n' + code
+    return code
+
+
+def benchmark(func, args, n_run):
+    times = []
+    for _ in range(n_run):
+        start = cp.cuda.Event()
+        end = cp.cuda.Event()
+        start.record()
+        func(*args)
+        end.record()
+        end.synchronize()
+        times.append(cp.cuda.get_elapsed_time(start, end))  # milliseconds
+    return times
diff --git a/examples/gmm/README.md b/examples/gmm/README.md
new file mode 100644
index 0000000..917f721
--- /dev/null
+++ b/examples/gmm/README.md
@@ -0,0 +1,19 @@
+# GMM example
+
+This example contains implementation of Gaussian Mixture Model (GMM).
+
+
+### How to demo
+The demo contains a script that partitions data into groups using Gaussian Mixture Model.
+The demo can be run by the following command.
+
+```
+python gmm.py [--gpu-id GPU_ID] [--num NUM] [--dim DIM]
+              [--max-iter MAX_ITER] [--tol TOL] [--output-image OUTPUT]
+```
+
+If you run this script on environment without matplotlib renderers (e.g., non-GUI environment), setting the environmental variable `MPLBACKEND` to `Agg` may be required to use `matplotlib`. For example,
+
+```
+MPLBACKEND=Agg python gmm.py ...
+```
diff --git a/examples/gmm/gmm.py b/examples/gmm/gmm.py
new file mode 100644
index 0000000..c297304
--- /dev/null
+++ b/examples/gmm/gmm.py
@@ -0,0 +1,177 @@
+import argparse
+import contextlib
+import time
+
+import matplotlib.pyplot as plt
+import numpy as np
+from scipy import stats
+
+import cupy
+
+
+@contextlib.contextmanager
+def timer(message):
+    cupy.cuda.Stream.null.synchronize()
+    start = time.time()
+    yield
+    cupy.cuda.Stream.null.synchronize()
+    end = time.time()
+    print('%s:  %f sec' % (message, end - start))
+
+
+def estimate_log_prob(X, inv_cov, means):
+    xp = cupy.get_array_module(X)
+    n_features = X.shape[1]
+    log_det = xp.sum(xp.log(inv_cov), axis=1)
+    precisions = inv_cov ** 2
+    log_prob = xp.sum((means ** 2 * precisions), 1) - \
+        2 * xp.dot(X, (means * precisions).T) + xp.dot(X ** 2, precisions.T)
+    return -0.5 * (n_features * xp.log(2 * np.pi) + log_prob) + log_det
+
+
+def m_step(X, resp):
+    xp = cupy.get_array_module(X)
+    nk = xp.sum(resp, axis=0)
+    means = xp.dot(resp.T, X) / nk[:, None]
+    X2 = xp.dot(resp.T, X * X) / nk[:, None]
+    covariances = X2 - means ** 2
+    return nk / len(X), means, covariances
+
+
+def e_step(X, inv_cov, means, weights):
+    xp = cupy.get_array_module(X)
+    weighted_log_prob = estimate_log_prob(X, inv_cov, means) + \
+        xp.log(weights)
+    log_prob_norm = xp.log(xp.sum(xp.exp(weighted_log_prob), axis=1))
+    log_resp = weighted_log_prob - log_prob_norm[:, None]
+    return xp.mean(log_prob_norm), log_resp
+
+
+def train_gmm(X, max_iter, tol, means, covariances):
+    xp = cupy.get_array_module(X)
+    lower_bound = -np.inf
+    converged = False
+    weights = xp.array([0.5, 0.5], dtype=np.float32)
+    inv_cov = 1 / xp.sqrt(covariances)
+
+    for n_iter in range(max_iter):
+        prev_lower_bound = lower_bound
+        log_prob_norm, log_resp = e_step(X, inv_cov, means, weights)
+        weights, means, covariances = m_step(X, xp.exp(log_resp))
+        inv_cov = 1 / xp.sqrt(covariances)
+        lower_bound = log_prob_norm
+        change = lower_bound - prev_lower_bound
+        if abs(change) < tol:
+            converged = True
+            break
+
+    if not converged:
+        print('Failed to converge. Increase max-iter or tol.')
+
+    return inv_cov, means, weights, covariances
+
+
+def predict(X, inv_cov, means, weights):
+    xp = cupy.get_array_module(X)
+    log_prob = estimate_log_prob(X, inv_cov, means)
+    return (log_prob + xp.log(weights)).argmax(axis=1)
+
+
+def calc_acc(X_train, y_train, X_test, y_test, max_iter, tol, means,
+             covariances):
+    xp = cupy.get_array_module(X_train)
+    inv_cov, means, weights, cov = \
+        train_gmm(X_train, max_iter, tol, means, covariances)
+    y_train_pred = predict(X_train, inv_cov, means, weights)
+    train_accuracy = xp.mean(y_train_pred == y_train) * 100
+    y_test_pred = predict(X_test, inv_cov, means, weights)
+    test_accuracy = xp.mean(y_test_pred == y_test) * 100
+    print('train_accuracy : %f' % train_accuracy)
+    print('test_accuracy : %f' % test_accuracy)
+    return y_test_pred, means, cov
+
+
+def draw(X, pred, means, covariances, output):
+    xp = cupy.get_array_module(X)
+    for i in range(2):
+        labels = X[pred == i]
+        if xp is cupy:
+            labels = labels.get()
+        plt.scatter(labels[:, 0], labels[:, 1], c=np.random.rand(1, 3))
+    if xp is cupy:
+        means = means.get()
+        covariances = covariances.get()
+    plt.scatter(means[:, 0], means[:, 1], s=120, marker='s', facecolors='y',
+                edgecolors='k')
+    x = np.linspace(-5, 5, 1000)
+    y = np.linspace(-5, 5, 1000)
+    X, Y = np.meshgrid(x, y)
+    for i in range(2):
+        dist = stats.multivariate_normal(means[i], covariances[i])
+        Z = dist.pdf(np.stack([X, Y], axis=-1))
+        plt.contour(X, Y, Z)
+    plt.savefig(output)
+
+
+def run(gpuid, num, dim, max_iter, tol, output):
+    """CuPy Gaussian Mixture Model example
+
+    Compute GMM parameters, weights, means and covariance matrix, depending on
+    sampled data. There are two main components, e_step and m_step.
+    In e_step, compute burden rate, which is expressed `resp`, by latest
+    weights, means and covariance matrix.
+    In m_step, compute weights, means and covariance matrix by latest `resp`.
+
+    """
+    scale = np.ones(dim)
+    train1 = np.random.normal(1, scale, size=(num, dim)).astype(np.float32)
+    train2 = np.random.normal(-1, scale, size=(num, dim)).astype(np.float32)
+    X_train = np.r_[train1, train2]
+    test1 = np.random.normal(1, scale, size=(100, dim)).astype(np.float32)
+    test2 = np.random.normal(-1, scale, size=(100, dim)).astype(np.float32)
+    X_test = np.r_[test1, test2]
+    y_train = np.r_[np.zeros(num), np.ones(num)].astype(np.int32)
+    y_test = np.r_[np.zeros(100), np.ones(100)].astype(np.int32)
+
+    mean1 = np.random.normal(1, scale, size=dim)
+    mean2 = np.random.normal(-1, scale, size=dim)
+    means = np.stack([mean1, mean2])
+    covariances = np.random.rand(2, dim)
+    print('Running CPU...')
+    with timer(' CPU '):
+        y_test_pred, means, cov = \
+            calc_acc(X_train, y_train, X_test, y_test, max_iter, tol,
+                     means, covariances)
+
+    with cupy.cuda.Device(gpuid):
+        X_train_gpu = cupy.array(X_train)
+        y_train_gpu = cupy.array(y_train)
+        y_test_gpu = cupy.array(y_test)
+        X_test_gpu = cupy.array(X_test)
+        means = cupy.array(means)
+        covariances = cupy.array(covariances)
+        print('Running GPU...')
+        with timer(' GPU '):
+            y_test_pred, means, cov = \
+                calc_acc(X_train_gpu, y_train_gpu, X_test_gpu, y_test_gpu,
+                         max_iter, tol, means, covariances)
+        if output is not None:
+            draw(X_test_gpu, y_test_pred, means, cov, output)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu-id', '-g', default=0, type=int,
+                        help='ID of GPU.')
+    parser.add_argument('--num', '-n', default=500000, type=int,
+                        help='number of train data')
+    parser.add_argument('--dim', '-d', default=2, type=int,
+                        help='dimension of each data')
+    parser.add_argument('--max-iter', '-m', default=30, type=int,
+                        help='number of iterations')
+    parser.add_argument('--tol', '-t', default=1e-3, type=float,
+                        help='error tolerance to stop iterations')
+    parser.add_argument('--output-image', '-o', default=None, type=str,
+                        dest='output', help='output image file name')
+    args = parser.parse_args()
+    run(args.gpu_id, args.num, args.dim, args.max_iter, args.tol, args.output)
diff --git a/examples/interoperability/mpi4py_multiple_devices.py b/examples/interoperability/mpi4py_multiple_devices.py
new file mode 100644
index 0000000..cbf9c97
--- /dev/null
+++ b/examples/interoperability/mpi4py_multiple_devices.py
@@ -0,0 +1,53 @@
+# Run this script with the following command:
+#
+#   mpiexec -n 2 python multple_devices.py
+#
+# This script executes simple communication and computation with 2 MPI
+# processes, each of which uses a different GPU
+
+import cupy
+from mpi4py import MPI
+
+
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+size = comm.Get_size()
+if size != 2:
+    raise RuntimeError("run this script with 2 processes: mpiexec -n 2 ...")
+device_count = cupy.cuda.runtime.getDeviceCount()
+if device_count < 2:
+    raise RuntimeError("this script requires 2 GPUs")
+
+# Select device based on local MPI rank.
+# Caveat: for simplicity we assume local_rank == rank here, which may or may
+# not be the case depending how MPI processes are lauched and how your code
+# is written. For more robust usage, you may need to consult the user manual
+# for your MPI library. For example:
+# local_rank = int(os.getenv("OMPI_COMM_WORLD_LOCAL_RANK"))  # Open MPI
+# local_rank = int(os.getenv("MV2_COMM_WORLD_LOCAL_RANK"))   # MVAPICH2
+local_rank = rank
+cupy.cuda.Device(local_rank).use()
+
+# send-recv
+if rank == 0:
+    arr = cupy.empty(100, dtype=cupy.int64)
+    comm.Recv(arr, source=1, tag=87)
+    assert (arr == cupy.arange(100).astype(cupy.int64)).all()
+else:
+    arr = cupy.arange(100).astype(cupy.int64)
+    comm.Send(arr, dest=0, tag=87)
+
+# allreduce
+arr1 = cupy.empty(1000)
+arr2 = cupy.random.random(1000)
+arr_total = arr2.copy()
+comm.Allreduce(MPI.IN_PLACE, arr_total)  # in-place reduction
+if rank == 0:
+    comm.Recv(arr1, source=1, tag=88)
+    comm.Send(arr2, dest=1, tag=89)
+else:
+    comm.Send(arr2, dest=0, tag=88)
+    comm.Recv(arr1, source=0, tag=89)
+assert (arr1 + arr2 == arr_total).all()
+
+print("process {}: finished".format(rank))
diff --git a/examples/jit/elmentwise_op.py b/examples/jit/elmentwise_op.py
new file mode 100644
index 0000000..41d6347
--- /dev/null
+++ b/examples/jit/elmentwise_op.py
@@ -0,0 +1,21 @@
+import cupy
+from cupyx import jit
+
+
+@jit.rawkernel()
+def elementwise_copy(x, y, size):
+    tid = jit.blockIdx.x * jit.blockDim.x + jit.threadIdx.x
+    ntid = jit.gridDim.x * jit.blockDim.x
+    for i in range(tid, size, ntid):
+        y[i] = x[i]
+
+
+size = cupy.uint32(2 ** 22)
+x = cupy.random.normal(size=(size,), dtype=cupy.float32)
+y = cupy.empty((size,), dtype=cupy.float32)
+
+elementwise_copy((128,), (1024,), (x, y, size))
+
+elementwise_copy[128, 1024](x, y, size)
+
+assert (x == y).all()
diff --git a/examples/jit/reduction_atomic.py b/examples/jit/reduction_atomic.py
new file mode 100644
index 0000000..17e10b6
--- /dev/null
+++ b/examples/jit/reduction_atomic.py
@@ -0,0 +1,33 @@
+import cupy
+from cupyx import jit
+
+
+@jit.rawkernel()
+def reduction(x, y, size):
+    tid = jit.blockIdx.x * jit.blockDim.x + jit.threadIdx.x
+    ntid = jit.blockDim.x * jit.gridDim.x
+
+    value = cupy.float32(0)
+    for i in range(tid, size, ntid):
+        value += x[i]
+
+    smem = jit.shared_memory(cupy.float32, 1024)
+    smem[jit.threadIdx.x] = value
+
+    jit.syncthreads()
+
+    if jit.threadIdx.x == cupy.uint32(0):
+        value = cupy.float32(0)
+        for i in range(jit.blockDim.x):
+            value += smem[i]
+        jit.atomic_add(y, 0, value)
+
+
+size = cupy.uint32(2 ** 22)
+x = cupy.random.normal(size=(size,), dtype=cupy.float32)
+y = cupy.empty((1,), dtype=cupy.float32)
+
+reduction[64, 1024](x, y, size)
+
+print(y[0])
+print(x.sum())
diff --git a/examples/jit/reduction_simple.py b/examples/jit/reduction_simple.py
new file mode 100644
index 0000000..f747dd6
--- /dev/null
+++ b/examples/jit/reduction_simple.py
@@ -0,0 +1,33 @@
+import cupy
+from cupyx import jit
+
+
+@jit.rawkernel()
+def reduction(x, y, size):
+    tid = jit.threadIdx.x
+    ntid = jit.blockDim.x
+
+    value = cupy.float32(0)
+    for i in range(tid, size, ntid):
+        value += x[i]
+
+    smem = jit.shared_memory(cupy.float32, 1024)
+    smem[tid] = value
+
+    jit.syncthreads()
+
+    if tid == cupy.uint32(0):
+        value = cupy.float32(0)
+        for i in range(ntid):
+            value += smem[i]
+        y[0] = value
+
+
+size = cupy.uint32(2 ** 22)
+x = cupy.random.normal(size=(size,), dtype=cupy.float32)
+y = cupy.empty((1,), dtype=cupy.float32)
+
+reduction[1, 1024](x, y, size)
+
+print(y[0])
+print(x.sum())
diff --git a/examples/kmeans/README.md b/examples/kmeans/README.md
new file mode 100644
index 0000000..5ee5a71
--- /dev/null
+++ b/examples/kmeans/README.md
@@ -0,0 +1,20 @@
+# kmeans example
+
+This example contains implementation of K-means clustering.
+
+
+### How to demo
+The demo contains a script that partitions data into groups using K-means clustering.
+The demo can be run by the following command.
+
+```
+python kmeans.py [--gpu-id GPU_ID] [--n-clusters N_CLUSTERS] [--num NUM]
+                 [--max-iter MAX_ITER] [--use-custom-kernel]
+                 [--output-image OUTPUT_IMAGE]
+```
+
+If you run this script on environment without matplotlib renderers (e.g., non-GUI environment), setting the environmental variable `MPLBACKEND` to `Agg` may be required to use `matplotlib`. For example,
+
+```
+MPLBACKEND=Agg python kmeans.py ...
+```
diff --git a/examples/kmeans/kmeans.py b/examples/kmeans/kmeans.py
new file mode 100644
index 0000000..a0bdd43
--- /dev/null
+++ b/examples/kmeans/kmeans.py
@@ -0,0 +1,152 @@
+import argparse
+import contextlib
+import time
+
+import cupy
+import matplotlib.pyplot as plt
+import numpy
+
+
+@contextlib.contextmanager
+def timer(message):
+    cupy.cuda.Stream.null.synchronize()
+    start = time.time()
+    yield
+    cupy.cuda.Stream.null.synchronize()
+    end = time.time()
+    print('%s:  %f sec' % (message, end - start))
+
+
+var_kernel = cupy.ElementwiseKernel(
+    'T x0, T x1, T c0, T c1', 'T out',
+    'out = (x0 - c0) * (x0 - c0) + (x1 - c1) * (x1 - c1)',
+    'var_kernel'
+)
+sum_kernel = cupy.ReductionKernel(
+    'T x, S mask', 'T out',
+    'mask ? x : 0',
+    'a + b', 'out = a', '0',
+    'sum_kernel'
+)
+count_kernel = cupy.ReductionKernel(
+    'T mask', 'float32 out',
+    'mask ? 1.0 : 0.0',
+    'a + b', 'out = a', '0.0',
+    'count_kernel'
+)
+
+
+def fit_xp(X, n_clusters, max_iter):
+    assert X.ndim == 2
+
+    # Get NumPy or CuPy module from the supplied array.
+    xp = cupy.get_array_module(X)
+
+    n_samples = len(X)
+
+    # Make an array to store the labels indicating which cluster each sample is
+    # contained.
+    pred = xp.zeros(n_samples)
+
+    # Choose the initial centroid for each cluster.
+    initial_indexes = xp.random.choice(n_samples, n_clusters, replace=False)
+    centers = X[initial_indexes]
+
+    for _ in range(max_iter):
+        # Compute the new label for each sample.
+        distances = xp.linalg.norm(X[:, None, :] - centers[None, :, :], axis=2)
+        new_pred = xp.argmin(distances, axis=1)
+
+        # If the label is not changed for each sample, we suppose the
+        # algorithm has converged and exit from the loop.
+        if xp.all(new_pred == pred):
+            break
+        pred = new_pred
+
+        # Compute the new centroid for each cluster.
+        i = xp.arange(n_clusters)
+        mask = pred == i[:, None]
+        sums = xp.where(mask[:, :, None], X, 0).sum(axis=1)
+        counts = xp.count_nonzero(mask, axis=1).reshape((n_clusters, 1))
+        centers = sums / counts
+
+    return centers, pred
+
+
+def fit_custom(X, n_clusters, max_iter):
+    assert X.ndim == 2
+
+    n_samples = len(X)
+
+    pred = cupy.zeros(n_samples)
+
+    initial_indexes = cupy.random.choice(n_samples, n_clusters, replace=False)
+    centers = X[initial_indexes]
+
+    for _ in range(max_iter):
+        distances = var_kernel(X[:, None, 0], X[:, None, 1],
+                               centers[None, :, 1], centers[None, :, 0])
+        new_pred = cupy.argmin(distances, axis=1)
+        if cupy.all(new_pred == pred):
+            break
+        pred = new_pred
+
+        i = cupy.arange(n_clusters)
+        mask = pred == i[:, None]
+        sums = sum_kernel(X, mask[:, :, None], axis=1)
+        counts = count_kernel(mask, axis=1).reshape((n_clusters, 1))
+        centers = sums / counts
+
+    return centers, pred
+
+
+def draw(X, n_clusters, centers, pred, output):
+    # Plot the samples and centroids of the fitted clusters into an image file.
+    for i in range(n_clusters):
+        labels = X[pred == i]
+        plt.scatter(labels[:, 0], labels[:, 1], c=numpy.random.rand(3))
+    plt.scatter(
+        centers[:, 0], centers[:, 1], s=120, marker='s', facecolors='y',
+        edgecolors='k')
+    plt.savefig(output)
+
+
+def run(gpuid, n_clusters, num, max_iter, use_custom_kernel, output):
+    samples = numpy.random.randn(num, 2)
+    X_train = numpy.r_[samples + 1, samples - 1]
+
+    with timer(' CPU '):
+        centers, pred = fit_xp(X_train, n_clusters, max_iter)
+
+    with cupy.cuda.Device(gpuid):
+        X_train = cupy.asarray(X_train)
+
+        with timer(' GPU '):
+            if use_custom_kernel:
+                centers, pred = fit_custom(X_train, n_clusters, max_iter)
+            else:
+                centers, pred = fit_xp(X_train, n_clusters, max_iter)
+
+        if output is not None:
+            index = numpy.random.choice(10000000, 300, replace=False)
+            draw(X_train[index].get(), n_clusters, centers.get(),
+                 pred[index].get(), output)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu-id', '-g', default=0, type=int,
+                        help='ID of GPU.')
+    parser.add_argument('--n-clusters', '-n', default=2, type=int,
+                        help='number of clusters')
+    parser.add_argument('--num', default=5000000, type=int,
+                        help='number of samples')
+    parser.add_argument('--max-iter', '-m', default=10, type=int,
+                        help='number of iterations')
+    parser.add_argument('--use-custom-kernel', action='store_true',
+                        default=False, help='use Elementwise kernel')
+    parser.add_argument('--output-image', '-o', default=None, type=str,
+                        help='output image file name')
+    args = parser.parse_args()
+    run(args.gpu_id, args.n_clusters, args.num, args.max_iter,
+        args.use_custom_kernel, args.output_image)
diff --git a/examples/peer/peer_matrix.py b/examples/peer/peer_matrix.py
new file mode 100644
index 0000000..c6047eb
--- /dev/null
+++ b/examples/peer/peer_matrix.py
@@ -0,0 +1,17 @@
+import cupy
+
+
+def main():
+    gpus = cupy.cuda.runtime.getDeviceCount()
+    for peerDevice in range(gpus):
+        for device in range(gpus):
+            if peerDevice == device:
+                continue
+            flag = cupy.cuda.runtime.deviceCanAccessPeer(device, peerDevice)
+            print(
+                f'Can access #{peerDevice} memory from #{device}: '
+                f'{flag == 1}')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/examples/stream/cublas.py b/examples/stream/cublas.py
new file mode 100644
index 0000000..e6dcd19
--- /dev/null
+++ b/examples/stream/cublas.py
@@ -0,0 +1,19 @@
+# nvprof --print-gpu-trace python examples/stream/cublas.py
+import cupy
+
+x = cupy.array([1, 2, 3])
+y = cupy.array([[1], [2], [3]])
+expected = cupy.matmul(x, y)
+cupy.cuda.Device().synchronize()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    z = cupy.matmul(x, y)
+stream.synchronize()
+cupy.testing.assert_array_equal(z, expected)
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+z = cupy.matmul(x, y)
+stream.synchronize()
+cupy.testing.assert_array_equal(z, expected)
diff --git a/examples/stream/cudnn.py b/examples/stream/cudnn.py
new file mode 100644
index 0000000..0e4cb8f
--- /dev/null
+++ b/examples/stream/cudnn.py
@@ -0,0 +1,21 @@
+# nvprof --print-gpu-trace python examples/stream/cudnn.py
+import cupy
+import cupy.cuda.cudnn
+import cupyx.cudnn
+
+x = cupy.array([1.0, 2.0, 3.0])
+mode = cupy.cuda.cudnn.CUDNN_ACTIVATION_RELU
+expected = cupyx.cudnn.activation_forward(x, mode)
+cupy.cuda.Device().synchronize()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    y = cupyx.cudnn.activation_forward(x, mode)
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+y = cupyx.cudnn.activation_forward(x, mode)
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
diff --git a/examples/stream/cufft.py b/examples/stream/cufft.py
new file mode 100644
index 0000000..620999e
--- /dev/null
+++ b/examples/stream/cufft.py
@@ -0,0 +1,18 @@
+# nvprof --print-gpu-trace python examples/stream/cufft.py
+import cupy
+
+x = cupy.array([1, 0, 3, 0, 5, 0, 7, 0, 9], dtype=float)
+expected_f = cupy.fft.fft(x)
+cupy.cuda.Device().synchronize()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    f = cupy.fft.fft(x)
+stream.synchronize()
+cupy.testing.assert_array_equal(f, expected_f)
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+f = cupy.fft.fft(x)
+stream.synchronize()
+cupy.testing.assert_array_equal(f, expected_f)
diff --git a/examples/stream/cupy_event.py b/examples/stream/cupy_event.py
new file mode 100644
index 0000000..d0cd3ba
--- /dev/null
+++ b/examples/stream/cupy_event.py
@@ -0,0 +1,32 @@
+# nvprof --print-gpu-trace python examples/stream/cupy_event.py
+import cupy
+
+x = cupy.array([1, 2, 3])
+
+start_event = cupy.cuda.stream.Event()
+stop_event = cupy.cuda.stream.Event()
+
+
+def _norm_with_elapsed_time(x):
+    start_event.record()
+    y = cupy.linalg.norm(x)
+    stop_event.record()
+    stop_event.synchronize()
+    print(cupy.cuda.get_elapsed_time(start_event, stop_event))
+    return y
+
+
+expected = _norm_with_elapsed_time(x)
+cupy.cuda.Device().synchronize()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    y = _norm_with_elapsed_time(x)
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+y = _norm_with_elapsed_time(x)
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
diff --git a/examples/stream/cupy_kernel.py b/examples/stream/cupy_kernel.py
new file mode 100644
index 0000000..a963a34
--- /dev/null
+++ b/examples/stream/cupy_kernel.py
@@ -0,0 +1,18 @@
+# nvprof --print-gpu-trace python examples/stream/cupy_kernel.py
+import cupy
+
+x = cupy.array([1, 2, 3])
+expected = cupy.linalg.norm(x)
+cupy.cuda.Device().synchronize()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    y = cupy.linalg.norm(x)
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+y = cupy.linalg.norm(x)
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
diff --git a/examples/stream/cupy_memcpy.py b/examples/stream/cupy_memcpy.py
new file mode 100644
index 0000000..0bf12d7
--- /dev/null
+++ b/examples/stream/cupy_memcpy.py
@@ -0,0 +1,50 @@
+# nvprof --print-gpu-trace python examples/stream/cupy_memcpy.py
+import cupy
+import numpy
+
+pinned_memory_pool = cupy.cuda.PinnedMemoryPool()
+cupy.cuda.set_pinned_memory_allocator(pinned_memory_pool.malloc)
+
+
+def _pin_memory(array):
+    mem = cupy.cuda.alloc_pinned_memory(array.nbytes)
+    ret = numpy.frombuffer(mem, array.dtype, array.size).reshape(array.shape)
+    ret[...] = array
+    return ret
+
+
+SIZE = 1024 * 1024
+x_cpu_src = numpy.arange(SIZE, dtype=numpy.float32)
+x_gpu_src = cupy.arange(SIZE, dtype=numpy.float32)
+
+
+# synchronous
+stream = cupy.cuda.Stream.null
+start = stream.record()
+x_gpu_dst = cupy.empty(x_cpu_src.shape, x_cpu_src.dtype)
+x_gpu_dst.set(x_cpu_src)
+x_cpu_dst = x_gpu_src.get()
+end = stream.record()
+
+print('Synchronous Device to Host / Host to Device (ms)')
+print(cupy.cuda.get_elapsed_time(start, end))
+
+
+# asynchronous
+x_gpu_dst = cupy.empty(x_cpu_src.shape, x_cpu_src.dtype)
+x_cpu_dst = numpy.empty(x_gpu_src.shape, x_gpu_src.dtype)
+
+x_pinned_cpu_src = _pin_memory(x_cpu_src)
+x_pinned_cpu_dst = _pin_memory(x_cpu_dst)
+
+with cupy.cuda.stream.Stream() as stream_htod:
+    start = stream_htod.record()
+    x_gpu_dst.set(x_pinned_cpu_src)
+    with cupy.cuda.stream.Stream() as stream_dtoh:
+        x_gpu_src.get(out=x_pinned_cpu_dst)
+        stream_dtoh.synchronize()
+    stream_htod.synchronize()
+    end = stream_htod.record()
+
+print('Asynchronous Device to Host / Host to Device (ms)')
+print(cupy.cuda.get_elapsed_time(start, end))
diff --git a/examples/stream/curand.py b/examples/stream/curand.py
new file mode 100644
index 0000000..c76c127
--- /dev/null
+++ b/examples/stream/curand.py
@@ -0,0 +1,12 @@
+# nvprof --print-gpu-trace python examples/stream/curand.py
+import cupy
+
+rand = cupy.random.RandomState()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    y = rand.lognormal(size=(1, 3))
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+y = rand.lognormal(size=(1, 3))
diff --git a/examples/stream/cusolver.py b/examples/stream/cusolver.py
new file mode 100644
index 0000000..f23f202
--- /dev/null
+++ b/examples/stream/cusolver.py
@@ -0,0 +1,20 @@
+# nvprof --print-gpu-trace python examples/stream/cusolver.py
+import cupy
+
+x = cupy.array([[1, 0, 3], [0, 5, 0], [7, 0, 9]], float)
+expected_w, expected_v = cupy.linalg.eigh(x, UPLO='U')
+cupy.cuda.Device().synchronize()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    w, v = cupy.linalg.eigh(x, UPLO='U')
+stream.synchronize()
+cupy.testing.assert_array_equal(w, expected_w)
+cupy.testing.assert_array_equal(v, expected_v)
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+w, v = cupy.linalg.eigh(x, UPLO='U')
+stream.synchronize()
+cupy.testing.assert_array_equal(w, expected_w)
+cupy.testing.assert_array_equal(v, expected_v)
diff --git a/examples/stream/cusparse.py b/examples/stream/cusparse.py
new file mode 100644
index 0000000..c39279e
--- /dev/null
+++ b/examples/stream/cusparse.py
@@ -0,0 +1,30 @@
+# nvprof --print-gpu-trace python examples/stream/cusparse.py
+import cupy
+import cupyx
+
+
+def _make(xp, sp, dtype):
+    data = xp.array([0, 1, 3, 2], dtype)
+    indices = xp.array([0, 0, 2, 1], 'i')
+    indptr = xp.array([0, 1, 2, 3, 4], 'i')
+    # 0, 1, 0, 0
+    # 0, 0, 0, 2
+    # 0, 0, 3, 0
+    return sp.csc_matrix((data, indices, indptr), shape=(3, 4))
+
+
+x = _make(cupy, cupyx.scipy.sparse, float)
+expected = cupyx.cusparse.cscsort(x)
+cupy.cuda.Device().synchronize()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    y = cupyx.cusparse.cscsort(x)
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+y = cupyx.cusparse.cscsort(x)
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
diff --git a/examples/stream/map_reduce.py b/examples/stream/map_reduce.py
new file mode 100644
index 0000000..a110757
--- /dev/null
+++ b/examples/stream/map_reduce.py
@@ -0,0 +1,47 @@
+import cupy
+import time
+
+device = cupy.cuda.Device()
+memory_pool = cupy.cuda.MemoryPool()
+cupy.cuda.set_allocator(memory_pool.malloc)
+rand = cupy.random.RandomState(seed=1)
+
+n = 10
+zs = []
+map_streams = []
+stop_events = []
+reduce_stream = cupy.cuda.stream.Stream()
+for i in range(n):
+    map_streams.append(cupy.cuda.stream.Stream())
+
+start_time = time.time()
+
+# Map
+for stream in map_streams:
+    with stream:
+        x = rand.normal(size=(1, 1024**2))
+        y = rand.normal(size=(1024**2, 1))
+        z = cupy.matmul(x, y)
+        zs.append(z)
+    stop_event = stream.record()
+    stop_events.append(stop_event)
+
+# Block the `reduce_stream` until all events occur. This does not block host.
+# This is not required when reduction is performed in the default (Stream.null)
+# stream unless streams are created with `non_blocking=True` flag.
+for i in range(n):
+    reduce_stream.wait_event(stop_events[i])
+
+# Reduce
+with reduce_stream:
+    z = sum(zs)
+
+device.synchronize()
+elapsed_time = time.time() - start_time
+print('elapsed time', elapsed_time)
+print('total bytes', memory_pool.total_bytes())
+
+# Free all blocks in the memory pool of streams
+for stream in map_streams:
+    memory_pool.free_all_blocks(stream=stream)
+print('total bytes', memory_pool.total_bytes())
diff --git a/examples/stream/thrust.py b/examples/stream/thrust.py
new file mode 100644
index 0000000..4049692
--- /dev/null
+++ b/examples/stream/thrust.py
@@ -0,0 +1,18 @@
+# nvprof --print-gpu-trace python examples/stream/thrust.py
+import cupy
+
+x = cupy.array([1, 3, 2])
+expected = x.sort()
+cupy.cuda.Device().synchronize()
+
+stream = cupy.cuda.stream.Stream()
+with stream:
+    y = x.sort()
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
+
+stream = cupy.cuda.stream.Stream()
+stream.use()
+y = x.sort()
+stream.synchronize()
+cupy.testing.assert_array_equal(y, expected)
diff --git a/install/cupy_builder/__init__.py b/install/cupy_builder/__init__.py
new file mode 100644
index 0000000..fc5d777
--- /dev/null
+++ b/install/cupy_builder/__init__.py
@@ -0,0 +1,24 @@
+import typing as _typing
+
+# Legacy modules
+from cupy_builder import cupy_setup_build  # NOQA
+from cupy_builder import install_build  # NOQA
+from cupy_builder import install_utils  # NOQA
+
+import cupy_builder._command  # NOQA
+from cupy_builder._context import Context  # NOQA
+from cupy_builder._preflight import preflight_check  # NOQA
+from cupy_builder._features import get_features  # NOQA
+
+
+_context: _typing.Optional[Context] = None
+
+
+def initialize(context: Context) -> None:
+    global _context
+    _context = context
+
+
+def get_context() -> Context:
+    assert _context is not None, 'cupy_builder is not initialized'
+    return _context
diff --git a/install/cupy_builder/_command.py b/install/cupy_builder/_command.py
new file mode 100644
index 0000000..03d4842
--- /dev/null
+++ b/install/cupy_builder/_command.py
@@ -0,0 +1,209 @@
+import json
+import os
+import os.path
+import subprocess
+import sys
+from typing import Any, Dict, List, Tuple
+
+import setuptools
+import setuptools.command.build_ext
+
+import cupy_builder
+import cupy_builder.install_build as build
+from cupy_builder._context import Context
+from cupy_builder._compiler import DeviceCompilerUnix, DeviceCompilerWin32
+
+
+def filter_files_by_extension(
+        sources: List[str],
+        extension: str,
+) -> Tuple[List[str], List[str]]:
+    sources_selected = []
+    sources_others = []
+    for src in sources:
+        if os.path.splitext(src)[1] == extension:
+            sources_selected.append(src)
+        else:
+            sources_others.append(src)
+    return sources_selected, sources_others
+
+
+def compile_device_code(
+        ctx: Context,
+        ext: setuptools.Extension
+) -> Tuple[List[str], List[str]]:
+    """Compiles device code ("*.cu").
+
+    This method invokes the device compiler (nvcc/hipcc) to build object
+    files from device code, then returns the tuple of:
+    - list of remaining (non-device) source files ("*.cpp")
+    - list of compiled object files for device code ("*.o")
+    """
+    sources_cu, sources_cpp = filter_files_by_extension(
+        ext.sources, '.cu')
+    if len(sources_cu) == 0:
+        # No device code used in this extension.
+        return ext.sources, []
+
+    if sys.platform == 'win32':
+        compiler = DeviceCompilerWin32(ctx)
+    else:
+        compiler = DeviceCompilerUnix(ctx)
+
+    objects = []
+    for src in sources_cu:
+        print(f'{ext.name}: Device code: {src}')
+        obj_ext = 'obj' if sys.platform == 'win32' else 'o'
+        # TODO(kmaehashi): embed CUDA version in path
+        obj = f'build/temp.device_objects/{src}.{obj_ext}'
+        if os.path.exists(obj) and (_get_timestamp(src) < _get_timestamp(obj)):
+            print(f'{ext.name}: Reusing cached object file: {obj}')
+        else:
+            os.makedirs(os.path.dirname(obj), exist_ok=True)
+            print(f'{ext.name}: Building: {obj}')
+            compiler.compile(obj, src, ext)
+        objects.append(obj)
+
+    return sources_cpp, objects
+
+
+def _get_timestamp(path: str) -> float:
+    stat = os.lstat(path)
+    return max(stat.st_atime, stat.st_mtime, stat.st_ctime)
+
+
+def dumpbin_dependents(dumpbin: str, path: str) -> List[str]:
+    args = [dumpbin, '/nologo', '/dependents', path]
+    try:
+        p = subprocess.run(args, stdout=subprocess.PIPE)
+    except FileNotFoundError:
+        print(f'*** DUMPBIN not found: {args}')
+        return []
+    if p.returncode != 0:
+        print(f'*** DUMPBIN failed ({p.returncode}): {args}')
+        return []
+    sections = p.stdout.decode().split('\r\n\r\n')
+    for num, section in enumerate(sections):
+        if 'Image has the following dependencies:' in section:
+            return [line.strip() for line in sections[num+1].splitlines()]
+    print(f'*** DUMPBIN output could not be parsed: {args}')
+    return []
+
+
+class custom_build_ext(setuptools.command.build_ext.build_ext):
+
+    """Custom `build_ext` command to include CUDA C source files."""
+
+    def _cythonize(self, nthreads: int) -> None:
+        # Defer importing Cython as it may be installed via setup_requires if
+        # the user does not have Cython installed.
+        import Cython.Build
+
+        ctx = cupy_builder.get_context()
+        compiler_directives = {
+            'linetrace': ctx.linetrace,
+            'profile': ctx.profile,
+            # Embed signatures for Sphinx documentation.
+            'embedsignature': True,
+        }
+
+        # Compile-time constants to be used in Cython code
+        compile_time_env: Dict[str, Any] = {}
+
+        # Enable CUDA Python.
+        # TODO: add `cuda` to `setup_requires` only when this flag is set
+        use_cuda_python = ctx.use_cuda_python
+        compile_time_env['CUPY_USE_CUDA_PYTHON'] = use_cuda_python
+        if use_cuda_python:
+            print('Using CUDA Python')
+
+        compile_time_env['CUPY_CUFFT_STATIC'] = False
+        compile_time_env['CUPY_CYTHON_VERSION'] = Cython.__version__
+        if ctx.use_stub:  # on RTD
+            compile_time_env['CUPY_CUDA_VERSION'] = 0
+            compile_time_env['CUPY_HIP_VERSION'] = 0
+        elif ctx.use_hip:  # on ROCm/HIP
+            compile_time_env['CUPY_CUDA_VERSION'] = 0
+            compile_time_env['CUPY_HIP_VERSION'] = build.get_hip_version()
+        else:  # on CUDA
+            compile_time_env['CUPY_CUDA_VERSION'] = (
+                ctx.features['cuda'].get_version())
+            compile_time_env['CUPY_HIP_VERSION'] = 0
+
+        print('Compile-time constants: ' +
+              json.dumps(compile_time_env, indent=4))
+
+        if sys.platform == 'win32':
+            # Disable multiprocessing on Windows (spawn)
+            nthreads = 0
+
+        Cython.Build.cythonize(
+            self.extensions, verbose=True, nthreads=nthreads, language_level=3,
+            compiler_directives=compiler_directives, annotate=ctx.annotate,
+            compile_time_env=compile_time_env)
+
+    def build_extensions(self) -> None:
+        num_jobs = int(os.environ.get('CUPY_NUM_BUILD_JOBS', '4'))
+        if num_jobs > 1:
+            self.parallel = num_jobs
+            if hasattr(self.compiler, 'initialize'):
+                # Workarounds a bug in setuptools/distutils on Windows by
+                # initializing the compiler before starting a thread.
+                # By default, MSVCCompiler performs initialization in the
+                # first compilation. However, in parallel compilation mode,
+                # the init code runs in each thread and messes up the internal
+                # state as the init code is not locked and is not idempotent.
+                # https://github.com/pypa/setuptools/blob/v60.0.0/setuptools/_distutils/_msvccompiler.py#L322-L327
+                self.compiler.initialize()
+
+        # Compile "*.pyx" files into "*.cpp" files.
+        print('Cythonizing...')
+        self._cythonize(num_jobs)
+
+        # Change an extension in each source filenames from "*.pyx" to "*.cpp".
+        # c.f. `Cython.Distutils.old_build_ext`
+        for ext in self.extensions:
+            sources_pyx, sources_others = filter_files_by_extension(
+                ext.sources, '.pyx')
+            sources_cpp = ['{}.cpp'.format(os.path.splitext(src)[0])
+                           for src in sources_pyx]
+            ext.sources = sources_cpp + sources_others
+            for src in ext.sources:
+                if not os.path.isfile(src):
+                    raise RuntimeError(f'Fatal error: missing file: {src}')
+
+        print('Building extensions...')
+        super().build_extensions()
+
+        if sys.platform == 'win32':
+            print('Generating DLL dependency list...')
+
+            # Find "dumpbin.exe" next to "lib.exe".
+            dumpbin = os.path.join(
+                os.path.dirname(self.compiler.lib), 'dumpbin.exe')
+
+            # Save list of dependent DLLs for all extension modules.
+            depends = sorted(set(sum([
+                dumpbin_dependents(dumpbin, f) for f in self.get_outputs()
+            ], [])))
+
+            depends_json = os.path.join(
+                self.build_lib, 'cupy', '.data', '_depends.json')
+            os.makedirs(os.path.dirname(depends_json), exist_ok=True)
+            with open(depends_json, 'w') as f:
+                json.dump({'depends': depends}, f)
+
+    def build_extension(self, ext: setuptools.Extension) -> None:
+        ctx = cupy_builder.get_context()
+
+        # Compile "*.cu" files into object files.
+        sources_cpp, extra_objects = compile_device_code(ctx, ext)
+
+        # Remove device code from list of sources, and instead add compiled
+        # object files to link.
+        ext.sources = sources_cpp
+        ext.extra_objects += extra_objects
+
+        # Let setuptools do the rest of the build process, i.e., compile
+        # "*.cpp" files and link object files generated from "*.cu".
+        super().build_extension(ext)
diff --git a/install/cupy_builder/_compiler.py b/install/cupy_builder/_compiler.py
new file mode 100644
index 0000000..360b7c5
--- /dev/null
+++ b/install/cupy_builder/_compiler.py
@@ -0,0 +1,292 @@
+import distutils.ccompiler
+import os
+import os.path
+import platform
+import shutil
+import sys
+import subprocess
+from typing import Any, Optional, List
+
+import setuptools
+import setuptools.msvc
+from setuptools import Extension
+
+from cupy_builder._context import Context
+import cupy_builder.install_build as build
+
+
+def _nvcc_gencode_options(cuda_version: int) -> List[str]:
+    """Returns NVCC GPU code generation options."""
+
+    if sys.argv == ['setup.py', 'develop']:
+        return []
+
+    envcfg = os.getenv('CUPY_NVCC_GENERATE_CODE', None)
+    if envcfg is not None and envcfg != 'current':
+        return ['--generate-code={}'.format(arch)
+                for arch in envcfg.split(';') if len(arch) > 0]
+    if envcfg == 'current' and build.get_compute_capabilities() is not None:
+        ccs = build.get_compute_capabilities()
+        arch_list = [
+            f'compute_{cc}' if cc < 60 else (f'compute_{cc}', f'sm_{cc}')
+            for cc in ccs]
+    else:
+        # The arch_list specifies virtual architectures, such as 'compute_61',
+        # and real architectures, such as 'sm_61', for which the CUDA
+        # input files are to be compiled.
+        #
+        # The syntax of an entry of the list is
+        #
+        #     entry ::= virtual_arch | (virtual_arch, real_arch)
+        #
+        # where virtual_arch is a string which means a virtual architecture and
+        # real_arch is a string which means a real architecture.
+        #
+        # If a virtual architecture is supplied, NVCC generates a PTX code
+        # the virtual architecture. If a pair of a virtual architecture and a
+        # real architecture is supplied, NVCC generates a PTX code for the
+        # virtual architecture as well as a cubin code for the real one.
+        #
+        # For example, making NVCC generate a PTX code for 'compute_60' virtual
+        # architecture, the arch_list has an entry of 'compute_60'.
+        #
+        #     arch_list = ['compute_60']
+        #
+        # For another, making NVCC generate a PTX code for 'compute_61' virtual
+        # architecture and a cubin code for 'sm_61' real architecture, the
+        # arch_list has an entry of ('compute_61', 'sm_61').
+        #
+        #     arch_list = [('compute_61', 'sm_61')]
+        #
+        # See the documentation of each CUDA version for the list of supported
+        # architectures:
+        #
+        #   https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-steering-gpu-code-generation
+
+        aarch64 = (platform.machine() == 'aarch64')
+        if cuda_version >= 12000:
+            arch_list = [('compute_50', 'sm_50'),
+                         ('compute_52', 'sm_52'),
+                         ('compute_60', 'sm_60'),
+                         ('compute_61', 'sm_61'),
+                         ('compute_70', 'sm_70'),
+                         ('compute_75', 'sm_75'),
+                         ('compute_80', 'sm_80'),
+                         ('compute_86', 'sm_86'),
+                         ('compute_89', 'sm_89'),
+                         ('compute_90', 'sm_90'),
+                         'compute_90']
+            if aarch64:
+                # Jetson TX1/TX2 are excluded as they don't support JetPack 5
+                # (CUDA 11.4).
+                arch_list += [
+                    # ('compute_53', 'sm_53'),  # Jetson (TX1 / Nano)
+                    # ('compute_62', 'sm_62'),  # Jetson (TX2)
+                    ('compute_72', 'sm_72'),  # Jetson (Xavier)
+                    ('compute_87', 'sm_87'),  # Jetson (Orin)
+                ]
+        elif cuda_version >= 11080:
+            arch_list = [('compute_35', 'sm_35'),
+                         ('compute_37', 'sm_37'),
+                         ('compute_50', 'sm_50'),
+                         ('compute_52', 'sm_52'),
+                         ('compute_60', 'sm_60'),
+                         ('compute_61', 'sm_61'),
+                         ('compute_70', 'sm_70'),
+                         ('compute_75', 'sm_75'),
+                         ('compute_80', 'sm_80'),
+                         ('compute_86', 'sm_86'),
+                         ('compute_89', 'sm_89'),
+                         ('compute_90', 'sm_90'),
+                         'compute_90']
+            if aarch64:
+                # Jetson TX1/TX2 are excluded as they don't support JetPack 5
+                # (CUDA 11.4).
+                arch_list += [
+                    # ('compute_53', 'sm_53'),  # Jetson (TX1 / Nano)
+                    # ('compute_62', 'sm_62'),  # Jetson (TX2)
+                    ('compute_72', 'sm_72'),  # Jetson (Xavier)
+                    ('compute_87', 'sm_87'),  # Jetson (Orin)
+                ]
+        elif cuda_version >= 11040:
+            # To utilize CUDA Minor Version Compatibility (`cupy-cuda11x`),
+            # CUBIN must be generated for all supported compute capabilities
+            # instead of PTX:
+            # https://docs.nvidia.com/deploy/cuda-compatibility/index.html#application-considerations
+            arch_list = [('compute_35', 'sm_35'),
+                         ('compute_37', 'sm_37'),
+                         ('compute_50', 'sm_50'),
+                         ('compute_52', 'sm_52'),
+                         ('compute_60', 'sm_60'),
+                         ('compute_61', 'sm_61'),
+                         ('compute_70', 'sm_70'),
+                         ('compute_75', 'sm_75'),
+                         ('compute_80', 'sm_80'),
+                         ('compute_86', 'sm_86'),
+                         'compute_86']
+            if aarch64:
+                # Jetson TX1/TX2 are excluded as they don't support JetPack 5
+                # (CUDA 11.4).
+                arch_list += [
+                    # ('compute_53', 'sm_53'),  # Jetson (TX1 / Nano)
+                    # ('compute_62', 'sm_62'),  # Jetson (TX2)
+                    ('compute_72', 'sm_72'),  # Jetson (Xavier)
+                    ('compute_87', 'sm_87'),  # Jetson (Orin)
+                ]
+        elif cuda_version >= 11010:
+            arch_list = ['compute_35',
+                         'compute_50',
+                         ('compute_60', 'sm_60'),
+                         ('compute_61', 'sm_61'),
+                         ('compute_70', 'sm_70'),
+                         ('compute_75', 'sm_75'),
+                         ('compute_80', 'sm_80'),
+                         ('compute_86', 'sm_86'),
+                         'compute_86']
+        elif cuda_version >= 11000:
+            arch_list = ['compute_35',
+                         'compute_50',
+                         ('compute_60', 'sm_60'),
+                         ('compute_61', 'sm_61'),
+                         ('compute_70', 'sm_70'),
+                         ('compute_75', 'sm_75'),
+                         ('compute_80', 'sm_80'),
+                         'compute_80']
+        elif cuda_version >= 10000:
+            arch_list = ['compute_30',
+                         'compute_50',
+                         ('compute_60', 'sm_60'),
+                         ('compute_61', 'sm_61'),
+                         ('compute_70', 'sm_70'),
+                         ('compute_75', 'sm_75'),
+                         'compute_70']
+        else:
+            # This should not happen.
+            assert False
+
+    options = []
+    for arch in arch_list:
+        if type(arch) is tuple:
+            virtual_arch, real_arch = arch
+            options.append('--generate-code=arch={},code={}'.format(
+                virtual_arch, real_arch))
+        else:
+            options.append('--generate-code=arch={},code={}'.format(
+                arch, arch))
+
+    return options
+
+
+class DeviceCompilerBase:
+    """A class that invokes NVCC or HIPCC."""
+
+    def __init__(self, ctx: Context):
+        self._context = ctx
+
+    def _get_preprocess_options(self, ext: Extension) -> List[str]:
+        # https://setuptools.pypa.io/en/latest/deprecated/distutils/apiref.html#distutils.core.Extension
+        # https://github.com/pypa/setuptools/blob/v60.0.0/setuptools/_distutils/command/build_ext.py#L524-L526
+        incdirs = ext.include_dirs[:]
+        macros: List[Any] = ext.define_macros[:]
+        for undef in ext.undef_macros:
+            macros.append((undef,))
+        return distutils.ccompiler.gen_preprocess_options(macros, incdirs)
+
+    def spawn(self, commands: List[str]) -> None:
+        print('Command:', commands)
+        subprocess.check_call(commands)
+
+
+class DeviceCompilerUnix(DeviceCompilerBase):
+
+    def compile(self, obj: str, src: str, ext: Extension) -> None:
+        if self._context.use_hip:
+            self._compile_unix_hipcc(obj, src, ext)
+        else:
+            self._compile_unix_nvcc(obj, src, ext)
+
+    def _compile_unix_nvcc(self, obj: str, src: str, ext: Extension) -> None:
+        cc_args = self._get_preprocess_options(ext) + ['-c']
+
+        # For CUDA C source files, compile them with NVCC.
+        nvcc_path = build.get_nvcc_path()
+        base_opts = build.get_compiler_base_options(nvcc_path)
+        compiler_so = nvcc_path
+
+        cuda_version = self._context.features['cuda'].get_version()
+        postargs = _nvcc_gencode_options(cuda_version) + [
+            '-Xfatbin=-compress-all', '-O2', '--compiler-options="-fPIC"']
+        if cuda_version >= 11020:
+            postargs += ['--std=c++14']
+            num_threads = int(os.environ.get('CUPY_NUM_NVCC_THREADS', '2'))
+            postargs += [f'-t{num_threads}']
+        else:
+            postargs += ['--std=c++11']
+        postargs += ['-Xcompiler=-fno-gnu-unique']
+        print('NVCC options:', postargs)
+        self.spawn(compiler_so + base_opts + cc_args + [src, '-o', obj] +
+                   postargs)
+
+    def _compile_unix_hipcc(self, obj: str, src: str, ext: Extension) -> None:
+        cc_args = self._get_preprocess_options(ext) + ['-c']
+
+        # For CUDA C source files, compile them with HIPCC.
+        rocm_path = build.get_hipcc_path()
+        base_opts = build.get_compiler_base_options(rocm_path)
+        compiler_so = rocm_path
+
+        hip_version = build.get_hip_version()
+        postargs = ['-O2', '-fPIC', '--include', 'hip_runtime.h']
+        if hip_version >= 402:
+            postargs += ['--std=c++14']
+        else:
+            postargs += ['--std=c++11']
+        print('HIPCC options:', postargs)
+        self.spawn(compiler_so + base_opts + cc_args + [src, '-o', obj] +
+                   postargs)
+
+
+class DeviceCompilerWin32(DeviceCompilerBase):
+
+    def compile(self, obj: str, src: str, ext: Extension) -> None:
+        if self._context.use_hip:
+            raise RuntimeError('ROCm is not supported on Windows')
+
+        compiler_so = build.get_nvcc_path()
+        cc_args = self._get_preprocess_options(ext) + ['-c']
+        cuda_version = self._context.features['cuda'].get_version()
+        postargs = _nvcc_gencode_options(cuda_version) + [
+            '-Xfatbin=-compress-all', '-O2']
+        if cuda_version >= 11020:
+            # MSVC 14.0 (2015) is deprecated for CUDA 11.2 but we need it
+            # to build CuPy because some Python versions were built using it.
+            # REF: https://wiki.python.org/moin/WindowsCompilers
+            postargs += ['-allow-unsupported-compiler']
+        postargs += ['-Xcompiler', '/MD', '-D_USE_MATH_DEFINES']
+        # This is to compile thrust with MSVC2015
+        if cuda_version >= 11020:
+            postargs += ['--std=c++14']
+            num_threads = int(os.environ.get('CUPY_NUM_NVCC_THREADS', '2'))
+            postargs += [f'-t{num_threads}']
+        cl_exe_path = self._find_host_compiler_path()
+        if cl_exe_path is not None:
+            print(f'Using host compiler at {cl_exe_path}')
+            postargs += ['--compiler-bindir', cl_exe_path]
+        print('NVCC options:', postargs)
+        self.spawn(compiler_so + cc_args + [src, '-o', obj] + postargs)
+
+    def _find_host_compiler_path(self) -> Optional[str]:
+        # c.f. cupy.cuda.compiler._get_extra_path_for_msvc
+        cl_exe = shutil.which('cl.exe')
+        if cl_exe:
+            # The compiler is already on PATH, no extra path needed.
+            return None
+
+        vctools: List[str] = setuptools.msvc.EnvironmentInfo(
+            platform.machine()).VCTools
+        for path in vctools:
+            cl_exe = os.path.join(path, 'cl.exe')
+            if os.path.exists(cl_exe):
+                return path
+        print(f'Warning: cl.exe could not be found in {vctools}')
+        return None
diff --git a/install/cupy_builder/_context.py b/install/cupy_builder/_context.py
new file mode 100644
index 0000000..40cfc96
--- /dev/null
+++ b/install/cupy_builder/_context.py
@@ -0,0 +1,93 @@
+import argparse
+import os
+import sys
+from typing import Any, List, Mapping, Optional, Tuple
+
+import cupy_builder
+
+
+def _get_env_bool(name: str, default: bool, env: Mapping[str, str]) -> bool:
+    return env[name] != '0' if name in env else default
+
+
+def _get_env_path(name: str, env: Mapping[str, str]) -> List[str]:
+    paths = env.get(name, None)
+    if paths is None:
+        return []
+    return [x for x in paths.split(os.pathsep) if len(x) != 0]
+
+
+class Context:
+    def __init__(
+            self, source_root: str, *,
+            _env: Mapping[str, str] = os.environ,
+            _argv: List[str] = sys.argv):
+        self.source_root = source_root
+
+        self.use_cuda_python = _get_env_bool(
+            'CUPY_USE_CUDA_PYTHON', False, _env)
+        self.use_hip = _get_env_bool(
+            'CUPY_INSTALL_USE_HIP', False, _env)
+        self.include_dirs = _get_env_path('CUPY_INCLUDE_PATH', _env)
+        self.library_dirs = _get_env_path('CUPY_LIBRARY_PATH', _env)
+
+        cmdopts, _argv[:] = parse_args(_argv)
+        self.package_name: str = cmdopts.cupy_package_name
+        self.long_description_path: Optional[str] = (
+            cmdopts.cupy_long_description)
+        self.wheel_libs: List[str] = cmdopts.cupy_wheel_lib
+        self.wheel_includes: List[str] = cmdopts.cupy_wheel_include
+        self.wheel_metadata_path: Optional[str] = (
+            cmdopts.cupy_wheel_metadata)
+        self.no_rpath: bool = cmdopts.cupy_no_rpath
+        self.profile: bool = cmdopts.cupy_profile
+        self.linetrace: bool = cmdopts.cupy_coverage
+        self.annotate: bool = cmdopts.cupy_coverage
+        self.use_stub: bool = cmdopts.cupy_no_cuda
+
+        if _get_env_bool('CUPY_INSTALL_NO_RPATH', False, _env):
+            self.no_rpath = True
+
+        if os.environ.get('READTHEDOCS', None) == 'True':
+            self.use_stub = True
+
+        self.features = cupy_builder.get_features(self)
+
+
+def parse_args(argv: List[str]) -> Tuple[Any, List[str]]:
+    parser = argparse.ArgumentParser(add_help=False)
+
+    parser.add_argument(
+        '--cupy-package-name', type=str, default='cupy',
+        help='alternate package name')
+    parser.add_argument(
+        '--cupy-long-description', type=str, default=None,
+        help='path to the long description file (reST)')
+    parser.add_argument(
+        '--cupy-wheel-lib', type=str, action='append', default=[],
+        help='shared library to copy into the wheel '
+             '(can be specified for multiple times)')
+    parser.add_argument(
+        '--cupy-wheel-include', type=str, action='append', default=[],
+        help='An include file to copy into the wheel. '
+             'Delimited by a colon. '
+             'The former part is a full path of the source include file and '
+             'the latter is the relative path within cupy wheel. '
+             '(can be specified for multiple times)')
+    parser.add_argument(
+        '--cupy-wheel-metadata', type=str, default=None,
+        help='wheel metadata (cupy/.data/_wheel.json)')
+    parser.add_argument(
+        '--cupy-no-rpath', action='store_true', default=False,
+        help='disable adding default library directories to RPATH')
+    parser.add_argument(
+        '--cupy-profile', action='store_true', default=False,
+        help='enable profiling for Cython code')
+    parser.add_argument(
+        '--cupy-coverage', action='store_true', default=False,
+        help='enable coverage for Cython code')
+    parser.add_argument(
+        '--cupy-no-cuda', action='store_true', default=False,
+        help='build CuPy with stub header file')
+
+    return parser.parse_known_args(argv)
diff --git a/install/cupy_builder/_environment.py b/install/cupy_builder/_environment.py
new file mode 100644
index 0000000..914b62c
--- /dev/null
+++ b/install/cupy_builder/_environment.py
@@ -0,0 +1,37 @@
+import functools
+import glob
+import os
+import sys
+from typing import Any, Callable, Optional
+
+
+def _memoize(f: Callable) -> Callable:
+    memo = {}
+
+    @functools.wraps(f)
+    def ret(*args: Any, **kwargs: Any) -> Any:
+        key = (args, frozenset(kwargs.items()))
+        if key not in memo:
+            memo[key] = f(*args, **kwargs)
+        return memo[key]
+    return ret
+
+
+@_memoize
+def get_nvtx_path() -> Optional[str]:
+    assert sys.platform == 'win32'
+
+    prog = os.environ.get('ProgramFiles', 'C:\\Program Files')
+    pattern = os.path.join(
+        prog, 'NVIDIA Corporation', 'Nsight Systems *', 'target-windows-x64',
+        'nvtx',
+    )
+    print(f'Looking for NVTX: {pattern}')
+    candidates = sorted(glob.glob(pattern))
+    if len(candidates) != 0:
+        # Pick the latest one
+        nvtx = candidates[-1]
+        print(f'Using NVTX at: {nvtx}')
+        return nvtx
+    print('NVTX could not be found')
+    return None
diff --git a/install/cupy_builder/_features.py b/install/cupy_builder/_features.py
new file mode 100644
index 0000000..d12de78
--- /dev/null
+++ b/install/cupy_builder/_features.py
@@ -0,0 +1,490 @@
+from typing import Any, Dict, List
+
+import cupy_builder.install_build as build
+import cupy_builder.install_utils as utils
+from cupy_builder import Context
+
+
+class Feature:
+    NOT_AVAILABLE = -1
+    _UNDETERMINED = -100
+
+    def __init__(self, ctx: Context):
+        # Name of the feature.
+        self.name = ''
+
+        # When True, fail the build if the feature is unavailable.
+        self.required = False
+
+        # List of Cython modules.
+        self.modules: List[str] = []
+
+        # C/C++ headers required for the feature.
+        # This is used only for testing availability of the feature.
+        self.includes: List[str] = []
+
+        # Shared libraries to be linked.
+        self.libraries: List[str] = []
+
+        # Version of the feature.
+        self._version: Any = self._UNDETERMINED
+
+    def configure(self, compiler: Any, settings: Any) -> bool:
+        # Fill `self._version` with the version or NOT_AVAILABLE
+        self._version = None
+        return True
+
+    def get_version(self) -> Any:
+        assert self._version != self._UNDETERMINED, 'not configured yet'
+        return self._version
+
+    def __contains__(self, key: Any) -> bool:
+        # TODO(kmaehashi): Remove this transient function.
+        if not isinstance(key, str):
+            return False
+        try:
+            self.__getitem__(key)
+        except AttributeError:
+            return False
+        return True
+
+    def __getitem__(self, key: str) -> Any:
+        # TODO(kmaehashi): Remove this transient function.
+        if key == 'file':
+            return self.modules
+        elif key == 'include':
+            return self.includes
+        return getattr(self, key)
+
+
+def _from_dict(d: Dict[str, Any], ctx: Context) -> Feature:
+    # Define a feature from dict.
+    # TODO(kmaehashi): Remove this transient function.
+    f = Feature(ctx)
+    f.name = d['name']
+    f.required = d.get('required', False)
+    f.libraries = d['libraries']
+
+    # Note: the followings are renamed
+    f.modules = d['file']
+    f.includes = d['include']
+    if 'check_method' in d:
+        f.configure = d['check_method']  # type: ignore
+        f._version = None
+        if 'version_method' in d:
+            f.get_version = d['version_method']  # type: ignore
+    return f
+
+
+# The value of the key 'file' is a list that contains extension names
+# or tuples of an extension name and a list of other souces files
+# required to build the extension such as .cpp files and .cu files.
+#
+#   <extension name> | (<extension name>, a list of <other source>)
+#
+# The extension name is also interpreted as the name of the Cython
+# source file required to build the extension with appending '.pyx'
+# file extension.
+
+_cuda_files = [
+    'cupy_backends.cuda.api.driver',
+    'cupy_backends.cuda.api._driver_enum',
+    'cupy_backends.cuda.api.runtime',
+    'cupy_backends.cuda.api._runtime_enum',
+    'cupy_backends.cuda.libs.cublas',
+    'cupy_backends.cuda.libs.curand',
+    'cupy_backends.cuda.libs.cusparse',
+    'cupy_backends.cuda.libs.nvrtc',
+    'cupy_backends.cuda.libs.profiler',
+    'cupy_backends.cuda.stream',
+    'cupy_backends.cuda._softlink',
+    'cupy._core._accelerator',
+    'cupy._core._carray',
+    'cupy._core._cub_reduction',
+    'cupy._core._dtype',
+    'cupy._core._fusion_kernel',
+    'cupy._core._fusion_thread_local',
+    'cupy._core._fusion_trace',
+    'cupy._core._fusion_variable',
+    'cupy._core._kernel',
+    'cupy._core._memory_range',
+    'cupy._core._optimize_config',
+    'cupy._core._reduction',
+    'cupy._core._routines_binary',
+    'cupy._core._routines_indexing',
+    'cupy._core._routines_linalg',
+    'cupy._core._routines_logic',
+    'cupy._core._routines_manipulation',
+    'cupy._core._routines_math',
+    'cupy._core._routines_sorting',
+    'cupy._core._routines_statistics',
+    'cupy._core._scalar',
+    'cupy._core.core',
+    'cupy._core.flags',
+    'cupy._core.internal',
+    'cupy._core.fusion',
+    'cupy._core.new_fusion',
+    'cupy._core.raw',
+    'cupy.cuda.common',
+    'cupy.cuda.cufft',
+    'cupy.cuda.device',
+    'cupy.cuda.memory',
+    'cupy.cuda.memory_hook',
+    'cupy.cuda.pinned_memory',
+    'cupy.cuda.function',
+    'cupy.cuda.stream',
+    'cupy.cuda.graph',
+    'cupy.cuda.texture',
+    'cupy.fft._cache',
+    'cupy.fft._callback',
+    'cupy.lib._polynomial',
+    'cupy._util',
+]
+
+
+def get_features(ctx: Context) -> Dict[str, Feature]:
+    # We handle nvtx (and likely any other future support) here, because
+    # the HIP stubs (hip/cupy_*.h) would cause many symbols
+    # to leak into all these modules even if unused. It's easier for all of
+    # them to link to the same set of shared libraries.
+    HIP_cuda_nvtx_cusolver = {
+        # TODO(leofang): call this "rocm" or "hip" to avoid confusion?
+        'name': 'cuda',
+        'required': True,
+        'file': _cuda_files + [
+            'cupy_backends.cuda.libs.nvtx',
+            'cupy_backends.cuda.libs.cusolver',
+            'cupyx.cusolver',
+        ],
+        'include': [
+            'hip/hip_runtime_api.h',
+            'hip/hiprtc.h',
+            'hipblas.h',
+            'hiprand/hiprand.h',
+            'hipsparse.h',
+            'hipfft.h',
+            'roctx.h',
+            'rocsolver.h',
+        ],
+        'libraries': [
+            'amdhip64',  # was hiprtc and hip_hcc before ROCm 3.8.0
+            'hipblas',
+            ('hipfft', lambda hip_version: hip_version >= 401),
+            'hiprand',
+            'hipsparse',
+            'rocfft',
+            'roctx64',
+            'rocblas',
+            'rocsolver',
+            'rocsparse',
+        ],
+        'check_method': build.check_hip_version,
+        'version_method': build.get_hip_version,
+    }
+    CUDA_cusolver = {
+        'name': 'cusolver',
+        'required': True,
+        'file': [
+            'cupy_backends.cuda.libs.cusolver',
+            'cupyx.cusolver',
+        ],
+        'include': [
+            'cusolverDn.h',
+        ],
+        'libraries': [
+            'cusolver',
+        ],
+    }
+    CUDA_cudnn = {
+        'name': 'cudnn',
+        'file': [
+            'cupy_backends.cuda.libs.cudnn',
+            'cupyx.cudnn',
+        ],
+        'include': [
+            'cudnn.h',
+        ],
+        'libraries': [
+            'cudnn',
+        ],
+        'check_method': build.check_cudnn_version,
+        'version_method': build.get_cudnn_version,
+    }
+    CUDA_nccl = {
+        'name': 'nccl',
+        'file': [
+            'cupy_backends.cuda.libs.nccl',
+        ],
+        'include': [
+            'nccl.h',
+        ],
+        'libraries': [
+            'nccl',
+        ],
+        'check_method': build.check_nccl_version,
+        'version_method': build.get_nccl_version,
+    }
+    CUDA_nvtx = {
+        'name': 'nvtx',
+        'file': [
+            'cupy_backends.cuda.libs.nvtx',
+        ],
+        'include': [
+            'nvtx3/nvToolsExt.h',
+        ],
+        'libraries': [
+        ],
+        'check_method': build.check_nvtx,
+    }
+    CUDA_cutensor = {
+        'name': 'cutensor',
+        'file': [
+            'cupy_backends.cuda.libs.cutensor',
+            'cupyx.cutensor',
+        ],
+        'include': [
+            'cutensor.h',
+        ],
+        'libraries': [
+            'cutensor',
+            'cublas',
+        ],
+        'check_method': build.check_cutensor_version,
+        'version_method': build.get_cutensor_version,
+    }
+    CUDA_cub = {
+        'name': 'cub',
+        'required': True,
+        'file': [
+            ('cupy.cuda.cub', ['cupy/cuda/cupy_cub.cu']),
+        ],
+        'include': [
+            'cub/util_namespace.cuh',  # dummy
+        ],
+        'libraries': [
+            # Dependency from CUB header files
+            'cudart',
+        ],
+        'check_method': build.check_cub_version,
+        'version_method': build.get_cub_version,
+    }
+    CUDA_jitify = {
+        'name': 'jitify',
+        'required': True,
+        'file': [
+            'cupy.cuda.jitify',
+        ],
+        'include': [
+            'cuda.h',
+            'cuda_runtime.h',
+            'nvrtc.h',
+        ],
+        'libraries': [
+            # Dependency from Jitify header files
+            'cuda',
+            'cudart',
+            'nvrtc',
+        ],
+        'check_method': build.check_jitify_version,
+        'version_method': build.get_jitify_version,
+    }
+    CUDA_random = {
+        'name': 'random',
+        'required': True,
+        'file': [
+            'cupy.random._bit_generator',
+            ('cupy.random._generator_api',
+             ['cupy/random/cupy_distributions.cu']),
+        ],
+        'include': [
+        ],
+        'libraries': [
+            # Dependency from cuRAND header files
+            'cudart',
+            'curand',
+        ],
+    }
+    HIP_random = {
+        'name': 'random',
+        'required': True,
+        'file': [
+            'cupy.random._bit_generator',
+            ('cupy.random._generator_api',
+             ['cupy/random/cupy_distributions.cu']),
+        ],
+        'include': [
+            'hiprand/hiprand.h',
+        ],
+        'libraries': [
+            # Dependency from cuRAND header files
+            'amdhip64',  # was hiprtc and hip_hcc before ROCm 3.8.0
+            'hiprand',
+        ],
+        'check_method': build.check_hip_version,
+        'version_method': build.get_hip_version,
+    }
+    CUDA_cusparselt = {
+        'name': 'cusparselt',
+        'file': [
+            'cupy_backends.cuda.libs.cusparselt',
+        ],
+        'include': [
+            'cusparseLt.h',
+        ],
+        'libraries': [
+            'cusparseLt',
+        ],
+        'check_method': build.check_cusparselt_version,
+        'version_method': build.get_cusparselt_version,
+    }
+    HIP_cub = {
+        'name': 'cub',
+        'required': True,
+        'file': [
+            ('cupy.cuda.cub', ['cupy/cuda/cupy_cub.cu']),
+        ],
+        'include': [
+            'hipcub/hipcub_version.hpp',  # dummy
+        ],
+        'libraries': [
+            'amdhip64',  # was hiprtc and hip_hcc before ROCm 3.8.0
+        ],
+        'check_method': build.check_cub_version,
+        'version_method': build.get_cub_version,
+    }
+    HIP_nccl = {
+        'name': 'nccl',
+        'file': [
+            'cupy_backends.cuda.libs.nccl',
+        ],
+        'include': [
+            'rccl.h',
+        ],
+        'libraries': [
+            'rccl',
+        ],
+        'check_method': build.check_nccl_version,
+        'version_method': build.get_nccl_version,
+    }
+    HIP_thrust = {
+        'name': 'thrust',
+        'required': True,
+        'file': [
+            ('cupy.cuda.thrust', ['cupy/cuda/cupy_thrust.cu']),
+        ],
+        'include': [
+            'thrust/version.h',
+        ],
+        'libraries': [
+            'amdhip64',  # was hiprtc and hip_hcc before ROCm 3.8.0
+        ],
+    }
+    CUDA_thrust = {
+        'name': 'thrust',
+        'required': True,
+        'file': [
+            ('cupy.cuda.thrust', ['cupy/cuda/cupy_thrust.cu']),
+        ],
+        'include': [
+            'thrust/device_ptr.h',
+            'thrust/sequence.h',
+            'thrust/sort.h',
+        ],
+        'libraries': [
+            # Dependency from Thrust header files
+            'cudart',
+        ],
+        'check_method': build.check_thrust_version,
+        'version_method': build.get_thrust_version,
+    }
+    COMMON_dlpack = {
+        'name': 'dlpack',
+        'required': True,
+        'file': [
+            'cupy._core.dlpack',
+        ],
+        'include': [
+            'cupy/dlpack/dlpack.h',
+        ],
+        'libraries': [],
+    }
+
+    if ctx.use_hip:
+        features = [
+            _from_dict(HIP_cuda_nvtx_cusolver, ctx),
+            _from_dict(HIP_cub, ctx),
+            _from_dict(HIP_nccl, ctx),
+            _from_dict(HIP_random, ctx),
+            _from_dict(HIP_thrust, ctx),
+            _from_dict(COMMON_dlpack, ctx),
+        ]
+    else:
+        features = [
+            CUDA_cuda(ctx),
+            _from_dict(CUDA_cusolver, ctx),
+            _from_dict(CUDA_cudnn, ctx),
+            _from_dict(CUDA_nccl, ctx),
+            _from_dict(CUDA_nvtx, ctx),
+            _from_dict(CUDA_cutensor, ctx),
+            _from_dict(CUDA_cub, ctx),
+            _from_dict(CUDA_jitify, ctx),
+            _from_dict(CUDA_random, ctx),
+            _from_dict(CUDA_thrust, ctx),
+            _from_dict(CUDA_cusparselt, ctx),
+            _from_dict(COMMON_dlpack, ctx),
+        ]
+    return {f.name: f for f in features}
+
+
+class CUDA_cuda(Feature):
+    minimum_cuda_version = 10020
+
+    def __init__(self, ctx: Context):
+        self.name = 'cuda'
+        self.required = True
+        self.modules = _cuda_files
+        self.includes = [
+            'cublas_v2.h',
+            'cuda.h',
+            'cuda_profiler_api.h',
+            'cuda_runtime.h',
+            'cufft.h',
+            'curand.h',
+            'cusparse.h',
+            'nvrtc.h',
+        ]
+        # TODO(kmaehashi): Split profiler module so that dependency to
+        # `cudart` can be removed when using CUDA Python.
+        self.libraries = (
+            (['cudart'] if ctx.use_cuda_python else ['cuda', 'cudart']) + [
+                'cublas',
+                'cufft',
+                'curand',
+                'cusparse',
+                'nvrtc',
+            ]
+        )
+        self._version = self._UNDETERMINED
+
+    def configure(self, compiler: Any, settings: Any) -> bool:
+        try:
+            out = build.build_and_run(compiler, '''
+            #include <cuda.h>
+            #include <stdio.h>
+            int main() {
+              printf("%d", CUDA_VERSION);
+              return 0;
+            }
+            ''', include_dirs=settings['include_dirs'])  # type: ignore[no-untyped-call] # NOQA
+        except Exception as e:
+            utils.print_warning('Cannot check CUDA version', str(e))
+            return False
+
+        self._version = int(out)
+
+        if self._version < self.minimum_cuda_version:
+            utils.print_warning(
+                'CUDA version is too old: %d' % self._version,
+                'CUDA 10.2 or newer is required')
+            return False
+        return True
diff --git a/install/cupy_builder/_preflight.py b/install/cupy_builder/_preflight.py
new file mode 100644
index 0000000..7cc66e3
--- /dev/null
+++ b/install/cupy_builder/_preflight.py
@@ -0,0 +1,39 @@
+import os
+import sys
+
+from cupy_builder import Context
+
+
+def preflight_check(ctx: Context) -> bool:
+    source_root = ctx.source_root
+    is_git = os.path.isdir(os.path.join(source_root, '.git'))
+    for submodule in ('cupy/_core/include/cupy/cub',
+                      'cupy/_core/include/cupy/jitify'):
+        if 0 < len(os.listdir(os.path.join(source_root, submodule))):
+            continue
+        if is_git:
+            msg = f'''
+===========================================================================
+The directory {submodule} is a git submodule but is currently empty.
+Please use the command:
+
+    $ git submodule update --init
+
+to populate the directory before building from source.
+===========================================================================
+        '''
+        else:
+            msg = f'''
+===========================================================================
+The directory {submodule} is a git submodule but is currently empty.
+Instead of using ZIP/TAR archive downloaded from GitHub, use
+
+    $ git clone --recursive https://github.com/cupy/cupy.git
+
+to get a buildable CuPy source tree.
+===========================================================================
+        '''
+
+        print(msg, file=sys.stderr)
+        return False
+    return True
diff --git a/install/cupy_builder/cupy_setup_build.py b/install/cupy_builder/cupy_setup_build.py
new file mode 100644
index 0000000..8023e46
--- /dev/null
+++ b/install/cupy_builder/cupy_setup_build.py
@@ -0,0 +1,451 @@
+# mypy: ignore-errors
+
+import copy
+from distutils import ccompiler
+from distutils import sysconfig
+import os
+import shutil
+import sys
+
+import setuptools
+
+import cupy_builder.install_build as build
+from cupy_builder._context import Context
+from cupy_builder.install_build import PLATFORM_LINUX
+from cupy_builder.install_build import PLATFORM_WIN32
+
+
+def ensure_module_file(file):
+    if isinstance(file, tuple):
+        return file
+    else:
+        return file, []
+
+
+def module_extension_name(file):
+    return ensure_module_file(file)[0]
+
+
+def module_extension_sources(file, use_cython, no_cuda):
+    pyx, others = ensure_module_file(file)
+    base = os.path.join(*pyx.split('.'))
+    pyx = base + ('.pyx' if use_cython else '.cpp')
+
+    # If CUDA SDK is not available, remove CUDA C files from extension sources
+    # and use stubs defined in header files.
+    if no_cuda:
+        others1 = []
+        for source in others:
+            base, ext = os.path.splitext(source)
+            if ext == '.cu':
+                continue
+            others1.append(source)
+        others = others1
+
+    return [pyx] + others
+
+
+def get_required_modules(MODULES):
+    return [m['name'] for m in MODULES if m.required]
+
+
+def check_library(compiler, includes=(), libraries=(),
+                  include_dirs=(), library_dirs=(), define_macros=None,
+                  extra_compile_args=()):
+
+    source = ''.join(['#include <%s>\n' % header for header in includes])
+    source += 'int main() {return 0;}'
+    try:
+        # We need to try to build a shared library because distutils
+        # uses different option to build an executable and a shared library.
+        # Especially when a user build an executable, distutils does not use
+        # LDFLAGS environment variable.
+        build.build_shlib(compiler, source, libraries,
+                          include_dirs, library_dirs, define_macros,
+                          extra_compile_args)
+    except Exception as e:
+        print(e)
+        sys.stdout.flush()
+        return False
+    return True
+
+
+def canonicalize_hip_libraries(hip_version, libraries):
+    def ensure_tuple(x):
+        return x if isinstance(x, tuple) else (x, None)
+    new_libraries = []
+    for library in libraries:
+        lib_name, pred = ensure_tuple(library)
+        if pred is None:
+            new_libraries.append(lib_name)
+        elif pred(hip_version):
+            new_libraries.append(lib_name)
+    libraries.clear()
+    libraries.extend(new_libraries)
+
+
+def preconfigure_modules(ctx: Context, MODULES, compiler, settings):
+    """Returns a list of modules buildable in given environment and settings.
+
+    For each module in MODULES list, this function checks if the module
+    can be built in the current environment and reports it.
+    Returns a list of module names available.
+    """
+
+    nvcc_path = build.get_nvcc_path()
+    hipcc_path = build.get_hipcc_path()
+    summary = [
+        '',
+        '************************************************************',
+        '* CuPy Configuration Summary                               *',
+        '************************************************************',
+        '',
+        'Build Environment:',
+        '  Include directories: {}'.format(str(settings['include_dirs'])),
+        '  Library directories: {}'.format(str(settings['library_dirs'])),
+        '  nvcc command       : {}'.format(
+            nvcc_path if nvcc_path else '(not found)'),
+        '  hipcc command      : {}'.format(
+            hipcc_path if hipcc_path else '(not found)'),
+        '',
+        'Environment Variables:',
+    ]
+
+    for key in ['CFLAGS', 'LDFLAGS', 'LIBRARY_PATH',
+                'CUDA_PATH', 'NVCC', 'HIPCC',
+                'ROCM_HOME']:
+        summary += ['  {:<16}: {}'.format(key, os.environ.get(key, '(none)'))]
+
+    summary += [
+        '',
+        'Modules:',
+    ]
+
+    ret = []
+    for module in MODULES:
+        installed = False
+        status = 'No'
+        errmsg = []
+
+        if module['name'] == 'cutensor':
+            cutensor_path = os.environ.get('CUTENSOR_PATH', '')
+            inc_path = os.path.join(cutensor_path, 'include')
+            if os.path.exists(inc_path):
+                settings['include_dirs'].append(inc_path)
+            cuda_version = ctx.features['cuda'].get_version()
+            cuda_major = str(cuda_version // 1000)
+            cuda_major_minor = cuda_major + '.' + \
+                str((cuda_version // 10) % 100)
+            for cuda_ver in (cuda_major_minor, cuda_major):
+                lib_path = os.path.join(cutensor_path, 'lib', cuda_ver)
+                if os.path.exists(lib_path):
+                    settings['library_dirs'].append(lib_path)
+                    break
+
+        # In ROCm 4.1 and later, we need to use the independent version of
+        # hipfft as well as rocfft. We configure the lists of include
+        # directories and libraries to link here depending on ROCm version
+        # before the configuration process following.
+        if ctx.use_hip and module['name'] == 'cuda':
+            if module.configure(compiler, settings):
+                hip_version = module.get_version()
+                if hip_version >= 401:
+                    rocm_path = build.get_rocm_path()
+                    inc_path = os.path.join(rocm_path, 'hipfft', 'include')
+                    settings['include_dirs'].insert(0, inc_path)
+                    lib_path = os.path.join(rocm_path, 'hipfft', 'lib')
+                    settings['library_dirs'].insert(0, lib_path)
+                # n.b., this modifieds MODULES['cuda']['libraries'] inplace
+                canonicalize_hip_libraries(hip_version, module['libraries'])
+
+        print('')
+        print('-------- Configuring Module: {} --------'.format(
+            module['name']))
+        sys.stdout.flush()
+        if not check_library(
+                compiler,
+                includes=module['include'],
+                include_dirs=settings['include_dirs'],
+                define_macros=settings['define_macros'],
+                extra_compile_args=settings['extra_compile_args']):
+            errmsg = ['Include files not found: %s' % module['include'],
+                      'Check your CFLAGS environment variable.']
+        elif not check_library(
+                compiler,
+                libraries=module['libraries'],
+                library_dirs=settings['library_dirs'],
+                define_macros=settings['define_macros'],
+                extra_compile_args=settings['extra_compile_args']):
+            errmsg = ['Cannot link libraries: %s' % module['libraries'],
+                      'Check your LDFLAGS environment variable.']
+        elif not module.configure(compiler, settings):
+            # Fail on per-library condition check (version requirements etc.)
+            installed = True
+            errmsg = ['The library is installed but not supported.']
+        elif (module['name'] in ('thrust', 'cub', 'random')
+                and (nvcc_path is None and hipcc_path is None)):
+            installed = True
+            cmd = 'nvcc' if not ctx.use_hip else 'hipcc'
+            errmsg = ['{} command could not be found in PATH.'.format(cmd),
+                      'Check your PATH environment variable.']
+        else:
+            installed = True
+            status = 'Yes'
+            ret.append(module['name'])
+
+        if installed:
+            version = module.get_version()
+            if version is not None:
+                status += f' (version {version})'
+
+        summary += [
+            '  {:<10}: {}'.format(module['name'], status)
+        ]
+
+        # If error message exists...
+        if len(errmsg) != 0:
+            summary += ['    -> {}'.format(m) for m in errmsg]
+
+            # Skip checking other modules when CUDA is unavailable.
+            if module['name'] == 'cuda':
+                break
+
+    # Get a list of the CC of the devices connected to this node
+    if not ctx.use_hip:
+        build.check_compute_capabilities(compiler, settings)
+
+    if len(ret) != len(MODULES):
+        if 'cuda' in ret:
+            lines = [
+                'WARNING: Some modules could not be configured.',
+                'CuPy will be installed without these modules.',
+            ]
+        else:
+            lines = [
+                'ERROR: CUDA could not be found on your system.',
+                '',
+                'HINT: You are trying to build CuPy from source, '
+                'which is NOT recommended for general use.',
+                '      Please consider using binary packages instead.',
+                '',
+
+            ]
+        summary += [
+            '',
+        ] + lines + [
+            'Please refer to the Installation Guide for details:',
+            'https://docs.cupy.dev/en/stable/install.html',
+            '',
+        ]
+
+    summary += [
+        '************************************************************',
+        '',
+    ]
+
+    print('\n'.join(summary))
+    return ret, settings
+
+
+def _rpath_base():
+    if PLATFORM_LINUX:
+        return '$ORIGIN'
+    else:
+        raise Exception('not supported on this platform')
+
+
+def make_extensions(ctx: Context, compiler, use_cython):
+    """Produce a list of Extension instances which passed to cythonize()."""
+
+    MODULES = ctx.features.values()
+
+    no_cuda = ctx.use_stub
+    use_hip = not no_cuda and ctx.use_hip
+    settings = build.get_compiler_setting(ctx, use_hip)
+
+    include_dirs = settings['include_dirs']
+
+    settings['include_dirs'] = [
+        x for x in include_dirs if os.path.exists(x)]
+    settings['library_dirs'] = [
+        x for x in settings['library_dirs'] if os.path.exists(x)]
+
+    # Adjust rpath to use CUDA libraries in `cupy/.data/lib/*.so`) from CuPy.
+    use_wheel_libs_rpath = (
+        0 < len(ctx.wheel_libs) and not PLATFORM_WIN32)
+
+    # In the environment with CUDA 7.5 on Ubuntu 16.04, gcc5.3 does not
+    # automatically deal with memcpy because string.h header file has
+    # been changed. This is a workaround for that environment.
+    # See details in the below discussions:
+    # https://github.com/BVLC/caffe/issues/4046
+    # https://groups.google.com/forum/#!topic/theano-users/3ihQYiTRG4E
+    settings['define_macros'].append(('_FORCE_INLINES', '1'))
+
+    if ctx.linetrace:
+        settings['define_macros'].append(('CYTHON_TRACE', '1'))
+        settings['define_macros'].append(('CYTHON_TRACE_NOGIL', '1'))
+    if no_cuda:
+        settings['define_macros'].append(('CUPY_NO_CUDA', '1'))
+    if use_hip:
+        settings['define_macros'].append(('CUPY_USE_HIP', '1'))
+        # introduced since ROCm 4.2.0
+        settings['define_macros'].append(('__HIP_PLATFORM_AMD__', '1'))
+        # deprecated since ROCm 4.2.0
+        settings['define_macros'].append(('__HIP_PLATFORM_HCC__', '1'))
+
+    available_modules = []
+    if no_cuda:
+        available_modules = [m['name'] for m in MODULES]
+    else:
+        available_modules, settings = preconfigure_modules(
+            ctx, MODULES, compiler, settings)
+        required_modules = get_required_modules(MODULES)
+        if not (set(required_modules) <= set(available_modules)):
+            raise Exception('Your CUDA environment is invalid. '
+                            'Please check above error log.')
+
+    ret = []
+    for module in MODULES:
+        if module['name'] not in available_modules:
+            continue
+
+        s = copy.deepcopy(settings)
+        if not no_cuda:
+            s['libraries'] = module['libraries']
+
+        compile_args = s.setdefault('extra_compile_args', [])
+        link_args = s.setdefault('extra_link_args', [])
+
+        if module['name'] == 'cusolver':
+            # cupy_backends/cupy_lapack.h has C++ template code
+            compile_args.append('--std=c++11')
+            # openmp is required for cusolver
+            if use_hip:
+                pass
+            elif compiler.compiler_type == 'unix':
+                compile_args.append('-fopenmp')
+                link_args.append('-fopenmp')
+            elif compiler.compiler_type == 'msvc':
+                compile_args.append('/openmp')
+
+        if module['name'] == 'random':
+            if compiler.compiler_type == 'msvc':
+                compile_args.append('-D_USE_MATH_DEFINES')
+
+        if module['name'] == 'jitify':
+            # this fixes RTD (no_cuda) builds...
+            compile_args.append('--std=c++11')
+            # Uncomment to diagnose Jitify issues.
+            # compile_args.append('-DJITIFY_PRINT_ALL')
+
+            # if any change is made to the Jitify header, we force recompiling
+            s['depends'] = ['./cupy/_core/include/cupy/jitify/jitify.hpp']
+
+        if module['name'] == 'dlpack':
+            # if any change is made to the DLPack header, we force recompiling
+            s['depends'] = ['./cupy/_core/include/cupy/dlpack/dlpack.h']
+
+        for f in module['file']:
+            s_file = copy.deepcopy(s)
+            name = module_extension_name(f)
+
+            if name.endswith('fft._callback') and not PLATFORM_LINUX:
+                continue
+
+            rpath = []
+            if not ctx.no_rpath:
+                # Add library directories (e.g., `/usr/local/cuda/lib64`) to
+                # RPATH.
+                rpath += s_file['library_dirs']
+
+            if use_wheel_libs_rpath:
+                # Add `cupy/.data/lib` (where shared libraries included in
+                # wheels reside) to RPATH.
+                # The path is resolved relative to the module, e.g., use
+                # `$ORIGIN/../cupy/.data/lib` for `cupy/cudnn.so` and
+                # `$ORIGIN/../../../cupy/.data/lib` for
+                # `cupy_backends/cuda/libs/cudnn.so`.
+                depth = name.count('.')
+                rpath.append(
+                    '{}{}/cupy/.data/lib'.format(_rpath_base(), '/..' * depth))
+
+            if not PLATFORM_WIN32 and not PLATFORM_LINUX:
+                assert False, "macOS is no longer supported"
+            if (PLATFORM_LINUX and len(rpath) != 0):
+                ldflag = '-Wl,'
+                if PLATFORM_LINUX:
+                    ldflag += '--disable-new-dtags,'
+                ldflag += ','.join('-rpath,' + p for p in rpath)
+                args = s_file.setdefault('extra_link_args', [])
+                args.append(ldflag)
+
+            sources = module_extension_sources(f, use_cython, no_cuda)
+            extension = setuptools.Extension(name, sources, **s_file)
+            ret.append(extension)
+
+    return ret
+
+
+def prepare_wheel_libs(ctx: Context):
+    """Prepare shared libraries and include files for wheels.
+
+    Shared libraries are placed under `cupy/.data/lib` and
+    RUNPATH will be set to this directory later (Linux only).
+    Include files are placed under `cupy/.data/include`.
+
+    Returns the list of files (path relative to `cupy` module) to add to
+    the sdist/wheel distribution.
+    """
+    data_dir = os.path.abspath(os.path.join('cupy', '.data'))
+    if os.path.exists(data_dir):
+        print('Clearing directory: {}'.format(data_dir))
+        shutil.rmtree(data_dir)
+    os.mkdir(data_dir)
+
+    # Generate list files to copy
+    # tuple of (src_path, dst_path)
+    files_to_copy = []
+
+    # Library files
+    for srcpath in ctx.wheel_libs:
+        relpath = os.path.basename(srcpath)
+        dstpath = os.path.join(data_dir, 'lib', relpath)
+        files_to_copy.append((srcpath, dstpath))
+
+    # Include files
+    for include_path_spec in ctx.wheel_includes:
+        srcpath, relpath = include_path_spec.rsplit(':', 1)
+        dstpath = os.path.join(data_dir, 'include', relpath)
+        files_to_copy.append((srcpath, dstpath))
+
+    # Wheel meta data
+    if ctx.wheel_metadata_path:
+        files_to_copy.append(
+            (ctx.wheel_metadata_path, os.path.join(data_dir, '_wheel.json')))
+
+    # Copy
+    for srcpath, dstpath in files_to_copy:
+        # Note: symlink is resolved by shutil.copy2.
+        print('Copying file for wheel: {}'.format(srcpath))
+        dirpath = os.path.dirname(dstpath)
+        if not os.path.isdir(dirpath):
+            os.makedirs(dirpath)
+        shutil.copy2(srcpath, dstpath)
+
+    package_files = [x[1] for x in files_to_copy] + [
+        'cupy/.data/_depends.json',
+    ]
+    return [os.path.relpath(f, 'cupy') for f in package_files]
+
+
+def get_ext_modules(use_cython: bool, ctx: Context):
+    # We need to call get_config_vars to initialize _config_vars in distutils
+    # see #1849
+    sysconfig.get_config_vars()
+    compiler = ccompiler.new_compiler()
+    sysconfig.customize_compiler(compiler)
+
+    extensions = make_extensions(ctx, compiler, use_cython)
+
+    return extensions
diff --git a/install/cupy_builder/install_build.py b/install/cupy_builder/install_build.py
new file mode 100644
index 0000000..d0317e3
--- /dev/null
+++ b/install/cupy_builder/install_build.py
@@ -0,0 +1,748 @@
+# mypy: ignore-errors
+
+import contextlib
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import List, Set
+
+import cupy_builder
+import cupy_builder.install_utils as utils
+from cupy_builder import _environment
+from cupy_builder._context import Context
+
+
+PLATFORM_LINUX = sys.platform.startswith('linux')
+PLATFORM_WIN32 = sys.platform.startswith('win32')
+
+minimum_cudnn_version = 7600
+
+minimum_hip_version = 305  # for ROCm 3.5.0+
+
+_cuda_path = 'NOT_INITIALIZED'
+_rocm_path = 'NOT_INITIALIZED'
+_compiler_base_options = None
+
+
+# Using tempfile.TemporaryDirectory would cause an error during cleanup
+# due to a bug: https://bugs.python.org/issue26660
+@contextlib.contextmanager
+def _tempdir():
+    temp_dir = tempfile.mkdtemp()
+    try:
+        yield temp_dir
+    finally:
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+def get_rocm_path():
+    global _rocm_path
+
+    # Use a magic word to represent the cache not filled because None is a
+    # valid return value.
+    if _rocm_path != 'NOT_INITIALIZED':
+        return _rocm_path
+
+    _rocm_path = os.environ.get('ROCM_HOME', '')
+    return _rocm_path
+
+
+def get_cuda_path():
+    global _cuda_path
+
+    # Use a magic word to represent the cache not filled because None is a
+    # valid return value.
+    if _cuda_path != 'NOT_INITIALIZED':
+        return _cuda_path
+
+    nvcc_path = utils.search_on_path(('nvcc', 'nvcc.exe'))
+    cuda_path_default = None
+    if nvcc_path is not None:
+        cuda_path_default = os.path.normpath(
+            os.path.join(os.path.dirname(nvcc_path), '..'))
+
+    cuda_path = os.environ.get('CUDA_PATH', '')  # Nvidia default on Windows
+    if len(cuda_path) > 0 and cuda_path != cuda_path_default:
+        utils.print_warning(
+            'nvcc path != CUDA_PATH',
+            'nvcc path: %s' % cuda_path_default,
+            'CUDA_PATH: %s' % cuda_path)
+
+    if os.path.exists(cuda_path):
+        _cuda_path = cuda_path
+    elif cuda_path_default is not None:
+        _cuda_path = cuda_path_default
+    elif os.path.exists('/usr/local/cuda'):
+        _cuda_path = '/usr/local/cuda'
+    else:
+        _cuda_path = None
+
+    return _cuda_path
+
+
+def get_nvcc_path() -> List[str]:
+    nvcc = os.environ.get('NVCC', None)
+    if nvcc:
+        return shlex.split(nvcc)
+
+    cuda_path = get_cuda_path()
+    if cuda_path is None:
+        return None
+
+    if PLATFORM_WIN32:
+        nvcc_bin = 'bin/nvcc.exe'
+    else:
+        nvcc_bin = 'bin/nvcc'
+
+    nvcc_path = os.path.join(cuda_path, nvcc_bin)
+    if os.path.exists(nvcc_path):
+        return [nvcc_path]
+    else:
+        return None
+
+
+def get_hipcc_path() -> List[str]:
+    hipcc = os.environ.get('HIPCC', None)
+    if hipcc:
+        return shlex.split(hipcc)
+
+    rocm_path = get_rocm_path()
+    if rocm_path is None:
+        return None
+
+    if PLATFORM_WIN32:
+        hipcc_bin = 'bin/hipcc.exe'
+    else:
+        hipcc_bin = 'bin/hipcc'
+
+    hipcc_path = os.path.join(rocm_path, hipcc_bin)
+    if os.path.exists(hipcc_path):
+        return [hipcc_path]
+    else:
+        return None
+
+
+def get_compiler_setting(ctx: Context, use_hip):
+    cuda_path = None
+    rocm_path = None
+
+    if use_hip:
+        rocm_path = get_rocm_path()
+    else:
+        cuda_path = get_cuda_path()
+
+    include_dirs = ctx.include_dirs.copy()
+    library_dirs = ctx.library_dirs.copy()
+    define_macros = []
+    extra_compile_args = []
+
+    if cuda_path:
+        include_dirs.append(os.path.join(cuda_path, 'include'))
+        if PLATFORM_WIN32:
+            library_dirs.append(os.path.join(cuda_path, 'bin'))
+            library_dirs.append(os.path.join(cuda_path, 'lib', 'x64'))
+        else:
+            library_dirs.append(os.path.join(cuda_path, 'lib64'))
+            library_dirs.append(os.path.join(cuda_path, 'lib'))
+
+    if rocm_path:
+        include_dirs.append(os.path.join(rocm_path, 'include'))
+        include_dirs.append(os.path.join(rocm_path, 'include', 'hip'))
+        include_dirs.append(os.path.join(rocm_path, 'include', 'rocrand'))
+        include_dirs.append(os.path.join(rocm_path, 'include', 'hiprand'))
+        include_dirs.append(os.path.join(rocm_path, 'include', 'roctracer'))
+        library_dirs.append(os.path.join(rocm_path, 'lib'))
+
+    if use_hip:
+        extra_compile_args.append('-std=c++11')
+
+    if PLATFORM_WIN32:
+        nvtx_path = _environment.get_nvtx_path()
+        if nvtx_path is not None and os.path.exists(nvtx_path):
+            include_dirs.append(os.path.join(nvtx_path, 'include'))
+        else:
+            define_macros.append(('CUPY_NO_NVTX', '1'))
+
+    # For CUB, we need the complex and CUB headers. The search precedence for
+    # the latter is:
+    #   1. built-in CUB (for CUDA 11+ and ROCm)
+    #   2. CuPy's CUB bundle
+    # Note that starting CuPy v8 we no longer use CUB_PATH
+
+    # for <cupy/complex.cuh>
+    cupy_header = os.path.join(
+        cupy_builder.get_context().source_root, 'cupy/_core/include')
+    global _jitify_path
+    _jitify_path = os.path.join(cupy_header, 'cupy/jitify')
+    if cuda_path:
+        cuda_cub_path = os.path.join(cuda_path, 'include', 'cub')
+        if not os.path.exists(cuda_cub_path):
+            cuda_cub_path = None
+    elif rocm_path:
+        cuda_cub_path = os.path.join(rocm_path, 'include', 'hipcub')
+        if not os.path.exists(cuda_cub_path):
+            cuda_cub_path = None
+    else:
+        cuda_cub_path = None
+    global _cub_path
+    if cuda_cub_path:
+        _cub_path = cuda_cub_path
+    elif not use_hip:  # CuPy's bundle doesn't work for ROCm
+        _cub_path = os.path.join(cupy_header, 'cupy', 'cub')
+    else:
+        raise Exception('Please install hipCUB and retry')
+    include_dirs.insert(0, _cub_path)
+    include_dirs.insert(1, cupy_header)
+
+    return {
+        'include_dirs': include_dirs,
+        'library_dirs': library_dirs,
+        'define_macros': define_macros,
+        'language': 'c++',
+        'extra_compile_args': extra_compile_args,
+    }
+
+
+def _match_output_lines(output_lines, regexs):
+    # Matches regular expressions `regexs` against `output_lines` and finds the
+    # consecutive matching lines from `output_lines`.
+    # `None` is returned if no match is found.
+    if len(output_lines) < len(regexs):
+        return None
+
+    matches = [None] * len(regexs)
+    for i in range(len(output_lines) - len(regexs)):
+        for j in range(len(regexs)):
+            m = re.match(regexs[j], output_lines[i + j])
+            if not m:
+                break
+            matches[j] = m
+        else:
+            # Match found
+            return matches
+
+    # No match
+    return None
+
+
+def get_compiler_base_options(compiler_path: List[str]) -> List[str]:
+    """Returns base options for nvcc compiler.
+
+    """
+    global _compiler_base_options
+    if _compiler_base_options is None:
+        _compiler_base_options = _get_compiler_base_options(compiler_path)
+    return _compiler_base_options
+
+
+def _get_compiler_base_options(compiler_path):
+    # Try compiling a dummy code.
+    # If the compilation fails, try to parse the output of compilation
+    # and try to compose base options according to it.
+    # compiler_path is the path to nvcc (CUDA) or hipcc (ROCm/HIP)
+    with _tempdir() as temp_dir:
+        test_cu_path = os.path.join(temp_dir, 'test.cu')
+        test_out_path = os.path.join(temp_dir, 'test.out')
+        with open(test_cu_path, 'w') as f:
+            f.write('int main() { return 0; }')
+        proc = subprocess.Popen(
+            compiler_path + ['-o', test_out_path, test_cu_path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+        stdoutdata, stderrdata = proc.communicate()
+        stderrlines = stderrdata.split(b'\n')
+        if proc.returncode != 0:
+
+            # No supported host compiler
+            matches = _match_output_lines(
+                stderrlines,
+                [
+                    b'^ERROR: No supported gcc/g\\+\\+ host compiler found, '
+                    b'but .* is available.$',
+                    b'^ *Use \'nvcc (.*)\' to use that instead.$',
+                ])
+            if matches is not None:
+                base_opts = matches[1].group(1)
+                base_opts = base_opts.decode('utf8').split(' ')
+                return base_opts
+
+            # Unknown error
+            raise RuntimeError(
+                'Encountered unknown error while testing nvcc:\n' +
+                stderrdata.decode('utf8'))
+
+    return []
+
+
+_hip_version = None
+_thrust_version = None
+_cudnn_version = None
+_nccl_version = None
+_cutensor_version = None
+_cub_path = None
+_cub_version = None
+_jitify_path = None
+_jitify_version = None
+_compute_capabilities = None
+_cusparselt_version = None
+
+
+def check_hip_version(compiler, settings):
+    global _hip_version
+    try:
+        out = build_and_run(compiler, '''
+        #include <hip/hip_version.h>
+        #include <stdio.h>
+        int main() {
+          printf("%d", HIP_VERSION);
+          return 0;
+        }
+        ''', include_dirs=settings['include_dirs'])
+
+    except Exception as e:
+        utils.print_warning('Cannot check HIP version', str(e))
+        return False
+
+    _hip_version = int(out)
+
+    if _hip_version < minimum_hip_version:
+        utils.print_warning(
+            'ROCm/HIP version is too old: %d' % _hip_version,
+            'ROCm 3.5.0 or newer is required')
+        return False
+
+    return True
+
+
+def get_hip_version(formatted: bool = False) -> int:
+    """Return ROCm version cached in check_hip_version()."""
+    global _hip_version
+    if _hip_version is None:
+        msg = 'check_hip_version() must be called first.'
+        raise RuntimeError(msg)
+    if formatted:
+        return str(_hip_version)
+    return _hip_version
+
+
+def check_compute_capabilities(compiler, settings):
+    """Return compute capabilities of the installed devices."""
+    global _compute_capabilities
+    try:
+        src = '''
+        #include <cuda_runtime_api.h>
+        #include <stdio.h>
+        #define CHECK_CUDART(x) { if ((x) != cudaSuccess) return 1; }
+
+        int main() {
+          int device_count;
+          CHECK_CUDART(cudaGetDeviceCount(&device_count));
+          for (int i = 0; i < device_count; i++) {
+              cudaDeviceProp prop;
+              CHECK_CUDART(cudaGetDeviceProperties(&prop, i));
+              printf("%d%d ", prop.major, prop.minor);
+          }
+          return 0;
+        }
+        '''
+        out = build_and_run(
+            compiler, src,
+            include_dirs=settings['include_dirs'],
+            libraries=('cudart',),
+            library_dirs=settings['library_dirs'])
+        _compute_capabilities = set([int(o) for o in out.split()])
+    except Exception as e:
+        utils.print_warning('Cannot check compute capability\n{0}'.format(e))
+        return False
+
+    return True
+
+
+def get_compute_capabilities(formatted: bool = False) -> Set[int]:
+    return _compute_capabilities
+
+
+def check_thrust_version(compiler, settings):
+    global _thrust_version
+
+    try:
+        out = build_and_run(compiler, '''
+        #include <thrust/version.h>
+        #include <stdio.h>
+
+        int main() {
+          printf("%d", THRUST_VERSION);
+          return 0;
+        }
+        ''', include_dirs=settings['include_dirs'])
+    except Exception as e:
+        utils.print_warning('Cannot check Thrust version\n{0}'.format(e))
+        return False
+
+    _thrust_version = int(out)
+
+    return True
+
+
+def get_thrust_version(formatted=False):
+    """Return Thrust version cached in check_thrust_version()."""
+    global _thrust_version
+    if _thrust_version is None:
+        msg = 'check_thrust_version() must be called first.'
+        raise RuntimeError(msg)
+    if formatted:
+        return str(_thrust_version)
+    return _thrust_version
+
+
+def check_cudnn_version(compiler, settings):
+    global _cudnn_version
+    try:
+        out = build_and_run(compiler, '''
+        #include <cudnn.h>
+        #include <stdio.h>
+        int main() {
+          printf("%d", CUDNN_VERSION);
+          return 0;
+        }
+        ''', include_dirs=settings['include_dirs'])
+
+    except Exception as e:
+        utils.print_warning('Cannot check cuDNN version\n{0}'.format(e))
+        return False
+
+    _cudnn_version = int(out)
+
+    if not minimum_cudnn_version <= _cudnn_version:
+        min_major = str(minimum_cudnn_version)
+        utils.print_warning(
+            'Unsupported cuDNN version: {}'.format(str(_cudnn_version)),
+            'cuDNN >=v{} is required'.format(min_major))
+        return False
+
+    return True
+
+
+def get_cudnn_version(formatted=False):
+    """Return cuDNN version cached in check_cudnn_version()."""
+    global _cudnn_version
+    if _cudnn_version is None:
+        msg = 'check_cudnn_version() must be called first.'
+        raise RuntimeError(msg)
+    if formatted:
+        return str(_cudnn_version)
+    return _cudnn_version
+
+
+def check_nccl_version(compiler, settings):
+    global _nccl_version
+
+    # NCCL 1.x does not provide version information.
+    try:
+        out = build_and_run(compiler,
+                            '''
+                            #ifndef CUPY_USE_HIP
+                            #include <nccl.h>
+                            #else
+                            #include <rccl.h>
+                            #endif
+                            #include <stdio.h>
+                            #ifdef NCCL_MAJOR
+                            #ifndef NCCL_VERSION_CODE
+                            #  define NCCL_VERSION_CODE \
+                            (NCCL_MAJOR * 1000 + NCCL_MINOR * 100 + NCCL_PATCH)
+                            #endif
+                            #else
+                            #  define NCCL_VERSION_CODE 0
+                            #endif
+                            int main() {
+                              printf("%d", NCCL_VERSION_CODE);
+                              return 0;
+                            }
+                            ''',
+                            include_dirs=settings['include_dirs'],
+                            define_macros=settings['define_macros'])
+
+    except Exception as e:
+        utils.print_warning('Cannot include NCCL\n{0}'.format(e))
+        return False
+
+    _nccl_version = int(out)
+
+    return True
+
+
+def get_nccl_version(formatted=False):
+    """Return NCCL version cached in check_nccl_version()."""
+    global _nccl_version
+    if _nccl_version is None:
+        msg = 'check_nccl_version() must be called first.'
+        raise RuntimeError(msg)
+    if formatted:
+        if _nccl_version == 0:
+            return '1.x'
+        return str(_nccl_version)
+    return _nccl_version
+
+
+def check_nvtx(compiler, settings):
+    if PLATFORM_WIN32:
+        if _environment.get_nvtx_path() is None:
+            utils.print_warning('NVTX unavailable')
+            return False
+    return True
+
+
+def check_cub_version(compiler, settings):
+    global _cub_version
+    global _cub_path
+
+    # This is guaranteed to work for any CUB source because the search
+    # precedence follows that of include paths.
+    # - On CUDA, CUB < 1.9.9 does not provide version.cuh and would error out
+    # - On ROCm, hipCUB has the same version as rocPRIM (as of ROCm 3.5.0)
+    try:
+        out = build_and_run(compiler,
+                            '''
+                            #ifndef CUPY_USE_HIP
+                            #include <cub/version.cuh>
+                            #else
+                            #include <hipcub/hipcub_version.hpp>
+                            #endif
+                            #include <stdio.h>
+
+                            int main() {
+                              #ifndef CUPY_USE_HIP
+                              printf("%d", CUB_VERSION);
+                              #else
+                              printf("%d", HIPCUB_VERSION);
+                              #endif
+                              return 0;
+                            }''',
+                            include_dirs=settings['include_dirs'],
+                            define_macros=settings['define_macros'])
+    except Exception as e:
+        # could be in a git submodule?
+        try:
+            # CuPy's bundle
+            cupy_cub_include = _cub_path
+            a = subprocess.run(' '.join(['git', 'describe', '--tags']),
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                               shell=True, cwd=cupy_cub_include)
+            if a.returncode == 0:
+                tag = a.stdout.decode()[:-1]
+
+                # CUB's tag convention changed after 1.8.0: "v1.9.0" -> "1.9.0"
+                # In any case, we normalize it to be in line with CUB_VERSION
+                if tag.startswith('v'):
+                    tag = tag[1:]
+                tag = tag.split('.')
+                out = int(tag[0]) * 100000 + int(tag[1]) * 100
+                try:
+                    out += int(tag[2])
+                except ValueError:
+                    # there're local commits so tag is like 1.8.0-1-gdcbb288f,
+                    # we add the number of commits to the version
+                    local_patch = tag[2].split('-')
+                    out += int(local_patch[0]) + int(local_patch[1])
+            else:
+                raise RuntimeError('Cannot determine CUB version from git tag'
+                                   '\n{0}'.format(e))
+        except Exception as e:
+            utils.print_warning('Cannot determine CUB version\n{0}'.format(e))
+            # 0: CUB is not built (makes no sense), -1: built with unknown ver
+            out = -1
+
+    _cub_version = int(out)
+    settings['define_macros'].append(('CUPY_CUB_VERSION_CODE', _cub_version))
+    return True  # we always build CUB
+
+
+def get_cub_version(formatted=False):
+    """Return CUB version cached in check_cub_version()."""
+    global _cub_version
+    if _cub_version is None:
+        msg = 'check_cub_version() must be called first.'
+        raise RuntimeError(msg)
+    if formatted:
+        if _cub_version == -1:
+            return '<unknown>'
+        return str(_cub_version)
+    return _cub_version
+
+
+def check_jitify_version(compiler, settings):
+    global _jitify_version
+
+    try:
+        cupy_jitify_include = _jitify_path
+        # Unfortunately Jitify does not have any identifiable name (branch,
+        # tag, etc), so we must use the commit here
+        a = subprocess.run(' '.join(['git', 'rev-parse', '--short', 'HEAD']),
+                           stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                           shell=True, cwd=cupy_jitify_include)
+        if a.returncode == 0:
+            out = a.stdout.decode()[:-1]  # unlike elsewhere, out is a str here
+        else:
+            raise RuntimeError('Cannot determine Jitify version from git')
+    except Exception as e:
+        utils.print_warning('Cannot determine Jitify version\n{}'.format(e))
+        # 0: Jitify is not built (makes no sense), -1: built with unknown ver
+        out = -1
+
+    _jitify_version = out
+    settings['define_macros'].append(('CUPY_JITIFY_VERSION_CODE',
+                                      _jitify_version))
+    return True  # we always build Jitify
+
+
+def get_jitify_version(formatted=True):
+    """Return Jitify version cached in check_jitify_version()."""
+    global _jitify_version
+    if _jitify_version is None:
+        msg = 'check_jitify_version() must be called first.'
+        raise RuntimeError(msg)
+    if formatted:
+        if _jitify_version == -1:
+            return '<unknown>'
+        return _jitify_version
+    raise RuntimeError('Jitify version is a commit string')
+
+
+def check_cutensor_version(compiler, settings):
+    global _cutensor_version
+    try:
+        out = build_and_run(compiler, '''
+        #include <cutensor.h>
+        #include <stdio.h>
+        #ifdef CUTENSOR_MAJOR
+        #ifndef CUTENSOR_VERSION
+        #define CUTENSOR_VERSION \
+                (CUTENSOR_MAJOR * 1000 + CUTENSOR_MINOR * 100 + CUTENSOR_PATCH)
+        #endif
+        #else
+        #  define CUTENSOR_VERSION 0
+        #endif
+        int main(int argc, char* argv[]) {
+          printf("%d", CUTENSOR_VERSION);
+          return 0;
+        }
+        ''', include_dirs=settings['include_dirs'])
+
+    except Exception as e:
+        utils.print_warning('Cannot check cuTENSOR version\n{0}'.format(e))
+        return False
+
+    _cutensor_version = int(out)
+
+    if _cutensor_version < 1000:
+        utils.print_warning(
+            'Unsupported cuTENSOR version: {}'.format(_cutensor_version)
+        )
+        return False
+
+    return True
+
+
+def get_cutensor_version(formatted=False):
+    """Return cuTENSOR version cached in check_cutensor_version()."""
+    global _cutensor_version
+    if _cutensor_version is None:
+        msg = 'check_cutensor_version() must be called first.'
+        raise RuntimeError(msg)
+    return _cutensor_version
+
+
+def check_cusparselt_version(compiler, settings):
+    global _cusparselt_version
+    try:
+        out = build_and_run(compiler, '''
+        #include <cusparseLt.h>
+        #include <stdio.h>
+        #ifndef CUSPARSELT_VERSION
+        #define CUSPARSELT_VERSION 0
+        #endif
+        int main(int argc, char* argv[]) {
+          printf("%d", CUSPARSELT_VERSION);
+          return 0;
+        }
+        ''', include_dirs=settings['include_dirs'])
+
+    except Exception as e:
+        utils.print_warning('Cannot check cuSPARSELt version\n{0}'.format(e))
+        return False
+
+    _cusparselt_version = int(out)
+    return True
+
+
+def get_cusparselt_version(formatted=False):
+    """Return cuSPARSELt version cached in check_cusparselt_version()."""
+    global _cusparselt_version
+    if _cusparselt_version is None:
+        msg = 'check_cusparselt_version() must be called first.'
+        raise RuntimeError(msg)
+    return _cusparselt_version
+
+
+def build_shlib(compiler, source, libraries=(),
+                include_dirs=(), library_dirs=(), define_macros=None,
+                extra_compile_args=()):
+    with _tempdir() as temp_dir:
+        fname = os.path.join(temp_dir, 'a.cpp')
+        with open(fname, 'w') as f:
+            f.write(source)
+        objects = compiler.compile([fname], output_dir=temp_dir,
+                                   include_dirs=include_dirs,
+                                   macros=define_macros,
+                                   extra_postargs=list(extra_compile_args))
+
+        try:
+            postargs = ['/MANIFEST'] if PLATFORM_WIN32 else []
+            compiler.link_shared_lib(objects,
+                                     os.path.join(temp_dir, 'a'),
+                                     libraries=libraries,
+                                     library_dirs=library_dirs,
+                                     extra_postargs=postargs,
+                                     target_lang='c++')
+        except Exception as e:
+            msg = 'Cannot build a stub file.\nOriginal error: {0}'.format(e)
+            raise Exception(msg)
+
+
+def build_and_run(compiler, source, libraries=(),
+                  include_dirs=(), library_dirs=(), define_macros=None,
+                  extra_compile_args=()):
+    with _tempdir() as temp_dir:
+        fname = os.path.join(temp_dir, 'a.cpp')
+        with open(fname, 'w') as f:
+            f.write(source)
+
+        objects = compiler.compile([fname], output_dir=temp_dir,
+                                   include_dirs=include_dirs,
+                                   macros=define_macros,
+                                   extra_postargs=list(extra_compile_args))
+
+        try:
+            postargs = ['/MANIFEST'] if PLATFORM_WIN32 else []
+            compiler.link_executable(objects,
+                                     os.path.join(temp_dir, 'a'),
+                                     libraries=libraries,
+                                     library_dirs=library_dirs,
+                                     extra_postargs=postargs,
+                                     target_lang='c++')
+        except Exception as e:
+            msg = 'Cannot build a stub file.\nOriginal error: {0}'.format(e)
+            raise Exception(msg)
+
+        try:
+            out = subprocess.check_output(os.path.join(temp_dir, 'a'))
+            return out
+
+        except Exception as e:
+            msg = 'Cannot execute a stub file.\nOriginal error: {0}'.format(e)
+            raise Exception(msg)
diff --git a/install/cupy_builder/install_utils.py b/install/cupy_builder/install_utils.py
new file mode 100644
index 0000000..b7dd550
--- /dev/null
+++ b/install/cupy_builder/install_utils.py
@@ -0,0 +1,22 @@
+import os
+from typing import List, Optional
+
+
+def print_warning(*lines: str) -> None:
+    print('**************************************************')
+    for line in lines:
+        print('*** WARNING: %s' % line)
+    print('**************************************************')
+
+
+def get_path(key: str) -> List[str]:
+    return os.environ.get(key, '').split(os.pathsep)
+
+
+def search_on_path(filenames: List[str]) -> Optional[str]:
+    for p in get_path('PATH'):
+        for filename in filenames:
+            full = os.path.join(p, filename)
+            if os.path.exists(full):
+                return os.path.abspath(full)
+    return None
diff --git a/install/mypy.ini b/install/mypy.ini
new file mode 100644
index 0000000..bd68657
--- /dev/null
+++ b/install/mypy.ini
@@ -0,0 +1,23 @@
+[mypy]
+check_untyped_defs = True
+disallow_any_decorated = False
+disallow_any_generics = False
+disallow_any_unimported = False
+disallow_incomplete_defs = True
+disallow_subclassing_any = True
+disallow_untyped_calls = True
+disallow_untyped_decorators = False
+disallow_untyped_defs = True
+ignore_errors = False
+ignore_missing_imports = True
+no_implicit_optional = True
+python_version = 3.8
+show_error_codes = True
+strict_equality = True
+strict_optional = True
+warn_redundant_casts = True
+warn_return_any = True
+warn_unreachable = True
+warn_unused_configs = True
+warn_unused_ignores = False
+
diff --git a/install/universal_pkg/RELEASE.md b/install/universal_pkg/RELEASE.md
new file mode 100644
index 0000000..71a1366
--- /dev/null
+++ b/install/universal_pkg/RELEASE.md
@@ -0,0 +1,5 @@
+# Release Steps
+
+* Bump `VERSION` in `setup.py`
+* Build sdist: `docker run -u "$(id -u):$(id -g)" -v "${PWD}:${PWD}" -w "${PWD}" -e CUPY_UNIVERSAL_PKG_BUILD=1 python:3.10 python setup.py sdist`
+* Upload sdist: `twine upload dist/cupy-wheel-*.tar.gz`
diff --git a/install/universal_pkg/__init__.py b/install/universal_pkg/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/install/universal_pkg/setup.py b/install/universal_pkg/setup.py
new file mode 100644
index 0000000..34b60a4
--- /dev/null
+++ b/install/universal_pkg/setup.py
@@ -0,0 +1,285 @@
+import ctypes
+import pkg_resources
+import os
+import sys
+from typing import List, Optional
+
+from setuptools import setup
+
+
+VERSION = '12.3.0'
+META_VERSION = VERSION
+
+# List of packages supported by this version of CuPy.
+PACKAGES = [
+    'cupy-cuda102',
+    'cupy-cuda110',
+    'cupy-cuda111',
+    'cupy-cuda11x',
+    'cupy-cuda12x',
+    'cupy-rocm-4-3',
+    'cupy-rocm-5-0',
+]
+
+# List of packages NOT supported by this version of CuPy.
+PACKAGES_OUTDATED = [
+    'cupy-cuda80',
+    'cupy-cuda90',
+    'cupy-cuda91',
+    'cupy-cuda92',
+    'cupy-cuda100',
+    'cupy-cuda101',
+    'cupy-cuda112',
+    'cupy-cuda113',
+    'cupy-cuda114',
+    'cupy-cuda115',
+    'cupy-cuda116',
+    'cupy-cuda117',
+    'cupy-rocm-4-0',
+    'cupy-rocm-4-2',
+]
+
+# List of sdist packages.
+PACKAGES_SDIST = [
+    'cupy',
+]
+
+
+class AutoDetectionFailed(Exception):
+    def __str__(self) -> str:
+        return f'''
+============================================================
+{super().__str__()}
+============================================================
+'''
+
+
+def _log(msg: str) -> None:
+    sys.stdout.write(f'[cupy-wheel] {msg}\n')
+    sys.stdout.flush()
+
+
+def _get_version_from_library(
+        libnames: List[str],
+        funcname: str,
+        nvrtc: bool = False,
+) -> Optional[int]:
+    """Returns the library version from list of candidate libraries."""
+
+    for libname in libnames:
+        try:
+            _log(f'Looking for library: {libname}')
+            runtime_so = ctypes.CDLL(libname)
+            break
+        except Exception as e:
+            _log(f'Failed to open {libname}: {e}')
+    else:
+        _log('No more candidate library to find')
+        return None
+
+    func = getattr(runtime_so, funcname, None)
+    if func is None:
+        raise AutoDetectionFailed(
+            f'{libname}: {func} could not be found')
+    func.restype = ctypes.c_int
+
+    if nvrtc:
+        # nvrtcVersion
+        func.argtypes = [
+            ctypes.POINTER(ctypes.c_int),
+            ctypes.POINTER(ctypes.c_int),
+        ]
+        major = ctypes.c_int()
+        minor = ctypes.c_int()
+        retval = func(major, minor)
+        version = major.value * 1000 + minor.value * 10
+    else:
+        # cudaRuntimeGetVersion
+        func.argtypes = [
+            ctypes.POINTER(ctypes.c_int),
+        ]
+        version_ref = ctypes.c_int()
+        retval = func(version_ref)
+        version = version_ref.value
+
+    if retval != 0:  # NVRTC_SUCCESS or cudaSuccess
+        raise AutoDetectionFailed(
+            f'{libname}: {func} returned error: {retval}')
+    _log(f'Detected version: {version}')
+    return version
+
+
+def _setup_win32_dll_directory() -> None:
+    if not hasattr(os, 'add_dll_directory'):
+        # Python 3.7 or earlier.
+        return
+    cuda_path = os.environ.get('CUDA_PATH', None)
+    if cuda_path is None:
+        _log('CUDA_PATH is not set.'
+             'cupy-wheel may not be able to discover NVRTC to probe version')
+        return
+    os.add_dll_directory(os.path.join(cuda_path, 'bin'))  # type: ignore[attr-defined] # NOQA
+
+
+def _get_cuda_version() -> Optional[int]:
+    """Returns the detected CUDA version or None."""
+
+    if sys.platform == 'linux':
+        libnames = [
+            'libnvrtc.so.12',
+            'libnvrtc.so.11.2',
+            'libnvrtc.so.11.1',
+            'libnvrtc.so.11.0',
+            'libnvrtc.so.10.2',
+        ]
+    elif sys.platform == 'win32':
+        libnames = [
+            'nvrtc64_120_0.dll',
+            'nvrtc64_112_0.dll',
+            'nvrtc64_111_0.dll',
+            'nvrtc64_110_0.dll',
+            'nvrtc64_102_0.dll',
+        ]
+        _setup_win32_dll_directory()
+    else:
+        _log(f'CUDA detection unsupported on platform: {sys.platform}')
+        return None
+    _log(f'Trying to detect CUDA version from libraries: {libnames}')
+    version = _get_version_from_library(libnames, 'nvrtcVersion', True)
+    return version
+
+
+def _get_rocm_version() -> Optional[int]:
+    """Returns the detected ROCm version or None."""
+    if sys.platform == 'linux':
+        libnames = ['libamdhip64.so']
+    else:
+        _log(f'ROCm detection unsupported on platform: {sys.platform}')
+        return None
+    version = _get_version_from_library(libnames, 'hipRuntimeGetVersion')
+    return version
+
+
+def _find_installed_packages() -> List[str]:
+    """Returns the list of CuPy packages installed in the environment."""
+
+    found = []
+    for pkg in (PACKAGES + PACKAGES_OUTDATED + PACKAGES_SDIST):
+        try:
+            pkg_resources.get_distribution(pkg)
+            found.append(pkg)
+        except pkg_resources.DistributionNotFound:
+            pass
+    return found
+
+
+def _cuda_version_to_package(ver: int) -> str:
+    if ver < 10020:
+        raise AutoDetectionFailed(
+            f'Your CUDA version ({ver}) is too old.')
+    elif ver < 11000:
+        # CUDA 10.2
+        suffix = '102'
+    elif ver < 11010:
+        # CUDA 11.0
+        suffix = '110'
+    elif ver < 11020:
+        # CUDA 11.1
+        suffix = '111'
+    elif ver < 12000:
+        # CUDA 11.2 ~ 11.x
+        suffix = '11x'
+    elif ver < 13000:
+        # CUDA 12.x
+        suffix = '12x'
+    else:
+        raise AutoDetectionFailed(
+            f'Your CUDA version ({ver}) is too new.')
+    return f'cupy-cuda{suffix}'
+
+
+def _rocm_version_to_package(ver: int) -> str:
+    """
+    ROCm 4.0.x = 3212
+    ROCm 4.1.x = 3241
+    ROCm 4.2.0 = 3275
+    ROCm 4.3.0 = 40321300
+    ROCm 4.3.1 = 40321331
+    ROCm 4.5.0 = 40421401
+    ROCm 4.5.1 = 40421432
+    ROCm 5.0.0 = 50013601
+    ROCm 5.1.0 = 50120531
+    """
+    if 4_03_00000 <= ver < 4_04_00000:
+        # ROCm 4.3
+        suffix = '4-3'
+    elif 5_00_00000 <= ver < 5_01_00000:
+        # ROCm 5.0
+        suffix = '5-0'
+    else:
+        raise AutoDetectionFailed(
+            f'Your ROCm version ({ver}) is unsupported.')
+    return f'cupy-rocm-{suffix}'
+
+
+def infer_best_package() -> str:
+    """Returns the appropriate CuPy wheel package name for the environment."""
+
+    # Find the existing CuPy wheel installation.
+    installed = _find_installed_packages()
+    if 1 < len(installed):
+        raise AutoDetectionFailed(
+            'You have multiple CuPy packages installed: \n'
+            f'  {installed}\n'
+            'Please uninstall all of them first, then try reinstalling.')
+
+    elif 1 == len(installed):
+        if installed[0] in PACKAGES_SDIST:
+            raise AutoDetectionFailed(
+                'You already have CuPy installed via source'
+                ' (pip install cupy).')
+        if installed[0] in PACKAGES_OUTDATED:
+            raise AutoDetectionFailed(
+                f'You have CuPy package "{installed[0]}" installed, but the'
+                f' package is not available for version {VERSION}.\n'
+                'Hint: cupy-cuda{112~117} has been merged to cupy-cuda11x in '
+                'CuPy v11. Uninstall the package and try again.')
+        return installed[0]
+
+    # Try CUDA.
+    version = _get_cuda_version()
+    if version is not None:
+        return _cuda_version_to_package(version)
+
+    # Try ROCm.
+    version = _get_rocm_version()
+    if version is not None:
+        return _rocm_version_to_package(version)
+
+    raise AutoDetectionFailed(
+        'Unable to detect NVIDIA CUDA or AMD ROCm installation.')
+
+
+#
+# Entrypoint
+#
+
+def main() -> None:
+    if os.environ.get('CUPY_UNIVERSAL_PKG_BUILD', None) is None:
+        package = infer_best_package()
+        requires = f'{package}=={VERSION}'
+        _log(f'Installing package: {requires}')
+        install_requires = [requires]
+    else:
+        _log('Building cupy-wheel package for release.')
+        install_requires = []
+
+    setup(
+        name='cupy-wheel',
+        version=META_VERSION,
+        install_requires=install_requires,
+    )
+
+
+if __name__ == '__main__':
+    main()
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..4e5a3ed
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,61 @@
+[metadata]
+license_files = docs/source/license.rst
+
+[flake8]  # for flake8 & autopep8
+# Exclude patterns need to match with absolute/relative file/directory path.
+exclude = *.git*, *.eggs*, *cupy/array_api*, *tests/cupy_tests/array_api_tests*, *docs/source*
+per-file-ignores =
+    # ignore long lines containing arrays of numerical constants
+    cupyx/scipy/special/_gammainc.py:E501
+
+[mypy]
+# Keep in sync with the list in .pre-commit-config.yaml
+files = cupy, cupyx, cupy_backends, .github, .pfnci
+ignore_missing_imports = True
+disable_error_code = attr-defined
+
+[tool:pytest]
+addopts = --strict-markers
+markers =
+    slow
+    multi_gpu
+filterwarnings =
+    error::FutureWarning
+    # ignore FutureWarning from cupy._util.experimental
+    ignore:.* is experimental\.:FutureWarning:cupy
+    ignore::UserWarning
+    error::DeprecationWarning
+    error::PendingDeprecationWarning
+    error::numpy.VisibleDeprecationWarning
+    error::numpy.ComplexWarning
+    # distutils (Python 3.10)
+    ignore:The distutils(.+) is deprecated:DeprecationWarning
+    ignore:dep_util is Deprecated:DeprecationWarning
+    # pkg_resources
+    ignore::DeprecationWarning:pkg_resources
+    ignore:pkg_resources is deprecated
+    # importing old SciPy is warned because it tries to
+    # import nose via numpy.testing
+    ignore::DeprecationWarning:scipy\._lib\._numpy_compat
+    # importing stats from old SciPy is warned because it tries to
+    # import numpy.testing.decorators
+    ignore::DeprecationWarning:scipy\.stats\.morestats
+    # Using `scipy.sparse` against NumPy 1.15+ raises warning
+    # as it uses `np.matrix` which is pending deprecation.
+    # Also exclude `numpy.matrixlib.defmatrix` as SciPy and our
+    # test code uses `np.asmatrix`, which internally calls
+    # `np.matrix`.
+    ignore::PendingDeprecationWarning:scipy\.sparse\.\w+
+    ignore::PendingDeprecationWarning:numpy\.matrixlib\.defmatrix
+    # pyreadline (dependency from optuna -> cliff -> cmd2) uses deprecated ABCs
+    ignore:Using or importing the ABCs from:DeprecationWarning:pyreadline
+    # Ignore warnings from Optuna 3.0 internal code
+    # TODO(kmaehashi): Remove after the issue is fixed in Optuna.
+    ignore:(.+?) has been deprecated in v3.0.0:FutureWarning:optuna
+    # Ignore warnings from SQLAlchemy on which Optuna depends
+    ignore:Deprecated API features detected! These feature\(s\) are not compatible with SQLAlchemy 2\.0\.:DeprecationWarning:optuna
+    ignore:(.+?)SQLAlchemy 2\.0(.+?):DeprecationWarning
+    # setuptools 65+
+    # TODO(kmaehashi): Remove distutils from cupy_builder to remove this
+    ignore:Absolute path '(.+?)' is being replaced with a relative path '(.+?)' for outputs:DeprecationWarning:distutils
+xfail_strict=true
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..4fe77b3
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+
+import glob
+import os
+from setuptools import setup, find_packages
+import sys
+
+source_root = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(source_root, 'install'))
+
+import cupy_builder  # NOQA
+from cupy_builder import cupy_setup_build  # NOQA
+
+ctx = cupy_builder.Context(source_root)
+cupy_builder.initialize(ctx)
+if not cupy_builder.preflight_check(ctx):
+    sys.exit(1)
+
+
+# TODO(kmaehashi): migrate to pyproject.toml (see #4727, #4619)
+setup_requires = [
+    'Cython>=0.29.22,<3',
+    'fastrlock>=0.5',
+]
+install_requires = [
+    'numpy>=1.20,<1.29',  # see #4773
+    'fastrlock>=0.5',
+]
+extras_require = {
+    'all': [
+        'scipy>=1.6,<1.14',  # see #4773
+        'Cython>=0.29.22,<3',
+        'optuna>=2.0',
+    ],
+    # TODO(kmaehashi): remove stylecheck and update the contribution guide
+    'stylecheck': [
+        'autopep8==1.5.5',
+        'flake8==3.8.4',
+        'pbr==5.5.1',
+        'pycodestyle==2.6.0',
+
+        'mypy==1.4.1',
+        'types-setuptools==57.4.14',
+    ],
+    'test': [
+        # 4.2 <= pytest < 6.2 is slow collecting tests and times out on CI.
+        # pytest < 7.2 has some different behavior that makes our CI fail
+        'pytest>=7.2',
+        'hypothesis>=6.37.2,<6.55.0',
+    ],
+}
+tests_require = extras_require['test']
+
+
+# List of files that needs to be in the distribution (sdist/wheel).
+# Notes:
+# - Files only needed in sdist should be added to `MANIFEST.in`.
+# - The following glob (`**`) ignores items starting with `.`.
+cupy_package_data = [
+    'cupy/cuda/cupy_thrust.cu',
+    'cupy/cuda/cupy_cub.cu',
+    'cupy/cuda/cupy_cufftXt.cu',  # for cuFFT callback
+    'cupy/cuda/cupy_cufftXt.h',  # for cuFFT callback
+    'cupy/cuda/cupy_cufft.h',  # for cuFFT callback
+    'cupy/cuda/cufft.pxd',  # for cuFFT callback
+    'cupy/cuda/cufft.pyx',  # for cuFFT callback
+    'cupy/random/cupy_distributions.cu',
+    'cupy/random/cupy_distributions.cuh',
+] + [
+    x for x in glob.glob('cupy/_core/include/cupy/**', recursive=True)
+    if os.path.isfile(x)
+]
+
+package_data = {
+    'cupy': [
+        os.path.relpath(x, 'cupy') for x in cupy_package_data
+    ],
+}
+
+package_data['cupy'] += cupy_setup_build.prepare_wheel_libs(ctx)
+
+
+if len(sys.argv) < 2 or sys.argv[1] == 'egg_info':
+    # Extensions are unnecessary for egg_info generation as all sources files
+    # can be enumerated via MANIFEST.in.
+    ext_modules = []
+else:
+    ext_modules = cupy_setup_build.get_ext_modules(True, ctx)
+
+
+# Get __version__ variable
+with open(os.path.join(source_root, 'cupy', '_version.py')) as f:
+    exec(f.read())
+
+long_description = None
+if ctx.long_description_path is not None:
+    with open(ctx.long_description_path) as f:
+        long_description = f.read()
+
+
+CLASSIFIERS = """\
+Development Status :: 5 - Production/Stable
+Intended Audience :: Science/Research
+Intended Audience :: Developers
+License :: OSI Approved :: MIT License
+Programming Language :: Python
+Programming Language :: Python :: 3
+Programming Language :: Python :: 3.8
+Programming Language :: Python :: 3.9
+Programming Language :: Python :: 3.10
+Programming Language :: Python :: 3.11
+Programming Language :: Python :: 3.12
+Programming Language :: Python :: 3 :: Only
+Programming Language :: Cython
+Topic :: Software Development
+Topic :: Scientific/Engineering
+Operating System :: POSIX
+Operating System :: Microsoft :: Windows
+"""
+
+
+setup(
+    name=ctx.package_name,
+    version=__version__,  # NOQA
+    description='CuPy: NumPy & SciPy for GPU',
+    long_description=long_description,
+    long_description_content_type='text/x-rst',
+    author='Seiya Tokui',
+    author_email='tokui@preferred.jp',
+    maintainer='CuPy Developers',
+    url='https://cupy.dev/',
+    license='MIT License',
+    project_urls={
+        "Bug Tracker": "https://github.com/cupy/cupy/issues",
+        "Documentation": "https://docs.cupy.dev/",
+        "Source Code": "https://github.com/cupy/cupy",
+    },
+    classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f],
+    packages=find_packages(exclude=['install', 'tests']),
+    package_data=package_data,
+    zip_safe=False,
+    python_requires='>=3.8',
+    setup_requires=setup_requires,
+    install_requires=install_requires,
+    tests_require=tests_require,
+    extras_require=extras_require,
+    ext_modules=ext_modules,
+    cmdclass={'build_ext': cupy_builder._command.custom_build_ext},
+)
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..4e7f4a2
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,74 @@
+import collections
+import os
+import subprocess
+import sys
+
+import pytest
+
+
+# Enable `testdir` fixture to test `cupy.testing`.
+# `pytest_plugins` cannot be locally configured. See also
+# https://docs.pytest.org/en/stable/deprecations.html#pytest-plugins-in-non-top-level-conftest-files
+pytest_plugins = ['pytester']
+
+
+def _is_pip_installed():
+    try:
+        import pip  # NOQA
+        return True
+    except ImportError:
+        return False
+
+
+def _is_in_ci():
+    ci_name = os.environ.get('CUPY_CI', '')
+    return ci_name != ''
+
+
+def pytest_configure(config):
+    # Print installed packages
+    if _is_in_ci() and _is_pip_installed():
+        print("***** Installed packages *****", flush=True)
+        subprocess.check_call([sys.executable, '-m', 'pip', 'freeze', '--all'])
+    if config.pluginmanager.hasplugin("xdist"):
+        config.pluginmanager.register(DeferPlugin())
+
+
+# https://docs.pytest.org/en/latest/how-to/writing_hook_functions.html#optionally-using-hooks-from-3rd-party-plugins
+class DeferPlugin:
+    """Simple plugin to defer pytest-xdist hook functions."""
+
+    # Edit the environment variable `CUDA_VISIBLE_DEVICES` for each session.
+    # Cannot use `pytest_configure_node` nor `pytest_testnodeready` hook,
+    # because they are called in the `master` node (process).
+    # See also https://github.com/pytest-dev/pytest-xdist/issues/179.
+    @pytest.fixture(autouse=True, scope='session')
+    def _rotate_cuda_visible_devices(self, worker_id):
+        if worker_id == 'master':
+            # `worker_id` can be `master` if `pytest-xdist` is installed and
+            # run without `-n` option.
+            return
+
+        n_gpu = os.environ.get('CUPY_TEST_GPU_LIMIT')
+        if n_gpu is None:
+            print('Tip: when using pytest-xdist, you can automatically rotate'
+                  ' CUDA_VISIBLE_DEVICES for each test worker by setting'
+                  ' CUPY_TEST_GPU_LIMIT environment variable.')
+            return
+        n_gpu = int(n_gpu)
+
+        assert worker_id.startswith('gw')
+        w = int(worker_id[2:])
+
+        devices = os.environ.get('CUDA_VISIBLE_DEVICES')
+        if devices is None:
+            devices = [str(k) for k in range(n_gpu)]
+        else:
+            devices = devices.split(',')[:n_gpu]
+        devices = collections.deque(devices)
+        devices.rotate(w)
+        devices = ','.join(devices)
+        os.environ['CUDA_VISIBLE_DEVICES'] = devices
+        # With PyTest's default, the print will be shown as
+        # "--- Captured stdout setup ---" on failure.
+        print(f'CUDA_VISIBLE_DEVICES={devices}')
diff --git a/tests/cupy_tests/__init__.py b/tests/cupy_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/array_api_tests/__init__.py b/tests/cupy_tests/array_api_tests/__init__.py
new file mode 100644
index 0000000..f082a3a
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/__init__.py
@@ -0,0 +1,26 @@
+"""
+Tests for the array API namespace.
+
+Note, full compliance with the array API can be tested with the official array API test
+suite https://github.com/data-apis/array-api-tests. This test suite primarily
+focuses on those things that are not tested by the official test suite.
+"""
+
+import sys
+
+import pytest
+
+from cupy_backends.cuda.api import runtime
+
+
+if sys.version_info < (3, 8):
+    pytest.skip('Python array API requires Python 3.8+',
+                allow_module_level=True)
+
+
+# hiprtc seems to have a bug and it causes errors in some tests later. We
+# temporarily skip the Python array API tests on ROCm until it is fixed.
+# See #5843
+if runtime.is_hip:
+    pytest.skip('Python array API tests are temporarily skipped on ROCm',
+                allow_module_level=True)
diff --git a/tests/cupy_tests/array_api_tests/test_array_object.py b/tests/cupy_tests/array_api_tests/test_array_object.py
new file mode 100644
index 0000000..7df6cfb
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/test_array_object.py
@@ -0,0 +1,366 @@
+import operator
+
+from numpy.testing import assert_raises
+import cupy as cp
+import pytest
+
+from cupy.array_api import ones, asarray, reshape, result_type, all, equal
+from cupy.array_api._dtypes import (
+    _all_dtypes,
+    _boolean_dtypes,
+    _floating_dtypes,
+    _integer_dtypes,
+    _integer_or_boolean_dtypes,
+    _numeric_dtypes,
+    int8,
+    int16,
+    int32,
+    int64,
+    uint64,
+    bool as bool_,
+)
+from cupy.array_api._array_object import Array
+
+
+def test_validate_index():
+    # The indexing tests in the official array API test suite test that the
+    # array object correctly handles the subset of indices that are required
+    # by the spec. But the NumPy array API implementation specifically
+    # disallows any index not required by the spec, via Array._validate_index.
+    # This test focuses on testing that non-valid indices are correctly
+    # rejected. See
+    # https://data-apis.org/array-api/latest/API_specification/indexing.html
+    # and the docstring of Array._validate_index for the exact indexing
+    # behavior that should be allowed. This does not test indices that are
+    # already invalid in NumPy itself because Array will generally just pass
+    # such indices directly to the underlying cp.ndarray.
+
+    a = ones((3, 4))
+
+    # Out of bounds slices are not allowed
+    assert_raises(IndexError, lambda: a[:4])
+    assert_raises(IndexError, lambda: a[:-4])
+    assert_raises(IndexError, lambda: a[:3:-1])
+    assert_raises(IndexError, lambda: a[:-5:-1])
+    assert_raises(IndexError, lambda: a[4:])
+    assert_raises(IndexError, lambda: a[-4:])
+    assert_raises(IndexError, lambda: a[4::-1])
+    assert_raises(IndexError, lambda: a[-4::-1])
+
+    assert_raises(IndexError, lambda: a[...,:5])
+    assert_raises(IndexError, lambda: a[...,:-5])
+    assert_raises(IndexError, lambda: a[...,:5:-1])
+    assert_raises(IndexError, lambda: a[...,:-6:-1])
+    assert_raises(IndexError, lambda: a[...,5:])
+    assert_raises(IndexError, lambda: a[...,-5:])
+    assert_raises(IndexError, lambda: a[...,5::-1])
+    assert_raises(IndexError, lambda: a[...,-5::-1])
+
+    # Boolean indices cannot be part of a larger tuple index
+    assert_raises(IndexError, lambda: a[a[:,0]==1,0])
+    assert_raises(IndexError, lambda: a[a[:,0]==1,...])
+    assert_raises(IndexError, lambda: a[..., a[0]==1])
+    assert_raises(IndexError, lambda: a[[True, True, True]])
+    assert_raises(IndexError, lambda: a[(True, True, True),])
+
+    # Integer array indices are not allowed (except for 0-D)
+    idx = asarray([[0, 1]])
+    assert_raises(IndexError, lambda: a[idx])
+    assert_raises(IndexError, lambda: a[idx,])
+    assert_raises(IndexError, lambda: a[[0, 1]])
+    assert_raises(IndexError, lambda: a[(0, 1), (0, 1)])
+    assert_raises(IndexError, lambda: a[[0, 1]])
+    assert_raises(IndexError, lambda: a[cp.array([[0, 1]])])
+
+    # Multiaxis indices must contain exactly as many indices as dimensions
+    assert_raises(IndexError, lambda: a[()])
+    assert_raises(IndexError, lambda: a[0,])
+    assert_raises(IndexError, lambda: a[0])
+    assert_raises(IndexError, lambda: a[:])
+
+def test_operators():
+    # For every operator, we test that it works for the required type
+    # combinations and raises TypeError otherwise
+    binary_op_dtypes = {
+        "__add__": "numeric",
+        "__and__": "integer_or_boolean",
+        "__eq__": "all",
+        "__floordiv__": "numeric",
+        "__ge__": "numeric",
+        "__gt__": "numeric",
+        "__le__": "numeric",
+        "__lshift__": "integer",
+        "__lt__": "numeric",
+        "__mod__": "numeric",
+        "__mul__": "numeric",
+        "__ne__": "all",
+        "__or__": "integer_or_boolean",
+        "__pow__": "numeric",
+        "__rshift__": "integer",
+        "__sub__": "numeric",
+        "__truediv__": "floating",
+        "__xor__": "integer_or_boolean",
+    }
+
+    # Recompute each time because of in-place ops
+    def _array_vals():
+        for d in _integer_dtypes:
+            yield asarray(1, dtype=d)
+        for d in _boolean_dtypes:
+            yield asarray(False, dtype=d)
+        for d in _floating_dtypes:
+            yield asarray(1.0, dtype=d)
+
+    for op, dtypes in binary_op_dtypes.items():
+        ops = [op]
+        if op not in ["__eq__", "__ne__", "__le__", "__ge__", "__lt__", "__gt__"]:
+            rop = "__r" + op[2:]
+            iop = "__i" + op[2:]
+            ops += [rop, iop]
+        for s in [1, 1.0, False]:
+            for _op in ops:
+                for a in _array_vals():
+                    # Test array op scalar. From the spec, the following combinations
+                    # are supported:
+
+                    # - Python bool for a bool array dtype,
+                    # - a Python int within the bounds of the given dtype for integer array dtypes,
+                    # - a Python int or float for floating-point array dtypes
+
+                    # We do not do bounds checking for int scalars, but rather use the default
+                    # NumPy behavior for casting in that case.
+
+                    if ((dtypes == "all"
+                         or dtypes == "numeric" and a.dtype in _numeric_dtypes
+                         or dtypes == "integer" and a.dtype in _integer_dtypes
+                         or dtypes == "integer_or_boolean" and a.dtype in _integer_or_boolean_dtypes
+                         or dtypes == "boolean" and a.dtype in _boolean_dtypes
+                         or dtypes == "floating" and a.dtype in _floating_dtypes
+                        )
+                        # bool is a subtype of int, which is why we avoid
+                        # isinstance here.
+                        and (a.dtype in _boolean_dtypes and type(s) == bool
+                             or a.dtype in _integer_dtypes and type(s) == int
+                             or a.dtype in _floating_dtypes and type(s) in [float, int]
+                        )):
+                        # Only test for no error
+                        getattr(a, _op)(s)
+                    else:
+                        assert_raises(TypeError, lambda: getattr(a, _op)(s))
+
+                # Test array op array.
+                for _op in ops:
+                    for x in _array_vals():
+                        for y in _array_vals():
+                            # See the promotion table in NEP 47 or the array
+                            # API spec page on type promotion. Mixed kind
+                            # promotion is not defined.
+                            if (x.dtype == uint64 and y.dtype in [int8, int16, int32, int64]
+                                or y.dtype == uint64 and x.dtype in [int8, int16, int32, int64]
+                                or x.dtype in _integer_dtypes and y.dtype not in _integer_dtypes
+                                or y.dtype in _integer_dtypes and x.dtype not in _integer_dtypes
+                                or x.dtype in _boolean_dtypes and y.dtype not in _boolean_dtypes
+                                or y.dtype in _boolean_dtypes and x.dtype not in _boolean_dtypes
+                                or x.dtype in _floating_dtypes and y.dtype not in _floating_dtypes
+                                or y.dtype in _floating_dtypes and x.dtype not in _floating_dtypes
+                                ):
+                                assert_raises(TypeError, lambda: getattr(x, _op)(y))
+                            # Ensure in-place operators only promote to the same dtype as the left operand.
+                            elif (
+                                _op.startswith("__i")
+                                and result_type(x.dtype, y.dtype) != x.dtype
+                            ):
+                                assert_raises(TypeError, lambda: getattr(x, _op)(y))
+                            # Ensure only those dtypes that are required for every operator are allowed.
+                            elif (dtypes == "all" and (x.dtype in _boolean_dtypes and y.dtype in _boolean_dtypes
+                                                      or x.dtype in _numeric_dtypes and y.dtype in _numeric_dtypes)
+                                or (dtypes == "numeric" and x.dtype in _numeric_dtypes and y.dtype in _numeric_dtypes)
+                                or dtypes == "integer" and x.dtype in _integer_dtypes and y.dtype in _numeric_dtypes
+                                or dtypes == "integer_or_boolean" and (x.dtype in _integer_dtypes and y.dtype in _integer_dtypes
+                                                                       or x.dtype in _boolean_dtypes and y.dtype in _boolean_dtypes)
+                                or dtypes == "boolean" and x.dtype in _boolean_dtypes and y.dtype in _boolean_dtypes
+                                or dtypes == "floating" and x.dtype in _floating_dtypes and y.dtype in _floating_dtypes
+                            ):
+                                getattr(x, _op)(y)
+                            else:
+                                assert_raises(TypeError, lambda: getattr(x, _op)(y))
+
+    unary_op_dtypes = {
+        "__abs__": "numeric",
+        "__invert__": "integer_or_boolean",
+        "__neg__": "numeric",
+        "__pos__": "numeric",
+    }
+    for op, dtypes in unary_op_dtypes.items():
+        for a in _array_vals():
+            if (
+                dtypes == "numeric"
+                and a.dtype in _numeric_dtypes
+                or dtypes == "integer_or_boolean"
+                and a.dtype in _integer_or_boolean_dtypes
+            ):
+                # Only test for no error
+                getattr(a, op)()
+            else:
+                assert_raises(TypeError, lambda: getattr(a, op)())
+
+    # Finally, matmul() must be tested separately, because it works a bit
+    # different from the other operations.
+    def _matmul_array_vals():
+        for a in _array_vals():
+            yield a
+        for d in _all_dtypes:
+            yield ones((3, 4), dtype=d)
+            yield ones((4, 2), dtype=d)
+            yield ones((4, 4), dtype=d)
+
+    # Scalars always error
+    for _op in ["__matmul__", "__rmatmul__", "__imatmul__"]:
+        for s in [1, 1.0, False]:
+            for a in _matmul_array_vals():
+                if (type(s) in [float, int] and a.dtype in _floating_dtypes
+                    or type(s) == int and a.dtype in _integer_dtypes):
+                    # Type promotion is valid, but @ is not allowed on 0-D
+                    # inputs, so the error is a ValueError
+                    assert_raises(ValueError, lambda: getattr(a, _op)(s))
+                else:
+                    assert_raises(TypeError, lambda: getattr(a, _op)(s))
+
+    for x in _matmul_array_vals():
+        for y in _matmul_array_vals():
+            if (x.dtype == uint64 and y.dtype in [int8, int16, int32, int64]
+                or y.dtype == uint64 and x.dtype in [int8, int16, int32, int64]
+                or x.dtype in _integer_dtypes and y.dtype not in _integer_dtypes
+                or y.dtype in _integer_dtypes and x.dtype not in _integer_dtypes
+                or x.dtype in _floating_dtypes and y.dtype not in _floating_dtypes
+                or y.dtype in _floating_dtypes and x.dtype not in _floating_dtypes
+                or x.dtype in _boolean_dtypes
+                or y.dtype in _boolean_dtypes
+                ):
+                assert_raises(TypeError, lambda: x.__matmul__(y))
+                assert_raises(TypeError, lambda: y.__rmatmul__(x))
+                assert_raises(TypeError, lambda: x.__imatmul__(y))
+            elif x.shape == () or y.shape == () or x.shape[1] != y.shape[0]:
+                assert_raises(ValueError, lambda: x.__matmul__(y))
+                assert_raises(ValueError, lambda: y.__rmatmul__(x))
+                if result_type(x.dtype, y.dtype) != x.dtype:
+                    assert_raises(TypeError, lambda: x.__imatmul__(y))
+                else:
+                    assert_raises(ValueError, lambda: x.__imatmul__(y))
+            else:
+                x.__matmul__(y)
+                y.__rmatmul__(x)
+                if result_type(x.dtype, y.dtype) != x.dtype:
+                    assert_raises(TypeError, lambda: x.__imatmul__(y))
+                elif y.shape[0] != y.shape[1]:
+                    # This one fails because x @ y has a different shape from x
+                    assert_raises(ValueError, lambda: x.__imatmul__(y))
+                else:
+                    x.__imatmul__(y)
+
+
+def test_python_scalar_construtors():
+    b = asarray(False)
+    i = asarray(0)
+    f = asarray(0.0)
+
+    assert bool(b) == False
+    assert int(i) == 0
+    assert float(f) == 0.0
+    assert operator.index(i) == 0
+
+    # bool/int/float should only be allowed on 0-D arrays.
+    assert_raises(TypeError, lambda: bool(asarray([False])))
+    assert_raises(TypeError, lambda: int(asarray([0])))
+    assert_raises(TypeError, lambda: float(asarray([0.0])))
+    assert_raises(TypeError, lambda: operator.index(asarray([0])))
+
+    # bool/int/float should only be allowed on arrays of the corresponding
+    # dtype
+    assert_raises(ValueError, lambda: bool(i))
+    assert_raises(ValueError, lambda: bool(f))
+
+    assert_raises(ValueError, lambda: int(b))
+    assert_raises(ValueError, lambda: int(f))
+
+    assert_raises(ValueError, lambda: float(b))
+    assert_raises(ValueError, lambda: float(i))
+
+    assert_raises(TypeError, lambda: operator.index(b))
+    assert_raises(TypeError, lambda: operator.index(f))
+
+
+def test_array_properties():
+    a = ones((1, 2, 3))
+    b = ones((2, 3))
+    assert_raises(ValueError, lambda: a.T)
+
+    assert isinstance(b.T, Array)
+    assert b.T.shape == (3, 2)
+
+    assert isinstance(a.mT, Array)
+    assert a.mT.shape == (1, 3, 2)
+    assert isinstance(b.mT, Array)
+    assert b.mT.shape == (3, 2)
+
+
+def test___array__():
+    a = ones((2, 3), dtype=int16)
+    assert cp.asarray(a) is a._array
+    b = cp.asarray(a, dtype=cp.float64)
+    assert cp.all(cp.equal(b, cp.ones((2, 3), dtype=cp.float64)))
+    assert b.dtype == cp.float64
+
+def test_allow_newaxis():
+    a = ones(5)
+    indexed_a = a[None, :]
+    assert indexed_a.shape == (1, 5)
+
+def test_disallow_flat_indexing_with_newaxis():
+    a = ones((3, 3, 3))
+    with pytest.raises(IndexError):
+        a[None, 0, 0]
+
+def test_disallow_mask_with_newaxis():
+    a = ones((3, 3, 3))
+    with pytest.raises(IndexError):
+        a[None, asarray(True)]
+
+@pytest.mark.parametrize("shape", [(), (5,), (3, 3, 3)])
+@pytest.mark.parametrize("index", ["string", False, True])
+def test_error_on_invalid_index(shape, index):
+    a = ones(shape)
+    with pytest.raises(IndexError):
+        a[index]
+
+def test_mask_0d_array_without_errors():
+    a = ones(())
+    a[asarray(True)]
+
+@pytest.mark.parametrize(
+    "i", [slice(5), slice(5, 0), asarray(True), asarray([0, 1])]
+)
+def test_error_on_invalid_index_with_ellipsis(i):
+    a = ones((3, 3, 3))
+    with pytest.raises(IndexError):
+        a[..., i]
+    with pytest.raises(IndexError):
+        a[i, ...]
+
+def test_array_keys_use_private_array():
+    """
+    Indexing operations convert array keys before indexing the internal array
+
+    Fails when array_api array keys are not converted into NumPy-proper arrays
+    in __getitem__(). This is achieved by passing array_api arrays with 0-sized
+    dimensions, which NumPy-proper treats erroneously - not sure why!
+
+    TODO: Find and use appropiate __setitem__() case.
+    """
+    a = ones((0, 0), dtype=bool_)
+    assert a[a].shape == (0,)
+
+    a = ones((0,), dtype=bool_)
+    key = ones((0, 0), dtype=bool_)
+    with pytest.raises(IndexError):
+        a[key]
diff --git a/tests/cupy_tests/array_api_tests/test_creation_functions.py b/tests/cupy_tests/array_api_tests/test_creation_functions.py
new file mode 100644
index 0000000..35adaab
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/test_creation_functions.py
@@ -0,0 +1,155 @@
+from numpy.testing import assert_raises, assert_equal
+import cupy as cp
+# due to the module structure we can't import it from cupy.array_api._typing
+from cupy.cuda import Device
+
+from cupy.array_api import all
+from cupy.array_api._creation_functions import (
+    asarray,
+    arange,
+    empty,
+    empty_like,
+    eye,
+    full,
+    full_like,
+    linspace,
+    meshgrid,
+    ones,
+    ones_like,
+    zeros,
+    zeros_like,
+)
+from cupy.array_api._dtypes import float32, float64
+from cupy.array_api._array_object import Array
+
+
+def test_asarray_errors():
+    # Test various protections against incorrect usage
+    assert_raises(TypeError, lambda: Array([1]))
+#    assert_raises(TypeError, lambda: asarray(["a"]))  # TODO(leofang): fix this?
+    assert_raises(ValueError, lambda: asarray([1.0], dtype=cp.float16))
+    assert_raises(OverflowError, lambda: asarray(2**100))
+    # Preferably this would be OverflowError
+    # assert_raises(OverflowError, lambda: asarray([2**100]))
+#    assert_raises(TypeError, lambda: asarray([2**100]))  # TODO(leofang): fix this?
+    assert_raises(ValueError, lambda: asarray([1], device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: asarray([1], device="gpu"))
+
+    assert_raises(ValueError, lambda: asarray([1], dtype=int))
+    assert_raises(ValueError, lambda: asarray([1], dtype="i"))
+    asarray([1], device=Device())  # on current device
+
+
+def test_asarray_copy():
+    a = asarray([1])
+    b = asarray(a, copy=True)
+    a[0] = 0
+    assert all(b[0] == 1)
+    assert all(a[0] == 0)
+    # Once copy=False is implemented, replace this with
+    # a = asarray([1])
+    # b = asarray(a, copy=False)
+    # a[0] = 0
+    # assert all(b[0] == 0)
+    assert_raises(NotImplementedError, lambda: asarray(a, copy=False))
+
+
+def test_asarray_nested():
+    a = asarray([[ones(5), ones(5)], [ones(5), ones(5)]])
+    assert_equal(a.shape, (2, 2, 5))
+
+
+def test_arange_errors():
+    assert_raises(ValueError, lambda: arange(1, device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: arange(1, device="gpu"))
+    assert_raises(ValueError, lambda: arange(1, dtype=int))
+    assert_raises(ValueError, lambda: arange(1, dtype="i"))
+    arange(1, device=Device())  # on current device
+
+
+def test_empty_errors():
+    assert_raises(ValueError, lambda: empty((1,), device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: empty((1,), device="gpu"))
+    assert_raises(ValueError, lambda: empty((1,), dtype=int))
+    assert_raises(ValueError, lambda: empty((1,), dtype="i"))
+    empty((1,), device=Device())  # on current device
+
+
+def test_empty_like_errors():
+    assert_raises(ValueError, lambda: empty_like(asarray(1), device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: empty_like(asarray(1), device="gpu"))
+    assert_raises(ValueError, lambda: empty_like(asarray(1), dtype=int))
+    assert_raises(ValueError, lambda: empty_like(asarray(1), dtype="i"))
+    empty_like(asarray(1), device=Device())  # on current device
+
+
+def test_eye_errors():
+    assert_raises(ValueError, lambda: eye(1, device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: eye(1, device="gpu"))
+    assert_raises(ValueError, lambda: eye(1, dtype=int))
+    assert_raises(ValueError, lambda: eye(1, dtype="i"))
+    eye(1, device=Device())  # on current device
+
+
+def test_full_errors():
+    assert_raises(ValueError, lambda: full((1,), 0, device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: full((1,), 0, device="gpu"))
+    assert_raises(ValueError, lambda: full((1,), 0, dtype=int))
+    assert_raises(ValueError, lambda: full((1,), 0, dtype="i"))
+    full((1,), 0, device=Device())  # on current device
+
+
+def test_full_like_errors():
+    assert_raises(ValueError, lambda: full_like(asarray(1), 0, device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: full_like(asarray(1), 0, device="gpu"))
+    assert_raises(ValueError, lambda: full_like(asarray(1), 0, dtype=int))
+    assert_raises(ValueError, lambda: full_like(asarray(1), 0, dtype="i"))
+    full_like(asarray(1), 0, device=Device())  # on current device
+
+
+def test_linspace_errors():
+    assert_raises(ValueError, lambda: linspace(0, 1, 10, device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: linspace(0, 1, 10, device="gpu"))
+    assert_raises(ValueError, lambda: linspace(0, 1, 10, dtype=float))
+    assert_raises(ValueError, lambda: linspace(0, 1, 10, dtype="f"))
+    linspace(0, 1, 10, device=Device())  # on current device
+
+
+def test_ones_errors():
+    assert_raises(ValueError, lambda: ones((1,), device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: ones((1,), device="gpu"))
+    assert_raises(ValueError, lambda: ones((1,), dtype=int))
+    assert_raises(ValueError, lambda: ones((1,), dtype="i"))
+    ones((1,), device=Device())  # on current device
+
+
+def test_ones_like_errors():
+    assert_raises(ValueError, lambda: ones_like((1,), device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: ones_like(asarray(1), device="gpu"))
+    assert_raises(ValueError, lambda: ones_like(asarray(1), dtype=int))
+    assert_raises(ValueError, lambda: ones_like(asarray(1), dtype="i"))
+    ones_like(asarray(1), device=Device())  # on current device
+
+
+def test_zeros_errors():
+    assert_raises(ValueError, lambda: zeros((1,), device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: zeros((1,), device="gpu"))
+    assert_raises(ValueError, lambda: zeros((1,), dtype=int))
+    assert_raises(ValueError, lambda: zeros((1,), dtype="i"))
+    zeros((1,), device=Device())  # on current device
+
+
+def test_zeros_like_errors():
+    assert_raises(ValueError, lambda: zeros_like((1,), device="cpu"))  # numpy's pick
+    assert_raises(ValueError, lambda: zeros_like(asarray(1), device="gpu"))
+    assert_raises(ValueError, lambda: zeros_like(asarray(1), dtype=int))
+    assert_raises(ValueError, lambda: zeros_like(asarray(1), dtype="i"))
+    zeros_like(asarray(1), device=Device())  # on current device
+
+def test_meshgrid_dtype_errors():
+    # Doesn't raise
+    meshgrid()
+    meshgrid(asarray([1.], dtype=float32))
+    meshgrid(asarray([1.], dtype=float32), asarray([1.], dtype=float32))
+
+    assert_raises(ValueError, lambda: meshgrid(asarray([1.], dtype=float32), asarray([1.], dtype=float64)))
diff --git a/tests/cupy_tests/array_api_tests/test_data_type_functions.py b/tests/cupy_tests/array_api_tests/test_data_type_functions.py
new file mode 100644
index 0000000..5da43f4
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/test_data_type_functions.py
@@ -0,0 +1,19 @@
+import pytest
+
+from cupy import array_api as xp
+
+
+@pytest.mark.parametrize(
+    "from_, to, expected",
+    [
+        (xp.int8, xp.int16, True),
+        (xp.int16, xp.int8, False),
+        (xp.bool, xp.int8, False),
+        (xp.asarray(0, dtype=xp.uint8), xp.int8, False),
+    ],
+)
+def test_can_cast(from_, to, expected):
+    """
+    can_cast() returns correct result
+    """
+    assert xp.can_cast(from_, to) == expected
diff --git a/tests/cupy_tests/array_api_tests/test_elementwise_functions.py b/tests/cupy_tests/array_api_tests/test_elementwise_functions.py
new file mode 100644
index 0000000..77c064e
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/test_elementwise_functions.py
@@ -0,0 +1,111 @@
+from inspect import getfullargspec
+
+from numpy.testing import assert_raises
+
+from cupy.array_api import asarray, _elementwise_functions
+from cupy.array_api._elementwise_functions import bitwise_left_shift, bitwise_right_shift
+from cupy.array_api._dtypes import (
+    _dtype_categories,
+    _boolean_dtypes,
+    _floating_dtypes,
+    _integer_dtypes,
+)
+
+
+def nargs(func):
+    return len(getfullargspec(func).args)
+
+
+def test_function_types():
+    # Test that every function accepts only the required input types. We only
+    # test the negative cases here (error). The positive cases are tested in
+    # the array API test suite.
+
+    elementwise_function_input_types = {
+        "abs": "numeric",
+        "acos": "floating-point",
+        "acosh": "floating-point",
+        "add": "numeric",
+        "asin": "floating-point",
+        "asinh": "floating-point",
+        "atan": "floating-point",
+        "atan2": "floating-point",
+        "atanh": "floating-point",
+        "bitwise_and": "integer or boolean",
+        "bitwise_invert": "integer or boolean",
+        "bitwise_left_shift": "integer",
+        "bitwise_or": "integer or boolean",
+        "bitwise_right_shift": "integer",
+        "bitwise_xor": "integer or boolean",
+        "ceil": "numeric",
+        "cos": "floating-point",
+        "cosh": "floating-point",
+        "divide": "floating-point",
+        "equal": "all",
+        "exp": "floating-point",
+        "expm1": "floating-point",
+        "floor": "numeric",
+        "floor_divide": "numeric",
+        "greater": "numeric",
+        "greater_equal": "numeric",
+        "isfinite": "numeric",
+        "isinf": "numeric",
+        "isnan": "numeric",
+        "less": "numeric",
+        "less_equal": "numeric",
+        "log": "floating-point",
+        "logaddexp": "floating-point",
+        "log10": "floating-point",
+        "log1p": "floating-point",
+        "log2": "floating-point",
+        "logical_and": "boolean",
+        "logical_not": "boolean",
+        "logical_or": "boolean",
+        "logical_xor": "boolean",
+        "multiply": "numeric",
+        "negative": "numeric",
+        "not_equal": "all",
+        "positive": "numeric",
+        "pow": "numeric",
+        "remainder": "numeric",
+        "round": "numeric",
+        "sign": "numeric",
+        "sin": "floating-point",
+        "sinh": "floating-point",
+        "sqrt": "floating-point",
+        "square": "numeric",
+        "subtract": "numeric",
+        "tan": "floating-point",
+        "tanh": "floating-point",
+        "trunc": "numeric",
+    }
+
+    def _array_vals():
+        for d in _integer_dtypes:
+            yield asarray(1, dtype=d)
+        for d in _boolean_dtypes:
+            yield asarray(False, dtype=d)
+        for d in _floating_dtypes:
+            yield asarray(1.0, dtype=d)
+
+    for x in _array_vals():
+        for func_name, types in elementwise_function_input_types.items():
+            dtypes = _dtype_categories[types]
+            func = getattr(_elementwise_functions, func_name)
+            if nargs(func) == 2:
+                for y in _array_vals():
+                    if x.dtype not in dtypes or y.dtype not in dtypes:
+                        assert_raises(TypeError, lambda: func(x, y))
+            else:
+                if x.dtype not in dtypes:
+                    assert_raises(TypeError, lambda: func(x))
+
+
+def test_bitwise_shift_error():
+    # bitwise shift functions should raise when the second argument is negative
+    assert_raises(
+        ValueError, lambda: bitwise_left_shift(asarray([1, 1]), asarray([1, -1]))
+    )
+    assert_raises(
+        ValueError, lambda: bitwise_right_shift(asarray([1, 1]), asarray([1, -1]))
+    )
diff --git a/tests/cupy_tests/array_api_tests/test_indexing_functions.py b/tests/cupy_tests/array_api_tests/test_indexing_functions.py
new file mode 100644
index 0000000..6bfbca6
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/test_indexing_functions.py
@@ -0,0 +1,22 @@
+import pytest
+
+from cupy import array_api as xp
+
+
+@pytest.mark.parametrize(
+    "obj, ind, axis, expected",
+    [
+        ([0, 1, 2, 3], [1, 3], -1, [1, 3]),
+        ([0, 1, 2, 3], [2, 0], 0, [2, 0]),
+        ([[0, 1, 2, 3]], [0, 0, 0], 0, [[0, 1, 2, 3]] * 3),
+        ([[0, 1, 2, 3]], [1, 2, 1], 1, [[1, 2, 1]]),
+    ],
+)
+def test_take(obj, ind, axis, expected):
+    """
+    Tests xp.take function
+    """
+    x = xp.asarray(obj)
+    ind = xp.asarray(ind)
+    out = xp.take(x, ind, axis=axis)
+    assert xp.all(out == xp.asarray(expected))
diff --git a/tests/cupy_tests/array_api_tests/test_set_functions.py b/tests/cupy_tests/array_api_tests/test_set_functions.py
new file mode 100644
index 0000000..ce1eb8d
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/test_set_functions.py
@@ -0,0 +1,21 @@
+import pytest
+from hypothesis import given
+from hypothesis.extra.array_api import make_strategies_namespace
+from hypothesis import settings, HealthCheck
+
+from cupy import array_api as xp
+
+xps = make_strategies_namespace(xp)
+
+
+@pytest.mark.parametrize("func", [xp.unique_all, xp.unique_inverse])
+@given(xps.arrays(dtype=xps.scalar_dtypes(), shape=xps.array_shapes()))
+@settings(suppress_health_check=[HealthCheck.too_slow], deadline=None)
+def test_inverse_indices_shape(func, x):
+    """
+    Inverse indices share shape of input array
+
+    See https://github.com/numpy/numpy/issues/20638
+    """
+    out = func(x)
+    assert out.inverse_indices.shape == x.shape
diff --git a/tests/cupy_tests/array_api_tests/test_sorting_functions.py b/tests/cupy_tests/array_api_tests/test_sorting_functions.py
new file mode 100644
index 0000000..475f0c9
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/test_sorting_functions.py
@@ -0,0 +1,23 @@
+import pytest
+
+from cupy import array_api as xp
+
+
+@pytest.mark.parametrize(
+    "obj, axis, expected",
+    [
+        ([0, 0], -1, [0, 1]),
+        ([0, 1, 0], -1, [1, 0, 2]),
+        ([[0, 1], [1, 1]], 0, [[1, 0], [0, 1]]),
+        ([[0, 1], [1, 1]], 1, [[1, 0], [0, 1]]),
+    ],
+)
+def test_stable_desc_argsort(obj, axis, expected):
+    """
+    Indices respect relative order of a descending stable-sort
+
+    See https://github.com/numpy/numpy/issues/20778
+    """
+    x = xp.asarray(obj)
+    out = xp.argsort(x, axis=axis, stable=True, descending=True)
+    assert xp.all(out == xp.asarray(expected))
diff --git a/tests/cupy_tests/array_api_tests/test_validation.py b/tests/cupy_tests/array_api_tests/test_validation.py
new file mode 100644
index 0000000..e603779
--- /dev/null
+++ b/tests/cupy_tests/array_api_tests/test_validation.py
@@ -0,0 +1,27 @@
+from typing import Callable
+
+import pytest
+
+from cupy import array_api as xp
+
+
+def p(func: Callable, *args, **kwargs):
+    f_sig = ", ".join(
+        [str(a) for a in args] + [f"{k}={v}" for k, v in kwargs.items()]
+    )
+    id_ = f"{func.__name__}({f_sig})"
+    return pytest.param(func, args, kwargs, id=id_)
+
+
+@pytest.mark.parametrize(
+    "func, args, kwargs",
+    [
+        p(xp.can_cast, 42, xp.int8),
+        p(xp.can_cast, xp.int8, 42),
+        p(xp.result_type, 42),
+    ],
+)
+def test_raises_on_invalid_types(func, args, kwargs):
+    """Function raises TypeError when passed invalidly-typed inputs"""
+    with pytest.raises(TypeError):
+        func(*args, **kwargs)
diff --git a/tests/cupy_tests/binary_tests/__init__.py b/tests/cupy_tests/binary_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/binary_tests/test_elementwise.py b/tests/cupy_tests/binary_tests/test_elementwise.py
new file mode 100644
index 0000000..8618804
--- /dev/null
+++ b/tests/cupy_tests/binary_tests/test_elementwise.py
@@ -0,0 +1,37 @@
+import unittest
+
+from cupy import testing
+
+
+class TestElementwise(unittest.TestCase):
+
+    @testing.for_int_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def check_unary_int(self, name, xp, dtype):
+        a = xp.array([-3, -2, -1, 0, 1, 2, 3]).astype(dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_int_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def check_binary_int(self, name, xp, dtype):
+        a = xp.array([-3, -2, -1, 0, 1, 2, 3]).astype(dtype)
+        b = xp.array([0, 1, 2, 3, 4, 5, 6]).astype(dtype)
+        return getattr(xp, name)(a, b)
+
+    def test_bitwise_and(self):
+        self.check_binary_int('bitwise_and')
+
+    def test_bitwise_or(self):
+        self.check_binary_int('bitwise_or')
+
+    def test_bitwise_xor(self):
+        self.check_binary_int('bitwise_xor')
+
+    def test_invert(self):
+        self.check_unary_int('invert')
+
+    def test_left_shift(self):
+        self.check_binary_int('left_shift')
+
+    def test_right_shift(self):
+        self.check_binary_int('right_shift')
diff --git a/tests/cupy_tests/binary_tests/test_packing.py b/tests/cupy_tests/binary_tests/test_packing.py
new file mode 100644
index 0000000..0f6a574
--- /dev/null
+++ b/tests/cupy_tests/binary_tests/test_packing.py
@@ -0,0 +1,78 @@
+import numpy
+import unittest
+import pytest
+import cupy
+from cupy import testing
+
+
+class TestPacking(unittest.TestCase):
+
+    @testing.for_int_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def check_packbits(self, data, xp, dtype, bitorder='big'):
+        # Note numpy <= 1.9 raises an Exception when an input array is bool.
+        # See https://github.com/numpy/numpy/issues/5377
+        a = xp.array(data, dtype=dtype)
+        return xp.packbits(a, bitorder=bitorder)
+
+    @testing.numpy_cupy_array_equal()
+    def check_unpackbits(self, data, xp, bitorder='big'):
+        a = xp.array(data, dtype=xp.uint8)
+        return xp.unpackbits(a, bitorder=bitorder)
+
+    def test_packbits(self):
+        self.check_packbits([0])
+        self.check_packbits([1])
+        self.check_packbits([0, 1])
+        self.check_packbits([1, 0, 1, 1, 0, 1, 1, 1])
+        self.check_packbits([1, 0, 1, 1, 0, 1, 1, 1, 1])
+        self.check_packbits(numpy.arange(24).reshape((2, 3, 4)) % 2)
+
+    def test_packbits_order(self):
+        for bo in ['big', 'little']:
+            self.check_packbits([0], bitorder=bo)
+            self.check_packbits([1], bitorder=bo)
+            self.check_packbits([0, 1], bitorder=bo)
+            self.check_packbits([1, 0, 1, 1, 0, 1, 1, 1], bitorder=bo)
+            self.check_packbits([1, 0, 1, 1, 0, 1, 1, 1, 1], bitorder=bo)
+            self.check_packbits(numpy.arange(24).reshape((2, 3, 4)) % 2,
+                                bitorder=bo)
+
+    def test_packbits_empty(self):
+        # Note packbits of numpy <= 1.11 has a bug against empty arrays.
+        # See https://github.com/numpy/numpy/issues/8324
+        self.check_packbits([])
+
+    def test_pack_invalid_order(self):
+        a = cupy.array([10, 20, 30])
+        pytest.raises(ValueError, cupy.packbits, a, bitorder='ascendant')
+        pytest.raises(ValueError, cupy.packbits, a, bitorder=10.4)
+
+    def test_pack_invalid_array(self):
+        fa = cupy.array([10, 20, 30], dtype=float)
+        pytest.raises(TypeError, cupy.packbits, fa)
+
+    def test_unpackbits(self):
+        self.check_unpackbits([])
+        self.check_unpackbits([0])
+        self.check_unpackbits([1])
+        self.check_unpackbits([255])
+        self.check_unpackbits([100, 200, 123, 213])
+
+    def test_unpack_invalid_array(self):
+        a = cupy.array([10, 20, 30])
+        pytest.raises(TypeError, cupy.unpackbits, a)
+        pytest.raises(TypeError, cupy.unpackbits, a.astype(float))
+
+    def test_pack_unpack_order(self):
+        for bo in ['big', 'little']:
+            self.check_unpackbits([], bitorder=bo)
+            self.check_unpackbits([0], bitorder=bo)
+            self.check_unpackbits([1], bitorder=bo)
+            self.check_unpackbits([255], bitorder=bo)
+            self.check_unpackbits([100, 200, 123, 213], bitorder=bo)
+
+    def test_unpack_invalid_order(self):
+        a = cupy.array([10, 20, 30], dtype=cupy.uint8)
+        pytest.raises(ValueError, cupy.unpackbits, a, bitorder='r')
+        pytest.raises(ValueError, cupy.unpackbits, a, bitorder=10)
diff --git a/tests/cupy_tests/core_tests/__init__.py b/tests/cupy_tests/core_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/core_tests/fusion_tests/__init__.py b/tests/cupy_tests/core_tests/fusion_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/core_tests/fusion_tests/fusion_utils.py b/tests/cupy_tests/core_tests/fusion_tests/fusion_utils.py
new file mode 100644
index 0000000..2573b44
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/fusion_utils.py
@@ -0,0 +1,144 @@
+import numpy
+
+import cupy
+from cupy import testing
+
+
+scalar_types = (numpy.generic, int, float, complex)
+
+
+def check_fusion(
+        generate_inputs_name='generate_inputs',
+        generate_inputs_args=None,
+        check_array=None,
+        check_array_kwargs=None,
+        accept_error=()):
+    """Decorator for tests for ``cupy.fuse``.
+
+    This decorator checks the results of the original function is equals to
+    that of the fused function.
+
+    Args:
+        generate_input_args(tuple): argument tuple passed to
+            ``generate_input``. Defaults to ``()``.
+        check_array(function): testing function which compares
+            ``{numpy/cupy}.ndarray`` objects.
+            Defaults to ``testing.assert_allclose``.
+        check_array_kwargs(dict): keyword arguments passed to
+            ``check_array``. Defaults to ``{'rtol': 3e-3, 'atol': 3e-3}``.
+        accept_error(Exception or tuple of Exception):
+            Specify acceptable errors.
+    """
+    if generate_inputs_args is None:
+        generate_inputs_args = ()
+    if check_array is None:
+        check_array = testing.assert_allclose
+    if check_array_kwargs is None:
+        # TODO(imanishi): Relax tolerances only when comparing float16 arrays.
+        check_array_kwargs = {'rtol': 3e-3, 'atol': 3e-3}
+    if not isinstance(accept_error, (tuple, list)):
+        accept_error = (accept_error,)
+
+    def check(xp, actual, expected):
+        if expected is None:
+            assert actual is None
+
+        elif isinstance(expected, scalar_types + (numpy.ndarray,)):
+            assert isinstance(actual, scalar_types + (xp.ndarray,))
+            check_array(actual, expected, **check_array_kwargs)
+
+        elif isinstance(expected, (list, tuple)):
+            assert type(actual) == type(expected)
+            for item_actual, item_expected in zip(actual, expected):
+                check(xp, item_actual, item_expected)
+
+        elif isinstance(expected, dict):
+            assert isinstance(actual, dict)
+            for key, expected_value in expected:
+                actual_value = actual.pop(key)
+                check_array(xp, actual_value, expected_value)
+            assert len(actual) == 0
+
+        else:
+            assert False
+
+    # Calls `func` with the arguments `args` and `kwargs`, and returns
+    # the tuple of the return value and the exception object raised by the
+    # function.
+    def call(func, args, kwargs):
+        try:
+            ret = func(*args, **kwargs)
+            err = None
+        except Exception as e:
+            if not isinstance(e, accept_error):
+                raise
+            ret = None
+            err = e
+        return ret, err
+
+    def check_result(xp, actual, expected):
+        ret_a, err_a = actual
+        ret_e, err_e = expected
+
+        if err_e is None:
+            # Exception not raised
+            if err_a is not None:
+                raise err_a
+            check(xp, ret_a, ret_e)
+        else:
+            if err_a is None:
+                raise err_e
+
+    def deco(func):
+        def wrapper(self, **generate_inputs_kwargs):
+            generate_inputs = getattr(self, generate_inputs_name)
+
+            impl_np = func(self, numpy, **generate_inputs_kwargs)
+            impl_cp = func(self, cupy, **generate_inputs_kwargs)
+
+            # TODO(imanishi): Fix these workaround after `cupy.fuse`
+            # supports lambda function.
+            # If `cupy.fuse` supports lambda function, these lines can be
+            # written more simply (as `impl_fuse_np = cupy.fuse(impl_np)`).
+            @cupy.fuse()
+            def impl_fuse_np(*args, **kwargs):
+                return impl_np(*args, **kwargs)
+
+            @cupy.fuse()
+            def impl_fuse_cp(*args, **kwargs):
+                return impl_cp(*args, **kwargs)
+
+            args_np, kwargs_np = generate_inputs(
+                numpy, *generate_inputs_args, **generate_inputs_kwargs)
+            args_cp, kwargs_cp = generate_inputs(
+                cupy, *generate_inputs_args, **generate_inputs_kwargs)
+            args_fuse_np, kwargs_fuse_np = generate_inputs(
+                numpy, *generate_inputs_args, **generate_inputs_kwargs)
+            args_fuse_cp, kwargs_fuse_cp = generate_inputs(
+                cupy, *generate_inputs_args, **generate_inputs_kwargs)
+
+            result_np = call(impl_np, args_np, kwargs_np)
+            result_cp = call(impl_cp, args_cp, kwargs_cp)
+            result_fuse_np = call(impl_fuse_np, args_fuse_np, kwargs_fuse_np)
+            result_fuse_cp = call(impl_fuse_cp, args_fuse_cp, kwargs_fuse_cp)
+
+            check_result(cupy, result_cp, result_np)
+            check_result(numpy, result_fuse_np, result_np)
+            check_result(cupy, result_fuse_cp, result_np)
+
+            _, err = result_np
+            if err is None:
+                check(cupy, args_cp, args_np)
+                check(numpy, args_fuse_np, args_np)
+                check(cupy, args_fuse_cp, args_np)
+
+        return wrapper
+    return deco
+
+
+def can_use_grid_synchronization():
+    return (
+        not cupy.cuda.runtime.is_hip and
+        cupy.cuda.runtime.runtimeGetVersion() >= 9000 and
+        int(cupy.cuda.device.get_compute_capability()) >= 70
+    )
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_array.py b/tests/cupy_tests/core_tests/fusion_tests/test_array.py
new file mode 100644
index 0000000..8b838fc
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_array.py
@@ -0,0 +1,309 @@
+import unittest
+
+import numpy
+
+from cupy import testing
+from cupy_tests.core_tests.fusion_tests import fusion_utils
+
+
+class FusionArrayTestBase(unittest.TestCase):
+
+    def _get_argument(self, xp, dtype, seed, value_type):
+        dtype = numpy.dtype(dtype)
+        if value_type == 'array':
+            x = testing.shaped_random((3, 4), xp, dtype, scale=5, seed=seed)
+            # Avoid zero-division
+            # TODO(imanishi): Doing this only in division tests.
+            x[x == 0] = 1
+            return x
+        if value_type == 'scalar':
+            return dtype.type(3)
+        if value_type == 'primitive':
+            return dtype.type(3).tolist()
+        assert False
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = self._get_argument(xp, dtype1, 0, self.left_value)
+        y = self._get_argument(xp, dtype2, 1, self.right_value)
+        return (x, y), {}
+
+
+@testing.parameterize(*testing._parameterized.product_dict(
+    [
+        {'name': 'neg', 'func': lambda x, y: -x},
+        {'name': 'add', 'func': lambda x, y: x + y},
+        {'name': 'sub', 'func': lambda x, y: x - y},
+        {'name': 'mul', 'func': lambda x, y: x * y},
+        {'name': 'div', 'func': lambda x, y: x / y},
+        {'name': 'pow', 'func': lambda x, y: x ** y},
+        {'name': 'eq', 'func': lambda x, y: x == y},
+        {'name': 'ne', 'func': lambda x, y: x != y},
+        {'name': 'lt', 'func': lambda x, y: x < y},
+        {'name': 'le', 'func': lambda x, y: x <= y},
+        {'name': 'gt', 'func': lambda x, y: x > y},
+        {'name': 'ge', 'func': lambda x, y: x >= y},
+    ],
+    [
+        {'left_value': 'array', 'right_value': 'array'},
+        {'left_value': 'array', 'right_value': 'scalar'},
+        {'left_value': 'array', 'right_value': 'primitive'},
+        {'left_value': 'scalar', 'right_value': 'array'},
+        {'left_value': 'primitive', 'right_value': 'array'},
+    ]
+))
+class TestFusionArrayOperator(FusionArrayTestBase):
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_operator(self, xp, dtype1, dtype2):
+        return self.func
+
+
+@testing.parameterize(*testing._parameterized.product_dict(
+    [
+        {'name': 'lshift', 'func': lambda x, y: x << y},
+        {'name': 'rshift', 'func': lambda x, y: x >> y},
+        {'name': 'and', 'func': lambda x, y: x & y},
+        {'name': 'or', 'func': lambda x, y: x | y},
+        {'name': 'xor', 'func': lambda x, y: x ^ y},
+        {'name': 'invert', 'func': lambda x, y: ~x},
+    ],
+    [
+        {'left_value': 'array', 'right_value': 'array'},
+        {'left_value': 'array', 'right_value': 'scalar'},
+        {'left_value': 'array', 'right_value': 'primitive'},
+        {'left_value': 'scalar', 'right_value': 'array'},
+        {'left_value': 'primitive', 'right_value': 'array'},
+    ]
+))
+class TestFusionArrayBitwiseOperator(FusionArrayTestBase):
+
+    def _is_uint64(self, x):
+        return not isinstance(x, int) and x.dtype == 'uint64'
+
+    def _is_signed_int(self, x):
+        return isinstance(x, int) or x.dtype.kind == 'i'
+
+    @testing.for_int_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_operator(self, xp, dtype1, dtype2):
+        def func(x, y):
+            if ((self._is_uint64(x) and self._is_signed_int(y))
+                    or (self._is_uint64(y) and self._is_signed_int(x))):
+                # Skip TypeError case.
+                return
+            return self.func(x, y)
+
+        return func
+
+
+@testing.parameterize(
+    {'left_value': 'array', 'right_value': 'array'},
+    {'left_value': 'array', 'right_value': 'scalar'},
+    {'left_value': 'array', 'right_value': 'primitive'},
+    {'left_value': 'scalar', 'right_value': 'array'},
+    {'left_value': 'primitive', 'right_value': 'array'},
+)
+class TestFusionArrayFloorDivide(FusionArrayTestBase):
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_bool=True, no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_floor_divide(self, xp, dtype1, dtype2):
+        return lambda x, y: x // y
+
+
+# TODO(imanishi): Fix TypeError in use of dtypes_combination test.
+@testing.parameterize(*testing._parameterized.product_dict(
+    [
+        {'left_value': 'array', 'right_value': 'array'},
+        {'left_value': 'array', 'right_value': 'scalar'},
+        {'left_value': 'array', 'right_value': 'primitive'},
+    ]
+))
+class TestFusionArrayInplaceOperator(FusionArrayTestBase):
+
+    def generate_inputs(self, xp, dtype):
+        x = self._get_argument(xp, dtype, 0, self.left_value)
+        y = self._get_argument(xp, dtype, 1, self.right_value)
+        return (x, y), {}
+
+    @testing.for_all_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_iadd(self, xp, dtype):
+        def func(x, y):
+            x += y
+
+        return func
+
+    @testing.for_all_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_isub(self, xp, dtype):
+        def func(x, y):
+            x -= y
+
+        return func
+
+    @testing.for_all_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_imul(self, xp, dtype):
+        def func(x, y):
+            x *= y
+
+        return func
+
+    @testing.for_float_dtypes()
+    @fusion_utils.check_fusion()
+    def test_itruediv_py3(self, xp, dtype):
+        def func(x, y):
+            x /= y
+
+        return func
+
+    @testing.for_int_dtypes(no_bool=True)
+    @fusion_utils.check_fusion(accept_error=(TypeError,))
+    def test_int_itruediv_py3_raises(self, xp, dtype):
+        def func(x, y):
+            x /= y
+
+        return func
+
+    @testing.for_int_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_imod(self, xp, dtype):
+        def func(x, y):
+            x %= y
+
+        return func
+
+    @testing.for_all_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_ipow(self, xp, dtype):
+        def func(x, y):
+            x **= y
+
+        return func
+
+    @testing.for_int_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_ilshift(self, xp, dtype):
+        def func(x, y):
+            x <<= y
+
+        return func
+
+    @testing.for_int_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_irshift(self, xp, dtype):
+        def func(x, y):
+            x >>= y
+
+        return func
+
+    @testing.for_int_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_iand(self, xp, dtype):
+        def func(x, y):
+            x &= y
+
+        return func
+
+    @testing.for_int_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_ior(self, xp, dtype):
+        def func(x, y):
+            x |= y
+
+        return func
+
+    @testing.for_int_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_ixor(self, xp, dtype):
+        def func(x, y):
+            x ^= y
+
+        return func
+
+
+class TestFusionArraySetItem(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=1)
+        return (x, y), {}
+
+    # TODO(imanishi): Fix TypeError in use of dtypes_combination test.
+    @fusion_utils.check_fusion(accept_error=TypeError)
+    def test_setitem_ellipsis(self, xp):
+        def func(x, y):
+            y[...] = x
+            return y
+
+        return func
+
+    # TODO(imanishi): Fix TypeError in use of dtypes_combination test.
+    @fusion_utils.check_fusion(accept_error=TypeError)
+    def test_setitem_non_slice(self, xp):
+        def func(x, y):
+            y[:] = x
+            return y
+
+        return func
+
+
+class TestFusionArrayMethods(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp, dtype, scale=10, seed=0)
+        return (x,), {}
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_copy(self, xp, dtype):
+        return lambda x: x.copy()
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_sum(self, xp, dtype):
+        return lambda x: x.sum()
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_prod(self, xp, dtype):
+        return lambda x: x.prod()
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_max(self, xp, dtype):
+        return lambda x: x.max()
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_min(self, xp, dtype):
+        return lambda x: x.min()
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_all(self, xp, dtype):
+        return lambda x: x.all()
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_any(self, xp, dtype):
+        return lambda x: x.any()
+
+
+class TestFusionArrayAsType(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        return (x,), {}
+
+    # TODO(asi1024): Raise complex warnings.
+    @testing.for_all_dtypes(name='dtype1', no_complex=True)
+    @testing.for_all_dtypes(name='dtype2')
+    @fusion_utils.check_fusion()
+    def test_astype(self, xp, dtype1, dtype2):
+        return lambda x: x.astype(dtype2)
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_example.py b/tests/cupy_tests/core_tests/fusion_tests/test_example.py
new file mode 100644
index 0000000..50ec20c
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_example.py
@@ -0,0 +1,46 @@
+import unittest
+import pytest
+
+import cupy
+from cupy import testing
+from cupy_tests.core_tests.fusion_tests import fusion_utils
+
+
+@testing.slow
+@pytest.mark.skipif(
+    cupy.cuda.runtime.is_hip, reason='HIP does not support this')
+class TestFusionExample(unittest.TestCase):
+    def generate_inputs(self, xp):
+        shape = (8, 64, 112, 112)
+        _, chan, _, _ = shape
+        x = testing.shaped_random(shape, xp, 'float32', scale=10, seed=0)
+        gamma = xp.ones(chan)
+        beta = xp.zeros(chan)
+        running_mean = xp.zeros(chan)
+        running_var = xp.ones(chan)
+        size = x.size // gamma.size
+        adjust = size / max(size - 1., 1.)
+        return (x, gamma, beta, running_mean, running_var, size, adjust), {}
+
+    @fusion_utils.check_fusion()
+    def test_batchnorm(self, xp):
+        def batchnorm(x, gamma, beta, running_mean, running_var, size, adjust):
+            decay = 0.9
+            eps = 2e-5
+            expander = (None, slice(None), None, None)
+
+            gamma = gamma[expander]
+            beta = beta[expander]
+            mean = xp.sum(x, axis=(0, 2, 3)) / size
+            diff = x - mean[expander]
+            var = xp.sum(diff * diff, axis=(0, 2, 3)) / size
+            inv_std = 1. / xp.sqrt(var + eps)
+            y = gamma * diff * inv_std[expander] + beta
+
+            running_mean *= decay
+            running_mean += (1 - decay) * mean
+            running_var *= decay
+            running_var += (1 - decay) * adjust * var
+            return y
+
+        return batchnorm
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_indexing.py b/tests/cupy_tests/core_tests/fusion_tests/test_indexing.py
new file mode 100644
index 0000000..1ba0be9
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_indexing.py
@@ -0,0 +1,117 @@
+import unittest
+
+from cupy import testing
+from cupy_tests.core_tests.fusion_tests import fusion_utils
+
+
+@testing.parameterize(
+    # Integer
+    {'shape': (2, 3, 4), 'indices': 1},
+    {'shape': (2, 3, 4), 'indices': -1},
+    {'shape': (2, 3, 4), 'indices': (1,)},
+    {'shape': (2, 3, 4), 'indices': (1, 0)},
+    {'shape': (2, 3, 4), 'indices': (1, 0, 2)},
+    {'shape': (2, 3, 4), 'indices': (-1, 0, -2)},
+    # Slice
+    {'shape': (2, 3, 4), 'indices': slice(None)},
+    {'shape': (2, 3, 4), 'indices': slice(None, None, 1)},
+    {'shape': (2, 3, 4), 'indices': slice(None, None, -1)},
+    {'shape': (2, 3, 4), 'indices': (slice(None), slice(None, None, -1))},
+    # Ellipsis
+    {'shape': (2, 3, 4), 'indices': Ellipsis},
+    {'shape': (2, 3, 4), 'indices': (Ellipsis,)},
+    # Newaxis
+    {'shape': (2, 3, 4), 'indices': None},
+    {'shape': (2, 3, 4), 'indices': (None,)},
+    {'shape': (2, 3, 4), 'indices': (None, None, None)},
+    # Tuple
+    {'shape': (2, 3, 4), 'indices': (slice(None), 0, slice(None, None, -1))},
+    {'shape': (2, 3, 4), 'indices': (1, None, slice(None, None, -1), None, 2)},
+    {'shape': (2,), 'indices': (slice(None), None)},
+    {'shape': (2, 3, 4), 'indices': (Ellipsis, 2)},
+    {'shape': (2, 3, 4), 'indices': (1, Ellipsis)},
+    {'shape': (2, 3, 4, 5), 'indices': (1, Ellipsis, 3)},
+)
+class TestIndexing(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype, scale=10, seed=0)
+        return (x,), {}
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_getitem(self, xp, dtype):
+        return lambda x: x[self.indices]
+
+
+@testing.parameterize(
+    {'shape': (), 'indices': 0},
+    {'shape': (2, 3), 'indices': (0, 0, 0)},
+    {'shape': (2, 3, 4), 'indices': -3},
+    {'shape': (2, 3, 4), 'indices': 3},
+)
+@testing.with_requires('numpy>=1.12.0')
+class TestArrayInvalidIndex(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype)
+        return (x,), {}
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion(accept_error=(IndexError,))
+    def test_invalid_getitem(self, xp, dtype):
+        return lambda x: x[self.indices]
+
+
+class TestIndexingCombination(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        y = testing.shaped_random((4,), xp, dtype2, scale=10, seed=1)
+        z = testing.shaped_random((1,), xp, dtype1, scale=10, seed=2)
+        return (x, y, z), {}
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_indexing_and_add_1(self, xp, dtype1, dtype2):
+        return lambda x, y, z: x + y[1]
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_indexing_and_add_2(self, xp, dtype1, dtype2):
+        return lambda x, y, z: x + z[0] + y
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_indexing_and_add_3(self, xp, dtype1, dtype2):
+        return lambda x, y, z: x + x[0] + x[1]
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_indexing_and_add_4(self, xp, dtype1, dtype2):
+        return lambda x, y, z: x + x[0, 1] + x[1] + x + x[2, 1]
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_indexing_twice_1(self, xp, dtype1, dtype2):
+        return lambda x, y, z: x[0][1]
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_indexing_twice_2(self, xp, dtype1, dtype2):
+        return lambda x, y, z: x[0][1] + x[1][0]
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_indexing_twice_3(self, xp, dtype1, dtype2):
+        return lambda x, y, z: x[0][1] + x[1] + y[0] + x[1][0] + x
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_kernel_cache.py b/tests/cupy_tests/core_tests/fusion_tests/test_kernel_cache.py
new file mode 100644
index 0000000..fc0df34
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_kernel_cache.py
@@ -0,0 +1,176 @@
+import unittest
+from unittest import mock
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+class CreateMock(object):
+
+    def __init__(self, target):
+        self.target = eval(target)
+        self.call_count = 0
+
+    def __call__(self, *args, **kwargs):
+        self.call_count += 1
+        return self.target(*args, **kwargs)
+
+    def check_call_count(self, xp, count):
+        # TODO(asi1024): Uncomment after replace fusion implementaiton.
+
+        # assert xp in (numpy, cupy)
+        # assert isinstance(count, int)
+        # if xp is cupy:
+        #     assert self.call_count == count
+        pass
+
+
+def mock_fusion_history():
+    def wrapper(impl):
+        def new_impl(self):
+            target = 'cupy._core._fusion_trace.TraceImpl'
+            with mock.patch(target, CreateMock(target)) as m:
+                numpy_result = impl(self, numpy, m)
+            with mock.patch(target, CreateMock(target)) as m:
+                cupy_result = impl(self, cupy, m)
+            testing.assert_array_list_equal(numpy_result, cupy_result)
+        return new_impl
+    return wrapper
+
+
+class TestFusionCache(unittest.TestCase):
+
+    @mock_fusion_history()
+    def test_same_array(self, xp, m):
+        @cupy.fuse()
+        def f(x, y):
+            return x + y
+
+        result = []
+        m.check_call_count(xp, 0)
+
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=1)
+        result.append(f(x, y))
+        m.check_call_count(xp, 1)
+
+        result.append(f(x, y))
+        m.check_call_count(xp, 1)
+
+        return result
+
+    @mock_fusion_history()
+    def test_dtype_combinations(self, xp, m):
+        @cupy.fuse()
+        def f(x, y):
+            return x + y
+
+        result = []
+        m.check_call_count(xp, 0)
+
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=1)
+        result.append(f(x, y))
+        m.check_call_count(xp, 1)
+
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=2)
+        y = testing.shaped_random((3, 4), xp, 'int16', scale=10, seed=3)
+        result.append(f(x, y))
+        m.check_call_count(xp, 2)
+
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=4)
+        y = testing.shaped_random((3, 4), xp, 'int16', scale=10, seed=5)
+        result.append(f(x, y))
+        m.check_call_count(xp, 2)
+
+        x = testing.shaped_random((3, 4), xp, 'int16', scale=10, seed=6)
+        y = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=7)
+        result.append(f(x, y))
+        m.check_call_count(xp, 3)
+
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=8)
+        y = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=9)
+        result.append(f(x, y))
+        m.check_call_count(xp, 3)
+
+        return result
+
+    @mock_fusion_history()
+    def test_shape_combinations(self, xp, m):
+        @cupy.fuse()
+        def f(x, y):
+            return x + y
+
+        result = []
+        m.check_call_count(xp, 0)
+
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=1)
+        result.append(f(x, y))
+        m.check_call_count(xp, 1)
+
+        x = testing.shaped_random((4, 5), xp, 'int32', scale=10, seed=2)
+        y = testing.shaped_random((4, 5), xp, 'int32', scale=10, seed=3)
+        result.append(f(x, y))
+        m.check_call_count(xp, 1)
+
+        x = testing.shaped_random((5,), xp, 'int32', scale=10, seed=4)
+        y = testing.shaped_random((4, 5), xp, 'int32', scale=10, seed=5)
+        result.append(f(x, y))
+        m.check_call_count(xp, 2)
+
+        x = testing.shaped_random((4, 3), xp, 'int32', scale=10, seed=6)
+        y = testing.shaped_random((4, 3), xp, 'int32', scale=10, seed=7)
+        result.append(f(x, y))
+        m.check_call_count(xp, 2)
+
+        x = testing.shaped_random((4, 1), xp, 'int32', scale=10, seed=8)
+        y = testing.shaped_random((4, 5), xp, 'int32', scale=10, seed=9)
+        result.append(f(x, y))
+        m.check_call_count(xp, 3)
+
+        x = testing.shaped_random((1, 1), xp, 'int32', scale=10, seed=8)
+        y = testing.shaped_random((1, 1), xp, 'int32', scale=10, seed=9)
+        result.append(f(x, y))
+        m.check_call_count(xp, 3)
+
+        x = testing.shaped_random((2, 5), xp, 'int32', scale=10, seed=10)
+        y = testing.shaped_random((4, 5), xp, 'int32', scale=10, seed=11)
+        with self.assertRaises(ValueError, msg='could not be broadcast'):
+            f(x, y)
+        m.check_call_count(xp, 4)
+
+        return result
+
+    @mock_fusion_history()
+    def test_memoryspace_combinations(self, xp, m):
+        @cupy.fuse()
+        def f(x, y):
+            return x + y
+
+        result = []
+        m.check_call_count(xp, 0)
+
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=0)
+        y = x
+        result.append(f(x, y))
+        m.check_call_count(xp, 1)
+
+        x = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=2)
+        y = testing.shaped_random((3, 4), xp, 'int32', scale=10, seed=3)
+        result.append(f(x, y))
+        m.check_call_count(xp, 2)
+
+        x = testing.shaped_random((3, 3), xp, 'int32', scale=10, seed=4)
+        y = x
+        result.append(f(x, y))
+        m.check_call_count(xp, 2)
+
+        x = testing.shaped_random((3, 3), xp, 'int32', scale=10, seed=6)
+        y = x.T
+        result.append(f(x, y))
+        m.check_call_count(xp, 3)
+
+        return result
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_misc.py b/tests/cupy_tests/core_tests/fusion_tests/test_misc.py
new file mode 100644
index 0000000..7c6f3f2
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_misc.py
@@ -0,0 +1,422 @@
+import threading
+import unittest
+from unittest import mock
+
+import cupy
+from cupy import testing
+from cupy_tests.core_tests.fusion_tests import fusion_utils
+
+
+class FusionTestBase(unittest.TestCase):
+    def generate_inputs(self, xp, nargs, dtype):
+        inputs = [
+            testing.shaped_random((3, 4), xp, dtype, scale=10, seed=seed)
+            for seed in range(nargs)
+        ]
+        return inputs, {}
+
+    def dtype_combination(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, dtype2, scale=10, seed=1)
+        return (x, y), {}
+
+
+class TestFusionInplaceUpdate(FusionTestBase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(5,))
+    def test_outarg_mixed(self, xp, dtype):
+        def func(x, y, z, u, v):
+            xp.add(x, y, out=z)
+            xp.subtract(z, x, out=u)
+            xp.multiply(z, x, out=v)
+            return u
+
+        return func
+
+    @testing.for_all_dtypes(no_bool=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(2,))
+    def test_iadd_multiple_times(self, xp, dtype):
+        def func(x, y):
+            x += y
+            x += y
+            x += y
+            return x
+
+        return func
+
+
+class TestFusionTuple(FusionTestBase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(3,))
+    def test_tuple(self, xp, dtype):
+        def func(x, y, z):
+            w = x * y + z
+            (x, w) = (w, x)
+            return z * w + y + x
+
+        return func
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(3,))
+    def test_return_tuple(self, xp, dtype):
+        def func(x, y, z):
+            return x + y, y + z, z * x
+
+        return func
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(3,))
+    def test_multiple_outputdifferent_type_same_ufunc(self, xp, dtype):
+        def func(x, y, z):
+            x = x.astype('int32')
+            y = x.astype('float32')
+            return x + y, y + z, z + x
+
+        return func
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(1,))
+    def test_return_empty_tuple(self, xp, dtype):
+        def func(x):
+            return ()
+
+        return func
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(1,))
+    def test_return_singleton_tuple(self, xp, dtype):
+        def func(x):
+            return (x,)
+
+        return func
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(2,))
+    def test_various_shape(self, xp, dtype):
+        def func(x, y):
+            a = x + y
+            b = xp.sum(a, axis=0)
+            x += b
+            c = xp.sum(x, axis=0)
+            y += c
+            return c, b, x, y, a
+        return func
+
+
+class TestReturnNone(FusionTestBase):
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion(generate_inputs_args=(0,))
+    def test_pass(self, xp, dtype):
+        def func():
+            pass
+        return func
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(1,))
+    def test_iadd_once(self, xp, dtype):
+        def func(x):
+            x += 2
+        return func
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @fusion_utils.check_fusion(generate_inputs_args=(1,))
+    def test_iadd_twice(self, xp, dtype):
+        def impl(x):
+            x += x
+            x += x
+        return impl
+
+
+class TestFusionNoneParams(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_python_none_parameter(self, xp, dtype):
+        @cupy.fuse()
+        def f(x, y, z):
+            if y is None:
+                return x * z
+            return x + y + z
+
+        x = testing.shaped_arange((10,), xp, dtype)
+        y = testing.shaped_arange((10,), xp, dtype)
+        z = testing.shaped_arange((10,), xp, dtype)
+        return f(x, None, z) + f(x, y, z)
+
+
+class TestSpecialValues(FusionTestBase):
+
+    @testing.for_float_dtypes()
+    @fusion_utils.check_fusion(generate_inputs_args=(1,))
+    def test_nan(self, xp, dtype):
+        def func(x):
+            x[:] = cupy.nan
+            return x
+        return func
+
+    @testing.for_float_dtypes()
+    @fusion_utils.check_fusion(generate_inputs_args=(1,))
+    def test_inf(self, xp, dtype):
+        def func(x):
+            x[:] = cupy.inf
+            return x
+        return func
+
+    @testing.for_float_dtypes()
+    @fusion_utils.check_fusion(generate_inputs_args=(1,))
+    def test_neginf(self, xp, dtype):
+        def func(x):
+            x[:] = -cupy.inf
+            return x
+        return func
+
+
+class TestFusionDecorator(unittest.TestCase):
+    def test_without_paren(self):
+        @cupy.fuse
+        def func_wo_paren(x):
+            """Fuse without parentheses"""
+            return x + x
+
+        assert func_wo_paren.__name__ == 'func_wo_paren'
+        assert func_wo_paren.__doc__ == 'Fuse without parentheses'
+
+    def test_with_paren(self):
+        @cupy.fuse()
+        def func_w_paren(x):
+            """Fuse with parentheses"""
+            return x + x
+
+        assert func_w_paren.__name__ == 'func_w_paren'
+        assert func_w_paren.__doc__ == 'Fuse with parentheses'
+
+
+class TestFusionKernelName(unittest.TestCase):
+
+    def check(self, xp, func, expected_name, is_elementwise):
+        a = xp.arange(0, 12, dtype='d').reshape(3, 4)
+        b = xp.arange(5, 17, dtype='d').reshape(3, 4)
+        c = xp.arange(13, 25, dtype='d').reshape(3, 4)
+
+        # Test kernel name (with mock)
+        if xp is cupy:
+            target = cupy._core._fusion_kernel.FusedKernel
+            target_full_name = '{}.{}'.format(
+                target.__module__, target.__name__)
+
+            with mock.patch(target_full_name) as kernel:  # NOQA
+                func(a, b, c)
+                # TODO(asi1024): Uncomment after replace fusion implementaiton.
+                # kernel.assert_called_once()
+                # self.assertEqual(kernel.call_args.args[0], expected_name)
+
+        # Test there's no error in computation (without mock)
+        return func(a, b, c)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_elementwise(self, xp):
+        def func(a, b, c):
+            @cupy.fuse()
+            def func_a1(x, y, z):
+                return (x + y) * z
+
+            return func_a1(a, b, c)
+
+        return self.check(xp, func, 'func_a1', True)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_elementwise_with_name(self, xp):
+        def func(a, b, c):
+            @cupy.fuse(kernel_name='abc')
+            def func_a1(x, y, z):
+                return (x + y) * z
+
+            return func_a1(a, b, c)
+
+        return self.check(xp, func, 'abc', True)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_reduction_premap(self, xp):
+        def func(a, b, c):
+            @cupy.fuse()
+            def func_a1(x, y, z):
+                return xp.sum((x + y) * z)
+
+            return func_a1(a, b, c)
+
+        return self.check(xp, func, 'func_a1', False)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_reduction_postmap(self, xp):
+        def func(a, b, c):
+            @cupy.fuse()
+            def func_a1(x):
+                return xp.sqrt(xp.sum(x) + 10)
+
+            return func_a1(a)
+
+        return self.check(xp, func, 'func_a1', False)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_reduction_01(self, xp):
+        def func(a, b, c):
+            @cupy.fuse()
+            def func_a1(x, y, z):
+                return xp.sqrt(xp.prod(x + y * z, axis=1) + 10)
+
+            return func_a1(a, b, c)
+
+        return self.check(xp, func, 'func_a1', False)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_reduction_with_name(self, xp):
+        def func(a, b, c):
+            @cupy.fuse(kernel_name='abc')
+            def func_a1(x, y, z):
+                return xp.sum((x + y) * z)
+
+            return func_a1(a, b, c)
+
+        return self.check(xp, func, 'abc', False)
+
+
+class TestFusionComposition(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_composition(self, xp, dtype):
+        @cupy.fuse()
+        def f(x, y):
+            return x - y * 2, x + y
+
+        @cupy.fuse()
+        def g(x, y, z):
+            a, b = f(x + z, z - x * 3)
+            c, d = f(x - y, y - z)
+            return a + b * c - d
+
+        @cupy.fuse()
+        def h(x, y):
+            a, b = f(x + y * 2, y * 3)
+            return a - b * g(x - 2, x - 3, -y)
+
+        x = testing.shaped_arange((3, 3), xp, dtype)
+        y = testing.shaped_arange((3, 3), xp, dtype)
+        return h(x, y)
+
+
+class TestFusionCompile(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_clear_cache(self, xp, dtype):
+        @cupy.fuse()
+        def f(x, y):
+            return x - y * 2
+
+        x = testing.shaped_arange((3, 3), xp, dtype)
+        y = testing.shaped_arange((3, 3), xp, dtype)
+        f.clear_cache()
+        return f(x, y)
+
+
+class TestFusionGetArrayModule(FusionTestBase):
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion(generate_inputs_args=(1,))
+    def test_get_array_module(self, xp, dtype):
+        def func(x):
+            assert xp == cupy.get_array_module(x)
+            return x
+
+        return func
+
+
+class TestFusionThread(unittest.TestCase):
+
+    def test_thread(self):
+        x = testing.shaped_arange((3, 3), cupy, cupy.int64)
+        y = testing.shaped_arange((3, 3), cupy, cupy.int64)
+        out = [None]
+
+        @cupy.fuse()
+        def f(x, y):
+            return x + y * 2
+
+        def _target(x, y):
+            cupy.cuda.Device(0).use()
+            out[0] = f(x, y)
+
+        t = threading.Thread(target=_target, args=(x, y))
+        t.daemon = True
+        t.start()
+        t.join()
+        assert (out[0] == f(x, y)).all()
+
+    @testing.numpy_cupy_array_equal()
+    def test_thread_multiple_dtypes(self, xp):
+        x1 = testing.shaped_arange((3, 3), xp, xp.int64)
+        y1 = testing.shaped_arange((3, 3), xp, xp.int64)
+        x2 = x1.astype(xp.float64)
+        y2 = y1.astype(xp.float64)
+        threads = [None] * 100
+        out = [None] * 100
+
+        @cupy.fuse()
+        def f(x, y):
+            return x + y * 2
+
+        def _target(tid, x, y):
+            if xp is cupy:
+                xp.cuda.Device(0).use()
+            out[tid] = f(x, y).astype(xp.int64)
+
+        def run_thread(tid):
+            x, y = (x1, y1) if tid % 2 == 0 else (x2, y2)
+            t = threading.Thread(target=_target, args=(tid, x, y))
+            threads[tid] = t
+            t.daemon = True
+            t.start()
+
+        for tid in range(0, 50):
+            run_thread(tid)
+
+        for tid in range(0, 50):
+            threads[tid].join()
+
+        for tid in range(50, 100):
+            run_thread(tid)
+
+        for tid in range(50, 100):
+            threads[tid].join()
+
+        return xp.concatenate(out)
+
+
+class TestFusionMultiDevice(unittest.TestCase):
+
+    @testing.multi_gpu(2)
+    @testing.numpy_cupy_array_equal()
+    def test_multi_device(self, xp):
+
+        @cupy.fuse()
+        def f(x, y):
+            return x + y * 2
+
+        with cupy.cuda.Device(0):
+            x1 = testing.shaped_random((3, 3), xp, xp.int64, seed=0)
+            y1 = testing.shaped_random((3, 3), xp, xp.int64, seed=1)
+            out1 = f(x1, y1)
+
+        with cupy.cuda.Device(1):
+            x2 = testing.shaped_random((3, 3), xp, xp.int64, seed=2)
+            y2 = testing.shaped_random((3, 3), xp, xp.int64, seed=3)
+            out2 = f(x2, y2)
+
+        return out1, out2
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_optimization.py b/tests/cupy_tests/core_tests/fusion_tests/test_optimization.py
new file mode 100644
index 0000000..ce25e19
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_optimization.py
@@ -0,0 +1,199 @@
+import unittest
+from unittest import mock
+
+import cupy  # NOQA
+from cupy import testing
+from cupy_tests.core_tests.fusion_tests import fusion_utils
+
+
+class CreateMock(object):
+
+    def __init__(self, target):
+        self.target = eval(target)
+        self.retvals = []
+
+    def __call__(self, *args, **kwargs):
+        ret = self.target(*args, **kwargs)
+        self.retvals.append(ret)
+        return ret
+
+    def check_number_of_ops(
+            self, loops, memories, variables, lookup, mutate):
+        # TODO(asi1024): Uncomment after replace fusion implementaiton.
+
+        # assert isinstance(loops, int)
+        # assert isinstance(memories, int)
+        # assert isinstance(variables, int)
+        # assert isinstance(lookup, list)
+        # assert isinstance(mutate, list)
+        # assert len(lookup) == len(mutate) == loops
+
+        # assert len(self.retvals) == 1
+        # history = self.retvals[0]
+        # assert len(history.op_list) == loops
+        # memory_space_set = set([p.memory for p in history.kernel_params])
+        # assert len(memory_space_set) == memories
+        # assert len(history.kernel_params) == variables
+        # for op, r, w in zip(history.op_list, lookup, mutate):
+        #     assert len(op.in_params) == r
+        #     assert len(op.out_params) == w
+        pass
+
+
+def check_number_of_ops(
+        loops, memories, variables, lookup, mutate):
+    def wrapper(test_method):
+        def new_impl(self, *args, **kwargs):
+            target = 'cupy._core._fusion_trace.TraceImpl'
+            with mock.patch(target, CreateMock(target)) as m:
+                result = test_method(self, *args, **kwargs)
+                m.check_number_of_ops(
+                    loops, memories, variables, lookup, mutate)
+            return result
+        return new_impl
+    return wrapper
+
+
+class TestOptimizations(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=1)
+        return (x, y), {}
+
+    def generate_input_broadcast(self, xp):
+        x = testing.shaped_random((3, 1, 4), xp, 'int64', scale=10, seed=0)
+        y = testing.shaped_random((3, 5, 4), xp, 'int64', scale=10, seed=1)
+        return (x, y), {}
+
+    def generate_input_same_memory(self, xp):
+        x = testing.shaped_random((4, 4), xp, 'int64', scale=10, seed=0)
+        return (x, x, x.T), {}
+
+    @check_number_of_ops(
+        loops=1, memories=3, variables=3, lookup=[2], mutate=[1])
+    @fusion_utils.check_fusion()
+    def test_one_elementwise_op(self, xp):
+        return lambda x, y: x + y
+
+    @check_number_of_ops(
+        loops=1, memories=3, variables=3, lookup=[2], mutate=[1])
+    @fusion_utils.check_fusion()
+    def test_fuse_elementwise_op_1(self, xp):
+        def impl(x, y):
+            return x + x + y + y
+        return impl
+
+    @check_number_of_ops(
+        loops=1, memories=3, variables=3, lookup=[2], mutate=[1])
+    @fusion_utils.check_fusion()
+    def test_fuse_elementwise_op_2(self, xp):
+        def impl(x, y):
+            z = x + y
+            return z + z
+        return impl
+
+    @check_number_of_ops(
+        # TODO(asi1024): memory space = 3.
+        loops=1, memories=4, variables=4, lookup=[3], mutate=[1])
+    @fusion_utils.check_fusion()
+    def test_fuse_elementwise_ops_4(self, xp):
+        def impl(x, y):
+            res = 0
+            for i in range(10):
+                res += x + y
+            return res
+        return impl
+
+    @check_number_of_ops(
+        loops=0, memories=1, variables=1, lookup=[], mutate=[])
+    @fusion_utils.check_fusion()
+    def test_ignore_op(self, xp):
+        def impl(x, y):
+            z = x + y  # NOQA
+            return x
+        return impl
+
+    @check_number_of_ops(
+        loops=1, memories=4, variables=4, lookup=[2], mutate=[2])
+    @fusion_utils.check_fusion()
+    def test_returns_tuple(self, xp):
+        def impl(x, y):
+            return x + y, y - x
+        return impl
+
+    @check_number_of_ops(
+        loops=2, memories=4, variables=6, lookup=[1, 3], mutate=[1, 1])
+    @fusion_utils.check_fusion(
+        generate_inputs_name='generate_input_broadcast')
+    def test_different_shapes(self, xp):
+        def impl(x, y):
+            r1 = x + x
+            r2 = x + y
+            r3 = y + y
+            return r1 * r2 * r3
+        return impl
+
+    @check_number_of_ops(
+        loops=1, memories=2, variables=2, lookup=[2], mutate=[2])
+    @fusion_utils.check_fusion()
+    def test_inplace_elementwise_1(self, xp):
+        def impl(x, y):
+            x += y
+            y += x
+            x += y
+        return impl
+
+    @check_number_of_ops(
+        loops=1, memories=3, variables=3, lookup=[2], mutate=[2])
+    @fusion_utils.check_fusion()
+    def test_inplace_elementwise_2(self, xp):
+        def impl(x, y):
+            x += y
+            return x + x
+        return impl
+
+    @check_number_of_ops(
+        loops=1, memories=1, variables=1, lookup=[1], mutate=[1])
+    @fusion_utils.check_fusion(
+        generate_inputs_name='generate_input_same_memory')
+    def test_inplace_same_variable(self, xp):
+        def impl(x, y, z):
+            x += y
+            x += y
+        return impl
+
+    @check_number_of_ops(
+        loops=4, memories=3, variables=4,
+        lookup=[1, 2, 1, 2], mutate=[1, 1, 1, 1])
+    @fusion_utils.check_fusion(
+        generate_inputs_name='generate_input_same_memory')
+    def test_inplace_same_memory_space(self, xp):
+        def impl(x, y, z):
+            x += z
+            x += z
+        return impl
+
+    @check_number_of_ops(
+        loops=1, memories=2, variables=2, lookup=[1], mutate=[1])
+    @fusion_utils.check_fusion()
+    def test_one_reduction_op(self, xp):
+        return lambda x, y: xp.sum(x, axis=0)
+
+    @check_number_of_ops(
+        loops=1, memories=2, variables=3, lookup=[1], mutate=[1])
+    @fusion_utils.check_fusion()
+    def test_one_reduction_op_rotate(self, xp):
+        return lambda x, y: xp.sum(x, axis=1)
+
+    # TODO(asi1024): Fix parameters after optimization.
+    @check_number_of_ops(
+        loops=2, memories=4, variables=4, lookup=[2, 1], mutate=[1, 1])
+    @fusion_utils.check_fusion()
+    def test_one_fuse_reduction_premap(self, xp):
+        def impl(x, y):
+            premap = x + y
+            return xp.sum(premap, axis=0)
+        return impl
+
+    # TODO(asi1024): Add tests for reduction.
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_reduction.py b/tests/cupy_tests/core_tests/fusion_tests/test_reduction.py
new file mode 100644
index 0000000..0216f64
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_reduction.py
@@ -0,0 +1,242 @@
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+from cupy_tests.core_tests.fusion_tests import fusion_utils
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(1,), (3, 4), (2, 1, 4), (2, 0, 3)],
+    'axis': [-4, -3, -2, -1, 0, 1, 2, 3, 4],
+}))
+class TestFusionReductionAxis(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random(self.shape, xp, 'int64', scale=10, seed=0)
+        return (x,), {}
+
+    @fusion_utils.check_fusion(accept_error=numpy.AxisError)
+    def test_sum_axis(self, xp):
+        return lambda x: cupy.sum(x, self.axis)
+
+    @fusion_utils.check_fusion(accept_error=numpy.AxisError)
+    def test_sum_kwargs_axis(self, xp):
+        return lambda x: cupy.sum(x, axis=self.axis)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(1,), (3, 4), (2, 1, 4), (2, 0, 3), (2, 3, 2, 2, 3)],
+    'axis': [
+        None, (0,), (1,), (0, 1), (1, 2), (0, 2), (1, 3)
+        # TODO(asi1024): Fix core.simple_reduction_kernel to raise Error.
+        # (0, 0), (-1, 1)
+    ],
+}))
+class TestFusionReductionMultiAxis(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random(self.shape, xp, 'int64', scale=10, seed=0)
+        return (x,), {}
+
+    @fusion_utils.check_fusion(accept_error=(ValueError, numpy.AxisError))
+    def test_sum_axis(self, xp):
+        return lambda x: cupy.sum(x, self.axis)
+
+    @fusion_utils.check_fusion(accept_error=(ValueError, numpy.AxisError))
+    def test_sum_kwargs_axis(self, xp):
+        return lambda x: cupy.sum(x, axis=self.axis)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [
+        (120, 128, 144),
+        (119, 127, 143),
+        (128, 128, 128),
+        (32, 1024, 1024)
+    ],
+    'axis': [None, 0, 1, 2, (0, 1), (0, 2), (1, 2)],
+}))
+@testing.slow
+class TestFusionReductionLarge(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random(self.shape, xp, 'int64', scale=10, seed=0)
+        return (x,), {}
+
+    @fusion_utils.check_fusion()
+    def test_sum_kwargs_axis(self, xp):
+        return lambda x: cupy.sum(x, axis=self.axis)
+
+
+# TODO(asi1024): Support for bool and complex dtypes.
+class TestFusionReductionSpecifyDtype(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        return (x,), {}
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_bool=True, no_complex=True)
+    @fusion_utils.check_fusion(accept_error=TypeError)
+    def test_sum(self, xp, dtype1, dtype2):
+        return lambda x: x.sum(axis=0, dtype=dtype2)
+
+
+@testing.parameterize(*testing.product({
+    'axis': [None, 0, 1],
+}))
+class TestFusionReductionAndElementwise(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=1)
+        return (x, y), {}
+
+    @fusion_utils.check_fusion()
+    def test_premap_one_array(self, xp):
+        return lambda x, y: xp.sum(x * 3, self.axis)
+
+    @fusion_utils.check_fusion()
+    def test_premap_two_arrays(self, xp):
+        return lambda x, y: xp.sum(x + y, self.axis)
+
+    @fusion_utils.check_fusion()
+    def test_postmap_one_array(self, xp):
+        return lambda x, y: xp.sum(x, self.axis) + 3
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion(accept_error=ValueError)
+    def test_postmap_two_arrays(self, xp):
+        return lambda x, y: xp.sum(x, self.axis) + y
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion(accept_error=ValueError)
+    def test_premap_postmap(self, xp):
+        return lambda x, y: xp.sum(xp.sqrt(x) + y, self.axis) * 2 + y
+
+    # TODO(asi1024): Uncomment after replace fusion implementaiton.
+    # @fusion_utils.check_fusion()
+    # def test_premap_inplace(self, xp):
+    #     def impl(x, y):
+    #         x += 2
+    #         y += x
+    #         return xp.sum(y, self.axis)
+    #     return impl
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion(accept_error=ValueError)
+    def test_postmap_inplace(self, xp):
+        def impl(x, y):
+            y += x
+            res = xp.sum(x, self.axis)
+            y += res
+        return impl
+
+
+@testing.parameterize(*testing.product({
+    'axis1': [None, 0, 1],
+    'axis2': [None, 0, 1],
+}))
+class TestFusionMultipleReductions(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=1)
+        return (x, y), {}
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion()
+    def test_two_distinct_reductions(self, xp):
+        return lambda x, y: (x.sum(self.axis1), y.sum(self.axis2))
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion(accept_error=ValueError)
+    def test_two_reductions_and_elementwise(self, xp):
+        return lambda x, y: x.sum(self.axis1) + y.sum(self.axis2)
+
+
+class TestFusionMultistageReductions(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((3, 4, 5), xp, 'int64', scale=10, seed=0)
+        return (x,), {}
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion()
+    def test_multistage_reductions(self, xp):
+        return lambda x: x.prod(axis=1).sum(axis=1)
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion()
+    def test_multistage_reductions_and_elementwise(self, xp):
+        return lambda x: (xp.sqrt(x).prod(axis=0) + x).sum(axis=1) * 2
+
+
+class TestFusionMultistageReductionsMultiAxis(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((3, 4, 5, 6), xp, 'int64', scale=10, seed=0)
+        return (x,), {}
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion()
+    def test_multistage_reductions(self, xp):
+        return lambda x: x.prod(axis=(-1, 1)).sum(axis=(0, 1))
+
+
+class TestFusionReductionRoutines(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((30,), xp, 'int64', scale=10, seed=0)
+        return (x,), {}
+
+    @fusion_utils.check_fusion()
+    def test_sum(self, xp):
+        return lambda x: xp.sum(x)
+
+    @fusion_utils.check_fusion()
+    def test_prod(self, xp):
+        return lambda x: xp.prod(x)
+
+    @fusion_utils.check_fusion()
+    def test_nansum(self, xp):
+        return lambda x: xp.nansum(x)
+
+    @fusion_utils.check_fusion()
+    def test_nanprod(self, xp):
+        return lambda x: xp.nanprod(x)
+
+
+class TestFusionMisc(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=0)
+        return (x,), {}
+
+    @unittest.skipUnless(
+        fusion_utils.can_use_grid_synchronization(),
+        'Requires CUDA grid synchronization')
+    @fusion_utils.check_fusion()
+    def test_sum_div_clip(self, xp):
+        def impl(x):
+            x = x / xp.sum(x)
+            return xp.clip(x, 2, 7)
+        return impl
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_routines.py b/tests/cupy_tests/core_tests/fusion_tests/test_routines.py
new file mode 100644
index 0000000..3e93b34
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_routines.py
@@ -0,0 +1,412 @@
+import unittest
+
+import numpy
+import pytest
+
+from cupy import testing
+from cupy_tests.core_tests.fusion_tests import fusion_utils
+
+
+class FusionUnaryUfuncTestBase(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp, dtype, scale=10, seed=0)
+        return (x,), {}
+
+
+class FusionBinaryUfuncTestBase(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, dtype2, scale=10, seed=1)
+        return (x, y), {}
+
+
+@testing.parameterize(*testing.product({
+    'func': [
+        'bitwise_and', 'bitwise_or', 'bitwise_xor', 'left_shift', 'right_shift'
+    ]
+}))
+class TestFusionBitwiseBinary(FusionBinaryUfuncTestBase):
+
+    @testing.for_int_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_bitwise(self, xp, dtype1, dtype2):
+        def impl(x, y):
+            if ((x.dtype == 'uint64' and y.dtype.kind == 'i')
+                    or (y.dtype == 'uint64' and x.dtype.kind == 'i')):
+                # Skip TypeError case.
+                return
+            return getattr(xp, self.func)(x, y)
+        return impl
+
+
+class TestFusionBitwiseUnary(FusionUnaryUfuncTestBase):
+
+    @testing.for_int_dtypes()
+    @fusion_utils.check_fusion()
+    def test_invert(self, xp, dtype):
+        return lambda x: xp.invert(x)
+
+
+@testing.parameterize(*testing.product({
+    'func': [
+        'greater', 'greater_equal', 'less', 'less_equal', 'equal', 'not_equal',
+        'logical_and', 'logical_or', 'logical_xor',
+        'maximum', 'minimum', 'fmax', 'fmin',
+    ]
+}))
+class TestFusionComparisonBinary(FusionBinaryUfuncTestBase):
+
+    @testing.for_all_dtypes_combination(
+        no_complex=True, names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_comparison(self, xp, dtype1, dtype2):
+        return lambda x, y: getattr(xp, self.func)(x, y)
+
+
+class TestFusionComparisonUnary(FusionUnaryUfuncTestBase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_comparison(self, xp, dtype):
+        return lambda x: xp.logical_not(x)
+
+
+class TestFusionArrayContents(FusionUnaryUfuncTestBase):
+
+    def generate_inputs(self, xp, has_nan, dtype):
+        if numpy.dtype(dtype).kind not in ('f', 'c'):
+            return super(TestFusionArrayContents, self).generate_inputs(
+                xp, dtype)
+
+        nan = numpy.nan
+        inf = dtype(float('inf'))
+
+        if has_nan:
+            x = xp.array([-3, nan, -1, nan, 0, nan, inf], dtype=dtype)
+        else:
+            x = xp.array([-3, inf, -1, -inf, 0, 1, 2], dtype=dtype)
+        return (x,), {}
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion(generate_inputs_args=(False,))
+    def test_isfinite(self, xp, dtype):
+        return lambda x: xp.isfinite(x)
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion(generate_inputs_args=(False,))
+    def test_isinf(self, xp, dtype):
+        return lambda x: xp.isinf(x)
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion(generate_inputs_args=(True,))
+    def test_isnan(self, xp, dtype):
+        return lambda x: xp.isnan(x)
+
+
+@testing.parameterize(*testing.product({
+    'func': [
+        'sin', 'cos', 'tan', 'arcsin', 'arccos', 'arctan',
+        'sinh', 'cosh', 'tanh', 'arcsinh', 'arccosh', 'arctanh',
+    ],
+}))
+class TestFusionTrigonometricUnary(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype):
+        if numpy.dtype(dtype).kind not in ('f', 'c'):
+            x = xp.array([0, 1])
+        else:
+            x = testing.shaped_random((3, 4), xp, dtype, scale=1, seed=0)
+        return (x,), {}
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_trigonometric(self, xp, dtype):
+        def impl(x):
+            with numpy.errstate(divide='ignore', invalid='ignore'):
+                return getattr(xp, self.func)(x)
+        return impl
+
+
+@testing.parameterize(*testing.product({
+    'func': ['arctan2', 'hypot']
+}))
+class TestFusionTrigonometricBinary(FusionBinaryUfuncTestBase):
+
+    @testing.for_all_dtypes_combination(
+        no_complex=True, names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_trigonometric(self, xp, dtype1, dtype2):
+        return lambda x, y: getattr(xp, self.func)(x, y)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['deg2rad', 'rad2deg', 'degrees', 'radians']
+}))
+class TestFusionDegRad(FusionUnaryUfuncTestBase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_trigonometric(self, xp, dtype):
+        return lambda x: getattr(xp, self.func)(x)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['around', 'round', 'rint', 'floor', 'ceil', 'trunc', 'fix']
+}))
+class TestFusionRounding(FusionUnaryUfuncTestBase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_rounding(self, xp, dtype):
+        return lambda x: getattr(xp, self.func)(x)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['exp', 'expm1', 'exp2', 'log', 'log10', 'log2', 'log1p']
+}))
+class TestFusionExpLogUnary(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp, dtype, scale=10, seed=0) + 1
+        return (x,), {}
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_explog(self, xp, dtype):
+        return lambda x: getattr(xp, self.func)(x)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['logaddexp', 'logaddexp2']
+}))
+class TestFusionExpLogBinary(FusionBinaryUfuncTestBase):
+
+    @testing.for_all_dtypes_combination(
+        no_complex=True, names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_explog(self, xp, dtype1, dtype2):
+        return lambda x, y: getattr(xp, self.func)(x, y)
+
+
+class TestFusionLdexp(FusionBinaryUfuncTestBase):
+
+    @testing.for_float_dtypes(name='dtype1')
+    @testing.for_dtypes(['i', 'l'], name='dtype2')
+    @fusion_utils.check_fusion()
+    def test_explog(self, xp, dtype1, dtype2):
+        return lambda x, y: xp.ldexp(x, y)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['signbit', 'frexp']
+}))
+class TestFusionFloatingUnary(FusionUnaryUfuncTestBase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_floating_point_routine(self, xp, dtype):
+        return lambda x: getattr(xp, self.func)(x)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['copysign', 'nextafter']
+}))
+class TestFusionFloatingBinary(FusionBinaryUfuncTestBase):
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_floating_point_routine(self, xp, dtype1, dtype2):
+        return lambda x, y: getattr(xp, self.func)(x, y)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['reciprocal', 'negative', 'angle', 'conj', 'real', 'imag']
+}))
+class TestArithmeticUnary(FusionUnaryUfuncTestBase):
+
+    def generate_inputs(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp, dtype, scale=10, seed=0)
+        x[x == 0] = 1
+        return (x,), {}
+
+    @testing.for_all_dtypes(no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_arithmetic(self, xp, dtype):
+        return lambda x: getattr(xp, self.func)(x)
+
+
+class TestModf(FusionUnaryUfuncTestBase):
+
+    def generate_inputs(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp, dtype, scale=10, seed=0)
+        return (x,), {}
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_arithmetic(self, xp, dtype):
+        return lambda x: xp.modf(x)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['add', 'subtract', 'multiply', 'power']
+}))
+class TestArithmeticBinary(FusionBinaryUfuncTestBase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=5, seed=0)
+        y = testing.shaped_random((3, 4), xp, dtype2, scale=5, seed=0)
+        return (x, y), {}
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_complex=True, no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_arithmetic(self, xp, dtype1, dtype2):
+        # TODO(unno): boolean subtract causes DeprecationWarning in numpy>=1.13
+        return lambda x, y: getattr(xp, self.func)(x, y)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['divide', 'true_divide', 'floor_divide', 'fmod', 'remainder']
+}))
+class TestDivide(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, dtype2, scale=10, seed=1)
+        y[y == 0] = 1
+        return (x, y), {}
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_divide(self, xp, dtype1, dtype2):
+        return lambda x, y: getattr(xp, self.func)(x, y)
+
+
+class TestDivmod(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, dtype2, scale=10, seed=1)
+        y[y == 0] = 1
+        return (x, y), {}
+
+    @testing.with_requires('numpy>=1.13')
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_divmod(self, xp, dtype1, dtype2):
+        return lambda x, y: xp.divmod(x, y)
+
+
+class TestFusionMisc(FusionUnaryUfuncTestBase):
+
+    @testing.with_requires('numpy>=1.11.2')
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_sqrt(self, xp, dtype):
+        return lambda x: xp.sqrt(x)
+
+    @testing.with_requires('numpy>=1.10')
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_cbrt(self, xp, dtype):
+        return lambda x: xp.cbrt(x)
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_square(self, xp, dtype):
+        return lambda x: xp.square(x)
+
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_absolute(self, xp, dtype):
+        return lambda x: xp.absolute(x)
+
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_abs(self, xp, dtype):
+        return lambda x: xp.abs(x)
+
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @fusion_utils.check_fusion()
+    def test_sign(self, xp, dtype):
+        return lambda x: xp.sign(x)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_clip(self, xp, dtype):
+        return lambda x: xp.clip(x, dtype(2), dtype(4))
+
+
+@testing.parameterize(*testing.product({
+    'func': ['i0', 'sinc']
+}))
+class TestFusionSpecialMath(FusionUnaryUfuncTestBase):
+
+    # TODO(imanishi): Fix for integer tests
+    @testing.for_float_dtypes()
+    @fusion_utils.check_fusion()
+    def test_special_math(self, xp, dtype):
+        return lambda x: getattr(xp, self.func)(x)
+
+
+class TestFusionManipulation(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        cond = testing.shaped_random((3, 4), xp, 'bool_', seed=0)
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=1)
+        y = testing.shaped_random((3, 4), xp, dtype2, scale=10, seed=2)
+        return (cond, x, y), {}
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_where(self, xp, dtype1, dtype2):
+        return lambda cond, x, y: xp.where(cond, x, y)
+
+    # TODO(imanishi): Supoort complex dtypes
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_complex=True)
+    @fusion_utils.check_fusion(accept_error=(TypeError,))
+    def test_copyto(self, xp, dtype1, dtype2):
+        return lambda cond, x, y: xp.copyto(x, y)
+
+    # TODO(imanishi): Supoort complex dtypes
+    @pytest.mark.xfail(reason='Issue #5848')
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_complex=True)
+    @fusion_utils.check_fusion(accept_error=(TypeError,))
+    def test_copyto_compat_broadcast(self, xp, dtype1, dtype2):
+        return lambda cond, x, y: xp.copyto(x, y[None])
+
+    # TODO(imanishi): Supoort complex dtypes
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_complex=True)
+    @fusion_utils.check_fusion(accept_error=(TypeError,))
+    def test_copyto_where(self, xp, dtype1, dtype2):
+        return lambda cond, x, y: xp.where(x, y, where=cond)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['sum', 'prod', 'amax', 'amin', 'max', 'min']
+}))
+class TestFusionNumericalReduction(FusionUnaryUfuncTestBase):
+
+    @testing.for_all_dtypes()
+    @fusion_utils.check_fusion()
+    def test_reduction(self, xp, dtype):
+        return lambda x: getattr(xp, self.func)(x)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['all', 'any']
+}))
+class TestFusionLogicalReduction(FusionUnaryUfuncTestBase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @fusion_utils.check_fusion()
+    def test_reduction(self, xp, dtype):
+        return lambda x: getattr(xp, self.func)(x)
diff --git a/tests/cupy_tests/core_tests/fusion_tests/test_ufunc.py b/tests/cupy_tests/core_tests/fusion_tests/test_ufunc.py
new file mode 100644
index 0000000..49397ec
--- /dev/null
+++ b/tests/cupy_tests/core_tests/fusion_tests/test_ufunc.py
@@ -0,0 +1,316 @@
+import itertools
+import unittest
+
+from cupy import testing
+from cupy_tests.core_tests.fusion_tests import fusion_utils
+
+
+def _permutate_shapes(shapes_list):
+    # Permutates input shapes
+    permutated_shapes_set = set()
+    for shapes in shapes_list:
+        for permutated_shapes in itertools.permutations(shapes):
+            permutated_shapes_set.add(permutated_shapes)
+    return list(permutated_shapes_set)
+
+
+@testing.parameterize(*testing.product({
+    'shapes': _permutate_shapes([
+        # Same shapes
+        ((1,), (1,)),
+        ((3, 4), (3, 4)),
+
+        # Broadcast
+        ((10,), (1,)),
+        ((3, 4), (3, 1)),
+        ((3, 4), (1, 4)),
+        ((3, 4), (4,)),
+        ((3, 4), (1, 1)),
+        ((3, 4), (1,)),
+        ((2, 3, 4), (1, 1, 1)),
+        ((3, 1), (1, 4)),
+        ((2, 1, 4), (3, 1)),
+
+        # TODO(asi1024): Fix testing.shaped_random to support 0-dim array.
+        # # 0-dim shape
+        # ((), ()),
+        # ((1,), ()),
+        # ((3,), ()),
+        # ((2, 3), ()),
+
+        # 0-size shape
+        ((0,), (0,)),
+        ((0,), (1,)),
+        ((2, 0, 3), (2, 0, 3)),
+        ((2, 0, 3), (0, 1)),
+
+        # Large case
+        ((256, 256), (256,)),
+        ((256, 256), (256, 1)),
+    ])
+}))
+class TestFusionBroadcast(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        shape1, shape2 = self.shapes
+        x = testing.shaped_random(shape1, xp, 'int64', scale=10, seed=0)
+        y = testing.shaped_random(shape2, xp, 'int64', scale=10, seed=1)
+        return (x, y), {}
+
+    @fusion_utils.check_fusion()
+    def test_broadcast(self, xp):
+        return lambda x, y: x + y
+
+    # TODO(asi1024): Uncomment after replace fusion implementaiton.
+
+    # @fusion_utils.check_fusion(accept_error=ValueError)
+    # def test_broadcast_inplace(self, xp):
+    #     def impl(x, y):
+    #         x += y
+    #     return impl
+
+
+@testing.parameterize(*testing.product({
+    'shapes': _permutate_shapes([
+        ((2,), (3,)),
+        ((2,), (0,)),
+        ((3, 2), (3, 3)),
+        ((3, 2), (2, 2)),
+        ((3,), (1, 2)),
+    ])
+}))
+class TestFusionBroadcastInvalid(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        shape1, shape2 = self.shapes
+        x = testing.shaped_random(shape1, xp, 'int64', scale=10, seed=0)
+        y = testing.shaped_random(shape2, xp, 'int64', scale=10, seed=1)
+        return (x, y), {}
+
+    @fusion_utils.check_fusion(accept_error=ValueError)
+    def test_broadcast(self, xp):
+        return lambda x, y: x + y
+
+    @fusion_utils.check_fusion(accept_error=ValueError)
+    def test_broadcast_inplace(self, xp):
+        def impl(x, y):
+            x += y
+        return impl
+
+
+class TestFusionParseInput(unittest.TestCase):
+
+    def generate_inputs(self, xp):
+        x = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=0)
+        return (x,), {}
+
+    @fusion_utils.check_fusion()
+    def test_add(self, xp):
+        return lambda x: x + x
+
+    # TODO(asi1024): Should fix cupy.ufunc
+    @fusion_utils.check_fusion(accept_error=(ValueError, TypeError))
+    def test_add_too_less_param(self, xp):
+        return lambda x: xp.add(x)
+
+    # TODO(asi1024): Should fix cupy.ufunc
+    @fusion_utils.check_fusion(accept_error=(ValueError, TypeError))
+    def test_add_too_much_param(self, xp):
+        return lambda x: xp.add(x, x, x, x)
+
+    @fusion_utils.check_fusion(accept_error=TypeError)
+    def test_add_none(self, xp):
+        return lambda x: x + None
+
+    @fusion_utils.check_fusion(accept_error=TypeError)
+    def test_add_object(self, xp):
+        return lambda x: x + object()
+
+    # TODO(asi1024): Should fix cupy.ufunc
+    # @fusion_utils.check_fusion()
+    # def test_add_out_none(self, xp):
+    #     def impl(x):
+    #         xp.add(x, x, None)
+    #         return x
+    #     return impl
+
+    @fusion_utils.check_fusion()
+    def test_add_kwargs_out_none(self, xp):
+        def impl(x):
+            xp.add(x, x, out=None)
+        return impl
+
+    # TODO(asi1024): Should fix cupy.ufunc
+    # @fusion_utils.check_fusion(accept_error=ValueError)
+    # def test_add_both_out_none(self, xp):
+    #     def impl(x):
+    #         xp.add(x, x, None, out=None)
+    #     return impl
+
+    @fusion_utils.check_fusion(accept_error=TypeError)
+    def test_add_out_object(self, xp):
+        def impl(x):
+            xp.add(x, x, object())
+            return x
+        return impl
+
+    @fusion_utils.check_fusion(accept_error=TypeError)
+    def test_add_kwargs_out_object(self, xp):
+        def impl(x):
+            xp.add(x, x, out=object())
+            return x
+        return impl
+
+    @fusion_utils.check_fusion()
+    def test_divmod(self, xp):
+        return lambda x: xp.divmod(x, x)
+
+
+class TestFusionOutDtype(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        x = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        y = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=1)
+        z = testing.shaped_random((3, 4), xp, dtype2, scale=10, seed=2)
+        return (x, y, z), {}
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), full=True, no_complex=True)
+    @fusion_utils.check_fusion(accept_error=TypeError)
+    @testing.with_requires('numpy>=1.13')
+    def test_outarg(self, xp, dtype1, dtype2):
+        def impl(x, y, z):
+            xp.add(x, y, out=z)
+            return z
+        return impl
+
+
+class TestFusionScalar(unittest.TestCase):
+
+    def generate_inputs(self, xp, dtype1, dtype2):
+        array = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        return (array,), {}
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_python_scalar_r(self, xp, dtype1, dtype2):
+        def func(array):
+            py_scalar = dtype2(1).item()
+            return array + py_scalar
+
+        return func
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_numpy_scalar_r(self, xp, dtype1, dtype2):
+        def func(array):
+            np_scalar = dtype2(1)
+            return array + np_scalar
+
+        return func
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_python_scalar_l(self, xp, dtype1, dtype2):
+        def func(array):
+            py_scalar = dtype2(1).item()
+            return py_scalar + array
+
+        return func
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion()
+    def test_numpy_scalar_l(self, xp, dtype1, dtype2):
+        def func(array):
+            np_scalar = dtype2(1)
+            return np_scalar + array
+
+        return func
+
+    def python_scalar_param_r(self, xp, dtype1, dtype2):
+        array = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        py_scalar = dtype2(1).item()
+        return (array, py_scalar), {}
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion(generate_inputs_name='python_scalar_param_r')
+    def test_python_scalar_param_r(self, xp, dtype1, dtype2):
+        def func(array, py_scalar):
+            return array + py_scalar
+
+        return func
+
+    def python_scalar_param_l(self, xp, dtype1, dtype2):
+        array = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        py_scalar = dtype2(1).item()
+        return (py_scalar, array), {}
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion(generate_inputs_name='python_scalar_param_l')
+    def test_python_scalar_param_l(self, xp, dtype1, dtype2):
+        def func(py_scalar, array):
+            return py_scalar + array
+
+        return func
+
+    def numpy_scalar_param_r(self, xp, dtype1, dtype2):
+        array = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        np_scalar = dtype2(1)
+        return (array, np_scalar), {}
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion(generate_inputs_name='numpy_scalar_param_r')
+    def test_numpy_scalar_param_r(self, xp, dtype1, dtype2):
+        def func(array, np_scalar):
+            return array + np_scalar
+
+        return func
+
+    def numpy_scalar_param_l(self, xp, dtype1, dtype2):
+        array = testing.shaped_random((3, 4), xp, dtype1, scale=10, seed=0)
+        np_scalar = dtype2(1)
+        return (np_scalar, array), {}
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion(generate_inputs_name='numpy_scalar_param_l')
+    def test_numpy_scalar_param_l(self, xp, dtype1, dtype2):
+        def func(np_scalar, array):
+            return np_scalar + array
+
+        return func
+
+    def numpy_scalar_params_binop(self, xp, dtype1, dtype2):
+        scalar1 = dtype1(1)
+        scalar2 = dtype2(1)
+        array = testing.shaped_random((3, 4), xp, 'int64', scale=10, seed=0)
+        return (scalar1, scalar2, array), {}
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion(
+        generate_inputs_name='numpy_scalar_params_binop')
+    def test_numpy_scalar_params_binop(self, xp, dtype1, dtype2):
+        def func(scalar1, scalar2, array):
+            dtype = (scalar1 + scalar2).dtype
+            return array.astype(dtype)
+
+        return func
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion(
+        generate_inputs_name='numpy_scalar_params_binop')
+    def test_scalar_inplace_update(self, xp, dtype1, dtype2):
+        def func(scalar1, scalar2, array):
+            scalar1_copy = scalar1
+            scalar1 += scalar2
+            return array + scalar1 + scalar1_copy
+
+        return func
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @fusion_utils.check_fusion(generate_inputs_name='numpy_scalar_param_r')
+    def test_scalar_inplace_update_with_array(self, xp, dtype1, dtype2):
+        def func(array, scalar):
+            scalar += array
+            return scalar
+
+        return func
diff --git a/tests/cupy_tests/core_tests/test_array_function.py b/tests/cupy_tests/core_tests/test_array_function.py
new file mode 100644
index 0000000..b8180a5
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_array_function.py
@@ -0,0 +1,58 @@
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+class TestArrayFunction(unittest.TestCase):
+
+    @testing.with_requires('numpy>=1.17.0')
+    def test_array_function(self):
+        a = numpy.random.randn(100, 100)
+        a_cpu = numpy.asarray(a)
+        a_gpu = cupy.asarray(a)
+
+        # The numpy call for both CPU and GPU arrays is intentional to test the
+        # __array_function__ protocol
+        qr_cpu = numpy.linalg.qr(a_cpu)
+        qr_gpu = numpy.linalg.qr(a_gpu)
+
+        if isinstance(qr_cpu, tuple):
+            for b_cpu, b_gpu in zip(qr_cpu, qr_gpu):
+                assert b_cpu.dtype == b_gpu.dtype
+                cupy.testing.assert_allclose(b_cpu, b_gpu, atol=1e-4)
+        else:
+            assert qr_cpu.dtype == qr_gpu.dtype
+            cupy.testing.assert_allclose(qr_cpu, qr_gpu, atol=1e-4)
+
+    @testing.with_requires('numpy>=1.17.0')
+    def test_array_function2(self):
+        a = numpy.random.randn(100, 100)
+        a_cpu = numpy.asarray(a)
+        a_gpu = cupy.asarray(a)
+
+        # The numpy call for both CPU and GPU arrays is intentional to test the
+        # __array_function__ protocol
+        out_cpu = numpy.sum(a_cpu, axis=1)
+        out_gpu = numpy.sum(a_gpu, axis=1)
+
+        assert out_cpu.dtype == out_gpu.dtype
+        cupy.testing.assert_allclose(out_cpu, out_gpu, atol=1e-4)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.numpy_cupy_equal()
+    def test_array_function_can_cast(self, xp):
+        return numpy.can_cast(xp.arange(2), 'f4')
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.numpy_cupy_equal()
+    def test_array_function_common_type(self, xp):
+        return numpy.common_type(xp.arange(2, dtype='f8'),
+                                 xp.arange(2, dtype='f4'))
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.numpy_cupy_equal()
+    def test_array_function_result_type(self, xp):
+        return numpy.result_type(3, xp.arange(2, dtype='f8'))
diff --git a/tests/cupy_tests/core_tests/test_carray.py b/tests/cupy_tests/core_tests/test_carray.py
new file mode 100644
index 0000000..f7384c9
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_carray.py
@@ -0,0 +1,85 @@
+import unittest
+
+import cupy
+from cupy import testing
+
+
+class TestCArray(unittest.TestCase):
+
+    def test_size(self):
+        x = cupy.arange(3).astype('i')
+        y = cupy.ElementwiseKernel(
+            'raw int32 x', 'int32 y', 'y = x.size()', 'test_carray_size',
+        )(x, size=1)
+        assert int(y[0]) == 3
+
+    def test_shape(self):
+        x = cupy.arange(6).reshape((2, 3)).astype('i')
+        y = cupy.ElementwiseKernel(
+            'raw int32 x', 'int32 y', 'y = x.shape()[i]', 'test_carray_shape',
+        )(x, size=2)
+        testing.assert_array_equal(y, (2, 3))
+
+    def test_strides(self):
+        x = cupy.arange(6).reshape((2, 3)).astype('i')
+        y = cupy.ElementwiseKernel(
+            'raw int32 x', 'int32 y', 'y = x.strides()[i]',
+            'test_carray_strides',
+        )(x, size=2)
+        testing.assert_array_equal(y, (12, 4))
+
+    def test_getitem_int(self):
+        x = cupy.arange(24).reshape((2, 3, 4)).astype('i')
+        y = cupy.empty_like(x)
+        y = cupy.ElementwiseKernel(
+            'raw T x', 'int32 y', 'y = x[i]', 'test_carray_getitem_int',
+        )(x, y)
+        testing.assert_array_equal(y, x)
+
+    def test_getitem_idx(self):
+        x = cupy.arange(24).reshape((2, 3, 4)).astype('i')
+        y = cupy.empty_like(x)
+        y = cupy.ElementwiseKernel(
+            'raw T x', 'int32 y',
+            'ptrdiff_t idx[] = {i / 12, i / 4 % 3, i % 4}; y = x[idx]',
+            'test_carray_getitem_idx',
+        )(x, y)
+        testing.assert_array_equal(y, x)
+
+
+@testing.parameterize(
+    {'size': 2 ** 31 - 1024},
+    {'size': 2 ** 31},
+    {'size': 2 ** 31 + 1024},
+    {'size': 2 ** 32 - 1024},
+    {'size': 2 ** 32},
+    {'size': 2 ** 32 + 1024},
+)
+@testing.slow
+class TestCArray32BitBoundary(unittest.TestCase):
+    # This test case is intended to confirm CArray indexing work correctly
+    # with input/output arrays whose size is so large that it crosses the
+    # 32-bit boundary (in terms of both number of elements and size in bytes).
+    # This test requires approx. 8 GiB GPU memory to run.
+    # See https://github.com/cupy/cupy/pull/882 for detailed discussions.
+
+    def tearDown(self):
+        # Free huge memory for slow test
+        cupy.get_default_memory_pool().free_all_blocks()
+
+    # HIP is known to fail with sizes > 2**32-1024
+    @unittest.skipIf(cupy.cuda.runtime.is_hip, 'HIP does not support this')
+    def test(self):
+        # Elementwise
+        a = cupy.full((1, self.size), 7, dtype=cupy.int8)
+        # Reduction
+        result = a.sum(axis=0, dtype=cupy.int8)
+        # Explicitly specify the dtype to absorb Linux/Windows difference.
+        assert result.sum(dtype=cupy.int64) == self.size * 7
+
+    # HIP is known to fail with sizes > 2**32-1024
+    @unittest.skipIf(cupy.cuda.runtime.is_hip, 'HIP does not support this')
+    def test_assign(self):
+        a = cupy.zeros(self.size, dtype=cupy.int8)
+        a[-1] = 1.0
+        assert a.sum() == 1
diff --git a/tests/cupy_tests/core_tests/test_core.py b/tests/cupy_tests/core_tests/test_core.py
new file mode 100644
index 0000000..344f8e3
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_core.py
@@ -0,0 +1,120 @@
+import unittest
+import sys
+
+import numpy
+import pytest
+
+import cupy
+from cupy._core import core
+from cupy import testing
+from cupy_tests.core_tests import test_raw
+
+
+class TestSize(unittest.TestCase):
+
+    def tearDown(self):
+        # Free huge memory for slow test
+        cupy.get_default_memory_pool().free_all_blocks()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_size(self, xp, dtype):
+        a = xp.ndarray((2, 3), dtype=dtype)
+        return xp.size(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_size_axis(self, xp, dtype):
+        a = xp.ndarray((2, 3), dtype=dtype)
+        return xp.size(a, axis=1)
+
+    @testing.for_all_dtypes()
+    def test_size_axis_error(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.ndarray((2, 3), dtype=dtype)
+            with pytest.raises(IndexError):
+                return xp.size(a, axis=3)
+
+    @testing.numpy_cupy_equal()
+    @testing.slow
+    def test_size_huge(self, xp):
+        a = xp.ndarray(2 ** 32, 'b')  # 4 GiB
+        return xp.size(a)
+
+
+_orders = {
+    order_arg: order_expect
+    for order_expect, order_args in [
+        ('C', ['C', 'c', 'CONTIGUOUS', '', None]),
+        ('F', ['F', 'f', 'FORTRAN']),
+    ]
+    for order_arg in order_args
+}
+
+
+class TestOrder(unittest.TestCase):
+
+    @testing.for_orders(_orders.keys())
+    def test_ndarray(self, order):
+        order_expect = _orders[order]
+        a = core.ndarray((2, 3), order=order)
+        expect_c = order_expect == 'C'
+        expect_f = order_expect == 'F'
+        assert a.flags.c_contiguous == expect_c
+        assert a.flags.f_contiguous == expect_f
+
+
+class TestMinScalarType:
+    def test_scalar(self):
+        for v in (-129, -128, 0, 1.2, numpy.inf):
+            assert cupy.min_scalar_type(v) is numpy.min_scalar_type(v)
+
+    @testing.for_all_dtypes()
+    def test_numpy_scalar(self, dtype):
+        sc = dtype(1)
+        for v in (sc, [sc, sc]):
+            assert cupy.min_scalar_type(v) is numpy.min_scalar_type(v)
+
+    @testing.for_all_dtypes()
+    def test_cupy_scalar(self, dtype):
+        sc = cupy.array(-1).astype(dtype)
+        for v in (sc, [sc, sc]):
+            assert cupy.min_scalar_type(v) is sc.dtype
+
+    @testing.for_all_dtypes()
+    def test_numpy_ndarray(self, dtype):
+        arr = numpy.array([[-1, 1]]).astype(dtype)
+        for v in (arr, (arr, arr)):
+            assert cupy.min_scalar_type(v) is numpy.min_scalar_type(v)
+
+    @testing.for_all_dtypes()
+    def test_cupy_ndarray(self, dtype):
+        arr = cupy.array([[-1, 1]]).astype(dtype)
+        for v in (arr, (arr, arr)):
+            assert cupy.min_scalar_type(v) is arr.dtype
+
+
+@testing.parameterize(*testing.product({
+    'cxx': (None, '--std=c++11'),
+}))
+class TestCuPyHeaders(unittest.TestCase):
+
+    def setUp(self):
+        self.temporary_cache_dir_context = test_raw.use_temporary_cache_dir()
+        self.cache_dir = self.temporary_cache_dir_context.__enter__()
+        self.header = '\n'.join(['#include <' + h + '>'
+                                 for h in core._cupy_header_list])
+
+    def tearDown(self):
+        self.temporary_cache_dir_context.__exit__(*sys.exc_info())
+
+    def test_compiling_core_header(self):
+        code = r'''
+        extern "C" __global__ void _test_ker_() { }
+        '''
+        code = self.header + code
+        options = () if self.cxx is None else (self.cxx,)
+        ker = cupy.RawKernel(code, '_test_ker_',
+                             options=options, backend='nvrtc')
+        ker((1,), (1,), ())
+        cupy.cuda.Device().synchronize()
diff --git a/tests/cupy_tests/core_tests/test_cub_reduction.py b/tests/cupy_tests/core_tests/test_cub_reduction.py
new file mode 100644
index 0000000..1e02071
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_cub_reduction.py
@@ -0,0 +1,148 @@
+from itertools import combinations
+import unittest
+import sys
+
+import cupy
+from cupy import _environment
+from cupy import testing
+from cupy._core import _accelerator
+from cupy._core import _cub_reduction
+from cupy.cuda import memory
+
+
+# This test class and its children below only test if CUB backend can be used
+# or not; they don't verify its correctness as it's already extensively covered
+# by existing tests
+@unittest.skipIf(_environment.get_cub_path() is None, 'CUB not found')
+class CubReductionTestBase(unittest.TestCase):
+    """
+    Note: call self.can_use() when arrays are already allocated, otherwise
+    call self._test_can_use().
+    """
+
+    def setUp(self):
+        if cupy.cuda.runtime.is_hip:
+            if _environment.get_hipcc_path() is None:
+                self.skipTest('hipcc is not found')
+
+        self.can_use = cupy._core._cub_reduction._can_use_cub_block_reduction
+
+        self.old_accelerators = _accelerator.get_reduction_accelerators()
+        _accelerator.set_reduction_accelerators(['cub'])
+
+    def tearDown(self):
+        _accelerator.set_reduction_accelerators(self.old_accelerators)
+
+    def _test_can_use(
+            self, i_shape, o_shape, r_axis, o_axis, order, expected):
+        in_args = [cupy.testing.shaped_arange(i_shape, order=order), ]
+        out_args = [cupy.testing.shaped_arange(o_shape, order=order), ]
+        result = self.can_use(in_args, out_args, r_axis, o_axis) is not None
+        assert result is expected
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(2,), (2, 3), (2, 3, 4), (2, 3, 4, 5)],
+    'order': ('C', 'F'),
+}))
+class TestSimpleCubReductionKernelContiguity(CubReductionTestBase):
+
+    @testing.for_contiguous_axes()
+    def test_can_use_cub_contiguous(self, axis):
+        r_axis = axis
+        i_shape = self.shape
+        o_axis = tuple(i for i in range(len(i_shape)) if i not in r_axis)
+        o_shape = tuple(self.shape[i] for i in o_axis)
+        self._test_can_use(i_shape, o_shape, r_axis, o_axis, self.order, True)
+
+    @testing.for_contiguous_axes()
+    def test_can_use_cub_non_contiguous(self, axis):
+        # array is contiguous, but reduce_axis is not
+        dim = len(self.shape)
+        r_dim = len(axis)
+        non_contiguous_axes = [i for i in combinations(range(dim), r_dim)
+                               if i != axis]
+
+        i_shape = self.shape
+        for r_axis in non_contiguous_axes:
+            o_axis = tuple(i for i in range(dim) if i not in r_axis)
+            o_shape = tuple(self.shape[i] for i in o_axis)
+            self._test_can_use(i_shape, o_shape, r_axis, o_axis,
+                               self.order, False)
+
+
+class TestSimpleCubReductionKernelMisc(CubReductionTestBase):
+
+    def test_can_use_cub_nonsense_input1(self):
+        # two inputs are not allowed
+        a = cupy.random.random((2, 3, 4))
+        b = cupy.random.random((2, 3, 4))
+        c = cupy.empty((2, 3, ))
+        assert self.can_use([a, b], [c], (2,), (0, 1)) is None
+
+    def test_can_use_cub_nonsense_input2(self):
+        # reduce_axis and out_axis do not add up to full axis set
+        self._test_can_use((2, 3, 4), (2, 3), (2,), (0,), 'C', False)
+
+    def test_can_use_cub_nonsense_input3(self):
+        # array is neither C- nor F- contiguous
+        a = cupy.random.random((3, 4, 5))
+        a = a[:, 0:-1:2, 0:-1:3]
+        assert not a.flags.forc
+        b = cupy.empty((3,))
+        assert self.can_use([a], [b], (1, 2), (0,)) is None
+
+    def test_can_use_cub_zero_size_input(self):
+        self._test_can_use((2, 0, 3), (), (0, 1, 2), (), 'C', False)
+
+    # We actually just wanna test shapes, no need to allocate large memory.
+    def test_can_use_cub_oversize_input1(self):
+        # full reduction with array size > 64 GB
+        mem = memory.alloc(100)
+        a = cupy.ndarray((2**6 * 1024**3 + 1,), dtype=cupy.int8, memptr=mem)
+        b = cupy.empty((), dtype=cupy.int8)
+        assert self.can_use([a], [b], (0,), ()) is None
+
+    def test_can_use_cub_oversize_input2(self):
+        # full reduction with array size = 64 GB should work!
+        mem = memory.alloc(100)
+        a = cupy.ndarray((2**6 * 1024**3,), dtype=cupy.int8, memptr=mem)
+        b = cupy.empty((), dtype=cupy.int8)
+        assert self.can_use([a], [b], (0,), ()) is not None
+
+    def test_can_use_cub_oversize_input3(self):
+        # full reduction with 2^63-1 elements
+        mem = memory.alloc(100)
+        max_num = sys.maxsize
+        a = cupy.ndarray((max_num,), dtype=cupy.int8, memptr=mem)
+        b = cupy.empty((), dtype=cupy.int8)
+        assert self.can_use([a], [b], (0,), ()) is None
+
+    def test_can_use_cub_oversize_input4(self):
+        # partial reduction with too many (2^31) blocks
+        mem = memory.alloc(100)
+        a = cupy.ndarray((2**31, 8), dtype=cupy.int8, memptr=mem)
+        b = cupy.empty((), dtype=cupy.int8)
+        assert self.can_use([a], [b], (1,), (0,)) is None
+
+    def test_can_use_accelerator_set_unset(self):
+        # ensure we use CUB block reduction and not CUB device reduction
+        old_routine_accelerators = _accelerator.get_routine_accelerators()
+        _accelerator.set_routine_accelerators([])
+
+        a = cupy.random.random((10, 10))
+        # this is the only function we can mock; the rest is cdef'd
+        func_name = ''.join(('cupy._core._cub_reduction.',
+                             '_SimpleCubReductionKernel_get_cached_function'))
+        func = _cub_reduction._SimpleCubReductionKernel_get_cached_function
+        with testing.AssertFunctionIsCalled(
+                func_name, wraps=func, times_called=2):  # two passes
+            a.sum()
+        with testing.AssertFunctionIsCalled(
+                func_name, wraps=func, times_called=1):  # one pass
+            a.sum(axis=1)
+        with testing.AssertFunctionIsCalled(
+                func_name, wraps=func, times_called=0):  # not used
+            a.sum(axis=0)
+
+        _accelerator.set_routine_accelerators(old_routine_accelerators)
diff --git a/tests/cupy_tests/core_tests/test_dlpack.py b/tests/cupy_tests/core_tests/test_dlpack.py
new file mode 100644
index 0000000..3116506
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_dlpack.py
@@ -0,0 +1,150 @@
+import unittest
+
+import pytest
+
+import cupy
+from cupy import cuda
+from cupy import testing
+
+
+def _gen_array(dtype):
+    if cupy.issubdtype(dtype, cupy.unsignedinteger):
+        array = cupy.random.randint(
+            0, 10, size=(2, 3)).astype(dtype)
+    elif cupy.issubdtype(dtype, cupy.integer):
+        array = cupy.random.randint(
+            -10, 10, size=(2, 3)).astype(dtype)
+    elif cupy.issubdtype(dtype, cupy.floating):
+        array = cupy.random.rand(
+            2, 3).astype(dtype)
+    elif cupy.issubdtype(dtype, cupy.complexfloating):
+        array = cupy.random.random((2, 3)).astype(dtype)
+    elif dtype == cupy.bool_:
+        array = cupy.random.randint(0, 2, size=(2, 3)).astype(cupy.bool_)
+    else:
+        assert False, f'unrecognized dtype: {dtype}'
+    return array
+
+
+class TestDLPackConversion(unittest.TestCase):
+
+    @pytest.mark.filterwarnings('ignore::DeprecationWarning')
+    @testing.for_all_dtypes(no_bool=False)
+    def test_conversion(self, dtype):
+        orig_array = _gen_array(dtype)
+        tensor = orig_array.toDlpack()
+        out_array = cupy.fromDlpack(tensor)
+        testing.assert_array_equal(orig_array, out_array)
+        testing.assert_array_equal(orig_array.data.ptr, out_array.data.ptr)
+
+
+@testing.parameterize(*testing.product({
+    'memory': ('device', 'managed')
+}))
+class TestNewDLPackConversion(unittest.TestCase):
+
+    old_pool = None
+    new_pool = None
+
+    def setUp(self):
+        if self.memory == 'managed':
+            if cuda.runtime.is_hip:
+                pytest.skip('HIP does not support managed memory')
+            self.old_pool = cupy.get_default_memory_pool()
+            self.new_pool = cuda.MemoryPool(cuda.malloc_managed)
+            cuda.set_allocator(self.new_pool.malloc)
+
+    def tearDown(self):
+        if self.old_pool is not None:
+            cuda.set_allocator(self.old_pool.malloc)
+
+    def _get_stream(self, stream_name):
+        if stream_name == 'null':
+            return cuda.Stream.null
+        elif stream_name == 'ptds':
+            return cuda.Stream.ptds
+        else:
+            return cuda.Stream()
+
+    @testing.for_all_dtypes(no_bool=False)
+    def test_conversion(self, dtype):
+        orig_array = _gen_array(dtype)
+        out_array = cupy.from_dlpack(orig_array)
+        testing.assert_array_equal(orig_array, out_array)
+        testing.assert_array_equal(
+            orig_array.data.ptr, out_array.data.ptr)
+
+    def test_stream(self):
+        allowed_streams = ['null', True]
+        if not cuda.runtime.is_hip:
+            allowed_streams.append('ptds')
+
+        # stream order is automatically established via DLPack protocol
+        for src_s in [self._get_stream(s) for s in allowed_streams]:
+            for dst_s in [self._get_stream(s) for s in allowed_streams]:
+                with src_s:
+                    orig_array = _gen_array(cupy.float32)
+                    # If src_s != dst_s, dst_s waits until src_s complete.
+                    # Null stream (0) must be passed as streamLegacy (1)
+                    # on CUDA.
+                    if not cuda.runtime.is_hip and dst_s.ptr == 0:
+                        s_ptr = 1
+                    else:
+                        s_ptr = dst_s.ptr
+                    dltensor = orig_array.__dlpack__(s_ptr)
+
+                with dst_s:
+                    out_array = cupy.from_dlpack(dltensor)
+                    testing.assert_array_equal(orig_array, out_array)
+                    testing.assert_array_equal(
+                        orig_array.data.ptr, out_array.data.ptr)
+
+
+class TestDLTensorMemory(unittest.TestCase):
+
+    def setUp(self):
+        self.old_pool = cupy.get_default_memory_pool()
+        self.pool = cupy.cuda.MemoryPool()
+        cupy.cuda.set_allocator(self.pool.malloc)
+
+    def tearDown(self):
+        self.pool.free_all_blocks()
+        cupy.cuda.set_allocator(self.old_pool.malloc)
+
+    def test_deleter(self):
+        # memory is freed when tensor is deleted, as it's not consumed
+        array = cupy.empty(10)
+        tensor = array.toDlpack()
+        # str(tensor): <capsule object "dltensor" at 0x7f7c4c835330>
+        assert "\"dltensor\"" in str(tensor)
+        assert self.pool.n_free_blocks() == 0
+        del array
+        assert self.pool.n_free_blocks() == 0
+        del tensor
+        assert self.pool.n_free_blocks() == 1
+
+    @pytest.mark.filterwarnings('ignore::DeprecationWarning')
+    def test_deleter2(self):
+        # memory is freed when array2 is deleted, as tensor is consumed
+        array = cupy.empty(10)
+        tensor = array.toDlpack()
+        assert "\"dltensor\"" in str(tensor)
+        array2 = cupy.fromDlpack(tensor)
+        assert "\"used_dltensor\"" in str(tensor)
+        assert self.pool.n_free_blocks() == 0
+        del array
+        assert self.pool.n_free_blocks() == 0
+        del array2
+        assert self.pool.n_free_blocks() == 1
+        del tensor
+        assert self.pool.n_free_blocks() == 1
+
+    @pytest.mark.filterwarnings('ignore::DeprecationWarning')
+    def test_multiple_consumption_error(self):
+        # Prevent segfault, see #3611
+        array = cupy.empty(10)
+        tensor = array.toDlpack()
+        array2 = cupy.fromDlpack(tensor)  # noqa
+        with pytest.raises(ValueError) as e:
+            array3 = cupy.fromDlpack(tensor)  # noqa
+        assert 'consumed multiple times' in str(e.value)
diff --git a/tests/cupy_tests/core_tests/test_elementwise.py b/tests/cupy_tests/core_tests/test_elementwise.py
new file mode 100644
index 0000000..5561373
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_elementwise.py
@@ -0,0 +1,145 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import _core
+from cupy import cuda
+from cupy import testing
+
+
+class TestElementwise(unittest.TestCase):
+
+    def check_copy(self, dtype, src_id, dst_id):
+        with cuda.Device(src_id):
+            src = testing.shaped_arange((2, 3, 4), dtype=dtype)
+        with cuda.Device(dst_id):
+            dst = cupy.empty((2, 3, 4), dtype=dtype)
+        _core.elementwise_copy(src, dst)
+        testing.assert_allclose(src, dst)
+
+    @testing.for_all_dtypes()
+    def test_copy(self, dtype):
+        device_id = cuda.Device().id
+        self.check_copy(dtype, device_id, device_id)
+
+    @testing.multi_gpu(2)
+    @testing.for_all_dtypes()
+    def test_copy_multigpu_nopeer(self, dtype):
+        if cuda.runtime.deviceCanAccessPeer(0, 1) == 1:
+            pytest.skip('peer access is available')
+        with self.assertRaises(ValueError):
+            self.check_copy(dtype, 0, 1)
+
+    @testing.multi_gpu(2)
+    @testing.for_all_dtypes()
+    def test_copy_multigpu_peer(self, dtype):
+        if cuda.runtime.deviceCanAccessPeer(0, 1) != 1:
+            pytest.skip('peer access is unavailable')
+        with pytest.warns(cupy._util.PerformanceWarning):
+            self.check_copy(dtype, 0, 1)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_copy_zero_sized_array1(self, xp, dtype, order):
+        src = xp.empty((0,), dtype=dtype)
+        res = xp.copy(src, order=order)
+        assert src is not res
+        return res
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_copy_zero_sized_array2(self, xp, dtype, order):
+        src = xp.empty((1, 0, 2), dtype=dtype)
+        res = xp.copy(src, order=order)
+        assert src is not res
+        return res
+
+    @testing.for_orders('CFAK')
+    def test_copy_orders(self, order):
+        a = cupy.empty((2, 3, 4))
+        b = cupy.copy(a, order)
+
+        a_cpu = numpy.empty((2, 3, 4))
+        b_cpu = numpy.copy(a_cpu, order)
+
+        assert b.strides == b_cpu.strides
+
+
+class TestElementwiseInvalidShape(unittest.TestCase):
+
+    def test_invalid_shape(self):
+        with self.assertRaisesRegex(ValueError, 'Out shape is mismatched'):
+            f = cupy.ElementwiseKernel('T x', 'T y', 'y += x')
+            x = cupy.arange(12).reshape(3, 4)
+            y = cupy.arange(4)
+            f(x, y)
+
+
+class TestElementwiseInvalidArgument(unittest.TestCase):
+
+    def test_invalid_kernel_name(self):
+        with self.assertRaisesRegex(ValueError, 'Invalid kernel name'):
+            cupy.ElementwiseKernel('T x', '', '', '1')
+
+
+class TestElementwiseType(unittest.TestCase):
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_large_int_upper_1(self, xp, dtype):
+        a = xp.array([0], dtype=xp.int8)
+        b = xp.iinfo(dtype).max
+        return a + b
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_large_int_upper_2(self, xp, dtype):
+        a = xp.array([1], dtype=xp.int8)
+        b = xp.iinfo(dtype).max - 1
+        return a + b
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_large_int_upper_3(self, xp, dtype):
+        a = xp.array([xp.iinfo(dtype).max], dtype=dtype)
+        b = xp.int8(0)
+        return a + b
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_large_int_upper_4(self, xp, dtype):
+        a = xp.array([xp.iinfo(dtype).max - 1], dtype=dtype)
+        b = xp.int8(1)
+        return a + b
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_large_int_lower_1(self, xp, dtype):
+        a = xp.array([0], dtype=xp.int8)
+        b = xp.iinfo(dtype).min
+        return a + b
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_large_int_lower_2(self, xp, dtype):
+        a = xp.array([-1], dtype=xp.int8)
+        b = xp.iinfo(dtype).min + 1
+        return a + b
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_large_int_lower_3(self, xp, dtype):
+        a = xp.array([xp.iinfo(dtype).min], dtype=dtype)
+        b = xp.int8(0)
+        return a + b
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_large_int_lower_4(self, xp, dtype):
+        a = xp.array([xp.iinfo(dtype).min + 1], dtype=dtype)
+        b = xp.int8(-1)
+        return a + b
diff --git a/tests/cupy_tests/core_tests/test_flags.py b/tests/cupy_tests/core_tests/test_flags.py
new file mode 100644
index 0000000..0860006
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_flags.py
@@ -0,0 +1,67 @@
+import unittest
+
+from cupy._core import flags
+from cupy import testing
+
+
+class TestFlags(unittest.TestCase):
+
+    def setUp(self):
+        self.flags = flags.Flags(1, 2, 3)
+
+    def test_c_contiguous(self):
+        assert 1 == self.flags['C_CONTIGUOUS']
+
+    def test_f_contiguous(self):
+        assert 2 == self.flags['F_CONTIGUOUS']
+
+    def test_owndata(self):
+        assert 3 == self.flags['OWNDATA']
+
+    def test_key_error(self):
+        with self.assertRaises(KeyError):
+            self.flags['unknown key']
+
+    def test_repr(self):
+        assert '''  C_CONTIGUOUS : 1
+  F_CONTIGUOUS : 2
+  OWNDATA : 3''' == repr(self.flags)
+
+
+@testing.parameterize(
+    *testing.product({
+        'order': ['C', 'F', 'non-contiguous'],
+        'shape': [(8, ), (4, 8)],
+    })
+)
+class TestContiguityFlags(unittest.TestCase):
+
+    def setUp(self):
+        self.flags = None
+
+    def init_flags(self, xp):
+        if self.order == 'non-contiguous':
+            a = xp.empty(self.shape)[::2]
+        else:
+            a = xp.empty(self.shape, order=self.order)
+        self.flags = a.flags
+
+    @testing.numpy_cupy_equal()
+    def test_fnc(self, xp):
+        self.init_flags(xp)
+        return self.flags.fnc
+
+    @testing.numpy_cupy_equal()
+    def test_forc(self, xp):
+        self.init_flags(xp)
+        return self.flags.forc
+
+    @testing.numpy_cupy_equal()
+    def test_f_contiguous(self, xp):
+        self.init_flags(xp)
+        return self.flags.f_contiguous
+
+    @testing.numpy_cupy_equal()
+    def test_c_contiguous(self, xp):
+        self.init_flags(xp)
+        return self.flags.c_contiguous
diff --git a/tests/cupy_tests/core_tests/test_function.py b/tests/cupy_tests/core_tests/test_function.py
new file mode 100644
index 0000000..da0e8b6
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_function.py
@@ -0,0 +1,218 @@
+import unittest
+import pytest
+
+import numpy
+
+import cupy
+from cupy._core import core
+from cupy.cuda import compiler
+from cupy.cuda import runtime
+from cupy import testing
+
+
+def _compile_func(kernel_name, code):
+    # workaround for hipRTC
+    extra_source = core._get_header_source() if runtime.is_hip else None
+    mod = compiler._compile_module_with_cache(
+        code, options=('--std=c++11',), extra_source=extra_source)
+    return mod.get_function(kernel_name)
+
+
+class TestFunction(unittest.TestCase):
+
+    def test_python_scalar(self):
+        code = '''
+extern "C" __global__ void test_kernel(const double* a, double b, double* x) {
+  int i = threadIdx.x;
+  x[i] = a[i] + b;
+}
+'''
+
+        a_cpu = numpy.arange(24, dtype=numpy.float64).reshape((4, 6))
+        a = cupy.array(a_cpu)
+        b = float(2)
+        x = cupy.empty_like(a)
+
+        func = _compile_func('test_kernel', code)
+
+        func.linear_launch(a.size, (a, b, x))
+
+        expected = a_cpu + b
+        testing.assert_array_equal(x, expected)
+
+    def test_numpy_scalar(self):
+        code = '''
+extern "C" __global__ void test_kernel(const double* a, double b, double* x) {
+  int i = threadIdx.x;
+  x[i] = a[i] + b;
+}
+'''
+
+        a_cpu = numpy.arange(24, dtype=numpy.float64).reshape((4, 6))
+        a = cupy.array(a_cpu)
+        b = numpy.float64(2)
+        x = cupy.empty_like(a)
+
+        func = _compile_func('test_kernel', code)
+
+        func.linear_launch(a.size, (a, b, x))
+
+        expected = a_cpu + b
+        testing.assert_array_equal(x, expected)
+
+    def test_numpy_dtype(self):
+        code = '''
+extern "C" __global__ void test_kernel(const double* a, double3 b, double* x) {
+  int i = threadIdx.x;
+  x[i] = a[i] + b.x + b.y + b.z;
+}
+'''
+
+        double3 = numpy.dtype({'names': ['x', 'y', 'z'],
+                               'formats': [numpy.float64]*3})
+        a_cpu = numpy.arange(24, dtype=numpy.float64)
+        a = cupy.array(a_cpu)
+        b = numpy.random.rand(3).view(double3)
+        x = cupy.empty_like(a)
+
+        func = _compile_func('test_kernel', code)
+
+        func.linear_launch(a.size, (a, b, x))
+
+        expected = a_cpu + b['x'] + b['y'] + b['z']
+        testing.assert_array_equal(x, expected)
+
+    def test_static_array(self):
+        code = '''
+struct double5 {
+    double value[5];
+    __device__ const double& operator[](size_t i) const { return value[i]; }
+};
+
+extern "C" __global__ void test_kernel(const double* a, double5 b, double* x) {
+  int i = threadIdx.x;
+  x[i] = a[i] + b[0] + b[1] + b[2] + b[3] + b[4];
+}
+'''
+
+        a_cpu = numpy.arange(24, dtype=numpy.float64)
+        a = cupy.array(a_cpu)
+        x = cupy.empty_like(a)
+
+        func = _compile_func('test_kernel', code)
+
+        # We cannot pass np.ndarray kernel arguments of size > 1
+        b = numpy.arange(5).astype(numpy.float64)
+        with pytest.raises(TypeError):
+            func.linear_launch(a.size, (a, b, x))
+
+        double5 = numpy.dtype({
+            'names': ['dummy'],
+            'formats': [(numpy.float64, (5,))]
+        })
+        func.linear_launch(a.size, (a, b.view(double5), x))
+
+        expected = a_cpu + b.sum()
+        testing.assert_array_equal(x, expected)
+
+    def test_custom_user_struct(self):
+        struct_definition = '''
+struct custom_user_struct {
+    int4 a;
+    char b;
+    double c[2];
+    short1 d;
+    unsigned long long int e[3];
+};
+'''
+
+        # first step is to determine struct memory layout
+        struct_layout_code = '''
+{struct_definition}
+extern "C" __global__ void get_struct_layout(
+                                unsigned long long *itemsize,
+                                unsigned long long *sizes,
+                                unsigned long long *offsets) {{
+    const custom_user_struct* ptr = nullptr;
+
+    itemsize[0] = sizeof(custom_user_struct);
+
+    sizes[0] = sizeof(ptr->a);
+    sizes[1] = sizeof(ptr->b);
+    sizes[2] = sizeof(ptr->c);
+    sizes[3] = sizeof(ptr->d);
+    sizes[4] = sizeof(ptr->e);
+
+    offsets[0] = (unsigned long long)&ptr->a;
+    offsets[1] = (unsigned long long)&ptr->b;
+    offsets[2] = (unsigned long long)&ptr->c;
+    offsets[3] = (unsigned long long)&ptr->d;
+    offsets[4] = (unsigned long long)&ptr->e;
+}}
+'''.format(struct_definition=struct_definition)
+
+        itemsize = cupy.ndarray(shape=(1,), dtype=numpy.uint64)
+        sizes = cupy.ndarray(shape=(5,), dtype=numpy.uint64)
+        offsets = cupy.ndarray(shape=(5,), dtype=numpy.uint64)
+        func = _compile_func('get_struct_layout', struct_layout_code)
+        func.linear_launch(1, (itemsize, sizes, offsets))
+
+        # Build structure data type recursively
+        names = list('abcde')
+        itemsize = cupy.asnumpy(itemsize).item()
+        sizes = cupy.asnumpy(sizes).tolist()
+        offsets = cupy.asnumpy(offsets).tolist()
+
+        def make_packed(basetype, N, itemsize):
+            assert 0 < N <= 4, N
+            names = list('xyzw')[:N]
+            formats = [basetype]*N
+            return numpy.dtype(dict(names=names,
+                                    formats=formats,
+                                    itemsize=itemsize))
+
+        # structure member data types
+        int4 = make_packed(numpy.int32, 4, sizes[0])
+        char = make_packed(numpy.int8, 1, sizes[1])
+        double2 = make_packed(numpy.float64, 2, sizes[2])
+        short1 = make_packed(numpy.int16, 1, sizes[3])
+        ulong3 = make_packed(numpy.uint64, 3, sizes[4])
+
+        formats = [int4, char, double2, short1, ulong3]
+        struct_dtype = numpy.dtype(dict(names=names,
+                                        formats=formats,
+                                        offsets=offsets,
+                                        itemsize=itemsize))
+
+        s = numpy.empty(shape=(1,), dtype=struct_dtype)
+        s['a'] = numpy.arange(0, 4).astype(numpy.int32).view(int4)
+        s['b'] = numpy.arange(4, 5).astype(numpy.int8).view(char)
+        s['c'] = numpy.arange(5, 7).astype(numpy.float64).view(double2)
+        s['d'] = numpy.arange(7, 8).astype(numpy.int16).view(short1)
+        s['e'] = numpy.arange(8, 11).astype(numpy.uint64).view(ulong3)
+
+        # test kernel code
+        code = '''
+{struct_definition}
+extern "C" __global__ void test_kernel(const double* a,
+                                       custom_user_struct s,
+                                       double* x) {{
+    int i = threadIdx.x;
+    double sum = s.a.x + s.a.y + s.a.z + s.a.w;
+    sum += s.b;
+    sum += s.c[0] + s.c[1];
+    sum += s.d.x;
+    sum += s.e[0] + s.e[1] + s.e[2];
+    x[i] = a[i] + sum;
+}}
+'''.format(struct_definition=struct_definition)
+
+        a_cpu = numpy.arange(24, dtype=numpy.float64)
+        a = cupy.array(a_cpu)
+        x = cupy.empty_like(a)
+
+        func = _compile_func('test_kernel', code)
+        func.linear_launch(a.size, (a, s, x))
+
+        expected = a_cpu + 55.0
+        testing.assert_array_equal(x, expected)
diff --git a/tests/cupy_tests/core_tests/test_gufuncs.py b/tests/cupy_tests/core_tests/test_gufuncs.py
new file mode 100644
index 0000000..02c5030
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_gufuncs.py
@@ -0,0 +1,282 @@
+import numpy
+import pytest
+
+import cupy
+
+from cupy import testing
+from cupy._core._gufuncs import _GUFunc
+
+
+class TestGUFuncSignature:
+    @pytest.mark.parametrize('signature', [
+        ('(i,j)->(i,j)', [('i', 'j')], [('i', 'j')]),
+        ('->(i)', [()], [('i',)]),
+        ('(i,j),(j,k)->(k,l)', [('i', 'j'), ('j', 'k')], [('k', 'l')]),
+        ('()->()', [()], [()])])
+    def test_signature_parsing(self, signature):
+        i, o = cupy._core._gufuncs._parse_gufunc_signature(signature[0])
+        assert i == signature[1]
+        assert o == signature[2]
+
+    @pytest.mark.parametrize('signature', [
+        '(i,j)(i,j)',
+        '(i,j)-(i,j)',
+        '(i,j)(i,j)->(i,j)',
+        'j->(i',
+        '',
+        '()->()->'])
+    def test_invalid_signature_parsing(self, signature):
+        with pytest.raises(ValueError):
+            cupy._core._gufuncs._parse_gufunc_signature(signature)
+
+
+class TestGUFuncAxes:
+    def _get_gufunc(self, signature):
+        def func(x):
+            return x
+        return _GUFunc(func, signature)
+
+    def _get_gufunc_scalar(self, signature):
+        def func(x):
+            return x.sum()
+        return _GUFunc(func, signature)
+
+    @pytest.mark.parametrize('axes', [
+        ((-1, -2), (-1, -2)),
+        ((0, 1), (0, 1)),
+        ((0, 1), (-1, -2)),
+        ((1, 2), (-1, -2)),
+        ((1, 2), (1, 2)),
+        ((1, 2), (2, 3)),
+        ((2, 3), (-1, -2)),
+        ((2, 3), (0, 1)),
+        ((2, 3), (1, 2)),
+        ((0, 3), (1, 2)),
+        ((0, 3), (2, 0)),
+    ])
+    @testing.numpy_cupy_array_equal()
+    def test_axes_selection(self, xp, axes):
+        x = testing.shaped_arange((2, 3, 4, 5), xp=xp)
+        if xp is cupy:
+            return self._get_gufunc('(i,j)->(i,j)')(x, axes=list(axes))
+        else:
+            return numpy.moveaxis(x, axes[0], axes[1])
+
+    @pytest.mark.parametrize('axes', [
+        (-1, -2),
+        (0, 1),
+        (1, 2),
+        (2, 3),
+        (0, 2),
+        (0, 3),
+        (1, 3),
+        (3, 0),
+        (2, 0),
+        (2, 1),
+        (1, 0),
+    ])
+    @testing.numpy_cupy_array_equal()
+    def test_axes_selection_single(self, xp, axes):
+        x = testing.shaped_arange((2, 3, 4, 5), xp=xp)
+        if xp is cupy:
+            return self._get_gufunc('(i)->(i)')(x, axes=list(axes))
+        else:
+            return numpy.moveaxis(x, axes[0], axes[1])
+
+    @pytest.mark.parametrize('axis', [0, 1, 2, 3])
+    @testing.numpy_cupy_array_equal()
+    def test_axis(self, xp, axis):
+        x = testing.shaped_arange((2, 3, 4, 5), xp=xp)
+        if xp is cupy:
+            return self._get_gufunc_scalar('(i)->()')(x, axis=axis)
+        else:
+            return x.sum(axis=axis)
+
+    def test_axis_invalid(self):
+        x = testing.shaped_arange((2, 3, 4, 5))
+        with pytest.raises(ValueError):
+            self._get_gufunc('(i, j)->(i, j)')(x, axis=((0, 1), (0, 1)))
+
+    @pytest.mark.parametrize('supports_batched', [True, False])
+    def test_supports_batched(self, supports_batched):
+        x = testing.shaped_arange((2, 3, 4, 5))
+
+        def func(x):
+            nonlocal supports_batched
+            if supports_batched:
+                assert x.ndim == 4
+            else:
+                assert x.ndim == 2
+            return x
+        gu_func = _GUFunc(func, '(i,j)->(i,j)',
+                          supports_batched=supports_batched)
+        gu_func(x)
+
+
+class TestGUFuncOut:
+    def _get_gufunc(self):
+        def func(x):
+            return x
+        return _GUFunc(func, '(i,j)->(i,j)')
+
+    def test_out_array(self):
+        x = testing.shaped_arange((2, 3, 4, 5))
+        out = cupy.empty((2, 3, 4, 5))
+        self._get_gufunc()(x, out=out)
+        testing.assert_allclose(x, out)
+
+    def test_supports_out(self):
+        x = testing.shaped_arange((2, 3, 4, 5))
+        out = cupy.empty((2, 3, 4, 5))
+        out_ptr = out.data.ptr
+
+        def func(x, out=None):
+            nonlocal out_ptr
+            # Base is a view of the output due to the batching
+            assert out.base.data.ptr == out_ptr
+            out[:] = x
+        gu_func = _GUFunc(func, '(i,j)->(i,j)', supports_out=True)
+        gu_func(x, out=out)
+        testing.assert_allclose(x, out)
+
+    def test_invalid_output_shape(self):
+        x = testing.shaped_arange((2, 3, 4, 5))
+        out = cupy.empty((3, 3, 4, 5))
+        with pytest.raises(ValueError):
+            self._get_gufunc()(x, out=out)
+
+    def test_invalid_output_dtype(self):
+        x = testing.shaped_arange((2, 3, 4, 5))
+        out = cupy.empty((2, 3, 4, 5), dtype='h')
+        with pytest.raises(TypeError):
+            self._get_gufunc()(x, out=out)
+
+
+class TestGUFuncDtype:
+    @testing.for_all_dtypes(name='dtype_i', no_bool=True, no_complex=True)
+    @testing.for_all_dtypes(name='dtype_o', no_bool=True, no_complex=True)
+    def test_dtypes(self, dtype_i, dtype_o):
+        x = testing.shaped_arange((2, 3, 4, 5), dtype=dtype_i)
+        if numpy.can_cast(dtype_o, x.dtype):
+            def func(x):
+                return x
+            gufunc = _GUFunc(func, '(i,j)->(i,j)')
+            z = gufunc(x, dtype=dtype_o)
+            assert z.dtype == dtype_o
+            testing.assert_allclose(z, x)
+
+
+class TestGUFuncOrder():
+
+    @pytest.mark.parametrize('order', ['C', 'F', 'K'])
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_order(self, xp, order):
+        x = testing.shaped_arange((2, 3, 4), xp=xp)
+        if xp is cupy:
+            def default(x):
+                return x
+            gu_func = _GUFunc(default, '(i, j, k)->(i, j, k)')
+            return gu_func(x, order=order)
+        else:
+            return xp.asarray(x, order=order)
+
+    @pytest.mark.parametrize('order', [('F', 'C', 'C'), ('F', 'F', 'F')])
+    def test_order_a(self, order):
+        x = testing.shaped_arange((2, 3, 4), order=order[0])
+        y = testing.shaped_arange((2, 3, 4), order=order[1])
+
+        def default(x, y):
+            return x
+
+        gu_func = _GUFunc(default, '(i,j,k),(i,j,k)->(i,j,k)')
+        z = gu_func(x, y, order='A')
+        if order[2] == 'C':
+            assert z.flags.c_contiguous
+        else:
+            assert z.flags.f_contiguous
+
+
+class TestGUFuncSignatures():
+    def test_signatures(self):
+        dtypes = 'fdihq'
+        dtypes_access = {d: None for d in dtypes}
+
+        def integers(x, y):
+            nonlocal dtypes_access
+            dtypes_access[numpy.dtype(x.dtype).char] = integers
+            return x + y
+
+        def floats(x, y):
+            nonlocal dtypes_access
+            dtypes_access[numpy.dtype(x.dtype).char] = floats
+            return x + y
+
+        def default(x, y):
+            nonlocal dtypes_access
+            dtypes_access[numpy.dtype(x.dtype).char] = default
+            return x + y
+
+        sigs = (('ii->i', integers), ('dd->d', floats))
+        gu_func = _GUFunc(default, '(i),(i)->(i)', signatures=sigs)
+        for dtype in dtypes:
+            x = cupy.array([10], dtype=dtype)
+            y = x
+            gu_func(x, y, casting='no')
+            if dtype in 'i':
+                assert dtypes_access[dtype] == integers
+            elif dtype in 'd':
+                assert dtypes_access[dtype] == floats
+            else:
+                assert dtypes_access[dtype] == default
+
+    @pytest.mark.parametrize('sig,', ['ii->i', 'i', ('i', 'i', 'i')])
+    def test_signature_lookup(self, sig):
+        called = False
+
+        def func(x, y):
+            nonlocal called
+            called = True
+            return x + y
+
+        def default(x, y):
+            return x + y
+
+        dtypes = 'fdhq'
+
+        sigs = (('ii->i', func),)
+        gu_func = _GUFunc(default, '(i),(i)->(i)', signatures=sigs)
+        for dtype in dtypes:
+            x = cupy.array([10], dtype=dtype)
+            y = x
+            gu_func(x, y, casting='no')
+            assert not called
+
+        x = cupy.array([10], dtype='d')
+        y = x
+        z = gu_func(x, y, casting='unsafe', signature=sig)
+        assert z.dtype == numpy.int32
+        assert called
+
+    @pytest.mark.parametrize('sigs,', [('i',), ('',), ('iii->i',), ('ii->',)])
+    def test_invalid_signatures(self, sigs):
+
+        def default(x, y):
+            return x + y
+
+        with pytest.raises(ValueError):
+            _GUFunc(default, '(i),(i)->(i)', signatures=sigs)
+
+    @pytest.mark.parametrize('sig,', ['i->i', 'id->i', ''])
+    def test_invalid_lookup(self, sig):
+
+        def default(x, y):
+            return x + y
+
+        sigs = (('ii->i', default),)
+        gu_func = _GUFunc(default, '(i),(i)->(i)', signatures=sigs)
+        _GUFunc(default, '(i),(i)->(i)', signatures=sigs)
+
+        x = cupy.array([10], dtype='d')
+        y = x
+        with pytest.raises(TypeError):
+            gu_func(x, y, casting='unsafe', signature=sig)
diff --git a/tests/cupy_tests/core_tests/test_include.py b/tests/cupy_tests/core_tests/test_include.py
new file mode 100644
index 0000000..641cb83
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_include.py
@@ -0,0 +1,86 @@
+import os
+from unittest import mock
+
+import cupy
+
+import pytest
+
+
+_code_base = '''
+#include <cuda_fp16.h>
+
+#include <cupy/carray.cuh>
+#include <cupy/complex.cuh>
+#include <cupy/cuComplex_bridge.h>
+#include <cupy/math_constants.h>
+#include <cupy/atomics.cuh>
+#include <cupy/hip_workaround.cuh>
+#include <cupy/swap.cuh>
+#include <cupy/tuple.cuh>
+'''
+
+_code_nvcc = _code_base + '''
+#include <cupy/type_dispatcher.cuh>
+
+int main() {
+    return 0;
+}
+'''
+
+_code_nvrtc = _code_base + '''
+
+__device__ void kernel() {
+}
+'''
+
+
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason='for CUDA')
+class TestIncludesCompileCUDA:
+    def _get_cuda_archs(self):
+        cuda_ver = cupy.cuda.runtime.runtimeGetVersion()
+        to_exclude = set((int(a) for a in cupy.cuda.compiler._tegra_archs))
+        if cuda_ver < 11000:
+            # CUDA 10.2 (Tegra excluded)
+            archs = (30, 35, 50, 52, 60, 61, 70, 75)
+        elif cuda_ver < 11010:
+            # CUDA 11.0
+            archs = (35, 37, 50, 52, 53, 60, 61, 62, 70, 72, 75, 80)
+        elif cuda_ver < 11020:
+            # CUDA 11.1
+            archs = (35, 37, 50, 52, 53, 60, 61, 62, 70, 72, 75, 80, 86)
+        else:
+            # CUDA 11.2+
+            archs = cupy.cuda.nvrtc.getSupportedArchs()
+            if cuda_ver == 11020 or cuda_ver >= 12000:
+                to_exclude.add(69)
+        archs = tuple(set(archs) - to_exclude)
+
+        return archs
+
+    def _get_options(self):
+        return (
+            '-I{}'.format(cupy._core.core._get_header_dir_path()),
+            '-I{}'.format(os.path.join(cupy.cuda.get_cuda_path(), 'include')),
+        )
+
+    def test_nvcc(self):
+        options = self._get_options()
+        for arch in self._get_cuda_archs():
+            cupy.cuda.compiler.compile_using_nvcc(
+                _code_nvcc, options=options, arch=arch)
+
+    def test_nvrtc(self):
+        cuda_ver = cupy.cuda.runtime.runtimeGetVersion()
+        options = self._get_options()
+        for arch in self._get_cuda_archs():
+            with mock.patch(
+                    'cupy.cuda.compiler._get_arch_for_options_for_nvrtc',
+                    lambda _: (f'-arch=compute_{arch}', 'ptx')):
+                cupy.cuda.compiler.compile_using_nvrtc(
+                    _code_nvrtc, options=options)
+            if cuda_ver >= 11010:
+                with mock.patch(
+                        'cupy.cuda.compiler._get_arch_for_options_for_nvrtc',
+                        lambda _: (f'-arch=sm_{arch}', 'cubin')):
+                    cupy.cuda.compiler.compile_using_nvrtc(
+                        _code_nvrtc, options=options)
diff --git a/tests/cupy_tests/core_tests/test_internal.py b/tests/cupy_tests/core_tests/test_internal.py
new file mode 100644
index 0000000..df94c98
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_internal.py
@@ -0,0 +1,245 @@
+import math
+import unittest
+
+import numpy
+import pytest
+
+from cupy._core import internal
+from cupy import testing
+
+
+class TestProd(unittest.TestCase):
+
+    def test_empty(self):
+        assert internal.prod([]) == 1
+
+    def test_one(self):
+        assert internal.prod([2]) == 2
+
+    def test_two(self):
+        assert internal.prod([2, 3]) == 6
+
+
+class TestProdSequence(unittest.TestCase):
+
+    def test_empty(self):
+        assert internal.prod_sequence(()) == 1
+
+    def test_one(self):
+        assert internal.prod_sequence((2,)) == 2
+
+    def test_two(self):
+        assert internal.prod_sequence((2, 3)) == 6
+
+
+class TestGetSize:
+
+    def test_none(self):
+        with testing.assert_warns(DeprecationWarning):
+            assert internal.get_size(None) == ()
+
+    def check_collection(self, a):
+        assert internal.get_size(a) == tuple(a)
+
+    def test_list(self):
+        self.check_collection([1, 2, 3])
+
+    def test_tuple(self):
+        self.check_collection((1, 2, 3))
+
+    def test_int(self):
+        assert internal.get_size(1) == (1,)
+
+    def test_numpy_int(self):
+        assert internal.get_size(numpy.int32(1)) == (1,)
+
+    def test_numpy_zero_dim_ndarray(self):
+        assert internal.get_size(numpy.array(1)) == (1,)
+
+    def test_tuple_of_numpy_scalars(self):
+        assert internal.get_size((numpy.int32(1), numpy.array(1))) == (1, 1)
+
+    @pytest.mark.parametrize(
+        'value', [True, numpy.bool_(True), numpy.array(True, dtype='?')])
+    def test_bool(self, value):
+        with pytest.raises(TypeError):
+            internal.get_size(value)
+        with pytest.raises(TypeError):
+            internal.get_size((value, value))
+
+    def test_float(self):
+        # `internal.get_size` is not responsible to interpret values as
+        # integers.
+        assert internal.get_size(1.0) == (1.0,)
+
+
+class TestVectorEqual(unittest.TestCase):
+
+    def test_empty(self):
+        assert internal.vector_equal([], []) is True
+
+    def test_not_equal(self):
+        assert internal.vector_equal([1, 2, 3], [1, 2, 0]) is False
+
+    def test_equal(self):
+        assert internal.vector_equal([-1, 0, 1], [-1, 0, 1]) is True
+
+    def test_different_size(self):
+        assert internal.vector_equal([1, 2, 3], [1, 2]) is False
+
+
+class TestGetCContiguity(unittest.TestCase):
+
+    def test_zero_in_shape(self):
+        assert internal.get_c_contiguity((1, 0, 1), (1, 1, 1), 3)
+
+    def test_all_one_shape(self):
+        assert internal.get_c_contiguity((1, 1, 1), (1, 1, 1), 3)
+
+    def test_normal1(self):
+        assert internal.get_c_contiguity((3, 4, 3), (24, 6, 2), 2)
+
+    def test_normal2(self):
+        assert internal.get_c_contiguity((3, 1, 3), (6, 100, 2), 2)
+
+    def test_normal3(self):
+        assert internal.get_c_contiguity((3,), (4, ), 4)
+
+    def test_normal4(self):
+        assert internal.get_c_contiguity((), (), 4)
+
+    def test_normal5(self):
+        assert internal.get_c_contiguity((3, 1), (4, 8), 4)
+
+    def test_no_contiguous1(self):
+        assert not internal.get_c_contiguity((3, 4, 3), (30, 6, 2), 2)
+
+    def test_no_contiguous2(self):
+        assert not internal.get_c_contiguity((3, 1, 3), (24, 6, 2), 2)
+
+    def test_no_contiguous3(self):
+        assert not internal.get_c_contiguity((3, 1, 3), (6, 6, 4), 2)
+
+
+class TestInferUnknownDimension(unittest.TestCase):
+
+    def test_known_all(self):
+        assert internal.infer_unknown_dimension((1, 2, 3), 6) == [1, 2, 3]
+
+    def test_multiple_unknown(self):
+        with self.assertRaises(ValueError):
+            internal.infer_unknown_dimension((-1, 1, -1), 10)
+
+    def test_infer(self):
+        assert internal.infer_unknown_dimension((-1, 2, 3), 12) == [2, 2, 3]
+
+
+@testing.parameterize(
+    {'slice': (2, 8, 1),    'expect': (2, 8, 1)},
+    {'slice': (2, None, 1), 'expect': (2, 10, 1)},
+    {'slice': (2, 1, 1),    'expect': (2, 2, 1)},
+    {'slice': (2, -1, 1),   'expect': (2, 9, 1)},
+
+    {'slice': (None, 8, 1),  'expect': (0, 8, 1)},
+    {'slice': (-3, 8, 1),    'expect': (7, 8, 1)},
+    {'slice': (11, 8, 1),    'expect': (10, 10, 1)},
+    {'slice': (11, 11, 1),   'expect': (10, 10, 1)},
+    {'slice': (-11, 8, 1),   'expect': (0, 8, 1)},
+    {'slice': (-11, -11, 1), 'expect': (0, 0, 1)},
+
+    {'slice': (8, 2, -1),    'expect': (8, 2, -1)},
+    {'slice': (8, None, -1), 'expect': (8, -1, -1)},
+    {'slice': (8, 9, -1),    'expect': (8, 8, -1)},
+    {'slice': (8, -3, -1),   'expect': (8, 7, -1)},
+
+    {'slice': (None, 8, -1),  'expect': (9, 8, -1)},
+    {'slice': (-3, 6, -1),    'expect': (7, 6, -1)},
+
+    {'slice': (10, 10, -1),   'expect': (9, 9, -1)},
+    {'slice': (10, 8, -1),    'expect': (9, 8, -1)},
+    {'slice': (9, 10, -1),    'expect': (9, 9, -1)},
+    {'slice': (9, 9, -1),     'expect': (9, 9, -1)},
+    {'slice': (9, 8, -1),     'expect': (9, 8, -1)},
+    {'slice': (8, 8, -1),     'expect': (8, 8, -1)},
+    {'slice': (-9, -8, -1),   'expect': (1, 1, -1)},
+    {'slice': (-9, -9, -1),   'expect': (1, 1, -1)},
+    {'slice': (-9, -10, -1),  'expect': (1, 0, -1)},
+    {'slice': (-9, -11, -1),  'expect': (1, -1, -1)},
+    {'slice': (-9, -12, -1),  'expect': (1, -1, -1)},
+    {'slice': (-10, -9, -1),  'expect': (0, 0, -1)},
+    {'slice': (-10, -10, -1), 'expect': (0, 0, -1)},
+    {'slice': (-10, -11, -1), 'expect': (0, -1, -1)},
+    {'slice': (-10, -12, -1), 'expect': (0, -1, -1)},
+    {'slice': (-11, 8, -1),   'expect': (-1, -1, -1)},
+    {'slice': (-11, -9, -1),  'expect': (-1, -1, -1)},
+    {'slice': (-11, -10, -1), 'expect': (-1, -1, -1)},
+    {'slice': (-11, -11, -1), 'expect': (-1, -1, -1)},
+    {'slice': (-11, -12, -1), 'expect': (-1, -1, -1)},
+)
+class TestCompleteSlice(unittest.TestCase):
+
+    def test_complete_slice(self):
+        assert internal.complete_slice(
+            slice(*self.slice), 10) == slice(*self.expect)
+
+
+class TestCompleteSliceError(unittest.TestCase):
+
+    def test_invalid_step_value(self):
+        with self.assertRaises(ValueError):
+            internal.complete_slice(slice(1, 1, 0), 1)
+
+    def test_invalid_step_type(self):
+        with self.assertRaises(TypeError):
+            internal.complete_slice(slice(1, 1, (1, 2)), 1)
+
+    def test_invalid_start_type(self):
+        with self.assertRaises(TypeError):
+            internal.complete_slice(slice((1, 2), 1, 1), 1)
+        with self.assertRaises(TypeError):
+            internal.complete_slice(slice((1, 2), 1, -1), 1)
+
+    def test_invalid_stop_type(self):
+        with self.assertRaises(TypeError):
+            internal.complete_slice(slice((1, 2), 1, 1), 1)
+        with self.assertRaises(TypeError):
+            internal.complete_slice(slice((1, 2), 1, -1), 1)
+
+
+@testing.parameterize(
+    {'x': 0, 'expect': 0},
+    {'x': 1, 'expect': 1},
+    {'x': 2, 'expect': 2},
+    {'x': 3, 'expect': 4},
+    {'x': 2 ** 10,     'expect': 2 ** 10},
+    {'x': 2 ** 10 - 1, 'expect': 2 ** 10},
+    {'x': 2 ** 10 + 1, 'expect': 2 ** 11},
+    {'x': 2 ** 40,     'expect': 2 ** 40},
+    {'x': 2 ** 40 - 1, 'expect': 2 ** 40},
+    {'x': 2 ** 40 + 1, 'expect': 2 ** 41},
+)
+class TestClp2(unittest.TestCase):
+
+    def test_clp2(self):
+        assert internal.clp2(self.x) == self.expect
+
+
+@testing.parameterize(*testing.product({
+    'value': [0.0, 1.0, -1.0,
+              0.25, -0.25,
+              11.0, -11.0,
+              2 ** -15, -(2 ** -15),  # Denormalized Number
+              float('inf'), float('-inf')],
+}))
+class TestConvertFloat16(unittest.TestCase):
+
+    def test_conversion(self):
+        half = internal.to_float16(self.value)
+        assert internal.from_float16(half) == self.value
+
+
+class TestConvertFloat16Nan(unittest.TestCase):
+
+    def test_conversion(self):
+        half = internal.to_float16(float('nan'))
+        assert math.isnan(internal.from_float16(half))
diff --git a/tests/cupy_tests/core_tests/test_iter.py b/tests/cupy_tests/core_tests/test_iter.py
new file mode 100644
index 0000000..9b417c5
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_iter.py
@@ -0,0 +1,42 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(*testing.product(
+    {'shape': [(3,), (2, 3, 4), (0,), (0, 2), (3, 0)]},
+))
+class TestIter(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_list(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype)
+        return list(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_len(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype)
+        return len(x)
+
+
+class TestIterInvalid(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_iter(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((), xp, dtype)
+            with pytest.raises(TypeError):
+                iter(x)
+
+    @testing.for_all_dtypes()
+    def test_len(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((), xp, dtype)
+            with pytest.raises(TypeError):
+                len(x)
diff --git a/tests/cupy_tests/core_tests/test_ndarray.py b/tests/cupy_tests/core_tests/test_ndarray.py
new file mode 100644
index 0000000..968812a
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray.py
@@ -0,0 +1,648 @@
+import copy
+import unittest
+
+import numpy
+import pytest
+
+from cupy_backends.cuda.api import driver
+from cupy_backends.cuda.api import runtime
+from cupy_backends.cuda import stream as stream_module
+import cupy
+from cupy import _util
+from cupy import _core
+from cupy import cuda
+from cupy import get_array_module
+from cupy import testing
+
+
+def wrap_take(array, *args, **kwargs):
+    if get_array_module(array) == numpy:
+        kwargs['mode'] = 'wrap'
+
+    return array.take(*args, **kwargs)
+
+
+class TestNdarrayInit(unittest.TestCase):
+
+    def test_shape_none(self):
+        with testing.assert_warns(DeprecationWarning):
+            a = cupy.ndarray(None)
+        assert a.shape == ()
+
+    def test_shape_int(self):
+        a = cupy.ndarray(3)
+        assert a.shape == (3,)
+
+    def test_shape_not_integer(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(TypeError):
+                xp.ndarray(1.0)
+            with pytest.raises(TypeError):
+                xp.ndarray((1.0,))
+
+    def test_shape_int_with_strides(self):
+        dummy = cupy.ndarray(3)
+        a = cupy.ndarray(3, strides=(0,), memptr=dummy.data)
+        assert a.shape == (3,)
+        assert a.strides == (0,)
+
+    def test_memptr(self):
+        a = cupy.arange(6).astype(numpy.float32).reshape((2, 3))
+        memptr = a.data
+
+        b = cupy.ndarray((2, 3), numpy.float32, memptr)
+        testing.assert_array_equal(a, b)
+
+        b += 1
+        testing.assert_array_equal(a, b)
+
+    def test_memptr_with_strides(self):
+        buf = cupy.ndarray(20, numpy.uint8)
+        memptr = buf.data
+
+        # self-overlapping strides
+        a = cupy.ndarray((2, 3), numpy.float32, memptr, strides=(8, 4))
+        assert a.strides == (8, 4)
+
+        a[:] = 1
+        a[0, 2] = 4
+        assert float(a[1, 0]) == 4
+
+    def test_strides_without_memptr(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.ndarray((2, 3), numpy.float32, strides=(20, 4))
+
+    def test_strides_is_given_and_order_is_ignored(self):
+        buf = cupy.ndarray(20, numpy.uint8)
+        a = cupy.ndarray(
+            (2, 3), numpy.float32, buf.data, strides=(8, 4), order='C')
+        assert a.strides == (8, 4)
+
+    @testing.with_requires('numpy>=1.19')
+    def test_strides_is_given_but_order_is_invalid(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.ndarray((2, 3), numpy.float32, strides=(8, 4), order='!')
+
+    def test_order(self):
+        shape = (2, 3, 4)
+        a = _core.ndarray(shape, order='F')
+        a_cpu = numpy.ndarray(shape, order='F')
+        assert a.strides == a_cpu.strides
+        assert a.flags.f_contiguous
+        assert not a.flags.c_contiguous
+
+    def test_order_none(self):
+        shape = (2, 3, 4)
+        a = _core.ndarray(shape, order=None)
+        a_cpu = numpy.ndarray(shape, order=None)
+        assert a.flags.c_contiguous == a_cpu.flags.c_contiguous
+        assert a.flags.f_contiguous == a_cpu.flags.f_contiguous
+        assert a.strides == a_cpu.strides
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(), (1,), (1, 2), (1, 2, 3)],
+        'order': ['C', 'F'],
+        'dtype': [
+            numpy.uint8,  # itemsize=1
+            numpy.uint16,  # itemsize=2
+        ],
+    }))
+class TestNdarrayInitStrides(unittest.TestCase):
+
+    # Check the strides given shape, itemsize and order.
+
+    @testing.numpy_cupy_equal()
+    def test_strides(self, xp):
+        arr = xp.ndarray(self.shape, dtype=self.dtype, order=self.order)
+        return (
+            arr.strides,
+            arr.flags.c_contiguous,
+            arr.flags.f_contiguous)
+
+
+class TestNdarrayInitRaise(unittest.TestCase):
+
+    def test_unsupported_type(self):
+        arr = numpy.ndarray((2, 3), dtype=object)
+        with pytest.raises(ValueError):
+            _core.array(arr)
+
+    def test_excessive_ndim(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.ndarray(
+                    shape=[1 for i in range(33)], dtype=xp.int8)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(), (0,), (1,), (0, 0, 2), (2, 3)],
+    })
+)
+class TestNdarrayDeepCopy(unittest.TestCase):
+
+    def _check_deepcopy(self, arr, arr2):
+        assert arr.data is not arr2.data
+        assert arr.shape == arr2.shape
+        assert arr.size == arr2.size
+        assert arr.dtype == arr2.dtype
+        assert arr.strides == arr2.strides
+        testing.assert_array_equal(arr, arr2)
+
+    def test_deepcopy(self):
+        arr = _core.ndarray(self.shape)
+        arr2 = copy.deepcopy(arr)
+        self._check_deepcopy(arr, arr2)
+
+    @testing.multi_gpu(2)
+    def test_deepcopy_multi_device(self):
+        arr = _core.ndarray(self.shape)
+        with cuda.Device(1):
+            arr2 = copy.deepcopy(arr)
+        self._check_deepcopy(arr, arr2)
+        assert arr2.device == arr.device
+
+
+_test_copy_multi_device_with_stream_src = r'''
+extern "C" __global__
+void wait_and_write(long long *x) {
+  clock_t start = clock();
+  clock_t now;
+  for (;;) {
+    now = clock();
+    clock_t cycles = now > start ? now - start : now + (0xffffffff - start);
+    if (cycles >= 1000000000) {
+      break;
+    }
+  }
+  x[0] = 1;
+  x[1] = now;  // in case the compiler optimizing away the entire loop
+}
+'''
+
+
+class TestNdarrayCopy:
+
+    @testing.multi_gpu(2)
+    @testing.for_orders('CFA')
+    def test_copy_multi_device_non_contiguous(self, order):
+        arr = _core.ndarray((20,))[::2]
+        dev1 = cuda.Device(1)
+        with dev1:
+            arr2 = arr.copy(order)
+        assert arr2.device == dev1
+        testing.assert_array_equal(arr, arr2)
+
+    @testing.multi_gpu(2)
+    def test_copy_multi_device_non_contiguous_K(self):
+        arr = _core.ndarray((20,))[::2]
+        with cuda.Device(1):
+            with pytest.raises(NotImplementedError):
+                arr.copy('K')
+
+    # See cupy/cupy#5004
+    @testing.multi_gpu(2)
+    @pytest.mark.xfail(
+        runtime.is_hip,
+        reason='ROCm may work differently in async D2D copy with streams')
+    def test_copy_multi_device_with_stream(self):
+        # Kernel that takes long enough then finally writes values.
+        src = _test_copy_multi_device_with_stream_src
+        if runtime.is_hip and driver.get_build_version() >= 5_00_00000:
+            src = '#include <ctime>\n' + src
+        kern = cupy.RawKernel(src, 'wait_and_write')
+
+        # Allocates a memory and launches the kernel on a device with its
+        # stream.
+        with cuda.Device(0):
+            # Keep this stream alive over the D2D copy below for HIP
+            with cuda.Stream() as s1:  # NOQA
+                a = cupy.zeros((2,), dtype=numpy.uint64)
+                kern((1,), (1,), a)
+
+        # D2D copy to another device with another stream should get the
+        # original values of the memory before the kernel on the first device
+        # finally makes the write.
+        with cuda.Device(1):
+            with cuda.Stream():
+                b = a.copy()
+                testing.assert_array_equal(
+                    b, numpy.array([0, 0], dtype=numpy.uint64))
+
+
+class TestNdarrayShape(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_shape_set(self, xp):
+        arr = xp.ndarray((2, 3))
+        arr.shape = (3, 2)
+        return xp.array(arr.shape)
+
+    @testing.numpy_cupy_array_equal()
+    def test_shape_set_infer(self, xp):
+        arr = xp.ndarray((2, 3))
+        arr.shape = (3, -1)
+        return xp.array(arr.shape)
+
+    @testing.numpy_cupy_array_equal()
+    def test_shape_set_int(self, xp):
+        arr = xp.ndarray((2, 3))
+        arr.shape = 6
+        return xp.array(arr.shape)
+
+    def test_shape_need_copy(self):
+        # from cupy/cupy#5470
+        for xp in (numpy, cupy):
+            arr = xp.ndarray((2, 3), order='F')
+            with pytest.raises(AttributeError) as e:
+                arr.shape = (3, 2)
+            assert 'incompatible shape' in str(e.value).lower()
+
+
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestNdarrayCudaInterface(unittest.TestCase):
+
+    def test_cuda_array_interface(self):
+        arr = cupy.zeros(shape=(2, 3), dtype=cupy.float64)
+        iface = arr.__cuda_array_interface__
+        assert iface['version'] == 3
+        assert (set(iface.keys()) ==
+                set(['shape', 'typestr', 'data', 'version', 'descr',
+                     'stream', 'strides']))
+        assert iface['shape'] == (2, 3)
+        assert iface['typestr'] == '<f8'
+        assert isinstance(iface['data'], tuple)
+        assert len(iface['data']) == 2
+        assert iface['data'][0] == arr.data.ptr
+        assert not iface['data'][1]
+        assert iface['descr'] == [('', '<f8')]
+        assert iface['strides'] is None
+        assert iface['stream'] == stream_module.get_default_stream_ptr()
+
+    def test_cuda_array_interface_view(self):
+        arr = cupy.zeros(shape=(10, 20), dtype=cupy.float64)
+        view = arr[::2, ::5]
+        iface = view.__cuda_array_interface__
+        assert iface['version'] == 3
+        assert (set(iface.keys()) ==
+                set(['shape', 'typestr', 'data', 'version', 'descr',
+                     'stream', 'strides']))
+        assert iface['shape'] == (5, 4)
+        assert iface['typestr'] == '<f8'
+        assert isinstance(iface['data'], tuple)
+        assert len(iface['data']) == 2
+        assert iface['data'][0] == arr.data.ptr
+        assert not iface['data'][1]
+        assert iface['strides'] == (320, 40)
+        assert iface['descr'] == [('', '<f8')]
+        assert iface['stream'] == stream_module.get_default_stream_ptr()
+
+    def test_cuda_array_interface_zero_size(self):
+        arr = cupy.zeros(shape=(10,), dtype=cupy.float64)
+        view = arr[0:3:-1]
+        iface = view.__cuda_array_interface__
+        assert iface['version'] == 3
+        assert (set(iface.keys()) ==
+                set(['shape', 'typestr', 'data', 'version', 'descr',
+                     'stream', 'strides']))
+        assert iface['shape'] == (0,)
+        assert iface['typestr'] == '<f8'
+        assert isinstance(iface['data'], tuple)
+        assert len(iface['data']) == 2
+        assert iface['data'][0] == 0
+        assert not iface['data'][1]
+        assert iface['strides'] is None
+        assert iface['descr'] == [('', '<f8')]
+        assert iface['stream'] == stream_module.get_default_stream_ptr()
+
+
+@testing.parameterize(*testing.product({
+    'stream': ('null', 'new', 'ptds'),
+    'ver': (2, 3),
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestNdarrayCudaInterfaceStream(unittest.TestCase):
+    def setUp(self):
+        if self.stream == 'null':
+            self.stream = cuda.Stream.null
+        elif self.stream == 'new':
+            self.stream = cuda.Stream()
+        elif self.stream == 'ptds':
+            self.stream = cuda.Stream.ptds
+
+        self.old_ver = _util.CUDA_ARRAY_INTERFACE_EXPORT_VERSION
+        _util.CUDA_ARRAY_INTERFACE_EXPORT_VERSION = self.ver
+
+    def tearDown(self):
+        _util.CUDA_ARRAY_INTERFACE_EXPORT_VERSION = self.old_ver
+
+    def test_cuda_array_interface_stream(self):
+        # this tests exporting CAI with a given stream
+        arr = cupy.zeros(shape=(10,), dtype=cupy.float64)
+        stream = self.stream
+        with stream:
+            iface = arr.__cuda_array_interface__
+        assert iface['version'] == self.ver
+        attrs = ['shape', 'typestr', 'data', 'version', 'descr', 'strides']
+        if self.ver == 3:
+            attrs.append('stream')
+        assert set(iface.keys()) == set(attrs)
+        assert iface['shape'] == (10,)
+        assert iface['typestr'] == '<f8'
+        assert isinstance(iface['data'], tuple)
+        assert len(iface['data']) == 2
+        assert iface['data'] == (arr.data.ptr, False)
+        assert iface['descr'] == [('', '<f8')]
+        assert iface['strides'] is None
+        if self.ver == 3:
+            if stream.ptr == 0:
+                ptr = stream_module.get_default_stream_ptr()
+                assert iface['stream'] == ptr
+            else:
+                assert iface['stream'] == stream.ptr
+
+
+@pytest.mark.skipif(not cupy.cuda.runtime.is_hip,
+                    reason='This is supported on CUDA')
+class TestNdarrayCudaInterfaceNoneCUDA(unittest.TestCase):
+
+    def setUp(self):
+        self.arr = cupy.zeros(shape=(2, 3), dtype=cupy.float64)
+
+    def test_cuda_array_interface_hasattr(self):
+        assert not hasattr(self.arr, '__cuda_array_interface__')
+
+    def test_cuda_array_interface_getattr(self):
+        with pytest.raises(AttributeError) as e:
+            getattr(self.arr, '__cuda_array_interface__')
+        assert 'HIP' in str(e.value)
+
+
+@testing.parameterize(
+    *testing.product({
+        'indices_shape': [(2,), (2, 3)],
+        'axis': [None, 0, 1, 2, -1, -2],
+    })
+)
+class TestNdarrayTake(unittest.TestCase):
+
+    shape = (3, 4, 5)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_take(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        if self.axis is None:
+            m = a.size
+        else:
+            m = a.shape[self.axis]
+        i = testing.shaped_arange(self.indices_shape, xp, numpy.int32) % m
+        return wrap_take(a, i, self.axis)
+
+
+@testing.parameterize(
+    *testing.product({
+        'indices': [2, [0, 1], -1, [-1, -2]],
+        'axis': [None, 0, 1, -1, -2],
+    })
+)
+class TestNdarrayTakeWithInt(unittest.TestCase):
+
+    shape = (3, 4, 5)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_take(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        return wrap_take(a, self.indices, self.axis)
+
+
+@testing.parameterize(
+    *testing.product({
+        'indices': [2, [0, 1], -1, [-1, -2]],
+        'axis': [None, 0, 1, -1, -2],
+    })
+)
+class TestNdarrayTakeWithIntWithOutParam(unittest.TestCase):
+
+    shape = (3, 4, 5)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_take(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        r1 = wrap_take(a, self.indices, self.axis)
+        r2 = xp.zeros_like(r1)
+        wrap_take(a, self.indices, self.axis, out=r2)
+        testing.assert_array_equal(r1, r2)
+        return r2
+
+
+@testing.parameterize(
+    *testing.product({
+        'indices': [0, -1, [0], [0, -1]],
+        'axis': [None, 0, -1],
+    })
+)
+class TestScalaNdarrayTakeWithIntWithOutParam(unittest.TestCase):
+
+    shape = ()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_take(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        r1 = wrap_take(a, self.indices, self.axis)
+        r2 = xp.zeros_like(r1)
+        wrap_take(a, self.indices, self.axis, out=r2)
+        testing.assert_array_equal(r1, r2)
+        return r2
+
+
+@testing.parameterize(
+    {'shape': (3, 4, 5), 'indices': (2,), 'axis': 3},
+    {'shape': (), 'indices': (0,), 'axis': 2}
+)
+class TestNdarrayTakeErrorAxisOverRun(unittest.TestCase):
+
+    def test_axis_overrun1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp)
+            with pytest.raises(numpy.AxisError):
+                wrap_take(a, self.indices, axis=self.axis)
+
+    def test_axis_overrun2(self):
+        a = testing.shaped_arange(self.shape, cupy)
+        with pytest.raises(numpy.AxisError):
+            wrap_take(a, self.indices, axis=self.axis)
+
+
+@testing.parameterize(
+    {'shape': (3, 4, 5), 'indices': (2, 3), 'out_shape': (2, 4)},
+    {'shape': (), 'indices': (), 'out_shape': (1,)}
+)
+class TestNdarrayTakeErrorShapeMismatch(unittest.TestCase):
+
+    def test_shape_mismatch(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp)
+            i = testing.shaped_arange(self.indices, xp, numpy.int32) % 3
+            o = testing.shaped_arange(self.out_shape, xp)
+            with pytest.raises(ValueError):
+                wrap_take(a, i, out=o)
+
+
+@testing.parameterize(
+    {'shape': (3, 4, 5), 'indices': (2, 3), 'out_shape': (2, 3)},
+    {'shape': (), 'indices': (), 'out_shape': ()}
+)
+class TestNdarrayTakeErrorTypeMismatch(unittest.TestCase):
+
+    def test_output_type_mismatch(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, numpy.int32)
+            i = testing.shaped_arange(self.indices, xp, numpy.int32) % 3
+            o = testing.shaped_arange(self.out_shape, xp, numpy.float32)
+            with pytest.raises(TypeError):
+                wrap_take(a, i, out=o)
+
+
+@testing.parameterize(
+    {'shape': (0,), 'indices': (0,), 'axis': None},
+    {'shape': (0,), 'indices': (0, 1), 'axis': None},
+    {'shape': (3, 0), 'indices': (2,), 'axis': 0},
+)
+class TestZeroSizedNdarrayTake(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_output_type_mismatch(self, xp):
+        a = testing.shaped_arange(self.shape, xp, numpy.int32)
+        i = testing.shaped_arange(self.indices, xp, numpy.int32)
+        return wrap_take(a, i, axis=self.axis)
+
+
+@testing.parameterize(
+    {'shape': (0,), 'indices': (1,)},
+    {'shape': (0,), 'indices': (1, 1)},
+)
+class TestZeroSizedNdarrayTakeIndexError(unittest.TestCase):
+
+    def test_output_type_mismatch(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, numpy.int32)
+            i = testing.shaped_arange(self.indices, xp, numpy.int32)
+            with pytest.raises(IndexError):
+                wrap_take(a, i)
+
+
+class TestSize(unittest.TestCase):
+
+    @testing.numpy_cupy_equal()
+    def test_size_without_axis(self, xp):
+        x = testing.shaped_arange((3, 4, 5), xp, numpy.int32)
+        return xp.size(x)
+
+    @testing.numpy_cupy_equal()
+    def test_size_with_axis(self, xp):
+        x = testing.shaped_arange((3, 4, 5), xp, numpy.int32)
+        return xp.size(x, 0)
+
+    @testing.numpy_cupy_equal()
+    def test_size_with_negative_axis(self, xp):
+        x = testing.shaped_arange((3, 4, 5), xp, numpy.int32)
+        return xp.size(x, -1)
+
+    @testing.numpy_cupy_equal()
+    def test_size_zero_dim_array(self, xp):
+        x = testing.shaped_arange((), xp, numpy.int32)
+        return xp.size(x)
+
+    def test_size_axis_too_large(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3, 4, 5), xp, numpy.int32)
+            with pytest.raises(IndexError):
+                xp.size(x, 3)
+
+    def test_size_axis_too_small(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3, 4, 5), xp, numpy.int32)
+            with pytest.raises(IndexError):
+                xp.size(x, -4)
+
+    def test_size_zero_dim_array_with_axis(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((), xp, numpy.int32)
+            with pytest.raises(IndexError):
+                xp.size(x, 0)
+
+
+class TestPythonInterface(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_bytes_tobytes(self, xp, dtype):
+        x = testing.shaped_arange((3, 4, 5), xp, dtype)
+        return bytes(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_bytes_tobytes_empty(self, xp, dtype):
+        x = xp.empty((0,), dtype)
+        return bytes(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_bytes_tobytes_empty2(self, xp, dtype):
+        x = xp.empty((3, 0, 4), dtype)
+        return bytes(x)
+
+    # The result of bytes(numpy.array(scalar)) is the same as bytes(scalar)
+    # if scalar is of an integer dtype including bool_. It's spec is
+    # bytes(int): bytes object of size given by the parameter initialized with
+    # null bytes.
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_bytes_tobytes_scalar_array(self, xp, dtype):
+        x = xp.array(3, dtype)
+        return bytes(x)
+
+    @testing.numpy_cupy_equal()
+    def test_format(self, xp):
+        x = xp.array(1.12345)
+        return format(x, '.2f')
+
+
+class TestNdarrayImplicitConversion(unittest.TestCase):
+
+    def test_array(self):
+        a = testing.shaped_arange((3, 4, 5), cupy, numpy.int64)
+        with pytest.raises(TypeError):
+            numpy.asarray(a)
+
+
+class C(cupy.ndarray):
+
+    def __new__(cls, *args, info=None, **kwargs):
+        obj = super().__new__(cls, *args, **kwargs)
+        obj.info = info
+        return obj
+
+    def __array_finalize__(self, obj):
+        if obj is None:
+            return
+        self.info = getattr(obj, 'info', None)
+
+
+class TestNdarraySubclass:
+
+    def test_explicit_constructor_call(self):
+        a = C([0, 1, 2, 3], info='information')
+        assert type(a) is C
+        assert issubclass(type(a), cupy.ndarray)
+        assert a.info == 'information'
diff --git a/tests/cupy_tests/core_tests/test_ndarray_adv_indexing.py b/tests/cupy_tests/core_tests/test_ndarray_adv_indexing.py
new file mode 100644
index 0000000..7b1ea54
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_adv_indexing.py
@@ -0,0 +1,709 @@
+import itertools
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+def perm(iterable):
+    return list(itertools.permutations(iterable))
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(4, 4, 4)],
+        'indexes': (
+            perm(([1, 0], slice(None))) +
+            perm(([1, 0], Ellipsis)) +
+            perm(([1, 2], None, slice(None))) +
+            perm(([1, 0], 1, slice(None))) +
+            perm(([1, 2], slice(0, 2), slice(None))) +
+            perm((1, [1, 2], 1)) +
+            perm(([[1, -1], [0, 3]], slice(None), slice(None))) +
+            perm(([1, 0], [3, 2], slice(None))) +
+            perm((slice(0, 3, 2), [1, 2], [1, 0])) +
+            perm(([1, 0], [2, 1], [3, 1])) +
+            perm(([1, 0], 1, [3, 1])) +
+            perm(([1, 2], [[1, 0], [0, 1], [-1, 1]], slice(None))) +
+            perm((None, [1, 2], [1, 0])) +
+            perm((numpy.array(0), numpy.array(-1))) +
+            perm((numpy.array(0), None)) +
+            perm((1, numpy.array(2), slice(None)))
+        )
+    })
+)
+class TestArrayAdvancedIndexingGetitemPerm:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_adv_getitem(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        return a[self.indexes]
+
+
+@testing.parameterize(
+    {'shape': (2, 3, 4), 'indexes': numpy.array(-1)},
+    {'shape': (2, 3, 4), 'indexes': (None, [1, 0], [0, 2], slice(None))},
+    {'shape': (2, 3, 4), 'indexes': (None, [0, 1], None, [2, 1], slice(None))},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([1, 0])},
+    {'shape': (2, 3, 4), 'indexes': [1]},
+    {'shape': (2, 3, 4), 'indexes': [1, 1]},
+    {'shape': (2, 3, 4), 'indexes': [1, -1]},
+    {'shape': (2, 3, 4), 'indexes': ([0, 1], slice(None), [[2, 1], [3, 1]])},
+    # mask
+    {'shape': (10,), 'indexes': (numpy.random.choice([False, True], (10,)),)},
+    {'shape': (2, 3, 4), 'indexes': (1, numpy.array([True, False, True]))},
+    {'shape': (2, 3, 4), 'indexes': (numpy.array([True, False]), 1)},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), 2, numpy.array([True, False, True, False]))},
+    {'shape': (2, 3, 4), 'indexes': (slice(None), 2, False)},
+    {'shape': (2, 3, 4),
+     'indexes': (numpy.random.choice([False, True], (2, 3, 4)),)},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), numpy.array([True, False, True]))},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), slice(None),
+                 numpy.array([True, False, False, True]))},
+    {'shape': (2, 3, 4),
+     'indexes': (1, 2,
+                 numpy.array([True, False, False, True]))},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), numpy.random.choice([False, True], (3, 4)))},
+    {'shape': (2, 3, 4),
+     'indexes': numpy.random.choice([False, True], (2, 3))},
+    {'shape': (2, 3, 4),
+     'indexes': (1, None, numpy.array([True, False, True]))},
+    # empty arrays
+    {'shape': (2, 3, 4), 'indexes': []},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([], dtype=numpy.int32)},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([[]], dtype=numpy.int32)},
+    {'shape': (2, 3, 4), 'indexes': (slice(None), [])},
+    {'shape': (2, 3, 4), 'indexes': ([], [])},
+    {'shape': (2, 3, 4), 'indexes': ([[]],)},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([], dtype=numpy.bool_)},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), numpy.array([], dtype=numpy.bool_))},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([[], []], dtype=numpy.bool_)},
+    {'shape': (2, 3, 4), 'indexes': numpy.empty((0, 0, 4), bool)},
+    # multiple masks
+    {'shape': (2, 3, 4), 'indexes': (True, [True, False])},
+    {'shape': (2, 3, 4), 'indexes': (False, [True, False])},
+    {'shape': (2, 3, 4), 'indexes': (True, [[1]], slice(1, 2))},
+    {'shape': (2, 3, 4), 'indexes': (False, [[1]], slice(1, 2))},
+    {'shape': (2, 3, 4), 'indexes': (True, [[1]], slice(1, 2), True)},
+    {'shape': (2, 3, 4), 'indexes': (True, [[1]], slice(1, 2), False)},
+    {'shape': (2, 3, 4),
+     'indexes': (Ellipsis, [[1, 1, -3], [0, 2, 2]], [True, False, True, True])
+     },
+    {'shape': (2, 3, 4),
+     'indexes': (numpy.empty((0, 3), bool), numpy.empty(0, bool))},
+    # zero-dim and zero-sized arrays
+    {'shape': (), 'indexes': Ellipsis},
+    {'shape': (), 'indexes': ()},
+    {'shape': (), 'indexes': None},
+    {'shape': (), 'indexes': True},
+    {'shape': (), 'indexes': (True,)},
+    {'shape': (), 'indexes': (False, True, True)},
+    {'shape': (), 'indexes': numpy.ones((), dtype=numpy.bool_)},
+    {'shape': (), 'indexes': numpy.zeros((), dtype=numpy.bool_)},
+    {'shape': (0,), 'indexes': None},
+    {'shape': (0,), 'indexes': ()},
+    {'shape': (2, 0), 'indexes': ([1],)},
+    {'shape': (0, 3), 'indexes': (slice(None), [1])},
+    {'shape': (0,), 'indexes': True},
+    {'shape': (0,), 'indexes': (True,)},
+    {'shape': (0,), 'indexes': (False, True, True)},
+    {'shape': (0,), 'indexes': numpy.ones((), dtype=numpy.bool_)},
+    {'shape': (0,), 'indexes': numpy.zeros((), dtype=numpy.bool_)},
+    # ellipsis
+    {'shape': (2, 3, 4), 'indexes': (1, Ellipsis, 2)},
+    # issue #1512
+    {'shape': (2, 3, 4), 'indexes': (Ellipsis, numpy.array(False))},
+    {'shape': (2, 3, 4), 'indexes': (Ellipsis, numpy.ones((3, 4), bool))},
+    # issue #4799
+    {'shape': (3, 4, 5),
+     'indexes': (slice(None), [0, 1], Ellipsis, [0, 1])},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), [1, 0], Ellipsis, numpy.ones((5, 2), int))},
+    _ids=False,  # Do not generate ids from randomly generated params
+)
+class TestArrayAdvancedIndexingGetitemParametrized:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_adv_getitem(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        return a[self.indexes]
+
+
+@testing.parameterize(
+    # empty arrays (list indexes)
+    {'shape': (2, 3, 4), 'indexes': [[]]},
+    {'shape': (2, 3, 4), 'indexes': [[[]]]},
+    {'shape': (2, 3, 4), 'indexes': [[[[]]]]},
+    {'shape': (2, 3, 4, 5), 'indexes': [[[[]]]]},
+    {'shape': (2, 3, 4, 5), 'indexes': [[[[[]]]]]},
+    # list indexes
+    {'shape': (2, 3, 4), 'indexes': [[1]]},
+    {'shape': (2, 3, 4), 'indexes': [[1, 1]]},
+    {'shape': (2, 3, 4), 'indexes': [[1], [1]]},
+)
+@testing.with_requires('numpy>=1.23')
+class TestArrayAdvancedIndexingGetitemParametrized2:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_adv_getitem(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        return a[self.indexes]
+
+
+@testing.parameterize(
+    # list indexes
+    {'shape': (2, 3, 4), 'indexes': [[1, 1], 1]},
+    {'shape': (2, 3, 4), 'indexes': [[1], slice(1, 2)]},
+    {'shape': (2, 3, 4), 'indexes': [[[1]], slice(1, 2)]},
+)
+@testing.with_requires('numpy>=1.24')
+class TestArrayAdvancedIndexingGetitemParametrizedValueError:
+
+    @testing.for_all_dtypes()
+    def test_adv_getitem(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, dtype)
+            with pytest.raises(ValueError):
+                a[self.indexes]
+
+
+@testing.parameterize(
+    {'shape': (2, 3, 4), 'transpose': (1, 2, 0),
+     'indexes': (slice(None), [1, 0])},
+    {'shape': (2, 3, 4), 'transpose': (1, 0, 2),
+     'indexes': (None, [1, 2], [0, -1])},
+)
+class TestArrayAdvancedIndexingGetitemParametrizedTransp:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_adv_getitem(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        if self.transpose:
+            a = a.transpose(self.transpose)
+        return a[self.indexes]
+
+
+class TestArrayAdvancedIndexingGetitemCupyIndices:
+
+    shape = (2, 3, 4)
+
+    def test_adv_getitem_cupy_indices1(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([1, 0])
+        original_index = index.copy()
+        b = a[index]
+        b_cpu = a.get()[index.get()]
+        testing.assert_array_equal(b, b_cpu)
+        testing.assert_array_equal(original_index, index)
+
+    def test_adv_getitem_cupy_indices2(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([1, 0])
+        original_index = index.copy()
+        b = a[(slice(None), index)]
+        b_cpu = a.get()[(slice(None), index.get())]
+        testing.assert_array_equal(b, b_cpu)
+        testing.assert_array_equal(original_index, index)
+
+    def test_adv_getitem_cupy_indices3(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([True, False])
+        original_index = index.copy()
+        b = a[index]
+        b_cpu = a.get()[index.get()]
+        testing.assert_array_equal(b, b_cpu)
+        testing.assert_array_equal(original_index, index)
+
+    def test_adv_getitem_cupy_indices4(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([4, -5])
+        original_index = index.copy()
+        b = a[index]
+        b_cpu = a.get()[index.get() % self.shape[1]]
+        testing.assert_array_equal(b, b_cpu)
+        testing.assert_array_equal(original_index, index)
+
+    def test_adv_getitem_cupy_indices5(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([4, -5])
+        original_index = index.copy()
+        b = a[[1, 0], index]
+        b_cpu = a.get()[[1, 0], index.get() % self.shape[1]]
+        testing.assert_array_equal(b, b_cpu)
+        testing.assert_array_equal(original_index, index)
+
+
+@testing.parameterize(
+    {'shape': (2**3 + 1, 2**4), 'indexes': (
+        numpy.array([2**3], dtype=numpy.int8),
+        numpy.array([1], dtype=numpy.int8))},
+    {'shape': (2**4 + 1, 2**4), 'indexes': (
+        numpy.array([2**4], dtype=numpy.uint8),
+        numpy.array([1], dtype=numpy.uint8))},
+    {'shape': (2**7 + 1, 2**8), 'indexes': (
+        numpy.array([2**7], dtype=numpy.int16),
+        numpy.array([1], dtype=numpy.int16))},
+    {'shape': (2**8 + 1, 2**8), 'indexes': (
+        numpy.array([2**8], dtype=numpy.uint16),
+        numpy.array([1], dtype=numpy.uint16))},
+    {'shape': (2**7 + 1, 2**8), 'indexes': (
+        numpy.array([2**7], dtype=numpy.int16),
+        numpy.array([1], dtype=numpy.int32))},
+    {'shape': (2**7 + 1, 2**8), 'indexes': (
+        numpy.array([2**7], dtype=numpy.int16),
+        numpy.array([1], dtype=numpy.int8))},
+    # Three-dimensional case
+    {'shape': (2**3 + 1, 3, 2**4), 'indexes': (
+        numpy.array([2**3], dtype=numpy.int8),
+        slice(None),
+        numpy.array([1], dtype=numpy.int8))},
+)
+class TestArrayAdvancedIndexingOverflow:
+
+    def test_getitem(self):
+        a = cupy.arange(numpy.prod(self.shape)).reshape(self.shape)
+        indexes_gpu = []
+        for s in self.indexes:
+            if isinstance(s, numpy.ndarray):
+                s = cupy.array(s)
+            indexes_gpu.append(s)
+        indexes_gpu = tuple(indexes_gpu)
+        b = a[indexes_gpu]
+        b_cpu = a.get()[self.indexes]
+        testing.assert_array_equal(b, b_cpu)
+
+    def test_setitem(self):
+        a_cpu = numpy.arange(numpy.prod(self.shape)).reshape(self.shape)
+        a = cupy.array(a_cpu)
+        indexes_gpu = []
+        for s in self.indexes:
+            if isinstance(s, numpy.ndarray):
+                s = cupy.array(s)
+            indexes_gpu.append(s)
+        indexes_gpu = tuple(indexes_gpu)
+        a[indexes_gpu] = -1
+        a_cpu[self.indexes] = -1
+        testing.assert_array_equal(a, a_cpu)
+
+
+@testing.parameterize(
+    {'shape': (), 'indexes': (-1,)},
+    {'shape': (), 'indexes': (0,)},
+    {'shape': (), 'indexes': (1,)},
+    {'shape': (), 'indexes': ([0],)},
+    {'shape': (), 'indexes': (numpy.array([0]),)},
+    {'shape': (), 'indexes': (numpy.array(0),)},
+    {'shape': (), 'indexes': numpy.array([True])},
+    {'shape': (), 'indexes': numpy.array([False, True, True])},
+    {'shape': (), 'indexes': ([False],)},
+    {'shape': (0,), 'indexes': (-1,)},
+    {'shape': (0,), 'indexes': (0,)},
+    {'shape': (0,), 'indexes': (1,)},
+    {'shape': (0,), 'indexes': ([0],)},
+    {'shape': (0,), 'indexes': (numpy.array([0]),)},
+    {'shape': (0,), 'indexes': (numpy.array(0),)},
+    {'shape': (0,), 'indexes': numpy.array([True])},
+    {'shape': (0,), 'indexes': numpy.array([False, True, True])},
+    {'shape': (0, 1), 'indexes': (0, Ellipsis)},
+    {'shape': (2, 3), 'indexes': (slice(None), [1, 2], slice(None))},
+    {'shape': (2, 3), 'indexes': numpy.array([], dtype=numpy.float64)},
+    {'shape': (3, 4), 'indexes': ([1, 0], [True, True])},
+    {'shape': (2, 3, 4),
+     'indexes': ([True, True], [[True, True, False, False]])},
+    {'shape': (2, 3, 4),
+     'indexes': ([True, True], [[True], [True], [False]])},
+    {'shape': (2, 3, 4), 'indexes': numpy.empty((0, 1), bool)},
+    {'shape': (2, 3, 4),
+     'indexes': (numpy.empty(0, bool), numpy.empty((0, 2), bool))},
+)
+class TestArrayInvalidIndexAdvGetitem:
+
+    def test_invalid_adv_getitem(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp)
+            with pytest.raises(IndexError):
+                a[self.indexes]
+
+
+@testing.parameterize(
+    {'shape': (0,), 'indexes': ([False],)},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), numpy.random.choice([False, True], (3, 1)))},
+    {'shape': (2, 3, 4),
+     'indexes': numpy.random.choice([False, True], (1, 3))},
+    _ids=False,  # Do not generate ids from randomly generated params
+)
+class TestArrayInvalidIndexAdvGetitem2:
+
+    def test_invalid_adv_getitem(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp)
+            with pytest.raises(IndexError):
+                a[self.indexes]
+
+
+@testing.parameterize(
+    {'shape': (2, 3, 4), 'indexes': [1, [1, [1]]]},
+)
+@testing.with_requires('numpy>=1.24')
+class TestArrayInvalidValueAdvGetitem:
+
+    def test_invalid_adv_getitem(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp)
+            with pytest.raises(ValueError):
+                a[self.indexes]
+
+
+@testing.parameterize(
+    # array only
+    {'shape': (2, 3, 4), 'indexes': numpy.array(-1), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([1, 0]), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [1, 0], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [1, -1], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': (slice(None), [1, 2]), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), [[1, 2], [0, -1]],), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), slice(None), [[1, 2], [0, -1]]), 'value': 1},
+    # slice and array
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), slice(1, 2), [[1, 2], [0, -1]]), 'value': 1},
+    # None and array
+    {'shape': (2, 3, 4),
+     'indexes': (None, [1, -1]), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (None, [1, -1], None), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (None, None, None, [1, -1]), 'value': 1},
+    # None, slice and array
+    {'shape': (2, 3, 4),
+     'indexes': (slice(0, 1), None, [1, -1]), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(0, 1), slice(1, 2), [1, -1]), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(0, 1), None, slice(1, 2), [1, -1]), 'value': 1},
+    # mask
+    {'shape': (2, 3, 4),
+     'indexes': numpy.array([True, False]), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (1, numpy.array([True, False, True])), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (numpy.array([True, False]), 1), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), numpy.array([True, False, True])), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), 2, numpy.array([True, False, True, False])),
+     'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), 2, False), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), slice(None),
+                 numpy.random.choice([False, True], (4,))),
+     'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (numpy.random.choice([False, True], (2, 3)),), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), numpy.random.choice([False, True], (3, 4)),),
+     'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (numpy.random.choice([False, True], (2, 3, 4)),), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (1, None, numpy.array([True, False, True])), 'value': 1},
+    # multiple arrays
+    {'shape': (2, 3, 4), 'indexes': ([0, -1], [1, -1]), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': ([0, -1], [1, -1], [2, 1]), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': ([0, -1], 1), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': ([0, -1], slice(None), [1, -1]),
+     'value': 1},
+    {'shape': (2, 3, 4), 'indexes': ([0, -1], 1, 2), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': ([1, 0], slice(None), [[2, 0], [3, 1]]),
+     'value': 1},
+    # multiple arrays and basic indexing
+    {'shape': (2, 3, 4), 'indexes': ([0, -1], None, [1, 0]), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': ([0, -1], slice(0, 2), [1, 0]),
+     'value': 1},
+    {'shape': (2, 3, 4), 'indexes': ([0, -1], None, slice(0, 2), [1, 0]),
+     'value': 1},
+    {'shape': (1, 1, 2, 3, 4),
+     'indexes': (None, slice(None), slice(None), [1, 0], [2, -1], 1),
+     'value': 1},
+    {'shape': (1, 1, 2, 3, 4),
+     'indexes': (None, slice(None), 0, [1, 0], slice(0, 2, 2), [2, -1]),
+     'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), [0, -1], [[1, 0], [0, 1], [-1, 1]]), 'value': 1},
+    # empty arrays
+    {'shape': (2, 3, 4), 'indexes': [], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [],
+     'value': numpy.array([1, 1, 1, 1])},
+    {'shape': (2, 3, 4), 'indexes': [],
+     'value': numpy.random.uniform(size=(3, 4))},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([], dtype=numpy.int32),
+     'value': 1},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([[]], dtype=numpy.int32),
+     'value': numpy.random.uniform(size=(3, 4))},
+    {'shape': (2, 3, 4), 'indexes': (slice(None), []),
+     'value': 1},
+    {'shape': (2, 3, 4), 'indexes': ([], []),
+     'value': 1},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([], dtype=numpy.bool_),
+     'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), numpy.array([], dtype=numpy.bool_)),
+     'value': 1},
+    {'shape': (2, 3, 4), 'indexes': numpy.array([[], []], dtype=numpy.bool_),
+     'value': numpy.random.uniform(size=(4,))},
+    {'shape': (2, 3, 4), 'indexes': numpy.empty((0, 0, 4), bool), 'value': 1},
+    # multiple masks
+    {'shape': (2, 3, 4), 'indexes': (True, [True, False]), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': (False, [True, False]), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': (True, [[1]], slice(1, 2)), 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': (False, [[1]], slice(1, 2)), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (True, [[1]], slice(1, 2), True), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (True, [[1]], slice(1, 2), False), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (Ellipsis, [[1, 1, -3], [0, 2, 2]], [True, False, True, True]),
+     'value': [[1, 2, 3], [4, 5, 6]]},
+    {'shape': (2, 3, 4),
+     'indexes': (numpy.empty((0, 3), bool), numpy.empty(0, bool)),
+     'value': 1},
+    # zero-dim and zero-sized arrays
+    {'shape': (), 'indexes': Ellipsis, 'value': 1},
+    {'shape': (), 'indexes': (), 'value': 1},
+    {'shape': (), 'indexes': None, 'value': 1},
+    {'shape': (), 'indexes': True, 'value': 1},
+    {'shape': (), 'indexes': (True,), 'value': 1},
+    {'shape': (), 'indexes': (False, True, True), 'value': 1},
+    {'shape': (), 'indexes': numpy.ones((), dtype=numpy.bool_), 'value': 1},
+    {'shape': (), 'indexes': numpy.zeros((), dtype=numpy.bool_), 'value': 1},
+    {'shape': (0,), 'indexes': None, 'value': 1},
+    {'shape': (0,), 'indexes': (), 'value': 1},
+    {'shape': (0,), 'indexes': True, 'value': 1},
+    {'shape': (0,), 'indexes': (True,), 'value': 1},
+    {'shape': (0,), 'indexes': (False, True, True), 'value': 1},
+    {'shape': (0,), 'indexes': numpy.ones((), dtype=numpy.bool_), 'value': 1},
+    {'shape': (0,), 'indexes': numpy.zeros((), dtype=numpy.bool_), 'value': 1},
+    # ellipsis
+    {'shape': (2, 3, 4), 'indexes': (1, Ellipsis, 2), 'value': 1},
+    # issue #1512
+    {'shape': (2, 3, 4),
+     'indexes': (Ellipsis, numpy.array(False)), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (Ellipsis, numpy.ones((3, 4), bool)), 'value': 1},
+    # issue #4799
+    {'shape': (3, 4, 5),
+     'indexes': (slice(None), [0, 1], Ellipsis, [0, 1]), 'value': 1},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), [1, 0], Ellipsis, numpy.ones((5, 2), int)),
+     'value': 1},
+    _ids=False,  # Do not generate ids from randomly generated params
+)
+class TestArrayAdvancedIndexingSetitemScalarValue:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_adv_setitem(self, xp, dtype):
+        a = xp.zeros(self.shape, dtype=dtype)
+        a[self.indexes] = self.value
+        return a
+
+
+@testing.parameterize(
+    # empty arrays (list indexes)
+    {'shape': (2, 3, 4), 'indexes': [[]], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [[[]]], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [[[[]]]], 'value': 1},
+    {'shape': (2, 3, 4, 5), 'indexes': [[[[]]]], 'value': 1},
+    {'shape': (2, 3, 4, 5), 'indexes': [[[[[]]]]], 'value': 1},
+    # list indexes
+    {'shape': (2, 3, 4), 'indexes': [[1]], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [[1, 0]], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [[1], [0]], 'value': 1},
+)
+@testing.with_requires('numpy>=1.23')
+class TestArrayAdvancedIndexingSetitemScalarValue2:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_adv_setitem(self, xp, dtype):
+        a = xp.zeros(self.shape, dtype=dtype)
+        a[self.indexes] = self.value
+        return a
+
+
+@testing.parameterize(
+    # zero-dim and zero-sized arrays
+    {'shape': (), 'indexes': numpy.array([True]), 'value': 1},
+    {'shape': (), 'indexes': numpy.array([False, True, True]), 'value': 1},
+    {'shape': (0,), 'indexes': numpy.array([True]), 'value': 1},
+    {'shape': (0,), 'indexes': numpy.array([False, True, True]), 'value': 1},
+)
+class TestArrayAdvancedIndexingSetitemScalarValueIndexError:
+
+    def test_adv_setitem(self):
+        for xp in (numpy, cupy):
+            a = xp.zeros(self.shape)
+            with pytest.raises(IndexError):
+                a[self.indexes] = self.value
+
+
+@testing.parameterize(
+    # list indexes
+    {'shape': (2, 3, 4), 'indexes': [[1, 0], 2], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [[1], slice(1, 2)], 'value': 1},
+    {'shape': (2, 3, 4), 'indexes': [[[1]], slice(1, 2)], 'value': 1},
+)
+@testing.with_requires('numpy>=1.24')
+class TestArrayAdvancedIndexingSetitemScalarValueValueError2:
+
+    @testing.for_all_dtypes()
+    def test_adv_setitem(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.zeros(self.shape, dtype=dtype)
+            with pytest.raises(ValueError):
+                a[self.indexes] = self.value
+
+
+@testing.parameterize(
+    {'shape': (2, 3, 4), 'indexes': numpy.array(1),
+     'value': numpy.array([1])},
+    {'shape': (2, 3, 4), 'indexes': numpy.array(1),
+     'value': numpy.array([1, 2, 3, 4])},
+    {'shape': (2, 3, 4), 'indexes': (slice(None), [0, -1]),
+     'value': numpy.arange(2 * 2 * 4).reshape(2, 2, 4)},
+    {'shape': (2, 5, 4), 'indexes': (slice(None), [[0, 2], [1, -1]]),
+     'value': numpy.arange(2 * 2 * 2 * 4).reshape(2, 2, 2, 4)},
+    # mask
+    {'shape': (2, 3, 4), 'indexes': numpy.random.choice([False, True], (2, 3)),
+     'value': numpy.arange(4)},
+    {'shape': (2, 3, 4),
+     'indexes': (slice(None), numpy.array([True, False, True])),
+     'value': numpy.arange(2 * 2 * 4).reshape(2, 2, 4)},
+    {'shape': (2, 3, 4),
+     'indexes': (numpy.array([[True, False, False], [False, True, True]]),),
+     'value': numpy.arange(3 * 4).reshape(3, 4)},
+    {'shape': (2, 2, 2),
+     'indexes': (slice(None), numpy.array([[True, False], [False, True]]),),
+     'value': numpy.arange(2 * 2).reshape(2, 2)},
+    {'shape': (2, 2, 2),
+     'indexes': (numpy.array(
+         [[[True, False], [True, False]], [[True, True], [False, False]]]),),
+     'value': numpy.arange(4)},
+    {'shape': (5,),
+     'indexes': numpy.array([True, False, False, True, True]),
+     'value': numpy.arange(3)},
+    # multiple arrays
+    {'shape': (2, 3, 4), 'indexes': ([1, 0], [2, 1]),
+     'value': numpy.arange(2 * 4).reshape(2, 4)},
+    {'shape': (2, 3, 4), 'indexes': ([1, 0], slice(None), [2, 1]),
+     'value': numpy.arange(2 * 3).reshape(2, 3)},
+    {'shape': (2, 3, 4), 'indexes': ([1, 0], slice(None), [[2, 0], [3, 1]]),
+     'value': numpy.arange(2 * 2 * 3).reshape(2, 2, 3)},
+    {'shape': (2, 3, 4),
+     'indexes': ([[1, 0], [1, 0]], slice(None), [[2, 0], [3, 1]]),
+     'value': numpy.arange(2 * 2 * 3).reshape(2, 2, 3)},
+    {'shape': (2, 3, 4),
+     'indexes': (1, slice(None), [[2, 0], [3, 1]]),
+     'value': numpy.arange(2 * 2 * 3).reshape(2, 2, 3)},
+    # list indexes
+    {'shape': (2, 3, 4), 'indexes': [1],
+     'value': numpy.arange(3 * 4).reshape(3, 4)},
+    _ids=False,  # Do not generate ids from randomly generated params
+)
+class TestArrayAdvancedIndexingVectorValue:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_adv_setitem(self, xp, dtype):
+        a = xp.zeros(self.shape, dtype=dtype)
+        a[self.indexes] = self.value.astype(a.dtype)
+        return a
+
+
+class TestArrayAdvancedIndexingSetitemCupyIndices:
+
+    shape = (2, 3)
+
+    def test_cupy_indices_integer_array_1(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([0, 1])
+        original_index = index.copy()
+        a[:, index] = cupy.array(1.)
+        testing.assert_array_equal(
+            a, cupy.array([[1., 1., 0.], [1., 1., 0.]]))
+        testing.assert_array_equal(index, original_index)
+
+    def test_cupy_indices_integer_array_2(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([3, -5])
+        original_index = index.copy()
+        a[:, index] = cupy.array(1.)
+        testing.assert_array_equal(
+            a, cupy.array([[1., 1., 0.], [1., 1., 0.]]))
+        testing.assert_array_equal(index, original_index)
+
+    def test_cupy_indices_integer_array_3(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([3, -5])
+        original_index = index.copy()
+        a[[1, 1], index] = cupy.array(1.)
+        testing.assert_array_equal(
+            a, cupy.array([[0., 0., 0.], [1., 1., 0.]]))
+        testing.assert_array_equal(index, original_index)
+
+    def test_cupy_indices_boolean_array(self):
+        a = cupy.zeros(self.shape)
+        index = cupy.array([True, False])
+        original_index = index.copy()
+        a[index] = cupy.array(1.)
+        testing.assert_array_equal(
+            a, cupy.array([[1., 1., 1.], [0., 0., 0.]]))
+        testing.assert_array_almost_equal(original_index, index)
+
+
+class TestArrayAdvancedIndexingSetitemDifferentDtypes:
+
+    @testing.for_all_dtypes_combination(names=['src_dtype', 'dst_dtype'],
+                                        no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_differnt_dtypes(self, xp, src_dtype, dst_dtype):
+        shape = (2, 3)
+        a = xp.zeros(shape, dtype=src_dtype)
+        indexes = xp.array([0, 1])
+        a[:, indexes] = xp.array(1, dtype=dst_dtype)
+        return a
+
+    @testing.for_all_dtypes_combination(names=['src_dtype', 'dst_dtype'],
+                                        no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_differnt_dtypes_mask(self, xp, src_dtype, dst_dtype):
+        shape = (2, 3)
+        a = xp.zeros(shape, dtype=src_dtype)
+        indexes = xp.array([True, False])
+        a[indexes] = xp.array(1, dtype=dst_dtype)
+        return a
+
+
+class TestArrayAdvancedIndexingSetitemTranspose:
+
+    @testing.numpy_cupy_array_equal()
+    def test_adv_setitem_transp(self, xp):
+        shape = (2, 3, 4)
+        a = xp.zeros(shape).transpose(0, 2, 1)
+        slices = (numpy.array([1, 0]), slice(None), numpy.array([2, 1]))
+        a[slices] = 1
+        return a
diff --git a/tests/cupy_tests/core_tests/test_ndarray_complex_ops.py b/tests/cupy_tests/core_tests/test_ndarray_complex_ops.py
new file mode 100644
index 0000000..7885d90
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_complex_ops.py
@@ -0,0 +1,166 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestConj(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal()
+    def test_conj(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        return x.conj()
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_almost_equal()
+    def test_conj_pass(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        y = x.conj()
+        assert x is y
+        return y
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal()
+    def test_conjugate(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        return x.conjugate()
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_almost_equal()
+    def test_conjugate_pass(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        y = x.conjugate()
+        assert x is y
+        return y
+
+
+class TestAngle(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal()
+    def test_angle(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.angle(x)
+
+
+class TestRealImag(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_real(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        return x.real
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_real_zero_dim(self, xp, dtype):
+        x = xp.array(1, dtype=dtype)
+        return x.real
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_real_non_contiguous(self, xp, dtype):
+        x = testing.shaped_arange((2, 3, 2), xp, dtype).transpose(0, 2, 1)
+        return x.real
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_imag(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        return x.imag
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_imag_zero_dim(self, xp, dtype):
+        x = xp.array(1, dtype=dtype)
+        return x.imag
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_imag_non_contiguous(self, xp, dtype):
+        x = testing.shaped_arange((2, 3, 2), xp, dtype).transpose(0, 2, 1)
+        return x.imag
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_real_setter(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        x.real = testing.shaped_reverse_arange((2, 3), xp, dtype).real
+        return x
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_real_setter_zero_dim(self, xp, dtype):
+        x = xp.array(1, dtype=dtype)
+        x.real = 2
+        return x
+
+    @testing.for_dtypes('FD')
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_real_setter_non_contiguous(self, xp, dtype):
+        x = testing.shaped_arange((2, 3, 2), xp, dtype).transpose(0, 2, 1)
+        x.real = testing.shaped_reverse_arange((2, 2, 3), xp, dtype).real
+        return x
+
+    @testing.for_dtypes('FD')
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_imag_setter(self, xp, dtype):
+        x = testing.shaped_arange((2, 3), xp, dtype)
+        x.imag = testing.shaped_reverse_arange((2, 3), xp, dtype).real
+        return x
+
+    @testing.for_dtypes('FD')
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_imag_setter_zero_dim(self, xp, dtype):
+        x = xp.array(1, dtype=dtype)
+        x.imag = 2
+        return x
+
+    @testing.for_dtypes('FD')
+    @testing.numpy_cupy_array_almost_equal(accept_error=False)
+    def test_imag_setter_non_contiguous(self, xp, dtype):
+        x = testing.shaped_arange((2, 3, 2), xp, dtype).transpose(0, 2, 1)
+        x.imag = testing.shaped_reverse_arange((2, 2, 3), xp, dtype).real
+        return x
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_imag_setter_raise(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((2, 3), xp, dtype)
+            with pytest.raises(TypeError):
+                x.imag = testing.shaped_reverse_arange((2, 3), xp, dtype)
+
+    @testing.for_all_dtypes()
+    def test_real_inplace(self, dtype):
+        x = cupy.zeros((2, 3), dtype=dtype)
+        x.real[:] = 1
+        expected = cupy.ones((2, 3), dtype=dtype)
+        assert cupy.all(x == expected)
+
+    @testing.for_all_dtypes()
+    def test_imag_inplace(self, dtype):
+        x = cupy.zeros((2, 3), dtype=dtype)
+
+        # TODO(kmaehashi) The following line should raise error for real
+        # dtypes, but currently ignored silently.
+        x.imag[:] = 1
+
+        expected = cupy.zeros((2, 3), dtype=dtype) + (
+            1j if x.dtype.kind == 'c' else 0)
+        assert cupy.all(x == expected)
+
+
+class TestScalarConversion(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_scalar_conversion(self, dtype):
+        scalar = 1 + 1j if numpy.dtype(dtype).kind == 'c' else 1
+        x_1d = cupy.array(scalar).astype(dtype)
+        assert complex(x_1d) == scalar
+
+        x_0d = x_1d.reshape(())
+        assert complex(x_0d) == scalar
diff --git a/tests/cupy_tests/core_tests/test_ndarray_contiguity.py b/tests/cupy_tests/core_tests/test_ndarray_contiguity.py
new file mode 100644
index 0000000..b1c7702
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_contiguity.py
@@ -0,0 +1,16 @@
+import unittest
+
+from cupy import testing
+
+
+class TestArrayContiguity(unittest.TestCase):
+
+    def test_is_contiguous(self):
+        a = testing.shaped_arange((2, 3, 4))
+        assert a.flags.c_contiguous is True
+        b = a.transpose(2, 0, 1)
+        assert b.flags.c_contiguous is False
+        c = a[::-1]
+        assert c.flags.c_contiguous is False
+        d = a[:, :, ::2]
+        assert d.flags.c_contiguous is False
diff --git a/tests/cupy_tests/core_tests/test_ndarray_conversion.py b/tests/cupy_tests/core_tests/test_ndarray_conversion.py
new file mode 100644
index 0000000..806e6cc
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_conversion.py
@@ -0,0 +1,59 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupy import testing
+
+
+@testing.parameterize(
+    {'shape': ()},
+    {'shape': (1,)},
+    {'shape': (1, 1, 1)},
+)
+class TestNdarrayItem(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_item(self, xp, dtype):
+        a = xp.full(self.shape, 3, dtype)
+        return a.item()
+
+
+@testing.parameterize(
+    {'shape': (0,)},
+    {'shape': (2, 3)},
+    {'shape': (1, 0, 1)},
+)
+class TestNdarrayItemRaise(unittest.TestCase):
+
+    def test_item(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, xp.float32)
+            with pytest.raises(ValueError):
+                a.item()
+
+
+@testing.parameterize(
+    {'shape': ()},
+    {'shape': (1,)},
+    {'shape': (2, 3)},
+    {'shape': (2, 3), 'order': 'C'},
+    {'shape': (2, 3), 'order': 'F'},
+)
+class TestNdarrayToBytes(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_item(self, xp, dtype):
+        if (runtime.is_hip and
+            (self.shape == (1,) or
+             (self.shape == (2, 3) and not hasattr(self, 'order')))):
+            pytest.xfail('ROCm/HIP may have a bug')
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        if hasattr(self, 'order'):
+            return a.tobytes(self.order)
+        else:
+            return a.tobytes()
diff --git a/tests/cupy_tests/core_tests/test_ndarray_copy_and_view.py b/tests/cupy_tests/core_tests/test_ndarray_copy_and_view.py
new file mode 100644
index 0000000..19fe1be
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_copy_and_view.py
@@ -0,0 +1,460 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy import _util
+
+
+def astype_without_warning(x, dtype, *args, **kwargs):
+    dtype = numpy.dtype(dtype)
+    if x.dtype.kind == 'c' and dtype.kind not in ['b', 'c']:
+        with testing.assert_warns(numpy.ComplexWarning):
+            return x.astype(dtype, *args, **kwargs)
+    else:
+        return x.astype(dtype, *args, **kwargs)
+
+
+class TestView:
+
+    @testing.numpy_cupy_array_equal()
+    def test_view(self, xp):
+        a = testing.shaped_arange((4,), xp, dtype=numpy.float32)
+        b = a.view(dtype=numpy.int32)
+        b[:] = 0
+        return a
+
+    @testing.for_dtypes([numpy.int16, numpy.int64])
+    @testing.numpy_cupy_array_equal()
+    def test_view_itemsize(self, xp, dtype):
+        a = testing.shaped_arange((4,), xp, dtype=numpy.int32)
+        b = a.view(dtype=dtype)
+        return b
+
+    @testing.numpy_cupy_array_equal()
+    def test_view_0d(self, xp):
+        a = xp.array(1.5, dtype=numpy.float32)
+        return a.view(dtype=numpy.int32)
+
+    @testing.for_dtypes([numpy.int16, numpy.int64])
+    def test_view_0d_raise(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.array(3, dtype=numpy.int32)
+            with pytest.raises(ValueError):
+                a.view(dtype=dtype)
+
+    @testing.for_dtypes([numpy.int16, numpy.int64])
+    def test_view_non_contiguous_raise(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 2, 2), xp, dtype=numpy.int32)
+            a = a.transpose(0, 2, 1)
+            with pytest.raises(ValueError):
+                a.view(dtype=dtype)
+
+    @testing.for_dtypes([numpy.int16, numpy.int64])
+    @testing.with_requires('numpy>=1.23')
+    def test_view_f_contiguous(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 2, 2), xp, dtype=numpy.float32)
+            a = a.T
+            with pytest.raises(ValueError):
+                a.view(dtype=dtype)
+
+    def test_view_assert_divisible(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3,), xp, dtype=numpy.int32)
+            with pytest.raises(ValueError):
+                a.view(dtype=numpy.int64)
+
+    @testing.for_dtypes([numpy.float32, numpy.float64])
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_view_relaxed_contiguous(self, xp, dtype):
+        a = testing.shaped_arange((1, 3, 5), xp, dtype=dtype)
+        a = xp.moveaxis(a, 0, 2)  # (3, 5, 1)
+        b = a.view(dtype=numpy.int32)
+        return b
+
+    @pytest.mark.parametrize(('order', 'shape'), [
+        ('C', (3,)),
+        ('C', (3, 5)),
+        ('C', (0,)),
+        ('C', (1, 3)),
+        ('C', (3, 1)),
+    ], ids=str)
+    @testing.numpy_cupy_equal()
+    def test_view_flags_smaller(self, xp, order, shape):
+        a = xp.zeros(shape, numpy.int32, order)
+        b = a.view(numpy.int16)
+        return b.flags.c_contiguous, b.flags.f_contiguous, b.flags.owndata
+
+    @pytest.mark.parametrize(('order', 'shape'), [
+        ('F', (3, 5)),
+    ], ids=str)
+    @testing.with_requires('numpy>=1.23')
+    def test_view_flags_smaller_invalid(self, order, shape):
+        for xp in (numpy, cupy):
+            a = xp.zeros(shape, numpy.int32, order)
+            with pytest.raises(ValueError):
+                a.view(numpy.int16)
+
+    @pytest.mark.parametrize(('order', 'shape'), [
+        ('C', (6,)),
+        ('C', (3, 10)),
+        ('C', (0,)),
+        ('C', (1, 6)),
+        ('C', (3, 2)),
+    ], ids=str)
+    @testing.numpy_cupy_equal()
+    def test_view_flags_larger(self, xp, order, shape):
+        a = xp.zeros(shape, numpy.int16, order)
+        b = a.view(numpy.int32)
+        return b.flags.c_contiguous, b.flags.f_contiguous, b.flags.owndata
+
+    @pytest.mark.parametrize(('order', 'shape'), [
+        ('F', (6, 5)),
+        ('F', (2, 3)),
+    ], ids=str)
+    @testing.with_requires('numpy>=1.23')
+    def test_view_flags_larger_invalid(self, order, shape):
+        for xp in (numpy, cupy):
+            a = xp.zeros(shape, numpy.int16, order)
+            with pytest.raises(ValueError):
+                a.view(numpy.int32)
+
+    @testing.with_requires('numpy>=1.23')
+    @testing.numpy_cupy_array_equal()
+    def test_view_smaller_dtype_multiple(self, xp):
+        # x is non-contiguous
+        x = xp.arange(10, dtype=xp.int32)[::2]
+        with pytest.raises(ValueError):
+            x.view(xp.int16)
+        return x[:, xp.newaxis].view(xp.int16)
+
+    @testing.with_requires('numpy>=1.23')
+    @testing.numpy_cupy_array_equal()
+    def test_view_smaller_dtype_multiple2(self, xp):
+        # x is non-contiguous, and stride[-1] != 0
+        x = xp.ones((3, 4), xp.int32)[:, :1:2]
+        return x.view(xp.int16)
+
+    @testing.with_requires('numpy>=1.23')
+    @testing.numpy_cupy_array_equal()
+    def test_view_larger_dtype_multiple(self, xp):
+        # x is non-contiguous in the first dimension, contiguous in the last
+        x = xp.arange(20, dtype=xp.int16).reshape(10, 2)[::2, :]
+        return x.view(xp.int32)
+
+    @testing.with_requires('numpy>=1.23')
+    @testing.numpy_cupy_array_equal()
+    def test_view_non_c_contiguous(self, xp):
+        # x is contiguous in axis=-1, but not C-contiguous in other axes
+        x = xp.arange(2 * 3 * 4, dtype=xp.int8).reshape(
+            2, 3, 4).transpose(1, 0, 2)
+        return x.view(xp.int16)
+
+    @testing.numpy_cupy_array_equal()
+    def test_view_larger_dtype_zero_sized(self, xp):
+        x = xp.ones((3, 20), xp.int16)[:0, ::2]
+        return x.view(xp.int32)
+
+
+class TestArrayCopy:
+
+    @pytest.mark.skipif(
+        not _util.ENABLE_SLICE_COPY, reason='Special copy disabled')
+    @testing.for_orders('CF')
+    @testing.for_dtypes([numpy.int16, numpy.int64,
+                         numpy.float16, numpy.float64])
+    @testing.numpy_cupy_array_equal()
+    def test_isinstance_numpy_copy(self, xp, dtype, order):
+        a = numpy.arange(100, dtype=dtype).reshape(10, 10, order=order)
+        b = xp.empty(a.shape, dtype=dtype, order=order)
+        b[:] = a
+        return b
+
+    @pytest.mark.skipif(
+        not _util.ENABLE_SLICE_COPY, reason='Special copy disabled')
+    def test_isinstance_numpy_copy_wrong_dtype(self):
+        a = numpy.arange(100, dtype=numpy.float64).reshape(10, 10)
+        b = cupy.empty(a.shape, dtype=numpy.int32)
+        with pytest.raises(ValueError):
+            b[:] = a
+
+    @pytest.mark.skipif(
+        not _util.ENABLE_SLICE_COPY, reason='Special copy disabled')
+    def test_isinstance_numpy_copy_wrong_shape(self):
+        for xp in (numpy, cupy):
+            a = numpy.arange(100, dtype=numpy.float64).reshape(10, 10)
+            b = cupy.empty(100, dtype=a.dtype)
+            with pytest.raises(ValueError):
+                b[:] = a
+
+    @pytest.mark.skipif(
+        not _util.ENABLE_SLICE_COPY, reason='Special copy disabled')
+    @testing.numpy_cupy_array_equal()
+    def test_isinstance_numpy_copy_not_slice(self, xp):
+        a = xp.arange(5, dtype=numpy.float64)
+        a[a < 3] = 0
+        return a
+
+    @pytest.mark.skipif(
+        not _util.ENABLE_SLICE_COPY, reason='Special copy disabled')
+    def test_copy_host_to_device_view(self):
+        dev = cupy.empty((10, 10), dtype=numpy.float32)[2:5, 1:8]
+        host = numpy.arange(3 * 7, dtype=numpy.float32).reshape(3, 7)
+        with pytest.raises(ValueError):
+            dev[:] = host
+
+
+class TestArrayFlatten:
+
+    @testing.numpy_cupy_array_equal()
+    def test_flatten(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.flatten()
+
+    @testing.numpy_cupy_array_equal()
+    def test_flatten_copied(self, xp):
+        a = testing.shaped_arange((4,), xp)
+        b = a.flatten()
+        a[:] = 1
+        return b
+
+    @testing.numpy_cupy_array_equal()
+    def test_flatten_transposed(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp).transpose(2, 0, 1)
+        return a.flatten()
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_flatten_order(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.flatten(order)
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_flatten_order_copied(self, xp, order):
+        a = testing.shaped_arange((4,), xp)
+        b = a.flatten(order=order)
+        a[:] = 1
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_flatten_order_transposed(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp).transpose(2, 0, 1)
+        return a.flatten(order=order)
+
+
+class TestArrayFill:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_fill(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        a.fill(1)
+        return a
+
+    @testing.with_requires('numpy>=1.24.0')
+    @testing.for_all_dtypes_combination(('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=numpy.ComplexWarning)
+    def test_fill_with_numpy_scalar_ndarray(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype1)
+        a.fill(numpy.ones((), dtype=dtype2))
+        return a
+
+    @testing.with_requires('numpy>=1.24.0')
+    @testing.for_all_dtypes_combination(('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=numpy.ComplexWarning)
+    def test_fill_with_cupy_scalar_ndarray(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype1)
+        b = xp.ones((), dtype=dtype2)
+        a.fill(b)
+        return a
+
+    @testing.for_all_dtypes()
+    def test_fill_with_nonscalar_ndarray(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                a.fill(xp.ones((1,), dtype=dtype))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_transposed_fill(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = a.transpose(2, 0, 1)
+        b.fill(1)
+        return b
+
+
+class TestArrayAsType:
+
+    @testing.for_orders(['C', 'F', 'A', 'K', None])
+    @testing.for_all_dtypes_combination(('src_dtype', 'dst_dtype'))
+    @testing.numpy_cupy_array_equal()
+    def test_astype(self, xp, src_dtype, dst_dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, src_dtype)
+        return astype_without_warning(a, dst_dtype, order=order)
+
+    @testing.for_orders(['C', 'F', 'A', 'K', None])
+    @testing.for_all_dtypes_combination(('src_dtype', 'dst_dtype'))
+    @testing.numpy_cupy_array_equal()
+    def test_astype_empty(self, xp, src_dtype, dst_dtype, order):
+        a = testing.shaped_arange((2, 0, 4), xp, src_dtype)
+        return astype_without_warning(a, dst_dtype, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes_combination(('src_dtype', 'dst_dtype'))
+    def test_astype_type(self, src_dtype, dst_dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, src_dtype)
+        b = astype_without_warning(a, dst_dtype, order=order)
+        a_cpu = testing.shaped_arange((2, 3, 4), numpy, src_dtype)
+        b_cpu = astype_without_warning(a_cpu, dst_dtype, order=order)
+        assert b.dtype.type == b_cpu.dtype.type
+
+    @testing.for_orders('CAK')
+    @testing.for_all_dtypes()
+    def test_astype_type_c_contiguous_no_copy(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = a.astype(dtype, order=order, copy=False)
+        assert b is a
+
+    @testing.for_orders('FAK')
+    @testing.for_all_dtypes()
+    def test_astype_type_f_contiguous_no_copy(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        a = cupy.asfortranarray(a)
+        b = a.astype(dtype, order=order, copy=False)
+        assert b is a
+
+    @testing.for_all_dtypes_combination(('src_dtype', 'dst_dtype'))
+    @testing.numpy_cupy_equal()
+    def test_astype_strides(self, xp, src_dtype, dst_dtype):
+        src = xp.empty((1, 2, 3), dtype=src_dtype)
+        return astype_without_warning(src, dst_dtype, order='K').strides
+
+    @testing.for_all_dtypes_combination(('src_dtype', 'dst_dtype'))
+    @testing.numpy_cupy_equal()
+    def test_astype_strides_negative(self, xp, src_dtype, dst_dtype):
+        src = xp.empty((2, 3), dtype=src_dtype)[::-1, :]
+        return astype_without_warning(src, dst_dtype, order='K').strides
+
+    @testing.for_all_dtypes_combination(('src_dtype', 'dst_dtype'))
+    @testing.numpy_cupy_equal()
+    def test_astype_strides_swapped(self, xp, src_dtype, dst_dtype):
+        src = xp.swapaxes(xp.empty((2, 3, 4), dtype=src_dtype), 1, 0)
+        return astype_without_warning(src, dst_dtype, order='K').strides
+
+    @testing.for_all_dtypes_combination(('src_dtype', 'dst_dtype'))
+    @testing.numpy_cupy_equal()
+    def test_astype_strides_broadcast(self, xp, src_dtype, dst_dtype):
+        src, _ = xp.broadcast_arrays(xp.empty((2,), dtype=src_dtype),
+                                     xp.empty((2, 3, 2), dtype=src_dtype))
+        return astype_without_warning(src, dst_dtype, order='K').strides
+
+    @testing.numpy_cupy_array_equal()
+    def test_astype_boolean_view(self, xp):
+        # See #4354
+        a = xp.array([0, 1, 2], dtype=numpy.int8).view(dtype=numpy.bool_)
+        return a.astype(numpy.int8)
+
+
+class TestArrayDiagonal:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_diagonal1(self, xp, dtype):
+        a = testing.shaped_arange((3, 4, 5), xp, dtype)
+        return a.diagonal(1, 2, 0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_diagonal2(self, xp, dtype):
+        a = testing.shaped_arange((3, 4, 5), xp, dtype)
+        return a.diagonal(-1, 2, 0)
+
+
+@testing.parameterize(
+    {'src_order': 'C'},
+    {'src_order': 'F'},
+)
+class TestNumPyArrayCopyView:
+    @pytest.mark.skipif(
+        not _util.ENABLE_SLICE_COPY, reason='Special copy disabled')
+    @testing.for_orders('CF')
+    @testing.for_dtypes([numpy.int16, numpy.int64,
+                         numpy.float16, numpy.float64])
+    @testing.numpy_cupy_array_equal()
+    def test_isinstance_numpy_view_copy_f(self, xp, dtype, order):
+        a = numpy.arange(100, dtype=dtype).reshape(
+            10, 10, order=self.src_order)
+        a = a[2:5, 1:8]
+        b = xp.empty(a.shape, dtype=dtype, order=order)
+        b[:] = a
+        return b
+
+
+class C_cp(cupy.ndarray):
+
+    def __new__(cls, *args, info=None, **kwargs):
+        obj = super().__new__(cls, *args, **kwargs)
+        obj.info = info
+        return obj
+
+    def __array_finalize__(self, obj):
+        if obj is None:
+            return
+        self.info = getattr(obj, 'info', None)
+
+
+class C_np(numpy.ndarray):
+
+    def __new__(cls, *args, info=None, **kwargs):
+        obj = super().__new__(cls, *args, **kwargs)
+        obj.info = info
+        return obj
+
+    def __array_finalize__(self, obj):
+        if obj is None:
+            return
+        self.info = getattr(obj, 'info', None)
+
+
+class TestSubclassArrayView:
+
+    def test_view_casting(self):
+        for xp, C in [(numpy, C_np), (cupy, C_cp)]:
+            a = xp.arange(5, dtype='i').view('f')
+            assert type(a) is xp.ndarray
+            assert a.dtype == xp.float32
+
+            a = xp.arange(5, dtype='i').view(dtype='f')
+            assert type(a) is xp.ndarray
+            assert a.dtype == xp.float32
+
+            with pytest.raises(TypeError):
+                xp.arange(5, dtype='i').view('f', dtype='f')
+
+            a = xp.arange(5, dtype='i').view(C)
+            assert type(a) is C
+            assert a.dtype == xp.int32
+            assert a.info is None
+
+            a = xp.arange(5, dtype='i').view(type=C)
+            assert type(a) is C
+            assert a.dtype == xp.int32
+            assert a.info is None
+
+            # When an instance of ndarray's subclass is supplied to `dtype`,
+            # view() interprets it as if it is supplied to `type`
+            a = xp.arange(5, dtype='i').view(dtype=C)
+            assert type(a) is C
+            assert a.dtype == xp.int32
+            assert a.info is None
+
+            with pytest.raises(TypeError):
+                xp.arange(5).view('f', C, type=C)
+
+        with pytest.raises(ValueError):
+            cupy.arange(5).view(type=numpy.ndarray)
diff --git a/tests/cupy_tests/core_tests/test_ndarray_cuda_array_interface.py b/tests/cupy_tests/core_tests/test_ndarray_cuda_array_interface.py
new file mode 100644
index 0000000..6412271
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_cuda_array_interface.py
@@ -0,0 +1,314 @@
+import unittest
+import pytest
+
+from cupy_backends.cuda import stream as stream_module
+import cupy
+from cupy import _core
+from cupy import testing
+
+
+# TODO(leofang): test PTDS in this file
+
+
+class DummyObjectWithCudaArrayInterface(object):
+
+    def __init__(self, a, ver=3):
+        self.a = a
+        self.ver = ver
+
+    @property
+    def __cuda_array_interface__(self):
+        desc = {
+            'shape': self.a.shape,
+            'strides': self.a.strides,
+            'typestr': self.a.dtype.str,
+            'descr': self.a.dtype.descr,
+            'data': (self.a.data.ptr, False),
+            'version': self.ver,
+        }
+        if self.ver == 3:
+            stream = cupy.cuda.get_current_stream()
+            desc['stream'] = 1 if stream.ptr == 0 else stream.ptr
+        return desc
+
+
+@testing.parameterize(*testing.product({
+    'stream': ('null', 'new'),
+    'ver': (2, 3),
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestArrayUfunc(unittest.TestCase):
+
+    def setUp(self):
+        if self.stream == 'null':
+            self.stream = cupy.cuda.Stream.null
+        elif self.stream == 'new':
+            self.stream = cupy.cuda.Stream()
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(rtol=1e-6, accept_error=TypeError,
+                                 contiguous_check=False)
+    def check_array_scalar_op(self, op, xp, x_type, y_type, trans=False):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        if trans:
+            a = a.T
+
+        if xp is cupy:
+            with self.stream:
+                a = DummyObjectWithCudaArrayInterface(a, self.ver)
+                return getattr(xp, op)(a, y_type(3))
+        else:
+            return getattr(xp, op)(a, y_type(3))
+
+    def test_add_scalar(self):
+        self.check_array_scalar_op('add')
+
+    def test_add_scalar_with_strides(self):
+        self.check_array_scalar_op('add', trans=True)
+
+
+@testing.parameterize(*testing.product({
+    'stream': ('null', 'new'),
+    'ver': (2, 3),
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestElementwiseKernel(unittest.TestCase):
+
+    def setUp(self):
+        if self.stream == 'null':
+            self.stream = cupy.cuda.Stream.null
+        elif self.stream == 'new':
+            self.stream = cupy.cuda.Stream()
+
+    @testing.for_all_dtypes_combination()
+    @testing.numpy_cupy_allclose(rtol=1e-6, accept_error=TypeError,
+                                 contiguous_check=False)
+    def check_array_scalar_op(self, op, xp, dtyes, trans=False):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], dtyes)
+        if trans:
+            a = a.T
+
+        if xp is cupy:
+            with self.stream:
+                a = DummyObjectWithCudaArrayInterface(a, self.ver)
+                f = cupy.ElementwiseKernel('T x, T y', 'T z', 'z = x + y')
+                return f(a, dtyes(3))
+        else:
+            return a + dtyes(3)
+
+    def test_add_scalar(self):
+        self.check_array_scalar_op('add')
+
+    def test_add_scalar_with_strides(self):
+        self.check_array_scalar_op('add', trans=True)
+
+
+@testing.parameterize(*testing.product({
+    'stream': ('null', 'new'),
+    'ver': (2, 3),
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestSimpleReductionFunction(unittest.TestCase):
+
+    def setUp(self):
+        if self.stream == 'null':
+            self.stream = cupy.cuda.Stream.null
+        elif self.stream == 'new':
+            self.stream = cupy.cuda.Stream()
+
+        self.my_int8_sum = _core.create_reduction_func(
+            'my_sum', ('b->b',), ('in0', 'a + b', 'out0 = a', None))
+
+    @testing.numpy_cupy_allclose()
+    def check_int8_sum(self, shape, xp, axis=None, keepdims=False,
+                       trans=False):
+        a = testing.shaped_random(shape, xp, 'b')
+        if trans:
+            a = a.T
+
+        if xp == cupy:
+            with self.stream:
+                a = DummyObjectWithCudaArrayInterface(a, self.ver)
+                return self.my_int8_sum(
+                    a, axis=axis, keepdims=keepdims)
+        else:
+            return a.sum(axis=axis, keepdims=keepdims, dtype='b')
+
+    def test_shape(self):
+        self.check_int8_sum((2 ** 10,))
+
+    def test_shape_with_strides(self):
+        self.check_int8_sum((2 ** 10, 16), trans=True)
+
+
+@testing.parameterize(*testing.product({
+    'stream': ('null', 'new'),
+    'ver': (2, 3),
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestReductionKernel(unittest.TestCase):
+
+    def setUp(self):
+        if self.stream == 'null':
+            self.stream = cupy.cuda.Stream.null
+        elif self.stream == 'new':
+            self.stream = cupy.cuda.Stream()
+
+        self.my_sum = _core.ReductionKernel(
+            'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum')
+
+    @testing.numpy_cupy_allclose()
+    def check_int8_sum(self, shape, xp, axis=None, keepdims=False,
+                       trans=False):
+        a = testing.shaped_random(shape, xp, 'b')
+        if trans:
+            a = a.T
+
+        if xp == cupy:
+            with self.stream:
+                a = DummyObjectWithCudaArrayInterface(a, self.ver)
+                return self.my_sum(
+                    a, axis=axis, keepdims=keepdims)
+        else:
+            return a.sum(axis=axis, keepdims=keepdims, dtype='b')
+
+    def test_shape(self):
+        self.check_int8_sum((2 ** 10,))
+
+    def test_shape_with_strides(self):
+        self.check_int8_sum((2 ** 10, 16), trans=True)
+
+
+@testing.parameterize(
+    {'shape': (10,), 'slices': (slice(0, None),)},
+    {'shape': (10,), 'slices': (slice(2, None),)},
+    {'shape': (10, 10), 'slices': (slice(0, None), slice(0, None))},
+    {'shape': (10, 10), 'slices': (slice(0, None), slice(2, None))},
+    {'shape': (10, 10), 'slices': (slice(2, None), slice(0, None))},
+    {'shape': (10, 10), 'slices': (slice(2, None), slice(2, None))},
+    {'shape': (10, 10), 'slices': (slice(2, None), slice(4, None))},
+)
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestSlicingMemoryPointer(unittest.TestCase):
+
+    @testing.for_all_dtypes_combination(names=['dtype'])
+    @testing.for_orders('CF')
+    def test_shape_with_strides(self, dtype, order):
+        x = cupy.zeros(self.shape, dtype=dtype, order=order)
+
+        start = [s.start for s in self.slices]
+        itemsize = cupy.dtype(dtype).itemsize
+        dimsize = [s * itemsize for s in start]
+        if len(self.shape) == 1:
+            offset = start[0] * itemsize
+        else:
+            if order == 'C':
+                offset = self.shape[0] * dimsize[0] + dimsize[1]
+            else:
+                offset = self.shape[0] * dimsize[1] + dimsize[0]
+
+        cai_ptr, _ = x.__cuda_array_interface__['data']
+        slice_cai_ptr, _ = x[self.slices].__cuda_array_interface__['data']
+        cupy_data_ptr = x.data.ptr
+        sliced_cupy_data_ptr = x[self.slices].data.ptr
+
+        assert cai_ptr == cupy_data_ptr
+        assert slice_cai_ptr == sliced_cupy_data_ptr
+        assert slice_cai_ptr == cai_ptr+offset
+
+
+test_cases = [
+    {'shape': (10,), 'slices': (slice(0, None),)},
+    {'shape': (10,), 'slices': (slice(2, None),)},
+    {'shape': (10, 10), 'slices': (slice(0, None), slice(0, None))},
+    {'shape': (10, 10), 'slices': (slice(0, None), slice(2, None))},
+    {'shape': (10, 10), 'slices': (slice(2, None), slice(0, None))},
+    {'shape': (10, 10), 'slices': (slice(2, None), slice(2, None))},
+    {'shape': (10, 10), 'slices': (slice(2, None), slice(4, None))},
+]
+test_streams = ('null', 'new')
+test_cases_with_stream = [
+    {'stream': s, **t} for t in test_cases for s in test_streams]
+
+
+@testing.parameterize(*test_cases_with_stream)
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestCUDAArrayInterfaceCompliance(unittest.TestCase):
+
+    def setUp(self):
+        if self.stream == 'null':
+            self.stream = cupy.cuda.Stream.null
+        elif self.stream == 'new':
+            self.stream = cupy.cuda.Stream()
+
+    @testing.for_all_dtypes_combination(names=['dtype'])
+    @testing.for_orders('CF')
+    def test_value_type(self, dtype, order):
+        x = cupy.zeros(self.shape, dtype=dtype, order=order)
+        y = x[self.slices]
+
+        # mandatory entries
+        with self.stream:
+            CAI = y.__cuda_array_interface__
+        shape = CAI['shape']
+        typestr = CAI['typestr']
+        ptr, readonly = CAI['data']
+        version = CAI['version']
+        strides = CAI['strides']
+
+        # optional entries
+        descr = CAI['descr'] if 'descr' in CAI else None
+        stream = CAI['stream'] if 'stream' in CAI else None
+
+        # Don't validate correctness of data here, just their types
+        assert version == 3  # bump this when the protocol is updated!
+        assert isinstance(CAI, dict)
+        assert isinstance(shape, tuple)
+        assert isinstance(typestr, str)
+        assert isinstance(ptr, int)
+        assert isinstance(readonly, bool)
+        assert (strides is None) or isinstance(strides, tuple)
+        assert (descr is None) or isinstance(descr, list)
+        if isinstance(descr, list):
+            for item in descr:
+                assert isinstance(item, tuple)
+        assert (stream is None) or isinstance(stream, int)
+
+
+@testing.parameterize(*testing.product({
+    'stream': ('null', 'new', 'ptds'),
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestCUDAArrayInterfaceStream(unittest.TestCase):
+    def setUp(self):
+        if self.stream == 'null':
+            self.stream = cupy.cuda.Stream.null
+        elif self.stream == 'new':
+            self.stream = cupy.cuda.Stream()
+        elif self.stream == 'ptds':
+            self.stream = cupy.cuda.Stream.ptds
+
+    def test_stream_export(self):
+        a = cupy.empty(100)
+
+        # the stream context should export the stream
+        with self.stream:
+            stream_ptr = a.__cuda_array_interface__['stream']
+
+        if self.stream is cupy.cuda.Stream.null:
+            assert stream_ptr == stream_module.get_default_stream_ptr()
+        elif self.stream is cupy.cuda.Stream.ptds:
+            assert stream_ptr == 2
+        else:
+            assert stream_ptr == self.stream.ptr
+
+        # without a stream context, it's always the default stream
+        stream_ptr = a.__cuda_array_interface__['stream']
+        assert stream_ptr == stream_module.get_default_stream_ptr()
diff --git a/tests/cupy_tests/core_tests/test_ndarray_elementwise_op.py b/tests/cupy_tests/core_tests/test_ndarray_elementwise_op.py
new file mode 100644
index 0000000..7902d37
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_elementwise_op.py
@@ -0,0 +1,775 @@
+import operator
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestArrayElementwiseOp:
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(rtol=1e-6, accept_error=TypeError)
+    def check_array_scalar_op(self, op, xp, x_type, y_type, swap=False,
+                              no_bool=False, no_complex=False):
+        x_dtype = numpy.dtype(x_type)
+        y_dtype = numpy.dtype(y_type)
+        if no_bool and x_dtype == '?' and y_dtype == '?':
+            return xp.array(True)
+        if no_complex and (x_dtype.kind == 'c' or y_dtype.kind == 'c'):
+            return xp.array(True)
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        if swap:
+            return op(y_type(3), a)
+        else:
+            return op(a, y_type(3))
+
+    def test_add_scalar(self):
+        self.check_array_scalar_op(operator.add)
+
+    def test_radd_scalar(self):
+        self.check_array_scalar_op(operator.add, swap=True)
+
+    def test_iadd_scalar(self):
+        self.check_array_scalar_op(operator.iadd)
+
+    def test_sub_scalar(self):
+        self.check_array_scalar_op(operator.sub, no_bool=True)
+
+    def test_rsub_scalar(self):
+        self.check_array_scalar_op(operator.sub, swap=True, no_bool=True)
+
+    def test_isub_scalar(self):
+        self.check_array_scalar_op(operator.isub, no_bool=True)
+
+    def test_mul_scalar(self):
+        self.check_array_scalar_op(operator.mul)
+
+    def test_rmul_scalar(self):
+        self.check_array_scalar_op(operator.mul, swap=True)
+
+    def test_imul_scalar(self):
+        self.check_array_scalar_op(operator.imul)
+
+    def test_truediv_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(operator.truediv)
+
+    def test_rtruediv_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(operator.truediv, swap=True)
+
+    def test_itruediv_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(operator.itruediv)
+
+    def test_floordiv_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(operator.floordiv, no_complex=True)
+
+    def test_rfloordiv_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(operator.floordiv, swap=True,
+                                       no_complex=True)
+
+    def test_ifloordiv_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(operator.ifloordiv, no_complex=True)
+
+    def test_pow_scalar(self):
+        self.check_array_scalar_op(operator.pow)
+
+    def test_rpow_scalar(self):
+        self.check_array_scalar_op(operator.pow, swap=True)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(atol=1.0, accept_error=TypeError)
+    def check_ipow_scalar(self, xp, x_type, y_type):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        return operator.ipow(a, y_type(3))
+
+    def test_ipow_scalar(self):
+        self.check_ipow_scalar()
+
+    def test_divmod0_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(lambda x, y: divmod(x, y)[0],
+                                       no_complex=True)
+
+    def test_divmod1_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(lambda x, y: divmod(x, y)[1],
+                                       no_complex=True)
+
+    def test_rdivmod0_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(lambda x, y: divmod(x, y)[0], swap=True,
+                                       no_complex=True)
+
+    def test_rdivmod1_scalar(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_scalar_op(lambda x, y: divmod(x, y)[1], swap=True,
+                                       no_complex=True)
+
+    def test_lt_scalar(self):
+        self.check_array_scalar_op(operator.lt, no_complex=False)
+
+    def test_le_scalar(self):
+        self.check_array_scalar_op(operator.le, no_complex=False)
+
+    def test_gt_scalar(self):
+        self.check_array_scalar_op(operator.gt, no_complex=False)
+
+    def test_ge_scalar(self):
+        self.check_array_scalar_op(operator.ge, no_complex=False)
+
+    def test_eq_scalar(self):
+        self.check_array_scalar_op(operator.eq)
+
+    def test_ne_scalar(self):
+        self.check_array_scalar_op(operator.ne)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_array_array_op(self, op, xp, x_type, y_type,
+                             no_bool=False, no_complex=False):
+        x_dtype = numpy.dtype(x_type)
+        y_dtype = numpy.dtype(y_type)
+        if no_bool and x_dtype == '?' and y_dtype == '?':
+            return xp.array(True)
+        if no_complex and (x_dtype.kind == 'c' or y_dtype.kind == 'c'):
+            return xp.array(True)
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        b = xp.array([[6, 5, 4], [3, 2, 1]], y_type)
+        return op(a, b)
+
+    def test_add_array(self):
+        self.check_array_array_op(operator.add)
+
+    def test_iadd_array(self):
+        self.check_array_array_op(operator.iadd)
+
+    def test_sub_array(self):
+        self.check_array_array_op(operator.sub, no_bool=True)
+
+    def test_isub_array(self):
+        self.check_array_array_op(operator.isub, no_bool=True)
+
+    def test_mul_array(self):
+        self.check_array_array_op(operator.mul)
+
+    def test_imul_array(self):
+        self.check_array_array_op(operator.imul)
+
+    def test_truediv_array(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_array_op(operator.truediv)
+
+    def test_itruediv_array(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_array_op(operator.itruediv)
+
+    def test_floordiv_array(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_array_op(operator.floordiv, no_complex=True)
+
+    def test_ifloordiv_array(self):
+        if '1.16.1' <= numpy.lib.NumpyVersion(numpy.__version__) < '1.18.0':
+            self.skipTest("NumPy Issue #12927")
+        with numpy.errstate(divide='ignore'):
+            self.check_array_array_op(operator.ifloordiv, no_complex=True)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-6, accept_error=TypeError)
+    def check_pow_array(self, xp, x_type, y_type):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        b = xp.array([[6, 5, 4], [3, 2, 1]], y_type)
+        return operator.pow(a, b)
+
+    def test_pow_array(self):
+        # There are some precission issues in HIP that prevent
+        # checking with atol=0
+        if cupy.cuda.runtime.is_hip:
+            self.check_pow_array()
+        else:
+            self.check_array_array_op(operator.pow)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(atol=1.0, accept_error=TypeError)
+    def check_ipow_array(self, xp, x_type, y_type):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        b = xp.array([[6, 5, 4], [3, 2, 1]], y_type)
+        return operator.ipow(a, b)
+
+    def test_ipow_array(self):
+        self.check_ipow_array()
+
+    def test_divmod0_array(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_array_op(lambda x, y: divmod(x, y)[0])
+
+    def test_divmod1_array(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_array_op(lambda x, y: divmod(x, y)[1])
+
+    def test_lt_array(self):
+        self.check_array_array_op(operator.lt, no_complex=True)
+
+    def test_le_array(self):
+        self.check_array_array_op(operator.le, no_complex=True)
+
+    def test_gt_array(self):
+        self.check_array_array_op(operator.gt, no_complex=True)
+
+    def test_ge_array(self):
+        self.check_array_array_op(operator.ge, no_complex=True)
+
+    def test_eq_array(self):
+        self.check_array_array_op(operator.eq)
+
+    def test_ne_array(self):
+        self.check_array_array_op(operator.ne)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_array_broadcasted_op(self, op, xp, x_type, y_type,
+                                   no_bool=False, no_complex=False):
+        x_dtype = numpy.dtype(x_type)
+        y_dtype = numpy.dtype(y_type)
+        if no_bool and x_dtype == '?' and y_dtype == '?':
+            return xp.array(True)
+        if no_complex and (x_dtype.kind == 'c' or y_dtype.kind == 'c'):
+            return xp.array(True)
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        b = xp.array([[1], [2]], y_type)
+        return op(a, b)
+
+    def test_broadcasted_add(self):
+        self.check_array_broadcasted_op(operator.add)
+
+    def test_broadcasted_iadd(self):
+        self.check_array_broadcasted_op(operator.iadd)
+
+    def test_broadcasted_sub(self):
+        # TODO(unno): sub for boolean array is deprecated in numpy>=1.13
+        self.check_array_broadcasted_op(operator.sub, no_bool=True)
+
+    def test_broadcasted_isub(self):
+        # TODO(unno): sub for boolean array is deprecated in numpy>=1.13
+        self.check_array_broadcasted_op(operator.isub, no_bool=True)
+
+    def test_broadcasted_mul(self):
+        self.check_array_broadcasted_op(operator.mul)
+
+    def test_broadcasted_imul(self):
+        self.check_array_broadcasted_op(operator.imul)
+
+    def test_broadcasted_truediv(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_broadcasted_op(operator.truediv)
+
+    def test_broadcasted_itruediv(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_broadcasted_op(operator.itruediv)
+
+    def test_broadcasted_floordiv(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_broadcasted_op(operator.floordiv, no_complex=True)
+
+    def test_broadcasted_ifloordiv(self):
+        if '1.16.1' <= numpy.lib.NumpyVersion(numpy.__version__) < '1.18.0':
+            self.skipTest("NumPy Issue #12927")
+        with numpy.errstate(divide='ignore'):
+            self.check_array_broadcasted_op(operator.ifloordiv,
+                                            no_complex=True)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-6, accept_error=TypeError)
+    def check_broadcasted_pow(self, xp, x_type, y_type):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        b = xp.array([[1], [2]], y_type)
+        return operator.pow(a, b)
+
+    def test_broadcasted_pow(self):
+        # There are some precission issues in HIP that prevent
+        # checking with atol=0
+        if cupy.cuda.runtime.is_hip:
+            self.check_broadcasted_pow()
+        else:
+            self.check_array_broadcasted_op(operator.pow)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(atol=1.0, accept_error=TypeError)
+    def check_broadcasted_ipow(self, xp, x_type, y_type):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
+        b = xp.array([[1], [2]], y_type)
+        return operator.ipow(a, b)
+
+    def test_broadcasted_ipow(self):
+        self.check_broadcasted_ipow()
+
+    def test_broadcasted_divmod0(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_broadcasted_op(lambda x, y: divmod(x, y)[0],
+                                            no_complex=True)
+
+    def test_broadcasted_divmod1(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_broadcasted_op(lambda x, y: divmod(x, y)[1],
+                                            no_complex=True)
+
+    def test_broadcasted_lt(self):
+        self.check_array_broadcasted_op(operator.lt, no_complex=True)
+
+    def test_broadcasted_le(self):
+        self.check_array_broadcasted_op(operator.le, no_complex=True)
+
+    def test_broadcasted_gt(self):
+        self.check_array_broadcasted_op(operator.gt, no_complex=True)
+
+    def test_broadcasted_ge(self):
+        self.check_array_broadcasted_op(operator.ge, no_complex=True)
+
+    def test_broadcasted_eq(self):
+        self.check_array_broadcasted_op(operator.eq)
+
+    def test_broadcasted_ne(self):
+        self.check_array_broadcasted_op(operator.ne)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def check_array_doubly_broadcasted_op(self, op, xp, x_type, y_type,
+                                          no_bool=False, no_complex=False):
+        x_dtype = numpy.dtype(x_type)
+        y_dtype = numpy.dtype(y_type)
+        if no_bool and x_dtype == '?' and y_dtype == '?':
+            return xp.array(True)
+        if no_complex and (x_dtype.kind == 'c' or y_dtype.kind == 'c'):
+            return xp.array(True)
+        a = xp.array([[[1, 2, 3]], [[4, 5, 6]]], x_type)
+        b = xp.array([[1], [2], [3]], y_type)
+        return op(a, b)
+
+    def test_doubly_broadcasted_add(self):
+        self.check_array_doubly_broadcasted_op(operator.add)
+
+    def test_doubly_broadcasted_sub(self):
+        self.check_array_doubly_broadcasted_op(operator.sub, no_bool=True)
+
+    def test_doubly_broadcasted_mul(self):
+        self.check_array_doubly_broadcasted_op(operator.mul)
+
+    def test_doubly_broadcasted_truediv(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_doubly_broadcasted_op(operator.truediv)
+
+    def test_doubly_broadcasted_floordiv(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_doubly_broadcasted_op(operator.floordiv,
+                                                   no_complex=True)
+
+    def test_doubly_broadcasted_pow(self):
+        self.check_array_doubly_broadcasted_op(operator.pow)
+
+    def test_doubly_broadcasted_divmod0(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_doubly_broadcasted_op(
+                lambda x, y: divmod(x, y)[0],
+                no_complex=True)
+
+    def test_doubly_broadcasted_divmod1(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_array_doubly_broadcasted_op(
+                lambda x, y: divmod(x, y)[1],
+                no_complex=True)
+
+    def test_doubly_broadcasted_lt(self):
+        self.check_array_doubly_broadcasted_op(operator.lt, no_complex=True)
+
+    def test_doubly_broadcasted_le(self):
+        self.check_array_doubly_broadcasted_op(operator.le, no_complex=True)
+
+    def test_doubly_broadcasted_gt(self):
+        self.check_array_doubly_broadcasted_op(operator.gt, no_complex=True)
+
+    def test_doubly_broadcasted_ge(self):
+        self.check_array_doubly_broadcasted_op(operator.ge, no_complex=True)
+
+    def test_doubly_broadcasted_eq(self):
+        self.check_array_doubly_broadcasted_op(operator.eq)
+
+    def test_doubly_broadcasted_ne(self):
+        self.check_array_doubly_broadcasted_op(operator.ne)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose()
+    def check_array_reversed_op(self, op, xp, x_type, y_type, no_bool=False):
+        if no_bool and x_type == numpy.bool_ and y_type == numpy.bool_:
+            return xp.array(True)
+        a = xp.array([1, 2, 3, 4, 5], x_type)
+        b = xp.array([1, 2, 3, 4, 5], y_type)
+        return op(a, b[::-1])
+
+    def test_array_reversed_add(self):
+        self.check_array_reversed_op(operator.add)
+
+    def test_array_reversed_sub(self):
+        self.check_array_reversed_op(operator.sub, no_bool=True)
+
+    def test_array_reversed_mul(self):
+        self.check_array_reversed_op(operator.mul)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def check_typecast(self, val, dtype):
+        operators = [
+            operator.add, operator.sub, operator.mul, operator.truediv]
+
+        for op in operators:
+            with numpy.errstate(divide='ignore', invalid='ignore'):
+                a = op(val, (testing.shaped_arange((5,), numpy, dtype) - 2))
+            b = op(val, (testing.shaped_arange((5,), cupy, dtype) - 2))
+            assert a.dtype == b.dtype
+
+    def test_typecast_bool1(self):
+        self.check_typecast(True)
+
+    def test_typecast_bool2(self):
+        self.check_typecast(False)
+
+    def test_typecast_int1(self):
+        self.check_typecast(0)
+
+    def test_typecast_int2(self):
+        self.check_typecast(-127)
+
+    def test_typecast_int3(self):
+        self.check_typecast(255)
+
+    def test_typecast_int4(self):
+        self.check_typecast(-32768)
+
+    def test_typecast_int5(self):
+        self.check_typecast(65535)
+
+    def test_typecast_int6(self):
+        self.check_typecast(-2147483648)
+
+    def test_typecast_int7(self):
+        self.check_typecast(4294967295)
+
+    def test_typecast_float1(self):
+        self.check_typecast(0.0)
+
+    def test_typecast_float2(self):
+        self.check_typecast(100000.0)
+
+    # Skip float16 because of NumPy #19514
+    @testing.for_all_dtypes(name='x_type', no_float16=True)
+    @testing.numpy_cupy_allclose()
+    def check_array_boolarray_op(self, op, xp, x_type):
+        a = xp.array([[2, 7, 1], [8, 2, 8]], x_type)
+        # Cast from np.bool8 array should not read bytes
+        b = xp.array([[3, 1, 4], [-1, -5, -9]], numpy.int8).view(bool)
+        return op(a, b)
+
+    def test_add_array_boolarray(self):
+        self.check_array_boolarray_op(operator.add)
+
+    def test_iadd_array_boolarray(self):
+        self.check_array_boolarray_op(operator.iadd)
+
+
+class TestArrayIntElementwiseOp:
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_array_scalar_op(self, op, xp, x_type, y_type, swap=False):
+        a = xp.array([[0, 1, 2], [1, 0, 2]], dtype=x_type)
+        if swap:
+            return op(y_type(2), a)
+        else:
+            return op(a, y_type(2))
+
+    def test_lshift_scalar(self):
+        self.check_array_scalar_op(operator.lshift)
+
+    def test_rlshift_scalar(self):
+        self.check_array_scalar_op(operator.lshift, swap=True)
+
+    def test_rshift_scalar(self):
+        self.check_array_scalar_op(operator.rshift)
+
+    def test_rrshift_scalar(self):
+        self.check_array_scalar_op(operator.rshift, swap=True)
+
+    def test_and_scalar(self):
+        self.check_array_scalar_op(operator.and_)
+
+    def test_rand_scalar(self):
+        self.check_array_scalar_op(operator.and_, swap=True)
+
+    def test_or_scalar(self):
+        self.check_array_scalar_op(operator.or_)
+
+    def test_ror_scalar(self):
+        self.check_array_scalar_op(operator.or_, swap=True)
+
+    def test_xor_scalar(self):
+        self.check_array_scalar_op(operator.xor)
+
+    def test_rxor_scalar(self):
+        self.check_array_scalar_op(operator.xor, swap=True)
+
+    def test_mod_scalar(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_scalar_op(operator.mod)
+
+    def test_rmod_scalar(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_scalar_op(operator.mod, swap=True)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_array_scalarzero_op(self, op, xp, x_type, y_type, swap=False):
+        a = xp.array([[0, 1, 2], [1, 0, 2]], dtype=x_type)
+        if swap:
+            return op(y_type(0), a)
+        else:
+            return op(a, y_type(0))
+
+    def test_lshift_scalarzero(self):
+        self.check_array_scalarzero_op(operator.lshift)
+
+    def test_rlshift_scalarzero(self):
+        self.check_array_scalarzero_op(operator.lshift, swap=True)
+
+    def test_rshift_scalarzero(self):
+        self.check_array_scalarzero_op(operator.rshift)
+
+    def test_rrshift_scalarzero(self):
+        self.check_array_scalarzero_op(operator.rshift, swap=True)
+
+    def test_and_scalarzero(self):
+        self.check_array_scalarzero_op(operator.and_)
+
+    def test_rand_scalarzero(self):
+        self.check_array_scalarzero_op(operator.and_, swap=True)
+
+    def test_or_scalarzero(self):
+        self.check_array_scalarzero_op(operator.or_)
+
+    def test_ror_scalarzero(self):
+        self.check_array_scalarzero_op(operator.or_, swap=True)
+
+    def test_xor_scalarzero(self):
+        self.check_array_scalarzero_op(operator.xor)
+
+    def test_rxor_scalarzero(self):
+        self.check_array_scalarzero_op(operator.xor, swap=True)
+
+    def test_mod_scalarzero(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_scalarzero_op(operator.mod)
+
+    def test_rmod_scalarzero(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_scalarzero_op(operator.mod, swap=True)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_array_array_op(self, op, xp, x_type, y_type):
+        a = xp.array([[0, 1, 2], [1, 0, 2]], dtype=x_type)
+        b = xp.array([[0, 0, 1], [0, 1, 2]], dtype=y_type)
+        return op(a, b)
+
+    def test_lshift_array(self):
+        self.check_array_array_op(operator.lshift)
+
+    def test_ilshift_array(self):
+        self.check_array_array_op(operator.ilshift)
+
+    def test_rshift_array(self):
+        self.check_array_array_op(operator.rshift)
+
+    def test_irshift_array(self):
+        self.check_array_array_op(operator.irshift)
+
+    def test_and_array(self):
+        self.check_array_array_op(operator.and_)
+
+    def test_iand_array(self):
+        self.check_array_array_op(operator.iand)
+
+    def test_or_array(self):
+        self.check_array_array_op(operator.or_)
+
+    def test_ior_array(self):
+        self.check_array_array_op(operator.ior)
+
+    def test_xor_array(self):
+        self.check_array_array_op(operator.xor)
+
+    def test_ixor_array(self):
+        self.check_array_array_op(operator.ixor)
+
+    def test_mod_array(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_array_op(operator.mod)
+
+    def test_imod_array(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_array_op(operator.imod)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_array_broadcasted_op(self, op, xp, x_type, y_type):
+        a = xp.array([[0, 1, 2], [1, 0, 2], [2, 1, 0]], dtype=x_type)
+        b = xp.array([[0, 0, 1]], dtype=y_type)
+        return op(a, b)
+
+    def test_broadcasted_lshift(self):
+        self.check_array_broadcasted_op(operator.lshift)
+
+    def test_broadcasted_ilshift(self):
+        self.check_array_broadcasted_op(operator.ilshift)
+
+    def test_broadcasted_rshift(self):
+        self.check_array_broadcasted_op(operator.rshift)
+
+    def test_broadcasted_irshift(self):
+        self.check_array_broadcasted_op(operator.irshift)
+
+    def test_broadcasted_and(self):
+        self.check_array_broadcasted_op(operator.and_)
+
+    def test_broadcasted_iand(self):
+        self.check_array_broadcasted_op(operator.iand)
+
+    def test_broadcasted_or(self):
+        self.check_array_broadcasted_op(operator.or_)
+
+    def test_broadcasted_ior(self):
+        self.check_array_broadcasted_op(operator.ior)
+
+    def test_broadcasted_xor(self):
+        self.check_array_broadcasted_op(operator.xor)
+
+    def test_broadcasted_ixor(self):
+        self.check_array_broadcasted_op(operator.ixor)
+
+    def test_broadcasted_mod(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_broadcasted_op(operator.mod)
+
+    def test_broadcasted_imod(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_broadcasted_op(operator.imod)
+
+    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_array_doubly_broadcasted_op(self, op, xp, x_type, y_type):
+        a = xp.array([[[0, 1, 2]], [[1, 0, 2]]], dtype=x_type)
+        b = xp.array([[0], [0], [1]], dtype=y_type)
+        return op(a, b)
+
+    def test_doubly_broadcasted_lshift(self):
+        self.check_array_doubly_broadcasted_op(operator.lshift)
+
+    def test_doubly_broadcasted_rshift(self):
+        self.check_array_doubly_broadcasted_op(operator.rshift)
+
+    def test_doubly_broadcasted_and(self):
+        self.check_array_doubly_broadcasted_op(operator.and_)
+
+    def test_doubly_broadcasted_or(self):
+        self.check_array_doubly_broadcasted_op(operator.or_)
+
+    def test_doubly_broadcasted_xor(self):
+        self.check_array_doubly_broadcasted_op(operator.xor)
+
+    def test_doubly_broadcasted_mod(self):
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            self.check_array_doubly_broadcasted_op(operator.mod)
+
+
+@pytest.mark.parametrize('value', [
+    None,
+    Ellipsis,
+    object(),
+    numpy._NoValue,
+])
+class TestArrayObjectComparison:
+
+    @pytest.mark.parametrize('swap', [False, True])
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_eq_object(self, xp, dtype, value, swap):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], dtype=dtype)
+        if swap:
+            return value == a
+        else:
+            return a == value
+
+    @pytest.mark.parametrize('swap', [False, True])
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_ne_object(self, xp, dtype, value, swap):
+        a = xp.array([[1, 2, 3], [4, 5, 6]], dtype=dtype)
+        if swap:
+            return value != a
+        else:
+            return a != value
+
+
+class HasEq:
+    def __eq__(self, other):
+        return (other == 2) | (other == 4)
+
+
+class HasNe:
+    def __ne__(self, other):
+        return (other == 2) | (other == 4)
+
+
+class HasEqSub(HasEq):
+    pass
+
+
+class CustomInt(int):
+    pass
+
+
+@pytest.mark.parametrize('dtype', ['int32', 'float64'])
+@pytest.mark.parametrize('value', [
+    HasEq(),
+    HasNe(),  # eq test passes because `==` does not fall back to `__ne__`.
+    HasEqSub(),
+    CustomInt(3),
+])
+class TestArrayObjectComparisonDifficult:
+
+    # OK to raise TypeError.
+    # If CuPy returns a result, it should match with NumPy's result.
+
+    def test_eq_object(self, dtype, value):
+        expected = numpy.array([[1, 2, 3], [4, 5, 6]], dtype=dtype) == value
+
+        a = cupy.array([[1, 2, 3], [4, 5, 6]], dtype=dtype)
+        try:
+            res = a == value
+        except TypeError:
+            pytest.skip()
+
+        cupy.testing.assert_array_equal(res, expected)
+
+    def test_ne_object(self, dtype, value):
+        expected = numpy.array([[1, 2, 3], [4, 5, 6]], dtype=dtype) != value
+
+        a = cupy.array([[1, 2, 3], [4, 5, 6]], dtype=dtype)
+        try:
+            res = a != value
+        except TypeError:
+            pytest.skip()
+
+        cupy.testing.assert_array_equal(res, expected)
diff --git a/tests/cupy_tests/core_tests/test_ndarray_get.py b/tests/cupy_tests/core_tests/test_ndarray_get.py
new file mode 100644
index 0000000..307f556
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_get.py
@@ -0,0 +1,142 @@
+import unittest
+
+import cupy
+from cupy import cuda
+from cupy import testing
+import numpy
+from numpy import testing as np_testing
+
+
+class TestArrayGet(unittest.TestCase):
+
+    def setUp(self):
+        self.stream = cuda.Stream.null
+
+    def check_get(self, f, stream, order='C'):
+        a_gpu = f(cupy)
+        a_cpu = a_gpu.get(stream, order=order)
+        if stream:
+            stream.synchronize()
+        b_cpu = f(numpy)
+        np_testing.assert_array_equal(a_cpu, b_cpu)
+        if order == 'F' or (order == 'A' and a_gpu.flags.f_contiguous):
+            assert a_cpu.flags.f_contiguous
+        else:
+            assert a_cpu.flags.c_contiguous
+
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    def test_contiguous_array(self, dtype, order):
+        def contiguous_array(xp):
+            return testing.shaped_arange((3,), xp, dtype, order)
+        self.check_get(contiguous_array, None, order)
+
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    def test_non_contiguous_array(self, dtype, order):
+        def non_contiguous_array(xp):
+            return testing.shaped_arange((3, 3), xp, dtype, order)[0::2, 0::2]
+        self.check_get(non_contiguous_array, None, order)
+
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    def test_contiguous_array_stream(self, dtype, order):
+        def contiguous_array(xp):
+            return testing.shaped_arange((3,), xp, dtype, order)
+        self.check_get(contiguous_array, self.stream, order)
+
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    def test_non_contiguous_array_stream(self, dtype, order):
+        def non_contiguous_array(xp):
+            return testing.shaped_arange((3, 3), xp, dtype, order)[0::2, 0::2]
+        self.check_get(non_contiguous_array, self.stream)
+
+    @testing.multi_gpu(2)
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    def test_get_multigpu(self, dtype, order):
+        with cuda.Device(1):
+            src = testing.shaped_arange((2, 3), cupy, dtype, order)
+            src = cupy.asfortranarray(src)
+        with cuda.Device(0):
+            dst = src.get()
+        expected = testing.shaped_arange((2, 3), numpy, dtype, order)
+        np_testing.assert_array_equal(dst, expected)
+
+
+class TestArrayGetWithOut(unittest.TestCase):
+
+    def setUp(self):
+        self.stream = cuda.Stream.null
+
+    def check_get(self, f, out, stream):
+        a_gpu = f(cupy)
+        a_cpu = a_gpu.get(stream, out=out)
+        if stream:
+            stream.synchronize()
+        b_cpu = f(numpy)
+        assert a_cpu is out
+        np_testing.assert_array_equal(a_cpu, b_cpu)
+
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    def test_contiguous_array(self, dtype, order):
+        def contiguous_array(xp):
+            return testing.shaped_arange((3,), xp, dtype, order)
+        out = numpy.empty((3,), dtype, order)
+        self.check_get(contiguous_array, out, None)
+
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    def test_contiguous_array_cross(self, dtype, order):
+        def contiguous_array(xp):
+            return testing.shaped_arange((3,), xp, dtype, order)
+        out_order = 'C' if order == 'F' else 'F'
+        out = numpy.empty((3,), dtype, out_order)
+        self.check_get(contiguous_array, out, None)
+
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    def test_contiguous_array_with_error(self, dtype, order):
+        out = numpy.empty((3, 3), dtype)[0:2, 0:2]
+        with self.assertRaises(RuntimeError):
+            a_gpu = testing.shaped_arange((3, 3), cupy, dtype, order)[0:2, 0:2]
+            a_gpu.get(out=out)
+
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    def test_non_contiguous_array(self, dtype, order):
+        def non_contiguous_array(xp):
+            return testing.shaped_arange((3, 3), xp, dtype, order)[0::2, 0::2]
+        out = numpy.empty((2, 2), dtype, order)
+        self.check_get(non_contiguous_array, out, None)
+
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    def test_contiguous_array_stream(self, dtype, order):
+        def contiguous_array(xp):
+            return testing.shaped_arange((3,), xp, dtype, order)
+        out = numpy.empty((3,), dtype, order)
+        self.check_get(contiguous_array, out, self.stream)
+
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    def test_non_contiguous_array_stream(self, dtype, order):
+        def non_contiguous_array(xp):
+            return testing.shaped_arange((3, 3), xp, dtype, order)[0::2, 0::2]
+        out = numpy.empty((2, 2), dtype, order)
+        self.check_get(non_contiguous_array, out, self.stream)
+
+    @testing.multi_gpu(2)
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    def test_get_multigpu(self, dtype, order):
+        with cuda.Device(1):
+            src = testing.shaped_arange((2, 3), cupy, dtype, order)
+            src = cupy.asfortranarray(src)
+        with cuda.Device(0):
+            dst = numpy.empty((2, 3), dtype, order)
+            src.get(out=dst)
+        expected = testing.shaped_arange((2, 3), numpy, dtype, order)
+        np_testing.assert_array_equal(dst, expected)
diff --git a/tests/cupy_tests/core_tests/test_ndarray_indexing.py b/tests/cupy_tests/core_tests/test_ndarray_indexing.py
new file mode 100644
index 0000000..3551879
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_indexing.py
@@ -0,0 +1,232 @@
+import unittest
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(
+    {'shape': (2, 3, 4), 'transpose': None, 'indexes': (1, 0, 2)},
+    {'shape': (2, 3, 4), 'transpose': None, 'indexes': (-1, 0, -2)},
+    {'shape': (2, 3, 4), 'transpose': (2, 0, 1), 'indexes': (1, 0, 2)},
+    {'shape': (2, 3, 4), 'transpose': (2, 0, 1), 'indexes': (-1, 0, -2)},
+    {'shape': (2, 3, 4), 'transpose': None,
+     'indexes': (slice(None), slice(None, 1), slice(2))},
+    {'shape': (2, 3, 4), 'transpose': None,
+     'indexes': (slice(None), slice(None, -1), slice(-2))},
+    {'shape': (2, 3, 4), 'transpose': (2, 0, 1),
+     'indexes': (slice(None), slice(None, 1), slice(2))},
+    {'shape': (2, 3, 5), 'transpose': None,
+     'indexes': (slice(None, None, -1), slice(1, None, -1), slice(4, 1, -2))},
+    {'shape': (2, 3, 5), 'transpose': (2, 0, 1),
+     'indexes': (slice(4, 1, -2), slice(None, None, -1), slice(1, None, -1))},
+    {'shape': (2, 3, 4), 'transpose': None, 'indexes': (Ellipsis, 2)},
+    {'shape': (2, 3, 4), 'transpose': None, 'indexes': (1, Ellipsis)},
+    {'shape': (2, 3, 4, 5), 'transpose': None, 'indexes': (1, Ellipsis, 3)},
+    {'shape': (2, 3, 4), 'transpose': None,
+     'indexes': (1, None, slice(2), None, 2)},
+    {'shape': (2, 3), 'transpose': None, 'indexes': (None,)},
+    {'shape': (2,), 'transpose': None, 'indexes': (slice(None,), None)},
+    {'shape': (), 'transpose': None, 'indexes': (None,)},
+    {'shape': (), 'transpose': None, 'indexes': (None, None)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(10, -9, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-9, -10, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-1, -10, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-1, -11, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-11, -11, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(10, -9, -3),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-1, -11, -3),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(1, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(0, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-1, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-4, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-6, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-10, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-11, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-12, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, 1, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, 0, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, -1, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, -4, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, -5, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, -6, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, -10, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, -11, -1),)},
+    {'shape': (10,), 'transpose': None, 'indexes': (slice(-5, -12, -1),)},
+    # reversing indexing on empty dimension
+    {'shape': (0,), 'transpose': None, 'indexes': (slice(None, None, -1),)},
+    {'shape': (0, 0), 'transpose': None,
+     'indexes': (slice(None, None, -1), slice(None, None, -1))},
+    {'shape': (0, 0), 'transpose': None,
+     'indexes': (None, slice(None, None, -1))},
+    {'shape': (0, 0), 'transpose': None,
+     'indexes': (slice(None, None, -1), None)},
+    {'shape': (0, 1), 'transpose': None,
+     'indexes': (slice(None, None, -1), None)},
+    {'shape': (1, 0), 'transpose': None,
+     'indexes': (None, slice(None, None, -1))},
+    {'shape': (1, 0, 1), 'transpose': None,
+     'indexes': (None, slice(None, None, -1), None)},
+    #
+    {'shape': (2, 0), 'transpose': None,
+     'indexes': (1, slice(None, None, None))},
+)
+class TestArrayIndexingParameterized(unittest.TestCase):
+
+    _getitem_hip_skip_condition = [
+        ((1, 0, 2), (2, 3, 4), None),
+        ((-1, 0, -2), (2, 3, 4), None),
+        ((1, 0, 2), (2, 3, 4), (2, 0, 1)),
+        ((-1, 0, -2), (2, 3, 4), (2, 0, 1)),
+        ((slice(None, None, None), None), (2,), None),
+        ((slice(-9, -10, -1),), (10,), None),
+        ((slice(-4, -5, -1),), (10,), None),
+        ((slice(-5, -6, -1),), (10,), None),
+    ]
+
+    def _check_getitem_hip_skip_condition(self):
+        return (self.indexes, self.shape, self.transpose) in \
+            self._getitem_hip_skip_condition
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_getitem(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip:
+            if self._check_getitem_hip_skip_condition():
+                pytest.xfail('HIP may have a bug')
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        if self.transpose:
+            a = a.transpose(self.transpose)
+        return a[self.indexes]
+
+
+@testing.parameterize(
+    {'shape': (), 'transpose': None, 'indexes': 0},
+    # Do not test which class will be raised:
+    # - too many indices (IndexError)
+    # - slice step cannot be zero (ValueError)
+    # {'shape': (), 'transpose': None, 'indexes': (slice(0, 1, 0),)},
+    {'shape': (2, 3), 'transpose': None, 'indexes': (0, 0, 0)},
+    {'shape': (2, 3, 4), 'transpose': None, 'indexes': -3},
+    {'shape': (2, 3, 4), 'transpose': (2, 0, 1), 'indexes': -5},
+    {'shape': (2, 3, 4), 'transpose': None, 'indexes': 3},
+    {'shape': (2, 3, 4), 'transpose': (2, 0, 1), 'indexes': 5},
+    {'shape': (2, 3, 4), 'transpose': None,
+     'indexes': (Ellipsis, Ellipsis, 1)},
+)
+class TestArrayIndexIndexError(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_invalid_getitem(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, dtype)
+            if self.transpose:
+                a = a.transpose(self.transpose)
+            with pytest.raises(IndexError):
+                a[self.indexes]
+
+
+@testing.parameterize(
+    {'error_class': ValueError, 'indexes': (slice(0, 1, 0),)},
+    {'error_class': TypeError,
+     'indexes': (slice((0, 0), None, None), )},
+    {'error_class': TypeError,
+     'indexes': (slice(None, (0, 0), None), )},
+    {'error_class': TypeError,
+     'indexes': (slice(None, None, (0, 0)), )},
+)
+class TestArrayIndexOtherError(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_invalid_getitem(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp, dtype)
+            with pytest.raises(self.error_class):
+                a[self.indexes]
+
+
+class TestArrayIndex(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_constant(self, xp, dtype):
+        a = xp.zeros((2, 3, 4), dtype=dtype)
+        a[:] = 1
+        return a
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_partial_constant(self, xp, dtype):
+        a = xp.zeros((2, 3, 4), dtype=dtype)
+        a[1, 1:3] = 1
+        return a
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_copy(self, xp, dtype):
+        a = xp.zeros((2, 3, 4), dtype=dtype)
+        b = testing.shaped_arange((2, 3, 4), xp, dtype)
+        a[:] = b
+        return a
+
+    @testing.for_all_dtypes_combination(('src_type', 'dst_type'))
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_different_type(self, xp, src_type, dst_type):
+        a = xp.zeros((2, 3, 4), dtype=dst_type)
+        b = testing.shaped_arange((2, 3, 4), xp, src_type)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', numpy.ComplexWarning)
+            a[:] = b
+        return a
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_partial_copy(self, xp, dtype):
+        a = xp.zeros((2, 3, 4), dtype=dtype)
+        b = testing.shaped_arange((3, 2), xp, dtype)
+        a[1, ::-1, 1:4:2] = b
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_T(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.T
+
+    @testing.numpy_cupy_array_equal()
+    def test_T_vector(self, xp):
+        a = testing.shaped_arange((4,), xp)
+        return a.T
+
+
+class TestSetItemCompatBroadcast:
+    @testing.numpy_cupy_array_equal()
+    def test_simple(self, xp):
+        dtype = int
+        a = xp.zeros(4, dtype)
+        a[:] = testing.shaped_arange((1, 4), xp, dtype)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_other1(self, xp):
+        dtype = int
+        a = xp.zeros((2, 1, 3), dtype)
+        a[:] = testing.shaped_arange((1, 2, 1, 3), xp, dtype)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_0d(self, xp):
+        dtype = int
+        a = xp.zeros((), dtype)
+        a[...] = testing.shaped_arange((1, 1), xp, dtype)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_remain0d(self, xp):
+        dtype = int
+        a = xp.zeros((2, 3, 4), dtype)
+        a[0, 1, 2] = testing.shaped_arange((), xp, dtype)
+        return a
diff --git a/tests/cupy_tests/core_tests/test_ndarray_math.py b/tests/cupy_tests/core_tests/test_ndarray_math.py
new file mode 100644
index 0000000..a91f149
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_math.py
@@ -0,0 +1,114 @@
+import unittest
+
+import numpy
+
+from cupy import testing
+
+
+@testing.parameterize(*testing.product({
+    'decimals': [-2, -1, 0, 1, 2],
+}))
+class TestRound(unittest.TestCase):
+
+    shape = (20,)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_round(self, xp, dtype):
+        if dtype == numpy.bool_:
+            # avoid cast problem
+            a = testing.shaped_random(self.shape, xp, scale=10, dtype=dtype)
+            return a.round(0)
+        if dtype == numpy.float16:
+            # avoid accuracy problem
+            a = testing.shaped_random(self.shape, xp, scale=10, dtype=dtype)
+            return a.round(0)
+        a = testing.shaped_random(self.shape, xp, scale=100, dtype=dtype)
+        return a.round(self.decimals)
+
+    @testing.numpy_cupy_array_equal()
+    def test_round_out(self, xp):
+        a = testing.shaped_random(self.shape, xp, scale=100, dtype='d')
+        out = xp.empty_like(a)
+        a.round(self.decimals, out)
+        return out
+
+
+@testing.parameterize(*testing.product({
+    # limit to:
+    # * <=0: values like 0.35 and 0.035 cannot be expressed exactly in IEEE 754
+    # * >-4: to avoid float16 overflow
+    'decimals': [-3, -2, -1, 0],
+}))
+class TestRoundHalfway(unittest.TestCase):
+
+    shape = (20,)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_round_halfway_float(self, xp, dtype):
+        # generate [..., -1.5, -0.5, 0.5, 1.5, ...] * 10^{-decimals}
+        a = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        a *= 2
+        a -= a.size + 1
+        scale = 10**abs(self.decimals)
+        if self.decimals < 0:
+            a *= scale
+        else:
+            a /= scale
+        a /= 2
+
+        return a.round(self.decimals)
+
+    @testing.for_signed_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_round_halfway_int(self, xp, dtype):
+        # generate [..., -1.5, -0.5, 0.5, 1.5, ...] * 10^{-decimals}
+        a = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        a *= 2
+        a -= a.size + 1
+        scale = 10**abs(self.decimals)
+        if self.decimals < 0:
+            a *= xp.array(scale).astype(dtype)
+        a >>= 1
+
+        return a.round(self.decimals)
+
+    @testing.for_unsigned_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_round_halfway_uint(self, xp, dtype):
+        # generate [0.5, 1.5, ...] * 10^{-decimals}
+        a = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        a *= 2
+        a -= 1
+        scale = 10**abs(self.decimals)
+        if self.decimals < 0:
+            a *= xp.array(scale).astype(dtype)
+        a >>= 1
+
+        return a.round(self.decimals)
+
+
+@testing.parameterize(*testing.product({
+    'decimals': [-5, -4, -3, -2, -1, 0]
+}))
+class TestRoundMinMax(unittest.TestCase):
+
+    @unittest.skip('Known incompatibility: see core.pyx')
+    @testing.numpy_cupy_array_equal()
+    def _test_round_int64(self, xp):
+        a = xp.array([-2**62, 2**62], dtype=xp.int64)
+        return a.round(self.decimals)
+
+    @unittest.skip('Known incompatibility: see core.pyx')
+    @testing.numpy_cupy_array_equal()
+    def test_round_uint64(self, xp):
+        a = xp.array([2**63], dtype=xp.uint64)
+        return a.round(self.decimals)
+
+    @unittest.skip('Known incompatibility: see core.pyx')
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_round_minmax(self, xp, dtype):
+        a = xp.array([xp.iinfo(dtype).min, xp.iinfo(dtype).max], dtype=dtype)
+        return a.round(self.decimals)
diff --git a/tests/cupy_tests/core_tests/test_ndarray_owndata.py b/tests/cupy_tests/core_tests/test_ndarray_owndata.py
new file mode 100644
index 0000000..66d616e
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_owndata.py
@@ -0,0 +1,20 @@
+import unittest
+
+from cupy import _core
+
+
+class TestArrayOwndata(unittest.TestCase):
+
+    def setUp(self):
+        self.a = _core.ndarray(())
+
+    def test_original_array(self):
+        assert self.a.flags.owndata is True
+
+    def test_view_array(self):
+        v = self.a.view()
+        assert v.flags.owndata is False
+
+    def test_reshaped_array(self):
+        r = self.a.reshape(())
+        assert r.flags.owndata is False
diff --git a/tests/cupy_tests/core_tests/test_ndarray_reduction.py b/tests/cupy_tests/core_tests/test_ndarray_reduction.py
new file mode 100644
index 0000000..d54eca7
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_reduction.py
@@ -0,0 +1,498 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+import cupy._core._accelerator as _acc
+from cupy._core import _cub_reduction
+from cupy import testing
+
+
+@testing.parameterize(*testing.product({
+    'order': ('C', 'F'),
+}))
+class TestArrayReduction(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, order=self.order)
+        return a.max()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_all_keepdims(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, order=self.order)
+        return a.max(keepdims=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype, order=self.order)
+        return a.max(axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.max(axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.max(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.max(axis=2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_multiple_axes(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.max(axis=(1, 2))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_multiple_axes_keepdims(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.max(axis=(1, 2), keepdims=True)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_nan(self, xp, dtype):
+        if _acc.ACCELERATOR_CUTENSOR in _acc.get_routine_accelerators():
+            pytest.skip()
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.max()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_nan_real(self, xp, dtype):
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.max()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_max_nan_imag(self, xp, dtype):
+        a = xp.array([float('nan')*1.j, 1.j, -1.j], dtype, order=self.order)
+        return a.max()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, order=self.order)
+        return a.min()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_all_keepdims(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, order=self.order)
+        return a.min(keepdims=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype, order=self.order)
+        return a.min(axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.min(axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.min(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.min(axis=2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_multiple_axes(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.min(axis=(1, 2))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_multiple_axes_keepdims(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.min(axis=(1, 2), keepdims=True)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_nan(self, xp, dtype):
+        if _acc.ACCELERATOR_CUTENSOR in _acc.get_routine_accelerators():
+            pytest.skip()
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.min()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_nan_real(self, xp, dtype):
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.min()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_min_nan_imag(self, xp, dtype):
+        a = xp.array([float('nan')*1.j, 1.j, -1.j], dtype, order=self.order)
+        return a.min()
+
+    # skip bool: numpy's ptp raises a TypeError on bool inputs
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, order=self.order)
+        return a.ptp()
+
+    @testing.with_requires('numpy>=1.15')
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_all_keepdims(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, order=self.order)
+        return a.ptp(keepdims=True)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype, order=self.order)
+        return a.ptp(axis=0)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.ptp(axis=0)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.ptp(axis=1)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.ptp(axis=2)
+
+    @testing.with_requires('numpy>=1.15')
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_multiple_axes(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.ptp(axis=(1, 2))
+
+    @testing.with_requires('numpy>=1.15')
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_multiple_axes_keepdims(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.ptp(axis=(1, 2), keepdims=True)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_nan(self, xp, dtype):
+        if _acc.ACCELERATOR_CUTENSOR in _acc.get_routine_accelerators():
+            pytest.skip()
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.ptp()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_nan_real(self, xp, dtype):
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.ptp()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ptp_nan_imag(self, xp, dtype):
+        a = xp.array([float('nan')*1.j, 1.j, -1.j], dtype, order=self.order)
+        return a.ptp()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmax_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, order=self.order)
+        return a.argmax()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmax_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype, order=self.order)
+        return a.argmax(axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmax_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.argmax(axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmax_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.argmax(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmax_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.argmax(axis=2)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmax_nan(self, xp, dtype):
+        if _acc.ACCELERATOR_CUTENSOR in _acc.get_routine_accelerators():
+            pytest.skip()
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.argmax()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmax_nan_real(self, xp, dtype):
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.argmax()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmax_nan_imag(self, xp, dtype):
+        a = xp.array([float('nan')*1.j, 1.j, -1.j], dtype, order=self.order)
+        return a.argmax()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmin_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, order=self.order)
+        return a.argmin()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmin_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype, order=self.order)
+        return a.argmin(axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmin_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.argmin(axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmin_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.argmin(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmin_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype, order=self.order)
+        return a.argmin(axis=2)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmin_nan(self, xp, dtype):
+        if _acc.ACCELERATOR_CUTENSOR in _acc.get_routine_accelerators():
+            pytest.skip()
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.argmin()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmin_nan_real(self, xp, dtype):
+        a = xp.array([float('nan'), 1, -1], dtype, order=self.order)
+        return a.argmin()
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_argmin_nan_imag(self, xp, dtype):
+        a = xp.array([float('nan')*1.j, 1.j, -1.j], dtype, order=self.order)
+        return a.argmin()
+
+
+@testing.parameterize(*testing.product({
+    # TODO(leofang): make a @testing.for_all_axes decorator
+    'shape_and_axis': [
+        ((), None),
+        ((0,), (0,)),
+        ((0, 2), (0,)),
+        ((0, 2), (1,)),
+        ((0, 2), (0, 1)),
+        ((2, 0), (0,)),
+        ((2, 0), (1,)),
+        ((2, 0), (0, 1)),
+        ((0, 2, 3), (0,)),
+        ((0, 2, 3), (1,)),
+        ((0, 2, 3), (2,)),
+        ((0, 2, 3), (0, 1)),
+        ((0, 2, 3), (1, 2)),
+        ((0, 2, 3), (0, 2)),
+        ((0, 2, 3), (0, 1, 2)),
+        ((2, 0, 3), (0,)),
+        ((2, 0, 3), (1,)),
+        ((2, 0, 3), (2,)),
+        ((2, 0, 3), (0, 1)),
+        ((2, 0, 3), (1, 2)),
+        ((2, 0, 3), (0, 2)),
+        ((2, 0, 3), (0, 1, 2)),
+        ((2, 3, 0), (0,)),
+        ((2, 3, 0), (1,)),
+        ((2, 3, 0), (2,)),
+        ((2, 3, 0), (0, 1)),
+        ((2, 3, 0), (1, 2)),
+        ((2, 3, 0), (0, 2)),
+        ((2, 3, 0), (0, 1, 2)),
+    ],
+    'order': ('C', 'F'),
+    'func': ('min', 'max', 'argmax', 'argmin'),
+}))
+class TestArrayReductionZeroSize:
+
+    @testing.numpy_cupy_allclose(
+        contiguous_check=False, accept_error=ValueError)
+    def test_zero_size(self, xp):
+        shape, axis = self.shape_and_axis
+        # NumPy only supports axis being an int
+        if self.func in ('argmax', 'argmin'):
+            if axis is not None and len(axis) == 1:
+                axis = axis[0]
+            else:
+                pytest.skip(
+                    f"NumPy does not support axis={axis} for {self.func}")
+        # dtype is irrelevant here, just pick one
+        a = testing.shaped_random(shape, xp, xp.float32, order=self.order)
+        return getattr(a, self.func)(axis=axis)
+
+
+# This class compares CUB results against NumPy's. ("fallback" is CuPy's
+# original kernel, also tested here to reduce code duplication.)
+@testing.parameterize(*testing.product({
+    'shape': [(10,), (10, 20), (10, 20, 30), (10, 20, 30, 40),
+              # skip (2, 3, 0) because it would not hit the CUB code path
+              (0,), (2, 0), (0, 2), (0, 2, 3), (2, 3, 0)],
+    'order': ('C', 'F'),
+    'backend': ('device', 'block', 'fallback'),
+}))
+@pytest.mark.skipif(
+    not cupy.cuda.cub.available, reason='The CUB routine is not enabled')
+class TestCubReduction:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.old_routine_accelerators = _acc.get_routine_accelerators()
+        self.old_reduction_accelerators = _acc.get_reduction_accelerators()
+        if self.backend == 'device':
+            _acc.set_routine_accelerators(['cub'])
+            _acc.set_reduction_accelerators([])
+        elif self.backend == 'block':
+            _acc.set_routine_accelerators([])
+            _acc.set_reduction_accelerators(['cub'])
+        elif self.backend == 'fallback':
+            _acc.set_routine_accelerators([])
+            _acc.set_reduction_accelerators([])
+        yield
+        _acc.set_routine_accelerators(self.old_routine_accelerators)
+        _acc.set_reduction_accelerators(self.old_reduction_accelerators)
+
+    @testing.for_contiguous_axes()
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(
+        contiguous_check=False, accept_error=ValueError)
+    def test_cub_min(self, xp, dtype, axis):
+        a = testing.shaped_random(self.shape, xp, dtype, order=self.order)
+
+        if xp is numpy:
+            return a.min(axis=axis)
+
+        # xp is cupy, first ensure we really use CUB
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        if self.backend == 'device':
+            func_name = 'cupy._core._routines_statistics.cub.'
+            if len(axis) == len(self.shape):
+                func_name += 'device_reduce'
+            else:
+                func_name += 'device_segmented_reduce'
+            with testing.AssertFunctionIsCalled(func_name, return_value=ret):
+                a.min(axis=axis)
+        elif self.backend == 'block':
+            # this is the only function we can mock; the rest is cdef'd
+            func_name = 'cupy._core._cub_reduction.'
+            func_name += '_SimpleCubReductionKernel_get_cached_function'
+            func = _cub_reduction._SimpleCubReductionKernel_get_cached_function
+            if len(axis) == len(self.shape):
+                times_called = 2  # two passes
+            else:
+                times_called = 1  # one pass
+            if a.size == 0:
+                times_called = 0  # _reduction.pyx has an early return path
+            with testing.AssertFunctionIsCalled(
+                    func_name, wraps=func, times_called=times_called):
+                a.min(axis=axis)
+        elif self.backend == 'fallback':
+            pass
+        # ...then perform the actual computation
+        return a.min(axis=axis)
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_cub_min_empty_axis(self, xp, dtype, contiguous_check=False):
+        a = testing.shaped_random(self.shape, xp, dtype, order=self.order)
+        return a.min(axis=())
+
+    @testing.for_contiguous_axes()
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(
+        contiguous_check=False, accept_error=ValueError)
+    def test_cub_max(self, xp, dtype, axis):
+        a = testing.shaped_random(self.shape, xp, dtype, order=self.order)
+
+        if xp is numpy:
+            return a.max(axis=axis)
+
+        # xp is cupy, first ensure we really use CUB
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        if self.backend == 'device':
+            func_name = 'cupy._core._routines_statistics.cub.'
+            if len(axis) == len(self.shape):
+                func_name += 'device_reduce'
+            else:
+                func_name += 'device_segmented_reduce'
+            with testing.AssertFunctionIsCalled(func_name, return_value=ret):
+                a.max(axis=axis)
+        elif self.backend == 'block':
+            # this is the only function we can mock; the rest is cdef'd
+            func_name = 'cupy._core._cub_reduction.'
+            func_name += '_SimpleCubReductionKernel_get_cached_function'
+            func = _cub_reduction._SimpleCubReductionKernel_get_cached_function
+            if len(axis) == len(self.shape):
+                times_called = 2  # two passes
+            else:
+                times_called = 1  # one pass
+            if a.size == 0:
+                times_called = 0  # _reduction.pyx has an early return path
+            with testing.AssertFunctionIsCalled(
+                    func_name, wraps=func, times_called=times_called):
+                a.max(axis=axis)
+        elif self.backend == 'fallback':
+            pass
+        # ...then perform the actual computation
+        return a.max(axis=axis)
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_cub_max_empty_axis(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype, order=self.order)
+        return a.max(axis=())
diff --git a/tests/cupy_tests/core_tests/test_ndarray_scatter.py b/tests/cupy_tests/core_tests/test_ndarray_scatter.py
new file mode 100644
index 0000000..c23fdd1
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_scatter.py
@@ -0,0 +1,283 @@
+import pytest
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(
+    # array only
+    {'shape': (2, 3, 4), 'slices': numpy.array(-1), 'value': 1},
+    {'shape': (2, 3, 4), 'slices': numpy.array([1, 0]), 'value': 1},
+    {'shape': (2, 3, 4), 'slices': (slice(None), [1, 2]), 'value': 1},
+    {'shape': (3, 4, 5),
+     'slices': (slice(None), [[1, 2], [0, -1]],), 'value': 1},
+    {'shape': (3, 4, 5),
+     'slices': (slice(None), slice(None), [[1, 2], [0, 3]]), 'value': 1},
+    # array with duplicate indices
+    {'shape': (2, 3), 'slices': ([1, 1], slice(None)), 'value': 1},
+    {'shape': (2, 3), 'slices': ([1, 0, 1], slice(None)), 'value': 1},
+    {'shape': (2, 3), 'slices': (slice(1, 2), [1, 0, 1]), 'value': 1},
+    # slice and array
+    {'shape': (3, 4, 5),
+     'slices': (slice(None), slice(1, 2), [[1, 3], [0, 2]]), 'value': 1},
+    # None and array
+    {'shape': (3, 4, 5),
+     'slices': (None, [1, -1]), 'value': 1},
+    {'shape': (3, 4, 5),
+     'slices': (None, [1, -1], None), 'value': 1},
+    {'shape': (3, 4, 5),
+     'slices': (None, None, None, [1, -1]), 'value': 1},
+    # None, slice and array
+    {'shape': (3, 4, 5),
+     'slices': (slice(0, 1), None, [1, -1]), 'value': 1},
+    {'shape': (3, 4, 5),
+     'slices': (slice(0, 1), slice(1, 2), [1, -1]), 'value': 1},
+    {'shape': (3, 4, 5),
+     'slices': (slice(0, 1), None, slice(1, 2), [1, -1]), 'value': 1},
+    # broadcasting
+    {'shape': (3, 4, 5), 'slices': (slice(None), [[1, 2], [0, -1]],),
+     'value': numpy.arange(3 * 2 * 2 * 5).reshape(3, 2, 2, 5)},
+    # multiple integer arrays
+    {'shape': (2, 3, 4), 'slices': ([1, 0], [2, 1]),
+     'value': numpy.arange(2 * 4).reshape(2, 4)},
+    {'shape': (2, 3, 4), 'slices': ([1, 0], slice(None), [2, 1]),
+     'value': numpy.arange(2 * 3).reshape(2, 3)},
+    {'shape': (2, 3, 4), 'slices': ([1, 0], slice(None), [[2, 0], [3, 1]]),
+     'value': numpy.arange(2 * 2 * 3).reshape(2, 2, 3)},
+    {'shape': (1, 1, 2, 3, 4),
+     'slices': (None, slice(None), 0, [1, 0], slice(0, 2, 2), [2, -1]),
+     'value': 1},
+    # multiple integer arrays duplicate
+    {'shape': (2, 3, 4), 'slices': ([1, 1], [1, 1]),
+     'value': numpy.arange(2 * 4).reshape(2, 4)},
+    {'shape': (2, 3, 4), 'slices': ([1, 1], slice(None), [[2, 2], [3, 1]]),
+     'value': numpy.arange(2 * 2 * 3).reshape(2, 2, 3)},
+    {'shape': (2, 3, 4), 'slices': ([1, 1], 1, [[2, 2], [3, 1]]),
+     'value': numpy.arange(2 * 2).reshape(2, 2)},
+    # mask
+    {'shape': (3, 4, 5),
+     'slices': (numpy.random.choice([False, True], (3, 4, 5)),),
+     'value': 1},
+    {'shape': (3, 4, 5),
+     'slices': (numpy.random.choice([False, True], (3,)),),
+     'value': numpy.arange(4 * 5).reshape(4, 5)},
+    {'shape': (3, 4, 5),
+     'slices': (slice(None), numpy.array([True, False, False, True]),),
+     'value': numpy.arange(3 * 2 * 5).reshape(3, 2, 5)},
+    # empty arrays
+    {'shape': (2, 3, 4), 'slices': [], 'value': 1},
+    {'shape': (2, 3, 4), 'slices': [],
+     'value': numpy.array([1, 1, 1, 1])},
+    {'shape': (2, 3, 4), 'slices': [],
+     'value': numpy.random.uniform(size=(3, 4))},
+    {'shape': (2, 3, 4), 'slices': numpy.array([], dtype=numpy.int32),
+     'value': 1},
+    {'shape': (2, 3, 4), 'slices': ([],),
+     'value': 1},
+    {'shape': (2, 3, 4), 'slices': numpy.array([[]], dtype=numpy.int32),
+     'value': numpy.random.uniform(size=(3, 4))},
+    {'shape': (2, 3, 4), 'slices': ([[]],),
+     'value': 1},
+    {'shape': (2, 3, 4), 'slices': ([[[]]],),
+     'value': 1},
+    {'shape': (2, 3, 4, 5), 'slices': ([[[]]],),
+     'value': 1},
+    {'shape': (2, 3, 4, 5), 'slices': ([[[[]]]],),
+     'value': 1},
+    {'shape': (2, 3, 4), 'slices': (slice(None), []),
+     'value': 1},
+    {'shape': (2, 3, 4), 'slices': ([], []),
+     'value': 1},
+    {'shape': (2, 3, 4), 'slices': numpy.array([], dtype=numpy.bool_),
+     'value': 1},
+    {'shape': (2, 3, 4),
+     'slices': (slice(None), numpy.array([], dtype=numpy.bool_)),
+     'value': 1},
+    {'shape': (2, 3, 4), 'slices': numpy.array([[], []], dtype=numpy.bool_),
+     'value': numpy.random.uniform(size=(4,))},
+    # list indexes
+    {'shape': (2, 3, 4), 'slices': [1], 'value': 1},
+    {'shape': (2, 3, 4), 'slices': [1, 1],
+     'value': numpy.arange(2 * 3 * 4).reshape(2, 3, 4)},
+    {'shape': (2, 3, 4), 'slices': ([1],), 'value': 1},
+    {'shape': (2, 3, 4), 'slices': ([1, 1],), 'value': 1},
+    {'shape': (2, 3, 4), 'slices': ([1], [1]), 'value': 1},
+    {'shape': (2, 3, 4), 'slices': ([1, 1], 1), 'value': 1},
+    {'shape': (2, 3, 4), 'slices': ([1], slice(1, 2)), 'value': 1},
+    {'shape': (2, 3, 4), 'slices': ([[1]], slice(1, 2)), 'value': 1},
+    _ids=False,  # Do not generate ids from randomly generated params
+)
+class TestScatterParametrized:
+
+    @testing.for_dtypes([numpy.float32, numpy.int32, numpy.uint32,
+                         numpy.uint64, numpy.ulonglong, numpy.float16,
+                         numpy.float64])
+    @testing.numpy_cupy_array_equal()
+    def test_scatter_add(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip and dtype == numpy.float16:
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        a = xp.zeros(self.shape, dtype)
+        xp.add.at(a, self.slices, self.value)
+        return a
+
+    @testing.for_dtypes([numpy.float32, numpy.int32, numpy.uint32,
+                         numpy.uint64, numpy.ulonglong, numpy.float64])
+    @testing.numpy_cupy_array_equal()
+    def test_scatter_max(self, xp, dtype):
+        a = xp.zeros(self.shape, dtype)
+        xp.maximum.at(a, self.slices, self.value)
+        return a
+
+    @testing.for_dtypes([numpy.float32, numpy.int32, numpy.uint32,
+                         numpy.uint64, numpy.ulonglong, numpy.float64])
+    @testing.numpy_cupy_array_equal()
+    def test_scatter_min(self, xp, dtype):
+        a = xp.zeros(self.shape, dtype)
+        xp.minimum.at(a, self.slices, self.value)
+        return a
+
+
+class TestScatterAdd:
+
+    @testing.for_dtypes([numpy.float32, numpy.int32, numpy.uint32,
+                         numpy.uint64, numpy.ulonglong, numpy.float16,
+                         numpy.float64])
+    def test_scatter_add_cupy_arguments(self, dtype):
+        if cupy.cuda.runtime.is_hip and dtype == numpy.float16:
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        shape = (2, 3)
+        a = cupy.zeros(shape, dtype)
+        slices = (cupy.array([1, 1]), slice(None))
+        cupy.add.at(a, slices, cupy.array(1.))
+        testing.assert_array_equal(
+            a, cupy.array([[0., 0., 0.], [2., 2., 2.]], dtype))
+
+    @testing.for_dtypes([numpy.float32, numpy.int32, numpy.uint32,
+                         numpy.uint64, numpy.ulonglong, numpy.float16,
+                         numpy.float64])
+    def test_scatter_add_cupy_arguments_mask(self, dtype):
+        if cupy.cuda.runtime.is_hip and dtype == numpy.float16:
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        shape = (2, 3)
+        a = cupy.zeros(shape, dtype)
+        slices = (cupy.array([True, False]), slice(None))
+        cupy.add.at(a, slices, cupy.array(1.))
+        testing.assert_array_equal(
+            a, cupy.array([[1., 1., 1.], [0., 0., 0.]], dtype))
+
+    @testing.for_dtypes_combination(
+        [numpy.float32, numpy.int32, numpy.uint32, numpy.uint64,
+         numpy.ulonglong, numpy.float16, numpy.float64],
+        names=['src_dtype', 'dst_dtype'])
+    def test_scatter_add_differnt_dtypes(self, src_dtype, dst_dtype):
+        if (
+                cupy.cuda.runtime.is_hip
+                and (src_dtype == numpy.float16
+                     or dst_dtype == numpy.float16)):
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        shape = (2, 3)
+        a = cupy.zeros(shape, dtype=src_dtype)
+        value = cupy.array(1, dtype=dst_dtype)
+        slices = ([1, 1], slice(None))
+        cupy.add.at(a, slices, value)
+
+        numpy.testing.assert_almost_equal(
+            a.get(),
+            numpy.array([[0, 0, 0], [2, 2, 2]], dtype=src_dtype))
+
+    @testing.for_dtypes_combination(
+        [numpy.float32, numpy.int32, numpy.uint32, numpy.uint64,
+         numpy.ulonglong, numpy.float16, numpy.float64],
+        names=['src_dtype', 'dst_dtype'])
+    def test_scatter_add_differnt_dtypes_mask(self, src_dtype, dst_dtype):
+        if (
+                cupy.cuda.runtime.is_hip
+                and (src_dtype == numpy.float16
+                     or dst_dtype == numpy.float16)):
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        shape = (2, 3)
+        a = cupy.zeros(shape, dtype=src_dtype)
+        value = cupy.array(1, dtype=dst_dtype)
+        slices = (numpy.array([[True, False, False], [False, True, True]]))
+        cupy.add.at(a, slices, value)
+
+        numpy.testing.assert_almost_equal(
+            a.get(),
+            numpy.array([[1, 0, 0], [0, 1, 1]], dtype=src_dtype))
+
+
+class TestScatterMinMax:
+
+    @testing.for_dtypes([numpy.float32, numpy.int32, numpy.uint32,
+                         numpy.uint64, numpy.ulonglong, numpy.float64])
+    def test_scatter_minmax_cupy_arguments(self, dtype):
+        shape = (2, 3)
+        a = cupy.zeros(shape, dtype)
+        slices = (cupy.array([1, 1]), slice(None))
+        cupy.maximum.at(a, slices, cupy.array(1.))
+        testing.assert_array_equal(
+            a, cupy.array([[0., 0., 0.], [1., 1., 1.]], dtype))
+
+        a = cupy.ones(shape, dtype)
+        cupy.minimum.at(a, slices, cupy.array(0.))
+        testing.assert_array_equal(
+            a, cupy.array([[1., 1., 1.], [0., 0., 0.]], dtype))
+
+    @testing.for_dtypes([numpy.float32, numpy.int32, numpy.uint32,
+                         numpy.uint64, numpy.ulonglong, numpy.float64])
+    def test_scatter_minmax_cupy_arguments_mask(self, dtype):
+        shape = (2, 3)
+        a = cupy.zeros(shape, dtype)
+        slices = (cupy.array([True, False]), slice(None))
+        cupy.maximum.at(a, slices, cupy.array(1.))
+        testing.assert_array_equal(
+            a, cupy.array([[1., 1., 1.], [0., 0., 0.]], dtype))
+
+        a = cupy.ones(shape, dtype)
+        cupy.minimum.at(a, slices, cupy.array(0.))
+        testing.assert_array_equal(
+            a, cupy.array([[0., 0., 0.], [1., 1., 1.]], dtype))
+
+    @testing.for_dtypes_combination(
+        [numpy.float32, numpy.int32, numpy.uint32, numpy.uint64,
+         numpy.ulonglong, numpy.float64],
+        names=['src_dtype', 'dst_dtype'])
+    def test_scatter_minmax_differnt_dtypes(self, src_dtype, dst_dtype):
+        shape = (2, 3)
+        a = cupy.zeros(shape, dtype=src_dtype)
+        value = cupy.array(1, dtype=dst_dtype)
+        slices = ([1, 1], slice(None))
+        cupy.maximum.at(a, slices, value)
+        numpy.testing.assert_almost_equal(
+            a.get(),
+            numpy.array([[0, 0, 0], [1, 1, 1]], dtype=src_dtype))
+
+        a = cupy.ones(shape, dtype=src_dtype)
+        value = cupy.array(0, dtype=dst_dtype)
+        cupy.minimum.at(a, slices, value)
+        numpy.testing.assert_almost_equal(
+            a.get(),
+            numpy.array([[1, 1, 1], [0, 0, 0]], dtype=src_dtype))
+
+    @testing.for_dtypes_combination(
+        [numpy.float32, numpy.int32, numpy.uint32, numpy.uint64,
+         numpy.ulonglong, numpy.float16, numpy.float64],
+        names=['src_dtype', 'dst_dtype'])
+    def test_scatter_minmax_differnt_dtypes_mask(self, src_dtype, dst_dtype):
+        shape = (2, 3)
+        a = cupy.zeros(shape, dtype=src_dtype)
+        value = cupy.array(1, dtype=dst_dtype)
+        slices = (numpy.array([[True, False, False], [False, True, True]]))
+        cupy.maximum.at(a, slices, value)
+        numpy.testing.assert_almost_equal(
+            a.get(),
+            numpy.array([[1, 0, 0], [0, 1, 1]], dtype=src_dtype))
+
+        a = cupy.ones(shape, dtype=src_dtype)
+        value = cupy.array(0, dtype=dst_dtype)
+        cupy.minimum.at(a, slices, value)
+        numpy.testing.assert_almost_equal(
+            a.get(),
+            numpy.array([[0, 1, 1], [1, 0, 0]], dtype=src_dtype))
diff --git a/tests/cupy_tests/core_tests/test_ndarray_ufunc.py b/tests/cupy_tests/core_tests/test_ndarray_ufunc.py
new file mode 100644
index 0000000..e58e6ed
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_ufunc.py
@@ -0,0 +1,270 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class C(cupy.ndarray):
+
+    def __new__(cls, *args, info=None, **kwargs):
+        obj = super().__new__(cls, *args, **kwargs)
+        obj.info = info
+        return obj
+
+    def __array_finalize__(self, obj):
+        if obj is None:
+            return
+        self.info = getattr(obj, 'info', None)
+
+
+class TestArrayUfunc:
+
+    @testing.for_all_dtypes()
+    def test_unary_op(self, dtype):
+        a = cupy.array(numpy.array([0, 1, 2]), dtype=dtype)
+        outa = numpy.sin(a)
+        # numpy operation produced a cupy array
+        assert isinstance(outa, cupy.ndarray)
+        b = a.get()
+        outb = numpy.sin(b)
+        assert numpy.allclose(outa.get(), outb)
+
+    @testing.for_all_dtypes()
+    def test_unary_op_out(self, dtype):
+        a = cupy.array(numpy.array([0, 1, 2]), dtype=dtype)
+        b = a.get()
+        outb = numpy.sin(b)
+        # pre-make output with same type as input
+        outa = cupy.array(numpy.array([0, 1, 2]), dtype=outb.dtype)
+        numpy.sin(a, out=outa)
+        assert numpy.allclose(outa.get(), outb)
+
+    @testing.for_all_dtypes()
+    def test_binary_op(self, dtype):
+        a1 = cupy.array(numpy.array([0, 1, 2]), dtype=dtype)
+        a2 = cupy.array(numpy.array([0, 1, 2]), dtype=dtype)
+        outa = numpy.add(a1, a2)
+        # numpy operation produced a cupy array
+        assert isinstance(outa, cupy.ndarray)
+        b1 = a1.get()
+        b2 = a2.get()
+        outb = numpy.add(b1, b2)
+        assert numpy.allclose(outa.get(), outb)
+
+    @testing.for_all_dtypes()
+    def test_binary_op_out(self, dtype):
+        a1 = cupy.array(numpy.array([0, 1, 2]), dtype=dtype)
+        a2 = cupy.array(numpy.array([0, 1, 2]), dtype=dtype)
+        outa = cupy.array(numpy.array([0, 1, 2]), dtype=dtype)
+        numpy.add(a1, a2, out=outa)
+        b1 = a1.get()
+        b2 = a2.get()
+        outb = numpy.add(b1, b2)
+        assert numpy.allclose(outa.get(), outb)
+
+    @testing.for_all_dtypes()
+    def test_binary_mixed_op(self, dtype):
+        a1 = cupy.array(numpy.array([0, 1, 2]), dtype=dtype)
+        a2 = cupy.array(numpy.array([0, 1, 2]), dtype=dtype).get()
+        with pytest.raises(TypeError):
+            # attempt to add cupy and numpy arrays
+            numpy.add(a1, a2)
+        with pytest.raises(TypeError):
+            # check reverse order
+            numpy.add(a2, a1)
+        with pytest.raises(TypeError):
+            # reject numpy output from cupy
+            numpy.add(a1, a1, out=a2)
+        with pytest.raises(TypeError):
+            # reject cupy output from numpy
+            numpy.add(a2, a2, out=a1)
+        with pytest.raises(ValueError):
+            # bad form for out=
+            # this is also an error with numpy array
+            numpy.sin(a1, out=())
+        with pytest.raises(ValueError):
+            # bad form for out=
+            # this is also an error with numpy array
+            numpy.sin(a1, out=(a1, a1))
+
+    @testing.numpy_cupy_array_equal()
+    def test_indexing(self, xp):
+        a = cupy.testing.shaped_arange((3, 1), xp)[:, :, None]
+        b = cupy.testing.shaped_arange((3, 2), xp)[:, None, :]
+        return a * b
+
+    @testing.numpy_cupy_array_equal()
+    def test_shares_memory(self, xp):
+        a = cupy.testing.shaped_arange((1000, 1000), xp, 'int64')
+        b = xp.transpose(a)
+        a += b
+        return a
+
+    def test_subclass_unary_op(self):
+        a = cupy.array([0, 1, 2]).view(C)
+        a.info = 1
+        outa = cupy.sin(a)
+        assert isinstance(outa, C)
+        assert outa.info is not None and outa.info == 1
+
+        b = a.get()
+        outb = numpy.sin(b)
+        testing.assert_allclose(outa, outb)
+
+    def test_subclass_binary_op(self):
+        a0 = cupy.array([0, 1, 2]).view(C)
+        a0.info = 1
+        a1 = cupy.array([3, 4, 5]).view(C)
+        a1.info = 2
+        outa = cupy.add(a0, a1)
+        assert isinstance(outa, C)
+        # a0 is used to initialize outa.info
+        assert outa.info is not None and outa.info == 1
+
+        b0 = a0.get()
+        b1 = a1.get()
+        outb = numpy.add(b0, b1)
+        testing.assert_allclose(outa, outb)
+
+    def test_subclass_binary_op_mixed(self):
+        a0 = cupy.array([0, 1, 2])
+        a1 = cupy.array([3, 4, 5]).view(C)
+        a1.info = 1
+        outa = cupy.add(a0, a1)
+        assert isinstance(outa, C)
+        # The first appearance of C's instance is used to initialize outa.info
+        assert outa.info is not None and outa.info == 1
+
+        b0 = a0.get()
+        b1 = a1.get()
+        outb = numpy.add(b0, b1)
+        testing.assert_allclose(outa, outb)
+
+    @testing.numpy_cupy_array_equal()
+    def test_ufunc_outer(self, xp):
+        a = cupy.testing.shaped_arange((3, 4), xp)
+        b = cupy.testing.shaped_arange((5, 6), xp)
+        return numpy.add.outer(a, b)
+
+    @testing.numpy_cupy_array_equal()
+    def test_ufunc_at(self, xp):
+        a = cupy.testing.shaped_arange((10,), xp)
+        b = cupy.testing.shaped_arange((5,), xp)
+        indices = xp.array([0, 3, 6, 7, 9])
+        numpy.add.at(a, indices, b)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_ufunc_at_scalar(self, xp):
+        a = cupy.testing.shaped_arange((10,), xp)
+        b = 7
+        indices = xp.array([0, 3, 6, 7, 9])
+        numpy.add.at(a, indices, b)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_ufunc_reduce(self, xp):
+        a = cupy.testing.shaped_arange((10, 12), xp)
+        return numpy.add.reduce(a, axis=-1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_ufunc_accumulate(self, xp):
+        a = cupy.testing.shaped_arange((10, 12), xp)
+        return numpy.add.accumulate(a, axis=-1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_ufunc_reduceat(self, xp):
+        a = cupy.testing.shaped_arange((10, 12), xp)
+        indices = xp.array([0, 3, 6, 7, 9])
+        return numpy.add.reduceat(a, indices, axis=-1)
+
+
+class TestUfunc:
+    @pytest.mark.parametrize('ufunc', [
+        'add',
+        'sin',
+    ])
+    @testing.numpy_cupy_equal()
+    def test_types(self, xp, ufunc):
+        types = getattr(xp, ufunc).types
+        if xp == numpy:
+            assert isinstance(types, list)
+            types = list(dict.fromkeys(  # remove dups: numpy/numpy#7897
+                sig for sig in types
+                # CuPy does not support the following dtypes:
+                # (c)longdouble, datetime, timedelta, and object.
+                if not any(t in sig for t in 'GgMmO')
+            ))
+        return types
+
+    @testing.numpy_cupy_allclose()
+    def test_unary_out_tuple(self, xp):
+        dtype = xp.float64
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        out = xp.zeros((2, 3), dtype)
+        ret = xp.sin(a, out=(out,))
+        assert ret is out
+        return ret
+
+    @testing.numpy_cupy_allclose()
+    def test_unary_out_positional_none(self, xp):
+        dtype = xp.float64
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.sin(a, None)
+
+    @testing.numpy_cupy_allclose()
+    def test_binary_out_tuple(self, xp):
+        dtype = xp.float64
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = xp.ones((2, 3), dtype)
+        out = xp.zeros((2, 3), dtype)
+        ret = xp.add(a, b, out=(out,))
+        assert ret is out
+        return ret
+
+    @testing.numpy_cupy_allclose()
+    def test_biary_out_positional_none(self, xp):
+        dtype = xp.float64
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = xp.ones((2, 3), dtype)
+        return xp.add(a, b, None)
+
+    @testing.numpy_cupy_allclose()
+    def test_divmod_out_tuple(self, xp):
+        dtype = xp.float64
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype)
+        out0 = xp.zeros((2, 3), dtype)
+        out1 = xp.zeros((2, 3), dtype)
+        ret = xp.divmod(a, b, out=(out0, out1))
+        assert ret[0] is out0
+        assert ret[1] is out1
+        return ret
+
+    @testing.numpy_cupy_allclose()
+    def test_divmod_out_positional_none(self, xp):
+        dtype = xp.float64
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = xp.ones((2, 3), dtype)
+        return xp.divmod(a, b, None, None)
+
+    @testing.numpy_cupy_allclose()
+    def test_divmod_out_partial(self, xp):
+        dtype = xp.float64
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype)
+        out0 = xp.zeros((2, 3), dtype)
+        ret = xp.divmod(a, b, out0)  # out1 is None
+        assert ret[0] is out0
+        return ret
+
+    @testing.numpy_cupy_allclose()
+    def test_divmod_out_partial_tuple(self, xp):
+        dtype = xp.float64
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype)
+        out1 = xp.zeros((2, 3), dtype)
+        ret = xp.divmod(a, b, out=(None, out1))
+        assert ret[1] is out1
+        return ret
diff --git a/tests/cupy_tests/core_tests/test_ndarray_unary_op.py b/tests/cupy_tests/core_tests/test_ndarray_unary_op.py
new file mode 100644
index 0000000..3647d4b
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ndarray_unary_op.py
@@ -0,0 +1,145 @@
+import operator
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestArrayBoolOp(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_bool_empty(self, dtype):
+        with testing.assert_warns(DeprecationWarning):
+            assert not bool(cupy.array((), dtype=dtype))
+
+    def test_bool_scalar_bool(self):
+        assert bool(cupy.array(True, dtype=numpy.bool_))
+        assert not bool(cupy.array(False, dtype=numpy.bool_))
+
+    @testing.for_all_dtypes()
+    def test_bool_scalar(self, dtype):
+        assert bool(cupy.array(1, dtype=dtype))
+        assert not bool(cupy.array(0, dtype=dtype))
+
+    def test_bool_one_element_bool(self):
+        assert bool(cupy.array([True], dtype=numpy.bool_))
+        assert not bool(cupy.array([False], dtype=numpy.bool_))
+
+    @testing.for_all_dtypes()
+    def test_bool_one_element(self, dtype):
+        assert bool(cupy.array([1], dtype=dtype))
+        assert not bool(cupy.array([0], dtype=dtype))
+
+    @testing.for_all_dtypes()
+    def test_bool_two_elements(self, dtype):
+        with self.assertRaises(ValueError):
+            bool(cupy.array([1, 2], dtype=dtype))
+
+
+class TestArrayUnaryOp(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def check_array_op(self, op, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return op(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def check_array_op_full(self, op, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return op(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_neg_array(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return operator.neg(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_pos_array(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        assert a is not +a
+        return +a
+
+    @testing.with_requires('numpy<1.25')
+    def test_pos_boolarray(self):
+        for xp in (numpy, cupy):
+            a = xp.array(True, dtype=xp.bool_)
+            with pytest.deprecated_call():
+                assert a is not +a
+
+    @testing.with_requires('numpy<1.16')
+    def test_pos_array_full(self):
+        self.check_array_op_full(operator.pos)
+
+    def test_abs_array(self):
+        self.check_array_op_full(operator.abs)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def check_zerodim_op(self, op, xp, dtype):
+        a = xp.array(-2).astype(dtype)
+        return op(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def check_zerodim_op_full(self, op, xp, dtype):
+        a = xp.array(-2).astype(dtype)
+        return op(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_neg_zerodim(self, xp, dtype):
+        a = xp.array(-2).astype(dtype)
+        return operator.neg(a)
+
+    def test_pos_zerodim(self):
+        self.check_zerodim_op(operator.pos)
+
+    def test_abs_zerodim(self):
+        self.check_zerodim_op_full(operator.abs)
+
+    def test_abs_zerodim_full(self):
+        self.check_zerodim_op_full(operator.abs)
+
+
+class TestArrayIntUnaryOp(unittest.TestCase):
+
+    @testing.for_int_dtypes()
+    @testing.numpy_cupy_allclose()
+    def check_array_op(self, op, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return op(a)
+
+    def test_invert_array(self):
+        self.check_array_op(operator.invert)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_zerodim_op(self, op, xp, dtype):
+        a = xp.array(-2).astype(dtype)
+        return op(a)
+
+    def test_invert_zerodim(self):
+        self.check_zerodim_op(operator.invert)
+
+
+@testing.parameterize(*testing.product({
+    'xp': [numpy, cupy],
+    'shape': [(3, 2), (), (3, 0, 2)]
+}))
+class TestBoolNeg(unittest.TestCase):
+
+    def test_bool_neg(self):
+        xp = self.xp
+        if xp is numpy and not testing.numpy_satisfies('>=1.13.0'):
+            raise unittest.SkipTest('NumPy<1.13.0')
+        shape = self.shape
+        x = testing.shaped_random(shape, xp, dtype=numpy.bool_)
+        with pytest.raises(TypeError):
+            -x
diff --git a/tests/cupy_tests/core_tests/test_raw.py b/tests/cupy_tests/core_tests/test_raw.py
new file mode 100644
index 0000000..d3fb884
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_raw.py
@@ -0,0 +1,1367 @@
+import contextlib
+import io
+import os
+import pickle
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+import pytest
+
+import cupy
+from cupy import testing
+from cupy import _util
+from cupy._core import _accelerator
+from cupy.cuda import compiler
+from cupy.cuda import memory
+
+
+_test_source1 = r'''
+extern "C" __global__
+void test_sum(const float* x1, const float* x2, float* y, unsigned int N) {
+    int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N)
+        y[tid] = x1[tid] + x2[tid];
+}
+'''
+
+_test_compile_src = r'''
+extern "C" __global__
+void test_op(const float* x1, const float* x2, float* y, unsigned int N) {
+    int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    int j;  // To generate a warning to appear in the log stream
+    if (tid < N)
+        y[tid] = x1[tid] OP x2[tid];
+}
+'''
+
+# test compiling and invoking multiple kernels in one single .cubin
+_test_source2 = r'''
+extern "C"{
+
+__global__ void test_sum(const float* x1, const float* x2, float* y, \
+                         unsigned int N)
+{
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N)
+    {
+        y[tid] = x1[tid] + x2[tid];
+    }
+}
+
+__global__ void test_multiply(const float* x1, const float* x2, float* y, \
+                              unsigned int N)
+{
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N)
+    {
+        y[tid] = x1[tid] * x2[tid];
+    }
+}
+
+}
+'''
+
+# test C macros
+_test_source3 = r'''
+#ifndef PRECISION
+    #define PRECISION 2
+#endif
+
+#if PRECISION == 2
+    #define TYPE double
+#elif PRECISION == 1
+    #define TYPE float
+#else
+    #error precision not supported
+#endif
+
+extern "C"{
+
+__global__ void test_sum(const TYPE* x1, const TYPE* x2, TYPE* y, \
+                         unsigned int N)
+{
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N)
+    {
+        y[tid] = x1[tid] + x2[tid];
+    }
+}
+
+__global__ void test_multiply(const TYPE* x1, const TYPE* x2, TYPE* y, \
+                              unsigned int N)
+{
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N)
+    {
+        y[tid] = x1[tid] * x2[tid];
+    }
+}
+
+}
+'''
+
+# dynamic parallelism
+_test_source4 = r'''
+extern "C"{
+
+__global__ void test_kernel_inner(float *arr, int N)
+{
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if (tid < N)
+        arr[tid] = 1.0;
+}
+
+__global__ void test_kernel(float *arr, int N, int inner_blk)
+{
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if (tid < N/inner_blk)
+        test_kernel_inner<<<1, inner_blk>>>(arr+tid*inner_blk, inner_blk);
+}
+
+}
+'''
+
+# to generate cubin/ptx
+_test_source5 = r'''
+extern "C" __global__
+void test_div(const float* x1, const float* x2, float* y, unsigned int N) {
+    int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N)
+        y[tid] = x1[tid] / (x2[tid] + 1.0);
+}
+'''
+
+_test_cuComplex = r'''
+#include <cuComplex.h>
+#define N 100
+
+extern "C"{
+/* ------------------- double complex ------------------- */
+
+__global__ void test_add(cuDoubleComplex* arr1, cuDoubleComplex* arr2,
+                         cuDoubleComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCadd(arr1[tid], arr2[tid]);
+    }
+}
+
+__global__ void test_sub(cuDoubleComplex* arr1, cuDoubleComplex* arr2,
+                         cuDoubleComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCsub(arr1[tid], arr2[tid]);
+    }
+}
+
+__global__ void test_mul(cuDoubleComplex* arr1, cuDoubleComplex* arr2,
+                         cuDoubleComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCmul(arr1[tid], arr2[tid]);
+    }
+}
+
+__global__ void test_div(cuDoubleComplex* arr1, cuDoubleComplex* arr2,
+                         cuDoubleComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCdiv(arr1[tid], arr2[tid]);
+    }
+}
+
+__global__ void test_conj(cuDoubleComplex* arr, cuDoubleComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuConj(arr[tid]);
+    }
+}
+
+__global__ void test_abs(cuDoubleComplex* arr, double* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCabs(arr[tid]);
+    }
+}
+
+__global__ void test_fma(cuDoubleComplex* A, cuDoubleComplex* B,
+                         cuDoubleComplex* C, cuDoubleComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCfma(A[tid], B[tid], C[tid]);
+    }
+}
+
+__global__ void test_make(cuDoubleComplex* arr) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    cuDoubleComplex out = make_cuDoubleComplex(1.8, 2.9);
+    if (tid < N) {
+        arr[tid] = make_cuDoubleComplex(cuCreal(out), -3.*cuCimag(out));
+    }
+}
+
+__global__ void test_downcast(cuDoubleComplex* arr, cuComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuComplexDoubleToFloat(arr[tid]);
+    }
+}
+
+__global__ void test_add_scalar(cuDoubleComplex* arr, cuDoubleComplex scalar,
+                                cuDoubleComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCadd(arr[tid], scalar);
+    }
+}
+
+/* ------------------- single complex ------------------- */
+
+__global__ void test_addf(cuComplex* arr1, cuComplex* arr2,
+                          cuComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCaddf(arr1[tid], arr2[tid]);
+    }
+}
+
+__global__ void test_subf(cuComplex* arr1, cuComplex* arr2,
+                          cuComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCsubf(arr1[tid], arr2[tid]);
+    }
+}
+
+__global__ void test_mulf(cuComplex* arr1, cuComplex* arr2,
+                          cuComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCmulf(arr1[tid], arr2[tid]);
+    }
+}
+
+__global__ void test_divf(cuComplex* arr1, cuComplex* arr2,
+                          cuComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCdivf(arr1[tid], arr2[tid]);
+    }
+}
+
+__global__ void test_conjf(cuComplex* arr, cuComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuConjf(arr[tid]);
+    }
+}
+
+__global__ void test_absf(cuFloatComplex* arr, float* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCabsf(arr[tid]);
+    }
+}
+
+__global__ void test_fmaf(cuFloatComplex* A, cuFloatComplex* B,
+                          cuFloatComplex* C, cuFloatComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCfmaf(A[tid], B[tid], C[tid]);
+    }
+}
+
+__global__ void test_makef(cuComplex* arr) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    cuComplex out = make_cuFloatComplex(1.8, 2.9);
+    if (tid < N) {
+        arr[tid] = make_cuFloatComplex(cuCrealf(out), -3.*cuCimagf(out));
+    }
+}
+
+__global__ void test_upcast(cuComplex* arr, cuDoubleComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuComplexFloatToDouble(arr[tid]);
+    }
+}
+
+__global__ void test_addf_scalar(cuComplex* arr, cuComplex scalar,
+                                 cuComplex* out) {
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    if (tid < N) {
+        out[tid] = cuCadd(arr[tid], scalar);
+    }
+}
+
+}
+'''
+
+test_const_mem = r'''
+extern "C"{
+__constant__ float some_array[100];
+
+__global__ void multiply_by_const(float* x, int N) {
+    int id = threadIdx.x + blockIdx.x * blockDim.x;
+
+    if (id < N) {
+        x[id] *= some_array[id];
+    }
+}
+}
+'''
+
+test_cxx_template = r'''
+#include <cupy/complex.cuh>
+
+template<typename T>
+__global__ void my_sqrt(T* input, int N) {
+  unsigned int x = blockIdx.x * blockDim.x + threadIdx.x;
+  if (x < N) {
+    input[x] *= input[x];
+  }
+}
+
+__global__ void my_func(double* input, int N) {
+  unsigned int x = blockIdx.x * blockDim.x + threadIdx.x;
+  if (x < N) {
+    input[x] *= input[x];
+  }
+}
+'''
+
+test_cast = r'''
+extern "C" __global__ void my_func(void* input, int N) {
+  double* arr = (double*)(input);
+  unsigned int x = blockIdx.x * blockDim.x + threadIdx.x;
+  if (x < N) {
+    arr[x] = 3.0 * arr[x] - 8.0;
+  }
+}
+'''
+
+
+@contextlib.contextmanager
+def use_temporary_cache_dir():
+    target1 = 'cupy.cuda.compiler.get_cache_dir'
+    target2 = 'cupy.cuda.compiler._empty_file_preprocess_cache'
+    temp_cache = {}
+    with tempfile.TemporaryDirectory() as path:
+        with mock.patch(target1, lambda: path):
+            with mock.patch(target2, temp_cache):
+                yield path
+
+
+@contextlib.contextmanager
+def compile_in_memory(in_memory):
+    target = 'cupy.cuda.compiler._get_bool_env_variable'
+
+    def new_target(name, default):
+        if name == 'CUPY_CACHE_IN_MEMORY':
+            return in_memory
+        else:
+            # below is the source code of _get_bool_env_variable
+            val = os.environ.get(name)
+            if val is None or len(val) == 0:
+                return default
+            try:
+                return int(val) == 1
+            except ValueError:
+                return False
+
+    with mock.patch(target, new_target) as m:
+        yield m
+
+
+@testing.parameterize(
+    # First test NVRTC
+    {'backend': 'nvrtc', 'in_memory': False},
+    # this run will read from in-memory cache
+    {'backend': 'nvrtc', 'in_memory': True},
+    # this run will force recompilation
+    {'backend': 'nvrtc', 'in_memory': True, 'clean_up': True},
+    # Below is the same set of NVRTC tests, with Jitify turned on. For tests
+    # that can already pass, it shouldn't matter whether Jitify is on or not,
+    # and the only side effect is to add overhead. It doesn't make sense to
+    # test NVCC + Jitify.
+    {'backend': 'nvrtc', 'in_memory': False, 'jitify': True},
+    {'backend': 'nvrtc', 'in_memory': True, 'jitify': True},
+    {'backend': 'nvrtc', 'in_memory': True, 'clean_up': True, 'jitify': True},
+    # Finally, we test NVCC
+    {'backend': 'nvcc', 'in_memory': False},
+)
+class TestRaw(unittest.TestCase):
+
+    def setUp(self):
+        if hasattr(self, 'clean_up'):
+            if cupy.cuda.runtime.is_hip:
+                # Clearing memo triggers recompiling kernels using name
+                # expressions in other tests, e.g. dot and matmul, which
+                # hits a nvrtc bug. See #5843, #5945 and #6725.
+                self.skipTest('Clearing memo hits a nvrtc bug in other tests')
+            _util.clear_memo()
+        self.dev = cupy.cuda.runtime.getDevice()
+        assert self.dev != 1
+        if not hasattr(self, 'jitify'):
+            self.jitify = False
+        if cupy.cuda.runtime.is_hip and self.jitify:
+            self.skipTest('Jitify does not support ROCm/HIP')
+
+        self.temporary_cache_dir_context = use_temporary_cache_dir()
+        self.in_memory_context = compile_in_memory(self.in_memory)
+        self.cache_dir = self.temporary_cache_dir_context.__enter__()
+        self.in_memory_context.__enter__()
+
+        self.kern = cupy.RawKernel(
+            _test_source1, 'test_sum',
+            backend=self.backend, jitify=self.jitify)
+        self.mod2 = cupy.RawModule(
+            code=_test_source2,
+            backend=self.backend, jitify=self.jitify)
+        self.mod3 = cupy.RawModule(
+            code=_test_source3,
+            options=('-DPRECISION=2',),
+            backend=self.backend, jitify=self.jitify)
+
+    def tearDown(self):
+        if (self.in_memory
+                and _accelerator.ACCELERATOR_CUB not in
+                _accelerator.get_reduction_accelerators()):
+            # should not write any file to the cache dir, but the CUB reduction
+            # kernel uses nvcc, with which I/O cannot be avoided
+            files = os.listdir(self.cache_dir)
+            for f in files:
+                if f == 'test_load_cubin.cu':
+                    count = 1
+                    break
+            else:
+                count = 0
+            assert len(files) == count
+
+        self.in_memory_context.__exit__(*sys.exc_info())
+        self.temporary_cache_dir_context.__exit__(*sys.exc_info())
+
+    def _helper(self, kernel, dtype):
+        N = 10
+        x1 = cupy.arange(N**2, dtype=dtype).reshape(N, N)
+        x2 = cupy.ones((N, N), dtype=dtype)
+        y = cupy.zeros((N, N), dtype=dtype)
+        kernel((N,), (N,), (x1, x2, y, N**2))
+        return x1, x2, y
+
+    def test_basic(self):
+        x1, x2, y = self._helper(self.kern, cupy.float32)
+        assert cupy.allclose(y, x1 + x2)
+
+    def test_kernel_attributes(self):
+        attrs = self.kern.attributes
+        for attribute in ['binary_version',
+                          'cache_mode_ca',
+                          'const_size_bytes',
+                          'local_size_bytes',
+                          'max_dynamic_shared_size_bytes',
+                          'max_threads_per_block',
+                          'num_regs',
+                          'preferred_shared_memory_carveout',
+                          'ptx_version',
+                          'shared_size_bytes']:
+            assert attribute in attrs
+        # TODO(leofang): investigate why this fails on ROCm 3.5.0
+        if not cupy.cuda.runtime.is_hip:
+            assert self.kern.num_regs > 0
+        assert self.kern.max_threads_per_block > 0
+        assert self.kern.shared_size_bytes == 0
+
+    def test_module(self):
+        module = self.mod2
+        ker_sum = module.get_function('test_sum')
+        ker_times = module.get_function('test_multiply')
+
+        x1, x2, y = self._helper(ker_sum, cupy.float32)
+        assert cupy.allclose(y, x1 + x2)
+
+        x1, x2, y = self._helper(ker_times, cupy.float32)
+        assert cupy.allclose(y, x1 * x2)
+
+    def test_compiler_flag(self):
+        module = self.mod3
+        ker_sum = module.get_function('test_sum')
+        ker_times = module.get_function('test_multiply')
+
+        x1, x2, y = self._helper(ker_sum, cupy.float64)
+        assert cupy.allclose(y, x1 + x2)
+
+        x1, x2, y = self._helper(ker_times, cupy.float64)
+        assert cupy.allclose(y, x1 * x2)
+
+    def test_invalid_compiler_flag(self):
+        if cupy.cuda.runtime.is_hip and self.backend == 'nvrtc':
+            self.skipTest('hiprtc does not handle #error macro properly')
+
+        if self.jitify:
+            ex_type = cupy.cuda.compiler.JitifyException
+        else:
+            ex_type = cupy.cuda.compiler.CompileException
+
+        with pytest.raises(ex_type) as ex:
+            mod = cupy.RawModule(code=_test_source3,
+                                 options=('-DPRECISION=3',),
+                                 backend=self.backend,
+                                 jitify=self.jitify)
+            mod.get_function('test_sum')  # enforce compilation
+
+        if not self.jitify:
+            assert 'precision not supported' in str(ex.value)
+
+    def _generate_file(self, ext: str):
+        # generate cubin/ptx by calling nvcc/hipcc
+
+        if not cupy.cuda.runtime.is_hip:
+            cc = cupy.cuda.get_nvcc_path()
+            arch = '-gencode=arch=compute_{CC},code=sm_{CC}'.format(
+                CC=compiler._get_arch())
+            code = _test_source5
+        else:
+            # TODO(leofang): expose get_hipcc_path() to cupy.cuda?
+            cc = cupy._environment.get_hipcc_path()
+            arch = '-v'  # dummy
+            code = compiler._convert_to_hip_source(_test_source5, None, False)
+        # split() is needed because nvcc could come from the env var NVCC
+        cmd = cc.split()
+        source = '{}/test_load_cubin.cu'.format(self.cache_dir)
+        file_path = self.cache_dir + 'test_load_cubin'
+        with open(source, 'w') as f:
+            f.write(code)
+        if not cupy.cuda.runtime.is_hip:
+            if ext == 'cubin':
+                file_path += '.cubin'
+                flag = '-cubin'
+            elif ext == 'ptx':
+                file_path += '.ptx'
+                flag = '-ptx'
+            else:
+                raise ValueError
+        else:
+            file_path += '.hsaco'
+            flag = '--genco'
+        cmd += [arch, flag, source, '-o', file_path]
+        cc = 'nvcc' if not cupy.cuda.runtime.is_hip else 'hipcc'
+        compiler._run_cc(cmd, self.cache_dir, cc)
+
+        return file_path
+
+    @unittest.skipIf(cupy.cuda.runtime.is_hip, 'HIP uses hsaco, not cubin')
+    def test_load_cubin(self):
+        # generate cubin in the temp dir
+        file_path = self._generate_file('cubin')
+
+        # load cubin and test the kernel
+        mod = cupy.RawModule(path=file_path, backend=self.backend)
+        ker = mod.get_function('test_div')
+        x1, x2, y = self._helper(ker, cupy.float32)
+        assert cupy.allclose(y, x1 / (x2 + 1.0))
+
+    @unittest.skipIf(cupy.cuda.runtime.is_hip, 'HIP uses hsaco, not ptx')
+    def test_load_ptx(self):
+        # generate ptx in the temp dir
+        file_path = self._generate_file('ptx')
+
+        # load ptx and test the kernel
+        mod = cupy.RawModule(path=file_path, backend=self.backend)
+        ker = mod.get_function('test_div')
+        x1, x2, y = self._helper(ker, cupy.float32)
+        assert cupy.allclose(y, x1 / (x2 + 1.0))
+
+    @unittest.skipIf(not cupy.cuda.runtime.is_hip,
+                     'CUDA uses cubin/ptx, not hsaco')
+    def test_load_hsaco(self):
+        # generate hsaco in the temp dir
+        file_path = self._generate_file('hsaco')
+
+        # load cubin and test the kernel
+        mod = cupy.RawModule(path=file_path, backend=self.backend)
+        ker = mod.get_function('test_div')
+        x1, x2, y = self._helper(ker, cupy.float32)
+        assert cupy.allclose(y, x1 / (x2 + 1.0))
+
+    def test_module_load_failure(self):
+        # in principle this test is better done in test_driver.py, but
+        # this error is more likely to appear when using RawModule, so
+        # let us do it here
+        with pytest.raises(cupy.cuda.driver.CUDADriverError) as ex:
+            mod = cupy.RawModule(
+                path=os.path.expanduser('~/this_does_not_exist.cubin'),
+                backend=self.backend)
+            mod.get_function('nonexisting_kernel')  # enforce loading
+        assert ('CUDA_ERROR_FILE_NOT_FOUND' in str(ex.value)  # CUDA
+                or 'hipErrorFileNotFound' in str(ex.value))  # HIP
+
+    def test_module_neither_code_nor_path(self):
+        with pytest.raises(TypeError):
+            cupy.RawModule()
+
+    def test_module_both_code_and_path(self):
+        with pytest.raises(TypeError):
+            cupy.RawModule(
+                code=_test_source1,
+                path='test.cubin')
+
+    def test_get_function_failure(self):
+        # in principle this test is better done in test_driver.py, but
+        # this error is more likely to appear when using RawModule, so
+        # let us do it here
+        with pytest.raises(cupy.cuda.driver.CUDADriverError) as ex:
+            self.mod2.get_function('no_such_kernel')
+        assert ('CUDA_ERROR_NOT_FOUND' in str(ex.value)  # for CUDA
+                or 'hipErrorNotFound' in str(ex.value))  # for HIP
+
+    @unittest.skipIf(cupy.cuda.runtime.is_hip,
+                     'ROCm/HIP does not support dynamic parallelism')
+    def test_dynamical_parallelism(self):
+        ker = cupy.RawKernel(_test_source4, 'test_kernel', options=('-dc',),
+                             backend=self.backend, jitify=self.jitify)
+        N = 169
+        inner_chunk = 13
+        x = cupy.zeros((N,), dtype=cupy.float32)
+        ker((1,), (N//inner_chunk,), (x, N, inner_chunk))
+        assert (x == 1.0).all()
+
+    def test_dynamical_parallelism_compile_failure(self):
+        # no option for separate compilation is given should cause an error
+        ker = cupy.RawKernel(_test_source4, 'test_kernel',
+                             backend=self.backend, jitify=self.jitify)
+        N = 10
+        inner_chunk = 2
+        x = cupy.zeros((N,), dtype=cupy.float32)
+        use_ptx = os.environ.get(
+            'CUPY_COMPILE_WITH_PTX', False)
+        if self.backend == 'nvrtc' and (
+                use_ptx or
+                (cupy.cuda.driver._is_cuda_python()
+                 and cupy.cuda.runtime.runtimeGetVersion() < 11010) or
+                (not cupy.cuda.driver._is_cuda_python()
+                 and not cupy.cuda.runtime.is_hip
+                 and cupy.cuda.driver.get_build_version() < 11010)):
+            # raised when calling ls.complete()
+            error = cupy.cuda.driver.CUDADriverError
+        else:  # nvcc, hipcc, hiprtc
+            error = cupy.cuda.compiler.CompileException
+        with pytest.raises(error):
+            ker((1,), (N//inner_chunk,), (x, N, inner_chunk))
+
+    @unittest.skipIf(cupy.cuda.runtime.is_hip,
+                     'HIP code should not use cuFloatComplex')
+    def test_cuFloatComplex(self):
+        N = 100
+        block = 32
+        grid = (N + block - 1) // block
+        dtype = cupy.complex64
+
+        mod = cupy.RawModule(
+            code=_test_cuComplex,
+            translate_cucomplex=True,
+            jitify=self.jitify)
+        a = cupy.random.random((N,)) + 1j*cupy.random.random((N,))
+        a = a.astype(dtype)
+        b = cupy.random.random((N,)) + 1j*cupy.random.random((N,))
+        b = b.astype(dtype)
+        c = cupy.random.random((N,)) + 1j*cupy.random.random((N,))
+        c = c.astype(dtype)
+        out = cupy.zeros((N,), dtype=dtype)
+        out_float = cupy.zeros((N,), dtype=cupy.float32)
+        out_up = cupy.zeros((N,), dtype=cupy.complex128)
+
+        ker = mod.get_function('test_addf')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a + b).all()
+
+        ker = mod.get_function('test_subf')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a - b).all()
+
+        ker = mod.get_function('test_mulf')
+        ker((grid,), (block,), (a, b, out))
+        assert cupy.allclose(out, a * b)
+
+        ker = mod.get_function('test_divf')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a / b).all()
+
+        ker = mod.get_function('test_conjf')
+        ker((grid,), (block,), (a, out))
+        assert (out == cupy.conj(a)).all()
+
+        ker = mod.get_function('test_absf')
+        ker((grid,), (block,), (a, out_float))
+        assert (out_float == cupy.abs(a)).all()
+
+        ker = mod.get_function('test_fmaf')
+        ker((grid,), (block,), (a, b, c, out))
+        assert cupy.allclose(out, a * b + c)
+
+        ker = mod.get_function('test_makef')
+        ker((grid,), (block,), (out,))
+        # because of precision issue, the (A==B).all() semantics would fail
+        assert cupy.allclose(out, 1.8 - 1j * 8.7)
+
+        ker = mod.get_function('test_upcast')
+        ker((grid,), (block,), (a, out_up))
+        assert (out_up == a.astype(cupy.complex128)).all()
+
+        # NumPy scalars.
+        b = cupy.complex64(2 + 3j)
+        ker = mod.get_function('test_addf_scalar')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a + b).all()
+
+    @unittest.skipIf(cupy.cuda.runtime.is_hip,
+                     'HIP code should not use cuDoubleComplex')
+    def test_cuDoubleComplex(self):
+        N = 100
+        block = 32
+        grid = (N + block - 1) // block
+        dtype = cupy.complex128
+
+        mod = cupy.RawModule(
+            code=_test_cuComplex,
+            translate_cucomplex=True,
+            jitify=self.jitify)
+        a = cupy.random.random((N,)) + 1j*cupy.random.random((N,))
+        a = a.astype(dtype)
+        b = cupy.random.random((N,)) + 1j*cupy.random.random((N,))
+        b = b.astype(dtype)
+        c = cupy.random.random((N,)) + 1j*cupy.random.random((N,))
+        c = c.astype(dtype)
+        out = cupy.zeros((N,), dtype=dtype)
+        out_float = cupy.zeros((N,), dtype=cupy.float64)
+        out_down = cupy.zeros((N,), dtype=cupy.complex64)
+
+        ker = mod.get_function('test_add')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a + b).all()
+
+        ker = mod.get_function('test_sub')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a - b).all()
+
+        ker = mod.get_function('test_mul')
+        ker((grid,), (block,), (a, b, out))
+        assert cupy.allclose(out, a * b)
+
+        ker = mod.get_function('test_div')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a / b).all()
+
+        ker = mod.get_function('test_conj')
+        ker((grid,), (block,), (a, out))
+        assert (out == cupy.conj(a)).all()
+
+        ker = mod.get_function('test_abs')
+        ker((grid,), (block,), (a, out_float))
+        assert (out_float == cupy.abs(a)).all()
+
+        ker = mod.get_function('test_fma')
+        ker((grid,), (block,), (a, b, c, out))
+        assert cupy.allclose(out, a * b + c)
+
+        ker = mod.get_function('test_make')
+        ker((grid,), (block,), (out,))
+        assert (out == 1.8 - 1j * 8.7).all()
+
+        ker = mod.get_function('test_downcast')
+        ker((grid,), (block,), (a, out_down))
+        assert (out_down == a.astype(cupy.complex64)).all()
+
+        # NumPy scalars.
+        b = cupy.complex128(2 + 3j)
+        ker = mod.get_function('test_add_scalar')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a + b).all()
+
+        # Python scalars.
+        b = 2 + 3j
+        ker = mod.get_function('test_add_scalar')
+        ker((grid,), (block,), (a, b, out))
+        assert (out == a + b).all()
+
+    def test_const_memory(self):
+        mod = cupy.RawModule(code=test_const_mem,
+                             backend=self.backend,
+                             jitify=self.jitify)
+        ker = mod.get_function('multiply_by_const')
+        mem_ptr = mod.get_global('some_array')
+        const_arr = cupy.ndarray((100,), cupy.float32, mem_ptr)
+        data = cupy.arange(100, dtype=cupy.float32)
+        const_arr[...] = data
+        output_arr = cupy.ones(100, dtype=cupy.float32)
+        ker((1,), (100,), (output_arr, cupy.int32(100)))
+        assert (data == output_arr).all()
+
+    def test_template_specialization(self):
+        if self.backend == 'nvcc':
+            self.skipTest('nvcc does not support template specialization')
+
+        # TODO(leofang): investigate why hiprtc generates a wrong source code
+        # when the same code is compiled and discarded. It seems hiprtc has
+        # an internal cache that conflicts with the 2nd compilation attempt.
+        if cupy.cuda.runtime.is_hip and hasattr(self, 'clean_up'):
+            self.skipTest('skip a potential hiprtc bug')
+
+        # compile code
+        if cupy.cuda.runtime.is_hip:
+            # ROCm 5.0 returns HIP_HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID for
+            # my_sqrt<complex<double>>, so we use thrust::complex<double>
+            # instead.
+            name_expressions = ['my_sqrt<int>', 'my_sqrt<float>',
+                                'my_sqrt<thrust::complex<double>>', 'my_func']
+        else:
+            name_expressions = ['my_sqrt<int>', 'my_sqrt<float>',
+                                'my_sqrt<complex<double>>', 'my_func']
+        mod = cupy.RawModule(code=test_cxx_template,
+                             options=('--std=c++11',),
+                             name_expressions=name_expressions,
+                             jitify=self.jitify)
+
+        dtypes = (cupy.int32, cupy.float32, cupy.complex128, cupy.float64)
+        for ker_T, dtype in zip(name_expressions, dtypes):
+            # get specialized kernels
+            if cupy.cuda.runtime.is_hip:
+                # TODO(leofang): investigate why getLoweredName has no error
+                # but returns an empty string for my_sqrt<complex<double>>
+                mangled_name = mod.module.mapping.get(ker_T)
+                if mangled_name == '':
+                    continue
+            ker = mod.get_function(ker_T)
+
+            # prepare inputs & expected outputs
+            in_arr = cupy.testing.shaped_random((10,), dtype=dtype)
+            out_arr = in_arr**2
+
+            # run
+            ker((1,), (10,), (in_arr, 10))
+
+            # check results
+            assert cupy.allclose(in_arr, out_arr)
+
+    def test_template_failure(self):
+        name_expressions = ['my_sqrt<int>']
+
+        # 1. nvcc is disabled for this feature
+        if self.backend == 'nvcc':
+            with pytest.raises(ValueError) as e:
+                cupy.RawModule(code=test_cxx_template, backend=self.backend,
+                               options=('--std=c++11',),
+                               name_expressions=name_expressions)
+            assert 'nvrtc' in str(e.value)
+            return  # the rest of tests do not apply to nvcc
+
+        # 2. compile code without specializations
+        mod = cupy.RawModule(code=test_cxx_template,
+                             options=('--std=c++11',),
+                             jitify=self.jitify)
+        # ...try to get a specialized kernel
+        match = ('named symbol not found' if not cupy.cuda.runtime.is_hip else
+                 'hipErrorNotFound')
+        with pytest.raises(cupy.cuda.driver.CUDADriverError, match=match):
+            mod.get_function('my_sqrt<int>')
+
+        # 3. compile code without specifying C++ standard
+        with pytest.raises(ValueError):
+            cupy.RawModule(code=test_cxx_template,
+                           name_expressions=name_expressions,
+                           jitify=self.jitify)
+
+        # 4. try to fetch something we didn't specialize for
+        mod = cupy.RawModule(code=test_cxx_template,
+                             options=('--std=c++11',),
+                             name_expressions=name_expressions,
+                             jitify=self.jitify)
+        if cupy.cuda.runtime.is_hip:
+            msg = 'hipErrorNotFound'
+        else:
+            msg = 'named symbol not found'
+        with pytest.raises(cupy.cuda.driver.CUDADriverError, match=msg):
+            mod.get_function('my_sqrt<double>')
+
+    def test_raw_pointer(self):
+        mod = cupy.RawModule(code=test_cast,
+                             backend=self.backend,
+                             jitify=self.jitify)
+        ker = mod.get_function('my_func')
+
+        a = cupy.ones((100,), dtype=cupy.float64)
+        memptr = memory.alloc(100 * a.dtype.itemsize)
+        memptr.copy_from(a.data, 100 * a.dtype.itemsize)  # one-initialize
+        b = cupy.ndarray((100,), cupy.float64, memptr=memptr)
+
+        ker((1,), (100,), (memptr, 100))
+        a = 3. * a - 8.
+        assert (a == b).all()
+
+    @testing.multi_gpu(2)
+    def test_context_switch_RawKernel(self):
+        # run test_basic() on another device
+
+        # we need to launch it once to force compiling
+        x1, x2, y = self._helper(self.kern, cupy.float32)
+
+        with cupy.cuda.Device(1):
+            x1, x2, y = self._helper(self.kern, cupy.float32)
+            assert cupy.allclose(y, x1 + x2)
+
+    @testing.multi_gpu(2)
+    def test_context_switch_RawModule1(self):
+        # run test_module() on another device
+        # in this test, re-compiling happens at 2nd get_function()
+        module = self.mod2
+        with cupy.cuda.Device(0):
+            module.get_function('test_sum')
+
+        with cupy.cuda.Device(1):
+            ker_sum = module.get_function('test_sum')
+            x1, x2, y = self._helper(ker_sum, cupy.float32)
+            assert cupy.allclose(y, x1 + x2)
+
+    @testing.multi_gpu(2)
+    def test_context_switch_RawModule2(self):
+        # run test_module() on another device
+        # in this test, re-compiling happens at kernel launch
+        module = self.mod2
+        with cupy.cuda.Device(0):
+            ker_sum = module.get_function('test_sum')
+
+        with cupy.cuda.Device(1):
+            x1, x2, y = self._helper(ker_sum, cupy.float32)
+            assert cupy.allclose(y, x1 + x2)
+
+    @testing.multi_gpu(2)
+    def test_context_switch_RawModule3(self):
+        # run test_load_cubin() on another device
+        # generate cubin in the temp dir and load it on device 0
+
+        device0 = cupy.cuda.Device(0)
+        device1 = cupy.cuda.Device(1)
+        if device0.compute_capability != device1.compute_capability:
+            raise pytest.skip()
+
+        with device0:
+            file_path = self._generate_file('cubin')
+            mod = cupy.RawModule(path=file_path, backend=self.backend)
+            mod.get_function('test_div')
+
+        # in this test, reloading happens at 2nd get_function()
+        with device1:
+            ker = mod.get_function('test_div')
+            x1, x2, y = self._helper(ker, cupy.float32)
+            assert cupy.allclose(y, x1 / (x2 + 1.0))
+
+    @testing.multi_gpu(2)
+    def test_context_switch_RawModule4(self):
+        # run test_load_cubin() on another device
+        # generate cubin in the temp dir and load it on device 0
+
+        device0 = cupy.cuda.Device(0)
+        device1 = cupy.cuda.Device(1)
+        if device0.compute_capability != device1.compute_capability:
+            raise pytest.skip()
+
+        with device0:
+            file_path = self._generate_file('cubin')
+            mod = cupy.RawModule(path=file_path, backend=self.backend)
+            ker = mod.get_function('test_div')
+
+        # in this test, reloading happens at kernel launch
+        with device1:
+            x1, x2, y = self._helper(ker, cupy.float32)
+            assert cupy.allclose(y, x1 / (x2 + 1.0))
+
+    @testing.multi_gpu(2)
+    def test_context_switch_RawModule5(self):
+        # run test_template_specialization() on another device
+        # in this test, re-compiling happens at get_function()
+        if self.backend == 'nvcc':
+            self.skipTest('nvcc does not support template specialization')
+
+        # compile code
+        name_expressions = ['my_sqrt<unsigned int>']
+        name = name_expressions[0]
+        with cupy.cuda.Device(0):
+            mod = cupy.RawModule(code=test_cxx_template,
+                                 options=('--std=c++11',),
+                                 name_expressions=name_expressions,
+                                 jitify=self.jitify)
+
+            # get specialized kernels
+            mod.get_function(name)
+
+        # switch device
+        with cupy.cuda.Device(1):
+            # get specialized kernels
+            ker = mod.get_function(name)
+
+            # prepare inputs & expected outputs
+            in_arr = cupy.testing.shaped_random((10,), dtype=cupy.uint32)
+            out_arr = in_arr**2
+
+            # run
+            ker((1,), (10,), (in_arr, 10))
+
+            # check results
+            assert cupy.allclose(in_arr, out_arr)
+
+    @testing.multi_gpu(2)
+    def test_context_switch_RawModule6(self):
+        # run test_template_specialization() on another device
+        # in this test, re-compiling happens at kernel launch
+        if self.backend == 'nvcc':
+            self.skipTest('nvcc does not support template specialization')
+
+        # compile code
+        name_expressions = ['my_sqrt<unsigned int>']
+        name = name_expressions[0]
+        with cupy.cuda.Device(0):
+            mod = cupy.RawModule(code=test_cxx_template,
+                                 options=('--std=c++11',),
+                                 name_expressions=name_expressions,
+                                 jitify=self.jitify)
+
+            # get specialized kernels
+            ker = mod.get_function(name)
+
+        # switch device
+        with cupy.cuda.Device(1):
+            # prepare inputs & expected outputs
+            in_arr = cupy.testing.shaped_random((10,), dtype=cupy.uint32)
+            out_arr = in_arr**2
+
+            # run
+            ker((1,), (10,), (in_arr, 10))
+
+            # check results
+            assert cupy.allclose(in_arr, out_arr)
+
+    @unittest.skipUnless(not cupy.cuda.runtime.is_hip,
+                         'only CUDA raises warning')
+    def test_compile_kernel(self):
+        kern = cupy.RawKernel(
+            _test_compile_src, 'test_op',
+            options=('-DOP=+',),
+            backend=self.backend,
+            jitify=self.jitify)
+        log = io.StringIO()
+        with use_temporary_cache_dir():
+            kern.compile(log_stream=log)
+        assert 'warning' in log.getvalue()
+        x1, x2, y = self._helper(kern, cupy.float32)
+        assert cupy.allclose(y, x1 + x2)
+
+    @unittest.skipUnless(not cupy.cuda.runtime.is_hip,
+                         'only CUDA raises warning')
+    def test_compile_module(self):
+        module = cupy.RawModule(
+            code=_test_compile_src,
+            backend=self.backend,
+            options=('-DOP=+',),
+            jitify=self.jitify)
+        log = io.StringIO()
+        with use_temporary_cache_dir():
+            module.compile(log_stream=log)
+        assert 'warning' in log.getvalue()
+        kern = module.get_function('test_op')
+        x1, x2, y = self._helper(kern, cupy.float32)
+        assert cupy.allclose(y, x1 + x2)
+
+
+_test_grid_sync = r'''
+#include <cooperative_groups.h>
+
+extern "C" __global__
+void test_grid_sync(const float* x1, const float* x2, float* y, int n) {
+    namespace cg = cooperative_groups;
+    cg::grid_group grid = cg::this_grid();
+    int size = gridDim.x * blockDim.x;
+    int tid = blockDim.x * blockIdx.x + threadIdx.x;
+    for (int i = tid; i < n; i += size) {
+        y[i] = x1[i];
+    }
+    cg::sync(grid);
+    for (int i = n - 1 - tid; i >= 0; i -= size) {
+        y[i] += x2[i];
+    }
+}
+'''
+
+
+@testing.parameterize(*testing.product({
+    'n': [10, 100, 1000],
+    'block': [64, 256],
+}))
+@unittest.skipUnless(
+    9000 <= cupy.cuda.runtime.runtimeGetVersion(),
+    'Requires CUDA 9.x or later')
+@unittest.skipUnless(
+    60 <= int(cupy.cuda.device.get_compute_capability()),
+    'Requires compute capability 6.0 or later')
+@unittest.skipIf(cupy.cuda.runtime.is_hip, 'Skip on HIP')
+class TestRawGridSync(unittest.TestCase):
+
+    def test_grid_sync_rawkernel(self):
+        n = self.n
+        with use_temporary_cache_dir():
+            kern_grid_sync = cupy.RawKernel(
+                _test_grid_sync, 'test_grid_sync', backend='nvcc',
+                enable_cooperative_groups=True)
+            x1 = cupy.arange(n ** 2, dtype='float32').reshape(n, n)
+            x2 = cupy.ones((n, n), dtype='float32')
+            y = cupy.zeros((n, n), dtype='float32')
+            block = self.block
+            grid = (n * n + block - 1) // block
+            kern_grid_sync((grid,), (block,), (x1, x2, y, n ** 2))
+            assert cupy.allclose(y, x1 + x2)
+
+    def test_grid_sync_rawmodule(self):
+        n = self.n
+        with use_temporary_cache_dir():
+            mod_grid_sync = cupy.RawModule(
+                code=_test_grid_sync, backend='nvcc',
+                enable_cooperative_groups=True)
+            x1 = cupy.arange(n ** 2, dtype='float32').reshape(n, n)
+            x2 = cupy.ones((n, n), dtype='float32')
+            y = cupy.zeros((n, n), dtype='float32')
+            kern = mod_grid_sync.get_function('test_grid_sync')
+            block = self.block
+            grid = (n * n + block - 1) // block
+            kern((grid,), (block,), (x1, x2, y, n ** 2))
+            assert cupy.allclose(y, x1 + x2)
+
+
+_test_script = r'''
+import pickle
+import sys
+
+import cupy as cp
+
+
+N = 100
+a = cp.random.random(N, dtype=cp.float32)
+b = cp.random.random(N, dtype=cp.float32)
+c = cp.empty_like(a)
+with open('raw.pkl', 'rb') as f:
+    ker = pickle.load(f)
+
+if len(sys.argv) == 2:
+    ker = ker.get_function(sys.argv[1])
+
+ker((1,), (100,), (a, b, c, N))
+assert cp.allclose(a + b, c)
+assert ker.enable_cooperative_groups
+'''
+
+
+# Pickling/unpickling a RawModule should always success, whereas
+# pickling/unpickling a RawKernel would fail if we don't enforce
+# recompiling after unpickling it.
+@testing.parameterize(*testing.product({
+    'compile': (False, True),
+    'raw': ('ker', 'mod', 'mod_ker'),
+}))
+@unittest.skipUnless(
+    60 <= int(cupy.cuda.device.get_compute_capability()),
+    'Requires compute capability 6.0 or later')
+@unittest.skipIf(cupy.cuda.runtime.is_hip,
+                 'HIP does not support enable_cooperative_groups')
+class TestRawPicklable(unittest.TestCase):
+
+    def setUp(self):
+        self.temporary_dir_context = use_temporary_cache_dir()
+        self.temp_dir = self.temporary_dir_context.__enter__()
+
+        # test if kw-only arguments are properly handled or not
+        if self.raw == 'ker':
+            self.ker = cupy.RawKernel(_test_source1, 'test_sum',
+                                      backend='nvcc',
+                                      enable_cooperative_groups=True)
+        else:
+            self.mod = cupy.RawModule(code=_test_source1,
+                                      backend='nvcc',
+                                      enable_cooperative_groups=True)
+
+    def tearDown(self):
+        self.temporary_dir_context.__exit__(*sys.exc_info())
+
+    def _helper(self):
+        N = 10
+        x1 = cupy.arange(N**2, dtype=cupy.float32).reshape(N, N)
+        x2 = cupy.ones((N, N), dtype=cupy.float32)
+        y = cupy.zeros((N, N), dtype=cupy.float32)
+        if self.raw == 'ker':
+            ker = self.ker
+        else:
+            ker = self.mod.get_function('test_sum')
+        ker((N,), (N,), (x1, x2, y, N**2))
+        assert cupy.allclose(x1 + x2, y)
+
+    def test_raw_picklable(self):
+        # force compiling before pickling
+        if self.compile:
+            self._helper()
+
+        if self.raw == 'ker':
+            # pickle the RawKernel
+            obj = self.ker
+        elif self.raw == 'mod':
+            # pickle the RawModule
+            obj = self.mod
+        elif self.raw == 'mod_ker':
+            # pickle the RawKernel fetched from the RawModule
+            obj = self.mod.get_function('test_sum')
+        with open(self.temp_dir + '/raw.pkl', 'wb') as f:
+            pickle.dump(obj, f)
+
+        # dump test script to temp dir
+        with open(self.temp_dir + '/TestRawPicklable.py', 'w') as f:
+            f.write(_test_script)
+        test_args = ['test_sum'] if self.raw == 'mod' else []
+
+        # run another process to check the pickle
+        s = subprocess.run([sys.executable, 'TestRawPicklable.py'] + test_args,
+                           cwd=self.temp_dir)
+        s.check_returncode()  # raise if unsuccess
+
+
+# a slightly more realistic kernel involving std utilities
+std_code = r'''
+#include <type_traits>
+
+template<typename T,
+         typename = typename std::enable_if<std::is_integral<T>::value>::type>
+__global__ void shift (T* a, int N) {
+    unsigned int tid = blockIdx.x * blockDim.x + threadIdx.x;
+    if (tid < N) {
+        a[tid] += 100;
+    }
+}
+'''
+
+
+@testing.parameterize(*testing.product({
+    'jitify': (False, True),
+}))
+@unittest.skipIf(cupy.cuda.runtime.is_hip,
+                 'Jitify does not support ROCm/HIP')
+class TestRawJitify(unittest.TestCase):
+
+    def setUp(self):
+        self.temporary_dir_context = use_temporary_cache_dir()
+        self.temp_dir = self.temporary_dir_context.__enter__()
+
+    def tearDown(self):
+        self.temporary_dir_context.__exit__(*sys.exc_info())
+
+    def _helper(self, header, options=()):
+        code = header
+        code += _test_source1
+        mod1 = cupy.RawModule(code=code,
+                              backend='nvrtc',
+                              options=options,
+                              jitify=self.jitify)
+
+        N = 10
+        x1 = cupy.arange(N**2, dtype=cupy.float32).reshape(N, N)
+        x2 = cupy.ones((N, N), dtype=cupy.float32)
+        y = cupy.zeros((N, N), dtype=cupy.float32)
+        ker = mod1.get_function('test_sum')
+        ker((N,), (N,), (x1, x2, y, N**2))
+        assert cupy.allclose(x1 + x2, y)
+
+    def _helper2(self, type_str):
+        mod2 = cupy.RawModule(code=std_code,
+                              jitify=self.jitify,
+                              name_expressions=['shift<%s>' % type_str, ],
+                              options=('--std=c++11',))
+        ker = mod2.get_function('shift<%s>' % type_str)
+        N = 256
+        a = cupy.random.random_integers(0, 7, N).astype(cupy.int32)
+        b = a.copy()
+        ker((1,), (N,), (a, N))
+        assert cupy.allclose(a, b+100)
+
+    def test_jitify1(self):
+        # simply prepend an unused header
+        hdr = '#include <cupy/cub/cub/block/block_reduce.cuh>\n'
+
+        if self.jitify:
+            if sys.platform.startswith('win32'):
+                pytest.xfail('macro preprocessing in NVRTC is likely buggy')
+            # Jitify will make it work
+            self._helper(hdr)
+        else:
+            # NVRTC cannot find C++ std headers without Jitify
+            with pytest.raises(cupy.cuda.compiler.CompileException) as ex:
+                self._helper(hdr)
+            assert 'cannot open source file' in str(ex.value)
+
+    def test_jitify2(self):
+        # NVRTC cannot compile any code involving std
+        if self.jitify:
+            # Jitify will make it work
+            self._helper2('int')
+        else:
+            with pytest.raises(cupy.cuda.compiler.CompileException) as ex:
+                self._helper2('int')
+            assert 'cannot open source file' in str(ex.value)
+
+    def test_jitify3(self):
+        # We supply a type impossible to specialize. Jitify is still able to
+        # locate the headers, but when it comes to the actual compilation,
+        # NVRTC fails (raising the same exception) with different error
+        # messages.
+        ex_type = cupy.cuda.compiler.CompileException
+        with pytest.raises(ex_type) as ex:
+            self._helper2('float')
+        if self.jitify:
+            assert 'Error in parsing name expression' in str(ex.value)
+        else:
+            assert 'cannot open source file' in str(ex.value)
+
+    def test_jitify4(self):
+        # ensure JitifyException is raised with a broken code
+        code = r'''
+        __global__ void i_am_broken() {
+        '''
+
+        if self.jitify:
+            ex_type = cupy.cuda.compiler.JitifyException
+        else:
+            ex_type = cupy.cuda.compiler.CompileException
+
+        with pytest.raises(ex_type):
+            mod = cupy.RawModule(code=code, jitify=self.jitify)
+            ker = mod.get_function('i_am_broken')  # noqa
+        # if Jitify could redirect its output, we would be able to check
+        # the error log here as well (NVIDIA/jitify#79)
+
+    def test_jitify5(self):
+        # If including a header that does not exist, Jitify would attempt to
+        # comment it out and proceed. If this header is actually unused, then
+        # everything would run just fine.
+
+        hdr = 'I_INCLUDE_SOMETHING.h'
+        with open(self.temp_dir + '/' + hdr, 'w') as f:
+            dummy = '#include <cupy/I_DO_NOT_EXIST_WAH_HA_HA.h>\n'
+            f.write(dummy)
+        hdr = '#include "' + hdr + '"\n'
+
+        if self.jitify:
+            # Jitify would print a warning "[jitify] File not found" to stdout,
+            # but as mentioned above and elsewhere, we can't capture it.
+            self._helper(hdr, options=('-I'+self.temp_dir,))
+        else:
+            with pytest.raises(cupy.cuda.compiler.CompileException) as ex:
+                self._helper(hdr, options=('-I'+self.temp_dir,))
+            assert 'cannot open source file' in str(ex.value)
diff --git a/tests/cupy_tests/core_tests/test_reduction.py b/tests/cupy_tests/core_tests/test_reduction.py
new file mode 100644
index 0000000..aceecce
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_reduction.py
@@ -0,0 +1,233 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+import cupy._core._accelerator as _acc
+from cupy import _core
+from cupy import testing
+
+
+_noncontiguous_params = [
+    # reduce at head axes
+    {'shape': (2, 4, 3), 'trans': (2, 1, 0), 'axis': (0, 1)},
+    # reduce at middle axes
+    {'shape': (2, 4, 5, 3), 'trans': (3, 2, 1, 0), 'axis': (1, 2)},
+    # reduce at tail axes
+    {'shape': (2, 4, 3), 'trans': (2, 1, 0), 'axis': (1, 2)},
+    # out_axis = (0,)
+    {'shape': (0, 4, 3), 'trans': (2, 1, 0), 'axis': (0, 1)},
+    # out_axis = ()
+    {'shape': (2, 4, 3), 'trans': (2, 1, 0), 'axis': (0, 1, 2)},
+]
+
+
+class AbstractReductionTestBase:
+
+    def get_sum_func(self):
+        raise NotImplementedError()
+
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def check_int8_sum(self, shape, xp, axis=None, keepdims=False, trans=None):
+        a = testing.shaped_random(shape, xp, 'b')
+        if trans:
+            a = a.transpose(*trans)
+        sum_func = self.get_sum_func()
+        if xp == cupy:
+            return sum_func(
+                a, axis=axis, keepdims=keepdims)
+        else:
+            return a.sum(axis=axis, keepdims=keepdims, dtype='b')
+
+
+class SimpleReductionFunctionTestBase(AbstractReductionTestBase):
+
+    def get_sum_func(self):
+        return _core.create_reduction_func(
+            'my_sum', ('b->b',), ('in0', 'a + b', 'out0 = a', None), 0)
+
+
+class TestSimpleReductionFunction(
+        unittest.TestCase, SimpleReductionFunctionTestBase):
+    def test_shape1(self):
+        for i in range(1, 10):
+            self.check_int8_sum((2 ** i,))
+            self.check_int8_sum((2 ** i - 1,))
+            self.check_int8_sum((2 ** i + 1,))
+
+    def test_shape2(self):
+        for i in range(1, 10):
+            self.check_int8_sum((2 ** i, 1000), axis=0)
+            self.check_int8_sum((2 ** i - 1, 1000), axis=0)
+            self.check_int8_sum((2 ** i + 1, 1000), axis=0)
+
+    def test_shape3(self):
+        for i in range(1, 10):
+            self.check_int8_sum((2 ** i, 1000), axis=1)
+            self.check_int8_sum((2 ** i - 1, 1000), axis=1)
+            self.check_int8_sum((2 ** i + 1, 1000), axis=1)
+
+    def test_shape4(self):
+        self.check_int8_sum((512, 256 * 256), axis=0)
+        self.check_int8_sum((512, 256 * 256), axis=1)
+
+        self.check_int8_sum((512 + 1, 256 * 256 + 1), axis=0)
+        self.check_int8_sum((512 + 1, 256 * 256 + 1), axis=1)
+
+    def test_shape5(self):
+        block_size = 512
+        size = ((2 << 32) // block_size)
+        self.check_int8_sum((size, 1), axis=1)
+        self.check_int8_sum((size, 1), axis=0)
+
+
+@testing.parameterize(*_noncontiguous_params)
+class TestSimpleReductionFunctionNonContiguous(
+        SimpleReductionFunctionTestBase, unittest.TestCase):
+
+    def test_noncontiguous(self):
+        self.check_int8_sum(self.shape, trans=self.trans, axis=self.axis)
+
+
+@testing.parameterize(*testing.product({
+    'backend': ([], ['cub']),
+}))
+class TestSimpleReductionFunctionComplexWarning(unittest.TestCase):
+
+    def setUp(self):
+        self.accelerators = _core.get_reduction_accelerators()
+        _core.set_reduction_accelerators(self.backend)
+
+    def tearDown(self):
+        _core.set_reduction_accelerators(self.accelerators)
+
+    @testing.for_complex_dtypes(name='c_dtype')
+    @testing.for_float_dtypes(name='f_dtype')
+    @testing.numpy_cupy_allclose()
+    def test_warns(self, xp, c_dtype, f_dtype):
+        with pytest.warns(numpy.ComplexWarning):
+            out = xp.ones((8,), dtype=c_dtype).sum(dtype=f_dtype)
+        return out
+
+
+class TestSimpleReductionFunctionInvalidAxis:
+    @pytest.mark.parametrize('axis', [
+        2,
+        (-3,),
+        (0, 7),
+    ])
+    def test_axis_overrun(self, axis):
+        for xp in (numpy, cupy):
+            a = xp.ones((2, 2))
+            with pytest.raises(numpy.AxisError):
+                a.sum(axis=axis)
+
+    @pytest.mark.parametrize('axis', [
+        (1, 1),
+        (0, -2),
+    ])
+    def test_axis_repeated(self, axis):
+        for xp in (numpy, cupy):
+            a = xp.ones((2, 2))
+            with pytest.raises(ValueError):
+                a.sum(axis=axis)
+
+
+class ReductionKernelTestBase(AbstractReductionTestBase):
+
+    def get_sum_func(self):
+        return cupy.ReductionKernel(
+            'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum')
+
+
+class TestReductionKernel(ReductionKernelTestBase, unittest.TestCase):
+
+    def test_shape1(self):
+        for i in range(1, 10):
+            self.check_int8_sum((2 ** i,))
+            self.check_int8_sum((2 ** i - 1,))
+            self.check_int8_sum((2 ** i + 1,))
+
+    def test_shape2(self):
+        for i in range(1, 10):
+            self.check_int8_sum((2 ** i, 1000), axis=0)
+            self.check_int8_sum((2 ** i - 1, 1000), axis=0)
+            self.check_int8_sum((2 ** i + 1, 1000), axis=0)
+
+    def test_shape3(self):
+        for i in range(1, 10):
+            self.check_int8_sum((2 ** i, 1000), axis=1)
+            self.check_int8_sum((2 ** i - 1, 1000), axis=1)
+            self.check_int8_sum((2 ** i + 1, 1000), axis=1)
+
+    def test_shape4(self):
+        self.check_int8_sum((512, 256 * 256), axis=0)
+        self.check_int8_sum((512, 256 * 256), axis=1)
+        self.check_int8_sum((512 + 1, 256 * 256 + 1), axis=0)
+        self.check_int8_sum((512 + 1, 256 * 256 + 1), axis=1)
+
+
+@testing.parameterize(*_noncontiguous_params)
+class TestReductionKernelNonContiguous(
+        ReductionKernelTestBase, unittest.TestCase):
+
+    def test_noncontiguous(self):
+        self.check_int8_sum(self.shape, trans=self.trans, axis=self.axis)
+
+
+class TestReductionKernelInvalidArgument(unittest.TestCase):
+
+    def test_invalid_kernel_name(self):
+        with self.assertRaisesRegex(ValueError, 'Invalid kernel name'):
+            cupy.ReductionKernel(
+                'T x', 'T y', 'x', 'a + b', 'y = a', '0', name='1')
+
+
+class TestReductionKernelCachedCode:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.old_routine_accelerators = _acc.get_routine_accelerators()
+        self.old_reduction_accelerators = _acc.get_reduction_accelerators()
+        # Disable CUB
+        _acc.set_reduction_accelerators([])
+        _acc.set_routine_accelerators([])
+        yield
+        _acc.set_routine_accelerators(self.old_routine_accelerators)
+        _acc.set_reduction_accelerators(self.old_reduction_accelerators)
+
+    def test_cached_code(self):
+        kernel = cupy.ReductionKernel(
+            'T x', 'T y', 'x', 'a + b', 'y = a', '0', name='cached_code')
+        assert len(kernel._cached_codes) == 0
+        x = cupy.arange(10)
+        kernel(x)
+        assert len(kernel._cached_codes) == 1
+        kernel(x)
+        assert len(kernel._cached_codes) == 1
+        kernel(x.astype(cupy.float32))
+        assert len(kernel._cached_codes) == 2
+
+    def test_simple_cached_code(self):
+        kernel = _core.create_reduction_func(
+            'my_sum', ('q->q', 'f->f'), ('in0', 'a + b', 'out0 = a', None), 0)
+        assert len(kernel._cached_codes) == 0
+        x = cupy.arange(10)
+        kernel(x)
+        assert len(kernel._cached_codes) == 1
+        kernel(x)
+        assert len(kernel._cached_codes) == 1
+        kernel(x.astype(cupy.float32))
+        assert len(kernel._cached_codes) == 2
+
+
+class TestLargeMultiDimReduction(
+        ReductionKernelTestBase, unittest.TestCase):
+
+    def test_large_dims_keep_kernels(self):
+        # This test creates a CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES
+        # if the output array dims are not reduced
+        shape = (4, 3, 2, 4, 3, 2, 2)
+        axis = (1, 4, 3, 6)
+        self.check_int8_sum(shape, axis=axis, keepdims=True)
diff --git a/tests/cupy_tests/core_tests/test_scan.py b/tests/cupy_tests/core_tests/test_scan.py
new file mode 100644
index 0000000..b2732ee
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_scan.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+
+import unittest
+
+import cupy
+from cupy import cuda
+from cupy._core._routines_math import _scan_for_test as scan
+from cupy import testing
+
+
+class TestScan(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_scan(self, dtype):
+        element_num = 10000
+
+        if dtype in {cupy.int8, cupy.uint8, cupy.float16}:
+            element_num = 100
+
+        a = cupy.ones((element_num,), dtype=dtype)
+        prefix_sum = scan(a)
+        expect = cupy.arange(start=1, stop=element_num + 1).astype(dtype)
+
+        testing.assert_array_equal(prefix_sum, expect)
+
+    def test_check_1d_array(self):
+        with self.assertRaises(TypeError):
+            a = cupy.zeros((2, 2))
+            scan(a)
+
+    @testing.multi_gpu(2)
+    def test_multi_gpu(self):
+        with cuda.Device(0):
+            a = cupy.zeros((10,))
+            scan(a)
+        with cuda.Device(1):
+            a = cupy.zeros((10,))
+            scan(a)
+
+    @testing.for_all_dtypes()
+    def test_scan_out(self, dtype):
+        element_num = 10000
+
+        if dtype in {cupy.int8, cupy.uint8, cupy.float16}:
+            element_num = 100
+
+        a = cupy.ones((element_num,), dtype=dtype)
+        b = cupy.zeros_like(a)
+        scan(a, b)
+        expect = cupy.arange(start=1, stop=element_num + 1).astype(dtype)
+
+        testing.assert_array_equal(b, expect)
+
+        scan(a, a)
+        testing.assert_array_equal(a, expect)
diff --git a/tests/cupy_tests/core_tests/test_syncdetect.py b/tests/cupy_tests/core_tests/test_syncdetect.py
new file mode 100644
index 0000000..f726506
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_syncdetect.py
@@ -0,0 +1,33 @@
+import unittest
+
+import pytest
+
+import cupy
+import cupyx
+
+
+class TestSyncDetect(unittest.TestCase):
+
+    def test_disallowed(self):
+        a = cupy.array([2, 3])
+        with cupyx.allow_synchronize(False):
+            with pytest.raises(cupyx.DeviceSynchronized):
+                a.get()
+
+    def test_allowed(self):
+        a = cupy.array([2, 3])
+        with cupyx.allow_synchronize(True):
+            a.get()
+
+    def test_nested_disallowed(self):
+        a = cupy.array([2, 3])
+        with cupyx.allow_synchronize(True):
+            with cupyx.allow_synchronize(False):
+                with pytest.raises(cupyx.DeviceSynchronized):
+                    a.get()
+
+    def test_nested_allowed(self):
+        a = cupy.array([2, 3])
+        with cupyx.allow_synchronize(False):
+            with cupyx.allow_synchronize(True):
+                a.get()
diff --git a/tests/cupy_tests/core_tests/test_ufunc_methods.py b/tests/cupy_tests/core_tests/test_ufunc_methods.py
new file mode 100644
index 0000000..5f65de1
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_ufunc_methods.py
@@ -0,0 +1,227 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestUfuncOuter:
+
+    @testing.numpy_cupy_array_equal()
+    def test_add_outer(self, xp):
+        x = testing.shaped_random((2, 3), xp=xp, dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((4, 1, 5), xp=xp, dtype=numpy.int32, seed=1)
+        return xp.add.outer(x, y)
+
+    @testing.numpy_cupy_array_equal()
+    def test_add_outer_scalar(self, xp):
+        return xp.add.outer(2, 3)
+
+
+class TestUfuncAtAtomicOps:
+
+    @testing.for_dtypes('iIQefd')
+    @testing.numpy_cupy_array_equal()
+    def test_at_add(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip and dtype == numpy.float16:
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        mask = testing.shaped_random(shape, xp=xp, dtype=bool, seed=1)
+        indices = xp.nonzero(mask)[0]
+        xp.add.at(x, indices, 3)
+        return x
+
+    @testing.for_dtypes('iIQefd')
+    @testing.numpy_cupy_array_equal()
+    def test_at_add_duplicate_indices(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip and dtype == numpy.float16:
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        indices = testing.shaped_random(
+            shape, xp=xp, dtype=numpy.int32, scale=shape[0], seed=1)
+        xp.add.at(x, indices, 3)
+        return x
+
+    @testing.for_dtypes('iI')
+    @testing.numpy_cupy_array_equal()
+    def test_at_subtract(self, xp, dtype):
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        mask = testing.shaped_random(shape, xp=xp, dtype=bool, seed=1)
+        indices = xp.nonzero(mask)[0]
+        xp.subtract.at(x, indices, 3)
+        return x
+
+    @testing.for_dtypes('iI')
+    @testing.numpy_cupy_array_equal()
+    def test_at_subtract_duplicate_indices(self, xp, dtype):
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        indices = testing.shaped_random(
+            shape, xp=xp, dtype=numpy.int32, scale=shape[0], seed=1)
+        xp.subtract.at(x, indices, 3)
+        return x
+
+    @testing.for_dtypes('iIQfd')
+    @testing.numpy_cupy_allclose()
+    def test_at_min(self, xp, dtype):
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        mask = testing.shaped_random(shape, xp=xp, dtype=bool, seed=1)
+        indices = xp.nonzero(mask)[0]
+        xp.minimum.at(x, indices, 3)
+        return x
+
+    @testing.for_dtypes('iIQfd')
+    @testing.numpy_cupy_allclose()
+    def test_at_min_duplicate_indices(self, xp, dtype):
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        indices = testing.shaped_random(
+            shape, xp=xp, dtype=numpy.int32, scale=shape[0], seed=1)
+        values = testing.shaped_random(
+            indices.shape, xp=xp, dtype=dtype, seed=2)
+        xp.minimum.at(x, indices, values)
+        return x
+
+    @testing.for_dtypes('iIQfd')
+    @testing.numpy_cupy_allclose()
+    def test_at_max(self, xp, dtype):
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        mask = testing.shaped_random(shape, xp=xp, dtype=bool, seed=1)
+        indices = xp.nonzero(mask)[0]
+        xp.maximum.at(x, indices, 3)
+        return x
+
+    @testing.for_dtypes('iIQfd')
+    @testing.numpy_cupy_allclose()
+    def test_at_max_duplicate_indices(self, xp, dtype):
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        indices = testing.shaped_random(
+            shape, xp=xp, dtype=numpy.int32, scale=shape[0], seed=1)
+        values = testing.shaped_random(
+            indices.shape, xp=xp, dtype=dtype, seed=2)
+        xp.maximum.at(x, indices, values)
+        return x
+
+    @testing.for_dtypes('iIlLqQ')
+    @testing.numpy_cupy_array_equal()
+    def test_at_bitwise_and(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip and numpy.dtype(dtype).char in 'lq':
+            pytest.skip('atomicOr does not support int64 in HIP')
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        indices = testing.shaped_random(
+            shape, xp=xp, dtype=numpy.int32, scale=shape[0], seed=1)
+        values = testing.shaped_random(
+            indices.shape, xp=xp, dtype=dtype, seed=2)
+        xp.bitwise_and.at(x, indices, values)
+        return x
+
+    @testing.for_dtypes('iIlLqQ')
+    @testing.numpy_cupy_array_equal()
+    def test_at_bitwise_or(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip and numpy.dtype(dtype).char in 'lq':
+            pytest.skip('atomicOr does not support int64 in HIP')
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        indices = testing.shaped_random(
+            shape, xp=xp, dtype=numpy.int32, scale=shape[0], seed=1)
+        values = testing.shaped_random(
+            indices.shape, xp=xp, dtype=dtype, seed=2)
+        xp.bitwise_or.at(x, indices, values)
+        return x
+
+    @testing.for_dtypes('iIlLqQ')
+    @testing.numpy_cupy_array_equal()
+    def test_at_bitwise_xor(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip and numpy.dtype(dtype).char in 'lq':
+            pytest.skip('atomicXor does not support int64 in HIP')
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        indices = testing.shaped_random(
+            shape, xp=xp, dtype=numpy.int32, scale=shape[0], seed=1)
+        values = testing.shaped_random(
+            indices.shape, xp=xp, dtype=dtype, seed=2)
+        xp.bitwise_xor.at(x, indices, values)
+        return x
+
+    @testing.for_dtypes('iIQefd')
+    @testing.numpy_cupy_array_equal()
+    def test_at_boolean_mask(self, xp, dtype):
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        mask = testing.shaped_random(shape, xp=xp, dtype=bool, seed=1)
+        xp.add.at(x, mask, 3)
+        return x
+
+    @testing.for_dtypes('iIQefd')
+    @testing.numpy_cupy_array_equal()
+    def test_at_array_values(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip and dtype == numpy.float16:
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        shape = (50,)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        mask = testing.shaped_random(shape, xp=xp, dtype=bool, seed=1)
+        indices = xp.nonzero(mask)[0]
+        values = testing.shaped_random(
+            indices.shape, xp=xp, dtype=numpy.int32, seed=2)
+        xp.add.at(x, indices, values)
+        return x
+
+    @testing.for_dtypes('iIQefd')
+    @testing.numpy_cupy_array_equal()
+    def test_at_multi_dimensional(self, xp, dtype):
+        if cupy.cuda.runtime.is_hip and dtype == numpy.float16:
+            pytest.skip('atomicAdd does not support float16 in HIP')
+        shape = (20, 30)
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, seed=0)
+        mask = testing.shaped_random(shape, xp=xp, dtype=bool, seed=1)
+        indices = xp.nonzero(mask)
+        xp.add.at(x, indices, 3)
+        return x
+
+
+class TestUfuncReduce:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-3, 'default': 1e-6})
+    def test_reduce_add(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp=xp, dtype=dtype, seed=0)
+        return xp.add.reduce(x, axis=-1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-3, 'default': 1e-6})
+    def test_multiply_add(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp=xp, dtype=dtype, seed=0)
+        return xp.multiply.reduce(x, axis=-1)
+
+
+class TestUfuncAccumulate:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-3, 'default': 1e-6})
+    def test_reduce_add(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp=xp, dtype=dtype, seed=0)
+        return xp.add.accumulate(x, axis=-1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-3, 'default': 1e-6})
+    def test_multiply_add(self, xp, dtype):
+        x = testing.shaped_random((3, 4), xp=xp, dtype=dtype, seed=0)
+        return xp.multiply.accumulate(x, axis=-1)
+
+
+class TestUfuncReduceAt:
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_reduce_add(self, xp, dtype):
+        x = testing.shaped_random((3, 4, 5), xp=xp, dtype=dtype, seed=0)
+        indices = testing.shaped_random(
+            (20,), xp=xp, dtype=numpy.int32, scale=4, seed=1)
+        return xp.add.reduceat(x, indices, axis=1)
diff --git a/tests/cupy_tests/core_tests/test_userkernel.py b/tests/cupy_tests/core_tests/test_userkernel.py
new file mode 100644
index 0000000..9ae2434
--- /dev/null
+++ b/tests/cupy_tests/core_tests/test_userkernel.py
@@ -0,0 +1,359 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.cuda import runtime
+from cupy.cuda.texture import (ChannelFormatDescriptor, CUDAarray,
+                               ResourceDescriptor, TextureDescriptor,
+                               TextureObject,)
+
+
+class TestUserkernel(unittest.TestCase):
+
+    def test_manual_indexing(self, n=100):
+        in1 = cupy.random.uniform(-1, 1, n).astype(cupy.float32)
+        in2 = cupy.random.uniform(-1, 1, n).astype(cupy.float32)
+        uesr_kernel_1 = cupy.ElementwiseKernel(
+            'T x, T y',
+            'T z',
+            '''
+                z = x + y;
+            ''',
+            'uesr_kernel_1')
+        out1 = uesr_kernel_1(in1, in2)
+
+        uesr_kernel_2 = cupy.ElementwiseKernel(
+            'raw T x, raw T y',
+            'raw T z',
+            '''
+                z[i] = x[i] + y[i];
+            ''',
+            'uesr_kernel_2')
+        out2 = uesr_kernel_2(in1, in2, size=n)
+
+        testing.assert_array_equal(out1, out2)
+
+    def test_python_scalar(self):
+        for typ in (int, float, bool):
+            dtype = numpy.dtype(typ).type
+            in1_cpu = numpy.random.randint(0, 1, (4, 5)).astype(dtype)
+            in1 = cupy.array(in1_cpu)
+            scalar_value = typ(2)
+            uesr_kernel_1 = cupy.ElementwiseKernel(
+                'T x, T y',
+                'T z',
+                '''
+                    z = x + y;
+                ''',
+                'uesr_kernel_1')
+            out1 = uesr_kernel_1(in1, scalar_value)
+
+            expected = in1_cpu + dtype(2)
+            testing.assert_array_equal(out1, expected)
+
+    @testing.for_all_dtypes()
+    def test_numpy_scalar(self, dtype):
+        in1_cpu = numpy.random.randint(0, 1, (4, 5)).astype(dtype)
+        in1 = cupy.array(in1_cpu)
+        scalar_value = dtype(2)
+        uesr_kernel_1 = cupy.ElementwiseKernel(
+            'T x, T y',
+            'T z',
+            '''
+                z = x + y;
+            ''',
+            'uesr_kernel_1')
+        out1 = uesr_kernel_1(in1, scalar_value)
+
+        expected = in1_cpu + dtype(2)
+        testing.assert_array_equal(out1, expected)
+
+    def test_cached_code(self):
+        in1 = cupy.random.uniform(-1, 1, 100).astype(cupy.float32)
+        in2 = cupy.random.uniform(-1, 1, 100).astype(cupy.float32)
+        user_kernel_1 = cupy.ElementwiseKernel(
+            'T x, T y',
+            'T z',
+            '''
+                z = x + y;
+            ''',
+            'uesr_kernel_1')
+        assert len(user_kernel_1._cached_codes) == 0
+        user_kernel_1(in1, in2)
+        assert len(user_kernel_1._cached_codes) == 1
+        user_kernel_1(in1, in2)
+        assert len(user_kernel_1._cached_codes) == 1
+        user_kernel_1(in1.astype(cupy.float64), in2.astype(cupy.float64))
+        assert len(user_kernel_1._cached_codes) == 2
+
+
+class TestElementwiseKernelSize(unittest.TestCase):
+    # Tests to check whether size argument raises ValueError correctly
+    # depending on the raw specifiers of a user kernel.
+
+    def setUp(self):
+        self.arr1 = cupy.array([1, 2], dtype='float32')
+        self.arr2 = cupy.array([3, 4], dtype='float32')
+
+    def raises_size_not_allowed(self):
+        return pytest.raises(ValueError, match=r'^Specified \'size\' can')
+
+    def raises_size_required(self):
+        return pytest.raises(ValueError, match=r'^Loop size is undecided\.')
+
+    def create_kernel(self, input_raw, output_raw):
+        # Creates a no-op kernel with given parameter specification.
+        # input_raw and output_raw are tuples of True/False whose
+        # corresponding parameter will be designated as 'raw' if True.
+        input_types = (
+            ', '.join([
+                '{}float32 x{}'.format(
+                    ('raw ' if raw else ''), i)
+                for i, raw in enumerate(input_raw)]))
+        output_types = (
+            ', '.join([
+                '{}float32 y{}'.format(
+                    ('raw ' if raw else ''), i)
+                for i, raw in enumerate(output_raw)]))
+        return cupy.ElementwiseKernel(input_types, output_types, '', 'kernel')
+
+    def test_all_raws(self):
+        # Input arrays are all raw -> size required
+        kernel1 = self.create_kernel((True, True), (False,))
+        kernel1(self.arr1, self.arr2, size=2)
+        with self.raises_size_required():
+            kernel1(self.arr1, self.arr2)
+        kernel2 = self.create_kernel((True, True), (True,))
+        kernel2(self.arr1, self.arr2, size=2)
+        with self.raises_size_required():
+            kernel2(self.arr1, self.arr2)
+
+    def test_all_nonraws(self):
+        # All arrays are not raw -> size not allowed
+        kernel1 = self.create_kernel((False, False), (False,))
+        with self.raises_size_not_allowed():
+            kernel1(self.arr1, self.arr2, size=2)
+        kernel2 = self.create_kernel((False, False), (True,))
+        with self.raises_size_not_allowed():
+            kernel2(self.arr1, self.arr2, size=2)
+
+    def test_some_nonraws(self):
+        # Some arrays are not raw -> size not allowed
+        kernel1 = self.create_kernel((True, False), (False,))
+        with self.raises_size_not_allowed():
+            kernel1(self.arr1, self.arr2, size=2)
+        kernel2 = self.create_kernel((False, True), (False,))
+        with self.raises_size_not_allowed():
+            kernel2(self.arr1, self.arr2, size=2)
+        kernel3 = self.create_kernel((True, False), (True,))
+        with self.raises_size_not_allowed():
+            kernel3(self.arr1, self.arr2, size=2)
+        kernel4 = self.create_kernel((False, True), (True,))
+        with self.raises_size_not_allowed():
+            kernel4(self.arr1, self.arr2, size=2)
+
+    def test_scalars_and_nonraws(self):
+        # Combination of scalars and non-raw arrays -> size not allowed
+        kernel1 = self.create_kernel((False, False), (False,))
+        with self.raises_size_not_allowed():
+            kernel1(self.arr1, 7, size=2)
+        kernel2 = self.create_kernel((False, False), (False,))
+        with self.raises_size_not_allowed():
+            kernel2(7, self.arr1, size=2)
+        kernel3 = self.create_kernel((False, False), (True,))
+        with self.raises_size_not_allowed():
+            kernel3(self.arr1, 7, size=2)
+        kernel4 = self.create_kernel((False, False), (True,))
+        with self.raises_size_not_allowed():
+            kernel4(7, self.arr1, size=2)
+
+    def test_scalars_and_raws_and_nonraws(self):
+        # Combination of scalars and raw arrays and non-raw arrays
+        #                                                   -> size not allowed
+        kernel1 = self.create_kernel((False, False, True), (False,))
+        with self.raises_size_not_allowed():
+            kernel1(self.arr1, 7, self.arr2, size=2)
+        kernel2 = self.create_kernel((False, False, True), (True,))
+        with self.raises_size_not_allowed():
+            kernel2(self.arr1, 7, self.arr2, size=2)
+
+    def test_scalars_and_raws(self):
+        # Combination of scalars and raw arrays -> size required
+        kernel1 = self.create_kernel((True, False), (False,))
+        kernel1(self.arr1, 7, size=2)
+        with self.raises_size_required():
+            kernel1(self.arr1, 7)
+        kernel2 = self.create_kernel((False, True), (False,))
+        kernel2(7, self.arr1, size=2)
+        with self.raises_size_required():
+            kernel2(7, self.arr1)
+        kernel3 = self.create_kernel((True, False), (True,))
+        kernel3(self.arr1, 7, size=2)
+        with self.raises_size_required():
+            kernel3(self.arr1, 7)
+        kernel4 = self.create_kernel((False, True), (True,))
+        kernel4(7, self.arr1, size=2)
+        with self.raises_size_required():
+            kernel4(7, self.arr1)
+
+    def test_size_determined_by_output(self):
+        # All the input args are unsized, but the size can be determined by the
+        # output arg. size argument is not allowed.
+
+        # Raw input
+        kernel1 = self.create_kernel((True,), (False,))
+        kernel1(self.arr1, self.arr2)
+        with self.raises_size_not_allowed():
+            kernel1(self.arr1, self.arr2, size=2)
+
+        # Scalar input
+        kernel2 = self.create_kernel((False,), (False,))
+        kernel2(self.arr1, self.arr2)
+        with self.raises_size_not_allowed():
+            kernel2(7, self.arr2, size=2)
+
+        # No input
+        kernel3 = self.create_kernel((), (False,))
+        kernel3(self.arr1)
+        with self.raises_size_not_allowed():
+            kernel3(self.arr1, size=2)
+
+    def test_no_input_and_raw_output(self):
+        # No input and the given output is raw -> size required
+        kernel1 = self.create_kernel((), (True,))
+        kernel1(self.arr1, size=2)
+        with self.raises_size_required():
+            kernel1(self.arr1)
+
+
+@testing.parameterize(*testing.product({
+    'value': [-1, 2 ** 32, 2 ** 63 - 1, -(2 ** 63)],
+}))
+class TestUserkernelScalar(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_scalar(self, xp, dtype):
+        x = testing.shaped_arange((2, 3, 4), xp, dtype)
+        if xp is numpy:
+            y = numpy.array(self.value).astype(dtype)
+            return x + y
+        else:
+            kernel = cupy.ElementwiseKernel('T x, T y', 'T z', 'z = x + y')
+            return kernel(x, self.value)
+
+
+class TestUserkernelManualBlockSize(unittest.TestCase):
+
+    def test_invalid_block_size(self):
+        x = testing.shaped_arange((2, 3, 4), cupy, cupy.float32)
+        kernel = cupy.ElementwiseKernel('T x, T y', 'T z', 'z = x + y')
+        with pytest.raises(ValueError):
+            kernel(x, 1, block_size=0)
+
+    def test_block_size(self):
+        x = testing.shaped_arange((2, 3, 4), cupy, cupy.float32)
+        kernel = cupy.ElementwiseKernel('T x, T y', 'T z', 'z = x + y')
+        y = kernel(x, 1, block_size=1)
+        testing.assert_array_equal(y, x + 1)
+
+
+@testing.parameterize(*testing.product({
+    'dimensions': ((64, 0, 0), (64, 32, 0), (64, 32, 19)),
+}))
+@pytest.mark.skipif(runtime.is_hip,
+                    reason='texture support on HIP is not yet implemented')
+class TestElementwiseKernelTexture(unittest.TestCase):
+
+    def _prep_texture(self):
+        width, height, depth = self.dimensions
+        dim = 3 if depth != 0 else 2 if height != 0 else 1
+
+        # generate input data and allocate output buffer
+        shape = (depth, height, width) if dim == 3 else \
+                (height, width) if dim == 2 else \
+                (width,)
+        self.shape = shape
+
+        # prepare input, output, and texture memory
+        # self.data holds the data stored in the texture memory
+        tex_data = cupy.random.random(shape, dtype=cupy.float32)
+        ch = ChannelFormatDescriptor(32, 0, 0, 0,
+                                     runtime.cudaChannelFormatKindFloat)
+        arr = CUDAarray(ch, width, height, depth)
+        arr.copy_from(tex_data)
+        self.data = tex_data
+
+        # create resource and texture descriptors
+        res = ResourceDescriptor(runtime.cudaResourceTypeArray, cuArr=arr)
+        address_mode = (runtime.cudaAddressModeClamp,
+                        runtime.cudaAddressModeClamp)
+        tex = TextureDescriptor(address_mode, runtime.cudaFilterModePoint,
+                                runtime.cudaReadModeElementType)
+
+        # create a texture object
+        return TextureObject(res, tex)
+
+    def _prep_kernel1D(self):
+        return cupy.ElementwiseKernel(
+            'T x, U texObj',
+            'T y',
+            '''
+            T temp = tex1D<T>(texObj,
+                              float(i)
+                              );
+            y = temp + x;
+            ''', name='test_tex1D')
+
+    def _prep_kernel2D(self):
+        return cupy.ElementwiseKernel(
+            'T x, U texObj, uint64 width',
+            'T y',
+            '''
+            T temp = tex2D<T>(texObj,
+                              (float)(i % width),
+                              (float)(i / width)
+                              );
+            y = temp + x;
+            ''', name='test_tex2D')
+
+    def _prep_kernel3D(self):
+        return cupy.ElementwiseKernel(
+            'T x, U texObj, uint64 width, uint64 height',
+            'T y',
+            '''
+            T temp = tex3D<T>(texObj,
+                              (float)((i % (width * height)) % width),
+                              (float)((i % (width * height)) / width),
+                              (float)((i / (width * height)))
+                              );
+            y = temp + x;
+            ''', name='test_tex3D')
+
+    def test_texture_input(self):
+        width, height, depth = self.dimensions
+        dim = 3 if depth != 0 else 2 if height != 0 else 1
+
+        texobj = self._prep_texture()
+        ker = getattr(self, f'_prep_kernel{dim}D')()
+
+        # prepare input
+        args = [None, texobj]
+        size = width
+        if height > 0:
+            size *= height
+            args.append(width)
+        if depth > 0:
+            size *= depth
+            args.append(height)
+        in_arr = cupy.arange(size, dtype=cupy.float32)
+        in_arr = in_arr.reshape(self.shape)
+        args[0] = in_arr
+
+        # compute and validate output
+        out_arr = ker(*args)
+        expected = in_arr + self.data
+        testing.assert_allclose(out_arr, expected)
diff --git a/tests/cupy_tests/creation_tests/__init__.py b/tests/cupy_tests/creation_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/creation_tests/test_basic.py b/tests/cupy_tests/creation_tests/test_basic.py
new file mode 100644
index 0000000..8fd3eba
--- /dev/null
+++ b/tests/cupy_tests/creation_tests/test_basic.py
@@ -0,0 +1,496 @@
+import numpy
+import pytest
+import warnings
+
+import cupy
+from cupy import testing
+
+
+class TestBasic:
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty(self, xp, dtype, order):
+        a = xp.empty((2, 3, 4), dtype=dtype, order=order)
+        a.fill(0)
+        return a
+
+    @testing.slow
+    def test_empty_huge_size(self):
+        a = cupy.empty((1024, 2048, 1024), dtype='b')
+        a.fill(123)
+        assert (a == 123).all()
+        # Free huge memory for slow test
+        del a
+        cupy.get_default_memory_pool().free_all_blocks()
+
+    @testing.slow
+    def test_empty_huge_size_fill0(self):
+        a = cupy.empty((1024, 2048, 1024), dtype='b')
+        a.fill(0)
+        assert (a == 0).all()
+        # Free huge memory for slow test
+        del a
+        cupy.get_default_memory_pool().free_all_blocks()
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_scalar(self, xp, dtype, order):
+        a = xp.empty((), dtype=dtype, order=order)
+        a.fill(0)
+        return a
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_scalar_none(self, xp, dtype, order):
+        with testing.assert_warns(DeprecationWarning):
+            a = xp.empty(None, dtype=dtype, order=order)
+        a.fill(0)
+        return a
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_int(self, xp, dtype, order):
+        a = xp.empty(3, dtype=dtype, order=order)
+        a.fill(0)
+        return a
+
+    @testing.slow
+    def test_empty_int_huge_size(self):
+        a = cupy.empty(2 ** 31, dtype='b')
+        a.fill(123)
+        assert (a == 123).all()
+        # Free huge memory for slow test
+        del a
+        cupy.get_default_memory_pool().free_all_blocks()
+
+    @testing.slow
+    def test_empty_int_huge_size_fill0(self):
+        a = cupy.empty(2 ** 31, dtype='b')
+        a.fill(0)
+        assert (a == 0).all()
+        # Free huge memory for slow test
+        del a
+        cupy.get_default_memory_pool().free_all_blocks()
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_like(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.empty_like(a, order=order)
+        b.fill(0)
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_like_contiguity(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.empty_like(a, order=order)
+        b.fill(0)
+        if order in ['f', 'F']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_like_contiguity2(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        a = xp.asfortranarray(a)
+        b = xp.empty_like(a, order=order)
+        b.fill(0)
+        if order in ['c', 'C']:
+            assert b.flags.c_contiguous
+        else:
+            assert b.flags.f_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_like_contiguity3(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        # test strides that are both non-contiguous and non-descending
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = xp.empty_like(a, order=order)
+        b.fill(0)
+        if order in ['k', 'K', None]:
+            assert not b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        elif order in ['f', 'F']:
+            assert not b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        return b
+
+    @testing.for_all_dtypes()
+    def test_empty_like_K_strides(self, dtype):
+        # test strides that are both non-contiguous and non-descending
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = numpy.empty_like(a, order='K')
+        b.fill(0)
+
+        # GPU case
+        ag = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        ag = ag[:, ::2, :].swapaxes(0, 1)
+        bg = cupy.empty_like(ag, order='K')
+        bg.fill(0)
+
+        # make sure NumPy and CuPy strides agree
+        assert b.strides == bg.strides
+        return
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.for_all_dtypes()
+    def test_empty_like_invalid_order(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.empty_like(a, order='Q')
+
+    def test_empty_like_subok(self):
+        a = testing.shaped_arange((2, 3, 4), cupy)
+        with pytest.raises(TypeError):
+            cupy.empty_like(a, subok=True)
+
+    @testing.for_CF_orders()
+    @testing.with_requires('numpy>=1.23')
+    def test_empty_zero_sized_array_strides(self, order):
+        a = numpy.empty((1, 0, 2), dtype='d', order=order)
+        b = cupy.empty((1, 0, 2), dtype='d', order=order)
+        assert b.strides == a.strides
+
+    @pytest.mark.parametrize('offset', [1, -1, 1 << 63, -(1 << 63)])
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_eye(self, xp, dtype, order, offset):
+        return xp.eye(5, 4, offset, dtype, order=order)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_identity(self, xp, dtype):
+        return xp.identity(4, dtype)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_zeros(self, xp, dtype, order):
+        return xp.zeros((2, 3, 4), dtype=dtype, order=order)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_zeros_scalar(self, xp, dtype, order):
+        return xp.zeros((), dtype=dtype, order=order)
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_zeros_scalar_none(self, xp, dtype, order):
+        with testing.assert_warns(DeprecationWarning):
+            return xp.zeros(None, dtype=dtype, order=order)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_zeros_int(self, xp, dtype, order):
+        return xp.zeros(3, dtype=dtype, order=order)
+
+    @testing.for_CF_orders()
+    def test_zeros_strides(self, order):
+        a = numpy.zeros((2, 3), dtype='d', order=order)
+        b = cupy.zeros((2, 3), dtype='d', order=order)
+        assert b.strides == a.strides
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_zeros_like(self, xp, dtype, order):
+        a = xp.ndarray((2, 3, 4), dtype=dtype)
+        return xp.zeros_like(a, order=order)
+
+    def test_zeros_like_subok(self):
+        a = cupy.ndarray((2, 3, 4))
+        with pytest.raises(TypeError):
+            cupy.zeros_like(a, subok=True)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_ones(self, xp, dtype, order):
+        return xp.ones((2, 3, 4), dtype=dtype, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_ones_like(self, xp, dtype, order):
+        a = xp.ndarray((2, 3, 4), dtype=dtype)
+        return xp.ones_like(a, order=order)
+
+    def test_ones_like_subok(self):
+        a = cupy.ndarray((2, 3, 4))
+        with pytest.raises(TypeError):
+            cupy.ones_like(a, subok=True)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_full(self, xp, dtype, order):
+        return xp.full((2, 3, 4), 1, dtype=dtype, order=order)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_full_default_dtype(self, xp, dtype, order):
+        return xp.full((2, 3, 4), xp.array(1, dtype=dtype), order=order)
+
+    @testing.for_all_dtypes_combination(('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal()
+    def test_full_dtypes_cpu_input(self, xp, dtype1, dtype2):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', numpy.ComplexWarning)
+            return xp.full(
+                (2, 3, 4), numpy.array(1, dtype=dtype1), dtype=dtype2)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_full_like(self, xp, dtype, order):
+        a = xp.ndarray((2, 3, 4), dtype=dtype)
+        return xp.full_like(a, 1, order=order)
+
+    @testing.for_all_dtypes_combination(('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal()
+    def test_full_like_dtypes_cpu_input(self, xp, dtype1, dtype2):
+        a = xp.ndarray((2, 3, 4), dtype=dtype1)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', numpy.ComplexWarning)
+            return xp.full_like(a, numpy.array(1, dtype=dtype1))
+
+    def test_full_like_subok(self):
+        a = cupy.ndarray((2, 3, 4))
+        with pytest.raises(TypeError):
+            cupy.full_like(a, 1, subok=True)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [4, (4, ), (4, 2), (4, 2, 3), (5, 4, 2, 3)],
+    })
+)
+class TestBasicReshape:
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_like_reshape(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.empty_like(a, order=order, shape=self.shape)
+        b.fill(0)
+        return b
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    def test_empty_like_reshape_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupy.empty_like(a, shape=self.shape)
+        b.fill(0)
+        c = cupy.empty(self.shape, order=order, dtype=dtype)
+        c.fill(0)
+
+        testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_like_reshape_contiguity(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.empty_like(a, order=order, shape=self.shape)
+        b.fill(0)
+        if order in ['f', 'F']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    def test_empty_like_reshape_contiguity_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupy.empty_like(a, order=order, shape=self.shape)
+        b.fill(0)
+        c = cupy.empty(self.shape)
+        c.fill(0)
+        if order in ['f', 'F']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_like_reshape_contiguity2(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        a = xp.asfortranarray(a)
+        b = xp.empty_like(a, order=order, shape=self.shape)
+        b.fill(0)
+        shape = self.shape if not numpy.isscalar(self.shape) else (self.shape,)
+        if (order in ['c', 'C'] or
+                (order in ['k', 'K', None] and len(shape) != a.ndim)):
+            assert b.flags.c_contiguous
+        else:
+            assert b.flags.f_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    def test_empty_like_reshape_contiguity2_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        a = cupy.asfortranarray(a)
+        b = cupy.empty_like(a, order=order, shape=self.shape)
+        b.fill(0)
+        c = cupy.empty(self.shape)
+        c.fill(0)
+        shape = self.shape if not numpy.isscalar(self.shape) else (self.shape,)
+        if (order in ['c', 'C'] or
+                (order in ['k', 'K', None] and len(shape) != a.ndim)):
+            assert b.flags.c_contiguous
+        else:
+            assert b.flags.f_contiguous
+        testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_empty_like_reshape_contiguity3(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        # test strides that are both non-contiguous and non-descending
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = xp.empty_like(a, order=order, shape=self.shape)
+        b.fill(0)
+        shape = self.shape if not numpy.isscalar(self.shape) else (self.shape,)
+        if len(shape) == 1:
+            assert b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        elif order in ['k', 'K', None] and len(shape) == a.ndim:
+            assert not b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        elif order in ['f', 'F']:
+            assert not b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    def test_empty_like_reshape_contiguity3_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        # test strides that are both non-contiguous and non-descending
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = cupy.empty_like(a, order=order, shape=self.shape)
+        b.fill(0)
+        shape = self.shape if not numpy.isscalar(self.shape) else (self.shape,)
+        if len(shape) == 1:
+            assert b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        elif order in ['k', 'K', None] and len(shape) == a.ndim:
+            assert not b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        elif order in ['f', 'F']:
+            assert not b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+
+        c = cupy.zeros(self.shape)
+        c.fill(0)
+        testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_all_dtypes()
+    def test_empty_like_K_strides_reshape(self, dtype):
+        # test strides that are both non-contiguous and non-descending
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = numpy.empty_like(a, order='K', shape=self.shape)
+        b.fill(0)
+
+        # GPU case
+        ag = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        ag = ag[:, ::2, :].swapaxes(0, 1)
+        bg = cupy.empty_like(ag, order='K', shape=self.shape)
+        bg.fill(0)
+
+        # make sure NumPy and CuPy strides agree
+        assert b.strides == bg.strides
+        return
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_zeros_like_reshape(self, xp, dtype, order):
+        a = xp.ndarray((2, 3, 4), dtype=dtype)
+        return xp.zeros_like(a, order=order, shape=self.shape)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    def test_zeros_like_reshape_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupy.zeros_like(a, shape=self.shape)
+        c = cupy.zeros(self.shape, order=order, dtype=dtype)
+
+        testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_ones_like_reshape(self, xp, dtype, order):
+        a = xp.ndarray((2, 3, 4), dtype=dtype)
+        return xp.ones_like(a, order=order, shape=self.shape)
+
+    @testing.for_all_dtypes()
+    def test_ones_like_reshape_cupy_only(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupy.ones_like(a, shape=self.shape)
+        c = cupy.ones(self.shape, dtype=dtype)
+
+        testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_full_like_reshape(self, xp, dtype, order):
+        a = xp.ndarray((2, 3, 4), dtype=dtype)
+        return xp.full_like(a, 1, order=order, shape=self.shape)
+
+    @testing.for_all_dtypes()
+    def test_full_like_reshape_cupy_only(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupy.full_like(a, 1, shape=self.shape)
+        c = cupy.full(self.shape, 1, dtype=dtype)
+
+        testing.assert_array_equal(b, c)
diff --git a/tests/cupy_tests/creation_tests/test_from_data.py b/tests/cupy_tests/creation_tests/test_from_data.py
new file mode 100644
index 0000000..567f8f9
--- /dev/null
+++ b/tests/cupy_tests/creation_tests/test_from_data.py
@@ -0,0 +1,779 @@
+import tempfile
+import unittest
+
+import pytest
+
+import cupy
+from cupy import cuda
+from cupy import testing
+import numpy
+
+
+class TestFromData(unittest.TestCase):
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array(self, xp, dtype, order):
+        return xp.array([[1, 2, 3], [2, 3, 4]], dtype=dtype, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_from_empty_list(self, xp, dtype, order):
+        return xp.array([], dtype=dtype, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_from_nested_empty_list(self, xp, dtype, order):
+        return xp.array([[], []], dtype=dtype, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_from_numpy(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_from_numpy_scalar(self, xp, dtype, order):
+        a = numpy.array(2, dtype=dtype)
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_numpy_c_and_f(self, xp, dtype, order):
+        a = numpy.ones((1, 3, 1), dtype=dtype)
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_from_numpy_broad_cast(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 1, 4), numpy, dtype)
+        a = numpy.broadcast_to(a, (2, 3, 4))
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_list_of_numpy(self, xp, dtype, src_order, dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of numpy.ndarray>)
+        a = [
+            testing.shaped_arange((3, 4), numpy, dtype, src_order) + (12 * i)
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_list_of_numpy_view(self, xp, dtype, src_order,
+                                           dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of numpy.ndarray>)
+
+        # create a list of view of ndarrays
+        a = [
+            (testing.shaped_arange((3, 8), numpy,
+                                   dtype, src_order) + (24 * i))[:, ::2]
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_list_of_numpy_scalar(self, xp, dtype, order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of numpy.ndarray>)
+        a = [numpy.array(i, dtype=dtype) for i in range(2)]
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_nested_list_of_numpy(self, xp, dtype, src_order,
+                                             dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of numpy.ndarray>)
+        a = [
+            [testing.shaped_arange(
+                (3, 4), numpy, dtype, src_order) + (12 * i)]
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_list_of_cupy(
+            self, xp, dtype1, dtype2, src_order, dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+        a = [
+            testing.shaped_arange((3, 4), xp, dtype1, src_order),
+            testing.shaped_arange((3, 4), xp, dtype2, src_order),
+        ]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_list_of_cupy_view(self, xp, dtype, src_order,
+                                          dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+
+        # create a list of view of ndarrays
+        a = [
+            (testing.shaped_arange((3, 8), xp,
+                                   dtype, src_order) + (24 * i))[:, ::2]
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_nested_list_of_cupy(self, xp, dtype, src_order,
+                                            dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+        a = [
+            [testing.shaped_arange((3, 4), xp, dtype, src_order) + (12 * i)]
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_from_list_of_cupy_scalar(self, xp, dtype, order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+        a = [xp.array(i, dtype=dtype) for i in range(2)]
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_from_nested_list_of_cupy_scalar(self, xp, dtype, order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+        a = [[xp.array(i, dtype=dtype)] for i in range(2)]
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_copy(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_both_c_and_f_contig_copy(self, xp, dtype, order):
+        a = testing.shaped_arange((1, 4, 1), xp, dtype, order="C")
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_both_c_and_f_contig_f_strides_copy(self, xp, dtype, order):
+        a = testing.shaped_arange((1, 4, 1), xp, dtype, order="F")
+        return xp.array(a, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_copy_is_copied(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.array(a, order=order)
+        a.fill(0)
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes(name='dtype1', no_complex=True)
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_array_equal()
+    def test_array_copy_with_dtype(self, xp, dtype1, dtype2, order):
+        # complex to real makes no sense
+        a = testing.shaped_arange((2, 3, 4), xp, dtype1)
+        return xp.array(a, dtype=dtype2, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes(name='dtype1', no_complex=True)
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_array_equal()
+    def test_array_copy_with_dtype_char(self, xp, dtype1, dtype2, order):
+        # complex to real makes no sense
+        a = testing.shaped_arange((2, 3, 4), xp, dtype1)
+        return xp.array(a, dtype=numpy.dtype(dtype2).char, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_array_copy_with_dtype_being_none(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.array(a, dtype=None, order=order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes(name='dtype1', no_complex=True)
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_copy_list_of_numpy_with_dtype(self, xp, dtype1, dtype2,
+                                                 src_order, dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of numpy.ndarray>)
+        a = [
+            testing.shaped_arange((3, 4), numpy, dtype1, src_order) + (12 * i)
+            for i in range(2)]
+        return xp.array(a, dtype=dtype2, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes(name='dtype1', no_complex=True)
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_copy_list_of_numpy_with_dtype_char(self, xp, dtype1,
+                                                      dtype2, src_order,
+                                                      dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of numpy.ndarray>)
+        a = [
+            testing.shaped_arange((3, 4), numpy, dtype1, src_order) + (12 * i)
+            for i in range(2)]
+        return xp.array(a, dtype=numpy.dtype(dtype2).char, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes(name='dtype1', no_complex=True)
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_copy_list_of_cupy_with_dtype(self, xp, dtype1, dtype2,
+                                                src_order, dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+        a = [
+            testing.shaped_arange((3, 4), xp, dtype1, src_order) + (12 * i)
+            for i in range(2)]
+        return xp.array(a, dtype=dtype2, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes(name='dtype1', no_complex=True)
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_array_copy_list_of_cupy_with_dtype_char(self, xp, dtype1, dtype2,
+                                                     src_order, dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+        a = [
+            testing.shaped_arange((3, 4), xp, dtype1, src_order) + (12 * i)
+            for i in range(2)]
+        return xp.array(a, dtype=numpy.dtype(dtype2).char, order=dst_order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_no_copy(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.array(a, copy=False, order=order)
+        a.fill(0)
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_f_contiguous_input(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype, order='F')
+        b = xp.array(a, copy=False, order=order)
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_f_contiguous_output(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.array(a, copy=False, order='F')
+        assert b.flags.f_contiguous
+        return b
+
+    @testing.multi_gpu(2)
+    def test_array_multi_device(self):
+        with cuda.Device(0):
+            x = testing.shaped_arange((2, 3, 4), cupy, dtype='f')
+        with cuda.Device(1):
+            y = cupy.array(x)
+        assert isinstance(y, cupy.ndarray)
+        assert x is not y  # Do copy
+        assert int(x.device) == 0
+        assert int(y.device) == 1
+        testing.assert_array_equal(x, y)
+
+    @testing.multi_gpu(2)
+    def test_array_multi_device_zero_size(self):
+        with cuda.Device(0):
+            x = testing.shaped_arange((0,), cupy, dtype='f')
+        with cuda.Device(1):
+            y = cupy.array(x)
+        assert isinstance(y, cupy.ndarray)
+        assert x is not y  # Do copy
+        assert x.device.id == 0
+        assert y.device.id == 1
+        testing.assert_array_equal(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_no_copy_ndmin(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.array(a, copy=False, ndmin=5)
+        assert a.shape == (2, 3, 4)
+        a.fill(0)
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_from_big_endian(self, xp, dtype):
+        dtype = numpy.dtype(dtype).newbyteorder('>')
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        b = xp.array(a)
+        # Make a computation here as just moving big-endian data back and forth
+        # happens to work before the change in #5828
+        return b + b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_array_from_list_of_numpy_big_endian(self, xp, dtype):
+        dtype = numpy.dtype(dtype).newbyteorder('>')
+        a = [testing.shaped_arange((3, 4), numpy, dtype) for i in range(2)]
+        b = xp.array(a)
+        # Make a computation here as just moving big-endian data back and forth
+        # happens to work before the change in #5828
+        return b + b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asarray(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.asarray(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_is_not_copied(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.asarray(a)
+        a.fill(0)
+        return b
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_with_order(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.asarray(a, order=order)
+        if order in ['F', 'f']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        return b
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_preserves_numpy_array_order(self, xp, dtype, order):
+        a_numpy = testing.shaped_arange((2, 3, 4), numpy, dtype, order)
+        b = xp.asarray(a_numpy)
+        assert b.flags.f_contiguous == a_numpy.flags.f_contiguous
+        assert b.flags.c_contiguous == a_numpy.flags.c_contiguous
+        return b
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asanyarray_with_order(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.asanyarray(a, order=order)
+        if order in ['F', 'f']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asanyarray_from_big_endian(self, xp, dtype):
+        dtype = numpy.dtype(dtype).newbyteorder('>')
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        b = xp.asanyarray(a)
+        # Make a computation here as just moving big-endian data back and forth
+        # happens to work before the change in #5828
+        return b + b
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_from_numpy(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        b = xp.asarray(a, order=order)
+        if order in ['F', 'f']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        return b
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_with_order_copy_behavior(self, xp, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.asarray(a, order=order)
+        a.fill(0)
+        return b
+
+    def test_ascontiguousarray_on_noncontiguous_array(self):
+        a = testing.shaped_arange((2, 3, 4))
+        b = a.transpose(2, 0, 1)
+        c = cupy.ascontiguousarray(b)
+        assert c.flags.c_contiguous
+        testing.assert_array_equal(b, c)
+
+    def test_ascontiguousarray_on_contiguous_array(self):
+        a = testing.shaped_arange((2, 3, 4))
+        b = cupy.ascontiguousarray(a)
+        assert a is b
+
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_cuda_array_zero_dim(self, xp):
+        a = xp.ones(())
+        return xp.ascontiguousarray(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_cuda_array_zero_dim_dtype(self, xp):
+        a = xp.ones((), dtype=numpy.float64)
+        return xp.ascontiguousarray(a, dtype=numpy.int64)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_from_big_endian(self, xp, dtype):
+        dtype = numpy.dtype(dtype).newbyteorder('>')
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        b = xp.asarray(a)
+        # Make a computation here as just moving big-endian data back and forth
+        # happens to work before the change in #5828
+        return b + b
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copy(self, xp, dtype, order):
+        a = xp.zeros((2, 3, 4), dtype=dtype)
+        b = xp.copy(a, order=order)
+        a[1] = 1
+        return b
+
+    @testing.multi_gpu(2)
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    def test_copy_multigpu(self, dtype, order):
+        with cuda.Device(0):
+            src = cupy.random.uniform(-1, 1, (2, 3)).astype(dtype)
+        with cuda.Device(1):
+            dst = cupy.copy(src, order)
+        testing.assert_allclose(src, dst, rtol=0, atol=0)
+
+    @testing.for_CF_orders()
+    @testing.numpy_cupy_equal()
+    def test_copy_order(self, xp, order):
+        a = xp.zeros((2, 3, 4), order=order)
+        b = xp.copy(a)
+        return (b.flags.c_contiguous, b.flags.f_contiguous)
+
+    @testing.numpy_cupy_array_equal()
+    def test_asfortranarray_cuda_array_zero_dim(self, xp):
+        a = xp.ones(())
+        return xp.asfortranarray(a)
+
+    @testing.for_all_dtypes_combination(['dtype_a', 'dtype_b'],
+                                        no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_asfortranarray_cuda_array_zero_dim_dtype(
+            self, xp, dtype_a, dtype_b):
+        a = xp.ones((), dtype=dtype_a)
+        return xp.asfortranarray(a, dtype=dtype_b)
+
+    @testing.numpy_cupy_array_equal()
+    def test_fromfile(self, xp):
+        with tempfile.TemporaryFile() as fh:
+            fh.write(b"\x00\x01\x02\x03\x04")
+            fh.flush()
+            fh.seek(0)
+            return xp.fromfile(fh, dtype="u1")
+
+    @testing.numpy_cupy_array_equal()
+    def test_fromfunction(self, xp):
+        def function(i, j): return i == j
+        return xp.fromfunction(function, shape=(3, 3), dtype=int)
+
+    @testing.numpy_cupy_array_equal()
+    def test_fromiter(self, xp):
+        iterable = (x*x for x in range(5))
+        return xp.fromiter(iterable, float)
+
+    @testing.numpy_cupy_array_equal()
+    def test_fromstring(self, xp):
+        return xp.fromstring('1 2', dtype=int, sep=' ')
+
+    @testing.numpy_cupy_array_equal()
+    def test_frombuffer(self, xp):
+        return xp.frombuffer(b'\x01\x02', dtype=numpy.uint8)
+
+    @testing.numpy_cupy_array_equal()
+    def test_loadtxt(self, xp):
+        with tempfile.TemporaryFile() as fh:
+            fh.write(b"0 1\n2 3")
+            fh.flush()
+            fh.seek(0)
+            return xp.loadtxt(fh, dtype="u1")
+
+    @testing.numpy_cupy_array_equal()
+    def test_genfromtxt(self, xp):
+        with tempfile.TemporaryFile() as fh:
+            fh.write(b"0 1\n2 3")
+            fh.flush()
+            fh.seek(0)
+            return xp.genfromtxt(fh, dtype="u1")
+
+    @testing.numpy_cupy_array_equal()
+    def test_fromfile_big_endian(self, xp):
+        with tempfile.TemporaryFile() as fh:
+            fh.write(b"\x00\x00\x00\x01")
+            fh.flush()
+            fh.seek(0)
+            a = xp.fromfile(fh, dtype='>u4')
+            # Make a computation here as just moving big-endian data back and
+            # forth happens to work before the change in #5828
+            return a + a
+
+
+max_cuda_array_interface_version = 3
+
+
+@testing.parameterize(*testing.product({
+    'ver': tuple(range(max_cuda_array_interface_version+1)),
+    'strides': (False, None, True),
+}))
+@pytest.mark.skipif(
+    cupy.cuda.runtime.is_hip, reason='HIP does not support this')
+class TestCudaArrayInterface(unittest.TestCase):
+    @testing.for_all_dtypes()
+    def test_base(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupy.asarray(
+            DummyObjectWithCudaArrayInterface(a, self.ver, self.strides))
+        testing.assert_array_equal(a, b)
+
+    @testing.for_all_dtypes()
+    def test_not_copied(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupy.asarray(
+            DummyObjectWithCudaArrayInterface(a, self.ver, self.strides))
+        a.fill(0)
+        testing.assert_array_equal(a, b)
+
+    @testing.for_all_dtypes()
+    def test_order(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupy.asarray(
+            DummyObjectWithCudaArrayInterface(a, self.ver, self.strides),
+            order='F')
+        assert b.flags.f_contiguous
+        testing.assert_array_equal(a, b)
+
+    @testing.for_all_dtypes()
+    def test_with_strides(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype).T
+        b = cupy.asarray(
+            DummyObjectWithCudaArrayInterface(a, self.ver, self.strides))
+        assert a.strides == b.strides
+        assert a.nbytes == b.data.mem.size
+
+    @testing.for_all_dtypes()
+    def test_with_zero_size_array(self, dtype):
+        a = testing.shaped_arange((0,), cupy, dtype)
+        b = cupy.asarray(
+            DummyObjectWithCudaArrayInterface(a, self.ver, self.strides))
+        assert a.strides == b.strides
+        assert a.nbytes == b.data.mem.size
+        assert a.data.ptr == 0
+        assert a.size == 0
+
+    @testing.for_all_dtypes()
+    def test_asnumpy(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = DummyObjectWithCudaArrayInterface(a, self.ver, self.strides)
+        a_cpu = cupy.asnumpy(a)
+        b_cpu = cupy.asnumpy(b)
+        testing.assert_array_equal(a_cpu, b_cpu)
+
+    def test_big_endian(self):
+        a = cupy.array([0x1, 0x0, 0x0, 0x0], dtype=numpy.int8)
+        dtype = numpy.dtype('>i4')
+        shape = 1,
+        strides = 4,
+        data = a.data.ptr
+        b = DummyObjectWithCudaArrayInterface(
+            (shape, strides, dtype.str, dtype.descr, data),
+            self.ver, self.strides)
+        with pytest.raises(ValueError):
+            cupy.asarray(b)
+
+
+@testing.parameterize(*testing.product({
+    'ver': tuple(range(1, max_cuda_array_interface_version+1)),
+    'strides': (False, None, True),
+}))
+@pytest.mark.skipif(
+    cupy.cuda.runtime.is_hip, reason='HIP does not support this')
+class TestCudaArrayInterfaceMaskedArray(unittest.TestCase):
+    # TODO(leofang): update this test when masked array is supported
+    @testing.for_all_dtypes()
+    def test_masked_array(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        mask = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        a = DummyObjectWithCudaArrayInterface(a, self.ver, self.strides, mask)
+        with pytest.raises(ValueError) as ex:
+            b = cupy.asarray(a)  # noqa
+        assert 'does not support' in str(ex.value)
+
+
+# marked slow as either numpy or cupy could go OOM in this test
+@testing.slow
+@pytest.mark.skipif(
+    cupy.cuda.runtime.is_hip, reason='HIP does not support this')
+class TestCudaArrayInterfaceBigArray(unittest.TestCase):
+    def test_with_over_size_array(self):
+        # real example from #3009
+        size = 5 * 10**8
+        a = testing.shaped_random((size,), cupy, cupy.float64)
+        b = cupy.asarray(DummyObjectWithCudaArrayInterface(a, 2, None))
+        testing.assert_array_equal(a, b)
+
+
+@pytest.mark.skipif(
+    cupy.cuda.runtime.is_hip, reason='HIP does not support this')
+class DummyObjectWithCudaArrayInterface(object):
+    def __init__(self, a, ver, include_strides=False, mask=None, stream=None):
+        assert ver in tuple(range(max_cuda_array_interface_version+1))
+        self.a = None
+        if isinstance(a, cupy.ndarray):
+            self.a = a
+        else:
+            self.shape, self.strides, self.typestr, self.descr, self.data = a
+        self.ver = ver
+        self.include_strides = include_strides
+        self.mask = mask
+        self.stream = stream
+
+    @property
+    def __cuda_array_interface__(self):
+        if self.a is not None:
+            desc = {
+                'shape': self.a.shape,
+                'typestr': self.a.dtype.str,
+                'descr': self.a.dtype.descr,
+                'data': (self.a.data.ptr, False),
+                'version': self.ver,
+            }
+            if self.a.flags.c_contiguous:
+                if self.include_strides is True:
+                    desc['strides'] = self.a.strides
+                elif self.include_strides is None:
+                    desc['strides'] = None
+                else:  # self.include_strides is False
+                    pass
+            else:  # F contiguous or neither
+                desc['strides'] = self.a.strides
+        else:
+            desc = {
+                'shape': self.shape,
+                'typestr': self.typestr,
+                'descr': self.descr,
+                'data': (self.data, False),
+                'version': self.ver,
+            }
+            if self.include_strides is True:
+                desc['strides'] = self.strides
+            elif self.include_strides is None:
+                desc['strides'] = None
+            else:  # self.include_strides is False
+                pass
+        if self.mask is not None:
+            desc['mask'] = self.mask
+        # The stream field is kept here for compliance. However, since the
+        # synchronization is done via calling a cpdef function, which cannot
+        # be mock-tested.
+        if self.stream is not None:
+            if self.stream is cuda.Stream.null:
+                desc['stream'] = cuda.runtime.streamLegacy
+            elif (not cuda.runtime.is_hip) and self.stream is cuda.Stream.ptds:
+                desc['stream'] = cuda.runtime.streamPerThread
+            else:
+                desc['stream'] = self.stream.ptr
+        return desc
+
+
+@testing.parameterize(
+    *testing.product({
+        'ndmin': [0, 1, 2, 3],
+        'copy': [True, False],
+        'xp': [numpy, cupy]
+    })
+)
+class TestArrayPreservationOfShape(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_cupy_array(self, dtype):
+        shape = 2, 3
+        a = testing.shaped_arange(shape, self.xp, dtype)
+        cupy.array(a, copy=self.copy, ndmin=self.ndmin)
+
+        # Check if cupy.ndarray does not alter
+        # the shape of the original array.
+        assert a.shape == shape
+
+
+@testing.parameterize(
+    *testing.product({
+        'ndmin': [0, 1, 2, 3],
+        'copy': [True, False],
+        'xp': [numpy, cupy]
+    })
+)
+class TestArrayCopy(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_cupy_array(self, dtype):
+        a = testing.shaped_arange((2, 3), self.xp, dtype)
+        actual = cupy.array(a, copy=self.copy, ndmin=self.ndmin)
+
+        should_copy = (self.xp is numpy) or self.copy
+        # TODO(Kenta Oono): Better determination of copy.
+        is_copied = not ((actual is a) or (actual.base is a) or
+                         (actual.base is a.base and a.base is not None))
+        assert should_copy == is_copied
+
+
+class TestArrayInvalidObject(unittest.TestCase):
+
+    def test_invalid_type(self):
+        a = numpy.array([1, 2, 3], dtype=object)
+        with self.assertRaises(ValueError):
+            cupy.array(a)
diff --git a/tests/cupy_tests/creation_tests/test_matrix.py b/tests/cupy_tests/creation_tests/test_matrix.py
new file mode 100644
index 0000000..3818434
--- /dev/null
+++ b/tests/cupy_tests/creation_tests/test_matrix.py
@@ -0,0 +1,220 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestMatrix(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag1(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        return xp.diag(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag2(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        return xp.diag(a, 1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag3(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        return xp.diag(a, -2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag_extraction_from_nested_list(self, xp):
+        a = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
+        r = xp.diag(a, 1)
+        assert isinstance(r, xp.ndarray)
+        return r
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag_extraction_from_nested_tuple(self, xp):
+        a = ((1, 2, 3), (4, 5, 6), (7, 8, 9))
+        r = xp.diag(a, -1)
+        assert isinstance(r, xp.ndarray)
+        return r
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag_construction(self, xp):
+        a = testing.shaped_arange((3,), xp)
+        r = xp.diag(a)
+        assert isinstance(r, xp.ndarray)
+        return r
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag_construction_from_list(self, xp):
+        a = [1, 2, 3]
+        r = xp.diag(a)
+        assert isinstance(r, xp.ndarray)
+        return r
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag_construction_from_tuple(self, xp):
+        a = (1, 2, 3)
+        r = xp.diag(a)
+        assert isinstance(r, xp.ndarray)
+        return r
+
+    def test_diag_scaler(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.diag(1)
+
+    def test_diag_0dim(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.diag(xp.zeros(()))
+
+    def test_diag_3dim(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.diag(xp.zeros((2, 2, 2)))
+
+    @testing.numpy_cupy_array_equal()
+    def test_diagflat1(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        return xp.diagflat(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_diagflat2(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        return xp.diagflat(a, 1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_diagflat3(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        return xp.diagflat(a, -2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_diagflat_from_scalar(self, xp):
+        return xp.diagflat(3)
+
+    @testing.numpy_cupy_array_equal()
+    def test_diagflat_from_scalar_with_k0(self, xp):
+        return xp.diagflat(3, 0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_diagflat_from_scalar_with_k1(self, xp):
+        return xp.diagflat(3, 1)
+
+
+@testing.parameterize(
+    {'shape': (2,)},
+    {'shape': (3, 3)},
+    {'shape': (4, 3)},
+)
+class TestTri(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_tri(self, xp, dtype):
+        return xp.tri(*self.shape, k=0, dtype=dtype)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_tri_nega(self, xp, dtype):
+        return xp.tri(*self.shape, k=-1, dtype=dtype)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_tri_posi(self, xp, dtype):
+        return xp.tri(*self.shape, k=1, dtype=dtype)
+
+
+@testing.parameterize(
+    {'shape': (2,)},
+    {'shape': (3, 3)},
+    {'shape': (4, 3)},
+    {'shape': (2, 3, 4)},
+)
+class TestTriLowerAndUpper(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_tril(self, xp, dtype):
+        m = testing.shaped_arange(self.shape, xp, dtype)
+        return xp.tril(m)
+
+    @testing.numpy_cupy_array_equal()
+    def test_tril_array_like(self, xp):
+        return xp.tril([[1, 2], [3, 4]])
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_tril_nega(self, xp, dtype):
+        m = testing.shaped_arange(self.shape, xp, dtype)
+        return xp.tril(m, -1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_tril_posi(self, xp, dtype):
+        m = testing.shaped_arange(self.shape, xp, dtype)
+        return xp.tril(m, 1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_triu(self, xp, dtype):
+        m = testing.shaped_arange(self.shape, xp, dtype)
+        return xp.triu(m)
+
+    @testing.numpy_cupy_array_equal()
+    def test_triu_array_like(self, xp):
+        return xp.triu([[1, 2], [3, 4]])
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_triu_nega(self, xp, dtype):
+        m = testing.shaped_arange(self.shape, xp, dtype)
+        return xp.triu(m, -1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_triu_posi(self, xp, dtype):
+        m = testing.shaped_arange(self.shape, xp, dtype)
+        return xp.triu(m, 1)
+
+
+@testing.parameterize(
+    *testing.product({
+        'N': [None, 0, 1, 2, 3],
+        'increasing': [False, True]
+    })
+)
+class TestVander(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_vander(self, xp, dtype):
+        a = testing.shaped_arange((3,), xp, dtype=dtype)
+        return xp.vander(a, N=self.N, increasing=self.increasing)
+
+    @testing.numpy_cupy_allclose()
+    def test_vander_array_like_list(self, xp):
+        a = [0, 1, 2, 3, 4, 5, 6]
+        return xp.vander(a, N=self.N, increasing=self.increasing)
+
+    @testing.numpy_cupy_allclose()
+    def test_vander_array_like_tuple(self, xp):
+        a = (0, 1, 2, 3, 4, 5, 6)
+        return xp.vander(a, N=self.N, increasing=self.increasing)
+
+    def test_vander_scalar(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.vander(1, N=self.N, increasing=self.increasing)
+
+    def test_vander_0dim(self):
+        for xp in (numpy, cupy):
+            a = xp.zeros(())
+            with pytest.raises(ValueError):
+                xp.vander(a, N=self.N, increasing=self.increasing)
+
+    def test_vander_2dim(self):
+        for xp in (numpy, cupy):
+            m = xp.zeros((2, 2))
+            with pytest.raises(ValueError):
+                xp.vander(m, N=self.N, increasing=self.increasing)
diff --git a/tests/cupy_tests/creation_tests/test_ranges.py b/tests/cupy_tests/creation_tests/test_ranges.py
new file mode 100644
index 0000000..42a84df
--- /dev/null
+++ b/tests/cupy_tests/creation_tests/test_ranges.py
@@ -0,0 +1,401 @@
+import functools
+import math
+import sys
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+def skip_int_equality_before_numpy_1_20(names=('dtype',)):
+    """Require numpy/numpy#16841 or skip the equality check."""
+    def decorator(wrapped):
+        if numpy.lib.NumpyVersion(numpy.__version__) >= '1.20.0':
+            return wrapped
+
+        @functools.wraps(wrapped)
+        def wrapper(self, *args, **kwargs):
+            xp = kwargs['xp']
+            dtypes = [kwargs[name] for name in names]
+            ret = wrapped(self, *args, **kwargs)
+            if any(numpy.issubdtype(dtype, numpy.integer) for dtype in dtypes):
+                ret = xp.zeros_like(ret)
+            return ret
+
+        return wrapper
+
+    return decorator
+
+
+class TestRanges(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_arange(self, xp, dtype):
+        return xp.arange(10, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_arange2(self, xp, dtype):
+        return xp.arange(5, 10, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_arange3(self, xp, dtype):
+        return xp.arange(1, 11, 2, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_arange4(self, xp, dtype):
+        return xp.arange(20, 2, -3).astype(dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_arange5(self, xp, dtype):
+        return xp.arange(0, 100, None, dtype=dtype)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_arange6(self, xp, dtype):
+        return xp.arange(0, 2, dtype=dtype)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_arange7(self, xp, dtype):
+        return xp.arange(10, 11, dtype=dtype)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_arange8(self, xp, dtype):
+        return xp.arange(10, 8, -1).astype(dtype)
+
+    @testing.with_requires('numpy>=1.24')
+    def test_arange9(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(TypeError):
+                xp.arange(10, dtype=xp.bool_)
+
+    @testing.numpy_cupy_array_equal()
+    def test_arange_no_dtype_int(self, xp):
+        return xp.arange(1, 11, 2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_arange_no_dtype_float(self, xp):
+        return xp.arange(1.0, 11.0, 2.0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_arange_negative_size(self, xp):
+        return xp.arange(3, 1)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace(self, xp, dtype):
+        return xp.linspace(0, 10, 5, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace2(self, xp, dtype):
+        return xp.linspace(10, 0, 5, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    @skip_int_equality_before_numpy_1_20()
+    def test_linspace3(self, xp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        return xp.linspace(-10, 8, 9, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_zero_num(self, xp, dtype):
+        return xp.linspace(0, 10, 0, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_zero_num_no_endopoint_with_retstep(self, xp, dtype):
+        x, step = xp.linspace(0, 10, 0, dtype=dtype, endpoint=False,
+                              retstep=True)
+        assert math.isnan(step)
+        return x
+
+    @testing.with_requires('numpy>=1.18')
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_one_num_no_endopoint_with_retstep(self, xp, dtype):
+        start, stop = 3, 7
+        x, step = xp.linspace(start, stop, 1, dtype=dtype, endpoint=False,
+                              retstep=True)
+        assert step == stop - start
+        return x
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_one_num(self, xp, dtype):
+        return xp.linspace(0, 2, 1, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_no_endpoint(self, xp, dtype):
+        return xp.linspace(0, 10, 5, dtype=dtype, endpoint=False)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_with_retstep(self, xp, dtype):
+        x, step = xp.linspace(0, 10, 5, dtype=dtype, retstep=True)
+        assert step == 2.5
+        return x
+
+    @testing.numpy_cupy_allclose()
+    def test_linspace_no_dtype_int(self, xp):
+        return xp.linspace(0, 10)
+
+    @testing.numpy_cupy_allclose()
+    def test_linspace_no_dtype_float(self, xp):
+        return xp.linspace(0.0, 10.0)
+
+    @testing.numpy_cupy_allclose()
+    def test_linspace_float_args_with_int_dtype(self, xp):
+        return xp.linspace(0.1, 9.1, 11, dtype=int)
+
+    def test_linspace_neg_num(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.linspace(0, 10, -1)
+
+    @testing.numpy_cupy_allclose()
+    def test_linspace_float_overflow(self, xp):
+        return xp.linspace(0., sys.float_info.max / 5, 10, dtype=float)
+
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_float_underflow(self, xp):
+        # find minimum subnormal number
+        x = sys.float_info.min
+        while x / 2 > 0:
+            x /= 2
+        return xp.linspace(0., x, 10, dtype=float)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_all_dtypes_combination(names=('dtype_range', 'dtype_out'),
+                                        no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_array_start_stop(self, xp, dtype_range, dtype_out):
+        start = xp.array([0, 120], dtype=dtype_range)
+        stop = xp.array([100, 0], dtype=dtype_range)
+        return xp.linspace(start, stop, num=50, dtype=dtype_out)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_all_dtypes_combination(names=('dtype_range', 'dtype_out'),
+                                        no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    @skip_int_equality_before_numpy_1_20(names=('dtype_range', 'dtype_out'))
+    def test_linspace_mixed_start_stop(self, xp, dtype_range, dtype_out):
+        start = 0.0
+        if xp.dtype(dtype_range).kind in 'u':
+            stop = xp.array([100, 16], dtype=dtype_range)
+        else:
+            stop = xp.array([100, -100], dtype=dtype_range)
+        return xp.linspace(start, stop, num=50, dtype=dtype_out)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_all_dtypes_combination(names=('dtype_range', 'dtype_out'),
+                                        no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    @skip_int_equality_before_numpy_1_20(names=('dtype_range', 'dtype_out'))
+    def test_linspace_mixed_start_stop2(self, xp, dtype_range, dtype_out):
+        if xp.dtype(dtype_range).kind in 'u':
+            start = xp.array([160, 120], dtype=dtype_range)
+        else:
+            start = xp.array([-120, 120], dtype=dtype_range)
+        stop = 0
+        return xp.linspace(start, stop, num=50, dtype=dtype_out)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_all_dtypes_combination(names=('dtype_range', 'dtype_out'),
+                                        no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_array_start_stop_axis1(self, xp, dtype_range, dtype_out):
+        start = xp.array([0, 120], dtype=dtype_range)
+        stop = xp.array([100, 0], dtype=dtype_range)
+        return xp.linspace(start, stop, num=50, dtype=dtype_out, axis=1)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_complex_start_stop(self, xp, dtype):
+        start = xp.array([0, 120], dtype=dtype)
+        stop = xp.array([100, 0], dtype=dtype)
+        return xp.linspace(start, stop, num=50, dtype=dtype)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_linspace_start_stop_list(self, xp, dtype):
+        start = [0, 0]
+        stop = [100, 16]
+        return xp.linspace(start, stop, num=50, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_logspace(self, xp, dtype):
+        return xp.logspace(0, 2, 5, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_logspace2(self, xp, dtype):
+        return xp.logspace(2, 0, 5, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_logspace_zero_num(self, xp, dtype):
+        return xp.logspace(0, 2, 0, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_logspace_one_num(self, xp, dtype):
+        return xp.logspace(0, 2, 1, dtype=dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_logspace_no_endpoint(self, xp, dtype):
+        return xp.logspace(0, 2, 5, dtype=dtype, endpoint=False)
+
+    @testing.numpy_cupy_allclose()
+    def test_logspace_no_dtype_int(self, xp):
+        return xp.logspace(0, 2)
+
+    @testing.numpy_cupy_allclose()
+    def test_logspace_no_dtype_float(self, xp):
+        return xp.logspace(0.0, 2.0)
+
+    @testing.numpy_cupy_allclose()
+    def test_logspace_float_args_with_int_dtype(self, xp):
+        return xp.logspace(0.1, 2.1, 11, dtype=int)
+
+    def test_logspace_neg_num(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.logspace(0, 10, -1)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_logspace_base(self, xp, dtype):
+        return xp.logspace(0, 2, 5, base=2.0, dtype=dtype)
+
+    # See #7946 and https://github.com/numpy/numpy/issues/24957
+    @testing.with_requires('numpy>=1.16, !=1.25.*, !=1.26.*')
+    @testing.for_all_dtypes_combination(names=('dtype_range', 'dtype_out'),
+                                        no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6, contiguous_check=False)
+    def test_logspace_array_start_stop_axis1(self, xp, dtype_range, dtype_out):
+        start = xp.array([0, 2], dtype=dtype_range)
+        stop = xp.array([2, 0], dtype=dtype_range)
+        return xp.logspace(start, stop, num=5, dtype=dtype_out, axis=1)
+
+
+@testing.parameterize(
+    *testing.product({
+        'indexing': ['xy', 'ij'],
+        'sparse': [False, True],
+        'copy': [False, True],
+    })
+)
+class TestMeshgrid(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_meshgrid0(self, dtype):
+        out = cupy.meshgrid(indexing=self.indexing, sparse=self.sparse,
+                            copy=self.copy)
+        assert (out == [])
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_meshgrid1(self, xp, dtype):
+        x = xp.arange(2).astype(dtype)
+        return xp.meshgrid(x, indexing=self.indexing, sparse=self.sparse,
+                           copy=self.copy)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_meshgrid2(self, xp, dtype):
+        x = xp.arange(2).astype(dtype)
+        y = xp.arange(3).astype(dtype)
+        return xp.meshgrid(x, y, indexing=self.indexing, sparse=self.sparse,
+                           copy=self.copy)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_meshgrid3(self, xp, dtype):
+        x = xp.arange(2).astype(dtype)
+        y = xp.arange(3).astype(dtype)
+        z = xp.arange(4).astype(dtype)
+        return xp.meshgrid(x, y, z, indexing=self.indexing, sparse=self.sparse,
+                           copy=self.copy)
+
+
+class TestMgrid(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_mgrid0(self, xp):
+        return xp.mgrid[0:]
+
+    @testing.numpy_cupy_array_equal()
+    def test_mgrid1(self, xp):
+        return xp.mgrid[-10:10]
+
+    @testing.numpy_cupy_array_equal()
+    def test_mgrid2(self, xp):
+        return xp.mgrid[-10:10:10j]
+
+    @testing.numpy_cupy_array_equal()
+    def test_mgrid3(self, xp):
+        x = xp.zeros(10)[:, None]
+        y = xp.ones(10)[:, None]
+        return xp.mgrid[x:y:10j]
+
+    @testing.numpy_cupy_array_equal()
+    def test_mgrid4(self, xp):
+        # check len(keys) > 1
+        return xp.mgrid[-10:10:10j, -10:10:10j]
+
+    @testing.numpy_cupy_array_equal()
+    def test_mgrid5(self, xp):
+        # check len(keys) > 1
+        x = xp.zeros(10)[:, None]
+        y = xp.ones(10)[:, None]
+        return xp.mgrid[x:y:10j, x:y:10j]
+
+
+class TestOgrid(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_ogrid0(self, xp):
+        return xp.ogrid[0:]
+
+    @testing.numpy_cupy_array_equal()
+    def test_ogrid1(self, xp):
+        return xp.ogrid[-10:10]
+
+    @testing.numpy_cupy_array_equal()
+    def test_ogrid2(self, xp):
+        return xp.ogrid[-10:10:10j]
+
+    @testing.numpy_cupy_array_equal()
+    def test_ogrid3(self, xp):
+        x = xp.zeros(10)[:, None]
+        y = xp.ones(10)[:, None]
+        return xp.ogrid[x:y:10j]
+
+    @testing.numpy_cupy_array_equal()
+    def test_ogrid4(self, xp):
+        # check len(keys) > 1
+        return xp.ogrid[-10:10:10j, -10:10:10j]
+
+    @testing.numpy_cupy_array_equal()
+    def test_ogrid5(self, xp):
+        # check len(keys) > 1
+        x = xp.zeros(10)[:, None]
+        y = xp.ones(10)[:, None]
+        return xp.ogrid[x:y:10j, x:y:10j]
diff --git a/tests/cupy_tests/cuda_tests/__init__.py b/tests/cupy_tests/cuda_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/cuda_tests/memory_hooks_tests/__init__.py b/tests/cupy_tests/cuda_tests/memory_hooks_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/cuda_tests/memory_hooks_tests/test_debug_print.py b/tests/cupy_tests/cuda_tests/memory_hooks_tests/test_debug_print.py
new file mode 100644
index 0000000..a3e5a0e
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/memory_hooks_tests/test_debug_print.py
@@ -0,0 +1,49 @@
+import io
+import json
+import unittest
+
+import cupy.cuda
+from cupy.cuda import memory
+from cupy.cuda import memory_hooks
+
+
+class TestDebugPrintHook(unittest.TestCase):
+
+    def setUp(self):
+        self.io = io.StringIO()
+        self.hook = memory_hooks.DebugPrintHook(file=self.io)
+        self.pool = memory.MemoryPool()
+
+    def test_print(self):
+        device_id = 0
+        size = 1
+        unit = 512
+        with cupy.cuda.Device(device_id):
+            with self.hook:
+                mem = self.pool.malloc(size)
+                ptr1, pmem1 = mem.ptr, id(mem.mem)
+                del mem
+                mem = self.pool.malloc(size)
+                ptr2, pmem2 = mem.ptr, id(mem.mem)
+                del mem
+        actual_lines = self.io.getvalue().splitlines()
+
+        expect = {'hook': 'alloc', 'device_id': device_id,
+                  'mem_size': unit, 'mem_ptr': ptr1}
+        assert expect == json.loads(actual_lines[0])
+
+        expect = {'hook': 'malloc', 'device_id': device_id, 'size': size,
+                  'mem_size': unit, 'mem_ptr': ptr1, 'pmem_id': hex(pmem1)}
+        assert expect == json.loads(actual_lines[1])
+
+        expect = {'hook': 'free', 'device_id': device_id,
+                  'mem_size': unit, 'mem_ptr': ptr1, 'pmem_id': hex(pmem1)}
+        assert expect == json.loads(actual_lines[2])
+
+        expect = {'hook': 'malloc', 'device_id': device_id, 'size': size,
+                  'mem_size': unit, 'mem_ptr': ptr2, 'pmem_id': hex(pmem2)}
+        assert expect == json.loads(actual_lines[3])
+
+        expect = {'hook': 'free', 'device_id': device_id,
+                  'mem_size': unit, 'mem_ptr': ptr2, 'pmem_id': hex(pmem2)}
+        assert expect == json.loads(actual_lines[4])
diff --git a/tests/cupy_tests/cuda_tests/memory_hooks_tests/test_line_profile.py b/tests/cupy_tests/cuda_tests/memory_hooks_tests/test_line_profile.py
new file mode 100644
index 0000000..252bfc8
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/memory_hooks_tests/test_line_profile.py
@@ -0,0 +1,50 @@
+import io
+import unittest
+import re
+
+from cupy.cuda import memory
+from cupy.cuda import memory_hooks
+
+
+class TestLineProfileHook(unittest.TestCase):
+
+    def setUp(self):
+        self.pool = memory.MemoryPool()
+
+    def test_print_report(self):
+        hook = memory_hooks.LineProfileHook()
+        p = self.pool.malloc(1000)
+        del p
+        with hook:
+            p1 = self.pool.malloc(1000)
+            p2 = self.pool.malloc(2000)
+        del p1
+        del p2
+        f = io.StringIO()
+        hook.print_report(file=f)
+        actual = f.getvalue()
+        expect = r'\A_root \(3\.00KB, 2\.00KB\)'
+        assert re.search(expect, actual)
+        expect = r'.*\.py:[0-9]+:test_print_report \(1\.00KB, 0\.00B\)'
+        assert re.search(expect, actual)
+        expect = r'.*\.py:[0-9]+:test_print_report \(2\.00KB, 2\.00KB\)'
+        assert re.search(expect, actual)
+
+    def test_print_report_max_depth(self):
+        hook = memory_hooks.LineProfileHook(max_depth=1)
+        with hook:
+            p = self.pool.malloc(1000)
+        del p
+        f = io.StringIO()
+        hook.print_report(file=f)
+        actual = f.getvalue()
+        assert 2 == len(actual.split('\n'))
+
+        hook = memory_hooks.LineProfileHook(max_depth=2)
+        with hook:
+            p = self.pool.malloc(1000)
+        del p
+        f = io.StringIO()
+        hook.print_report(file=f)
+        actual = f.getvalue()
+        assert 3 == len(actual.split('\n'))
diff --git a/tests/cupy_tests/cuda_tests/test_compiler.py b/tests/cupy_tests/cuda_tests/test_compiler.py
new file mode 100644
index 0000000..7d91190
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_compiler.py
@@ -0,0 +1,136 @@
+import pickle
+import unittest
+from unittest import mock
+
+import pytest
+
+import cupy
+from cupy.cuda import compiler
+
+
+def cuda_version():
+    return cupy.cuda.runtime.runtimeGetVersion()
+
+
+@unittest.skipIf(cupy.cuda.runtime.is_hip, 'CUDA specific tests')
+class TestNvrtcArch(unittest.TestCase):
+    def setUp(self):
+        cupy.clear_memo()  # _get_arch result is cached
+
+    def _check_get_arch(self, device_cc, expected_arch):
+        with mock.patch('cupy.cuda.device.Device') as device_class:
+            device_class.return_value.compute_capability = device_cc
+            assert compiler._get_arch() == expected_arch
+        cupy.clear_memo()  # _get_arch result is cached
+
+    @unittest.skipUnless(9000 <= cuda_version(), 'Requires CUDA 9.x or later')
+    def test_get_arch_cuda9(self):
+        self._check_get_arch('62', '62')  # Tegra
+        self._check_get_arch('70', '70')
+        self._check_get_arch('72', '72')  # Tegra
+
+    @unittest.skipUnless(10010 <= cuda_version(),
+                         'Requires CUDA 10.1 or later')
+    def test_get_arch_cuda101(self):
+        self._check_get_arch('75', '75')
+
+    @unittest.skipUnless(11000 <= cuda_version(),
+                         'Requires CUDA 11.0 or later')
+    def test_get_arch_cuda11(self):
+        self._check_get_arch('80', '80')
+
+    def _compile(self, arch):
+        compiler.compile_using_nvrtc('', arch=arch)
+
+    @unittest.skipUnless(9000 <= cuda_version(), 'Requires CUDA 9.0 or later')
+    def test_compile_cuda9(self):
+        # This test is intended to detect specification change in NVRTC API.
+
+        # It should not fail.
+        # (Do not test `compute_72` as it is for Tegra.)
+        self._compile('70')
+
+        # It should fail.
+        self.assertRaises(
+            compiler.CompileException, self._compile, '73')
+
+    @unittest.skipUnless(10010 <= cuda_version() < 11000,
+                         'Requires CUDA 10.1 or 10.2')
+    def test_compile_cuda101(self):
+        # This test is intended to detect specification change in NVRTC API.
+
+        # It should not fail.
+        # (Do not test `compute_72` as it is for Tegra.)
+        self._compile('75')
+
+        # It should fail. (compute_80 is not supported until CUDA 11)
+        self.assertRaises(
+            compiler.CompileException, self._compile, '80')
+
+    @unittest.skipUnless(11000 <= cuda_version(),
+                         'Requires CUDA 11.0 or later')
+    def test_compile_cuda11(self):
+        # This test is intended to detect specification change in NVRTC API.
+
+        # It should not fail.
+        self._compile('80')
+
+        # It should fail.
+        self.assertRaises(
+            compiler.CompileException, self._compile, '83')
+
+
+class TestNvrtcStderr(unittest.TestCase):
+
+    @unittest.skipIf(cupy.cuda.runtime.is_hip,
+                     'HIPRTC has different error message')
+    def test1(self):
+        # An error message contains the file name `kern.cu`
+        with self.assertRaisesRegex(compiler.CompileException, 'kern.cu'):
+            compiler.compile_using_nvrtc('a')
+
+    @unittest.skipIf(not cupy.cuda.runtime.is_hip,
+                     'NVRTC has different error message')
+    def test2(self):
+        with self.assertRaises(compiler.CompileException) as e:
+            compiler.compile_using_nvrtc('a')
+            assert "unknown type name 'a'" in e
+
+
+class TestIsValidKernelName(unittest.TestCase):
+
+    def test_valid(self):
+        assert compiler.is_valid_kernel_name('valid_name_1')
+
+    def test_empty(self):
+        assert not compiler.is_valid_kernel_name('')
+
+    def test_start_with_digit(self):
+        assert not compiler.is_valid_kernel_name('0_invalid')
+
+    def test_new_line(self):
+        assert not compiler.is_valid_kernel_name('invalid\nname')
+
+    def test_symbol(self):
+        assert not compiler.is_valid_kernel_name('invalid$name')
+
+    def test_space(self):
+        assert not compiler.is_valid_kernel_name('invalid name')
+
+
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = compiler.CompileException('msg', 'fn.cu', 'fn', ('-ftz=true',))
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
+
+
+class TestCompileWithCache:
+    def test_compile_module_with_cache(self):
+        compiler._compile_module_with_cache('__device__ void func() {}')
+
+    def test_deprecated_compile_with_cache(self):
+        with pytest.warns(UserWarning):
+            compiler.compile_with_cache('__device__ void func() {}')
diff --git a/tests/cupy_tests/cuda_tests/test_cublas.py b/tests/cupy_tests/cuda_tests/test_cublas.py
new file mode 100644
index 0000000..dd19eba
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_cublas.py
@@ -0,0 +1,13 @@
+import pickle
+import unittest
+
+from cupy.cuda import cublas
+
+
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = cublas.CUBLASError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_cudnn.py b/tests/cupy_tests/cuda_tests/test_cudnn.py
new file mode 100644
index 0000000..3ee62b2
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_cudnn.py
@@ -0,0 +1,17 @@
+import pickle
+import unittest
+
+from cupy.cuda import cudnn
+
+
+cudnn_available = cudnn.available
+
+
+@unittest.skipUnless(cudnn_available, 'cuDNN is unavailable')
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = cudnn.CuDNNError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_cufft.py b/tests/cupy_tests/cuda_tests/test_cufft.py
new file mode 100644
index 0000000..4cfdac7
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_cufft.py
@@ -0,0 +1,232 @@
+import pickle
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.cuda import cufft
+from cupy.fft import config
+from cupy.fft._fft import _convert_fft_type
+
+from ..fft_tests.test_fft import (multi_gpu_config, _skip_multi_gpu_bug)
+
+
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = cufft.CuFFTError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
+
+
+# This class tests multi-GPU Plan1d with data sitting on host.
+# Internally, we use cuFFT's data transfer API to ensure the
+# data is in order.
+
+@testing.parameterize(*testing.product({
+    'shape': [(64,), (4, 16), (128,), (8, 32)],
+}))
+@testing.multi_gpu(2)
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason='not supported by hipFFT')
+class TestMultiGpuPlan1dNumPy(unittest.TestCase):
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    def test_fft(self, dtype):
+        _skip_multi_gpu_bug(self.shape, self.gpus)
+
+        a = testing.shaped_random(self.shape, numpy, dtype)
+
+        if len(self.shape) == 1:
+            batch = 1
+            nx = self.shape[0]
+        elif len(self.shape) == 2:
+            batch = self.shape[0]
+            nx = self.shape[1]
+
+        # compute via cuFFT
+        cufft_type = _convert_fft_type(a.dtype, 'C2C')
+        plan = cufft.Plan1d(nx, cufft_type, batch, devices=config._devices)
+        out_cp = numpy.empty_like(a)
+        plan.fft(a, out_cp, cufft.CUFFT_FORWARD)
+
+        out_np = numpy.fft.fft(a)
+        # np.fft.fft alway returns np.complex128
+        if dtype is numpy.complex64:
+            out_np = out_np.astype(dtype)
+
+        assert numpy.allclose(out_cp, out_np, rtol=1e-4, atol=1e-7)
+
+        # compute it again to ensure Plan1d's internal state is reset
+        plan.fft(a, out_cp, cufft.CUFFT_FORWARD)
+
+        assert numpy.allclose(out_cp, out_np, rtol=1e-4, atol=1e-7)
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    def test_ifft(self, dtype):
+        _skip_multi_gpu_bug(self.shape, self.gpus)
+
+        a = testing.shaped_random(self.shape, numpy, dtype)
+
+        if len(self.shape) == 1:
+            batch = 1
+            nx = self.shape[0]
+        elif len(self.shape) == 2:
+            batch = self.shape[0]
+            nx = self.shape[1]
+
+        # compute via cuFFT
+        cufft_type = _convert_fft_type(a.dtype, 'C2C')
+        plan = cufft.Plan1d(nx, cufft_type, batch, devices=config._devices)
+        out_cp = numpy.empty_like(a)
+        plan.fft(a, out_cp, cufft.CUFFT_INVERSE)
+        # normalization
+        out_cp /= nx
+
+        out_np = numpy.fft.ifft(a)
+        # np.fft.fft alway returns np.complex128
+        if dtype is numpy.complex64:
+            out_np = out_np.astype(dtype)
+
+        assert numpy.allclose(out_cp, out_np, rtol=1e-4, atol=1e-7)
+
+        # compute it again to ensure Plan1d's internal state is reset
+        plan.fft(a, out_cp, cufft.CUFFT_INVERSE)
+        # normalization
+        out_cp /= nx
+
+        assert numpy.allclose(out_cp, out_np, rtol=1e-4, atol=1e-7)
+
+
+# This class contains a few simple tests for the new XtPlanNd wrapper. A full-
+# scale test will be done (with little cost) when this low-level API is
+# integrated with the high-level APIs such as cupy.fft.fft() (see #4406), so
+# don't make our life too hard here. Note that we don't have cupy.complex32
+# yet, so its dtype is represented by a single character 'E' (in contrast to
+# 'e' for float16) for the time being (see #3370).
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 16), (4, 16, 16)],
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason='not supported by hipFFT')
+class TestXtPlanNd(unittest.TestCase):
+
+    @testing.for_complex_dtypes()
+    def test_forward_fft(self, dtype):
+        t = dtype
+        idtype = odtype = edtype = cupy.dtype(t)
+        shape = self.shape
+        length = cupy._core.internal.prod(shape[1:])
+
+        a = testing.shaped_random(shape, cupy, dtype)
+        out = cupy.empty_like(a)
+
+        plan = cufft.XtPlanNd(shape[1:],
+                              shape[1:], 1, length, idtype,
+                              shape[1:], 1, length, odtype,
+                              shape[0], edtype,
+                              order='C', last_axis=-1, last_size=None)
+        plan.fft(a, out, cufft.CUFFT_FORWARD)
+
+        if len(shape) <= 2:
+            out_cp = cupy.fft.fft(a)
+        else:
+            out_cp = cupy.fft.fftn(a, axes=(-1, -2))
+        testing.assert_allclose(out, out_cp)
+
+    @testing.for_complex_dtypes()
+    def test_backward_fft(self, dtype):
+        t = dtype
+        idtype = odtype = edtype = cupy.dtype(t)
+        shape = self.shape
+        length = cupy._core.internal.prod(shape[1:])
+
+        a = testing.shaped_random(shape, cupy, dtype)
+        out = cupy.empty_like(a)
+
+        plan = cufft.XtPlanNd(shape[1:],
+                              shape[1:], 1, length, idtype,
+                              shape[1:], 1, length, odtype,
+                              shape[0], edtype,
+                              order='C', last_axis=-1, last_size=None)
+        plan.fft(a, out, cufft.CUFFT_INVERSE)
+
+        if len(shape) <= 2:
+            out_cp = cupy.fft.ifft(a)
+        else:
+            out_cp = cupy.fft.ifftn(a, axes=(-1, -2))
+        testing.assert_allclose(out/length, out_cp)
+
+    @pytest.mark.skipif(int(cupy.cuda.device.get_compute_capability()) < 53,
+                        reason='half-precision complex FFT is not supported')
+    def test_forward_fft_complex32(self):
+        t = 'E'
+        idtype = odtype = edtype = t
+        old_shape = self.shape
+        shape = list(self.shape)
+        shape[-1] = 2*shape[-1]
+        shape = tuple(shape)
+
+        a = testing.shaped_random(shape, cupy, cupy.float16)
+        out = cupy.empty_like(a)
+
+        shape = old_shape
+        length = cupy._core.internal.prod(shape[1:])
+        plan = cufft.XtPlanNd(shape[1:],
+                              shape[1:], 1, length, idtype,
+                              shape[1:], 1, length, odtype,
+                              shape[0], edtype,
+                              order='C', last_axis=-1, last_size=None)
+        plan.fft(a, out, cufft.CUFFT_FORWARD)
+
+        a_cp = a.astype(cupy.float32)  # upcast
+        a_cp = a_cp.view(cupy.complex64)
+        if len(shape) <= 2:
+            out_cp = cupy.fft.fft(a_cp)
+        else:
+            out_cp = cupy.fft.fftn(a_cp, axes=(-1, -2))
+        out_cp = out_cp.view(cupy.float32)
+        out_cp = out_cp.astype(cupy.float16)  # downcast
+
+        # We shouldn't expect getting high accuracy, as both computations have
+        # precision losses...
+        testing.assert_allclose(out, out_cp, rtol=1E-1, atol=1E-1)
+
+    @pytest.mark.skipif(int(cupy.cuda.device.get_compute_capability()) < 53,
+                        reason='half-precision complex FFT is not supported')
+    def test_backward_fft_complex32(self):
+        t = 'E'
+        idtype = odtype = edtype = t
+        old_shape = self.shape
+        shape = list(self.shape)
+        shape[-1] = 2*shape[-1]
+        shape = tuple(shape)
+
+        a = testing.shaped_random(shape, cupy, cupy.float16)
+        out = cupy.empty_like(a)
+
+        shape = old_shape
+        length = cupy._core.internal.prod(shape[1:])
+        plan = cufft.XtPlanNd(shape[1:],
+                              shape[1:], 1, length, idtype,
+                              shape[1:], 1, length, odtype,
+                              shape[0], edtype,
+                              order='C', last_axis=-1, last_size=None)
+        plan.fft(a, out, cufft.CUFFT_INVERSE)
+
+        a_cp = a.astype(cupy.float32)  # upcast
+        a_cp = a_cp.view(cupy.complex64)
+        if len(shape) <= 2:
+            out_cp = cupy.fft.ifft(a_cp)
+        else:
+            out_cp = cupy.fft.ifftn(a_cp, axes=(-1, -2))
+        out_cp = out_cp.view(cupy.float32)
+        out_cp = out_cp.astype(cupy.float16)  # downcast
+
+        # We shouldn't expect getting high accuracy, as both computations have
+        # precision losses...
+        testing.assert_allclose(out/length, out_cp, rtol=1E-1, atol=1E-1)
diff --git a/tests/cupy_tests/cuda_tests/test_curand.py b/tests/cupy_tests/cuda_tests/test_curand.py
new file mode 100644
index 0000000..44c36f4
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_curand.py
@@ -0,0 +1,47 @@
+import pickle
+import unittest
+
+import numpy
+
+import cupy
+from cupy.cuda import curand
+
+
+class TestGenerateNormal(unittest.TestCase):
+
+    def setUp(self):
+        self.generator = curand.createGenerator(
+            curand.CURAND_RNG_PSEUDO_DEFAULT)
+
+    def test_invalid_argument_normal_float(self):
+        out = cupy.empty((1,), dtype=numpy.float32)
+        with self.assertRaises(ValueError):
+            curand.generateNormal(
+                self.generator, out.data.ptr, 1, 0.0, 1.0)
+
+    def test_invalid_argument_normal_double(self):
+        out = cupy.empty((1,), dtype=numpy.float64)
+        with self.assertRaises(ValueError):
+            curand.generateNormalDouble(
+                self.generator, out.data.ptr, 1, 0.0, 1.0)
+
+    def test_invalid_argument_log_normal_float(self):
+        out = cupy.empty((1,), dtype=numpy.float32)
+        with self.assertRaises(ValueError):
+            curand.generateLogNormal(
+                self.generator, out.data.ptr, 1, 0.0, 1.0)
+
+    def test_invalid_argument_log_normal_double(self):
+        out = cupy.empty((1,), dtype=numpy.float64)
+        with self.assertRaises(ValueError):
+            curand.generateLogNormalDouble(
+                self.generator, out.data.ptr, 1, 0.0, 1.0)
+
+
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = curand.CURANDError(100)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_cusolver.py b/tests/cupy_tests/cuda_tests/test_cusolver.py
new file mode 100644
index 0000000..d2859be
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_cusolver.py
@@ -0,0 +1,13 @@
+import pickle
+import unittest
+
+from cupy import cuda
+
+
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = cuda.cusolver.CUSOLVERError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_cusparse.py b/tests/cupy_tests/cuda_tests/test_cusparse.py
new file mode 100644
index 0000000..e6299e0
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_cusparse.py
@@ -0,0 +1,27 @@
+import pickle
+import unittest
+
+import cupy
+from cupy.cuda import cusparse
+
+
+class TestException(unittest.TestCase):
+
+    def test_error_message(self):
+        e = cusparse.CuSparseError(1)
+        if cupy.cuda.runtime.is_hip:
+            assert str(e) == (
+                'HIPSPARSE_STATUS_NOT_INITIALIZED: '
+                'HIPSPARSE_STATUS_NOT_INITIALIZED'
+            )
+        else:
+            assert str(e) == (
+                'CUSPARSE_STATUS_NOT_INITIALIZED: '
+                'initialization error'
+            )
+
+    def test_pickle(self):
+        e1 = cusparse.CuSparseError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_cutensor.py b/tests/cupy_tests/cuda_tests/test_cutensor.py
new file mode 100644
index 0000000..437d194
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_cutensor.py
@@ -0,0 +1,14 @@
+import pickle
+import unittest
+
+from cupy.cuda import cutensor
+
+
+@unittest.skipUnless(cutensor.available, 'cuTensor is unavailable')
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = cutensor.CuTensorError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_device.py b/tests/cupy_tests/cuda_tests/test_device.py
new file mode 100644
index 0000000..93a40c3
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_device.py
@@ -0,0 +1,238 @@
+import re
+import threading
+import unittest
+
+import pytest
+
+import cupy
+from cupy import cuda
+from cupy import testing
+
+
+class TestDeviceComparison(unittest.TestCase):
+
+    def check_eq(self, result, obj1, obj2):
+        if result:
+            assert obj1 == obj2
+            assert obj2 == obj1
+            assert not (obj1 != obj2)
+            assert not (obj2 != obj1)
+        else:
+            assert obj1 != obj2
+            assert obj2 != obj1
+            assert not (obj1 == obj2)
+            assert not (obj2 == obj1)
+
+    def test_equality(self):
+        self.check_eq(True, cuda.Device(0), cuda.Device(0))
+        self.check_eq(True, cuda.Device(1), cuda.Device(1))
+        self.check_eq(False, cuda.Device(0), cuda.Device(1))
+        self.check_eq(False, cuda.Device(0), 0)
+        self.check_eq(False, cuda.Device(0), None)
+        self.check_eq(False, cuda.Device(0), object())
+
+    def test_lt_device(self):
+        assert cuda.Device(0) < cuda.Device(1)
+        assert not (cuda.Device(0) < cuda.Device(0))
+        assert not (cuda.Device(1) < cuda.Device(0))
+
+    def test_le_device(self):
+        assert cuda.Device(0) <= cuda.Device(1)
+        assert cuda.Device(0) <= cuda.Device(0)
+        assert not (cuda.Device(1) <= cuda.Device(0))
+
+    def test_gt_device(self):
+        assert not (cuda.Device(0) > cuda.Device(0))
+        assert not (cuda.Device(0) > cuda.Device(0))
+        assert cuda.Device(1) > cuda.Device(0)
+
+    def test_ge_device(self):
+        assert not (cuda.Device(0) >= cuda.Device(1))
+        assert cuda.Device(0) >= cuda.Device(0)
+        assert cuda.Device(1) >= cuda.Device(0)
+
+    def check_comparison_other_type(self, obj1, obj2):
+        with pytest.raises(TypeError):
+            obj1 < obj2
+        with pytest.raises(TypeError):
+            obj1 <= obj2
+        with pytest.raises(TypeError):
+            obj1 > obj2
+        with pytest.raises(TypeError):
+            obj1 >= obj2
+        with pytest.raises(TypeError):
+            obj2 < obj1
+        with pytest.raises(TypeError):
+            obj2 <= obj1
+        with pytest.raises(TypeError):
+            obj2 > obj1
+        with pytest.raises(TypeError):
+            obj2 >= obj1
+
+    def test_comparison_other_type(self):
+        self.check_comparison_other_type(cuda.Device(0), 0)
+        self.check_comparison_other_type(cuda.Device(0), 1)
+        self.check_comparison_other_type(cuda.Device(1), 0)
+        self.check_comparison_other_type(cuda.Device(1), None)
+        self.check_comparison_other_type(cuda.Device(1), object())
+
+
+class TestDeviceAttributes(unittest.TestCase):
+
+    def test_device_attributes(self):
+        d = cuda.Device()
+        attributes = d.attributes
+        assert isinstance(attributes, dict)
+        assert all(isinstance(a, int) for a in attributes.values())
+        # test a specific attribute that would be present on any supported GPU
+        assert 'MaxThreadsPerBlock' in attributes
+
+    def test_device_attributes_error(self):
+        with pytest.raises(cuda.runtime.CUDARuntimeError):
+            # try to retrieve attributes from a non-existent device
+            cuda.device.Device(cuda.runtime.getDeviceCount()).attributes
+
+
+class TestDevicePCIBusId(unittest.TestCase):
+    def test_device_get_pci_bus_id(self):
+        d = cuda.Device()
+        pci_bus_id = d.pci_bus_id
+        assert re.match(
+            '^[a-fA-F0-9]{4}:[a-fA-F0-9]{2}:[a-fA-F0-9]{2}.[a-fA-F0-9]',
+            pci_bus_id
+        )
+
+    def test_device_by_pci_bus_id(self):
+        d1 = cuda.Device()
+        d2 = cuda.Device.from_pci_bus_id(d1.pci_bus_id)
+        assert d1 == d2
+        d3 = cuda.Device(d2)
+        assert d2 == d3
+
+        with pytest.raises(cuda.runtime.CUDARuntimeError) as excinfo:
+            cuda.Device.from_pci_bus_id('fake:id')
+            assert excinfo == 'cudaErrorInvalidValue: invalid argument'
+
+        with pytest.raises(cuda.runtime.CUDARuntimeError) as excinfo:
+            cuda.Device.from_pci_bus_id('FFFF:FF:FF.F')
+            assert excinfo == 'cudaErrorInvalidDevice: invalid device ordinal'
+
+
+class TestDeviceHandles(unittest.TestCase):
+    def _check_handle(self, func):
+        handles = [func(), None, None]
+
+        def _subthread():
+            cupy.cuda.Device().use()
+            handles[1] = func()
+            handles[2] = func()
+
+        t = threading.Thread(target=_subthread)
+        t.start()
+        t.join()
+        assert handles[0] is not None
+        assert handles[0] != handles[1]
+        assert handles[1] == handles[2]
+
+    def test_cublas_handle(self):
+        self._check_handle(cuda.get_cublas_handle)
+
+    def test_cusolver_handle(self):
+        self._check_handle(cuda.device.get_cusolver_handle)
+
+    def test_cusolver_sp_handle(self):
+        self._check_handle(cuda.device.get_cublas_handle)
+
+    def test_cusparse_handle(self):
+        self._check_handle(cuda.device.get_cusparse_handle)
+
+
+class TestDeviceFromPointer(unittest.TestCase):
+    def test_from_pointer(self):
+        assert cuda.device.from_pointer(cupy.empty(1).data.ptr).id == 0
+
+
+@testing.multi_gpu(2)
+class TestDeviceSwitch(unittest.TestCase):
+
+    def setUp(self):
+        self._prev_device = cuda.Device()
+
+    def tearDown(self):
+        self._prev_device.use()
+
+    def test_use(self):
+        dev0 = cuda.Device(0)
+        dev1 = cuda.Device(1)
+
+        assert dev1.use() is dev1
+        assert 1 == cuda.Device().id
+        assert dev0.use() is dev0
+        assert 0 == cuda.Device().id
+
+    def test_context(self):
+        dev0 = cuda.Device(0)
+        dev1 = cuda.Device(1)
+
+        with dev0:
+            assert 0 == cuda.Device().id
+            with dev1:
+                assert 1 == cuda.Device().id
+            assert 0 == cuda.Device().id
+
+    def test_context_and_use(self):
+        dev0 = cuda.Device(0)
+        dev1 = cuda.Device(1)
+
+        dev1.use()
+        with dev0:
+            assert 0 == cuda.Device().id
+            dev1.use()
+            with dev1:
+                assert 1 == cuda.Device().id
+            assert 1 == cuda.Device().id
+        assert 1 == cuda.Device().id
+
+    def test_thread_safe(self):
+        dev0 = cuda.Device(0)
+        dev1 = cuda.Device(1)
+
+        t0_setup = threading.Event()
+        t1_setup = threading.Event()
+        t0_first_exit = threading.Event()
+
+        t0_exit_device = []
+        t1_exit_device = []
+
+        def t0_seq():
+            with dev0:
+                with dev0:
+                    t0_setup.set()
+                    t1_setup.wait()
+                t0_exit_device.append(cuda.Device().id)
+                t0_first_exit.set()
+
+        def t1_seq():
+            t0_setup.wait()
+            with dev1:
+                with dev0:
+                    t1_setup.set()
+                    t0_first_exit.wait()
+                t1_exit_device.append(cuda.Device().id)
+
+        t1 = threading.Thread(target=t1_seq)
+        t1.start()
+        t0_seq()
+        t1.join()
+        assert t0_exit_device[0] == 0
+        assert t1_exit_device[0] == 1
+
+    def test_invalid(self):
+        d = cuda.Device(100)
+        with pytest.raises(cuda.runtime.CUDARuntimeError):
+            d.use()
+        with pytest.raises(cuda.runtime.CUDARuntimeError):
+            with d:
+                pass
+        with cuda.Device(0):
+            pass
diff --git a/tests/cupy_tests/cuda_tests/test_driver.py b/tests/cupy_tests/cuda_tests/test_driver.py
new file mode 100644
index 0000000..88cb8a9
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_driver.py
@@ -0,0 +1,64 @@
+import pickle
+import threading
+import unittest
+
+import cupy
+from cupy import testing
+from cupy.cuda import driver
+
+
+@unittest.skipIf(cupy.cuda.runtime.is_hip, 'Context API is dperecated in HIP')
+class TestDriver(unittest.TestCase):
+    def test_ctxGetCurrent(self):
+        # Make sure to create context.
+        cupy.arange(1)
+        assert 0 != driver.ctxGetCurrent()
+
+    def test_ctxGetCurrent_thread(self):
+        # Make sure to create context in main thread.
+        cupy.arange(1)
+
+        def f(self):
+            self._result0 = driver.ctxGetCurrent()
+            cupy.cuda.Device().use()
+            cupy.arange(1)
+            self._result1 = driver.ctxGetCurrent()
+
+        self._result0 = None
+        self._result1 = None
+        t = threading.Thread(target=f, args=(self,))
+        t.daemon = True
+        t.start()
+        t.join()
+
+        # The returned context pointer must be NULL on sub thread
+        # without valid context.
+        assert 0 == self._result0
+
+        # After the context is created, it should return the valid
+        # context pointer.
+        assert 0 != self._result1
+
+    @testing.multi_gpu(2)
+    def test_ctxGetDevice(self):
+        with cupy.cuda.Device(1):
+            dev = driver.ctxGetDevice()
+            assert dev == 1
+        with cupy.cuda.Device(0):
+            dev = driver.ctxGetDevice()
+            assert dev == 0
+
+    def test_streamGetCtx(self):
+        s = cupy.cuda.Stream()
+        ctx = driver.streamGetCtx(s.ptr)
+        ctx2 = driver.ctxGetCurrent()
+        assert ctx == ctx2
+
+
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = driver.CUDADriverError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_graph.py b/tests/cupy_tests/cuda_tests/test_graph.py
new file mode 100644
index 0000000..6d70f85
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_graph.py
@@ -0,0 +1,345 @@
+import pytest
+
+import cupy
+from cupy import cuda
+from cupy import testing
+import cupyx
+
+
+@pytest.mark.skipif(cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+@pytest.mark.skipif(cuda.driver.get_build_version() < 10010,
+                    reason='Only CUDA 10.1+ supports this')
+class TestGraph:
+
+    def _helper1(self, a):
+        # this tests ufuncs involving simple arithmetic
+        a = a + 3
+        a = a * 7.6
+        return a**2
+
+    def _helper2(self, a):
+        # this tests ufuncs involving math API calls
+        a = 3 * cupy.sin(a)
+        return cupy.sqrt(a)
+
+    def _helper3(self, a):
+        # this tests CUDA library calls
+        a = a * cupy.fft.fft(a)
+        return cupy.fft.ifft(a)
+
+    def _helper4(self, a):
+        # this tests a common pattern in CuPy internal in which the host
+        # operation depends on intermediate outcome on GPU (and thus requires
+        # synchronization)
+        result = cupy.zeros((1,), dtype=cupy.int32)
+        if a.sum() > 0:  # synchronize!
+            result += 1
+        if a[-1] >= 0:  # synchronize!
+            result += 2
+        return result
+
+    @pytest.mark.parametrize('upload', (True, False))
+    def test_capture_run_on_same_stream(self, upload):
+        s = cupy.cuda.Stream(non_blocking=True)
+
+        for n in range(3):
+            func = getattr(self, '_helper{}'.format(n+1))
+            a = cupy.random.random((100,))
+
+            with s:
+                s.begin_capture()
+                out1 = func(a)
+                g = s.end_capture()
+                if upload and cuda.runtime.runtimeGetVersion() >= 11010:
+                    g.upload()
+                g.launch()
+            s.synchronize()
+
+            out2 = func(a)
+            testing.assert_array_equal(out1, out2)
+
+    @pytest.mark.parametrize('upload', (True, False))
+    def test_capture_run_on_different_streams(self, upload):
+        s1 = cupy.cuda.Stream(non_blocking=True)
+        s2 = cupy.cuda.Stream(non_blocking=True)
+
+        for n in range(3):
+            func = getattr(self, '_helper{}'.format(n+1))
+            a = cupy.random.random((100,))
+
+            with s1:
+                s1.begin_capture()
+                out1 = func(a)
+                g = s1.end_capture()
+            with s2:
+                if upload and cuda.runtime.runtimeGetVersion() >= 11010:
+                    g.upload()
+                g.launch()
+            s2.synchronize()
+
+            out2 = func(a)
+            testing.assert_array_equal(out1, out2)
+
+    @pytest.mark.parametrize('upload', (True, False))
+    def test_stream_is_capturing(self, upload):
+        s = cupy.cuda.Stream(non_blocking=True)
+        a = cupy.random.random((100,))
+
+        with s:
+            s.begin_capture()
+            assert s.is_capturing()
+            assert not cuda.Stream.null.is_capturing()
+            b = a * 3
+            g = s.end_capture()
+        assert not s.is_capturing()
+        assert not cuda.Stream.null.is_capturing()
+
+        # check the graph integrity
+        if upload and cuda.runtime.runtimeGetVersion() >= 11010:
+            g.upload()
+        g.launch()
+        s.synchronize()
+        testing.assert_array_equal(b, 3 * a)
+
+    @pytest.mark.parametrize('upload', (True, False))
+    def test_stream_fork_join(self, upload):
+        s1 = cupy.cuda.Stream(non_blocking=True)
+        s2 = cupy.cuda.Stream(non_blocking=True)
+        e1 = cupy.cuda.Event()
+        e2 = cupy.cuda.Event()
+        a = cupy.random.random((100,))
+
+        def func(x):
+            return 3 * x + 1
+
+        with s1:
+            s1.begin_capture()
+            out1 = a * 100
+            e1.record(s1)
+            s2.wait_event(e1)
+            with s2:
+                out2 = func(out1)
+                e2.record(s2)
+            s1.wait_event(e2)
+            g = s1.end_capture()
+
+        # check integrity
+        assert not s1.is_capturing()
+        assert not s2.is_capturing()
+        if upload and cuda.runtime.runtimeGetVersion() >= 11010:
+            g.upload()
+        g.launch()
+        s1.synchronize()
+        testing.assert_array_equal(out2, func(a * 100))
+
+    @pytest.mark.parametrize('upload', (True, False))
+    def test_null_stream_cannot_capture(self, upload):
+        s = cupy.cuda.Stream(non_blocking=False)
+        a = cupy.random.random((100,))
+
+        with s:
+            s.begin_capture()
+            b = a + 4
+            assert s.is_capturing()
+            # cudaStreamLegacy is unhappy when a blocking stream is capturing
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                cuda.Stream.null.is_capturing()
+            assert 'cudaErrorStreamCaptureImplicit' in str(e.value)
+            g = s.end_capture()
+        assert not s.is_capturing()
+        assert not cuda.Stream.null.is_capturing()
+
+        # check the graph integrity
+        if upload and cuda.runtime.runtimeGetVersion() >= 11010:
+            g.upload()
+        g.launch()
+        s.synchronize()
+        testing.assert_array_equal(b, a + 4)
+
+    def test_stream_capture_failure1(self):
+        s = cupy.cuda.Stream(non_blocking=True)
+
+        with s:
+            s.begin_capture()
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                s.synchronize()
+            assert 'cudaErrorStreamCaptureUnsupported' in str(e.value)
+            # invalid operation causes the capture sequence to be invalidated
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                g = s.end_capture()  # noqa
+            assert 'cudaErrorStreamCaptureInvalidated' in str(e.value)
+
+        # check s left the capture mode and permits normal usage
+        assert not s.is_capturing()
+        s.synchronize()
+
+    def test_stream_capture_failure2(self):
+        s1 = cupy.cuda.Stream(non_blocking=True)
+        s2 = cupy.cuda.Stream(non_blocking=True)
+        e2 = cupy.cuda.Event()
+        a = cupy.random.random((100,))
+
+        with s1:
+            s1.begin_capture()
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                g = s2.end_capture()
+            assert 'cudaErrorIllegalState' in str(e.value)
+            e2.record(s1)
+            s2.wait_event(e2)
+            with s2:
+                b = a**3  # noqa
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                g = s2.end_capture()
+            assert 'cudaErrorStreamCaptureUnmatched' in str(e.value)
+            # invalid operation causes the capture sequence to be invalidated
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                g = s1.end_capture()  # noqa
+            assert 'cudaErrorStreamCaptureInvalidated' in str(e.value)
+
+        # check both s1 and s2 left the capture mode and permit normal usage
+        assert not s1.is_capturing()
+        assert not s2.is_capturing()
+        s1.synchronize()
+        s2.synchronize()
+
+    def test_stream_capture_failure3(self):
+        s1 = cupy.cuda.Stream(non_blocking=True)
+        s2 = cupy.cuda.Stream(non_blocking=True)
+        e2 = cupy.cuda.Event()
+        a = cupy.random.random((100,))
+
+        with s1:
+            s1.begin_capture()
+            e2.record(s1)
+            s2.wait_event(e2)
+            with s2:
+                # internally the function requires synchronization, which is
+                # incompatible with stream capturing and so we raise
+                with pytest.raises(RuntimeError) as e:
+                    b = cupy.where(a > 0.5)  # noqa
+                assert 'is capturing' in str(e.value)
+            # invalid operation causes the capture sequence to be invalidated
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                g = s1.end_capture()  # noqa
+            assert 'cudaErrorStreamCaptureUnjoined' in str(e.value)
+
+        # check both s1 and s2 left the capture mode and permit normal usage
+        assert not s1.is_capturing()
+        assert not s2.is_capturing()
+        s1.synchronize()
+        s2.synchronize()
+
+    def test_stream_capture_failure4(self):
+        s = cupy.cuda.Stream(non_blocking=True)
+
+        with s:
+            s.begin_capture()
+            # query the stream status is illegal during capturing
+            s.done
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                s.end_capture()
+            assert 'cudaErrorStreamCaptureInvalidated' in str(e.value)
+
+        # check s left the capture mode and permits normal usage
+        assert not s.is_capturing()
+        s.synchronize()
+
+    def test_stream_capture_failure5(self):
+        s = cupy.cuda.Stream(non_blocking=True)
+        func = self._helper4
+        a = cupy.random.random((100,))
+
+        with s:
+            s.begin_capture()
+            # internally the function requires synchronization, which is
+            # incompatible with stream capturing and so we raise
+            with pytest.raises(RuntimeError) as e:
+                func(a)
+            assert 'is capturing' in str(e.value)
+            s.end_capture()
+
+        # check s left the capture mode and permits normal usage
+        assert not s.is_capturing()
+        s.synchronize()
+
+    def test_stream_capture_failure6(self):
+        s = cupy.cuda.Stream(non_blocking=True)
+
+        with s:
+            s.begin_capture()
+            # synchronize the stream is illegal during capturing
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                s.synchronize()
+            assert 'cudaErrorStreamCaptureUnsupported' in str(e.value)
+            with pytest.raises(cuda.runtime.CUDARuntimeError) as e:
+                s.end_capture()
+            assert 'cudaErrorStreamCaptureInvalidated' in str(e.value)
+
+        # check s left the capture mode and permits normal usage
+        assert not s.is_capturing()
+        s.synchronize()
+
+    def test_stream_capture_failure_cublas(self):
+        s = cupy.cuda.Stream(non_blocking=True)
+        a = cupy.random.random((3, 4))
+        b = cupy.random.random((4, 5))
+
+        with s:
+            s.begin_capture()
+            with pytest.raises(NotImplementedError) as e:
+                cupy.matmul(a, b)
+            assert 'cuBLAS' in str(e.value)
+            s.end_capture()
+
+        # check s left the capture mode and permits normal usage
+        assert not s.is_capturing()
+        s.synchronize()
+
+    def test_stream_capture_failure_cusolver(self):
+        s = cupy.cuda.Stream(non_blocking=True)
+        a = cupy.random.random((8, 8))
+        a += a.T
+
+        with s:
+            s.begin_capture()
+            with pytest.raises(NotImplementedError) as e:
+                cupy.linalg.svd(a)
+            assert 'cuSOLVER' in str(e.value)
+            s.end_capture()
+
+        # check s left the capture mode and permits normal usage
+        assert not s.is_capturing()
+        s.synchronize()
+
+    def test_stream_capture_failure_curand(self):
+        s = cupy.cuda.Stream(non_blocking=True)
+
+        with s:
+            s.begin_capture()
+            with pytest.raises(NotImplementedError) as e:
+                cupy.random.random(100)
+            assert 'cuRAND' in str(e.value)
+            s.end_capture()
+
+        # check s left the capture mode and permits normal usage
+        assert not s.is_capturing()
+        s.synchronize()
+
+    def test_stream_capture_failure_cusparse(self):
+        s = cupy.cuda.Stream(non_blocking=True)
+        a = cupy.zeros((3, 4))
+        a[0] = 1
+        a = cupyx.scipy.sparse.csr_matrix(a)
+        a.has_canonical_format  # avoid launching custom kernels during capture
+
+        with s:
+            s.begin_capture()
+            with pytest.raises(NotImplementedError) as e:
+                a * a.T
+            assert 'cuSPARSE' in str(e.value)
+            s.end_capture()
+
+        # check s left the capture mode and permits normal usage
+        assert not s.is_capturing()
+        s.synchronize()
diff --git a/tests/cupy_tests/cuda_tests/test_memory.py b/tests/cupy_tests/cuda_tests/test_memory.py
new file mode 100644
index 0000000..9111acb
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_memory.py
@@ -0,0 +1,1341 @@
+import ctypes
+import gc
+import pickle
+import threading
+import unittest
+
+import fastrlock
+import pytest
+
+import cupy.cuda
+from cupy.cuda import device
+from cupy.cuda import memory
+from cupy.cuda import runtime
+from cupy.cuda import stream as stream_module
+from cupy import testing
+
+
+class MockMemory(memory.Memory):
+    cur_ptr = 1
+
+    def __init__(self, size):
+        self.ptr = MockMemory.cur_ptr
+        MockMemory.cur_ptr += size
+        self.size = size
+        self.device_id = 0
+
+    def __del__(self):
+        self.ptr = 0
+        pass
+
+
+def mock_alloc(size):
+    mem = MockMemory(size)
+    return memory.MemoryPointer(mem, 0)
+
+
+class TestUnownedMemoryClass(unittest.TestCase):
+
+    def test_inherits_base_memory(self):
+        assert issubclass(memory.UnownedMemory, memory.BaseMemory)
+
+
+@testing.parameterize(*testing.product({
+    'allocator': [memory._malloc, memory.malloc_managed, memory.malloc_async],
+    'specify_device_id': [True, False],
+}))
+class TestUnownedMemory(unittest.TestCase):
+
+    def check(self, device_id):
+        if cupy.cuda.runtime.is_hip:
+            if self.allocator is memory.malloc_managed:
+                if cupy.cuda.driver.get_build_version() < 40300000:
+                    raise unittest.SkipTest(
+                        'Managed memory requires ROCm 4.3+')
+                else:
+                    raise unittest.SkipTest(
+                        'hipPointerGetAttributes does not support managed '
+                        'memory')
+            if self.allocator is memory.malloc_async:
+                raise unittest.SkipTest('HIP does not support async mempool')
+        else:
+            if self.allocator is memory.malloc_async:
+                if cupy.cuda.driver._is_cuda_python():
+                    version = cupy.cuda.runtime.runtimeGetVersion()
+                else:
+                    version = cupy.cuda.driver.get_build_version()
+                if version < 11020:
+                    raise unittest.SkipTest('malloc_async is supported since '
+                                            'CUDA 11.2')
+                elif runtime.deviceGetAttribute(
+                        runtime.cudaDevAttrMemoryPoolsSupported, 0) == 0:
+                    raise unittest.SkipTest(
+                        'malloc_async is not supported on device 0')
+
+        size = 24
+        shape = (2, 3)
+        dtype = cupy.float32
+        with device.Device(device_id):
+            src_mem_ptr = self.allocator(size)
+        src_ptr = src_mem_ptr.ptr
+
+        args = (src_ptr, size, src_mem_ptr)
+        kwargs = {}
+        if self.specify_device_id:
+            kwargs = {'device_id': device_id}
+
+        if cupy.cuda.runtime.is_hip and self.allocator is memory._malloc:
+            # In ROCm, it seems that `hipPointerGetAttributes()`, which is
+            # called in `UnownedMemory()`, requires an unmanaged device pointer
+            # that the current device must be the one on which the memory
+            # referred to by the pointer physically resides.
+            with device.Device(device_id):
+                unowned_mem = memory.UnownedMemory(*args, **kwargs)
+        else:
+            unowned_mem = memory.UnownedMemory(*args, **kwargs)
+        assert unowned_mem.size == size
+        assert unowned_mem.ptr == src_ptr
+        assert unowned_mem.device_id == device_id
+
+        arr = cupy.ndarray(shape, dtype, memory.MemoryPointer(unowned_mem, 0))
+
+        # Delete the source object
+        del src_mem_ptr
+
+        with device.Device(device_id):
+            arr[:] = 2
+            assert (arr == 2).all()
+
+    def test_device0(self):
+        self.check(0)
+
+    @testing.multi_gpu(2)
+    def test_device1(self):
+        self.check(1)
+
+
+class TestMemoryPointer(unittest.TestCase):
+
+    def test_int(self):
+        pval = MockMemory.cur_ptr
+        memptr = mock_alloc(1)
+        assert pval == int(memptr)
+
+    def test_add(self):
+        pval = MockMemory.cur_ptr
+        memptr = mock_alloc(8)
+
+        memptr2 = memptr + 4
+        assert isinstance(memptr2, memory.MemoryPointer)
+        assert pval + 4 == int(memptr2)
+
+        memptr3 = 4 + memptr
+        assert isinstance(memptr3, memory.MemoryPointer)
+        assert pval + 4 == int(memptr3)
+
+        memptr += 4
+        assert isinstance(memptr, memory.MemoryPointer)
+        assert pval + 4 == int(memptr)
+
+    def test_sub(self):
+        pval = MockMemory.cur_ptr
+        memptr = mock_alloc(8) + 4
+
+        memptr2 = memptr - 4
+        assert isinstance(memptr2, memory.MemoryPointer)
+        assert pval == int(memptr2)
+
+        memptr -= 4
+        assert isinstance(memptr, memory.MemoryPointer)
+        assert pval == int(memptr)
+
+    def test_copy_to_and_from_host(self):
+        a_gpu = memory.alloc(4)
+        a_cpu = ctypes.c_int(100)
+        a_gpu.copy_from(ctypes.cast(ctypes.byref(a_cpu), ctypes.c_void_p), 4)
+
+        b_cpu = ctypes.c_int()
+        a_gpu.copy_to_host(
+            ctypes.cast(ctypes.byref(b_cpu), ctypes.c_void_p), 4)
+        assert b_cpu.value == a_cpu.value
+
+    def test_copy_from_device(self):
+        a_gpu = memory.alloc(4)
+        a_cpu = ctypes.c_int(100)
+        a_gpu.copy_from(ctypes.cast(ctypes.byref(a_cpu), ctypes.c_void_p), 4)
+
+        b_gpu = memory.alloc(4)
+        b_gpu.copy_from(a_gpu, 4)
+        b_cpu = ctypes.c_int()
+        b_gpu.copy_to_host(
+            ctypes.cast(ctypes.byref(b_cpu), ctypes.c_void_p), 4)
+        assert b_cpu.value == a_cpu.value
+
+    def test_copy_to_and_from_host_using_raw_ptr(self):
+        a_gpu = memory.alloc(4)
+        a_cpu = ctypes.c_int(100)
+        a_cpu_ptr = ctypes.cast(ctypes.byref(a_cpu), ctypes.c_void_p)
+        a_gpu.copy_from(a_cpu_ptr.value, 4)
+
+        b_cpu = ctypes.c_int()
+        b_cpu_ptr = ctypes.cast(ctypes.byref(b_cpu), ctypes.c_void_p)
+        a_gpu.copy_to_host(b_cpu_ptr.value, 4)
+        assert b_cpu.value == a_cpu.value
+
+    def test_copy_from_device_using_raw_ptr(self):
+        a_gpu = memory.alloc(4)
+        a_cpu = ctypes.c_int(100)
+        a_cpu_ptr = ctypes.cast(ctypes.byref(a_cpu), ctypes.c_void_p)
+        a_gpu.copy_from(a_cpu_ptr.value, 4)
+
+        b_gpu = memory.alloc(4)
+        b_gpu.copy_from(a_gpu, 4)
+        b_cpu = ctypes.c_int()
+        b_cpu_ptr = ctypes.cast(ctypes.byref(b_cpu), ctypes.c_void_p)
+        b_gpu.copy_to_host(b_cpu_ptr.value, 4)
+        assert b_cpu.value == a_cpu.value
+
+    def test_memset(self):
+        a_gpu = memory.alloc(4)
+        a_gpu.memset(1, 4)
+        a_cpu = ctypes.c_ubyte()
+        for i in range(4):
+            a_gpu.copy_to_host(
+                ctypes.cast(ctypes.byref(a_cpu), ctypes.c_void_p), 1)
+            assert a_cpu.value == 1
+            a_gpu += 1
+
+
+@testing.parameterize(*testing.product({
+    'use_streams': [True, False],
+}))
+class TestMemoryPointerAsync(unittest.TestCase):
+
+    def setUp(self):
+        self.stream = stream_module.Stream() if self.use_streams else None
+
+    def test_copy_to_and_from_host_async(self):
+        a_gpu = memory.alloc(4)
+        a_cpu = ctypes.c_int(100)
+        a_gpu.copy_from_async(ctypes.cast(ctypes.byref(
+            a_cpu), ctypes.c_void_p), 4, stream=self.stream)
+
+        b_cpu = ctypes.c_int()
+        a_gpu.copy_to_host_async(
+            ctypes.cast(ctypes.byref(b_cpu), ctypes.c_void_p),
+            4, stream=self.stream)
+        if self.stream is not None:
+            self.stream.synchronize()
+        else:
+            stream_module.get_current_stream().synchronize()
+        assert b_cpu.value == a_cpu.value
+
+    def test_copy_from_device_async(self):
+        a_gpu = memory.alloc(4)
+        a_cpu = ctypes.c_int(100)
+        a_gpu.copy_from_async(ctypes.cast(ctypes.byref(
+            a_cpu), ctypes.c_void_p), 4, stream=self.stream)
+
+        b_gpu = memory.alloc(4)
+        b_gpu.copy_from_async(a_gpu, 4, stream=self.stream)
+        b_cpu = ctypes.c_int()
+        b_gpu.copy_to_host_async(
+            ctypes.cast(ctypes.byref(b_cpu), ctypes.c_void_p),
+            4, stream=self.stream)
+        if self.stream is not None:
+            self.stream.synchronize()
+        else:
+            stream_module.get_current_stream().synchronize()
+        assert b_cpu.value == a_cpu.value
+
+    def test_copy_to_and_from_host_async_using_raw_ptr(self):
+        a_gpu = memory.alloc(4)
+        a_cpu = ctypes.c_int(100)
+        a_cpu_ptr = ctypes.cast(ctypes.byref(a_cpu), ctypes.c_void_p)
+        a_gpu.copy_from_async(a_cpu_ptr.value, 4, stream=self.stream)
+
+        b_cpu = ctypes.c_int()
+        b_cpu_ptr = ctypes.cast(ctypes.byref(b_cpu), ctypes.c_void_p)
+        a_gpu.copy_to_host_async(b_cpu_ptr.value, 4, stream=self.stream)
+        if self.stream is not None:
+            self.stream.synchronize()
+        else:
+            stream_module.get_current_stream().synchronize()
+        assert b_cpu.value == a_cpu.value
+
+    def test_copy_from_device_async_using_raw_ptr(self):
+        a_gpu = memory.alloc(4)
+        a_cpu = ctypes.c_int(100)
+        a_cpu_ptr = ctypes.cast(ctypes.byref(a_cpu), ctypes.c_void_p)
+        a_gpu.copy_from_async(a_cpu_ptr.value, 4, stream=self.stream)
+
+        b_gpu = memory.alloc(4)
+        b_gpu.copy_from_async(a_gpu, 4, stream=self.stream)
+        b_cpu = ctypes.c_int()
+        b_cpu_ptr = ctypes.cast(ctypes.byref(b_cpu), ctypes.c_void_p)
+        b_gpu.copy_to_host_async(b_cpu_ptr.value, 4, stream=self.stream)
+        if self.stream is not None:
+            self.stream.synchronize()
+        else:
+            stream_module.get_current_stream().synchronize()
+        assert b_cpu.value == a_cpu.value
+
+
+# -----------------------------------------------------------------------------
+# Memory pool
+
+class TestSingleDeviceMemoryPool(unittest.TestCase):
+
+    def setUp(self):
+        self.pool = memory.SingleDeviceMemoryPool(allocator=mock_alloc)
+        self.unit = memory._allocation_unit_size
+        self.stream = stream_module.Stream()
+        self.stream_ident = self.stream.ptr
+
+    def test_round_size(self):
+        assert memory._round_size(self.unit - 1) == self.unit
+        assert memory._round_size(self.unit) == self.unit
+        assert memory._round_size(self.unit + 1) == self.unit * 2
+
+    def test_bin_index_from_size(self):
+        assert memory._bin_index_from_size(self.unit - 1) == 0
+        assert memory._bin_index_from_size(self.unit) == 0
+        assert memory._bin_index_from_size(self.unit + 1) == 1
+
+    def test_split(self):
+        mem = MockMemory(self.unit * 4)
+        chunk = memory._Chunk(mem, 0, mem.size, self.stream_ident)
+        tail = chunk.split(self.unit * 2)
+        assert chunk.ptr() == mem.ptr
+        assert chunk.offset == 0
+        assert chunk.size == self.unit * 2
+        assert chunk.prev is None
+        assert chunk.next.ptr() == tail.ptr()
+        assert chunk.stream_ident == self.stream_ident
+        assert tail.ptr() == mem.ptr + self.unit * 2
+        assert tail.offset == self.unit * 2
+        assert tail.size == self.unit * 2
+        assert tail.prev.ptr() == chunk.ptr()
+        assert tail.next is None
+        assert tail.stream_ident == self.stream_ident
+
+        tail_of_head = chunk.split(self.unit)
+        assert chunk.ptr() == mem.ptr
+        assert chunk.offset == 0
+        assert chunk.size == self.unit
+        assert chunk.prev is None
+        assert chunk.next.ptr() == tail_of_head.ptr()
+        assert chunk.stream_ident == self.stream_ident
+        assert tail_of_head.ptr() == mem.ptr + self.unit
+        assert tail_of_head.offset == self.unit
+        assert tail_of_head.size == self.unit
+        assert tail_of_head.prev.ptr() == chunk.ptr()
+        assert tail_of_head.next.ptr() == tail.ptr()
+        assert tail_of_head.stream_ident == self.stream_ident
+
+        tail_of_tail = tail.split(self.unit)
+        assert tail.ptr() == chunk.ptr() + self.unit * 2
+        assert tail.offset == self.unit * 2
+        assert tail.size == self.unit
+        assert tail.prev.ptr() == tail_of_head.ptr()
+        assert tail.next.ptr() == tail_of_tail.ptr()
+        assert tail.stream_ident == self.stream_ident
+        assert tail_of_tail.ptr() == mem.ptr + self.unit * 3
+        assert tail_of_tail.offset == self.unit * 3
+        assert tail_of_tail.size == self.unit
+        assert tail_of_tail.prev.ptr() == tail.ptr()
+        assert tail_of_tail.next is None
+        assert tail_of_tail.stream_ident == self.stream_ident
+
+    def test_merge(self):
+        mem = MockMemory(self.unit * 4)
+        chunk = memory._Chunk(mem, 0, mem.size, self.stream_ident)
+        chunk_ptr = chunk.ptr()
+        chunk_offset = chunk.offset
+        chunk_size = chunk.size
+
+        tail = chunk.split(self.unit * 2)
+        head = chunk
+        head_ptr = head.ptr()
+        head_offset = head.offset
+        head_size = head.size
+        tail_ptr = tail.ptr()
+        tail_offset = tail.offset
+        tail_size = tail.size
+
+        tail_of_head = head.split(self.unit)
+        tail_of_tail = tail.split(self.unit)
+
+        head.merge(tail_of_head)
+        assert head.ptr() == head_ptr
+        assert head.offset == head_offset
+        assert head.size == head_size
+        assert head.prev is None
+        assert head.next.ptr() == tail_ptr
+        assert head.stream_ident == self.stream_ident
+
+        tail.merge(tail_of_tail)
+        assert tail.ptr() == tail_ptr
+        assert tail.offset == tail_offset
+        assert tail.size == tail_size
+        assert tail.prev.ptr() == head_ptr
+        assert tail.next is None
+        assert tail.stream_ident == self.stream_ident
+
+        head.merge(tail)
+        assert head.ptr() == chunk_ptr
+        assert head.offset == chunk_offset
+        assert head.size == chunk_size
+        assert head.prev is None
+        assert head.next is None
+        assert head.stream_ident == self.stream_ident
+
+    def test_alloc(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        p2 = self.pool.malloc(self.unit * 4)
+        p3 = self.pool.malloc(self.unit * 8)
+        assert p1.ptr != p2.ptr
+        assert p1.ptr != p3.ptr
+        assert p2.ptr != p3.ptr
+
+    def test_alloc_split(self):
+        p = self.pool.malloc(self.unit * 4)
+        ptr = p.ptr
+        del p
+        head = self.pool.malloc(self.unit * 2)
+        tail = self.pool.malloc(self.unit * 2)
+        assert ptr == head.ptr
+        assert ptr + self.unit * 2 == tail.ptr
+
+    def test_alloc_limit(self):
+        self.pool.set_limit(size=(self.unit * 6))
+
+        p1 = self.pool.malloc(self.unit * 5)
+        p2 = self.pool.malloc(self.unit * 1)
+        with self.assertRaises(memory.OutOfMemoryError):
+            self.pool.malloc(self.unit)
+
+        self.pool.set_limit(size=(self.unit * 7))
+        p3 = self.pool.malloc(self.unit)
+        del p1, p2, p3
+
+    def test_alloc_limit_fragmented(self):
+        # Test for #7678.
+        self.pool.set_limit(size=(self.unit * 6))
+
+        p1 = self.pool.malloc(self.unit * 5)
+        del p1
+        p2 = self.pool.malloc(self.unit * 6)
+        del p2
+
+    def test_free(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        ptr1 = p1.ptr
+        del p1
+        p2 = self.pool.malloc(self.unit * 4)
+        assert ptr1 == p2.ptr
+
+    def test_free_stream(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        ptr1 = p1.ptr
+        del p1
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 4)
+        assert ptr1 != p2.ptr
+
+    def test_free_merge(self):
+        p = self.pool.malloc(self.unit * 4)
+        ptr = p.ptr
+        del p
+
+        # merge head into tail
+        head = self.pool.malloc(self.unit * 2)
+        tail = self.pool.malloc(self.unit * 2)
+        assert ptr == head.ptr
+        del tail
+        del head
+        p = self.pool.malloc(self.unit * 4)
+        assert ptr == p.ptr
+        del p
+
+        # merge tail into head
+        head = self.pool.malloc(self.unit * 2)
+        tail = self.pool.malloc(self.unit * 2)
+        assert ptr == head.ptr
+        del head
+        del tail
+        p = self.pool.malloc(self.unit * 4)
+        assert ptr == p.ptr
+        del p
+
+    def test_free_different_size(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        ptr1 = p1.ptr
+        del p1
+        p2 = self.pool.malloc(self.unit * 8)
+        assert ptr1 != p2.ptr
+
+    def test_free_all_blocks(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        ptr1 = p1.ptr
+        del p1
+        self.pool.free_all_blocks()
+        p2 = self.pool.malloc(self.unit * 4)
+        assert ptr1 != p2.ptr
+        del p2
+
+    def test_free_all_blocks_split(self):
+        # do not free splitted blocks
+        p = self.pool.malloc(self.unit * 4)
+        del p
+        head = self.pool.malloc(self.unit * 2)
+        tail = self.pool.malloc(self.unit * 2)
+        tailptr = tail.ptr
+        del tail
+        self.pool.free_all_blocks()
+        p = self.pool.malloc(self.unit * 2)
+        assert tailptr == p.ptr
+        del head
+
+    def test_free_all_blocks_stream(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        ptr1 = p1.ptr
+        del p1
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 4)
+            ptr2 = p2.ptr
+            del p2
+        self.pool.free_all_blocks(stream=stream_module.Stream.null)
+        p3 = self.pool.malloc(self.unit * 4)
+        assert ptr1 != p3.ptr
+        assert ptr2 != p3.ptr
+        with self.stream:
+            p4 = self.pool.malloc(self.unit * 4)
+            assert ptr1 != p4.ptr
+            assert ptr2 == p4.ptr
+
+    def test_free_all_blocks_all_streams(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        ptr1 = p1.ptr
+        del p1
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 4)
+            ptr2 = p2.ptr
+            del p2
+        self.pool.free_all_blocks()
+        p3 = self.pool.malloc(self.unit * 4)
+        assert ptr1 != p3.ptr
+        assert ptr2 != p3.ptr
+        with self.stream:
+            p4 = self.pool.malloc(self.unit * 4)
+            assert ptr1 != p4.ptr
+            assert ptr2 != p4.ptr
+
+    def test_free_all_free(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        ptr1 = p1.ptr
+        del p1
+        with testing.assert_warns(DeprecationWarning):
+            self.pool.free_all_free()
+        p2 = self.pool.malloc(self.unit * 4)
+        assert ptr1 != p2.ptr
+
+    def test_used_bytes(self):
+        p1 = self.pool.malloc(self.unit * 2)
+        assert self.unit * 2 == self.pool.used_bytes()
+        p2 = self.pool.malloc(self.unit * 4)
+        assert self.unit * 6 == self.pool.used_bytes()
+        del p2
+        assert self.unit * 2 == self.pool.used_bytes()
+        del p1
+        assert self.unit * 0 == self.pool.used_bytes()
+        p3 = self.pool.malloc(self.unit * 1)
+        assert self.unit * 1 == self.pool.used_bytes()
+        del p3
+
+    def test_used_bytes_stream(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        del p1
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 2)
+        assert self.unit * 2 == self.pool.used_bytes()
+        del p2
+
+    def test_free_bytes(self):
+        p1 = self.pool.malloc(self.unit * 2)
+        assert self.unit * 0 == self.pool.free_bytes()
+        p2 = self.pool.malloc(self.unit * 4)
+        assert self.unit * 0 == self.pool.free_bytes()
+        del p2
+        assert self.unit * 4 == self.pool.free_bytes()
+        del p1
+        assert self.unit * 6 == self.pool.free_bytes()
+        p3 = self.pool.malloc(self.unit * 1)
+        assert self.unit * 5 == self.pool.free_bytes()
+        del p3
+
+    def test_free_bytes_stream(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        del p1
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 2)
+        assert self.unit * 4 == self.pool.free_bytes()
+        del p2
+
+    def test_total_bytes(self):
+        p1 = self.pool.malloc(self.unit * 2)
+        assert self.unit * 2 == self.pool.total_bytes()
+        p2 = self.pool.malloc(self.unit * 4)
+        assert self.unit * 6 == self.pool.total_bytes()
+        del p1
+        assert self.unit * 6 == self.pool.total_bytes()
+        del p2
+        assert self.unit * 6 == self.pool.total_bytes()
+        p3 = self.pool.malloc(self.unit * 1)
+        assert self.unit * 6 == self.pool.total_bytes()
+
+        assert (self.pool.used_bytes() + self.pool.free_bytes()
+                == self.pool.total_bytes())
+
+        del p3
+
+        self.pool.free_all_blocks()
+        assert 0 == self.pool.total_bytes()
+
+    def test_total_bytes_stream(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        del p1
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 2)
+        assert self.unit * 6 == self.pool.total_bytes()
+        del p2
+
+    def test_get_limit(self):
+        # limit is disabled by default
+        assert 0 == self.pool.get_limit()
+
+    def test_set_limit_size(self):
+        self.pool.set_limit(size=1024)
+        assert 1024 == self.pool.get_limit()
+
+        self.pool.set_limit(size=2**33)
+        assert 2**33 == self.pool.get_limit()
+
+        self.pool.set_limit(size=0)
+        assert 0 == self.pool.get_limit()
+
+        with self.assertRaises(ValueError):
+            self.pool.set_limit(size=-1)
+
+    def test_set_limit_fraction(self):
+        _, total = cupy.cuda.runtime.memGetInfo()
+
+        self.pool.set_limit(fraction=0)
+        assert 0 == self.pool.get_limit()
+
+        self.pool.set_limit(fraction=0.5)
+        assert total * 0.5 == self.pool.get_limit()
+
+        self.pool.set_limit(fraction=1.0)
+        assert total == self.pool.get_limit()
+
+        with self.assertRaises(ValueError):
+            self.pool.set_limit(fraction=-1)
+
+        with self.assertRaises(ValueError):
+            self.pool.set_limit(fraction=1.1)
+
+
+class TestParseMempoolLimitEnvVar(unittest.TestCase):
+    def test_parse_limit_string(self):
+        parse_limit_string = memory._parse_limit_string
+
+        # size
+        param = parse_limit_string('0')
+        assert 0 == param['size']
+        assert None is param['fraction']
+
+        param = parse_limit_string('1073741824')
+        assert 1073741824 == param['size']
+        assert None is param['fraction']
+
+        # fraction
+        param = parse_limit_string('0%')
+        assert None is param['size']
+        assert 0.0 == param['fraction']
+
+        param = parse_limit_string('40%')
+        assert None is param['size']
+        assert 0.4 == param['fraction']
+
+        param = parse_limit_string('70.5%')
+        assert None is param['size']
+        assert 0.705 == param['fraction']
+
+        param = parse_limit_string('100%')
+        assert None is param['size']
+        assert 1.0 == param['fraction']
+
+
+@testing.parameterize(*testing.product({
+    'allocator': [memory._malloc, memory.malloc_managed],
+}))
+class TestMemoryPool(unittest.TestCase):
+
+    def setUp(self):
+        if (
+            cupy.cuda.runtime.is_hip and
+            cupy.cuda.driver.get_build_version() < 40300000 and
+            self.allocator is memory.malloc_managed
+        ):
+            raise unittest.SkipTest('Managed memory requires ROCm 4.3+')
+        self.pool = memory.MemoryPool(self.allocator)
+
+    def tearDown(self):
+        self.pool.free_all_blocks()
+
+    def test_zero_size_alloc(self):
+        with cupy.cuda.Device():
+            mem = self.pool.malloc(0).mem
+            assert isinstance(mem, memory.Memory)
+            assert not isinstance(mem, memory.PooledMemory)
+
+    def test_double_free(self):
+        with cupy.cuda.Device():
+            mem = self.pool.malloc(1).mem
+            mem.free()
+            mem.free()
+
+    def test_free_all_blocks(self):
+        with cupy.cuda.Device():
+            mem = self.pool.malloc(1).mem
+            assert isinstance(mem, memory.BaseMemory)
+            assert isinstance(mem, memory.PooledMemory)
+            assert self.pool.n_free_blocks() == 0
+            mem.free()
+            assert self.pool.n_free_blocks() == 1
+            self.pool.free_all_blocks()
+            assert self.pool.n_free_blocks() == 0
+
+    def test_free_all_blocks_without_malloc(self):
+        with cupy.cuda.Device():
+            # call directly without malloc.
+            self.pool.free_all_blocks()
+            assert self.pool.n_free_blocks() == 0
+
+    def test_free_all_free(self):
+        with cupy.cuda.Device():
+            mem = self.pool.malloc(1).mem
+            assert isinstance(mem, memory.BaseMemory)
+            assert isinstance(mem, memory.PooledMemory)
+            assert self.pool.n_free_blocks() == 0
+            mem.free()
+            assert self.pool.n_free_blocks() == 1
+            with testing.assert_warns(DeprecationWarning):
+                self.pool.free_all_free()
+            assert self.pool.n_free_blocks() == 0
+
+    def test_free_all_free_without_malloc(self):
+        with cupy.cuda.Device():
+            # call directly without malloc.
+            with testing.assert_warns(DeprecationWarning):
+                self.pool.free_all_free()
+            assert self.pool.n_free_blocks() == 0
+
+    def test_n_free_blocks_without_malloc(self):
+        with cupy.cuda.Device():
+            # call directly without malloc/free_all_free.
+            assert self.pool.n_free_blocks() == 0
+
+    def test_used_bytes(self):
+        with cupy.cuda.Device():
+            assert 0 == self.pool.used_bytes()
+
+    def test_free_bytes(self):
+        with cupy.cuda.Device():
+            assert 0 == self.pool.free_bytes()
+
+    def test_total_bytes(self):
+        with cupy.cuda.Device():
+            assert 0 == self.pool.total_bytes()
+
+
+# TODO(leofang): test MemoryAsyncPool. We currently remove the test because
+# this test class requires the ability of creating a new pool, which we do
+# not support yet for MemoryAsyncPool.
+@testing.parameterize(*testing.product({
+    'mempool': ('MemoryPool',),
+}))
+class TestAllocator(unittest.TestCase):
+
+    def setUp(self):
+        if self.mempool == 'MemoryAsyncPool':
+            if cupy.cuda.runtime.is_hip:
+                pytest.skip('HIP does not support async allocator')
+            if cupy.cuda.driver._is_cuda_python():
+                version = cupy.cuda.runtime.runtimeGetVersion()
+            else:
+                version = cupy.cuda.driver.get_build_version()
+            if version < 11020:
+                pytest.skip('malloc_async is supported since CUDA 11.2')
+            if cupy.cuda.runtime.driverGetVersion() < 11030:
+                pytest.skip('pool statistics is supported with driver 11.3+')
+        self.old_pool = cupy.get_default_memory_pool()
+        self.pool = getattr(memory, self.mempool)()
+        memory.set_allocator(self.pool.malloc)
+
+    def tearDown(self):
+        self.pool.set_limit(size=0)
+        self.pool.free_all_blocks()
+        memory.set_allocator(self.old_pool.malloc)
+
+    def test_set_allocator(self):
+        with cupy.cuda.Device():
+            assert 0 == self.pool.used_bytes()
+            arr = cupy.arange(128, dtype=cupy.int64)
+            assert 1024 == arr.data.mem.size
+            assert 1024 == self.pool.used_bytes()
+
+    def test_get_allocator(self):
+        assert memory.get_allocator() == self.pool.malloc
+
+    def test_allocator_context_manager(self):
+        new_pool = memory.MemoryPool()
+        with cupy.cuda.using_allocator(new_pool.malloc):
+            assert memory.get_allocator() == new_pool.malloc
+        assert memory.get_allocator() == self.pool.malloc
+
+    def test_set_allocator_cm(self):
+        new_pool = memory.MemoryPool()
+        new_pool2 = memory.MemoryPool()
+        with cupy.cuda.using_allocator(new_pool.malloc):
+            with self.assertRaises(ValueError):
+                memory.set_allocator(new_pool2.malloc)
+
+    def test_allocator_nested_context_manager(self):
+        new_pool = memory.MemoryPool()
+        with cupy.cuda.using_allocator(new_pool.malloc):
+            new_pool2 = memory.MemoryPool()
+            assert memory.get_allocator() == new_pool.malloc
+            with cupy.cuda.using_allocator(new_pool2.malloc):
+                assert memory.get_allocator() == new_pool2.malloc
+            assert memory.get_allocator() == new_pool.malloc
+        assert memory.get_allocator() == self.pool.malloc
+
+    def test_allocator_thread_local(self):
+        barrier = threading.Barrier(2)
+
+        def thread_body(self):
+            cupy.cuda.Device().use()
+            new_pool = memory.MemoryPool()
+            with cupy.cuda.using_allocator(new_pool.malloc):
+                assert memory.get_allocator() == new_pool.malloc
+                barrier.wait()
+                arr = cupy.zeros(128, dtype=cupy.int64)
+                barrier.wait()
+                assert arr.data.mem.size == new_pool.used_bytes()
+                barrier.wait()
+            assert memory.get_allocator() == self.pool.malloc
+            self._success = True
+
+        with cupy.cuda.Device():
+            self._success = False
+            t = threading.Thread(target=thread_body, args=(self,), daemon=True)
+            t.start()
+            barrier.wait()
+            assert memory.get_allocator() == self.pool.malloc
+            arr = cupy.ones(256, dtype=cupy.int64)
+            barrier.wait()
+            assert arr.data.mem.size == self.pool.used_bytes()
+            barrier.wait()
+            t.join()
+            assert self._success
+
+    def test_thread_local_valid(self):
+        new_pool = memory.MemoryPool()
+        arr = None
+        with cupy.cuda.using_allocator(new_pool.malloc):
+            arr = cupy.zeros(128, dtype=cupy.int64)
+            arr += 1
+        # Check that arr and the pool have not ben released
+        assert arr.data.mem.size == new_pool.used_bytes()
+        assert arr.sum() == 128
+
+    def _reuse_between_thread(self, stream_main, stream_sub):
+        new_pool = memory.MemoryPool()
+
+        def job(stream):
+            cupy.cuda.Device().use()
+            with cupy.cuda.using_allocator(new_pool.malloc):
+                with stream:
+                    arr = cupy.arange(16)
+            self._ptr = arr.data.ptr
+            del arr
+            self._error = False
+
+        # Run in main thread.
+        self._ptr = -1
+        self._error = True
+        job(stream_main)
+        assert not self._error
+        main_ptr = self._ptr
+
+        # Run in sub thread.
+        self._ptr = -1
+        self._error = True
+        with cupy.cuda.Device():
+            t = threading.Thread(target=job, args=(stream_sub,))
+            t.daemon = True
+            t.start()
+            t.join()
+        assert not self._error
+        return main_ptr, self._ptr
+
+    def test_reuse_between_thread(self):
+        stream = cupy.cuda.Stream.null
+        main_ptr, sub_ptr = self._reuse_between_thread(stream, stream)
+        assert main_ptr == sub_ptr
+
+    def test_reuse_between_thread_same_stream(self):
+        stream = cupy.cuda.Stream()
+        main_ptr, sub_ptr = self._reuse_between_thread(stream, stream)
+        assert main_ptr == sub_ptr
+
+    def test_reuse_between_thread_different_stream(self):
+        stream1 = cupy.cuda.Stream()
+        stream2 = cupy.cuda.Stream()
+        main_ptr, sub_ptr = self._reuse_between_thread(stream1, stream2)
+        assert main_ptr != sub_ptr
+
+    @pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason='No PTDS on HIP')
+    def test_reuse_between_thread_ptds(self):
+        stream = cupy.cuda.Stream.ptds
+        main_ptr, sub_ptr = self._reuse_between_thread(stream, stream)
+        assert main_ptr != sub_ptr
+
+
+class TestAllocatorDisabled(unittest.TestCase):
+
+    def setUp(self):
+        self.pool = cupy.get_default_memory_pool()
+
+    def tearDown(self):
+        memory.set_allocator(self.pool.malloc)
+
+    def _check_pool_not_used(self):
+        used_bytes = self.pool.used_bytes()
+        with cupy.cuda.Device():
+            arr = cupy.arange(128, dtype=cupy.int64)
+            assert 0 == self.pool.used_bytes() - used_bytes
+            del arr
+
+    def test(self):
+        memory.set_allocator()
+        self._check_pool_not_used()
+
+    def test_none(self):
+        memory.set_allocator(None)
+        self._check_pool_not_used()
+
+
+class PythonAllocator(object):
+    def __init__(self):
+        self.malloc_called = False
+        self.free_called = False
+
+    def malloc(self, size, device_id):
+        self.malloc_called = True
+        return cupy.cuda.runtime.malloc(size)
+
+    def free(self, size, device_id):
+        self.free_called = True
+        cupy.cuda.runtime.free(size)
+
+
+class TestPythonFunctionAllocator(unittest.TestCase):
+    def setUp(self):
+        self.old_pool = cupy.get_default_memory_pool()
+        self.alloc = PythonAllocator()
+        python_alloc = memory.PythonFunctionAllocator(
+            self.alloc.malloc, self.alloc.free)
+        memory.set_allocator(python_alloc.malloc)
+
+    def tearDown(self):
+        memory.set_allocator(self.old_pool.malloc)
+
+    def test_allocator(self):
+        assert not self.alloc.malloc_called and not self.alloc.free_called
+        cupy.zeros(10)
+        assert self.alloc.malloc_called and self.alloc.free_called
+
+
+class TestMemInfo(unittest.TestCase):
+
+    def test_mem_info(self):
+        d = cupy.cuda.Device()
+        mem_info = d.mem_info
+        assert isinstance(mem_info, tuple)
+        assert len(mem_info) == 2
+        assert all(isinstance(m, int) for m in mem_info)
+        assert all(m > 0 for m in mem_info)
+
+
+class TestLockAndNoGc(unittest.TestCase):
+
+    def test(self):
+        lock = fastrlock.rlock.FastRLock()
+        ctx = memory.LockAndNoGc(lock)
+
+        assert gc.isenabled()
+        self.assertRaises(Exception, lock.release)
+        with ctx:
+            assert not gc.isenabled()
+            lock.release()
+            lock.acquire()
+        assert gc.isenabled()
+        self.assertRaises(Exception, lock.release)
+
+
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = memory.OutOfMemoryError(124, 1024, 1024)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
+
+
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support async allocator')
+@pytest.mark.skipif(cupy.cuda.driver._is_cuda_python()
+                    and cupy.cuda.runtime.runtimeGetVersion() < 11020,
+                    reason='malloc_async is supported since CUDA 11.2')
+@pytest.mark.skipif(not cupy.cuda.driver._is_cuda_python()
+                    and cupy.cuda.driver.get_build_version() < 11020,
+                    reason='malloc_async is supported since CUDA 11.2')
+class TestMallocAsync(unittest.TestCase):
+
+    def setUp(self):
+        if cupy.cuda.runtime.deviceGetAttribute(
+                cupy.cuda.runtime.cudaDevAttrMemoryPoolsSupported, 0) == 0:
+            pytest.skip('malloc_async is not supported on device 0')
+        self.old_pool = cupy.get_default_memory_pool()
+        memory.set_allocator(memory.malloc_async)
+
+    def tearDown(self):
+        memory.set_allocator(self.old_pool.malloc)
+
+    def _check_pool_not_used(self):
+        used_bytes = self.old_pool.used_bytes()
+        with cupy.cuda.Device():
+            arr = cupy.arange(128, dtype=cupy.int64)
+            assert 0 == self.old_pool.used_bytes() - used_bytes
+            del arr
+
+    def test(self):
+        self._check_pool_not_used()
+
+    def test_stream1(self):
+        # Check: pool is not used when on a stream
+        s = cupy.cuda.Stream()
+        with s:
+            self._check_pool_not_used()
+
+    def test_stream2(self):
+        # Check: the memory was allocated on the right stream
+        s = cupy.cuda.Stream()
+        with s:
+            memptr = memory.alloc(100)
+            assert memptr.mem.stream_ref().ptr == s.ptr
+
+    def test_stream3(self):
+        # Check: destory stream does not affect memory deallocation
+        s = cupy.cuda.Stream()
+        with s:
+            memptr = memory.alloc(100)
+
+        del s
+        gc.collect()
+        del memptr
+
+    def test_stream4(self):
+        # Check: free on the same stream
+        s = cupy.cuda.Stream()
+        with s:
+            memptr = memory.alloc(100)
+            del memptr
+
+    def test_stream5(self):
+        # Check: free on another stream
+        s1 = cupy.cuda.Stream()
+        with s1:
+            memptr = memory.alloc(100)
+        del s1
+
+        s2 = cupy.cuda.Stream()
+        with s2:
+            del memptr
+
+
+# the mempool is reused across tests, so we need to keep track
+# of the status after each test is done
+used_bytes_watermark = 0
+free_bytes_watermark = 0
+
+
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support async allocator')
+@pytest.mark.skipif(cupy.cuda.driver._is_cuda_python()
+                    and cupy.cuda.runtime.runtimeGetVersion() < 11020,
+                    reason='malloc_async is supported since CUDA 11.2')
+@pytest.mark.skipif(not cupy.cuda.driver._is_cuda_python()
+                    and cupy.cuda.driver.get_build_version() < 11020,
+                    reason='malloc_async is supported since CUDA 11.2')
+class TestMemoryAsyncPool(unittest.TestCase):
+
+    def setUp(self):
+        if cupy.cuda.runtime.deviceGetAttribute(
+                cupy.cuda.runtime.cudaDevAttrMemoryPoolsSupported, 0) == 0:
+            pytest.skip('malloc_async is not supported on device 0')
+        self.pool = memory.MemoryAsyncPool()
+        self.unit = memory._allocation_unit_size
+        self.stream = stream_module.Stream()
+        self.stream_ident = self.stream.ptr
+        cupy.get_default_memory_pool().free_all_blocks()
+        cupy.cuda.Device().synchronize()
+
+    def tearDown(self):
+        self.pool.set_limit(size=0)
+        self.pool.free_all_blocks()
+        global used_bytes_watermark, free_bytes_watermark
+        used_bytes_watermark = self.pool.used_bytes()
+        free_bytes_watermark = self.pool.free_bytes()
+
+    def test_zero_size_alloc(self):
+        with cupy.cuda.Device():
+            mem = self.pool.malloc(0).mem
+            assert isinstance(mem, memory.MemoryAsync)
+            assert not isinstance(mem, memory.PooledMemory)
+
+    def test_alloc(self):
+        with cupy.cuda.Device():
+            mem = self.pool.malloc(100).mem
+            assert isinstance(mem, memory.MemoryAsync)
+            assert not isinstance(mem, memory.PooledMemory)
+
+    @testing.slow
+    def test_alloc_large_chunk(self):
+        self.pool.free_all_blocks()
+        with cupy.cuda.Device() as d:
+            _, mem_total = d.mem_info
+            mem = self.pool.malloc(int(0.7 * mem_total)).mem  # 70% memory
+            del mem
+            mem = self.pool.malloc(int(0.3 * mem_total)).mem  # 30% memory # noqa
+
+    def test_free_all_blocks(self):
+        with cupy.cuda.Device():
+            mem = self.pool.malloc(1).mem
+            del mem
+            self.pool.free_all_blocks()
+
+    @testing.slow
+    def test_free_all_blocks_large_chunk(self):
+        # When memory is returned to the async mempool, it is not immediately
+        # visible to normal malloc routines until after a sync happens.
+        default_pool = cupy.get_default_memory_pool()
+        with cupy.cuda.Device() as d:
+            _, mem_total = d.mem_info
+            mem = self.pool.malloc(int(0.7 * mem_total)).mem  # 70% memory
+            del mem
+            with pytest.raises(memory.OutOfMemoryError):
+                default_pool.malloc(int(0.3 * mem_total))  # 30% memory
+            self.pool.free_all_blocks()  # synchronize
+            default_pool.malloc(int(0.3 * mem_total))  # this time it'd work
+
+    @testing.slow
+    def test_interaction_with_CuPy_default_pool(self):
+        # Test saneness of cudaMallocAsync
+        default_pool = cupy.get_default_memory_pool()
+        with cupy.cuda.Device() as d:
+            _, mem_total = d.mem_info
+            mem = default_pool.malloc(int(0.7 * mem_total)).mem  # 70% memory
+            del mem
+            with pytest.raises(memory.OutOfMemoryError):
+                self.pool.malloc(int(0.3 * mem_total))  # 30% memory
+            default_pool.free_all_blocks()
+            self.pool.malloc(int(0.3 * mem_total))  # this time it'd work
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='used_bytes is supported with driver 11.3+')
+    def test_used_bytes(self):
+        with cupy.cuda.Device():
+            assert used_bytes_watermark + 0 == self.pool.used_bytes()
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='used_bytes is supported with driver 11.3+')
+    def test_used_bytes2(self):
+        p1 = self.pool.malloc(self.unit * 2)
+        assert used_bytes_watermark + self.unit * 2 == self.pool.used_bytes()
+        p2 = self.pool.malloc(self.unit * 4)
+        assert used_bytes_watermark + self.unit * 6 == self.pool.used_bytes()
+        del p2
+        assert used_bytes_watermark + self.unit * 2 == self.pool.used_bytes()
+        del p1
+        assert used_bytes_watermark + self.unit * 0 == self.pool.used_bytes()
+        p3 = self.pool.malloc(self.unit * 1)
+        assert used_bytes_watermark + self.unit * 1 == self.pool.used_bytes()
+        del p3
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='used_bytes is supported with driver 11.3+')
+    def test_used_bytes_stream(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        del p1
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 2)
+        assert used_bytes_watermark + self.unit * 2 == self.pool.used_bytes()
+        del p2
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='free_bytes is supported with driver 11.3+')
+    def test_free_bytes(self):
+        with cupy.cuda.Device():
+            assert free_bytes_watermark + 0 == self.pool.free_bytes()
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='free_bytes is supported with driver 11.3+')
+    def test_free_bytes2(self):
+        # Note: MemoryAsyncPool works differently from MemoryPool. The first
+        # allocation would be much bigger than requested, and the pool size
+        # increases as needed. As a result, this test method is very different
+        # from TestSingleDeviceMemoryPool.test_free_bytes(), in that the pool
+        # size is a fixed value (outside of our control).
+        p1 = self.pool.malloc(self.unit * 2)
+        assert self.pool.free_bytes() == (
+            self.pool.total_bytes() - self.pool.used_bytes())  # always true
+
+        # current_size is fixed throughout this test, as no synchronization
+        # (such as free_all_blocks()) is done
+        current_size = self.pool.total_bytes()
+        free_size = self.pool.free_bytes()
+
+        p2 = self.pool.malloc(self.unit * 4)
+        free_size -= self.unit * 4
+        assert self.pool.free_bytes() == free_size
+        del p2
+        free_size += self.unit * 4
+        assert self.pool.free_bytes() == free_size
+        del p1
+        free_size += self.unit * 2
+        assert self.pool.free_bytes() == free_size
+        p3 = self.pool.malloc(self.unit * 1)
+        free_size -= self.unit * 1
+        assert self.pool.free_bytes() == free_size
+        del p3
+        assert self.pool.total_bytes() == current_size
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='free_bytes is supported with driver 11.3+')
+    def test_free_bytes_stream(self):
+        p1 = self.pool.malloc(self.unit * 4)
+        del p1
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 2)
+        assert self.pool.free_bytes() == (
+            self.pool.total_bytes() - self.pool.used_bytes())  # always true
+        del p2
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='total_bytes is supported with driver 11.3+')
+    def test_total_bytes(self):
+        with cupy.cuda.Device():
+            assert (used_bytes_watermark + free_bytes_watermark + 0
+                    == self.pool.total_bytes())
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='total_bytes is supported with driver 11.3+')
+    def test_total_bytes2(self):
+        # Note: MemoryAsyncPool works differently from MemoryPool. The first
+        # allocation would be much bigger than requested, and the pool size
+        # increases as needed. As a result, this test method is very different
+        # from TestSingleDeviceMemoryPool.test_total_bytes(), in that the pool
+        # size is either 0 or a fixed value (outside of our control).
+        p1 = self.pool.malloc(self.unit * 2)
+        current_size = self.pool.total_bytes()
+        total_size = used_bytes_watermark + free_bytes_watermark
+
+        assert current_size == self.pool.total_bytes()
+        p2 = self.pool.malloc(self.unit * 4)
+        assert current_size == self.pool.total_bytes()
+        del p1
+        assert current_size == self.pool.total_bytes()
+        del p2
+        assert current_size == self.pool.total_bytes()
+        self.pool.free_all_blocks()
+        assert total_size == self.pool.total_bytes()
+        p3 = self.pool.malloc(self.unit * 1)
+        assert current_size == self.pool.total_bytes()
+
+        assert (self.pool.used_bytes() + self.pool.free_bytes()
+                == self.pool.total_bytes())
+
+        del p3
+
+        self.pool.free_all_blocks()
+        assert total_size == self.pool.total_bytes()
+
+    @pytest.mark.skipif(cupy.cuda.runtime.driverGetVersion() < 11030,
+                        reason='total_bytes is supported with driver 11.3+')
+    def test_total_bytes_stream(self):
+        # Note: MemoryAsyncPool works differently from MemoryPool. The first
+        # allocation would be much bigger than requested, and the pool size
+        # increases as needed. As a result, this test method is very different
+        # from TestSingleDeviceMemoryPool.test_total_bytes_stream(), in that
+        # the pool size is either 0 or a fixed value (outside of our control).
+        p1 = self.pool.malloc(self.unit * 4)
+        current_size = self.pool.total_bytes()
+        assert current_size > 0
+        del p1
+        assert self.pool.total_bytes() > 0
+        self.pool.free_all_blocks()
+        total_size = used_bytes_watermark + free_bytes_watermark
+        assert total_size == self.pool.total_bytes()
+        with self.stream:
+            p2 = self.pool.malloc(self.unit * 2)
+        assert current_size == self.pool.total_bytes()
+        del p2
+
+    def test_get_limit(self):
+        # limit is disabled by default
+        assert 2**64-1 == self.pool.get_limit()
+
+    def test_set_limit_size(self):
+        self.pool.set_limit(size=1024)
+        assert 1024 == self.pool.get_limit()
+
+        self.pool.set_limit(size=2**33)
+        assert 2**33 == self.pool.get_limit()
+
+        self.pool.set_limit(size=0)
+        assert 2**64-1 == self.pool.get_limit()
+
+        with self.assertRaises(ValueError):
+            self.pool.set_limit(size=-1)
+
+    def test_set_limit_fraction(self):
+        _, total = cupy.cuda.runtime.memGetInfo()
+
+        self.pool.set_limit(fraction=0)
+        assert 2**64-1 == self.pool.get_limit()
+
+        self.pool.set_limit(fraction=0.5)
+        assert total * 0.5 == self.pool.get_limit()
+
+        self.pool.set_limit(fraction=1.0)
+        assert total == self.pool.get_limit()
+
+        with self.assertRaises(ValueError):
+            self.pool.set_limit(fraction=-1)
+
+        with self.assertRaises(ValueError):
+            self.pool.set_limit(fraction=1.1)
diff --git a/tests/cupy_tests/cuda_tests/test_memory_hook.py b/tests/cupy_tests/cuda_tests/test_memory_hook.py
new file mode 100644
index 0000000..62ee953
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_memory_hook.py
@@ -0,0 +1,85 @@
+import unittest
+
+import cupy.cuda
+from cupy.cuda import memory
+from cupy.cuda import memory_hook
+
+
+class SimpleMemoryHook(memory_hook.MemoryHook):
+    name = 'SimpleMemoryHook'
+
+    def __init__(self):
+        self.alloc_preprocess_history = []
+        self.alloc_postprocess_history = []
+        self.malloc_preprocess_history = []
+        self.malloc_postprocess_history = []
+        self.free_preprocess_history = []
+        self.free_postprocess_history = []
+
+    def alloc_preprocess(self, **kwargs):
+        self.alloc_preprocess_history.append(kwargs)
+
+    def alloc_postprocess(self, **kwargs):
+        self.alloc_postprocess_history.append(kwargs)
+
+    def malloc_preprocess(self, **kwargs):
+        self.malloc_preprocess_history.append(kwargs)
+
+    def malloc_postprocess(self, **kwargs):
+        self.malloc_postprocess_history.append(kwargs)
+
+    def free_preprocess(self, **kwargs):
+        self.free_preprocess_history.append(kwargs)
+
+    def free_postprocess(self, **kwargs):
+        self.free_postprocess_history.append(kwargs)
+
+
+class TestMemoryHook(unittest.TestCase):
+
+    def setUp(self):
+        self.pool = memory.MemoryPool()
+        self.unit = 512
+
+    def test_hook(self):
+        hook = SimpleMemoryHook()
+        with cupy.cuda.Device(0):
+            with hook:
+                mem = self.pool.malloc(1)
+                ptr1, pmem1 = (mem.ptr, id(mem.mem))
+                del mem
+                mem = self.pool.malloc(1)
+                ptr2, pmem2 = (mem.ptr, id(mem.mem))
+                del mem
+        assert 1 == len(hook.alloc_preprocess_history)
+        assert 1 == len(hook.alloc_postprocess_history)
+        assert 2 == len(hook.malloc_preprocess_history)
+        assert 2 == len(hook.malloc_postprocess_history)
+        assert 2 == len(hook.free_preprocess_history)
+        assert 2 == len(hook.free_postprocess_history)
+        assert {'device_id': 0,
+                'mem_size': self.unit} == hook.alloc_preprocess_history[0]
+        assert {'device_id': 0, 'mem_size': self.unit,
+                'mem_ptr': ptr1} == hook.alloc_postprocess_history[0]
+        assert {'device_id': 0, 'size': 1,
+                'mem_size': self.unit} == hook.malloc_preprocess_history[0]
+        assert {'device_id': 0, 'size': 1, 'mem_size': self.unit,
+                'mem_ptr': ptr1, 'pmem_id': pmem1
+                } == hook.malloc_postprocess_history[0]
+        assert {'device_id': 0, 'size': 1,
+                'mem_size': self.unit} == hook.malloc_preprocess_history[1]
+        assert {'device_id': 0, 'size': 1, 'mem_size': self.unit,
+                'mem_ptr': ptr2, 'pmem_id': pmem2
+                } == hook.malloc_postprocess_history[1]
+        assert {'device_id': 0, 'mem_size': self.unit,
+                'mem_ptr': ptr1, 'pmem_id': pmem1
+                } == hook.free_preprocess_history[0]
+        assert {'device_id': 0, 'mem_size': self.unit,
+                'mem_ptr': ptr1, 'pmem_id': pmem1
+                } == hook.free_postprocess_history[0]
+        assert {'device_id': 0, 'mem_size': self.unit,
+                'mem_ptr': ptr2, 'pmem_id': pmem2
+                } == hook.free_preprocess_history[1]
+        assert {'device_id': 0, 'mem_size': self.unit,
+                'mem_ptr': ptr2, 'pmem_id': pmem2
+                } == hook.free_postprocess_history[1]
diff --git a/tests/cupy_tests/cuda_tests/test_nccl.py b/tests/cupy_tests/cuda_tests/test_nccl.py
new file mode 100644
index 0000000..99441ed
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_nccl.py
@@ -0,0 +1,101 @@
+import pickle
+import unittest
+
+import cupy
+from cupy import cuda
+from cupy.cuda import nccl
+from cupy import testing
+
+
+nccl_available = nccl.available
+
+if nccl_available:
+    nccl_version = nccl.get_version()
+else:
+    nccl_version = -1
+
+
+@unittest.skipUnless(nccl_available, 'nccl is not installed')
+class TestNCCL(unittest.TestCase):
+
+    def test_single_proc_ring(self):
+        id = nccl.get_unique_id()
+        comm = nccl.NcclCommunicator(1, id, 0)
+        assert 0 == comm.rank_id()
+        comm.destroy()
+
+    @unittest.skipUnless(nccl_version >= 2400, 'Using old NCCL')
+    def test_abort(self):
+        id = nccl.get_unique_id()
+        comm = nccl.NcclCommunicator(1, id, 0)
+        comm.abort()
+
+    @unittest.skipUnless(nccl_version >= 2400, 'Using old NCCL')
+    def test_check_async_error(self):
+        id = nccl.get_unique_id()
+        comm = nccl.NcclCommunicator(1, id, 0)
+        comm.check_async_error()
+        comm.destroy()
+
+    def test_init_all(self):
+        comms = nccl.NcclCommunicator.initAll(1)
+        for i, comm in enumerate(comms):
+            assert i == comms[i].rank_id()
+        for i, comm in enumerate(comms):
+            comms[i].destroy()
+
+    def test_single_proc_single_dev(self):
+        comms = nccl.NcclCommunicator.initAll(1)
+        nccl.groupStart()
+        for comm in comms:
+            cuda.Device(comm.device_id()).use()
+            sendbuf = cupy.arange(10)
+            recvbuf = cupy.zeros_like(sendbuf)
+            comm.allReduce(sendbuf.data.ptr, recvbuf.data.ptr, 10,
+                           nccl.NCCL_INT64, nccl.NCCL_SUM,
+                           cuda.Stream.null.ptr)
+        nccl.groupEnd()
+        assert cupy.allclose(sendbuf, recvbuf)
+
+    def test_comm_size(self):
+        id = nccl.get_unique_id()
+        comm = nccl.NcclCommunicator(1, id, 0)
+        assert 1 == comm.size()
+
+    @testing.multi_gpu(2)
+    @unittest.skipUnless(nccl_version >= 2700, 'Using old NCCL')
+    def test_send_recv(self):
+        devs = [0, 1]
+        comms = nccl.NcclCommunicator.initAll(devs)
+        nccl.groupStart()
+        for comm in comms:
+            dev_id = comm.device_id()
+            rank = comm.rank_id()
+            assert rank == dev_id
+
+            if rank == 0:
+                with cuda.Device(dev_id):
+                    sendbuf = cupy.arange(10, dtype=cupy.int64)
+                    comm.send(sendbuf.data.ptr, 10, nccl.NCCL_INT64,
+                              1, cuda.Stream.null.ptr)
+            elif rank == 1:
+                with cuda.Device(dev_id):
+                    recvbuf = cupy.zeros(10, dtype=cupy.int64)
+                    comm.recv(recvbuf.data.ptr, 10, nccl.NCCL_INT64,
+                              0, cuda.Stream.null.ptr)
+        nccl.groupEnd()
+
+        # check result
+        with cuda.Device(1):
+            expected = cupy.arange(10, dtype=cupy.int64)
+            assert (recvbuf == expected).all()
+
+
+@unittest.skipUnless(nccl_available, 'nccl is not installed')
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = nccl.NcclError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_nvrtc.py b/tests/cupy_tests/cuda_tests/test_nvrtc.py
new file mode 100644
index 0000000..ab6a5e9
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_nvrtc.py
@@ -0,0 +1,13 @@
+import pickle
+import unittest
+
+from cupy.cuda import nvrtc
+
+
+class TestExceptionPicklable(unittest.TestCase):
+
+    def test(self):
+        e1 = nvrtc.NVRTCError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
diff --git a/tests/cupy_tests/cuda_tests/test_nvtx.py b/tests/cupy_tests/cuda_tests/test_nvtx.py
new file mode 100644
index 0000000..5fa6743
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_nvtx.py
@@ -0,0 +1,21 @@
+import unittest
+
+from cupy import cuda
+
+
+@unittest.skipUnless(cuda.nvtx.available, 'nvtx is not installed')
+class TestNVTX(unittest.TestCase):
+
+    def test_Mark(self):
+        cuda.nvtx.Mark('test:Mark', 0)
+
+    def test_MarkC(self):
+        cuda.nvtx.MarkC('test:MarkC', 0xFF000000)
+
+    def test_RangePush(self):
+        cuda.nvtx.RangePush('test:RangePush', 1)
+        cuda.nvtx.RangePop()
+
+    def test_RangePushC(self):
+        cuda.nvtx.RangePushC('test:RangePushC', 0xFF000000)
+        cuda.nvtx.RangePop()
diff --git a/tests/cupy_tests/cuda_tests/test_pinned_memory.py b/tests/cupy_tests/cuda_tests/test_pinned_memory.py
new file mode 100644
index 0000000..134be19
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_pinned_memory.py
@@ -0,0 +1,129 @@
+import unittest
+
+from cupy.cuda import pinned_memory
+
+
+class MockMemory(pinned_memory.PinnedMemory):
+    cur_ptr = 1
+
+    def __init__(self, size):
+        self.ptr = MockMemory.cur_ptr
+        MockMemory.cur_ptr += size
+        self.size = size
+
+    def __del__(self):
+        self.ptr = 0
+        pass
+
+
+def mock_alloc(size):
+    mem = MockMemory(size)
+    return pinned_memory.PinnedMemoryPointer(mem, 0)
+
+
+# -----------------------------------------------------------------------------
+# Memory pointer
+
+class TestMemoryPointer(unittest.TestCase):
+
+    def test_int(self):
+        pval = MockMemory.cur_ptr
+        memptr = mock_alloc(1)
+        assert pval == int(memptr)
+
+    def test_add(self):
+        pval = MockMemory.cur_ptr
+        memptr = mock_alloc(8)
+
+        memptr2 = memptr + 4
+        assert isinstance(memptr2, pinned_memory.PinnedMemoryPointer)
+        assert pval + 4 == int(memptr2)
+
+        memptr3 = 4 + memptr
+        assert isinstance(memptr3, pinned_memory.PinnedMemoryPointer)
+        assert pval + 4 == int(memptr3)
+
+        memptr += 4
+        assert isinstance(memptr, pinned_memory.PinnedMemoryPointer)
+        assert pval + 4 == int(memptr)
+
+    def test_sub(self):
+        pval = MockMemory.cur_ptr
+        memptr = mock_alloc(8) + 4
+
+        memptr2 = memptr - 4
+        assert isinstance(memptr2, pinned_memory.PinnedMemoryPointer)
+        assert pval == int(memptr2)
+
+        memptr -= 4
+        assert isinstance(memptr, pinned_memory.PinnedMemoryPointer)
+        assert pval == int(memptr)
+
+
+# -----------------------------------------------------------------------------
+# Memory pool
+
+
+class TestSingleDeviceMemoryPool(unittest.TestCase):
+
+    def setUp(self):
+        self.pool = pinned_memory.PinnedMemoryPool(allocator=mock_alloc)
+
+    def test_alloc(self):
+        p1 = self.pool.malloc(1000)
+        p2 = self.pool.malloc(1000)
+        p3 = self.pool.malloc(2000)
+        assert p1.ptr != p2.ptr
+        assert p1.ptr != p3.ptr
+        assert p2.ptr != p3.ptr
+
+    def test_free(self):
+        p1 = self.pool.malloc(1000)
+        ptr1 = p1.ptr
+        del p1
+        p2 = self.pool.malloc(1000)
+        assert ptr1 == p2.ptr
+
+    def test_free_different_size(self):
+        p1 = self.pool.malloc(1000)
+        ptr1 = p1.ptr
+        del p1
+        p2 = self.pool.malloc(2000)
+        assert ptr1 != p2.ptr
+
+    def test_free_all_blocks(self):
+        p1 = self.pool.malloc(1000)
+        ptr1 = p1.ptr
+        del p1
+        self.pool.free_all_blocks()
+        p2 = self.pool.malloc(1000)
+        assert ptr1 != p2.ptr
+
+    def test_free_all_blocks2(self):
+        mem = self.pool.malloc(1).mem
+        assert isinstance(mem, pinned_memory.PinnedMemory)
+        assert isinstance(mem, pinned_memory.PooledPinnedMemory)
+        assert self.pool.n_free_blocks() == 0
+        mem.free()
+        assert self.pool.n_free_blocks() == 1
+        self.pool.free_all_blocks()
+        assert self.pool.n_free_blocks() == 0
+
+    def test_zero_size_alloc(self):
+        mem = self.pool.malloc(0).mem
+        assert isinstance(mem, pinned_memory.PinnedMemory)
+        assert not isinstance(mem, pinned_memory.PooledPinnedMemory)
+
+    def test_double_free(self):
+        mem = self.pool.malloc(1).mem
+        mem.free()
+        mem.free()
+
+    def test_free_all_blocks_without_malloc(self):
+        # call directly without malloc.
+        self.pool.free_all_blocks()
+        assert self.pool.n_free_blocks() == 0
+
+    def test_n_free_blocks_without_malloc(self):
+        # call directly without malloc/free_all_blocks.
+        assert self.pool.n_free_blocks() == 0
diff --git a/tests/cupy_tests/cuda_tests/test_profile.py b/tests/cupy_tests/cuda_tests/test_profile.py
new file mode 100644
index 0000000..1467822
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_profile.py
@@ -0,0 +1,29 @@
+import unittest
+from unittest import mock
+
+from cupy import cuda
+
+
+class TestProfile(unittest.TestCase):
+
+    def test_profile(self):
+        start_patch = mock.patch('cupy.cuda.profiler.start')
+        stop_patch = mock.patch('cupy.cuda.profiler.stop')
+        with start_patch as start, stop_patch as stop:
+            with cuda.profile():
+                pass
+            start.assert_called_once_with()
+            stop.assert_called_once_with()
+
+    def test_err_case(self):
+        start_patch = mock.patch('cupy.cuda.profiler.start')
+        stop_patch = mock.patch('cupy.cuda.profiler.stop')
+        with start_patch as start, stop_patch as stop:
+            try:
+                with cuda.profile():
+                    raise Exception()
+            except Exception:
+                # ignore
+                pass
+            start.assert_called_once_with()
+            stop.assert_called_once_with()
diff --git a/tests/cupy_tests/cuda_tests/test_runtime.py b/tests/cupy_tests/cuda_tests/test_runtime.py
new file mode 100644
index 0000000..e3b01bb
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_runtime.py
@@ -0,0 +1,57 @@
+import pickle
+
+import pytest
+
+import cupy
+from cupy.cuda import driver
+from cupy.cuda import nvrtc
+from cupy.cuda import runtime
+
+
+class TestExceptionPicklable:
+
+    def test(self):
+        e1 = runtime.CUDARuntimeError(1)
+        e2 = pickle.loads(pickle.dumps(e1))
+        assert e1.args == e2.args
+        assert str(e1) == str(e2)
+
+
+class TestMemPool:
+
+    @pytest.mark.skipif(runtime.is_hip,
+                        reason='HIP does not support async allocator')
+    @pytest.mark.skipif(driver._is_cuda_python()
+                        and runtime.runtimeGetVersion() < 11020,
+                        reason='cudaMemPool_t is supported since CUDA 11.2')
+    @pytest.mark.skipif(not driver._is_cuda_python()
+                        and driver.get_build_version() < 11020,
+                        reason='cudaMemPool_t is supported since CUDA 11.2')
+    @pytest.mark.skipif(runtime.deviceGetAttribute(
+        runtime.cudaDevAttrMemoryPoolsSupported, 0) == 0,
+        reason='cudaMemPool_t is not supported on device 0')
+    def test_mallocFromPoolAsync(self):
+        # also test create/destroy a pool
+        props = runtime.MemPoolProps(
+            runtime.cudaMemAllocationTypePinned,
+            runtime.cudaMemHandleTypeNone,
+            runtime.cudaMemLocationTypeDevice,
+            0)  # on device 0
+        pool = runtime.memPoolCreate(props)
+        assert pool > 0
+        s = cupy.cuda.Stream()
+        ptr = runtime.mallocFromPoolAsync(128, pool, s.ptr)
+        assert ptr > 0
+        runtime.freeAsync(ptr, s.ptr)
+        runtime.memPoolDestroy(pool)
+
+
+@pytest.mark.skipif(runtime.is_hip,
+                    reason='This assumption is correct only in CUDA')
+def test_assumed_runtime_version():
+    # When CUDA Python is enabled, CuPy calculates the CUDA runtime version
+    # from NVRTC version. This test ensures that the assumption is correct
+    # by running the same logic in non-CUDA Python environment.
+    # When this fails, `runtime.runtimeGetVersion()` logic needs to be fixed.
+    (major, minor) = nvrtc.getVersion()
+    assert runtime.runtimeGetVersion() == major * 1000 + minor * 10
diff --git a/tests/cupy_tests/cuda_tests/test_stream.py b/tests/cupy_tests/cuda_tests/test_stream.py
new file mode 100644
index 0000000..d3b08aa
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_stream.py
@@ -0,0 +1,286 @@
+import gc
+import threading
+import unittest
+
+import pytest
+
+import cupy
+from cupy._creation import from_data
+from cupy import cuda
+from cupy import testing
+
+
+@testing.parameterize(
+    *testing.product({
+        'stream_name': ['null', 'ptds'],
+    }))
+class TestStream(unittest.TestCase):
+
+    def setUp(self):
+        if cuda.runtime.is_hip and self.stream_name == 'ptds':
+            self.skipTest('HIP does not support PTDS')
+
+        self._prev_stream = cuda.get_current_stream()
+
+        if self.stream_name == 'null':
+            self.stream = cuda.Stream.null
+        elif self.stream_name == 'ptds':
+            self.stream = cuda.Stream.ptds
+        self.stream.use()
+
+    def tearDown(self):
+        self._prev_stream.use()
+
+    @unittest.skipIf(cuda.runtime.is_hip, 'This test is only for CUDA')
+    def test_eq_cuda(self):
+        null0 = self.stream
+        if self.stream == cuda.Stream.null:
+            null1 = cuda.Stream(True)
+            null2 = cuda.Stream(True)
+            null3 = cuda.Stream(ptds=True)
+        else:
+            null1 = cuda.Stream(ptds=True)
+            null2 = cuda.Stream(ptds=True)
+            null3 = cuda.Stream(True)
+        null4 = cuda.Stream()
+
+        assert null0 == null1
+        assert null1 == null2
+        assert null2 != null3
+        assert null2 != null4
+
+    @unittest.skipIf(not cuda.runtime.is_hip, 'This test is only for HIP')
+    def test_eq_hip(self):
+        null0 = self.stream
+        null1 = cuda.Stream(True)
+        null2 = cuda.Stream(True)
+        null3 = cuda.Stream()
+
+        assert null0 == null1
+        assert null1 == null2
+        assert null2 != null3
+
+    def test_hash(self):
+        hash(self.stream)
+        hash(cuda.Stream(True))
+        hash(cuda.Stream(False))
+        mapping = {cuda.Stream(): 1, cuda.Stream(): 2}  # noqa
+
+    def check_del(self, null, ptds):
+        stream = cuda.Stream(null=null, ptds=ptds).use()
+        assert stream is cuda.get_current_stream()
+        stream_ptr = stream.ptr
+        x = from_data.array([1, 2, 3])
+        del stream
+        assert stream_ptr == cuda.get_current_stream().ptr
+        cuda.Stream.null.use()
+        assert cuda.Stream.null is cuda.get_current_stream()
+        # Want to test cudaStreamDestory is issued, but
+        # runtime.streamQuery(stream_ptr) causes SEGV. We cannot test...
+        del x
+
+    def test_del_default(self):
+        self.check_del(null=False, ptds=False)
+
+    def test_del(self):
+        null = self.stream == cuda.Stream.null
+        if cuda.runtime.is_hip:
+            ptds = False
+        else:
+            ptds = self.stream == cuda.Stream.ptds
+
+        self.check_del(null=null, ptds=ptds)
+
+    def test_get_and_add_callback(self):
+        N = 100
+        cupy_arrays = [testing.shaped_random((2, 3)) for _ in range(N)]
+
+        if not cuda.runtime.is_hip:
+            stream = self.stream
+        else:
+            # adding callbacks to the null stream in HIP would segfault...
+            stream = cuda.Stream()
+
+        out = []
+        stream_list = []
+
+        def _callback(s, _, t):
+            out.append(t[0])
+            stream_list.append(s.ptr)
+
+        for i in range(N):
+            numpy_array = cupy_arrays[i].get(stream=stream)
+            stream.add_callback(
+                _callback,
+                (i, numpy_array))
+
+        stream.synchronize()
+        assert out == list(range(N))
+        assert all(s == stream.ptr for s in stream_list)
+
+    @unittest.skipIf(cuda.runtime.is_hip,
+                     'HIP does not support launch_host_func')
+    def test_launch_host_func(self):
+        N = 100
+        cupy_arrays = [testing.shaped_random((2, 3)) for _ in range(N)]
+
+        stream = cuda.Stream.null
+
+        out = []
+        for i in range(N):
+            numpy_array = cupy_arrays[i].get(stream=stream)
+            stream.launch_host_func(
+                lambda t: out.append(t[0]), (i, numpy_array))
+
+        stream.synchronize()
+        assert out == list(range(N))
+
+    def test_with_statement(self):
+        stream1 = cuda.Stream()
+        stream2 = cuda.Stream()
+        assert self.stream == cuda.get_current_stream()
+        with stream1:
+            assert stream1 == cuda.get_current_stream()
+            with stream2:
+                assert stream2 == cuda.get_current_stream()
+            assert stream1 == cuda.get_current_stream()
+        # self.stream is "forgotten"!
+        assert cuda.Stream.null == cuda.get_current_stream()
+
+    def test_use(self):
+        stream1 = cuda.Stream().use()
+        assert stream1 == cuda.get_current_stream()
+        self.stream.use()
+        assert self.stream == cuda.get_current_stream()
+
+    @testing.multi_gpu(2)
+    def test_per_device(self):
+        with cuda.Device(0):
+            stream0 = cuda.Stream()
+            with stream0:
+                assert stream0 == cuda.get_current_stream()
+                with cuda.Device(1):
+                    assert stream0 != cuda.get_current_stream()
+                    assert cuda.Stream.null == cuda.get_current_stream()
+                assert stream0 == cuda.get_current_stream()
+
+    @testing.multi_gpu(2)
+    def test_per_device_failure(self):
+        with cuda.Device(0):
+            stream0 = cuda.Stream()
+        with cuda.Device(1):
+            with pytest.raises(RuntimeError):
+                with stream0:
+                    pass
+            with pytest.raises(RuntimeError):
+                stream0.use()
+
+    def test_mix_use_context(self):
+        # See cupy/cupy#5143
+        s1 = cuda.Stream()
+        s2 = cuda.Stream()
+        s3 = cuda.Stream()
+        assert cuda.get_current_stream() == self.stream
+        with s1:
+            assert cuda.get_current_stream() == s1
+            s2.use()
+            assert cuda.get_current_stream() == s2
+            with s3:
+                assert cuda.get_current_stream() == s3
+                del s2
+            assert cuda.get_current_stream() == s1
+        # self.stream is "forgotten"!
+        assert cuda.get_current_stream() == cuda.Stream.null
+
+    def test_stream_thread(self):
+        s1 = None
+
+        def f1(barrier, errors):
+            global s1
+            tid = barrier.wait()
+            try:
+                s1 = cuda.Stream()
+                barrier.wait()  # until t2 starts
+                s1.use()
+                barrier.wait()  # until t2 uses the stream
+                s1 = None
+                gc.collect()
+                barrier.wait()  # until t2 decrefs the stream
+                assert cuda.get_current_stream() is not None
+                cupy.arange(10)
+                errors[tid] = False
+            except Exception as e:
+                print(f'error in {tid}: {e}')
+
+        def f2(barrier, errors):
+            global s1
+            tid = barrier.wait()
+            try:
+                barrier.wait()  # until t1 creates the stream
+                s1.use()
+                barrier.wait()  # until t1 uses the stream
+                s1 = None
+                gc.collect()
+                barrier.wait()  # until t1 decrefs the stream
+                assert cuda.get_current_stream() is not None
+                cupy.arange(10)
+                errors[tid] = False
+            except Exception as e:
+                print(f'error in {tid}: {e}')
+
+        barrier = threading.Barrier(2)
+        errors = [True, True]
+        threads = [
+            threading.Thread(target=f1, args=(barrier, errors), daemon=True),
+            threading.Thread(target=f2, args=(barrier, errors), daemon=True),
+        ]
+        del s1
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+        for err in errors:
+            assert err is False
+
+
+class TestExternalStream(unittest.TestCase):
+
+    def setUp(self):
+        self.stream_ptr = cuda.runtime.streamCreate()
+        self.stream = cuda.ExternalStream(self.stream_ptr)
+
+    def tearDown(self):
+        cuda.runtime.streamDestroy(self.stream_ptr)
+
+    def test_get_and_add_callback(self):
+        N = 100
+        cupy_arrays = [testing.shaped_random((2, 3)) for _ in range(N)]
+
+        stream = self.stream
+
+        out = []
+        for i in range(N):
+            numpy_array = cupy_arrays[i].get(stream=stream)
+            stream.add_callback(
+                lambda _, __, t: out.append(t[0]),
+                (i, numpy_array))
+
+        stream.synchronize()
+        assert out == list(range(N))
+
+    @unittest.skipIf(cuda.runtime.is_hip,
+                     'HIP does not support launch_host_func')
+    def test_launch_host_func(self):
+        N = 100
+        cupy_arrays = [testing.shaped_random((2, 3)) for _ in range(N)]
+
+        stream = self.stream
+
+        out = []
+        for i in range(N):
+            numpy_array = cupy_arrays[i].get(stream=stream)
+            stream.launch_host_func(
+                lambda t: out.append(t[0]), (i, numpy_array))
+
+        stream.synchronize()
+        assert out == list(range(N))
diff --git a/tests/cupy_tests/cuda_tests/test_texture.py b/tests/cupy_tests/cuda_tests/test_texture.py
new file mode 100644
index 0000000..8ce48c4
--- /dev/null
+++ b/tests/cupy_tests/cuda_tests/test_texture.py
@@ -0,0 +1,425 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.cuda import runtime
+from cupy.cuda.texture import (ChannelFormatDescriptor, CUDAarray,
+                               ResourceDescriptor, TextureDescriptor,
+                               TextureObject, SurfaceObject)
+
+if cupy.cuda.runtime.is_hip:
+    pytest.skip('HIP texture support is not yet ready',
+                allow_module_level=True)
+
+
+@testing.parameterize(*testing.product({
+    'xp': ('numpy', 'cupy'),
+    'stream': (True, False),
+    'dimensions': ((68, 0, 0), (68, 19, 0), (68, 19, 31)),
+    'n_channels': (1, 2, 4),
+    'dtype': (numpy.float16, numpy.float32, numpy.int8, numpy.int16,
+              numpy.int32, numpy.uint8, numpy.uint16, numpy.uint32),
+    'c_contiguous': (True, False),
+}))
+class TestCUDAarray:
+    def test_array_gen_cpy(self):
+        xp = numpy if self.xp == 'numpy' else cupy
+        stream = None if not self.stream else cupy.cuda.Stream()
+        width, height, depth = self.dimensions
+        n_channel = self.n_channels
+
+        dim = 3 if depth != 0 else 2 if height != 0 else 1
+        shape = (depth, height, n_channel*width) if dim == 3 else \
+                (height, n_channel*width) if dim == 2 else \
+                (n_channel*width,)
+
+        # generate input data and allocate output buffer
+        if self.dtype in (numpy.float16, numpy.float32):
+            arr = xp.random.random(shape).astype(self.dtype)
+            kind = runtime.cudaChannelFormatKindFloat
+        else:  # int
+            arr = xp.random.randint(100, size=shape, dtype=self.dtype)
+            if self.dtype in (numpy.int8, numpy.int16, numpy.int32):
+                kind = runtime.cudaChannelFormatKindSigned
+            else:
+                kind = runtime.cudaChannelFormatKindUnsigned
+
+        if self.c_contiguous:
+            arr2 = xp.zeros_like(arr)
+            assert arr.flags.c_contiguous
+            assert arr2.flags.c_contiguous
+        else:
+            arr = arr[..., ::2]
+            arr2 = xp.zeros_like(arr)
+            width = arr.shape[-1] // n_channel
+            assert not arr.flags.c_contiguous
+            assert arr2.flags.c_contiguous
+            assert arr.shape[-1] == n_channel*width
+
+        # create a CUDA array
+        ch_bits = [0, 0, 0, 0]
+        for i in range(n_channel):
+            ch_bits[i] = arr.dtype.itemsize*8
+        ch = ChannelFormatDescriptor(*ch_bits, kind)
+        cu_arr = CUDAarray(ch, width, height, depth)
+
+        # need to wait for the current stream to finish initialization
+        if stream is not None:
+            s = cupy.cuda.get_current_stream()
+            e = s.record()
+            stream.wait_event(e)
+
+        # copy from input to CUDA array, and back to output
+        cu_arr.copy_from(arr, stream)
+        cu_arr.copy_to(arr2, stream)
+
+        # check input and output are identical
+        if stream is not None:
+            stream.synchronize()
+        assert (arr == arr2).all()
+
+
+source_texobj = r'''
+extern "C"{
+__global__ void copyKernel1Dfetch(float* output,
+                                  cudaTextureObject_t texObj,
+                                  int width)
+{
+    unsigned int x = blockIdx.x * blockDim.x + threadIdx.x;
+
+    // Read from texture and write to global memory
+    if (x < width)
+        output[x] = tex1Dfetch<float>(texObj, x);
+}
+
+__global__ void copyKernel1D(float* output,
+                             cudaTextureObject_t texObj,
+                             int width)
+{
+    unsigned int x = blockIdx.x * blockDim.x + threadIdx.x;
+
+    // Read from texture and write to global memory
+    float u = x;
+    if (x < width)
+        output[x] = tex1D<float>(texObj, u);
+}
+
+__global__ void copyKernel2D(float* output,
+                             cudaTextureObject_t texObj,
+                             int width, int height)
+{
+    unsigned int x = blockIdx.x * blockDim.x + threadIdx.x;
+    unsigned int y = blockIdx.y * blockDim.y + threadIdx.y;
+
+    // Read from texture and write to global memory
+    float u = x;
+    float v = y;
+    if (x < width && y < height)
+        output[y * width + x] = tex2D<float>(texObj, u, v);
+}
+
+__global__ void copyKernel3D(float* output,
+                             cudaTextureObject_t texObj,
+                             int width, int height, int depth)
+{
+    unsigned int x = blockIdx.x * blockDim.x + threadIdx.x;
+    unsigned int y = blockIdx.y * blockDim.y + threadIdx.y;
+    unsigned int z = blockIdx.z * blockDim.z + threadIdx.z;
+
+    // Read from texture and write to global memory
+    float u = x;
+    float v = y;
+    float w = z;
+    if (x < width && y < height && z < depth)
+        output[z*width*height+y*width+x] = tex3D<float>(texObj, u, v, w);
+}
+
+__global__ void copyKernel3D_4ch(float* output_x,
+                                 float* output_y,
+                                 float* output_z,
+                                 float* output_w,
+                                 cudaTextureObject_t texObj,
+                                 int width, int height, int depth)
+{
+    unsigned int x = blockIdx.x * blockDim.x + threadIdx.x;
+    unsigned int y = blockIdx.y * blockDim.y + threadIdx.y;
+    unsigned int z = blockIdx.z * blockDim.z + threadIdx.z;
+    float4 data;
+
+    // Read from texture, separate channels, and write to global memory
+    float u = x;
+    float v = y;
+    float w = z;
+    if (x < width && y < height && z < depth) {
+        data = tex3D<float4>(texObj, u, v, w);
+        output_x[z*width*height+y*width+x] = data.x;
+        output_y[z*width*height+y*width+x] = data.y;
+        output_z[z*width*height+y*width+x] = data.z;
+        output_w[z*width*height+y*width+x] = data.w;
+    }
+}
+}
+'''
+
+
+@testing.parameterize(*testing.product({
+    'dimensions': ((64, 0, 0), (64, 32, 0), (64, 32, 19)),
+    'mem_type': ('CUDAarray', 'linear', 'pitch2D'),
+    'target': ('object',),
+}))
+class TestTexture:
+    def test_fetch_float_texture(self):
+        width, height, depth = self.dimensions
+        dim = 3 if depth != 0 else 2 if height != 0 else 1
+
+        if (self.mem_type == 'linear' and dim != 1) or \
+           (self.mem_type == 'pitch2D' and dim != 2):
+            pytest.skip('The test case {0} is inapplicable for {1} and thus '
+                        'skipped.'.format(self.dimensions, self.mem_type))
+
+        # generate input data and allocate output buffer
+        shape = (depth, height, width) if dim == 3 else \
+                (height, width) if dim == 2 else \
+                (width,)
+
+        # prepare input, output, and texture memory
+        tex_data = cupy.random.random(shape, dtype=cupy.float32)
+        real_output = cupy.zeros_like(tex_data)
+        ch = ChannelFormatDescriptor(32, 0, 0, 0,
+                                     runtime.cudaChannelFormatKindFloat)
+        assert tex_data.flags['C_CONTIGUOUS']
+        assert real_output.flags['C_CONTIGUOUS']
+        if self.mem_type == 'CUDAarray':
+            arr = CUDAarray(ch, width, height, depth)
+            expected_output = cupy.zeros_like(tex_data)
+            assert expected_output.flags['C_CONTIGUOUS']
+            # test bidirectional copy
+            arr.copy_from(tex_data)
+            arr.copy_to(expected_output)
+        else:  # linear are pitch2D are backed by ndarray
+            arr = tex_data
+            expected_output = tex_data
+
+        # create resource and texture descriptors
+        if self.mem_type == 'CUDAarray':
+            res = ResourceDescriptor(runtime.cudaResourceTypeArray, cuArr=arr)
+        elif self.mem_type == 'linear':
+            res = ResourceDescriptor(runtime.cudaResourceTypeLinear,
+                                     arr=arr,
+                                     chDesc=ch,
+                                     sizeInBytes=arr.size*arr.dtype.itemsize)
+        else:  # pitch2D
+            # In this case, we rely on the fact that the hand-picked array
+            # shape meets the alignment requirement. This is CUDA's limitation,
+            # see CUDA Runtime API reference guide. "TexturePitchAlignment" is
+            # assumed to be 32, which should be applicable for most devices.
+            res = ResourceDescriptor(runtime.cudaResourceTypePitch2D,
+                                     arr=arr,
+                                     chDesc=ch,
+                                     width=width,
+                                     height=height,
+                                     pitchInBytes=width*arr.dtype.itemsize)
+        address_mode = (runtime.cudaAddressModeClamp,
+                        runtime.cudaAddressModeClamp)
+        tex = TextureDescriptor(address_mode, runtime.cudaFilterModePoint,
+                                runtime.cudaReadModeElementType)
+
+        if self.target == 'object':
+            # create a texture object
+            texobj = TextureObject(res, tex)
+            mod = cupy.RawModule(code=source_texobj)
+        else:
+            assert False
+
+        # get and launch the kernel
+        ker_name = 'copyKernel'
+        ker_name += '3D' if dim == 3 else '2D' if dim == 2 else '1D'
+        ker_name += 'fetch' if self.mem_type == 'linear' else ''
+        ker = mod.get_function(ker_name)
+        block = (4, 4, 2) if dim == 3 else (4, 4) if dim == 2 else (4,)
+        grid = ()
+        args = (real_output,)
+        if self.target == 'object':
+            args = args + (texobj,)
+        if dim >= 1:
+            grid_x = (width + block[0] - 1)//block[0]
+            grid = grid + (grid_x,)
+            args = args + (width,)
+        if dim >= 2:
+            grid_y = (height + block[1] - 1)//block[1]
+            grid = grid + (grid_y,)
+            args = args + (height,)
+        if dim == 3:
+            grid_z = (depth + block[2] - 1)//block[2]
+            grid = grid + (grid_z,)
+            args = args + (depth,)
+        ker(grid, block, args)
+
+        # validate result
+        assert (real_output == expected_output).all()
+
+
+@testing.parameterize(*testing.product({
+    'target': ('object',),
+}))
+class TestTextureVectorType:
+    def test_fetch_float4_texture(self):
+        width = 47
+        height = 39
+        depth = 11
+        n_channel = 4
+
+        # generate input data and allocate output buffer
+        in_shape = (depth, height, n_channel*width)
+        out_shape = (depth, height, width)
+
+        # prepare input, output, and texture memory
+        tex_data = cupy.random.random(in_shape, dtype=cupy.float32)
+        real_output_x = cupy.zeros(out_shape, dtype=cupy.float32)
+        real_output_y = cupy.zeros(out_shape, dtype=cupy.float32)
+        real_output_z = cupy.zeros(out_shape, dtype=cupy.float32)
+        real_output_w = cupy.zeros(out_shape, dtype=cupy.float32)
+        ch = ChannelFormatDescriptor(32, 32, 32, 32,
+                                     runtime.cudaChannelFormatKindFloat)
+        arr = CUDAarray(ch, width, height, depth)
+        arr.copy_from(tex_data)
+
+        # create resource and texture descriptors
+        res = ResourceDescriptor(runtime.cudaResourceTypeArray, cuArr=arr)
+        address_mode = (runtime.cudaAddressModeClamp,
+                        runtime.cudaAddressModeClamp)
+        tex = TextureDescriptor(address_mode, runtime.cudaFilterModePoint,
+                                runtime.cudaReadModeElementType)
+
+        if self.target == 'object':
+            # create a texture object
+            texobj = TextureObject(res, tex)
+            mod = cupy.RawModule(code=source_texobj)
+        else:
+            assert False
+
+        # get and launch the kernel
+        ker_name = 'copyKernel3D_4ch'
+        ker = mod.get_function(ker_name)
+        block = (4, 4, 2)
+        grid = ((width + block[0] - 1)//block[0],
+                (height + block[1] - 1)//block[1],
+                (depth + block[2] - 1)//block[2])
+        args = (real_output_x, real_output_y, real_output_z, real_output_w)
+        if self.target == 'object':
+            args = args + (texobj,)
+        args = args + (width, height, depth)
+        ker(grid, block, args)
+
+        # validate result
+        assert (real_output_x == tex_data[..., 0::4]).all()
+        assert (real_output_y == tex_data[..., 1::4]).all()
+        assert (real_output_z == tex_data[..., 2::4]).all()
+        assert (real_output_w == tex_data[..., 3::4]).all()
+
+
+source_surfobj = r"""
+extern "C" {
+__global__ void writeKernel1D(cudaSurfaceObject_t surf,
+                              int width)
+{
+    unsigned int w = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (w < width)
+    {
+        float value = w;
+        value *= 3.0;
+        surf1Dwrite(value, surf, w * 4);
+    }
+}
+
+__global__ void writeKernel2D(cudaSurfaceObject_t surf,
+                              int width, int height)
+{
+    unsigned int w = blockIdx.x * blockDim.x + threadIdx.x;
+    unsigned int h = blockIdx.y * blockDim.y + threadIdx.y;
+
+    if (w < width && h < height)
+    {
+        float value = h * width + w;
+        value *= 3.0;
+        surf2Dwrite(value, surf, w * 4, h);
+    }
+}
+
+__global__ void writeKernel3D(cudaSurfaceObject_t surf,
+                              int width, int height, int depth)
+{
+    unsigned int w = blockIdx.x * blockDim.x + threadIdx.x;
+    unsigned int h = blockIdx.y * blockDim.y + threadIdx.y;
+    unsigned int d = blockIdx.z * blockDim.z + threadIdx.z;
+
+    if (w < width && h < height && d < depth)
+    {
+        float value = d * width * height + h * width + w;
+        value *= 3.0;
+        surf3Dwrite(value, surf, w * 4, h, d);
+    }
+}
+}
+"""
+
+
+@testing.parameterize(*testing.product({
+    'dimensions': ((64, 0, 0), (64, 32, 0), (64, 32, 32)),
+}))
+class TestSurface:
+    def test_write_float_surface(self):
+        width, height, depth = self.dimensions
+        dim = 3 if depth != 0 else 2 if height != 0 else 1
+
+        # generate input data and allocate output buffer
+        shape = (depth, height, width) if dim == 3 else \
+                (height, width) if dim == 2 else \
+                (width,)
+
+        # prepare input, output, and surface memory
+        real_output = cupy.zeros(shape, dtype=cupy.float32)
+        assert real_output.flags['C_CONTIGUOUS']
+        ch = ChannelFormatDescriptor(32, 0, 0, 0,
+                                     runtime.cudaChannelFormatKindFloat)
+        expected_output = cupy.arange(numpy.prod(shape), dtype=cupy.float32)
+        expected_output = expected_output.reshape(shape) * 3.0
+        assert expected_output.flags['C_CONTIGUOUS']
+
+        # create resource descriptor
+        # note that surface memory only support CUDA array
+        arr = CUDAarray(ch, width, height, depth,
+                        runtime.cudaArraySurfaceLoadStore)
+        arr.copy_from(real_output)  # init to zero
+        res = ResourceDescriptor(runtime.cudaResourceTypeArray, cuArr=arr)
+
+        # create a surface object; currently we don't support surface reference
+        surfobj = SurfaceObject(res)
+        mod = cupy.RawModule(code=source_surfobj)
+
+        # get and launch the kernel
+        ker_name = 'writeKernel'
+        ker_name += '3D' if dim == 3 else '2D' if dim == 2 else '1D'
+        ker = mod.get_function(ker_name)
+        block = (4, 4, 2) if dim == 3 else (4, 4) if dim == 2 else (4,)
+        grid = ()
+        args = (surfobj,)
+        if dim >= 1:
+            grid_x = (width + block[0] - 1)//block[0]
+            grid = grid + (grid_x,)
+            args = args + (width,)
+        if dim >= 2:
+            grid_y = (height + block[1] - 1)//block[1]
+            grid = grid + (grid_y,)
+            args = args + (height,)
+        if dim == 3:
+            grid_z = (depth + block[2] - 1)//block[2]
+            grid = grid + (grid_z,)
+            args = args + (depth,)
+        ker(grid, block, args)
+
+        # validate result
+        arr.copy_to(real_output)
+        assert (real_output == expected_output).all()
diff --git a/tests/cupy_tests/fft_tests/__init__.py b/tests/cupy_tests/fft_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/fft_tests/test_cache.py b/tests/cupy_tests/fft_tests/test_cache.py
new file mode 100644
index 0000000..b80da86
--- /dev/null
+++ b/tests/cupy_tests/fft_tests/test_cache.py
@@ -0,0 +1,493 @@
+import contextlib
+import io
+import queue
+import threading
+import unittest
+
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.cuda import cufft
+from cupy.cuda import device
+from cupy.cuda import runtime
+from cupy.fft import config
+
+from .test_fft import (multi_gpu_config, _skip_multi_gpu_bug)
+
+
+def intercept_stdout(func):
+    with io.StringIO() as buf, contextlib.redirect_stdout(buf):
+        func()
+        stdout = buf.getvalue()
+    return stdout
+
+
+n_devices = runtime.getDeviceCount()
+
+
+class TestPlanCache(unittest.TestCase):
+    def setUp(self):
+        self.caches = []
+        self.old_sizes = []
+        for i in range(n_devices):
+            with device.Device(i):
+                cache = config.get_plan_cache()
+                self.old_sizes.append(cache.get_size())
+                cache.clear()
+                cache.set_memsize(-1)
+                cache.set_size(2)
+            self.caches.append(cache)
+
+    def tearDown(self):
+        for i in range(n_devices):
+            with device.Device(i):
+                cache = config.get_plan_cache()
+                cache.clear()
+                cache.set_size(self.old_sizes[i])
+                cache.set_memsize(-1)
+
+    def test_LRU_cache1(self):
+        # test if insertion and clean-up works
+        cache = config.get_plan_cache()
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+        a = testing.shaped_random((10,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+
+        cache.clear()
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+    def test_LRU_cache2(self):
+        # test if plan is reused
+        cache = config.get_plan_cache()
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+        # run once and fetch the cached plan
+        a = testing.shaped_random((10,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        iterator = iter(cache)
+        plan0 = next(iterator)[1].plan
+
+        # repeat
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        iterator = iter(cache)
+        plan1 = next(iterator)[1].plan
+
+        # we should get the same plan
+        assert plan0 is plan1
+
+    def test_LRU_cache3(self):
+        # test if cache size is limited
+        cache = config.get_plan_cache()
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+        # run once and fetch the cached plan
+        a = testing.shaped_random((10,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        iterator = iter(cache)
+        plan = next(iterator)[1].plan
+
+        # run another two FFTs with different sizes so that the first
+        # plan is discarded from the cache
+        a = testing.shaped_random((20,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 2 <= cache.get_size()
+        a = testing.shaped_random((30,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 2 <= cache.get_size()
+
+        # check if the first plan is indeed not cached
+        for _, node in cache:
+            assert plan is not node.plan
+
+    def test_LRU_cache4(self):
+        # test if fetching the plan will reorder it to the top
+        cache = config.get_plan_cache()
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+        # this creates a Plan1d
+        a = testing.shaped_random((10,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+
+        # this creates a PlanNd
+        a = testing.shaped_random((10, 20), cupy, cupy.float32)
+        cupy.fft.fftn(a)
+        assert cache.get_curr_size() == 2 <= cache.get_size()
+
+        # The first in the cache is the most recently used one;
+        # using an iterator to access the linked list guarantees that
+        # we don't alter the cache order
+        iterator = iter(cache)
+        assert isinstance(next(iterator)[1].plan, cufft.PlanNd)
+        assert isinstance(next(iterator)[1].plan, cufft.Plan1d)
+        with pytest.raises(StopIteration):
+            next(iterator)
+
+        # this brings Plan1d to the top
+        a = testing.shaped_random((10,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 2 <= cache.get_size()
+        iterator = iter(cache)
+        assert isinstance(next(iterator)[1].plan, cufft.Plan1d)
+        assert isinstance(next(iterator)[1].plan, cufft.PlanNd)
+        with pytest.raises(StopIteration):
+            next(iterator)
+
+        # An LRU cache guarantees that such a silly operation never
+        # raises StopIteration
+        iterator = iter(cache)
+        for i in range(100):
+            cache[next(iterator)[0]]
+
+    @testing.multi_gpu(2)
+    def test_LRU_cache5(self):
+        # test if the LRU cache is thread-local
+
+        def init_caches(gpus):
+            for i in gpus:
+                with device.Device(i):
+                    config.get_plan_cache()
+
+        # Testing in the current thread: in setUp() we ensure all caches
+        # are initialized
+        stdout = intercept_stdout(config.show_plan_cache_info)
+        assert 'uninitialized' not in stdout
+
+        def thread_show_plan_cache_info(queue):
+            # allow output from another thread to be accessed by the
+            # main thread
+            cupy.cuda.Device().use()
+            stdout = intercept_stdout(config.show_plan_cache_info)
+            queue.put(stdout)
+
+        # When starting a new thread, the cache is uninitialized there
+        # (for both devices)
+        q = queue.Queue()
+        thread = threading.Thread(target=thread_show_plan_cache_info,
+                                  args=(q,))
+        thread.start()
+        thread.join()
+        stdout = q.get()
+        assert stdout.count('uninitialized') == n_devices
+
+        def thread_init_caches(gpus, queue):
+            cupy.cuda.Device().use()
+            init_caches(gpus)
+            thread_show_plan_cache_info(queue)
+
+        # Now let's try initializing device 0 on another thread
+        thread = threading.Thread(target=thread_init_caches,
+                                  args=([0], q,))
+        thread.start()
+        thread.join()
+        stdout = q.get()
+        assert stdout.count('uninitialized') == n_devices - 1
+
+        # ...and this time both devices
+        thread = threading.Thread(target=thread_init_caches,
+                                  args=([0, 1], q,))
+        thread.start()
+        thread.join()
+        stdout = q.get()
+        assert stdout.count('uninitialized') == n_devices - 2
+
+    @testing.multi_gpu(2)
+    def test_LRU_cache6(self):
+        # test if each device has a separate cache
+        cache0 = self.caches[0]
+        cache1 = self.caches[1]
+
+        # ensure a fresh state
+        assert cache0.get_curr_size() == 0 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+
+        # do some computation on GPU 0
+        with device.Device(0):
+            a = testing.shaped_random((10,), cupy, cupy.float32)
+            cupy.fft.fft(a)
+        assert cache0.get_curr_size() == 1 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+
+        # do some computation on GPU 1
+        with device.Device(1):
+            c = testing.shaped_random((16,), cupy, cupy.float64)
+            cupy.fft.fft(c)
+        assert cache0.get_curr_size() == 1 <= cache0.get_size()
+        assert cache1.get_curr_size() == 1 <= cache1.get_size()
+
+        # reset device 0
+        cache0.clear()
+        assert cache0.get_curr_size() == 0 <= cache0.get_size()
+        assert cache1.get_curr_size() == 1 <= cache1.get_size()
+
+        # reset device 1
+        cache1.clear()
+        assert cache0.get_curr_size() == 0 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+
+    @testing.multi_gpu(2)
+    @pytest.mark.skipif(runtime.is_hip,
+                        reason="hipFFT doesn't support multi-GPU")
+    def test_LRU_cache7(self):
+        # test accessing a multi-GPU plan
+        cache0 = self.caches[0]
+        cache1 = self.caches[1]
+
+        # ensure a fresh state
+        assert cache0.get_curr_size() == 0 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+
+        # do some computation on GPU 0
+        with device.Device(0):
+            a = testing.shaped_random((10,), cupy, cupy.float32)
+            cupy.fft.fft(a)
+        assert cache0.get_curr_size() == 1 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+
+        # do a multi-GPU FFT
+        config.use_multi_gpus = True
+        config.set_cufft_gpus([0, 1])
+        c = testing.shaped_random((128,), cupy, cupy.complex64)
+        cupy.fft.fft(c)
+        assert cache0.get_curr_size() == 2 <= cache0.get_size()
+        assert cache1.get_curr_size() == 1 <= cache1.get_size()
+
+        # check both devices' caches see the same multi-GPU plan
+        plan0 = next(iter(cache0))[1].plan
+        plan1 = next(iter(cache1))[1].plan
+        assert plan0 is plan1
+
+        # reset
+        config.use_multi_gpus = False
+        config._device = None
+
+        # do some computation on GPU 1
+        with device.Device(1):
+            e = testing.shaped_random((20,), cupy, cupy.complex128)
+            cupy.fft.fft(e)
+        assert cache0.get_curr_size() == 2 <= cache0.get_size()
+        assert cache1.get_curr_size() == 2 <= cache1.get_size()
+
+        # by this time, the multi-GPU plan remains the most recently
+        # used one on GPU 0, but not on GPU 1
+        assert plan0 is next(iter(cache0))[1].plan
+        assert plan1 is not next(iter(cache1))[1].plan
+
+        # now use it again to make it the most recent
+        config.use_multi_gpus = True
+        config.set_cufft_gpus([0, 1])
+        c = testing.shaped_random((128,), cupy, cupy.complex64)
+        cupy.fft.fft(c)
+        assert cache0.get_curr_size() == 2 <= cache0.get_size()
+        assert cache1.get_curr_size() == 2 <= cache1.get_size()
+        assert plan0 is next(iter(cache0))[1].plan
+        assert plan1 is next(iter(cache1))[1].plan
+        # reset
+        config.use_multi_gpus = False
+        config._device = None
+
+        # Do 2 more different FFTs on one of the devices, and the
+        # multi-GPU plan would be discarded from both caches
+        with device.Device(1):
+            x = testing.shaped_random((30,), cupy, cupy.complex128)
+            cupy.fft.fft(x)
+            y = testing.shaped_random((40, 40), cupy, cupy.complex64)
+            cupy.fft.fftn(y)
+        for _, node in cache0:
+            assert plan0 is not node.plan
+        for _, node in cache1:
+            assert plan1 is not node.plan
+        assert cache0.get_curr_size() == 1 <= cache0.get_size()
+        assert cache1.get_curr_size() == 2 <= cache1.get_size()
+
+    def test_LRU_cache8(self):
+        # test if Plan1d and PlanNd can coexist in the same cache
+        cache = config.get_plan_cache()
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+        # do a 1D FFT
+        a = testing.shaped_random((10,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        assert isinstance(next(iter(cache))[1].plan, cufft.Plan1d)
+
+        # then a 3D FFT
+        a = testing.shaped_random((8, 8, 8), cupy, cupy.complex128)
+        cupy.fft.fftn(a)
+        assert cache.get_curr_size() == 2 <= cache.get_size()
+        iterator = iter(cache)
+
+        # the cached order is 1. PlanNd, 2. Plan1d
+        assert isinstance(next(iterator)[1].plan, cufft.PlanNd)
+        assert isinstance(next(iterator)[1].plan, cufft.Plan1d)
+
+    def test_LRU_cache9(self):
+        # test if memsizes in the cache adds up
+        cache = config.get_plan_cache()
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+        memsize = 0
+        a = testing.shaped_random((10,), cupy, cupy.float32)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        memsize += next(iter(cache))[1].plan.work_area.mem.size
+
+        a = testing.shaped_random((48,), cupy, cupy.complex64)
+        cupy.fft.fft(a)
+        assert cache.get_curr_size() == 2 <= cache.get_size()
+        memsize += next(iter(cache))[1].plan.work_area.mem.size
+
+        assert memsize == cache.get_curr_memsize()
+
+    def test_LRU_cache10(self):
+        # test if deletion works and if show_info() is consistent with data
+        cache = config.get_plan_cache()
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+        curr_size = 0
+        size = 2
+        curr_memsize = 0
+        memsize = '(unlimited)'  # default
+
+        a = testing.shaped_random((16, 16), cupy, cupy.float32)
+        cupy.fft.fft2(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        node1 = next(iter(cache))[1]
+        curr_size += 1
+        curr_memsize += node1.plan.work_area.mem.size
+        stdout = intercept_stdout(cache.show_info)
+        assert '{0} / {1} (counts)'.format(curr_size, size) in stdout
+        assert '{0} / {1} (bytes)'.format(curr_memsize, memsize) in stdout
+        assert str(node1) in stdout
+
+        a = testing.shaped_random((1024,), cupy, cupy.complex64)
+        cupy.fft.ifft(a)
+        assert cache.get_curr_size() == 2 <= cache.get_size()
+        node2 = next(iter(cache))[1]
+        curr_size += 1
+        curr_memsize += node2.plan.work_area.mem.size
+        stdout = intercept_stdout(cache.show_info)
+        assert '{0} / {1} (counts)'.format(curr_size, size) in stdout
+        assert '{0} / {1} (bytes)'.format(curr_memsize, memsize) in stdout
+        assert str(node2) + '\n' + str(node1) in stdout
+
+        # test deletion
+        key = node2.key
+        del cache[key]
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        curr_size -= 1
+        curr_memsize -= node2.plan.work_area.mem.size
+        stdout = intercept_stdout(cache.show_info)
+        assert '{0} / {1} (counts)'.format(curr_size, size) in stdout
+        assert '{0} / {1} (bytes)'.format(curr_memsize, memsize) in stdout
+        assert str(node2) not in stdout
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.multi_gpu(2)
+    @pytest.mark.skipif(runtime.is_hip,
+                        reason="hipFFT doesn't support multi-GPU")
+    def test_LRU_cache11(self):
+        # test if collectively deleting a multi-GPU plan works
+        _skip_multi_gpu_bug((128,), self.gpus)
+        cache0 = self.caches[0]
+        cache1 = self.caches[1]
+
+        # ensure a fresh state
+        assert cache0.get_curr_size() == 0 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+
+        # do a multi-GPU FFT
+        c = testing.shaped_random((128,), cupy, cupy.complex64)
+        cupy.fft.fft(c)
+        assert cache0.get_curr_size() == 1 <= cache0.get_size()
+        assert cache1.get_curr_size() == 1 <= cache1.get_size()
+
+        node0 = next(iter(cache0))[1]
+        node1 = next(iter(cache1))[1]
+        assert node0.key == node1.key
+        assert node0.plan is node1.plan
+        assert cache0.get_curr_memsize() == node0.memsize > 0
+        assert cache1.get_curr_memsize() == node1.memsize > 0
+
+        # delete
+        del cache0[node0.key]
+        assert cache0.get_curr_size() == 0 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+        assert cache0.get_curr_memsize() == 0
+        assert cache1.get_curr_memsize() == 0
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.multi_gpu(2)
+    @pytest.mark.skipif(runtime.is_hip,
+                        reason="hipFFT doesn't support multi-GPU")
+    def test_LRU_cache12(self):
+        # test if an error is raise when one of the caches is unable
+        # to fit it a multi-GPU plan
+        cache0 = self.caches[0]
+        cache1 = self.caches[1]
+
+        # ensure a fresh state
+        assert cache0.get_curr_size() == 0 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+
+        # make it impossible to cache
+        cache1.set_memsize(1)
+
+        # do a multi-GPU FFT
+        with pytest.raises(RuntimeError) as e:
+            c = testing.shaped_random((128,), cupy, cupy.complex64)
+            cupy.fft.fft(c)
+        assert 'plan memsize is too large for device 1' in str(e.value)
+        assert cache0.get_curr_size() == 0 <= cache0.get_size()
+        assert cache1.get_curr_size() == 0 <= cache1.get_size()
+
+    @unittest.skipIf(runtime.is_hip, "rocFFT has different plan sizes")
+    @unittest.skipIf(runtime.runtimeGetVersion() >= 11080,
+                     "CUDA 11.8 has different plan size")
+    def test_LRU_cache13(self):
+        # test if plan insertion respect the memory size limit
+        cache = config.get_plan_cache()
+        cache.set_memsize(1024)
+
+        # ensure a fresh state
+        assert cache.get_curr_size() == 0 <= cache.get_size()
+
+        # On CUDA 10.0 + sm75, this generates a plan of size 1024 bytes
+        a = testing.shaped_random((128,), cupy, cupy.complex64)
+        cupy.fft.ifft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        assert cache.get_curr_memsize() == 1024 == cache.get_memsize()
+
+        # a second plan (of same size) is generated, but the cache is full,
+        # so the first plan is evicted
+        a = testing.shaped_random((64,), cupy, cupy.complex128)
+        cupy.fft.ifft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        assert cache.get_curr_memsize() == 1024 == cache.get_memsize()
+        plan = next(iter(cache))[1].plan
+
+        # this plan is twice as large, so won't fit in
+        a = testing.shaped_random((128,), cupy, cupy.complex128)
+        with pytest.raises(RuntimeError) as e:
+            cupy.fft.ifft(a)
+        assert 'memsize is too large' in str(e.value)
+        # the cache remains intact
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        assert cache.get_curr_memsize() == 1024 == cache.get_memsize()
+        plan1 = next(iter(cache))[1].plan
+        assert plan1 is plan
+
+        # double the cache size would make the plan just fit (and evict
+        # the existing one)
+        cache.set_memsize(2048)
+        cupy.fft.ifft(a)
+        assert cache.get_curr_size() == 1 <= cache.get_size()
+        assert cache.get_curr_memsize() == 2048 == cache.get_memsize()
+        plan2 = next(iter(cache))[1].plan
+        assert plan2 is not plan
diff --git a/tests/cupy_tests/fft_tests/test_callback.py b/tests/cupy_tests/fft_tests/test_callback.py
new file mode 100644
index 0000000..ed6683b
--- /dev/null
+++ b/tests/cupy_tests/fft_tests/test_callback.py
@@ -0,0 +1,708 @@
+import contextlib
+import string
+import sys
+import tempfile
+from unittest import mock
+
+import numpy as np
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@contextlib.contextmanager
+def use_temporary_cache_dir():
+    target = 'cupy.fft._callback.get_cache_dir'
+    with tempfile.TemporaryDirectory() as path:
+        with mock.patch(target, lambda: path):
+            yield path
+
+
+_load_callback = r'''
+__device__ ${data_type} CB_ConvertInput(
+    void* dataIn, size_t offset, void* callerInfo, void* sharedPtr)
+{
+    ${data_type} x = ((${data_type}*)dataIn)[offset];
+    ${element} *= 2.5;
+    return x;
+}
+
+__device__ ${load_type} d_loadCallbackPtr = CB_ConvertInput;
+'''
+
+_load_callback_with_aux = r'''
+__device__ ${data_type} CB_ConvertInput(
+    void* dataIn, size_t offset, void* callerInfo, void* sharedPtr)
+{
+    ${data_type} x = ((${data_type}*)dataIn)[offset];
+    ${element} *= *((${aux_type}*)callerInfo);
+    return x;
+}
+
+__device__ ${load_type} d_loadCallbackPtr = CB_ConvertInput;
+'''
+
+_load_callback_with_aux2 = r'''
+__device__ ${data_type} CB_ConvertInput(
+    void* dataIn, size_t offset, void* callerInfo, void* sharedPtr)
+{
+    ${data_type} x = ((${data_type}*)dataIn)[offset];
+    ${element} *= ((${aux_type}*)callerInfo)[offset];
+    return x;
+}
+
+__device__ ${load_type} d_loadCallbackPtr = CB_ConvertInput;
+'''
+
+_store_callback = r'''
+__device__ void CB_ConvertOutput(
+    void *dataOut, size_t offset, ${data_type} element,
+    void *callerInfo, void *sharedPointer)
+{
+    ${data_type} x = element;
+    ${element} /= 3.8;
+    ((${data_type}*)dataOut)[offset] = x;
+}
+
+__device__ ${store_type} d_storeCallbackPtr = CB_ConvertOutput;
+'''
+
+_store_callback_with_aux = r'''
+__device__ void CB_ConvertOutput(
+    void *dataOut, size_t offset, ${data_type} element,
+    void *callerInfo, void *sharedPointer)
+{
+    ${data_type} x = element;
+    ${element} /= *((${aux_type}*)callerInfo);
+    ((${data_type}*)dataOut)[offset] = x;
+}
+
+__device__ ${store_type} d_storeCallbackPtr = CB_ConvertOutput;
+'''
+
+
+def _set_load_cb(code, element, data_type, callback_type, aux_type=None):
+    return string.Template(code).substitute(
+        data_type=data_type,
+        aux_type=aux_type,
+        load_type=callback_type,
+        element=element)
+
+
+def _set_store_cb(code, element, data_type, callback_type, aux_type=None):
+    return string.Template(code).substitute(
+        data_type=data_type,
+        aux_type=aux_type,
+        store_type=callback_type,
+        element=element)
+
+
+@testing.parameterize(*testing.product({
+    'n': [None, 5, 10, 15],
+    'shape': [(10, 7), (10,), (10, 10)],
+    'norm': [None, 'ortho'],
+}))
+@testing.with_requires('cython>=0.29.0')
+@pytest.mark.skipif(not sys.platform.startswith('linux'),
+                    reason='callbacks are only supported on Linux')
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='hipFFT does not support callbacks')
+class Test1dCallbacks:
+
+    def _test_load_helper(self, xp, dtype, fft_func):
+        fft = getattr(xp.fft, fft_func)
+        code = _load_callback
+        if dtype == np.complex64:
+            types = ('x.x', 'cufftComplex', 'cufftCallbackLoadC')
+        elif dtype == np.complex128:
+            types = ('x.x', 'cufftDoubleComplex', 'cufftCallbackLoadZ')
+        elif dtype == np.float32:
+            types = ('x', 'cufftReal', 'cufftCallbackLoadR')
+        else:
+            types = ('x', 'cufftDoubleReal', 'cufftCallbackLoadD')
+        cb_load = _set_load_cb(code, *types)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is np:
+            a.real *= 2.5
+            out = fft(a, n=self.n, norm=self.norm)
+            if dtype in (np.float32, np.complex64):
+                if fft_func != 'irfft':
+                    out = out.astype(np.complex64)
+                else:
+                    out = out.astype(np.float32)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(cb_load=cb_load):
+                    out = fft(a, n=self.n, norm=self.norm)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_fft_load(self, xp, dtype):
+        return self._test_load_helper(xp, dtype, 'fft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_ifft_load(self, xp, dtype):
+        return self._test_load_helper(xp, dtype, 'ifft')
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_rfft_load(self, xp, dtype):
+        return self._test_load_helper(xp, dtype, 'rfft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_irfft_load(self, xp, dtype):
+        return self._test_load_helper(xp, dtype, 'irfft')
+
+    def _test_store_helper(self, xp, dtype, fft_func):
+        fft = getattr(xp.fft, fft_func)
+        code = _store_callback
+        if dtype == np.complex64:
+            if fft_func != 'irfft':
+                types = ('x.y', 'cufftComplex', 'cufftCallbackStoreC')
+            else:
+                types = ('x', 'cufftReal', 'cufftCallbackStoreR')
+        elif dtype == np.complex128:
+            if fft_func != 'irfft':
+                types = ('x.y', 'cufftDoubleComplex', 'cufftCallbackStoreZ')
+            else:
+                types = ('x', 'cufftDoubleReal', 'cufftCallbackStoreD')
+        elif dtype == np.float32:
+            types = ('x.y', 'cufftComplex', 'cufftCallbackStoreC')
+        elif dtype == np.float64:
+            types = ('x.y', 'cufftDoubleComplex', 'cufftCallbackStoreZ')
+        cb_store = _set_store_cb(code, *types)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is np:
+            out = fft(a, n=self.n, norm=self.norm)
+            if fft_func != 'irfft':
+                out.imag /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.complex64)
+            else:
+                out /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.float32)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(cb_store=cb_store):
+                    out = fft(a, n=self.n, norm=self.norm)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_fft_store(self, xp, dtype):
+        return self._test_store_helper(xp, dtype, 'fft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_ifft_store(self, xp, dtype):
+        return self._test_store_helper(xp, dtype, 'ifft')
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_rfft_store(self, xp, dtype):
+        return self._test_store_helper(xp, dtype, 'rfft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_irfft_store(self, xp, dtype):
+        return self._test_store_helper(xp, dtype, 'irfft')
+
+    def _test_load_store_helper(self, xp, dtype, fft_func):
+        fft = getattr(xp.fft, fft_func)
+        load_code = _load_callback
+        store_code = _store_callback
+        if fft_func in ('fft', 'ifft'):
+            if dtype == np.complex64:
+                load_types = ('x.x', 'cufftComplex', 'cufftCallbackLoadC')
+                store_types = ('x.y', 'cufftComplex', 'cufftCallbackStoreC')
+            else:
+                load_types = (
+                    'x.x', 'cufftDoubleComplex', 'cufftCallbackLoadZ')
+                store_types = (
+                    'x.y', 'cufftDoubleComplex', 'cufftCallbackStoreZ')
+        elif fft_func == 'rfft':
+            if dtype == np.float32:
+                load_types = ('x', 'cufftReal', 'cufftCallbackLoadR')
+                store_types = ('x.y', 'cufftComplex', 'cufftCallbackStoreC')
+            else:
+                load_types = ('x', 'cufftDoubleReal', 'cufftCallbackLoadD')
+                store_types = (
+                    'x.y', 'cufftDoubleComplex', 'cufftCallbackStoreZ')
+        else:  # irfft
+            if dtype == np.complex64:
+                load_types = ('x.x', 'cufftComplex', 'cufftCallbackLoadC')
+                store_types = ('x', 'cufftReal', 'cufftCallbackStoreR')
+            else:
+                load_types = (
+                    'x.x', 'cufftDoubleComplex', 'cufftCallbackLoadZ')
+                store_types = ('x', 'cufftDoubleReal', 'cufftCallbackStoreD')
+        cb_load = _set_load_cb(load_code, *load_types)
+        cb_store = _set_store_cb(store_code, *store_types)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is np:
+            a.real *= 2.5
+            out = fft(a, n=self.n, norm=self.norm)
+            if fft_func != 'irfft':
+                out.imag /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.complex64)
+            else:
+                out /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.float32)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(
+                        cb_load=cb_load, cb_store=cb_store):
+                    out = fft(a, n=self.n, norm=self.norm)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_fft_load_store(self, xp, dtype):
+        return self._test_load_store_helper(xp, dtype, 'fft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_ifft_load_store(self, xp, dtype):
+        return self._test_load_store_helper(xp, dtype, 'ifft')
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_rfft_load_store(self, xp, dtype):
+        return self._test_load_store_helper(xp, dtype, 'rfft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_irfft_load_store(self, xp, dtype):
+        return self._test_load_store_helper(xp, dtype, 'irfft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_fft_load_aux(self, xp, dtype):
+        fft = xp.fft.fft
+        c = _load_callback_with_aux2
+        if dtype == np.complex64:
+            cb_load = _set_load_cb(
+                c, 'x.x', 'cufftComplex', 'cufftCallbackLoadC', 'float')
+        else:
+            cb_load = _set_load_cb(
+                c, 'x.x', 'cufftDoubleComplex', 'cufftCallbackLoadZ', 'double')
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out_last = self.n if self.n is not None else self.shape[-1]
+        out_shape = list(self.shape)
+        out_shape[-1] = out_last
+        last_min = min(self.shape[-1], out_last)
+        b = xp.arange(np.prod(out_shape), dtype=xp.dtype(dtype).char.lower())
+        b = b.reshape(out_shape)
+        if xp is np:
+            x = np.zeros(out_shape, dtype=dtype)
+            x[..., 0:last_min] = a[..., 0:last_min]
+            x.real *= b
+            out = fft(x, n=self.n, norm=self.norm)
+            if dtype in (np.float32, np.complex64):
+                out = out.astype(np.complex64)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(
+                        cb_load=cb_load, cb_load_aux_arr=b):
+                    out = fft(a, n=self.n, norm=self.norm)
+
+        return out
+
+    def _test_load_store_aux_helper(self, xp, dtype, fft_func):
+        fft = getattr(xp.fft, fft_func)
+        load_code = _load_callback_with_aux
+        store_code = _store_callback_with_aux
+        if xp is cupy:
+            load_aux = xp.asarray(2.5, dtype=xp.dtype(dtype).char.lower())
+            store_aux = xp.asarray(3.8, dtype=xp.dtype(dtype).char.lower())
+
+        if fft_func in ('fft', 'ifft'):
+            if dtype == np.complex64:
+                load_types = (
+                    'x.x', 'cufftComplex', 'cufftCallbackLoadC', 'float')
+                store_types = (
+                    'x.y', 'cufftComplex', 'cufftCallbackStoreC', 'float')
+            else:
+                load_types = ('x.x', 'cufftDoubleComplex',
+                              'cufftCallbackLoadZ', 'double')
+                store_types = ('x.y', 'cufftDoubleComplex',
+                               'cufftCallbackStoreZ', 'double')
+        elif fft_func == 'rfft':
+            if dtype == np.float32:
+                load_types = (
+                    'x', 'cufftReal', 'cufftCallbackLoadR', 'float')
+                store_types = (
+                    'x.y', 'cufftComplex', 'cufftCallbackStoreC', 'float')
+            else:
+                load_types = (
+                    'x', 'cufftDoubleReal', 'cufftCallbackLoadD', 'double')
+                store_types = ('x.y', 'cufftDoubleComplex',
+                               'cufftCallbackStoreZ', 'double')
+        else:  # irfft
+            if dtype == np.complex64:
+                load_types = (
+                    'x.x', 'cufftComplex', 'cufftCallbackLoadC', 'float')
+                store_types = (
+                    'x', 'cufftReal', 'cufftCallbackStoreR', 'float')
+            else:
+                load_types = ('x.x', 'cufftDoubleComplex',
+                              'cufftCallbackLoadZ', 'double')
+                store_types = ('x', 'cufftDoubleReal',
+                               'cufftCallbackStoreD', 'double')
+        cb_load = _set_load_cb(load_code, *load_types)
+        cb_store = _set_store_cb(store_code, *store_types)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is np:
+            a.real *= 2.5
+            out = fft(a, n=self.n, norm=self.norm)
+            if fft_func != 'irfft':
+                out.imag /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.complex64)
+            else:
+                out /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.float32)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(
+                        cb_load=cb_load, cb_store=cb_store,
+                        cb_load_aux_arr=load_aux, cb_store_aux_arr=store_aux):
+                    out = fft(a, n=self.n, norm=self.norm)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_fft_load_store_aux(self, xp, dtype):
+        return self._test_load_store_aux_helper(xp, dtype, 'fft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_ifft_load_store_aux(self, xp, dtype):
+        return self._test_load_store_aux_helper(xp, dtype, 'ifft')
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_rfft_load_store_aux(self, xp, dtype):
+        return self._test_load_store_aux_helper(xp, dtype, 'rfft')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_irfft_load_store_aux(self, xp, dtype):
+        return self._test_load_store_aux_helper(xp, dtype, 'irfft')
+
+
+@testing.parameterize(
+    {'shape': (3, 4), 's': None, 'axes': None, 'norm': None},
+    {'shape': (3, 4), 's': (1, None), 'axes': None, 'norm': None},
+    {'shape': (3, 4), 's': (1, 5), 'axes': None, 'norm': None},
+    {'shape': (3, 4), 's': None, 'axes': (-2, -1), 'norm': None},
+    {'shape': (3, 4), 's': None, 'axes': None, 'norm': 'ortho'},
+    {'shape': (2, 3, 4), 's': None, 'axes': None, 'norm': None},
+    {'shape': (2, 3, 4), 's': (1, 4, None), 'axes': None, 'norm': None},
+    {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None, 'norm': None},
+    {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1), 'norm': None},
+    {'shape': (2, 3, 4), 's': None, 'axes': None, 'norm': 'ortho'},
+    {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2), 'norm': 'ortho'},
+)
+@testing.with_requires('cython>=0.29.0')
+@pytest.mark.skipif(not sys.platform.startswith('linux'),
+                    reason='callbacks are only supported on Linux')
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='hipFFT does not support callbacks')
+class TestNdCallbacks:
+
+    def _test_load_helper(self, xp, dtype, fft_func):
+        fft = getattr(xp.fft, fft_func)
+        load_code = _load_callback
+        if dtype == np.complex64:
+            types = ('x.x', 'cufftComplex', 'cufftCallbackLoadC')
+        elif dtype == np.complex128:
+            types = ('x.x', 'cufftDoubleComplex', 'cufftCallbackLoadZ')
+        elif dtype == np.float32:
+            types = ('x', 'cufftReal', 'cufftCallbackLoadR')
+        else:
+            types = ('x', 'cufftDoubleReal', 'cufftCallbackLoadD')
+        cb_load = _set_load_cb(load_code, *types)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is np:
+            a.real *= 2.5
+            out = fft(a, s=self.s, axes=self.axes, norm=self.norm)
+            if dtype in (np.float32, np.complex64):
+                if fft_func != 'irfftn':
+                    out = out.astype(np.complex64)
+                else:
+                    out = out.astype(np.float32)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(cb_load=cb_load):
+                    out = fft(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_load(self, xp, dtype):
+        return self._test_load_helper(xp, dtype, 'fftn')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_load(self, xp, dtype):
+        return self._test_load_helper(xp, dtype, 'ifftn')
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_load(self, xp, dtype):
+        return self._test_load_helper(xp, dtype, 'rfftn')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_load(self, xp, dtype):
+        return self._test_load_helper(xp, dtype, 'irfftn')
+
+    def _test_store_helper(self, xp, dtype, fft_func):
+        fft = getattr(xp.fft, fft_func)
+        store_code = _store_callback
+        if dtype == np.complex64:
+            if fft_func != 'irfftn':
+                types = ('x.y', 'cufftComplex', 'cufftCallbackStoreC')
+            else:
+                types = ('x', 'cufftReal', 'cufftCallbackStoreR')
+        elif dtype == np.complex128:
+            if fft_func != 'irfftn':
+                types = ('x.y', 'cufftDoubleComplex', 'cufftCallbackStoreZ')
+            else:
+                types = ('x', 'cufftDoubleReal', 'cufftCallbackStoreD')
+        elif dtype == np.float32:
+            types = ('x.y', 'cufftComplex', 'cufftCallbackStoreC')
+        elif dtype == np.float64:
+            types = ('x.y', 'cufftDoubleComplex', 'cufftCallbackStoreZ')
+        cb_store = _set_store_cb(store_code, *types)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is np:
+            out = fft(a, s=self.s, axes=self.axes, norm=self.norm)
+            if fft_func != 'irfftn':
+                out.imag /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.complex64)
+            else:
+                out /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.float32)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(cb_store=cb_store):
+                    out = fft(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_store(self, xp, dtype):
+        return self._test_store_helper(xp, dtype, 'fftn')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_store(self, xp, dtype):
+        return self._test_store_helper(xp, dtype, 'ifftn')
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_store(self, xp, dtype):
+        return self._test_store_helper(xp, dtype, 'rfftn')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_store(self, xp, dtype):
+        return self._test_store_helper(xp, dtype, 'irfftn')
+
+    def _test_load_store_helper(self, xp, dtype, fft_func):
+        fft = getattr(xp.fft, fft_func)
+        load_code = _load_callback
+        store_code = _store_callback
+        if fft_func in ('fftn', 'ifftn'):
+            if dtype == np.complex64:
+                load_types = ('x.x', 'cufftComplex', 'cufftCallbackLoadC')
+                store_types = ('x.y', 'cufftComplex', 'cufftCallbackStoreC')
+            else:
+                load_types = (
+                    'x.x', 'cufftDoubleComplex', 'cufftCallbackLoadZ')
+                store_types = (
+                    'x.y', 'cufftDoubleComplex', 'cufftCallbackStoreZ')
+        elif fft_func == 'rfftn':
+            if dtype == np.float32:
+                load_types = ('x', 'cufftReal', 'cufftCallbackLoadR')
+                store_types = ('x.y', 'cufftComplex', 'cufftCallbackStoreC')
+            else:
+                load_types = ('x', 'cufftDoubleReal', 'cufftCallbackLoadD')
+                store_types = (
+                    'x.y', 'cufftDoubleComplex', 'cufftCallbackStoreZ')
+        else:  # irfft
+            if dtype == np.complex64:
+                load_types = ('x.x', 'cufftComplex', 'cufftCallbackLoadC')
+                store_types = ('x', 'cufftReal', 'cufftCallbackStoreR')
+            else:
+                load_types = (
+                    'x.x', 'cufftDoubleComplex', 'cufftCallbackLoadZ')
+                store_types = ('x', 'cufftDoubleReal', 'cufftCallbackStoreD')
+        cb_load = _set_load_cb(load_code, *load_types)
+        cb_store = _set_store_cb(store_code, *store_types)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is np:
+            a.real *= 2.5
+            out = fft(a, s=self.s, axes=self.axes, norm=self.norm)
+            if fft_func != 'irfftn':
+                out.imag /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.complex64)
+            else:
+                out /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.float32)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(
+                        cb_load=cb_load, cb_store=cb_store):
+                    out = fft(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_load_store(self, xp, dtype):
+        return self._test_load_store_helper(xp, dtype, 'fftn')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_load_store(self, xp, dtype):
+        return self._test_load_store_helper(xp, dtype, 'ifftn')
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_load_store(self, xp, dtype):
+        return self._test_load_store_helper(xp, dtype, 'rfftn')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_load_store(self, xp, dtype):
+        return self._test_load_store_helper(xp, dtype, 'irfftn')
+
+    def _test_load_store_aux_helper(self, xp, dtype, fft_func):
+        fft = getattr(xp.fft, fft_func)
+        load_code = _load_callback_with_aux
+        store_code = _store_callback_with_aux
+        if xp is cupy:
+            load_aux = xp.asarray(2.5, dtype=xp.dtype(dtype).char.lower())
+            store_aux = xp.asarray(3.8, dtype=xp.dtype(dtype).char.lower())
+
+        if fft_func in ('fftn', 'ifftn'):
+            if dtype == np.complex64:
+                load_types = (
+                    'x.x', 'cufftComplex', 'cufftCallbackLoadC', 'float')
+                store_types = (
+                    'x.y', 'cufftComplex', 'cufftCallbackStoreC', 'float')
+            else:
+                load_types = ('x.x', 'cufftDoubleComplex',
+                              'cufftCallbackLoadZ', 'double')
+                store_types = ('x.y', 'cufftDoubleComplex',
+                               'cufftCallbackStoreZ', 'double')
+        elif fft_func == 'rfftn':
+            if dtype == np.float32:
+                load_types = (
+                    'x', 'cufftReal', 'cufftCallbackLoadR', 'float')
+                store_types = (
+                    'x.y', 'cufftComplex', 'cufftCallbackStoreC', 'float')
+            else:
+                load_types = (
+                    'x', 'cufftDoubleReal', 'cufftCallbackLoadD', 'double')
+                store_types = ('x.y', 'cufftDoubleComplex',
+                               'cufftCallbackStoreZ', 'double')
+        else:  # irfftn
+            if dtype == np.complex64:
+                load_types = (
+                    'x.x', 'cufftComplex', 'cufftCallbackLoadC', 'float')
+                store_types = (
+                    'x', 'cufftReal', 'cufftCallbackStoreR', 'float')
+            else:
+                load_types = ('x.x', 'cufftDoubleComplex',
+                              'cufftCallbackLoadZ', 'double')
+                store_types = ('x', 'cufftDoubleReal',
+                               'cufftCallbackStoreD', 'double')
+        cb_load = _set_load_cb(load_code, *load_types)
+        cb_store = _set_store_cb(store_code, *store_types)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is np:
+            a.real *= 2.5
+            out = fft(a, s=self.s, axes=self.axes, norm=self.norm)
+            if fft_func != 'irfftn':
+                out.imag /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.complex64)
+            else:
+                out /= 3.8
+                if dtype in (np.float32, np.complex64):
+                    out = out.astype(np.float32)
+        else:
+            with use_temporary_cache_dir():
+                with xp.fft.config.set_cufft_callbacks(
+                        cb_load=cb_load, cb_store=cb_store,
+                        cb_load_aux_arr=load_aux, cb_store_aux_arr=store_aux):
+                    out = fft(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_load_store_aux(self, xp, dtype):
+        return self._test_load_store_aux_helper(xp, dtype, 'fftn')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_load_store_aux(self, xp, dtype):
+        return self._test_load_store_aux_helper(xp, dtype, 'ifftn')
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_load_store_aux(self, xp, dtype):
+        return self._test_load_store_aux_helper(xp, dtype, 'rfftn')
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_load_store_aux(self, xp, dtype):
+        return self._test_load_store_aux_helper(xp, dtype, 'irfftn')
diff --git a/tests/cupy_tests/fft_tests/test_fft.py b/tests/cupy_tests/fft_tests/test_fft.py
new file mode 100644
index 0000000..e9e2528
--- /dev/null
+++ b/tests/cupy_tests/fft_tests/test_fft.py
@@ -0,0 +1,1362 @@
+import functools
+
+import numpy as np
+import pytest
+
+import cupy
+from cupy.fft import config
+from cupy.fft._fft import (_default_fft_func, _fft, _fftn,
+                           _size_last_transform_axis)
+from cupy import testing
+from cupy.testing._loops import _wraps_partial
+
+
+@pytest.fixture
+def skip_forward_backward(request):
+    if request.instance.norm in ('backward', 'forward'):
+        if not (np.lib.NumpyVersion(np.__version__) >= '1.20.0'):
+            pytest.skip('forward/backward is supported by NumPy 1.20+')
+
+
+def nd_planning_states(states=[True, False], name='enable_nd'):
+    """Decorator for parameterized tests with and wihout nd planning
+
+    Tests are repeated with config.enable_nd_planning set to True and False
+
+    Args:
+         states(list of bool): The boolean cases to test.
+         name(str): Argument name to which specified dtypes are passed.
+
+    This decorator adds a keyword argument specified by ``name``
+    to the test fixture. Then, it runs the fixtures in parallel
+    by passing the each element of ``dtypes`` to the named
+    argument.
+    """
+    def decorator(impl):
+        @_wraps_partial(impl, name)
+        def test_func(self, *args, **kw):
+            # get original global planning state
+            planning_state = config.enable_nd_planning
+            try:
+                for nd_planning in states:
+                    try:
+                        # enable or disable nd planning
+                        config.enable_nd_planning = nd_planning
+
+                        kw[name] = nd_planning
+                        impl(self, *args, **kw)
+                    except Exception:
+                        print(name, 'is', nd_planning)
+                        raise
+            finally:
+                # restore original global planning state
+                config.enable_nd_planning = planning_state
+
+        return test_func
+    return decorator
+
+
+def multi_gpu_config(gpu_configs=None):
+    """Decorator for parameterized tests with different GPU configurations.
+
+    Args:
+        gpu_configs (list of list): The GPUs to test.
+
+    .. notes:
+        The decorated tests are skipped if no or only one GPU is available.
+    """
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kw):
+            use_multi_gpus = config.use_multi_gpus
+            _devices = config._devices
+
+            try:
+                for gpus in gpu_configs:
+                    try:
+                        nGPUs = len(gpus)
+                        assert nGPUs >= 2, 'Must use at least two gpus'
+                        config.use_multi_gpus = True
+                        config.set_cufft_gpus(gpus)
+                        self.gpus = gpus
+
+                        impl(self, *args, **kw)
+                    except Exception:
+                        print('GPU config is:', gpus)
+                        raise
+            finally:
+                config.use_multi_gpus = use_multi_gpus
+                config._devices = _devices
+                del self.gpus
+
+        return test_func
+    return decorator
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*testing.product({
+    'n': [None, 0, 5, 10, 15],
+    'shape': [(0,), (10, 0), (10,), (10, 10)],
+    'norm': [None, 'backward', 'ortho', 'forward', ''],
+}))
+class TestFft:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.fft(a, n=self.n, norm=self.norm)
+
+        # np.fft.fft always returns np.complex128
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    # NumPy 1.17.0 and 1.17.1 raises ZeroDivisonError due to a bug
+    @testing.with_requires('numpy!=1.17.0')
+    @testing.with_requires('numpy!=1.17.1')
+    def test_ifft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.ifft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(0, 10), (10, 0, 10), (10, 10), (10, 5, 10)],
+    'data_order': ['F', 'C'],
+    'axis': [0, 1, -1],
+}))
+class TestFftOrder:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.data_order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.fft(a, axis=self.axis)
+
+        # np.fft.fft always returns np.complex128
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.data_order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.ifft(a, axis=self.axis)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+
+# See #3757 and NVIDIA internal ticket 3093094
+def _skip_multi_gpu_bug(shape, gpus):
+    # avoid CUDA 11.0 (will be fixed by CUDA 11.2) bug triggered by
+    # - batch = 1
+    # - gpus = [1, 0]
+    if (11000 <= cupy.cuda.runtime.runtimeGetVersion() < 11020
+            and len(shape) == 1
+            and gpus == [1, 0]):
+        pytest.skip('avoid CUDA 11 bug')
+
+
+# Almost identical to the TestFft class, except that
+# 1. multi-GPU cuFFT is used
+# 2. the tested parameter combinations are adjusted to meet the requirements
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*testing.product({
+    'n': [None, 0, 64],
+    'shape': [(0,), (0, 10), (64,), (4, 64)],
+    'norm': [None, 'backward', 'ortho', 'forward', ''],
+}))
+@testing.multi_gpu(2)
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='hipFFT does not support multi-GPU FFT')
+class TestMultiGpuFft:
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft(self, xp, dtype):
+        _skip_multi_gpu_bug(self.shape, self.gpus)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.fft(a, n=self.n, norm=self.norm)
+
+        # np.fft.fft always returns np.complex128
+        if xp is np and dtype is np.complex64:
+            out = out.astype(dtype)
+
+        return out
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    # NumPy 1.17.0 and 1.17.1 raises ZeroDivisonError due to a bug
+    @testing.with_requires('numpy!=1.17.0')
+    @testing.with_requires('numpy!=1.17.1')
+    def test_ifft(self, xp, dtype):
+        _skip_multi_gpu_bug(self.shape, self.gpus)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.ifft(a, n=self.n, norm=self.norm)
+
+        # np.fft.fft always returns np.complex128
+        if xp is np and dtype is np.complex64:
+            out = out.astype(dtype)
+
+        return out
+
+
+# Almost identical to the TestFftOrder class, except that
+# 1. multi-GPU cuFFT is used
+# 2. the tested parameter combinations are adjusted to meet the requirements
+@testing.parameterize(*testing.product({
+    'shape': [(10, 10), (10, 5, 10)],
+    'data_order': ['F', 'C'],
+    'axis': [0, 1, -1],
+}))
+@testing.multi_gpu(2)
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='hipFFT does not support multi-GPU FFT')
+class TestMultiGpuFftOrder:
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft(self, xp, dtype):
+        _skip_multi_gpu_bug(self.shape, self.gpus)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.data_order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.fft(a, axis=self.axis)
+
+        # np.fft.fft always returns np.complex128
+        if xp is np and dtype is np.complex64:
+            out = out.astype(dtype)
+
+        return out
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft(self, xp, dtype):
+        _skip_multi_gpu_bug(self.shape, self.gpus)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.data_order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.ifft(a, axis=self.axis)
+
+        # np.fft.fft always returns np.complex128
+        if xp is np and dtype is np.complex64:
+            out = out.astype(dtype)
+
+        return out
+
+
+class TestDefaultPlanType:
+
+    @nd_planning_states()
+    def test_default_fft_func(self, enable_nd):
+        # test cases where nd cuFFT plan is possible
+        ca = cupy.ones((16, 16, 16))
+        for axes in [(0, 1), (1, 2), None, (0, 1, 2)]:
+            fft_func = _default_fft_func(ca, axes=axes)
+            if enable_nd:
+                # TODO(leofang): test newer ROCm versions
+                if axes == (0, 1) and cupy.cuda.runtime.is_hip:
+                    assert fft_func is _fft
+                else:
+                    assert fft_func is _fftn
+            else:
+                assert fft_func is _fft
+
+        # only a single axis is transformed -> 1d plan preferred
+        for axes in [(0, ), (1, ), (2, )]:
+            assert _default_fft_func(ca, axes=axes) is _fft
+
+        # non-contiguous axes -> nd plan not possible
+        assert _default_fft_func(ca, axes=(0, 2)) is _fft
+
+        # >3 axes transformed -> nd plan not possible
+        ca = cupy.ones((2, 4, 6, 8))
+        assert _default_fft_func(ca) is _fft
+
+        # first or last axis not included -> nd plan not possible
+        assert _default_fft_func(ca, axes=(1, )) is _fft
+
+        # for rfftn
+        ca = cupy.random.random((4, 2, 6))
+        for s, axes in zip([(3, 4), None, (8, 7, 5)],
+                           [(-2, -1), (0, 1), None]):
+            fft_func = _default_fft_func(ca, s=s, axes=axes, value_type='R2C')
+            if enable_nd:
+                # TODO(leofang): test newer ROCm versions
+                if axes == (0, 1) and cupy.cuda.runtime.is_hip:
+                    assert fft_func is _fft
+                else:
+                    assert fft_func is _fftn
+            else:
+                assert fft_func is _fft
+
+        # nd plan not possible if last axis is not 0 or ndim-1
+        assert _default_fft_func(ca, axes=(2, 1), value_type='R2C') is _fft
+
+        # for irfftn
+        ca = cupy.random.random((4, 2, 6)).astype(cupy.complex128)
+        for s, axes in zip([(3, 4), None, (8, 7, 5)],
+                           [(-2, -1), (0, 1), None]):
+            fft_func = _default_fft_func(ca, s=s, axes=axes, value_type='C2R')
+            if enable_nd:
+                # To get around hipFFT's bug, we don't use PlanNd for C2R
+                # TODO(leofang): test newer ROCm versions
+                if cupy.cuda.runtime.is_hip:
+                    assert fft_func is _fft
+                else:
+                    assert fft_func is _fftn
+            else:
+                assert fft_func is _fft
+
+        # nd plan not possible if last axis is not 0 or ndim-1
+        assert _default_fft_func(ca, axes=(2, 1), value_type='C2R') is _fft
+
+
+@pytest.mark.skipif(10010 <= cupy.cuda.runtime.runtimeGetVersion() <= 11010,
+                    reason='avoid a cuFFT bug (cupy/cupy#3777)')
+@testing.slow
+class TestFftAllocate:
+
+    def test_fft_allocate(self):
+        # Check CuFFTError is not raised when the GPU memory is enough.
+        # See https://github.com/cupy/cupy/issues/1063
+        # TODO(mizuno): Simplify "a" after memory compaction is implemented.
+        a = []
+        for i in range(10):
+            a.append(cupy.empty(100000000))
+        del a
+        b = cupy.empty(100000007, dtype=cupy.float32)
+        cupy.fft.fft(b)
+        # Free huge memory for slow test
+        del b
+        cupy.get_default_memory_pool().free_all_blocks()
+        # Clean up FFT plan cache
+        cupy.fft.config.clear_plan_cache()
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (1, None), 'axes': None},
+        {'shape': (3, 4), 's': (1, 5), 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': ()},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, None), 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': ()},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2)},
+        {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+        {'shape': (0, 5), 's': None, 'axes': None},
+        {'shape': (2, 0, 5), 's': None, 'axes': None},
+        {'shape': (0, 0, 5), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (0, 5), 'axes': None},
+        {'shape': (3, 4), 's': (1, 0), 'axes': None},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestFft2:
+
+    @nd_planning_states()
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft2(self, xp, dtype, order, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.fft2(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if self.axes is not None and not self.axes:
+            assert out is a
+            return out
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @nd_planning_states()
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft2(self, xp, dtype, order, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.ifft2(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if self.axes is not None and not self.axes:
+            assert out is a
+            return out
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (1, None), 'axes': None},
+        {'shape': (3, 4), 's': (1, 5), 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': [-1, -2]},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': ()},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, None), 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': ()},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2)},
+        {'shape': (2, 3, 4), 's': (4, 3, 2), 'axes': (2, 0, 1)},
+        {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+        {'shape': (0, 5), 's': None, 'axes': None},
+        {'shape': (2, 0, 5), 's': None, 'axes': None},
+        {'shape': (0, 0, 5), 's': None, 'axes': None},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestFftn:
+
+    @nd_planning_states()
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn(self, xp, dtype, order, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.fftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if self.axes is not None and not self.axes:
+            assert out is a
+            return out
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @nd_planning_states()
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn(self, xp, dtype, order, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.ifftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if self.axes is not None and not self.axes:
+            assert out is a
+            return out
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (1, 5), 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': None},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2)},
+        {'shape': (0, 5), 's': None, 'axes': None},
+        {'shape': (2, 0, 5), 's': None, 'axes': None},
+        {'shape': (0, 0, 5), 's': None, 'axes': None},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward']})
+    )
+))
+class TestPlanCtxManagerFftn:
+
+    @pytest.fixture(autouse=True)
+    def skip_buggy(self):
+        if cupy.cuda.runtime.is_hip:
+            # TODO(leofang): test newer ROCm versions
+            if (self.axes == (0, 1) and self.shape == (2, 3, 4)):
+                pytest.skip("hipFFT's PlanNd for this case "
+                            "is buggy, so Plan1d is generated "
+                            "instead")
+
+    @nd_planning_states()
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn(self, xp, dtype, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            plan = get_fft_plan(a, self.s, self.axes)
+            with plan:
+                out = xp.fft.fftn(a, s=self.s, axes=self.axes, norm=self.norm)
+        else:
+            out = xp.fft.fftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if xp is np and dtype is np.complex64:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @nd_planning_states()
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn(self, xp, dtype, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            plan = get_fft_plan(a, self.s, self.axes)
+            with plan:
+                out = xp.fft.ifftn(a, s=self.s, axes=self.axes, norm=self.norm)
+        else:
+            out = xp.fft.ifftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if xp is np and dtype is np.complex64:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @nd_planning_states()
+    @testing.for_complex_dtypes()
+    def test_fftn_error_on_wrong_plan(self, dtype, enable_nd):
+        if 0 in self.shape:
+            pytest.skip('0 in shape')
+        # This test ensures the context manager plan is picked up
+
+        from cupyx.scipy.fftpack import get_fft_plan
+        from cupy.fft import fftn
+        assert config.enable_nd_planning == enable_nd
+
+        # can't get a plan, so skip
+        if self.axes is not None:
+            if self.s is not None:
+                if len(self.s) != len(self.axes):
+                    return
+            elif len(self.shape) != len(self.axes):
+                return
+
+        a = testing.shaped_random(self.shape, cupy, dtype)
+        bad_in_shape = tuple(2*i for i in self.shape)
+        if self.s is None:
+            bad_out_shape = bad_in_shape
+        else:
+            bad_out_shape = tuple(2*i for i in self.s)
+        b = testing.shaped_random(bad_in_shape, cupy, dtype)
+        plan_wrong = get_fft_plan(b, bad_out_shape, self.axes)
+
+        with pytest.raises(ValueError) as ex, plan_wrong:
+            fftn(a, s=self.s, axes=self.axes, norm=self.norm)
+        # targeting a particular error
+        assert 'The cuFFT plan and a.shape do not match' in str(ex.value)
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*testing.product({
+    'n': [None, 5, 10, 15],
+    'shape': [(10,), ],
+    'norm': [None, 'backward', 'ortho', 'forward'],
+}))
+class TestPlanCtxManagerFft:
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            shape = (self.n,) if self.n is not None else None
+            plan = get_fft_plan(a, shape=shape)
+            assert isinstance(plan, cupy.cuda.cufft.Plan1d)
+            with plan:
+                out = xp.fft.fft(a, n=self.n, norm=self.norm)
+        else:
+            out = xp.fft.fft(a, n=self.n, norm=self.norm)
+
+        # np.fft.fft always returns np.complex128
+        if xp is np and dtype is np.complex64:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            shape = (self.n,) if self.n is not None else None
+            plan = get_fft_plan(a, shape=shape)
+            assert isinstance(plan, cupy.cuda.cufft.Plan1d)
+            with plan:
+                out = xp.fft.ifft(a, n=self.n, norm=self.norm)
+        else:
+            out = xp.fft.ifft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype is np.complex64:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    def test_fft_error_on_wrong_plan(self, dtype):
+        # This test ensures the context manager plan is picked up
+
+        from cupyx.scipy.fftpack import get_fft_plan
+        from cupy.fft import fft
+
+        a = testing.shaped_random(self.shape, cupy, dtype)
+        bad_shape = tuple(5*i for i in self.shape)
+        b = testing.shaped_random(bad_shape, cupy, dtype)
+        plan_wrong = get_fft_plan(b)
+        assert isinstance(plan_wrong, cupy.cuda.cufft.Plan1d)
+
+        with pytest.raises(ValueError) as ex, plan_wrong:
+            fft(a, n=self.n, norm=self.norm)
+        # targeting a particular error
+        assert 'Target array size does not match the plan.' in str(ex.value)
+
+
+# Almost identical to the TestPlanCtxManagerFft class, except that
+# 1. multi-GPU cuFFT is used
+# 2. the tested parameter combinations are adjusted to meet the requirements
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*testing.product({
+    'n': [None, 64],
+    'shape': [(64,), (128,)],
+    'norm': [None, 'backward', 'ortho', 'forward', ''],
+}))
+@testing.multi_gpu(2)
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='hipFFT does not support multi-GPU FFT')
+class TestMultiGpuPlanCtxManagerFft:
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft(self, xp, dtype):
+        _skip_multi_gpu_bug(self.shape, self.gpus)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            shape = (self.n,) if self.n is not None else None
+            plan = get_fft_plan(a, shape=shape)
+            assert isinstance(plan, cupy.cuda.cufft.Plan1d)
+            with plan:
+                out = xp.fft.fft(a, n=self.n, norm=self.norm)
+        else:
+            out = xp.fft.fft(a, n=self.n, norm=self.norm)
+
+        # np.fft.fft always returns np.complex128
+        if xp is np and dtype is np.complex64:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft(self, xp, dtype):
+        _skip_multi_gpu_bug(self.shape, self.gpus)
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            shape = (self.n,) if self.n is not None else None
+            plan = get_fft_plan(a, shape=shape)
+            assert isinstance(plan, cupy.cuda.cufft.Plan1d)
+            with plan:
+                out = xp.fft.ifft(a, n=self.n, norm=self.norm)
+        else:
+            out = xp.fft.ifft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype is np.complex64:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @multi_gpu_config(gpu_configs=[[0, 1], [1, 0]])
+    @testing.for_complex_dtypes()
+    def test_fft_error_on_wrong_plan(self, dtype):
+        # This test ensures the context manager plan is picked up
+
+        from cupyx.scipy.fftpack import get_fft_plan
+        from cupy.fft import fft
+
+        a = testing.shaped_random(self.shape, cupy, dtype)
+        bad_shape = tuple(4*i for i in self.shape)
+        b = testing.shaped_random(bad_shape, cupy, dtype)
+        plan_wrong = get_fft_plan(b)
+        assert isinstance(plan_wrong, cupy.cuda.cufft.Plan1d)
+
+        with pytest.raises(ValueError) as ex, plan_wrong:
+            fft(a, n=self.n, norm=self.norm)
+        # targeting a particular error
+        if self.norm == '':
+            # if norm is invalid, we still get ValueError, but it's raised
+            # when checking norm, earlier than the plan check
+            return  # skip
+        assert 'Target array size does not match the plan.' in str(ex.value)
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, None), 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4, 5), 's': None, 'axes': (-3, -2, -1)},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestFftnContiguity:
+
+    @nd_planning_states([True])
+    @testing.for_all_dtypes()
+    def test_fftn_orders(self, dtype, enable_nd):
+        for order in ['C', 'F']:
+            a = testing.shaped_random(self.shape, cupy, dtype)
+            if order == 'F':
+                a = cupy.asfortranarray(a)
+            out = cupy.fft.fftn(a, s=self.s, axes=self.axes)
+
+            fft_func = _default_fft_func(a, s=self.s, axes=self.axes)
+            if fft_func is _fftn:
+                # nd plans have output with contiguity matching the input
+                assert out.flags.c_contiguous == a.flags.c_contiguous
+                assert out.flags.f_contiguous == a.flags.f_contiguous
+            else:
+                # 1d planning case doesn't guarantee preserved contiguity
+                pass
+
+    @nd_planning_states([True])
+    @testing.for_all_dtypes()
+    def test_ifftn_orders(self, dtype, enable_nd):
+        for order in ['C', 'F']:
+
+            a = testing.shaped_random(self.shape, cupy, dtype)
+            if order == 'F':
+                a = cupy.asfortranarray(a)
+            out = cupy.fft.ifftn(a, s=self.s, axes=self.axes)
+
+            fft_func = _default_fft_func(a, s=self.s, axes=self.axes)
+            if fft_func is _fftn:
+                # nd plans have output with contiguity matching the input
+                assert out.flags.c_contiguous == a.flags.c_contiguous
+                assert out.flags.f_contiguous == a.flags.f_contiguous
+            else:
+                # 1d planning case doesn't guarantee preserved contiguity
+                pass
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*testing.product({
+    'n': [None, 5, 10, 15],
+    'shape': [(10,), (10, 10)],
+    'norm': [None, 'backward', 'ortho', 'forward', ''],
+}))
+class TestRfft:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.rfft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.irfft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.float32)
+
+        return out
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*testing.product({
+    'n': [None, 5, 10, 15],
+    'shape': [(10,)],
+    'norm': [None, 'backward', 'ortho', 'forward'],
+}))
+class TestPlanCtxManagerRfft:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            shape = (self.n,) if self.n is not None else None
+            plan = get_fft_plan(a, shape=shape, value_type='R2C')
+            assert isinstance(plan, cupy.cuda.cufft.Plan1d)
+            with plan:
+                out = xp.fft.rfft(a, n=self.n, norm=self.norm)
+        else:
+            out = xp.fft.rfft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            shape = (self.n,) if self.n is not None else None
+            plan = get_fft_plan(a, shape=shape, value_type='C2R')
+            assert isinstance(plan, cupy.cuda.cufft.Plan1d)
+            with plan:
+                out = xp.fft.irfft(a, n=self.n, norm=self.norm)
+        else:
+            out = xp.fft.irfft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.float32)
+
+        return out
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_rfft_error_on_wrong_plan(self, dtype):
+        # This test ensures the context manager plan is picked up
+
+        from cupyx.scipy.fftpack import get_fft_plan
+        from cupy.fft import rfft
+
+        a = testing.shaped_random(self.shape, cupy, dtype)
+        bad_shape = tuple(5*i for i in self.shape)
+        b = testing.shaped_random(bad_shape, cupy, dtype)
+        plan_wrong = get_fft_plan(b, value_type='R2C')
+        assert isinstance(plan_wrong, cupy.cuda.cufft.Plan1d)
+
+        with pytest.raises(ValueError) as ex, plan_wrong:
+            rfft(a, n=self.n, norm=self.norm)
+        # targeting a particular error
+        assert 'Target array size does not match the plan.' in str(ex.value)
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (1, None), 'axes': None},
+        {'shape': (3, 4), 's': (1, 5), 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, None), 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2)},
+        {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestRfft2:
+
+    @nd_planning_states()
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft2(self, xp, dtype, order, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.rfft2(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+        return out
+
+    @nd_planning_states()
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft2(self, xp, dtype, order, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        if (10020 >= cupy.cuda.runtime.runtimeGetVersion() >= 10010
+                and int(cupy.cuda.device.get_compute_capability()) < 70
+                and _size_last_transform_axis(
+                    self.shape, self.s, self.axes) == 2):
+            pytest.skip('work-around for cuFFT issue')
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.irfft2(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.float32)
+        return out
+
+
+@testing.parameterize(
+    {'shape': (3, 4), 's': None, 'axes': (), 'norm': None},
+    {'shape': (2, 3, 4), 's': None, 'axes': (), 'norm': None},
+)
+class TestRfft2EmptyAxes:
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_rfft2(self, dtype):
+        for xp in (np, cupy):
+            a = testing.shaped_random(self.shape, xp, dtype)
+            with pytest.raises(IndexError):
+                xp.fft.rfft2(a, s=self.s, axes=self.axes, norm=self.norm)
+
+    @testing.for_all_dtypes()
+    def test_irfft2(self, dtype):
+        for xp in (np, cupy):
+            a = testing.shaped_random(self.shape, xp, dtype)
+            with pytest.raises(IndexError):
+                xp.fft.irfft2(a, s=self.s, axes=self.axes, norm=self.norm)
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (1, None), 'axes': None},
+        {'shape': (3, 4), 's': (1, 5), 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, None), 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2)},
+        {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestRfftn:
+
+    @nd_planning_states()
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn(self, xp, dtype, order, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.rfftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @nd_planning_states()
+    @testing.for_orders('CF')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn(self, xp, dtype, order, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        if (10020 >= cupy.cuda.runtime.runtimeGetVersion() >= 10010
+                and int(cupy.cuda.device.get_compute_capability()) < 70
+                and _size_last_transform_axis(
+                    self.shape, self.s, self.axes) == 2):
+            pytest.skip('work-around for cuFFT issue')
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if order == 'F':
+            a = xp.asfortranarray(a)
+        out = xp.fft.irfftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.float32)
+
+        return out
+
+
+# Only those tests in which a legit plan can be obtained are kept
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (1, None), 'axes': None},
+        {'shape': (3, 4), 's': (1, 5), 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, None), 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2)},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestPlanCtxManagerRfftn:
+
+    @pytest.fixture(autouse=True)
+    def skip_buggy(self):
+        if cupy.cuda.runtime.is_hip:
+            # TODO(leofang): test newer ROCm versions
+            if (self.axes == (0, 1) and self.shape == (2, 3, 4)):
+                pytest.skip("hipFFT's PlanNd for this case "
+                            "is buggy, so Plan1d is generated "
+                            "instead")
+
+    @nd_planning_states()
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn(self, xp, dtype, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            plan = get_fft_plan(a, self.s, self.axes, value_type='R2C')
+            with plan:
+                out = xp.fft.rfftn(a, s=self.s, axes=self.axes, norm=self.norm)
+        else:
+            out = xp.fft.rfftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+    @pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                        reason="hipFFT's PlanNd for C2R is buggy")
+    @nd_planning_states()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn(self, xp, dtype, enable_nd):
+        assert config.enable_nd_planning == enable_nd
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cupy:
+            from cupyx.scipy.fftpack import get_fft_plan
+            plan = get_fft_plan(a, self.s, self.axes, value_type='C2R')
+            with plan:
+                out = xp.fft.irfftn(
+                    a, s=self.s, axes=self.axes, norm=self.norm)
+        else:
+            out = xp.fft.irfftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.float32)
+
+        return out
+
+    # TODO(leofang): write test_rfftn_error_on_wrong_plan()?
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, None), 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestRfftnContiguity:
+
+    @nd_planning_states([True])
+    @testing.for_float_dtypes()
+    def test_rfftn_orders(self, dtype, enable_nd):
+        for order in ['C', 'F']:
+            a = testing.shaped_random(self.shape, cupy, dtype)
+            if order == 'F':
+                a = cupy.asfortranarray(a)
+            out = cupy.fft.rfftn(a, s=self.s, axes=self.axes)
+
+            fft_func = _default_fft_func(a, s=self.s, axes=self.axes,
+                                         value_type='R2C')
+            if fft_func is _fftn:
+                # nd plans have output with contiguity matching the input
+                assert out.flags.c_contiguous == a.flags.c_contiguous
+                assert out.flags.f_contiguous == a.flags.f_contiguous
+            else:
+                # 1d planning case doesn't guarantee preserved contiguity
+                pass
+
+    @nd_planning_states([True])
+    @testing.for_all_dtypes()
+    def test_ifftn_orders(self, dtype, enable_nd):
+        for order in ['C', 'F']:
+
+            a = testing.shaped_random(self.shape, cupy, dtype)
+            if order == 'F':
+                a = cupy.asfortranarray(a)
+            out = cupy.fft.irfftn(a, s=self.s, axes=self.axes)
+
+            fft_func = _default_fft_func(a, s=self.s, axes=self.axes,
+                                         value_type='C2R')
+            if fft_func is _fftn:
+                # nd plans have output with contiguity matching the input
+                assert out.flags.c_contiguous == a.flags.c_contiguous
+                assert out.flags.f_contiguous == a.flags.f_contiguous
+            else:
+                # 1d planning case doesn't guarantee preserved contiguity
+                pass
+
+
+@testing.parameterize(
+    {'shape': (3, 4), 's': None, 'axes': (), 'norm': None},
+    {'shape': (2, 3, 4), 's': None, 'axes': (), 'norm': None},
+)
+class TestRfftnEmptyAxes:
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_rfftn(self, dtype):
+        for xp in (np, cupy):
+            a = testing.shaped_random(self.shape, xp, dtype)
+            with pytest.raises(IndexError):
+                xp.fft.rfftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+    @testing.for_all_dtypes()
+    def test_irfftn(self, dtype):
+        for xp in (np, cupy):
+            a = testing.shaped_random(self.shape, xp, dtype)
+            with pytest.raises(IndexError):
+                xp.fft.irfftn(a, s=self.s, axes=self.axes, norm=self.norm)
+
+
+@pytest.mark.usefixtures('skip_forward_backward')
+@testing.parameterize(*testing.product({
+    'n': [None, 5, 10, 15],
+    'shape': [(10,), (10, 10)],
+    'norm': [None, 'backward', 'ortho', 'forward', ''],
+}))
+class TestHfft:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_hfft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.hfft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.float32)
+
+        return out
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ihfft(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.ihfft(a, n=self.n, norm=self.norm)
+
+        if xp is np and dtype in [np.float16, np.float32, np.complex64]:
+            out = out.astype(np.complex64)
+
+        return out
+
+
+@testing.parameterize(
+    {'n': 1, 'd': 1},
+    {'n': 10, 'd': 0.5},
+    {'n': 100, 'd': 2},
+)
+class TestFftfreq:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_fftfreq(self, xp, dtype):
+        out = xp.fft.fftfreq(self.n, self.d)
+
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_rfftfreq(self, xp, dtype):
+        out = xp.fft.rfftfreq(self.n, self.d)
+
+        return out
+
+
+@testing.parameterize(
+    {'shape': (5,), 'axes': None},
+    {'shape': (5,), 'axes': 0},
+    {'shape': (10,), 'axes': None},
+    {'shape': (10,), 'axes': 0},
+    {'shape': (10, 10), 'axes': None},
+    {'shape': (10, 10), 'axes': 0},
+    {'shape': (10, 10), 'axes': (0, 1)},
+)
+class TestFftshift:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_fftshift(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.fftshift(x, self.axes)
+
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False)
+    def test_ifftshift(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        out = xp.fft.ifftshift(x, self.axes)
+
+        return out
+
+
+class TestThreading:
+
+    def test_threading1(self):
+        import threading
+        from cupy.cuda.cufft import get_current_plan
+
+        def thread_get_curr_plan():
+            cupy.cuda.Device().use()
+            return get_current_plan()
+
+        new_thread = threading.Thread(target=thread_get_curr_plan)
+        new_thread.start()
+
+    def test_threading2(self):
+        import threading
+
+        a = cupy.arange(100, dtype=cupy.complex64).reshape(10, 10)
+
+        def thread_do_fft():
+            cupy.cuda.Device().use()
+            b = cupy.fft.fftn(a)
+            return b
+
+        new_thread = threading.Thread(target=thread_do_fft)
+        new_thread.start()
diff --git a/tests/cupy_tests/functional_tests/__init__.py b/tests/cupy_tests/functional_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/functional_tests/test_piecewise.py b/tests/cupy_tests/functional_tests/test_piecewise.py
new file mode 100644
index 0000000..2e3ee39
--- /dev/null
+++ b/tests/cupy_tests/functional_tests/test_piecewise.py
@@ -0,0 +1,119 @@
+import unittest
+
+import pytest
+
+import numpy
+import cupy
+from cupy import testing
+
+
+class TestPiecewise(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise(self, xp, dtype):
+        x = xp.linspace(2.5, 12.5, 6, dtype=dtype)
+        condlist = [x < 0, x >= 0, x < 5, x >= 1.5]
+        funclist = xp.array([-1, 1, 2, 5])
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_scalar_input(self, xp, dtype):
+        x = dtype(2)
+        condlist = [x < 0, x >= 0]
+        funclist = [1, 10]
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_scalar_condition(self, xp, dtype):
+        x = testing.shaped_random(shape=(2, 3, 5), xp=xp, dtype=dtype)
+        condlist = True
+        funclist = xp.array([-10, 10])
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_signed_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_otherwise_condition1(self, xp, dtype):
+        x = xp.linspace(-2, 20, 12, dtype=dtype)
+        condlist = [x > 15, x <= 5, x == 0, x == 10]
+        funclist = xp.array([-1, 0, 2, 3, -5])
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_otherwise_condition2(self, xp, dtype):
+        x = xp.array([-10, 20, 30, 40]).astype(dtype)
+        condlist = [
+            xp.array([True, False, False, True]),
+            xp.array([True, False, False, True]),
+        ]
+        funclist = xp.array([-1, 1, 2])
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_zero_dim_input(self, xp, dtype):
+        x = testing.shaped_random(shape=(), xp=xp, dtype=dtype)
+        condlist = [x < 0, x > 0]
+        funclist = [10, 1, 2]
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_ndim_input(self, xp, dtype):
+        x = testing.shaped_random(shape=(2, 3, 5), xp=xp, dtype=dtype)
+        condlist = [x < 0, x > 0]
+        funclist = [10, 1, 2]
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_zero_dim_condlist(self, xp, dtype):
+        x = testing.shaped_random(shape=(), xp=xp, dtype=dtype)
+        condlist = [testing.shaped_random(shape=(), xp=xp, dtype=bool)]
+        funclist = [1, 2]
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_ndarray_condlist_funclist(self, xp, dtype):
+        x = xp.linspace(1, 20, 12, dtype=dtype)
+        condlist = xp.array([x > 15, x <= 5, x == 0, x == 10])
+        funclist = xp.array([-1, 0, 2, 3, -5]).astype(dtype)
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_piecewise_diff_types_funclist(self, xp, dtype1, dtype2):
+        x = xp.linspace(1, 20, 12, dtype=dtype1)
+        condlist = [x > 15, x <= 5, x == 0, x == 10]
+        funclist = xp.array([1, 0, 2, 3, 5], dtype=dtype2)
+        return xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    def test_mismatched_lengths(self, dtype):
+        funclist = [-1, 0, 2, 4, 5]
+        for xp in (numpy, cupy):
+            x = xp.linspace(-2, 4, 6, dtype=dtype)
+            condlist = [x < 0, x >= 0]
+            with pytest.raises(ValueError):
+                xp.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    def test_callable_funclist(self, dtype):
+        x = cupy.linspace(-2, 4, 6, dtype=dtype)
+        condlist = [x < 0, x > 0]
+        funclist = [lambda x: -x, lambda x: x]
+        with pytest.raises(NotImplementedError):
+            cupy.piecewise(x, condlist, funclist)
+
+    @testing.for_all_dtypes()
+    def test_mixed_funclist(self, dtype):
+        x = cupy.linspace(-2, 2, 6, dtype=dtype)
+        condlist = [x < 0, x == 0, x > 0]
+        funclist = [-10, lambda x: -x, 10, lambda x: x]
+        with pytest.raises(NotImplementedError):
+            cupy.piecewise(x, condlist, funclist)
diff --git a/tests/cupy_tests/functional_tests/test_vectorize.py b/tests/cupy_tests/functional_tests/test_vectorize.py
new file mode 100644
index 0000000..b6c31b5
--- /dev/null
+++ b/tests/cupy_tests/functional_tests/test_vectorize.py
@@ -0,0 +1,668 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.cuda import runtime
+
+
+class TestVectorizeOps(unittest.TestCase):
+
+    def _run(self, func, xp, dtypes):
+        f = xp.vectorize(func)
+        args = [
+            testing.shaped_random((20, 30), xp, dtype, seed=seed)
+            for seed, dtype in enumerate(dtypes)
+        ]
+        return f(*args)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_vectorize_reciprocal(self, xp, dtype):
+        def my_reciprocal(x):
+            scalar = xp.dtype(dtype).type(10)
+            return xp.reciprocal(x + scalar)
+
+        return self._run(my_reciprocal, xp, [dtype])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_add(self, xp, dtype1, dtype2):
+        def my_add(x, y):
+            return x + y
+
+        return self._run(my_add, xp, [dtype1, dtype2])
+
+    @testing.for_dtypes('bhilqefdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_sub(self, xp, dtype):
+        def my_sub(x, y):
+            return x - y
+
+        return self._run(my_sub, xp, [dtype, dtype])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_vectorize_mul(self, xp, dtype1, dtype2):
+        def my_mul(x, y):
+            return x * y
+
+        return self._run(my_mul, xp, [dtype1, dtype2])
+
+    @testing.for_dtypes('qQefdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_vectorize_pow(self, xp, dtype):
+        def my_pow(x, y):
+            return x ** y
+
+        f = xp.vectorize(my_pow)
+        x1 = testing.shaped_random((20, 30), xp, dtype, seed=0)
+        x2 = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        x1[x1 == 0] = 1
+        return f(x1, x2)
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_vectorize_minmax(self, xp, dtype1, dtype2):
+        def my_minmax(x, y):
+            return max(x, y) - min(x, y)
+
+        f = xp.vectorize(my_minmax)
+        x1 = testing.shaped_random((20, 30), xp, dtype1, seed=0)
+        x2 = testing.shaped_random((20, 30), xp, dtype2, seed=1)
+        x1[x1 == 0] = 1
+        return f(x1, x2)
+
+    def run_div(self, func, xp, dtypes):
+        dtype1, dtype2 = dtypes
+        f = xp.vectorize(func)
+        x1 = testing.shaped_random((20, 30), xp, dtype1, seed=0)
+        x2 = testing.shaped_random((20, 30), xp, dtype2, seed=1)
+        x2[x2 == 0] = 1
+        return f(x1, x2)
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    @testing.with_requires('numpy>=1.23', 'numpy!=1.24.0', 'numpy!=1.24.1')
+    def test_vectorize_div(self, xp, dtype1, dtype2):
+        def my_div(x, y):
+            return x / y
+
+        return self.run_div(my_div, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_vectorize_floor_div(self, xp, dtype1, dtype2):
+        def my_floor_div(x, y):
+            return x // y
+
+        return self.run_div(my_floor_div, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_allclose(
+        rtol=1e-6, atol=1e-6, accept_error=TypeError)
+    def test_vectorize_mod(self, xp, dtype1, dtype2):
+        def my_mod(x, y):
+            return x % y
+
+        return self.run_div(my_mod, xp, [dtype1, dtype2])
+
+    @testing.for_dtypes('iIlLqQ')
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_lshift(self, xp, dtype):
+        def my_lshift(x, y):
+            return x << y
+
+        return self._run(my_lshift, xp, [dtype, dtype])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_rshift(self, xp, dtype1, dtype2):
+        def my_lshift(x, y):
+            return x >> y
+
+        return self._run(my_lshift, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_bit_or(self, xp, dtype1, dtype2):
+        def my_bit_or(x, y):
+            return x | y
+
+        return self._run(my_bit_or, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_bit_and(self, xp, dtype1, dtype2):
+        def my_bit_and(x, y):
+            return x & y
+
+        return self._run(my_bit_and, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_bit_xor(self, xp, dtype1, dtype2):
+        def my_bit_xor(x, y):
+            return x ^ y
+
+        return self._run(my_bit_xor, xp, [dtype1, dtype2])
+
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_bit_invert(self, xp):
+        def my_bit_invert(x):
+            return ~x
+
+        return self._run(my_bit_invert, xp, [numpy.int64])
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_logical_not(self, xp, dtype):
+        def my_logical_not(x):
+            return not x
+
+        return self._run(my_logical_not, xp, [dtype])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_eq(self, xp, dtype1, dtype2):
+        def my_eq(x, y):
+            return x == y
+
+        return self._run(my_eq, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_neq(self, xp, dtype1, dtype2):
+        def my_neq(x, y):
+            return x != y
+
+        return self._run(my_neq, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_lt(self, xp, dtype1, dtype2):
+        def my_lt(x, y):
+            return x < y
+
+        return self._run(my_lt, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_le(self, xp, dtype1, dtype2):
+        def my_le(x, y):
+            return x <= y
+
+        return self._run(my_le, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_gt(self, xp, dtype1, dtype2):
+        def my_gt(x, y):
+            return x > y
+
+        return self._run(my_gt, xp, [dtype1, dtype2])
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_ge(self, xp, dtype1, dtype2):
+        def my_ge(x, y):
+            return x >= y
+
+        return self._run(my_ge, xp, [dtype1, dtype2])
+
+    @testing.for_dtypes('bhilqefdFD')
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_usub(self, xp, dtype):
+        def my_usub(x):
+            return -x
+
+        return self._run(my_usub, xp, [dtype])
+
+
+class TestVectorizeExprs(unittest.TestCase):
+
+    @testing.for_all_dtypes(name='cond_dtype', no_complex=True)
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_vectorize_ifexp(self, xp, dtype, cond_dtype):
+        def my_ifexp(c, x, y):
+            return x if c else y
+
+        f = xp.vectorize(my_ifexp)
+        cond = testing.shaped_random((20, 30), xp, cond_dtype, seed=0)
+        x = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        y = testing.shaped_random((20, 30), xp, dtype, seed=2)
+        return f(cond, x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_vectorize_incr(self, xp, dtype):
+        def my_incr(x):
+            return x + 1
+
+        f = xp.vectorize(my_incr)
+        x = testing.shaped_random((20, 30), xp, dtype, seed=0)
+        return f(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_vectorize_ufunc_call(self, xp, dtype):
+        def my_ufunc_add(x, y):
+            return xp.add(x, y)
+
+        f = xp.vectorize(my_ufunc_add)
+        x = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        y = testing.shaped_random((20, 30), xp, dtype, seed=2)
+        return f(x, y)
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_allclose(
+        rtol={numpy.float16: 1e3, 'default': 1e-7}, accept_error=TypeError)
+    def test_vectorize_ufunc_call_dtype(self, xp, dtype1, dtype2):
+        def my_ufunc_add(x, y):
+            return xp.add(x, y, dtype=dtype2)
+
+        f = xp.vectorize(my_ufunc_add)
+        x = testing.shaped_random((20, 30), xp, dtype1, seed=1)
+        y = testing.shaped_random((20, 30), xp, dtype1, seed=2)
+        return f(x, y)
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'), full=True)
+    @testing.numpy_cupy_array_equal(
+        accept_error=(TypeError, numpy.ComplexWarning))
+    def test_vectorize_typecast(self, xp, dtype1, dtype2):
+        typecast = xp.dtype(dtype2).type
+
+        def my_typecast(x):
+            return typecast(x)
+
+        f = xp.vectorize(my_typecast)
+        x = testing.shaped_random((20, 30), xp, dtype1, seed=1)
+        return f(x)
+
+
+class TestVectorizeInstructions(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_vectorize_assign_new(self, xp, dtype):
+        def my_assign(x):
+            y = x + x
+            return x + y
+
+        f = xp.vectorize(my_assign)
+        x = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        return f(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_vectorize_assign_update(self, xp, dtype):
+        def my_assign(x):
+            x = x + x
+            return x + x
+
+        f = xp.vectorize(my_assign)
+        x = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        return f(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_vectorize_augassign(self, xp, dtype):
+        def my_augassign(x):
+            x += x
+            return x + x
+
+        f = xp.vectorize(my_augassign)
+        x = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_const_assign(self, xp):
+        def my_typecast(x):
+            typecast = xp.dtype('f').type
+            return typecast(x)
+
+        f = xp.vectorize(my_typecast)
+        x = testing.shaped_random((20, 30), xp, numpy.int32, seed=1)
+        return f(x)
+
+    def test_vectorize_const_typeerror(self):
+        def my_invalid_type(x):
+            x = numpy.dtype('f').type
+            return x
+
+        f = cupy.vectorize(my_invalid_type)
+        x = testing.shaped_random((20, 30), cupy, numpy.int32, seed=1)
+        with pytest.raises(TypeError):
+            f(x)
+
+    def test_vectorize_const_non_toplevel(self):
+        def my_invalid_type(x):
+            if x == 3:
+                typecast = numpy.dtype('f').type  # NOQA
+            return x
+
+        f = cupy.vectorize(my_invalid_type)
+        x = cupy.array([1, 2, 3, 4, 5])
+        with pytest.raises(TypeError):
+            f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_nonconst_for_value(self, xp):
+        def my_nonconst_result(x):
+            result = numpy.int32(0)
+            result = x
+            return result
+
+        f = xp.vectorize(my_nonconst_result)
+        x = testing.shaped_random((20, 30), xp, numpy.int32, seed=1)
+        return f(x)
+
+
+class TestVectorizeStmts(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_if(self, xp):
+        def func_if(x):
+            if x % 2 == 0:
+                y = x
+            else:
+                y = -x
+            return y
+
+        f = xp.vectorize(func_if)
+        x = xp.array([1, 2, 3, 4, 5])
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_if_no_orlese(self, xp):
+        def func_if(x):
+            y = 0
+            if x % 2 == 0:
+                y = x
+            return y
+
+        f = xp.vectorize(func_if)
+        x = xp.array([1, 2, 3, 4, 5])
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_elif(self, xp):
+        def func_if(x):
+            y = 0
+            if x % 2 == 0:
+                y = x
+            elif x % 3 == 0:
+                y = -x
+            return y
+
+        f = xp.vectorize(func_if)
+        x = xp.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_while(self, xp):
+        def func_while(x):
+            y = 0
+            while x > 0:
+                y += x
+                x -= 1
+            return y
+
+        f = xp.vectorize(func_while)
+        x = xp.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+        return f(x)
+
+    @testing.for_dtypes('qQ')
+    @testing.numpy_cupy_array_equal()
+    def test_for(self, xp, dtype):
+        def func_for(x):
+            y = 0
+            for i in range(x):
+                y += i
+            return y
+
+        f = xp.vectorize(func_for)
+        x = xp.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype)
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_for_const_range(self, xp):
+        def func_for(x):
+            for i in range(3, 10):
+                x += i
+            return x
+
+        f = xp.vectorize(func_for)
+        x = xp.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_for_range_step(self, xp):
+        def func_for(x, y, z):
+            res = 0
+            for i in range(x, y, z):
+                res += i * i
+            return res
+
+        f = xp.vectorize(func_for)
+        start = xp.array([0, 1, 2, 3, 4, 5])
+        stop = xp.array([-21, -23, -19, 17, 27, 24])
+        step = xp.array([-3, -2, -1, 1, 2, 3])
+        return f(start, stop, step)
+
+    @testing.numpy_cupy_array_equal()
+    def test_for_update_counter(self, xp):
+        def func_for(x):
+            for i in range(10):
+                x += i
+                i += 1
+            return x
+
+        f = xp.vectorize(func_for)
+        x = xp.array([0, 1, 2, 3, 4])
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_for_counter_after_loop(self, xp):
+        def func_for(x):
+            for i in range(10):
+                pass
+            return x + i
+
+        f = xp.vectorize(func_for)
+        x = xp.array([0, 1, 2, 3, 4])
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_for_compound_expression_param(self, xp):
+        def func_for(x, y):
+            res = 0
+            for i in range(x * y):
+                res += i
+            return res
+
+        f = xp.vectorize(func_for)
+        x = xp.array([0, 1, 2, 3, 4])
+        return f(x, x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_for_update_loop_condition(self, xp):
+        def func_for(x):
+            res = 0
+            for i in range(x):
+                res += i
+                x -= 1
+            return res
+
+        f = xp.vectorize(func_for)
+        x = xp.array([0, 1, 2, 3, 4])
+        return f(x)
+
+    @testing.numpy_cupy_array_equal()
+    def test_tuple(self, xp):
+        def func_tuple(x, y):
+            x, y = y, x
+            z = x, y
+            a, b = z
+            return a * a + b
+
+        f = xp.vectorize(func_tuple)
+        x = xp.array([0, 1, 2, 3, 4])
+        y = xp.array([5, 6, 7, 8, 9])
+        return f(x, y)
+
+    @testing.numpy_cupy_array_equal()
+    def test_tuple_pattern_match(self, xp):
+        def func_pattern_match(x, y):
+            x, y = y, x
+            z = x, y
+            (a, b), y = z, x
+            return a * a + b + y
+
+        f = xp.vectorize(func_pattern_match)
+        x = xp.array([0, 1, 2, 3, 4])
+        y = xp.array([5, 6, 7, 8, 9])
+        return f(x, y)
+
+    def test_tuple_pattern_match_type_error(self):
+        def func_pattern_match(x, y):
+            x, y = y, x
+            z = x, y
+            (a, b), z = z, x
+            return a * a + b
+
+        f = cupy.vectorize(func_pattern_match)
+        x = cupy.array([0, 1, 2, 3, 4])
+        y = cupy.array([5, 6, 7, 8, 9])
+        with pytest.raises(TypeError, match='Data type mismatch of variable:'):
+            return f(x, y)
+
+    @testing.numpy_cupy_array_equal()
+    def test_return_tuple(self, xp):
+        def func_tuple(x, y):
+            return x + y, x / y
+
+        f = xp.vectorize(func_tuple)
+        x = xp.array([0, 1, 2, 3, 4])
+        y = xp.array([5, 6, 7, 8, 9])
+        return f(x, y)
+
+
+class _MyClass:
+
+    def __init__(self, x):
+        self.x = x
+
+
+class TestVectorizeConstants(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_const_value(self, xp):
+
+        def my_func(x1, x2):
+            return x1 - x2 + const
+
+        const = 8
+        f = xp.vectorize(my_func)
+        x1 = testing.shaped_random((20, 30), xp, xp.int64, seed=1)
+        x2 = testing.shaped_random((20, 30), xp, xp.int64, seed=2)
+        return f(x1, x2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_const_attr(self, xp):
+
+        def my_func(x1, x2):
+            return x1 - x2 + const.x
+
+        const = _MyClass(10)
+        f = xp.vectorize(my_func)
+        x1 = testing.shaped_random((20, 30), xp, xp.int64, seed=1)
+        x2 = testing.shaped_random((20, 30), xp, xp.int64, seed=2)
+        return f(x1, x2)
+
+
+class TestVectorizeBroadcast(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_vectorize_broadcast(self, xp, dtype):
+        def my_func(x1, x2):
+            return x1 + x2
+
+        f = xp.vectorize(my_func)
+        x1 = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        x2 = testing.shaped_random((30,), xp, dtype, seed=2)
+        return f(x1, x2)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_vectorize_python_scalar_input(self, xp, dtype):
+        def my_func(x1, x2):
+            return x1 + x2
+
+        f = xp.vectorize(my_func)
+        x1 = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        x2 = 1
+        return f(x1, x2)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_vectorize_numpy_scalar_input(self, xp, dtype):
+        def my_func(x1, x2):
+            return x1 + x2
+
+        f = xp.vectorize(my_func)
+        x1 = testing.shaped_random((20, 30), xp, dtype, seed=1)
+        x2 = dtype(1)
+        return f(x1, x2)
+
+
+class TestVectorize(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(
+        rtol={'default': 1e-5,
+              numpy.float16: 1e-3 if runtime.is_hip else 1e-5})
+    def test_vectorize_arithmetic_ops(self, xp, dtype):
+        def my_func(x1, x2, x3):
+            y = x1 + x2 * x3 ** x1
+            x2 = y + x3 * x1
+            return x1 + x2 + x3
+
+        f = xp.vectorize(my_func)
+        x1 = testing.shaped_random((20, 30), xp, dtype, seed=1, scale=4)
+        x2 = testing.shaped_random((20, 30), xp, dtype, seed=2, scale=4)
+        x3 = testing.shaped_random((20, 30), xp, dtype, seed=3, scale=4)
+        return f(x1, x2, x3)
+
+    @testing.numpy_cupy_array_equal()
+    def test_vectorize_lambda(self, xp):
+        f = xp.vectorize(lambda a, b, c: a + b * c)
+        x1 = testing.shaped_random((20, 30), xp, numpy.int64, seed=1)
+        x2 = testing.shaped_random((20, 30), xp, numpy.int64, seed=2)
+        x3 = testing.shaped_random((20, 30), xp, numpy.int64, seed=3)
+        return f(x1, x2, x3)
+
+    def test_vectorize_lambda_xfail(self):
+        functions = [lambda a, b: a + b, lambda a, b: a * b]
+        f = cupy.vectorize(functions[0])
+        x1 = testing.shaped_random((20, 30), cupy, numpy.int64, seed=1)
+        x2 = testing.shaped_random((20, 30), cupy, numpy.int64, seed=2)
+        with pytest.raises(ValueError, match='Multiple callables are found'):
+            return f(x1, x2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_relu(self, xp):
+        f = xp.vectorize(lambda x: x if x > 0.0 else 0.0)
+        a = xp.array([0.4, -0.2, 1.8, -1.2], dtype=xp.float32)
+        return f(a)  # float32
+
+    def test_relu_type_error(self):
+        f = cupy.vectorize(lambda x: x if x > 0.0 else cupy.float64(0.0))
+        a = cupy.array([0.4, -0.2, 1.8, -1.2], dtype=cupy.float32)
+        with pytest.raises(TypeError):
+            return f(a)
diff --git a/tests/cupy_tests/indexing_tests/__init__.py b/tests/cupy_tests/indexing_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/indexing_tests/test_generate.py b/tests/cupy_tests/indexing_tests/test_generate.py
new file mode 100644
index 0000000..6e3f4cf
--- /dev/null
+++ b/tests/cupy_tests/indexing_tests/test_generate.py
@@ -0,0 +1,410 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import driver
+from cupy.cuda import runtime
+from cupy._indexing import generate
+from cupy import testing
+
+
+class TestIndices(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_indices_list0(self, xp, dtype):
+        return xp.indices((0,), dtype)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_indices_list1(self, xp, dtype):
+        return xp.indices((1, 2), dtype)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_indices_list2(self, xp, dtype):
+        return xp.indices((1, 2, 3, 4), dtype)
+
+    @testing.with_requires('numpy>=1.24')
+    def test_indices_list3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(TypeError):
+                xp.indices((1, 2, 3, 4), dtype=xp.bool_)
+
+
+class TestIX_(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_ix_list(self, xp):
+        return xp.ix_([0, 1], [2, 4])
+
+    @pytest.mark.xfail(runtime.is_hip and driver.get_build_version() < 402,
+                       reason='HIP may have a bug')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_ix_ndarray(self, xp, dtype):
+        return xp.ix_(xp.array([0, 1], dtype), xp.array([2, 3], dtype))
+
+    @testing.numpy_cupy_array_equal(type_check=False)
+    def test_ix_empty_ndarray(self, xp):
+        return xp.ix_(xp.array([]))
+
+    @testing.numpy_cupy_array_equal()
+    def test_ix_bool_ndarray(self, xp):
+        return xp.ix_(xp.array([True, False] * 2))
+
+
+class TestR_(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_r_1(self, xp, dtype):
+        a = testing.shaped_arange((3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 4), xp, dtype)
+        return xp.r_[a, b]
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_r_8(self, xp, dtype):
+        a = testing.shaped_arange((3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 4), xp, dtype)
+        c = testing.shaped_reverse_arange((1, 4), xp, dtype)
+        return xp.r_[a, b, c]
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal(
+        type_check=(numpy.lib.NumpyVersion(numpy.__version__) >= "1.25.0"))
+    def test_r_2(self, xp, dtype):
+        a = xp.array([1, 2, 3], dtype)
+        return xp.r_[a, 0, 0, a]
+
+    def test_r_3(self):
+        with self.assertRaises(NotImplementedError):
+            cupy.r_[-1:1:6j, [0] * 3, 5, 6]
+
+    @testing.for_all_dtypes()
+    def test_r_4(self, dtype):
+        a = testing.shaped_arange((1, 3), cupy, dtype)
+        with self.assertRaises(NotImplementedError):
+            cupy.r_['-1', a, a]
+
+    def test_r_5(self):
+        with self.assertRaises(NotImplementedError):
+            cupy.r_['0,2', [1, 2, 3], [4, 5, 6]]
+
+    def test_r_6(self):
+        with self.assertRaises(NotImplementedError):
+            cupy.r_['0,2,0', [1, 2, 3], [4, 5, 6]]
+
+    def test_r_7(self):
+        with self.assertRaises(NotImplementedError):
+            cupy.r_['r', [1, 2, 3], [4, 5, 6]]
+
+    @testing.for_all_dtypes()
+    def test_r_9(self, dtype):
+        a = testing.shaped_arange((3, 4), cupy, dtype)
+        b = testing.shaped_reverse_arange((2, 5), cupy, dtype)
+        with self.assertRaises(ValueError):
+            cupy.r_[a, b]
+
+
+class TestC_(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_c_1(self, xp, dtype):
+        a = testing.shaped_arange((4, 2), xp, dtype)
+        b = testing.shaped_reverse_arange((4, 3), xp, dtype)
+        return xp.c_[a, b]
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_c_2(self, xp, dtype):
+        a = testing.shaped_arange((4, 2), xp, dtype)
+        b = testing.shaped_reverse_arange((4, 3), xp, dtype)
+        c = testing.shaped_reverse_arange((4, 1), xp, dtype)
+        return xp.c_[a, b, c]
+
+    @testing.for_all_dtypes()
+    def test_c_3(self, dtype):
+        a = testing.shaped_arange((3, 4), cupy, dtype)
+        b = testing.shaped_reverse_arange((2, 5), cupy, dtype)
+        with self.assertRaises(ValueError):
+            cupy.c_[a, b]
+
+
+class TestAxisConcatenator(unittest.TestCase):
+
+    def test_AxisConcatenator_init1(self):
+        with self.assertRaises(TypeError):
+            generate.AxisConcatenator.__init__()
+
+    def test_len(self):
+        a = generate.AxisConcatenator()
+        assert len(a) == 0
+
+
+class TestUnravelIndex(unittest.TestCase):
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes()
+    @testing.numpy_cupy_array_equal(type_check=False)
+    def test(self, xp, order, dtype):
+        a = testing.shaped_arange((4, 3, 2), xp, dtype)
+        a = xp.minimum(a, 6 * 4 - 1)
+        return xp.unravel_index(a, (6, 4), order=order)
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.for_int_dtypes()
+    def test_invalid_order(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((4, 3, 2), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.unravel_index(a, (6, 4), order='V')
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes(no_bool=True)
+    def test_invalid_index(self, order, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((4, 3, 2), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.unravel_index(a, (6, 4), order=order)
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_float_dtypes()
+    def test_invalid_dtype(self, order, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((4, 3, 2), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.unravel_index(a, (6, 4), order=order)
+
+
+class TestRavelMultiIndex(unittest.TestCase):
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_basic(self, xp, order, dtype):
+        dims = (8, 4)
+        a = [xp.ones(5, dtype=dtype)] * len(dims)
+        return xp.ravel_multi_index(a, dims, order=order)
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_multi_index_broadcasting(self, xp, order, dtype):
+        dims = (3, 5)
+        x, y = xp.meshgrid(*[xp.arange(s, dtype=dtype) for s in dims],
+                           sparse=True)
+        return xp.ravel_multi_index((x, y), dims, order=order)
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_basic_nd_coords(self, xp, order, dtype):
+        dims = (8, 4)
+        a = [xp.ones((3, 3, 3), dtype=dtype)] * len(dims)
+        return xp.ravel_multi_index(a, dims, order=order)
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_basic_clip(self, xp, order, dtype):
+        dims = (8, 4, 2)
+        a = [xp.arange(max(dims), dtype=dtype)] * len(dims)
+        return xp.ravel_multi_index(a, dims, order=order, mode='clip')
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_basic_wrap(self, xp, order, dtype):
+        dims = (8, 4, 2)
+        a = [xp.arange(max(dims), dtype=dtype)] * len(dims)
+        return xp.ravel_multi_index(a, dims, order=order, mode='wrap')
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes(no_bool=True)
+    def test_basic_raise(self, order, dtype):
+        for xp in (numpy, cupy):
+            dims = (8, 4, 2)
+            a = [xp.arange(max(dims), dtype=dtype)] * len(dims)
+            with pytest.raises(ValueError):
+                return xp.ravel_multi_index(a, dims, order=order, mode='raise')
+
+    @testing.for_int_dtypes()
+    def test_invalid_float_dims(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.ones((3, 5), dtype=dtype)
+            with pytest.raises(TypeError):
+                xp.ravel_multi_index(a, (2., 4, 8.))
+
+    @testing.for_float_dtypes()
+    def test_invalid_multi_index_dtype(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.ones((3, 5), dtype=dtype)
+            with pytest.raises(TypeError):
+                xp.ravel_multi_index(a, (2, 4, 8))
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes(no_bool=True)
+    def test_invalid_multi_index_shape(self, order, dtype):
+        for xp in (numpy, cupy):
+            # a.shape[0] != len(dims)
+            dims = (2, 4)
+            a = xp.ones((len(dims) + 1, 5), dtype=dtype)
+            with pytest.raises(ValueError):
+                xp.ravel_multi_index(a, dims, order=order)
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.for_int_dtypes(no_bool=True)
+    def test_invalid_order(self, dtype):
+        for xp in (numpy, cupy):
+            dims = (8, 4)
+            a = tuple([xp.arange(min(dims), dtype=dtype) for d in dims])
+            with pytest.raises(ValueError):
+                xp.ravel_multi_index(a, dims, order='V')
+
+    @testing.for_orders(['C', 'F', None])
+    @testing.for_int_dtypes(no_bool=True)
+    def test_dims_overflow(self, order, dtype):
+        for xp in (numpy, cupy):
+            dims = (8, 4)
+            a = tuple([xp.arange(min(dims), dtype=dtype) for d in dims])
+            with pytest.raises(ValueError):
+                xp.ravel_multi_index(
+                    a, (xp.iinfo(xp.int64).max, 4), order=order)
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.for_int_dtypes(no_bool=True)
+    def test_invalid_mode(self, dtype):
+        for xp in (numpy, cupy):
+            dims = (8, 4)
+            a = tuple([xp.arange(min(dims), dtype=dtype) for d in dims])
+            with pytest.raises(ValueError):
+                xp.ravel_multi_index(a, dims, mode='invalid')
+
+
+class TestMaskIndices:
+
+    @testing.numpy_cupy_array_equal()
+    def test_mask_indices(self, xp):
+        # arr is a square matrix with 50% density
+        multiplier = testing.shaped_random((10, 10), xp=xp, dtype=xp.bool_)
+        arr = testing.shaped_random((10, 10), xp=xp) * multiplier
+        return xp.mask_indices(10, lambda n, k=None: arr)
+
+    @testing.numpy_cupy_array_equal()
+    def test_mask_indices_k(self, xp):
+        return xp.mask_indices(10, xp.triu, k=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_empty(self, xp):
+        return xp.mask_indices(0, xp.triu)
+
+
+class TestTrilIndices:
+
+    @testing.numpy_cupy_array_equal()
+    def test_tril_indices_1(self, xp):
+        return xp.tril_indices(n=29, k=0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_tril_indices_2(self, xp):
+        return xp.tril_indices(n=11, k=4, m=4)
+
+    @testing.numpy_cupy_array_equal()
+    def test_tril_indices_3(self, xp):
+        return xp.tril_indices(n=4, k=4, m=3)
+
+    @testing.for_all_dtypes()
+    def test_tril_indices(self, dtype):
+        for xp in (numpy, cupy):
+            arr = testing.shaped_random((10, 10), xp=xp, dtype=dtype)
+            if xp is numpy:
+                error = ValueError
+            else:
+                error = TypeError
+            with pytest.raises(error):
+                xp.tril_indices(arr, k=0)
+
+
+class TestTrilIndicesForm:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_tril_indices_from_1(self, xp, dtype):
+        arr = testing.shaped_random((10, 10), xp=xp, dtype=dtype)
+        return xp.tril_indices_from(arr, k=4)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_tril_indices_from_2(self, xp, dtype):
+        arr = testing.shaped_random((10, 20), xp=xp, dtype=dtype)
+        return xp.tril_indices_from(arr, k=13)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_tril_indices_from_3(self, xp, dtype):
+        arr = testing.shaped_random((4, 6), xp=xp, dtype=dtype)
+        return xp.tril_indices_from(arr)
+
+    @testing.for_all_dtypes()
+    def test_tril_indices_from_4(self, dtype):
+        for xp in (numpy, cupy):
+            with pytest.raises(AttributeError):
+                xp.tril_indices_from(4, k=1)
+
+
+class TestTriuIndices:
+
+    @testing.numpy_cupy_array_equal()
+    def test_triu_indices_1(self, xp):
+        return xp.triu_indices(n=10, k=0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_triu_indices_2(self, xp):
+        return xp.triu_indices(n=23, k=3, m=4)
+
+    @testing.numpy_cupy_array_equal()
+    def test_triu_indices_3(self, xp):
+        return xp.triu_indices(n=4, k=4, m=4)
+
+    @testing.for_all_dtypes()
+    def test_triu_indices_4(self, dtype):
+        for xp in (numpy, cupy):
+            arr = testing.shaped_random((10, 10), xp=xp, dtype=dtype)
+            if xp is numpy:
+                error = ValueError
+            else:
+                error = TypeError
+            with pytest.raises(error):
+                xp.triu_indices(arr, k=0)
+
+
+class TestTriuIndicesFrom:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_triu_indices_from_1(self, xp, dtype):
+        arr = testing.shaped_random((20, 20), xp=xp, dtype=dtype)
+        return xp.triu_indices_from(arr, k=11)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_triu_indices_from_2(self, xp, dtype):
+        arr = testing.shaped_random((20, 5), xp=xp, dtype=dtype)
+        return xp.triu_indices_from(arr, k=32)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_triu_indices_from_3(self, xp, dtype):
+        arr = testing.shaped_random((4, 6), xp=xp, dtype=dtype)
+        return xp.triu_indices_from(arr)
+
+    @testing.for_all_dtypes()
+    def test_triu_indices_from_4(self, dtype):
+        for xp in (numpy, cupy):
+            with pytest.raises(AttributeError):
+                xp.triu_indices_from(4, k=1)
diff --git a/tests/cupy_tests/indexing_tests/test_indexing.py b/tests/cupy_tests/indexing_tests/test_indexing.py
new file mode 100644
index 0000000..5ecca12
--- /dev/null
+++ b/tests/cupy_tests/indexing_tests/test_indexing.py
@@ -0,0 +1,383 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestIndexing(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_take_by_scalar(self, xp):
+        a = testing.shaped_arange((2, 4, 3), xp)
+        return a.take(2, axis=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_take_by_scalar(self, xp):
+        a = testing.shaped_arange((2, 4, 3), xp)
+        return xp.take(a, 2, axis=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_take_by_array(self, xp):
+        a = testing.shaped_arange((2, 4, 3), xp)
+        b = xp.array([[1, 3], [2, 0]])
+        return a.take(b, axis=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_take_no_axis(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        b = xp.array([[10, 5], [3, 20]])
+        return a.take(b)
+
+    # see cupy#3017
+    # mark slow as NumPy could go OOM on the Windows CI
+    @testing.slow
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_take_index_range_overflow(self, xp, dtype):
+        # Skip for too large dimensions
+        if numpy.dtype(dtype) in (numpy.int64, numpy.uint64):
+            pytest.skip()
+        # Skip because NumPy actually allocates a contiguous array in the
+        # `take` below to require much time.
+        if dtype in (numpy.int32, numpy.uint32):
+            pytest.skip()
+        iinfo = numpy.iinfo(dtype)
+        a = xp.broadcast_to(xp.ones(1), (iinfo.max + 1,))
+        b = xp.array([0], dtype=dtype)
+        return a.take(b)
+
+    @testing.numpy_cupy_array_equal()
+    def test_take_along_axis(self, xp):
+        a = testing.shaped_random((2, 4, 3), xp, dtype='float32')
+        b = testing.shaped_random((2, 6, 3), xp, dtype='int64', scale=4)
+        return xp.take_along_axis(a, b, axis=-2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_take_along_axis_none_axis(self, xp):
+        a = testing.shaped_random((2, 4, 3), xp, dtype='float32')
+        b = testing.shaped_random((30,), xp, dtype='int64', scale=24)
+        return xp.take_along_axis(a, b, axis=None)
+
+    @testing.numpy_cupy_array_equal()
+    def test_compress(self, xp):
+        a = testing.shaped_arange((3, 4, 5), xp)
+        b = xp.array([True, False, True])
+        return xp.compress(b, a, axis=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_compress_no_axis(self, xp):
+        a = testing.shaped_arange((3, 4, 5), xp)
+        b = xp.array([True, False, True])
+        return xp.compress(b, a)
+
+    @testing.for_int_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_compress_no_bool(self, xp, dtype):
+        a = testing.shaped_arange((3, 4, 5), xp)
+        b = testing.shaped_arange((3,), xp, dtype)
+        return xp.compress(b, a, axis=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_compress_overrun_false(self, xp):
+        a = testing.shaped_arange((3,), xp)
+        b = xp.array([True, False, True, False, False, False])
+        return xp.compress(b, a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_compress_empty_1dim(self, xp):
+        a = testing.shaped_arange((3, 4, 5), xp)
+        b = xp.array([])
+        return xp.compress(b, a, axis=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_compress_empty_1dim_no_axis(self, xp):
+        a = testing.shaped_arange((3, 4, 5), xp)
+        b = xp.array([])
+        return xp.compress(b, a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_compress_0dim(self, xp):
+        a = xp.array(3)
+        b = xp.array([True])
+        return xp.compress(b, a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_diagonal(self, xp, dtype):
+        a = testing.shaped_arange((3, 4, 5), xp, dtype)
+        return a.diagonal(1, 2, 0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_external_diagonal(self, xp, dtype):
+        a = testing.shaped_arange((3, 4, 5), xp, dtype)
+        return xp.diagonal(a, 1, 2, 0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_diagonal_negative1(self, xp, dtype):
+        a = testing.shaped_arange((3, 4, 5), xp, dtype)
+        return a.diagonal(-1, 2, 0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_diagonal_negative2(self, xp, dtype):
+        a = testing.shaped_arange((3, 3, 3), xp, dtype)
+        return a.diagonal(0, -1, -2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_diagonal_negative3(self, xp, dtype):
+        a = testing.shaped_arange((3, 3, 3), xp, dtype)
+        return a.diagonal(0, -1, 1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_diagonal_negative4(self, xp, dtype):
+        a = testing.shaped_arange((3, 3, 3), xp, dtype)
+        return a.diagonal(0, -3, -1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_diagonal_negative5(self, xp, dtype):
+        a = testing.shaped_arange((3, 3, 3), xp, dtype)
+        return a.diagonal(0, -1, -3)
+
+    def test_diagonal_invalid1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 3, 3), xp)
+            with pytest.raises(IndexError):
+                a.diagonal(0, 1, 3)
+
+    def test_diagonal_invalid2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 3, 3), xp)
+            with pytest.raises(IndexError):
+                a.diagonal(0, 2, -4)
+
+    @testing.numpy_cupy_array_equal()
+    def test_extract(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        b = xp.array([[True, False, True],
+                      [False, True, False],
+                      [True, False, True]])
+        return xp.extract(b, a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_extract_no_bool(self, xp, dtype):
+        a = testing.shaped_arange((3, 3), xp)
+        b = xp.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]], dtype=dtype)
+        return xp.extract(b, a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_extract_shape_mismatch(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        b = xp.array([[True, False],
+                      [True, False],
+                      [True, False]])
+        return xp.extract(b, a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_extract_size_mismatch(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        b = xp.array([[True, False, True],
+                      [False, True, False]])
+        return xp.extract(b, a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_extract_size_mismatch2(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        b = xp.array([[True, False, True, False],
+                      [False, True, False, True]])
+        return xp.extract(b, a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_extract_empty_1dim(self, xp):
+        a = testing.shaped_arange((3, 3), xp)
+        b = xp.array([])
+        return xp.extract(b, a)
+
+
+class TestChoose(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_choose(self, xp, dtype):
+        a = xp.array([0, 2, 1, 2])
+        c = testing.shaped_arange((3, 4), xp, dtype)
+        return a.choose(c)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_choose_broadcast(self, xp, dtype):
+        a = xp.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]])
+        c = xp.array([-10, 10]).astype(dtype)
+        return a.choose(c)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_choose_broadcast2(self, xp, dtype):
+        a = xp.array([0, 1])
+        c = testing.shaped_arange((3, 5, 2), xp, dtype)
+        return a.choose(c)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_choose_wrap(self, xp, dtype):
+        a = xp.array([0, 3, -1, 5])
+        c = testing.shaped_arange((3, 4), xp, dtype)
+        return a.choose(c, mode='wrap')
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_choose_clip(self, xp, dtype):
+        a = xp.array([0, 3, -1, 5])
+        c = testing.shaped_arange((3, 4), xp, dtype)
+        return a.choose(c, mode='clip')
+
+    @testing.with_requires('numpy>=1.19')
+    def test_unknown_clip(self):
+        for xp in (numpy, cupy):
+            a = xp.array([0, 3, -1, 5])
+            c = testing.shaped_arange((3, 4), xp, numpy.float32)
+            with pytest.raises(ValueError):
+                a.choose(c, mode='unknow')
+
+    def test_raise(self):
+        a = cupy.array([2])
+        c = cupy.array([[0, 1]])
+        with self.assertRaises(ValueError):
+            a.choose(c)
+
+    @testing.for_all_dtypes()
+    def test_choose_broadcast_fail(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.array([0, 1])
+            c = testing.shaped_arange((3, 5, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                return a.choose(c)
+
+
+class TestSelect(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_select(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype)
+        condlist = [a > 3, a < 5]
+        choicelist = [a, a**2]
+        return xp.select(condlist, choicelist)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_almost_equal()
+    def test_select_complex(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype)
+        condlist = [a > 3, a < 5]
+        choicelist = [a, a**2]
+        return xp.select(condlist, choicelist)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_select_default(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype)
+        condlist = [a > 3, a < 5]
+        choicelist = [a, a**2]
+        default = 3
+        return xp.select(condlist, choicelist, default)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_almost_equal()
+    def test_select_default_complex(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype)
+        condlist = [a > 3, a < 5]
+        choicelist = [a, a**2]
+        default = 3
+        return xp.select(condlist, choicelist, default)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_select_odd_shaped_broadcastable(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype)
+        b = xp.arange(30, dtype=dtype).reshape(3, 10)
+        condlist = [a < 3, b > 8]
+        choicelist = [a, b]
+        return xp.select(condlist, choicelist)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_select_odd_shaped_broadcastable_complex(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype)
+        b = xp.arange(20, dtype=dtype).reshape(2, 10)
+        condlist = [a < 3, b > 8]
+        choicelist = [a, b**2]
+        return xp.select(condlist, choicelist)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_select_1D_choicelist(self, xp, dtype):
+        a = xp.array(1)
+        b = xp.array(3)
+        condlist = [a < 3, b > 8]
+        choicelist = [a, b]
+        return xp.select(condlist, choicelist)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_select_choicelist_condlist_broadcast(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype)
+        b = xp.arange(20, dtype=dtype).reshape(2, 10)
+        condlist = [a < 4, b > 8]
+        choicelist = [xp.repeat(a, 2).reshape(2, 10), b]
+        return xp.select(condlist, choicelist)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_select_length_error(self, dtype):
+        a = cupy.arange(10, dtype=dtype)
+        condlist = [a > 3]
+        choicelist = [a, a**2]
+        with pytest.raises(ValueError):
+            cupy.select(condlist, choicelist)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_select_type_error_condlist(self, dtype):
+        a = cupy.arange(10, dtype=dtype)
+        condlist = [[3] * 10, [2] * 10]
+        choicelist = [a, a**2]
+        with pytest.raises(AttributeError):
+            cupy.select(condlist, choicelist)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_select_type_error_choicelist(self, dtype):
+        a, b = list(range(10)), list(range(-10, 0))
+        condlist = [0] * 10
+        choicelist = [a, b]
+        with pytest.raises(ValueError):
+            cupy.select(condlist, choicelist)
+
+    def test_select_empty_lists(self):
+        condlist = []
+        choicelist = []
+        with pytest.raises(ValueError):
+            cupy.select(condlist, choicelist)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_select_odd_shaped_non_broadcastable(self, dtype):
+        a = cupy.arange(10, dtype=dtype)
+        b = cupy.arange(20, dtype=dtype)
+        condlist = [a < 3, b > 8]
+        choicelist = [a, b]
+        with pytest.raises(ValueError):
+            cupy.select(condlist, choicelist)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_select_default_scalar(self, dtype):
+        a = cupy.arange(10)
+        b = cupy.arange(20)
+        condlist = [a < 3, b > 8]
+        choicelist = [a, b]
+        with pytest.raises(TypeError):
+            cupy.select(condlist, choicelist, [dtype(2)])
diff --git a/tests/cupy_tests/indexing_tests/test_insert.py b/tests/cupy_tests/indexing_tests/test_insert.py
new file mode 100644
index 0000000..76b67a9
--- /dev/null
+++ b/tests/cupy_tests/indexing_tests/test_insert.py
@@ -0,0 +1,305 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(7,), (2, 3), (4, 3, 2)],
+    'n_vals': [0, 1, 3, 15],
+}))
+class TestPlace(unittest.TestCase):
+
+    # NumPy 1.9 don't wraps values.
+    # https://github.com/numpy/numpy/pull/5821
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_place(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        if self.n_vals == 0:
+            mask = xp.zeros(self.shape, dtype=numpy.bool_)
+        else:
+            mask = testing.shaped_random(self.shape, xp, numpy.bool_)
+        vals = testing.shaped_random((self.n_vals,), xp, dtype)
+        xp.place(a, mask, vals)
+        return a
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(7,), (2, 3), (4, 3, 2)],
+}))
+class TestPlaceRaises(unittest.TestCase):
+
+    # NumPy 1.9 performs illegal memory access.
+    # https://github.com/numpy/numpy/pull/5821
+    @testing.with_requires('numpy>=1.10')
+    @testing.for_all_dtypes()
+    def test_place_empty_value_error(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, dtype)
+            mask = testing.shaped_arange(self.shape, xp, int) % 2 == 0
+            vals = testing.shaped_random((0,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.place(a, mask, vals)
+
+    # Before NumPy 1.12 it was TypeError.
+    # https://github.com/numpy/numpy/pull/7003
+    @testing.for_all_dtypes()
+    def test_place_shape_unmatch_error(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, dtype)
+            mask = testing.shaped_random((3, 4), xp, numpy.bool_)
+            vals = testing.shaped_random((1,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.place(a, mask, vals)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(7,), (2, 3), (4, 3, 2)],
+    'mode': ['raise', 'wrap', 'clip'],
+    'n_vals': [0, 1, 3, 4, 5],
+}))
+class TestPut(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_put(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        # Take care so that actual indices don't overlap.
+        if self.mode == 'raise':
+            inds = xp.array([2, -1, 3, 0])
+        else:
+            inds = xp.array([2, -8, 3, 7])
+        vals = testing.shaped_random((self.n_vals,), xp, dtype)
+        xp.put(a, inds, vals, self.mode)
+        return a
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(7,), (2, 3), (4, 3, 2)],
+}))
+class TestPutScalars(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_put_index_scalar(self, xp):
+        dtype = cupy.float32
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        inds = 4
+        vals = testing.shaped_random((4,), xp, dtype)
+        xp.put(a, inds, vals)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_put_values_scalar(self, xp):
+        dtype = cupy.float32
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        # Take care so that actual indices don't overlap.
+        inds = xp.array([2, 3, 5])
+        vals = 3.0
+        xp.put(a, inds, vals)
+        return a
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(7,), (2, 3)],
+}))
+class TestPutRaises(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_put_inds_underflow_error(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, dtype)
+            inds = xp.array([2, -8, 3, 0])
+            vals = testing.shaped_random((4,), xp, dtype)
+            with pytest.raises(IndexError):
+                xp.put(a, inds, vals, mode='raise')
+
+    @testing.for_all_dtypes()
+    def test_put_inds_overflow_error(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, dtype)
+            inds = xp.array([2, -1, 3, 7])
+            vals = testing.shaped_random((4,), xp, dtype)
+            with pytest.raises(IndexError):
+                xp.put(a, inds, vals, mode='raise')
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.for_all_dtypes()
+    def test_put_mode_error(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, dtype)
+            inds = xp.array([2, -1, 3, 0])
+            vals = testing.shaped_random((4,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.put(a, inds, vals, mode='unknown')
+
+
+@testing.parameterize(
+    *testing.product(
+        {'shape': [(0,), (1,), (2, 3), (2, 3, 4)]}))
+class TestPutmaskSameShape(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_putmask(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype=dtype, seed=0)
+        mask = testing.shaped_random(self.shape, xp, dtype=numpy.bool_, seed=1)
+        values = testing.shaped_random(self.shape, xp, dtype=dtype, seed=2)
+        ret = xp.putmask(a, mask, values)
+        assert ret is None
+        return a
+
+
+@testing.parameterize(
+    *testing.product(
+        {'shape': [(0,), (1,), (2, 3), (2, 3, 4)],
+         'values_shape': [(2,), (3, 1), (5,)]}))
+class TestPutmaskDifferentShapes(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_putmask(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype=dtype, seed=3)
+        mask = testing.shaped_random(self.shape, xp, dtype=numpy.bool_, seed=4)
+        values = testing.shaped_random(self.values_shape,
+                                       xp, dtype=dtype, seed=5)
+        ret = xp.putmask(a, mask, values)
+        assert ret is None
+        return a
+
+
+class TestPutmask(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_putmask_scalar_values(self, xp):
+        shape = (2, 3)
+        a = testing.shaped_arange(shape, xp)
+        xp.putmask(a, a > 1, 30)
+        return a
+
+    def test_putmask_non_equal_shape_raises(self):
+        for xp in (numpy, cupy):
+            a = xp.array([1, 2, 3])
+            mask = xp.array([True, False])
+            with pytest.raises(ValueError):
+                xp.putmask(a, mask, a**2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_putmask_int_mask_scalar_values(self, xp):
+        a = xp.array([1, 2, 3, 3])
+        mask = xp.array([0, 1, 0, 2])
+        xp.putmask(a, mask, 0)
+        return a
+
+
+class TestPutmaskDifferentDtypes(unittest.TestCase):
+
+    @testing.for_all_dtypes_combination(names=['a_dtype', 'val_dtype'])
+    def test_putmask_differnt_dtypes_raises(self, a_dtype, val_dtype):
+        shape = (2, 3)
+        for xp in (numpy, cupy):
+            a = testing.shaped_random(shape, xp, dtype=a_dtype)
+            mask = testing.shaped_random(shape, xp, dtype=numpy.bool_)
+            values = testing.shaped_random((3,), xp, dtype=val_dtype)
+            if not numpy.can_cast(val_dtype, a_dtype):
+                with pytest.raises(TypeError):
+                    xp.putmask(a, mask, values)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_putmask_differnt_dtypes_mask(self, xp, dtype):
+        shape = (2, 3)
+        a = testing.shaped_random(shape, xp, dtype=numpy.int64)
+        mask = testing.shaped_random(shape, xp, dtype=dtype)
+        values = testing.shaped_random((3,), xp, dtype=numpy.int64)
+        xp.putmask(a, mask, values)
+        return a
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(3, 3), (2, 2, 2), (3, 5), (5, 3)],
+    'val': [1, 0, (2,), (2, 2)],
+    'wrap': [True, False],
+}))
+class TestFillDiagonal(unittest.TestCase):
+
+    def _compute_val(self, xp):
+        if type(self.val) is int:
+            return self.val
+        else:
+            return xp.arange(numpy.prod(self.val)).reshape(self.val)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_fill_diagonal(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        val = self._compute_val(xp)
+        xp.fill_diagonal(a, val=val, wrap=self.wrap)
+        return a
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_columnar_slice(self, xp, dtype):  # see cupy#2970
+        if self.shape == (2, 2, 2):
+            pytest.skip(
+                'The length of each dimension must be the same after slicing')
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        val = self._compute_val(xp)
+        xp.fill_diagonal(a[:, 1:], val=val, wrap=self.wrap)
+        return a
+
+    @testing.for_all_dtypes()
+    def test_1darray(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((5,), xp, dtype)
+            val = self._compute_val(xp)
+            with pytest.raises(ValueError):
+                xp.fill_diagonal(a, val=val, wrap=self.wrap)
+
+
+@testing.parameterize(*testing.product({
+    'n': [2, 4, -3, 0],
+    'ndim': [2, 3, 1, 0, -2],
+}))
+class TestDiagIndices(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag_indices(self, xp):
+        return xp.diag_indices(self.n, self.ndim)
+
+
+@testing.parameterize(*testing.product({
+    'n': [-3, 0],
+    'ndim': [1, 0, -2],
+}))
+class TestDiagIndicesInvalidValues(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag_indices(self, xp):
+        return xp.diag_indices(self.n, self.ndim)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(3, 3), (0, 0), (2, 2, 2)],
+}))
+class TestDiagIndicesFrom(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_diag_indices_from(self, xp):
+        arr = testing.shaped_arange(self.shape, xp)
+        return xp.diag_indices_from(arr)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(3, 5), (3, 3, 4), (5,), (0,), (-1,)],
+}))
+class TestDiagIndicesFromRaises(unittest.TestCase):
+
+    def test_non_equal_dims(self):
+        for xp in (numpy, cupy):
+            arr = testing.shaped_arange(self.shape, xp)
+            with pytest.raises(ValueError):
+                xp.diag_indices_from(arr)
diff --git a/tests/cupy_tests/indexing_tests/test_iterate.py b/tests/cupy_tests/indexing_tests/test_iterate.py
new file mode 100644
index 0000000..98109a7
--- /dev/null
+++ b/tests/cupy_tests/indexing_tests/test_iterate.py
@@ -0,0 +1,138 @@
+import unittest
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestFlatiter(unittest.TestCase):
+
+    def test_base(self):
+        for xp in (numpy, cupy):
+            a = xp.zeros((2, 3, 4))
+            assert a.flat.base is a
+
+    def test_iter(self):
+        for xp in (numpy, cupy):
+            it = xp.zeros((2, 3, 4)).flat
+            assert iter(it) is it
+
+    def test_next(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            e = a.flatten()
+            for ai, ei in zip(a.flat, e):
+                assert ai == ei
+
+    def test_len(self):
+        for xp in (numpy, cupy):
+            a = xp.zeros((2, 3, 4))
+            assert len(a.flat) == 24
+            assert len(a[::2].flat) == 12
+
+    @testing.numpy_cupy_array_equal()
+    def test_copy(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        o = a.flat.copy()
+        assert a is not o
+        return a.flat.copy()
+
+    @testing.numpy_cupy_array_equal()
+    def test_copy_next(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        it = a.flat
+        it.__next__()
+        return it.copy()  # Returns the flattened copy of whole `a`
+
+
+@testing.parameterize(
+    {'shape': (2, 3, 4), 'index': Ellipsis},
+    {'shape': (2, 3, 4), 'index': 0},
+    {'shape': (2, 3, 4), 'index': 10},
+    {'shape': (2, 3, 4), 'index': slice(None)},
+    {'shape': (2, 3, 4), 'index': slice(None, 10)},
+    {'shape': (2, 3, 4), 'index': slice(None, None, 2)},
+    {'shape': (2, 3, 4), 'index': slice(None, None, -1)},
+    {'shape': (2, 3, 4), 'index': slice(10, None, -1)},
+    {'shape': (2, 3, 4), 'index': slice(10, None, -2)},
+    {'shape': (), 'index': slice(None)},
+    {'shape': (10,), 'index': slice(None)},
+)
+class TestFlatiterSubscript(unittest.TestCase):
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_getitem(self, xp, dtype, order):
+        a = testing.shaped_arange(self.shape, xp, dtype, order)
+        return a.flat[self.index]
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_scalar(self, xp, dtype, order):
+        a = xp.zeros(self.shape, dtype=dtype, order=order)
+        a.flat[self.index] = 1
+        return a
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_ndarray_1d(self, xp, dtype, order):
+        if numpy.isscalar(self.index):
+            pytest.skip()
+        a = xp.zeros(self.shape, dtype=dtype, order=order)
+        v = testing.shaped_arange((3,), xp, dtype, order)
+        a.flat[self.index] = v
+        return a
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_ndarray_nd(self, xp, dtype, order):
+        if numpy.isscalar(self.index):
+            pytest.skip()
+        a = xp.zeros(self.shape, dtype=dtype, order=order)
+        v = testing.shaped_arange((2, 3), xp, dtype, order)
+        a.flat[self.index] = v
+        return a
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes_combination(('a_dtype', 'v_dtype'))
+    @testing.numpy_cupy_array_equal()
+    def test_setitem_ndarray_different_types(
+            self, xp, a_dtype, v_dtype, order):
+        if numpy.isscalar(self.index):
+            pytest.skip()
+        a = xp.zeros(self.shape, dtype=a_dtype, order=order)
+        v = testing.shaped_arange((3,), xp, v_dtype, order)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', numpy.ComplexWarning)
+            a.flat[self.index] = v
+        return a
+
+
+@testing.parameterize(
+    {'shape': (2, 3, 4), 'index': None},
+    {'shape': (2, 3, 4), 'index': (0,)},
+    {'shape': (2, 3, 4), 'index': True},
+    {'shape': (2, 3, 4), 'index': cupy.array([0])},
+    {'shape': (2, 3, 4), 'index': [0]},
+)
+class TestFlatiterSubscriptIndexError(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_getitem(self, dtype):
+        a = testing.shaped_arange(self.shape, cupy, dtype)
+        with pytest.raises(IndexError):
+            a.flat[self.index]
+
+    @testing.for_all_dtypes()
+    def test_setitem(self, dtype):
+        a = testing.shaped_arange(self.shape, cupy, dtype)
+        v = testing.shaped_arange((1,), cupy, dtype)
+        with pytest.raises(IndexError):
+            a.flat[self.index] = v
diff --git a/tests/cupy_tests/io_tests/__init__.py b/tests/cupy_tests/io_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/io_tests/test_base_n.py b/tests/cupy_tests/io_tests/test_base_n.py
new file mode 100644
index 0000000..1460c92
--- /dev/null
+++ b/tests/cupy_tests/io_tests/test_base_n.py
@@ -0,0 +1,48 @@
+from cupy import testing
+
+
+class TestBinaryRepr(testing.NumpyAliasBasicTestBase):
+
+    func = 'binary_repr'
+
+
+@testing.parameterize(
+    *testing.product({
+        'args': [
+            (0,), (3,), (-3,),
+            (0, 0),
+            (3, 5),
+            (-3, 5),
+            # TODO(unno): Insuffisicent width is deprecated in numpy>=1.13.
+            # We need to check if it cause a warning, and maybe it causes an
+            # error in the future.
+            # (3, 0),
+            # (-3, 0),
+        ]}))
+class TestBinaryReprValues(testing.NumpyAliasValuesTestBase):
+
+    func = 'binary_repr'
+
+
+class TestBaseRepr(testing.NumpyAliasBasicTestBase):
+
+    func = 'base_repr'
+
+
+@testing.parameterize(
+    *testing.product({
+        'args': [
+            (0,), (5,), (-5,),
+            (0, 2), (0, 10), (0, 36),
+            (5, 2), (5, 10), (5, 36),
+            (-5, 2), (-5, 10), (-5, 36),
+            (-5, 2, 0),
+            (-5, 2, 2),
+            (-5, 2, 10),
+            (5, 2, 0),
+            (5, 2, 2),
+            (5, 2, 10),
+        ]}))
+class TestBaseReprValues(testing.NumpyAliasValuesTestBase):
+
+    func = 'base_repr'
diff --git a/tests/cupy_tests/io_tests/test_formatting.py b/tests/cupy_tests/io_tests/test_formatting.py
new file mode 100644
index 0000000..83024a4
--- /dev/null
+++ b/tests/cupy_tests/io_tests/test_formatting.py
@@ -0,0 +1,41 @@
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+class TestFormatting(unittest.TestCase):
+
+    def test_array_repr(self):
+        a = testing.shaped_arange((2, 3, 4), cupy)
+        b = testing.shaped_arange((2, 3, 4), numpy)
+        assert cupy.array_repr(a) == numpy.array_repr(b)
+
+    def test_array_str(self):
+        a = testing.shaped_arange((2, 3, 4), cupy)
+        b = testing.shaped_arange((2, 3, 4), numpy)
+        assert cupy.array_str(a) == numpy.array_str(b)
+
+    def test_array2string(self):
+        a = testing.shaped_arange((2, 3, 4), cupy)
+        b = testing.shaped_arange((2, 3, 4), numpy)
+        assert cupy.array2string(a) == numpy.array2string(b)
+
+    def test_format_float_positional_python_scalar(self):
+        x = 1.0
+        assert cupy.format_float_positional(
+            x) == numpy.format_float_positional(x)
+
+    def test_format_float_positional(self):
+        a = testing.shaped_arange((), cupy)
+        b = testing.shaped_arange((), numpy)
+        assert cupy.format_float_positional(
+            a) == numpy.format_float_positional(b)
+
+    def test_format_float_scientific(self):
+        a = testing.shaped_arange((), cupy)
+        b = testing.shaped_arange((), numpy)
+        assert cupy.format_float_scientific(
+            a) == numpy.format_float_scientific(b)
diff --git a/tests/cupy_tests/io_tests/test_npz.py b/tests/cupy_tests/io_tests/test_npz.py
new file mode 100644
index 0000000..a26a607
--- /dev/null
+++ b/tests/cupy_tests/io_tests/test_npz.py
@@ -0,0 +1,100 @@
+import io
+import pickle
+import unittest
+
+import cupy
+from cupy import testing
+
+
+class TestNpz(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_save_load(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), dtype=dtype)
+        sio = io.BytesIO()
+        cupy.save(sio, a)
+        s = sio.getvalue()
+        sio.close()
+
+        sio = io.BytesIO(s)
+        b = cupy.load(sio)
+        sio.close()
+
+        testing.assert_array_equal(a, b)
+
+    def test_save_pickle(self):
+        data = object()
+
+        sio = io.BytesIO()
+        with self.assertRaises(ValueError):
+            cupy.save(sio, data, allow_pickle=False)
+        sio.close()
+
+        sio = io.BytesIO()
+        cupy.save(sio, data, allow_pickle=True)
+        sio.close()
+
+    def test_load_pickle(self):
+        a = testing.shaped_arange((2, 3, 4), dtype=cupy.float32)
+
+        sio = io.BytesIO()
+        a.dump(sio)
+        s = sio.getvalue()
+        sio.close()
+
+        sio = io.BytesIO(s)
+        b = cupy.load(sio, allow_pickle=True)
+        testing.assert_array_equal(a, b)
+        sio.close()
+
+        sio = io.BytesIO(s)
+        with self.assertRaises(ValueError):
+            cupy.load(sio, allow_pickle=False)
+        sio.close()
+
+    @testing.for_all_dtypes()
+    def check_savez(self, savez, dtype):
+        a1 = testing.shaped_arange((2, 3, 4), dtype=dtype)
+        a2 = testing.shaped_arange((3, 4, 5), dtype=dtype)
+
+        sio = io.BytesIO()
+        savez(sio, a1, a2)
+        s = sio.getvalue()
+        sio.close()
+
+        sio = io.BytesIO(s)
+        with cupy.load(sio) as d:
+            b1 = d['arr_0']
+            b2 = d['arr_1']
+        sio.close()
+
+        testing.assert_array_equal(a1, b1)
+        testing.assert_array_equal(a2, b2)
+
+    def test_savez(self):
+        self.check_savez(cupy.savez)
+
+    def test_savez_compressed(self):
+        self.check_savez(cupy.savez_compressed)
+
+    @testing.for_all_dtypes()
+    def test_pickle(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), dtype=dtype)
+        s = pickle.dumps(a)
+        b = pickle.loads(s)
+        testing.assert_array_equal(a, b)
+
+    @testing.for_all_dtypes()
+    def test_dump(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), dtype=dtype)
+
+        sio = io.BytesIO()
+        a.dump(sio)
+        s = sio.getvalue()
+        sio.close()
+
+        sio = io.BytesIO(s)
+        b = cupy.load(sio, allow_pickle=True)
+        sio.close()
+
+        testing.assert_array_equal(a, b)
diff --git a/tests/cupy_tests/io_tests/test_text.py b/tests/cupy_tests/io_tests/test_text.py
new file mode 100644
index 0000000..449bdb0
--- /dev/null
+++ b/tests/cupy_tests/io_tests/test_text.py
@@ -0,0 +1,25 @@
+import filecmp
+import os
+import tempfile
+import unittest
+
+import numpy
+
+import cupy
+
+
+class TestText(unittest.TestCase):
+
+    def test_savetxt(self):
+        tmp_cupy = tempfile.NamedTemporaryFile(delete=False)
+        tmp_numpy = tempfile.NamedTemporaryFile(delete=False)
+        try:
+            tmp_cupy.close()
+            tmp_numpy.close()
+            array = [[1, 2, 3], [2, 3, 4]]
+            cupy.savetxt(tmp_cupy.name, cupy.array(array))
+            numpy.savetxt(tmp_numpy.name, numpy.array(array))
+            assert filecmp.cmp(tmp_cupy.name, tmp_numpy.name)
+        finally:
+            os.remove(tmp_cupy.name)
+            os.remove(tmp_numpy.name)
diff --git a/tests/cupy_tests/lib_tests/__init__.py b/tests/cupy_tests/lib_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/lib_tests/test_polynomial.py b/tests/cupy_tests/lib_tests/test_polynomial.py
new file mode 100644
index 0000000..ed0cab9
--- /dev/null
+++ b/tests/cupy_tests/lib_tests/test_polynomial.py
@@ -0,0 +1,898 @@
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import driver
+from cupy.cuda import runtime
+import cupyx
+from cupy import testing
+
+
+@testing.parameterize(
+    {'variable': None},
+    {'variable': 'y'},
+)
+class TestPoly1dInit:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_numpy_array(self, xp, dtype):
+        a = numpy.arange(5, dtype=dtype)
+        with cupyx.allow_synchronize(False):
+            out = xp.poly1d(a, variable=self.variable)
+        assert out.variable == (self.variable or 'x')
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_cupy_array(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        with cupyx.allow_synchronize(False):
+            out = xp.poly1d(a, variable=self.variable)
+        assert out.variable == (self.variable or 'x')
+        return out
+
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_list(self, xp):
+        with cupyx.allow_synchronize(False):
+            out = xp.poly1d([1, 2, 3, 4], variable=self.variable)
+        assert out.variable == (self.variable or 'x')
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_numpy_poly1d(self, xp, dtype):
+        array = testing.shaped_arange((5,), numpy, dtype)
+        a = numpy.poly1d(array)
+        with cupyx.allow_synchronize(False):
+            out = xp.poly1d(a, variable=self.variable)
+        assert out.variable == (self.variable or 'x')
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_numpy_poly1d_variable(self, xp, dtype):
+        array = testing.shaped_arange((5,), numpy, dtype)
+        a = numpy.poly1d(array, variable='z')
+        with cupyx.allow_synchronize(False):
+            out = xp.poly1d(a, variable=self.variable)
+        assert out.variable == (self.variable or 'z')
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_cupy_poly1d(self, xp, dtype):
+        array = testing.shaped_arange((5,), xp, dtype)
+        a = xp.poly1d(array)
+        out = xp.poly1d(a, variable=self.variable)
+        assert out.variable == (self.variable or 'x')
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_cupy_poly1d_variable(self, xp, dtype):
+        array = testing.shaped_arange((5,), xp, dtype)
+        a = xp.poly1d(array, variable='z')
+        out = xp.poly1d(a, variable=self.variable)
+        assert out.variable == (self.variable or 'z')
+        return out
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_zero_dim(self, xp, dtype):
+        a = testing.shaped_arange((), xp, dtype)
+        with cupyx.allow_synchronize(False):
+            out = xp.poly1d(a, variable=self.variable)
+        assert out.variable == (self.variable or 'x')
+        return out
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_zero_size(self, xp, dtype):
+        a = testing.shaped_arange((0,), xp, dtype)
+        with cupyx.allow_synchronize(False):
+            out = xp.poly1d(a, variable=self.variable)
+        assert out.variable == (self.variable or 'x')
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5)
+    def test_poly1d_roots(self, xp, dtype):
+        a = testing.shaped_arange((4,), xp, dtype)
+        out = xp.poly1d(a, True, variable=self.variable)
+        assert out.variable == (self.variable or 'x')
+        return out.coeffs
+
+
+class TestPoly1d:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_leading_zeros(self, xp, dtype):
+        a = xp.array([0, 0, 1, 2, 3], dtype)
+        return xp.poly1d(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_neg(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return -xp.poly1d(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_order(self, xp, dtype):
+        a = testing.shaped_arange((10,), xp, dtype)
+        return xp.poly1d(a).order
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_order_leading_zeros(self, xp, dtype):
+        a = xp.array([0, 0, 1, 2, 3, 0], dtype)
+        return xp.poly1d(a).order
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 402,
+                        reason='syevj not available')
+    @testing.for_signed_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_poly1d_roots(self, xp, dtype):
+        a = xp.array([-3, -2.5, 3], dtype)
+        out = xp.poly1d(a).roots
+        # The current `cupy.roots` doesn't guarantee the order of results.
+        return xp.sort(out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_getitem1(self, xp, dtype):
+        a = testing.shaped_arange((10,), xp, dtype)
+        with cupyx.allow_synchronize(False):
+            return xp.poly1d(a)[-1]
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_getitem2(self, xp, dtype):
+        a = testing.shaped_arange((10,), xp, dtype)
+        with cupyx.allow_synchronize(False):
+            return xp.poly1d(a)[5]
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_getitem3(self, xp, dtype):
+        a = testing.shaped_arange((10,), xp, dtype)
+        with cupyx.allow_synchronize(False):
+            return xp.poly1d(a)[100]
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_getitem4(self, xp, dtype):
+        a = xp.array([0, 0, 1, 2, 3, 0], dtype)
+        with cupyx.allow_synchronize(False):
+            return xp.poly1d(a)[2]
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_setitem(self, xp, dtype):
+        a = testing.shaped_arange((10,), xp, dtype)
+        b = xp.poly1d(a)
+        with cupyx.allow_synchronize(False):
+            b[100] = 20
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_poly1d_setitem_leading_zeros(self, xp, dtype):
+        a = xp.array([0, 0, 0, 2, 3, 0], dtype)
+        b = xp.poly1d(a)
+        with cupyx.allow_synchronize(False):
+            b[1] = 10
+        return b
+
+    @testing.for_all_dtypes()
+    def test_poly1d_setitem_neg(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((10,), xp, dtype)
+            b = xp.poly1d(a)
+            with pytest.raises(ValueError):
+                b[-1] = 20
+
+    @testing.for_all_dtypes()
+    def test_poly1d_get1(self, dtype):
+        a1 = testing.shaped_arange((10,), cupy, dtype)
+        a2 = testing.shaped_arange((10,), numpy, dtype)
+        b1 = cupy.poly1d(a1, variable='z').get()
+        b2 = numpy.poly1d(a2, variable='z')
+        assert b1 == b2
+
+    @testing.for_all_dtypes()
+    def test_poly1d_get2(self, dtype):
+        a1 = testing.shaped_arange((), cupy, dtype)
+        a2 = testing.shaped_arange((), numpy, dtype)
+        b1 = cupy.poly1d(a1).get()
+        b2 = numpy.poly1d(a2)
+        assert b1 == b2
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_poly1d_set(self, dtype):
+        arr1 = testing.shaped_arange((10,), cupy, dtype)
+        arr2 = numpy.ones(10, dtype=dtype)
+        a = cupy.poly1d(arr1)
+        b = numpy.poly1d(arr2, variable='z')
+        a.set(b)
+        assert a.variable == b.variable
+        testing.assert_array_equal(a.coeffs, b.coeffs)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_repr(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return repr(xp.poly1d(a))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_str(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return str(xp.poly1d(a))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-2, 'default': 1e-6})
+    def test_poly1d_call(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        b = xp.poly1d(a)
+        return b(a)
+
+
+class TestPoly:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_poly_1d(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.poly(a)
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_poly_2d_symmetric_real(self, xp, dtype):
+        a = xp.array([[6, 3, 1],
+                      [3, 0, 5],
+                      [1, 5, 6]], dtype)
+        return xp.poly(a)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_poly_2d_hermitian_complex(self, xp, dtype):
+        a = xp.array([[2, -1j], [1j, 1]], dtype)
+        return xp.poly(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_poly_2d_square(self, dtype):
+        a = testing.shaped_arange((3, 3), cupy, dtype)
+        with pytest.raises(NotImplementedError):
+            cupy.poly(a)
+
+    @testing.for_all_dtypes()
+    def test_poly_2d_general(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.poly(a)
+
+    @testing.for_all_dtypes()
+    def test_poly_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((5, 4, 3), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.poly(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_poly_zero_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), xp, dtype)
+            with pytest.raises(TypeError):
+                numpy.poly(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly_empty(self, xp, dtype):
+        a = xp.zeros((0), dtype)
+        return xp.poly(a)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(), (0,), (5,)],
+    'exp': [0, 4, 5, numpy.int32(5), numpy.int64(5)],
+}))
+class TestPoly1dPow:
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-1)
+    def test_poly1d_pow_scalar(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        return xp.poly1d(a) ** self.exp
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(5,), (5, 2)],
+    'exp': [-10, 3.5, [1, 2, 3]],
+}))
+class TestPoly1dPowInvalidValue:
+
+    @testing.for_all_dtypes()
+    def test_poly1d_pow(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(self.shape, xp, dtype)
+            with pytest.raises(ValueError):
+                xp.poly1d(a) ** self.exp
+
+
+@testing.parameterize(*testing.product({
+    'exp': [3.0, numpy.float64(5)],
+}))
+class TestPoly1dPowInvalidType:
+
+    @testing.for_all_dtypes()
+    def test_poly1d_pow(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((5,), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.poly1d(a) ** self.exp
+
+
+class Poly1dTestBase:
+
+    def _get_input(self, xp, in_type, dtype, *, size=10):
+        if in_type in ('ndarray', 'poly1d'):
+            array = testing.shaped_arange((size,), xp, dtype)
+            if array.dtype == numpy.bool_:
+                # Avoid leading zero-coefficients.
+                array = xp.logical_not(array)
+            if in_type == 'poly1d':
+                array = xp.poly1d(array)
+            return array
+        if in_type == 'python_scalar':
+            return dtype(5).item()
+        if in_type == 'numpy_scalar':
+            return dtype(5)
+        assert False
+
+
+@testing.parameterize(*testing.product({
+    'func': [
+        lambda x, y: x + y,
+        lambda x, y: x - y,
+        lambda x, y: x * y,
+    ],
+    'type_l': ['poly1d', 'ndarray', 'python_scalar', 'numpy_scalar'],
+    'type_r': ['poly1d', 'ndarray', 'python_scalar', 'numpy_scalar'],
+}))
+class TestPoly1dPolynomialArithmetic(Poly1dTestBase):
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, accept_error=TypeError)
+    def test_poly1d_arithmetic(self, xp, dtype):
+        if self.type_l == 'numpy_scalar' and self.type_r == 'poly1d':
+            pytest.skip('Avoid numpy bug.')
+        a1 = self._get_input(xp, self.type_l, dtype)
+        a2 = self._get_input(xp, self.type_r, dtype)
+        return self.func(a1, a2)
+
+
+@testing.parameterize(*testing.product({
+    'fname': ['add', 'subtract', 'multiply', 'divide', 'power'],
+    'type_l': ['poly1d', 'ndarray', 'python_scalar', 'numpy_scalar'],
+    'type_r': ['poly1d', 'ndarray', 'python_scalar', 'numpy_scalar'],
+}))
+class TestPoly1dMathArithmetic(Poly1dTestBase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_poly1d_arithmetic(self, xp, dtype):
+        func = getattr(xp, self.fname)
+        a1 = self._get_input(xp, self.type_l, dtype)
+        a2 = self._get_input(xp, self.type_r, dtype)
+        return func(a1, a2)
+
+
+@testing.parameterize(*testing.product({
+    'fname': ['polyadd', 'polysub', 'polymul'],
+    'type_l': ['poly1d', 'ndarray', 'python_scalar', 'numpy_scalar'],
+    'type_r': ['poly1d', 'ndarray', 'python_scalar', 'numpy_scalar'],
+}))
+class TestPoly1dRoutines(Poly1dTestBase):
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, accept_error=TypeError)
+    def test_poly1d_routine(self, xp, dtype):
+        func = getattr(xp, self.fname)
+        a1 = self._get_input(xp, self.type_l, dtype)
+        a2 = self._get_input(xp, self.type_r, dtype)
+        return func(a1, a2)
+
+
+class TestPoly1dEquality:
+
+    def make_poly1d1(self, xp, dtype):
+        a1 = testing.shaped_arange((4,), xp, dtype)
+        a2 = xp.zeros((4,), dtype)
+        b1 = xp.poly1d(a1)
+        b2 = xp.poly1d(a2)
+        return b1, b2
+
+    def make_poly1d2(self, xp, dtype):
+        a1 = testing.shaped_arange((4,), xp, dtype)
+        a2 = testing.shaped_arange((4,), xp, dtype)
+        b1 = xp.poly1d(a1)
+        b2 = xp.poly1d(a2)
+        return b1, b2
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_eq1(self, xp, dtype):
+        a, b = self.make_poly1d1(xp, dtype)
+        return a == b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_eq2(self, xp, dtype):
+        a, b = self.make_poly1d2(xp, dtype)
+        return a == b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_ne1(self, xp, dtype):
+        a, b = self.make_poly1d1(xp, dtype)
+        return a != b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_poly1d_ne2(self, xp, dtype):
+        a, b = self.make_poly1d2(xp, dtype)
+        return a != b
+
+
+@testing.parameterize(*testing.product({
+    'fname': ['polyadd', 'polysub', 'polymul'],
+    'shape1': [(), (0,), (3,), (5,)],
+    'shape2': [(), (0,), (3,), (5,)],
+}))
+class TestPolyArithmeticShapeCombination:
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_polyroutine(self, xp, dtype):
+        func = getattr(xp, self.fname)
+        a = testing.shaped_arange(self.shape1, xp, dtype)
+        b = testing.shaped_arange(self.shape2, xp, dtype)
+        return func(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'fname': ['polyadd', 'polysub', 'polymul'],
+}))
+class TestPolyArithmeticDiffTypes:
+
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'])
+    def test_polyroutine_diff_types_array(self, dtype1, dtype2):
+        def f(xp):
+            func = getattr(xp, self.fname)
+            a = testing.shaped_arange((10,), xp, dtype1)
+            b = testing.shaped_arange((5,), xp, dtype2)
+            return func(a, b)
+        rtol = 1e-5
+        if runtime.is_hip and self.fname == 'polymul':
+            rtol = 1e-4
+        try:
+            testing.assert_allclose(f(cupy), f(numpy), rtol=rtol)
+        except TypeError:
+            pass
+
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'])
+    def test_polyroutine_diff_types_poly1d(self, dtype1, dtype2):
+        def f(xp):
+            func = getattr(xp, self.fname)
+            a = testing.shaped_arange((10,), xp, dtype1)
+            b = testing.shaped_arange((5,), xp, dtype2)
+            a = xp.poly1d(a, variable='z')
+            b = xp.poly1d(b, variable='y')
+            out = func(a, b)
+            assert out.variable == 'x'
+            return out
+        rtol = 1e-5
+        if runtime.is_hip and self.fname == 'polymul':
+            rtol = 1e-4
+        try:
+            testing.assert_allclose(f(cupy), f(numpy), rtol=rtol)
+        except TypeError:
+            pass
+
+
+@testing.parameterize(*testing.product({
+    'shape1': [(3,)],
+    'shape2': [(3,), (3, 2)],
+    'deg': [0, 3, 3.0, 3.5, 10],
+    'rcond': [None, 0.5, 1e-15],
+    'weighted': [True, False]
+}))
+class TestPolyfitParametersCombinations:
+
+    def _full_fit(self, xp, dtype):
+        x = testing.shaped_arange(self.shape1, xp, dtype)
+        y = testing.shaped_arange(self.shape2, xp, dtype)
+        w = x if self.weighted else None
+        return xp.polyfit(x, y, self.deg, self.rcond, True, w)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(
+        atol=1e-9, accept_error=TypeError, contiguous_check=False)
+    def test_polyfit_default(self, xp, dtype):
+        if runtime.is_hip and self.deg == 0:
+            pytest.xfail('ROCm/HIP may have a bug')
+        x = testing.shaped_arange(self.shape1, xp, dtype)
+        y = testing.shaped_arange(self.shape2, xp, dtype)
+        w = x if self.weighted else None
+        return xp.polyfit(x, y, self.deg, self.rcond, w=w)
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_full(self, dtype):
+        if runtime.is_hip and self.deg == 0:
+            pytest.xfail('ROCm/HIP may have a bug')
+
+        cp_c, cp_resids, cp_rank, cp_s, cp_rcond = self._full_fit(cupy, dtype)
+        np_c, np_resids, np_rank, np_s, np_rcond = self._full_fit(numpy, dtype)
+
+        testing.assert_allclose(cp_c, np_c, atol=1e-9)
+        testing.assert_allclose(cp_resids, np_resids, atol=1e-9)
+        testing.assert_allclose(cp_s, np_s, atol=1e-9)
+        assert cp_rank == np_rank
+        if self.rcond is not None:
+            assert cp_rcond == np_rcond
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(3,), (3, 2)],
+    'deg': [0, 1],
+    'rcond': [None, 1e-15],
+    'weighted': [True, False],
+    'cov': ['unscaled', True]
+}))
+class TestPolyfitCovMode:
+
+    def _cov_fit(self, xp, dtype):
+        x = xp.array([0.008, 0.01, 0.015], dtype)
+        y = testing.shaped_arange(self.shape, xp, dtype)
+        w = x if self.weighted else None
+        return xp.polyfit(x, y, self.deg, self.rcond, w=w, cov=self.cov)
+
+    @testing.for_float_dtypes(no_float16=True)
+    def test_polyfit_cov(self, dtype):
+        if runtime.is_hip and self.deg == 0:
+            pytest.xfail('ROCm/HIP may have a bug')
+        cp_c, cp_cov = self._cov_fit(cupy, dtype)
+        np_c, np_cov = self._cov_fit(numpy, dtype)
+        testing.assert_allclose(cp_c, np_c, rtol=1e-5)
+        testing.assert_allclose(cp_cov, np_cov, rtol=1e-3)
+
+
+class TestPolyfit:
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_poor_fit(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5,), xp, dtype)
+            y = testing.shaped_arange((5,), xp, dtype)
+            with pytest.warns(numpy.RankWarning):
+                xp.polyfit(x, y, 6)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(), (0,), (5, 3, 3)],
+}))
+class TestPolyfitInvalidShapes:
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_x_invalid_shape(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange(self.shape, xp, dtype)
+            y = testing.shaped_arange((5,), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polyfit(x, y, 5)
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_y_invalid_shape(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5,), xp, dtype)
+            y = testing.shaped_arange(self.shape, xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polyfit(x, y, 5)
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_w_invalid_shape(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5,), xp, dtype)
+            w = testing.shaped_arange(self.shape, xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polyfit(x, x, 5, w=w)
+
+
+class TestPolyfitInvalid:
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_neg_degree(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5,), xp, dtype)
+            y = testing.shaped_arange((5,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polyfit(x, y, -4)
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_complex_degree(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5,), xp, dtype)
+            y = testing.shaped_arange((5,), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polyfit(x, y, 5j)
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_xy_mismatched_length(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((10,), xp, dtype)
+            y = testing.shaped_arange((5,), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polyfit(x, y, 5)
+
+    @testing.for_all_dtypes(no_float16=True)
+    def test_polyfit_yw_mismatched_length(self, dtype):
+        for xp in (numpy, cupy):
+            y = testing.shaped_arange((10,), xp, dtype)
+            w = testing.shaped_arange((5,), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polyfit(y, y, 5, w=w)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    def test_polyfit_cov_invalid(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5,), xp, dtype)
+            y = testing.shaped_arange((5,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polyfit(x, y, 5, cov=True)
+
+    def test_polyfit_float16(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5,), xp, numpy.float16)
+            with pytest.raises(TypeError):
+                xp.polyfit(x, x, 4)
+
+
+class TestPolyfitDiffTypes:
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True, full=True)
+    @testing.numpy_cupy_allclose(rtol=1e-1, atol=1e-1, accept_error=TypeError)
+    def test_polyfit_unweighted_diff_types(self, xp, dtype1, dtype2):
+        x = testing.shaped_arange((5,), xp, dtype1)
+        y = testing.shaped_arange((5,), xp, dtype2)
+        return xp.polyfit(x, y, 5)
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2', 'dtype3'], no_bool=True, full=True)
+    @testing.numpy_cupy_allclose(atol=1e-0, accept_error=TypeError)
+    def test_polyfit_weighted_diff_types(self, xp, dtype1, dtype2, dtype3):
+        x = testing.shaped_arange((5,), xp, dtype1)
+        y = testing.shaped_arange((5,), xp, dtype2)
+        w = testing.shaped_arange((5,), xp, dtype3)
+        return xp.polyfit(x, y, 5, w=w)
+
+
+@testing.parameterize(*testing.product({
+    'type_l': ['poly1d', 'ndarray'],
+    'type_r': ['poly1d', 'ndarray', 'numpy_scalar', 'python_scalar'],
+}))
+class TestPolyval(Poly1dTestBase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-2, 'default': 1e-3})
+    def test_polyval(self, xp, dtype):
+        a1 = self._get_input(xp, self.type_l, dtype, size=5)
+        a2 = self._get_input(xp, self.type_r, dtype, size=5)
+        return xp.polyval(a1, a2)
+
+
+@testing.parameterize(*testing.product({
+    'type_l': ['numpy_scalar', 'python_scalar'],
+    'type_r': ['poly1d', 'ndarray', 'python_scalar', 'numpy_scalar'],
+}))
+class TestPolyvalInvalidTypes(Poly1dTestBase):
+
+    @testing.for_all_dtypes()
+    def test_polyval(self, dtype):
+        for xp in (numpy, cupy):
+            a1 = self._get_input(xp, self.type_l, dtype)
+            a2 = self._get_input(xp, self.type_r, dtype)
+            with pytest.raises(TypeError):
+                xp.polyval(a1, a2)
+
+
+@testing.parameterize(*testing.product({
+    'shape1': [(0,), (3,), (5,)],
+    'shape2': [(), (0,), (3,), (5,)]
+}))
+class TestPolyvalShapeCombination:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-2, 'default': 1e-6})
+    def test_polyval(self, xp, dtype):
+        a = testing.shaped_arange(self.shape1, xp, dtype)
+        b = testing.shaped_arange(self.shape2, xp, dtype)
+        return xp.polyval(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(), (0,), (3,), (5,)]
+}))
+class TestPolyvalInvalidShapeCombination:
+
+    @testing.for_all_dtypes()
+    def test_polyval(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), xp, dtype)
+            b = testing.shaped_arange(self.shape, xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polyval(a, b)
+
+
+class TestPolyvalDtypesCombination:
+
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'], full=True)
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-2, 'default': 1e-6})
+    def test_polyval_diff_types_array_array(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((10,), xp, dtype1)
+        b = testing.shaped_arange((3,), xp, dtype2)
+        return xp.polyval(a, b)
+
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'], full=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_polyval_diff_types_array_scalar(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((10,), xp, dtype1)
+        b = dtype2(3)
+        return xp.polyval(a, b)
+
+
+class TestPolyvalMultiDimensional:
+
+    @testing.for_all_dtypes()
+    def test_polyval_ndim_values(self, dtype):
+        a = testing.shaped_arange((10,), cupy, dtype)
+        b = testing.shaped_arange((2, 4), cupy, dtype)
+        return cupy.polyval(a, b)
+
+    @testing.for_all_dtypes()
+    def test_polyval_poly1d_values(self, dtype):
+        a = testing.shaped_arange((5,), cupy, dtype)
+        b = testing.shaped_arange((3,), cupy, dtype)
+        b = cupy.poly1d(b)
+        return cupy.polyval(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'fname': ['polyadd', 'polysub', 'polymul', 'polyval'],
+}))
+class TestPolyRoutinesNdim:
+
+    @testing.for_all_dtypes()
+    def test_polyroutine_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            func = getattr(xp, self.fname)
+            a = testing.shaped_arange((2, 3, 4), xp, dtype)
+            b = testing.shaped_arange((10, 5), xp, dtype)
+            with pytest.raises(ValueError):
+                func(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'input': [[2, -1, -2], [-4, 10, 4]],
+}))
+@pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 402,
+                    reason='syevj not available')
+class TestRootsReal:
+
+    @testing.for_signed_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_roots_array(self, xp, dtype):
+        a = xp.array(self.input, dtype)
+        out = xp.roots(a)
+        return xp.sort(out)
+
+    @testing.for_signed_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_roots_poly1d(self, xp, dtype):
+        a = xp.array(self.input, dtype)
+        out = xp.roots(xp.poly1d(a))
+        return xp.sort(out)
+
+
+@testing.parameterize(*testing.product({
+    'input': [[3j, 1.5j, -3j], [3 + 2j, 5], [3j, 0], [0, 3j]],
+}))
+class TestRootsComplex:
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_roots_array(self, xp, dtype):
+        if runtime.is_hip and self.input == [3j, 1.5j, -3j]:
+            pytest.xfail('rocBLAS not implemented')
+        a = xp.array(self.input, dtype)
+        out = xp.roots(a)
+        return xp.sort(out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_roots_poly1d(self, xp, dtype):
+        if runtime.is_hip and self.input == [3j, 1.5j, -3j]:
+            pytest.xfail('rocBLAS not implemented')
+        a = xp.array(self.input, dtype)
+        out = xp.roots(xp.poly1d(a))
+        return xp.sort(out)
+
+
+@testing.parameterize(*testing.product({
+    'input': [[5, 10], [5, 0], [0, 5], [0, 0], [5]],
+}))
+class TestRootsSpecialCases:
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_roots_array(self, xp, dtype):
+        a = xp.array(self.input, dtype)
+        return xp.roots(a)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_roots_poly1d(self, xp, dtype):
+        a = xp.array(self.input, dtype)
+        return xp.roots(xp.poly1d(a))
+
+
+class TestRoots:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_roots_zero_sized(self, xp, dtype):
+        a = xp.zeros((0,), dtype)
+        return xp.roots(a)
+
+    @testing.with_requires('numpy>1.17')
+    @testing.for_all_dtypes(no_bool=True)
+    def test_roots_zero_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.roots(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_roots_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.roots(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_roots_zeros(self, xp, dtype):
+        a = xp.zeros((3,), dtype)
+        return xp.roots(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_roots_zeros_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.zeros((2, 1), dtype)
+            with pytest.raises(ValueError):
+                xp.roots(a)
+
+    def test_roots_bool_symmetric(self):
+        a = cupy.array([5, -1, -5], bool)
+        with pytest.raises(NotImplementedError):
+            cupy.roots(a)
diff --git a/tests/cupy_tests/lib_tests/test_shape_base.py b/tests/cupy_tests/lib_tests/test_shape_base.py
new file mode 100644
index 0000000..6297070
--- /dev/null
+++ b/tests/cupy_tests/lib_tests/test_shape_base.py
@@ -0,0 +1,108 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(*(testing.product({'axis': [0, 1, -1]})))
+class TestApplyAlongAxis(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_simple(self, xp):
+        a = xp.ones((20, 10), 'd')
+        return xp.apply_along_axis(len, self.axis, a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_3d(self, xp, dtype):
+        a = xp.arange(27, dtype=dtype).reshape((3, 3, 3))
+        return xp.apply_along_axis(xp.sum, self.axis, a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_0d_array(self, xp):
+
+        def sum_to_0d(x):
+            """ Sum x, returning a 0d array of the same class """
+            assert x.ndim == 1
+            return xp.squeeze(xp.sum(x, keepdims=True))
+
+        a = xp.ones((6, 3))
+        return xp.apply_along_axis(sum_to_0d, self.axis, a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_axis_insertion_2d(self, xp):
+
+        def f1to2(x):
+            """produces an asymmetric non-square matrix from x"""
+            assert x.ndim == 1
+            return (x[::-1] * x[1:, None])
+
+        # 2d insertion
+        a2d = xp.arange(6 * 3).reshape((6, 3))
+        return xp.apply_along_axis(f1to2, self.axis, a2d)
+
+    @testing.numpy_cupy_array_equal()
+    def test_axis_insertion_3d(self, xp):
+
+        def f1to2(x):
+            """produces an asymmetric non-square matrix from x"""
+            assert x.ndim == 1
+            return (x[::-1] * x[1:, None])
+
+        # 3d insertion
+        a3d = xp.arange(6 * 5 * 3).reshape((6, 5, 3))
+        return xp.apply_along_axis(f1to2, self.axis, a3d)
+
+    def test_empty1(self):
+        # can't apply_along_axis when there's no chance to call the function
+        def never_call(x):
+            assert False  # should never be reached
+
+        for xp in [numpy, cupy]:
+            a = xp.empty((0, 0))
+            with pytest.raises(ValueError):
+                xp.apply_along_axis(never_call, self.axis, a)
+
+    def test_empty2(self):
+        # but it's sometimes ok with some non-zero dimensions
+        def empty_to_1(x):
+            assert len(x) == 0
+            return 1
+
+        for xp in [numpy, cupy]:
+            shape = [10, 10]
+            shape[self.axis] = 0
+            shape = tuple(shape)
+            a = xp.empty(shape)
+            if self.axis == 0:
+                other_axis = 1
+            else:
+                other_axis = 0
+            with pytest.raises(ValueError):
+                xp.apply_along_axis(empty_to_1, other_axis, a)
+
+            # okay to call along the shape 0 axis
+            testing.assert_array_equal(
+                xp.apply_along_axis(empty_to_1, self.axis, a),
+                xp.ones((10,))
+            )
+
+    @testing.numpy_cupy_array_equal()
+    def test_tuple_outs(self, xp):
+        def func(x):
+            return x.sum(axis=-1), x.prod(axis=-1), x.max(axis=-1)
+
+        a = testing.shaped_arange((2, 2, 2), xp, cupy.int64)
+        return xp.apply_along_axis(func, 1, a)
+
+
+@testing.with_requires('numpy>=1.16')
+def test_apply_along_axis_invalid_axis():
+    for xp in [numpy, cupy]:
+        a = xp.ones((8, 4))
+        for axis in [-3, 2]:
+            with pytest.raises(numpy.AxisError):
+                xp.apply_along_axis(xp.sum, axis, a)
diff --git a/tests/cupy_tests/lib_tests/test_strided_tricks.py b/tests/cupy_tests/lib_tests/test_strided_tricks.py
new file mode 100644
index 0000000..16a0425
--- /dev/null
+++ b/tests/cupy_tests/lib_tests/test_strided_tricks.py
@@ -0,0 +1,46 @@
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+from cupy.lib import stride_tricks
+
+
+class TestAsStrided(unittest.TestCase):
+    def test_as_strided(self):
+        a = cupy.array([1, 2, 3, 4])
+        a_view = stride_tricks.as_strided(
+            a, shape=(2,), strides=(2 * a.itemsize,))
+        expected = cupy.array([1, 3])
+        testing.assert_array_equal(a_view, expected)
+
+        a = cupy.array([1, 2, 3, 4])
+        a_view = stride_tricks.as_strided(
+            a, shape=(3, 4), strides=(0, 1 * a.itemsize))
+        expected = cupy.array([[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]])
+        testing.assert_array_equal(a_view, expected)
+
+    @testing.numpy_cupy_array_equal()
+    def test_rolling_window(self, xp):
+        a = testing.shaped_arange((3, 4), xp)
+        a_rolling = rolling_window(a, 2, 0)
+
+        return a_rolling
+
+
+def rolling_window(a, window, axis=-1):
+    """
+    Make an ndarray with a rolling window along axis.
+    This function is taken from https://github.com/numpy/numpy/pull/31
+    but slightly modified to accept axis option.
+    """
+    a = numpy.swapaxes(a, axis, -1)
+    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
+    strides = a.strides + (a.strides[-1],)
+    if isinstance(a, numpy.ndarray):
+        rolling = numpy.lib.stride_tricks.as_strided(
+            a, shape=shape, strides=strides)
+    elif isinstance(a, cupy.ndarray):
+        rolling = stride_tricks.as_strided(a, shape=shape, strides=strides)
+    return rolling.swapaxes(-2, axis)
diff --git a/tests/cupy_tests/linalg_tests/__init__.py b/tests/cupy_tests/linalg_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/linalg_tests/test_decomposition.py b/tests/cupy_tests/linalg_tests/test_decomposition.py
new file mode 100644
index 0000000..5b8cc94
--- /dev/null
+++ b/tests/cupy_tests/linalg_tests/test_decomposition.py
@@ -0,0 +1,368 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupyx import cusolver
+from cupy.cuda import driver
+from cupy.cuda import runtime
+from cupy.linalg import _util
+from cupy import testing
+from cupy.testing import _condition
+import cupyx
+
+
+def random_matrix(shape, dtype, scale, sym=False):
+    m, n = shape[-2:]
+    dtype = numpy.dtype(dtype)
+    assert dtype.kind in 'iufc'
+    low_s, high_s = scale
+    bias = None
+    if dtype.kind in 'iu':
+        # For an m \times n matrix M whose element is in [-0.5, 0.5], it holds
+        # (singular value of M) <= \sqrt{mn} / 2
+        err = numpy.sqrt(m * n) / 2.
+        low_s += err
+        high_s -= err
+        if dtype.kind in 'u':
+            assert sym, (
+                'generating nonsymmetric matrix with uint cells is not'
+                ' supported')
+            # (singular value of numpy.ones((m, n))) <= \sqrt{mn}
+            high_s = bias = high_s / (1 + numpy.sqrt(m * n))
+    assert low_s <= high_s
+    a = numpy.random.standard_normal(shape)
+    if dtype.kind == 'c':
+        a = a + 1j * numpy.random.standard_normal(shape)
+    u, s, vh = numpy.linalg.svd(a)
+    if sym:
+        assert m == n
+        vh = u.conj().swapaxes(-1, -2)
+    new_s = numpy.random.uniform(low_s, high_s, s.shape)
+    new_a = numpy.einsum('...ij,...j,...jk->...ik', u, new_s, vh)
+    if bias is not None:
+        new_a += bias
+    if dtype.kind in 'iu':
+        new_a = numpy.rint(new_a)
+    return new_a.astype(dtype)
+
+
+class TestCholeskyDecomposition:
+
+    @testing.numpy_cupy_allclose(atol=1e-3)
+    def check_L(self, array, xp):
+        a = xp.asarray(array)
+        return xp.linalg.cholesky(a)
+
+    @testing.for_dtypes([
+        numpy.int32, numpy.int64, numpy.uint32, numpy.uint64,
+        numpy.float32, numpy.float64, numpy.complex64, numpy.complex128])
+    def test_decomposition(self, dtype):
+        # A positive definite matrix
+        A = random_matrix((5, 5), dtype, scale=(10, 10000), sym=True)
+        self.check_L(A)
+        # np.linalg.cholesky only uses a lower triangle of an array
+        self.check_L(numpy.array([[1, 2], [1, 9]], dtype))
+
+    @testing.for_dtypes([
+        numpy.int32, numpy.int64, numpy.uint32, numpy.uint64,
+        numpy.float32, numpy.float64, numpy.complex64, numpy.complex128])
+    def test_batched_decomposition(self, dtype):
+        if not cusolver.check_availability('potrfBatched'):
+            pytest.skip('potrfBatched is not available')
+        Ab1 = random_matrix((3, 5, 5), dtype, scale=(10, 10000), sym=True)
+        self.check_L(Ab1)
+        Ab2 = random_matrix((2, 2, 5, 5), dtype, scale=(10, 10000), sym=True)
+        self.check_L(Ab2)
+
+    @pytest.mark.parametrize('shape', [
+        # empty square
+        (0, 0),
+        (3, 0, 0),
+        # empty batch
+        (2, 0, 3, 4, 4),
+    ])
+    @testing.for_dtypes([
+        numpy.int32, numpy.uint16,
+        numpy.float32, numpy.float64, numpy.complex64, numpy.complex128])
+    @testing.numpy_cupy_allclose()
+    def test_empty(self, shape, xp, dtype):
+        a = xp.empty(shape, dtype)
+        return xp.linalg.cholesky(a)
+
+
+class TestCholeskyInvalid(unittest.TestCase):
+
+    def check_L(self, array):
+        for xp in (numpy, cupy):
+            a = xp.asarray(array)
+            with cupyx.errstate(linalg='raise'):
+                with pytest.raises(numpy.linalg.LinAlgError):
+                    xp.linalg.cholesky(a)
+
+    @testing.for_dtypes([
+        numpy.int32, numpy.int64, numpy.uint32, numpy.uint64,
+        numpy.float32, numpy.float64])
+    def test_decomposition(self, dtype):
+        A = numpy.array([[1, -2], [-2, 1]]).astype(dtype)
+        self.check_L(A)
+
+
+@testing.parameterize(*testing.product({
+    'mode': ['r', 'raw', 'complete', 'reduced'],
+}))
+class TestQRDecomposition(unittest.TestCase):
+
+    @testing.for_dtypes('fdFD')
+    def check_mode(self, array, mode, dtype):
+        if runtime.is_hip and driver.get_build_version() < 307:
+            if dtype in (numpy.complex64, numpy.complex128):
+                pytest.skip('ungqr unsupported')
+
+        a_cpu = numpy.asarray(array, dtype=dtype)
+        a_gpu = cupy.asarray(array, dtype=dtype)
+        result_gpu = cupy.linalg.qr(a_gpu, mode=mode)
+        if (
+            mode != 'raw' or
+            numpy.lib.NumpyVersion(numpy.__version__) >= '1.22.0rc1'
+        ):
+            result_cpu = numpy.linalg.qr(a_cpu, mode=mode)
+            self._check_result(result_cpu, result_gpu)
+
+    def _check_result(self, result_cpu, result_gpu):
+        if isinstance(result_cpu, tuple):
+            for b_cpu, b_gpu in zip(result_cpu, result_gpu):
+                assert b_cpu.dtype == b_gpu.dtype
+                cupy.testing.assert_allclose(b_cpu, b_gpu, atol=1e-4)
+        else:
+            assert result_cpu.dtype == result_gpu.dtype
+            cupy.testing.assert_allclose(result_cpu, result_gpu, atol=1e-4)
+
+    @testing.fix_random()
+    @_condition.repeat(3, 10)
+    def test_mode(self):
+        self.check_mode(numpy.random.randn(2, 4), mode=self.mode)
+        self.check_mode(numpy.random.randn(3, 3), mode=self.mode)
+        self.check_mode(numpy.random.randn(5, 4), mode=self.mode)
+
+    @testing.with_requires('numpy>=1.22')
+    @testing.fix_random()
+    def test_mode_rank3(self):
+        self.check_mode(numpy.random.randn(3, 2, 4), mode=self.mode)
+        self.check_mode(numpy.random.randn(4, 3, 3), mode=self.mode)
+        self.check_mode(numpy.random.randn(2, 5, 4), mode=self.mode)
+
+    @testing.with_requires('numpy>=1.22')
+    @testing.fix_random()
+    def test_mode_rank4(self):
+        self.check_mode(numpy.random.randn(2, 3, 2, 4), mode=self.mode)
+        self.check_mode(numpy.random.randn(2, 4, 3, 3), mode=self.mode)
+        self.check_mode(numpy.random.randn(2, 2, 5, 4), mode=self.mode)
+
+    @testing.with_requires('numpy>=1.16')
+    def test_empty_array(self):
+        self.check_mode(numpy.empty((0, 3)), mode=self.mode)
+        self.check_mode(numpy.empty((3, 0)), mode=self.mode)
+
+    @testing.with_requires('numpy>=1.22')
+    def test_empty_array_rank3(self):
+        self.check_mode(numpy.empty((0, 3, 2)), mode=self.mode)
+        self.check_mode(numpy.empty((3, 0, 2)), mode=self.mode)
+        self.check_mode(numpy.empty((3, 2, 0)), mode=self.mode)
+        self.check_mode(numpy.empty((0, 3, 3)), mode=self.mode)
+        self.check_mode(numpy.empty((3, 0, 3)), mode=self.mode)
+        self.check_mode(numpy.empty((3, 3, 0)), mode=self.mode)
+        self.check_mode(numpy.empty((0, 2, 3)), mode=self.mode)
+        self.check_mode(numpy.empty((2, 0, 3)), mode=self.mode)
+        self.check_mode(numpy.empty((2, 3, 0)), mode=self.mode)
+
+
+@testing.parameterize(*testing.product({
+    'full_matrices': [True, False],
+}))
+@testing.fix_random()
+class TestSVD(unittest.TestCase):
+
+    def setUp(self):
+        self.seed = testing.generate_seed()
+
+    @testing.for_dtypes([
+        numpy.int32, numpy.int64, numpy.uint32, numpy.uint64,
+        numpy.float32, numpy.float64, numpy.complex64, numpy.complex128,
+    ])
+    def check_usv(self, shape, dtype):
+        array = testing.shaped_random(
+            shape, numpy, dtype=dtype, seed=self.seed)
+        a_cpu = numpy.asarray(array, dtype=dtype)
+        a_gpu = cupy.asarray(array, dtype=dtype)
+        result_cpu = numpy.linalg.svd(a_cpu, full_matrices=self.full_matrices)
+        result_gpu = cupy.linalg.svd(a_gpu, full_matrices=self.full_matrices)
+        # Check if the input matrix is not broken
+        cupy.testing.assert_allclose(a_gpu, a_cpu)
+
+        assert len(result_gpu) == 3
+        for i in range(3):
+            assert result_gpu[i].shape == result_cpu[i].shape
+            assert result_gpu[i].dtype == result_cpu[i].dtype
+        u_cpu, s_cpu, vh_cpu = result_cpu
+        u_gpu, s_gpu, vh_gpu = result_gpu
+        cupy.testing.assert_allclose(s_gpu, s_cpu, rtol=1e-5, atol=1e-4)
+
+        # reconstruct the matrix
+        k = s_cpu.shape[-1]
+        if len(shape) == 2:
+            if self.full_matrices:
+                a_gpu_usv = cupy.dot(u_gpu[:, :k] * s_gpu, vh_gpu[:k, :])
+            else:
+                a_gpu_usv = cupy.dot(u_gpu * s_gpu, vh_gpu)
+        else:
+            if self.full_matrices:
+                a_gpu_usv = cupy.matmul(u_gpu[..., :k] * s_gpu[..., None, :],
+                                        vh_gpu[..., :k, :])
+            else:
+                a_gpu_usv = cupy.matmul(u_gpu*s_gpu[..., None, :], vh_gpu)
+        cupy.testing.assert_allclose(a_gpu, a_gpu_usv, rtol=1e-4, atol=1e-4)
+
+        # assert unitary
+        u_len = u_gpu.shape[-1]
+        vh_len = vh_gpu.shape[-2]
+        cupy.testing.assert_allclose(
+            cupy.matmul(u_gpu.swapaxes(-1, -2).conj(), u_gpu),
+            _util.stacked_identity(shape[:-2], u_len, dtype),
+            atol=1e-4)
+        cupy.testing.assert_allclose(
+            cupy.matmul(vh_gpu, vh_gpu.swapaxes(-1, -2).conj()),
+            _util.stacked_identity(shape[:-2], vh_len, dtype),
+            atol=1e-4)
+
+    @testing.for_dtypes([
+        numpy.int32, numpy.int64, numpy.uint32, numpy.uint64,
+        numpy.float32, numpy.float64, numpy.complex64, numpy.complex128,
+    ])
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-4)
+    def check_singular(self, shape, xp, dtype):
+        array = testing.shaped_random(shape, xp, dtype=dtype, seed=self.seed)
+        a = xp.asarray(array, dtype=dtype)
+        a_copy = a.copy()
+        result = xp.linalg.svd(
+            a, full_matrices=self.full_matrices, compute_uv=False)
+        # Check if the input matrix is not broken
+        assert (a == a_copy).all()
+        return result
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank2(self):
+        self.check_usv((3, 7))
+        self.check_usv((2, 2))
+        self.check_usv((7, 3))
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank2_no_uv(self):
+        self.check_singular((3, 7))
+        self.check_singular((2, 2))
+        self.check_singular((7, 3))
+
+    @testing.with_requires('numpy>=1.16')
+    def test_svd_rank2_empty_array(self):
+        self.check_usv((0, 3))
+        self.check_usv((3, 0))
+        self.check_usv((1, 0))
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.numpy_cupy_array_equal()
+    def test_svd_rank2_empty_array_compute_uv_false(self, xp):
+        array = xp.empty((3, 0))
+        return xp.linalg.svd(
+            array, full_matrices=self.full_matrices, compute_uv=False)
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank3(self):
+        self.check_usv((2, 3, 4))
+        self.check_usv((2, 3, 7))
+        self.check_usv((2, 4, 4))
+        self.check_usv((2, 7, 3))
+        self.check_usv((2, 4, 3))
+        self.check_usv((2, 32, 32))  # still use _gesvdj_batched
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank3_loop(self):
+        # This tests the loop-based batched gesvd on CUDA (_gesvd_batched)
+        self.check_usv((2, 64, 64))
+        self.check_usv((2, 64, 32))
+        self.check_usv((2, 32, 64))
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank3_no_uv(self):
+        self.check_singular((2, 3, 4))
+        self.check_singular((2, 3, 7))
+        self.check_singular((2, 4, 4))
+        self.check_singular((2, 7, 3))
+        self.check_singular((2, 4, 3))
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank3_no_uv_loop(self):
+        # This tests the loop-based batched gesvd on CUDA (_gesvd_batched)
+        self.check_singular((2, 64, 64))
+        self.check_singular((2, 64, 32))
+        self.check_singular((2, 32, 64))
+
+    @testing.with_requires('numpy>=1.16')
+    def test_svd_rank3_empty_array(self):
+        self.check_usv((0, 3, 4))
+        self.check_usv((3, 0, 4))
+        self.check_usv((3, 4, 0))
+        self.check_usv((3, 0, 0))
+        self.check_usv((0, 3, 0))
+        self.check_usv((0, 0, 3))
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.numpy_cupy_array_equal()
+    def test_svd_rank3_empty_array_compute_uv_false1(self, xp):
+        array = xp.empty((3, 0, 4))
+        return xp.linalg.svd(
+            array, full_matrices=self.full_matrices, compute_uv=False)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.numpy_cupy_array_equal()
+    def test_svd_rank3_empty_array_compute_uv_false2(self, xp):
+        array = xp.empty((0, 3, 4))
+        return xp.linalg.svd(
+            array, full_matrices=self.full_matrices, compute_uv=False)
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank4(self):
+        self.check_usv((2, 2, 3, 4))
+        self.check_usv((2, 2, 3, 7))
+        self.check_usv((2, 2, 4, 4))
+        self.check_usv((2, 2, 7, 3))
+        self.check_usv((2, 2, 4, 3))
+        self.check_usv((2, 2, 32, 32))  # still use _gesvdj_batched
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank4_loop(self):
+        # This tests the loop-based batched gesvd on CUDA (_gesvd_batched)
+        self.check_usv((3, 2, 64, 64))
+        self.check_usv((3, 2, 64, 32))
+        self.check_usv((3, 2, 32, 64))
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank4_no_uv(self):
+        self.check_singular((2, 2, 3, 4))
+        self.check_singular((2, 2, 3, 7))
+        self.check_singular((2, 2, 4, 4))
+        self.check_singular((2, 2, 7, 3))
+        self.check_singular((2, 2, 4, 3))
+
+    @_condition.repeat(3, 10)
+    def test_svd_rank4_no_uv_loop(self):
+        # This tests the loop-based batched gesvd on CUDA (_gesvd_batched)
+        self.check_singular((3, 2, 64, 64))
+        self.check_singular((3, 2, 64, 32))
+        self.check_singular((3, 2, 32, 64))
+
+    @testing.with_requires('numpy>=1.16')
+    def test_svd_rank4_empty_array(self):
+        self.check_usv((0, 2, 3, 4))
+        self.check_usv((1, 2, 0, 4))
+        self.check_usv((1, 2, 3, 0))
diff --git a/tests/cupy_tests/linalg_tests/test_eigenvalue.py b/tests/cupy_tests/linalg_tests/test_eigenvalue.py
new file mode 100644
index 0000000..70fc334
--- /dev/null
+++ b/tests/cupy_tests/linalg_tests/test_eigenvalue.py
@@ -0,0 +1,181 @@
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import driver
+from cupy.cuda import runtime
+from cupy import testing
+
+
+def _get_hermitian(xp, a, UPLO):
+    if UPLO == 'U':
+        return xp.triu(a) + xp.triu(a, 1).swapaxes(-2, -1).conj()
+    else:
+        return xp.tril(a) + xp.tril(a, -1).swapaxes(-2, -1).conj()
+
+
+@testing.parameterize(*testing.product({
+    'UPLO': ['U', 'L'],
+}))
+@pytest.mark.skipif(
+    runtime.is_hip and driver.get_build_version() < 402,
+    reason='eigensolver not added until ROCm 4.2.0')
+class TestEigenvalue:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4, contiguous_check=False)
+    def test_eigh(self, xp, dtype):
+        if xp == numpy and dtype == numpy.float16:
+            # NumPy's eigh does not support float16
+            _dtype = 'f'
+        else:
+            _dtype = dtype
+        if numpy.dtype(_dtype).kind == 'c':
+            a = xp.array([[1, 2j, 3], [4j, 5, 6j], [7, 8j, 9]], _dtype)
+        else:
+            a = xp.array([[1, 0, 3], [0, 5, 0], [7, 0, 9]], _dtype)
+        w, v = xp.linalg.eigh(a, UPLO=self.UPLO)
+
+        # Changed the verification method to check if Av and vw match, since
+        # the eigenvectors of eigh() with CUDA 11.6 are mathematically correct
+        # but may not match NumPy.
+        A = _get_hermitian(xp, a, self.UPLO)
+        if _dtype == numpy.float16:
+            tol = 1e-3
+        else:
+            tol = 1e-5
+        testing.assert_allclose(A @ v, v @ xp.diag(w), atol=tol, rtol=tol)
+        # Check if v @ vt is an identity matrix
+        testing.assert_allclose(v @ v.swapaxes(-2, -1).conj(),
+                                xp.identity(A.shape[-1], _dtype), atol=tol,
+                                rtol=tol)
+        if xp == numpy and dtype == numpy.float16:
+            w = w.astype('e')
+        return w
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4, contiguous_check=False)
+    def test_eigh_batched(self, xp, dtype):
+        a = xp.array([[[1, 0, 3], [0, 5, 0], [7, 0, 9]],
+                      [[3, 0, 3], [0, 7, 0], [7, 0, 11]]], dtype)
+        w, v = xp.linalg.eigh(a, UPLO=self.UPLO)
+
+        # NumPy, cuSOLVER, rocSOLVER all sort in ascending order,
+        # so w's should be directly comparable. However, both cuSOLVER
+        # and rocSOLVER pick a different convention for constructing
+        # eigenvectors, so v's are not directly comparible and we verify
+        # them through the eigen equation A*v=w*v.
+        A = _get_hermitian(xp, a, self.UPLO)
+        for i in range(a.shape[0]):
+            testing.assert_allclose(
+                A[i].dot(v[i]), w[i]*v[i], rtol=1e-5, atol=1e-5)
+        return w
+
+    @testing.for_dtypes('FD')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4, contiguous_check=False)
+    def test_eigh_complex_batched(self, xp, dtype):
+        a = xp.array([[[1, 2j, 3], [4j, 5, 6j], [7, 8j, 9]],
+                      [[0, 2j, 3], [4j, 4, 6j], [7, 8j, 8]]], dtype)
+        w, v = xp.linalg.eigh(a, UPLO=self.UPLO)
+
+        # NumPy, cuSOLVER, rocSOLVER all sort in ascending order,
+        # so w's should be directly comparable. However, both cuSOLVER
+        # and rocSOLVER pick a different convention for constructing
+        # eigenvectors, so v's are not directly comparible and we verify
+        # them through the eigen equation A*v=w*v.
+        A = _get_hermitian(xp, a, self.UPLO)
+        for i in range(a.shape[0]):
+            testing.assert_allclose(
+                A[i].dot(v[i]), w[i]*v[i], rtol=1e-5, atol=1e-5)
+        return w
+
+    @testing.for_all_dtypes(no_float16=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_eigvalsh(self, xp, dtype):
+        a = xp.array([[1, 0, 3], [0, 5, 0], [7, 0, 9]], dtype)
+        w = xp.linalg.eigvalsh(a, UPLO=self.UPLO)
+        # NumPy, cuSOLVER, rocSOLVER all sort in ascending order,
+        # so they should be directly comparable
+        return w
+
+    @testing.for_all_dtypes(no_float16=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_eigvalsh_batched(self, xp, dtype):
+        a = xp.array([[[1, 0, 3], [0, 5, 0], [7, 0, 9]],
+                      [[3, 0, 3], [0, 7, 0], [7, 0, 11]]], dtype)
+        w = xp.linalg.eigvalsh(a, UPLO=self.UPLO)
+        # NumPy, cuSOLVER, rocSOLVER all sort in ascending order,
+        # so they should be directly comparable
+        return w
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_eigvalsh_complex(self, xp, dtype):
+        a = xp.array([[1, 2j, 3], [4j, 5, 6j], [7, 8j, 9]], dtype)
+        w = xp.linalg.eigvalsh(a, UPLO=self.UPLO)
+        # NumPy, cuSOLVER, rocSOLVER all sort in ascending order,
+        # so they should be directly comparable
+        return w
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_eigvalsh_complex_batched(self, xp, dtype):
+        a = xp.array([[[1, 2j, 3], [4j, 5, 6j], [7, 8j, 9]],
+                      [[0, 2j, 3], [4j, 4, 6j], [7, 8j, 8]]], dtype)
+        w = xp.linalg.eigvalsh(a, UPLO=self.UPLO)
+        # NumPy, cuSOLVER, rocSOLVER all sort in ascending order,
+        # so they should be directly comparable
+        return w
+
+
+@pytest.mark.parametrize('UPLO', ['U', 'L'])
+@pytest.mark.parametrize('shape', [
+    (0, 0),
+    (2, 0, 0),
+    (0, 3, 3),
+])
+@pytest.mark.skipif(
+    runtime.is_hip and driver.get_build_version() < 402,
+    reason='eigensolver not added until ROCm 4.2.0')
+class TestEigenvalueEmpty:
+
+    @testing.for_dtypes('ifdFD')
+    @testing.numpy_cupy_allclose()
+    def test_eigh(self, xp, dtype, shape, UPLO):
+        a = xp.empty(shape, dtype)
+        assert a.size == 0
+        return xp.linalg.eigh(a, UPLO=UPLO)
+
+    @testing.for_dtypes('ifdFD')
+    @testing.numpy_cupy_allclose()
+    def test_eigvalsh(self, xp, dtype, shape, UPLO):
+        a = xp.empty(shape, dtype)
+        assert a.size == 0
+        return xp.linalg.eigvalsh(a, UPLO=UPLO)
+
+
+@pytest.mark.parametrize('UPLO', ['U', 'L'])
+@pytest.mark.parametrize('shape', [
+    (),
+    (3,),
+    (2, 3),
+    (4, 0),
+    (2, 2, 3),
+    (0, 2, 3),
+])
+@pytest.mark.skipif(
+    runtime.is_hip and driver.get_build_version() < 402,
+    reason='eigensolver not added until ROCm 4.2.0')
+class TestEigenvalueInvalid:
+
+    def test_eigh_shape_error(self, UPLO, shape):
+        for xp in (numpy, cupy):
+            a = xp.zeros(shape)
+            with pytest.raises(numpy.linalg.LinAlgError):
+                xp.linalg.eigh(a, UPLO)
+
+    def test_eigvalsh_shape_error(self, UPLO, shape):
+        for xp in (numpy, cupy):
+            a = xp.zeros(shape)
+            with pytest.raises(numpy.linalg.LinAlgError):
+                xp.linalg.eigvalsh(a, UPLO)
diff --git a/tests/cupy_tests/linalg_tests/test_einsum.py b/tests/cupy_tests/linalg_tests/test_einsum.py
new file mode 100644
index 0000000..b638ae5
--- /dev/null
+++ b/tests/cupy_tests/linalg_tests/test_einsum.py
@@ -0,0 +1,563 @@
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+rng = numpy.random.default_rng(seed=0)
+
+
+def _dec_shape(shape, dec):
+    # Test smaller shape
+    return tuple(1 if s == 1 else max(0, s - dec) for s in shape)
+
+
+def _rand1_shape(shape, prob):
+    # Test broadcast
+    # If diagonals are "broadcasted" we can simply:
+    # return tuple(1 if rng.uniform() < prob else s for s in shape)
+    table = {}
+    new_shape = []
+    for s in shape:
+        if s not in table:
+            table[s] = 1 if rng.uniform() < prob else s
+        new_shape.append(table[s])
+    return tuple(new_shape)
+
+
+def augment_einsum_testcases(*params):
+    """Modify shapes in einsum tests
+
+    Shape parameter should be starts with 'shape_'.
+    The original parameter is stored as '_raw_params'.
+
+    Args:
+        params (sequence of dicts)
+
+    Yields:
+        dict: parameter with modified shapes.
+
+    """
+    for dec in range(3):
+        for drop in [False, True]:
+            for param in params:
+                param_new = param.copy()
+                for k in param.keys():
+                    if k.startswith('shape_'):
+                        new_shape = _dec_shape(param[k], dec)
+                        if drop:
+                            prob = rng.uniform()
+                            new_shape = _rand1_shape(new_shape, prob)
+                        param_new[k] = new_shape
+                param_new['_raw_params'] = {
+                    'orig': param,
+                    'dec': dec,
+                    'drop': drop,
+                }
+                yield param_new
+
+
+class TestEinSumError:
+
+    def test_irregular_ellipsis1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('..', xp.zeros((2, 2, 2)))
+
+    def test_irregular_ellipsis2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('...i...', xp.zeros((2, 2, 2)))
+
+    def test_irregular_ellipsis3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i...->...i...', xp.zeros((2, 2, 2)))
+
+    def test_irregular_ellipsis4(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('...->', xp.zeros((2, 2, 2)))
+
+    def test_no_arguments(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum()
+
+    def test_one_argument(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('')
+
+    def test_not_string_subject(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(TypeError):
+                xp.einsum(0, 0)
+
+    def test_bad_argument(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(TypeError):
+                xp.einsum('', 0, bad_arg=0)
+
+    def test_too_many_operands1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('', 0, 0)
+
+    def test_too_many_operands2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(
+                    'i,j',
+                    xp.array([0, 0]),
+                    xp.array([0, 0]),
+                    xp.array([0, 0]))
+
+    def test_too_few_operands1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(',', 0)
+
+    def test_too_many_dimension1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i', 0)
+
+    def test_too_many_dimension2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('ij', xp.array([0, 0]))
+
+    def test_too_many_dimension3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('ijk...->...', xp.arange(6).reshape(2, 3))
+
+    def test_too_few_dimension(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i->i', xp.arange(6).reshape(2, 3))
+
+    def test_invalid_char1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i%', xp.array([0, 0]))
+
+    def test_invalid_char2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('j$', xp.array([0, 0]))
+
+    def test_invalid_char3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i->&', xp.array([0, 0]))
+
+    # output subscripts must appear in inumpy.t
+    def test_invalid_output_subscripts1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i->ij', xp.array([0, 0]))
+
+    # output subscripts may only be specified once
+    def test_invalid_output_subscripts2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('ij->jij', xp.array([[0, 0], [0, 0]]))
+
+    # output subscripts must not incrudes comma
+    def test_invalid_output_subscripts3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('ij->i,j', xp.array([[0, 0], [0, 0]]))
+
+    # dimensions much match when being collapsed
+    def test_invalid_diagonal1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('ii', xp.arange(6).reshape(2, 3))
+
+    def test_invalid_diagonal2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('ii->', xp.arange(6).reshape(2, 3))
+
+    def test_invalid_diagonal3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('ii', xp.arange(3).reshape(1, 3))
+
+    def test_dim_mismatch_char1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i,i', xp.arange(2), xp.arange(3))
+
+    def test_dim_mismatch_ellipsis1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('...,...', xp.arange(2), xp.arange(3))
+
+    def test_dim_mismatch_ellipsis2(self):
+        for xp in (numpy, cupy):
+            a = xp.arange(12).reshape(2, 3, 2)
+            with pytest.raises(ValueError):
+                xp.einsum('i...,...i', a, a)
+
+    def test_dim_mismatch_ellipsis3(self):
+        for xp in (numpy, cupy):
+            a = xp.arange(12).reshape(2, 3, 2)
+            with pytest.raises(ValueError):
+                xp.einsum('...,...', a, a[:, :2])
+
+    # invalid -> operator
+    def test_invalid_arrow1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i-i', xp.array([0, 0]))
+
+    def test_invalid_arrow2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i>i', xp.array([0, 0]))
+
+    def test_invalid_arrow3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i->->i', xp.array([0, 0]))
+
+    def test_invalid_arrow4(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum('i-', xp.array([0, 0]))
+
+
+class TestListArgEinSumError:
+
+    @testing.with_requires('numpy>=1.19')
+    def test_invalid_sub1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(TypeError):
+                xp.einsum(xp.arange(2), [None])
+
+    def test_invalid_sub2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(xp.arange(2), [0], [1])
+
+    def test_invalid_sub3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(xp.arange(2), [Ellipsis, 0, Ellipsis])
+
+    def test_dim_mismatch1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(xp.arange(2), [0], xp.arange(3), [0])
+
+    def test_dim_mismatch2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(xp.arange(2), [0], xp.arange(3), [0], [0])
+
+    def test_dim_mismatch3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(xp.arange(6).reshape(2, 3), [0, 0])
+
+    def test_too_many_dims1(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(3, [0])
+
+    def test_too_many_dims2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(xp.arange(2), [0, 1])
+
+    def test_too_many_dims3(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                xp.einsum(xp.arange(6).reshape(2, 3), [Ellipsis, 0, 1, 2])
+
+
+@pytest.mark.parametrize('do_opt', [False, True])
+class TestListArgEinSum:
+    # numpy/numpy#15961: should accept int64 type in subscript list
+    @testing.with_requires('numpy>=1.19')
+    @testing.numpy_cupy_allclose()
+    def test_numpy_15961_array(self, xp, do_opt):
+        n = 3
+        a = testing.shaped_arange((n, n), xp)
+        sub = numpy.array([0, 0])
+        return xp.einsum(a, sub, optimize=do_opt)
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.numpy_cupy_allclose()
+    def test_numpy_15961_list(self, xp, do_opt):
+        n = 3
+        a = testing.shaped_arange((n, n), xp)
+        sub = list(numpy.array([0, 0]))
+        return xp.einsum(a, sub, optimize=do_opt)
+
+
+@testing.parameterize(*augment_einsum_testcases(
+    {'shape_a': (2, 3), 'subscripts': 'ij'},  # do nothing
+    {'shape_a': (2, 3), 'subscripts': '...'},  # do nothing
+    {'shape_a': (2, 3), 'subscripts': 'ji'},  # transpose
+    {'shape_a': (3, 3), 'subscripts': 'ii->i'},  # diagonal 2d
+    {'shape_a': (3, 3, 3), 'subscripts': 'jii->ij'},  # partial diagonal 3d
+    {'shape_a': (3, 3, 3), 'subscripts': 'iji->ij'},  # partial diagonal 3d
+    {'shape_a': (3, 3, 3), 'subscripts': '...ii->...i'},  # partial diagonal 3d
+    {'shape_a': (3, 3, 3), 'subscripts': 'iii->i'},  # diagonal 3d
+    {'shape_a': (2, 3, 4), 'subscripts': 'ijk->jik'},  # swap axes
+    {'shape_a': (2, 3, 4), 'subscripts': 'ijk->kij'},  # swap axes
+    {'shape_a': (2, 3, 4), 'subscripts': 'ijk->ikj'},  # swap axes
+    {'shape_a': (2, 3, 4), 'subscripts': 'kji->ikj'},  # swap axes
+    {'shape_a': (2, 3, 4), 'subscripts': 'j...i->i...j'},  # swap axes
+    {'shape_a': (3,), 'subscripts': 'i->'},  # sum
+    {'shape_a': (3, 3), 'subscripts': 'ii'},  # trace
+    {'shape_a': (2, 2, 2, 2), 'subscripts': 'ijkj->kij'},
+    {'shape_a': (2, 2, 2, 2), 'subscripts': 'ijij->ij'},
+    {'shape_a': (2, 2, 2, 2), 'subscripts': 'jiji->ij'},
+    {'shape_a': (2, 2, 2, 2), 'subscripts': 'ii...->...'},  # trace
+    {'shape_a': (2, 2, 2, 2), 'subscripts': 'i...i->...'},  # trace
+    {'shape_a': (2, 2, 2, 2), 'subscripts': '...ii->...'},  # trace
+    {'shape_a': (2, 2, 2, 2), 'subscripts': 'j...i->...'},  # sum
+
+    {'shape_a': (2, 3), 'subscripts': 'ij->ij...'},  # do nothing
+    {'shape_a': (2, 3), 'subscripts': 'ij->i...j'},  # do nothing
+    {'shape_a': (2, 3), 'subscripts': 'ij->...ij'},  # do nothing
+    {'shape_a': (2, 3), 'subscripts': 'ij...->ij'},  # do nothing
+    {'shape_a': (2, 3), 'subscripts': 'i...j->ij'},  # do nothing
+
+    {'shape_a': (), 'subscripts': ''},  # do nothing
+    {'shape_a': (), 'subscripts': '->'},  # do nothing
+))
+class TestEinSumUnaryOperation:
+
+    @testing.for_all_dtypes(no_bool=False)
+    @testing.numpy_cupy_allclose(
+        rtol={numpy.float16: 1e-1, 'default': 1e-7}, contiguous_check=False)
+    def test_einsum_unary(self, xp, dtype):
+        a = testing.shaped_arange(self.shape_a, xp, dtype)
+        out = xp.einsum(self.subscripts, a)
+        if xp is not numpy:
+            optimized_out = xp.einsum(self.subscripts, a, optimize=True)
+            testing.assert_allclose(optimized_out, out)
+        return out
+
+    @testing.for_all_dtypes(no_bool=False)
+    @testing.numpy_cupy_equal()
+    def test_einsum_unary_views(self, xp, dtype):
+        a = testing.shaped_arange(self.shape_a, xp, dtype)
+        b = xp.einsum(self.subscripts, a)
+
+        return b.ndim == 0 or b.base is a
+
+    @testing.for_all_dtypes_combination(
+        ['dtype_a', 'dtype_out'],
+        no_bool=False,
+        no_complex=True)  # avoid ComplexWarning
+    @testing.numpy_cupy_allclose(
+        rtol={numpy.float16: 1e-1, 'default': 1e-7}, contiguous_check=False)
+    def test_einsum_unary_dtype(self, xp, dtype_a, dtype_out):
+        if not numpy.can_cast(dtype_a, dtype_out):
+            pytest.skip()
+        a = testing.shaped_arange(self.shape_a, xp, dtype_a)
+        return xp.einsum(self.subscripts, a, dtype=dtype_out)
+
+
+class TestEinSumUnaryOperationWithScalar:
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_scalar_int(self, xp, dtype):
+        return xp.asarray(xp.einsum('->', 2, dtype=dtype))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_scalar_float(self, xp, dtype):
+        return xp.asarray(xp.einsum('', 2.0, dtype=dtype))
+
+
+@testing.parameterize(*augment_einsum_testcases(
+    # dot vecvec
+    {'shape_a': (3,), 'shape_b': (3,),
+     'subscripts': 'i,i'},
+    # outer
+    {'shape_a': (2,), 'shape_b': (3,),
+     'subscripts': 'i,j'},
+    # dot matvec
+    {'shape_a': (2, 3), 'shape_b': (3,),
+     'subscripts': 'ij,j'},
+    {'shape_a': (2, 3), 'shape_b': (2,),
+     'subscripts': 'ij,i'},
+    # dot matmat
+    {'shape_a': (2, 3), 'shape_b': (3, 4),
+     'subscripts': 'ij,jk'},
+    # tensordot
+    {'shape_a': (3, 4, 2), 'shape_b': (4, 3, 2),
+     'subscripts': 'ijk, jil -> kl'},
+    {'shape_a': (3, 4, 2), 'shape_b': (4, 2, 3),
+     'subscripts': 'i...,...k->ki...'},
+    {'shape_a': (3, 4, 2), 'shape_b': (4, 3, 2),
+     'subscripts': 'ij...,ji...->i...'},
+    # trace and tensordot and diagonal
+    {'shape_a': (2, 3, 2, 4), 'shape_b': (3, 2, 2),
+     'subscripts': 'ijil,jkk->kj'},
+    {'shape_a': (2, 4, 2, 3), 'shape_b': (3, 2, 4),
+     'subscripts': 'i...ij,ji...->...j'},
+    # broadcast
+    {'shape_a': (2, 3, 4), 'shape_b': (3,),
+     'subscripts': 'ij...,j...->ij...'},
+    {'shape_a': (2, 3, 4), 'shape_b': (3,),
+     'subscripts': 'ij...,...j->ij...'},
+    {'shape_a': (2, 3, 4), 'shape_b': (3,),
+     'subscripts': 'ij...,j->ij...'},
+    {'shape_a': (4, 3), 'shape_b': (3, 2),
+     'subscripts': 'ik...,k...->i...'},
+    {'shape_a': (4, 3), 'shape_b': (3, 2),
+     'subscripts': 'ik...,...kj->i...j'},
+    {'shape_a': (4, 3), 'shape_b': (3, 2),
+     'subscripts': '...k,kj'},
+    {'shape_a': (4, 3), 'shape_b': (3, 2),
+     'subscripts': 'ik,k...->i...'},
+    {'shape_a': (2, 3, 4, 5), 'shape_b': (4,),
+     'subscripts': 'ijkl,k'},
+    {'shape_a': (2, 3, 4, 5), 'shape_b': (4,),
+     'subscripts': '...kl,k'},
+    {'shape_a': (2, 3, 4, 5), 'shape_b': (4,),
+     'subscripts': '...kl,k...'},
+    {'shape_a': (1, 1, 1, 2, 3, 2), 'shape_b': (2, 3, 2, 2),
+     'subscripts': '...lmn,lmno->...o'},
+))
+class TestEinSumBinaryOperation:
+    @testing.for_all_dtypes_combination(
+        ['dtype_a', 'dtype_b'],
+        no_bool=False,
+        no_float16=False)
+    @testing.numpy_cupy_allclose(
+        rtol={numpy.float16: 1e-2, 'default': 1e-7}, contiguous_check=False)
+    def test_einsum_binary(self, xp, dtype_a, dtype_b):
+        a = testing.shaped_arange(self.shape_a, xp, dtype_a)
+        b = testing.shaped_arange(self.shape_b, xp, dtype_b)
+        return xp.einsum(self.subscripts, a, b)
+
+
+class TestEinSumBinaryOperationWithScalar:
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_scalar_1(self, xp, dtype):
+        shape_a = (2,)
+        a = testing.shaped_arange(shape_a, xp, dtype)
+        return xp.asarray(xp.einsum(',i->', 3, a))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_scalar_2(self, xp, dtype):
+        shape_a = (2,)
+        a = testing.shaped_arange(shape_a, xp, dtype)
+        return xp.asarray(xp.einsum('i,->', a, 4))
+
+
+@testing.parameterize(*augment_einsum_testcases(
+    {'shape_a': (2, 3), 'shape_b': (3, 4), 'shape_c': (4, 5),
+     'subscripts': 'ij,jk,kl'},
+    {'shape_a': (2, 4), 'shape_b': (2, 3), 'shape_c': (2,),
+     'subscripts': 'ij,ik,i->ijk'},
+    {'shape_a': (2, 4), 'shape_b': (3, 2), 'shape_c': (2,),
+     'subscripts': 'ij,ki,i->jk'},
+    {'shape_a': (2, 3, 4), 'shape_b': (2,), 'shape_c': (3, 4, 2),
+     'subscripts': 'i...,i,...i->...i'},
+    {'shape_a': (2, 3, 4), 'shape_b': (4, 3), 'shape_c': (3, 3, 4),
+     'subscripts': 'a...,...b,c...->abc...'},
+    {'shape_a': (2, 3, 4), 'shape_b': (3, 4), 'shape_c': (3, 3, 4),
+     'subscripts': 'a...,...,c...->ac...'},
+    {'shape_a': (3, 3, 4), 'shape_b': (4, 3), 'shape_c': (2, 3, 4),
+     'subscripts': 'a...,...b,c...->abc...'},
+    {'shape_a': (3, 3, 4), 'shape_b': (3, 4), 'shape_c': (2, 3, 4),
+     'subscripts': 'a...,...,c...->ac...'},
+))
+class TestEinSumTernaryOperation:
+    @testing.for_all_dtypes_combination(
+        ['dtype_a', 'dtype_b', 'dtype_c'],
+        no_bool=False,
+        no_float16=False)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_einsum_ternary(self, xp, dtype_a, dtype_b, dtype_c):
+        a = testing.shaped_arange(self.shape_a, xp, dtype_a)
+        b = testing.shaped_arange(self.shape_b, xp, dtype_b)
+        c = testing.shaped_arange(self.shape_c, xp, dtype_c)
+
+        try:
+            out = xp.einsum(self.subscripts, a, b, c, optimize=False)
+        except TypeError:
+            assert xp is numpy
+            out = xp.einsum(self.subscripts, a, b, c)
+
+        if xp is not numpy:  # Avoid numpy issues #11059, #11060
+            for optimize in [
+                    True,  # 'greedy'
+                    'optimal',
+                    ['einsum_path', (0, 1), (0, 1)],
+                    ['einsum_path', (0, 2), (0, 1)],
+                    ['einsum_path', (1, 2), (0, 1)],
+            ]:
+                optimized_out = xp.einsum(
+                    self.subscripts, a, b, c, optimize=optimize)
+                testing.assert_allclose(optimized_out, out)
+        return out
+
+
+@testing.parameterize(*([
+    # memory constraint
+    {'subscript': 'a,b,c->abc', 'opt': ('greedy', 0)},
+    {'subscript': 'acdf,jbje,gihb,hfac', 'opt': ('greedy', 0)},
+] + testing.product({'subscript': [
+    # long paths
+    'acdf,jbje,gihb,hfac,gfac,gifabc,hfac',
+    'chd,bde,agbc,hiad,bdi,cgh,agdb',
+    # edge cases
+    'eb,cb,fb->cef',
+    'dd,fb,be,cdb->cef',
+    'bca,cdb,dbf,afc->',
+    'dcc,fce,ea,dbf->ab',
+    'a,ac,ab,ad,cd,bd,bc->',
+], 'opt': ['greedy', 'optimal'],
+})))
+class TestEinSumLarge:
+
+    chars = 'abcdefghij'
+    sizes = (2, 3, 4, 5, 4, 3, 2, 6, 5, 4, 3)
+    size_dict = {}
+    for size, char in zip(sizes, chars):
+        size_dict[char] = size
+
+    @pytest.fixture()
+    def shapes(self):
+        size_dict = self.size_dict
+        terms = self.subscript.split('->')[0].split(',')
+        return tuple([
+            tuple([size_dict[x] for x in term])
+            for term in terms
+        ])
+
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_einsum(self, xp, shapes):
+        arrays = [
+            testing.shaped_random(shape, xp, float, scale=1)
+            for shape in shapes
+        ]
+        # TODO(kataoka): support memory efficient cupy.einsum
+        with warnings.catch_warnings(record=True) as ws:
+            # I hope there's no problem with np.einsum for these cases...
+            out = xp.einsum(self.subscript, *arrays, optimize=self.opt)
+            if xp is not numpy and \
+                    isinstance(self.opt, tuple):  # with memory limit
+                for w in ws:
+                    assert 'memory' in str(w.message)
+            else:
+                assert len(ws) == 0
+        return out
diff --git a/tests/cupy_tests/linalg_tests/test_norms.py b/tests/cupy_tests/linalg_tests/test_norms.py
new file mode 100644
index 0000000..0ce1c66
--- /dev/null
+++ b/tests/cupy_tests/linalg_tests/test_norms.py
@@ -0,0 +1,208 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx
+
+
+class TestTrace(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_trace(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4, 5), xp, dtype)
+        return a.trace(1, 3, 2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_trace(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4, 5), xp, dtype)
+        return xp.trace(a, 1, 3, 2)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(1,), (2,)],
+    'ord': [-numpy.inf, -2, -1, 0, 1, 2, 3, numpy.inf],
+    'axis': [0, None],
+    'keepdims': [True, False],
+}) + testing.product({
+    'shape': [(1, 2), (2, 2)],
+    'ord': [-numpy.inf, -2, -1, 1, 2, numpy.inf, 'fro', 'nuc'],
+    'axis': [(0, 1), None],
+    'keepdims': [True, False],
+}) + testing.product({
+    'shape': [(2, 2, 2)],
+    'ord': [-numpy.inf, -2, -1, 0, 1, 2, 3, numpy.inf],
+    'axis': [0, 1, 2],
+    'keepdims': [True, False],
+}) + testing.product({
+    'shape': [(2, 2, 2)],
+    'ord': [-numpy.inf, -1, 1, numpy.inf, 'fro'],
+    'axis': [(0, 1), (0, 2), (1, 2)],
+    'keepdims': [True, False],
+})
+)
+class TestNorm(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_norm(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        res = xp.linalg.norm(a, self.ord, self.axis, self.keepdims)
+        if xp == numpy and not isinstance(res, numpy.ndarray):
+            real_dtype = a.real.dtype
+            if issubclass(real_dtype.type, numpy.inexact):
+                # Avoid numpy bug. See numpy/numpy#10667
+                res = res.astype(a.real.dtype)
+        return res
+
+
+@testing.parameterize(*testing.product({
+    'array': [
+        [[1, 2], [3, 4]],
+        [[1, 2], [1, 2]],
+        [[0, 0], [0, 0]],
+        [1, 2],
+        [0, 1],
+        [0, 0],
+    ],
+    'tol': [None, 1]
+}))
+class TestMatrixRank(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_float16=True, no_complex=True)
+    @testing.numpy_cupy_array_equal(type_check=True)
+    def test_matrix_rank(self, xp, dtype):
+        a = xp.array(self.array, dtype=dtype)
+        y = xp.linalg.matrix_rank(a, tol=self.tol)
+        if xp is cupy:
+            assert isinstance(y, cupy.ndarray)
+            assert y.shape == ()
+        else:
+            # Note numpy returns numpy scalar or python int
+            y = xp.array(y)
+        return y
+
+
+class TestDet(unittest.TestCase):
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_det(self, xp, dtype):
+        a = testing.shaped_arange((2, 2), xp, dtype) + 1
+        return xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_det_3(self, xp, dtype):
+        a = testing.shaped_arange((2, 2, 2), xp, dtype) + 1
+        return xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_det_4(self, xp, dtype):
+        a = testing.shaped_arange((2, 2, 2, 2), xp, dtype) + 1
+        return xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_det_empty_batch(self, xp, dtype):
+        a = xp.empty((2, 0, 3, 3), dtype)
+        return xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_det_empty_matrix(self, xp, dtype):
+        a = xp.empty((0, 0), dtype)
+        return xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_det_empty_matrices(self, xp, dtype):
+        a = xp.empty((2, 3, 0, 0), dtype)
+        return xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    def test_det_different_last_two_dims(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 2), xp, dtype)
+            with pytest.raises(numpy.linalg.LinAlgError):
+                xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    def test_det_different_last_two_dims_empty_batch(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.empty((0, 3, 2), dtype)
+            with pytest.raises(numpy.linalg.LinAlgError):
+                xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    def test_det_one_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2,), xp, dtype)
+            with pytest.raises(numpy.linalg.LinAlgError):
+                xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    def test_det_zero_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), xp, dtype)
+            with pytest.raises(numpy.linalg.LinAlgError):
+                xp.linalg.det(a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_det_singular(self, xp, dtype):
+        a = xp.zeros((2, 3, 3), dtype)
+        return xp.linalg.det(a)
+
+
+class TestSlogdet(unittest.TestCase):
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_slogdet(self, xp, dtype):
+        a = testing.shaped_arange((2, 2), xp, dtype) + 1
+        sign, logdet = xp.linalg.slogdet(a)
+        return sign, logdet
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_slogdet_3(self, xp, dtype):
+        a = testing.shaped_arange((2, 2, 2), xp, dtype) + 1
+        sign, logdet = xp.linalg.slogdet(a)
+        return sign, logdet
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_slogdet_4(self, xp, dtype):
+        a = testing.shaped_arange((2, 2, 2, 2), xp, dtype) + 1
+        sign, logdet = xp.linalg.slogdet(a)
+        return sign, logdet
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_slogdet_singular(self, xp, dtype):
+        a = xp.zeros((3, 3), dtype)
+        sign, logdet = xp.linalg.slogdet(a)
+        return sign, logdet
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4)
+    def test_slogdet_singular_errstate(self, xp, dtype):
+        a = xp.zeros((3, 3), dtype)
+        with cupyx.errstate(linalg='raise'):
+            # `cupy.linalg.slogdet` internally catches `dev_info < 0` from
+            # cuSOLVER, which should not affect `dev_info > 0` cases.
+            sign, logdet = xp.linalg.slogdet(a)
+        return sign, logdet
+
+    @testing.for_dtypes('fdFD')
+    def test_slogdet_one_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2,), xp, dtype)
+            with pytest.raises(numpy.linalg.LinAlgError):
+                xp.linalg.slogdet(a)
diff --git a/tests/cupy_tests/linalg_tests/test_product.py b/tests/cupy_tests/linalg_tests/test_product.py
new file mode 100644
index 0000000..f00013e
--- /dev/null
+++ b/tests/cupy_tests/linalg_tests/test_product.py
@@ -0,0 +1,478 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(*testing.product({
+    'shape': [
+        ((2, 3, 4), (3, 4, 2)),
+        ((1, 1), (1, 1)),
+        ((1, 1), (1, 2)),
+        ((1, 2), (2, 1)),
+        ((2, 1), (1, 1)),
+        ((1, 2), (2, 3)),
+        ((2, 1), (1, 3)),
+        ((2, 3), (3, 1)),
+        ((2, 3), (3, 4)),
+        ((0, 3), (3, 4)),
+        ((2, 3), (3, 0)),
+        ((0, 3), (3, 0)),
+        ((3, 0), (0, 4)),
+        ((2, 3, 0), (3, 0, 2)),
+        ((0, 0), (0, 0)),
+        ((3,), (3,)),
+        ((2,), (2, 4)),
+        ((4, 2), (2,)),
+    ],
+    'trans_a': [True, False],
+    'trans_b': [True, False],
+}))
+class TestDot(unittest.TestCase):
+
+    @testing.for_all_dtypes_combination(['dtype_a', 'dtype_b'])
+    @testing.numpy_cupy_allclose()
+    def test_dot(self, xp, dtype_a, dtype_b):
+        shape_a, shape_b = self.shape
+        if self.trans_a:
+            a = testing.shaped_arange(shape_a[::-1], xp, dtype_a).T
+        else:
+            a = testing.shaped_arange(shape_a, xp, dtype_a)
+        if self.trans_b:
+            b = testing.shaped_arange(shape_b[::-1], xp, dtype_b).T
+        else:
+            b = testing.shaped_arange(shape_b, xp, dtype_b)
+        return xp.dot(a, b)
+
+    @testing.for_float_dtypes(name='dtype_a')
+    @testing.for_float_dtypes(name='dtype_b')
+    @testing.for_float_dtypes(name='dtype_c')
+    @testing.numpy_cupy_allclose(accept_error=ValueError)
+    def test_dot_with_out(self, xp, dtype_a, dtype_b, dtype_c):
+        shape_a, shape_b = self.shape
+        if self.trans_a:
+            a = testing.shaped_arange(shape_a[::-1], xp, dtype_a).T
+        else:
+            a = testing.shaped_arange(shape_a, xp, dtype_a)
+        if self.trans_b:
+            b = testing.shaped_arange(shape_b[::-1], xp, dtype_b).T
+        else:
+            b = testing.shaped_arange(shape_b, xp, dtype_b)
+        if a.ndim == 0 or b.ndim == 0:
+            shape_c = shape_a + shape_b
+        else:
+            shape_c = shape_a[:-1] + shape_b[:-2] + shape_b[-1:]
+        c = xp.empty(shape_c, dtype=dtype_c)
+        out = xp.dot(a, b, out=c)
+        assert out is c
+        return c
+
+
+@testing.parameterize(*testing.product({
+    'params': [
+        #  Test for 0 dimension
+        ((3, ), (3, ), -1, -1, -1),
+        #  Test for basic cases
+        ((1, 2), (1, 2), -1, -1, 1),
+        ((1, 3), (1, 3), 1, -1, -1),
+        ((1, 2), (1, 3), -1, -1, 1),
+        ((2, 2), (1, 3), -1, -1, 0),
+        ((3, 3), (1, 2), 0, -1, -1),
+        ((0, 3), (0, 3), -1, -1, -1),
+        #  Test for higher dimensions
+        ((2, 0, 3), (2, 0, 3), 0, 0, 0),
+        ((2, 4, 5, 3), (2, 4, 5, 3), -1, -1, 0),
+        ((2, 4, 5, 2), (2, 4, 5, 2), 0, 0, -1),
+    ],
+}))
+class TestCrossProduct(unittest.TestCase):
+
+    @testing.for_all_dtypes_combination(['dtype_a', 'dtype_b'])
+    @testing.numpy_cupy_allclose()
+    def test_cross(self, xp, dtype_a, dtype_b):
+        if dtype_a == dtype_b == numpy.bool_:
+            # cross does not support bool-bool inputs.
+            return xp.array(True)
+        shape_a, shape_b, axisa, axisb, axisc = self.params
+        a = testing.shaped_arange(shape_a, xp, dtype_a)
+        b = testing.shaped_arange(shape_b, xp, dtype_b)
+        return xp.cross(a, b, axisa, axisb, axisc)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [
+        ((), ()),
+        ((), (2, 4)),
+        ((4, 2), ()),
+    ],
+    'trans_a': [True, False],
+    'trans_b': [True, False],
+}))
+class TestDotFor0Dim(unittest.TestCase):
+
+    @testing.for_all_dtypes_combination(['dtype_a', 'dtype_b'])
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_dot(self, xp, dtype_a, dtype_b):
+        shape_a, shape_b = self.shape
+        if self.trans_a:
+            a = testing.shaped_arange(shape_a[::-1], xp, dtype_a).T
+        else:
+            a = testing.shaped_arange(shape_a, xp, dtype_a)
+        if self.trans_b:
+            b = testing.shaped_arange(shape_b[::-1], xp, dtype_b).T
+        else:
+            b = testing.shaped_arange(shape_b, xp, dtype_b)
+        return xp.dot(a, b)
+
+
+class TestProduct(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_dot_vec1(self, xp, dtype):
+        a = testing.shaped_arange((2,), xp, dtype)
+        b = testing.shaped_arange((2,), xp, dtype)
+        return xp.dot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_dot_vec2(self, xp, dtype):
+        a = testing.shaped_arange((2,), xp, dtype)
+        b = testing.shaped_arange((2, 1), xp, dtype)
+        return xp.dot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_dot_vec3(self, xp, dtype):
+        a = testing.shaped_arange((1, 2), xp, dtype)
+        b = testing.shaped_arange((2,), xp, dtype)
+        return xp.dot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_transposed_dot(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype).transpose(1, 0, 2)
+        b = testing.shaped_arange((2, 3, 4), xp, dtype).transpose(0, 2, 1)
+        return xp.dot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_transposed_dot_with_out(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype).transpose(1, 0, 2)
+        b = testing.shaped_arange((4, 2, 3), xp, dtype).transpose(2, 0, 1)
+        c = xp.ndarray((3, 2, 3, 2), dtype=dtype)
+        xp.dot(a, b, out=c)
+        return c
+
+    @testing.for_all_dtypes()
+    def test_transposed_dot_with_out_f_contiguous(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp, dtype).transpose(1, 0, 2)
+            b = testing.shaped_arange((4, 2, 3), xp, dtype).transpose(2, 0, 1)
+            c = xp.ndarray((3, 2, 3, 2), dtype=dtype, order='F')
+            with pytest.raises(ValueError):
+                # Only C-contiguous array is acceptable
+                xp.dot(a, b, out=c)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_dot_with_single_elem_array1(self, xp, dtype):
+        a = testing.shaped_arange((3, 1), xp, dtype)
+        b = xp.array([[2]], dtype=dtype)
+        return xp.dot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_dot_with_single_elem_array2(self, xp, dtype):
+        a = xp.array([[2]], dtype=dtype)
+        b = testing.shaped_arange((1, 3), xp, dtype)
+        return xp.dot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_vdot(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        b = testing.shaped_reverse_arange((5,), xp, dtype)
+        return xp.vdot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_reversed_vdot(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)[::-1]
+        b = testing.shaped_reverse_arange((5,), xp, dtype)[::-1]
+        return xp.vdot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_multidim_vdot(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_arange((2, 2, 2, 3), xp, dtype)
+        return xp.vdot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_transposed_multidim_vdot(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype).transpose(2, 0, 1)
+        b = testing.shaped_arange(
+            (2, 2, 2, 3), xp, dtype).transpose(1, 3, 0, 2)
+        return xp.vdot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_inner(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        b = testing.shaped_reverse_arange((5,), xp, dtype)
+        return xp.inner(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_reversed_inner(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)[::-1]
+        b = testing.shaped_reverse_arange((5,), xp, dtype)[::-1]
+        return xp.inner(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_multidim_inner(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_arange((3, 2, 4), xp, dtype)
+        return xp.inner(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_transposed_higher_order_inner(self, xp, dtype):
+        a = testing.shaped_arange((2, 4, 3), xp, dtype).transpose(2, 0, 1)
+        b = testing.shaped_arange((4, 2, 3), xp, dtype).transpose(1, 2, 0)
+        return xp.inner(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_outer(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        b = testing.shaped_arange((4,), xp, dtype)
+        return xp.outer(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_reversed_outer(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        b = testing.shaped_arange((4,), xp, dtype)
+        return xp.outer(a[::-1], b[::-1])
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_multidim_outer(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.outer(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_tensordot(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_arange((3, 4, 5), xp, dtype)
+        return xp.tensordot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_transposed_tensordot(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype).transpose(1, 0, 2)
+        b = testing.shaped_arange((4, 3, 2), xp, dtype).transpose(2, 0, 1)
+        return xp.tensordot(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_tensordot_with_int_axes(self, xp, dtype):
+        if dtype in (numpy.uint8, numpy.int8, numpy.uint16, numpy.int16):
+            a = testing.shaped_arange((1, 2, 3), xp, dtype)
+            b = testing.shaped_arange((2, 3, 1), xp, dtype)
+            return xp.tensordot(a, b, axes=2)
+        else:
+            a = testing.shaped_arange((2, 3, 4, 5), xp, dtype)
+            b = testing.shaped_arange((3, 4, 5, 2), xp, dtype)
+            return xp.tensordot(a, b, axes=3)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_transposed_tensordot_with_int_axes(self, xp, dtype):
+        if dtype in (numpy.uint8, numpy.int8, numpy.uint16, numpy.int16):
+            # Avoid overflow
+            a = testing.shaped_arange(
+                (1, 2, 3), xp, dtype).transpose(2, 0, 1)
+            b = testing.shaped_arange(
+                (3, 2, 1), xp, dtype).transpose(2, 1, 0)
+            return xp.tensordot(a, b, axes=2)
+        else:
+            a = testing.shaped_arange(
+                (2, 3, 4, 5), xp, dtype).transpose(2, 0, 3, 1)
+            b = testing.shaped_arange(
+                (5, 4, 3, 2), xp, dtype).transpose(3, 0, 2, 1)
+            return xp.tensordot(a, b, axes=3)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_tensordot_with_list_axes(self, xp, dtype):
+        if dtype in (numpy.uint8, numpy.int8, numpy.uint16, numpy.int16):
+            # Avoid overflow
+            a = testing.shaped_arange((1, 2, 3), xp, dtype)
+            b = testing.shaped_arange((3, 1, 2), xp, dtype)
+            return xp.tensordot(a, b, axes=([2, 1], [0, 2]))
+        else:
+            a = testing.shaped_arange((2, 3, 4, 5), xp, dtype)
+            b = testing.shaped_arange((3, 5, 4, 2), xp, dtype)
+            return xp.tensordot(a, b, axes=([3, 2, 1], [1, 2, 0]))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_transposed_tensordot_with_list_axes(self, xp, dtype):
+        if dtype in (numpy.uint8, numpy.int8, numpy.uint16, numpy.int16):
+            # Avoid overflow
+            a = testing.shaped_arange(
+                (1, 2, 3), xp, dtype).transpose(2, 0, 1)
+            b = testing.shaped_arange(
+                (2, 3, 1), xp, dtype).transpose(0, 2, 1)
+            return xp.tensordot(a, b, axes=([2, 0], [0, 2]))
+        else:
+            a = testing.shaped_arange(
+                (2, 3, 4, 5), xp, dtype).transpose(2, 0, 3, 1)
+            b = testing.shaped_arange(
+                (3, 5, 4, 2), xp, dtype).transpose(3, 0, 2, 1)
+            return xp.tensordot(a, b, axes=([2, 0, 3], [3, 2, 1]))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_tensordot_zero_dim(self, xp, dtype):
+        a = xp.array(2, dtype=dtype)
+        b = testing.shaped_arange((3, 4, 2), xp, dtype)
+        return xp.tensordot(a, b, axes=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_kron(self, xp, dtype):
+        a = testing.shaped_arange((4,), xp, dtype)
+        b = testing.shaped_arange((5,), xp, dtype)
+        return xp.kron(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_reversed_kron(self, xp, dtype):
+        a = testing.shaped_arange((4,), xp, dtype)
+        b = testing.shaped_arange((5,), xp, dtype)
+        return xp.kron(a[::-1], b[::-1])
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_multidim_kron(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_arange((4, 2, 3), xp, dtype)
+        return xp.kron(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_zerodim_kron(self, xp, dtype):
+        a = xp.array(2, dtype=dtype)
+        b = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.kron(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'params': [
+        ((0, 0), 2),
+        ((0, 0), (1, 0)),
+        ((0, 0, 0), 2),
+        ((0, 0, 0), 3),
+        ((0, 0, 0), ([2, 1], [0, 2])),
+        ((0, 0, 0), ([0, 2, 1], [1, 2, 0])),
+    ],
+}))
+class TestProductZeroLength(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_tensordot_zero_length(self, xp, dtype):
+        shape, axes = self.params
+        a = testing.shaped_arange(shape, xp, dtype)
+        return xp.tensordot(a, a, axes=axes)
+
+
+class TestMatrixPower(unittest.TestCase):
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_matrix_power_0(self, xp, dtype):
+        a = testing.shaped_arange((3, 3), xp, dtype)
+        return xp.linalg.matrix_power(a, 0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_matrix_power_1(self, xp, dtype):
+        a = testing.shaped_arange((3, 3), xp, dtype)
+        return xp.linalg.matrix_power(a, 1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_matrix_power_2(self, xp, dtype):
+        a = testing.shaped_arange((3, 3), xp, dtype)
+        return xp.linalg.matrix_power(a, 2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_matrix_power_3(self, xp, dtype):
+        a = testing.shaped_arange((3, 3), xp, dtype)
+        return xp.linalg.matrix_power(a, 3)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_matrix_power_inv1(self, xp, dtype):
+        a = testing.shaped_arange((3, 3), xp, dtype)
+        a = a * a % 30
+        return xp.linalg.matrix_power(a, -1)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_matrix_power_inv2(self, xp, dtype):
+        a = testing.shaped_arange((3, 3), xp, dtype)
+        a = a * a % 30
+        return xp.linalg.matrix_power(a, -2)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_matrix_power_inv3(self, xp, dtype):
+        a = testing.shaped_arange((3, 3), xp, dtype)
+        a = a * a % 30
+        return xp.linalg.matrix_power(a, -3)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_matrix_power_of_two(self, xp, dtype):
+        a = xp.eye(23, k=17, dtype=dtype) + xp.eye(23, k=-6, dtype=dtype)
+        return xp.linalg.matrix_power(a, 1 << 50)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_matrix_power_large(self, xp, dtype):
+        a = xp.eye(23, k=17, dtype=dtype) + xp.eye(23, k=-6, dtype=dtype)
+        return xp.linalg.matrix_power(a, 123456789123456789)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose()
+    def test_matrix_power_invlarge(self, xp, dtype):
+        a = xp.eye(23, k=17, dtype=dtype) + xp.eye(23, k=-6, dtype=dtype)
+        return xp.linalg.matrix_power(a, -987654321987654321)
+
+
+@pytest.mark.parametrize('shape', [
+    (2, 3, 3),
+    (3, 0, 0),
+])
+@pytest.mark.parametrize('n', [0, 5, -7])
+class TestMatrixPowerBatched:
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=5e-5)
+    def test_matrix_power_batched(self, xp, dtype, shape, n):
+        a = testing.shaped_arange(shape, xp, dtype)
+        a += xp.identity(shape[-1], dtype)
+        return xp.linalg.matrix_power(a, n)
diff --git a/tests/cupy_tests/linalg_tests/test_solve.py b/tests/cupy_tests/linalg_tests/test_solve.py
new file mode 100644
index 0000000..e677d5b
--- /dev/null
+++ b/tests/cupy_tests/linalg_tests/test_solve.py
@@ -0,0 +1,334 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.testing import _condition
+import cupyx
+from cupy.cublas import get_batched_gesv_limit, set_batched_gesv_limit
+
+
+@testing.parameterize(*testing.product({
+    'batched_gesv_limit': [None, 0],
+    'order': ['C', 'F'],
+}))
+@testing.fix_random()
+class TestSolve(unittest.TestCase):
+
+    def setUp(self):
+        if self.batched_gesv_limit is not None:
+            self.old_limit = get_batched_gesv_limit()
+            set_batched_gesv_limit(self.batched_gesv_limit)
+
+    def tearDown(self):
+        if self.batched_gesv_limit is not None:
+            set_batched_gesv_limit(self.old_limit)
+
+    @testing.for_dtypes('ifdFD')
+    # TODO(kataoka): Fix contiguity
+    @testing.numpy_cupy_allclose(atol=1e-3, contiguous_check=False)
+    def check_x(self, a_shape, b_shape, xp, dtype):
+        a = testing.shaped_random(a_shape, xp, dtype=dtype, seed=0, scale=20)
+        b = testing.shaped_random(b_shape, xp, dtype=dtype, seed=1)
+        a = a.copy(order=self.order)
+        b = b.copy(order=self.order)
+        a_copy = a.copy()
+        b_copy = b.copy()
+        result = xp.linalg.solve(a, b)
+        cupy.testing.assert_array_equal(a_copy, a)
+        cupy.testing.assert_array_equal(b_copy, b)
+        return result
+
+    def test_solve(self):
+        self.check_x((4, 4), (4,))
+        self.check_x((5, 5), (5, 2))
+        self.check_x((2, 4, 4), (2, 4,))
+        self.check_x((2, 5, 5), (2, 5, 2))
+        self.check_x((2, 3, 2, 2), (2, 3, 2,))
+        self.check_x((2, 3, 3, 3), (2, 3, 3, 2))
+        self.check_x((0, 0), (0,))
+        self.check_x((0, 0), (0, 2))
+        self.check_x((0, 2, 2), (0, 2,))
+        self.check_x((0, 2, 2), (0, 2, 3))
+
+    def check_shape(self, a_shape, b_shape, error_type):
+        for xp in (numpy, cupy):
+            a = xp.random.rand(*a_shape)
+            b = xp.random.rand(*b_shape)
+            with pytest.raises(error_type):
+                xp.linalg.solve(a, b)
+
+    @testing.numpy_cupy_allclose()
+    def test_solve_singular_empty(self, xp):
+        a = xp.zeros((3, 3))  # singular
+        b = xp.empty((3, 0))  # nrhs = 0
+        # LinAlgError("Singular matrix") is not raised
+        return xp.linalg.solve(a, b)
+
+    def test_invalid_shape(self):
+        self.check_shape((2, 3), (4,), numpy.linalg.LinAlgError)
+        self.check_shape((3, 3), (2,), ValueError)
+        self.check_shape((3, 3), (2, 2), ValueError)
+        self.check_shape((3, 3, 4), (3,), numpy.linalg.LinAlgError)
+        self.check_shape((2, 3, 3), (3,), ValueError)
+        self.check_shape((3, 3), (0,), ValueError)
+        self.check_shape((0, 3, 4), (3,), numpy.linalg.LinAlgError)
+
+
+@testing.parameterize(*testing.product({
+    'a_shape': [(2, 3, 6), (3, 4, 4, 3)],
+    'axes': [None, (0, 2)],
+}))
+@testing.fix_random()
+class TestTensorSolve(unittest.TestCase):
+
+    @testing.for_dtypes('ifdFD')
+    @testing.numpy_cupy_allclose(atol=0.02)
+    def test_tensorsolve(self, xp, dtype):
+        a_shape = self.a_shape
+        b_shape = self.a_shape[:2]
+        a = testing.shaped_random(a_shape, xp, dtype=dtype, seed=0)
+        b = testing.shaped_random(b_shape, xp, dtype=dtype, seed=1)
+        return xp.linalg.tensorsolve(a, b, axes=self.axes)
+
+
+@testing.parameterize(*testing.product({
+    'order': ['C', 'F'],
+}))
+class TestInv(unittest.TestCase):
+
+    @testing.for_dtypes('ifdFD')
+    @_condition.retry(10)
+    def check_x(self, a_shape, dtype):
+        a_cpu = numpy.random.randint(0, 10, size=a_shape)
+        a_cpu = a_cpu.astype(dtype, order=self.order)
+        a_gpu = cupy.asarray(a_cpu, order=self.order)
+        a_gpu_copy = a_gpu.copy()
+        result_cpu = numpy.linalg.inv(a_cpu)
+        result_gpu = cupy.linalg.inv(a_gpu)
+        assert result_cpu.dtype == result_gpu.dtype
+        cupy.testing.assert_allclose(result_cpu, result_gpu, atol=1e-3)
+        cupy.testing.assert_array_equal(a_gpu_copy, a_gpu)
+
+    def check_shape(self, a_shape):
+        a = cupy.random.rand(*a_shape)
+        with self.assertRaises(numpy.linalg.LinAlgError):
+            cupy.linalg.inv(a)
+
+    def test_inv(self):
+        self.check_x((3, 3))
+        self.check_x((4, 4))
+        self.check_x((5, 5))
+        self.check_x((2, 5, 5))
+        self.check_x((3, 4, 4))
+        self.check_x((4, 2, 3, 3))
+        self.check_x((0, 0))
+        self.check_x((3, 0, 0))
+        self.check_x((2, 0, 3, 4, 4))
+
+    def test_invalid_shape(self):
+        self.check_shape((2, 3))
+        self.check_shape((4, 1))
+        self.check_shape((4, 3, 2))
+        self.check_shape((2, 4, 3))
+        self.check_shape((2, 0))
+        self.check_shape((0, 2, 3))
+
+
+class TestInvInvalid(unittest.TestCase):
+
+    @testing.for_dtypes('ifdFD')
+    def test_inv(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.array([[1, 2], [2, 4]]).astype(dtype)
+            with cupyx.errstate(linalg='raise'):
+                with pytest.raises(numpy.linalg.LinAlgError):
+                    xp.linalg.inv(a)
+
+    @testing.for_dtypes('ifdFD')
+    def test_batched_inv(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.array([[[1, 2], [2, 4]]]).astype(dtype)
+            assert a.ndim >= 3  # CuPy internally uses a batched function.
+            with cupyx.errstate(linalg='raise'):
+                with pytest.raises(numpy.linalg.LinAlgError):
+                    xp.linalg.inv(a)
+
+
+class TestPinv(unittest.TestCase):
+
+    @testing.for_dtypes('ifdFD')
+    @_condition.retry(10)
+    def check_x(self, a_shape, rcond, dtype):
+        a_gpu = testing.shaped_random(a_shape, dtype=dtype)
+        a_cpu = cupy.asnumpy(a_gpu)
+        a_gpu_copy = a_gpu.copy()
+        if not isinstance(rcond, float):
+            rcond = numpy.asarray(rcond)
+        result_cpu = numpy.linalg.pinv(a_cpu, rcond=rcond)
+        if not isinstance(rcond, float):
+            rcond = cupy.asarray(rcond)
+        result_gpu = cupy.linalg.pinv(a_gpu, rcond=rcond)
+
+        assert result_cpu.dtype == result_gpu.dtype
+        cupy.testing.assert_allclose(result_cpu, result_gpu, atol=1e-3)
+        cupy.testing.assert_array_equal(a_gpu_copy, a_gpu)
+
+    def test_pinv(self):
+        self.check_x((3, 3), rcond=1e-15)
+        self.check_x((2, 4), rcond=1e-15)
+        self.check_x((3, 2), rcond=1e-15)
+
+        self.check_x((4, 4), rcond=0.3)
+        self.check_x((2, 5), rcond=0.5)
+        self.check_x((5, 3), rcond=0.6)
+
+    def test_pinv_batched(self):
+        self.check_x((2, 3, 4), rcond=1e-15)
+        self.check_x((2, 3, 4, 5), rcond=1e-15)
+
+    def test_pinv_batched_vector_rcond(self):
+        self.check_x((2, 3, 4), rcond=[0.2, 0.8])
+        self.check_x((2, 3, 4, 5),
+                     rcond=[[0.2, 0.9, 0.1],
+                            [0.7, 0.2, 0.5]])
+
+    def test_pinv_size_0(self):
+        self.check_x((3, 0), rcond=1e-15)
+        self.check_x((0, 3), rcond=1e-15)
+        self.check_x((0, 0), rcond=1e-15)
+        self.check_x((0, 2, 3), rcond=1e-15)
+        self.check_x((2, 0, 3), rcond=1e-15)
+
+
+class TestLstsq:
+
+    @testing.for_dtypes('ifdFD')
+    @testing.numpy_cupy_allclose(atol=1e-3)
+    def check_lstsq_solution(self, a_shape, b_shape, seed, rcond, xp, dtype,
+                             singular=False):
+        if singular:
+            m, n = a_shape
+            rank = min(m, n) - 1
+            a = xp.matmul(
+                testing.shaped_random(
+                    (m, rank), xp, dtype=dtype, scale=3, seed=seed),
+                testing.shaped_random(
+                    (rank, n), xp, dtype=dtype, scale=3, seed=seed+42),
+            )
+        else:
+            a = testing.shaped_random(a_shape, xp, dtype=dtype, seed=seed)
+        b = testing.shaped_random(b_shape, xp, dtype=dtype, seed=seed+37)
+        a_copy = a.copy()
+        b_copy = b.copy()
+        results = xp.linalg.lstsq(a, b, rcond)
+        if xp is cupy:
+            testing.assert_array_equal(a_copy, a)
+            testing.assert_array_equal(b_copy, b)
+        return results
+
+    def check_invalid_shapes(self, a_shape, b_shape):
+        a = cupy.random.rand(*a_shape)
+        b = cupy.random.rand(*b_shape)
+        with pytest.raises(numpy.linalg.LinAlgError):
+            cupy.linalg.lstsq(a, b, rcond=None)
+
+    def test_lstsq_solutions(self):
+        # Comapres numpy.linalg.lstsq and cupy.linalg.lstsq solutions for:
+        #   a shapes range from (3, 3) to (5, 3) and (3, 5)
+        #   b shapes range from (i, 3) to (i, )
+        #   sets a random seed for deterministic testing
+        for i in range(3, 6):
+            for j in range(3, 6):
+                for k in range(2, 4):
+                    seed = i + j + k
+                    # check when b has shape (i, k)
+                    self.check_lstsq_solution((i, j), (i, k), seed,
+                                              rcond=-1)
+                    self.check_lstsq_solution((i, j), (i, k), seed,
+                                              rcond=None)
+                    self.check_lstsq_solution((i, j), (i, k), seed,
+                                              rcond=0.5)
+                    self.check_lstsq_solution((i, j), (i, k), seed,
+                                              rcond=1e-6, singular=True)
+                # check when b has shape (i, )
+                self.check_lstsq_solution((i, j), (i, ), seed+1, rcond=-1)
+                self.check_lstsq_solution((i, j), (i, ), seed+1, rcond=None)
+                self.check_lstsq_solution((i, j), (i, ), seed+1, rcond=0.5)
+                self.check_lstsq_solution((i, j), (i, ), seed+1, rcond=1e-6,
+                                          singular=True)
+
+    @pytest.mark.parametrize("rcond", [-1, None, 0.5])
+    @pytest.mark.parametrize("k", [None, 0, 1, 4])
+    @pytest.mark.parametrize(("i", "j"), [(0, 0), (3, 0), (0, 7)])
+    @testing.for_dtypes('ifdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_lstsq_empty_matrix(self, xp, dtype, i, j, k, rcond):
+        a = xp.empty((i, j), dtype)
+        assert a.size == 0
+        b_shape = (i,) if k is None else (i, k)
+        b = testing.shaped_random(b_shape, xp, dtype)
+        return xp.linalg.lstsq(a, b, rcond)
+
+    def test_invalid_shapes(self):
+        self.check_invalid_shapes((4, 3), (3, ))
+        self.check_invalid_shapes((3, 3, 3), (2, 2))
+        self.check_invalid_shapes((3, 3, 3), (3, 3))
+        self.check_invalid_shapes((3, 3), (3, 3, 3))
+        self.check_invalid_shapes((2, 2), (10, ))
+        self.check_invalid_shapes((3, 3), (2, 2))
+        self.check_invalid_shapes((4, 3), (10, 3, 3))
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-3)
+    def test_warn_rcond(self, xp, dtype):
+        a = testing.shaped_random((3, 3), xp, dtype)
+        b = testing.shaped_random((3,), xp, dtype)
+        with testing.assert_warns(FutureWarning):
+            return xp.linalg.lstsq(a, b)
+
+
+class TestTensorInv(unittest.TestCase):
+
+    @testing.for_dtypes('ifdFD')
+    @_condition.retry(10)
+    def check_x(self, a_shape, ind, dtype):
+        a_cpu = numpy.random.randint(0, 10, size=a_shape).astype(dtype)
+        a_gpu = cupy.asarray(a_cpu)
+        a_gpu_copy = a_gpu.copy()
+        result_cpu = numpy.linalg.tensorinv(a_cpu, ind=ind)
+        result_gpu = cupy.linalg.tensorinv(a_gpu, ind=ind)
+        assert result_cpu.dtype == result_gpu.dtype
+        cupy.testing.assert_allclose(result_cpu, result_gpu, atol=1e-3)
+        cupy.testing.assert_array_equal(a_gpu_copy, a_gpu)
+
+    def check_shape(self, a_shape, ind):
+        a = cupy.random.rand(*a_shape)
+        with self.assertRaises(numpy.linalg.LinAlgError):
+            cupy.linalg.tensorinv(a, ind=ind)
+
+    def check_ind(self, a_shape, ind):
+        a = cupy.random.rand(*a_shape)
+        with self.assertRaises(ValueError):
+            cupy.linalg.tensorinv(a, ind=ind)
+
+    def test_tensorinv(self):
+        self.check_x((12, 3, 4), ind=1)
+        self.check_x((3, 8, 24), ind=2)
+        self.check_x((18, 3, 3, 2), ind=1)
+        self.check_x((1, 4, 2, 2), ind=2)
+        self.check_x((2, 3, 5, 30), ind=3)
+        self.check_x((24, 2, 2, 3, 2), ind=1)
+        self.check_x((3, 4, 2, 3, 2), ind=2)
+        self.check_x((1, 2, 3, 2, 3), ind=3)
+        self.check_x((3, 2, 1, 2, 12), ind=4)
+
+    def test_invalid_shape(self):
+        self.check_shape((2, 3, 4), ind=1)
+        self.check_shape((1, 2, 3, 4), ind=3)
+
+    def test_invalid_index(self):
+        self.check_ind((12, 3, 4), ind=-1)
+        self.check_ind((18, 3, 3, 2), ind=0)
diff --git a/tests/cupy_tests/logic_tests/__init__.py b/tests/cupy_tests/logic_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/logic_tests/test_comparison.py b/tests/cupy_tests/logic_tests/test_comparison.py
new file mode 100644
index 0000000..924eb4f
--- /dev/null
+++ b/tests/cupy_tests/logic_tests/test_comparison.py
@@ -0,0 +1,291 @@
+import operator
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+class TestComparison(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_binary(self, name, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a, b)
+
+    def test_greater(self):
+        self.check_binary('greater')
+
+    def test_greater_equal(self):
+        self.check_binary('greater_equal')
+
+    def test_less(self):
+        self.check_binary('less')
+
+    def test_less_equal(self):
+        self.check_binary('less_equal')
+
+    def test_not_equal(self):
+        self.check_binary('not_equal')
+
+    def test_equal(self):
+        self.check_binary('equal')
+
+
+class TestComparisonOperator(unittest.TestCase):
+
+    operators = [
+        operator.lt, operator.le,
+        operator.eq, operator.ne,
+        operator.gt, operator.ge,
+    ]
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_binary_npscalar_array(self, xp, dtype):
+        a = numpy.int16(3)
+        b = testing.shaped_arange((2, 3), xp, dtype)
+        return [op(a, b) for op in self.operators]
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_binary_pyscalar_array(self, xp, dtype):
+        a = 3.0
+        b = testing.shaped_arange((2, 3), xp, dtype)
+        return [op(a, b) for op in self.operators]
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_binary_array_npscalar(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = numpy.float32(3.0)
+        return [op(a, b) for op in self.operators]
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_binary_array_pyscalar(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = 3
+        return [op(a, b) for op in self.operators]
+
+
+class TestArrayEqual(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equal_not_equal(self, xp, dtype):
+        a = xp.array([1, 2, 3, 4], dtype=dtype)
+        b = xp.array([1, 2, 4, 5], dtype=dtype)
+        return xp.array_equal(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equal_is_equal(self, xp, dtype):
+        a = xp.array([1, 2, 3, 4], dtype=dtype)
+        b = xp.array([1, 2, 3, 4], dtype=dtype)
+        return xp.array_equal(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equal_diff_length(self, xp, dtype):
+        a = xp.array([1, 2, 3, 4], dtype=dtype)
+        b = xp.array([1, 2, 3], dtype=dtype)
+        return xp.array_equal(a, b)
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equal_infinite_equal_nan(self, xp, dtype):
+        nan = float('nan')
+        inf = float('inf')
+        ninf = float('-inf')
+        a = xp.array([0, nan, inf, ninf], dtype=dtype)
+        b = xp.array([0, nan, inf, ninf], dtype=dtype)
+        return xp.array_equal(a, b, equal_nan=True)
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equal_complex_equal_nan(self, xp, dtype):
+        a = xp.array([1+2j], dtype=dtype)
+        b = a.copy()
+        b.imag = xp.nan
+        a.real = xp.nan
+        return xp.array_equal(a, b, equal_nan=True)
+
+    @testing.numpy_cupy_equal()
+    def test_array_equal_diff_dtypes_not_equal(self, xp):
+        a = xp.array([0.9e-5, 1.1e-5, 100.5, 10.5])
+        b = xp.array([0, 0, 1000, 1000])
+        return xp.array_equal(a, b)
+
+    @testing.numpy_cupy_equal()
+    def test_array_equal_diff_dtypes_is_equal(self, xp):
+        a = xp.array([0.0, 1.0, 100.0, 10.0])
+        b = xp.array([0, 1, 100, 10])
+        return xp.array_equal(a, b)
+
+    @testing.numpy_cupy_equal()
+    def test_array_equal_broadcast_not_allowed(self, xp):
+        a = xp.array([1, 1, 1, 1])
+        b = xp.array([1])
+        return xp.array_equal(a, b)
+
+
+class TestArrayEquiv(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equiv_equal_not_equal(self, xp, dtype):
+        a = xp.array([0, 4, 1, 1], dtype=dtype)
+        b = xp.array([0, 4, 2, 3], dtype=dtype)
+        return xp.array_equiv(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equiv_equal_is_equal(self, xp, dtype):
+        a = xp.array([0, 4, 1, 1], dtype=dtype)
+        b = xp.array([0, 4, 1, 1], dtype=dtype)
+        return xp.array_equiv(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equiv_diff_length(self, xp, dtype):
+        a = xp.array([0, 4, 0, 5], dtype=dtype)
+        b = xp.array([0, 4, 0], dtype=dtype)
+        return xp.array_equiv(a, b)
+
+    @testing.numpy_cupy_equal()
+    def test_array_equiv_diff_dtypes_not_equal(self, xp):
+        a = xp.array([0.9e-5, 1.1e-5, 100.5, 10.5])
+        b = xp.array([0, 0, 1000, 1000])
+        return xp.array_equiv(a, b)
+
+    @testing.numpy_cupy_equal()
+    def test_array_equiv_diff_dtypes_is_equal(self, xp):
+        a = xp.array([0.0, 1.0, 100.0, 10.0])
+        b = xp.array([0, 1, 100, 10])
+        return xp.array_equiv(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equiv_broadcast1(self, xp, dtype):
+        a = xp.array([0, 4], dtype)
+        b = xp.array([[0, 4], [0, 4]], dtype)
+        return xp.array_equiv(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equiv_broadcast2(self, xp, dtype):
+        a = xp.array([0, 4], dtype=dtype)
+        b = xp.array([[0, 4], [0, 5]], dtype=dtype)
+        return xp.array_equiv(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_array_equiv_non_broadcast(self, xp, dtype):
+        a = xp.array([0, 4], dtype=dtype)
+        b = xp.array([[0, 4, 0, 4], [0, 4, 0, 4]], dtype=dtype)
+        return xp.array_equiv(a, b)
+
+
+class TestAllclose(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_allclose_finite(self, xp, dtype):
+        a = xp.array([0.9e-5, 1.1e-5, 1000 + 1e-4, 1000 - 1e-4]).astype(dtype)
+        b = xp.array([0, 0, 1000, 1000]).astype(dtype)
+        return xp.allclose(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_allclose_min_int(self, xp, dtype):
+        a = xp.array([0]).astype(dtype)
+        b = xp.array([numpy.iinfo('i').min]).astype(dtype)
+        return xp.allclose(a, b)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_allclose_infinite(self, xp, dtype):
+        nan = float('nan')
+        inf = float('inf')
+        ninf = float('-inf')
+        a = xp.array([0, nan, nan, 0, inf, ninf]).astype(dtype)
+        b = xp.array([0, nan, 0, nan, inf, ninf]).astype(dtype)
+        return xp.allclose(a, b)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_allclose_infinite_equal_nan(self, xp, dtype):
+        nan = float('nan')
+        inf = float('inf')
+        ninf = float('-inf')
+        a = xp.array([0, nan, inf, ninf]).astype(dtype)
+        b = xp.array([0, nan, inf, ninf]).astype(dtype)
+        return xp.allclose(a, b, equal_nan=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_allclose_array_scalar(self, xp, dtype):
+        a = xp.array([0.9e-5, 1.1e-5]).astype(dtype)
+        b = xp.dtype(xp.dtype).type(0)
+        return xp.allclose(a, b)
+
+
+class TestIsclose(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_is_close_finite(self, xp, dtype):
+        # In numpy<1.10 this test fails when dtype is bool
+        a = xp.array([0.9e-5, 1.1e-5, 1000 + 1e-4, 1000 - 1e-4]).astype(dtype)
+        b = xp.array([0, 0, 1000, 1000]).astype(dtype)
+        return xp.isclose(a, b)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_is_close_min_int(self, xp, dtype):
+        # In numpy<1.10 this test fails when dtype is bool
+        a = xp.array([0]).astype(dtype)
+        b = xp.array([numpy.iinfo('i').min]).astype(dtype)
+        return xp.isclose(a, b)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_is_close_infinite(self, xp, dtype):
+        nan = float('nan')
+        inf = float('inf')
+        ninf = float('-inf')
+        a = xp.array([0, nan, nan, 0, inf, ninf]).astype(dtype)
+        b = xp.array([0, nan, 0, nan, inf, ninf]).astype(dtype)
+        return xp.isclose(a, b)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_is_close_infinite_equal_nan(self, xp, dtype):
+        nan = float('nan')
+        inf = float('inf')
+        ninf = float('-inf')
+        a = xp.array([0, nan, inf, ninf]).astype(dtype)
+        b = xp.array([0, nan, inf, ninf]).astype(dtype)
+        return xp.isclose(a, b, equal_nan=True)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_is_close_array_scalar(self, xp, dtype):
+        a = xp.array([0.9e-5, 1.1e-5]).astype(dtype)
+        b = xp.dtype(xp.dtype).type(0)
+        return xp.isclose(a, b)
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_is_close_scalar_scalar(self, dtype):
+        # cupy.isclose always returns ndarray
+        a = cupy.dtype(cupy.dtype).type(0)
+        b = cupy.dtype(cupy.dtype).type(0)
+        cond = cupy.isclose(a, b)
+        assert cond.shape == ()
+        assert bool(cond)
diff --git a/tests/cupy_tests/logic_tests/test_content.py b/tests/cupy_tests/logic_tests/test_content.py
new file mode 100644
index 0000000..549c743
--- /dev/null
+++ b/tests/cupy_tests/logic_tests/test_content.py
@@ -0,0 +1,46 @@
+import unittest
+
+import numpy
+
+from cupy import testing
+
+
+class TestContent(unittest.TestCase):
+
+    @testing.for_dtypes('efFdD')
+    @testing.numpy_cupy_array_equal()
+    def check_unary_inf(self, name, xp, dtype):
+        a = xp.array([-3, numpy.inf, -1, -numpy.inf, 0, 1, 2],
+                     dtype=dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_dtypes('efFdD')
+    @testing.numpy_cupy_array_equal()
+    def check_unary_nan(self, name, xp, dtype):
+        a = xp.array(
+            [-3, numpy.NAN, -1, numpy.NAN, 0, numpy.NAN, numpy.inf],
+            dtype=dtype)
+        return getattr(xp, name)(a)
+
+    def test_isfinite(self):
+        self.check_unary_inf('isfinite')
+
+    def test_isinf(self):
+        self.check_unary_inf('isinf')
+
+    def test_isnan(self):
+        self.check_unary_nan('isnan')
+
+
+class TestUfuncLike(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def check_unary(self, name, xp):
+        a = xp.array([-3, xp.inf, -1, -xp.inf, 0, 1, 2, xp.nan])
+        return getattr(xp, name)(a)
+
+    def test_isneginf(self):
+        self.check_unary('isneginf')
+
+    def test_isposinf(self):
+        self.check_unary('isposinf')
diff --git a/tests/cupy_tests/logic_tests/test_ops.py b/tests/cupy_tests/logic_tests/test_ops.py
new file mode 100644
index 0000000..40fff40
--- /dev/null
+++ b/tests/cupy_tests/logic_tests/test_ops.py
@@ -0,0 +1,31 @@
+import unittest
+
+from cupy import testing
+
+
+class TestOps(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary(self, name, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_binary(self, name, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a, b)
+
+    def test_logical_and(self):
+        self.check_binary('logical_and')
+
+    def test_logical_or(self):
+        self.check_binary('logical_or')
+
+    def test_logical_xor(self):
+        self.check_binary('logical_xor')
+
+    def test_logical_not(self):
+        self.check_unary('logical_not')
diff --git a/tests/cupy_tests/logic_tests/test_truth.py b/tests/cupy_tests/logic_tests/test_truth.py
new file mode 100644
index 0000000..75be9ac
--- /dev/null
+++ b/tests/cupy_tests/logic_tests/test_truth.py
@@ -0,0 +1,305 @@
+import warnings
+
+import numpy
+
+from cupy import testing
+
+
+def _calc_out_shape(shape, axis, keepdims):
+    if axis is None:
+        axis = list(range(len(shape)))
+    elif isinstance(axis, int):
+        axis = [axis]
+    else:
+        axis = list(axis)
+
+    shape = numpy.array(shape)
+
+    if keepdims:
+        shape[axis] = 1
+    else:
+        shape[axis] = -1
+        shape = filter(lambda x: x != -1, shape)
+    return tuple(shape)
+
+
+@testing.parameterize(
+    *testing.product(
+        {'f': ['all', 'any'],
+         'x': [numpy.arange(24).reshape(2, 3, 4) - 10,
+               numpy.zeros((2, 3, 4)),
+               numpy.ones((2, 3, 4)),
+               numpy.zeros((0, 3, 4)),
+               numpy.ones((0, 3, 4))],
+         'axis': [None, (0, 1, 2), 0, 1, 2, (0, 1)],
+         'keepdims': [False, True]}))
+class TestAllAny:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_without_out(self, xp, dtype):
+        x = xp.asarray(self.x).astype(dtype)
+        return getattr(xp, self.f)(x, self.axis, None, self.keepdims)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_with_out(self, xp, dtype):
+        x = xp.asarray(self.x).astype(dtype)
+        out_shape = _calc_out_shape(x.shape, self.axis, self.keepdims)
+        out = xp.empty(out_shape, dtype=x.dtype)
+        getattr(xp, self.f)(x, self.axis, out, self.keepdims)
+        return out
+
+
+@testing.parameterize(
+    *testing.product(
+        {'f': ['all', 'any'],
+         'x': [numpy.array([[[numpy.nan]]]),
+               numpy.array([[[numpy.nan, 0]]]),
+               numpy.array([[[numpy.nan, 1]]]),
+               numpy.array([[[numpy.nan, 0, 1]]])],
+         'axis': [None, (0, 1, 2), 0, 1, 2, (0, 1)],
+         'keepdims': [False, True]}))
+class TestAllAnyWithNaN:
+
+    @testing.for_dtypes(
+        (numpy.float64, numpy.float32, numpy.float16, numpy.bool_))
+    @testing.numpy_cupy_array_equal()
+    def test_without_out(self, xp, dtype):
+        x = xp.asarray(self.x).astype(dtype)
+        return getattr(xp, self.f)(x, self.axis, None, self.keepdims)
+
+    @testing.for_dtypes(
+        (numpy.float64, numpy.float32, numpy.float16, numpy.bool_))
+    @testing.numpy_cupy_array_equal()
+    def test_with_out(self, xp, dtype):
+        x = xp.asarray(self.x).astype(dtype)
+        out_shape = _calc_out_shape(x.shape, self.axis, self.keepdims)
+        out = xp.empty(out_shape, dtype=x.dtype)
+        getattr(xp, self.f)(x, self.axis, out, self.keepdims)
+        return out
+
+
+class TestAllAnyAlias:
+    @testing.numpy_cupy_array_equal()
+    def test_alltrue(self, xp):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            return xp.alltrue(xp.array([1, 2, 3]))
+
+    @testing.numpy_cupy_array_equal()
+    def test_sometrue(self, xp):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            return xp.sometrue(xp.array([0]))
+
+
+@testing.parameterize(
+    *testing.product(
+        {'f': ['in1d', 'isin'],
+         'shape_x': [
+             (0, ),
+             (3, ),
+             (2, 3),
+             (2, 1, 3),
+             (2, 0, 1),
+             (2, 0, 1, 1)
+        ],
+            'shape_y': [
+             (0, ),
+             (3, ),
+             (2, 3),
+             (2, 1, 3),
+             (2, 0, 1),
+             (2, 0, 1, 1)
+        ],
+            'assume_unique': [False, True],
+            'invert': [False, True]}))
+class TestIn1DIsIn:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test(self, xp, dtype):
+        x = testing.shaped_arange(self.shape_x, xp, dtype)
+        y = testing.shaped_arange(self.shape_y, xp, dtype)
+        if xp is numpy and self.f == 'isin':
+            return xp.in1d(x, y, self.assume_unique, self.invert)\
+                .reshape(x.shape)
+        return getattr(xp, self.f)(x, y, self.assume_unique, self.invert)
+
+
+class TestSetdiff1d:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setdiff1d_same_arrays(self, xp, dtype):
+        x = xp.array([1, 2, 3, 4, 5], dtype=dtype)
+        y = xp.array([1, 2, 3, 4, 5], dtype=dtype)
+        return xp.setdiff1d(x, y, assume_unique=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setdiff1d_diff_size_arr_inputs(self, xp, dtype):
+        x = xp.array([3, 4, 9, 1, 5, 4], dtype=dtype)
+        y = xp.array([8, 7, 3, 9, 0], dtype=dtype)
+        return xp.setdiff1d(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setdiff1d_diff_elements(self, xp, dtype):
+        x = xp.array([3, 4, 9, 1, 5, 4], dtype=dtype)
+        y = xp.array([8, 7, 3, 9, 0], dtype=dtype)
+        return xp.setdiff1d(x, y, assume_unique=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setdiff1d_with_2d(self, xp, dtype):
+        x = testing.shaped_random((2, 3), xp, dtype=dtype)
+        y = testing.shaped_random((3, 5), xp, dtype=dtype)
+        return xp.setdiff1d(x, y, assume_unique=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setdiff1d_with_duplicate_elements(self, xp, dtype):
+        x = xp.array([1, 2, 3, 2, 2, 6], dtype=dtype)
+        y = xp.array([3, 4, 2, 1, 1, 9], dtype=dtype)
+        return xp.setdiff1d(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setdiff1d_empty_arr(self, xp, dtype):
+        x = xp.array([], dtype=dtype)
+        y = xp.array([], dtype=dtype)
+        return xp.setdiff1d(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setdiff1d_more_dim(self, xp, dtype):
+        x = testing.shaped_arange((2, 3, 4, 8), xp, dtype=dtype)
+        y = testing.shaped_arange((5, 4, 2), xp, dtype=dtype)
+        return xp.setdiff1d(x, y, assume_unique=True)
+
+    @testing.numpy_cupy_array_equal()
+    def test_setdiff1d_bool_val(self, xp):
+        x = xp.array([True, False, True])
+        y = xp.array([False])
+        return xp.setdiff1d(x, y)
+
+
+class TestSetxor1d:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setxor1d_same_arrays(self, xp, dtype):
+        x = xp.array([1, 2, 3, 4, 5], dtype=dtype)
+        y = xp.array([1, 2, 3, 4, 5], dtype=dtype)
+        return xp.setxor1d(x, y, assume_unique=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setxor1d_diff_size_arr_inputs(self, xp, dtype):
+        x = xp.array([3, 4, 9, 1, 5, 4], dtype=dtype)
+        y = xp.array([8, 7, 3, 9, 0], dtype=dtype)
+        return xp.setxor1d(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setxor1d_diff_elements(self, xp, dtype):
+        x = xp.array([3, 4, 9, 1, 5, 4], dtype=dtype)
+        y = xp.array([8, 7, 3, 9, 0], dtype=dtype)
+        return xp.setxor1d(x, y, assume_unique=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setxor1d_with_2d(self, xp, dtype):
+        x = testing.shaped_random((2, 3), xp, dtype=dtype)
+        y = testing.shaped_random((3, 5), xp, dtype=dtype)
+        return xp.setxor1d(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setxor1d_with_duplicate_elements(self, xp, dtype):
+        x = xp.array([1, 2, 3, 2, 2, 6], dtype=dtype)
+        y = xp.array([3, 4, 2, 1, 1, 9], dtype=dtype)
+        return xp.setxor1d(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setxor1d_empty_arr(self, xp, dtype):
+        x = xp.array([], dtype=dtype)
+        y = xp.array([], dtype=dtype)
+        return xp.setxor1d(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_setxor1d_more_dim(self, xp, dtype):
+        x = testing.shaped_arange((2, 3, 4, 8), xp, dtype=dtype)
+        y = testing.shaped_arange((5, 4, 2), xp, dtype=dtype)
+        return xp.setxor1d(x, y)
+
+    @testing.numpy_cupy_array_equal()
+    def test_setxor1d_bool_val(self, xp):
+        x = xp.array([True, False, True])
+        y = xp.array([False])
+        return xp.setxor1d(x, y)
+
+
+class TestIntersect1d:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_one_dim_with_unique_values(self, xp, dtype):
+        a = xp.array([1, 2, 3, 4, 5], dtype=dtype)
+        b = xp.array([1, 2, 3, 4, 5], dtype=dtype)
+        return xp.intersect1d(a, b, assume_unique=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_with_random_val(self, xp, dtype):
+        a = xp.array([3, 4, 9, 1, 5, 4], dtype=dtype)
+        b = xp.array([8, 7, 3, 9, 0], dtype=dtype)
+        return xp.intersect1d(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_more_dim(self, xp, dtype):
+        a = testing.shaped_random((3, 4), xp, dtype=dtype)
+        b = testing.shaped_random((5, 2), xp, dtype=dtype)
+        return xp.intersect1d(a, b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_return_indices(self, xp, dtype):
+        a = xp.array([2, 3, 4, 1, 9, 4], dtype=dtype)
+        b = xp.array([7, 5, 1, 2, 9, 3], dtype=dtype)
+        return xp.intersect1d(a, b, return_indices=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_multiple_instances(self, xp, dtype):
+        a = xp.array([2, 4, 5, 2, 1, 5], dtype=dtype)
+        b = xp.array([4, 6, 2, 5, 7, 6], dtype=dtype)
+        return xp.intersect1d(a, b, return_indices=True)
+
+
+class TestUnion1d:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_union1d(self, xp, dtype):
+        x = xp.array([4, 1, 1, 1, 9, 9, 9], dtype=dtype)
+        y = xp.array([4, 0, 5, 2, 0, 0, 5], dtype=dtype)
+        return xp.union1d(x, y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_union1d_2(self, xp, dtype):
+        x = testing.shaped_arange((5, 2), xp, dtype=dtype)
+        y = testing.shaped_arange((2, 3, 4), xp, dtype=dtype)
+        return xp.union1d(x, y)
+
+    @testing.numpy_cupy_array_equal()
+    def test_union1d_3(self, xp):
+        x = xp.zeros((2, 2), dtype=xp.complex_)
+        y = xp.array([[1+1j, 2+3j], [4+1j, 0+7j]])
+        return xp.union1d(x, y)
diff --git a/tests/cupy_tests/logic_tests/test_type_test.py b/tests/cupy_tests/logic_tests/test_type_test.py
new file mode 100644
index 0000000..95bf359
--- /dev/null
+++ b/tests/cupy_tests/logic_tests/test_type_test.py
@@ -0,0 +1,106 @@
+import unittest
+
+import numpy
+
+from cupy import testing
+
+
+class TestIsScalar(testing.NumpyAliasBasicTestBase):
+
+    func = 'isscalar'
+
+    @testing.with_requires('numpy>=1.18')
+    def test_argspec(self):
+        super().test_argspec()
+
+
+@testing.parameterize(
+    *testing.product({
+        'value': [
+            0, 0.0, True,
+            numpy.int32(1), numpy.array([1, 2], numpy.int32),
+            numpy.complex128(1), numpy.complex128(1j),
+            numpy.complex128(1 + 1j),
+            None, 'abc', '', int, numpy.int32]}))
+class TestIsScalarValues(testing.NumpyAliasValuesTestBase):
+
+    func = 'isscalar'
+
+    def setUp(self):
+        self.args = (self.value,)
+
+
+class TestIsScalarValues2(testing.NumpyAliasValuesTestBase):
+
+    func = 'isscalar'
+
+    def setUp(self):
+        value = object()
+        self.args = (value,)
+
+
+@testing.parameterize(
+    *testing.product({
+        'value': [
+            # C and F
+            numpy.ones(24, order='C'),
+            # C and not F
+            numpy.ones((4, 6), order='C'),
+            # not C and F
+            numpy.ones((4, 6), order='F'),
+            # not C and not F
+            numpy.ones((4, 6), order='C')[1:3][1:3],
+        ]
+    })
+)
+class TestIsFortran(unittest.TestCase):
+
+    @testing.numpy_cupy_equal()
+    def test(self, xp):
+        return xp.isfortran(xp.asarray(self.value))
+
+
+@testing.parameterize(
+    {'func': 'iscomplex'},
+    {'func': 'isreal'},
+)
+class TestTypeTestingFunctions(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test(self, xp, dtype):
+        return getattr(xp, self.func)(xp.ones(5, dtype=dtype))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_scalar(self, xp, dtype):
+        return getattr(xp, self.func)(dtype(3))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_list(self, xp, dtype):
+        return getattr(xp, self.func)(
+            testing.shaped_arange((2, 3), xp, dtype).tolist())
+
+
+@testing.parameterize(
+    {'func': 'iscomplexobj'},
+    {'func': 'isrealobj'},
+)
+class TestTypeTestingObjFunctions(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test(self, xp, dtype):
+        return getattr(xp, self.func)(xp.ones(5, dtype=dtype))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_scalar(self, xp, dtype):
+        return getattr(xp, self.func)(dtype(3))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_list(self, xp, dtype):
+        return getattr(xp, self.func)(
+            testing.shaped_arange((2, 3), xp, dtype).tolist())
diff --git a/tests/cupy_tests/manipulation_tests/__init__.py b/tests/cupy_tests/manipulation_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/manipulation_tests/test_add_remove.py b/tests/cupy_tests/manipulation_tests/test_add_remove.py
new file mode 100644
index 0000000..74bb7b7
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_add_remove.py
@@ -0,0 +1,211 @@
+import unittest
+
+import pytest
+
+import numpy
+import cupy
+from cupy import testing
+
+
+class TestAppend(unittest.TestCase):
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test(self, xp, dtype1, dtype2):
+        a = testing.shaped_random((3, 4, 5), xp, dtype1)
+        b = testing.shaped_random((6, 7), xp, dtype2)
+        return xp.append(a, b)
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_scalar_lhs(self, xp, dtype1, dtype2):
+        scalar = xp.dtype(dtype1).type(10).item()
+        return xp.append(scalar, xp.arange(20, dtype=dtype2))
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_scalar_rhs(self, xp, dtype1, dtype2):
+        scalar = xp.dtype(dtype2).type(10).item()
+        return xp.append(xp.arange(20, dtype=dtype1), scalar)
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_numpy_scalar_lhs(self, xp, dtype1, dtype2):
+        scalar = xp.dtype(dtype1).type(10)
+        return xp.append(scalar, xp.arange(20, dtype=dtype2))
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_numpy_scalar_rhs(self, xp, dtype1, dtype2):
+        scalar = xp.dtype(dtype2).type(10)
+        return xp.append(xp.arange(20, dtype=dtype1), scalar)
+
+    @testing.numpy_cupy_array_equal()
+    def test_scalar_both(self, xp):
+        return xp.append(10, 10)
+
+    @testing.numpy_cupy_array_equal()
+    def test_axis(self, xp):
+        a = testing.shaped_random((3, 4, 5), xp, xp.float32)
+        b = testing.shaped_random((3, 10, 5), xp, xp.float32)
+        return xp.append(a, b, axis=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_zerodim(self, xp):
+        return xp.append(xp.array(0), xp.arange(10))
+
+    @testing.numpy_cupy_array_equal()
+    def test_empty(self, xp):
+        return xp.append(xp.array([]), xp.arange(10))
+
+
+class TestResize(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test(self, xp):
+        return xp.resize(xp.arange(10), (10, 10))
+
+    @testing.numpy_cupy_array_equal()
+    def test_remainder(self, xp):
+        return xp.resize(xp.arange(8), (10, 10))
+
+    @testing.numpy_cupy_array_equal()
+    def test_shape_int(self, xp):
+        return xp.resize(xp.arange(10), 15)
+
+    @testing.numpy_cupy_array_equal()
+    def test_scalar(self, xp):
+        return xp.resize(2, (10, 10))
+
+    @testing.numpy_cupy_array_equal()
+    def test_scalar_shape_int(self, xp):
+        return xp.resize(2, 10)
+
+    @testing.numpy_cupy_array_equal()
+    def test_typed_scalar(self, xp):
+        return xp.resize(xp.float32(10.0), (10, 10))
+
+    @testing.numpy_cupy_array_equal()
+    def test_zerodim(self, xp):
+        return xp.resize(xp.array(0), (10, 10))
+
+    @testing.numpy_cupy_array_equal()
+    def test_empty(self, xp):
+        return xp.resize(xp.array([]), (10, 10))
+
+
+class TestUnique:
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_unique(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return xp.unique(a)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_unique_index(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return xp.unique(a, return_index=True)[1]
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_unique_inverse(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return xp.unique(a, return_inverse=True)[1]
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_unique_counts(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return xp.unique(a, return_counts=True)[1]
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_unique_return_all(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return xp.unique(
+            a, return_index=True, return_inverse=True, return_counts=True)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_unique_empty(self, xp, dtype):
+        a = xp.empty((0,), dtype)
+        return xp.unique(a)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_unique_empty_return_all(self, xp, dtype):
+        a = xp.empty((3, 0, 2), dtype)
+        return xp.unique(
+            a, return_index=True, return_inverse=True, return_counts=True)
+
+    @pytest.mark.parametrize('equal_nan', [True, False])
+    @pytest.mark.parametrize('dtype', 'efdFD')
+    @testing.numpy_cupy_array_equal()
+    @testing.with_requires('numpy>=1.23.1')
+    def test_unique_equal_nan(self, xp, dtype, equal_nan):
+        if xp.dtype(dtype).kind == 'c':
+            # Nan and Nan+Nan*1j are collapsed when equal_nan=True
+            a = xp.array([
+                complex(xp.nan, 3), 2, complex(7, xp.nan), xp.nan,
+                complex(xp.nan, xp.nan), 2, xp.nan, 1
+            ], dtype=dtype)
+        else:
+            a = xp.array([2, xp.nan, 2, xp.nan, 1], dtype=dtype)
+        return xp.unique(a, equal_nan=equal_nan)
+
+
+@testing.parameterize(*testing.product({
+    'trim': ['fb', 'f', 'b']
+}))
+class TestTrim_zeros(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trim_non_zeros(self, xp, dtype):
+        a = xp.array([-1, 2, -3, 7]).astype(dtype)
+        return xp.trim_zeros(a, trim=self.trim)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trim_trimmed(self, xp, dtype):
+        a = xp.array([1, 0, 2, 3, 0, 5], dtype=dtype)
+        return xp.trim_zeros(a, trim=self.trim)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trim_all_zeros(self, xp, dtype):
+        a = xp.zeros(shape=(1000,), dtype=dtype)
+        return xp.trim_zeros(a, trim=self.trim)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trim_front_zeros(self, xp, dtype):
+        a = xp.array([0, 0, 4, 1, 0, 2, 3, 0, 5], dtype=dtype)
+        return xp.trim_zeros(a, trim=self.trim)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trim_back_zeros(self, xp, dtype):
+        a = xp.array([1, 0, 2, 3, 0, 5, 0, 0, 0], dtype=dtype)
+        return xp.trim_zeros(a, trim=self.trim)
+
+    @testing.for_all_dtypes()
+    def test_trim_zero_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.trim_zeros(a, trim=self.trim)
+
+    @testing.for_all_dtypes()
+    def test_trim_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3), xp, dtype=dtype)
+            with pytest.raises(ValueError):
+                xp.trim_zeros(a, trim=self.trim)
diff --git a/tests/cupy_tests/manipulation_tests/test_basic.py b/tests/cupy_tests/manipulation_tests/test_basic.py
new file mode 100644
index 0000000..fbff7ed
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_basic.py
@@ -0,0 +1,259 @@
+import itertools
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy import cuda
+from cupy import testing
+
+
+class TestBasic:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.empty((2, 3, 4), dtype=dtype)
+        xp.copyto(b, a)
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_different_contiguity(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 2), xp, dtype).T
+        b = xp.empty((2, 3, 2), dtype=dtype)
+        xp.copyto(b, a)
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_dtype(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype='?')
+        b = xp.empty((2, 3, 4), dtype=dtype)
+        xp.copyto(b, a)
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_broadcast(self, xp, dtype):
+        a = testing.shaped_arange((3, 1), xp, dtype)
+        b = xp.empty((2, 3, 4), dtype=dtype)
+        xp.copyto(b, a)
+        return b
+
+    @pytest.mark.parametrize(('dst_shape', 'src_shape'), [
+        ((), (2,)),
+        ((2, 0, 5, 4), (2, 0, 3, 4)),
+        ((6,), (2, 3)),
+        ((2, 3), (6,)),
+    ])
+    def test_copyto_raises_shape(self, dst_shape, src_shape):
+        for xp in (numpy, cupy):
+            dst = xp.zeros(dst_shape, int)
+            src = xp.zeros(src_shape, int)
+            with pytest.raises(ValueError):
+                xp.copyto(dst, src)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_squeeze(self, xp, dtype):
+        a = testing.shaped_arange((1, 1, 3, 4), xp, dtype)
+        b = xp.empty((3, 4), dtype=dtype)
+        xp.copyto(b, a)
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_squeeze_different_contiguity(self, xp, dtype):
+        a = testing.shaped_arange((1, 1, 3, 4), xp, dtype)
+        b = xp.empty((4, 3), dtype=dtype).T
+        xp.copyto(b, a)
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_squeeze_broadcast(self, xp, dtype):
+        a = testing.shaped_arange((1, 2, 1, 4), xp, dtype)
+        b = xp.empty((2, 3, 4), dtype=dtype)
+        xp.copyto(b, a)
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_where(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3, 4), xp, dtype)
+        c = testing.shaped_arange((2, 3, 4), xp, '?')
+        xp.copyto(a, b, where=c)
+        return a
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_where_squeeze_broadcast(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((1, 2, 1, 4), xp, dtype)
+        c = testing.shaped_arange((3, 4), xp, '?')
+        xp.copyto(a, b, where=c)
+        return a
+
+    @pytest.mark.parametrize('shape', [(2, 3, 4), (0,)])
+    @testing.for_all_dtypes(no_bool=True)
+    def test_copyto_where_raises(self, dtype, shape):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(shape, xp, 'i')
+            b = testing.shaped_reverse_arange(shape, xp, 'i')
+            c = testing.shaped_arange(shape, xp, dtype)
+            with pytest.raises(TypeError):
+                xp.copyto(a, b, where=c)
+
+    def _check_copyto_where_multigpu_raises(self, dtype, ngpus):
+        def get_numpy():
+            a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+            b = testing.shaped_reverse_arange((2, 3, 4), numpy, dtype)
+            c = testing.shaped_arange((2, 3, 4), numpy, '?')
+            numpy.copyto(a, b, where=c)
+            return a
+
+        for dev1, dev2, dev3, dev4 in itertools.product(*[range(ngpus)] * 4):
+            if dev1 == dev2 == dev3 == dev4:
+                continue
+            if not dev1 <= dev2 <= dev3 <= dev4:
+                continue
+
+            with cuda.Device(dev1):
+                a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+            with cuda.Device(dev2):
+                b = testing.shaped_reverse_arange((2, 3, 4), cupy, dtype)
+            with cuda.Device(dev3):
+                c = testing.shaped_arange((2, 3, 4), cupy, '?')
+            with cuda.Device(dev4):
+                if all([(peer == dev4) or
+                        (cuda.runtime.deviceCanAccessPeer(dev4, peer) == 1)
+                        for peer in (dev1, dev2, dev3)]):
+                    with pytest.warns(cupy._util.PerformanceWarning):
+                        cupy.copyto(a, b, where=c)
+                else:
+                    with pytest.raises(
+                            ValueError,
+                            match='Peer access is unavailable'):
+                        cupy.copyto(a, b, where=c)
+
+    @testing.multi_gpu(2)
+    @testing.for_all_dtypes()
+    def test_copyto_where_multigpu_raises(self, dtype):
+        self._check_copyto_where_multigpu_raises(dtype, 2)
+
+    @testing.multi_gpu(4)
+    @testing.for_all_dtypes()
+    def test_copyto_where_multigpu_raises_4(self, dtype):
+        self._check_copyto_where_multigpu_raises(dtype, 4)
+
+    @testing.multi_gpu(6)
+    @testing.for_all_dtypes()
+    def test_copyto_where_multigpu_raises_6(self, dtype):
+        self._check_copyto_where_multigpu_raises(dtype, 6)
+
+    @testing.multi_gpu(2)
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copyto_multigpu(self, xp, dtype):
+        with cuda.Device(0):
+            a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        with cuda.Device(1):
+            b = xp.empty((2, 3, 4), dtype=dtype)
+        xp.copyto(b, a)
+        return b
+
+    @testing.multi_gpu(2)
+    @testing.for_all_dtypes()
+    def test_copyto_multigpu_noncontinguous(self, dtype):
+        with cuda.Device(0):
+            src = testing.shaped_arange((2, 3, 4), cupy, dtype)
+            src = src.swapaxes(0, 1)
+        with cuda.Device(1):
+            dst = cupy.empty_like(src)
+            cupy.copyto(dst, src)
+
+        expected = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        expected = expected.swapaxes(0, 1)
+
+        testing.assert_array_equal(expected, src.get())
+        testing.assert_array_equal(expected, dst.get())
+
+
+@testing.parameterize(
+    *testing.product(
+        {'src': [float(3.2), int(0), int(4), int(-4), True, False, 1 + 1j],
+         'dst_shape': [(), (0,), (1,), (1, 1), (2, 2)]}))
+class TestCopytoFromScalar:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_copyto(self, xp, dtype):
+        dst = xp.ones(self.dst_shape, dtype=dtype)
+        xp.copyto(dst, self.src)
+        return dst
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_copyto_where(self, xp, dtype):
+        dst = xp.ones(self.dst_shape, dtype=dtype)
+        mask = (testing.shaped_arange(
+            self.dst_shape, xp, dtype) % 2).astype(xp.bool_)
+        xp.copyto(dst, self.src, where=mask)
+        return dst
+
+
+@pytest.mark.parametrize(
+    'casting', ['no', 'equiv', 'safe', 'same_kind', 'unsafe'])
+class TestCopytoFromNumpyScalar:
+
+    @testing.for_all_dtypes_combination(('dtype1', 'dtype2'))
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_copyto(self, xp, dtype1, dtype2, casting):
+        dst = xp.zeros((2, 3, 4), dtype=dtype1)
+        src = numpy.array(1, dtype=dtype2)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', numpy.ComplexWarning)
+            xp.copyto(dst, src, casting)
+        return dst
+
+    @testing.for_all_dtypes()
+    @pytest.mark.parametrize('make_src',
+                             [lambda dtype: numpy.array([1], dtype=dtype),
+                              lambda dtype: dtype(1)])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_copyto2(self, xp, make_src, dtype, casting):
+        dst = xp.zeros((2, 3, 4), dtype=dtype)
+        src = make_src(dtype)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', numpy.ComplexWarning)
+            xp.copyto(dst, src, casting)
+        return dst
+
+    @testing.for_all_dtypes_combination(('dtype1', 'dtype2'))
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_copyto_where(self, xp, dtype1, dtype2, casting):
+        shape = (2, 3, 4)
+        dst = xp.ones(shape, dtype=dtype1)
+        src = numpy.array(1, dtype=dtype2)
+        mask = (testing.shaped_arange(shape, xp, dtype1) % 2).astype(xp.bool_)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', numpy.ComplexWarning)
+            xp.copyto(dst, src, casting=casting, where=mask)
+        return dst
+
+
+@pytest.mark.parametrize('shape', [(3, 2), (0,)])
+@pytest.mark.parametrize('where', [
+    float(3.2), int(0), int(4), int(-4), True, False, 1 + 1j
+])
+@testing.for_all_dtypes()
+@testing.numpy_cupy_allclose()
+def test_copyto_scalarwhere(xp, dtype, where, shape):
+    dst = xp.zeros(shape, dtype=dtype)
+    src = xp.ones(shape, dtype=dtype)
+    xp.copyto(dst, src, where=where)
+    return dst
diff --git a/tests/cupy_tests/manipulation_tests/test_dims.py b/tests/cupy_tests/manipulation_tests/test_dims.py
new file mode 100644
index 0000000..0bdd098
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_dims.py
@@ -0,0 +1,344 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupy import testing
+
+
+class TestDims(unittest.TestCase):
+
+    def check_atleast(self, func, xp):
+        a = testing.shaped_arange((), xp)
+        b = testing.shaped_arange((2,), xp)
+        c = testing.shaped_arange((2, 2), xp)
+        d = testing.shaped_arange((4, 3, 2), xp)
+        e = 1
+        f = numpy.float32(1)
+        return func(a, b, c, d, e, f)
+
+    @testing.numpy_cupy_array_equal()
+    def test_atleast_1d1(self, xp):
+        return self.check_atleast(xp.atleast_1d, xp)
+
+    @testing.numpy_cupy_array_equal()
+    def test_atleast_1d2(self, xp):
+        a = testing.shaped_arange((1, 3, 2), xp)
+        return xp.atleast_1d(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_atleast_2d1(self, xp):
+        return self.check_atleast(xp.atleast_2d, xp)
+
+    @testing.numpy_cupy_array_equal()
+    def test_atleast_2d2(self, xp):
+        a = testing.shaped_arange((1, 3, 2), xp)
+        return xp.atleast_2d(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_atleast_3d1(self, xp):
+        return self.check_atleast(xp.atleast_3d, xp)
+
+    @testing.numpy_cupy_array_equal()
+    def test_atleast_3d2(self, xp):
+        a = testing.shaped_arange((1, 3, 2), xp)
+        return xp.atleast_3d(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_broadcast_to(self, xp, dtype):
+        # Note that broadcast_to is only supported on numpy>=1.10
+        a = testing.shaped_arange((3, 1, 4), xp, dtype)
+        b = xp.broadcast_to(a, (2, 3, 3, 4))
+        return b
+
+    @testing.numpy_cupy_array_equal()
+    def test_broadcast_to_int(self, xp):
+        # Broadcast 0-dim array to 1-dim.
+        a = xp.array(10, dtype=xp.float32)
+        b = xp.broadcast_to(a, 10)
+        return b
+
+    @testing.for_all_dtypes()
+    def test_broadcast_to_fail(self, dtype):
+        for xp in (numpy, cupy):
+            # Note that broadcast_to is only supported on numpy>=1.10
+            a = testing.shaped_arange((3, 1, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.broadcast_to(a, (1, 3, 4))
+
+    @testing.for_all_dtypes()
+    def test_broadcast_to_short_shape(self, dtype):
+        for xp in (numpy, cupy):
+            # Note that broadcast_to is only supported on numpy>=1.10
+            a = testing.shaped_arange((1, 3, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.broadcast_to(a, (3, 4))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_broadcast_to_numpy19(self, xp, dtype):
+        # Note that broadcast_to is only supported on numpy>=1.10
+        a = testing.shaped_arange((3, 1, 4), xp, dtype)
+        if xp is cupy:
+            b = xp.broadcast_to(a, (2, 3, 3, 4))
+        else:
+            dummy = xp.empty((2, 3, 3, 4))
+            b, _ = xp.broadcast_arrays(a, dummy)
+        return b
+
+    @testing.for_all_dtypes()
+    def test_broadcast_to_fail_numpy19(self, dtype):
+        for xp in (numpy, cupy):
+            # Note that broadcast_to is only supported on numpy>=1.10
+            a = testing.shaped_arange((3, 1, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.broadcast_to(a, (1, 3, 4))
+
+    @testing.for_all_dtypes()
+    def test_broadcast_to_short_shape_numpy19(self, dtype):
+        for xp in (numpy, cupy):
+            # Note that broadcast_to is only supported on numpy>=1.10
+            a = testing.shaped_arange((1, 3, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.broadcast_to(a, (3, 4))
+
+    @testing.numpy_cupy_array_equal()
+    def test_expand_dims0(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.expand_dims(a, 0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_expand_dims1(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.expand_dims(a, 1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_expand_dims2(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.expand_dims(a, 2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_expand_dims_negative1(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.expand_dims(a, -2)
+
+    @testing.with_requires('numpy>=1.18')
+    def test_expand_dims_negative2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.expand_dims(a, -4)
+
+    @testing.with_requires('numpy>=1.18')
+    @testing.numpy_cupy_array_equal()
+    def test_expand_dims_tuple_axis(self, xp):
+        a = testing.shaped_arange((2, 2, 2), xp)
+        return [xp.expand_dims(a, axis) for axis in [
+            (0, 1, 2),
+            (0, -1, -2),
+            (0, 3, 5),
+            (0, -3, -5),
+            (),
+            (1,),
+        ]]
+
+    @testing.with_requires('numpy>=1.18')
+    def test_expand_dims_out_of_range(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 2, 2), xp)
+            for axis in [(1, -6), (1, 5)]:
+                with pytest.raises(numpy.AxisError):
+                    xp.expand_dims(a, axis)
+
+    @testing.with_requires('numpy>=1.18')
+    def test_expand_dims_repeated_axis(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 2, 2), xp)
+            with pytest.raises(ValueError):
+                xp.expand_dims(a, (1, 1))
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze1(self, xp):
+        a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+        return a.squeeze()
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze2(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.squeeze()
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze_int_axis1(self, xp):
+        a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+        return a.squeeze(axis=2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze_int_axis2(self, xp):
+        a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+        return a.squeeze(axis=-3)
+
+    def test_squeeze_int_axis_failure1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+            with pytest.raises(ValueError):
+                a.squeeze(axis=-9)
+
+    def test_squeeze_int_axis_failure2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+            with pytest.raises(numpy.AxisError):
+                a.squeeze(axis=-9)
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze_tuple_axis1(self, xp):
+        a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+        return a.squeeze(axis=(2, 4))
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze_tuple_axis2(self, xp):
+        a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+        return a.squeeze(axis=(-4, -3))
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze_tuple_axis3(self, xp):
+        a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+        return a.squeeze(axis=(4, 2))
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze_tuple_axis4(self, xp):
+        a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+        return a.squeeze(axis=())
+
+    def test_squeeze_tuple_axis_failure1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+            with pytest.raises(ValueError):
+                a.squeeze(axis=(-9,))
+
+    def test_squeeze_tuple_axis_failure2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+            with pytest.raises(ValueError):
+                a.squeeze(axis=(2, 2))
+
+    def test_squeeze_tuple_axis_failure3(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+            with pytest.raises(numpy.AxisError):
+                a.squeeze(axis=(-9,))
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze_scalar1(self, xp):
+        a = testing.shaped_arange((), xp)
+        return a.squeeze(axis=0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_squeeze_scalar2(self, xp):
+        a = testing.shaped_arange((), xp)
+        return a.squeeze(axis=-1)
+
+    def test_squeeze_scalar_failure1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), xp)
+            with pytest.raises(ValueError):
+                a.squeeze(axis=-2)
+
+    def test_squeeze_scalar_failure2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), xp)
+            with pytest.raises(ValueError):
+                a.squeeze(axis=1)
+
+    def test_squeeze_scalar_failure3(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), xp)
+            with pytest.raises(numpy.AxisError):
+                a.squeeze(axis=-2)
+
+    def test_squeeze_scalar_failure4(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), cupy)
+            with pytest.raises(numpy.AxisError):
+                a.squeeze(axis=1)
+
+    def test_squeeze_failure(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 1, 3, 4), xp)
+            with pytest.raises(ValueError):
+                a.squeeze(axis=2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_squeeze(self, xp):
+        a = testing.shaped_arange((1, 2, 1, 3, 1, 1, 4, 1), xp)
+        return xp.squeeze(a)
+
+
+@testing.parameterize(
+    {'shapes': [(), ()]},
+    {'shapes': [(0,), (0,)]},
+    {'shapes': [(1,), (1,)]},
+    {'shapes': [(2,), (2,)]},
+    {'shapes': [(0,), (1,)]},
+    {'shapes': [(2, 3), (1, 3)]},
+    {'shapes': [(2, 1, 3, 4), (3, 1, 4)]},
+    {'shapes': [(4, 3, 2, 3), (2, 3)]},
+    {'shapes': [(2, 0, 1, 1, 3), (2, 1, 0, 0, 3)]},
+    {'shapes': [(0, 1, 1, 3), (2, 1, 0, 0, 3)]},
+    {'shapes': [(0, 1, 1, 0, 3), (5, 2, 0, 1, 0, 0, 3), (2, 1, 0, 0, 0, 3)]},
+)
+class TestBroadcast(unittest.TestCase):
+
+    def _broadcast(self, xp, dtype, shapes):
+        arrays = [
+            testing.shaped_arange(s, xp, dtype) for s in shapes]
+        return xp.broadcast(*arrays)
+
+    @testing.for_all_dtypes()
+    def test_broadcast(self, dtype):
+        broadcast_np = self._broadcast(numpy, dtype, self.shapes)
+        broadcast_cp = self._broadcast(cupy, dtype, self.shapes)
+        assert broadcast_np.shape == broadcast_cp.shape
+        assert broadcast_np.size == broadcast_cp.size
+        assert broadcast_np.nd == broadcast_cp.nd
+
+    def _hip_skip_invalid_broadcast(self):
+        invalid_shapes = [
+            [(1,), (1,)],
+            [(2,), (2,)],
+        ]
+        if runtime.is_hip and self.shapes in invalid_shapes:
+            pytest.xfail('HIP/ROCm may have a bug')
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_broadcast_arrays(self, xp, dtype):
+        self._hip_skip_invalid_broadcast()
+        arrays = [
+            testing.shaped_arange(s, xp, dtype) for s in self.shapes]
+        return xp.broadcast_arrays(*arrays)
+
+
+@testing.parameterize(
+    {'shapes': [(3,), (2,)]},
+    {'shapes': [(3, 2), (2, 3,)]},
+    {'shapes': [(3, 2), (3, 4,)]},
+    {'shapes': [(0,), (2,)]},
+)
+class TestInvalidBroadcast(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_invalid_broadcast(self, dtype):
+        for xp in (numpy, cupy):
+            arrays = [testing.shaped_arange(s, xp, dtype) for s in self.shapes]
+            with pytest.raises(ValueError):
+                xp.broadcast(*arrays)
+
+    @testing.for_all_dtypes()
+    def test_invalid_broadcast_arrays(self, dtype):
+        for xp in (numpy, cupy):
+            arrays = [testing.shaped_arange(s, xp, dtype) for s in self.shapes]
+            with pytest.raises(ValueError):
+                xp.broadcast_arrays(*arrays)
diff --git a/tests/cupy_tests/manipulation_tests/test_join.py b/tests/cupy_tests/manipulation_tests/test_join.py
new file mode 100644
index 0000000..6fd40b9
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_join.py
@@ -0,0 +1,452 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy import cuda
+
+
+class TestJoin:
+
+    @testing.for_all_dtypes(name='dtype1')
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_array_equal()
+    def test_column_stack(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((4, 3), xp, dtype1)
+        b = testing.shaped_arange((4,), xp, dtype2)
+        c = testing.shaped_arange((4, 2), xp, dtype1)
+        return xp.column_stack((a, b, c))
+
+    def test_column_stack_wrong_ndim1(self):
+        a = cupy.zeros(())
+        b = cupy.zeros((3,))
+        with pytest.raises(ValueError):
+            cupy.column_stack((a, b))
+
+    def test_column_stack_wrong_ndim2(self):
+        a = cupy.zeros((3, 2, 3))
+        b = cupy.zeros((3, 2))
+        with pytest.raises(ValueError):
+            cupy.column_stack((a, b))
+
+    def test_column_stack_wrong_shape(self):
+        a = cupy.zeros((3, 2))
+        b = cupy.zeros((4, 3))
+        with pytest.raises(ValueError):
+            cupy.column_stack((a, b))
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate1(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3, 2), xp, dtype)
+        c = testing.shaped_arange((2, 3, 3), xp, dtype)
+        return xp.concatenate((a, b, c), axis=2)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate2(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3, 2), xp, dtype)
+        c = testing.shaped_arange((2, 3, 3), xp, dtype)
+        return xp.concatenate((a, b, c), axis=-1)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_axis_none(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((3, 5, 2), xp, dtype)
+        c = testing.shaped_arange((7, ), xp, dtype)
+        return xp.concatenate((a, b, c), axis=None)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_large_2(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3, 2), xp, dtype)
+        c = testing.shaped_arange((2, 3, 3), xp, dtype)
+        d = testing.shaped_arange((2, 3, 5), xp, dtype)
+        e = testing.shaped_arange((2, 3, 2), xp, dtype)
+        return xp.concatenate((a, b, c, d, e) * 2, axis=-1)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_large_3(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 1), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3, 1), xp, dtype)
+        return xp.concatenate((a, b) * 10, axis=-1)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_large_4(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3, 4), xp, dtype)
+        return xp.concatenate((a, b) * 10, axis=-1)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_large_5(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3, 4), xp, 'i')
+        return xp.concatenate((a, b) * 10, axis=-1)
+
+    @testing.multi_gpu(2)
+    def test_concatenate_large_different_devices(self):
+        arrs = []
+        for i in range(10):
+            with cuda.Device(i % 2):
+                arrs.append(cupy.empty((2, 3, 4)))
+        if cuda.runtime.deviceCanAccessPeer(0, 1) == 1:
+            with pytest.warns(cupy._util.PerformanceWarning):
+                cupy.concatenate(arrs)
+        else:
+            with pytest.raises(ValueError):
+                cupy.concatenate(arrs)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_f_contiguous(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_arange((2, 3, 2), xp, dtype).T
+        c = testing.shaped_arange((2, 3, 3), xp, dtype)
+        return xp.concatenate((a, b, c), axis=-1)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_large_f_contiguous(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = testing.shaped_arange((2, 3, 2), xp, dtype).T
+        c = testing.shaped_arange((2, 3, 3), xp, dtype)
+        d = testing.shaped_arange((2, 3, 2), xp, dtype).T
+        e = testing.shaped_arange((2, 3, 2), xp, dtype)
+        return xp.concatenate((a, b, c, d, e) * 2, axis=-1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_many_multi_dtype(self, xp):
+        a = testing.shaped_arange((2, 1), xp, 'i')
+        b = testing.shaped_arange((2, 1), xp, 'f')
+        return xp.concatenate((a, b) * 1024, axis=1)
+
+    @testing.slow
+    def test_concatenate_32bit_boundary(self):
+        a = cupy.zeros((2 ** 30,), dtype=cupy.int8)
+        b = cupy.zeros((2 ** 30,), dtype=cupy.int8)
+        ret = cupy.concatenate([a, b])
+        del a
+        del b
+        del ret
+        # Free huge memory for slow test
+        cupy.get_default_memory_pool().free_all_blocks()
+
+    def test_concatenate_wrong_ndim(self):
+        a = cupy.empty((2, 3))
+        b = cupy.empty((2,))
+        with pytest.raises(ValueError):
+            cupy.concatenate((a, b))
+
+    def test_concatenate_wrong_shape(self):
+        a = cupy.empty((2, 3, 4))
+        b = cupy.empty((3, 3, 4))
+        c = cupy.empty((4, 4, 4))
+        with pytest.raises(ValueError):
+            cupy.concatenate((a, b, c))
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_out(self, xp, dtype):
+        a = testing.shaped_arange((3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((3, 4), xp, dtype)
+        c = testing.shaped_arange((3, 4), xp, dtype)
+        out = xp.zeros((3, 12), dtype=dtype)
+        xp.concatenate((a, b, c), axis=1, out=out)
+        return out
+
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_out_same_kind(self, xp):
+        a = testing.shaped_arange((3, 4), xp, xp.float64)
+        b = testing.shaped_reverse_arange((3, 4), xp, xp.float64)
+        c = testing.shaped_arange((3, 4), xp, xp.float64)
+        out = xp.zeros((3, 12), dtype=xp.float32)
+        xp.concatenate((a, b, c), axis=1, out=out)
+        return out
+
+    def test_concatenate_out_invalid_shape(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 4), xp, xp.float64)
+            b = testing.shaped_reverse_arange((3, 4), xp, xp.float64)
+            c = testing.shaped_arange((3, 4), xp, xp.float64)
+            out = xp.zeros((4, 10), dtype=xp.float64)
+            with pytest.raises(ValueError):
+                xp.concatenate((a, b, c), axis=1, out=out)
+
+    def test_concatenate_out_invalid_shape_2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 4), xp, xp.float64)
+            b = testing.shaped_reverse_arange((3, 4), xp, xp.float64)
+            c = testing.shaped_arange((3, 4), xp, xp.float64)
+            out = xp.zeros((2, 2, 10), dtype=xp.float64)
+            with pytest.raises(ValueError):
+                xp.concatenate((a, b, c), axis=1, out=out)
+
+    def test_concatenate_out_invalid_dtype(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 4), xp, xp.float64)
+            b = testing.shaped_reverse_arange((3, 4), xp, xp.float64)
+            c = testing.shaped_arange((3, 4), xp, xp.float64)
+            out = xp.zeros((3, 12), dtype=xp.int64)
+            with pytest.raises(TypeError):
+                xp.concatenate((a, b, c), axis=1, out=out)
+
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'])
+    @testing.numpy_cupy_array_equal()
+    def test_concatenate_different_dtype(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((3, 4), xp, dtype1)
+        b = testing.shaped_arange((3, 4), xp, dtype2)
+        return xp.concatenate((a, b))
+
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'])
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_concatenate_out_different_dtype(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((3, 4), xp, dtype1)
+        b = testing.shaped_arange((3, 4), xp, dtype1)
+        out = xp.zeros((6, 4), dtype=dtype2)
+        return xp.concatenate((a, b), out=out)
+
+    @testing.with_requires('numpy>=1.20.0')
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'])
+    @testing.numpy_cupy_array_equal(accept_error=TypeError)
+    def test_concatenate_dtype(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((3, 4), xp, dtype1)
+        b = testing.shaped_arange((3, 4), xp, dtype1)
+        return xp.concatenate((a, b), dtype=dtype2)
+
+    @testing.with_requires('numpy>=1.20.0')
+    def test_concatenate_dtype_invalid_out(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 4), xp, xp.float64)
+            b = testing.shaped_arange((3, 4), xp, xp.float64)
+            out = xp.zeros((6, 4), dtype=xp.int64)
+            with pytest.raises(TypeError):
+                xp.concatenate((a, b), out=out, dtype=xp.int64)
+
+    @testing.with_requires('numpy>=1.20.0')
+    @pytest.mark.filterwarnings('error::numpy.ComplexWarning')
+    @pytest.mark.parametrize('casting', [
+        'no',
+        'equiv',
+        'safe',
+        'same_kind',
+        'unsafe',
+    ])
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'])
+    @testing.numpy_cupy_array_equal(
+        accept_error=(TypeError, numpy.ComplexWarning))
+    def test_concatenate_casting(self, xp, dtype1, dtype2, casting):
+        a = testing.shaped_arange((3, 4), xp, dtype1)
+        b = testing.shaped_arange((3, 4), xp, dtype1)
+        # may raise TypeError or ComplexWarning
+        return xp.concatenate((a, b), dtype=dtype2, casting=casting)
+
+    @testing.numpy_cupy_array_equal()
+    def test_dstack(self, xp):
+        a = testing.shaped_arange((1, 3, 2), xp)
+        b = testing.shaped_arange((3,), xp)
+        c = testing.shaped_arange((1, 3), xp)
+        return xp.dstack((a, b, c))
+
+    @testing.numpy_cupy_array_equal()
+    def test_dstack_single_element(self, xp):
+        a = testing.shaped_arange((1, 2, 3), xp)
+        return xp.dstack((a,))
+
+    @testing.numpy_cupy_array_equal()
+    def test_dstack_single_element_2(self, xp):
+        a = testing.shaped_arange((1, 2), xp)
+        return xp.dstack((a,))
+
+    @testing.numpy_cupy_array_equal()
+    def test_dstack_single_element_3(self, xp):
+        a = testing.shaped_arange((1,), xp)
+        return xp.dstack((a,))
+
+    @testing.numpy_cupy_array_equal()
+    def test_hstack_vectors(self, xp):
+        a = xp.arange(3)
+        b = xp.arange(2, -1, -1)
+        return xp.hstack((a, b))
+
+    @testing.numpy_cupy_array_equal()
+    def test_hstack_scalars(self, xp):
+        a = testing.shaped_arange((), xp)
+        b = testing.shaped_arange((), xp)
+        c = testing.shaped_arange((), xp)
+        return xp.hstack((a, b, c))
+
+    @testing.numpy_cupy_array_equal()
+    def test_hstack(self, xp):
+        a = testing.shaped_arange((2, 1), xp)
+        b = testing.shaped_arange((2, 2), xp)
+        c = testing.shaped_arange((2, 3), xp)
+        return xp.hstack((a, b, c))
+
+    @testing.numpy_cupy_array_equal()
+    def test_vstack_vectors(self, xp):
+        a = xp.arange(3)
+        b = xp.arange(2, -1, -1)
+        return xp.vstack((a, b))
+
+    @testing.numpy_cupy_array_equal()
+    def test_vstack_single_element(self, xp):
+        a = xp.arange(3)
+        return xp.vstack((a,))
+
+    def test_vstack_wrong_ndim(self):
+        a = cupy.empty((3,))
+        b = cupy.empty((3, 1))
+        with pytest.raises(ValueError):
+            cupy.vstack((a, b))
+
+    @testing.numpy_cupy_array_equal()
+    def test_stack(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        b = testing.shaped_arange((2, 3), xp)
+        c = testing.shaped_arange((2, 3), xp)
+        return xp.stack((a, b, c))
+
+    def test_stack_value(self):
+        a = testing.shaped_arange((2, 3), cupy)
+        b = testing.shaped_arange((2, 3), cupy)
+        c = testing.shaped_arange((2, 3), cupy)
+        s = cupy.stack((a, b, c))
+        assert s.shape == (3, 2, 3)
+        cupy.testing.assert_array_equal(s[0], a)
+        cupy.testing.assert_array_equal(s[1], b)
+        cupy.testing.assert_array_equal(s[2], c)
+
+    @testing.numpy_cupy_array_equal()
+    def test_stack_with_axis1(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.stack((a, a), axis=1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_stack_with_axis2(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.stack((a, a), axis=2)
+
+    def test_stack_with_axis_over(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3), xp)
+            with pytest.raises(ValueError):
+                xp.stack((a, a), axis=3)
+
+    def test_stack_with_axis_value(self):
+        a = testing.shaped_arange((2, 3), cupy)
+        s = cupy.stack((a, a), axis=1)
+
+        assert s.shape == (2, 2, 3)
+        cupy.testing.assert_array_equal(s[:, 0, :], a)
+        cupy.testing.assert_array_equal(s[:, 1, :], a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_stack_with_negative_axis(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.stack((a, a), axis=-1)
+
+    def test_stack_with_negative_axis_value(self):
+        a = testing.shaped_arange((2, 3), cupy)
+        s = cupy.stack((a, a), axis=-1)
+
+        assert s.shape == (2, 3, 2)
+        cupy.testing.assert_array_equal(s[:, :, 0], a)
+        cupy.testing.assert_array_equal(s[:, :, 1], a)
+
+    def test_stack_different_shape(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3), xp)
+            b = testing.shaped_arange((2, 4), xp)
+            with pytest.raises(ValueError):
+                xp.stack([a, b])
+
+    def test_stack_out_of_bounds1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3), xp)
+            with pytest.raises(ValueError):
+                xp.stack([a, a], axis=3)
+
+    def test_stack_out_of_bounds2(self):
+        a = testing.shaped_arange((2, 3), cupy)
+        with pytest.raises(numpy.AxisError):
+            return cupy.stack([a, a], axis=3)
+
+    @testing.for_all_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_stack_out(self, xp, dtype):
+        a = testing.shaped_arange((3, 4), xp, dtype)
+        b = testing.shaped_reverse_arange((3, 4), xp, dtype)
+        c = testing.shaped_arange((3, 4), xp, dtype)
+        out = xp.zeros((3, 3, 4), dtype=dtype)
+        xp.stack((a, b, c), axis=1, out=out)
+        return out
+
+    @testing.numpy_cupy_array_equal()
+    def test_stack_out_same_kind(self, xp):
+        a = testing.shaped_arange((3, 4), xp, xp.float64)
+        b = testing.shaped_reverse_arange((3, 4), xp, xp.float64)
+        c = testing.shaped_arange((3, 4), xp, xp.float64)
+        out = xp.zeros((3, 3, 4), dtype=xp.float32)
+        xp.stack((a, b, c), axis=1, out=out)
+        return out
+
+    def test_stack_out_invalid_shape(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 4), xp, xp.float64)
+            b = testing.shaped_reverse_arange((3, 4), xp, xp.float64)
+            c = testing.shaped_arange((3, 4), xp, xp.float64)
+            out = xp.zeros((3, 3, 10), dtype=xp.float64)
+            with pytest.raises(ValueError):
+                xp.stack((a, b, c), axis=1, out=out)
+
+    def test_stack_out_invalid_shape_2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 4), xp, xp.float64)
+            b = testing.shaped_reverse_arange((3, 4), xp, xp.float64)
+            c = testing.shaped_arange((3, 4), xp, xp.float64)
+            out = xp.zeros((3, 3, 3, 10), dtype=xp.float64)
+            with pytest.raises(ValueError):
+                xp.stack((a, b, c), axis=1, out=out)
+
+    def test_stack_out_invalid_dtype(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((3, 4), xp, xp.float64)
+            b = testing.shaped_reverse_arange((3, 4), xp, xp.float64)
+            c = testing.shaped_arange((3, 4), xp, xp.float64)
+            out = xp.zeros((3, 3, 4), dtype=xp.int64)
+            with pytest.raises(TypeError):
+                xp.stack((a, b, c), axis=1, out=out)
+
+    @testing.for_all_dtypes(name='dtype1')
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_array_equal()
+    def test_row_stack(self, xp, dtype1, dtype2):
+        a = testing.shaped_arange((4, 3), xp, dtype1)
+        b = testing.shaped_arange((3,), xp, dtype2)
+        c = testing.shaped_arange((2, 3), xp, dtype1)
+        return xp.row_stack((a, b, c))
+
+    def test_row_stack_wrong_ndim1(self):
+        a = cupy.zeros(())
+        b = cupy.zeros((3,))
+        with pytest.raises(ValueError):
+            cupy.row_stack((a, b))
+
+    def test_row_stack_wrong_ndim2(self):
+        a = cupy.zeros((3, 2, 3))
+        b = cupy.zeros((3, 2))
+        with pytest.raises(ValueError):
+            cupy.row_stack((a, b))
+
+    def test_row_stack_wrong_shape(self):
+        a = cupy.zeros((3, 2))
+        b = cupy.zeros((4, 3))
+        with pytest.raises(ValueError):
+            cupy.row_stack((a, b))
diff --git a/tests/cupy_tests/manipulation_tests/test_kind.py b/tests/cupy_tests/manipulation_tests/test_kind.py
new file mode 100644
index 0000000..7921a14
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_kind.py
@@ -0,0 +1,127 @@
+import unittest
+import pytest
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+class TestKind(unittest.TestCase):
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_asarray_chkfinite(self, xp, dtype, order):
+        a = [0, 4, 0, 5]
+        return xp.asarray_chkfinite(a, dtype=dtype, order=order)
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes(no_bool=True)
+    def test_asarray_chkfinite_non_finite_vals(self, dtype, order):
+        a = [-numpy.inf, 0., numpy.inf, numpy.nan]
+        for xp in (numpy, cupy):
+            if xp.issubdtype(dtype, xp.integer):
+                error = OverflowError
+            else:
+                error = ValueError
+            with pytest.raises(error):
+                xp.asarray_chkfinite(a, dtype=dtype, order=order)
+
+    @testing.for_all_dtypes()
+    def test_asfarray(self, dtype):
+        a = cupy.asarray([1, 2, 3])
+        a_gpu = cupy.asfarray(a, dtype)
+        a_cpu = numpy.asfarray(a.get(), dtype)
+        assert a_cpu.dtype == a_gpu.dtype
+
+    @testing.for_all_dtypes()
+    def test_asfortranarray1(self, dtype):
+        def func(xp):
+            x = xp.zeros((2, 3), dtype)
+            ret = xp.asfortranarray(x)
+            assert x.flags.c_contiguous
+            assert ret.flags.f_contiguous
+            return ret.strides
+        assert func(numpy) == func(cupy)
+
+    @testing.for_all_dtypes()
+    def test_asfortranarray2(self, dtype):
+        def func(xp):
+            x = xp.zeros((2, 3, 4), dtype)
+            ret = xp.asfortranarray(x)
+            assert x.flags.c_contiguous
+            assert ret.flags.f_contiguous
+            return ret.strides
+        assert func(numpy) == func(cupy)
+
+    @testing.for_all_dtypes()
+    def test_asfortranarray3(self, dtype):
+        def func(xp):
+            x = xp.zeros((2, 3, 4), dtype)
+            ret = xp.asfortranarray(xp.asfortranarray(x))
+            assert x.flags.c_contiguous
+            assert ret.flags.f_contiguous
+            return ret.strides
+        assert func(numpy) == func(cupy)
+
+    @testing.for_all_dtypes()
+    def test_asfortranarray4(self, dtype):
+        def func(xp):
+            x = xp.zeros((2, 3), dtype)
+            x = xp.transpose(x, (1, 0))
+            ret = xp.asfortranarray(x)
+            assert ret.flags.f_contiguous
+            return ret.strides
+        assert func(numpy) == func(cupy)
+
+    @testing.for_all_dtypes()
+    def test_asfortranarray5(self, dtype):
+        def func(xp):
+            x = testing.shaped_arange((2, 3), xp, dtype)
+            ret = xp.asfortranarray(x)
+            assert x.flags.c_contiguous
+            assert ret.flags.f_contiguous
+            return ret.strides
+        assert func(numpy) == func(cupy)
+
+    @testing.for_all_dtypes()
+    def test_require_flag_check(self, dtype):
+        possible_flags = [['C_CONTIGUOUS'], ['F_CONTIGUOUS']]
+        x = cupy.zeros((2, 3, 4), dtype)
+        for flags in possible_flags:
+            arr = cupy.require(x, dtype, flags)
+            for parameter in flags:
+                assert arr.flags[parameter]
+                assert arr.dtype == dtype
+
+    @testing.for_all_dtypes()
+    def test_require_owndata(self, dtype):
+        x = cupy.zeros((2, 3, 4), dtype)
+        arr = x.view()
+        arr = cupy.require(arr, dtype, ['O'])
+        assert arr.flags['OWNDATA']
+
+    @testing.for_all_dtypes()
+    def test_require_C_and_F_flags(self, dtype):
+        x = cupy.zeros((2, 3, 4), dtype)
+        with pytest.raises(ValueError):
+            cupy.require(x, dtype, ['C', 'F'])
+
+    @testing.for_all_dtypes()
+    def test_require_incorrect_requirments(self, dtype):
+        x = cupy.zeros((2, 3, 4), dtype)
+        with pytest.raises(ValueError):
+            cupy.require(x, dtype, ['W'])
+
+    @testing.for_all_dtypes()
+    def test_require_incorrect_dtype(self, dtype):
+        x = cupy.zeros((2, 3, 4), dtype)
+        with pytest.raises(ValueError):
+            cupy.require(x, 'random', 'C')
+
+    @testing.for_all_dtypes()
+    def test_require_empty_requirements(self, dtype):
+        x = cupy.zeros((2, 3, 4), dtype)
+        x = cupy.require(x, dtype, [])
+        assert x.flags['C_CONTIGUOUS']
diff --git a/tests/cupy_tests/manipulation_tests/test_rearrange.py b/tests/cupy_tests/manipulation_tests/test_rearrange.py
new file mode 100644
index 0000000..f09d257
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_rearrange.py
@@ -0,0 +1,272 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(
+    {'shape': (10,), 'shift': 2, 'axis': None},
+    {'shape': (5, 2), 'shift': 1, 'axis': None},
+    {'shape': (5, 2), 'shift': -2, 'axis': None},
+    {'shape': (5, 2), 'shift': 1, 'axis': 0},
+    {'shape': (5, 2), 'shift': 1, 'axis': -1},
+    {'shape': (10,), 'shift': 35, 'axis': None},
+    {'shape': (5, 2), 'shift': 11, 'axis': 0},
+    {'shape': (), 'shift': 5, 'axis': None},
+    {'shape': (5, 2), 'shift': 1, 'axis': (0, 1)},
+    {'shape': (5, 2), 'shift': 1, 'axis': (0, 0)},
+    {'shape': (5, 2), 'shift': 50, 'axis': 0},
+    {'shape': (5, 2), 'shift': (2, 1), 'axis': (0, 1)},
+    {'shape': (5, 2), 'shift': (2, 1), 'axis': (0, -1)},
+    {'shape': (5, 2), 'shift': (2, 1), 'axis': (1, -1)},
+    {'shape': (5, 2), 'shift': (2, 1, 3), 'axis': 0},
+    {'shape': (5, 2), 'shift': (2, 1, 3), 'axis': None},
+)
+class TestRoll(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_roll(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype)
+        return xp.roll(x, self.shift, axis=self.axis)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_roll_cupy_shift(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype)
+        shift = self.shift
+        if xp is cupy:
+            shift = cupy.array(shift)
+        return xp.roll(x, shift, axis=self.axis)
+
+
+class TestRollTypeError(unittest.TestCase):
+
+    def test_roll_invalid_shift(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5, 2), xp)
+            with pytest.raises(TypeError):
+                xp.roll(x, '0', axis=0)
+
+    def test_roll_invalid_axis_type(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((5, 2), xp)
+            with pytest.raises(TypeError):
+                xp.roll(x, 2, axis='0')
+
+
+@testing.parameterize(
+    {'shape': (5, 2, 3), 'shift': (2, 2, 2), 'axis': (0, 1)},
+    {'shape': (5, 2), 'shift': 1, 'axis': 2},
+    {'shape': (5, 2), 'shift': 1, 'axis': -3},
+    {'shape': (5, 2, 2), 'shift': (1, 0), 'axis': (0, 1, 2)},
+    {'shape': (5, 2), 'shift': 1, 'axis': -3},
+    {'shape': (5, 2), 'shift': 1, 'axis': (1, -3)},
+)
+class TestRollValueError(unittest.TestCase):
+    def test_roll_invalid(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange(self.shape, xp)
+            with pytest.raises(ValueError):
+                xp.roll(x, self.shift, axis=self.axis)
+
+    def test_roll_invalid_cupy_shift(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange(self.shape, xp)
+            shift = self.shift
+            if xp is cupy:
+                shift = cupy.array(shift)
+            with pytest.raises(ValueError):
+                xp.roll(x, shift, axis=self.axis)
+
+
+class TestFliplr(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_fliplr_2(self, xp, dtype):
+        x = testing.shaped_arange((3, 4), xp, dtype)
+        return xp.fliplr(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_fliplr_3(self, xp, dtype):
+        x = testing.shaped_arange((3, 4, 2), xp, dtype)
+        return xp.fliplr(x)
+
+    @testing.for_all_dtypes()
+    def test_fliplr_insufficient_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.fliplr(x)
+
+
+class TestFlipud(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flipud_1(self, xp, dtype):
+        x = testing.shaped_arange((3,), xp, dtype)
+        return xp.flipud(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flipud_2(self, xp, dtype):
+        x = testing.shaped_arange((3, 4), xp, dtype)
+        return xp.flipud(x)
+
+    @testing.for_all_dtypes()
+    def test_flipud_insufficient_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.flipud(x)
+
+
+class TestFlip(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_1(self, xp, dtype):
+        x = testing.shaped_arange((3,), xp, dtype)
+        return xp.flip(x, 0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_2(self, xp, dtype):
+        x = testing.shaped_arange((3, 4), xp, dtype)
+        return xp.flip(x, 1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_all(self, xp, dtype):
+        x = testing.shaped_arange((3, 4), xp, dtype)
+        return xp.flip(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_with_negative_axis(self, xp, dtype):
+        x = testing.shaped_arange((3, 4, 2), xp, dtype)
+        return xp.flip(x, -1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_with_axis_tuple(self, xp, dtype):
+        x = testing.shaped_arange((3, 4, 2), xp, dtype)
+        return xp.flip(x, (0, 2))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_dim_0(self, xp, dtype):
+        x = testing.shaped_arange((), xp, dtype)
+        return xp.flip(x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_empty_dim_1(self, xp, dtype):
+        x = xp.array([], dtype).reshape((0,))
+        return xp.flip(x, 0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_empty_dim_2(self, xp, dtype):
+        x = xp.array([], dtype).reshape((0, 0))
+        return xp.flip(x, 1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_empty_dim_3(self, xp, dtype):
+        x = xp.array([], dtype).reshape((1, 0, 1))
+        return xp.flip(x, 1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flip_empty_dim_all(self, xp, dtype):
+        x = xp.array([], dtype).reshape((1, 0, 1))
+        return xp.flip(x)
+
+    @testing.for_all_dtypes()
+    def test_flip_insufficient_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.flip(x, 0)
+
+    @testing.for_all_dtypes()
+    def test_flip_invalid_axis(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.flip(x, 2)
+
+    @testing.for_all_dtypes()
+    def test_flip_invalid_negative_axis(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.flip(x, -3)
+
+
+class TestRot90(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_rot90_none(self, xp, dtype):
+        x = testing.shaped_arange((3, 4), xp, dtype)
+        return xp.rot90(x, 0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_rot90_twice(self, xp, dtype):
+        x = testing.shaped_arange((3, 4, 2), xp, dtype)
+        return xp.rot90(x, 2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_rot90_negative(self, xp, dtype):
+        x = testing.shaped_arange((3, 4, 2), xp, dtype)
+        return xp.rot90(x, -1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_rot90_with_axes(self, xp, dtype):
+        x = testing.shaped_arange((3, 4, 2), xp, dtype)
+        return xp.rot90(x, 1, axes=(1, 2))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_rot90_with_negative_axes(self, xp, dtype):
+        x = testing.shaped_arange((3, 4, 2), xp, dtype)
+        return xp.rot90(x, 1, axes=(1, -1))
+
+    @testing.for_all_dtypes()
+    def test_rot90_insufficient_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.rot90(x)
+
+    @testing.for_all_dtypes()
+    def test_rot90_too_much_axes(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3, 4, 2), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.rot90(x, 1, axes=(0, 1, 2))
+
+    @testing.for_all_dtypes()
+    def test_rot90_invalid_axes(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3, 4, 2), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.rot90(x, 1, axes=(1, 3))
+
+    @testing.for_all_dtypes()
+    def test_rot90_invalid_negative_axes(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3, 4, 2), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.rot90(x, 1, axes=(1, -2))
diff --git a/tests/cupy_tests/manipulation_tests/test_shape.py b/tests/cupy_tests/manipulation_tests/test_shape.py
new file mode 100644
index 0000000..b8c8566
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_shape.py
@@ -0,0 +1,243 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@pytest.mark.parametrize('shape', [(2, 3), (), (4,)])
+class TestShape:
+
+    def test_shape(self, shape):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange(shape, xp)
+            assert cupy.shape(a) == shape
+
+    def test_shape_list(self, shape):
+        a = testing.shaped_arange(shape, numpy)
+        a = a.tolist()
+        assert cupy.shape(a) == shape
+
+
+class TestReshape:
+
+    def test_reshape_strides(self):
+        def func(xp):
+            a = testing.shaped_arange((1, 1, 1, 2, 2), xp)
+            return a.strides
+        assert func(numpy) == func(cupy)
+
+    def test_reshape2(self):
+        def func(xp):
+            a = xp.zeros((8,), dtype=xp.float32)
+            return a.reshape((1, 1, 1, 4, 1, 2)).strides
+        assert func(numpy) == func(cupy)
+
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_nocopy_reshape(self, xp, dtype, order):
+        a = xp.zeros((2, 3, 4), dtype=dtype)
+        b = a.reshape(4, 3, 2, order=order)
+        b[1] = 1
+        return a
+
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_nocopy_reshape_with_order(self, xp, dtype, order):
+        a = xp.zeros((2, 3, 4), dtype=dtype)
+        b = a.reshape(4, 3, 2, order=order)
+        b[1] = 1
+        return a
+
+    @testing.for_orders('CFA')
+    @testing.numpy_cupy_array_equal()
+    def test_transposed_reshape2(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp).transpose(2, 0, 1)
+        return a.reshape(2, 3, 4, order=order)
+
+    @testing.for_orders('CFA')
+    @testing.numpy_cupy_array_equal()
+    def test_reshape_with_unknown_dimension(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.reshape(3, -1, order=order)
+
+    def test_reshape_with_multiple_unknown_dimensions(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                a.reshape(3, -1, -1)
+
+    def test_reshape_with_changed_arraysize(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                a.reshape(2, 4, 4)
+
+    def test_reshape_invalid_order(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                a.reshape(2, 4, 4, order='K')
+
+    def test_reshape_zerosize_invalid(self):
+        for xp in (numpy, cupy):
+            a = xp.zeros((0,))
+            with pytest.raises(ValueError):
+                a.reshape(())
+
+    def test_reshape_zerosize_invalid_unknown(self):
+        for xp in (numpy, cupy):
+            a = xp.zeros((0,))
+            with pytest.raises(ValueError):
+                a.reshape((-1, 0))
+
+    @testing.numpy_cupy_array_equal()
+    def test_reshape_zerosize(self, xp):
+        a = xp.zeros((0,))
+        b = a.reshape((0,))
+        assert b.base is a
+        return b
+
+    @testing.for_orders('CFA')
+    @testing.numpy_cupy_array_equal(strides_check=True)
+    def test_reshape_zerosize2(self, xp, order):
+        a = xp.zeros((2, 0, 3))
+        b = a.reshape((5, 0, 4), order=order)
+        assert b.base is a
+        return b
+
+    @testing.for_orders('CFA')
+    @testing.numpy_cupy_array_equal()
+    def test_external_reshape(self, xp, order):
+        a = xp.zeros((8,), dtype=xp.float32)
+        return xp.reshape(a, (1, 1, 1, 4, 1, 2), order=order)
+
+    def _test_ndim_limit(self, xp, ndim, dtype, order):
+        idx = [1]*ndim
+        idx[-1] = ndim
+        a = xp.ones(ndim, dtype=dtype)
+        a = a.reshape(idx, order=order)
+        assert a.ndim == ndim
+        return a
+
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_ndim_limit1(self, xp, dtype, order):
+        # from cupy/cupy#4193
+        a = self._test_ndim_limit(xp, 32, dtype, order)
+        return a
+
+    @testing.for_orders('CFA')
+    @testing.for_all_dtypes()
+    def test_ndim_limit2(self, dtype, order):
+        # from cupy/cupy#4193
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                self._test_ndim_limit(xp, 33, dtype, order)
+
+
+class TestRavel:
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_ravel(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        a = a.transpose(2, 0, 1)
+        return a.ravel(order)
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_ravel2(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.ravel(order)
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_ravel3(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        a = xp.asfortranarray(a)
+        return a.ravel(order)
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_ravel4(self, xp, order):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        a = a.transpose(0, 2, 1)[:, ::-2]
+        return a.ravel(order)
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_ravel_non_contiguous(self, xp, order):
+        a = xp.arange(10)[::2]
+        assert not a.flags.c_contiguous and not a.flags.f_contiguous
+        b = a.ravel(order)
+        assert b.flags.c_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_ravel_broadcasted(self, xp, order):
+        a = xp.array([1])
+        b = xp.broadcast_to(a, (10,))
+        assert not b.flags.c_contiguous and not b.flags.f_contiguous
+        b = b.ravel(order)
+        assert b.flags.c_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.numpy_cupy_array_equal()
+    def test_ravel_broadcasted2(self, xp, order):
+        a = testing.shaped_arange((2, 1), xp)
+        b = xp.broadcast_to(a, (3, 2, 4))
+        assert not b.flags.c_contiguous and not b.flags.f_contiguous
+        b = b.ravel(order)
+        assert b.flags.c_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_orders('CF', name='a_order')
+    @testing.numpy_cupy_array_equal()
+    def test_ravel_broadcasted3(self, xp, order, a_order):
+        a = testing.shaped_arange((2, 1, 3), xp, order=a_order)
+        b = xp.broadcast_to(a, (2, 4, 3))
+        b = b.ravel(order)
+        assert b.flags.c_contiguous
+        return b
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_ravel(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        a = a.transpose(2, 0, 1)
+        return xp.ravel(a)
+
+
+@pytest.mark.parametrize('order_init', ['C', 'F'])
+@pytest.mark.parametrize('order_reshape', ['C', 'F', 'A', 'c', 'f', 'a'])
+@pytest.mark.parametrize('shape_in_out', [
+    ((2, 3), (1, 6, 1)),  # (shape_init, shape_final)
+    ((6,), (2, 3)),
+    ((3, 3, 3), (9, 3)),
+])
+class TestReshapeOrder:
+
+    def test_reshape_contiguity(
+        self, order_init, order_reshape, shape_in_out
+    ):
+        shape_init, shape_final = shape_in_out
+
+        a_cupy = testing.shaped_arange(shape_init, xp=cupy)
+        a_cupy = cupy.asarray(a_cupy, order=order_init)
+        b_cupy = a_cupy.reshape(shape_final, order=order_reshape)
+
+        a_numpy = testing.shaped_arange(shape_init, xp=numpy)
+        a_numpy = numpy.asarray(a_numpy, order=order_init)
+        b_numpy = a_numpy.reshape(shape_final, order=order_reshape)
+
+        assert b_cupy.flags.f_contiguous == b_numpy.flags.f_contiguous
+        assert b_cupy.flags.c_contiguous == b_numpy.flags.c_contiguous
+
+        testing.assert_array_equal(b_cupy.strides, b_numpy.strides)
+        testing.assert_array_equal(b_cupy, b_numpy)
diff --git a/tests/cupy_tests/manipulation_tests/test_split.py b/tests/cupy_tests/manipulation_tests/test_split.py
new file mode 100644
index 0000000..ad9d269
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_split.py
@@ -0,0 +1,96 @@
+import unittest
+
+from cupy import testing
+
+
+class TestSplit(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_split1(self, xp):
+        a = testing.shaped_arange((3, 11), xp)
+        return xp.array_split(a, 4, 1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_split2(self, xp):
+        a = testing.shaped_arange((3, 11), xp)
+        return xp.array_split(a, 4, -1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_split_empty_array(self, xp):
+        a = testing.shaped_arange((5, 0), xp)
+        return xp.array_split(a, [2, 4], 0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_split_empty_sections(self, xp):
+        a = testing.shaped_arange((3, 11), xp)
+        return xp.array_split(a, [])
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_split_out_of_bound1(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.array_split(a, [3])
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_split_out_of_bound2(self, xp):
+        a = testing.shaped_arange((0,), xp)
+        return xp.array_split(a, [1])
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_split_unordered_sections(self, xp):
+        a = testing.shaped_arange((5,), xp)
+        return xp.array_split(a, [4, 2])
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_split_non_divisible(self, xp):
+        a = testing.shaped_arange((5, 3), xp)
+        return xp.array_split(a, 4)
+
+    @testing.numpy_cupy_array_equal()
+    def test_dsplit(self, xp):
+        a = testing.shaped_arange((3, 3, 12), xp)
+        return xp.dsplit(a, 4)
+
+    @testing.numpy_cupy_array_equal()
+    def test_hsplit_vectors(self, xp):
+        a = testing.shaped_arange((12,), xp)
+        return xp.hsplit(a, 4)
+
+    @testing.numpy_cupy_array_equal()
+    def test_hsplit(self, xp):
+        a = testing.shaped_arange((3, 12), xp)
+        return xp.hsplit(a, 4)
+
+    @testing.numpy_cupy_array_equal()
+    def test_split_by_sections1(self, xp):
+        a = testing.shaped_arange((3, 11), xp)
+        return xp.split(a, (2, 4, 9), 1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_split_by_sections2(self, xp):
+        a = testing.shaped_arange((3, 11), xp)
+        return xp.split(a, (2, 4, 9), -1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_split_by_sections3(self, xp):
+        a = testing.shaped_arange((3, 11), xp)
+        return xp.split(a, (-9, 4, -2), 1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_split_out_of_bound1(self, xp):
+        a = testing.shaped_arange((2, 3), xp)
+        return xp.split(a, [3])
+
+    @testing.numpy_cupy_array_equal()
+    def test_split_out_of_bound2(self, xp):
+        a = testing.shaped_arange((0,), xp)
+        return xp.split(a, [1])
+
+    @testing.numpy_cupy_array_equal()
+    def test_split_unordered_sections(self, xp):
+        a = testing.shaped_arange((5,), xp)
+        return xp.split(a, [4, 2])
+
+    @testing.numpy_cupy_array_equal()
+    def test_vsplit(self, xp):
+        a = testing.shaped_arange((12, 3), xp)
+        return xp.vsplit(a, 4)
diff --git a/tests/cupy_tests/manipulation_tests/test_tiling.py b/tests/cupy_tests/manipulation_tests/test_tiling.py
new file mode 100644
index 0000000..45417ce
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_tiling.py
@@ -0,0 +1,131 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(
+    {'repeats': 0, 'axis': None},
+    {'repeats': 2, 'axis': None},
+    {'repeats': 2, 'axis': 1},
+    {'repeats': 2, 'axis': -1},
+    {'repeats': [0, 0, 0], 'axis': 1},
+    {'repeats': [1, 2, 3], 'axis': 1},
+    {'repeats': [1, 2, 3], 'axis': -2},
+)
+class TestRepeat(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_repeat(self, xp):
+        x = testing.shaped_arange((2, 3, 4), xp)
+        return xp.repeat(x, self.repeats, self.axis)
+
+
+class TestRepeatRepeatsNdarray(unittest.TestCase):
+
+    def test_func(self):
+        a = testing.shaped_arange((2, 3, 4), cupy)
+        repeats = cupy.array([2, 3], dtype=cupy.int32)
+        with pytest.raises(ValueError, match=r'repeats'):
+            cupy.repeat(a, repeats)
+
+    def test_method(self):
+        a = testing.shaped_arange((2, 3, 4), cupy)
+        repeats = cupy.array([2, 3], dtype=cupy.int32)
+        with pytest.raises(ValueError, match=r'repeats'):
+            a.repeat(repeats)
+
+
+@testing.parameterize(
+    {'repeats': [2], 'axis': None},
+    {'repeats': [2], 'axis': 1},
+)
+class TestRepeatListBroadcast(unittest.TestCase):
+
+    """Test for `repeats` argument using single element list.
+
+    This feature is only supported in NumPy 1.10 or later.
+    """
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_repeat(self, xp):
+        x = testing.shaped_arange((2, 3, 4), xp)
+        return xp.repeat(x, self.repeats, self.axis)
+
+
+@testing.parameterize(
+    {'repeats': 0, 'axis': None},
+    {'repeats': 2, 'axis': None},
+    {'repeats': 2, 'axis': 0},
+    {'repeats': [1, 2, 3, 4], 'axis': None},
+    {'repeats': [1, 2, 3, 4], 'axis': 0},
+)
+class TestRepeat1D(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_repeat(self, xp):
+        x = testing.shaped_arange((4,), xp)
+        return xp.repeat(x, self.repeats, self.axis)
+
+
+@testing.parameterize(
+    {'repeats': [2], 'axis': None},
+    {'repeats': [2], 'axis': 0},
+)
+class TestRepeat1DListBroadcast(unittest.TestCase):
+
+    """See comment in TestRepeatListBroadcast class."""
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_repeat(self, xp):
+        x = testing.shaped_arange((4,), xp)
+        return xp.repeat(x, self.repeats, self.axis)
+
+
+@testing.parameterize(
+    {'repeats': -3, 'axis': None},
+    {'repeats': [-3, -3], 'axis': 0},
+    {'repeats': [1, 2, 3], 'axis': None},
+    {'repeats': [1, 2], 'axis': 1},
+    {'repeats': 2, 'axis': -4},
+    {'repeats': 2, 'axis': 3},
+)
+class TestRepeatFailure(unittest.TestCase):
+
+    def test_repeat_failure(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.repeat(x, self.repeats, self.axis)
+
+
+@testing.parameterize(
+    {'reps': 0},
+    {'reps': 1},
+    {'reps': 2},
+    {'reps': (0, 1)},
+    {'reps': (2, 3)},
+    {'reps': (2, 3, 4, 5)},
+)
+class TestTile(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_tile(self, xp):
+        x = testing.shaped_arange((2, 3, 4), xp)
+        return xp.tile(x, self.reps)
+
+
+@testing.parameterize(
+    {'reps': -1},
+    {'reps': (-1, -2)},
+)
+class TestTileFailure(unittest.TestCase):
+
+    def test_tile_failure(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.tile(x, -3)
diff --git a/tests/cupy_tests/manipulation_tests/test_transpose.py b/tests/cupy_tests/manipulation_tests/test_transpose.py
new file mode 100644
index 0000000..53669e3
--- /dev/null
+++ b/tests/cupy_tests/manipulation_tests/test_transpose.py
@@ -0,0 +1,170 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestTranspose(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_moveaxis1(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.moveaxis(a, [0, 1], [1, 2])
+
+    @testing.numpy_cupy_array_equal()
+    def test_moveaxis2(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.moveaxis(a, 1, -1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_moveaxis3(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.moveaxis(a, [0, 2], [1, 0])
+
+    @testing.numpy_cupy_array_equal()
+    def test_moveaxis4(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.moveaxis(a, [2, 0], [1, 0])
+
+    @testing.numpy_cupy_array_equal()
+    def test_moveaxis5(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.moveaxis(a, [2, 0], [0, 1])
+
+    @testing.numpy_cupy_array_equal()
+    def test_moveaxis6(self, xp):
+        a = testing.shaped_arange((2, 3, 4, 5, 6), xp)
+        return xp.moveaxis(a, [0, 2, 1], [3, 4, 0])
+
+    # dim is too large
+    def test_moveaxis_invalid1_1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.moveaxis(a, [0, 1], [1, 3])
+
+    def test_moveaxis_invalid1_2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.moveaxis(a, [0, 1], [1, 3])
+
+    def test_moveaxis_invalid1_3(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.moveaxis(a, 0, 3)
+
+    # dim is too small
+    def test_moveaxis_invalid2_1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.moveaxis(a, [0, -4], [1, 2])
+
+    def test_moveaxis_invalid2_2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.moveaxis(a, [0, -4], [1, 2])
+
+    def test_moveaxis_invalid2_3(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.moveaxis(a, -4, 0)
+
+    # len(source) != len(destination)
+    def test_moveaxis_invalid3_1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.moveaxis(a, [0, 1, 2], [1, 2])
+
+    def test_moveaxis_invalid3_2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.moveaxis(a, 0, [1, 2])
+
+    # len(source) != len(destination)
+    def test_moveaxis_invalid4_1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.moveaxis(a, [0, 1], [1, 2, 0])
+
+    def test_moveaxis_invalid4_2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.moveaxis(a, [0, 1], 1)
+
+    # Use the same axis twice
+    def test_moveaxis_invalid5_1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.moveaxis(a, [1, -1], [1, 3])
+
+    def test_moveaxis_invalid5_2(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.moveaxis(a, [0, 1], [-1, 2])
+
+    def test_moveaxis_invalid5_3(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.moveaxis(a, [0, 1], [1, 1])
+
+    @testing.numpy_cupy_array_equal()
+    def test_rollaxis(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.rollaxis(a, 2)
+
+    def test_rollaxis_failure(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.rollaxis(a, 3)
+
+    @testing.numpy_cupy_array_equal()
+    def test_swapaxes(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.swapaxes(a, 2, 0)
+
+    def test_swapaxes_failure(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(ValueError):
+                xp.swapaxes(a, 3, 0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_transpose(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.transpose(-1, 0, 1)
+
+    @testing.numpy_cupy_array_equal()
+    def test_transpose_empty(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.transpose()
+
+    @testing.numpy_cupy_array_equal()
+    def test_transpose_none(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return a.transpose(None)
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_transpose(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.transpose(a, (-1, 0, 1))
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_transpose_all(self, xp):
+        a = testing.shaped_arange((2, 3, 4), xp)
+        return xp.transpose(a)
diff --git a/tests/cupy_tests/math_tests/__init__.py b/tests/cupy_tests/math_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/math_tests/test_arithmetic.py b/tests/cupy_tests/math_tests/test_arithmetic.py
new file mode 100644
index 0000000..36e63b0
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_arithmetic.py
@@ -0,0 +1,533 @@
+import itertools
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+float_types = [numpy.float16, numpy.float32, numpy.float64]
+complex_types = [numpy.complex64, numpy.complex128]
+signed_int_types = [numpy.int8, numpy.int16, numpy.int32, numpy.int64]
+unsigned_int_types = [numpy.uint8, numpy.uint16, numpy.uint32, numpy.uint64]
+int_types = signed_int_types + unsigned_int_types
+all_types = [numpy.bool_] + float_types + int_types + complex_types
+negative_types = (
+    [numpy.bool_] + float_types + signed_int_types + complex_types)
+negative_types_wo_fp16 = (
+    [numpy.bool_] + [numpy.float32, numpy.float64]
+    + [numpy.int16, numpy.int32, numpy.int64] + complex_types)
+negative_no_complex_types = [numpy.bool_] + float_types + signed_int_types
+no_complex_types = [numpy.bool_] + float_types + int_types
+
+
+@testing.parameterize(*(
+    testing.product({
+        'nargs': [1],
+        'name': ['reciprocal', 'conj', 'conjugate', 'angle'],
+    }) + testing.product({
+        'nargs': [2],
+        'name': [
+            'add', 'multiply', 'divide', 'power', 'subtract', 'true_divide',
+            'floor_divide', 'float_power', 'fmod', 'remainder'],
+    })
+))
+class TestArithmeticRaisesWithNumpyInput:
+
+    def test_raises_with_numpy_input(self):
+        nargs = self.nargs
+        name = self.name
+
+        # Check TypeError is raised if numpy.ndarray is given as input
+        func = getattr(cupy, name)
+        for input_xp_list in itertools.product(*[[numpy, cupy]] * nargs):
+            if all(xp is cupy for xp in input_xp_list):
+                # We don't test all-cupy-array inputs here
+                continue
+            arys = [xp.array([2, -3]) for xp in input_xp_list]
+            with pytest.raises(TypeError):
+                func(*arys)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d)
+                  for d in all_types
+                  ] + [0, 0.0j, 0j, 2, 2.0, 2j, True, False]),
+        'name': ['conj', 'conjugate', 'real', 'imag'],
+    }) + testing.product({
+        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d)
+                  for d in all_types
+                  ] + [0, 0.0j, 0j, 2, 2.0, 2j, True, False]),
+        'deg': [True, False],
+        'name': ['angle'],
+    }) + testing.product({
+        'arg1': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                  for d in negative_types_wo_fp16
+                  ] + [0, 0.0j, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False]),
+        'deg': [True, False],
+        'name': ['angle'],
+    }) + testing.product({
+        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d) + 1
+                  for d in all_types
+                  ] + [2, 2.0, 2j, True]),
+        'name': ['reciprocal'],
+    })
+))
+class TestArithmeticUnary:
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_unary(self, xp):
+        arg1 = self.arg1
+        if isinstance(arg1, numpy.ndarray):
+            arg1 = xp.asarray(arg1)
+
+        if self.name in {'angle'}:
+            y = getattr(xp, self.name)(arg1, self.deg)
+        else:
+            y = getattr(xp, self.name)(arg1)
+
+        if self.name in ('real', 'imag'):
+            # Some NumPy functions return Python scalars for Python scalar
+            # inputs.
+            # We need to convert them to arrays to compare with CuPy outputs.
+            if xp is numpy and isinstance(arg1, (bool, int, float, complex)):
+                y = xp.asarray(y)
+
+            # TODO(niboshi): Fix this
+            # numpy.real and numpy.imag return Python int if the input is
+            # Python bool. CuPy should return an array of dtype=int32 or
+            # dtype=int64 (depending on the platform) in such cases, instead
+            # of an array of dtype=bool.
+            if xp is cupy and isinstance(arg1, bool):
+                y = y.astype(int)
+
+        return y
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(3, 2), (), (3, 0, 2)],
+}))
+class TestComplex:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_real_ndarray_nocomplex(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        real = x.real
+        assert real is x  # real returns self
+        return real
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_real_nocomplex(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        real = xp.real(x)
+        assert real is x  # real returns self
+        return real
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_imag_ndarray_nocomplex(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        imag = x.imag
+        return imag
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_imag_nocomplex(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        imag = xp.imag(x)
+        return imag
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_real_ndarray_complex(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        x_ = x.copy()
+        real = x_.real
+        # real returns a view
+        assert real.base is x_
+        x_ += 1 + 1j
+        testing.assert_array_equal(real, x.real + 1)
+        return real
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_real_complex(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        x_ = x.copy()
+        real = xp.real(x_)
+        # real returns a view
+        assert real.base is x_
+        x_ += 1 + 1j
+        testing.assert_array_equal(real, x.real + 1)
+        return real
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_imag_ndarray_complex(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        x_ = x.copy()
+        imag = x_.imag
+        # imag returns a view
+        assert imag.base is x_
+        x_ += 1 + 1j
+        testing.assert_array_equal(imag, x.imag + 1)
+        return imag
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_imag_complex(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype=dtype)
+        x_ = x.copy()
+        imag = xp.imag(x_)
+        # imag returns a view
+        assert imag.base is x_
+        x_ += 1 + 1j
+        testing.assert_array_equal(imag, x.imag + 1)
+        return imag
+
+
+class ArithmeticBinaryBase:
+
+    @testing.numpy_cupy_allclose(atol=1e-4)
+    def check_binary(self, xp):
+        arg1 = self.arg1
+        arg2 = self.arg2
+        np1 = numpy.asarray(arg1)
+        np2 = numpy.asarray(arg2)
+        dtype1 = np1.dtype
+        dtype2 = np2.dtype
+
+        if self.name == 'power' or self.name == 'float_power':
+            # TODO(niboshi): Fix this: power(0, 1j)
+            #     numpy => 1+0j
+            #     cupy => 0j
+            if dtype2 in complex_types and (np1 == 0).any():
+                return xp.array(True)
+
+        # TODO(niboshi): Fix this: xp.add(0j, xp.array([2.], 'f')).dtype
+        #     numpy => complex64
+        #     cupy => complex128
+        if isinstance(arg1, complex):
+            if dtype2 in (numpy.float16, numpy.float32):
+                return xp.array(True)
+
+        if isinstance(arg1, numpy.ndarray):
+            arg1 = xp.asarray(arg1)
+        if isinstance(arg2, numpy.ndarray):
+            arg2 = xp.asarray(arg2)
+
+        # Subtraction between booleans is not allowed.
+        if (self.name == 'subtract'
+                and dtype1 == numpy.bool_
+                and dtype2 == numpy.bool_):
+            return xp.array(True)
+
+        func = getattr(xp, self.name)
+        with numpy.errstate(divide='ignore'):
+            with warnings.catch_warnings():
+                warnings.filterwarnings('ignore')
+                if self.use_dtype:
+                    y = func(arg1, arg2, dtype=self.dtype)
+                else:
+                    y = func(arg1, arg2)
+
+        # TODO(niboshi): Fix this. If rhs is a Python complex,
+        #    numpy returns complex64
+        #    cupy returns complex128
+        if (xp is cupy and isinstance(arg2, complex)
+                and self.name != 'float_power'):
+            if dtype1 in (numpy.float16, numpy.float32):
+                y = y.astype(numpy.complex64)
+
+        # NumPy returns different values (nan/inf) on division by zero
+        # depending on the architecture.
+        # As it is not possible for CuPy to replicate this behavior, we ignore
+        # the difference here.
+        if self.name in ('floor_divide', 'remainder'):
+            if y.dtype in (float_types + complex_types) and (np2 == 0).any():
+                y = xp.asarray(y)
+                y[y == numpy.inf] = numpy.nan
+                y[y == -numpy.inf] = numpy.nan
+
+        return y
+
+
+@testing.parameterize(*(
+    testing.product({
+        # TODO(unno): boolean subtract causes DeprecationWarning in numpy>=1.13
+        'arg1': [testing.shaped_arange((2, 3), numpy, dtype=d)
+                 for d in all_types
+                 ] + [0, 0.0, 0j, 2, 2.0, 2j, True, False],
+        'arg2': [testing.shaped_reverse_arange((2, 3), numpy, dtype=d)
+                 for d in all_types
+                 ] + [0, 0.0, 0j, 2, 2.0, 2j, True, False],
+        'name': ['add', 'multiply', 'power', 'subtract', 'float_power'],
+    }) + testing.product({
+        'arg1': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                 for d in negative_types
+                 ] + [0, 0.0, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False],
+        'arg2': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                 for d in negative_types
+                 ] + [0, 0.0, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False],
+        'name': ['divide', 'true_divide', 'subtract'],
+    })
+))
+class TestArithmeticBinary(ArithmeticBinaryBase):
+
+    def test_binary(self):
+        self.use_dtype = False
+        self.check_binary()
+
+
+@testing.parameterize(*(
+    testing.product({
+        'arg1': [numpy.array([3, 2, 1, 1, 2, 3], dtype=d)
+                 for d in unsigned_int_types
+                 ] + [0, 0.0, 2, 2.0, -2, -2.0, True, False],
+        'arg2': [numpy.array([3, 2, 1, 1, 2, 3], dtype=d)
+                 for d in unsigned_int_types
+                 ] + [0, 0.0, 2, 2.0, -2, -2.0, True, False],
+        'name': ['true_divide'],
+        'dtype': [numpy.float64],
+        'use_dtype': [True, False],
+    }) + testing.product({
+        'arg1': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                 for d in signed_int_types
+                 ] + [0, 0.0, 2, 2.0, -2, -2.0, True, False],
+        'arg2': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                 for d in signed_int_types
+                 ] + [0, 0.0, 2, 2.0, -2, -2.0, True, False],
+        'name': ['true_divide'],
+        'dtype': [numpy.float64],
+        'use_dtype': [True, False],
+    }) + testing.product({
+        'arg1': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                 for d in float_types] + [0.0, 2.0, -2.0],
+        'arg2': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                 for d in float_types] + [0.0, 2.0, -2.0],
+        'name': ['power', 'true_divide', 'subtract', 'float_power'],
+        'dtype': [numpy.float64],
+        'use_dtype': [True, False],
+    }) + testing.product({
+        'arg1': [testing.shaped_arange((2, 3), numpy, dtype=d)
+                 for d in no_complex_types
+                 ] + [0, 0.0, 2, 2.0, -2, -2.0, True, False],
+        'arg2': [testing.shaped_reverse_arange((2, 3), numpy, dtype=d)
+                 for d in no_complex_types
+                 ] + [0, 0.0, 2, 2.0, -2, -2.0, True, False],
+        'name': ['floor_divide', 'fmod', 'remainder'],
+        'dtype': [numpy.float64],
+        'use_dtype': [True, False],
+    }) + testing.product({
+        'arg1': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                 for d in negative_no_complex_types
+                 ] + [0, 0.0, 2, 2.0, -2, -2.0, True, False],
+        'arg2': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
+                 for d in negative_no_complex_types
+                 ] + [0, 0.0, 2, 2.0, -2, -2.0, True, False],
+        'name': ['floor_divide', 'fmod', 'remainder'],
+        'dtype': [numpy.float64],
+        'use_dtype': [True, False],
+    })
+))
+class TestArithmeticBinary2(ArithmeticBinaryBase):
+
+    def test_binary(self):
+        self.check_binary()
+
+
+class UfuncTestBase:
+
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_casting_out(self, in0_type, in1_type, out_type, casting, xp):
+        a = testing.shaped_arange((2, 3), xp, in0_type)
+        b = testing.shaped_arange((2, 3), xp, in1_type)
+        c = xp.zeros((2, 3), out_type)
+        if casting != 'unsafe':
+            # may raise TypeError
+            return xp.add(a, b, out=c, casting=casting)
+
+        with warnings.catch_warnings(record=True) as ws:
+            warnings.simplefilter('always')
+            ret = xp.add(a, b, out=c, casting=casting)
+        ws = [w.category for w in ws]
+        assert all([w == numpy.ComplexWarning for w in ws]), str(ws)
+        return ret, xp.array(len(ws))
+
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def check_casting_dtype(self, in0_type, in1_type, dtype, casting, xp):
+        a = testing.shaped_arange((2, 3), xp, in0_type)
+        b = testing.shaped_arange((2, 3), xp, in1_type)
+        if casting != 'unsafe':
+            # may raise TypeError
+            return xp.add(a, b, dtype=dtype, casting=casting)
+
+        with warnings.catch_warnings(record=True) as ws:
+            warnings.simplefilter('always')
+            ret = xp.add(a, b, dtype=dtype, casting='unsafe')
+        ws = [w.category for w in ws]
+        assert all([w == numpy.ComplexWarning for w in ws]), str(ws)
+        return ret, xp.array(len(ws))
+
+    # delete this, once check_casting_dtype passes
+    @testing.numpy_cupy_allclose()
+    def check_casting_dtype_unsafe_ignore_warnings(
+            self, in0_type, in1_type, dtype, xp):
+        a = testing.shaped_arange((2, 3), xp, in0_type)
+        b = testing.shaped_arange((2, 3), xp, in1_type)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            return xp.add(a, b, dtype=dtype, casting='unsafe')
+
+
+class TestUfunc(UfuncTestBase):
+
+    @pytest.mark.parametrize('casting', [
+        'no',
+        'equiv',
+        'safe',
+        'same_kind',
+        'unsafe',
+    ])
+    @testing.for_all_dtypes_combination(
+        names=['in_type', 'out_type'])
+    def test_casting_out_only(self, in_type, out_type, casting):
+        self.check_casting_out(in_type, in_type, out_type, casting)
+
+    @pytest.mark.parametrize('casting', [
+        pytest.param('no', marks=pytest.mark.skip('flaky xfail')),
+        pytest.param('equiv', marks=pytest.mark.skip('flaky xfail')),
+        'safe',
+        'same_kind',
+        'unsafe',
+    ])
+    @testing.for_all_dtypes_combination(
+        names=['in0_type', 'in1_type', 'out_type'], full=False)
+    def test_casting_in_out(self, in0_type, in1_type, out_type, casting):
+        self.check_casting_out(in0_type, in1_type, out_type, casting)
+
+    @pytest.mark.xfail()
+    @pytest.mark.parametrize('casting', [
+        'no',
+        'equiv',
+    ])
+    @pytest.mark.parametrize(('in0_type', 'in1_type', 'out_type'), [
+        (numpy.int16, numpy.int32, numpy.int32),
+    ])
+    def test_casting_in_xfail1(self, in0_type, in1_type, out_type, casting):
+        self.check_casting_out(in0_type, in1_type, out_type, casting)
+
+    @pytest.mark.skip('flaky xfail')
+    @pytest.mark.parametrize('casting', [
+        'no',
+        'equiv',
+        'safe',
+        'same_kind',
+        'unsafe',
+    ])
+    @testing.for_all_dtypes_combination(
+        names=['in0_type', 'in1_type', 'dtype'], full=False)
+    def test_casting_dtype(self, in0_type, in1_type, dtype, casting):
+        self.check_casting_dtype(in0_type, in1_type, dtype, casting)
+
+    @pytest.mark.xfail()
+    @pytest.mark.parametrize('casting', [
+        'no',
+        'equiv',
+    ])
+    @pytest.mark.parametrize(('in0_type', 'in1_type', 'dtype'), [
+        (numpy.int16, numpy.int32, numpy.int32),
+    ])
+    def test_casting_dtype_xfail1(self, in0_type, in1_type, dtype, casting):
+        self.check_casting_dtype(in0_type, in1_type, dtype, casting)
+
+    @pytest.mark.xfail()
+    @pytest.mark.parametrize('casting', [
+        'no',
+        'equiv',
+        'safe',
+        'same_kind',
+    ])
+    @pytest.mark.parametrize(('in0_type', 'in1_type', 'dtype'), [
+        (numpy.int32, numpy.int32, numpy.bool_),
+        (numpy.float64, numpy.float64, numpy.int32),
+    ])
+    def test_casting_dtype_xfail2(self, in0_type, in1_type, dtype, casting):
+        self.check_casting_dtype(in0_type, in1_type, dtype, casting)
+
+    @testing.for_all_dtypes_combination(
+        names=['in0_type', 'in1_type', 'dtype'], full=False)
+    def test_casting_dtype_unsafe_ignore_warnings(
+            self, in0_type, in1_type, dtype):
+        self.check_casting_dtype_unsafe_ignore_warnings(
+            in0_type, in1_type, dtype
+        )
+
+
+@testing.slow
+class TestUfuncSlow(UfuncTestBase):
+    @pytest.mark.parametrize('casting', [
+        pytest.param('no', marks=pytest.mark.xfail()),
+        pytest.param('equiv', marks=pytest.mark.xfail()),
+        'safe',
+        'same_kind',
+        'unsafe',
+    ])
+    @testing.for_all_dtypes_combination(
+        names=['in0_type', 'in1_type', 'out_type'], full=True)
+    def test_casting_out(self, in0_type, in1_type, out_type, casting):
+        self.check_casting_out(in0_type, in1_type, out_type, casting)
+
+    @pytest.mark.xfail()
+    @pytest.mark.parametrize('casting', [
+        'no',
+        'equiv',
+        'safe',
+        'same_kind',
+        'unsafe',
+    ])
+    @testing.for_all_dtypes_combination(
+        names=['in0_type', 'in1_type', 'dtype'], full=True)
+    def test_casting_dtype(self, in0_type, in1_type, dtype, casting):
+        self.check_casting_dtype(in0_type, in1_type, dtype, casting)
+
+    @testing.for_all_dtypes_combination(
+        names=['in0_type', 'in1_type', 'dtype'], full=True)
+    def test_casting_dtype_unsafe_ignore_warnings(
+            self, in0_type, in1_type, dtype):
+        self.check_casting_dtype_unsafe_ignore_warnings(
+            in0_type, in1_type, dtype
+        )
+
+
+class TestArithmeticModf:
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_modf(self, xp, dtype):
+        a = xp.array([-2.5, -1.5, -0.5, 0, 0.5, 1.5, 2.5], dtype=dtype)
+        b, c = xp.modf(a)
+        d = xp.empty((2, 7), dtype=dtype)
+        d[0] = b
+        d[1] = c
+        return d
+
+
+@testing.parameterize(*testing.product({
+    'xp': [numpy, cupy],
+    'shape': [(3, 2), (), (3, 0, 2)]
+}))
+class TestBoolSubtract:
+
+    def test_bool_subtract(self):
+        xp = self.xp
+        shape = self.shape
+        x = testing.shaped_random(shape, xp, dtype=numpy.bool_)
+        y = testing.shaped_random(shape, xp, dtype=numpy.bool_)
+        with pytest.raises(TypeError):
+            xp.subtract(x, y)
diff --git a/tests/cupy_tests/math_tests/test_explog.py b/tests/cupy_tests/math_tests/test_explog.py
new file mode 100644
index 0000000..86f3de9
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_explog.py
@@ -0,0 +1,70 @@
+import numpy
+import pytest
+
+from cupy import testing
+
+
+class TestExplog:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary(self, name, xp, dtype, no_complex=False):
+        if no_complex:
+            if numpy.dtype(dtype).kind == 'c':
+                return xp.array(True)
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_binary(self, name, xp, dtype, no_complex=False):
+        if no_complex:
+            if numpy.dtype(dtype).kind == 'c':
+                return xp.array(True)
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a, b)
+
+    def test_exp(self):
+        self.check_unary('exp')
+
+    def test_expm1(self):
+        self.check_unary('expm1')
+
+    def test_exp2(self):
+        self.check_unary('exp2')
+
+    def test_log(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_unary('log')
+
+    def test_log10(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_unary('log10')
+
+    def test_log2(self):
+        with numpy.errstate(divide='ignore'):
+            self.check_unary('log2')
+
+    def test_log1p(self):
+        self.check_unary('log1p')
+
+    def test_logaddexp(self):
+        self.check_binary('logaddexp', no_complex=True)
+
+    @pytest.mark.parametrize('val', [numpy.inf, -numpy.inf])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_logaddexp_infinities(self, xp, dtype, val):
+        a = xp.full((2, 3), val, dtype=dtype)
+        return xp.logaddexp(a, a)
+
+    def test_logaddexp2(self):
+        self.check_binary('logaddexp2', no_complex=True)
+
+    @pytest.mark.parametrize('val', [numpy.inf, -numpy.inf])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_logaddexp2_infinities(self, xp, dtype, val):
+        a = xp.full((2, 3), val, dtype=dtype)
+        return xp.logaddexp2(a, a)
diff --git a/tests/cupy_tests/math_tests/test_floating.py b/tests/cupy_tests/math_tests/test_floating.py
new file mode 100644
index 0000000..b2223d0
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_floating.py
@@ -0,0 +1,67 @@
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+class TestFloating(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_signbit(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.signbit(a)
+
+    @testing.for_all_dtypes_combination(
+        ('dtype_a', 'dtype_b'), no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_copysign_combination(self, xp, dtype_a, dtype_b):
+        a = testing.shaped_arange((2, 3), xp, dtype_a)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype_b)
+        return xp.copysign(a, b)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_copysign_float(self, xp, dtype):
+        a = xp.array([-xp.inf, -3, -0.0, 0, 3, xp.inf], dtype=dtype)[:, None]
+        b = xp.array([-xp.inf, -3, -0.0, 0, 3, xp.inf], dtype=dtype)[None, :]
+        return xp.copysign(a, b)
+
+    @testing.for_float_dtypes(name='ftype')
+    @testing.for_dtypes(['i', 'l'], name='itype')
+    @testing.numpy_cupy_array_equal()
+    def test_ldexp(self, xp, ftype, itype):
+        a = xp.array([-3, -2, -1, 0, 1, 2, 3], dtype=ftype)
+        b = xp.array([-3, -2, -1, 0, 1, 2, 3], dtype=itype)
+        return xp.ldexp(a, b)
+
+    @testing.for_float_dtypes()
+    def test_frexp(self, dtype):
+        numpy_a = numpy.array([-300, -20, -10, -1, 0, 1, 10, 20, 300],
+                              dtype=dtype)
+        numpy_b, numpy_c = numpy.frexp(numpy_a)
+
+        cupy_a = cupy.array(numpy_a)
+        cupy_b, cupy_c = cupy.frexp(cupy_a)
+
+        testing.assert_array_equal(cupy_b, numpy_b)
+        testing.assert_array_equal(cupy_c, numpy_c)
+
+    @testing.for_all_dtypes_combination(
+        ('dtype_a', 'dtype_b'), no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_nextafter_combination(self, xp, dtype_a, dtype_b):
+        a = testing.shaped_arange((2, 3), xp, dtype_a)
+        # skip 0 because cupy (may) handle denormals differently (-ftz=true)
+        a[a == 0] = 1
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype_b)
+        return xp.nextafter(a, b)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_nextafter_float(self, xp, dtype):
+        a = xp.array([-5, -3, 3, 5], dtype=dtype)[:, None]
+        b = xp.array([-xp.inf, -4, 0, 4, xp.inf], dtype=dtype)[None, :]
+        return xp.nextafter(a, b)
diff --git a/tests/cupy_tests/math_tests/test_hyperbolic.py b/tests/cupy_tests/math_tests/test_hyperbolic.py
new file mode 100644
index 0000000..146cf53
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_hyperbolic.py
@@ -0,0 +1,39 @@
+import unittest
+
+from cupy import testing
+
+
+class TestHyperbolic(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary(self, name, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_unit(self, name, xp, dtype):
+        a = xp.array([0.2, 0.4, 0.6, 0.8], dtype=dtype)
+        return getattr(xp, name)(a)
+
+    def test_sinh(self):
+        self.check_unary('sinh')
+
+    def test_cosh(self):
+        self.check_unary('cosh')
+
+    def test_tanh(self):
+        self.check_unary('tanh')
+
+    def test_arcsinh(self):
+        self.check_unary('arcsinh')
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_arccosh(self, xp, dtype):
+        a = xp.array([1, 2, 3], dtype=dtype)
+        return xp.arccosh(a)
+
+    def test_arctanh(self):
+        self.check_unary_unit('arctanh')
diff --git a/tests/cupy_tests/math_tests/test_matmul.py b/tests/cupy_tests/math_tests/test_matmul.py
new file mode 100644
index 0000000..ef6509a
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_matmul.py
@@ -0,0 +1,400 @@
+import operator
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy._core import _routines_linalg as _linalg
+from cupy import testing
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape_pair': [
+            # dot test
+            ((3, 2), (2, 4)),
+            ((3, 0), (0, 4)),
+            ((0, 2), (2, 4)),
+            ((3, 2), (2, 0)),
+            ((2,), (2, 4)),
+            ((0,), (0, 4)),
+            ((3, 2), (2,)),
+            ((3, 0), (0,)),
+            ((2,), (2,)),
+            ((0,), (0,)),
+            # matmul test
+            ((5, 3, 2), (5, 2, 4)),
+            ((0, 3, 2), (0, 2, 4)),
+            ((5, 3, 2), (2, 4)),
+            ((0, 3, 2), (2, 4)),
+            ((3, 2), (5, 2, 4)),
+            ((3, 2), (0, 2, 4)),
+            ((5, 3, 2), (1, 2, 4)),
+            ((0, 3, 2), (1, 2, 4)),
+            ((1, 3, 2), (5, 2, 4)),
+            ((1, 3, 2), (0, 2, 4)),
+            ((5, 3, 2), (2,)),
+            ((5, 3, 0), (0,)),
+            ((2,), (5, 2, 4)),
+            ((0,), (5, 0, 4)),
+            ((2, 2, 3, 2), (2, 2, 2, 4)),
+            ((5, 0, 3, 2), (5, 0, 2, 4)),
+            ((6, 5, 3, 2), (2, 4)),
+            ((5, 0, 3, 2), (2, 4)),
+            ((3, 2), (6, 5, 2, 4)),
+            ((3, 2), (5, 0, 2, 4)),
+            ((1, 5, 3, 2), (6, 1, 2, 4)),
+            ((1, 0, 3, 2), (6, 1, 2, 4)),
+            ((6, 1, 3, 2), (1, 5, 2, 4)),
+            ((6, 1, 3, 2), (1, 0, 2, 4)),
+            ((6, 5, 3, 2), (2,)),
+            ((6, 5, 3, 0), (0,)),
+            ((2,), (6, 5, 2, 4)),
+            ((0,), (6, 5, 0, 4)),
+            ((1, 3, 3), (10, 1, 3, 1)),
+        ],
+    }))
+class TestMatmul(unittest.TestCase):
+
+    @testing.for_all_dtypes(name='dtype1')
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_operator_matmul(self, xp, dtype1, dtype2):
+        x1 = testing.shaped_arange(self.shape_pair[0], xp, dtype1)
+        x2 = testing.shaped_arange(self.shape_pair[1], xp, dtype2)
+        return operator.matmul(x1, x2)
+
+    @testing.for_all_dtypes(name='dtype1')
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_cupy_matmul(self, xp, dtype1, dtype2):
+        x1 = testing.shaped_arange(self.shape_pair[0], xp, dtype1)
+        x2 = testing.shaped_arange(self.shape_pair[1], xp, dtype2)
+        return xp.matmul(x1, x2)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape_pair': [
+            # dot test
+            ((2, 3), (3, 4), (2, 4)),
+            # ((0,), (0,), (0,)),  # TODO: fix GUFunc bug?
+            # matmul test
+            ((5, 3, 2), (5, 2, 4), (5, 3, 4)),
+            ((0, 3, 2), (0, 2, 4), (0, 3, 4)),
+        ],
+    }))
+class TestMatmulOut(unittest.TestCase):
+
+    @testing.for_all_dtypes(name='dtype1')
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_allclose(
+        rtol=1e-3, atol=1e-3,  # required for uint8
+        accept_error=TypeError)
+    def test_cupy_matmul_noncontiguous(self, xp, dtype1, dtype2):
+        x1 = testing.shaped_arange(self.shape_pair[0], xp, dtype1)
+        x2 = testing.shaped_arange(self.shape_pair[1], xp, dtype2)
+        out = xp.zeros(self.shape_pair[2], dtype1)[::-1]
+        ret = xp.matmul(x1, x2, out=out)
+        # TODO: Fix GUFunc bug
+        # assert ret is out
+        assert xp.allclose(ret, out)
+        return ret
+
+    @testing.for_all_dtypes(name='dtype1')
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_cupy_matmul_out_cast(self, xp, dtype1, dtype2):
+        x1 = testing.shaped_arange(self.shape_pair[0], xp, dtype1)
+        x2 = testing.shaped_arange(self.shape_pair[1], xp, dtype2)
+        out = xp.zeros(self.shape_pair[2], bool)
+        ret = xp.matmul(x1, x2, out=out, casting='unsafe')
+        # TODO: Fix GUFunc bug
+        # assert ret is out
+        assert xp.allclose(ret, out)
+        return ret
+
+
+class TestMatmulOutOverlap:
+
+    @pytest.mark.parametrize('shape', [
+        (900, 900),
+        (2, 600, 600),
+    ])
+    @testing.for_dtypes([numpy.int32, numpy.float64])
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5)
+    def test_overlap_both(self, xp, dtype, shape):
+        a = xp.ones(shape, dtype)
+        return xp.matmul(a, a, out=a)
+
+
+class TestMatmulStrides:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_relaxed_c_contiguous_input(self, xp, dtype):
+        x1 = testing.shaped_arange((2, 2, 3), xp, dtype)[:, None, :, :]
+        x2 = testing.shaped_arange((2, 1, 3, 1), xp, dtype)
+        return x1 @ x2
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape_pair': [
+            ((6, 5, 3, 2), (6, 5, 2, 4)),
+            ((6, 5, 3, 2), (6, 1, 2, 4)),
+            ((6, 5, 3, 2), (1, 5, 2, 4)),
+            ((6, 5, 3, 2), (1, 1, 2, 4)),
+            ((6, 1, 3, 2), (6, 5, 2, 4)),
+            ((1, 5, 3, 2), (6, 5, 2, 4)),
+            ((1, 1, 3, 2), (6, 5, 2, 4)),
+            ((3, 2), (6, 5, 2, 4)),
+            ((6, 5, 3, 2), (2, 4)),
+            ((2,), (6, 5, 2, 4)),
+            ((6, 5, 3, 2), (2,)),
+        ],
+    }))
+class TestMatmulLarge(unittest.TestCase):
+
+    # Avoid overflow
+    skip_dtypes = {
+        (numpy.int8, numpy.uint8),
+        (numpy.int8, numpy.int16),
+        (numpy.int8, numpy.float16),
+        (numpy.uint8, numpy.uint8),
+        (numpy.uint8, numpy.int16),
+        (numpy.uint8, numpy.uint16),
+        (numpy.int16, numpy.int16),
+        (numpy.uint16, numpy.uint16),
+    }
+
+    @testing.for_all_dtypes(name='dtype1')
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_operator_matmul(self, xp, dtype1, dtype2):
+        if ((dtype1, dtype2) in self.skip_dtypes or
+                (dtype2, dtype1) in self.skip_dtypes):
+            pytest.skip()
+        x1 = testing.shaped_random(self.shape_pair[0], xp, dtype1)
+        x2 = testing.shaped_random(self.shape_pair[1], xp, dtype2)
+        return operator.matmul(x1, x2)
+
+    @testing.for_all_dtypes(name='dtype1')
+    @testing.for_all_dtypes(name='dtype2')
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_cupy_matmul(self, xp, dtype1, dtype2):
+        if ((dtype1, dtype2) in self.skip_dtypes or
+                (dtype2, dtype1) in self.skip_dtypes):
+            pytest.skip()
+        shape1, shape2 = self.shape_pair
+        x1 = testing.shaped_random(shape1, xp, dtype1)
+        x2 = testing.shaped_random(shape2, xp, dtype2)
+        return xp.matmul(x1, x2)
+
+
+@pytest.mark.parametrize('shape1,shape2', [
+    ((256, 256, 3, 2), (256, 256, 2, 4)),
+    ((256, 256, 3, 2), (2, 4)),
+    ((3, 2), (256, 256, 2, 4))
+])
+class TestMatmulIntegralLargeBatch:
+
+    @testing.for_int_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_operator_matmul(self, xp, dtype, shape1, shape2):
+        x1 = testing.shaped_random(shape1, xp, dtype)
+        x2 = testing.shaped_random(shape2, xp, dtype)
+        return operator.matmul(x1, x2)
+
+    @testing.for_int_dtypes(name='dtype')
+    @testing.numpy_cupy_array_equal()
+    def test_cupy_matmul(self, xp, dtype, shape1, shape2):
+        x1 = testing.shaped_random(shape1, xp, dtype)
+        x2 = testing.shaped_random(shape2, xp, dtype)
+        return xp.matmul(x1, x2)
+
+
+class TestMatmulOverflow(unittest.TestCase):
+
+    @testing.for_int_dtypes(name='dtype', no_bool=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_overflow(self, xp, dtype):
+        value = numpy.iinfo(dtype).max
+        a = xp.array([value - 10]).astype(dtype)
+        b = xp.array([value - 10]).astype(dtype)
+        return xp.matmul(a, b)
+
+
+class _TestMatmulComputeTypes(unittest.TestCase):
+
+    def setUp(self):
+        self.old_compute_type = cupy._core.get_compute_type(self.dtype)
+        cupy._core.set_compute_type(self.dtype, self.compute_type)
+
+    def tearDown(self):
+        cupy._core.set_compute_type(self.dtype, self.old_compute_type)
+
+    def make_x1_x2(self, xp, shapes, dtypes):
+        x1 = testing.shaped_random(shapes[0], xp, dtypes[0])
+        x2 = testing.shaped_random(shapes[1], xp, dtypes[1])
+        return x1, x2
+
+
+@testing.parameterize(
+    *testing.product({
+        'compute_type': [
+            _linalg.COMPUTE_TYPE_DEFAULT,
+            _linalg.COMPUTE_TYPE_PEDANTIC,
+        ],
+        'shape_pair': [
+            ((32, 64), (64, 96)),
+            ((64, 96), (96, 32)),
+            ((96, 32), (32, 64)),
+        ],
+    }))
+class TestMatmulFp16ComputeTypes(_TestMatmulComputeTypes):
+    dtype = numpy.float16
+
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)
+    def test_operator_matmul(self, xp):
+        x1, x2 = self.make_x1_x2(xp, self.shape_pair, (self.dtype, self.dtype))
+        return operator.matmul(x1, x2)
+
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)
+    def test_cupy_matmul(self, xp):
+        x1, x2 = self.make_x1_x2(xp, self.shape_pair, (self.dtype, self.dtype))
+        return xp.matmul(x1, x2)
+
+
+@testing.parameterize(
+    *testing.product({
+        'compute_type': [
+            _linalg.COMPUTE_TYPE_DEFAULT,
+            _linalg.COMPUTE_TYPE_PEDANTIC,
+            _linalg.COMPUTE_TYPE_TF32,
+        ],
+        'shape_pair': [
+            ((100, 200), (200, 300)),
+            ((200, 300), (300, 100)),
+            ((300, 100), (100, 200)),
+        ],
+        'dtype_pair': [
+            (numpy.float16, numpy.float32),
+            (numpy.float32, numpy.float32),
+            (numpy.float16, numpy.complex64),
+            (numpy.float32, numpy.complex64),
+            (numpy.complex64, numpy.complex64),
+        ],
+    }))
+class TestMatmulFp32ComputeTypes(_TestMatmulComputeTypes):
+    dtype = numpy.float32
+
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)
+    def test_operator_matmul(self, xp):
+        x1, x2 = self.make_x1_x2(xp, self.shape_pair, self.dtype_pair)
+        return operator.matmul(x1, x2)
+
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)
+    def test_cupy_matmul(self, xp):
+        x1, x2 = self.make_x1_x2(xp, self.shape_pair, self.dtype_pair)
+        return xp.matmul(x1, x2)
+
+
+@testing.parameterize(
+    *testing.product({
+        'compute_type': [
+            _linalg.COMPUTE_TYPE_DEFAULT,
+            _linalg.COMPUTE_TYPE_PEDANTIC,
+        ],
+        'shape_pair': [
+            ((100, 200), (200, 300)),
+            ((200, 300), (300, 100)),
+            ((300, 100), (100, 200)),
+        ],
+        'dtype_pair': [
+            (numpy.float32, numpy.float64),
+            (numpy.float64, numpy.float64),
+            (numpy.float32, numpy.complex128),
+            (numpy.float64, numpy.complex128),
+            (numpy.complex64, numpy.complex128),
+            (numpy.complex128, numpy.complex128),
+        ],
+    }))
+class TestMatmulFp64ComputeTypes(_TestMatmulComputeTypes):
+    dtype = numpy.float64
+
+    @testing.numpy_cupy_allclose()
+    def test_operator_matmul(self, xp):
+        x1, x2 = self.make_x1_x2(xp, self.shape_pair, self.dtype_pair)
+        return operator.matmul(x1, x2)
+
+    @testing.numpy_cupy_allclose()
+    def test_cupy_matmul(self, xp):
+        x1, x2 = self.make_x1_x2(xp, self.shape_pair, self.dtype_pair)
+        return xp.matmul(x1, x2)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape_pair': [
+            ((5, 3, 1), (3, 1, 4)),
+            ((3, 2, 3), (3, 2, 4)),
+            ((3, 2), ()),
+            ((), (3, 2)),
+            ((), ()),
+            ((3, 2), (1,)),
+            ((0, 2), (3, 0)),
+            ((0, 1, 1), (2, 1, 1)),
+        ],
+    }))
+class TestMatmulInvalidShape(unittest.TestCase):
+
+    def test_invalid_shape(self):
+        for xp in (numpy, cupy):
+            shape1, shape2 = self.shape_pair
+            x1 = testing.shaped_arange(shape1, xp, numpy.float32)
+            x2 = testing.shaped_arange(shape2, xp, numpy.float32)
+            with pytest.raises(ValueError):
+                xp.matmul(x1, x2)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shapes_axes': [
+            (((2, 5, 3, 2, 3, 4),  (3, 5, 1, 1, 1, 4), (5, 5, 2, 2, 3, 4)),
+             [(1, 2), (0, 1), (0, 1)]),
+            (((2, 5, 3, 2, 3, 4),  (2, 5, 3, 1, 4, 1), (3, 1, 2, 5, 3, 2)),
+             [(-2, -1), (-2, -1), (0, 1)]),
+            (((3, 2, 4, 4), (4, 4, 3, 2), (4, 4, 3, 3)),
+             [(0, 1), (-1, -2), (-2, -1)]),
+            (((3, 2, 4, 4), (2, 3, 4, 4), (4, 3, 3, 4)),
+             [(0, 1), (0, 1), (1, 2)]),
+        ],
+    }))
+class TestMatmulAxes(unittest.TestCase):
+
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_cupy_matmul_axes(self, xp):
+        x1 = testing.shaped_arange(self.shapes_axes[0][0], xp)
+        x2 = testing.shaped_arange(self.shapes_axes[0][1], xp)
+        return xp.matmul(x1, x2, axes=self.shapes_axes[1])
+
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-3)  # required for uint8
+    def test_cupy_matmul_axes_out(self, xp):
+        x1 = testing.shaped_arange(self.shapes_axes[0][0], xp)
+        x2 = testing.shaped_arange(self.shapes_axes[0][1], xp)
+        out = xp.zeros(self.shapes_axes[0][2])
+        xp.matmul(x1, x2, axes=self.shapes_axes[1], out=out)
+        return out
+
+
+class TestMatmulDispatch(unittest.TestCase):
+
+    def test_matmul_dispatch(self):
+        x1 = testing.shaped_arange((2, 10, 5), cupy)
+        x2 = testing.shaped_arange((10, 2, 5), cupy)
+        o_np = numpy.matmul(x1, x2, axes=[(0, 1), (0, 1), (0, 1)])
+        assert isinstance(o_np, cupy.ndarray)
+        o_cp = cupy.matmul(x1, x2, axes=[(0, 1), (0, 1), (0, 1)])
+        testing.assert_allclose(o_np, o_cp)
diff --git a/tests/cupy_tests/math_tests/test_misc.py b/tests/cupy_tests/math_tests/test_misc.py
new file mode 100644
index 0000000..01c2d27
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_misc.py
@@ -0,0 +1,537 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestMisc:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary(self, name, xp, dtype, no_bool=False):
+        if no_bool and numpy.dtype(dtype).char == '?':
+            return numpy.int_(0)
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_binary(self, name, xp, dtype, no_bool=False):
+        if no_bool and numpy.dtype(dtype).char == '?':
+            return numpy.int_(0)
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a, b)
+
+    @testing.for_dtypes(['?', 'b', 'h', 'i', 'q', 'e', 'f', 'd', 'F', 'D'])
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_negative(self, name, xp, dtype, no_bool=False):
+        if no_bool and numpy.dtype(dtype).char == '?':
+            return numpy.int_(0)
+        a = xp.array([-3, -2, -1, 1, 2, 3], dtype=dtype)
+        if numpy.dtype(dtype).kind == 'c':
+            a += (a * 1j).astype(dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_dtypes(['e', 'f', 'd', 'F', 'D'])
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_inf(self, name, xp, dtype, **kwargs):
+        inf = numpy.inf
+        if numpy.dtype(dtype).kind != 'c':
+            a = xp.array([0, -1, 1, -inf, inf], dtype=dtype)
+        else:
+            a = xp.array([complex(x, y)
+                          for x in [0, -1, 1, -inf, inf]
+                          for y in [0, -1, 1, -inf, inf]],
+                         dtype=dtype)
+        return getattr(xp, name)(a, **kwargs)
+
+    @testing.for_dtypes(['e', 'f', 'd', 'F', 'D'])
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_nan(self, name, xp, dtype, **kwargs):
+        nan = numpy.nan
+        if numpy.dtype(dtype).kind != 'c':
+            a = xp.array([0, -1, 1, -nan, nan], dtype=dtype)
+        else:
+            a = xp.array([complex(x, y)
+                          for x in [0, -1, 1, -nan, nan]
+                          for y in [0, -1, 1, -nan, nan]],
+                         dtype=dtype)
+        return getattr(xp, name)(a, **kwargs)
+
+    @testing.for_dtypes(['e', 'f', 'd', 'F', 'D'])
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_inf_nan(self, name, xp, dtype):
+        inf = numpy.inf
+        nan = numpy.nan
+        if numpy.dtype(dtype).kind != 'c':
+            a = xp.array([0, -1, 1, -inf, inf, -nan, nan], dtype=dtype)
+        else:
+            a = xp.array([complex(x, y)
+                          for x in [0, -1, 1, -inf, inf, -nan, nan]
+                          for y in [0, -1, 1, -inf, inf, -nan, nan]],
+                         dtype=dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_dtypes(['e', 'f', 'd', 'F', 'D'])
+    @testing.numpy_cupy_array_equal()
+    def check_binary_nan(self, name, xp, dtype):
+        a = xp.array([-3, numpy.NAN, -1, numpy.NAN, 0, numpy.NAN, 2],
+                     dtype=dtype)
+        b = xp.array([numpy.NAN, numpy.NAN, 1, 0, numpy.NAN, -1, -2],
+                     dtype=dtype)
+        return getattr(xp, name)(a, b)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_clip1(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.clip(3, 13)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_clip3(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.clip(3, 13)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_clip_min_none(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.clip(None, 3)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_clip_max_none(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.clip(3, None)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    def test_clip_min_max_none(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                a.clip(None, None)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_external_clip1(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.clip(a, 3, 13)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_external_clip2(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.clip(a, 3, 13)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_external_clip3(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.clip(a, 8, 4)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    def test_external_clip4(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.clip(a, 3)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_clip2(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        a_min = xp.array([3, 4, 5, 6], dtype=dtype)
+        a_max = xp.array([[10], [9], [8]], dtype=dtype)
+        return a.clip(a_min, a_max)
+
+    def test_sqrt(self):
+        self.check_unary('sqrt')
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_cbrt(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.cbrt(a)
+
+    def test_square(self):
+        self.check_unary('square')
+
+    def test_absolute(self):
+        self.check_unary('absolute')
+
+    def test_absolute_negative(self):
+        self.check_unary_negative('absolute')
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_fabs(self, xp, dtype):
+        a = xp.array([2, 3, 4], dtype=dtype)
+        return xp.fabs(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_fabs_negative(self, xp, dtype):
+        a = xp.array([-2.0, -4.0, 0.0, 4.0], dtype=dtype)
+        return xp.fabs(a)
+
+    def test_sign(self):
+        self.check_unary('sign', no_bool=True)
+
+    def test_sign_negative(self):
+        self.check_unary_negative('sign', no_bool=True)
+
+    def test_maximum(self):
+        self.check_binary('maximum')
+
+    def test_maximum_nan(self):
+        self.check_binary_nan('maximum')
+
+    def test_minimum(self):
+        self.check_binary('minimum')
+
+    def test_minimum_nan(self):
+        self.check_binary_nan('minimum')
+
+    def test_fmax(self):
+        self.check_binary('fmax')
+
+    def test_fmax_nan(self):
+        self.check_binary_nan('fmax')
+
+    def test_fmin(self):
+        self.check_binary('fmin')
+
+    def test_fmin_nan(self):
+        self.check_binary_nan('fmin')
+
+    def test_nan_to_num(self):
+        self.check_unary('nan_to_num')
+
+    def test_nan_to_num_negative(self):
+        self.check_unary_negative('nan_to_num')
+
+    def test_nan_to_num_for_old_numpy(self):
+        self.check_unary('nan_to_num', no_bool=True)
+
+    def test_nan_to_num_negative_for_old_numpy(self):
+        self.check_unary_negative('nan_to_num', no_bool=True)
+
+    def test_nan_to_num_inf(self):
+        self.check_unary_inf('nan_to_num')
+
+    def test_nan_to_num_nan(self):
+        self.check_unary_nan('nan_to_num')
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_nan_to_num_scalar_nan(self, xp):
+        return xp.nan_to_num(xp.nan)
+
+    def test_nan_to_num_inf_nan(self):
+        self.check_unary_inf_nan('nan_to_num')
+
+    def test_nan_to_num_nan_arg(self):
+        self.check_unary_nan('nan_to_num', nan=1.0)
+
+    def test_nan_to_num_inf_arg(self):
+        self.check_unary_inf('nan_to_num', posinf=1.0, neginf=-1.0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_nan_to_num_copy(self, xp):
+        x = xp.asarray([0, 1, xp.nan, 4], dtype=xp.float64)
+        y = xp.nan_to_num(x, copy=True)
+        assert x is not y
+        return y
+
+    @testing.numpy_cupy_array_equal()
+    def test_nan_to_num_inplace(self, xp):
+        x = xp.asarray([0, 1, xp.nan, 4], dtype=xp.float64)
+        y = xp.nan_to_num(x, copy=False)
+        assert x is y
+        return y
+
+    @pytest.mark.parametrize('kwarg', ['nan', 'posinf', 'neginf'])
+    def test_nan_to_num_broadcast(self, kwarg):
+        for xp in (numpy, cupy):
+            x = xp.asarray([0, 1, xp.nan, 4], dtype=xp.float64)
+            y = xp.zeros((2, 4), dtype=xp.float64)
+            with pytest.raises(ValueError):
+                xp.nan_to_num(x, **{kwarg: y})
+            with pytest.raises(ValueError):
+                xp.nan_to_num(0.0, **{kwarg: y})
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_real_if_close_real_dtypes(self, xp, dtype):
+        x = testing.shaped_random((10,), xp, dtype)
+        return xp.real_if_close(x)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_real_if_close_with_tol_real_dtypes(self, xp, dtype):
+        x = testing.shaped_random((10,), xp, dtype)
+        return xp.real_if_close(x, tol=1e-6)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_real_if_close_true(self, xp, dtype):
+        dtype = numpy.dtype(dtype).char.lower()
+        tol = numpy.finfo(dtype).eps * 90
+        x = testing.shaped_random((10,), xp, dtype) + tol * 1j
+        out = xp.real_if_close(x)
+        assert x.dtype != out.dtype
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_real_if_close_false(self, xp, dtype):
+        dtype = numpy.dtype(dtype).char.lower()
+        tol = numpy.finfo(dtype).eps * 110
+        x = testing.shaped_random((10,), xp, dtype) + tol * 1j
+        out = xp.real_if_close(x)
+        assert x.dtype == out.dtype
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_real_if_close_with_integer_tol_true(self, xp, dtype):
+        dtype = numpy.dtype(dtype).char.lower()
+        tol = numpy.finfo(dtype).eps * 140
+        x = testing.shaped_random((10,), xp, dtype) + tol * 1j
+        out = xp.real_if_close(x, tol=150)
+        assert x.dtype != out.dtype
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_real_if_close_with_integer_tol_false(self, xp, dtype):
+        dtype = numpy.dtype(dtype).char.lower()
+        tol = numpy.finfo(dtype).eps * 50
+        x = testing.shaped_random((10,), xp, dtype) + tol * 1j
+        out = xp.real_if_close(x, tol=30)
+        assert x.dtype == out.dtype
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_real_if_close_with_float_tol_true(self, xp, dtype):
+        dtype = numpy.dtype(dtype).char.lower()
+        x = testing.shaped_random((10,), xp, dtype) + 3e-4j
+        out = xp.real_if_close(x, tol=1e-3)
+        assert x.dtype != out.dtype
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_real_if_close_with_float_tol_false(self, xp, dtype):
+        dtype = numpy.dtype(dtype).char.lower()
+        x = testing.shaped_random((10,), xp, dtype) + 3e-3j
+        out = xp.real_if_close(x, tol=1e-3)
+        assert x.dtype == out.dtype
+        return out
+
+    @testing.for_all_dtypes(name='dtype_x', no_bool=True, no_complex=True)
+    @testing.for_all_dtypes(name='dtype_y', no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        return xp.interp(x, fx, fy)
+
+    @testing.for_all_dtypes(name='dtype_x', no_bool=True, no_complex=True)
+    @testing.for_all_dtypes(name='dtype_y', no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_period(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        return xp.interp(x, fx, fy, period=5)
+
+    @testing.for_all_dtypes(name='dtype_x', no_bool=True, no_complex=True)
+    @testing.for_all_dtypes(name='dtype_y', no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_left_right(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        left = 10
+        right = 20
+        return xp.interp(x, fx, fy, left, right)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_all_dtypes(name='dtype_x', no_bool=True, no_complex=True)
+    @testing.for_dtypes('efdFD', name='dtype_y')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_nan_fy(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        fy[0] = fy[2] = fy[-1] = numpy.nan
+        return xp.interp(x, fx, fy)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_float_dtypes(name='dtype_x')
+    @testing.for_dtypes('efdFD', name='dtype_y')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_nan_fx(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        fx[-1] = numpy.nan  # x and fx must remain sorted (NaNs are the last)
+        return xp.interp(x, fx, fy)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_float_dtypes(name='dtype_x')
+    @testing.for_dtypes('efdFD', name='dtype_y')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_nan_x(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        x[-1] = numpy.nan  # x and fx must remain sorted (NaNs are the last)
+        return xp.interp(x, fx, fy)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_all_dtypes(name='dtype_x', no_bool=True, no_complex=True)
+    @testing.for_dtypes('efdFD', name='dtype_y')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_inf_fy(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        fy[0] = fy[2] = fy[-1] = numpy.inf
+        return xp.interp(x, fx, fy)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_float_dtypes(name='dtype_x')
+    @testing.for_dtypes('efdFD', name='dtype_y')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_inf_fx(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        fx[-1] = numpy.inf  # x and fx must remain sorted
+        return xp.interp(x, fx, fy)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_float_dtypes(name='dtype_x')
+    @testing.for_dtypes('efdFD', name='dtype_y')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_inf_x(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([1, 3, 5, 7, 9], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        x[-1] = numpy.inf  # x and fx must remain sorted
+        return xp.interp(x, fx, fy)
+
+    @testing.for_all_dtypes(name='dtype_x', no_bool=True, no_complex=True)
+    @testing.for_all_dtypes(name='dtype_y', no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_size1(self, xp, dtype_y, dtype_x):
+        # interpolate at points on and outside the boundaries
+        x = xp.asarray([0, 1, 2, 4, 6, 8, 9, 10], dtype=dtype_x)
+        fx = xp.asarray([5], dtype=dtype_x)
+        fy = xp.sin(fx).astype(dtype_y)
+        left = 10
+        right = 20
+        return xp.interp(x, fx, fy, left, right)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_float_dtypes(name='dtype_x')
+    @testing.for_dtypes('efdFD', name='dtype_y')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_interp_inf_to_nan(self, xp, dtype_y, dtype_x):
+        # from NumPy's test_non_finite_inf
+        x = xp.asarray([0.5], dtype=dtype_x)
+        fx = xp.asarray([-numpy.inf, numpy.inf], dtype=dtype_x)
+        fy = xp.asarray([0, 10], dtype=dtype_y)
+        return xp.interp(x, fx, fy)
+
+    @testing.for_all_dtypes(name='dtype_2', no_bool=True, no_complex=True)
+    @testing.for_all_dtypes(name='dtype_1', no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_heaviside(self, xp, dtype_1, dtype_2):
+        x = testing.shaped_random((10, ), xp, dtype_1)
+        h = xp.asarray([10], dtype=dtype_2)
+        return xp.heaviside(x, h)
+
+    @testing.for_all_dtypes(name='dtype_2', no_bool=True, no_complex=True)
+    @testing.for_float_dtypes(name='dtype_1')
+    @testing.numpy_cupy_array_equal()
+    def test_heaviside_nan_inf(self, xp, dtype_1, dtype_2):
+        x = xp.asarray([-2., 0., 3., xp.nan, xp.inf, -xp.inf], dtype=dtype_1)
+        h = xp.asarray([10], dtype=dtype_2)
+        return xp.heaviside(x, h)
+
+
+@testing.parameterize(*testing.product({
+    'mode': ['valid', 'same', 'full'],
+    'shape1': [(), (5,), (6,), (20,), (21,)],
+    'shape2': [(), (5,), (6,), (20,), (21,)],
+}))
+class TestConvolveShapeCombination:
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3)
+    def test_convolve(self, xp, dtype):
+        a = testing.shaped_arange(self.shape1, xp, dtype)
+        b = testing.shaped_arange(self.shape2, xp, dtype)
+        return xp.convolve(a, b, mode=self.mode)
+
+
+@pytest.mark.parametrize('mode', ['valid', 'same', 'full'])
+class TestConvolve:
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_convolve_non_contiguous(self, xp, dtype, mode):
+        a = testing.shaped_arange((300,), xp, dtype)
+        b = testing.shaped_arange((100,), xp, dtype)
+        return xp.convolve(a[::200], b[10::70], mode=mode)
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=5e-4)
+    def test_convolve_large_non_contiguous(self, xp, dtype, mode):
+        a = testing.shaped_arange((10000,), xp, dtype)
+        b = testing.shaped_arange((100,), xp, dtype)
+        return xp.convolve(a[200::], b[10::70], mode=mode)
+
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'])
+    @testing.numpy_cupy_allclose(rtol=1e-2)
+    def test_convolve_diff_types(self, xp, dtype1, dtype2, mode):
+        a = testing.shaped_random((200,), xp, dtype1)
+        b = testing.shaped_random((100,), xp, dtype2)
+        return xp.convolve(a, b, mode=mode)
+
+
+@testing.parameterize(*testing.product({
+    'mode': ['valid', 'same', 'full']
+}))
+class TestConvolveInvalid:
+
+    @testing.for_all_dtypes()
+    def test_convolve_empty(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.zeros((0,), dtype)
+            with pytest.raises(ValueError):
+                xp.convolve(a, a, mode=self.mode)
+
+    @testing.for_all_dtypes()
+    def test_convolve_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp, dtype)
+            b = testing.shaped_arange((10, 5), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.convolve(a, b, mode=self.mode)
diff --git a/tests/cupy_tests/math_tests/test_rational.py b/tests/cupy_tests/math_tests/test_rational.py
new file mode 100644
index 0000000..3210fe9
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_rational.py
@@ -0,0 +1,37 @@
+import unittest
+
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestRational(unittest.TestCase):
+
+    @testing.for_dtypes(['?', 'e', 'f', 'd', 'F', 'D'])
+    def test_gcd_dtype_check(self, dtype):
+        a = cupy.random.randint(-10, 10, size=(10, 10)).astype(dtype)
+        b = cupy.random.randint(-10, 10, size=(10, 10)).astype(dtype)
+        with pytest.raises(TypeError):
+            cupy.gcd(a, b)
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_gcd_check_boundary_cases(self, xp, dtype):
+        a = xp.array([0, -10, -5, 10, 410, 1, 6, 33])
+        b = xp.array([0, 5, -10, -5, 20, 51, 6, 42])
+        return xp.gcd(a, b)
+
+    @testing.for_dtypes(['?', 'e', 'f', 'd', 'F', 'D'])
+    def test_lcm_dtype_check(self, dtype):
+        a = cupy.random.randint(-10, 10, size=(10, 10)).astype(dtype)
+        b = cupy.random.randint(-10, 10, size=(10, 10)).astype(dtype)
+        with pytest.raises(TypeError):
+            cupy.lcm(a, b)
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_lcm_check_boundary_cases(self, xp, dtype):
+        a = xp.array([0, -10, -5, 10, 410, 1, 6, 33])
+        b = xp.array([0, 5, -10, -5, 20, 51, 6, 42])
+        return xp.lcm(a, b)
diff --git a/tests/cupy_tests/math_tests/test_rounding.py b/tests/cupy_tests/math_tests/test_rounding.py
new file mode 100644
index 0000000..ce690a0
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_rounding.py
@@ -0,0 +1,165 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestRounding(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary(self, name, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_complex(self, name, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_complex_dtypes()
+    def check_unary_complex_unsupported(self, name, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3), xp, dtype)
+            with pytest.raises(TypeError):
+                getattr(xp, name)(a)
+
+    @testing.for_dtypes(['?', 'b', 'h', 'i', 'q', 'e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_negative(self, name, xp, dtype):
+        a = xp.array([-3, -2, -1, 1, 2, 3], dtype=dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_negative_complex(self, name, xp, dtype):
+        a = xp.array([-3-3j, -2-2j, -1-1j, 1+1j, 2+2j, 3+3j], dtype=dtype)
+        return getattr(xp, name)(a)
+
+    def test_rint(self):
+        self.check_unary('rint')
+        self.check_unary_complex('rint')
+
+    def test_rint_negative(self):
+        self.check_unary_negative('rint')
+        self.check_unary_negative_complex('rint')
+
+    def test_floor(self):
+        self.check_unary('floor')
+        self.check_unary_complex_unsupported('floor')
+
+    def test_ceil(self):
+        self.check_unary('ceil')
+        self.check_unary_complex_unsupported('ceil')
+
+    def test_trunc(self):
+        self.check_unary('trunc')
+        self.check_unary_complex_unsupported('trunc')
+
+    def test_fix(self):
+        self.check_unary('fix')
+        self.check_unary_complex_unsupported('fix')
+
+    def test_around(self):
+        self.check_unary('around')
+        self.check_unary_complex('around')
+
+    @testing.with_requires('numpy<1.25')
+    def test_round_(self):
+        self.check_unary('round_')
+        self.check_unary_complex('round_')
+
+    def test_round(self):
+        self.check_unary('round')
+        self.check_unary_complex('round')
+
+
+@testing.parameterize(*testing.product({
+    'decimals': [-2, -1, 0, 1, 2],
+}))
+class TestRound(unittest.TestCase):
+
+    shape = (20,)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_round(self, xp, dtype):
+        if dtype == numpy.bool_:
+            # avoid cast problem
+            a = testing.shaped_random(self.shape, xp, scale=10, dtype=dtype)
+            return xp.around(a, 0)
+        if dtype == numpy.float16:
+            # avoid accuracy problem
+            a = testing.shaped_random(self.shape, xp, scale=10, dtype=dtype)
+            return xp.around(a, 0)
+        a = testing.shaped_random(self.shape, xp, scale=100, dtype=dtype)
+        return xp.around(a, self.decimals)
+
+    @testing.numpy_cupy_array_equal()
+    def test_round_out(self, xp):
+        a = testing.shaped_random(self.shape, xp, scale=100, dtype='d')
+        out = xp.empty_like(a)
+        xp.around(a, self.decimals, out)
+        return out
+
+
+@testing.parameterize(*testing.product({
+    'decimals': [-100, -99, -90, 0, 90, 99, 100],
+}))
+class TestRoundExtreme(unittest.TestCase):
+
+    shape = (20,)
+
+    @testing.for_dtypes([numpy.float64, numpy.complex128])
+    @testing.numpy_cupy_allclose()
+    def test_round_large(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, scale=1e100, dtype=dtype)
+        return xp.around(a, self.decimals)
+
+    @testing.for_dtypes([numpy.float64, numpy.complex128])
+    @testing.numpy_cupy_allclose()
+    def test_round_small(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, scale=1e-100, dtype=dtype)
+        return xp.around(a, self.decimals)
+
+
+@testing.parameterize(*testing.product({
+    'value': [
+        (14, -1),
+        (15, -1),
+        (16, -1),
+        (14.0, -1),
+        (15.0, -1),
+        (16.0, -1),
+        (1.4, 0),
+        (1.5, 0),
+        (1.6, 0),
+    ]
+}))
+class TestRoundBorder(unittest.TestCase):
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_around_positive1(self, xp):
+        a, decimals = self.value
+        return xp.around(a, decimals)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_around_positive2(self, xp):
+        a, decimals = self.value
+        a = xp.asarray(a)
+        return xp.around(a, decimals)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_around_negative1(self, xp):
+        a, decimals = self.value
+        return xp.around(-a, decimals)
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_around_negative2(self, xp):
+        a, decimals = self.value
+        a = xp.asarray(a)
+        return xp.around(-a, decimals)
diff --git a/tests/cupy_tests/math_tests/test_special.py b/tests/cupy_tests/math_tests/test_special.py
new file mode 100644
index 0000000..5d483db
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_special.py
@@ -0,0 +1,24 @@
+import unittest
+
+import cupy
+from cupy import testing
+
+
+class TestSpecial(unittest.TestCase):
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(rtol=1e-3)
+    def test_i0(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.i0(a)
+
+    @testing.for_dtypes(['e', 'f', 'd', 'F', 'D'])
+    @testing.numpy_cupy_allclose(atol=1e-3)
+    def test_sinc(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype, scale=1)
+        return xp.sinc(a)
+
+    @testing.for_dtypes(['e', 'f', 'd', 'F', 'D'])
+    def test_sinc_zero(self, dtype):
+        a = cupy.sinc(cupy.zeros(1, dtype=dtype))
+        testing.assert_array_equal(a, cupy.ones(1, dtype=dtype))
diff --git a/tests/cupy_tests/math_tests/test_sumprod.py b/tests/cupy_tests/math_tests/test_sumprod.py
new file mode 100644
index 0000000..ff6113b
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_sumprod.py
@@ -0,0 +1,1150 @@
+import math
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+import cupy._core._accelerator as _acc
+import cupy.cuda.cutensor
+from cupy._core import _cub_reduction
+from cupy import testing
+
+
+class TestSumprod:
+
+    @pytest.fixture(autouse=True)
+    def tearDown(self):
+        yield
+        # Free huge memory for slow test
+        cupy.get_default_memory_pool().free_all_blocks()
+        cupy.get_default_pinned_memory_pool().free_all_blocks()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.sum()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_all_keepdims(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.sum(keepdims=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_sum_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.sum(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_all2(self, xp, dtype):
+        a = testing.shaped_arange((20, 30, 40), xp, dtype)
+        return a.sum()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_all_transposed(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype).transpose(2, 0, 1)
+        return a.sum()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_all_transposed2(self, xp, dtype):
+        a = testing.shaped_arange((20, 30, 40), xp, dtype).transpose(2, 0, 1)
+        return a.sum()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.sum(axis=1)
+
+    @testing.slow
+    @testing.numpy_cupy_allclose()
+    def test_sum_axis_huge(self, xp):
+        a = testing.shaped_random((2048, 1, 1024), xp, 'b')
+        a = xp.broadcast_to(a, (2048, 1024, 1024))
+        return a.sum(axis=2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_sum_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.sum(a, axis=1)
+
+    # float16 is omitted, since NumPy's sum on float16 arrays has more error
+    # than CuPy's.
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose()
+    def test_sum_axis2(self, xp, dtype):
+        a = testing.shaped_arange((20, 30, 40), xp, dtype)
+        return a.sum(axis=1)
+
+    def test_sum_axis2_float16(self):
+        # Note that the above test example overflows in float16. We use a
+        # smaller array instead.
+        a = testing.shaped_arange((2, 30, 4), dtype='e')
+        sa = a.sum(axis=1)
+        b = testing.shaped_arange((2, 30, 4), numpy, dtype='f')
+        sb = b.sum(axis=1)
+        testing.assert_allclose(sa, sb.astype('e'))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_sum_axis_transposed(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype).transpose(2, 0, 1)
+        return a.sum(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_sum_axis_transposed2(self, xp, dtype):
+        a = testing.shaped_arange((20, 30, 40), xp, dtype).transpose(2, 0, 1)
+        return a.sum(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_axes(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4, 5), xp, dtype)
+        return a.sum(axis=(1, 3))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_sum_axes2(self, xp, dtype):
+        a = testing.shaped_arange((20, 30, 40, 50), xp, dtype)
+        return a.sum(axis=(1, 3))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_sum_axes3(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4, 5), xp, dtype)
+        return a.sum(axis=(0, 2, 3))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_sum_axes4(self, xp, dtype):
+        a = testing.shaped_arange((20, 30, 40, 50), xp, dtype)
+        return a.sum(axis=(0, 2, 3))
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_empty_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4, 5), xp, dtype)
+        return a.sum(axis=())
+
+    @testing.for_all_dtypes_combination(names=['src_dtype', 'dst_dtype'])
+    @testing.numpy_cupy_allclose()
+    def test_sum_dtype(self, xp, src_dtype, dst_dtype):
+        if not xp.can_cast(src_dtype, dst_dtype):
+            pytest.skip()
+        a = testing.shaped_arange((2, 3, 4), xp, src_dtype)
+        return a.sum(dtype=dst_dtype)
+
+    @testing.for_all_dtypes_combination(names=['src_dtype', 'dst_dtype'])
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-2, 'default': 1e-7})
+    def test_sum_keepdims_and_dtype(self, xp, src_dtype, dst_dtype):
+        if not xp.can_cast(src_dtype, dst_dtype):
+            pytest.skip()
+        a = testing.shaped_arange((2, 3, 4), xp, src_dtype)
+        return a.sum(axis=2, dtype=dst_dtype, keepdims=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_keepdims_multiple_axes(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.sum(axis=(1, 2), keepdims=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_sum_out(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        b = xp.empty((2, 4), dtype=dtype)
+        a.sum(axis=1, out=b)
+        return b
+
+    def test_sum_out_wrong_shape(self):
+        a = testing.shaped_arange((2, 3, 4))
+        b = cupy.empty((2, 3))
+        with pytest.raises(ValueError):
+            a.sum(axis=1, out=b)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_prod_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return a.prod()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_prod_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.prod(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_prod_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.prod(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_prod_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.prod(a, axis=1)
+
+    @testing.for_all_dtypes_combination(names=['src_dtype', 'dst_dtype'])
+    @testing.numpy_cupy_allclose()
+    def test_prod_dtype(self, xp, src_dtype, dst_dtype):
+        if not xp.can_cast(src_dtype, dst_dtype):
+            pytest.skip()
+        a = testing.shaped_arange((2, 3), xp, src_dtype)
+        return a.prod(dtype=dst_dtype)
+
+    @testing.numpy_cupy_allclose()
+    def test_product_alias(self, xp):
+        a = testing.shaped_arange((2, 3), xp, xp.float32)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            return xp.product(a)
+
+
+# This class compares CUB results against NumPy's
+@testing.parameterize(*testing.product({
+    'shape': [(10,), (10, 20), (10, 20, 30), (10, 20, 30, 40)],
+    'order': ('C', 'F'),
+    'backend': ('device', 'block'),
+}))
+@pytest.mark.skipif(
+    not cupy.cuda.cub.available, reason='The CUB routine is not enabled')
+class TestCubReduction:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        old_routine_accelerators = _acc.get_routine_accelerators()
+        old_reduction_accelerators = _acc.get_reduction_accelerators()
+        if self.backend == 'device':
+            _acc.set_routine_accelerators(['cub'])
+            _acc.set_reduction_accelerators([])
+        elif self.backend == 'block':
+            _acc.set_routine_accelerators([])
+            _acc.set_reduction_accelerators(['cub'])
+        yield
+        _acc.set_routine_accelerators(old_routine_accelerators)
+        _acc.set_reduction_accelerators(old_reduction_accelerators)
+
+    @testing.for_contiguous_axes()
+    # sum supports less dtypes; don't test float16 as it's not as accurate?
+    @testing.for_dtypes('qQfdFD')
+    @testing.numpy_cupy_allclose(rtol=1E-5)
+    def test_cub_sum(self, xp, dtype, axis):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order in ('c', 'C'):
+            a = xp.ascontiguousarray(a)
+        elif self.order in ('f', 'F'):
+            a = xp.asfortranarray(a)
+
+        if xp is numpy:
+            return a.sum(axis=axis)
+
+        # xp is cupy, first ensure we really use CUB
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        if self.backend == 'device':
+            func_name = 'cupy._core._routines_math.cub.'
+            if len(axis) == len(self.shape):
+                func_name += 'device_reduce'
+            else:
+                func_name += 'device_segmented_reduce'
+            with testing.AssertFunctionIsCalled(func_name, return_value=ret):
+                a.sum(axis=axis)
+        elif self.backend == 'block':
+            # this is the only function we can mock; the rest is cdef'd
+            func_name = 'cupy._core._cub_reduction.'
+            func_name += '_SimpleCubReductionKernel_get_cached_function'
+            func = _cub_reduction._SimpleCubReductionKernel_get_cached_function
+            if len(axis) == len(self.shape):
+                times_called = 2  # two passes
+            else:
+                times_called = 1  # one pass
+            with testing.AssertFunctionIsCalled(
+                    func_name, wraps=func, times_called=times_called):
+                a.sum(axis=axis)
+        # ...then perform the actual computation
+        return a.sum(axis=axis)
+
+    # sum supports less dtypes; don't test float16 as it's not as accurate?
+    @testing.for_dtypes('qQfdFD')
+    @testing.numpy_cupy_allclose(rtol=1E-5, contiguous_check=False)
+    def test_cub_sum_empty_axis(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order in ('c', 'C'):
+            a = xp.ascontiguousarray(a)
+        elif self.order in ('f', 'F'):
+            a = xp.asfortranarray(a)
+        return a.sum(axis=())
+
+    @testing.for_contiguous_axes()
+    # prod supports less dtypes; don't test float16 as it's not as accurate?
+    @testing.for_dtypes('qQfdFD')
+    @testing.numpy_cupy_allclose(rtol=1E-5)
+    def test_cub_prod(self, xp, dtype, axis):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order in ('c', 'C'):
+            a = xp.ascontiguousarray(a)
+        elif self.order in ('f', 'F'):
+            a = xp.asfortranarray(a)
+
+        if xp is numpy:
+            return a.prod(axis=axis)
+
+        # xp is cupy, first ensure we really use CUB
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        if self.backend == 'device':
+            func_name = 'cupy._core._routines_math.cub.'
+            if len(axis) == len(self.shape):
+                func_name += 'device_reduce'
+            else:
+                func_name += 'device_segmented_reduce'
+            with testing.AssertFunctionIsCalled(func_name, return_value=ret):
+                a.prod(axis=axis)
+        elif self.backend == 'block':
+            # this is the only function we can mock; the rest is cdef'd
+            func_name = 'cupy._core._cub_reduction.'
+            func_name += '_SimpleCubReductionKernel_get_cached_function'
+            func = _cub_reduction._SimpleCubReductionKernel_get_cached_function
+            if len(axis) == len(self.shape):
+                times_called = 2  # two passes
+            else:
+                times_called = 1  # one pass
+            with testing.AssertFunctionIsCalled(
+                    func_name, wraps=func, times_called=times_called):
+                a.prod(axis=axis)
+        # ...then perform the actual computation
+        return a.prod(axis=axis)
+
+    # TODO(leofang): test axis after support is added
+    # don't test float16 as it's not as accurate?
+    @testing.for_dtypes('bhilBHILfdF')
+    @testing.numpy_cupy_allclose(rtol=1E-4)
+    def test_cub_cumsum(self, xp, dtype):
+        if self.backend == 'block':
+            pytest.skip('does not support')
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order in ('c', 'C'):
+            a = xp.ascontiguousarray(a)
+        elif self.order in ('f', 'F'):
+            a = xp.asfortranarray(a)
+
+        if xp is numpy:
+            return a.cumsum()
+
+        # xp is cupy, first ensure we really use CUB
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        func = 'cupy._core._routines_math.cub.device_scan'
+        with testing.AssertFunctionIsCalled(func, return_value=ret):
+            a.cumsum()
+        # ...then perform the actual computation
+        return a.cumsum()
+
+    # TODO(leofang): test axis after support is added
+    # don't test float16 as it's not as accurate?
+    @testing.for_dtypes('bhilBHILfdF')
+    @testing.numpy_cupy_allclose(rtol=1E-4)
+    def test_cub_cumprod(self, xp, dtype):
+        if self.backend == 'block':
+            pytest.skip('does not support')
+
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order in ('c', 'C'):
+            a = xp.ascontiguousarray(a)
+        elif self.order in ('f', 'F'):
+            a = xp.asfortranarray(a)
+
+        if xp is numpy:
+            result = a.cumprod()
+            return self._mitigate_cumprod(xp, dtype, result)
+
+        # xp is cupy, first ensure we really use CUB
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        func = 'cupy._core._routines_math.cub.device_scan'
+        with testing.AssertFunctionIsCalled(func, return_value=ret):
+            a.cumprod()
+        # ...then perform the actual computation
+        result = a.cumprod()
+        return self._mitigate_cumprod(xp, dtype, result)
+
+    def _mitigate_cumprod(self, xp, dtype, result):
+        # for testing cumprod against complex arrays, the gotcha is CuPy may
+        # produce only Inf at the position where NumPy starts to give NaN. So,
+        # an error would be raised during assert_allclose where the positions
+        # of NaNs are examined. Since this is both algorithm and architecture
+        # dependent, we have no control over this behavior and can only
+        # circumvent the issue by manually converting Inf to NaN
+        if dtype in (numpy.complex64, numpy.complex128):
+            pos = xp.where(xp.isinf(result))
+            result[pos] = xp.nan + 1j * xp.nan
+        return result
+
+
+# This class compares cuTENSOR results against NumPy's
+@testing.parameterize(*testing.product({
+    'shape': [(10,), (10, 20), (10, 20, 30), (10, 20, 30, 40)],
+    'order': ('C', 'F'),
+}))
+@pytest.mark.skipif(
+    not cupy.cuda.cutensor.available,
+    reason='The cuTENSOR routine is not enabled')
+class TestCuTensorReduction:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        old_accelerators = cupy._core.get_routine_accelerators()
+        cupy._core.set_routine_accelerators(['cutensor'])
+        yield
+        cupy._core.set_routine_accelerators(old_accelerators)
+
+    @testing.for_contiguous_axes()
+    # sum supports less dtypes; don't test float16 as it's not as accurate?
+    @testing.for_dtypes('qQfdFD')
+    @testing.numpy_cupy_allclose(rtol=1E-5, contiguous_check=False)
+    def test_cutensor_sum(self, xp, dtype, axis):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order in ('c', 'C'):
+            a = xp.ascontiguousarray(a)
+        elif self.order in ('f', 'F'):
+            a = xp.asfortranarray(a)
+
+        if xp is numpy:
+            return a.sum(axis=axis)
+
+        # xp is cupy, first ensure we really use cuTENSOR
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        func = 'cupyx.cutensor._try_reduction_routine'
+        with testing.AssertFunctionIsCalled(func, return_value=ret):
+            a.sum(axis=axis)
+        # ...then perform the actual computation
+        return a.sum(axis=axis)
+
+    # sum supports less dtypes; don't test float16 as it's not as accurate?
+    @testing.for_dtypes('qQfdFD')
+    @testing.numpy_cupy_allclose(rtol=1E-5, contiguous_check=False)
+    def test_cutensor_sum_empty_axis(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order in ('c', 'C'):
+            a = xp.ascontiguousarray(a)
+        elif self.order in ('f', 'F'):
+            a = xp.asfortranarray(a)
+        return a.sum(axis=())
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(2, 3, 4), (20, 30, 40)],
+        'axis': [0, 1],
+        'transpose_axes': [True, False],
+        'keepdims': [True, False],
+        'func': ['nansum', 'nanprod']
+    })
+)
+class TestNansumNanprodLong:
+
+    def _do_transposed_axis_test(self):
+        return not self.transpose_axes and self.axis != 1
+
+    def _numpy_nanprod_implemented(self):
+        return (self.func == 'nanprod' and
+                numpy.__version__ >= numpy.lib.NumpyVersion('1.10.0'))
+
+    def _test(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        if self.transpose_axes:
+            a = a.transpose(2, 0, 1)
+        if not issubclass(dtype, xp.integer):
+            a[:, 1] = xp.nan
+        func = getattr(xp, self.func)
+        return func(a, axis=self.axis, keepdims=self.keepdims)
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True)
+    @testing.numpy_cupy_allclose()
+    def test_nansum_all(self, xp, dtype):
+        if (not self._numpy_nanprod_implemented() or
+                not self._do_transposed_axis_test()):
+            return xp.array(())
+        return self._test(xp, dtype)
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True)
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_nansum_axis_transposed(self, xp, dtype):
+        if (not self._numpy_nanprod_implemented() or
+                not self._do_transposed_axis_test()):
+            return xp.array(())
+        return self._test(xp, dtype)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(2, 3, 4), (20, 30, 40)],
+    })
+)
+class TestNansumNanprodExtra:
+
+    def test_nansum_axis_float16(self):
+        # Note that the above test example overflows in float16. We use a
+        # smaller array instead, just return if array is too large.
+        if (numpy.prod(self.shape) > 24):
+            return
+        a = testing.shaped_arange(self.shape, dtype='e')
+        a[:, 1] = cupy.nan
+        sa = cupy.nansum(a, axis=1)
+        b = testing.shaped_arange(self.shape, numpy, dtype='f')
+        b[:, 1] = numpy.nan
+        sb = numpy.nansum(b, axis=1)
+        testing.assert_allclose(sa, sb.astype('e'))
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True)
+    @testing.numpy_cupy_allclose()
+    def test_nansum_out(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        if not issubclass(dtype, xp.integer):
+            a[:, 1] = xp.nan
+        b = xp.empty((self.shape[0], self.shape[2]), dtype=dtype)
+        xp.nansum(a, axis=1, out=b)
+        return b
+
+    def test_nansum_out_wrong_shape(self):
+        a = testing.shaped_arange(self.shape)
+        a[:, 1] = cupy.nan
+        b = cupy.empty((2, 3))
+        with pytest.raises(ValueError):
+            cupy.nansum(a, axis=1, out=b)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(2, 3, 4, 5), (20, 30, 40, 50)],
+        'axis': [(1, 3), (0, 2, 3)],
+    })
+)
+class TestNansumNanprodAxes:
+    @testing.for_all_dtypes(no_bool=True, no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nansum_axes(self, xp, dtype):
+        a = testing.shaped_arange(self.shape, xp, dtype)
+        if not issubclass(dtype, xp.integer):
+            a[:, 1] = xp.nan
+        return xp.nansum(a, axis=self.axis)
+
+
+class TestNansumNanprodHuge:
+    def _test(self, xp, nan_slice):
+        a = testing.shaped_random((2048, 1, 1024), xp, 'f')
+        a[nan_slice] = xp.nan
+        a = xp.broadcast_to(a, (2048, 1024, 1024))
+        return xp.nansum(a, axis=2)
+
+    @testing.slow
+    @testing.numpy_cupy_allclose(atol=1e-1)
+    def test_nansum_axis_huge(self, xp):
+        return self._test(
+            xp, (slice(None, None), slice(None, None), slice(1, 2)))
+
+    @testing.slow
+    @testing.numpy_cupy_allclose(atol=1e-2)
+    def test_nansum_axis_huge_halfnan(self, xp):
+        return self._test(
+            xp, (slice(None, None), slice(None, None), slice(0, 512)))
+
+
+axes = [0, 1, 2]
+
+
+@testing.parameterize(*testing.product({'axis': axes}))
+class TestCumsum:
+
+    def _cumsum(self, xp, a, *args, **kwargs):
+        b = a.copy()
+        res = xp.cumsum(a, *args, **kwargs)
+        testing.assert_array_equal(a, b)  # Check if input array is overwritten
+        return res
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumsum(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return self._cumsum(xp, a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumsum_out(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        out = xp.zeros((5,), dtype=dtype)
+        self._cumsum(xp, a, out=out)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumsum_out_noncontiguous(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        out = xp.zeros((10,), dtype=dtype)[::2]  # Non contiguous view
+        self._cumsum(xp, a, out=out)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumsum_2dim(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return self._cumsum(xp, a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_cumsum_axis(self, xp, dtype):
+        n = len(axes)
+        a = testing.shaped_arange(tuple(range(4, 4 + n)), xp, dtype)
+        return self._cumsum(xp, a, axis=self.axis)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumsum_axis_out(self, xp, dtype):
+        n = len(axes)
+        shape = tuple(range(4, 4 + n))
+        a = testing.shaped_arange(shape, xp, dtype)
+        out = xp.zeros(shape, dtype=dtype)
+        self._cumsum(xp, a, axis=self.axis, out=out)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumsum_axis_out_noncontiguous(self, xp, dtype):
+        n = len(axes)
+        shape = tuple(range(4, 4 + n))
+        a = testing.shaped_arange(shape, xp, dtype)
+        out = xp.zeros((8,)+shape[1:], dtype=dtype)[::2]  # Non contiguous view
+        self._cumsum(xp, a, axis=self.axis, out=out)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(contiguous_check=False)
+    def test_ndarray_cumsum_axis(self, xp, dtype):
+        n = len(axes)
+        a = testing.shaped_arange(tuple(range(4, 4 + n)), xp, dtype)
+        return a.cumsum(axis=self.axis)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumsum_axis_empty(self, xp, dtype):
+        n = len(axes)
+        a = testing.shaped_arange(tuple(range(0, n)), xp, dtype)
+        return self._cumsum(xp, a, axis=self.axis)
+
+    @testing.for_all_dtypes()
+    def test_invalid_axis_lower1(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((4, 5), xp, dtype)
+            with pytest.raises(numpy.AxisError):
+                xp.cumsum(a, axis=-a.ndim - 1)
+
+    @testing.for_all_dtypes()
+    def test_invalid_axis_lower2(self, dtype):
+        a = testing.shaped_arange((4, 5), cupy, dtype)
+        with pytest.raises(numpy.AxisError):
+            return cupy.cumsum(a, axis=-a.ndim - 1)
+
+    @testing.for_all_dtypes()
+    def test_invalid_axis_upper1(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((4, 5), xp, dtype)
+            with pytest.raises(numpy.AxisError):
+                xp.cumsum(a, axis=a.ndim + 1)
+
+    @testing.for_all_dtypes()
+    def test_invalid_axis_upper2(self, dtype):
+        a = testing.shaped_arange((4, 5), cupy, dtype)
+        with pytest.raises(numpy.AxisError):
+            return cupy.cumsum(a, axis=a.ndim + 1)
+
+    def test_cumsum_arraylike(self):
+        with pytest.raises(TypeError):
+            return cupy.cumsum((1, 2, 3))
+
+    @testing.for_float_dtypes()
+    def test_cumsum_numpy_array(self, dtype):
+        a_numpy = numpy.arange(8, dtype=dtype)
+        with pytest.raises(TypeError):
+            return cupy.cumsum(a_numpy)
+
+
+class TestCumprod:
+
+    def _cumprod(self, xp, a, *args, **kwargs):
+        b = a.copy()
+        res = xp.cumprod(a, *args, **kwargs)
+        testing.assert_array_equal(a, b)  # Check if input array is overwritten
+        return res
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumprod_1dim(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return self._cumprod(xp, a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumprod_out(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        out = xp.zeros((5,), dtype=dtype)
+        self._cumprod(xp, a, out=out)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumprod_out_noncontiguous(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        out = xp.zeros((10,), dtype=dtype)[::2]  # Non contiguous view
+        self._cumprod(xp, a, out=out)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_cumprod_2dim_without_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return self._cumprod(xp, a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_cumprod_2dim_with_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return self._cumprod(xp, a, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_ndarray_cumprod_2dim_with_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return a.cumprod(axis=1)
+
+    @testing.slow
+    def test_cumprod_huge_array(self):
+        size = 2 ** 32
+        # Free huge memory for slow test
+        cupy.get_default_memory_pool().free_all_blocks()
+        a = cupy.ones(size, 'b')
+        result = cupy.cumprod(a, dtype='b')
+        del a
+        assert (result == 1).all()
+        # Free huge memory for slow test
+        del result
+        cupy.get_default_memory_pool().free_all_blocks()
+
+    @testing.for_all_dtypes()
+    def test_invalid_axis_lower1(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((4, 5), xp, dtype)
+            with pytest.raises(numpy.AxisError):
+                xp.cumprod(a, axis=-a.ndim - 1)
+
+    @testing.for_all_dtypes()
+    def test_invalid_axis_lower2(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((4, 5), xp, dtype)
+            with pytest.raises(numpy.AxisError):
+                xp.cumprod(a, axis=-a.ndim - 1)
+
+    @testing.for_all_dtypes()
+    def test_invalid_axis_upper1(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((4, 5), xp, dtype)
+            with pytest.raises(numpy.AxisError):
+                return xp.cumprod(a, axis=a.ndim)
+
+    @testing.for_all_dtypes()
+    def test_invalid_axis_upper2(self, dtype):
+        a = testing.shaped_arange((4, 5), cupy, dtype)
+        with pytest.raises(numpy.AxisError):
+            return cupy.cumprod(a, axis=a.ndim)
+
+    def test_cumprod_arraylike(self):
+        with pytest.raises(TypeError):
+            return cupy.cumprod((1, 2, 3))
+
+    @testing.for_float_dtypes()
+    def test_cumprod_numpy_array(self, dtype):
+        a_numpy = numpy.arange(1, 6, dtype=dtype)
+        with pytest.raises(TypeError):
+            return cupy.cumprod(a_numpy)
+
+    @testing.numpy_cupy_allclose()
+    def test_cumproduct_alias(self, xp):
+        a = testing.shaped_arange((2, 3), xp, xp.float32)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            return xp.cumproduct(a)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(20,), (7, 6), (3, 4, 5)],
+    'axis': [None, 0, 1, 2],
+    'func': ('nancumsum', 'nancumprod'),
+}))
+class TestNanCumSumProd:
+
+    zero_density = 0.25
+
+    def _make_array(self, dtype):
+        dtype = numpy.dtype(dtype)
+        if dtype.char in 'efdFD':
+            r_dtype = dtype.char.lower()
+            a = testing.shaped_random(self.shape, numpy, dtype=r_dtype,
+                                      scale=1)
+            if dtype.char in 'FD':
+                ai = a
+                aj = testing.shaped_random(self.shape, numpy, dtype=r_dtype,
+                                           scale=1)
+                ai[ai < math.sqrt(self.zero_density)] = 0
+                aj[aj < math.sqrt(self.zero_density)] = 0
+                a = ai + 1j * aj
+            else:
+                a[a < self.zero_density] = 0
+            a = a / a
+        else:
+            a = testing.shaped_random(self.shape, numpy, dtype=dtype)
+        return a
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nancumsumprod(self, xp, dtype):
+        if self.axis is not None and self.axis >= len(self.shape):
+            pytest.skip()
+        a = xp.array(self._make_array(dtype))
+        out = getattr(xp, self.func)(a, axis=self.axis)
+        return xp.ascontiguousarray(out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nancumsumprod_out(self, xp, dtype):
+        dtype = numpy.dtype(dtype)
+        if self.axis is not None and self.axis >= len(self.shape):
+            pytest.skip()
+        if len(self.shape) > 1 and self.axis is None:
+            # Skip the cases where np.nancum{sum|prod} raise AssertionError.
+            pytest.skip()
+        a = xp.array(self._make_array(dtype))
+        out = xp.empty(self.shape, dtype=dtype)
+        getattr(xp, self.func)(a, axis=self.axis, out=out)
+        return xp.ascontiguousarray(out)
+
+
+class TestDiff:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_diff_1dim(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.diff(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_diff_1dim_with_n(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.diff(a, n=3)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_diff_2dim_without_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.diff(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_diff_2dim_with_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.diff(a, axis=-2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_diff_2dim_with_n_and_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.diff(a, 2, 1)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_diff_2dim_with_prepend(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        b = testing.shaped_arange((4, 1), xp, dtype)
+        return xp.diff(a, axis=-1, prepend=b)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_diff_2dim_with_append(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        b = testing.shaped_arange((1, 5), xp, dtype)
+        return xp.diff(a, axis=0, append=b, n=2)
+
+    @testing.with_requires('numpy>=1.16')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_diff_2dim_with_scalar_append(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.diff(a, prepend=1, append=0)
+
+    @testing.with_requires('numpy>=1.16')
+    def test_diff_invalid_axis(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((2, 3, 4), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.diff(a, axis=3)
+            with pytest.raises(numpy.AxisError):
+                xp.diff(a, axis=-4)
+
+
+# This class compares CUB results against NumPy's
+@testing.parameterize(*testing.product_dict(
+    testing.product({
+        'shape': [()],
+        'axis': [None, ()],
+        'spacing': [(), (1.2,)],
+    })
+    + testing.product({
+        'shape': [(33,)],
+        'axis': [None, 0, -1, (0,)],
+        'spacing': [(), (1.2,), 'sequence of int', 'arrays'],
+    })
+    + testing.product({
+        'shape': [(10, 20), (10, 20, 30)],
+        'axis': [None, 0, -1, (0, -1), (1, 0)],
+        'spacing': [(), (1.2,), 'sequence of int', 'arrays', 'mixed'],
+    }),
+    testing.product({
+        'edge_order': [1, 2],
+    }),
+))
+class TestGradient:
+
+    def _gradient(self, xp, dtype, shape, spacing, axis, edge_order):
+        x = testing.shaped_random(shape, xp, dtype=dtype)
+        if axis is None:
+            normalized_axes = tuple(range(x.ndim))
+        else:
+            normalized_axes = axis
+            if not isinstance(normalized_axes, tuple):
+                normalized_axes = normalized_axes,
+            normalized_axes = tuple(ax % x.ndim for ax in normalized_axes)
+        if spacing == 'sequence of int':
+            # one scalar per axis
+            spacing = tuple((ax + 1) / x.ndim for ax in normalized_axes)
+        elif spacing == 'arrays':
+            # one array per axis
+            spacing = tuple(
+                xp.arange(x.shape[ax]) * (ax + 0.5) for ax in normalized_axes
+            )
+            # make at one of the arrays have non-constant spacing
+            spacing[-1][5:] *= 2.0
+        elif spacing == 'mixed':
+            # mixture of arrays and scalars
+            spacing = [xp.arange(x.shape[normalized_axes[0]])]
+            spacing = spacing + [0.5] * (len(normalized_axes) - 1)
+        return xp.gradient(x, *spacing, axis=axis, edge_order=edge_order)
+
+    @testing.for_dtypes('fFdD')
+    @testing.numpy_cupy_allclose(atol=1e-6, rtol=1e-5)
+    def test_gradient_floating(self, xp, dtype):
+        return self._gradient(xp, dtype, self.shape, self.spacing, self.axis,
+                              self.edge_order)
+
+    # unsigned int behavior fixed in 1.18.1
+    # https://github.com/numpy/numpy/issues/15207
+    @testing.with_requires('numpy>=1.18.1')
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-6, rtol=1e-5)
+    def test_gradient_int(self, xp, dtype):
+        return self._gradient(xp, dtype, self.shape, self.spacing, self.axis,
+                              self.edge_order)
+
+    @testing.numpy_cupy_allclose(atol=2e-2, rtol=1e-3)
+    def test_gradient_float16(self, xp):
+        return self._gradient(xp, numpy.float16, self.shape, self.spacing,
+                              self.axis, self.edge_order)
+
+
+class TestGradientErrors:
+
+    def test_gradient_invalid_spacings1(self):
+        # more spacings than axes
+        spacing = (1.0, 2.0, 3.0)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random((32, 16), xp)
+            with pytest.raises(TypeError):
+                xp.gradient(x, *spacing)
+
+    def test_gradient_invalid_spacings2(self):
+        # wrong length array in spacing
+        shape = (32, 16)
+        spacing = (15, cupy.arange(shape[1] + 1))
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            with pytest.raises(ValueError):
+                xp.gradient(x, *spacing)
+
+    def test_gradient_invalid_spacings3(self):
+        # spacing array with ndim != 1
+        shape = (32, 16)
+        spacing = (15, cupy.arange(shape[0]).reshape(4, -1))
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            with pytest.raises(ValueError):
+                xp.gradient(x, *spacing)
+
+    def test_gradient_invalid_edge_order1(self):
+        # unsupported edge order
+        shape = (32, 16)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            with pytest.raises(ValueError):
+                xp.gradient(x, edge_order=3)
+
+    def test_gradient_invalid_edge_order2(self):
+        # shape cannot be < edge_order
+        shape = (1, 16)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            with pytest.raises(ValueError):
+                xp.gradient(x, axis=0, edge_order=2)
+
+    @testing.with_requires('numpy>=1.16')
+    def test_gradient_invalid_axis(self):
+        # axis out of range
+        shape = (4, 16)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            for axis in [-3, 2]:
+                with pytest.raises(numpy.AxisError):
+                    xp.gradient(x, axis=axis)
+
+    def test_gradient_bool_input(self):
+        # axis out of range
+        shape = (4, 16)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp, dtype=numpy.bool_)
+            with pytest.raises(TypeError):
+                xp.gradient(x)
+
+
+class TestEdiff1d:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_1dim(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.ediff1d(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_2dim(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.ediff1d(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_3dim(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.ediff1d(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_to_begin1(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.ediff1d(a, to_begin=xp.array([0], dtype=dtype))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_to_begin2(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.ediff1d(a, to_begin=xp.array([4, 4], dtype=dtype))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_to_begin3(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.ediff1d(a, to_begin=xp.array([1, 1], dtype=dtype))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_to_end1(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.ediff1d(a, to_end=xp.array([0], dtype=dtype))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_to_end2(self, xp, dtype):
+        a = testing.shaped_arange((4, 1), xp, dtype)
+        return xp.ediff1d(a, to_end=xp.array([1, 2], dtype=dtype))
+
+    @testing.for_dtypes('bhilqefdFD')
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_ed1(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4, 5), xp, dtype)
+        return xp.ediff1d(a, to_begin=xp.array([-1], dtype=dtype),
+                          to_end=xp.array([0], dtype=dtype))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ediff1d_ed2(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.ediff1d(a, to_begin=xp.array([0, 4], dtype=dtype),
+                          to_end=xp.array([1, 1], dtype=dtype))
+
+
+class TestTrapz:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    def test_trapz_1dim(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.trapz(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    def test_trapz_1dim_with_x(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        x = testing.shaped_arange((5,), xp, dtype)
+        return xp.trapz(a, x=x)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    def test_trapz_1dim_with_dx(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        return xp.trapz(a, dx=0.1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    def test_trapz_2dim_without_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.trapz(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    def test_trapz_2dim_with_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.trapz(a, axis=-2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    def test_trapz_2dim_with_x_and_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        x = testing.shaped_arange((5,), xp, dtype)
+        return xp.trapz(a, x=x, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    def test_trapz_2dim_with_dx_and_axis(self, xp, dtype):
+        a = testing.shaped_arange((4, 5), xp, dtype)
+        return xp.trapz(a, dx=0.1, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    def test_trapz_1dim_with_x_and_dx(self, xp, dtype):
+        a = testing.shaped_arange((5,), xp, dtype)
+        x = testing.shaped_arange((5,), xp, dtype)
+        return xp.trapz(a, x=x, dx=0.1)
diff --git a/tests/cupy_tests/math_tests/test_trigonometric.py b/tests/cupy_tests/math_tests/test_trigonometric.py
new file mode 100644
index 0000000..776b3bb
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_trigonometric.py
@@ -0,0 +1,113 @@
+import unittest
+
+from cupy import testing
+
+
+class TestTrigonometric(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary(self, name, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_binary(self, name, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_reverse_arange((2, 3), xp, dtype)
+        return getattr(xp, name)(a, b)
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def check_unary_unit(self, name, xp, dtype):
+        a = xp.array([0.2, 0.4, 0.6, 0.8], dtype=dtype)
+        return getattr(xp, name)(a)
+
+    def test_sin(self):
+        self.check_unary('sin')
+
+    def test_cos(self):
+        self.check_unary('cos')
+
+    def test_tan(self):
+        self.check_unary('tan')
+
+    def test_arcsin(self):
+        self.check_unary_unit('arcsin')
+
+    def test_arccos(self):
+        self.check_unary_unit('arccos')
+
+    def test_arctan(self):
+        self.check_unary('arctan')
+
+    def test_arctan2(self):
+        self.check_binary('arctan2')
+
+    def test_hypot(self):
+        self.check_binary('hypot')
+
+    def test_deg2rad(self):
+        self.check_unary('deg2rad')
+
+    def test_rad2deg(self):
+        self.check_unary('rad2deg')
+
+
+@testing.with_requires('numpy>=1.21.0')
+class TestUnwrap(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_1dim(self, xp, dtype):
+        a = testing.shaped_random((5,), xp, dtype)
+        return xp.unwrap(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_1dim_with_discont(self, xp, dtype):
+        a = testing.shaped_random((5,), xp, dtype)
+        return xp.unwrap(a, discont=1.0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_1dim_with_period(self, xp, dtype):
+        a = testing.shaped_random((5,), xp, dtype)
+        return xp.unwrap(a, period=1.2)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_1dim_with_discont_and_period(self, xp, dtype):
+        a = testing.shaped_random((5,), xp, dtype)
+        return xp.unwrap(a, discont=1.0, period=1.2)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_2dim_without_axis(self, xp, dtype):
+        a = testing.shaped_random((4, 5), xp, dtype)
+        return xp.unwrap(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_2dim_with_axis(self, xp, dtype):
+        a = testing.shaped_random((4, 5), xp, dtype)
+        return xp.unwrap(a, axis=1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_2dim_with_discont(self, xp, dtype):
+        a = testing.shaped_random((4, 5), xp, dtype)
+        return xp.unwrap(a, discont=5.0, axis=1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_2dim_with_period(self, xp, dtype):
+        a = testing.shaped_random((4, 5), xp, dtype)
+        return xp.unwrap(a, axis=1, period=4.5)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_unwrap_2dim_with_discont_and_period(self, xp, dtype):
+        a = testing.shaped_random((4, 5), xp, dtype)
+        return xp.unwrap(a, discont=5.0, axis=1, period=4.5)
diff --git a/tests/cupy_tests/math_tests/test_window.py b/tests/cupy_tests/math_tests/test_window.py
new file mode 100644
index 0000000..9a671f6
--- /dev/null
+++ b/tests/cupy_tests/math_tests/test_window.py
@@ -0,0 +1,38 @@
+import unittest
+
+from cupy import testing
+
+
+@testing.parameterize(
+    *testing.product({
+        'm': [0, 1, -1, 1024],
+        'name': ['bartlett', 'blackman', 'hamming', 'hanning'],
+    })
+)
+class TestWindow(unittest.TestCase):
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_window(self, xp):
+        return getattr(xp, self.name)(self.m)
+
+
+@testing.parameterize(
+    *testing.product({
+        'm': [10, 30, 1024],
+        'beta': [-3.4, 0, 5, 6, 8.6],
+        'name': ['kaiser'],
+    })
+)
+class TestKaiser(unittest.TestCase):
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_kaiser_parametric(self, xp):
+        return getattr(xp, self.name)(self.m, self.beta)
+
+
+@testing.parameterize(*testing.product({'m': [-1, 0, 1]}))
+class TestKaiserBoundary(unittest.TestCase):
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_kaiser(self, xp):
+        return xp.kaiser(self.m, 1.5)
diff --git a/tests/cupy_tests/misc_tests/__init__.py b/tests/cupy_tests/misc_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/misc_tests/test_byte_bounds.py b/tests/cupy_tests/misc_tests/test_byte_bounds.py
new file mode 100644
index 0000000..7354c46
--- /dev/null
+++ b/tests/cupy_tests/misc_tests/test_byte_bounds.py
@@ -0,0 +1,76 @@
+import cupy
+from cupy import testing
+
+
+class TestByteBounds:
+
+    @testing.for_all_dtypes()
+    def test_1d_contiguous(self, dtype):
+        a = cupy.zeros(12, dtype=dtype)
+        itemsize = a.itemsize
+        a_low = a.data.ptr
+        a_high = a.data.ptr + 12 * itemsize
+        assert cupy.byte_bounds(a) == (a_low, a_high)
+
+    @testing.for_all_dtypes()
+    def test_2d_contiguous(self, dtype):
+        a = cupy.zeros((4, 7), dtype=dtype)
+        itemsize = a.itemsize
+        a_low = a.data.ptr
+        a_high = a.data.ptr + 4 * 7 * itemsize
+        assert cupy.byte_bounds(a) == (a_low, a_high)
+
+    @testing.for_all_dtypes()
+    def test_1d_noncontiguous_pos_stride(self, dtype):
+        a = cupy.zeros(12, dtype=dtype)
+        itemsize = a.itemsize
+        b = a[::2]
+        b_low = b.data.ptr
+        b_high = b.data.ptr + 11 * itemsize  # a[10]
+        assert cupy.byte_bounds(b) == (b_low, b_high)
+
+    @testing.for_all_dtypes()
+    def test_2d_noncontiguous_pos_stride(self, dtype):
+        a = cupy.zeros((4, 7), dtype=dtype)
+        b = a[::2, ::2]
+        itemsize = b.itemsize
+        b_low = a.data.ptr
+        b_high = b.data.ptr + 3 * 7 * itemsize  # a[2][6]
+        assert cupy.byte_bounds(b) == (b_low, b_high)
+
+    @testing.for_all_dtypes()
+    def test_1d_contiguous_neg_stride(self, dtype):
+        a = cupy.zeros(12, dtype=dtype)
+        b = a[::-1]
+        itemsize = b.itemsize
+        b_low = b.data.ptr - 11 * itemsize
+        b_high = b.data.ptr + 1 * itemsize
+        assert cupy.byte_bounds(b) == (b_low, b_high)
+
+    @testing.for_all_dtypes()
+    def test_2d_noncontiguous_neg_stride(self, dtype):
+        a = cupy.zeros((4, 7), dtype=dtype)
+        b = a[::-2, ::-2]  # strides = (-56, -8), shape = (2, 4)
+        itemsize = b.itemsize
+        b_low = b.data.ptr - 2 * 7 * itemsize * \
+            (2 - 1) - 2 * itemsize * (4 - 1)
+        b_high = b.data.ptr + 1 * itemsize
+        assert cupy.byte_bounds(b) == (b_low, b_high)
+
+    @testing.for_all_dtypes()
+    def test_2d_noncontiguous_posneg_stride_1(self, dtype):
+        a = cupy.zeros((4, 7), dtype=dtype)
+        b = a[::1, ::-1]  # strides = (28, -4), shape=(4, 7)
+        itemsize = b.itemsize
+        b_low = b.data.ptr - itemsize * (7 - 1)
+        b_high = b.data.ptr + 1 * itemsize + 7 * itemsize * (4 - 1)
+        assert cupy.byte_bounds(b) == (b_low, b_high)
+
+    @testing.for_all_dtypes()
+    def test_2d_noncontiguous_posneg_stride_2(self, dtype):
+        a = cupy.zeros((4, 7), dtype=dtype)
+        b = a[::2, ::-2]  # strides = (56, -8), shape=(2, 4)
+        itemsize = b.itemsize
+        b_low = b.data.ptr - 2 * itemsize * (4 - 1)
+        b_high = b.data.ptr + 1 * itemsize + 2 * 7 * itemsize * (2 - 1)
+        assert cupy.byte_bounds(b) == (b_low, b_high)
diff --git a/tests/cupy_tests/misc_tests/test_memory_ranges.py b/tests/cupy_tests/misc_tests/test_memory_ranges.py
new file mode 100644
index 0000000..9c07c7a
--- /dev/null
+++ b/tests/cupy_tests/misc_tests/test_memory_ranges.py
@@ -0,0 +1,152 @@
+import numpy
+import unittest
+
+import cupy
+from cupy import testing
+
+
+class TestMayShareMemory(unittest.TestCase):
+
+    @testing.numpy_cupy_equal()
+    def test_different_arrays(self, xp):
+        a = xp.array([1, 2, 3])
+        b = xp.array([1, 2, 3])
+        assert xp.may_share_memory(a, b) is False
+
+    @testing.numpy_cupy_equal()
+    def test_same_array(self, xp):
+        a = xp.array([1, 2, 3])
+        assert xp.may_share_memory(a, a) is True
+
+    @testing.numpy_cupy_equal()
+    def test_zero_size(self, xp):
+        a = xp.array([])
+        assert xp.may_share_memory(a, a) is False
+
+    @testing.numpy_cupy_equal()
+    def test_shares_memory(self, xp):
+        x = xp.arange(12)
+        a = x[0:7]
+        b = x[6:12]
+        assert xp.may_share_memory(a, b) is True
+
+    @testing.numpy_cupy_equal()
+    def test_cover(self, xp):
+        x = xp.arange(12)
+        a = x[1:10]
+        b = x[4:6]
+        assert xp.may_share_memory(a, b) is True
+
+    @testing.numpy_cupy_equal()
+    def test_away(self, xp):
+        x = xp.arange(12)
+        a = x[1:6]
+        b = x[8:11]
+        assert xp.may_share_memory(a, b) is False
+
+    @testing.numpy_cupy_equal()
+    def test_touch_edge_true(self, xp):
+        x = xp.arange(12)
+        a = x[1:10]
+        b = x[7:10]
+        assert xp.may_share_memory(a, b) is True
+
+    def test_negative_strides(self):
+        for xp in (numpy, cupy):
+            a = xp.zeros((3, 3))
+            assert xp.may_share_memory(a[:2, 1::-1], a[1:, 1:]) is True
+
+    @testing.numpy_cupy_equal()
+    def test_touch_edge_false(self, xp):
+        x = xp.arange(12)
+        a = x[1:7]
+        b = x[7:10]
+        assert xp.may_share_memory(a, b) is False
+
+    def _get_slices(self, size):
+        slices = []
+        for start in range(0, size + 1):
+            for end in range(start, size + 1):
+                for step in range(-2, 2):
+                    if step != 0:
+                        slices.append(slice(start, end, step))
+        return slices
+
+    def test_combination(self):
+        size = 4
+        slices = self._get_slices(size)
+        memory_np = numpy.empty(size * size)
+        memory_cp = cupy.empty(size * size)
+
+        arrays = []
+
+        array_1d_np = memory_np[5:5+size]
+        array_1d_cp = memory_cp[5:5+size]
+        for s in slices:
+            arrays.append((array_1d_np[s], array_1d_cp[s], s))
+
+        array_2d_np = memory_np.reshape(size, size)
+        array_2d_cp = memory_cp.reshape(size, size)
+        for s1 in slices:
+            for s2 in slices:
+                arrays.append((
+                    array_2d_np[s1, s2], array_2d_cp[s1, s2], (s1, s2)))
+
+        for array1_np, array1_cp, sl1 in arrays:
+            for array2_np, array2_cp, sl2 in arrays:
+                ret_np = numpy.may_share_memory(array1_np, array2_np)
+                ret_cp = cupy.may_share_memory(array1_cp, array2_cp)
+                assert ret_np == ret_cp, \
+                    'Failed in case of {} and {}'.format(sl1, sl2)
+
+
+class TestSharesMemory(unittest.TestCase):
+
+    def test_different_arrays(self):
+        for xp in (numpy, cupy):
+            a = xp.array([1, 2, 3])
+            b = xp.array([1, 2, 3])
+            assert xp.shares_memory(a, b) is False
+
+    def test_same_array(self):
+        for xp in (numpy, cupy):
+            a = xp.array([1, 2, 3])
+            assert xp.shares_memory(a, a) is True
+
+    def test_zero_size_array(self):
+        for xp in (numpy, cupy):
+            a = xp.array([])
+            assert xp.shares_memory(a, a) is False
+
+    def test_contiguous_arrays(self):
+        for xp in (numpy, cupy):
+            x = xp.arange(12)
+            # shares memory
+            assert xp.shares_memory(x[0:7], x[6:12]) is True
+            # covers
+            assert xp.shares_memory(x[1:10], x[4:6]) is True
+            assert xp.shares_memory(x[4:6], x[1:10]) is True
+            # detached
+            assert xp.shares_memory(x[1:6], x[8:11]) is False
+            # touch
+            assert xp.shares_memory(x[1:10], x[7:10]) is True
+            assert xp.shares_memory(x[1:7], x[7:10]) is False
+
+    def test_non_contiguous_case(self):
+        for xp in (numpy, cupy):
+            x = xp.arange(100)
+            assert xp.shares_memory(x, x[1::4]) is True
+            assert xp.shares_memory(x[0::2], x[1::4]) is False
+            assert xp.shares_memory(x[0::9], x[1::11]) is True
+
+    def test_multi_dimension_case(self):
+        for xp in (numpy, cupy):
+            x = xp.arange(100).reshape(10, 10)
+            assert xp.shares_memory(x[0::2], x[1::3]) is True
+            assert xp.shares_memory(x[0::2], x[1::4]) is False
+            assert xp.shares_memory(x[0::2], x[::, 1::2]) is True
+
+    def test_complex_type_case(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_random((2, 3, 4), xp, numpy.complex128)
+            assert xp.shares_memory(x, x.imag) is True
diff --git a/tests/cupy_tests/misc_tests/test_who.py b/tests/cupy_tests/misc_tests/test_who.py
new file mode 100644
index 0000000..0a4efc9
--- /dev/null
+++ b/tests/cupy_tests/misc_tests/test_who.py
@@ -0,0 +1,65 @@
+import cupy
+
+
+class TestWho:
+    def test_who_empty(self, capsys):
+        cupy.who()
+        out, err = capsys.readouterr()
+        lines = out.split('\n')
+        assert len(lines) == 3
+        assert lines[1] == 'Upper bound on total bytes  =       0'
+
+    def test_who_local_var(self, capsys):
+        # Variables declared inside an object function are not visible
+        # this is true also for numpy
+        x = cupy.ones(10)  # NOQA
+        cupy.who()
+        out, err = capsys.readouterr()
+        lines = out.split('\n')
+        assert len(lines) == 3
+        assert lines[1] == 'Upper bound on total bytes  =       0'
+
+    def test_who_global(self, capsys):
+        global x
+        x = cupy.ones(10)  # NOQA
+        cupy.who()
+        out, err = capsys.readouterr()
+        lines = out.split('\n')
+        assert lines[-4].split() == ['x', '10', '80', 'float64']
+        assert lines[-2] == 'Upper bound on total bytes  =       80'
+
+    def test_who_global_multi(self, capsys):
+        global x
+        global y
+        x = cupy.ones(10)  # NOQA
+        y = cupy.ones(20, dtype=cupy.int32)  # NOQA
+        cupy.who()
+        out, err = capsys.readouterr()
+        lines = out.split('\n')
+        # depending on the env, the order in which vars are print
+        # might be different
+        var_1 = lines[-5].split()
+        var_2 = lines[-4].split()
+        if var_1[0] == 'x':
+            assert var_1 == ['x', '10', '80', 'float64']
+            assert var_2 == ['y', '20', '80', 'int32']
+        else:
+            assert var_2 == ['x', '10', '80', 'float64']
+            assert var_1 == ['y', '20', '80', 'int32']
+        assert lines[-2] == 'Upper bound on total bytes  =       160'
+
+    def test_who_dict_arrays(self, capsys):
+        var_dict = {'x': cupy.ones(10)}
+        cupy.who(var_dict)
+        out, err = capsys.readouterr()
+        lines = out.split('\n')
+        assert lines[-4].split() == ['x', '10', '80', 'float64']
+        assert lines[-2] == 'Upper bound on total bytes  =       80'
+
+    def test_who_dict_empty(self, capsys):
+        global x
+        x = cupy.ones(10)  # NOQA
+        cupy.who({})
+        out, err = capsys.readouterr()
+        lines = out.split('\n')
+        assert lines[-2] == 'Upper bound on total bytes  =       0'
diff --git a/tests/cupy_tests/padding_tests/__init__.py b/tests/cupy_tests/padding_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/padding_tests/test_pad.py b/tests/cupy_tests/padding_tests/test_pad.py
new file mode 100644
index 0000000..b510c59
--- /dev/null
+++ b/tests/cupy_tests/padding_tests/test_pad.py
@@ -0,0 +1,310 @@
+import unittest
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(
+    *testing.product({
+        'array': [numpy.arange(6).reshape([2, 3])],
+        'pad_width': [1, [1, 2], [[1, 2], [3, 4]]],
+        # mode 'mean' is non-exact, so it is tested in a separate class
+        'mode': ['constant', 'edge', 'linear_ramp', 'maximum',
+                 'minimum', 'reflect', 'symmetric', 'wrap'],
+    })
+)
+class TestPadDefault(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_pad_default(self, xp, dtype):
+        array = xp.array(self.array, dtype=dtype)
+
+        if (xp.dtype(dtype).kind in ['i', 'u'] and
+                self.mode == 'linear_ramp'):
+            # TODO: can remove this skip once cupy/cupy/#2330 is merged
+            return array
+
+        # Older version of NumPy(<1.12) can emit ComplexWarning
+        def f():
+            return xp.pad(array, self.pad_width, mode=self.mode)
+
+        if xp is numpy:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', numpy.ComplexWarning)
+                return f()
+        else:
+            return f()
+
+
+@testing.parameterize(
+    *testing.product({
+        'array': [numpy.arange(6).reshape([2, 3])],
+        'pad_width': [1, [1, 2], [[1, 2], [3, 4]]],
+    })
+)
+class TestPadDefaultMean(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_almost_equal(decimal=5)
+    def test_pad_default(self, xp, dtype):
+        array = xp.array(self.array, dtype=dtype)
+
+        if xp.dtype(dtype).kind in ['i', 'u']:
+            # TODO: can remove this skip once cupy/cupy/#2330 is merged
+            return array
+
+        # Older version of NumPy(<1.12) can emit ComplexWarning
+        def f():
+            return xp.pad(array, self.pad_width, mode='mean')
+
+        if xp is numpy:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', numpy.ComplexWarning)
+                return f()
+        else:
+            return f()
+
+
+@testing.parameterize(
+    # mode='constant'
+    {'array': numpy.arange(6).reshape([2, 3]), 'pad_width': 1,
+     'mode': 'constant', 'constant_values': 3},
+    {'array': numpy.arange(6).reshape([2, 3]),
+     'pad_width': [1, 2], 'mode': 'constant',
+     'constant_values': [3, 4]},
+    {'array': numpy.arange(6).reshape([2, 3]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'constant',
+     'constant_values': [[3, 4], [5, 6]]},
+    # mode='reflect'
+    {'array': numpy.arange(6).reshape([2, 3]), 'pad_width': 1,
+     'mode': 'reflect', 'reflect_type': 'odd'},
+    {'array': numpy.arange(6).reshape([2, 3]),
+     'pad_width': [1, 2], 'mode': 'reflect', 'reflect_type': 'odd'},
+    {'array': numpy.arange(6).reshape([2, 3]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'reflect',
+     'reflect_type': 'odd'},
+    # mode='symmetric'
+    {'array': numpy.arange(6).reshape([2, 3]), 'pad_width': 1,
+     'mode': 'symmetric', 'reflect_type': 'odd'},
+    {'array': numpy.arange(6).reshape([2, 3]),
+     'pad_width': [1, 2], 'mode': 'symmetric', 'reflect_type': 'odd'},
+    {'array': numpy.arange(6).reshape([2, 3]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'symmetric',
+     'reflect_type': 'odd'},
+    # mode='minimum'
+    {'array': numpy.arange(60).reshape([5, 12]), 'pad_width': 1,
+     'mode': 'minimum', 'stat_length': 2},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [1, 2], 'mode': 'minimum', 'stat_length': (2, 4)},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'minimum',
+     'stat_length': ((2, 4), (3, 5))},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'minimum',
+     'stat_length': None},
+    # mode='maximum'
+    {'array': numpy.arange(60).reshape([5, 12]), 'pad_width': 1,
+     'mode': 'maximum', 'stat_length': 2},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [1, 2], 'mode': 'maximum', 'stat_length': (2, 4)},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'maximum',
+     'stat_length': ((2, 4), (3, 5))},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'maximum',
+     'stat_length': None},
+)
+# Old numpy does not work with multi-dimensional constant_values
+class TestPad(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_pad(self, xp, dtype):
+        array = xp.array(self.array, dtype=dtype)
+
+        # Older version of NumPy(<1.12) can emit ComplexWarning
+        def f():
+            if self.mode == 'constant':
+                return xp.pad(array, self.pad_width, mode=self.mode,
+                              constant_values=self.constant_values)
+            elif self.mode in ['minimum', 'maximum']:
+                return xp.pad(array, self.pad_width, mode=self.mode,
+                              stat_length=self.stat_length)
+            elif self.mode in ['reflect', 'symmetric']:
+                return xp.pad(array, self.pad_width, mode=self.mode,
+                              reflect_type=self.reflect_type)
+
+        if xp is numpy:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', numpy.ComplexWarning)
+                return f()
+        else:
+            return f()
+
+
+@testing.parameterize(
+    # mode='mean'
+    {'array': numpy.arange(60).reshape([5, 12]), 'pad_width': 1,
+     'mode': 'mean', 'stat_length': 2},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [1, 2], 'mode': 'mean', 'stat_length': (2, 4)},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'mean',
+     'stat_length': ((2, 4), (3, 5))},
+    {'array': numpy.arange(60).reshape([5, 12]),
+     'pad_width': [[1, 2], [3, 4]], 'mode': 'mean',
+     'stat_length': None},
+)
+# Old numpy does not work with multi-dimensional constant_values
+class TestPadMean(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_almost_equal(decimal=5)
+    def test_pad(self, xp, dtype):
+        array = xp.array(self.array, dtype=dtype)
+
+        if xp.dtype(dtype).kind in ['i', 'u']:
+            # TODO: can remove this skip once cupy/cupy/#2330 is merged
+            return array
+
+        # Older version of NumPy(<1.12) can emit ComplexWarning
+        def f():
+            return xp.pad(array, self.pad_width, mode=self.mode,
+                          stat_length=self.stat_length)
+
+        if xp is numpy:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', numpy.ComplexWarning)
+                return f()
+        else:
+            return f()
+
+
+class TestPadNumpybug(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_pad_highdim_default(self, xp, dtype):
+        array = xp.arange(6, dtype=dtype).reshape([2, 3])
+        pad_width = [[1, 2], [3, 4]]
+        constant_values = [[1, 2], [3, 4]]
+        a = xp.pad(array, pad_width, mode='constant',
+                   constant_values=constant_values)
+        return a
+
+
+class TestPadEmpty(unittest.TestCase):
+
+    @testing.with_requires('numpy>=1.17')
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_pad_empty(self, xp, dtype):
+        array = xp.arange(6, dtype=dtype).reshape([2, 3])
+        pad_width = 2
+        a = xp.pad(array, pad_width=pad_width, mode='empty')
+        # omit uninitialized "empty" boundary from the comparison
+        return a[pad_width:-pad_width, pad_width:-pad_width]
+
+
+class TestPadCustomFunction(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_pad_via_func(self, xp, dtype):
+        def _padwithtens(vector, pad_width, iaxis, kwargs):
+            vector[:pad_width[0]] = 10
+            vector[-pad_width[1]:] = 10
+        a = xp.arange(6, dtype=dtype).reshape(2, 3)
+        a = xp.pad(a, 2, _padwithtens)
+        return a
+
+
+@testing.parameterize(
+    # mode='constant'
+    {'array': [], 'pad_width': 1, 'mode': 'constant', 'constant_values': 3},
+    {'array': 1, 'pad_width': 1, 'mode': 'constant', 'constant_values': 3},
+    {'array': [0, 1, 2, 3], 'pad_width': 1, 'mode': 'constant',
+     'constant_values': 3},
+    {'array': [0, 1, 2, 3], 'pad_width': [1, 2], 'mode': 'constant',
+     'constant_values': 3},
+    # mode='edge'
+    {'array': 1, 'pad_width': 1, 'mode': 'edge'},
+    {'array': [0, 1, 2, 3], 'pad_width': 1, 'mode': 'edge'},
+    {'array': [0, 1, 2, 3], 'pad_width': [1, 2], 'mode': 'edge'},
+    # mode='reflect'
+    {'array': 1, 'pad_width': 1, 'mode': 'reflect'},
+    {'array': [0, 1, 2, 3], 'pad_width': 1, 'mode': 'reflect'},
+    {'array': [0, 1, 2, 3], 'pad_width': [1, 2], 'mode': 'reflect'},
+)
+class TestPadSpecial(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_pad_special(self, xp):
+        array = xp.array(self.array)
+
+        if self.mode == 'constant':
+            a = xp.pad(array, self.pad_width, mode=self.mode,
+                       constant_values=self.constant_values)
+        elif self.mode in ['edge', 'reflect']:
+            a = xp.pad(array, self.pad_width, mode=self.mode)
+        return a
+
+
+@testing.parameterize(
+    {'array': [0, 1, 2, 3], 'pad_width': [-1, 1], 'mode': 'constant',
+     'kwargs': {'constant_values': 3}},
+    {'array': [0, 1, 2, 3], 'pad_width': [[3, 4], [5, 6]], 'mode': 'constant',
+     'kwargs': {'constant_values': 3}},
+    {'array': [0, 1, 2, 3], 'pad_width': [1], 'mode': 'constant',
+     'kwargs': {'notallowedkeyword': 3}},
+    # edge
+    {'array': [], 'pad_width': 1, 'mode': 'edge',
+     'kwargs': {}},
+    {'array': [0, 1, 2, 3], 'pad_width': [-1, 1], 'mode': 'edge',
+     'kwargs': {}},
+    {'array': [0, 1, 2, 3], 'pad_width': [[3, 4], [5, 6]], 'mode': 'edge',
+     'kwargs': {}},
+    {'array': [0, 1, 2, 3], 'pad_width': [1], 'mode': 'edge',
+     'kwargs': {'notallowedkeyword': 3}},
+    # mode='reflect'
+    {'array': [], 'pad_width': 1, 'mode': 'reflect',
+     'kwargs': {}},
+    {'array': [0, 1, 2, 3], 'pad_width': [-1, 1], 'mode': 'reflect',
+     'kwargs': {}},
+    {'array': [0, 1, 2, 3], 'pad_width': [[3, 4], [5, 6]], 'mode': 'reflect',
+     'kwargs': {}},
+    {'array': [0, 1, 2, 3], 'pad_width': [1], 'mode': 'reflect',
+     'kwargs': {'notallowedkeyword': 3}},
+)
+@testing.with_requires('numpy>=1.17')
+class TestPadValueError(unittest.TestCase):
+
+    def test_pad_failure(self):
+        for xp in (numpy, cupy):
+            array = xp.array(self.array)
+            with pytest.raises(ValueError):
+                xp.pad(array, self.pad_width, self.mode, **self.kwargs)
+
+
+@testing.parameterize(
+    {'array': [0, 1, 2, 3], 'pad_width': [], 'mode': 'constant',
+     'kwargs': {'constant_values': 3}},
+    # edge
+    {'array': [0, 1, 2, 3], 'pad_width': [], 'mode': 'edge',
+     'kwargs': {}},
+    # mode='reflect'
+    {'array': [0, 1, 2, 3], 'pad_width': [], 'mode': 'reflect',
+     'kwargs': {}},
+)
+class TestPadTypeError(unittest.TestCase):
+
+    def test_pad_failure(self):
+        for xp in (numpy, cupy):
+            array = xp.array(self.array)
+            with pytest.raises(TypeError):
+                xp.pad(array, self.pad_width, self.mode, **self.kwargs)
diff --git a/tests/cupy_tests/polynomial_tests/test_polynomial.py b/tests/cupy_tests/polynomial_tests/test_polynomial.py
new file mode 100644
index 0000000..8f60b17
--- /dev/null
+++ b/tests/cupy_tests/polynomial_tests/test_polynomial.py
@@ -0,0 +1,134 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestPolynomial(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_polyvander1(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        return xp.polynomial.polynomial.polyvander(a, 20)
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_polyvander2(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        return xp.polynomial.polynomial.polyvander(a, 10)
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_polyvander3(self, xp, dtype):
+        a = testing.shaped_random((100,), xp, dtype)
+        return xp.polynomial.polynomial.polyvander(a, 10)
+
+    @testing.for_all_dtypes()
+    def test_polyvander_negative_degree(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((10,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polynomial.polyvander(a, -3)
+
+    @testing.with_requires('numpy>=1.17')
+    @testing.for_all_dtypes()
+    def test_polyvander_non_integral_float_degree(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((10,), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polynomial.polynomial.polyvander(a, 2.6)
+
+    @testing.with_requires('numpy>=1.17')
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_polyvander_integral_float_degree(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        with testing.assert_warns(DeprecationWarning):
+            return xp.polynomial.polynomial.polyvander(a, 5.0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_polyvander_zeros(self, xp, dtype):
+        a = xp.zeros(10, dtype)
+        return xp.polynomial.polynomial.polyvander(a, 10)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_polyvander_ndim(self, xp, dtype):
+        a = testing.shaped_random((3, 2, 1), xp, dtype)
+        return xp.polynomial.polynomial.polyvander(a, 2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_polyvander_zero_dim(self, xp, dtype):
+        a = testing.shaped_random((), xp, dtype)
+        return xp.polynomial.polynomial.polyvander(a, 5)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_polycompanion(self, xp, dtype):
+        a = testing.shaped_random((1000,), xp, dtype)
+        return xp.polynomial.polynomial.polycompanion(a)
+
+    @testing.for_all_dtypes()
+    def test_polycompanion_zeros(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.zeros(10, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polynomial.polycompanion(a)
+
+    @testing.for_all_dtypes()
+    def test_polycompanion_zerosize(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.zeros((0,), dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polynomial.polycompanion(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_polycompanion_trailing_zeros(self, xp, dtype):
+        a = xp.array([3, 5, 7, 0, 0, 0], dtype)
+        return xp.polynomial.polynomial.polycompanion(a)
+
+    @testing.for_all_dtypes()
+    def test_polycompanion_single_value1(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.array([3, 0, 0, 0], dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polynomial.polycompanion(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_polycompanion_single_value2(self, xp, dtype):
+        a = xp.array([0, 0, 0, 3], dtype)
+        return xp.polynomial.polynomial.polycompanion(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_polycompanion_single_value3(self, xp, dtype):
+        a = xp.array([0, 3, 0, 0], dtype)
+        return xp.polynomial.polynomial.polycompanion(a)
+
+    @testing.for_all_dtypes()
+    def test_polycompanion_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((3, 4, 5), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polynomial.polycompanion(a)
+
+    @testing.for_all_dtypes()
+    def test_polycompanion_zero_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polynomial.polycompanion(a)
+
+    def test_polycompanion_nocommon_types(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((5,), xp, dtype=bool)
+            with pytest.raises(Exception):
+                xp.polynomial.polynomial.polycompanion(a)
diff --git a/tests/cupy_tests/polynomial_tests/test_polyutils.py b/tests/cupy_tests/polynomial_tests/test_polyutils.py
new file mode 100644
index 0000000..9b57a9b
--- /dev/null
+++ b/tests/cupy_tests/polynomial_tests/test_polyutils.py
@@ -0,0 +1,172 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(*testing.product({
+    'trim': [True, False]
+}))
+class TestAsSeries(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_as_series(self, xp, dtype):
+        a = testing.shaped_random((1000,), xp, dtype)
+        return xp.polynomial.polyutils.as_series(a, trim=self.trim)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_as_series_zeros(self, xp, dtype):
+        a = xp.zeros(10, dtype)
+        return xp.polynomial.polyutils.as_series(a, trim=self.trim)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_as_series_trailing_zeros(self, xp, dtype):
+        a = xp.array([3, 5, 7, 0, 0, 0], dtype)
+        return xp.polynomial.polyutils.as_series(a, trim=self.trim)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_as_series_list(self, xp, dtype):
+        a = [xp.array([3, 5, 7, -4, 1, 2]).astype(dtype)]
+        return xp.polynomial.polyutils.as_series(a, trim=self.trim)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_as_series_2dim(self, xp, dtype):
+        a = testing.shaped_random((4, 5), xp, dtype)
+        return xp.polynomial.polyutils.as_series(a, trim=self.trim)
+
+    @testing.for_all_dtypes()
+    def test_as_series_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((3, 4, 5), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polyutils.as_series(a, trim=self.trim)
+
+    def test_as_series_nocommon_types(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((5,), xp, dtype=bool)
+            with pytest.raises(ValueError):
+                xp.polynomial.polyutils.as_series(a, trim=self.trim)
+
+
+class TestTrimseq(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trimseq(self, xp, dtype):
+        a = testing.shaped_random((1000,), xp, dtype)
+        return xp.polynomial.polyutils.trimseq(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trimseq_zeros(self, xp, dtype):
+        a = xp.zeros(10, dtype)
+        b = xp.polynomial.polyutils.trimseq(a)
+        assert xp.shares_memory(a[:1], b) is True
+        return b
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trimseq_trailing_zeros(self, xp, dtype):
+        a = xp.array([1, 5, 2, 0, 0, 0], dtype)
+        return xp.polynomial.polyutils.trimseq(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_trimseq_zerosize(self, xp, dtype):
+        a = xp.zeros((0,), dtype)
+        return xp.polynomial.polyutils.trimseq(a)
+
+    @testing.for_all_dtypes()
+    def test_trimseq_zero_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp, dtype)
+            with pytest.raises(TypeError):
+                xp.polynomial.polyutils.trimseq(a)
+
+    @testing.for_all_dtypes()
+    def test_trimseq_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((3, 4, 5), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polyutils.trimseq(a)
+
+
+@testing.parameterize(*testing.product({
+    'tol': [0, 1e-3]
+}))
+class TestTrimcoef(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_trimcoef(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        return xp.polynomial.polyutils.trimcoef(a, dtype(self.tol))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_trimcoef_trailing_zeros(self, xp, dtype):
+        a = xp.array([1, 2, 0, 3, 0, 3e-4, 1e-3], dtype)
+        return xp.polynomial.polyutils.trimcoef(a, dtype(self.tol))
+
+    @testing.for_all_dtypes_combination(
+        names=['dtype1', 'dtype2'], no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_trimcoef_diff_types(self, xp, dtype1, dtype2):
+        a = xp.array([1, 2, 0, 3, 0], dtype1)
+        return xp.polynomial.polyutils.trimcoef(a, dtype2(self.tol))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_trimcoef_zeros(self, xp, dtype):
+        a = xp.zeros(10, dtype)
+        return xp.polynomial.polyutils.trimcoef(a, dtype(self.tol))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_trimcoef_almost_zeros(self, xp, dtype):
+        a = xp.array([1e-3, 1e-5, 1e-4], dtype)
+        return xp.polynomial.polyutils.trimcoef(a, dtype(self.tol))
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_trimcoef_zero_dim(self, xp, dtype):
+        a = testing.shaped_random((), xp, dtype)
+        return xp.polynomial.polyutils.trimcoef(a, dtype(self.tol))
+
+
+class TestTrimcoefInvalid(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_trimcoef_neg_tol(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polyutils.trimcoef(a, -1e-3)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_trimcoef_zero_size(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polyutils.trimcoef(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_trimcoef_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 3), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.polynomial.polyutils.trimcoef(a)
+
+    def test_trimcoef_bool(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((5,), xp, bool)
+            with pytest.raises(ValueError):
+                xp.polynomial.polyutils.trimcoef(a)
diff --git a/tests/cupy_tests/prof_tests/__init__.py b/tests/cupy_tests/prof_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/prof_tests/test_range.py b/tests/cupy_tests/prof_tests/test_range.py
new file mode 100644
index 0000000..2d3799e
--- /dev/null
+++ b/tests/cupy_tests/prof_tests/test_range.py
@@ -0,0 +1,98 @@
+import unittest
+from unittest import mock
+
+from cupy import cuda
+from cupy import prof
+
+
+@unittest.skipUnless(cuda.nvtx.available, 'nvtx is required for time_range')
+class TestTimeRange(unittest.TestCase):
+
+    def test_time_range(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePush')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            with prof.time_range('test:time_range', color_id=-1):
+                pass
+            push.assert_called_once_with('test:time_range', -1)
+            pop.assert_called_once_with()
+
+    def test_time_range_with_ARGB(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePushC')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            with prof.time_range('test:time_range_with_ARGB',
+                                 argb_color=0xFF00FF00):
+                pass
+            push.assert_called_once_with(
+                'test:time_range_with_ARGB', 0xFF00FF00)
+            pop.assert_called_once_with()
+
+    def test_time_range_err(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePush')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            try:
+                with prof.time_range('test:time_range_error', -1):
+                    raise Exception()
+            except Exception:
+                pass
+            push.assert_called_once_with('test:time_range_error', -1)
+            pop.assert_called_once_with()
+
+    def test_TimeRangeDecorator(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePush')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            @prof.TimeRangeDecorator()
+            def f():
+                pass
+            f()
+            # Default value of color id is -1
+            push.assert_called_once_with('f', -1)
+            pop.assert_called_once_with()
+
+    def test_TimeRangeDecorator_with_ARGB(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePushC')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            @prof.TimeRangeDecorator(argb_color=0xFFFF0000)
+            def f():
+                pass
+            f()
+            push.assert_called_once_with('f', 0xFFFF0000)
+            pop.assert_called_once_with()
+
+    def test_TimeRangeDecorator_err(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePush')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            @prof.TimeRangeDecorator()
+            def f():
+                raise Exception()
+            try:
+                f()
+            except Exception:
+                pass
+            # Default value of color id is -1
+            push.assert_called_once_with('f', -1)
+            pop.assert_called_once_with()
+
+
+class TestTimeRangeNVTXUnavailable(unittest.TestCase):
+
+    def setUp(self):
+        self.nvtx_available = cuda.nvtx.available
+        cuda.nvtx.available = False
+
+    def tearDown(self):
+        cuda.nvtx.available = self.nvtx_available
+
+    def test_time_range(self):
+        with self.assertRaises(RuntimeError):
+            with prof.time_range(''):
+                pass
+
+    def test_time_range_decorator(self):
+        with self.assertRaises(RuntimeError):
+            prof.TimeRangeDecorator()
diff --git a/tests/cupy_tests/random_tests/__init__.py b/tests/cupy_tests/random_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/random_tests/common_distributions.py b/tests/cupy_tests/random_tests/common_distributions.py
new file mode 100644
index 0000000..48672fd
--- /dev/null
+++ b/tests/cupy_tests/random_tests/common_distributions.py
@@ -0,0 +1,521 @@
+import functools
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+from cupy.testing import _condition
+
+
+def two_sample_Kolmogorov_Smirnov_test(observed1, observed2):
+    """Computes the Kolmogorov-Smirnov statistic on 2 samples
+
+    Unlike `scipy.stats.ks_2samp`, the returned p-value is not accurate
+    for large p.
+    """
+    assert observed1.dtype == observed2.dtype
+    n1, = observed1.shape
+    n2, = observed2.shape
+    assert n1 >= 100 and n2 >= 100
+    observed = numpy.concatenate([observed1, observed2])
+    indices = numpy.argsort(observed)
+    observed = observed[indices]  # sort
+    ds = numpy.cumsum(numpy.where(indices < n1, -n2, n1).astype(numpy.int64))
+    assert ds[-1] == 0
+    check = numpy.concatenate([observed[:-1] < observed[1:], [True]])
+    ds = ds[check]
+    d_plus = float(ds.max()) / (n1 * n2)
+    d_minus = -float(ds.min()) / (n1 * n2)
+    d = max(d_plus, d_minus)
+    # Approximate p = special.kolmogorov(d * numpy.sqrt(n1 * n2 / (n1 + n2)))
+    p = min(1.0, 2.0 * numpy.exp(-2.0 * d**2 * n1 * n2 / (n1 + n2)))
+    return d_plus, d_minus, p
+
+
+class BaseGeneratorTestCase(unittest.TestCase):
+
+    target_method = None
+
+    def get_rng(self, xp, seed):
+        pass
+
+    def set_rng_seed(self, seed):
+        pass
+
+    def setUp(self):
+        self.__seed = testing.generate_seed()
+        # rng will be a new or old generator API object
+        self.rng = self.get_rng(cupy, self.__seed)
+
+    def _get_generator_func(self, *args, **kwargs):
+        assert isinstance(self.target_method, str), (
+            'generate_method must be overridden')
+        f = getattr(self.rng, self.target_method)
+        return lambda: f(*args, **kwargs)
+
+    def _generate_check_repro(self, func, seed):
+        # Sample a random array while checking reproducibility
+        self.set_rng_seed(seed)
+        x = func()
+        self.set_rng_seed(seed)
+        y = func()
+        testing.assert_array_equal(
+            x, y,
+            'Randomly generated arrays with the same seed did not match')
+        return x
+
+    def generate(self, *args, **kwargs):
+        # Pick one sample from generator.
+        # Reproducibility is checked by repeating seed-and-sample cycle twice.
+        func = self._get_generator_func(*args, **kwargs)
+        return self._generate_check_repro(func, self.__seed)
+
+    def generate_many(self, *args, **kwargs):
+        # Pick many samples from generator.
+        # Reproducibility is checked only for the first sample,
+        # because it's very slow to set seed every time.
+        _count = kwargs.pop('_count', None)
+        assert _count is not None, '_count is required'
+        func = self._get_generator_func(*args, **kwargs)
+
+        if _count == 0:
+            return []
+
+        vals = [self._generate_check_repro(func, self.__seed)]
+        for _ in range(1, _count):
+            vals.append(func())
+        return vals
+
+    def check_ks(self, significance_level, cupy_len=100, numpy_len=1000):
+        return functools.partial(
+            self._check_ks, significance_level, cupy_len, numpy_len)
+
+    def _check_ks(
+            self, significance_level, cupy_len, numpy_len,
+            *args, **kwargs):
+        assert 'size' in kwargs
+
+        # cupy
+        func = self._get_generator_func(*args, **kwargs)
+        vals_cupy = func()
+        assert vals_cupy.size > 0
+        count = 1 + (cupy_len - 1) // vals_cupy.size
+        vals_cupy = [vals_cupy]
+        for _ in range(1, count):
+            vals_cupy.append(func())
+        vals_cupy = cupy.stack(vals_cupy).ravel()
+
+        # numpy
+        kwargs['size'] = numpy_len
+        dtype = kwargs.pop('dtype', None)
+        numpy_rng = self.get_rng(numpy, self.__seed)
+        vals_numpy = getattr(numpy_rng, self.target_method)(*args, **kwargs)
+        if dtype is not None:
+            vals_numpy = vals_numpy.astype(dtype, copy=False)
+
+        # test
+        d_plus, d_minus, p_value = \
+            two_sample_Kolmogorov_Smirnov_test(
+                cupy.asnumpy(vals_cupy), vals_numpy)
+        if p_value < significance_level:
+            message = '''Rejected null hypothesis:
+p: %f
+D+ (cupy < numpy): %f
+D- (cupy > numpy): %f''' % (p_value, d_plus, d_minus)
+            raise AssertionError(message)
+
+
+uniform_params = [
+    {'low': 1, 'high': 10.0, 'size': (3, 5)},
+    {'low': [1, 2], 'high': 3, 'size': None},
+    {'low': 20, 'high': 20.1, 'size': 1000}
+]
+
+
+class Uniform:
+    target_method = 'uniform'
+
+    def test_uniform(self):
+        low = self.low
+        if isinstance(low, list):
+            low = cupy.array(low)
+        high = self.high
+        if isinstance(high, list):
+            high = cupy.array(high)
+
+        result = self.generate(low, high, self.size)
+        assert cupy.all(result >= cupy.asarray(low).min())
+        assert cupy.all(result < cupy.asarray(high).max())
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_uniform_ks(self):
+        if (isinstance(self.low, list) or isinstance(self.high, list)):
+            self.skipTest('Stastical checks only for scalar args')
+        self.check_ks(0.05)(low=self.low, high=self.low, size=2000)
+
+
+beta_params = [
+    {'a': 1.0, 'b': 3.0},
+    {'a': 3.0, 'b': 3.0},
+    {'a': 3.0, 'b': 1.0},
+    {'a': [1.0, 3.0, 5.0, 6.0, 9.0], 'b': 7.0},
+    {'a': 5.0, 'b': [1.0, 5.0, 8.0, 1.0, 3.0]},
+    {'a': [8.0, 6.0, 2.0, 4.0, 7.0], 'b': [3.0, 1.0, 2.0, 8.0, 1.0]}]
+
+
+class Beta:
+
+    target_method = 'beta'
+
+    def test_beta(self):
+        a = self.a
+        b = self.b
+        if (isinstance(self.a, list) or isinstance(self.b, list)):
+            a = cupy.array(self.a)
+            b = cupy.array(self.b)
+        self.generate(a, b, size=(3, 5))
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_beta_ks(self):
+        if (isinstance(self.a, list) or isinstance(self.b, list)):
+            self.skipTest('Stastical checks only for scalar args')
+        self.check_ks(0.05)(a=self.a, b=self.b, size=2000)
+
+
+class StandardExponential:
+
+    target_method = 'standard_exponential'
+
+    def test_standard_exponential(self):
+        self.generate(size=(3, 2))
+
+    @testing.slow
+    @_condition.repeat(10)
+    def test_standard_exponential_isfinite(self):
+        x = self.generate(size=10**7)
+        assert cupy.isfinite(x).all()
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_standard_exponential_ks(self, dtype):
+        self.check_ks(0.05)(size=2000, dtype=dtype)
+
+
+standard_gamma_params = [
+    {'shape': 0.5},
+    {'shape': 1.0},
+    {'shape': 3.0}]
+
+
+class StandardGamma:
+
+    target_method = 'standard_gamma'
+
+    def test_standard_gamma(self):
+        self.generate(shape=self.shape, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_standard_gamma_ks(self, dtype):
+        self.check_ks(0.05)(
+            shape=self.shape, size=2000, dtype=dtype)
+
+
+standard_normal_params = [
+    {'size': None},
+    {'size': (1, 2, 3)},
+    {'size': 3},
+    {'size': (1000, 1000)},
+    {'size': (3, 3)},
+    {'size': ()}]
+
+
+class StandardNormal:
+
+    target_method = 'standard_normal'
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_normal_ks(self, dtype):
+        self.check_ks(0.05)(size=self.size, dtype=dtype)
+
+
+exponential_params = [
+    {'scale': 0.5},
+    {'scale': 1},
+    {'scale': 10}]
+
+
+class Exponential:
+
+    target_method = 'exponential'
+
+    def test_exponential(self):
+        self.generate(scale=self.scale, size=(3, 2))
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_exponential_ks(self):
+        self.check_ks(0.05)(
+            self.scale, size=2000)
+
+
+poisson_params = [
+    {'lam': 1.0},
+    {'lam': 3.0},
+    {'lam': 10.0}]
+
+
+class Poisson:
+
+    target_method = 'poisson'
+
+    def test_poisson(self):
+        self.generate(lam=self.lam, size=(3, 2))
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_poisson_ks(self):
+        self.check_ks(0.05)(
+            lam=self.lam, size=2000)
+
+    def test_poisson_large(self):
+        self.generate(lam=self.lam, size=(1000, 1000))
+
+
+gamma_params = [
+    {'shape': 0.5, 'scale': 0.5},
+    {'shape': 1.0, 'scale': 0.5},
+    {'shape': 3.0, 'scale': 0.5},
+    {'shape': 0.5, 'scale': 1.0},
+    {'shape': 1.0, 'scale': 1.0},
+    {'shape': 3.0, 'scale': 1.0},
+    {'shape': 0.5, 'scale': 3.0},
+    {'shape': 1.0, 'scale': 3.0},
+    {'shape': 3.0, 'scale': 3.0}]
+
+
+class Gamma:
+
+    target_method = 'gamma'
+
+    def test_gamma_1(self):
+        self.generate(shape=self.shape, scale=self.scale, size=(3, 2))
+
+    def test_gamma_2(self):
+        self.generate(shape=self.shape, size=(3, 2))
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_gamma_ks(self):
+        self.check_ks(0.05)(
+            self.shape, self.scale, size=2000)
+
+
+binomial_params = [
+    {'n': 2, 'p': 0.5},
+    {'n': 5, 'p': 0.5},
+    {'n': 10, 'p': 0.5},
+    {'n': 2, 'p': 0.1},
+    {'n': 5, 'p': 0.1},
+    {'n': 10, 'p': 0.1},
+    {'n': 2, 'p': 1.0},
+    {'n': 2, 'p': 1.0},
+    {'n': 2, 'p': 1.0}]
+
+
+class Binomial:
+
+    target_method = 'binomial'
+
+    def test_binomial(self):
+        self.generate(n=self.n, p=self.p, size=(3, 2))
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_binomial_ks(self):
+        self.check_ks(0.05)(self.n, self.p, size=2000)
+
+
+geometric_params = [
+    {'p': 0.5},
+    {'p': 0.1},
+    {'p': 1.0},
+    {'p': [0.1, 0.5]},
+]
+
+
+class Geometric:
+
+    target_method = 'geometric'
+
+    def test_geometric(self):
+        p = self.p
+        if not isinstance(self.p, float):
+            p = cupy.array(self.p)
+        self.generate(p=p, size=(3, 2))
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_geometric_ks(self):
+        if not isinstance(self.p, float):
+            self.skipTest('Statistical checks only for scalar `p`')
+        self.check_ks(0.05)(
+            p=self.p, size=2000)
+
+
+hypergeometric_params = [
+    {'ngood': 5, 'nbad': 5, 'nsample': 5},
+    {'ngood': 10, 'nbad': 10, 'nsample': 10},
+    {'ngood': 100, 'nbad': 2, 'nsample': 10},
+    {'ngood': [0, 5, 8], 'nbad': [5, 0, 3], 'nsample': [2, 1, 8]},
+    {'ngood': [1, 4, 2, 7, 6], 'nbad': 5.0, 'nsample': [2, 7, 4, 6, 5]},
+]
+
+
+class Hypergeometric:
+
+    target_method = 'hypergeometric'
+
+    def test_hypergeometric(self):
+        ngood = self.ngood
+        nbad = self.nbad
+        nsample = self.nsample
+        if (isinstance(self.ngood, list) or isinstance(self.nbad, list)
+                or isinstance(self.nsample, list)):
+            ngood = cupy.array(self.ngood)
+            nbad = cupy.array(self.nbad)
+            nsample = cupy.array(self.nsample)
+        self.generate(ngood, nbad, nsample)
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_hypergeometric_ks(self):
+        if (isinstance(self.ngood, list) or isinstance(self.nbad, list)
+                or isinstance(self.nsample, list)):
+            self.skipTest('Stastical checks only for scalar args')
+        self.check_ks(0.05)(self.ngood, self.nbad, self.nsample, size=2000)
+
+
+power_params = [
+    {'a': 0.5},
+    {'a': 1},
+    {'a': 5},
+    {'a': [0.8, 0.7, 1, 2, 5]},
+]
+
+
+class Power:
+
+    target_method = 'power'
+
+    def test_power(self):
+        a = self.a
+        if not isinstance(self.a, float):
+            a = cupy.array(self.a)
+        self.generate(a=a)
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_power_ks(self):
+        if not isinstance(self.a, float):
+            self.skipTest('Statistical checks only for scalar `a`')
+        self.check_ks(0.05)(
+            a=self.a, size=2000)
+
+
+logseries_params = [
+    {'p': 0.5},
+    {'p': 0.1},
+    {'p': 0.9},
+    {'p': [0.8, 0.7]},
+]
+
+
+class Logseries:
+
+    target_method = 'logseries'
+
+    def test_logseries(self):
+        p = self.p
+        if not isinstance(self.p, float):
+            p = cupy.array(self.p)
+        self.generate(p=p, size=(3, 2))
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_geometric_ks(self):
+        if not isinstance(self.p, float):
+            self.skipTest('Statistical checks only for scalar `p`')
+        self.check_ks(0.05)(p=self.p, size=2000)
+
+
+chisquare_params = [
+    {'df': 1.0},
+    {'df': 3.0},
+    {'df': 10.0},
+    {'df': [2, 5, 8]},
+]
+
+
+class Chisquare:
+
+    target_method = 'chisquare'
+
+    def test_chisquare(self):
+        df = self.df
+        if not isinstance(self.df, float):
+            df = cupy.array(self.df)
+        self.generate(df=df)
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_chisquare_ks(self):
+        if not isinstance(self.df, float):
+            self.skipTest('Statistical checks only for scalar `df`')
+        self.check_ks(0.05)(
+            df=self.df, size=2000)
+
+
+f_params = [
+    {'dfnum': 1.0, 'dfden': 3.0},
+    {'dfnum': 3.0, 'dfden': 3.0},
+    {'dfnum': 3.0, 'dfden': 1.0},
+    {'dfnum': [1.0, 3.0, 3.0], 'dfden': [3.0, 3.0, 1.0]},
+]
+
+
+class F:
+
+    target_method = 'f'
+
+    def test_f(self):
+        dfnum = self.dfnum
+        dfden = self.dfden
+        if isinstance(self.dfnum, list) or isinstance(self.dfden, list):
+            dfnum = cupy.array(self.dfnum)
+            dfden = cupy.array(self.dfden)
+        self.generate(dfnum, dfden)
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_f_ks(self):
+        if isinstance(self.dfnum, list) or isinstance(self.dfden, list):
+            self.skipTest('Stastical checks only for scalar args')
+        self.check_ks(0.05)(self.dfnum, self.dfden, size=2000)
+
+
+dirichlet_params = [
+    {'alpha': 5},
+    {'alpha': 1},
+    {'alpha': [2, 5, 8]}
+]
+
+
+class Dirichlet:
+    target_method = 'dirichlet'
+
+    def test_dirichlet(self):
+        alpha = self.alpha
+        if not isinstance(self.alpha, float):
+            alpha = cupy.array(self.alpha)
+        self.generate(alpha=alpha, size=(3, 2))
+
+    def test_dirichlet_int_shape(self):
+        alpha = self.alpha
+        if not isinstance(self.alpha, int):
+            alpha = cupy.array(self.alpha)
+        self.generate(alpha=alpha, size=5)
+
+    # TODO(kataoka): add distribution test
diff --git a/tests/cupy_tests/random_tests/test_bit_generator.py b/tests/cupy_tests/random_tests/test_bit_generator.py
new file mode 100644
index 0000000..ff0d62b
--- /dev/null
+++ b/tests/cupy_tests/random_tests/test_bit_generator.py
@@ -0,0 +1,75 @@
+import unittest
+import pytest
+
+import numpy
+
+import cupy
+from cupy import random
+from cupy import testing
+
+
+class BitGeneratorTestCase:
+
+    def setUp(self):
+        self.seed = testing.generate_seed()
+
+    def check_seed(self, seed):
+        bg1 = self.bg(seed)
+        bg2 = self.bg(seed)
+        bg3 = self.bg(None)
+
+        xs1 = bg1.random_raw(10)
+        xs2 = bg2.random_raw(10)
+        xs3 = bg3.random_raw(10)
+
+        # Random state must be reproducible
+        assert cupy.array_equal(xs1, xs2)
+        # Random state must be initialized randomly with seed=None
+        assert not cupy.array_equal(xs1, xs3)
+
+    @testing.for_int_dtypes(no_bool=True)
+    def test_seed_not_none(self, dtype):
+        self.check_seed(dtype(0))
+
+    @testing.for_dtypes([numpy.complex_])
+    def test_seed_invalid_type_complex(self, dtype):
+        with self.assertRaises(TypeError):
+            self.bg(dtype(0))
+
+    @testing.for_float_dtypes()
+    def test_seed_invalid_type_float(self, dtype):
+        with self.assertRaises(TypeError):
+            self.bg(dtype(0))
+
+    def test_array_seed(self):
+        self.check_seed(numpy.random.randint(0, 2**31, size=10))
+
+
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestBitGeneratorXORWOW(BitGeneratorTestCase, unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+        self.bg = random._bit_generator.XORWOW
+
+
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestBitGeneratorMRG32k3a(BitGeneratorTestCase, unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+        self.bg = random._bit_generator.MRG32k3a
+
+
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                    reason='HIP does not support this')
+class TestBitGeneratorPhilox4x3210(BitGeneratorTestCase, unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+        self.bg = random._bit_generator.Philox4x3210
diff --git a/tests/cupy_tests/random_tests/test_distributions.py b/tests/cupy_tests/random_tests/test_distributions.py
new file mode 100644
index 0000000..e605a57
--- /dev/null
+++ b/tests/cupy_tests/random_tests/test_distributions.py
@@ -0,0 +1,852 @@
+import numpy
+import pytest
+
+import cupy
+from cupy.random import _distributions
+from cupy import testing
+
+
+_regular_float_dtypes = (numpy.float64, numpy.float32)
+_float_dtypes = _regular_float_dtypes + (numpy.float16,)
+_signed_dtypes = tuple(numpy.dtype(i).type for i in 'bhilq')
+_unsigned_dtypes = tuple(numpy.dtype(i).type for i in 'BHILQ')
+_int_dtypes = _signed_dtypes + _unsigned_dtypes
+
+
+class RandomDistributionsTestCase:
+    def check_distribution(self, dist_name, params, dtype):
+        cp_params = {k: cupy.asarray(params[k]) for k in params}
+        np_out = numpy.asarray(
+            getattr(numpy.random, dist_name)(size=self.shape, **params),
+            dtype)
+        cp_out = getattr(_distributions, dist_name)(
+            size=self.shape, dtype=dtype, **cp_params)
+        assert cp_out.shape == np_out.shape
+        assert cp_out.dtype == np_out.dtype
+
+    def check_generator_distribution(self, dist_name, params, dtype):
+        cp_params = {k: cupy.asarray(params[k]) for k in params}
+        np_gen = numpy.random.default_rng(0)
+        cp_gen = cupy.random.default_rng(0)
+        np_out = numpy.asarray(
+            getattr(np_gen, dist_name)(size=self.shape, **params))
+        cp_out = getattr(cp_gen, dist_name)(
+            size=self.shape, **cp_params)
+        assert cp_out.shape == np_out.shape
+        assert cp_out.dtype == np_out.dtype
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'a_shape': [(), (3, 2)],
+    'b_shape': [(), (3, 2)],
+    'dtype': _float_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsBeta(RandomDistributionsTestCase):
+
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['a_dtype', 'b_dtype'])
+    def test_beta(self, a_dtype, b_dtype):
+        a = numpy.full(self.a_shape, 3, dtype=a_dtype)
+        b = numpy.full(self.b_shape, 3, dtype=b_dtype)
+        self.check_distribution('beta',
+                                {'a': a, 'b': b}, self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'n_shape': [(), (3, 2)],
+    'p_shape': [(), (3, 2)],
+    'dtype': _int_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsBinomial(RandomDistributionsTestCase):
+
+    @cupy.testing.for_signed_dtypes('n_dtype')
+    @cupy.testing.for_float_dtypes('p_dtype')
+    def test_binomial(self, n_dtype, p_dtype):
+        if numpy.dtype('l') == numpy.int32 and n_dtype == numpy.int64:
+            pytest.skip('n must be able to cast to long')
+        n = numpy.full(self.n_shape, 5, dtype=n_dtype)
+        p = numpy.full(self.p_shape, 0.5, dtype=p_dtype)
+        self.check_distribution('binomial',
+                                {'n': n, 'p': p}, self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'df_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsChisquare:
+
+    def check_distribution(self, dist_func, df_dtype, dtype):
+        df = cupy.full(self.df_shape, 5, dtype=df_dtype)
+        out = dist_func(df, self.shape, dtype)
+        assert self.shape == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_float_dtypes('df_dtype')
+    @cupy.testing.for_float_dtypes('dtype')
+    def test_chisquare(self, df_dtype, dtype):
+        self.check_distribution(_distributions.chisquare, df_dtype, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2, 3), (3, 2, 3)],
+    'alpha_shape': [(3,)],
+})
+)
+class TestDistributionsDirichlet(RandomDistributionsTestCase):
+
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['alpha_dtype', 'dtype'])
+    def test_dirichlet(self, alpha_dtype, dtype):
+        alpha = numpy.ones(self.alpha_shape, dtype=alpha_dtype)
+        self.check_distribution('dirichlet',
+                                {'alpha': alpha}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'scale_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsExponential(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('scale_dtype')
+    def test_exponential(self, scale_dtype, dtype):
+        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
+        self.check_distribution('exponential',
+                                {'scale': scale}, dtype)
+
+
+class TestDistributionsExponentialError(RandomDistributionsTestCase):
+
+    def test_negative_scale(self):
+        scale = cupy.array([2, -1, 3], dtype=numpy.float32)
+        with pytest.raises(ValueError):
+            cupy.random.exponential(scale)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'dfnum_shape': [(), (3, 2)],
+    'dfden_shape': [(), (3, 2)],
+    'dtype': _float_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsF:
+
+    def check_distribution(self, dist_func, dfnum_dtype, dfden_dtype, dtype):
+        dfnum = cupy.ones(self.dfnum_shape, dtype=dfnum_dtype)
+        dfden = cupy.ones(self.dfden_shape, dtype=dfden_dtype)
+        out = dist_func(dfnum, dfden, self.shape, dtype)
+        assert self.shape == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_float_dtypes('dfnum_dtype')
+    @cupy.testing.for_float_dtypes('dfden_dtype')
+    def test_f(self, dfnum_dtype, dfden_dtype):
+        self.check_distribution(_distributions.f,
+                                dfnum_dtype, dfden_dtype, self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'shape_shape': [(), (3, 2)],
+    'scale_shape': [(), (3, 2)],
+    'dtype': _float_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsGamma:
+
+    def check_distribution(self, dist_func, shape_dtype, scale_dtype,
+                           dtype=None):
+        shape = cupy.ones(self.shape_shape, dtype=shape_dtype)
+        scale = cupy.ones(self.scale_shape, dtype=scale_dtype)
+        if dtype is None:
+            out = dist_func(shape, scale, self.shape)
+        else:
+            out = dist_func(shape, scale, self.shape, dtype)
+        out_shape = self.shape
+        if self.shape is None:
+            out_shape = shape.shape if shape.shape != () else scale.shape
+        assert out_shape == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['shape_dtype', 'scale_dtype'])
+    def test_gamma_legacy(self, shape_dtype, scale_dtype):
+        self.check_distribution(_distributions.gamma,
+                                shape_dtype, scale_dtype, self.dtype)
+
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['shape_dtype', 'scale_dtype'])
+    @pytest.mark.skipif(
+        cupy.cuda.runtime.is_hip
+        and (int(str(cupy.cuda.runtime.runtimeGetVersion())[:3]) < 403),
+        reason="HIP<4.3 not supported ",
+    )
+    def test_gamma_generator(self, shape_dtype, scale_dtype):
+        self.check_distribution(cupy.random.default_rng().gamma,
+                                shape_dtype, scale_dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'p_shape': [(), (3, 2)],
+    'dtype': _int_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsGeometric:
+
+    def check_distribution(self, dist_func, p_dtype, dtype):
+        p = 0.5 * cupy.ones(self.p_shape, dtype=p_dtype)
+        out = dist_func(p, self.shape, dtype)
+        assert self.shape == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_float_dtypes('p_dtype')
+    def test_geometric(self, p_dtype):
+        self.check_distribution(_distributions.geometric,
+                                p_dtype, self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'loc_shape': [(), (3, 2)],
+    'scale_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsGumbel(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['loc_dtype', 'scale_dtype'])
+    def test_gumbel(self, loc_dtype, scale_dtype, dtype):
+        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
+        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
+        self.check_distribution('gumbel',
+                                {'loc': loc, 'scale': scale}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'ngood_shape': [(), (3, 2)],
+    'nbad_shape': [(), (3, 2)],
+    'nsample_shape': [(), (3, 2)],
+    'nsample_dtype': [numpy.int32, numpy.int64],  # to escape timeout
+    'dtype': [numpy.int32, numpy.int64],  # to escape timeout
+})
+)
+class TestDistributionsHyperGeometric:
+
+    def check_distribution(self, dist_func, ngood_dtype, nbad_dtype,
+                           nsample_dtype, dtype):
+        ngood = cupy.ones(self.ngood_shape, dtype=ngood_dtype)
+        nbad = cupy.ones(self.nbad_shape, dtype=nbad_dtype)
+        nsample = cupy.ones(self.nsample_shape, dtype=nsample_dtype)
+        out = dist_func(ngood, nbad, nsample, self.shape, dtype)
+        assert self.shape == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_dtypes_combination(
+        [numpy.int32, numpy.int64], names=['ngood_dtype', 'nbad_dtype'])
+    def test_hypergeometric(self, ngood_dtype, nbad_dtype):
+        self.check_distribution(_distributions.hypergeometric, ngood_dtype,
+                                nbad_dtype, self.nsample_dtype, self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'loc_shape': [(), (3, 2)],
+    'scale_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsuLaplace(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['loc_dtype', 'scale_dtype'])
+    def test_laplace(self, loc_dtype, scale_dtype, dtype):
+        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
+        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
+        self.check_distribution('laplace',
+                                {'loc': loc, 'scale': scale}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'loc_shape': [(), (3, 2)],
+    'scale_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsLogistic(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['loc_dtype', 'scale_dtype'])
+    def test_logistic(self, loc_dtype, scale_dtype, dtype):
+        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
+        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
+        self.check_distribution('logistic',
+                                {'loc': loc, 'scale': scale}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'mean_shape': [(), (3, 2)],
+    'sigma_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsLognormal(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['mean_dtype', 'sigma_dtype'])
+    def test_lognormal(self, mean_dtype, sigma_dtype, dtype):
+        mean = numpy.ones(self.mean_shape, dtype=mean_dtype)
+        sigma = numpy.ones(self.sigma_shape, dtype=sigma_dtype)
+        self.check_distribution('lognormal',
+                                {'mean': mean, 'sigma': sigma}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'p_shape': [()],
+})
+)
+class TestDistributionsLogseries(RandomDistributionsTestCase):
+
+    @cupy.testing.for_dtypes([numpy.int64, numpy.int32], 'dtype')
+    @cupy.testing.for_float_dtypes('p_dtype', no_float16=True)
+    def test_logseries(self, p_dtype, dtype):
+        p = numpy.full(self.p_shape, 0.5, dtype=p_dtype)
+        self.check_distribution('logseries',
+                                {'p': p}, dtype)
+
+    @cupy.testing.for_dtypes([numpy.int64, numpy.int32], 'dtype')
+    @cupy.testing.for_float_dtypes('p_dtype', no_float16=True)
+    def test_logseries_for_invalid_p(self, p_dtype, dtype):
+        with pytest.raises(ValueError):
+            cp_params = {'p': cupy.zeros(self.p_shape, dtype=p_dtype)}
+            _distributions.logseries(size=self.shape, dtype=dtype, **cp_params)
+        with pytest.raises(ValueError):
+            cp_params = {'p': cupy.ones(self.p_shape, dtype=p_dtype)}
+            _distributions.logseries(size=self.shape, dtype=dtype, **cp_params)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'd': [2, 4],
+})
+)
+class TestDistributionsMultivariateNormal:
+
+    def check_distribution(self, dist_func, mean_dtype, cov_dtype, dtype):
+        mean = cupy.zeros(self.d, dtype=mean_dtype)
+        cov = cupy.random.normal(size=(self.d, self.d), dtype=cov_dtype)
+        cov = cov.T.dot(cov)
+        out = dist_func(mean, cov, self.shape, dtype=dtype)
+        assert self.shape+(self.d,) == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('mean_dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('cov_dtype', no_float16=True)
+    def test_normal(self, mean_dtype, cov_dtype, dtype):
+        self.check_distribution(_distributions.multivariate_normal,
+                                mean_dtype, cov_dtype, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'n_shape': [(), (3, 2)],
+    'p_shape': [(), (3, 2)],
+    'dtype': _int_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsNegativeBinomial(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('n_dtype')
+    @cupy.testing.for_float_dtypes('p_dtype')
+    def test_negative_binomial(self, n_dtype, p_dtype):
+        n = numpy.full(self.n_shape, 5, dtype=n_dtype)
+        p = numpy.full(self.p_shape, 0.5, dtype=p_dtype)
+        self.check_distribution('negative_binomial',
+                                {'n': n, 'p': p}, self.dtype)
+
+    @cupy.testing.for_float_dtypes('n_dtype')
+    @cupy.testing.for_float_dtypes('p_dtype')
+    def test_negative_binomial_for_noninteger_n(self, n_dtype, p_dtype):
+        n = numpy.full(self.n_shape, 5.5, dtype=n_dtype)
+        p = numpy.full(self.p_shape, 0.5, dtype=p_dtype)
+        self.check_distribution('negative_binomial',
+                                {'n': n, 'p': p}, self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'df_shape': [(), (3, 2)],
+    'nonc_shape': [(), (3, 2)],
+    'dtype': _int_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsNoncentralChisquare(RandomDistributionsTestCase):
+
+    @cupy.testing.for_dtypes_combination(
+        _regular_float_dtypes, names=['df_dtype', 'nonc_dtype'])
+    def test_noncentral_chisquare(self, df_dtype, nonc_dtype):
+        df = numpy.full(self.df_shape, 1, dtype=df_dtype)
+        nonc = numpy.full(self.nonc_shape, 1, dtype=nonc_dtype)
+        self.check_distribution('noncentral_chisquare',
+                                {'df': df, 'nonc': nonc}, self.dtype)
+
+    @cupy.testing.for_float_dtypes('param_dtype', no_float16=True)
+    def test_noncentral_chisquare_for_invalid_params(self, param_dtype):
+        df = cupy.full(self.df_shape, -1, dtype=param_dtype)
+        nonc = cupy.full(self.nonc_shape, 1, dtype=param_dtype)
+        with pytest.raises(ValueError):
+            _distributions.noncentral_chisquare(
+                df, nonc, size=self.shape, dtype=self.dtype)
+
+        df = cupy.full(self.df_shape, 1, dtype=param_dtype)
+        nonc = cupy.full(self.nonc_shape, -1, dtype=param_dtype)
+        with pytest.raises(ValueError):
+            _distributions.noncentral_chisquare(
+                df, nonc, size=self.shape, dtype=self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'dfnum_shape': [(), (3, 2)],
+    'dfden_shape': [(), (3, 2)],
+    'nonc_shape': [(), (3, 2)],
+    'dtype': _int_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsNoncentralF(RandomDistributionsTestCase):
+
+    @cupy.testing.for_dtypes_combination(
+        _regular_float_dtypes,
+        names=['dfnum_dtype', 'dfden_dtype', 'nonc_dtype'])
+    def test_noncentral_f(self, dfnum_dtype, dfden_dtype, nonc_dtype):
+        dfnum = numpy.full(self.dfnum_shape, 1, dtype=dfnum_dtype)
+        dfden = numpy.full(self.dfden_shape, 1, dtype=dfden_dtype)
+        nonc = numpy.full(self.nonc_shape, 1, dtype=nonc_dtype)
+        self.check_distribution('noncentral_f',
+                                {'dfnum': dfnum, 'dfden': dfden, 'nonc': nonc},
+                                self.dtype)
+
+    @cupy.testing.for_float_dtypes('param_dtype', no_float16=True)
+    def test_noncentral_f_for_invalid_params(self, param_dtype):
+        dfnum = numpy.full(self.dfnum_shape, -1, dtype=param_dtype)
+        dfden = numpy.full(self.dfden_shape, 1, dtype=param_dtype)
+        nonc = numpy.full(self.nonc_shape, 1, dtype=param_dtype)
+        with pytest.raises(ValueError):
+            _distributions.noncentral_f(
+                dfnum, dfden, nonc, size=self.shape, dtype=self.dtype)
+
+        dfnum = numpy.full(self.dfnum_shape, 1, dtype=param_dtype)
+        dfden = numpy.full(self.dfden_shape, -1, dtype=param_dtype)
+        nonc = numpy.full(self.nonc_shape, 1, dtype=param_dtype)
+        with pytest.raises(ValueError):
+            _distributions.noncentral_f(
+                dfnum, dfden, nonc, size=self.shape, dtype=self.dtype)
+
+        dfnum = numpy.full(self.dfnum_shape, 1, dtype=param_dtype)
+        dfden = numpy.full(self.dfden_shape, 1, dtype=param_dtype)
+        nonc = numpy.full(self.nonc_shape, -1, dtype=param_dtype)
+        with pytest.raises(ValueError):
+            _distributions.noncentral_f(
+                dfnum, dfden, nonc, size=self.shape, dtype=self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'loc_shape': [(), (3, 2)],
+    'scale_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsNormal(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['loc_dtype', 'scale_dtype'])
+    def test_normal(self, loc_dtype, scale_dtype, dtype):
+        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
+        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
+        self.check_distribution('normal',
+                                {'loc': loc, 'scale': scale}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'a_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsPareto:
+
+    def check_distribution(self, dist_func, a_dtype, dtype):
+        a = cupy.ones(self.a_shape, dtype=a_dtype)
+        out = dist_func(a, self.shape, dtype)
+        if self.shape is not None:
+            assert self.shape == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('a_dtype')
+    def test_pareto(self, a_dtype, dtype):
+        self.check_distribution(_distributions.pareto,
+                                a_dtype, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'lam_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsPoisson:
+
+    def check_distribution(self, dist_func, lam_dtype, dtype=None):
+        lam = cupy.full(self.lam_shape, 5, dtype=lam_dtype)
+        if dtype is not None:
+            out = dist_func(lam, self.shape, dtype)
+            assert out.dtype == dtype
+        else:
+            out = dist_func(lam, self.shape)
+        if self.shape is not None:
+            assert self.shape == out.shape
+        else:
+            assert lam.shape == out.shape
+
+    @cupy.testing.for_int_dtypes('dtype')
+    @cupy.testing.for_float_dtypes('lam_dtype')
+    def test_poisson_legacy(self, lam_dtype, dtype):
+        self.check_distribution(_distributions.poisson, lam_dtype, dtype)
+
+    @cupy.testing.for_float_dtypes('lam_dtype')
+    @pytest.mark.skipif(
+        cupy.cuda.runtime.is_hip
+        and (int(str(cupy.cuda.runtime.runtimeGetVersion())[:3]) < 403),
+        reason="HIP<4.3 not supported ",
+    )
+    def test_poisson_generator(self, lam_dtype):
+        self.check_distribution(cupy.random.default_rng(0).poisson,
+                                lam_dtype)
+
+
+class TestDistributionsPoissonInvalid:
+
+    @pytest.mark.skipif(
+        cupy.cuda.runtime.is_hip
+        and (int(str(cupy.cuda.runtime.runtimeGetVersion())[:3]) < 403),
+        reason="HIP<4.3 not supported ",
+    )
+    def test_none_lam_generator(self):
+        with pytest.raises(TypeError):
+            cupy.random.default_rng(0).poisson(None)
+
+    def test_none_lam_legacy(self):
+        with pytest.raises(ValueError):
+            _distributions.poisson(None)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'a_shape': [()],
+})
+)
+class TestDistributionsPower(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('a_dtype')
+    def test_power(self, a_dtype, dtype):
+        a = numpy.full(self.a_shape, 0.5, dtype=a_dtype)
+        self.check_distribution('power', {'a': a}, dtype)
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('a_dtype')
+    def test_power_for_negative_a(self, a_dtype, dtype):
+        a = numpy.full(self.a_shape, -0.5, dtype=a_dtype)
+        with pytest.raises(ValueError):
+            cp_params = {'a': cupy.asarray(a)}
+            getattr(_distributions, 'power')(
+                size=self.shape, dtype=dtype, **cp_params)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'scale_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsRayleigh(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('scale_dtype')
+    def test_rayleigh(self, scale_dtype, dtype):
+        scale = numpy.full(self.scale_shape, 3, dtype=scale_dtype)
+        self.check_distribution('rayleigh',
+                                {'scale': scale}, dtype)
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('scale_dtype')
+    def test_rayleigh_for_zero_scale(self, scale_dtype, dtype):
+        scale = numpy.zeros(self.scale_shape, dtype=scale_dtype)
+        self.check_distribution('rayleigh',
+                                {'scale': scale}, dtype)
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('scale_dtype')
+    def test_rayleigh_for_negative_scale(self, scale_dtype, dtype):
+        scale = numpy.full(self.scale_shape, -0.5, dtype=scale_dtype)
+        with pytest.raises(ValueError):
+            cp_params = {'scale': cupy.asarray(scale)}
+            _distributions.rayleigh(size=self.shape, dtype=dtype, **cp_params)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+})
+)
+class TestDistributionsStandardCauchy(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    def test_standard_cauchy(self, dtype):
+        self.check_distribution('standard_cauchy', {}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+})
+)
+class TestDistributionsStandardExponential(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    def test_standard_exponential(self, dtype):
+        self.check_distribution('standard_exponential', {}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+    'shape_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsStandardGamma(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('shape_dtype')
+    def test_standard_gamma_legacy(self, shape_dtype, dtype):
+        shape = numpy.ones(self.shape_shape, dtype=shape_dtype)
+        self.check_distribution('standard_gamma',
+                                {'shape': shape}, dtype)
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('shape_dtype')
+    @pytest.mark.skipif(
+        cupy.cuda.runtime.is_hip
+        and (int(str(cupy.cuda.runtime.runtimeGetVersion())[:3]) < 403),
+        reason="HIP<4.3 not supported ",
+    )
+    def test_standard_gamma_generator(self, shape_dtype, dtype):
+        shape = numpy.ones(self.shape_shape, dtype=shape_dtype)
+        self.check_generator_distribution('standard_gamma',
+                                          {'shape': shape},
+                                          dtype)
+
+
+class TestDistributionsStandardGammaInvalid(RandomDistributionsTestCase):
+
+    @pytest.mark.skipif(
+        cupy.cuda.runtime.is_hip
+        and (int(str(cupy.cuda.runtime.runtimeGetVersion())[:3]) < 403),
+        reason="HIP<4.3 not supported ",
+    )
+    def test_none_shape_generator(self):
+        with pytest.raises(TypeError):
+            cupy.random.default_rng(0).standard_gamma(None)
+
+    def test_none_shape_legacy(self):
+        with pytest.raises(ValueError):
+            _distributions.standard_gamma(None)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2), None],
+})
+)
+class TestDistributionsStandardNormal(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    def test_standard_normal(self, dtype):
+        self.check_distribution('standard_normal', {}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'df_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsStandardT:
+
+    def check_distribution(self, dist_func, df_dtype, dtype):
+        df = cupy.ones(self.df_shape, dtype=df_dtype)
+        out = dist_func(df, self.shape, dtype)
+        assert self.shape == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('df_dtype')
+    def test_standard_t(self, df_dtype, dtype):
+        self.check_distribution(_distributions.standard_t, df_dtype, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'left_shape': [(), (3, 2)],
+    'mode_shape': [(), (3, 2)],
+    'right_shape': [(), (3, 2)],
+    'dtype': _regular_float_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsTriangular(RandomDistributionsTestCase):
+
+    @cupy.testing.for_dtypes_combination(
+        _regular_float_dtypes,
+        names=['left_dtype', 'mode_dtype', 'right_dtype'])
+    def test_triangular(self, left_dtype, mode_dtype, right_dtype):
+        left = numpy.full(self.left_shape, -1, dtype=left_dtype)
+        mode = numpy.full(self.mode_shape, 0, dtype=mode_dtype)
+        right = numpy.full(self.right_shape, 2, dtype=right_dtype)
+        self.check_distribution('triangular',
+                                {'left': left, 'mode': mode, 'right': right},
+                                self.dtype)
+
+    @cupy.testing.for_float_dtypes('param_dtype', no_float16=True)
+    def test_triangular_for_invalid_params(self, param_dtype):
+        left = cupy.full(self.left_shape, 1, dtype=param_dtype)
+        mode = cupy.full(self.mode_shape, 0, dtype=param_dtype)
+        right = cupy.full(self.right_shape, 2, dtype=param_dtype)
+        with pytest.raises(ValueError):
+            _distributions.triangular(
+                left, mode, right, size=self.shape, dtype=self.dtype)
+
+        left = cupy.full(self.left_shape, -2, dtype=param_dtype)
+        mode = cupy.full(self.mode_shape, 0, dtype=param_dtype)
+        right = cupy.full(self.right_shape, -1, dtype=param_dtype)
+        with pytest.raises(ValueError):
+            _distributions.triangular(
+                left, mode, right, size=self.shape, dtype=self.dtype)
+
+        left = cupy.full(self.left_shape, 0, dtype=param_dtype)
+        mode = cupy.full(self.mode_shape, 0, dtype=param_dtype)
+        right = cupy.full(self.right_shape, 0, dtype=param_dtype)
+        with pytest.raises(ValueError):
+            _distributions.triangular(
+                left, mode, right, size=self.shape, dtype=self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'low_shape': [(), (3, 2)],
+    'high_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsUniform(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['low_dtype', 'high_dtype'])
+    def test_uniform(self, low_dtype, high_dtype, dtype):
+        low = numpy.ones(self.low_shape, dtype=low_dtype)
+        high = numpy.ones(self.high_shape, dtype=high_dtype) * 2.
+        self.check_distribution('uniform',
+                                {'low': low, 'high': high}, dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'mu_shape': [(), (3, 2)],
+    'kappa_shape': [(), (3, 2)],
+    'dtype': _float_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsVonmises:
+
+    def check_distribution(self, dist_func, mu_dtype, kappa_dtype, dtype):
+        mu = cupy.ones(self.mu_shape, dtype=mu_dtype)
+        kappa = cupy.ones(self.kappa_shape, dtype=kappa_dtype)
+        out = dist_func(mu, kappa, self.shape, dtype)
+        assert self.shape == out.shape
+        assert out.dtype == dtype
+
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['mu_dtype', 'kappa_dtype'])
+    def test_vonmises(self, mu_dtype, kappa_dtype):
+        self.check_distribution(_distributions.vonmises,
+                                mu_dtype, kappa_dtype, self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'mean_shape': [(), (3, 2)],
+    'scale_shape': [(), (3, 2)],
+    'dtype': _regular_float_dtypes,  # to escape timeout
+})
+)
+class TestDistributionsWald(RandomDistributionsTestCase):
+
+    @cupy.testing.for_dtypes_combination(
+        _float_dtypes, names=['mean_dtype', 'scale_dtype'])
+    def test_wald(self, mean_dtype, scale_dtype):
+        mean = numpy.full(self.mean_shape, 3, dtype=mean_dtype)
+        scale = numpy.full(self.scale_shape, 3, dtype=scale_dtype)
+        self.check_distribution('wald',
+                                {'mean': mean, 'scale': scale}, self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'a_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsWeibull(RandomDistributionsTestCase):
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('a_dtype')
+    def test_weibull(self, a_dtype, dtype):
+        a = numpy.ones(self.a_shape, dtype=a_dtype)
+        self.check_distribution('weibull',
+                                {'a': a}, dtype)
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('a_dtype')
+    def test_weibull_for_inf_a(self, a_dtype, dtype):
+        a = numpy.full(self.a_shape, numpy.inf, dtype=a_dtype)
+        self.check_distribution('weibull', {'a': a}, dtype)
+
+    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
+    @cupy.testing.for_float_dtypes('a_dtype')
+    def test_weibull_for_negative_a(self, a_dtype, dtype):
+        a = numpy.full(self.a_shape, -0.5, dtype=a_dtype)
+        with pytest.raises(ValueError):
+            cp_params = {'a': cupy.asarray(a)}
+            getattr(_distributions, 'weibull')(
+                size=self.shape, dtype=dtype, **cp_params)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 3, 2), (3, 2)],
+    'a_shape': [(), (3, 2)],
+})
+)
+class TestDistributionsZipf(RandomDistributionsTestCase):
+
+    @cupy.testing.for_dtypes([numpy.int32, numpy.int64], 'dtype')
+    @cupy.testing.for_float_dtypes('a_dtype')
+    def test_zipf(self, a_dtype, dtype):
+        a = numpy.full(self.a_shape, 2, dtype=a_dtype)
+        self.check_distribution('zipf', {'a': a}, dtype)
diff --git a/tests/cupy_tests/random_tests/test_generator.py b/tests/cupy_tests/random_tests/test_generator.py
new file mode 100644
index 0000000..908dbc7
--- /dev/null
+++ b/tests/cupy_tests/random_tests/test_generator.py
@@ -0,0 +1,1290 @@
+import functools
+import os
+import threading
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import cuda
+from cupy.cuda import runtime
+from cupy.random import _generator
+from cupy import testing
+from cupy.testing import _condition
+from cupy.testing import _hypothesis
+
+from cupy_tests.random_tests import common_distributions
+
+
+def numpy_cupy_equal_continuous_distribution(significance_level, name='xp'):
+    """Decorator that tests the distributions of NumPy samples and CuPy ones.
+
+    Args:
+        significance_level (float): The test fails if p-value is lower than
+            this argument.
+        name(str): Argument name whose value is either
+            ``numpy`` or ``cupy`` module.
+
+    Decorated test fixture is required to return samples from the same
+    distribution even if ``xp`` is ``numpy`` or ``cupy``.
+
+    """
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kw):
+            kw[name] = cupy
+            cupy_result = impl(self, *args, **kw)
+
+            kw[name] = numpy
+            numpy_result = impl(self, *args, **kw)
+
+            assert cupy_result is not None
+            assert numpy_result is not None
+            d_plus, d_minus, p_value = \
+                common_distributions.two_sample_Kolmogorov_Smirnov_test(
+                    cupy.asnumpy(cupy_result), numpy_result)
+            if p_value < significance_level:
+                message = '''Rejected null hypothesis:
+p: %f
+D+ (cupy < numpy): %f
+D- (cupy > numpy): %f''' % (p_value, d_plus, d_minus)
+                raise AssertionError(message)
+        return test_func
+    return decorator
+
+
+def _get_size(size):
+    # CuPy returns an ndarray of shape () even if size=None.
+    # cf. NumPy returns a Python scalar if size=None.
+    if size is None:
+        return ()
+    return cupy._core.get_size(size)
+
+
+class RandomGeneratorTestCase(common_distributions.BaseGeneratorTestCase):
+
+    target_method = None
+
+    def get_rng(self, xp, seed):
+        return xp.random.RandomState(seed=seed)
+
+    def set_rng_seed(self, seed):
+        self.rng.seed(seed)
+
+
+def _xp_random(xp, method_name):
+    method = getattr(xp.random.RandomState(), method_name)
+    if xp == cupy:
+        return method
+
+    def f(*args, **kwargs):
+        dtype = kwargs.pop('dtype', None)
+        ret = method(*args, **kwargs)
+        if dtype is not None:
+            ret = ret.astype(dtype, copy=False)
+        return ret
+
+    return f
+
+
+@testing.fix_random()
+class TestRandomState(unittest.TestCase):
+
+    def setUp(self):
+        self.rs = _generator.RandomState(seed=testing.generate_seed())
+
+    def check_seed(self, seed):
+        rs = self.rs
+
+        rs.seed(seed)
+        xs1 = [rs.uniform() for _ in range(100)]
+
+        rs.seed(seed)
+        xs2 = [rs.uniform() for _ in range(100)]
+
+        rs.seed(seed)
+        rs.seed(None)
+        xs3 = [rs.uniform() for _ in range(100)]
+
+        # Random state must be reproducible
+        assert xs1 == xs2
+        # Random state must be initialized randomly with seed=None
+        assert xs1 != xs3
+
+    @testing.for_int_dtypes()
+    def test_seed_not_none(self, dtype):
+        self.check_seed(dtype(0))
+
+    @testing.for_dtypes([numpy.complex_])
+    def test_seed_invalid_type_complex(self, dtype):
+        with self.assertRaises(TypeError):
+            self.rs.seed(dtype(0))
+
+    @testing.for_float_dtypes()
+    def test_seed_invalid_type_float(self, dtype):
+        with self.assertRaises(TypeError):
+            self.rs.seed(dtype(0))
+
+    def test_array_seed(self):
+        self.check_seed(numpy.random.randint(0, 2**31, size=40))
+
+    def test_methods(self):
+        methods = [
+            cuda.curand.CURAND_RNG_PSEUDO_DEFAULT,
+            cuda.curand.CURAND_RNG_PSEUDO_MRG32K3A,
+            cupy.cuda.curand.CURAND_RNG_PSEUDO_MT19937,
+            cupy.cuda.curand.CURAND_RNG_PSEUDO_PHILOX4_32_10,
+            cupy.cuda.curand.CURAND_RNG_PSEUDO_MTGP32,
+            cupy.cuda.curand.CURAND_RNG_PSEUDO_XORWOW
+        ]
+
+        for method in methods:
+            if (runtime.is_hip and
+                    method == cupy.cuda.curand.CURAND_RNG_PSEUDO_MT19937):
+                # hipRAND fails for MT19937 with the status code 1000,
+                # HIPRAND_STATUS_NOT_IMPLEMENTED. We use `pytest.raises` here
+                # so that we will be able to find it once hipRAND implement
+                # MT19937 as the imperative `pytest.xfail` immediately rewinds
+                # the control flow and does not run the test.
+                with pytest.raises(KeyError) as e:
+                    rs = cupy.random.RandomState(method=method)
+                assert e.value.args == (1000,)
+                continue
+            rs = cupy.random.RandomState(method=method)
+            rs.normal()
+
+
+@testing.parameterize(*common_distributions.beta_params)
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestBeta(
+    common_distributions.Beta,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(
+    {'n': 5, 'p': 0.5},
+    {'n': 5, 'p': 0.0},
+    {'n': 5, 'p': 1.0},
+)
+@testing.fix_random()
+class TestBinomial(RandomGeneratorTestCase):
+    # TODO(niboshi):
+    #   Test soundness of distribution.
+    #   Currently only reprocibility is checked.
+
+    target_method = 'binomial'
+
+    def test_binomial(self):
+        self.generate(n=self.n, p=self.p, size=(3, 2))
+
+
+@testing.parameterize(*common_distributions.chisquare_params)
+@testing.fix_random()
+class TestChisquare(
+    common_distributions.Chisquare,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.dirichlet_params)
+@testing.fix_random()
+class TestDirichlet(
+    common_distributions.Dirichlet,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.exponential_params)
+@testing.fix_random()
+class TestExponential(
+    common_distributions.Exponential,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.f_params)
+@testing.fix_random()
+class TestF(
+    common_distributions.F,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.gamma_params)
+@testing.fix_random()
+class TestGamma(
+    common_distributions.Gamma,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.geometric_params)
+@testing.fix_random()
+class TestGeometric(
+    common_distributions.Geometric,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.hypergeometric_params)
+@testing.fix_random()
+class TestHypergeometric(
+    common_distributions.Hypergeometric,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.fix_random()
+class TestLaplace(RandomGeneratorTestCase):
+
+    target_method = 'laplace'
+
+    def test_laplace_1(self):
+        self.generate()
+
+    def test_laplace_2(self):
+        self.generate(0.0, 1.0, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_laplace_ks_1(self, dtype):
+        self.check_ks(0.05)(
+            size=2000, dtype=dtype)
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_laplace_ks_2(self, dtype):
+        self.check_ks(0.05)(
+            2.3, 4.5, size=2000, dtype=dtype)
+
+
+@testing.fix_random()
+class TestLogistic(RandomGeneratorTestCase):
+
+    target_method = 'logistic'
+
+    def test_logistic_1(self):
+        self.generate()
+
+    def test_logistic_2(self):
+        self.generate(0.0, 1.0, size=(3, 2))
+
+    @testing.slow
+    @_condition.repeat(10)
+    def test_standard_logistic_isfinite(self):
+        x = self.generate(size=10**7)
+        assert cupy.isfinite(x).all()
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_logistic_ks_1(self, dtype):
+        self.check_ks(0.05)(
+            size=2000, dtype=dtype)
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_logistic_ks_2(self, dtype):
+        self.check_ks(0.05)(
+            2.3, 4.5, size=2000, dtype=dtype)
+
+
+@testing.parameterize(*[
+    {'args': (0.0, 1.0), 'size': None},
+    {'args': (10.0, 20.0), 'size': None},
+    {'args': (0.0, 1.0), 'size': 10},
+    {'args': (0.0, 1.0), 'size': (1, 2, 3)},
+    {'args': (0.0, 1.0), 'size': 3},
+    {'args': (0.0, 1.0), 'size': (3, 3)},
+    {'args': (0.0, 1.0), 'size': ()},
+])
+@testing.fix_random()
+class TestLogNormal(RandomGeneratorTestCase):
+
+    target_method = 'lognormal'
+
+    def check_lognormal(self, dtype):
+        vals = self.generate_many(
+            self.args[0], self.args[1], self.size, dtype, _count=10)
+
+        shape = _get_size(self.size)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype == dtype
+            assert val.shape == shape
+            assert (0 <= val).all()
+
+    def test_lognormal_float(self):
+        self.check_lognormal(float)
+
+    def test_lognormal_float32(self):
+        self.check_lognormal(numpy.float32)
+
+    def test_lognormal_float64(self):
+        self.check_lognormal(numpy.float64)
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_lognormal_ks(self, dtype):
+        self.check_ks(0.05)(
+            *self.args, size=self.size, dtype=dtype)
+
+
+@testing.parameterize(*common_distributions.logseries_params)
+@testing.fix_random()
+class TestLogseries(
+    common_distributions.Logseries,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*[
+    {'args': ([0., 0.], [[1., 0.], [0., 1.]]), 'size': None, 'tol': 1e-6},
+    {'args': ([10., 10.], [[20., 10.], [10., 20.]]),
+     'size': None, 'tol': 1e-6},
+    {'args': ([0., 0.], [[1., 0.], [0., 1.]]), 'size': 10, 'tol': 1e-6},
+    {'args': ([0., 0.], [[1., 0.], [0., 1.]]), 'size': (1, 2, 3), 'tol': 1e-6},
+    {'args': ([0., 0.], [[1., 0.], [0., 1.]]), 'size': 3, 'tol': 1e-6},
+    {'args': ([0., 0.], [[1., 0.], [0., 1.]]), 'size': (3, 3), 'tol': 1e-6},
+    {'args': ([0., 0.], [[1., 0.], [0., 1.]]), 'size': (), 'tol': 1e-6},
+])
+@testing.fix_random()
+class TestMultivariateNormal(RandomGeneratorTestCase):
+
+    target_method = 'multivariate_normal'
+
+    def check_multivariate_normal(self, dtype):
+        vals = self.generate_many(
+            mean=self.args[0], cov=self.args[1], size=self.size, tol=self.tol,
+            dtype=dtype, _count=10)
+
+        shape = _get_size(self.size)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype == dtype
+            assert val.shape == shape + (2,)
+
+    def test_multivariate_normal_float32(self):
+        self.check_multivariate_normal(numpy.float32)
+
+    def test_multivariate_normal_float64(self):
+        self.check_multivariate_normal(numpy.float64)
+
+    # TODO(kataoka): add distribution test
+
+
+@testing.parameterize(
+    {'n': 5, 'p': 0.5},
+)
+@testing.fix_random()
+class TestNegativeBinomial(RandomGeneratorTestCase):
+    target_method = 'negative_binomial'
+
+    def test_negative_binomial(self):
+        self.generate(n=self.n, p=self.p, size=(3, 2))
+
+    # TODO(kataoka): add distribution test
+
+
+@testing.parameterize(
+    {'df': 1.5, 'nonc': 2.0},
+    {'df': 2.0, 'nonc': 0.0},
+)
+@testing.fix_random()
+class TestNoncentralChisquare(RandomGeneratorTestCase):
+
+    target_method = 'noncentral_chisquare'
+
+    def test_noncentral_chisquare(self):
+        self.generate(df=self.df, nonc=self.nonc, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_noncentral_chisquare_ks(self, dtype):
+        self.check_ks(0.05)(
+            self.df, self.nonc, size=2000, dtype=dtype)
+
+
+@testing.parameterize(
+    {'dfnum': 2.0, 'dfden': 3.0, 'nonc': 4.0},
+    {'dfnum': 2.5, 'dfden': 1.5, 'nonc': 0.0},
+)
+@testing.fix_random()
+class TestNoncentralF(RandomGeneratorTestCase):
+
+    target_method = 'noncentral_f'
+
+    def test_noncentral_f(self):
+        self.generate(
+            dfnum=self.dfnum, dfden=self.dfden, nonc=self.nonc, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_noncentral_f_ks(self, dtype):
+        self.check_ks(0.05)(
+            self.dfnum, self.dfden, self.nonc, size=2000, dtype=dtype)
+
+
+@testing.parameterize(*[
+    {'args': (0.0, 1.0), 'size': None},
+    {'args': (10.0, 20.0), 'size': None},
+    {'args': (0.0, 1.0), 'size': 10},
+    {'args': (0.0, 1.0), 'size': (1, 2, 3)},
+    {'args': (0.0, 1.0), 'size': 3},
+    {'args': (0.0, 1.0), 'size': (3, 3)},
+    {'args': (0.0, 1.0), 'size': ()},
+])
+@testing.fix_random()
+class TestNormal(RandomGeneratorTestCase):
+
+    target_method = 'normal'
+
+    def check_normal(self, dtype):
+        vals = self.generate_many(
+            self.args[0], self.args[1], self.size, dtype, _count=10)
+
+        shape = _get_size(self.size)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype == dtype
+            assert val.shape == shape
+
+    def test_normal_float32(self):
+        self.check_normal(numpy.float32)
+
+    def test_normal_float64(self):
+        self.check_normal(numpy.float64)
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_normal_ks(self, dtype):
+        self.check_ks(0.05)(
+            *self.args, size=self.size, dtype=dtype)
+
+
+@testing.parameterize(
+    {'a': 1.0},
+    {'a': 3.0},
+    {'a': 10.0},
+)
+@testing.fix_random()
+class TestPareto(RandomGeneratorTestCase):
+
+    target_method = 'pareto'
+
+    def test_pareto(self):
+        self.generate(a=self.a, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_pareto_ks(self, dtype):
+        self.check_ks(0.05)(
+            a=self.a, size=2000, dtype=dtype)
+
+
+@testing.parameterize(*common_distributions.poisson_params)
+@testing.fix_random()
+class TestPoisson(
+    common_distributions.Poisson,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(
+    {'df': 1.0},
+    {'df': 3.0},
+    {'df': 10.0},
+)
+@testing.fix_random()
+class TestStandardT(RandomGeneratorTestCase):
+
+    target_method = 'standard_t'
+
+    def test_standard_t(self):
+        self.generate(df=self.df, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_standard_t_ks(self, dtype):
+        self.check_ks(0.05)(
+            df=self.df, size=2000, dtype=dtype)
+
+
+@testing.parameterize(*[
+    {'size': None},
+    {'size': 10},
+    {'size': (1, 2, 3)},
+    {'size': 3},
+    {'size': ()},
+])
+@testing.fix_random()
+class TestRandomSample(unittest.TestCase):
+
+    def setUp(self):
+        self.rs = _generator.RandomState(seed=testing.generate_seed())
+
+    def check_random_sample(self, dtype):
+        vals = [self.rs.random_sample(self.size, dtype) for _ in range(10)]
+
+        shape = _get_size(self.size)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype == dtype
+            assert val.shape == shape
+            assert (0 <= val).all()
+            assert (val < 1).all()
+
+    def test_random_sample_float32(self):
+        self.check_random_sample(numpy.float32)
+
+    def test_random_sample_float64(self):
+        self.check_random_sample(numpy.float64)
+
+
+@testing.fix_random()
+class TestRandomSampleDistrib(unittest.TestCase):
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    @numpy_cupy_equal_continuous_distribution(0.05)
+    def test_random_sample_ks(self, xp, dtype):
+        return _xp_random(xp, 'random_sample')(size=2000, dtype=dtype)
+
+
+@testing.fix_random()
+class TestRandAndRandN(unittest.TestCase):
+
+    def setUp(self):
+        self.rs = _generator.RandomState(seed=testing.generate_seed())
+
+    def test_rand_invalid_argument(self):
+        with self.assertRaises(TypeError):
+            self.rs.rand(1, 2, 3, unnecessary='unnecessary_argument')
+
+    def test_randn_invalid_argument(self):
+        with self.assertRaises(TypeError):
+            self.rs.randn(1, 2, 3, unnecessary='unnecessary_argument')
+
+
+@testing.parameterize(*common_distributions.power_params)
+@testing.fix_random()
+class TestPower(
+    common_distributions.Power,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(
+    {'scale': 1.0},
+    {'scale': 3.0},
+)
+@testing.fix_random()
+class TestRayleigh(RandomGeneratorTestCase):
+
+    target_method = 'rayleigh'
+
+    def test_rayleigh(self):
+        self.generate(scale=self.scale, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_rayleigh_ks(self, dtype):
+        self.check_ks(0.05)(
+            scale=self.scale, size=2000, dtype=dtype)
+
+
+@testing.fix_random()
+class TestStandardCauchy(RandomGeneratorTestCase):
+
+    target_method = 'standard_cauchy'
+
+    def test_standard_cauchy(self):
+        self.generate(size=(3, 2))
+
+    @testing.slow
+    @_condition.repeat(10)
+    def test_standard_cauchy_isfinite(self):
+        x = self.generate(size=10**7)
+        assert cupy.isfinite(x).all()
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_standard_cauchy_ks(self, dtype):
+        self.check_ks(0.05)(
+            size=2000, dtype=dtype)
+
+
+@testing.parameterize(*common_distributions.standard_gamma_params)
+@testing.fix_random()
+class TestStandardGamma(
+    common_distributions.StandardGamma,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.fix_random()
+class TestInterval(RandomGeneratorTestCase):
+
+    target_method = '_interval'
+
+    def test_zero(self):
+        shape = (2, 3)
+        vals = self.generate_many(0, shape, _count=10)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype.kind in 'iu'
+            assert val.shape == shape
+            assert (val == 0).all()
+
+    def test_shape_zero(self):
+        mx = 10
+        vals = self.generate_many(mx, None, _count=10)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype.kind in 'iu'
+            assert val.shape == ()
+            assert (0 <= val).all()
+            assert (val <= mx).all()
+        # TODO(niboshi): Distribution test
+
+    def test_shape_one_dim(self):
+        mx = 10
+        size = 20
+        vals = self.generate_many(mx, size, _count=10)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype.kind in 'iu'
+            assert val.shape == (size,)
+            assert (0 <= val).all()
+            assert (val <= mx).all()
+        # TODO(niboshi): Distribution test
+
+    def test_shape_multi_dim(self):
+        mx = 10
+        shape = (1, 2)
+        vals = self.generate_many(mx, shape, _count=10)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype.kind in 'iu'
+            assert val.shape == shape
+            assert (0 <= val).all()
+            assert (val <= mx).all()
+        # TODO(niboshi): Distribution test
+
+    def test_bound_1(self):
+        vals = self.generate_many(10, (2, 3), _count=10)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype.kind in 'iu'
+            assert val.shape == (2, 3)
+            assert (0 <= val).all()
+            assert (val <= 10).all()
+
+    def test_bound_2(self):
+        vals = self.generate_many(2, None, _count=20)
+        for val in vals:
+            assert isinstance(val, cupy.ndarray)
+            assert val.dtype.kind in 'iu'
+            assert val.shape == ()
+            assert (0 <= val).all()
+            assert (val <= 2).all()
+
+    @_condition.repeat(3, 10)
+    def test_goodness_of_fit(self):
+        mx = 5
+        trial = 300
+        vals = self.generate_many(mx, None, _count=trial)
+        vals = [val.get() for val in vals]
+        counts = numpy.histogram(vals, bins=numpy.arange(mx + 2))[0]
+        expected = numpy.array([float(trial) / (mx + 1)] * (mx + 1))
+        assert _hypothesis.chi_square_test(counts, expected)
+
+    @_condition.repeat(3)
+    def test_goodness_of_fit_2(self):
+        mx = 5
+        vals = self.generate(mx, (5, 5)).get()
+        counts = numpy.histogram(vals, bins=numpy.arange(mx + 2))[0]
+        expected = numpy.array([float(vals.size) / (mx + 1)] * (mx + 1))
+        assert _hypothesis.chi_square_test(counts, expected)
+
+
+@testing.fix_random()
+class TestTomaxint(RandomGeneratorTestCase):
+
+    target_method = 'tomaxint'
+
+    def test_tomaxint_none(self):
+        x = self.generate()
+        assert x.shape == ()
+        assert (0 <= x).all()
+        assert (x <= cupy.iinfo(cupy.int_).max).all()
+
+    def test_tomaxint_int(self):
+        x = self.generate(3)
+        assert x.shape == (3,)
+        assert (0 <= x).all()
+        assert (x <= cupy.iinfo(cupy.int_).max).all()
+
+    def test_tomaxint_tuple(self):
+        x = self.generate((2, 3))
+        assert x.shape == (2, 3)
+        assert (0 <= x).all()
+        assert (x <= cupy.iinfo(cupy.int_).max).all()
+
+
+@testing.parameterize(
+    {'a': 3, 'size': 2, 'p': None},
+    {'a': 3, 'size': 2, 'p': [0.3, 0.3, 0.4]},
+    {'a': 3, 'size': (5, 5), 'p': [0.3, 0.3, 0.4]},
+    {'a': 3, 'size': (5, 5), 'p': numpy.array([0.3, 0.3, 0.4])},
+    {'a': 3, 'size': (), 'p': None},
+    {'a': numpy.array([0.0, 1.0, 2.0]), 'size': 2, 'p': [0.3, 0.3, 0.4]},
+    {'a': 0, 'size': 0, 'p': None},
+    {'a': numpy.array([]), 'size': 0, 'p': None},
+)
+@testing.fix_random()
+class TestChoice1(RandomGeneratorTestCase):
+
+    target_method = 'choice'
+
+    def test_dtype_shape(self):
+        v = self.generate(a=self.a, size=self.size, p=self.p)
+        if isinstance(self.size, int):
+            expected_shape = (self.size,)
+        else:
+            expected_shape = self.size
+        if isinstance(self.a, numpy.ndarray):
+            expected_dtype = 'float'
+        else:
+            expected_dtype = 'int64'
+        assert v.dtype == expected_dtype
+        assert v.shape == expected_shape
+
+    @_condition.repeat(3, 10)
+    def test_bound(self):
+        vals = self.generate_many(
+            a=self.a, size=self.size, p=self.p, _count=20)
+        vals = [val.get() for val in vals]
+        size_ = self.size if isinstance(self.size, tuple) else (self.size,)
+        if size_ == (0, ):
+            self.skipTest('no bound check for empty `random.choice`')
+        for val in vals:
+            assert val.shape == size_
+        assert min(val.min() for val in vals) == 0
+        assert max(val.max() for val in vals) == 2
+
+
+@testing.parameterize(
+    {'a': [0, 1, 2], 'size': 2, 'p': [0.3, 0.3, 0.4]},
+)
+@testing.fix_random()
+class TestChoice2(RandomGeneratorTestCase):
+
+    target_method = 'choice'
+
+    def test_dtype_shape(self):
+        v = self.generate(a=self.a, size=self.size, p=self.p)
+        if isinstance(self.size, int):
+            expected_shape = (self.size,)
+        else:
+            expected_shape = self.size
+        if isinstance(self.a, numpy.ndarray):
+            expected_dtype = 'float'
+        else:
+            expected_dtype = 'int'
+        assert v.dtype == expected_dtype
+        assert v.shape == expected_shape
+
+    @_condition.repeat(3, 10)
+    def test_bound(self):
+        vals = self.generate_many(
+            a=self.a, size=self.size, p=self.p, _count=20)
+        vals = [val.get() for val in vals]
+        size_ = self.size if isinstance(self.size, tuple) else (self.size,)
+        for val in vals:
+            assert val.shape == size_
+        assert min(val.min() for val in vals) == 0
+        assert max(val.max() for val in vals) == 2
+
+
+@testing.fix_random()
+class TestChoiceChi(RandomGeneratorTestCase):
+
+    target_method = 'choice'
+
+    @_condition.repeat(3, 10)
+    def test_goodness_of_fit(self):
+        trial = 100
+        vals = self.generate_many(3, 1, True, [0.3, 0.3, 0.4], _count=trial)
+        vals = [val.get() for val in vals]
+        counts = numpy.histogram(vals, bins=numpy.arange(4))[0]
+        expected = numpy.array([30, 30, 40])
+        assert _hypothesis.chi_square_test(counts, expected)
+
+    @_condition.repeat(3, 10)
+    @pytest.mark.xfail(runtime.is_hip, reason='ROCm/HIP may have a bug')
+    def test_goodness_of_fit_2(self):
+        vals = self.generate(3, (5, 20), True, [0.3, 0.3, 0.4]).get()
+        counts = numpy.histogram(vals, bins=numpy.arange(4))[0]
+        expected = numpy.array([30, 30, 40])
+        assert _hypothesis.chi_square_test(counts, expected)
+
+
+@testing.fix_random()
+class TestChoiceMultinomial(unittest.TestCase):
+
+    @_condition.repeat(3, 10)
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=0.02)
+    def test_choice_multinomial(self, xp, dtype):
+        p = xp.array([0.5, 0.25, 0.125, 0.125], dtype)
+        trial = 10000
+        x = xp.random.choice(len(p), trial, p=p)
+        y = xp.bincount(x).astype('f') / trial
+        return y
+
+
+@testing.parameterize(
+    {'a': 3.1, 'size': 1, 'p': [0.1, 0.1, 0.8]},
+    {'a': None, 'size': 1, 'p': [0.1, 0.1, 0.8]},
+    {'a': -3, 'size': 1, 'p': [0.1, 0.1, 0.8]},
+    {'a': [[0, 1], [2, 3]], 'size': 1, 'p': [[0.1, 0.2], [0.3, 0.4]]},
+    {'a': [[0, 1], [2, 3]], 'size': 1, 'p': [0.3, 0.7]},
+    {'a': [], 'size': 1, 'p': [0.1, 0.1, 0.8]},
+    {'a': 4, 'size': 1, 'p': [[0.1, 0.2], [0.3, 0.4]]},
+    {'a': 2, 'size': 1, 'p': [0.1, 0.1, 0.8]},
+    {'a': 3, 'size': 1, 'p': [-0.1, 0.3, 0.8]},
+    {'a': 3, 'size': 1, 'p': [0.1, 0.1, 0.7]},
+)
+@testing.fix_random()
+class TestChoiceFailure(unittest.TestCase):
+
+    def setUp(self):
+        self.rs = _generator.RandomState(seed=testing.generate_seed())
+
+    def test_choice_invalid_value(self):
+        with self.assertRaises(ValueError):
+            self.rs.choice(a=self.a, size=self.size, p=self.p)
+
+
+@testing.parameterize(
+    {'a': 5, 'size': 2},
+    {'a': 5, 'size': (2, 2)},
+    {'a': 5, 'size': ()},
+    {'a': numpy.array([0.0, 2.0, 4.0]), 'size': 2},
+)
+@testing.fix_random()
+class TestChoiceReplaceFalse(RandomGeneratorTestCase):
+
+    target_method = 'choice'
+
+    def test_dtype_shape(self):
+        v = self.generate(a=self.a, size=self.size, replace=False)
+        if isinstance(self.size, int):
+            expected_shape = (self.size,)
+        else:
+            expected_shape = self.size
+        if isinstance(self.a, numpy.ndarray):
+            expected_dtype = 'float'
+        else:
+            expected_dtype = 'int'
+        assert v.dtype == expected_dtype
+        assert v.shape == expected_shape
+
+    @_condition.repeat(3, 10)
+    def test_bound(self):
+        val = self.generate(a=self.a, size=self.size, replace=False).get()
+        size = self.size if isinstance(self.size, tuple) else (self.size,)
+        assert val.shape == size
+        assert (0 <= val).all()
+        assert (val < 5).all()
+        val = numpy.asarray(val)
+        assert numpy.unique(val).size == val.size
+
+
+@testing.fix_random()
+class TestGumbel(RandomGeneratorTestCase):
+
+    target_method = 'gumbel'
+
+    def test_gumbel_1(self):
+        self.generate()
+
+    def test_gumbel_2(self):
+        self.generate(0.0, 1.0, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_gumbel_ks_1(self, dtype):
+        self.check_ks(0.05)(
+            size=2000, dtype=dtype)
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_gumbel_ks_2(self, dtype):
+        self.check_ks(0.05)(
+            2.3, 4.5, size=2000, dtype=dtype)
+
+
+@testing.fix_random()
+class TestRandint(RandomGeneratorTestCase):
+    # TODO(niboshi):
+    #   Test soundness of distribution.
+    #   Currently only reprocibility is checked.
+
+    target_method = 'randint'
+
+    def test_randint_1(self):
+        self.generate(3)
+
+    def test_randint_2(self):
+        self.generate(3, 4, size=(3, 2))
+
+    def test_randint_empty1(self):
+        self.generate(3, 10, size=0)
+
+    def test_randint_empty2(self):
+        self.generate(3, size=(4, 0, 5))
+
+    def test_randint_overflow(self):
+        self.generate(numpy.int8(-100), numpy.int8(100))
+
+    def test_randint_float1(self):
+        self.generate(-1.2, 3.4, 5)
+
+    def test_randint_float2(self):
+        self.generate(6.7, size=(2, 3))
+
+    def test_randint_int64_1(self):
+        self.generate(2**34, 2**40, 3, dtype='q')
+
+    def test_randint_array(self):
+        self.generate([[[-1], [0]], [[-2], [1]], [[3], [4]]], [[10, 11, 12]])
+
+
+@testing.fix_random()
+class TestUniform(RandomGeneratorTestCase):
+
+    target_method = 'uniform'
+
+    def test_uniform_1(self):
+        self.generate()
+
+    def test_uniform_2(self):
+        self.generate(-4.2, 2.4, size=(3, 2))
+
+    def test_uniform_broadcast(self):
+        self.generate([[2, 3]], [4])
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_uniform_ks_1(self, dtype):
+        self.check_ks(0.05)(
+            size=2000, dtype=dtype)
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_uniform_ks_2(self, dtype):
+        self.check_ks(0.05)(
+            -4.2, 2.4, size=2000, dtype=dtype)
+
+
+@testing.parameterize(
+    {'mu': 0.0, 'kappa': 1.0},
+    {'mu': 3.0, 'kappa': 3.0},
+    {'mu': 3.0, 'kappa': 1.0},
+)
+@testing.fix_random()
+class TestVonmises(RandomGeneratorTestCase):
+
+    target_method = 'vonmises'
+
+    def test_vonmises(self):
+        self.generate(mu=self.mu, kappa=self.kappa, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_vonmises_ks(self, dtype):
+        self.check_ks(0.05)(
+            self.mu, self.kappa, size=2000, dtype=dtype)
+
+
+@testing.parameterize(
+    {'mean': 1.0, 'scale': 3.0},
+    {'mean': 3.0, 'scale': 3.0},
+    {'mean': 3.0, 'scale': 1.0},
+)
+@testing.fix_random()
+class TestWald(RandomGeneratorTestCase):
+
+    target_method = 'wald'
+
+    def test_wald(self):
+        self.generate(mean=self.mean, scale=self.scale, size=(3, 2))
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_wald_ks(self, dtype):
+        self.check_ks(0.05)(
+            self.mean, self.scale, size=2000, dtype=dtype)
+
+
+@testing.parameterize(
+    {'a': 0.5},
+    {'a': 1.0},
+    {'a': 3.0},
+    {'a': numpy.inf},
+)
+@testing.fix_random()
+class TestWeibull(RandomGeneratorTestCase):
+
+    target_method = 'weibull'
+
+    def test_weibull(self):
+        self.generate(a=self.a, size=(3, 2))
+
+    def test_weibull_size_none(self):
+        self.generate([[0.5, 1.0, 3.0]], size=None)
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_weibull_ks(self, dtype):
+        self.check_ks(0.05)(
+            a=self.a, size=2000, dtype=dtype)
+
+
+@testing.parameterize(
+    {'a': 2.0},
+)
+@testing.fix_random()
+class TestZipf(RandomGeneratorTestCase):
+
+    target_method = 'zipf'
+
+    def test_zipf(self):
+        self.generate(a=self.a, size=(3, 2))
+
+    # TODO(kataoka): add distribution test
+
+
+@testing.parameterize(
+    {'a': 3, 'size': 5},
+    {'a': [1, 2, 3], 'size': 5},
+)
+@testing.fix_random()
+class TestChoiceReplaceFalseFailure(unittest.TestCase):
+
+    def test_choice_invalid_value(self):
+        for xp in (numpy, cupy):
+            rs = xp.random.RandomState(seed=testing.generate_seed())
+            with pytest.raises(ValueError):
+                rs.choice(a=self.a, size=self.size, replace=False)
+
+
+class TestResetStates(unittest.TestCase):
+
+    def test_reset_states(self):
+        _generator._random_states = 'dummy'
+        _generator.reset_states()
+        assert {} == _generator._random_states
+
+
+class TestGetRandomState(unittest.TestCase):
+
+    def setUp(self):
+        self.device_id = cuda.Device().id
+        self.rs_tmp = _generator._random_states
+
+    def tearDown(self, *args):
+        _generator._random_states = self.rs_tmp
+
+    def test_get_random_state_initialize(self):
+        _generator._random_states = {}
+        rs = _generator.get_random_state()
+        assert _generator._random_states[self.device_id] == rs
+
+    def test_get_random_state_memoized(self):
+        _generator._random_states = {self.device_id: 'expected',
+                                     self.device_id + 1: 'dummy'}
+        rs = _generator.get_random_state()
+        assert 'expected' == _generator._random_states[self.device_id]
+        assert 'dummy' == _generator._random_states[self.device_id + 1]
+        assert 'expected' == rs
+
+
+class TestSetRandomState(unittest.TestCase):
+
+    def setUp(self):
+        self.rs_tmp = _generator._random_states
+
+    def tearDown(self, *args):
+        _generator._random_states = self.rs_tmp
+
+    def test_set_random_state(self):
+        rs = _generator.RandomState()
+        _generator.set_random_state(rs)
+        assert _generator.get_random_state() is rs
+
+    def test_set_random_state_call_multiple_times(self):
+        _generator.set_random_state(_generator.RandomState())
+        rs = _generator.RandomState()
+        _generator.set_random_state(rs)
+        assert _generator.get_random_state() is rs
+
+
+@testing.fix_random()
+class TestStandardExponential(
+    common_distributions.StandardExponential,
+    RandomGeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(
+    {'left': -1.0, 'mode': 0.0, 'right': 2.0},
+)
+@testing.fix_random()
+class TestTriangular(RandomGeneratorTestCase):
+
+    target_method = 'triangular'
+
+    def test_triangular(self):
+        self.generate(
+            left=self.left, mode=self.mode, right=self.right, size=(3, 2))
+
+
+class TestRandomStateThreadSafe(unittest.TestCase):
+
+    def setUp(self):
+        cupy.random.reset_states()
+
+    def test_get_random_state_thread_safe(self):
+        def _f(func, args=()):
+            cupy.cuda.Device().use()
+            func(*args)
+
+        seed = 10
+        threads = [
+            threading.Thread(
+                target=_f, args=(cupy.random.seed, (seed,))),
+            threading.Thread(
+                target=_f, args=(cupy.random.get_random_state,)),
+            threading.Thread(
+                target=_f, args=(cupy.random.get_random_state,)),
+            threading.Thread(
+                target=_f, args=(cupy.random.get_random_state,)),
+            threading.Thread(
+                target=_f, args=(cupy.random.get_random_state,)),
+            threading.Thread(
+                target=_f, args=(cupy.random.get_random_state,)),
+            threading.Thread(
+                target=_f, args=(cupy.random.get_random_state,)),
+        ]
+
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+        actual = cupy.random.uniform()
+        cupy.random.seed(seed)
+        expected = cupy.random.uniform()
+        assert actual == expected
+
+    def test_set_random_state_thread_safe(self):
+        def _f(func, args=()):
+            cupy.cuda.Device().use()
+            func(*args)
+
+        rs = cupy.random.RandomState()
+        threads = [
+            threading.Thread(
+                target=_f, args=(cupy.random.set_random_state, (rs,))),
+            threading.Thread(
+                target=_f, args=(cupy.random.set_random_state, (rs,))),
+        ]
+
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+        assert cupy.random.get_random_state() is rs
+
+
+class TestGetRandomState2(unittest.TestCase):
+
+    def setUp(self):
+        self.rs_dict = _generator._random_states
+        _generator._random_states = {}
+        self.cupy_seed = os.getenv('CUPY_SEED')
+
+    def tearDown(self, *args):
+        _generator._random_states = self.rs_dict
+        if self.cupy_seed is None:
+            os.environ.pop('CUPY_SEED', None)
+        else:
+            os.environ['CUPY_SEED'] = self.cupy_seed
+
+    def test_get_random_state_no_cupy(self):
+        os.environ.pop('CUPY_SEED', None)
+        rvs0 = self._get_rvs_reset()
+        rvs1 = self._get_rvs_reset()
+
+        self._check_different(rvs0, rvs1)
+
+    def test_get_random_state_with_cupy(self):
+        rvs0 = self._get_rvs(_generator.RandomState(6))
+
+        os.environ['CUPY_SEED'] = '6'
+        rvs1 = self._get_rvs_reset()
+
+        self._check_same(rvs0, rvs1)
+
+    def _get_rvs(self, rs):
+        rvu = rs.rand(4)
+        rvn = rs.randn(4)
+        return rvu, rvn
+
+    def _get_rvs_reset(self):
+        _generator.reset_states()
+        return self._get_rvs(_generator.get_random_state())
+
+    def _check_same(self, rvs0, rvs1):
+        for rv0, rv1 in zip(rvs0, rvs1):
+            testing.assert_array_equal(rv0, rv1)
+
+    def _check_different(self, rvs0, rvs1):
+        for rv0, rv1 in zip(rvs0, rvs1):
+            for r0, r1 in zip(rv0, rv1):
+                assert r0 != r1
+
+
+class TestCheckAndGetDtype(unittest.TestCase):
+
+    @testing.for_float_dtypes(no_float16=True)
+    def test_float32_64_type(self, dtype):
+        assert _generator._check_and_get_dtype(dtype) == numpy.dtype(dtype)
+
+    def test_float16(self):
+        with self.assertRaises(TypeError):
+            _generator._check_and_get_dtype(numpy.float16)
+
+    @testing.for_int_dtypes()
+    def test_int_type(self, dtype):
+        with self.assertRaises(TypeError):
+            _generator._check_and_get_dtype(dtype)
diff --git a/tests/cupy_tests/random_tests/test_generator_api.py b/tests/cupy_tests/random_tests/test_generator_api.py
new file mode 100644
index 0000000..842b7be
--- /dev/null
+++ b/tests/cupy_tests/random_tests/test_generator_api.py
@@ -0,0 +1,370 @@
+import threading
+import unittest
+import pytest
+
+import numpy
+
+import cupy
+from cupy import random
+from cupy import testing
+from cupy.testing import _condition
+
+from cupy_tests.random_tests import common_distributions
+
+
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip
+                    and (int(
+                        str(cupy.cuda.runtime.runtimeGetVersion())[:3]) < 403),
+                    reason='HIP<4.3 not supported ')
+class GeneratorTestCase(common_distributions.BaseGeneratorTestCase):
+
+    target_method = None
+
+    def get_rng(self, xp, seed):
+        if xp is cupy:
+            return cupy.random._generator_api.Generator(
+                random._bit_generator.Philox4x3210(seed=seed))
+        else:
+            return numpy.random.Generator(numpy.random.MT19937(seed))
+
+    def set_rng_seed(self, seed):
+        self.rng.bit_generator = random._bit_generator.Philox4x3210(seed=seed)
+
+
+class InvalidOutsMixin:
+
+    def invalid_dtype_out(self, **kwargs):
+        out = cupy.zeros((3, 2), dtype=cupy.float32)
+        with pytest.raises(TypeError):
+            self.generate(size=(3, 2), out=out, **kwargs)
+
+    def invalid_contiguity(self, **kwargs):
+        out = cupy.zeros((4, 6), dtype=cupy.float64)[0:3:, 0:2:]
+        with pytest.raises(ValueError):
+            self.generate(size=(3, 2), out=out, **kwargs)
+
+    def invalid_shape(self, **kwargs):
+        out = cupy.zeros((3, 3), dtype=cupy.float64)
+        with pytest.raises(ValueError):
+            self.generate(size=(3, 2), out=out, **kwargs)
+
+    def test_invalid_dtype_out(self):
+        self.invalid_dtype_out()
+
+    def test_invalid_contiguity(self):
+        self.invalid_contiguity()
+
+    def test_invalid_shape(self):
+        self.invalid_shape()
+
+
+@testing.parameterize(*common_distributions.uniform_params)
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestUniform(
+    common_distributions.Uniform,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.exponential_params)
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestExponential(
+    common_distributions.Exponential,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.poisson_params)
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestPoisson(
+    common_distributions.Poisson,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.binomial_params)
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestBinomial(
+    common_distributions.Binomial,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.beta_params)
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestBeta(
+    common_distributions.Beta,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestStandardExponential(
+    InvalidOutsMixin,
+    common_distributions.StandardExponential,
+    GeneratorTestCase,
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.gamma_params)
+@testing.fix_random()
+class TestGamma(
+    common_distributions.Gamma,
+    GeneratorTestCase,
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.standard_gamma_params)
+@testing.fix_random()
+class TestStandardGamma(
+    common_distributions.StandardGamma,
+    GeneratorTestCase,
+):
+    pass
+
+
+@testing.fix_random()
+class TestStandardGammaInvalid(InvalidOutsMixin, GeneratorTestCase):
+
+    target_method = 'standard_gamma'
+
+    def test_invalid_dtype_out(self):
+        self.invalid_dtype_out(shape=1.0)
+
+    def test_invalid_contiguity(self):
+        self.invalid_contiguity(shape=1.0)
+
+        out = cupy.zeros((4, 6), order='F', dtype=cupy.float64)
+        with pytest.raises(ValueError):
+            self.generate(size=(4, 6), out=out, shape=1.0)
+
+    def test_invalid_shape(self):
+        self.invalid_shape(shape=1.0)
+
+    def test_invalid_dtypes(self):
+        for dtype in 'bhiqleFD':
+            with pytest.raises(TypeError):
+                self.generate(size=(3, 2), shape=1.0, dtype=dtype)
+
+
+@testing.fix_random()
+class TestStandardGammaEmpty(GeneratorTestCase):
+
+    target_method = 'standard_gamma'
+
+    def test_empty_shape(self):
+        y = self.generate(shape=cupy.empty((1, 0)))
+        assert y.shape == (1, 0)
+
+    def test_empty_size(self):
+        y = self.generate(1.0, size=(1, 0))
+        assert y.shape == (1, 0)
+
+    def test_empty_out(self):
+        out = cupy.empty((1, 0))
+        y = self.generate(cupy.empty((1, 0)), out=out)
+        assert y is out
+        assert y.shape == (1, 0)
+
+
+@testing.with_requires('numpy>=1.17.0')
+@testing.parameterize(*common_distributions.standard_normal_params)
+@testing.fix_random()
+class TestStandardNormal(
+    common_distributions.StandardNormal,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestStandardNormalInvalid(InvalidOutsMixin, GeneratorTestCase):
+
+    target_method = 'standard_normal'
+
+    def test_invalid_dtypes(self):
+        for dtype in 'bhiqleFD':
+            with pytest.raises(TypeError):
+                self.generate(size=(3, 2), dtype=dtype)
+
+
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestIntegers(GeneratorTestCase):
+    target_method = 'integers'
+
+    def test_integers_1(self):
+        self.generate(3)
+
+    def test_integers_2(self):
+        self.generate(3, 4, size=(3, 2))
+
+    def test_integers_empty1(self):
+        self.generate(3, 10, size=0)
+
+    def test_integers_empty2(self):
+        self.generate(3, size=(4, 0, 5))
+
+    def test_integers_overflow(self):
+        self.generate(numpy.int8(-100), numpy.int8(100))
+
+    def test_integers_float1(self):
+        self.generate(-1.2, 3.4, 5)
+
+    def test_integers_float2(self):
+        self.generate(6.7, size=(2, 3))
+
+    def test_integers_int64_1(self):
+        self.generate(2**34, 2**40, 3)
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_integers_ks(self):
+        self.check_ks(0.05)(
+            low=100, high=1000, size=2000)
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_integers_ks_low(self):
+        self.check_ks(0.05)(
+            low=100, size=2000)
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_integers_ks_large(self):
+        self.check_ks(0.05)(
+            low=2**34, high=2**40, size=2000)
+
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_integers_ks_large2(self):
+        self.check_ks(0.05)(
+            2**40, size=2000)
+
+
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestRandom(InvalidOutsMixin, GeneratorTestCase):
+    # TODO(niboshi):
+    #   Test soundness of distribution.
+    #   Currently only reprocibility is checked.
+
+    target_method = 'random'
+
+    def test_random(self):
+        self.generate(3)
+
+    @testing.for_dtypes('fd')
+    @_condition.repeat_with_success_at_least(10, 3)
+    def test_random_ks(self, dtype):
+        self.check_ks(0.05)(size=2000, dtype=dtype)
+
+
+@testing.parameterize(*common_distributions.geometric_params)
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestGeometric(
+    common_distributions.Geometric,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.hypergeometric_params)
+@testing.with_requires('numpy>=1.17.0')
+@testing.fix_random()
+class TestHypergeometric(
+    common_distributions.Hypergeometric,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.power_params)
+@testing.fix_random()
+class TestPower(
+    common_distributions.Power,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.with_requires('numpy>=1.17.0')
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip
+                    and (int(
+                        str(cupy.cuda.runtime.runtimeGetVersion())[:3]) < 403),
+                    reason='HIP<4.3 not supported ')
+class TestRandomStateThreadSafe(unittest.TestCase):
+
+    def test_default_rng_thread_safe(self):
+        def _f(func, args=()):
+            cupy.cuda.Device().use()
+            func(*args)
+
+        seed = 10
+        threads = [
+            threading.Thread(
+                target=_f, args=(cupy.random.default_rng, (seed,))),
+            threading.Thread(target=_f, args=(cupy.random.default_rng)),
+            threading.Thread(target=_f, args=(cupy.random.default_rng)),
+            threading.Thread(target=_f, args=(cupy.random.default_rng)),
+            threading.Thread(target=_f, args=(cupy.random.default_rng)),
+            threading.Thread(target=_f, args=(cupy.random.default_rng)),
+            threading.Thread(target=_f, args=(cupy.random.default_rng)),
+        ]
+
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+        actual = cupy.random.default_rng(seed).standard_exponential()
+        expected = cupy.random.default_rng(seed).standard_exponential()
+        assert actual == expected
+
+
+@testing.parameterize(*common_distributions.logseries_params)
+@testing.fix_random()
+class TestLogseries(
+    common_distributions.Logseries,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.chisquare_params)
+@testing.fix_random()
+class TestChisquare(
+    common_distributions.Chisquare,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.f_params)
+@testing.fix_random()
+class TestF(
+    common_distributions.F,
+    GeneratorTestCase
+):
+    pass
+
+
+@testing.parameterize(*common_distributions.dirichlet_params)
+@testing.fix_random()
+class TestDrichlet(
+    common_distributions.Dirichlet,
+
+    GeneratorTestCase
+):
+    pass
diff --git a/tests/cupy_tests/random_tests/test_init.py b/tests/cupy_tests/random_tests/test_init.py
new file mode 100644
index 0000000..ad89bfa
--- /dev/null
+++ b/tests/cupy_tests/random_tests/test_init.py
@@ -0,0 +1,6 @@
+import cupy
+
+
+def test_bytes():
+    out = cupy.random.bytes(10)
+    assert isinstance(out, bytes)
diff --git a/tests/cupy_tests/random_tests/test_permutations.py b/tests/cupy_tests/random_tests/test_permutations.py
new file mode 100644
index 0000000..62f1750
--- /dev/null
+++ b/tests/cupy_tests/random_tests/test_permutations.py
@@ -0,0 +1,182 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.testing import _condition
+
+
+@testing.parameterize(
+    {'seed': None},
+    {'seed': 0},
+)
+class TestPermutations(unittest.TestCase):
+
+    def _xp_random(self, xp):
+        if self.seed is None:
+            return xp.random
+        else:
+            return xp.random.RandomState(seed=self.seed)
+
+    # Test ranks
+
+    # TODO(niboshi): Fix xfail
+    @pytest.mark.xfail(reason='Explicit error types required')
+    def test_permutation_zero_dim(self):
+        for xp in (numpy, cupy):
+            xp_random = self._xp_random(xp)
+            a = testing.shaped_random((), xp)
+            with pytest.raises(IndexError):
+                xp_random.permutation(a)
+
+    # Test same values
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_permutation_sort_1dim(self, dtype):
+        cupy_random = self._xp_random(cupy)
+        a = cupy.arange(10, dtype=dtype)
+        b = cupy.copy(a)
+        c = cupy_random.permutation(a)
+        testing.assert_allclose(a, b)
+        testing.assert_allclose(b, cupy.sort(c))
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_permutation_sort_ndim(self, dtype):
+        cupy_random = self._xp_random(cupy)
+        a = cupy.arange(15, dtype=dtype).reshape(5, 3)
+        b = cupy.copy(a)
+        c = cupy_random.permutation(a)
+        testing.assert_allclose(a, b)
+        testing.assert_allclose(b, cupy.sort(c, axis=0))
+
+    # Test seed
+
+    @testing.for_all_dtypes()
+    def test_permutation_seed1(self, dtype):
+        a = testing.shaped_random((10,), cupy, dtype)
+        b = cupy.copy(a)
+
+        cupy_random = self._xp_random(cupy)
+        if self.seed is None:
+            cupy_random.seed(0)
+        pa = cupy_random.permutation(a)
+        cupy_random = self._xp_random(cupy)
+        if self.seed is None:
+            cupy_random.seed(0)
+        pb = cupy_random.permutation(b)
+
+        testing.assert_allclose(pa, pb)
+
+
+class TestShuffle(unittest.TestCase):
+
+    # Test ranks
+
+    def test_shuffle_zero_dim(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            with pytest.raises(TypeError):
+                xp.random.shuffle(a)
+
+    # Test same values
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_shuffle_sort_1dim(self, dtype):
+        a = cupy.arange(10, dtype=dtype)
+        b = cupy.copy(a)
+        cupy.random.shuffle(a)
+        testing.assert_allclose(cupy.sort(a), b)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_shuffle_sort_ndim(self, dtype):
+        a = cupy.arange(15, dtype=dtype).reshape(5, 3)
+        b = cupy.copy(a)
+        cupy.random.shuffle(a)
+        testing.assert_allclose(cupy.sort(a, axis=0), b)
+
+    # Test seed
+
+    @testing.for_all_dtypes()
+    def test_shuffle_seed1(self, dtype):
+        a = testing.shaped_random((10,), cupy, dtype)
+        b = cupy.copy(a)
+        cupy.random.seed(0)
+        cupy.random.shuffle(a)
+        cupy.random.seed(0)
+        cupy.random.shuffle(b)
+        testing.assert_allclose(a, b)
+
+
+@testing.parameterize(*(testing.product({
+    'num': [0, 1, 100, 1000, 10000, 100000],
+})))
+class TestPermutationSoundness(unittest.TestCase):
+
+    def setUp(self):
+        a = cupy.random.permutation(self.num)
+        self.a = a.get()
+
+    # Test soundness
+
+    @_condition.repeat(3)
+    def test_permutation_soundness(self):
+        assert (numpy.sort(self.a) == numpy.arange(self.num)).all()
+
+
+@testing.parameterize(*(testing.product({
+    'offset': [0, 17, 34, 51],
+    'gap': [1, 2, 3, 5, 7],
+    'mask': [1, 2, 4, 8, 16, 32, 64, 128],
+})))
+class TestPermutationRandomness(unittest.TestCase):
+
+    num = 256
+
+    def setUp(self):
+        a = cupy.random.permutation(self.num)
+        self.a = a.get()
+        self.num_half = int(self.num / 2)
+
+    # Simple bit proportion test
+
+    # This test is to check kind of randomness of permutation.
+    # An intuition behind this test is that, when you make a sub-array
+    # by regularly extracting half elements from the permuted array,
+    # the sub-array should also hold randomeness and accordingly
+    # frequency of appearance of 0 and 1 at each bit position of
+    # whole elements in the sub-array should become similar
+    # when elements count of original array is 2^N.
+    # Note that this is not an establishd method to check randomness.
+    # TODO(anaruse): implement randomness check using some established methods.
+    @_condition.repeat_with_success_at_least(5, 3)
+    def test_permutation_randomness(self):
+        if self.mask > self.num_half:
+            return
+        index = numpy.arange(self.num_half)
+        index = (index * self.gap + self.offset) % self.num
+        samples = self.a[index]
+        ret = (samples & self.mask > 0)
+        count = numpy.count_nonzero(ret)  # expectation: self.num_half / 2
+        if count > self.num_half - count:
+            count = self.num_half - count
+        prob_le_count = self._calc_probability(count)
+        if prob_le_count < 0.001:
+            raise
+
+    def _calc_probability(self, count):
+        comb_all = self._comb(self.num, self.num_half)
+        comb_le_count = 0
+        for i in range(count + 1):
+            tmp = self._comb(self.num_half, i)
+            comb_i = tmp * tmp
+            comb_le_count += comb_i
+        prob = comb_le_count / comb_all
+        return prob
+
+    def _comb(self, N, k):
+        val = numpy.float64(1)
+        for i in range(k):
+            val *= (N - i) / (k - i)
+        return val
diff --git a/tests/cupy_tests/random_tests/test_random.py b/tests/cupy_tests/random_tests/test_random.py
new file mode 100644
index 0000000..a9389bb
--- /dev/null
+++ b/tests/cupy_tests/random_tests/test_random.py
@@ -0,0 +1,19 @@
+import unittest
+
+from cupy import random
+from cupy import testing
+
+
+class TestResetSeed(unittest.TestCase):
+
+    @testing.for_float_dtypes(no_float16=True)
+    def test_reset_seed(self, dtype):
+        rs = random.get_random_state()
+        rs.seed(0)
+        l1 = rs.rand(10, dtype=dtype)
+
+        rs = random.get_random_state()
+        rs.seed(0)
+        l2 = rs.rand(10, dtype=dtype)
+
+        testing.assert_array_equal(l1, l2)
diff --git a/tests/cupy_tests/random_tests/test_sample.py b/tests/cupy_tests/random_tests/test_sample.py
new file mode 100644
index 0000000..ee8a3a1
--- /dev/null
+++ b/tests/cupy_tests/random_tests/test_sample.py
@@ -0,0 +1,300 @@
+import unittest
+from unittest import mock
+
+import numpy
+import pytest
+
+import cupy
+from cupy import cuda
+from cupy.cuda import runtime
+from cupy import random
+from cupy import testing
+from cupy.testing import _condition
+from cupy.testing import _hypothesis
+
+
+class TestRandint(unittest.TestCase):
+
+    def test_lo_hi_reversed(self):
+        with self.assertRaises(ValueError):
+            random.randint(100, 1)
+
+    def test_lo_hi_equal(self):
+        with self.assertRaises(ValueError):
+            random.randint(3, 3, size=0)
+
+        with self.assertRaises(ValueError):
+            # int(-0.2) is not less than int(0.3)
+            random.randint(-0.2, 0.3)
+
+    def test_lo_hi_nonrandom(self):
+        a = random.randint(-0.9, 1.1, size=3)
+        testing.assert_array_equal(a, cupy.full((3,), 0))
+
+        a = random.randint(-1.1, -0.9, size=(2, 2))
+        testing.assert_array_equal(a, cupy.full((2, 2), -1))
+
+    def test_zero_sizes(self):
+        a = random.randint(10, size=(0,))
+        testing.assert_array_equal(a, cupy.array(()))
+
+        a = random.randint(10, size=0)
+        testing.assert_array_equal(a, cupy.array(()))
+
+
+@testing.fix_random()
+class TestRandint2(unittest.TestCase):
+
+    @_condition.repeat(3, 10)
+    def test_bound_1(self):
+        vals = [random.randint(0, 10, (2, 3)).get() for _ in range(20)]
+        for val in vals:
+            assert val.shape == (2, 3)
+        assert min(_.min() for _ in vals) == 0
+        assert max(_.max() for _ in vals) == 9
+
+    @_condition.repeat(3, 10)
+    def test_bound_2(self):
+        vals = [random.randint(0, 2).get() for _ in range(20)]
+        for val in vals:
+            assert val.shape == ()
+        assert min(vals) == 0
+        assert max(vals) == 1
+
+    @_condition.repeat(3, 10)
+    def test_bound_overflow(self):
+        # 100 - (-100) exceeds the range of int8
+        val = random.randint(numpy.int8(-100), numpy.int8(100), size=20).get()
+        assert val.shape == (20,)
+        assert val.min() >= -100
+        assert val.max() < 100
+
+    @_condition.repeat(3, 10)
+    def test_bound_float1(self):
+        # generate floats s.t. int(low) < int(high)
+        low, high = sorted(numpy.random.uniform(-5, 5, size=2))
+        low -= 1
+        high += 1
+        vals = [random.randint(low, high, (2, 3)).get() for _ in range(10)]
+        for val in vals:
+            assert val.shape == (2, 3)
+        assert min(_.min() for _ in vals) == int(low)
+        assert max(_.max() for _ in vals) == int(high) - 1
+
+    def test_bound_float2(self):
+        vals = [random.randint(-1.0, 1.0, (2, 3)).get() for _ in range(10)]
+        for val in vals:
+            assert val.shape == (2, 3)
+        assert min(_.min() for _ in vals) == -1
+        assert max(_.max() for _ in vals) == 0
+
+    @_condition.repeat(3, 10)
+    def test_goodness_of_fit(self):
+        mx = 5
+        trial = 100
+        vals = [random.randint(mx).get() for _ in range(trial)]
+        counts = numpy.histogram(vals, bins=numpy.arange(mx + 1))[0]
+        expected = numpy.array([float(trial) / mx] * mx)
+        assert _hypothesis.chi_square_test(counts, expected)
+
+    @_condition.repeat(3, 10)
+    @pytest.mark.xfail(runtime.is_hip, reason='ROCm/HIP may have a bug')
+    def test_goodness_of_fit_2(self):
+        mx = 5
+        vals = random.randint(mx, size=(5, 20)).get()
+        counts = numpy.histogram(vals, bins=numpy.arange(mx + 1))[0]
+        expected = numpy.array([float(vals.size) / mx] * mx)
+        assert _hypothesis.chi_square_test(counts, expected)
+
+
+class TestRandintDtype(unittest.TestCase):
+
+    @testing.for_dtypes([
+        numpy.int8, numpy.uint8, numpy.int16, numpy.uint16, numpy.int32])
+    def test_dtype(self, dtype):
+        size = (1000,)
+        low = numpy.iinfo(dtype).min
+        high = numpy.iinfo(dtype).max + 1
+        x = random.randint(low, high, size, dtype).get()
+        assert low <= min(x)
+        assert max(x) <= high
+
+    @testing.for_int_dtypes(no_bool=True)
+    def test_dtype2(self, dtype):
+        dtype = numpy.dtype(dtype)
+
+        # randint does not support 64 bit integers
+        if dtype in (numpy.int64, numpy.uint64):
+            return
+
+        iinfo = numpy.iinfo(dtype)
+        size = (10000,)
+
+        x = random.randint(iinfo.min, iinfo.max + 1, size, dtype).get()
+        assert x.dtype == dtype
+        assert iinfo.min <= min(x)
+        assert max(x) <= iinfo.max
+
+        # Lower bound check
+        with self.assertRaises(ValueError):
+            random.randint(iinfo.min - 1, iinfo.min + 10, size, dtype)
+
+        # Upper bound check
+        with self.assertRaises(ValueError):
+            random.randint(iinfo.max - 10, iinfo.max + 2, size, dtype)
+
+
+class TestRandomIntegers(unittest.TestCase):
+
+    def test_normal(self):
+        with mock.patch('cupy.random._sample.randint') as m:
+            random.random_integers(3, 5)
+        m.assert_called_with(3, 6, None)
+
+    def test_high_is_none(self):
+        with mock.patch('cupy.random._sample.randint') as m:
+            random.random_integers(3, None)
+        m.assert_called_with(1, 4, None)
+
+    def test_size_is_not_none(self):
+        with mock.patch('cupy.random._sample.randint') as m:
+            random.random_integers(3, 5, (1, 2, 3))
+        m.assert_called_with(3, 6, (1, 2, 3))
+
+
+@testing.fix_random()
+class TestRandomIntegers2(unittest.TestCase):
+
+    @_condition.repeat(3, 10)
+    def test_bound_1(self):
+        vals = [random.random_integers(0, 10, (2, 3)).get() for _ in range(10)]
+        for val in vals:
+            assert val.shape == (2, 3)
+        assert min(_.min() for _ in vals) == 0
+        assert max(_.max() for _ in vals) == 10
+
+    @_condition.repeat(3, 10)
+    def test_bound_2(self):
+        vals = [random.random_integers(0, 2).get() for _ in range(20)]
+        for val in vals:
+            assert val.shape == ()
+        assert min(vals) == 0
+        assert max(vals) == 2
+
+    @_condition.repeat(3, 10)
+    def test_goodness_of_fit(self):
+        mx = 5
+        trial = 100
+        vals = [random.randint(0, mx).get() for _ in range(trial)]
+        counts = numpy.histogram(vals, bins=numpy.arange(mx + 1))[0]
+        expected = numpy.array([float(trial) / mx] * mx)
+        assert _hypothesis.chi_square_test(counts, expected)
+
+    @_condition.repeat(3, 10)
+    @pytest.mark.xfail(runtime.is_hip, reason='ROCm/HIP may have a bug')
+    def test_goodness_of_fit_2(self):
+        mx = 5
+        vals = random.randint(0, mx, (5, 20)).get()
+        counts = numpy.histogram(vals, bins=numpy.arange(mx + 1))[0]
+        expected = numpy.array([float(vals.size) / mx] * mx)
+        assert _hypothesis.chi_square_test(counts, expected)
+
+
+class TestChoice(unittest.TestCase):
+
+    def setUp(self):
+        self.rs_tmp = random._generator._random_states
+        device_id = cuda.Device().id
+        self.m = mock.Mock()
+        self.m.choice.return_value = 0
+        random._generator._random_states = {device_id: self.m}
+
+    def tearDown(self):
+        random._generator._random_states = self.rs_tmp
+
+    def test_size_and_replace_and_p_are_none(self):
+        random.choice(3)
+        self.m.choice.assert_called_with(3, None, True, None)
+
+    def test_size_and_replace_are_none(self):
+        random.choice(3, None, None, [0.1, 0.1, 0.8])
+        self.m.choice.assert_called_with(3, None, None, [0.1, 0.1, 0.8])
+
+    def test_size_and_p_are_none(self):
+        random.choice(3, None, True)
+        self.m.choice.assert_called_with(3, None, True, None)
+
+    def test_replace_and_p_are_none(self):
+        random.choice(3, 1)
+        self.m.choice.assert_called_with(3, 1, True, None)
+
+    def test_size_is_none(self):
+        random.choice(3, None, True, [0.1, 0.1, 0.8])
+        self.m.choice.assert_called_with(3, None, True, [0.1, 0.1, 0.8])
+
+    def test_replace_is_none(self):
+        random.choice(3, 1, None, [0.1, 0.1, 0.8])
+        self.m.choice.assert_called_with(3, 1, None, [0.1, 0.1, 0.8])
+
+    def test_p_is_none(self):
+        random.choice(3, 1, True)
+        self.m.choice.assert_called_with(3, 1, True, None)
+
+    def test_no_none(self):
+        random.choice(3, 1, True, [0.1, 0.1, 0.8])
+        self.m.choice.assert_called_with(3, 1, True, [0.1, 0.1, 0.8])
+
+
+class TestRandomSample(unittest.TestCase):
+
+    def test_rand(self):
+        with mock.patch('cupy.random._sample.random_sample') as m:
+            random.rand(1, 2, 3, dtype=numpy.float32)
+        m.assert_called_once_with(
+            size=(1, 2, 3), dtype=numpy.float32)
+
+    def test_rand_default_dtype(self):
+        with mock.patch('cupy.random._sample.random_sample') as m:
+            random.rand(1, 2, 3)
+        m.assert_called_once_with(
+            size=(1, 2, 3), dtype=float)
+
+    def test_rand_invalid_argument(self):
+        with self.assertRaises(TypeError):
+            random.rand(1, 2, 3, unnecessary='unnecessary_argument')
+
+    def test_randn(self):
+        with mock.patch('cupy.random._distributions.normal') as m:
+            random.randn(1, 2, 3, dtype=numpy.float32)
+        m.assert_called_once_with(
+            size=(1, 2, 3), dtype=numpy.float32)
+
+    def test_randn_default_dtype(self):
+        with mock.patch('cupy.random._distributions.normal') as m:
+            random.randn(1, 2, 3)
+        m.assert_called_once_with(
+            size=(1, 2, 3), dtype=float)
+
+    def test_randn_invalid_argument(self):
+        with self.assertRaises(TypeError):
+            random.randn(1, 2, 3, unnecessary='unnecessary_argument')
+
+
+@testing.parameterize(
+    {'size': None},
+    {'size': ()},
+    {'size': 4},
+    {'size': (0,)},
+    {'size': (1, 0)},
+)
+@testing.fix_random()
+class TestMultinomial(unittest.TestCase):
+
+    @_condition.repeat(3, 10)
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(rtol=0.05)
+    def test_multinomial(self, xp, dtype):
+        pvals = xp.array([0.2, 0.3, 0.5], dtype)
+        x = xp.random.multinomial(100000, pvals, self.size)
+        assert x.dtype == 'l'
+        return x / 100000
diff --git a/tests/cupy_tests/sorting_tests/__init__.py b/tests/cupy_tests/sorting_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/sorting_tests/test_count.py b/tests/cupy_tests/sorting_tests/test_count.py
new file mode 100644
index 0000000..22e17cb
--- /dev/null
+++ b/tests/cupy_tests/sorting_tests/test_count.py
@@ -0,0 +1,60 @@
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+class TestCount(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_count_nonzero(self, dtype):
+        def func(xp):
+            m = testing.shaped_random((2, 3), xp, xp.bool_)
+            a = testing.shaped_random((2, 3), xp, dtype) * m
+            c = xp.count_nonzero(a)
+            if xp is cupy:
+                # CuPy returns zero-dimensional array instead of
+                # returning a scalar value
+                assert isinstance(c, xp.ndarray)
+                assert c.dtype == 'l'
+                assert c.shape == ()
+            return int(c)
+        assert func(numpy) == func(cupy)
+
+    @testing.for_all_dtypes()
+    def test_count_nonzero_zero_dim(self, dtype):
+        def func(xp):
+            a = xp.array(1.0, dtype=dtype)
+            c = xp.count_nonzero(a)
+            if xp is cupy:
+                # CuPy returns zero-dimensional array instead of
+                # returning a scalar value
+                assert isinstance(c, xp.ndarray)
+                assert c.dtype == 'l'
+                assert c.shape == ()
+            return int(c)
+        assert func(numpy) == func(cupy)
+
+    @testing.for_all_dtypes()
+    def test_count_nonzero_int_axis(self, dtype):
+        for ax in range(3):
+            def func(xp):
+                m = testing.shaped_random((2, 3, 4), xp, xp.bool_)
+                a = testing.shaped_random((2, 3, 4), xp, dtype) * m
+                return xp.count_nonzero(a, axis=ax)
+            testing.assert_allclose(func(numpy), func(cupy))
+
+    @testing.for_all_dtypes()
+    def test_count_nonzero_tuple_axis(self, dtype):
+        for ax in range(3):
+            for ay in range(3):
+                if ax == ay:
+                    continue
+
+                def func(xp):
+                    m = testing.shaped_random((2, 3, 4), xp, xp.bool_)
+                    a = testing.shaped_random((2, 3, 4), xp, dtype) * m
+                    return xp.count_nonzero(a, axis=(ax, ay))
+                testing.assert_allclose(func(numpy), func(cupy))
diff --git a/tests/cupy_tests/sorting_tests/test_search.py b/tests/cupy_tests/sorting_tests/test_search.py
new file mode 100644
index 0000000..94603b0
--- /dev/null
+++ b/tests/cupy_tests/sorting_tests/test_search.py
@@ -0,0 +1,832 @@
+import numpy
+import pytest
+
+import cupy
+import cupy._core._accelerator as _acc
+from cupy._core import _cub_reduction
+from cupy import testing
+
+
+class TestSearch:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmax_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return a.argmax()
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_external_argmax_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.argmax(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(accept_error=ValueError)
+    def test_argmax_nan(self, xp, dtype):
+        a = xp.array([float('nan'), -1, 1], dtype)
+        return a.argmax()
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmax_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return a.argmax(axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_external_argmax_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return xp.argmax(a, axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmax_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return a.argmax(axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmax_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return a.argmax(axis=1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmax_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return a.argmax(axis=2)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmax_tie(self, xp, dtype):
+        a = xp.array([0, 5, 2, 3, 4, 5], dtype)
+        return a.argmax()
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_argmax_zero_size(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                a.argmax()
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_argmax_zero_size_axis0(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                a.argmax(axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmax_zero_size_axis1(self, xp, dtype):
+        a = testing.shaped_random((0, 1), xp, dtype)
+        return a.argmax(axis=1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmin_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return a.argmin()
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_argmin_nan(self, xp, dtype):
+        a = xp.array([float('nan'), -1, 1], dtype)
+        return a.argmin()
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_external_argmin_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.argmin(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmin_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return a.argmin(axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_external_argmin_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return xp.argmin(a, axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmin_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return a.argmin(axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmin_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return a.argmin(axis=1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmin_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return a.argmin(axis=2)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmin_tie(self, xp, dtype):
+        a = xp.array([0, 1, 2, 3, 0, 5], dtype)
+        return a.argmin()
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_argmin_zero_size(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                return a.argmin()
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_argmin_zero_size_axis0(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                a.argmin(axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_argmin_zero_size_axis1(self, xp, dtype):
+        a = testing.shaped_random((0, 1), xp, dtype)
+        return a.argmin(axis=1)
+
+
+# TODO(leofang): remove this once CUDA 9.0 is dropped
+def _skip_cuda90(dtype):
+    ver = cupy.cuda.runtime.runtimeGetVersion()
+    if dtype == cupy.float16 and ver == 9000:
+        pytest.skip('CUB does not support fp16 on CUDA 9.0')
+
+
+# This class compares CUB results against NumPy's
+# TODO(leofang): test axis after support is added
+@testing.parameterize(*testing.product({
+    'shape': [(10,), (10, 20), (10, 20, 30), (10, 20, 30, 40)],
+    'order_and_axis': (('C', -1), ('C', None), ('F', 0), ('F', None)),
+    'backend': ('device', 'block'),
+}))
+@pytest.mark.skipif(
+    not cupy.cuda.cub.available, reason='The CUB routine is not enabled')
+class TestCubReduction:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.order, self.axis = self.order_and_axis
+        old_routine_accelerators = _acc.get_routine_accelerators()
+        old_reduction_accelerators = _acc.get_reduction_accelerators()
+        if self.backend == 'device':
+            if self.axis is not None:
+                pytest.skip('does not support')
+            _acc.set_routine_accelerators(['cub'])
+            _acc.set_reduction_accelerators([])
+        elif self.backend == 'block':
+            _acc.set_routine_accelerators([])
+            _acc.set_reduction_accelerators(['cub'])
+        yield
+        _acc.set_routine_accelerators(old_routine_accelerators)
+        _acc.set_reduction_accelerators(old_reduction_accelerators)
+
+    @testing.for_dtypes('bhilBHILefdFD')
+    @testing.numpy_cupy_allclose(rtol=1E-5, contiguous_check=False)
+    def test_cub_argmin(self, xp, dtype):
+        _skip_cuda90(dtype)
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order == 'C':
+            a = xp.ascontiguousarray(a)
+        else:
+            a = xp.asfortranarray(a)
+
+        if xp is numpy:
+            return a.argmin(axis=self.axis)
+
+        # xp is cupy, first ensure we really use CUB
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        if self.backend == 'device':
+            func_name = 'cupy._core._routines_statistics.cub.'
+            func_name += 'device_reduce'
+            with testing.AssertFunctionIsCalled(func_name, return_value=ret):
+                a.argmin(axis=self.axis)
+        elif self.backend == 'block':
+            # this is the only function we can mock; the rest is cdef'd
+            func_name = 'cupy._core._cub_reduction.'
+            func_name += '_SimpleCubReductionKernel_get_cached_function'
+            func = _cub_reduction._SimpleCubReductionKernel_get_cached_function
+            if self.axis is not None and len(self.shape) > 1:
+                times_called = 1  # one pass
+            else:
+                times_called = 2  # two passes
+            with testing.AssertFunctionIsCalled(
+                    func_name, wraps=func, times_called=times_called):
+                a.argmin(axis=self.axis)
+        # ...then perform the actual computation
+        return a.argmin(axis=self.axis)
+
+    @testing.for_dtypes('bhilBHILefdFD')
+    @testing.numpy_cupy_allclose(rtol=1E-5, contiguous_check=False)
+    def test_cub_argmax(self, xp, dtype):
+        _skip_cuda90(dtype)
+        a = testing.shaped_random(self.shape, xp, dtype)
+        if self.order == 'C':
+            a = xp.ascontiguousarray(a)
+        else:
+            a = xp.asfortranarray(a)
+
+        if xp is numpy:
+            return a.argmax(axis=self.axis)
+
+        # xp is cupy, first ensure we really use CUB
+        ret = cupy.empty(())  # Cython checks return type, need to fool it
+        if self.backend == 'device':
+            func_name = 'cupy._core._routines_statistics.cub.'
+            func_name += 'device_reduce'
+            with testing.AssertFunctionIsCalled(func_name, return_value=ret):
+                a.argmax(axis=self.axis)
+        elif self.backend == 'block':
+            # this is the only function we can mock; the rest is cdef'd
+            func_name = 'cupy._core._cub_reduction.'
+            func_name += '_SimpleCubReductionKernel_get_cached_function'
+            func = _cub_reduction._SimpleCubReductionKernel_get_cached_function
+            if self.axis is not None and len(self.shape) > 1:
+                times_called = 1  # one pass
+            else:
+                times_called = 2  # two passes
+            with testing.AssertFunctionIsCalled(
+                    func_name, wraps=func, times_called=times_called):
+                a.argmax(axis=self.axis)
+        # ...then perform the actual computation
+        return a.argmax(axis=self.axis)
+
+
+@testing.parameterize(*testing.product({
+    'func': ['argmin', 'argmax'],
+    'is_module': [True, False],
+    'shape': [(3, 4), ()],
+}))
+class TestArgMinMaxDtype:
+
+    @testing.for_dtypes(
+        dtypes=[numpy.int8, numpy.int16, numpy.int32, numpy.int64],
+        name='result_dtype')
+    @testing.for_all_dtypes(name='in_dtype')
+    def test_argminmax_dtype(self, in_dtype, result_dtype):
+        a = testing.shaped_random(self.shape, cupy, in_dtype)
+        if self.is_module:
+            func = getattr(cupy, self.func)
+            y = func(a, dtype=result_dtype)
+        else:
+            func = getattr(a, self.func)
+            y = func(dtype=result_dtype)
+        assert y.shape == ()
+        assert y.dtype == result_dtype
+
+
+@testing.parameterize(
+    {'cond_shape': (2, 3, 4), 'x_shape': (2, 3, 4), 'y_shape': (2, 3, 4)},
+    {'cond_shape': (4,),      'x_shape': (2, 3, 4), 'y_shape': (2, 3, 4)},
+    {'cond_shape': (2, 3, 4), 'x_shape': (2, 3, 4), 'y_shape': (3, 4)},
+    {'cond_shape': (3, 4),    'x_shape': (2, 3, 4), 'y_shape': (4,)},
+)
+class TestWhereTwoArrays:
+
+    @testing.for_all_dtypes_combination(
+        names=['cond_type', 'x_type', 'y_type'])
+    @testing.numpy_cupy_allclose()
+    def test_where_two_arrays(self, xp, cond_type, x_type, y_type):
+        m = testing.shaped_random(self.cond_shape, xp, xp.bool_)
+        # Almost all values of a matrix `shaped_random` makes are not zero.
+        # To make a sparse matrix, we need multiply `m`.
+        cond = testing.shaped_random(self.cond_shape, xp, cond_type) * m
+        x = testing.shaped_random(self.x_shape, xp, x_type, seed=0)
+        y = testing.shaped_random(self.y_shape, xp, y_type, seed=1)
+        return xp.where(cond, x, y)
+
+
+@testing.parameterize(
+    {'cond_shape': (2, 3, 4)},
+    {'cond_shape': (4,)},
+    {'cond_shape': (2, 3, 4)},
+    {'cond_shape': (3, 4)},
+)
+class TestWhereCond:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_where_cond(self, xp, dtype):
+        m = testing.shaped_random(self.cond_shape, xp, xp.bool_)
+        cond = testing.shaped_random(self.cond_shape, xp, dtype) * m
+        return xp.where(cond)
+
+
+class TestWhereError:
+
+    def test_one_argument(self):
+        for xp in (numpy, cupy):
+            cond = testing.shaped_random((3, 4), xp, dtype=xp.bool_)
+            x = testing.shaped_random((2, 3, 4), xp, xp.int32)
+            with pytest.raises(ValueError):
+                xp.where(cond, x)
+
+
+@testing.parameterize(
+    {'array': numpy.random.randint(0, 2, (20,))},
+    {'array': numpy.random.randn(3, 2, 4)},
+    {'array': numpy.empty((0,))},
+    {'array': numpy.empty((0, 2))},
+    {'array': numpy.empty((0, 2, 0))},
+    _ids=False,  # Do not generate ids from randomly generated params
+)
+class TestNonzero:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_nonzero(self, xp, dtype):
+        array = xp.array(self.array, dtype=dtype)
+        return xp.nonzero(array)
+
+
+@testing.parameterize(
+    {'array': numpy.array(0)},
+    {'array': numpy.array(1)},
+)
+@testing.with_requires('numpy>=1.17.0')
+class TestNonzeroZeroDimension:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_nonzero(self, xp, dtype):
+        array = xp.array(self.array, dtype=dtype)
+        with testing.assert_warns(DeprecationWarning):
+            return xp.nonzero(array)
+
+
+@testing.parameterize(
+    {'array': numpy.random.randint(0, 2, (20,))},
+    {'array': numpy.random.randn(3, 2, 4)},
+    {'array': numpy.array(0)},
+    {'array': numpy.array(1)},
+    {'array': numpy.empty((0,))},
+    {'array': numpy.empty((0, 2))},
+    {'array': numpy.empty((0, 2, 0))},
+    _ids=False,  # Do not generate ids from randomly generated params
+)
+class TestFlatNonzero:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_flatnonzero(self, xp, dtype):
+        array = xp.array(self.array, dtype=dtype)
+        return xp.flatnonzero(array)
+
+
+@testing.parameterize(
+    {'array': numpy.random.randint(0, 2, (20,))},
+    {'array': numpy.random.randn(3, 2, 4)},
+    {'array': numpy.empty((0,))},
+    {'array': numpy.empty((0, 2))},
+    {'array': numpy.empty((0, 2, 0))},
+    _ids=False,  # Do not generate ids from randomly generated params
+)
+class TestArgwhere:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_argwhere(self, xp, dtype):
+        array = xp.array(self.array, dtype=dtype)
+        return xp.argwhere(array)
+
+
+@testing.parameterize(
+    {'value': 0},
+    {'value': 3},
+)
+@testing.with_requires('numpy>=1.18')
+class TestArgwhereZeroDimension:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_argwhere(self, xp, dtype):
+        array = xp.array(self.value, dtype=dtype)
+        return xp.argwhere(array)
+
+
+class TestNanArgMin:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.nanargmin(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_nan(self, xp, dtype):
+        a = xp.array([float('nan'), -1, 1], dtype)
+        return xp.nanargmin(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_nan2(self, xp, dtype):
+        a = xp.array([float('nan'), float('nan'), -1, 1], dtype)
+        return xp.nanargmin(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_nan3(self, xp, dtype):
+        a = xp.array([float('nan'), float('nan'), -1, 1, 1.0, -2.0], dtype)
+        return xp.nanargmin(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_nan4(self, xp, dtype):
+        a = xp.array([-1, 1, 1.0, -2.0, float('nan'), float('nan')],
+                     dtype)
+        return xp.nanargmin(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_nan5(self, xp, dtype):
+        a = xp.array([-1, 1, 1.0, -2.0, float('nan'), float('nan'), -1, 1],
+                     dtype)
+        return xp.nanargmin(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return xp.nanargmin(a, axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanargmin(a, axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanargmin(a, axis=1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanargmin(a, axis=2)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_tie(self, xp, dtype):
+        a = xp.array([0, 5, 2, 3, 4, 5], dtype)
+        return xp.nanargmin(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_nanargmin_zero_size(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.nanargmin(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_nanargmin_zero_size_axis0(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                return xp.nanargmin(a, axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmin_zero_size_axis1(self, xp, dtype):
+        a = testing.shaped_random((0, 1), xp, dtype)
+        return xp.nanargmin(a, axis=1)
+
+
+class TestNanArgMax:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.nanargmax(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_nan(self, xp, dtype):
+        a = xp.array([float('nan'), -1, 1], dtype)
+        return xp.nanargmax(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_nan2(self, xp, dtype):
+        a = xp.array([float('nan'), float('nan'), -1, 1], dtype)
+        return xp.nanargmax(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_nan3(self, xp, dtype):
+        a = xp.array([float('nan'), float('nan'), -1, 1, 1.0, -2.0], dtype)
+        return xp.nanargmax(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_nan4(self, xp, dtype):
+        a = xp.array([-1, 1, 1.0, -2.0, float('nan'), float('nan')],
+                     dtype)
+        return xp.nanargmax(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_nan5(self, xp, dtype):
+        a = xp.array([-1, 1, 1.0, -2.0, float('nan'), float('nan'), -1, 1],
+                     dtype)
+        return xp.nanargmax(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return xp.nanargmax(a, axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanargmax(a, axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanargmax(a, axis=1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanargmax(a, axis=2)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_tie(self, xp, dtype):
+        a = xp.array([0, 5, 2, 3, 4, 5], dtype)
+        return xp.nanargmax(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_nanargmax_zero_size(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.nanargmax(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_nanargmax_zero_size_axis0(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((0, 1), xp, dtype)
+            with pytest.raises(ValueError):
+                return xp.nanargmax(a, axis=0)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanargmax_zero_size_axis1(self, xp, dtype):
+        a = testing.shaped_random((0, 1), xp, dtype)
+        return xp.nanargmax(a, axis=1)
+
+
+@testing.parameterize(*testing.product(
+    {'bins': [
+        [],
+        [0, 1, 2, 4, 10],
+        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+        [0.0, 1.0, 2.5, 4.0, 10.0],
+        [-1.0, 1.0, 2.5, 4.0, 20.0],
+        [1.5, 2.5, 4.0, 6.0],
+        [float('-inf'), 1.5, 2.5, 4.0, 6.0],
+        [1.5, 2.5, 4.0, 6.0, float('inf')],
+        [float('-inf'), 1.5, 2.5, 4.0, 6.0, float('inf')],
+        [0.0, 1.0, 1.0, 4.0, 4.0, 10.0],
+        [0.0, 1.0, 1.0, 4.0, 4.0, 4.0, 4.0, 10.0],
+    ],
+        'side': ['left', 'right'],
+        'shape': [(), (10,), (6, 3, 3)]})
+)
+class TestSearchSorted:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype)
+        bins = xp.array(self.bins)
+        y = xp.searchsorted(bins, x, side=self.side)
+        return y,
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_ndarray_searchsorted(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype)
+        bins = xp.array(self.bins)
+        y = bins.searchsorted(x, side=self.side)
+        return y,
+
+
+@testing.parameterize(
+    {'side': 'left'},
+    {'side': 'right'})
+class TestSearchSortedNanInf:
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_nanbins(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        bins = xp.array([0, 1, 2, 4, 10, float('nan')])
+        y = xp.searchsorted(bins, x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_nan(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[5] = float('nan')
+        bins = xp.array([0, 1, 2, 4, 10])
+        y = xp.searchsorted(bins, x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_nan_last(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[-1] = float('nan')
+        bins = xp.array([0, 1, 2, 4, float('nan')])
+        y = xp.searchsorted(bins, x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_nan_last_repeat(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[-1] = float('nan')
+        bins = xp.array([0, 1, 2, float('nan'), float('nan')])
+        y = xp.searchsorted(bins, x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_all_nans(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[-1] = float('nan')
+        bins = xp.array([float('nan'), float('nan'), float('nan'),
+                         float('nan'), float('nan')])
+        y = xp.searchsorted(bins, x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_inf(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[5] = float('inf')
+        bins = xp.array([0, 1, 2, 4, 10])
+        y = xp.searchsorted(bins, x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_minf(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[5] = float('-inf')
+        bins = xp.array([0, 1, 2, 4, 10])
+        y = xp.searchsorted(bins, x, side=self.side)
+        return y,
+
+
+class TestSearchSortedInvalid:
+
+    # Cant test unordered bins due to numpy undefined
+    # behavior for searchsorted
+
+    def test_searchsorted_ndbins(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((10,), xp, xp.float64)
+            bins = xp.array([[10, 4], [2, 1], [7, 8]])
+            with pytest.raises(ValueError):
+                xp.searchsorted(bins, x)
+
+    def test_ndarray_searchsorted_ndbins(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((10,), xp, xp.float64)
+            bins = xp.array([[10, 4], [2, 1], [7, 8]])
+            with pytest.raises(ValueError):
+                bins.searchsorted(x)
+
+
+class TestSearchSortedWithSorter:
+
+    @testing.numpy_cupy_array_equal()
+    def test_sorter(self, xp):
+        x = testing.shaped_arange((12,), xp, xp.float64)
+        bins = xp.array([10, 4, 2, 1, 8])
+        sorter = xp.array([3, 2, 1, 4, 0])
+        y = xp.searchsorted(bins, x, sorter=sorter)
+        return y,
+
+    def test_invalid_sorter(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((12,), xp, xp.float64)
+            bins = xp.array([10, 4, 2, 1, 8])
+            sorter = xp.array([0])
+            with pytest.raises(ValueError):
+                xp.searchsorted(bins, x, sorter=sorter)
+
+    def test_nonint_sorter(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((12,), xp, xp.float64)
+            bins = xp.array([10, 4, 2, 1, 8])
+            sorter = xp.array([], dtype=xp.float64)
+            with pytest.raises(TypeError):
+                xp.searchsorted(bins, x, sorter=sorter)
+
+
+@testing.parameterize(
+    {'side': 'left'},
+    {'side': 'right'})
+class TestNdarraySearchSortedNanInf:
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_nanbins(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        bins = xp.array([0, 1, 2, 4, 10, float('nan')])
+        y = bins.searchsorted(x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_nan(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[5] = float('nan')
+        bins = xp.array([0, 1, 2, 4, 10])
+        y = bins.searchsorted(x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_nan_last(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[-1] = float('nan')
+        bins = xp.array([0, 1, 2, 4, float('nan')])
+        y = bins.searchsorted(x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_nan_last_repeat(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[-1] = float('nan')
+        bins = xp.array([0, 1, 2, float('nan'), float('nan')])
+        y = bins.searchsorted(x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_all_nans(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[-1] = float('nan')
+        bins = xp.array([float('nan'), float('nan'), float('nan'),
+                         float('nan'), float('nan')])
+        y = bins.searchsorted(x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_inf(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[5] = float('inf')
+        bins = xp.array([0, 1, 2, 4, 10])
+        y = bins.searchsorted(x, side=self.side)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_minf(self, xp):
+        x = testing.shaped_arange((10,), xp, xp.float64)
+        x[5] = float('-inf')
+        bins = xp.array([0, 1, 2, 4, 10])
+        y = bins.searchsorted(x, side=self.side)
+        return y,
+
+
+class TestNdarraySearchSortedWithSorter:
+
+    @testing.numpy_cupy_array_equal()
+    def test_sorter(self, xp):
+        x = testing.shaped_arange((12,), xp, xp.float64)
+        bins = xp.array([10, 4, 2, 1, 8])
+        sorter = xp.array([3, 2, 1, 4, 0])
+        y = bins.searchsorted(x, sorter=sorter)
+        return y,
+
+    def test_invalid_sorter(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((12,), xp, xp.float64)
+            bins = xp.array([10, 4, 2, 1, 8])
+            sorter = xp.array([0])
+            with pytest.raises(ValueError):
+                bins.searchsorted(x, sorter=sorter)
+
+    def test_nonint_sorter(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((12,), xp, xp.float64)
+            bins = xp.array([10, 4, 2, 1, 8])
+            sorter = xp.array([], dtype=xp.float64)
+            with pytest.raises(TypeError):
+                bins.searchsorted(x, sorter=sorter)
diff --git a/tests/cupy_tests/sorting_tests/test_sort.py b/tests/cupy_tests/sorting_tests/test_sort.py
new file mode 100644
index 0000000..bc652db
--- /dev/null
+++ b/tests/cupy_tests/sorting_tests/test_sort.py
@@ -0,0 +1,783 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestSort(unittest.TestCase):
+
+    # Test ranks
+
+    def test_sort_zero_dim(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            with pytest.raises(numpy.AxisError):
+                a.sort()
+
+    def test_external_sort_zero_dim(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.sort(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_sort_two_or_more_dim(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        a.sort()
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_sort_two_or_more_dim(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        return xp.sort(a)
+
+    # Test dtypes
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_sort_dtype(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        a.sort()
+        return a
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_external_sort_dtype(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        return xp.sort(a)
+
+    # Test contiguous arrays
+
+    @testing.numpy_cupy_array_equal()
+    def test_sort_contiguous(self, xp):
+        a = testing.shaped_random((10,), xp)  # C contiguous view
+        a.sort()
+        return a
+
+    def test_sort_non_contiguous(self):
+        a = testing.shaped_random((10,), cupy)[::2]  # Non contiguous view
+        with self.assertRaises(NotImplementedError):
+            a.sort()
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_sort_contiguous(self, xp):
+        a = testing.shaped_random((10,), xp)  # C contiguous view
+        return xp.sort(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_sort_non_contiguous(self, xp):
+        a = testing.shaped_random((10,), xp)[::2]  # Non contiguous view
+        return xp.sort(a)
+
+    # Test axis
+
+    @testing.numpy_cupy_array_equal()
+    def test_sort_axis1(self, xp):
+        a = testing.shaped_random((2, 3, 4), xp)
+        a.sort(axis=0)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_sort_axis2(self, xp):
+        a = testing.shaped_random((2, 3, 4), xp)
+        a.sort(axis=1)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_sort_axis3(self, xp):
+        a = testing.shaped_random((2, 3, 4), xp)
+        a.sort(axis=2)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_sort_axis(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        return xp.sort(a, axis=0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_sort_negative_axis(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        a.sort(axis=-2)
+        return a
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_sort_negative_axis(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        return xp.sort(a, axis=-2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_external_sort_none_axis(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        return xp.sort(a, axis=None)
+
+    def test_sort_invalid_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 3, 3), xp)
+            with pytest.raises(numpy.AxisError):
+                a.sort(axis=3)
+
+    def test_sort_invalid_axis2(self):
+        a = testing.shaped_random((2, 3, 3), cupy)
+        with self.assertRaises(numpy.AxisError):
+            a.sort(axis=3)
+
+    def test_external_sort_invalid_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 3, 3), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.sort(a, axis=3)
+
+    def test_external_sort_invalid_axis2(self):
+        a = testing.shaped_random((2, 3, 3), cupy)
+        with self.assertRaises(numpy.AxisError):
+            cupy.sort(a, axis=3)
+
+    def test_sort_invalid_negative_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 3, 3), xp)
+            with pytest.raises(numpy.AxisError):
+                a.sort(axis=-4)
+
+    def test_sort_invalid_negative_axis2(self):
+        a = testing.shaped_random((2, 3, 3), cupy)
+        with self.assertRaises(numpy.AxisError):
+            a.sort(axis=-4)
+
+    def test_external_sort_invalid_negative_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 3, 3), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.sort(a, axis=-4)
+
+    def test_external_sort_invalid_negative_axis2(self):
+        a = testing.shaped_random((2, 3, 3), cupy)
+        with self.assertRaises(numpy.AxisError):
+            cupy.sort(a, axis=-4)
+
+    # Test NaN ordering
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan1(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        a[2] = a[6] = xp.nan
+        out = xp.sort(a)
+        return out
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        a[0, 2, 1] = a[1, 0, 3] = xp.nan
+        out = xp.sort(a, axis=0)
+        return out
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan3(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        a[0, 2, 1] = a[1, 0, 3] = xp.nan
+        out = xp.sort(a, axis=1)
+        return out
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan4(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        a[0, 2, 1] = a[1, 0, 3] = xp.nan
+        out = xp.sort(a, axis=2)
+        return out
+
+    # Large case
+
+    @testing.slow
+    @testing.numpy_cupy_array_equal()
+    def test_large(self, xp):
+        a = testing.shaped_random((17, 1023, 1023), xp)
+        return xp.sort(a, axis=-1)
+
+
+class TestLexsort(unittest.TestCase):
+
+    # Test ranks
+
+    # TODO(niboshi): Fix xfail
+    @pytest.mark.xfail(reason='Explicit error types required')
+    def test_lexsort_zero_dim(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            with pytest.raises(numpy.AxisError):
+                return xp.lexsort(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_lexsort_one_dim(self, xp):
+        a = testing.shaped_random((2,), xp)
+        return xp.lexsort(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_lexsort_two_dim(self, xp):
+        a = xp.array([[9, 4, 0, 4, 0, 2, 1],
+                      [1, 5, 1, 4, 3, 4, 4]])  # from numpy.lexsort example
+        return xp.lexsort(a)
+
+    def test_lexsort_three_or_more_dim(self):
+        a = testing.shaped_random((2, 10, 10), cupy)
+        with self.assertRaises(NotImplementedError):
+            return cupy.lexsort(a)
+
+    # Test dtypes
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_lexsort_dtype(self, xp, dtype):
+        a = testing.shaped_random((2, 10), xp, dtype)
+        return xp.lexsort(a)
+
+    # Test NaN ordering
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan1(self, xp, dtype):
+        a = testing.shaped_random((2, 10), xp, dtype)
+        a[0, 2] = a[0, 6] = xp.nan
+        return xp.lexsort(a)
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan2(self, xp, dtype):
+        a = testing.shaped_random((2, 10), xp, dtype)
+        a[1, 2] = a[0, 6] = xp.nan
+        return xp.lexsort(a)
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan3(self, xp, dtype):
+        a = testing.shaped_random((2, 10), xp, dtype)
+        a[1, 2] = a[1, 6] = xp.nan
+        return xp.lexsort(a)
+
+    # Test non C-contiguous input
+
+    @testing.numpy_cupy_array_equal()
+    def test_view(self, xp):
+        # from #3232
+        a = testing.shaped_random((4, 8), xp, dtype=xp.float64)
+        a = a.T[::-1]
+        return xp.lexsort(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_F_order(self, xp):
+        a = testing.shaped_random((4, 8), xp, dtype=xp.float64)
+        a = xp.asfortranarray(a)
+        assert a.flags.f_contiguous
+        assert not a.flags.c_contiguous
+        return xp.lexsort(a)
+
+
+@testing.parameterize(*testing.product({
+    'external': [False, True],
+}))
+class TestArgsort(unittest.TestCase):
+
+    def argsort(self, a, axis=-1):
+        if self.external:
+            xp = cupy.get_array_module(a)
+            return xp.argsort(a, axis=axis)
+        else:
+            return a.argsort(axis=axis)
+
+    # Test base cases
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_argsort_zero_dim(self, xp, dtype):
+        a = testing.shaped_random((), xp, dtype)
+        return self.argsort(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_argsort_one_dim(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        return self.argsort(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_argsort_multi_dim(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 3), xp, dtype)
+        return self.argsort(a)
+
+    @testing.numpy_cupy_array_equal()
+    def test_argsort_non_contiguous(self, xp):
+        a = xp.array([1, 0, 2, 3])[::2]
+        return self.argsort(a)
+
+    # Test axis
+
+    @testing.numpy_cupy_array_equal()
+    def test_argsort_axis(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        return self.argsort(a, axis=0)
+
+    @testing.numpy_cupy_array_equal()
+    def test_argsort_negative_axis(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        return self.argsort(a, axis=2)
+
+    @testing.numpy_cupy_array_equal()
+    def test_argsort_none_axis(self, xp):
+        a = testing.shaped_random((2, 3, 3), xp)
+        return self.argsort(a, axis=None)
+
+    def test_argsort_invalid_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 3, 3), xp)
+            with pytest.raises(numpy.AxisError):
+                self.argsort(a, axis=3)
+
+    def test_argsort_invalid_axis2(self):
+        a = testing.shaped_random((2, 3, 3), cupy)
+        with self.assertRaises(numpy.AxisError):
+            return self.argsort(a, axis=3)
+
+    @testing.numpy_cupy_array_equal()
+    def test_argsort_zero_dim_axis(self, xp):
+        a = testing.shaped_random((), xp)
+        return self.argsort(a, axis=0)
+
+    def test_argsort_zero_dim_invalid_axis(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            with pytest.raises(numpy.AxisError):
+                self.argsort(a, axis=1)
+
+    def test_argsort_invalid_negative_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 3, 3), xp)
+            with pytest.raises(numpy.AxisError):
+                self.argsort(a, axis=-4)
+
+    def test_argsort_invalid_negative_axis2(self):
+        a = testing.shaped_random((2, 3, 3), cupy)
+        with self.assertRaises(numpy.AxisError):
+            return self.argsort(a, axis=-4)
+
+    # Misc tests
+
+    def test_argsort_original_array_not_modified_one_dim(self):
+        a = testing.shaped_random((10,), cupy)
+        b = cupy.array(a)
+        self.argsort(a)
+        testing.assert_allclose(a, b)
+
+    def test_argsort_original_array_not_modified_multi_dim(self):
+        a = testing.shaped_random((2, 3, 3), cupy)
+        b = cupy.array(a)
+        self.argsort(a)
+        testing.assert_allclose(a, b)
+
+    # Test NaN ordering
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan1(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        a[2] = a[6] = xp.nan
+        return self.argsort(a)
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_nan2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        a[0, 2, 1] = a[1, 1, 3] = xp.nan
+        return self.argsort(a)
+
+
+@pytest.mark.filterwarnings('ignore:.*msort.*:DeprecationWarning')
+class TestMsort(unittest.TestCase):
+
+    # Test base cases
+
+    def test_msort_zero_dim(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.msort(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_msort_one_dim(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype)
+        return xp.msort(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_msort_multi_dim(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.msort(a)
+
+
+class TestSort_complex(unittest.TestCase):
+
+    def test_sort_complex_zero_dim(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            with pytest.raises(numpy.AxisError):
+                xp.sort_complex(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_sort_complex_1dim(self, xp, dtype):
+        a = testing.shaped_random((100,), xp, dtype)
+        return a, xp.sort_complex(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_sort_complex_ndim(self, xp, dtype):
+        a = testing.shaped_random((2, 5, 3), xp, dtype)
+        return a, xp.sort_complex(a)
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_array_equal()
+    def test_sort_complex_nan(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 5), xp, dtype)
+        a[0, 2, 1] = a[1, 0, 3] = xp.nan
+        return a, xp.sort_complex(a)
+
+
+@testing.parameterize(*testing.product({
+    'external': [False, True],
+    'length': [10, 20000],
+}))
+class TestPartition(unittest.TestCase):
+
+    def partition(self, a, kth, axis=-1):
+        if self.external:
+            xp = cupy.get_array_module(a)
+            return xp.partition(a, kth, axis=axis)
+        else:
+            a.partition(kth, axis=axis)
+            return a
+
+    # Test base cases
+
+    def test_partition_zero_dim(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            kth = 2
+            with pytest.raises(numpy.AxisError):
+                self.partition(a, kth)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_partition_one_dim(self, xp, dtype):
+        a = testing.shaped_random((self.length,), xp, dtype)
+        kth = 2
+        x = self.partition(a, kth)
+        assert xp.all(x[0:kth] <= x[kth:kth + 1])
+        assert xp.all(x[kth:kth + 1] <= x[kth + 1:])
+        return x[kth]
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_partition_multi_dim(self, xp, dtype):
+        a = testing.shaped_random((10, 10, self.length), xp, dtype)
+        kth = 2
+        x = self.partition(a, kth)
+        assert xp.all(x[:, :, 0:kth] <= x[:, :, kth:kth + 1])
+        assert xp.all(x[:, :, kth:kth + 1] <= x[:, :, kth + 1:])
+        return x[:, :, kth:kth + 1]
+
+    # Test non-contiguous array
+
+    @testing.numpy_cupy_equal()
+    def test_partition_non_contiguous(self, xp):
+        a = testing.shaped_random((self.length,), xp)[::-1]
+        kth = 2
+        if not self.external:
+            if xp is cupy:
+                with self.assertRaises(NotImplementedError):
+                    return self.partition(a, kth)
+            return 0  # dummy
+        else:
+            x = self.partition(a, kth)
+            assert xp.all(x[0:kth] <= x[kth:kth + 1])
+            assert xp.all(x[kth:kth + 1] <= x[kth + 1:])
+            return x[kth]
+
+    # Test kth
+
+    @testing.numpy_cupy_equal()
+    def test_partition_sequence_kth(self, xp):
+        a = testing.shaped_random((self.length,), xp)
+        kth = (2, 4)
+        x = self.partition(a, kth)
+        return x[kth[0]], x[kth[1]]
+
+    @testing.numpy_cupy_equal()
+    def test_partition_negative_kth(self, xp):
+        a = testing.shaped_random((self.length,), xp)
+        kth = -3
+        x = self.partition(a, kth)
+        return x[kth]
+
+    def test_partition_invalid_kth(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((self.length,), xp)
+            kth = self.length
+            with pytest.raises(ValueError):
+                self.partition(a, kth)
+
+    def test_partition_invalid_negative_kth(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((self.length,), xp)
+            kth = -self.length - 1
+            with pytest.raises(ValueError):
+                self.partition(a, kth)
+
+    # Test axis
+
+    @testing.numpy_cupy_array_equal()
+    def test_partition_axis(self, xp):
+        a = testing.shaped_random((self.length, 10, 10), xp)
+        kth = 2
+        axis = 0
+        x = self.partition(a, kth, axis=axis)
+        return x[kth, :, :]
+
+    @testing.numpy_cupy_array_equal()
+    def test_partition_negative_axis(self, xp):
+        a = testing.shaped_random((10, 10, self.length), xp)
+        kth = 2
+        axis = -1
+        x = self.partition(a, kth, axis=axis)
+        return x[:, :, kth]
+
+    @testing.numpy_cupy_equal()
+    def test_partition_none_axis(self, xp):
+        if self.external:
+            a = testing.shaped_random((2, self.length), xp)
+            kth = 2
+            axis = None
+            x = self.partition(a, kth, axis=axis)
+            return x[kth]
+        else:
+            return None
+
+    def test_partition_invalid_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 2, self.length), xp)
+            kth = 2
+            axis = 3
+            with pytest.raises(numpy.AxisError):
+                self.partition(a, kth, axis=axis)
+
+    def test_partition_invalid_axis2(self):
+        a = testing.shaped_random((2, 2, self.length), cupy)
+        with self.assertRaises(numpy.AxisError):
+            kth = 2
+            axis = 3
+            return self.partition(a, kth, axis=axis)
+
+    def test_partition_invalid_negative_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 2, self.length), xp)
+            kth = 2
+            axis = -4
+            with pytest.raises(numpy.AxisError):
+                self.partition(a, kth, axis=axis)
+
+    def test_partition_invalid_negative_axis2(self):
+        a = testing.shaped_random((2, 2, self.length), cupy)
+        with self.assertRaises(numpy.AxisError):
+            kth = 2
+            axis = -4
+            return self.partition(a, kth, axis=axis)
+
+
+@testing.parameterize(*testing.product({
+    'external': [False, True],
+}))
+class TestArgpartition(unittest.TestCase):
+
+    def argpartition(self, a, kth, axis=-1):
+        if self.external:
+            xp = cupy.get_array_module(a)
+            return xp.argpartition(a, kth, axis=axis)
+        else:
+            return a.argpartition(kth, axis=axis)
+
+    # Test base cases
+
+    def test_argpartition_zero_dim(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((), xp)
+            kth = 2
+            with pytest.raises(ValueError):
+                self.argpartition(a, kth)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_equal()
+    def test_argpartition_one_dim(self, xp, dtype):
+        a = testing.shaped_random((10,), xp, dtype, 100)
+        kth = 2
+        idx = self.argpartition(a, kth)
+        assert (a[idx[:kth]] <= a[idx[kth]]).all()
+        assert (a[idx[kth]] <= a[idx[kth + 1:]]).all()
+        return a[idx[kth]]
+
+    # TODO(leofang): test all dtypes -- this workaround needs to be kept,
+    # likely due to #3287? Need investigation.
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_argpartition_multi_dim(self, xp, dtype):
+        a = testing.shaped_random((3, 3, 10), xp, dtype, 100)
+        kth = 2
+        idx = self.argpartition(a, kth)
+        rows = [[[0]], [[1]], [[2]]]
+        cols = [[[0], [1], [2]]]
+        assert (a[rows, cols, idx[:, :, :kth]] <
+                a[rows, cols, idx[:, :, kth:kth + 1]]).all()
+        assert (a[rows, cols, idx[:, :, kth:kth + 1]] <
+                a[rows, cols, idx[:, :, kth + 1:]]).all()
+        return idx[:, :, kth:kth + 1]
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_argpartition_multi_dim_kernel(self, xp, dtype):
+        # Use a larger scale for shaped_random to avoid duplicated numbers,
+        # which may make different indices at kth between NumPy and CuPy. Skip
+        # if int8 and uint8 not to overflow.
+        if dtype in (xp.int8, xp.uint8):
+            pytest.skip()
+        a = testing.shaped_random((3, 3, 256), xp, dtype, 10000)
+        kth = 20
+        idx = self.argpartition(a, kth, axis=-1)
+
+        rows = [[[0]], [[1]], [[2]]]
+        cols = [[[0], [1], [2]]]
+
+        assert (a[rows, cols, idx[:, :, :kth]] <=
+                a[rows, cols, idx[:, :, kth:kth + 1]]).all()
+        assert (a[rows, cols, idx[:, :, kth:kth + 1]] <=
+                a[rows, cols, idx[:, :, kth + 1:]]).all()
+        return idx[:, :, kth:kth + 1]
+
+    # Test non-contiguous array
+
+    @testing.numpy_cupy_equal()
+    def test_argpartition_non_contiguous(self, xp):
+        a = testing.shaped_random((10,), xp, 'i', 100)[::2]
+        kth = 2
+        idx = self.argpartition(a, kth)
+        assert (a[idx[:kth]] < a[idx[kth]]).all()
+        assert (a[idx[kth]] < a[idx[kth + 1:]]).all()
+        return idx[kth]
+
+    # Test kth
+
+    @testing.numpy_cupy_equal()
+    def test_argpartition_sequence_kth(self, xp):
+        a = testing.shaped_random((10,), xp, scale=100)
+        kth = (2, 4)
+        idx = self.argpartition(a, kth)
+        for _kth in kth:
+            assert (a[idx[:_kth]] < a[idx[_kth]]).all()
+            assert (a[idx[_kth]] < a[idx[_kth + 1:]]).all()
+        return (idx[2], idx[4])
+
+    @testing.numpy_cupy_equal()
+    def test_argpartition_negative_kth(self, xp):
+        a = testing.shaped_random((10,), xp, scale=100)
+        kth = -3
+        idx = self.argpartition(a, kth)
+        assert (a[idx[:kth]] < a[idx[kth]]).all()
+        assert (a[idx[kth]] < a[idx[kth + 1:]]).all()
+        return idx[kth]
+
+    def test_argpartition_invalid_kth(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((10,), xp, scale=100)
+            kth = 10
+            with pytest.raises(ValueError):
+                self.argpartition(a, kth)
+
+    def test_argpartition_invalid_negative_kth(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((10,), xp, scale=100)
+            kth = -11
+            with pytest.raises(ValueError):
+                self.argpartition(a, kth)
+
+    # Test axis
+
+    @testing.numpy_cupy_array_equal()
+    def test_argpartition_axis(self, xp):
+        a = testing.shaped_random((10, 3, 3), xp, scale=100)
+        kth = 2
+        axis = 0
+        idx = self.argpartition(a, kth, axis=axis)
+        rows = [[[0], [1], [2]]]
+        cols = [[[0, 1, 2]]]
+        assert (a[idx[:kth, :, :], rows, cols] <
+                a[idx[kth:kth + 1, :, :], rows, cols]).all()
+        assert (a[idx[kth:kth + 1, :, :], rows, cols] <
+                a[idx[kth + 1:, :, :], rows, cols]).all()
+        return idx[kth:kth + 1, :, :]
+
+    @testing.numpy_cupy_array_equal()
+    def test_argpartition_negative_axis(self, xp):
+        a = testing.shaped_random((3, 3, 10), xp, scale=100)
+        kth = 2
+        axis = -1
+        idx = self.argpartition(a, kth, axis=axis)
+        rows = [[[0]], [[1]], [[2]]]
+        cols = [[[0], [1], [2]]]
+        assert (a[rows, cols, idx[:, :, :kth]] <
+                a[rows, cols, idx[:, :, kth:kth + 1]]).all()
+        assert (a[rows, cols, idx[:, :, kth:kth + 1]] <
+                a[rows, cols, idx[:, :, kth + 1:]]).all()
+        return idx[:, :, kth:kth + 1]
+
+    @testing.numpy_cupy_equal()
+    def test_argpartition_none_axis(self, xp):
+        a = testing.shaped_random((2, 2), xp, scale=100)
+        kth = 2
+        axis = None
+        idx = self.argpartition(a, kth, axis=axis)
+        a1 = a.flatten()
+        assert (a1[idx[:kth]] < a1[idx[kth]]).all()
+        assert (a1[idx[kth]] < a1[idx[kth + 1:]]).all()
+        return idx[kth]
+
+    def test_argpartition_invalid_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 2, 2), xp, scale=100)
+            kth = 1
+            axis = 3
+            with pytest.raises(numpy.AxisError):
+                self.argpartition(a, kth, axis=axis)
+
+    def test_argpartition_invalid_axis2(self):
+        a = testing.shaped_random((2, 2, 2), cupy, scale=100)
+        kth = 1
+        axis = 3
+        with self.assertRaises(numpy.AxisError):
+            self.argpartition(a, kth, axis=axis)
+
+    def test_argpartition_invalid_negative_axis1(self):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((2, 2, 2), xp, scale=100)
+            kth = 1
+            axis = -4
+            with pytest.raises(numpy.AxisError):
+                self.argpartition(a, kth, axis=axis)
+
+    def test_argpartition_invalid_negative_axis2(self):
+        a = testing.shaped_random((2, 2, 2), cupy, scale=100)
+        kth = 1
+        axis = -4
+        with self.assertRaises(numpy.AxisError):
+            self.argpartition(a, kth, axis=axis)
diff --git a/tests/cupy_tests/statistics_tests/__init__.py b/tests/cupy_tests/statistics_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/statistics_tests/test_correlation.py b/tests/cupy_tests/statistics_tests/test_correlation.py
new file mode 100644
index 0000000..d3973ef
--- /dev/null
+++ b/tests/cupy_tests/statistics_tests/test_correlation.py
@@ -0,0 +1,184 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestCorrcoef(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_corrcoef(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.corrcoef(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_corrcoef_diag_exception(self, xp, dtype):
+        a = testing.shaped_arange((1, 3), xp, dtype)
+        return xp.corrcoef(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_corrcoef_y(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        y = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.corrcoef(a, y=y)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_corrcoef_rowvar(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        y = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.corrcoef(a, y=y, rowvar=False)
+
+    @testing.with_requires('numpy>=1.20')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(accept_error=True)
+    def test_corrcoef_dtype(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        y = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.corrcoef(a, y=y, dtype=dtype)
+
+
+class TestCov(unittest.TestCase):
+
+    def generate_input(self, a_shape, y_shape, xp, dtype):
+        a = testing.shaped_arange(a_shape, xp, dtype)
+        y = None
+        if y_shape is not None:
+            y = testing.shaped_arange(y_shape, xp, dtype)
+        return a, y
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(accept_error=True)
+    def check(self, a_shape, y_shape=None, rowvar=True, bias=False,
+              ddof=None, xp=None, dtype=None,
+              fweights=None, aweights=None, name=None):
+        a, y = self.generate_input(a_shape, y_shape, xp, dtype)
+        if fweights is not None:
+            fweights = name.asarray(fweights)
+        if aweights is not None:
+            aweights = name.asarray(aweights)
+        # print(type(fweights))
+        return xp.cov(a, y, rowvar, bias, ddof,
+                      fweights, aweights, dtype=dtype)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(accept_error=True)
+    def check_warns(self, a_shape, y_shape=None, rowvar=True, bias=False,
+                    ddof=None, xp=None, dtype=None,
+                    fweights=None, aweights=None):
+        with testing.assert_warns(RuntimeWarning):
+            a, y = self.generate_input(a_shape, y_shape, xp, dtype)
+            return xp.cov(a, y, rowvar, bias, ddof,
+                          fweights, aweights, dtype=dtype)
+
+    @testing.for_all_dtypes()
+    def check_raises(self, a_shape, y_shape=None,
+                     rowvar=True, bias=False, ddof=None,
+                     dtype=None, fweights=None, aweights=None):
+        for xp in (numpy, cupy):
+            a, y = self.generate_input(a_shape, y_shape, xp, dtype)
+            with pytest.raises(ValueError):
+                xp.cov(a, y, rowvar, bias, ddof,
+                       fweights, aweights, dtype=dtype)
+
+    def test_cov(self):
+        self.check((2, 3))
+        self.check((2,), (2,))
+        self.check((1, 3), (1, 3), rowvar=False)
+        self.check((2, 3), (2, 3), rowvar=False)
+        self.check((2, 3), bias=True)
+        self.check((2, 3), ddof=2)
+        self.check((2, 3))
+        self.check((1, 3), fweights=(1, 4, 1))
+        self.check((1, 3), aweights=(1.0, 4.0, 1.0))
+        self.check((1, 3), bias=True, aweights=(1.0, 4.0, 1.0))
+        self.check((1, 3), fweights=(1, 4, 1),
+                   aweights=(1.0, 4.0, 1.0))
+
+    def test_cov_warns(self):
+        self.check_warns((2, 3), ddof=3)
+        self.check_warns((2, 3), ddof=4)
+
+    def test_cov_raises(self):
+        self.check_raises((2, 3), ddof=1.2)
+        self.check_raises((3, 4, 2))
+        self.check_raises((2, 3), (3, 4, 2))
+
+    def test_cov_empty(self):
+        self.check((0, 1))
+
+
+@testing.parameterize(*testing.product({
+    'mode': ['valid', 'same', 'full'],
+    'shape1': [(5,), (6,), (20,), (21,)],
+    'shape2': [(5,), (6,), (20,), (21,)],
+}))
+class TestCorrelateShapeCombination(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_correlate(self, xp, dtype):
+        a = testing.shaped_arange(self.shape1, xp, dtype)
+        b = testing.shaped_arange(self.shape2, xp, dtype)
+        return xp.correlate(a, b, mode=self.mode)
+
+
+@pytest.mark.parametrize('mode', ['valid', 'full', 'same'])
+class TestCorrelate:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_correlate_non_contiguous(self, xp, dtype, mode):
+        a = testing.shaped_arange((300,), xp, dtype)
+        b = testing.shaped_arange((100,), xp, dtype)
+        return xp.correlate(a[::200], b[10::70], mode=mode)
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_correlate_large_non_contiguous(self, xp, dtype, mode):
+        a = testing.shaped_arange((10000,), xp, dtype)
+        b = testing.shaped_arange((1000,), xp, dtype)
+        return xp.correlate(a[200::], b[10::700], mode=mode)
+
+    @testing.for_all_dtypes_combination(names=['dtype1', 'dtype2'])
+    @testing.numpy_cupy_allclose(rtol=1e-2)
+    def test_correlate_diff_types(self, xp, dtype1, dtype2, mode):
+        a = testing.shaped_random((200,), xp, dtype1)
+        b = testing.shaped_random((100,), xp, dtype2)
+        return xp.correlate(a, b, mode=mode)
+
+
+@testing.parameterize(*testing.product({
+    'mode': ['valid', 'same', 'full']
+}))
+class TestCorrelateInvalid(unittest.TestCase):
+
+    @testing.with_requires('numpy>=1.18')
+    @testing.for_all_dtypes()
+    def test_correlate_empty(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.zeros((0,), dtype)
+            with pytest.raises(ValueError):
+                xp.correlate(a, a, mode=self.mode)
+
+    @testing.for_all_dtypes()
+    def test_correlate_ndim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((5, 10, 2), xp, dtype)
+            b = testing.shaped_arange((3, 4, 4), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.correlate(a, b, mode=self.mode)
+
+    @testing.for_all_dtypes()
+    def test_correlate_zero_dim(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_arange((), xp, dtype)
+            b = testing.shaped_arange((1,), xp, dtype)
+            with pytest.raises(ValueError):
+                xp.correlate(a, b, mode=self.mode)
diff --git a/tests/cupy_tests/statistics_tests/test_histogram.py b/tests/cupy_tests/statistics_tests/test_histogram.py
new file mode 100644
index 0000000..f54d089
--- /dev/null
+++ b/tests/cupy_tests/statistics_tests/test_histogram.py
@@ -0,0 +1,566 @@
+import sys
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+# Note that numpy.bincount does not support uint64 on 64-bit environment
+# as it casts an input array to intp.
+# And it does not support uint32, int64 and uint64 on 32-bit environment.
+_all_types = (
+    numpy.float16, numpy.float32, numpy.float64,
+    numpy.int8, numpy.int16, numpy.int32,
+    numpy.uint8, numpy.uint16,
+    numpy.bool_)
+_signed_types = (
+    numpy.int8, numpy.int16, numpy.int32,
+    numpy.bool_)
+
+if sys.maxsize > 2 ** 32:
+    _all_types = _all_types + (numpy.int64, numpy.uint32)
+    _signed_types = _signed_types + (numpy.int64,)
+
+
+def for_all_dtypes_bincount(name='dtype'):
+    return testing.for_dtypes(_all_types, name=name)
+
+
+def for_signed_dtypes_bincount(name='dtype'):
+    return testing.for_dtypes(_signed_types, name=name)
+
+
+def for_all_dtypes_combination_bincount(names):
+    return testing.for_dtypes_combination(_all_types, names=names)
+
+
+class TestHistogram(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram(self, xp, dtype):
+        x = testing.shaped_arange((10,), xp, dtype)
+        y, bin_edges = xp.histogram(x)
+        return y, bin_edges
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_same_value(self, xp, dtype):
+        x = xp.zeros(10, dtype)
+        y, bin_edges = xp.histogram(x, 3)
+        return y, bin_edges
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_density(self, xp, dtype):
+        x = testing.shaped_arange((10,), xp, dtype)
+        y, bin_edges = xp.histogram(x, density=True)
+        # check normalization
+        area = xp.sum(y * xp.diff(bin_edges))
+        testing.assert_allclose(area, 1)
+        return y, bin_edges
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_range_lower_outliers(self, xp, dtype):
+        # Check that lower outliers are not tallied
+        a = xp.arange(10, dtype=dtype) + .5
+        h, b = xp.histogram(a, range=[0, 9])
+        assert int(h.sum()) == 9
+        return h, b
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_range_upper_outliers(self, xp, dtype):
+        # Check that upper outliers are not tallied
+        a = xp.arange(10, dtype=dtype) + .5
+        h, b = xp.histogram(a, range=[1, 10])
+        assert int(h.sum()) == 9
+        return h, b
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_histogram_range_with_density(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype) + .5
+        h, b = xp.histogram(a, range=[1, 9], density=True)
+        # check normalization
+        testing.assert_allclose(float((h * xp.diff(b)).sum()), 1)
+        return h
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_histogram_range_with_weights_and_density(self, xp, dtype):
+        a = xp.arange(10, dtype=dtype) + .5
+        w = xp.arange(10, dtype=dtype) + .5
+        h, b = xp.histogram(a, range=[1, 9], weights=w, density=True)
+        testing.assert_allclose(float((h * xp.diff(b)).sum()), 1)
+        return h
+
+    def test_histogram_invalid_range(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(ValueError):
+                # range must be None or have two elements
+                xp.histogram(xp.arange(10), range=[1, 9, 15])
+
+    def test_histogram_invalid_range2(self):
+        for xp in (numpy, cupy):
+            with pytest.raises(TypeError):
+                xp.histogram(xp.arange(10), range=10)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    def test_histogram_weights_mismatch(self, dtype):
+        for xp in (numpy, cupy):
+            a = xp.arange(10, dtype=dtype) + .5
+            w = xp.arange(11, dtype=dtype) + .5
+            with pytest.raises(ValueError):
+                xp.histogram(a, range=[1, 9], weights=w, density=True)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_histogram_int_weights_dtype(self, xp, dtype):
+        # Check the type of the returned histogram
+        a = xp.arange(10, dtype=dtype)
+        h, b = xp.histogram(a, weights=xp.ones(10, int))
+        assert xp.issubdtype(h.dtype, xp.integer)
+        return h
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_histogram_float_weights_dtype(self, xp, dtype):
+        # Check the type of the returned histogram
+        a = xp.arange(10, dtype=dtype)
+        h, b = xp.histogram(a, weights=xp.ones(10, float))
+        assert xp.issubdtype(h.dtype, xp.floating)
+        return h
+
+    def test_histogram_weights_basic(self):
+        v = cupy.random.rand(100)
+        w = cupy.ones(100) * 5
+        a, b = cupy.histogram(v)
+        na, nb = cupy.histogram(v, density=True)
+        wa, wb = cupy.histogram(v, weights=w)
+        nwa, nwb = cupy.histogram(v, weights=w, density=True)
+        testing.assert_array_almost_equal(a * 5, wa)
+        testing.assert_array_almost_equal(na, nwa)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_histogram_float_weights(self, xp, dtype):
+        # Check weights are properly applied.
+        v = xp.linspace(0, 10, 10, dtype=dtype)
+        w = xp.concatenate((xp.zeros(5, dtype=dtype), xp.ones(5, dtype=dtype)))
+        wa, wb = xp.histogram(v, bins=xp.arange(11), weights=w)
+        testing.assert_array_almost_equal(wa, w)
+        return wb
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal(type_check=False)
+    def test_histogram_int_weights(self, xp, dtype):
+        # Check with integer weights
+        v = xp.asarray([1, 2, 2, 4], dtype=dtype)
+        w = xp.asarray([4, 3, 2, 1], dtype=dtype)
+        wa, wb = xp.histogram(v, bins=4, weights=w)
+        testing.assert_array_equal(wa, [4, 5, 0, 1])
+        return wa, wb
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_histogram_int_weights_normalized(self, xp, dtype):
+        v = xp.asarray([1, 2, 2, 4], dtype=dtype)
+        w = xp.asarray([4, 3, 2, 1], dtype=dtype)
+        wa, wb = xp.histogram(v, bins=4, weights=w, density=True)
+        testing.assert_array_almost_equal(
+            wa, xp.asarray([4, 5, 0, 1]) / 10. / 3. * 4)
+        return wb
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_int_weights_nonuniform_bins(self, xp, dtype):
+        # Check weights with non-uniform bin widths
+        a, b = xp.histogram(
+            xp.arange(9, dtype=dtype),
+            xp.asarray([0, 1, 3, 6, 10], dtype=dtype),
+            weights=xp.asarray([2, 1, 1, 1, 1, 1, 1, 1, 1], dtype=dtype),
+            density=True)
+        testing.assert_array_almost_equal(a, [.2, .1, .1, .075])
+        return a, b
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal(type_check=False)
+    def test_histogram_complex_weights(self, xp, dtype):
+        values = xp.asarray([1.3, 2.5, 2.3])
+        weights = xp.asarray([1, -1, 2]) + 1j * xp.asarray([2, 1, 2])
+        weights = weights.astype(dtype)
+        a, b = xp.histogram(
+            values, bins=2, weights=weights)
+        return a, b
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_array_equal(type_check=False)
+    def test_histogram_complex_weights_uneven_bins(self, xp, dtype):
+        values = xp.asarray([1.3, 2.5, 2.3])
+        weights = xp.asarray([1, -1, 2]) + 1j * xp.asarray([2, 1, 2])
+        weights = weights.astype(dtype)
+        a, b = xp.histogram(
+            values, bins=xp.asarray([0, 2, 3]), weights=weights)
+        return a, b
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_empty(self, xp, dtype):
+        x = xp.array([], dtype)
+        y, bin_edges = xp.histogram(x)
+        return y, bin_edges
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_int_bins(self, xp, dtype):
+        x = testing.shaped_arange((10,), xp, dtype)
+        y, bin_edges = xp.histogram(x, 4)
+        return y, bin_edges
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_array_bins(self, xp, dtype):
+        x = testing.shaped_arange((10,), xp, dtype)
+        bins = testing.shaped_arange((3,), xp, dtype)
+        y, bin_edges = xp.histogram(x, bins)
+        return y, bin_edges
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_numpy_bins(self, xp, dtype):
+        x = testing.shaped_arange((10,), xp, dtype)
+        bins = testing.shaped_arange((3,), numpy, dtype)
+        y, bin_edges = xp.histogram(x, bins)
+        return y, bin_edges
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_histogram_list_bins(self, xp, dtype):
+        x = testing.shaped_arange((10,), xp, dtype)
+        bins = list(testing.shaped_arange((3,), numpy, dtype))
+        y, bin_edges = xp.histogram(x, bins)
+        return y, bin_edges
+
+    # numpy 1.13.1 does not check this error correctly with unsigned int.
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    def test_histogram_bins_not_ordered(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((10,), xp, dtype)
+            bins = xp.array([1, 3, 2], dtype)
+            with pytest.raises(ValueError):
+                xp.histogram(x, bins)
+
+    @for_all_dtypes_bincount()
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_bincount(self, xp, dtype):
+        x = testing.shaped_arange((3,), xp, dtype)
+        return xp.bincount(x)
+
+    @for_all_dtypes_bincount()
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_bincount_duplicated_value(self, xp, dtype):
+        x = xp.array([1, 2, 2, 1, 2, 4], dtype)
+        return xp.bincount(x)
+
+    @for_all_dtypes_combination_bincount(names=['x_type', 'w_type'])
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_bincount_with_weight(self, xp, x_type, w_type):
+        x = testing.shaped_arange((3,), xp, x_type)
+        w = testing.shaped_arange((3,), xp, w_type)
+        return xp.bincount(x, weights=w)
+
+    @for_all_dtypes_bincount()
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_bincount_with_minlength(self, xp, dtype):
+        x = testing.shaped_arange((3,), xp, dtype)
+        return xp.bincount(x, minlength=5)
+
+    @for_all_dtypes_combination_bincount(names=['x_type', 'w_type'])
+    def test_bincount_invalid_weight_length(self, x_type, w_type):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((1,), xp, x_type)
+            w = testing.shaped_arange((2,), xp, w_type)
+            # TODO(imanishi): Split this test into a test for ValueError and
+            # a test for TypeError.
+            with pytest.raises((ValueError, TypeError)):
+                xp.bincount(x, weights=w)
+
+    @for_signed_dtypes_bincount()
+    def test_bincount_negative(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3,), xp, dtype) - 2
+            with pytest.raises(ValueError):
+                xp.bincount(x)
+
+    @for_all_dtypes_bincount()
+    def test_bincount_too_deep(self, dtype):
+        for xp in (numpy, cupy):
+            x = xp.array([[1]], dtype)
+            with pytest.raises(ValueError):
+                xp.bincount(x)
+
+    @for_all_dtypes_bincount()
+    def test_bincount_too_small(self, dtype):
+        for xp in (numpy, cupy):
+            x = xp.zeros((), dtype)
+            with pytest.raises(ValueError):
+                xp.bincount(x)
+
+    @for_all_dtypes_bincount()
+    @testing.numpy_cupy_allclose(accept_error=TypeError)
+    def test_bincount_zero(self, xp, dtype):
+        x = testing.shaped_arange((3,), xp, dtype)
+        return xp.bincount(x, minlength=0)
+
+    @for_all_dtypes_bincount()
+    def test_bincount_too_small_minlength(self, dtype):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((3,), xp, dtype)
+            # TODO(imanishi): Split this test into a test for ValueError and
+            # a test for TypeError.
+            with pytest.raises((ValueError, TypeError)):
+                xp.bincount(x, minlength=-1)
+
+
+# TODO(leofang): we temporarily remove CUB histogram support for now,
+# see cupy/cupy#7698. When it's ready, revert the commit that checked
+# in this comment to restore the support.
+
+
+@testing.parameterize(*testing.product(
+    {'bins': [
+        # Test monotonically increasing with in-bounds values
+        [1.5, 2.5, 4.0, 6.0],
+        # Explicit out-of-bounds for x values
+        [-1.0, 1.0, 2.5, 4.0, 20.0],
+        # Repeated values should yield right-most or left-most indexes
+        [0.0, 1.0, 1.0, 4.0, 4.0, 10.0],
+    ],
+        'increasing': [True, False],
+        'right': [True, False],
+        'shape': [(), (10,), (6, 3, 3)]})
+)
+class TestDigitize:
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_array_equal()
+    def test_digitize(self, xp, dtype):
+        x = testing.shaped_arange(self.shape, xp, dtype)
+        bins = self.bins
+        if not self.increasing:
+            bins = bins[::-1]
+        bins = xp.array(bins)
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+
+@testing.parameterize(
+    {'right': True},
+    {'right': False})
+class TestDigitizeNanInf(unittest.TestCase):
+
+    @testing.numpy_cupy_array_equal()
+    def test_digitize_nan(self, xp):
+        x = testing.shaped_arange((14,), xp, xp.float32)
+        x[5] = float('nan')
+        bins = xp.array([1.0, 3.0, 5.0, 8.0, 12.0], xp.float32)
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_digitize_nan_bins(self, xp):
+        x = testing.shaped_arange((14,), xp, xp.float32)
+        bins = xp.array([1.0, 3.0, 5.0, 8.0, float('nan')], xp.float32)
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_digitize_nan_bins_repeated(self, xp):
+        x = testing.shaped_arange((14,), xp, xp.float32)
+        x[5] = float('nan')
+        bins = [1.0, 3.0, 5.0, 8.0, float('nan'), float('nan')]
+        bins = xp.array(bins, xp.float32)
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_digitize_nan_bins_decreasing(self, xp):
+        x = testing.shaped_arange((14,), xp, xp.float32)
+        x[5] = float('nan')
+        bins = [float('nan'), 8.0, 5.0, 3.0, 1.0]
+        bins = xp.array(bins, xp.float32)
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_digitize_nan_bins_decreasing_repeated(self, xp):
+        x = testing.shaped_arange((14,), xp, xp.float32)
+        x[5] = float('nan')
+        bins = [float('nan'), float('nan'), float('nan'), 5.0, 3.0, 1.0]
+        bins = xp.array(bins, xp.float32)
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_digitize_all_nan_bins(self, xp):
+        x = testing.shaped_arange((14,), xp, xp.float32)
+        x[5] = float('nan')
+        bins = [float('nan'), float('nan'), float('nan'), float('nan')]
+        bins = xp.array(bins, xp.float32)
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_inf(self, xp):
+        x = testing.shaped_arange((14,), xp, xp.float64)
+        x[5] = float('inf')
+        bins = xp.array([0, 1, 2, 4, 10])
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+    @testing.numpy_cupy_array_equal()
+    def test_searchsorted_minf(self, xp):
+        x = testing.shaped_arange((14,), xp, xp.float64)
+        x[5] = float('-inf')
+        bins = xp.array([0, 1, 2, 4, 10])
+        y = xp.digitize(x, bins, right=self.right)
+        return y,
+
+
+class TestDigitizeInvalid(unittest.TestCase):
+
+    def test_digitize_complex(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((14,), xp, complex)
+            bins = xp.array([1.0, 3.0, 5.0, 8.0, 12.0], complex)
+            with pytest.raises(TypeError):
+                xp.digitize(x, bins)
+
+    def test_digitize_nd_bins(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_arange((14,), xp, xp.float64)
+            bins = xp.array([[1], [2]])
+            with pytest.raises(ValueError):
+                xp.digitize(x, bins)
+
+
+@testing.parameterize(
+    *testing.product(
+        {'weights': [None, 1, 2],
+         'weights_dtype': [numpy.int32, numpy.float64],
+         'density': [True, False],
+         'bins': [10, (8, 16, 12), (16, 8, 12), (16, 12, 8), (12, 8, 16),
+                  'array_list'],
+         'range': [None, ((20, 50), (10, 100), (0, 40))]}
+    )
+)
+class TestHistogramdd:
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-7, rtol=1e-7)
+    def test_histogramdd(self, xp, dtype):
+        x = testing.shaped_random((100, 3), xp, dtype, scale=100)
+        if self.bins == 'array_list':
+            bins = [xp.arange(0, 100, 4),
+                    xp.arange(0, 100, 10),
+                    xp.arange(25)]
+        else:
+            bins = self.bins
+        if self.weights is not None:
+            weights = xp.ones((x.shape[0],), dtype=self.weights_dtype)
+        else:
+            weights = None
+        y, bin_edges = xp.histogramdd(x, bins=bins, range=self.range,
+                                      weights=weights, density=self.density)
+        return [y, ] + [e for e in bin_edges]
+
+
+class TestHistogramddErrors(unittest.TestCase):
+
+    def test_histogramdd_invalid_bins(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_random((16, 2), xp, scale=100)
+            bins = [xp.arange(0, 100, 10), ] * 3
+            with pytest.raises(ValueError):
+                y, bin_edges = xp.histogramdd(x, bins)
+
+    def test_histogramdd_invalid_bins2(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_random((16, 2), xp, scale=100)
+            with pytest.raises(ValueError):
+                y, bin_edges = xp.histogramdd(x, bins=0)
+
+    def test_histogramdd_invalid_bins3(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_random((16, 2), xp, scale=100)
+            bins = xp.arange(100)
+            bins[30] = 99  # non-ascending bins
+            with pytest.raises(ValueError):
+                y, bin_edges = xp.histogramdd(x, bins=bins)
+
+    def test_histogramdd_invalid_bins4(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_random((16, 2), xp, scale=100)
+            bins = xp.arange(64).reshape((8, 8))  # too many dimensions
+            with pytest.raises(ValueError):
+                y, bin_edges = xp.histogramdd(x, bins=bins)
+
+    def test_histogramdd_invalid_range(self):
+        for xp in (numpy, cupy):
+            x = testing.shaped_random((16, 2), xp, scale=100)
+            r = ((0, 100),) * 3
+            with pytest.raises(ValueError):
+                y, bin_edges = xp.histogramdd(x, range=r)
+
+    def test_histogramdd_disallow_arraylike_bins(self):
+        x = testing.shaped_random((16, 2), cupy, scale=100)
+        bins = [[0, 10, 20, 50, 90]] * 2  # too many dimensions
+        with pytest.raises(ValueError):
+            y, bin_edges = cupy.histogramdd(x, bins=bins)
+
+
+@testing.parameterize(
+    *testing.product(
+        {'weights': [None, 1, 2],
+         'weights_dtype': [numpy.int32, numpy.float64],
+         'density': [True, False],
+         'bins': [10, (8, 16), (16, 8), 'array_list', 'array'],
+         'range': [None, ((20, 50), (10, 100))]}
+    )
+)
+class TestHistogram2d:
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-7, rtol=1e-7)
+    def test_histogram2d(self, xp, dtype):
+        x = testing.shaped_random((100, ), xp, dtype, scale=100)
+        y = testing.shaped_random((100, ), xp, dtype, scale=100)
+        if self.bins == 'array_list':
+            bins = [xp.arange(0, 100, 4), xp.arange(0, 100, 10)]
+        elif self.bins == 'array':
+            bins = xp.arange(0, 100, 4)
+        else:
+            bins = self.bins
+        if self.weights is not None:
+            weights = xp.ones((x.shape[0],), dtype=self.weights_dtype)
+        else:
+            weights = None
+        y, edges0, edges1 = xp.histogram2d(x, y, bins=bins,
+                                           range=self.range, weights=weights,
+                                           density=self.density)
+        return y, edges0, edges1
+
+
+class TestHistogram2dErrors(unittest.TestCase):
+
+    def test_histogram2d_disallow_arraylike_bins(self):
+        x = testing.shaped_random((16, ), cupy, scale=100)
+        y = testing.shaped_random((16, ), cupy, scale=100)
+        bins = [0, 10, 20, 50, 90]
+        with pytest.raises(ValueError):
+            y, bin_edges = cupy.histogram2d(x, y, bins=bins)
diff --git a/tests/cupy_tests/statistics_tests/test_meanvar.py b/tests/cupy_tests/statistics_tests/test_meanvar.py
new file mode 100644
index 0000000..b605a33
--- /dev/null
+++ b/tests/cupy_tests/statistics_tests/test_meanvar.py
@@ -0,0 +1,514 @@
+import math
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+ignore_runtime_warnings = pytest.mark.filterwarnings(
+    "ignore", category=RuntimeWarning)
+
+
+class TestMedian:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_median_noaxis(self, xp, dtype):
+        a = testing.shaped_random((3, 4, 5), xp, dtype)
+        return xp.median(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_median_axis1(self, xp, dtype):
+        a = testing.shaped_random((3, 4, 5), xp, dtype)
+        return xp.median(a, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_median_axis2(self, xp, dtype):
+        a = testing.shaped_random((3, 4, 5), xp, dtype)
+        return xp.median(a, axis=2)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_median_overwrite_input(self, xp, dtype):
+        a = testing.shaped_random((3, 4, 5), xp, dtype)
+        return xp.median(a, overwrite_input=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_median_keepdims_axis1(self, xp, dtype):
+        a = testing.shaped_random((3, 4, 5), xp, dtype)
+        return xp.median(a, axis=1, keepdims=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_median_keepdims_noaxis(self, xp, dtype):
+        a = testing.shaped_random((3, 4, 5), xp, dtype)
+        return xp.median(a, keepdims=True)
+
+    def test_median_invalid_axis(self):
+        for xp in [numpy, cupy]:
+            a = testing.shaped_random((3, 4, 5), xp)
+            with pytest.raises(numpy.AxisError):
+                return xp.median(a, -a.ndim - 1, keepdims=False)
+
+            with pytest.raises(numpy.AxisError):
+                return xp.median(a, a.ndim, keepdims=False)
+
+            with pytest.raises(numpy.AxisError):
+                return xp.median(a, (-a.ndim - 1, 1), keepdims=False)
+
+            with pytest.raises(numpy.AxisError):
+                return xp.median(a, (0, a.ndim,), keepdims=False)
+
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_allclose()
+    def test_median_nan(self, xp, dtype):
+        a = xp.array(
+            [[xp.nan, 1, 2, 3],
+             [4, 5, 6, 7],
+             [8, 9, 10, xp.nan]],
+            dtype=dtype,
+        )
+        return xp.median(a, axis=1)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(3, 4, 5)],
+        'axis': [(0, 1), (0, -1), (1, 2), (1,)],
+        'keepdims': [True, False]
+    })
+)
+class TestMedianAxis:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_median_axis_sequence(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        return xp.median(a, self.axis, keepdims=self.keepdims)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(3, 4, 5)],
+        'axis': [None, 0, 1, -1, (0, 1), (0, 2), (-1, -2), [0, 1]],
+        'keepdims': [True, False],
+        'overwrite_input': [True, False]
+    })
+)
+class TestNanMedian:
+
+    zero_density = 0.25
+
+    def _make_array(self, dtype):
+        dtype = numpy.dtype(dtype)
+        if dtype.char in 'efdFD':
+            r_dtype = dtype.char.lower()
+            a = testing.shaped_random(self.shape, numpy, dtype=r_dtype,
+                                      scale=1)
+            if dtype.char in 'FD':
+                ai = a
+                aj = testing.shaped_random(self.shape, numpy, dtype=r_dtype,
+                                           scale=1)
+                ai[ai < math.sqrt(self.zero_density)] = 0
+                aj[aj < math.sqrt(self.zero_density)] = 0
+                a = ai + 1j * aj
+            else:
+                a[a < self.zero_density] = 0
+            a = a / a
+        else:
+            a = testing.shaped_random(self.shape, numpy, dtype=dtype)
+        return a
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmedian(self, xp, dtype):
+        a = xp.array(self._make_array(dtype))
+        out = xp.nanmedian(a, self.axis, keepdims=self.keepdims,
+                           overwrite_input=self.overwrite_input)
+        return xp.ascontiguousarray(out)
+
+
+class TestAverage:
+
+    _multiprocess_can_split_ = True
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_average_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.average(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_average_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.average(a, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_average_weights(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        w = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.average(a, weights=w)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-1, 'default': 1e-7})
+    @pytest.mark.parametrize(
+        'axis,weights', [(1, False), (None, True), (1, True)])
+    def test_returned(self, xp, dtype, axis, weights):
+        a = testing.shaped_arange((2, 3), numpy, dtype)
+        if weights:
+            w = testing.shaped_arange((2, 3), numpy, dtype)
+        else:
+            w = None
+        return xp.average(a, axis=axis, weights=w, returned=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-2, 'default': 5e-7})
+    @pytest.mark.parametrize('returned', [True, False])
+    @testing.with_requires('numpy>=1.23.1')
+    def test_average_keepdims_axis1(self, xp, dtype, returned):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        w = testing.shaped_random((2, 3), xp, dtype)
+        return xp.average(
+            a, axis=1, weights=w, returned=returned, keepdims=True)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol={'default': 1e-7, numpy.float16: 1e-3})
+    @pytest.mark.parametrize('returned', [True, False])
+    @testing.with_requires('numpy>=1.23.1')
+    def test_average_keepdims_noaxis(self, xp, dtype, returned):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        w = testing.shaped_random((2, 3), xp, dtype)
+        return xp.average(a, weights=w, returned=returned, keepdims=True)
+
+
+class TestMeanVar:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_mean_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return a.mean()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_mean_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.mean(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_mean_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.mean(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_mean_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.mean(a, axis=1)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_mean_all_float64_dtype(self, xp, dtype):
+        a = xp.full((2, 3, 4), 123456789, dtype=dtype)
+        return xp.mean(a, dtype=numpy.float64)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_mean_all_int64_dtype(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.mean(a, dtype=numpy.int64)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_mean_all_complex_dtype(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.mean(a, dtype=numpy.complex64)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_var_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return a.var()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_var_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.var(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_var_all_ddof(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return a.var(ddof=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_var_all_ddof(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.var(a, ddof=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_var_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.var(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_var_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.var(a, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_var_axis_ddof(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.var(axis=1, ddof=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_var_axis_ddof(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.var(a, axis=1, ddof=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_std_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return a.std()
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_std_all(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.std(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_std_all_ddof(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return a.std(ddof=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_std_all_ddof(self, xp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return xp.std(a, ddof=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_std_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.std(axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_std_axis(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.std(a, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_std_axis_ddof(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return a.std(axis=1, ddof=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_external_std_axis_ddof(self, xp, dtype):
+        a = testing.shaped_arange((2, 3, 4), xp, dtype)
+        return xp.std(a, axis=1, ddof=1)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(3, 4), (30, 40, 50)],
+        'axis': [None, 0, 1],
+        'keepdims': [True, False]
+    })
+)
+class TestNanMean:
+
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanmean_without_nan(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+        return xp.nanmean(a, axis=self.axis, keepdims=self.keepdims)
+
+    @ignore_runtime_warnings
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanmean_with_nan_float(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype)
+
+        if a.dtype.kind not in 'biu':
+            a[1, :] = xp.nan
+            a[:, 3] = xp.nan
+
+        return xp.nanmean(a, axis=self.axis, keepdims=self.keepdims)
+
+
+class TestNanMeanAdditional:
+
+    @ignore_runtime_warnings
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanmean_out(self, xp, dtype):
+        a = testing.shaped_random((10, 20, 30), xp, dtype)
+        z = xp.zeros((20, 30), dtype=dtype)
+
+        if a.dtype.kind not in 'biu':
+            a[1, :] = xp.nan
+            a[:, 3] = xp.nan
+
+        xp.nanmean(a, axis=0, out=z)
+        return z
+
+    @testing.slow
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanmean_huge(self, xp, dtype):
+        a = testing.shaped_random((1024, 512), xp, dtype)
+
+        if a.dtype.kind not in 'biu':
+            a[:512, :256] = xp.nan
+
+        return xp.nanmean(a, axis=1)
+
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_nanmean_float16(self, xp):
+        a = testing.shaped_arange((2, 3), xp, numpy.float16)
+        a[0][0] = xp.nan
+        return xp.nanmean(a)
+
+    @ignore_runtime_warnings
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanmean_all_nan(self, xp):
+        a = xp.zeros((3, 4))
+        a[:] = xp.nan
+        return xp.nanmean(a)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [(3, 4), (4, 3, 5)],
+        'axis': [None, 0, 1],
+        'keepdims': [True, False],
+        'ddof': [0, 1]
+    }))
+class TestNanVarStd:
+
+    @ignore_runtime_warnings
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanvar(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype=dtype)
+        if a.dtype.kind not in 'biu':
+            a[0, :] = xp.nan
+        return xp.nanvar(
+            a, axis=self.axis, ddof=self.ddof, keepdims=self.keepdims)
+
+    @ignore_runtime_warnings
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanstd(self, xp, dtype):
+        a = testing.shaped_random(self.shape, xp, dtype=dtype)
+        if a.dtype.kind not in 'biu':
+            a[0, :] = xp.nan
+        return xp.nanstd(
+            a, axis=self.axis, ddof=self.ddof, keepdims=self.keepdims)
+
+
+class TestNanVarStdAdditional:
+
+    @ignore_runtime_warnings
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanvar_out(self, xp, dtype):
+        a = testing.shaped_random((10, 20, 30), xp, dtype)
+        z = xp.zeros((20, 30))
+
+        if a.dtype.kind not in 'biu':
+            a[1, :] = xp.nan
+            a[:, 3] = xp.nan
+
+        xp.nanvar(a, axis=0, out=z)
+        return z
+
+    @testing.slow
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanvar_huge(self, xp, dtype):
+        a = testing.shaped_random((1024, 512), xp, dtype)
+
+        if a.dtype.kind not in 'biu':
+            a[:512, :256] = xp.nan
+
+        return xp.nanvar(a, axis=1)
+
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_nanvar_float16(self, xp):
+        a = testing.shaped_arange((4, 5), xp, numpy.float16)
+        a[0][0] = xp.nan
+        return xp.nanvar(a, axis=0)
+
+    @ignore_runtime_warnings
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanstd_out(self, xp, dtype):
+        a = testing.shaped_random((10, 20, 30), xp, dtype)
+        z = xp.zeros((20, 30))
+
+        if a.dtype.kind not in 'biu':
+            a[1, :] = xp.nan
+            a[:, 3] = xp.nan
+
+        xp.nanstd(a, axis=0, out=z)
+        return z
+
+    @testing.slow
+    @testing.for_all_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_nanstd_huge(self, xp, dtype):
+        a = testing.shaped_random((1024, 512), xp, dtype)
+
+        if a.dtype.kind not in 'biu':
+            a[:512, :256] = xp.nan
+
+        return xp.nanstd(a, axis=1)
+
+    @testing.numpy_cupy_allclose(rtol=1e-4)
+    def test_nanstd_float16(self, xp):
+        a = testing.shaped_arange((4, 5), xp, numpy.float16)
+        a[0][0] = xp.nan
+        return xp.nanstd(a, axis=1)
+
+
+@testing.parameterize(*testing.product({
+    'params': [
+        ((), None),
+        ((0,), None),
+        ((0, 0), None),
+        ((0, 0), 1),
+        ((0, 0, 0), None),
+        ((0, 0, 0), (0, 2)),
+    ],
+    'func': ['mean', 'std', 'var'],
+}))
+class TestProductZeroLength:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_external_mean_zero_len(self, xp, dtype):
+        shape, axis = self.params
+        a = testing.shaped_arange(shape, xp, dtype)
+        f = getattr(xp, self.func)
+        return f(a, axis=axis)
diff --git a/tests/cupy_tests/statistics_tests/test_order.py b/tests/cupy_tests/statistics_tests/test_order.py
new file mode 100644
index 0000000..20e94b0
--- /dev/null
+++ b/tests/cupy_tests/statistics_tests/test_order.py
@@ -0,0 +1,430 @@
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+import cupy._core._accelerator as _acc
+from cupy import cuda
+from cupy import testing
+
+
+_all_methods = (
+    # 'inverted_cdf',               # TODO(takagi) Not implemented
+    # 'averaged_inverted_cdf',      # TODO(takagi) Not implemented
+    # 'closest_observation',        # TODO(takagi) Not implemented
+    # 'interpolated_inverted_cdf',  # TODO(takagi) Not implemented
+    # 'hazen',                      # TODO(takagi) Not implemented
+    # 'weibull',                    # TODO(takagi) Not implemented
+    'linear',
+    # 'median_unbiased',            # TODO(takagi) Not implemented
+    # 'normal_unbiased',            # TODO(takagi) Not implemented
+    'lower',
+    'higher',
+    'midpoint',
+    # 'nearest',                    # TODO(hvy): Not implemented
+)
+
+
+def for_all_methods(name='method'):
+    return pytest.mark.parametrize(name, _all_methods)
+
+
+@testing.with_requires('numpy>=1.22.0rc1')
+class TestQuantile:
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_percentile_unexpected_method(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((4, 2, 3, 2), xp, dtype)
+            q = testing.shaped_random((5,), xp, dtype=dtype, scale=100)
+            with pytest.raises(ValueError):
+                xp.percentile(a, q, axis=-1, method='deadbeef')
+
+    # See gh-4453
+    @testing.for_float_dtypes()
+    def test_percentile_memory_access(self, dtype):
+        # Create an allocator that guarantees array allocated in
+        # cupy.percentile call will be followed by a NaN
+        original_allocator = cuda.get_allocator()
+
+        def controlled_allocator(size):
+            memptr = original_allocator(size)
+            base_size = memptr.mem.size
+            assert base_size % 512 == 0
+            item_size = dtype().itemsize
+            shape = (base_size // item_size,)
+            x = cupy.ndarray(memptr=memptr, shape=shape, dtype=dtype)
+            x.fill(cupy.nan)
+            return memptr
+
+        # Check that percentile still returns non-NaN results
+        a = testing.shaped_random((5,), cupy, dtype)
+        q = cupy.array((0, 100), dtype=dtype)
+
+        cuda.set_allocator(controlled_allocator)
+        try:
+            percentiles = cupy.percentile(a, q, axis=None,
+                                          method='linear')
+        finally:
+            cuda.set_allocator(original_allocator)
+
+        assert not cupy.any(cupy.isnan(percentiles))
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_quantile_unexpected_method(self, dtype):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((4, 2, 3, 2), xp, dtype)
+            q = testing.shaped_random((5,), xp, dtype=dtype, scale=1)
+            with pytest.raises(ValueError):
+                xp.quantile(a, q, axis=-1, method='deadbeef')
+
+
+@testing.with_requires('numpy>=1.22.0rc1')
+@for_all_methods()
+class TestQuantileMethods:
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_percentile_defaults(self, xp, dtype, method):
+        a = testing.shaped_random((2, 3, 8), xp, dtype)
+        q = testing.shaped_random((3,), xp, dtype=dtype, scale=100)
+        return xp.percentile(a, q, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_percentile_q_list(self, xp, dtype, method):
+        a = testing.shaped_arange((1001,), xp, dtype)
+        q = [99, 99.9]
+        return xp.percentile(a, q, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_percentile_no_axis(self, xp, dtype, method):
+        a = testing.shaped_random((10, 2, 4, 8), xp, dtype)
+        q = testing.shaped_random((5,), xp, dtype=dtype, scale=100)
+        return xp.percentile(a, q, axis=None, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_percentile_neg_axis(self, xp, dtype, method):
+        a = testing.shaped_random((4, 3, 10, 2, 8), xp, dtype)
+        q = testing.shaped_random((5,), xp, dtype=dtype, scale=100)
+        return xp.percentile(a, q, axis=-1, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_percentile_tuple_axis(self, xp, dtype, method):
+        a = testing.shaped_random((1, 6, 3, 2), xp, dtype)
+        q = testing.shaped_random((5,), xp, dtype=dtype, scale=100)
+        return xp.percentile(a, q, axis=(0, 1, 2), method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_percentile_scalar_q(self, xp, dtype, method):
+        a = testing.shaped_random((2, 3, 8), xp, dtype)
+        q = 13.37
+        return xp.percentile(a, q, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_percentile_keepdims(self, xp, dtype, method):
+        a = testing.shaped_random((7, 2, 9, 2), xp, dtype)
+        q = testing.shaped_random((5,), xp, dtype=dtype, scale=100)
+        return xp.percentile(
+            a, q, axis=None, keepdims=True, method=method)
+
+    @testing.for_float_dtypes(no_float16=True)  # NumPy raises error on int8
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_percentile_out(self, xp, dtype, method):
+        a = testing.shaped_random((10, 2, 3, 2), xp, dtype)
+        q = testing.shaped_random((5,), xp, dtype=dtype, scale=100)
+        out = testing.shaped_random((5, 10, 2, 3), xp, dtype)
+        return xp.percentile(
+            a, q, axis=-1, method=method, out=out)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_percentile_overwrite(self, xp, dtype, method):
+        a = testing.shaped_random((10, 2, 3, 2), xp, dtype)
+        ap = a.copy()
+        q = testing.shaped_random((5,), xp, dtype=dtype, scale=100)
+        res = xp.percentile(ap, q, axis=-1, method=method,
+                            overwrite_input=True)
+
+        assert not xp.all(ap == a)
+        return res
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_percentile_bad_q(self, dtype, method):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((4, 2, 3, 2), xp, dtype)
+            q = testing.shaped_random((1, 2, 3), xp, dtype=dtype, scale=100)
+            with pytest.raises(ValueError):
+                xp.percentile(a, q, axis=-1, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_percentile_out_of_range_q(self, dtype, method):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((4, 2, 3, 2), xp, dtype)
+            for q in [[-0.1], [100.1]]:
+                with pytest.raises(ValueError):
+                    xp.percentile(a, q, axis=-1, method=method)
+
+    @testing.for_all_dtypes()
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_quantile_defaults(self, xp, dtype, method):
+        a = testing.shaped_random((2, 3, 8), xp, dtype)
+        q = testing.shaped_random((3,), xp, scale=1)
+        return xp.quantile(a, q, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_quantile_q_list(self, xp, dtype, method):
+        a = testing.shaped_arange((1001,), xp, dtype)
+        q = [.99, .999]
+        return xp.quantile(a, q, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_quantile_no_axis(self, xp, dtype, method):
+        a = testing.shaped_random((10, 2, 4, 8), xp, dtype)
+        q = testing.shaped_random((5,), xp, scale=1)
+        return xp.quantile(a, q, axis=None, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_quantile_neg_axis(self, xp, dtype, method):
+        a = testing.shaped_random((4, 3, 10, 2, 8), xp, dtype)
+        q = testing.shaped_random((5,), xp, scale=1)
+        return xp.quantile(a, q, axis=-1, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_quantile_tuple_axis(self, xp, dtype, method):
+        a = testing.shaped_random((1, 6, 3, 2), xp, dtype)
+        q = testing.shaped_random((5,), xp, scale=1)
+        return xp.quantile(a, q, axis=(0, 1, 2), method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_quantile_scalar_q(self, xp, dtype, method):
+        a = testing.shaped_random((2, 3, 8), xp, dtype)
+        q = .1337
+        return xp.quantile(a, q, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-5)
+    def test_quantile_keepdims(self, xp, dtype, method):
+        a = testing.shaped_random((7, 2, 9, 2), xp, dtype)
+        q = testing.shaped_random((5,), xp, scale=1)
+        return xp.quantile(
+            a, q, axis=None, keepdims=True, method=method)
+
+    @testing.for_float_dtypes(no_float16=True)  # NumPy raises error on int8
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_quantile_out(self, xp, dtype, method):
+        a = testing.shaped_random((10, 2, 3, 2), xp, dtype)
+        q = testing.shaped_random((5,), xp, dtype=dtype, scale=1)
+        out = testing.shaped_random((5, 10, 2, 3), xp, dtype)
+        return xp.quantile(
+            a, q, axis=-1, method=method, out=out)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_quantile_overwrite(self, xp, dtype, method):
+        a = testing.shaped_random((10, 2, 3, 2), xp, dtype)
+        ap = a.copy()
+        q = testing.shaped_random((5,), xp, dtype=dtype, scale=1)
+
+        res = xp.quantile(a, q, axis=-1, method=method, overwrite_input=True)
+
+        assert not xp.all(ap == a)
+        return res
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_quantile_bad_q(self, dtype, method):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((4, 2, 3, 2), xp, dtype)
+            q = testing.shaped_random((1, 2, 3), xp, dtype=dtype, scale=1)
+            with pytest.raises(ValueError):
+                xp.quantile(a, q, axis=-1, method=method)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    def test_quantile_out_of_range_q(self, dtype, method):
+        for xp in (numpy, cupy):
+            a = testing.shaped_random((4, 2, 3, 2), xp, dtype)
+            for q in [[-0.1], [1.1]]:
+                with pytest.raises(ValueError):
+                    xp.quantile(a, q, axis=-1, method=method)
+
+
+class TestOrder:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose()
+    def test_nanmax_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.nanmax(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmax_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return xp.nanmax(a, axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmax_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanmax(a, axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmax_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanmax(a, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmax_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanmax(a, axis=2)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmax_nan(self, xp, dtype):
+        a = xp.array([float('nan'), 1, -1], dtype)
+        with warnings.catch_warnings():
+            return xp.nanmax(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmax_all_nan(self, xp, dtype):
+        a = xp.array([float('nan'), float('nan')], dtype)
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always')
+            m = xp.nanmax(a)
+        assert len(w) == 1
+        assert w[0].category is RuntimeWarning
+        return m
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmin_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.nanmin(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmin_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return xp.nanmin(a, axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmin_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanmin(a, axis=0)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmin_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanmin(a, axis=1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmin_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.nanmin(a, axis=2)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmin_nan(self, xp, dtype):
+        a = xp.array([float('nan'), 1, -1], dtype)
+        with warnings.catch_warnings():
+            return xp.nanmin(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_nanmin_all_nan(self, xp, dtype):
+        a = xp.array([float('nan'), float('nan')], dtype)
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always')
+            m = xp.nanmin(a)
+        assert len(w) == 1
+        assert w[0].category is RuntimeWarning
+        return m
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ptp_all(self, xp, dtype):
+        a = testing.shaped_random((2, 3), xp, dtype)
+        return xp.ptp(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ptp_axis_large(self, xp, dtype):
+        a = testing.shaped_random((3, 1000), xp, dtype)
+        return xp.ptp(a, axis=0)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ptp_axis0(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.ptp(a, axis=0)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ptp_axis1(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.ptp(a, axis=1)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose()
+    def test_ptp_axis2(self, xp, dtype):
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return xp.ptp(a, axis=2)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_ptp_nan(self, xp, dtype):
+        if _acc.ACCELERATOR_CUTENSOR in _acc.get_routine_accelerators():
+            pytest.skip()
+        a = xp.array([float('nan'), 1, -1], dtype)
+        return xp.ptp(a)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_ptp_all_nan(self, xp, dtype):
+        if _acc.ACCELERATOR_CUTENSOR in _acc.get_routine_accelerators():
+            pytest.skip()
+        a = xp.array([float('nan'), float('nan')], dtype)
+        return xp.ptp(a)
+
+
+# See gh-4607
+# "Magic" values used in this test were empirically found to result in
+# non-monotonicity for less accurate linear interpolation formulas
+@testing.parameterize(*testing.product({
+    'magic_value': (-29, -53, -207, -16373, -99999,)
+}))
+class TestPercentileMonotonic:
+
+    @testing.with_requires('numpy>=1.22.0rc1')
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose()
+    def test_percentile_monotonic(self, dtype, xp):
+        a = testing.shaped_random((5,), xp, dtype)
+
+        a[0] = self.magic_value
+        a[1] = self.magic_value
+        q = xp.linspace(0, 100, 21)
+        percentiles = xp.percentile(a, q, method='linear')
+
+        # Assert that percentile output increases monotonically
+        assert xp.all(xp.diff(percentiles) >= 0)
+
+        return percentiles
diff --git a/tests/cupy_tests/test_cublas.py b/tests/cupy_tests/test_cublas.py
new file mode 100644
index 0000000..717f8fc
--- /dev/null
+++ b/tests/cupy_tests/test_cublas.py
@@ -0,0 +1,629 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import cublas
+from cupy import testing
+
+
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'n': [10, 33, 100],
+    'bs': [None, 1, 10],
+    'nrhs': [None, 1, 10],
+}))
+class TestBatchedGesv:
+    _tol = {'f': 5e-5, 'd': 1e-12}
+
+    def _make_random_matrices(self, shape, xp):
+        a = testing.shaped_random(shape, xp, dtype=self.r_dtype, scale=1)
+        if self.dtype.char in 'FD':
+            a = a + 1j * testing.shaped_random(shape, xp, dtype=self.r_dtype,
+                                               scale=1)
+        return a
+
+    def _make_well_conditioned_matrices(self, shape):
+        a = self._make_random_matrices(shape, numpy)
+        u, s, vh = numpy.linalg.svd(a)
+        s = testing.shaped_random(s.shape, numpy, dtype=self.r_dtype,
+                                  scale=1) + 1
+        a = numpy.einsum('...ik,...k,...kj->...ij', u, s, vh)
+        return cupy.array(a)
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.dtype = numpy.dtype(self.dtype)
+        if self.dtype.char in 'fF':
+            self.r_dtype = numpy.float32
+        else:
+            self.r_dtype = numpy.float64
+        n = self.n
+        bs = 1 if self.bs is None else self.bs
+        nrhs = 1 if self.nrhs is None else self.nrhs
+        a = self._make_well_conditioned_matrices((bs, n, n))
+        x = self._make_random_matrices((bs, n, nrhs), cupy)
+        b = cupy.matmul(a, x)
+        a_shape = (n, n) if self.bs is None else (bs, n, n)
+        b_shape = [n]
+        if self.bs is not None:
+            b_shape.insert(0, bs)
+        if self.nrhs is not None:
+            b_shape.append(nrhs)
+        self.a = a.reshape(a_shape)
+        self.b = b.reshape(b_shape)
+        self.x_ref = x.reshape(b_shape)
+        if self.r_dtype == numpy.float32:
+            self.tol = self._tol['f']
+        elif self.r_dtype == numpy.float64:
+            self.tol = self._tol['d']
+
+    def test_batched_gesv(self):
+        x = cublas.batched_gesv(self.a, self.b)
+        cupy.testing.assert_allclose(x, self.x_ref,
+                                     rtol=self.tol, atol=self.tol)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'n': [10, 100],
+    'mode': [None, numpy, cupy],
+}))
+class TestLevel1Functions:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.dtype = numpy.dtype(self.dtype)
+        self.tol = self._tol[self.dtype.char.lower()]
+
+    def _make_random_vector(self):
+        return testing.shaped_random((self.n,), cupy, dtype=self.dtype)
+
+    def _make_out(self, dtype):
+        out = None
+        if self.mode is not None:
+            out = self.mode.empty([], dtype=dtype)
+        return out
+
+    def _check_pointer(self, a, b):
+        if a is not None and b is not None:
+            assert self._get_pointer(a) == self._get_pointer(b)
+
+    def _get_pointer(self, a):
+        if isinstance(a, cupy.ndarray):
+            return a.data.ptr
+        else:
+            return a.ctypes.data
+
+    def test_iamax(self):
+        x = self._make_random_vector()
+        ref = cupy.argmax(cupy.absolute(x.real) + cupy.absolute(x.imag))
+        out = self._make_out('i')
+        res = cublas.iamax(x, out=out)
+        self._check_pointer(res, out)
+        # Note: iamax returns 1-based index
+        cupy.testing.assert_array_equal(res - 1, ref)
+
+    def test_iamin(self):
+        x = self._make_random_vector()
+        ref = cupy.argmin(cupy.absolute(x.real) + cupy.absolute(x.imag))
+        out = self._make_out('i')
+        res = cublas.iamin(x, out=out)
+        self._check_pointer(res, out)
+        # Note: iamin returns 1-based index
+        cupy.testing.assert_array_equal(res - 1, ref)
+
+    def test_asum(self):
+        x = self._make_random_vector()
+        ref = cupy.sum(cupy.absolute(x.real) + cupy.absolute(x.imag))
+        out = self._make_out(self.dtype.char.lower())
+        res = cublas.asum(x, out=out)
+        self._check_pointer(res, out)
+        cupy.testing.assert_allclose(res, ref, rtol=self.tol, atol=self.tol)
+
+    def test_axpy(self):
+        x = self._make_random_vector()
+        y = self._make_random_vector()
+        a = 1.1
+        if self.dtype.char in 'FD':
+            a = a - 1j * 0.9
+        ref = a * x + y
+        if self.mode is not None:
+            a = self.mode.array(a, dtype=self.dtype)
+        cublas.axpy(a, x, y)
+        cupy.testing.assert_allclose(y, ref, rtol=self.tol, atol=self.tol)
+
+    def test_dot(self):
+        x = self._make_random_vector()
+        y = self._make_random_vector()
+        ref = x.dot(y)
+        out = self._make_out(self.dtype)
+        if self.dtype.char in 'FD':
+            with pytest.raises(TypeError):
+                res = cublas.dot(x, y, out=out)
+            return
+        res = cublas.dot(x, y, out=out)
+        self._check_pointer(res, out)
+        cupy.testing.assert_allclose(res, ref, rtol=self.tol, atol=self.tol)
+
+    def test_dotu(self):
+        x = self._make_random_vector()
+        y = self._make_random_vector()
+        ref = x.dot(y)
+        out = self._make_out(self.dtype)
+        res = cublas.dotu(x, y, out=out)
+        self._check_pointer(res, out)
+        cupy.testing.assert_allclose(res, ref, rtol=self.tol, atol=self.tol)
+
+    def test_dotc(self):
+        x = self._make_random_vector()
+        y = self._make_random_vector()
+        ref = x.conj().dot(y)
+        out = self._make_out(self.dtype)
+        res = cublas.dotc(x, y, out=out)
+        self._check_pointer(res, out)
+        cupy.testing.assert_allclose(res, ref, rtol=self.tol, atol=self.tol)
+
+    def test_nrm2(self):
+        x = self._make_random_vector()
+        ref = cupy.linalg.norm(x)
+        out = self._make_out(self.dtype.char.lower())
+        res = cublas.nrm2(x, out=out)
+        self._check_pointer(res, out)
+        cupy.testing.assert_allclose(res, ref, rtol=self.tol, atol=self.tol)
+
+    def test_scal(self):
+        x = self._make_random_vector()
+        a = 1.1
+        if self.dtype.char in 'FD':
+            a = a - 1j * 0.9
+        ref = a * x
+        if self.mode is not None:
+            a = self.mode.array(a, dtype=self.dtype)
+        cublas.scal(a, x)
+        cupy.testing.assert_allclose(x, ref, rtol=self.tol, atol=self.tol)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'shape': [(10, 9), (9, 10)],
+    'trans': ['N', 'T', 'H'],
+    'order': ['C', 'F'],
+    'mode': [None, numpy, cupy],
+}))
+class TestGemv:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.dtype = numpy.dtype(self.dtype)
+        self.tol = self._tol[self.dtype.char.lower()]
+
+    def test_gemv(self):
+        a = testing.shaped_random(self.shape, cupy, dtype=self.dtype,
+                                  order=self.order)
+        if self.trans == 'N':
+            ylen, xlen = self.shape
+        else:
+            xlen, ylen = self.shape
+        x = testing.shaped_random((xlen,), cupy, dtype=self.dtype)
+        y = testing.shaped_random((ylen,), cupy, dtype=self.dtype)
+        alpha = 0.9
+        beta = 0.8
+        if self.dtype.char in 'FD':
+            alpha = alpha - 1j * 0.7
+            beta = beta - 1j * 0.6
+        if self.trans == 'N':
+            ref = alpha * a.dot(x) + beta * y
+        elif self.trans == 'T':
+            ref = alpha * a.T.dot(x) + beta * y
+        elif self.trans == 'H':
+            ref = alpha * a.T.conj().dot(x) + beta * y
+        if self.mode is not None:
+            alpha = self.mode.array(alpha)
+            beta = self.mode.array(beta)
+        cupy.cublas.gemv(self.trans, alpha, a, x, beta, y)
+        cupy.testing.assert_allclose(y, ref, rtol=self.tol, atol=self.tol)
+
+
+@testing.parameterize(*testing.product({
+    'rank': [5, 9],
+    'band': [0, 1, 3],
+    'lower': [0, 1],
+    'order': ['C', 'F'],
+    'mode': [None, numpy, cupy],
+}))
+class TestSbmv:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _gen2band(self, A, ku=0, kl=0, order='C'):
+        assert A.ndim == 2
+        n, m = A.shape
+        ldm, lda = n, 1 + ku + kl
+        B = numpy.zeros((lda, ldm), dtype=A.dtype, order=order)
+        for j in range(n):
+            k = ku - j
+            for i in range(max(0, j-ku), min(m, j + kl + 1)):
+                B[(k + i), j] = A[i, j]
+        return B
+
+    @testing.for_dtypes('fd')
+    def test_sbmv(self, dtype):
+        dtype = numpy.dtype(dtype)
+        alpha, beta = 3.0, 2.0
+        n, k = self.rank, self.band
+        a = numpy.eye(n, n, 0, dtype, self.order)
+        a *= numpy.random.randint(20)
+        for i in range(1, k+1):
+            band = numpy.random.randint(20, size=n-i)
+            a += numpy.diag(band, k=+i)
+            a += numpy.diag(band, k=-i)
+        x = numpy.random.randint(20, size=n).astype(a.dtype)
+        y = numpy.random.randint(20, size=n).astype(a.dtype)
+        ku, kl = k, 0
+        if self.lower == 1:
+            ku, kl = kl, ku
+        b = self._gen2band(a, ku, kl)
+        a, b = cupy.asarray(a), cupy.asarray(b)
+        x, y = cupy.asarray(x), cupy.asarray(y)
+        ref = alpha * a.dot(x) + beta * y
+        if self.mode is not None:
+            alpha = self.mode.array(alpha)
+            beta = self.mode.array(beta)
+        y_ret = cupy.cublas.sbmv(k, alpha, b, x, beta, y, lower=self.lower)
+        tol = self._tol[dtype.char.lower()]
+        cupy.testing.assert_allclose(y, ref, rtol=tol, atol=tol)
+        cupy.testing.assert_allclose(y_ret, ref, rtol=tol, atol=tol)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'shape': [(10, 9), (9, 10)],
+    'order': ['C', 'F'],
+    'mode': [None, numpy, cupy],
+}))
+class TestGer:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.dtype = numpy.dtype(self.dtype)
+        self.tol = self._tol[self.dtype.char.lower()]
+        self.a = testing.shaped_random(self.shape, cupy, dtype=self.dtype,
+                                       order=self.order)
+        self.x = testing.shaped_random((self.shape[0],), cupy,
+                                       dtype=self.dtype)
+        self.y = testing.shaped_random((self.shape[1],), cupy,
+                                       dtype=self.dtype)
+        self.alpha = 1.1
+        if self.dtype.char in 'FD':
+            self.alpha = self.alpha - 1j * 0.9
+
+    def test_ger(self):
+        if self.dtype.char in 'FD':
+            with pytest.raises(TypeError):
+                cublas.ger(self.alpha, self.x, self.y, self.a)
+            return
+        ref = self.alpha * cupy.outer(self.x, self.y) + self.a
+        if self.mode is not None:
+            self.alpha = self.mode.array(self.alpha)
+        cublas.ger(self.alpha, self.x, self.y, self.a)
+        cupy.testing.assert_allclose(self.a, ref, rtol=self.tol, atol=self.tol)
+
+    def test_geru(self):
+        ref = self.alpha * cupy.outer(self.x, self.y) + self.a
+        if self.mode is not None:
+            self.alpha = self.mode.array(self.alpha)
+        cublas.geru(self.alpha, self.x, self.y, self.a)
+        cupy.testing.assert_allclose(self.a, ref, rtol=self.tol, atol=self.tol)
+
+    def test_gerc(self):
+        ref = self.alpha * cupy.outer(self.x, self.y.conj()) + self.a
+        if self.mode is not None:
+            self.alpha = self.mode.array(self.alpha)
+        cublas.gerc(self.alpha, self.x, self.y, self.a)
+        cupy.testing.assert_allclose(self.a, ref, rtol=self.tol, atol=self.tol)
+
+
+@testing.parameterize(*testing.product({
+    'nk': [(5, 9), (9, 5)],
+    'transa': ['N', 'T'],
+    'ordera': ['F', 'C'],
+    'orderc': ['F', 'C'],
+    'lower': [0, 1],
+    'mode': [None, numpy, cupy]
+}))
+class TestSyrk:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _make_matrix(self, m, n, trans, order, dtype):
+        if trans == 'N':
+            shape = (m, n)
+        else:
+            shape = (n, m)
+        return testing.shaped_random(shape, cupy, dtype=dtype, order=order,
+                                     scale=1.0)
+
+    def _trans_matrix(self, a, trans):
+        if trans == 'N':
+            return a
+        return a.T
+
+    @testing.for_dtypes('fdFD')
+    def test_syrk(self, dtype):
+        dtype = numpy.dtype(dtype)
+        tol = self._tol[dtype.char.lower()]
+        alpha, beta = 3.0, 2.0
+        if dtype.char in 'FD':
+            alpha = alpha - 1j * 2.0
+            beta = beta + 1j * 5.0
+        n, k = self.nk
+        a = self._make_matrix(n, k, self.transa, self.ordera, dtype)
+        aa = self._trans_matrix(a, self.transa)
+        ref = alpha * aa.dot(aa.T)  # beta is used as a placeholder only
+        c = cublas.syrk(self.transa, a, alpha=alpha, beta=beta,
+                        lower=self.lower)
+        rr, cc = cupy.asnumpy(ref), cupy.asnumpy(c)
+        if self.lower:
+            rr[numpy.triu_indices_from(rr,  1)] = 0
+        else:
+            rr[numpy.tril_indices_from(rr, -1)] = 0
+        rru = rr[numpy.triu_indices_from(rr)]
+        ccu = cc[numpy.triu_indices_from(cc)]
+        rrl = rr[numpy.tril_indices_from(rr)]
+        ccl = cc[numpy.tril_indices_from(cc)]
+        cupy.testing.assert_allclose(ccu, rru, rtol=tol, atol=tol)
+        cupy.testing.assert_allclose(ccl, rrl, rtol=tol, atol=tol)
+
+    @testing.for_dtypes('fdFD')
+    def test_syrk_out(self, dtype):
+        dtype = numpy.dtype(dtype)
+        tol = self._tol[dtype.char.lower()]
+        alpha, beta = 2.3, 1.7
+        if dtype.char in 'FD':
+            alpha = alpha - 1j * 0.7
+            beta = beta + 1j * 2.3
+        n, k = self.nk
+        a = self._make_matrix(n, k, self.transa, self.ordera, dtype)
+        aa = self._trans_matrix(a, self.transa)
+        m = aa.shape[0]
+        c = self._make_matrix(m, m, 'N', self.orderc, dtype)
+        c0 = cupy.array(c)
+        ref = alpha * aa.dot(aa.T) + beta * c
+        cublas.syrk(self.transa, a, out=c, alpha=alpha, beta=beta,
+                    lower=self.lower)
+        rr, c0, cc = cupy.asnumpy(ref), cupy.asnumpy(c0), cupy.asnumpy(c)
+        if self.lower:
+            trii = numpy.triu_indices_from(rr,  1)
+        else:
+            trii = numpy.tril_indices_from(rr, -1)
+        rr[trii] = c0[trii]
+        rru = rr[numpy.triu_indices_from(rr)]
+        ccu = cc[numpy.triu_indices_from(cc)]
+        rrl = rr[numpy.tril_indices_from(rr)]
+        ccl = cc[numpy.tril_indices_from(cc)]
+        cupy.testing.assert_allclose(ccu, rru, rtol=tol, atol=tol)
+        cupy.testing.assert_allclose(ccl, rrl, rtol=tol, atol=tol)
+
+
+@testing.parameterize(*testing.product({
+    'mnk': [(8, 9, 10), (10, 9, 8)],
+    'transa': ['N', 'T', 'H'],
+    'transb': ['N', 'T', 'H'],
+    'ordera': ['C', 'F'],
+    'orderb': ['C', 'F'],
+    'orderc': ['C', 'F'],
+    'mode': [None, numpy, cupy],
+}))
+class TestGemmAndGeam:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _make_matrix(self, m, n, trans, order, dtype):
+        if trans == 'N':
+            shape = (m, n)
+        else:
+            shape = (n, m)
+        return testing.shaped_random(shape, cupy, dtype=dtype, order=order,
+                                     scale=1.0)
+
+    def _trans_matrix(self, a, trans):
+        if trans == 'T':
+            a = a.T
+        elif trans == 'H':
+            a = a.T.conj()
+        return a
+
+    @testing.for_dtypes('fdFD')
+    def test_gemm(self, dtype):
+        if not (self.mode is None and self.orderc == 'C'):
+            pytest.skip()
+        dtype = numpy.dtype(dtype)
+        tol = self._tol[dtype.char.lower()]
+        m, n, k = self.mnk
+        a = self._make_matrix(m, k, self.transa, self.ordera, dtype)
+        b = self._make_matrix(k, n, self.transb, self.orderb, dtype)
+        aa = self._trans_matrix(a, self.transa)
+        bb = self._trans_matrix(b, self.transb)
+        ref = aa.dot(bb)
+        c = cublas.gemm(self.transa, self.transb, a, b)
+        cupy.testing.assert_allclose(c, ref, rtol=tol, atol=tol)
+
+    @testing.for_dtypes('fdFD')
+    def test_gemm_out(self, dtype):
+        dtype = numpy.dtype(dtype)
+        tol = self._tol[dtype.char.lower()]
+        m, n, k = self.mnk
+        a = self._make_matrix(m, k, self.transa, self.ordera, dtype)
+        b = self._make_matrix(k, n, self.transb, self.orderb, dtype)
+        c = self._make_matrix(m, n, 'N', self.orderc, dtype)
+        alpha = 0.9
+        beta = 0.8
+        if dtype.char in 'FD':
+            alpha = alpha - 1j * 0.7
+            beta = beta - 1j * 0.6
+        aa = self._trans_matrix(a, self.transa)
+        bb = self._trans_matrix(b, self.transb)
+        ref = alpha * aa.dot(bb) + beta * c
+        if self.mode is not None:
+            alpha = self.mode.array(alpha)
+            beta = self.mode.array(beta)
+        cublas.gemm(self.transa, self.transb, a, b, out=c,
+                    alpha=alpha, beta=beta)
+        cupy.testing.assert_allclose(c, ref, rtol=tol, atol=tol)
+
+    @testing.for_dtypes('fdFD')
+    def test_geam(self, dtype):
+        if self.orderc != 'F':
+            pytest.skip()
+        dtype = numpy.dtype(dtype)
+        tol = self._tol[dtype.char.lower()]
+        m, n, _ = self.mnk
+        a = self._make_matrix(m, n, self.transa, self.ordera, dtype)
+        b = self._make_matrix(m, n, self.transb, self.orderb, dtype)
+        alpha = 0.9
+        beta = 0.8
+        if dtype.char in 'FD':
+            alpha = alpha - 1j * 0.7
+            beta = beta - 1j * 0.6
+        aa = self._trans_matrix(a, self.transa)
+        bb = self._trans_matrix(b, self.transb)
+        ref = alpha * aa + beta * bb
+        if self.mode is not None:
+            alpha = self.mode.array(alpha)
+            beta = self.mode.array(beta)
+        c = cublas.geam(self.transa, self.transb, alpha, a, beta, b)
+        cupy.testing.assert_allclose(c, ref, rtol=tol, atol=tol)
+
+    @testing.for_dtypes('fdFD')
+    def test_geam_out(self, dtype):
+        dtype = numpy.dtype(dtype)
+        tol = self._tol[dtype.char.lower()]
+        m, n, _ = self.mnk
+        a = self._make_matrix(m, n, self.transa, self.ordera, dtype)
+        b = self._make_matrix(m, n, self.transb, self.orderb, dtype)
+        c = self._make_matrix(m, n, 'N', self.orderc, dtype)
+        alpha = 0.9
+        beta = 0.8
+        if dtype.char in 'FD':
+            alpha = alpha - 1j * 0.7
+            beta = beta - 1j * 0.6
+        aa = self._trans_matrix(a, self.transa)
+        bb = self._trans_matrix(b, self.transb)
+        ref = alpha * aa + beta * bb
+        if self.mode is not None:
+            alpha = self.mode.array(alpha)
+            beta = self.mode.array(beta)
+        cublas.geam(self.transa, self.transb, alpha, a, beta, b, out=c)
+        cupy.testing.assert_allclose(c, ref, rtol=tol, atol=tol)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(9, 10), (10, 9)],
+    'side': ['L', 'R'],
+    'ordera': ['C', 'F'],
+    'orderc': ['C', 'F'],
+}))
+class TestDgmm:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _setup(self, dtype, xdim=1):
+        self.dtype = numpy.dtype(dtype)
+        self.tol = self._tol[self.dtype.char.lower()]
+        self.a = testing.shaped_random(self.shape, cupy, dtype=dtype,
+                                       order=self.ordera, scale=1.0)
+        if self.side == 'L':
+            xlen = self.shape[0]
+        elif self.side == 'R':
+            xlen = self.shape[1]
+        if xdim == 0:
+            self.x = cupy.array(1.1, dtype=dtype)
+        elif xdim == 1:
+            self.x = testing.shaped_random(
+                (xlen,), cupy, dtype=dtype, scale=1.0)
+        elif xdim == 2:
+            self.x = testing.shaped_random(
+                (xlen, xlen), cupy, dtype=dtype, scale=1.0)
+
+    @testing.for_dtypes('fdFD')
+    def test_dgmm(self, dtype):
+        if self.orderc != 'F':
+            pytest.skip()
+        self._setup(dtype)
+        if self.side == 'L':
+            ref = cupy.diag(self.x) @ self.a
+        elif self.side == 'R':
+            ref = self.a @ cupy.diag(self.x)
+        c = cublas.dgmm(self.side, self.a, self.x)
+        cupy.testing.assert_allclose(c, ref, rtol=self.tol, atol=self.tol)
+
+    @testing.for_dtypes('fdFD')
+    def test_dgmm_out(self, dtype):
+        self._setup(dtype)
+        if self.side == 'L':
+            ref = cupy.diag(self.x) @ self.a
+        elif self.side == 'R':
+            ref = self.a @ cupy.diag(self.x)
+        c = cupy.empty(self.shape, order=self.orderc, dtype=dtype)
+        cublas.dgmm(self.side, self.a, self.x, out=c)
+        cupy.testing.assert_allclose(c, ref, rtol=self.tol, atol=self.tol)
+
+    @testing.for_dtypes('fdFD')
+    def test_dgmm_inplace(self, dtype):
+        if self.orderc != 'F':
+            pytest.skip()
+        self._setup(dtype)
+        if self.side == 'L':
+            ref = cupy.diag(self.x) @ self.a
+        elif self.side == 'R':
+            ref = self.a @ cupy.diag(self.x)
+        cublas.dgmm(self.side, self.a, self.x, out=self.a)
+        cupy.testing.assert_allclose(self.a, ref, rtol=self.tol, atol=self.tol)
+
+    _dgmm_incx_minus_one_hip_skip_condition = [
+        ('C', 'F', (9, 10), 'R'),
+        ('C', 'F', (10, 9), 'R'),
+        ('F', 'F', (9, 10), 'L'),
+        ('F', 'F', (10, 9), 'L'),
+    ]
+
+    def _check_dgmm_incx_minus_one_hip_skip_condition(self):
+        return (self.ordera, self.orderc, self.shape, self.side) in \
+            self._dgmm_incx_minus_one_hip_skip_condition
+
+    @testing.for_dtypes('fdFD')
+    def test_dgmm_incx_minus_one(self, dtype):
+        if self.orderc != 'F':
+            pytest.skip()
+        if cupy.cuda.runtime.is_hip:
+            if self._check_dgmm_incx_minus_one_hip_skip_condition():
+                pytest.xfail('HIP dgmm may have a bug')
+        self._setup(dtype)
+        if self.side == 'L':
+            ref = cupy.diag(self.x[::-1]) @ self.a
+        elif self.side == 'R':
+            ref = self.a @ cupy.diag(self.x[::-1])
+        c = cublas.dgmm(self.side, self.a, self.x, incx=-1)
+        cupy.testing.assert_allclose(c, ref, rtol=self.tol, atol=self.tol)
+
+    @testing.for_dtypes('fdFD')
+    def test_dgmm_x_scalar(self, dtype):
+        if self.orderc != 'F':
+            pytest.skip()
+        self._setup(dtype, xdim=0)
+        ref = self.x * self.a
+        c = cublas.dgmm(self.side, self.a, self.x, incx=0)
+        cupy.testing.assert_allclose(c, ref, rtol=self.tol, atol=self.tol)
+
+    @testing.for_dtypes('fdFD')
+    def test_dgmm_x_matrix(self, dtype):
+        if self.orderc != 'F':
+            pytest.skip()
+        self._setup(dtype, xdim=2)
+        if self.side == 'L':
+            ref = cupy.diag(cupy.diag(self.x)) @ self.a
+            incx = self.shape[0] + 1
+        elif self.side == 'R':
+            ref = self.a @ cupy.diag(cupy.diag(self.x))
+            incx = self.shape[1] + 1
+        c = cublas.dgmm(self.side, self.a, self.x, incx=incx)
+        cupy.testing.assert_allclose(c, ref, rtol=self.tol, atol=self.tol)
diff --git a/tests/cupy_tests/test_init.py b/tests/cupy_tests/test_init.py
new file mode 100644
index 0000000..0084180
--- /dev/null
+++ b/tests/cupy_tests/test_init.py
@@ -0,0 +1,153 @@
+import operator
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+import numpy
+import pytest
+
+import cupy
+import cupyx
+
+
+def _run_script(code):
+    # subprocess is required not to interfere with cupy module imported in top
+    # of this file
+    temp_dir = tempfile.mkdtemp()
+    try:
+        script_path = os.path.join(temp_dir, 'script.py')
+        with open(script_path, 'w') as f:
+            f.write(code)
+        proc = subprocess.Popen(
+            [sys.executable, script_path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+        stdoutdata, stderrdata = proc.communicate()
+    finally:
+        shutil.rmtree(temp_dir, ignore_errors=True)
+    return proc.returncode, stdoutdata, stderrdata
+
+
+def _test_cupy_available(self):
+    returncode, stdoutdata, stderrdata = _run_script('''
+import cupy
+print(cupy.is_available())''')
+    assert returncode == 0, 'stderr: {!r}'.format(stderrdata)
+    assert stdoutdata in (b'True\n', b'True\r\n', b'False\n', b'False\r\n')
+    return stdoutdata == b'True\n' or stdoutdata == b'True\r\n'
+
+
+class TestImportError(unittest.TestCase):
+
+    def test_import_error(self):
+        returncode, stdoutdata, stderrdata = _run_script('''
+try:
+    import cupy
+except Exception as e:
+    print(type(e).__name__)
+''')
+        assert returncode == 0, 'stderr: {!r}'.format(stderrdata)
+        assert stdoutdata in (b'', b'RuntimeError\n')
+
+
+if not cupy.cuda.runtime.is_hip:
+    visible = 'CUDA_VISIBLE_DEVICES'
+else:
+    visible = 'HIP_VISIBLE_DEVICES'
+
+
+class TestAvailable(unittest.TestCase):
+
+    def test_available(self):
+        available = _test_cupy_available(self)
+        assert available
+
+
+class TestNotAvailable(unittest.TestCase):
+
+    def setUp(self):
+        self.old = os.environ.get(visible)
+
+    def tearDown(self):
+        if self.old is None:
+            os.environ.pop(visible)
+        else:
+            os.environ[visible] = self.old
+
+    @unittest.skipIf(cupy.cuda.runtime.is_hip,
+                     'HIP handles empty HIP_VISIBLE_DEVICES differently')
+    def test_no_device_1(self):
+        os.environ['CUDA_VISIBLE_DEVICES'] = ' '
+        available = _test_cupy_available(self)
+        assert not available
+
+    def test_no_device_2(self):
+        os.environ[visible] = '-1'
+        available = _test_cupy_available(self)
+        assert not available
+
+
+class TestMemoryPool(unittest.TestCase):
+
+    def test_get_default_memory_pool(self):
+        p = cupy.get_default_memory_pool()
+        assert isinstance(p, cupy.cuda.memory.MemoryPool)
+
+    def test_get_default_pinned_memory_pool(self):
+        p = cupy.get_default_pinned_memory_pool()
+        assert isinstance(p, cupy.cuda.pinned_memory.PinnedMemoryPool)
+
+
+class TestShowConfig(unittest.TestCase):
+
+    def test_show_config(self):
+        with mock.patch('sys.stdout.write') as write_func:
+            cupy.show_config()
+        write_func.assert_called_once_with(
+            str(cupyx.get_runtime_info(full=False)))
+
+    def test_show_config_with_handles(self):
+        with mock.patch('sys.stdout.write') as write_func:
+            cupy.show_config(_full=True)
+        write_func.assert_called_once_with(
+            str(cupyx.get_runtime_info(full=True)))
+
+
+class TestAliases(unittest.TestCase):
+
+    def test_abs_is_absolute(self):
+        for xp in (numpy, cupy):
+            assert xp.abs is xp.absolute
+
+    def test_conj_is_conjugate(self):
+        for xp in (numpy, cupy):
+            assert xp.conj is xp.conjugate
+
+    def test_bitwise_not_is_invert(self):
+        for xp in (numpy, cupy):
+            assert xp.bitwise_not is xp.invert
+
+
+@pytest.mark.parametrize('name', [
+    'AxisError',
+    'ComplexWarning',
+    'ModuleDeprecationWarning',
+    'RankWarning',
+    'TooHardError',
+    'VisibleDeprecationWarning',
+    'linalg.LinAlgError'
+])
+def test_error_classes(name):
+    get = operator.attrgetter(name)
+    assert issubclass(get(cupy), get(numpy))
+
+
+# This is copied from chainer/testing/__init__.py, so should be replaced in
+# some way.
+if __name__ == '__main__':
+    import pytest
+    pytest.main([__file__, '-vvs', '-x', '--pdb'])
diff --git a/tests/cupy_tests/test_ndim.py b/tests/cupy_tests/test_ndim.py
new file mode 100644
index 0000000..bb657c2
--- /dev/null
+++ b/tests/cupy_tests/test_ndim.py
@@ -0,0 +1,66 @@
+import numpy
+import unittest
+
+import cupy
+from cupy import testing
+
+
+class TestNdim(unittest.TestCase):
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_ndarray1d(self, xp):
+        return xp.ndim(xp.arange(5))
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_ndarray2d(self, xp):
+        return xp.ndim(xp.ones((2, 4)))
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_ndarray0d(self, xp):
+        return xp.ndim(xp.asarray(5))
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_scalar(self, xp):
+        return xp.ndim(5)
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_none(self, xp):
+        return xp.ndim(None)
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_string(self, xp):
+        return xp.ndim('abc')
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_list1(self, xp):
+        return xp.ndim([1, 2, 3])
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_list2(self, xp):
+        return xp.ndim([[1, 2, 3], [4, 5, 6]])
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_tuple(self, xp):
+        return xp.ndim(((1, 2, 3), (4, 5, 6)))
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_set(self, xp):
+        return xp.ndim({1, 2, 3})
+
+    @testing.numpy_cupy_equal()
+    def test_ndim_object(self, xp):
+        return xp.ndim(dict(a=5, b='b'))
+
+    # numpy.dim works on CuPy arrays and cupy.ndim works on NumPy arrays
+    def test_ndim_array_function(self):
+        a = cupy.ones((4, 4))
+        assert numpy.ndim(a) == 2
+
+        a = cupy.asarray(5)
+        assert numpy.ndim(a) == 0
+
+        a = numpy.ones((4, 4))
+        assert cupy.ndim(a) == 2
+
+        a = numpy.asarray(5)
+        assert cupy.ndim(a) == 0
diff --git a/tests/cupy_tests/test_numpy_interop.py b/tests/cupy_tests/test_numpy_interop.py
new file mode 100644
index 0000000..f0cb1c9
--- /dev/null
+++ b/tests/cupy_tests/test_numpy_interop.py
@@ -0,0 +1,150 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx
+
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+
+class TestGetArrayModule(unittest.TestCase):
+
+    def test_get_array_module_1(self):
+        n1 = numpy.array([2], numpy.float32)
+        c1 = cupy.array([2], numpy.float32)
+        csr1 = cupyx.scipy.sparse.csr_matrix((5, 3), dtype=numpy.float32)
+
+        assert numpy is cupy.get_array_module()
+        assert numpy is cupy.get_array_module(n1)
+        assert cupy is cupy.get_array_module(c1)
+        assert cupy is cupy.get_array_module(csr1)
+
+        assert numpy is cupy.get_array_module(n1, n1)
+        assert cupy is cupy.get_array_module(c1, c1)
+        assert cupy is cupy.get_array_module(csr1, csr1)
+
+        assert cupy is cupy.get_array_module(n1, csr1)
+        assert cupy is cupy.get_array_module(csr1, n1)
+        assert cupy is cupy.get_array_module(c1, n1)
+        assert cupy is cupy.get_array_module(n1, c1)
+        assert cupy is cupy.get_array_module(c1, csr1)
+        assert cupy is cupy.get_array_module(csr1, c1)
+
+        if scipy_available:
+            csrn1 = scipy.sparse.csr_matrix((5, 3), dtype=numpy.float32)
+
+            assert numpy is cupy.get_array_module(csrn1)
+            assert cupy is cupy.get_array_module(csrn1, csr1)
+            assert cupy is cupy.get_array_module(csr1, csrn1)
+            assert cupy is cupy.get_array_module(c1, csrn1)
+            assert cupy is cupy.get_array_module(csrn1, c1)
+            assert numpy is cupy.get_array_module(n1, csrn1)
+            assert numpy is cupy.get_array_module(csrn1, n1)
+
+
+class MockArray(numpy.lib.mixins.NDArrayOperatorsMixin):
+    __array_priority__ = 20  # less than cupy.ndarray.__array_priority__
+
+    def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
+        assert method == '__call__'
+        name = ufunc.__name__
+        return name, inputs, kwargs
+
+
+class TestArrayUfunc:
+
+    def test_add(self):
+        x = cupy.array([3, 7])
+        y = MockArray()
+        assert x + y == ('add', (x, y), {})
+        assert y + x == ('add', (y, x), {})
+        y2 = y
+        y2 += x
+        assert y2 == ('add', (y, x), {'out': y})
+        with pytest.raises(TypeError):
+            x += y
+
+    @pytest.mark.xfail(
+        reason='cupy.ndarray.__array_ufunc__ does not support gufuncs yet')
+    def test_matmul(self):
+        x = cupy.array([3, 7])
+        y = MockArray()
+        assert x @ y == ('matmul', (x, y), {})
+        assert y @ x == ('matmul', (y, x), {})
+        y2 = y
+        y2 @= x
+        assert y2 == ('matmul', (y, x), {'out': y})
+        with pytest.raises(TypeError):
+            x @= y
+
+    def test_lt(self):
+        x = cupy.array([3, 7])
+        y = MockArray()
+        assert (x < y) == ('less', (x, y), {})
+        assert (y < x) == ('less', (y, x), {})
+
+
+class MockArray2:
+    __array_ufunc__ = None
+
+    def __add__(self, other):
+        return 'add'
+
+    def __radd__(self, other):
+        return 'radd'
+
+    def __matmul__(self, other):
+        return 'matmul'
+
+    def __rmatmul__(self, other):
+        return 'rmatmul'
+
+    def __lt__(self, other):
+        return 'lt'
+
+    def __gt__(self, other):
+        return 'gt'
+
+
+class TestArrayUfuncOptout:
+
+    def test_add(self):
+        x = cupy.array([3, 7])
+        y = MockArray2()
+        assert x + y == 'radd'
+        assert y + x == 'add'
+
+    def test_matmul(self):
+        x = cupy.array([3, 7])
+        y = MockArray2()
+        assert x @ y == 'rmatmul'
+        assert y @ x == 'matmul'
+
+    def test_lt(self):
+        x = cupy.array([3, 7])
+        y = MockArray2()
+        assert (x < y) == 'gt'
+        assert (y < x) == 'lt'
+
+
+class TestAsnumpy:
+
+    def test_asnumpy(self):
+        x = testing.shaped_random((2, 3, 4), cupy, cupy.float64)
+        y = cupy.asnumpy(x)
+        testing.assert_array_equal(x, y)
+
+    def test_asnumpy_out(self):
+        x = testing.shaped_random((2, 3, 4), cupy, cupy.float64)
+        y = cupyx.empty_like_pinned(x)
+        y = cupy.asnumpy(x, out=y)
+        testing.assert_array_equal(x, y)
+        assert isinstance(y.base, cupy.cuda.PinnedMemoryPointer)
+        assert y.base.ptr == y.ctypes.data
diff --git a/tests/cupy_tests/test_type_routines.py b/tests/cupy_tests/test_type_routines.py
new file mode 100644
index 0000000..c74638d
--- /dev/null
+++ b/tests/cupy_tests/test_type_routines.py
@@ -0,0 +1,91 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+def _generate_type_routines_input(xp, dtype, obj_type):
+    dtype = numpy.dtype(dtype)
+    if obj_type == 'dtype':
+        return dtype
+    if obj_type == 'specifier':
+        return str(dtype)
+    if obj_type == 'scalar':
+        return dtype.type(3)
+    if obj_type == 'array':
+        return xp.zeros(3, dtype=dtype)
+    if obj_type == 'primitive':
+        return type(dtype.type(3).tolist())
+    assert False
+
+
+@testing.parameterize(
+    *testing.product({
+        'obj_type': ['dtype', 'specifier', 'scalar', 'array', 'primitive'],
+    })
+)
+class TestCanCast(unittest.TestCase):
+
+    @testing.for_all_dtypes_combination(names=('from_dtype', 'to_dtype'))
+    @testing.numpy_cupy_equal()
+    def test_can_cast(self, xp, from_dtype, to_dtype):
+        from_obj = _generate_type_routines_input(xp, from_dtype, self.obj_type)
+        ret = xp.can_cast(from_obj, to_dtype)
+        assert isinstance(ret, bool)
+        return ret
+
+
+class TestCommonType(unittest.TestCase):
+
+    @testing.numpy_cupy_equal()
+    def test_common_type_empty(self, xp):
+        ret = xp.common_type()
+        assert type(ret) == type
+        return ret
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_equal()
+    def test_common_type_single_argument(self, xp, dtype):
+        array = _generate_type_routines_input(xp, dtype, 'array')
+        ret = xp.common_type(array)
+        assert type(ret) == type
+        return ret
+
+    @testing.for_all_dtypes_combination(
+        names=('dtype1', 'dtype2'), no_bool=True)
+    @testing.numpy_cupy_equal()
+    def test_common_type_two_arguments(self, xp, dtype1, dtype2):
+        array1 = _generate_type_routines_input(xp, dtype1, 'array')
+        array2 = _generate_type_routines_input(xp, dtype2, 'array')
+        ret = xp.common_type(array1, array2)
+        assert type(ret) == type
+        return ret
+
+    @testing.for_all_dtypes()
+    def test_common_type_bool(self, dtype):
+        for xp in (numpy, cupy):
+            array1 = _generate_type_routines_input(xp, dtype, 'array')
+            array2 = _generate_type_routines_input(xp, 'bool_', 'array')
+            with pytest.raises(TypeError):
+                xp.common_type(array1, array2)
+
+
+@testing.parameterize(
+    *testing.product({
+        'obj_type1': ['dtype', 'specifier', 'scalar', 'array', 'primitive'],
+        'obj_type2': ['dtype', 'specifier', 'scalar', 'array', 'primitive'],
+    })
+)
+class TestResultType(unittest.TestCase):
+
+    @testing.for_all_dtypes_combination(names=('dtype1', 'dtype2'))
+    @testing.numpy_cupy_equal()
+    def test_result_type(self, xp, dtype1, dtype2):
+        input1 = _generate_type_routines_input(xp, dtype1, self.obj_type1)
+        input2 = _generate_type_routines_input(xp, dtype2, self.obj_type2)
+        ret = xp.result_type(input1, input2)
+        assert isinstance(ret, numpy.dtype)
+        return ret
diff --git a/tests/cupy_tests/test_typing.py b/tests/cupy_tests/test_typing.py
new file mode 100644
index 0000000..cad0c74
--- /dev/null
+++ b/tests/cupy_tests/test_typing.py
@@ -0,0 +1,8 @@
+import cupy
+
+
+class TestClassGetItem:
+
+    def test_class_getitem(self):
+        from typing import Any
+        cupy.ndarray[Any, Any]
diff --git a/tests/cupy_tests/testing_tests/__init__.py b/tests/cupy_tests/testing_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupy_tests/testing_tests/test_array.py b/tests/cupy_tests/testing_tests/test_array.py
new file mode 100644
index 0000000..a94e336
--- /dev/null
+++ b/tests/cupy_tests/testing_tests/test_array.py
@@ -0,0 +1,116 @@
+import copy
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+
+
+@testing.parameterize(
+    *testing.product({
+        'assertion': ['assert_allclose', 'assert_array_almost_equal',
+                      'assert_array_almost_equal_nulp',
+                      'assert_array_max_ulp', 'assert_array_equal'],
+        'array_module_x': [numpy, cupy],
+        'array_module_y': [numpy, cupy]
+    })
+)
+class TestEqualityAssertion(unittest.TestCase):
+
+    def setUp(self):
+        self.assertion = getattr(testing, self.assertion)
+        val = numpy.random.uniform(-1, 1, (2, 3))
+        self.x = self.array_module_x.array(val, val.dtype, copy=True)
+        self.y = self.array_module_y.array(val, val.dtype, copy=True)
+
+    def test_equality(self):
+        self.assertion(self.x, self.y)
+
+    def test_inequality(self):
+        self.y += 1
+        with self.assertRaises(AssertionError):
+            self.assertion(self.x, self.y)
+
+
+def _convert_array(xs, array_module):
+    if array_module == 'all_numpy':
+        return xs
+    elif array_module == 'all_cupy':
+        return [
+            cupy.asarray(x)
+            for x in xs
+        ]
+    else:
+        return [
+            cupy.asarray(x) if numpy.random.randint(0, 2) else x
+            for x in xs
+        ]
+
+
+@testing.parameterize(
+    *testing.product({
+        'array_module_x': ['all_numpy', 'all_cupy', 'random'],
+        'array_module_y': ['all_numpy', 'all_cupy', 'random']
+    })
+)
+class TestListEqualityAssertion(unittest.TestCase):
+
+    def setUp(self):
+        xs = [numpy.random.uniform(-1, 1, (2, 3)) for _ in range(10)]
+        ys = copy.deepcopy(xs)
+        self.xs = _convert_array(xs, self.array_module_x)
+        self.ys = _convert_array(ys, self.array_module_y)
+
+    def test_equality_numpy(self):
+        testing.assert_array_list_equal(self.xs, self.ys)
+
+    def test_inequality_numpy(self):
+        self.xs[0] += 1
+        with self.assertRaisesRegex(
+                AssertionError, '^\nArrays are not equal'):
+            testing.assert_array_list_equal(self.xs, self.ys)
+
+
+@testing.parameterize(
+    *testing.product({
+        'array_module_x': [numpy, cupy],
+        'array_module_y': [numpy, cupy]
+    })
+)
+class TestStridesEqualityAssertion(unittest.TestCase):
+
+    def setUp(self):
+        val = numpy.random.uniform(-1, 1, (2, 3))
+        self.x = self.array_module_x.array(val, val.dtype, copy=True)
+        self.y = self.array_module_y.array(val, val.dtype, copy=True)
+
+    def test_equality_numpy(self):
+        testing.assert_array_equal(self.x, self.y, strides_check=True)
+
+    def test_inequality_numpy(self):
+        self.y = self.array_module_y.asfortranarray(self.y)
+        with self.assertRaises(AssertionError):
+            testing.assert_array_equal(self.x, self.y, strides_check=True)
+
+
+@testing.parameterize(
+    *testing.product({
+        'array_module_x': [numpy, cupy],
+        'array_module_y': [numpy, cupy]
+    })
+)
+class TestLessAssertion(unittest.TestCase):
+
+    def setUp(self):
+        val = numpy.random.uniform(-1, 1, (2, 3))
+        self.x = self.array_module_x.array(val, val.dtype, copy=True)
+        self.y = self.array_module_y.array(val + 1, val.dtype, copy=True)
+
+    def test_equality_numpy(self):
+        testing.assert_array_less(self.x, self.y)
+
+    def test_inequality_numpy(self):
+        self.x[0] += 100
+        with self.assertRaises(AssertionError):
+            testing.assert_array_less(self.x, self.y)
diff --git a/tests/cupy_tests/testing_tests/test_condition.py b/tests/cupy_tests/testing_tests/test_condition.py
new file mode 100644
index 0000000..272dfc7
--- /dev/null
+++ b/tests/cupy_tests/testing_tests/test_condition.py
@@ -0,0 +1,190 @@
+import unittest
+
+from cupy.testing import _condition
+
+
+SKIP_REASON = 'test skip reason'
+
+
+# The test fixtures of this TestCase is used to be decorated by
+# decorator in test. So we do not run them alone.
+class MockUnitTest(unittest.TestCase):
+
+    failure_case_counter = 0
+    success_case_counter = 0
+    skip_case_counter = 0
+    probabilistic_case_counter = 0
+    probabilistic_case_success_counter = 0
+    probabilistic_case_failure_counter = 0
+
+    @staticmethod
+    def clear_counter():
+        MockUnitTest.failure_case_counter = 0
+        MockUnitTest.success_case_counter = 0
+        MockUnitTest.skip_case_counter = 0
+        MockUnitTest.probabilistic_case_counter = 0
+        MockUnitTest.probabilistic_case_success_counter = 0
+        MockUnitTest.probabilistic_case_failure_counter = 0
+
+    def failure_case(self):
+        MockUnitTest.failure_case_counter += 1
+        self.fail()
+
+    def success_case(self):
+        MockUnitTest.success_case_counter += 1
+        assert True
+
+    def skip_case(self):
+        MockUnitTest.skip_case_counter += 1
+        self.skipTest(SKIP_REASON)
+
+    def error_case(self):
+        raise Exception()
+
+    def probabilistic_case(self):
+        MockUnitTest.probabilistic_case_counter += 1
+        if MockUnitTest.probabilistic_case_counter % 2 == 0:
+            MockUnitTest.probabilistic_case_success_counter += 1
+            assert True
+        else:
+            MockUnitTest.probabilistic_case_failure_counter += 1
+            self.fail()
+
+    def runTest(self):
+        pass
+
+
+def _should_fail(self, f):
+    try:
+        f(self.unit_test)
+        self.fail(
+            'AssertionError is expected to be raised, but none is raised')
+    except AssertionError as e:
+        # check if the detail is included in the error object
+        assert 'first error message:' in str(e)
+
+
+def _should_pass(self, f):
+    f(self.unit_test)
+
+
+def _should_skip(self, f):
+    try:
+        f(self.unit_test)
+        self.fail(
+            'SkipTest is expected to be raised, but none is raised')
+    except unittest.SkipTest as e:
+        assert SKIP_REASON in str(e)
+
+
+class TestRepeatWithSuccessAtLeast(unittest.TestCase):
+
+    def _decorate(self, f, times, min_success):
+        return _condition.repeat_with_success_at_least(
+            times, min_success)(f)
+
+    def setUp(self):
+        self.unit_test = MockUnitTest()
+        MockUnitTest.clear_counter()
+
+    def test_all_trials_fail(self):
+        f = self._decorate(MockUnitTest.failure_case, 10, 1)
+        _should_fail(self, f)
+        assert self.unit_test.failure_case_counter == 10
+
+    def test_all_trials_fail2(self):
+        f = self._decorate(MockUnitTest.failure_case, 10, 0)
+        _should_pass(self, f)
+        assert self.unit_test.failure_case_counter <= 10
+
+    def test_all_trials_error(self):
+        f = self._decorate(MockUnitTest.error_case, 10, 1)
+        _should_fail(self, f)
+
+    def test_all_trials_succeed(self):
+        f = self._decorate(MockUnitTest.success_case, 10, 10)
+        _should_pass(self, f)
+        assert self.unit_test.success_case_counter == 10
+
+    def test_all_trials_succeed2(self):
+        self.assertRaises(AssertionError,
+                          _condition.repeat_with_success_at_least,
+                          10, 11)
+
+    def test_half_of_trials_succeed(self):
+        f = self._decorate(MockUnitTest.probabilistic_case, 10, 5)
+        _should_pass(self, f)
+        assert self.unit_test.probabilistic_case_counter <= 10
+        assert self.unit_test.probabilistic_case_success_counter >= 5
+        assert self.unit_test.probabilistic_case_failure_counter <= 5
+
+    def test_half_of_trials_succeed2(self):
+        f = self._decorate(MockUnitTest.probabilistic_case, 10, 6)
+        _should_fail(self, f)
+        assert self.unit_test.probabilistic_case_counter <= 10
+        assert self.unit_test.probabilistic_case_success_counter < 6
+        assert self.unit_test.probabilistic_case_failure_counter >= 5
+
+
+class TestRepeat(unittest.TestCase):
+
+    def _decorate(self, f, times):
+        return _condition.repeat(times)(f)
+
+    def setUp(self):
+        self.unit_test = MockUnitTest()
+        MockUnitTest.clear_counter()
+
+    def test_failure_case(self):
+        f = self._decorate(MockUnitTest.failure_case, 10)
+        _should_fail(self, f)
+        assert self.unit_test.failure_case_counter <= 10
+
+    def test_success_case(self):
+        f = self._decorate(MockUnitTest.success_case, 10)
+        _should_pass(self, f)
+        assert self.unit_test.success_case_counter == 10
+
+    def test_skip_case(self):
+        f = self._decorate(MockUnitTest.skip_case, 10)
+        _should_skip(self, f)
+        assert self.unit_test.skip_case_counter == 1
+
+    def test_probabilistic_case(self):
+        f = self._decorate(MockUnitTest.probabilistic_case, 10)
+        _should_fail(self, f)
+        assert self.unit_test.probabilistic_case_counter <= 10
+        assert self.unit_test.probabilistic_case_success_counter < 10
+        assert self.unit_test.probabilistic_case_failure_counter > 0
+
+
+class TestRetry(unittest.TestCase):
+
+    def _decorate(self, f, times):
+        return _condition.retry(times)(f)
+
+    def setUp(self):
+        self.unit_test = MockUnitTest()
+        MockUnitTest.clear_counter()
+
+    def test_failure_case(self):
+        f = self._decorate(MockUnitTest.failure_case, 10)
+        _should_fail(self, f)
+        assert self.unit_test.failure_case_counter == 10
+
+    def test_success_case(self):
+        f = self._decorate(MockUnitTest.success_case, 10)
+        _should_pass(self, f)
+        assert self.unit_test.success_case_counter <= 10
+
+    def test_skip_case(self):
+        f = self._decorate(MockUnitTest.skip_case, 10)
+        _should_skip(self, f)
+        assert self.unit_test.skip_case_counter == 1
+
+    def test_probabilistic_case(self):
+        f = self._decorate(MockUnitTest.probabilistic_case, 10)
+        _should_pass(self, f)
+        assert self.unit_test.probabilistic_case_counter <= 10
+        assert self.unit_test.probabilistic_case_success_counter > 0
+        assert self.unit_test.probabilistic_case_failure_counter < 10
diff --git a/tests/cupy_tests/testing_tests/test_helper.py b/tests/cupy_tests/testing_tests/test_helper.py
new file mode 100644
index 0000000..3c6c0b3
--- /dev/null
+++ b/tests/cupy_tests/testing_tests/test_helper.py
@@ -0,0 +1,185 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+
+
+class TestPackageRequirements:
+    def test_installed(self):
+        assert testing.installed('cupy')
+        assert testing.installed('cupy>9', 'numpy>=1.12')
+        assert testing.installed('numpy>=1.10,<=2.0')
+        assert not testing.installed('numpy>=2.0')
+        assert not testing.installed('numpy>1.10,<1.9')
+
+    def test_numpy_satisfies(self):
+        assert testing.numpy_satisfies('>1.10')
+        assert not testing.numpy_satisfies('>=2.10')
+
+    @testing.with_requires('numpy>2.0')
+    def test_with_requires(self):
+        assert False, 'this should not happen'
+
+
+@testing.parameterize(*testing.product({
+    'xp': [numpy, cupy],
+    'shape': [(3, 2), (), (3, 0, 2)],
+}))
+class TestShapedRandom(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    def test_shape_and_dtype(self, dtype):
+        a = testing.shaped_random(self.shape, self.xp, dtype)
+        assert isinstance(a, self.xp.ndarray)
+        assert a.shape == self.shape
+        assert a.dtype == dtype
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    def test_value_range(self, dtype):
+        a = testing.shaped_random(self.shape, self.xp, dtype)
+        assert self.xp.all(0 <= a)
+        assert self.xp.all(a < 10)
+
+    @testing.for_complex_dtypes()
+    def test_complex(self, dtype):
+        a = testing.shaped_random(self.shape, self.xp, dtype)
+        assert self.xp.all(0 <= a.real)
+        assert self.xp.all(a.real < 10)
+        assert self.xp.all(0 <= a.imag)
+        assert self.xp.all(a.imag < 10)
+        if 0 not in self.shape:
+            assert self.xp.any(a.imag)
+
+
+@testing.parameterize(*testing.product({
+    'xp': [numpy, cupy],
+}))
+class TestShapedRandomBool(unittest.TestCase):
+
+    def test_bool(self):
+        a = testing.shaped_random(10000, self.xp, numpy.bool_)
+        assert 4000 < self.xp.sum(a) < 6000
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [
+        numpy.float16, numpy.float32, numpy.float64,
+        numpy.complex64, numpy.complex128,
+    ],
+    'xp': [numpy, cupy],
+    'x_s_shapes': [
+        ((0, 0), (0,)),
+        ((2, 2), (2,)),
+        ((2, 3), (2,)),
+        ((3, 2), (2,)),
+        # broadcast
+        ((2, 2), ()),
+    ],
+}))
+class TestGenerateMatrix(unittest.TestCase):
+
+    def test_generate_matrix(self):
+        dtype = self.dtype
+        x_shape, s_shape = self.x_s_shapes
+        sv = self.xp.random.uniform(
+            0.5, 1.5, s_shape).astype(dtype().real.dtype)
+        x = testing.generate_matrix(
+            x_shape, xp=self.xp, dtype=dtype, singular_values=sv)
+        assert x.shape == x_shape
+
+        if 0 in x_shape:
+            return
+
+        s = self.xp.linalg.svd(
+            x.astype(numpy.complex128), full_matrices=False, compute_uv=False,
+        )
+        sv = self.xp.broadcast_to(sv, s.shape)
+        sv_sorted = self.xp.sort(sv, axis=-1)[..., ::-1]
+
+        rtol = 1e-3 if dtype == numpy.float16 else 1e-7
+        self.xp.testing.assert_allclose(s, sv_sorted, rtol=rtol)
+
+
+class TestGenerateMatrixInvalid(unittest.TestCase):
+
+    def test_no_singular_values(self):
+        with self.assertRaises(TypeError):
+            testing.generate_matrix((2, 2))
+
+    def test_invalid_shape(self):
+        with self.assertRaises(ValueError):
+            testing.generate_matrix((2,), singular_values=1)
+
+    def test_invalid_dtype_singular_values(self):
+        with self.assertRaises(TypeError):
+            testing.generate_matrix((2, 2), singular_values=1 + 0j)
+
+    def test_invalid_dtype(self):
+        with self.assertRaises(TypeError):
+            testing.generate_matrix(
+                (2, 2), dtype=numpy.int32, singular_values=1)
+
+    def test_negative_singular_values(self):
+        with self.assertRaises(ValueError):
+            testing.generate_matrix((2, 2), singular_values=[1, -1])
+
+    def test_shape_mismatch(self):
+        with self.assertRaises(ValueError):
+            testing.generate_matrix(
+                (2, 2), singular_values=numpy.ones(3))
+
+    def test_shape_mismatch_2(self):
+        with self.assertRaises(ValueError):
+            testing.generate_matrix(
+                (0, 2, 2), singular_values=numpy.ones(3))
+
+
+class TestAssertFunctionIsCalled(unittest.TestCase):
+
+    def test_patch_ndarray(self):
+        orig = cupy.ndarray
+        with testing.AssertFunctionIsCalled('cupy.ndarray'):
+            a = cupy.ndarray((2, 3), numpy.float32)
+        assert cupy.ndarray is orig
+        assert not isinstance(a, cupy.ndarray)
+
+    def test_spy_ndarray(self):
+        orig = cupy.ndarray
+        with testing.AssertFunctionIsCalled(
+                'cupy.ndarray', wraps=cupy.ndarray):
+            a = cupy.ndarray((2, 3), numpy.float32)
+        assert cupy.ndarray is orig
+        assert isinstance(a, cupy.ndarray)
+
+    def test_fail_not_called(self):
+        orig = cupy.ndarray
+        with pytest.raises(AssertionError):
+            with testing.AssertFunctionIsCalled('cupy.ndarray'):
+                pass
+        assert cupy.ndarray is orig
+
+    def test_fail_called_twice(self):
+        orig = cupy.ndarray
+        with pytest.raises(AssertionError):
+            with testing.AssertFunctionIsCalled('cupy.ndarray'):
+                cupy.ndarray((2, 3), numpy.float32)
+                cupy.ndarray((2, 3), numpy.float32)
+        assert cupy.ndarray is orig
+
+    def test_times_called(self):
+        orig = cupy.ndarray
+        with testing.AssertFunctionIsCalled('cupy.ndarray', times_called=2):
+            cupy.ndarray((2, 3), numpy.float32)
+            cupy.ndarray((2, 3), numpy.float32)
+        assert cupy.ndarray is orig
+
+    def test_inner_error(self):
+        orig = cupy.ndarray
+        with pytest.raises(numpy.AxisError):
+            with testing.AssertFunctionIsCalled('cupy.ndarray'):
+                cupy.ndarray((2, 3), numpy.float32)
+                raise numpy.AxisError('foo')
+        assert cupy.ndarray is orig
diff --git a/tests/cupy_tests/testing_tests/test_loops.py b/tests/cupy_tests/testing_tests/test_loops.py
new file mode 100644
index 0000000..8c8269c
--- /dev/null
+++ b/tests/cupy_tests/testing_tests/test_loops.py
@@ -0,0 +1,510 @@
+import re
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.testing import _loops
+
+
+class _Exception1(Exception):
+    pass
+
+
+class _Exception2(Exception):
+    pass
+
+
+class TestContainsSignedAndUnsigned(unittest.TestCase):
+
+    def test_include(self):
+        kw = {'x': numpy.int32, 'y': numpy.uint32}
+        assert _loops._contains_signed_and_unsigned(kw)
+
+        kw = {'x': numpy.float32, 'y': numpy.uint32}
+        assert _loops._contains_signed_and_unsigned(kw)
+
+    def test_signed_only(self):
+        kw = {'x': numpy.int32}
+        assert not _loops._contains_signed_and_unsigned(kw)
+
+        kw = {'x': numpy.float32}
+        assert not _loops._contains_signed_and_unsigned(kw)
+
+    def test_unsigned_only(self):
+        kw = {'x': numpy.uint32}
+        assert not _loops._contains_signed_and_unsigned(kw)
+
+    def test_ignore_not_dtype(self):
+        kw = {'x': numpy.int32, 'y': numpy.uint32, 'a': [0, 1]}
+        assert _loops._contains_signed_and_unsigned(kw)
+
+
+class TestCheckCupyNumpyError(unittest.TestCase):
+
+    tbs = {
+        cupy: 'xxxx',
+        numpy: 'yyyy'
+    }
+
+    def test_both_success(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_both_success(self, xp):
+                pass
+
+        with self.assertRaises(AssertionError):
+            dummy_both_success(self)
+
+    def test_cupy_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_cupy_error(self, xp):
+                if xp is cupy:
+                    raise Exception(self.tbs.get(cupy))
+
+        with self.assertRaisesRegex(AssertionError, self.tbs.get(cupy)):
+            dummy_cupy_error(self)
+
+    def test_numpy_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_numpy_error(self, xp):
+                if xp is numpy:
+                    raise Exception(self.tbs.get(numpy))
+
+        with self.assertRaisesRegex(AssertionError, self.tbs.get(numpy)):
+            dummy_numpy_error(self)
+
+    def test_cupy_numpy_different_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_cupy_numpy_different_error(self, xp):
+                if xp is cupy:
+                    raise TypeError(self.tbs.get(cupy))
+                elif xp is numpy:
+                    raise ValueError(self.tbs.get(numpy))
+
+        # Use re.S mode to ignore new line characters
+        pattern = re.compile(
+            self.tbs.get(cupy) + '.*' + self.tbs.get(numpy), re.S)
+        with self.assertRaisesRegex(AssertionError, pattern):
+            dummy_cupy_numpy_different_error(self)
+
+    def test_cupy_derived_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_cupy_derived_error(self, xp):
+                if xp is cupy:
+                    raise _Exception1(self.tbs.get(cupy))
+                elif xp is numpy:
+                    raise _Exception2(self.tbs.get(numpy))
+
+        dummy_cupy_derived_error(self)  # Assert no exceptions
+
+    def test_numpy_derived_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_numpy_derived_error(self, xp):
+                if xp is cupy:
+                    raise Exception(self.tbs.get(cupy))
+                elif xp is numpy:
+                    raise IndexError(self.tbs.get(numpy))
+
+        # NumPy errors may not derive from CuPy errors, i.e. CuPy errors should
+        # be at least as explicit as the NumPy error
+        pattern = re.compile(
+            self.tbs.get(cupy) + '.*' + self.tbs.get(numpy), re.S)
+        with self.assertRaisesRegex(AssertionError, pattern):
+            dummy_numpy_derived_error(self)
+
+    def test_same_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises(accept_error=Exception)
+            def dummy_same_error(self, xp):
+                raise Exception(self.tbs.get(xp))
+
+        dummy_same_error(self)
+
+    def test_cupy_derived_unaccept_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises(accept_error=ValueError)
+            def dummy_cupy_derived_unaccept_error(self, xp):
+                if xp is cupy:
+                    raise IndexError(self.tbs.get(cupy))
+                elif xp is numpy:
+                    raise Exception(self.tbs.get(numpy))
+
+        # Neither `IndexError` nor `Exception` is derived from `ValueError`,
+        # therefore expect an error
+        pattern = re.compile(
+            self.tbs.get(cupy) + '.*' + self.tbs.get(numpy), re.S)
+        with self.assertRaisesRegex(AssertionError, pattern):
+            dummy_cupy_derived_unaccept_error(self)
+
+    def test_numpy_derived_unaccept_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises(accept_error=ValueError)
+            def dummy_numpy_derived_unaccept_error(self, xp):
+                if xp is cupy:
+                    raise Exception(self.tbs.get(cupy))
+                elif xp is numpy:
+                    raise ValueError(self.tbs.get(numpy))
+
+        # `Exception` is not derived from `ValueError`, therefore expect an
+        # error
+        pattern = re.compile(
+            self.tbs.get(cupy) + '.*' + self.tbs.get(numpy), re.S)
+        with self.assertRaisesRegex(AssertionError, pattern):
+            dummy_numpy_derived_unaccept_error(self)
+
+    def test_forbidden_error(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises(accept_error=False)
+            def dummy_forbidden_error(self, xp):
+                raise Exception(self.tbs.get(xp))
+
+        pattern = re.compile(
+            self.tbs.get(cupy) + '.*' + self.tbs.get(numpy), re.S)
+        with self.assertRaisesRegex(AssertionError, pattern):
+            dummy_forbidden_error(self)
+
+    def test_axis_error_different_type(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_axis_error(self, xp):
+                if xp is cupy:
+                    raise numpy.AxisError(self.tbs.get(cupy))
+                elif xp is numpy:
+                    raise TypeError(self.tbs.get(numpy))
+
+        pattern = re.compile(
+            self.tbs.get(cupy) + '.*' + self.tbs.get(numpy), re.S)
+        with self.assertRaisesRegex(AssertionError, pattern):
+            dummy_axis_error(self)
+
+    def test_axis_error_value_different_type(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_axis_error(self, xp):
+                if xp is cupy:
+                    raise numpy.AxisError(self.tbs.get(cupy))
+                elif xp is numpy:
+                    raise ValueError(self.tbs.get(numpy))
+
+        pattern = re.compile(
+            self.tbs.get(cupy) + '.*' + self.tbs.get(numpy), re.S)
+        with self.assertRaisesRegex(AssertionError, pattern):
+            dummy_axis_error(self)
+
+    def test_axis_error_index_different_type(self):
+        with testing.assert_warns(DeprecationWarning):
+            @testing.numpy_cupy_raises()
+            def dummy_axis_error(self, xp):
+                if xp is cupy:
+                    raise numpy.AxisError(self.tbs.get(cupy))
+                elif xp is numpy:
+                    raise IndexError(self.tbs.get(numpy))
+
+        pattern = re.compile(
+            self.tbs.get(cupy) + '.*' + self.tbs.get(numpy), re.S)
+        with self.assertRaisesRegex(AssertionError, pattern):
+            dummy_axis_error(self)
+
+
+class NumPyCuPyDecoratorBase(object):
+
+    def test_valid(self):
+        decorator = getattr(testing, self.decorator)()
+        decorated_func = decorator(type(self).valid_func)
+        decorated_func(self)
+
+    def test_invalid(self):
+        decorator = getattr(testing, self.decorator)()
+        decorated_func = decorator(type(self).invalid_func)
+        with self.assertRaises(AssertionError):
+            decorated_func(self)
+
+    def test_name(self):
+        decorator = getattr(testing, self.decorator)(name='foo')
+        decorated_func = decorator(type(self).strange_kw_func)
+        decorated_func(self)
+
+
+def numpy_error(_, xp):
+    if xp == numpy:
+        raise ValueError()
+    elif xp == cupy:
+        return cupy.array(1)
+
+
+def cupy_error(_, xp):
+    if xp == numpy:
+        return numpy.array(1)
+    elif xp == cupy:
+        raise ValueError()
+
+
+class NumPyCuPyDecoratorBase2(object):
+
+    def test_accept_error_numpy(self):
+        decorator = getattr(testing, self.decorator)(accept_error=False)
+        decorated_func = decorator(numpy_error)
+        with self.assertRaises(AssertionError):
+            decorated_func(self)
+
+    def test_accept_error_cupy(self):
+        decorator = getattr(testing, self.decorator)(accept_error=False)
+        decorated_func = decorator(cupy_error)
+        with self.assertRaises(AssertionError):
+            decorated_func(self)
+
+
+def make_result(xp, np_result, cp_result):
+    if xp == numpy:
+        return np_result
+    elif xp == cupy:
+        return cp_result
+
+
+@testing.parameterize(
+    {'decorator': 'numpy_cupy_allclose'},
+    {'decorator': 'numpy_cupy_array_almost_equal'},
+    {'decorator': 'numpy_cupy_array_almost_equal_nulp'},
+    {'decorator': 'numpy_cupy_array_max_ulp'},
+    {'decorator': 'numpy_cupy_array_equal'}
+)
+class TestNumPyCuPyEqual(unittest.TestCase, NumPyCuPyDecoratorBase,
+                         NumPyCuPyDecoratorBase2):
+
+    def valid_func(self, xp):
+        return make_result(xp, numpy.array(1), cupy.array(1))
+
+    def invalid_func(self, xp):
+        return make_result(xp, numpy.array(1), cupy.array(2))
+
+    def strange_kw_func(self, foo):
+        return make_result(foo, numpy.array(1), cupy.array(1))
+
+
+@testing.parameterize(
+    {'decorator': 'numpy_cupy_array_equal'}
+)
+class TestNumPyCuPyListEqual(unittest.TestCase, NumPyCuPyDecoratorBase):
+
+    def valid_func(self, xp):
+        return make_result(xp, [numpy.array(1)], [cupy.array(1)])
+
+    def invalid_func(self, xp):
+        return make_result(xp, [numpy.array(1)], [cupy.array(2)])
+
+    def strange_kw_func(self, foo):
+        return make_result(foo, [numpy.array(1)], [cupy.array(1)])
+
+
+@testing.parameterize(
+    {'decorator': 'numpy_cupy_array_less'}
+)
+class TestNumPyCuPyLess(unittest.TestCase, NumPyCuPyDecoratorBase,
+                        NumPyCuPyDecoratorBase2):
+
+    def valid_func(self, xp):
+        return make_result(xp, numpy.array(2), cupy.array(1))
+
+    def invalid_func(self, xp):
+        return make_result(xp, numpy.array(1), cupy.array(2))
+
+    def strange_kw_func(self, foo):
+        return make_result(foo, numpy.array(2), cupy.array(1))
+
+
+class TestNumPyCuPyAllCloseTolPerDtype(unittest.TestCase):
+
+    def _test_rtol(self, xp, dtype):
+        if xp is numpy:
+            return numpy.array(1, dtype=dtype)
+        else:
+            finfo = numpy.finfo(dtype)
+            return cupy.array(1 + finfo.eps, dtype=dtype)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(rtol={numpy.float16: 1e-3, 'default': 1e-6})
+    def test_rtol_per_dtype(self, xp, dtype):
+        return self._test_rtol(xp, dtype)
+
+    @pytest.mark.xfail(strict=True)
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6)
+    def test_rtol_fail(self, xp, dtype):
+        return self._test_rtol(xp, dtype)
+
+    def test_rtol_invalid_key(self):
+        with self.assertRaises(TypeError):
+            testing.numpy_cupy_allclose(rtol={'float16': 1e-3})
+
+    def _test_atol(self, xp, dtype):
+        if xp is numpy:
+            return numpy.array(0, dtype=dtype)
+        else:
+            finfo = numpy.finfo(dtype)
+            return cupy.array(finfo.eps, dtype=dtype)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol={numpy.float16: 1e-3, 'default': 1e-6})
+    def test_atol_per_dtype(self, xp, dtype):
+        return self._test_atol(xp, dtype)
+
+    @pytest.mark.xfail(strict=True)
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-6)
+    def test_atol_fail(self, xp, dtype):
+        return self._test_atol(xp, dtype)
+
+    def test_atol_invalid_key(self):
+        with self.assertRaises(TypeError):
+            testing.numpy_cupy_allclose(atol={'float16': 1e-3})
+
+
+class TestIgnoreOfNegativeValueDifferenceOnCpuAndGpu(unittest.TestCase):
+
+    @testing.numpy_cupy_allclose()
+    def correct_failure(self, dtype1, dtype2, xp):
+        if xp == numpy:
+            return xp.array(-1, dtype=numpy.float32)
+        else:
+            return xp.array(-2, dtype=numpy.float32)
+
+    @testing.for_unsigned_dtypes('dtype1')
+    @testing.for_signed_dtypes('dtype2')
+    def test_correct_failure(self, dtype1, dtype2):
+        with pytest.raises(AssertionError):
+            self.correct_failure(dtype1, dtype2)
+
+    @testing.for_unsigned_dtypes('dtype1')
+    @testing.for_signed_dtypes('dtype2')
+    @testing.numpy_cupy_allclose()
+    def test_correct_success(self, xp, dtype1, dtype2):
+        # Behavior of assigning a negative value to an unsigned integer
+        # variable is undefined.
+        # nVidia GPUs and Intel CPUs behave differently.
+        # To avoid this difference, we need to ignore dimensions whose
+        # values are negative.
+        if xp == numpy:
+            return xp.array(-1).astype(dtype1)
+        else:
+            return xp.array(-2).astype(dtype1)
+
+
+@testing.parameterize(*testing.product({
+    'framework': ['unittest', 'pytest']
+}))
+class TestSkip(unittest.TestCase):
+
+    def _skip(self, reason):
+        if self.framework == 'unittest':
+            self.skipTest(reason)
+        else:
+            pytest.skip(reason)
+
+    @testing.numpy_cupy_allclose()
+    def test_allclose(self, xp):
+        self._skip('Test for skip with @numpy_cupy_allclose')
+        assert False
+
+    @testing.numpy_cupy_array_almost_equal()
+    def test_array_almost_equal(self, xp):
+        raise self._skip('Test for skip with @numpy_cupy_array_almost_equal')
+        assert False
+
+    @testing.numpy_cupy_array_almost_equal_nulp()
+    def test_array_almost_equal_nulp(self, xp):
+        raise self._skip(
+            'Test for skip with @numpy_cupy_array_almost_equal_nulp')
+        assert False
+
+    @testing.numpy_cupy_array_max_ulp()
+    def test_array_max_ulp(self, xp):
+        raise self._skip('Test for skip with @numpy_cupy_array_max_ulp')
+        assert False
+
+    @testing.numpy_cupy_array_equal()
+    def test_array_equal(self, xp):
+        raise self._skip('Test for skip with @numpy_cupy_array_equal')
+        assert False
+
+    @testing.numpy_cupy_array_less()
+    def test_less(self, xp):
+        raise self._skip('Test for skip with @numpy_cupy_array_less')
+        assert False
+
+    @testing.numpy_cupy_equal()
+    def test_equal(self, xp):
+        raise self._skip('Test for skip with @numpy_cupy_equal')
+        assert False
+
+    @testing.for_all_dtypes()
+    def test_dtypes(self, dtype):
+        if dtype is cupy.float32:
+            raise self._skip('Test for skipping a dtype in @for_all_dtypes')
+            assert False
+        else:
+            assert True
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_dtypes_allclose(self, xp, dtype):
+        if dtype is xp.float32:
+            raise self._skip('Test for skipping a dtype in @for_all_dtypes')
+            assert False
+        else:
+            return xp.array(True)
+
+
+@testing.parameterize(*testing.product({
+    'framework': ['unittest', 'pytest']
+}))
+class TestSkipFail(unittest.TestCase):
+
+    def _skip(self, reason):
+        if self.framework == 'unittest':
+            raise unittest.SkipTest(reason)
+        else:
+            pytest.skip(reason)
+
+    @pytest.mark.xfail(strict=True)
+    @testing.numpy_cupy_allclose()
+    def test_different_reason(self, xp):
+        if xp is numpy:
+            self._skip('skip1')
+        else:
+            self._skip('skip2')
+
+    @pytest.mark.xfail(strict=True)
+    @testing.numpy_cupy_allclose()
+    def test_only_numpy(self, xp):
+        if xp is numpy:
+            self._skip('skip')
+        else:
+            return xp.array(True)
+
+    @pytest.mark.xfail(strict=True)
+    @testing.numpy_cupy_allclose()
+    def test_only_cupy(self, xp):
+        if xp is numpy:
+            return xp.array(True)
+        else:
+            self._skip('skip')
+
+    @pytest.mark.xfail(strict=True)
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose()
+    def test_dtype_only_cupy(self, xp, dtype):
+        if dtype is not xp.float32:
+            return xp.array(True)
+
+        if xp is numpy:
+            return xp.array(True)
+        else:
+            self._skip('skip')
diff --git a/tests/cupy_tests/testing_tests/test_parameterized.py b/tests/cupy_tests/testing_tests/test_parameterized.py
new file mode 100644
index 0000000..a77a5e3
--- /dev/null
+++ b/tests/cupy_tests/testing_tests/test_parameterized.py
@@ -0,0 +1,232 @@
+import collections
+import re
+import textwrap
+import unittest
+
+import pytest
+
+from cupy import testing
+
+
+@testing.parameterize(
+    {'actual': {'a': [1, 2], 'b': [3, 4, 5]},
+     'expect': [{'a': 1, 'b': 3}, {'a': 1, 'b': 4}, {'a': 1, 'b': 5},
+                {'a': 2, 'b': 3}, {'a': 2, 'b': 4}, {'a': 2, 'b': 5}]},
+    {'actual': {'a': [1, 2]}, 'expect': [{'a': 1}, {'a': 2}]},
+    {'actual': {'a': [1, 2], 'b': []}, 'expect': []},
+    {'actual': {'a': []}, 'expect': []},
+    {'actual': {}, 'expect': [{}]})
+class TestProduct(unittest.TestCase):
+
+    def test_product(self):
+        assert testing.product(self.actual) == self.expect
+
+
+@testing.parameterize(
+    {'actual': [[{'a': 1, 'b': 3}, {'a': 2, 'b': 4}], [{'c': 5}, {'c': 6}]],
+     'expect': [{'a': 1, 'b': 3, 'c': 5}, {'a': 1, 'b': 3, 'c': 6},
+                {'a': 2, 'b': 4, 'c': 5}, {'a': 2, 'b': 4, 'c': 6}]},
+    {'actual': [[{'a': 1}, {'a': 2}], [{'b': 3}, {'b': 4}, {'b': 5}]],
+     'expect': [{'a': 1, 'b': 3}, {'a': 1, 'b': 4}, {'a': 1, 'b': 5},
+                {'a': 2, 'b': 3}, {'a': 2, 'b': 4}, {'a': 2, 'b': 5}]},
+    {'actual': [[{'a': 1}, {'a': 2}]], 'expect': [{'a': 1}, {'a': 2}]},
+    {'actual': [[{'a': 1}, {'a': 2}], []], 'expect': []},
+    {'actual': [[]], 'expect': []},
+    {'actual': [], 'expect': [{}]})
+class TestProductDict(unittest.TestCase):
+
+    def test_product_dict(self):
+        assert testing.product_dict(*self.actual) == self.expect
+
+
+def f(x):
+    return x
+
+
+class C(object):
+
+    def __repr__(self):
+        return '<C object>'
+
+    def __call__(self, x):
+        return x
+
+    def method(self, x):
+        return x
+
+
+@testing.parameterize(
+    {'callable': f},
+    {'callable': lambda x: x},
+    {'callable': C()},
+    {'callable': C().method}
+)
+class TestParameterize(unittest.TestCase):
+
+    def test_callable(self):
+        y = self.callable(1)
+        assert y == 1
+
+    def test_skip(self):
+        # Skipping the test case should not report error.
+        self.skipTest('skip')
+
+
+@testing.parameterize(
+    {'callable': f},
+    {'callable': lambda x: x},
+    {'callable': C()},
+    {'callable': C().method}
+)
+class TestParameterizePytestImpl:
+
+    def test_callable(self):
+        y = self.callable(1)
+        assert y == 1
+
+    def test_skip(self):
+        # Skipping the test case should not report error.
+        pytest.skip('skip')
+
+
+@pytest.mark.parametrize(('src', 'outcomes'), [
+    (  # simple
+        textwrap.dedent('''\
+        @testing.parameterize({'a': 1}, {'a': 2})
+        class TestA:
+            def test_a(self):
+                assert self.a > 0
+        '''), [
+            ("::TestA::test_a[_param_0_{a=1}]", 'PASSED'),
+            ("::TestA::test_a[_param_1_{a=2}]", 'PASSED'),
+        ]
+    ),
+    (  # simple fail
+        textwrap.dedent('''\
+        @testing.parameterize({'a': 1}, {'a': 2})
+        class TestA:
+            def test_a(self):
+                assert self.a == 1
+        '''), [
+            ("::TestA::test_a[_param_0_{a=1}]", 'PASSED'),
+            ("::TestA::test_a[_param_1_{a=2}]", 'FAILED'),
+        ]
+    ),
+    (  # set of params can be different
+        textwrap.dedent('''\
+        @testing.parameterize({'a': 1}, {'b': 2})
+        class TestA:
+            def test_a(self):
+                a = getattr(self, 'a', 3)
+                b = getattr(self, 'b', 4)
+                assert (a, b) in [(1, 4), (3, 2)]
+        '''), [
+            ("::TestA::test_a[_param_0_{a=1}]", 'PASSED'),
+            ("::TestA::test_a[_param_1_{b=2}]", 'PASSED'),
+        ]
+    ),
+    (  # do not copy each value but do not share the param dict
+        textwrap.dedent('''\
+        import numpy
+        @testing.parameterize({'a': numpy.array(1)}, {'a': 1})
+        class TestA:
+            def test_first(self):
+                assert self.a == 1
+                self.a += 2
+            def test_second(self):
+                assert self.a == 1
+                self.a += 2
+        '''), [
+            ("::TestA::test_first[_param_0_{a=array(1)}]", 'PASSED'),
+            ("::TestA::test_first[_param_1_{a=1}]", 'PASSED'),
+            ("::TestA::test_second[_param_0_{a=array(1)}]", 'FAILED'),
+            ("::TestA::test_second[_param_1_{a=1}]", 'PASSED'),
+        ]
+    ),
+    (  # multiple params and class attr
+        textwrap.dedent('''\
+        @testing.parameterize({'a': 1, 'b': 4}, {'a': 2, 'b': 3})
+        class TestA:
+            c = 5
+            def test_a(self):
+                assert self.a + self.b == self.c
+        '''), [
+            ("::TestA::test_a[_param_0_{a=1, b=4}]", 'PASSED'),
+            ("::TestA::test_a[_param_1_{a=2, b=3}]", 'PASSED'),
+        ]
+    ),
+    (  # combine pytest.mark.parameterize
+        textwrap.dedent('''\
+        import pytest
+        @pytest.mark.parametrize("outer", ["E", "e"])
+        @testing.parameterize({"x": "D"}, {"x": "d"})
+        @pytest.mark.parametrize("inner", ["c", "C"])
+        class TestA:
+            @pytest.mark.parametrize(
+                ("fn1", "fn2"), [("A", "b"), ("a", "B")])
+            def test_a(self, fn2, inner, outer, fn1):
+                assert (
+                    (fn1 + fn2 + inner + self.x + outer).lower()
+                    == "abcde")
+            @pytest.mark.parametrize(
+                "fn", ["A", "a"])
+            def test_b(self, outer, fn, inner):
+                assert sum(
+                    c.isupper() for c in [fn, inner, self.x, outer]
+                ) != 2
+        '''), [
+            ("::TestA::test_a[A-b-_param_0_{x='D'}-E-c]", 'PASSED'),
+            ("::TestA::test_a[A-b-_param_0_{x='D'}-E-C]", 'PASSED'),
+            ("::TestA::test_a[A-b-_param_0_{x='D'}-e-c]", 'PASSED'),
+            ("::TestA::test_a[A-b-_param_0_{x='D'}-e-C]", 'PASSED'),
+            ("::TestA::test_a[A-b-_param_1_{x='d'}-E-c]", 'PASSED'),
+            ("::TestA::test_a[A-b-_param_1_{x='d'}-E-C]", 'PASSED'),
+            ("::TestA::test_a[A-b-_param_1_{x='d'}-e-c]", 'PASSED'),
+            ("::TestA::test_a[A-b-_param_1_{x='d'}-e-C]", 'PASSED'),
+            ("::TestA::test_a[a-B-_param_0_{x='D'}-E-c]", 'PASSED'),
+            ("::TestA::test_a[a-B-_param_0_{x='D'}-E-C]", 'PASSED'),
+            ("::TestA::test_a[a-B-_param_0_{x='D'}-e-c]", 'PASSED'),
+            ("::TestA::test_a[a-B-_param_0_{x='D'}-e-C]", 'PASSED'),
+            ("::TestA::test_a[a-B-_param_1_{x='d'}-E-c]", 'PASSED'),
+            ("::TestA::test_a[a-B-_param_1_{x='d'}-E-C]", 'PASSED'),
+            ("::TestA::test_a[a-B-_param_1_{x='d'}-e-c]", 'PASSED'),
+            ("::TestA::test_a[a-B-_param_1_{x='d'}-e-C]", 'PASSED'),
+            ("::TestA::test_b[A-_param_0_{x='D'}-E-c]", 'PASSED'),
+            ("::TestA::test_b[A-_param_0_{x='D'}-E-C]", 'PASSED'),
+            ("::TestA::test_b[A-_param_0_{x='D'}-e-c]", 'FAILED'),
+            ("::TestA::test_b[A-_param_0_{x='D'}-e-C]", 'PASSED'),
+            ("::TestA::test_b[A-_param_1_{x='d'}-E-c]", 'FAILED'),
+            ("::TestA::test_b[A-_param_1_{x='d'}-E-C]", 'PASSED'),
+            ("::TestA::test_b[A-_param_1_{x='d'}-e-c]", 'PASSED'),
+            ("::TestA::test_b[A-_param_1_{x='d'}-e-C]", 'FAILED'),
+            ("::TestA::test_b[a-_param_0_{x='D'}-E-c]", 'FAILED'),
+            ("::TestA::test_b[a-_param_0_{x='D'}-E-C]", 'PASSED'),
+            ("::TestA::test_b[a-_param_0_{x='D'}-e-c]", 'PASSED'),
+            ("::TestA::test_b[a-_param_0_{x='D'}-e-C]", 'FAILED'),
+            ("::TestA::test_b[a-_param_1_{x='d'}-E-c]", 'PASSED'),
+            ("::TestA::test_b[a-_param_1_{x='d'}-E-C]", 'FAILED'),
+            ("::TestA::test_b[a-_param_1_{x='d'}-e-c]", 'PASSED'),
+            ("::TestA::test_b[a-_param_1_{x='d'}-e-C]", 'PASSED'),
+        ]
+    ),
+])
+# Pytest 7.0.1 creates test with names test_b[A-c-_param_0_{x='D'}-e]
+#  instead of test_b[A-_param_0_{x='D'}-e-c]
+@pytest.mark.skipif(
+    pytest.__version__ > "7.4.2", reason="test name not compatible"
+)
+def test_parameterize_pytest_impl(testdir, src, outcomes):
+    testdir.makepyfile('from cupy import testing\n' + src)
+    result = testdir.runpytest('-v', '--tb=no')
+    expected_lines = [
+        '.*{} {}.*'.format(re.escape(name), res)
+        for name, res in outcomes
+    ]
+    print("Result", pytest.__version__)
+    print("---")
+    print("Expected", '\n'.join(expected_lines))
+    result.stdout.re_match_lines(expected_lines)
+    expected_count = collections.Counter(
+        [res.lower() for _, res in outcomes]
+    )
+    result.assert_outcomes(**expected_count)
diff --git a/tests/cupyx_tests/__init__.py b/tests/cupyx_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/distributed_tests/comm_runner.py b/tests/cupyx_tests/distributed_tests/comm_runner.py
new file mode 100644
index 0000000..e5d1130
--- /dev/null
+++ b/tests/cupyx_tests/distributed_tests/comm_runner.py
@@ -0,0 +1,607 @@
+import sys
+import time
+import warnings
+
+import cupy
+from cupy import cuda
+from cupy.cuda import nccl
+from cupy import testing
+
+from cupyx.distributed import init_process_group
+from cupyx.distributed._nccl_comm import NCCLBackend
+from cupyx.distributed._store import ExceptionAwareProcess
+from cupyx.scipy import sparse
+
+
+nccl_available = nccl.available
+
+
+N_WORKERS = 2
+
+
+def _launch_workers(func, args=(), n_workers=N_WORKERS):
+    processes = []
+    # TODO catch exceptions
+    for rank in range(n_workers):
+        p = ExceptionAwareProcess(
+            target=func,
+            args=(rank,) + args)
+        p.start()
+        processes.append(p)
+
+    for p in processes:
+        p.join()
+
+
+def broadcast(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_broadcast(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        expected = cupy.arange(2 * 3 * 4, dtype=dtype).reshape((2, 3, 4))
+        if rank == root:
+            in_array = expected
+        else:
+            in_array = cupy.zeros((2, 3, 4), dtype=dtype)
+        comm.broadcast(in_array, root)
+        testing.assert_allclose(in_array, expected)
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_broadcast(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_broadcast(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_broadcast, (0, dtype))
+        _launch_workers(run_broadcast, (1, dtype))
+
+
+def reduce(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_reduce(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = cupy.arange(2 * 3 * 4, dtype='f').reshape(2, 3, 4)
+        out_array = cupy.zeros((2, 3, 4), dtype='f')
+        comm.reduce(in_array, out_array, root)
+        if rank == root:
+            testing.assert_allclose(out_array, 2 * in_array)
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_reduce(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_reduce(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_reduce, (0, dtype))
+        _launch_workers(run_reduce, (1, dtype))
+
+
+def all_reduce(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_all_reduce(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = cupy.arange(2 * 3 * 4, dtype='f').reshape(2, 3, 4)
+        out_array = cupy.zeros((2, 3, 4), dtype='f')
+
+        comm.all_reduce(in_array, out_array)
+        testing.assert_allclose(out_array, 2 * in_array)
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_all_reduce(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_all_reduce, (dtype,))
+
+
+def reduce_scatter(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_reduce_scatter(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = 1 + cupy.arange(
+            N_WORKERS * 10, dtype='f').reshape(N_WORKERS, 10)
+        out_array = cupy.zeros((10,), dtype='f')
+
+        comm.reduce_scatter(in_array, out_array, 10)
+        testing.assert_allclose(out_array, 2 * in_array[rank])
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_reduce_scatter(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_reduce_scatter, (dtype,))
+
+
+def all_gather(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_all_gather(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = (rank + 1) * cupy.arange(
+            N_WORKERS * 10, dtype='f').reshape(N_WORKERS, 10)
+        out_array = cupy.zeros((N_WORKERS, 10), dtype='f')
+        comm.all_gather(in_array, out_array, 10)
+        expected = 1 + cupy.arange(N_WORKERS).reshape(N_WORKERS, 1)
+        expected = expected * cupy.broadcast_to(
+            cupy.arange(10, dtype='f'), (N_WORKERS, 10))
+        testing.assert_allclose(out_array, expected)
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_all_gather(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_all_gather, (dtype,))
+
+
+def send_and_recv(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_send_and_recv(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = cupy.arange(10, dtype='f')
+        out_array = cupy.zeros((10,), dtype='f')
+        if rank == 0:
+            comm.send(in_array, 1)
+        else:
+            comm.recv(out_array, 0)
+            testing.assert_allclose(out_array, in_array)
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_send_and_recv(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_send_and_recv, (dtype,))
+
+
+def send_recv(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_send_recv(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = cupy.arange(10, dtype='f')
+        for i in range(N_WORKERS):
+            out_array = cupy.zeros((10,), dtype='f')
+            comm.send_recv(in_array, out_array, i)
+            testing.assert_allclose(out_array, in_array)
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_send_recv(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_send_recv, (dtype,))
+
+
+def scatter(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_scatter(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = 1 + cupy.arange(
+            N_WORKERS * 10, dtype='f').reshape(N_WORKERS, 10)
+        out_array = cupy.zeros((10,), dtype='f')
+
+        comm.scatter(in_array, out_array, root)
+        if rank > 0:
+            testing.assert_allclose(out_array, in_array[rank])
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_scatter(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_scatter(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_scatter, (0, dtype))
+        _launch_workers(run_scatter, (1, dtype))
+
+
+def gather(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_gather(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = (rank + 1) * cupy.arange(10, dtype='f')
+        out_array = cupy.zeros((N_WORKERS, 10), dtype='f')
+        comm.gather(in_array, out_array, root)
+        if rank == root:
+            expected = 1 + cupy.arange(N_WORKERS).reshape(N_WORKERS, 1)
+            expected = expected * cupy.broadcast_to(
+                cupy.arange(10, dtype='f'), (N_WORKERS, 10))
+            testing.assert_allclose(out_array, expected)
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_gather(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_gather(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_gather, (0, dtype))
+        _launch_workers(run_gather, (1, dtype))
+
+
+def all_to_all(dtype, use_mpi=False):
+    if dtype in 'hH':
+        return  # nccl does not support int16
+
+    def run_all_to_all(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = cupy.arange(
+            N_WORKERS * 10, dtype='f').reshape(N_WORKERS, 10)
+        out_array = cupy.zeros((N_WORKERS, 10), dtype='f')
+        comm.all_to_all(in_array, out_array)
+        expected = (10 * rank) + cupy.broadcast_to(
+            cupy.arange(10, dtype='f'), (N_WORKERS, 10))
+        testing.assert_allclose(out_array, expected)
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_all_to_all(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_all_to_all, (dtype,))
+
+
+def barrier(use_mpi=False):
+    def run_barrier(rank, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        comm.barrier()
+        before = time.time()
+        if rank == 0:
+            time.sleep(2)
+        comm.barrier()
+        after = time.time()
+        assert int(after - before) == 2
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_barrier(MPI.COMM_WORLD.Get_rank(), True)
+    else:
+        _launch_workers(run_barrier)
+
+
+def init(use_mpi=False):
+    def run_init(rank, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = init_process_group(N_WORKERS, rank, use_mpi=use_mpi)
+        # Do a simple call to verify we got a valid comm
+        in_array = cupy.zeros(1)
+        if rank == 0:
+            in_array = in_array + 1
+        comm.broadcast(in_array, 0)
+        testing.assert_allclose(in_array, cupy.ones(1))
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_init(MPI.COMM_WORLD.Get_rank(), dtype, True)
+        run_init(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_init)
+
+
+def _make_sparse(dtype):
+    data = cupy.array([1, 3, 2, 5, 1, 1], dtype)
+    indices = cupy.array([0, 3, 1, 3, 0, 2], 'i')
+    indptr = cupy.array([0, 2, 3, 4, 6], 'i')
+    return sparse.csr_matrix((data, indices, indptr), shape=(4, 4))
+
+
+def _make_sparse_empty(dtype):
+    data = cupy.array([0], dtype)
+    indices = cupy.array([0], 'i')
+    indptr = cupy.array([0], 'i')
+    return sparse.csr_matrix((data, indices, indptr), shape=(0, 0))
+
+
+def sparse_send_and_recv(dtype, use_mpi=False):
+    def run_send_and_recv(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = _make_sparse(dtype)
+        out_array = _make_sparse_empty(dtype)
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        if rank == 0:
+            comm.send(in_array, 1)
+        else:
+            comm.recv(out_array, 0)
+            testing.assert_allclose(out_array.todense(), in_array.todense())
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_send_and_recv(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_send_and_recv, (dtype,))
+
+
+def sparse_send_recv(dtype, use_mpi=False):
+    def run_send_recv(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = _make_sparse(dtype)
+        out_array = _make_sparse_empty(dtype)
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        if rank == 0:
+            comm.send_recv(in_array, out_array, 1)
+        else:
+            comm.send_recv(in_array, out_array, 0)
+            testing.assert_allclose(out_array.todense(), in_array.todense())
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_send_recv(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_send_recv, (dtype,))
+
+
+def sparse_broadcast(dtype, use_mpi=False):
+
+    def run_broadcast(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        expected = _make_sparse(dtype)
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        if rank == root:
+            in_array = expected
+        else:
+            in_array = _make_sparse_empty(dtype)
+        comm.broadcast(in_array, root)
+        testing.assert_allclose(in_array.todense(), expected.todense())
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_broadcast(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_broadcast(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_broadcast, (0, dtype,))
+        _launch_workers(run_broadcast, (1, dtype,))
+
+
+def sparse_reduce(dtype, use_mpi=False):
+
+    def run_reduce(rank, root, dtype, op, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = _make_sparse(dtype)
+        out_array = _make_sparse_empty(dtype)
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        comm.reduce(in_array, out_array, root, op)
+        if rank == root:
+            if op == 'sum':
+                testing.assert_allclose(
+                    out_array.todense(), 2 * in_array.todense())
+            else:
+                testing.assert_allclose(
+                    out_array.todense(),
+                    cupy.matmul(in_array.todense(), in_array.todense()))
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_reduce(MPI.COMM_WORLD.Get_rank(), 0, dtype, 'sum', True)
+        run_reduce(MPI.COMM_WORLD.Get_rank(), 1, dtype, 'prod', True)
+    else:
+        _launch_workers(run_reduce, (0, dtype, 'sum'))
+        _launch_workers(run_reduce, (1, dtype, 'prod'))
+
+
+def sparse_all_reduce(dtype, use_mpi=False):
+
+    def run_all_reduce(rank, dtype, op, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = _make_sparse(dtype)
+        out_array = _make_sparse_empty(dtype)
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        comm.all_reduce(in_array, out_array, op)
+        if op == 'sum':
+            testing.assert_allclose(
+                out_array.todense(), 2 * in_array.todense())
+        else:
+            testing.assert_allclose(
+                out_array.todense(),
+                cupy.matmul(in_array.todense(), in_array.todense()))
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_all_reduce(MPI.COMM_WORLD.Get_rank(), dtype, 'sum', True)
+        run_all_reduce(MPI.COMM_WORLD.Get_rank(), dtype, 'prod', True)
+    else:
+        _launch_workers(run_all_reduce, (dtype, 'sum'))
+        _launch_workers(run_all_reduce, (dtype, 'prod'))
+
+
+def sparse_scatter(dtype, use_mpi=False):
+
+    def run_scatter(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_arrays = [_make_sparse(dtype), 2*_make_sparse(dtype)]
+        out_array = _make_sparse_empty(dtype)
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        comm.scatter(in_arrays, out_array, root)
+        testing.assert_allclose(
+            out_array.todense(), in_arrays[rank].todense())
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_scatter(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_scatter(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_scatter, (0, dtype))
+        _launch_workers(run_scatter, (1, dtype))
+
+
+def sparse_gather(dtype, use_mpi=False):
+
+    def run_gather(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = (rank + 1) * _make_sparse(dtype)
+        out_arrays = []
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        comm.gather(in_array, out_arrays, root)
+        if rank == root:
+            expected = [_make_sparse(dtype), 2 * _make_sparse(dtype)]
+            testing.assert_allclose(
+                out_arrays[0].todense(), expected[0].todense())
+            testing.assert_allclose(
+                out_arrays[1].todense(), expected[1].todense())
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_gather(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_gather(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_gather, (0, dtype))
+        _launch_workers(run_gather, (1, dtype))
+
+
+def sparse_all_gather(dtype, use_mpi=False):
+
+    def run_all_gather(rank, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_array = (rank + 1) * _make_sparse(dtype)
+        out_arrays = []
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        comm.all_gather(in_array, out_arrays, 0)
+        expected = [_make_sparse(dtype), 2 * _make_sparse(dtype)]
+        testing.assert_allclose(
+            out_arrays[0].todense(), expected[0].todense())
+        testing.assert_allclose(
+            out_arrays[1].todense(), expected[1].todense())
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_all_gather(MPI.COMM_WORLD.Get_rank(), dtype, True)
+        run_all_gather(MPI.COMM_WORLD.Get_rank(), dtype, True)
+    else:
+        _launch_workers(run_all_gather, (dtype,))
+        _launch_workers(run_all_gather, (dtype,))
+
+
+def sparse_all_to_all(dtype, use_mpi=False):
+
+    def run_all_to_all(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_arrays = [_make_sparse(dtype), 2 * _make_sparse(dtype)]
+        out_array = []
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        comm.all_to_all(in_arrays, out_array)
+        testing.assert_allclose(
+            out_array[0].todense(), (rank + 1) * in_arrays[0].todense())
+        testing.assert_allclose(
+            out_array[1].todense(), (rank + 1) * in_arrays[0].todense())
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_all_to_all(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_all_to_all(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_all_to_all, (0, dtype))
+        _launch_workers(run_all_to_all, (1, dtype))
+
+
+def sparse_reduce_scatter(dtype, use_mpi=False):
+
+    def run_reduce_scatter(rank, root, dtype, use_mpi=False):
+        dev = cuda.Device(rank)
+        dev.use()
+        comm = NCCLBackend(N_WORKERS, rank, use_mpi=use_mpi)
+        in_arrays = [(rank + 1) * _make_sparse(dtype),
+                     (rank + 2) * _make_sparse(dtype)]
+        out_array = _make_sparse_empty(dtype)
+        warnings.filterwarnings(
+            'ignore', '.*transferring sparse.*', UserWarning)
+        comm.reduce_scatter(in_arrays, out_array, 2)
+        target = ((rank + 1) * _make_sparse(dtype)
+                  + (rank + 2) * _make_sparse(dtype))
+        testing.assert_allclose(
+            out_array.todense(), target.todense())
+
+    if use_mpi:
+        from mpi4py import MPI
+        # This process was run with mpiexec
+        run_reduce_scatter(MPI.COMM_WORLD.Get_rank(), 0, dtype, True)
+        run_reduce_scatter(MPI.COMM_WORLD.Get_rank(), 1, dtype, True)
+    else:
+        _launch_workers(run_reduce_scatter, (0, dtype))
+        _launch_workers(run_reduce_scatter, (1, dtype))
+
+
+if __name__ == '__main__':
+    # Run the templatized test
+    func = globals()[sys.argv[1]]
+    # dtype is the char representation
+    use_mpi = True if sys.argv[2] == "mpi" else False
+    dtype = sys.argv[3] if len(sys.argv) == 4 else None
+    if dtype is not None:
+        func(dtype, use_mpi)
+    else:
+        func(use_mpi)
diff --git a/tests/cupyx_tests/distributed_tests/test_array.py b/tests/cupyx_tests/distributed_tests/test_array.py
new file mode 100644
index 0000000..46562e4
--- /dev/null
+++ b/tests/cupyx_tests/distributed_tests/test_array.py
@@ -0,0 +1,106 @@
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx.distributed._array
+
+
+@pytest.fixture
+def mem_pool():
+    try:
+        old_pool = cupy.get_default_memory_pool()
+        pool = cupy.cuda.memory.MemoryPool()
+        cupy.cuda.memory.set_allocator(pool.malloc)
+        yield pool
+    finally:
+        pool.set_limit(size=0)
+        pool.free_all_blocks()
+        cupy.cuda.memory.set_allocator(old_pool.malloc)
+
+
+@testing.multi_gpu(2)
+class TestDistributedArray:
+    def test_array_creation_from_numpy(self, mem_pool):
+        array = numpy.arange(64, dtype='q').reshape(8, 8)
+        assert mem_pool.used_bytes() == 0
+        da = cupyx.distributed._array.array(array, (0, 1), (4, 8))
+        assert da.device.id == -1
+        # Ensure no memory allocation other than the chunks
+        assert da.data.ptr == 0
+        assert da.shape == (8, 8)
+        assert mem_pool.used_bytes() == array.nbytes
+        testing.assert_array_equal(da._chunks[0], array[:4, :])
+        testing.assert_array_equal(da._chunks[1], array[4:, :])
+
+    def test_array_creation_from_cupy(self, mem_pool):
+        array = cupy.arange(64, dtype='q').reshape(8, 8)
+        assert mem_pool.used_bytes() == array.nbytes
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', cupy._util.PerformanceWarning)
+            da = cupyx.distributed._array.array(array, (0, 1), (4, 8))
+        assert da.device.id == -1
+        # Ensure no memory allocation other than chunks & original array
+        assert da.data.ptr == 0
+        assert da.shape == (8, 8)
+        assert mem_pool.used_bytes() == 2 * array.nbytes
+        testing.assert_array_equal(da._chunks[0], array[:4, :])
+        testing.assert_array_equal(da._chunks[1], array[4:, :])
+
+    def test_invalid_array_creation(self):
+        with pytest.raises(TypeError, match=r'numpy or cupy'):
+            cupyx.distributed._array.array([1, 2, 3], (0, 1), (2, 8))
+
+    def test_array_creation_invalid_splits(self):
+        array = cupy.arange(64).reshape(8, 8)
+        with pytest.raises(RuntimeError, match=r'single axis'):
+            cupyx.distributed._array.array(array, (0, 1), (4, 4))
+
+    def test_array_creation_more_chunks_than_devices(self):
+        array = cupy.arange(64).reshape(8, 8)
+        with pytest.raises(RuntimeError, match=r'amount of devices'):
+            cupyx.distributed._array.array(array, (0, 1), (2, 8))
+
+    @pytest.mark.parametrize('split_shape', [(8, 4), (4, 8)])
+    def test_ufuncs(self, split_shape):
+        # TODO parameterize split shape
+        np_a = numpy.arange(64).reshape(8, 8)
+        np_b = numpy.arange(64).reshape(8, 8) * 2
+        np_r = numpy.sin(np_a * np_b)
+        d_a = cupyx.distributed._array.array(np_a, (0, 1), split_shape)
+        d_b = cupyx.distributed._array.array(np_b, (0, 1), split_shape)
+        d_r = cupy.sin(d_a * d_b)
+        testing.assert_array_almost_equal(d_r.asnumpy(), np_r)
+
+    @pytest.mark.parametrize('split_shape', [(8, 4), (4, 8)])
+    def test_elementwise_kernel(self, split_shape):
+        custom_kernel = cupy.ElementwiseKernel(
+            'float32 x, float32 y',
+            'float32 z',
+            'z = (x - y) * (x - y)',
+            'custom')
+        # TODO parameterize split shape
+        np_a = numpy.arange(64).reshape(8, 8).astype(numpy.float32)
+        np_b = (numpy.arange(64).reshape(8, 8) * 2.0).astype(numpy.float32)
+        np_r = (np_a - np_b) * (np_a - np_b)
+        d_a = cupyx.distributed._array.array(np_a, (0, 1), split_shape)
+        d_b = cupyx.distributed._array.array(np_b, (0, 1), split_shape)
+        d_r = custom_kernel(d_a, d_b)
+        testing.assert_array_almost_equal(d_r.asnumpy(), np_r)
+
+    def test_incompatible_chunk_shapes(self):
+        np_a = numpy.arange(64).reshape(8, 8)
+        np_b = numpy.arange(64).reshape(8, 8) * 2
+        d_a = cupyx.distributed._array.array(np_a, (0, 1), (4, 8))
+        d_b = cupyx.distributed._array.array(np_b, (0, 1), (8, 4))
+        with pytest.raises(RuntimeError, match=r'chunk sizes'):
+            cupy.sin(d_a * d_b)
+
+    def test_incompatible_operand(self):
+        np_a = numpy.arange(64).reshape(8, 8)
+        cp_b = cupy.arange(64).reshape(8, 8)
+        d_a = cupyx.distributed._array.array(np_a, (0, 1), (4, 8))
+        with pytest.raises(RuntimeError, match=r'Mix `cupy.ndarray'):
+            cupy.sin(d_a * cp_b)
diff --git a/tests/cupyx_tests/distributed_tests/test_comm.py b/tests/cupyx_tests/distributed_tests/test_comm.py
new file mode 100644
index 0000000..49215e2
--- /dev/null
+++ b/tests/cupyx_tests/distributed_tests/test_comm.py
@@ -0,0 +1,189 @@
+import pathlib
+import subprocess
+import sys
+import unittest
+import os
+
+import numpy
+import pytest
+
+from cupy.cuda import nccl
+from cupy import testing
+
+from cupyx.distributed import init_process_group
+from cupyx.distributed._nccl_comm import _mpi_available
+
+
+nccl_available = nccl.available
+
+
+def _run_test(test_name, dtype=None):
+    # subprocess is required not to interfere with cupy module imported in top
+    # of this file
+    runner_path = pathlib.Path(__file__).parent / 'comm_runner.py'
+    args = [sys.executable, runner_path, test_name, 'store']
+    if dtype is not None:
+        args.append(numpy.dtype(dtype).char)
+    proc = subprocess.Popen(
+        args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    stdoutdata, stderrdata = proc.communicate()
+    assert stderrdata.decode() == ''
+    assert proc.returncode == 0
+
+
+def _run_test_with_mpi(test_name, dtype=None):
+    # subprocess is required not to interfere with cupy module imported in top
+    # of this file
+    runner_path = pathlib.Path(__file__).parent / 'comm_runner.py'
+    args = ['mpiexec', '-n', '2', '--allow-run-as-root',
+            sys.executable, runner_path, test_name, 'mpi']
+    if dtype is not None:
+        args.append(numpy.dtype(dtype).char)
+    proc = subprocess.Popen(
+        args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=os.environ
+    )
+    stdoutdata, stderrdata = proc.communicate()
+    assert stderrdata.decode() == ''
+    assert proc.returncode == 0
+
+
+@pytest.mark.skipif(not nccl_available, reason='nccl is not installed')
+@testing.multi_gpu(2)
+class TestNCCLBackend:
+    def _run_test(self, test, dtype):
+        _run_test(test, dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_broadcast(self, dtype):
+        self._run_test('broadcast', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_reduce(self, dtype):
+        self._run_test('reduce', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_all_reduce(self, dtype):
+        self._run_test('all_reduce', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_reduce_scatter(self, dtype):
+        self._run_test('reduce_scatter', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_all_gather(self, dtype):
+        self._run_test('all_gather', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_send_and_recv(self, dtype):
+        self._run_test('send_and_recv', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_send_recv(self, dtype):
+        self._run_test('send_recv', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_scatter(self, dtype):
+        self._run_test('scatter', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_gather(self, dtype):
+        self._run_test('gather', dtype)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_all_to_all(self, dtype):
+        self._run_test('all_to_all', dtype)
+
+    def test_barrier(self):
+        self._run_test('barrier', None)
+
+
+@pytest.mark.skipif(not _mpi_available, reason='mpi is not installed')
+@testing.multi_gpu(2)
+class TestNCCLBackendWithMPI(TestNCCLBackend):
+    def _run_test(self, test, dtype):
+        _run_test_with_mpi(test, dtype)
+
+
+@pytest.mark.skipif(not nccl_available, reason='nccl is not installed')
+@testing.multi_gpu(2)
+class TestNCCLBackendSparse:
+    def _run_test(self, test, dtype):
+        _run_test(test, dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_send_and_recv(self, dtype):
+        self._run_test('sparse_send_and_recv', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_broadcast(self, dtype):
+        self._run_test('sparse_broadcast', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_reduce(self, dtype):
+        self._run_test('sparse_reduce', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_all_reduce(self, dtype):
+        self._run_test('sparse_all_reduce', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_scatter(self, dtype):
+        self._run_test('sparse_scatter', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_gather(self, dtype):
+        self._run_test('sparse_gather', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_all_gather(self, dtype):
+        self._run_test('sparse_all_gather', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_all_to_all(self, dtype):
+        self._run_test('sparse_all_to_all', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_reduce_scatter(self, dtype):
+        self._run_test('sparse_reduce_scatter', dtype)
+
+    @testing.for_dtypes('fdFD')
+    def test_send_recv(self, dtype):
+        self._run_test('sparse_send_recv', dtype)
+
+
+@pytest.mark.skipif(not _mpi_available, reason='mpi is not installed')
+@testing.multi_gpu(2)
+class TestNCCLBackendSparseWithMPI(TestNCCLBackendSparse):
+    def _run_test(self, test, dtype):
+        _run_test_with_mpi(test, dtype)
+
+
+@pytest.mark.skipif(not nccl_available, reason='nccl is not installed')
+class TestInitDistributed(unittest.TestCase):
+
+    @testing.multi_gpu(2)
+    def test_init(self):
+        _run_test('init')
+
+    def test_invalid_backend(self):
+        with pytest.raises(ValueError):
+            init_process_group(1, 0, backend='mpi')
+
+    def test_invalid_n_devices(self):
+        with pytest.raises(ValueError):
+            init_process_group(0, 0)
+
+        with pytest.raises(ValueError):
+            init_process_group(-1, 0)
+
+    def test_invalid_rank(self):
+        with pytest.raises(ValueError):
+            init_process_group(2, -1)
+
+        with pytest.raises(ValueError):
+            init_process_group(2, 3)
diff --git a/tests/cupyx_tests/distributed_tests/test_store.py b/tests/cupyx_tests/distributed_tests/test_store.py
new file mode 100644
index 0000000..6d8a735
--- /dev/null
+++ b/tests/cupyx_tests/distributed_tests/test_store.py
@@ -0,0 +1,43 @@
+import unittest
+
+import pytest
+
+from cupy.cuda import nccl
+from cupy.testing import _condition
+from cupyx.distributed import _store
+
+
+nccl_available = nccl.available
+
+
+@pytest.mark.skipif(not nccl_available, reason='nccl is not installed')
+class TestTCPStore(unittest.TestCase):
+
+    @_condition.retry(10)
+    def test_store_get_set(self):
+        store = _store.TCPStore(1)
+        store.run()
+        try:
+            proxy = _store.TCPStoreProxy()
+            proxy['test-value'] = 1234
+            assert proxy['test-value'] == 1234
+            proxy['test-bytes'] = b'123\x00123'
+            assert proxy['test-bytes'] == b'123\x00123'
+        finally:
+            store.stop()
+
+    @_condition.retry(10)
+    def test_store_invalid_get_set(self):
+        store = _store.TCPStore(1)
+        try:
+            store.run()
+            proxy = _store.TCPStoreProxy()
+            with pytest.raises(ValueError):
+                proxy['test-value'] = 1234.0
+            with pytest.raises(ValueError):
+                proxy[123] = 1234
+            with pytest.raises(ValueError):
+                a = proxy[123]  # NOQA
+        finally:
+            store.stop()
+    # Barrier is tested directly in the communicators
diff --git a/tests/cupyx_tests/fallback_mode_tests/__init__.py b/tests/cupyx_tests/fallback_mode_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/fallback_mode_tests/test_fallback.py b/tests/cupyx_tests/fallback_mode_tests/test_fallback.py
new file mode 100644
index 0000000..f784704
--- /dev/null
+++ b/tests/cupyx_tests/fallback_mode_tests/test_fallback.py
@@ -0,0 +1,657 @@
+import functools
+import operator
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupyx import fallback_mode
+from cupyx.fallback_mode import fallback
+from cupyx.fallback_mode.notification import FallbackWarning
+
+
+ignore_fallback_warnings = pytest.mark.filterwarnings(
+    "ignore", category=FallbackWarning)
+
+
+def numpy_fallback_equal(name='xp'):
+    """
+    Decorator that checks fallback_mode results are equal to NumPy ones.
+    Checks results that are non-ndarray.
+
+    Args:
+        name(str): Argument name whose value is either
+        ``numpy`` or ``cupy`` module.
+    """
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kwargs):
+
+            kwargs[name] = fallback_mode.numpy
+            fallback_result = impl(self, *args, **kwargs)
+
+            kwargs[name] = numpy
+            numpy_result = impl(self, *args, **kwargs)
+
+            assert numpy_result == fallback_result
+
+        return test_func
+    return decorator
+
+
+def numpy_fallback_array_equal(name='xp'):
+    """
+    Decorator that checks fallback_mode results are equal to NumPy ones.
+    Checks ndarrays.
+
+    Args:
+        name(str): Argument name whose value is either
+        ``numpy`` or ``cupy`` module.
+    """
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kwargs):
+
+            kwargs[name] = fallback_mode.numpy
+            fallback_result = impl(self, *args, **kwargs)
+
+            kwargs[name] = numpy
+            numpy_result = impl(self, *args, **kwargs)
+
+            if isinstance(numpy_result, numpy.ndarray):
+                # if numpy returns ndarray, cupy must return ndarray
+                assert isinstance(fallback_result, fallback.ndarray)
+
+                fallback_mode.numpy.testing.assert_array_equal(
+                    numpy_result, fallback_result)
+
+                assert fallback_result.dtype == numpy_result.dtype
+
+            elif isinstance(numpy_result, numpy.ScalarType):
+                # if numpy returns scalar
+                # cupy may return 0-dim array
+                assert numpy_result == fallback_result._cupy_array.item() or \
+                    (numpy_result == fallback_result._numpy_array).all()
+
+            else:
+                assert False
+
+        return test_func
+    return decorator
+
+
+def numpy_fallback_array_allclose(name='xp', rtol=1e-07):
+    """
+    Decorator that checks fallback_mode results are almost equal to NumPy ones.
+    Checks ndarrays.
+
+    Args:
+        name(str): Argument name whose value is either
+        ``numpy`` or ``cupy`` module.
+    """
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kwargs):
+
+            kwargs[name] = fallback_mode.numpy
+            fallback_result = impl(self, *args, **kwargs)
+
+            kwargs[name] = numpy
+            numpy_result = impl(self, *args, **kwargs)
+
+            assert isinstance(fallback_result, fallback.ndarray)
+
+            fallback_mode.numpy.testing.assert_allclose(
+                numpy_result, fallback_result, rtol=rtol)
+
+            assert fallback_result.dtype == numpy_result.dtype
+
+        return test_func
+    return decorator
+
+
+def enable_slice_copy(func):
+    """
+    Decorator that enables CUPY_EXPERIMENTAL_SLICE_COPY.
+    And then restores it to previous state.
+    """
+    def decorator(*args, **kwargs):
+        old = cupy._util.ENABLE_SLICE_COPY
+        cupy._util.ENABLE_SLICE_COPY = True
+        func(*args, **kwargs)
+        cupy._util.ENABLE_SLICE_COPY = old
+
+    return decorator
+
+
+def get_numpy_version():
+    v = numpy.lib.NumpyVersion(numpy.__version__)
+    return (v.major, v.minor, v.bugfix)
+
+
+@ignore_fallback_warnings
+class TestFallbackMode(unittest.TestCase):
+
+    def test_module_not_callable(self):
+
+        pytest.raises(TypeError, fallback_mode.numpy)
+
+        pytest.raises(TypeError, fallback_mode.numpy.linalg)
+
+    def test_numpy_scalars(self):
+
+        assert fallback_mode.numpy.inf is numpy.inf
+
+        assert fallback_mode.numpy.pi is numpy.pi
+
+        # True, because 'is' checks for reference
+        # fallback_mode.numpy.nan and numpy.nan both have same reference
+        assert fallback_mode.numpy.nan is numpy.nan
+
+    def test_cupy_specific_func(self):
+
+        with pytest.raises(AttributeError):
+            func = fallback_mode.numpy.ElementwiseKernel  # NOQA
+
+    def test_func_not_in_numpy(self):
+
+        with pytest.raises(AttributeError):
+            func = fallback_mode.numpy.dummy  # NOQA
+
+    def test_same_reference(self):
+
+        assert fallback_mode.numpy.int64 is numpy.int64
+
+        assert fallback_mode.numpy.float32 is numpy.float32
+
+    def test_isinstance(self):
+
+        a = fallback_mode.numpy.float64(3)
+        assert isinstance(a, fallback_mode.numpy.float64)
+
+        abs = fallback_mode.numpy.vectorize(fallback_mode.numpy.abs)
+        assert isinstance(abs, fallback_mode.numpy.vectorize)
+
+        date = fallback_mode.numpy.datetime64('2019-07-18')
+        assert isinstance(date, fallback_mode.numpy.datetime64)
+
+
+@testing.parameterize(
+    {'func': 'min', 'shape': (3, 4), 'args': (), 'kwargs': {'axis': 0}},
+    {'func': 'argmin', 'shape': (3, 4), 'args': (), 'kwargs': {}},
+    {'func': 'roots', 'shape': (3,), 'args': (), 'kwargs': {}},
+    {'func': 'resize', 'shape': (2, 6), 'args': ((6, 2),), 'kwargs': {}},
+    {'func': 'resize', 'shape': (3, 4), 'args': ((4, 9),), 'kwargs': {}},
+    {'func': 'delete', 'shape': (5, 4), 'args': (1, 0), 'kwargs': {}},
+    {'func': 'append', 'shape': (2, 3), 'args': ([[7, 8, 9]],),
+     'kwargs': {'axis': 0}},
+    {'func': 'asarray_chkfinite', 'shape': (2, 4), 'args': (),
+     'kwargs': {'dtype': numpy.float64}}
+)
+@ignore_fallback_warnings
+class TestFallbackMethodsArrayExternal(unittest.TestCase):
+
+    @numpy_fallback_array_equal()
+    def test_fallback_methods_array_external(self, xp):
+
+        a = testing.shaped_random(self.shape, xp=xp, dtype=numpy.int64)
+        return getattr(xp, self.func)(a, *self.args, **self.kwargs)
+
+
+@testing.parameterize(
+    {'func': 'min', 'shape': (3, 4), 'args': (), 'kwargs': {'axis': 0},
+     'numpy_version': None},
+    {'func': 'argmin', 'shape': (3, 4), 'args': (), 'kwargs': {},
+     'numpy_version': (1, 10, 0)},
+    {'func': 'arccos', 'shape': (2, 3), 'args': (), 'kwargs': {},
+     'numpy_version': None},
+    {'func': 'fabs', 'shape': (2, 3), 'args': (), 'kwargs': {},
+     'numpy_version': None},
+    {'func': 'nancumsum', 'shape': (5, 3), 'args': (), 'kwargs': {'axis': 1},
+     'numpy_version': (1, 12, 0)},
+    {'func': 'nanpercentile', 'shape': (3, 4), 'args': (50,),
+     'kwargs': {'axis': 0}, 'numpy_version': None}
+)
+@ignore_fallback_warnings
+class TestFallbackMethodsArrayExternalOut(unittest.TestCase):
+
+    @numpy_fallback_array_equal()
+    def test_fallback_methods_array_external_out(self, xp):
+        if self.numpy_version and get_numpy_version() < self.numpy_version:
+            self.skipTest('Test not supported for this version of numpy')
+
+        a = testing.shaped_random(self.shape, xp=xp)
+        kwargs = self.kwargs.copy()
+        res = getattr(xp, self.func)(a, *self.args, **kwargs)
+
+        # to get the shape of out
+        out = xp.zeros(res.shape, dtype=res.dtype)
+        kwargs['out'] = out
+        getattr(xp, self.func)(a, *self.args, **kwargs)
+        return out
+
+
+@testing.parameterize(
+    {'object': 'ndarray'},
+    {'object': 'ndarray.__add__'},
+    {'object': 'vectorize'},
+    {'object': 'linalg.eig'},
+)
+class TestDocs(unittest.TestCase):
+
+    @numpy_fallback_equal()
+    def test_docs(self, xp):
+        return operator.attrgetter(self.object)(xp).__doc__
+
+
+class TestFallbackArray(unittest.TestCase):
+
+    def test_ndarray_creation_compatible(self):
+
+        a = fallback_mode.numpy.array([[1, 2], [3, 4]])
+        assert isinstance(a, fallback.ndarray)
+        assert a._supports_cupy
+
+        b = fallback_mode.numpy.arange(9)
+        assert isinstance(b, fallback.ndarray)
+        assert a._supports_cupy
+
+    def test_ndarray_creation_not_compatible(self):
+
+        a = fallback_mode.numpy.array([1, 2, 3], dtype=object)
+        assert isinstance(a, fallback.ndarray)
+        assert not a._supports_cupy
+
+        b = fallback_mode.numpy.array(['a', 'b', 'c', 'd'], dtype='|S1')
+        assert isinstance(b, fallback.ndarray)
+        assert not b._supports_cupy
+
+        # Structured array will automatically be _numpy_array
+        c = fallback_mode.numpy.array(
+            [('Rex', 9, 81.0), ('Fido', 3, 27.0)],
+            dtype=[('name', 'U10'), ('age', 'i4'), ('weight', 'f4')])
+
+        assert isinstance(c, fallback.ndarray)
+        assert not c._supports_cupy
+
+    def test_getitem(self):
+
+        x = fallback_mode.numpy.array([1, 2, 3])
+
+        # single element
+        assert int(x[2]) == 3
+
+        # slicing
+        res = cupy.array([1, 2, 3])
+        testing.assert_array_equal(x[:2]._cupy_array, res[:2])
+
+    def test_setitem(self):
+
+        x = fallback_mode.numpy.array([1, 2, 3])
+
+        # single element
+        x[2] = 99
+        res = cupy.array([1, 2, 99])
+        testing.assert_array_equal(x._cupy_array, res)
+
+        # slicing
+        y = fallback_mode.numpy.array([11, 22])
+        x[:2] = y
+        res = cupy.array([11, 22, 99])
+        testing.assert_array_equal(x._cupy_array, res)
+
+    @numpy_fallback_equal()
+    def test_ndarray_shape(self, xp):
+
+        x = xp.arange(20)
+        x = x.reshape(4, 5)
+
+        return x.shape
+
+    @numpy_fallback_equal()
+    def test_ndarray_init(self, xp):
+        a = xp.ndarray((3, 4))
+        return a.shape
+
+    @numpy_fallback_equal()
+    def test_ndarray_shape_creation(self, xp):
+        a = xp.ndarray((4, 5))
+        return a.shape
+
+    def test_instancecheck_ndarray(self):
+
+        a = fallback_mode.numpy.array([1, 2, 3])
+        assert isinstance(a, fallback_mode.numpy.ndarray)
+
+        b = fallback_mode.numpy.ndarray((2, 3))
+        assert isinstance(b, fallback_mode.numpy.ndarray)
+
+    def test_instancecheck_type(self):
+        a = fallback_mode.numpy.arange(3)
+        assert isinstance(a, type(a))
+
+    @numpy_fallback_equal()
+    def test_type_call(self, xp):
+        a = xp.array([1])
+        b = type(a)((2, 3))
+        return b.shape
+
+    @numpy_fallback_equal()
+    def test_type_assert(self, xp):
+        a = xp.array([1, 2, 3])
+        return type(a) == xp.ndarray
+
+    @numpy_fallback_equal()
+    def test_base(self, xp):
+        a = xp.arange(7)
+        b = a[2:]
+        return b.base is a
+
+
+@testing.parameterize(
+    {'func': 'min', 'shape': (5,), 'args': (), 'kwargs': {}},
+    {'func': 'argmax', 'shape': (5, 3), 'args': (), 'kwargs': {'axis': 0}},
+    {'func': 'ptp', 'shape': (3, 3), 'args': (), 'kwargs': {'axis': 1}},
+    {'func': 'compress', 'shape': (3, 2), 'args': ([False, True],),
+     'kwargs': {'axis': 0}}
+)
+class TestFallbackArrayMethodsInternal(unittest.TestCase):
+
+    @numpy_fallback_array_equal()
+    def test_fallback_array_methods_internal(self, xp):
+
+        a = testing.shaped_random(self.shape, xp=xp)
+        return getattr(a, self.func)(*self.args, **self.kwargs)
+
+    @numpy_fallback_array_equal()
+    def test_fallback_array_methods_internal_out(self, xp):
+
+        a = testing.shaped_random(self.shape, xp=xp)
+        kwargs = self.kwargs.copy()
+        res = getattr(a, self.func)(*self.args, **kwargs)
+
+        # to get the shape of out
+        out = xp.zeros(res.shape, dtype=res.dtype)
+        kwargs['out'] = out
+        getattr(a, self.func)(*self.args, **kwargs)
+        return out
+
+
+@testing.parameterize(
+    {'func': '__eq__', 'shape': (3, 4)},
+    {'func': '__ne__', 'shape': (3, 1)},
+    {'func': '__gt__', 'shape': (4,)},
+    {'func': '__lt__', 'shape': (1, 1)},
+    {'func': '__ge__', 'shape': (1, 2)},
+    {'func': '__le__', 'shape': (1,)}
+)
+class TestArrayComparison(unittest.TestCase):
+
+    @numpy_fallback_array_equal()
+    def test_ndarray_comparison(self, xp):
+
+        a = testing.shaped_random(self.shape, xp=xp)
+        b = testing.shaped_random(self.shape, xp=xp, seed=3)
+
+        return getattr(a, self.func)(b)
+
+
+@testing.parameterize(
+    {'func': '__str__', 'shape': (5, 6)},
+    {'func': '__repr__', 'shape': (3, 4)},
+    {'func': '__int__', 'shape': ()},
+    {'func': '__float__', 'shape': ()},
+    {'func': '__len__', 'shape': (3, 3)},
+    {'func': '__bool__', 'shape': ()},
+)
+class TestArrayUnaryMethods(unittest.TestCase):
+
+    @numpy_fallback_equal()
+    def test_unary_methods(self, xp):
+        a = testing.shaped_random(self.shape, xp=xp)
+        return getattr(a, self.func)()
+
+
+@testing.parameterize(
+    {'func': '__abs__', 'shape': (5, 6), 'dtype': numpy.float32},
+    {'func': '__copy__', 'shape': (3, 4), 'dtype': numpy.float32},
+    {'func': '__neg__', 'shape': (3, 3), 'dtype': numpy.float32},
+    {'func': '__invert__', 'shape': (2, 4), 'dtype': numpy.int32}
+)
+class TestArrayUnaryMethodsArray(unittest.TestCase):
+
+    @numpy_fallback_array_equal()
+    def test_unary_methods_array(self, xp):
+
+        a = testing.shaped_random(self.shape, xp=xp, dtype=self.dtype)
+
+        return getattr(a, self.func)()
+
+
+@testing.parameterize(
+    {'func': '__add__', 'shape': (3, 4), 'dtype': numpy.float32},
+    {'func': '__sub__', 'shape': (2, 2), 'dtype': numpy.float32},
+    {'func': '__mul__', 'shape': (5, 6), 'dtype': numpy.float32},
+    {'func': '__mod__', 'shape': (3, 4), 'dtype': numpy.float32},
+    {'func': '__iadd__', 'shape': (1,), 'dtype': numpy.float32},
+    {'func': '__imul__', 'shape': (1, 1), 'dtype': numpy.float32},
+    {'func': '__and__', 'shape': (3, 3), 'dtype': numpy.int32},
+    {'func': '__ipow__', 'shape': (4, 5), 'dtype': numpy.int32},
+    {'func': '__xor__', 'shape': (4, 4), 'dtype': numpy.int32},
+    {'func': '__lshift__', 'shape': (2,), 'dtype': numpy.int32},
+    {'func': '__irshift__', 'shape': (3, 2), 'dtype': numpy.int32},
+)
+class TestArrayArithmeticMethods(unittest.TestCase):
+
+    @numpy_fallback_array_allclose(rtol=1e-6)
+    def test_arithmetic_methods(self, xp):
+        a = testing.shaped_random(self.shape, xp=xp, dtype=self.dtype)
+        b = testing.shaped_random(self.shape, xp=xp, dtype=self.dtype, seed=5)
+        return getattr(a, self.func)(b)
+
+
+class TestArrayMatmul(unittest.TestCase):
+
+    @testing.with_requires('numpy>=1.16')
+    @numpy_fallback_array_allclose(rtol=1e-05)
+    def test_mm_matmul(self, xp):
+        a = testing.shaped_random((4, 5), xp)
+        b = testing.shaped_random((5, 3), xp, seed=5)
+
+        return a.__matmul__(b)
+
+
+class TestVectorizeWrapper(unittest.TestCase):
+
+    @numpy_fallback_array_equal()
+    def test_pyfunc_custom_list(self, xp):
+
+        def function(a, b):
+            if a > b:
+                return a - b
+            return a + b
+
+        return xp.vectorize(function)([1, 2, 3, 4], 2)
+
+    @numpy_fallback_array_equal()
+    def test_pyfunc_builtin(self, xp):
+        a = testing.shaped_random((4, 5), xp)
+        vabs = xp.vectorize(abs)
+        return vabs(a)
+
+    @numpy_fallback_array_equal()
+    def test_pyfunc_numpy(self, xp):
+        a = testing.shaped_random((4, 5), xp)
+        vabs = xp.vectorize(numpy.abs)
+        return vabs(a)
+
+    @numpy_fallback_equal()
+    def test_getattr(self, xp):
+        vabs = xp.vectorize(numpy.abs)
+        return vabs.pyfunc
+
+    @numpy_fallback_array_equal()
+    def test_setattr(self, xp):
+        a = xp.array([-1, 2, -3])
+        vabs = xp.vectorize(abs)
+        vabs.otypes = ['float']
+        return vabs(a)
+
+    @numpy_fallback_equal()
+    def test_doc(self, xp):
+        vabs = xp.vectorize(abs)
+        return vabs.__doc__
+
+
+@ignore_fallback_warnings
+class TestInplaceSpecialMethods(unittest.TestCase):
+
+    @numpy_fallback_array_equal()
+    def test_resize_internal(self, xp):
+        a = testing.shaped_random((3, 4), xp)
+        a.resize(4, 5, refcheck=False)
+        return a
+
+    @numpy_fallback_array_equal()
+    def test_ndarray_byteswap(self, xp):
+        a = testing.shaped_random((4,), xp, dtype=xp.int16)
+        return a.byteswap()
+
+    @unittest.skipIf(get_numpy_version() < (1, 13, 0),
+                     'inplace kwarg for byteswap was added in numpy v1.13.0')
+    @numpy_fallback_array_equal()
+    def test_ndarray_byteswap_inplace(self, xp):
+        a = testing.shaped_random((4,), xp, dtype=xp.int16)
+        a.byteswap(inplace=True)
+        return a
+
+    @numpy_fallback_array_equal()
+    def test_putmask(self, xp):
+        a = testing.shaped_random((3, 4), xp, dtype=xp.int8)
+        xp.putmask(a, a > 2, a**2)
+        return a
+
+    @unittest.skipIf(get_numpy_version() < (1, 15, 0),
+                     'put_along_axis introduced in numpy v1.15.0')
+    @numpy_fallback_array_equal()
+    def test_put_along_axis(self, xp):
+        a = xp.array([[10, 30, 20], [60, 40, 50]])
+        ai = xp.expand_dims(xp.argmax(a, axis=1), axis=1)
+        xp.put_along_axis(a, ai, 99, axis=1)
+        return a
+
+    @unittest.skipIf(get_numpy_version() < (1, 15, 0),
+                     'quantile introduced in numpy v1.15.0')
+    @numpy_fallback_array_equal()
+    def test_out_is_returned_when_fallbacked(self, xp):
+        a = testing.shaped_random((3, 4), xp)
+        z = xp.zeros((4, ))
+        res = xp.quantile(a, 0.5, axis=0, out=z)
+        assert res is z
+        return res
+
+    @numpy_fallback_array_allclose()
+    def test_out_is_returned_when_not_fallbacked(self, xp):
+        a = testing.shaped_random((3, 4), xp, dtype=xp.float64)
+        z = xp.zeros((4,))
+        res = xp.var(a, axis=0, out=z)
+        assert res is z
+        return res
+
+
+@ignore_fallback_warnings
+class TestArrayVariants(unittest.TestCase):
+
+    @numpy_fallback_array_equal()
+    def test_creation_masked(self, xp):
+        mx = xp.ma.array([1, 2, 3, 4], mask=[1, 0, 1, 0])
+        return mx
+
+    @numpy_fallback_equal()
+    def test_method_internal(self, xp):
+        mx = xp.ma.array([1, 2, 3, 4], mask=[1, 0, 1, 0])
+        return mx.min()
+
+    @numpy_fallback_equal()
+    def test_method_internal_not_callable(self, xp):
+        mx = xp.ma.array([1, 2, 3, 4], mask=[1, 0, 1, 0])
+        return mx.shape
+
+    @numpy_fallback_equal()
+    def test_method_external_masked(self, xp):
+        mx = xp.ma.array([1, 2, 3, 4], mask=[1, 0, 1, 0])
+        return xp.mean(mx)
+
+    @numpy_fallback_array_equal()
+    def test_magic_method_masked(self, xp):
+        mx = xp.ma.array([1, 2, 3, 4], mask=[1, 0, 1, 0])
+        my = xp.ma.array([4, 2, 3, 1], mask=[1, 0, 1, 0])
+        return mx >= my
+
+    @numpy_fallback_array_equal()
+    def test_creation_char(self, xp):
+        cx = xp.char.array(['a', 'b', 'c'], itemsize=3)
+        return cx
+
+    @numpy_fallback_array_equal()
+    def test_method_external_char(self, xp):
+        cx = xp.char.array(['a', 'b', 'c'], itemsize=3)
+        cy = xp.char.array(['a', 'b', 'c'], itemsize=3)
+        return xp.char.add(cx, cy)
+
+    @numpy_fallback_array_equal()
+    def test_magic_method_char(self, xp):
+        cx = xp.char.array(['a', 'b', 'c'], itemsize=3)
+        cy = xp.char.array(['a', 'b', 'c'], itemsize=3)
+        return cx == cy
+
+    @numpy_fallback_array_equal()
+    def test_inplace(self, xp):
+        x = xp.arange(12).reshape((3, 4))
+        mask = xp.zeros_like(x)
+        mask[0, :] = 1
+        mx = xp.ma.array(x, mask=mask)
+        z = xp.ma.zeros((4,))
+        xp.nanmean(mx, axis=0, out=z)
+        return z
+
+    @numpy_fallback_array_equal()
+    def test_matrix_returned(self, xp):
+        x = testing.shaped_random((2, 3), xp=xp)
+        y = xp.asmatrix(x)
+
+        if xp is fallback_mode.numpy:
+            assert x._supports_cupy
+            assert isinstance(y, fallback.ndarray)
+            assert not y._supports_cupy
+            assert y._numpy_array.__class__ is numpy.matrix
+
+        return y
+
+    @numpy_fallback_array_equal()
+    def test_record_array(self, xp):
+        ra = xp.rec.array([1, 2, 3])
+        return ra
+
+    # changes in MaskedArray should be reflected in base ndarray
+    @numpy_fallback_array_equal()
+    def test_ma_func(self, xp):
+        x = xp.array([1, 2, 3, 4])
+        x += x
+        mx = xp.ma.array(x, mask=[1, 0, 1, 0])
+        assert mx.base is x
+        mx += mx
+        return x
+
+    # changes in base ndarray should be reflected in MaskedArray
+    @enable_slice_copy
+    @numpy_fallback_array_equal()
+    def test_ma_func_inverse(self, xp):
+        x = xp.array([1, 2, 3, 4])
+        mx = xp.ma.array(x, mask=[1, 0, 1, 0])
+        assert mx.base is x
+        mx += mx
+        x += x
+        return mx
diff --git a/tests/cupyx_tests/fallback_mode_tests/test_notifications.py b/tests/cupyx_tests/fallback_mode_tests/test_notifications.py
new file mode 100644
index 0000000..18ef61a
--- /dev/null
+++ b/tests/cupyx_tests/fallback_mode_tests/test_notifications.py
@@ -0,0 +1,172 @@
+import contextlib
+import io
+
+import numpy
+import pytest
+
+from cupy import testing
+from cupyx import fallback_mode
+from cupyx import _ufunc_config
+from cupyx_tests.fallback_mode_tests import test_fallback as test_utils
+
+
+class NotificationTestBase:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        old_config = _ufunc_config.geterr()
+        yield
+        _ufunc_config.seterr(**old_config)
+
+
+class TestNotifications(NotificationTestBase):
+
+    def test_seterr_geterr(self):
+
+        default = _ufunc_config.geterr()
+        assert default['fallback_mode'] == 'ignore'
+
+        old = _ufunc_config.seterr(fallback_mode='warn')
+        current = _ufunc_config.geterr()
+        assert old['fallback_mode'] == 'ignore'
+        assert current['fallback_mode'] == 'warn'
+        _ufunc_config.seterr(**old)
+
+    def test_errstate(self):
+
+        old = _ufunc_config.seterr(fallback_mode='print')
+        before = _ufunc_config.geterr()
+
+        with _ufunc_config.errstate(fallback_mode='raise'):
+            inside = _ufunc_config.geterr()
+            assert inside['fallback_mode'] == 'raise'
+
+        after = _ufunc_config.geterr()
+        assert before['fallback_mode'] == after['fallback_mode']
+        _ufunc_config.seterr(**old)
+
+
+@testing.parameterize(
+    {'func_name': 'get_include'},
+)
+class TestNotificationModes(NotificationTestBase):
+
+    @property
+    def func(self):
+        if self.func_name == 'get_include':
+            return fallback_mode.numpy.get_include
+        assert False
+
+    def test_notification_ignore(self):
+
+        old = _ufunc_config.seterr(fallback_mode='ignore')
+        saved_stdout = io.StringIO()
+
+        with contextlib.redirect_stdout(saved_stdout):
+            self.func()
+
+        _ufunc_config.seterr(**old)
+        output = saved_stdout.getvalue().strip()
+        assert output == ""
+
+    def test_notification_print(self):
+
+        old = _ufunc_config.seterr(fallback_mode='print')
+        saved_stdout = io.StringIO()
+
+        with contextlib.redirect_stdout(saved_stdout):
+            self.func()
+
+        _ufunc_config.seterr(**old)
+        nf = self.func._numpy_object
+        output = saved_stdout.getvalue().strip()
+        msg1 = "'{}' method not in cupy, ".format(nf.__name__)
+        msg2 = "falling back to '{}.{}'".format(nf.__module__, nf.__name__)
+        assert output == ("Warning: " + msg1 + msg2)
+
+    def test_notification_warn(self):
+
+        _ufunc_config.seterr(fallback_mode='warn')
+
+        with pytest.warns(fallback_mode.notification.FallbackWarning):
+            self.func()
+
+    def test_notification_raise(self):
+
+        old = _ufunc_config.seterr(fallback_mode='raise')
+
+        with pytest.raises(AttributeError):
+            self.func()
+
+        _ufunc_config.seterr(**old)
+
+
+class TestNotificationVectorize(NotificationTestBase):
+
+    @test_utils.enable_slice_copy
+    def test_custom_or_builtin_pyfunc(self):
+
+        old = _ufunc_config.seterr(fallback_mode='print')
+        saved_stdout = io.StringIO()
+
+        with contextlib.redirect_stdout(saved_stdout):
+            def custom_abs(x):
+                if x >= 0:
+                    return x
+                return -x
+
+            a = testing.shaped_random((3, 4), fallback_mode.numpy)
+            vec_abs = fallback_mode.numpy.vectorize(custom_abs)
+            vec_abs(a)
+
+            # built-in
+            vec_abs = fallback_mode.numpy.vectorize(abs)
+            vec_abs(a)
+
+        _ufunc_config.seterr(**old)
+        output = saved_stdout.getvalue().strip()
+        msg = "'vectorize' method not in cupy, "
+        msg += "falling back to '"
+        msg += numpy.vectorize.__module__ + ".vectorize'"
+        assert output == ("Warning: " + msg + "\nWarning: " + msg)
+
+    @test_utils.enable_slice_copy
+    def test_cupy_supported_pyfunc(self):
+
+        old = _ufunc_config.seterr(fallback_mode='print')
+        saved_stdout = io.StringIO()
+
+        with contextlib.redirect_stdout(saved_stdout):
+            a = testing.shaped_random((3, 4), fallback_mode.numpy)
+            vec_abs = fallback_mode.numpy.vectorize(fallback_mode.numpy.abs)
+            vec_abs(a)
+
+        _ufunc_config.seterr(**old)
+        output = saved_stdout.getvalue().strip()
+        msg1 = "'vectorize' method not in cupy, "
+        msg1 += "falling back to '"
+        msg1 += numpy.vectorize.__module__ + ".vectorize'"
+        msg2 = "'absolute' method is available in cupy but cannot be used, "
+        msg2 += "falling back to its numpy implementation"
+        assert output == ("Warning: " + msg1 + "\nWarning: " + msg2)
+
+    @pytest.mark.skip(reason='#6282 implemented cupy.fabs')
+    @test_utils.enable_slice_copy
+    def test_numpy_only_pyfunc(self):
+
+        old = _ufunc_config.seterr(fallback_mode='print')
+        saved_stdout = io.StringIO()
+
+        with contextlib.redirect_stdout(saved_stdout):
+            a = testing.shaped_random((3, 4), fallback_mode.numpy)
+            vec_abs = fallback_mode.numpy.vectorize(fallback_mode.numpy.fabs)
+            vec_abs(a)
+
+        _ufunc_config.seterr(**old)
+        output = saved_stdout.getvalue().strip()
+        msg1 = "'vectorize' method not in cupy, "
+        msg1 += "falling back to '"
+        msg1 += numpy.vectorize.__module__ + ".vectorize'"
+        msg2 = "'fabs' method not in cupy, "
+        msg2 += "falling back to its numpy implementation"
+        assert output == ("Warning: " + msg1 + "\nWarning: " + msg2)
diff --git a/tests/cupyx_tests/jit_tests/__init__.py b/tests/cupyx_tests/jit_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/jit_tests/test_cooperative_groups.py b/tests/cupyx_tests/jit_tests/test_cooperative_groups.py
new file mode 100644
index 0000000..60a4ec4
--- /dev/null
+++ b/tests/cupyx_tests/jit_tests/test_cooperative_groups.py
@@ -0,0 +1,185 @@
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupyx import jit
+
+
+@pytest.mark.skipif(runtime.is_hip, reason="not supported on HIP")
+class TestCooperativeGroups:
+
+    def test_thread_block_group(self):
+        @jit.rawkernel()
+        def test_thread_block(x):
+            g = jit.cg.this_thread_block()
+            if g.thread_rank() == 0:
+                x[0] += 101
+            if g.thread_rank() == 1:
+                x[1] = g.size()
+            # test dim3
+            if g.thread_rank() == 2:
+                g_idx = g.group_index()
+                x[2], x[3], x[4] = g_idx.x, g_idx.y, g_idx.z
+            if g.thread_rank() == 3:
+                t_idx = g.thread_index()
+                x[5], x[6], x[7] = t_idx.x, t_idx.y, t_idx.z
+            if g.thread_rank() == 4:
+                g_dim = g.group_dim()
+                x[8], x[9], x[10] = g_dim.x, g_dim.y, g_dim.z
+            g.sync()
+
+        x = cupy.empty((16,), dtype=cupy.int64)
+        x[:] = -1
+        test_thread_block[1, 32](x)
+        assert x[0] == 100
+        assert x[1] == 32
+        assert (x[2], x[3], x[4]) == (0, 0, 0)
+        assert (x[5], x[6], x[7]) == (3, 0, 0)
+        assert (x[8], x[9], x[10]) == (32, 1, 1)
+        assert (x[11:] == -1).all()
+
+    @pytest.mark.skipif(
+        runtime.runtimeGetVersion() < 11060
+        or (cupy.cuda.driver._is_cuda_python()
+            and cupy.cuda.nvrtc.getVersion() < (11, 6)),
+        reason='not supported until CUDA 11.6')
+    def test_thread_block_group_cu116_new_APIs(self):
+        @jit.rawkernel()
+        def test_thread_block(x):
+            g = jit.cg.this_thread_block()
+            if g.thread_rank() == 0:
+                x[0] = g.num_threads()
+            if g.thread_rank() == 1:
+                d_th = g.dim_threads()
+                x[1], x[2], x[3] = d_th.x, d_th.y, d_th.z
+            g.sync()
+
+        x = cupy.empty((16,), dtype=cupy.int64)
+        x[:] = -1
+        test_thread_block[1, 32](x)
+        assert x[0] == 32
+        assert (x[1], x[2], x[3]) == (32, 1, 1)
+        assert (x[4:] == -1).all()
+
+    @pytest.mark.skipif(
+        runtime.runtimeGetVersion() < 11000,
+        reason='we do not support it')
+    @pytest.mark.skipif(runtime.deviceGetAttribute(
+        runtime.cudaDevAttrCooperativeLaunch, 0) == 0,
+        reason='cooperative launch is not supported on device 0')
+    def test_grid_group(self):
+        @jit.rawkernel()
+        def test_grid(x):
+            g = jit.cg.this_grid()
+            if g.thread_rank() == 0:
+                x[0] = g.is_valid()
+            if g.thread_rank() == 1:
+                x[1] = g.size()
+            if g.thread_rank() == 32:  # on the 2nd group
+                # Note: this is not yet possible...
+                # x[2], x[3], x[4] == g.group_dim()
+                g_dim = g.group_dim()
+                x[2], x[3], x[4] = g_dim.x, g_dim.y, g_dim.z
+            g.sync()  # this should just work!
+
+        x = cupy.empty((16,), dtype=cupy.uint64)
+        x[:] = -1  # = 2**64-1
+        test_grid[2, 32](x)
+        assert x[0] == 1
+        assert x[1] == 64
+        assert (x[2], x[3], x[4]) == (2, 1, 1)
+        assert (x[5:] == 2**64-1).all()
+
+    @pytest.mark.skipif(
+        runtime.runtimeGetVersion() < 11060
+        or (cupy.cuda.driver._is_cuda_python()
+            and cupy.cuda.nvrtc.getVersion() < (11, 6)),
+        reason='not supported until CUDA 11.6')
+    @pytest.mark.skipif(runtime.deviceGetAttribute(
+        runtime.cudaDevAttrCooperativeLaunch, 0) == 0,
+        reason='cooperative launch is not supported on device 0')
+    def test_grid_group_cu116_new_APIs(self):
+        @jit.rawkernel()
+        def test_grid(x):
+            g = jit.cg.this_grid()
+            if g.thread_rank() == 1:
+                x[1] = g.num_threads()
+            if g.thread_rank() == 32:
+                g_dim = g.dim_blocks()
+                x[2], x[3], x[4] = g_dim.x, g_dim.y, g_dim.z
+            if g.thread_rank() == 33:  # on the 2nd block
+                x[5] = g.block_rank()
+            if g.thread_rank() == 2:
+                x[6] = g.num_blocks()
+            if g.thread_rank() == 34:  # on the 2nd block
+                b_idx = g.block_index()
+                x[7], x[8], x[9] = b_idx.x, b_idx.y, b_idx.z
+            g.sync()  # this should just work!
+
+        x = cupy.empty((16,), dtype=cupy.uint64)
+        x[:] = -1  # = 2**64-1
+        test_grid[2, 32](x)
+        assert x[1] == 64
+        assert (x[2], x[3], x[4]) == (2, 1, 1)
+        assert x[5] == 1
+        assert x[6] == 2
+        assert (x[7], x[8], x[9]) == (1, 0, 0)
+        assert (x[10:] == 2**64-1).all()
+
+    @pytest.mark.skipif(
+        runtime.runtimeGetVersion() < 11000,
+        reason='we do not support it')
+    @pytest.mark.skipif(runtime.deviceGetAttribute(
+        runtime.cudaDevAttrCooperativeLaunch, 0) == 0,
+        reason='cooperative launch is not supported on device 0')
+    def test_cg_sync(self):
+        @jit.rawkernel()
+        def test_sync():
+            b = jit.cg.this_thread_block()
+            g = jit.cg.this_grid()
+            jit.cg.sync(b)
+            jit.cg.sync(g)
+
+        test_sync[2, 64]()
+
+    # We also skip CUDA 11.0 due to missing support of memcpy_async
+    # and aligned_size_t...
+    @pytest.mark.skipif(
+        runtime.runtimeGetVersion() < 11010,
+        reason='not supported until CUDA 11.0')
+    @pytest.mark.parametrize(
+        'test_aligned', (True, False),
+    )
+    def test_cg_memcpy_async_wait_for_wait(self, test_aligned):
+        @jit.rawkernel()
+        def test_copy(x, y):
+            # do two batches of copies to test relevant APIs
+            if test_aligned:
+                smem = jit.shared_memory(cupy.int32, 32*2, alignment=16)
+            else:
+                smem = jit.shared_memory(cupy.int32, 32*2)
+            g = jit.cg.this_thread_block()
+            tid = g.thread_rank()
+            # int32 is 4 bytes
+            if test_aligned:
+                # CuPy ensures x is 256B-aligned
+                jit.cg.memcpy_async(
+                    g, smem, 0, x, 0, 4*32, aligned_size=16)
+                jit.cg.memcpy_async(
+                    g, smem, 32, x, 32, 4*32, aligned_size=16)
+            else:
+                jit.cg.memcpy_async(
+                    g, smem, 0, x, 0, 4*32)
+                jit.cg.memcpy_async(
+                    g, smem, 32, x, 32, 4*32)
+            jit.cg.wait_prior(g, 1)
+            if tid < 32:
+                y[tid] = smem[tid]
+            jit.cg.wait(g)
+            if 32 <= tid and tid < 64:  # can't do "32 <= tid < 64" yet...
+                y[tid] = smem[tid]
+
+        x = cupy.arange(64, dtype=cupy.int32)
+        y = cupy.zeros(64, dtype=cupy.int32)
+        test_copy[2, 64](x, y)
+        assert (x == y).all()
diff --git a/tests/cupyx_tests/jit_tests/test_cub.py b/tests/cupyx_tests/jit_tests/test_cub.py
new file mode 100644
index 0000000..ca4bb54
--- /dev/null
+++ b/tests/cupyx_tests/jit_tests/test_cub.py
@@ -0,0 +1,128 @@
+import cupy
+from cupy import testing
+from cupy_backends.cuda.api import runtime
+from cupyx import jit
+
+
+class TestCubWarpReduce:
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_sum(self, dtype):
+
+        @jit.rawkernel()
+        def warp_reduce_sum(x, y):
+            WarpReduce = jit.cub.WarpReduce[dtype]
+            temp_storage = jit.shared_memory(
+                dtype=WarpReduce.TempStorage, size=1)
+            i, j = jit.blockIdx.x, jit.threadIdx.x
+            value = x[i, j]
+            aggregater = WarpReduce(temp_storage[0])
+            aggregate = aggregater.Sum(value)
+            if j == 0:
+                y[i] = aggregate
+
+        warp_size = 64 if runtime.is_hip else 32
+        h, w = (32, warp_size)
+        x = testing.shaped_random((h, w), dtype=dtype)
+        y = testing.shaped_random((h,), dtype=dtype)
+        expected = cupy.asnumpy(x).sum(axis=-1, dtype=dtype)
+
+        warp_reduce_sum[h, w](x, y)
+        testing.assert_allclose(y, expected, rtol=1e-6)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_reduce_sum(self, dtype):
+
+        @jit.rawkernel()
+        def warp_reduce_sum(x, y):
+            WarpReduce = jit.cub.WarpReduce[dtype]
+            temp_storage = jit.shared_memory(
+                dtype=WarpReduce.TempStorage, size=1)
+            i, j = jit.blockIdx.x, jit.threadIdx.x
+            value = x[i, j]
+            aggregater = WarpReduce(temp_storage[0])
+            aggregate = aggregater.Reduce(value, jit.cub.Sum())
+            if j == 0:
+                y[i] = aggregate
+
+        warp_size = 64 if runtime.is_hip else 32
+        h, w = (32, warp_size)
+        x = testing.shaped_random((h, w), dtype=dtype)
+        y = testing.shaped_random((h,), dtype=dtype)
+        expected = cupy.asnumpy(x).sum(axis=-1, dtype=dtype)
+
+        warp_reduce_sum[h, w](x, y)
+        testing.assert_allclose(y, expected, rtol=1e-6)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_reduce_max(self, dtype):
+
+        @jit.rawkernel()
+        def warp_reduce_max(x, y):
+            WarpReduce = jit.cub.WarpReduce[dtype]
+            temp_storage = jit.shared_memory(
+                dtype=WarpReduce.TempStorage, size=1)
+            i, j = jit.blockIdx.x, jit.threadIdx.x
+            value = x[i, j]
+            aggregater = WarpReduce(temp_storage[0])
+            aggregate = aggregater.Reduce(value, jit.cub.Max())
+            if j == 0:
+                y[i] = aggregate
+
+        warp_size = 64 if runtime.is_hip else 32
+        h, w = (32, warp_size)
+        x = testing.shaped_random((h, w), dtype=dtype)
+        y = testing.shaped_random((h,), dtype=dtype)
+        expected = cupy.asnumpy(x).max(axis=-1)
+
+        warp_reduce_max[h, w](x, y)
+        testing.assert_allclose(y, expected, rtol=1e-6)
+
+
+class TestCubBlockReduce:
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_sum(self, dtype):
+
+        @jit.rawkernel()
+        def block_reduce_sum(x, y):
+            BlockReduce = jit.cub.BlockReduce[dtype, 256]
+            temp_storage = jit.shared_memory(
+                dtype=BlockReduce.TempStorage, size=1)
+            i, j = jit.blockIdx.x, jit.threadIdx.x
+            value = x[i, j]
+            aggregater = BlockReduce(temp_storage[0])
+            aggregate = aggregater.Sum(value)
+            if j == 0:
+                y[i] = aggregate
+
+        h, w = (32, 256)
+        x = testing.shaped_random((h, w), dtype=dtype)
+        y = testing.shaped_random((h,), dtype=dtype)
+        expected = cupy.asnumpy(x).sum(axis=-1, dtype=dtype)
+
+        block_reduce_sum[h, w](x, y)
+        testing.assert_allclose(y, expected, rtol=1e-6)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_reduce_min(self, dtype):
+
+        @jit.rawkernel()
+        def block_reduce_min(x, y):
+            BlockReduce = jit.cub.BlockReduce[dtype, 256]
+            temp_storage = jit.shared_memory(
+                dtype=BlockReduce.TempStorage, size=1)
+            i, j = jit.blockIdx.x, jit.threadIdx.x
+            value = x[i, j]
+            aggregater = BlockReduce(temp_storage[0])
+            aggregate = aggregater.Reduce(value, jit.cub.Min())
+            if j == 0:
+                y[i] = aggregate
+
+        h, w = (32, 256)
+        x = testing.shaped_random((h, w), dtype=dtype)
+        y = testing.shaped_random((h,), dtype=dtype)
+        expected = cupy.asnumpy(x).min(axis=-1)
+
+        block_reduce_min[h, w](x, y)
+        testing.assert_allclose(y, expected, rtol=1e-6)
diff --git a/tests/cupyx_tests/jit_tests/test_device_function.py b/tests/cupyx_tests/jit_tests/test_device_function.py
new file mode 100644
index 0000000..43246fe
--- /dev/null
+++ b/tests/cupyx_tests/jit_tests/test_device_function.py
@@ -0,0 +1,103 @@
+import unittest
+import numpy
+
+import pytest
+
+import cupy
+from cupyx import jit
+from cupy import testing
+
+
+class TestDeviceFunction(unittest.TestCase):
+
+    def test_device_function(self):
+        @jit.rawkernel()
+        def f(x, y, z):
+            tid = jit.threadIdx.x
+            z[tid] = g(x[tid], y[tid]) + x[tid] + y[tid]
+
+        @jit.rawkernel(device=True)
+        def g(x, y):
+            x += 1
+            y += 1
+            return x + y
+
+        x = testing.shaped_random((30,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((30,), dtype=numpy.int32, seed=1)
+        z = testing.shaped_random((30,), dtype=numpy.int32, seed=2)
+        f((1,), (30,), (x, y, z))
+        assert (z == (x + y + 1) * 2).all()
+
+    def test_device_function_duplicated_names(self):
+        @jit.rawkernel()
+        def f(x, y, z):
+            tid = jit.threadIdx.x
+            z[tid] = g(10)(x[tid], y[tid])
+            z[tid] += g(20)(x[tid], y[tid])
+            z[tid] += g(30)(x[tid], y[tid])
+
+        def g(const):
+            @jit.rawkernel(device=True)
+            def f(x, y):
+                return x + y + const
+            return f
+
+        x = testing.shaped_random((30,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((30,), dtype=numpy.int32, seed=1)
+        z = testing.shaped_random((30,), dtype=numpy.int32, seed=2)
+        f((1,), (30,), (x, y, z))
+        assert (z == (x + y) * 3 + 60).all()
+
+    def test_device_function_recursive(self):
+        @jit.rawkernel()
+        def f(x, y, z):
+            tid = jit.threadIdx.x
+            z[tid] = g(x[tid], y[tid])
+
+        @jit.rawkernel(device=True)
+        def g(x, y):
+            return x + y + g(x, y)
+
+        x = testing.shaped_random((30,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((30,), dtype=numpy.int32, seed=1)
+        z = testing.shaped_random((30,), dtype=numpy.int32, seed=2)
+        with pytest.raises(ValueError):
+            f((1,), (30,), (x, y, z))
+
+    def test_device_function_template_recursion(self):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x
+            y[tid] = x[tid]
+            jit.syncthreads()
+            g(1)(y)
+
+        def g(step):
+            @jit.rawkernel(device=True)
+            def f(x):
+                if step < 256:
+                    tid = jit.threadIdx.x
+                    if tid % (step * 2) == 0:
+                        x[tid] += x[tid + step]
+                        jit.syncthreads()
+                        g(step * 2)(x)
+            return f
+
+        x = testing.shaped_random((256,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((256,), dtype=numpy.int32, seed=1)
+        f((1,), (256,), (x, y))
+        assert y[0] == x.sum()
+
+    def test_device_function_called_once(self):
+        @jit.rawkernel(device=True)
+        def g(x):
+            x[0] += 1
+            return 1
+
+        @jit.rawkernel()
+        def f(x):
+            x[g(x)] += 1
+
+        x = cupy.array([0, 0])
+        f((1,), (1,), (x,))
+        testing.assert_array_equal(x, [1, 1])
diff --git a/tests/cupyx_tests/jit_tests/test_raw.py b/tests/cupyx_tests/jit_tests/test_raw.py
new file mode 100644
index 0000000..c13125f
--- /dev/null
+++ b/tests/cupyx_tests/jit_tests/test_raw.py
@@ -0,0 +1,770 @@
+import numpy
+
+import pytest
+
+import cupy
+from cupyx import jit
+from cupy import testing
+from cupy.cuda import device
+from cupy.cuda import runtime
+
+
+class TestRaw:
+
+    def test_raw_grid_1D(self):
+        @jit.rawkernel()
+        def f(arr1, arr2):
+            x = jit.grid(1)
+            if x < arr1.size:
+                arr2[x] = arr1[x]
+
+        x = cupy.arange(10)
+        y = cupy.empty_like(x)
+        f((1,), (10,), (x, y))
+        assert (x == y).all()
+
+    def test_raw_grid_2D(self):
+        @jit.rawkernel()
+        def f(arr1, arr2, n, m):
+            x, y = jit.grid(2)
+            # TODO(leofang): make it possible to write this:
+            # if x < arr1.shape[0] and y < arr1.shape[1]:
+            if x < n and y < m:
+                arr2[x, y] = arr1[x, y]
+
+        x = cupy.arange(20).reshape(4, 5)
+        y = cupy.empty_like(x)
+        f((1,), (4, 5), (x, y, x.shape[0], x.shape[1]))
+        assert (x == y).all()
+
+    def test_raw_grid_3D(self):
+        @jit.rawkernel()
+        def f(arr1, arr2, k, m, n):
+            x, y, z = jit.grid(3)
+            if x < k and y < m and z < n:
+                arr2[x, y, z] = arr1[x, y, z]
+
+        l, m, n = (2, 3, 4)
+        x = cupy.arange(24).reshape(l, m, n)
+        y = cupy.empty_like(x)
+        f(((l+1)//2, (m+1)//2, (n+1)//2), (2, 2, 2), (x, y, l, m, n))
+        assert (x == y).all()
+
+    def test_raw_grid_invalid1(self):
+        @jit.rawkernel()
+        def f():
+            x, = jit.grid(1)  # cannot unpack an int
+
+        with pytest.raises(ValueError):
+            f((1,), (1,), ())
+
+    def test_raw_grid_invalid2(self):
+        @jit.rawkernel()
+        def f():
+            x = jit.grid(2)
+            y = cupy.int64(x)  # <- x is a tuple  # NOQA
+
+        # we don't care the exception type as long as something is raised
+        with pytest.raises(Exception):
+            f((1,), (1,), ())
+
+    def test_raw_grid_invalid3(self):
+        for n in (0, 4, 'abc', [0], (1,)):
+            @jit.rawkernel()
+            def f():
+                x = jit.grid(n)  # n can only be 1, 2, 3 (as int)  # NOQA
+
+            err = ValueError if isinstance(n, int) else TypeError
+            with pytest.raises(err):
+                f((1,), (1,), ())
+
+    def test_raw_gridsize_1D(self):
+        @jit.rawkernel()
+        def f(arr):
+            x = jit.grid(1)
+            gx = jit.gridsize(1)
+            if x == 0:
+                arr[x] = gx
+
+        x = cupy.zeros(1)
+        grid = 8
+        block = 10
+        f((grid,), (block,), (x,))
+        assert x == grid * block
+
+    def test_raw_gridsize_2D(self):
+        @jit.rawkernel()
+        def f(arr):
+            x, y = jit.grid(2)
+            gx, gy = jit.gridsize(2)
+            if x == 0 and y == 0:
+                arr[0] = gx
+                arr[1] = gy
+
+        x = cupy.zeros(2)
+        grid = (2, 7)
+        block = (3, 5)
+        f(grid, block, (x,))
+        assert x[0] == grid[0] * block[0]
+        assert x[1] == grid[1] * block[1]
+
+    def test_raw_gridsize_3D(self):
+        @jit.rawkernel()
+        def f(arr):
+            x, y, z = jit.grid(3)
+            gx, gy, gz = jit.gridsize(3)
+            if x == 0 and y == 0 and z == 0:
+                arr[0] = gx
+                arr[1] = gy
+                arr[2] = gz
+
+        x = cupy.zeros(3)
+        grid = (2, 7, 4)
+        block = (3, 5, 11)
+        f(grid, block, (x,))
+        assert x[0] == grid[0] * block[0]
+        assert x[1] == grid[1] * block[1]
+        assert x[2] == grid[2] * block[2]
+
+    def test_raw_one_thread(self):
+        @jit.rawkernel()
+        def f(x, y):
+            y[0] = x[0]
+
+        x = cupy.array([10], dtype=numpy.int32)
+        y = cupy.array([20], dtype=numpy.int32)
+        f((1,), (1,), (x, y))
+        assert int(y[0]) == 10
+
+    def test_raw_elementwise_single_op(self):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            y[tid] = x[tid]
+
+        x = testing.shaped_random((30,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((30,), dtype=numpy.int32, seed=1)
+        f((5,), (6,), (x, y))
+        assert bool((x == y).all())
+
+    def test_raw_elementwise_loop(self):
+        @jit.rawkernel()
+        def f(x, y, size):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            ntid = jit.blockDim.x * jit.gridDim.x
+            for i in range(tid, size, ntid):
+                y[i] = x[i]
+
+        x = testing.shaped_random((1024,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((1024,), dtype=numpy.int32, seed=1)
+        f((5,), (6,), (x, y, numpy.uint32(1024)))
+        assert bool((x == y).all())
+
+    def test_raw_multidimensional_array(self):
+        @jit.rawkernel()
+        def f(x, y, n_row, n_col):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            ntid = jit.blockDim.x * jit.gridDim.x
+            size = n_row * n_col
+            for i in range(tid, size, ntid):
+                i_row = i // n_col
+                i_col = i % n_col
+                y[i_row, i_col] = x[i_row, i_col]
+
+        n, m = numpy.uint32(12), numpy.uint32(13)
+        x = testing.shaped_random((n, m), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((n, m), dtype=numpy.int32, seed=1)
+        f((5,), (6,), (x, y, n, m))
+        assert bool((x == y).all())
+
+    def test_raw_multidimensional_array_with_attr(self):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            ntid = jit.blockDim.x * jit.gridDim.x
+            n_col = x.size // len(x)
+            for i in range(tid, x.size, ntid):
+                i_row = i // n_col
+                i_col = i % n_col
+                y[i_row, i_col] = x[i_row, i_col]
+
+        n, m = numpy.uint32(12), numpy.uint32(13)
+        x = testing.shaped_random((n, m), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((n, m), dtype=numpy.int32, seed=1)
+        f((5,), (6,), (x, y))
+        assert bool((x == y).all())
+
+    def test_raw_ndim(self):
+        @jit.rawkernel()
+        def f(x, y):
+            y[0] = x.ndim
+
+        x = cupy.empty((1, 1, 1, 1, 1, 1, 1), dtype=numpy.int32)
+        y = cupy.zeros((1,), dtype=numpy.int64)
+        f((1,), (1,), (x, y))
+        assert y.item() == 7
+
+    def test_raw_0dim_array(self):
+        @jit.rawkernel()
+        def f(x, y):
+            y[()] = x[()]
+
+        x = testing.shaped_random((), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((), dtype=numpy.int32, seed=1)
+        f((1,), (1,), (x, y))
+        assert bool((x == y).all())
+
+    def test_min(self):
+        @jit.rawkernel()
+        def f(x, y, z, r):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            r[tid] = min(x[tid], y[tid], z[tid])
+
+        x = testing.shaped_random((1024,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((1024,), dtype=numpy.int32, seed=1)
+        z = testing.shaped_random((1024,), dtype=numpy.int32, seed=2)
+        r = testing.shaped_random((1024,), dtype=numpy.int32, seed=3)
+        f((8,), (128,), (x, y, z, r))
+        expected = cupy.minimum(x, cupy.minimum(y, z))
+        assert bool((r == expected).all())
+
+    def test_max(self):
+        @jit.rawkernel()
+        def f(x, y, z, r):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            r[tid] = max(x[tid], y[tid], z[tid])
+
+        x = testing.shaped_random((1024,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((1024,), dtype=numpy.int32, seed=1)
+        z = testing.shaped_random((1024,), dtype=numpy.int32, seed=2)
+        r = testing.shaped_random((1024,), dtype=numpy.int32, seed=3)
+        f((8,), (128,), (x, y, z, r))
+        expected = cupy.maximum(x, cupy.maximum(y, z))
+        assert bool((r == expected).all())
+
+    def test_syncthreads(self):
+        @jit.rawkernel()
+        def f(x, y, buf):
+            tid = jit.threadIdx.x + jit.threadIdx.y * jit.blockDim.x
+            ntid = jit.blockDim.x * jit.blockDim.y
+            buf[tid] = x[ntid - tid - 1]
+            jit.syncthreads()
+            y[tid] = buf[ntid - tid - 1]
+
+        x = testing.shaped_random((1024,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((1024,), dtype=numpy.int32, seed=1)
+        buf = testing.shaped_random((1024,), dtype=numpy.int32, seed=2)
+        f((1,), (32, 32), (x, y, buf))
+        assert bool((x == y).all())
+
+    def test_syncwarp(self):
+        @jit.rawkernel()
+        def f(x):
+            laneId = jit.threadIdx.x & 0x1f
+            if laneId < 16:
+                x[laneId] = 1
+            else:
+                x[laneId] = 2
+            jit.syncwarp()
+
+        x = cupy.zeros((32,), dtype=numpy.int32)
+        y = cupy.ones_like(x)
+        f((1,), (32,), (x,))
+        y[16:] += 1
+        assert bool((x == y).all())
+
+    def test_syncwarp_mask(self):
+        @jit.rawkernel()
+        def f(x, m):
+            laneId = jit.threadIdx.x & 0x1f
+            if laneId < m:
+                x[laneId] = 1
+                jit.syncwarp(mask=(1 << m) - 1)
+
+        for mask in (2, 4, 8, 16, 32):
+            x = cupy.zeros((32,), dtype=numpy.int32)
+            y = cupy.zeros_like(x)
+            f((1,), (32,), (x, mask))
+            y[:mask] += 1
+            assert bool((x == y).all())
+
+    def test_shared_memory_static(self):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x
+            ntid = jit.blockDim.x
+            bid = jit.blockIdx.x
+            i = tid + bid * ntid
+
+            smem = jit.shared_memory(numpy.int32, 32)
+            smem[tid] = x[i]
+            jit.syncthreads()
+            y[i] = smem[ntid - tid - 1]
+
+        x = testing.shaped_random((1024,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((1024,), dtype=numpy.int32, seed=1)
+        f((32,), (32,), (x, y))
+        expected = x.reshape(32, 32)[:, ::-1].ravel()
+        assert bool((y == expected).all())
+
+    def test_shared_memory_dynamic(self):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x
+            ntid = jit.blockDim.x
+            bid = jit.blockIdx.x
+            i = tid + bid * ntid
+
+            smem = jit.shared_memory(numpy.int32, None)
+            smem[tid] = x[i]
+            jit.syncthreads()
+            y[i] = smem[ntid - tid - 1]
+
+        x = testing.shaped_random((1024,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((1024,), dtype=numpy.int32, seed=1)
+        f((32,), (32,), (x, y), shared_mem=128)
+        expected = x.reshape(32, 32)[:, ::-1].ravel()
+        assert bool((y == expected).all())
+
+    @staticmethod
+    def _check(a, e):
+        if a.dtype == numpy.float16:
+            testing.assert_allclose(a, e, rtol=3e-2, atol=3e-2)
+        else:
+            testing.assert_allclose(a, e, rtol=1e-5, atol=1e-5)
+
+    @testing.for_dtypes('iILQfd' if runtime.is_hip else 'iILQefd')
+    def test_atomic_add(self, dtype):
+        @jit.rawkernel()
+        def f(x, index, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            jit.atomic_add(out, index[tid], x[tid])
+
+        x = testing.shaped_random((1024,), dtype=dtype, seed=0)
+        index = testing.shaped_random(
+            (1024,), dtype=numpy.bool_, seed=1).astype(numpy.int32)
+        out = cupy.zeros((2,), dtype=dtype)
+        f((32,), (32,), (x, index, out))
+
+        expected = cupy.zeros((2,), dtype=dtype)
+        cupy.add.at(expected, index, x)
+        self._check(out, expected)
+
+    @testing.for_dtypes('iI')
+    def test_atomic_sub(self, dtype):
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                jit.atomic_sub(out, 0, x[tid])
+
+        x = cupy.ones((1024,), dtype=dtype)
+        out = cupy.sum(x, dtype=dtype).reshape(1,)
+        f((32,), (32,), (x, out))
+        expected = cupy.zeros_like(out)
+        self._check(out, expected)
+
+    @testing.for_dtypes('iILQf')
+    def test_atomic_exch(self, dtype):
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                jit.atomic_exch(out, x.size - tid - 1, x[tid])
+
+        x = cupy.arange(1024, dtype=dtype)
+        out = cupy.zeros_like(x)
+        f((32,), (32,), (x, out))
+        expected = x[::-1]
+        self._check(out, expected)
+
+    @testing.for_dtypes('iILQ')
+    def test_atomic_min(self, dtype):
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                jit.atomic_min(out, tid, x[tid])
+
+        x = cupy.arange(1024, dtype=dtype)
+        out = x + 1
+        f((32,), (32,), (x, out))  # effectively copy x to out
+        self._check(out, x)
+
+    @testing.for_dtypes('iILQ')
+    def test_atomic_max(self, dtype):
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                jit.atomic_max(out, tid, x[tid])
+
+        x = cupy.arange(1, 1025, dtype=dtype)
+        out = x - 1
+        f((32,), (32,), (x, out))  # effectively copy x to out
+        self._check(out, x)
+
+    def test_atomic_inc(self):
+        dtype = cupy.uint32  # atomic_inc only supports 1 dtype
+
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                # = 0 if out[tid] >= x[tid] else out[tid] + 1
+                jit.atomic_inc(out, tid, x[tid])
+
+        x = cupy.arange(1, 1025, dtype=dtype)
+        out = x - 1
+        f((32,), (32,), (x, out))
+        expected = x
+        self._check(out, expected)
+
+    def test_atomic_dec(self):
+        dtype = cupy.uint32  # atomic_dec only supports 1 dtype
+
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                # = x[tid] if (out[tid] == 0 or out[tid] >= x[tid]) \
+                # else out[tid] - 1
+                jit.atomic_dec(out, tid, x[tid])
+
+        x = cupy.zeros(1024, dtype=dtype)
+        out = cupy.arange(1024, dtype=dtype)
+        f((32,), (32,), (x, out))
+        expected = x
+        self._check(out, expected)
+
+    @testing.for_dtypes('iILQ' if runtime.is_hip else 'iHILQ')
+    def test_atomic_cas(self, dtype):
+        if dtype == cupy.uint16:
+            if (
+                runtime.is_hip or
+                runtime.runtimeGetVersion() < 10010 or
+                int(device.get_compute_capability()) < 70
+            ):
+                self.skipTest('not supported')
+
+        @jit.rawkernel()
+        def f(x, y, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                # = y[tid] if out[tid] == x[tid] else out[tid]
+                jit.atomic_cas(out, tid, x[tid], y[tid])
+
+        x = cupy.arange(1024, dtype=dtype)
+        y = x.copy()
+        y[512:] = 0
+        out = x.copy()
+        out[:512] = 0
+        f((32,), (32,), (x, y, out))
+        expected = cupy.zeros_like(out)
+        self._check(out, expected)
+
+    @testing.for_dtypes('iILQ')
+    def test_atomic_and(self, dtype):
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                jit.atomic_and(out, tid, x[tid])
+
+        x = cupy.arange(1024, dtype=dtype)
+        out = cupy.ones_like(x)
+        f((32,), (32,), (x, out))
+        expected = cupy.zeros_like(out)
+        expected[1::2] = 1
+        self._check(out, expected)
+
+    @testing.for_dtypes('iILQ')
+    def test_atomic_or(self, dtype):
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                jit.atomic_or(out, tid, x[tid])
+
+        x = testing.shaped_random((1024,), dtype=dtype, seed=0)
+        out = testing.shaped_random((1024,), dtype=dtype, seed=1)
+        y = out.copy()
+        f((32,), (32,), (x, out))
+        expected = x | y
+        self._check(out, expected)
+
+    @testing.for_dtypes('iILQ')
+    def test_atomic_xor(self, dtype):
+        @jit.rawkernel()
+        def f(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                jit.atomic_xor(out, tid, x[tid])
+
+        @jit.rawkernel()
+        def f_no_atomic(x, out):
+            tid = jit.blockDim.x * jit.blockIdx.x + jit.threadIdx.x
+            if tid < x.size:
+                out[tid] = out[tid] ^ x[tid]
+
+        x = testing.shaped_random((1024,), dtype=dtype, seed=0)
+        out = testing.shaped_random((1024,), dtype=dtype, seed=1)
+        expected = out.copy()
+        f((32,), (32,), (x, out))
+        f_no_atomic((32,), (32,), (x, expected))
+        self._check(out, expected)
+
+    def test_raw_grid_block_interface(self):
+        @jit.rawkernel()
+        def f(x, y, size):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            ntid = jit.blockDim.x * jit.gridDim.x
+            for i in range(tid, size, ntid):
+                y[i] = x[i]
+
+        x = testing.shaped_random((1024,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((1024,), dtype=numpy.int32, seed=1)
+        f[5, 6](x, y, numpy.uint32(1024))
+        assert bool((x == y).all())
+
+    # TODO(leofang): test float16 ('e') once cupy/cupy#5346 is resolved
+    @testing.for_dtypes('iIlqfd' if runtime.is_hip else 'iIlLqQfd')
+    def test_shfl(self, dtype):
+        # strictly speaking this function is invalid in Python (see the
+        # discussion in cupy/cupy#5340), but it serves for our purpose
+        @jit.rawkernel()
+        def f(a, b):
+            laneId = jit.threadIdx.x & 0x1f
+            if laneId == 0:
+                value = a
+            value = jit.shfl_sync(0xffffffff, value, 0)
+            b[laneId] = value
+
+        a = dtype(100)
+        b = cupy.empty((32,), dtype=dtype)
+        f[1, 32](a, b)
+        assert (b == a * cupy.ones((32,), dtype=dtype)).all()
+
+    def test_shfl_width(self):
+        @jit.rawkernel()
+        def f(a, b, w):
+            laneId = jit.threadIdx.x & 0x1f
+            value = jit.shfl_sync(0xffffffff, b[jit.threadIdx.x], 0, width=w)
+            b[laneId] = value
+
+        c = cupy.arange(32, dtype=cupy.int32)
+        for w in (2, 4, 8, 16, 32):
+            a = cupy.int32(100)
+            b = cupy.arange(32, dtype=cupy.int32)
+            f[1, 32](a, b, w)
+            c[c % w != 0] = c[c % w == 0]
+            assert (b == c).all()
+
+    # TODO(leofang): test float16 ('e') once cupy/cupy#5346 is resolved
+    @testing.for_dtypes('iIlqfd' if runtime.is_hip else 'iIlLqQfd')
+    def test_shfl_up(self, dtype):
+        N = 5
+
+        @jit.rawkernel()
+        def f(a):
+            value = jit.shfl_up_sync(0xffffffff, a[jit.threadIdx.x], N)
+            a[jit.threadIdx.x] = value
+
+        a = cupy.arange(32, dtype=dtype)
+        f[1, 32](a)
+        expected = [i for i in range(N)] + [i for i in range(32-N)]
+        assert (a == cupy.asarray(expected, dtype=dtype)).all()
+
+    # TODO(leofang): test float16 ('e') once cupy/cupy#5346 is resolved
+    @testing.for_dtypes('iIlqfd' if runtime.is_hip else 'iIlLqQfd')
+    def test_shfl_down(self, dtype):
+        N = 5
+        # __shfl_down() on HIP does not seem to have the same behavior...
+        block = cupy._core._get_warpsize()
+
+        @jit.rawkernel()
+        def f(a):
+            value = jit.shfl_down_sync(0xffffffff, a[jit.threadIdx.x], N)
+            a[jit.threadIdx.x] = value
+
+        a = cupy.arange(block, dtype=dtype)
+        f[1, block](a)
+        expected = [i for i in range(N, block)]
+        expected += [(block-N+i) for i in range(N)]
+        assert (a == cupy.asarray(expected, dtype=dtype)).all()
+
+    # TODO(leofang): test float16 ('e') once cupy/cupy#5346 is resolved
+    @testing.for_dtypes('iIlqfd' if runtime.is_hip else 'iIlLqQfd')
+    def test_shfl_xor(self, dtype):
+        @jit.rawkernel()
+        def f_shfl_xor(a):
+            laneId = jit.threadIdx.x & 0x1f
+            value = 31 - laneId
+            i = 16
+            while i >= 1:
+                value += jit.shfl_xor_sync(0xffffffff, value, i)
+                i //= 2
+            a[jit.threadIdx.x] = value
+
+        a = cupy.arange(32, dtype=dtype)
+        b = a.copy()
+        f_shfl_xor[1, 32](a)
+        assert (a == b.sum() * cupy.ones(32, dtype=dtype)).all()
+
+    def test_error_msg(self):
+        @jit.rawkernel()
+        def f(x):
+            return unknown_var  # NOQA
+
+        x = cupy.zeros((10,), dtype=numpy.float32)
+        with pytest.raises(NameError, match='Unbound name: unknown_var'):
+            f((1,), (1,), (x,))
+
+    def test_laneid(self):
+        @jit.rawkernel()
+        def f(arr):
+            x = jit.grid(1)
+            if x < arr.size:
+                arr[x] = jit.laneid()
+
+        N = cupy._core._get_warpsize()
+        x = cupy.zeros((N*2,), dtype=cupy.uint32)
+        f((1,), (N*2,), (x,))
+        y = cupy.arange(N*2, dtype=cupy.uint32) % N
+        assert (x == y).all()
+
+    def test_warpsize(self):
+        @jit.rawkernel()
+        def f(arr):
+            x = jit.grid(1)
+            if x == 0:
+                arr[0] = jit.warpsize
+
+        N = cupy._core._get_warpsize()
+        x = cupy.zeros((1,), dtype=cupy.uint32)
+        f((1,), (1,), (x,))
+        assert x == N
+
+    @testing.for_CF_orders()
+    def test_shape_strides(self, order):
+        @jit.rawkernel()
+        def f(arr):
+            x = jit.grid(1)
+            if x == 0:
+                shapes = arr.shape
+                strides = arr.strides
+                arr[0, 0, 0] = shapes[0]
+                arr[0, 0, 1] = shapes[1]
+                arr[0, 0, 2] = shapes[2]
+                arr[0, 0, 3] = strides[0]
+                arr[0, 0, 4] = strides[1]
+                arr[0, 0, 5] = strides[2]
+
+        a = cupy.zeros((3, 4, 6), order=order)
+        f[1, 1](a)
+        assert (a[0, 0, 0:3] == cupy.asarray(a.shape)).all()
+        assert (a[0, 0, 3:6] == cupy.asarray(a.strides)).all()
+
+    @testing.multi_gpu(2)
+    def test_device_cache(self):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            y[tid] = x[tid]
+
+        with device.Device(0):
+            x = testing.shaped_random((30,), dtype=numpy.int32, seed=0)
+            y = testing.shaped_random((30,), dtype=numpy.int32, seed=1)
+            f((5,), (6,), (x, y))
+            assert bool((x == y).all())
+        with device.Device(1):
+            x = testing.shaped_random((30,), dtype=numpy.int32, seed=2)
+            y = testing.shaped_random((30,), dtype=numpy.int32, seed=3)
+            f((5,), (6,), (x, y))
+            assert bool((x == y).all())
+
+    def test_cached_code(self):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            y[tid] = x[tid]
+
+        assert isinstance(f.cached_codes, dict)
+        assert len(f.cached_codes) == 0
+
+        x = testing.shaped_random((30,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((30,), dtype=numpy.int32, seed=1)
+        f((5,), (6,), (x, y))
+        assert bool((x == y).all())
+
+        assert len(f.cached_codes) == 1
+        assert isinstance(f.cached_code, str)
+
+        x = testing.shaped_random((30,), dtype=numpy.float32, seed=0)
+        y = testing.shaped_random((30,), dtype=numpy.float32, seed=1)
+        f((5,), (6,), (x, y))
+        assert bool((x == y).all())
+
+        assert len(f.cached_codes) == 2
+
+        x = testing.shaped_random((30,), dtype=numpy.int32, seed=0)
+        y = testing.shaped_random((30,), dtype=numpy.int32, seed=1)
+        f((5,), (6,), (x, y))
+        assert bool((x == y).all())
+
+        assert len(f.cached_codes) == 2
+
+    @pytest.mark.parametrize(
+        'unroll', (True, False, None, 5)
+    )
+    def test_range(self, unroll):
+
+        @jit.rawkernel()
+        def f(x):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            y = x[tid]
+            # workaround: can't use "is" here...
+            if unroll == None:  # noqa: E711
+                for i in jit.range(5):
+                    y += 2
+            else:
+                for i in jit.range(5, unroll=unroll):
+                    y += 2
+            x[tid] = y
+
+        x = testing.shaped_random((30,), dtype=numpy.float32, seed=0)
+        y = x.copy()
+        f[3, 10](x)
+        assert (x == y + 10).all()
+        if unroll is True:
+            assert '#pragma unroll\n' in f.cached_code
+        elif unroll is False:
+            assert '#pragma unroll(1)\n' in f.cached_code
+        elif unroll is None:
+            assert 'unroll' not in f.cached_code
+        elif unroll >= 0:
+            assert f'#pragma unroll({unroll})\n' in f.cached_code
+
+    @pytest.mark.parametrize('ctype', (bool, int, float, complex))
+    def test_scalar_args(self, ctype):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            x[tid] = y
+
+        x = cupy.zeros((30), dtype=ctype)
+        y = ctype(1)
+        f((5,), (6,), (x, y))
+        testing.assert_array_equal(x, numpy.full_like(x, 1))
+
+    @testing.for_all_dtypes()
+    def test_numpy_scalar_args(self, dtype):
+        @jit.rawkernel()
+        def f(x, y):
+            tid = jit.threadIdx.x + jit.blockDim.x * jit.blockIdx.x
+            x[tid] = y
+
+        x = cupy.zeros((30), dtype=dtype)
+        y = numpy.dtype(dtype).type(1)
+        f((5,), (6,), (x, y))
+        testing.assert_array_equal(x, numpy.full_like(x, 1))
diff --git a/tests/cupyx_tests/jit_tests/test_thrust.py b/tests/cupyx_tests/jit_tests/test_thrust.py
new file mode 100644
index 0000000..867d769
--- /dev/null
+++ b/tests/cupyx_tests/jit_tests/test_thrust.py
@@ -0,0 +1,1125 @@
+import numpy
+
+import pytest
+
+import cupy
+from cupy import testing
+from cupy_backends.cuda.api import runtime
+from cupyx import jit
+
+
+class TestThrust:
+    def test_count_shared_memory(self):
+        @jit.rawkernel()
+        def count(x, y):
+            tid = jit.threadIdx.x
+            smem = jit.shared_memory(numpy.int32, 32)
+            smem[tid] = x[tid]
+            jit.syncthreads()
+            y[tid] = jit.thrust.count(jit.thrust.device, smem, smem + 32, tid)
+
+        size = cupy.uint32(32)
+        x = cupy.arange(size, dtype=cupy.int32)
+        y = cupy.zeros(size, dtype=cupy.int32)
+        count[1, 32](x, y)
+        testing.assert_array_equal(y, cupy.ones(size, dtype=cupy.int32))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_adjacent_difference(self, order):
+        @jit.rawkernel()
+        def adjacent_difference(x, y):
+            i = jit.threadIdx.x
+            array = x[i]
+            result = y[i]
+            jit.thrust.adjacent_difference(
+                jit.thrust.device, array.begin(), array.end(), result.begin())
+
+        h, w = (128, 128)
+        x = testing.shaped_random(
+            (h, w), dtype=numpy.int32, order=order)
+        y = cupy.zeros(h * w, dtype=cupy.int32).reshape(h, w)
+        adjacent_difference[1, 128](x, y)
+        testing.assert_array_equal(y[:, 0], x[:, 0])
+        testing.assert_array_equal(y[:, 1:], x[:, 1:] - x[:, :-1])
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_binary_search(self, order):
+        @jit.rawkernel()
+        def binary_search(x, y):
+            i = jit.threadIdx.x
+            array = x[i]
+            y[i] = jit.thrust.binary_search(
+                jit.thrust.seq, array.begin(), array.end(), 100)
+
+        n1, n2 = (128, 160)
+        x = testing.shaped_random(
+            (n1, n2), dtype=numpy.int32, scale=200, order=order)
+        x = cupy.sort(x, axis=-1)
+        y = cupy.zeros(n1, dtype=cupy.bool_)
+        binary_search[1, n1](x, y)
+
+        expected = (x == 100).any(axis=-1)
+        assert 70 < expected.sum().item() < 80
+        testing.assert_array_equal(y, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_binary_search_vec(self, order):
+        @jit.rawkernel()
+        def binary_search(x, y, z):
+            i = jit.threadIdx.x
+            array = x[i]
+            value = y[i]
+            output = z[i]
+            jit.thrust.binary_search(
+                jit.thrust.seq,
+                array.begin(), array.end(),
+                value.begin(), value.end(),
+                output.begin())
+
+        n1, n2, n3 = (128, 160, 200)
+        x = testing.shaped_random(
+            (n1, n2), dtype=numpy.int32, scale=200, order=order, seed=0)
+        x = cupy.sort(x, axis=-1)
+        y = testing.shaped_random(
+            (n1, n3), dtype=numpy.int32, scale=200, order=order, seed=1)
+        z = cupy.zeros((n1, n3), dtype=cupy.bool_)
+        binary_search[1, n1](x, y, z)
+
+        expected = (x[:, :, None] == y[:, None, :]).any(axis=1)
+        testing.assert_array_equal(z, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_count_iterator(self, order):
+        @jit.rawkernel()
+        def count(x, y):
+            i = jit.threadIdx.x
+            j = jit.threadIdx.y
+            k = jit.threadIdx.z
+            array = x[i, j]
+            y[i, j, k] = jit.thrust.count(
+                jit.thrust.device, array.begin(), array.end(), k)
+
+        h, w, c = (16, 16, 128)
+        x = testing.shaped_random(
+            (h, w, c), dtype=numpy.int32, scale=4, order=order)
+        y = cupy.zeros(h * w * 4, dtype=cupy.int32).reshape(h, w, 4)
+        count[1, (16, 16, 4)](x, y)
+        expected = (x[..., None] == cupy.arange(4)).sum(axis=2)
+        testing.assert_array_equal(y, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_copy_iterator(self, order):
+        @jit.rawkernel()
+        def copy(x, y):
+            i = jit.threadIdx.x
+            x_array = x[i]
+            y_array = y[i]
+            jit.thrust.copy(
+                jit.thrust.device,
+                x_array.begin(),
+                x_array.end(),
+                y_array.begin(),
+            )
+
+        h, w = (256, 256)
+        x = testing.shaped_random((h, w), dtype=numpy.int32, order=order)
+        y = cupy.zeros((h, w), dtype=numpy.int32)
+        copy[1, 256](x, y)
+        testing.assert_array_equal(x, y)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_equal(self, order):
+        if runtime.is_hip:
+            # TODO(asi1024): Fix compile error.
+            pytest.skip('Skip rocm thrust::equal test')
+
+        @jit.rawkernel()
+        def equal(x, y, z):
+            i = jit.threadIdx.x
+            x_array = x[i]
+            y_array = y[i]
+            z[i] = jit.thrust.equal(
+                jit.thrust.device,
+                x_array.begin(),
+                x_array.end(),
+                y_array.begin(),
+            )
+
+        n1, n2 = (256, 256)
+        x = testing.shaped_random(
+            (n1, n2), dtype=numpy.int32, order=order, seed=0)
+        y = x.copy()
+        y[100][200] = 0
+        z = cupy.zeros((n1,), dtype=numpy.int32)
+        equal[1, 256](x, y, z)
+        testing.assert_array_equal(z, (x == y).all(axis=1))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_equal_range(self, order):
+        @jit.rawkernel()
+        def equal_range(x, y):
+            i = jit.threadIdx.x
+            array = x[i]
+            start, end = jit.thrust.equal_range(
+                jit.thrust.seq, array.begin(), array.end(), 100)
+            y[i] = end - start
+
+        n1, n2 = (128, 160)
+        x = testing.shaped_random(
+            (n1, n2), dtype=numpy.int32, scale=200, order=order)
+        x = cupy.sort(x, axis=-1)
+        y = cupy.zeros(n1, dtype=cupy.int32)
+        equal_range[1, n1](x, y)
+
+        expected = (x == 100).sum(axis=-1)
+        testing.assert_array_equal(y, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_exclusive_scan(self, order):
+        @jit.rawkernel()
+        def exclusive_scan(x, y):
+            i = jit.threadIdx.x
+            array = x[i]
+            result = y[i]
+            jit.thrust.exclusive_scan(
+                jit.thrust.seq, array.begin(), array.end(), result.begin())
+
+        n1, n2 = (128, 160)
+        x = testing.shaped_random(
+            (n1, n2), dtype=numpy.int32, order=order)
+        y = cupy.zeros((n1, n2), dtype=cupy.int32)
+        exclusive_scan[1, n1](x, y)
+
+        expected = x.cumsum(axis=-1)[:, :-1]
+        testing.assert_array_equal(y[:, 1:], expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_exclusive_scan_init(self, order):
+        @jit.rawkernel()
+        def exclusive_scan_init(x, y):
+            i = jit.threadIdx.x
+            array = x[i]
+            result = y[i]
+            jit.thrust.exclusive_scan(
+                jit.thrust.seq, array.begin(), array.end(), result.begin(), 10)
+
+        n1, n2 = (128, 160)
+        x = testing.shaped_random(
+            (n1, n2), dtype=numpy.int32, order=order)
+        y = cupy.zeros((n1, n2), dtype=cupy.int32)
+        exclusive_scan_init[1, n1](x, y)
+
+        expected = x.cumsum(axis=-1)[:, :-1] + 10
+        testing.assert_array_equal(y[:, 1:], expected)
+
+    def test_exclusive_scan_by_key(self):
+        @jit.rawkernel()
+        def exclusive_scan_by_key(key, value):
+            jit.thrust.exclusive_scan_by_key(
+                jit.thrust.device, key.begin(), key.end(),
+                value.begin(), value.begin(), 5)
+
+        key = cupy.array([0, 0, 0, 1, 1, 2, 3, 3, 3, 3])
+        value = cupy.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
+        exclusive_scan_by_key[1, 1](key, value)
+
+        expected = cupy.array([5, 6, 7, 5, 6, 5, 5, 6, 7, 8])
+        testing.assert_array_equal(value, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_fill(self, order):
+        @jit.rawkernel()
+        def fill(x):
+            i = jit.threadIdx.x
+            array = x[i]
+            jit.thrust.fill(jit.thrust.device, array.begin(), array.end(), 10)
+
+        n1, n2 = (128, 160)
+        x = cupy.zeros((n1, n2), dtype=numpy.int32, order=order)
+        fill[1, n1](x)
+        expected = cupy.full((n1, n2), 10, dtype=numpy.int32)
+        testing.assert_array_equal(x, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_find(self, order):
+        @jit.rawkernel()
+        def find(x, y):
+            i, j, k = jit.threadIdx.x, jit.threadIdx.y, jit.threadIdx.z
+            array = x[i, j]
+            iterator = jit.thrust.find(
+                jit.thrust.device, array.begin(), array.end(), k)
+            y[i, j, k] = iterator - array.begin()
+
+        h, w, c = (16, 16, 128)
+        x = testing.shaped_random(
+            (h, w, c), dtype=numpy.int32, scale=4, order=order)
+        y = cupy.zeros(h * w * 4, dtype=cupy.int32).reshape(h, w, 4)
+        find[1, (16, 16, 4)](x, y)
+
+        expected = numpy.full((h, w, 4), c, y.dtype)
+        x_numpy = cupy.asnumpy(x)
+        for i in range(h):
+            for j in range(w):
+                for k in range(c):
+                    elem = x_numpy[i, j, k]
+                    expected[i, j, elem] = min(expected[i, j, elem], k)
+        testing.assert_array_equal(y, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_gather(self, order):
+        @jit.rawkernel()
+        def gather(x, y, z):
+            i = jit.threadIdx.x
+            map_ = x[i]
+            input_ = y
+            result = z[i]
+            jit.thrust.gather(
+                jit.thrust.device, map_.begin(), map_.end(),
+                input_.begin(), result.begin())
+
+        n1, n2 = (128, 160)
+        x = testing.shaped_random(
+            (n1, n2), dtype=cupy.int32, scale=n2, order=order, seed=0)
+        y = testing.shaped_random(
+            (n2,), dtype=cupy.float32, order=order, seed=1)
+        z = cupy.zeros((n1, n2), cupy.float32)
+        gather[1, n1](x, y, z)
+        expected = y[x]
+        testing.assert_array_equal(z, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_inclusive_scan(self, order):
+        @jit.rawkernel()
+        def inclusive_scan(x, y):
+            i = jit.threadIdx.x
+            array = x[i]
+            result = y[i]
+            jit.thrust.inclusive_scan(
+                jit.thrust.seq, array.begin(), array.end(), result.begin())
+
+        n1, n2 = (128, 160)
+        x = testing.shaped_random(
+            (n1, n2), dtype=numpy.int32, order=order)
+        y = cupy.zeros((n1, n2), dtype=cupy.int32)
+        inclusive_scan[1, n1](x, y)
+
+        expected = x.cumsum(axis=-1)
+        testing.assert_array_equal(y, expected)
+
+    def test_inclusive_scan_by_key(self):
+        @jit.rawkernel()
+        def inclusive_scan_by_key(key, value):
+            jit.thrust.inclusive_scan_by_key(
+                jit.thrust.device, key.begin(), key.end(),
+                value.begin(), value.begin())
+
+        key = cupy.array([0, 0, 0, 1, 1, 2, 3, 3, 3, 3])
+        value = cupy.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
+        inclusive_scan_by_key[1, 1](key, value)
+
+        expected = cupy.array([1, 2, 3, 1, 2, 1, 1, 2, 3, 4])
+        testing.assert_array_equal(value, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_inner_product(self, order):
+        @jit.rawkernel()
+        def inner_product(a, b, c):
+            i = jit.threadIdx.x
+            c[i] = jit.thrust.inner_product(
+                jit.thrust.device, a[i].begin(), a[i].end(),
+                b[i].begin(), 0.)
+
+        (n1, n2) = (128, 160)
+        x = testing.shaped_random(
+            (n1, n2), dtype=numpy.float32, order=order, seed=0)
+        y = testing.shaped_random(
+            (n1, n2), dtype=numpy.float32, order=order, seed=1)
+        z = cupy.zeros((n1,), dtype=numpy.float32)
+        inner_product[1, n1](x, y, z)
+
+        expected = (x * y).sum(axis=1)
+        testing.assert_allclose(z, expected, rtol=1e-6)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_is_sorted(self, order):
+        @jit.rawkernel()
+        def is_sorted(x, out):
+            i = jit.threadIdx.x
+            out[i] = jit.thrust.is_sorted(
+                jit.thrust.device, x[i].begin(), x[i].end())
+
+        x = cupy.array([
+            [1, 4, 2, 8, 5, 7],
+            [1, 2, 4, 5, 7, 8],
+        ], order=order)
+        out = cupy.array([False, False], numpy.bool_)
+        is_sorted[1, 2](x, out)
+
+        expected = cupy.array([False, True])
+        testing.assert_array_equal(out, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_is_sorted_until(self, order):
+        @jit.rawkernel()
+        def is_sorted_until(x, out):
+            i = jit.threadIdx.x
+            it = jit.thrust.is_sorted_until(
+                jit.thrust.device, x[i].begin(), x[i].end())
+            out[i] = it - x[i].begin()
+
+        x = cupy.array([
+            [1, 2, 3, 4, 1, 2, 3, 4],
+            [1, 2, 4, 1, 2, 3, 5, 5],
+        ], order=order)
+        out = cupy.zeros((2,), dtype=numpy.int64)
+        is_sorted_until[1, 2](x, out)
+
+        expected = cupy.array([4, 3])
+        testing.assert_array_equal(out, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_lower_bound(self, order):
+        @jit.rawkernel()
+        def lower_bound(x, v, out):
+            i = jit.threadIdx.x
+            it = jit.thrust.lower_bound(
+                jit.thrust.seq, x.begin(), x.end(), v[i])
+            out[i] = it - x.begin()
+
+        n1, n2 = (128, 160)
+        x = testing.shaped_random((n2,), dtype=numpy.int32, order=order)
+        x = cupy.sort(x)
+        values = testing.shaped_random((n1,), dtype=numpy.int32, order=order)
+        out = cupy.zeros(n1, dtype=numpy.int32)
+        lower_bound[1, n1](x, values, out)
+
+        expected = cupy.searchsorted(x, values, side='left')
+        testing.assert_array_equal(out, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_lower_bound_vec(self, order):
+        @jit.rawkernel()
+        def lower_bound(x, v, out):
+            i = jit.threadIdx.x
+            jit.thrust.lower_bound(
+                jit.thrust.seq, x.begin(), x.end(),
+                v[i].begin(), v[i].end(), out[i].begin())
+
+        n1, n2, n3 = (128, 160, 200)
+        x = testing.shaped_random(
+            (n2,), dtype=numpy.int32, scale=200, order=order, seed=0)
+        x = cupy.sort(x)
+        values = testing.shaped_random(
+            (n1, n3), dtype=numpy.int32, scale=200, order=order, seed=1)
+        out = cupy.zeros((n1, n3), dtype=numpy.int32)
+        lower_bound[1, n1](x, values, out)
+
+        expected = cupy.searchsorted(x, values, side='left')
+        testing.assert_array_equal(out, expected)
+
+    def test_make_constant_iterator(self):
+        @jit.rawkernel()
+        def make_constant_iterator(x):
+            i = jit.threadIdx.x
+            it = jit.thrust.make_constant_iterator(i * i)
+            x[i] = it[i]
+
+        n = 128
+        out = cupy.zeros((n,), numpy.int32)
+        make_constant_iterator[1, n](out)
+
+        expected = cupy.arange(n) ** 2
+        testing.assert_array_equal(out, expected)
+
+    def test_make_counting_iterator(self):
+        @jit.rawkernel()
+        def make_counting_iterator(x):
+            i = jit.threadIdx.x
+            it = jit.thrust.make_counting_iterator(i * i)
+            x[i] = it[i]
+
+        n = 128
+        out = cupy.zeros((n,), numpy.int32)
+        make_counting_iterator[1, n](out)
+
+        expected = cupy.arange(n) ** 2 + cupy.arange(n)
+        testing.assert_array_equal(out, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_mismatch_iterator(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support pair of pointer type')
+
+        @jit.rawkernel()
+        def mismatch(x1, x2, out1, out2):
+            i = jit.threadIdx.x
+            x1_array = x1[i]
+            x2_array = x2[i]
+            pair = jit.thrust.mismatch(
+                jit.thrust.device,
+                x1_array.begin(),
+                x1_array.end(),
+                x2_array.begin(),
+            )
+            out1[i] = pair[0] - x1_array.begin()
+            out2[i] = pair[1] - x2_array.begin()
+
+        h, w = (5, 256)
+        x1 = testing.shaped_random(
+            (h, w), dtype=numpy.float32, scale=20000, order=order, seed=0)
+        x2 = x1.copy()
+        x2[0][0] = -1
+        x2[1][100] = -1
+        x2[2][30] = -1
+        x2[3][200] = -1
+        out1 = cupy.zeros(5, numpy.int32)
+        out2 = cupy.zeros(5, numpy.int32)
+        mismatch[1, 5](x1, x2, out1, out2)
+
+        testing.assert_array_equal(out1, [0, 100, 30, 200, w])
+        testing.assert_array_equal(out2, [0, 100, 30, 200, w])
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_reduce(self, order):
+        @jit.rawkernel()
+        def reduce(x, y):
+            i = jit.threadIdx.x
+            y[i] = jit.thrust.reduce(
+                jit.thrust.device, x[i].begin(), x[i].end())
+
+        (n1, n2) = (128, 160)
+        x = testing.shaped_random((n1, n2), dtype=numpy.int64, order=order)
+        out = cupy.zeros((n1,), numpy.int32)
+        reduce[1, n1](x, out)
+
+        expected = x.sum(axis=-1)
+        testing.assert_array_equal(out, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_reduce_init(self, order):
+        @jit.rawkernel()
+        def reduce(x, y):
+            i = jit.threadIdx.x
+            y[i] = jit.thrust.reduce(
+                jit.thrust.device, x[i].begin(), x[i].end(), 10)
+
+        (n1, n2) = (128, 160)
+        x = testing.shaped_random((n1, n2), dtype=numpy.int64, order=order)
+        out = cupy.zeros((n1,), numpy.int32)
+        reduce[1, n1](x, out)
+
+        expected = x.sum(axis=-1) + 10
+        testing.assert_array_equal(out, expected)
+
+    def test_reduce_by_key(self):
+        @jit.rawkernel()
+        def reduce_by_key(keys, values, keys_out, values_out, size1, size2):
+            ret = jit.thrust.reduce_by_key(
+                jit.thrust.device, keys.begin(), keys.end(),
+                values.begin(), keys_out.begin(), values_out.begin())
+            size1[0] = ret[0] - keys_out.begin()
+            size2[0] = ret[1] - values_out.begin()
+
+        keys = cupy.array([1, 3, 3, 3, 2, 2, 1], dtype=numpy.int32)
+        values = cupy.array([9, 8, 7, 6, 5, 4, 3], dtype=numpy.int32)
+        keys_out = cupy.zeros((7,), dtype=numpy.int32)
+        values_out = cupy.zeros((7,), dtype=numpy.int32)
+        size1 = cupy.zeros((1,), numpy.int32)
+        size2 = cupy.zeros((2,), numpy.int32)
+        reduce_by_key[1, 1](keys, values, keys_out, values_out, size1, size2)
+
+        testing.assert_array_equal(size1[0], 4)
+        testing.assert_array_equal(size2[0], 4)
+        testing.assert_array_equal(
+            keys_out[:4], cupy.array([1, 3, 2, 1], dtype=numpy.int32))
+        testing.assert_array_equal(
+            values_out[:4], cupy.array([9, 21, 9, 3], dtype=numpy.int32))
+
+    def test_remove(self):
+        @jit.rawkernel()
+        def remove_(x, size):
+            ptr = jit.thrust.remove(
+                jit.thrust.device, x.begin(), x.end(), 1)
+            size[0] = ptr - x.begin()
+
+        x = cupy.array([3, 1, 4, 1, 5, 9], dtype=numpy.int32)
+        size = cupy.zeros((1,), numpy.int32)
+        remove_[1, 1](x, size)
+
+        testing.assert_array_equal(size[0], 4)
+        testing.assert_array_equal(
+            x[:4], cupy.array([3, 4, 5, 9], dtype=numpy.int32))
+
+    def test_remove_copy(self):
+        @jit.rawkernel()
+        def remove_copy(x, result, size):
+            ptr = jit.thrust.remove_copy(
+                jit.thrust.device, x.begin(), x.end(), result.begin(), 1)
+            size[0] = ptr - result.begin()
+
+        x = cupy.array([3, 1, 4, 1, 5, 9], dtype=numpy.int32)
+        result = cupy.zeros((6,), dtype=numpy.int32)
+        size = cupy.zeros((1,), numpy.int32)
+        remove_copy[1, 1](x, result, size)
+
+        testing.assert_array_equal(size[0], 4)
+        testing.assert_array_equal(
+            result[:4], cupy.array([3, 4, 5, 9], dtype=numpy.int32))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_replace(self, order):
+        @jit.rawkernel()
+        def replace(x):
+            i = jit.threadIdx.x
+            jit.thrust.replace(
+                jit.thrust.device, x[i].begin(), x[i].end(), 0, 999)
+
+        (n1, n2) = (128, 160)
+        x = testing.shaped_random((n1, n2), dtype=numpy.int32, order=order)
+        expected = x.copy()
+        replace[1, n1](x)
+
+        mask = expected == 0
+        assert (mask.sum(axis=1) > 0).all()
+        expected[mask] = 999
+        testing.assert_array_equal(x, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_replace_copy(self, order):
+        @jit.rawkernel()
+        def replace_copy(x, out):
+            i = jit.threadIdx.x
+            jit.thrust.replace_copy(
+                jit.thrust.device, x[i].begin(), x[i].end(),
+                out[i].begin(), 0, 999)
+
+        (n1, n2) = (128, 160)
+        x = testing.shaped_random((n1, n2), dtype=numpy.int32, order=order)
+        out = cupy.zeros((n1, n2), dtype=numpy.int32)
+        replace_copy[1, n1](x, out)
+
+        expected = x.copy()
+        mask = x == 0
+        assert (mask.sum(axis=1) > 0).all()
+        expected[mask] = 999
+        testing.assert_array_equal(out, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_reverse(self, order):
+        @jit.rawkernel()
+        def reverse(x):
+            i = jit.threadIdx.x
+            jit.thrust.reverse(
+                jit.thrust.device, x[i].begin(), x[i].end())
+
+        (n1, n2) = (128, 160)
+        x = testing.shaped_random((n1, n2), dtype=numpy.int32, order=order)
+        expected = x[:, ::-1].copy()
+        reverse[1, n1](x)
+
+        testing.assert_array_equal(x, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_reverse_copy(self, order):
+        @jit.rawkernel()
+        def reverse_copy(x, out):
+            i = jit.threadIdx.x
+            jit.thrust.reverse_copy(
+                jit.thrust.device, x[i].begin(), x[i].end(), out[i].begin())
+
+        (n1, n2) = (128, 160)
+        x = testing.shaped_random((n1, n2), dtype=numpy.int32, order=order)
+        out = cupy.zeros((n1, n2), dtype=numpy.int32)
+        reverse_copy[1, n1](x, out)
+
+        expected = x[:, ::-1]
+        testing.assert_array_equal(out, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_scatter(self, order):
+        @jit.rawkernel()
+        def scatter(values, map, result):
+            jit.thrust.scatter(
+                jit.thrust.device, values.begin(), values.end(),
+                map.begin(), result.begin())
+
+        values = cupy.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=numpy.int32)
+        map = cupy.array([0, 5, 1, 6, 2, 7, 3, 8, 4, 9], dtype=numpy.int32)
+        result = cupy.zeros((10,), dtype=numpy.int64)
+        scatter[1, 1](values, map, result)
+
+        testing.assert_array_equal(
+            result,
+            cupy.array([1, 3, 5, 7, 9, 2, 4, 6, 8, 10]),
+        )
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_sequence(self, order):
+        @jit.rawkernel()
+        def sequence(x):
+            i = jit.threadIdx.x
+            jit.thrust.sequence(jit.thrust.device, x[i].begin(), x[i].end())
+
+        (n1, n2) = (128, 160)
+        x = cupy.zeros((n1, n2), dtype=numpy.int32)
+        sequence[1, n1](x)
+
+        expected = cupy.arange(n2, dtype=numpy.int32)
+        expected = expected + cupy.zeros((n1, n2), dtype=numpy.int32)
+        testing.assert_array_equal(x, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_sequence_init(self, order):
+        @jit.rawkernel()
+        def sequence(x):
+            i = jit.threadIdx.x
+            jit.thrust.sequence(jit.thrust.device, x[i].begin(), x[i].end(), 1)
+
+        (n1, n2) = (128, 160)
+        x = cupy.zeros((n1, n2), dtype=numpy.int32)
+        sequence[1, n1](x)
+
+        expected = cupy.arange(n2, dtype=numpy.int32)
+        expected = expected + cupy.ones((n1, n2), dtype=numpy.int32)
+        testing.assert_array_equal(x, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_sequence_step(self, order):
+        @jit.rawkernel()
+        def sequence(x):
+            i = numpy.int32(jit.threadIdx.x)
+            jit.thrust.sequence(
+                jit.thrust.device, x[i].begin(), x[i].end(), i, 10)
+
+        (n1, n2) = (128, 160)
+        x = cupy.zeros((n1, n2), dtype=numpy.int32)
+        sequence[1, n1](x)
+
+        expected = cupy.arange(n2, dtype=numpy.int32) * 10
+        expected = expected + cupy.arange(n1, dtype=numpy.int32)[:, None]
+        testing.assert_array_equal(x, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_set_difference(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support thrust set operations.')
+
+        @jit.rawkernel()
+        def set_difference(x, y, out, size):
+            it = jit.thrust.set_difference(
+                jit.thrust.device,
+                x.begin(), x.end(),
+                y.begin(), y.end(),
+                out.begin())
+            size[0] = it - out.begin()
+
+        x = cupy.array([0, 1, 3, 4, 5, 6, 9])
+        y = cupy.array([1, 3, 5, 7, 9])
+        out = cupy.zeros((10,), dtype=numpy.int64)
+        size = cupy.zeros((1,), dtype=numpy.int64)
+        set_difference[1, 1](x, y, out, size)
+
+        testing.assert_array_equal(size, cupy.array([3]))
+        testing.assert_array_equal(out[:3], cupy.array([0, 4, 6]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_set_difference_by_key(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support thrust set operations.')
+
+        @jit.rawkernel()
+        def set_difference_by_key(
+                keys1, keys2, values1, values2, keys_out, values_out, size):
+            it0, it1 = jit.thrust.set_difference_by_key(
+                jit.thrust.device,
+                keys1.begin(), keys1.end(),
+                keys2.begin(), keys2.end(),
+                values1.begin(), values2.begin(),
+                keys_out.begin(), values_out.begin()
+            )
+            size[0] = it0 - keys_out.begin()
+            size[1] = it1 - values_out.begin()
+
+        a_keys = cupy.array([0, 1, 3, 4, 5, 6, 9])
+        a_vals = cupy.array([2, 2, 2, 2, 2, 2, 2])
+        b_keys = cupy.array([1, 3, 5, 7, 9])
+        b_vals = cupy.array([1, 1, 1, 1, 1])
+        keys_out = cupy.zeros((10,), dtype=numpy.int64)
+        vals_out = cupy.zeros((10,), dtype=numpy.int64)
+        size = cupy.zeros((2,), dtype=numpy.int64)
+        set_difference_by_key[1, 1](
+            a_keys, b_keys, a_vals, b_vals, keys_out, vals_out, size)
+
+        testing.assert_array_equal(size, cupy.array([3, 3]))
+        testing.assert_array_equal(keys_out[:3], cupy.array([0, 4, 6]))
+        testing.assert_array_equal(vals_out[:3], cupy.array([2, 2, 2]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_set_intersection(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support thrust set operations.')
+
+        @jit.rawkernel()
+        def set_intersection(x, y, out, size):
+            it = jit.thrust.set_intersection(
+                jit.thrust.device,
+                x.begin(), x.end(),
+                y.begin(), y.end(),
+                out.begin())
+            size[0] = it - out.begin()
+
+        x = cupy.array([1, 3, 5, 7, 9, 11])
+        y = cupy.array([1, 1, 2, 3, 5, 8, 13])
+        out = cupy.zeros((10,), dtype=numpy.int64)
+        size = cupy.zeros((1,), dtype=numpy.int64)
+        set_intersection[1, 1](x, y, out, size)
+
+        testing.assert_array_equal(size, cupy.array([3]))
+        testing.assert_array_equal(out[:3], cupy.array([1, 3, 5]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_set_intersection_by_key(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support thrust set operations.')
+
+        @jit.rawkernel()
+        def set_intersection_by_key(
+                keys1, keys2, values1, keys_out, values_out, size):
+            it0, it1 = jit.thrust.set_intersection_by_key(
+                jit.thrust.device,
+                keys1.begin(), keys1.end(),
+                keys2.begin(), keys2.end(),
+                values1.begin(), keys_out.begin(), values_out.begin()
+            )
+            size[0] = it0 - keys_out.begin()
+            size[1] = it1 - values_out.begin()
+
+        a_keys = cupy.array([1, 3, 5, 7, 9, 11])
+        a_vals = cupy.array([2, 2, 2, 2, 2, 2, 2])
+        b_keys = cupy.array([1, 1, 2, 3, 5, 8, 13])
+        keys_out = cupy.zeros((10,), dtype=numpy.int64)
+        vals_out = cupy.zeros((10,), dtype=numpy.int64)
+        size = cupy.zeros((2,), dtype=numpy.int64)
+        set_intersection_by_key[1, 1](
+            a_keys, b_keys, a_vals, keys_out, vals_out, size)
+
+        testing.assert_array_equal(size, cupy.array([3, 3]))
+        testing.assert_array_equal(keys_out[:3], cupy.array([1, 3, 5]))
+        testing.assert_array_equal(vals_out[:3], cupy.array([2, 2, 2]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_set_symmetric_difference(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support thrust set operations.')
+
+        @jit.rawkernel()
+        def set_symmetric_difference(x, y, out, size):
+            it = jit.thrust.set_symmetric_difference(
+                jit.thrust.device,
+                x.begin(), x.end(),
+                y.begin(), y.end(),
+                out.begin())
+            size[0] = it - out.begin()
+
+        x = cupy.array([0, 1, 2, 4, 6, 7])
+        y = cupy.array([1, 2, 5, 8])
+        out = cupy.zeros((10,), dtype=numpy.int64)
+        size = cupy.zeros((1,), dtype=numpy.int64)
+        set_symmetric_difference[1, 1](x, y, out, size)
+
+        testing.assert_array_equal(size, cupy.array([6]))
+        testing.assert_array_equal(out[:6], cupy.array([0, 4, 5, 6, 7, 8]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_set_symmetric_difference_by_key(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support thrust set operations.')
+
+        @jit.rawkernel()
+        def set_symmetric_difference_by_key(
+                keys1, keys2, values1, values2, keys_out, values_out, size):
+            it0, it1 = jit.thrust.set_symmetric_difference_by_key(
+                jit.thrust.device,
+                keys1.begin(), keys1.end(),
+                keys2.begin(), keys2.end(),
+                values1.begin(), values2.begin(),
+                keys_out.begin(), values_out.begin()
+            )
+            size[0] = it0 - keys_out.begin()
+            size[1] = it1 - values_out.begin()
+
+        a_keys = cupy.array([0, 1, 2, 4, 6, 7])
+        a_vals = cupy.array([1, 1, 1, 1, 1, 1])
+        b_keys = cupy.array([1, 2, 5, 8])
+        b_vals = cupy.array([2, 2, 2, 2])
+        keys_out = cupy.zeros((10,), dtype=numpy.int64)
+        vals_out = cupy.zeros((10,), dtype=numpy.int64)
+        size = cupy.zeros((2,), dtype=numpy.int64)
+        set_symmetric_difference_by_key[1, 1](
+            a_keys, b_keys, a_vals, b_vals, keys_out, vals_out, size)
+
+        testing.assert_array_equal(size, cupy.array([6, 6]))
+        testing.assert_array_equal(
+            keys_out[:6], cupy.array([0, 4, 5, 6, 7, 8]))
+        testing.assert_array_equal(
+            vals_out[:6], cupy.array([1, 1, 2, 1, 1, 2]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_set_union(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support thrust set operations.')
+
+        @jit.rawkernel()
+        def set_union(x, y, out, size):
+            it = jit.thrust.set_union(
+                jit.thrust.device,
+                x.begin(), x.end(),
+                y.begin(), y.end(),
+                out.begin())
+            size[0] = it - out.begin()
+
+        x = cupy.array([0, 1, 2, 4, 6, 7])
+        y = cupy.array([1, 2, 5, 8])
+        out = cupy.zeros((10,), dtype=numpy.int64)
+        size = cupy.zeros((1,), dtype=numpy.int64)
+        set_union[1, 1](x, y, out, size)
+
+        testing.assert_array_equal(size, cupy.array([8]))
+        testing.assert_array_equal(
+            out[:8], cupy.array([0, 1, 2, 4, 5, 6, 7, 8]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_set_union_by_key(self, order):
+        if runtime.is_hip:
+            pytest.xfail('HIP does not support thrust set operations.')
+
+        @jit.rawkernel()
+        def set_union_by_key(
+                keys1, keys2, values1, values2, keys_out, values_out, size):
+            it0, it1 = jit.thrust.set_union_by_key(
+                jit.thrust.device,
+                keys1.begin(), keys1.end(),
+                keys2.begin(), keys2.end(),
+                values1.begin(), values2.begin(),
+                keys_out.begin(), values_out.begin()
+            )
+            size[0] = it0 - keys_out.begin()
+            size[1] = it1 - values_out.begin()
+
+        a_keys = cupy.array([0, 1, 2, 4, 6, 7])
+        a_vals = cupy.array([1, 1, 1, 1, 1, 1])
+        b_keys = cupy.array([1, 2, 5, 8])
+        b_vals = cupy.array([2, 2, 2, 2])
+        keys_out = cupy.zeros((10,), dtype=numpy.int64)
+        vals_out = cupy.zeros((10,), dtype=numpy.int64)
+        size = cupy.zeros((2,), dtype=numpy.int64)
+        set_union_by_key[1, 1](
+            a_keys, b_keys, a_vals, b_vals, keys_out, vals_out, size)
+
+        testing.assert_array_equal(size, cupy.array([8, 8]))
+        testing.assert_array_equal(
+            keys_out[:8], cupy.array([0, 1, 2, 4, 5, 6, 7, 8]))
+        testing.assert_array_equal(
+            vals_out[:8], cupy.array([1, 1, 1, 1, 2, 1, 1, 2]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_sort(self, order):
+        if runtime.is_hip:
+            pytest.skip('See https://github.com/cupy/cupy/pull/7162')
+
+        @jit.rawkernel()
+        def sort(x):
+            i = jit.threadIdx.x
+            array = x[i]
+            jit.thrust.sort(jit.thrust.device, array.begin(), array.end())
+
+        h, w = (256, 256)
+        x = testing.shaped_random(
+            (h, w), dtype=numpy.int32, scale=4, order=order)
+        x_numpy = cupy.asnumpy(x)
+        sort[1, 256](x)
+
+        testing.assert_array_equal(x, numpy.sort(x_numpy, axis=-1))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_sort_by_key(self, order):
+        if runtime.is_hip:
+            pytest.skip('See https://github.com/cupy/cupy/pull/7162')
+
+        @jit.rawkernel()
+        def sort_by_key(x, y):
+            i = jit.threadIdx.x
+            x_array = x[i]
+            y_array = y[i]
+            jit.thrust.sort_by_key(
+                jit.thrust.device,
+                x_array.begin(),
+                x_array.end(),
+                y_array.begin(),
+            )
+
+        h, w = (256, 256)
+        x = cupy.arange(h * w, dtype=numpy.int32)
+        cupy.random.shuffle(x)
+        x = x.reshape(h, w)
+        y = testing.shaped_random(
+            (h, w), dtype=numpy.int32, scale=20000, order=order, seed=1)
+        x_numpy = cupy.asnumpy(x)
+        y_numpy = cupy.asnumpy(y)
+        sort_by_key[1, 256](x, y)
+
+        indices = numpy.argsort(x_numpy, axis=-1)
+        x_expected = numpy.array([a[i] for a, i in zip(x_numpy, indices)])
+        y_expected = numpy.array([a[i] for a, i in zip(y_numpy, indices)])
+        testing.assert_array_equal(x, x_expected)
+        testing.assert_array_equal(y, y_expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_stable_sort(self, order):
+        if runtime.is_hip:
+            pytest.skip('See https://github.com/cupy/cupy/pull/7162')
+
+        @jit.rawkernel()
+        def stable_sort(x):
+            i = jit.threadIdx.x
+            array = x[i]
+            jit.thrust.stable_sort(
+                jit.thrust.device, array.begin(), array.end())
+
+        h, w = (256, 256)
+        x = testing.shaped_random(
+            (h, w), dtype=numpy.int32, scale=4, order=order)
+        x_numpy = cupy.asnumpy(x)
+        stable_sort[1, 256](x)
+
+        testing.assert_array_equal(x, numpy.sort(x_numpy, axis=-1))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_stable_sort_by_key(self, order):
+        if runtime.is_hip:
+            pytest.skip('See https://github.com/cupy/cupy/pull/7162')
+
+        @jit.rawkernel()
+        def stable_sort_by_key(x, y):
+            i = jit.threadIdx.x
+            x_array = x[i]
+            y_array = y[i]
+            jit.thrust.stable_sort_by_key(
+                jit.thrust.device,
+                x_array.begin(),
+                x_array.end(),
+                y_array.begin(),
+            )
+
+        h, w = (256, 256)
+        x = cupy.arange(h * w, dtype=numpy.int32)
+        cupy.random.shuffle(x)
+        x = x.reshape(h, w)
+        y = testing.shaped_random(
+            (h, w), dtype=numpy.int32, scale=20000, order=order, seed=1)
+        x_numpy = cupy.asnumpy(x)
+        y_numpy = cupy.asnumpy(y)
+        stable_sort_by_key[1, 256](x, y)
+
+        indices = numpy.argsort(x_numpy, axis=-1)
+        x_expected = numpy.array([a[i] for a, i in zip(x_numpy, indices)])
+        y_expected = numpy.array([a[i] for a, i in zip(y_numpy, indices)])
+        testing.assert_array_equal(x, x_expected)
+        testing.assert_array_equal(y, y_expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_swap_ranges(self, order):
+        if runtime.is_hip:
+            pytest.skip('See https://github.com/cupy/cupy/pull/7162')
+
+        @jit.rawkernel()
+        def swap_ranges(x, y):
+            i = jit.threadIdx.x
+            jit.thrust.swap_ranges(
+                jit.thrust.device, x[i].begin(), x[i].end(), y[i].begin())
+
+        h, w = (256, 256)
+        x = testing.shaped_random(
+            (h, w), dtype=numpy.int32, scale=4, order=order, seed=0)
+        y = testing.shaped_random(
+            (h, w), dtype=numpy.int32, scale=4, order=order, seed=1)
+        x_expected = y.copy()
+        y_expected = x.copy()
+        swap_ranges[1, 256](x, y)
+
+        testing.assert_array_equal(x_expected, x)
+        testing.assert_array_equal(y_expected, y)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_unique(self, order):
+
+        @jit.rawkernel()
+        def unique(x, size):
+            it = jit.thrust.unique(jit.thrust.device, x.begin(), x.end())
+            size[0] = it - x.begin()
+
+        x = cupy.array([1, 3, 3, 3, 2, 2, 1])
+        size = cupy.zeros((1,), dtype=numpy.int32)
+        unique[1, 1](x, size)
+
+        testing.assert_array_equal(size, cupy.array([4], dtype=numpy.int32))
+        testing.assert_array_equal(x[:4], cupy.array([1, 3, 2, 1]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_unique_by_key(self, order):
+
+        @jit.rawkernel()
+        def unique_by_key(keys, values, size):
+            it0, it1 = jit.thrust.unique_by_key(
+                jit.thrust.device, keys.begin(), keys.end(), values.begin())
+            size[0] = it0 - keys.begin()
+            size[1] = it1 - values.begin()
+
+        keys = cupy.array([1, 3, 3, 3, 2, 2, 1])
+        values = cupy.array([9, 8, 7, 6, 5, 4, 3])
+        size = cupy.zeros((2,), dtype=numpy.int32)
+        unique_by_key[1, 1](keys, values, size)
+
+        testing.assert_array_equal(size, cupy.array([4, 4], dtype=numpy.int32))
+        testing.assert_array_equal(keys[:4], cupy.array([1, 3, 2, 1]))
+        testing.assert_array_equal(values[:4], cupy.array([9, 8, 5, 3]))
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_upper_bound(self, order):
+        @jit.rawkernel()
+        def upper_bound(x, v, out):
+            i = jit.threadIdx.x
+            it = jit.thrust.upper_bound(
+                jit.thrust.seq, x.begin(), x.end(), v[i])
+            out[i] = it - x.begin()
+
+        n1, n2 = (128, 160)
+        x = testing.shaped_random((n2,), dtype=numpy.int32, order=order)
+        x = cupy.sort(x)
+        values = testing.shaped_random((n1,), dtype=numpy.int32, order=order)
+        out = cupy.zeros(n1, dtype=numpy.int32)
+        upper_bound[1, n1](x, values, out)
+
+        expected = cupy.searchsorted(x, values, side='right')
+        testing.assert_array_equal(out, expected)
+
+    @pytest.mark.parametrize('order', ['C', 'F'])
+    def test_upper_bound_vec(self, order):
+        @jit.rawkernel()
+        def upper_bound(x, v, out):
+            i = jit.threadIdx.x
+            jit.thrust.upper_bound(
+                jit.thrust.seq, x.begin(), x.end(),
+                v[i].begin(), v[i].end(), out[i].begin())
+
+        n1, n2, n3 = (128, 160, 200)
+        x = testing.shaped_random(
+            (n2,), dtype=numpy.int32, scale=200, order=order, seed=0)
+        x = cupy.sort(x)
+        values = testing.shaped_random(
+            (n1, n3), dtype=numpy.int32, scale=200, order=order, seed=1)
+        out = cupy.zeros((n1, n3), dtype=numpy.int32)
+        upper_bound[1, n1](x, values, out)
+
+        expected = cupy.searchsorted(x, values, side='right')
+        testing.assert_array_equal(out, expected)
diff --git a/tests/cupyx_tests/linalg_tests/__init__.py b/tests/cupyx_tests/linalg_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/linalg_tests/sparse_tests/__init__.py b/tests/cupyx_tests/linalg_tests/sparse_tests/__init__.py
new file mode 100644
index 0000000..611a436
--- /dev/null
+++ b/tests/cupyx_tests/linalg_tests/sparse_tests/__init__.py
@@ -0,0 +1,8 @@
+import pytest
+
+import cupy
+
+
+if cupy.cuda.runtime.is_hip:
+    pytest.skip('HIP sparse support is not yet ready',
+                allow_module_level=True)
diff --git a/tests/cupyx_tests/linalg_tests/sparse_tests/test_solve.py b/tests/cupyx_tests/linalg_tests/sparse_tests/test_solve.py
new file mode 100644
index 0000000..7d8b402
--- /dev/null
+++ b/tests/cupyx_tests/linalg_tests/sparse_tests/test_solve.py
@@ -0,0 +1,67 @@
+import unittest
+
+import numpy
+import pytest
+try:
+    import scipy.linalg
+    import scipy.sparse
+    import scipy.stats
+
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+import cupy as cp
+from cupy import testing
+from cupy.testing import _condition
+import cupyx
+import cupyx.scipy.sparse as sp
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+}))
+@unittest.skipUnless(scipy_available, 'requires scipy')
+class TestLschol(unittest.TestCase):
+
+    def setUp(self):
+        rvs = scipy.stats.randint(0, 15).rvs
+        self.A = scipy.sparse.random(
+            50, 50, density=0.2, data_rvs=rvs, dtype=self.dtype)
+        self.b = numpy.random.randint(5, size=50)
+        # symmetric and positive definite
+        self.A = self.A.T*self.A + 10*scipy.sparse.eye(50)
+        self.b = self.A.T*self.b
+        # initial scipy results by dense cholesky method.
+        L = scipy.linalg.cho_factor(self.A.todense())
+        self.x = scipy.linalg.cho_solve(L, self.b)
+        if self.dtype == numpy.float64:
+            self.decimal = 8
+        else:
+            self.decimal = 3
+
+    def test_size(self):
+        A = sp.csr_matrix(self.A, dtype=self.dtype)
+        b = cp.array(numpy.append(self.b, [1]), dtype=self.dtype)
+        with pytest.raises(ValueError):
+            cupyx.linalg.sparse.lschol(A, b)
+
+    def test_shape(self):
+        A = sp.csr_matrix(self.A, dtype=self.dtype)
+        b = cp.array(numpy.tile(self.b, (2, 1)), dtype=self.dtype)
+        with pytest.raises(ValueError):
+            cupyx.linalg.sparse.lschol(A, b)
+
+    @_condition.retry(10)
+    def test_csrmatrix(self):
+        A = sp.csr_matrix(self.A, dtype=self.dtype)
+        b = cp.array(self.b, dtype=self.dtype)
+        x = cupyx.linalg.sparse.lschol(A, b)
+        testing.assert_array_almost_equal(x, self.x, decimal=self.decimal)
+
+    @_condition.retry(10)
+    def test_ndarray(self):
+        A = cp.array(self.A.A, dtype=self.dtype)
+        b = cp.array(self.b, dtype=self.dtype)
+        x = cupyx.linalg.sparse.lschol(A, b)
+        testing.assert_array_almost_equal(x, self.x, decimal=self.decimal)
diff --git a/tests/cupyx_tests/linalg_tests/test_solve.py b/tests/cupyx_tests/linalg_tests/test_solve.py
new file mode 100644
index 0000000..b7d0053
--- /dev/null
+++ b/tests/cupyx_tests/linalg_tests/test_solve.py
@@ -0,0 +1,84 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupyx import cusolver
+from cupy import testing
+import cupyx
+
+
+@testing.parameterize(*testing.product({
+    'size': [5, 9, 17, 33],
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@pytest.mark.xfail(runtime.is_hip,
+                   reason='rocSOLVER does not implement potrs yet.')
+class TestInvh(unittest.TestCase):
+
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_invh(self, xp):
+        a = self._create_symmetric_matrix(xp, self.size, self.dtype)
+        if xp == cupy:
+            return cupyx.linalg.invh(a)
+        else:
+            return numpy.linalg.inv(a)
+
+    def _create_symmetric_matrix(self, xp, n, dtype):
+        if dtype == numpy.complex128:
+            f_dtype = numpy.float64
+        elif dtype == numpy.complex64:
+            f_dtype = numpy.float32
+        else:
+            f_dtype = dtype
+        a = testing.shaped_random((n, n), xp, f_dtype, scale=1)
+        a = a + a.T + xp.eye(n, dtype=f_dtype) * n
+        if dtype in (numpy.complex64, numpy.complex128):
+            b = testing.shaped_random((n, n), xp, f_dtype, scale=1)
+            b = b - b.T
+            a = a + 1j * b
+        return a
+
+
+@testing.parameterize(*testing.product({
+    'size': [8],
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+class TestErrorInvh(unittest.TestCase):
+
+    @pytest.mark.skipif(
+        cupy.cuda.runtime.runtimeGetVersion() == 12000,
+        reason='This fails with CUDA 12.0.0 but pass in CUDA 12.0.1. (#7309)')
+    def test_invh(self):
+        a = self._create_symmetric_matrix(self.size, self.dtype)
+        with cupyx.errstate(linalg='raise'):
+            with self.assertRaises(numpy.linalg.LinAlgError):
+                cupyx.linalg.invh(a)
+
+    def _create_symmetric_matrix(self, n, dtype):
+        a = testing.shaped_random((n, n), cupy, dtype, scale=1)
+        a = a + a.T - cupy.eye(n, dtype=dtype)
+        return a
+
+
+# TODO: cusolver does not support nrhs > 1 for potrsBatched
+@testing.parameterize(*testing.product({
+    'shape': [(2, 3, 3)],
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+class TestXFailBatchedInvh(unittest.TestCase):
+
+    def test_invh(self):
+        if not cusolver.check_availability('potrsBatched'):
+            pytest.skip('potrsBatched is not available')
+        a = self._create_symmetric_matrix(self.shape, self.dtype)
+        with cupyx.errstate(linalg='raise'):
+            with self.assertRaises(numpy.linalg.LinAlgError):
+                cupyx.linalg.invh(a)
+
+    def _create_symmetric_matrix(self, shape, dtype):
+        a = testing.shaped_random(shape, cupy, dtype, scale=1)
+        a = a @ a.transpose(0, 2, 1)
+        return a
diff --git a/tests/cupyx_tests/profiler_tests/__init__.py b/tests/cupyx_tests/profiler_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/profiler_tests/test_benchmark.py b/tests/cupyx_tests/profiler_tests/test_benchmark.py
new file mode 100644
index 0000000..7889df3
--- /dev/null
+++ b/tests/cupyx_tests/profiler_tests/test_benchmark.py
@@ -0,0 +1,132 @@
+import unittest
+from unittest import mock
+
+import numpy
+
+import cupy
+from cupy import testing
+from cupyx import profiler
+from cupyx.profiler import _time
+
+
+class TestBenchmark(unittest.TestCase):
+
+    def test_cpu_routine(self):
+        with mock.patch('time.perf_counter',
+                        mock.Mock(side_effect=[2.4, 3.8, 3.8] * 10)):
+            with mock.patch('cupy.cuda.get_elapsed_time',
+                            mock.Mock(return_value=2500)):
+                mock_func = mock.Mock()
+                mock_func.__name__ = 'test_name_xxx'
+                x = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                y = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                assert mock_func.call_count == 0
+
+                perf = profiler.benchmark(
+                    mock_func, (x, y), n_repeat=10, n_warmup=3)
+
+                assert perf.name == 'test_name_xxx'
+                assert mock_func.call_count == 13
+                assert perf.cpu_times.shape == (10,)
+                assert perf.gpu_times.shape == (1, 10,)
+                assert (perf.cpu_times == 1.4).all()
+                assert (perf.gpu_times == 2.5).all()
+
+    @testing.multi_gpu(2)
+    def test_multigpu_routine(self):
+        with mock.patch('time.perf_counter',
+                        mock.Mock(side_effect=[2.4, 3.8, 3.8] * 10)):
+            with mock.patch('cupy.cuda.get_elapsed_time',
+                            mock.Mock(return_value=2500)):
+                mock_func = mock.Mock()
+                mock_func.__name__ = 'test_name_xxx'
+                x = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                y = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                assert mock_func.call_count == 0
+
+                perf = profiler.benchmark(
+                    mock_func, (x, y), n_repeat=10, n_warmup=3, devices=(0, 1))
+
+                assert perf.name == 'test_name_xxx'
+                assert mock_func.call_count == 13
+                assert perf.cpu_times.shape == (10,)
+                assert perf.gpu_times.shape == (2, 10,)
+                assert (perf.cpu_times == 1.4).all()
+                assert (perf.gpu_times == 2.5).all()
+
+    def test_benchmark_max_duration(self):
+        with mock.patch('time.perf_counter',
+                        mock.Mock(side_effect=[1., 2., 2.] * 6)):
+            with mock.patch('cupy.cuda.get_elapsed_time',
+                            mock.Mock(return_value=2500)):
+                mock_func = mock.Mock()
+                mock_func.__name__ = 'test_name_xxx'
+                x = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                y = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                assert mock_func.call_count == 0
+
+                perf = profiler.benchmark(
+                    mock_func, (x, y), n_warmup=3, max_duration=2.5)
+
+                assert perf.name == 'test_name_xxx'
+                assert mock_func.call_count == 6
+                assert perf.cpu_times.shape == (3,)
+                assert perf.gpu_times.shape == (1, 3)
+                assert (perf.cpu_times == 1.).all()
+                assert (perf.gpu_times == 2.5).all()
+
+    def test_benchmark_kwargs(self):
+        x = cupy.random.rand(5)
+        profiler.benchmark(
+            cupy.nonzero, kwargs={'a': x}, n_repeat=1, n_warmup=1)
+
+
+class TestPerfCaseResult(unittest.TestCase):
+    def test_show_gpu(self):
+        times = numpy.array([
+            [5.4, 7.1, 6.0, 5.4, 4.2],
+            [6.4, 4.3, 8.9, 9.6, 3.8],
+        ]) * 1e-6
+        perf = _time._PerfCaseResult('test_name_xxx', times, (0,))
+        expected = (
+            'test_name_xxx       :'
+            '    CPU:     5.620 us   +/-  0.943 '
+            '(min:     4.200 / max:     7.100) us '
+            '    GPU-0:     6.600 us   +/-  2.344 '
+            '(min:     3.800 / max:     9.600) us'
+        )
+        assert str(perf) == expected
+
+    def test_no_show_gpu(self):
+        times = numpy.array([
+            [5.4, 7.1, 6.0, 5.4, 4.2],
+            [6.4, 4.3, 8.9, 9.6, 3.8],
+        ]) * 1e-6
+        perf = _time._PerfCaseResult('test_name_xxx', times, (0,))
+        expected = (
+            'test_name_xxx       :'
+            '    CPU:     5.620 us   +/-  0.943 '
+            '(min:     4.200 / max:     7.100) us'
+        )
+        assert perf.to_str() == expected
+        # Checks if the result does not change.
+        assert perf.to_str() == expected
+
+    def test_single_show_gpu(self):
+        times = numpy.array([[5.4], [6.4]]) * 1e-6
+        perf = _time._PerfCaseResult('test_name_xxx', times, (0,))
+        assert str(perf) == ('test_name_xxx       :    CPU:     5.400 us '
+                             '    GPU-0:     6.400 us')
+
+    def test_single_no_show_gpu(self):
+        times = numpy.array([[5.4], [6.4]]) * 1e-6
+        perf = _time._PerfCaseResult('test_name_xxx', times, (0,))
+        assert perf.to_str() == 'test_name_xxx       :    CPU:     5.400 us'
+
+    def test_show_multigpu(self):
+        times = numpy.array([[5.4], [6.4], [7.0], [8.1]]) * 1e-6
+        perf = _time._PerfCaseResult('test_name_xxx', times, (0, 1, 2))
+        assert str(perf) == ('test_name_xxx       :    CPU:     5.400 us '
+                             '    GPU-0:     6.400 us '
+                             '    GPU-1:     7.000 us '
+                             '    GPU-2:     8.100 us')
diff --git a/tests/cupyx_tests/profiler_tests/test_profile.py b/tests/cupyx_tests/profiler_tests/test_profile.py
new file mode 100644
index 0000000..262e8b3
--- /dev/null
+++ b/tests/cupyx_tests/profiler_tests/test_profile.py
@@ -0,0 +1,29 @@
+import unittest
+from unittest import mock
+
+from cupyx import profiler
+
+
+class TestProfile(unittest.TestCase):
+
+    def test_profile(self):
+        start_patch = mock.patch('cupy_backends.cuda.libs.profiler.start')
+        stop_patch = mock.patch('cupy_backends.cuda.libs.profiler.stop')
+        with start_patch as start, stop_patch as stop:
+            with profiler.profile():
+                pass
+            start.assert_called_once_with()
+            stop.assert_called_once_with()
+
+    def test_err_case(self):
+        start_patch = mock.patch('cupy_backends.cuda.libs.profiler.start')
+        stop_patch = mock.patch('cupy_backends.cuda.libs.profiler.stop')
+        with start_patch as start, stop_patch as stop:
+            try:
+                with profiler.profile():
+                    raise Exception()
+            except Exception:
+                # ignore
+                pass
+            start.assert_called_once_with()
+            stop.assert_called_once_with()
diff --git a/tests/cupyx_tests/profiler_tests/test_time_range.py b/tests/cupyx_tests/profiler_tests/test_time_range.py
new file mode 100644
index 0000000..49041a3
--- /dev/null
+++ b/tests/cupyx_tests/profiler_tests/test_time_range.py
@@ -0,0 +1,98 @@
+import unittest
+from unittest import mock
+
+from cupy import cuda
+from cupyx import profiler
+
+
+@unittest.skipUnless(cuda.nvtx.available, 'nvtx is required for time_range')
+class TestTimeRange(unittest.TestCase):
+
+    def test_time_range(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePush')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            with profiler.time_range('test:time_range', color_id=-1):
+                pass
+            push.assert_called_once_with('test:time_range', -1)
+            pop.assert_called_once_with()
+
+    def test_time_range_with_ARGB(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePushC')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            with profiler.time_range('test:time_range_with_ARGB',
+                                     argb_color=0xFF00FF00):
+                pass
+            push.assert_called_once_with(
+                'test:time_range_with_ARGB', 0xFF00FF00)
+            pop.assert_called_once_with()
+
+    def test_time_range_err(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePush')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            try:
+                with profiler.time_range('test:time_range_error', -1):
+                    raise Exception()
+            except Exception:
+                pass
+            push.assert_called_once_with('test:time_range_error', -1)
+            pop.assert_called_once_with()
+
+    def test_time_range_as_decorator(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePush')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            @profiler.time_range()
+            def f():
+                pass
+            f()
+            # Default value of color id is -1
+            push.assert_called_once_with('f', -1)
+            pop.assert_called_once_with()
+
+    def test_time_range_as_decorator_with_ARGB(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePushC')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            @profiler.time_range(argb_color=0xFFFF0000)
+            def f():
+                pass
+            f()
+            push.assert_called_once_with('f', 0xFFFF0000)
+            pop.assert_called_once_with()
+
+    def test_time_range_as_decorator_err(self):
+        push_patch = mock.patch('cupy.cuda.nvtx.RangePush')
+        pop_patch = mock.patch('cupy.cuda.nvtx.RangePop')
+        with push_patch as push, pop_patch as pop:
+            @profiler.time_range()
+            def f():
+                raise Exception()
+            try:
+                f()
+            except Exception:
+                pass
+            # Default value of color id is -1
+            push.assert_called_once_with('f', -1)
+            pop.assert_called_once_with()
+
+
+class TestTimeRangeNVTXUnavailable(unittest.TestCase):
+
+    def setUp(self):
+        self.nvtx_available = cuda.nvtx.available
+        cuda.nvtx.available = False
+
+    def tearDown(self):
+        cuda.nvtx.available = self.nvtx_available
+
+    def test_time_range(self):
+        with self.assertRaises(RuntimeError):
+            with profiler.time_range(''):
+                pass
+
+    def test_time_range_decorator(self):
+        with self.assertRaises(RuntimeError):
+            profiler.time_range()
diff --git a/tests/cupyx_tests/scipy_tests/__init__.py b/tests/cupyx_tests/scipy_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/fft_tests/__init__.py b/tests/cupyx_tests/scipy_tests/fft_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/fft_tests/test_fft.py b/tests/cupyx_tests/scipy_tests/fft_tests/test_fft.py
new file mode 100644
index 0000000..b31f742
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/fft_tests/test_fft.py
@@ -0,0 +1,1786 @@
+import numpy as np
+try:
+    # scipy.fft is available since scipy v1.4.0+
+    import scipy.fft as scipy_fft
+except ImportError:
+    scipy_fft = None
+    scipy = None
+else:
+    import scipy
+import pytest
+
+import cupy as cp
+from cupy import testing
+from cupy.fft._fft import _default_fft_func, _fftn
+import cupyx.scipy.fft as cp_fft
+
+
+_irfft_skip_condition = (
+    int(cp.cuda.device.get_compute_capability()) < 70 and
+    10020 >= cp.cuda.runtime.runtimeGetVersion() >= 10010)
+
+
+def _fft_module(xp):
+    if xp is not np:
+        return cp_fft
+    else:
+        if scipy_fft is not None:
+            return scipy_fft
+        else:  # fallback to numpy when scipy is unavailable
+            return np.fft
+
+
+def _correct_np_dtype(xp, dtype, out):
+    # NumPy always transforms in double precision, cast output to correct type
+    if xp is np and scipy_fft is None:
+        if dtype in [np.float16, np.float32, np.complex64]:
+            if out.dtype.kind == 'f':
+                return out.astype(np.float32)
+            else:
+                return out.astype(np.complex64)
+    return out
+
+
+def _skip_forward_backward(norm):
+    if norm in ('backward', 'forward'):
+        if (scipy_fft is not None
+                and not (np.lib.NumpyVersion(scipy.__version__) >= '1.6.0')):
+            pytest.skip('forward/backward is supported by SciPy 1.6.0+')
+        elif (scipy_fft is None
+                and not (np.lib.NumpyVersion(np.__version__) >= '1.20.0')):
+            pytest.skip('forward/backward is supported by NumPy 1.20+')
+
+
+@testing.parameterize(*testing.product({
+    'n': [None, 0, 5, 10, 15],
+    'shape': [(9,), (10,), (10, 9), (10, 10)],
+    'axis': [-1, 0],
+    'norm': [None, 'backward', 'ortho', 'forward', '']
+}))
+class TestFft:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).fft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).fft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                  **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis)}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).fft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                  **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis), 'overwrite_x': True}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).fft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                  **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            plan = _fft_module(xp).get_fft_plan(x, shape=self.n,
+                                                axes=self.axis)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).fft(x, n=self.n, axis=self.axis)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).fft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.fft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.5.0')
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft_backend_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis)}
+            backend = cp_fft
+        else:
+            # scipy raises NotImplementedError if plan is not None
+            overwrite_kw = {'plan': None}
+            backend = 'scipy'
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.fft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).ifft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).ifft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                   **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis)}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).ifft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                   **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis), 'overwrite_x': True}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).ifft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                   **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            plan = _fft_module(xp).get_fft_plan(x, shape=self.n,
+                                                axes=self.axis)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).ifft(x, n=self.n, axis=self.axis)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).ifft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ifft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.5.0')
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft_backend_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis)}
+            backend = cp_fft
+        else:
+            # scipy raises NotImplementedError if plan is not None
+            overwrite_kw = {'plan': None}
+            backend = 'scipy'
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ifft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                 **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'shape': [(3, 4)],
+            's': [None, (1, 5)],
+            'axes': [None, (-2, -1), (-1, -2), (0,)],
+        })
+        + testing.product({
+            'shape': [(2, 3, 4)],
+            's': [None, (1, 5), (1, 4, 10)],
+            'axes': [None, (-2, -1), (-1, -2, -3)],
+        }),
+        testing.product({
+            'norm': [None, 'backward', 'ortho', 'forward', '']
+        })
+    )
+))
+class TestFft2:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft2(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).fft2(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft2_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).fft2(x, s=self.s, axes=self.axes,
+                                   norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft2_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes)}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).fft2(x, s=self.s, axes=self.axes, norm=self.norm,
+                                   **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft2_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes), 'overwrite_x': True}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).fft2(x, s=self.s, axes=self.axes, norm=self.norm,
+                                   **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft2_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            plan = _fft_module(xp).get_fft_plan(x, shape=self.s,
+                                                axes=self.axes)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).fft2(x, s=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).fft2(x, s=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft2_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.fft2(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.5.0')
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fft2_backend_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes)}
+            backend = cp_fft
+        else:
+            # scipy raises NotImplementedError if plan is not None
+            overwrite_kw = {'plan': None}
+            backend = 'scipy'
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.fft2(x, s=self.s, axes=self.axes, norm=self.norm,
+                                 **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft2(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).ifft2(
+            x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft2_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).ifft2(x, s=self.s, axes=self.axes,
+                                    norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft2_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes)}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).ifft2(x, s=self.s, axes=self.axes,
+                                    norm=self.norm, **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft2_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes), 'overwrite_x': True}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).ifft2(x, s=self.s, axes=self.axes,
+                                    norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft2_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            plan = _fft_module(xp).get_fft_plan(x, shape=self.s,
+                                                axes=self.axes)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).ifft2(x, s=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).ifft2(x, s=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft2_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ifft2(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.5.0')
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifft2_backend_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes)}
+            backend = cp_fft
+        else:
+            # scipy raises NotImplementedError if plan is not None
+            overwrite_kw = {'plan': None}
+            backend = 'scipy'
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ifft2(x, s=self.s, axes=self.axes, norm=self.norm,
+                                  **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'shape': [(3, 4)],
+            's': [None, (1, 5)],
+            'axes': [None, (-2, -1), (-1, -2), (0,)],
+        })
+        + testing.product({
+            'shape': [(2, 3, 4)],
+            's': [None, (1, 5), (1, 4, 10)],
+            'axes': [None, (0, 1), (-2, -1), (-1, -2, -3)],
+        })
+        + testing.product({
+            'shape': [(2, 3, 4, 5)],
+            's': [None],
+            'axes': [None, (0, 1, 2, 3)],
+        }),
+        testing.product({
+            'norm': [None, 'backward', 'ortho', 'forward', '']
+        })
+    )
+))
+class TestFftn:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).fftn(x, s=self.s, axes=self.axes,
+                                   norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).fftn(x, s=self.s, axes=self.axes,
+                                   norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes)}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).fftn(x, s=self.s, axes=self.axes, norm=self.norm,
+                                   **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes), 'overwrite_x': True}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).fftn(x, s=self.s, axes=self.axes, norm=self.norm,
+                                   **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            plan = _fft_module(xp).get_fft_plan(x, shape=self.s,
+                                                axes=self.axes)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).fftn(x, s=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).fftn(x, s=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.fftn(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.5.0')
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_fftn_backend_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes)}
+            backend = cp_fft
+        else:
+            # scipy raises NotImplementedError if plan is not None
+            overwrite_kw = {'plan': None}
+            backend = 'scipy'
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.fftn(x, s=self.s, axes=self.axes, norm=self.norm,
+                                 **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).ifftn(x, s=self.s, axes=self.axes,
+                                    norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).ifftn(x, s=self.s, axes=self.axes,
+                                    norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes)}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).ifftn(x, s=self.s, axes=self.axes,
+                                    norm=self.norm, **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes), 'overwrite_x': True}
+        else:
+            overwrite_kw = {}
+        out = _fft_module(xp).ifftn(x, s=self.s, axes=self.axes,
+                                    norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            plan = _fft_module(xp).get_fft_plan(x, shape=self.s,
+                                                axes=self.axes)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).ifftn(x, s=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).ifftn(x, s=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ifftn(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.5.0')
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ifftn_backend_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        x_orig = x.copy()
+        if xp is cp:
+            overwrite_kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.s, axes=self.axes)}
+            backend = cp_fft
+        else:
+            # scipy raises NotImplementedError if plan is not None
+            overwrite_kw = {'plan': None}
+            backend = 'scipy'
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ifftn(x, s=self.s, axes=self.axes, norm=self.norm,
+                                  **overwrite_kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+@testing.parameterize(*testing.product({
+    'n': [None, 5, 10, 15],
+    'shape': [(9,), (10,), (10, 9), (10, 10)],
+    'axis': [-1, 0],
+    'norm': [None, 'backward', 'ortho', 'forward', '']
+}))
+class TestRfft:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).rfft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).rfft(x, n=self.n, axis=self.axis,
+                                   norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        if xp is cp:
+            kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis, value_type='R2C')}
+        else:
+            kw = {}
+        out = _fft_module(xp).rfft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                   **kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.rfft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cp:
+            kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis, value_type='R2C'),
+                'overwrite_x': True}
+        else:
+            kw = {}
+        out = _fft_module(xp).rfft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                   **kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            plan = _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis, value_type='R2C')
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).rfft(x, n=self.n, axis=self.axis)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).rfft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    # the irfft tests show a slightly different results in CUDA 11.0 when
+    # compared to SciPy 1.6.1
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).irfft(x, n=self.n, axis=self.axis,
+                                    norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).irfft(x, n=self.n, axis=self.axis,
+                                    norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        if xp is cp:
+            kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis, value_type='C2R')}
+        else:
+            kw = {}
+        out = _fft_module(xp).irfft(
+            x, n=self.n, axis=self.axis, norm=self.norm, **kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        if xp is cp:
+            kw = {'plan': _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis, value_type='C2R'),
+                'overwrite_x': True}
+        else:
+            kw = {}
+        out = _fft_module(xp).irfft(
+            x, n=self.n, axis=self.axis, norm=self.norm, **kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            plan = _fft_module(xp).get_fft_plan(
+                x, shape=self.n, axes=self.axis, value_type='C2R')
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).irfft(x, n=self.n, axis=self.axis)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).irfft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.irfft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+def _skip_hipFFT_PlanNd_bug(axes, shape):
+    if cp.cuda.runtime.is_hip:
+        # TODO(leofang): test newer ROCm versions
+        if (axes == (0, 1) and shape == (2, 3, 4)):
+            pytest.skip(
+                "hipFFT's PlanNd for this case is buggy, "
+                "so Plan1d is generated instead")
+
+
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (1, 5), 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2)},
+        {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestRfft2:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft2(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).rfft2(x, s=self.s, axes=self.axes,
+                                    norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft2_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).rfft2(x, s=self.s, axes=self.axes,
+                                    norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft2_plan(self, xp, dtype):
+        _skip_hipFFT_PlanNd_bug(self.axes, self.shape)
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='R2C')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            kw = {'plan': plan}
+        else:
+            kw = {}
+        out = _fft_module(xp).rfft2(
+            x, s=self.s, axes=self.axes, norm=self.norm, **kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft2_overwrite_plan(self, xp, dtype):
+        _skip_hipFFT_PlanNd_bug(self.axes, self.shape)
+        x = testing.shaped_random(self.shape, xp, dtype)
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='R2C')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            kw = {'plan': plan, 'overwrite_x': True}
+        else:
+            kw = {}
+        out = _fft_module(xp).rfft2(
+            x, s=self.s, axes=self.axes, norm=self.norm, **kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft2_plan_manager(self, xp, dtype):
+        _skip_hipFFT_PlanNd_bug(self.axes, self.shape)
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='R2C')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).rfft2(x, s=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).rfft2(x, s=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfft2_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.rfft2(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft2(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).irfft2(x, s=self.s, axes=self.axes,
+                                     norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft2_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).irfft2(x, s=self.s, axes=self.axes,
+                                     norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(int(cp.cuda.device.get_compute_capability()) < 70,
+                        reason="Known to fail with Pascal or older")
+    @pytest.mark.skipif(cp.cuda.runtime.is_hip,
+                        reason="hipFFT's PlanNd for C2R is buggy")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft2_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='C2R')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            kw = {'plan': plan}
+        else:
+            kw = {}
+        out = _fft_module(xp).irfft2(
+            x, s=self.s, axes=self.axes, norm=self.norm, **kw)
+        testing.assert_array_equal(x, x_orig)
+
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(int(cp.cuda.device.get_compute_capability()) < 70,
+                        reason="Known to fail with Pascal or older")
+    @pytest.mark.skipif(cp.cuda.runtime.is_hip,
+                        reason="hipFFT's PlanNd for C2R is buggy")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft2_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='C2R')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            kw = {'plan': plan, 'overwrite_x': True}
+        else:
+            kw = {}
+        out = _fft_module(xp).irfft2(
+            x, s=self.s, axes=self.axes, norm=self.norm, **kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(int(cp.cuda.device.get_compute_capability()) < 70,
+                        reason="Known to fail with Pascal or older")
+    @pytest.mark.skipif(cp.cuda.runtime.is_hip,
+                        reason="hipFFT's PlanNd for C2R is buggy")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft2_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='C2R')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).irfft2(x, s=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).irfft2(x, s=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfft2_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.irfft2(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (3, 4), 's': (1, 5), 'axes': None},
+        {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+        {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 4), 's': None, 'axes': (0,)},
+        {'shape': (3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+        {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+        {'shape': (2, 3, 4), 's': None, 'axes': None},
+        {'shape': (2, 3, 4), 's': (2, 3), 'axes': (0, 1, 2)},
+        {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward', '']})
+    )
+))
+class TestRfftn:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).rfftn(x, s=self.s, axes=self.axes,
+                                    norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).rfftn(x, s=self.s, axes=self.axes,
+                                    norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_plan(self, xp, dtype):
+        _skip_hipFFT_PlanNd_bug(self.axes, self.shape)
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='R2C')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            kw = {'plan': plan}
+        else:
+            kw = {}
+        out = _fft_module(xp).rfftn(
+            x, s=self.s, axes=self.axes, norm=self.norm, **kw)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_overwrite_plan(self, xp, dtype):
+        _skip_hipFFT_PlanNd_bug(self.axes, self.shape)
+        x = testing.shaped_random(self.shape, xp, dtype)
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='R2C')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            kw = {'plan': plan, 'overwrite_x': True}
+        else:
+            kw = {}
+        out = _fft_module(xp).rfftn(
+            x, s=self.s, axes=self.axes, norm=self.norm, **kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_plan_manager(self, xp, dtype):
+        _skip_hipFFT_PlanNd_bug(self.axes, self.shape)
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='R2C')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).rfftn(x, s=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).rfftn(x, s=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_rfftn_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.rfftn(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).irfftn(x, s=self.s, axes=self.axes,
+                                     norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).irfftn(x, s=self.s, axes=self.axes,
+                                     norm=self.norm, **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(int(cp.cuda.device.get_compute_capability()) < 70,
+                        reason="Known to fail with Pascal or older")
+    @pytest.mark.skipif(cp.cuda.runtime.is_hip,
+                        reason="hipFFT's PlanNd for C2R is buggy")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='C2R')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            kw = {'plan': plan}
+        else:
+            kw = {}
+        out = _fft_module(xp).irfftn(
+            x, s=self.s, axes=self.axes, norm=self.norm, **kw)
+        testing.assert_array_equal(x, x_orig)
+
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(int(cp.cuda.device.get_compute_capability()) < 70,
+                        reason="Known to fail with Pascal or older")
+    @pytest.mark.skipif(cp.cuda.runtime.is_hip,
+                        reason="hipFFT's PlanNd for C2R is buggy")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_overwrite_plan(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='C2R')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            kw = {'plan': plan, 'overwrite_x': True}
+        else:
+            kw = {}
+        out = _fft_module(xp).irfftn(
+            x, s=self.s, axes=self.axes, norm=self.norm, **kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(int(cp.cuda.device.get_compute_capability()) < 70,
+                        reason="Known to fail with Pascal or older")
+    @pytest.mark.skipif(cp.cuda.runtime.is_hip,
+                        reason="hipFFT's PlanNd for C2R is buggy")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_plan_manager(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+
+        # hack: skip testing if getting a cuFFT plan is impossible
+        try:
+            plan = _fft_module(cp).get_fft_plan(
+                x, shape=self.s, axes=self.axes, value_type='C2R')
+        except ValueError:
+            return x
+
+        if xp is cp:
+            from cupy.cuda.cufft import get_current_plan
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = _fft_module(xp).irfftn(x, s=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:
+            out = _fft_module(xp).irfftn(x, s=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_irfftn_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.irfftn(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+@testing.parameterize(*testing.product({
+    'n': [None, 5, 10, 15],
+    'shape': [(10,), (10, 10)],
+    'axis': [0, -1],
+    'norm': [None, 'backward', 'ortho', 'forward', ''],
+}))
+class TestHfft:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=4e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_hfft(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).hfft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=4e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_hfft_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).hfft(x, n=self.n, axis=self.axis, norm=self.norm,
+                                   **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    # TODO(leofang): rewrite this test when we support R2C/C2R cuFFT plans
+    @testing.for_all_dtypes()
+    def test_hfft_plan(self, dtype):
+        x = testing.shaped_random(self.shape, cp, dtype)
+        with pytest.raises(NotImplementedError, match='not yet supported'):
+            _fft_module(cp).hfft(x, n=self.n, axis=self.axis,
+                                 norm=self.norm, plan='abc')
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=4e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_hfft_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.hfft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ihfft(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).ihfft(x, n=self.n, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ihfft_overwrite(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        overwrite_kw = {} if xp is np else {'overwrite_x': True}
+        out = _fft_module(xp).ihfft(x, n=self.n, norm=self.norm,
+                                    **overwrite_kw)
+        return _correct_np_dtype(xp, dtype, out)
+
+    # TODO(leofang): rewrite this test when we support R2C/C2R cuFFT plans
+    @testing.for_all_dtypes(no_complex=True)
+    def test_ihfft_plan(self, dtype):
+        x = testing.shaped_random(self.shape, cp, dtype)
+        with pytest.raises(NotImplementedError, match='not yet supported'):
+            _fft_module(cp).ihfft(x, n=self.n, axis=self.axis,
+                                  norm=self.norm, plan='abc')
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ihfft_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ihfft(x, n=self.n, axis=self.axis, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (2, 5), 's': None, 'axes': None},
+        {'shape': (2, 10), 's': (1, 5), 'axes': None},
+        {'shape': (2, 30), 's': None, 'axes': (-2, -1)},
+        {'shape': (2, 50), 's': None, 'axes': (-1, -2)},
+        {'shape': (2, 100), 's': (2, 50), 'axes': (0,)},
+        {'shape': (2, 3, 10), 's': None, 'axes': None},
+        {'shape': (2, 5, 20), 's': None, 'axes': (0, 1, 2)},
+        {'shape': (2, 10, 100), 's': (2, 10), 'axes': (0, -1, -2)},
+        {'shape': (2, 5, 10), 's': None, 'axes': (-2, -1, 0)},
+        {'shape': (2, 10, 50, 100), 's': (2, 10, 50), 'axes': (0,)},
+
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward']})
+    )
+))
+@testing.with_requires('scipy>=1.4.0')
+class TestHfft2:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=4e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_hfft2(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).hfft2(x, s=self.s, axes=self.axes,
+                                    norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=4e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_hfft2_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.hfft2(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.xfail(cp.cuda.runtime.is_hip, strict=False,  # see #6427
+                       reason="Flaky in HIP when running with other tests")
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ihfft2(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).ihfft2(x, s=self.s, axes=self.axes,
+                                     norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.xfail(cp.cuda.runtime.is_hip, strict=False,  # see #6427
+                       reason="Flaky in HIP when running with other tests")
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ihfft2_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ihfft2(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+@testing.parameterize(*(
+    testing.product_dict([
+        {'shape': (2, 5), 's': None, 'axes': None},
+        {'shape': (2, 10), 's': (1, 5), 'axes': None},
+        {'shape': (2, 30), 's': None, 'axes': (-2, -1)},
+        {'shape': (2, 50), 's': None, 'axes': (-1, -2)},
+        {'shape': (3, 100), 's': (2, 50), 'axes': (0,)},
+        {'shape': (3, 3, 10), 's': None, 'axes': None},
+        {'shape': (3, 5, 20), 's': None, 'axes': (0, 1, 2)},
+        {'shape': (5, 10, 100), 's': (2, 10), 'axes': (0, -1, -2)},
+        {'shape': (6, 5, 10), 's': None, 'axes': (-2, -1, 0)},
+        {'shape': (7, 10, 50, 100), 's': (2, 10, 50), 'axes': (0,)},
+
+    ],
+        testing.product({'norm': [None, 'backward', 'ortho', 'forward']})
+    )
+))
+@testing.with_requires('scipy>=1.4.0')
+class TestHfftn:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        _skip_forward_backward(self.norm)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=4e-4, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_hfftn(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).hfftn(x, s=self.s, axes=self.axes,
+                                    norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.skipif(_irfft_skip_condition,
+                        reason="Known to fail with Pascal or older")
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=4e-4, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_hfftn_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.hfftn(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.xfail(cp.cuda.runtime.is_hip, strict=False,  # see #6427
+                       reason="Flaky in HIP when running with other tests")
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ihfftn(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = _fft_module(xp).ihfftn(x, s=self.s, axes=self.axes,
+                                     norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+    @pytest.mark.xfail(cp.cuda.runtime.is_hip, strict=False,  # see #6427
+                       reason="Flaky in HIP when running with other tests")
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_ihfftn_backend(self, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            out = scipy_fft.ihfftn(x, s=self.s, axes=self.axes, norm=self.norm)
+        testing.assert_array_equal(x, x_orig)
+        return _correct_np_dtype(xp, dtype, out)
+
+
+@pytest.mark.parametrize('func', [
+    cp_fft.fft2, cp_fft.ifft2, cp_fft.rfft2, cp_fft.irfft2,
+    cp_fft.fftn, cp_fft.ifftn, cp_fft.rfftn, cp_fft.irfftn])
+def test_scalar_shape_axes(func):
+    x = testing.shaped_random((10, 10), cp)
+    y_scalar = func(x, s=5, axes=-1)
+    y_normal = func(x, s=(5,), axes=(-1,))
+    testing.assert_allclose(y_scalar, y_normal)
diff --git a/tests/cupyx_tests/scipy_tests/fft_tests/test_fftlog.py b/tests/cupyx_tests/scipy_tests/fft_tests/test_fftlog.py
new file mode 100644
index 0000000..95656fc
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/fft_tests/test_fftlog.py
@@ -0,0 +1,75 @@
+import numpy as np
+
+import cupy
+import cupyx.scipy.fft as cp_fft
+from cupy import testing
+
+try:
+    # scipy.fft is available since scipy v1.4.0+
+    import scipy.fft as scipy_fft  # noqa
+except ImportError:
+    scipy_fft = None
+
+try:
+    # fht, ifht, fhtoffset are available in scipy v1.7.0+
+    from scipy.fft import fhtoffset
+except ImportError:
+    fhtoffset = None
+
+atol = {cupy.float64: 1e-10, 'default': 1e-5}
+rtol = {cupy.float64: 1e-10, 'default': 1e-5}
+
+
+@testing.parameterize(*testing.product({
+    'mu': [0.3, 0.5, -1.2],
+    'shape': [(9,), (10,), (10, 9), (8, 10)],
+    'offset': [0.0, 0.1, 'optimal'],
+    'bias': [0.0, 0.8, -0.5],
+    'function': ['fht', 'ifht'],
+}))
+class TestFftlog:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=rtol, atol=atol, scipy_name='scp')
+    @testing.with_requires('scipy>=1.7.0')
+    def test_fht(self, xp, scp, dtype):
+
+        # test function, analytical Hankel transform is of the same form
+        def f(r, mu):
+            return r**(mu + 1) * xp.exp(-r**2 / 2)
+
+        shape = self.shape
+        r = xp.logspace(-4, 4, shape[-1])
+        dln = xp.log(r[1]/r[0])
+
+        if len(shape) == 2:
+            r = xp.stack([r] * shape[0], axis=0)
+
+        mu = self.mu
+        bias = self.bias
+        if self.offset == 'optimal':
+            offset = fhtoffset(dln.item(), mu, bias)
+        else:
+            offset = self.offset
+
+        a = f(r, mu).astype(dtype)
+        func = getattr(scp.fft, self.function)
+        return func(a, dln, mu, offset=offset, bias=bias)
+
+
+@testing.parameterize(*testing.product({
+    'function': ['fht', 'ifht'],
+}))
+class TestFftlogScipyBackend:
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    @testing.with_requires('scipy>=1.9.0')
+    def test_dct_backend(self, xp, dtype):
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            fft_func = getattr(scipy_fft, self.function)
+            r = xp.logspace(-2, 2, 10)
+            dln = xp.log(r[1]/r[0])
+            return fft_func(r, dln, mu=0.5)
diff --git a/tests/cupyx_tests/scipy_tests/fft_tests/test_helper.py b/tests/cupyx_tests/scipy_tests/fft_tests/test_helper.py
new file mode 100644
index 0000000..9a65e2e
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/fft_tests/test_helper.py
@@ -0,0 +1,21 @@
+import unittest
+
+import cupyx.scipy.fft as cp_fft
+
+
+class TestNextFastLen(unittest.TestCase):
+
+    def _is_fast_len(self, n):
+        if n == 0:
+            return True
+        for p in (2, 3, 5, 7):
+            while n % p == 0:
+                n //= p
+        return n == 1
+
+    def test_next_fast_len(self):
+        for in_value in range(2000):
+            out_value = cp_fft.next_fast_len(in_value)
+            assert self._is_fast_len(out_value)
+            for i in range(in_value, out_value):
+                assert not self._is_fast_len(i)
diff --git a/tests/cupyx_tests/scipy_tests/fft_tests/test_realtransforms.py b/tests/cupyx_tests/scipy_tests/fft_tests/test_realtransforms.py
new file mode 100644
index 0000000..33a94ee
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/fft_tests/test_realtransforms.py
@@ -0,0 +1,173 @@
+import numpy as np
+import pytest
+
+from cupyx.scipy import fft as cp_fft
+from cupy import testing
+
+try:
+    # scipy.fft is available since scipy v1.4.0+
+    import scipy.fft as scipy_fft  # noqa
+except ImportError:
+    scipy_fft = None
+
+# used to avoid SciPy bug in complex dtype cases with output_x=True
+# https://github.com/scipy/scipy/pull/11904
+scipy_cplx_bug = not cp_fft._scipy_150
+
+if cp_fft._scipy_160:
+    # additional normalization options available for SciPy>=1.6
+    all_dct_norms = [None, 'ortho', 'forward', 'backward']
+else:
+    all_dct_norms = [None, 'ortho']
+
+
+@testing.parameterize(
+    *testing.product(
+        {
+            'n': [None, 0, 5, 15],
+            'type': [1, 2, 3, 4],
+            'shape': [(9,), (10,), (10, 9)],
+            'axis': [-1, 0],
+            'norm': ['ortho'],
+            'overwrite_x': [False],
+            'function': ['dct', 'dst', 'idct', 'idst'],
+        }
+    )
+    # test all overwrite_x and norm combinations on a smaller subset of shapes
+    + testing.product(
+        {
+            'n': [None, 15],
+            'type': [2, 3],
+            'shape': [(10, 9)],
+            'axis': [-1, 0],
+            'norm': all_dct_norms,
+            'overwrite_x': [False, True],
+            'function': ['dct', 'dst', 'idct', 'idst'],
+        }
+    )
+)
+@testing.with_requires('scipy>=1.4')
+class TestDctDst:
+
+    def _run_transform(self, dct_func, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        if scipy_cplx_bug and x.dtype.kind == 'c':
+            # skip cases where SciPy has a bug
+            return x
+        x_orig = x.copy()
+        kwargs = dict(type=self.type,
+                      n=self.n,
+                      axis=self.axis,
+                      norm=self.norm,
+                      overwrite_x=self.overwrite_x)
+        if self.type in [1, 4]:
+            if xp != np:
+                # type 1 and 4 real-to-real transforms not implemented
+                with pytest.raises(NotImplementedError):
+                    dct_func(x, **kwargs)
+            return xp.zeros([])
+        out = dct_func(x, **kwargs)
+        if not self.overwrite_x:
+            testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(
+        scipy_name='scp', rtol=1e-4, atol=1e-5, accept_error=ValueError,
+        contiguous_check=False
+    )
+    def test_dct(self, xp, scp, dtype):
+        fft_func = getattr(scp.fft, self.function)
+        return self._run_transform(fft_func, xp, dtype)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_dct_backend(self, xp, dtype):
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            fft_func = getattr(scipy_fft, self.function)
+            return self._run_transform(fft_func, xp, dtype)
+
+
+@testing.parameterize(
+    *(
+        # 2D cases
+        testing.product(
+            {
+                'shape': [(3, 4)],
+                'type': [2, 3],
+                # Note: non-integer s or s == 0 will cause a ValueError
+                's': [None, (1, 5), (-1, -1), (0, 5), (1.5, 2.5)],
+                'axes': [None, (-2, -1), (-1, -2), (0,)],
+                'norm': ['ortho'],
+                'overwrite_x': [False],
+                'function': ['dctn', 'dstn', 'idctn', 'idstn'],
+            }
+        )
+        # 3D cases
+        + testing.product(
+            {
+                'shape': [(2, 3, 4)],
+                'type': [2, 3],
+                # Note: len(s) < ndim is allowed
+                #       len(s) > ndim raises a ValueError
+                's': [None, (1, 5), (1, 4, 10), (2, 2, 2, 2)],
+                'axes': [None, (-2, -1), (-1, -2, -3)],
+                'norm': ['ortho'],
+                'overwrite_x': [False],
+                'function': ['dctn', 'dstn', 'idctn', 'idstn'],
+            }
+        )
+        # 4D cases
+        + testing.product(
+            {
+                'shape': [(2, 3, 4, 5)],
+                'type': [2, 3],
+                's': [None],
+                'axes': [None],
+                'norm': all_dct_norms,
+                'overwrite_x': [True, False],
+                'function': ['dctn', 'dstn', 'idctn', 'idstn'],
+            }
+        )
+    )
+)
+@testing.with_requires('scipy>=1.4')
+class TestDctnDstn():
+
+    def _run_transform(self, dct_func, xp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        if scipy_cplx_bug and x.dtype.kind == 'c':
+            # skip cases where SciPy has a bug
+            return x
+        x_orig = x.copy()
+        out = dct_func(
+            x,
+            type=self.type,
+            s=self.s,
+            axes=self.axes,
+            norm=self.norm,
+            overwrite_x=self.overwrite_x,
+        )
+        if not self.overwrite_x:
+            testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(
+        scipy_name='scp', rtol=1e-4, atol=1e-5, accept_error=ValueError,
+        contiguous_check=False
+    )
+    def test_dctn(self, xp, scp, dtype):
+        fft_func = getattr(scp.fft, self.function)
+        return self._run_transform(fft_func, xp, dtype)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-5, accept_error=ValueError,
+                                 contiguous_check=False)
+    def test_dctn_backend(self, xp, dtype):
+        backend = 'scipy' if xp is np else cp_fft
+        with scipy_fft.set_backend(backend):
+            fft_func = getattr(scipy_fft, self.function)
+            return self._run_transform(fft_func, xp, dtype)
diff --git a/tests/cupyx_tests/scipy_tests/fftpack_tests/__init__.py b/tests/cupyx_tests/scipy_tests/fftpack_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/fftpack_tests/test_fftpack.py b/tests/cupyx_tests/scipy_tests/fftpack_tests/test_fftpack.py
new file mode 100644
index 0000000..5df72bb
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/fftpack_tests/test_fftpack.py
@@ -0,0 +1,689 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx.scipy.fftpack  # NOQA
+from cupy.fft._fft import _default_fft_func, _fftn
+
+if cupyx.scipy._scipy_available:
+    import scipy.fftpack  # NOQA
+
+
+@testing.parameterize(*testing.product({
+    'n': [None, 0, 5, 10, 15],
+    'shape': [(9,), (10,), (10, 9), (10, 10)],
+    'axis': [-1, 0],
+}))
+@testing.with_requires('scipy>=0.19.0')
+class TestFft(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = scp.fftpack.fft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft_overwrite(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return scp.fftpack.fft(x, n=self.n, axis=self.axis,
+                               overwrite_x=True)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if scp is cupyx.scipy:
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis)
+            out = scp.fftpack.fft(x, n=self.n, axis=self.axis, plan=plan)
+        else:  # scipy
+            out = scp.fftpack.fft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft_overwrite_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        if scp is cupyx.scipy:
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis)
+            x = scp.fftpack.fft(x, n=self.n, axis=self.axis,
+                                overwrite_x=True, plan=plan)
+        else:  # scipy
+            x = scp.fftpack.fft(x, n=self.n, axis=self.axis,
+                                overwrite_x=True)
+        return x
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft_plan_manager(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if scp is cupyx.scipy:
+            from cupy.cuda.cufft import get_current_plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = scp.fftpack.fft(x, n=self.n, axis=self.axis)
+            assert get_current_plan() is None
+        else:  # scipy
+            out = scp.fftpack.fft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = scp.fftpack.ifft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft_overwrite(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return scp.fftpack.ifft(x, n=self.n, axis=self.axis,
+                                overwrite_x=True)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if scp is cupyx.scipy:
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis)
+            out = scp.fftpack.ifft(x, n=self.n, axis=self.axis, plan=plan)
+        else:  # scipy
+            out = scp.fftpack.ifft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft_overwrite_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        if scp is cupyx.scipy:
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis)
+            x = scp.fftpack.ifft(x, n=self.n, axis=self.axis,
+                                 overwrite_x=True, plan=plan)
+        else:  # scipy
+            x = scp.fftpack.ifft(x, n=self.n, axis=self.axis,
+                                 overwrite_x=True)
+        return x
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft_plan_manager(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return x
+        x_orig = x.copy()
+        if scp is cupyx.scipy:
+            from cupy.cuda.cufft import get_current_plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = scp.fftpack.ifft(x, n=self.n, axis=self.axis)
+            assert get_current_plan() is None
+        else:  # scipy
+            out = scp.fftpack.ifft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_complex_dtypes()
+    def test_fft_multiple_plan_error(self, dtype):
+        # hack: avoid testing the cases when the output array is of size 0
+        # because cuFFT and numpy raise different kinds of exceptions
+        if self.n == 0:
+            return
+        import cupy
+        import cupyx.scipy.fftpack as fftpack
+        x = testing.shaped_random(self.shape, cupy, dtype)
+        plan = fftpack.get_fft_plan(x, shape=self.n, axes=self.axis)
+        with pytest.raises(RuntimeError) as ex, plan:
+            fftpack.fft(x, n=self.n, axis=self.axis, plan=plan)
+        assert 'Use the cuFFT plan either as' in str(ex.value)
+
+
+@testing.parameterize(
+    {'shape': (3, 4), 's': None, 'axes': None},
+    {'shape': (3, 4), 's': (1, 5), 'axes': None},
+    {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+    {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+    {'shape': (3, 4), 's': None, 'axes': (0,)},
+    {'shape': (2, 3, 4), 's': None, 'axes': None},
+    {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+    {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+    {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+    {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+    {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+)
+@testing.with_requires('scipy>=0.19.0')
+class TestFft2(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft2(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = scp.fftpack.fft2(x, shape=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft2_overwrite(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return scp.fftpack.fft2(x, shape=self.s, axes=self.axes,
+                                overwrite_x=True)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft2_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            import cupy.fft.config as config
+            config.enable_nd_planning = False  # use explicit plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            out = scp.fftpack.fft2(x, shape=self.s, axes=self.axes, plan=plan)
+            config.enable_nd_planning = True  # default
+        else:  # scipy
+            out = scp.fftpack.fft2(x, shape=self.s, axes=self.axes)
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft2_overwrite_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            import cupy.fft.config as config
+            config.enable_nd_planning = False  # use explicit plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            x = scp.fftpack.fft2(x, shape=self.s, axes=self.axes,
+                                 overwrite_x=True, plan=plan)
+            config.enable_nd_planning = True  # default
+        else:  # scipy
+            x = scp.fftpack.fft2(x, shape=self.s, axes=self.axes,
+                                 overwrite_x=True)
+        return x
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fft2_plan_manager(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            from cupy.cuda.cufft import get_current_plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = scp.fftpack.fft2(x, shape=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:  # scipy
+            out = scp.fftpack.fft2(x, shape=self.s, axes=self.axes)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft2(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = scp.fftpack.ifft2(x, shape=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft2_overwrite(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return scp.fftpack.ifft2(x, shape=self.s, axes=self.axes,
+                                 overwrite_x=True)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft2_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            import cupy.fft.config as config
+            config.enable_nd_planning = False  # use explicit plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            out = scp.fftpack.ifft2(x, shape=self.s, axes=self.axes, plan=plan)
+            config.enable_nd_planning = True  # default
+        else:  # scipy
+            out = scp.fftpack.ifft2(x, shape=self.s, axes=self.axes)
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft2_overwrite_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            import cupy.fft.config as config
+            config.enable_nd_planning = False  # use explicit plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            x = scp.fftpack.ifft2(x, shape=self.s, axes=self.axes,
+                                  overwrite_x=True, plan=plan)
+            config.enable_nd_planning = True  # default
+        else:  # scipy
+            x = scp.fftpack.ifft2(x, shape=self.s, axes=self.axes,
+                                  overwrite_x=True)
+        return x
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifft2_plan_manager(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            from cupy.cuda.cufft import get_current_plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = scp.fftpack.ifft2(x, shape=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:  # scipy
+            out = scp.fftpack.ifft2(x, shape=self.s, axes=self.axes)
+        return out
+
+
+@testing.parameterize(
+    {'shape': (3, 4), 's': None, 'axes': None},
+    {'shape': (3, 4), 's': (1, 5), 'axes': None},
+    {'shape': (3, 4), 's': None, 'axes': (-2, -1)},
+    {'shape': (3, 4), 's': None, 'axes': (-1, -2)},
+    {'shape': (3, 4), 's': None, 'axes': (0,)},
+    {'shape': (2, 3, 4), 's': None, 'axes': None},
+    {'shape': (2, 3, 4), 's': (1, 4, 10), 'axes': None},
+    {'shape': (2, 3, 4), 's': None, 'axes': (-3, -2, -1)},
+    {'shape': (2, 3, 4), 's': None, 'axes': (-1, -2, -3)},
+    {'shape': (2, 3, 4), 's': None, 'axes': (0, 1)},
+    {'shape': (2, 3, 4, 5), 's': None, 'axes': None},
+)
+@testing.with_requires('scipy>=0.19.0')
+class TestFftn(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fftn(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = scp.fftpack.fftn(x, shape=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fftn_overwrite(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return scp.fftpack.fftn(x, shape=self.s, axes=self.axes,
+                                overwrite_x=True)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fftn_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            import cupy.fft.config as config
+            config.enable_nd_planning = False  # use explicit plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            out = scp.fftpack.fftn(x, shape=self.s, axes=self.axes, plan=plan)
+            config.enable_nd_planning = True  # default
+        else:  # scipy
+            out = scp.fftpack.fftn(x, shape=self.s, axes=self.axes)
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fftn_overwrite_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            import cupy.fft.config as config
+            config.enable_nd_planning = False  # use explicit plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            x = scp.fftpack.fftn(x, shape=self.s, axes=self.axes,
+                                 overwrite_x=True, plan=plan)
+            config.enable_nd_planning = True  # default
+        else:  # scipy
+            x = scp.fftpack.fftn(x, shape=self.s, axes=self.axes,
+                                 overwrite_x=True)
+        return x
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_fftn_plan_manager(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            from cupy.cuda.cufft import get_current_plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = scp.fftpack.fftn(x, shape=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:  # scipy
+            out = scp.fftpack.fftn(x, shape=self.s, axes=self.axes)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifftn(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = scp.fftpack.ifftn(x, shape=self.s, axes=self.axes)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifftn_overwrite(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return scp.fftpack.ifftn(x, shape=self.s, axes=self.axes,
+                                 overwrite_x=True)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifftn_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            import cupy.fft.config as config
+            config.enable_nd_planning = False  # use explicit plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            out = scp.fftpack.ifftn(x, shape=self.s, axes=self.axes, plan=plan)
+            config.enable_nd_planning = True  # default
+        else:  # scipy
+            out = scp.fftpack.ifftn(x, shape=self.s, axes=self.axes)
+        return out
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifftn_overwrite_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            import cupy.fft.config as config
+            config.enable_nd_planning = False  # use explicit plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            x = scp.fftpack.ifftn(x, shape=self.s, axes=self.axes,
+                                  overwrite_x=True, plan=plan)
+            config.enable_nd_planning = True  # default
+        else:  # scipy
+            x = scp.fftpack.ifftn(x, shape=self.s, axes=self.axes,
+                                  overwrite_x=True)
+        return x
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_ifftn_plan_manager(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return x
+        if scp is cupyx.scipy:
+            from cupy.cuda.cufft import get_current_plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = scp.fftpack.ifftn(x, shape=self.s, axes=self.axes)
+            assert get_current_plan() is None
+        else:  # scipy
+            out = scp.fftpack.ifftn(x, shape=self.s, axes=self.axes)
+        return out
+
+    @testing.for_complex_dtypes()
+    def test_fftn_multiple_plan_error(self, dtype):
+        import cupy
+        import cupyx.scipy.fftpack as fftpack
+        x = testing.shaped_random(self.shape, cupy, dtype)
+        # hack: avoid testing the cases when getting a cuFFT plan is impossible
+        if _default_fft_func(x, s=self.s, axes=self.axes) is not _fftn:
+            return
+        plan = fftpack.get_fft_plan(x, shape=self.s, axes=self.axes)
+        with pytest.raises(RuntimeError) as ex, plan:
+            fftpack.fftn(x, shape=self.s, axes=self.axes, plan=plan)
+        assert 'Use the cuFFT plan either as' in str(ex.value)
+
+
+@testing.parameterize(*testing.product({
+    'n': [None, 5, 10, 15],
+    'shape': [(9,), (10,), (10, 9), (10, 10)],
+    'axis': [-1, 0],
+}))
+@testing.with_requires('scipy>=0.19.0')
+class TestRfft(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_rfft(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = scp.fftpack.rfft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_rfft_overwrite(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return scp.fftpack.rfft(x, n=self.n, axis=self.axis,
+                                overwrite_x=True)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_rfft_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        if scp is cupyx.scipy:
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis,
+                                            value_type='R2C')
+            out = scp.fftpack.rfft(x, n=self.n, axis=self.axis, plan=plan)
+        else:  # scipy
+            out = scp.fftpack.rfft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_rfft_overwrite_plan(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        if scp is cupyx.scipy:
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis,
+                                            value_type='R2C')
+            x = scp.fftpack.rfft(x, n=self.n, axis=self.axis,
+                                 overwrite_x=True, plan=plan)
+        else:  # scipy
+            x = scp.fftpack.rfft(x, n=self.n, axis=self.axis,
+                                 overwrite_x=True)
+        return x
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-6, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_rfft_plan_manager(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        if scp is cupyx.scipy:
+            from cupy.cuda.cufft import get_current_plan
+            plan = scp.fftpack.get_fft_plan(x, shape=self.n, axes=self.axis,
+                                            value_type='R2C')
+            with plan:
+                assert id(plan) == id(get_current_plan())
+                out = scp.fftpack.rfft(x, n=self.n, axis=self.axis)
+            assert get_current_plan() is None
+        else:  # scipy
+            out = scp.fftpack.rfft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_irfft(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        x_orig = x.copy()
+        out = scp.fftpack.irfft(x, n=self.n, axis=self.axis)
+        testing.assert_array_equal(x, x_orig)
+        return out
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, accept_error=ValueError,
+                                 contiguous_check=False, scipy_name='scp')
+    def test_irfft_overwrite(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return scp.fftpack.irfft(x, n=self.n, axis=self.axis,
+                                 overwrite_x=True)
+
+
+@testing.parameterize(
+    {'shape': (32, 16, 4), 'data_order': 'F'},
+    {'shape': (4, 32, 16), 'data_order': 'C'},
+)
+class TestFftnView(unittest.TestCase):
+
+    @testing.for_complex_dtypes()
+    def test_contiguous_view(self, dtype):
+        # Fortran-ordered case tests: https://github.com/cupy/cupy/issues/3079
+        a = testing.shaped_random(self.shape, cupy, dtype)
+        if self.data_order == 'F':
+            a = cupy.asfortranarray(a)
+            sl = numpy.s_[..., 0]
+        else:
+            sl = numpy.s_[0, ...]
+
+        # transform a contiguous view without pre-planning
+        view = a[sl]
+        expected = cupyx.scipy.fftpack.fftn(view)
+
+        # create plan and then apply it to a contiguous view
+        plan = cupyx.scipy.fftpack.get_fft_plan(view)
+        with plan:
+            out = cupyx.scipy.fftpack.fftn(view)
+        testing.assert_allclose(expected, out)
+
+    @testing.for_complex_dtypes()
+    def test_noncontiguous_view(self, dtype):
+        a = testing.shaped_random(self.shape, cupy, dtype)
+        if self.data_order == 'F':
+            a = cupy.asfortranarray(a)
+            sl = numpy.s_[..., ::2]
+        else:
+            sl = numpy.s_[::2, ...]
+
+        # transform a non-contiguous view without pre-planning
+        view = a[sl]
+        expected = cupyx.scipy.fftpack.fftn(view)
+
+        # create plan and then apply it to a non-contiguous view
+        plan = cupyx.scipy.fftpack.get_fft_plan(view.copy())
+        with plan:
+            out = cupyx.scipy.fftpack.fftn(view)
+        testing.assert_allclose(expected, out)
+
+    @testing.for_complex_dtypes()
+    def test_overwrite_x_with_contiguous_view(self, dtype):
+        # Test case for: https://github.com/cupy/cupy/issues/3079
+        a = testing.shaped_random(self.shape, cupy, dtype)
+        if self.data_order == 'C':
+            # C-contiguous view
+            b = a[:a.shape[0] // 2, ...]
+        else:
+            # F-contiguous view
+            a = cupy.asfortranarray(a)
+            b = a[..., :a.shape[-1] // 2]
+        b_ptr = b.data.ptr
+        out = cupyx.scipy.fftpack.fftn(b, overwrite_x=True)
+        assert out.data.ptr == b_ptr
diff --git a/tests/cupyx_tests/scipy_tests/interpolate_tests/__init__.py b/tests/cupyx_tests/scipy_tests/interpolate_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/interpolate_tests/test_bspline.py b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_bspline.py
new file mode 100644
index 0000000..23a7ba2
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_bspline.py
@@ -0,0 +1,489 @@
+import pytest
+import inspect
+
+from cupy import testing
+from cupy_backends.cuda.api import driver
+from cupy_backends.cuda.api import runtime
+import numpy as np
+import cupyx.scipy.interpolate  # NOQA
+
+try:
+    from scipy import interpolate  # NOQA
+except ImportError:
+    pass
+
+
+@testing.parameterize(*testing.product({
+    'extrapolate': [True, False, 'periodic'],
+    'c': [[-1, 2, 0, -1], [[-1, 2, 0, -1]] * 5]}))
+@testing.with_requires("scipy")
+class TestBSpline:
+
+    def _make_random_spline(self, xp, scp, n=35, k=3):
+        t = xp.sort(testing.shaped_random((n + k + 1,), xp, dtype=np.float64))
+        c = testing.shaped_random((n,), xp, dtype=np.float64)
+        return scp.interpolate.BSpline.construct_fast(t, c, k)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=True)
+    def test_ctor(self, xp, scp):
+        # knots should be an ordered 1-D array of finite real numbers
+        t = [1, 1.j]
+        c = [1.0]
+        k = 0
+        scp.interpolate.BSpline(t, c, k)
+
+        t = [1, xp.nan]
+        scp.interpolate.BSpline(t, c, k)
+
+        t = [1, xp.inf]
+        scp.interpolate.BSpline(t, c, k)
+
+        t = [1, -1]
+        scp.interpolate.BSpline(t, c, k)
+
+        t = [[1], [1]]
+        scp.interpolate.BSpline(t, c, k)
+
+        # for n+k+1 knots and degree k need at least n coefficients
+        t = [0, 1, 2]
+        c = [1]
+        scp.interpolate.BSpline(t, c, k)
+
+        t = [0, 1, 2, 3, 4]
+        c = [1., 1.]
+        k = 2
+        scp.interpolate.BSpline(t, c, k)
+
+        # non-integer orders
+        t = [0., 0., 1., 2., 3., 4.]
+        c = [1., 1., 1.]
+        k = "cubic"
+        scp.interpolate.BSpline(t, c, k)
+
+        t = [0., 0., 1., 2., 3., 4.]
+        c = [1., 1., 1.]
+        k = 2.5
+        scp.interpolate.BSpline(t, c, k)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        k = 2
+        t = xp.arange(7, dtype=dtype)
+        c = xp.asarray(self.c, dtype=dtype)
+        test_xs = xp.linspace(-5, 10, 100, dtype=dtype)
+        B = scp.interpolate.BSpline(t, c, k, extrapolate=self.extrapolate)
+        return B(test_xs)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_degree_1(self, xp, scp):
+        t = xp.asarray([0, 1, 2, 3, 4])
+        c = xp.asarray([1, 2, 3])
+        k = 1
+
+        b = scp.interpolate.BSpline(t, c, k)
+        x = xp.linspace(1, 3, 50)
+        return b(x)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_rndm_unity(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+
+        b = self._make_random_spline(xp, scp)
+        b.c = xp.ones_like(b.c)
+        xx = xp.linspace(b.t[b.k], b.t[-b.k-1], 100)
+        return b(xx)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_vectorization(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+
+        n, k = 22, 3
+        t = xp.sort(xp.random.random(n))
+        c = xp.random.random(size=(n, 6, 7))
+        b = scp.interpolate.BSpline(t, c, k)
+        tm, tp = t[k], t[-k-1]
+        xx = tm + (tp - tm) * xp.random.random((3, 4, 5))
+        return b(xx).shape
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-3)
+    def test_bspline_len_c(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+
+        # for n+k+1 knots, only first n coefs are used.
+        n, k = 33, 3
+        t = xp.sort(testing.shaped_random((n + k + 1,), xp))
+        c = testing.shaped_random((n,), xp)
+
+        # pad coefficients with random garbage
+        c_pad = xp.r_[c, testing.shaped_random((k + 1,), xp)]
+
+        BSpline = scp.interpolate.BSpline
+        b, b_pad = BSpline(t, c, k), BSpline(t, c_pad, k)
+
+        dt = t[-1] - t[0]
+
+        # Locally, linspace produces relatively different values (1e-7) between
+        # NumPy and CuPy during testing. Such difference result in a 1e-3
+        # tolerance
+        xx = xp.linspace(t[0] - dt, t[-1] + dt, 50, dtype=dtype)
+        return b(xx), b_pad(xx)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_endpoints(self, xp, scp):
+        # base interval is closed
+        b = self._make_random_spline(xp, scp)
+        t, _, k = b.tck
+        tm, tp = t[k], t[-k-1]
+        return (
+            b(xp.asarray([tm, tp]), extrapolate=self.extrapolate),
+            b(xp.asarray([tm + 1e-10, tp - 1e-10]),
+              extrapolate=self.extrapolate))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_continuity(self, xp, scp):
+        # assert continuity at internal knots
+        b = self._make_random_spline(xp, scp)
+        t, _, k = b.tck
+        return b(t[k+1:-k-1] - 1e-10), b(t[k+1:-k-1] + 1e-10)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_extrap(self, xp, scp):
+        b = self._make_random_spline(xp, scp)
+        t, c, k = b.tck
+        dt = t[-1] - t[0]
+        xx = xp.linspace(t[k] - dt, t[-k-1] + dt, 50)
+        mask = (t[k] < xx) & (xx < t[-k-1])
+
+        return (b(xx[mask], extrapolate=self.extrapolate),
+                b(xx, extrapolate=self.extrapolate))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_default_extrap(self, xp, scp):
+        # BSpline defaults to extrapolate=True
+        b = self._make_random_spline(xp, scp)
+        t, _, k = b.tck
+        xx = [t[0] - 1, t[-1] + 1]
+        yy = b(xx)
+        return not xp.all(xp.isnan(yy))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_periodic_extrap(self, xp, scp):
+        b = self._make_random_spline(xp, scp, n=4, k=3)
+        t, c, k = b.tck
+        n = t.size - (k + 1)
+
+        # Direct check
+        xx = xp.asarray([-1, 0, 0.5, 1])
+        xy = t[k] + (xx - t[k]) % (t[n] - t[k])
+        return b(xx, extrapolate='periodic'), b(xy, extrapolate=True)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_derivative_rndm(self, xp, scp):
+        b = self._make_random_spline(xp, scp)
+        t, _, k = b.tck
+        xx = xp.linspace(t[0], t[-1], 50)
+        xx = xp.r_[xx, t]
+
+        derivatives = []
+        for der in range(1, k+2):
+            derivatives.append(b(xx, nu=der))
+
+        return derivatives
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bspline_derivative_jumps(self, xp, scp):
+        # example from de Boor, Chap IX, example (24)
+        # NB: knots augmented & corresp coefs are zeroed out
+        # in agreement with the convention (29)
+        k = 2
+        t = xp.asarray([-1, -1, 0, 1, 1, 3, 4, 6, 6, 6, 7, 7])
+        c = xp.r_[0, 0, testing.shaped_random((5,), xp), 0, 0]
+        b = scp.interpolate.BSpline(t, c, k)
+
+        comp = []
+        # b is continuous at x != 6 (triple knot)
+        x = xp.asarray([1, 3, 4, 6])
+        comp.append(b(x[x != 6] - 1e-10))
+        comp.append(b(x[x != 6] + 1e-10))
+        comp.append(b(6. - 1e-10))
+        comp.append(b(6. + 1e-10))
+
+        # 1st derivative jumps at double knots, 1 & 6:
+        x0 = xp.asarray([3, 4])
+        comp.append(b(x0 - 1e-10, nu=1))
+        comp.append(b(x0 + 1e-10, nu=1))
+
+        x1 = xp.asarray([1, 6])
+        comp.append(b(x1 - 1e-10, nu=1))
+        comp.append(b(x1 + 1e-10, nu=1))
+
+        # 2nd derivative is not guaranteed to be continuous either
+        comp.append(b(x - 1e-10, nu=2))
+        comp.append(b(x + 1e-10, nu=2))
+
+        return comp
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_basis_element(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        t = xp.arange(7, dtype=dtype)
+        b = scp.interpolate.BSpline.basis_element(
+            t, extrapolate=self.extrapolate)
+        return b.tck
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_basis_element_quadratic(self, xp, scp):
+        xx = xp.linspace(-1, 4, 20)
+        b = scp.interpolate.BSpline.basis_element(
+            t=xp.asarray([0, 1, 2, 3]))
+        r1 = b(xx)
+
+        b = scp.interpolate.BSpline.basis_element(
+            t=xp.asarray([0, 1, 1, 2]))
+        xx = xp.linspace(0, 2, 10)
+        r2 = b(xx)
+        return r1, r2
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_basis_element_rndm(self, xp, scp):
+        b = self._make_random_spline(xp, scp)
+        t, c, k = b.tck
+        xx = xp.linspace(t[k], t[-k-1], 20)
+        n = len(t) - (k+1)
+        s = 0.
+        for i in range(n):
+            b = scp.interpolate.BSpline.basis_element(
+                t[i:i+k+2], extrapolate=False)(xx)
+            # zero out out-of-bounds elements
+            s += c[i] * xp.nan_to_num(b)
+        return s
+
+    @pytest.mark.xfail(
+        runtime.is_hip and driver.get_build_version() < 5_00_00000,
+        reason='name_expression with ROCm 4.3 may not work')
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_cmplx(self, xp, scp):
+        b = self._make_random_spline(xp, scp)
+        t, c, k = b.tck
+        cc = c * (1. + 3.j)
+
+        b = scp.interpolate.BSpline(t, cc, k)
+        b_re = scp.interpolate.BSpline(t, b.c.real, k)
+        b_im = scp.interpolate.BSpline(t, b.c.imag, k)
+
+        xx = xp.linspace(t[k], t[-k-1], 20)
+        return b(xx), b_re(xx) + 1j * b_im(xx)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_nan(self, xp, scp):
+        # nan in, nan out.
+        b = scp.interpolate.BSpline.basis_element(
+            xp.asarray([0, 1, 1, 2]))
+        return b(xp.asarray([xp.nan]))
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @testing.with_requires('scipy>=1.8.0')
+    def test_design_matrix(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+
+        t = xp.arange(-1, 7, dtype=dtype)
+        x = xp.arange(1, 5, dtype=dtype)
+        k = 2
+
+        mat = scp.interpolate.BSpline.design_matrix(x, t, k)
+        return mat.todense()
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_single_derivative(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+
+        k = 2
+        t = xp.arange(7, dtype=dtype)
+        c = xp.asarray(self.c, dtype=dtype)
+
+        b = scp.interpolate.BSpline(t, c, k, extrapolate=self.extrapolate)
+        return b.derivative().tck
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_multiple_derivative(self, xp, scp):
+        b = self._make_random_spline(xp, scp, k=5)
+        t, c, k = b.tck
+        xx = xp.linspace(t[k], t[-k-1], 20)
+        comp = []
+        for j in range(1, k):
+            b = b.derivative()
+            comp.append(b(xx))
+        return comp
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_antiderivative_tck(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+
+        k = 2
+        t = xp.arange(7, dtype=dtype)
+        c = xp.asarray(self.c, dtype=dtype)
+
+        b = scp.interpolate.BSpline(t, c, k, extrapolate=self.extrapolate)
+        return b.antiderivative().tck
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_antiderivative(self, xp, scp):
+        b = self._make_random_spline(xp, scp)
+        t, c, k = b.tck
+        xx = xp.linspace(t[k], t[-k-1], 20)
+        r1 = b.antiderivative().derivative()(xx)
+
+        # repeat with N-D array for c
+        c = xp.c_[c, c, c]
+        c = xp.dstack((c, c))
+        b = scp.interpolate.BSpline(t, c, k)
+        r2 = b.antiderivative().derivative()(xx)
+        return r1, r2
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_integral(self, xp, scp):
+        # x for x < 1 else 2 - x
+        b = scp.interpolate.BSpline.basis_element(xp.asarray([0, 1, 2]))
+
+        ret = []
+        ret.append(b.integrate(0, 1))
+        ret.append(b.integrate(1, 0))
+        ret.append(b.integrate(1, 0))
+
+        # extrapolate or zeros outside of [0, 2]; default is yes
+        ret.append(b.integrate(-1, 1))
+        ret.append(b.integrate(-1, 1, extrapolate=True))
+        ret.append(b.integrate(-1, 1, extrapolate=False))
+        ret.append(b.integrate(1, -1, extrapolate=False))
+
+        # Test ``_fitpack._splint()``
+        ret.append(b.integrate(1, -1, extrapolate=False))
+
+        # Test ``extrapolate='periodic'``.
+        b.extrapolate = 'periodic'
+
+        ret.append(b.integrate(0, 2))
+        ret.append(b.integrate(2, 0))
+        ret.append(b.integrate(-9, -7))
+        ret.append(b.integrate(-8, -4))
+        ret.append(b.integrate(0.5, 1.5))
+        ret.append(b.integrate(1.5, 3))
+        ret.append(b.integrate(1.5 + 12, 3 + 12))
+        ret.append(b.integrate(1.5, 3 + 12))
+        ret.append(b.integrate(0, -1))
+        ret.append(b.integrate(-9, -10))
+        ret.append(b.integrate(0, -9))
+        return [xp.asarray(x) for x in ret]
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_integrate(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+
+        k = 2
+        t = xp.arange(7, dtype=dtype)
+        c = xp.asarray(self.c, dtype=dtype)
+
+        b = scp.interpolate.BSpline(t, c, k, extrapolate=self.extrapolate)
+        return xp.asarray(b.integrate(0, 5))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=True)
+    def test_axis(self, xp, scp):
+        n, k = 22, 3
+        t = xp.linspace(0, 1, n + k + 1)
+        x = testing.shaped_random((3, 4, 5), xp)
+
+        ret = []
+        for ax in range(-4, 4):
+            sh = [6, 7, 8]
+            # We need the positive axis for some of the indexing and
+            # slices used in this test.
+            pos_axis = self.axis % 4
+            sh.insert(pos_axis, n)   # [22, 6, 7, 8] etc
+            c = testing.shaped_random(sh, xp)
+            b = scp.interpolate.BSpline(t, c, k, axis=ax)
+            ret.append(b(x))
+
+        # -c.ndim <= axis < c.ndim
+        for ax in [-c.ndim - 1, c.ndim]:
+            scp.interpolate.BSpline(t, c, k, axis=ax)
+
+        # derivative, antiderivative keeps the axis
+        BSpline = scp.interpolate.BSpline
+        for b1 in [BSpline(t, c, k, axis=self.axis).derivative(),
+                   BSpline(t, c, k, axis=self.axis).derivative(2),
+                   BSpline(t, c, k, axis=self.axis).antiderivative(),
+                   BSpline(t, c, k, axis=self.axis).antiderivative(2)]:
+            ret.append(b1.axis)
+        return b1
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_neg_axis(self, xp, scp):
+        k = 2
+        t = xp.asarray([0, 1, 2, 3, 4, 5, 6])
+        c = xp.asarray([[-1, 2, 0, -1], [2, 0, -3, 1]])
+
+        spl = scp.interpolate.BSpline(t, c, k, axis=-1)
+        return spl(xp.asarray([2.5]))
+
+    @testing.numpy_cupy_allclose(
+        scipy_name='scp', rtol=3e-5, atol=3e-5)
+    @testing.with_requires('scipy>=1.8.0')
+    def test_design_matrix_same_as_BSpline_call(self, xp, scp):
+        """Test that design_matrix(x) is equivalent to BSpline(..)(x)."""
+        ret = []
+
+        has_extrapolate = True
+        for mod in [cupyx.scipy.interpolate, interpolate]:
+            sig = inspect.signature(mod.BSpline.design_matrix)
+            has_extrapolate = has_extrapolate and (
+                'extrapolate' in sig.parameters)
+
+        kwargs = {}
+        if has_extrapolate:
+            kwargs = {'extrapolate': self.extrapolate}
+
+        for k in range(0, 5):
+            x = testing.shaped_random((10 * (k + 1),), xp, scale=1, seed=1234)
+            xmin, xmax = xp.amin(x), xp.amax(x)
+
+            t = xp.r_[xp.linspace(xmin - 2, xmin - 1, k),
+                      xp.linspace(xmin, xmax, 2 * (k + 1)),
+                      xp.linspace(xmax + 1, xmax + 2, k)]
+            c = xp.eye(len(t) - k - 1)
+            bspline = scp.interpolate.BSpline(t, c, k, self.extrapolate)
+            ret.append(bspline(x))
+            ret.append(scp.interpolate.BSpline.design_matrix(
+                x, t, k, **kwargs).todense())
+
+            # extrapolation regime
+            if has_extrapolate:
+                x = xp.array([xmin - 10, xmin - 1, xmax + 1.5, xmax + 10])
+                if not self.extrapolate:
+                    with pytest.raises(ValueError):
+                        scp.interpolate.BSpline.design_matrix(
+                            x, t, k, self.extrapolate)
+                else:
+                    ret.append(bspline(x))
+                    ret.append(scp.interpolate.BSpline.design_matrix(
+                        x, t, k, self.extrapolate).todense())
+
+        return ret
diff --git a/tests/cupyx_tests/scipy_tests/interpolate_tests/test_bspline2.py b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_bspline2.py
new file mode 100644
index 0000000..6522e08
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_bspline2.py
@@ -0,0 +1,458 @@
+import pytest
+import cupy
+from cupy import testing
+from cupy.cuda import runtime
+
+import numpy as _np
+import cupyx.scipy.interpolate as csi  # NOQA
+
+try:
+    from scipy import interpolate  # NOQA
+except ImportError:
+    pass
+
+
+@pytest.mark.skipif(runtime.is_hip, reason='csrlsvqr not available')
+@testing.with_requires("scipy")
+class TestInterp:
+    #
+    # Test basic ways of constructing interpolating splines.
+    #
+    def get_xy(self, xp):
+        xx = xp.linspace(0., 2.*cupy.pi, 11)
+        yy = xp.sin(xx)
+        return xx, yy
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=TypeError)
+    def test_non_int_order(self, xp, scp):
+        x, y = self.get_xy(xp)
+        return scp.interpolate.make_interp_spline(x, y, k=2.5)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_order_0(self, xp, scp):
+        x, y = self.get_xy(xp)
+        return (scp.interpolate.make_interp_spline(x, y, k=0)(x),
+                scp.interpolate.make_interp_spline(x, y, k=0, axis=-1)(x))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_linear(self, xp, scp):
+        x, y = self.get_xy(xp)
+        return (scp.interpolate.make_interp_spline(x, y, k=1)(x),
+                scp.interpolate.make_interp_spline(x, y, k=1, axis=-1)(x))
+
+    @testing.with_requires("scipy >= 1.10")
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    @pytest.mark.parametrize('k', [0, 1, 2, 3])
+    def test_incompatible_x_y(self, xp, scp, k):
+        x = [0, 1, 2, 3, 4, 5]
+        y = [0, 1, 2, 3, 4, 5, 6, 7]
+        scp.interpolate.make_interp_spline(x, y, k=k)
+
+    @testing.with_requires("scipy >= 1.10")
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    @pytest.mark.parametrize('k', [0, 1, 2, 3])
+    def test_broken_x(self, xp, scp, k):
+        x = [0, 1, 1, 2, 3, 4]      # duplicates
+        y = [0, 1, 2, 3, 4, 5]
+        scp.interpolate.make_interp_spline(x, y, k=k)
+
+    @testing.with_requires("scipy >= 1.10")
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    @pytest.mark.parametrize('k', [0, 1, 2, 3])
+    def test_broken_x_2(self, xp, scp, k):
+        x = [0, 2, 1, 3, 4, 5]      # unsorted
+        y = [0, 1, 2, 3, 4, 5]
+        scp.interpolate.make_interp_spline(x, y, k=k)
+
+    @testing.with_requires("scipy >= 1.10")
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    @pytest.mark.parametrize('k', [0, 1, 2, 3])
+    def test_broken_x_3(self, xp, scp, k):
+        x = xp.asarray([0, 1, 2, 3, 4, 5]).reshape((1, -1))     # 1D
+        y = [0, 1, 2, 3, 4, 5]
+        scp.interpolate.make_interp_spline(x, y, k=1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    @pytest.mark.parametrize('k', [3, 5])
+    def test_not_a_knot(self, xp, scp, k):
+        x, y = self.get_xy(xp)
+        return scp.interpolate.make_interp_spline(x, y, k=k)(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    @pytest.mark.parametrize('k', [0, 1, 3, 5])
+    def test_int_xy(self, xp, scp, k):
+        x = xp.arange(10).astype(int)
+        y = xp.arange(10).astype(int)
+        return scp.interpolate.make_interp_spline(x, y, k=k)(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('k', [0, 1, 2, 3])
+    def test_sliced_input(self, xp, scp, k):
+        #  non C contiguous arrays
+        xx = xp.linspace(-1, 1, 100)
+        x = xx[::5]
+        y = xx[::5]
+        return scp.interpolate.make_interp_spline(x, y, k=k)(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    @pytest.mark.parametrize('k', [1, 2, 3, 5])
+    def test_list_input(self, xp, scp, k):
+        # regression test for gh-8714: TypeError for x, y being lists and k=2
+        x = list(range(10))
+        y = [a**2 for a in x]
+        return scp.interpolate.make_interp_spline(x, y, k=k)(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_quadratic_deriv_right(self, xp, scp):
+        x, y = self.get_xy(xp)
+        der = [(1, 8.)]  # order, value: f'(x) = 8.
+
+        # derivative at right-hand edge
+        b = scp.interpolate.make_interp_spline(x, y, k=2, bc_type=(None, der))
+        return b(x), b(x[-1], 1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_quadratic_deriv_left(self, xp, scp):
+        x, y = self.get_xy(xp)
+        der = [(1, 8.)]  # order, value: f'(x) = 8.
+
+        # derivative at left-hand edge
+        b = scp.interpolate.make_interp_spline(x, y, k=2, bc_type=(der, None))
+        return b(x), b(x[0], 1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_cubic_deriv_deriv(self, xp, scp):
+        x, y = self.get_xy(xp)
+        # first derivatives at left & right edges:
+        der_l, der_r = [(1, 3.)], [(1, 4.)]
+        b = scp.interpolate.make_interp_spline(
+            x, y, k=3, bc_type=(der_l, der_r))
+        return b(x), b(x[0], 1), b(x[-1], 1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_cubic_deriv_natural(self, xp, scp):
+        x, y = self.get_xy(xp)
+
+        # 'natural' cubic spline, zero out 2nd derivatives at the boundaries
+        der_l, der_r = [(2, 0)], [(2, 0)]
+        b = scp.interpolate.make_interp_spline(x, y, k=3,
+                                               bc_type=(der_l, der_r))
+        return b(x), b(x[0], 2), b(x[-1], 2)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_quintic_derivs(self, xp, scp):
+        k, n = 5, 7
+        x = xp.arange(n).astype(xp.float_)
+        y = xp.sin(x)
+        der_l = [(1, -12.), (2, 1)]
+        der_r = [(1, 8.), (2, 3.)]
+        b = scp.interpolate.make_interp_spline(x, y, k=k,
+                                               bc_type=(der_l, der_r))
+        return b(x), b(x[0], 1), b(x[0], 2), b(x[-1], 1), b(x[-1], 2)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_knots_not_data_sites(self, xp, scp):
+        # Knots need not coincide with the data sites.
+        # use a quadratic spline, knots are at data averages,
+        # two additional constraints are zero 2nd derivatives at edges
+        k = 2
+        x, y = self.get_xy(xp)
+        t = xp.r_[(x[0],)*(k+1),
+                  (x[1:] + x[:-1]) / 2.,
+                  (x[-1],)*(k+1)]
+        b = scp.interpolate.make_interp_spline(x, y, k, t,
+                                               bc_type=([(2, 0)], [(2, 0)]))
+        return b(x), b(x[0], 2), b(x[-1], 2)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_minimum_points_and_deriv(self, xp, scp):
+        # interpolation of f(x) = x**3 between 0 and 1. f'(x) = 3 * xx**2 and
+        # f'(0) = 0, f'(1) = 3.
+        k = 3
+        x = [0., 1.]
+        y = [0., 1.]
+        b = scp.interpolate.make_interp_spline(x, y, k,
+                                               bc_type=([(1, 0.)], [(1, 3.)]))
+        xx = xp.linspace(0., 1.)
+        return b(xx)    # assert_allclose(b(xx), xx**3)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_complex(self, xp, scp):
+        x, y = self.get_xy(xp)
+        y = y + 1j*y
+
+        # first derivatives at left & right edges:
+        der_l, der_r = [(1, 3.j)], [(1, 4.+2.j)]
+        b = scp.interpolate.make_interp_spline(x, y, k=3,
+                                               bc_type=(der_l, der_r))
+        return b(x), b(x[0], 1), b(x[-1], 1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_complex_01(self, xp, scp):
+        # also test zero and first order
+        x, y = self.get_xy(xp)
+        y = y + 1j*y
+        return (scp.interpolate.make_interp_spline(x, y, k=0)(x),
+                scp.interpolate.make_interp_spline(x, y, k=1)(x))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_multiple_rhs(self, xp, scp):
+        x, _ = self.get_xy(xp)
+        yy = xp.c_[xp.sin(x), xp.cos(x)]
+        der_l = [(1, [1., 2.])]
+        der_r = [(1, [3., 4.])]
+
+        b = scp.interpolate.make_interp_spline(x, yy, k=3,
+                                               bc_type=(der_l, der_r))
+        return b(x), b(x[0], 1), b(x[-1], 1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_shapes(self, xp, scp):
+        xp.random.seed(1234)
+        k, n = 3, 22
+        x = xp.sort(xp.random.random(size=n))
+        y = xp.random.random(size=(n, 5, 6, 7))
+
+        b1 = scp.interpolate.make_interp_spline(x, y, k)
+        assert b1.c.shape == (n, 5, 6, 7)
+
+        # now throw in some derivatives
+        d_l = [(1, xp.random.random((5, 6, 7)))]
+        d_r = [(1, xp.random.random((5, 6, 7)))]
+        b2 = scp.interpolate.make_interp_spline(x, y, k, bc_type=(d_l, d_r))
+        assert b2.c.shape == (n + k - 1, 5, 6, 7)
+
+        return b1.c.shape + b2.c.shape
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_string_aliases_1(self, xp, scp):
+        x, _ = self.get_xy(xp)
+        y = xp.sin(x)
+
+        # a single string is duplicated
+        b1 = scp.interpolate.make_interp_spline(x, y, k=3,
+                                                bc_type='natural')
+        b2 = scp.interpolate.make_interp_spline(x, y, k=3,
+                                                bc_type=([(2, 0)], [(2, 0)]))
+        return b1.c, b2.c
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_string_aliases_2(self, xp, scp):
+        x, _ = self.get_xy(xp)
+        y = xp.sin(x)
+        # two strings are handled
+        b1 = scp.interpolate.make_interp_spline(x, y, k=3,
+                                                bc_type=('natural', 'clamped'))
+        b2 = scp.interpolate.make_interp_spline(x, y, k=3,
+                                                bc_type=([(2, 0)], [(1, 0)]))
+        return b1.c, b2.c
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_string_aliases_3(self, xp, scp):
+        x, _ = self.get_xy(xp)
+        y = xp.sin(x)
+
+        # one-sided BCs are OK
+        b1 = scp.interpolate.make_interp_spline(x, y, k=2,
+                                                bc_type=(None, 'clamped'))
+        b2 = scp.interpolate.make_interp_spline(x, y, k=2,
+                                                bc_type=(None, [(1, 0.0)]))
+        return b1.c, b2.c
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_string_aliases_4(self, xp, scp):
+        x, _ = self.get_xy(xp)
+        y = xp.sin(x)
+
+        # 'not-a-knot' is equivalent to None
+        b1 = scp.interpolate.make_interp_spline(x, y, k=3,
+                                                bc_type='not-a-knot')
+        b2 = scp.interpolate.make_interp_spline(x, y, k=3,
+                                                bc_type=None)
+        return b1.c, b2.c
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_string_aliases_5(self, xp, scp):
+        x, _ = self.get_xy(xp)
+        y = xp.sin(x)
+
+        # unknown strings do not pass
+        scp.interpolate.make_interp_spline(x, y, k=3, bc_type='typo')
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_string_aliases_6(self, xp, scp):
+        x, _ = self.get_xy(xp)
+
+        # string aliases are handled for 2D values
+        yy = xp.c_[xp.sin(x), xp.cos(x)]
+        der_l = [(1, [0., 0.])]
+        der_r = [(2, [0., 0.])]
+
+        b2 = scp.interpolate.make_interp_spline(x, yy, k=3,
+                                                bc_type=(der_l, der_r))
+        b1 = scp.interpolate.make_interp_spline(x, yy, k=3,
+                                                bc_type=('clamped', 'natural'))
+        return b1.c, b2.c
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_string_aliases_7(self, xp, scp):
+        # ... and for N-D values:
+
+        # cupy and numpy random streams differ for the same seed,
+        # hence use numpy.random for both numpy and cupy
+        rng = _np.random.RandomState(1234)
+        k, n = 3, 22
+        x = _np.sort(rng.uniform(size=n))
+        y = rng.uniform(size=(n, 5, 6, 7))
+        if xp is cupy:
+            x = cupy.asarray(x)
+            y = cupy.asarray(y)
+
+        # now throw in some derivatives
+        d_l = [(1, xp.zeros((5, 6, 7)))]
+        d_r = [(1, xp.zeros((5, 6, 7)))]
+        b1 = scp.interpolate.make_interp_spline(x, y, k, bc_type=(d_l, d_r))
+        b2 = scp.interpolate.make_interp_spline(x, y, k, bc_type='clamped')
+        return b1.c, b2.c
+
+    def test_deriv_spec(self):
+        # If one of the derivatives is omitted, the spline definition is
+        # incomplete.
+        x = y = [1.0, 2, 3, 4, 5, 6]
+
+        with pytest.raises(ValueError):
+            csi.make_interp_spline(x, y, bc_type=([(1, 0.)], None))
+
+        with pytest.raises(ValueError):
+            csi.make_interp_spline(x, y, bc_type=(1, 0.))
+
+        with pytest.raises(ValueError):
+            csi.make_interp_spline(x, y, bc_type=[(1, 0.)])
+
+        with pytest.raises(ValueError):
+            csi.make_interp_spline(x, y, bc_type=42)
+
+        # CubicSpline expects`bc_type=(left_pair, right_pair)`, while
+        # here we expect `bc_type=(iterable, iterable)`.
+        l, r = (1, 0.0), (1, 0.0)
+        with pytest.raises(ValueError):
+            csi.make_interp_spline(x, y, bc_type=(l, r))
+
+    def test_full_matrix(self):
+        from cupyx.scipy.interpolate._bspline2 import (
+            _make_interp_spline_full_matrix)
+        cupy.random.seed(1234)
+        k, n = 3, 7
+        x = cupy.sort(cupy.random.random(size=n))
+        y = cupy.random.random(size=n)
+
+        # test not-a-knot
+        b = csi.make_interp_spline(x, y, k=3)
+        bf = _make_interp_spline_full_matrix(x, y, k, b.t, bc_type=None)
+        cupy.testing.assert_allclose(b.c, bf.c, atol=1e-14, rtol=1e-14)
+
+        # test with some b.c.
+        b = csi.make_interp_spline(x, y, k=3, bc_type='natural')
+        bf = _make_interp_spline_full_matrix(x, y, k, b.t, bc_type='natural')
+        cupy.testing.assert_allclose(b.c, bf.c, atol=1e-13)
+
+    # test periodic constructor #
+
+
+@testing.with_requires("scipy>=1.7")
+@pytest.mark.skipif(runtime.is_hip, reason='csrlsvqr not available')
+class TestInterpPeriodic:
+    #
+    # Test basic ways of constructing interpolating splines w/periodic BCs.
+    #
+    def get_xy(self, xp):
+        xx = xp.linspace(0., 2.*cupy.pi, 11)
+        yy = xp.sin(xx)
+        return xx, yy
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_periodic(self, xp, scp):
+        x, y = self.get_xy(xp)
+        # k = 5 here for more derivatives
+        b = scp.interpolate.make_interp_spline(x, y, k=5, bc_type='periodic')
+        for i in range(1, 5):
+            xp.testing.assert_allclose(b(x[0], nu=i),
+                                       b(x[-1], nu=i), atol=1e-11)
+        return b(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_periodic_axis1(self, xp, scp):
+        x, y = self.get_xy(xp)
+        b = scp.interpolate.make_interp_spline(x, y, k=5,
+                                               bc_type='periodic', axis=-1)
+        for i in range(1, 5):
+            xp.testing.assert_allclose(b(x[0], nu=i),
+                                       b(x[-1], nu=i), atol=1e-11)
+        return b(x)
+
+    @pytest.mark.parametrize('k', [2, 3, 4, 5, 6, 7])
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_periodic_random(self, xp, scp, k):
+        # tests for both cases (k > n and k <= n)
+        n = 15
+        _np.random.seed(1234)
+        x = _np.sort(_np.random.random_sample(n) * 10)
+        y = _np.random.random_sample(n) * 100
+        if xp is cupy:
+            x = cupy.asarray(x)
+            y = cupy.asarray(y)
+
+        y[0] = y[-1]
+        b = scp.interpolate.make_interp_spline(x, y, k=k, bc_type='periodic')
+
+        return b(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_periodic_axis(self, xp, scp):
+        x, y = self.get_xy(xp)
+        n = x.shape[0]
+        _np.random.seed(1234)
+        x = _np.random.random_sample(n) * 2 * _np.pi
+        x = _np.sort(x)
+        if xp is cupy:
+            x = cupy.asarray(x)
+        x[0] = 0.
+        x[-1] = 2 * xp.pi
+        y = xp.zeros((2, n))
+        y[0] = xp.sin(x)
+        y[1] = xp.cos(x)
+        b = scp.interpolate.make_interp_spline(x, y, k=5,
+                                               bc_type='periodic', axis=1)
+        return b(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_periodic_points_exception(self, xp, scp):
+        # first and last points should match when periodic case expected
+        n, k = 8, 5
+        x = xp.linspace(0, n, n)
+        y = x
+        return scp.interpolate.make_interp_spline(x, y, k=k,
+                                                  bc_type='periodic')
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_periodic_knots_exception(self, xp, scp):
+        # `periodic` case does not work with passed vector of knots
+        n, k = 7, 3
+        x = xp.linspace(0, n, n)
+        y = x**2
+        t = xp.zeros(n + 2 * k)
+        return scp.interpolate.make_interp_spline(x, y, k, t, 'periodic')
+
+    def test_periodic_cubic(self):
+        # edge case: Cubic interpolation on 3 points
+        # TODO: refactor when PPoly / CubicHermiteSpline is available
+        n = 3
+        cupy.random.seed(1234)
+        x = cupy.sort(cupy.random.random_sample(n) * 10)
+        y = cupy.random.random_sample(n) * 100
+        y[0] = y[-1]
+        b = csi.make_interp_spline(x, y, k=3, bc_type='periodic')
+
+        cub = interpolate.CubicSpline(x.get(), y.get(), bc_type='periodic')
+        cupy.testing.assert_allclose(b(x),
+                                     cub(x.get()), atol=1e-14)
diff --git a/tests/cupyx_tests/scipy_tests/interpolate_tests/test_polyint.py b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_polyint.py
new file mode 100644
index 0000000..d5fa098
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_polyint.py
@@ -0,0 +1,617 @@
+
+import io
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupy import testing
+import cupyx.scipy.interpolate  # NOQA
+from cupyx.scipy.interpolate import CubicHermiteSpline
+
+try:
+    from scipy import interpolate  # NOQA
+except ImportError:
+    pass
+
+
+if cupy.cuda.runtime.runtimeGetVersion() < 11000:
+    # Workarounds precison issues in CUDA 10.2 + float16
+    default_atol = 5e-2
+    default_rtol = 1e-1
+else:
+    default_atol = 0
+    default_rtol = 1e-7
+
+
+@testing.with_requires("scipy")
+class TestBarycentric:
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_lagrange(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 100, dtype=dtype)
+        xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.BarycentricInterpolator(xs, ys)
+        return P(test_xs)
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-5, rtol=1e-4)
+    def test_scalar(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'iu':
+            pytest.skip()
+        true_poly = numpy.poly1d([-1, 2, 6, -3, 2])
+        xs = numpy.linspace(-1, 1, 10, dtype=dtype)
+        ys = true_poly(xs)
+        if xp is cupy:
+            xs = cupy.asarray(xs)
+            ys = cupy.asarray(ys)
+        P = scp.interpolate.BarycentricInterpolator(xs, ys)
+        return P(xp.array(7, dtype=dtype))
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_delayed(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 100, dtype=dtype)
+        xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.BarycentricInterpolator(xs)
+        P.set_yi(ys)
+        return P(test_xs)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_append(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 100, dtype=dtype)
+        xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.BarycentricInterpolator(xs[:3], ys[:3])
+        P.add_xi(xs[3:], ys[3:])
+        return P(test_xs)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_vector(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        xs = xp.array([0, 1, 2], dtype=dtype)
+        ys = xp.array([[0, 1], [1, 0], [2, 1]], dtype=dtype)
+        test_xs = xp.linspace(-1, 3, 100, dtype=dtype)
+        P = scp.interpolate.BarycentricInterpolator(xs, ys)
+        return P(test_xs)
+
+    @pytest.mark.parametrize('test_xs', [0, [0], [0, 1]])
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_shapes_scalarvalue(self, xp, scp, test_xs):
+        true_poly = xp.poly1d([-2, 3, 5, 1, -3])
+        xs = xp.linspace(-1, 10, 10)
+        ys = true_poly(xs)
+        P = scp.interpolate.BarycentricInterpolator(xs, ys)
+        test_xs = xp.array(test_xs)
+        return xp.shape(P(test_xs))
+
+    @pytest.mark.parametrize('test_xs', [0, [0], [0, 1]])
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_shapes_vectorvalue(self, xp, scp, test_xs):
+        true_poly = xp.poly1d([4, -5, 3, 2, -4])
+        xs = xp.linspace(-10, 10, 20)
+        ys = true_poly(xs)
+        P = scp.interpolate.BarycentricInterpolator(
+            xs, xp.outer(ys, xp.arange(3)))
+        test_xs = xp.array(test_xs)
+        return xp.shape(P(test_xs))
+
+    @pytest.mark.parametrize('test_xs', [0, [0], [0, 1]])
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_shapes_1d_vectorvalue(self, xp, scp, test_xs):
+        true_poly = xp.poly1d([-3, -1, 4, 9, 8])
+        xs = xp.linspace(-1, 10, 10)
+        ys = true_poly(xs)
+        P = scp.interpolate.BarycentricInterpolator(
+            xs, xp.outer(ys, xp.array([1])))
+        test_xs = xp.array(test_xs)
+        return xp.shape(P(test_xs))
+
+    @testing.with_requires("scipy>=1.8.0")
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_large_chebyshev(self, xp, scp, dtype):
+        n = 100
+        j = numpy.arange(n + 1, dtype=dtype).astype(numpy.float64)
+        x = numpy.cos(j * numpy.pi / n)
+
+        if xp is cupy:
+            j = cupy.asarray(j)
+            x = cupy.asarray(x)
+        # The weights for Chebyshev points against SciPy counterpart
+        return scp.interpolate.BarycentricInterpolator(x).wi
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_complex(self, xp, scp):
+        x = xp.array([1, 2, 3, 4])
+        y = xp.array([1, 2, 1j, 3])
+        xi = xp.array([0, 8, 1, 5])
+        return scp.interpolate.BarycentricInterpolator(x, y)(xi)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_wrapper(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-2, 2, 5, dtype=dtype)
+        xs = xp.linspace(-2, 2, 5, dtype=dtype)
+        ys = true_poly(xs)
+        return scp.interpolate.barycentric_interpolate(xs, ys, test_xs)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_array_input(self, xp, scp, dtype):
+        x = 1000 * xp.arange(1, 11, dtype=dtype)
+        y = xp.arange(1, 11, dtype=dtype)
+        xi = xp.array(1000 * 9.5)
+        return scp.interpolate.barycentric_interpolate(x, y, xi)
+
+
+@testing.with_requires("scipy")
+class TestKrogh:
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_lagrange(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        xs = xp.linspace(-1, 1, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        out = P(test_xs)
+        print(out.dtype)
+        return out
+
+    @testing.for_all_dtypes(no_bool=True, no_float16=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-6, rtol=1e-6)
+    def test_scalar(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        true_poly = numpy.poly1d([-1, 2, 6, -3, 2])
+        xs = numpy.linspace(-1, 1, 10, dtype=dtype)
+        ys = true_poly(xs)
+        if xp is cupy:
+            xs = cupy.asarray(xs)
+            ys = cupy.asarray(ys)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        return P(xp.array(7, dtype=dtype))
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_derivatives(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        xs = xp.linspace(-1, 1, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        D = P.derivatives(test_xs)
+        return D
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_low_derivatives(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        xs = xp.linspace(-1, 1, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        D = P.derivatives(test_xs, len(xs) + 2)
+        return D
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_derivative(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        xs = xp.linspace(-1, 1, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        m = 10
+        return [P.derivative(test_xs, i) for i in range(m)]
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-7, rtol=1e-7)
+    def test_high_derivative(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        xs = xp.linspace(-1, 1, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        D = P.derivative(test_xs, 2 * len(xs))
+        return D
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_hermite(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        xs = xp.linspace(-1, 1, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        D = P(test_xs)
+        return D
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-7, rtol=1e-7)
+    def test_hermite_derivative(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        xs = xp.array([0, 0, 0], dtype=dtype)
+        ys = xp.array([1, 2, 3], dtype=dtype)
+        test_xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        return P(test_xs)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-7, rtol=1e-7)
+    def test_vector(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        xs = xp.array([0, 1, 2], dtype=dtype)
+        ys = xp.array([[0, 1], [1, 0], [2, 1]], dtype=dtype)
+        test_xs = xp.linspace(-1, 3, 5, dtype=dtype)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        return P(test_xs)
+
+    @testing.for_dtypes('bhilfd')
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_empty(self, xp, scp, dtype):
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        xs = xp.linspace(-5, 5, 5, dtype=dtype)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        return P(xp.array([]))
+
+    @pytest.mark.parametrize('test_xs', [0, [0], [0, 1]])
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_shapes_scalarvalue(self, xp, scp, test_xs):
+        true_poly = xp.poly1d([-2, 3, 5, 1, -3])
+        xs = xp.linspace(-1, 10, 10)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        test_xs = xp.array(test_xs)
+        return xp.shape(P(test_xs))
+
+    @pytest.mark.parametrize('test_xs', [0, [0], [0, 1]])
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_shapes_scalarvalue_derivatives(self, xp, scp, test_xs):
+        true_poly = xp.poly1d([-2, 3, 5, 1, -3])
+        xs = xp.linspace(-1, 10, 10)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(xs, ys)
+        test_xs = xp.array(test_xs)
+        return xp.shape(P.derivatives(test_xs))
+
+    @pytest.mark.parametrize('test_xs', [0, [0], [0, 1]])
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_shapes_vectorvalue(self, xp, scp, test_xs):
+        true_poly = xp.poly1d([4, -5, 3, 2, -4])
+        xs = xp.linspace(-10, 10, 20)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(
+            xs, xp.outer(ys, xp.arange(3)))
+        test_xs = xp.array(test_xs)
+        return xp.shape(P(test_xs))
+
+    @pytest.mark.parametrize('test_xs', [0, [0], [0, 1]])
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_shapes_vectorvalue_derivative(self, xp, scp, test_xs):
+        true_poly = xp.poly1d([4, -5, 3, 2, -4])
+        xs = xp.linspace(-10, 10, 20)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(
+            xs, xp.outer(ys, xp.arange(3)))
+        test_xs = xp.array(test_xs)
+        return xp.shape(P.derivatives(test_xs))
+
+    @pytest.mark.parametrize('test_xs', [0, [0], [0, 1]])
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_shapes_1d_vectorvalue(self, xp, scp, test_xs):
+        true_poly = xp.poly1d([-3, -1, 4, 9, 8])
+        xs = xp.linspace(-1, 10, 10)
+        ys = true_poly(xs)
+        P = scp.interpolate.KroghInterpolator(
+            xs, xp.outer(ys, xp.array([1])))
+        test_xs = xp.array(test_xs)
+        return xp.shape(P(test_xs))
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_wrapper(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-2, 2, 5, dtype=dtype)
+        xs = xp.linspace(-1, 1, 5, dtype=dtype)
+        ys = true_poly(xs)
+        return scp.interpolate.krogh_interpolate(xs, ys, test_xs)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(
+        atol=default_atol, rtol=default_rtol, scipy_name='scp')
+    def test_wrapper2(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            pytest.skip()
+        true_poly = xp.poly1d([-2, 3, 1, 5, -4])
+        test_xs = xp.linspace(-2, 2, 5, dtype=dtype)
+        xs = xp.linspace(-1, 1, 5, dtype=dtype)
+        ys = true_poly(xs)
+        return scp.interpolate.krogh_interpolate(xs, ys, test_xs, der=3)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_derivatives_complex(self, xp, scp):
+        x = xp.array([-1, -1, 0, 1, 1])
+        y = xp.array([1, 1.0j, 0, -1, 1.0j])
+        P = scp.interpolate.KroghInterpolator(x, y)
+        D = P.derivatives(xp.array(0))
+        return D
+
+
+@testing.with_requires("scipy>=1.10.0")
+class TestZeroSizeArrays:
+    # regression tests for gh-17241 : CubicSpline et al must not segfault
+    # when y.size == 0
+    # The two methods below are _almost_ the same, but not quite:
+    # one is for objects which have the `bc_type` argument (CubicSpline)
+    # and the other one is for those which do not (Pchip, Akima1D)
+
+    # XXX: add CubicSpline to the test loop, when implemented
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('y_shape', [(10, 0, 5), (10, 5, 0)])
+    @pytest.mark.parametrize('bc_type',
+                             ['not-a-knot', 'periodic', 'natural', 'clamped'])
+    @pytest.mark.parametrize('axis', [0, 1, 2])
+    @pytest.mark.parametrize('klass', ['make_interp_spline', ])
+    def test_zero_size(self, xp, scp, klass, y_shape, bc_type, axis):
+        if runtime.is_hip and bc_type == 'periodic':
+            pytest.xfail('Not implemented on HIP/ROCm')
+        x = xp.arange(10)
+        y = xp.zeros(y_shape)
+        xval = xp.arange(3)
+
+        cls = getattr(scp.interpolate, klass)
+        obj = cls(x, y, bc_type=bc_type)
+        r1 = obj(xval)
+        assert r1.size == 0
+        assert r1.shape == xval.shape + y.shape[1:]
+
+        # Also check with an explicit non-default axis
+        yt = xp.moveaxis(y, 0, axis)  # (10, 0, 5) --> (0, 10, 5) if axis=1 etc
+
+        obj = cls(x, yt, bc_type=bc_type, axis=axis)
+        sh = yt.shape[:axis] + (xval.size, ) + yt.shape[axis+1:]
+        r2 = obj(xval)
+        assert r2.size == 0
+        assert r2.shape == sh
+        return r1, r2
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('y_shape', [(10, 0, 5), (10, 5, 0)])
+    @pytest.mark.parametrize('axis', [0, 1, 2])
+    @pytest.mark.parametrize('klass',
+                             ['PchipInterpolator', 'Akima1DInterpolator'])
+    def test_zero_size_2(self, xp, scp, klass, y_shape, axis):
+        x = xp.arange(10)
+        y = xp.zeros(y_shape)
+        xval = xp.arange(3)
+
+        cls = getattr(scp.interpolate, klass)
+        obj = cls(x, y)
+        r1 = obj(xval)
+        assert r1.size == 0
+        assert r1.shape == xval.shape + y.shape[1:]
+
+        # Also check with an explicit non-default axis
+        yt = xp.moveaxis(y, 0, axis)  # (10, 0, 5) --> (0, 10, 5) if axis=1 etc
+
+        obj = cls(x, yt, axis=axis)
+        sh = yt.shape[:axis] + (xval.size, ) + yt.shape[axis+1:]
+        r2 = obj(xval)
+        assert r2.size == 0
+        assert r2.shape == sh
+        return r1, r2
+
+
+@testing.with_requires("scipy")
+class TestCubicHermiteSpline:
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_correctness(self, xp, scp):
+        x = xp.asarray([0, 2, 7])
+        y = xp.asarray([-1, 2, 3])
+        dydx = xp.asarray([0, 3, 7])
+        s = scp.interpolate.CubicHermiteSpline(x, y, dydx)
+        return s(x), s(x, 1)
+
+    def test_ctor_error_handling(self):
+        x = cupy.asarray([1, 2, 3])
+        y = cupy.asarray([0, 3, 5])
+        dydx = cupy.asarray([1, -1, 2, 3])
+        dydx_with_nan = cupy.asarray([1, 0, cupy.nan])
+
+        with pytest.raises(ValueError):
+            CubicHermiteSpline(x, y, dydx)
+
+        with pytest.raises(ValueError):
+            CubicHermiteSpline(x, y, dydx_with_nan)
+
+
+@testing.with_requires("scipy")
+class TestPCHIP:
+    def _make_random(self, xp, scp, npts=20):
+        xi = xp.sort(testing.shaped_random((npts,), xp))
+        yi = testing.shaped_random((npts,), xp)
+        return scp.interpolate.PchipInterpolator(xi, yi), xi, yi
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-3)
+    def test_overshoot(self, xp, scp):
+        # PCHIP should not overshoot
+        p, xi, _ = self._make_random(xp, scp)
+        results = []
+        for i in range(len(xi) - 1):
+            x1, x2 = xi[i], xi[i+1]
+            x = xp.linspace(x1, x2, 10)
+            yp = p(x)
+            results.append(yp)
+        return results
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-3)
+    def test_monotone(self, xp, scp):
+        # PCHIP should preserve monotonicty
+        p, xi, _ = self._make_random(xp, scp)
+        results = []
+        for i in range(len(xi) - 1):
+            x1, x2 = xi[i], xi[i+1]
+            x = xp.linspace(x1, x2, 10)
+            yp = p(x)
+            results.append(yp)
+        return results
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_cast(self, xp, scp):
+        # regression test for integer input data, see gh-3453
+        data = xp.array([[0, 4, 12, 27, 47, 60, 79, 87, 99, 100],
+                         [-33, -33, -19, -2, 12, 26, 38, 45, 53, 55]])
+        xx = xp.arange(100)
+        curve = scp.interpolate.PchipInterpolator(data[0], data[1])(xx)
+
+        data1 = data * 1.0
+        curve1 = scp.interpolate.PchipInterpolator(data1[0], data1[1])(xx)
+        return curve, curve1
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_nag(self, xp, scp):
+        # Example from NAG C implementation,
+        # http://nag.com/numeric/cl/nagdoc_cl25/html/e01/e01bec.html
+        # suggested in scipy/gh-5326 as a smoke test for the way the
+        # derivatives are computed (see also scipy/gh-3453)
+        dataStr = '''
+          7.99   0.00000E+0
+          8.09   0.27643E-4
+          8.19   0.43750E-1
+          8.70   0.16918E+0
+          9.20   0.46943E+0
+         10.00   0.94374E+0
+         12.00   0.99864E+0
+         15.00   0.99992E+0
+         20.00   0.99999E+0
+        '''
+        data = xp.loadtxt(io.StringIO(dataStr))
+        pch = scp.interpolate.PchipInterpolator(data[:, 0], data[:, 1])
+
+        resultStr = '''
+           7.9900       0.0000
+           9.1910       0.4640
+          10.3920       0.9645
+          11.5930       0.9965
+          12.7940       0.9992
+          13.9950       0.9998
+          15.1960       0.9999
+          16.3970       1.0000
+          17.5980       1.0000
+          18.7990       1.0000
+          20.0000       1.0000
+        '''
+        result = xp.loadtxt(io.StringIO(resultStr))
+        return pch(result[:, 0])
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_endslopes(self, xp, scp):
+        # this is a smoke test for scipy/gh-3453: PCHIP interpolator should not
+        # set edge slopes to zero if the data do not suggest zero
+        # edge derivatives
+        x = xp.array([0.0, 0.1, 0.25, 0.35])
+        y1 = xp.array([279.35, 0.5e3, 1.0e3, 2.5e3])
+        y2 = xp.array([279.35, 2.5e3, 1.50e3, 1.0e3])
+        pchip = scp.interpolate.PchipInterpolator
+        results = []
+        for pp in (pchip(x, y1), pchip(x, y2)):
+            for t in (x[0], x[-1]):
+                results.append(pp(t, 1))
+        return results
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_all_zeros(self, xp, scp):
+        x = xp.arange(10)
+        y = xp.zeros_like(x)
+
+        # this should work and not generate any warnings
+        with warnings.catch_warnings():
+            warnings.filterwarnings('error')
+            pch = scp.interpolate.PchipInterpolator(x, y)
+
+        xx = xp.linspace(0, 9, 101)
+        return pch(xx)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_two_points(self, xp, scp):
+        # regression test for gh-6222: pchip([0, 1], [0, 1]) fails because
+        # it tries to use a three-point scheme to estimate edge derivatives,
+        # while there are only two points available.
+        # Instead, it should construct a linear interpolator.
+        x = xp.linspace(0, 1, 11)
+        p = scp.interpolate.PchipInterpolator([0, 1], [0, 2])
+        return p(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_pchip_interpolate(self, xp, scp):
+        r1 = scp.interpolate.pchip_interpolate(
+            [1, 2, 3], [4, 5, 6], [0.5], der=1)
+        r2 = scp.interpolate.pchip_interpolate(
+            [1, 2, 3], [4, 5, 6], [0.5], der=0)
+        r3 = scp.interpolate.pchip_interpolate(
+            [1, 2, 3], [4, 5, 6], [0.5], der=[0, 1])
+        return r1, r2, xp.asarray(r3)
+
+
+@testing.with_requires("scipy")
+class TestAkima1D:
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_correctness(self, xp, scp):
+        x = xp.asarray([-1, 0, 1, 2, 3, 4])
+        # y = xp.asarray([-1, 2, 3])
+        y = testing.shaped_random((6, 1), xp)
+        s = scp.interpolate.Akima1DInterpolator(x, y)
+        return s(x), s(x, 1)
diff --git a/tests/cupyx_tests/scipy_tests/interpolate_tests/test_ppoly.py b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_ppoly.py
new file mode 100644
index 0000000..ac91607
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_ppoly.py
@@ -0,0 +1,1315 @@
+import pytest
+import numpy
+import numpy as np
+from operator import attrgetter
+
+import cupy
+from cupy import testing
+from cupy.testing import assert_allclose
+from cupy_backends.cuda.api import driver
+from cupy_backends.cuda.api import runtime
+import cupyx.scipy
+import cupyx.scipy.interpolate  # NOQA
+
+try:
+    import scipy
+    from scipy import interpolate as sc_interpolate  # NOQA
+    from scipy import special as sc_special   # NOQA
+except ImportError:
+    pass
+
+interpolate_cls = ['PPoly', 'BPoly']
+
+
+@testing.with_requires("scipy")
+class TestPPolyCommon:
+    """Test basic functionality for PPoly and BPoly."""
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    def test_sort_check(self, cls):
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            c = xp.array([[1, 4], [2, 5], [3, 6]])
+            x = xp.array([0, 1, 0.5])
+            cls1 = getattr(scp.interpolate, cls)
+            try:
+                cls1(c, x)
+            except ValueError:
+                pass
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    def test_ctor_c(self, cls):
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            # wrong shape: `c` must be at least 2D
+            cls1 = getattr(scp.interpolate, cls)
+            try:
+                cls1(c=xp.asarray([1, 2]),
+                     x=xp.asarray([0, 1]))
+            except ValueError:
+                pass
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_extend(self, xp, scp, cls):
+        # Test adding new points to the piecewise polynomial
+        cls = getattr(scp.interpolate, cls)
+        numpy.random.seed(1234)
+
+        order = 3
+        x = numpy.unique(numpy.r_[0, 10 * numpy.random.rand(30), 10])
+        x = xp.asarray(x)
+        c = 2*numpy.random.rand(order+1, len(x)-1, 2, 3) - 1
+        c = xp.asarray(c)
+
+        pp = cls(c[:, :9], x[:10])
+        pp.extend(c[:, 9:], x[10:])
+
+        pp2 = cls(c[:, 10:], x[10:])
+        pp2.extend(c[:, :10], x[:10])
+
+        pp3 = cls(c, x)
+
+        return pp.c, pp.x, pp2.c, pp2.x, pp3.c, pp3.x
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_extend_diff_orders(self, xp, scp, cls):
+        # Test extending polynomial with different order one
+        cls = getattr(scp.interpolate, cls)
+        numpy.random.seed(1234)
+
+        x = xp.linspace(0, 1, 6)
+        c = xp.asarray(numpy.random.rand(2, 5))
+
+        x2 = xp.linspace(1, 2, 6)
+        c2 = xp.asarray(numpy.random.rand(4, 5))
+
+        pp1 = cls(c, x)
+        pp2 = cls(c2, x2)
+
+        pp_comb = cls(c, x)
+        pp_comb.extend(c2, x2[1:])
+
+        # NB. doesn't match to pp1 at the endpoint, because pp1 is not
+        #     continuous with pp2 as we took random coefs.
+        xi1 = xp.linspace(0, 1, 300, endpoint=False)
+        xi2 = xp.linspace(1, 2, 300)
+
+        return pp1(xi1), pp_comb(xi1), pp2(xi2), pp_comb(xi2)
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_extend_descending(self, xp, scp, cls):
+        cls = getattr(scp.interpolate, cls)
+        numpy.random.seed(0)
+
+        order = 3
+        x = numpy.sort(numpy.random.uniform(0, 10, 20))
+        x = xp.asarray(x)
+        c = numpy.random.rand(order + 1, x.shape[0] - 1, 2, 3)
+        c = xp.asarray(c)
+
+        p1 = cls(c[:, :9], x[:10])
+        p1.extend(c[:, 9:], x[10:])
+
+        p2 = cls(c[:, 10:], x[10:])
+        p2.extend(c[:, :10], x[:10])
+
+        return p1.c, p1.x, p2.c, p2.x
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_shape(self, xp, scp, cls):
+        cls = getattr(scp.interpolate, cls)
+        numpy.random.seed(1234)
+
+        c = numpy.random.rand(8, 12, 5, 6, 7)
+        c = xp.asarray(c)
+        x = numpy.sort(numpy.random.rand(13))
+        x = xp.asarray(x)
+        xpts = numpy.random.rand(3, 4)
+        xpts = xp.asarray(xpts)
+
+        p = cls(c, x)
+        return p(xpts).shape
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    def test_shape_2(self, cls):
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            cls1 = getattr(scp.interpolate, cls)
+            numpy.random.seed(1234)
+
+            c = numpy.random.rand(8, 12, 5, 6, 7)
+            c = xp.asarray(c)
+            x = numpy.sort(numpy.random.rand(13))
+            x = xp.asarray(x)
+
+            # 'scalars'
+            p = cls1(c[..., 0, 0, 0], x)
+
+            assert p(0.5).shape == ()
+            assert p(xp.array(0.5)).shape == ()
+
+            with pytest.raises(ValueError):
+                xxx = xp.array([[0.1, 0.2], [0.4]], dtype=object)
+                p(xxx)   # raises ValueError
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    @pytest.mark.xfail(
+        runtime.is_hip and driver.get_build_version() < 5_00_00000,
+        reason='Fails on ROCm 4.3')
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_complex_coef(self, xp, scp, cls):
+        cls = getattr(scp.interpolate, cls)
+        numpy.random.seed(12345)
+        x = numpy.sort(numpy.random.random(13))
+        x = xp.array(x)
+        c = numpy.random.random((8, 12)) * (1. + 0.3j)
+        c = xp.array(c)
+        # c_re, c_im = c.real, c.imag
+        xpt = xp.array(numpy.random.random(5))
+
+        p = cls(c, x)
+        return [p(xpt, nu) for nu in [0, 1, 2]]
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    @pytest.mark.parametrize('axis', [0, 1, 2, 3])
+    def test_axis(self, cls, axis):
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            cls1 = getattr(scp.interpolate, cls)
+            numpy.random.seed(12345)
+            c = numpy.random.rand(3, 4, 5, 6, 7, 8)
+            c = xp.asarray(c)
+            c_s = c.shape
+            xpt = numpy.random.random((1, 2))
+            xpt = xp.asarray(xpt)
+
+            m = c.shape[axis+1]
+            x = numpy.sort(numpy.random.rand(m+1))
+            x = xp.asarray(x)
+
+            p = cls1(c, x, axis=axis)
+            assert (p.c.shape ==
+                    c_s[axis:axis+2] + c_s[:axis] + c_s[axis+2:])
+
+            res = p(xpt)
+            targ_shape = c_s[:axis] + xpt.shape + c_s[2+axis:]
+            assert res.shape == targ_shape
+
+            # deriv/antideriv does not drop the axis
+            for p1 in [cls1(c, x, axis=axis).derivative(),
+                       cls1(c, x, axis=axis).derivative(2),
+                       cls1(c, x, axis=axis).antiderivative(),
+                       cls1(c, x, axis=axis).antiderivative(2)]:
+                assert p1.axis == p.axis
+
+    @pytest.mark.parametrize('cls', interpolate_cls)
+    @pytest.mark.parametrize('axis', [-1, 4, 5, 6])
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_axis_2(self, xp, scp, cls, axis):
+        cls = getattr(scp.interpolate, cls)
+        numpy.random.seed(12345)
+        c = numpy.random.rand(3, 4, 5, 6, 7, 8)
+        c = xp.asarray(c)
+
+        x = numpy.sort(numpy.random.rand(c.shape[-1]))
+        x = xp.asarray(x)
+
+        # c array needs two axes for the coefficients and intervals, so
+        # 0 <= axis < c.ndim-1; raise otherwise
+        instance = cls(c=c, x=x, axis=axis)
+        return instance.c, instance.x, instance.axis
+
+
+@testing.with_requires("scipy")
+class TestPPoly:
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_simple(self, xp, scp):
+        c = xp.array([[1, 4], [2, 5], [3, 6]])
+        x = xp.array([0, 0.5, 1])
+        p = scp.interpolate.PPoly(c, x)
+        # testing.assert_allclose(p(0.3), 1*0.3**2 + 2*0.3 + 3)
+        # testing.assert_allclose(p(0.7), 4*(0.7-0.5)**2 + 5*(0.7-0.5) + 6)
+        return p(0.3), p(0.7)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_periodic(self, xp, scp):
+        c = xp.array([[1, 4], [2, 5], [3, 6]])
+        x = xp.array([0, 0.5, 1])
+        p = scp.interpolate.PPoly(c, x, extrapolate='periodic')
+        return p(1.3), p(-0.3), p(1.3, 1), p(-0.3, 1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_read_only(self, xp, scp):
+        c = xp.array([[1, 4], [2, 5], [3, 6]])
+        x = xp.array([0, 0.5, 1])
+        xnew = xp.array([0, 0.1, 0.2])
+        scp.interpolate.PPoly(c, x, extrapolate='periodic')
+
+        lst = []
+        for writeable in (True, False):
+            x.flags.writeable = writeable
+            f = scp.interpolate.PPoly(c, x)
+            vals = f(xnew)
+            assert xp.isfinite(vals).all()
+            lst.append(vals)
+        return vals
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_multi_shape(self, xp, scp):
+        c = numpy.random.rand(6, 2, 1, 2, 3)
+        c = xp.asarray(c)
+        x = xp.array([0, 0.5, 1])
+        p = scp.interpolate.PPoly(c, x)
+        assert p.x.shape == x.shape
+        assert p.c.shape == c.shape
+        assert p(0.3).shape == c.shape[2:]
+
+        assert p(xp.random.rand(5, 6)).shape == (5, 6) + c.shape[2:]
+
+        dp = p.derivative()
+        assert dp.c.shape == (5, 2, 1, 2, 3)
+        ip = p.antiderivative()
+        assert ip.c.shape == (7, 2, 1, 2, 3)
+        return True
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_construct_fast(self, xp, scp):
+        c = xp.array([[1, 4], [2, 5], [3, 6]], dtype=float)
+        x = xp.array([0, 0.5, 1])
+        p = scp.interpolate.PPoly.construct_fast(c, x)
+        # testing.assert_allclose(p(0.3), 1*0.3**2 + 2*0.3 + 3)
+        # testing.assert_allclose(p(0.7), 4*(0.7-0.5)**2 + 5*(0.7-0.5) + 6)
+        return p(0.3), p(0.7)
+
+    def test_from_spline(self):
+        numpy.random.seed(1234)
+        x = numpy.sort(numpy.r_[0, numpy.random.rand(11), 1])
+        y = numpy.random.rand(len(x))
+
+        spl = sc_interpolate.splrep(x, y, s=0)
+        spl = (cupy.asarray(spl[0]), cupy.asarray(spl[1]), spl[2])
+        sc_spl = tuple([x.get() if isinstance(x, cupy.ndarray) else x
+                        for x in spl])
+
+        pp = cupyx.scipy.interpolate.PPoly.from_spline(spl)
+
+        xi = np.linspace(0, 1, 200)
+        testing.assert_allclose(
+            pp(cupy.asarray(xi)), sc_interpolate.splev(xi, sc_spl))
+
+        # make sure .from_spline accepts BSpline objects
+        b = cupyx.scipy.interpolate.BSpline(*spl)
+        ppp = cupyx.scipy.interpolate.PPoly.from_spline(b)
+        testing.assert_allclose(ppp(xi), b(xi))
+
+        # BSpline's extrapolate attribute propagates unless overridden
+        t, c, k = spl
+        for extrap in (None, True, False):
+            b = cupyx.scipy.interpolate.BSpline(t, c, k, extrapolate=extrap)
+            p = cupyx.scipy.interpolate.PPoly.from_spline(b)
+            assert p.extrapolate == b.extrapolate
+
+    def test_derivative_simple(self):
+        c = cupy.array([[4, 3, 2, 1]]).T
+        dc = cupy.array([[3*4, 2*3, 2]]).T
+        ddc = cupy.array([[2*3*4, 1*2*3]]).T
+        x = cupy.array([0, 1])
+
+        pp = cupyx.scipy.interpolate.PPoly(c, x)
+        dpp = cupyx.scipy.interpolate.PPoly(dc, x)
+        ddpp = cupyx.scipy.interpolate.PPoly(ddc, x)
+
+        testing.assert_allclose(pp.derivative().c, dpp.c)
+        testing.assert_allclose(pp.derivative(2).c, ddpp.c)
+
+    def test_derivative_eval(self):
+        numpy.random.seed(1234)
+        x = numpy.sort(numpy.r_[0, numpy.random.rand(11), 1])
+        y = numpy.random.rand(len(x))
+
+        spl = sc_interpolate.splrep(x, y, s=0)
+        spl_cupy = (cupy.asarray(spl[0]), cupy.asarray(spl[1]), spl[2])
+        pp = cupyx.scipy.interpolate.PPoly.from_spline(spl_cupy)
+
+        xi = cupy.linspace(0, 1, 200)
+        for dx in range(0, 3):
+            testing.assert_allclose(pp(xi, dx),
+                                    sc_interpolate.splev(xi.get(), spl, dx))
+
+    def test_derivative(self):
+        numpy.random.seed(1234)
+        x = numpy.sort(numpy.r_[0, numpy.random.rand(11), 1])
+        y = numpy.random.rand(len(x))
+
+        spl = sc_interpolate.splrep(x, y, s=0, k=5)
+        spl_cupy = (cupy.asarray(spl[0]), cupy.asarray(spl[1]), spl[2])
+        pp = cupyx.scipy.interpolate.PPoly.from_spline(spl_cupy)
+
+        xi = cupy.linspace(0, 1, 200)
+        for dx in range(0, 10):
+            testing.assert_allclose(pp(xi, dx), pp.derivative(dx)(xi),
+                                    err_msg="dx=%d" % (dx,))
+
+    def test_antiderivative_of_constant(self):
+        # https://github.com/scipy/scipy/issues/4216
+        PPoly = cupyx.scipy.interpolate.PPoly
+        p = PPoly(cupy.asarray([[1.]]), cupy.asarray([0, 1]))
+        testing.assert_allclose(p.antiderivative().c,
+                                PPoly(c=cupy.asarray([[1], [0]]),
+                                      x=cupy.asarray([0, 1])).c, atol=1e-15)
+        testing.assert_allclose(p.antiderivative().x,
+                                PPoly(c=cupy.asarray([[1], [0]]),
+                                      x=cupy.asarray([0, 1])).x, atol=1e-15)
+
+    def test_antiderivative_regression_4355(self):
+        # https://github.com/scipy/scipy/issues/4355
+        PPoly = cupyx.scipy.interpolate.PPoly
+        p = PPoly(cupy.asarray([[1., 0.5]]), cupy.asarray([0, 1, 2]))
+        q = p.antiderivative()
+        testing.assert_allclose(q.c, cupy.asarray([[1, 0.5], [0, 1]]))
+        testing.assert_allclose(q.x, cupy.asarray([0, 1, 2]))
+        testing.assert_allclose(p.integrate(0, 2), 1.5)
+        testing.assert_allclose(q(2) - q(0), 1.5)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_antiderivative_simple(self, xp, scp):
+        # [ p1(x) = 3*x**2 + 2*x + 1,
+        #   p2(x) = 1.6875]
+        c = xp.array([[3, 2, 1], [0, 0, 1.6875]]).T
+        # [ pp1(x) = x**3 + x**2 + x,
+        #   pp2(x) = 1.6875*(x - 0.25) + pp1(0.25)]
+        # ic = xp.array([[1, 1, 1, 0], [0, 0, 1.6875, 0.328125]]).T
+        # [ ppp1(x) = (1/4)*x**4 + (1/3)*x**3 + (1/2)*x**2,
+        #   ppp2(x) = (1.6875/2)*(x - 0.25)**2 + pp1(0.25)*x + ppp1(0.25)]
+        # iic = xp.array([[1/4, 1/3, 1/2, 0, 0],
+        #                 [0, 0, 1.6875/2, 0.328125, 0.037434895833333336]]).T
+        x = xp.array([0, 0.25, 1])
+
+        pp = scp.interpolate.PPoly(c, x)
+        ipp = pp.antiderivative()
+        iipp = pp.antiderivative(2)
+        iipp2 = ipp.antiderivative()
+
+        return ipp.x, ipp.c, iipp.x, iipp.c, iipp2.x, iipp2.c
+
+    def test_antiderivative_vs_derivative(self):
+        numpy.random.seed(1234)
+        x = numpy.linspace(0, 1, 30)**2
+        y = numpy.random.rand(len(x))
+        spl = sc_interpolate.splrep(x, y, s=0, k=5)
+        spl = (cupy.asarray(spl[0]), cupy.asarray(spl[1]), spl[2])
+        pp = cupyx.scipy.interpolate.PPoly.from_spline(spl)
+
+        for dx in range(0, 10):
+            ipp = pp.antiderivative(dx)
+
+            # check that derivative is inverse op
+            pp2 = ipp.derivative(dx)
+            assert_allclose(pp.c, pp2.c)
+
+            # check continuity
+            for k in range(dx):
+                pp2 = ipp.derivative(k)
+
+                r = 1e-13
+                endpoint = r*pp2.x[:-1] + (1 - r)*pp2.x[1:]
+
+                assert_allclose(pp2(pp2.x[1:]), pp2(endpoint),
+                                rtol=1e-7, err_msg="dx=%d k=%d" % (dx, k))
+
+    def test_antiderivative_vs_spline(self):
+        numpy.random.seed(1234)
+        x = numpy.sort(numpy.r_[0, numpy.random.rand(11), 1])
+        y = numpy.random.rand(len(x))
+
+        spl = sc_interpolate.splrep(x, y, s=0, k=5)
+        spl_cupy = (cupy.asarray(spl[0]), cupy.asarray(spl[1]), spl[2])
+        pp = cupyx.scipy.interpolate.PPoly.from_spline(spl_cupy)
+
+        for dx in range(0, 10):
+            pp2 = pp.antiderivative(dx)
+            spl2 = cupyx.scipy.interpolate.splantider(spl_cupy, dx)
+            spl2_np = (spl2[0].get(), spl2[1].get(), spl2[2])
+
+            xi = cupy.linspace(0, 1, 200)
+            testing.assert_allclose(pp2(xi),
+                                    sc_interpolate.splev(xi.get(), spl2_np),
+                                    rtol=1e-7)
+
+    def test_antiderivative_continuity(self):
+        c = cupy.array([[2, 1, 2, 2], [2, 1, 3, 3]]).T
+        x = cupy.array([0, 0.5, 1])
+
+        p = cupyx.scipy.interpolate.PPoly(c, x)
+        ip = p.antiderivative()
+
+        # check continuity
+        testing.assert_allclose(ip(0.5 - 1e-9), ip(0.5 + 1e-9), rtol=1e-8)
+
+        # check that only lowest order coefficients were changed
+        p2 = ip.derivative()
+        testing.assert_allclose(p2.c, p.c)
+
+    def test_integrate(self):
+        numpy.random.seed(1234)
+        x = numpy.sort(numpy.r_[0, numpy.random.rand(11), 1])
+        y = numpy.random.rand(len(x))
+
+        spl = sc_interpolate.splrep(x, y, s=0, k=5)
+        cp_spl = tuple([cupy.asarray(x) if isinstance(x, np.ndarray) else x
+                        for x in spl])
+        # spl_cupy = (cupy.asarray(spl[0]), cupy.asarray(spl[1]), spl[2])
+        pp = cupyx.scipy.interpolate.PPoly.from_spline(cp_spl)
+
+        a, b = 0.3, 0.9
+        ig = pp.integrate(a, b)
+
+        # ipp = pp.antiderivative()
+        # testing.assert_allclose(ig, ipp(b) - ipp(a))
+        testing.assert_allclose(ig, sc_interpolate.splint(a, b, spl))
+
+        a, b = -0.3, 0.9
+        ig = pp.integrate(a, b, extrapolate=True)
+        # testing.assert_allclose(ig, ipp(b) - ipp(a))
+
+        assert (cupy.isnan(pp.integrate(a, b, extrapolate=False)).all())
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_integrate_readonly(self, xp, scp):
+        x = xp.array([1, 2, 4])
+        c = xp.array([[0., 0.], [-1., -1.], [2., -0.], [1., 2.]])
+
+        ret = []
+        for writeable in (True, False):
+            x.flags.writeable = writeable
+
+            P = scp.interpolate.PPoly(c, x)
+            vals = P.integrate(1, 4)
+            ret.append(vals)
+
+        return ret
+
+    def test_integrate_periodic(self):
+        x = cupy.array([1, 2, 4])
+        c = cupy.array([[0., 0.], [-1., -1.], [2., -0.], [1., 2.]])
+
+        P = cupyx.scipy.interpolate.PPoly(c, x, extrapolate='periodic')
+        poly_int = P.antiderivative()
+
+        period_int = poly_int(4) - poly_int(1)
+
+        assert_allclose(P.integrate(1, 4), period_int)
+        assert_allclose(P.integrate(-10, -7), period_int)
+        assert_allclose(P.integrate(-10, -4), 2 * period_int)
+
+        assert_allclose(P.integrate(1.5, 2.5), poly_int(2.5) - poly_int(1.5))
+        assert_allclose(
+            P.integrate(3.5, 5), poly_int(2) - poly_int(1) +
+            poly_int(4) - poly_int(3.5))
+        assert_allclose(P.integrate(3.5 + 12, 5 + 12),
+                        poly_int(2) - poly_int(1) + poly_int(4) -
+                        poly_int(3.5))
+        assert_allclose(P.integrate(3.5, 5 + 12),
+                        poly_int(2) - poly_int(1) + poly_int(4) -
+                        poly_int(3.5) + 4 * period_int)
+
+        assert_allclose(P.integrate(0, -1), poly_int(2) - poly_int(3))
+        assert_allclose(P.integrate(-9, -10), poly_int(2) - poly_int(3))
+        assert_allclose(
+            P.integrate(0, -10), poly_int(2) - poly_int(3) - 3 * period_int)
+
+    @pytest.mark.skip(reason='There is not an asymmetric eigenvalue solver')
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_roots(self, xp, scp):
+        x = xp.linspace(0, 1, 31)**2
+        y = xp.sin(30*x)
+
+        if xp is cupy:
+            spl = sc_interpolate.splrep(x.get(), y.get(), s=0, k=3)
+            spl = (cupy.asarray(spl[0]), cupy.asarray(spl[1]), spl[2])
+        else:
+            spl = sc_interpolate.splrep(x, y, s=0, k=3)
+
+        pp = scp.interpolate.PPoly.from_spline(spl)
+
+        r = pp.roots()
+        return r
+
+    @pytest.mark.skip(reason='There is not an asymmetric eigenvalue solver')
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-14)
+    def test_roots_idzero(self, xp, scp):
+        # Roots for piecewise polynomials with identically zero
+        # sections.
+        c = xp.array([[-1, 0.25], [0, 0], [-1, 0.25]]).T
+        x = xp.array([0, 0.4, 0.6, 1.0])
+        pp = scp.interpolate.PPoly(c, x)
+
+        # ditto for p.solve(const) with sections identically equal const
+        const = 2.
+        c1 = c.copy()
+        c1[1, :] += const
+        pp1 = scp.interpolate.PPoly(c1, x)
+
+        return pp.roots(), pp1.roots()
+
+    @pytest.mark.skip(reason='There is not an asymmetric eigenvalue solver')
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_roots_all_zero(self, xp, scp):
+        # test the code path for the polynomial being
+        # identically zero everywhere
+        c = xp.asarray([[0], [0]])
+        x = xp.asarray([0, 1])
+        p = scp.interpolate.PPoly(c, x)
+        return p.roots(), p.solve(0), p.solve(1)
+
+    @pytest.mark.skip(reason='There is not an asymmetric eigenvalue solver')
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_roots_all_zero_1(self, xp, scp):
+        # test the code path for the polynomial being
+        # identically zero everywhere
+        c = xp.asarray([[0, 0], [0, 0]])
+        x = xp.asarray([0, 1, 2])
+        p = scp.interpolate.PPoly(c, x)
+        return p.roots(), p.solve(0), p.solve(1)
+
+    @pytest.mark.skip(reason='There is not an asymmetric eigenvalue solver')
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_roots_repeated(self, xp, scp):
+        # Check roots repeated in multiple sections are reported only
+        # once.
+
+        # [(x + 1)**2 - 1, -x**2] ; x == 0 is a repeated root
+        c = xp.array([[1, 0, -1], [-1, 0, 0]]).T
+        x = xp.array([-1, 0, 1])
+
+        pp = scp.interpolate.PPoly(c, x)
+        return pp.roots(), pp.roots(extrapolate=False)
+
+    @pytest.mark.skip(reason='There is not an asymmetric eigenvalue solver')
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_roots_discont(self, xp, scp):
+        # Check that a discontinuity across zero is reported as root
+        c = xp.array([[1], [-1]]).T
+        x = xp.array([0, 0.5, 1])
+        pp = scp.interpolate.PPoly(c, x)
+        return (pp.roots(), pp.roots(discontinuity=False),
+                pp.solve(0.5), pp.solve(0.5, discontinuity=False),
+                pp.solve(1.5), pp.solve(1.5, discontinuity=False))
+
+    @pytest.mark.skip(reason='There is not an asymmetric eigenvalue solver')
+    def test_roots_random(self):
+        # Check high-order polynomials with random coefficients
+        numpy.random.seed(1234)
+
+        num = 0
+
+        for extrapolate in (True, False):
+            for order in range(0, 20):
+                x = numpy.unique(numpy.r_[0, 10 * numpy.random.rand(30), 10])
+                x = cupy.asarray(x)
+                c = 2*numpy.random.rand(order+1, len(x)-1, 2, 3) - 1
+                c = cupy.asarray(c)
+
+                pp = cupyx.scipy.interpolate.PPoly(c, x)
+                for y in [0, numpy.random.random()]:
+                    r = pp.solve(y, discontinuity=False,
+                                 extrapolate=extrapolate)
+
+                    for i in range(2):
+                        for j in range(3):
+                            r1 = r[i]
+                            rr = r1[j]
+                            if rr.size > 0:
+                                # Check that the reported roots
+                                # indeed are roots
+                                num += rr.size
+                                val = pp(rr, extrapolate=extrapolate)[:, i, j]
+                                cmpval = pp(rr, nu=1,
+                                            extrapolate=extrapolate)[:, i, j]
+                                msg = "(%r) r = %s" % (extrapolate, repr(rr),)
+                                assert_allclose((val-y) / cmpval, 0, atol=1e-7,
+                                                err_msg=msg)
+
+        # Check that we checked a number of roots
+        assert num > 100, repr(num)
+
+    @pytest.mark.skip(reason='There is not a complex root solver available')
+    def test_roots_croots(self):
+        # Test the complex root finding algorithm
+        testing.shaped_seed(1234)
+
+        for k in range(1, 15):
+            c = testing.shaped_rand(k, 1, 130)
+
+            if k == 3:
+                # add a case with zero discriminant
+                c[:, 0, 0] = 1, 2, 1
+
+            for y in [0, testing.shaped_random()]:
+                w = np.empty(c.shape, dtype=complex)
+                # _ppoly._croots_poly1(c, w)
+
+                if k == 1:
+                    assert np.isnan(w).all()
+                    continue
+
+                res = 0
+                cres = 0
+                for i in range(k):
+                    res += c[i, None] * w**(k-1-i)
+                    cres += abs(c[i, None] * w**(k-1-i))
+                with np.errstate(invalid='ignore'):
+                    res /= cres
+                res = res.ravel()
+                res = res[~np.isnan(res)]
+                assert_allclose(res, 0, atol=1e-10)
+
+    @pytest.mark.parametrize('extrapolate', [True, False, None])
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_extrapolate_attr(self, xp, scp, extrapolate):
+        # [ 1 - x**2 ]
+        c = xp.array([[-1, 0, 1]]).T
+        x = xp.array([0, 1])
+
+        pp = scp.interpolate.PPoly(c, x, extrapolate=extrapolate)
+        pp_d = pp.derivative()
+        pp_i = pp.antiderivative()
+
+        xx = xp.asarray([-0.1, 1.1])
+
+        return pp(xx), pp_i(xx), pp_d(xx)
+
+    def binom_matrix(self, power, xp):
+        n = numpy.arange(power + 1).reshape(-1, 1)
+        k = numpy.arange(power + 1)
+        B = sc_special.binom(n, k)
+        if xp is cupy:
+            B = cupy.asarray(B)
+        return B[::-1, ::-1]
+
+    def _prepare_descending(self, m, xp, scp):
+        power = 3
+        x = numpy.sort(numpy.random.uniform(0, 10, m + 1))
+        x = xp.asarray(x)
+        ca = numpy.random.uniform(-2, 2, size=(power + 1, m))
+        ca = xp.asarray(ca)
+
+        h = xp.diff(x)
+        h_powers = h[None, :] ** xp.arange(power + 1)[::-1, None]
+        B = self.binom_matrix(power, xp)
+        cap = ca * h_powers
+        cdp = xp.dot(B.T, cap)
+        cd = cdp / h_powers
+
+        pa = scp.interpolate.PPoly(ca, x, extrapolate=True)
+        pd = scp.interpolate.PPoly(cd[:, ::-1], x[::-1], extrapolate=True)
+        return pa, pd
+
+    @pytest.mark.parametrize('m', [10, 20, 30])
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-13)
+    def test_descending(self, m, xp, scp):
+        numpy.random.seed(0)
+        pa, pd = self._prepare_descending(m, xp, scp)
+
+        x_test = numpy.random.uniform(-10, 20, 100)
+        x_test = xp.asarray(x_test)
+        return pa(x_test), pa(x_test, 1)
+
+    @pytest.mark.parametrize('m', [10, 20, 30])
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-13)
+    def test_descending_derivative(self, m, xp, scp):
+        numpy.random.seed(0)
+        pa, pd = self._prepare_descending(m, xp, scp)
+        pa_d = pa.derivative()
+        pd_d = pd.derivative()
+
+        x_test = numpy.random.uniform(-10, 20, 100)
+        x_test = xp.asarray(x_test)
+        return pa_d(x_test), pd_d(x_test)
+
+    @pytest.mark.parametrize('m', [10, 20, 30])
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_descending_antiderivative(self, m, xp, scp):
+        numpy.random.seed(0)
+        pa, pd = self._prepare_descending(m, xp, scp)
+
+        # Antiderivatives won't be equal because fixing continuity is
+        # done in the reverse order, but surely the differences should be
+        # equal.
+        pa_i = pa.antiderivative()
+        pd_i = pd.antiderivative()
+        results = []
+        for a, b in numpy.random.uniform(-10, 20, (5, 2)):
+            int_a = pa.integrate(a, b)
+            int_d = pd.integrate(a, b)
+            results += [int_a, int_d]
+            results += [pa_i(b) - pa_i(a), pd_i(b) - pd_i(a)]
+        return results
+
+    @pytest.mark.skip(reason='There is not an asymmetric eigenvalue solver')
+    @pytest.mark.parametrize('m', [10, 20, 30])
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-12)
+    def test_descending_roots(self, m, xp, scp):
+        numpy.random.seed(0)
+        pa, pd = self._prepare_descending(m, xp, scp)
+
+        roots_d = pd.roots()
+        roots_a = pa.roots()
+        return roots_a, roots_d
+
+
+@testing.with_requires("scipy")
+class TestBPoly:
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_simple(self, xp, scp):
+        x = xp.asarray([0, 1])
+        c = xp.asarray([[3]])
+        bp = scp.interpolate.BPoly(c, x)
+        return bp(0.1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_simple2(self, xp, scp):
+        x = xp.asarray([0, 1])
+        c = xp.asarray([[3], [1]])
+        bp = scp.interpolate.BPoly(c, x)   # 3*(1-x) + 1*x
+        # bp(0.1) = 3*0.9 + 1.*0.1
+        return bp(0.1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_simple3(self, xp, scp):
+        x = xp.asarray([0, 1])
+        c = xp.asarray([[3], [1], [4]])
+        # 3 * (1-x)**2 + 2 * x (1-x) + 4 * x**2
+        bp = scp.interpolate.BPoly(c, x)
+        # bp(0.2) = 3 * 0.8*0.8 + 1 * 2*0.2*0.8 + 4 * 0.2*0.2)
+        return bp(0.2)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_simple4(self, xp, scp):
+        x = xp.asarray([0, 1])
+        c = xp.asarray([[1], [1], [1], [2]])
+        bp = scp.interpolate.BPoly(c, x)
+        # bp(0.3) = 0.7**3 + 3 * 0.7**2 * 0.3 + 3 * 0.7 * 0.3**2 + 2 * 0.3**3)
+        return bp(0.3)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_simple5(self, xp, scp):
+        x = xp.asarray([0, 1])
+        c = xp.asarray([[1], [1], [8], [2], [1]])
+        bp = scp.interpolate.BPoly(c, x)
+        # bp(0.3) = 0.7**4 + 4 * 0.7**3 * 0.3 + 8 * 6 * 0.7**2 * 0.3**2 +
+        #           2 * 4 * 0.7 * 0.3**3 + 0.3**4)
+        return bp(0.3)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_periodic(self, xp, scp):
+        x = xp.asarray([0, 1, 3])
+        c = xp.asarray([[3, 0], [0, 0], [0, 2]])
+        # [3*(1-x)**2, 2*((x-1)/2)**2]
+        bp = scp.interpolate.BPoly(c, x, extrapolate='periodic')
+
+        # bp(3.4) = 3 * 0.6**2, bp(-1.3) = 2 * (0.7/2)**2
+        # bp(3.4, 1) = -6 * 0.6, bp(-1.3, 1) = 2 * (0.7/2)
+        return bp(3.4), bp(-1.3), bp(3.4, 1), bp(-1.3, 1)
+
+    @pytest.mark.parametrize('m', [10, 20, 30])
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_descending(self, xp, scp, m):
+        res = []
+        power = 3
+        x = xp.sort(testing.shaped_random((m + 1,), xp))
+        ca = testing.shaped_random((power + 1, m), xp)
+        ca = ca * 2 - 1
+        # We need only to flip coefficients to get it right!
+        cd = ca[::-1].copy()
+
+        pa = scp.interpolate.BPoly(ca, x, extrapolate=True)
+        pd = scp.interpolate.BPoly(cd[:, ::-1], x[::-1], extrapolate=True)
+
+        x_test = testing.shaped_random((100,), xp)
+        x_test = x_test * 30 - 10
+
+        res += [pa(x_test), pd(x_test)]
+        res += [pa(x_test, 1), pd(x_test, 1)]
+
+        pa_d = pa.derivative()
+        pd_d = pd.derivative()
+
+        res += [pa_d(x_test), pd_d(x_test)]
+
+        # Antiderivatives won't be equal because fixing continuity is
+        # done in the reverse order, but surely the differences should be
+        # equal.
+        pa_i = pa.antiderivative()
+        pd_i = pd.antiderivative()
+        points = testing.shaped_random((5, 2), xp)
+        points = points * 30 - 10
+        for a, b in points:
+            int_a = pa.integrate(a, b)
+            int_d = pd.integrate(a, b)
+            res += [int_a, int_d]
+            res += [pa_i(b) - pa_i(a), pd_i(b) - pd_i(a)]
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_multi_shape(self, xp, scp):
+        c = testing.shaped_random((6, 2, 1, 2, 3), xp)
+        x = xp.array([0, 0.5, 1])
+        p = scp.interpolate.BPoly(c, x)
+        x1 = testing.shaped_random((5, 6), xp)
+        dp = p.derivative()
+        return p(x1), dp(x1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_interval_length(self, xp, scp):
+        x = xp.asarray([0, 2])
+        c = xp.asarray([[3], [1], [4]])
+        bp = scp.interpolate.BPoly(c, x)
+        xval = xp.asarray([0.1])
+        # s = xval / 2  # s = (x - xa) / (xb - xa)
+        # bp(sval) = 3 * (1-s)*(1-s) + 1 * 2*s*(1-s) + 4 * s*s
+        return bp(xval)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_two_intervals(self, xp, scp):
+        x = xp.asarray([0, 1, 3])
+        c = xp.asarray([[3, 0], [0, 0], [0, 2]])
+        bp = scp.interpolate.BPoly(c, x)  # [3*(1-x)**2, 2*((x-1)/2)**2]
+        return bp(xp.asarray([0.4, 1.7]))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('extrapolate', [True, False, None])
+    def test_extrapolate_attr(self, xp, scp, extrapolate):
+        x = xp.asarray([0, 2])
+        c = xp.asarray([[3], [1], [4]])
+        x1 = xp.asarray([-0.1, 2.1])
+        bp = scp.interpolate.BPoly(c, x, extrapolate=extrapolate)
+        bp_d = bp.derivative()
+        return bp(x1), bp_d(x1)
+
+
+@testing.with_requires("scipy")
+class TestBPolyCalculus:
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_derivative(self, xp, scp):
+        res = []
+        x = xp.asarray([0, 1, 3])
+        c = xp.asarray([[3, 0], [0, 0], [0, 2]])
+        # [3*(1-x)**2, 2*((x-1)/2)**2]
+        bp = scp.interpolate.BPoly(c, x)
+        bp_der = bp.derivative()
+
+        # bp_der(0.4) = -6 * 0.6
+        # bp_der(1.7) = 0.7
+        res += [bp_der(0.4), bp_der(1.7)]
+
+        # derivatives in-place
+        # bp(0.4, nu=1) = -6 * (1 - 0.4)
+        # bp(0.4, nu=2) = 6
+        # bp(0.4, nu=3) = 0
+        res += [bp(0.4, nu=1), bp(0.4, nu=2), bp(0.4, nu=3)]
+
+        # bp(1.7, nu=1) = 0.7
+        # bp(1.7, nu=2) = 1
+        # bp(1.7, nu=3) = 0
+        res += [bp(1.7, nu=1), bp(1.7, nu=2), bp(1.7, nu=3)]
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-3)
+    def test_derivative_ppoly(self, xp, scp):
+        # make sure it's consistent w/ power basis
+        m, k = 5, 8   # number of intervals, order
+        x = xp.sort(testing.shaped_random((m,), xp))
+        c = testing.shaped_random((k, m-1), xp)
+        bp = scp.interpolate.BPoly(c, x)
+        pp = scp.interpolate.PPoly.from_bernstein_basis(bp)
+
+        res = []
+        for _ in range(k):
+            bp = bp.derivative()
+            pp = pp.derivative()
+            xi = xp.linspace(x[0], x[-1], 21)
+            res += [bp(xi), pp(xi)]
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-3)
+    def test_deriv_inplace(self, xp, scp):
+        m, k = 5, 8   # number of intervals, order
+        x = xp.sort(testing.shaped_random((m,), xp))
+        c = testing.shaped_random((k, m - 1), xp)
+
+        # test both real and complex coefficients
+        res = []
+        for cc in [c.copy(), c*(1. + 2.j)]:
+            # Skip for complex on older ROCm
+            if (
+                runtime.is_hip and
+                driver.get_build_version() < 5_00_00000 and
+                cc.dtype.kind == 'c'
+            ):
+                continue
+            bp = scp.interpolate.BPoly(cc, x)
+            xi = xp.linspace(x[0], x[-1], 21)
+            for i in range(k):
+                res += [bp(xi, i), bp.derivative(i)(xi)]
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_antiderivative_simple(self, xp, scp):
+        # f(x) = x        for x \in [0, 1),
+        #        (x-1)/2  for x \in [1, 3]
+        #
+        # antiderivative is then
+        # F(x) = x**2 / 2            for x \in [0, 1),
+        #        0.5*x*(x/2 - 1) + A  for x \in [1, 3]
+        # where A = 3/4 for continuity at x = 1.
+        x = xp.asarray([0, 1, 3])
+        c = xp.asarray([[0, 0], [1, 1]])
+
+        bp = scp.interpolate.BPoly(c, x)
+        bi = bp.antiderivative()
+
+        xx = xp.linspace(0, 3, 11)
+        return bi(xx)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=4e-6)
+    def test_der_antider(self, xp, scp):
+        x = xp.sort(testing.shaped_random((11,), xp))
+        c = testing.shaped_random((4, 10, 2, 3), xp)
+        bp = scp.interpolate.BPoly(c, x)
+
+        xx = xp.linspace(x[0], x[-1], 100)
+        return bp.antiderivative().derivative()(xx), bp(xx)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=3e-7)
+    def test_antider_ppoly(self, xp, scp):
+        x = xp.sort(testing.shaped_random((11,), xp))
+        c = testing.shaped_random((4, 10, 2, 3), xp)
+        bp = scp.interpolate.BPoly(c, x)
+        pp = scp.interpolate.PPoly.from_bernstein_basis(bp)
+
+        xx = xp.linspace(x[0], x[-1], 10)
+        return bp.antiderivative(2)(xx), pp.antiderivative(2)(xx)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_antider_continuous(self, xp, scp):
+        x = xp.sort(testing.shaped_random((11,), xp))
+        c = testing.shaped_random((4, 10), xp)
+        bp = scp.interpolate.BPoly(c, x).antiderivative()
+
+        xx = bp.x[1:-1]
+        return bp(xx - 1e-14), bp(xx + 1e-14)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_integrate(self, xp, scp):
+        x = xp.sort(testing.shaped_random((11,), xp))
+        c = testing.shaped_random((4, 10), xp)
+        bp = scp.interpolate.BPoly(c, x)
+        pp = scp.interpolate.PPoly.from_bernstein_basis(bp)
+        return bp.integrate(0, 1), pp.integrate(0, 1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_integrate_extrap(self, xp, scp):
+        c = xp.asarray([[1]])
+        x = xp.asarray([0, 1])
+
+        # default is extrapolate=True
+        b = scp.interpolate.BPoly(c, x)
+        # .integrate argument overrides self.extrapolate
+        b1 = scp.interpolate.BPoly(c, x, extrapolate=False)
+
+        return (b.integrate(0, 2), b1.integrate(0, 2),
+                b1.integrate(0, 2, extrapolate=True))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_integrate_periodic(self, xp, scp):
+        res = []
+        x = xp.asarray([1, 2, 4])
+        c = xp.asarray([[0., 0.], [-1., -1.], [2., -0.], [1., 2.]])
+
+        P = scp.interpolate.BPoly.from_power_basis(
+            scp.interpolate.PPoly(c, x), extrapolate='periodic')
+
+        # I = P.antiderivative()
+        # period_int = I(4) - I(1)
+        res += [
+            # period_int
+            P.integrate(1, 4),
+            # period_int
+            P.integrate(-10, -7),
+            # 2 * period_int
+            P.integrate(-10, -4),
+            # I(2.5) - I(1.5)
+            P.integrate(1.5, 2.5),
+            # I(2) - I(1) + I(4) - I(3.5)
+            P.integrate(3.5, 5),
+            # I(2) - I(1) + I(4) - I(3.5)
+            P.integrate(3.5 + 12, 5 + 12),
+            # I(2) - I(1) + I(4) - I(3.5) + 4 * period_int
+            P.integrate(3.5, 5 + 12),
+            # I(2) - I(3)
+            P.integrate(0, -1),
+            # I(2) - I(3)
+            P.integrate(-9, -10),
+            # I(2) - I(3) - 3 * period_int
+            P.integrate(0, -10)
+        ]
+
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_antider_neg(self, xp, scp):
+        # .derivative(-nu) ==> .andiderivative(nu) and vice versa
+        c = xp.asarray([[1]])
+        x = xp.asarray([0, 1])
+        b = scp.interpolate.BPoly(c, x)
+
+        xx = xp.linspace(0, 1, 21)
+
+        return (b.derivative(-1)(xx), b.antiderivative()(xx),
+                b.derivative(1)(xx), b.antiderivative(-1)(xx))
+
+
+@testing.with_requires("scipy")
+class TestPolyConversions:
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_bp_from_pp(self, xp, scp):
+        x = xp.asarray([0, 1, 3])
+        c = xp.asarray([[3, 2], [1, 8], [4, 3]])
+        pp = scp.interpolate.PPoly(c, x)
+        bp = scp.interpolate.BPoly.from_power_basis(pp)
+        pp1 = scp.interpolate.PPoly.from_bernstein_basis(bp)
+
+        x1 = xp.asarray([0.1, 1.4])
+        return pp(x1), bp(x1), pp(x1), pp1(x1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=6e-7)
+    def test_bp_from_pp_random(self, xp, scp):
+        m, k = 5, 8   # number of intervals, order
+        x = xp.sort(testing.shaped_random((m,), xp))
+        c = testing.shaped_random((k, m - 1), xp)
+        pp = scp.interpolate.PPoly(c, x)
+        bp = scp.interpolate.BPoly.from_power_basis(pp)
+        pp1 = scp.interpolate.PPoly.from_bernstein_basis(bp)
+
+        x1 = xp.linspace(x[0], x[-1], 21)
+        return pp(x1), bp(x1), pp(x1), pp1(x1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_pp_from_bp(self, xp, scp):
+        x = xp.asarray([0, 1, 3])
+        c = xp.asarray([[3, 3], [1, 1], [4, 2]])
+        bp = scp.interpolate.BPoly(c, x)
+        pp = scp.interpolate.PPoly.from_bernstein_basis(bp)
+        bp1 = scp.interpolate.BPoly.from_power_basis(pp)
+
+        x1 = xp.asarray([0.1, 1.4])
+        return bp(x1), pp(x1), bp(x1), bp1(x1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=True)
+    @pytest.mark.parametrize('comb', [
+        (attrgetter('PPoly'), attrgetter('PPoly.from_bernstein_basis')),
+        (attrgetter('BPoly'), attrgetter('PPoly.from_power_basis'))])
+    def test_broken_conversions(self, xp, scp, comb):
+        # regression test for scipy/gh-10597: from_power_basis only
+        # accepts PPoly etc.
+        x = xp.asarray([0, 1, 3])
+        c = xp.asarray([[3, 3], [1, 1], [4, 2]])
+
+        get_interp_cls, get_conv_meth = comb
+        pp = get_interp_cls(scp.interpolate)(c, x)
+        get_conv_meth(scp.interpolate)(pp)
+
+
+@testing.with_requires("scipy")
+class TestBPolyFromDerivatives:
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_make_poly_1(self, xp, scp):
+        c1 = scp.interpolate.BPoly._construct_from_derivatives(
+            0, 1, [2], [3])
+        return c1
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_make_poly_2(self, xp, scp):
+        c1 = scp.interpolate.BPoly._construct_from_derivatives(
+            0, 1, [1, 0], [1])
+        # f'(0) = 3
+        c2 = scp.interpolate.BPoly._construct_from_derivatives(
+            0, 1, [2, 3], [1])
+        # f'(1) = 3
+        c3 = scp.interpolate.BPoly._construct_from_derivatives(
+            0, 1, [2], [1, 3])
+        return c1, c2, c3
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_make_poly_3(self, xp, scp):
+        # f'(0)=2, f''(0)=3
+        c1 = scp.interpolate.BPoly._construct_from_derivatives(
+            0, 1, [1, 2, 3], [4])
+        # f'(1)=2, f''(1)=3
+        c2 = scp.interpolate.BPoly._construct_from_derivatives(
+            0, 1, [1], [4, 2, 3])
+        # f'(0)=2, f'(1)=3
+        c3 = scp.interpolate.BPoly._construct_from_derivatives(
+            0, 1, [1, 2], [4, 3])
+        return c1, c2, c3
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_make_poly_12(self, xp, scp):
+        ya = xp.r_[0, testing.shaped_random((5,), xp)]
+        yb = xp.r_[0, testing.shaped_random((5,), xp)]
+
+        c = scp.interpolate.BPoly._construct_from_derivatives(0, 1, ya, yb)
+        pp = scp.interpolate.BPoly(c[:, None], xp.asarray([0, 1]))
+        res = []
+        for _ in range(6):
+            res += [pp(0.), pp(1.)]
+            pp = pp.derivative()
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_raise_degree(self, xp, scp):
+        x = xp.asarray([0, 1])
+        k, d = 8, 5
+        c = testing.shaped_random((k, 1, 2, 3, 4), xp)
+        bp = scp.interpolate.BPoly(c, x)
+
+        c1 = scp.interpolate.BPoly._raise_degree(c, d)
+        bp1 = scp.interpolate.BPoly(c1, x)
+
+        x1 = xp.linspace(0, 1, 11)
+        return bp(x1), bp1(x1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_xi_yi(self, xp, scp):
+        scp.interpolate.BPoly.from_derivatives(
+            xp.asarray([0, 1]), xp.asarray([0]))
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_coords_order(self, xp, scp):
+        xi = xp.asarray([0, 0, 1])
+        yi = xp.asarray([[0], [0], [0]])
+        scp.interpolate.BPoly.from_derivatives(xi, yi)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_zeros(self, xp, scp):
+        xi = xp.asarray([0, 1, 2, 3])
+        # NB: will have to raise the degree
+        yi = [[0, 0], [0], [0, 0], [0, 0]]
+        pp = scp.interpolate.BPoly.from_derivatives(xi, yi)
+        assert pp.c.shape == (4, 3)
+
+        ppd = pp.derivative()
+        res = []
+        for x1 in [0., 0.1, 1., 1.1, 1.9, 2., 2.5]:
+            res += [pp(x1), ppd(x1)]
+        return res
+
+    def _make_random_mk(self, m, k, xp):
+        # k derivatives at each breakpoint
+        xi = xp.asarray([1. * j**2 for j in range(m+1)])
+        yi = [testing.shaped_random((k,), xp) for j in range(m+1)]
+        return xi, yi
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_random_12(self, xp, scp):
+        m, k = 5, 12
+        xi, yi = self._make_random_mk(m, k, xp)
+        pp = scp.interpolate.BPoly.from_derivatives(xi, yi)
+
+        res = []
+        for _ in range(k//2):
+            res.append(pp(xi))
+            pp = pp.derivative()
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_order_zero(self, xp, scp):
+        m, k = 5, 12
+        xi, yi = self._make_random_mk(m, k, xp)
+        scp.interpolate.BPoly.from_derivatives(xi, yi, 0)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_orders_too_high(self, xp, scp):
+        m, k = 5, 12
+        xi, yi = self._make_random_mk(m, k, xp)
+
+        # this is still ok
+        scp.interpolate.BPoly.from_derivatives(xi, yi, orders=2*k-1)
+        with pytest.raises(ValueError):
+            # but this is not
+            scp.interpolate.BPoly.from_derivatives(xi, yi, orders=2*k)
+        return True
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_orders_global(self, xp, scp):
+        m, k = 5, 12
+        xi, yi = self._make_random_mk(m, k, xp)
+
+        # ok, this is confusing. Local polynomials will be of the order 5
+        # which means that up to the 2nd derivatives will be used at each point
+        order = 5
+        pp = scp.interpolate.BPoly.from_derivatives(xi, yi, orders=order)
+
+        res = []
+        for _ in range(order//2+1):
+            res += [pp(xi[1:-1] - 1e-12), pp(xi[1:-1] + 1e-12)]
+            pp = pp.derivative()
+        # assert_(not xp.allclose(pp(xi[1:-1] - 1e-12), pp(xi[1:-1] + 1e-12)))
+
+        # now repeat with `order` being even: on each interval, it uses
+        # order//2 'derivatives' @ the right-hand endpoint and
+        # order//2+1 @ 'derivatives' the left-hand endpoint
+        order = 6
+        pp = scp.interpolate.BPoly.from_derivatives(xi, yi, orders=order)
+        for _ in range(order//2):
+            res += [pp(xi[1:-1] - 1e-12), pp(xi[1:-1] + 1e-12)]
+            pp = pp.derivative()
+        # assert_(not xp.allclose(pp(xi[1:-1] - 1e-12), pp(xi[1:-1] + 1e-12)))
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_orders_local(self, xp, scp):
+        m, k = 7, 12
+        xi, yi = self._make_random_mk(m, k, xp)
+
+        orders = [o + 1 for o in range(m)]
+        res = []
+        for i, x in enumerate(xi[1:-1]):
+            pp = scp.interpolate.BPoly.from_derivatives(xi, yi, orders=orders)
+            for _ in range(orders[i] // 2 + 1):
+                res += [pp(x - 1e-12), pp(x + 1e-12)]
+                pp = pp.derivative()
+            # assert_(not xp.allclose(pp(x - 1e-12), pp(x + 1e-12)))
+        return res
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_yi_trailing_dims(self, xp, scp):
+        m, k = 7, 5
+        xi = xp.sort(testing.shaped_random((m+1,), xp))
+        yi = testing.shaped_random((m+1, k, 6, 7, 8), xp)
+        pp = scp.interpolate.BPoly.from_derivatives(xi, yi)
+        return pp.c.shape
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_scipy_gh_5430(self, xp, scp):
+        # At least one of these raises an error unless gh-5430 is
+        # fixed. In py2k an int is implemented using a C long, so
+        # which one fails depends on your system. In py3k there is only
+        # one arbitrary precision integer type, so both should fail.
+        res = []
+        orders = xp.int32(1)
+        p = scp.interpolate.BPoly.from_derivatives(
+            xp.asarray([0, 1]), [[0], [0]], orders=orders)
+        res.append(p(0))
+        orders = xp.int64(1)
+        p = scp.interpolate.BPoly.from_derivatives(
+            xp.asarray([0, 1]), [[0], [0]], orders=orders)
+        res.append(p(0))
+        orders = 1
+        # This worked before; make sure it still works
+        p = scp.interpolate.BPoly.from_derivatives(
+            xp.asarray([0, 1]), [[0], [0]], orders=orders)
+        res.append(p(0))
+        orders = 1
+        return res
diff --git a/tests/cupyx_tests/scipy_tests/interpolate_tests/test_rbfinterp.py b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_rbfinterp.py
new file mode 100644
index 0000000..0d24627
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_rbfinterp.py
@@ -0,0 +1,576 @@
+import pickle
+import pytest
+import warnings
+
+import numpy as _np
+from numpy.linalg import LinAlgError
+import cupy as cp
+import cupyx
+from cupy import testing
+import cupyx.scipy.interpolate  # NOQA
+
+
+try:
+    from scipy import interpolate  # NOQA
+except ImportError:
+    pass
+
+try:
+    from scipy.stats.qmc import Halton
+except ImportError:
+    # qmc.Halton is only available in SciPy >= 1.70
+    pass
+
+
+from cupyx.scipy.interpolate._rbfinterp import (
+    _AVAILABLE, _SCALE_INVARIANT, _NAME_TO_MIN_DEGREE, NAME_TO_FUNC,
+    _monomial_powers, polynomial_matrix, kernel_matrix)
+
+
+def _kernel_matrix(x, kernel):
+    """Return RBFs, with centers at `x`, evaluated at `x`."""
+    out = cp.empty((x.shape[0], x.shape[0]), dtype=float)
+    kernel_func = NAME_TO_FUNC[kernel]
+    kernel_matrix(x, kernel_func, out)
+    return out
+
+
+def _polynomial_matrix(x, powers):
+    """Return monomials, with exponents from `powers`, evaluated at `x`."""
+    out = cp.empty((x.shape[0], powers.shape[0]), dtype=float)
+    polynomial_matrix(x, powers, out)
+    return out
+
+
+def _vandermonde(x, degree):
+    # Returns a matrix of monomials that span polynomials with the specified
+    # degree evaluated at x.
+    powers = _monomial_powers(x.shape[1], degree)
+    return _polynomial_matrix(x, powers)
+
+
+def _1d_test_function(x, xp):
+    # Test function used in Wahba's "Spline Models for Observational Data".
+    # domain ~= (0, 3), range ~= (-1.0, 0.2)
+    x = x[:, 0]
+    y = 4.26*(xp.exp(-x) - 4*xp.exp(-2*x) + 3*xp.exp(-3*x))
+    return y
+
+
+def _2d_test_function(x, xp):
+    # Franke's test function.
+    # domain ~= (0, 1) X (0, 1), range ~= (0.0, 1.2)
+    x1, x2 = x[:, 0], x[:, 1]
+    term1 = 0.75 * xp.exp(-(9*x1-2)**2/4 - (9*x2-2)**2/4)
+    term2 = 0.75 * xp.exp(-(9*x1+1)**2/49 - (9*x2+1)/10)
+    term3 = 0.5 * xp.exp(-(9*x1-7)**2/4 - (9*x2-3)**2/4)
+    term4 = -0.2 * xp.exp(-(9*x1-4)**2 - (9*x2-7)**2)
+    y = term1 + term2 + term3 + term4
+    return y
+
+
+def _is_conditionally_positive_definite(kernel, m, xp, scp):
+    # Tests whether the kernel is conditionally positive definite of order m.
+    # See chapter 7 of Fasshauer's "Meshfree Approximation Methods with
+    # MATLAB".
+    nx = 10
+    ntests = 100
+    for ndim in [1, 2, 3, 4, 5]:
+        # Generate sample points with a Halton sequence to avoid samples that
+        # are too close to eachother, which can make the matrix singular.
+        seq = Halton(ndim, scramble=False, seed=_np.random.RandomState())
+        for _ in range(ntests):
+            x = xp.asarray(2*seq.random(nx)) - 1
+            A = _kernel_matrix(x, kernel)
+            P = _vandermonde(x, m - 1)
+            Q, R = cp.linalg.qr(P, mode='complete')
+            # Q2 forms a basis spanning the space where P.T.dot(x) = 0. Project
+            # A onto this space, and then see if it is positive definite using
+            # the Cholesky decomposition. If not, then the kernel is not c.p.d.
+            # of order m.
+            Q2 = Q[:, P.shape[1]:]
+            B = Q2.T.dot(A).dot(Q2)
+            try:
+                cp.linalg.cholesky(B)
+            except cp.linalg.LinAlgError:
+                return False
+
+    return True
+
+
+# Sorting the parametrize arguments is necessary to avoid a parallelization
+# issue described here: https://github.com/pytest-dev/pytest-xdist/issues/432.
+@testing.with_requires("scipy>=1.7.0")
+@pytest.mark.skip(reason='conditionally posdef: skip for now')
+@testing.numpy_cupy_allclose(scipy_name='scp')
+@pytest.mark.parametrize('kernel', sorted(_AVAILABLE))
+def test_conditionally_positive_definite(xp, scp, kernel):
+    # Test if each kernel in _AVAILABLE is conditionally positive definite of
+    # order m, where m comes from _NAME_TO_MIN_DEGREE. This is a necessary
+    # condition for the smoothed RBF interpolant to be well-posed in general.
+    m = _NAME_TO_MIN_DEGREE.get(kernel, -1) + 1
+    assert _is_conditionally_positive_definite(kernel, m, xp, scp)
+
+
+@testing.with_requires("scipy>=1.7.0")
+class _TestRBFInterpolator:
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('kernel', sorted(_SCALE_INVARIANT))
+    def test_scale_invariance_1d(self, xp, scp, kernel):
+        # Verify that the functions in _SCALE_INVARIANT are insensitive to the
+        # shape parameter (when smoothing == 0) in 1d.
+        seq = Halton(1, scramble=False, seed=_np.random.RandomState())
+        x = xp.asarray(3*seq.random(50))
+        y = _1d_test_function(x, xp)
+        xitp = xp.asarray(3*seq.random(50))
+        yitp1 = self.build(scp, x, y, epsilon=1.0, kernel=kernel)(xitp)
+        yitp2 = self.build(scp, x, y, epsilon=2.0, kernel=kernel)(xitp)
+        return yitp1, yitp2
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('kernel', sorted(_SCALE_INVARIANT))
+    def test_scale_invariance_2d(self, xp, scp, kernel):
+        # Verify that the functions in _SCALE_INVARIANT are insensitive to the
+        # shape parameter (when smoothing == 0) in 2d.
+        seq = Halton(2, scramble=False, seed=_np.random.RandomState())
+        x = xp.asarray(seq.random(100))
+        y = _2d_test_function(x, xp)
+        xitp = xp.asarray(seq.random(100))
+        yitp1 = self.build(scp, x, y, epsilon=1.0, kernel=kernel)(xitp)
+        yitp2 = self.build(scp, x, y, epsilon=2.0, kernel=kernel)(xitp)
+        return yitp1, yitp2
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('kernel', sorted(_AVAILABLE))
+    def test_extreme_domains(self, xp, scp, kernel):
+        # Make sure the interpolant remains numerically stable for very
+        # large/small domains.
+        seq = Halton(2, scramble=False, seed=_np.random.RandomState())
+        scale = 1e50
+        shift = 1e55
+
+        x = xp.asarray(seq.random(100))
+        y = _2d_test_function(x, xp)
+        xitp = xp.asarray(seq.random(100))
+
+        if kernel in _SCALE_INVARIANT:
+            yitp1 = self.build(scp, x, y, kernel=kernel)(xitp)
+            yitp2 = self.build(scp,
+                               x*scale + shift, y,
+                               kernel=kernel
+                               )(xitp*scale + shift)
+        else:
+            yitp1 = self.build(scp, x, y, epsilon=5.0, kernel=kernel)(xitp)
+            yitp2 = self.build(scp,
+                               x*scale + shift, y,
+                               epsilon=5.0/scale,
+                               kernel=kernel
+                               )(xitp*scale + shift)
+
+        return yitp1, yitp2
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_polynomial_reproduction(self, xp, scp):
+        # If the observed data comes from a polynomial, then the interpolant
+        # should be able to reproduce the polynomial exactly, provided that
+        # `degree` is sufficiently high.
+        rng = _np.random.RandomState(0)
+        seq = Halton(2, scramble=False, seed=rng)
+        degree = 3
+
+        x = xp.asarray(seq.random(50))
+        xitp = xp.asarray(seq.random(50))
+
+        if xp is _np:
+            P = _vandermonde(cp.asarray(x), degree).get()
+            Pitp = _vandermonde(cp.asarray(xitp), degree).get()
+        else:
+            P = _vandermonde(x, degree)
+            Pitp = _vandermonde(xitp, degree)
+
+        poly_coeffs = rng.normal(0.0, 1.0, P.shape[1])
+        poly_coeffs = xp.asarray(poly_coeffs)
+
+        y = P.dot(poly_coeffs)
+        yitp1 = Pitp.dot(poly_coeffs)
+        yitp2 = self.build(scp, x, y, degree=degree)(xitp)
+
+        return yitp1, yitp2
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_vector_data(self, xp, scp):
+        # Make sure interpolating a vector field is the same as interpolating
+        # each component separately.
+        seq = Halton(2, scramble=False, seed=_np.random.RandomState())
+
+        x = xp.asarray(seq.random(100))
+        xitp = xp.asarray(seq.random(100))
+
+        y = xp.array([_2d_test_function(x, xp),
+                      _2d_test_function(x[:, ::-1], xp)]).T
+
+        yitp1 = self.build(scp, x, y)(xitp)
+        yitp2 = self.build(scp, x, y[:, 0])(xitp)
+        yitp3 = self.build(scp, x, y[:, 1])(xitp)
+
+        return yitp1, yitp2, yitp3
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_complex_data(self, xp, scp):
+        # Interpolating complex input should be the same as interpolating the
+        # real and complex components.
+        seq = Halton(2, scramble=False, seed=_np.random.RandomState())
+
+        x = xp.asarray(seq.random(100))
+        xitp = xp.asarray(seq.random(100))
+
+        y = _2d_test_function(x, xp) + 1j*_2d_test_function(x[:, ::-1], xp)
+
+        yitp1 = self.build(scp, x, y)(xitp)
+        yitp2 = self.build(scp, x, y.real)(xitp)
+        yitp3 = self.build(scp, x, y.imag)(xitp)
+
+        return yitp1, yitp2, yitp3
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('kernel', sorted(_AVAILABLE))
+    def test_interpolation_misfit_1d(self, xp, scp, kernel):
+        # Make sure that each kernel, with its default `degree` and an
+        # appropriate `epsilon`, does a good job at interpolation in 1d.
+        seq = Halton(1, scramble=False, seed=_np.random.RandomState())
+
+        x = xp.asarray(3*seq.random(50))
+        xitp = xp.asarray(3*seq.random(50))
+
+        y = _1d_test_function(x, xp)
+        ytrue = _1d_test_function(xitp, xp)
+        yitp = self.build(scp, x, y, epsilon=5.0, kernel=kernel)(xitp)
+
+        mse = xp.mean((yitp - ytrue)**2)
+        assert mse < 1.0e-4
+        return yitp
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('kernel', sorted(_AVAILABLE))
+    def test_interpolation_misfit_2d(self, xp, scp, kernel):
+        # Make sure that each kernel, with its default `degree` and an
+        # appropriate `epsilon`, does a good job at interpolation in 2d.
+        seq = Halton(2, scramble=False, seed=_np.random.RandomState())
+
+        x = xp.asarray(seq.random(100))
+        xitp = xp.asarray(seq.random(100))
+
+        y = _2d_test_function(x, xp)
+        ytrue = _2d_test_function(xitp, xp)
+        yitp = self.build(scp, x, y, epsilon=5.0, kernel=kernel)(xitp)
+
+        mse = xp.mean((yitp - ytrue)**2)
+        assert mse < 2.0e-4
+        return yitp
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-8)
+    @pytest.mark.parametrize('kernel', sorted(_AVAILABLE))
+    def test_smoothing_misfit(self, xp, scp, kernel):
+        # Make sure we can find a smoothing parameter for each kernel that
+        # removes a sufficient amount of noise.
+        rng = _np.random.RandomState(0)
+        seq = Halton(1, scramble=False, seed=rng)
+
+        noise = 0.2
+        rmse_tol = 0.1
+        smoothing_range = 10**xp.linspace(-4, 1, 20)
+
+        x = xp.asarray(3*seq.random(100))
+        y = _1d_test_function(
+            x, xp) + xp.asarray(rng.normal(0.0, noise, (100,)))
+        ytrue = _1d_test_function(x, xp)
+        rmse_within_tol = False
+        for smoothing in smoothing_range:
+            ysmooth = self.build(scp,
+                                 x, y,
+                                 epsilon=1.0,
+                                 smoothing=smoothing,
+                                 kernel=kernel)(x)
+            rmse = xp.sqrt(xp.mean((ysmooth - ytrue)**2))
+            if rmse < rmse_tol:
+                rmse_within_tol = True
+                break
+
+        assert rmse_within_tol
+        return ysmooth
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_array_smoothing(self, xp, scp):
+        # Test using an array for `smoothing` to give less weight to a known
+        # outlier.
+        rng = _np.random.RandomState(0)
+        seq = Halton(1, scramble=False, seed=rng)
+        degree = 2
+
+        x = xp.asarray(seq.random(50))
+        if xp is _np:
+            P = _vandermonde(cp.asarray(x), degree).get()
+        else:
+            P = _vandermonde(x, degree)
+        poly_coeffs = xp.asarray(rng.normal(0.0, 1.0, P.shape[1]))
+        y = P.dot(poly_coeffs)
+        y_with_outlier = xp.copy(y)
+        y_with_outlier[10] += 1.0
+        smoothing = xp.zeros((50,))
+        smoothing[10] = 1000.0
+        yitp = self.build(scp, x, y_with_outlier, smoothing=smoothing)(x)
+        # Should be able to reproduce the uncorrupted data almost exactly.
+        return yitp, y
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_inconsistent_x_dimensions_error(self, xp, scp):
+        # ValueError should be raised if the observation points and evaluation
+        # points have a different number of dimensions.
+        y = Halton(2, scramble=False, seed=_np.random.RandomState()).random(10)
+        y = xp.asarray(y)
+        d = _2d_test_function(y, xp)
+        x = Halton(1, scramble=False, seed=_np.random.RandomState()).random(10)
+        x = xp.asarray(x)
+        self.build(scp, y, d)(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_inconsistent_d_length_error(self, xp, scp):
+        y = xp.linspace(0, 1, 5)[:, None]
+        d = xp.zeros(1)
+        self.build(scp, y, d)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_y_not_2d_error(self, xp, scp):
+        y = xp.linspace(0, 1, 5)
+        d = xp.zeros(5)
+        self.build(scp, y, d)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_inconsistent_smoothing_length_error(self, xp, scp):
+        y = xp.linspace(0, 1, 5)[:, None]
+        d = xp.zeros(5)
+        smoothing = xp.ones(1)
+        self.build(scp, y, d, smoothing=smoothing)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_invalid_kernel_name_error(self, xp, scp):
+        y = xp.linspace(0, 1, 5)[:, None]
+        d = xp.zeros(5)
+        self.build(scp, y, d, kernel='test')
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    @pytest.mark.parametrize('kernel', sorted(_AVAILABLE))
+    def test_epsilon_not_specified_error(self, xp, scp, kernel):
+        if kernel in _SCALE_INVARIANT:
+            return True
+
+        y = xp.linspace(0, 1, 5)[:, None]
+        d = xp.zeros(5)
+        self.build(scp, y, d, kernel=kernel)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_x_not_2d_error(self, xp, scp):
+        y = xp.linspace(0, 1, 5)[:, None]
+        x = xp.linspace(0, 1, 5)
+        d = xp.zeros(5)
+        self.build(scp, y, d)(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=ValueError)
+    def test_not_enough_observations_error(self, xp, scp):
+        y = xp.linspace(0, 1, 1)[:, None]
+        d = xp.zeros(1)
+        self.build(scp, y, d, kernel='thin_plate_spline')
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=UserWarning)
+    @pytest.mark.parametrize('kernel',
+                             [kl for kl in _NAME_TO_MIN_DEGREE])
+    def test_degree_warning(self, xp, scp, kernel):
+        y = xp.linspace(0, 1, 5)[:, None]
+        d = xp.zeros(5)
+        deg = _NAME_TO_MIN_DEGREE[kernel]
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            self.build(scp, y, d, epsilon=1.0, kernel=kernel, degree=deg-1)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', accept_error=LinAlgError)
+    def test_rank_error(self, xp, scp):
+        # An error should be raised when `kernel` is "thin_plate_spline" and
+        # observations are 2-D and collinear.
+        y = xp.array([[2.0, 0.0], [1.0, 0.0], [0.0, 0.0]])
+        d = xp.array([0.0, 0.0, 0.0])
+        with cupyx.errstate(linalg='raise'):
+            self.build(scp, y, d, kernel='thin_plate_spline')(y)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('dim', [1, 2, 3])
+    def test_single_point(self, xp, scp, dim):
+        # Make sure interpolation still works with only one point (in 1, 2, and
+        # 3 dimensions).
+        y = xp.zeros((1, dim))
+        d = xp.ones((1,))
+        f = self.build(scp, y, d, kernel='linear')(y)
+        return d, f
+
+    def test_pickleable(self):
+        # Make sure we can pickle and unpickle the interpolant without any
+        # changes in the behavior.
+        seq = Halton(1, scramble=False,
+                     seed=_np.random.RandomState(2305982309))
+
+        x = cp.asarray(3*seq.random(50))
+        xitp = cp.asarray(3*seq.random(50))
+        y = _1d_test_function(x, cp)
+
+        interp = cupyx.scipy.interpolate.RBFInterpolator(x, y)
+
+        yitp1 = interp(xitp)
+        yitp2 = pickle.loads(pickle.dumps(interp))(xitp)
+
+        testing.assert_array_equal(yitp1, yitp2)
+
+
+@testing.with_requires("scipy>=1.7.0")
+class TestRBFInterpolatorNeighborsNone(_TestRBFInterpolator):
+    def build(self, scp, *args, **kwargs):
+        return scp.interpolate.RBFInterpolator(*args, **kwargs)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_smoothing_limit_1d(self, xp, scp):
+        # For large smoothing parameters, the interpolant should approach a
+        # least squares fit of a polynomial with the specified degree.
+        seq = Halton(1, scramble=False, seed=_np.random.RandomState())
+
+        degree = 3
+        smoothing = 1e8
+
+        x = xp.asarray(3*seq.random(50))
+        xitp = xp.asarray(3*seq.random(50))
+
+        y = _1d_test_function(x, xp)
+
+        yitp1 = self.build(scp,
+                           x, y,
+                           degree=degree,
+                           smoothing=smoothing
+                           )(xitp)
+
+        if xp is _np:
+            P = _vandermonde(cp.asarray(x), degree).get()
+            Pitp = _vandermonde(cp.asarray(xitp), degree).get()
+        else:
+            P = _vandermonde(x, degree)
+            Pitp = _vandermonde(xitp, degree)
+
+        yitp2 = Pitp.dot(xp.linalg.lstsq(P, y, rcond=None)[0])
+
+        return yitp1, yitp2
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_smoothing_limit_2d(self, xp, scp):
+        # For large smoothing parameters, the interpolant should approach a
+        # least squares fit of a polynomial with the specified degree.
+        seq = Halton(2, scramble=False, seed=_np.random.RandomState())
+
+        degree = 3
+        smoothing = 1e8
+
+        x = xp.asarray(seq.random(100))
+        xitp = xp.asarray(seq.random(100))
+
+        y = _2d_test_function(x, xp)
+
+        yitp1 = self.build(scp,
+                           x, y,
+                           degree=degree,
+                           smoothing=smoothing
+                           )(xitp)
+
+        if xp is _np:
+            P = _vandermonde(cp.asarray(x), degree).get()
+            Pitp = _vandermonde(cp.asarray(xitp), degree).get()
+        else:
+            P = _vandermonde(x, degree)
+            Pitp = _vandermonde(xitp, degree)
+        yitp2 = Pitp.dot(xp.linalg.lstsq(P, y, rcond=None)[0])
+
+        return yitp1, yitp2
+
+    @pytest.mark.slow
+    def test_chunking(self):
+        # If the observed data comes from a polynomial, then the interpolant
+        # should be able to reproduce the polynomial exactly, provided that
+        # `degree` is sufficiently high.
+        rng = _np.random.RandomState(0)
+        seq = Halton(2, scramble=False, seed=rng)
+        degree = 3
+
+        largeN = 1000 + 33
+        # this is large to check that chunking of the RBFInterpolator is tested
+        x = cp.asarray(seq.random(50))
+        xitp = cp.asarray(seq.random(largeN))
+
+        P = _vandermonde(x, degree)
+        Pitp = _vandermonde(xitp, degree)
+
+        poly_coeffs = cp.asarray(rng.normal(0.0, 1.0, P.shape[1]))
+
+        y = P.dot(poly_coeffs)
+        yitp1 = Pitp.dot(poly_coeffs)
+        interp = cupyx.scipy.interpolate.RBFInterpolator(x, y, degree=degree)
+        ce_real = interp._chunk_evaluator
+
+        def _chunk_evaluator(*args, **kwargs):
+            kwargs.update(memory_budget=100)
+            return ce_real(*args, **kwargs)
+
+        # monkeypatch.setattr(interp, '_chunk_evaluator', _chunk_evaluator)
+        interp._chunk_evaluator = _chunk_evaluator
+        yitp2 = interp(xitp)
+        testing.assert_allclose(yitp1, yitp2, atol=1e-8)
+
+
+"""
+# Disable `all neighbors not None` tests : they need KDTree
+
+class TestRBFInterpolatorNeighbors20(_TestRBFInterpolator):
+    # RBFInterpolator using 20 nearest neighbors.
+    def build(self, *args, **kwargs):
+        return RBFInterpolator(*args, **kwargs, neighbors=20)
+
+    def test_equivalent_to_rbf_interpolator(self):
+        seq = Halton(2, scramble=False, seed=_np.random.RandomState())
+
+        x = cp.asarray(seq.random(100))
+        xitp = cp.asarray(seq.random(100))
+
+        y = _2d_test_function(x)
+
+        yitp1 = self.build(x, y)(xitp)
+
+        yitp2 = []
+        tree = cKDTree(x)
+        for xi in xitp:
+            _, nbr = tree.query(xi, 20)
+            yitp2.append(RBFInterpolator(x[nbr], y[nbr])(xi[None])[0])
+
+        assert_allclose(yitp1, yitp2, atol=1e-8)
+
+
+class TestRBFInterpolatorNeighborsInf(TestRBFInterpolatorNeighborsNone):
+    # RBFInterpolator using neighbors=np.inf. This should give exactly the same
+    # results as neighbors=None, but it will be slower.
+    def build(self, *args, **kwargs):
+        return RBFInterpolator(*args, **kwargs, neighbors=cp.inf)
+
+    def test_equivalent_to_rbf_interpolator(self):
+        seq = Halton(1, scramble=False, seed=_np.random.RandomState())
+
+        x = cp.asarray(3*seq.random(50))
+        xitp = cp.asarray(3*seq.random(50))
+
+        y = _1d_test_function(x)
+        yitp1 = self.build(x, y)(xitp)
+        yitp2 = RBFInterpolator(x, y)(xitp)
+
+        assert_allclose(yitp1, yitp2, atol=1e-8)
+"""
diff --git a/tests/cupyx_tests/scipy_tests/interpolate_tests/test_rgi.py b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_rgi.py
new file mode 100644
index 0000000..87e5c23
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/interpolate_tests/test_rgi.py
@@ -0,0 +1,741 @@
+import pytest
+import cupy as cp
+
+from cupy.testing import (assert_allclose, assert_array_equal,
+                          assert_array_almost_equal)
+from cupy_backends.cuda.api import runtime
+from pytest import raises as assert_raises
+
+from cupyx.scipy.interpolate import RegularGridInterpolator, interpn
+
+methods = ['linear', 'nearest']
+if not runtime.is_hip:
+    methods += ["slinear", "cubic", "quintic", 'pchip']
+
+parametrize_rgi_interp_methods = pytest.mark.parametrize("method", methods)
+
+
+class TestRegularGridInterpolator:
+    def _get_sample_4d(self):
+        # create a 4-D grid of 3 points in each dimension
+        points = [(0., .5, 1.)] * 4
+        values = cp.asarray([0., .5, 1.])
+        values0 = values[:, cp.newaxis, cp.newaxis, cp.newaxis]
+        values1 = values[cp.newaxis, :, cp.newaxis, cp.newaxis]
+        values2 = values[cp.newaxis, cp.newaxis, :, cp.newaxis]
+        values3 = values[cp.newaxis, cp.newaxis, cp.newaxis, :]
+        values = (values0 + values1 * 10 + values2 * 100 + values3 * 1000)
+        return points, values
+
+    def _get_sample_4d_2(self):
+        # create another 4-D grid of 3 points in each dimension
+        points = [(0., .5, 1.)] * 2 + [(0., 5., 10.)] * 2
+        values = cp.asarray([0., .5, 1.])
+        values0 = values[:, cp.newaxis, cp.newaxis, cp.newaxis]
+        values1 = values[cp.newaxis, :, cp.newaxis, cp.newaxis]
+        values2 = values[cp.newaxis, cp.newaxis, :, cp.newaxis]
+        values3 = values[cp.newaxis, cp.newaxis, cp.newaxis, :]
+        values = (values0 + values1 * 10 + values2 * 100 + values3 * 1000)
+        return points, values
+
+    def _get_sample_4d_3(self):
+        # create another 4-D grid of 7 points in each dimension
+        points = [(0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0)] * 4
+        values = cp.asarray([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0])
+        values0 = values[:, cp.newaxis, cp.newaxis, cp.newaxis]
+        values1 = values[cp.newaxis, :, cp.newaxis, cp.newaxis]
+        values2 = values[cp.newaxis, cp.newaxis, :, cp.newaxis]
+        values3 = values[cp.newaxis, cp.newaxis, cp.newaxis, :]
+        values = (values0 + values1 * 10 + values2 * 100 + values3 * 1000)
+        return points, values
+
+    def _get_sample_4d_4(self):
+        # create another 4-D grid of 2 points in each dimension
+        points = [(0.0, 1.0)] * 4
+        values = cp.asarray([0.0, 1.0])
+        values0 = values[:, cp.newaxis, cp.newaxis, cp.newaxis]
+        values1 = values[cp.newaxis, :, cp.newaxis, cp.newaxis]
+        values2 = values[cp.newaxis, cp.newaxis, :, cp.newaxis]
+        values3 = values[cp.newaxis, cp.newaxis, cp.newaxis, :]
+        values = (values0 + values1 * 10 + values2 * 100 + values3 * 1000)
+        return points, values
+
+    @parametrize_rgi_interp_methods
+    def test_complex(self, method):
+        points, values = self._get_sample_4d_3()
+        values = values - 2j*values
+        sample = cp.asarray([[0.1, 0.1, 1., .9], [0.2, 0.1, .45, .8],
+                             [0.5, 0.5, .5, .5]])
+
+        interp = RegularGridInterpolator(points, values, method=method)
+        rinterp = RegularGridInterpolator(points, values.real, method=method)
+        iinterp = RegularGridInterpolator(points, values.imag, method=method)
+
+        v1 = interp(sample)
+        v2 = rinterp(sample) + 1j*iinterp(sample)
+        assert_allclose(v1, v2)
+
+    def test_linear_xi1d(self):
+        points, values = self._get_sample_4d_2()
+        interp = RegularGridInterpolator(points, values)
+        sample = cp.asarray([0.1, 0.1, 10., 9.])
+        wanted = 1001.1
+        assert_array_almost_equal(interp(sample), wanted)
+
+    def test_linear_xi3d(self):
+        points, values = self._get_sample_4d()
+        interp = RegularGridInterpolator(points, values)
+        sample = cp.asarray([[0.1, 0.1, 1., .9], [0.2, 0.1, .45, .8],
+                             [0.5, 0.5, .5, .5]])
+        wanted = cp.asarray([1001.1, 846.2, 555.5])
+        assert_array_almost_equal(interp(sample), wanted)
+
+    @pytest.mark.parametrize(
+        "sample, wanted",
+        [
+            (cp.asarray([0.1, 0.1, 0.9, 0.9]), 1100.0),
+            (cp.asarray([0.1, 0.1, 0.1, 0.1]), 0.0),
+            (cp.asarray([0.0, 0.0, 0.0, 0.0]), 0.0),
+            (cp.asarray([1.0, 1.0, 1.0, 1.0]), 1111.0),
+            (cp.asarray([0.1, 0.4, 0.6, 0.9]), 1055.0),
+        ],
+    )
+    def test_nearest(self, sample, wanted):
+        points, values = self._get_sample_4d()
+        interp = RegularGridInterpolator(points, values, method="nearest")
+        assert_array_almost_equal(interp(sample), wanted)
+
+    def test_linear_edges(self):
+        points, values = self._get_sample_4d()
+        interp = RegularGridInterpolator(points, values)
+        sample = cp.asarray([[0., 0., 0., 0.], [1., 1., 1., 1.]])
+        wanted = cp.asarray([0., 1111.])
+        assert_array_almost_equal(interp(sample), wanted)
+
+    def test_valid_create(self):
+        # create a 2-D grid of 3 points in each dimension
+        points = [(0., .5, 1.), (0., 1., .5)]
+        values = cp.asarray([0., .5, 1.])
+        values0 = values[:, cp.newaxis]
+        values1 = values[cp.newaxis, :]
+        values = (values0 + values1 * 10)
+        assert_raises(ValueError, RegularGridInterpolator, points, values)
+        points = [((0., .5, 1.), ), (0., .5, 1.)]
+        assert_raises(ValueError, RegularGridInterpolator, points, values)
+        points = [(0., .5, .75, 1.), (0., .5, 1.)]
+        assert_raises(ValueError, RegularGridInterpolator, points, values)
+        points = [(0., .5, 1.), (0., .5, 1.), (0., .5, 1.)]
+        assert_raises(ValueError, RegularGridInterpolator, points, values)
+        points = [(0., .5, 1.), (0., .5, 1.)]
+        assert_raises(ValueError, RegularGridInterpolator, points, values,
+                      method="undefmethod")
+
+    def test_valid_call(self):
+        points, values = self._get_sample_4d()
+        interp = RegularGridInterpolator(points, values)
+        sample = cp.asarray([[0., 0., 0., 0.], [1., 1., 1., 1.]])
+        assert_raises(ValueError, interp, sample, "undefmethod")
+        sample = cp.asarray([[0., 0., 0.], [1., 1., 1.]])
+        assert_raises(ValueError, interp, sample)
+        sample = cp.asarray([[0., 0., 0., 0.], [1., 1., 1., 1.1]])
+        assert_raises(ValueError, interp, sample)
+
+    def test_out_of_bounds_extrap(self):
+        points, values = self._get_sample_4d()
+        interp = RegularGridInterpolator(points, values, bounds_error=False,
+                                         fill_value=None)
+        sample = cp.asarray([[-.1, -.1, -.1, -.1], [1.1, 1.1, 1.1, 1.1],
+                             [21, 2.1, -1.1, -11], [2.1, 2.1, -1.1, -1.1]])
+        wanted = cp.asarray([0., 1111., 11., 11.])
+        assert_array_almost_equal(interp(sample, method="nearest"), wanted)
+        wanted = cp.asarray([-111.1, 1222.1, -11068., -1186.9])
+        assert_array_almost_equal(interp(sample, method="linear"), wanted)
+
+    def test_out_of_bounds_extrap2(self):
+        points, values = self._get_sample_4d_2()
+        interp = RegularGridInterpolator(points, values, bounds_error=False,
+                                         fill_value=None)
+        sample = cp.asarray([[-.1, -.1, -.1, -.1], [1.1, 1.1, 1.1, 1.1],
+                             [21, 2.1, -1.1, -11], [2.1, 2.1, -1.1, -1.1]])
+        wanted = cp.asarray([0., 11., 11., 11.])
+        assert_array_almost_equal(interp(sample, method="nearest"), wanted)
+        wanted = cp.asarray([-12.1, 133.1, -1069., -97.9])
+        assert_array_almost_equal(interp(sample, method="linear"), wanted)
+
+    def test_out_of_bounds_fill(self):
+        points, values = self._get_sample_4d()
+        interp = RegularGridInterpolator(points, values, bounds_error=False,
+                                         fill_value=cp.nan)
+        sample = cp.asarray([[-.1, -.1, -.1, -.1], [1.1, 1.1, 1.1, 1.1],
+                             [2.1, 2.1, -1.1, -1.1]])
+        wanted = cp.asarray([cp.nan, cp.nan, cp.nan])
+        assert_array_almost_equal(interp(sample, method="nearest"), wanted)
+        assert_array_almost_equal(interp(sample, method="linear"), wanted)
+        sample = cp.asarray([[0.1, 0.1, 1., .9], [0.2, 0.1, .45, .8],
+                             [0.5, 0.5, .5, .5]])
+        wanted = cp.asarray([1001.1, 846.2, 555.5])
+        assert_array_almost_equal(interp(sample), wanted)
+
+    def test_invalid_fill_value(self):
+        cp.random.seed(1234)
+        x = cp.linspace(0, 2, 5)
+        y = cp.linspace(0, 1, 7)
+        values = cp.random.rand(5, 7)
+
+        # integers can be cast to floats
+        RegularGridInterpolator((x, y), values, fill_value=1)
+
+        # complex values cannot
+        assert_raises(ValueError, RegularGridInterpolator,
+                      (x, y), values, fill_value=1+2j)
+
+    def test_fillvalue_type(self):
+        # from #3703; test that interpolator object construction succeeds
+        values = cp.ones((10, 20, 30), dtype='>f4')
+        points = [cp.arange(n) for n in values.shape]
+        # xi = [(1, 1, 1)]
+        RegularGridInterpolator(points, values)
+        RegularGridInterpolator(points, values, fill_value=0.)
+
+    def test_length_one_axis(self):
+        # gh-5890, gh-9524 : length-1 axis is legal for method='linear'.
+        # Along the axis it's linear interpolation; away from the length-1
+        # axis, it's an extrapolation, so fill_value should be used.
+        def f(x, y):
+            return x + y
+        x = cp.linspace(1, 1, 1)
+        y = cp.linspace(1, 10, 10)
+        data = f(*cp.meshgrid(x, y, indexing="ij", sparse=True))
+
+        interp = RegularGridInterpolator((x, y), data, method="linear",
+                                         bounds_error=False, fill_value=101)
+
+        # check values at the grid
+        assert_allclose(interp(cp.array([[1, 1], [1, 5], [1, 10]])),
+                        [2, 6, 11],
+                        atol=1e-14)
+
+        # check off-grid interpolation is indeed linear
+        assert_allclose(interp(cp.array([[1, 1.4], [1, 5.3], [1, 10]])),
+                        [2.4, 6.3, 11],
+                        atol=1e-14)
+
+        # check exrapolation w/ fill_value
+        assert_allclose(interp(cp.array([1.1, 2.4])),
+                        interp.fill_value,
+                        atol=1e-14)
+
+        # check extrapolation: linear along the `y` axis, const along `x`
+        interp.fill_value = None
+        assert_allclose(interp([[1, 0.3], [1, 11.5]]),
+                        [1.3, 12.5], atol=1e-15)
+
+        assert_allclose(interp([[1.5, 0.3], [1.9, 11.5]]),
+                        [1.3, 12.5], atol=1e-15)
+
+        # extrapolation with method='nearest'
+        interp = RegularGridInterpolator((x, y), data, method="nearest",
+                                         bounds_error=False, fill_value=None)
+        assert_allclose(interp([[1.5, 1.8], [-4, 5.1]]),
+                        [3, 6],
+                        atol=1e-15)
+
+    @pytest.mark.parametrize("fill_value", [None, cp.nan, cp.pi])
+    @pytest.mark.parametrize("method", ['linear', 'nearest'])
+    def test_length_one_axis2(self, fill_value, method):
+        options = {"fill_value": fill_value, "bounds_error": False,
+                   "method": method}
+
+        x = cp.linspace(0, 2*cp.pi, 20)
+        z = cp.sin(x)
+
+        fa = RegularGridInterpolator((x,), z[:], **options)
+        fb = RegularGridInterpolator((x, [0]), z[:, None], **options)
+
+        x1a = cp.linspace(-1, 2*cp.pi+1, 100)
+        za = fa(x1a)
+
+        # evaluated at provided y-value, fb should behave exactly as fa
+        y1b = cp.zeros(100)
+        zb = fb(cp.vstack([x1a, y1b]).T)
+        assert_allclose(zb, za)
+
+        # evaluated at a different y-value, fb should return fill value
+        y1b = cp.ones(100)
+        zb = fb(cp.vstack([x1a, y1b]).T)
+        if fill_value is None:
+            assert_allclose(zb, za)
+        else:
+            assert_allclose(zb, fill_value)
+
+    @pytest.mark.parametrize("method", ['nearest', 'linear'])
+    def test_nan_x_1d(self, method):
+        # gh-6624 : if x is nan, result should be nan
+        f = RegularGridInterpolator((cp.array([1, 2, 3]),),
+                                    cp.array([10, 20, 30]), fill_value=1,
+                                    bounds_error=False, method=method)
+        assert cp.isnan(f([cp.nan]))
+
+        # test arbitrary nan pattern
+        rng = cp.random.default_rng(8143215468)
+        x = rng.random(size=100)*4
+        i = rng.random(size=100) > 0.5
+        x[i] = cp.nan
+
+        # out-of-bounds comparisons, `out_of_bounds += x < grid[0]`,
+        # generate numpy warnings if `x` contains nans.
+        # These warnings should propagate to user
+        res = f(x)
+
+        assert_array_equal(res[i], cp.nan)
+        assert_array_equal(res[~i], f(x[~i]))
+
+        # also test the length-one axis f(nan)
+        x = [1, 2, 3]
+        y = [1, ]
+        data = cp.ones((3, 1))
+        f = RegularGridInterpolator((x, y), data, fill_value=1,
+                                    bounds_error=False, method=method)
+        assert cp.isnan(f([cp.nan, 1]))
+        assert cp.isnan(f([1, cp.nan]))
+
+    @pytest.mark.parametrize("method", ['nearest', 'linear'])
+    def test_nan_x_2d(self, method):
+        x, y = cp.array([0, 1, 2]), cp.array([1, 3, 7])
+
+        def f(x, y):
+            return x**2 + y**2
+
+        xg, yg = cp.meshgrid(x, y, indexing='ij', sparse=True)
+        data = f(xg, yg)
+        interp = RegularGridInterpolator((x, y), data,
+                                         method=method, bounds_error=False)
+
+        res = interp([[1.5, cp.nan], [1, 1]])
+        assert_allclose(res[1], 2, atol=1e-14)
+        assert cp.isnan(res[0])
+
+        # test arbitrary nan pattern
+        rng = cp.random.default_rng(8143215468)
+        x = rng.random(size=100)*4-1
+        y = rng.random(size=100)*8
+        i1 = rng.random(size=100) > 0.5
+        i2 = rng.random(size=100) > 0.5
+        i = i1 | i2
+        x[i1] = cp.nan
+        y[i2] = cp.nan
+        z = cp.array([x, y]).T
+        # out-of-bounds comparisons, `out_of_bounds += x < grid[0]`,
+        # generate numpy warnings if `x` contains nans.
+        # These warnings should propagate to user (since `x` is user
+        # input)
+        res = interp(z)
+
+        assert_array_equal(res[i], cp.nan)
+        assert_array_equal(res[~i], interp(z[~i]))
+
+    @parametrize_rgi_interp_methods
+    def test_descending_points(self, method):
+        def val_func_3d(x, y, z):
+            return 2 * x ** 3 + 3 * y ** 2 - z
+
+        x = cp.linspace(1, 4, 11)
+        y = cp.linspace(4, 7, 22)
+        z = cp.linspace(7, 9, 33)
+        points = (x, y, z)
+        values = val_func_3d(
+            *cp.meshgrid(*points, indexing='ij', sparse=True))
+        my_interpolating_function = RegularGridInterpolator(points,
+                                                            values,
+                                                            method=method)
+        pts = cp.array([[2.1, 6.2, 8.3], [3.3, 5.2, 7.1]])
+        correct_result = my_interpolating_function(pts)
+
+        # descending data
+        x_descending = x[::-1]
+        y_descending = y[::-1]
+        z_descending = z[::-1]
+        points_shuffled = (x_descending, y_descending, z_descending)
+        values_shuffled = val_func_3d(
+            *cp.meshgrid(*points_shuffled, indexing='ij', sparse=True))
+        my_interpolating_function = RegularGridInterpolator(
+            points_shuffled, values_shuffled, method=method)
+        test_result = my_interpolating_function(pts)
+
+        assert_array_equal(correct_result, test_result)
+
+    def test_invalid_points_order(self):
+        def val_func_2d(x, y):
+            return 2 * x ** 3 + 3 * y ** 2
+
+        x = cp.array([.5, 2., 0., 4., 5.5])  # not ascending or descending
+        y = cp.array([.5, 2., 3., 4., 5.5])
+        points = (x, y)
+        values = val_func_2d(*cp.meshgrid(*points, indexing='ij',
+                                          sparse=True))
+        match = "must be strictly ascending or descending"
+        with pytest.raises(ValueError, match=match):
+            RegularGridInterpolator(points, values)
+
+    @parametrize_rgi_interp_methods
+    def test_fill_value(self, method):
+        interp = RegularGridInterpolator([cp.arange(6)], cp.ones(6),
+                                         method=method, bounds_error=False)
+        assert cp.isnan(interp([10]))
+
+    @parametrize_rgi_interp_methods
+    def test_nonscalar_values(self, method):
+        # Verify that non-scalar valued values also works
+        points = [(0.0, 0.5, 1.0, 1.5, 2.0, 2.5)] * 2 + [
+            (0.0, 5.0, 10.0, 15.0, 20, 25.0)
+        ] * 2
+
+        rng = cp.random.default_rng(1234)
+        values = rng.random((6, 6, 6, 6, 8))
+        sample = rng.random((7, 3, 4))
+
+        interp = RegularGridInterpolator(points, values, method=method,
+                                         bounds_error=False)
+        v = interp(sample)
+        assert_array_equal(v.shape, (7, 3, 8), err_msg=method)
+
+        vs = []
+        for j in range(8):
+            interp = RegularGridInterpolator(points, values[..., j],
+                                             method=method,
+                                             bounds_error=False)
+            vs.append(interp(sample))
+        v2 = cp.array(vs).transpose(1, 2, 0)
+
+        assert_allclose(v, v2, atol=1e-14, err_msg=method)
+
+    @parametrize_rgi_interp_methods
+    @pytest.mark.parametrize("flip_points", [False, True])
+    def test_nonscalar_values_2(self, method, flip_points):
+        # Verify that non-scalar valued values also work : use different
+        # lengths of axes to simplify tracing the internals
+        points = [(0.0, 0.5, 1.0, 1.5, 2.0, 2.5),
+                  (0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0),
+                  (0.0, 5.0, 10.0, 15.0, 20, 25.0, 35.0, 36.0),
+                  (0.0, 5.0, 10.0, 15.0, 20, 25.0, 35.0, 36.0, 47)]
+
+        # verify, that strictly decreasing dimensions work
+        if flip_points:
+            points = [tuple(reversed(p)) for p in points]
+
+        rng = cp.random.default_rng(1234)
+
+        trailing_points = (3, 2)
+        # NB: values has a `num_trailing_dims` trailing dimension
+        values = rng.random((6, 7, 8, 9, *trailing_points))
+        sample = rng.random(4)   # a single sample point !
+
+        interp = RegularGridInterpolator(points, values, method=method,
+                                         bounds_error=False)
+        v = interp(sample)
+
+        # v has a single sample point *per entry in the trailing dimensions*
+        assert v.shape == (1, *trailing_points)
+
+        # check the values, too : manually loop over the trailing dimensions
+        vs = cp.empty((values.shape[-2:]))
+        for i in range(values.shape[-2]):
+            for j in range(values.shape[-1]):
+                interp = RegularGridInterpolator(points, values[..., i, j],
+                                                 method=method,
+                                                 bounds_error=False)
+                vs[i, j] = interp(sample)
+        v2 = cp.expand_dims(vs, axis=0)
+        assert_allclose(v, v2, atol=1e-14, err_msg=method)
+
+    def test_nonscalar_values_linear_2D(self):
+        # Verify that non-scalar values work in the 2D fast path
+        method = 'linear'
+        points = [(0.0, 0.5, 1.0, 1.5, 2.0, 2.5),
+                  (0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0), ]
+
+        rng = cp.random.default_rng(1234)
+
+        trailing_points = (3, 4)
+        # NB: values has a `num_trailing_dims` trailing dimension
+        values = rng.random((6, 7, *trailing_points))
+        sample = rng.random(2)   # a single sample point !
+
+        interp = RegularGridInterpolator(points, values, method=method,
+                                         bounds_error=False)
+        v = interp(sample)
+
+        # v has a single sample point *per entry in the trailing dimensions*
+        assert v.shape == (1, *trailing_points)
+
+        # check the values, too : manually loop over the trailing dimensions
+        vs = cp.empty((values.shape[-2:]))
+        for i in range(values.shape[-2]):
+            for j in range(values.shape[-1]):
+                interp = RegularGridInterpolator(points, values[..., i, j],
+                                                 method=method,
+                                                 bounds_error=False)
+                vs[i, j] = interp(sample)
+        v2 = cp.expand_dims(vs, axis=0)
+        assert_allclose(v, v2, atol=1e-14, err_msg=method)
+
+
+class MyValue:
+    """
+    Minimal indexable object
+    """
+
+    def __init__(self, shape):
+        self.ndim = 2
+        self.shape = shape
+        self._v = cp.arange(cp.prod(cp.array(shape))).reshape(shape)
+
+    def __getitem__(self, idx):
+        return self._v[idx]
+
+    def __array_interface__(self):
+        return None
+
+    def __array__(self):
+        raise RuntimeError("No array representation")
+
+
+class TestInterpN:
+    def _sample_2d_data(self):
+        x = cp.array([.5, 2., 3., 4., 5.5, 6.])
+        y = cp.array([.5, 2., 3., 4., 5.5, 6.])
+        z = cp.array(
+            [
+                [1, 2, 1, 2, 1, 1],
+                [1, 2, 1, 2, 1, 1],
+                [1, 2, 3, 2, 1, 1],
+                [1, 2, 2, 2, 1, 1],
+                [1, 2, 1, 2, 1, 1],
+                [1, 2, 2, 2, 1, 1],
+            ]
+        )
+        return x, y, z
+
+    def _sample_4d_data(self):
+        points = [(0., .5, 1.)] * 2 + [(0., 5., 10.)] * 2
+        values = cp.asarray([0., .5, 1.])
+        values0 = values[:, cp.newaxis, cp.newaxis, cp.newaxis]
+        values1 = values[cp.newaxis, :, cp.newaxis, cp.newaxis]
+        values2 = values[cp.newaxis, cp.newaxis, :, cp.newaxis]
+        values3 = values[cp.newaxis, cp.newaxis, cp.newaxis, :]
+        values = (values0 + values1 * 10 + values2 * 100 + values3 * 1000)
+        return points, values
+
+    def test_linear_4d(self):
+        # create a 4-D grid of 3 points in each dimension
+        points, values = self._sample_4d_data()
+        interp_rg = RegularGridInterpolator(points, values)
+        sample = cp.asarray([[0.1, 0.1, 10., 9.]])
+        wanted = interpn(points, values, sample, method="linear")
+        assert_array_almost_equal(interp_rg(sample), wanted)
+
+    def test_4d_linear_outofbounds(self):
+        # create a 4-D grid of 3 points in each dimension
+        points, values = self._sample_4d_data()
+        sample = cp.asarray([[0.1, -0.1, 10.1, 9.]])
+        wanted = 999.99
+        actual = interpn(points, values, sample, method="linear",
+                         bounds_error=False, fill_value=999.99)
+        assert_array_almost_equal(actual, wanted)
+
+    def test_nearest_4d(self):
+        # create a 4-D grid of 3 points in each dimension
+        points, values = self._sample_4d_data()
+        interp_rg = RegularGridInterpolator(points, values, method="nearest")
+        sample = cp.asarray([[0.1, 0.1, 10., 9.]])
+        wanted = interpn(points, values, sample, method="nearest")
+        assert_array_almost_equal(interp_rg(sample), wanted)
+
+    def test_4d_nearest_outofbounds(self):
+        # create a 4-D grid of 3 points in each dimension
+        points, values = self._sample_4d_data()
+        sample = cp.asarray([[0.1, -0.1, 10.1, 9.]])
+        wanted = 999.99
+        actual = interpn(points, values, sample, method="nearest",
+                         bounds_error=False, fill_value=999.99)
+        assert_array_almost_equal(actual, wanted)
+
+    def test_xi_1d(self):
+        # verify that 1-D xi works as expected
+        points, values = self._sample_4d_data()
+        sample = cp.asarray([0.1, 0.1, 10., 9.])
+        v1 = interpn(points, values, sample, bounds_error=False)
+        v2 = interpn(points, values, sample[None, :], bounds_error=False)
+        assert_allclose(v1, v2)
+
+    def test_xi_nd(self):
+        # verify that higher-d xi works as expected
+        points, values = self._sample_4d_data()
+
+        cp.random.seed(1234)
+        sample = cp.random.rand(2, 3, 4)
+
+        v1 = interpn(points, values, sample, method='nearest',
+                     bounds_error=False)
+        assert_array_equal(v1.shape, (2, 3))
+
+        v2 = interpn(points, values, sample.reshape(-1, 4),
+                     method='nearest', bounds_error=False)
+        assert_allclose(v1, v2.reshape(v1.shape))
+
+    @parametrize_rgi_interp_methods
+    def test_xi_broadcast(self, method):
+        # verify that the interpolators broadcast xi
+        x, y, values = self._sample_2d_data()
+        points = (x, y)
+
+        xi = cp.linspace(0, 1, 2)
+        yi = cp.linspace(0, 3, 3)
+
+        sample = (xi[:, None], yi[None, :])
+        v1 = interpn(points, values, sample, method=method, bounds_error=False)
+        assert_array_equal(v1.shape, (2, 3))
+
+        xx, yy = cp.meshgrid(xi, yi)
+        sample = cp.c_[xx.T.ravel(), yy.T.ravel()]
+
+        v2 = interpn(points, values, sample,
+                     method=method, bounds_error=False)
+        assert_allclose(v1, v2.reshape(v1.shape))
+
+    @parametrize_rgi_interp_methods
+    def test_nonscalar_values(self, method):
+        # Verify that non-scalar valued values also works
+        points = [(0.0, 0.5, 1.0, 1.5, 2.0, 2.5)] * 2 + [
+            (0.0, 5.0, 10.0, 15.0, 20, 25.0)
+        ] * 2
+
+        rng = cp.random.default_rng(1234)
+        values = rng.random((6, 6, 6, 6, 8))
+        sample = rng.random((7, 3, 4))
+
+        v = interpn(points, values, sample, method=method,
+                    bounds_error=False)
+        assert_array_equal(v.shape, (7, 3, 8), err_msg=method)
+
+        vs = [interpn(points, values[..., j], sample, method=method,
+                      bounds_error=False) for j in range(8)]
+        v2 = cp.array(vs).transpose(1, 2, 0)
+
+        assert_allclose(v, v2, atol=1e-14, err_msg=method)
+
+    @parametrize_rgi_interp_methods
+    def test_nonscalar_values_2(self, method):
+        # Verify that non-scalar valued values also work : use different
+        # lengths of axes to simplify tracing the internals
+        points = [(0.0, 0.5, 1.0, 1.5, 2.0, 2.5),
+                  (0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0),
+                  (0.0, 5.0, 10.0, 15.0, 20, 25.0, 35.0, 36.0),
+                  (0.0, 5.0, 10.0, 15.0, 20, 25.0, 35.0, 36.0, 47)]
+
+        rng = cp.random.default_rng(1234)
+
+        trailing_points = (3, 2)
+        # NB: values has a `num_trailing_dims` trailing dimension
+        values = rng.random((6, 7, 8, 9, *trailing_points))
+        sample = rng.random(4)   # a single sample point !
+
+        v = interpn(points, values, sample, method=method, bounds_error=False)
+
+        # v has a single sample point *per entry in the trailing dimensions*
+        assert v.shape == (1, *trailing_points)
+
+        # check the values, too : manually loop over the trailing dimensions
+        vs = [[
+            interpn(points, values[..., i, j], sample, method=method,
+                    bounds_error=False) for i in range(values.shape[-2])
+        ] for j in range(values.shape[-1])]
+
+        assert_allclose(v, cp.asarray(vs).T, atol=1e-14, err_msg=method)
+
+    @parametrize_rgi_interp_methods
+    def test_complex(self, method):
+        x, y, values = self._sample_2d_data()
+        points = (x, y)
+        values = values - 2j*values
+
+        sample = cp.array([[1, 2.3, 5.3, 0.5, 3.3, 1.2, 3],
+                           [1, 3.3, 1.2, 4.0, 5.0, 1.0, 3]]).T
+
+        v1 = interpn(points, values, sample, method=method)
+        v2r = interpn(points, values.real, sample, method=method)
+        v2i = interpn(points, values.imag, sample, method=method)
+        v2 = v2r + 1j*v2i
+        assert_allclose(v1, v2)
+
+    def test_length_one_axis(self):
+        # gh-5890, gh-9524 : length-1 axis is legal for method='linear'.
+        # Along the axis it's linear interpolation; away from the length-1
+        # axis, it's an extrapolation, so fill_value should be used.
+
+        values = cp.array([[0.1, 1, 10]])
+        xi = cp.array([[1, 2.2], [1, 3.2], [1, 3.8]])
+
+        res = interpn(([1], [2, 3, 4]), values, xi)
+        wanted = [0.9*0.2 + 0.1,   # on [2, 3) it's 0.9*(x-2) + 0.1
+                  9*0.2 + 1,       # on [3, 4] it's 9*(x-3) + 1
+                  9*0.8 + 1]
+
+        assert_allclose(res, wanted, atol=1e-15)
+
+        # check extrapolation
+        xi = cp.array([[1.1, 2.2], [1.5, 3.2], [-2.3, 3.8]])
+        res = interpn(([1], [2, 3, 4]), values, xi,
+                      bounds_error=False, fill_value=None)
+
+        assert_allclose(res, wanted, atol=1e-15)
+
+    def test_descending_points(self):
+        def value_func_4d(x, y, z, a):
+            return 2 * x ** 3 + 3 * y ** 2 - z - a
+
+        x1 = cp.array([0, 1, 2, 3])
+        x2 = cp.array([0, 10, 20, 30])
+        x3 = cp.array([0, 10, 20, 30])
+        x4 = cp.array([0, .1, .2, .30])
+        points = (x1, x2, x3, x4)
+        values = value_func_4d(
+            *cp.meshgrid(*points, indexing='ij', sparse=True))
+        pts = (cp.array(0.1),
+               cp.array(0.3),
+               cp.transpose(cp.linspace(0, 30, 4)),
+               cp.linspace(0, 0.3, 4))
+        correct_result = interpn(points, values, pts)
+
+        x1_descend = x1[::-1]
+        x2_descend = x2[::-1]
+        x3_descend = x3[::-1]
+        x4_descend = x4[::-1]
+        points_shuffled = (x1_descend, x2_descend, x3_descend, x4_descend)
+        values_shuffled = value_func_4d(
+            *cp.meshgrid(*points_shuffled, indexing='ij', sparse=True))
+        test_result = interpn(points_shuffled, values_shuffled, pts)
+
+        assert_array_equal(correct_result, test_result)
+
+    def test_invalid_points_order(self):
+        x = cp.array([.5, 2., 0., 4., 5.5])  # not ascending or descending
+        y = cp.array([.5, 2., 3., 4., 5.5])
+        z = cp.array([[1, 2, 1, 2, 1], [1, 2, 1, 2, 1], [1, 2, 3, 2, 1],
+                      [1, 2, 2, 2, 1], [1, 2, 1, 2, 1]])
+        xi = cp.array([[1, 2.3, 6.3, 0.5, 3.3, 1.2, 3],
+                       [1, 3.3, 1.2, -4.0, 5.0, 1.0, 3]]).T
+
+        match = "must be strictly ascending or descending"
+        with pytest.raises(ValueError, match=match):
+            interpn((x, y), z, xi)
+
+    def test_invalid_xi_dimensions(self):
+        # https://github.com/scipy/scipy/issues/16519
+        points = [cp.array((0, 1))]
+        values = cp.array([0, 1])
+        xi = cp.ones((1, 1, 3))
+        msg = ("The requested sample points xi have dimension 3, but this "
+               "RegularGridInterpolator has dimension 1")
+        with assert_raises(ValueError, match=msg):
+            interpn(points, values, xi)
diff --git a/tests/cupyx_tests/scipy_tests/linalg_tests/__init__.py b/tests/cupyx_tests/scipy_tests/linalg_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/linalg_tests/test_decomp_lu.py b/tests/cupyx_tests/scipy_tests/linalg_tests/test_decomp_lu.py
new file mode 100644
index 0000000..e414370
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/linalg_tests/test_decomp_lu.py
@@ -0,0 +1,155 @@
+import unittest
+import warnings
+
+import numpy
+import cupy
+from cupy import testing
+import cupyx.scipy.linalg
+if cupyx.scipy._scipy_available:
+    import scipy.linalg
+
+
+# TODO: After the feature is released
+# requires_scipy_linalg_backend = testing.with_requires('scipy>=1.x.x')
+requires_scipy_linalg_backend = unittest.skip(
+    'scipy.linalg backend feature has not been released'
+)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(1, 1), (2, 2), (3, 3), (5, 5), (1, 5), (5, 1), (2, 5), (5, 2)],
+}))
+@testing.fix_random()
+@testing.with_requires('scipy')
+class TestLUFactor(unittest.TestCase):
+
+    @testing.for_dtypes('fdFD')
+    def test_lu_factor(self, dtype):
+        if self.shape[0] != self.shape[1]:
+            self.skipTest(
+                'skip non-square tests since scipy.lu_factor requires square')
+        a_cpu = testing.shaped_random(self.shape, numpy, dtype=dtype)
+        a_gpu = cupy.asarray(a_cpu)
+        result_cpu = scipy.linalg.lu_factor(a_cpu)
+        result_gpu = cupyx.scipy.linalg.lu_factor(a_gpu)
+        assert len(result_cpu) == len(result_gpu)
+        assert result_cpu[0].dtype == result_gpu[0].dtype
+        assert result_cpu[1].dtype == result_gpu[1].dtype
+        cupy.testing.assert_allclose(result_cpu[0], result_gpu[0], atol=1e-5)
+        cupy.testing.assert_array_equal(result_cpu[1], result_gpu[1])
+
+    def check_lu_factor_reconstruction(self, A):
+        m, n = self.shape
+        lu, piv = cupyx.scipy.linalg.lu_factor(A)
+        # extract ``L`` and ``U`` from ``lu``
+        L = cupy.tril(lu, k=-1)
+        cupy.fill_diagonal(L, 1.)
+        L = L[:, :m]
+        U = cupy.triu(lu)
+        U = U[:n, :]
+        # check output shapes
+        assert lu.shape == (m, n)
+        assert L.shape == (m, min(m, n))
+        assert U.shape == (min(m, n), n)
+        assert piv.shape == (min(m, n),)
+        # apply pivot (on CPU since slaswp is not available in cupy)
+        piv = cupy.asnumpy(piv)
+        rows = numpy.arange(m)
+        for i, row in enumerate(piv):
+            if i != row:
+                rows[i], rows[row] = rows[row], rows[i]
+        PA = A[rows]
+        # check that reconstruction is close to original
+        LU = L.dot(U)
+        cupy.testing.assert_allclose(LU, PA, atol=1e-5)
+
+    @testing.for_dtypes('fdFD')
+    def test_lu_factor_reconstruction(self, dtype):
+        A = testing.shaped_random(self.shape, cupy, dtype=dtype)
+        self.check_lu_factor_reconstruction(A)
+
+    @testing.for_dtypes('fdFD')
+    def test_lu_factor_reconstruction_singular(self, dtype):
+        if self.shape[0] != self.shape[1]:
+            self.skipTest(
+                'skip non-square tests since scipy.lu_factor requires square')
+        A = testing.shaped_random(self.shape, cupy, dtype=dtype)
+        A -= A.mean(axis=0, keepdims=True)
+        A -= A.mean(axis=1, keepdims=True)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', RuntimeWarning)
+            self.check_lu_factor_reconstruction(A)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(1, 1), (2, 2), (3, 3), (5, 5), (1, 5), (5, 1), (2, 5), (5, 2)],
+    'permute_l': [False, True],
+}))
+@testing.fix_random()
+@testing.with_requires('scipy')
+class TestLU(unittest.TestCase):
+
+    @testing.for_dtypes('fdFD')
+    def test_lu(self, dtype):
+        a_cpu = testing.shaped_random(self.shape, numpy, dtype=dtype)
+        a_gpu = cupy.asarray(a_cpu)
+        result_cpu = scipy.linalg.lu(a_cpu, permute_l=self.permute_l)
+        result_gpu = cupyx.scipy.linalg.lu(a_gpu, permute_l=self.permute_l)
+        assert len(result_cpu) == len(result_gpu)
+        if not self.permute_l:
+            # check permutation matrix
+            result_cpu = list(result_cpu)
+            result_gpu = list(result_gpu)
+            P_cpu = result_cpu.pop(0)
+            P_gpu = result_gpu.pop(0)
+            cupy.testing.assert_array_equal(P_gpu, P_cpu)
+        cupy.testing.assert_allclose(result_gpu[0], result_cpu[0], atol=1e-5)
+        cupy.testing.assert_allclose(result_gpu[1], result_cpu[1], atol=1e-5)
+
+    @testing.for_dtypes('fdFD')
+    def test_lu_reconstruction(self, dtype):
+        m, n = self.shape
+        A = testing.shaped_random(self.shape, cupy, dtype=dtype)
+        if self.permute_l:
+            PL, U = cupyx.scipy.linalg.lu(A, permute_l=self.permute_l)
+            PLU = PL @ U
+        else:
+            P, L, U = cupyx.scipy.linalg.lu(A, permute_l=self.permute_l)
+            PLU = P @ L @ U
+        # check that reconstruction is close to original
+        cupy.testing.assert_allclose(PLU, A, atol=1e-5)
+
+
+@testing.parameterize(*testing.product({
+    'trans': [0, 1, 2],
+    'shapes': [((4, 4), (4,)), ((5, 5), (5, 2))],
+}))
+@testing.fix_random()
+@testing.with_requires('scipy')
+class TestLUSolve(unittest.TestCase):
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_lu_solve(self, xp, scp, dtype):
+        a_shape, b_shape = self.shapes
+        A = testing.shaped_random(a_shape, xp, dtype=dtype)
+        b = testing.shaped_random(b_shape, xp, dtype=dtype)
+        lu = scp.linalg.lu_factor(A)
+        return scp.linalg.lu_solve(lu, b, trans=self.trans)
+
+    @requires_scipy_linalg_backend
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_lu_solve_backend(self, xp, dtype):
+        a_shape, b_shape = self.shapes
+        A = testing.shaped_random(a_shape, xp, dtype=dtype)
+        b = testing.shaped_random(b_shape, xp, dtype=dtype)
+        if xp is numpy:
+            lu = scipy.linalg.lu_factor(A)
+            backend = 'scipy'
+        else:
+            lu = cupyx.scipy.linalg.lu_factor(A)
+            backend = cupyx.scipy.linalg
+        with scipy.linalg.set_backend(backend):
+            out = scipy.linalg.lu_solve(lu, b, trans=self.trans)
+        return out
diff --git a/tests/cupyx_tests/scipy_tests/linalg_tests/test_solve_triangular.py b/tests/cupyx_tests/scipy_tests/linalg_tests/test_solve_triangular.py
new file mode 100644
index 0000000..57b6d33
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/linalg_tests/test_solve_triangular.py
@@ -0,0 +1,95 @@
+import unittest
+
+import cupy
+from cupy import testing
+import cupyx.scipy.linalg
+
+import numpy
+import pytest
+
+try:
+    import scipy.linalg
+    _scipy_available = True
+except ImportError:
+    _scipy_available = False
+
+
+@testing.parameterize(*testing.product({
+    'trans': [0, 1, 2, 'N', 'T', 'C'],
+    'lower': [True, False],
+    'unit_diagonal': [True, False],
+    'overwrite_b': [True, False],
+    'check_finite': [True, False],
+}))
+@testing.with_requires('scipy')
+class TestSolveTriangular(unittest.TestCase):
+
+    @testing.for_dtypes('fdFD')
+    def check_x(self, a_shape, b_shape, dtype):
+        a_cpu = numpy.random.randint(1, 10, size=a_shape).astype(dtype)
+        b_cpu = numpy.random.randint(1, 10, size=b_shape).astype(dtype)
+        a_cpu = numpy.tril(a_cpu)
+
+        if self.lower is False:
+            a_cpu = a_cpu.T
+        if self.unit_diagonal is True:
+            numpy.fill_diagonal(a_cpu, 1)
+
+        a_gpu = cupy.asarray(a_cpu)
+        b_gpu = cupy.asarray(b_cpu)
+        a_gpu_copy = a_gpu.copy()
+        b_gpu_copy = b_gpu.copy()
+        result_cpu = scipy.linalg.solve_triangular(
+            a_cpu, b_cpu, trans=self.trans, lower=self.lower,
+            unit_diagonal=self.unit_diagonal, overwrite_b=self.overwrite_b,
+            check_finite=self.check_finite)
+        result_gpu = cupyx.scipy.linalg.solve_triangular(
+            a_gpu, b_gpu, trans=self.trans, lower=self.lower,
+            unit_diagonal=self.unit_diagonal, overwrite_b=self.overwrite_b,
+            check_finite=self.check_finite)
+        assert result_cpu.dtype == result_gpu.dtype
+        cupy.testing.assert_allclose(result_cpu, result_gpu, atol=1e-3)
+        cupy.testing.assert_array_equal(a_gpu_copy, a_gpu)
+        if not self.overwrite_b:
+            cupy.testing.assert_array_equal(b_gpu_copy, b_gpu)
+
+    def test_solve(self):
+        self.check_x((4, 4), (4,))
+        self.check_x((5, 5), (5, 2))
+        self.check_x((5, 5), (5, 5))
+
+    def check_shape(self, a_shape, b_shape):
+        for xp, sp in ((numpy, scipy), (cupy, cupyx.scipy)):
+            a = xp.random.rand(*a_shape)
+            b = xp.random.rand(*b_shape)
+            with pytest.raises(ValueError):
+                sp.linalg.solve_triangular(
+                    a, b, trans=self.trans, lower=self.lower,
+                    unit_diagonal=self.unit_diagonal,
+                    overwrite_b=self.overwrite_b,
+                    check_finite=self.check_finite)
+
+    def test_invalid_shape(self):
+        self.check_shape((2, 3), (4,))
+        self.check_shape((3, 3), (2,))
+        self.check_shape((3, 3), (2, 2))
+        self.check_shape((3, 3, 4), (3,))
+
+    def check_infinite(self, a_shape, b_shape):
+        for xp, sp in ((numpy, scipy), (cupy, cupyx.scipy)):
+            a = xp.random.rand(*a_shape)
+            b = xp.random.rand(*b_shape)
+            a[(0,) * a.ndim] = numpy.inf
+            b[(0,) * b.ndim] = numpy.inf
+            with pytest.raises(ValueError):
+                sp.linalg.solve_triangular(
+                    a, b, trans=self.trans, lower=self.lower,
+                    unit_diagonal=self.unit_diagonal,
+                    overwrite_b=self.overwrite_b,
+                    check_finite=self.check_finite)
+
+    def test_infinite(self):
+        if self.check_finite:
+            self.check_infinite((4, 4), (4,))
+            self.check_infinite((5, 5), (5, 2))
+            self.check_infinite((5, 5), (5, 5))
diff --git a/tests/cupyx_tests/scipy_tests/linalg_tests/test_special_matrices.py b/tests/cupyx_tests/scipy_tests/linalg_tests/test_special_matrices.py
new file mode 100644
index 0000000..c1aea2e
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/linalg_tests/test_special_matrices.py
@@ -0,0 +1,107 @@
+import unittest
+
+from cupy import testing
+import cupyx.scipy.linalg  # NOQA
+
+try:
+    import scipy.linalg  # NOQA
+except ImportError:
+    pass
+
+
+class TestSpecialMatricesBase(unittest.TestCase):
+    def _get_arg(self, xp, arg):
+        if isinstance(arg, tuple):
+            # Allocate array with the given shape
+            return testing.shaped_random(arg, xp)
+
+        # Otherwise just pass the arg back
+        return arg
+
+
+@testing.parameterize(*(
+    testing.product({
+        # 1 argument: 1D array
+        'function': ['circulant', 'toeplitz', 'hankel', 'companion'],
+        'args': [((1,),), ((2,),), ((4,),), ((10,),), ((25,),)],
+    }) + testing.product({
+        # 2 arguments: both 1D arrays
+        # For leslie, second array should be 1 less long than first
+        'function': ['toeplitz', 'hankel', 'leslie'],
+        'args': [((1,), (1,)), ((2,), (1,)), ((4,), (5,)),
+                 ((10,), (9,)), ((25,), (24,))],
+    }) + testing.product({
+        # 1 argument: int
+        'function': ['hadamard', 'helmert', 'hilbert', 'dft'],
+        'args': [(1,), (2,), (4,), (10,), (25,)],
+    }) + testing.product({
+        # 2 arguments: int, dtype
+        'function': ['hadamard'],
+        'args': [(4, 'int32'), (8, 'float64'), (6, 'float64')],
+    }) + testing.product({
+        # 2 arguments: int, bool
+        'function': ['helmert'],
+        'args': [(4, True), (5, False)],
+    }) + testing.product({
+        # 2 arguments: int, str
+        'function': ['dft'],
+        'args': [(4, 'sqrtn'), (5, 'n')],
+    }) + testing.product({
+        # 2 arguments: both 2D arrays
+        'function': ['kron'],
+        'args': [((5, 5), (4, 5),), ((5, 4), (4, 4),), ((1, 2), (4, 5))],
+    }) + testing.product({
+        # 0 or more arguments: all 1 or 2D arrays
+        'function': ['block_diag'],
+        'args': [(), ((5,),), ((4, 5),), ((5,), (4, 4),), ((1, 2), (4, 5)),
+                 ((1,), (4, 5), (3, 6), (7, 2), (8, 9))],
+    })
+))
+@testing.with_requires('scipy')
+class TestSpecialMatrices(TestSpecialMatricesBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_special_matrix(self, xp, scp):
+        function = getattr(scp.linalg, self.function)
+        return function(*[self._get_arg(xp, arg) for arg in self.args])
+
+
+@testing.parameterize(*(
+    testing.product({
+        # 1 argument: 1D array
+        'function': ['fiedler', 'fiedler_companion'],
+        'args': [((0,),), ((1,),), ((2,),), ((4,),), ((10,),), ((25,),)],
+    })
+))
+@testing.with_requires('scipy>=1.3.0')
+class TestSpecialMatrices_1_3_0(TestSpecialMatricesBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_special_matrix(self, xp, scp):
+        function = getattr(scp.linalg, self.function)
+        return function(*[self._get_arg(xp, arg) for arg in self.args])
+
+    def _get_arg(self, xp, arg):
+        if isinstance(arg, tuple):
+            # Allocate array with the given shape
+            return testing.shaped_random(arg, xp)
+
+        # Otherwise just pass the arg back
+        return arg
+
+
+@testing.parameterize(*(
+    testing.product({
+        # 2-3 arguments: 1D array, 1 int, 1 optional str
+        'function': ['convolution_matrix'],
+        'args': [((1,), 5), ((2,), 3), ((4,), 10), ((10,), 15), ((25,), 25),
+                 ((4,), 6, 'full'), ((10,), 8, 'same'), ((25,), 25, 'valid')],
+    })
+))
+@testing.with_requires('scipy>=1.5.0')
+class TestSpecialMatrices_1_5_0(TestSpecialMatricesBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_special_matrix(self, xp, scp):
+        function = getattr(scp.linalg, self.function)
+        return function(*[self._get_arg(xp, arg) for arg in self.args])
diff --git a/tests/cupyx_tests/scipy_tests/linalg_tests/test_uarray.py b/tests/cupyx_tests/scipy_tests/linalg_tests/test_uarray.py
new file mode 100644
index 0000000..0751e46
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/linalg_tests/test_uarray.py
@@ -0,0 +1,9 @@
+from cupy import testing
+import cupyx.scipy.linalg._uarray
+
+
+@testing.with_requires('scipy>=1.5')
+def test_implements_names():
+    # With the newest SciPy, the decorator `@implements` must find the
+    # matching scipy functions.
+    assert not cupyx.scipy.linalg._uarray._notfound
diff --git a/tests/cupyx_tests/scipy_tests/ndimage_tests/__init__.py b/tests/cupyx_tests/scipy_tests/ndimage_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/ndimage_tests/test_filters.py b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_filters.py
new file mode 100644
index 0000000..a76e07b
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_filters.py
@@ -0,0 +1,824 @@
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupy import testing
+import cupyx.scipy.ndimage  # NOQA
+
+try:
+    import scipy.ndimage  # NOQA
+except ImportError:
+    pass
+
+
+class FilterTestCaseBase:
+    """
+    Add some utility methods for the parameterized tests for filters. these
+    assume there are the "parameters" self.filter, self.wdtype or self.dtype,
+    and self.ndim, self.kshape, or self.shape. Other optional "parameters" are
+    also used if available like self.footprint when the filter is a filter
+    that uses the footprint. These methods allow testing across multiple
+    filter types much more easily.
+    """
+
+    # default param values if not provided
+    filter = 'convolve'
+    shape = (4, 5)
+    ksize = 3
+    dtype = numpy.float64
+    footprint = True
+    order = 'C'
+    mode = 'reflect'
+
+    # Params that need to be possibly shortened to the right number of dims
+    DIMS_PARAMS = ('origin', 'sigma')
+
+    # Params that need no processing and just go into kwargs
+    KWARGS_PARAMS = ('output', 'axis', 'mode', 'cval', 'truncate')+DIMS_PARAMS
+
+    # Params that need no processing go before weights in the arguments
+    ARGS_PARAMS = ('rank', 'percentile',
+                   'derivative', 'derivative2', 'function')
+
+    def _filter(self, xp, scp):
+        """
+        The function that all tests end up calling, possibly after a few
+        adjustments to the class "parameters".
+        """
+        # The filter function
+        filter = getattr(scp.ndimage, self.filter)
+
+        # The kwargs to pass to the filter function
+        kwargs = {param: getattr(self, param)
+                  for param in FilterTestCaseBase.KWARGS_PARAMS
+                  if hasattr(self, param)}
+
+        if 'cval' in kwargs and kwargs['cval'] is cupy.nan:
+            # skip comparisons in cases where CuPy doesn't support NaN cval
+            if self.filter in ['minimum_filter', 'minimum_filter1d',
+                               'maximum_filter', 'maximum_filter1d',
+                               'median_filter', 'percentile_filter',
+                               'rank_filter']:
+                return xp.asarray([])
+
+        is_1d = self.filter.endswith('1d')
+        for param in FilterTestCaseBase.DIMS_PARAMS:
+            if param in kwargs and isinstance(kwargs[param], tuple):
+                value = kwargs[param]
+                kwargs[param] = value[0] if is_1d else value[:self._ndim]
+
+        # The array we are filtering
+        arr = testing.shaped_random(self.shape, xp, self.dtype)
+        if self.order == 'F':
+            arr = xp.asfortranarray(arr)
+
+        # The weights we are using to filter
+        wghts = self._get_weights(xp)
+        if isinstance(wghts, tuple) and len(wghts) == 2 and wghts[0] is None:
+            # w is actually a tuple of (None, footprint)
+            wghts, kwargs['footprint'] = wghts
+
+        # Bulid the arguments
+        args = [getattr(self, param)
+                for param in FilterTestCaseBase.ARGS_PARAMS
+                if hasattr(self, param)]
+        if wghts is not None:
+            args.append(wghts)
+
+        # Actually perform filtering
+        return filter(arr, *args, **kwargs)
+
+    def _get_weights(self, xp):
+        # Gets the second argument to the filter functions.
+        # For convolve/correlate/convolve1d/correlate1d this is the weights.
+        # For minimum_filter1d/maximum_filter1d this is the kernel size.
+        #
+        # For filters that take a footprint this is a bit more complicated and
+        # is either the kernel size or a tuple of None and the footprint. The
+        # _filter() method knows about this and handles it automatically.
+        #
+        # Many specialized filters have no weights and this returns None.
+
+        if self.filter in ('convolve', 'correlate'):
+            return testing.shaped_random(self._kshape, xp, self._dtype)
+
+        if self.filter in ('convolve1d', 'correlate1d'):
+            return testing.shaped_random((self.ksize,), xp, self._dtype)
+
+        if self.filter in ('minimum_filter', 'maximum_filter', 'median_filter',
+                           'rank_filter', 'percentile_filter', 'generic_filter'
+                           ):
+            if not self.footprint:
+                return self.ksize
+            kshape = self._kshape
+            footprint = testing.shaped_random(kshape, xp, scale=1) > 0.5
+            if not footprint.any():
+                footprint = xp.ones(kshape)
+            return None, footprint
+
+        if self.filter in ('uniform_filter1d', 'generic_filter1d',
+                           'minimum_filter1d', 'maximum_filter1d'):
+            return self.ksize
+
+        if self.filter == 'uniform_filter':
+            return self._kshape
+
+        # gaussian_filter*, prewitt, sobel, *laplace, and *_gradient_magnitude
+        # all have no 'weights' or similar argument so return None
+        return None
+
+    @property
+    def _dtype(self):
+        return getattr(self, 'wdtype', None) or self.dtype
+
+    @property
+    def _ndim(self):
+        return getattr(self, 'ndim', len(getattr(self, 'shape', [])))
+
+    @property
+    def _kshape(self):
+        return getattr(self, 'kshape', (self.ksize,) * self._ndim)
+
+
+# Parameters common across all modes (with some overrides)
+COMMON_PARAMS = {
+    'shape': [(4, 5), (3, 4, 5), (1, 3, 4, 5)],
+    'ksize': [3, 4],
+    'dtype': [numpy.uint8, numpy.float64],
+}
+
+# Floating point dtypes only
+COMMON_FLOAT_PARAMS = dict(COMMON_PARAMS)
+COMMON_FLOAT_PARAMS['dtype'] = [numpy.float32, numpy.float64]
+
+
+# The bulk of the tests are done with this class
+@testing.parameterize(*(
+    testing.product_dict(
+        # Filter-function specific params
+        testing.product({
+            'filter': ['convolve', 'correlate'],
+        }) + testing.product({
+            'filter': ['convolve1d', 'correlate1d', 'minimum_filter1d'],
+            'axis': [0, 1, -1],
+        }) + testing.product({
+            'filter': ['minimum_filter', 'median_filter'],
+            'footprint': [False, True],
+        }),
+
+        # Mode-specific params
+        testing.product({
+            **COMMON_PARAMS,
+            'mode': ['reflect'],
+            # With reflect test some of the other parameters as well
+            'origin': [0, 1, (-1, 1, -1, 1)],
+            'output': [None, numpy.uint8, numpy.float64],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'mode': ['constant'], 'cval': [-1.0, 0.0, 1.0],
+        }) + testing.product({
+            **COMMON_FLOAT_PARAMS,
+            'mode': ['constant'], 'cval': [numpy.nan, numpy.inf, -numpy.inf],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'mode': ['nearest', 'wrap'],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'shape': [(4, 5), (3, 4, 5)],  # no (1,3,4,5) due to scipy bug
+            'mode': ['mirror'],
+        })
+    )
+))
+@testing.with_requires('scipy')
+class TestFilter(FilterTestCaseBase):
+
+    def _hip_skip_invalid_condition(self):
+        if not runtime.is_hip:
+            return
+        if self.dtype != numpy.float64:
+            return
+        if self.filter != 'median_filter':
+            return
+        if self.mode == 'mirror':
+            invalids = [(False, 3, (4, 5)),
+                        (False, 3, (3, 4, 5)),
+                        (False, 4, (4, 5)),
+                        (True, 4, (4, 5))]
+            if (self.footprint, self.ksize, self.shape) in invalids:
+                pytest.xfail('ROCm/HIP may have a bug')
+        elif self.mode == 'reflect':
+            if (self.footprint
+                    and self.ksize == 3
+                    and self.shape == (3, 4, 5)
+                    and self.output != numpy.float64):
+                pytest.xfail('ROCm/HIP may have a bug')
+        elif self.mode == 'nearest' or self.mode == 'wrap':
+            if (self.footprint
+                    and self.ksize == 3
+                    and self.shape == (3, 4, 5)):
+                pytest.xfail('ROCm/HIP may have a bug')
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        self._hip_skip_invalid_condition()
+        if self.dtype == getattr(self, 'output', None):
+            pytest.skip("redundant")
+        return self._filter(xp, scp)
+
+
+@testing.with_requires('scipy')
+class TestNearestFilterEdgeCase:
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        # Use a large input to check any compilation error, see cupy/cupy#6048
+        return scp.ndimage.gaussian_filter(
+            testing.shaped_random((26, 3, 52, 70, 50), xp, xp.float32),
+            [1, 1, 1, 1, 1], mode='nearest')
+
+
+def dummy_deriv_func(input, axis, output, mode, cval, *args, **kwargs):
+    # For testing generic_laplace and generic_gradient_magnitude. Doesn't test
+    # mode, cval, or extra argument but those are tested indirectly with
+    # laplace, gaussian_laplace, and gaussian_gradient_magnitude.
+    if output is not None and not isinstance(output, numpy.dtype):
+        output[...] = input + axis
+    else:
+        output = input + axis
+    return output
+
+
+# This tests filters that are very similar to other filters so we only check
+# the basics with them.
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'filter': ['maximum_filter1d', 'maximum_filter'],
+        }) + testing.product({
+            'filter': ['percentile_filter'],
+            'percentile': [0, 25, 50, -25, 100],
+        }) + testing.product({
+            'filter': ['rank_filter'],
+            'rank': [0, 1, -1],
+        }),
+        testing.product(COMMON_PARAMS)
+    ) + testing.product_dict(
+        # All of these filters have no ksize and evoke undef behavior
+        # outputting to uint8
+        testing.product({
+            'filter': ['gaussian_filter1d'],
+            'axis': [0, 1, -1],
+            'sigma': [1.5, 2],
+            'truncate': [2.75, 4],
+        }) + testing.product({
+            'filter': ['gaussian_filter', 'gaussian_laplace',
+                       'gaussian_gradient_magnitude'],
+            'sigma': [1.5, 2.25, (1.5, 2.25, 1.0, 3.0)],
+            'truncate': [2.75, 4],
+        }) + testing.product({
+            'filter': ['prewitt', 'sobel'],
+            'axis': [0, 1, -1],
+        }) + testing.product({
+            'filter': ['laplace'],
+        }),
+        testing.product({
+            'shape': [(4, 5), (3, 4, 5), (1, 3, 4, 5)],
+            'dtype': [numpy.uint8, numpy.float64],
+            'output': [numpy.float64],
+        })
+    ) + testing.product_dict(
+        # These take derivative functions to compute part of the process
+        testing.product({
+            'filter': ['generic_laplace'],
+            'derivative': [dummy_deriv_func],
+        }) + testing.product({
+            'filter': ['generic_gradient_magnitude'],
+            'derivative': [dummy_deriv_func],
+        }),
+        testing.product({
+            'shape': [(4, 5), (3, 4, 5), (1, 3, 4, 5)],
+            'dtype': [numpy.uint8, numpy.float64],
+        })
+    )
+))
+@testing.with_requires('scipy')
+class TestFilterFast(FilterTestCaseBase):
+
+    def _hip_skip_invalid_condition(self):
+        if not runtime.is_hip:
+            return
+        if self.filter != 'percentile_filter':
+            return
+        if self.ksize != 3:
+            return
+        invalids = [(25, (1, 3, 4, 5)),
+                    (50, (3, 4, 5)),
+                    (-25, (1, 3, 4, 5))]
+        if (self.percentile, self.shape) in invalids:
+            pytest.xfail('ROCm/HIP may have a bug')
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        self._hip_skip_invalid_condition()
+        return self._filter(xp, scp)
+
+
+# This tests filters that are very similar to other filters so we only check
+# the basics with them.
+@testing.parameterize(*(
+    testing.product_dict(
+        # All of these filters have no ksize and evoke undef behavior
+        # outputting to uint8
+        testing.product({
+            'filter': ['gaussian_filter1d'],
+            'axis': [0, 1, -1],
+            'sigma': [1.5, 2],
+            'truncate': [2.75, 4],
+        }) + testing.product({
+            'filter': ['gaussian_filter', 'gaussian_laplace',
+                       'gaussian_gradient_magnitude'],
+            'sigma': [1.5, 2.25, (1.5, 2.25, 1.0, 3.0)],
+            'truncate': [2.75, 4],
+        }) + testing.product({
+            'filter': ['prewitt', 'sobel'],
+            'axis': [0, 1, -1],
+        }) + testing.product({
+            'filter': ['laplace'],
+        }),
+        testing.product({
+            'shape': [(4, 5), (3, 4, 5), (1, 3, 4, 5)],
+            'dtype': [numpy.complex64, numpy.complex128],
+            'output': [None],
+        })
+    ) + testing.product_dict(
+        # These take derivative functions to compute part of the process
+        testing.product({
+            'filter': ['generic_laplace'],
+            'derivative': [dummy_deriv_func],
+        }) + testing.product({
+            'filter': ['generic_gradient_magnitude'],
+            'derivative': [dummy_deriv_func],
+        }),
+        testing.product({
+            'shape': [(4, 5), (3, 4, 5), (1, 3, 4, 5)],
+            'dtype': [numpy.complex64, numpy.complex128],
+        })
+    )
+))
+@testing.with_requires('scipy')
+class TestFilterComplexFast(FilterTestCaseBase):
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy>=1.6.0')
+    def test_filter(self, xp, scp):
+        return self._filter(xp, scp)
+
+
+# Kernels and Functions for testing generic_filter
+rms_raw = cupy.RawKernel('''extern "C" __global__
+void rms(const double* x, int filter_size, double* y) {
+    double ss = 0;
+    for (int i = 0; i < filter_size; ++i) { ss += x[i]*x[i]; }
+    y[0] = ss/filter_size;
+}''', 'rms')
+rms_red = cupy.ReductionKernel('X x', 'Y y', 'x*x',
+                               'a + b', 'y = a/_in_ind.size()', '0', 'rms')
+
+
+def rms_pyfunc(x):
+    return (x * x).sum()/len(x)
+
+
+lt_raw = cupy.RawKernel('''extern "C" __global__
+void lt(const double* x, int filter_size, double* y) {
+    int n = 0;
+    double c = x[filter_size/2];
+    for (int i = 0; i < filter_size; ++i) { n += c>x[i]; }
+    y[0] = n;
+}''', 'lt')
+lt_red = cupy.ReductionKernel('X x', 'Y y', '_raw_x[_in_ind.size()/2]>x',
+                              'a + b', 'y = a', '0', 'lt', reduce_type='int')
+
+
+def lt_pyfunc(x):
+    return (x[len(x)//2] > x).sum()
+
+
+# This tests generic_filter.
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'filter': ['generic_filter'],
+            'func_or_kernel': [
+                ('rms_raw', 'rms_pyfunc'),
+                ('lt_red', 'lt_pyfunc'),
+            ],
+            'footprint': [False, True],
+        }),
+
+        # Mode-specific params
+        testing.product({
+            **COMMON_PARAMS,
+            'mode': ['reflect'],
+            # With reflect test some of the other parameters as well
+            'origin': [0, 1, (-1, 1, -1, 1)],
+            'output': [None, numpy.uint8, numpy.float64],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'mode': ['constant'], 'cval': [0.0, 1.0],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'mode': ['nearest', 'wrap'],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'shape': [(4, 5), (3, 4, 5)],
+            'mode': ['mirror'],
+        })
+    ) + testing.product({
+        'filter': ['generic_filter'],
+        'func_or_kernel': [
+            ('rms_red', 'rms_pyfunc'),
+            ('lt_raw', 'lt_pyfunc'),
+        ],
+        'footprint': [False, True],
+        **COMMON_PARAMS,
+        'dtype': [numpy.float64],
+    })
+))
+@testing.with_requires('scipy')
+class TestGenericFilter(FilterTestCaseBase):
+
+    _func_or_kernels = {
+        'rms_raw': rms_raw,
+        'rms_red': rms_red,
+        'rms_pyfunc': rms_pyfunc,
+        'lt_raw': lt_raw,
+        'lt_red': lt_red,
+        'lt_pyfunc': lt_pyfunc,
+    }
+
+    def get_func_or_kernel(self, xp):
+        return self._func_or_kernels[
+            self.func_or_kernel[1 if xp == numpy else 0]]
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        # Need to deal with the different versions of the functions given to
+        # numpy vs cupy
+        self.function = self.get_func_or_kernel(xp)
+        return self._filter(xp, scp)
+
+
+# Kernels and functions for use with generic_filter1d
+def shift_pyfunc(src, dst):
+    dst[:] = src[:dst.size]
+
+
+shift_raw = cupy.RawKernel('''extern "C" __global__
+void shift(const double* in, ptrdiff_t in_length,
+          double* out, ptrdiff_t out_length) {
+    for (int i = 0; i < out_length; ++i) { out[i] = in[i]; }
+}''', 'shift')
+
+
+# This tests generic_filter1d.
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'filter': ['generic_filter1d'],
+            'func_or_kernel': [('shift_raw', 'shift_pyfunc')],
+            'axis': [0, 1, -1],
+        }),
+
+        # Mode-specific params
+        testing.product({
+            **COMMON_PARAMS,
+            'mode': ['reflect'],
+            # With reflect test some of the other parameters as well
+            'origin': [0, 1, (-1, 1, -1, 1)],
+            'output': [None, numpy.uint8, numpy.float64],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'mode': ['constant'], 'cval': [0.0, 1.0],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'mode': ['nearest', 'wrap'],
+        }) + testing.product({
+            **COMMON_PARAMS,
+            'shape': [(4, 5), (3, 4, 5)],
+            'mode': ['mirror'],
+        })
+    )
+))
+@testing.with_requires('scipy')
+class TestGeneric1DFilter(FilterTestCaseBase):
+    _func_or_kernels = {
+        'shift_raw': shift_raw,
+        'shift_pyfunc': shift_pyfunc,
+    }
+
+    def get_func_or_kernel(self, xp):
+        return self._func_or_kernels[
+            self.func_or_kernel[1 if xp == numpy else 0]]
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        # Need to deal with the different versions of the functions given to
+        # numpy vs cupy
+        self.function = self.get_func_or_kernel(xp)
+        return self._filter(xp, scp)
+
+
+# Tests things requiring scipy >= 1.5.0
+@testing.parameterize(*(
+    testing.product_dict(
+        # Filter-function specific params
+        testing.product({
+            'filter': ['convolve', 'correlate',
+                       'minimum_filter', 'maximum_filter'],
+        }) + testing.product({
+            'filter': ['convolve1d', 'correlate1d',
+                       'minimum_filter1d', 'maximum_filter1d'],
+            'axis': [0, 1, -1],
+        }),
+
+        # Mode-specific params
+        testing.product({
+            **COMMON_PARAMS,
+            'shape': [(1, 3, 4, 5)],
+            'mode': ['mirror'],
+        })
+    )
+))
+# SciPy behavior fixed in 1.5.0: https://github.com/scipy/scipy/issues/11661
+@testing.with_requires('scipy>=1.5.0')
+class TestMirrorWithDim1(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        return self._filter(xp, scp)
+
+
+# Tests kernels large enough to trigger shell sort in rank-based filters
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'filter': ['median_filter'],
+        }) + testing.product({
+            'filter': ['percentile_filter'],
+            'percentile': [25, -25],
+        }) + testing.product({
+            'filter': ['rank_filter'],
+            'rank': [1],
+        }),
+        testing.product({
+            'fooprint': [False, True],
+            'ksize': [16, 17],
+            'shape': [(20, 21)],
+        })
+    )
+))
+@testing.with_requires('scipy')
+class TestShellSort(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        return self._filter(xp, scp)
+
+
+# Tests with Fortran-ordered arrays
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'filter': ['convolve', 'correlate',
+                       'minimum_filter', 'maximum_filter'],
+        }) + testing.product({
+            'filter': ['convolve1d', 'correlate1d',
+                       'minimum_filter1d', 'maximum_filter1d'],
+            'axis': [0, 1, -1],
+        }),
+        testing.product({
+            **COMMON_PARAMS,
+            'shape': [(4, 5), (3, 4, 5)],
+            'order': ['F'],
+        })
+    )
+))
+@testing.with_requires('scipy')
+class TestFortranOrder(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        return self._filter(xp, scp)
+
+
+# Tests with weight dtypes that are distinct from the input and output dtypes
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'filter': ['convolve', 'correlate'],
+        }) + testing.product({
+            'filter': ['convolve1d', 'correlate1d'],
+            'axis': [0, 1, -1],
+        }),
+        testing.product({
+            **COMMON_PARAMS,
+            'mode': ['reflect'],
+            'output': [None, numpy.uint8, numpy.float64],
+            'dtype': [numpy.uint8, numpy.int16, numpy.int32,
+                      numpy.float32, numpy.float64],
+            'wdtype': [numpy.uint8, numpy.float64],
+        })
+    )
+))
+@testing.with_requires('scipy')
+class TestWeightDtype(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_filter(self, xp, scp):
+        if self.dtype == self.wdtype:
+            pytest.skip("redundant")
+        return self._filter(xp, scp)
+
+
+# Tests with weight dtypes that are distinct from the input and output dtypes
+@testing.parameterize(*(
+    testing.product_dict(
+        testing.product({
+            'filter': ['convolve', 'correlate'],
+        }) + testing.product({
+            'filter': ['convolve1d', 'correlate1d'],
+            'axis': [-1],
+        }),
+        testing.product({
+            'shape': [(8, 12)],
+            'ksize': [3],
+            'mode': ['reflect'],
+            'output': [None],
+            'dtype': [numpy.uint8, numpy.float32, numpy.float64,
+                      numpy.complex64, numpy.complex128],
+            'wdtype': [numpy.uint8, numpy.float32, numpy.float64,
+                       numpy.complex64, numpy.complex128],
+        })
+    )
+))
+@testing.with_requires('scipy>=1.5.9')
+class TestWeightComplexDtype(FilterTestCaseBase):
+
+    def _skip_noncomplex(self):
+        if not (numpy.dtype(self.dtype).kind == 'c' or
+                numpy.dtype(self.wdtype).kind == 'c'):
+            pytest.skip("non-complex")
+
+    def _get_array_and_weights(self, xp):
+        arr = testing.shaped_random(self.shape, xp, self.dtype)
+        weights = self._get_weights(xp)
+        return arr, weights
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy>=1.6.0')
+    def test_filter_complex(self, xp, scp):
+        self._skip_noncomplex()
+        return self._filter(xp, scp)
+
+    @testing.with_requires('scipy>=1.6.0')
+    def test_filter_complex_output_dtype_error(self):
+        # raises RuntimeError if provided a real-valued output array
+        self._skip_noncomplex()
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            func = getattr(scp.ndimage, self.filter)
+            arr, weights = self._get_array_and_weights(xp)
+            output = xp.empty(self.shape, dtype=numpy.float64)
+            with pytest.raises(RuntimeError):
+                func(arr, weights, output=output)
+
+    @testing.with_requires('scipy>=1.6.0')
+    def test_filter_complex_output_dtype_warns(self):
+        # warns if a real-valued dtype is specified for the output
+        self._skip_noncomplex()
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            func = getattr(scp.ndimage, self.filter)
+            arr, weights = self._get_array_and_weights(xp)
+            with pytest.warns(UserWarning):
+                func(arr, weights, output=numpy.float64)
+
+
+# Tests special weights (ND)
+@testing.parameterize(*testing.product({
+    'filter': ['convolve', 'correlate', 'minimum_filter', 'maximum_filter'],
+    'shape': [(3, 3), (3, 3, 3)],
+    'dtype': [numpy.uint8, numpy.float64],
+}))
+@testing.with_requires('scipy')
+class TestSpecialWeightCases(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_extra_0_dim(self, xp, scp):
+        # NOTE: minimum/maximum_filter raise ValueError but convolve/correlate
+        # return an array of zeroes the same shape as the input. This will
+        # handle both and only pass is both numpy and cupy do the same thing.
+        self.kshape = (0,) + self.shape
+        return self._filter(xp, scp)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=RuntimeError)
+    def test_missing_dim(self, xp, scp):
+        self.kshape = self.shape[1:]
+        return self._filter(xp, scp)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=RuntimeError)
+    def test_extra_dim(self, xp, scp):
+        self.kshape = self.shape[:1] + self.shape
+        return self._filter(xp, scp)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=(RuntimeError, ValueError))
+    def test_replace_dim_with_0(self, xp, scp):
+        self.kshape = (0,) + self.shape[1:]
+        return self._filter(xp, scp)
+
+
+# Tests special weights (1D)
+@testing.parameterize(*testing.product({
+    'filter': ['convolve1d', 'correlate1d',
+               'minimum_filter1d', 'maximum_filter1d'],
+    'shape': [(3, 3), (3, 3, 3)],
+    'dtype': [numpy.uint8, numpy.float64],
+}))
+@testing.with_requires('scipy')
+class TestSpecialCases1D(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=RuntimeError)
+    def test_0_dim(self, xp, scp):
+        self.ksize = 0
+        return self._filter(xp, scp)
+
+
+# Tests invalid axis value
+@testing.parameterize(*testing.product({
+    'filter': ['convolve1d', 'correlate1d',
+               'minimum_filter1d', 'maximum_filter1d'],
+    'shape': [(4, 5), (3, 4, 5), (1, 3, 4, 5)],
+}))
+@testing.with_requires('scipy')
+class TestInvalidAxis(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_invalid_axis_pos(self, xp, scp):
+        self.axis = len(self.shape)
+        try:
+            return self._filter(xp, scp)
+        except numpy.AxisError:
+            # numpy.AxisError is a subclass of ValueError
+            # currently cupyx is raising numpy.AxisError but scipy is still
+            # raising ValueError
+            raise ValueError('invalid axis')
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_invalid_axis_neg(self, xp, scp):
+        self.axis = -len(self.shape) - 1
+        try:
+            return self._filter(xp, scp)
+        except numpy.AxisError:
+            raise ValueError('invalid axis')
+
+
+# Tests invalid mode value
+@testing.parameterize(*testing.product({
+    'filter': ['convolve', 'correlate',
+               'convolve1d', 'correlate1d',
+               'minimum_filter', 'maximum_filter',
+               'minimum_filter1d', 'maximum_filter1d'],
+    'mode': ['unknown'],
+    'shape': [(4, 5)],
+}))
+@testing.with_requires('scipy')
+class TestInvalidMode(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=RuntimeError)
+    def test_invalid_mode(self, xp, scp):
+        return self._filter(xp, scp)
+
+
+# Tests invalid origin values
+@testing.parameterize(*testing.product({
+    'filter': ['convolve', 'correlate',
+               'convolve1d', 'correlate1d',
+               'minimum_filter', 'maximum_filter',
+               'minimum_filter1d', 'maximum_filter1d'],
+    'ksize': [3, 4],
+    'shape': [(4, 5)], 'dtype': [numpy.float64],
+}))
+# SciPy behavior fixed in 1.2.0: https://github.com/scipy/scipy/issues/822
+@testing.with_requires('scipy>=1.2.0')
+class TestInvalidOrigin(FilterTestCaseBase):
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_invalid_origin_neg(self, xp, scp):
+        self.origin = -self.ksize // 2 - 1
+        return self._filter(xp, scp)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_invalid_origin_pos(self, xp, scp):
+        self.origin = self.ksize - self.ksize // 2
+        return self._filter(xp, scp)
diff --git a/tests/cupyx_tests/scipy_tests/ndimage_tests/test_fourier.py b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_fourier.py
new file mode 100644
index 0000000..2cbd1b4
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_fourier.py
@@ -0,0 +1,421 @@
+import cupy
+import numpy
+import pytest
+
+from cupy import testing
+# TODO (grlee77): use fft instead of fftpack once min. supported scipy >= 1.4
+import cupyx.scipy.fft  # NOQA
+import cupyx.scipy.fftpack  # NOQA
+import cupyx.scipy.ndimage  # NOQA
+
+try:
+    # scipy.fft only available since SciPy 1.4.0
+    import scipy.fft  # NOQA
+except ImportError:
+    pass
+
+try:
+    # These modules will be present in all supported SciPy versions
+    import scipy
+    import scipy.fftpack  # NOQA
+    import scipy.ndimage  # NOQA
+    scipy_version = numpy.lib.NumpyVersion(scipy.__version__)
+except ImportError:
+    pass
+
+
+@testing.parameterize(
+    *(
+        testing.product(
+            {
+                "shape": [(32, 16), (31, 15)],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "shift": [1, -3, (5, 5.3), (3, 5)],
+            }
+        )
+        + testing.product(
+            {
+                "shape": [(5, 16, 7), ],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "shift": [3, (-1, 2.5, 1)],
+            }
+        )
+        + testing.product(
+            {
+                "shape": [(15, ), ],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "shift": [8.5, (5,)],
+            }
+        )
+    )
+)
+@testing.with_requires("scipy")
+class TestFourierShift:
+
+    def _test_real_nd(self, xp, scp, x, real_axis):
+        a = scp.fft.rfft(x, axis=real_axis)
+        # complex-valued FFTs on all other axes
+        complex_axes = tuple([ax for ax in range(x.ndim) if ax != real_axis])
+        if complex_axes:
+            a = scp.fft.fftn(a, axes=complex_axes)
+
+        a = scp.ndimage.fourier_shift(
+            a, self.shift, n=x.shape[real_axis], axis=real_axis)
+
+        if complex_axes:
+            a = scp.fft.ifftn(a, axes=complex_axes)
+        a = scp.fft.irfft(a, axis=real_axis)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+    @testing.with_requires("scipy>=1.4.0")
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_real_fft_axis0(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        if x.dtype.kind == 'c':
+            # skip: can't use rfft on complex-valued x
+            return x
+        return self._test_real_nd(xp, scp, x, 0)
+
+    @testing.with_requires("scipy>=1.4.0")
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_real_fft_axis1(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        if x.dtype.kind == 'c' or x.ndim < 2:
+            # skip: can't use rfft along axis 1 on complex-valued x or 1d x
+            return x
+        return self._test_real_nd(xp, scp, x, 1)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_complex_fft(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        a = scp.fftpack.fftn(x)
+        a = scp.ndimage.fourier_shift(a, self.shift)
+        a = scp.fftpack.ifftn(a)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_complex_fft_with_output(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        a = scp.fftpack.fftn(x)
+        scp.ndimage.fourier_shift(a, self.shift, output=a)
+        a = scp.fftpack.ifftn(a)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+
+@testing.parameterize(
+    *(
+        testing.product(
+            {
+                "shape": [(32, 16), (31, 15)],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "sigma": [1, (5, 5.3), (3, 5)],
+            }
+        )
+        + testing.product(
+            {
+                "shape": [(5, 16, 7), ],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "sigma": [3, (1, 2.5, 3)],
+            }
+        )
+        + testing.product(
+            {
+                "shape": [(15, ), ],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "sigma": [8.5, (5,)],
+            }
+        )
+    )
+)
+@testing.with_requires("scipy")
+class TestFourierGaussian:
+
+    def _test_real_nd(self, xp, scp, x, real_axis):
+        a = scp.fft.rfft(x, axis=real_axis)
+        # complex-valued FFTs on all other axes
+        complex_axes = tuple([ax for ax in range(x.ndim) if ax != real_axis])
+        if complex_axes:
+            a = scp.fft.fftn(a, axes=complex_axes)
+
+        a = scp.ndimage.fourier_gaussian(
+            a, self.sigma, n=x.shape[real_axis], axis=real_axis)
+
+        if complex_axes:
+            a = scp.fft.ifftn(a, axes=complex_axes)
+        a = scp.fft.irfft(a, axis=real_axis)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+    @testing.with_requires("scipy>=1.4.0")
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_real_fft_axis0(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        if x.dtype.kind == 'c':
+            # skip: can't use rfft on complex-valued x
+            return x
+        return self._test_real_nd(xp, scp, x, 0)
+
+    @testing.with_requires("scipy>=1.4.0")
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_real_fft_axis1(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        if x.dtype.kind == 'c' or x.ndim < 2:
+            # skip: can't use rfft along axis 1 on complex-valued x or 1d x
+            return x
+        return self._test_real_nd(xp, scp, x, 1)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_complex_fft(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        a = scp.fftpack.fftn(x)
+        a = scp.ndimage.fourier_gaussian(a, self.sigma)
+        a = scp.fftpack.ifftn(a)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_complex_fft_with_output(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        a = scp.fftpack.fftn(x)
+        scp.ndimage.fourier_gaussian(a, self.sigma, output=a)
+        a = scp.fftpack.ifftn(a)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+
+@testing.parameterize(
+    *(
+        testing.product(
+            {
+                "shape": [(32, 16), (31, 15)],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "size": [1, (5, 5.3), (3, 5)],
+            }
+        )
+        + testing.product(
+            {
+                "shape": [(5, 16, 7), ],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "size": [3, (1, 2.5, 3)],
+            }
+        )
+        + testing.product(
+            {
+                "shape": [(15, ), ],
+                "dtype": [numpy.float32, numpy.float64, numpy.complex64,
+                          numpy.complex128],
+                "size": [8.5, (5,)],
+            }
+        )
+    )
+)
+@testing.with_requires("scipy")
+class TestFourierUniform:
+
+    def _test_real_nd(self, xp, scp, x, real_axis):
+        a = scp.fft.rfft(x, axis=real_axis)
+        # complex-valued FFTs on all other axes
+        complex_axes = tuple([ax for ax in range(x.ndim) if ax != real_axis])
+        if complex_axes:
+            a = scp.fft.fftn(a, axes=complex_axes)
+
+        a = scp.ndimage.fourier_uniform(
+            a, self.size, n=x.shape[real_axis], axis=real_axis)
+
+        if complex_axes:
+            a = scp.fft.ifftn(a, axes=complex_axes)
+        a = scp.fft.irfft(a, axis=real_axis)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+    @testing.with_requires("scipy>=1.4.0")
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_real_fft_axis0(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        if x.dtype.kind == 'c':
+            # skip: can't use rfft on complex-valued x
+            return x
+        return self._test_real_nd(xp, scp, x, 0)
+
+    @testing.with_requires("scipy>=1.4.0")
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_real_fft_axis1(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        if x.dtype.kind == 'c' or x.ndim < 2:
+            # skip: can't use rfft along axis 1 on complex-valued x or 1d x
+            return x
+        return self._test_real_nd(xp, scp, x, 1)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_complex_fft(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        a = scp.fftpack.fftn(x)
+        a = scp.ndimage.fourier_uniform(a, self.size)
+        a = scp.fftpack.ifftn(a)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name="scp")
+    def test_complex_fft_with_output(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.dtype)
+        a = scp.fftpack.fftn(x)
+        scp.ndimage.fourier_uniform(a, self.size, output=a)
+        a = scp.fftpack.ifftn(a)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+
+@testing.parameterize(
+    *(
+        testing.product(
+            {
+                'shape': [(32, 16), (31, 15)],
+                'size': [1, (5, 5), (3, 5)],
+            }
+        )
+        + testing.product(
+            {
+                'shape': [(5, 16, 7)],
+                'size': [3, (1, 2, 4)],
+            }
+        )
+        + testing.product(
+            {
+                'shape': [(15, ), ],
+                'size': [8, (5,)],
+            }
+        )
+    )
+)
+@testing.with_requires('scipy')
+class TestFourierEllipsoid():
+    def _test_real_nd(self, xp, scp, x, real_axis):
+        if x.ndim == 1 and scipy_version < '1.5.3':
+            # 1D case gives an incorrect result in SciPy < 1.5.3
+            pytest.skip('scipy version to old')
+
+        a = scp.fft.rfft(x, axis=real_axis)
+        # complex-valued FFTs on all other axes
+        complex_axes = tuple([ax for ax in range(x.ndim) if ax != real_axis])
+        if complex_axes:
+            a = scp.fft.fftn(a, axes=complex_axes)
+
+        a = scp.ndimage.fourier_ellipsoid(
+            a, self.size, n=x.shape[real_axis], axis=real_axis
+        )
+
+        if complex_axes:
+            a = scp.fft.ifftn(a, axes=complex_axes)
+        a = scp.fft.irfft(a, axis=real_axis)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_real_fft_axis0(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        return self._test_real_nd(xp, scp, x, 0)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_real_fft_axis1(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        if x.ndim < 2:
+            # skip: there is no axis=1 on 1d arrays
+            return x
+        return self._test_real_nd(xp, scp, x, 1)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_complex_fft(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        if x.ndim == 1 and scipy_version < '1.5.3':
+            # 1D case gives an incorrect result in SciPy < 1.5.3
+            pytest.skip('scipy version to old')
+        a = scp.fftpack.fftn(x)
+        a = scp.ndimage.fourier_ellipsoid(a, self.size)
+        a = scp.fftpack.ifftn(a)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_complex_fft_with_output(self, xp, scp, dtype):
+        x = testing.shaped_random(self.shape, xp, dtype)
+        if x.ndim == 1 and scipy_version < '1.5.3':
+            # 1D case gives an incorrect result in SciPy < 1.5.3
+            pytest.skip('scipy version to old')
+        a = scp.fftpack.fftn(x)
+        scp.ndimage.fourier_ellipsoid(a.copy(), self.size, output=a)
+        a = scp.fftpack.ifftn(a)
+        if not x.dtype.kind == 'c':
+            a = a.real
+        return xp.ascontiguousarray(a)
+
+
+@testing.with_requires('scipy')
+class TestFourierEllipsoidInvalid():
+
+    # SciPy < 1.5 raises ValueError instead of AxisError
+    @testing.with_requires('scipy>=1.5.0')
+    def test_0d_input(self):
+        for xp, scp in zip((numpy, cupy), (scipy, cupyx.scipy)):
+            with pytest.raises(numpy.AxisError):
+                scp.ndimage.fourier_ellipsoid(xp.asarray(5.0), size=2)
+        return
+
+    def test_4d_input(self):
+        # SciPy should raise here too because >3d isn't implemented, but
+        # as of 1.5.4, it does not.
+        shape = (4, 6, 8, 10)
+        for xp, scp in zip((cupy,), (cupyx.scipy,)):
+            with pytest.raises(NotImplementedError):
+                scp.ndimage.fourier_ellipsoid(xp.ones(shape), size=2)
+        return
+
+    # SciPy < 1.5 raises ValueError instead of AxisError
+    @testing.with_requires('scipy>=1.5.0')
+    def test_invalid_axis(self):
+        # SciPy should raise here too because >3d isn't implemented, but
+        # as of 1.5.4, it does not.
+        shape = (6, 8)
+        for xp, scp in zip((numpy, cupy), (scipy, cupyx.scipy)):
+            with pytest.raises(numpy.AxisError):
+                scp.ndimage.fourier_ellipsoid(xp.ones(shape), 2, axis=2)
+            with pytest.raises(numpy.AxisError):
+                scp.ndimage.fourier_ellipsoid(xp.ones(shape), 2, axis=-3)
+        return
+
+    def test_invalid_size(self):
+        # test size length mismatch
+        shape = (6, 8)
+        for xp, scp in zip((numpy, cupy), (scipy, cupyx.scipy)):
+            with pytest.raises(RuntimeError):
+                scp.ndimage.fourier_ellipsoid(xp.ones(shape), size=(2, 3, 4))
+            with pytest.raises(RuntimeError):
+                scp.ndimage.fourier_ellipsoid(xp.ones(shape), size=(4,))
+        return
diff --git a/tests/cupyx_tests/scipy_tests/ndimage_tests/test_interpolation.py b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_interpolation.py
new file mode 100644
index 0000000..a44e634
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_interpolation.py
@@ -0,0 +1,1029 @@
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupy import testing
+import cupyx.scipy.ndimage
+from cupyx.scipy.ndimage import _util
+
+try:
+    import scipy
+    import scipy.ndimage
+    scipy_version = numpy.lib.NumpyVersion(scipy.__version__)
+except ImportError:
+    pass
+
+try:
+    import cv2
+except ImportError:
+    pass
+
+# testing these modes can only be tested against SciPy >= 1.6.0+
+scipy16_modes = ['wrap', 'grid-wrap', 'reflect', 'grid-mirror',
+                 'grid-constant']
+# these modes are okay to test on older SciPy
+legacy_modes = ['constant', 'nearest', 'mirror']
+
+
+def _conditional_scipy_version_skip(mode, order):
+    if ((mode in scipy16_modes or (mode != 'mirror' and order > 1)) and
+            (scipy_version < '1.6.0')):
+        pytest.skip(
+            'SciPy >= 1.6.0 needed to test this mode/order combination.')
+
+
+@testing.parameterize(*testing.product({
+    'output': [None, numpy.float64, 'f', float, 'empty'],
+    'order': [0, 1, 2, 3, 4, 5],
+    'mode': ['constant', 'nearest', 'mirror'] + scipy16_modes,
+    'cval': [1.0],
+    'prefilter': [True],
+}))
+@testing.with_requires('scipy')
+class TestMapCoordinates:
+
+    _multiprocess_can_split = True
+
+    def _map_coordinates(self, xp, scp, a, coordinates):
+        _conditional_scipy_version_skip(self.mode, self.order)
+        map_coordinates = scp.ndimage.map_coordinates
+        if self.output == 'empty':
+            output = xp.empty(coordinates.shape[1:], dtype=a.dtype)
+            return_value = map_coordinates(a, coordinates, output, self.order,
+                                           self.mode, self.cval,
+                                           self.prefilter)
+            assert return_value is None or return_value is output
+            return output
+        else:
+            return map_coordinates(a, coordinates, self.output, self.order,
+                                   self.mode, self.cval, self.prefilter)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-4, scipy_name='scp')
+    def test_map_coordinates_float(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        coordinates = testing.shaped_random((a.ndim, 100), xp, dtype)
+        return self._map_coordinates(xp, scp, a, coordinates)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-4, scipy_name='scp')
+    @testing.with_requires('scipy>=1.6.0')
+    def test_map_coordinates_complex_float(self, xp, scp, dtype):
+        # promote output to a complex dtype
+        if self.output == numpy.float64:
+            self.output = numpy.complex128
+        elif self.output == float:
+            self.output = complex
+        elif self.output == 'f':
+            self.output = 'F'
+        a = testing.shaped_random((100, 100), xp, dtype)
+        coordinates = testing.shaped_random((a.ndim, 100), xp, xp.float64)
+        return self._map_coordinates(xp, scp, a, coordinates)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-4, scipy_name='scp')
+    def test_map_coordinates_fortran_order(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        coordinates = testing.shaped_random((a.ndim, 100), xp, dtype)
+        a = xp.asfortranarray(a)
+        coordinates = xp.asfortranarray(coordinates)
+        return self._map_coordinates(xp, scp, a, coordinates)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-4, scipy_name='scp')
+    def test_map_coordinates_float_nd_coords(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        coordinates = testing.shaped_random((a.ndim, 10, 10), xp, dtype,
+                                            scale=99.0)
+        return self._map_coordinates(xp, scp, a, coordinates)
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-4, scipy_name='scp')
+    def test_map_coordinates_int(self, xp, scp, dtype):
+        if numpy.lib.NumpyVersion(scipy.__version__) < '1.0.0':
+            if dtype in (numpy.dtype('l'), numpy.dtype('q')):
+                dtype = numpy.int64
+            elif dtype in (numpy.dtype('L'), numpy.dtype('Q')):
+                dtype = numpy.uint64
+
+        a = testing.shaped_random((100, 100), xp, dtype)
+        coordinates = testing.shaped_random((a.ndim, 100), xp, dtype)
+        out = self._map_coordinates(xp, scp, a, coordinates)
+        float_out = self._map_coordinates(xp, scp, a.astype(xp.float64),
+                                          coordinates) % 1
+        half = xp.full_like(float_out, 0.5)
+        out[xp.isclose(float_out, half, atol=1e-5)] = 0
+        return out
+
+
+@testing.parameterize(*testing.product({
+    'order': [0, 1, 2, 3, 4, 5],
+    'mode': ['constant', 'nearest', 'mirror'] + scipy16_modes,
+}))
+@testing.with_requires('scipy')
+class TestMapCoordinatesHalfInteger:
+
+    def _map_coordinates(self, xp, scp, a, coordinates):
+        _conditional_scipy_version_skip(self.mode, self.order)
+        map_coordinates = scp.ndimage.map_coordinates
+        return map_coordinates(a, coordinates, None, self.order, self.mode)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-4, scipy_name='scp')
+    def test_map_coordinates_float(self, xp, scp, dtype):
+        # Half integer coordinate rounding test case from:
+        # https://github.com/cupy/cupy/issues/4550
+        a = testing.shaped_arange((4, 3), xp, dtype)
+        coordinates = xp.array([[0.5, 2], [0.5, 1]])
+        return self._map_coordinates(xp, scp, a, coordinates)
+
+
+@testing.parameterize(*testing.product({
+    'matrix_shape': [(2,), (2, 2), (2, 3), (3, 3)],
+    'offset': [0.3, [-1.3, 1.3]],
+    'output_shape': [None],
+    'output': [None, numpy.float64, 'empty'],
+    'order': [0, 1, 2, 3, 4, 5],
+    'mode': legacy_modes + scipy16_modes,
+    'cval': [1.0],
+    'prefilter': [False, True],
+}))
+@testing.with_requires('scipy')
+class TestAffineTransform:
+
+    _multiprocess_can_split = True
+
+    def _affine_transform(self, xp, scp, a, matrix):
+        _conditional_scipy_version_skip(self.mode, self.order)
+        ver = numpy.lib.NumpyVersion(scipy.__version__)
+        if ver < '1.0.0' and matrix.ndim == 2 and matrix.shape[1] == 3:
+            return xp.empty(0)
+
+        if matrix.shape == (3, 3):
+            matrix[-1, 0:-1] = 0
+            matrix[-1, -1] = 1
+        affine_transform = scp.ndimage.affine_transform
+        if self.output == 'empty':
+            output = xp.empty_like(a)
+            return_value = affine_transform(a, matrix, self.offset,
+                                            self.output_shape, output,
+                                            self.order, self.mode, self.cval,
+                                            self.prefilter)
+            assert return_value is None or return_value is output
+            return output
+        else:
+            return affine_transform(a, matrix, self.offset, self.output_shape,
+                                    self.output, self.order, self.mode,
+                                    self.cval, self.prefilter)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_affine_transform_float(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        matrix = testing.shaped_random(self.matrix_shape, xp, dtype)
+        return self._affine_transform(xp, scp, a, matrix)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(rtol=1e-6, atol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy>=1.6.0')
+    def test_affine_transform_complex_float(self, xp, scp, dtype):
+        if self.output == numpy.float64:
+            # must promote output to a complex dtype
+            self.output = numpy.complex128
+        a = testing.shaped_random((100, 100), xp, dtype)
+        matrix = testing.shaped_random(self.matrix_shape, xp, xp.float64)
+        return self._affine_transform(xp, scp, a, matrix)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_affine_transform_fortran_order(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        a = xp.asfortranarray(a)
+        matrix = testing.shaped_random(self.matrix_shape, xp, dtype)
+        matrix = xp.asfortranarray(matrix)
+        return self._affine_transform(xp, scp, a, matrix)
+
+    def _hip_skip_invalid_condition(self):
+        if (runtime.is_hip
+                and self.matrix_shape in [(2,), (2, 2)]
+                and self.order in [2, 3, 4, 5]
+                and self.output in [None, 'empty']
+                and self.prefilter):
+            pytest.xfail('ROCm/HIP may have a bug')
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_affine_transform_int(self, xp, scp, dtype):
+        self._hip_skip_invalid_condition()
+
+        if numpy.lib.NumpyVersion(scipy.__version__) < '1.0.0':
+            if dtype in (numpy.dtype('l'), numpy.dtype('q')):
+                dtype = numpy.int64
+            elif dtype in (numpy.dtype('L'), numpy.dtype('Q')):
+                dtype = numpy.uint64
+
+        a = testing.shaped_random((100, 100), xp, dtype)
+        matrix = testing.shaped_random(self.matrix_shape, xp, dtype)
+        out = self._affine_transform(xp, scp, a, matrix)
+        float_out = self._affine_transform(xp, scp, a.astype(xp.float64),
+                                           matrix) % 1
+        half = xp.full_like(float_out, 0.5)
+        out[xp.isclose(float_out, half, atol=1e-5)] = 0
+        return out
+
+
+@testing.with_requires('scipy')
+class TestAffineExceptions:
+
+    def test_invalid_affine_ndim(self):
+        ndimage_modules = (scipy.ndimage, cupyx.scipy.ndimage)
+        for (xp, ndi) in zip((numpy, cupy), ndimage_modules):
+            x = xp.ones((8, 8, 8))
+            with pytest.raises(RuntimeError):
+                ndi.affine_transform(x, xp.ones((3, 3, 3)))
+            with pytest.raises(RuntimeError):
+                ndi.affine_transform(x, xp.ones(()))
+
+    def test_invalid_affine_shape(self):
+        ndimage_modules = (scipy.ndimage, cupyx.scipy.ndimage)
+        for (xp, ndi) in zip((numpy, cupy), ndimage_modules):
+            x = xp.ones((8, 8, 8))
+            with pytest.raises(RuntimeError):
+                ndi.affine_transform(x, xp.ones((0, 3)))
+            with pytest.raises(RuntimeError):
+                ndi.affine_transform(x, xp.eye(x.ndim - 1))
+            with pytest.raises(RuntimeError):
+                ndi.affine_transform(x, xp.eye(x.ndim + 2))
+            with pytest.raises(RuntimeError):
+                ndi.affine_transform(x, xp.eye(x.ndim)[:, :-1])
+
+    @testing.with_requires('scipy>=1.6.0')
+    def test_invalid_output_dtype(self):
+        # real output array with complex input is not allowed
+        ndimage_modules = (scipy.ndimage, cupyx.scipy.ndimage)
+        for (xp, ndi) in zip((numpy, cupy), ndimage_modules):
+            x = xp.ones((8, 8, 8), dtype=numpy.complex128)
+            output = xp.ones_like(x, dtype=x.real.dtype)
+            with pytest.raises(RuntimeError):
+                ndi.affine_transform(x, xp.ones((0, 3)), output=output)
+
+    def test_invalid_texture_arguments(self):
+        if runtime.is_hip:
+            pytest.skip('texture memory not supported yet')
+
+        aft = cupyx.scipy.ndimage.affine_transform
+        x = [cupy.ones((8, ) * n, dtype=cupy.float32) for n in range(1, 5)]
+
+        # (ndim < 2) and (ndim > 3) must fail
+        for i in [0, 3]:
+            with pytest.raises(ValueError):
+                aft(x[i], cupy.eye(i + 1, dtype=cupy.float32),
+                    texture_memory=True)
+        # wrong input dtype
+        for dt in [cupy.float16, cupy.float64, cupy.int32, cupy.int64]:
+            with pytest.raises(ValueError):
+                aft(cupy.ones((8, 8), dtype=dt),
+                    cupy.eye(3, dtype=cupy.float32), texture_memory=True)
+        # wrong matrix shape
+        for i in range(len(x)):
+            with pytest.raises(ValueError):
+                aft(x[i], cupy.eye(i, dtype=cupy.float32),
+                    texture_memory=True)
+        # wrong output
+        with pytest.raises(ValueError):
+            aft(x[2], cupy.eye(3, dtype=cupy.float32), output='wrong',
+                texture_memory=True)
+        # wrong mode
+        for m in ['mirror', 'reflect', 'wrap', 'grid-mirror',
+                  'grid-wrap', 'grid-constant', 'opencv']:
+            with pytest.raises(ValueError):
+                aft(x[2], cupy.eye(3, dtype=cupy.float32), mode=m,
+                    texture_memory=True)
+        # non matching output_shape and output's shape
+        with pytest.raises(ValueError):
+            output = cupy.empty((7, 7, 7), dtype=cupy.float32)
+            aft(x[2], cupy.eye(3, dtype=cupy.float32), output_shape=(8, 8, 8),
+                output=output, texture_memory=True)
+        # non matching output_shape and input shape
+        with pytest.raises(ValueError):
+            aft(x[2], cupy.eye(3, dtype=cupy.float32), output_shape=(7, 7, 7),
+                texture_memory=True)
+
+
+@pytest.mark.skipif(runtime.is_hip, reason='texture memory not supported yet')
+@testing.parameterize(*testing.product({
+    'output': [None, numpy.float32, 'empty'],
+    'output_shape': [None, 10],
+    'order': [0, 1],
+    'mode': ['constant', 'nearest'],
+    'shape': [(100, 100), (10, 20), (10, 10, 10), (10, 20, 30)],
+    'theta': [0, 90, 180, 270]
+}))
+@testing.with_requires('scipy')
+class TestAffineTransformTextureMemory:
+
+    _multiprocess_can_split = True
+
+    def _2d_rotation_matrix(self, theta, rotation_center):
+        import scipy.special
+        c, s = scipy.special.cosdg(theta), scipy.special.sindg(theta)
+        m = numpy.array([
+            [1, 0, rotation_center[0]],
+            [0, 1, rotation_center[1]],
+            [0, 0, 1]
+        ], numpy.float32)
+        m = numpy.dot(m, numpy.array([
+            [c, -s, 0],
+            [s, c, 0],
+            [0, 0, 1]
+        ], numpy.float32))
+        m = numpy.dot(m, numpy.array([
+            [1, 0, -rotation_center[0]],
+            [0, 1, -rotation_center[1]],
+            [0, 0, 1]
+        ], numpy.float32))
+        return m
+
+    def _3d_rotation_matrix(self, theta, rotation_center):
+        c, s = scipy.special.cosdg(theta), scipy.special.sindg(theta)
+        m = numpy.array([
+            [1, 0, 0, rotation_center[0]],
+            [0, 1, 0, rotation_center[1]],
+            [0, 0, 1, rotation_center[2]],
+            [0, 0, 0, 1]
+        ], numpy.float32)
+        m = numpy.dot(m, numpy.array([
+            [1, 0, 0, 0],
+            [0, c, -s, 0],
+            [0, s, c, 0],
+            [0, 0, 0, 1]
+        ], numpy.float32))
+        m = numpy.dot(m, numpy.array([
+            [1, 0, 0, -rotation_center[0]],
+            [0, 1, 0, -rotation_center[1]],
+            [0, 0, 1, -rotation_center[2]],
+            [0, 0, 0, 1]
+        ], numpy.float32))
+        return m
+
+    @testing.numpy_cupy_allclose(atol=0.1, scipy_name='scp')
+    def test_affine_transform_texture_memory(self, xp, scp):
+        a = xp.ones(self.shape, dtype=xp.float32)
+        center = numpy.divide(numpy.subtract(self.shape, 1), 2)
+
+        if len(self.shape) == 2:
+            matrix = self._2d_rotation_matrix(self.theta, center)
+        elif len(self.shape) == 3:
+            matrix = self._3d_rotation_matrix(self.theta, center)
+        else:
+            return pytest.xfail('Unsupported shape')
+
+        if self.output == 'empty':
+            output = xp.empty(self.shape, dtype=xp.float32)
+            if self.output_shape:
+                return pytest.skip('This combination is tested in '
+                                   'test_invalid_texture_arguments')
+        else:
+            output = self.output
+
+        if self.output_shape:
+            output_shape = (self.output_shape, ) * len(self.shape)
+        else:
+            output_shape = self.output_shape
+
+        if xp == cupy:
+            m = cupyx.scipy.ndimage.affine_transform
+            matrix = cupy.array(matrix)
+            return m(a, matrix, output_shape=output_shape,
+                     output=output, order=self.order,
+                     mode=self.mode, texture_memory=True)
+        else:
+            m = scp.ndimage.affine_transform
+            return m(a, matrix, output_shape=output_shape,
+                     output=output, order=self.order,
+                     mode=self.mode)
+
+
+@testing.with_requires('opencv-python')
+class TestAffineTransformOpenCV:
+
+    _multiprocess_can_split = True
+
+    @testing.for_float_dtypes(no_float16=True)
+    # The precision of cv2.warpAffine is not good because it uses fixed-point
+    # arithmetic.
+    @testing.numpy_cupy_allclose(atol=0.2)
+    def test_affine_transform_opencv(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        matrix = testing.shaped_random((2, 3), xp, dtype)
+        if xp == cupy:
+            return cupyx.scipy.ndimage.affine_transform(a, matrix, order=1,
+                                                        mode='opencv')
+        else:
+            return cv2.warpAffine(a, matrix, (a.shape[1], a.shape[0]))
+
+
+@testing.parameterize(*(
+    testing.product({
+        'angle': [-10, 1000],
+        'axes': [(1, 0)],
+        'reshape': [False, True],
+        'output': [None, numpy.float64, 'empty'],
+        'order': [0, 1],
+        'mode': legacy_modes,
+        'cval': [1.0],
+        'prefilter': [True],
+    }) + testing.product({
+        'angle': [-15],
+        'axes': [(1, 0)],
+        'reshape': [False],
+        'output': [None],
+        'order': [0, 1, 3],
+        'mode': legacy_modes + scipy16_modes,
+        'cval': [1.0],
+        'prefilter': [True],
+    })
+))
+@testing.with_requires('scipy')
+class TestRotate:
+
+    _multiprocess_can_split = True
+
+    def _rotate(self, xp, scp, a):
+        _conditional_scipy_version_skip(self.mode, self.order)
+        rotate = scp.ndimage.rotate
+        if self.output == 'empty':
+            output = rotate(a, self.angle, self.axes,
+                            self.reshape, None, self.order,
+                            self.mode, self.cval, self.prefilter)
+            return_value = rotate(a, self.angle, self.axes,
+                                  self.reshape, output, self.order,
+                                  self.mode, self.cval, self.prefilter)
+            assert return_value is None or return_value is output
+            return output
+        else:
+            return rotate(a, self.angle, self.axes,
+                          self.reshape, self.output, self.order,
+                          self.mode, self.cval, self.prefilter)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_rotate_float(self, xp, scp, dtype):
+        a = testing.shaped_random((10, 10), xp, dtype)
+        return self._rotate(xp, scp, a)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy>=1.6.0')
+    def test_rotate_complex_float(self, xp, scp, dtype):
+        if self.output == numpy.float64:
+            self.output = numpy.complex128
+        a = testing.shaped_random((10, 10), xp, dtype)
+        return self._rotate(xp, scp, a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_rotate_fortran_order(self, xp, scp, dtype):
+        a = testing.shaped_random((10, 10), xp, dtype)
+        a = xp.asfortranarray(a)
+        return self._rotate(xp, scp, a)
+
+    def _hip_skip_invalid_condition(self):
+        if runtime.is_hip:
+            if (self.angle in [-10, 1000]
+                    and self.mode in ['constant', 'nearest', 'mirror']
+                    and self.output == numpy.float64
+                    and self.reshape):
+                pytest.xfail('ROCm/HIP may have a bug')
+            if (self.angle == -15
+                    and self.mode in [
+                        'nearest', 'grid-wrap', 'reflect', 'grid-mirror']
+                    and self.order == 3):
+                pytest.xfail('ROCm/HIP may have a bug')
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_rotate_int(self, xp, scp, dtype):
+        self._hip_skip_invalid_condition()
+
+        if numpy.lib.NumpyVersion(scipy.__version__) < '1.0.0':
+            if dtype in (numpy.dtype('l'), numpy.dtype('q')):
+                dtype = numpy.int64
+            elif dtype in (numpy.dtype('L'), numpy.dtype('Q')):
+                dtype = numpy.uint64
+
+        a = testing.shaped_random((10, 10), xp, dtype)
+        out = self._rotate(xp, scp, a)
+        float_out = self._rotate(xp, scp, a.astype(xp.float64)) % 1
+        half = xp.full_like(float_out, 0.5)
+        out[xp.isclose(float_out, half, atol=1e-5)] = 0
+        return out
+
+
+# Scipy older than 1.3.0 raises IndexError instead of ValueError
+@testing.with_requires('scipy>=1.3.0')
+class TestRotateExceptions:
+
+    def test_rotate_invalid_plane(self):
+        ndimage_modules = (scipy.ndimage, cupyx.scipy.ndimage)
+        for (xp, ndi) in zip((numpy, cupy), ndimage_modules):
+            x = xp.ones((8, 8, 8))
+            angle = 15
+            with pytest.raises(ValueError):
+                ndi.rotate(x, angle, [0, x.ndim])
+            with pytest.raises(ValueError):
+                ndi.rotate(x, angle, [-(x.ndim + 1), 1])
+
+
+@testing.parameterize(
+    {'axes': (-1, -2)},
+    {'axes': (0, 1)},
+    {'axes': (2, 0)},
+    {'axes': (-2, 2)},
+)
+@testing.with_requires('scipy')
+class TestRotateAxes:
+
+    _multiprocess_can_split = True
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_rotate_axes(self, xp, scp, dtype):
+        a = testing.shaped_random((10, 10, 10), xp, dtype)
+        rotate = scp.ndimage.rotate
+        return rotate(a, 1, self.axes, order=1)
+
+
+@testing.with_requires('opencv-python')
+class TestRotateOpenCV:
+
+    _multiprocess_can_split = True
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=0.3)
+    def test_rotate_opencv(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        if xp == cupy:
+            return cupyx.scipy.ndimage.rotate(a, 10, reshape=False,
+                                              order=1, mode='opencv')
+        else:
+            matrix = cv2.getRotationMatrix2D((49.5, 49.5), 10, 1)
+            return cv2.warpAffine(a, matrix, (a.shape[1], a.shape[0]))
+
+
+@testing.parameterize(*(
+    testing.product({
+        'shift': [0.1, -10, (5, -5)],
+        'output': [None, numpy.float64, 'empty'],
+        'order': [0, 1, 3],
+        'mode': legacy_modes + scipy16_modes,
+        'cval': [1.0],
+        'prefilter': [True],
+    }) + testing.product({
+        'shift': [0.1, ],
+        'output': [None, numpy.float64, 'empty'],
+        'order': [0, 1, 3],
+        'mode': ['constant', ],
+        'cval': [cupy.nan, cupy.inf, -cupy.inf],
+        'prefilter': [True],
+    })
+))
+@testing.with_requires('scipy')
+class TestShift:
+
+    _multiprocess_can_split = True
+
+    def _shift(self, xp, scp, a):
+        shift = scp.ndimage.shift
+        _conditional_scipy_version_skip(self.mode, self.order)
+        if self.output == 'empty':
+            output = xp.empty_like(a)
+            return_value = shift(a, self.shift, output, self.order,
+                                 self.mode, self.cval, self.prefilter)
+            assert return_value is None or return_value is output
+            return output
+        else:
+            return shift(a, self.shift, self.output, self.order,
+                         self.mode, self.cval, self.prefilter)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_shift_float(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return self._shift(xp, scp, a)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy>=1.6.0')
+    def test_shift_complex_float(self, xp, scp, dtype):
+        if self.output == numpy.float64:
+            self.output = numpy.complex128
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return self._shift(xp, scp, a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_shift_fortran_order(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        a = xp.asfortranarray(a)
+        return self._shift(xp, scp, a)
+
+    def _hip_skip_invalid_condition(self):
+        if (runtime.is_hip
+                and self.cval == 1.0
+                and self.order == 3
+                and self.output in [None, 'empty']
+                and self.shift == 0.1):
+            pytest.xfail('ROCm/HIP may have a bug')
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_shift_int(self, xp, scp, dtype):
+        self._hip_skip_invalid_condition()
+
+        if self.mode == 'constant' and not xp.isfinite(self.cval):
+            if self.output is None or self.output == 'empty':
+                # Non-finite cval with integer output array is not supported
+                # CuPy exception is tested in TestInterpolationInvalidCval
+                return xp.asarray([])
+
+        if numpy.lib.NumpyVersion(scipy.__version__) < '1.0.0':
+            if dtype in (numpy.dtype('l'), numpy.dtype('q')):
+                dtype = numpy.int64
+            elif dtype in (numpy.dtype('L'), numpy.dtype('Q')):
+                dtype = numpy.uint64
+
+        a = testing.shaped_random((100, 100), xp, dtype)
+        out = self._shift(xp, scp, a)
+        float_out = self._shift(xp, scp, a.astype(xp.float64)) % 1
+        half = xp.full_like(float_out, 0.5)
+        out[xp.isclose(float_out, half, atol=1e-5)] = 0
+        return out
+
+
+# non-finite cval with integer valued output is not allowed for CuPy
+@testing.parameterize(*testing.product({
+    'output': [None, numpy.float64, numpy.int32, 'empty'],
+    'order': [0, 1],
+    'mode': ['constant', 'nearest'],
+    'cval': [cupy.nan, cupy.inf, -cupy.inf],
+}))
+class TestInterpolationInvalidCval:
+
+    def _prep_output(self, a):
+        if self.output == 'empty':
+            return cupy.zeros_like(a)
+        return self.output
+
+    @testing.for_int_dtypes(no_bool=True)
+    def test_shift(self, dtype):
+        a = cupy.ones((32,), dtype=dtype)
+        shift = cupyx.scipy.ndimage.shift
+        output = self._prep_output(a)
+        if _util._is_integer_output(output, a) and self.mode == 'constant':
+            with pytest.raises(NotImplementedError):
+                shift(a, 1, output=output, order=self.order, mode=self.mode,
+                      cval=self.cval)
+        else:
+            shift(a, 1, output=output, order=self.order, mode=self.mode,
+                  cval=self.cval)
+
+    @testing.for_int_dtypes(no_bool=True)
+    def test_zoom(self, dtype):
+        a = cupy.ones((32,), dtype=dtype)
+        zoom = cupyx.scipy.ndimage.zoom
+        output = self._prep_output(a)
+        if _util._is_integer_output(output, a) and self.mode == 'constant':
+            with pytest.raises(NotImplementedError):
+                # zoom of 1.0 to keep same shape
+                zoom(a, 1, output=output, order=self.order, mode=self.mode,
+                     cval=self.cval)
+        else:
+            zoom(a, 1, output=output, order=self.order, mode=self.mode,
+                 cval=self.cval)
+
+    @testing.for_int_dtypes(no_bool=True)
+    def test_rotate(self, dtype):
+        a = cupy.ones((16, 16), dtype=dtype)
+        rotate = cupyx.scipy.ndimage.rotate
+        output = self._prep_output(a)
+        if _util._is_integer_output(output, a) and self.mode == 'constant':
+            with pytest.raises(NotImplementedError):
+                # rotate by 0 to keep same shape
+                rotate(a, 0, output=output, order=self.order, mode=self.mode,
+                       cval=self.cval)
+        else:
+            rotate(a, 0, output=output, order=self.order, mode=self.mode,
+                   cval=self.cval)
+
+    @testing.for_int_dtypes(no_bool=True)
+    def test_affine(self, dtype):
+        a = cupy.ones((16, 16), dtype=dtype)
+        affine = cupy.eye(2)
+        affine_transform = cupyx.scipy.ndimage.affine_transform
+        output = self._prep_output(a)
+        if _util._is_integer_output(output, a) and self.mode == 'constant':
+            with pytest.raises(NotImplementedError):
+                affine_transform(a, affine, output=output, order=self.order,
+                                 mode=self.mode, cval=self.cval)
+        else:
+            affine_transform(a, affine, output=output, order=self.order,
+                             mode=self.mode, cval=self.cval)
+
+    @testing.for_int_dtypes(no_bool=True)
+    def test_map_coordinates(self, dtype):
+        a = cupy.ones((32,), dtype=dtype)
+        coords = cupy.arange(32)[cupy.newaxis, :] + 2.5
+        map_coordinates = cupyx.scipy.ndimage.map_coordinates
+        output = self._prep_output(a)
+        if _util._is_integer_output(output, a) and self.mode == 'constant':
+            with pytest.raises(NotImplementedError):
+                map_coordinates(a, coords, output=output, order=self.order,
+                                mode=self.mode, cval=self.cval)
+        else:
+            map_coordinates(a, coords, output=output, order=self.order,
+                            mode=self.mode, cval=self.cval)
+
+
+@testing.with_requires('opencv-python')
+class TestShiftOpenCV:
+
+    _multiprocess_can_split = True
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=0.2)
+    def test_shift_opencv(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        shift = testing.shaped_random((2,), xp, dtype)
+        if xp == cupy:
+            return cupyx.scipy.ndimage.shift(a, shift, order=1,
+                                             mode='opencv')
+        else:
+            matrix = numpy.array([[1, 0, shift[1]], [0, 1, shift[0]]])
+            return cv2.warpAffine(a, matrix, (a.shape[1], a.shape[0]))
+
+
+@testing.parameterize(*testing.product({
+    'zoom': [0.1, 10, (0.1, 10)],
+    'output': [None, numpy.float64, 'empty'],
+    'order': [0, 1],
+    'mode': legacy_modes,
+    'cval': [1.0],
+    'prefilter': [True],
+}))
+@testing.with_requires('scipy')
+class TestZoom:
+
+    _multiprocess_can_split = True
+
+    def _zoom(self, xp, scp, a):
+        _conditional_scipy_version_skip(self.mode, self.order)
+        zoom = scp.ndimage.zoom
+        if self.output == 'empty':
+            output = zoom(a, self.zoom, None, self.order,
+                          self.mode, self.cval, self.prefilter)
+            return_value = zoom(a, self.zoom, output, self.order,
+                                self.mode, self.cval, self.prefilter)
+            assert return_value is None or return_value is output
+            return output
+        else:
+            return zoom(a, self.zoom, self.output, self.order,
+                        self.mode, self.cval, self.prefilter)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_zoom_float(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return self._zoom(xp, scp, a)
+
+    @testing.for_complex_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy>=1.6.0')
+    def test_zoom_complex_float(self, xp, scp, dtype):
+        if self.output == numpy.float64:
+            self.output = numpy.complex128
+        a = testing.shaped_random((100, 100), xp, dtype)
+        return self._zoom(xp, scp, a)
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_zoom_fortran_order(self, xp, scp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        a = xp.asfortranarray(a)
+        return self._zoom(xp, scp, a)
+
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_zoom_int(self, xp, scp, dtype):
+        if numpy.lib.NumpyVersion(scipy.__version__) < '1.0.0':
+            if dtype in (numpy.dtype('l'), numpy.dtype('q')):
+                dtype = numpy.int64
+            elif dtype in (numpy.dtype('L'), numpy.dtype('Q')):
+                dtype = numpy.uint64
+
+        a = testing.shaped_random((100, 100), xp, dtype)
+        out = self._zoom(xp, scp, a)
+        float_out = self._zoom(xp, scp, a.astype(xp.float64)) % 1
+        half = xp.full_like(float_out, 0.5)
+        out[xp.isclose(float_out, half, atol=1e-5)] = 0
+        return out
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(2, 3), (4, 4)],
+    'zoom': [(1, 1), (3, 5), (8, 2), (8, 8)],
+    'mode': ['nearest', 'reflect', 'mirror', 'grid-wrap', 'grid-constant'],
+}))
+class TestZoomOrder0IntegerGrid():
+
+    def test_zoom_grid_by_int_order0(self):
+        # When grid_mode is True,  order 0 zoom should be the same as
+        # replication via a Kronecker product. The only exceptions to this are
+        # the non-grid modes 'constant' and 'wrap'.
+        size = numpy.prod(self.shape)
+        x = cupy.arange(size, dtype=float).reshape(self.shape)
+        testing.assert_array_almost_equal(
+            cupyx.scipy.ndimage.zoom(
+                x, self.zoom, order=0, mode=self.mode, grid_mode=True
+            ),
+            cupy.kron(x, cupy.ones(self.zoom)),
+        )
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(5, 5, 2)],
+    'zoom': [(2, 2, 0.5)],  # selected to give output.shape[-1] == 1
+    'mode': legacy_modes + scipy16_modes,
+    'order': [0, 1, 2, 3, 4, 5],
+    'grid_mode': [False, True],
+}))
+class TestZoomOutputSize1():
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy>=1.6.0')
+    def test_zoom_output_size1(self, xp, scp, dtype):
+        x = xp.zeros(self.shape, dtype=dtype)
+        x[1, 1, 1] = 1
+        return scp.ndimage.zoom(x, self.zoom, order=self.order, mode=self.mode,
+                                grid_mode=self.grid_mode)
+
+
+@testing.parameterize(
+    {'zoom': 3},
+    {'zoom': 0.3},
+)
+@testing.with_requires('opencv-python')
+class TestZoomOpenCV:
+
+    _multiprocess_can_split = True
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-4)
+    def test_zoom_opencv(self, xp, dtype):
+        a = testing.shaped_random((100, 100), xp, dtype)
+        if xp == cupy:
+            return cupyx.scipy.ndimage.zoom(a, self.zoom, order=1,
+                                            mode='opencv')
+        else:
+            output_shape = numpy.rint(numpy.multiply(a.shape, self.zoom))
+            return cv2.resize(a, tuple(output_shape.astype(int)))
+
+    @testing.for_float_dtypes(no_float16=True)
+    @testing.numpy_cupy_allclose(atol=1e-4)
+    def test_zoom_opencv_output_size1(self, xp, dtype):
+        a = testing.shaped_random((100, 100, 1), xp, dtype)
+        if xp == cupy:
+            return cupyx.scipy.ndimage.zoom(a, (self.zoom, self.zoom, 1),
+                                            order=1, mode='opencv')
+        else:
+            output_shape = numpy.rint(numpy.multiply(a.shape[:2], self.zoom))
+            return numpy.expand_dims(
+                cv2.resize(a, tuple(output_shape.astype(int))), axis=-1)
+
+
+@testing.parameterize(*testing.product({
+    # these 3 modes have analytical spline boundary conditions
+    'mode': ['mirror', 'grid-wrap', 'reflect'],
+    'order': [0, 1, 2, 3, 4, 5],
+    'dtype': [numpy.uint8, numpy.float64],
+    'output': [numpy.float64, numpy.float32],
+    'axis': [0, 1, 2, -1],
+}))
+@testing.with_requires('scipy')
+class TestSplineFilter1d:
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_spline_filter1d(self, xp, scp):
+        if self.mode == 'grid-wrap' and scipy_version < '1.6.0':
+            pytest.skip('testing mode grid-wrap requires scipy >= 1.6.0')
+        x = testing.shaped_random((16, 12, 11), dtype=self.dtype, xp=xp)
+        return scp.ndimage.spline_filter1d(x, order=self.order, axis=self.axis,
+                                           output=self.output, mode=self.mode)
+
+    @testing.for_CF_orders(name='array_order')
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_spline_filter1d_output(self, xp, scp, array_order):
+        if self.mode == 'grid-wrap' and scipy_version < '1.6.0':
+            pytest.skip('testing mode grid-wrap requires scipy >= 1.6.0')
+        x = testing.shaped_random((16, 12, 11), dtype=self.dtype, xp=xp,
+                                  order=array_order)
+        output = xp.empty(x.shape, dtype=self.output, order=array_order)
+        scp.ndimage.spline_filter1d(x, order=self.order, axis=self.axis,
+                                    output=output, mode=self.mode)
+        return output
+
+
+# See #5537
+@testing.slow
+@testing.with_requires('scipy')
+class TestSplineFilter1dLargeArray:
+
+    @pytest.mark.parametrize('mode', ['mirror', 'grid-wrap', 'reflect'])
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_spline_filter1d_large_array(self, xp, scp, mode):
+        if mode == 'grid-wrap' and scipy_version < '1.6.0':
+            pytest.skip('testing mode grid-wrap requires scipy >= 1.6.0')
+        # Test input that cannot be indexed by int32 in bytes, i.e.
+        # x.size * dtype.itemsize >= 2 ** 31
+        x = xp.empty((2**20, 2**9), numpy.float32)
+        # To speed-up, do not generate many random numbers
+        x = x[::2**15, :]
+        x[...] = testing.shaped_random(
+            (2**5, 2**9), dtype=numpy.float32, xp=xp)
+        return scp.ndimage.spline_filter1d(x, mode=mode)
+
+
+@testing.parameterize(*testing.product({
+    # these 3 modes have analytical spline boundary conditions
+    'mode': ['mirror', 'grid-wrap', 'reflect'],
+    'order': [0, 1, 2, 3, 4, 5],
+    'dtype': [numpy.uint8, numpy.float64],
+    'output': [numpy.float64, numpy.float32],
+}))
+@testing.with_requires('scipy')
+class TestSplineFilter:
+    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-4, scipy_name='scp')
+    def test_spline_filter(self, xp, scp):
+        if self.mode == 'grid-wrap' and scipy_version < '1.6.0':
+            pytest.skip('testing mode grid-wrap requires scipy >= 1.6.0')
+        x = testing.shaped_random((16, 12, 11), dtype=self.dtype, xp=xp)
+        if self.order < 2:
+            with pytest.raises(RuntimeError):
+                scp.ndimage.spline_filter(x, order=self.order,
+                                          output=self.output, mode=self.mode)
+            return xp.asarray([])
+        return scp.ndimage.spline_filter(x, order=self.order,
+                                         output=self.output, mode=self.mode)
+
+    @testing.for_CF_orders(name='array_order')
+    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-4, scipy_name='scp')
+    def test_spline_filter_with_output(self, xp, scp, array_order):
+        if self.mode == 'grid-wrap' and scipy_version < '1.6.0':
+            pytest.skip('testing mode grid-wrap requires scipy >= 1.6.0')
+        x = testing.shaped_random((16, 12, 11), dtype=self.dtype, xp=xp,
+                                  order=array_order)
+        output = xp.empty(x.shape, dtype=self.output, order=array_order)
+        if self.order < 2:
+            with pytest.raises(RuntimeError):
+                scp.ndimage.spline_filter(x, order=self.order, output=output,
+                                          mode=self.mode)
+            return xp.asarray([])
+        scp.ndimage.spline_filter(x, order=self.order, output=output,
+                                  mode=self.mode)
+        return output
+
+
+@testing.parameterize(*testing.product({
+    'mode': ['mirror', 'wrap', 'reflect'],
+    'order': [2, 3, 4, 5],
+    'dtype': [numpy.complex64, numpy.complex128],
+    'output': [numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestSplineFilterComplex:
+
+    @testing.with_requires('scipy>=1.6')
+    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-4, scipy_name='scp')
+    def test_spline_filter_complex(self, xp, scp):
+        x = testing.shaped_random((16, 12, 11), dtype=self.dtype, xp=xp)
+        return scp.ndimage.spline_filter(x, order=self.order,
+                                         output=self.output, mode=self.mode)
+
+    # the following test case is for SciPy versions lacking complex support
+    @testing.with_requires('scipy<1.6')
+    def test_spline_filter_complex2(self):
+        if self.mode == 'wrap':
+            pytest.skip('mode cannot be tested against SciPy < 1.6')
+        cpu_func = scipy.ndimage.spline_filter
+        gpu_func = cupyx.scipy.ndimage.spline_filter
+        x = testing.shaped_random((16, 12, 11), dtype=self.dtype, xp=numpy)
+        x_gpu = cupy.asarray(x)
+
+        kwargs_gpu = dict(order=self.order, output=self.output, mode=self.mode)
+        res_gpu = gpu_func(x_gpu, **kwargs_gpu)
+
+        output_real = numpy.empty((1,), dtype=self.output).real.dtype
+        kwargs = dict(order=self.order, output=output_real, mode=self.mode)
+        res_cpu = cpu_func(x.real, **kwargs) + 1j * cpu_func(x.imag, **kwargs)
+        testing.assert_allclose(res_cpu, res_gpu, atol=1e-4, rtol=1e-4)
diff --git a/tests/cupyx_tests/scipy_tests/ndimage_tests/test_measurements.py b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_measurements.py
new file mode 100644
index 0000000..71f42b0
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_measurements.py
@@ -0,0 +1,452 @@
+import warnings
+
+import numpy
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupy import testing
+from cupy import _util
+from cupy._core import _accelerator
+import cupyx.scipy.ndimage  # NOQA
+
+try:
+    import scipy
+    import scipy.ndimage  # NOQA
+    scipy_version = numpy.lib.NumpyVersion(scipy.__version__)
+except ImportError:
+    scipy_version = numpy.lib.NumpyVersion('0.0.0')
+    pass
+
+stats_ops = ['sum', 'mean', 'variance', 'standard_deviation', 'center_of_mass']
+if scipy_version >= '1.6.0':
+    # 'scipy 1.6 added a copy of sum under the name sum_labels'
+    stats_ops += ['sum_labels']
+
+
+def _generate_binary_structure(rank, connectivity):
+    if connectivity < 1:
+        connectivity = 1
+    if rank < 1:
+        return numpy.array(True, dtype=bool)
+    output = numpy.fabs(numpy.indices([3] * rank) - 1)
+    output = numpy.add.reduce(output, 0)
+    return output <= connectivity
+
+
+@testing.parameterize(*testing.product({
+    'ndim': [1, 2, 3, 4],
+    'size': [50, 100],
+    'density': [0.2, 0.3, 0.4],
+    'connectivity': [None, 2, 3],
+    'x_dtype': [bool, numpy.int8, numpy.int32, numpy.int64,
+                numpy.float32, numpy.float64],
+    'output': [None, numpy.int32, numpy.int64],
+    'o_type': [None, 'ndarray']
+}))
+@testing.with_requires('scipy')
+class TestLabel:
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_label(self, xp, scp):
+        size = int(pow(self.size, 1 / self.ndim))
+        x_shape = range(size, size + self.ndim)
+        x = xp.zeros(x_shape, dtype=self.x_dtype)
+        x[testing.shaped_random(x_shape, xp) < self.density] = 1
+        if self.connectivity is None:
+            structure = None
+        else:
+            structure = _generate_binary_structure(self.ndim,
+                                                   self.connectivity)
+        if self.o_type == 'ndarray' and self.output is not None:
+            output = xp.empty(x_shape, dtype=self.output)
+            num_features = scp.ndimage.label(x, structure=structure,
+                                             output=output)
+            return output
+        labels, num_features = scp.ndimage.label(x, structure=structure,
+                                                 output=self.output)
+        return labels
+
+
+@testing.with_requires('scipy')
+class TestLabelSpecialCases:
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_label_empty(self, xp, scp):
+        x = xp.empty(0)
+        labels, num_features = scp.ndimage.label(x)
+        return labels
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_label_0d_zero(self, xp, scp):
+        x = xp.zeros([])
+        labels, num_features = scp.ndimage.label(x)
+        return labels
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_label_0d_one(self, xp, scp):
+        x = xp.ones([])
+        labels, num_features = scp.ndimage.label(x)
+        return labels
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_label_swirl(self, xp, scp):
+        x = [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
+             [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1],
+             [1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1],
+             [1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1],
+             [1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1],
+             [1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1],
+             [1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1],
+             [1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1],
+             [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1],
+             [1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+             [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
+        x = xp.array(x)
+        labels, num_features = scp.ndimage.label(x)
+        return labels
+
+
+@testing.parameterize(*testing.product({
+    'op': stats_ops,
+}))
+@testing.with_requires('scipy')
+class TestStats:
+
+    def _make_image(self, shape, xp, dtype):
+        if dtype == xp.bool_:
+            return testing.shaped_random(shape, xp, dtype=xp.bool_)
+        else:
+            return testing.shaped_arange(shape, xp, dtype=dtype)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_single_dim(self, xp, scp, dtype):
+        image = self._make_image((100,), xp, dtype)
+        labels = testing.shaped_random((100,), xp, dtype=xp.int32, scale=4)
+        index = xp.array([1, 2, 3])
+        op = getattr(scp.ndimage, self.op)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image, labels, index)
+        if self.op == 'center_of_mass':
+            assert isinstance(result, list)
+            # assert isinstance(result[0], tuple)
+            assert len(result[0]) == image.ndim
+            result = xp.asarray(result)
+        return result
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_multi_dim(self, xp, scp, dtype):
+        image = self._make_image((8, 8, 8), xp, dtype)
+        labels = testing.shaped_random((8, 8, 8), xp, dtype=xp.int32, scale=4)
+        index = xp.array([1, 2, 3])
+        op = getattr(scp.ndimage, self.op)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image, labels, index)
+        if self.op == 'center_of_mass':
+            assert isinstance(result, list)
+            # assert isinstance(result[0], tuple)
+            assert len(result[0]) == image.ndim
+            result = xp.asarray(result)
+        return result
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_broadcast_labels(self, xp, scp, dtype):
+        # 1d label will be broadcast to 2d
+        image = self._make_image((16, 6), xp, dtype)
+        labels = xp.asarray([1, 0, 2, 2, 2, 0], dtype=xp.int32)
+        op = getattr(scp.ndimage, self.op)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image, labels)
+        return result
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_broadcast_labels2(self, xp, scp, dtype):
+        # 1d label will be broadcast to 2d
+        image = self._make_image((16, 6), xp, dtype)
+        labels = xp.asarray([1, 0, 2, 2, 2, 0], dtype=xp.int32)
+        index = 2
+        op = getattr(scp.ndimage, self.op)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image, labels, index)
+        return result
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_zero_dim(self, xp, scp, dtype):
+        image = self._make_image((), xp, dtype)
+        labels = testing.shaped_random((), xp, dtype=xp.int32, scale=4)
+        index = xp.array([1, 2, 3])
+        op = getattr(scp.ndimage, self.op)
+        if self.op == 'center_of_mass':
+            # SciPy doesn't handle 0-dimensional array input for center_of_mass
+            with pytest.raises(IndexError):
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', _util.PerformanceWarning)
+                    op(image, labels, index)
+            return xp.array([])
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image, labels, index)
+        return result
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_only_input(self, xp, scp, dtype):
+        image = self._make_image((100,), xp, dtype)
+        op = getattr(scp.ndimage, self.op)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image)
+        return result
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_no_index(self, xp, scp, dtype):
+        image = self._make_image((100,), xp, dtype)
+        labels = testing.shaped_random((100,), xp, dtype=xp.int32, scale=4)
+        op = getattr(scp.ndimage, self.op)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image, labels)
+        return result
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_scalar_index(self, xp, scp, dtype):
+        image = self._make_image((100,), xp, dtype)
+        labels = testing.shaped_random((100,), xp, dtype=xp.int32, scale=4)
+        op = getattr(scp.ndimage, self.op)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image, labels, 1)
+        return result
+
+    @testing.for_complex_dtypes()
+    def test_invalid_image_dtype(self, dtype):
+        image = self._make_image((100,), cupy, dtype)
+        labels = testing.shaped_random((100,), cupy, dtype=cupy.int32, scale=4)
+        index = cupy.array([1, 2, 3])
+        op = getattr(cupyx.scipy.ndimage, self.op)
+        with pytest.raises(TypeError):
+            op(image, labels, index)
+
+    def test_invalid_image_type(self):
+        image = list(range(100))
+        labels = testing.shaped_random((100,), cupy, dtype=cupy.int32, scale=4)
+        index = cupy.array([1, 2, 3])
+        op = getattr(cupyx.scipy.ndimage, self.op)
+        with pytest.raises(TypeError):
+            op(image, labels, index)
+
+    def test_invalid_labels_shape(self):
+        image = self._make_image((100,), cupy, cupy.int32)
+        labels = testing.shaped_random((50,), cupy, dtype=cupy.int32, scale=4)
+        index = cupy.array([1, 2, 3])
+        op = getattr(cupyx.scipy.ndimage, self.op)
+        with pytest.raises(ValueError):
+            op(image, labels, index)
+
+    def test_invalid_labels_type(self):
+        image = self._make_image((100,), cupy, cupy.int32)
+        labels = numpy.random.randint(0, 4, dtype=numpy.int32, size=100)
+        index = cupy.array([1, 2, 3])
+        op = getattr(cupyx.scipy.ndimage, self.op)
+        with pytest.raises(TypeError):
+            op(image, labels, index)
+
+    def test_invalid_index_type(self):
+        image = self._make_image((100,), cupy, cupy.int32)
+        labels = testing.shaped_random((100,), cupy, dtype=cupy.int32, scale=4)
+        index = [1, 2, 3]
+        op = getattr(cupyx.scipy.ndimage, self.op)
+        with pytest.raises(TypeError):
+            op(image, labels, index)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_no_values(self, xp, scp, dtype):
+        image = xp.array([], dtype=dtype)
+        labels = xp.array([])
+        index = xp.array([])
+        op = getattr(scp.ndimage, self.op)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', _util.PerformanceWarning)
+            result = op(image, labels, index)
+        return result
+
+
+@testing.parameterize(*testing.product({
+    'op': ['maximum', 'median', 'minimum', 'maximum_position',
+           'minimum_position', 'extrema'],
+    'labels': [None, 5, 50],
+    'index': [None, 1, 'all', 'subset'],
+    'shape': [(512,), (32, 64)],
+    'enable_cub': [True, False],
+}))
+@testing.with_requires('scipy')
+class TestMeasurementsSelect:
+
+    @pytest.fixture(autouse=True)
+    def with_accelerators(self):
+        old_accelerators = _accelerator.get_routine_accelerators()
+        if self.enable_cub:
+            _accelerator.set_routine_accelerators(['cub'])
+        else:
+            _accelerator.set_routine_accelerators([])
+        yield
+        _accelerator.set_routine_accelerators(old_accelerators)
+
+    def _hip_skip_invalid_condition(self):
+        if (runtime.is_hip
+                and self.op == 'extrema'
+                and (self.index is None
+                     or (self.index == 1 and self.labels in [None, 5])
+                     or (self.index in ['all', 'subset']
+                         and self.labels is None))):
+            pytest.xfail('ROCm/HIP may have a bug')
+
+    # no_bool=True due to https://github.com/scipy/scipy/issues/12836
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_measurements_select(self, xp, scp, dtype):
+        self._hip_skip_invalid_condition()
+
+        shape = self.shape
+        rstate = numpy.random.RandomState(0)
+        # scale must be small enough to avoid potential integer overflow due to
+        # https://github.com/scipy/scipy/issues/12836
+        x = testing.shaped_random(shape, xp=xp, dtype=dtype, scale=32)
+        non_unique = xp.unique(x).size < x.size
+
+        if (self.op in ['minimum_position', 'maximum_position'] and
+                non_unique and self.index is not None):
+            # skip cases with non-unique min or max position
+            return xp.array([])
+
+        if self.labels is None:
+            labels = self.labels
+        else:
+            labels = rstate.choice(self.labels, x.size).reshape(shape) + 1
+            labels = xp.asarray(labels)
+        if self.index is None or isinstance(self.index, int):
+            index = self.index
+        elif self.index == 'all':
+            if self.labels is not None:
+                index = xp.arange(1, self.labels + 1, dtype=cupy.intp)
+            else:
+                index = None
+        elif self.index == 'subset':
+            if self.labels is not None:
+                index = xp.arange(1, self.labels + 1, dtype=cupy.intp)[1::2]
+            else:
+                index = None
+        func = getattr(scp.ndimage, self.op)
+        result = func(x, labels, index)
+        if self.op == 'extrema':
+            if non_unique and self.index is not None:
+                # omit comparison of minimum_position, maximum_position
+                result = [xp.asarray(r) for r in result[:2]]
+            else:
+                result = [xp.asarray(r) for r in result]
+        else:
+            if isinstance(result, list):
+                # convert list of coordinate tuples to an array for comparison
+                result = xp.asarray(result)
+        return result
+
+
+@testing.parameterize(*testing.product({
+    'labels': [None, 4, 6],
+    'index': [None, [0, 2], [3, 1, 0], [1]],
+    'shape': [(200,), (16, 20)],
+}))
+@testing.with_requires('scipy')
+class TestHistogram():
+
+    def _make_image(self, shape, xp, dtype, scale):
+        return testing.shaped_random(shape, xp, dtype=dtype, scale=scale)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_histogram(self, xp, scp, dtype):
+        nbins = 5
+        minval = 0
+        maxval = 10
+        image = self._make_image(self.shape, xp, dtype, scale=maxval)
+        labels = self.labels
+        index = self.index
+        if labels is not None:
+            labels = testing.shaped_random(self.shape, xp, dtype=xp.int32,
+                                           scale=self.labels)
+        if index is not None:
+            index = xp.array(index)
+        op = getattr(scp.ndimage, 'histogram')
+        if index is not None and labels is None:
+            # cannot give an index array without labels
+            with pytest.raises(ValueError):
+                op(image, minval, maxval, nbins, labels, index)
+            return xp.asarray([])
+        result = op(image, minval, maxval, nbins, labels, index)
+        if index is None:
+            return result
+        # stack 1d arrays into a single array for comparison
+        return xp.stack(result)
+
+
+@testing.parameterize(*testing.product({
+    'labels': [None, 4],
+    'index': [None, [0, 2], [3, 1, 0], [1]],
+    'shape': [(200,), (16, 20)],
+    'dtype': [numpy.float64, 'same'],
+    'default': [0, 3],
+    'pass_positions': [True, False],
+}))
+@testing.with_requires('scipy')
+class TestLabeledComprehension():
+
+    def _make_image(self, shape, xp, dtype, scale):
+        if dtype == xp.bool_:
+            return testing.shaped_random(shape, xp, dtype=xp.bool_)
+        else:
+            return testing.shaped_random(shape, xp, dtype=dtype, scale=scale)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True, no_float16=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-4, atol=1e-4)
+    def test_labeled_comprehension(self, xp, scp, dtype):
+        image = self._make_image(self.shape, xp, dtype, scale=101)
+        labels = self.labels
+        index = self.index
+        if labels is not None:
+            labels = testing.shaped_random(self.shape, xp, dtype=xp.int32,
+                                           scale=4)
+        if index is not None:
+            index = xp.array(index)
+
+        if self.pass_positions:
+            # simple function that takes a positions argument
+            def func(x, pos):
+                return xp.sum(x + pos > 50)
+        else:
+            # simple function to apply to each lable
+            func = xp.sum
+
+        op = getattr(scp.ndimage, 'labeled_comprehension')
+        dtype == image.dtype if self.dtype == 'same' else self.dtype
+        if index is not None and labels is None:
+            # cannot give an index array without labels
+            with pytest.raises(ValueError):
+                op(image, labels, index, func, dtype, self.default,
+                   self.pass_positions)
+            return xp.asarray([])
+        return op(image, labels, index, func, dtype, self.default,
+                  self.pass_positions)
diff --git a/tests/cupyx_tests/scipy_tests/ndimage_tests/test_morphology.py b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_morphology.py
new file mode 100644
index 0000000..7772ca3
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/ndimage_tests/test_morphology.py
@@ -0,0 +1,610 @@
+import numpy
+import pytest
+
+from cupy import testing
+import cupyx.scipy.ndimage  # NOQA
+
+try:
+    import scipy.ndimage  # NOQA
+except ImportError:
+    pass
+
+
+@testing.parameterize(
+    {'rank': 0, 'connectivity': 1},
+    {'rank': 1, 'connectivity': 1},
+    {'rank': 2, 'connectivity': 1},
+    {'rank': 2, 'connectivity': 2},
+    {'rank': 3, 'connectivity': 1},
+    {'rank': 3, 'connectivity': 2},
+    {'rank': 3, 'connectivity': 3},
+    # edge cases
+    {'rank': -1, 'connectivity': 0},
+    {'rank': 3, 'connectivity': 0},
+    {'rank': 3, 'connectivity': 500})
+@testing.with_requires('scipy')
+class TestGenerateBinaryStructure:
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_generate_binary_structure(self, xp, scp):
+        return scp.ndimage.generate_binary_structure(self.rank,
+                                                     self.connectivity)
+
+
+@testing.with_requires('scipy')
+class TestIterateStructure:
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_iterate_structure1(self, xp, scp):
+        struct = xp.asarray([[0, 1, 0],
+                             [1, 1, 1],
+                             [0, 1, 0]])
+        return scp.ndimage.iterate_structure(struct, 2)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_iterate_structure2(self, xp, scp):
+        struct = xp.asarray([[0, 1],
+                             [1, 1],
+                             [0, 1]])
+        return scp.ndimage.iterate_structure(struct, 2)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_iterate_structure3(self, xp, scp):
+        struct = xp.asarray([[0, 1, 0],
+                             [1, 1, 1],
+                             [0, 1, 0]])
+        out, origin = scp.ndimage.iterate_structure(struct, 2, 1)
+        assert origin == [2, 2]
+        return out
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.bool_, numpy.int8, numpy.uint8, numpy.float32,
+                    numpy.float64],
+        'border_value': [0, 1],
+        'structure': [None, [1, 0, 1], [1, 1, 0]],
+        'origin': [-1, 0, 1],
+        'data': [[], [1, 1, 0, 1, 1]],
+        'filter': ['binary_erosion', 'binary_dilation'],
+        'output': [None, numpy.float32, numpy.int8, 'array']}
+    ))
+)
+@testing.with_requires('scipy')
+class TestBinaryErosionAndDilation1d:
+    def _filter(self, xp, scp, x):
+        filter = getattr(scp.ndimage, self.filter)
+        structure = self.structure
+        if structure is not None:
+            structure = xp.asarray(structure)
+        if self.output == 'array':
+            output = xp.zeros_like(x)
+            filter(x, structure, iterations=1, mask=None,
+                   output=output, border_value=self.border_value,
+                   origin=self.origin, brute_force=True)
+            return output
+        return filter(x, structure, iterations=1, mask=None,
+                      output=self.output, border_value=self.border_value,
+                      origin=self.origin, brute_force=True)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_binary_erosion_and_dilation_1d(self, xp, scp):
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        x = xp.asarray(self.data, dtype=self.x_dtype)
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.bool_, numpy.float64],
+        'border_value': [0, 1],
+        'connectivity': [1, 2],
+        'origin': [0, 1],
+        'data': [[[0, 1, 0, 0, 0, 0, 0, 0],
+                  [1, 1, 1, 0, 0, 0, 0, 0],
+                  [0, 1, 0, 0, 0, 1, 0, 0],
+                  [0, 0, 0, 1, 1, 1, 1, 0],
+                  [0, 0, 1, 1, 0, 1, 0, 0],
+                  [0, 1, 1, 1, 1, 1, 1, 0],
+                  [0, 0, 1, 0, 0, 1, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+
+                 [[1, 1, 1, 0, 0, 0, 0, 0],
+                  [1, 1, 1, 0, 0, 0, 0, 0],
+                  [1, 1, 1, 1, 1, 1, 1, 0],
+                  [0, 0, 1, 1, 1, 1, 1, 0],
+                  [0, 1, 1, 1, 0, 1, 1, 0],
+                  [0, 1, 1, 1, 1, 1, 1, 0],
+                  [0, 1, 1, 1, 1, 1, 1, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+                 ],
+        'filter': ['binary_opening', 'binary_closing'],
+        'output': [None, numpy.float32, numpy.int8]}
+    ))
+)
+@testing.with_requires('scipy>=1.1.0')
+class TestBinaryOpeningAndClosing:
+    def _filter(self, xp, scp, x):
+        filter = getattr(scp.ndimage, self.filter)
+        structure = scp.ndimage.generate_binary_structure(x.ndim,
+                                                          self.connectivity)
+        return filter(x, structure, iterations=1, output=self.output,
+                      origin=self.origin, mask=None,
+                      border_value=self.border_value, brute_force=True)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_binary_opening_and_closing(self, xp, scp):
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        x = xp.asarray(self.data, dtype=self.x_dtype)
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.bool_, numpy.float64],
+        'connectivity': [1, 2],
+        'origin': [-1, 0, 1],
+        'data': [[[0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 1, 1, 1, 1, 0, 0],
+                  [0, 0, 1, 0, 0, 1, 0, 0],
+                  [0, 0, 1, 0, 0, 1, 0, 0],
+                  [0, 0, 1, 0, 0, 1, 0, 0],
+                  [0, 0, 1, 1, 1, 1, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+
+                 [[0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 1, 1, 0, 0, 0],
+                  [0, 0, 1, 0, 0, 1, 0, 0],
+                  [0, 0, 1, 0, 0, 1, 0, 0],
+                  [0, 0, 1, 0, 0, 1, 0, 0],
+                  [0, 0, 0, 1, 1, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+
+                 [[0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 1, 0, 0, 0, 0, 0],
+                  [0, 1, 0, 1, 0, 1, 1, 1],
+                  [0, 1, 0, 1, 0, 1, 0, 1],
+                  [0, 1, 0, 1, 0, 1, 0, 1],
+                  [0, 0, 1, 0, 0, 1, 1, 1],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+                 ],
+        'output': [None, numpy.float32, numpy.int8]}
+    ))
+)
+@testing.with_requires('scipy')
+class TestBinaryFillHoles:
+    def _filter(self, xp, scp, x):
+        filter = scp.ndimage.binary_fill_holes
+        structure = scp.ndimage.generate_binary_structure(x.ndim,
+                                                          self.connectivity)
+        return filter(x, structure, output=self.output, origin=self.origin)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_binary_fill_holes(self, xp, scp):
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        x = xp.asarray(self.data, dtype=self.x_dtype)
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.bool_, numpy.float64],
+        'struct': ['same', 'separate'],
+        'origins': [((0, 0), (0, 0)),
+                    ((0, 1), (-1, 0))],
+        'data': [[[0, 1, 0, 0, 0],
+                  [1, 1, 1, 0, 0],
+                  [0, 1, 0, 1, 1],
+                  [0, 0, 1, 1, 1],
+                  [0, 1, 1, 1, 0],
+                  [0, 1, 1, 1, 1],
+                  [0, 1, 1, 1, 1],
+                  [0, 0, 0, 0, 0]],
+
+                 [[0, 1, 0, 0, 1, 1, 1, 0],
+                  [1, 1, 1, 0, 0, 1, 0, 0],
+                  [0, 1, 0, 1, 1, 1, 1, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+
+                 [[0, 1, 0, 0, 1, 1, 1, 0],
+                  [1, 1, 1, 0, 0, 0, 0, 0],
+                  [0, 1, 0, 1, 1, 1, 1, 0],
+                  [0, 0, 1, 1, 1, 1, 1, 0],
+                  [0, 1, 1, 1, 0, 1, 1, 0],
+                  [0, 0, 0, 0, 1, 1, 1, 0],
+                  [0, 1, 1, 1, 1, 1, 1, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+                 ],
+        'output': [None, numpy.float32, numpy.int8]}
+    ))
+)
+@testing.with_requires('scipy')
+class TestBinaryHitOrMiss:
+    def _filter(self, xp, scp, x):
+        filter = scp.ndimage.binary_hit_or_miss
+        if self.struct == 'same':
+            structure1 = scp.ndimage.generate_binary_structure(x.ndim, 1)
+            structure2 = structure1
+        elif self.struct == 'separate':
+            structure1 = xp.asarray([[0, 0, 0],
+                                     [1, 1, 1],
+                                     [0, 0, 0]])
+
+            structure2 = xp.asarray([[1, 1, 1],
+                                     [0, 0, 0],
+                                     [1, 1, 1]])
+        origin1, origin2 = self.origins
+        return filter(x, structure1, structure2, output=self.output,
+                      origin1=origin1, origin2=origin2)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_binary_hit_or_miss(self, xp, scp):
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        x = xp.asarray(self.data, dtype=self.x_dtype)
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.bool_, numpy.float64],
+        'border_value': [0, 1],
+        'connectivity': [1, 2],
+        'origin': [0, 1],
+        'mask': [[[0, 1, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 1, 0],
+                  [0, 0, 0, 0, 1, 1, 0, 0],
+                  [0, 0, 1, 1, 1, 0, 0, 0],
+                  [0, 1, 1, 0, 1, 1, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+
+                 [[0, 1, 0, 0, 0, 0, 0, 0],
+                  [0, 1, 1, 0, 0, 0, 0, 0],
+                  [0, 0, 1, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 1, 0, 0],
+                  [0, 0, 0, 1, 1, 0, 0, 0],
+                  [0, 0, 1, 0, 0, 1, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+                 ],
+        'data': [[[0, 1, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 1, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+
+                 [[0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0],
+                  [0, 0, 0, 0, 0, 0, 0, 0]],
+                 ],
+        'output': [None, numpy.float32, numpy.int8]}
+    ))
+)
+@testing.with_requires('scipy')
+class TestBinaryPropagation:
+    def _filter(self, xp, scp, x):
+        filter = scp.ndimage.binary_propagation
+        structure = scp.ndimage.generate_binary_structure(x.ndim,
+                                                          self.connectivity)
+        mask = xp.asarray(self.mask)
+        return filter(x, structure, mask=mask, output=self.output,
+                      border_value=self.border_value, origin=self.origin)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_binary_propagation(self, xp, scp):
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        x = xp.asarray(self.data, dtype=self.x_dtype)
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.int8, numpy.float32],
+        'border_value': [0, 1],
+        'connectivity': [1, 2],
+        'origin': [0, -1],
+        'shape': [(64,), (16, 15), (5, 7, 9)],
+        'density': [0.1, 0.5, 0.9],
+        'filter': ['binary_erosion', 'binary_dilation'],
+        'iterations': [1, 2, 0],
+        'output': [None, numpy.float32, 'array']}
+    ))
+)
+@testing.with_requires('scipy')
+class TestBinaryErosionAndDilation:
+    def _filter(self, xp, scp, x):
+        filter = getattr(scp.ndimage, self.filter)
+        ndim = len(self.shape)
+        structure = scp.ndimage.generate_binary_structure(ndim,
+                                                          self.connectivity)
+        if self.output == 'array':
+            output = xp.zeros_like(x)
+            filter(x, structure, iterations=self.iterations, mask=None,
+                   output=output, border_value=self.border_value,
+                   origin=self.origin, brute_force=True)
+            return output
+        return filter(x, structure, iterations=self.iterations, mask=None,
+                      output=self.output, border_value=self.border_value,
+                      origin=self.origin, brute_force=True)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_binary_erosion_and_dilation(self, xp, scp):
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        rstate = numpy.random.RandomState(5)
+        x = rstate.randn(*self.shape) > self.density
+        x = xp.asarray(x, dtype=self.x_dtype)
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.int8, numpy.float32],
+        'shape': [(16, 24)],
+        'filter': ['binary_erosion', 'binary_dilation'],
+        'iterations': [1, 2],
+        'contiguity': ['C', 'F', 'none']}
+    ))
+)
+@testing.with_requires('scipy')
+class TestBinaryErosionAndDilationContiguity:
+    def _filter(self, xp, scp, x):
+        filter = getattr(scp.ndimage, self.filter)
+        ndim = len(self.shape)
+        structure = scp.ndimage.generate_binary_structure(ndim, 1)
+        return filter(x, structure, iterations=self.iterations, mask=None,
+                      output=None, border_value=0, origin=0, brute_force=True)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_binary_erosion_and_dilation_input_contiguity(self, xp, scp):
+        rstate = numpy.random.RandomState(5)
+        x = rstate.randn(*self.shape) > 0.3
+        x = xp.asarray(x, dtype=self.x_dtype)
+        if self.contiguity == 'F':
+            x = xp.asfortranarray(x)
+        elif self.contiguity == 'none':
+            x = x[::2, ::3]
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'shape': [(3, 4), (2, 3, 4), (1, 2, 3, 4)],
+        'size': [3, 4],
+        'footprint': [None, 'random'],
+        'structure': [None, 'random'],
+        'mode': ['reflect'],
+        'cval': [0.0],
+        'origin': [0, 1, None],
+        'x_dtype': [numpy.int8, numpy.int16, numpy.int32,
+                    numpy.float32, numpy.float64],
+        'output': [None, numpy.int32, numpy.float64],
+        'filter': ['grey_erosion', 'grey_dilation']
+    }) + testing.product({
+        'shape': [(3, 4), (2, 3, 4), (1, 2, 3, 4)],
+        'size': [3, 4],
+        'footprint': [None, 'random'],
+        'structure': [None, 'random'],
+        'mode': ['constant'],
+        'cval': [-1.0, 0.0, 1.0],
+        'origin': [0],
+        'x_dtype': [numpy.int32, numpy.float64],
+        'output': [None],
+        'filter': ['grey_erosion', 'grey_dilation']
+    }) + testing.product({
+        'shape': [(3, 4), (2, 3, 4), (1, 2, 3, 4)],
+        'size': [3, 4],
+        'footprint': [None, 'random'],
+        'structure': [None, 'random'],
+        'mode': ['nearest', 'mirror', 'wrap'],
+        'cval': [0.0],
+        'origin': [0],
+        'x_dtype': [numpy.int32, numpy.float64],
+        'output': [None],
+        'filter': ['grey_erosion', 'grey_dilation']
+    })
+))
+@testing.with_requires('scipy')
+class TestGreyErosionAndDilation:
+
+    def _filter(self, xp, scp, x):
+        filter = getattr(scp.ndimage, self.filter)
+        if self.origin is None:
+            origin = (-1, 1, -1, 1)[:x.ndim]
+        else:
+            origin = self.origin
+        if self.footprint is None:
+            footprint = None
+        else:
+            shape = (self.size, ) * x.ndim
+            r = testing.shaped_random(shape, xp, scale=1)
+            footprint = xp.where(r < .5, 1, 0)
+            if not footprint.any():
+                footprint = xp.ones(shape)
+        if self.structure is None:
+            structure = None
+        else:
+            shape = (self.size, ) * x.ndim
+            structure = testing.shaped_random(shape, xp, dtype=xp.int32)
+        return filter(x, size=self.size, footprint=footprint,
+                      structure=structure, output=self.output,
+                      mode=self.mode, cval=self.cval, origin=origin)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_grey_erosion_and_dilation(self, xp, scp):
+        if self.mode == 'mirror' and 1 in self.shape:
+            pytest.skip('not testable against scipy')
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        x = testing.shaped_random(self.shape, xp, self.x_dtype)
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*testing.product({
+    'size': [3, 4],
+    'structure': [None, 'random'],
+    'mode': ['reflect', 'constant', 'nearest', 'mirror', 'wrap'],
+    'origin': [0, None],
+    'x_dtype': [numpy.int32, numpy.float32],
+    'output': [None, numpy.float64],
+    'filter': ['grey_closing', 'grey_opening']
+}))
+@testing.with_requires('scipy')
+class TestGreyClosingAndOpening:
+
+    shape = (4, 5)
+    footprint = None
+    cval = 0.0
+
+    def _filter(self, xp, scp, x):
+        filter = getattr(scp.ndimage, self.filter)
+        if self.origin is None:
+            origin = (-1, 1, -1, 1)[:x.ndim]
+        else:
+            origin = self.origin
+        if self.structure is None:
+            structure = None
+        else:
+            shape = (self.size, ) * x.ndim
+            structure = testing.shaped_random(shape, xp, dtype=xp.int32)
+        return filter(x, size=self.size, footprint=self.footprint,
+                      structure=structure, output=self.output,
+                      mode=self.mode, cval=self.cval, origin=origin)
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_grey_closing_and_opening(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.x_dtype)
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.int32],
+        'origin': [-1, 0, 1],
+        'filter': ['morphological_gradient', 'morphological_laplace'],
+        'mode': ['reflect', 'constant'],
+        'output': [None],
+        'size': [(3, 3), (4, 3)],
+        'footprint': [None, 'random'],
+        'structure': [None, 'random']}
+    ) + testing.product({
+        'x_dtype': [numpy.int32, numpy.float64],
+        'origin': [0],
+        'filter': ['morphological_gradient', 'morphological_laplace'],
+        'mode': ['reflect', 'constant', 'nearest', 'mirror', 'wrap'],
+        'output': [None, numpy.float32, 'zeros'],
+        'size': [3],
+        'footprint': [None, 'random'],
+        'structure': [None, 'random']}
+    ))
+)
+@testing.with_requires('scipy')
+class TestMorphologicalGradientAndLaplace:
+
+    def _filter(self, xp, scp, x):
+        filter = getattr(scp.ndimage, self.filter)
+        if xp.isscalar(self.size):
+            shape = (self.size,) * x.ndim
+        else:
+            shape = tuple(self.size)
+        if self.footprint is None:
+            footprint = None
+        else:
+            r = testing.shaped_random(shape, xp, scale=1)
+            footprint = xp.where(r < .5, 1, 0)
+            if not footprint.any():
+                footprint = xp.ones(shape)
+        if self.structure is None:
+            structure = None
+        else:
+            structure = testing.shaped_random(shape, xp, dtype=xp.int32)
+        if self.output == 'zeros':
+            output = xp.zeros_like(x)
+        else:
+            output = self.output
+        return filter(x, self.size, footprint, structure,
+                      output=output, mode=self.mode, cval=0.0,
+                      origin=self.origin)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_morphological_gradient_and_laplace(self, xp, scp):
+        x = xp.zeros((7, 7), dtype=self.x_dtype)
+        x[2:5, 2:5] = 1
+        x[4, 4] = 2
+        x[2, 3] = 3
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        return self._filter(xp, scp, x)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'x_dtype': [numpy.int32],
+        'shape': [(5, 7)],
+        'origin': [-1, 0, 1],
+        'filter': ['white_tophat', 'black_tophat'],
+        'mode': ['reflect', 'constant'],
+        'output': [None],
+        'size': [(3, 3), (4, 3)],
+        'footprint': [None, 'random'],
+        'structure': [None, 'random']}
+    ) + testing.product({
+        'x_dtype': [numpy.int32, numpy.float64],
+        'shape': [(6, 8)],
+        'origin': [0],
+        'filter': ['white_tophat', 'black_tophat'],
+        'mode': ['reflect', 'constant', 'nearest', 'mirror', 'wrap'],
+        'output': [None, numpy.float32, 'zeros'],
+        'size': [3],
+        'footprint': [None, 'random'],
+        'structure': [None, 'random']}
+    ))
+)
+@testing.with_requires('scipy')
+class TestWhiteTophatAndBlackTopHat:
+
+    def _filter(self, xp, scp, x):
+        filter = getattr(scp.ndimage, self.filter)
+        if xp.isscalar(self.size):
+            shape = (self.size,) * x.ndim
+        else:
+            shape = tuple(self.size)
+        if self.footprint is None:
+            footprint = None
+        else:
+            r = testing.shaped_random(shape, xp, scale=1)
+            footprint = xp.where(r < .5, 1, 0)
+            if not footprint.any():
+                footprint = xp.ones(shape)
+        if self.structure is None:
+            structure = None
+        else:
+            structure = testing.shaped_random(shape, xp, dtype=xp.int32)
+        if self.output == 'zeros':
+            output = xp.zeros_like(x)
+        else:
+            output = self.output
+        return filter(x, self.size, footprint, structure,
+                      output=output, mode=self.mode, cval=0.0,
+                      origin=self.origin)
+
+    @testing.numpy_cupy_array_equal(scipy_name='scp')
+    def test_white_tophat_and_black_tophat(self, xp, scp):
+        x = testing.shaped_random(self.shape, xp, self.x_dtype)
+        if self.x_dtype == self.output:
+            pytest.skip('redundant')
+        return self._filter(xp, scp, x)
diff --git a/tests/cupyx_tests/scipy_tests/signal_tests/test_bsplines.py b/tests/cupyx_tests/scipy_tests/signal_tests/test_bsplines.py
new file mode 100644
index 0000000..08c866d
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/signal_tests/test_bsplines.py
@@ -0,0 +1,33 @@
+import sys
+import unittest
+
+from cupy import testing
+
+import cupyx.scipy.signal  # NOQA
+
+try:
+    import scipy.signal  # NOQA
+except ImportError:
+    pass
+
+
+@testing.parameterize(*testing.product({
+    'input': [(256, 256), (4, 512), (512, 3)],
+    'hrow': [1, 3],
+    'hcol': [1, 3],
+}))
+@testing.with_requires('scipy')
+class TestSepFIR2d(unittest.TestCase):
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_sepfir2d(self, xp, scp, dtype):
+        if sys.platform.startswith('win32') and xp.dtype(dtype).kind in 'c':
+            # it's likely a SciPy bug (ValueError: incorrect type), so we
+            # do this to effectively skip testing it
+            return xp.zeros(10)
+
+        input = testing.shaped_random(self.input, xp, dtype)
+        hrow = testing.shaped_random((self.hrow,), xp, dtype)
+        hcol = testing.shaped_random((self.hcol,), xp, dtype)
+        return scp.signal.sepfir2d(input, hrow, hcol)
diff --git a/tests/cupyx_tests/scipy_tests/signal_tests/test_signaltools.py b/tests/cupyx_tests/scipy_tests/signal_tests/test_signaltools.py
new file mode 100644
index 0000000..f8f85fa
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/signal_tests/test_signaltools.py
@@ -0,0 +1,362 @@
+import sys
+
+import numpy as np
+import pytest
+
+import cupy
+from cupy.cuda import runtime
+from cupy import testing
+import cupyx.scipy.signal
+
+try:
+    import scipy.signal  # NOQA
+except ImportError:
+    pass
+
+
+@testing.parameterize(*testing.product({
+    'size1': [(10,), (5, 10), (10, 3), (3, 4, 10)],
+    'size2': [3, 4, 5, 10],
+    'mode': ['full', 'same', 'valid'],
+}))
+@testing.with_requires('scipy')
+class TestConvolveCorrelate:
+    def _filter(self, func, dtype, xp, scp):
+        in1 = testing.shaped_random(self.size1, xp, dtype)
+        in2 = testing.shaped_random((self.size2,)*in1.ndim, xp, dtype)
+        return getattr(scp.signal, func)(in1, in2, self.mode, method='direct')
+
+    tols = {np.float32: 1e-5, np.complex64: 1e-5,
+            np.float16: 1e-3, 'default': 1e-10}
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_convolve(self, xp, scp, dtype):
+        return self._filter('convolve', dtype, xp, scp)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_correlate(self, xp, scp, dtype):
+        return self._filter('correlate', dtype, xp, scp)
+
+
+@testing.parameterize(*testing.product({
+    'size1': [(10,), (5, 10), (10, 3), (3, 4, 10)],
+    'size2': [3, 4, 5, 10],
+    'mode': ['full', 'same', 'valid'],
+}))
+@testing.with_requires('scipy')
+class TestFFTConvolve:
+    def _filter(self, func, dtype, xp, scp, **kwargs):
+        in1 = testing.shaped_random(self.size1, xp, dtype)
+        in2 = testing.shaped_random((self.size2,)*in1.ndim, xp, dtype)
+        return getattr(scp.signal, func)(in1, in2, self.mode, **kwargs)
+
+    tols = {np.float32: 1e-3, np.complex64: 1e-3,
+            np.float16: 1e-3, 'default': 1e-8}
+
+    def _hip_skip_invalid_condition(self):
+        invalid_condition = [
+            ('full', 3), ('full', 4), ('full', 5), ('full', 10),
+            ('same', 3), ('same', 4), ('same', 5), ('same', 10),
+            ('valid', 3), ('valid', 10)]
+        if (runtime.is_hip and self.size1 == (3, 4, 10)
+                and (self.mode, self.size2) in invalid_condition):
+            pytest.xfail('ROCm/HIP may have a bug')
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_fftconvolve(self, xp, scp, dtype):
+        self._hip_skip_invalid_condition()
+        return self._filter('fftconvolve', dtype, xp, scp)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_convolve_fft(self, xp, scp, dtype):
+        self._hip_skip_invalid_condition()
+        return self._filter('convolve', dtype, xp, scp, method='fft')
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_correlate_fft(self, xp, scp, dtype):
+        self._hip_skip_invalid_condition()
+        return self._filter('correlate', dtype, xp, scp, method='fft')
+
+
+def tupleid(shape):
+    return ''.join(str(s) for s in shape)
+
+
+@testing.with_requires('scipy')
+class TestFFTConvolveFastShape:
+    @pytest.mark.parametrize('mode', ['full', 'same', 'valid'])
+    @pytest.mark.parametrize(('shape1', 'shape2'), [
+        ((1,), (7,)),
+        ((3,), (1,)),
+        ((5, 4), (3, 1)),
+        ((1, 1), (2, 4)),
+        ((), ()),
+        ((5, 1, 1), (1, 4, 1)),
+    ], ids=tupleid)
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp')
+    def test_fftconvolve1(self, xp, scp, dtype, shape1, shape2, mode):
+        in1 = testing.shaped_random(shape1, xp, dtype)
+        in2 = testing.shaped_random(shape2, xp, dtype)
+        return scp.signal.fftconvolve(in1, in2, mode=mode)
+
+    # SciPy says "For 'valid' mode, one must be at least as large as the
+    # other in every dimension". So 'valid' is excluded from the testcases,
+    # while SciPy fails to reject them.
+    @pytest.mark.parametrize('mode', ['full', 'same'])
+    @pytest.mark.parametrize(('shape1', 'shape2'), [
+        ((5, 1), (1, 4)),
+        ((5, 1, 1), (1, 4, 1)),
+    ], ids=tupleid)
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp')
+    def test_fftconvolve1_incomparable_shape(
+            self, xp, scp, dtype, shape1, shape2, mode):
+        in1 = testing.shaped_random(shape1, xp, dtype)
+        in2 = testing.shaped_random(shape2, xp, dtype)
+        return scp.signal.fftconvolve(in1, in2, mode=mode)
+
+    @pytest.mark.parametrize('mode', ['full', 'same', 'valid'])
+    @pytest.mark.parametrize(('shape1', 'shape2', 'axes'), [
+        ((1, 4), (2, 4), (0,)),
+        # ((3, 3), (3, 3), ()), => ValueError. Only reduced axes can be empty.
+        ((2, 5, 5), (2, 1, 3), (1, 2)),
+        ((2, 5, 5), (2, 1, 1), (1, 2)),
+        ((1, 5, 5), (2, 1, 3), (1, 2)),  # broadcast
+    ], ids=tupleid)
+    @testing.for_dtypes('efdFD')
+    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp')
+    def test_fftconvolve1_axes(
+            self, xp, scp, dtype, shape1, shape2, axes, mode):
+        in1 = testing.shaped_random(shape1, xp, dtype)
+        in2 = testing.shaped_random(shape2, xp, dtype)
+        return scp.signal.fftconvolve(in1, in2, mode=mode, axes=axes)
+
+
+@testing.parameterize(*testing.product({
+    'size1': [(10,), (5, 10), (10, 3), (3, 10, 15)],
+    'size2': [3, 4, 5, 10, None],
+    'mode': ['full', 'same', 'valid'],
+}))
+@testing.with_requires('scipy')
+class TestOAConvolve:
+    tols = {np.float32: 1e-3, np.complex64: 1e-3,
+            np.float16: 1e-3, 'default': 1e-8}
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_oaconvolve(self, xp, scp, dtype):
+        if runtime.is_hip and self.size2 in [5, None]:
+            pytest.xfail('ROCm/HIP may have a bug')
+        in1 = testing.shaped_random(self.size1, xp, dtype)
+        shape2 = self.size1 if self.size2 is None else (self.size2,)*in1.ndim
+        in2 = testing.shaped_random(shape2, xp, dtype)
+        return scp.signal.oaconvolve(in1, in2, self.mode)
+
+
+@testing.parameterize(*(testing.product({
+    'size1': [(5, 10), (10, 7)],
+    'size2': [(3, 2), (3, 3), (2, 2), (10, 10), (11, 11)],
+    'mode': ['full', 'same', 'valid'],
+    'boundary': ['fill'],
+    'fillvalue': [0, 1, -1],
+}) + testing.product({
+    'size1': [(5, 10), (10, 7)],
+    'size2': [(3, 2), (3, 3), (2, 2), (10, 10), (11, 11)],
+    'mode': ['full', 'same', 'valid'],
+    'boundary': ['wrap', 'symm'],
+    'fillvalue': [0],
+})))
+@testing.with_requires('scipy')
+class TestConvolveCorrelate2D:
+    def _filter(self, func, dtype, xp, scp):
+        if self.mode == 'full' and self.boundary != 'fill':
+            # See https://github.com/scipy/scipy/issues/12685
+            pytest.skip('broken in scipy')
+        if np.dtype(dtype).kind == 'u' and self.fillvalue < 0:
+            pytest.skip('fillvalue overflow')
+        in1 = testing.shaped_random(self.size1, xp, dtype)
+        in2 = testing.shaped_random(self.size2, xp, dtype)
+        return getattr(scp.signal, func)(in1, in2, self.mode, self.boundary,
+                                         self.fillvalue)
+
+    tols = {np.float32: 5e-4, np.complex64: 5e-4,
+            np.float16: 1e-3, 'default': 1e-10}
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_convolve2d(self, xp, scp, dtype):
+        return self._filter('convolve2d', dtype, xp, scp)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp',
+                                 accept_error=ValueError)
+    def test_correlate2d(self, xp, scp, dtype):
+        return self._filter('correlate2d', dtype, xp, scp)
+
+
+@testing.with_requires('scipy')
+class TestConvolve2DEdgeCase:
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy<1.10')
+    def test_convolve2d_1(self, xp, scp):
+        # Meant a gray-scale image
+        data = testing.shaped_random(
+            (512, 512), xp=xp, dtype=xp.uint8, scale=256)
+        scharr = xp.array(
+            [[-3-3j, 0-10j, +3-3j],
+             [-10+0j, 0+0j, +10+0j],
+             [-3+3j, 0+10j, +3+3j]])  # Gx + j*Gy
+        return scp.signal.convolve2d(
+            data, scharr, boundary='symm', mode='same')
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_convolve2d_2(self, xp, scp):
+        # see cupy/cupy#6047
+        a = xp.array([[257]], dtype="uint64")
+        b = xp.array([[1]], dtype="uint8")
+        return scp.signal.convolve2d(a, b, mode="same")
+
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_convolve2d_3(self, xp, scp):
+        # see cupy/cupy#6047
+        a = xp.array([[257]], dtype="uint64")
+        b = xp.array([[1]], dtype="uint8")
+        return scp.signal.convolve2d(b, a, mode="same")
+
+
+@testing.parameterize(*testing.product({
+    'mode': ['valid', 'same', 'full']
+}))
+class TestChooseConvMethod:
+
+    @testing.for_dtypes('efdFD')
+    def test_choose_conv_method1(self, dtype):
+        a = testing.shaped_arange((10000,), cupy, dtype)
+        b = testing.shaped_arange((5000,), cupy, dtype)
+        assert cupyx.scipy.signal.choose_conv_method(
+            a, b, mode=self.mode) == 'fft'
+
+    @testing.for_dtypes('efdFD')
+    def test_choose_conv_method2(self, dtype):
+        a = testing.shaped_arange((5000,), cupy, dtype)
+        b = testing.shaped_arange((10000,), cupy, dtype)
+        assert cupyx.scipy.signal.choose_conv_method(
+            a, b, mode=self.mode) == 'fft'
+
+    @testing.for_int_dtypes()
+    def test_choose_conv_method_int(self, dtype):
+        a = testing.shaped_arange((10,), cupy, dtype)
+        b = testing.shaped_arange((5,), cupy, dtype)
+        assert cupyx.scipy.signal.choose_conv_method(
+            a, b, mode=self.mode) == 'direct'
+
+    @testing.for_all_dtypes()
+    def test_choose_conv_method_ndim(self, dtype):
+        a = testing.shaped_arange((3, 4, 5), cupy, dtype)
+        b = testing.shaped_arange((1, 2), cupy, dtype)
+        with pytest.raises(NotImplementedError):
+            cupyx.scipy.signal.choose_conv_method(a, b, mode=self.mode)
+
+    @testing.for_all_dtypes()
+    def test_choose_conv_method_zero_dim(self, dtype):
+        a = testing.shaped_arange((), cupy, dtype)
+        b = testing.shaped_arange((5,), cupy, dtype)
+        with pytest.raises(NotImplementedError):
+            cupyx.scipy.signal.choose_conv_method(a, b, mode=self.mode)
+
+
+@testing.parameterize(*testing.product({
+    'im': [(10,), (5, 10), (10, 3), (3, 4, 10)],
+    'mysize': [3, 4, (3, 4, 5)],
+    'noise': [False, True],
+}))
+@testing.with_requires('scipy')
+class TestWiener:
+    tols = {np.float32: 1e-5, np.complex64: 1e-5,
+            np.float16: 1e-3, 'default': 1e-10}
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=tols, rtol=tols, scipy_name='scp')
+    def test_wiener(self, xp, scp, dtype):
+        im = testing.shaped_random(self.im, xp, dtype)
+        mysize = self.mysize
+        if isinstance(mysize, tuple):
+            mysize = mysize[:im.ndim]
+        noise = (testing.shaped_random(self.im, xp, dtype)
+                 if self.noise else None)
+        out = scp.signal.wiener(im, mysize, noise)
+        # Always returns float64 or complex128 data  in both scipy and
+        # cupyx.scipy. Per-datatype tolerances are based on the output
+        # data type but quality is based on input data type (if floating point)
+        assert out.dtype == (np.complex128 if out.dtype.kind == 'c' else
+                             np.float64)
+        return out.astype(dtype, copy=False) if dtype in self.tols else out
+
+
+@testing.parameterize(*testing.product({
+    'a': [(10,), (5, 10), (10, 3), (3, 4, 10)],
+    'domain': [3, 4, (3, 3, 5)],
+    'rank': [0, 1, 2],
+}))
+@testing.with_requires('scipy')
+class TestOrderFilter:
+    @testing.for_all_dtypes(no_float16=True, no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-8, rtol=1e-8, scipy_name='scp',
+                                 accept_error=ValueError)  # for even kernels
+    def test_order_filter(self, xp, scp, dtype):
+        a = testing.shaped_random(self.a, xp, dtype)
+        d = self.domain
+        d = d[:a.ndim] if isinstance(d, tuple) else (d,)*a.ndim
+        domain = testing.shaped_random(d, xp) > 0.25
+        rank = min(self.rank, domain.sum())
+        return scp.signal.order_filter(a, domain, rank)
+
+
+@testing.parameterize(*testing.product({
+    'volume': [(10,), (5, 10), (10, 5), (5, 6, 10)],
+    'kernel_size': [3, 4, (3, 3, 5)],
+}))
+@testing.with_requires('scipy>=1.7.0', 'scipy<1.11.0')
+class TestMedFilt:
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-8, rtol=1e-8, scipy_name='scp',
+                                 accept_error=ValueError)  # for even kernels
+    def test_medfilt(self, xp, scp, dtype):
+        if sys.platform == 'win32':
+            pytest.xfail('medfilt broken for Scipy 1.7.0 in windows')
+        volume = testing.shaped_random(self.volume, xp, dtype)
+        kernel_size = self.kernel_size
+        if isinstance(kernel_size, tuple):
+            kernel_size = kernel_size[:volume.ndim]
+        return scp.signal.medfilt(volume, kernel_size)
+
+
+@testing.parameterize(*testing.product({
+    'input': [(5, 10), (10, 5)],
+    'kernel_size': [3, 4, (3, 5)],
+}))
+@testing.with_requires('scipy>=1.7.0', 'scipy<1.11.0')
+class TestMedFilt2d:
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-8, rtol=1e-8, scipy_name='scp',
+                                 accept_error=ValueError)  # for even kernels
+    def test_medfilt2d(self, xp, scp, dtype):
+        if sys.platform == 'win32':
+            pytest.xfail('medfilt2d broken for Scipy 1.7.0 in windows')
+        input = testing.shaped_random(self.input, xp, dtype)
+        kernel_size = self.kernel_size
+        return scp.signal.medfilt2d(input, kernel_size)
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/__init__.py b/tests/cupyx_tests/scipy_tests/sparse_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/csgraph_tests/test_traversal.py b/tests/cupyx_tests/scipy_tests/sparse_tests/csgraph_tests/test_traversal.py
new file mode 100644
index 0000000..22c895e
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/csgraph_tests/test_traversal.py
@@ -0,0 +1,63 @@
+import unittest
+
+import numpy
+try:
+    import scipy.sparse  # NOQA
+    import scipy.sparse.csgraph  # NOQA
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+import cupyx.scipy.sparse.csgraph  # NOQA
+try:
+    import pylibcugraph  # NOQA
+    pylibcugraph_available = True
+except ImportError:
+    pylibcugraph_available = False
+from cupy import testing
+
+
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64'],
+    'm': [5, 10, 20],
+    'nnz_per_row': [1, 2, 3],
+    'directed': [True, False],
+    'connection': ['weak', 'strong'],
+    'return_labels': [True, False],
+}))
+@unittest.skipUnless(scipy_available and pylibcugraph_available,
+                     'requires scipy and pylibcugraph')
+class TestConnectedComponents(unittest.TestCase):
+
+    def _make_matrix(self, dtype, xp):
+        shape = (self.m, self.m)
+        density = self.nnz_per_row / self.m
+        a = testing.shaped_random(shape, xp, dtype=dtype, scale=1)
+        a = a / density
+        a[a > 1] = 0
+        return a
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_connected_components(self, xp, sp):
+        a = self._make_matrix(self.dtype, xp)
+        a = sp.csr_matrix(a)
+        if self.return_labels:
+            n, labels = sp.csgraph.connected_components(
+                a, directed=self.directed, connection=self.connection,
+                return_labels=self.return_labels)
+            # Note: CuPy returns un-ordered results in both strong and
+            # weak connection case while SciPy returns un-ordered labels
+            # in strong connection case therefore. Since in most cases
+            # the labels returned by both cuPy and Scipy are un-ordered
+            # therefore, always adjust the labels.
+            table = xp.zeros((n,), dtype=numpy.int32) - 1
+            j = 0
+            for i in range(labels.size):
+                if table[labels[i]] < 0:
+                    table[labels[i]] = j
+                    j = j + 1
+                labels[i] = table[labels[i]]
+            return n, labels
+        else:
+            return sp.csgraph.connected_components(
+                a, directed=self.directed, connection=self.connection,
+                return_labels=self.return_labels)
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_base.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_base.py
new file mode 100644
index 0000000..1c46291
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_base.py
@@ -0,0 +1,84 @@
+import unittest
+
+import pytest
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+from cupy import testing
+from cupyx.scipy import sparse
+
+
+if scipy_available:
+    class DummySparseCPU(scipy.sparse.spmatrix):
+
+        def __init__(self, maxprint=50, shape=None, nnz=0):
+            super(DummySparseCPU, self).__init__(maxprint)
+            self._shape = shape
+            self._nnz = nnz
+
+        def getnnz(self):
+            return self._nnz
+
+
+class DummySparseGPU(sparse.spmatrix):
+
+    def __init__(self, maxprint=50, shape=None, nnz=0):
+        super(DummySparseGPU, self).__init__(maxprint)
+        self._shape = shape
+        self._nnz = nnz
+
+    def get_shape(self):
+        return self._shape
+
+    def getnnz(self):
+        return self._nnz
+
+
+@testing.with_requires('scipy<1.11')
+class TestSpmatrix(unittest.TestCase):
+
+    def dummy_class(self, sp):
+        if sp is sparse:
+            return DummySparseGPU
+        else:
+            return DummySparseCPU
+
+    def test_instantiation(self):
+        for sp in (scipy.sparse, sparse):
+            with pytest.raises(ValueError):
+                sp.spmatrix()
+
+    def test_len(self):
+        for sp in (scipy.sparse, sparse):
+            s = self.dummy_class(sp)()
+            with pytest.raises(TypeError):
+                len(s)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_bool_true(self, xp, sp):
+        s = self.dummy_class(sp)(shape=(1, 1), nnz=1)
+        return bool(s)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_bool_false(self, xp, sp):
+        s = self.dummy_class(sp)(shape=(1, 1), nnz=0)
+        return bool(s)
+
+    def test_bool_invalid(self):
+        for sp in (scipy.sparse, sparse):
+            s = self.dummy_class(sp)(shape=(2, 1))
+            with pytest.raises(ValueError):
+                bool(s)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_asformat_none(self, xp, sp):
+        s = self.dummy_class(sp)()
+        assert s.asformat(None) is s
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_maxprint(self, xp, sp):
+        s = self.dummy_class(sp)(maxprint=30)
+        return s.getmaxprint()
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_construct.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_construct.py
new file mode 100644
index 0000000..9ee0b77
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_construct.py
@@ -0,0 +1,476 @@
+import functools
+import re
+from unittest import mock
+
+import numpy
+import pytest
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+import cupy
+from cupy import testing
+from cupy.cuda import runtime
+from cupyx.scipy import sparse
+from cupyx.scipy.sparse import _construct
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'format': ['csr', 'csc', 'coo'],
+    'm': [3],
+    'n': [None, 3, 2],
+    'k': [0, 1],
+}))
+@testing.with_requires('scipy')
+class TestEye:
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_eye(self, xp, sp):
+        x = sp.eye(
+            self.m, n=self.n, k=self.k, dtype=self.dtype, format=self.format)
+        assert isinstance(x, sp.spmatrix)
+        assert x.format == self.format
+        return x
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'format': ['csr', 'csc', 'coo'],
+}))
+@testing.with_requires('scipy')
+class TestIdentity:
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_eye(self, xp, sp):
+        x = sp.identity(3, dtype=self.dtype, format=self.format)
+        assert isinstance(x, sp.spmatrix)
+        assert x.format == self.format
+        return x
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestSpdiags:
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_spdiags(self, xp, sp):
+        data = xp.arange(12, dtype=self.dtype).reshape(3, 4)
+        diags = xp.array([0, -1, 2], dtype='i')
+        x = sp.spdiags(data, diags, 3, 4)
+        return x
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64]
+}))
+class TestVstack:
+
+    def data(self):
+
+        A = sparse.coo_matrix((cupy.asarray([1.0, 2.0, 3.0, 4.0]),
+                               (cupy.asarray([0, 0, 1, 1]),
+                                cupy.asarray([0, 1, 0, 1]))))
+        B = sparse.coo_matrix((cupy.asarray([5.0, 6.0]),
+                               (cupy.asarray([0, 0]),
+                                cupy.asarray([0, 1]))))
+
+        return A, B
+
+    def expected(self):
+
+        return cupy.asarray([[1, 2],
+                             [3, 4],
+                             [5, 6]], self.dtype)
+
+    def test_basic_vstack(self):
+
+        A, B = self.data()
+
+        actual = _construct.vstack([A, B]).todense()
+        testing.assert_array_equal(actual, self.expected())
+
+    def test_dtype(self):
+
+        A, B = self.data()
+
+        actual = _construct.vstack([A, B], dtype=self.dtype)
+        assert actual.dtype == self.dtype
+
+    def test_csr(self):
+
+        A, B = self.data()
+
+        actual = _construct.vstack([A.tocsr(), B.tocsr()]).todense()
+        testing.assert_array_equal(actual, self.expected())
+
+    def test_csr_with_dtype(self):
+
+        A, B = self.data()
+
+        actual = _construct.vstack([A.tocsr(), B.tocsr()],
+                                   dtype=self.dtype)
+        assert actual.dtype == self.dtype
+        assert actual.indices.dtype == cupy.int32
+        assert actual.indptr.dtype == cupy.int32
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64]
+}))
+class TestHstack:
+
+    def data(self):
+
+        A = sparse.coo_matrix((cupy.asarray([1.0, 2.0, 3.0, 4.0]),
+                               (cupy.asarray([0, 0, 1, 1]),
+                                cupy.asarray([0, 1, 0, 1]))))
+        B = sparse.coo_matrix((cupy.asarray([5.0, 6.0]),
+                               (cupy.asarray([0, 1]),
+                                cupy.asarray([0, 0]))))
+
+        return A, B
+
+    def expected(self):
+
+        return cupy.asarray([[1, 2, 5],
+                             [3, 4, 6]])
+
+    def test_basic_hstack(self):
+
+        A, B = self.data()
+        actual = _construct.hstack([A, B], dtype=self.dtype).todense()
+        testing.assert_array_equal(actual, self.expected())
+        assert actual.dtype == self.dtype
+
+    def test_csc(self):
+        A, B = self.data()
+        actual = _construct.hstack([A.tocsc(), B.tocsc()],
+                                   dtype=self.dtype).todense()
+        testing.assert_array_equal(actual, self.expected())
+        assert actual.dtype == self.dtype
+
+    def test_csc_with_dtype(self):
+
+        A, B = self.data()
+
+        actual = _construct.hstack([A.tocsc(), B.tocsc()],
+                                   dtype=self.dtype)
+        assert actual.indices.dtype == cupy.int32
+        assert actual.indptr.dtype == cupy.int32
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64]
+}))
+class TestBmat:
+
+    def data(self):
+        A = sparse.csr_matrix(cupy.asarray([[1, 2], [3, 4]],
+                                           self.dtype)).tocoo()
+        B = sparse.csr_matrix(cupy.asarray([[5], [6]],
+                                           self.dtype)).tocoo()
+        C = sparse.csr_matrix(cupy.asarray([[7]],
+                                           self.dtype)).tocoo()
+        D = sparse.coo_matrix((0, 0), dtype=self.dtype)
+
+        return A, B, C, D
+
+    def test_basic_inputs(self):
+
+        A, B, C, D = self.data()
+
+        expected = cupy.asarray([[1, 2, 5],
+                                 [3, 4, 6],
+                                 [0, 0, 7]], dtype=self.dtype)
+
+        testing.assert_array_equal(
+            _construct.bmat([[A, B], [None, C]]).todense(), expected
+        )
+
+        expected = cupy.asarray([[1, 2, 0],
+                                 [3, 4, 0],
+                                 [0, 0, 7]])
+        testing.assert_array_equal(
+            _construct.bmat([[A, None], [None, C]]).todense(), expected
+        )
+
+        expected = cupy.asarray([[0, 5],
+                                 [0, 6],
+                                 [7, 0]])
+
+        testing.assert_array_equal(
+            _construct.bmat([[None, B], [C, None]]).todense(), expected
+        )
+
+    def test_empty(self):
+
+        A, B, C, D = self.data()
+
+        expected = cupy.empty((0, 0), dtype=self.dtype)
+        testing.assert_array_equal(_construct.bmat([[None, None]]).todense(),
+                                   expected)
+        testing.assert_array_equal(_construct.bmat([[None, D], [D, None]])
+                                   .todense(), expected)
+
+    def test_edge_cases(self):
+        """Catch-all for small edge cases"""
+
+        A, B, C, D = self.data()
+
+        expected = cupy.asarray([[7]], dtype=self.dtype)
+        testing.assert_array_equal(_construct.bmat([[None, D], [C, None]])
+                                   .todense(), expected)
+
+    def test_failure_cases(self):
+
+        A, B, C, D = self.data()
+
+        match = r'.*Got blocks\[{}\]\.shape\[{}\] == 1, expected 2'
+
+        # test failure cases
+        message1 = re.compile(match.format('1,0', '1'))
+        with pytest.raises(ValueError, match=message1):
+            _construct.bmat([[A], [B]], dtype=self.dtype)
+
+        message2 = re.compile(match.format('0,1', '0'))
+        with pytest.raises(ValueError, match=message2):
+            _construct.bmat([[A, C]], dtype=self.dtype)
+
+
+@testing.parameterize(*testing.product({
+    'random_method': ['random', 'rand'],
+    'dtype': [numpy.float32, numpy.float64],
+    'format': ['csr', 'csc', 'coo'],
+}))
+class TestRandom:
+
+    def test_random(self):
+        x = getattr(sparse, self.random_method)(
+            3, 4, density=0.1,
+            format=self.format, dtype=self.dtype)
+        assert x.shape == (3, 4)
+        assert x.dtype == self.dtype
+        assert x.format == self.format
+
+    def test_random_with_seed(self):
+        x = getattr(sparse, self.random_method)(
+            3, 4, density=0.1,
+            format=self.format, dtype=self.dtype,
+            random_state=1)
+        assert x.shape == (3, 4)
+        assert x.dtype == self.dtype
+        assert x.format == self.format
+
+        y = getattr(sparse, self.random_method)(
+            3, 4, density=0.1,
+            format=self.format, dtype=self.dtype,
+            random_state=1)
+
+        testing.assert_array_equal(x.toarray(), y.toarray())
+
+    def test_random_with_state(self):
+        state1 = cupy.random.RandomState(1)
+        x = getattr(sparse, self.random_method)(
+            3, 4, density=0.1,
+            format=self.format, dtype=self.dtype,
+            random_state=state1)
+        assert x.shape == (3, 4)
+        assert x.dtype == self.dtype
+        assert x.format == self.format
+
+        state2 = cupy.random.RandomState(1)
+        y = getattr(sparse, self.random_method)(
+            3, 4, density=0.1,
+            format=self.format, dtype=self.dtype,
+            random_state=state2)
+
+        testing.assert_array_equal(x.toarray(), y.toarray())
+
+    def test_random_with_data_rvs(self):
+        if self.random_method == 'rand':
+            pytest.skip('cupyx.scipy.sparse.rand does not support data_rvs')
+        data_rvs = mock.MagicMock(side_effect=cupy.zeros)
+        x = getattr(sparse, self.random_method)(
+            3, 4, density=0.1, data_rvs=data_rvs,
+            format=self.format, dtype=self.dtype)
+        assert x.shape == (3, 4)
+        assert x.dtype == self.dtype
+        assert x.format == self.format
+
+        assert data_rvs.call_count == 1
+        # Note that its value is generated randomly
+        assert isinstance(data_rvs.call_args[0][0], int)
+
+
+@testing.with_requires('scipy')
+class TestRandomInvalidArgument:
+
+    def test_too_small_density(self):
+        for sp in (scipy.sparse, sparse):
+            with pytest.raises(ValueError):
+                sp.random(3, 4, density=-0.1)
+
+    def test_too_large_density(self):
+        for sp in (scipy.sparse, sparse):
+            with pytest.raises(ValueError):
+                sp.random(3, 4, density=1.1)
+
+    def test_invalid_dtype(self):
+        # Note: SciPy 1.12+ accepts integer.
+        with pytest.raises(NotImplementedError):
+            sparse.random(3, 4, dtype='i')
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'format': ['dia', 'csr', 'csc', 'coo'],
+}))
+@testing.with_requires('scipy')
+class TestDiags:
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_diags_scalar_offset(self, xp, sp):
+        x = sp.diags(
+            xp.arange(16), offsets=0, dtype=self.dtype, format=self.format)
+        assert isinstance(x, sp.spmatrix)
+        assert x.format == self.format
+        return x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_diags_single_element_lists(self, xp, sp):
+        x = sp.diags(
+            [xp.arange(16)], offsets=[0], dtype=self.dtype, format=self.format)
+        assert isinstance(x, sp.spmatrix)
+        assert x.format == self.format
+        return x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_diags_multiple(self, xp, sp):
+        x = sp.diags(
+            [xp.arange(15), xp.arange(16), xp.arange(15), xp.arange(13)],
+            offsets=[-1, 0, 1, 3],
+            dtype=self.dtype, format=self.format)
+        assert isinstance(x, sp.spmatrix)
+        assert x.format == self.format
+        return x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_diags_offsets_as_array(self, xp, sp):
+        x = sp.diags(
+            [xp.arange(15), xp.arange(16), xp.arange(15), xp.arange(13)],
+            offsets=xp.array([-1, 0, 1, 3]),
+            dtype=self.dtype, format=self.format)
+        assert isinstance(x, sp.spmatrix)
+        assert x.format == self.format
+        return x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_diags_non_square(self, xp, sp):
+        x = sp.diags(
+            [xp.arange(5), xp.arange(3)],
+            offsets=[0, -2], shape=(5, 10),
+            dtype=self.dtype, format=self.format)
+        assert isinstance(x, sp.spmatrix)
+        assert x.format == self.format
+        return x
+
+
+# borrowed from scipy:
+_arrs_kron = [
+    [[0]],
+    [[-1]],
+    [[4]],
+    [[10]],
+    [[0], [0]],
+    [[0, 0]],
+    [[1, 2], [3, 4]],
+    [[0, 2], [5, 0]],
+    [[0, 2, -6], [8, 0, 14]],
+    [[5, 4], [0, 0], [6, 0]],
+    [[5, 4, 4], [1, 0, 0], [6, 0, 8]],
+    [[0, 1, 0, 2, 0, 5, 8]],
+    [[0.5, 0.125, 0, 3.25], [0, 2.5, 0, 0]], ]
+
+
+def skip_HIP_0_size_matrix():
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kw):
+            try:
+                impl(self, *args, **kw)
+            except AssertionError as e:
+                if runtime.is_hip:
+                    assert 'ValueError: hipSPARSE' in str(e)
+                    pytest.xfail('may be buggy')
+                raise
+        return test_func
+    return decorator
+
+
+@testing.parameterize(*testing.product({
+    'dtype': (numpy.float32, numpy.float64, numpy.complex64, numpy.complex128),
+    'format': ('csr', 'csc', 'coo'),
+    'arrA': _arrs_kron,
+    'arrB': _arrs_kron,
+}))
+@testing.with_requires('scipy>=1.6')
+class TestKron:
+
+    def _make_sp_mat(self, xp, sp, arr, dtype):
+        a = xp.array(arr, dtype=dtype)
+        a = sp.csr_matrix(a)
+        return a
+
+    @skip_HIP_0_size_matrix()
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_kron(self, xp, sp):
+        a = self._make_sp_mat(xp, sp, self.arrA, self.dtype)
+        b = self._make_sp_mat(xp, sp, self.arrB, self.dtype)
+        kron = sp.kron(a, b, format=self.format)
+        assert kron.shape == (a.shape[0] * b.shape[0], a.shape[1] * b.shape[1])
+        assert kron.nnz == a.nnz * b.nnz
+        return kron
+
+    # TODO(leofang): check oversize inputs as in scipy/scipy#11879 after
+    # #3513 is fixed
+
+
+_arrs_kronsum = [
+    [[0]],
+    [[-1]],
+    [[4]],
+    [[10]],
+    [[1, 2], [3, 4]],
+    [[0, 2], [5, 0]],
+    [[0, 2, -6], [8, 0, 14], [0, 3, 0]],
+    [[5, 4, 4], [1, 0, 0], [6, 0, 8]]]
+
+
+@testing.parameterize(*testing.product({
+    'dtype': (numpy.float32, numpy.float64, numpy.complex64, numpy.complex128),
+    'format': ('csr', 'csc', 'coo'),
+    'arrA': _arrs_kronsum,
+    'arrB': _arrs_kronsum,
+}))
+@testing.with_requires('scipy>=1.6')
+class TestKronsum:
+
+    def _make_sp_mat(self, xp, sp, arr, dtype):
+        a = xp.array(arr, dtype=dtype)
+        a = sp.csr_matrix(a)
+        return a
+
+    @skip_HIP_0_size_matrix()
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_kronsum(self, xp, sp):
+        a = self._make_sp_mat(xp, sp, self.arrA, self.dtype)
+        b = self._make_sp_mat(xp, sp, self.arrB, self.dtype)
+        kronsum = sp.kronsum(a, b, format=self.format)
+        assert kronsum.shape == (a.shape[0] * b.shape[0],
+                                 a.shape[1] * b.shape[1])
+        return kronsum
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_coo.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_coo.py
new file mode 100644
index 0000000..29b9f39
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_coo.py
@@ -0,0 +1,1191 @@
+import pickle
+import sys
+
+import numpy
+import pytest
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+import cupy
+from cupy import testing
+from cupy.cuda import driver
+from cupy.cuda import runtime
+import cupyx.cusparse
+from cupyx.scipy import sparse
+
+
+def _make(xp, sp, dtype):
+    data = xp.array([0, 1, 2, 3], dtype)
+    row = xp.array([0, 0, 1, 2], 'i')
+    col = xp.array([0, 1, 3, 2], 'i')
+    # 0, 1, 0, 0
+    # 0, 0, 0, 2
+    # 0, 0, 3, 0
+    return sp.coo_matrix((data, (row, col)), shape=(3, 4))
+
+
+def _make_complex(xp, sp, dtype):
+    data = xp.array([0, 1, 2, 3], dtype)
+    if dtype in [numpy.complex64, numpy.complex128]:
+        data = data - 1j
+    row = xp.array([0, 0, 1, 2], 'i')
+    col = xp.array([0, 1, 3, 2], 'i')
+    # 0, 1 - 1j, 0, 0
+    # 0, 0, 0, 2 - 1j
+    # 0, 0, 3 - 1j, 0
+    return sp.coo_matrix((data, (row, col)), shape=(3, 4))
+
+
+def _make2(xp, sp, dtype):
+    data = xp.array([1, 2, 3, 4], dtype)
+    row = xp.array([0, 1, 1, 2], 'i')
+    col = xp.array([2, 1, 2, 2], 'i')
+    # 0, 0, 1, 0
+    # 0, 2, 3, 0
+    # 0, 0, 4, 0
+    return sp.coo_matrix((data, (row, col)), shape=(3, 4))
+
+
+def _make3(xp, sp, dtype):
+    data = xp.array([1, 2, 3, 4, 5], dtype)
+    row = xp.array([0, 1, 1, 3, 3], 'i')
+    col = xp.array([0, 2, 1, 0, 2], 'i')
+    # 1, 0, 0
+    # 0, 3, 2
+    # 0, 0, 0
+    # 4, 0, 5
+    return sp.coo_matrix((data, (row, col)), shape=(4, 3))
+
+
+def _make_unordered(xp, sp, dtype):
+    data = xp.array([1, 4, 3, 2], dtype)
+    row = xp.array([0, 2, 1, 0], 'i')
+    col = xp.array([0, 2, 3, 1], 'i')
+    # 1, 2, 0, 0
+    # 0, 0, 0, 3
+    # 0, 0, 4, 0
+    return sp.coo_matrix((data, (row, col)), shape=(3, 4))
+
+
+def _make_duplicate(xp, sp, dtype):
+    data = xp.array([0, 1, 2, 3, 4, 5], dtype)
+    row = xp.array([1, 1, 1, 1, 0, 1], 'i')
+    col = xp.array([0, 0, 2, 0, 0, 2], 'i')
+    # 4, 0, 0, 0
+    # 4, 0, 7, 0
+    # 0, 0, 0, 0
+    return sp.coo_matrix((data, (row, col)), shape=(3, 4))
+
+
+def _make_empty(xp, sp, dtype):
+    data = xp.array([], dtype)
+    row = xp.array([], 'i')
+    col = xp.array([], 'i')
+    return sp.coo_matrix((data, (row, col)), shape=(3, 4))
+
+
+def _make_square(xp, sp, dtype):
+    data = xp.array([0, 1, 2, 3], dtype)
+    row = xp.array([0, 0, 1, 2], 'i')
+    col = xp.array([0, 2, 0, 2], 'i')
+    # 0, 1, 0
+    # 2, 0, 0
+    # 0, 0, 3
+    return sp.coo_matrix((data, (row, col)), shape=(3, 3))
+
+
+def _make_shape(xp, sp, dtype):
+    return sp.coo_matrix((3, 4))
+
+
+def _make_sum_dup(xp, sp, dtype):
+    # 1 0 0
+    # 1 1 0
+    # 1 1 1
+    data = xp.array([1, 1, 1, 1, 1, 1], dtype)
+    row = xp.array([0, 1, 1, 2, 2, 2], 'i')
+    col = xp.array([0, 0, 1, 0, 1, 2], 'i')
+    return sp.coo_matrix((data, (row, col)), shape=(3, 3))
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+class TestCooMatrix:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.m = _make(cupy, sparse, self.dtype)
+
+    def test_dtype(self):
+        assert self.m.dtype == self.dtype
+
+    def test_data(self):
+        assert self.m.data.dtype == self.dtype
+        testing.assert_array_equal(
+            self.m.data, cupy.array([0, 1, 2, 3], self.dtype))
+
+    def test_row(self):
+        assert self.m.row.dtype == numpy.int32
+        testing.assert_array_equal(
+            self.m.row, cupy.array([0, 0, 1, 2], self.dtype))
+
+    def test_col(self):
+        assert self.m.col.dtype == numpy.int32
+        testing.assert_array_equal(
+            self.m.col, cupy.array([0, 1, 3, 2], self.dtype))
+
+    def test_init_copy(self):
+        n = sparse.coo_matrix(self.m)
+        assert n is not self.m
+        cupy.testing.assert_array_equal(n.toarray(), self.m.toarray())
+
+    def test_init_copy_other_sparse(self):
+        n = sparse.coo_matrix(self.m.tocsr())
+        cupy.testing.assert_array_equal(n.toarray(), self.m.toarray())
+
+    @testing.with_requires('scipy')
+    def test_init_copy_scipy_sparse(self):
+        m = _make(numpy, scipy.sparse, self.dtype)
+        n = sparse.coo_matrix(m)
+        assert isinstance(n.data, cupy.ndarray)
+        assert isinstance(n.row, cupy.ndarray)
+        assert isinstance(n.col, cupy.ndarray)
+        cupy.testing.assert_array_equal(n.data, m.data)
+        cupy.testing.assert_array_equal(n.row, m.row)
+        cupy.testing.assert_array_equal(n.col, m.col)
+        assert n.shape == m.shape
+
+    @testing.with_requires('scipy')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_init_copy_other_scipy_sparse(self, xp, sp):
+        m = _make(numpy, scipy.sparse, self.dtype)
+        n = sp.coo_matrix(m.tocsc())
+        assert len(n.data) == len(m.data)
+        assert len(n.row) == len(m.row)
+        assert len(n.col) == len(m.col)
+        assert n.shape == m.shape
+        return n
+
+    def test_pickle_roundtrip(self):
+        s = _make(cupy, sparse, self.dtype)
+        s2 = pickle.loads(pickle.dumps(s))
+        assert s.shape == s2.shape
+        assert s.dtype == s2.dtype
+        if scipy_available:
+            assert (s.get() != s2.get()).count_nonzero() == 0
+
+    def test_shape(self):
+        assert self.m.shape == (3, 4)
+
+    def test_ndim(self):
+        assert self.m.ndim == 2
+
+    def test_nnz(self):
+        assert self.m.nnz == 4
+
+    def test_conj(self):
+        n = _make_complex(cupy, sparse, self.dtype)
+        cupy.testing.assert_array_equal(n.conj().data, n.data.conj())
+
+    def test_has_canonical_format(self):
+        assert self.m.has_canonical_format is False
+
+    @testing.with_requires('scipy')
+    def test_get(self):
+        m = self.m.get()
+        assert isinstance(m, scipy.sparse.coo_matrix)
+        expect = [
+            [0, 1, 0, 0],
+            [0, 0, 0, 2],
+            [0, 0, 3, 0]
+        ]
+        numpy.testing.assert_allclose(m.toarray(), expect)
+
+    @testing.with_requires('scipy')
+    def test_str(self):
+        if numpy.dtype(self.dtype).kind == 'f':
+            expect = '''  (0, 0)\t0.0
+  (0, 1)\t1.0
+  (1, 3)\t2.0
+  (2, 2)\t3.0'''
+        elif numpy.dtype(self.dtype).kind == 'c':
+            expect = '''  (0, 0)\t0j
+  (0, 1)\t(1+0j)
+  (1, 3)\t(2+0j)
+  (2, 2)\t(3+0j)'''
+        assert str(self.m) == expect
+
+    def test_toarray(self):
+        m = self.m.toarray()
+        expect = [
+            [0, 1, 0, 0],
+            [0, 0, 0, 2],
+            [0, 0, 3, 0]
+        ]
+        cupy.testing.assert_allclose(m, expect)
+
+    # reshape
+    def test_reshape_0(self):
+        assert self.m.reshape((12, 1)).shape == (12, 1)
+
+    def test_reshape_1(self):
+        m = self.m.reshape((1, 12)).toarray()
+        expect = [[0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0]]
+        cupy.testing.assert_allclose(m, expect)
+
+    def test_reshape_2(self):
+        m = self.m.reshape((1, 12), order='F').toarray()
+        expect = [[1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0]]
+        cupy.testing.assert_allclose(m, expect)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCooMatrixInit:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.shape = (3, 4)
+
+    def data(self, xp):
+        return xp.array([0, 1, 2, 3], self.dtype)
+
+    def row(self, xp):
+        return xp.array([0, 0, 1, 2], 'i')
+
+    def col(self, xp):
+        return xp.array([0, 1, 3, 2], 'i')
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_shape_none(self, xp, sp):
+        x = sp.coo_matrix(
+            (self.data(xp), (self.row(xp), self.col(xp))), shape=None)
+        assert x.shape == (3, 4)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_dtype(self, xp, sp):
+        data = self.data(xp).real.astype('i')
+        x = sp.coo_matrix(
+            (data, (self.row(xp), self.col(xp))), dtype=self.dtype)
+        assert x.dtype == self.dtype
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_copy_true(self, xp, sp):
+        data = self.data(xp)
+        row = self.row(xp)
+        col = self.col(xp)
+        x = sp.coo_matrix((data, (row, col)), copy=True)
+
+        assert data is not x.data
+        assert row is not x.row
+        assert col is not x.col
+
+    def test_init_dense(self):
+        m = cupy.array([[0, 1, 0, 2],
+                        [0, 0, 0, 0],
+                        [0, 0, 3, 0]], dtype=self.dtype)
+        n = sparse.coo_matrix(m)
+        assert n.nnz == 3
+        assert n.shape == (3, 4)
+        cupy.testing.assert_array_equal(n.data, [1, 2, 3])
+        cupy.testing.assert_array_equal(n.row, [0, 0, 2])
+        cupy.testing.assert_array_equal(n.col, [1, 3, 2])
+
+    def test_init_dense_allzero(self):
+        m = cupy.array([[0, 0, 0, 0],
+                        [0, 0, 0, 0],
+                        [0, 0, 0, 0]], dtype=self.dtype)
+        n = sparse.coo_matrix(m)
+        assert n.nnz == 0
+        assert n.shape == (3, 4)
+        cupy.testing.assert_array_equal(n.data, [])
+        cupy.testing.assert_array_equal(n.row, [])
+        cupy.testing.assert_array_equal(n.col, [])
+
+    def test_init_dense_check_if_row_major(self):
+        rows, cols = 10, 9
+        for order in ('C', 'F'):
+            d = testing.shaped_random((rows, cols), dtype=self.dtype,
+                                      order=order)
+            mask = testing.shaped_random((rows, cols), scale=1.0)
+            d[mask > 0.5] = 0
+            s = sparse.coo_matrix(d)
+            for i in range(s.nnz):
+                assert 0 <= s.row[i] < rows
+                assert 0 <= s.col[i] < cols
+                assert s.data[i] == d[s.row[i], s.col[i]]
+                if i == 0:
+                    continue
+                assert ((s.row[i-1] < s.row[i]) or
+                        (s.row[i-1] == s.row[i] and s.col[i-1] < s.col[i]))
+            assert s.has_canonical_format
+
+    def test_invalid_format(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(TypeError):
+                sp.coo_matrix(
+                    (self.data(xp), self.row(xp)), shape=self.shape)
+
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=1e-5)
+    def test_intlike_shape(self, xp, sp):
+        s = sp.coo_matrix((self.data(xp), (self.row(xp), self.col(xp))),
+                          shape=(xp.array(self.shape[0]),
+                                 xp.int32(self.shape[1])))
+        assert isinstance(s.shape[0], int)
+        assert isinstance(s.shape[1], int)
+        return s
+
+    def test_shape_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.coo_matrix(
+                    (self.data(xp), (self.row(xp), self.col(xp))),
+                    shape=(2,))
+
+    def test_data_invalid(self):
+        with pytest.raises(ValueError):
+            sparse.coo_matrix(
+                ('invalid', (self.row(cupy), self.col(cupy))),
+                shape=self.shape)
+
+    def test_data_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.coo_matrix(
+                    (self.data(xp)[None], (self.row(xp), self.col(xp))),
+                    shape=self.shape)
+
+    def test_row_invalid(self):
+        with pytest.raises(ValueError):
+            sparse.coo_matrix(
+                (self.data(cupy), ('invalid', self.col(cupy))),
+                shape=self.shape)
+
+    def test_row_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.coo_matrix(
+                    (self.data(xp), (self.row(xp)[None], self.col(xp))),
+                    shape=self.shape)
+
+    def test_col_invalid(self):
+        with pytest.raises(ValueError):
+            sparse.coo_matrix(
+                (self.data(cupy), (self.row(cupy), 'invalid')),
+                shape=self.shape)
+
+    def test_col_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.coo_matrix(
+                    (self.data(xp), (self.row(xp), self.col(xp)[None])),
+                    shape=self.shape)
+
+    def test_data_different_length(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            data = xp.arange(5, dtype=self.dtype)
+            with pytest.raises(TypeError):
+                sp.coo_matrix(
+                    (data(xp), (self.row(xp), self.col(xp))),
+                    shape=self.shape)
+
+    def test_row_different_length(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            row = xp.arange(5, dtype=self.dtype)
+            with pytest.raises(TypeError):
+                sp.coo_matrix(
+                    (self.data(xp), (row(xp), self.col(xp))),
+                    shape=self.shape)
+
+    def test_col_different_length(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            col = xp.arange(5, dtype=self.dtype)
+            with pytest.raises(TypeError):
+                sp.coo_matrix(
+                    (self.data(xp), (self.row(xp), col(xp))),
+                    shape=self.shape)
+
+    def test_fail_to_infer_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            data = xp.array([], dtype=self.dtype)
+            row = xp.array([], dtype='i')
+            col = xp.array([], dtype='i')
+            with pytest.raises(ValueError):
+                sp.coo_matrix((data, (row, col)), shape=None)
+
+    def test_row_too_large(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            row = xp.array([0, 0, 1, 3], 'i')
+            with pytest.raises(ValueError):
+                sp.coo_matrix(
+                    (self.data(xp), (row, self.col(xp))),
+                    shape=self.shape)
+
+    def test_row_too_small(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            row = xp.array([0, -1, 1, 2], 'i')
+            with pytest.raises(ValueError):
+                sp.coo_matrix(
+                    (self.data(xp), (row, self.col(xp))),
+                    shape=self.shape)
+
+    def test_col_too_large(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            col = xp.array([0, 1, 4, 2], 'i')
+            with pytest.raises(ValueError):
+                sp.coo_matrix(
+                    (self.data(xp), (self.row(xp), col)),
+                    shape=self.shape)
+
+    def test_col_too_small(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            col = xp.array([0, -1, 3, 2], 'i')
+            with pytest.raises(ValueError):
+                sp.coo_matrix(
+                    (self.data(xp), (self.row(xp), col)),
+                    shape=self.shape)
+
+    def test_unsupported_dtype(self):
+        with pytest.raises(ValueError):
+            sparse.coo_matrix(
+                (self.data(cupy), (self.row(cupy), self.col(cupy))),
+                shape=self.shape, dtype='i')
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_conj(self, xp, sp):
+        n = _make_complex(xp, sp, self.dtype)
+        cupy.testing.assert_array_equal(n.conj().data, n.data.conj())
+
+
+@testing.parameterize(*testing.product({
+    'make_method': [
+        '_make', '_make_unordered', '_make_empty', '_make_duplicate',
+        '_make_shape'],
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCooMatrixScipyComparison:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if (sys.platform == 'win32' and
+                cupyx.cusparse.getVersion() == 11301 and
+                self.dtype == cupy.complex128):
+            pytest.xfail('Known to fail on CUDA 11.2 for Windows')
+
+    @property
+    def make(self):
+        return globals()[self.make_method]
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_dtype(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.dtype
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_nnz(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.getnnz()
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_asfptype(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.asfptype()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_toarray(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_A(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.A
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocoo(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.tocoo()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocoo_copy(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = m.tocoo(copy=True)
+        assert m.data is not n.data
+        assert m.row is not n.row
+        assert m.col is not n.col
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        out = m.tocsc()
+        assert out.has_canonical_format
+        return out
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsc_copy(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = m.tocsc(copy=True)
+        assert m.data is not n.data
+        assert n.has_canonical_format
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        out = m.tocsr()
+        assert out.has_canonical_format
+        return out
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsr_copy(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = m.tocsr(copy=True)
+        assert m.data is not n.data
+        assert n.has_canonical_format
+        return n
+
+    # dot
+    @testing.with_requires('scipy!=1.8.0')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.dot(2.0)
+
+    @testing.with_requires('scipy!=1.8.0')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_numpy_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.dot(numpy.dtype(self.dtype).type(2.0))
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_csr(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return m.dot(x)
+
+    def test_dot_csr_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = sp.csr_matrix((5, 3), dtype=self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_csc(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return m.dot(x)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_sparse(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return m.dot(x)
+
+    @testing.with_requires('scipy>=1.8.0rc1')
+    def test_dot_zero_dim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = xp.array(2, dtype=self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_dense_vector(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = xp.arange(4).astype(self.dtype)
+        return m.dot(x)
+
+    def test_dot_dense_vector_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = xp.arange(5).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_dense_matrix(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = xp.arange(8).reshape(4, 2).astype(self.dtype)
+        return m.dot(x)
+
+    def test_dot_dense_matrix_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = xp.arange(10).reshape(5, 2).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    def test_dot_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    def test_dot_unsupported(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.dot(None)
+
+    # __add__
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_add_zero(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m + 0
+
+    def test_add_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                m + 1
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_csr(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype)
+        return m + n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_coo(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype).tocoo()
+        return m + n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_dense(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return m + n
+
+    # __radd__
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_radd_zero(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return 0 + m
+
+    def test_radd_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                1 + m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_radd_dense(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return n + m
+
+    # __sub__
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_sub_zero(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m - 0
+
+    def test_sub_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                m - 1
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_csr(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype)
+        return m - n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_coo(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype).tocoo()
+        return m - n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_dense(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return m - n
+
+    # __rsub__
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_rsub_zero(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return 0 - m
+
+    def test_rsub_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                1 - m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rsub_dense(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return n - m
+
+    # __mul__
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_mul_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m * 2.0
+
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_mul_numpy_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m * numpy.dtype(self.dtype).type(2.0)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_csr(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return m * x
+
+    def test_mul_csr_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = sp.csr_matrix((5, 3), dtype=self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_csc(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_sparse(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_mul_zero_dim(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = xp.array(2, dtype=self.dtype)
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_dense_vector(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = xp.arange(4).astype(self.dtype)
+        return m * x
+
+    def test_mul_dense_vector_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = xp.arange(5).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_dense_matrix(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = xp.arange(8).reshape(4, 2).astype(self.dtype)
+        return m * x
+
+    def test_mul_dense_matrix_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = xp.arange(10).reshape(5, 2).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    def test_mul_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    def test_mul_unsupported(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m * None
+
+    # __rmul__
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_rmul_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return 2.0 * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_rmul_numpy_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return numpy.dtype(self.dtype).type(2.0) * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_csr(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return x * m
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_rmul_csc(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return x * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_sparse(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return x * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_rmul_zero_dim(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = xp.array(2, dtype=self.dtype)
+        return x * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_dense_matrix(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        x = xp.arange(12).reshape(4, 3).astype(self.dtype)
+        return x * m
+
+    def test_rmul_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                x * m
+
+    @pytest.mark.xfail(
+        scipy_available and
+        numpy.lib.NumpyVersion(scipy.__version__) >= '1.8.0rc1' and
+        numpy.lib.NumpyVersion(scipy.__version__) <= '1.9.0rc1',
+        reason='See scipy/15210')
+    def test_rmul_unsupported(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                None * m
+
+    # Note: '@' operator is almost equivalent to '*' operator. Only test the
+    # cases where '@' raises an exception and '*' does not.
+    def test_matmul_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = 2.0
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    def test_matmul_numpy_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = numpy.dtype(self.dtype).type(2.0)
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    def test_matmul_scalar_like_array(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.array(2.0, self.dtype)
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    # __pow__
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_pow_0(self, xp, sp):
+        m = _make_square(xp, sp, self.dtype)
+        return m ** 0
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_pow_1(self, xp, sp):
+        m = _make_square(xp, sp, self.dtype)
+        return m ** 1
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_pow_2(self, xp, sp):
+        m = _make_square(xp, sp, self.dtype)
+        return m ** 2
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_pow_3(self, xp, sp):
+        m = _make_square(xp, sp, self.dtype)
+        return m ** 3
+
+    def test_pow_neg(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make_square(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m ** -1
+
+    def test_sum_tuple_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.sum(axis=(0, 1))
+
+    def test_sum_float_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.sum(axis=0.0)
+
+    def test_sum_too_large_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.sum(axis=3)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_transpose(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.transpose()
+
+    def test_transpose_axes_int(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.transpose(axes=0)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_eliminate_zeros(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.eliminate_zeros()
+        return m.nnz
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'ret_dtype': [None, numpy.float32, numpy.float64],
+    'axis': [None, 0, 1, -1, -2],
+}))
+@testing.with_requires('scipy')
+class TestCooMatrixSum:
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.sum(axis=self.axis, dtype=self.ret_dtype)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_with_out(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if self.axis is None:
+            shape = ()
+        else:
+            shape = list(m.shape)
+            shape[self.axis] = 1
+            shape = tuple(shape)
+        out = xp.empty(shape, dtype=self.ret_dtype)
+        if xp is numpy:
+            # TODO(unno): numpy.matrix is used for scipy.sparse though
+            # cupy.ndarray is used for cupyx.scipy.sparse.
+            out = xp.asmatrix(out)
+        return m.sum(axis=self.axis, dtype=self.ret_dtype, out=out)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCooMatrixSumDuplicates:
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_duplicates(self, xp, sp):
+        m = _make_duplicate(xp, sp, self.dtype)
+        assert not m.has_canonical_format
+        m.sum_duplicates()
+        assert m.has_canonical_format
+        assert m.nnz == 3
+
+        m.sum_duplicates()
+        assert m.has_canonical_format
+        return m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_duplicates_canonical(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        assert not m.has_canonical_format
+        m.sum_duplicates()
+        assert m.has_canonical_format
+        assert m.nnz == 4
+        return m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_duplicates_empty(self, xp, sp):
+        m = _make_empty(xp, sp, self.dtype)
+        assert not m.has_canonical_format
+        m.sum_duplicates()
+        assert m.has_canonical_format
+        assert m.nnz == 0
+        return m
+
+    @testing.with_requires('scipy>=1.11.0')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_duplicates_compatibility(self, xp, sp):
+        m = _make_sum_dup(xp, sp, self.dtype)
+        row = m.row.copy()
+        col = m.col.copy()
+        assert not m.has_canonical_format
+        m.sum_duplicates()
+        assert m.has_canonical_format
+        testing.assert_array_equal(m.row, row)
+        testing.assert_array_equal(m.col, col)
+        assert m.has_canonical_format
+        return m
+
+    @testing.with_requires('scipy<1.11.0')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_duplicates_incompatibility(self, xp, sp):
+        # See #3620 and #3624. CuPy's and SciPy's COO indices could mismatch
+        # due to the order of lexsort, but the matrix is correct.
+        m = _make_sum_dup(xp, sp, self.dtype)
+        if xp is cupy:
+            sorted_first = m.row.copy()
+        else:
+            sorted_first = m.col.copy()
+        assert not m.has_canonical_format
+        m.sum_duplicates()
+        assert m.has_canonical_format
+        # Here we ensure this sorting order is not altered by future PRs...
+        sorted_first.sort()
+        if xp is cupy:
+            testing.assert_array_equal(m.row, sorted_first)
+        else:
+            testing.assert_array_equal(m.col, sorted_first)
+        assert m.has_canonical_format
+        # ...and now we make sure the dense matrix is the same
+        return m
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'ufunc': [
+        'arcsin', 'arcsinh', 'arctan', 'arctanh', 'ceil', 'deg2rad', 'expm1',
+        'floor', 'log1p', 'rad2deg', 'rint', 'sign', 'sin', 'sinh', 'sqrt',
+        'tan', 'tanh', 'trunc',
+    ],
+}))
+@testing.with_requires('scipy')
+class TestUfunc:
+
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=1e-5)
+    def test_ufun(self, xp, sp):
+        x = _make(xp, sp, self.dtype)
+        x.data *= 0.1
+        func = getattr(x, self.ufunc)
+        complex_unsupported = {'ceil', 'deg2rad', 'floor', 'rad2deg', 'trunc'}
+        if (numpy.dtype(self.dtype).kind == 'c' and
+                self.ufunc in complex_unsupported):
+            with pytest.raises(TypeError):
+                func()
+            return xp.array(0)
+        else:
+            return func()
+
+
+class TestIsspmatrixCoo:
+
+    def test_coo(self):
+        x = sparse.coo_matrix(
+            (cupy.array([0], 'f'),
+             (cupy.array([0], 'i'), cupy.array([0], 'i'))),
+            shape=(1, 1), dtype='f')
+        assert sparse.isspmatrix_coo(x) is True
+
+    def test_csr(self):
+        x = sparse.csr_matrix(
+            (cupy.array([], 'f'),
+             cupy.array([], 'i'),
+             cupy.array([0], 'i')),
+            shape=(0, 0), dtype='f')
+        assert sparse.isspmatrix_coo(x) is False
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(8, 5), (5, 5), (5, 8)],
+}))
+@testing.with_requires('scipy>=1.5.0')
+class TestCooMatrixDiagonal:
+    density = 0.5
+
+    def _make_matrix(self, dtype):
+        a = testing.shaped_random(self.shape, numpy, dtype=dtype)
+        mask = testing.shaped_random(self.shape, numpy, dtype='f', scale=1.0)
+        a[mask > self.density] = 0
+        scipy_a = scipy.sparse.coo_matrix(a)
+        cupyx_a = sparse.coo_matrix(cupy.array(a))
+        return scipy_a, cupyx_a
+
+    @testing.for_dtypes('fdFD')
+    def test_diagonal(self, dtype):
+        scipy_a, cupyx_a = self._make_matrix(dtype)
+        m, n = self.shape
+        for k in range(-m, n+1):
+            scipy_diag = scipy_a.diagonal(k=k)
+            cupyx_diag = cupyx_a.diagonal(k=k)
+            testing.assert_allclose(scipy_diag, cupyx_diag)
+        scipy_a.has_canonical_format = False
+        cupyx_a.has_canonical_format = False
+        for k in range(-m, n+1):
+            scipy_diag = scipy_a.diagonal(k=k)
+            cupyx_diag = cupyx_a.diagonal(k=k)
+            testing.assert_allclose(scipy_diag, cupyx_diag)
+
+    def _test_setdiag(self, scipy_a, cupyx_a, x, k):
+        scipy_a = scipy_a.copy()
+        cupyx_a = cupyx_a.copy()
+        scipy_a.setdiag(x, k=k)
+        cupyx_a.setdiag(cupy.array(x), k=k)
+        testing.assert_allclose(scipy_a.data, cupyx_a.data)
+        testing.assert_array_equal(scipy_a.row, cupyx_a.row)
+        testing.assert_array_equal(scipy_a.col, cupyx_a.col)
+
+    @testing.for_dtypes('fdFD')
+    def test_setdiag(self, dtype):
+        scipy_a, cupyx_a = self._make_matrix(dtype)
+        m, n = self.shape
+        for k in range(-m+1, n):
+            m_st, n_st = max(0, -k), max(0, k)
+            for d in (-1, 0, 1):
+                x_len = min(m - m_st, n - n_st) + d
+                if x_len <= 0:
+                    continue
+                x = numpy.ones((x_len,), dtype=dtype)
+                self._test_setdiag(scipy_a, cupyx_a, x, k)
+
+    @testing.for_dtypes('fdFD')
+    def test_setdiag_scalar(self, dtype):
+        scipy_a, cupyx_a = self._make_matrix(dtype)
+        x = numpy.array(1.0, dtype=dtype)
+        m, n = self.shape
+        for k in range(-m+1, n):
+            self._test_setdiag(scipy_a, cupyx_a, x, k)
+
+    def test_setdiag_invalid(self):
+        dtype = 'f'
+        scipy_a, cupyx_a = self._make_matrix(dtype)
+        x = numpy.array(1.0, dtype=dtype)
+        m, n = self.shape
+        for k in (-m, n):
+            with pytest.raises(ValueError):
+                scipy_a.setdiag(x, k=k)
+            with pytest.raises(ValueError):
+                cupyx_a.setdiag(x, k=k)
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_csc.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_csc.py
new file mode 100644
index 0000000..c902fc3
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_csc.py
@@ -0,0 +1,1560 @@
+import pickle
+import sys
+
+import numpy
+import pytest
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+import cupy
+from cupy import testing
+from cupy.cuda import driver
+from cupy.cuda import runtime
+import cupyx.cusparse
+from cupyx.scipy import sparse
+
+
+def _make(xp, sp, dtype):
+    data = xp.array([0, 1, 3, 2], dtype)
+    indices = xp.array([0, 0, 2, 1], 'i')
+    indptr = xp.array([0, 1, 2, 3, 4], 'i')
+    # 0, 1, 0, 0
+    # 0, 0, 0, 2
+    # 0, 0, 3, 0
+    return sp.csc_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_complex(xp, sp, dtype):
+    data = xp.array([0, 1, 2, 3], dtype)
+    if dtype in [numpy.complex64, numpy.complex128]:
+        data = data - 1j
+    indices = xp.array([0, 1, 3, 2], 'i')
+    indptr = xp.array([0, 2, 3, 4], 'i')
+    # 0, 1 - 1j, 0, 0
+    # 0, 0, 0, 2 - 1j
+    # 0, 0, 3 - 1j, 0
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make2(xp, sp, dtype):
+    data = xp.array([2, 1, 3, 4], dtype)
+    indices = xp.array([1, 0, 1, 2], 'i')
+    indptr = xp.array([0, 0, 1, 4, 4], 'i')
+    # 0, 0, 1, 0
+    # 0, 2, 3, 0
+    # 0, 0, 4, 0
+    return sp.csc_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make3(xp, sp, dtype):
+    data = xp.array([1, 4, 3, 2, 5], dtype)
+    indices = xp.array([0, 3, 1, 1, 3], 'i')
+    indptr = xp.array([0, 2, 3, 5], 'i')
+    # 1, 0, 0
+    # 0, 3, 2
+    # 0, 0, 0
+    # 4, 0, 5
+    return sp.csc_matrix((data, indices, indptr), shape=(4, 3))
+
+
+def _make_unordered(xp, sp, dtype):
+    data = xp.array([1, 2, 3, 4], dtype)
+    indices = xp.array([1, 0, 1, 2], 'i')
+    indptr = xp.array([0, 0, 0, 2, 4], 'i')
+    return sp.csc_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_duplicate(xp, sp, dtype):
+    data = xp.array([1, 4, 3, 0, 2, 5], dtype)
+    indices = xp.array([0, 1, 0, 2, 1, 1], 'i')
+    indptr = xp.array([0, 3, 4, 6, 6], 'i')
+    # 4, 0, 0, 0
+    # 4, 0, 7, 0
+    # 0, 0, 0, 0
+    return sp.csc_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_empty(xp, sp, dtype):
+    data = xp.array([], dtype)
+    indices = xp.array([], 'i')
+    indptr = xp.array([0, 0, 0, 0, 0], 'i')
+    return sp.csc_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_shape(xp, sp, dtype):
+    return sp.csc_matrix((3, 4))
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+class TestCscMatrix:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.m = _make(cupy, sparse, self.dtype)
+
+    def test_dtype(self):
+        assert self.m.dtype == self.dtype
+
+    def test_data(self):
+        assert self.m.data.dtype == self.dtype
+        testing.assert_array_equal(
+            self.m.data, cupy.array([0, 1, 3, 2], self.dtype))
+
+    def test_indices(self):
+        assert self.m.indices.dtype == numpy.int32
+        testing.assert_array_equal(
+            self.m.indices, cupy.array([0, 0, 2, 1], self.dtype))
+
+    def test_indptr(self):
+        assert self.m.indptr.dtype == numpy.int32
+        testing.assert_array_equal(
+            self.m.indptr, cupy.array([0, 1, 2, 3, 4], self.dtype))
+
+    def test_init_copy(self):
+        n = sparse.csc_matrix(self.m)
+        assert n is not self.m
+        cupy.testing.assert_array_equal(n.data, self.m.data)
+        cupy.testing.assert_array_equal(n.indices, self.m.indices)
+        cupy.testing.assert_array_equal(n.indptr, self.m.indptr)
+        assert n.shape == self.m.shape
+
+    def test_init_copy_other_sparse(self):
+        n = sparse.csc_matrix(self.m.tocsr())
+        cupy.testing.assert_array_equal(n.data, self.m.data)
+        cupy.testing.assert_array_equal(n.indices, self.m.indices)
+        cupy.testing.assert_array_equal(n.indptr, self.m.indptr)
+        assert n.shape == self.m.shape
+
+    @testing.with_requires('scipy')
+    def test_init_copy_scipy_sparse(self):
+        m = _make(numpy, scipy.sparse, self.dtype)
+        n = sparse.csc_matrix(m)
+        assert isinstance(n.data, cupy.ndarray)
+        assert isinstance(n.indices, cupy.ndarray)
+        assert isinstance(n.indptr, cupy.ndarray)
+        cupy.testing.assert_array_equal(n.data, m.data)
+        cupy.testing.assert_array_equal(n.indices, m.indices)
+        cupy.testing.assert_array_equal(n.indptr, m.indptr)
+        assert n.shape == m.shape
+
+    @testing.with_requires('scipy')
+    def test_init_copy_other_scipy_sparse(self):
+        m = _make(numpy, scipy.sparse, self.dtype)
+        n = sparse.csc_matrix(m.tocsr())
+        assert isinstance(n.data, cupy.ndarray)
+        assert isinstance(n.indices, cupy.ndarray)
+        assert isinstance(n.indptr, cupy.ndarray)
+        cupy.testing.assert_array_equal(n.data, m.data)
+        cupy.testing.assert_array_equal(n.indices, m.indices)
+        cupy.testing.assert_array_equal(n.indptr, m.indptr)
+        assert n.shape == m.shape
+
+    def test_init_dense(self):
+        m = cupy.array([[0, 1, 0, 2],
+                        [0, 0, 0, 0],
+                        [0, 0, 0, 3]], dtype=self.dtype)
+        n = sparse.csc_matrix(m)
+        assert n.nnz == 3
+        assert n.shape == (3, 4)
+        cupy.testing.assert_array_equal(n.data, [1, 2, 3])
+        cupy.testing.assert_array_equal(n.indices, [0, 0, 2])
+        cupy.testing.assert_array_equal(n.indptr, [0, 0, 1, 1, 3])
+
+    @pytest.mark.xfail(runtime.is_hip, reason='hipSPARSE handles nnz=0 badly')
+    def test_init_dense_empty(self):
+        m = cupy.array([[0, 0, 0, 0],
+                        [0, 0, 0, 0],
+                        [0, 0, 0, 0]], dtype=self.dtype)
+        n = sparse.csc_matrix(m)
+        assert n.nnz == 0
+        assert n.shape == (3, 4)
+        cupy.testing.assert_array_equal(n.data, [])
+        cupy.testing.assert_array_equal(n.indices, [])
+        cupy.testing.assert_array_equal(n.indptr, [0, 0, 0, 0, 0])
+
+    def test_init_dense_one_dim(self):
+        m = cupy.array([0, 1, 0, 2], dtype=self.dtype)
+        n = sparse.csc_matrix(m)
+        assert n.nnz == 2
+        assert n.shape == (1, 4)
+        cupy.testing.assert_array_equal(n.data, [1, 2])
+        cupy.testing.assert_array_equal(n.indices, [0, 0])
+        cupy.testing.assert_array_equal(n.indptr, [0, 0, 1, 1, 2])
+
+    def test_init_dense_zero_dim(self):
+        m = cupy.array(1, dtype=self.dtype)
+        n = sparse.csc_matrix(m)
+        assert n.nnz == 1
+        assert n.shape == (1, 1)
+        cupy.testing.assert_array_equal(n.data, [1])
+        cupy.testing.assert_array_equal(n.indices, [0])
+        cupy.testing.assert_array_equal(n.indptr, [0, 1])
+
+    def test_init_data_row_col(self):
+        o = self.m.tocoo()
+        n = sparse.csc_matrix((o.data, (o.row, o.col)))
+        cupy.testing.assert_array_equal(n.data, self.m.data)
+        cupy.testing.assert_array_equal(n.indices, self.m.indices)
+        cupy.testing.assert_array_equal(n.indptr, self.m.indptr)
+        assert n.shape == self.m.shape
+
+    @testing.with_requires('scipy')
+    def test_init_dense_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(TypeError):
+                m = xp.zeros((1, 1, 1), dtype=self.dtype)
+                sp.csc_matrix(m)
+
+    def test_copy(self):
+        n = self.m.copy()
+        assert isinstance(n, sparse.csc_matrix)
+        assert n is not self.m
+        assert n.data is not self.m.data
+        assert n.indices is not self.m.indices
+        assert n.indptr is not self.m.indptr
+        cupy.testing.assert_array_equal(n.data, self.m.data)
+        cupy.testing.assert_array_equal(n.indices, self.m.indices)
+        cupy.testing.assert_array_equal(n.indptr, self.m.indptr)
+        assert n.shape == self.m.shape
+
+    def test_shape(self):
+        assert self.m.shape == (3, 4)
+
+    def test_ndim(self):
+        assert self.m.ndim == 2
+
+    def test_nnz(self):
+        assert self.m.nnz == 4
+
+    def test_conj(self):
+        n = _make_complex(cupy, sparse, self.dtype)
+        cupy.testing.assert_array_equal(n.conj().data, n.data.conj())
+
+    @testing.with_requires('scipy')
+    def test_get(self):
+        m = self.m.get()
+        assert isinstance(m, scipy.sparse.csc_matrix)
+        expect = [
+            [0, 1, 0, 0],
+            [0, 0, 0, 2],
+            [0, 0, 3, 0]
+        ]
+        numpy.testing.assert_allclose(m.toarray(), expect)
+
+    @testing.with_requires('scipy')
+    def test_str(self):
+        if numpy.dtype(self.dtype).kind == 'f':
+            expect = '''  (0, 0)\t0.0
+  (0, 1)\t1.0
+  (2, 2)\t3.0
+  (1, 3)\t2.0'''
+        elif numpy.dtype(self.dtype).kind == 'c':
+            expect = '''  (0, 0)\t0j
+  (0, 1)\t(1+0j)
+  (2, 2)\t(3+0j)
+  (1, 3)\t(2+0j)'''
+
+        assert str(self.m) == expect
+
+    def test_toarray(self):
+        m = self.m.toarray()
+        expect = [
+            [0, 1, 0, 0],
+            [0, 0, 0, 2],
+            [0, 0, 3, 0]
+        ]
+        assert m.flags.c_contiguous
+        cupy.testing.assert_allclose(m, expect)
+
+    def test_pickle_roundtrip(self):
+        s = _make(cupy, sparse, self.dtype)
+        s2 = pickle.loads(pickle.dumps(s))
+        assert s._descr.descriptor != s2._descr.descriptor
+        assert s.shape == s2.shape
+        assert s.dtype == s2.dtype
+        if scipy_available:
+            assert (s.get() != s2.get()).count_nonzero() == 0
+
+    def test_reshape_0(self):
+        assert self.m.reshape((12, 1)).shape == (12, 1)
+
+    def test_reshape_1(self):
+        m = self.m.reshape((1, 12)).toarray()
+        expect = [[0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0]]
+        cupy.testing.assert_allclose(m, expect)
+
+    def test_reshape_2(self):
+        m = self.m.reshape((1, 12), order='F').toarray()
+        expect = [[1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0]]
+        cupy.testing.assert_allclose(m, expect)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCscMatrixInit:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.shape = (3, 4)
+
+    def data(self, xp):
+        return xp.array([1, 2, 3, 4], self.dtype)
+
+    def indices(self, xp):
+        return xp.array([0, 0, 2, 1], 'i')
+
+    def indptr(self, xp):
+        return xp.array([0, 1, 2, 3, 4], 'i')
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_shape_none(self, xp, sp):
+        x = sp.csc_matrix(
+            (self.data(xp), self.indices(xp), self.indptr(xp)), shape=None)
+        assert x.shape == (3, 4)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_dtype(self, xp, sp):
+        data = self.data(xp).real.astype('i')
+        x = sp.csc_matrix(
+            (data, self.indices(xp), self.indptr(xp)), dtype=self.dtype)
+        assert x.dtype == self.dtype
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_copy_true(self, xp, sp):
+        data = self.data(xp)
+        indices = self.indices(xp)
+        indptr = self.indptr(xp)
+        x = sp.csc_matrix((data, indices, indptr), copy=True)
+
+        assert data is not x.data
+        assert indices is not x.indices
+        assert indptr is not x.indptr
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_init_with_shape(self, xp, sp):
+        s = sp.csc_matrix(self.shape)
+        assert s.shape == self.shape
+        assert s.dtype == 'd'
+        assert s.size == 0
+        return s
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_init_with_shape_and_dtype(self, xp, sp):
+        s = sp.csc_matrix(self.shape, dtype=self.dtype)
+        assert s.shape == self.shape
+        assert s.dtype == self.dtype
+        assert s.size == 0
+        return s
+
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=1e-5)
+    def test_intlike_shape(self, xp, sp):
+        s = sp.csc_matrix((self.data(xp), self.indices(xp), self.indptr(xp)),
+                          shape=(xp.array(self.shape[0]),
+                                 xp.int32(self.shape[1])))
+        assert isinstance(s.shape[0], int)
+        assert isinstance(s.shape[1], int)
+        return s
+
+    def test_shape_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    (self.data(xp), self.indices(xp), self.indptr(xp)),
+                    shape=(2,))
+
+    def test_data_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    ('invalid', self.indices(xp), self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_data_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    (self.data(xp)[None], self.indices(xp),
+                     self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_indices_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    (self.data(xp), 'invalid', self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_indices_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    (self.data(xp), self.indices(xp)[None], self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_indptr_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    (self.data(xp), self.indices(xp), 'invalid'),
+                    shape=self.shape)
+
+    def test_indptr_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    (self.data(xp), self.indices(xp), self.indptr(xp)[None]),
+                    shape=self.shape)
+
+    def test_data_indices_different_length(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            data = xp.arange(5, dtype=self.dtype)
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    (data, self.indices(xp), self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_indptr_invalid_length(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            indptr = xp.array([0, 1], 'i')
+            with pytest.raises(ValueError):
+                sp.csc_matrix(
+                    (self.data(xp), self.indices(xp), indptr),
+                    shape=self.shape)
+
+    def test_unsupported_dtype(self):
+        with pytest.raises(ValueError):
+            sparse.csc_matrix(
+                (self.data(cupy), self.indices(cupy), self.indptr(cupy)),
+                shape=self.shape, dtype='i')
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_conj(self, xp, sp):
+        n = _make_complex(xp, sp, self.dtype)
+        cupy.testing.assert_array_equal(n.conj().data, n.data.conj())
+
+
+@testing.parameterize(*testing.product({
+    'make_method': [
+        '_make', '_make_unordered', '_make_empty', '_make_duplicate',
+        '_make_shape'],
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCscMatrixScipyComparison:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if (sys.platform == 'win32' and
+                cupyx.cusparse.getVersion() == 11301 and
+                self.dtype == cupy.complex128):
+            pytest.xfail('Known to fail on CUDA 11.2 for Windows')
+        if runtime.is_hip:
+            if self.make_method in ('_make_empty', '_make_shape'):
+                # xcsr2coo, xcsrgemm2Nnz, csrmm2, nnz_compress, ... could raise
+                # HIPSPARSE_STATUS_INVALID_VALUE, maybe because we have a zero
+                # matrix (nnz=0)?
+                pytest.xfail('may be buggy')
+
+    @property
+    def make(self):
+        return globals()[self.make_method]
+
+    def test_len(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                len(m)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_asfptype(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.asfptype()
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_toarray(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        a = m.toarray()
+        if sp is sparse:
+            assert a.flags.c_contiguous
+        return a
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_toarray_c_order(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        a = m.toarray(order='C')
+        assert a.flags.c_contiguous
+        return a
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_toarray_f_order(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        a = m.toarray(order='F')
+        assert a.flags.f_contiguous
+        return a
+
+    @testing.with_requires('numpy>=1.19')
+    def test_toarray_unknown_order(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.toarray(order='#')
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_A(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.A
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocoo(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.tocoo()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocoo_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocoo(copy=True)
+        assert m.data is not n.data
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.tocsc()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsc_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocsc(copy=True)
+        assert m.data is not n.data
+        assert m.indices is not n.indices
+        assert m.indptr is not n.indptr
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.tocsr()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsr_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocsr(copy=True)
+        assert m.data is not n.data
+        assert m.indices is not n.indices
+        assert m.indptr is not n.indptr
+        return n
+
+    # dot
+    @testing.with_requires('scipy!=1.8.0')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.dot(2.0)
+
+    @testing.with_requires('scipy!=1.8.0')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_numpy_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.dot(numpy.dtype(self.dtype).type(2.0))
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return m.dot(x)
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    def test_dot_csr_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = sp.csr_matrix((5, 3), dtype=self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_csc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return m.dot(x)
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_sparse(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return m.dot(x)
+
+    @testing.with_requires('scipy>=1.8.0rc1')
+    def test_dot_zero_dim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.array(2, dtype=self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_dense_vector(self, xp, sp):
+        if runtime.is_hip:
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(4).astype(self.dtype)
+        return m.dot(x)
+
+    def test_dot_dense_vector_invalid_shape(self):
+        if runtime.is_hip:
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(5).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_dot_dense_matrix(self, xp, sp):
+        if runtime.is_hip:
+            if driver.get_build_version() < 400:
+                pytest.skip('no working implementation')
+            # no idea what's wrong...
+            elif self.make_method in (
+                    '_make', '_make_unordered', '_make_duplicate'):
+                pytest.xfail('spMM raises HIPSPARSE_STATUS_INVALID_VALUE')
+
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(8).reshape(4, 2).astype(self.dtype)
+        return m.dot(x)
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() <= 400,
+                        reason='no working implementation')
+    def test_dot_dense_matrix_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(10).reshape(5, 2).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    def test_dot_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    def test_dot_unsupported(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.dot(None)
+
+    # __add__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_zero(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m + 0
+
+    def test_add_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                m + 1
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype)
+        return m + n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_coo(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype).tocoo()
+        return m + n
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_add_dense(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return m + n
+
+    # __radd__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_radd_zero(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return 0 + m
+
+    def test_radd_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                1 + m
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_radd_dense(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return n + m
+
+    # __sub__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_zero(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m - 0
+
+    def test_sub_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                m - 1
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype)
+        return m - n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_coo(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype).tocoo()
+        return m - n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_dense(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return m - n
+
+    # __rsub__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rsub_zero(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return 0 - m
+
+    def test_rsub_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                1 - m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rsub_dense(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return n - m
+
+    # __mul__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m * 2.0
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_numpy_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m * numpy.dtype(self.dtype).type(2.0)
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_mul_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return m * x
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_mul_csc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return m * x
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_mul_sparse(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_zero_dim(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.array(2, dtype=self.dtype)
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_dense_vector(self, xp, sp):
+        if runtime.is_hip:
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(4).astype(self.dtype)
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_mul_dense_matrix(self, xp, sp):
+        if runtime.is_hip:
+            if driver.get_build_version() < 400:
+                pytest.skip('no working implementation')
+            # no idea what's wrong...
+            elif self.make_method in (
+                    '_make', '_make_unordered', '_make_duplicate'):
+                pytest.xfail('spMM raises HIPSPARSE_STATUS_INVALID_VALUE')
+
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(8).reshape(4, 2).astype(self.dtype)
+        return m * x
+
+    def test_mul_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    def test_mul_unsupported(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m * None
+
+    # __rmul__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return 2.0 * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_numpy_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return numpy.dtype(self.dtype).type(2.0) * m
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_rmul_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return x * m
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_rmul_csc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return x * m
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_sparse(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return x * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_zero_dim(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.array(2, dtype=self.dtype)
+        return x * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_rmul_dense_matrix(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(12).reshape(4, 3).astype(self.dtype)
+        return x * m
+
+    def test_rmul_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                x * m
+
+    def test_rmul_unsupported(self):
+        if (
+            numpy.lib.NumpyVersion(scipy.__version__) >= '1.8.0rc1' and
+            self.make_method not in ['_make_empty', '_make_shape']
+        ):
+            pytest.xfail('See scipy/15210')
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            # TODO(unno): When a sparse matrix has no element, scipy.sparse
+            # does not raise an error.
+            if m.nnz == 0:
+                continue
+            with pytest.raises(TypeError):
+                None * m
+
+    # Note: '@' operator is almost equivalent to '*' operator. Only test the
+    # cases where '@' raises an exception and '*' does not.
+    def test_matmul_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = 2.0
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    def test_matmul_numpy_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = numpy.dtype(self.dtype).type(2.0)
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    def test_matmul_scalar_like_array(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.array(2.0, self.dtype)
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_has_canonical_format(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.has_canonical_format
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_canonical_format2(self, xp, sp):
+        # this test is adopted from SciPy's
+        M = sp.csc_matrix((xp.array([2], dtype=self.dtype),
+                           xp.array([0]), xp.array([0, 1])))
+        assert M.has_canonical_format is True
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_canonical_format3(self, xp, sp):
+        # this test is adopted from SciPy's
+        indices = xp.array([0, 0])  # contains duplicate
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+
+        M = sp.csc_matrix((data, indices, indptr))
+        assert M.has_canonical_format is False
+
+        # set by deduplicating
+        M.sum_duplicates()
+        assert M.has_canonical_format is True
+        assert 1 == len(M.indices)
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_canonical_format4(self, xp, sp):
+        # this test is adopted from SciPy's
+        indices = xp.array([0, 0])  # contains duplicate
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+
+        M = sp.csc_matrix((data, indices, indptr))
+        # set manually (although underlyingly duplicated)
+        M.has_canonical_format = True
+        assert M.has_canonical_format
+        assert 2 == len(M.indices)  # unaffected content
+
+        # ensure deduplication bypassed when has_canonical_format == True
+        M.sum_duplicates()
+        assert 2 == len(M.indices)  # unaffected content
+        return M
+
+    @testing.with_requires('scipy>1.6.0')
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_has_sorted_indices(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.has_sorted_indices
+
+    # TODO(asi1024): Remove test after the fixed version is released.
+    # https://github.com/scipy/scipy/pull/13426
+    @testing.with_requires('scipy<=1.6.0')
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_has_sorted_indices_for_old_scipy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return bool(m.has_sorted_indices)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_sorted_indices2(self, xp, sp):
+        # this test is adopted from SciPy's
+        sorted_inds = xp.array([0, 1])
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+        M = sp.csc_matrix((data, sorted_inds, indptr))
+        assert M.has_sorted_indices
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_sorted_indices3(self, xp, sp):
+        # this test is adopted from SciPy's
+        sorted_inds = xp.array([0, 1])
+        unsorted_inds = xp.array([1, 0])
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+        M = sp.csc_matrix((data, unsorted_inds, indptr))
+        assert not M.has_sorted_indices
+
+        # set by sorting
+        M.sort_indices()
+        assert M.has_sorted_indices
+        assert (M.indices == sorted_inds).all()
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_sorted_indices4(self, xp, sp):
+        # this test is adopted from SciPy's
+        unsorted_inds = xp.array([1, 0])
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+        M = sp.csc_matrix((data, unsorted_inds, indptr))
+
+        # set manually (although underlyingly unsorted)
+        M.has_sorted_indices = True
+        assert M.has_sorted_indices
+        assert (M.indices == unsorted_inds).all()
+
+        # ensure sort bypassed when has_sorted_indices == True
+        M.sort_indices()
+        assert (M.indices == unsorted_inds).all()
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sort_indices(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.sort_indices()
+        assert m.has_sorted_indices
+        return m
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_sort_indices2(self, xp, sp):
+        # 1. this test is adopted from SciPy's.
+        # 2. we don't check the contiguity flag because SciPy and CuPy handle
+        #    the underlying data differently
+        data = xp.arange(5).astype(xp.float32)
+        indices = xp.array([7, 2, 1, 5, 4])
+        indptr = xp.array([0, 3, 5])
+        asp = sp.csc_matrix((data, indices, indptr), shape=(10, 2))
+        asp.sort_indices()
+        assert (asp.indices == xp.array([1, 2, 7, 4, 5])).all()
+        return asp.todense()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sorted_indices(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m = m.sorted_indices()
+        assert m.has_sorted_indices
+        return m
+
+    def test_sum_tuple_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.sum(axis=(0, 1))
+
+    def test_sum_too_large_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.sum(axis=3)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_duplicates(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.sum_duplicates()
+        assert m.has_canonical_format
+        return m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_transpose(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.transpose()
+
+    def test_transpose_axes_int(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.transpose(axes=0)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_eliminate_zeros(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.eliminate_zeros()
+        return m
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    @pytest.mark.skipif(
+        not runtime.is_hip and cupy.cuda.runtime.runtimeGetVersion() < 8000,
+        reason='CUDA <8 cannot keep number of non-zero entries ')
+    def test_eliminate_zeros_nnz(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.eliminate_zeros()
+        return m.nnz
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'ret_dtype': [None, numpy.float32, numpy.float64],
+    'axis': [None, 0, 1, -1, -2],
+}))
+@testing.with_requires('scipy')
+class TestCscMatrixSum:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if runtime.is_hip and self.axis in (None, -1, 1):
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.sum(axis=self.axis, dtype=self.ret_dtype)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_with_out(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if self.axis is None:
+            shape = ()
+        else:
+            shape = list(m.shape)
+            shape[self.axis] = 1
+            shape = tuple(shape)
+        out = xp.empty(shape, dtype=self.ret_dtype)
+        if xp is numpy:
+            # TODO(unno): numpy.matrix is used for scipy.sparse though
+            # cupy.ndarray is used for cupyx.scipy.sparse.
+            out = xp.asmatrix(out)
+        return m.sum(axis=self.axis, dtype=self.ret_dtype, out=out)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCscMatrixScipyCompressed:
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_get_shape(self, xp, sp):
+        return _make(xp, sp, self.dtype).get_shape()
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_getnnz(self, xp, sp):
+        return _make(xp, sp, self.dtype).getnnz()
+
+
+@testing.parameterize(*testing.product({
+    # TODO(takagi): Test dtypes
+    'axis': [None, -2, -1, 0, 1],
+    'dense': [False, True],  # means a sparse matrix but all elements filled
+}))
+@testing.with_requires('scipy>=0.19.0')
+class TestCscMatrixScipyCompressedMinMax:
+
+    def _make_data_min(self, xp, sp, dense=False):
+        dm_data = testing.shaped_random((10, 20), xp=xp, scale=1.0)
+        if not dense:
+            dm_data[abs(dm_data) < 0.95] = 0
+        return sp.csc_matrix(xp.array(dm_data))
+
+    def _make_data_max(self, xp, sp, dense=False):
+        return -self._make_data_min(xp, sp, dense=dense)
+
+    def _make_data_min_explicit(self, xp, sp, axis):
+        dm_data = testing.shaped_random((10, 20), xp=xp, scale=1.0)
+        if xp is cupy:
+            dm_data[dm_data < 0.95] = 0
+        else:
+            # As SciPy sparse matrix does not have `explicit` parameter, we
+            # make SciPy inputs such that SciPy's spmatrix.min(axis=axis)
+            # returns the same value as CuPy's spmatrix.min(axis=axis,
+            # explicit=True).
+
+            # Put infinity instead of zeros so spmatrix.min(axis=axis) returns
+            # the smallest numbers except for zero.
+            dm_data[dm_data < 0.95] = numpy.inf
+
+            if axis is None:
+                # If all elements in the array are set to infinity, we make it
+                # have at least a zero so SciPy's spmatrix.min(axis=None)
+                # returns zero.
+                if numpy.isinf(dm_data).all():
+                    dm_data[0, 0] = 0
+            else:
+                if axis < 0:
+                    axis += 2
+
+                # If all elements in a row/column are set to infinity, we make
+                # it have at least a zero so spmatrix.min(axis=axis) returns
+                # zero for the row/column.
+                mask = numpy.zeros_like(dm_data, dtype=numpy.bool_)
+                if axis == 0:
+                    rows = dm_data.argmin(axis=0)
+                    cols = numpy.arange(20)
+                else:
+                    rows = numpy.arange(10)
+                    cols = dm_data.argmin(axis=1)
+                mask[rows, cols] = numpy.isinf(dm_data[rows, cols])
+                dm_data[mask] = 0
+
+        return sp.csc_matrix(xp.array(dm_data))
+
+    def _make_data_max_explicit(self, xp, sp, axis):
+        return -self._make_data_min_explicit(xp, sp, axis=axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_min(self, xp, sp):
+        data = self._make_data_min(xp, sp, dense=self.dense)
+        return data.min(axis=self.axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_min_explicit(self, xp, sp):
+        data = self._make_data_min_explicit(xp, sp, axis=self.axis)
+        if xp is cupy:
+            return data.min(axis=self.axis, explicit=True)
+        else:
+            return data.min(axis=self.axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_max(self, xp, sp):
+        data = self._make_data_max(xp, sp, dense=self.dense)
+        return data.max(axis=self.axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_max_explicit(self, xp, sp):
+        data = self._make_data_max_explicit(xp, sp, axis=self.axis)
+        if xp is cupy:
+            return data.max(axis=self.axis, explicit=True)
+        else:
+            return data.max(axis=self.axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp', type_check=False)
+    def test_argmin(self, xp, sp):
+        data = self._make_data_min(xp, sp, dense=self.dense)
+        # Due to a SciPy bug, the argmin output is different from the expected
+        # one
+        if self.axis is None and self.dense:
+            pytest.skip()
+        return xp.array(data.argmin(axis=self.axis))
+
+    @testing.numpy_cupy_array_equal(sp_name='sp', type_check=False)
+    def test_argmax(self, xp, sp):
+        data = self._make_data_max(xp, sp, dense=self.dense)
+        # Due to a SciPy bug, the argmin output is different from the expected
+        # one
+        if self.axis is None and self.dense:
+            pytest.skip()
+        return xp.array(data.argmax(axis=self.axis))
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCscMatrixData:
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_dtype(self, xp, sp):
+        return _make(xp, sp, self.dtype).dtype
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_abs(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return abs(m)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_neg(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return (-m)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_astype(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if numpy.dtype(self.dtype).kind == 'c':
+            t = 'D'
+        else:
+            t = 'd'
+        return m.astype(t)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_count_nonzero(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.count_nonzero()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_power(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.power(2)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_power_with_dtype(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if numpy.dtype(self.dtype).kind == 'c':
+            t = 'D'
+        else:
+            t = 'd'
+        return m.power(2, t)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_None(self, xp, sp):
+        if runtime.is_hip:
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=None)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_0(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=0)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_1(self, xp, sp):
+        if runtime.is_hip:
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=1)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_negative_1(self, xp, sp):
+        if runtime.is_hip:
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=-1)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_negative_2(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=-2)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'ufunc': [
+        'arcsin', 'arcsinh', 'arctan', 'arctanh', 'ceil', 'deg2rad', 'expm1',
+        'floor', 'log1p', 'rad2deg', 'rint', 'sign', 'sin', 'sinh', 'sqrt',
+        'tan', 'tanh', 'trunc',
+    ],
+}))
+@testing.with_requires('scipy')
+class TestUfunc:
+
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=1e-5)
+    def test_ufun(self, xp, sp):
+        x = _make(xp, sp, self.dtype)
+        x.data *= 0.1
+        func = getattr(x, self.ufunc)
+        complex_unsupported = {'ceil', 'deg2rad', 'floor', 'rad2deg', 'trunc'}
+        if (numpy.dtype(self.dtype).kind == 'c' and
+                self.ufunc in complex_unsupported):
+            with pytest.raises(TypeError):
+                func()
+            return xp.array(0)
+        else:
+            return func()
+
+
+class TestIsspmatrixCsc:
+
+    def test_csr(self):
+        x = sparse.csr_matrix(
+            (cupy.array([], 'f'),
+             cupy.array([], 'i'),
+             cupy.array([0], 'i')),
+            shape=(0, 0), dtype='f')
+        assert not sparse.isspmatrix_csc(x)
+
+    def test_csc(self):
+        x = sparse.csc_matrix(
+            (cupy.array([], 'f'),
+             cupy.array([], 'i'),
+             cupy.array([0], 'i')),
+            shape=(0, 0), dtype='f')
+        assert sparse.isspmatrix_csc(x)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestCsrMatrixGetitem:
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_getitem_int_int(self, xp, sp):
+        assert _make(xp, sp, self.dtype)[0, 1] == 1
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_getitem_int_int_not_found(self, xp, sp):
+        assert _make(xp, sp, self.dtype)[1, 1] == 0
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_getitem_int_int_negative(self, xp, sp):
+        assert _make(xp, sp, self.dtype)[-1, -2] == 3
+
+    def test_getitem_int_int_too_small_row(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[-4, 0]
+
+    def test_getitem_int_int_too_large_row(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[3, 0]
+
+    def test_getitem_int_int_too_small_col(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[0, -5]
+
+    def test_getitem_int_int_too_large_col(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[0, 4]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_int(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, 1]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_negative_int(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, -1]
+
+    def test_getitem_int_too_small(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[:, -5]
+
+    def test_getitem_int_too_large(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[:, 4]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, 1:3]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_negative(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, -2:-1]
+
+    # SciPy prior to 1.4 has bugs where either an IndexError is raised or a
+    # segfault occurs instead of returning an empty slice.
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_start_larger_than_stop(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, 3:2]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_rowslice_all(self, xp, sp):
+        # This test is adapted from Scipy's CSC tests
+        return _make(xp, sp, self.dtype)[slice(None, None, None)]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    @testing.with_requires('scipy>=1.9.3')
+    def test_getitem_rowslice_negative_stop(self, xp, sp):
+        # This test is adapted from Scipy's CSC tests
+        return _make(xp, sp, self.dtype)[slice(1, -2, 2)]
+
+    def test_getrow(self):
+
+        # This test is adapted from Scipy's CSC tests
+        N = 10
+        X = testing.shaped_random((N, N), cupy, seed=0)
+        X[X > 0.7] = 0
+        Xcsc = sparse.csc_matrix(X)
+
+        for i in range(N):
+            arr_row = X[i:i + 1, :]
+            csc_row = Xcsc.getrow(i)
+
+            assert sparse.isspmatrix_csr(csc_row)
+            assert (arr_row == csc_row.toarray()).all()
+
+    def test_getcol(self):
+        # This test is adapted from Scipy's CSC tests
+        N = 10
+        X = testing.shaped_random((N, N), cupy, seed=0)
+        X[X > 0.7] = 0
+        Xcsc = sparse.csc_matrix(X)
+
+        for i in range(N):
+            arr_col = X[:, i:i + 1]
+            csc_col = Xcsc.getcol(i)
+
+            assert sparse.isspmatrix_csc(csc_col)
+            assert (arr_col == csc_col.toarray()).all()
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestCsrMatrixGetitem2:
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_start_too_small(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, -5:None]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_start_too_large(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, 5:None]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_stop_too_small(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, None:-5]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_stop_too_large(self, xp, sp):
+        return _make(xp, sp, self.dtype)[:, None:5]
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_csr.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_csr.py
new file mode 100644
index 0000000..c00d264
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_csr.py
@@ -0,0 +1,2170 @@
+import contextlib
+import pickle
+import sys
+import warnings
+
+import numpy
+import pytest
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+from cupy_backends.cuda.api import driver
+from cupy_backends.cuda.api import runtime
+import cupy
+from cupy._core import _accelerator
+from cupy import testing
+import cupyx.cusparse
+from cupyx.scipy import sparse
+
+
+def _make(xp, sp, dtype):
+    data = xp.array([0, 1, 2, 3], dtype)
+    indices = xp.array([0, 1, 3, 2], 'i')
+    indptr = xp.array([0, 2, 3, 4], 'i')
+    # 0, 1, 0, 0
+    # 0, 0, 0, 2
+    # 0, 0, 3, 0
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_complex(xp, sp, dtype):
+    data = xp.array([0, 1, 2, 3], dtype)
+    if dtype in [numpy.complex64, numpy.complex128]:
+        data = data - 1j
+    indices = xp.array([0, 1, 3, 2], 'i')
+    indptr = xp.array([0, 2, 3, 4], 'i')
+    # 0, 1 - 1j, 0, 0
+    # 0, 0, 0, 2 - 1j
+    # 0, 0, 3 - 1j, 0
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make2(xp, sp, dtype):
+    data = xp.array([1, 2, 3, 4], dtype)
+    indices = xp.array([2, 1, 2, 2], 'i')
+    indptr = xp.array([0, 1, 3, 4], 'i')
+    # 0, 0, 1, 0
+    # 0, 2, 3, 0
+    # 0, 0, 4, 0
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make3(xp, sp, dtype):
+    data = xp.array([1, 2, 3, 4, 5], dtype)
+    indices = xp.array([0, 2, 1, 0, 2], 'i')
+    indptr = xp.array([0, 1, 3, 3, 5], 'i')
+    # 1, 0, 0
+    # 0, 3, 2
+    # 0, 0, 0
+    # 4, 0, 5
+    return sp.csr_matrix((data, indices, indptr), shape=(4, 3))
+
+
+def _make4(xp, sp, dtype):
+    data = xp.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype)
+    indices = xp.array([0, 2, 3, 0, 1, 3, 0, 1, 2], 'i')
+    indptr = xp.array([0, 3, 6, 9], 'i')
+    # 1, 0, 2, 3
+    # 4, 5, 0, 6
+    # 7, 8, 9, 0
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_unordered(xp, sp, dtype):
+    data = xp.array([1, 2, 3, 4], dtype)
+    indices = xp.array([1, 0, 1, 2], 'i')
+    indptr = xp.array([0, 2, 3, 4], 'i')
+    # 2, 1, 0, 0
+    # 0, 3, 0, 0
+    # 0, 0, 4, 0
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_duplicate(xp, sp, dtype):
+    data = xp.array([0, 1, 3, 2, 4, 5], dtype)
+    indices = xp.array([0, 0, 0, 2, 0, 2], 'i')
+    indptr = xp.array([0, 3, 6, 6], 'i')
+    # 4, 0, 0, 0
+    # 4, 0, 7, 0
+    # 0, 0, 0, 0
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_empty(xp, sp, dtype):
+    data = xp.array([], dtype)
+    indices = xp.array([], 'i')
+    indptr = xp.array([0, 0, 0, 0], 'i')
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 4))
+
+
+def _make_square(xp, sp, dtype):
+    data = xp.array([0, 1, 2, 3], dtype)
+    indices = xp.array([0, 1, 0, 2], 'i')
+    indptr = xp.array([0, 2, 3, 4], 'i')
+    # 0, 1, 0
+    # 2, 0, 0
+    # 0, 0, 3
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 3))
+
+
+def _make_row(xp, sp, dtype):
+    data = xp.array([1, 2, 3], dtype)
+    indices = xp.array([0, 2, 3], 'i')
+    indptr = xp.array([0, 3], 'i')
+    # 1, 0, 2, 3
+    return sp.csr_matrix((data, indices, indptr), shape=(1, 4))
+
+
+def _make_col(xp, sp, dtype):
+    data = xp.array([1, 2], dtype)
+    indices = xp.array([0, 0], 'i')
+    indptr = xp.array([0, 1, 1, 2], 'i')
+    # 1
+    # 0
+    # 2
+    return sp.csr_matrix((data, indices, indptr), shape=(3, 1))
+
+
+def _make_shape(xp, sp, dtype):
+    return sp.csr_matrix((3, 4), dtype=dtype)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+class TestCsrMatrix:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.m = _make(cupy, sparse, self.dtype)
+
+    def test_dtype(self):
+        assert self.m.dtype == self.dtype
+
+    def test_data(self):
+        assert self.m.data.dtype == self.dtype
+        testing.assert_array_equal(
+            self.m.data, cupy.array([0, 1, 2, 3], self.dtype))
+
+    def test_indices(self):
+        assert self.m.indices.dtype == numpy.int32
+        testing.assert_array_equal(
+            self.m.indices, cupy.array([0, 1, 3, 2], self.dtype))
+
+    def test_indptr(self):
+        assert self.m.indptr.dtype == numpy.int32
+        testing.assert_array_equal(
+            self.m.indptr, cupy.array([0, 2, 3, 4], self.dtype))
+
+    def test_init_copy(self):
+        n = sparse.csr_matrix(self.m)
+        assert n is not self.m
+        cupy.testing.assert_array_equal(n.data, self.m.data)
+        cupy.testing.assert_array_equal(n.indices, self.m.indices)
+        cupy.testing.assert_array_equal(n.indptr, self.m.indptr)
+        assert n.shape == self.m.shape
+
+    def test_init_copy_other_sparse(self):
+        n = sparse.csr_matrix(self.m.tocsc())
+        cupy.testing.assert_array_equal(n.data, self.m.data)
+        cupy.testing.assert_array_equal(n.indices, self.m.indices)
+        cupy.testing.assert_array_equal(n.indptr, self.m.indptr)
+        assert n.shape == self.m.shape
+
+    @testing.with_requires('scipy')
+    def test_init_copy_scipy_sparse(self):
+        m = _make(numpy, scipy.sparse, self.dtype)
+        n = sparse.csr_matrix(m)
+        assert isinstance(n.data, cupy.ndarray)
+        assert isinstance(n.indices, cupy.ndarray)
+        assert isinstance(n.indptr, cupy.ndarray)
+        cupy.testing.assert_array_equal(n.data, m.data)
+        cupy.testing.assert_array_equal(n.indices, m.indices)
+        cupy.testing.assert_array_equal(n.indptr, m.indptr)
+        assert n.shape == m.shape
+
+    @testing.with_requires('scipy')
+    def test_init_copy_other_scipy_sparse(self):
+        m = _make(numpy, scipy.sparse, self.dtype)
+        n = sparse.csr_matrix(m.tocsc())
+        assert isinstance(n.data, cupy.ndarray)
+        assert isinstance(n.indices, cupy.ndarray)
+        assert isinstance(n.indptr, cupy.ndarray)
+        cupy.testing.assert_array_equal(n.data, m.data)
+        cupy.testing.assert_array_equal(n.indices, m.indices)
+        cupy.testing.assert_array_equal(n.indptr, m.indptr)
+        assert n.shape == m.shape
+
+    def test_init_dense(self):
+        m = cupy.array([[0, 1, 0, 2],
+                        [0, 0, 0, 0],
+                        [0, 0, 3, 0]], dtype=self.dtype)
+        n = sparse.csr_matrix(m)
+        assert n.nnz == 3
+        assert n.shape == (3, 4)
+        cupy.testing.assert_array_equal(n.data, [1, 2, 3])
+        cupy.testing.assert_array_equal(n.indices, [1, 3, 2])
+        cupy.testing.assert_array_equal(n.indptr, [0, 2, 2, 3])
+
+    @pytest.mark.xfail(runtime.is_hip, reason='hipSPARSE handles nnz=0 badly')
+    def test_init_dense_empty(self):
+        m = cupy.array([[0, 0, 0, 0],
+                        [0, 0, 0, 0],
+                        [0, 0, 0, 0]], dtype=self.dtype)
+        n = sparse.csr_matrix(m)
+        assert n.nnz == 0
+        assert n.shape == (3, 4)
+        cupy.testing.assert_array_equal(n.data, [])
+        cupy.testing.assert_array_equal(n.indices, [])
+        cupy.testing.assert_array_equal(n.indptr, [0, 0, 0, 0])
+
+    def test_init_dense_one_dim(self):
+        m = cupy.array([0, 1, 0, 2], dtype=self.dtype)
+        n = sparse.csr_matrix(m)
+        assert n.nnz == 2
+        assert n.shape == (1, 4)
+        cupy.testing.assert_array_equal(n.data, [1, 2])
+        cupy.testing.assert_array_equal(n.indices, [1, 3])
+        cupy.testing.assert_array_equal(n.indptr, [0, 2])
+
+    def test_init_dense_zero_dim(self):
+        m = cupy.array(1, dtype=self.dtype)
+        n = sparse.csr_matrix(m)
+        assert n.nnz == 1
+        assert n.shape == (1, 1)
+        cupy.testing.assert_array_equal(n.data, [1])
+        cupy.testing.assert_array_equal(n.indices, [0])
+        cupy.testing.assert_array_equal(n.indptr, [0, 1])
+
+    def test_init_data_row_col(self):
+        o = self.m.tocoo()
+        n = sparse.csr_matrix((o.data, (o.row, o.col)))
+        cupy.testing.assert_array_equal(n.data, self.m.data)
+        cupy.testing.assert_array_equal(n.indices, self.m.indices)
+        cupy.testing.assert_array_equal(n.indptr, self.m.indptr)
+        assert n.shape == self.m.shape
+
+    @testing.with_requires('scipy')
+    def test_init_dense_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = xp.zeros((1, 1, 1), dtype=self.dtype)
+            with pytest.raises(TypeError):
+                sp.csr_matrix(m)
+
+    def test_copy(self):
+        n = self.m.copy()
+        assert isinstance(n, sparse.csr_matrix)
+        assert n is not self.m
+        assert n.data is not self.m.data
+        assert n.indices is not self.m.indices
+        assert n.indptr is not self.m.indptr
+        cupy.testing.assert_array_equal(n.data, self.m.data)
+        cupy.testing.assert_array_equal(n.indices, self.m.indices)
+        cupy.testing.assert_array_equal(n.indptr, self.m.indptr)
+        assert n.shape == self.m.shape
+
+    def test_shape(self):
+        assert self.m.shape == (3, 4)
+
+    def test_ndim(self):
+        assert self.m.ndim == 2
+
+    def test_nnz(self):
+        assert self.m.nnz == 4
+
+    def test_conj(self):
+        n = _make_complex(cupy, sparse, self.dtype)
+        cupy.testing.assert_array_equal(n.conj().data, n.data.conj())
+
+    @testing.with_requires('scipy')
+    def test_get(self):
+        m = self.m.get()
+        assert isinstance(m, scipy.sparse.csr_matrix)
+        expect = [
+            [0, 1, 0, 0],
+            [0, 0, 0, 2],
+            [0, 0, 3, 0]
+        ]
+        numpy.testing.assert_allclose(m.toarray(), expect)
+
+    @testing.with_requires('scipy')
+    def test_str(self):
+        if numpy.dtype(self.dtype).kind == 'f':
+            expect = '''  (0, 0)\t0.0
+  (0, 1)\t1.0
+  (1, 3)\t2.0
+  (2, 2)\t3.0'''
+        elif numpy.dtype(self.dtype).kind == 'c':
+            expect = '''  (0, 0)\t0j
+  (0, 1)\t(1+0j)
+  (1, 3)\t(2+0j)
+  (2, 2)\t(3+0j)'''
+
+        assert str(self.m) == expect
+
+    def test_toarray(self):
+        m = self.m.toarray()
+        expect = [
+            [0, 1, 0, 0],
+            [0, 0, 0, 2],
+            [0, 0, 3, 0]
+        ]
+        assert m.flags.c_contiguous
+        cupy.testing.assert_allclose(m, expect)
+
+    def test_pickle_roundtrip(self):
+        s = _make(cupy, sparse, self.dtype)
+
+        s2 = pickle.loads(pickle.dumps(s))
+        assert s._descr.descriptor != s2._descr.descriptor
+        assert s.shape == s2.shape
+        assert s.dtype == s2.dtype
+        if scipy_available:
+            assert (s.get() != s2.get()).count_nonzero() == 0
+
+    def test_reshape_0(self):
+        assert self.m.reshape((12, 1)).shape == (12, 1)
+
+    def test_reshape_1(self):
+        m = self.m.reshape((1, 12)).toarray()
+        expect = [[0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0]]
+        cupy.testing.assert_allclose(m, expect)
+
+    def test_reshape_2(self):
+        m = self.m.reshape((1, 12), order='F').toarray()
+        expect = [[1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0]]
+        cupy.testing.assert_allclose(m, expect)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCsrMatrixInit:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.shape = (3, 4)
+
+    def data(self, xp):
+        return xp.array([1, 2, 3, 4], self.dtype)
+
+    def indices(self, xp):
+        return xp.array([0, 1, 3, 2], 'i')
+
+    def indptr(self, xp):
+        return xp.array([0, 2, 3, 4], 'i')
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_shape_none(self, xp, sp):
+        x = sp.csr_matrix(
+            (self.data(xp), self.indices(xp), self.indptr(xp)), shape=None)
+        assert x.shape == (3, 4)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_dtype(self, xp, sp):
+        data = self.data(xp).real.astype('i')
+        x = sp.csr_matrix(
+            (data, self.indices(xp), self.indptr(xp)), dtype=self.dtype)
+        assert x.dtype == self.dtype
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_copy_true(self, xp, sp):
+        data = self.data(xp)
+        indices = self.indices(xp)
+        indptr = self.indptr(xp)
+        x = sp.csr_matrix((data, indices, indptr), copy=True)
+
+        assert data is not x.data
+        assert indices is not x.indices
+        assert indptr is not x.indptr
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_init_with_shape(self, xp, sp):
+        s = sp.csr_matrix(self.shape)
+        assert s.shape == self.shape
+        assert s.dtype == 'd'
+        assert s.size == 0
+        return s
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_init_with_shape_and_dtype(self, xp, sp):
+        s = sp.csr_matrix(self.shape, dtype=self.dtype)
+        assert s.shape == self.shape
+        assert s.dtype == self.dtype
+        assert s.size == 0
+        return s
+
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=1e-5)
+    def test_intlike_shape(self, xp, sp):
+        s = sp.csr_matrix((self.data(xp), self.indices(xp), self.indptr(xp)),
+                          shape=(xp.array(self.shape[0]),
+                                 xp.int32(self.shape[1])))
+        assert isinstance(s.shape[0], int)
+        assert isinstance(s.shape[1], int)
+        return s
+
+    def test_shape_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    (self.data(xp), self.indices(xp), self.indptr(xp)),
+                    shape=(2,))
+
+    def test_data_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    ('invalid', self.indices(xp), self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_data_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    (self.data(xp)[None], self.indices(xp), self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_indices_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    (self.data(xp), 'invalid', self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_indices_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    (self.data(xp), self.indices(xp)[None], self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_indptr_invalid(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    (self.data(xp), self.indices(xp), 'invalid'),
+                    shape=self.shape)
+
+    def test_indptr_invalid_ndim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    (self.data(xp), self.indices(xp), self.indptr(xp)[None]),
+                    shape=self.shape)
+
+    def test_data_indices_different_length(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            data = xp.arange(5, dtype=self.dtype)
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    (data, self.indices(xp), self.indptr(xp)),
+                    shape=self.shape)
+
+    def test_indptr_invalid_length(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            indptr = xp.array([0, 1], 'i')
+            with pytest.raises(ValueError):
+                sp.csr_matrix(
+                    (self.data(xp), self.indices(xp), indptr),
+                    shape=self.shape)
+
+    def test_unsupported_dtype(self):
+        with pytest.raises(ValueError):
+            sparse.csr_matrix(
+                (self.data(cupy), self.indices(cupy), self.indptr(cupy)),
+                shape=self.shape, dtype='i')
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_conj(self, xp, sp):
+        n = _make_complex(xp, sp, self.dtype)
+        cupy.testing.assert_array_equal(n.conj().data, n.data.conj())
+
+
+@testing.parameterize(*testing.product({
+    'make_method': [
+        '_make', '_make_unordered', '_make_empty', '_make_duplicate',
+        '_make_shape'],
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCsrMatrixScipyComparison:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if (sys.platform == 'win32' and
+                cupyx.cusparse.getVersion() == 11301 and
+                self.dtype == cupy.complex128):
+            pytest.xfail('Known to fail on CUDA 11.2 for Windows')
+        if runtime.is_hip:
+            if self.make_method in ('_make_empty', '_make_shape'):
+                # xcsr2coo, xcsrgemm2Nnz, csrmm2, nnz_compress, ... could raise
+                # HIPSPARSE_STATUS_INVALID_VALUE, maybe because we have a zero
+                # matrix (nnz=0)?
+                pytest.xfail('may be buggy')
+
+    @property
+    def make(self):
+        return globals()[self.make_method]
+
+    def test_len(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                len(m)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_iter(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        rows = []
+        for r in m:
+            rows.append(r)
+            assert isinstance(r, sp.spmatrix)
+        assert len(rows) == 3
+        return xp.concatenate([r.toarray() for r in rows])
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_asfptype(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.asfptype().toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_toarray(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        a = m.toarray()
+        assert a.flags.c_contiguous
+        return a
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_toarray_c_order(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        a = m.toarray(order='C')
+        assert a.flags.c_contiguous
+        return a
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_toarray_f_order(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        a = m.toarray(order='F')
+        assert a.flags.f_contiguous
+        return a
+
+    @testing.with_requires('numpy>=1.19')
+    def test_toarray_unknown_order(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.toarray(order='#')
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_A(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.A
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocoo(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.tocoo()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocoo_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocoo(copy=True)
+        assert m.data is not n.data
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.tocsc()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsc_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocsc(copy=True)
+        assert m.data is not n.data
+        assert m.indices is not n.indices
+        assert m.indptr is not n.indptr
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.tocsr()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsr_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocsr(copy=True)
+        assert m.data is not n.data
+        assert m.indices is not n.indices
+        assert m.indptr is not n.indptr
+        return n
+
+    # dot
+    @testing.with_requires('scipy!=1.8.0')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.dot(2.0)
+
+    @testing.with_requires('scipy!=1.8.0')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_dot_numpy_scalar(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.dot(numpy.dtype(self.dtype).type(2.0))
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return m.dot(x)
+
+    def test_dot_csr_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = sp.csr_matrix((5, 3), dtype=self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_csc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return m.dot(x)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_sparse(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return m.dot(x)
+
+    @testing.with_requires('scipy>=1.8.0rc1')
+    def test_dot_zero_dim(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.array(2, dtype=self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_dot_dense_vector(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(4).astype(self.dtype)
+        return m.dot(x)
+
+    def test_dot_dense_vector_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(5).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_dot_dense_matrix(self, xp, sp):
+        if (cupyx.cusparse.getVersion() == 11702
+                and self.make_method == '_make_duplicate'):
+            pytest.xfail('Known to fail on CUDA 11.6.1/11.6.2')
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(8).reshape(4, 2).astype(self.dtype)
+        return m.dot(x)
+
+    def test_dot_dense_matrix_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(10).reshape(5, 2).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    def test_dot_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m.dot(x)
+
+    def test_dot_unsupported(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.dot(None)
+
+    # __add__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_zero(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m + 0
+
+    def test_add_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                m + 1
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype)
+        return m + n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_coo(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype).tocoo()
+        return m + n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_add_dense(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return m + n
+
+    # __radd__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_radd_zero(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return (0 + m).toarray()
+
+    def test_radd_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                1 + m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_radd_dense(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return n + m
+
+    # __sub__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_zero(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return (m - 0).toarray()
+
+    def test_sub_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                m - 1
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype)
+        return (m - n).toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_coo(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = _make2(xp, sp, self.dtype).tocoo()
+        return m - n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sub_dense(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return m - n
+
+    # __rsub__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rsub_zero(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return 0 - m
+
+    def test_rsub_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                1 - m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rsub_dense(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = xp.arange(12).reshape(3, 4)
+        return n - m
+
+    # __mul__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m * 2.0
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_numpy_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m * numpy.dtype(self.dtype).type(2.0)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return m * x
+
+    def test_mul_csr_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = sp.csr_matrix((5, 3), dtype=self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_csc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_sparse(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_zero_dim(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.array(2, dtype=self.dtype)
+        return m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_dense_vector(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(4).astype(self.dtype)
+        return m * x
+
+    def test_mul_dense_vector_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(5).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_mul_dense_matrix(self, xp, sp):
+        if (cupyx.cusparse.getVersion() == 11702
+                and self.make_method == '_make_duplicate'):
+            pytest.xfail('Known to fail on CUDA 11.6.1/11.6.2')
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(8).reshape(4, 2).astype(self.dtype)
+        return m * x
+
+    def test_mul_dense_matrix_invalid_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(10).reshape(5, 2).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    def test_mul_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                m * x
+
+    def test_mul_unsupported(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m * None
+
+    # __rmul__
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return 2.0 * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_numpy_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return numpy.dtype(self.dtype).type(2.0) * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_csr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype)
+        return x * m
+
+    @pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 400,
+                        reason='no working implementation')
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_rmul_csc(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocsc()
+        return x * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_sparse(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make3(xp, sp, self.dtype).tocoo()
+        return x * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_rmul_zero_dim(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.array(2, dtype=self.dtype)
+        return x * m
+
+    @testing.numpy_cupy_allclose(sp_name='sp', contiguous_check=False)
+    def test_rmul_dense_matrix(self, xp, sp):
+        if runtime.is_hip:
+            if driver.get_build_version() < 400:
+                pytest.skip('no working implementation')
+            # no idea what's wrong...
+            elif self.make_method in (
+                    '_make', '_make_unordered', '_make_duplicate'):
+                pytest.xfail('spMM raises HIPSPARSE_STATUS_INVALID_VALUE')
+
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(12).reshape(4, 3).astype(self.dtype)
+        return x * m
+
+    def test_rmul_dense_ndim3(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.arange(24).reshape(4, 2, 3).astype(self.dtype)
+            with pytest.raises(ValueError):
+                x * m
+
+    def test_rmul_unsupported(self):
+        if (
+            numpy.lib.NumpyVersion(scipy.__version__) >= '1.8.0rc1' and
+            self.make_method not in ['_make_empty', '_make_shape']
+        ):
+            pytest.xfail('See scipy/15210')
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            if m.nnz == 0:
+                # When there is no element, a SciPy's sparse matrix does
+                # not raise an error when it is multiplied with None.
+                continue
+            with pytest.raises(TypeError):
+                None * m
+
+    # Note: '@' operator is almost equivalent to '*' operator. Only test the
+    # cases where '@' raises an exception and '*' does not.
+    def test_matmul_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = 2.0
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    def test_matmul_numpy_scalar(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = numpy.dtype(self.dtype).type(2.0)
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    def test_matmul_scalar_like_array(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            x = xp.array(2.0, self.dtype)
+            with pytest.raises(ValueError):
+                m @ x
+            with pytest.raises(ValueError):
+                x @ m
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_has_canonical_format(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.has_canonical_format
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_canonical_format2(self, xp, sp):
+        # this test is adopted from SciPy's
+        M = sp.csr_matrix((xp.array([2], dtype=self.dtype),
+                           xp.array([0]), xp.array([0, 1])))
+        assert M.has_canonical_format is True
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_canonical_format3(self, xp, sp):
+        # this test is adopted from SciPy's
+        indices = xp.array([0, 0])  # contains duplicate
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+
+        M = sp.csr_matrix((data, indices, indptr))
+        assert M.has_canonical_format is False
+
+        # set by deduplicating
+        M.sum_duplicates()
+        assert M.has_canonical_format is True
+        assert 1 == len(M.indices)
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_canonical_format4(self, xp, sp):
+        # this test is adopted from SciPy's
+        indices = xp.array([0, 0])  # contains duplicate
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+
+        M = sp.csr_matrix((data, indices, indptr))
+        # set manually (although underlyingly duplicated)
+        M.has_canonical_format = True
+        assert M.has_canonical_format
+        assert 2 == len(M.indices)  # unaffected content
+
+        # ensure deduplication bypassed when has_canonical_format == True
+        M.sum_duplicates()
+        assert 2 == len(M.indices)  # unaffected content
+        return M
+
+    @testing.with_requires('scipy>1.6.0')
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_has_sorted_indices(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.has_sorted_indices
+
+    # TODO(asi1024): Remove test after the fixed version is released.
+    # https://github.com/scipy/scipy/pull/13426
+    @testing.with_requires('scipy<=1.6.0')
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_has_sorted_indices_for_old_scipy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return bool(m.has_sorted_indices)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_sorted_indices2(self, xp, sp):
+        # this test is adopted from SciPy's
+        sorted_inds = xp.array([0, 1])
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+        M = sp.csr_matrix((data, sorted_inds, indptr))
+        assert M.has_sorted_indices
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_sorted_indices3(self, xp, sp):
+        # this test is adopted from SciPy's
+        sorted_inds = xp.array([0, 1])
+        unsorted_inds = xp.array([1, 0])
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+        M = sp.csr_matrix((data, unsorted_inds, indptr))
+        assert not M.has_sorted_indices
+
+        # set by sorting
+        M.sort_indices()
+        assert M.has_sorted_indices
+        assert (M.indices == sorted_inds).all()
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_has_sorted_indices4(self, xp, sp):
+        # this test is adopted from SciPy's
+        unsorted_inds = xp.array([1, 0])
+        data = xp.array([1, 1], dtype=self.dtype)
+        indptr = xp.array([0, 2])
+        M = sp.csr_matrix((data, unsorted_inds, indptr))
+
+        # set manually (although underlyingly unsorted)
+        M.has_sorted_indices = True
+        assert M.has_sorted_indices
+        assert (M.indices == unsorted_inds).all()
+
+        # ensure sort bypassed when has_sorted_indices == True
+        M.sort_indices()
+        assert (M.indices == unsorted_inds).all()
+        return M
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sort_indices(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.sort_indices()
+        assert m.has_sorted_indices
+        return m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sort_indices2(self, xp, sp):
+        # this test is adopted from SciPy's
+        data = xp.arange(5).astype(xp.float32)
+        indices = xp.array([7, 2, 1, 5, 4])
+        indptr = xp.array([0, 3, 5])
+        asp = sp.csr_matrix((data, indices, indptr), shape=(2, 10))
+        asp.sort_indices()
+        assert (asp.indices == xp.array([1, 2, 7, 4, 5])).all()
+        return asp.todense()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sorted_indices(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m = m.sorted_indices()
+        assert m.has_sorted_indices
+        return m
+
+    def test_sum_tuple_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.sum(axis=(0, 1))
+
+    def test_sum_str_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.sum(axis='test')
+
+    def test_sum_too_large_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.sum(axis=3)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_duplicates(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.sum_duplicates()
+        assert m.has_canonical_format
+        return m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_transpose(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.transpose()
+
+    def test_transpose_axes_int(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.transpose(axes=0)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_eliminate_zeros(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.eliminate_zeros()
+        return m
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    @pytest.mark.skipif(
+        not runtime.is_hip and cupy.cuda.runtime.runtimeGetVersion() < 8000,
+        reason='CUDA <8 cannot keep number of non-zero entries ')
+    def test_eliminate_zeros_nnz(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        m.eliminate_zeros()
+        return m.nnz
+
+    # multiply
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_multiply_scalar(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.multiply(2).toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_multiply_dense_row(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(4, dtype=self.dtype)
+        return m.multiply(x).toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_multiply_dense_col(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(3, dtype=self.dtype).reshape(3, 1)
+        return m.multiply(x).toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_multiply_dense_matrix(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(12, dtype=self.dtype).reshape(3, 4)
+        return m.multiply(x).toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_multiply_csr_matrix(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make4(xp, sp, self.dtype)
+        return m.multiply(x).toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_multiply_csr_row(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make_row(xp, sp, self.dtype)
+        return m.multiply(x).toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_multiply_csr_col(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make_col(xp, sp, self.dtype)
+        return m.multiply(x).toarray()
+
+    def _make_scalar(self, dtype):
+        if numpy.issubdtype(dtype, numpy.integer):
+            return dtype(2)
+        elif numpy.issubdtype(dtype, numpy.floating):
+            return dtype(2.5)
+        else:
+            return dtype(2.5 - 1.5j)
+
+    # divide
+    @testing.for_dtypes('ifdFD')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_divide_scalar(self, xp, sp, dtype):
+        m = self.make(xp, sp, self.dtype)
+        y = m / self._make_scalar(dtype)
+        return y.toarray()
+
+    @testing.for_dtypes('ifdFD')
+    # type promotion rules are different for ()-shaped arrays
+    @testing.numpy_cupy_allclose(sp_name='sp', type_check=False)
+    def test_divide_scalarlike(self, xp, sp, dtype):
+        m = self.make(xp, sp, self.dtype)
+        y = m / xp.array(self._make_scalar(dtype))
+        return y.toarray()
+
+    @testing.with_requires('scipy<1.11')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_divide_dense_row(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(4, dtype=self.dtype)
+        return m / x
+
+    @testing.with_requires('scipy<1.11')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_divide_dense_col(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(3, dtype=self.dtype).reshape(3, 1)
+        return m / x
+
+    @testing.with_requires('scipy<1.11')
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_divide_dense_matrix(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(12, dtype=self.dtype).reshape(3, 4)
+        return m / x
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_divide_csr_matrix(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = _make4(xp, sp, self.dtype)
+        return m / x
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCsrMatrixPowScipyComparison:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if (sys.platform == 'win32' and
+                cupyx.cusparse.getVersion() == 11301 and
+                self.dtype == cupy.complex128):
+            pytest.xfail('Known to fail on CUDA 11.2 for Windows')
+
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_pow_0(self, xp, sp):
+        m = _make_square(xp, sp, self.dtype)
+        return m ** 0
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_pow_1(self, xp, sp):
+        m = _make_square(xp, sp, self.dtype)
+        return m ** 1
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_pow_2(self, xp, sp):
+        m = _make_square(xp, sp, self.dtype)
+        return m ** 2
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_pow_3(self, xp, sp):
+        m = _make_square(xp, sp, self.dtype)
+        return m ** 3
+        return m ** 3
+
+    def test_pow_neg(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make_square(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m ** -1
+
+    def test_pow_not_square(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m ** 2
+
+    def test_pow_float(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make_square(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m ** 1.5
+
+    def test_pow_list(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make_square(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m ** []
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'ret_dtype': [None, numpy.float32, numpy.float64],
+    'axis': [None, 0, 1, -1, -2],
+}))
+@testing.with_requires('scipy')
+class TestCsrMatrixSum:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if runtime.is_hip and self.axis in (0, -2):
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.sum(axis=self.axis, dtype=self.ret_dtype)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_with_out(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if self.axis is None:
+            shape = ()
+        else:
+            shape = list(m.shape)
+            shape[self.axis] = 1
+            shape = tuple(shape)
+        out = xp.empty(shape, dtype=self.ret_dtype)
+        if xp is numpy:
+            # TODO(unno): numpy.matrix is used for scipy.sparse though
+            # cupy.ndarray is used for cupyx.scipy.sparse.
+            out = xp.asmatrix(out)
+        return m.sum(axis=self.axis, dtype=self.ret_dtype, out=out)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=self.axis, dtype=self.ret_dtype)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_with_out(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if self.axis is None:
+            shape = ()
+        else:
+            shape = list(m.shape)
+            shape[self.axis] = 1
+            shape = tuple(shape)
+        out = xp.empty(shape, dtype=self.ret_dtype)
+        if xp is numpy:
+            # TODO(unno): numpy.matrix is used for scipy.sparse though
+            # cupy.ndarray is used for cupyx.scipy.sparse.
+            out = xp.asmatrix(out)
+        return m.mean(axis=self.axis, dtype=self.ret_dtype, out=out)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCsrMatrixScipyCompressed:
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_get_shape(self, xp, sp):
+        return _make(xp, sp, self.dtype).get_shape()
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_getnnz(self, xp, sp):
+        return _make(xp, sp, self.dtype).getnnz()
+
+
+@testing.parameterize(*testing.product({
+    # TODO(takagi): Test dtypes
+    'axis': [None, -2, -1, 0, 1],
+    'dense': [False, True],  # means a sparse matrix but all elements filled
+}))
+@testing.with_requires('scipy>=0.19.0')
+class TestCsrMatrixScipyCompressedMinMax:
+    def _make_data_min(self, xp, sp, dense=False):
+        dm_data = testing.shaped_random((10, 20), xp=xp, scale=1.0)
+        if not dense:
+            dm_data[abs(dm_data) < 0.95] = 0
+        return sp.csr_matrix(xp.array(dm_data))
+
+    def _make_data_max(self, xp, sp, dense=False):
+        return -self._make_data_min(xp, sp, dense=dense)
+
+    def _make_data_min_explicit(self, xp, sp, axis):
+        dm_data = testing.shaped_random((10, 20), xp=xp, scale=1.0)
+        if xp is cupy:
+            dm_data[dm_data < 0.95] = 0
+        else:
+            # As SciPy sparse matrix does not have `explicit` parameter, we
+            # make SciPy inputs such that SciPy's spmatrix.min(axis=axis)
+            # returns the same value as CuPy's spmatrix.min(axis=axis,
+            # explicit=True).
+
+            # Put infinity instead of zeros so spmatrix.min(axis=axis) returns
+            # the smallest numbers except for zero.
+            dm_data[dm_data < 0.95] = numpy.inf
+
+            if axis is None:
+                # If all elements in the array are set to infinity, we make it
+                # have at least a zero so SciPy's spmatrix.min(axis=None)
+                # returns zero.
+                if numpy.isinf(dm_data).all():
+                    dm_data[0, 0] = 0
+            else:
+                if axis < 0:
+                    axis += 2
+
+                # If all elements in a row/column are set to infinity, we make
+                # it have at least a zero so spmatrix.min(axis=axis) returns
+                # zero for the row/column.
+                mask = numpy.zeros_like(dm_data, dtype=numpy.bool_)
+                if axis == 0:
+                    rows = dm_data.argmin(axis=0)
+                    cols = numpy.arange(20)
+                else:
+                    rows = numpy.arange(10)
+                    cols = dm_data.argmin(axis=1)
+                mask[rows, cols] = numpy.isinf(dm_data[rows, cols])
+                dm_data[mask] = 0
+
+        return sp.csr_matrix(xp.array(dm_data))
+
+    def _make_data_max_explicit(self, xp, sp, axis):
+        return -self._make_data_min_explicit(xp, sp, axis=axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_min(self, xp, sp):
+        data = self._make_data_min(xp, sp, dense=self.dense)
+        return data.min(axis=self.axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_min_explicit(self, xp, sp):
+        data = self._make_data_min_explicit(xp, sp, axis=self.axis)
+        if xp is cupy:
+            return data.min(axis=self.axis, explicit=True)
+        else:
+            return data.min(axis=self.axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_max(self, xp, sp):
+        data = self._make_data_max(xp, sp, dense=self.dense)
+        return data.max(axis=self.axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_max_explicit(self, xp, sp):
+        data = self._make_data_max_explicit(xp, sp, axis=self.axis)
+        if xp is cupy:
+            return data.max(axis=self.axis, explicit=True)
+        else:
+            return data.max(axis=self.axis)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp', type_check=False)
+    def test_argmin(self, xp, sp):
+        data = self._make_data_min(xp, sp, dense=self.dense)
+        # Due to a SciPy bug, the argmin output is different from the expected
+        # one
+        if self.axis is None and self.dense:
+            pytest.skip()
+        return xp.array(data.argmin(axis=self.axis))
+
+    @testing.numpy_cupy_array_equal(sp_name='sp', type_check=False)
+    def test_argmax(self, xp, sp):
+        data = self._make_data_max(xp, sp, dense=self.dense)
+        # Due to a SciPy bug, the argmin output is different from the expected
+        # one
+        if self.axis is None and self.dense:
+            pytest.skip()
+        return xp.array(data.argmax(axis=self.axis))
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@testing.with_requires('scipy')
+class TestCsrMatrixData:
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_dtype(self, xp, sp):
+        return _make(xp, sp, self.dtype).dtype
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_abs(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return abs(m)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_neg(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return -m
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_astype(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if numpy.dtype(self.dtype).kind == 'c':
+            t = 'D'
+        else:
+            t = 'd'
+        return m.astype(t)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_count_nonzero(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.count_nonzero()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_power(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.power(2)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_power_with_dtype(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if numpy.dtype(self.dtype).kind == 'c':
+            t = 'D'
+        else:
+            t = 'd'
+        return m.power(2, t)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_None(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=None)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_0(self, xp, sp):
+        if runtime.is_hip:
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=0)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_1(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=1)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_negative_1(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=-1)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mean_axis_negative_2(self, xp, sp):
+        if runtime.is_hip:
+            HIP_version = driver.get_build_version()
+            if HIP_version < 400:
+                pytest.skip('no working implementation')
+            elif HIP_version < 5_00_00000:
+                # I got HIPSPARSE_STATUS_INTERNAL_ERROR...
+                pytest.xfail('spmv is buggy (trans=True)')
+
+        m = _make(xp, sp, self.dtype)
+        return m.mean(axis=-2)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'ufunc': [
+        'arcsin', 'arcsinh', 'arctan', 'arctanh', 'ceil', 'deg2rad', 'expm1',
+        'floor', 'log1p', 'rad2deg', 'rint', 'sign', 'sin', 'sinh', 'sqrt',
+        'tan', 'tanh', 'trunc',
+    ],
+}))
+@testing.with_requires('scipy')
+class TestUfunc:
+
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=1e-5)
+    def test_ufun(self, xp, sp):
+        x = _make(xp, sp, self.dtype)
+        x.data *= 0.1
+        func = getattr(x, self.ufunc)
+        complex_unsupported = {'ceil', 'deg2rad', 'floor', 'rad2deg', 'trunc'}
+        if (numpy.dtype(self.dtype).kind == 'c' and
+                self.ufunc in complex_unsupported):
+            with pytest.raises(TypeError):
+                func()
+            return numpy.array(0)
+        else:
+            return func()
+
+
+class TestIsspmatrixCsr:
+
+    def test_csr(self):
+        x = sparse.csr_matrix(
+            (cupy.array([], 'f'),
+             cupy.array([], 'i'),
+             cupy.array([0], 'i')),
+            shape=(0, 0), dtype='f')
+        assert sparse.isspmatrix_csr(x) is True
+
+    def test_csc(self):
+        x = sparse.csr_matrix(
+            (cupy.array([], 'f'),
+             cupy.array([], 'i'),
+             cupy.array([0], 'i')),
+            shape=(0, 0), dtype='f')
+        assert sparse.isspmatrix_csc(x) is False
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, cupy.complex64, cupy.complex128],
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestCsrMatrixGetitem:
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_getitem_int_int(self, xp, sp):
+        assert _make(xp, sp, self.dtype)[0, 1] == 1
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_getitem_int_int_not_found(self, xp, sp):
+        assert _make(xp, sp, self.dtype)[1, 1] == 0
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_getitem_int_int_negative(self, xp, sp):
+        assert _make(xp, sp, self.dtype)[-1, -2] == 3
+
+    def test_getitem_int_int_too_small_row(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[-4, 0]
+
+    def test_getitem_int_int_too_large_row(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[3, 0]
+
+    def test_getitem_int_int_too_small_col(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[0, -5]
+
+    def test_getitem_int_int_too_large_col(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[0, 4]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_int(self, xp, sp):
+        return _make(xp, sp, self.dtype)[1]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_int_negative(self, xp, sp):
+        return _make(xp, sp, self.dtype)[-1]
+
+    def test_getitem_int_to_small(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[-4]
+
+    def test_getitem_int_to_large(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[3]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_int_none_slice(self, xp, sp):
+        return _make(xp, sp, self.dtype)[1, :]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_negative_int_none_slice(self, xp, sp):
+        return _make(xp, sp, self.dtype)[-1, :]
+
+    def test_getitem_int_too_small_none_slice(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[-4, :]
+
+    def test_getitem_int_too_large_none_slice(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(IndexError):
+                _make(xp, sp, self.dtype)[3, :]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice(self, xp, sp):
+        return _make(xp, sp, self.dtype)[1:3]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_negative(self, xp, sp):
+        return _make(xp, sp, self.dtype)[-2:-1]
+
+    # SciPy prior to 1.4 has bugs where either an IndexError is raised or a
+    # segfault occurs instead of returning an empty slice.
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_start_larger_than_stop(self, xp, sp):
+        return _make(xp, sp, self.dtype)[3:2]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_ellipsis(self, xp, sp):
+        return _make(xp, sp, self.dtype)[...]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_int_ellipsis(self, xp, sp):
+        return _make(xp, sp, self.dtype)[1, ...]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_rowslice_all(self, xp, sp):
+        # This test is adapted from Scipy
+        return _make(xp, sp, self.dtype)[slice(None, None, None)]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    @testing.with_requires('scipy>=1.9.3')
+    def test_getitem_rowslice_negative_stop(self, xp, sp):
+        # This test is adapted from Scipy
+        return _make(xp, sp, self.dtype)[slice(1, -2, 2)]
+
+    def test_getrow(self):
+
+        # This test is adapted from Scipy's CSR tests
+        N = 10
+        X = testing.shaped_random((N, N), cupy, seed=0)
+        X[X > 0.7] = 0
+        Xcsr = sparse.csr_matrix(X)
+
+        for i in range(N):
+            arr_row = X[i:i + 1, :]
+            csr_row = Xcsr.getrow(i)
+            assert sparse.isspmatrix_csr(csr_row)
+            assert (arr_row == csr_row.toarray()).all()
+
+    def test_getcol(self):
+        # This test is adapted from Scipy's CSR tests
+        N = 10
+        X = testing.shaped_random((N, N), cupy, seed=0)
+        X[X > 0.7] = 0
+        Xcsr = sparse.csr_matrix(X)
+
+        for i in range(N):
+            arr_col = X[:, i:i + 1]
+            csr_col = Xcsr.getcol(i)
+
+            assert sparse.isspmatrix_csr(csr_col)
+            assert (arr_col == csr_col.toarray()).all()
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, cupy.complex64, cupy.complex128],
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestCsrMatrixGetitem2:
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_start_too_small(self, xp, sp):
+        return _make(xp, sp, self.dtype)[-4:None]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_start_too_large(self, xp, sp):
+        return _make(xp, sp, self.dtype)[4:None]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_stop_too_small(self, xp, sp):
+        return _make(xp, sp, self.dtype)[None:-4]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_getitem_slice_stop_too_large(self, xp, sp):
+        return _make(xp, sp, self.dtype)[None:4]
+
+
+# CUB SpMV works only when the matrix size is nonzero
+@testing.parameterize(*testing.product({
+    'make_method': ['_make', '_make_unordered', '_make_duplicate'],
+    'dtype': [numpy.float32, numpy.float64, cupy.complex64, cupy.complex128],
+}))
+@testing.with_requires('scipy')
+@pytest.mark.skipif(
+    cupy.cuda.cub._get_cuda_build_version() >= 11000,
+    reason='CUDA built-in CUB SpMV is buggy, see cupy/cupy#3822')
+@pytest.mark.skipif(runtime.is_hip, reason='hipCUB does not provide spmv')
+@pytest.mark.skipif(
+    not cupy.cuda.cub.available,
+    reason='The CUB routine is not enabled')
+class TestCubSpmv:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def cub(self):
+        old_accelerators = _accelerator.get_routine_accelerators()
+        _accelerator.set_routine_accelerators(['cub'])
+        yield
+        _accelerator.set_routine_accelerators(old_accelerators)
+
+    @property
+    def make(self):
+        return globals()[self.make_method]
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_mul_dense_vector(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        x = xp.arange(4).astype(self.dtype)
+        if xp is numpy:
+            return m * x
+
+        # xp is cupy, first ensure we really use CUB
+        func = 'cupyx.scipy.sparse._csr.cub.device_csrmv'
+        with testing.AssertFunctionIsCalled(func):
+            m * x
+        # ...then perform the actual computation
+        return m * x
+
+
+@testing.parameterize(*testing.product({
+    'a_dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'b_dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'shape': [(4, 25), (10, 10), (25, 4)],
+    'nz_rate': [0.1, 0.5],
+    'opt': ['maximum', 'minimum'],
+}))
+@testing.with_requires('scipy')
+class TestCsrMatrixMaximumMinimum:
+
+    def _make_array(self, shape, dtype, xp):
+        dtype = numpy.dtype(dtype)
+        if dtype.char in 'fF':
+            real_dtype = 'float32'
+        elif dtype.char in 'dD':
+            real_dtype = 'float64'
+        a = testing.shaped_random(shape, xp, dtype=real_dtype, scale=2)
+        a = (a - 1) / self.nz_rate
+        a[a > 1] = 0
+        a[a < -1] = 0
+        return a
+
+    def _make_matrix(self, shape, dtype, xp):
+        dtype = numpy.dtype(dtype)
+        a = self._make_array(shape, dtype, xp)
+        if dtype.char in 'FD':
+            a = a + 1j * self._make_array(shape, dtype, xp)
+        return a
+
+    def _make_sp_matrix(self, dtype, xp, sp):
+        return sp.csr_matrix(self._make_matrix(self.shape, dtype, xp))
+
+    def _make_sp_matrix_row(self, dtype, xp, sp):
+        shape = 1, self.shape[1]
+        return sp.csr_matrix(self._make_matrix(shape, dtype, xp))
+
+    def _make_sp_matrix_col(self, dtype, xp, sp):
+        shape = self.shape[0], 1
+        return sp.csr_matrix(self._make_matrix(shape, dtype, xp))
+
+    def _make_sp_matrix_shape(self, shape, dtype, xp, sp):
+        return sp.csr_matrix(self._make_matrix(shape, dtype, xp))
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_sparse(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix(self.b_dtype, xp, sp)
+        return getattr(a, self.opt)(b)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_sparse_row(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix_row(self.b_dtype, xp, sp)
+        if xp == numpy:
+            # SciPy does not support sparse broadcasting
+            return getattr(a, self.opt)(b.toarray())
+        else:
+            return getattr(a, self.opt)(b).toarray()
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_sparse_col(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix_col(self.b_dtype, xp, sp)
+        if xp == numpy:
+            # SciPy does not support sparse broadcasting
+            return getattr(a, self.opt)(b.toarray())
+        else:
+            return getattr(a, self.opt)(b).toarray()
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_dense(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix(self.b_dtype, xp, sp).toarray()
+        return getattr(a, self.opt)(b)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_dense_row(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix_row(self.b_dtype, xp, sp).toarray()
+        return getattr(a, self.opt)(b)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_dense_col(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix_col(self.b_dtype, xp, sp).toarray()
+        return getattr(a, self.opt)(b)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_scalar_plus(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        return getattr(a, self.opt)(0.5)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_scalar_minus(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        return getattr(a, self.opt)(-0.5)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_scalar_zero(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        return getattr(a, self.opt)(0)
+
+    def test_ng_shape(self):
+        xp, sp = cupy, sparse
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        for i, j in ((-1, 0), (1, 0), (0, -1), (0, 1)):
+            shape = self.shape[0] + i, self.shape[1] + j
+            b = self._make_sp_matrix_shape(shape, self.b_dtype, xp, sp)
+            with pytest.raises(ValueError):
+                getattr(a, self.opt)(b)
+
+
+@testing.parameterize(*testing.product({
+    'a_dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'b_dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'shape': [(6, 15), (15, 6)],
+    'opt': ['_eq_', '_ne_', '_lt_', '_gt_', '_le_', '_ge_'],
+}))
+@testing.with_requires('scipy>=1.2')
+class TestCsrMatrixComparison:
+    nz_rate = 0.3
+
+    def _make_array(self, shape, dtype, xp):
+        dtype = numpy.dtype(dtype)
+        if dtype.char in 'fF':
+            real_dtype = 'float32'
+        elif dtype.char in 'dD':
+            real_dtype = 'float64'
+        a = testing.shaped_random(shape, xp, dtype=real_dtype, scale=2)
+        a = (a - 1) / self.nz_rate
+        a[a > 1] = 0
+        a[a < -1] = 0
+        return a
+
+    def _make_matrix(self, shape, dtype, xp):
+        dtype = numpy.dtype(dtype)
+        a = self._make_array(shape, dtype, xp)
+        if dtype.char in 'FD':
+            a = a + 1j * self._make_array(shape, dtype, xp)
+        return a
+
+    def _make_sp_matrix(self, dtype, xp, sp):
+        return sp.csr_matrix(self._make_matrix(self.shape, dtype, xp))
+
+    def _make_sp_matrix_row(self, dtype, xp, sp):
+        shape = 1, self.shape[1]
+        return sp.csr_matrix(self._make_matrix(shape, dtype, xp))
+
+    def _make_sp_matrix_col(self, dtype, xp, sp):
+        shape = self.shape[0], 1
+        return sp.csr_matrix(self._make_matrix(shape, dtype, xp))
+
+    def _make_sp_matrix_shape(self, shape, dtype, xp, sp):
+        return sp.csr_matrix(self._make_matrix(shape, dtype, xp))
+
+    def _compare(self, a, b):
+        if self.opt == '_eq_':
+            return a == b
+        elif self.opt == '_ne_':
+            return a != b
+        elif self.opt == '_lt_':
+            return a < b
+        elif self.opt == '_gt_':
+            return a > b
+        elif self.opt == '_le_':
+            return a <= b
+        elif self.opt == '_ge_':
+            return a >= b
+
+    @contextlib.contextmanager
+    def _ignore_efficiency_warning(self):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', sparse.SparseEfficiencyWarning)
+            yield
+
+    @contextlib.contextmanager
+    def _assert_warns_efficiency(self, sp, scalar_rhs=None):
+        if scalar_rhs is None and self._compare(0, 0):
+            with testing.assert_warns(sp.SparseEfficiencyWarning):
+                yield
+        elif scalar_rhs is not None and self._compare(0, scalar_rhs):
+            if sp is sparse:  # cupy
+                # TODO(kataoka): Test it, too. But, it seems the current
+                # implementation does not depend on the scalar value.
+                with self._ignore_efficiency_warning():
+                    yield
+            else:  # scipy
+                with testing.assert_warns(sp.SparseEfficiencyWarning):
+                    yield
+        else:
+            yield
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_sparse(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix(self.b_dtype, xp, sp)
+        with self._assert_warns_efficiency(sp):
+            return self._compare(a, b)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_sparse_row(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix_row(self.b_dtype, xp, sp)
+        if xp == numpy:
+            # SciPy does not support sparse broadcasting
+            return self._compare(a, b.toarray())
+        else:
+            with self._assert_warns_efficiency(sp):
+                return self._compare(a, b).toarray()
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_sparse_col(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix_col(self.b_dtype, xp, sp)
+        if xp == numpy:
+            # SciPy does not support sparse broadcasting
+            return self._compare(a, b.toarray())
+        else:
+            with self._assert_warns_efficiency(sp):
+                return self._compare(a, b).toarray()
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_dense(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix(self.b_dtype, xp, sp).toarray()
+        return self._compare(a, b)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_dense_row(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix_row(self.b_dtype, xp, sp).toarray()
+        return self._compare(a, b)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_dense_col(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        b = self._make_sp_matrix_col(self.b_dtype, xp, sp).toarray()
+        return self._compare(a, b)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_scalar_plus(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        with self._assert_warns_efficiency(sp, 0.5):
+            return self._compare(a, 0.5)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_scalar_minus(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        with self._assert_warns_efficiency(sp, -0.5):
+            return self._compare(a, -0.5)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_scalar_zero(self, xp, sp):
+        if self.opt in ('_le_', '_ge_'):
+            # <= and >= with 0 are not supported by SciPy
+            pytest.skip()
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        with self._assert_warns_efficiency(sp, 0):
+            return self._compare(a, 0)
+
+    @testing.numpy_cupy_array_equal(sp_name='sp')
+    def test_scalar_nan(self, xp, sp):
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        with self._assert_warns_efficiency(sp, numpy.nan):
+            return self._compare(a, numpy.nan)
+
+    def test_ng_shape(self):
+        xp, sp = cupy, sparse
+        a = self._make_sp_matrix(self.a_dtype, xp, sp)
+        for i, j in ((-1, 0), (1, 0), (0, -1), (0, 1)):
+            shape = self.shape[0] + i, self.shape[1] + j
+            b = self._make_sp_matrix_shape(shape, self.b_dtype, xp, sp)
+            with pytest.raises(ValueError):
+                with self._ignore_efficiency_warning():
+                    self._compare(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(8, 5), (5, 5), (5, 8)],
+}))
+@testing.with_requires('scipy>=1.5.0')
+class TestCsrMatrixDiagonal:
+    density = 0.5
+
+    def _make_matrix(self, dtype):
+        a = testing.shaped_random(self.shape, numpy, dtype=dtype)
+        mask = testing.shaped_random(self.shape, numpy, dtype='f', scale=1.0)
+        a[mask > self.density] = 0
+        scipy_a = scipy.sparse.csr_matrix(a)
+        cupyx_a = sparse.csr_matrix(cupy.array(a))
+        return scipy_a, cupyx_a
+
+    @testing.for_dtypes('fdFD')
+    def test_diagonal(self, dtype):
+        scipy_a, cupyx_a = self._make_matrix(dtype)
+        m, n = self.shape
+        for k in range(-m, n+1):
+            scipy_diag = scipy_a.diagonal(k=k)
+            cupyx_diag = cupyx_a.diagonal(k=k)
+            testing.assert_allclose(scipy_diag, cupyx_diag)
+
+    def _test_setdiag(self, scipy_a, cupyx_a, x, k):
+        scipy_a = scipy_a.copy()
+        cupyx_a = cupyx_a.copy()
+        scipy_a.setdiag(x, k=k)
+        cupyx_a.setdiag(cupy.array(x), k=k)
+        testing.assert_allclose(scipy_a.data, cupyx_a.data)
+        testing.assert_array_equal(scipy_a.indices, cupyx_a.indices)
+        testing.assert_array_equal(scipy_a.indptr, cupyx_a.indptr)
+
+    @testing.for_dtypes('fdFD')
+    def test_setdiag(self, dtype):
+        scipy_a, cupyx_a = self._make_matrix(dtype)
+        m, n = self.shape
+        for k in range(-m+1, n):
+            m_st, n_st = max(0, -k), max(0, k)
+            for d in (-1, 0, 1):
+                x_len = min(m - m_st, n - n_st) + d
+                if x_len <= 0:
+                    continue
+                x = numpy.ones((x_len,), dtype=dtype)
+                self._test_setdiag(scipy_a, cupyx_a, x, k)
+
+    @testing.for_dtypes('fdFD')
+    def test_setdiag_scalar(self, dtype):
+        scipy_a, cupyx_a = self._make_matrix(dtype)
+        x = numpy.array(1.0, dtype=dtype)
+        m, n = self.shape
+        for k in range(-m+1, n):
+            self._test_setdiag(scipy_a, cupyx_a, x, k)
+
+    def test_setdiag_invalid(self):
+        dtype = 'f'
+        scipy_a, cupyx_a = self._make_matrix(dtype)
+        x = numpy.array(1.0, dtype=dtype)
+        m, n = self.shape
+        for k in (-m, n):
+            with pytest.raises(ValueError):
+                scipy_a.setdiag(x, k=k)
+            with pytest.raises(ValueError):
+                cupyx_a.setdiag(x, k=k)
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_dia.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_dia.py
new file mode 100644
index 0000000..7a53233
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_dia.py
@@ -0,0 +1,370 @@
+import pickle
+import unittest
+
+
+import numpy
+import pytest
+try:
+    import scipy.sparse  # NOQA
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+import cupy
+from cupy import testing
+from cupy.cuda import driver
+from cupy.cuda import runtime
+from cupyx.scipy import sparse
+
+
+def _make(xp, sp, dtype):
+    data = xp.array([[0, 1, 2], [3, 4, 5]], dtype)
+    offsets = xp.array([0, -1], 'i')
+    # 0, 0, 0, 0
+    # 3, 1, 0, 0
+    # 0, 4, 2, 0
+    return sp.dia_matrix((data, offsets), shape=(3, 4))
+
+
+def _make_complex(xp, sp, dtype):
+    data = xp.array([[0, 1, 2], [3, 4, 5]], dtype)
+    if dtype in [numpy.complex64, numpy.complex128]:
+        data = data - 1j
+    offsets = xp.array([0, -1], 'i')
+    # 0, 0, 0, 0
+    # 3 - 1j, 1 - 1j, 0, 0
+    # 0, 4 - 1j, 2 - 1j, 0
+    return sp.dia_matrix((data, offsets), shape=(3, 4))
+
+
+def _make_empty(xp, sp, dtype):
+    data = xp.array([[]], 'f')
+    offsets = xp.array([0], 'i')
+    return sp.dia_matrix((data, offsets), shape=(3, 4))
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+class TestDiaMatrix(unittest.TestCase):
+
+    def setUp(self):
+        self.m = _make(cupy, sparse, self.dtype)
+
+    def test_dtype(self):
+        assert self.m.dtype == self.dtype
+
+    def test_data(self):
+        assert self.m.data.dtype == self.dtype
+        testing.assert_array_equal(
+            self.m.data, cupy.array([[0, 1, 2], [3, 4, 5]], self.dtype))
+
+    def test_offsets(self):
+        if (runtime.is_hip and self.dtype == numpy.float32
+                and driver.get_build_version() == 400):
+            pytest.xfail('generated wrong result -- may be buggy?')
+        assert self.m.offsets.dtype == numpy.int32
+        testing.assert_array_equal(
+            self.m.offsets, cupy.array([0, -1], self.dtype))
+
+    def test_shape(self):
+        assert self.m.shape == (3, 4)
+
+    def test_ndim(self):
+        assert self.m.ndim == 2
+
+    def test_nnz(self):
+        assert self.m.nnz == 5
+
+    def test_conj(self):
+        n = _make_complex(cupy, sparse, self.dtype)
+        cupy.testing.assert_array_equal(n.conj().data, n.data.conj())
+
+    def test_conjugate(self):
+        n = _make_complex(cupy, sparse, self.dtype)
+        cupy.testing.assert_array_equal(n.conjugate().data, n.data.conj())
+
+    @unittest.skipUnless(scipy_available, 'requires scipy')
+    def test_str(self):
+        if numpy.dtype(self.dtype).kind == 'f':
+            expect = '''  (1, 1)\t1.0
+  (2, 2)\t2.0
+  (1, 0)\t3.0
+  (2, 1)\t4.0'''
+        else:
+            expect = '''  (1, 1)\t(1+0j)
+  (2, 2)\t(2+0j)
+  (1, 0)\t(3+0j)
+  (2, 1)\t(4+0j)'''
+        assert str(self.m) == expect
+
+    def test_toarray(self):
+        m = self.m.toarray()
+        expect = [
+            [0, 0, 0, 0],
+            [3, 1, 0, 0],
+            [0, 4, 2, 0]
+        ]
+        assert m.flags.c_contiguous
+        cupy.testing.assert_allclose(m, expect)
+
+    def test_pickle_roundtrip(self):
+        s = _make(cupy, sparse, self.dtype)
+        s2 = pickle.loads(pickle.dumps(s))
+        assert s.shape == s2.shape
+        assert s.dtype == s2.dtype
+        if scipy_available:
+            assert (s.get() != s2.get()).count_nonzero() == 0
+
+    def test_diagonal(self):
+        testing.assert_array_equal(
+            self.m.diagonal(-2), cupy.array([0], self.dtype))
+        testing.assert_array_equal(
+            self.m.diagonal(-1), cupy.array([3, 4], self.dtype))
+        testing.assert_array_equal(
+            self.m.diagonal(), cupy.array([0, 1, 2], self.dtype))
+        testing.assert_array_equal(
+            self.m.diagonal(1), cupy.array([0, 0, 0], self.dtype))
+        testing.assert_array_equal(
+            self.m.diagonal(2), cupy.array([0, 0], self.dtype))
+        testing.assert_array_equal(
+            self.m.diagonal(3), cupy.array([0], self.dtype))
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@unittest.skipUnless(scipy_available, 'requires scipy')
+class TestDiaMatrixInit(unittest.TestCase):
+
+    def setUp(self):
+        self.shape = (3, 4)
+
+    def data(self, xp):
+        return xp.array([[1, 2, 3], [4, 5, 6]], self.dtype)
+
+    def offsets(self, xp):
+        return xp.array([0, -1], 'i')
+
+    def test_shape_none(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.dia_matrix(
+                    (self.data(xp), self.offsets(xp)), shape=None)
+
+    def test_scipy_sparse(self):
+        s_h = scipy.sparse.dia_matrix((self.data(numpy), self.offsets(numpy)),
+                                      shape=self.shape)
+        s_d = sparse.dia_matrix(s_h)
+        s_h2 = s_d.get()
+        assert s_h.shape == s_d.shape
+        assert s_h.dtype == s_d.dtype
+        assert s_h.shape == s_h2.shape
+        assert s_h.dtype == s_h2.dtype
+        assert (s_h.data == s_h2.data).all()
+        assert (s_h.offsets == s_h2.offsets).all()
+
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=1e-5)
+    def test_intlike_shape(self, xp, sp):
+        s = sp.dia_matrix((self.data(xp), self.offsets(xp)),
+                          shape=(xp.array(self.shape[0]),
+                                 xp.int32(self.shape[1])))
+        assert isinstance(s.shape[0], int)
+        assert isinstance(s.shape[1], int)
+        return s
+
+    def test_large_rank_offset(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.dia_matrix(
+                    (self.data(xp), self.offsets(xp)[None]), shape=self.shape)
+
+    def test_large_rank_data(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            with pytest.raises(ValueError):
+                sp.dia_matrix(
+                    (self.data(xp)[None], self.offsets(xp)), shape=self.shape)
+
+    def test_data_offsets_different_size(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            offsets = xp.array([0, -1, 1], 'i')
+            with pytest.raises(ValueError):
+                sp.dia_matrix(
+                    (self.data(xp), offsets), shape=self.shape)
+
+    def test_duplicated_offsets(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            offsets = xp.array([1, 1], 'i')
+            with pytest.raises(ValueError):
+                sp.dia_matrix(
+                    (self.data(xp), offsets), shape=self.shape)
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_conj(self, xp, sp):
+        n = _make_complex(xp, sp, self.dtype)
+        cupy.testing.assert_array_equal(n.conj().data, n.data.conj())
+
+
+@testing.parameterize(*testing.product({
+    'make_method': ['_make', '_make_empty'],
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+@unittest.skipUnless(scipy_available, 'requires scipy')
+class TestDiaMatrixScipyComparison(unittest.TestCase):
+
+    def setUp(self):
+        if runtime.is_hip:
+            if self.make_method in ('_make_empty',):
+                # xcsr2coo could raise HIPSPARSE_STATUS_INVALID_VALUE, maybe
+                # because we have a zero matrix (nnz=0)?
+                pytest.xfail('may be buggy')
+
+    @property
+    def make(self):
+        return globals()[self.make_method]
+
+    @testing.numpy_cupy_equal(sp_name='sp')
+    def test_nnz_axis(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.nnz
+
+    def test_nnz_axis_not_none(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = self.make(xp, sp, self.dtype)
+            with pytest.raises(NotImplementedError):
+                m.getnnz(axis=0)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_toarray(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.toarray()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_A(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.A
+
+    def test_sum_tuple_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.sum(axis=(0, 1))
+
+    def test_sum_float_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(TypeError):
+                m.sum(axis=0.0)
+
+    def test_sum_too_large_axis(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            with pytest.raises(ValueError):
+                m.sum(axis=3)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocoo(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.tocoo()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocoo_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocoo(copy=True)
+        assert m.data is not n.data
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsc(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.tocsc()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsc_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocsc(copy=True)
+        assert m.data is not n.data
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsr(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.tocsr()
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_tocsr_copy(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        n = m.tocsr(copy=True)
+        assert m.data is not n.data
+        return n
+
+    @testing.numpy_cupy_allclose(sp_name='sp', _check_sparse_format=False)
+    def test_transpose(self, xp, sp):
+        m = self.make(xp, sp, self.dtype)
+        return m.transpose()
+
+    @testing.with_requires('scipy>=1.5.0')
+    def test_diagonal_error(self):
+        # Before scipy 1.5.0 dia_matrix diagonal raised
+        # `ValueError`, now returns empty array.
+        # Check #3469
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            m = _make(xp, sp, self.dtype)
+            d = m.diagonal(k=10)
+            assert d.size == 0
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'ret_dtype': [None, numpy.float32, numpy.float64],
+    'axis': [None, 0, 1, -1, -2],
+}))
+@unittest.skipUnless(scipy_available, 'requires scipy')
+class TestDiaMatrixSum(unittest.TestCase):
+
+    def setUp(self):
+        if runtime.is_hip and self.axis in (0, -2):
+            HIP_version = driver.get_build_version()
+            if HIP_version < 5_00_00000:
+                # internally a temporary CSC matrix is generated and thus
+                # casues problems (see test_csc.py)
+                pytest.xfail('spmv is buggy (trans=True)')
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        return m.sum(axis=self.axis, dtype=self.ret_dtype)
+
+    @testing.numpy_cupy_allclose(sp_name='sp')
+    def test_sum_with_out(self, xp, sp):
+        m = _make(xp, sp, self.dtype)
+        if self.axis is None:
+            shape = ()
+        else:
+            shape = list(m.shape)
+            shape[self.axis] = 1
+            shape = tuple(shape)
+        out = xp.empty(shape, dtype=self.ret_dtype)
+        if xp is numpy:
+            # TODO(unno): numpy.matrix is used for scipy.sparse though
+            # cupy.ndarray is used for cupyx.scipy.sparse.
+            out = xp.asmatrix(out)
+        return m.sum(axis=self.axis, dtype=self.ret_dtype, out=out)
+
+
+class TestIsspmatrixDia(unittest.TestCase):
+
+    def test_dia(self):
+        x = sparse.dia_matrix(
+            (cupy.array([], 'f'),
+             cupy.array([0], 'i')),
+            shape=(0, 0), dtype='f')
+        assert sparse.isspmatrix_dia(x) is True
+
+    def test_csr(self):
+        x = sparse.csr_matrix(
+            (cupy.array([], 'f'),
+             cupy.array([], 'i'),
+             cupy.array([0], 'i')),
+            shape=(0, 0), dtype='f')
+        assert sparse.isspmatrix_dia(x) is False
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_extract.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_extract.py
new file mode 100644
index 0000000..17c171d
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_extract.py
@@ -0,0 +1,75 @@
+import unittest
+
+import numpy
+try:
+    import scipy.sparse
+    scipy_available = True
+except ImportError:
+    scipy_available = False
+
+import cupy
+from cupy import testing
+from cupyx.scipy import sparse
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(8, 3), (4, 4), (3, 8)],
+    'a_format': ['dense', 'csr', 'csc', 'coo'],
+    'out_format': [None, 'csr', 'csc'],
+}))
+@unittest.skipUnless(scipy_available, 'requires scipy')
+class TestExtract(unittest.TestCase):
+
+    density = 0.75
+
+    def _make_matrix(self, dtype):
+        a = testing.shaped_random(self.shape, numpy, dtype=dtype)
+        a[a > self.density] = 0
+        b = cupy.array(a)
+        if self.a_format == 'csr':
+            a = scipy.sparse.csr_matrix(a)
+            b = sparse.csr_matrix(b)
+        elif self.a_format == 'csc':
+            a = scipy.sparse.csc_matrix(a)
+            b = sparse.csc_matrix(b)
+        elif self.a_format == 'coo':
+            a = scipy.sparse.coo_matrix(a)
+            b = sparse.coo_matrix(b)
+        return a, b
+
+    @testing.for_dtypes('fdFD')
+    def test_tril(self, dtype):
+        np_a, cp_a = self._make_matrix(dtype)
+        m, n = self.shape
+        for k in range(-m+1, n):
+            np_out = scipy.sparse.tril(np_a, k=k, format=self.out_format)
+            cp_out = sparse.tril(cp_a, k=k, format=self.out_format)
+            assert np_out.format == cp_out.format
+            assert np_out.nnz == cp_out.nnz
+            cupy.testing.assert_allclose(np_out.todense(), cp_out.todense())
+
+    @testing.for_dtypes('fdFD')
+    def test_triu(self, dtype):
+        np_a, cp_a = self._make_matrix(dtype)
+        m, n = self.shape
+        for k in range(-m+1, n):
+            np_out = scipy.sparse.triu(np_a, k=k, format=self.out_format)
+            cp_out = sparse.triu(cp_a, k=k, format=self.out_format)
+            assert np_out.format == cp_out.format
+            assert np_out.nnz == cp_out.nnz
+            cupy.testing.assert_allclose(np_out.todense(), cp_out.todense())
+
+    @testing.for_dtypes('fdFD')
+    def test_find(self, dtype):
+        if self.out_format is not None:
+            unittest.SkipTest()
+        np_a, cp_a = self._make_matrix(dtype)
+        np_row, np_col, np_data = scipy.sparse.find(np_a)
+        cp_row, cp_col, cp_data = sparse.find(cp_a)
+        # Note: Check the results by reconstructing the sparse matrix from the
+        # results of find, as SciPy and CuPy differ in the data order.
+        np_out = scipy.sparse.coo_matrix((np_data, (np_row, np_col)),
+                                         shape=self.shape)
+        cp_out = sparse.coo_matrix((cp_data, (cp_row, cp_col)),
+                                   shape=self.shape)
+        cupy.testing.assert_allclose(np_out.todense(), cp_out.todense())
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_index.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_index.py
new file mode 100644
index 0000000..dc57abd
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_index.py
@@ -0,0 +1,579 @@
+import functools
+import itertools
+
+import numpy
+import pytest
+try:
+    import scipy.sparse
+except ImportError:
+    pass
+
+import cupy
+import cupyx
+from cupy import testing
+from cupy.cuda import runtime
+from cupyx.scipy import sparse
+
+
+def _get_index_combos(idx):
+    return [dict['arr_fn'](idx, dtype=dict['dtype'])
+            for dict in testing.product({
+                "arr_fn": [numpy.array, cupy.array],
+                "dtype": [numpy.int32, numpy.int64]
+            })]
+
+
+def _check_shares_memory(xp, sp, x, y):
+    if sp.issparse(x) and sp.issparse(y):
+        assert not xp.shares_memory(x.indptr, y.indptr)
+        assert not xp.shares_memory(x.indices, y.indices)
+        assert not xp.shares_memory(x.data, y.data)
+
+
+@testing.parameterize(*testing.product({
+    'format': ['csr', 'csc'],
+    'density': [0.9],
+    'dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'n_rows': [25, 150],
+    'n_cols': [25, 150]
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestSetitemIndexing:
+
+    def _run(self, maj, min=None, data=5):
+
+        import scipy.sparse
+        for i in range(2):
+            shape = self.n_rows, self.n_cols
+            a = testing.shaped_sparse_random(
+                shape, sparse, self.dtype, self.density, self.format)
+            expected = testing.shaped_sparse_random(
+                shape, scipy.sparse, self.dtype, self.density, self.format)
+
+            maj_h = maj.get() if isinstance(maj, cupy.ndarray) else maj
+            min_h = min.get() if isinstance(min, cupy.ndarray) else min
+
+            data_is_cupy = isinstance(data, (cupy.ndarray, sparse.spmatrix))
+            data_h = data.get() if data_is_cupy else data
+
+            if min is not None:
+                actual = a
+                actual[maj, min] = data
+
+                expected[maj_h, min_h] = data_h
+            else:
+                actual = a
+                actual[maj] = data
+
+                expected[maj_h] = data_h
+
+        if cupyx.scipy.sparse.isspmatrix(actual):
+            actual.sort_indices()
+            expected.sort_indices()
+
+            cupy.testing.assert_array_equal(
+                actual.indptr, expected.indptr)
+            cupy.testing.assert_array_equal(
+                actual.indices, expected.indices)
+            cupy.testing.assert_array_equal(
+                actual.data, expected.data)
+
+        else:
+
+            cupy.testing.assert_array_equal(
+                actual.ravel(), cupy.array(expected).ravel())
+
+    def test_set_sparse(self):
+
+        x = cupyx.scipy.sparse.random(1, 5, format='csr', density=0.8)
+
+        # Test inner indexing with sparse data
+        for maj, min in zip(_get_index_combos([0, 1, 2, 3, 5]),
+                            _get_index_combos([1, 2, 3, 4, 5])):
+            self._run(maj, min, data=x)
+        self._run([0, 1, 2, 3, 5], [1, 2, 3, 4, 5], data=x)
+
+        # Test 2d major indexing 1d minor indexing with sparse data
+        for maj, min in zip(_get_index_combos([[0], [1], [2], [3], [5]]),
+                            _get_index_combos([1, 2, 3, 4, 5])):
+            self._run(maj, min, data=x)
+        self._run([[0], [1], [2], [3], [5]], [1, 2, 3, 4, 5], data=x)
+
+        # Test 1d major indexing 2d minor indexing with sparse data
+        for maj, min in zip(_get_index_combos([0, 1, 2, 3, 5]),
+                            _get_index_combos([[1], [2], [3], [4], [5]])):
+            self._run(maj, min, data=x)
+        self._run([0, 1, 2, 3, 5], [[1], [2], [3], [4], [5]], data=x)
+
+        # Test minor indexing numpy scalar / cupy 0-dim
+        for maj, min in zip(_get_index_combos([0, 2, 4, 5, 6]),
+                            _get_index_combos(1)):
+            self._run(maj, min, data=x)
+
+    @testing.with_requires('scipy>=1.5.0')
+    def test_set_zero_dim_bool_mask(self):
+
+        zero_dim_data = [numpy.array(5), cupy.array(5)]
+
+        for data in zero_dim_data:
+            self._run([False, True], data=data)
+
+    def test_set_zero_dim_scalar(self):
+
+        zero_dim_data = [numpy.array(5), cupy.array(5)]
+
+        for data in zero_dim_data:
+            self._run(slice(5, 10000), data=data)
+            self._run([1, 5, 4, 5], data=data)
+            self._run(0, 2, data=data)
+
+    def test_major_slice(self):
+        self._run(slice(5, 10000), data=5)
+        self._run(slice(5, 4), data=5)
+        self._run(slice(4, 5, 2), data=5)
+        self._run(slice(5, 4, -2), data=5)
+        self._run(slice(2, 4), slice(0, 2), [[4], [1]])
+        self._run(slice(2, 4), slice(0, 2), [[4, 5], [6, 7]])
+        self._run(slice(2, 4), 0, [[4], [6]])
+
+        self._run(slice(5, 9))
+        self._run(slice(9, 5))
+
+    def test_major_all(self):
+        self._run(slice(None))
+
+    def test_major_scalar(self):
+        self._run(10)
+
+    def test_major_fancy(self):
+        self._run([1, 5, 4])
+        self._run([10, 2])
+        self._run([2])
+
+    def test_major_slice_minor_slice(self):
+        self._run(slice(1, 5), slice(1, 5))
+
+    def test_major_slice_minor_all(self):
+        self._run(slice(1, 5), slice(None))
+        self._run(slice(5, 1), slice(None))
+
+    def test_major_slice_minor_scalar(self):
+        self._run(slice(1, 5), 5)
+        self._run(slice(5, 1), 5)
+        self._run(slice(5, 1, -1), 5)
+
+    def test_major_slice_minor_fancy(self):
+        self._run(slice(1, 10, 2), [1, 5, 4])
+
+    def test_major_scalar_minor_slice(self):
+        self._run(5, slice(1, 5))
+
+    def test_major_scalar_minor_all(self):
+        self._run(5, slice(None))
+
+    def test_major_scalar_minor_scalar(self):
+        self._run(5, 5)
+        self._run(10, 24, 5)
+
+    def test_major_scalar_minor_fancy(self):
+        self._run(5, [1, 5, 4])
+
+    def test_major_all_minor_all(self):
+        self._run(slice(None), slice(None))
+
+    def test_major_all_minor_fancy(self):
+        for min in _get_index_combos(
+                [0, 3, 4, 1, 1, 5, 5, 2, 3, 4, 5, 4, 1, 5]):
+            self._run(slice(None), min)
+
+        self._run(slice(None), [0, 3, 4, 1, 1, 5, 5, 2, 3, 4, 5, 4, 1, 5])
+
+    def test_major_fancy_minor_fancy(self):
+
+        for maj, min in zip(_get_index_combos([1, 2, 3, 4, 1, 6, 1, 8, 9]),
+                            _get_index_combos([1, 5, 2, 3, 4, 5, 4, 1, 5])):
+            self._run(maj, min)
+
+        self._run([1, 2, 3, 4, 1, 6, 1, 8, 9], [1, 5, 2, 3, 4, 5, 4, 1, 5])
+
+        for idx in _get_index_combos([1, 5, 4]):
+            self._run(idx, idx)
+        self._run([1, 5, 4], [1, 5, 4])
+
+        for maj, min in zip(_get_index_combos([2, 0, 10]),
+                            _get_index_combos([9, 2, 1])):
+            self._run(maj, min)
+        self._run([2, 0, 10], [9, 2, 1])
+
+        for maj, min in zip(_get_index_combos([2, 9]),
+                            _get_index_combos([2, 1])):
+            self._run(maj, min)
+        self._run([2, 0], [2, 1])
+
+    def test_major_fancy_minor_all(self):
+
+        for idx in _get_index_combos([1, 5, 4]):
+            self._run(idx, slice(None))
+        self._run([1, 5, 4], slice(None))
+
+    def test_major_fancy_minor_scalar(self):
+
+        for idx in _get_index_combos([1, 5, 4]):
+            self._run(idx, 5)
+        self._run([1, 5, 4], 5)
+
+    def test_major_fancy_minor_slice(self):
+
+        for idx in _get_index_combos([1, 5, 4]):
+            self._run(idx, slice(1, 5))
+            self._run(idx, slice(5, 1, -1))
+        self._run([1, 5, 4], slice(1, 5))
+        self._run([1, 5, 4], slice(5, 1, -1))
+
+    def test_major_bool_fancy(self):
+        rand_bool = testing.shaped_random(self.n_rows, dtype=bool)
+        self._run(rand_bool)
+
+    def test_major_slice_with_step(self):
+
+        # positive step
+        self._run(slice(1, 10, 2))
+        self._run(slice(2, 10, 5))
+        self._run(slice(0, 10, 10))
+
+        self._run(slice(1, None, 2))
+        self._run(slice(2, None, 5))
+        self._run(slice(0, None,  10))
+
+        # negative step
+        self._run(slice(10, 1, -2))
+        self._run(slice(10, 2, -5))
+        self._run(slice(10, 0, -10))
+
+        self._run(slice(10, None, -2))
+        self._run(slice(10, None, -5))
+        self._run(slice(10, None, -10))
+
+    def test_major_slice_with_step_minor_slice_with_step(self):
+
+        # positive step
+        self._run(slice(1, 10, 2), slice(1, 10, 2))
+        self._run(slice(2, 10, 5), slice(2, 10, 5))
+        self._run(slice(0, 10, 10), slice(0, 10, 10))
+
+        # negative step
+        self._run(slice(10, 1, 2), slice(10, 1, 2))
+        self._run(slice(10, 2, 5), slice(10, 2, 5))
+        self._run(slice(10, 0, 10), slice(10, 0, 10))
+
+    def test_major_slice_with_step_minor_all(self):
+
+        # positive step
+        self._run(slice(1, 10, 2), slice(None))
+        self._run(slice(2, 10, 5), slice(None))
+        self._run(slice(0, 10, 10), slice(None))
+
+        # negative step
+        self._run(slice(10, 1, 2), slice(None))
+        self._run(slice(10, 2, 5), slice(None))
+        self._run(slice(10, 0, 10), slice(None))
+
+    @testing.with_requires('scipy>=1.5.0')
+    def test_fancy_setting_bool(self):
+        # Unfortunately, boolean setting is implemented slightly
+        # differently between Scipy 1.4 and 1.5. Using the most
+        # up-to-date version in CuPy.
+
+        for maj in _get_index_combos(
+                [[True], [False], [False], [True], [True], [True]]):
+            self._run(maj, data=5)
+        self._run([[True], [False], [False], [True], [True], [True]], data=5)
+
+        for maj in _get_index_combos([True, False, False, True, True, True]):
+            self._run(maj, data=5)
+        self._run([True, False, False, True, True, True], data=5)
+
+        for maj in _get_index_combos([[True], [False], [True]]):
+            self._run(maj, data=5)
+        self._run([[True], [False], [True]], data=5)
+
+    def test_fancy_setting(self):
+
+        for maj, data in zip(_get_index_combos([0, 5, 10, 2]),
+                             _get_index_combos([1, 2, 3, 2])):
+            self._run(maj, 0, data)
+        self._run([0, 5, 10, 2], 0, [1, 2, 3, 2])
+
+        # Indexes with duplicates should follow 'last-in-wins'
+        # But Cupy dense indexing doesn't support this yet:
+        # ref: https://github.com/cupy/cupy/issues/3836
+        # Starting with an empty array for now, since insertions
+        # use `last-in-wins`.
+        self.density = 0.0  # Zeroing out density to force only insertions
+        for maj, min, data in zip(_get_index_combos([0, 5, 10, 2, 0, 10]),
+                                  _get_index_combos([1, 2, 3, 4, 1, 3]),
+                                  _get_index_combos([1, 2, 3, 4, 5, 6])):
+            self._run(maj, min, data)
+
+
+class IndexingTestBase:
+
+    def _make_matrix(self, sp, dtype):
+        shape = self.n_rows, self.n_cols
+        return testing.shaped_sparse_random(
+            shape, sp, dtype, self.density, self.format)
+
+    def _make_indices(self, xp, dtype=None):
+        indices = []
+        for ind in self.indices:
+            if isinstance(ind, slice):
+                indices.append(ind)
+            else:
+                indices.append(xp.array(ind, dtype=dtype))
+
+        return tuple(indices)
+
+
+_int_index = [0, -1, 10, -10]
+_slice_index = [
+    slice(0, 0), slice(None), slice(3, 17), slice(17, 3, -1)
+]
+_slice_index_full = [
+    slice(0, 0), slice(0, 1), slice(5, 6), slice(None),
+    slice(3, 17, 1), slice(17, 3, 1), slice(2, -1, 1), slice(-1, 2, 1),
+    slice(3, 17, -1), slice(17, 3, -1), slice(2, -1, -1), slice(-1, 2, -1),
+    slice(3, 17, 2), slice(17, 3, 2), slice(3, 17, -2), slice(17, 3, -2),
+]
+_int_array_index = [
+    [], [0], [0, 0], [1, 5, 4, 5, 2, 4, 1]
+]
+
+
+@testing.parameterize(*testing.product({
+    'format': ['csr', 'csc'],
+    'density': [0.0, 0.5],
+    'n_rows': [1, 25],
+    'n_cols': [1, 25],
+    'indices': (
+        # Int
+        _int_index
+        # Slice
+        + _slice_index_full
+        # Int x Int
+        + list(itertools.product(_int_index, _int_index))
+        # Slice x Slice
+        + list(itertools.product(_slice_index, _slice_index))
+        # Int x Slice
+        + list(itertools.product(_int_index, _slice_index))
+        + list(itertools.product(_slice_index, _int_index))
+        # Ellipsis
+        + [
+            (Ellipsis,),
+            (Ellipsis, slice(None)),
+            (slice(None), Ellipsis),
+            (Ellipsis, 1),
+            (1, Ellipsis),
+            (slice(1, None), Ellipsis),
+            (Ellipsis, slice(1, None)),
+        ]
+    ),
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestSliceIndexing(IndexingTestBase):
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_array_equal(
+        sp_name='sp', type_check=False, accept_error=IndexError)
+    def test_indexing(self, xp, sp, dtype):
+        a = self._make_matrix(sp, dtype)
+        res = a[self.indices]
+        _check_shares_memory(xp, sp, a, res)
+        return res
+
+
+def skip_HIP_0_size_matrix():
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kw):
+            try:
+                impl(self, *args, **kw)
+            except AssertionError as e:
+                if runtime.is_hip:
+                    assert 'ValueError: hipSPARSE' in str(e)
+                    pytest.xfail('may be buggy')
+                raise
+        return test_func
+    return decorator
+
+
+def _check_bounds(indices, n_rows, n_cols, **kwargs):
+    if not isinstance(indices, tuple):
+        indices = (indices,)
+    for index, size in zip(indices, [n_rows, n_cols]):
+        if isinstance(index, list):
+            for ind in index:
+                if not (0 <= ind < size):
+                    # CuPy does not check boundaries.
+                    # pytest.skip('Out of bounds')
+                    return False
+    return True
+
+
+@testing.parameterize(*[params for params in testing.product({
+    'format': ['csr', 'csc'],
+    'density': [0.0, 0.5],
+    'n_rows': [1, 25],
+    'n_cols': [1, 25],
+    'indices': (
+        # Array
+        _int_array_index
+        # Array x Int
+        + list(itertools.product(_int_array_index, _int_index))
+        + list(itertools.product(_int_index, _int_array_index))
+        # Array x Slice
+        + list(itertools.product(_slice_index, _int_array_index))
+        # SciPy chose inner indexing for int-array x slice inputs.
+        # + list(itertools.product(_int_array_index, _slice_index))
+        # Array x Array (Inner indexing)
+        + [
+            ([], []),
+            ([0], [0]),
+            ([1, 5, 4], [1, 5, 4]),
+            ([2, 0, 10, 0, 2], [9, 2, 1, 0, 2]),
+            ([2, 0, 10, 0], [9, 2, 1, 0]),
+            ([2, 0, 2], [2, 1, 1]),
+            ([2, 0, 2], [2, 1, 2]),
+        ]
+    )
+}) if _check_bounds(**params)])
+@testing.with_requires('scipy>=1.4.0')
+class TestArrayIndexing(IndexingTestBase):
+
+    @skip_HIP_0_size_matrix()
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_array_equal(
+        sp_name='sp', type_check=False, accept_error=IndexError)
+    def test_list_indexing(self, xp, sp, dtype):
+        a = self._make_matrix(sp, dtype)
+        res = a[self.indices]
+        _check_shares_memory(xp, sp, a, res)
+        return res
+
+    @skip_HIP_0_size_matrix()
+    @testing.for_dtypes('fdFD')
+    @testing.for_dtypes('il', name='ind_dtype')
+    @testing.numpy_cupy_array_equal(
+        sp_name='sp', type_check=False, accept_error=IndexError)
+    def test_numpy_ndarray_indexing(self, xp, sp, dtype, ind_dtype):
+        a = self._make_matrix(sp, dtype)
+        indices = self._make_indices(numpy, ind_dtype)
+        res = a[indices]
+        _check_shares_memory(xp, sp, a, res)
+        return res
+
+    @skip_HIP_0_size_matrix()
+    @testing.for_dtypes('fdFD')
+    @testing.for_dtypes('il', name='ind_dtype')
+    @testing.numpy_cupy_array_equal(
+        sp_name='sp', type_check=False, accept_error=IndexError)
+    def test_cupy_ndarray_indexing(self, xp, sp, dtype, ind_dtype):
+        a = self._make_matrix(sp, dtype)
+        indices = self._make_indices(xp, ind_dtype)
+        res = a[indices]
+        _check_shares_memory(xp, sp, a, res)
+        return res
+
+
+@testing.parameterize(*testing.product({
+    'format': ['csr', 'csc'],
+    'density': [0.0, 0.5],
+    'indices': [
+        # Bool array x Int
+        ([True, False, True], 3),
+        (2, [True, False, True, False, True]),
+        # Bool array x Slice
+        ([True, False, True], slice(None)),
+        ([True, False, True], slice(1, 4)),
+        (slice(None), [True, False, True, False, True]),
+        (slice(1, 4), [True, False, True, False, True]),
+        # Bool array x Bool array
+        # SciPy chose inner indexing for int-array x slice inputs.
+        ([True, False, True], [True, False, True]),
+    ],
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestBoolMaskIndexing(IndexingTestBase):
+
+    n_rows = 3
+    n_cols = 5
+
+    # In older environments (e.g., py35, scipy 1.4), scipy sparse arrays are
+    # crashing when indexed with native Python boolean list.
+    @testing.with_requires('scipy>=1.5.0')
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_array_equal(sp_name='sp', type_check=False)
+    def test_bool_mask(self, xp, sp, dtype):
+        a = self._make_matrix(sp, dtype)
+        res = a[self.indices]
+        _check_shares_memory(xp, sp, a, res)
+        return res
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_array_equal(sp_name='sp', type_check=False)
+    def test_numpy_bool_mask(self, xp, sp, dtype):
+        a = self._make_matrix(sp, dtype)
+        indices = self._make_indices(numpy)
+        res = a[indices]
+        _check_shares_memory(xp, sp, a, res)
+        return res
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_array_equal(sp_name='sp', type_check=False)
+    def test_cupy_bool_mask(self, xp, sp, dtype):
+        a = self._make_matrix(sp, dtype)
+        indices = self._make_indices(xp)
+        res = a[indices]
+        _check_shares_memory(xp, sp, a, res)
+        return res
+
+
+@testing.parameterize(*testing.product({
+    'format': ['csr', 'csc'],
+    'density': [0.4],
+    'dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'n_rows': [25, 150],
+    'n_cols': [25, 150],
+    'indices': [
+        ('foo',),
+        (2, 'foo'),
+        ([[0, 0], [1, 1]]),
+    ],
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestIndexingIndexError(IndexingTestBase):
+
+    def test_indexing_index_error(self):
+        for xp, sp in [(numpy, scipy.sparse), (cupy, sparse)]:
+            a = self._make_matrix(sp, numpy.float32)
+            with pytest.raises(IndexError):
+                a[self.indices]
+
+
+@testing.parameterize(*testing.product({
+    'format': ['csr', 'csc'],
+    'density': [0.4],
+    'dtype': ['float32', 'float64', 'complex64', 'complex128'],
+    'n_rows': [25, 150],
+    'n_cols': [25, 150],
+    'indices': [
+        ([1, 2, 3], [1, 2, 3, 4]),
+    ],
+}))
+@testing.with_requires('scipy>=1.4.0')
+class TestIndexingValueError(IndexingTestBase):
+
+    def test_indexing_value_error(self):
+        for xp, sp in [(numpy, scipy.sparse), (cupy, sparse)]:
+            a = self._make_matrix(sp, numpy.float32)
+            with pytest.raises(ValueError):
+                a[self.indices]
diff --git a/tests/cupyx_tests/scipy_tests/sparse_tests/test_linalg.py b/tests/cupyx_tests/scipy_tests/sparse_tests/test_linalg.py
new file mode 100644
index 0000000..1973903
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/sparse_tests/test_linalg.py
@@ -0,0 +1,1697 @@
+import contextlib
+import functools
+import re
+import io
+import unittest
+import warnings
+
+import numpy
+import pytest
+try:
+    import scipy.sparse
+    import scipy.sparse.linalg
+    import scipy.stats
+except ImportError:
+    pass
+
+import cupy
+from cupyx import cusparse, cusolver
+from cupy import testing
+from cupy.cuda import driver
+from cupy.cuda import runtime
+from cupy.testing import _condition
+from cupyx.scipy import sparse
+import cupyx.scipy.sparse.linalg  # NOQA
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+}))
+@testing.with_requires('scipy')
+@pytest.mark.skipif(runtime.is_hip, reason='lsqr not supported')
+class TestLsqr(unittest.TestCase):
+
+    def setUp(self):
+        rvs = scipy.stats.randint(0, 15).rvs
+        self.A = scipy.sparse.random(50, 50, density=0.2, data_rvs=rvs)
+        self.b = numpy.random.randint(15, size=50)
+
+    def test_size(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            A = sp.csr_matrix(self.A, dtype=self.dtype)
+            b = xp.array(numpy.append(self.b, [1]), dtype=self.dtype)
+            with pytest.raises(ValueError):
+                sp.linalg.lsqr(A, b)
+
+    def test_shape(self):
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            A = sp.csr_matrix(self.A, dtype=self.dtype)
+            b = xp.array(numpy.tile(self.b, (2, 1)), dtype=self.dtype)
+            with pytest.raises(ValueError):
+                sp.linalg.lsqr(A, b)
+
+    @_condition.retry(10)
+    @testing.numpy_cupy_allclose(atol=1e-1, sp_name='sp')
+    def test_csrmatrix(self, xp, sp):
+        A = sp.csr_matrix(self.A, dtype=self.dtype)
+        b = xp.array(self.b, dtype=self.dtype)
+        x = sp.linalg.lsqr(A, b)
+        return x[0]
+
+    @_condition.retry(10)
+    @testing.numpy_cupy_allclose(atol=1e-1, sp_name='sp')
+    def test_ndarray(self, xp, sp):
+        A = xp.array(self.A.A, dtype=self.dtype)
+        b = xp.array(self.b, dtype=self.dtype)
+        x = sp.linalg.lsqr(A, b)
+        return x[0]
+
+
+@testing.parameterize(*testing.product({
+    'ord': [None, -numpy.inf, -2, -1, 0, 1, 2, 3, numpy.inf, 'fro'],
+    'dtype': [
+        numpy.float32,
+        numpy.float64,
+        numpy.complex64,
+        numpy.complex128
+    ],
+    'axis': [None, (0, 1), (1, -2)],
+}))
+@testing.with_requires('scipy')
+class TestMatrixNorm:
+
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4, sp_name='sp',
+                                 accept_error=(ValueError,
+                                               NotImplementedError))
+    def test_matrix_norm(self, xp, sp):
+        if runtime.is_hip and self.ord in (1, -1, numpy.inf, -numpy.inf):
+            pytest.xfail('csc spmv is buggy')
+        if self.ord == 2:
+            pytest.xfail('ord=2 is not implemented in cupy')
+        a = xp.arange(9, dtype=self.dtype) - 4
+        b = a.reshape((3, 3))
+        b = sp.csr_matrix(b, dtype=self.dtype)
+        return sp.linalg.norm(b, ord=self.ord, axis=self.axis)
+
+
+@testing.parameterize(*testing.product({
+    'ord': [None, -numpy.inf, -2, -1, 0, 1, 2, numpy.inf, 'fro'],
+    'dtype': [
+        numpy.float32,
+        numpy.float64,
+        numpy.complex64,
+        numpy.complex128
+    ],
+    'transpose': [True, False],
+    'axis': [0, (1,), (-2,), -1],
+})
+)
+@testing.with_requires('scipy')
+class TestVectorNorm:
+
+    @testing.numpy_cupy_allclose(rtol=1e-3, atol=1e-4, sp_name='sp',
+                                 accept_error=(ValueError,))
+    def test_vector_norm(self, xp, sp):
+        if runtime.is_hip:
+            if (self.axis in (0, (-2,))
+                    and self.ord in (-2, -1, 0, 1, 2, None)):
+                pytest.xfail('csc spmv is buggy')
+
+        a = xp.arange(9, dtype=self.dtype) - 4
+        b = a.reshape((3, 3))
+        b = sp.csr_matrix(b, dtype=self.dtype)
+        if self.transpose:
+            b = b.T
+        return sp.linalg.norm(b, ord=self.ord, axis=self.axis)
+
+# TODO : TestVsNumpyNorm
+
+
+@testing.parameterize(*testing.product({
+    'which': ['LM', 'LA', 'SA'],
+    'k': [3, 6, 12],
+    'return_eigenvectors': [True, False],
+    'use_linear_operator': [True, False],
+}))
+@testing.with_requires('scipy')
+class TestEigsh:
+    n = 30
+    density = 0.33
+    tol = {numpy.float32: 1e-5, numpy.complex64: 1e-5, 'default': 1e-12}
+    res_tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _make_matrix(self, dtype, xp):
+        shape = (self.n, self.n)
+        a = testing.shaped_random(shape, xp, dtype=dtype)
+        mask = testing.shaped_random(shape, xp, dtype='f', scale=1)
+        a[mask > self.density] = 0
+        a = a * a.conj().T
+        return a
+
+    def _test_eigsh(self, a, xp, sp):
+        ret = sp.linalg.eigsh(a, k=self.k, which=self.which,
+                              return_eigenvectors=self.return_eigenvectors)
+        if self.return_eigenvectors:
+            w, x = ret
+            # Check the residuals to see if eigenvectors are correct.
+            ax_xw = a @ x - xp.multiply(x, w.reshape(1, self.k))
+            res = xp.linalg.norm(ax_xw) / xp.linalg.norm(w)
+            tol = self.res_tol[numpy.dtype(a.dtype).char.lower()]
+            assert (res < tol)
+        else:
+            w = ret
+        return xp.sort(w)
+
+    @pytest.mark.parametrize('format', ['csr', 'csc', 'coo'])
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=tol, atol=tol, sp_name='sp')
+    def test_sparse(self, format, dtype, xp, sp):
+        if runtime.is_hip and format == 'csc':
+            pytest.xfail('may be buggy')  # trans=True
+
+        a = self._make_matrix(dtype, xp)
+        a = sp.coo_matrix(a).asformat(format)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        return self._test_eigsh(a, xp, sp)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=tol, atol=tol, sp_name='sp')
+    def test_dense(self, dtype, xp, sp):
+        a = self._make_matrix(dtype, xp)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        return self._test_eigsh(a, xp, sp)
+
+    @pytest.mark.xfail(
+        reason='eigsh works wrong (#5001)',
+        raises=AssertionError,
+    )
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=tol, atol=tol, sp_name='sp')
+    def test_dense_low_rank(self, dtype, xp, sp):
+        n = self.n
+        rank = 5
+        # density is ignored.
+        a = testing.shaped_random((n, rank), xp, dtype=dtype, scale=1).dot(
+            testing.shaped_random((rank, n), xp, dtype=dtype, scale=1))
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        return self._test_eigsh(a, xp, sp)
+
+    def test_invalid(self):
+        if self.use_linear_operator is True:
+            pytest.skip()
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            a = xp.diag(xp.ones((self.n, ), dtype='f'))
+            with pytest.raises(ValueError):
+                sp.linalg.eigsh(xp.ones((2, 1), dtype='f'))
+            with pytest.raises(ValueError):
+                sp.linalg.eigsh(a, k=0)
+        xp, sp = cupy, sparse
+        a = xp.diag(xp.ones((self.n, ), dtype='f'))
+        with pytest.raises(ValueError):
+            sp.linalg.eigsh(xp.ones((1,), dtype='f'))
+        with pytest.raises(TypeError):
+            sp.linalg.eigsh(xp.ones((2, 2), dtype='i'))
+        with pytest.raises(ValueError):
+            sp.linalg.eigsh(a, k=self.n)
+        with pytest.raises(ValueError):
+            sp.linalg.eigsh(a, k=self.k, which='SM')
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(30, 29), (29, 29), (29, 30)],
+    'k': [3, 6, 12],
+    'return_vectors': [True, False],
+    'use_linear_operator': [True, False],
+}))
+@testing.with_requires('scipy')
+class TestSvds:
+    density = 0.33
+    tol = {numpy.float32: 1e-4, numpy.complex64: 2e-4, 'default': 1e-12}
+
+    def _make_matrix(self, dtype, xp):
+        a = testing.shaped_random(self.shape, xp, dtype=dtype)
+        mask = testing.shaped_random(self.shape, xp, dtype='f', scale=1)
+        a[mask > self.density] = 0
+        return a
+
+    def _test_svds(self, a, xp, sp):
+        ret = sp.linalg.svds(a, k=self.k,
+                             return_singular_vectors=self.return_vectors)
+        if self.return_vectors:
+            u, s, vt = ret
+            # Check the results with u @ s @ vt, as singular vectors don't
+            # necessarily match.
+            return u @ xp.diag(s) @ vt
+        else:
+            return xp.sort(ret)
+
+    @pytest.mark.parametrize('format', ['csr', 'csc', 'coo'])
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=tol, atol=tol, sp_name='sp')
+    def test_sparse(self, format, dtype, xp, sp):
+        if runtime.is_hip and format in ('csr', 'csc'):
+            pytest.xfail('may be buggy')  # trans=True
+
+        a = self._make_matrix(dtype, xp)
+        a = sp.coo_matrix(a).asformat(format)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        return self._test_svds(a, xp, sp)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=tol, atol=tol, sp_name='sp')
+    def test_dense(self, dtype, xp, sp):
+        a = self._make_matrix(dtype, xp)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        return self._test_svds(a, xp, sp)
+
+    @pytest.mark.xfail(
+        reason='eigsh works wrong (#5001)',
+        raises=AssertionError,
+    )
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=tol, atol=tol, sp_name='sp')
+    def test_dense_low_rank(self, dtype, xp, sp):
+        m, n = self.shape
+        rank = 5
+        # density is ignored.
+        a = testing.shaped_random((m, rank), xp, dtype=dtype, scale=1).dot(
+            testing.shaped_random((rank, n), xp, dtype=dtype, scale=1))
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        return self._test_svds(a, xp, sp)
+
+    def test_invalid(self):
+        if self.use_linear_operator is True:
+            pytest.skip()
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            a = xp.diag(xp.ones(self.shape, dtype='f'))
+            with pytest.raises(ValueError):
+                sp.linalg.svds(a, k=0)
+        xp, sp = cupy, sparse
+        a = xp.diag(xp.ones(self.shape, dtype='f'))
+        with pytest.raises(ValueError):
+            sp.linalg.svds(xp.ones((1,), dtype='f'))
+        with pytest.raises(TypeError):
+            sp.linalg.svds(xp.ones((2, 2), dtype='i'))
+        with pytest.raises(ValueError):
+            sp.linalg.svds(a, k=min(self.shape))
+        with pytest.raises(ValueError):
+            sp.linalg.svds(a, k=self.k, which='SM')
+
+
+@testing.parameterize(*testing.product({
+    'x0': [None, 'ones'],
+    'M': [None, 'jacobi'],
+    'atol': [None, 'select-by-dtype'],
+    'b_ndim': [1, 2],
+    'use_linear_operator': [False, True],
+}))
+@testing.with_requires('scipy')
+class TestCg:
+    n = 30
+    density = 0.33
+    _atol = {'f': 1e-5, 'd': 1e-12}
+
+    def _make_matrix(self, dtype, xp):
+        dtype = numpy.dtype(dtype)
+        shape = (self.n, 10)
+        a = testing.shaped_random(shape, xp, dtype=dtype.char.lower(), scale=1)
+        if dtype.char in 'FD':
+            a = a + 1j * testing.shaped_random(
+                shape, xp, dtype=dtype.char.lower(), scale=1)
+        mask = testing.shaped_random(shape, xp, dtype='f', scale=1)
+        a[mask > self.density] = 0
+        a = a @ a.conj().T
+        a = a + xp.diag(xp.ones((self.n,), dtype=dtype.char.lower()))
+        M = None
+        if self.M == 'jacobi':
+            M = xp.diag(1.0 / xp.diag(a))
+        return a, M
+
+    def _make_normalized_vector(self, dtype, xp):
+        b = testing.shaped_random((self.n,), xp, dtype=dtype)
+        return b / xp.linalg.norm(b)
+
+    def _test_cg(self, dtype, xp, sp, a, M):
+        dtype = numpy.dtype(dtype)
+        b = self._make_normalized_vector(dtype, xp)
+        if self.b_ndim == 2:
+            b = b.reshape(self.n, 1)
+        x0 = None
+        if self.x0 == 'ones':
+            x0 = xp.ones((self.n,), dtype=dtype)
+        atol = None
+        if self.atol == 'select-by-dtype':
+            atol = self._atol[dtype.char.lower()]
+        if atol is None and xp == numpy:
+            # Note: If atol is None or not specified, Scipy (at least 1.5.3)
+            # raises DeprecationWarning
+            with pytest.deprecated_call():
+                return sp.linalg.cg(a, b, x0=x0, M=M, atol=atol)
+        else:
+            return sp.linalg.cg(a, b, x0=x0, M=M, atol=atol)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_dense(self, dtype, xp, sp):
+        a, M = self._make_matrix(dtype, xp)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+            if M is not None:
+                M = sp.linalg.aslinearoperator(M)
+        return self._test_cg(dtype, xp, sp, a, M)
+
+    @pytest.mark.parametrize('format', ['csr', 'csc', 'coo'])
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_sparse(self, format, dtype, xp, sp):
+        if runtime.is_hip and format == 'csc':
+            pytest.xfail('may be buggy')  # trans=True
+
+        a, M = self._make_matrix(dtype, xp)
+        a = sp.coo_matrix(a).asformat(format)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        if M is not None:
+            M = sp.coo_matrix(M).asformat(format)
+            if self.use_linear_operator:
+                M = sp.linalg.aslinearoperator(M)
+        return self._test_cg(dtype, xp, sp, a, M)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_empty(self, dtype, xp, sp):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.use_linear_operator is False):
+            pytest.skip()
+        a = xp.empty((0, 0), dtype=dtype)
+        b = xp.empty((0,), dtype=dtype)
+        if self.atol is None and xp == numpy:
+            # Note: If atol is None or not specified, Scipy (at least 1.5.3)
+            # raises DeprecationWarning
+            with pytest.deprecated_call():
+                return sp.linalg.cg(a, b)
+        else:
+            return sp.linalg.cg(a, b)
+
+    @testing.for_dtypes('fdFD')
+    def test_callback(self, dtype):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.use_linear_operator is False):
+            pytest.skip()
+        xp, sp = cupy, sparse
+        a, M = self._make_matrix(dtype, xp)
+        b = self._make_normalized_vector(dtype, xp)
+        is_called = False
+
+        def callback(x):
+            print(xp.linalg.norm(b - a @ x))
+            nonlocal is_called
+            is_called = True
+        sp.linalg.cg(a, b, callback=callback)
+        assert is_called
+
+    def test_invalid(self):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.use_linear_operator is False):
+            pytest.skip()
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            a, M = self._make_matrix('f', xp)
+            b = self._make_normalized_vector('f', xp)
+            ng_a = xp.ones((self.n, ), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cg(ng_a, b, atol=self.atol)
+            ng_a = xp.ones((self.n, self.n + 1), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cg(ng_a, b, atol=self.atol)
+            ng_a = xp.ones((self.n, self.n, 1), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cg(ng_a, b, atol=self.atol)
+            ng_b = xp.ones((self.n + 1,), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cg(a, ng_b, atol=self.atol)
+            ng_b = xp.ones((self.n, 2), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cg(a, ng_b, atol=self.atol)
+            ng_x0 = xp.ones((self.n + 1,), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cg(a, b, x0=ng_x0, atol=self.atol)
+            ng_M = xp.diag(xp.ones((self.n + 1,), dtype='f'))
+            with pytest.raises(ValueError):
+                sp.linalg.cg(a, b, M=ng_M, atol=self.atol)
+        xp, sp = cupy, sparse
+        b = self._make_normalized_vector('f', xp)
+        ng_a = xp.ones((self.n, self.n), dtype='i')
+        with pytest.raises(TypeError):
+            sp.linalg.cg(ng_a, b, atol=self.atol)
+
+
+@testing.parameterize(*testing.product({
+    'x0': [None, 'ones'],
+    'M': [None, 'jacobi'],
+    'atol': [None, 'select-by-dtype'],
+    'b_ndim': [1, 2],
+    'restart': [None, 10],
+    'use_linear_operator': [False, True],
+}))
+@testing.with_requires('scipy>=1.4')
+class TestGmres:
+    n = 30
+    density = 0.2
+    _atol = {'f': 1e-5, 'd': 1e-12}
+
+    # TODO(kataoka): Fix the `lstsq` call in CuPy's `gmres`
+    @pytest.fixture(autouse=True)
+    def ignore_futurewarning(self):
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                'ignore', '`rcond` parameter will change', FutureWarning,
+            )
+            yield
+
+    def _make_matrix(self, dtype, xp):
+        dtype = numpy.dtype(dtype)
+        shape = (self.n, self.n)
+        a = testing.shaped_random(shape, xp, dtype=dtype, scale=1)
+        mask = testing.shaped_random(shape, xp, dtype='f', scale=1)
+        a[mask > self.density] = 0
+        diag = xp.diag(testing.shaped_random(
+            (self.n,), xp, dtype=dtype.char.lower(), scale=1) + 1)
+        a[diag > 0] = 0
+        a = a + diag
+        M = None
+        if self.M == 'jacobi':
+            M = xp.diag(1.0 / xp.diag(a))
+        return a, M
+
+    def _make_normalized_vector(self, dtype, xp):
+        b = testing.shaped_random((self.n,), xp, dtype=dtype, scale=1)
+        return b / xp.linalg.norm(b)
+
+    def _test_gmres(self, dtype, xp, sp, a, M):
+        dtype = numpy.dtype(dtype)
+        b = self._make_normalized_vector(dtype, xp)
+        if self.b_ndim == 2:
+            b = b.reshape(self.n, 1)
+        x0 = None
+        if self.x0 == 'ones':
+            x0 = xp.ones((self.n,), dtype=dtype)
+        atol = None
+        if self.atol == 'select-by-dtype':
+            atol = self._atol[dtype.char.lower()]
+        if atol is None and xp == numpy:
+            # Note: If atol is None or not specified, Scipy (at least 1.5.3)
+            # raises DeprecationWarning
+            with pytest.deprecated_call():
+                return sp.linalg.gmres(
+                    a, b, x0=x0, restart=self.restart, M=M, atol=atol)
+        else:
+            return sp.linalg.gmres(
+                a, b, x0=x0, restart=self.restart, M=M, atol=atol)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_dense(self, dtype, xp, sp):
+        a, M = self._make_matrix(dtype, xp)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+            if M is not None:
+                M = sp.linalg.aslinearoperator(M)
+        return self._test_gmres(dtype, xp, sp, a, M)
+
+    @pytest.mark.parametrize('format', ['csr', 'csc', 'coo'])
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_sparse(self, format, dtype, xp, sp):
+        if runtime.is_hip and format == 'csc':
+            pytest.xfail('may be buggy')  # trans=True
+
+        a, M = self._make_matrix(dtype, xp)
+        a = sp.coo_matrix(a).asformat(format)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        if M is not None:
+            M = sp.coo_matrix(M).asformat(format)
+            if self.use_linear_operator:
+                M = sp.linalg.aslinearoperator(M)
+        return self._test_gmres(dtype, xp, sp, a, M)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_empty(self, dtype, xp, sp):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.restart is None and self.use_linear_operator is False):
+            pytest.skip()
+        a = xp.empty((0, 0), dtype=dtype)
+        b = xp.empty((0,), dtype=dtype)
+        if self.atol is None and xp == numpy:
+            # Note: If atol is None or not specified, Scipy (at least 1.5.3)
+            # raises DeprecationWarning
+            with pytest.deprecated_call():
+                return sp.linalg.gmres(a, b)
+        else:
+            return sp.linalg.gmres(a, b)
+
+    @testing.for_dtypes('fdFD')
+    def test_callback(self, dtype):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.restart is None and self.use_linear_operator is False):
+            pytest.skip()
+        xp, sp = cupy, sparse
+        a, M = self._make_matrix(dtype, xp)
+        b = self._make_normalized_vector(dtype, xp)
+        is_called = False
+
+        def callback1(x):
+            print(xp.linalg.norm(b - a @ x))
+            nonlocal is_called
+            is_called = True
+        sp.linalg.gmres(a, b, callback=callback1, callback_type='x')
+        assert is_called
+        is_called = False
+
+        def callback2(pr_norm):
+            print(pr_norm)
+            nonlocal is_called
+            is_called = True
+        sp.linalg.gmres(a, b, callback=callback2, callback_type='pr_norm')
+        assert is_called
+
+    def test_invalid(self):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.restart is None and self.use_linear_operator is False):
+            pytest.skip()
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            a, M = self._make_matrix('f', xp)
+            b = self._make_normalized_vector('f', xp)
+            ng_a = xp.ones((self.n, ), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.gmres(ng_a, b)
+            ng_a = xp.ones((self.n, self.n + 1), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.gmres(ng_a, b)
+            ng_a = xp.ones((self.n, self.n, 1), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.gmres(ng_a, b)
+            ng_b = xp.ones((self.n + 1,), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.gmres(a, ng_b)
+            ng_b = xp.ones((self.n, 2), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.gmres(a, ng_b)
+            ng_x0 = xp.ones((self.n + 1,), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.gmres(a, b, x0=ng_x0)
+            ng_M = xp.diag(xp.ones((self.n + 1,), dtype='f'))
+            with pytest.raises(ValueError):
+                sp.linalg.gmres(a, b, M=ng_M)
+            ng_callback_type = '?'
+            with pytest.raises(ValueError):
+                sp.linalg.gmres(a, b, callback_type=ng_callback_type)
+        xp, sp = cupy, sparse
+        b = self._make_normalized_vector('f', xp)
+        ng_a = xp.ones((self.n, self.n), dtype='i')
+        with pytest.raises(TypeError):
+            sp.linalg.gmres(ng_a, b)
+
+
+def skip_HIP_spMM_error(outer=()):
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kw):
+            if (runtime.is_hip and self.inner_modification == 'sparse'
+                    and self.outer_modification in outer):
+                pytest.xfail('spMM is buggy')  # trans=True
+            impl(self, *args, **kw)
+        return test_func
+    return decorator
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'outer_modification': [
+        'normal', 'transpose', 'hermitian'],
+    'inner_modification': [
+        'normal', 'sparse', 'linear_operator', 'class_matvec', 'class_matmat'],
+    'M': [1, 6],
+    'N': [1, 7],
+}))
+@testing.with_requires('scipy>=1.4')
+class TestLinearOperator:
+
+    # modified from scipy
+    # class that defines parametrized custom cases
+    # adapted from scipy's analogous tests
+    def _inner_cases(self, xp, sp, A):
+        # creating base-matrix-like class with default
+        # matrix-vector and adjoint-matrix-vector impl
+
+        def mv(x):
+            return A.dot(x)
+
+        def rmv(x):
+            return A.T.conj().dot(x)
+
+        # defining the user-defined classes
+        class BaseMatlike(sp.linalg.LinearOperator):
+
+            def __init__(self):
+                self.dtype = A.dtype
+                self.shape = A.shape
+
+            def _adjoint(self):
+                shape = self.shape[1], self.shape[0]
+                return sp.linalg.LinearOperator(
+                    matvec=rmv, rmatvec=mv, dtype=self.dtype, shape=shape)
+
+        class HasMatvec(BaseMatlike):
+
+            def _matvec(self, x):
+                return mv(x)
+
+        class HasMatmat(BaseMatlike):
+
+            def _matmat(self, x):
+                return mv(x)
+
+        if self.inner_modification == 'normal':
+            return sp.linalg.aslinearoperator(A)
+        if self.inner_modification == 'sparse':
+            # TODO(asi1024): Fix to return contiguous matrix.
+            return sp.linalg.aslinearoperator(sp.csr_matrix(A))
+        if self.inner_modification == 'linear_operator':
+            return sp.linalg.LinearOperator(
+                matvec=mv, rmatvec=rmv, dtype=A.dtype, shape=A.shape)
+        if self.inner_modification == 'class_matvec':
+            return HasMatvec()
+        if self.inner_modification == 'class_matmat':
+            return HasMatmat()
+        assert False
+
+    def _generate_linear_operator(self, xp, sp):
+        A = testing.shaped_random((self.M, self.N), xp, self.dtype)
+
+        if self.outer_modification == 'normal':
+            return self._inner_cases(xp, sp, A)
+        if self.outer_modification == 'transpose':
+            # From SciPy 1.4 (scipy/scipy#9064)
+            return self._inner_cases(xp, sp, A.T).T
+        if self.outer_modification == 'hermitian':
+            return self._inner_cases(xp, sp, A.T.conj()).H
+        assert False
+
+    @skip_HIP_spMM_error(outer=('transpose', 'hermitian'))
+    @testing.numpy_cupy_allclose(sp_name='sp', rtol=1e-6)
+    def test_matvec(self, xp, sp):
+        linop = self._generate_linear_operator(xp, sp)
+        x_1dim = testing.shaped_random((self.N,), xp, self.dtype)
+        x_2dim = testing.shaped_random((self.N, 1), xp, self.dtype)
+        return linop.matvec(x_1dim), linop.matvec(x_2dim)
+
+    @skip_HIP_spMM_error(outer=('transpose', 'hermitian'))
+    @testing.numpy_cupy_allclose(
+        sp_name='sp', rtol=1e-6, contiguous_check=False)
+    def test_matmat(self, xp, sp):
+        linop = self._generate_linear_operator(xp, sp)
+        x = testing.shaped_random((self.N, 8), xp, self.dtype)
+        return linop.matmat(x)
+
+    @skip_HIP_spMM_error(outer=('normal',))
+    @testing.numpy_cupy_allclose(sp_name='sp', rtol=1e-6)
+    def test_rmatvec(self, xp, sp):
+        linop = self._generate_linear_operator(xp, sp)
+        x_1dim = testing.shaped_random((self.M,), xp, self.dtype)
+        x_2dim = testing.shaped_random((self.M, 1), xp, self.dtype)
+        return linop.rmatvec(x_1dim), linop.rmatvec(x_2dim)
+
+    @skip_HIP_spMM_error(outer=('normal',))
+    @testing.numpy_cupy_allclose(
+        sp_name='sp', rtol=1e-6, contiguous_check=False)
+    def test_rmatmat(self, xp, sp):
+        linop = self._generate_linear_operator(xp, sp)
+        x = testing.shaped_random((self.M, 8), xp, self.dtype)
+        return linop.rmatmat(x)
+
+    @skip_HIP_spMM_error(outer=('transpose', 'hermitian'))
+    @testing.numpy_cupy_allclose(
+        sp_name='sp', rtol=1e-6, contiguous_check=False)
+    def test_dot(self, xp, sp):
+        linop = self._generate_linear_operator(xp, sp)
+        x0 = testing.shaped_random((self.N,), xp, self.dtype)
+        x1 = testing.shaped_random((self.N, 1), xp, self.dtype)
+        x2 = testing.shaped_random((self.N, 8), xp, self.dtype)
+        return linop.dot(x0), linop.dot(x1), linop.dot(x2)
+
+    @skip_HIP_spMM_error(outer=('transpose', 'hermitian'))
+    @testing.numpy_cupy_allclose(
+        sp_name='sp', rtol=1e-6, contiguous_check=False)
+    def test_mul(self, xp, sp):
+        linop = self._generate_linear_operator(xp, sp)
+        x0 = testing.shaped_random((self.N,), xp, self.dtype)
+        x1 = testing.shaped_random((self.N, 1), xp, self.dtype)
+        x2 = testing.shaped_random((self.N, 8), xp, self.dtype)
+        return linop * x0, linop * x1, linop * x2
+
+
+@testing.parameterize(*testing.product({
+    'lower': [True, False],
+    'unit_diagonal': [True, False],
+    'nrhs': [None, 1, 4],
+    'order': ['C', 'F']
+}))
+@testing.with_requires('scipy>=1.4.0')
+@pytest.mark.skipif(not cusparse.check_availability('csrsm2'),
+                    reason='no working implementation')
+class TestSpsolveTriangular:
+
+    n = 10
+    density = 0.5
+
+    def _make_matrix(self, dtype, xp):
+        a_shape = (self.n, self.n)
+        a = testing.shaped_random(a_shape, xp, dtype=dtype, scale=1)
+        mask = testing.shaped_random(a_shape, xp, dtype='f', scale=1)
+        a[mask > self.density] = 0
+        diag = xp.diag(xp.ones((self.n,), dtype=dtype))
+        a = a + diag
+        if self.lower:
+            a = xp.tril(a)
+        else:
+            a = xp.triu(a)
+        b_shape = (self.n,) if self.nrhs is None else (self.n, self.nrhs)
+        b = testing.shaped_random(b_shape, xp, dtype=dtype, order=self.order)
+        return a, b
+
+    def _test_spsolve_triangular(self, sp, a, b):
+        return sp.linalg.spsolve_triangular(a, b, lower=self.lower,
+                                            unit_diagonal=self.unit_diagonal)
+
+    @pytest.mark.parametrize('format', ['csr', 'csc', 'coo'])
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(
+        rtol=1e-5, atol=1e-5, sp_name='sp', contiguous_check=False)
+    def test_sparse(self, format, dtype, xp, sp):
+        a, b = self._make_matrix(dtype, xp)
+        a = sp.coo_matrix(a).asformat(format)
+        return self._test_spsolve_triangular(sp, a, b)
+
+    def test_invalid_cases(self):
+        dtype = 'float64'
+        if not (self.lower and self.unit_diagonal and self.nrhs == 4 and
+                self.order == 'C'):
+            pytest.skip()
+
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            a, b = self._make_matrix(dtype, xp)
+            a = sp.csr_matrix(a)
+
+            # a is not a square matrix
+            ng_a = sp.csr_matrix(xp.ones((self.n + 1, self.n), dtype=dtype))
+            with pytest.raises(ValueError):
+                self._test_spsolve_triangular(sp, ng_a, b)
+            # b is not a 1D/2D matrix
+            ng_b = xp.ones((1, self.n, self.nrhs), dtype=dtype)
+            with pytest.raises(ValueError):
+                self._test_spsolve_triangular(sp, a, ng_b)
+            # mismatched shape
+            ng_b = xp.ones((self.n + 1, self.nrhs), dtype=dtype)
+            with pytest.raises(ValueError):
+                self._test_spsolve_triangular(sp, a, ng_b)
+
+        xp, sp = cupy, sparse
+        a, b = self._make_matrix(dtype, xp)
+        a = sp.csr_matrix(a)
+
+        # unsupported dtype
+        ng_a = sp.csr_matrix(xp.ones((self.n, self.n), dtype='bool'))
+        with pytest.raises(TypeError):
+            self._test_spsolve_triangular(sp, ng_a, b)
+        # a is not spmatrix
+        ng_a = xp.ones((self.n, self.n), dtype=dtype)
+        with pytest.raises(TypeError):
+            self._test_spsolve_triangular(sp, ng_a, b)
+        # b is not cupy ndarray
+        ng_b = numpy.ones((self.n, self.nrhs), dtype=dtype)
+        with pytest.raises(TypeError):
+            self._test_spsolve_triangular(sp, a, ng_b)
+
+
+@testing.parameterize(*testing.product({
+    'tol': [0, 1e-5],
+    'reorder': [0, 1, 2, 3],
+}))
+@testing.with_requires('scipy')
+@pytest.mark.skipif(runtime.is_hip, reason='csrlsvqr not available')
+class TestCsrlsvqr:
+
+    n = 8
+    density = 0.75
+    _test_tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _setup(self, dtype):
+        dtype = numpy.dtype(dtype)
+        a_shape = (self.n, self.n)
+        a = testing.shaped_random(
+            a_shape, numpy, dtype=dtype, scale=2 / self.n)
+        a_mask = testing.shaped_random(a_shape, numpy, dtype='f', scale=1)
+        a[a_mask > self.density] = 0
+        a_diag = numpy.diag(numpy.ones((self.n,), dtype=dtype))
+        a = a + a_diag
+        b = testing.shaped_random((self.n,), numpy, dtype=dtype)
+        test_tol = self._test_tol[dtype.char.lower()]
+        return a, b, test_tol
+
+    @testing.for_dtypes('fdFD')
+    def test_csrlsvqr(self, dtype):
+        a, b, test_tol = self._setup(dtype)
+        ref_x = numpy.linalg.solve(a, b)
+        cp_a = cupy.array(a)
+        sp_a = cupyx.scipy.sparse.csr_matrix(cp_a)
+        cp_b = cupy.array(b)
+        x = cusolver.csrlsvqr(sp_a, cp_b, tol=self.tol,
+                              reorder=self.reorder)
+        cupy.testing.assert_allclose(x, ref_x, rtol=test_tol,
+                                     atol=test_tol)
+
+
+@pytest.mark.skipif(runtime.is_hip, reason='csrlsvqr not available')
+@testing.with_requires('scipy')
+class TestSpSolve:
+    def _check_spsolve(self, xp, sp, dtyp):
+        n, nb = 5, 3
+
+        a = xp.diag(xp.arange(n) + 1)
+        sa = sp.csr_matrix(a.astype(dtyp))
+
+        # prepare b to be non-contiguous
+        b = xp.arange((2*n*nb), dtype=dtyp).reshape((2*n, nb))
+        b = b[::2, :]
+        result = sp.linalg.spsolve(sa, b)
+        return result
+
+    @pytest.mark.parametrize('dtyp', ['float32', 'complex64'])
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=5e-7)
+    def test_spsolve_single(self, xp, sp, dtyp):
+        return self._check_spsolve(xp, sp, dtyp)
+
+    @pytest.mark.parametrize('dtyp', ['float64', 'complex128'])
+    @testing.numpy_cupy_allclose(sp_name='sp', atol=1e-14)
+    def test_spsolve_double(self, xp, sp, dtyp):
+        return self._check_spsolve(xp, sp, dtyp)
+
+
+def _eigen_vec_transform(block_vec, xp):
+    """Helper to swap sign of each eigen vector based on the first
+    non-zero element. ie, to standardize the first non-zero element
+    of eigen vector as positive. This helps in comparing equivalence
+    of eigen vectors"""
+    direction = testing.shaped_random((block_vec.shape[0], 1),
+                                      xp=xp, seed=123)
+    direction = xp.where(block_vec.T.dot(direction) >= 0, 1, -1).T
+    # shape of mask: (block_vec.shape[0], 1)
+    # each eigenvector is multiplied by a 1 or -1 (scalar)
+    # this is done by broadcasting mask
+    return block_vec * direction
+
+
+@testing.with_requires('scipy>=1.4')
+@pytest.mark.skipif(runtime.is_hip and driver.get_build_version() < 402,
+                    reason='syevj not available')
+# tests adapted from scipy's tests of lobpcg
+class TestLOBPCG:
+
+    def _generate_input_for_elastic_rod(self, n, xp):
+        """Build the matrices for the generalized eigenvalue problem of the
+        fixed-free elastic rod vibration model.
+        """
+        L = 1.0
+        le = L / n
+        rho = 7.85e3
+        S = 1.e-4
+        E = 2.1e11
+        mass = rho * S * le / 6.
+        k = E * S / le
+        A = k * (xp.diag(xp.r_[2. * xp.ones(n - 1), 1]) -
+                 xp.diag(xp.ones(n - 1), 1) - xp.diag(xp.ones(n - 1), -1))
+        B = mass * (xp.diag(xp.r_[4. * xp.ones(n - 1), 2]) +
+                    xp.diag(xp.ones(n - 1), 1) + xp.diag(xp.ones(n - 1), -1))
+        return A, B
+
+    def _generate_input_for_mikota_pair(self, n, xp):
+        """Build a pair of full diagonal matrices for the generalized eigenvalue
+        problem. The Mikota pair acts as a nice test since the eigenvalues are
+        the squares of the integers n, n=1,2,...
+        """  # NOQA
+        x = xp.arange(1, n + 1)
+        B = xp.diag(1. / x)
+        y = xp.arange(n - 1, 0, -1)
+        z = xp.arange(2 * n - 1, 0, -2)
+        A = xp.diag(z) - xp.diag(y, -1) - xp.diag(y, 1)
+        return A, B
+
+    def _generate_random_initial_ortho_eigvec(self, m, n, xp=numpy, seed=0):
+        """helper to generate orthogonal, random initial approximation for
+        eigen vectors.
+        """
+        V = testing.shaped_random((m, n), xp=numpy, seed=seed)
+        # TODO : use cupy's native linalg.orth() once implemented
+        X = scipy.linalg.orth(V)
+        return xp.asarray(X)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_small_generate_input_for_elastic_rod(self, xp, sp):
+        A, B = self._generate_input_for_elastic_rod(10, xp)
+        n = A.shape[0]
+        X = self._generate_random_initial_ortho_eigvec(n, 10, xp)
+        eigvals, eigvecs = sp.linalg.lobpcg(A,
+                                            X, B=B,
+                                            tol=1e-5, maxiter=30,
+                                            largest=False)
+        return eigvals, _eigen_vec_transform(eigvecs, xp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_small_generate_input_for_mikota_pair(self, xp, sp):
+        A, B = self._generate_input_for_mikota_pair(10, xp)
+        n = A.shape[0]
+        X = self._generate_random_initial_ortho_eigvec(n, 10, xp)
+        eigvals, eigvecs = sp.linalg.lobpcg(A,
+                                            X, B=B,
+                                            tol=1e-5, maxiter=30,
+                                            largest=False)
+        return eigvals, _eigen_vec_transform(eigvecs, xp)
+
+    @testing.with_requires('scipy<1.11')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_generate_input_for_elastic_rod(self, xp, sp):
+        A, B = self._generate_input_for_elastic_rod(100, xp)
+        n = A.shape[0]
+        X = self._generate_random_initial_ortho_eigvec(n, 20, xp)
+        eigvals, eigvecs = sp.linalg.lobpcg(A,
+                                            X, B=B,
+                                            tol=1e-5, maxiter=30,
+                                            largest=False)
+        return eigvals, _eigen_vec_transform(eigvecs, xp)
+
+    @testing.with_requires('scipy<1.11')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_generate_input_for_mikota_pair(self, xp, sp):
+        A, B = self._generate_input_for_mikota_pair(100, xp)
+        n = A.shape[0]
+        X = self._generate_random_initial_ortho_eigvec(n, 20, xp)
+        eigvals, eigvecs = sp.linalg.lobpcg(A,
+                                            X, B=B,
+                                            tol=1e-5, maxiter=30,
+                                            largest=False)
+        return eigvals, _eigen_vec_transform(eigvecs, xp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_regression(self, xp, sp):
+        """Check the eigenvalue of the identity matrix is one.
+        """
+        n = 10
+        X = xp.ones((n, 1))
+        A = xp.identity(n)
+        w, v = sp.linalg.lobpcg(A, X)
+        return w, _eigen_vec_transform(v, xp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_diagonal(self, xp, sp):
+        """Check for diagonal matrices.
+        """
+        # The system of interest is of size n x n.
+        n = 100
+        # We care about only m eigenpairs.
+        m = 4
+        # Define the generalized eigenvalue problem Av = cBv
+        # where (c, v) is a generalized eigenpair,
+        # We choose A to be the diagonal matrix whose entries are 1..n
+        # and where B is chosen to be the identity matrix.
+        vals = xp.arange(1, n + 1, dtype=float)
+        A = sp.diags([vals], [0], (n, n))
+        B = sp.eye(n)
+        # Let the preconditioner M be the inverse of A.
+        M = sp.diags([1. / vals], [0], (n, n))
+        # Pick random initial vectors.
+        X = testing.shaped_random((n, m), xp=xp, seed=1234)
+        # Require that the returned eigenvectors be in the orthogonal
+        # complement of the first few standard basis vectors (Y)
+        m_excluded = 3
+        Y = xp.eye(n, m_excluded)
+        eigvals, vecs = sp.linalg.lobpcg(A, X, B, M=M, Y=Y, tol=1e-4,
+                                         maxiter=40, largest=False)
+        return eigvals, _eigen_vec_transform(vecs, xp)
+
+    def _generate_A_for_fiedler(self, n, p, xp):
+        """Check for fiedler vector computation"""
+        # fiedler vector computation based on scipy's tests
+        # https://github.com/scipy/scipy/blob/ab1c0907fe9255582397db04592d6066745018d3/scipy/sparse/linalg/eigen/lobpcg/tests/test_lobpcg.py#L140
+        col = numpy.zeros(n)
+        col[1] = 1
+        A = scipy.linalg.toeplitz(col)
+        D = numpy.diag(A.sum(axis=1))
+        return xp.asarray(D - A)
+
+    def _generate_small_X_for_fiedler(self, n, p, xp):
+        tmp = xp.pi * xp.arange(n) / n
+        analytic_V = xp.cos(xp.outer(xp.arange(n) + 1 / 2, tmp))
+        return analytic_V[:, :p]
+
+    def _generate_large_X_for_fiedler(self, n, p, xp):
+        tmp = xp.pi * xp.arange(n) / n
+        analytic_V = xp.cos(xp.outer(xp.arange(n) + 1 / 2, tmp))
+        return analytic_V[:, -p:]
+
+    def _generate_approximate_X_for_fiedler(self, n, p, xp):
+        fiedler_guess = xp.concatenate((xp.ones(n // 2),
+                                        -xp.ones(n - n // 2)))
+        return xp.vstack((xp.ones(n), fiedler_guess)).T
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_fiedler_small_8(self, xp, sp):
+        """Check the dense workaround path for small matrices
+           for small fiedler eigen values and vectors
+        """
+        # This triggers the dense path because 8 < 2*5.
+        A = self._generate_A_for_fiedler(8, 2, xp)
+        X = self._generate_small_X_for_fiedler(8, 2, xp)
+        lobpcg_w, lobpcg_V = sp.linalg.lobpcg(A, X, largest=False)
+        return lobpcg_w, _eigen_vec_transform(lobpcg_V, xp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_fiedler_large_8(self, xp, sp):
+        """Check the dense workaround path for small matrices
+           for large fiedler eigen values and vectors
+        """
+        # This triggers the dense path because 8 < 2*5.
+        A = self._generate_A_for_fiedler(8, 2, xp)
+        X = self._generate_large_X_for_fiedler(8, 2, xp)
+        lobpcg_w, lobpcg_V = sp.linalg.lobpcg(A, X, largest=False)
+        return lobpcg_w, _eigen_vec_transform(lobpcg_V, xp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_fiedler_approximate_8(self, xp, sp):
+        """Check the dense workaround path for small matrices
+           for approximately-formed fiedler eigen values and vectors
+        """
+        # This triggers the dense path because 8 < 2*5.
+        A = self._generate_A_for_fiedler(8, 2, xp)
+        X = self._generate_approximate_X_for_fiedler(8, 2, xp)
+        lobpcg_w, lobpcg_V = sp.linalg.lobpcg(A, X, largest=False)
+        return lobpcg_w, _eigen_vec_transform(lobpcg_V, xp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_fiedler_small_12(self, xp, sp):
+        """Check the dense workaround path is avoided for non-small
+           fiedler matrices and small eigen values and vectors
+        """
+        A = self._generate_A_for_fiedler(12, 2, xp)
+        X = self._generate_small_X_for_fiedler(12, 2, xp)
+        lobpcg_w, lobpcg_V = sp.linalg.lobpcg(A, X, largest=False)
+        return lobpcg_w, _eigen_vec_transform(lobpcg_V, xp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_fiedler_large_12(self, xp, sp):
+        """Check the dense workaround path is avoided for non-small
+           fiedler matrices and large eigen values and vectors
+        """
+        A = self._generate_A_for_fiedler(12, 2, xp)
+        X = self._generate_large_X_for_fiedler(12, 2, xp)
+        lobpcg_w, lobpcg_V = sp.linalg.lobpcg(A, X, largest=False)
+        return lobpcg_w, _eigen_vec_transform(lobpcg_V, xp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    def test_fiedler_approximate_12(self, xp, sp):
+        """Check the dense workaround path is avoided for non-small,
+           approximately generated fiedler matrices
+        """
+        A = self._generate_A_for_fiedler(12, 2, xp)
+        X = self._generate_approximate_X_for_fiedler(12, 2, xp)
+        lobpcg_w, lobpcg_V = sp.linalg.lobpcg(A, X, largest=False)
+        return lobpcg_w, _eigen_vec_transform(lobpcg_V, xp)
+
+    def _verbosity_helper(self, xp, sp):
+        """Helper to capture the verbose output from stdout
+        """
+        A, B = self._generate_input_for_elastic_rod(100, xp)
+        n = A.shape[0]
+        m = 20
+        X = self._generate_random_initial_ortho_eigvec(n, m, xp)
+        saved_stdout = io.StringIO()
+        with contextlib.redirect_stdout(saved_stdout):
+            _, _ = sp.linalg.lobpcg(A, X, B=B, tol=1e-5,
+                                    maxiter=30, largest=False,
+                                    verbosityLevel=9)
+        output = saved_stdout.getvalue().strip()
+        return output
+
+    @testing.with_requires('scipy<1.10,>=1.8')
+    def test_verbosity(self):
+        """Check that nonzero verbosity level code runs
+           and is identical to scipy's output format.
+        """
+        stdout_cupy = self._verbosity_helper(cupy, cupyx.scipy.sparse)
+        stdout_numpy = self._verbosity_helper(numpy, scipy.sparse)
+        # getting rid of the numbers and whitespaces, we care only about
+        # format of printed output.
+        # also, due to the fact that there are unpredictable (but minor)
+        # differences in decimal digits between scipy and cupy verbose output
+        stdout_cupy = re.sub(r'[-+]?\d+\.?\d*[ ]*', '{number}', stdout_cupy)
+        stdout_numpy = re.sub(r'[-+]?\d+\.?\d*[ ]*', '{number}', stdout_numpy)
+        assert stdout_numpy == stdout_cupy, '''numpy: %s
+                                               cupy: %s''' % (stdout_numpy,
+                                                              stdout_cupy)
+
+    @testing.numpy_cupy_allclose(
+        rtol=1e-5, atol=5e-3 if runtime.is_hip else 1e-3, sp_name='sp',
+        contiguous_check=False)
+    def test_random_initial_float32(self, xp, sp):
+        """Check lobpcg in float32 for specific initial.
+        """
+        n = 50
+        m = 4
+        vals = -xp.arange(1, n + 1)
+        A = sp.diags([vals], [0], (n, n))
+        A = A.astype(xp.float32)
+        X = testing.shaped_random((n, m), xp=xp, seed=3)
+        eigvals, vecs = sp.linalg.lobpcg(A, X, tol=1e-3, maxiter=50,
+                                         verbosityLevel=1)
+        return eigvals, _eigen_vec_transform(vecs, xp)
+
+    @pytest.mark.xfail(
+        runtime.is_hip and driver.get_build_version() >= 5_00_00000,
+        reason='ROCm 5.0+ may have a bug')
+    @pytest.mark.xfail(
+        cupy.cuda.cusolver._getVersion() in (
+            (11, 4, 5),  # CUDA 12.1.1
+            (11, 5, 0),  # CUDA 12.2.0
+            (11, 5, 1),  # CUDA 12.2.1
+            (11, 5, 2),  # CUDA 12.2.2
+        ),
+        reason='cuSOLVER in CUDA 12.1+ may have a bug',
+        strict=False,  # Seems only failing with Volta (V100 / T4)
+    )
+    def test_maxit_None(self):
+        """Check lobpcg if maxit=None runs 20 iterations (the default)
+        by checking the size of the iteration history output, which should
+        be the number of iterations plus 2 (initial and final values).
+        """
+        def make(xp, sp):
+            vals = -xp.arange(1, n + 1)
+            A = sp.diags([vals], [0], (n, n))
+            A = A.astype(xp.float32)
+            X = testing.shaped_random((n, m), xp=xp, seed=1566950023)
+            return A, X
+        n = 50
+        m = 4
+        A, X = make(cupy, sparse)
+        w, _, l_h = sparse.linalg.lobpcg(
+            A, X, tol=1e-8, maxiter=None, retLambdaHistory=True)
+
+        # Assert the eigenavlues against SciPy
+        A_np, X_np = make(numpy, scipy.sparse)
+        w_np, _ = scipy.sparse.linalg.lobpcg(
+            A_np, X_np, tol=1e-8, maxiter=None)
+        testing.assert_allclose(w, w_np, rtol=1e-5)
+
+        # Assert the number of iterations
+        assert len(l_h) == 22
+
+
+@testing.with_requires('scipy>=1.4')
+@testing.parameterize(*testing.product({
+    'A_sparsity': [True, False],
+    'B_sparsity': [True, False],
+    'A_dtype': [cupy.float32, cupy.float64],
+    'preconditioner_sparsity': [True, False],
+    'preconditioner_dtype': [None, cupy.float32, cupy.float64],
+    'X_dtype': [cupy.float32, cupy.float64],
+    'Y_dtype': [cupy.float32, cupy.float64],
+    'sparse_format': ['coo', 'csr', 'csc']
+}))
+# test class for testing against diagonal matrices overall various data types
+class TestLOBPCGForDiagInput:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if runtime.is_hip:
+            if driver.get_build_version() < 402:
+                pytest.skip('syevj not available')
+            if (((self.A_sparsity is True) or (self.B_sparsity is True)
+                    or (self.preconditioner_sparsity is True))
+                    and self.sparse_format == 'csc'):
+                pytest.xfail('spMM not working')
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp',
+                                 contiguous_check=False)
+    @pytest.mark.slow
+    def test_diagonal_data_types(self, xp, sp):
+        """Check lobpcg for diagonal matrices for all matrix types.
+        """
+        n = 40
+        m = 4
+        # Define the generalized eigenvalue problem Av = cBv
+        # where (c, v) is a generalized eigenpair,
+        # and where we choose A  and B to be diagonal.
+        vals = xp.arange(1, n + 1)
+        # A and B matrices based on parametrization
+        A = sp.diags([vals * vals], [0], (n, n), format=self.sparse_format)
+        A = A.astype(xp.dtype(self.A_dtype))
+        A = A if self.A_sparsity is True else A.toarray()
+
+        B = sp.diags([vals], [0], (n, n), format=self.sparse_format)
+        B = B if self.B_sparsity is True else B.toarray()
+
+        M_LO = None
+        if self.preconditioner_dtype is not None:
+            M = sp.diags([1. / vals], [0], (n, n), format=self.sparse_format)
+            M = M if self.preconditioner_sparsity else M.toarray()
+
+            def fun(x):
+                return M @ x
+            # Define Preconditioner function as Linear Operator
+            M_LO = sp.linalg.LinearOperator(matvec=fun,
+                                            matmat=fun,
+                                            shape=(n, n),
+                                            dtype=xp.dtype(self.preconditioner_dtype))  # NOQA
+
+        # Cannot be sparse array
+        X = testing.shaped_random((n, m), xp=xp, dtype=xp.dtype(self.X_dtype),
+                                  seed=1234)
+
+        # Require tht returned eigenvectors be in the orthogonal
+        # complement of the first few standard basis vectors
+        # (Cannot be sparse array)
+        m_excluded = 3
+        Y = xp.eye(n, m_excluded, dtype=xp.dtype(self.Y_dtype))
+        # core call to lobpcg solver
+        eigvals, eigvecs = sp.linalg.lobpcg(A, X, B=B, M=M_LO, Y=Y,
+                                            tol=1e-4, maxiter=100,
+                                            largest=False)
+        return eigvals, _eigen_vec_transform(eigvecs, xp)
+
+
+@testing.parameterize(*testing.product({
+    'format': ['csr', 'csc', 'coo'],
+    'nrhs': [None, 1, 4],
+    'order': ['C', 'F']
+}))
+@testing.with_requires('scipy')
+@pytest.mark.skipif(not cusparse.check_availability('csrsm2'),
+                    reason='no working implementation')
+class TestSplu:
+
+    n = 10
+    density = 0.5
+
+    def _make_matrix(self, dtype, xp, sp, density=None):
+        if density is None:
+            density = self.density
+        a_shape = (self.n, self.n)
+        a = testing.shaped_random(a_shape, xp, dtype=dtype, scale=2 / self.n)
+        mask = testing.shaped_random(a_shape, xp, dtype='f', scale=1)
+        a[mask > density] = 0
+        diag = xp.diag(xp.ones((self.n,), dtype=dtype))
+        a = a + diag
+        if self.format == 'csr':
+            a = sp.csr_matrix(a)
+        elif self.format == 'csc':
+            a = sp.csc_matrix(a)
+        elif self.format == 'coo':
+            a = sp.coo_matrix(a)
+        b_shape = (self.n,) if self.nrhs is None else (self.n, self.nrhs)
+        b = testing.shaped_random(b_shape, xp, dtype=dtype, order=self.order)
+        return a, b
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_splu(self, dtype, xp, sp):
+        a, b = self._make_matrix(dtype, xp, sp)
+        return sp.linalg.splu(a).solve(b)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_factorized(self, dtype, xp, sp):
+        a, b = self._make_matrix(dtype, xp, sp)
+        return sp.linalg.factorized(a)(b)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_spilu(self, dtype, xp, sp):
+        a, b = self._make_matrix(dtype, xp, sp)
+        return sp.linalg.spilu(a).solve(b)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_spilu_0(self, dtype, xp, sp):
+        # Note: We don't know how to compute ILU(0) with
+        # scipy.sprase.linalg.spilu, so in this test we use a matrix where the
+        # format is a sparse matrix but is actually a dense matrix.
+        a, b = self._make_matrix(dtype, xp, sp, density=1.0)
+        if xp == cupy:
+            # Set fill_factor=1 to computes ILU(0) using cuSparse
+            ainv = sp.linalg.spilu(a, fill_factor=1)
+        else:
+            ainv = sp.linalg.spilu(a)
+        return ainv.solve(b)
+
+
+@testing.parameterize(*testing.product({
+    'damp': [0.0, 1.0, 2.0],
+    'format': ['coo', 'csr', 'csc'],
+    'm': [30, 40, 50],
+    'n': [20, 30],
+    'x0': [None, 'ones'],
+    'use_linear_operator': [False, True],
+}))
+@testing.with_requires('scipy')
+class TestLsmr:
+
+    density = 0.01
+
+    def _make_matrix(self, xp):
+        shape = (self.m, self.n)
+        a = testing.shaped_random(shape, xp, scale=1)
+        mask = testing.shaped_random(shape, xp, scale=1)
+        a[mask > self.density] = 0
+        return a
+
+    def _make_normalized_vector(self, xp):
+        b = testing.shaped_random((self.m,), xp, scale=1)
+        return b / xp.linalg.norm(b)
+
+    def _test_lsmr(self, xp, sp, a):
+        b = self._make_normalized_vector(xp)
+        x0 = None
+        if self.x0 == 'ones':
+            x0 = xp.ones((self.n,))
+        return sp.linalg.lsmr(a, b, x0=x0, damp=self.damp)
+
+    @testing.numpy_cupy_allclose(rtol=1e-1, atol=1e-1, sp_name='sp')
+    def test_sparse(self, xp, sp):
+        if runtime.is_hip and self.format in ('csr', 'csc'):
+            pytest.xfail('may be buggy')  # trans=True
+
+        if (self.damp == 0 and self.x0 == 'ones' and self.n != 20):
+            pytest.skip()
+        a = self._make_matrix(xp)
+        a = sp.coo_matrix(a).asformat(self.format)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        return self._test_lsmr(xp, sp, a)[0]
+
+    @testing.numpy_cupy_allclose(rtol=1e-1, atol=1e-1, sp_name='sp')
+    def test_dense(self, xp, sp):
+        if (self.damp == 0 and self.x0 == 'ones' and self.n != 20):
+            pytest.skip()
+        a = self._make_matrix(xp)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        return self._test_lsmr(xp, sp, a)[0]
+
+    def test_invalid(self):
+        if not (self.x0 is None and self.use_linear_operator is False):
+            pytest.skip()
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            a = self._make_matrix(xp)
+            b = self._make_normalized_vector(xp)
+            ng_a = xp.ones((self.m, ))
+            with pytest.raises(ValueError):
+                sp.linalg.lsmr(ng_a, b)
+            ng_a = xp.ones((self.m, self.n, 1))
+            with pytest.raises(ValueError):
+                sp.linalg.lsmr(ng_a, b)
+            ng_b = xp.ones((self.m + 1,))
+            with pytest.raises(ValueError):
+                sp.linalg.lsmr(a, ng_b)
+            ng_b = xp.ones((self.m, 2))
+            with pytest.raises(ValueError):
+                sp.linalg.lsmr(a, ng_b)
+            ng_x0 = xp.ones((self.n + 1,))
+            with pytest.raises(ValueError):
+                sp.linalg.lsmr(a, b, x0=ng_x0)
+
+
+@testing.parameterize(*testing.product({
+    'x0': [None, 'ones'],
+    'M': [None, 'jacobi'],
+    'atol': [None, 'select-by-dtype'],
+    'b_ndim': [1, 2],
+    'use_linear_operator': [False, True],
+}))
+@testing.with_requires('scipy')
+class TestCgs:
+    n = 30
+    density = 0.33
+    _atol = {'f': 1e-5, 'd': 1e-12}
+
+    def _make_matrix(self, dtype, xp):
+        dtype = numpy.dtype(dtype)
+        shape = (self.n, 10)
+        a = testing.shaped_random(shape, xp, dtype=dtype.char.lower(), scale=1)
+        if dtype.char in 'FD':
+            a = a + 1j * testing.shaped_random(
+                shape, xp, dtype=dtype.char.lower(), scale=1)
+        mask = testing.shaped_random(shape, xp, dtype='f', scale=1)
+        a[mask > self.density] = 0
+        a = a @ a.conj().T
+        a = a + xp.diag(xp.ones((self.n,), dtype=dtype.char.lower()))
+        M = None
+        if self.M == 'jacobi':
+            M = xp.diag(1.0 / xp.diag(a))
+        return a, M
+
+    def _make_normalized_vector(self, dtype, xp):
+        b = testing.shaped_random((self.n,), xp, dtype=dtype)
+        return b / xp.linalg.norm(b)
+
+    def _test_cgs(self, dtype, xp, sp, a, M):
+        dtype = numpy.dtype(dtype)
+        b = self._make_normalized_vector(dtype, xp)
+        if self.b_ndim == 2:
+            b = b.reshape(self.n, 1)
+        x0 = None
+        if self.x0 == 'ones':
+            x0 = xp.ones((self.n,), dtype=dtype)
+        atol = None
+        if self.atol == 'select-by-dtype':
+            atol = self._atol[dtype.char.lower()]
+        if atol is None and xp == numpy:
+            # Note: If atol is None or not specified, Scipy (at least 1.5.3)
+            # raises DeprecationWarning
+            with pytest.deprecated_call():
+                return sp.linalg.cgs(a, b, x0=x0, M=M, atol=atol)
+        else:
+            return sp.linalg.cgs(a, b, x0=x0, M=M, atol=atol)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_dense(self, dtype, xp, sp):
+        a, M = self._make_matrix(dtype, xp)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+            if M is not None:
+                M = sp.linalg.aslinearoperator(M)
+        return self._test_cgs(dtype, xp, sp, a, M)
+
+    @pytest.mark.parametrize('format', ['csr', 'csc', 'coo'])
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_sparse(self, format, dtype, xp, sp):
+        if runtime.is_hip and format == 'csc':
+            pytest.xfail('may be buggy')  # trans=True
+
+        a, M = self._make_matrix(dtype, xp)
+        a = sp.coo_matrix(a).asformat(format)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        if M is not None:
+            M = sp.coo_matrix(M).asformat(format)
+            if self.use_linear_operator:
+                M = sp.linalg.aslinearoperator(M)
+        return self._test_cgs(dtype, xp, sp, a, M)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_empty(self, dtype, xp, sp):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.use_linear_operator is False):
+            pytest.skip()
+        a = xp.empty((0, 0), dtype=dtype)
+        b = xp.empty((0,), dtype=dtype)
+        if self.atol is None and xp == numpy:
+            # Note: If atol is None or not specified, Scipy (at least 1.5.3)
+            # raises DeprecationWarning
+            with pytest.deprecated_call():
+                return sp.linalg.cgs(a, b)
+        else:
+            return sp.linalg.cgs(a, b)
+
+    @testing.for_dtypes('fdFD')
+    def test_callback(self, dtype):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.use_linear_operator is False):
+            pytest.skip()
+        xp, sp = cupy, sparse
+        a, M = self._make_matrix(dtype, xp)
+        b = self._make_normalized_vector(dtype, xp)
+        is_called = False
+
+        def callback(x):
+            print(xp.linalg.norm(b - a @ x))
+            nonlocal is_called
+            is_called = True
+        sp.linalg.cgs(a, b, callback=callback)
+        assert is_called
+
+    def test_invalid(self):
+        if not (self.x0 is None and self.M is None and self.atol is None and
+                self.use_linear_operator is False):
+            pytest.skip()
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            a, M = self._make_matrix('f', xp)
+            b = self._make_normalized_vector('f', xp)
+            ng_a = xp.ones((self.n, ), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cgs(ng_a, b, atol=self.atol)
+            ng_a = xp.ones((self.n, self.n + 1), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cgs(ng_a, b, atol=self.atol)
+            ng_a = xp.ones((self.n, self.n, 1), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cgs(ng_a, b, atol=self.atol)
+            ng_b = xp.ones((self.n + 1,), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cgs(a, ng_b, atol=self.atol)
+            ng_b = xp.ones((self.n, 2), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cgs(a, ng_b, atol=self.atol)
+            ng_x0 = xp.ones((self.n + 1,), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.cgs(a, b, x0=ng_x0, atol=self.atol)
+            ng_M = xp.diag(xp.ones((self.n + 1,), dtype='f'))
+            with pytest.raises(ValueError):
+                sp.linalg.cgs(a, b, M=ng_M, atol=self.atol)
+        xp, sp = cupy, sparse
+        b = self._make_normalized_vector('f', xp)
+        ng_a = xp.ones((self.n, self.n), dtype='i')
+        with pytest.raises(TypeError):
+            sp.linalg.cgs(ng_a, b, atol=self.atol)
+
+
+@testing.parameterize(*testing.product({
+    'format': ['coo', 'csr', 'csc'],
+    'm': [30, 40],
+    'x0': [None, 'ones'],
+    'M': [None, 'jacobi'],
+    'shift': [0, 1],
+    'use_linear_operator': [False, True],
+}))
+@testing.with_requires('scipy')
+class TestMinres:
+
+    density = 0.01
+
+    def _make_matrix(self, xp):
+        shape = (self.m, self.m)
+        a = testing.shaped_random(shape, xp, scale=1)
+        mask = testing.shaped_random(shape, xp, scale=1)
+        a[mask > self.density] = 0
+        M = None
+        if self.M == 'jacobi':
+            M = xp.diag(1.0 / xp.diag(a))
+        return a, M
+
+    def _make_normalized_vector(self, xp):
+        b = testing.shaped_random((self.m,), xp, scale=1)
+        return b / xp.linalg.norm(b)
+
+    def _test_minres(self, xp, sp, a, M):
+        b = self._make_normalized_vector(xp)
+        x0 = None
+        if self.x0 == 'ones':
+            x0 = xp.ones((self.m,))
+        return sp.linalg.minres(a, b, x0=x0, M=M)[0]
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_sparse(self, xp, sp):
+        if runtime.is_hip and self.format == 'csc':
+            pytest.xfail('may be buggy')  # trans=True
+        a, M = self._make_matrix(xp)
+        a = sp.coo_matrix(a).asformat(self.format)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+        if M is not None:
+            M = sp.coo_matrix(M).asformat(self.format)
+            if self.use_linear_operator:
+                M = sp.linalg.aslinearoperator(M)
+        return self._test_minres(xp, sp, a, M)
+
+    @testing.numpy_cupy_allclose(rtol=1e-5, atol=1e-5, sp_name='sp')
+    def test_dense(self, xp, sp):
+        a, M = self._make_matrix(xp)
+        if self.use_linear_operator:
+            a = sp.linalg.aslinearoperator(a)
+            if M is not None:
+                M = sp.linalg.aslinearoperator(M)
+        return self._test_minres(xp, sp, a, M)
+
+    def test_invalid(self):
+        if not (self.x0 is None and self.M is None
+                and self.use_linear_operator is False):
+            pytest.skip()
+        for xp, sp in ((numpy, scipy.sparse), (cupy, sparse)):
+            a, M = self._make_matrix(xp)
+            b = self._make_normalized_vector(xp)
+            ng_a = xp.ones((self.m, ))
+            with pytest.raises(ValueError):
+                sp.linalg.minres(ng_a, b)
+            ng_a = xp.ones((self.m, self.m + 1))
+            with pytest.raises(ValueError):
+                sp.linalg.minres(ng_a, b)
+            ng_a = xp.ones((self.m, self.m, 1), dtype='f')
+            with pytest.raises(ValueError):
+                sp.linalg.minres(ng_a, b)
+            ng_b = xp.ones((self.m + 1,))
+            with pytest.raises(ValueError):
+                sp.linalg.minres(a, ng_b)
+            ng_b = xp.ones((self.m, 2))
+            with pytest.raises(ValueError):
+                sp.linalg.minres(a, ng_b)
+            ng_x0 = xp.ones((self.m + 1,))
+            with pytest.raises(ValueError):
+                sp.linalg.minres(a, b, x0=ng_x0)
+            ng_M = xp.diag(xp.ones((self.m + 1,)))
+            with pytest.raises(ValueError):
+                sp.linalg.minres(a, b, M=ng_M)
+
+    def test_callback(self):
+        if not (self.x0 is None and self.M is None and
+                self.use_linear_operator is False):
+            pytest.skip()
+        xp, sp = cupy, sparse
+        a, M = self._make_matrix(xp)
+        b = self._make_normalized_vector(xp)
+        is_called = False
+
+        def callback(x):
+            nonlocal is_called
+            is_called = True
+        sp.linalg.minres(a, b, callback=callback)
+        assert is_called
diff --git a/tests/cupyx_tests/scipy_tests/spatial_tests/__init__.py b/tests/cupyx_tests/scipy_tests/spatial_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/spatial_tests/test_distance.py b/tests/cupyx_tests/scipy_tests/spatial_tests/test_distance.py
new file mode 100644
index 0000000..941e73e
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/spatial_tests/test_distance.py
@@ -0,0 +1,232 @@
+import unittest
+import pytest
+
+import numpy
+import cupy
+try:
+    import scipy.spatial  # NOQA
+    import scipy.spatial.distance  # NOQA
+    scipy_available = True
+except ModuleNotFoundError:
+    scipy_available = False
+import cupyx.scipy.spatial.distance  # NOQA
+try:
+    import pylibraft  # NOQA
+    pylibraft_available = True
+except ModuleNotFoundError:
+    pylibraft_available = False
+from cupy import testing
+
+
+@testing.with_requires("scipy")
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64'],
+    'rows': [20, 100],
+    'cols': [20, 100],
+    'metric': ['euclidean', 'cityblock', 'canberra', 'chebyshev',
+               'hamming', 'correlation', 'jensenshannon', 'russellrao',
+               "minkowski", "cosine", "sqeuclidean"],
+    'p': [2.0],
+    'order': ["C", "F"]
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason="tests for CUDA only")
+@pytest.mark.skipif(not scipy_available or not pylibraft_available,
+                    reason='requires scipy and pylibraft')
+class TestCdist(unittest.TestCase):
+
+    def _make_matrix(self, xp, dtype, order):
+        shape = (self.rows, self.cols)
+        return testing.shaped_random(shape, xp, dtype=dtype,
+                                     scale=1, order=order)
+
+    @testing.numpy_cupy_array_almost_equal(decimal=4, scipy_name='scp')
+    def test_cdist_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        b = self._make_matrix(xp, self.dtype, self.order)
+
+        # RussellRao expects boolean arrays
+        if self.metric == "russellrao":
+            a[a < 0.5] = 0
+            a[a >= 0.5] = 1
+            b[b < 0.5] = 0
+            b[b >= 0.5] = 1
+
+        # JensenShannon expects probability arrays
+        elif self.metric == "jensenshannon":
+            a_n = a
+            b_n = b
+            if xp == cupy:
+                a_n = a_n.get()
+                b_n = b_n.get()
+
+            # l1 normalization is different between cupy and numpy
+            # so use numpy.
+            norm = numpy.sum(a_n, axis=1)
+            a = (a_n.T / norm).T
+            if xp == cupy:
+                a = cupy.asarray(a)
+
+            norm = numpy.sum(b_n, axis=1)
+            b = (b_n.T / norm).T
+            if xp == cupy:
+                b = cupy.asarray(b)
+
+        if self.metric == 'minkowski':
+            out = scp.spatial.distance.cdist(a, b, metric=self.metric,
+                                             p=self.p).astype(self.dtype)
+        else:
+            out = scp.spatial.distance.cdist(a, b, metric=self.metric)\
+                .astype(self.dtype)
+
+        print(str(out.shape))
+        return out
+
+
+@testing.with_requires("scipy")
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64'],
+    'rows': [20, 100],
+    'cols': [20, 100],
+    'p': [1.0, 2.0, 3.0],
+    'order': ["C", "F"]
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason="tests for CUDA only")
+@pytest.mark.skipif(not scipy_available or not pylibraft_available,
+                    reason='requires scipy and pylibraft')
+class TestPdist:
+
+    def _make_matrix(self, xp, dtype, order):
+        shape = (self.rows, self.cols)
+        return testing.shaped_random(shape, xp, dtype=dtype,
+                                     scale=1, order=order)
+
+    @testing.numpy_cupy_array_almost_equal(decimal=4, scipy_name='scp')
+    def test_pdist_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.pdist(a).astype(self.dtype)
+        return out
+
+
+@testing.with_requires("scipy")
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64'],
+    'rows': [20, 100],
+    'cols': [20, 100],
+    'p': [1.0, 2.0, 3.0],
+    'order': ["C", "F"]
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason="tests for CUDA only")
+@pytest.mark.skipif(not scipy_available or not pylibraft_available,
+                    reason='requires scipy and pylibraft')
+class TestDistanceMatrix(unittest.TestCase):
+
+    def _make_matrix(self, xp, dtype, order):
+        shape = (self.rows, self.cols)
+        return testing.shaped_random(shape, xp, dtype=dtype,
+                                     scale=1, order=order)
+
+    @testing.numpy_cupy_array_almost_equal(decimal=4, scipy_name='scp')
+    def test_distance_matrix_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance_matrix(a, a, p=self.p).astype(self.dtype)
+        return out
+
+
+@testing.with_requires("scipy")
+@testing.parameterize(*testing.product({
+    'dtype': ['float32', 'float64'],
+    'cols': [20, 100],
+    'p': [1.0, 2.0, 3.0],
+    'order': ["C", "F"]
+}))
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason="tests for CUDA only")
+@pytest.mark.skipif(not scipy_available or not pylibraft_available,
+                    reason='requires scipy and pylibraft')
+class TestDistanceFunction(unittest.TestCase):
+
+    def _make_matrix(self, xp, dtype, order):
+        shape = (1, self.cols) if xp is cupy else (self.cols,)
+        return testing.shaped_random(shape, xp, dtype=dtype,
+                                     scale=1, order=order)
+
+    @testing.numpy_cupy_equal(scipy_name='scp')
+    def test_minkowski_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.minkowski(a, a, p=self.p)
+        return out
+
+    @testing.numpy_cupy_equal(scipy_name='scp')
+    def test_canberra_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.canberra(a, a)
+        return out
+
+    @testing.numpy_cupy_equal(scipy_name='scp')
+    def test_chebyshev_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.chebyshev(a, a)
+        return out
+
+    @testing.numpy_cupy_equal(scipy_name='scp')
+    def test_cityblock_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.cityblock(a, a)
+        return out
+
+    @testing.numpy_cupy_array_almost_equal(scipy_name='scp', type_check=False)
+    def test_correlation_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.correlation(a, a)
+        return xp.asarray(out)
+
+    @testing.numpy_cupy_array_almost_equal(scipy_name='scp', type_check=False)
+    def test_cosine_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.cosine(a, a)
+        return xp.asarray(out)
+
+    @testing.numpy_cupy_equal(scipy_name='scp')
+    def test_hamming_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.hamming(a, a)
+        return out
+
+    @testing.numpy_cupy_equal(scipy_name='scp')
+    def test_euclidean_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.euclidean(a, a)
+        return out
+
+    @testing.numpy_cupy_equal(scipy_name='scp')
+    def test_jensenshannon_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.jensenshannon(a, a)
+        return out
+
+    @testing.numpy_cupy_array_almost_equal(scipy_name='scp', type_check=False)
+    def test_russellrao_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        a[a < 0.5] = 0
+        a[a >= 0.5] = 1
+        out = scp.spatial.distance.russellrao(a, a)
+        return xp.asarray(out)
+
+    @testing.numpy_cupy_equal(scipy_name='scp')
+    def test_sqeuclidean_(self, xp, scp):
+
+        a = self._make_matrix(xp, self.dtype, self.order)
+        out = scp.spatial.distance.sqeuclidean(a, a)
+        return out
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/__init__.py b/tests/cupyx_tests/scipy_tests/special_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_basic.py b/tests/cupyx_tests/scipy_tests/special_tests/test_basic.py
new file mode 100644
index 0000000..f4952e3
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_basic.py
@@ -0,0 +1,218 @@
+import math
+
+import cupy
+import numpy
+import pytest
+
+import cupyx.scipy.special
+from cupy import testing
+from cupy.testing import (
+    assert_array_equal,
+    assert_array_almost_equal,
+)
+from cupy.testing import numpy_cupy_allclose
+
+rtol = {'default': 1e-5, cupy.float64: 1e-12, cupy.complex128: 1e-12}
+
+
+@testing.with_requires("scipy")
+class TestLegendreFunctions:
+
+    def test_lpmv_basic(self):
+        # specific values tested in the SciPy test suite
+        scp = cupyx.scipy
+        lp = scp.special.lpmv(0, 2, 0.5)
+        assert_array_almost_equal(lp, -0.125, 7)
+        lp = scp.special.lpmv(0, 40, 0.001)
+        assert_array_almost_equal(lp, 0.1252678976534484, 7)
+
+        # XXX: this is outside the domain of the current implementation,
+        #      so ensure it returns a NaN rather than a wrong answer.
+        olderr = numpy.seterr(all="ignore")
+        try:
+            lp = scp.special.lpmv(-1, -1, 0.001)
+        finally:
+            numpy.seterr(**olderr)
+        assert lp != 0 or cupy.isnan(lp)
+
+    @pytest.mark.parametrize("order", [0, 1, 2, 3, 4])
+    @pytest.mark.parametrize("degree", [0, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50])
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp", atol=1e-12)
+    def test_lpmv(self, xp, scp, dtype, order, degree):
+        vals = xp.linspace(-1, 1, 100, dtype=dtype)
+        return scp.special.lpmv(order, degree, vals)
+
+
+@testing.with_requires("scipy")
+class TestBasic:
+
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_gammasgn(self, xp, scp, dtype):
+        vals = xp.linspace(-4, 4, 100, dtype=dtype)
+        return scp.special.gammasgn(vals)
+
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-5)
+    def test_log1p_linspace(self, xp, scp, dtype):
+        vals = xp.linspace(-100, 100, 1000, dtype=dtype)
+        dtype = xp.dtype(dtype)
+        if dtype.kind == 'c':
+            # broadcast to mix large and small real vs. imaginary
+            vals = vals[::10, xp.newaxis] + 1j * vals[xp.newaxis, ::10]
+        return xp.abs(scp.special.log1p(vals))
+
+    @testing.for_dtypes("efFdD")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-5)
+    def test_log1p_logspace(self, xp, scp, dtype):
+        dtype = xp.dtype(dtype)
+        vals = xp.logspace(-10, 10, 1000, dtype=dtype)
+        if dtype.kind == 'c':
+            # broadcast to mix large and small real vs. imaginary
+            vals = vals[::10, xp.newaxis] + 1j * vals[xp.newaxis, ::10]
+        return xp.abs(scp.special.log1p(vals))
+
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=rtol)
+    def test_log1p_path2(self, xp, scp, dtype):
+        # test values for code path corresponding to range [1/sqrt(2), sqrt(2)]
+        vals = xp.linspace(1 / math.sqrt(2), math.sqrt(2), 1000, dtype=dtype)
+        return scp.special.log1p(vals)
+
+    def test_log1p_real(self):
+        log1p = cupyx.scipy.special.log1p
+        inf = cupy.inf
+        nan = cupy.nan
+        assert_array_equal(log1p(0), 0.0)
+        assert_array_equal(log1p(-1), -inf)
+        assert_array_equal(log1p(-2), nan)
+        assert_array_equal(log1p(inf), inf)
+
+    @pytest.mark.parametrize("function", ["xlogy", "xlog1py"])
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp", rtol={'default': 1e-3,
+                                                 cupy.float64: 1e-12})
+    def test_xlogy(self, xp, scp, dtype, function):
+        # only test with values > 0 to avoid NaNs
+        x = xp.linspace(-100, 100, 1000, dtype=dtype)
+        y = xp.linspace(0.001, 100, 1000, dtype=dtype)
+        if x.dtype.kind == 'c':
+            x -= 1j * x
+            y += 1j * y
+        return getattr(scp.special, function)(x, y)
+
+    @pytest.mark.parametrize("function", ["xlogy", "xlog1py"])
+    @testing.for_dtypes('efdFD')
+    @numpy_cupy_allclose(scipy_name="scp", rtol={'default': 1e-3,
+                                                 cupy.float64: 1e-12})
+    def test_xlogy_zeros(self, xp, scp, dtype, function):
+        # only test with values > 0 to avoid NaNs
+        x = xp.zeros((1, 100), dtype=dtype)
+        y = xp.linspace(-10, 10, 100, dtype=dtype)
+        if y.dtype.kind == 'c':
+            y += 1j * y
+        return getattr(scp.special, function)(x, y)
+
+    @pytest.mark.parametrize("function", ["xlogy", "xlog1py"])
+    @testing.for_all_dtypes()
+    def test_xlogy_nonfinite(self, dtype, function):
+        func = getattr(cupyx.scipy.special, function)
+        y = cupy.ones((5,), dtype=dtype)
+        assert cupy.all(cupy.isnan(func(cupy.nan, y)))
+        assert cupy.all(cupy.isnan(func(y, cupy.nan)))
+
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6)
+    def test_exp2(self, xp, scp, dtype):
+        vals = xp.linspace(-100, 100, 200, dtype=dtype)
+        return scp.special.exp2(vals)
+
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6)
+    def test_exp10(self, xp, scp, dtype):
+        if xp.dtype(dtype).char == 'd':
+            vals = xp.linspace(-100, 100, 100, dtype=dtype)
+        else:
+            # Note: comparisons start to fail outside this range
+            #       np.finfo(np.float32).max is 3.4028235e+38
+            vals = xp.linspace(-37, 37, 100, dtype=dtype)
+        return scp.special.exp10(vals)
+
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6)
+    def test_expm1(self, xp, scp, dtype):
+        vals = xp.linspace(-50, 50, 200, dtype=dtype)
+        if xp.dtype(dtype).kind == 'c':
+            # broadcast to mix small and large real and imaginary parts
+            vals = vals[:, xp.newaxis] + 1j * vals[xp.newaxis, :]
+        return scp.special.expm1(vals)
+
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6)
+    def test_cosm1(self, xp, scp, dtype):
+        vals = xp.linspace(-50, 50, 200, dtype=dtype)
+        return scp.special.cosm1(vals)
+
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6)
+    def test_cosm1_close_to_zero(self, xp, scp, dtype):
+        vals = xp.linspace(-1e-8, 1e-8, 200, dtype=dtype)
+        return scp.special.cosm1(vals)
+
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6)
+    def test_radian(self, xp, scp, dtype):
+        tmp = xp.linspace(-100, 100, 10, dtype=dtype)
+        d = tmp[:, xp.newaxis, xp.newaxis]
+        m = tmp[xp.newaxis, : xp.newaxis]
+        s = tmp[xp.newaxis, xp.newaxis, :]
+        return scp.special.radian(d, m, s)
+
+    @pytest.mark.parametrize('function', ['cosdg', 'sindg', 'tandg', 'cotdg'])
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp",
+                         atol={'default': 1e-6, cupy.float64: 1e-12},
+                         rtol=1e-6)
+    def test_trig_degrees(self, xp, scp, dtype, function):
+        vals = xp.linspace(-100, 100, 200, dtype=dtype)
+        # test at exact multiples of 45 degrees
+        vals = xp.concatenate((vals, xp.arange(-360, 361, 45, dtype=dtype)))
+        return getattr(scp.special, function)(vals)
+
+    @testing.for_dtypes("efd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6)
+    def test_cbrt(self, xp, scp, dtype):
+        vals = xp.linspace(-100, 100, 200, dtype=dtype)
+        return scp.special.cbrt(vals)
+
+    # TODO: omit "e" since SciPy will promote to "f", but CuPy does not
+    @testing.for_dtypes("fd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6)
+    def test_round(self, xp, scp, dtype):
+        vals = xp.concatenate(
+            (
+                xp.linspace(-2, 2, 100, dtype=dtype),
+                # test at half-integer locations
+                xp.asarray([-2.5, -1.5, -0.5, 0.5, 1.5, 2.5], dtype=dtype),
+            )
+        )
+        return scp.special.round(vals)
+
+    # Exclude 'e' here because of deficiency in the NumPy/SciPy
+    # implementation for float16 dtype. This was also noted in
+    # cupy_tests/math_tests/test_special.py
+    @testing.for_dtypes("fd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6, atol=1e-6)
+    def test_sinc(self, xp, scp, dtype):
+        vals = xp.linspace(-100, 100, 200, dtype=dtype)
+        return scp.special.sinc(vals)
+
+    # TODO: Should we make all int dtypes convert to float64 and test for that?
+    #       currently int8->float16, int16->float32, etc... but for
+    #       numpy.sinc/scipy.special.sinc any int type becomes float64.
+    @testing.for_dtypes("fd")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-6, atol=1e-6)
+    def test_sinc_integer_valued(self, xp, scp, dtype):
+        vals = xp.arange(0, 10, dtype=dtype)
+        return scp.special.sinc(vals)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_bessel.py b/tests/cupyx_tests/scipy_tests/special_tests/test_bessel.py
new file mode 100644
index 0000000..67f68c9
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_bessel.py
@@ -0,0 +1,135 @@
+import unittest
+
+import cupy
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+
+
+@testing.with_requires('scipy')
+class TestSpecial:
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(rtol=1e-5, scipy_name='scp')
+    def check_unary(self, name, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(scp.special, name)(a)
+
+    def test_j0(self):
+        self.check_unary('j0')
+
+    def test_j1(self):
+        self.check_unary('j1')
+
+    def test_y0(self):
+        self.check_unary('y0')
+
+    def test_y1(self):
+        self.check_unary('y1')
+
+    def test_i0(self):
+        self.check_unary('i0')
+
+    def test_i0e(self):
+        self.check_unary('i0e')
+
+    def test_i1(self):
+        self.check_unary('i1')
+
+    def test_i1e(self):
+        self.check_unary('i1e')
+
+    def test_k0(self):
+        self.check_unary('k0')
+
+    def test_k0e(self):
+        self.check_unary('k0e')
+
+    def test_k1(self):
+        self.check_unary('k1')
+
+    def test_k1e(self):
+        self.check_unary('k1e')
+
+    @testing.for_dtypes('iId', name='order_dtype')
+    @testing.for_dtypes('efd')
+    @testing.numpy_cupy_allclose(atol=1e-12, scipy_name='scp')
+    def test_yn(self, xp, scp, dtype, order_dtype):
+        import scipy.special  # NOQA
+
+        n = xp.arange(0, 10, dtype=order_dtype)
+        a = xp.linspace(-10, 10, 100, dtype=dtype)
+        return scp.special.yn(n[:, xp.newaxis], a[xp.newaxis, :])
+
+
+@testing.with_requires('scipy')
+class TestFusionSpecial(unittest.TestCase):
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(rtol=1e-5, scipy_name='scp')
+    def check_unary(self, name, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+
+        @cupy.fuse()
+        def f(x):
+            return getattr(scp.special, name)(x)
+
+        return f(a)
+
+    def test_j0(self):
+        self.check_unary('j0')
+
+    def test_j1(self):
+        self.check_unary('j1')
+
+    def test_y0(self):
+        self.check_unary('y0')
+
+    def test_y1(self):
+        self.check_unary('y1')
+
+    def test_i0(self):
+        self.check_unary('i0')
+
+    def test_i0e(self):
+        self.check_unary('i0e')
+
+    def test_i1(self):
+        self.check_unary('i1')
+
+    def test_i1e(self):
+        self.check_unary('i1e')
+
+    def test_k0(self):
+        self.check_unary('k0')
+
+    def test_k0e(self):
+        self.check_unary('k0e')
+
+    def test_k1(self):
+        self.check_unary('k1')
+
+    def test_k1e(self):
+        self.check_unary('k1e')
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(rtol=1e-5, scipy_name='scp')
+    def test_chbevl_dependent_fusion(self, dtype, xp, scp):
+        @cupy.fuse
+        def fused(x):
+            _k0 = scp.special.k0(x)
+            _k0e = scp.special.k0e(x)
+            _k1 = scp.special.k1(x)
+            _k1e = scp.special.k1e(x)
+
+            # rgamma depends on `chbevl` too
+            # see rgamma definition for additional info
+            _rgamma = scp.special.rgamma(x)
+
+            return _k0 + _k0e + _k1 + _k1e + _rgamma
+
+        a = xp.linspace(-10, 10, 50, dtype=dtype).reshape((2, -1))
+        return fused(a)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_beta.py b/tests/cupyx_tests/scipy_tests/special_tests/test_beta.py
new file mode 100644
index 0000000..14cfed1
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_beta.py
@@ -0,0 +1,144 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+
+
+def _get_logspace_max(dtype):
+    # keep logspace within range of the float dtype
+    if dtype.char == 'd':
+        return 200
+    elif dtype.char == 'f':
+        return 30
+    elif dtype.char == 'e':
+        return 5
+
+
+@testing.with_requires('scipy')
+class TestBeta:
+
+    @pytest.mark.parametrize('function', ['beta', 'betaln'])
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_arange(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, function)
+        a = testing.shaped_arange((1, 10), xp, dtype)
+        b = testing.shaped_arange((10, 1), xp, dtype)
+        return func(a, b)
+
+    @pytest.mark.skipif(
+        cupy.cuda.runtime.is_hip and
+        cupy.cuda.runtime.runtimeGetVersion() < 5_00_00000,
+        reason='ROCm/HIP fails in ROCm 4.x')
+    @pytest.mark.parametrize('function', ['beta', 'betaln'])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_linspace(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, function)
+        # TODO: Some choices of start/stop value can give mismatched location
+        #       of +inf or -inf values.
+        x = xp.linspace(-20, 21, 50, dtype=dtype)
+        return func(x[:, xp.newaxis], x[xp.newaxis, :])
+
+    @pytest.mark.parametrize('function', ['beta', 'betaln'])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_logspace(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, function)
+        lmax = _get_logspace_max(xp.dtype(dtype))
+        x = xp.logspace(-lmax, lmax, 32, dtype=dtype)
+        return func(x[:, xp.newaxis], x[xp.newaxis, :])
+
+    @pytest.mark.parametrize('function', ['beta', 'betaln'])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_inf_and_nan(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+        func = getattr(scp.special, function)
+        a = xp.array([-numpy.inf, numpy.nan, numpy.inf, 0], dtype=dtype)
+        return func(a[:, xp.newaxis], a[xp.newaxis, :])
+
+    def test_beta_specific_vals(self):
+        # specific values borrowed from SciPy test suite
+        special = cupyx.scipy.special
+        testing.assert_allclose(special.beta(1, 1), 1.0)
+        testing.assert_allclose(special.beta(-100.3, 1e-200),
+                                special.gamma(1e-200))
+        testing.assert_allclose(special.beta(0.0342, 171),
+                                24.070498359873497, rtol=1e-13, atol=0)
+
+    def test_betaln_specific_vals(self):
+        # specific values borrowed from SciPy test suite
+        special = cupyx.scipy.special
+        testing.assert_allclose(special.betaln(1, 1), 0.0, atol=1e-10)
+        testing.assert_allclose(special.betaln(-100.3, 1e-200),
+                                special.gammaln(1e-200))
+        testing.assert_allclose(special.betaln(0.0342, 170),
+                                3.1811881124242447, rtol=1e-14, atol=0)
+
+
+@testing.with_requires('scipy')
+class TestBetaInc:
+
+    @pytest.mark.parametrize('function', ['betainc', 'betaincinv'])
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_arange(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, function)
+        a = testing.shaped_arange((1, 10, 1), xp, dtype)
+        b = testing.shaped_arange((10, 1, 1), xp, dtype)
+        x = xp.asarray([0, 0.25, 0.5, 0.75, 1], dtype=dtype).reshape(1, 1, 5)
+        # return scp.special.betainc(a, b, x)
+        return func(a, b, x)
+
+    @pytest.mark.parametrize('function', ['betainc', 'betaincinv'])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_linspace(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, function)
+        # TODO: Some choices of start/stop value can give mismatched location
+        #       of +inf or -inf values.
+        tmp = xp.linspace(-20, 21, 10, dtype=dtype)
+        a = tmp[:, xp.newaxis, xp.newaxis]
+        b = tmp[xp.newaxis, :, xp.newaxis]
+        x = xp.linspace(0, 1, 5, dtype=dtype)[xp.newaxis, xp.newaxis, :]
+        return func(a, b, x)
+
+    @pytest.mark.parametrize('function', ['betainc', 'betaincinv'])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_beta_inf_and_nan(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+        func = getattr(scp.special, function)
+        a = xp.array([-numpy.inf, numpy.nan, numpy.inf, 0], dtype=dtype)
+        return func(a[:, xp.newaxis, xp.newaxis],
+                    a[xp.newaxis, :, xp.newaxis],
+                    a[xp.newaxis, xp.newaxis, :])
+
+    def test_betainc_specific_vals(self):
+        # specific values borrowed from SciPy test suite
+        special = cupyx.scipy.special
+
+        assert special.betainc(1, 1, 1) == 1.0
+        testing.assert_allclose(special.betainc(0.0342, 171, 1e-10),
+                                0.55269916901806648)
+
+    def test_betaincinv_specific_vals(self):
+        # specific values borrowed from SciPy test suite
+        special = cupyx.scipy.special
+
+        assert special.betaincinv(1, 1, 1) == 1.0
+        testing.assert_allclose(special.betaincinv(0.0342, 171, 0.25),
+                                8.4231316935498957e-21, rtol=3e-12, atol=0)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_convex_analysis.py b/tests/cupyx_tests/scipy_tests/special_tests/test_convex_analysis.py
new file mode 100644
index 0000000..34fcbd9
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_convex_analysis.py
@@ -0,0 +1,59 @@
+import itertools
+import unittest
+
+import cupy
+import numpy
+
+from cupy import testing
+import cupyx.scipy.special
+
+
+@testing.with_requires('scipy')
+class TestSpecialConvex(unittest.TestCase):
+
+    def test_huber_basic(self):
+        assert cupyx.scipy.special.huber(-1, 1.5) == cupy.inf
+        testing.assert_allclose(cupyx.scipy.special.huber(2, 1.5),
+                                0.5 * 1.5**2)
+        testing.assert_allclose(cupyx.scipy.special.huber(2, 2.5),
+                                2 * (2.5 - 0.5 * 2))
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_huber(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+        z = testing.shaped_random((10, 2), xp=xp, dtype=dtype)
+        return scp.special.huber(z[:, 0], z[:, 1])
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_entr(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+        values = (0, 0.5, 1.0, cupy.inf)
+        signs = [-1, 1]
+        arr = []
+        for sgn, v in itertools.product(signs, values):
+            arr.append(sgn * v)
+        z = xp.asarray(arr, dtype=dtype)
+        return scp.special.entr(z)
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_rel_entr(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+        values = (0, 0.5, 1.0)
+        signs = [-1, 1]
+        arr = []
+        for sgna, va, sgnb, vb in itertools.product(signs, values, signs,
+                                                    values):
+            arr.append((sgna*va, sgnb*vb))
+        z = xp.asarray(numpy.array(arr, dtype=dtype))
+        return scp.special.kl_div(z[:, 0], z[:, 1])
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_pseudo_huber(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+        z = testing.shaped_random((10, 2), xp=numpy, dtype=dtype).tolist()
+        z = xp.asarray(z + [[0, 0.5], [0.5, 0]], dtype=dtype)
+        return scp.special.pseudo_huber(z[:, 0], z[:, 1])
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_digamma.py b/tests/cupyx_tests/scipy_tests/special_tests/test_digamma.py
new file mode 100644
index 0000000..6c95965
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_digamma.py
@@ -0,0 +1,59 @@
+import unittest
+
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+import numpy
+
+
+@testing.with_requires('scipy')
+class TestDigamma(unittest.TestCase):
+
+    @testing.with_requires('scipy>=1.1.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_arange(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return scp.special.digamma(a)
+
+    @testing.with_requires('scipy>=1.1.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_linspace_positive(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = numpy.linspace(0, 30, 1000, dtype=dtype)
+        a = xp.asarray(a)
+        return scp.special.digamma(a)
+
+    @testing.with_requires('scipy>=1.1.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_linspace_negative(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = numpy.linspace(-30, 0, 1000, dtype=dtype)
+        a = xp.asarray(a)
+        return scp.special.digamma(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_scalar(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        return scp.special.digamma(dtype(1.5))
+
+    @testing.with_requires('scipy>=1.1.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_inf_and_nan(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = numpy.array([-numpy.inf, numpy.nan, numpy.inf]).astype(dtype)
+        a = xp.asarray(a)
+        return scp.special.digamma(a)
+
+    def test_psi(self):
+        """Verify that psi exists and is the same as digamma"""
+        assert cupyx.scipy.special.psi is cupyx.scipy.special.digamma
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_erf.py b/tests/cupyx_tests/scipy_tests/special_tests/test_erf.py
new file mode 100644
index 0000000..5e68bf8
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_erf.py
@@ -0,0 +1,148 @@
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+
+
+def _boundary_inputs(boundary, rtol, atol):
+    left = boundary * (1 - numpy.copysign(rtol, boundary)) - atol
+    right = boundary * (1 + numpy.copysign(rtol, boundary)) + atol
+    return [left, boundary, right]
+
+
+class _TestBase(object):
+
+    def test_erf(self):
+        self.check_unary('erf')
+
+    def test_erfc(self):
+        self.check_unary('erfc')
+
+    def test_erfcx(self):
+        self.check_unary('erfcx')
+
+    @testing.with_requires('scipy>=1.4.0')
+    def test_erfinv(self):
+        self.check_unary('erfinv')
+        self.check_unary_random('erfinv', scale=2, offset=-1)
+        self.check_unary_boundary('erfinv', boundary=-1)
+        self.check_unary_boundary('erfinv', boundary=1)
+
+    @testing.with_requires('scipy>=1.4.0')
+    def test_erfcinv(self):
+        self.check_unary('erfcinv')
+        self.check_unary_random('erfcinv', scale=2, offset=0)
+        self.check_unary_boundary('erfcinv', boundary=0)
+        self.check_unary_boundary('erfcinv', boundary=2)
+
+
+@testing.with_requires('scipy')
+class TestSpecial(unittest.TestCase, _TestBase):
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def check_unary(self, name, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return getattr(scp.special, name)(a)
+
+    @testing.for_dtypes(['f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def check_unary_random(self, name, xp, scp, dtype, scale, offset):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_random((2, 3), xp, dtype, scale=scale) + offset
+        return getattr(scp.special, name)(a)
+
+    @testing.for_dtypes(['f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def check_unary_boundary(self, name, xp, scp, dtype, boundary):
+        import scipy.special  # NOQA
+
+        a = _boundary_inputs(boundary, 1.0 / 1024, 1.0 / 1024)
+        a = xp.array(a, dtype=dtype)
+        return getattr(scp.special, name)(a)
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_dtypes(['f', 'd'])
+    def test_erfinv_behavior(self, dtype):
+        a = cupy.empty((1,), dtype=dtype)
+
+        a[:] = 1.0 + 1E-6
+        a = cupyx.scipy.special.erfinv(a)
+        assert cupy.isnan(a)
+        a[:] = -1.0 - 1E-6
+        a = cupyx.scipy.special.erfinv(a)
+        assert cupy.isnan(a)
+        a[:] = 1.0
+        a = cupyx.scipy.special.erfinv(a)
+        assert numpy.isposinf(cupy.asnumpy(a))
+        a[:] = -1.0
+        a = cupyx.scipy.special.erfinv(a)
+        assert numpy.isneginf(cupy.asnumpy(a))
+
+    @testing.with_requires('scipy>=1.4.0')
+    @testing.for_dtypes(['f', 'd'])
+    def test_erfcinv_behavior(self, dtype):
+        a = cupy.empty((1,), dtype=dtype)
+
+        a[:] = 2.0 + 1E-6
+        a = cupyx.scipy.special.erfcinv(a)
+        assert cupy.isnan(a)
+        a[:] = 0.0 - 1E-6
+        a = cupyx.scipy.special.erfcinv(a)
+        assert cupy.isnan(a)
+        a[:] = 0.0
+        a = cupyx.scipy.special.erfcinv(a)
+        assert numpy.isposinf(cupy.asnumpy(a))
+        a[:] = 2.0
+        a = cupyx.scipy.special.erfcinv(a)
+        assert numpy.isneginf(cupy.asnumpy(a))
+
+
+@testing.with_requires('scipy')
+class TestFusionSpecial(unittest.TestCase, _TestBase):
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def check_unary(self, name, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+
+        @cupy.fuse()
+        def f(x):
+            return getattr(scp.special, name)(x)
+
+        return f(a)
+
+    @testing.for_dtypes(['f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def check_unary_random(self, name, xp, scp, dtype, scale, offset):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_random((2, 3), xp, dtype, scale=scale) + offset
+
+        @cupy.fuse()
+        def f(x):
+            return getattr(scp.special, name)(x)
+
+        return f(a)
+
+    @testing.for_dtypes(['f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def check_unary_boundary(self, name, xp, scp, dtype, boundary):
+        import scipy.special  # NOQA
+
+        a = _boundary_inputs(boundary, 1.0 / 1024, 1.0 / 1024)
+        a = xp.array(a, dtype=dtype)
+
+        @cupy.fuse()
+        def f(x):
+            return getattr(scp.special, name)(x)
+
+        return f(a)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_exp1.py b/tests/cupyx_tests/scipy_tests/special_tests/test_exp1.py
new file mode 100644
index 0000000..d9c2580
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_exp1.py
@@ -0,0 +1,40 @@
+import cupyx.scipy.special  # NOQA
+
+from cupy import testing
+from cupy.testing import numpy_cupy_allclose
+
+
+@testing.with_requires("scipy")
+class TestExp1:
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_exp1_zero(self, xp, scp):
+        return scp.special.exp1(0.0)
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_exp1_negative(self, xp, scp):
+        return scp.special.exp1(-4.2)
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_exp1_large_negative(self, xp, scp):
+        return scp.special.exp1(-1000)
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_exp1_positive(self, xp, scp):
+        return scp.special.exp1(4.2)
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_exp1_large_positive(self, xp, scp):
+        return scp.special.exp1(1000)
+
+    @testing.for_dtypes(["f"])
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-5)
+    def test_exp1_linspace_float32(self, xp, scp, dtype):
+        x = xp.linspace(-10, 60, 1000, dtype=dtype)
+        return scp.special.exp1(x)
+
+    @testing.for_dtypes(["d"])
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-12)
+    def test_exp1_linspace_float64(self, xp, scp, dtype):
+        x = xp.linspace(-10, 100, 1000, dtype=dtype)
+        return scp.special.exp1(x)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_expi.py b/tests/cupyx_tests/scipy_tests/special_tests/test_expi.py
new file mode 100644
index 0000000..a134e24
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_expi.py
@@ -0,0 +1,40 @@
+import cupyx.scipy.special  # NOQA
+
+from cupy import testing
+from cupy.testing import numpy_cupy_allclose
+
+
+@testing.with_requires("scipy")
+class TestExpi:
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expi_zero(self, xp, scp):
+        return scp.special.expi(0.0)
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expi_negative(self, xp, scp):
+        return scp.special.expi(-4.2)
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expi_large_negative(self, xp, scp):
+        return scp.special.expi(-1000)
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expi_positive(self, xp, scp):
+        return scp.special.expi(4.2)
+
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expi_large_positive(self, xp, scp):
+        return scp.special.expi(1000)
+
+    @testing.for_dtypes("f")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-5)
+    def test_expi_linspace_float32(self, xp, scp, dtype):
+        x = xp.linspace(-10, 60, 1000, dtype=dtype)
+        return scp.special.expi(x)
+
+    @testing.for_dtypes("d")
+    @numpy_cupy_allclose(scipy_name="scp", rtol=1e-12)
+    def test_expi_linspace_float64(self, xp, scp, dtype):
+        x = xp.linspace(-10, 100, 1000, dtype=dtype)
+        return scp.special.expi(x)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_expn.py b/tests/cupyx_tests/scipy_tests/special_tests/test_expn.py
new file mode 100644
index 0000000..2121f93
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_expn.py
@@ -0,0 +1,52 @@
+import cupyx.scipy.special  # NOQA
+
+from cupy import testing
+from cupy.testing import numpy_cupy_allclose
+
+
+@testing.with_requires("scipy")
+class TestExpn:
+
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expn(self, xp, scp, dtype):
+        return scp.special.expn(-1, 1.0)
+
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expn_2(self, xp, scp, dtype):
+        return scp.special.expn(1, -1.0)
+
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expn_large_values(self, xp, scp, dtype):
+        return scp.special.expn(500, 10)
+
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expn_large_values_2(self, xp, scp, dtype):
+        return scp.special.expn(10, 500)
+
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expn_zero_values(self, xp, scp, dtype):
+        return scp.special.expn(1.0, 0)
+
+    @testing.for_dtypes("efdFD")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expn_zero_values_2(self, xp, scp, dtype):
+        return scp.special.expn(0.0, 2)
+
+    @testing.for_dtypes("edf")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expn_array_inputs(self, xp, scp, dtype):
+        x = testing.shaped_arange((5, 4, 2), xp, dtype)
+        n = testing.shaped_arange((5, 1, 2), xp, dtype)
+        return scp.special.expn(n, x)
+
+    @testing.for_dtypes("edf")
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_expn_array_inputs_2(self, xp, scp, dtype):
+        x = testing.shaped_random((5, 3), xp, dtype)
+        n = testing.shaped_random((5, 3), xp, dtype)
+        return scp.special.expn(n, x)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_gamma.py b/tests/cupyx_tests/scipy_tests/special_tests/test_gamma.py
new file mode 100644
index 0000000..34649db
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_gamma.py
@@ -0,0 +1,63 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+
+
+@testing.with_requires('scipy')
+class TestGamma:
+
+    @pytest.mark.parametrize('function', ['gamma', 'loggamma', 'rgamma'])
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_arange(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        func = getattr(scp.special, function)
+        return func(a)
+
+    @pytest.mark.skipif(
+        cupy.cuda.runtime.is_hip and
+        cupy.cuda.runtime.runtimeGetVersion() < 5_00_00000,
+        reason='ROCm/HIP fails in ROCm 4.x')
+    @pytest.mark.parametrize('function', ['gamma', 'loggamma', 'rgamma'])
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_linspace(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        a = numpy.linspace(-30, 30, 1000, dtype=dtype)
+        if a.dtype.kind == 'c':
+            a -= 1j * a
+        a = xp.asarray(a)
+        func = getattr(scp.special, function)
+        return func(a)
+
+    @pytest.mark.parametrize('function', ['gamma', 'loggamma', 'rgamma'])
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_scalar(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        if xp.dtype(dtype).kind == 'c':
+            val = dtype(1.5 + 1.0j)
+        else:
+            val = dtype(1.5)
+        func = getattr(scp.special, function)
+        return func(val)
+
+    # skip on SciPy < 1.5 due to: https://github.com/scipy/scipy/issues/11315
+    @pytest.mark.parametrize('function', ['gamma', 'loggamma', 'rgamma'])
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    @testing.with_requires('scipy>=1.5.0')
+    def test_inf_and_nan(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        a = numpy.array([-numpy.inf, numpy.nan, numpy.inf]).astype(dtype)
+        a = xp.asarray(a)
+        func = getattr(scp.special, function)
+        return func(a)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_gammainc.py b/tests/cupyx_tests/scipy_tests/special_tests/test_gammainc.py
new file mode 100644
index 0000000..eef270b
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_gammainc.py
@@ -0,0 +1,169 @@
+import cupy as cp
+import pytest
+
+import cupyx.scipy.special as sc
+from cupy.testing import assert_allclose, assert_array_equal
+
+# from scipy.special._testutils import FuncData
+
+
+INVALID_POINTS = [(1, -1), (0, 0), (-1, 1), (cp.nan, 1), (1, cp.nan)]
+
+
+class TestGammainc(object):
+    @pytest.mark.skipif(
+        cp.cuda.runtime.is_hip and
+        cp.cuda.runtime.runtimeGetVersion() < 5_00_00000,
+        reason='ROCm/HIP fails in ROCm 4.x')
+    @pytest.mark.parametrize("a, x", INVALID_POINTS)
+    def test_domain(self, a, x):
+        assert cp.isnan(sc.gammainc(a, x))
+
+    def test_a_eq_0_x_gt_0(self):
+        assert sc.gammainc(0, 1) == 1
+
+    @pytest.mark.parametrize(
+        "a, x, desired",
+        [
+            (cp.inf, 1, 0),
+            (cp.inf, 0, 0),
+            (cp.inf, cp.inf, cp.nan),
+            (1, cp.inf, 1),
+        ],
+    )
+    def test_infinite_arguments(self, a, x, desired):
+        result = sc.gammainc(a, x)
+        if cp.isnan(desired):
+            assert cp.isnan(result)
+        else:
+            assert result == desired
+
+    @pytest.mark.skipif(
+        cp.cuda.runtime.is_hip and
+        cp.cuda.runtime.runtimeGetVersion() < 5_00_00000,
+        reason='ROCm/HIP fails in ROCm 4.x')
+    def test_infinite_limits(self):
+        # Test that large arguments converge to the hard-coded limits
+        # at infinity.
+        assert_allclose(
+            sc.gammainc(1000, 100),
+            sc.gammainc(cp.inf, 100),
+            atol=1e-200,  # Use `atol` since the function converges to 0.
+            rtol=0,
+        )
+        assert sc.gammainc(100, 1000) == sc.gammainc(100, cp.inf)
+
+    def test_x_zero(self):
+        a = cp.arange(1, 10)
+        assert_array_equal(sc.gammainc(a, 0), 0)
+
+    @pytest.mark.skipif(
+        cp.cuda.runtime.is_hip and
+        cp.cuda.runtime.runtimeGetVersion() < 5_00_00000,
+        reason='ROCm/HIP fails in ROCm 4.x')
+    def test_limit_check(self):
+        result = sc.gammainc(1e-10, 1)
+        limit = sc.gammainc(0, 1)
+        assert cp.isclose(result, limit)
+
+    def gammainc_line(self, x):
+        # The line a = x where a simpler asymptotic expansion (analog
+        # of DLMF 8.12.15) is available.
+        c = cp.asarray(
+            [
+                -1 / 3,
+                -1 / 540,
+                25 / 6048,
+                101 / 155520,
+                -3184811 / 3695155200,
+                -2745493 / 8151736420,
+            ]
+        )
+        res = 0
+        xfac = 1
+        for ck in c:
+            res -= ck * xfac
+            xfac /= x
+        res /= cp.sqrt(2 * cp.pi * x)
+        res += 0.5
+        return res
+
+    # omit the following test as FuncData has not been implemented
+    # TODO: implement FuncData?
+    # def test_line(self):
+    #     x = cp.logspace(cp.log10(25), 300, 500)
+    #     a = x
+    #     dataset = cp.vstack((a, x, self.gammainc_line(x))).T
+    #     FuncData(sc.gammainc, dataset, (0, 1), 2, rtol=1e-11).check()
+
+    @pytest.mark.skipif(
+        cp.cuda.runtime.is_hip and
+        cp.cuda.runtime.runtimeGetVersion() < 5_00_00000,
+        reason='ROCm/HIP fails in ROCm 4.x')
+    def test_roundtrip(self):
+        a = cp.logspace(-5, 10, 100)
+        x = cp.logspace(-5, 10, 100)
+
+        y = sc.gammaincinv(a, sc.gammainc(a, x))
+        assert_allclose(x, y, rtol=1e-10)
+
+
+class TestGammaincc(object):
+    @pytest.mark.parametrize("a, x", INVALID_POINTS)
+    def test_domain(self, a, x):
+        assert cp.isnan(sc.gammaincc(a, x))
+
+    def test_a_eq_0_x_gt_0(self):
+        assert sc.gammaincc(0, 1) == 0
+
+    @pytest.mark.parametrize(
+        "a, x, desired",
+        [
+            (cp.inf, 1, 1),
+            (cp.inf, 0, 1),
+            (cp.inf, cp.inf, cp.nan),
+            (1, cp.inf, 0),
+        ],
+    )
+    def test_infinite_arguments(self, a, x, desired):
+        result = sc.gammaincc(a, x)
+        if cp.isnan(desired):
+            assert cp.isnan(result)
+        else:
+            assert result == desired
+
+    @pytest.mark.skipif(
+        cp.cuda.runtime.is_hip and
+        cp.cuda.runtime.runtimeGetVersion() < 5_00_00000,
+        reason='ROCm/HIP fails in ROCm 4.x')
+    def test_infinite_limits(self):
+        # Test that large arguments converge to the hard-coded limits
+        # at infinity.
+        assert sc.gammaincc(1000, 100) == sc.gammaincc(cp.inf, 100)
+        assert_allclose(
+            sc.gammaincc(100, 1000),
+            sc.gammaincc(100, cp.inf),
+            atol=1e-200,  # Use `atol` since the function converges to 0.
+            rtol=0,
+        )
+
+    def test_limit_check(self):
+        result = sc.gammaincc(1e-10, 1)
+        limit = sc.gammaincc(0, 1)
+        assert cp.isclose(result, limit)
+
+    def test_x_zero(self):
+        a = cp.arange(1, 10)
+        assert_array_equal(sc.gammaincc(a, 0), 1)
+
+    @pytest.mark.skipif(
+        cp.cuda.runtime.is_hip and
+        cp.cuda.runtime.runtimeGetVersion() < 5_00_00000,
+        reason='ROCm/HIP fails in ROCm 4.x')
+    def test_roundtrip(self):
+        a = cp.logspace(-5, 10, 100)
+        x = cp.logspace(-5, 10, 100)
+
+        y = sc.gammainccinv(a, sc.gammaincc(a, x))
+
+        assert_allclose(x, y, rtol=1e-14)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_gammaln.py b/tests/cupyx_tests/scipy_tests/special_tests/test_gammaln.py
new file mode 100644
index 0000000..aae2b47
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_gammaln.py
@@ -0,0 +1,119 @@
+import math
+
+import numpy
+import pytest
+
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+
+
+@testing.with_requires('scipy')
+class TestGammaln:
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_arange(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return scp.special.gammaln(a)
+
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-5, scipy_name='scp')
+    def test_linspace(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = numpy.linspace(-30, 30, 1000, dtype=dtype)
+        a = xp.asarray(a)
+        return scp.special.gammaln(a)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_scalar(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        return scp.special.gammaln(dtype(1.5))
+
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_inf_and_nan(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = xp.array([-numpy.inf, numpy.nan, numpy.inf]).astype(dtype)
+        return scp.special.gammaln(a)
+
+
+@testing.with_requires('scipy')
+class TestMultigammaln:
+
+    @pytest.mark.parametrize('d', [1, 5, 15])
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-5, scipy_name='scp')
+    def test_linspace(self, xp, scp, dtype, d):
+        import scipy.special  # NOQA
+
+        minval = math.ceil(0.5 * (d - 1) + .0001)
+        a = xp.linspace(minval, minval + 50, 1000, dtype=dtype)
+        return scp.special.multigammaln(a, d)
+
+    @pytest.mark.parametrize('d', [1, 5, 15])
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_scalar(self, xp, scp, dtype, d):
+        import scipy.special  # NOQA
+
+        return scp.special.multigammaln(dtype(30), d)
+
+    @pytest.mark.parametrize('d', [1, 5, 15])
+    @pytest.mark.parametrize('a', ['nan', 'inf'])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_nonfinite_scalar(self, xp, scp, dtype, a, d):
+        import scipy.special  # NOQA
+
+        a = getattr(xp, a)
+        return scp.special.multigammaln(dtype(a), d)
+
+    @pytest.mark.parametrize('d', [1, 5, 15])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_nonfinite_array(self, xp, scp, dtype, d):
+        import scipy.special  # NOQA
+
+        a = xp.array([-numpy.inf, numpy.nan, numpy.inf]).astype(dtype)
+
+        # Note: TypeError is only here because of a bug in SciPy's formatting
+        # of the ValueError message
+        with pytest.raises((ValueError, TypeError)):
+            scp.special.multigammaln(a, d)
+        return xp.zeros(0)
+
+    @pytest.mark.parametrize('d', ['array', 1.5])
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_invalid_d(self, xp, scp, dtype, d):
+        import scipy.special  # NOQA
+
+        a = xp.array([5.0, 10.0, 15.0]).astype(dtype)
+        if d == 'array':
+            d = xp.arange(10, 20, dtype=int)
+
+        with pytest.raises(ValueError):
+            scp.special.multigammaln(a, d)
+        return xp.zeros(0)
+
+    @pytest.mark.parametrize('d', [1, 5, 15])
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-5, scipy_name='scp')
+    def test_invalid_a(self, xp, scp, dtype, d):
+        import scipy.special  # NOQA
+
+        # all values must be > (0.5 * (d - 1)) so this minval is invalid
+        minval = 0.5 * (d - 1)
+        a = xp.linspace(minval, minval + 10, 10, dtype=dtype)
+
+        # Note: TypeError is only here because of a bug in SciPy's formatting
+        # of the ValueError message
+        with pytest.raises((ValueError, TypeError)):
+            scp.special.multigammaln(a, d)
+        return xp.zeros(0)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_log_softmax.py b/tests/cupyx_tests/scipy_tests/special_tests/test_log_softmax.py
new file mode 100644
index 0000000..c346074
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_log_softmax.py
@@ -0,0 +1,66 @@
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+
+
+atol = {'default': 1e-6, cupy.float64: 1e-14}
+rtol = {'default': 1e-6, cupy.float64: 1e-14}
+
+atol_low = {'default': 1e-6, cupy.float16: 1e-3, cupy.float64: 1e-14}
+rtol_low = {'default': 1e-6, cupy.float16: 1e-3, cupy.float64: 1e-14}
+
+
+@testing.with_requires('scipy')
+class TestLogSoftmax:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_log_softmax_ndarray_1(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            # Unsigned integers make underflows in numpy (eventually seen as
+            # overflows in `np.exp(tmp)`)
+            pytest.skip()
+        a = testing.shaped_random((40, 50), xp, dtype=dtype)
+        return scp.special.log_softmax(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_log_softmax_ndarray_2(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            # Unsigned integers make underflows in numpy (eventually seen as
+            # overflows in `np.exp(tmp)`)
+            pytest.skip()
+        a = xp.array([1000, 1]).astype(dtype)
+        return scp.special.log_softmax(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_log_softmax_2d(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            # Unsigned integers make underflows in numpy (eventually seen as
+            # overflows in `np.exp(tmp)`)
+            pytest.skip()
+        a = testing.shaped_random((5, 3), xp, dtype=dtype)
+        return scp.special.log_softmax(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_log_softmax_axis_arg(self, xp, scp, dtype):
+        a = xp.array([[100, 1000], [1e-10, 1e-10]])
+        return scp.special.log_softmax(a, axis=-1)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(
+        scipy_name='scp',
+        atol=atol_low,
+        rtol=rtol_low
+    )
+    def test_log_softmax_3d(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            # Unsigned integers make underflows in numpy (eventually seen as
+            # overflows in `np.exp(tmp)`)
+            pytest.skip()
+        a = testing.shaped_random((2, 3, 4), xp, dtype)
+        return scp.special.log_softmax(a, axis=1)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_logsumexp.py b/tests/cupyx_tests/scipy_tests/special_tests/test_logsumexp.py
new file mode 100644
index 0000000..991687c
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_logsumexp.py
@@ -0,0 +1,112 @@
+import pytest
+
+import numpy
+
+import cupy
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+
+try:
+    import scipy.special  # NOQA
+except ImportError:
+    pass
+
+
+@testing.with_requires('scipy')
+class TestLogsumexp:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_large_inputs(self, xp, scp, dtype):
+        a = xp.arange(400)
+        return scp.special.logsumexp(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_more_large_inputs(self, xp, scp, dtype):
+        a = xp.arange(10000)
+        return scp.special.logsumexp(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_large_numbers(self, xp, scp, dtype):
+        a = xp.array([1000, 1000]).astype(dtype)
+        return scp.special.logsumexp(a)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_keep_dimensions(self, xp, scp):
+        a = xp.array([[100, 1000], [1e10, 1e-10]])
+        return scp.special.logsumexp(a, axis=-1, keepdims=True)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-6)
+    def test_array_inputs(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        a = testing.shaped_random((100, 1000), xp, dtype=dtype)
+        return scp.special.logsumexp(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_sign_argument(self, xp, scp, dtype):
+        a = xp.array([1, 1, 1]).astype(dtype)
+        b = xp.array([1, -1, -1]).astype(dtype)
+        return scp.special.logsumexp(a, b=b, return_sign=True)
+
+    @testing.for_all_dtypes(no_bool=True, no_complex=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_sign_zero(self, xp, scp, dtype):
+        a = xp.array([1, 1]).astype(dtype)
+        b = xp.array([1, -1]).astype(dtype)
+        return scp.special.logsumexp(a, b=b, return_sign=True)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-6)
+    def test_sign_multi_dims(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        a = testing.shaped_random((1, 1, 3, 4), xp, dtype=dtype)
+        b = testing.shaped_random((1, 1, 1, 4), xp, dtype=dtype)
+        return scp.special.logsumexp(a, b=b, return_sign=True)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-6)
+    def test_sign_multi_dims_axis(self, xp, scp, dtype):
+        a = testing.shaped_random((1, 2, 3, 4), xp, dtype=dtype)
+        b = testing.shaped_random((1, 2, 3, 4), xp, dtype=dtype)
+        return scp.special.logsumexp(a, axis=2, b=b, return_sign=True)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-6)
+    def test_sign_multi_dims_axis_2d(self, xp, scp, dtype):
+        a = testing.shaped_random((1, 2, 3, 4), xp, dtype=dtype)
+        b = testing.shaped_random((1, 2, 3, 4), xp, dtype=dtype)
+        return scp.special.logsumexp(a, axis=(1, 3), b=b, return_sign=True)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_b_zero(self, xp, scp, dtype):
+        a = xp.array([1, 100], dtype=dtype)
+        b = xp.array([1, 0], dtype=dtype)
+        return scp.special.logsumexp(a, b=b)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_b_multi_dims(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind in 'u':
+            pytest.skip()
+        a = testing.shaped_arange((4, 1, 2, 1), xp, dtype=dtype)
+        b = testing.shaped_arange((3, 1, 5), xp, dtype=dtype)
+        return scp.special.logsumexp(a, b=b)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_special_values(self, xp, scp):
+        a = xp.array([cupy.inf, -cupy.inf, cupy.nan, -cupy.nan])
+        return scp.special.logsumexp(a)
+
+    @testing.for_all_dtypes(no_bool=True)
+    def test_empty_array_inputs(self, dtype):
+        a = numpy.array([], dtype=dtype)
+        for xp in (scipy, cupyx.scipy):
+            with pytest.raises(ValueError):
+                xp.special.logsumexp(a)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_polygamma.py b/tests/cupyx_tests/scipy_tests/special_tests/test_polygamma.py
new file mode 100644
index 0000000..6c54a4c
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_polygamma.py
@@ -0,0 +1,58 @@
+import unittest
+
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+import numpy
+
+import warnings
+
+
+@testing.with_requires('scipy')
+class TestPolygamma(unittest.TestCase):
+
+    @testing.with_requires('scipy>=1.1.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_arange(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_arange((2, 3), xp, dtype)
+        return scp.special.polygamma(a, b)
+
+    @testing.with_requires('scipy>=1.1.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp')
+    def test_linspace(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = numpy.tile(numpy.arange(5), 200).astype(dtype)
+        b = numpy.linspace(-30, 30, 1000, dtype=dtype)
+        a = xp.asarray(a)
+        b = xp.asarray(b)
+        return scp.special.polygamma(a, b)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_scalar(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        # polygamma in scipy returns numpy.float64 value when inputs scalar.
+        # whatever type input is.
+        return scp.special.polygamma(
+            dtype(2.), dtype(1.5)).astype(numpy.float32)
+
+    @testing.with_requires('scipy>=1.1.0')
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_inf_and_nan(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        x = numpy.array([-numpy.inf, numpy.nan, numpy.inf]).astype(dtype)
+        a = numpy.tile(x, 3)
+        b = numpy.repeat(x, 3)
+        a = xp.asarray(a)
+        b = xp.asarray(b)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            return scp.special.polygamma(a, b)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_softmax.py b/tests/cupyx_tests/scipy_tests/special_tests/test_softmax.py
new file mode 100644
index 0000000..5b4b681
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_softmax.py
@@ -0,0 +1,82 @@
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx
+import cupyx.scipy.special  # NOQA
+
+
+atol = {
+    'default': 1e-6,
+    cupy.float16: 1e-2,
+}
+rtol = {
+    'default': 1e-6,
+    cupy.float16: 1e-2,
+}
+
+
+@testing.with_requires('scipy')
+class TestSoftmax:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_1dim(self, xp, scp, dtype):
+        x = xp.arange(400)
+        return scp.special.softmax(x)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_2dim(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            # Unsigned integers make underflows in numpy (eventually seen as
+            # overflows in `np.exp(tmp)`)
+            pytest.skip()
+        x = testing.shaped_random((5, 4), xp, dtype=dtype, scale=8)
+        return scp.special.softmax(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_2dim_float16(self, xp, scp):
+        x = testing.shaped_random((5, 4), xp, dtype=xp.float16, scale=8)
+        return scp.special.softmax(x)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_multi_dim(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            # Unsigned integers make underflows in numpy (eventually seen as
+            # overflows in `np.exp(tmp)`)
+            pytest.skip()
+        x = testing.shaped_random((5, 6, 7, 4), xp, dtype=dtype, scale=8)
+        return scp.special.softmax(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-4, rtol=1e-4)
+    def test_multi_dim_float16(self, xp, scp):
+        x = testing.shaped_random((5, 6, 7, 4), xp, dtype=xp.float16, scale=8)
+        return scp.special.softmax(x)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=atol)
+    def test_2dim_with_axis(self, xp, scp, dtype):
+        if xp.dtype(dtype).kind == 'u':
+            # Unsigned integers make underflows in numpy (eventually seen as
+            # overflows in `np.exp(tmp)`)
+            pytest.skip()
+        x = testing.shaped_random((5, 4), xp, dtype=dtype, scale=8)
+        return scp.special.softmax(x, axis=1)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_empty(self, xp, scp, dtype):
+        x = testing.shaped_random((), xp, dtype=dtype)
+        return scp.special.softmax(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_zeros_ones(self, xp, scp):
+        x = xp.array([0.0, 1.0, -1.0, -0.0])
+        return scp.special.softmax(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_nans_infs(self, xp, scp):
+        x = xp.array([cupy.inf, cupy.nan, -cupy.nan, -cupy.inf])
+        return scp.special.softmax(x)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_sph_harm.py b/tests/cupyx_tests/scipy_tests/special_tests/test_sph_harm.py
new file mode 100644
index 0000000..b8b7ab4
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_sph_harm.py
@@ -0,0 +1,28 @@
+import cupy as cp
+import pytest
+
+import cupyx.scipy.special  # NOQA
+from cupy import testing
+from cupy.testing import numpy_cupy_allclose
+
+
+def _get_harmonic_list(degree_max):
+    """Generate list of all spherical harmonics up to degree_max."""
+    harmonic_list = []
+    for degree in range(degree_max + 1):
+        for order in range(-degree, degree + 1):
+            harmonic_list.append((order, degree))
+    return harmonic_list
+
+
+@testing.with_requires("scipy")
+class TestBasic():
+
+    @pytest.mark.parametrize("m, n", _get_harmonic_list(degree_max=5))
+    @testing.for_dtypes(["e", "f", "d"])
+    @numpy_cupy_allclose(scipy_name="scp")
+    def test_sph_harm(self, xp, scp, dtype, m, n):
+        theta = xp.linspace(0, 2 * cp.pi)
+        phi = xp.linspace(0, cp.pi)
+        theta, phi = xp.meshgrid(theta, phi)
+        return scp.special.sph_harm(m, n, theta, phi)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_statistics.py b/tests/cupyx_tests/scipy_tests/special_tests/test_statistics.py
new file mode 100644
index 0000000..ea0fdac
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_statistics.py
@@ -0,0 +1,302 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx.scipy.special
+
+
+class _TestBase:
+
+    def test_ndtr(self):
+        self.check_unary_linspace0_1('ndtr')
+
+    def test_log_ndtr(self):
+        self.check_unary_linspace0_1('log_ndtr')
+
+    def test_ndtri(self):
+        self.check_unary_linspace0_1('ndtri')
+
+    def test_logit(self):
+        self.check_unary_lower_precision('logit')
+
+    def test_expit(self):
+        self.check_unary_lower_precision('expit')
+
+    @testing.with_requires('scipy>=1.8.0rc0')
+    def test_log_expit(self):
+        self.check_unary_lower_precision('log_expit')
+
+
+atol = {'default': 1e-14, cupy.float64: 1e-14}
+rtol = {'default': 1e-5, cupy.float64: 1e-14}
+
+# not all functions pass at the stricter tolerances above
+atol_low = {'default': 5e-4, cupy.float64: 1e-12}
+rtol_low = {'default': 5e-4, cupy.float64: 1e-12}
+
+
+@testing.with_requires('scipy')
+class TestSpecial(_TestBase):
+
+    def _check_unary(self, a, name, scp):
+        import scipy.special  # NOQA
+
+        return getattr(scp.special, name)(a)
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=atol, rtol=rtol, scipy_name='scp')
+    def check_unary(self, name, xp, scp, dtype):
+        a = xp.linspace(-10, 10, 100, dtype=dtype)
+        return self._check_unary(a, name, scp)
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=atol_low, rtol=rtol_low,
+                                 scipy_name='scp')
+    def check_unary_lower_precision(self, name, xp, scp, dtype):
+        a = xp.linspace(-10, 10, 100, dtype=dtype)
+        return self._check_unary(a, name, scp)
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=atol, rtol=rtol, scipy_name='scp')
+    def check_unary_linspace0_1(self, name, xp, scp, dtype):
+        p = xp.linspace(0, 1, 1000, dtype=dtype)
+        return self._check_unary(p, name, scp)
+
+    def test_logit_nonfinite(self):
+        logit = cupyx.scipy.special.logit
+        assert float(logit(0)) == -numpy.inf
+        assert float(logit(1)) == numpy.inf
+        assert numpy.isnan(float(logit(1.1)))
+        assert numpy.isnan(float(logit(-0.1)))
+
+    @pytest.mark.parametrize('inverse', [False, True],
+                             ids=['boxcox', 'inv_boxcox'])
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=atol, rtol=rtol, scipy_name='scp')
+    def test_boxcox(self, xp, scp, dtype, inverse):
+        import scipy.special  # NOQA
+
+        # outputs are only finite over range (0, 1)
+        x = xp.linspace(0.001, 1000, 1000, dtype=dtype).reshape((1, 1000))
+        lmbda = xp.asarray([-5, 0, 5], dtype=dtype).reshape((3, 1))
+        result = scp.special.boxcox(x, lmbda)
+        if inverse:
+            result = scp.special.inv_boxcox(result, lmbda)
+        return result
+
+    def test_boxcox_nonfinite(self):
+        boxcox = cupyx.scipy.special.boxcox
+        assert float(boxcox(0, -5)) == -numpy.inf
+        assert numpy.isnan(float(boxcox(-0.1, 5)))
+
+    @pytest.mark.parametrize('inverse', [False, True],
+                             ids=['boxcox', 'inv_boxcox'])
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=atol, rtol=rtol, scipy_name='scp')
+    def test_boxcox1p(self, xp, scp, dtype, inverse):
+        import scipy.special  # NOQA
+
+        x = xp.linspace(-0.99, 1000, 1000, dtype=dtype).reshape((1, 1000))
+        lmbda = xp.asarray([-5, 0, 5], dtype=dtype).reshape((3, 1))
+        result = scp.special.boxcox1p(x, lmbda)
+        if inverse:
+            result = scp.special.inv_boxcox1p(result, lmbda)
+        return result
+
+    def test_boxcox1p_nonfinite(self):
+        boxcox1p = cupyx.scipy.special.boxcox1p
+        assert float(boxcox1p(-1, -5)) == -numpy.inf
+        assert numpy.isnan(float(boxcox1p(-1.1, 5)))
+
+
+@testing.with_requires('scipy')
+class TestFusionSpecial(_TestBase):
+
+    def _check_unary(self, a, name, scp):
+        import scipy.special  # NOQA
+
+        @cupy.fuse()
+        def f(x):
+            return getattr(scp.special, name)(x)
+
+        return f(a)
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=atol, rtol=rtol, scipy_name='scp')
+    def check_unary(self, name, xp, scp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return self._check_unary(a, name, scp)
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=atol_low, rtol=rtol_low,
+                                 scipy_name='scp')
+    def check_unary_lower_precision(self, name, xp, scp, dtype):
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        return self._check_unary(a, name, scp)
+
+    @testing.for_dtypes(['e', 'f', 'd'])
+    @testing.numpy_cupy_allclose(atol=atol, rtol=rtol, scipy_name='scp')
+    def check_unary_linspace0_1(self, name, xp, scp, dtype):
+        a = xp.linspace(0, 1, 1000, dtype)
+        return self._check_unary(a, name, scp)
+
+
+class _TestDistributionsBase:
+
+    def _test_scalar(self, function, args, expected, rtol=1e-12, atol=1e-12):
+        special = cupyx.scipy.special
+        function = getattr(special, function)
+        testing.assert_allclose(function(*args), expected, rtol=rtol,
+                                atol=atol)
+
+
+@testing.with_requires('scipy')
+class TestTwoArgumentDistribution(_TestDistributionsBase):
+
+    @pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                        reason="avoid failures observed on HIP")
+    @pytest.mark.parametrize('function', ['chdtr', 'chdtrc', 'chdtri',
+                                          'pdtr', 'pdtrc', 'pdtri'])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_linspace_broadcast(self, xp, scp, dtype, function):
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, function)
+        # chdtr* comparisons fail at < 1 degree of freedom
+        minval = 1 if function.startswith('chdtr') else -1
+        v = xp.arange(minval, 10, dtype=dtype)[:, xp.newaxis]
+        if function in ['chdtri', 'pdtri']:
+            # concentrate values around probability range ([0, 1])
+            x = xp.linspace(-.1, 1.3, 20, dtype=dtype)
+        else:
+            # concentrate mostly on valid, positive values
+            x = xp.linspace(-1, 10, 20, dtype=dtype)
+        return func(v, x[xp.newaxis, :])
+
+    @testing.for_float_dtypes()
+    @testing.for_int_dtypes(name='int_dtype', no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_linspace_pdtri(self, xp, scp, int_dtype, dtype):
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, 'pdtri')
+        k = xp.arange(1, 10, dtype=int_dtype)[:, xp.newaxis]
+        y = xp.linspace(0, 1, 20, dtype=dtype)[xp.newaxis, :]
+        return func(k, y)
+
+    @pytest.mark.parametrize(
+        'function, args, expected',
+        [('chdtr', (1, 0), 0.0),
+         ('chdtr', (0.7, cupy.inf), 1.0),
+         ('chdtr', (0.6, 3), 0.957890536704110),
+         ('chdtrc', (1, 0), 1.0),
+         ('chdtrc', (0.6, 3), 1 - 0.957890536704110),
+         ('chdtri', (1, 1), 0.0),
+         ('chdtri', (0.6, 1 - 0.957890536704110), 3),
+         ]
+    )
+    def test_scalar(self, function, args, expected):
+        self._test_scalar(function, args, expected)
+
+
+@testing.with_requires('scipy')
+class TestThreeArgumentDistributions(_TestDistributionsBase):
+
+    @pytest.mark.parametrize('function', ['btdtr', 'btdtri', 'fdtr', 'fdtrc',
+                                          'fdtri', 'gdtr', 'gdtrc'])
+    @testing.for_float_dtypes()
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_linspace_broadcasted(self, xp, scp, dtype, function):
+        """Linspace with three arguments.
+
+        This method uses first two arguments with mostly non-negative values.
+        In some cases, the last argument is constrained to range [0, 1]
+        """
+
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, function)
+        # a and b should be positive
+        a = xp.linspace(-1, 21, 30, dtype=dtype)[:, xp.newaxis, xp.newaxis]
+        b = xp.linspace(-1, 21, 30, dtype=dtype)[xp.newaxis, :, xp.newaxis]
+        if function in ['fdtri', 'btdtr', 'btdtri']:
+            # x should be in [0, 1] so concentrate values around that
+            x = xp.linspace(-0.1, 1.3, 20, dtype=dtype)
+        else:
+            # x should be non-negative, but test with at least 1 negative value
+            x = xp.linspace(-1, 10, 20, dtype=dtype)
+        x = x[xp.newaxis, xp.newaxis, :]
+        return func(a, b, x)
+
+    # omit test with scipy < 1.5 due to change in ufunc type signatures
+    @pytest.mark.parametrize('function', ['bdtr', 'bdtrc', 'bdtri'])
+    @testing.for_float_dtypes()
+    @testing.for_signed_dtypes(name='int_dtype')
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    @testing.with_requires('scipy>=1.5.0')
+    @pytest.mark.skipif(cupy.cuda.runtime.is_hip,
+                        reason="avoid failures observed on HIP")
+    def test_binomdist_linspace(self, xp, scp, function, dtype, int_dtype):
+        import scipy.special  # NOQA
+
+        # Skip cases deprecated in SciPy 1.5+ via this Cython code:
+        # https://github.com/scipy/scipy/blob/cdb9b034d46c7ba0cacf65a9b2848c5d49c286c4/scipy/special/_legacy.pxd#L39-L43  # NOQA
+        # All type casts except `dld->d` should raise a DeprecationWawrning
+        # However on the SciPy side, this shows up as a SystemError
+        #    SystemError: <class 'DeprecationWarning'> returned a result with an exception set  # NOQA
+        safe_cast = xp.result_type(int_dtype, 'l') == xp.dtype('l')
+        safe_cast &= xp.result_type(int_dtype, dtype) == xp.float64
+        if not safe_cast:
+            return xp.zeros((1,))
+
+        func = getattr(scp.special, function)
+        n = xp.linspace(0, 80, 80, dtype=int_dtype)[xp.newaxis, :, xp.newaxis]
+
+        # broadcast to create k <= n
+        k = xp.linspace(0, 1, 10, dtype=dtype)
+        k = k[:, xp.newaxis, xp.newaxis] * n
+        p = xp.linspace(0, 1, 5, dtype=dtype)[xp.newaxis, xp.newaxis, :]
+        return func(k, n, p)
+
+    @pytest.mark.parametrize('function', ['nbdtr', 'nbdtrc', 'nbdtri'])
+    @testing.for_float_dtypes()
+    @testing.for_signed_dtypes(name='int_dtype')
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
+    def test_negbinomdist_linspace(self, xp, scp, function, dtype, int_dtype):
+        import scipy.special  # NOQA
+
+        func = getattr(scp.special, function)
+        n = xp.linspace(0, 20, 20, dtype=int_dtype)[xp.newaxis, :, xp.newaxis]
+        k = xp.linspace(0, 20, 20, dtype=int_dtype)[:, xp.newaxis, xp.newaxis]
+        p = xp.linspace(0, 1, 5, dtype=dtype)[xp.newaxis, xp.newaxis, :]
+        return func(k, n, p)
+
+    @pytest.mark.parametrize(
+        'function, args, expected',
+        [('btdtr', (1, 1, 1), 1.0),
+         ('btdtri', (1, 1, 1), 1.0),
+         ('betainc', (1, 1, 0), 0.0),
+         # Computed using Wolfram Alpha: CDF[FRatioDistribution[1e-6, 5], 10]
+         ('fdtr', (1e-6, 5, 10), 0.9999940790193488),
+         ('fdtrc', (1, 1, 0), 1.0),
+         # Computed using Wolfram Alpha:
+         #   1 - CDF[FRatioDistribution[2, 1/10], 1e10]
+         ('fdtrc', (2, 0.1, 1e10), 0.2722378462129351),
+         # From Wolfram Alpha:
+         #   CDF[FRatioDistribution[1/10, 1], 3] = 0.8756751669632106...
+         ('fdtri', (0.1, 1, 0.8756751669632106), 3.0),
+         ('gdtr', (1, 1, 0), 0.0),
+         ('gdtr', (1, 1, cupy.inf), 1.0),
+         ('gdtrc', (1, 1, 0), 1.0),
+         ('bdtr', (1, 1, 0.5), 1.0),
+         ('bdtrc', (1, 3, 0.5), 0.5),
+         ('bdtri', (1, 3, 0.5), 0.5),
+         ('nbdtr', (1, 1, 1), 1.0),
+         ('nbdtrc', (1, 1, 1), 0.0),
+         ('nbdtri', (1, 1, 1), 1.0),
+         ]
+    )
+    def test_scalar(self, function, args, expected):
+        self._test_scalar(function, args, expected)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_ufunc_dispatch.py b/tests/cupyx_tests/scipy_tests/special_tests/test_ufunc_dispatch.py
new file mode 100644
index 0000000..3b58dd1
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_ufunc_dispatch.py
@@ -0,0 +1,54 @@
+import numpy
+import cupy
+import cupyx.scipy.special
+
+from cupy import testing
+import pytest
+
+try:
+    import scipy.special
+
+    scipy_ufuncs = {
+        f
+        for f in scipy.special.__all__
+        if isinstance(getattr(scipy.special, f), numpy.ufunc)
+    }
+    cupyx_scipy_ufuncs = {
+        f
+        for f in dir(cupyx.scipy.special)
+        if isinstance(getattr(cupyx.scipy.special, f), cupy.ufunc)
+    }
+
+except ImportError:
+    scipy_ufuncs = set()
+    cupyx_scipy_ufuncs = set()
+
+
+@testing.with_requires("scipy")
+@pytest.mark.parametrize("ufunc", sorted(cupyx_scipy_ufuncs & scipy_ufuncs))
+class TestUfunc:
+    def _should_skip(self, f):
+        if f.startswith("gammainc"):
+            if (
+                cupy.cuda.runtime.is_hip
+                and cupy.cuda.runtime.runtimeGetVersion() < 5_00_00000
+            ):
+                pytest.skip('ROCm/HIP fails in ROCm 4.x')
+
+    @testing.numpy_cupy_allclose(atol=1e-4)
+    def test_dispatch(self, xp, ufunc):
+        self._should_skip(ufunc)
+        ufunc = getattr(scipy.special, ufunc)
+        # some ufunc (like sph_harm) do not work with float inputs
+        # therefore we retrieve the types from the ufunc itself
+        if ufunc.__name__ in ['bdtr', 'bdtrc', 'bdtri']:
+            # Make sure non-deprecated types are used.
+            # (avoids DeprecationWarning from SciPy >=1.7)
+            types = 'dld->d'
+        else:
+            types = ufunc.types[0]
+        args = [
+            cupy.testing.shaped_random((5,), xp, dtype=types[i])
+            for i in range(ufunc.nin)
+        ]
+        return ufunc(*args)
diff --git a/tests/cupyx_tests/scipy_tests/special_tests/test_zeta.py b/tests/cupyx_tests/scipy_tests/special_tests/test_zeta.py
new file mode 100644
index 0000000..86e84a9
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/special_tests/test_zeta.py
@@ -0,0 +1,48 @@
+import unittest
+
+from cupy import testing
+import cupyx.scipy.special  # NOQA
+import numpy
+
+
+@testing.with_requires('scipy')
+class TestZeta(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_arange(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        b = testing.shaped_arange((2, 3), xp, dtype)
+        return scp.special.zeta(a, b)
+
+    @testing.for_all_dtypes(no_complex=True, no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-6, scipy_name='scp')
+    def test_linspace(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = numpy.linspace(-30, 30, 1000, dtype=dtype)
+        b = numpy.linspace(-30, 30, 1000, dtype=dtype)
+        a = xp.asarray(a)
+        b = xp.asarray(b)
+        return scp.special.zeta(a, b)
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_scalar(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        return scp.special.zeta(dtype(2.), dtype(1.5))
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(atol=1e-2, rtol=1e-3, scipy_name='scp')
+    def test_inf_and_nan(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        x = numpy.array([-numpy.inf, numpy.nan, numpy.inf]).astype(dtype)
+        a = numpy.tile(x, 3)
+        b = numpy.repeat(x, 3)
+        a = xp.asarray(a)
+        b = xp.asarray(b)
+        return scp.special.zeta(a, b)
diff --git a/tests/cupyx_tests/scipy_tests/stats_tests/test_distributions.py b/tests/cupyx_tests/scipy_tests/stats_tests/test_distributions.py
new file mode 100644
index 0000000..c55de9d
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/stats_tests/test_distributions.py
@@ -0,0 +1,138 @@
+import math
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx.scipy.stats  # NOQA
+from cupyx.scipy import stats
+from cupyx.scipy.stats import _distributions
+
+try:
+    import scipy.stats  # NOQA
+except ImportError:
+    pass
+
+
+class TestEntropyBasic(unittest.TestCase):
+    def test_entropy_positive(self):
+        # See ticket SciPy's gh-497
+        pk = cupy.asarray([0.5, 0.2, 0.3])
+        qk = cupy.asarray([0.1, 0.25, 0.65])
+        eself = stats.entropy(pk, pk)
+        edouble = stats.entropy(pk, qk)
+        assert 0.0 == eself
+        assert edouble >= 0.0
+
+    def test_entropy_base(self):
+        pk = cupy.ones(16, float)
+        s = stats.entropy(pk, base=2.0)
+        assert abs(s - 4.0) < 1.0e-5
+
+        qk = cupy.ones(16, float)
+        qk[:8] = 2.0
+        s = stats.entropy(pk, qk)
+        s2 = stats.entropy(pk, qk, base=2.0)
+        assert abs(s / s2 - math.log(2.0)) < 1.0e-5
+
+    def test_entropy_zero(self):
+        # Test for SciPy PR-479
+        s = stats.entropy(cupy.asarray([0, 1, 2]))
+        expected = 0.63651416829481278
+        assert abs(float(s) - expected) < 1e-12
+
+    def test_entropy_2d(self):
+        pk = cupy.asarray([[0.1, 0.2], [0.6, 0.3], [0.3, 0.5]])
+        qk = cupy.asarray([[0.2, 0.1], [0.3, 0.6], [0.5, 0.3]])
+        testing.assert_array_almost_equal(
+            stats.entropy(pk, qk), [0.1933259, 0.18609809]
+        )
+
+    def test_entropy_2d_zero(self):
+        pk = cupy.asarray([[0.1, 0.2], [0.6, 0.3], [0.3, 0.5]])
+        qk = cupy.asarray([[0.0, 0.1], [0.3, 0.6], [0.5, 0.3]])
+        testing.assert_array_almost_equal(stats.entropy(pk, qk),
+                                          [cupy.inf, 0.18609809])
+
+        pk[0][0] = 0.0
+        testing.assert_array_almost_equal(
+            stats.entropy(pk, qk), [0.17403988, 0.18609809]
+        )
+
+    def test_entropy_base_2d_nondefault_axis(self):
+        pk = cupy.asarray([[0.1, 0.2], [0.6, 0.3], [0.3, 0.5]])
+        testing.assert_array_almost_equal(
+            stats.entropy(pk, axis=1),
+            cupy.asarray([0.63651417, 0.63651417, 0.66156324]),
+        )
+
+    def test_entropy_2d_nondefault_axis(self):
+        pk = cupy.asarray([[0.1, 0.2], [0.6, 0.3], [0.3, 0.5]])
+        qk = cupy.asarray([[0.2, 0.1], [0.3, 0.6], [0.5, 0.3]])
+        testing.assert_array_almost_equal(
+            stats.entropy(pk, qk, axis=1),
+            cupy.asarray([0.231049, 0.231049, 0.127706]),
+        )
+
+    def test_entropy_raises_value_error(self):
+        pk = cupy.asarray([[0.1, 0.2], [0.6, 0.3], [0.3, 0.5]])
+        qk = cupy.asarray([[0.1, 0.2], [0.6, 0.3]])
+        with pytest.raises(ValueError):
+            stats.entropy(pk, qk)
+
+
+@testing.parameterize(*(
+    testing.product({
+        'shape': [(64, ), (16, 15), (14, 4, 10)],
+        'base': [None, 10],
+        'axis': [None, 0, -1],
+        'use_qk': [False, True],
+        'normalize': [False, True],
+    })
+))
+@testing.with_requires('scipy>=1.4.0')
+class TestEntropy(unittest.TestCase):
+
+    def _entropy(self, xp, scp, dtype, shape, use_qk, base, axis, normalize):
+        pk = testing.shaped_random(shape, xp, dtype=dtype)
+        is_float16 = pk.dtype.char == 'e'
+        if use_qk:
+            qk = testing.shaped_random(shape, xp, dtype=dtype)
+        else:
+            qk = None
+
+        if normalize and pk.dtype.kind != 'c':
+            # if we don't normalize pk and qk, entropy will do it internally
+            norm_axis = 0 if axis is None else axis
+            pk = _distributions._normalize(pk, norm_axis)
+            if qk is not None:
+                qk = _distributions._normalize(qk, norm_axis)
+        res = scp.stats.entropy(pk, qk=qk, base=base, axis=axis)
+
+        float_type = xp.float32 if pk.dtype.char in 'ef' else xp.float64
+        if res.ndim > 0:
+            # verify expected dtype
+            assert res.dtype == float_type
+
+        # Cast back to the floating precision of the input so that the
+        # correct rtol is used by numpy_cupy_allclose
+        res = xp.asarray(res, xp.float16 if is_float16 else float_type)
+        return res
+
+    @testing.for_all_dtypes(no_complex=True)
+    @testing.numpy_cupy_allclose(rtol={cupy.float16: 1e-3,
+                                       cupy.float32: 1e-6,
+                                       'default': 1e-15},
+                                 scipy_name='scp')
+    def test_entropy(self, xp, scp, dtype):
+        return self._entropy(xp, scp, dtype, self.shape, self.use_qk,
+                             self.base, self.axis, self.normalize)
+
+    @testing.for_complex_dtypes()
+    def test_entropy_complex(self, dtype):
+        for xp, scp in zip([numpy, cupy], [scipy, cupyx.scipy]):
+            with pytest.raises(TypeError):
+                return self._entropy(xp, scp, dtype, self.shape, self.use_qk,
+                                     self.base, self.axis, self.normalize)
diff --git a/tests/cupyx_tests/scipy_tests/stats_tests/test_morestats.py b/tests/cupyx_tests/scipy_tests/stats_tests/test_morestats.py
new file mode 100644
index 0000000..fea10c5
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/stats_tests/test_morestats.py
@@ -0,0 +1,145 @@
+import numpy
+
+import cupy
+from cupy import testing
+import cupyx
+import cupyx.scipy.stats  # NOQA
+
+
+atol = {
+    cupy.float16: 5e-3,
+    cupy.float32: 1e-6,
+    cupy.complex64: 1e-6,
+    cupy.float64: 1e-14,
+    cupy.complex128: 1e-14,
+}
+rtol = {
+    cupy.float16: 5e-3,
+    cupy.float32: 1e-6,
+    cupy.complex64: 1e-6,
+    cupy.float64: 1e-14,
+    cupy.complex128: 1e-14,
+}
+
+
+def _dtype(dtype, xp):
+    dtype = xp.dtype(dtype)
+    if dtype.kind in 'fc':
+        return dtype
+    if dtype in (xp.int8, xp.uint8):
+        return xp.float16
+    if dtype in (xp.int16, xp.uint16):
+        return xp.float32
+    return xp.float64
+
+
+def _make_data(shape, xp, dtype):
+    if dtype == xp.float16:
+        return testing.shaped_random(shape, xp, dtype=dtype, scale=3)
+    else:
+        return testing.shaped_arange(shape, xp, dtype=dtype)
+
+
+def _compute(xp, scp, lmb, data):
+    result = scp.stats.boxcox_llf(lmb, data)
+    if data.ndim == 1:
+        if data.dtype.kind == 'c':
+            assert result.dtype == xp.complex128
+        else:
+            assert result.dtype == xp.float64
+    else:
+        if data.dtype.kind in 'cf':
+            assert result.dtype == data.dtype
+        elif lmb == 0:
+            for dtype1 in [xp.float16, xp.float32, xp.float64]:
+                if xp.can_cast(data.dtype, dtype1):
+                    break
+            assert result.dtype == dtype1
+        else:
+            assert result.dtype == xp.float64
+    return result, _dtype(data.dtype, xp)
+
+
+@testing.with_requires('scipy')
+class TestBoxcox_llf:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_1dim(self, xp, scp, dtype):
+        data = _make_data((10,), xp, dtype)
+        lmb = 4.0
+        result, dtype1 = _compute(xp, scp, lmb, data)
+        return result.astype(dtype1, copy=False)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_2dim(self, xp, scp, dtype):
+        data = _make_data((3, 8), xp, dtype)
+        lmb = 6.0
+        result, dtype1 = _compute(xp, scp, lmb, data)
+        return result.astype(dtype1, copy=False)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_3dim(self, xp, scp, dtype):
+        data = _make_data((10, 3, 4), xp, dtype)
+        lmb = 3.0
+        result, dtype1 = _compute(xp, scp, lmb, data)
+        return result.astype(dtype1, copy=False)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_multi_dim(self, xp, scp, dtype):
+        dtype == xp.float16
+        if dtype == xp.float16:
+            data = _make_data((3, 2, 3, 2), xp, dtype)
+        else:
+            data = _make_data((3, 2, 4, 3), xp, dtype)
+        lmb = 3.0
+        result, dtype1 = _compute(xp, scp, lmb, data)
+        return result.astype(dtype1, copy=False)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_zero_lmb(self, xp, scp, dtype):
+        data = _make_data((9, 14), xp, dtype)
+        lmb = 0.0
+        result, dtype1 = _compute(xp, scp, lmb, data)
+        return result.astype(dtype1, copy=False)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_empty(self, xp, scp, dtype):
+        data = _make_data((0,), xp, dtype)
+        lmb = 3
+        result = scp.stats.boxcox_llf(lmb, data)
+        if xp is numpy:
+            return numpy.array(result)
+        else:
+            return result
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_lmb_neg(self, xp, scp, dtype):
+        data = xp.array([198.0, 233.0, 233.0, 392.0])
+        lmb = -45
+        return scp.stats.boxcox_llf(lmb, data)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_lmb_neg2(self, xp, scp, dtype):
+        data = _make_data((3, 5), xp, dtype)
+        lmb = -3.0
+        result, dtype1 = _compute(xp, scp, lmb, data)
+        return result.astype(dtype1, copy=False)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_array_empty_neg_lmb(self, xp, scp, dtype):
+        data = _make_data((0,), xp, dtype)
+        lmb = -1.0
+        result = scp.stats.boxcox_llf(lmb, data)
+        if xp is numpy:
+            return numpy.array(result)
+        else:
+            return result
diff --git a/tests/cupyx_tests/scipy_tests/stats_tests/test_stats.py b/tests/cupyx_tests/scipy_tests/stats_tests/test_stats.py
new file mode 100644
index 0000000..47aa0f3
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/stats_tests/test_stats.py
@@ -0,0 +1,244 @@
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx
+import cupyx.scipy.stats  # NOQA
+
+try:
+    import scipy.stats
+except ImportError:
+    pass
+
+
+atol = {'default': 1e-6, cupy.float64: 1e-14}
+rtol = {'default': 1e-6, cupy.float64: 1e-14}
+
+
+@testing.with_requires('scipy')
+class TestTrim:
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(
+        scipy_name='scp', rtol=1e-6, contiguous_check=False)
+    @pytest.mark.parametrize('shape', [(24,), (6, 4), (6, 4, 3), (4, 6)])
+    def test_base(self, xp, scp, dtype, order, shape):
+        a = testing.shaped_random(
+            shape, xp=xp, dtype=dtype, order=order, scale=100)
+        return scp.stats.trim_mean(a, 2 / 6.)
+
+    @testing.for_all_dtypes()
+    def test_zero_dim(self, dtype):
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            a = xp.array(0, dtype=dtype)
+            with pytest.raises(IndexError):
+                return scp.stats.trim_mean(a, 2 / 6.)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_zero_dim_axis_none(self, xp, scp, dtype):
+        a = xp.array(0, dtype=dtype)
+        return scp.stats.trim_mean(a, 2 / 6., axis=None)
+
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    @pytest.mark.parametrize('propotiontocut', [0.0, 0.6])
+    def test_empty(self, xp, scp, dtype, propotiontocut):
+        a = xp.array([])
+        return scp.stats.trim_mean(a, 2 / 6., propotiontocut)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_allclose(
+        scipy_name='scp', rtol=1e-6, contiguous_check=False)
+    @pytest.mark.parametrize('axis', [0, 1, 2, 3, -1, None])
+    def test_axis(self, xp, scp, dtype, order, axis):
+        a = testing.shaped_random(
+            (5, 6, 4, 7), xp=xp, dtype=dtype, order=order, scale=100)
+        return scp.stats.trim_mean(a, 2 / 6., axis=axis)
+
+    def test_propotion_too_big(self):
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            a = xp.array([4, 8, 2, 0, 9, 5, 10, 1, 7, 3, 6])
+            with pytest.raises(ValueError):
+                scp.stats.trim_mean(a, 0.6)
+
+
+@testing.with_requires('scipy')
+class TestZmap:
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zmap_1dim(self, xp, scp, dtype):
+        x = testing.shaped_random((10,), xp, dtype=dtype)
+        y = testing.shaped_random((8,), xp, dtype=dtype)
+        return scp.stats.zmap(x, y)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zmap_2dim(self, xp, scp, dtype):
+        x = testing.shaped_random((2, 6), xp, dtype=dtype)
+        y = testing.shaped_random((2, 1), xp, dtype=dtype)
+        return scp.stats.zmap(x, y)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zmap_multi_dim(self, xp, scp, dtype):
+        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=dtype)
+        y = testing.shaped_random((3, 4, 1, 1), xp, dtype=dtype)
+        return scp.stats.zmap(x, y)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zmap_multi_dim_2(self, xp, scp, dtype):
+        x = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=dtype)
+        y = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=dtype)
+        return scp.stats.zmap(x, y)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
+    def test_zmap_multi_dim_2_float16(self, xp, scp):
+        x = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=xp.float16)
+        y = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=xp.float16)
+        return scp.stats.zmap(x, y)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zmap_with_axis(self, xp, scp, dtype):
+        x = testing.shaped_random((2, 3), xp, dtype=dtype)
+        y = testing.shaped_random((1, 3), xp, dtype=dtype)
+        return scp.stats.zmap(x, y, axis=1)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zmap_with_axis_ddof(self, xp, scp, dtype):
+        x = testing.shaped_random((4, 5), xp, dtype=dtype)
+        y = testing.shaped_random((1, 5), xp, dtype=dtype)
+        return scp.stats.zmap(x, y, axis=1, ddof=2)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_zmap_empty(self, xp, scp, dtype):
+        x = xp.array([], dtype=dtype)
+        y = xp.array([1, 3, 5], dtype=dtype)
+        return scp.stats.zmap(x, y)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    @testing.with_requires('scipy>=1.7')
+    def test_zmap_nan_policy_propagate(self, xp, scp, dtype):
+        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
+        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
+        with numpy.errstate(invalid='ignore'):  # numpy warns with complex
+            return scp.stats.zmap(x, y, nan_policy='propagate')
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    @testing.with_requires('scipy>=1.7')
+    def test_zmap_nan_policy_omit(self, xp, scp, dtype):
+        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
+        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
+        return scp.stats.zmap(x, y, nan_policy='omit')
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    @testing.with_requires('scipy>=1.7')
+    def test_zmap_nan_policy_omit_axis_ddof(self, xp, scp, dtype):
+        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
+        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
+        return scp.stats.zmap(x, y, axis=0, ddof=1, nan_policy='omit')
+
+    @testing.for_dtypes('fdFD')
+    @testing.with_requires('scipy>=1.7')
+    def test_zmap_nan_policy_raise(self, dtype):
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            x = xp.array([1, 2, 3], dtype=dtype)
+            y = xp.array([8, -4, xp.nan, 4], dtype=dtype)
+            with pytest.raises(ValueError):
+                scp.stats.zmap(x, y, nan_policy='raise')
+
+
+@testing.with_requires('scipy')
+class TestZscore:
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zscore_1dim(self, xp, scp, dtype):
+        x = testing.shaped_random((10,), xp, dtype=dtype)
+        return scp.stats.zscore(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-3)
+    def test_zscore_1dim_float16(self, xp, scp):
+        x = testing.shaped_random((10,), xp, dtype=xp.float16)
+        return scp.stats.zscore(x)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zscore_2dim(self, xp, scp, dtype):
+        x = testing.shaped_random((5, 3), xp, dtype=dtype)
+        return scp.stats.zscore(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
+    def test_zscore_2dim_float16(self, xp, scp):
+        x = testing.shaped_random((5, 3), xp, dtype=xp.float16)
+        return scp.stats.zscore(x)
+
+    @testing.for_all_dtypes(no_float16=True, no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zscore_multi_dim(self, xp, scp, dtype):
+        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=dtype)
+        return scp.stats.zscore(x)
+
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
+    def test_zscore_multi_dim_float16(self, xp, scp):
+        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=xp.float16)
+        return scp.stats.zscore(x)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=atol)
+    def test_zscore_with_axis(self, xp, scp, dtype):
+        x = testing.shaped_random((5, 6), xp, dtype=dtype)
+        return scp.stats.zscore(x, axis=1)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    def test_zscore_with_axis_ddof(self, xp, scp, dtype):
+        x = testing.shaped_random((2, 3, 8), xp, dtype=dtype)
+        return scp.stats.zscore(x, axis=2, ddof=2)
+
+    @testing.for_all_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(scipy_name='scp')
+    def test_zscore_empty(self, xp, scp, dtype):
+        x = xp.array([], dtype=dtype)
+        return scp.stats.zscore(x)
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    @testing.with_requires('scipy>=1.7')
+    def test_zscore_nan_policy_propagate(self, xp, scp, dtype):
+        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
+        with numpy.errstate(invalid='ignore'):  # numpy warns with complex
+            return scp.stats.zscore(x, nan_policy='propagate')
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    @testing.with_requires('scipy>=1.7')
+    def test_zscore_nan_policy_omit(self, xp, scp, dtype):
+        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
+        return scp.stats.zscore(x, nan_policy='omit')
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
+    @testing.with_requires('scipy>=1.7')
+    def test_zscore_nan_policy_omit_axis_ddof(self, xp, scp, dtype):
+        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
+        return scp.stats.zscore(x, axis=0, ddof=1, nan_policy='omit')
+
+    @testing.for_dtypes('fdFD')
+    @testing.with_requires('scipy>=1.7')
+    def test_zscore_nan_policy_raise(self, dtype):
+        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
+            x = xp.array([1, 2, 3, xp.nan], dtype=dtype)
+            with pytest.raises(ValueError):
+                scp.stats.zscore(x, nan_policy='raise')
diff --git a/tests/cupyx_tests/scipy_tests/test_get_array_module.py b/tests/cupyx_tests/scipy_tests/test_get_array_module.py
new file mode 100644
index 0000000..8311295
--- /dev/null
+++ b/tests/cupyx_tests/scipy_tests/test_get_array_module.py
@@ -0,0 +1,28 @@
+import unittest
+
+from cupy import testing
+import cupyx.scipy.special
+
+
+@testing.with_requires('scipy')
+class TestSpecial(unittest.TestCase):
+
+    @testing.for_dtypes(['f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_get_array_module(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        module = cupyx.scipy.get_array_module(a)
+        assert module is scp
+        return module.special.j0(a)
+
+    @testing.for_dtypes(['f', 'd'])
+    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
+    def test_get_array_module_multiple_parameters(self, xp, scp, dtype):
+        import scipy.special  # NOQA
+
+        a = testing.shaped_arange((2, 3), xp, dtype)
+        module = cupyx.scipy.get_array_module(a, a)
+        assert module is scp
+        return module.special.j1(a)
diff --git a/tests/cupyx_tests/test_cudnn.py b/tests/cupyx_tests/test_cudnn.py
new file mode 100644
index 0000000..84ef7b0
--- /dev/null
+++ b/tests/cupyx_tests/test_cudnn.py
@@ -0,0 +1,457 @@
+import sys
+
+import numpy
+import pytest
+
+import cupy
+import cupy.cuda.cudnn as libcudnn
+from cupy import testing
+
+
+cudnn_enabled = libcudnn.available
+
+
+if cudnn_enabled:
+    modes = [
+        libcudnn.CUDNN_ACTIVATION_SIGMOID,
+        libcudnn.CUDNN_ACTIVATION_RELU,
+        libcudnn.CUDNN_ACTIVATION_TANH,
+    ]
+    coef_modes = [
+        libcudnn.CUDNN_ACTIVATION_CLIPPED_RELU,
+    ]
+    layouts = [
+        libcudnn.CUDNN_TENSOR_NCHW,
+        libcudnn.CUDNN_TENSOR_NHWC,
+    ]
+    cudnn_version = libcudnn.getVersion()
+    if cudnn_version >= 6000:
+        coef_modes.append(libcudnn.CUDNN_ACTIVATION_ELU)
+
+    from cupyx import cudnn
+else:
+    cudnn_version = -1
+    modes = []
+    coef_modes = []
+    layouts = []
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'mode': modes,
+}))
+@pytest.mark.skipif(not cudnn_enabled, reason='cuDNN is not available')
+class TestCudnnActivation:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.x = testing.shaped_arange((3, 4), cupy, self.dtype)
+        self.y = testing.shaped_arange((3, 4), cupy, self.dtype)
+        self.g = testing.shaped_arange((3, 4), cupy, self.dtype)
+
+    def test_activation_forward(self):
+        cudnn.activation_forward(self.x, self.mode)
+
+    def test_activation_backward(self):
+        cudnn.activation_backward(self.x, self.y, self.g, self.mode)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'mode': coef_modes,
+}))
+@pytest.mark.skipif(not cudnn_enabled, reason='cuDNN is not available')
+class TestCudnnActivationCoef:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.x = testing.shaped_arange((3, 4), cupy, self.dtype)
+        self.y = testing.shaped_arange((3, 4), cupy, self.dtype)
+        self.g = testing.shaped_arange((3, 4), cupy, self.dtype)
+        self.coef = self.dtype(0.75)
+
+    def test_activation_forward(self):
+        cudnn.activation_forward(self.x, self.mode, self.coef)
+
+    def test_activation_backward(self):
+        cudnn.activation_backward(self.x, self.y, self.g, self.mode,
+                                  self.coef)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'ratio': [0.0, 0.1, 0.2, 0.5],
+    'seed': [0, 100]
+}))
+@pytest.mark.skipif(not cudnn_enabled, reason='cuDNN is not available')
+class TestCudnnDropout:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.x = testing.shaped_arange((3, 4), cupy, self.dtype)
+        self.gy = testing.shaped_arange((3, 4), cupy, self.dtype)
+        self.states = cudnn.DropoutStates(None, self.seed)
+
+    def test_dropout_forward(self):
+        _, y = self.states.forward(None, self.x, self.ratio)
+        if self.ratio == 0:
+            assert cupy.all(self.x == y)
+        else:
+            assert cupy.all(self.x != y)
+
+    def test_dropout_backward(self):
+        rspace, y = self.states.forward(None, self.x, self.ratio)
+        gx = self.states.backward(
+            None, self.gy, self.ratio, rspace)
+
+        forward_mask = y / self.x
+        backward_mask = gx / self.gy
+
+        # backward_mask must be the same as forward_mask
+        assert cupy.all(forward_mask == backward_mask)
+
+    def test_dropout_seed(self):
+        # initialize Dropoutstates with the same seed
+        states2 = cudnn.DropoutStates(None, self.seed)
+
+        rspace, y = self.states.forward(None, self.x, self.ratio)
+        rspace2, y2 = states2.forward(None, self.x, self.ratio)
+        # forward results must be the same
+        assert cupy.all(y == y2)
+
+        gx = self.states.backward(None, self.gy, self.ratio, rspace)
+        gx2 = states2.backward(None, self.gy, self.ratio, rspace2)
+        # backward results must be the same
+        assert cupy.all(gx == gx2)
+
+
+@testing.parameterize(*(testing.product({
+    'tensor_core': ['always', 'auto', 'never'],
+    'dtype': [numpy.float16, numpy.float32, numpy.float64],
+    'dilate': [1, 2],
+    'groups': [1, 2],
+    'ndim': [2],
+    'max_workspace_size': [0, 2 ** 22],
+    'auto_tune': [True, False],
+    'bias': [True, False],
+    'layout': layouts,
+})))
+@pytest.mark.skipif(not cudnn_enabled, reason='cuDNN is not available')
+class TestConvolutionForward:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        ndim = self.ndim
+        dtype = self.dtype
+        batches = 2
+        if self.layout == libcudnn.CUDNN_TENSOR_NHWC:
+            # channel size must be multiple of 4
+            in_channels_a_group = 4
+            out_channels_a_group = 4
+        else:
+            in_channels_a_group = 3
+            out_channels_a_group = 2
+        in_channels = in_channels_a_group * self.groups
+        out_channels = out_channels_a_group * self.groups
+        # TODO(anaruse): increase test cases.
+        ksize = 3
+        stride = 2
+        pad = ksize // stride * self.dilate
+        self.strides = (stride,) * ndim
+        self.pads = (pad,) * ndim
+        self.dilations = (self.dilate,) * ndim
+        if self.layout == libcudnn.CUDNN_TENSOR_NHWC:
+            self.x = cupy.zeros(
+                (batches,) + (ksize,) * ndim + (in_channels,), dtype)
+            self.W = cupy.zeros(
+                (out_channels,) + (ksize,) * ndim + (in_channels_a_group,),
+                dtype)
+            self.y = cupy.ones(
+                (batches,) + (2,) * ndim + (out_channels,), dtype)
+        else:
+            self.x = cupy.zeros(
+                (batches, in_channels) + (ksize,) * ndim, dtype)
+            self.W = cupy.zeros(
+                (out_channels, in_channels_a_group) + (ksize,) * ndim, dtype)
+            self.y = cupy.ones((batches, out_channels) + (2,) * ndim, dtype)
+        self.b = None
+        if self.bias:
+            self.b = cupy.zeros((out_channels,), dtype)
+
+        version = libcudnn.getVersion()
+        self.err = None
+        if ((self.dilate > 1 and version < 6000) or
+                (self.groups > 1 and version < 7000)):
+            self.err = ValueError
+        elif ndim > 2 and self.dilate > 1:
+            self.err = libcudnn.CuDNNError
+        _workspace_size = cudnn.get_max_workspace_size()
+        cudnn.set_max_workspace_size(self.max_workspace_size)
+        yield
+        cudnn.set_max_workspace_size(_workspace_size)
+
+    def call(self):
+        cudnn.convolution_forward(
+            self.x, self.W, self.b, self.y,
+            self.pads, self.strides, self.dilations, self.groups,
+            auto_tune=self.auto_tune, tensor_core=self.tensor_core,
+            d_layout=self.layout, w_layout=self.layout)
+
+    def test_call(self):
+        if self.layout == libcudnn.CUDNN_TENSOR_NHWC:
+            version = libcudnn.getVersion()
+            if self.groups > 1:
+                pytest.skip()
+            if self.dilate > 1 and version < 7300:
+                pytest.skip()
+            if self.dtype is numpy.float64 and version < 7100:
+                pytest.skip()
+        if self.err is None:
+            self.call()
+            assert (self.y == 0).all()
+        else:
+            with pytest.raises(self.err):
+                self.call()
+
+
+@testing.parameterize(*(testing.product({
+    'tensor_core': ['always', 'auto', 'never'],
+    'dtype': [numpy.float16, numpy.float32, numpy.float64],
+    'dilate': [1, 2],
+    'groups': [1, 2],
+    'ndim': [2, 3],
+    'max_workspace_size': [0, 2 ** 22],
+    'auto_tune': [True, False],
+    'deterministic': [True, False],
+})))
+@pytest.mark.skipif(not cudnn_enabled, reason='cuDNN is not available')
+class TestConvolutionBackwardFilter:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        ndim = self.ndim
+        dtype = self.dtype
+        batches = 2
+        in_channels_a_group = 3
+        out_channels_a_group = 2
+        in_channels = in_channels_a_group * self.groups
+        out_channels = out_channels_a_group * self.groups
+        # TODO(anaruse): increase test cases.
+        ksize = 3
+        stride = 2
+        pad = ksize // stride * self.dilate
+        self.strides = (stride,) * ndim
+        self.pads = (pad,) * ndim
+        self.dilations = (self.dilate,) * ndim
+        self.x = cupy.zeros(
+            (batches, in_channels) + (ksize,) * ndim, dtype)
+        self.gy = cupy.zeros((batches, out_channels) + (2,) * ndim, dtype)
+
+        self.gW = cupy.ones(
+            (out_channels, in_channels_a_group) + (ksize,) * ndim, dtype)
+
+        version = libcudnn.getVersion()
+        deterministic = self.deterministic
+        self.err = None
+        if ((self.dilate > 1 and version < 6000) or
+                (self.groups > 1 and version < 7000)):
+            self.err = ValueError
+        elif deterministic and (
+                (self.dilate > 1 and version < 7000) or
+                (ndim > 2 and version < 6000) or
+                (ndim > 2 and self.dtype == numpy.float64 and version < 8100)):
+            self.err = libcudnn.CuDNNError
+        elif (8000 <= version < 8100 and
+              self.max_workspace_size == 0 and
+              int(cupy.cuda.device.get_compute_capability()) < 70 and
+              self.groups > 1 and ndim > 2 and
+              self.dtype == numpy.float16):
+            self.err = RuntimeError
+        _workspace_size = cudnn.get_max_workspace_size()
+        cudnn.set_max_workspace_size(self.max_workspace_size)
+        yield
+        cudnn.set_max_workspace_size(_workspace_size)
+
+    def call(self):
+        cudnn.convolution_backward_filter(
+            self.x, self.gy, self.gW,
+            self.pads, self.strides, self.dilations, self.groups,
+            deterministic=self.deterministic,
+            auto_tune=self.auto_tune,
+            tensor_core=self.tensor_core)
+
+    def test_call(self):
+        if self.deterministic and self.max_workspace_size == 0:
+            # This test case is very unstable
+            return
+        if self.err is None:
+            self.call()
+            assert (self.gW == 0).all()
+        else:
+            with pytest.raises(self.err):
+                self.call()
+
+
+@testing.parameterize(*(testing.product({
+    'tensor_core': ['always', 'auto', 'never'],
+    'dtype': [numpy.float16, numpy.float32, numpy.float64],
+    'dilate': [1, 2],
+    'groups': [1, 2],
+    'ndim': [2, 3],
+    'max_workspace_size': [0, 2 ** 22],
+    'auto_tune': [True, False],
+    'deterministic': [True, False],
+    'bias': [True, False],
+})))
+@pytest.mark.skipif(not cudnn_enabled, reason='cuDNN is not available')
+class TestConvolutionBackwardData:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        ndim = self.ndim
+        dtype = self.dtype
+        batches = 2
+        in_channels_a_group = 3
+        out_channels_a_group = 2
+        in_channels = in_channels_a_group * self.groups
+        out_channels = out_channels_a_group * self.groups
+        # TODO(anaruse): increase test cases.
+        ksize = 3
+        stride = 2
+        pad = ksize // stride * self.dilate
+        self.strides = (stride,) * ndim
+        self.pads = (pad,) * ndim
+        self.dilations = (self.dilate,) * ndim
+        self.W = cupy.zeros(
+            (out_channels, in_channels_a_group) + (ksize,) * ndim, dtype)
+        self.gy = cupy.zeros((batches, out_channels) + (2,) * ndim, dtype)
+        self.b = None
+        if self.bias:
+            self.b = cupy.zeros((in_channels,), dtype)
+
+        self.gx = cupy.ones(
+            (batches, in_channels) + (ksize,) * ndim, dtype)
+
+        version = libcudnn.getVersion()
+        deterministic = self.deterministic
+        self.err = None
+        if ((self.dilate > 1 and version < 6000) or
+                (self.groups > 1 and version < 7000)):
+            self.err = ValueError
+        elif (sys.platform.startswith('win32') and version == 7605
+                and deterministic and self.dtype == numpy.float16
+                and self.ndim == 3 and self.dilate == 2 and self.groups == 2):
+            # see https://github.com/cupy/cupy/pull/4893
+            self.err = RuntimeError
+        elif deterministic and (
+                (self.dilate > 1 and
+                 (ndim != 2 and version < 8100 or version < 7300)) or
+                (ndim > 2 and version < 6000) or
+                (ndim > 2 and self.dtype == numpy.float64 and version < 8100)):
+            self.err = libcudnn.CuDNNError
+        elif (8000 <= version < 8100 and
+              int(cupy.cuda.device.get_compute_capability()) < 70 and
+              self.dilate > 1 and self.groups > 1 and ndim > 2 and
+              self.dtype == numpy.float16):
+            self.err = RuntimeError
+        _workspace_size = cudnn.get_max_workspace_size()
+        cudnn.set_max_workspace_size(self.max_workspace_size)
+        yield
+        cudnn.set_max_workspace_size(_workspace_size)
+
+    def call(self):
+        cudnn.convolution_backward_data(
+            self.W, self.gy, self.b, self.gx,
+            self.pads, self.strides, self.dilations, self.groups,
+            deterministic=self.deterministic,
+            auto_tune=self.auto_tune,
+            tensor_core=self.tensor_core)
+
+    def test_call(self):
+        if self.deterministic and self.max_workspace_size == 0:
+            # This test case is very unstable
+            return
+        if self.err is None:
+            self.call()
+            assert (self.gx == 0).all()
+        else:
+            with pytest.raises(self.err):
+                self.call()
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'ksize': [1, 3, 5],
+    'stride': [2, 4],
+    'auto_tune': [True, False],
+}))
+@pytest.mark.skipif(
+    not cudnn_enabled or cudnn_version < 7500 or cudnn_version >= 8000,
+    reason='cuDNN 7.x (x >= 5) is required')
+class TestConvolutionNoAvailableAlgorithm:
+    '''Checks if an expected error is raised.
+
+    This checks if an expected error is raised when no available algorithm
+    is found by cuDNN for a configuration. This (no available algorithm found)
+    can occur when convolution_backward_data or convolution_backward_filter is
+    performed with NHWC layout.
+
+    Please notice that conditions that cause the error may change depending on
+    cuDNN version. The conditions below are set based on cuDNN 7.5.0 and 7.6.0.
+    '''
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.layout = libcudnn.CUDNN_TENSOR_NHWC
+        n = 16
+        x_c, y_c = 64, 64
+        x_h, x_w = 32, 32
+        y_h, y_w = x_h // self.stride, x_w // self.stride
+        self.pad = (self.ksize - 1) // 2
+        if self.layout == libcudnn.CUDNN_TENSOR_NHWC:
+            x_shape = (n, x_h, x_w, x_c)
+            y_shape = (n, y_h, y_w, y_c)
+            W_shape = (y_c, self.ksize, self.ksize, x_c)
+        else:
+            x_shape = (n, x_c, x_h, x_w)
+            y_shape = (n, y_c, y_h, y_w)
+            W_shape = (y_c, x_c, self.ksize, self.ksize)
+        self.x = cupy.ones(x_shape, dtype=self.dtype)
+        self.W = cupy.ones(W_shape, dtype=self.dtype)
+        self.y = cupy.empty(y_shape, dtype=self.dtype)
+        self.gx = cupy.empty(x_shape, dtype=self.dtype)
+        self.gW = cupy.empty(W_shape, dtype=self.dtype)
+        self.gy = cupy.ones(y_shape, dtype=self.dtype)
+        _workspace_size = cudnn.get_max_workspace_size()
+        cudnn.set_max_workspace_size(0)
+        yield
+        cudnn.set_max_workspace_size(_workspace_size)
+
+    def test_backward_filter(self):
+        if not (self.layout == libcudnn.CUDNN_TENSOR_NHWC and
+                self.dtype == numpy.float64):
+            pytest.skip()
+        with pytest.raises(RuntimeError):
+            cudnn.convolution_backward_filter(
+                self.x, self.gy, self.gW,
+                pad=(self.pad, self.pad), stride=(self.stride, self.stride),
+                dilation=(1, 1), groups=1, deterministic=False,
+                auto_tune=self.auto_tune, tensor_core='always',
+                d_layout=self.layout, w_layout=self.layout)
+
+    def test_backward_data(self):
+        if self.layout != libcudnn.CUDNN_TENSOR_NHWC:
+            pytest.skip()
+        with pytest.raises(RuntimeError):
+            cudnn.convolution_backward_data(
+                self.W, self.gy, None, self.gx,
+                pad=(self.pad, self.pad), stride=(self.stride, self.stride),
+                dilation=(1, 1), groups=1, deterministic=0,
+                auto_tune=self.auto_tune, tensor_core='always',
+                d_layout=self.layout, w_layout=self.layout)
+
+    def _get_error_type(self):
+        if self.auto_tune:
+            return RuntimeError
+        else:
+            return libcudnn.CuDNNError
diff --git a/tests/cupyx_tests/test_cupyx.py b/tests/cupyx_tests/test_cupyx.py
new file mode 100644
index 0000000..f17a268
--- /dev/null
+++ b/tests/cupyx_tests/test_cupyx.py
@@ -0,0 +1,30 @@
+import unittest
+
+import cupyx
+from cupy import testing
+
+
+@testing.parameterize(*testing.product({
+    'divide': [None],
+}))
+class TestErrState(unittest.TestCase):
+
+    def test_errstate(self):
+        orig = cupyx.geterr()
+        with cupyx.errstate(divide=self.divide):
+            state = cupyx.geterr()
+            assert state.pop('divide') == self.divide
+            orig.pop('divide')
+            assert state == orig
+
+    def test_seterr(self):
+        pass
+
+
+# TODO(hvy): Implement TestErrStateDivide
+
+# TODO(hvy): Implement TestErrStateOver
+
+# TODO(hvy): Implement TestErrStateUnder
+
+# TODO(hvy): Implement TestErrStateInvalid
diff --git a/tests/cupyx_tests/test_cusolver.py b/tests/cupyx_tests/test_cusolver.py
new file mode 100644
index 0000000..ffd3b1c
--- /dev/null
+++ b/tests/cupyx_tests/test_cusolver.py
@@ -0,0 +1,316 @@
+import numpy
+import pytest
+
+import cupy
+from cupyx import cusolver
+from cupy import testing
+from cupy._core import _routines_linalg as _linalg
+import cupyx
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'shape': [
+        # gesvdj tests
+        (5, 3), (4, 4), (3, 5),
+        # gesvdjBatched tests
+        (2, 5, 3), (2, 4, 4), (2, 3, 5),
+    ],
+    'order': ['C', 'F'],
+    'full_matrices': [True, False],
+    'overwrite_a': [True, False],
+}))
+class TestGesvdj:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusolver.check_availability('gesvdj'):
+            pytest.skip('gesvdj is not available')
+        shape = self.shape
+        if self.dtype == numpy.complex64:
+            a_real = numpy.random.random(shape).astype(numpy.float32)
+            a_imag = numpy.random.random(shape).astype(numpy.float32)
+            self.a = a_real + 1.j * a_imag
+        elif self.dtype == numpy.complex128:
+            a_real = numpy.random.random(shape).astype(numpy.float64)
+            a_imag = numpy.random.random(shape).astype(numpy.float64)
+            self.a = a_real + 1.j * a_imag
+        else:
+            self.a = numpy.random.random(shape).astype(self.dtype)
+
+    def test_gesvdj(self):
+        a = cupy.array(self.a, order=self.order)
+        u, s, v = cusolver.gesvdj(a, full_matrices=self.full_matrices,
+                                  overwrite_a=self.overwrite_a)
+
+        # sigma = diag(s)
+        shape = self.shape
+        mn = min(shape[-2:])
+        if self.full_matrices:
+            sigma_shape = shape
+        else:
+            sigma_shape = shape[:-2] + (mn, mn)
+        sigma = cupy.zeros(sigma_shape, self.dtype)
+        ix = numpy.arange(mn)
+        sigma[..., ix, ix] = s
+
+        vh = v.swapaxes(-2, -1).conjugate()
+        aa = cupy.matmul(cupy.matmul(u, sigma), vh)
+        if self.dtype in (numpy.float32, numpy.complex64):
+            decimal = 5
+        else:
+            decimal = 10
+        testing.assert_array_almost_equal(aa, self.a, decimal=decimal)
+
+    def test_gesvdj_no_uv(self):
+        a = cupy.array(self.a, order=self.order)
+        s = cusolver.gesvdj(a, full_matrices=self.full_matrices,
+                            compute_uv=False, overwrite_a=self.overwrite_a)
+        expect = numpy.linalg.svd(self.a, full_matrices=self.full_matrices,
+                                  compute_uv=False)
+        if self.dtype in (numpy.float32, numpy.complex64):
+            decimal = 5
+        else:
+            decimal = 10
+        testing.assert_array_almost_equal(s, expect, decimal=decimal)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'shape': [(5, 4), (1, 4, 3), (4, 3, 2)],
+}))
+class TestGesvda:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusolver.check_availability('gesvda'):
+            pytest.skip('gesvda is not available')
+        if self.dtype == numpy.complex64:
+            a_real = numpy.random.random(self.shape).astype(numpy.float32)
+            a_imag = numpy.random.random(self.shape).astype(numpy.float32)
+            self.a = a_real + 1.j * a_imag
+        elif self.dtype == numpy.complex128:
+            a_real = numpy.random.random(self.shape).astype(numpy.float64)
+            a_imag = numpy.random.random(self.shape).astype(numpy.float64)
+            self.a = a_real + 1.j * a_imag
+        else:
+            self.a = numpy.random.random(self.shape).astype(self.dtype)
+
+    def test_gesvda(self):
+        a = cupy.array(self.a)
+        u, s, v = cusolver.gesvda(a)
+        if a.ndim == 2:
+            batch_size = 1
+            a = a.reshape((1,) + a.shape)
+            u = u.reshape((1,) + u.shape)
+            s = s.reshape((1,) + s.shape)
+            v = v.reshape((1,) + v.shape)
+        else:
+            batch_size = a.shape[0]
+        for i in range(batch_size):
+            sigma = cupy.diag(s[i])
+            vh = v[i].T.conjugate()
+            aa = cupy.matmul(cupy.matmul(u[i], sigma), vh)
+            if self.dtype in (numpy.float32, numpy.complex64):
+                decimal = 5
+            else:
+                decimal = 10
+            testing.assert_array_almost_equal(aa, a[i], decimal=decimal)
+
+    def test_gesvda_no_uv(self):
+        a = cupy.array(self.a)
+        s = cusolver.gesvda(a, compute_uv=False)
+        expect = numpy.linalg.svd(self.a, compute_uv=False)
+        if self.dtype in (numpy.float32, numpy.complex64):
+            decimal = 5
+        else:
+            decimal = 10
+        testing.assert_array_almost_equal(s, expect, decimal=decimal)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'order': ['C', 'F'],
+    'UPLO': ['L', 'U'],
+}))
+class TestSyevj:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusolver.check_availability('syevj'):
+            pytest.skip('syevj is not available')
+        if self.dtype in (numpy.complex64, numpy.complex128):
+            self.a = numpy.array(
+                [[1, 2j, 3], [-2j, 5, 4j], [3, -4j, 9]], dtype=self.dtype)
+        else:
+            self.a = numpy.array(
+                [[1, 2, 3], [2, 5, 4], [3, 4, 9]], dtype=self.dtype)
+
+    def test_syevj(self):
+        a = cupy.array(self.a, order=self.order)
+        w, v = cusolver.syevj(a, UPLO=self.UPLO, with_eigen_vector=True)
+
+        # check eignvalue equation
+        testing.assert_allclose(self.a.dot(
+            v.get()), w * v, rtol=1e-3, atol=1e-4)
+
+    def test_syevjBatched(self):
+        lda, m = self.a.shape
+
+        na = numpy.stack([self.a, self.a + numpy.diag(numpy.ones(m))])
+        a = cupy.array(na, order=self.order)
+
+        w, v = cusolver.syevj(a, UPLO=self.UPLO, with_eigen_vector=True)
+
+        # check eignvalue equation
+        batch_size = a.shape[0]
+        for i in range(batch_size):
+            testing.assert_allclose(
+                na[i].dot(v[i].get()), w[i] * v[i], rtol=1e-3, atol=1e-4)
+
+        # check arbitrary batch dimension shape
+        na = numpy.stack([na, na, na])
+        a = cupy.array(na, order=self.order)
+
+        w, v = cusolver.syevj(a, UPLO=self.UPLO, with_eigen_vector=True)
+
+        assert v.shape == a.shape
+        assert w.shape == a.shape[:-1]
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'n': [10, 100],
+    'nrhs': [None, 1, 10],
+    'compute_type': [None,
+                     _linalg.COMPUTE_TYPE_FP16,
+                     _linalg.COMPUTE_TYPE_TF32,
+                     _linalg.COMPUTE_TYPE_FP32],
+}))
+class TestGesv:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _make_random_matrix(self, shape, xp):
+        a = testing.shaped_random(shape, xp, dtype=self.r_dtype, scale=1)
+        if self.dtype.char in 'FD':
+            a = a + 1j * testing.shaped_random(
+                shape, xp, dtype=self.r_dtype, scale=1)
+        return a
+
+    def _make_well_conditioned_matrix(self, shape):
+        a = self._make_random_matrix(shape, numpy)
+        u, s, vh = numpy.linalg.svd(a)
+        s = testing.shaped_random(s.shape, numpy, dtype=self.r_dtype,
+                                  scale=1) + 1
+        a = numpy.einsum('...ik,...k,...kj->...ij', u, s, vh)
+        return cupy.array(a)
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusolver.check_availability('gesv'):
+            pytest.skip('gesv is not available')
+        self.dtype = numpy.dtype(self.dtype)
+        self.r_dtype = self.dtype.char.lower()
+        a = self._make_well_conditioned_matrix((self.n, self.n))
+        if self.nrhs is None:
+            x_shape = (self.n, )
+        else:
+            x_shape = (self.n, self.nrhs)
+        self.x_ref = self._make_random_matrix(x_shape, cupy)
+        b = numpy.dot(a, self.x_ref)
+        self.tol = self._tol[self.r_dtype]
+        self.a = cupy.array(a)
+        self.b = cupy.array(b)
+        if self.compute_type is not None:
+            old_compute_type = _linalg.get_compute_type(self.dtype)
+            _linalg.set_compute_type(self.dtype, self.compute_type)
+            yield
+            _linalg.set_compute_type(self.dtype, old_compute_type)
+        else:
+            yield
+
+    def test_gesv(self):
+        x = cusolver.gesv(self.a, self.b)
+        cupy.testing.assert_allclose(x, self.x_ref,
+                                     rtol=self.tol, atol=self.tol)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'shape': [(32, 32), (37, 32)],
+    'nrhs': [None, 1, 4],
+    'compute_type': [None,
+                     _linalg.COMPUTE_TYPE_FP16,
+                     _linalg.COMPUTE_TYPE_TF32,
+                     _linalg.COMPUTE_TYPE_FP32],
+}))
+class TestGels:
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusolver.check_availability('gels'):
+            pytest.skip('gels is not available')
+        if self.compute_type is not None:
+            old_compute_type = _linalg.get_compute_type(self.dtype)
+            _linalg.set_compute_type(self.dtype, self.compute_type)
+            yield
+            _linalg.set_compute_type(self.dtype, old_compute_type)
+        else:
+            yield
+
+    def test_gels(self):
+        b_shape = [self.shape[0]]
+        if self.nrhs is not None:
+            b_shape.append(self.nrhs)
+        a = testing.shaped_random(self.shape, numpy, dtype=self.dtype)
+        b = testing.shaped_random(b_shape, numpy, dtype=self.dtype)
+        tol = self._tol[numpy.dtype(self.dtype).char.lower()]
+        x_lstsq = numpy.linalg.lstsq(a, b, rcond=None)[0]
+        x_gels = cusolver.gels(cupy.array(a), cupy.array(b))
+        cupy.testing.assert_allclose(x_gels, x_lstsq, rtol=tol, atol=tol)
+
+
+@testing.parameterize(*testing.product({
+    'tol': [0, 1e-5],
+    'reorder': [0, 1, 2, 3],
+    'b_contiguous': [True, False],
+}))
+@testing.with_requires('scipy')
+class TestCsrlsvqr:
+
+    n = 8
+    density = 0.75
+    _test_tol = {'f': 1e-5, 'd': 1e-12}
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusolver.check_availability('csrlsvqr'):
+            pytest.skip('csrlsvqr is not available')
+
+    def _setup(self, dtype):
+        dtype = numpy.dtype(dtype)
+        a_shape = (self.n, self.n)
+        a = testing.shaped_random(a_shape, numpy, dtype=dtype, scale=2/self.n)
+        a_mask = testing.shaped_random(a_shape, numpy, dtype='f', scale=1)
+        a[a_mask > self.density] = 0
+        a_diag = numpy.diag(numpy.ones((self.n,), dtype=dtype))
+        a = a + a_diag
+        b = testing.shaped_random((self.n,), numpy, dtype=dtype)
+        test_tol = self._test_tol[dtype.char.lower()]
+        return a, b, test_tol
+
+    @testing.for_dtypes('fdFD')
+    def test_csrlsvqr(self, dtype):
+        a, b, test_tol = self._setup(dtype)
+        cp_a = cupy.array(a)
+        sp_a = cupyx.scipy.sparse.csr_matrix(cp_a)
+        cp_b = cupy.array(b)
+        if not self.b_contiguous:
+            b = b[::-1]
+            cp_b = cp_b[::-1]
+        ref_x = numpy.linalg.solve(a, b)
+        x = cupyx.cusolver.csrlsvqr(sp_a, cp_b, tol=self.tol,
+                                    reorder=self.reorder)
+        cupy.testing.assert_allclose(x, ref_x, rtol=test_tol,
+                                     atol=test_tol)
diff --git a/tests/cupyx_tests/test_cusparse.py b/tests/cupyx_tests/test_cusparse.py
new file mode 100644
index 0000000..3c617b7
--- /dev/null
+++ b/tests/cupyx_tests/test_cusparse.py
@@ -0,0 +1,1118 @@
+import functools
+import pickle
+import sys
+
+import numpy
+import pytest
+try:
+    import scipy.sparse
+except ImportError:
+    pass
+
+import cupy
+from cupy import testing
+from cupyx import cusparse
+from cupy.cuda import driver
+from cupy.cuda import runtime
+from cupy_backends.cuda.libs import cusparse as _cusparse
+from cupyx.scipy import sparse
+
+
+class TestMatDescriptor:
+
+    def test_create(self):
+        md = cusparse.MatDescriptor.create()
+        assert isinstance(md.descriptor, int)
+
+    def test_pickle(self):
+        md = cusparse.MatDescriptor.create()
+        md2 = pickle.loads(pickle.dumps(md))
+        assert isinstance(md2.descriptor, int)
+        assert md.descriptor != md2.descriptor
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'transa': [True, False],
+}))
+@testing.with_requires('scipy')
+class TestCsrmm:
+
+    alpha = 0.5
+    beta = 0.25
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.op_a = scipy.sparse.random(2, 3, density=0.5, dtype=self.dtype)
+        if self.transa:
+            self.a = self.op_a.T
+        else:
+            self.a = self.op_a
+        self.b = numpy.random.uniform(-1, 1, (3, 4)).astype(self.dtype)
+        self.c = numpy.random.uniform(-1, 1, (2, 4)).astype(self.dtype)
+
+    def test_csrmm(self):
+        if not cusparse.check_availability('csrmm'):
+            pytest.skip('csrmm is not available')
+        if runtime.is_hip:
+            if self.transa:
+                pytest.xfail('may be buggy')
+
+        a = sparse.csr_matrix(self.a)
+        b = cupy.array(self.b, order='f')
+        y = cusparse.csrmm(a, b, alpha=self.alpha, transa=self.transa)
+        expect = self.alpha * self.op_a.dot(self.b)
+        testing.assert_array_almost_equal(y, expect)
+
+    def test_csrmm_with_c(self):
+        if not cusparse.check_availability('csrmm'):
+            pytest.skip('csrmm is not available')
+        if runtime.is_hip:
+            if self.transa:
+                pytest.xfail('may be buggy')
+
+        a = sparse.csr_matrix(self.a)
+        b = cupy.array(self.b, order='f')
+        c = cupy.array(self.c, order='f')
+        y = cusparse.csrmm(
+            a, b, c=c, alpha=self.alpha, beta=self.beta, transa=self.transa)
+        expect = self.alpha * self.op_a.dot(self.b) + self.beta * self.c
+        assert y is c
+        testing.assert_array_almost_equal(y, expect)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'trans': [(False, False), (True, False), (False, True)],
+}))
+@testing.with_requires('scipy')
+class TestCsrmm2:
+
+    alpha = 0.5
+    beta = 0.25
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.transa, self.transb = self.trans
+        self.op_a = scipy.sparse.random(2, 3, density=0.5, dtype=self.dtype)
+        if self.transa:
+            self.a = self.op_a.T
+        else:
+            self.a = self.op_a
+        self.op_b = numpy.random.uniform(-1, 1, (3, 4)).astype(self.dtype)
+        if self.transb:
+            self.b = self.op_b.T
+        else:
+            self.b = self.op_b
+        self.c = numpy.random.uniform(-1, 1, (2, 4)).astype(self.dtype)
+
+    def test_csrmm2(self):
+        if not cusparse.check_availability('csrmm2'):
+            pytest.skip('csrmm2 is not available')
+        if runtime.is_hip:
+            if self.transa:
+                pytest.xfail('may be buggy')
+
+        a = sparse.csr_matrix(self.a)
+        b = cupy.array(self.b, order='f')
+        y = cusparse.csrmm2(
+            a, b, alpha=self.alpha, transa=self.transa, transb=self.transb)
+        expect = self.alpha * self.op_a.dot(self.op_b)
+        testing.assert_array_almost_equal(y, expect)
+
+    def test_csrmm2_with_c(self):
+        if not cusparse.check_availability('csrmm2'):
+            pytest.skip('csrmm2 is not available')
+        if runtime.is_hip:
+            if self.transa:
+                pytest.xfail('may be buggy')
+
+        a = sparse.csr_matrix(self.a)
+        b = cupy.array(self.b, order='f')
+        c = cupy.array(self.c, order='f')
+        y = cusparse.csrmm2(
+            a, b, c=c, alpha=self.alpha, beta=self.beta,
+            transa=self.transa, transb=self.transb)
+        expect = self.alpha * self.op_a.dot(self.op_b) + self.beta * self.c
+        assert y is c
+        testing.assert_array_almost_equal(y, expect)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'shape': [(3, 4), (4, 3)]
+}))
+@testing.with_requires('scipy>=1.2.0')
+class TestCsrgeam:
+
+    alpha = 0.5
+    beta = 0.25
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        m, n = self.shape
+        self.a = scipy.sparse.random(m, n, density=0.3, dtype=self.dtype)
+        self.b = scipy.sparse.random(m, n, density=0.3, dtype=self.dtype)
+
+    def test_csrgeam(self):
+        if not cusparse.check_availability('csrgeam'):
+            pytest.skip('csrgeam is not available')
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        c = cusparse.csrgeam(a, b, alpha=self.alpha, beta=self.beta)
+        expect = self.alpha * self.a + self.beta * self.b
+        testing.assert_array_almost_equal(c.toarray(), expect.toarray())
+
+    def test_csrgeam2(self):
+        if not cusparse.check_availability('csrgeam2'):
+            pytest.skip('csrgeam2 is not available')
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        c = cusparse.csrgeam2(a, b, alpha=self.alpha, beta=self.beta)
+        expect = self.alpha * self.a + self.beta * self.b
+        testing.assert_array_almost_equal(c.toarray(), expect.toarray())
+
+
+@testing.with_requires('scipy')
+class TestCsrgeamInvalidCases:
+
+    dtype = numpy.float32
+    shape = (4, 3)
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        m, n = self.shape
+        self.a = scipy.sparse.random(m, n, density=0.3, dtype=self.dtype)
+        self.b = scipy.sparse.random(m, n, density=0.3, dtype=self.dtype)
+
+    def test_csrgeam_invalid_format(self):
+        if not cusparse.check_availability('csrgeam'):
+            pytest.skip('csrgeam is not available')
+        a = sparse.csc_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        with pytest.raises(TypeError):
+            cusparse.csrgeam(a, b)
+        with pytest.raises(TypeError):
+            cusparse.csrgeam(b, a)
+
+    def test_csrgeam_invalid_shape(self):
+        if not cusparse.check_availability('csrgeam'):
+            pytest.skip('csrgeam is not available')
+        a = sparse.csr_matrix(self.a.T)
+        b = sparse.csr_matrix(self.b)
+        with pytest.raises(ValueError):
+            cusparse.csrgeam(a, b)
+
+    def test_csrgeam_availability(self):
+        if not cusparse.check_availability('csrgeam'):
+            a = sparse.csr_matrix(self.a)
+            b = sparse.csr_matrix(self.b)
+            with pytest.raises(RuntimeError):
+                cusparse.csrgeam(a, b)
+
+    def test_csrgeam2_invalid_format(self):
+        if not cusparse.check_availability('csrgeam2'):
+            pytest.skip('csrgeam2 is not available')
+        a = sparse.csc_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        with pytest.raises(TypeError):
+            cusparse.csrgeam2(a, b)
+        with pytest.raises(TypeError):
+            cusparse.csrgeam2(b, a)
+
+    def test_csrgeam2_invalid_shape(self):
+        if not cusparse.check_availability('csrgeam2'):
+            pytest.skip('csrgeam2 is not available')
+        a = sparse.csr_matrix(self.a.T)
+        b = sparse.csr_matrix(self.b)
+        with pytest.raises(ValueError):
+            cusparse.csrgeam2(a, b)
+
+    def test_csrgeam2_availability(self):
+        if not cusparse.check_availability('csrgeam2'):
+            a = sparse.csr_matrix(self.a)
+            b = sparse.csr_matrix(self.b)
+            with pytest.raises(RuntimeError):
+                cusparse.csrgeam2(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'transa': [False, True],
+    'transb': [False, True],
+}))
+@testing.with_requires('scipy')
+class TestCsrgemm:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.op_a = scipy.sparse.random(2, 3, density=0.5, dtype=self.dtype)
+        if self.transa:
+            self.a = self.op_a.T
+        else:
+            self.a = self.op_a
+        self.op_b = scipy.sparse.random(3, 4, density=0.5, dtype=self.dtype)
+        if self.transb:
+            self.b = self.op_b.T
+        else:
+            self.b = self.op_b
+
+    def test_csrgemm(self):
+        if not cusparse.check_availability('csrgemm'):
+            pytest.skip('csrgemm is not available.')
+        if runtime.is_hip:
+            if self.transa or self.transb:
+                pytest.xfail('may be buggy')
+
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        y = cusparse.csrgemm(a, b, transa=self.transa, transb=self.transb)
+        expect = self.op_a.dot(self.op_b)
+        testing.assert_array_almost_equal(y.toarray(), expect.toarray())
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'shape': [(2, 3, 4), (4, 3, 2)]
+}))
+@testing.with_requires('scipy>=1.2.0')
+class TestCsrgemm2:
+
+    alpha = 0.5
+    beta = 0.25
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        m, n, k = self.shape
+        self.a = scipy.sparse.random(m, k, density=0.5, dtype=self.dtype)
+        self.b = scipy.sparse.random(k, n, density=0.5, dtype=self.dtype)
+        self.d = scipy.sparse.random(m, n, density=0.5, dtype=self.dtype)
+
+    def test_csrgemm2_ab(self):
+        if not cusparse.check_availability('csrgemm2'):
+            pytest.skip('csrgemm2 is not available.')
+
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        c = cusparse.csrgemm2(a, b, alpha=self.alpha)
+        expect = self.alpha * self.a.dot(self.b)
+        testing.assert_array_almost_equal(c.toarray(), expect.toarray())
+
+    def test_csrgemm2_abpd(self):
+        if not cusparse.check_availability('csrgemm2'):
+            pytest.skip('csrgemm2 is not available.')
+        if runtime.is_hip and driver.get_build_version() < 402:
+            pytest.xfail('csrgemm2 is buggy')
+
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        d = sparse.csr_matrix(self.d)
+        c = cusparse.csrgemm2(a, b, d=d, alpha=self.alpha, beta=self.beta)
+        expect = self.alpha * self.a.dot(self.b) + self.beta * self.d
+        testing.assert_array_almost_equal(c.toarray(), expect.toarray())
+
+
+@testing.with_requires('scipy')
+class TestCsrgemm2InvalidCases:
+
+    dtype = numpy.float32
+    shape = (2, 3, 4)
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        m, n, k = self.shape
+        self.a = scipy.sparse.random(m, k, density=0.5, dtype=self.dtype)
+        self.b = scipy.sparse.random(k, n, density=0.5, dtype=self.dtype)
+        self.d = scipy.sparse.random(m, n, density=0.5, dtype=self.dtype)
+
+    def test_csrgemm2_invalid_format(self):
+        if not cusparse.check_availability('csrgemm2'):
+            pytest.skip('csrgemm2 is not available.')
+        a = sparse.csc_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        with pytest.raises(TypeError):
+            cusparse.csrgemm2(a, b)
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csc_matrix(self.b)
+        with pytest.raises(TypeError):
+            cusparse.csrgemm2(a, b)
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        d = sparse.csc_matrix(self.d)
+        with pytest.raises(TypeError):
+            cusparse.csrgemm2(a, b, d=d)
+
+    def test_csrgemm2_invalid_shape(self):
+        if not cusparse.check_availability('csrgemm2'):
+            pytest.skip('csrgemm2 is not available.')
+        a = sparse.csc_matrix(self.a).T
+        b = sparse.csr_matrix(self.b)
+        with pytest.raises(ValueError):
+            cusparse.csrgemm2(a, b)
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csc_matrix(self.b).T
+        with pytest.raises(ValueError):
+            cusparse.csrgemm2(a, b)
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        d = sparse.csc_matrix(self.d).T
+        with pytest.raises(ValueError):
+            cusparse.csrgemm2(a, b, d=d)
+
+    def test_csrgemm2_availability(self):
+        if not cusparse.check_availability('csrgemm2'):
+            a = sparse.csr_matrix(self.a)
+            b = sparse.csr_matrix(self.b)
+            with pytest.raises(RuntimeError):
+                cusparse.csrgemm2(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'shape': [(2, 3, 4), (4, 3, 2)]
+}))
+@testing.with_requires('scipy>=1.2.0')
+class TestSpgemm:
+
+    alpha = 0.5
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusparse.check_availability('spgemm'):
+            pytest.skip('spgemm is not available.')
+
+        if (sys.platform == 'win32'
+            and cusparse.getVersion() == 11301
+                and self.dtype == numpy.complex128):
+            pytest.xfail('spgemm fails on CUDA 11.2 on Windows')
+
+        m, n, k = self.shape
+        self.a = scipy.sparse.random(m, k, density=0.5, dtype=self.dtype)
+        self.b = scipy.sparse.random(k, n, density=0.5, dtype=self.dtype)
+
+    def test_spgemm_ab(self):
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        c = cusparse.spgemm(a, b, alpha=self.alpha)
+        expect = self.alpha * self.a.dot(self.b)
+        testing.assert_array_almost_equal(c.toarray(), expect.toarray())
+
+
+@testing.with_requires('scipy')
+class TestSpgemmInvalidCases:
+
+    dtype = numpy.float32
+    shape = (2, 3, 4)
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        m, n, k = self.shape
+        self.a = scipy.sparse.random(m, k, density=0.5, dtype=self.dtype)
+        self.b = scipy.sparse.random(k, n, density=0.5, dtype=self.dtype)
+
+    def test_spgemm_invalid_format(self):
+        if not cusparse.check_availability('spgemm'):
+            pytest.skip('spgemm is not available.')
+        a = sparse.csc_matrix(self.a)
+        b = sparse.csr_matrix(self.b)
+        with pytest.raises(TypeError):
+            cusparse.spgemm(a, b)
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csc_matrix(self.b)
+        with pytest.raises(TypeError):
+            cusparse.spgemm(a, b)
+
+    def test_spgemm_invalid_shape(self):
+        if not cusparse.check_availability('spgemm'):
+            pytest.skip('spgemm is not available.')
+        a = sparse.csc_matrix(self.a).T
+        b = sparse.csr_matrix(self.b)
+        with pytest.raises(ValueError):
+            cusparse.spgemm(a, b)
+        a = sparse.csr_matrix(self.a)
+        b = sparse.csc_matrix(self.b).T
+        with pytest.raises(ValueError):
+            cusparse.spgemm(a, b)
+
+    def test_spgemm_availability(self):
+        if not cusparse.check_availability('spgemm'):
+            a = sparse.csr_matrix(self.a)
+            b = sparse.csr_matrix(self.b)
+            with pytest.raises(RuntimeError):
+                cusparse.spgemm(a, b)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64],
+    'transa': [False, True],
+}))
+@testing.with_requires('scipy')
+class TestCsrmv:
+
+    alpha = 0.5
+    beta = 0.25
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.op_a = scipy.sparse.random(2, 3, density=0.5, dtype=self.dtype)
+        if self.transa:
+            self.a = self.op_a.T
+        else:
+            self.a = self.op_a
+        self.x = numpy.random.uniform(-1, 1, 3).astype(self.dtype)
+        self.y = numpy.random.uniform(-1, 1, 2).astype(self.dtype)
+
+    def test_csrmv(self):
+        if not cusparse.check_availability('csrmv'):
+            pytest.skip('csrmv is not available')
+        if runtime.is_hip:
+            if self.transa:
+                pytest.xfail('may be buggy')
+
+        a = sparse.csr_matrix(self.a)
+        x = cupy.array(self.x, order='f')
+        y = cusparse.csrmv(
+            a, x, alpha=self.alpha, transa=self.transa)
+        expect = self.alpha * self.op_a.dot(self.x)
+        testing.assert_array_almost_equal(y, expect)
+
+    def test_csrmv_with_y(self):
+        if not cusparse.check_availability('csrmv'):
+            pytest.skip('csrmv is not available')
+        if runtime.is_hip:
+            if self.transa:
+                pytest.xfail('may be buggy')
+
+        a = sparse.csr_matrix(self.a)
+        x = cupy.array(self.x, order='f')
+        y = cupy.array(self.y, order='f')
+        z = cusparse.csrmv(
+            a, x, y=y, alpha=self.alpha, beta=self.beta, transa=self.transa)
+        expect = self.alpha * self.op_a.dot(self.x) + self.beta * self.y
+        assert y is z
+        testing.assert_array_almost_equal(y, expect)
+
+    def test_csrmvEx_aligned(self):
+        if not cusparse.check_availability('csrmvEx'):
+            pytest.skip('csrmvEx is not available')
+        a = sparse.csr_matrix(self.a)
+        x = cupy.array(self.x, order='f')
+
+        assert cusparse.csrmvExIsAligned(a, x)
+
+    def test_csrmvEx_not_aligned(self):
+        if not cusparse.check_availability('csrmvEx'):
+            pytest.skip('csrmvEx is not available')
+        a = sparse.csr_matrix(self.a)
+        tmp = cupy.array(numpy.hstack([self.x, self.y]), order='f')
+        x = tmp[0:len(self.x)]
+        y = tmp[len(self.x):]
+        assert not cusparse.csrmvExIsAligned(a, x, y)
+
+    def test_csrmvEx(self):
+        if not cusparse.check_availability('csrmvEx'):
+            pytest.skip('csrmvEx is not available')
+        if self.transa:
+            # no support for transa
+            return
+
+        a = sparse.csr_matrix(self.a)
+        x = cupy.array(self.x, order='f')
+        y = cusparse.csrmvEx(a, x, alpha=self.alpha)
+        expect = self.alpha * self.op_a.dot(self.x)
+        testing.assert_array_almost_equal(y, expect)
+
+    def test_csrmvEx_with_y(self):
+        if not cusparse.check_availability('csrmvEx'):
+            pytest.skip('csrmvEx is not available')
+        if self.transa:
+            # no support for transa
+            return
+        a = sparse.csr_matrix(self.a)
+        x = cupy.array(self.x, order='f')
+        y = cupy.array(self.y, order='f')
+        z = cusparse.csrmvEx(
+            a, x, y=y, alpha=self.alpha, beta=self.beta)
+        expect = self.alpha * self.op_a.dot(self.x) + self.beta * self.y
+        assert y is z
+        testing.assert_array_almost_equal(y, expect)
+
+
+@testing.with_requires('scipy')
+class TestCoosort:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusparse.check_availability('coosort'):
+            pytest.skip('coosort is not available')
+
+        self.a = scipy.sparse.random(
+            100, 100, density=0.9, dtype=numpy.float32, format='coo')
+        numpy.random.shuffle(self.a.row)
+        numpy.random.shuffle(self.a.col)
+
+    def test_coosort(self):
+        a = sparse.coo_matrix(self.a)
+        cusparse.coosort(a)
+        # lexsort by row first and col second
+        argsort = numpy.lexsort((self.a.col, self.a.row))
+        testing.assert_array_equal(self.a.row[argsort], a.row)
+        testing.assert_array_equal(self.a.col[argsort], a.col)
+        testing.assert_array_almost_equal(self.a.data[argsort], a.data)
+
+    def test_coosort_by_column(self):
+        a = sparse.coo_matrix(self.a)
+        cusparse.coosort(a, sort_by='c')
+        # lexsort by col first and row second
+        argsort = numpy.lexsort((self.a.row, self.a.col))
+        testing.assert_array_equal(self.a.row[argsort], a.row)
+        testing.assert_array_equal(self.a.col[argsort], a.col)
+        testing.assert_array_almost_equal(self.a.data[argsort], a.data)
+
+
+@testing.with_requires('scipy')
+class TestCsrsort:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusparse.check_availability('csrsort'):
+            pytest.skip('csrsort is not available')
+
+        self.a = scipy.sparse.random(
+            1, 1000, density=0.9, dtype=numpy.float32, format='csr')
+        numpy.random.shuffle(self.a.indices)
+        self.a.has_sorted_indices = False
+
+    def test_csrsort(self):
+        a = sparse.csr_matrix(self.a)
+        cusparse.csrsort(a)
+
+        self.a.sort_indices()
+        testing.assert_array_equal(self.a.indptr, a.indptr)
+        testing.assert_array_equal(self.a.indices, a.indices)
+        testing.assert_array_almost_equal(self.a.data, a.data)
+
+
+@testing.with_requires('scipy')
+class TestCscsort:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusparse.check_availability('cscsort'):
+            pytest.skip('cscsort is not available')
+
+        self.a = scipy.sparse.random(
+            1000, 1, density=0.9, dtype=numpy.float32, format='csc')
+        numpy.random.shuffle(self.a.indices)
+        self.a.has_sorted_indices = False
+
+    def test_cscsort(self):
+        a = sparse.csc_matrix(self.a)
+        cusparse.cscsort(a)
+
+        self.a.sort_indices()
+        testing.assert_array_equal(self.a.indptr, a.indptr)
+        testing.assert_array_equal(self.a.indices, a.indices)
+        testing.assert_array_almost_equal(self.a.data, a.data)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'transa': [False, True],
+    'shape': [(3, 2), (4, 3)],
+    'format': ['csr', 'csc', 'coo'],
+}))
+@testing.with_requires('scipy>=1.2.0')
+class TestSpmv:
+
+    alpha = 0.5
+    beta = 0.25
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        m, n = self.shape
+        self.op_a = scipy.sparse.random(m, n, density=0.5, format=self.format,
+                                        dtype=self.dtype)
+        if self.transa:
+            self.a = self.op_a.T
+        else:
+            self.a = self.op_a
+        self.x = numpy.random.uniform(-1, 1, n).astype(self.dtype)
+        self.y = numpy.random.uniform(-1, 1, m).astype(self.dtype)
+        if self.format == 'csr':
+            self.sparse_matrix = sparse.csr_matrix
+        elif self.format == 'csc':
+            self.sparse_matrix = sparse.csc_matrix
+        elif self.format == 'coo':
+            self.sparse_matrix = sparse.coo_matrix
+
+    def test_spmv(self):
+        if not cusparse.check_availability('spmv'):
+            pytest.skip('spmv is not available')
+        if runtime.is_hip:
+            if ((self.format == 'csr' and self.transa is True)
+                    or (self.format == 'csc' and self.transa is False)
+                    or (self.format == 'coo' and self.transa is True)):
+                pytest.xfail('may be buggy')
+
+        a = self.sparse_matrix(self.a)
+        if not a.has_canonical_format:
+            a.sum_duplicates()
+        x = cupy.array(self.x)
+        y = cusparse.spmv(a, x, alpha=self.alpha, transa=self.transa)
+        expect = self.alpha * self.op_a.dot(self.x)
+        testing.assert_array_almost_equal(y, expect)
+
+    def test_spmv_with_y(self):
+        if not cusparse.check_availability('spmv'):
+            pytest.skip('spmv is not available')
+        if runtime.is_hip:
+            if ((self.format == 'csr' and self.transa is True)
+                    or (self.format == 'csc' and self.transa is False)
+                    or (self.format == 'coo' and self.transa is True)):
+                pytest.xfail('may be buggy')
+
+        a = self.sparse_matrix(self.a)
+        if not a.has_canonical_format:
+            a.sum_duplicates()
+        x = cupy.array(self.x)
+        y = cupy.array(self.y)
+        z = cusparse.spmv(a, x, y=y, alpha=self.alpha, beta=self.beta,
+                          transa=self.transa)
+        expect = self.alpha * self.op_a.dot(self.x) + self.beta * self.y
+        assert y is z
+        testing.assert_array_almost_equal(y, expect)
+
+
+@testing.with_requires('scipy')
+class TestErrorSpmv:
+
+    dtype = numpy.float32
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        m, n = 2, 3
+        self.a = scipy.sparse.random(m, n, density=0.5,
+                                     dtype=self.dtype)
+        self.x = numpy.random.uniform(-1, 1, n).astype(self.dtype)
+        self.y = numpy.random.uniform(-1, 1, m).astype(self.dtype)
+
+    def test_error_shape(self):
+        if not cusparse.check_availability('spmv'):
+            pytest.skip('spmv is not available')
+
+        a = sparse.csr_matrix(self.a.T)
+        x = cupy.array(self.x)
+        with pytest.raises(ValueError):
+            cusparse.spmv(a, x)
+
+        a = sparse.csr_matrix(self.a)
+        x = cupy.array(self.x)
+        with pytest.raises(ValueError):
+            cusparse.spmv(a, x, transa=True)
+
+        a = sparse.csr_matrix(self.a)
+        x = cupy.array(self.y)
+        with pytest.raises(ValueError):
+            cusparse.spmv(a, x)
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'transa': [False, True],
+    'transb': [False, True],
+    'dims': [(2, 3, 4), (3, 4, 2)],
+    'format': ['csr', 'csc', 'coo'],
+}))
+@testing.with_requires('scipy>=1.2.0')
+class TestSpmm:
+
+    alpha = 0.5
+    beta = 0.25
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        if not cusparse.check_availability('spmm'):
+            pytest.skip('spmm is not available')
+        if cusparse.getVersion() == 11700 and self.format == 'coo':
+            pytest.skip('spmm fails on CUDA 11.5.x with COO')
+        if cusparse.getVersion() == 11702 and self.format in ('csr', 'csc'):
+            pytest.skip('spmm fails on CUDA 11.6.1/11.6.2 with CSR or CSC')
+        if runtime.is_hip:
+            if ((self.format == 'csr' and self.transa is True)
+                    or (self.format == 'csc' and self.transa is False)
+                    or (self.format == 'coo' and self.transa is True)):
+                pytest.xfail('may be buggy')
+        m, n, k = self.dims
+        self.op_a = scipy.sparse.random(m, k, density=0.5, format=self.format,
+                                        dtype=self.dtype)
+        if self.transa:
+            self.a = self.op_a.T
+        else:
+            self.a = self.op_a
+        self.op_b = numpy.random.uniform(-1, 1, (k, n)).astype(self.dtype)
+        if self.transb:
+            self.b = self.op_b.T
+        else:
+            self.b = self.op_b
+        self.c = numpy.random.uniform(-1, 1, (m, n)).astype(self.dtype)
+        if self.format == 'csr':
+            self.sparse_matrix = sparse.csr_matrix
+        elif self.format == 'csc':
+            self.sparse_matrix = sparse.csc_matrix
+        elif self.format == 'coo':
+            self.sparse_matrix = sparse.coo_matrix
+
+    def test_spmm(self):
+        a = self.sparse_matrix(self.a)
+        if not a.has_canonical_format:
+            a.sum_duplicates()
+        b = cupy.array(self.b, order='f')
+        c = cusparse.spmm(
+            a, b, alpha=self.alpha, transa=self.transa, transb=self.transb)
+        expect = self.alpha * self.op_a.dot(self.op_b)
+        testing.assert_array_almost_equal(c, expect)
+
+    def test_spmm_with_c(self):
+        a = self.sparse_matrix(self.a)
+        if not a.has_canonical_format:
+            a.sum_duplicates()
+        b = cupy.array(self.b, order='f')
+        c = cupy.array(self.c, order='f')
+        y = cusparse.spmm(
+            a, b, c=c, alpha=self.alpha, beta=self.beta,
+            transa=self.transa, transb=self.transb)
+        expect = self.alpha * self.op_a.dot(self.op_b) + self.beta * self.c
+        assert y is c
+        testing.assert_array_almost_equal(y, expect)
+
+
+@testing.with_requires('scipy')
+class TestErrorSpmm:
+
+    dtype = numpy.float32
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        m, n, k = 2, 3, 4
+        self.a = scipy.sparse.random(m, k, density=0.5,
+                                     dtype=self.dtype)
+        self.b = numpy.random.uniform(-1, 1, (k, n)).astype(self.dtype)
+        self.c = numpy.random.uniform(-1, 1, (m, n)).astype(self.dtype)
+
+    def test_error_shape(self):
+        if not cusparse.check_availability('spmm'):
+            pytest.skip('spmm is not available')
+
+        a = sparse.csr_matrix(self.a.T)
+        b = cupy.array(self.b, order='f')
+        with pytest.raises(ValueError):
+            cusparse.spmm(a, b)
+
+        a = sparse.csr_matrix(self.a)
+        b = cupy.array(self.b, order='f')
+        with pytest.raises(AssertionError):
+            cusparse.spmm(a, b.T)
+
+        a = sparse.csr_matrix(self.a)
+        b = cupy.array(self.b)
+        with pytest.raises(AssertionError):
+            cusparse.spmm(a, b)
+
+        a = sparse.csr_matrix(self.a)
+        b = cupy.array(self.c, order='f')
+        with pytest.raises(ValueError):
+            cusparse.spmm(a, b)
+
+        a = sparse.csr_matrix(self.a)
+        b = cupy.array(self.b, order='f')
+        c = cupy.array(self.b, order='f')
+        with pytest.raises(ValueError):
+            cusparse.spmm(a, b, c=c)
+
+
+@testing.parameterize(*testing.product({
+    'lower': [True, False],
+    'unit_diag': [True, False],
+    'transa': ['N', 'T', 'H'],
+    'blocking': [True, False],
+    'level_info': [True, False],
+    'format': ['csr', 'csc'],
+    'nrhs': [None, 1, 4],
+    'order': ['C', 'F']
+}))
+@testing.with_requires('scipy')
+class TestCsrsm2:
+
+    n = 6
+    alpha = 1.0
+    density = 0.75
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _setup(self, dtype):
+        dtype = numpy.dtype(dtype)
+        self.tol = self._tol[dtype.char.lower()]
+
+        a_shape = (self.n, self.n)
+        a = testing.shaped_random(a_shape, numpy, dtype=dtype, scale=1)
+        a_mask = testing.shaped_random(a_shape, numpy, dtype='f', scale=1)
+        a[a_mask > self.density] = 0
+        a_diag = numpy.diag(numpy.ones((self.n,), dtype=dtype))
+        if self.unit_diag:
+            a[a_diag > 0] = 0
+        a = a + a_diag
+        cp_a = cupy.array(a)
+        if self.unit_diag:
+            cp_a[a_diag > 0] = 0.1  # any number except 0
+        if self.format == 'csr':
+            self.a = sparse.csr_matrix(cp_a)
+        elif self.format == 'csc':
+            self.a = sparse.csc_matrix(cp_a)
+
+        b_shape = (self.n,) if self.nrhs is None else (self.n, self.nrhs)
+        b = numpy.arange(1, numpy.prod(b_shape) + 1,
+                         dtype=dtype).reshape(b_shape)
+        b = b.copy(order=self.order)
+        self.b = cupy.array(b, order=self.order)
+
+        if self.lower:
+            a = numpy.tril(a)
+        else:
+            a = numpy.triu(a)
+        if self.transa == 'T':
+            a = a.T
+        elif self.transa == 'H':
+            a = a.conj().T
+        self.ref_x = numpy.linalg.solve(a, self.alpha * b)
+
+    @testing.for_dtypes('fdFD')
+    def test_csrsm2(self, dtype):
+        if not cusparse.check_availability('csrsm2'):
+            pytest.skip('csrsm2 is not available')
+        if runtime.is_hip:
+            if (self.transa == 'H'
+                or (driver.get_build_version() < 400
+                    and ((self.format == 'csc' and self.transa == 'N')
+                         or (self.format == 'csr' and self.transa == 'T')))):
+                pytest.xfail('may be buggy')
+
+        if (self.format == 'csc' and numpy.dtype(dtype).char in 'FD' and
+                self.transa == 'H'):
+            pytest.skip('unsupported combination')
+        self._setup(dtype)
+        x = self.b.copy(order=self.order)
+        cusparse.csrsm2(self.a, x, alpha=self.alpha,
+                        lower=self.lower, unit_diag=self.unit_diag,
+                        transa=self.transa, blocking=self.blocking,
+                        level_info=self.level_info)
+        testing.assert_allclose(x, self.ref_x, atol=self.tol, rtol=self.tol)
+
+
+@testing.parameterize(*testing.product({
+    'n': [7, 10],
+    'level_info': [True, False],
+}))
+@testing.with_requires('scipy')
+class TestCsrilu02:
+
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _make_matrix(self, dtype):
+        if not cusparse.check_availability('csrilu02'):
+            pytest.skip('csrilu02 is not available')
+        a = testing.shaped_random((self.n, self.n), cupy, dtype=dtype,
+                                  scale=0.9) + 0.1
+        a = a + cupy.diag(cupy.ones((self.n,), dtype=dtype.char.lower()))
+        return a
+
+    @testing.for_dtypes('fdFD')
+    def test_csrilu02(self, dtype):
+        dtype = numpy.dtype(dtype)
+        a_ref = self._make_matrix(dtype)
+        a = sparse.csr_matrix(a_ref)
+        cusparse.csrilu02(a, level_info=self.level_info)
+        a = a.todense()
+        al = cupy.tril(a, k=-1)
+        al = al + cupy.diag(cupy.ones((self.n,), dtype=dtype.char.lower()))
+        au = cupy.triu(a)
+        a = al @ au
+        tol = self._tol[dtype.char.lower()]
+        cupy.testing.assert_allclose(a, a_ref, atol=tol, rtol=tol)
+
+    def test_invalid_cases(self):
+        dtype = numpy.dtype('d')
+        a_ref = self._make_matrix(dtype)
+
+        # invalid format
+        a = sparse.csc_matrix(a_ref)
+        with pytest.raises(TypeError):
+            cusparse.csrilu02(a, level_info=self.level_info)
+
+        # invalid shape
+        a = cupy.ones((self.n, self.n + 1), dtype=dtype)
+        a = sparse.csr_matrix(a)
+        with pytest.raises(ValueError):
+            cusparse.csrilu02(a, level_info=self.level_info)
+
+        # matrix with zero diagonal element
+        a = a_ref
+        a[-1, -1] = 0
+        a = sparse.csr_matrix(a)
+        with pytest.raises(ValueError):
+            cusparse.csrilu02(a, level_info=self.level_info)
+
+        # singular matrix
+        a = a_ref
+        a[1:] = a[0]
+        a = sparse.csr_matrix(a)
+        with pytest.raises(ValueError):
+            cusparse.csrilu02(a, level_info=self.level_info)
+
+
+def skip_HIP_0_size_matrix():
+    def decorator(impl):
+        @functools.wraps(impl)
+        def test_func(self, *args, **kw):
+            try:
+                impl(self, *args, **kw)
+            except ValueError as e:
+                if runtime.is_hip:
+                    assert 'hipSPARSE' in str(e)
+                    pytest.xfail('may be buggy')
+                raise
+        return test_func
+    return decorator
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(3, 4), (4, 4), (4, 3)],
+    'density': [0.0, 0.5, 1.0],
+    'format': ['csr', 'csc', 'coo']
+}))
+@testing.with_requires('scipy')
+class TestSparseMatrixConversion:
+
+    @skip_HIP_0_size_matrix()
+    @testing.for_dtypes('fdFD')
+    def test_denseToSparse(self, dtype):
+        if not cusparse.check_availability('denseToSparse'):
+            pytest.skip('denseToSparse is not available')
+        x = cupy.random.uniform(0, 1, self.shape).astype(dtype)
+        x[x < self.density] = 0
+        y = cusparse.denseToSparse(x, format=self.format)
+        assert y.format == self.format
+        testing.assert_array_equal(x, y.todense())
+
+    @skip_HIP_0_size_matrix()
+    @testing.for_dtypes('fdFD')
+    def test_sparseToDense(self, dtype):
+        if not cusparse.check_availability('sparseToDense'):
+            pytest.skip('sparseToDense is not available')
+        m, n = self.shape
+        x = scipy.sparse.random(m, n, density=self.density, format=self.format,
+                                dtype=dtype)
+        if self.format == 'csr':
+            x = sparse.csr_matrix(x)
+        elif self.format == 'csc':
+            x = sparse.csc_matrix(x)
+        elif self.format == 'coo':
+            x = sparse.coo_matrix(x)
+        y = cusparse.sparseToDense(x)
+        testing.assert_array_equal(x.todense(), y)
+
+
+@pytest.mark.parametrize('dims', [(3, 4), (4, 3), (3, None)])
+@pytest.mark.parametrize(
+    'dtype', [cupy.float32, cupy.float64, cupy.complex64, cupy.complex128])
+@pytest.mark.parametrize('format', ['csr', 'csc', 'coo'])
+@pytest.mark.parametrize(
+    'transa,lower,unit_diag,b_order',
+    [('N', True, False, 'f'),   # base
+     ('T', True, False, 'f'),   # transa == 'T'
+     ('H', True, False, 'f'),   # transa == 'H'
+     ('N', False, False, 'f'),  # lower == False
+     ('T', False, True, 'f'),   # transa == 'T', lower == False
+     ('H', False, True, 'f'),   # transa == 'H', lower == False
+     ('N', True, True, 'f'),    # unit_diag == True
+     ('N', True, False, 'c'),   # b_order == 'c'
+     ])
+@testing.with_requires('scipy')
+class TestSpsm:
+
+    alpha = 0.5
+
+    def _make_matrix(self, dtype, m, lower, unit_diag, format):
+        # Make a sparse m x m triangular non-singular matrix
+        a = scipy.sparse.random(
+            m, m, density=0.5, format=format, dtype=dtype)
+
+        if unit_diag:
+            diag = numpy.diag(numpy.ones(m).astype(dtype))
+        else:
+            diag = numpy.diag(numpy.random.uniform(0.1, 1, m).astype(dtype))
+        a = a - numpy.diag(a.diagonal()) + diag
+
+        if lower:
+            a = scipy.sparse.tril(a)
+        else:
+            a = scipy.sparse.triu(a)
+
+        if not a.has_canonical_format:
+            a.sum_duplicates()
+
+        return a
+
+    @pytest.fixture(autouse=True)
+    def setUp(self, dims, dtype, lower, unit_diag, transa, format):
+        m, n = dims
+
+        self.op_a = self._make_matrix(dtype, m, lower, unit_diag, format)
+        self.a = self.op_a
+
+        if n is None:
+            b_shape = m,
+        else:
+            b_shape = m, n
+        self.op_b = numpy.random.uniform(-1, 1, b_shape).astype(dtype)
+        self.b = self.op_b
+
+        if format == 'csr':
+            self.sparse_matrix = sparse.csr_matrix
+        elif format == 'csc':
+            self.sparse_matrix = sparse.csc_matrix
+        elif format == 'coo':
+            self.sparse_matrix = sparse.coo_matrix
+        else:
+            assert False
+
+    def test_spsm(self, lower, unit_diag, transa, b_order, dtype, format):
+        if not cusparse.check_availability('spsm'):
+            pytest.skip('spsm is not available')
+        if not runtime.is_hip and _cusparse.get_build_version() < 11701:
+            # eariler than CUDA 11.6
+            if b_order == 'c':
+                pytest.skip("Older CUDA has a bug")
+        if runtime.is_hip:
+            if format == 'coo' or b_order == 'c':
+                pytest.skip('may be buggy or not supported')
+        a = self.sparse_matrix(self.a)
+        b = cupy.array(self.b, order=b_order)
+        c = cusparse.spsm(
+            a, b, alpha=self.alpha, lower=lower, unit_diag=unit_diag,
+            transa=transa)
+
+        if transa == 'N':
+            op_a = self.op_a
+        elif transa == 'T':
+            op_a = self.op_a.T
+        else:
+            op_a = self.op_a.conj().T
+        lhs = op_a.dot(c.get())
+
+        rhs = self.alpha * self.op_b
+
+        if dtype in (cupy.float32, cupy.complex64):
+            tol = 1e-5
+        else:
+            tol = 1e-12
+        testing.assert_allclose(lhs, rhs, rtol=tol, atol=tol)
diff --git a/tests/cupyx_tests/test_cutensor.py b/tests/cupyx_tests/test_cutensor.py
new file mode 100644
index 0000000..60b3dea
--- /dev/null
+++ b/tests/cupyx_tests/test_cutensor.py
@@ -0,0 +1,370 @@
+import gc
+
+import numpy
+import pytest
+
+import cupy
+from cupy._core import _routines_linalg as _linalg
+from cupy import testing
+from cupy.cuda import device
+
+from cupy.cuda import cutensor as ct
+
+if ct.available:
+    from cupyx import cutensor
+
+
+@testing.parameterize(
+    {'dtype': numpy.float16, 'tol': 3e-3},
+    {'dtype': numpy.float32, 'tol': 1e-6},
+    {'dtype': numpy.float64, 'tol': 1e-12},
+    {'dtype': numpy.complex64, 'tol': 1e-6},
+    {'dtype': numpy.complex128, 'tol': 1e-12},
+)
+@pytest.mark.skipif(not ct.available, reason='cuTensor is unavailable')
+class TestCuTensor:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.a = testing.shaped_random(
+            (20, 40, 30), cupy, self.dtype, seed=0)
+        self.b = testing.shaped_random(
+            (40, 30, 20), cupy, self.dtype, seed=1)
+        self.c = testing.shaped_random(
+            (30, 20, 40), cupy, self.dtype, seed=2)
+        self.mode_a = ('y', 'z', 'x')
+        self.mode_b = ('z', 'x', 'y')
+        self.mode_c = ('x', 'y', 'z')
+        self.alpha = 1.1
+        self.beta = 1.2
+        self.gamma = 1.3
+        self.a_transposed = self.a.transpose(2, 0, 1).copy()
+        self.b_transposed = self.b.transpose(1, 2, 0).copy()
+        self.c_transposed = self.c.copy()
+
+    def test_elementwise_trinary(self):
+        desc_a = cutensor.create_tensor_descriptor(self.a)
+        desc_b = cutensor.create_tensor_descriptor(self.b)
+        desc_c = cutensor.create_tensor_descriptor(self.c)
+
+        d = cutensor.elementwise_trinary(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.beta, self.b, desc_b, self.mode_b,
+            self.gamma, self.c, desc_c, self.mode_c
+        )
+
+        assert d.dtype == self.dtype
+
+        testing.assert_allclose(
+            self.alpha * self.a_transposed +
+            self.beta * self.b_transposed +
+            self.gamma * self.c_transposed,
+            d,
+            rtol=self.tol, atol=self.tol
+        )
+
+    def test_elementwise_trinary_out(self):
+        out = testing.shaped_random(
+            (30, 20, 40), cupy, self.dtype, seed=3)
+
+        desc_a = cutensor.create_tensor_descriptor(self.a)
+        desc_b = cutensor.create_tensor_descriptor(self.b)
+        desc_c = cutensor.create_tensor_descriptor(self.c)
+
+        d = cutensor.elementwise_trinary(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.beta, self.b, desc_b, self.mode_b,
+            self.gamma, self.c, desc_c, self.mode_c, out=out
+        )
+
+        assert d is out
+        testing.assert_allclose(
+            self.alpha * self.a_transposed +
+            self.beta * self.b_transposed +
+            self.gamma * self.c,
+            d,
+            rtol=self.tol, atol=self.tol
+        )
+
+    def test_elementwise_binary(self):
+        desc_a = cutensor.create_tensor_descriptor(self.a)
+        desc_c = cutensor.create_tensor_descriptor(self.c)
+
+        d = cutensor.elementwise_binary(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.gamma, self.c, desc_c, self.mode_c
+        )
+
+        assert d.dtype == self.dtype
+
+        testing.assert_allclose(
+            self.alpha * self.a_transposed +
+            self.gamma * self.c_transposed,
+            d,
+            rtol=self.tol, atol=self.tol
+        )
+
+    def test_elementwise_binary_out(self):
+        out = testing.shaped_random(
+            (30, 20, 40), cupy, self.dtype, seed=3)
+        desc_a = cutensor.create_tensor_descriptor(self.a)
+        desc_c = cutensor.create_tensor_descriptor(self.c)
+
+        d = cutensor.elementwise_binary(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.gamma, self.c, desc_c, self.mode_c, out=out
+        )
+
+        assert d is out
+        testing.assert_allclose(
+            self.alpha * self.a_transposed +
+            self.gamma * self.c_transposed,
+            d,
+            rtol=self.tol, atol=self.tol
+        )
+
+    def test_contraction(self):
+        compute_capability = int(device.get_compute_capability())
+        if compute_capability < 70 and self.dtype == numpy.float16:
+            pytest.skip('Not supported.')
+
+        desc_a = cutensor.create_tensor_descriptor(self.a)
+        desc_b = cutensor.create_tensor_descriptor(self.b)
+        desc_c = cutensor.create_tensor_descriptor(self.c)
+
+        d = cutensor.contraction(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.b, desc_b, self.mode_b,
+            self.beta, self.c, desc_c, self.mode_c
+        )
+
+        assert self.c is d
+        testing.assert_allclose(
+            self.alpha * self.a_transposed * self.b_transposed +
+            self.beta * self.c_transposed,
+            d,
+            rtol=self.tol, atol=self.tol
+        )
+
+    def test_reduction(self):
+        if self.dtype == numpy.float16:
+            pytest.skip('Not supported.')
+
+        c = testing.shaped_random((30,), cupy, self.dtype, seed=2)
+        c_orig = c.copy()
+
+        desc_a = cutensor.create_tensor_descriptor(self.a)
+        desc_c = cutensor.create_tensor_descriptor(c)
+
+        d = cutensor.reduction(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.beta, c, desc_c, ('x',)
+        )
+
+        assert c is d
+        testing.assert_allclose(
+            self.alpha * self.a_transposed.sum(axis=(1, 2)) +
+            self.beta * c_orig,
+            d,
+            rtol=self.tol, atol=self.tol
+        )
+
+
+@pytest.mark.skipif(not ct.available, reason='cuTensor is unavailable')
+class TestMode:
+
+    def test_create_mode_int(self):
+        m = cutensor.create_mode(10, 11, 12)
+        assert m.ndim == 3
+        assert repr(m) == 'mode(10, 11, 12)'
+
+    def test_create_mode_ascii(self):
+        m = cutensor.create_mode('x', 'y')
+        assert m.ndim == 2
+        assert repr(m) == 'mode(120, 121)'
+
+    def test_mode_compare(self):
+        m1 = cutensor.create_mode(10, 11, 12)
+        m2 = cutensor.create_mode(10, 11, 12)
+        assert m1 == m2
+        assert m1.data == m2.data  # cached
+
+        m2 = cutensor.create_mode(12, 11, 10)
+        assert m1 != m2
+        assert m1.data != m2.data
+
+
+@pytest.mark.skipif(not ct.available, reason='cuTensor is unavailable')
+class TestScalar:
+
+    def test_create(self):
+        s = cutensor._Scalar(10, cupy.float32)
+        assert repr(s) == 'scalar(10.0, dtype=float32)'
+
+
+@pytest.mark.skipif(not ct.available, reason='cuTensor is unavailable')
+class TestCuTensorDescriptor:
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        self.a = testing.shaped_random(
+            (20, 40, 30), cupy, numpy.float32, seed=0)
+        self.b = testing.shaped_random(
+            (40, 30, 20), cupy, numpy.float32, seed=1)
+        self.c = testing.shaped_random(
+            (30, 20, 40), cupy, numpy.float32, seed=2)
+        self.mode_a = ('y', 'z', 'x')
+        self.mode_b = ('z', 'x', 'y')
+        self.mode_c = ('x', 'y', 'z')
+        self.alpha = 1.1
+        self.beta = 1.2
+        self.gamma = 1.3
+        self.a_transposed = self.a.transpose(2, 0, 1).copy()
+        self.b_transposed = self.b.transpose(1, 2, 0).copy()
+        self.c_transposed = self.c.copy()
+
+    def test_elementwise_trinary(self):
+        desc_a = cutensor.create_tensor_descriptor(self.a, ct.OP_SQRT)
+        desc_b = cutensor.create_tensor_descriptor(self.b, ct.OP_TANH)
+        desc_c = cutensor.create_tensor_descriptor(self.c, ct.OP_COS)
+
+        d = cutensor.elementwise_trinary(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.beta, self.b, desc_b, self.mode_b,
+            self.gamma, self.c, desc_c, self.mode_c,
+            op_AB=ct.OP_ADD, op_ABC=ct.OP_MUL
+        )
+
+        testing.assert_allclose(
+            (self.alpha * cupy.sqrt(self.a_transposed) +
+             self.beta * cupy.tanh(self.b_transposed)) *
+            self.gamma * cupy.cos(self.c),
+            d,
+            rtol=1e-6, atol=1e-6
+        )
+
+    def test_elementwise_binary(self):
+        desc_a = cutensor.create_tensor_descriptor(self.a, ct.OP_SIGMOID)
+        desc_c = cutensor.create_tensor_descriptor(self.c, ct.OP_ABS)
+
+        d = cutensor.elementwise_binary(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.gamma, self.c, desc_c, self.mode_c,
+            op_AC=ct.OP_MUL
+        )
+
+        testing.assert_allclose(
+            self.alpha * (1 / (1 + cupy.exp(-self.a_transposed))) *
+            self.gamma * cupy.abs(self.c),
+            d,
+            rtol=1e-6, atol=1e-6
+        )
+
+    def test_reduction(self):
+        c = testing.shaped_random((30,), cupy, numpy.float32, seed=2)
+        c_orig = c.copy()
+
+        desc_a = cutensor.create_tensor_descriptor(self.a, ct.OP_COS)
+        desc_c = cutensor.create_tensor_descriptor(c, ct.OP_TANH)
+
+        d = cutensor.reduction(
+            self.alpha, self.a, desc_a, self.mode_a,
+            self.beta, c, desc_c, ('x',),
+            reduce_op=ct.OP_MAX
+        )
+
+        assert c is d
+        testing.assert_allclose(
+            self.alpha * cupy.cos(self.a_transposed).max(axis=(1, 2)) +
+            self.beta * cupy.tanh(c_orig),
+            d,
+            rtol=1e-6, atol=1e-6
+        )
+
+
+@testing.parameterize(*testing.product({
+    'dtype_combo': ['eee', 'fff', 'ddd', 'FFF', 'DDD', 'dDD', 'DdD'],
+    'compute_type_hint': [None, 'down-convert', 'TF32'],
+    'shape': [(40, 20, 20)],  # let last two dim be the same for testing cache
+    'alpha': [1.0],
+    'beta': [0.0, 1.0],
+}))
+@pytest.mark.skipif(not ct.available, reason='cuTensor is unavailable')
+class TestCuTensorContraction:
+    _tol = {'e': 1e-3, 'f': 1e-6, 'd': 1e-12}
+
+    def make_random_array(self, shape, dtype):
+        return testing.shaped_random(shape, cupy, dtype=dtype, scale=1)
+
+    def make_matrix(self, shape, dtype):
+        r_dtype = dtype
+        if dtype == numpy.complex64:
+            r_dtype = numpy.float32
+        elif dtype == numpy.complex128:
+            r_dtype = numpy.float64
+        a = self.make_random_array(shape, r_dtype)
+        if dtype.char in 'FD':
+            a = a + 1j * self.make_random_array(shape, r_dtype)
+        return a
+
+    @pytest.fixture(autouse=True)
+    def setUp(self):
+        compute_capability = int(device.get_compute_capability())
+        if compute_capability < 70 and 'e' in self.dtype_combo:
+            pytest.skip("Not supported")
+        dtype_chars = list(self.dtype_combo)
+        self.a_dtype = numpy.dtype(dtype_chars[0])
+        self.b_dtype = numpy.dtype(dtype_chars[1])
+        self.c_dtype = numpy.dtype(dtype_chars[2])
+        self.tol = self._tol[dtype_chars[2].lower()]
+        self.compute_type = _linalg.COMPUTE_TYPE_DEFAULT
+        if self.compute_type_hint == 'down-convert':
+            if self.c_dtype.char in 'fF':
+                self.compute_type = _linalg.COMPUTE_TYPE_FP16
+                self.tol = self._tol['e']
+            elif self.c_dtype.char in 'dD':
+                self.compute_type = _linalg.COMPUTE_TYPE_FP32
+                self.tol = self._tol['f']
+        elif self.compute_type_hint == 'TF32':
+            if self.c_dtype.char in 'fF':
+                self.compute_type = _linalg.COMPUTE_TYPE_TF32
+                self.tol = self._tol['e']
+        m, n, k = self.shape
+        self.a = self.make_matrix((m, k), self.a_dtype)
+        self.b = self.make_matrix((k, n), self.b_dtype)
+        self.c = self.make_matrix((m, n), self.c_dtype)
+        self.c_ref = self.alpha * cupy.matmul(self.a, self.b)
+        self.c_ref += self.beta * self.c
+        old_compute_type = cupy._core.get_compute_type(self.c_dtype)
+        cupy._core.set_compute_type(self.c_dtype, self.compute_type)
+        yield
+        cupy._core.set_compute_type(self.c_dtype, old_compute_type)
+
+    def test_contraction(self):
+        desc_a = cutensor.create_tensor_descriptor(self.a)
+        desc_b = cutensor.create_tensor_descriptor(self.b)
+        desc_c = cutensor.create_tensor_descriptor(self.c)
+        mode_a = cutensor.create_mode('m', 'k')
+        mode_b = cutensor.create_mode('k', 'n')
+        mode_c = cutensor.create_mode('m', 'n')
+        cutensor.contraction(self.alpha,
+                             self.a, desc_a, mode_a,
+                             self.b, desc_b, mode_b,
+                             self.beta,
+                             self.c, desc_c, mode_c)
+        cupy.testing.assert_allclose(self.c, self.c_ref,
+                                     rtol=self.tol, atol=self.tol)
+
+        # test the contraction descriptor cache (issues #7318, #7812)
+        del mode_b
+        gc.collect()
+        mode_b = cutensor.create_mode('n', 'k')  # flipped
+        self.c_ref = self.alpha * cupy.matmul(self.a, self.b.T)
+        self.c_ref += self.beta * self.c
+        cutensor.contraction(self.alpha,
+                             self.a, desc_a, mode_a,
+                             self.b, desc_b, mode_b,
+                             self.beta,
+                             self.c, desc_c, mode_c)
+        cupy.testing.assert_allclose(self.c, self.c_ref,
+                                     rtol=self.tol, atol=self.tol)
diff --git a/tests/cupyx_tests/test_lapack.py b/tests/cupyx_tests/test_lapack.py
new file mode 100644
index 0000000..350dbc0
--- /dev/null
+++ b/tests/cupyx_tests/test_lapack.py
@@ -0,0 +1,167 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+import cupyx
+from cupyx import cusolver, lapack
+
+
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+    'n': [3],
+    'nrhs': [None, 1, 4],
+    'order': ['C', 'F'],
+}))
+class TestGesv(unittest.TestCase):
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    def _make_array(self, shape, alpha, beta):
+        a = testing.shaped_random(shape, cupy, dtype=self.dtype.char.lower(),
+                                  order=self.order, scale=alpha) + beta
+        return a
+
+    def _make_matrix(self, shape, alpha, beta):
+        a = self._make_array(shape, alpha, beta)
+        if self.dtype.char in 'FD':
+            a = a + 1j * self._make_array(shape, alpha, beta)
+        return a
+
+    def setUp(self):
+        self.dtype = numpy.dtype(self.dtype)
+        n = self.n
+        nrhs = 1 if self.nrhs is None else self.nrhs
+        # Diagonally dominant matrix is used as it is stable
+        alpha = 2.0 / n
+        a = self._make_matrix((n, n), alpha, -alpha / 2)
+        diag = cupy.diag(cupy.ones((n,), dtype=self.dtype.char.lower()))
+        a[diag > 0] = 0
+        a += diag
+        x = self._make_matrix((n, nrhs), 0.2, 0.9)
+        b = cupy.matmul(a, x)
+        b_shape = [n]
+        if self.nrhs is not None:
+            b_shape.append(nrhs)
+        b = b.reshape(b_shape)
+        self.a = a
+        if self.nrhs is None or self.nrhs == 1:
+            self.b = b.copy(order=self.order)
+        else:
+            self.b = b.copy(order='F')
+        self.x_ref = x.reshape(b_shape)
+        self.tol = self._tol[self.dtype.char.lower()]
+
+    def test_gesv(self):
+        lapack.gesv(self.a, self.b)
+        cupy.testing.assert_allclose(self.b, self.x_ref,
+                                     rtol=self.tol, atol=self.tol)
+
+    def test_invalid_cases(self):
+        if self.nrhs is None or self.nrhs == 1:
+            raise unittest.SkipTest()
+        ng_a = self.a.reshape(1, self.n, self.n)
+        with pytest.raises(ValueError):
+            lapack.gesv(ng_a, self.b)
+        ng_b = self.b.reshape(1, self.n, self.nrhs)
+        with pytest.raises(ValueError):
+            lapack.gesv(self.a, ng_b)
+        ng_a = cupy.ones((self.n, self.n+1), dtype=self.dtype)
+        with pytest.raises(ValueError):
+            lapack.gesv(ng_a, self.b)
+        ng_a = cupy.ones((self.n+1, self.n+1), dtype=self.dtype)
+        with pytest.raises(ValueError):
+            lapack.gesv(ng_a, self.b)
+        ng_a = cupy.ones(self.a.shape, dtype='i')
+        with pytest.raises(TypeError):
+            lapack.gesv(ng_a, self.b)
+        ng_a = cupy.ones((2, self.n, self.n), dtype=self.dtype, order='F')[0]
+        with pytest.raises(ValueError):
+            lapack.gesv(ng_a, self.b)
+        ng_b = self.b.copy(order='C')
+        with pytest.raises(ValueError):
+            lapack.gesv(self.a, ng_b)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(4, 4), (5, 4), (4, 5)],
+    'nrhs': [None, 1, 4],
+}))
+class TestGels(unittest.TestCase):
+    _tol = {'f': 1e-5, 'd': 1e-12}
+
+    @testing.for_dtypes('fdFD')
+    def test_gels(self, dtype):
+        b_shape = [self.shape[0]]
+        if self.nrhs is not None:
+            b_shape.append(self.nrhs)
+        a = testing.shaped_random(self.shape, numpy, dtype=dtype)
+        b = testing.shaped_random(b_shape, numpy, dtype=dtype)
+        tol = self._tol[numpy.dtype(dtype).char.lower()]
+        x_lstsq = numpy.linalg.lstsq(a, b, rcond=None)[0]
+        x_gels = lapack.gels(cupy.array(a), cupy.array(b))
+        cupy.testing.assert_allclose(x_gels, x_lstsq, rtol=tol, atol=tol)
+
+
+@testing.parameterize(*testing.product({
+    'shape': [(3, 4, 2, 2), (5, 3, 3), (7, 7)],
+    'nrhs': [None, 1, 4]
+}))
+class TestPosv(unittest.TestCase):
+
+    def setUp(self):
+        if not cusolver.check_availability('potrsBatched'):
+            pytest.skip('potrsBatched is not available')
+
+    @testing.for_dtypes('fdFD')
+    @testing.numpy_cupy_allclose(atol=1e-5)
+    def test_posv(self, xp, dtype):
+        # TODO: cusolver does not support nrhs > 1 for potrsBatched
+        if len(self.shape) > 2 and self.nrhs and self.nrhs > 1:
+            pytest.skip('cusolver does not support nrhs > 1 for potrsBatched')
+
+        a = self._create_posdef_matrix(xp, self.shape, dtype)
+        b_shape = list(self.shape[:-1])
+        if self.nrhs is not None:
+            b_shape.append(self.nrhs)
+        b = testing.shaped_random(b_shape, xp, dtype=dtype)
+
+        if xp == cupy:
+            return lapack.posv(a, b)
+        else:
+            return numpy.linalg.solve(a, b)
+
+    def _create_posdef_matrix(self, xp, shape, dtype):
+        n = shape[-1]
+        a = testing.shaped_random(shape, xp, dtype, scale=1)
+        a = a @ a.swapaxes(-2, -1).conjugate()
+        a = a + n * xp.eye(n)
+        return a
+
+
+# TODO: cusolver does not support nrhs > 1 for potrsBatched
+@testing.parameterize(*testing.product({
+    'shape': [(2, 3, 3)],
+    'dtype': [numpy.float32, numpy.float64, numpy.complex64, numpy.complex128],
+}))
+class TestXFailBatchedPosv(unittest.TestCase):
+
+    def test_posv(self):
+        if not cusolver.check_availability('potrsBatched'):
+            pytest.skip('potrsBatched is not available')
+        a = self._create_posdef_matrix(cupy, self.shape, self.dtype)
+        n = a.shape[-1]
+        identity_matrix = cupy.eye(n, dtype=a.dtype)
+        b = cupy.empty(a.shape, a.dtype)
+        b[...] = identity_matrix
+        with cupyx.errstate(linalg='ignore'):
+            with pytest.raises(cupy.cuda.cusolver.CUSOLVERError):
+                lapack.posv(a, b)
+
+    def _create_posdef_matrix(self, xp, shape, dtype):
+        n = shape[-1]
+        a = testing.shaped_random(shape, xp, dtype, scale=1)
+        a = a @ a.swapaxes(-2, -1).conjugate()
+        a = a + n * xp.eye(n)
+        return a
diff --git a/tests/cupyx_tests/test_optimize.py b/tests/cupyx_tests/test_optimize.py
new file mode 100644
index 0000000..aa5e308
--- /dev/null
+++ b/tests/cupyx_tests/test_optimize.py
@@ -0,0 +1,212 @@
+import os
+import tempfile
+import unittest
+from unittest import mock
+
+import pytest
+
+import cupy
+from cupy import testing
+from cupy._core import _accelerator
+
+
+try:
+    import warnings
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', category=DeprecationWarning)
+        import cupyx.optimizing
+        import cupyx.optimizing._optimize
+        import cupy._core._optimize_config
+except ImportError:
+    pass
+
+
+@testing.with_requires('optuna')
+class TestOptimize(unittest.TestCase):
+
+    def setUp(self):
+        cupy._core._optimize_config._clear_all_contexts_cache()
+
+    def test_optimize_reduction_kernel(self):
+        my_sum = cupy.ReductionKernel(
+            'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum')
+        x = testing.shaped_arange((3, 4), cupy)
+        y1 = my_sum(x, axis=1)
+        with cupyx.optimizing.optimize():
+            y2 = my_sum(x, axis=1)
+        testing.assert_array_equal(y1, y2)
+
+    def test_optimize_cache(self):
+        if (_accelerator.ACCELERATOR_CUB
+                in _accelerator.get_reduction_accelerators()):
+            pytest.skip('optimize cannot be mocked for CUB reduction')
+
+        target = cupyx.optimizing._optimize._optimize
+        target_full_name = '{}.{}'.format(target.__module__, target.__name__)
+
+        with mock.patch(target_full_name) as optimize_impl:
+            my_sum = cupy.ReductionKernel(
+                'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum')
+            my_sum_ = cupy.ReductionKernel(
+                'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum_')
+            x = testing.shaped_arange((3, 4), cupy)
+            x_ = testing.shaped_arange((3, 4), cupy)
+            y = testing.shaped_arange((4, 4), cupy)
+            z = testing.shaped_arange((3, 4), cupy)[::-1]
+            assert x.strides == y.strides
+            assert x.shape == z.shape
+
+            with cupyx.optimizing.optimize():
+                my_sum(x, axis=1)
+                assert optimize_impl.call_count == 1
+                my_sum(x, axis=1)
+                assert optimize_impl.call_count == 1
+                my_sum(x, axis=0)
+                assert optimize_impl.call_count == 2
+                my_sum(x_, axis=1)
+                assert optimize_impl.call_count == 2
+                my_sum(y, axis=1)
+                assert optimize_impl.call_count == 3
+                my_sum(z, axis=1)
+                assert optimize_impl.call_count == 4
+                my_sum_(x, axis=1)
+                assert optimize_impl.call_count == 5
+
+            with cupyx.optimizing.optimize(key='new_key'):
+                my_sum(x, axis=1)
+                assert optimize_impl.call_count == 6
+
+            with cupyx.optimizing.optimize(key=None):
+                my_sum(x, axis=1)
+                assert optimize_impl.call_count == 6
+                my_sum(x)
+                assert optimize_impl.call_count == 7
+
+    @testing.multi_gpu(2)
+    def test_optimize_cache_multi_gpus(self):
+        if (_accelerator.ACCELERATOR_CUB
+                in _accelerator.get_reduction_accelerators()):
+            pytest.skip('optimize cannot be mocked for CUB reduction')
+
+        target = cupyx.optimizing._optimize._optimize
+        target_full_name = '{}.{}'.format(target.__module__, target.__name__)
+
+        with mock.patch(target_full_name) as optimize_impl:
+            my_sum = cupy.ReductionKernel(
+                'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum')
+
+            with cupyx.optimizing.optimize():
+                with cupy.cuda.Device(0):
+                    x = testing.shaped_arange((3, 4), cupy)
+                    my_sum(x, axis=1)
+                    assert optimize_impl.call_count == 1
+
+                with cupy.cuda.Device(1):
+                    x = testing.shaped_arange((3, 4), cupy)
+                    my_sum(x, axis=1)
+                    assert optimize_impl.call_count == 2
+
+    def test_optimize_pickle(self):
+        my_sum = cupy.ReductionKernel(
+            'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum')
+        x = testing.shaped_arange((3, 4), cupy)
+
+        with tempfile.TemporaryDirectory() as directory:
+            filepath = directory + '/optimize_params'
+
+            with cupyx.optimizing.optimize() as context:
+                my_sum(x, axis=1)
+                params_map = context._params_map
+                context.save(filepath)
+
+            cupy._core._optimize_config._clear_all_contexts_cache()
+
+            with cupyx.optimizing.optimize() as context:
+                assert params_map.keys() != context._params_map.keys()
+                context.load(filepath)
+                assert params_map.keys() == context._params_map.keys()
+
+            with cupyx.optimizing.optimize(key='other_key') as context:
+                with pytest.raises(ValueError):
+                    context.load(filepath)
+
+    def test_optimize_autosave(self):
+        with tempfile.TemporaryDirectory() as directory:
+            filepath = directory + '/optimize_params'
+
+            # non-existing file, readonly=True
+            with testing.assert_warns(UserWarning):
+                with cupyx.optimizing.optimize(path=filepath, readonly=True):
+                    cupy.sum(cupy.arange(2))
+
+            # non-existing file, readonly=False
+            with cupyx.optimizing.optimize(path=filepath, readonly=False):
+                cupy.sum(cupy.arange(4))
+            filesize = os.stat(filepath).st_size
+            assert 0 < filesize
+
+            # existing file, readonly=True
+            with cupyx.optimizing.optimize(path=filepath, readonly=True):
+                cupy.sum(cupy.arange(6))
+            assert filesize == os.stat(filepath).st_size
+
+            # existing file, readonly=False
+            with cupyx.optimizing.optimize(path=filepath, readonly=False):
+                cupy.sum(cupy.arange(8))
+            assert filesize <= os.stat(filepath).st_size
+
+
+# TODO(leofang): check the optimizer is not applicable to the cutensor backend?
+@testing.parameterize(*testing.product({
+    'backend': ([], ['cub'])
+}))
+@testing.with_requires('optuna')
+class TestOptimizeBackends(unittest.TestCase):
+    """This class tests if optuna is in effect for create_reduction_func()"""
+
+    def setUp(self):
+        cupy._core._optimize_config._clear_all_contexts_cache()
+        self.old_reductions = _accelerator.get_reduction_accelerators()
+        _accelerator.set_reduction_accelerators(self.backend)
+
+        # avoid shadowed by the cub module
+        self.old_routines = _accelerator.get_routine_accelerators()
+        _accelerator.set_routine_accelerators([])
+
+        self.x = testing.shaped_arange((3, 4), cupy, dtype=cupy.float32)
+
+    def tearDown(self):
+        _accelerator.set_routine_accelerators(self.old_routines)
+        _accelerator.set_reduction_accelerators(self.old_reductions)
+
+    def test_optimize1(self):
+        # Ensure the optimizer is run 3 times for all backends.
+        func = 'cupyx.optimizing._optimize._optimize'
+        times_called = 3
+
+        # Setting "wraps" is necessary to avoid compilation errors.
+        with testing.AssertFunctionIsCalled(
+                func, times_called=times_called,
+                wraps=cupyx.optimizing._optimize._optimize):
+            with cupyx.optimizing.optimize():
+                self.x.sum()
+            with cupyx.optimizing.optimize():
+                self.x.sum(axis=1)
+            with cupyx.optimizing.optimize():
+                self.x.sum(axis=0)  # CUB falls back to the simple reduction
+
+    def test_optimize2(self):
+        # Ensure the CUB optimizer is not run when the CUB kernel is not used.
+        func = 'cupy._core._cub_reduction._get_cub_optimized_params'
+        times_called = 2 if ('cub' in self.backend) else 0
+
+        # Setting "wraps" is necessary to avoid errors being silently ignored.
+        with testing.AssertFunctionIsCalled(
+                func, times_called=times_called,
+                wraps=cupy._core._cub_reduction._get_cub_optimized_params):
+            with cupyx.optimizing.optimize():
+                self.x.sum()
+            with cupyx.optimizing.optimize():
+                self.x.sum(axis=1)
+            with cupyx.optimizing.optimize():
+                self.x.sum(axis=0)  # CUB optimizer not used
diff --git a/tests/cupyx_tests/test_pinned_array.py b/tests/cupyx_tests/test_pinned_array.py
new file mode 100644
index 0000000..4e8e402
--- /dev/null
+++ b/tests/cupyx_tests/test_pinned_array.py
@@ -0,0 +1,422 @@
+import unittest
+
+import numpy
+import pytest
+
+import cupy
+from cupy import testing
+from cupy.testing._loops import _wraps_partial
+import cupyx
+
+
+def numpy_cupyx_array_equal(target_func, name='func'):
+    _mod = (cupy, numpy)
+
+    _numpy_funcs = {
+        'empty': numpy.empty,
+        'empty_like': numpy.empty_like,
+        'zeros': numpy.zeros,
+        'zeros_like': numpy.zeros_like,
+    }
+
+    _cupy_funcs = {
+        'empty': cupyx.empty_pinned,
+        'empty_like': cupyx.empty_like_pinned,
+        'zeros': cupyx.zeros_pinned,
+        'zeros_like': cupyx.zeros_like_pinned,
+    }
+
+    def _get_test_func(xp, func):
+        if xp is numpy:
+            return _numpy_funcs[func]
+        elif xp is cupy:
+            return _cupy_funcs[func]
+        else:
+            assert False
+
+    def _check_pinned_mem_used(a, xp):
+        if xp is cupy:
+            assert isinstance(a.base, cupy.cuda.PinnedMemoryPointer)
+            assert a.base.ptr == a.ctypes.data
+
+    def decorator(impl):
+        @_wraps_partial(impl, name)
+        def test_func(self, *args, **kw):
+            out = []
+            for xp in _mod:
+                func = _get_test_func(xp, target_func)
+                kw[name] = func
+                a = impl(self, *args, **kw)
+                _check_pinned_mem_used(a, xp)
+                out.append(a)
+            numpy.testing.assert_array_equal(*out)
+        return test_func
+    return decorator
+
+
+# test_empty_scalar_none is removed
+# test_zeros_scalar_none is removed
+class TestBasic(unittest.TestCase):
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty')
+    def test_empty(self, dtype, order, func):
+        a = func((2, 3, 4), dtype=dtype, order=order)
+        a.fill(0)
+        return a
+
+    @testing.slow
+    def test_empty_huge_size(self):
+        a = cupyx.empty_pinned((1024, 2048, 1024), dtype='b')
+        a.fill(123)
+        assert (a == 123).all()
+        # Free huge memory for slow test
+        del a
+        cupy.get_default_pinned_memory_pool().free_all_blocks()
+
+    @testing.slow
+    def test_empty_huge_size_fill0(self):
+        a = cupyx.empty_pinned((1024, 2048, 1024), dtype='b')
+        a.fill(0)
+        assert (a == 0).all()
+        # Free huge memory for slow test
+        del a
+        cupy.get_default_pinned_memory_pool().free_all_blocks()
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty')
+    def test_empty_scalar(self, dtype, order, func):
+        a = func((), dtype=dtype, order=order)
+        a.fill(0)
+        return a
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty')
+    def test_empty_int(self, dtype, order, func):
+        a = func(3, dtype=dtype, order=order)
+        a.fill(0)
+        return a
+
+    @testing.slow
+    def test_empty_int_huge_size(self):
+        a = cupyx.empty_pinned(2 ** 31, dtype='b')
+        a.fill(123)
+        assert (a == 123).all()
+        # Free huge memory for slow test
+        del a
+        cupy.get_default_pinned_memory_pool().free_all_blocks()
+
+    @testing.slow
+    def test_empty_int_huge_size_fill0(self):
+        a = cupyx.empty_pinned(2 ** 31, dtype='b')
+        a.fill(0)
+        assert (a == 0).all()
+        # Free huge memory for slow test
+        del a
+        cupy.get_default_pinned_memory_pool().free_all_blocks()
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty_like')
+    def test_empty_like(self, dtype, order, func):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        b = func(a, order=order)
+        b.fill(0)
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty_like')
+    def test_empty_like_contiguity(self, dtype, order, func):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        b = func(a, order=order)
+        b.fill(0)
+        if order in ['f', 'F']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty_like')
+    def test_empty_like_contiguity2(self, dtype, order, func):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        a = numpy.asfortranarray(a)
+        b = func(a, order=order)
+        b.fill(0)
+        if order in ['c', 'C']:
+            assert b.flags.c_contiguous
+        else:
+            assert b.flags.f_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty_like')
+    def test_empty_like_contiguity3(self, dtype, order, func):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        # test strides that are both non-contiguous and non-descending
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = func(a, order=order)
+        b.fill(0)
+        if order in ['k', 'K', None]:
+            assert not b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        elif order in ['f', 'F']:
+            assert not b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        return b
+
+    @testing.for_all_dtypes()
+    def test_empty_like_K_strides(self, dtype):
+        # test strides that are both non-contiguous and non-descending;
+        # also test accepting cupy.ndarray
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = numpy.empty_like(a, order='K')
+        b.fill(0)
+
+        # GPU case
+        ag = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        ag = ag[:, ::2, :].swapaxes(0, 1)
+        bg = cupyx.empty_like_pinned(ag, order='K')
+        bg.fill(0)
+
+        # make sure NumPy and CuPy strides agree
+        assert b.strides == bg.strides
+
+    @testing.with_requires('numpy>=1.19')
+    @testing.for_all_dtypes()
+    def test_empty_like_invalid_order(self, dtype):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        with pytest.raises(ValueError):
+            cupyx.empty_like_pinned(a, order='Q')
+
+    def test_empty_like_subok(self):
+        a = testing.shaped_arange((2, 3, 4), numpy)
+        with pytest.raises(TypeError):
+            cupyx.empty_like_pinned(a, subok=True)
+
+    @testing.for_CF_orders()
+    def test_empty_zero_sized_array_strides(self, order):
+        a = numpy.empty((1, 0, 2), dtype='d', order=order)
+        b = cupyx.empty_pinned((1, 0, 2), dtype='d', order=order)
+        assert b.strides == a.strides
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='zeros')
+    def test_zeros(self, dtype, order, func):
+        return func((2, 3, 4), dtype=dtype, order=order)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='zeros')
+    def test_zeros_scalar(self, dtype, order, func):
+        return func((), dtype=dtype, order=order)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='zeros')
+    def test_zeros_int(self, dtype, order, func):
+        return func(3, dtype=dtype, order=order)
+
+    @testing.for_CF_orders()
+    def test_zeros_strides(self, order):
+        a = numpy.zeros((2, 3), dtype='d', order=order)
+        b = cupyx.zeros_pinned((2, 3), dtype='d', order=order)
+        assert b.strides == a.strides
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='zeros_like')
+    def test_zeros_like(self, dtype, order, func):
+        a = numpy.ndarray((2, 3, 4), dtype=dtype)
+        return func(a, order=order)
+
+    def test_zeros_like_subok(self):
+        a = numpy.ndarray((2, 3, 4))
+        with pytest.raises(TypeError):
+            cupyx.zeros_like_pinned(a, subok=True)
+
+
+@testing.parameterize(
+    *testing.product({
+        'shape': [4, (4, ), (4, 2), (4, 2, 3), (5, 4, 2, 3)],
+    })
+)
+class TestBasicReshape(unittest.TestCase):
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty_like')
+    def test_empty_like_reshape(self, dtype, order, func):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        b = func(a, order=order, shape=self.shape)
+        b.fill(0)
+        return b
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    def test_empty_like_reshape_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupyx.empty_like_pinned(a, shape=self.shape)
+        b.fill(0)
+        c = cupyx.empty_pinned(self.shape, order=order, dtype=dtype)
+        c.fill(0)
+        numpy.testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty_like')
+    def test_empty_like_reshape_contiguity(self, dtype, order, func):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        b = func(a, order=order, shape=self.shape)
+        b.fill(0)
+        if order in ['f', 'F']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    def test_empty_like_reshape_contiguity_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupyx.empty_like_pinned(a, order=order, shape=self.shape)
+        b.fill(0)
+        c = cupyx.empty_pinned(self.shape)
+        c.fill(0)
+        if order in ['f', 'F']:
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+        numpy.testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty_like')
+    def test_empty_like_reshape_contiguity2(self, dtype, order, func):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        a = numpy.asfortranarray(a)
+        b = func(a, order=order, shape=self.shape)
+        b.fill(0)
+        shape = self.shape if not numpy.isscalar(self.shape) else (self.shape,)
+        if (order in ['c', 'C'] or
+                (order in ['k', 'K', None] and len(shape) != a.ndim)):
+            assert b.flags.c_contiguous
+        else:
+            assert b.flags.f_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    def test_empty_like_reshape_contiguity2_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        a = cupy.asfortranarray(a)
+        b = cupyx.empty_like_pinned(a, order=order, shape=self.shape)
+        b.fill(0)
+        c = cupyx.empty_pinned(self.shape)
+        c.fill(0)
+        shape = self.shape if not numpy.isscalar(self.shape) else (self.shape,)
+        if (order in ['c', 'C'] or
+                (order in ['k', 'K', None] and len(shape) != a.ndim)):
+            assert b.flags.c_contiguous
+        else:
+            assert b.flags.f_contiguous
+        numpy.testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='empty_like')
+    def test_empty_like_reshape_contiguity3(self, dtype, order, func):
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        # test strides that are both non-contiguous and non-descending
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = func(a, order=order, shape=self.shape)
+        b.fill(0)
+        shape = self.shape if not numpy.isscalar(self.shape) else (self.shape,)
+        if len(shape) == 1:
+            assert b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        elif order in ['k', 'K', None] and len(shape) == a.ndim:
+            assert not b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        elif order in ['f', 'F']:
+            assert not b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        return b
+
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    def test_empty_like_reshape_contiguity3_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        # test strides that are both non-contiguous and non-descending
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = cupyx.empty_like_pinned(a, order=order, shape=self.shape)
+        b.fill(0)
+        shape = self.shape if not numpy.isscalar(self.shape) else (self.shape,)
+        if len(shape) == 1:
+            assert b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        elif order in ['k', 'K', None] and len(shape) == a.ndim:
+            assert not b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+        elif order in ['f', 'F']:
+            assert not b.flags.c_contiguous
+            assert b.flags.f_contiguous
+        else:
+            assert b.flags.c_contiguous
+            assert not b.flags.f_contiguous
+
+        c = cupyx.zeros_pinned(self.shape)
+        c.fill(0)
+        testing.assert_array_equal(b, c)
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_all_dtypes()
+    def test_empty_like_K_strides_reshape(self, dtype):
+        # test strides that are both non-contiguous and non-descending
+        a = testing.shaped_arange((2, 3, 4), numpy, dtype)
+        a = a[:, ::2, :].swapaxes(0, 1)
+        b = cupyx.empty_like_pinned(a, order='K', shape=self.shape)
+        b.fill(0)
+
+        # GPU case
+        ag = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        ag = ag[:, ::2, :].swapaxes(0, 1)
+        bg = cupyx.empty_like_pinned(ag, order='K', shape=self.shape)
+        bg.fill(0)
+
+        # make sure NumPy and CuPy strides agree
+        assert b.strides == bg.strides
+        return
+
+    @testing.with_requires('numpy>=1.17.0')
+    @testing.for_orders('CFAK')
+    @testing.for_all_dtypes()
+    @numpy_cupyx_array_equal(target_func='zeros_like')
+    def test_zeros_like_reshape(self, dtype, order, func):
+        a = numpy.ndarray((2, 3, 4), dtype=dtype)
+        return func(a, order=order, shape=self.shape)
+
+    @testing.for_CF_orders()
+    @testing.for_all_dtypes()
+    def test_zeros_like_reshape_cupy_only(self, dtype, order):
+        a = testing.shaped_arange((2, 3, 4), cupy, dtype)
+        b = cupyx.zeros_like_pinned(a, shape=self.shape)
+        c = cupyx.zeros_pinned(self.shape, order=order, dtype=dtype)
+        numpy.testing.assert_array_equal(b, c)
diff --git a/tests/cupyx_tests/test_rsqrt.py b/tests/cupyx_tests/test_rsqrt.py
new file mode 100644
index 0000000..87c86e0
--- /dev/null
+++ b/tests/cupyx_tests/test_rsqrt.py
@@ -0,0 +1,18 @@
+import unittest
+
+import numpy
+
+import cupy
+from cupy import testing
+import cupyx
+
+
+class TestRsqrt(unittest.TestCase):
+
+    @testing.for_all_dtypes(no_complex=True)
+    def test_rsqrt(self, dtype):
+        # Adding 1.0 to avoid division by zero.
+        a = testing.shaped_arange((2, 3), numpy, dtype) + 1.0
+        out = cupyx.rsqrt(cupy.array(a))
+        # numpy.sqrt is broken in numpy<1.11.2
+        testing.assert_allclose(out, 1.0 / numpy.sqrt(a))
diff --git a/tests/cupyx_tests/test_runtime.py b/tests/cupyx_tests/test_runtime.py
new file mode 100644
index 0000000..1829d84
--- /dev/null
+++ b/tests/cupyx_tests/test_runtime.py
@@ -0,0 +1,46 @@
+import unittest
+from unittest import mock
+
+import cupy
+import cupyx
+
+
+from cupy.cuda import cudnn
+
+
+def _get_error_func(error, *args, **kwargs):
+    def raise_error(*_args, **_kwargs):
+        raise error(*args, **kwargs)
+    return raise_error
+
+
+class TestRuntime(unittest.TestCase):
+    def test_runtime(self):
+        runtime = cupyx.get_runtime_info()
+        assert cupy.__version__ == runtime.cupy_version
+        assert cupy.__version__ in str(runtime)
+
+    @unittest.skipUnless(cudnn.available, 'cuDNN is required')
+    def test_error(self):
+        runtime = cupyx.get_runtime_info()
+        assert 'Error' not in str(runtime)
+
+        with mock.patch(
+                'cupy_backends.cuda.api.runtime.driverGetVersion',
+                side_effect=_get_error_func(
+                    cupy.cuda.runtime.CUDARuntimeError, 0)):
+            runtime = cupyx.get_runtime_info()
+            assert 'CUDARuntimeError' in str(runtime)
+
+        with mock.patch(
+                'cupy_backends.cuda.api.runtime.runtimeGetVersion',
+                side_effect=_get_error_func(
+                    cupy.cuda.runtime.CUDARuntimeError, 0)):
+            runtime = cupyx.get_runtime_info()
+            assert 'CUDARuntimeError' in str(runtime)
+
+        with mock.patch(
+                'cupy_backends.cuda.libs.cudnn.getVersion',
+                side_effect=_get_error_func(cudnn.CuDNNError, 0)):
+            runtime = cupyx.get_runtime_info()
+            assert 'CuDNNError' in str(runtime)
diff --git a/tests/cupyx_tests/test_time.py b/tests/cupyx_tests/test_time.py
new file mode 100644
index 0000000..cb7d78d
--- /dev/null
+++ b/tests/cupyx_tests/test_time.py
@@ -0,0 +1,131 @@
+import unittest
+from unittest import mock
+
+import numpy
+
+import cupy
+from cupy import testing
+import cupyx
+
+
+class TestRepeat(unittest.TestCase):
+
+    def test_cpu_routine(self):
+        with mock.patch('time.perf_counter',
+                        mock.Mock(side_effect=[2.4, 3.8, 3.8] * 10)):
+            with mock.patch('cupy.cuda.get_elapsed_time',
+                            mock.Mock(return_value=2500)):
+                mock_func = mock.Mock()
+                mock_func.__name__ = 'test_name_xxx'
+                x = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                y = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                assert mock_func.call_count == 0
+
+                perf = cupyx.time.repeat(
+                    mock_func, (x, y), n_repeat=10, n_warmup=3)
+
+                assert perf.name == 'test_name_xxx'
+                assert mock_func.call_count == 13
+                assert perf.cpu_times.shape == (10,)
+                assert perf.gpu_times.shape == (1, 10,)
+                assert (perf.cpu_times == 1.4).all()
+                assert (perf.gpu_times == 2.5).all()
+
+    @testing.multi_gpu(2)
+    def test_multigpu_routine(self):
+        with mock.patch('time.perf_counter',
+                        mock.Mock(side_effect=[2.4, 3.8, 3.8] * 10)):
+            with mock.patch('cupy.cuda.get_elapsed_time',
+                            mock.Mock(return_value=2500)):
+                mock_func = mock.Mock()
+                mock_func.__name__ = 'test_name_xxx'
+                x = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                y = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                assert mock_func.call_count == 0
+
+                perf = cupyx.time.repeat(
+                    mock_func, (x, y), n_repeat=10, n_warmup=3, devices=(0, 1))
+
+                assert perf.name == 'test_name_xxx'
+                assert mock_func.call_count == 13
+                assert perf.cpu_times.shape == (10,)
+                assert perf.gpu_times.shape == (2, 10,)
+                assert (perf.cpu_times == 1.4).all()
+                assert (perf.gpu_times == 2.5).all()
+
+    def test_repeat_max_duration(self):
+        with mock.patch('time.perf_counter',
+                        mock.Mock(side_effect=[1., 2., 2.] * 6)):
+            with mock.patch('cupy.cuda.get_elapsed_time',
+                            mock.Mock(return_value=2500)):
+                mock_func = mock.Mock()
+                mock_func.__name__ = 'test_name_xxx'
+                x = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                y = cupy.testing.shaped_random((2, 3), cupy, 'int32')
+                assert mock_func.call_count == 0
+
+                perf = cupyx.time.repeat(
+                    mock_func, (x, y), n_warmup=3, max_duration=2.5)
+
+                assert perf.name == 'test_name_xxx'
+                assert mock_func.call_count == 6
+                assert perf.cpu_times.shape == (3,)
+                assert perf.gpu_times.shape == (1, 3)
+                assert (perf.cpu_times == 1.).all()
+                assert (perf.gpu_times == 2.5).all()
+
+    def test_repeat_kwargs(self):
+        x = cupy.random.rand(5)
+        cupyx.time.repeat(
+            cupy.nonzero, kwargs={'a': x}, n_repeat=1, n_warmup=1)
+
+
+class TestPerfCaseResult(unittest.TestCase):
+    def test_show_gpu(self):
+        times = numpy.array([
+            [5.4, 7.1, 6.0, 5.4, 4.2],
+            [6.4, 4.3, 8.9, 9.6, 3.8],
+        ]) * 1e-6
+        perf = cupyx.time._PerfCaseResult('test_name_xxx', times, (0,))
+        expected = (
+            'test_name_xxx       :'
+            '    CPU:     5.620 us   +/-  0.943 '
+            '(min:     4.200 / max:     7.100) us '
+            '    GPU-0:     6.600 us   +/-  2.344 '
+            '(min:     3.800 / max:     9.600) us'
+        )
+        assert str(perf) == expected
+
+    def test_no_show_gpu(self):
+        times = numpy.array([
+            [5.4, 7.1, 6.0, 5.4, 4.2],
+            [6.4, 4.3, 8.9, 9.6, 3.8],
+        ]) * 1e-6
+        perf = cupyx.time._PerfCaseResult('test_name_xxx', times, (0,))
+        expected = (
+            'test_name_xxx       :'
+            '    CPU:     5.620 us   +/-  0.943 '
+            '(min:     4.200 / max:     7.100) us'
+        )
+        assert perf.to_str() == expected
+        # Checks if the result does not change.
+        assert perf.to_str() == expected
+
+    def test_single_show_gpu(self):
+        times = numpy.array([[5.4], [6.4]]) * 1e-6
+        perf = cupyx.time._PerfCaseResult('test_name_xxx', times, (0,))
+        assert str(perf) == ('test_name_xxx       :    CPU:     5.400 us '
+                             '    GPU-0:     6.400 us')
+
+    def test_single_no_show_gpu(self):
+        times = numpy.array([[5.4], [6.4]]) * 1e-6
+        perf = cupyx.time._PerfCaseResult('test_name_xxx', times, (0,))
+        assert perf.to_str() == 'test_name_xxx       :    CPU:     5.400 us'
+
+    def test_show_multigpu(self):
+        times = numpy.array([[5.4], [6.4], [7.0], [8.1]]) * 1e-6
+        perf = cupyx.time._PerfCaseResult('test_name_xxx', times, (0, 1, 2))
+        assert str(perf) == ('test_name_xxx       :    CPU:     5.400 us '
+                             '    GPU-0:     6.400 us '
+                             '    GPU-1:     7.000 us '
+                             '    GPU-2:     8.100 us')
diff --git a/tests/cupyx_tests/tools_tests/__init__.py b/tests/cupyx_tests/tools_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/cupyx_tests/tools_tests/test_install_library.py b/tests/cupyx_tests/tools_tests/test_install_library.py
new file mode 100644
index 0000000..0d3b4a7
--- /dev/null
+++ b/tests/cupyx_tests/tools_tests/test_install_library.py
@@ -0,0 +1,78 @@
+import os
+import platform
+import tempfile
+import urllib
+
+from cupy import testing
+from cupyx.tools import install_library
+
+import pytest
+
+
+_libraries = ['cudnn', 'nccl', 'cutensor']
+
+
+def _get_supported_cuda_versions(lib):
+    return sorted(set([
+        rec['cuda'] for rec in install_library.library_records[lib]]))
+
+
+class TestInstallLibrary:
+
+    @pytest.mark.parametrize('cuda', _get_supported_cuda_versions('cudnn'))
+    @testing.slow
+    def test_install_cudnn(self, cuda):
+        self._test_install('cudnn', cuda)
+
+    @pytest.mark.skipif(
+        platform.system() == 'Windows',
+        reason='NCCL is only available for Linux')
+    @pytest.mark.parametrize('cuda', _get_supported_cuda_versions('nccl'))
+    @testing.slow
+    def test_install_nccl(self, cuda):
+        self._test_install('nccl', cuda)
+
+    @pytest.mark.parametrize('cuda', _get_supported_cuda_versions('cutensor'))
+    @testing.slow
+    def test_install_cutensor(self, cuda):
+        self._test_install('cutensor', cuda)
+
+    def _test_install(self, library, cuda):
+        system = platform.system()
+        for rec in install_library.library_records[library]:
+            if rec['cuda'] != cuda:
+                continue
+            version = rec[library]
+            filenames = rec['assets'][system]['filenames']
+            with tempfile.TemporaryDirectory() as d:
+                install_library.install_lib(cuda, d, library)
+                self._check_installed(
+                    d, cuda, library, version, filenames)
+            break
+        else:
+            pytest.fail(f'unexpected CUDA version {cuda} for {library}')
+
+    def _check_installed(self, prefix, cuda, lib, version, filenames):
+        install_root = os.path.join(prefix, cuda, lib, version)
+        assert os.path.isdir(install_root)
+        for _x, _y, files in os.walk(install_root):
+            for filename in filenames:
+                if filename in files:
+                    return
+        pytest.fail('expected file cound not be found')
+
+    @pytest.mark.parametrize('library', _libraries)
+    def test_urls(self, library):
+        assets = [r['assets']
+                  for r in install_library.library_records[library]]
+        for asset in assets:
+            for system in asset.keys():
+                url = asset[system]['url']
+                with urllib.request.urlopen(
+                        urllib.request.Request(url, method='HEAD')) as resp:
+                    assert resp.getcode() == 200
+
+    @pytest.mark.parametrize('library', _libraries)
+    def test_main(self, library):
+        install_library.main(
+            ['--library', library, '--action', 'dump', '--cuda', 'null'])
diff --git a/tests/example_tests/__init__.py b/tests/example_tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/example_tests/example_test.py b/tests/example_tests/example_test.py
new file mode 100644
index 0000000..5c52a62
--- /dev/null
+++ b/tests/example_tests/example_test.py
@@ -0,0 +1,18 @@
+import os
+import subprocess
+import sys
+
+
+def run_example(path, *args):
+    examples_path = os.path.join(
+        os.path.dirname(__file__), '..', '..', 'examples')
+    fullpath = os.path.join(examples_path, path)
+
+    try:
+        return subprocess.check_output(
+            (sys.executable, fullpath) + args,
+            stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        print('Original error message:')
+        print(e.output.decode('utf-8'))
+        raise
diff --git a/tests/example_tests/test_custom_struct.py b/tests/example_tests/test_custom_struct.py
new file mode 100644
index 0000000..31693d1
--- /dev/null
+++ b/tests/example_tests/test_custom_struct.py
@@ -0,0 +1,34 @@
+import re
+import unittest
+
+from example_tests import example_test
+
+
+class TestCustomStruct(unittest.TestCase):
+
+    def test_builtin_vectors(self):
+        output = example_test.run_example('custom_struct/builtin_vectors.py')
+        assert re.match(
+            r"Kernel output matches expected value.",
+            output.decode('utf-8'),
+        )
+
+    def test_packed_matrix(self):
+        output = example_test.run_example('custom_struct/packed_matrix.py')
+        assert re.match(
+            r"Kernel output matches expected value for type 'float'.\r?\n"
+            r"Kernel output matches expected value for type 'double'.",
+            output.decode('utf-8'),
+        )
+
+    def test_complex_struct(self):
+        output = example_test.run_example('custom_struct/complex_struct.py')
+        assert re.match(
+            r"Overall structure itemsize: \d+ bytes\r?\n"
+            r"Structure members itemsize: \[(\s*\d+){5}]\r?\n"
+            r"Structure members offsets: \[(\s*\d+){5}]\r?\n"
+            r"Complex structure value:\r?\n"
+            r"\s+\[.*\]\r?\n"
+            r"Kernel output matches expected value.",
+            output.decode('utf-8'),
+        )
diff --git a/tests/example_tests/test_finance.py b/tests/example_tests/test_finance.py
new file mode 100644
index 0000000..77c77d0
--- /dev/null
+++ b/tests/example_tests/test_finance.py
@@ -0,0 +1,62 @@
+import os
+import re
+import unittest
+
+from example_tests import example_test
+
+from cupy import testing
+
+
+def _normalize_regexp_eol(pattern):
+    return pattern.replace(r'\n', re.escape(os.linesep))
+
+
+class TestBlackScholes(unittest.TestCase):
+
+    def test_black_scholes(self):
+        output = example_test.run_example(
+            'finance/black_scholes.py', '--n-options', '10')
+        pattern = _normalize_regexp_eol(
+            r'initializing...\n' +
+            r'start computation\n' +
+            r' CPU \(NumPy, Naive implementation\):\t[0-9\.]+ sec\n' +
+            r' GPU \(CuPy, Naive implementation\):\t[0-9\.]+ sec\n' +
+            r' GPU \(CuPy, Elementwise kernel\):\t[0-9\.]+ sec')
+        assert re.search(pattern, output.decode('utf-8'))
+
+
+class TestMonteCarlo(unittest.TestCase):
+
+    def test_monte_carlo(self):
+        output = example_test.run_example(
+            'finance/monte_carlo.py', '--n-options', '10',
+            '--n-samples-per-thread', '10',
+            '--n-threads-per-option', '10')
+        pattern = _normalize_regexp_eol(
+            r'initializing...\n' +
+            r'start computation\n' +
+            r'    # of options: 10\n' +
+            r'    # of samples per option: 100\n' +
+            r'GPU \(CuPy, Monte Carlo method\):\t[0-9\.]+ sec\n' +
+            r'Error: [0-9\.]+')
+        assert re.search(pattern, output.decode('utf-8'))
+
+
+class TestMonteCarloWithMultiGPU(unittest.TestCase):
+
+    @testing.multi_gpu(2)
+    def test_monte_carlo_multigpu(self):
+        output = example_test.run_example(
+            'finance/monte_carlo_multigpu.py', '--gpus', '0', '1',
+            '--n-options', '10',
+            '--n-samples-per-thread', '10',
+            '--n-threads-per-option', '10')
+        pattern = _normalize_regexp_eol(
+            r'initializing...\n' +
+            r'start computation\n' +
+            r'    # of gpus: 2\n' +
+            r'    # of options: 10\n' +
+            r'    # of samples per option: 200\n' +
+            r'GPU \(CuPy, Monte Carlo method\):\t[0-9\.]+ sec\n' +
+            r'Error: [0-9\.]+')
+        assert re.search(pattern, output.decode('utf-8'))
diff --git a/tests/example_tests/test_gemm.py b/tests/example_tests/test_gemm.py
new file mode 100644
index 0000000..758d585
--- /dev/null
+++ b/tests/example_tests/test_gemm.py
@@ -0,0 +1,9 @@
+import unittest
+
+from example_tests import example_test
+
+
+class TestGEMM(unittest.TestCase):
+
+    def test_sgemm(self):
+        example_test.run_example('gemm/sgemm.py')
diff --git a/tests/example_tests/test_gmm.py b/tests/example_tests/test_gmm.py
new file mode 100644
index 0000000..7616579
--- /dev/null
+++ b/tests/example_tests/test_gmm.py
@@ -0,0 +1,37 @@
+import os
+import re
+import shutil
+import tempfile
+import unittest
+
+from cupy import testing
+
+from example_tests import example_test
+
+
+os.environ['MPLBACKEND'] = 'Agg'
+
+
+@testing.with_requires('matplotlib')
+@testing.with_requires('scipy')
+class TestGMM(unittest.TestCase):
+
+    def test_gmm(self):
+        output = example_test.run_example('gmm/gmm.py', '--num', '10')
+        assert re.search(
+            r'Running CPU\.\.\.\s+train_accuracy : [0-9\.]+\s+' +
+            r'test_accuracy : [0-9\.]+\s+CPU :  [0-9\.]+ sec\s+' +
+            r'Running GPU\.\.\.\s+train_accuracy : [0-9\.]+\s+' +
+            r'test_accuracy : [0-9\.]+\s+GPU :  [0-9\.]+ sec',
+            output.decode('utf-8'),
+        )
+
+    def test_output_image(self):
+        dir_path = tempfile.mkdtemp()
+        try:
+            image_path = os.path.join(dir_path, 'gmm.png')
+            example_test.run_example(
+                'gmm/gmm.py', '--num', '10', '-o', image_path)
+            assert os.path.exists(image_path)
+        finally:
+            shutil.rmtree(dir_path, ignore_errors=True)
diff --git a/tests/example_tests/test_kmeans.py b/tests/example_tests/test_kmeans.py
new file mode 100644
index 0000000..c8ce4d8
--- /dev/null
+++ b/tests/example_tests/test_kmeans.py
@@ -0,0 +1,43 @@
+import os
+import re
+import shutil
+import tempfile
+import unittest
+
+from cupy import testing
+
+from example_tests import example_test
+
+
+os.environ['MPLBACKEND'] = 'Agg'
+
+
+@testing.with_requires('matplotlib')
+class TestKmeans(unittest.TestCase):
+
+    def test_default(self):
+        output = example_test.run_example(
+            'kmeans/kmeans.py', '-m', '1', '--num', '10')
+        assert re.search(
+            r' CPU :  [0-9\.]+ sec\s+GPU :  [0-9\.]+ sec',
+            output.decode('utf-8'),
+        )
+
+    def test_custom_kernel(self):
+        output = example_test.run_example(
+            'kmeans/kmeans.py', '-m', '1', '--num', '10',
+            '--use-custom-kernel')
+        assert re.search(
+            r' CPU :  [0-9\.]+ sec\s+GPU :  [0-9\.]+ sec',
+            output.decode('utf-8'),
+        )
+
+    def test_result_image(self):
+        dir_path = tempfile.mkdtemp()
+        try:
+            image_path = os.path.join(dir_path, 'kmeans.png')
+            example_test.run_example(
+                'kmeans/kmeans.py', '-m', '1', '--num', '10', '-o', image_path)
+            assert os.path.exists(image_path)
+        finally:
+            shutil.rmtree(dir_path, ignore_errors=True)
diff --git a/tests/install_tests/__init__.py b/tests/install_tests/__init__.py
new file mode 100644
index 0000000..a03182f
--- /dev/null
+++ b/tests/install_tests/__init__.py
@@ -0,0 +1,20 @@
+import importlib
+import os
+import sys
+
+
+source_root = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', '..'))
+
+
+def _from_install_import(name):
+    original_sys_path = sys.path.copy()
+    try:
+        sys.path.append(os.path.join(source_root, 'install'))
+        return importlib.import_module(name)
+    finally:
+        sys.path = original_sys_path
+
+
+cupy_builder = _from_install_import('cupy_builder')
+cupy_builder.initialize(cupy_builder.Context(source_root, _env={}, _argv=[]))
diff --git a/tests/install_tests/test_build.py b/tests/install_tests/test_build.py
new file mode 100644
index 0000000..070aeb3
--- /dev/null
+++ b/tests/install_tests/test_build.py
@@ -0,0 +1,39 @@
+from distutils import ccompiler
+from distutils import sysconfig
+import os
+import unittest
+
+import pytest
+
+from cupy_builder._context import Context
+from cupy_builder import install_build as build
+
+
+test_hip = bool(int(os.environ.get('CUPY_INSTALL_USE_HIP', '0')))
+
+
+class TestCheckVersion(unittest.TestCase):
+
+    def setUp(self):
+        ctx = Context('.', _env={}, _argv=[])
+        self.compiler = ccompiler.new_compiler()
+        sysconfig.customize_compiler(self.compiler)
+        self.settings = build.get_compiler_setting(ctx, False)
+
+    @pytest.mark.skipif(not test_hip, reason='For ROCm/HIP environment')
+    def test_check_hip_version(self):
+        with self.assertRaises(RuntimeError):
+            build.get_hip_version()
+        assert build.check_hip_version(
+            self.compiler, self.settings)
+        assert isinstance(build.get_hip_version(), int)
+        assert isinstance(build.get_hip_version(True), str)
+
+    @pytest.mark.skipif(test_hip,
+                        reason='ROCm/HIP DNN support is not ready')
+    def test_check_cudnn_version(self):
+        with self.assertRaises(RuntimeError):
+            build.get_cudnn_version()
+        if build.check_cudnn_version(self.compiler, self.settings):
+            assert isinstance(build.get_cudnn_version(), int)
+            assert isinstance(build.get_cudnn_version(True), str)
diff --git a/tests/install_tests/test_cupy_builder/__init__.py b/tests/install_tests/test_cupy_builder/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/install_tests/test_cupy_builder/test_command.py b/tests/install_tests/test_cupy_builder/test_command.py
new file mode 100644
index 0000000..b983f18
--- /dev/null
+++ b/tests/install_tests/test_cupy_builder/test_command.py
@@ -0,0 +1,17 @@
+from cupy_builder._command import filter_files_by_extension
+
+
+def test_filter_files_by_extension():
+    sources_cpp = ['a.cpp', 'b.cpp']
+    sources_pyx = ['c.pyx']
+    sources = sources_cpp + sources_pyx
+    assert filter_files_by_extension(
+        sources, '.cpp') == (sources_cpp, sources_pyx)
+    assert filter_files_by_extension(
+        sources, '.pyx') == (sources_pyx, sources_cpp)
+    assert filter_files_by_extension(
+        sources, '.cu') == ([], sources)
+    assert filter_files_by_extension(
+        sources_cpp, '.cpp') == (sources_cpp, [])
+    assert filter_files_by_extension(
+        sources_cpp, '.pyx') == ([], sources_cpp)
diff --git a/tests/install_tests/test_cupy_builder/test_context.py b/tests/install_tests/test_cupy_builder/test_context.py
new file mode 100644
index 0000000..e33e8a0
--- /dev/null
+++ b/tests/install_tests/test_cupy_builder/test_context.py
@@ -0,0 +1,86 @@
+import os
+
+from cupy_builder._context import (
+    _get_env_bool, _get_env_path, Context, parse_args)
+
+
+class TestGetEnvBool:
+    def test_true(self):
+        assert _get_env_bool('V', True, {})
+        assert _get_env_bool('V', True, {'V': '1'})
+        assert _get_env_bool('V', False, {'V': '1'})
+
+    def test_false(self):
+        assert not _get_env_bool('V', False, {})
+        assert not _get_env_bool('V', False, {'V': '0'})
+        assert not _get_env_bool('V', True, {'V': '0'})
+
+
+class TestGetEnvPath:
+    def test(self):
+        assert _get_env_path('P', {}) == []
+        assert _get_env_path('P', {'P': f'1{os.pathsep}'}) == ['1']
+        assert _get_env_path('P', {'P': f'1{os.pathsep}2'}) == ['1', '2']
+
+
+class TestContext:
+    def test_default(self):
+        ctx = Context('.', _env={}, _argv=[])
+
+        assert ctx.source_root == '.'
+
+        assert not ctx.use_cuda_python
+        assert not ctx.use_hip
+        assert ctx.include_dirs == []
+        assert ctx.library_dirs == []
+
+        assert ctx.package_name == 'cupy'
+        assert ctx.long_description_path is None
+        assert ctx.wheel_libs == []
+        assert ctx.wheel_includes == []
+        assert ctx.wheel_metadata_path is None
+        assert not ctx.no_rpath
+        assert not ctx.profile
+        assert not ctx.linetrace
+        assert not ctx.annotate
+        assert not ctx.use_stub
+
+    def test_env(self):
+        ctx = Context('.', _env={
+            'CUPY_USE_CUDA_PYTHON': '1',
+            'CUPY_INSTALL_USE_HIP': '1',
+            'CUPY_INCLUDE_PATH': f'/tmp/include{os.pathsep}/tmp2/include',
+            'CUPY_LIBRARY_PATH': f'/tmp/lib{os.pathsep}/tmp2/lib',
+        }, _argv=[])
+
+        assert ctx.use_cuda_python
+        assert ctx.use_hip
+        assert ctx.include_dirs == ['/tmp/include', '/tmp2/include']
+        assert ctx.library_dirs == ['/tmp/lib', '/tmp2/lib']
+
+
+class TestParseArgs:
+    def test_args(self):
+        args = [
+            'python', 'setup.py', 'develop',
+            '--cupy-package-name', 'cupy-cudaXXX',
+            '--cupy-long-description', 'foo.rst',
+            '--cupy-wheel-lib', 'lib1',
+            '--cupy-wheel-lib', 'lib2',
+            '--cupy-wheel-metadata', '_wheel.json',
+            '--cupy-no-rpath',
+            '--cupy-profile',
+            '--cupy-coverage',
+            '--cupy-no-cuda',
+            '--extra-option',
+        ]
+        options, remaining = parse_args(args)
+        assert options.cupy_package_name == 'cupy-cudaXXX'
+        assert options.cupy_long_description == 'foo.rst'
+        assert options.cupy_wheel_lib == ['lib1', 'lib2']
+        assert options.cupy_wheel_metadata == '_wheel.json'
+        assert options.cupy_no_rpath
+        assert options.cupy_profile
+        assert options.cupy_coverage
+        assert options.cupy_no_cuda
+        assert remaining == ['python', 'setup.py', 'develop', '--extra-option']
diff --git a/tests/install_tests/test_cupy_builder/test_features.py b/tests/install_tests/test_cupy_builder/test_features.py
new file mode 100644
index 0000000..f9790a9
--- /dev/null
+++ b/tests/install_tests/test_cupy_builder/test_features.py
@@ -0,0 +1,30 @@
+from distutils import ccompiler
+from distutils import sysconfig
+
+from cupy_builder._context import Context
+from cupy_builder._features import CUDA_cuda
+from cupy_builder import install_build as build
+
+import cupy
+import pytest
+
+
+def get_compiler_settings():
+    ctx = Context('.', _env={}, _argv=[])
+    sysconfig.get_config_vars()
+    compiler = ccompiler.new_compiler()
+    sysconfig.customize_compiler(compiler)
+    settings = build.get_compiler_setting(ctx, False)
+    return compiler, settings
+
+
+@pytest.mark.skipif(cupy.cuda.runtime.is_hip, reason='For CUDA environment')
+def test_CUDA_cuda():
+    ctx = Context('.', _env={}, _argv=[])
+    feat = CUDA_cuda(ctx)
+    compiler, settings = get_compiler_settings()
+    feat.configure(compiler, settings)
+    if not cupy.cuda.driver._is_cuda_python():
+        # In CUDA Python, `get_build_version()` returns CUDA Python's version
+        assert feat._version == cupy.cuda.driver.get_build_version()
+    assert feat._version == cupy.cuda.runtime.runtimeGetVersion()
diff --git a/tests/install_tests/test_universal_pkg/__init__.py b/tests/install_tests/test_universal_pkg/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/install_tests/test_universal_pkg/test_setup.py b/tests/install_tests/test_universal_pkg/test_setup.py
new file mode 100644
index 0000000..cf904f4
--- /dev/null
+++ b/tests/install_tests/test_universal_pkg/test_setup.py
@@ -0,0 +1,54 @@
+from .. import _from_install_import
+
+import subprocess
+import sys
+
+import pytest
+import cupy
+
+
+setup = _from_install_import('universal_pkg.setup')
+
+
+@pytest.mark.skipif(
+    cupy.cuda.runtime.is_hip or cupy.cuda.driver._is_cuda_python(),
+    reason='for CUDA')
+def test_get_cuda_version():
+    assert setup._get_cuda_version() == cupy.cuda.runtime.runtimeGetVersion()
+
+
+@pytest.mark.skipif(not cupy.cuda.runtime.is_hip, reason='for HIP')
+def test_get_rocm_version():
+    assert setup._get_rocm_version() == cupy.cuda.runtime.runtimeGetVersion()
+
+
+def test_cuda_version_to_package():
+    with pytest.raises(setup.AutoDetectionFailed):
+        assert setup._cuda_version_to_package(10019)
+    assert setup._cuda_version_to_package(10020) == 'cupy-cuda102'
+    assert setup._cuda_version_to_package(11060) == 'cupy-cuda11x'
+    with pytest.raises(setup.AutoDetectionFailed):
+        assert setup._cuda_version_to_package(99999)
+
+
+def test_rocm_version_to_package():
+    with pytest.raises(setup.AutoDetectionFailed):
+        assert setup._rocm_version_to_package(399)
+    assert setup._rocm_version_to_package(4_03_21300) == 'cupy-rocm-4-3'
+    assert setup._rocm_version_to_package(5_00_13601) == 'cupy-rocm-5-0'
+    with pytest.raises(setup.AutoDetectionFailed):
+        assert setup._rocm_version_to_package(9_00_00000)
+
+
+def test_infer_best_package():
+    pkgs = setup._find_installed_packages()
+    if 1 < len(pkgs) or pkgs == ['cupy']:
+        with pytest.raises(setup.AutoDetectionFailed):
+            setup.infer_best_package()
+    else:
+        assert setup.infer_best_package() == pkgs[0]
+
+
+def test_execute():
+    proc = subprocess.run([sys.executable, setup.__file__, 'help'])
+    assert proc.returncode == 1
diff --git a/tests/install_tests/test_utils.py b/tests/install_tests/test_utils.py
new file mode 100644
index 0000000..10d8059
--- /dev/null
+++ b/tests/install_tests/test_utils.py
@@ -0,0 +1,18 @@
+import unittest
+
+from . import _from_install_import
+
+
+utils = _from_install_import('cupy_builder.install_utils')
+
+
+class TestPrintWarning(unittest.TestCase):
+
+    def test_print_warning(self):
+        utils.print_warning('This is a test.')
+
+
+class TestSearchOnPath(unittest.TestCase):
+
+    def test_exec_not_found(self):
+        assert utils.search_on_path(['no_such_exec']) is None
diff --git a/third_party/cub/.cproject b/third_party/cub/.cproject
new file mode 100644
index 0000000..e76d1da
--- /dev/null
+++ b/third_party/cub/.cproject
@@ -0,0 +1,1223 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
+	<storageModule moduleId="org.eclipse.cdt.core.settings">
+		<cconfiguration id="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311" moduleId="org.eclipse.cdt.core.settings" name="Default">
+				<externalSettings>
+					<externalSetting languages="cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.1945715073"/>
+				</externalSettings>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.Cygwin_PE" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactName="B40CTrunk" buildProperties="" description="" id="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311" name="Default" parent="org.eclipse.cdt.build.core.emptycfg">
+					<folderInfo id="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113" name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.cygwin.base.481495889" name="Cygwin GCC" superClass="cdt.managedbuild.toolchain.gnu.cygwin.base">
+							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.Cygwin_PE" id="cdt.managedbuild.target.gnu.platform.cygwin.base.100038061" name="Debug Platform" osList="win32" superClass="cdt.managedbuild.target.gnu.platform.cygwin.base"/>
+							<builder buildPath="${workspace_loc:/PrivateCub}/Default" id="cdt.managedbuild.target.gnu.builder.cygwin.base.412463247" keepEnvironmentInBuildfile="false" name="Gnu Make Builder" superClass="cdt.managedbuild.target.gnu.builder.cygwin.base"/>
+							<tool id="cdt.managedbuild.tool.gnu.assembler.cygwin.base.996758685" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.cygwin.base">
+								<option id="gnu.both.asm.option.include.paths.900454792" name="Include paths (-I)" superClass="gnu.both.asm.option.include.paths" valueType="includePath">
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include/device_launch_parameters.h&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include/crt/device_functions.h&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include&quot;"/>
+								</option>
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.221302756" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.archiver.cygwin.base.1353653670" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.cygwin.base"/>
+							<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.1401626953" name="Cygwin C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base">
+								<option id="gnu.cpp.compiler.option.include.paths.1909687606" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include/device_launch_parameters.h&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include/device_functions.h&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include&quot;"/>
+								</option>
+								<option id="gnu.cpp.compiler.option.preprocessor.def.1893619952" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" useByScannerDiscovery="false" valueType="definedSymbols">
+									<listOptionValue builtIn="false" value="__device__"/>
+									<listOptionValue builtIn="false" value="__global__"/>
+									<listOptionValue builtIn="false" value="__shared__"/>
+									<listOptionValue builtIn="false" value="__forceinline__"/>
+									<listOptionValue builtIn="false" value="__host__"/>
+									<listOptionValue builtIn="false" value="__device_builtin__"/>
+									<listOptionValue builtIn="false" value="__device_builtin_texture_type__"/>
+									<listOptionValue builtIn="false" value="TEST_ARCH=200"/>
+									<listOptionValue builtIn="false" value="__launch_bounds__(...)"/>
+									<listOptionValue builtIn="false" value="__align__(...)"/>
+									<listOptionValue builtIn="false" value="__CUDA_ARCH__=350"/>
+									<listOptionValue builtIn="false" value="__CUDACC__=1"/>
+								</option>
+								<option id="gnu.cpp.compiler.option.dialect.std.49639338" name="Language standard" superClass="gnu.cpp.compiler.option.dialect.std" useByScannerDiscovery="true" value="gnu.cpp.compiler.dialect.default" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.1708330939" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.1940954787" name="Cygwin C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.cygwin.base">
+								<option id="gnu.c.compiler.option.include.paths.1945618846" name="Include paths (-I)" superClass="gnu.c.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include/device_launch_parameters.h&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include/crt/device_functions.h&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${CUDA_PATH}/include&quot;"/>
+								</option>
+								<option id="gnu.c.compiler.option.preprocessor.def.symbols.1005509663" name="Defined symbols (-D)" superClass="gnu.c.compiler.option.preprocessor.def.symbols" useByScannerDiscovery="false" valueType="definedSymbols">
+									<listOptionValue builtIn="false" value="__device__"/>
+									<listOptionValue builtIn="false" value="__global__"/>
+									<listOptionValue builtIn="false" value="__shared__"/>
+									<listOptionValue builtIn="false" value="__forceinline__"/>
+									<listOptionValue builtIn="false" value="__host__"/>
+									<listOptionValue builtIn="false" value="__device_builtin__"/>
+									<listOptionValue builtIn="false" value="__device_builtin_texture_type__"/>
+									<listOptionValue builtIn="false" value="TEST_ARCH=200"/>
+									<listOptionValue builtIn="false" value="__launch_bounds__(...)"/>
+									<listOptionValue builtIn="false" value="__align__(...)"/>
+									<listOptionValue builtIn="false" value="__CUDA_ARCH__=350"/>
+									<listOptionValue builtIn="false" value="__CUDACC__=1"/>
+								</option>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.469104331" superClass="cdt.managedbuild.tool.gnu.c.compiler.input.cygwin"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.c.linker.cygwin.base.1600375047" name="Cygwin C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.cygwin.base"/>
+							<tool id="cdt.managedbuild.tool.gnu.cpp.linker.cygwin.base.1176124124" name="Cygwin C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.cygwin.base">
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.958378367" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
+									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
+									<additionalInput kind="additionalinput" paths="$(LIBS)"/>
+								</inputType>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.pathentry"/>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+			<storageModule moduleId="org.eclipse.cdt.core.language.mapping"/>
+			<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
+		</cconfiguration>
+	</storageModule>
+	<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+		<project id="B40CTrunk.null.1404415602" name="B40CTrunk"/>
+	</storageModule>
+	<storageModule moduleId="refreshScope" versionNumber="2">
+		<configuration configurationName="Default">
+			<resource resourceType="PROJECT" workspacePath="/GIT_CUB"/>
+		</configuration>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
+	<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings">
+		<doc-comment-owner id="org.eclipse.cdt.ui.doxygen">
+			<path value=""/>
+		</doc-comment-owner>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets"/>
+	<storageModule moduleId="scannerConfiguration">
+		<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/>
+		<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+			<buildOutputProvider>
+				<openAction enabled="true" filePath=""/>
+				<parser enabled="true"/>
+			</buildOutputProvider>
+			<scannerInfoProvider id="specsFile">
+				<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+				<parser enabled="true"/>
+			</scannerInfoProvider>
+		</profile>
+		<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+			<buildOutputProvider>
+				<openAction enabled="true" filePath=""/>
+				<parser enabled="true"/>
+			</buildOutputProvider>
+			<scannerInfoProvider id="makefileGenerator">
+				<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+				<parser enabled="true"/>
+			</scannerInfoProvider>
+		</profile>
+		<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+			<buildOutputProvider>
+				<openAction enabled="true" filePath=""/>
+				<parser enabled="true"/>
+			</buildOutputProvider>
+			<scannerInfoProvider id="specsFile">
+				<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+				<parser enabled="true"/>
+			</scannerInfoProvider>
+		</profile>
+		<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+			<buildOutputProvider>
+				<openAction enabled="true" filePath=""/>
+				<parser enabled="true"/>
+			</buildOutputProvider>
+			<scannerInfoProvider id="specsFile">
+				<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+				<parser enabled="true"/>
+			</scannerInfoProvider>
+		</profile>
+		<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+			<buildOutputProvider>
+				<openAction enabled="true" filePath=""/>
+				<parser enabled="true"/>
+			</buildOutputProvider>
+			<scannerInfoProvider id="specsFile">
+				<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+				<parser enabled="true"/>
+			</scannerInfoProvider>
+		</profile>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.1940954787;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.469104331">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.1665401269;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.494265807">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileCPP"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.43985841;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.1045483126">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileC"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.1240277003;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.1264397663">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileC"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.459535216;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.2120860882">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.1758599759;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.466964704">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileC"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.1401626953;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.1708330939">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.1671954574;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.304556051">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.2110267806;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.903720746">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.1850250798;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.1752562149">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.1296776241;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.268633283">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileC"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.265387950;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.563557831">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileCPP"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.629007265;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.450470600">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileC"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.2085396856;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.1885998497">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileCPP"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.652522784;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.1098348915">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileCPP"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.1149397878;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.1156849140">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileC"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.586941236;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.1654082299">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileCPP"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.1214991320;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.332043455">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileCPP"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.cpp.compiler.cygwin.base.440957653;cdt.managedbuild.tool.gnu.cpp.compiler.input.cygwin.1117446939">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileCPP"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311;cdt.managedbuild.toolchain.gnu.cygwin.base.1260156311.1722659113;cdt.managedbuild.tool.gnu.c.compiler.cygwin.base.158380621;cdt.managedbuild.tool.gnu.c.compiler.input.cygwin.1945715073">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCWinManagedMakePerProjectProfileC"/>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerFileProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="makefileGenerator">
+					<runAction arguments="-f ${project_name}_scd.mk" command="make" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfile">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/${specs_file}" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.cpp" command="g++" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+			<profile id="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC">
+				<buildOutputProvider>
+					<openAction enabled="true" filePath=""/>
+					<parser enabled="true"/>
+				</buildOutputProvider>
+				<scannerInfoProvider id="specsFile">
+					<runAction arguments="-E -P -v -dD ${plugin_state_location}/specs.c" command="gcc" useDefault="true"/>
+					<parser enabled="true"/>
+				</scannerInfoProvider>
+			</profile>
+		</scannerConfigBuildInfo>
+	</storageModule>
+</cproject>
diff --git a/third_party/cub/.project b/third_party/cub/.project
new file mode 100644
index 0000000..7aca9e0
--- /dev/null
+++ b/third_party/cub/.project
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>GIT_CUB</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
+			<triggers>clean,full,incremental,</triggers>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
+			<triggers>full,incremental,</triggers>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.cdt.core.cnature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
+		<nature>org.eclipse.cdt.core.ccnature</nature>
+	</natures>
+</projectDescription>
diff --git a/third_party/cub/.settings/.gitignore b/third_party/cub/.settings/.gitignore
new file mode 100644
index 0000000..d81d4c4
--- /dev/null
+++ b/third_party/cub/.settings/.gitignore
@@ -0,0 +1 @@
+/language.settings.xml
diff --git a/third_party/cub/.settings/org.eclipse.cdt.codan.core.prefs b/third_party/cub/.settings/org.eclipse.cdt.codan.core.prefs
new file mode 100644
index 0000000..64da777
--- /dev/null
+++ b/third_party/cub/.settings/org.eclipse.cdt.codan.core.prefs
@@ -0,0 +1,72 @@
+eclipse.preferences.version=1
+org.eclipse.cdt.codan.checkers.errnoreturn=Warning
+org.eclipse.cdt.codan.checkers.errnoreturn.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},implicit\=>false}
+org.eclipse.cdt.codan.checkers.errreturnvalue=Error
+org.eclipse.cdt.codan.checkers.errreturnvalue.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.checkers.nocommentinside=-Error
+org.eclipse.cdt.codan.checkers.nocommentinside.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.checkers.nolinecomment=-Error
+org.eclipse.cdt.codan.checkers.nolinecomment.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.checkers.noreturn=Error
+org.eclipse.cdt.codan.checkers.noreturn.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},implicit\=>false}
+org.eclipse.cdt.codan.internal.checkers.AbstractClassCreation=Error
+org.eclipse.cdt.codan.internal.checkers.AbstractClassCreation.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.AmbiguousProblem=Error
+org.eclipse.cdt.codan.internal.checkers.AmbiguousProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.AssignmentInConditionProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.AssignmentInConditionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.AssignmentToItselfProblem=Error
+org.eclipse.cdt.codan.internal.checkers.AssignmentToItselfProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.CaseBreakProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.CaseBreakProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},no_break_comment\=>"no break",last_case_param\=>true,empty_case_param\=>false}
+org.eclipse.cdt.codan.internal.checkers.CatchByReference=Warning
+org.eclipse.cdt.codan.internal.checkers.CatchByReference.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},unknown\=>false,exceptions\=>()}
+org.eclipse.cdt.codan.internal.checkers.CircularReferenceProblem=Error
+org.eclipse.cdt.codan.internal.checkers.CircularReferenceProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.ClassMembersInitialization=Warning
+org.eclipse.cdt.codan.internal.checkers.ClassMembersInitialization.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},skip\=>true}
+org.eclipse.cdt.codan.internal.checkers.FieldResolutionProblem=Error
+org.eclipse.cdt.codan.internal.checkers.FieldResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.FunctionResolutionProblem=Error
+org.eclipse.cdt.codan.internal.checkers.FunctionResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.InvalidArguments=Error
+org.eclipse.cdt.codan.internal.checkers.InvalidArguments.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.InvalidTemplateArgumentsProblem=Error
+org.eclipse.cdt.codan.internal.checkers.InvalidTemplateArgumentsProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.LabelStatementNotFoundProblem=Error
+org.eclipse.cdt.codan.internal.checkers.LabelStatementNotFoundProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.MemberDeclarationNotFoundProblem=Error
+org.eclipse.cdt.codan.internal.checkers.MemberDeclarationNotFoundProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.MethodResolutionProblem=Error
+org.eclipse.cdt.codan.internal.checkers.MethodResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.NamingConventionFunctionChecker=-Info
+org.eclipse.cdt.codan.internal.checkers.NamingConventionFunctionChecker.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},pattern\=>"^[a-z]",macro\=>true,exceptions\=>()}
+org.eclipse.cdt.codan.internal.checkers.NonVirtualDestructorProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.NonVirtualDestructorProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.OverloadProblem=Error
+org.eclipse.cdt.codan.internal.checkers.OverloadProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.RedeclarationProblem=Error
+org.eclipse.cdt.codan.internal.checkers.RedeclarationProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.RedefinitionProblem=Error
+org.eclipse.cdt.codan.internal.checkers.RedefinitionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.ReturnStyleProblem=-Warning
+org.eclipse.cdt.codan.internal.checkers.ReturnStyleProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.ScanfFormatStringSecurityProblem=-Warning
+org.eclipse.cdt.codan.internal.checkers.ScanfFormatStringSecurityProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.StatementHasNoEffectProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.StatementHasNoEffectProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},macro\=>true,exceptions\=>()}
+org.eclipse.cdt.codan.internal.checkers.SuggestedParenthesisProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.SuggestedParenthesisProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},paramNot\=>false}
+org.eclipse.cdt.codan.internal.checkers.SuspiciousSemicolonProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.SuspiciousSemicolonProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},else\=>false,afterelse\=>false}
+org.eclipse.cdt.codan.internal.checkers.TypeResolutionProblem=Error
+org.eclipse.cdt.codan.internal.checkers.TypeResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+org.eclipse.cdt.codan.internal.checkers.UnusedFunctionDeclarationProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.UnusedFunctionDeclarationProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},macro\=>true}
+org.eclipse.cdt.codan.internal.checkers.UnusedStaticFunctionProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.UnusedStaticFunctionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},macro\=>true}
+org.eclipse.cdt.codan.internal.checkers.UnusedVariableDeclarationProblem=Warning
+org.eclipse.cdt.codan.internal.checkers.UnusedVariableDeclarationProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},macro\=>true,exceptions\=>("@(\#)","$Id")}
+org.eclipse.cdt.codan.internal.checkers.VariableResolutionProblem=Error
+org.eclipse.cdt.codan.internal.checkers.VariableResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}}
+useParentScope=false
diff --git a/third_party/cub/.settings/org.eclipse.cdt.core.prefs b/third_party/cub/.settings/org.eclipse.cdt.core.prefs
new file mode 100644
index 0000000..80b8e65
--- /dev/null
+++ b/third_party/cub/.settings/org.eclipse.cdt.core.prefs
@@ -0,0 +1,177 @@
+eclipse.preferences.version=1
+indexer/indexAllFiles=true
+indexer/indexAllHeaderVersions=false
+indexer/indexAllVersionsSpecificHeaders=
+indexer/indexOnOpen=false
+indexer/indexUnusedHeadersWithAlternateLang=false
+indexer/indexUnusedHeadersWithDefaultLang=true
+indexer/indexerId=org.eclipse.cdt.core.fastIndexer
+indexer/skipFilesLargerThanMB=8
+indexer/skipImplicitReferences=false
+indexer/skipIncludedFilesLargerThanMB=16
+indexer/skipMacroReferences=false
+indexer/skipReferences=false
+indexer/skipTypeReferences=false
+indexer/useHeuristicIncludeResolution=true
+org.eclipse.cdt.core.formatter.alignment_for_arguments_in_method_invocation=16
+org.eclipse.cdt.core.formatter.alignment_for_assignment=16
+org.eclipse.cdt.core.formatter.alignment_for_base_clause_in_type_declaration=48
+org.eclipse.cdt.core.formatter.alignment_for_binary_expression=16
+org.eclipse.cdt.core.formatter.alignment_for_compact_if=0
+org.eclipse.cdt.core.formatter.alignment_for_conditional_expression=48
+org.eclipse.cdt.core.formatter.alignment_for_conditional_expression_chain=18
+org.eclipse.cdt.core.formatter.alignment_for_constructor_initializer_list=0
+org.eclipse.cdt.core.formatter.alignment_for_declarator_list=16
+org.eclipse.cdt.core.formatter.alignment_for_enumerator_list=48
+org.eclipse.cdt.core.formatter.alignment_for_expression_list=0
+org.eclipse.cdt.core.formatter.alignment_for_expressions_in_array_initializer=16
+org.eclipse.cdt.core.formatter.alignment_for_member_access=0
+org.eclipse.cdt.core.formatter.alignment_for_overloaded_left_shift_chain=16
+org.eclipse.cdt.core.formatter.alignment_for_parameters_in_method_declaration=48
+org.eclipse.cdt.core.formatter.alignment_for_throws_clause_in_method_declaration=48
+org.eclipse.cdt.core.formatter.brace_position_for_array_initializer=next_line
+org.eclipse.cdt.core.formatter.brace_position_for_block=next_line
+org.eclipse.cdt.core.formatter.brace_position_for_block_in_case=end_of_line
+org.eclipse.cdt.core.formatter.brace_position_for_method_declaration=next_line
+org.eclipse.cdt.core.formatter.brace_position_for_namespace_declaration=end_of_line
+org.eclipse.cdt.core.formatter.brace_position_for_switch=end_of_line
+org.eclipse.cdt.core.formatter.brace_position_for_type_declaration=next_line
+org.eclipse.cdt.core.formatter.comment.min_distance_between_code_and_line_comment=1
+org.eclipse.cdt.core.formatter.comment.never_indent_line_comments_on_first_column=true
+org.eclipse.cdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=true
+org.eclipse.cdt.core.formatter.compact_else_if=true
+org.eclipse.cdt.core.formatter.continuation_indentation=1
+org.eclipse.cdt.core.formatter.continuation_indentation_for_array_initializer=1
+org.eclipse.cdt.core.formatter.format_guardian_clause_on_one_line=false
+org.eclipse.cdt.core.formatter.indent_access_specifier_compare_to_type_header=false
+org.eclipse.cdt.core.formatter.indent_access_specifier_extra_spaces=0
+org.eclipse.cdt.core.formatter.indent_body_declarations_compare_to_access_specifier=true
+org.eclipse.cdt.core.formatter.indent_body_declarations_compare_to_namespace_header=false
+org.eclipse.cdt.core.formatter.indent_breaks_compare_to_cases=true
+org.eclipse.cdt.core.formatter.indent_declaration_compare_to_template_header=false
+org.eclipse.cdt.core.formatter.indent_empty_lines=false
+org.eclipse.cdt.core.formatter.indent_statements_compare_to_block=true
+org.eclipse.cdt.core.formatter.indent_statements_compare_to_body=true
+org.eclipse.cdt.core.formatter.indent_switchstatements_compare_to_cases=true
+org.eclipse.cdt.core.formatter.indent_switchstatements_compare_to_switch=false
+org.eclipse.cdt.core.formatter.indentation.size=4
+org.eclipse.cdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_after_template_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_catch_in_try_statement=insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_colon_in_constructor_initializer_list=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_else_in_if_statement=insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_identifier_in_function_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_in_empty_block=insert
+org.eclipse.cdt.core.formatter.insert_space_after_assignment_operator=insert
+org.eclipse.cdt.core.formatter.insert_space_after_binary_operator=insert
+org.eclipse.cdt.core.formatter.insert_space_after_closing_angle_bracket_in_template_arguments=insert
+org.eclipse.cdt.core.formatter.insert_space_after_closing_angle_bracket_in_template_parameters=insert
+org.eclipse.cdt.core.formatter.insert_space_after_closing_brace_in_block=insert
+org.eclipse.cdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
+org.eclipse.cdt.core.formatter.insert_space_after_colon_in_base_clause=insert
+org.eclipse.cdt.core.formatter.insert_space_after_colon_in_case=insert
+org.eclipse.cdt.core.formatter.insert_space_after_colon_in_conditional=insert
+org.eclipse.cdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_base_types=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_declarator_list=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_expression_list=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_template_arguments=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_template_parameters=insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_angle_bracket_in_template_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_angle_bracket_in_template_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_bracket=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_exception_specification=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_postfix_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_prefix_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_question_in_conditional=insert
+org.eclipse.cdt.core.formatter.insert_space_after_semicolon_in_for=insert
+org.eclipse.cdt.core.formatter.insert_space_after_unary_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_assignment_operator=insert
+org.eclipse.cdt.core.formatter.insert_space_before_binary_operator=insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_angle_bracket_in_template_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_angle_bracket_in_template_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_bracket=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_exception_specification=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_base_clause=insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_case=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_conditional=insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_default=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_base_types=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_declarator_list=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_expression_list=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_template_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_template_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_angle_bracket_in_template_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_angle_bracket_in_template_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_block=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_namespace_declaration=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_bracket=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_exception_specification=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_for=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_if=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_while=insert
+org.eclipse.cdt.core.formatter.insert_space_before_postfix_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_prefix_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_question_in_conditional=insert
+org.eclipse.cdt.core.formatter.insert_space_before_semicolon=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_unary_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_brackets=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_exception_specification=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
+org.eclipse.cdt.core.formatter.join_wrapped_lines=true
+org.eclipse.cdt.core.formatter.keep_else_statement_on_same_line=false
+org.eclipse.cdt.core.formatter.keep_empty_array_initializer_on_one_line=false
+org.eclipse.cdt.core.formatter.keep_imple_if_on_one_line=true
+org.eclipse.cdt.core.formatter.keep_then_statement_on_same_line=false
+org.eclipse.cdt.core.formatter.lineSplit=80
+org.eclipse.cdt.core.formatter.number_of_empty_lines_to_preserve=1
+org.eclipse.cdt.core.formatter.put_empty_statement_on_new_line=true
+org.eclipse.cdt.core.formatter.tabulation.char=space
+org.eclipse.cdt.core.formatter.tabulation.size=4
+org.eclipse.cdt.core.formatter.use_tabs_only_for_leading_indentations=false
diff --git a/third_party/cub/.settings/org.eclipse.cdt.ui.prefs b/third_party/cub/.settings/org.eclipse.cdt.ui.prefs
new file mode 100644
index 0000000..ca73f82
--- /dev/null
+++ b/third_party/cub/.settings/org.eclipse.cdt.ui.prefs
@@ -0,0 +1,3 @@
+eclipse.preferences.version=1
+formatter_profile=_B40C
+formatter_settings_version=1
diff --git a/third_party/cub/.settings/org.eclipse.core.runtime.prefs b/third_party/cub/.settings/org.eclipse.core.runtime.prefs
new file mode 100644
index 0000000..2e6330e
--- /dev/null
+++ b/third_party/cub/.settings/org.eclipse.core.runtime.prefs
@@ -0,0 +1,4 @@
+content-types/enabled=true
+content-types/org.eclipse.cdt.core.cxxHeader/file-extensions=cuh
+content-types/org.eclipse.cdt.core.cxxSource/file-extensions=cu
+eclipse.preferences.version=1
diff --git a/third_party/cub/CHANGE_LOG.TXT b/third_party/cub/CHANGE_LOG.TXT
new file mode 100644
index 0000000..ed7f395
--- /dev/null
+++ b/third_party/cub/CHANGE_LOG.TXT
@@ -0,0 +1,403 @@
+1.8.0    02/15/2018
+    - API change: change to the interfaces of cub::ShuffleIndex, cub::ShuffleUp, and 
+      cub::ShuffleDown to better compute the PTX shfl control constant for 
+      logical warps smaller than 32 threads 
+    - Bug fixes: 
+        - Issue #112: Bug in WarpScan's broadcast of warp-wide aggregate for 
+          logical warps < 32 threads 
+        
+          		  
+//-----------------------------------------------------------------------------
+
+1.7.5    02/08/2018
+    - Added radix-sorting support for __half keys
+    - Updated sorting policies for improved 8b-key performance 
+    - Bug fixes: 
+        - Syntax tweaks to mollify Clang
+        - Issue #127: DeviceRunLengthEncode::Encode returns wrong results 
+        - Issue #128: 7-bit sorting passes fail for sm61 w/ large-values
+        
+          		  
+//-----------------------------------------------------------------------------
+
+1.7.4    09/20/2017
+    - Bug fixes: 
+        - Issue #114: Can't pair non-trivially-constructible values in radix sort
+        - Issue #115: WarpReduce segmented reduction broken in CUDA 9 for logical warp sizes < 32 
+          		  
+//-----------------------------------------------------------------------------
+
+1.7.3    08/28/2017
+    - Bug fixes: 
+        - Issue #110: DeviceHistogram null-pointer exception bug for iterator inputs
+          		  
+//-----------------------------------------------------------------------------
+
+1.7.2    08/26/2017
+    - Bug fixes: 
+        - Issue #104: Device-wide reduction is now "run-to-run" deterministic for 
+          pseudo-associative reduction operators (like floating point addition)
+          		  
+//-----------------------------------------------------------------------------
+
+1.7.1    08/18/2017
+    - Updated Volta radix sorting tuning policies 
+    - Bug fixes: 
+        - Issue #104 (uint64_t warp-reduce broken for cub 1.7.0 on cuda 8 and older)
+        - Issue #103 (Can't mix Thrust 9.0 and CUB)
+        - Issue #102 (CUB pulls in windows.h which defines min/max macros that conflict with std::min/std::max)
+        - Issue #99 (Radix sorting crashes NVCC on Windows 10 for SM52)
+        - Issue #98 (cuda-memcheck: --tool initcheck failed with lineOfSight)
+        - Issue #94 (Git clone size)
+        - Issue #93 (accept iterators for segment offsets)
+        - Issue #87 (CUB uses anonymous unions which is not valid C++)
+        - Issue #44 (Check for C++ 11 should be changed that Visual Studio 2013 is also recognized as C++ 11 capable)
+          		  
+//-----------------------------------------------------------------------------
+
+1.7.0    06/07/2017
+    - Compatible with CUDA9 and SM7.x (Volta) independent thread scheduling 
+    - API change: remove cub::WarpAll() and cub::WarpAny().  These functions served to 
+      emulate __all and __any functionality for SM1.x devices, which did not have those 
+      operations.  However, the SM1.x devices are now deprecated in CUDA, and the 
+      interfaces of the these two functions are now lacking the lane-mask needed 
+      for collectives to run on Volta SMs having independent thread scheduling. 
+    - Bug fixes: 
+        - Issue #86 Incorrect results with ReduceByKey
+          		  
+//-----------------------------------------------------------------------------
+
+1.6.4    12/06/2016
+    - Updated sm_5x, sm_6x tuning policies for radix sorting (3.5B and 3.4B 
+      32b keys/s on TitanX and GTX 1080, respectively)
+    - Bug fixes: 
+        - Restore fence work-around for scan (reduce-by-key, etc.) hangs 
+          in CUDA 8.5
+        - Issue 65: DeviceSegmentedRadixSort should allow inputs to have 
+          pointer-to-const type 
+        - Mollify Clang device-side warnings
+        - Remove out-dated VC project files
+          		  
+//-----------------------------------------------------------------------------
+
+1.6.3    11/20/2016
+    - API change: BlockLoad and BlockStore are now templated by the local
+      data type, instead of the Iterator type.  This allows for output iterators
+      having \p void as their \p value_type (e.g., discard iterators).
+    - Updated GP100 tuning policies for radix sorting (6.2B 32b keys/s)
+    - Bug fixes: 
+        - Issue #74: Warpreduce executes reduction operator for out-of-bounds items
+        - Issue #72 (cub:InequalityWrapper::operator() should be non-const)
+        - Issue #71 (KeyVairPair won't work if Key has non-trivial ctor)
+		- Issue #70 1.5.3 breaks BlockScan API.  Retroactively reversioned
+		  from v1.5.3 -> v1.6 to appropriately indicate API change.
+		- Issue #69 cub::BlockStore::Store doesn't compile if OutputIteratorT::value_type != T  
+        - Issue #68 (cub::TilePrefixCallbackOp::WarpReduce doesn't permit ptx 
+          arch specialization)
+		- Improved support for Win32 platforms (warnings, alignment, etc)
+		  
+//-----------------------------------------------------------------------------
+
+1.6.2 (was 1.5.5)    10/25/2016
+    - Updated Pascal tuning policies for radix sorting
+    - Bug fixes: 
+        - Fix for arm64 compilation of caching allocator
+
+//-----------------------------------------------------------------------------
+
+1.6.1 (was 1.5.4)    10/14/2016
+    - Bug fixes: 
+        - Fix for radix sorting bug introduced by scan refactorization
+
+//-----------------------------------------------------------------------------
+
+1.6.0 (was 1.5.3)    10/11/2016
+    - API change: Device/block/warp-wide exclusive scans have been revised to now 
+      accept an "initial value" (instead of an "identity value") for seeding the 
+      computation with an arbitrary prefix.  
+    - API change: Device-wide reductions and scans can now have input sequence types that are 
+      different from output sequence types (as long as they are coercible)
+      value") for seeding the computation with an arbitrary prefix
+    - Reduce repository size (move doxygen binary to doc repository)
+    - Minor reductions in block-scan instruction count
+    - Bug fixes: 
+        - Issue #55: warning in cub/device/dispatch/dispatch_reduce_by_key.cuh 
+        - Issue #59: cub::DeviceScan::ExclusiveSum can't prefix sum of float into double
+        - Issue #58: Infinite loop in cub::CachingDeviceAllocator::NearestPowerOf
+        - Issue #47: Caching allocator needs to clean up cuda error upon successful retry 
+        - Issue #46: Very high amount of needed memory from the cub::DeviceHistogram::HistogramEven routine
+        - Issue #45: Caching Device Allocator fails with debug output enabled
+        - Fix for generic-type reduce-by-key warpscan (sm3.x and newer)
+
+//-----------------------------------------------------------------------------
+
+1.5.2    03/21/2016
+	- Improved medium-size scan performance for sm5x (Maxwell)
+    - Refactored caching allocator for device memory
+   		- Spends less time locked
+		- Failure to allocate a block from the runtime will retry once after
+		  freeing cached allocations
+		- Now respects max-bin (issue where blocks in excess of max-bin were
+		  still being retained in free cache)
+		- Uses C++11 mutex when available
+    - Bug fixes: 
+        - Fix for generic-type reduce-by-key warpscan (sm3.x and newer)
+          
+//-----------------------------------------------------------------------------
+
+1.5.1    12/28/2015
+    - Bug fixes: 
+        - Fix for incorrect DeviceRadixSort output for some small problems on 
+          Maxwell SM52 architectures
+        - Fix for macro redefinition warnings when compiling with Thrust sort
+          
+//-----------------------------------------------------------------------------
+
+1.5.0    12/14/2015
+    - New Features:
+        - Added new segmented device-wide operations for device-wide sort and 
+          reduction primitives.
+    - Bug fixes: 
+        - Fix for Git Issue 36 (Compilation error with GCC 4.8.4 nvcc 7.0.27) and
+          Forums thread (ThreadLoad generates compiler errors when loading from 
+          pointer-to-const)
+        - Fix for Git Issue 29 (DeviceRadixSort::SortKeys<bool> yields compiler 
+          errors)
+        - Fix for Git Issue 26 (CUDA error: misaligned address after 
+          cub::DeviceRadixSort::SortKeys())
+        - Fix for incorrect/crash on 0-length problems, e.g., Git Issue 25 (Floating 
+          point exception (core dumped) during cub::DeviceRadixSort::SortKeys)
+        - Fix for CUDA 7.5 issues on SM 5.2 with SHFL-based warp-scan and warp-reduction 
+          on non-primitive data types (e.g., user-defined structs)
+        - Fix for small radix sorting problems where 0 temporary bytes were 
+          required and users code was invoking malloc(0) on some systems where
+          that returns NULL.  (Impl assumed was asking for size again and was not 
+          running the sort.)
+          
+//-----------------------------------------------------------------------------
+
+1.4.1    04/13/2015
+    - Bug fixes: 
+        - Fixes for CUDA 7.0 issues with SHFL-based warp-scan and warp-reduction 
+          on non-primitive data types (e.g., user-defined structs)
+        - Fixes for minor CUDA 7.0 performance regressions in cub::DeviceScan,
+          DeviceReduceByKey
+        - Fixes to allow cub::DeviceRadixSort and cub::BlockRadixSort on bool types
+        - Remove requirement for callers to define the CUB_CDP macro 
+          when invoking CUB device-wide rountines using CUDA dynamic parallelism
+        - Fix for headers not being included in the proper order (or missing includes)
+          for some block-wide functions
+          
+//-----------------------------------------------------------------------------
+
+1.4.0    03/18/2015
+    - New Features:
+		- Support and performance tuning for new Maxwell GPU architectures
+        - Updated cub::DeviceHistogram implementation that provides the same 
+          "histogram-even" and "histogram-range" functionality as IPP/NPP.
+          Provides extremely fast and, perhaps more importantly, very 
+          uniform performance response across diverse real-world datasets, 
+          including pathological (homogeneous) sample distributions (resilience)
+        - New cub::DeviceSpmv methods for multiplying sparse matrices by 
+          dense vectors, load-balanced using a merge-based parallel decomposition.
+        - New cub::DeviceRadixSort sorting entry-points that always return
+          the sorted output into the specified buffer (as opposed to the 
+          cub::DoubleBuffer in which it could end up in either buffer)
+        - New cub::DeviceRunLengthEncode::NonTrivialRuns for finding the starting 
+          offsets and lengths of all non-trivial runs (i.e., length > 1) of keys in 
+          a given sequence.  (Useful for top-down partitioning algorithms like 
+          MSD sorting of very-large keys.)
+          
+//-----------------------------------------------------------------------------
+
+1.3.2    07/28/2014
+    - Bug fixes: 
+        - Fix for cub::DeviceReduce where reductions of small problems 
+          (small enough to only dispatch a single thread block) would run in 
+          the default stream (stream zero) regardless of whether an alternate
+          stream was specified.  
+          
+//-----------------------------------------------------------------------------
+
+1.3.1    05/23/2014
+    - Bug fixes: 
+        - Workaround for a benign WAW race warning reported by cuda-memcheck
+          in BlockScan specialized for BLOCK_SCAN_WARP_SCANS algorithm.
+        - Fix for bug in DeviceRadixSort where the algorithm may sort more 
+          key bits than the caller specified (up to the nearest radix digit).
+        - Fix for ~3% DeviceRadixSort performance regression on Kepler and 
+          Fermi that was introduced in v1.3.0.  
+
+//-----------------------------------------------------------------------------
+
+1.3.0    05/12/2014
+    - New features:
+        - CUB's collective (block-wide, warp-wide) primitives underwent a minor 
+          interface refactoring:
+            - To provide the appropriate support for multidimensional thread blocks,
+              The interfaces for collective classes are now template-parameterized 
+              by X, Y, and Z block dimensions (with BLOCK_DIM_Y and BLOCK_DIM_Z being 
+              optional, and BLOCK_DIM_X replacing BLOCK_THREADS).  Furthermore, the 
+              constructors that accept remapped linear thread-identifiers have been 
+              removed: all primitives now assume a row-major thread-ranking for 
+              multidimensional thread blocks.  
+            - To allow the host program (compiled by the host-pass) to 
+              accurately determine the device-specific storage requirements for 
+              a given collective (compiled for each device-pass), the interfaces 
+              for collective classes are now (optionally) template-parameterized 
+              by the desired PTX compute capability. This is useful when 
+              aliasing collective storage to shared memory that has been 
+              allocated dynamically by the host at the kernel call site.   
+            - Most CUB programs having typical 1D usage should not require any 
+              changes to accomodate these updates.
+        - Added new "combination" WarpScan methods for efficiently computing 
+          both inclusive and exclusive prefix scans (and sums).
+    - Bug fixes: 
+        - Fixed bug in cub::WarpScan (which affected cub::BlockScan and 
+          cub::DeviceScan) where incorrect results (e.g., NAN) would often be 
+          returned when parameterized for floating-point types (fp32, fp64).
+        - Workaround-fix for ptxas error when compiling with with -G flag on Linux 
+          (for debug instrumentation) 
+        - Misc. workaround-fixes for certain scan scenarios (using custom 
+          scan operators) where code compiled for SM1x is run on newer 
+          GPUs of higher compute-capability: the compiler could not tell
+          which memory space was being used collective operations and was 
+          mistakenly using global ops instead of shared ops. 
+
+//-----------------------------------------------------------------------------
+
+1.2.3    04/01/2014
+    - Bug fixes: 
+        - Fixed access violation bug in DeviceReduce::ReduceByKey for non-primitive value types
+        - Fixed code-snippet bug in ArgIndexInputIteratorT documentation 
+
+//-----------------------------------------------------------------------------
+
+1.2.2    03/03/2014
+    - New features:
+        - Added MS VC++ project solutions for device-wide and block-wide examples 
+    - Performance:
+        - Added a third algorithmic variant of cub::BlockReduce for improved performance
+          when using commutative operators (e.g., numeric addition)
+    - Bug fixes: 
+        - Fixed bug where inclusion of Thrust headers in a certain order prevented CUB device-wide primitives from working properly
+
+//-----------------------------------------------------------------------------
+
+1.2.0    02/25/2014
+    - New features:
+        - Added device-wide reduce-by-key (DeviceReduce::ReduceByKey, DeviceReduce::RunLengthEncode) 
+    - Performance
+        - Improved DeviceScan, DeviceSelect, DevicePartition performance
+    - Documentation and testing:
+        - Compatible with CUDA 6.0
+        - Added performance-portability plots for many device-wide primitives to doc 
+        - Update doc and tests to reflect iterator (in)compatibilities with CUDA 5.0 (and older) and Thrust 1.6 (and older).
+    - Bug fixes 
+        - Revised the operation of temporary tile status bookkeeping for DeviceScan (and similar) to be safe for current code run on future platforms (now uses proper fences)  
+        - Fixed DeviceScan bug where Win32 alignment disagreements between host and device regarding user-defined data types would corrupt tile status
+        - Fixed BlockScan bug where certain exclusive scans on custom data types for the BLOCK_SCAN_WARP_SCANS variant would return incorrect results for the first thread in the block
+        - Added workaround for TexRefInputIteratorTto work with CUDA 6.0
+    
+//-----------------------------------------------------------------------------
+
+1.1.1    12/11/2013
+    - New features:
+        - Added TexObjInputIteratorT, TexRefInputIteratorT, CacheModifiedInputIteratorT, and CacheModifiedOutputIterator types for loading & storing arbitrary types through the cache hierarchy.  Compatible with Thrust API. 
+        - Added descending sorting to DeviceRadixSort and BlockRadixSort
+        - Added min, max, arg-min, and arg-max to DeviceReduce
+        - Added DeviceSelect (select-unique, select-if, and select-flagged)
+        - Added DevicePartition (partition-if, partition-flagged)
+        - Added generic cub::ShuffleUp(), cub::ShuffleDown(), and cub::ShuffleIndex() for warp-wide communication of arbitrary data types (SM3x+)
+        - Added cub::MaxSmOccupancy() for accurately determining SM occupancy for any given kernel function pointer
+    - Performance
+        - Improved DeviceScan and DeviceRadixSort performance for older architectures (SM10-SM30)
+    - Interface changes:
+        - Refactored block-wide I/O (BlockLoad and BlockStore), removing cache-modifiers from their interfaces.  The CacheModifiedInputIteratorTand CacheModifiedOutputIterator should now be used with BlockLoad and BlockStore to effect that behavior.
+        - Rename device-wide "stream_synchronous" param to "debug_synchronous" to avoid confusion about usage
+    - Documentation and testing:
+        - Added simple examples of device-wide methods
+        - Improved doxygen documentation and example snippets
+        - Improved test coverege to include up to 21,000 kernel variants and 851,000 unit tests (per architecture, per platform)
+    - Bug fixes 
+        - Fixed misc DeviceScan, BlockScan, DeviceReduce, and BlockReduce bugs when operating on non-primitive types for older architectures SM10-SM13
+        - Fixed DeviceScan / WarpReduction bug: SHFL-based segmented reduction producting incorrect results for multi-word types (size > 4B) on Linux 
+        - Fixed BlockScan bug: For warpscan-based scans, not all threads in the first warp were entering the prefix callback functor
+        - Fixed DeviceRadixSort bug: race condition with key-value pairs for pre-SM35 architectures
+        - Fixed DeviceRadixSort bug: incorrect bitfield-extract behavior with long keys on 64bit Linux
+        - Fixed BlockDiscontinuity bug: complation error in for types other than int32/uint32
+        - CDP (device-callable) versions of device-wide methods now report the same temporary storage allocation size requirement as their host-callable counterparts
+     
+
+//-----------------------------------------------------------------------------
+
+1.0.2    08/23/2013
+    - Corrections to code snippet examples for BlockLoad, BlockStore, and BlockDiscontinuity
+    - Cleaned up unnecessary/missing header includes.  You can now safely #inlude a specific .cuh (instead of cub.cuh)
+    - Bug/compilation fixes for BlockHistogram 
+
+//-----------------------------------------------------------------------------
+
+1.0.1    08/08/2013
+    - New collective interface idiom (specialize::construct::invoke).
+    - Added best-in-class DeviceRadixSort.  Implements short-circuiting for homogenous digit passes.
+    - Added best-in-class DeviceScan.  Implements single-pass "adaptive-lookback" strategy.
+    - Significantly improved documentation (with example code snippets) 
+    - More extensive regression test suit for aggressively testing collective variants
+    - Allow non-trially-constructed types (previously unions had prevented aliasing temporary storage of those types)
+    - Improved support for Kepler SHFL (collective ops now use SHFL for types larger than 32b)
+    - Better code generation for 64-bit addressing within BlockLoad/BlockStore
+    - DeviceHistogram now supports histograms of arbitrary bins
+    - Misc. fixes
+      - Workarounds for SM10 codegen issues in uncommonly-used WarpScan/Reduce specializations
+      - Updates to accommodate CUDA 5.5 dynamic parallelism   
+
+
+//-----------------------------------------------------------------------------
+
+0.9.4    05/07/2013
+
+    - Fixed compilation errors for SM10-SM13
+    - Fixed compilation errors for some WarpScan entrypoints on SM30+
+    - Added block-wide histogram (BlockHistogram256)
+    - Added device-wide histogram (DeviceHistogram256)
+    - Added new BlockScan algorithm variant BLOCK_SCAN_RAKING_MEMOIZE, which 
+      trades more register consumption for less shared memory I/O)
+    - Updates to BlockRadixRank to use BlockScan (which improves performance
+      on Kepler due to SHFL instruction)
+    - Allow types other than C++ primitives to be used in WarpScan::*Sum methods 
+      if they only have operator + overloaded.  (Previously they also required 
+      to support assignment from int(0).) 
+    - Update BlockReduce's BLOCK_REDUCE_WARP_REDUCTIONS algorithm to work even 
+      when block size is not an even multiple of warp size
+    - Added work management utility descriptors (GridQueue, GridEvenShare)
+    - Refactoring of DeviceAllocator interface and CachingDeviceAllocator 
+      implementation 
+    - Misc. documentation updates and corrections. 
+     
+//-----------------------------------------------------------------------------
+
+0.9.2    04/04/2013
+
+    - Added WarpReduce.  WarpReduce uses the SHFL instruction when applicable. 
+      BlockReduce now uses this WarpReduce instead of implementing its own.
+    - Misc. fixes for 64-bit Linux compilation warnings and errors.
+    - Misc. documentation updates and corrections. 
+
+//-----------------------------------------------------------------------------
+
+0.9.1    03/09/2013
+
+    - Fix for ambiguity in BlockScan::Reduce() between generic reduction and 
+      summation.  Summation entrypoints are now called ::Sum(), similar to the 
+      convention in BlockScan.
+    - Small edits to mainpage documentation and download tracking
+    
+//-----------------------------------------------------------------------------
+
+0.9.0    03/07/2013    
+
+    - Intial "preview" release.    CUB is the first durable, high-performance library 
+      of cooperative block-level, warp-level, and thread-level primitives for CUDA 
+      kernel programming.  More primitives and examples coming soon!
+    
\ No newline at end of file
diff --git a/third_party/cub/LICENSE.TXT b/third_party/cub/LICENSE.TXT
new file mode 100644
index 0000000..a678e64
--- /dev/null
+++ b/third_party/cub/LICENSE.TXT
@@ -0,0 +1,24 @@
+Copyright (c) 2010-2011, Duane Merrill.  All rights reserved.
+Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+   *  Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+   *  Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+   *  Neither the name of the NVIDIA CORPORATION nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/third_party/cub/README.md b/third_party/cub/README.md
new file mode 100644
index 0000000..d2a338e
--- /dev/null
+++ b/third_party/cub/README.md
@@ -0,0 +1,128 @@
+<hr>
+<h3>About CUB</h3>
+
+Current release: v1.8.0 (02/16/2018)
+
+We recommend the [CUB Project Website](http://nvlabs.github.com/cub) for further information and examples.
+
+CUB provides state-of-the-art, reusable software components for every layer 
+of the CUDA programming model:
+- [<b><em>Device-wide primitives</em></b>] (https://nvlabs.github.com/cub/group___device_module.html) 
+  - Sort, prefix scan, reduction, histogram, etc.  
+  - Compatible with CUDA dynamic parallelism
+- [<b><em>Block-wide "collective" primitives</em></b>] (https://nvlabs.github.com/cub/group___block_module.html)
+  - I/O, sort, prefix scan, reduction, histogram, etc.  
+  - Compatible with arbitrary thread block sizes and types 
+- [<b><em>Warp-wide "collective" primitives</em></b>] (https://nvlabs.github.com/cub/group___warp_module.html)
+  - Warp-wide prefix scan, reduction, etc.
+  - Safe and architecture-specific
+- [<b><em>Thread and resource utilities</em></b>](https://nvlabs.github.com/cub/group___thread_module.html)
+  - PTX intrinsics, device reflection, texture-caching iterators, caching memory allocators, etc. 
+
+![Orientation of collective primitives within the CUDA software stack](http://nvlabs.github.com/cub/cub_overview.png)
+
+<br><hr>
+<h3>A Simple Example</h3>
+
+```C++
+#include <cub/cub.cuh>
+ 
+// Block-sorting CUDA kernel
+__global__ void BlockSortKernel(int *d_in, int *d_out)
+{
+     using namespace cub;
+
+     // Specialize BlockRadixSort, BlockLoad, and BlockStore for 128 threads 
+     // owning 16 integer items each
+     typedef BlockRadixSort<int, 128, 16>                     BlockRadixSort;
+     typedef BlockLoad<int, 128, 16, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
+     typedef BlockStore<int, 128, 16, BLOCK_STORE_TRANSPOSE> BlockStore;
+ 
+     // Allocate shared memory
+     __shared__ union {
+         typename BlockRadixSort::TempStorage  sort;
+         typename BlockLoad::TempStorage       load; 
+         typename BlockStore::TempStorage      store; 
+     } temp_storage; 
+
+     int block_offset = blockIdx.x * (128 * 16);	  // OffsetT for this block's ment
+
+     // Obtain a segment of 2048 consecutive keys that are blocked across threads
+     int thread_keys[16];
+     BlockLoad(temp_storage.load).Load(d_in + block_offset, thread_keys);
+     __syncthreads();
+
+     // Collectively sort the keys
+     BlockRadixSort(temp_storage.sort).Sort(thread_keys);
+     __syncthreads();
+
+     // Store the sorted segment 
+     BlockStore(temp_storage.store).Store(d_out + block_offset, thread_keys);
+}
+```
+
+Each thread block uses cub::BlockRadixSort to collectively sort 
+its own input segment.  The class is specialized by the 
+data type being sorted, by the number of threads per block, by the number of 
+keys per thread, and implicitly by the targeted compilation architecture.  
+
+The cub::BlockLoad and cub::BlockStore classes are similarly specialized.    
+Furthermore, to provide coalesced accesses to device memory, these primitives are 
+configured to access memory using a striped access pattern (where consecutive threads 
+simultaneously access consecutive items) and then <em>transpose</em> the keys into 
+a [<em>blocked arrangement</em>](index.html#sec4sec3) of elements across threads. 
+
+Once specialized, these classes expose opaque \p TempStorage member types.  
+The thread block uses these storage types to statically allocate the union of 
+shared memory needed by the thread block.  (Alternatively these storage types 
+could be aliased to global memory allocations).
+
+<br><hr>
+<h3>Stable Releases</h3>
+
+CUB releases are labeled using version identifiers having three fields: 
+*epoch.feature.update*.  The *epoch* field corresponds to support for
+a major change in the CUDA programming model.  The *feature* field 
+corresponds to a stable set of features, functionality, and interface.  The
+*update* field corresponds to a bug-fix or performance update for that
+feature set.  At the moment, we do not publicly provide non-stable releases 
+such as development snapshots, beta releases or rolling releases.  (Feel free
+to contact us if you would like such things.)  See the 
+[CUB Project Website](http://nvlabs.github.com/cub) for more information.
+
+<br><hr>
+<h3>Contributors</h3>
+
+CUB is developed as an open-source project by [NVIDIA Research](http://research.nvidia.com).  The primary contributor is [Duane Merrill](http://github.com/dumerrill).
+
+<br><hr>
+<h3>Open Source License</h3>
+
+CUB is available under the "New BSD" open-source license:
+
+```
+Copyright (c) 2010-2011, Duane Merrill.  All rights reserved.
+Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+   *  Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+   *  Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+   *  Neither the name of the NVIDIA CORPORATION nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+```
diff --git a/third_party/cub/common.mk b/third_party/cub/common.mk
new file mode 100644
index 0000000..82893ab
--- /dev/null
+++ b/third_party/cub/common.mk
@@ -0,0 +1,233 @@
+#/******************************************************************************
+# * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+# * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+# * 
+# * Redistribution and use in source and binary forms, with or without
+# * modification, are permitted provided that the following conditions are met:
+# *	 * Redistributions of source code must retain the above copyright
+# *	   notice, this list of conditions and the following disclaimer.
+# *	 * Redistributions in binary form must reproduce the above copyright
+# *	   notice, this list of conditions and the following disclaimer in the
+# *	   documentation and/or other materials provided with the distribution.
+# *	 * Neither the name of the NVIDIA CORPORATION nor the
+# *	   names of its contributors may be used to endorse or promote products
+# *	   derived from this software without specific prior written permission.
+# * 
+# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+# * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *
+#******************************************************************************/
+
+
+#-------------------------------------------------------------------------------
+# Commandline Options
+#-------------------------------------------------------------------------------
+
+# [sm=<XXX,...>] Compute-capability to compile for, e.g., "sm=200,300,350" (SM20 by default).
+  
+COMMA = ,
+ifdef sm
+	SM_ARCH = $(subst $(COMMA),-,$(sm))
+else 
+    SM_ARCH = 200
+endif
+
+ifeq (700, $(findstring 700, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_70,code=\"sm_70,compute_70\" 
+    SM_DEF 		+= -DSM700
+    TEST_ARCH 	= 700
+endif
+ifeq (620, $(findstring 620, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_62,code=\"sm_62,compute_62\" 
+    SM_DEF 		+= -DSM620
+    TEST_ARCH 	= 620
+endif
+ifeq (610, $(findstring 610, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_61,code=\"sm_61,compute_61\" 
+    SM_DEF 		+= -DSM610
+    TEST_ARCH 	= 610
+endif
+ifeq (600, $(findstring 600, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_60,code=\"sm_60,compute_60\" 
+    SM_DEF 		+= -DSM600
+    TEST_ARCH 	= 600
+endif
+ifeq (520, $(findstring 520, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_52,code=\"sm_52,compute_52\" 
+    SM_DEF 		+= -DSM520
+    TEST_ARCH 	= 520
+endif
+ifeq (370, $(findstring 370, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_37,code=\"sm_37,compute_37\" 
+    SM_DEF 		+= -DSM370
+    TEST_ARCH 	= 370
+endif
+ifeq (350, $(findstring 350, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_35,code=\"sm_35,compute_35\" 
+    SM_DEF 		+= -DSM350
+    TEST_ARCH 	= 350
+endif
+ifeq (300, $(findstring 300, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_30,code=\"sm_30,compute_30\"
+    SM_DEF 		+= -DSM300
+    TEST_ARCH 	= 300
+endif
+ifeq (210, $(findstring 210, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_20,code=\"sm_21,compute_20\"
+    SM_DEF 		+= -DSM210
+    TEST_ARCH 	= 210
+endif
+ifeq (200, $(findstring 200, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_20,code=\"sm_20,compute_20\"
+    SM_DEF 		+= -DSM200
+    TEST_ARCH 	= 200
+endif
+ifeq (130, $(findstring 130, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_13,code=\"sm_13,compute_13\" 
+    SM_DEF 		+= -DSM130
+    TEST_ARCH 	= 130
+endif
+ifeq (120, $(findstring 120, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_12,code=\"sm_12,compute_12\" 
+    SM_DEF 		+= -DSM120
+    TEST_ARCH 	= 120
+endif
+ifeq (110, $(findstring 110, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_11,code=\"sm_11,compute_11\" 
+    SM_DEF 		+= -DSM110
+    TEST_ARCH 	= 110
+endif
+ifeq (100, $(findstring 100, $(SM_ARCH)))
+    SM_TARGETS 	+= -gencode=arch=compute_10,code=\"sm_10,compute_10\" 
+    SM_DEF 		+= -DSM100
+    TEST_ARCH 	= 100
+endif
+
+
+# [cdp=<0|1>] CDP enable option (default: no)
+ifeq ($(cdp), 1)
+	DEFINES += -DCUB_CDP
+	CDP_SUFFIX = cdp
+    NVCCFLAGS += -rdc=true -lcudadevrt
+else
+	CDP_SUFFIX = nocdp
+endif
+
+
+# [force32=<0|1>] Device addressing mode option (64-bit device pointers by default) 
+ifeq ($(force32), 1)
+	CPU_ARCH = -m32
+	CPU_ARCH_SUFFIX = i386
+else
+	CPU_ARCH = -m64
+	CPU_ARCH_SUFFIX = x86_64
+    NPPI = -lnppist
+endif
+
+
+# [abi=<0|1>] CUDA ABI option (enabled by default) 
+ifneq ($(abi), 0)
+	ABI_SUFFIX = abi
+else 
+	NVCCFLAGS += -Xptxas -abi=no
+	ABI_SUFFIX = noabi
+endif
+
+
+# [open64=<0|1>] Middle-end compiler option (nvvm by default)
+ifeq ($(open64), 1)
+	NVCCFLAGS += -open64
+	PTX_SUFFIX = open64
+else 
+	PTX_SUFFIX = nvvm
+endif
+
+
+# [verbose=<0|1>] Verbose toolchain output from nvcc option
+ifeq ($(verbose), 1)
+	NVCCFLAGS += -v
+endif
+
+
+# [keep=<0|1>] Keep intermediate compilation artifacts option
+ifeq ($(keep), 1)
+	NVCCFLAGS += -keep
+endif
+
+# [debug=<0|1>] Generate debug mode code
+ifeq ($(debug), 1)
+	NVCCFLAGS += -G
+endif
+
+
+#-------------------------------------------------------------------------------
+# Compiler and compilation platform
+#-------------------------------------------------------------------------------
+
+CUB_DIR = $(dir $(lastword $(MAKEFILE_LIST)))
+
+NVCC = "$(shell which nvcc)"
+ifdef nvccver
+    NVCC_VERSION = $(nvccver)
+else
+    NVCC_VERSION = $(strip $(shell nvcc --version | grep release | sed 's/.*release //' |  sed 's/,.*//'))
+endif
+
+# detect OS
+OSUPPER = $(shell uname -s 2>/dev/null | tr [:lower:] [:upper:])
+
+# Default flags: verbose kernel properties (regs, smem, cmem, etc.); runtimes for compilation phases 
+NVCCFLAGS += $(SM_DEF) -Xptxas -v -Xcudafe -\# 
+
+ifeq (WIN_NT, $(findstring WIN_NT, $(OSUPPER)))
+    # For MSVC
+    # Enable more warnings and treat as errors
+    NVCCFLAGS += -Xcompiler /W3 -Xcompiler /WX
+    # Disable excess x86 floating point precision that can lead to results being labeled incorrectly
+    NVCCFLAGS += -Xcompiler /fp:strict
+    # Help the compiler/linker work with huge numbers of kernels on Windows
+	NVCCFLAGS += -Xcompiler /bigobj -Xcompiler /Zm500
+	CC = cl
+	
+	# Multithreaded runtime
+	NVCCFLAGS += -Xcompiler /MT
+	
+ifneq ($(force32), 1)
+	CUDART_CYG = "$(shell dirname $(NVCC))/../lib/Win32/cudart.lib"
+else
+	CUDART_CYG = "$(shell dirname $(NVCC))/../lib/x64/cudart.lib"
+endif
+	CUDART = "$(shell cygpath -w $(CUDART_CYG))"
+else
+    # For g++
+    # Disable excess x86 floating point precision that can lead to results being labeled incorrectly
+    NVCCFLAGS += -Xcompiler -ffloat-store
+    CC = g++
+ifneq ($(force32), 1)
+    CUDART = "$(shell dirname $(NVCC))/../lib/libcudart_static.a"
+else
+    CUDART = "$(shell dirname $(NVCC))/../lib64/libcudart_static.a"
+endif
+endif
+
+# Suffix to append to each binary
+BIN_SUFFIX = sm$(SM_ARCH)_$(PTX_SUFFIX)_$(NVCC_VERSION)_$(ABI_SUFFIX)_$(CDP_SUFFIX)_$(CPU_ARCH_SUFFIX)
+
+
+#-------------------------------------------------------------------------------
+# Dependency Lists
+#-------------------------------------------------------------------------------
+
+rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
+
+CUB_DEPS = 	$(call rwildcard, $(CUB_DIR),*.cuh) \
+			$(CUB_DIR)common.mk
+		
diff --git a/third_party/cub/cub/agent/agent_histogram.cuh b/third_party/cub/cub/agent/agent_histogram.cuh
new file mode 100644
index 0000000..37b1ec9
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_histogram.cuh
@@ -0,0 +1,787 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::AgentHistogram implements a stateful abstraction of CUDA thread blocks for participating in device-wide histogram .
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "../util_type.cuh"
+#include "../block/block_load.cuh"
+#include "../grid/grid_queue.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy
+ ******************************************************************************/
+
+/**
+ *
+ */
+enum BlockHistogramMemoryPreference
+{
+    GMEM,
+    SMEM,
+    BLEND
+};
+
+
+/**
+ * Parameterizable tuning policy type for AgentHistogram
+ */
+template <
+    int                             _BLOCK_THREADS,                 ///< Threads per thread block
+    int                             _PIXELS_PER_THREAD,             ///< Pixels per thread (per tile of input)
+    BlockLoadAlgorithm              _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
+    CacheLoadModifier               _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
+    bool                            _RLE_COMPRESS,                  ///< Whether to perform localized RLE to compress samples before histogramming
+    BlockHistogramMemoryPreference  _MEM_PREFERENCE,                ///< Whether to prefer privatized shared-memory bins (versus privatized global-memory bins)
+    bool                            _WORK_STEALING>                 ///< Whether to dequeue tiles from a global work queue
+struct AgentHistogramPolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,                   ///< Threads per thread block
+        PIXELS_PER_THREAD       = _PIXELS_PER_THREAD,               ///< Pixels per thread (per tile of input)
+        IS_RLE_COMPRESS         = _RLE_COMPRESS,                    ///< Whether to perform localized RLE to compress samples before histogramming
+        MEM_PREFERENCE          = _MEM_PREFERENCE,                  ///< Whether to prefer privatized shared-memory bins (versus privatized global-memory bins)
+        IS_WORK_STEALING        = _WORK_STEALING,                   ///< Whether to dequeue tiles from a global work queue
+    };
+
+    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;          ///< The BlockLoad algorithm to use
+    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;           ///< Cache load modifier for reading input elements
+};
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+/**
+ * \brief AgentHistogram implements a stateful abstraction of CUDA thread blocks for participating in device-wide histogram .
+ */
+template <
+    typename    AgentHistogramPolicyT,     ///< Parameterized AgentHistogramPolicy tuning policy type
+    int         PRIVATIZED_SMEM_BINS,           ///< Number of privatized shared-memory histogram bins of any channel.  Zero indicates privatized counters to be maintained in device-accessible memory.
+    int         NUM_CHANNELS,                   ///< Number of channels interleaved in the input data.  Supports up to four channels.
+    int         NUM_ACTIVE_CHANNELS,            ///< Number of channels actively being histogrammed
+    typename    SampleIteratorT,                ///< Random-access input iterator type for reading samples
+    typename    CounterT,                       ///< Integer type for counting sample occurrences per histogram bin
+    typename    PrivatizedDecodeOpT,            ///< The transform operator type for determining privatized counter indices from samples, one for each channel
+    typename    OutputDecodeOpT,                ///< The transform operator type for determining output bin-ids from privatized counter indices, one for each channel
+    typename    OffsetT,                        ///< Signed integer type for global offsets
+    int         PTX_ARCH = CUB_PTX_ARCH>        ///< PTX compute capability
+struct AgentHistogram
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// The sample type of the input iterator
+    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+    /// The pixel type of SampleT
+    typedef typename CubVector<SampleT, NUM_CHANNELS>::Type PixelT;
+
+    /// The quad type of SampleT
+    typedef typename CubVector<SampleT, 4>::Type QuadT;
+
+    /// Constants
+    enum
+    {
+        BLOCK_THREADS           = AgentHistogramPolicyT::BLOCK_THREADS,
+
+        PIXELS_PER_THREAD       = AgentHistogramPolicyT::PIXELS_PER_THREAD,
+        SAMPLES_PER_THREAD      = PIXELS_PER_THREAD * NUM_CHANNELS,
+        QUADS_PER_THREAD        = SAMPLES_PER_THREAD / 4,
+
+        TILE_PIXELS             = PIXELS_PER_THREAD * BLOCK_THREADS,
+        TILE_SAMPLES            = SAMPLES_PER_THREAD * BLOCK_THREADS,
+
+        IS_RLE_COMPRESS            = AgentHistogramPolicyT::IS_RLE_COMPRESS,
+
+        MEM_PREFERENCE          = (PRIVATIZED_SMEM_BINS > 0) ?
+                                        AgentHistogramPolicyT::MEM_PREFERENCE :
+                                        GMEM,
+
+        IS_WORK_STEALING           = AgentHistogramPolicyT::IS_WORK_STEALING,
+    };
+
+    /// Cache load modifier for reading input elements
+    static const CacheLoadModifier LOAD_MODIFIER = AgentHistogramPolicyT::LOAD_MODIFIER;
+
+
+    /// Input iterator wrapper type (for applying cache modifier)
+    typedef typename If<IsPointer<SampleIteratorT>::VALUE,
+            CacheModifiedInputIterator<LOAD_MODIFIER, SampleT, OffsetT>,     // Wrap the native input pointer with CacheModifiedInputIterator
+            SampleIteratorT>::Type                                           // Directly use the supplied input iterator type
+        WrappedSampleIteratorT;
+
+    /// Pixel input iterator type (for applying cache modifier)
+    typedef CacheModifiedInputIterator<LOAD_MODIFIER, PixelT, OffsetT>
+        WrappedPixelIteratorT;
+
+    /// Qaud input iterator type (for applying cache modifier)
+    typedef CacheModifiedInputIterator<LOAD_MODIFIER, QuadT, OffsetT>
+        WrappedQuadIteratorT;
+
+    /// Parameterized BlockLoad type for samples
+    typedef BlockLoad<
+            SampleT,
+            BLOCK_THREADS,
+            SAMPLES_PER_THREAD,
+            AgentHistogramPolicyT::LOAD_ALGORITHM>
+        BlockLoadSampleT;
+
+    /// Parameterized BlockLoad type for pixels
+    typedef BlockLoad<
+            PixelT,
+            BLOCK_THREADS,
+            PIXELS_PER_THREAD,
+            AgentHistogramPolicyT::LOAD_ALGORITHM>
+        BlockLoadPixelT;
+
+    /// Parameterized BlockLoad type for quads
+    typedef BlockLoad<
+            QuadT,
+            BLOCK_THREADS,
+            QUADS_PER_THREAD,
+            AgentHistogramPolicyT::LOAD_ALGORITHM>
+        BlockLoadQuadT;
+
+    /// Shared memory type required by this thread block
+    struct _TempStorage
+    {
+        CounterT histograms[NUM_ACTIVE_CHANNELS][PRIVATIZED_SMEM_BINS + 1];     // Smem needed for block-privatized smem histogram (with 1 word of padding)
+
+        int tile_idx;
+
+        // Aliasable storage layout
+        union Aliasable
+        {
+            typename BlockLoadSampleT::TempStorage sample_load;     // Smem needed for loading a tile of samples
+            typename BlockLoadPixelT::TempStorage pixel_load;       // Smem needed for loading a tile of pixels
+            typename BlockLoadQuadT::TempStorage quad_load;         // Smem needed for loading a tile of quads
+
+        } aliasable;
+    };
+
+
+    /// Temporary storage type (unionable)
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    /// Reference to temp_storage
+    _TempStorage &temp_storage;
+
+    /// Sample input iterator (with cache modifier applied, if possible)
+    WrappedSampleIteratorT d_wrapped_samples;
+
+    /// Native pointer for input samples (possibly NULL if unavailable)
+    SampleT* d_native_samples;
+
+    /// The number of output bins for each channel
+    int (&num_output_bins)[NUM_ACTIVE_CHANNELS];
+
+    /// The number of privatized bins for each channel
+    int (&num_privatized_bins)[NUM_ACTIVE_CHANNELS];
+
+    /// Reference to gmem privatized histograms for each channel
+    CounterT* d_privatized_histograms[NUM_ACTIVE_CHANNELS];
+
+    /// Reference to final output histograms (gmem)
+    CounterT* (&d_output_histograms)[NUM_ACTIVE_CHANNELS];
+
+    /// The transform operator for determining output bin-ids from privatized counter indices, one for each channel
+    OutputDecodeOpT (&output_decode_op)[NUM_ACTIVE_CHANNELS];
+
+    /// The transform operator for determining privatized counter indices from samples, one for each channel
+    PrivatizedDecodeOpT (&privatized_decode_op)[NUM_ACTIVE_CHANNELS];
+
+    /// Whether to prefer privatized smem counters vs privatized global counters
+    bool prefer_smem;
+
+
+    //---------------------------------------------------------------------
+    // Initialize privatized bin counters
+    //---------------------------------------------------------------------
+
+    // Initialize privatized bin counters
+    __device__ __forceinline__ void InitBinCounters(CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS])
+    {
+        // Initialize histogram bin counts to zeros
+        #pragma unroll
+        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+        {
+            for (int privatized_bin = threadIdx.x; privatized_bin < num_privatized_bins[CHANNEL]; privatized_bin += BLOCK_THREADS)
+            {
+                privatized_histograms[CHANNEL][privatized_bin] = 0;
+            }
+        }
+
+        // Barrier to make sure all threads are done updating counters
+        CTA_SYNC();
+    }
+
+
+    // Initialize privatized bin counters.  Specialized for privatized shared-memory counters
+    __device__ __forceinline__ void InitSmemBinCounters()
+    {
+        CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS];
+
+        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+            privatized_histograms[CHANNEL] = temp_storage.histograms[CHANNEL];
+
+        InitBinCounters(privatized_histograms);
+    }
+
+
+    // Initialize privatized bin counters.  Specialized for privatized global-memory counters
+    __device__ __forceinline__ void InitGmemBinCounters()
+    {
+        InitBinCounters(d_privatized_histograms);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Update final output histograms
+    //---------------------------------------------------------------------
+
+    // Update final output histograms from privatized histograms
+    __device__ __forceinline__ void StoreOutput(CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS])
+    {
+        // Barrier to make sure all threads are done updating counters
+        CTA_SYNC();
+
+        // Apply privatized bin counts to output bin counts
+        #pragma unroll
+        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+        {
+            int channel_bins = num_privatized_bins[CHANNEL];
+            for (int privatized_bin = threadIdx.x; 
+                    privatized_bin < channel_bins;  
+                    privatized_bin += BLOCK_THREADS)
+            {
+                int         output_bin  = -1;
+                CounterT    count       = privatized_histograms[CHANNEL][privatized_bin];
+                bool        is_valid    = count > 0;
+
+                output_decode_op[CHANNEL].template BinSelect<LOAD_MODIFIER>((SampleT) privatized_bin, output_bin, is_valid);
+
+                if (output_bin >= 0)
+                {
+                    atomicAdd(&d_output_histograms[CHANNEL][output_bin], count);
+                }
+
+            }
+        }
+    }
+
+
+    // Update final output histograms from privatized histograms.  Specialized for privatized shared-memory counters
+    __device__ __forceinline__ void StoreSmemOutput()
+    {
+        CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS];
+        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+            privatized_histograms[CHANNEL] = temp_storage.histograms[CHANNEL];
+
+        StoreOutput(privatized_histograms);
+    }
+
+
+    // Update final output histograms from privatized histograms.  Specialized for privatized global-memory counters
+    __device__ __forceinline__ void StoreGmemOutput()
+    {
+        StoreOutput(d_privatized_histograms);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Tile accumulation
+    //---------------------------------------------------------------------
+
+    // Accumulate pixels.  Specialized for RLE compression.
+    __device__ __forceinline__ void AccumulatePixels(
+        SampleT             samples[PIXELS_PER_THREAD][NUM_CHANNELS],
+        bool                is_valid[PIXELS_PER_THREAD],
+        CounterT*           privatized_histograms[NUM_ACTIVE_CHANNELS],
+        Int2Type<true>      is_rle_compress)
+    {
+        #pragma unroll
+        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+        {
+            // Bin pixels
+            int bins[PIXELS_PER_THREAD];
+
+            #pragma unroll
+            for (int PIXEL = 0; PIXEL < PIXELS_PER_THREAD; ++PIXEL)
+            {
+                bins[PIXEL] = -1;
+                privatized_decode_op[CHANNEL].template BinSelect<LOAD_MODIFIER>(samples[PIXEL][CHANNEL], bins[PIXEL], is_valid[PIXEL]);
+            }
+
+            CounterT accumulator = 1;
+
+            #pragma unroll
+            for (int PIXEL = 0; PIXEL < PIXELS_PER_THREAD - 1; ++PIXEL)
+            {
+                if (bins[PIXEL] != bins[PIXEL + 1])
+                {
+                    if (bins[PIXEL] >= 0)
+                        atomicAdd(privatized_histograms[CHANNEL] + bins[PIXEL], accumulator);
+
+                     accumulator = 0;
+                }
+                accumulator++;
+            }
+
+            // Last pixel
+            if (bins[PIXELS_PER_THREAD - 1] >= 0)
+                atomicAdd(privatized_histograms[CHANNEL] + bins[PIXELS_PER_THREAD - 1], accumulator);
+        }
+    }
+
+
+    // Accumulate pixels.  Specialized for individual accumulation of each pixel.
+    __device__ __forceinline__ void AccumulatePixels(
+        SampleT             samples[PIXELS_PER_THREAD][NUM_CHANNELS],
+        bool                is_valid[PIXELS_PER_THREAD],
+        CounterT*           privatized_histograms[NUM_ACTIVE_CHANNELS],
+        Int2Type<false>     is_rle_compress)
+    {
+        #pragma unroll
+        for (int PIXEL = 0; PIXEL < PIXELS_PER_THREAD; ++PIXEL)
+        {
+            #pragma unroll
+            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+            {
+                int bin = -1;
+                privatized_decode_op[CHANNEL].template BinSelect<LOAD_MODIFIER>(samples[PIXEL][CHANNEL], bin, is_valid[PIXEL]);
+                if (bin >= 0)
+                    atomicAdd(privatized_histograms[CHANNEL] + bin, 1);
+            }
+        }
+    }
+
+
+    /**
+     * Accumulate pixel, specialized for smem privatized histogram
+     */
+    __device__ __forceinline__ void AccumulateSmemPixels(
+        SampleT             samples[PIXELS_PER_THREAD][NUM_CHANNELS],
+        bool                is_valid[PIXELS_PER_THREAD])
+    {
+        CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS];
+
+        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+            privatized_histograms[CHANNEL] = temp_storage.histograms[CHANNEL];
+
+        AccumulatePixels(samples, is_valid, privatized_histograms, Int2Type<IS_RLE_COMPRESS>());
+    }
+
+
+    /**
+     * Accumulate pixel, specialized for gmem privatized histogram
+     */
+    __device__ __forceinline__ void AccumulateGmemPixels(
+        SampleT             samples[PIXELS_PER_THREAD][NUM_CHANNELS],
+        bool                is_valid[PIXELS_PER_THREAD])
+    {
+        AccumulatePixels(samples, is_valid, d_privatized_histograms, Int2Type<IS_RLE_COMPRESS>());
+    }
+
+
+
+    //---------------------------------------------------------------------
+    // Tile loading
+    //---------------------------------------------------------------------
+
+    // Load full, aligned tile using pixel iterator (multi-channel)
+    template <int _NUM_ACTIVE_CHANNELS>
+    __device__ __forceinline__ void LoadFullAlignedTile(
+        OffsetT                         block_offset,
+        int                             valid_samples,
+        SampleT                         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
+        Int2Type<_NUM_ACTIVE_CHANNELS>  num_active_channels)
+    {
+        typedef PixelT AliasedPixels[PIXELS_PER_THREAD];
+
+        WrappedPixelIteratorT d_wrapped_pixels((PixelT*) (d_native_samples + block_offset));
+
+        // Load using a wrapped pixel iterator
+        BlockLoadPixelT(temp_storage.aliasable.pixel_load).Load(
+            d_wrapped_pixels,
+            reinterpret_cast<AliasedPixels&>(samples));
+    }
+
+    // Load full, aligned tile using quad iterator (single-channel)
+    __device__ __forceinline__ void LoadFullAlignedTile(
+        OffsetT                         block_offset,
+        int                             valid_samples,
+        SampleT                         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
+        Int2Type<1>                     num_active_channels)
+    {
+        typedef QuadT AliasedQuads[QUADS_PER_THREAD];
+
+        WrappedQuadIteratorT d_wrapped_quads((QuadT*) (d_native_samples + block_offset));
+
+        // Load using a wrapped quad iterator
+        BlockLoadQuadT(temp_storage.aliasable.quad_load).Load(
+            d_wrapped_quads,
+            reinterpret_cast<AliasedQuads&>(samples));
+    }
+
+    // Load full, aligned tile
+    __device__ __forceinline__ void LoadTile(
+        OffsetT         block_offset,
+        int             valid_samples,
+        SampleT         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
+        Int2Type<true>  is_full_tile,
+        Int2Type<true>  is_aligned)
+    {
+        LoadFullAlignedTile(block_offset, valid_samples, samples, Int2Type<NUM_ACTIVE_CHANNELS>());
+    }
+
+    // Load full, mis-aligned tile using sample iterator
+    __device__ __forceinline__ void LoadTile(
+        OffsetT         block_offset,
+        int             valid_samples,
+        SampleT         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
+        Int2Type<true>  is_full_tile,
+        Int2Type<false> is_aligned)
+    {
+        typedef SampleT AliasedSamples[SAMPLES_PER_THREAD];
+
+        // Load using sample iterator
+        BlockLoadSampleT(temp_storage.aliasable.sample_load).Load(
+            d_wrapped_samples + block_offset,
+            reinterpret_cast<AliasedSamples&>(samples));
+    }
+
+    // Load partially-full, aligned tile using the pixel iterator
+    __device__ __forceinline__ void LoadTile(
+        OffsetT         block_offset,
+        int             valid_samples,
+        SampleT         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
+        Int2Type<false> is_full_tile,
+        Int2Type<true>  is_aligned)
+    {
+        typedef PixelT AliasedPixels[PIXELS_PER_THREAD];
+
+        WrappedPixelIteratorT d_wrapped_pixels((PixelT*) (d_native_samples + block_offset));
+
+        int valid_pixels = valid_samples / NUM_CHANNELS;
+
+        // Load using a wrapped pixel iterator
+        BlockLoadPixelT(temp_storage.aliasable.pixel_load).Load(
+            d_wrapped_pixels,
+            reinterpret_cast<AliasedPixels&>(samples),
+            valid_pixels);
+    }
+
+    // Load partially-full, mis-aligned tile using sample iterator
+    __device__ __forceinline__ void LoadTile(
+        OffsetT         block_offset,
+        int             valid_samples,
+        SampleT         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
+        Int2Type<false> is_full_tile,
+        Int2Type<false> is_aligned)
+    {
+        typedef SampleT AliasedSamples[SAMPLES_PER_THREAD];
+
+        BlockLoadSampleT(temp_storage.aliasable.sample_load).Load(
+            d_wrapped_samples + block_offset,
+            reinterpret_cast<AliasedSamples&>(samples),
+            valid_samples);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Tile processing
+    //---------------------------------------------------------------------
+
+    // Consume a tile of data samples
+    template <
+        bool IS_ALIGNED,        // Whether the tile offset is aligned (quad-aligned for single-channel, pixel-aligned for multi-channel)
+        bool IS_FULL_TILE>      // Whether the tile is full
+    __device__ __forceinline__ void ConsumeTile(OffsetT block_offset, int valid_samples)
+    {
+        SampleT     samples[PIXELS_PER_THREAD][NUM_CHANNELS];
+        bool        is_valid[PIXELS_PER_THREAD];
+
+        // Load tile
+        LoadTile(
+            block_offset,
+            valid_samples,
+            samples,
+            Int2Type<IS_FULL_TILE>(),
+            Int2Type<IS_ALIGNED>());
+
+        // Set valid flags
+        #pragma unroll
+        for (int PIXEL = 0; PIXEL < PIXELS_PER_THREAD; ++PIXEL)
+            is_valid[PIXEL] = IS_FULL_TILE || (((threadIdx.x * PIXELS_PER_THREAD + PIXEL) * NUM_CHANNELS) < valid_samples);
+
+        // Accumulate samples
+#if CUB_PTX_ARCH >= 120
+        if (prefer_smem)
+            AccumulateSmemPixels(samples, is_valid);
+        else
+            AccumulateGmemPixels(samples, is_valid);
+#else
+        AccumulateGmemPixels(samples, is_valid);
+#endif
+
+    }
+
+
+    // Consume row tiles.  Specialized for work-stealing from queue
+    template <bool IS_ALIGNED>
+    __device__ __forceinline__ void ConsumeTiles(
+        OffsetT             num_row_pixels,             ///< The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                   ///< The number of rows in the region of interest
+        OffsetT             row_stride_samples,         ///< The number of samples between starts of consecutive rows in the region of interest
+        int                 tiles_per_row,              ///< Number of image tiles per row
+        GridQueue<int>      tile_queue,
+        Int2Type<true>      is_work_stealing)
+    {
+
+        int         num_tiles                   = num_rows * tiles_per_row;
+        int         tile_idx                    = (blockIdx.y  * gridDim.x) + blockIdx.x;
+        OffsetT     num_even_share_tiles        = gridDim.x * gridDim.y;
+
+        while (tile_idx < num_tiles)
+        {
+            int     row             = tile_idx / tiles_per_row;
+            int     col             = tile_idx - (row * tiles_per_row);
+            OffsetT row_offset      = row * row_stride_samples;
+            OffsetT col_offset      = (col * TILE_SAMPLES);
+            OffsetT tile_offset     = row_offset + col_offset;
+
+            if (col == tiles_per_row - 1)
+            {
+                // Consume a partially-full tile at the end of the row
+                OffsetT num_remaining = (num_row_pixels * NUM_CHANNELS) - col_offset;
+                ConsumeTile<IS_ALIGNED, false>(tile_offset, num_remaining);
+            } 
+            else
+            {
+                // Consume full tile
+                ConsumeTile<IS_ALIGNED, true>(tile_offset, TILE_SAMPLES);
+            }
+
+            CTA_SYNC();
+
+            // Get next tile
+            if (threadIdx.x == 0)
+                temp_storage.tile_idx = tile_queue.Drain(1) + num_even_share_tiles;
+
+            CTA_SYNC();
+
+            tile_idx = temp_storage.tile_idx;
+        }
+    }
+
+
+    // Consume row tiles.  Specialized for even-share (striped across thread blocks)
+    template <bool IS_ALIGNED>
+    __device__ __forceinline__ void ConsumeTiles(
+        OffsetT             num_row_pixels,             ///< The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                   ///< The number of rows in the region of interest
+        OffsetT             row_stride_samples,         ///< The number of samples between starts of consecutive rows in the region of interest
+        int                 tiles_per_row,              ///< Number of image tiles per row
+        GridQueue<int>      tile_queue,
+        Int2Type<false>     is_work_stealing)
+    {
+        for (int row = blockIdx.y; row < num_rows; row += gridDim.y)
+        {
+            OffsetT row_begin   = row * row_stride_samples;
+            OffsetT row_end     = row_begin + (num_row_pixels * NUM_CHANNELS);
+            OffsetT tile_offset = row_begin + (blockIdx.x * TILE_SAMPLES);
+
+            while (tile_offset < row_end)
+            {
+                OffsetT num_remaining = row_end - tile_offset;
+
+                if (num_remaining < TILE_SAMPLES)
+                {
+                    // Consume partial tile
+                    ConsumeTile<IS_ALIGNED, false>(tile_offset, num_remaining);
+                    break;
+                }
+
+                // Consume full tile
+                ConsumeTile<IS_ALIGNED, true>(tile_offset, TILE_SAMPLES);
+                tile_offset += gridDim.x * TILE_SAMPLES;
+            }
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Parameter extraction
+    //---------------------------------------------------------------------
+
+    // Return a native pixel pointer (specialized for CacheModifiedInputIterator types)
+    template <
+        CacheLoadModifier   _MODIFIER,
+        typename            _ValueT,
+        typename            _OffsetT>
+    __device__ __forceinline__ SampleT* NativePointer(CacheModifiedInputIterator<_MODIFIER, _ValueT, _OffsetT> itr)
+    {
+        return itr.ptr;
+    }
+
+    // Return a native pixel pointer (specialized for other types)
+    template <typename IteratorT>
+    __device__ __forceinline__ SampleT* NativePointer(IteratorT itr)
+    {
+        return NULL;
+    }
+
+
+
+    //---------------------------------------------------------------------
+    // Interface
+    //---------------------------------------------------------------------
+
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__ AgentHistogram(
+        TempStorage         &temp_storage,                                      ///< Reference to temp_storage
+        SampleIteratorT     d_samples,                                          ///< Input data to reduce
+        int                 (&num_output_bins)[NUM_ACTIVE_CHANNELS],            ///< The number bins per final output histogram
+        int                 (&num_privatized_bins)[NUM_ACTIVE_CHANNELS],        ///< The number bins per privatized histogram
+        CounterT*           (&d_output_histograms)[NUM_ACTIVE_CHANNELS],        ///< Reference to final output histograms
+        CounterT*           (&d_privatized_histograms)[NUM_ACTIVE_CHANNELS],    ///< Reference to privatized histograms
+        OutputDecodeOpT     (&output_decode_op)[NUM_ACTIVE_CHANNELS],           ///< The transform operator for determining output bin-ids from privatized counter indices, one for each channel
+        PrivatizedDecodeOpT (&privatized_decode_op)[NUM_ACTIVE_CHANNELS])       ///< The transform operator for determining privatized counter indices from samples, one for each channel
+    :
+        temp_storage(temp_storage.Alias()),
+        d_wrapped_samples(d_samples),
+        num_output_bins(num_output_bins),
+        num_privatized_bins(num_privatized_bins),
+        d_output_histograms(d_output_histograms),
+        privatized_decode_op(privatized_decode_op),
+        output_decode_op(output_decode_op),
+        d_native_samples(NativePointer(d_wrapped_samples)),
+        prefer_smem((MEM_PREFERENCE == SMEM) ?
+            true :                              // prefer smem privatized histograms
+            (MEM_PREFERENCE == GMEM) ?
+                false :                         // prefer gmem privatized histograms
+                blockIdx.x & 1)                 // prefer blended privatized histograms
+    {
+        int blockId = (blockIdx.y * gridDim.x) + blockIdx.x;
+
+        // Initialize the locations of this block's privatized histograms
+        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+            this->d_privatized_histograms[CHANNEL] = d_privatized_histograms[CHANNEL] + (blockId * num_privatized_bins[CHANNEL]);
+    }
+
+
+    /**
+     * Consume image
+     */
+    __device__ __forceinline__ void ConsumeTiles(
+        OffsetT             num_row_pixels,             ///< The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                   ///< The number of rows in the region of interest
+        OffsetT             row_stride_samples,         ///< The number of samples between starts of consecutive rows in the region of interest
+        int                 tiles_per_row,              ///< Number of image tiles per row
+        GridQueue<int>      tile_queue)                 ///< Queue descriptor for assigning tiles of work to thread blocks
+    {
+        // Check whether all row starting offsets are quad-aligned (in single-channel) or pixel-aligned (in multi-channel)
+        int     quad_mask           = AlignBytes<QuadT>::ALIGN_BYTES - 1;
+        int     pixel_mask          = AlignBytes<PixelT>::ALIGN_BYTES - 1;
+        size_t  row_bytes           = sizeof(SampleT) * row_stride_samples;
+
+        bool quad_aligned_rows      = (NUM_CHANNELS == 1) && (SAMPLES_PER_THREAD % 4 == 0) &&     // Single channel
+                                        ((size_t(d_native_samples) & quad_mask) == 0) &&        // ptr is quad-aligned
+                                        ((num_rows == 1) || ((row_bytes & quad_mask) == 0));    // number of row-samples is a multiple of the alignment of the quad
+
+        bool pixel_aligned_rows     = (NUM_CHANNELS > 1) &&                                     // Multi channel
+                                        ((size_t(d_native_samples) & pixel_mask) == 0) &&       // ptr is pixel-aligned
+                                        ((row_bytes & pixel_mask) == 0);                        // number of row-samples is a multiple of the alignment of the pixel
+
+        // Whether rows are aligned and can be vectorized
+        if ((d_native_samples != NULL) && (quad_aligned_rows || pixel_aligned_rows))
+            ConsumeTiles<true>(num_row_pixels, num_rows, row_stride_samples, tiles_per_row, tile_queue, Int2Type<IS_WORK_STEALING>());
+        else
+            ConsumeTiles<false>(num_row_pixels, num_rows, row_stride_samples, tiles_per_row, tile_queue, Int2Type<IS_WORK_STEALING>());
+    }
+
+
+    /**
+     * Initialize privatized bin counters.  Specialized for privatized shared-memory counters
+     */
+    __device__ __forceinline__ void InitBinCounters()
+    {
+        if (prefer_smem)
+            InitSmemBinCounters();
+        else
+            InitGmemBinCounters();
+    }
+
+
+    /**
+     * Store privatized histogram to device-accessible memory.  Specialized for privatized shared-memory counters
+     */
+    __device__ __forceinline__ void StoreOutput()
+    {
+        if (prefer_smem)
+            StoreSmemOutput();
+        else
+            StoreGmemOutput();
+    }
+
+
+};
+
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_radix_sort_downsweep.cuh b/third_party/cub/cub/agent/agent_radix_sort_downsweep.cuh
new file mode 100644
index 0000000..faea881
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_radix_sort_downsweep.cuh
@@ -0,0 +1,789 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * AgentRadixSortDownsweep implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort downsweep .
+ */
+
+
+#pragma once
+
+#include <stdint.h>
+
+#include "../thread/thread_load.cuh"
+#include "../block/block_load.cuh"
+#include "../block/block_store.cuh"
+#include "../block/block_radix_rank.cuh"
+#include "../block/block_exchange.cuh"
+#include "../util_type.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Radix ranking algorithm
+ */
+enum RadixRankAlgorithm
+{
+    RADIX_RANK_BASIC,
+    RADIX_RANK_MEMOIZE,
+    RADIX_RANK_MATCH
+};
+
+/**
+ * Parameterizable tuning policy type for AgentRadixSortDownsweep
+ */
+template <
+    int                         _BLOCK_THREADS,         ///< Threads per thread block
+    int                         _ITEMS_PER_THREAD,      ///< Items per thread (per tile of input)
+    BlockLoadAlgorithm          _LOAD_ALGORITHM,        ///< The BlockLoad algorithm to use
+    CacheLoadModifier           _LOAD_MODIFIER,         ///< Cache load modifier for reading keys (and values)
+    RadixRankAlgorithm          _RANK_ALGORITHM,        ///< The radix ranking algorithm to use
+    BlockScanAlgorithm          _SCAN_ALGORITHM,        ///< The block scan algorithm to use
+    int                         _RADIX_BITS>            ///< The number of radix bits, i.e., log2(bins)
+struct AgentRadixSortDownsweepPolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,           ///< Threads per thread block
+        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,        ///< Items per thread (per tile of input)
+        RADIX_BITS              = _RADIX_BITS,              ///< The number of radix bits, i.e., log2(bins)
+    };
+
+    static const BlockLoadAlgorithm  LOAD_ALGORITHM     = _LOAD_ALGORITHM;    ///< The BlockLoad algorithm to use
+    static const CacheLoadModifier   LOAD_MODIFIER      = _LOAD_MODIFIER;     ///< Cache load modifier for reading keys (and values)
+    static const RadixRankAlgorithm  RANK_ALGORITHM     = _RANK_ALGORITHM;    ///< The radix ranking algorithm to use
+    static const BlockScanAlgorithm  SCAN_ALGORITHM     = _SCAN_ALGORITHM;    ///< The BlockScan algorithm to use
+};
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+
+
+
+
+/**
+ * \brief AgentRadixSortDownsweep implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort downsweep .
+ */
+template <
+    typename AgentRadixSortDownsweepPolicy,     ///< Parameterized AgentRadixSortDownsweepPolicy tuning policy type
+    bool     IS_DESCENDING,                     ///< Whether or not the sorted-order is high-to-low
+    typename KeyT,                              ///< KeyT type
+    typename ValueT,                            ///< ValueT type
+    typename OffsetT>                           ///< Signed integer type for global offsets
+struct AgentRadixSortDownsweep
+{
+    //---------------------------------------------------------------------
+    // Type definitions and constants
+    //---------------------------------------------------------------------
+
+    // Appropriate unsigned-bits representation of KeyT
+    typedef typename Traits<KeyT>::UnsignedBits UnsignedBits;
+
+    static const UnsignedBits           LOWEST_KEY  = Traits<KeyT>::LOWEST_KEY;
+    static const UnsignedBits           MAX_KEY     = Traits<KeyT>::MAX_KEY;
+
+    static const BlockLoadAlgorithm     LOAD_ALGORITHM  = AgentRadixSortDownsweepPolicy::LOAD_ALGORITHM;
+    static const CacheLoadModifier      LOAD_MODIFIER   = AgentRadixSortDownsweepPolicy::LOAD_MODIFIER;
+    static const RadixRankAlgorithm     RANK_ALGORITHM  = AgentRadixSortDownsweepPolicy::RANK_ALGORITHM;
+    static const BlockScanAlgorithm     SCAN_ALGORITHM  = AgentRadixSortDownsweepPolicy::SCAN_ALGORITHM;
+
+    enum
+    {
+        BLOCK_THREADS           = AgentRadixSortDownsweepPolicy::BLOCK_THREADS,
+        ITEMS_PER_THREAD        = AgentRadixSortDownsweepPolicy::ITEMS_PER_THREAD,
+        RADIX_BITS              = AgentRadixSortDownsweepPolicy::RADIX_BITS,
+        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
+
+        RADIX_DIGITS            = 1 << RADIX_BITS,
+        KEYS_ONLY               = Equals<ValueT, NullType>::VALUE,
+    };
+
+    // Input iterator wrapper type (for applying cache modifier)s
+    typedef CacheModifiedInputIterator<LOAD_MODIFIER, UnsignedBits, OffsetT>    KeysItr;
+    typedef CacheModifiedInputIterator<LOAD_MODIFIER, ValueT, OffsetT>          ValuesItr;
+
+    // Radix ranking type to use
+    typedef typename If<(RANK_ALGORITHM == RADIX_RANK_BASIC),
+            BlockRadixRank<BLOCK_THREADS, RADIX_BITS, IS_DESCENDING, false, SCAN_ALGORITHM>,
+            typename If<(RANK_ALGORITHM == RADIX_RANK_MEMOIZE),
+                BlockRadixRank<BLOCK_THREADS, RADIX_BITS, IS_DESCENDING, true, SCAN_ALGORITHM>,
+                BlockRadixRankMatch<BLOCK_THREADS, RADIX_BITS, IS_DESCENDING, SCAN_ALGORITHM>
+            >::Type
+        >::Type BlockRadixRankT;
+
+    enum
+    {
+        /// Number of bin-starting offsets tracked per thread
+        BINS_TRACKED_PER_THREAD = BlockRadixRankT::BINS_TRACKED_PER_THREAD
+    };
+
+    // BlockLoad type (keys)
+    typedef BlockLoad<
+        UnsignedBits,
+        BLOCK_THREADS,
+        ITEMS_PER_THREAD,
+        LOAD_ALGORITHM> BlockLoadKeysT;
+
+    // BlockLoad type (values)
+    typedef BlockLoad<
+        ValueT,
+        BLOCK_THREADS,
+        ITEMS_PER_THREAD,
+        LOAD_ALGORITHM> BlockLoadValuesT;
+
+    // Value exchange array type
+    typedef ValueT ValueExchangeT[TILE_ITEMS];
+
+    /**
+     * Shared memory storage layout
+     */
+    union __align__(16) _TempStorage
+    {
+        typename BlockLoadKeysT::TempStorage    load_keys;
+        typename BlockLoadValuesT::TempStorage  load_values;
+        typename BlockRadixRankT::TempStorage   radix_rank;
+
+        struct
+        {
+            UnsignedBits                        exchange_keys[TILE_ITEMS];
+            OffsetT                             relative_bin_offsets[RADIX_DIGITS];
+        };
+
+        Uninitialized<ValueExchangeT>           exchange_values;
+
+        OffsetT                                 exclusive_digit_prefix[RADIX_DIGITS];
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Thread fields
+    //---------------------------------------------------------------------
+
+    // Shared storage for this CTA
+    _TempStorage    &temp_storage;
+
+    // Input and output device pointers
+    KeysItr         d_keys_in;
+    ValuesItr       d_values_in;
+    UnsignedBits    *d_keys_out;
+    ValueT          *d_values_out;
+
+    // The global scatter base offset for each digit (valid in the first RADIX_DIGITS threads)
+    OffsetT         bin_offset[BINS_TRACKED_PER_THREAD];
+
+    // The least-significant bit position of the current digit to extract
+    int             current_bit;
+
+    // Number of bits in current digit
+    int             num_bits;
+
+    // Whether to short-cirucit
+    int             short_circuit;
+
+    //---------------------------------------------------------------------
+    // Utility methods
+    //---------------------------------------------------------------------
+
+
+    /**
+     * Scatter ranked keys through shared memory, then to device-accessible memory
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__ void ScatterKeys(
+        UnsignedBits    (&twiddled_keys)[ITEMS_PER_THREAD],
+        OffsetT         (&relative_bin_offsets)[ITEMS_PER_THREAD],
+        int             (&ranks)[ITEMS_PER_THREAD],
+        OffsetT         valid_items)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            temp_storage.exchange_keys[ranks[ITEM]] = twiddled_keys[ITEM];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            UnsignedBits key            = temp_storage.exchange_keys[threadIdx.x + (ITEM * BLOCK_THREADS)];
+            UnsignedBits digit          = BFE(key, current_bit, num_bits);
+            relative_bin_offsets[ITEM]  = temp_storage.relative_bin_offsets[digit];
+
+            // Un-twiddle
+            key = Traits<KeyT>::TwiddleOut(key);
+
+            if (FULL_TILE || 
+                (static_cast<OffsetT>(threadIdx.x + (ITEM * BLOCK_THREADS)) < valid_items))
+            {
+                d_keys_out[relative_bin_offsets[ITEM] + threadIdx.x + (ITEM * BLOCK_THREADS)] = key;
+            }
+        }
+    }
+
+
+    /**
+     * Scatter ranked values through shared memory, then to device-accessible memory
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__ void ScatterValues(
+        ValueT      (&values)[ITEMS_PER_THREAD],
+        OffsetT     (&relative_bin_offsets)[ITEMS_PER_THREAD],
+        int         (&ranks)[ITEMS_PER_THREAD],
+        OffsetT     valid_items)
+    {
+        CTA_SYNC();
+
+        ValueExchangeT &exchange_values = temp_storage.exchange_values.Alias();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            exchange_values[ranks[ITEM]] = values[ITEM];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            ValueT value = exchange_values[threadIdx.x + (ITEM * BLOCK_THREADS)];
+
+            if (FULL_TILE ||
+                (static_cast<OffsetT>(threadIdx.x + (ITEM * BLOCK_THREADS)) < valid_items))
+            {
+                d_values_out[relative_bin_offsets[ITEM] + threadIdx.x + (ITEM * BLOCK_THREADS)] = value;
+            }
+        }
+    }
+
+    /**
+     * Load a tile of keys (specialized for full tile, any ranking algorithm)
+     */
+    template <int _RANK_ALGORITHM>
+    __device__ __forceinline__ void LoadKeys(
+        UnsignedBits                (&keys)[ITEMS_PER_THREAD],
+        OffsetT                     block_offset,
+        OffsetT                     valid_items,
+        UnsignedBits                oob_item,
+        Int2Type<true>              is_full_tile,
+        Int2Type<_RANK_ALGORITHM>   rank_algorithm)
+    {
+        BlockLoadKeysT(temp_storage.load_keys).Load(
+            d_keys_in + block_offset, keys);
+
+        CTA_SYNC();
+    }
+
+
+    /**
+     * Load a tile of keys (specialized for partial tile, any ranking algorithm)
+     */
+    template <int _RANK_ALGORITHM>
+    __device__ __forceinline__ void LoadKeys(
+        UnsignedBits                (&keys)[ITEMS_PER_THREAD],
+        OffsetT                     block_offset,
+        OffsetT                     valid_items,
+        UnsignedBits                oob_item,
+        Int2Type<false>             is_full_tile,
+        Int2Type<_RANK_ALGORITHM>   rank_algorithm)
+    {
+        // Register pressure work-around: moving valid_items through shfl prevents compiler
+        // from reusing guards/addressing from prior guarded loads
+        valid_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(valid_items, 0, 0xffffffff);
+
+        BlockLoadKeysT(temp_storage.load_keys).Load(
+            d_keys_in + block_offset, keys, valid_items, oob_item);
+
+        CTA_SYNC();
+    }
+
+
+    /**
+     * Load a tile of keys (specialized for full tile, match ranking algorithm)
+     */
+    __device__ __forceinline__ void LoadKeys(
+        UnsignedBits                (&keys)[ITEMS_PER_THREAD],
+        OffsetT                     block_offset,
+        OffsetT                     valid_items,
+        UnsignedBits                oob_item,
+        Int2Type<true>              is_full_tile,
+        Int2Type<RADIX_RANK_MATCH>  rank_algorithm)
+    {
+        LoadDirectWarpStriped(threadIdx.x, d_keys_in + block_offset, keys);
+    }
+
+
+    /**
+     * Load a tile of keys (specialized for partial tile, match ranking algorithm)
+     */
+    __device__ __forceinline__ void LoadKeys(
+        UnsignedBits                (&keys)[ITEMS_PER_THREAD],
+        OffsetT                     block_offset,
+        OffsetT                     valid_items,
+        UnsignedBits                oob_item,
+        Int2Type<false>             is_full_tile,
+        Int2Type<RADIX_RANK_MATCH>  rank_algorithm)
+    {
+        // Register pressure work-around: moving valid_items through shfl prevents compiler
+        // from reusing guards/addressing from prior guarded loads
+        valid_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(valid_items, 0, 0xffffffff);
+
+        LoadDirectWarpStriped(threadIdx.x, d_keys_in + block_offset, keys, valid_items, oob_item);
+    }
+
+
+    /**
+     * Load a tile of values (specialized for full tile, any ranking algorithm)
+     */
+    template <int _RANK_ALGORITHM>
+    __device__ __forceinline__ void LoadValues(
+        ValueT                      (&values)[ITEMS_PER_THREAD],
+        OffsetT                     block_offset,
+        OffsetT                     valid_items,
+        Int2Type<true>              is_full_tile,
+        Int2Type<_RANK_ALGORITHM>   rank_algorithm)
+    {
+        BlockLoadValuesT(temp_storage.load_values).Load(
+            d_values_in + block_offset, values);
+
+        CTA_SYNC();
+    }
+
+
+    /**
+     * Load a tile of values (specialized for partial tile, any ranking algorithm)
+     */
+    template <int _RANK_ALGORITHM>
+    __device__ __forceinline__ void LoadValues(
+        ValueT                      (&values)[ITEMS_PER_THREAD],
+        OffsetT                     block_offset,
+        OffsetT                     valid_items,
+        Int2Type<false>             is_full_tile,
+        Int2Type<_RANK_ALGORITHM>   rank_algorithm)
+    {
+        // Register pressure work-around: moving valid_items through shfl prevents compiler
+        // from reusing guards/addressing from prior guarded loads
+        valid_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(valid_items, 0, 0xffffffff);
+
+        BlockLoadValuesT(temp_storage.load_values).Load(
+            d_values_in + block_offset, values, valid_items);
+
+        CTA_SYNC();
+    }
+
+
+    /**
+     * Load a tile of items (specialized for full tile, match ranking algorithm)
+     */
+    __device__ __forceinline__ void LoadValues(
+        ValueT                      (&values)[ITEMS_PER_THREAD],
+        OffsetT                     block_offset,
+        OffsetT                     valid_items,
+        Int2Type<true>              is_full_tile,
+        Int2Type<RADIX_RANK_MATCH>  rank_algorithm)
+    {
+        LoadDirectWarpStriped(threadIdx.x, d_values_in + block_offset, values);
+    }
+
+
+    /**
+     * Load a tile of items (specialized for partial tile, match ranking algorithm)
+     */
+    __device__ __forceinline__ void LoadValues(
+        ValueT                      (&values)[ITEMS_PER_THREAD],
+        OffsetT                     block_offset,
+        OffsetT                     valid_items,
+        Int2Type<false>             is_full_tile,
+        Int2Type<RADIX_RANK_MATCH>  rank_algorithm)
+    {
+        // Register pressure work-around: moving valid_items through shfl prevents compiler
+        // from reusing guards/addressing from prior guarded loads
+        valid_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(valid_items, 0, 0xffffffff);
+
+        LoadDirectWarpStriped(threadIdx.x, d_values_in + block_offset, values, valid_items);
+    }
+
+
+    /**
+     * Truck along associated values
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__ void GatherScatterValues(
+        OffsetT         (&relative_bin_offsets)[ITEMS_PER_THREAD],
+        int             (&ranks)[ITEMS_PER_THREAD],
+        OffsetT         block_offset,
+        OffsetT         valid_items,
+        Int2Type<false> /*is_keys_only*/)
+    {
+        ValueT values[ITEMS_PER_THREAD];
+
+        CTA_SYNC();
+
+        LoadValues(
+            values,
+            block_offset,
+            valid_items,
+            Int2Type<FULL_TILE>(),
+            Int2Type<RANK_ALGORITHM>());
+
+        ScatterValues<FULL_TILE>(
+            values,
+            relative_bin_offsets,
+            ranks,
+            valid_items);
+    }
+
+
+    /**
+     * Truck along associated values (specialized for key-only sorting)
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__ void GatherScatterValues(
+        OffsetT         (&/*relative_bin_offsets*/)[ITEMS_PER_THREAD],
+        int             (&/*ranks*/)[ITEMS_PER_THREAD],
+        OffsetT         /*block_offset*/,
+        OffsetT         /*valid_items*/,
+        Int2Type<true>  /*is_keys_only*/)
+    {}
+
+
+    /**
+     * Process tile
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__ void ProcessTile(
+        OffsetT block_offset,
+        const OffsetT &valid_items = TILE_ITEMS)
+    {
+        UnsignedBits    keys[ITEMS_PER_THREAD];
+        int             ranks[ITEMS_PER_THREAD];
+        OffsetT         relative_bin_offsets[ITEMS_PER_THREAD];
+
+        // Assign default (min/max) value to all keys
+        UnsignedBits default_key = (IS_DESCENDING) ? LOWEST_KEY : MAX_KEY;
+
+        // Load tile of keys
+        LoadKeys(
+            keys,
+            block_offset,
+            valid_items, 
+            default_key,
+            Int2Type<FULL_TILE>(),
+            Int2Type<RANK_ALGORITHM>());
+
+        // Twiddle key bits if necessary
+        #pragma unroll
+        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
+        {
+            keys[KEY] = Traits<KeyT>::TwiddleIn(keys[KEY]);
+        }
+
+        // Rank the twiddled keys
+        int exclusive_digit_prefix[BINS_TRACKED_PER_THREAD];
+        BlockRadixRankT(temp_storage.radix_rank).RankKeys(
+            keys,
+            ranks,
+            current_bit,
+            num_bits,
+            exclusive_digit_prefix);
+
+        CTA_SYNC();
+
+        // Share exclusive digit prefix
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+            {
+                // Store exclusive prefix
+                temp_storage.exclusive_digit_prefix[bin_idx] =
+                    exclusive_digit_prefix[track];
+            }
+        }
+
+        CTA_SYNC();
+
+        // Get inclusive digit prefix
+        int inclusive_digit_prefix[BINS_TRACKED_PER_THREAD];
+
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+            {
+                if (IS_DESCENDING)
+                {
+                    // Get inclusive digit prefix from exclusive prefix (higher bins come first)
+                    inclusive_digit_prefix[track] = (bin_idx == 0) ?
+                        (BLOCK_THREADS * ITEMS_PER_THREAD) :
+                        temp_storage.exclusive_digit_prefix[bin_idx - 1];
+                }
+                else
+                {
+                    // Get inclusive digit prefix from exclusive prefix (lower bins come first)
+                    inclusive_digit_prefix[track] = (bin_idx == RADIX_DIGITS - 1) ?
+                        (BLOCK_THREADS * ITEMS_PER_THREAD) :
+                        temp_storage.exclusive_digit_prefix[bin_idx + 1];
+                }
+            }
+        }
+
+        CTA_SYNC();
+
+        // Update global scatter base offsets for each digit
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+            {
+                bin_offset[track] -= exclusive_digit_prefix[track];
+                temp_storage.relative_bin_offsets[bin_idx] = bin_offset[track];
+                bin_offset[track] += inclusive_digit_prefix[track];
+            }
+        }
+
+        CTA_SYNC();
+
+        // Scatter keys
+        ScatterKeys<FULL_TILE>(keys, relative_bin_offsets, ranks, valid_items);
+
+        // Gather/scatter values
+        GatherScatterValues<FULL_TILE>(relative_bin_offsets , ranks, block_offset, valid_items, Int2Type<KEYS_ONLY>());
+    }
+
+    //---------------------------------------------------------------------
+    // Copy shortcut
+    //---------------------------------------------------------------------
+
+    /**
+     * Copy tiles within the range of input
+     */
+    template <
+        typename InputIteratorT,
+        typename T>
+    __device__ __forceinline__ void Copy(
+        InputIteratorT  d_in,
+        T               *d_out,
+        OffsetT         block_offset,
+        OffsetT         block_end)
+    {
+        // Simply copy the input
+        while (block_offset + TILE_ITEMS <= block_end)
+        {
+            T items[ITEMS_PER_THREAD];
+
+            LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_in + block_offset, items);
+            CTA_SYNC();
+            StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_out + block_offset, items);
+
+            block_offset += TILE_ITEMS;
+        }
+
+        // Clean up last partial tile with guarded-I/O
+        if (block_offset < block_end)
+        {
+            OffsetT valid_items = block_end - block_offset;
+
+            T items[ITEMS_PER_THREAD];
+
+            LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_in + block_offset, items, valid_items);
+            CTA_SYNC();
+            StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_out + block_offset, items, valid_items);
+        }
+    }
+
+
+    /**
+     * Copy tiles within the range of input (specialized for NullType)
+     */
+    template <typename InputIteratorT>
+    __device__ __forceinline__ void Copy(
+        InputIteratorT  /*d_in*/,
+        NullType        * /*d_out*/,
+        OffsetT         /*block_offset*/,
+        OffsetT         /*block_end*/)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Interface
+    //---------------------------------------------------------------------
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__ AgentRadixSortDownsweep(
+        TempStorage     &temp_storage,
+        OffsetT         (&bin_offset)[BINS_TRACKED_PER_THREAD],
+        OffsetT         num_items,
+        const KeyT      *d_keys_in,
+        KeyT            *d_keys_out,
+        const ValueT    *d_values_in,
+        ValueT          *d_values_out,
+        int             current_bit,
+        int             num_bits)
+    :
+        temp_storage(temp_storage.Alias()),
+        d_keys_in(reinterpret_cast<const UnsignedBits*>(d_keys_in)),
+        d_values_in(d_values_in),
+        d_keys_out(reinterpret_cast<UnsignedBits*>(d_keys_out)),
+        d_values_out(d_values_out),
+        current_bit(current_bit),
+        num_bits(num_bits),
+        short_circuit(1)
+    {
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            this->bin_offset[track] = bin_offset[track];
+
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+            {
+                // Short circuit if the histogram has only bin counts of only zeros or problem-size
+                short_circuit = short_circuit && ((bin_offset[track] == 0) || (bin_offset[track] == num_items));
+            }
+        }
+
+        short_circuit = CTA_SYNC_AND(short_circuit);
+    }
+
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__ AgentRadixSortDownsweep(
+        TempStorage     &temp_storage,
+        OffsetT         num_items,
+        OffsetT         *d_spine,
+        const KeyT      *d_keys_in,
+        KeyT            *d_keys_out,
+        const ValueT    *d_values_in,
+        ValueT          *d_values_out,
+        int             current_bit,
+        int             num_bits)
+    :
+        temp_storage(temp_storage.Alias()),
+        d_keys_in(reinterpret_cast<const UnsignedBits*>(d_keys_in)),
+        d_values_in(d_values_in),
+        d_keys_out(reinterpret_cast<UnsignedBits*>(d_keys_out)),
+        d_values_out(d_values_out),
+        current_bit(current_bit),
+        num_bits(num_bits),
+        short_circuit(1)
+    {
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+
+            // Load digit bin offsets (each of the first RADIX_DIGITS threads will load an offset for that digit)
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+            {
+                if (IS_DESCENDING)
+                    bin_idx = RADIX_DIGITS - bin_idx - 1;
+
+                // Short circuit if the first block's histogram has only bin counts of only zeros or problem-size
+                OffsetT first_block_bin_offset = d_spine[gridDim.x * bin_idx];
+                short_circuit = short_circuit && ((first_block_bin_offset == 0) || (first_block_bin_offset == num_items));
+
+                // Load my block's bin offset for my bin
+                bin_offset[track] = d_spine[(gridDim.x * bin_idx) + blockIdx.x];
+            }
+        }
+
+        short_circuit = CTA_SYNC_AND(short_circuit);
+    }
+
+
+    /**
+     * Distribute keys from a segment of input tiles.
+     */
+    __device__ __forceinline__ void ProcessRegion(
+        OffsetT   block_offset,
+        OffsetT   block_end)
+    {
+        if (short_circuit)
+        {
+            // Copy keys
+            Copy(d_keys_in, d_keys_out, block_offset, block_end);
+
+            // Copy values
+            Copy(d_values_in, d_values_out, block_offset, block_end);
+        }
+        else
+        {
+            // Process full tiles of tile_items
+            #pragma unroll 1
+            while (block_offset + TILE_ITEMS <= block_end)
+            {
+                ProcessTile<true>(block_offset);
+                block_offset += TILE_ITEMS;
+
+                CTA_SYNC();
+            }
+
+            // Clean up last partial tile with guarded-I/O
+            if (block_offset < block_end)
+            {
+                ProcessTile<false>(block_offset, block_end - block_offset);
+            }
+
+        }
+    }
+
+};
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_radix_sort_upsweep.cuh b/third_party/cub/cub/agent/agent_radix_sort_upsweep.cuh
new file mode 100644
index 0000000..2081cef
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_radix_sort_upsweep.cuh
@@ -0,0 +1,526 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * AgentRadixSortUpsweep implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort upsweep .
+ */
+
+#pragma once
+
+#include "../thread/thread_reduce.cuh"
+#include "../thread/thread_load.cuh"
+#include "../warp/warp_reduce.cuh"
+#include "../block/block_load.cuh"
+#include "../util_type.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for AgentRadixSortUpsweep
+ */
+template <
+    int                 _BLOCK_THREADS,     ///< Threads per thread block
+    int                 _ITEMS_PER_THREAD,  ///< Items per thread (per tile of input)
+    CacheLoadModifier   _LOAD_MODIFIER,     ///< Cache load modifier for reading keys
+    int                 _RADIX_BITS>        ///< The number of radix bits, i.e., log2(bins)
+struct AgentRadixSortUpsweepPolicy
+{
+    enum
+    {
+        BLOCK_THREADS       = _BLOCK_THREADS,       ///< Threads per thread block
+        ITEMS_PER_THREAD    = _ITEMS_PER_THREAD,    ///< Items per thread (per tile of input)
+        RADIX_BITS          = _RADIX_BITS,          ///< The number of radix bits, i.e., log2(bins)
+    };
+
+    static const CacheLoadModifier LOAD_MODIFIER = _LOAD_MODIFIER;      ///< Cache load modifier for reading keys
+};
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+/**
+ * \brief AgentRadixSortUpsweep implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort upsweep .
+ */
+template <
+    typename AgentRadixSortUpsweepPolicy,   ///< Parameterized AgentRadixSortUpsweepPolicy tuning policy type
+    typename KeyT,                          ///< KeyT type
+    typename OffsetT>                       ///< Signed integer type for global offsets
+struct AgentRadixSortUpsweep
+{
+
+    //---------------------------------------------------------------------
+    // Type definitions and constants
+    //---------------------------------------------------------------------
+
+    typedef typename Traits<KeyT>::UnsignedBits UnsignedBits;
+
+    // Integer type for digit counters (to be packed into words of PackedCounters)
+    typedef unsigned char DigitCounter;
+
+    // Integer type for packing DigitCounters into columns of shared memory banks
+    typedef unsigned int PackedCounter;
+
+    static const CacheLoadModifier LOAD_MODIFIER = AgentRadixSortUpsweepPolicy::LOAD_MODIFIER;
+
+    enum
+    {
+        RADIX_BITS              = AgentRadixSortUpsweepPolicy::RADIX_BITS,
+        BLOCK_THREADS           = AgentRadixSortUpsweepPolicy::BLOCK_THREADS,
+        KEYS_PER_THREAD         = AgentRadixSortUpsweepPolicy::ITEMS_PER_THREAD,
+
+        RADIX_DIGITS            = 1 << RADIX_BITS,
+
+        LOG_WARP_THREADS        = CUB_PTX_LOG_WARP_THREADS,
+        WARP_THREADS            = 1 << LOG_WARP_THREADS,
+        WARPS                   = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+
+        TILE_ITEMS              = BLOCK_THREADS * KEYS_PER_THREAD,
+
+        BYTES_PER_COUNTER       = sizeof(DigitCounter),
+        LOG_BYTES_PER_COUNTER   = Log2<BYTES_PER_COUNTER>::VALUE,
+
+        PACKING_RATIO           = sizeof(PackedCounter) / sizeof(DigitCounter),
+        LOG_PACKING_RATIO       = Log2<PACKING_RATIO>::VALUE,
+
+        LOG_COUNTER_LANES       = CUB_MAX(0, RADIX_BITS - LOG_PACKING_RATIO),
+        COUNTER_LANES           = 1 << LOG_COUNTER_LANES,
+
+        // To prevent counter overflow, we must periodically unpack and aggregate the
+        // digit counters back into registers.  Each counter lane is assigned to a
+        // warp for aggregation.
+
+        LANES_PER_WARP          = CUB_MAX(1, (COUNTER_LANES + WARPS - 1) / WARPS),
+
+        // Unroll tiles in batches without risk of counter overflow
+        UNROLL_COUNT            = CUB_MIN(64, 255 / KEYS_PER_THREAD),
+        UNROLLED_ELEMENTS       = UNROLL_COUNT * TILE_ITEMS,
+    };
+
+
+    // Input iterator wrapper type (for applying cache modifier)s
+    typedef CacheModifiedInputIterator<LOAD_MODIFIER, UnsignedBits, OffsetT> KeysItr;
+
+    /**
+     * Shared memory storage layout
+     */
+    union __align__(16) _TempStorage
+    {
+        DigitCounter    thread_counters[COUNTER_LANES][BLOCK_THREADS][PACKING_RATIO];
+        PackedCounter   packed_thread_counters[COUNTER_LANES][BLOCK_THREADS];
+        OffsetT         block_counters[WARP_THREADS][RADIX_DIGITS];
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Thread fields (aggregate state bundle)
+    //---------------------------------------------------------------------
+
+    // Shared storage for this CTA
+    _TempStorage    &temp_storage;
+
+    // Thread-local counters for periodically aggregating composite-counter lanes
+    OffsetT         local_counts[LANES_PER_WARP][PACKING_RATIO];
+
+    // Input and output device pointers
+    KeysItr         d_keys_in;
+
+    // The least-significant bit position of the current digit to extract
+    int             current_bit;
+
+    // Number of bits in current digit
+    int             num_bits;
+
+
+
+    //---------------------------------------------------------------------
+    // Helper structure for templated iteration
+    //---------------------------------------------------------------------
+
+    // Iterate
+    template <int COUNT, int MAX>
+    struct Iterate
+    {
+        // BucketKeys
+        static __device__ __forceinline__ void BucketKeys(
+            AgentRadixSortUpsweep       &cta,
+            UnsignedBits                keys[KEYS_PER_THREAD])
+        {
+            cta.Bucket(keys[COUNT]);
+
+            // Next
+            Iterate<COUNT + 1, MAX>::BucketKeys(cta, keys);
+        }
+    };
+
+    // Terminate
+    template <int MAX>
+    struct Iterate<MAX, MAX>
+    {
+        // BucketKeys
+        static __device__ __forceinline__ void BucketKeys(AgentRadixSortUpsweep &/*cta*/, UnsignedBits /*keys*/[KEYS_PER_THREAD]) {}
+    };
+
+
+    //---------------------------------------------------------------------
+    // Utility methods
+    //---------------------------------------------------------------------
+
+    /**
+     * Decode a key and increment corresponding smem digit counter
+     */
+    __device__ __forceinline__ void Bucket(UnsignedBits key)
+    {
+        // Perform transform op
+        UnsignedBits converted_key = Traits<KeyT>::TwiddleIn(key);
+
+        // Extract current digit bits
+        UnsignedBits digit = BFE(converted_key, current_bit, num_bits);
+
+        // Get sub-counter offset
+        UnsignedBits sub_counter = digit & (PACKING_RATIO - 1);
+
+        // Get row offset
+        UnsignedBits row_offset = digit >> LOG_PACKING_RATIO;
+
+        // Increment counter
+        temp_storage.thread_counters[row_offset][threadIdx.x][sub_counter]++;
+    }
+
+
+    /**
+     * Reset composite counters
+     */
+    __device__ __forceinline__ void ResetDigitCounters()
+    {
+        #pragma unroll
+        for (int LANE = 0; LANE < COUNTER_LANES; LANE++)
+        {
+            temp_storage.packed_thread_counters[LANE][threadIdx.x] = 0;
+        }
+    }
+
+
+    /**
+     * Reset the unpacked counters in each thread
+     */
+    __device__ __forceinline__ void ResetUnpackedCounters()
+    {
+        #pragma unroll
+        for (int LANE = 0; LANE < LANES_PER_WARP; LANE++)
+        {
+            #pragma unroll
+            for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++)
+            {
+                local_counts[LANE][UNPACKED_COUNTER] = 0;
+            }
+        }
+    }
+
+
+    /**
+     * Extracts and aggregates the digit counters for each counter lane
+     * owned by this warp
+     */
+    __device__ __forceinline__ void UnpackDigitCounts()
+    {
+        unsigned int warp_id = threadIdx.x >> LOG_WARP_THREADS;
+        unsigned int warp_tid = LaneId();
+
+        #pragma unroll
+        for (int LANE = 0; LANE < LANES_PER_WARP; LANE++)
+        {
+            const int counter_lane = (LANE * WARPS) + warp_id;
+            if (counter_lane < COUNTER_LANES)
+            {
+                #pragma unroll
+                for (int PACKED_COUNTER = 0; PACKED_COUNTER < BLOCK_THREADS; PACKED_COUNTER += WARP_THREADS)
+                {
+                    #pragma unroll
+                    for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++)
+                    {
+                        OffsetT counter = temp_storage.thread_counters[counter_lane][warp_tid + PACKED_COUNTER][UNPACKED_COUNTER];
+                        local_counts[LANE][UNPACKED_COUNTER] += counter;
+                    }
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Processes a single, full tile
+     */
+    __device__ __forceinline__ void ProcessFullTile(OffsetT block_offset)
+    {
+        // Tile of keys
+        UnsignedBits keys[KEYS_PER_THREAD];
+
+        LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_keys_in + block_offset, keys);
+
+        // Prevent hoisting
+        CTA_SYNC();
+
+        // Bucket tile of keys
+        Iterate<0, KEYS_PER_THREAD>::BucketKeys(*this, keys);
+    }
+
+
+    /**
+     * Processes a single load (may have some threads masked off)
+     */
+    __device__ __forceinline__ void ProcessPartialTile(
+        OffsetT block_offset,
+        const OffsetT &block_end)
+    {
+        // Process partial tile if necessary using single loads
+        block_offset += threadIdx.x;
+        while (block_offset < block_end)
+        {
+            // Load and bucket key
+            UnsignedBits key = d_keys_in[block_offset];
+            Bucket(key);
+            block_offset += BLOCK_THREADS;
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Interface
+    //---------------------------------------------------------------------
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__ AgentRadixSortUpsweep(
+        TempStorage &temp_storage,
+        const KeyT  *d_keys_in,
+        int         current_bit,
+        int         num_bits)
+    :
+        temp_storage(temp_storage.Alias()),
+        d_keys_in(reinterpret_cast<const UnsignedBits*>(d_keys_in)),
+        current_bit(current_bit),
+        num_bits(num_bits)
+    {}
+
+
+    /**
+     * Compute radix digit histograms from a segment of input tiles.
+     */
+    __device__ __forceinline__ void ProcessRegion(
+        OffsetT          block_offset,
+        const OffsetT    &block_end)
+    {
+        // Reset digit counters in smem and unpacked counters in registers
+        ResetDigitCounters();
+        ResetUnpackedCounters();
+
+        // Unroll batches of full tiles
+        while (block_offset + UNROLLED_ELEMENTS <= block_end)
+        {
+            for (int i = 0; i < UNROLL_COUNT; ++i)
+            {
+                ProcessFullTile(block_offset);
+                block_offset += TILE_ITEMS;
+            }
+
+            CTA_SYNC();
+
+            // Aggregate back into local_count registers to prevent overflow
+            UnpackDigitCounts();
+
+            CTA_SYNC();
+
+            // Reset composite counters in lanes
+            ResetDigitCounters();
+        }
+
+        // Unroll single full tiles
+        while (block_offset + TILE_ITEMS <= block_end)
+        {
+            ProcessFullTile(block_offset);
+            block_offset += TILE_ITEMS;
+        }
+
+        // Process partial tile if necessary
+        ProcessPartialTile(
+            block_offset,
+            block_end);
+
+        CTA_SYNC();
+
+        // Aggregate back into local_count registers
+        UnpackDigitCounts();
+    }
+
+
+    /**
+     * Extract counts (saving them to the external array)
+     */
+    template <bool IS_DESCENDING>
+    __device__ __forceinline__ void ExtractCounts(
+        OffsetT     *counters,
+        int         bin_stride = 1,
+        int         bin_offset = 0)
+    {
+        unsigned int warp_id    = threadIdx.x >> LOG_WARP_THREADS;
+        unsigned int warp_tid   = LaneId();
+
+        // Place unpacked digit counters in shared memory
+        #pragma unroll
+        for (int LANE = 0; LANE < LANES_PER_WARP; LANE++)
+        {
+            int counter_lane = (LANE * WARPS) + warp_id;
+            if (counter_lane < COUNTER_LANES)
+            {
+                int digit_row = counter_lane << LOG_PACKING_RATIO;
+
+                #pragma unroll
+                for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++)
+                {
+                    int bin_idx = digit_row + UNPACKED_COUNTER;
+
+                    temp_storage.block_counters[warp_tid][bin_idx] =
+                        local_counts[LANE][UNPACKED_COUNTER];
+                }
+            }
+        }
+
+        CTA_SYNC();
+
+        // Rake-reduce bin_count reductions
+
+        // Whole blocks
+        #pragma unroll
+        for (int BIN_BASE   = RADIX_DIGITS % BLOCK_THREADS;
+            (BIN_BASE + BLOCK_THREADS) <= RADIX_DIGITS;
+            BIN_BASE += BLOCK_THREADS)
+        {
+            int bin_idx = BIN_BASE + threadIdx.x;
+
+            OffsetT bin_count = 0;
+            #pragma unroll
+            for (int i = 0; i < WARP_THREADS; ++i)
+                bin_count += temp_storage.block_counters[i][bin_idx];
+
+            if (IS_DESCENDING)
+                bin_idx = RADIX_DIGITS - bin_idx - 1;
+
+            counters[(bin_stride * bin_idx) + bin_offset] = bin_count;
+        }
+
+        // Remainder
+        if ((RADIX_DIGITS % BLOCK_THREADS != 0) && (threadIdx.x < RADIX_DIGITS))
+        {
+            int bin_idx = threadIdx.x;
+
+            OffsetT bin_count = 0;
+            #pragma unroll
+            for (int i = 0; i < WARP_THREADS; ++i)
+                bin_count += temp_storage.block_counters[i][bin_idx];
+
+            if (IS_DESCENDING)
+                bin_idx = RADIX_DIGITS - bin_idx - 1;
+
+            counters[(bin_stride * bin_idx) + bin_offset] = bin_count;
+        }
+    }
+
+
+    /**
+     * Extract counts
+     */
+    template <int BINS_TRACKED_PER_THREAD>
+    __device__ __forceinline__ void ExtractCounts(
+        OffsetT (&bin_count)[BINS_TRACKED_PER_THREAD])  ///< [out] The exclusive prefix sum for the digits [(threadIdx.x * BINS_TRACKED_PER_THREAD) ... (threadIdx.x * BINS_TRACKED_PER_THREAD) + BINS_TRACKED_PER_THREAD - 1]
+    {
+        unsigned int warp_id    = threadIdx.x >> LOG_WARP_THREADS;
+        unsigned int warp_tid   = LaneId();
+
+        // Place unpacked digit counters in shared memory
+        #pragma unroll
+        for (int LANE = 0; LANE < LANES_PER_WARP; LANE++)
+        {
+            int counter_lane = (LANE * WARPS) + warp_id;
+            if (counter_lane < COUNTER_LANES)
+            {
+                int digit_row = counter_lane << LOG_PACKING_RATIO;
+
+                #pragma unroll
+                for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++)
+                {
+                    int bin_idx = digit_row + UNPACKED_COUNTER;
+
+                    temp_storage.block_counters[warp_tid][bin_idx] =
+                        local_counts[LANE][UNPACKED_COUNTER];
+                }
+            }
+        }
+
+        CTA_SYNC();
+
+        // Rake-reduce bin_count reductions
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+            {
+                bin_count[track] = 0;
+
+                #pragma unroll
+                for (int i = 0; i < WARP_THREADS; ++i)
+                    bin_count[track] += temp_storage.block_counters[i][bin_idx];
+            }
+        }
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_reduce.cuh b/third_party/cub/cub/agent/agent_reduce.cuh
new file mode 100644
index 0000000..000a905
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_reduce.cuh
@@ -0,0 +1,385 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::AgentReduce implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduction .
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "../block/block_load.cuh"
+#include "../block/block_reduce.cuh"
+#include "../grid/grid_mapping.cuh"
+#include "../grid/grid_even_share.cuh"
+#include "../util_type.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for AgentReduce
+ */
+template <
+    int                     _BLOCK_THREADS,         ///< Threads per thread block
+    int                     _ITEMS_PER_THREAD,      ///< Items per thread (per tile of input)
+    int                     _VECTOR_LOAD_LENGTH,    ///< Number of items per vectorized load
+    BlockReduceAlgorithm    _BLOCK_ALGORITHM,       ///< Cooperative block-wide reduction algorithm to use
+    CacheLoadModifier       _LOAD_MODIFIER>         ///< Cache load modifier for reading input elements
+struct AgentReducePolicy
+{
+    enum
+    {
+        BLOCK_THREADS       = _BLOCK_THREADS,       ///< Threads per thread block
+        ITEMS_PER_THREAD    = _ITEMS_PER_THREAD,    ///< Items per thread (per tile of input)
+        VECTOR_LOAD_LENGTH  = _VECTOR_LOAD_LENGTH,  ///< Number of items per vectorized load
+    };
+
+    static const BlockReduceAlgorithm  BLOCK_ALGORITHM      = _BLOCK_ALGORITHM;     ///< Cooperative block-wide reduction algorithm to use
+    static const CacheLoadModifier     LOAD_MODIFIER        = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
+};
+
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+/**
+ * \brief AgentReduce implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduction .
+ *
+ * Each thread reduces only the values it loads. If \p FIRST_TILE, this
+ * partial reduction is stored into \p thread_aggregate.  Otherwise it is
+ * accumulated into \p thread_aggregate.
+ */
+template <
+    typename AgentReducePolicy,        ///< Parameterized AgentReducePolicy tuning policy type
+    typename InputIteratorT,           ///< Random-access iterator type for input
+    typename OutputIteratorT,          ///< Random-access iterator type for output
+    typename OffsetT,                  ///< Signed integer type for global offsets
+    typename ReductionOp>              ///< Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+struct AgentReduce
+{
+
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    /// The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    /// Vector type of InputT for data movement
+    typedef typename CubVector<InputT, AgentReducePolicy::VECTOR_LOAD_LENGTH>::Type VectorT;
+
+    /// Input iterator wrapper type (for applying cache modifier)
+    typedef typename If<IsPointer<InputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentReducePolicy::LOAD_MODIFIER, InputT, OffsetT>,      // Wrap the native input pointer with CacheModifiedInputIterator
+            InputIteratorT>::Type                                                               // Directly use the supplied input iterator type
+        WrappedInputIteratorT;
+
+    /// Constants
+    enum
+    {
+        BLOCK_THREADS       = AgentReducePolicy::BLOCK_THREADS,
+        ITEMS_PER_THREAD    = AgentReducePolicy::ITEMS_PER_THREAD,
+        VECTOR_LOAD_LENGTH  = CUB_MIN(ITEMS_PER_THREAD, AgentReducePolicy::VECTOR_LOAD_LENGTH),
+        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
+
+        // Can vectorize according to the policy if the input iterator is a native pointer to a primitive type
+        ATTEMPT_VECTORIZATION   = (VECTOR_LOAD_LENGTH > 1) &&
+                                    (ITEMS_PER_THREAD % VECTOR_LOAD_LENGTH == 0) &&
+                                    (IsPointer<InputIteratorT>::VALUE) && Traits<InputT>::PRIMITIVE,
+
+    };
+
+    static const CacheLoadModifier    LOAD_MODIFIER   = AgentReducePolicy::LOAD_MODIFIER;
+    static const BlockReduceAlgorithm BLOCK_ALGORITHM = AgentReducePolicy::BLOCK_ALGORITHM;
+
+    /// Parameterized BlockReduce primitive
+    typedef BlockReduce<OutputT, BLOCK_THREADS, AgentReducePolicy::BLOCK_ALGORITHM> BlockReduceT;
+
+    /// Shared memory type required by this thread block
+    struct _TempStorage
+    {
+        typename BlockReduceT::TempStorage  reduce;
+    };
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    _TempStorage&           temp_storage;       ///< Reference to temp_storage
+    InputIteratorT          d_in;               ///< Input data to reduce
+    WrappedInputIteratorT   d_wrapped_in;       ///< Wrapped input data to reduce
+    ReductionOp             reduction_op;       ///< Binary reduction operator
+
+
+    //---------------------------------------------------------------------
+    // Utility
+    //---------------------------------------------------------------------
+
+
+    // Whether or not the input is aligned with the vector type (specialized for types we can vectorize)
+    template <typename Iterator>
+    static __device__ __forceinline__ bool IsAligned(
+        Iterator        d_in,
+        Int2Type<true>  /*can_vectorize*/)
+    {
+        return (size_t(d_in) & (sizeof(VectorT) - 1)) == 0;
+    }
+
+    // Whether or not the input is aligned with the vector type (specialized for types we cannot vectorize)
+    template <typename Iterator>
+    static __device__ __forceinline__ bool IsAligned(
+        Iterator        /*d_in*/,
+        Int2Type<false> /*can_vectorize*/)
+    {
+        return false;
+    }
+
+
+    //---------------------------------------------------------------------
+    // Constructor
+    //---------------------------------------------------------------------
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__ AgentReduce(
+        TempStorage&            temp_storage,       ///< Reference to temp_storage
+        InputIteratorT          d_in,               ///< Input data to reduce
+        ReductionOp             reduction_op)       ///< Binary reduction operator
+    :
+        temp_storage(temp_storage.Alias()),
+        d_in(d_in),
+        d_wrapped_in(d_in),
+        reduction_op(reduction_op)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Tile consumption
+    //---------------------------------------------------------------------
+
+    /**
+     * Consume a full tile of input (non-vectorized)
+     */
+    template <int IS_FIRST_TILE>
+    __device__ __forceinline__ void ConsumeTile(
+        OutputT                 &thread_aggregate,
+        OffsetT                 block_offset,       ///< The offset the tile to consume
+        int                     /*valid_items*/,    ///< The number of valid items in the tile
+        Int2Type<true>          /*is_full_tile*/,   ///< Whether or not this is a full tile
+        Int2Type<false>         /*can_vectorize*/)  ///< Whether or not we can vectorize loads
+    {
+        OutputT items[ITEMS_PER_THREAD];
+
+        // Load items in striped fashion
+        LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_wrapped_in + block_offset, items);
+
+        // Reduce items within each thread stripe
+        thread_aggregate = (IS_FIRST_TILE) ?
+            internal::ThreadReduce(items, reduction_op) :
+            internal::ThreadReduce(items, reduction_op, thread_aggregate);
+    }
+
+
+    /**
+     * Consume a full tile of input (vectorized)
+     */
+    template <int IS_FIRST_TILE>
+    __device__ __forceinline__ void ConsumeTile(
+        OutputT                 &thread_aggregate,
+        OffsetT                 block_offset,       ///< The offset the tile to consume
+        int                     /*valid_items*/,    ///< The number of valid items in the tile
+        Int2Type<true>          /*is_full_tile*/,   ///< Whether or not this is a full tile
+        Int2Type<true>          /*can_vectorize*/)  ///< Whether or not we can vectorize loads
+    {
+        // Alias items as an array of VectorT and load it in striped fashion
+        enum { WORDS =  ITEMS_PER_THREAD / VECTOR_LOAD_LENGTH };
+
+        // Fabricate a vectorized input iterator
+        InputT *d_in_unqualified = const_cast<InputT*>(d_in) + block_offset + (threadIdx.x * VECTOR_LOAD_LENGTH);
+        CacheModifiedInputIterator<AgentReducePolicy::LOAD_MODIFIER, VectorT, OffsetT> d_vec_in(
+            reinterpret_cast<VectorT*>(d_in_unqualified));
+
+        // Load items as vector items
+        InputT input_items[ITEMS_PER_THREAD];
+        VectorT *vec_items = reinterpret_cast<VectorT*>(input_items);
+        #pragma unroll
+        for (int i = 0; i < WORDS; ++i)
+            vec_items[i] = d_vec_in[BLOCK_THREADS * i];
+
+        // Convert from input type to output type
+        OutputT items[ITEMS_PER_THREAD];
+        #pragma unroll
+        for (int i = 0; i < ITEMS_PER_THREAD; ++i)
+            items[i] = input_items[i];
+
+        // Reduce items within each thread stripe
+        thread_aggregate = (IS_FIRST_TILE) ?
+            internal::ThreadReduce(items, reduction_op) :
+            internal::ThreadReduce(items, reduction_op, thread_aggregate);
+    }
+
+
+    /**
+     * Consume a partial tile of input
+     */
+    template <int IS_FIRST_TILE, int CAN_VECTORIZE>
+    __device__ __forceinline__ void ConsumeTile(
+        OutputT                 &thread_aggregate,
+        OffsetT                 block_offset,       ///< The offset the tile to consume
+        int                     valid_items,        ///< The number of valid items in the tile
+        Int2Type<false>         /*is_full_tile*/,   ///< Whether or not this is a full tile
+        Int2Type<CAN_VECTORIZE> /*can_vectorize*/)  ///< Whether or not we can vectorize loads
+    {
+        // Partial tile
+        int thread_offset = threadIdx.x;
+
+        // Read first item
+        if ((IS_FIRST_TILE) && (thread_offset < valid_items))
+        {
+            thread_aggregate = d_wrapped_in[block_offset + thread_offset];
+            thread_offset += BLOCK_THREADS;
+        }
+
+        // Continue reading items (block-striped)
+        while (thread_offset < valid_items)
+        {
+            OutputT item        = d_wrapped_in[block_offset + thread_offset];
+            thread_aggregate    = reduction_op(thread_aggregate, item);
+            thread_offset       += BLOCK_THREADS;
+        }
+    }
+
+
+    //---------------------------------------------------------------
+    // Consume a contiguous segment of tiles
+    //---------------------------------------------------------------------
+
+    /**
+     * \brief Reduce a contiguous segment of input tiles
+     */
+    template <int CAN_VECTORIZE>
+    __device__ __forceinline__ OutputT ConsumeRange(
+        GridEvenShare<OffsetT> &even_share,          ///< GridEvenShare descriptor
+        Int2Type<CAN_VECTORIZE> can_vectorize)      ///< Whether or not we can vectorize loads
+    {
+        OutputT thread_aggregate;
+
+        if (even_share.block_offset + TILE_ITEMS > even_share.block_end)
+        {
+            // First tile isn't full (not all threads have valid items)
+            int valid_items = even_share.block_end - even_share.block_offset;
+            ConsumeTile<true>(thread_aggregate, even_share.block_offset, valid_items, Int2Type<false>(), can_vectorize);
+            return BlockReduceT(temp_storage.reduce).Reduce(thread_aggregate, reduction_op, valid_items);
+        }
+
+        // At least one full block
+        ConsumeTile<true>(thread_aggregate, even_share.block_offset, TILE_ITEMS, Int2Type<true>(), can_vectorize);
+        even_share.block_offset += even_share.block_stride;
+
+        // Consume subsequent full tiles of input
+        while (even_share.block_offset + TILE_ITEMS <= even_share.block_end)
+        {
+            ConsumeTile<false>(thread_aggregate, even_share.block_offset, TILE_ITEMS, Int2Type<true>(), can_vectorize);
+            even_share.block_offset += even_share.block_stride;
+        }
+
+        // Consume a partially-full tile
+        if (even_share.block_offset < even_share.block_end)
+        {
+            int valid_items = even_share.block_end - even_share.block_offset;
+            ConsumeTile<false>(thread_aggregate, even_share.block_offset, valid_items, Int2Type<false>(), can_vectorize);
+        }
+
+        // Compute block-wide reduction (all threads have valid items)
+        return BlockReduceT(temp_storage.reduce).Reduce(thread_aggregate, reduction_op);
+    }
+
+
+    /**
+     * \brief Reduce a contiguous segment of input tiles
+     */
+    __device__ __forceinline__ OutputT ConsumeRange(
+        OffsetT block_offset,                       ///< [in] Threadblock begin offset (inclusive)
+        OffsetT block_end)                          ///< [in] Threadblock end offset (exclusive)
+    {
+        GridEvenShare<OffsetT> even_share;
+        even_share.template BlockInit<TILE_ITEMS>(block_offset, block_end);
+
+        return (IsAligned(d_in + block_offset, Int2Type<ATTEMPT_VECTORIZATION>())) ?
+            ConsumeRange(even_share, Int2Type<true && ATTEMPT_VECTORIZATION>()) :
+            ConsumeRange(even_share, Int2Type<false && ATTEMPT_VECTORIZATION>());
+    }
+
+
+    /**
+     * Reduce a contiguous segment of input tiles
+     */
+    __device__ __forceinline__ OutputT ConsumeTiles(
+        GridEvenShare<OffsetT> &even_share)        ///< [in] GridEvenShare descriptor
+    {
+        // Initialize GRID_MAPPING_STRIP_MINE even-share descriptor for this thread block
+        even_share.template BlockInit<TILE_ITEMS, GRID_MAPPING_STRIP_MINE>();
+
+        return (IsAligned(d_in, Int2Type<ATTEMPT_VECTORIZATION>())) ?
+            ConsumeRange(even_share, Int2Type<true && ATTEMPT_VECTORIZATION>()) :
+            ConsumeRange(even_share, Int2Type<false && ATTEMPT_VECTORIZATION>());
+
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_reduce_by_key.cuh b/third_party/cub/cub/agent/agent_reduce_by_key.cuh
new file mode 100644
index 0000000..51964d3
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_reduce_by_key.cuh
@@ -0,0 +1,547 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::AgentReduceByKey implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key.
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "single_pass_scan_operators.cuh"
+#include "../block/block_load.cuh"
+#include "../block/block_store.cuh"
+#include "../block/block_scan.cuh"
+#include "../block/block_discontinuity.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../iterator/constant_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for AgentReduceByKey
+ */
+template <
+    int                         _BLOCK_THREADS,                 ///< Threads per thread block
+    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
+    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
+    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
+    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
+struct AgentReduceByKeyPolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
+        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
+    };
+
+    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
+    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
+    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
+};
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+/**
+ * \brief AgentReduceByKey implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key
+ */
+template <
+    typename    AgentReduceByKeyPolicyT,        ///< Parameterized AgentReduceByKeyPolicy tuning policy type
+    typename    KeysInputIteratorT,             ///< Random-access input iterator type for keys
+    typename    UniqueOutputIteratorT,          ///< Random-access output iterator type for keys
+    typename    ValuesInputIteratorT,           ///< Random-access input iterator type for values
+    typename    AggregatesOutputIteratorT,      ///< Random-access output iterator type for values
+    typename    NumRunsOutputIteratorT,         ///< Output iterator type for recording number of items selected
+    typename    EqualityOpT,                    ///< KeyT equality operator type
+    typename    ReductionOpT,                   ///< ValueT reduction operator type
+    typename    OffsetT>                        ///< Signed integer type for global offsets
+struct AgentReduceByKey
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    // The input keys type
+    typedef typename std::iterator_traits<KeysInputIteratorT>::value_type KeyInputT;
+
+    // The output keys type
+    typedef typename If<(Equals<typename std::iterator_traits<UniqueOutputIteratorT>::value_type, void>::VALUE),    // KeyOutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<KeysInputIteratorT>::value_type,                                              // ... then the input iterator's value type,
+        typename std::iterator_traits<UniqueOutputIteratorT>::value_type>::Type KeyOutputT;                         // ... else the output iterator's value type
+
+    // The input values type
+    typedef typename std::iterator_traits<ValuesInputIteratorT>::value_type ValueInputT;
+
+    // The output values type
+    typedef typename If<(Equals<typename std::iterator_traits<AggregatesOutputIteratorT>::value_type, void>::VALUE),    // ValueOutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<ValuesInputIteratorT>::value_type,                                                // ... then the input iterator's value type,
+        typename std::iterator_traits<AggregatesOutputIteratorT>::value_type>::Type ValueOutputT;                       // ... else the output iterator's value type
+
+    // Tuple type for scanning (pairs accumulated segment-value with segment-index)
+    typedef KeyValuePair<OffsetT, ValueOutputT> OffsetValuePairT;
+
+    // Tuple type for pairing keys and values
+    typedef KeyValuePair<KeyOutputT, ValueOutputT> KeyValuePairT;
+
+    // Tile status descriptor interface type
+    typedef ReduceByKeyScanTileState<ValueOutputT, OffsetT> ScanTileStateT;
+
+    // Guarded inequality functor
+    template <typename _EqualityOpT>
+    struct GuardedInequalityWrapper
+    {
+        _EqualityOpT     op;             ///< Wrapped equality operator
+        int             num_remaining;  ///< Items remaining
+
+        /// Constructor
+        __host__ __device__ __forceinline__
+        GuardedInequalityWrapper(_EqualityOpT op, int num_remaining) : op(op), num_remaining(num_remaining) {}
+
+        /// Boolean inequality operator, returns <tt>(a != b)</tt>
+        template <typename T>
+        __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b, int idx) const
+        {
+            if (idx < num_remaining)
+                return !op(a, b);   // In bounds
+
+            // Return true if first out-of-bounds item, false otherwise
+            return (idx == num_remaining);
+       }
+    };
+
+
+    // Constants
+    enum
+    {
+        BLOCK_THREADS       = AgentReduceByKeyPolicyT::BLOCK_THREADS,
+        ITEMS_PER_THREAD    = AgentReduceByKeyPolicyT::ITEMS_PER_THREAD,
+        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
+        TWO_PHASE_SCATTER   = (ITEMS_PER_THREAD > 1),
+
+        // Whether or not the scan operation has a zero-valued identity value (true if we're performing addition on a primitive type)
+        HAS_IDENTITY_ZERO   = (Equals<ReductionOpT, cub::Sum>::VALUE) && (Traits<ValueOutputT>::PRIMITIVE),
+    };
+
+    // Cache-modified Input iterator wrapper type (for applying cache modifier) for keys
+    typedef typename If<IsPointer<KeysInputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentReduceByKeyPolicyT::LOAD_MODIFIER, KeyInputT, OffsetT>,     // Wrap the native input pointer with CacheModifiedValuesInputIterator
+            KeysInputIteratorT>::Type                                                                   // Directly use the supplied input iterator type
+        WrappedKeysInputIteratorT;
+
+    // Cache-modified Input iterator wrapper type (for applying cache modifier) for values
+    typedef typename If<IsPointer<ValuesInputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentReduceByKeyPolicyT::LOAD_MODIFIER, ValueInputT, OffsetT>,   // Wrap the native input pointer with CacheModifiedValuesInputIterator
+            ValuesInputIteratorT>::Type                                                                 // Directly use the supplied input iterator type
+        WrappedValuesInputIteratorT;
+
+    // Cache-modified Input iterator wrapper type (for applying cache modifier) for fixup values
+    typedef typename If<IsPointer<AggregatesOutputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentReduceByKeyPolicyT::LOAD_MODIFIER, ValueInputT, OffsetT>,   // Wrap the native input pointer with CacheModifiedValuesInputIterator
+            AggregatesOutputIteratorT>::Type                                                            // Directly use the supplied input iterator type
+        WrappedFixupInputIteratorT;
+
+    // Reduce-value-by-segment scan operator
+    typedef ReduceBySegmentOp<ReductionOpT> ReduceBySegmentOpT;
+
+    // Parameterized BlockLoad type for keys
+    typedef BlockLoad<
+            KeyOutputT,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            AgentReduceByKeyPolicyT::LOAD_ALGORITHM>
+        BlockLoadKeysT;
+
+    // Parameterized BlockLoad type for values
+    typedef BlockLoad<
+            ValueOutputT,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            AgentReduceByKeyPolicyT::LOAD_ALGORITHM>
+        BlockLoadValuesT;
+
+    // Parameterized BlockDiscontinuity type for keys
+    typedef BlockDiscontinuity<
+            KeyOutputT,
+            BLOCK_THREADS>
+        BlockDiscontinuityKeys;
+
+    // Parameterized BlockScan type
+    typedef BlockScan<
+            OffsetValuePairT,
+            BLOCK_THREADS,
+            AgentReduceByKeyPolicyT::SCAN_ALGORITHM>
+        BlockScanT;
+
+    // Callback type for obtaining tile prefix during block scan
+    typedef TilePrefixCallbackOp<
+            OffsetValuePairT,
+            ReduceBySegmentOpT,
+            ScanTileStateT>
+        TilePrefixCallbackOpT;
+
+    // Key and value exchange types
+    typedef KeyOutputT    KeyExchangeT[TILE_ITEMS + 1];
+    typedef ValueOutputT  ValueExchangeT[TILE_ITEMS + 1];
+
+    // Shared memory type for this thread block
+    union _TempStorage
+    {
+        struct
+        {
+            typename BlockScanT::TempStorage                scan;           // Smem needed for tile scanning
+            typename TilePrefixCallbackOpT::TempStorage     prefix;         // Smem needed for cooperative prefix callback
+            typename BlockDiscontinuityKeys::TempStorage    discontinuity;  // Smem needed for discontinuity detection
+        };
+
+        // Smem needed for loading keys
+        typename BlockLoadKeysT::TempStorage load_keys;
+
+        // Smem needed for loading values
+        typename BlockLoadValuesT::TempStorage load_values;
+
+        // Smem needed for compacting key value pairs(allows non POD items in this union)
+        Uninitialized<KeyValuePairT[TILE_ITEMS + 1]> raw_exchange;
+    };
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    _TempStorage&                   temp_storage;       ///< Reference to temp_storage
+    WrappedKeysInputIteratorT       d_keys_in;          ///< Input keys
+    UniqueOutputIteratorT           d_unique_out;       ///< Unique output keys
+    WrappedValuesInputIteratorT     d_values_in;        ///< Input values
+    AggregatesOutputIteratorT       d_aggregates_out;   ///< Output value aggregates
+    NumRunsOutputIteratorT          d_num_runs_out;     ///< Output pointer for total number of segments identified
+    EqualityOpT                     equality_op;        ///< KeyT equality operator
+    ReductionOpT                    reduction_op;       ///< Reduction operator
+    ReduceBySegmentOpT              scan_op;            ///< Reduce-by-segment scan operator
+
+
+    //---------------------------------------------------------------------
+    // Constructor
+    //---------------------------------------------------------------------
+
+    // Constructor
+    __device__ __forceinline__
+    AgentReduceByKey(
+        TempStorage&                temp_storage,       ///< Reference to temp_storage
+        KeysInputIteratorT          d_keys_in,          ///< Input keys
+        UniqueOutputIteratorT       d_unique_out,       ///< Unique output keys
+        ValuesInputIteratorT        d_values_in,        ///< Input values
+        AggregatesOutputIteratorT   d_aggregates_out,   ///< Output value aggregates
+        NumRunsOutputIteratorT      d_num_runs_out,     ///< Output pointer for total number of segments identified
+        EqualityOpT                 equality_op,        ///< KeyT equality operator
+        ReductionOpT                reduction_op)       ///< ValueT reduction operator
+    :
+        temp_storage(temp_storage.Alias()),
+        d_keys_in(d_keys_in),
+        d_unique_out(d_unique_out),
+        d_values_in(d_values_in),
+        d_aggregates_out(d_aggregates_out),
+        d_num_runs_out(d_num_runs_out),
+        equality_op(equality_op),
+        reduction_op(reduction_op),
+        scan_op(reduction_op)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Scatter utility methods
+    //---------------------------------------------------------------------
+
+    /**
+     * Directly scatter flagged items to output offsets
+     */
+    __device__ __forceinline__ void ScatterDirect(
+        KeyValuePairT   (&scatter_items)[ITEMS_PER_THREAD],
+        OffsetT         (&segment_flags)[ITEMS_PER_THREAD],
+        OffsetT         (&segment_indices)[ITEMS_PER_THREAD])
+    {
+        // Scatter flagged keys and values
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            if (segment_flags[ITEM])
+            {
+                d_unique_out[segment_indices[ITEM]]     = scatter_items[ITEM].key;
+                d_aggregates_out[segment_indices[ITEM]] = scatter_items[ITEM].value;
+            }
+        }
+    }
+
+
+    /**
+     * 2-phase scatter flagged items to output offsets
+     *
+     * The exclusive scan causes each head flag to be paired with the previous
+     * value aggregate: the scatter offsets must be decremented for value aggregates
+     */
+    __device__ __forceinline__ void ScatterTwoPhase(
+        KeyValuePairT   (&scatter_items)[ITEMS_PER_THREAD],
+        OffsetT         (&segment_flags)[ITEMS_PER_THREAD],
+        OffsetT         (&segment_indices)[ITEMS_PER_THREAD],
+        OffsetT         num_tile_segments,
+        OffsetT         num_tile_segments_prefix)
+    {
+        CTA_SYNC();
+
+        // Compact and scatter pairs
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            if (segment_flags[ITEM])
+            {
+                temp_storage.raw_exchange.Alias()[segment_indices[ITEM] - num_tile_segments_prefix] = scatter_items[ITEM];
+            }
+        }
+
+        CTA_SYNC();
+
+        for (int item = threadIdx.x; item < num_tile_segments; item += BLOCK_THREADS)
+        {
+            KeyValuePairT pair                                  = temp_storage.raw_exchange.Alias()[item];
+            d_unique_out[num_tile_segments_prefix + item]       = pair.key;
+            d_aggregates_out[num_tile_segments_prefix + item]   = pair.value;
+        }
+    }
+
+
+    /**
+     * Scatter flagged items
+     */
+    __device__ __forceinline__ void Scatter(
+        KeyValuePairT   (&scatter_items)[ITEMS_PER_THREAD],
+        OffsetT         (&segment_flags)[ITEMS_PER_THREAD],
+        OffsetT         (&segment_indices)[ITEMS_PER_THREAD],
+        OffsetT         num_tile_segments,
+        OffsetT         num_tile_segments_prefix)
+    {
+        // Do a one-phase scatter if (a) two-phase is disabled or (b) the average number of selected items per thread is less than one
+        if (TWO_PHASE_SCATTER && (num_tile_segments > BLOCK_THREADS))
+        {
+            ScatterTwoPhase(
+                scatter_items,
+                segment_flags,
+                segment_indices,
+                num_tile_segments,
+                num_tile_segments_prefix);
+        }
+        else
+        {
+            ScatterDirect(
+                scatter_items,
+                segment_flags,
+                segment_indices);
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Cooperatively scan a device-wide sequence of tiles with other CTAs
+    //---------------------------------------------------------------------
+
+    /**
+     * Process a tile of input (dynamic chained scan)
+     */
+    template <bool IS_LAST_TILE>                ///< Whether the current tile is the last tile
+    __device__ __forceinline__ void ConsumeTile(
+        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
+        int                 tile_idx,           ///< Tile index
+        OffsetT             tile_offset,        ///< Tile offset
+        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
+    {
+        KeyOutputT          keys[ITEMS_PER_THREAD];             // Tile keys
+        KeyOutputT          prev_keys[ITEMS_PER_THREAD];        // Tile keys shuffled up
+        ValueOutputT        values[ITEMS_PER_THREAD];           // Tile values
+        OffsetT             head_flags[ITEMS_PER_THREAD];       // Segment head flags
+        OffsetT             segment_indices[ITEMS_PER_THREAD];  // Segment indices
+        OffsetValuePairT    scan_items[ITEMS_PER_THREAD];       // Zipped values and segment flags|indices
+        KeyValuePairT       scatter_items[ITEMS_PER_THREAD];    // Zipped key value pairs for scattering
+
+        // Load keys
+        if (IS_LAST_TILE)
+            BlockLoadKeysT(temp_storage.load_keys).Load(d_keys_in + tile_offset, keys, num_remaining);
+        else
+            BlockLoadKeysT(temp_storage.load_keys).Load(d_keys_in + tile_offset, keys);
+
+        // Load tile predecessor key in first thread
+        KeyOutputT tile_predecessor;
+        if (threadIdx.x == 0)
+        {
+            tile_predecessor = (tile_idx == 0) ?
+                keys[0] :                       // First tile gets repeat of first item (thus first item will not be flagged as a head)
+                d_keys_in[tile_offset - 1];     // Subsequent tiles get last key from previous tile
+        }
+
+        CTA_SYNC();
+
+        // Load values
+        if (IS_LAST_TILE)
+            BlockLoadValuesT(temp_storage.load_values).Load(d_values_in + tile_offset, values, num_remaining);
+        else
+            BlockLoadValuesT(temp_storage.load_values).Load(d_values_in + tile_offset, values);
+
+        CTA_SYNC();
+
+        // Initialize head-flags and shuffle up the previous keys
+        if (IS_LAST_TILE)
+        {
+            // Use custom flag operator to additionally flag the first out-of-bounds item
+            GuardedInequalityWrapper<EqualityOpT> flag_op(equality_op, num_remaining);
+            BlockDiscontinuityKeys(temp_storage.discontinuity).FlagHeads(
+                head_flags, keys, prev_keys, flag_op, tile_predecessor);
+        }
+        else
+        {
+            InequalityWrapper<EqualityOpT> flag_op(equality_op);
+            BlockDiscontinuityKeys(temp_storage.discontinuity).FlagHeads(
+                head_flags, keys, prev_keys, flag_op, tile_predecessor);
+        }
+
+        // Zip values and head flags
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            scan_items[ITEM].value  = values[ITEM];
+            scan_items[ITEM].key    = head_flags[ITEM];
+        }
+
+        // Perform exclusive tile scan
+        OffsetValuePairT    block_aggregate;        // Inclusive block-wide scan aggregate
+        OffsetT             num_segments_prefix;    // Number of segments prior to this tile
+        OffsetValuePairT    total_aggregate;        // The tile prefix folded with block_aggregate
+        if (tile_idx == 0)
+        {
+            // Scan first tile
+            BlockScanT(temp_storage.scan).ExclusiveScan(scan_items, scan_items, scan_op, block_aggregate);
+            num_segments_prefix     = 0;
+            total_aggregate         = block_aggregate;
+
+            // Update tile status if there are successor tiles
+            if ((!IS_LAST_TILE) && (threadIdx.x == 0))
+                tile_state.SetInclusive(0, block_aggregate);
+        }
+        else
+        {
+            // Scan non-first tile
+            TilePrefixCallbackOpT prefix_op(tile_state, temp_storage.prefix, scan_op, tile_idx);
+            BlockScanT(temp_storage.scan).ExclusiveScan(scan_items, scan_items, scan_op, prefix_op);
+
+            block_aggregate         = prefix_op.GetBlockAggregate();
+            num_segments_prefix     = prefix_op.GetExclusivePrefix().key;
+            total_aggregate         = prefix_op.GetInclusivePrefix();
+        }
+
+        // Rezip scatter items and segment indices
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            scatter_items[ITEM].key     = prev_keys[ITEM];
+            scatter_items[ITEM].value   = scan_items[ITEM].value;
+            segment_indices[ITEM]       = scan_items[ITEM].key;
+        }
+
+        // At this point, each flagged segment head has:
+        //  - The key for the previous segment
+        //  - The reduced value from the previous segment
+        //  - The segment index for the reduced value
+
+        // Scatter flagged keys and values
+        OffsetT num_tile_segments = block_aggregate.key;
+        Scatter(scatter_items, head_flags, segment_indices, num_tile_segments, num_segments_prefix);
+
+        // Last thread in last tile will output final count (and last pair, if necessary)
+        if ((IS_LAST_TILE) && (threadIdx.x == BLOCK_THREADS - 1))
+        {
+            OffsetT num_segments = num_segments_prefix + num_tile_segments;
+
+            // If the last tile is a whole tile, output the final_value
+            if (num_remaining == TILE_ITEMS)
+            {
+                d_unique_out[num_segments]      = keys[ITEMS_PER_THREAD - 1];
+                d_aggregates_out[num_segments]  = total_aggregate.value;
+                num_segments++;
+            }
+
+            // Output the total number of items selected
+            *d_num_runs_out = num_segments;
+        }
+    }
+
+
+    /**
+     * Scan tiles of items as part of a dynamic chained scan
+     */
+    __device__ __forceinline__ void ConsumeRange(
+        int                 num_items,          ///< Total number of input items
+        ScanTileStateT&     tile_state,         ///< Global tile state descriptor
+        int                 start_tile)         ///< The starting tile for the current grid
+    {
+        // Blocks are launched in increasing order, so just assign one tile per block
+        int     tile_idx        = start_tile + blockIdx.x;          // Current tile index
+        OffsetT tile_offset     = OffsetT(TILE_ITEMS) * tile_idx;   // Global offset for the current tile
+        OffsetT num_remaining   = num_items - tile_offset;          // Remaining items (including this tile)
+
+        if (num_remaining > TILE_ITEMS)
+        {
+            // Not last tile
+            ConsumeTile<false>(num_remaining, tile_idx, tile_offset, tile_state);
+        }
+        else if (num_remaining > 0)
+        {
+            // Last tile
+            ConsumeTile<true>(num_remaining, tile_idx, tile_offset, tile_state);
+        }
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_rle.cuh b/third_party/cub/cub/agent/agent_rle.cuh
new file mode 100644
index 0000000..cb7a4a6
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_rle.cuh
@@ -0,0 +1,837 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::AgentRle implements a stateful abstraction of CUDA thread blocks for participating in device-wide run-length-encode.
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "single_pass_scan_operators.cuh"
+#include "../block/block_load.cuh"
+#include "../block/block_store.cuh"
+#include "../block/block_scan.cuh"
+#include "../block/block_exchange.cuh"
+#include "../block/block_discontinuity.cuh"
+#include "../grid/grid_queue.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../iterator/constant_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for AgentRle
+ */
+template <
+    int                         _BLOCK_THREADS,                 ///< Threads per thread block
+    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
+    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
+    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
+    bool                        _STORE_WARP_TIME_SLICING,       ///< Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any store-related data transpositions (versus each warp having its own storage)
+    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
+struct AgentRlePolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
+        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
+        STORE_WARP_TIME_SLICING = _STORE_WARP_TIME_SLICING,     ///< Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any store-related data transpositions (versus each warp having its own storage)
+    };
+
+    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
+    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
+    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
+};
+
+
+
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+/**
+ * \brief AgentRle implements a stateful abstraction of CUDA thread blocks for participating in device-wide run-length-encode 
+ */
+template <
+    typename    AgentRlePolicyT,        ///< Parameterized AgentRlePolicyT tuning policy type
+    typename    InputIteratorT,         ///< Random-access input iterator type for data
+    typename    OffsetsOutputIteratorT, ///< Random-access output iterator type for offset values
+    typename    LengthsOutputIteratorT, ///< Random-access output iterator type for length values
+    typename    EqualityOpT,            ///< T equality operator type
+    typename    OffsetT>                ///< Signed integer type for global offsets
+struct AgentRle
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type T;
+
+    /// The lengths output value type
+    typedef typename If<(Equals<typename std::iterator_traits<LengthsOutputIteratorT>::value_type, void>::VALUE),   // LengthT =  (if output iterator's value type is void) ?
+        OffsetT,                                                                                                    // ... then the OffsetT type,
+        typename std::iterator_traits<LengthsOutputIteratorT>::value_type>::Type LengthT;                           // ... else the output iterator's value type
+
+    /// Tuple type for scanning (pairs run-length and run-index)
+    typedef KeyValuePair<OffsetT, LengthT> LengthOffsetPair;
+
+    /// Tile status descriptor interface type
+    typedef ReduceByKeyScanTileState<LengthT, OffsetT> ScanTileStateT;
+
+    // Constants
+    enum
+    {
+        WARP_THREADS            = CUB_WARP_THREADS(PTX_ARCH),
+        BLOCK_THREADS           = AgentRlePolicyT::BLOCK_THREADS,
+        ITEMS_PER_THREAD        = AgentRlePolicyT::ITEMS_PER_THREAD,
+        WARP_ITEMS              = WARP_THREADS * ITEMS_PER_THREAD,
+        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
+        WARPS                   = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+
+        /// Whether or not to sync after loading data
+        SYNC_AFTER_LOAD         = (AgentRlePolicyT::LOAD_ALGORITHM != BLOCK_LOAD_DIRECT),
+
+        /// Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any store-related data transpositions (versus each warp having its own storage)
+        STORE_WARP_TIME_SLICING = AgentRlePolicyT::STORE_WARP_TIME_SLICING,
+        ACTIVE_EXCHANGE_WARPS   = (STORE_WARP_TIME_SLICING) ? 1 : WARPS,
+    };
+
+
+    /**
+     * Special operator that signals all out-of-bounds items are not equal to everything else,
+     * forcing both (1) the last item to be tail-flagged and (2) all oob items to be marked
+     * trivial.
+     */
+    template <bool LAST_TILE>
+    struct OobInequalityOp
+    {
+        OffsetT         num_remaining;
+        EqualityOpT      equality_op;
+
+        __device__ __forceinline__ OobInequalityOp(
+            OffsetT     num_remaining,
+            EqualityOpT  equality_op)
+        :
+            num_remaining(num_remaining),
+            equality_op(equality_op)
+        {}
+
+        template <typename Index>
+        __host__ __device__ __forceinline__ bool operator()(T first, T second, Index idx)
+        {
+            if (!LAST_TILE || (idx < num_remaining))
+                return !equality_op(first, second);
+            else
+                return true;
+        }
+    };
+
+
+    // Cache-modified Input iterator wrapper type (for applying cache modifier) for data
+    typedef typename If<IsPointer<InputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentRlePolicyT::LOAD_MODIFIER, T, OffsetT>,      // Wrap the native input pointer with CacheModifiedVLengthnputIterator
+            InputIteratorT>::Type                                                       // Directly use the supplied input iterator type
+        WrappedInputIteratorT;
+
+    // Parameterized BlockLoad type for data
+    typedef BlockLoad<
+            T,
+            AgentRlePolicyT::BLOCK_THREADS,
+            AgentRlePolicyT::ITEMS_PER_THREAD,
+            AgentRlePolicyT::LOAD_ALGORITHM>
+        BlockLoadT;
+
+    // Parameterized BlockDiscontinuity type for data
+    typedef BlockDiscontinuity<T, BLOCK_THREADS> BlockDiscontinuityT;
+
+    // Parameterized WarpScan type
+    typedef WarpScan<LengthOffsetPair> WarpScanPairs;
+
+    // Reduce-length-by-run scan operator
+    typedef ReduceBySegmentOp<cub::Sum> ReduceBySegmentOpT;
+
+    // Callback type for obtaining tile prefix during block scan
+    typedef TilePrefixCallbackOp<
+            LengthOffsetPair,
+            ReduceBySegmentOpT,
+            ScanTileStateT>
+        TilePrefixCallbackOpT;
+
+    // Warp exchange types
+    typedef WarpExchange<LengthOffsetPair, ITEMS_PER_THREAD>        WarpExchangePairs;
+
+    typedef typename If<STORE_WARP_TIME_SLICING, typename WarpExchangePairs::TempStorage, NullType>::Type WarpExchangePairsStorage;
+
+    typedef WarpExchange<OffsetT, ITEMS_PER_THREAD>                 WarpExchangeOffsets;
+    typedef WarpExchange<LengthT, ITEMS_PER_THREAD>                 WarpExchangeLengths;
+
+    typedef LengthOffsetPair WarpAggregates[WARPS];
+
+    // Shared memory type for this thread block
+    struct _TempStorage
+    {
+        // Aliasable storage layout
+        union Aliasable
+        {
+            struct
+            {
+                typename BlockDiscontinuityT::TempStorage       discontinuity;              // Smem needed for discontinuity detection
+                typename WarpScanPairs::TempStorage             warp_scan[WARPS];           // Smem needed for warp-synchronous scans
+                Uninitialized<LengthOffsetPair[WARPS]>          warp_aggregates;            // Smem needed for sharing warp-wide aggregates
+                typename TilePrefixCallbackOpT::TempStorage     prefix;                     // Smem needed for cooperative prefix callback
+            };
+
+            // Smem needed for input loading
+            typename BlockLoadT::TempStorage                    load;
+
+            // Aliasable layout needed for two-phase scatter
+            union ScatterAliasable
+            {
+                unsigned long long                              align;
+                WarpExchangePairsStorage                        exchange_pairs[ACTIVE_EXCHANGE_WARPS];
+                typename WarpExchangeOffsets::TempStorage       exchange_offsets[ACTIVE_EXCHANGE_WARPS];
+                typename WarpExchangeLengths::TempStorage       exchange_lengths[ACTIVE_EXCHANGE_WARPS];
+
+            } scatter_aliasable;
+
+        } aliasable;
+
+        OffsetT             tile_idx;                   // Shared tile index
+        LengthOffsetPair    tile_inclusive;             // Inclusive tile prefix
+        LengthOffsetPair    tile_exclusive;             // Exclusive tile prefix
+    };
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    _TempStorage&                   temp_storage;       ///< Reference to temp_storage
+
+    WrappedInputIteratorT           d_in;               ///< Pointer to input sequence of data items
+    OffsetsOutputIteratorT          d_offsets_out;      ///< Input run offsets
+    LengthsOutputIteratorT          d_lengths_out;      ///< Output run lengths
+
+    EqualityOpT                     equality_op;        ///< T equality operator
+    ReduceBySegmentOpT              scan_op;            ///< Reduce-length-by-flag scan operator
+    OffsetT                         num_items;          ///< Total number of input items
+
+
+    //---------------------------------------------------------------------
+    // Constructor
+    //---------------------------------------------------------------------
+
+    // Constructor
+    __device__ __forceinline__
+    AgentRle(
+        TempStorage                 &temp_storage,      ///< [in] Reference to temp_storage
+        InputIteratorT              d_in,               ///< [in] Pointer to input sequence of data items
+        OffsetsOutputIteratorT      d_offsets_out,      ///< [out] Pointer to output sequence of run offsets
+        LengthsOutputIteratorT      d_lengths_out,      ///< [out] Pointer to output sequence of run lengths
+        EqualityOpT                 equality_op,        ///< [in] T equality operator
+        OffsetT                     num_items)          ///< [in] Total number of input items
+    :
+        temp_storage(temp_storage.Alias()),
+        d_in(d_in),
+        d_offsets_out(d_offsets_out),
+        d_lengths_out(d_lengths_out),
+        equality_op(equality_op),
+        scan_op(cub::Sum()),
+        num_items(num_items)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Utility methods for initializing the selections
+    //---------------------------------------------------------------------
+
+    template <bool FIRST_TILE, bool LAST_TILE>
+    __device__ __forceinline__ void InitializeSelections(
+        OffsetT             tile_offset,
+        OffsetT             num_remaining,
+        T                   (&items)[ITEMS_PER_THREAD],
+        LengthOffsetPair    (&lengths_and_num_runs)[ITEMS_PER_THREAD])
+    {
+        bool                head_flags[ITEMS_PER_THREAD];
+        bool                tail_flags[ITEMS_PER_THREAD];
+
+        OobInequalityOp<LAST_TILE> inequality_op(num_remaining, equality_op);
+
+        if (FIRST_TILE && LAST_TILE)
+        {
+            // First-and-last-tile always head-flags the first item and tail-flags the last item
+
+            BlockDiscontinuityT(temp_storage.aliasable.discontinuity).FlagHeadsAndTails(
+                head_flags, tail_flags, items, inequality_op);
+        }
+        else if (FIRST_TILE)
+        {
+            // First-tile always head-flags the first item
+
+            // Get the first item from the next tile
+            T tile_successor_item;
+            if (threadIdx.x == BLOCK_THREADS - 1)
+                tile_successor_item = d_in[tile_offset + TILE_ITEMS];
+
+            BlockDiscontinuityT(temp_storage.aliasable.discontinuity).FlagHeadsAndTails(
+                head_flags, tail_flags, tile_successor_item, items, inequality_op);
+        }
+        else if (LAST_TILE)
+        {
+            // Last-tile always flags the last item
+
+            // Get the last item from the previous tile
+            T tile_predecessor_item;
+            if (threadIdx.x == 0)
+                tile_predecessor_item = d_in[tile_offset - 1];
+
+            BlockDiscontinuityT(temp_storage.aliasable.discontinuity).FlagHeadsAndTails(
+                head_flags, tile_predecessor_item, tail_flags, items, inequality_op);
+        }
+        else
+        {
+            // Get the first item from the next tile
+            T tile_successor_item;
+            if (threadIdx.x == BLOCK_THREADS - 1)
+                tile_successor_item = d_in[tile_offset + TILE_ITEMS];
+
+            // Get the last item from the previous tile
+            T tile_predecessor_item;
+            if (threadIdx.x == 0)
+                tile_predecessor_item = d_in[tile_offset - 1];
+
+            BlockDiscontinuityT(temp_storage.aliasable.discontinuity).FlagHeadsAndTails(
+                head_flags, tile_predecessor_item, tail_flags, tile_successor_item, items, inequality_op);
+        }
+
+        // Zip counts and runs
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            lengths_and_num_runs[ITEM].key      = head_flags[ITEM] && (!tail_flags[ITEM]);
+            lengths_and_num_runs[ITEM].value    = ((!head_flags[ITEM]) || (!tail_flags[ITEM]));
+        }
+    }
+
+    //---------------------------------------------------------------------
+    // Scan utility methods
+    //---------------------------------------------------------------------
+
+    /**
+     * Scan of allocations
+     */
+    __device__ __forceinline__ void WarpScanAllocations(
+        LengthOffsetPair    &tile_aggregate,
+        LengthOffsetPair    &warp_aggregate,
+        LengthOffsetPair    &warp_exclusive_in_tile,
+        LengthOffsetPair    &thread_exclusive_in_warp,
+        LengthOffsetPair    (&lengths_and_num_runs)[ITEMS_PER_THREAD])
+    {
+        // Perform warpscans
+        unsigned int warp_id = ((WARPS == 1) ? 0 : threadIdx.x / WARP_THREADS);
+        int lane_id = LaneId();
+
+        LengthOffsetPair identity;
+        identity.key = 0;
+        identity.value = 0;
+
+        LengthOffsetPair thread_inclusive;
+        LengthOffsetPair thread_aggregate = internal::ThreadReduce(lengths_and_num_runs, scan_op);
+        WarpScanPairs(temp_storage.aliasable.warp_scan[warp_id]).Scan(
+            thread_aggregate,
+            thread_inclusive,
+            thread_exclusive_in_warp,
+            identity,
+            scan_op);
+
+        // Last lane in each warp shares its warp-aggregate
+        if (lane_id == WARP_THREADS - 1)
+            temp_storage.aliasable.warp_aggregates.Alias()[warp_id] = thread_inclusive;
+
+        CTA_SYNC();
+
+        // Accumulate total selected and the warp-wide prefix
+        warp_exclusive_in_tile          = identity;
+        warp_aggregate                  = temp_storage.aliasable.warp_aggregates.Alias()[warp_id];
+        tile_aggregate                  = temp_storage.aliasable.warp_aggregates.Alias()[0];
+
+        #pragma unroll
+        for (int WARP = 1; WARP < WARPS; ++WARP)
+        {
+            if (warp_id == WARP)
+                warp_exclusive_in_tile = tile_aggregate;
+
+            tile_aggregate = scan_op(tile_aggregate, temp_storage.aliasable.warp_aggregates.Alias()[WARP]);
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Utility methods for scattering selections
+    //---------------------------------------------------------------------
+
+    /**
+     * Two-phase scatter, specialized for warp time-slicing
+     */
+    template <bool FIRST_TILE>
+    __device__ __forceinline__ void ScatterTwoPhase(
+        OffsetT             tile_num_runs_exclusive_in_global,
+        OffsetT             warp_num_runs_aggregate,
+        OffsetT             warp_num_runs_exclusive_in_tile,
+        OffsetT             (&thread_num_runs_exclusive_in_warp)[ITEMS_PER_THREAD],
+        LengthOffsetPair    (&lengths_and_offsets)[ITEMS_PER_THREAD],
+        Int2Type<true>      is_warp_time_slice)
+    {
+        unsigned int warp_id = ((WARPS == 1) ? 0 : threadIdx.x / WARP_THREADS);
+        int lane_id = LaneId();
+
+        // Locally compact items within the warp (first warp)
+        if (warp_id == 0)
+        {
+            WarpExchangePairs(temp_storage.aliasable.scatter_aliasable.exchange_pairs[0]).ScatterToStriped(
+                lengths_and_offsets, thread_num_runs_exclusive_in_warp);
+        }
+
+        // Locally compact items within the warp (remaining warps)
+        #pragma unroll
+        for (int SLICE = 1; SLICE < WARPS; ++SLICE)
+        {
+            CTA_SYNC();
+
+            if (warp_id == SLICE)
+            {
+                WarpExchangePairs(temp_storage.aliasable.scatter_aliasable.exchange_pairs[0]).ScatterToStriped(
+                    lengths_and_offsets, thread_num_runs_exclusive_in_warp);
+            }
+        }
+
+        // Global scatter
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            if ((ITEM * WARP_THREADS) < warp_num_runs_aggregate - lane_id)
+            {
+                OffsetT item_offset =
+                    tile_num_runs_exclusive_in_global +
+                    warp_num_runs_exclusive_in_tile +
+                    (ITEM * WARP_THREADS) + lane_id;
+
+                // Scatter offset
+                d_offsets_out[item_offset] = lengths_and_offsets[ITEM].key;
+
+                // Scatter length if not the first (global) length
+                if ((!FIRST_TILE) || (ITEM != 0) || (threadIdx.x > 0))
+                {
+                    d_lengths_out[item_offset - 1] = lengths_and_offsets[ITEM].value;
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Two-phase scatter
+     */
+    template <bool FIRST_TILE>
+    __device__ __forceinline__ void ScatterTwoPhase(
+        OffsetT             tile_num_runs_exclusive_in_global,
+        OffsetT             warp_num_runs_aggregate,
+        OffsetT             warp_num_runs_exclusive_in_tile,
+        OffsetT             (&thread_num_runs_exclusive_in_warp)[ITEMS_PER_THREAD],
+        LengthOffsetPair    (&lengths_and_offsets)[ITEMS_PER_THREAD],
+        Int2Type<false>     is_warp_time_slice)
+    {
+        unsigned int warp_id = ((WARPS == 1) ? 0 : threadIdx.x / WARP_THREADS);
+        int lane_id = LaneId();
+
+        // Unzip
+        OffsetT run_offsets[ITEMS_PER_THREAD];
+        LengthT run_lengths[ITEMS_PER_THREAD];
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            run_offsets[ITEM] = lengths_and_offsets[ITEM].key;
+            run_lengths[ITEM] = lengths_and_offsets[ITEM].value;
+        }
+
+        WarpExchangeOffsets(temp_storage.aliasable.scatter_aliasable.exchange_offsets[warp_id]).ScatterToStriped(
+            run_offsets, thread_num_runs_exclusive_in_warp);
+
+        WARP_SYNC(0xffffffff);
+
+        WarpExchangeLengths(temp_storage.aliasable.scatter_aliasable.exchange_lengths[warp_id]).ScatterToStriped(
+            run_lengths, thread_num_runs_exclusive_in_warp);
+
+        // Global scatter
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            if ((ITEM * WARP_THREADS) + lane_id < warp_num_runs_aggregate)
+            {
+                OffsetT item_offset =
+                    tile_num_runs_exclusive_in_global +
+                    warp_num_runs_exclusive_in_tile +
+                    (ITEM * WARP_THREADS) + lane_id;
+
+                // Scatter offset
+                d_offsets_out[item_offset] = run_offsets[ITEM];
+
+                // Scatter length if not the first (global) length
+                if ((!FIRST_TILE) || (ITEM != 0) || (threadIdx.x > 0))
+                {
+                    d_lengths_out[item_offset - 1] = run_lengths[ITEM];
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Direct scatter
+     */
+    template <bool FIRST_TILE>
+    __device__ __forceinline__ void ScatterDirect(
+        OffsetT             tile_num_runs_exclusive_in_global,
+        OffsetT             warp_num_runs_aggregate,
+        OffsetT             warp_num_runs_exclusive_in_tile,
+        OffsetT             (&thread_num_runs_exclusive_in_warp)[ITEMS_PER_THREAD],
+        LengthOffsetPair    (&lengths_and_offsets)[ITEMS_PER_THREAD])
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            if (thread_num_runs_exclusive_in_warp[ITEM] < warp_num_runs_aggregate)
+            {
+                OffsetT item_offset =
+                    tile_num_runs_exclusive_in_global +
+                    warp_num_runs_exclusive_in_tile +
+                    thread_num_runs_exclusive_in_warp[ITEM];
+
+                // Scatter offset
+                d_offsets_out[item_offset] = lengths_and_offsets[ITEM].key;
+
+                // Scatter length if not the first (global) length
+                if (item_offset >= 1)
+                {
+                    d_lengths_out[item_offset - 1] = lengths_and_offsets[ITEM].value;
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Scatter
+     */
+    template <bool FIRST_TILE>
+    __device__ __forceinline__ void Scatter(
+        OffsetT             tile_num_runs_aggregate,
+        OffsetT             tile_num_runs_exclusive_in_global,
+        OffsetT             warp_num_runs_aggregate,
+        OffsetT             warp_num_runs_exclusive_in_tile,
+        OffsetT             (&thread_num_runs_exclusive_in_warp)[ITEMS_PER_THREAD],
+        LengthOffsetPair    (&lengths_and_offsets)[ITEMS_PER_THREAD])
+    {
+        if ((ITEMS_PER_THREAD == 1) || (tile_num_runs_aggregate < BLOCK_THREADS))
+        {
+            // Direct scatter if the warp has any items
+            if (warp_num_runs_aggregate)
+            {
+                ScatterDirect<FIRST_TILE>(
+                    tile_num_runs_exclusive_in_global,
+                    warp_num_runs_aggregate,
+                    warp_num_runs_exclusive_in_tile,
+                    thread_num_runs_exclusive_in_warp,
+                    lengths_and_offsets);
+            }
+        }
+        else
+        {
+            // Scatter two phase
+            ScatterTwoPhase<FIRST_TILE>(
+                tile_num_runs_exclusive_in_global,
+                warp_num_runs_aggregate,
+                warp_num_runs_exclusive_in_tile,
+                thread_num_runs_exclusive_in_warp,
+                lengths_and_offsets,
+                Int2Type<STORE_WARP_TIME_SLICING>());
+        }
+    }
+
+
+
+    //---------------------------------------------------------------------
+    // Cooperatively scan a device-wide sequence of tiles with other CTAs
+    //---------------------------------------------------------------------
+
+    /**
+     * Process a tile of input (dynamic chained scan)
+     */
+    template <
+        bool                LAST_TILE>
+    __device__ __forceinline__ LengthOffsetPair ConsumeTile(
+        OffsetT             num_items,          ///< Total number of global input items
+        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
+        int                 tile_idx,           ///< Tile index
+        OffsetT             tile_offset,        ///< Tile offset
+        ScanTileStateT      &tile_status)       ///< Global list of tile status
+    {
+        if (tile_idx == 0)
+        {
+            // First tile
+
+            // Load items
+            T items[ITEMS_PER_THREAD];
+            if (LAST_TILE)
+                BlockLoadT(temp_storage.aliasable.load).Load(d_in + tile_offset, items, num_remaining, T());
+            else
+                BlockLoadT(temp_storage.aliasable.load).Load(d_in + tile_offset, items);
+
+            if (SYNC_AFTER_LOAD)
+                CTA_SYNC();
+
+            // Set flags
+            LengthOffsetPair    lengths_and_num_runs[ITEMS_PER_THREAD];
+
+            InitializeSelections<true, LAST_TILE>(
+                tile_offset,
+                num_remaining,
+                items,
+                lengths_and_num_runs);
+
+            // Exclusive scan of lengths and runs
+            LengthOffsetPair tile_aggregate;
+            LengthOffsetPair warp_aggregate;
+            LengthOffsetPair warp_exclusive_in_tile;
+            LengthOffsetPair thread_exclusive_in_warp;
+
+            WarpScanAllocations(
+                tile_aggregate,
+                warp_aggregate,
+                warp_exclusive_in_tile,
+                thread_exclusive_in_warp,
+                lengths_and_num_runs);
+
+            // Update tile status if this is not the last tile
+            if (!LAST_TILE && (threadIdx.x == 0))
+                tile_status.SetInclusive(0, tile_aggregate);
+
+            // Update thread_exclusive_in_warp to fold in warp run-length
+            if (thread_exclusive_in_warp.key == 0)
+                thread_exclusive_in_warp.value += warp_exclusive_in_tile.value;
+
+            LengthOffsetPair    lengths_and_offsets[ITEMS_PER_THREAD];
+            OffsetT             thread_num_runs_exclusive_in_warp[ITEMS_PER_THREAD];
+            LengthOffsetPair    lengths_and_num_runs2[ITEMS_PER_THREAD];
+
+            // Downsweep scan through lengths_and_num_runs
+            internal::ThreadScanExclusive(lengths_and_num_runs, lengths_and_num_runs2, scan_op, thread_exclusive_in_warp);
+
+            // Zip
+
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                lengths_and_offsets[ITEM].value         = lengths_and_num_runs2[ITEM].value;
+                lengths_and_offsets[ITEM].key        = tile_offset + (threadIdx.x * ITEMS_PER_THREAD) + ITEM;
+                thread_num_runs_exclusive_in_warp[ITEM] = (lengths_and_num_runs[ITEM].key) ?
+                                                                lengths_and_num_runs2[ITEM].key :         // keep
+                                                                WARP_THREADS * ITEMS_PER_THREAD;            // discard
+            }
+
+            OffsetT tile_num_runs_aggregate              = tile_aggregate.key;
+            OffsetT tile_num_runs_exclusive_in_global    = 0;
+            OffsetT warp_num_runs_aggregate              = warp_aggregate.key;
+            OffsetT warp_num_runs_exclusive_in_tile      = warp_exclusive_in_tile.key;
+
+            // Scatter
+            Scatter<true>(
+                tile_num_runs_aggregate,
+                tile_num_runs_exclusive_in_global,
+                warp_num_runs_aggregate,
+                warp_num_runs_exclusive_in_tile,
+                thread_num_runs_exclusive_in_warp,
+                lengths_and_offsets);
+
+            // Return running total (inclusive of this tile)
+            return tile_aggregate;
+        }
+        else
+        {
+            // Not first tile
+
+            // Load items
+            T items[ITEMS_PER_THREAD];
+            if (LAST_TILE)
+                BlockLoadT(temp_storage.aliasable.load).Load(d_in + tile_offset, items, num_remaining, T());
+            else
+                BlockLoadT(temp_storage.aliasable.load).Load(d_in + tile_offset, items);
+
+            if (SYNC_AFTER_LOAD)
+                CTA_SYNC();
+
+            // Set flags
+            LengthOffsetPair    lengths_and_num_runs[ITEMS_PER_THREAD];
+
+            InitializeSelections<false, LAST_TILE>(
+                tile_offset,
+                num_remaining,
+                items,
+                lengths_and_num_runs);
+
+            // Exclusive scan of lengths and runs
+            LengthOffsetPair tile_aggregate;
+            LengthOffsetPair warp_aggregate;
+            LengthOffsetPair warp_exclusive_in_tile;
+            LengthOffsetPair thread_exclusive_in_warp;
+
+            WarpScanAllocations(
+                tile_aggregate,
+                warp_aggregate,
+                warp_exclusive_in_tile,
+                thread_exclusive_in_warp,
+                lengths_and_num_runs);
+
+            // First warp computes tile prefix in lane 0
+            TilePrefixCallbackOpT prefix_op(tile_status, temp_storage.aliasable.prefix, Sum(), tile_idx);
+            unsigned int warp_id = ((WARPS == 1) ? 0 : threadIdx.x / WARP_THREADS);
+            if (warp_id == 0)
+            {
+                prefix_op(tile_aggregate);
+                if (threadIdx.x == 0)
+                    temp_storage.tile_exclusive = prefix_op.exclusive_prefix;
+            }
+
+            CTA_SYNC();
+
+            LengthOffsetPair tile_exclusive_in_global = temp_storage.tile_exclusive;
+
+            // Update thread_exclusive_in_warp to fold in warp and tile run-lengths
+            LengthOffsetPair thread_exclusive = scan_op(tile_exclusive_in_global, warp_exclusive_in_tile);
+            if (thread_exclusive_in_warp.key == 0)
+                thread_exclusive_in_warp.value += thread_exclusive.value;
+
+            // Downsweep scan through lengths_and_num_runs
+            LengthOffsetPair    lengths_and_num_runs2[ITEMS_PER_THREAD];
+            LengthOffsetPair    lengths_and_offsets[ITEMS_PER_THREAD];
+            OffsetT             thread_num_runs_exclusive_in_warp[ITEMS_PER_THREAD];
+
+            internal::ThreadScanExclusive(lengths_and_num_runs, lengths_and_num_runs2, scan_op, thread_exclusive_in_warp);
+
+            // Zip
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                lengths_and_offsets[ITEM].value         = lengths_and_num_runs2[ITEM].value;
+                lengths_and_offsets[ITEM].key        = tile_offset + (threadIdx.x * ITEMS_PER_THREAD) + ITEM;
+                thread_num_runs_exclusive_in_warp[ITEM] = (lengths_and_num_runs[ITEM].key) ?
+                                                                lengths_and_num_runs2[ITEM].key :         // keep
+                                                                WARP_THREADS * ITEMS_PER_THREAD;            // discard
+            }
+
+            OffsetT tile_num_runs_aggregate              = tile_aggregate.key;
+            OffsetT tile_num_runs_exclusive_in_global    = tile_exclusive_in_global.key;
+            OffsetT warp_num_runs_aggregate              = warp_aggregate.key;
+            OffsetT warp_num_runs_exclusive_in_tile      = warp_exclusive_in_tile.key;
+
+            // Scatter
+            Scatter<false>(
+                tile_num_runs_aggregate,
+                tile_num_runs_exclusive_in_global,
+                warp_num_runs_aggregate,
+                warp_num_runs_exclusive_in_tile,
+                thread_num_runs_exclusive_in_warp,
+                lengths_and_offsets);
+
+            // Return running total (inclusive of this tile)
+            return prefix_op.inclusive_prefix;
+        }
+    }
+
+
+    /**
+     * Scan tiles of items as part of a dynamic chained scan
+     */
+    template <typename NumRunsIteratorT>            ///< Output iterator type for recording number of items selected
+    __device__ __forceinline__ void ConsumeRange(
+        int                 num_tiles,              ///< Total number of input tiles
+        ScanTileStateT&     tile_status,            ///< Global list of tile status
+        NumRunsIteratorT    d_num_runs_out)         ///< Output pointer for total number of runs identified
+    {
+        // Blocks are launched in increasing order, so just assign one tile per block
+        int     tile_idx        = (blockIdx.x * gridDim.y) + blockIdx.y;    // Current tile index
+        OffsetT tile_offset     = tile_idx * TILE_ITEMS;                  // Global offset for the current tile
+        OffsetT num_remaining   = num_items - tile_offset;                  // Remaining items (including this tile)
+
+        if (tile_idx < num_tiles - 1)
+        {
+            // Not the last tile (full)
+            ConsumeTile<false>(num_items, num_remaining, tile_idx, tile_offset, tile_status);
+        }
+        else if (num_remaining > 0)
+        {
+            // The last tile (possibly partially-full)
+            LengthOffsetPair running_total = ConsumeTile<true>(num_items, num_remaining, tile_idx, tile_offset, tile_status);
+
+            if (threadIdx.x == 0)
+            {
+                // Output the total number of items selected
+                *d_num_runs_out = running_total.key;
+
+                // The inclusive prefix contains accumulated length reduction for the last run
+                if (running_total.key > 0)
+                    d_lengths_out[running_total.key - 1] = running_total.value;
+            }
+        }
+    }
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_scan.cuh b/third_party/cub/cub/agent/agent_scan.cuh
new file mode 100644
index 0000000..9368615
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_scan.cuh
@@ -0,0 +1,471 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::AgentScan implements a stateful abstraction of CUDA thread blocks for participating in device-wide prefix scan .
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "single_pass_scan_operators.cuh"
+#include "../block/block_load.cuh"
+#include "../block/block_store.cuh"
+#include "../block/block_scan.cuh"
+#include "../grid/grid_queue.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for AgentScan
+ */
+template <
+    int                         _BLOCK_THREADS,                 ///< Threads per thread block
+    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
+    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
+    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
+    BlockStoreAlgorithm         _STORE_ALGORITHM,               ///< The BlockStore algorithm to use
+    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
+struct AgentScanPolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
+        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
+    };
+
+    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;          ///< The BlockLoad algorithm to use
+    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;           ///< Cache load modifier for reading input elements
+    static const BlockStoreAlgorithm    STORE_ALGORITHM         = _STORE_ALGORITHM;         ///< The BlockStore algorithm to use
+    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;          ///< The BlockScan algorithm to use
+};
+
+
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+/**
+ * \brief AgentScan implements a stateful abstraction of CUDA thread blocks for participating in device-wide prefix scan .
+ */
+template <
+    typename AgentScanPolicyT,      ///< Parameterized AgentScanPolicyT tuning policy type
+    typename InputIteratorT,        ///< Random-access input iterator type
+    typename OutputIteratorT,       ///< Random-access output iterator type
+    typename ScanOpT,               ///< Scan functor type
+    typename InitValueT,            ///< The init_value element for ScanOpT type (cub::NullType for inclusive scan)
+    typename OffsetT>               ///< Signed integer type for global offsets
+struct AgentScan
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    // Tile status descriptor interface type
+    typedef ScanTileState<OutputT> ScanTileStateT;
+
+    // Input iterator wrapper type (for applying cache modifier)
+    typedef typename If<IsPointer<InputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentScanPolicyT::LOAD_MODIFIER, InputT, OffsetT>,   // Wrap the native input pointer with CacheModifiedInputIterator
+            InputIteratorT>::Type                                                           // Directly use the supplied input iterator type
+        WrappedInputIteratorT;
+
+    // Constants
+    enum
+    {
+        IS_INCLUSIVE        = Equals<InitValueT, NullType>::VALUE,            // Inclusive scan if no init_value type is provided
+        BLOCK_THREADS       = AgentScanPolicyT::BLOCK_THREADS,
+        ITEMS_PER_THREAD    = AgentScanPolicyT::ITEMS_PER_THREAD,
+        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
+    };
+
+    // Parameterized BlockLoad type
+    typedef BlockLoad<
+            OutputT,
+            AgentScanPolicyT::BLOCK_THREADS,
+            AgentScanPolicyT::ITEMS_PER_THREAD,
+            AgentScanPolicyT::LOAD_ALGORITHM>
+        BlockLoadT;
+
+    // Parameterized BlockStore type
+    typedef BlockStore<
+            OutputT,
+            AgentScanPolicyT::BLOCK_THREADS,
+            AgentScanPolicyT::ITEMS_PER_THREAD,
+            AgentScanPolicyT::STORE_ALGORITHM>
+        BlockStoreT;
+
+    // Parameterized BlockScan type
+    typedef BlockScan<
+            OutputT,
+            AgentScanPolicyT::BLOCK_THREADS,
+            AgentScanPolicyT::SCAN_ALGORITHM>
+        BlockScanT;
+
+    // Callback type for obtaining tile prefix during block scan
+    typedef TilePrefixCallbackOp<
+            OutputT,
+            ScanOpT,
+            ScanTileStateT>
+        TilePrefixCallbackOpT;
+
+    // Stateful BlockScan prefix callback type for managing a running total while scanning consecutive tiles
+    typedef BlockScanRunningPrefixOp<
+            OutputT,
+            ScanOpT>
+        RunningPrefixCallbackOp;
+
+    // Shared memory type for this thread block
+    union _TempStorage
+    {
+        typename BlockLoadT::TempStorage    load;       // Smem needed for tile loading
+        typename BlockStoreT::TempStorage   store;      // Smem needed for tile storing
+
+        struct
+        {
+            typename TilePrefixCallbackOpT::TempStorage  prefix;     // Smem needed for cooperative prefix callback
+            typename BlockScanT::TempStorage             scan;       // Smem needed for tile scanning
+        };
+    };
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    _TempStorage&               temp_storage;       ///< Reference to temp_storage
+    WrappedInputIteratorT       d_in;               ///< Input data
+    OutputIteratorT             d_out;              ///< Output data
+    ScanOpT                     scan_op;            ///< Binary scan operator
+    InitValueT                  init_value;         ///< The init_value element for ScanOpT
+
+
+    //---------------------------------------------------------------------
+    // Block scan utility methods
+    //---------------------------------------------------------------------
+
+    /**
+     * Exclusive scan specialization (first tile)
+     */
+    __device__ __forceinline__
+    void ScanTile(
+        OutputT             (&items)[ITEMS_PER_THREAD],
+        OutputT             init_value,
+        ScanOpT             scan_op,
+        OutputT             &block_aggregate,
+        Int2Type<false>     /*is_inclusive*/)
+    {
+        BlockScanT(temp_storage.scan).ExclusiveScan(items, items, init_value, scan_op, block_aggregate);
+        block_aggregate = scan_op(init_value, block_aggregate);
+    }
+
+
+    /**
+     * Inclusive scan specialization (first tile)
+     */
+    __device__ __forceinline__
+    void ScanTile(
+        OutputT             (&items)[ITEMS_PER_THREAD],
+        InitValueT          /*init_value*/,
+        ScanOpT             scan_op,
+        OutputT             &block_aggregate,
+        Int2Type<true>      /*is_inclusive*/)
+    {
+        BlockScanT(temp_storage.scan).InclusiveScan(items, items, scan_op, block_aggregate);
+    }
+
+
+    /**
+     * Exclusive scan specialization (subsequent tiles)
+     */
+    template <typename PrefixCallback>
+    __device__ __forceinline__
+    void ScanTile(
+        OutputT             (&items)[ITEMS_PER_THREAD],
+        ScanOpT             scan_op,
+        PrefixCallback      &prefix_op,
+        Int2Type<false>     /*is_inclusive*/)
+    {
+        BlockScanT(temp_storage.scan).ExclusiveScan(items, items, scan_op, prefix_op);
+    }
+
+
+    /**
+     * Inclusive scan specialization (subsequent tiles)
+     */
+    template <typename PrefixCallback>
+    __device__ __forceinline__
+    void ScanTile(
+        OutputT             (&items)[ITEMS_PER_THREAD],
+        ScanOpT             scan_op,
+        PrefixCallback      &prefix_op,
+        Int2Type<true>      /*is_inclusive*/)
+    {
+        BlockScanT(temp_storage.scan).InclusiveScan(items, items, scan_op, prefix_op);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Constructor
+    //---------------------------------------------------------------------
+
+    // Constructor
+    __device__ __forceinline__
+    AgentScan(
+        TempStorage&    temp_storage,       ///< Reference to temp_storage
+        InputIteratorT  d_in,               ///< Input data
+        OutputIteratorT d_out,              ///< Output data
+        ScanOpT         scan_op,            ///< Binary scan operator
+        InitValueT      init_value)         ///< Initial value to seed the exclusive scan
+    :
+        temp_storage(temp_storage.Alias()),
+        d_in(d_in),
+        d_out(d_out),
+        scan_op(scan_op),
+        init_value(init_value)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Cooperatively scan a device-wide sequence of tiles with other CTAs
+    //---------------------------------------------------------------------
+
+    /**
+     * Process a tile of input (dynamic chained scan)
+     */
+    template <bool IS_LAST_TILE>                ///< Whether the current tile is the last tile
+    __device__ __forceinline__ void ConsumeTile(
+        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
+        int                 tile_idx,           ///< Tile index
+        OffsetT             tile_offset,        ///< Tile offset
+        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
+    {
+        // Load items
+        OutputT items[ITEMS_PER_THREAD];
+
+        if (IS_LAST_TILE)
+            BlockLoadT(temp_storage.load).Load(d_in + tile_offset, items, num_remaining);
+        else
+            BlockLoadT(temp_storage.load).Load(d_in + tile_offset, items);
+
+        CTA_SYNC();
+
+        // Perform tile scan
+        if (tile_idx == 0)
+        {
+            // Scan first tile
+            OutputT block_aggregate;
+            ScanTile(items, init_value, scan_op, block_aggregate, Int2Type<IS_INCLUSIVE>());
+            if ((!IS_LAST_TILE) && (threadIdx.x == 0))
+                tile_state.SetInclusive(0, block_aggregate);
+        }
+        else
+        {
+            // Scan non-first tile
+            TilePrefixCallbackOpT prefix_op(tile_state, temp_storage.prefix, scan_op, tile_idx);
+            ScanTile(items, scan_op, prefix_op, Int2Type<IS_INCLUSIVE>());
+        }
+
+        CTA_SYNC();
+
+        // Store items
+        if (IS_LAST_TILE)
+            BlockStoreT(temp_storage.store).Store(d_out + tile_offset, items, num_remaining);
+        else
+            BlockStoreT(temp_storage.store).Store(d_out + tile_offset, items);
+    }
+
+
+    /**
+     * Scan tiles of items as part of a dynamic chained scan
+     */
+    __device__ __forceinline__ void ConsumeRange(
+        int                 num_items,          ///< Total number of input items
+        ScanTileStateT&     tile_state,         ///< Global tile state descriptor
+        int                 start_tile)         ///< The starting tile for the current grid
+    {
+        // Blocks are launched in increasing order, so just assign one tile per block
+        int     tile_idx        = start_tile + blockIdx.x;          // Current tile index
+        OffsetT tile_offset     = OffsetT(TILE_ITEMS) * tile_idx;   // Global offset for the current tile
+        OffsetT num_remaining   = num_items - tile_offset;          // Remaining items (including this tile)
+
+        if (num_remaining > TILE_ITEMS)
+        {
+            // Not last tile
+            ConsumeTile<false>(num_remaining, tile_idx, tile_offset, tile_state);
+        }
+        else if (num_remaining > 0)
+        {
+            // Last tile
+            ConsumeTile<true>(num_remaining, tile_idx, tile_offset, tile_state);
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Scan an sequence of consecutive tiles (independent of other thread blocks)
+    //---------------------------------------------------------------------
+
+    /**
+     * Process a tile of input
+     */
+    template <
+        bool                        IS_FIRST_TILE,
+        bool                        IS_LAST_TILE>
+    __device__ __forceinline__ void ConsumeTile(
+        OffsetT                     tile_offset,                ///< Tile offset
+        RunningPrefixCallbackOp&    prefix_op,                  ///< Running prefix operator
+        int                         valid_items = TILE_ITEMS)   ///< Number of valid items in the tile
+    {
+        // Load items
+        OutputT items[ITEMS_PER_THREAD];
+
+        if (IS_LAST_TILE)
+            BlockLoadT(temp_storage.load).Load(d_in + tile_offset, items, valid_items);
+        else
+            BlockLoadT(temp_storage.load).Load(d_in + tile_offset, items);
+
+        CTA_SYNC();
+
+        // Block scan
+        if (IS_FIRST_TILE)
+        {
+            OutputT block_aggregate;
+            ScanTile(items, init_value, scan_op, block_aggregate, Int2Type<IS_INCLUSIVE>());
+            prefix_op.running_total = block_aggregate;
+        }
+        else
+        {
+            ScanTile(items, scan_op, prefix_op, Int2Type<IS_INCLUSIVE>());
+        }
+
+        CTA_SYNC();
+
+        // Store items
+        if (IS_LAST_TILE)
+            BlockStoreT(temp_storage.store).Store(d_out + tile_offset, items, valid_items);
+        else
+            BlockStoreT(temp_storage.store).Store(d_out + tile_offset, items);
+    }
+
+
+    /**
+     * Scan a consecutive share of input tiles
+     */
+    __device__ __forceinline__ void ConsumeRange(
+        OffsetT  range_offset,      ///< [in] Threadblock begin offset (inclusive)
+        OffsetT  range_end)         ///< [in] Threadblock end offset (exclusive)
+    {
+        BlockScanRunningPrefixOp<OutputT, ScanOpT> prefix_op(scan_op);
+
+        if (range_offset + TILE_ITEMS <= range_end)
+        {
+            // Consume first tile of input (full)
+            ConsumeTile<true, true>(range_offset, prefix_op);
+            range_offset += TILE_ITEMS;
+
+            // Consume subsequent full tiles of input
+            while (range_offset + TILE_ITEMS <= range_end)
+            {
+                ConsumeTile<false, true>(range_offset, prefix_op);
+                range_offset += TILE_ITEMS;
+            }
+
+            // Consume a partially-full tile
+            if (range_offset < range_end)
+            {
+                int valid_items = range_end - range_offset;
+                ConsumeTile<false, false>(range_offset, prefix_op, valid_items);
+            }
+        }
+        else
+        {
+            // Consume the first tile of input (partially-full)
+            int valid_items = range_end - range_offset;
+            ConsumeTile<true, false>(range_offset, prefix_op, valid_items);
+        }
+    }
+
+
+    /**
+     * Scan a consecutive share of input tiles, seeded with the specified prefix value
+     */
+    __device__ __forceinline__ void ConsumeRange(
+        OffsetT range_offset,                       ///< [in] Threadblock begin offset (inclusive)
+        OffsetT range_end,                          ///< [in] Threadblock end offset (exclusive)
+        OutputT prefix)                             ///< [in] The prefix to apply to the scan segment
+    {
+        BlockScanRunningPrefixOp<OutputT, ScanOpT> prefix_op(prefix, scan_op);
+
+        // Consume full tiles of input
+        while (range_offset + TILE_ITEMS <= range_end)
+        {
+            ConsumeTile<true, false>(range_offset, prefix_op);
+            range_offset += TILE_ITEMS;
+        }
+
+        // Consume a partially-full tile
+        if (range_offset < range_end)
+        {
+            int valid_items = range_end - range_offset;
+            ConsumeTile<false, false>(range_offset, prefix_op, valid_items);
+        }
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_segment_fixup.cuh b/third_party/cub/cub/agent/agent_segment_fixup.cuh
new file mode 100644
index 0000000..e2de58e
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_segment_fixup.cuh
@@ -0,0 +1,375 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::AgentSegmentFixup implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key.
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "single_pass_scan_operators.cuh"
+#include "../block/block_load.cuh"
+#include "../block/block_store.cuh"
+#include "../block/block_scan.cuh"
+#include "../block/block_discontinuity.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../iterator/constant_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for AgentSegmentFixup
+ */
+template <
+    int                         _BLOCK_THREADS,                 ///< Threads per thread block
+    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
+    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
+    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
+    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
+struct AgentSegmentFixupPolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
+        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
+    };
+
+    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
+    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
+    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
+};
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+/**
+ * \brief AgentSegmentFixup implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key
+ */
+template <
+    typename    AgentSegmentFixupPolicyT,       ///< Parameterized AgentSegmentFixupPolicy tuning policy type
+    typename    PairsInputIteratorT,            ///< Random-access input iterator type for keys
+    typename    AggregatesOutputIteratorT,      ///< Random-access output iterator type for values
+    typename    EqualityOpT,                    ///< KeyT equality operator type
+    typename    ReductionOpT,                   ///< ValueT reduction operator type
+    typename    OffsetT>                        ///< Signed integer type for global offsets
+struct AgentSegmentFixup
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    // Data type of key-value input iterator
+    typedef typename std::iterator_traits<PairsInputIteratorT>::value_type KeyValuePairT;
+
+    // Value type
+    typedef typename KeyValuePairT::Value ValueT;
+
+    // Tile status descriptor interface type
+    typedef ReduceByKeyScanTileState<ValueT, OffsetT> ScanTileStateT;
+
+    // Constants
+    enum
+    {
+        BLOCK_THREADS       = AgentSegmentFixupPolicyT::BLOCK_THREADS,
+        ITEMS_PER_THREAD    = AgentSegmentFixupPolicyT::ITEMS_PER_THREAD,
+        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
+
+        // Whether or not do fixup using RLE + global atomics
+        USE_ATOMIC_FIXUP    = (CUB_PTX_ARCH >= 350) && 
+                                (Equals<ValueT, float>::VALUE || 
+                                 Equals<ValueT, int>::VALUE ||
+                                 Equals<ValueT, unsigned int>::VALUE ||
+                                 Equals<ValueT, unsigned long long>::VALUE),
+
+        // Whether or not the scan operation has a zero-valued identity value (true if we're performing addition on a primitive type)
+        HAS_IDENTITY_ZERO   = (Equals<ReductionOpT, cub::Sum>::VALUE) && (Traits<ValueT>::PRIMITIVE),
+    };
+
+    // Cache-modified Input iterator wrapper type (for applying cache modifier) for keys
+    typedef typename If<IsPointer<PairsInputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentSegmentFixupPolicyT::LOAD_MODIFIER, KeyValuePairT, OffsetT>,    // Wrap the native input pointer with CacheModifiedValuesInputIterator
+            PairsInputIteratorT>::Type                                                                      // Directly use the supplied input iterator type
+        WrappedPairsInputIteratorT;
+
+    // Cache-modified Input iterator wrapper type (for applying cache modifier) for fixup values
+    typedef typename If<IsPointer<AggregatesOutputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentSegmentFixupPolicyT::LOAD_MODIFIER, ValueT, OffsetT>,    // Wrap the native input pointer with CacheModifiedValuesInputIterator
+            AggregatesOutputIteratorT>::Type                                                        // Directly use the supplied input iterator type
+        WrappedFixupInputIteratorT;
+
+    // Reduce-value-by-segment scan operator
+    typedef ReduceByKeyOp<cub::Sum> ReduceBySegmentOpT;
+
+    // Parameterized BlockLoad type for pairs
+    typedef BlockLoad<
+            KeyValuePairT,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            AgentSegmentFixupPolicyT::LOAD_ALGORITHM>
+        BlockLoadPairs;
+
+    // Parameterized BlockScan type
+    typedef BlockScan<
+            KeyValuePairT,
+            BLOCK_THREADS,
+            AgentSegmentFixupPolicyT::SCAN_ALGORITHM>
+        BlockScanT;
+
+    // Callback type for obtaining tile prefix during block scan
+    typedef TilePrefixCallbackOp<
+            KeyValuePairT,
+            ReduceBySegmentOpT,
+            ScanTileStateT>
+        TilePrefixCallbackOpT;
+
+    // Shared memory type for this thread block
+    union _TempStorage
+    {
+        struct
+        {
+            typename BlockScanT::TempStorage                scan;           // Smem needed for tile scanning
+            typename TilePrefixCallbackOpT::TempStorage     prefix;         // Smem needed for cooperative prefix callback
+        };
+
+        // Smem needed for loading keys
+        typename BlockLoadPairs::TempStorage load_pairs;
+    };
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    _TempStorage&                   temp_storage;       ///< Reference to temp_storage
+    WrappedPairsInputIteratorT      d_pairs_in;          ///< Input keys
+    AggregatesOutputIteratorT       d_aggregates_out;   ///< Output value aggregates
+    WrappedFixupInputIteratorT      d_fixup_in;         ///< Fixup input values
+    InequalityWrapper<EqualityOpT>  inequality_op;      ///< KeyT inequality operator
+    ReductionOpT                    reduction_op;       ///< Reduction operator
+    ReduceBySegmentOpT              scan_op;            ///< Reduce-by-segment scan operator
+
+
+    //---------------------------------------------------------------------
+    // Constructor
+    //---------------------------------------------------------------------
+
+    // Constructor
+    __device__ __forceinline__
+    AgentSegmentFixup(
+        TempStorage&                temp_storage,       ///< Reference to temp_storage
+        PairsInputIteratorT         d_pairs_in,          ///< Input keys
+        AggregatesOutputIteratorT   d_aggregates_out,   ///< Output value aggregates
+        EqualityOpT                 equality_op,        ///< KeyT equality operator
+        ReductionOpT                reduction_op)       ///< ValueT reduction operator
+    :
+        temp_storage(temp_storage.Alias()),
+        d_pairs_in(d_pairs_in),
+        d_aggregates_out(d_aggregates_out),
+        d_fixup_in(d_aggregates_out),
+        inequality_op(equality_op),
+        reduction_op(reduction_op),
+        scan_op(reduction_op)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Cooperatively scan a device-wide sequence of tiles with other CTAs
+    //---------------------------------------------------------------------
+
+
+    /**
+     * Process input tile.  Specialized for atomic-fixup
+     */
+    template <bool IS_LAST_TILE>
+    __device__ __forceinline__ void ConsumeTile(
+        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
+        int                 tile_idx,           ///< Tile index
+        OffsetT             tile_offset,        ///< Tile offset
+        ScanTileStateT&     tile_state,         ///< Global tile state descriptor
+        Int2Type<true>      use_atomic_fixup)   ///< Marker whether to use atomicAdd (instead of reduce-by-key)
+    {
+        KeyValuePairT   pairs[ITEMS_PER_THREAD];
+
+        // Load pairs
+        KeyValuePairT oob_pair;
+        oob_pair.key = -1;
+
+        if (IS_LAST_TILE)
+            BlockLoadPairs(temp_storage.load_pairs).Load(d_pairs_in + tile_offset, pairs, num_remaining, oob_pair);
+        else
+            BlockLoadPairs(temp_storage.load_pairs).Load(d_pairs_in + tile_offset, pairs);
+
+        // RLE 
+        #pragma unroll
+        for (int ITEM = 1; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            ValueT* d_scatter = d_aggregates_out + pairs[ITEM - 1].key;
+            if (pairs[ITEM].key != pairs[ITEM - 1].key)
+                atomicAdd(d_scatter, pairs[ITEM - 1].value);
+            else
+                pairs[ITEM].value = reduction_op(pairs[ITEM - 1].value, pairs[ITEM].value);
+        }
+
+        // Flush last item if valid
+        ValueT* d_scatter = d_aggregates_out + pairs[ITEMS_PER_THREAD - 1].key;
+        if ((!IS_LAST_TILE) || (pairs[ITEMS_PER_THREAD - 1].key >= 0))
+            atomicAdd(d_scatter, pairs[ITEMS_PER_THREAD - 1].value);
+    }
+
+
+    /**
+     * Process input tile.  Specialized for reduce-by-key fixup
+     */
+    template <bool IS_LAST_TILE>
+    __device__ __forceinline__ void ConsumeTile(
+        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
+        int                 tile_idx,           ///< Tile index
+        OffsetT             tile_offset,        ///< Tile offset
+        ScanTileStateT&     tile_state,         ///< Global tile state descriptor
+        Int2Type<false>     use_atomic_fixup)   ///< Marker whether to use atomicAdd (instead of reduce-by-key)
+    {
+        KeyValuePairT   pairs[ITEMS_PER_THREAD];
+        KeyValuePairT   scatter_pairs[ITEMS_PER_THREAD];
+
+        // Load pairs
+        KeyValuePairT oob_pair;
+        oob_pair.key = -1;
+
+        if (IS_LAST_TILE)
+            BlockLoadPairs(temp_storage.load_pairs).Load(d_pairs_in + tile_offset, pairs, num_remaining, oob_pair);
+        else
+            BlockLoadPairs(temp_storage.load_pairs).Load(d_pairs_in + tile_offset, pairs);
+
+        CTA_SYNC();
+
+        KeyValuePairT tile_aggregate;
+        if (tile_idx == 0)
+        {
+            // Exclusive scan of values and segment_flags
+            BlockScanT(temp_storage.scan).ExclusiveScan(pairs, scatter_pairs, scan_op, tile_aggregate);
+
+            // Update tile status if this is not the last tile
+            if (threadIdx.x == 0)
+            {
+                // Set first segment id to not trigger a flush (invalid from exclusive scan)
+                scatter_pairs[0].key = pairs[0].key;
+
+                if (!IS_LAST_TILE)
+                    tile_state.SetInclusive(0, tile_aggregate);
+
+            }
+        }
+        else
+        {
+            // Exclusive scan of values and segment_flags
+            TilePrefixCallbackOpT prefix_op(tile_state, temp_storage.prefix, scan_op, tile_idx);
+            BlockScanT(temp_storage.scan).ExclusiveScan(pairs, scatter_pairs, scan_op, prefix_op);
+            tile_aggregate = prefix_op.GetBlockAggregate();
+        }
+
+        // Scatter updated values
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            if (scatter_pairs[ITEM].key != pairs[ITEM].key)
+            {
+                // Update the value at the key location
+                ValueT value    = d_fixup_in[scatter_pairs[ITEM].key];
+                value           = reduction_op(value, scatter_pairs[ITEM].value);
+
+                d_aggregates_out[scatter_pairs[ITEM].key] = value;
+            }
+        }
+
+        // Finalize the last item
+        if (IS_LAST_TILE)
+        {
+            // Last thread will output final count and last item, if necessary
+            if (threadIdx.x == BLOCK_THREADS - 1)
+            {
+                // If the last tile is a whole tile, the inclusive prefix contains accumulated value reduction for the last segment
+                if (num_remaining == TILE_ITEMS)
+                {
+                    // Update the value at the key location
+                    OffsetT last_key = pairs[ITEMS_PER_THREAD - 1].key;
+                    d_aggregates_out[last_key] = reduction_op(tile_aggregate.value, d_fixup_in[last_key]);
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Scan tiles of items as part of a dynamic chained scan
+     */
+    __device__ __forceinline__ void ConsumeRange(
+        int                 num_items,          ///< Total number of input items
+        int                 num_tiles,          ///< Total number of input tiles
+        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
+    {
+        // Blocks are launched in increasing order, so just assign one tile per block
+        int     tile_idx        = (blockIdx.x * gridDim.y) + blockIdx.y;    // Current tile index
+        OffsetT tile_offset     = tile_idx * TILE_ITEMS;                    // Global offset for the current tile
+        OffsetT num_remaining   = num_items - tile_offset;                  // Remaining items (including this tile)
+
+        if (num_remaining > TILE_ITEMS)
+        {
+            // Not the last tile (full)
+            ConsumeTile<false>(num_remaining, tile_idx, tile_offset, tile_state, Int2Type<USE_ATOMIC_FIXUP>());
+        }
+        else if (num_remaining > 0)
+        {
+            // The last tile (possibly partially-full)
+            ConsumeTile<true>(num_remaining, tile_idx, tile_offset, tile_state, Int2Type<USE_ATOMIC_FIXUP>());
+        }
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_select_if.cuh b/third_party/cub/cub/agent/agent_select_if.cuh
new file mode 100644
index 0000000..52ca9fc
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_select_if.cuh
@@ -0,0 +1,703 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::AgentSelectIf implements a stateful abstraction of CUDA thread blocks for participating in device-wide select.
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "single_pass_scan_operators.cuh"
+#include "../block/block_load.cuh"
+#include "../block/block_store.cuh"
+#include "../block/block_scan.cuh"
+#include "../block/block_exchange.cuh"
+#include "../block/block_discontinuity.cuh"
+#include "../grid/grid_queue.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for AgentSelectIf
+ */
+template <
+    int                         _BLOCK_THREADS,                 ///< Threads per thread block
+    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
+    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
+    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
+    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
+struct AgentSelectIfPolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
+        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
+    };
+
+    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
+    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
+    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
+};
+
+
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+
+/**
+ * \brief AgentSelectIf implements a stateful abstraction of CUDA thread blocks for participating in device-wide selection
+ *
+ * Performs functor-based selection if SelectOpT functor type != NullType
+ * Otherwise performs flag-based selection if FlagsInputIterator's value type != NullType
+ * Otherwise performs discontinuity selection (keep unique)
+ */
+template <
+    typename    AgentSelectIfPolicyT,           ///< Parameterized AgentSelectIfPolicy tuning policy type
+    typename    InputIteratorT,                 ///< Random-access input iterator type for selection items
+    typename    FlagsInputIteratorT,            ///< Random-access input iterator type for selections (NullType* if a selection functor or discontinuity flagging is to be used for selection)
+    typename    SelectedOutputIteratorT,        ///< Random-access input iterator type for selection_flags items
+    typename    SelectOpT,                      ///< Selection operator type (NullType if selections or discontinuity flagging is to be used for selection)
+    typename    EqualityOpT,                    ///< Equality operator type (NullType if selection functor or selections is to be used for selection)
+    typename    OffsetT,                        ///< Signed integer type for global offsets
+    bool        KEEP_REJECTS>                   ///< Whether or not we push rejected items to the back of the output
+struct AgentSelectIf
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<SelectedOutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                                  // ... then the input iterator's value type,
+        typename std::iterator_traits<SelectedOutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    // The flag value type
+    typedef typename std::iterator_traits<FlagsInputIteratorT>::value_type FlagT;
+
+    // Tile status descriptor interface type
+    typedef ScanTileState<OffsetT> ScanTileStateT;
+
+    // Constants
+    enum
+    {
+        USE_SELECT_OP,
+        USE_SELECT_FLAGS,
+        USE_DISCONTINUITY,
+
+        BLOCK_THREADS           = AgentSelectIfPolicyT::BLOCK_THREADS,
+        ITEMS_PER_THREAD        = AgentSelectIfPolicyT::ITEMS_PER_THREAD,
+        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
+        TWO_PHASE_SCATTER       = (ITEMS_PER_THREAD > 1),
+
+        SELECT_METHOD           = (!Equals<SelectOpT, NullType>::VALUE) ?
+                                    USE_SELECT_OP :
+                                    (!Equals<FlagT, NullType>::VALUE) ?
+                                        USE_SELECT_FLAGS :
+                                        USE_DISCONTINUITY
+    };
+
+    // Cache-modified Input iterator wrapper type (for applying cache modifier) for items
+    typedef typename If<IsPointer<InputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentSelectIfPolicyT::LOAD_MODIFIER, InputT, OffsetT>,        // Wrap the native input pointer with CacheModifiedValuesInputIterator
+            InputIteratorT>::Type                                                               // Directly use the supplied input iterator type
+        WrappedInputIteratorT;
+
+    // Cache-modified Input iterator wrapper type (for applying cache modifier) for values
+    typedef typename If<IsPointer<FlagsInputIteratorT>::VALUE,
+            CacheModifiedInputIterator<AgentSelectIfPolicyT::LOAD_MODIFIER, FlagT, OffsetT>,    // Wrap the native input pointer with CacheModifiedValuesInputIterator
+            FlagsInputIteratorT>::Type                                                          // Directly use the supplied input iterator type
+        WrappedFlagsInputIteratorT;
+
+    // Parameterized BlockLoad type for input data
+    typedef BlockLoad<
+            OutputT,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            AgentSelectIfPolicyT::LOAD_ALGORITHM>
+        BlockLoadT;
+
+    // Parameterized BlockLoad type for flags
+    typedef BlockLoad<
+            FlagT,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            AgentSelectIfPolicyT::LOAD_ALGORITHM>
+        BlockLoadFlags;
+
+    // Parameterized BlockDiscontinuity type for items
+    typedef BlockDiscontinuity<
+            OutputT,
+            BLOCK_THREADS>
+        BlockDiscontinuityT;
+
+    // Parameterized BlockScan type
+    typedef BlockScan<
+            OffsetT,
+            BLOCK_THREADS,
+            AgentSelectIfPolicyT::SCAN_ALGORITHM>
+        BlockScanT;
+
+    // Callback type for obtaining tile prefix during block scan
+    typedef TilePrefixCallbackOp<
+            OffsetT,
+            cub::Sum,
+            ScanTileStateT>
+        TilePrefixCallbackOpT;
+
+    // Item exchange type
+    typedef OutputT ItemExchangeT[TILE_ITEMS];
+
+    // Shared memory type for this thread block
+    union _TempStorage
+    {
+        struct
+        {
+            typename BlockScanT::TempStorage                scan;           // Smem needed for tile scanning
+            typename TilePrefixCallbackOpT::TempStorage     prefix;         // Smem needed for cooperative prefix callback
+            typename BlockDiscontinuityT::TempStorage       discontinuity;  // Smem needed for discontinuity detection
+        };
+
+        // Smem needed for loading items
+        typename BlockLoadT::TempStorage load_items;
+
+        // Smem needed for loading values
+        typename BlockLoadFlags::TempStorage load_flags;
+
+        // Smem needed for compacting items (allows non POD items in this union)
+        Uninitialized<ItemExchangeT> raw_exchange;
+    };
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    _TempStorage&                   temp_storage;       ///< Reference to temp_storage
+    WrappedInputIteratorT           d_in;               ///< Input items
+    SelectedOutputIteratorT         d_selected_out;     ///< Unique output items
+    WrappedFlagsInputIteratorT      d_flags_in;         ///< Input selection flags (if applicable)
+    InequalityWrapper<EqualityOpT>  inequality_op;      ///< T inequality operator
+    SelectOpT                       select_op;          ///< Selection operator
+    OffsetT                         num_items;          ///< Total number of input items
+
+
+    //---------------------------------------------------------------------
+    // Constructor
+    //---------------------------------------------------------------------
+
+    // Constructor
+    __device__ __forceinline__
+    AgentSelectIf(
+        TempStorage                 &temp_storage,      ///< Reference to temp_storage
+        InputIteratorT              d_in,               ///< Input data
+        FlagsInputIteratorT         d_flags_in,         ///< Input selection flags (if applicable)
+        SelectedOutputIteratorT     d_selected_out,     ///< Output data
+        SelectOpT                   select_op,          ///< Selection operator
+        EqualityOpT                 equality_op,        ///< Equality operator
+        OffsetT                     num_items)          ///< Total number of input items
+    :
+        temp_storage(temp_storage.Alias()),
+        d_in(d_in),
+        d_flags_in(d_flags_in),
+        d_selected_out(d_selected_out),
+        select_op(select_op),
+        inequality_op(equality_op),
+        num_items(num_items)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Utility methods for initializing the selections
+    //---------------------------------------------------------------------
+
+    /**
+     * Initialize selections (specialized for selection operator)
+     */
+    template <bool IS_FIRST_TILE, bool IS_LAST_TILE>
+    __device__ __forceinline__ void InitializeSelections(
+        OffsetT                     /*tile_offset*/,
+        OffsetT                     num_tile_items,
+        OutputT                     (&items)[ITEMS_PER_THREAD],
+        OffsetT                     (&selection_flags)[ITEMS_PER_THREAD],
+        Int2Type<USE_SELECT_OP>     /*select_method*/)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            // Out-of-bounds items are selection_flags
+            selection_flags[ITEM] = 1;
+
+            if (!IS_LAST_TILE || (OffsetT(threadIdx.x * ITEMS_PER_THREAD) + ITEM < num_tile_items))
+                selection_flags[ITEM] = select_op(items[ITEM]);
+        }
+    }
+
+
+    /**
+     * Initialize selections (specialized for valid flags)
+     */
+    template <bool IS_FIRST_TILE, bool IS_LAST_TILE>
+    __device__ __forceinline__ void InitializeSelections(
+        OffsetT                     tile_offset,
+        OffsetT                     num_tile_items,
+        OutputT                     (&/*items*/)[ITEMS_PER_THREAD],
+        OffsetT                     (&selection_flags)[ITEMS_PER_THREAD],
+        Int2Type<USE_SELECT_FLAGS>  /*select_method*/)
+    {
+        CTA_SYNC();
+
+        FlagT flags[ITEMS_PER_THREAD];
+
+        if (IS_LAST_TILE)
+        {
+            // Out-of-bounds items are selection_flags
+            BlockLoadFlags(temp_storage.load_flags).Load(d_flags_in + tile_offset, flags, num_tile_items, 1);
+        }
+        else
+        {
+            BlockLoadFlags(temp_storage.load_flags).Load(d_flags_in + tile_offset, flags);
+        }
+
+        // Convert flag type to selection_flags type
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            selection_flags[ITEM] = flags[ITEM];
+        }
+    }
+
+
+    /**
+     * Initialize selections (specialized for discontinuity detection)
+     */
+    template <bool IS_FIRST_TILE, bool IS_LAST_TILE>
+    __device__ __forceinline__ void InitializeSelections(
+        OffsetT                     tile_offset,
+        OffsetT                     num_tile_items,
+        OutputT                     (&items)[ITEMS_PER_THREAD],
+        OffsetT                     (&selection_flags)[ITEMS_PER_THREAD],
+        Int2Type<USE_DISCONTINUITY> /*select_method*/)
+    {
+        if (IS_FIRST_TILE)
+        {
+            CTA_SYNC();
+
+            // Set head selection_flags.  First tile sets the first flag for the first item
+            BlockDiscontinuityT(temp_storage.discontinuity).FlagHeads(selection_flags, items, inequality_op);
+        }
+        else
+        {
+            OutputT tile_predecessor;
+            if (threadIdx.x == 0)
+                tile_predecessor = d_in[tile_offset - 1];
+
+            CTA_SYNC();
+
+            BlockDiscontinuityT(temp_storage.discontinuity).FlagHeads(selection_flags, items, inequality_op, tile_predecessor);
+        }
+
+        // Set selection flags for out-of-bounds items
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            // Set selection_flags for out-of-bounds items
+            if ((IS_LAST_TILE) && (OffsetT(threadIdx.x * ITEMS_PER_THREAD) + ITEM >= num_tile_items))
+                selection_flags[ITEM] = 1;
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Scatter utility methods
+    //---------------------------------------------------------------------
+
+    /**
+     * Scatter flagged items to output offsets (specialized for direct scattering)
+     */
+    template <bool IS_LAST_TILE, bool IS_FIRST_TILE>
+    __device__ __forceinline__ void ScatterDirect(
+        OutputT (&items)[ITEMS_PER_THREAD],
+        OffsetT (&selection_flags)[ITEMS_PER_THREAD],
+        OffsetT (&selection_indices)[ITEMS_PER_THREAD],
+        OffsetT num_selections)
+    {
+        // Scatter flagged items
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            if (selection_flags[ITEM])
+            {
+                if ((!IS_LAST_TILE) || selection_indices[ITEM] < num_selections)
+                {
+                    d_selected_out[selection_indices[ITEM]] = items[ITEM];
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Scatter flagged items to output offsets (specialized for two-phase scattering)
+     */
+    template <bool IS_LAST_TILE, bool IS_FIRST_TILE>
+    __device__ __forceinline__ void ScatterTwoPhase(
+        OutputT         (&items)[ITEMS_PER_THREAD],
+        OffsetT         (&selection_flags)[ITEMS_PER_THREAD],
+        OffsetT         (&selection_indices)[ITEMS_PER_THREAD],
+        int             /*num_tile_items*/,                         ///< Number of valid items in this tile
+        int             num_tile_selections,                        ///< Number of selections in this tile
+        OffsetT         num_selections_prefix,                      ///< Total number of selections prior to this tile
+        OffsetT         /*num_rejected_prefix*/,                    ///< Total number of rejections prior to this tile
+        Int2Type<false> /*is_keep_rejects*/)                        ///< Marker type indicating whether to keep rejected items in the second partition
+    {
+        CTA_SYNC();
+
+        // Compact and scatter items
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            int local_scatter_offset = selection_indices[ITEM] - num_selections_prefix;
+            if (selection_flags[ITEM])
+            {
+                temp_storage.raw_exchange.Alias()[local_scatter_offset] = items[ITEM];
+            }
+        }
+
+        CTA_SYNC();
+
+        for (int item = threadIdx.x; item < num_tile_selections; item += BLOCK_THREADS)
+        {
+            d_selected_out[num_selections_prefix + item] = temp_storage.raw_exchange.Alias()[item];
+        }
+    }
+
+
+    /**
+     * Scatter flagged items to output offsets (specialized for two-phase scattering)
+     */
+    template <bool IS_LAST_TILE, bool IS_FIRST_TILE>
+    __device__ __forceinline__ void ScatterTwoPhase(
+        OutputT         (&items)[ITEMS_PER_THREAD],
+        OffsetT         (&selection_flags)[ITEMS_PER_THREAD],
+        OffsetT         (&selection_indices)[ITEMS_PER_THREAD],
+        int             num_tile_items,                             ///< Number of valid items in this tile
+        int             num_tile_selections,                        ///< Number of selections in this tile
+        OffsetT         num_selections_prefix,                      ///< Total number of selections prior to this tile
+        OffsetT         num_rejected_prefix,                        ///< Total number of rejections prior to this tile
+        Int2Type<true>  /*is_keep_rejects*/)                        ///< Marker type indicating whether to keep rejected items in the second partition
+    {
+        CTA_SYNC();
+
+        int tile_num_rejections = num_tile_items - num_tile_selections;
+
+        // Scatter items to shared memory (rejections first)
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            int item_idx                = (threadIdx.x * ITEMS_PER_THREAD) + ITEM;
+            int local_selection_idx     = selection_indices[ITEM] - num_selections_prefix;
+            int local_rejection_idx     = item_idx - local_selection_idx;
+            int local_scatter_offset    = (selection_flags[ITEM]) ?
+                                            tile_num_rejections + local_selection_idx :
+                                            local_rejection_idx;
+
+            temp_storage.raw_exchange.Alias()[local_scatter_offset] = items[ITEM];
+        }
+
+        CTA_SYNC();
+
+        // Gather items from shared memory and scatter to global
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            int item_idx            = (ITEM * BLOCK_THREADS) + threadIdx.x;
+            int rejection_idx       = item_idx;
+            int selection_idx       = item_idx - tile_num_rejections;
+            OffsetT scatter_offset  = (item_idx < tile_num_rejections) ?
+                                        num_items - num_rejected_prefix - rejection_idx - 1 :
+                                        num_selections_prefix + selection_idx;
+
+            OutputT item = temp_storage.raw_exchange.Alias()[item_idx];
+
+            if (!IS_LAST_TILE || (item_idx < num_tile_items))
+            {
+                d_selected_out[scatter_offset] = item;
+            }
+        }
+    }
+
+
+    /**
+     * Scatter flagged items
+     */
+    template <bool IS_LAST_TILE, bool IS_FIRST_TILE>
+    __device__ __forceinline__ void Scatter(
+        OutputT         (&items)[ITEMS_PER_THREAD],
+        OffsetT         (&selection_flags)[ITEMS_PER_THREAD],
+        OffsetT         (&selection_indices)[ITEMS_PER_THREAD],
+        int             num_tile_items,                             ///< Number of valid items in this tile
+        int             num_tile_selections,                        ///< Number of selections in this tile
+        OffsetT         num_selections_prefix,                      ///< Total number of selections prior to this tile
+        OffsetT         num_rejected_prefix,                        ///< Total number of rejections prior to this tile
+        OffsetT         num_selections)                             ///< Total number of selections including this tile
+    {
+        // Do a two-phase scatter if (a) keeping both partitions or (b) two-phase is enabled and the average number of selection_flags items per thread is greater than one
+        if (KEEP_REJECTS || (TWO_PHASE_SCATTER && (num_tile_selections > BLOCK_THREADS)))
+        {
+            ScatterTwoPhase<IS_LAST_TILE, IS_FIRST_TILE>(
+                items,
+                selection_flags,
+                selection_indices,
+                num_tile_items,
+                num_tile_selections,
+                num_selections_prefix,
+                num_rejected_prefix,
+                Int2Type<KEEP_REJECTS>());
+        }
+        else
+        {
+            ScatterDirect<IS_LAST_TILE, IS_FIRST_TILE>(
+                items,
+                selection_flags,
+                selection_indices,
+                num_selections);
+        }
+    }
+
+    //---------------------------------------------------------------------
+    // Cooperatively scan a device-wide sequence of tiles with other CTAs
+    //---------------------------------------------------------------------
+
+
+    /**
+     * Process first tile of input (dynamic chained scan).  Returns the running count of selections (including this tile)
+     */
+    template <bool IS_LAST_TILE>
+    __device__ __forceinline__ OffsetT ConsumeFirstTile(
+        int                 num_tile_items,      ///< Number of input items comprising this tile
+        OffsetT             tile_offset,        ///< Tile offset
+        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
+    {
+        OutputT     items[ITEMS_PER_THREAD];
+        OffsetT     selection_flags[ITEMS_PER_THREAD];
+        OffsetT     selection_indices[ITEMS_PER_THREAD];
+
+        // Load items
+        if (IS_LAST_TILE)
+            BlockLoadT(temp_storage.load_items).Load(d_in + tile_offset, items, num_tile_items);
+        else
+            BlockLoadT(temp_storage.load_items).Load(d_in + tile_offset, items);
+
+        // Initialize selection_flags
+        InitializeSelections<true, IS_LAST_TILE>(
+            tile_offset,
+            num_tile_items,
+            items,
+            selection_flags,
+            Int2Type<SELECT_METHOD>());
+
+        CTA_SYNC();
+
+        // Exclusive scan of selection_flags
+        OffsetT num_tile_selections;
+        BlockScanT(temp_storage.scan).ExclusiveSum(selection_flags, selection_indices, num_tile_selections);
+
+        if (threadIdx.x == 0)
+        {
+            // Update tile status if this is not the last tile
+            if (!IS_LAST_TILE)
+                tile_state.SetInclusive(0, num_tile_selections);
+        }
+
+        // Discount any out-of-bounds selections
+        if (IS_LAST_TILE)
+            num_tile_selections -= (TILE_ITEMS - num_tile_items);
+
+        // Scatter flagged items
+        Scatter<IS_LAST_TILE, true>(
+            items,
+            selection_flags,
+            selection_indices,
+            num_tile_items,
+            num_tile_selections,
+            0,
+            0,
+            num_tile_selections);
+
+        return num_tile_selections;
+    }
+
+
+    /**
+     * Process subsequent tile of input (dynamic chained scan).  Returns the running count of selections (including this tile)
+     */
+    template <bool IS_LAST_TILE>
+    __device__ __forceinline__ OffsetT ConsumeSubsequentTile(
+        int                 num_tile_items,      ///< Number of input items comprising this tile
+        int                 tile_idx,           ///< Tile index
+        OffsetT             tile_offset,        ///< Tile offset
+        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
+    {
+        OutputT     items[ITEMS_PER_THREAD];
+        OffsetT     selection_flags[ITEMS_PER_THREAD];
+        OffsetT     selection_indices[ITEMS_PER_THREAD];
+
+        // Load items
+        if (IS_LAST_TILE)
+            BlockLoadT(temp_storage.load_items).Load(d_in + tile_offset, items, num_tile_items);
+        else
+            BlockLoadT(temp_storage.load_items).Load(d_in + tile_offset, items);
+
+        // Initialize selection_flags
+        InitializeSelections<false, IS_LAST_TILE>(
+            tile_offset,
+            num_tile_items,
+            items,
+            selection_flags,
+            Int2Type<SELECT_METHOD>());
+
+        CTA_SYNC();
+
+        // Exclusive scan of values and selection_flags
+        TilePrefixCallbackOpT prefix_op(tile_state, temp_storage.prefix, cub::Sum(), tile_idx);
+        BlockScanT(temp_storage.scan).ExclusiveSum(selection_flags, selection_indices, prefix_op);
+
+        OffsetT num_tile_selections     = prefix_op.GetBlockAggregate();
+        OffsetT num_selections          = prefix_op.GetInclusivePrefix();
+        OffsetT num_selections_prefix   = prefix_op.GetExclusivePrefix();
+        OffsetT num_rejected_prefix     = (tile_idx * TILE_ITEMS) - num_selections_prefix;
+
+        // Discount any out-of-bounds selections
+        if (IS_LAST_TILE)
+        {
+            int num_discount    = TILE_ITEMS - num_tile_items;
+            num_selections      -= num_discount;
+            num_tile_selections -= num_discount;
+        }
+
+        // Scatter flagged items
+        Scatter<IS_LAST_TILE, false>(
+            items,
+            selection_flags,
+            selection_indices,
+            num_tile_items,
+            num_tile_selections,
+            num_selections_prefix,
+            num_rejected_prefix,
+            num_selections);
+
+        return num_selections;
+    }
+
+
+    /**
+     * Process a tile of input
+     */
+    template <bool IS_LAST_TILE>
+    __device__ __forceinline__ OffsetT ConsumeTile(
+        int                 num_tile_items,         ///< Number of input items comprising this tile
+        int                 tile_idx,           ///< Tile index
+        OffsetT             tile_offset,        ///< Tile offset
+        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
+    {
+        OffsetT num_selections;
+        if (tile_idx == 0)
+        {
+            num_selections = ConsumeFirstTile<IS_LAST_TILE>(num_tile_items, tile_offset, tile_state);
+        }
+        else
+        {
+            num_selections = ConsumeSubsequentTile<IS_LAST_TILE>(num_tile_items, tile_idx, tile_offset, tile_state);
+        }
+
+        return num_selections;
+    }
+
+
+    /**
+     * Scan tiles of items as part of a dynamic chained scan
+     */
+    template <typename NumSelectedIteratorT>        ///< Output iterator type for recording number of items selection_flags
+    __device__ __forceinline__ void ConsumeRange(
+        int                     num_tiles,          ///< Total number of input tiles
+        ScanTileStateT&         tile_state,         ///< Global tile state descriptor
+        NumSelectedIteratorT    d_num_selected_out) ///< Output total number selection_flags
+    {
+        // Blocks are launched in increasing order, so just assign one tile per block
+        int     tile_idx        = (blockIdx.x * gridDim.y) + blockIdx.y;    // Current tile index
+        OffsetT tile_offset     = tile_idx * TILE_ITEMS;                    // Global offset for the current tile
+
+        if (tile_idx < num_tiles - 1)
+        {
+            // Not the last tile (full)
+            ConsumeTile<false>(TILE_ITEMS, tile_idx, tile_offset, tile_state);
+        }
+        else
+        {
+            // The last tile (possibly partially-full)
+            OffsetT num_remaining   = num_items - tile_offset;
+            OffsetT num_selections  = ConsumeTile<true>(num_remaining, tile_idx, tile_offset, tile_state);
+
+            if (threadIdx.x == 0)
+            {
+                // Output the total number of items selection_flags
+                *d_num_selected_out = num_selections;
+            }
+        }
+    }
+
+};
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/agent_spmv_orig.cuh b/third_party/cub/cub/agent/agent_spmv_orig.cuh
new file mode 100644
index 0000000..54e2a13
--- /dev/null
+++ b/third_party/cub/cub/agent/agent_spmv_orig.cuh
@@ -0,0 +1,670 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::AgentSpmv implements a stateful abstraction of CUDA thread blocks for participating in device-wide SpMV.
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "../util_type.cuh"
+#include "../block/block_reduce.cuh"
+#include "../block/block_scan.cuh"
+#include "../block/block_exchange.cuh"
+#include "../thread/thread_search.cuh"
+#include "../thread/thread_operators.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../iterator/counting_input_iterator.cuh"
+#include "../iterator/tex_ref_input_iterator.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Tuning policy
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for AgentSpmv
+ */
+template <
+    int                             _BLOCK_THREADS,                         ///< Threads per thread block
+    int                             _ITEMS_PER_THREAD,                      ///< Items per thread (per tile of input)
+    CacheLoadModifier               _ROW_OFFSETS_SEARCH_LOAD_MODIFIER,      ///< Cache load modifier for reading CSR row-offsets during search
+    CacheLoadModifier               _ROW_OFFSETS_LOAD_MODIFIER,             ///< Cache load modifier for reading CSR row-offsets
+    CacheLoadModifier               _COLUMN_INDICES_LOAD_MODIFIER,          ///< Cache load modifier for reading CSR column-indices
+    CacheLoadModifier               _VALUES_LOAD_MODIFIER,                  ///< Cache load modifier for reading CSR values
+    CacheLoadModifier               _VECTOR_VALUES_LOAD_MODIFIER,           ///< Cache load modifier for reading vector values
+    bool                            _DIRECT_LOAD_NONZEROS,                  ///< Whether to load nonzeros directly from global during sequential merging (vs. pre-staged through shared memory)
+    BlockScanAlgorithm              _SCAN_ALGORITHM>                        ///< The BlockScan algorithm to use
+struct AgentSpmvPolicy
+{
+    enum
+    {
+        BLOCK_THREADS                                                   = _BLOCK_THREADS,                       ///< Threads per thread block
+        ITEMS_PER_THREAD                                                = _ITEMS_PER_THREAD,                    ///< Items per thread (per tile of input)
+        DIRECT_LOAD_NONZEROS                                            = _DIRECT_LOAD_NONZEROS,                ///< Whether to load nonzeros directly from global during sequential merging (pre-staged through shared memory)
+    };
+
+    static const CacheLoadModifier  ROW_OFFSETS_SEARCH_LOAD_MODIFIER    = _ROW_OFFSETS_SEARCH_LOAD_MODIFIER;    ///< Cache load modifier for reading CSR row-offsets
+    static const CacheLoadModifier  ROW_OFFSETS_LOAD_MODIFIER           = _ROW_OFFSETS_LOAD_MODIFIER;           ///< Cache load modifier for reading CSR row-offsets
+    static const CacheLoadModifier  COLUMN_INDICES_LOAD_MODIFIER        = _COLUMN_INDICES_LOAD_MODIFIER;        ///< Cache load modifier for reading CSR column-indices
+    static const CacheLoadModifier  VALUES_LOAD_MODIFIER                = _VALUES_LOAD_MODIFIER;                ///< Cache load modifier for reading CSR values
+    static const CacheLoadModifier  VECTOR_VALUES_LOAD_MODIFIER         = _VECTOR_VALUES_LOAD_MODIFIER;         ///< Cache load modifier for reading vector values
+    static const BlockScanAlgorithm SCAN_ALGORITHM                      = _SCAN_ALGORITHM;                      ///< The BlockScan algorithm to use
+
+};
+
+
+/******************************************************************************
+ * Thread block abstractions
+ ******************************************************************************/
+
+template <
+    typename        ValueT,              ///< Matrix and vector value type
+    typename        OffsetT>             ///< Signed integer type for sequence offsets
+struct SpmvParams
+{
+    ValueT*         d_values;            ///< Pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix <b>A</b>.
+    OffsetT*        d_row_end_offsets;   ///< Pointer to the array of \p m offsets demarcating the end of every row in \p d_column_indices and \p d_values
+    OffsetT*        d_column_indices;    ///< Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix <b>A</b>.  (Indices are zero-valued.)
+    ValueT*         d_vector_x;          ///< Pointer to the array of \p num_cols values corresponding to the dense input vector <em>x</em>
+    ValueT*         d_vector_y;          ///< Pointer to the array of \p num_rows values corresponding to the dense output vector <em>y</em>
+    int             num_rows;            ///< Number of rows of matrix <b>A</b>.
+    int             num_cols;            ///< Number of columns of matrix <b>A</b>.
+    int             num_nonzeros;        ///< Number of nonzero elements of matrix <b>A</b>.
+    ValueT          alpha;               ///< Alpha multiplicand
+    ValueT          beta;                ///< Beta addend-multiplicand
+
+    TexRefInputIterator<ValueT, 66778899, OffsetT>  t_vector_x;
+};
+
+
+/**
+ * \brief AgentSpmv implements a stateful abstraction of CUDA thread blocks for participating in device-wide SpMV.
+ */
+template <
+    typename    AgentSpmvPolicyT,           ///< Parameterized AgentSpmvPolicy tuning policy type
+    typename    ValueT,                     ///< Matrix and vector value type
+    typename    OffsetT,                    ///< Signed integer type for sequence offsets
+    bool        HAS_ALPHA,                  ///< Whether the input parameter \p alpha is 1
+    bool        HAS_BETA,                   ///< Whether the input parameter \p beta is 0
+    int         PTX_ARCH = CUB_PTX_ARCH>    ///< PTX compute capability
+struct AgentSpmv
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// Constants
+    enum
+    {
+        BLOCK_THREADS           = AgentSpmvPolicyT::BLOCK_THREADS,
+        ITEMS_PER_THREAD        = AgentSpmvPolicyT::ITEMS_PER_THREAD,
+        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
+    };
+
+    /// 2D merge path coordinate type
+    typedef typename CubVector<OffsetT, 2>::Type CoordinateT;
+
+    /// Input iterator wrapper types (for applying cache modifiers)
+
+    typedef CacheModifiedInputIterator<
+            AgentSpmvPolicyT::ROW_OFFSETS_SEARCH_LOAD_MODIFIER,
+            OffsetT,
+            OffsetT>
+        RowOffsetsSearchIteratorT;
+
+    typedef CacheModifiedInputIterator<
+            AgentSpmvPolicyT::ROW_OFFSETS_LOAD_MODIFIER,
+            OffsetT,
+            OffsetT>
+        RowOffsetsIteratorT;
+
+    typedef CacheModifiedInputIterator<
+            AgentSpmvPolicyT::COLUMN_INDICES_LOAD_MODIFIER,
+            OffsetT,
+            OffsetT>
+        ColumnIndicesIteratorT;
+
+    typedef CacheModifiedInputIterator<
+            AgentSpmvPolicyT::VALUES_LOAD_MODIFIER,
+            ValueT,
+            OffsetT>
+        ValueIteratorT;
+
+    typedef CacheModifiedInputIterator<
+            AgentSpmvPolicyT::VECTOR_VALUES_LOAD_MODIFIER,
+            ValueT,
+            OffsetT>
+        VectorValueIteratorT;
+
+    // Tuple type for scanning (pairs accumulated segment-value with segment-index)
+    typedef KeyValuePair<OffsetT, ValueT> KeyValuePairT;
+
+    // Reduce-value-by-segment scan operator
+    typedef ReduceByKeyOp<cub::Sum> ReduceBySegmentOpT;
+
+    // BlockReduce specialization
+    typedef BlockReduce<
+            ValueT,
+            BLOCK_THREADS,
+            BLOCK_REDUCE_WARP_REDUCTIONS>
+        BlockReduceT;
+
+    // BlockScan specialization
+    typedef BlockScan<
+            KeyValuePairT,
+            BLOCK_THREADS,
+            AgentSpmvPolicyT::SCAN_ALGORITHM>
+        BlockScanT;
+
+    // BlockScan specialization
+    typedef BlockScan<
+            ValueT,
+            BLOCK_THREADS,
+            AgentSpmvPolicyT::SCAN_ALGORITHM>
+        BlockPrefixSumT;
+
+    // BlockExchange specialization
+    typedef BlockExchange<
+            ValueT,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD>
+        BlockExchangeT;
+
+    /// Merge item type (either a non-zero value or a row-end offset)
+    union MergeItem
+    {
+        // Value type to pair with index type OffsetT (NullType if loading values directly during merge)
+        typedef typename If<AgentSpmvPolicyT::DIRECT_LOAD_NONZEROS, NullType, ValueT>::Type MergeValueT;
+
+        OffsetT     row_end_offset;
+        MergeValueT nonzero;
+    };
+
+    /// Shared memory type required by this thread block
+    struct _TempStorage
+    {
+        CoordinateT tile_coords[2];
+
+        union Aliasable
+        {
+            // Smem needed for tile of merge items
+            MergeItem merge_items[ITEMS_PER_THREAD + TILE_ITEMS + 1];
+
+            // Smem needed for block exchange
+            typename BlockExchangeT::TempStorage exchange;
+
+            // Smem needed for block-wide reduction
+            typename BlockReduceT::TempStorage reduce;
+
+            // Smem needed for tile scanning
+            typename BlockScanT::TempStorage scan;
+
+            // Smem needed for tile prefix sum
+            typename BlockPrefixSumT::TempStorage prefix_sum;
+
+        } aliasable;
+    };
+
+    /// Temporary storage type (unionable)
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+
+    _TempStorage&                   temp_storage;         /// Reference to temp_storage
+
+    SpmvParams<ValueT, OffsetT>&    spmv_params;
+
+    ValueIteratorT                  wd_values;            ///< Wrapped pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix <b>A</b>.
+    RowOffsetsIteratorT             wd_row_end_offsets;   ///< Wrapped Pointer to the array of \p m offsets demarcating the end of every row in \p d_column_indices and \p d_values
+    ColumnIndicesIteratorT          wd_column_indices;    ///< Wrapped Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix <b>A</b>.  (Indices are zero-valued.)
+    VectorValueIteratorT            wd_vector_x;          ///< Wrapped Pointer to the array of \p num_cols values corresponding to the dense input vector <em>x</em>
+    VectorValueIteratorT            wd_vector_y;          ///< Wrapped Pointer to the array of \p num_cols values corresponding to the dense input vector <em>x</em>
+
+
+    //---------------------------------------------------------------------
+    // Interface
+    //---------------------------------------------------------------------
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__ AgentSpmv(
+        TempStorage&                    temp_storage,           ///< Reference to temp_storage
+        SpmvParams<ValueT, OffsetT>&    spmv_params)            ///< SpMV input parameter bundle
+    :
+        temp_storage(temp_storage.Alias()),
+        spmv_params(spmv_params),
+        wd_values(spmv_params.d_values),
+        wd_row_end_offsets(spmv_params.d_row_end_offsets),
+        wd_column_indices(spmv_params.d_column_indices),
+        wd_vector_x(spmv_params.d_vector_x),
+        wd_vector_y(spmv_params.d_vector_y)
+    {}
+
+
+
+
+    /**
+     * Consume a merge tile, specialized for direct-load of nonzeros
+     */
+    __device__ __forceinline__ KeyValuePairT ConsumeTile(
+        int             tile_idx,
+        CoordinateT     tile_start_coord,
+        CoordinateT     tile_end_coord,
+        Int2Type<true>  is_direct_load)     ///< Marker type indicating whether to load nonzeros directly during path-discovery or beforehand in batch
+    {
+        int         tile_num_rows           = tile_end_coord.x - tile_start_coord.x;
+        int         tile_num_nonzeros       = tile_end_coord.y - tile_start_coord.y;
+        OffsetT*    s_tile_row_end_offsets  = &temp_storage.aliasable.merge_items[0].row_end_offset;
+
+        // Gather the row end-offsets for the merge tile into shared memory
+        for (int item = threadIdx.x; item <= tile_num_rows; item += BLOCK_THREADS)
+        {
+            s_tile_row_end_offsets[item] = wd_row_end_offsets[tile_start_coord.x + item];
+        }
+
+        CTA_SYNC();
+
+        // Search for the thread's starting coordinate within the merge tile
+        CountingInputIterator<OffsetT>  tile_nonzero_indices(tile_start_coord.y);
+        CoordinateT                     thread_start_coord;
+
+        MergePathSearch(
+            OffsetT(threadIdx.x * ITEMS_PER_THREAD),    // Diagonal
+            s_tile_row_end_offsets,                     // List A
+            tile_nonzero_indices,                       // List B
+            tile_num_rows,
+            tile_num_nonzeros,
+            thread_start_coord);
+
+        CTA_SYNC();            // Perf-sync
+
+        // Compute the thread's merge path segment
+        CoordinateT     thread_current_coord = thread_start_coord;
+        KeyValuePairT   scan_segment[ITEMS_PER_THREAD];
+
+        ValueT          running_total = 0.0;
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            OffsetT nonzero_idx         = CUB_MIN(tile_nonzero_indices[thread_current_coord.y], spmv_params.num_nonzeros - 1);
+            OffsetT column_idx          = wd_column_indices[nonzero_idx];
+            ValueT  value               = wd_values[nonzero_idx];
+
+            ValueT  vector_value        = spmv_params.t_vector_x[column_idx];
+#if (CUB_PTX_ARCH >= 350)
+            vector_value                = wd_vector_x[column_idx];
+#endif
+            ValueT  nonzero             = value * vector_value;
+
+            OffsetT row_end_offset      = s_tile_row_end_offsets[thread_current_coord.x];
+
+            if (tile_nonzero_indices[thread_current_coord.y] < row_end_offset)
+            {
+                // Move down (accumulate)
+                running_total += nonzero;
+                scan_segment[ITEM].value    = running_total;
+                scan_segment[ITEM].key      = tile_num_rows;
+                ++thread_current_coord.y;
+            }
+            else
+            {
+                // Move right (reset)
+                scan_segment[ITEM].value    = running_total;
+                scan_segment[ITEM].key      = thread_current_coord.x;
+                running_total               = 0.0;
+                ++thread_current_coord.x;
+            }
+        }
+
+        CTA_SYNC();
+
+        // Block-wide reduce-value-by-segment
+        KeyValuePairT       tile_carry;
+        ReduceBySegmentOpT  scan_op;
+        KeyValuePairT       scan_item;
+
+        scan_item.value = running_total;
+        scan_item.key   = thread_current_coord.x;
+
+        BlockScanT(temp_storage.aliasable.scan).ExclusiveScan(scan_item, scan_item, scan_op, tile_carry);
+
+        if (tile_num_rows > 0)
+        {
+            if (threadIdx.x == 0)
+                scan_item.key = -1;
+
+            // Direct scatter
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+            {
+                if (scan_segment[ITEM].key < tile_num_rows)
+                {
+                    if (scan_item.key == scan_segment[ITEM].key)
+                        scan_segment[ITEM].value = scan_item.value + scan_segment[ITEM].value;
+
+                    if (HAS_ALPHA)
+                    {
+                        scan_segment[ITEM].value *= spmv_params.alpha;
+                    }
+
+                    if (HAS_BETA)
+                    {
+                        // Update the output vector element
+                        ValueT addend = spmv_params.beta * wd_vector_y[tile_start_coord.x + scan_segment[ITEM].key];
+                        scan_segment[ITEM].value += addend;
+                    }
+
+                    // Set the output vector element
+                    spmv_params.d_vector_y[tile_start_coord.x + scan_segment[ITEM].key] = scan_segment[ITEM].value;
+                }
+            }
+        }
+
+        // Return the tile's running carry-out
+        return tile_carry;
+    }
+
+
+
+    /**
+     * Consume a merge tile, specialized for indirect load of nonzeros
+     */
+    __device__ __forceinline__ KeyValuePairT ConsumeTile(
+        int             tile_idx,
+        CoordinateT     tile_start_coord,
+        CoordinateT     tile_end_coord,
+        Int2Type<false> is_direct_load)     ///< Marker type indicating whether to load nonzeros directly during path-discovery or beforehand in batch
+    {
+        int         tile_num_rows           = tile_end_coord.x - tile_start_coord.x;
+        int         tile_num_nonzeros       = tile_end_coord.y - tile_start_coord.y;
+
+#if (CUB_PTX_ARCH >= 520)
+
+        OffsetT*    s_tile_row_end_offsets  = &temp_storage.aliasable.merge_items[0].row_end_offset;
+        ValueT*     s_tile_nonzeros         = &temp_storage.aliasable.merge_items[tile_num_rows + ITEMS_PER_THREAD].nonzero;
+
+        // Gather the nonzeros for the merge tile into shared memory
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            int nonzero_idx = threadIdx.x + (ITEM * BLOCK_THREADS);
+
+            ValueIteratorT a                = wd_values + tile_start_coord.y + nonzero_idx;
+            ColumnIndicesIteratorT ci       = wd_column_indices + tile_start_coord.y + nonzero_idx;
+            ValueT* s                       = s_tile_nonzeros + nonzero_idx;
+
+            if (nonzero_idx < tile_num_nonzeros)
+            {
+
+                OffsetT column_idx              = *ci;
+                ValueT  value                   = *a;
+
+                ValueT  vector_value            = spmv_params.t_vector_x[column_idx];
+                vector_value                    = wd_vector_x[column_idx];
+
+                ValueT  nonzero                 = value * vector_value;
+
+                *s    = nonzero;
+            }
+        }
+
+
+#else
+
+        OffsetT*    s_tile_row_end_offsets  = &temp_storage.aliasable.merge_items[0].row_end_offset;
+        ValueT*     s_tile_nonzeros         = &temp_storage.aliasable.merge_items[tile_num_rows + ITEMS_PER_THREAD].nonzero;
+
+        // Gather the nonzeros for the merge tile into shared memory
+        if (tile_num_nonzeros > 0)
+        {
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+            {
+                int     nonzero_idx             = threadIdx.x + (ITEM * BLOCK_THREADS);
+                nonzero_idx                     = CUB_MIN(nonzero_idx, tile_num_nonzeros - 1);
+
+                OffsetT column_idx              = wd_column_indices[tile_start_coord.y + nonzero_idx];
+                ValueT  value                   = wd_values[tile_start_coord.y + nonzero_idx];
+
+                ValueT  vector_value            = spmv_params.t_vector_x[column_idx];
+#if (CUB_PTX_ARCH >= 350)
+                vector_value                    = wd_vector_x[column_idx];
+#endif
+                ValueT  nonzero                 = value * vector_value;
+
+                s_tile_nonzeros[nonzero_idx]    = nonzero;
+            }
+        }
+
+#endif
+
+        // Gather the row end-offsets for the merge tile into shared memory
+        #pragma unroll 1
+        for (int item = threadIdx.x; item <= tile_num_rows; item += BLOCK_THREADS)
+        {
+            s_tile_row_end_offsets[item] = wd_row_end_offsets[tile_start_coord.x + item];
+        }
+
+        CTA_SYNC();
+
+        // Search for the thread's starting coordinate within the merge tile
+        CountingInputIterator<OffsetT>  tile_nonzero_indices(tile_start_coord.y);
+        CoordinateT                     thread_start_coord;
+
+        MergePathSearch(
+            OffsetT(threadIdx.x * ITEMS_PER_THREAD),    // Diagonal
+            s_tile_row_end_offsets,                     // List A
+            tile_nonzero_indices,                       // List B
+            tile_num_rows,
+            tile_num_nonzeros,
+            thread_start_coord);
+
+        CTA_SYNC();            // Perf-sync
+
+        // Compute the thread's merge path segment
+        CoordinateT     thread_current_coord = thread_start_coord;
+        KeyValuePairT   scan_segment[ITEMS_PER_THREAD];
+        ValueT          running_total = 0.0;
+
+        OffsetT row_end_offset  = s_tile_row_end_offsets[thread_current_coord.x];
+        ValueT  nonzero         = s_tile_nonzeros[thread_current_coord.y];
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            if (tile_nonzero_indices[thread_current_coord.y] < row_end_offset)
+            {
+                // Move down (accumulate)
+                scan_segment[ITEM].value    = nonzero;
+                running_total               += nonzero;
+                ++thread_current_coord.y;
+                nonzero                     = s_tile_nonzeros[thread_current_coord.y];
+            }
+            else
+            {
+                // Move right (reset)
+                scan_segment[ITEM].value    = 0.0;
+                running_total               = 0.0;
+                ++thread_current_coord.x;
+                row_end_offset              = s_tile_row_end_offsets[thread_current_coord.x];
+            }
+
+            scan_segment[ITEM].key = thread_current_coord.x;
+        }
+
+        CTA_SYNC();
+
+        // Block-wide reduce-value-by-segment
+        KeyValuePairT       tile_carry;
+        ReduceBySegmentOpT  scan_op;
+        KeyValuePairT       scan_item;
+
+        scan_item.value = running_total;
+        scan_item.key = thread_current_coord.x;
+
+        BlockScanT(temp_storage.aliasable.scan).ExclusiveScan(scan_item, scan_item, scan_op, tile_carry);
+
+        if (threadIdx.x == 0)
+        {
+            scan_item.key = thread_start_coord.x;
+            scan_item.value = 0.0;
+        }
+
+        if (tile_num_rows > 0)
+        {
+
+            CTA_SYNC();
+
+            // Scan downsweep and scatter
+            ValueT* s_partials = &temp_storage.aliasable.merge_items[0].nonzero;
+
+            if (scan_item.key != scan_segment[0].key)
+            {
+                s_partials[scan_item.key] = scan_item.value;
+            }
+            else
+            {
+                scan_segment[0].value += scan_item.value;
+            }
+
+            #pragma unroll
+            for (int ITEM = 1; ITEM < ITEMS_PER_THREAD; ++ITEM)
+            {
+                if (scan_segment[ITEM - 1].key != scan_segment[ITEM].key)
+                {
+                    s_partials[scan_segment[ITEM - 1].key] = scan_segment[ITEM - 1].value;
+                }
+                else
+                {
+                    scan_segment[ITEM].value += scan_segment[ITEM - 1].value;
+                }
+            }
+
+            CTA_SYNC();
+
+            #pragma unroll 1
+            for (int item = threadIdx.x; item < tile_num_rows; item += BLOCK_THREADS)
+            {
+                spmv_params.d_vector_y[tile_start_coord.x + item] = s_partials[item];
+            }
+        }
+
+        // Return the tile's running carry-out
+        return tile_carry;
+    }
+
+
+    /**
+     * Consume input tile
+     */
+    __device__ __forceinline__ void ConsumeTile(
+        CoordinateT*    d_tile_coordinates,     ///< [in] Pointer to the temporary array of tile starting coordinates
+        KeyValuePairT*  d_tile_carry_pairs,     ///< [out] Pointer to the temporary array carry-out dot product row-ids, one per block
+        int             num_merge_tiles)        ///< [in] Number of merge tiles
+    {
+        int tile_idx = (blockIdx.x * gridDim.y) + blockIdx.y;    // Current tile index
+
+        if (tile_idx >= num_merge_tiles)
+            return;
+
+        // Read our starting coordinates
+        if (threadIdx.x < 2)
+        {
+            if (d_tile_coordinates == NULL)
+            {
+                // Search our starting coordinates
+                OffsetT                         diagonal = (tile_idx + threadIdx.x) * TILE_ITEMS;
+                CoordinateT                     tile_coord;
+                CountingInputIterator<OffsetT>  nonzero_indices(0);
+
+                // Search the merge path
+                MergePathSearch(
+                    diagonal,
+                    RowOffsetsSearchIteratorT(spmv_params.d_row_end_offsets),
+                    nonzero_indices,
+                    spmv_params.num_rows,
+                    spmv_params.num_nonzeros,
+                    tile_coord);
+
+                temp_storage.tile_coords[threadIdx.x] = tile_coord;
+            }
+            else
+            {
+                temp_storage.tile_coords[threadIdx.x] = d_tile_coordinates[tile_idx + threadIdx.x];
+            }
+        }
+
+        CTA_SYNC();
+
+        CoordinateT tile_start_coord     = temp_storage.tile_coords[0];
+        CoordinateT tile_end_coord       = temp_storage.tile_coords[1];
+
+        // Consume multi-segment tile
+        KeyValuePairT tile_carry = ConsumeTile(
+            tile_idx,
+            tile_start_coord,
+            tile_end_coord,
+            Int2Type<AgentSpmvPolicyT::DIRECT_LOAD_NONZEROS>());
+
+        // Output the tile's carry-out
+        if (threadIdx.x == 0)
+        {
+            if (HAS_ALPHA)
+                tile_carry.value *= spmv_params.alpha;
+
+            tile_carry.key += tile_start_coord.x;
+            d_tile_carry_pairs[tile_idx]    = tile_carry;
+        }
+    }
+
+
+};
+
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/agent/single_pass_scan_operators.cuh b/third_party/cub/cub/agent/single_pass_scan_operators.cuh
new file mode 100644
index 0000000..53409bd
--- /dev/null
+++ b/third_party/cub/cub/agent/single_pass_scan_operators.cuh
@@ -0,0 +1,815 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Callback operator types for supplying BlockScan prefixes
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../warp/warp_reduce.cuh"
+#include "../util_arch.cuh"
+#include "../util_device.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Prefix functor type for maintaining a running prefix while scanning a
+ * region independent of other thread blocks
+ ******************************************************************************/
+
+/**
+ * Stateful callback operator type for supplying BlockScan prefixes.
+ * Maintains a running prefix that can be applied to consecutive
+ * BlockScan operations.
+ */
+template <
+    typename T,                 ///< BlockScan value type
+    typename ScanOpT>            ///< Wrapped scan operator type
+struct BlockScanRunningPrefixOp
+{
+    ScanOpT     op;                 ///< Wrapped scan operator
+    T           running_total;      ///< Running block-wide prefix
+
+    /// Constructor
+    __device__ __forceinline__ BlockScanRunningPrefixOp(ScanOpT op)
+    :
+        op(op)
+    {}
+
+    /// Constructor
+    __device__ __forceinline__ BlockScanRunningPrefixOp(
+        T starting_prefix,
+        ScanOpT op)
+    :
+        op(op),
+        running_total(starting_prefix)
+    {}
+
+    /**
+     * Prefix callback operator.  Returns the block-wide running_total in thread-0.
+     */
+    __device__ __forceinline__ T operator()(
+        const T &block_aggregate)              ///< The aggregate sum of the BlockScan inputs
+    {
+        T retval = running_total;
+        running_total = op(running_total, block_aggregate);
+        return retval;
+    }
+};
+
+
+/******************************************************************************
+ * Generic tile status interface types for block-cooperative scans
+ ******************************************************************************/
+
+/**
+ * Enumerations of tile status
+ */
+enum ScanTileStatus
+{
+    SCAN_TILE_OOB,          // Out-of-bounds (e.g., padding)
+    SCAN_TILE_INVALID = 99, // Not yet processed
+    SCAN_TILE_PARTIAL,      // Tile aggregate is available
+    SCAN_TILE_INCLUSIVE,    // Inclusive tile prefix is available
+};
+
+
+/**
+ * Tile status interface.
+ */
+template <
+    typename    T,
+    bool        SINGLE_WORD = Traits<T>::PRIMITIVE>
+struct ScanTileState;
+
+
+/**
+ * Tile status interface specialized for scan status and value types
+ * that can be combined into one machine word that can be
+ * read/written coherently in a single access.
+ */
+template <typename T>
+struct ScanTileState<T, true>
+{
+    // Status word type
+    typedef typename If<(sizeof(T) == 8),
+        long long,
+        typename If<(sizeof(T) == 4),
+            int,
+            typename If<(sizeof(T) == 2),
+                short,
+                char>::Type>::Type>::Type StatusWord;
+
+
+    // Unit word type
+    typedef typename If<(sizeof(T) == 8),
+        longlong2,
+        typename If<(sizeof(T) == 4),
+            int2,
+            typename If<(sizeof(T) == 2),
+                int,
+                uchar2>::Type>::Type>::Type TxnWord;
+
+
+    // Device word type
+    struct TileDescriptor
+    {
+        StatusWord  status;
+        T           value;
+    };
+
+
+    // Constants
+    enum
+    {
+        TILE_STATUS_PADDING = CUB_PTX_WARP_THREADS,
+    };
+
+
+    // Device storage
+    TxnWord *d_tile_descriptors;
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    ScanTileState()
+    :
+        d_tile_descriptors(NULL)
+    {}
+
+
+    /// Initializer
+    __host__ __device__ __forceinline__
+    cudaError_t Init(
+        int     /*num_tiles*/,                      ///< [in] Number of tiles
+        void    *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t  /*temp_storage_bytes*/)             ///< [in] Size in bytes of \t d_temp_storage allocation
+    {
+        d_tile_descriptors = reinterpret_cast<TxnWord*>(d_temp_storage);
+        return cudaSuccess;
+    }
+
+
+    /**
+     * Compute device memory needed for tile status
+     */
+    __host__ __device__ __forceinline__
+    static cudaError_t AllocationSize(
+        int     num_tiles,                          ///< [in] Number of tiles
+        size_t  &temp_storage_bytes)                ///< [out] Size in bytes of \t d_temp_storage allocation
+    {
+        temp_storage_bytes = (num_tiles + TILE_STATUS_PADDING) * sizeof(TileDescriptor);       // bytes needed for tile status descriptors
+        return cudaSuccess;
+    }
+
+
+    /**
+     * Initialize (from device)
+     */
+    __device__ __forceinline__ void InitializeStatus(int num_tiles)
+    {
+        int tile_idx = (blockIdx.x * blockDim.x) + threadIdx.x;
+
+        TxnWord val = TxnWord();
+        TileDescriptor *descriptor = reinterpret_cast<TileDescriptor*>(&val);
+
+        if (tile_idx < num_tiles)
+        {
+            // Not-yet-set
+            descriptor->status = StatusWord(SCAN_TILE_INVALID);
+            d_tile_descriptors[TILE_STATUS_PADDING + tile_idx] = val;
+        }
+
+        if ((blockIdx.x == 0) && (threadIdx.x < TILE_STATUS_PADDING))
+        {
+            // Padding
+            descriptor->status = StatusWord(SCAN_TILE_OOB);
+            d_tile_descriptors[threadIdx.x] = val;
+        }
+    }
+
+
+    /**
+     * Update the specified tile's inclusive value and corresponding status
+     */
+    __device__ __forceinline__ void SetInclusive(int tile_idx, T tile_inclusive)
+    {
+        TileDescriptor tile_descriptor;
+        tile_descriptor.status = SCAN_TILE_INCLUSIVE;
+        tile_descriptor.value = tile_inclusive;
+
+        TxnWord alias;
+        *reinterpret_cast<TileDescriptor*>(&alias) = tile_descriptor;
+        ThreadStore<STORE_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias);
+    }
+
+
+    /**
+     * Update the specified tile's partial value and corresponding status
+     */
+    __device__ __forceinline__ void SetPartial(int tile_idx, T tile_partial)
+    {
+        TileDescriptor tile_descriptor;
+        tile_descriptor.status = SCAN_TILE_PARTIAL;
+        tile_descriptor.value = tile_partial;
+
+        TxnWord alias;
+        *reinterpret_cast<TileDescriptor*>(&alias) = tile_descriptor;
+        ThreadStore<STORE_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias);
+    }
+
+    /**
+     * Wait for the corresponding tile to become non-invalid
+     */
+    __device__ __forceinline__ void WaitForValid(
+        int             tile_idx,
+        StatusWord      &status,
+        T               &value)
+    {
+        TileDescriptor tile_descriptor;
+        do
+        {
+            __threadfence_block(); // prevent hoisting loads from loop
+            TxnWord alias = ThreadLoad<LOAD_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx);
+            tile_descriptor = reinterpret_cast<TileDescriptor&>(alias);
+
+        } while (WARP_ANY((tile_descriptor.status == SCAN_TILE_INVALID), 0xffffffff));
+
+        status = tile_descriptor.status;
+        value = tile_descriptor.value;
+    }
+
+};
+
+
+
+/**
+ * Tile status interface specialized for scan status and value types that
+ * cannot be combined into one machine word.
+ */
+template <typename T>
+struct ScanTileState<T, false>
+{
+    // Status word type
+    typedef char StatusWord;
+
+    // Constants
+    enum
+    {
+        TILE_STATUS_PADDING = CUB_PTX_WARP_THREADS,
+    };
+
+    // Device storage
+    StatusWord  *d_tile_status;
+    T           *d_tile_partial;
+    T           *d_tile_inclusive;
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    ScanTileState()
+    :
+        d_tile_status(NULL),
+        d_tile_partial(NULL),
+        d_tile_inclusive(NULL)
+    {}
+
+
+    /// Initializer
+    __host__ __device__ __forceinline__
+    cudaError_t Init(
+        int     num_tiles,                          ///< [in] Number of tiles
+        void    *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t  temp_storage_bytes)                 ///< [in] Size in bytes of \t d_temp_storage allocation
+    {
+        cudaError_t error = cudaSuccess;
+        do
+        {
+            void*   allocations[3];
+            size_t  allocation_sizes[3];
+
+            allocation_sizes[0] = (num_tiles + TILE_STATUS_PADDING) * sizeof(StatusWord);           // bytes needed for tile status descriptors
+            allocation_sizes[1] = (num_tiles + TILE_STATUS_PADDING) * sizeof(Uninitialized<T>);     // bytes needed for partials
+            allocation_sizes[2] = (num_tiles + TILE_STATUS_PADDING) * sizeof(Uninitialized<T>);     // bytes needed for inclusives
+
+            // Compute allocation pointers into the single storage blob
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+
+            // Alias the offsets
+            d_tile_status       = reinterpret_cast<StatusWord*>(allocations[0]);
+            d_tile_partial      = reinterpret_cast<T*>(allocations[1]);
+            d_tile_inclusive    = reinterpret_cast<T*>(allocations[2]);
+        }
+        while (0);
+
+        return error;
+    }
+
+
+    /**
+     * Compute device memory needed for tile status
+     */
+    __host__ __device__ __forceinline__
+    static cudaError_t AllocationSize(
+        int     num_tiles,                          ///< [in] Number of tiles
+        size_t  &temp_storage_bytes)                ///< [out] Size in bytes of \t d_temp_storage allocation
+    {
+        // Specify storage allocation requirements
+        size_t  allocation_sizes[3];
+        allocation_sizes[0] = (num_tiles + TILE_STATUS_PADDING) * sizeof(StatusWord);         // bytes needed for tile status descriptors
+        allocation_sizes[1] = (num_tiles + TILE_STATUS_PADDING) * sizeof(Uninitialized<T>);   // bytes needed for partials
+        allocation_sizes[2] = (num_tiles + TILE_STATUS_PADDING) * sizeof(Uninitialized<T>);   // bytes needed for inclusives
+
+        // Set the necessary size of the blob
+        void* allocations[3];
+        return CubDebug(AliasTemporaries(NULL, temp_storage_bytes, allocations, allocation_sizes));
+    }
+
+
+    /**
+     * Initialize (from device)
+     */
+    __device__ __forceinline__ void InitializeStatus(int num_tiles)
+    {
+        int tile_idx = (blockIdx.x * blockDim.x) + threadIdx.x;
+        if (tile_idx < num_tiles)
+        {
+            // Not-yet-set
+            d_tile_status[TILE_STATUS_PADDING + tile_idx] = StatusWord(SCAN_TILE_INVALID);
+        }
+
+        if ((blockIdx.x == 0) && (threadIdx.x < TILE_STATUS_PADDING))
+        {
+            // Padding
+            d_tile_status[threadIdx.x] = StatusWord(SCAN_TILE_OOB);
+        }
+    }
+
+
+    /**
+     * Update the specified tile's inclusive value and corresponding status
+     */
+    __device__ __forceinline__ void SetInclusive(int tile_idx, T tile_inclusive)
+    {
+        // Update tile inclusive value
+        ThreadStore<STORE_CG>(d_tile_inclusive + TILE_STATUS_PADDING + tile_idx, tile_inclusive);
+
+        // Fence
+        __threadfence();
+
+        // Update tile status
+        ThreadStore<STORE_CG>(d_tile_status + TILE_STATUS_PADDING + tile_idx, StatusWord(SCAN_TILE_INCLUSIVE));
+    }
+
+
+    /**
+     * Update the specified tile's partial value and corresponding status
+     */
+    __device__ __forceinline__ void SetPartial(int tile_idx, T tile_partial)
+    {
+        // Update tile partial value
+        ThreadStore<STORE_CG>(d_tile_partial + TILE_STATUS_PADDING + tile_idx, tile_partial);
+
+        // Fence
+        __threadfence();
+
+        // Update tile status
+        ThreadStore<STORE_CG>(d_tile_status + TILE_STATUS_PADDING + tile_idx, StatusWord(SCAN_TILE_PARTIAL));
+    }
+
+    /**
+     * Wait for the corresponding tile to become non-invalid
+     */
+    __device__ __forceinline__ void WaitForValid(
+        int             tile_idx,
+        StatusWord      &status,
+        T               &value)
+    {
+        do {
+            status = ThreadLoad<LOAD_CG>(d_tile_status + TILE_STATUS_PADDING + tile_idx);
+
+            __threadfence();    // prevent hoisting loads from loop or loads below above this one
+
+        } while (status == SCAN_TILE_INVALID);
+
+        if (status == StatusWord(SCAN_TILE_PARTIAL)) 
+            value = ThreadLoad<LOAD_CG>(d_tile_partial + TILE_STATUS_PADDING + tile_idx);
+        else
+            value = ThreadLoad<LOAD_CG>(d_tile_inclusive + TILE_STATUS_PADDING + tile_idx);
+    }
+};
+
+
+/******************************************************************************
+ * ReduceByKey tile status interface types for block-cooperative scans
+ ******************************************************************************/
+
+/**
+ * Tile status interface for reduction by key.
+ *
+ */
+template <
+    typename    ValueT,
+    typename    KeyT,
+    bool        SINGLE_WORD = (Traits<ValueT>::PRIMITIVE) && (sizeof(ValueT) + sizeof(KeyT) < 16)>
+struct ReduceByKeyScanTileState;
+
+
+/**
+ * Tile status interface for reduction by key, specialized for scan status and value types that
+ * cannot be combined into one machine word.
+ */
+template <
+    typename    ValueT,
+    typename    KeyT>
+struct ReduceByKeyScanTileState<ValueT, KeyT, false> :
+    ScanTileState<KeyValuePair<KeyT, ValueT> >
+{
+    typedef ScanTileState<KeyValuePair<KeyT, ValueT> > SuperClass;
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    ReduceByKeyScanTileState() : SuperClass() {}
+};
+
+
+/**
+ * Tile status interface for reduction by key, specialized for scan status and value types that
+ * can be combined into one machine word that can be read/written coherently in a single access.
+ */
+template <
+    typename ValueT,
+    typename KeyT>
+struct ReduceByKeyScanTileState<ValueT, KeyT, true>
+{
+    typedef KeyValuePair<KeyT, ValueT>KeyValuePairT;
+
+    // Constants
+    enum
+    {
+        PAIR_SIZE           = sizeof(ValueT) + sizeof(KeyT),
+        TXN_WORD_SIZE       = 1 << Log2<PAIR_SIZE + 1>::VALUE,
+        STATUS_WORD_SIZE    = TXN_WORD_SIZE - PAIR_SIZE,
+
+        TILE_STATUS_PADDING = CUB_PTX_WARP_THREADS,
+    };
+
+    // Status word type
+    typedef typename If<(STATUS_WORD_SIZE == 8),
+        long long,
+        typename If<(STATUS_WORD_SIZE == 4),
+            int,
+            typename If<(STATUS_WORD_SIZE == 2),
+                short,
+                char>::Type>::Type>::Type StatusWord;
+
+    // Status word type
+    typedef typename If<(TXN_WORD_SIZE == 16),
+        longlong2,
+        typename If<(TXN_WORD_SIZE == 8),
+            long long,
+            int>::Type>::Type TxnWord;
+
+    // Device word type (for when sizeof(ValueT) == sizeof(KeyT))
+    struct TileDescriptorBigStatus
+    {
+        KeyT        key;
+        ValueT      value;
+        StatusWord  status;
+    };
+
+    // Device word type (for when sizeof(ValueT) != sizeof(KeyT))
+    struct TileDescriptorLittleStatus
+    {
+        ValueT      value;
+        StatusWord  status;
+        KeyT        key;
+    };
+
+    // Device word type
+    typedef typename If<
+            (sizeof(ValueT) == sizeof(KeyT)),
+            TileDescriptorBigStatus,
+            TileDescriptorLittleStatus>::Type
+        TileDescriptor;
+
+
+    // Device storage
+    TxnWord *d_tile_descriptors;
+
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    ReduceByKeyScanTileState()
+    :
+        d_tile_descriptors(NULL)
+    {}
+
+
+    /// Initializer
+    __host__ __device__ __forceinline__
+    cudaError_t Init(
+        int     /*num_tiles*/,                      ///< [in] Number of tiles
+        void    *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t  /*temp_storage_bytes*/)             ///< [in] Size in bytes of \t d_temp_storage allocation
+    {
+        d_tile_descriptors = reinterpret_cast<TxnWord*>(d_temp_storage);
+        return cudaSuccess;
+    }
+
+
+    /**
+     * Compute device memory needed for tile status
+     */
+    __host__ __device__ __forceinline__
+    static cudaError_t AllocationSize(
+        int     num_tiles,                          ///< [in] Number of tiles
+        size_t  &temp_storage_bytes)                ///< [out] Size in bytes of \t d_temp_storage allocation
+    {
+        temp_storage_bytes = (num_tiles + TILE_STATUS_PADDING) * sizeof(TileDescriptor);       // bytes needed for tile status descriptors
+        return cudaSuccess;
+    }
+
+
+    /**
+     * Initialize (from device)
+     */
+    __device__ __forceinline__ void InitializeStatus(int num_tiles)
+    {
+        int             tile_idx    = (blockIdx.x * blockDim.x) + threadIdx.x;
+        TxnWord         val         = TxnWord();
+        TileDescriptor  *descriptor = reinterpret_cast<TileDescriptor*>(&val);
+
+        if (tile_idx < num_tiles)
+        {
+            // Not-yet-set
+            descriptor->status = StatusWord(SCAN_TILE_INVALID);
+            d_tile_descriptors[TILE_STATUS_PADDING + tile_idx] = val;
+        }
+
+        if ((blockIdx.x == 0) && (threadIdx.x < TILE_STATUS_PADDING))
+        {
+            // Padding
+            descriptor->status = StatusWord(SCAN_TILE_OOB);
+            d_tile_descriptors[threadIdx.x] = val;
+        }
+    }
+
+
+    /**
+     * Update the specified tile's inclusive value and corresponding status
+     */
+    __device__ __forceinline__ void SetInclusive(int tile_idx, KeyValuePairT tile_inclusive)
+    {
+        TileDescriptor tile_descriptor;
+        tile_descriptor.status  = SCAN_TILE_INCLUSIVE;
+        tile_descriptor.value   = tile_inclusive.value;
+        tile_descriptor.key     = tile_inclusive.key;
+
+        TxnWord alias;
+        *reinterpret_cast<TileDescriptor*>(&alias) = tile_descriptor;
+        ThreadStore<STORE_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias);
+    }
+
+
+    /**
+     * Update the specified tile's partial value and corresponding status
+     */
+    __device__ __forceinline__ void SetPartial(int tile_idx, KeyValuePairT tile_partial)
+    {
+        TileDescriptor tile_descriptor;
+        tile_descriptor.status  = SCAN_TILE_PARTIAL;
+        tile_descriptor.value   = tile_partial.value;
+        tile_descriptor.key     = tile_partial.key;
+
+        TxnWord alias;
+        *reinterpret_cast<TileDescriptor*>(&alias) = tile_descriptor;
+        ThreadStore<STORE_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias);
+    }
+
+    /**
+     * Wait for the corresponding tile to become non-invalid
+     */
+    __device__ __forceinline__ void WaitForValid(
+        int                     tile_idx,
+        StatusWord              &status,
+        KeyValuePairT           &value)
+    {
+//        TxnWord         alias           = ThreadLoad<LOAD_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx);
+//        TileDescriptor  tile_descriptor = reinterpret_cast<TileDescriptor&>(alias);
+//
+//        while (tile_descriptor.status == SCAN_TILE_INVALID)
+//        {
+//            __threadfence_block(); // prevent hoisting loads from loop
+//
+//            alias           = ThreadLoad<LOAD_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx);
+//            tile_descriptor = reinterpret_cast<TileDescriptor&>(alias);
+//        }
+//
+//        status      = tile_descriptor.status;
+//        value.value = tile_descriptor.value;
+//        value.key   = tile_descriptor.key;
+
+        TileDescriptor tile_descriptor;
+        do
+        {
+            __threadfence_block(); // prevent hoisting loads from loop
+            TxnWord alias = ThreadLoad<LOAD_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx);
+            tile_descriptor = reinterpret_cast<TileDescriptor&>(alias);
+
+        } while (WARP_ANY((tile_descriptor.status == SCAN_TILE_INVALID), 0xffffffff));
+
+        status      = tile_descriptor.status;
+        value.value = tile_descriptor.value;
+        value.key   = tile_descriptor.key;
+    }
+
+};
+
+
+/******************************************************************************
+ * Prefix call-back operator for coupling local block scan within a
+ * block-cooperative scan
+ ******************************************************************************/
+
+/**
+ * Stateful block-scan prefix functor.  Provides the the running prefix for
+ * the current tile by using the call-back warp to wait on on
+ * aggregates/prefixes from predecessor tiles to become available.
+ */
+template <
+    typename    T,
+    typename    ScanOpT,
+    typename    ScanTileStateT,
+    int         PTX_ARCH = CUB_PTX_ARCH>
+struct TilePrefixCallbackOp
+{
+    // Parameterized warp reduce
+    typedef WarpReduce<T, CUB_PTX_WARP_THREADS, PTX_ARCH> WarpReduceT;
+
+    // Temporary storage type
+    struct _TempStorage
+    {
+        typename WarpReduceT::TempStorage   warp_reduce;
+        T                                   exclusive_prefix;
+        T                                   inclusive_prefix;
+        T                                   block_aggregate;
+    };
+
+    // Alias wrapper allowing temporary storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+    // Type of status word
+    typedef typename ScanTileStateT::StatusWord StatusWord;
+
+    // Fields
+    _TempStorage&               temp_storage;       ///< Reference to a warp-reduction instance
+    ScanTileStateT&             tile_status;        ///< Interface to tile status
+    ScanOpT                     scan_op;            ///< Binary scan operator
+    int                         tile_idx;           ///< The current tile index
+    T                           exclusive_prefix;   ///< Exclusive prefix for the tile
+    T                           inclusive_prefix;   ///< Inclusive prefix for the tile
+
+    // Constructor
+    __device__ __forceinline__
+    TilePrefixCallbackOp(
+        ScanTileStateT       &tile_status,
+        TempStorage         &temp_storage,
+        ScanOpT              scan_op,
+        int                 tile_idx)
+    :
+        temp_storage(temp_storage.Alias()),
+        tile_status(tile_status),
+        scan_op(scan_op),
+        tile_idx(tile_idx) {}
+
+
+    // Block until all predecessors within the warp-wide window have non-invalid status
+    __device__ __forceinline__
+    void ProcessWindow(
+        int         predecessor_idx,        ///< Preceding tile index to inspect
+        StatusWord  &predecessor_status,    ///< [out] Preceding tile status
+        T           &window_aggregate)      ///< [out] Relevant partial reduction from this window of preceding tiles
+    {
+        T value;
+        tile_status.WaitForValid(predecessor_idx, predecessor_status, value);
+
+        // Perform a segmented reduction to get the prefix for the current window.
+        // Use the swizzled scan operator because we are now scanning *down* towards thread0.
+
+        int tail_flag = (predecessor_status == StatusWord(SCAN_TILE_INCLUSIVE));
+        window_aggregate = WarpReduceT(temp_storage.warp_reduce).TailSegmentedReduce(
+            value,
+            tail_flag,
+            SwizzleScanOp<ScanOpT>(scan_op));
+    }
+
+
+    // BlockScan prefix callback functor (called by the first warp)
+    __device__ __forceinline__
+    T operator()(T block_aggregate)
+    {
+
+        // Update our status with our tile-aggregate
+        if (threadIdx.x == 0)
+        {
+            temp_storage.block_aggregate = block_aggregate;
+            tile_status.SetPartial(tile_idx, block_aggregate);
+        }
+
+        int         predecessor_idx = tile_idx - threadIdx.x - 1;
+        StatusWord  predecessor_status;
+        T           window_aggregate;
+
+        // Wait for the warp-wide window of predecessor tiles to become valid
+        ProcessWindow(predecessor_idx, predecessor_status, window_aggregate);
+
+        // The exclusive tile prefix starts out as the current window aggregate
+        exclusive_prefix = window_aggregate;
+
+        // Keep sliding the window back until we come across a tile whose inclusive prefix is known
+        while (WARP_ALL((predecessor_status != StatusWord(SCAN_TILE_INCLUSIVE)), 0xffffffff))
+        {
+            predecessor_idx -= CUB_PTX_WARP_THREADS;
+
+            // Update exclusive tile prefix with the window prefix
+            ProcessWindow(predecessor_idx, predecessor_status, window_aggregate);
+            exclusive_prefix = scan_op(window_aggregate, exclusive_prefix);
+        }
+
+        // Compute the inclusive tile prefix and update the status for this tile
+        if (threadIdx.x == 0)
+        {
+            inclusive_prefix = scan_op(exclusive_prefix, block_aggregate);
+            tile_status.SetInclusive(tile_idx, inclusive_prefix);
+
+            temp_storage.exclusive_prefix = exclusive_prefix;
+            temp_storage.inclusive_prefix = inclusive_prefix;
+        }
+
+        // Return exclusive_prefix
+        return exclusive_prefix;
+    }
+
+    // Get the exclusive prefix stored in temporary storage
+    __device__ __forceinline__
+    T GetExclusivePrefix()
+    {
+        return temp_storage.exclusive_prefix;
+    }
+
+    // Get the inclusive prefix stored in temporary storage
+    __device__ __forceinline__
+    T GetInclusivePrefix()
+    {
+        return temp_storage.inclusive_prefix;
+    }
+
+    // Get the block aggregate stored in temporary storage
+    __device__ __forceinline__
+    T GetBlockAggregate()
+    {
+        return temp_storage.block_aggregate;
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_adjacent_difference.cuh b/third_party/cub/cub/block/block_adjacent_difference.cuh
new file mode 100644
index 0000000..acef9f0
--- /dev/null
+++ b/third_party/cub/cub/block/block_adjacent_difference.cuh
@@ -0,0 +1,596 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockDiscontinuity class provides [<em>collective</em>](index.html#sec0) methods for flagging discontinuities within an ordered set of items partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../util_type.cuh"
+#include "../util_ptx.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+template <
+    typename    T,
+    int         BLOCK_DIM_X,
+    int         BLOCK_DIM_Y     = 1,
+    int         BLOCK_DIM_Z     = 1,
+    int         PTX_ARCH        = CUB_PTX_ARCH>
+class BlockAdjacentDifference
+{
+private:
+
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+
+    /// Shared memory storage layout type (last element from each thread's input)
+    struct _TempStorage
+    {
+        T first_items[BLOCK_THREADS];
+        T last_items[BLOCK_THREADS];
+    };
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+    /// Specialization for when FlagOp has third index param
+    template <typename FlagOp, bool HAS_PARAM = BinaryOpHasIdxParam<T, FlagOp>::HAS_PARAM>
+    struct ApplyOp
+    {
+        // Apply flag operator
+        static __device__ __forceinline__ T FlagT(FlagOp flag_op, const T &a, const T &b, int idx)
+        {
+            return flag_op(b, a, idx);
+        }
+    };
+
+    /// Specialization for when FlagOp does not have a third index param
+    template <typename FlagOp>
+    struct ApplyOp<FlagOp, false>
+    {
+        // Apply flag operator
+        static __device__ __forceinline__ T FlagT(FlagOp flag_op, const T &a, const T &b, int /*idx*/)
+        {
+            return flag_op(b, a);
+        }
+    };
+
+    /// Templated unrolling of item comparison (inductive case)
+    template <int ITERATION, int MAX_ITERATIONS>
+    struct Iterate
+    {
+        // Head flags
+        template <
+            int             ITEMS_PER_THREAD,
+            typename        FlagT,
+            typename        FlagOp>
+        static __device__ __forceinline__ void FlagHeads(
+            int                     linear_tid,
+            FlagT                   (&flags)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
+            T                       (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+            T                       (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
+            FlagOp                  flag_op)                            ///< [in] Binary boolean flag predicate
+        {
+            preds[ITERATION] = input[ITERATION - 1];
+
+            flags[ITERATION] = ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                preds[ITERATION],
+                input[ITERATION],
+                (linear_tid * ITEMS_PER_THREAD) + ITERATION);
+
+            Iterate<ITERATION + 1, MAX_ITERATIONS>::FlagHeads(linear_tid, flags, input, preds, flag_op);
+        }
+
+        // Tail flags
+        template <
+            int             ITEMS_PER_THREAD,
+            typename        FlagT,
+            typename        FlagOp>
+        static __device__ __forceinline__ void FlagTails(
+            int                     linear_tid,
+            FlagT                   (&flags)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
+            T                       (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+            FlagOp                  flag_op)                            ///< [in] Binary boolean flag predicate
+        {
+            flags[ITERATION] = ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                input[ITERATION],
+                input[ITERATION + 1],
+                (linear_tid * ITEMS_PER_THREAD) + ITERATION + 1);
+
+            Iterate<ITERATION + 1, MAX_ITERATIONS>::FlagTails(linear_tid, flags, input, flag_op);
+        }
+
+    };
+
+    /// Templated unrolling of item comparison (termination case)
+    template <int MAX_ITERATIONS>
+    struct Iterate<MAX_ITERATIONS, MAX_ITERATIONS>
+    {
+        // Head flags
+        template <
+            int             ITEMS_PER_THREAD,
+            typename        FlagT,
+            typename        FlagOp>
+        static __device__ __forceinline__ void FlagHeads(
+            int                     /*linear_tid*/,
+            FlagT                   (&/*flags*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
+            T                       (&/*input*/)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+            T                       (&/*preds*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
+            FlagOp                  /*flag_op*/)                            ///< [in] Binary boolean flag predicate
+        {}
+
+        // Tail flags
+        template <
+            int             ITEMS_PER_THREAD,
+            typename        FlagT,
+            typename        FlagOp>
+        static __device__ __forceinline__ void FlagTails(
+            int                     /*linear_tid*/,
+            FlagT                   (&/*flags*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
+            T                       (&/*input*/)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+            FlagOp                  /*flag_op*/)                            ///< [in] Binary boolean flag predicate
+        {}
+    };
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+
+public:
+
+    /// \smemstorage{BlockDiscontinuity}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockAdjacentDifference()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockAdjacentDifference(
+        TempStorage &temp_storage)  ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Head flag operations
+     *********************************************************************/
+    //@{
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeads(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        T               (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share last item
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        if (linear_tid == 0)
+        {
+            // Set flag for first thread-item (preds[0] is undefined)
+            head_flags[0] = 1;
+        }
+        else
+        {
+            preds[0] = temp_storage.last_items[linear_tid - 1];
+            head_flags[0] = ApplyOp<FlagOp>::FlagT(flag_op, preds[0], input[0], linear_tid * ITEMS_PER_THREAD);
+        }
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+    }
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeads(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        T               (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
+        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
+        T               tile_predecessor_item)              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
+    {
+        // Share last item
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        // Set flag for first thread-item
+        preds[0] = (linear_tid == 0) ?
+            tile_predecessor_item :              // First thread
+            temp_storage.last_items[linear_tid - 1];
+
+        head_flags[0] = ApplyOp<FlagOp>::FlagT(flag_op, preds[0], input[0], linear_tid * ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+    }
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeads(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        T preds[ITEMS_PER_THREAD];
+        FlagHeads(head_flags, input, preds, flag_op);
+    }
+
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeads(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
+        T               tile_predecessor_item)              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
+    {
+        T preds[ITEMS_PER_THREAD];
+        FlagHeads(head_flags, input, preds, flag_op, tile_predecessor_item);
+    }
+
+
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagTails(
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first item
+        temp_storage.first_items[linear_tid] = input[0];
+
+        CTA_SYNC();
+
+        // Set flag for last thread-item
+        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
+            1 :                             // Last thread
+            ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                input[ITEMS_PER_THREAD - 1],
+                temp_storage.first_items[linear_tid + 1],
+                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagTails(
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
+        T               tile_successor_item)                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
+    {
+        // Share first item
+        temp_storage.first_items[linear_tid] = input[0];
+
+        CTA_SYNC();
+
+        // Set flag for last thread-item
+        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
+            tile_successor_item :              // Last thread
+            temp_storage.first_items[linear_tid + 1];
+
+        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            input[ITEMS_PER_THREAD - 1],
+            successor_item,
+            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeadsAndTails(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first and last items
+        temp_storage.first_items[linear_tid] = input[0];
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        T preds[ITEMS_PER_THREAD];
+
+        // Set flag for first thread-item
+        preds[0] = temp_storage.last_items[linear_tid - 1];
+        if (linear_tid == 0)
+        {
+            head_flags[0] = 1;
+        }
+        else
+        {
+            head_flags[0] = ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                preds[0],
+                input[0],
+                linear_tid * ITEMS_PER_THREAD);
+        }
+
+
+        // Set flag for last thread-item
+        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
+            1 :                             // Last thread
+            ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                input[ITEMS_PER_THREAD - 1],
+                temp_storage.first_items[linear_tid + 1],
+                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeadsAndTails(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               tile_successor_item,                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first and last items
+        temp_storage.first_items[linear_tid] = input[0];
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        T preds[ITEMS_PER_THREAD];
+
+        // Set flag for first thread-item
+        if (linear_tid == 0)
+        {
+            head_flags[0] = 1;
+        }
+        else
+        {
+            preds[0] = temp_storage.last_items[linear_tid - 1];
+            head_flags[0] = ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                preds[0],
+                input[0],
+                linear_tid * ITEMS_PER_THREAD);
+        }
+
+        // Set flag for last thread-item
+        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
+            tile_successor_item :              // Last thread
+            temp_storage.first_items[linear_tid + 1];
+
+        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            input[ITEMS_PER_THREAD - 1],
+            successor_item,
+            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeadsAndTails(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               tile_predecessor_item,              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first and last items
+        temp_storage.first_items[linear_tid] = input[0];
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        T preds[ITEMS_PER_THREAD];
+
+        // Set flag for first thread-item
+        preds[0] = (linear_tid == 0) ?
+            tile_predecessor_item :              // First thread
+            temp_storage.last_items[linear_tid - 1];
+
+        head_flags[0] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            preds[0],
+            input[0],
+            linear_tid * ITEMS_PER_THREAD);
+
+        // Set flag for last thread-item
+        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
+            1 :                             // Last thread
+            ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                input[ITEMS_PER_THREAD - 1],
+                temp_storage.first_items[linear_tid + 1],
+                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeadsAndTails(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               tile_predecessor_item,              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               tile_successor_item,                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first and last items
+        temp_storage.first_items[linear_tid] = input[0];
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        T preds[ITEMS_PER_THREAD];
+
+        // Set flag for first thread-item
+        preds[0] = (linear_tid == 0) ?
+            tile_predecessor_item :              // First thread
+            temp_storage.last_items[linear_tid - 1];
+
+        head_flags[0] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            preds[0],
+            input[0],
+            linear_tid * ITEMS_PER_THREAD);
+
+        // Set flag for last thread-item
+        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
+            tile_successor_item :              // Last thread
+            temp_storage.first_items[linear_tid + 1];
+
+        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            input[ITEMS_PER_THREAD - 1],
+            successor_item,
+            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/block/block_discontinuity.cuh b/third_party/cub/cub/block/block_discontinuity.cuh
new file mode 100644
index 0000000..503e3e0
--- /dev/null
+++ b/third_party/cub/cub/block/block_discontinuity.cuh
@@ -0,0 +1,1148 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockDiscontinuity class provides [<em>collective</em>](index.html#sec0) methods for flagging discontinuities within an ordered set of items partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../util_type.cuh"
+#include "../util_ptx.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief The BlockDiscontinuity class provides [<em>collective</em>](index.html#sec0) methods for flagging discontinuities within an ordered set of items partitioned across a CUDA thread block. ![](discont_logo.png)
+ * \ingroup BlockModule
+ *
+ * \tparam T                The data type to be flagged.
+ * \tparam BLOCK_DIM_X      The thread block length in threads along the X dimension
+ * \tparam BLOCK_DIM_Y      <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z      <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH         <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - A set of "head flags" (or "tail flags") is often used to indicate corresponding items
+ *   that differ from their predecessors (or successors).  For example, head flags are convenient
+ *   for demarcating disjoint data segments as part of a segmented scan or reduction.
+ * - \blocked
+ *
+ * \par Performance Considerations
+ * - \granularity
+ *
+ * \par A Simple Example
+ * \blockcollective{BlockDiscontinuity}
+ * \par
+ * The code snippet below illustrates the head flagging of 512 integer items that
+ * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+ * where each thread owns 4 consecutive items.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+ *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+ *
+ *     // Allocate shared memory for BlockDiscontinuity
+ *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+ *
+ *     // Obtain a segment of consecutive items that are blocked across threads
+ *     int thread_data[4];
+ *     ...
+ *
+ *     // Collectively compute head flags for discontinuities in the segment
+ *     int head_flags[4];
+ *     BlockDiscontinuity(temp_storage).FlagHeads(head_flags, thread_data, cub::Inequality());
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the block of threads is
+ * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }</tt>.
+ * The corresponding output \p head_flags in those threads will be
+ * <tt>{ [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
+ *
+ * \par Performance Considerations
+ * - Incurs zero bank conflicts for most types
+ *
+ */
+template <
+    typename    T,
+    int         BLOCK_DIM_X,
+    int         BLOCK_DIM_Y     = 1,
+    int         BLOCK_DIM_Z     = 1,
+    int         PTX_ARCH        = CUB_PTX_ARCH>
+class BlockDiscontinuity
+{
+private:
+
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+
+    /// Shared memory storage layout type (last element from each thread's input)
+    struct _TempStorage
+    {
+        T first_items[BLOCK_THREADS];
+        T last_items[BLOCK_THREADS];
+    };
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+    /// Specialization for when FlagOp has third index param
+    template <typename FlagOp, bool HAS_PARAM = BinaryOpHasIdxParam<T, FlagOp>::HAS_PARAM>
+    struct ApplyOp
+    {
+        // Apply flag operator
+        static __device__ __forceinline__ bool FlagT(FlagOp flag_op, const T &a, const T &b, int idx)
+        {
+            return flag_op(a, b, idx);
+        }
+    };
+
+    /// Specialization for when FlagOp does not have a third index param
+    template <typename FlagOp>
+    struct ApplyOp<FlagOp, false>
+    {
+        // Apply flag operator
+        static __device__ __forceinline__ bool FlagT(FlagOp flag_op, const T &a, const T &b, int /*idx*/)
+        {
+            return flag_op(a, b);
+        }
+    };
+
+    /// Templated unrolling of item comparison (inductive case)
+    template <int ITERATION, int MAX_ITERATIONS>
+    struct Iterate
+    {
+        // Head flags
+        template <
+            int             ITEMS_PER_THREAD,
+            typename        FlagT,
+            typename        FlagOp>
+        static __device__ __forceinline__ void FlagHeads(
+            int                     linear_tid,
+            FlagT                   (&flags)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
+            T                       (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+            T                       (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
+            FlagOp                  flag_op)                            ///< [in] Binary boolean flag predicate
+        {
+            preds[ITERATION] = input[ITERATION - 1];
+
+            flags[ITERATION] = ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                preds[ITERATION],
+                input[ITERATION],
+                (linear_tid * ITEMS_PER_THREAD) + ITERATION);
+
+            Iterate<ITERATION + 1, MAX_ITERATIONS>::FlagHeads(linear_tid, flags, input, preds, flag_op);
+        }
+
+        // Tail flags
+        template <
+            int             ITEMS_PER_THREAD,
+            typename        FlagT,
+            typename        FlagOp>
+        static __device__ __forceinline__ void FlagTails(
+            int                     linear_tid,
+            FlagT                   (&flags)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
+            T                       (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+            FlagOp                  flag_op)                            ///< [in] Binary boolean flag predicate
+        {
+            flags[ITERATION] = ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                input[ITERATION],
+                input[ITERATION + 1],
+                (linear_tid * ITEMS_PER_THREAD) + ITERATION + 1);
+
+            Iterate<ITERATION + 1, MAX_ITERATIONS>::FlagTails(linear_tid, flags, input, flag_op);
+        }
+
+    };
+
+    /// Templated unrolling of item comparison (termination case)
+    template <int MAX_ITERATIONS>
+    struct Iterate<MAX_ITERATIONS, MAX_ITERATIONS>
+    {
+        // Head flags
+        template <
+            int             ITEMS_PER_THREAD,
+            typename        FlagT,
+            typename        FlagOp>
+        static __device__ __forceinline__ void FlagHeads(
+            int                     /*linear_tid*/,
+            FlagT                   (&/*flags*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
+            T                       (&/*input*/)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+            T                       (&/*preds*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
+            FlagOp                  /*flag_op*/)                            ///< [in] Binary boolean flag predicate
+        {}
+
+        // Tail flags
+        template <
+            int             ITEMS_PER_THREAD,
+            typename        FlagT,
+            typename        FlagOp>
+        static __device__ __forceinline__ void FlagTails(
+            int                     /*linear_tid*/,
+            FlagT                   (&/*flags*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
+            T                       (&/*input*/)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+            FlagOp                  /*flag_op*/)                            ///< [in] Binary boolean flag predicate
+        {}
+    };
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+
+public:
+
+    /// \smemstorage{BlockDiscontinuity}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockDiscontinuity()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockDiscontinuity(
+        TempStorage &temp_storage)  ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Head flag operations
+     *********************************************************************/
+    //@{
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeads(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        T               (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share last item
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        if (linear_tid == 0)
+        {
+            // Set flag for first thread-item (preds[0] is undefined)
+            head_flags[0] = 1;
+        }
+        else
+        {
+            preds[0] = temp_storage.last_items[linear_tid - 1];
+            head_flags[0] = ApplyOp<FlagOp>::FlagT(flag_op, preds[0], input[0], linear_tid * ITEMS_PER_THREAD);
+        }
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+    }
+
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeads(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        T               (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
+        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
+        T               tile_predecessor_item)              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
+    {
+        // Share last item
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        // Set flag for first thread-item
+        preds[0] = (linear_tid == 0) ?
+            tile_predecessor_item :              // First thread
+            temp_storage.last_items[linear_tid - 1];
+
+        head_flags[0] = ApplyOp<FlagOp>::FlagT(flag_op, preds[0], input[0], linear_tid * ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+    }
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+    /**
+     * \brief Sets head flags indicating discontinuities between items partitioned across the thread block, for which the first item has no reference and is always flagged.
+     *
+     * \par
+     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
+     *   returns \p true (where <em>previous-item</em> is either the preceding item
+     *   in the same thread or the last item in the previous thread).
+     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is always flagged.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the head-flagging of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+     *
+     *     // Allocate shared memory for BlockDiscontinuity
+     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute head flags for discontinuities in the segment
+     *     int head_flags[4];
+     *     BlockDiscontinuity(temp_storage).FlagHeads(head_flags, thread_data, cub::Inequality());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }</tt>.
+     * The corresponding output \p head_flags in those threads will be
+     * <tt>{ [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
+     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeads(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        T preds[ITEMS_PER_THREAD];
+        FlagHeads(head_flags, input, preds, flag_op);
+    }
+
+
+    /**
+     * \brief Sets head flags indicating discontinuities between items partitioned across the thread block.
+     *
+     * \par
+     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
+     *   returns \p true (where <em>previous-item</em> is either the preceding item
+     *   in the same thread or the last item in the previous thread).
+     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is compared
+     *   against \p tile_predecessor_item.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the head-flagging of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+     *
+     *     // Allocate shared memory for BlockDiscontinuity
+     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Have thread0 obtain the predecessor item for the entire tile
+     *     int tile_predecessor_item;
+     *     if (threadIdx.x == 0) tile_predecessor_item == ...
+     *
+     *     // Collectively compute head flags for discontinuities in the segment
+     *     int head_flags[4];
+     *     BlockDiscontinuity(temp_storage).FlagHeads(
+     *         head_flags, thread_data, cub::Inequality(), tile_predecessor_item);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }</tt>,
+     * and that \p tile_predecessor_item is \p 0.  The corresponding output \p head_flags in those threads will be
+     * <tt>{ [0,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
+     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeads(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
+        T               tile_predecessor_item)              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
+    {
+        T preds[ITEMS_PER_THREAD];
+        FlagHeads(head_flags, input, preds, flag_op, tile_predecessor_item);
+    }
+
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Tail flag operations
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Sets tail flags indicating discontinuities between items partitioned across the thread block, for which the last item has no reference and is always flagged.
+     *
+     * \par
+     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
+     *   returns \p true (where <em>next-item</em> is either the next item
+     *   in the same thread or the first item in the next thread).
+     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
+     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is always flagged.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the tail-flagging of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+     *
+     *     // Allocate shared memory for BlockDiscontinuity
+     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute tail flags for discontinuities in the segment
+     *     int tail_flags[4];
+     *     BlockDiscontinuity(temp_storage).FlagTails(tail_flags, thread_data, cub::Inequality());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>.
+     * The corresponding output \p tail_flags in those threads will be
+     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,1] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
+     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagTails(
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first item
+        temp_storage.first_items[linear_tid] = input[0];
+
+        CTA_SYNC();
+
+        // Set flag for last thread-item
+        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
+            1 :                             // Last thread
+            ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                input[ITEMS_PER_THREAD - 1],
+                temp_storage.first_items[linear_tid + 1],
+                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    /**
+     * \brief Sets tail flags indicating discontinuities between items partitioned across the thread block.
+     *
+     * \par
+     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
+     *   returns \p true (where <em>next-item</em> is either the next item
+     *   in the same thread or the first item in the next thread).
+     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
+     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is compared
+     *   against \p tile_successor_item.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the tail-flagging of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+     *
+     *     // Allocate shared memory for BlockDiscontinuity
+     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Have thread127 obtain the successor item for the entire tile
+     *     int tile_successor_item;
+     *     if (threadIdx.x == 127) tile_successor_item == ...
+     *
+     *     // Collectively compute tail flags for discontinuities in the segment
+     *     int tail_flags[4];
+     *     BlockDiscontinuity(temp_storage).FlagTails(
+     *         tail_flags, thread_data, cub::Inequality(), tile_successor_item);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>
+     * and that \p tile_successor_item is \p 125.  The corresponding output \p tail_flags in those threads will be
+     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,0] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
+     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagTails(
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
+        T               tile_successor_item)                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
+    {
+        // Share first item
+        temp_storage.first_items[linear_tid] = input[0];
+
+        CTA_SYNC();
+
+        // Set flag for last thread-item
+        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
+            tile_successor_item :              // Last thread
+            temp_storage.first_items[linear_tid + 1];
+
+        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            input[ITEMS_PER_THREAD - 1],
+            successor_item,
+            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Head & tail flag operations
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Sets both head and tail flags indicating discontinuities between items partitioned across the thread block.
+     *
+     * \par
+     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
+     *   returns \p true (where <em>previous-item</em> is either the preceding item
+     *   in the same thread or the last item in the previous thread).
+     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is always flagged.
+     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
+     *   returns \p true (where <em>next-item</em> is either the next item
+     *   in the same thread or the first item in the next thread).
+     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
+     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is always flagged.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the head- and tail-flagging of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+     *
+     *     // Allocate shared memory for BlockDiscontinuity
+     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute head and flags for discontinuities in the segment
+     *     int head_flags[4];
+     *     int tail_flags[4];
+     *     BlockDiscontinuity(temp_storage).FlagTails(
+     *         head_flags, tail_flags, thread_data, cub::Inequality());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>
+     * and that the tile_successor_item is \p 125.  The corresponding output \p head_flags
+     * in those threads will be <tt>{ [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
+     * and the corresponding output \p tail_flags in those threads will be
+     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,1] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
+     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeadsAndTails(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first and last items
+        temp_storage.first_items[linear_tid] = input[0];
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        T preds[ITEMS_PER_THREAD];
+
+        // Set flag for first thread-item
+        preds[0] = temp_storage.last_items[linear_tid - 1];
+        if (linear_tid == 0)
+        {
+            head_flags[0] = 1;
+        }
+        else
+        {
+            head_flags[0] = ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                preds[0],
+                input[0],
+                linear_tid * ITEMS_PER_THREAD);
+        }
+
+
+        // Set flag for last thread-item
+        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
+            1 :                             // Last thread
+            ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                input[ITEMS_PER_THREAD - 1],
+                temp_storage.first_items[linear_tid + 1],
+                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    /**
+     * \brief Sets both head and tail flags indicating discontinuities between items partitioned across the thread block.
+     *
+     * \par
+     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
+     *   returns \p true (where <em>previous-item</em> is either the preceding item
+     *   in the same thread or the last item in the previous thread).
+     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is always flagged.
+     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
+     *   returns \p true (where <em>next-item</em> is either the next item
+     *   in the same thread or the first item in the next thread).
+     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
+     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is compared
+     *   against \p tile_predecessor_item.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the head- and tail-flagging of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+     *
+     *     // Allocate shared memory for BlockDiscontinuity
+     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Have thread127 obtain the successor item for the entire tile
+     *     int tile_successor_item;
+     *     if (threadIdx.x == 127) tile_successor_item == ...
+     *
+     *     // Collectively compute head and flags for discontinuities in the segment
+     *     int head_flags[4];
+     *     int tail_flags[4];
+     *     BlockDiscontinuity(temp_storage).FlagTails(
+     *         head_flags, tail_flags, tile_successor_item, thread_data, cub::Inequality());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>
+     * and that the tile_successor_item is \p 125.  The corresponding output \p head_flags
+     * in those threads will be <tt>{ [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
+     * and the corresponding output \p tail_flags in those threads will be
+     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,0] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
+     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeadsAndTails(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               tile_successor_item,                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first and last items
+        temp_storage.first_items[linear_tid] = input[0];
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        T preds[ITEMS_PER_THREAD];
+
+        // Set flag for first thread-item
+        if (linear_tid == 0)
+        {
+            head_flags[0] = 1;
+        }
+        else
+        {
+            preds[0] = temp_storage.last_items[linear_tid - 1];
+            head_flags[0] = ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                preds[0],
+                input[0],
+                linear_tid * ITEMS_PER_THREAD);
+        }
+
+        // Set flag for last thread-item
+        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
+            tile_successor_item :              // Last thread
+            temp_storage.first_items[linear_tid + 1];
+
+        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            input[ITEMS_PER_THREAD - 1],
+            successor_item,
+            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    /**
+     * \brief Sets both head and tail flags indicating discontinuities between items partitioned across the thread block.
+     *
+     * \par
+     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
+     *   returns \p true (where <em>previous-item</em> is either the preceding item
+     *   in the same thread or the last item in the previous thread).
+     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is compared
+     *   against \p tile_predecessor_item.
+     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
+     *   returns \p true (where <em>next-item</em> is either the next item
+     *   in the same thread or the first item in the next thread).
+     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
+     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is always flagged.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the head- and tail-flagging of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+     *
+     *     // Allocate shared memory for BlockDiscontinuity
+     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Have thread0 obtain the predecessor item for the entire tile
+     *     int tile_predecessor_item;
+     *     if (threadIdx.x == 0) tile_predecessor_item == ...
+     *
+     *     // Have thread127 obtain the successor item for the entire tile
+     *     int tile_successor_item;
+     *     if (threadIdx.x == 127) tile_successor_item == ...
+     *
+     *     // Collectively compute head and flags for discontinuities in the segment
+     *     int head_flags[4];
+     *     int tail_flags[4];
+     *     BlockDiscontinuity(temp_storage).FlagTails(
+     *         head_flags, tile_predecessor_item, tail_flags, tile_successor_item,
+     *         thread_data, cub::Inequality());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>,
+     * that the \p tile_predecessor_item is \p 0, and that the
+     * \p tile_successor_item is \p 125.  The corresponding output \p head_flags
+     * in those threads will be <tt>{ [0,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
+     * and the corresponding output \p tail_flags in those threads will be
+     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,1] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
+     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeadsAndTails(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               tile_predecessor_item,              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first and last items
+        temp_storage.first_items[linear_tid] = input[0];
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        T preds[ITEMS_PER_THREAD];
+
+        // Set flag for first thread-item
+        preds[0] = (linear_tid == 0) ?
+            tile_predecessor_item :              // First thread
+            temp_storage.last_items[linear_tid - 1];
+
+        head_flags[0] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            preds[0],
+            input[0],
+            linear_tid * ITEMS_PER_THREAD);
+
+        // Set flag for last thread-item
+        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
+            1 :                             // Last thread
+            ApplyOp<FlagOp>::FlagT(
+                flag_op,
+                input[ITEMS_PER_THREAD - 1],
+                temp_storage.first_items[linear_tid + 1],
+                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+    /**
+     * \brief Sets both head and tail flags indicating discontinuities between items partitioned across the thread block.
+     *
+     * \par
+     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
+     *   returns \p true (where <em>previous-item</em> is either the preceding item
+     *   in the same thread or the last item in the previous thread).
+     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is compared
+     *   against \p tile_predecessor_item.
+     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
+     *   <tt>input<sub><em>i</em></sub></tt> when
+     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
+     *   returns \p true (where <em>next-item</em> is either the next item
+     *   in the same thread or the first item in the next thread).
+     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
+     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is compared
+     *   against \p tile_successor_item.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the head- and tail-flagging of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
+     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
+     *
+     *     // Allocate shared memory for BlockDiscontinuity
+     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Have thread0 obtain the predecessor item for the entire tile
+     *     int tile_predecessor_item;
+     *     if (threadIdx.x == 0) tile_predecessor_item == ...
+     *
+     *     // Have thread127 obtain the successor item for the entire tile
+     *     int tile_successor_item;
+     *     if (threadIdx.x == 127) tile_successor_item == ...
+     *
+     *     // Collectively compute head and flags for discontinuities in the segment
+     *     int head_flags[4];
+     *     int tail_flags[4];
+     *     BlockDiscontinuity(temp_storage).FlagTails(
+     *         head_flags, tile_predecessor_item, tail_flags, tile_successor_item,
+     *         thread_data, cub::Inequality());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>,
+     * that the \p tile_predecessor_item is \p 0, and that the
+     * \p tile_successor_item is \p 125.  The corresponding output \p head_flags
+     * in those threads will be <tt>{ [0,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
+     * and the corresponding output \p tail_flags in those threads will be
+     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,0] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
+     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        FlagT,
+        typename        FlagOp>
+    __device__ __forceinline__ void FlagHeadsAndTails(
+        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
+        T               tile_predecessor_item,              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
+        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
+        T               tile_successor_item,                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
+        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
+        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
+    {
+        // Share first and last items
+        temp_storage.first_items[linear_tid] = input[0];
+        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        T preds[ITEMS_PER_THREAD];
+
+        // Set flag for first thread-item
+        preds[0] = (linear_tid == 0) ?
+            tile_predecessor_item :              // First thread
+            temp_storage.last_items[linear_tid - 1];
+
+        head_flags[0] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            preds[0],
+            input[0],
+            linear_tid * ITEMS_PER_THREAD);
+
+        // Set flag for last thread-item
+        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
+            tile_successor_item :              // Last thread
+            temp_storage.first_items[linear_tid + 1];
+
+        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
+            flag_op,
+            input[ITEMS_PER_THREAD - 1],
+            successor_item,
+            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
+
+        // Set head_flags for remaining items
+        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
+
+        // Set tail_flags for remaining items
+        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
+    }
+
+
+
+
+    //@}  end member group
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/block/block_exchange.cuh b/third_party/cub/cub/block/block_exchange.cuh
new file mode 100644
index 0000000..3ae9934
--- /dev/null
+++ b/third_party/cub/cub/block/block_exchange.cuh
@@ -0,0 +1,1248 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockExchange class provides [<em>collective</em>](index.html#sec0) methods for rearranging data partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../util_ptx.cuh"
+#include "../util_arch.cuh"
+#include "../util_macro.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief The BlockExchange class provides [<em>collective</em>](index.html#sec0) methods for rearranging data partitioned across a CUDA thread block. ![](transpose_logo.png)
+ * \ingroup BlockModule
+ *
+ * \tparam T                    The data type to be exchanged.
+ * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
+ * \tparam ITEMS_PER_THREAD     The number of items partitioned onto each thread.
+ * \tparam WARP_TIME_SLICING    <b>[optional]</b> When \p true, only use enough shared memory for a single warp's worth of tile data, time-slicing the block-wide exchange over multiple synchronized rounds.  Yields a smaller memory footprint at the expense of decreased parallelism.  (Default: false)
+ * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - It is commonplace for blocks of threads to rearrange data items between
+ *   threads.  For example, the device-accessible memory subsystem prefers access patterns
+ *   where data items are "striped" across threads (where consecutive threads access consecutive items),
+ *   yet most block-wide operations prefer a "blocked" partitioning of items across threads
+ *   (where consecutive items belong to a single thread).
+ * - BlockExchange supports the following types of data exchanges:
+ *   - Transposing between [<em>blocked</em>](index.html#sec5sec3) and [<em>striped</em>](index.html#sec5sec3) arrangements
+ *   - Transposing between [<em>blocked</em>](index.html#sec5sec3) and [<em>warp-striped</em>](index.html#sec5sec3) arrangements
+ *   - Scattering ranked items to a [<em>blocked arrangement</em>](index.html#sec5sec3)
+ *   - Scattering ranked items to a [<em>striped arrangement</em>](index.html#sec5sec3)
+ * - \rowmajor
+ *
+ * \par A Simple Example
+ * \blockcollective{BlockExchange}
+ * \par
+ * The code snippet below illustrates the conversion from a "blocked" to a "striped" arrangement
+ * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
+ *
+ * __global__ void ExampleKernel(int *d_data, ...)
+ * {
+ *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
+ *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
+ *
+ *     // Allocate shared memory for BlockExchange
+ *     __shared__ typename BlockExchange::TempStorage temp_storage;
+ *
+ *     // Load a tile of data striped across threads
+ *     int thread_data[4];
+ *     cub::LoadDirectStriped<128>(threadIdx.x, d_data, thread_data);
+ *
+ *     // Collectively exchange data into a blocked arrangement across threads
+ *     BlockExchange(temp_storage).StripedToBlocked(thread_data);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of striped input \p thread_data across the block of threads is
+ * <tt>{ [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] }</tt>.
+ * The corresponding output \p thread_data in those threads will be
+ * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
+ *
+ * \par Performance Considerations
+ * - Proper device-specific padding ensures zero bank conflicts for most types.
+ *
+ */
+template <
+    typename    InputT,
+    int         BLOCK_DIM_X,
+    int         ITEMS_PER_THREAD,
+    bool        WARP_TIME_SLICING   = false,
+    int         BLOCK_DIM_Y         = 1,
+    int         BLOCK_DIM_Z         = 1,
+    int         PTX_ARCH            = CUB_PTX_ARCH>
+class BlockExchange
+{
+private:
+
+    /******************************************************************************
+     * Constants
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        LOG_WARP_THREADS            = CUB_LOG_WARP_THREADS(PTX_ARCH),
+        WARP_THREADS                = 1 << LOG_WARP_THREADS,
+        WARPS                       = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+
+        LOG_SMEM_BANKS              = CUB_LOG_SMEM_BANKS(PTX_ARCH),
+        SMEM_BANKS                  = 1 << LOG_SMEM_BANKS,
+
+        TILE_ITEMS                  = BLOCK_THREADS * ITEMS_PER_THREAD,
+
+        TIME_SLICES                 = (WARP_TIME_SLICING) ? WARPS : 1,
+
+        TIME_SLICED_THREADS         = (WARP_TIME_SLICING) ? CUB_MIN(BLOCK_THREADS, WARP_THREADS) : BLOCK_THREADS,
+        TIME_SLICED_ITEMS           = TIME_SLICED_THREADS * ITEMS_PER_THREAD,
+
+        WARP_TIME_SLICED_THREADS    = CUB_MIN(BLOCK_THREADS, WARP_THREADS),
+        WARP_TIME_SLICED_ITEMS      = WARP_TIME_SLICED_THREADS * ITEMS_PER_THREAD,
+
+        // Insert padding to avoid bank conflicts during raking when items per thread is a power of two and > 4 (otherwise we can typically use 128b loads)
+        INSERT_PADDING              = (ITEMS_PER_THREAD > 4) && (PowerOfTwo<ITEMS_PER_THREAD>::VALUE),
+        PADDING_ITEMS               = (INSERT_PADDING) ? (TIME_SLICED_ITEMS >> LOG_SMEM_BANKS) : 0,
+    };
+
+    /******************************************************************************
+     * Type definitions
+     ******************************************************************************/
+
+    /// Shared memory storage layout type
+    struct __align__(16) _TempStorage
+    {
+        InputT buff[TIME_SLICED_ITEMS + PADDING_ITEMS];
+    };
+
+public:
+
+    /// \smemstorage{BlockExchange}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+private:
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+    unsigned int lane_id;
+    unsigned int warp_id;
+    unsigned int warp_offset;
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+    /**
+     * Transposes data items from <em>blocked</em> arrangement to <em>striped</em> arrangement.  Specialized for no timeslicing.
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void BlockedToStriped(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        Int2Type<false> /*time_slicing*/)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM;
+            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+            temp_storage.buff[item_offset] = input_items[ITEM];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
+            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+            output_items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+
+    /**
+     * Transposes data items from <em>blocked</em> arrangement to <em>striped</em> arrangement.  Specialized for warp-timeslicing.
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void BlockedToStriped(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        Int2Type<true>  /*time_slicing*/)
+    {
+        InputT temp_items[ITEMS_PER_THREAD];
+
+        #pragma unroll
+        for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++)
+        {
+            const int SLICE_OFFSET  = SLICE * TIME_SLICED_ITEMS;
+            const int SLICE_OOB     = SLICE_OFFSET + TIME_SLICED_ITEMS;
+
+            CTA_SYNC();
+
+            if (warp_id == SLICE)
+            {
+                #pragma unroll
+                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+                {
+                    int item_offset = (lane_id * ITEMS_PER_THREAD) + ITEM;
+                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                    temp_storage.buff[item_offset] = input_items[ITEM];
+                }
+            }
+
+            CTA_SYNC();
+
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                // Read a strip of items
+                const int STRIP_OFFSET  = ITEM * BLOCK_THREADS;
+                const int STRIP_OOB     = STRIP_OFFSET + BLOCK_THREADS;
+
+                if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET))
+                {
+                    int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET;
+                    if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS))
+                    {
+                        if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                        temp_items[ITEM] = temp_storage.buff[item_offset];
+                    }
+                }
+            }
+        }
+
+        // Copy
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            output_items[ITEM] = temp_items[ITEM];
+        }
+    }
+
+
+    /**
+     * Transposes data items from <em>blocked</em> arrangement to <em>warp-striped</em> arrangement. Specialized for no timeslicing
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void BlockedToWarpStriped(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        Int2Type<false> /*time_slicing*/)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = warp_offset + ITEM + (lane_id * ITEMS_PER_THREAD);
+            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+            temp_storage.buff[item_offset] = input_items[ITEM];
+        }
+
+        WARP_SYNC(0xffffffff);
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = warp_offset + (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
+            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+            output_items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+    /**
+     * Transposes data items from <em>blocked</em> arrangement to <em>warp-striped</em> arrangement. Specialized for warp-timeslicing
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void BlockedToWarpStriped(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        Int2Type<true>  /*time_slicing*/)
+    {
+        if (warp_id == 0)
+        {
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                int item_offset = ITEM + (lane_id * ITEMS_PER_THREAD);
+                if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                temp_storage.buff[item_offset] = input_items[ITEM];
+            }
+
+            WARP_SYNC(0xffffffff);
+
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                int item_offset = (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
+                if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                output_items[ITEM] = temp_storage.buff[item_offset];
+            }
+        }
+
+        #pragma unroll
+        for (unsigned int SLICE = 1; SLICE < TIME_SLICES; ++SLICE)
+        {
+            CTA_SYNC();
+
+            if (warp_id == SLICE)
+            {
+                #pragma unroll
+                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+                {
+                    int item_offset = ITEM + (lane_id * ITEMS_PER_THREAD);
+                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                    temp_storage.buff[item_offset] = input_items[ITEM];
+                }
+
+                WARP_SYNC(0xffffffff);
+
+                #pragma unroll
+                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+                {
+                    int item_offset = (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
+                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                    output_items[ITEM] = temp_storage.buff[item_offset];
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Transposes data items from <em>striped</em> arrangement to <em>blocked</em> arrangement.  Specialized for no timeslicing.
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void StripedToBlocked(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        Int2Type<false> /*time_slicing*/)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
+            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+            temp_storage.buff[item_offset] = input_items[ITEM];
+        }
+
+        CTA_SYNC();
+
+        // No timeslicing
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM;
+            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+            output_items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+
+    /**
+     * Transposes data items from <em>striped</em> arrangement to <em>blocked</em> arrangement.  Specialized for warp-timeslicing.
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void StripedToBlocked(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        Int2Type<true>  /*time_slicing*/)
+    {
+        // Warp time-slicing
+        InputT temp_items[ITEMS_PER_THREAD];
+
+        #pragma unroll
+        for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++)
+        {
+            const int SLICE_OFFSET  = SLICE * TIME_SLICED_ITEMS;
+            const int SLICE_OOB     = SLICE_OFFSET + TIME_SLICED_ITEMS;
+
+            CTA_SYNC();
+
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                // Write a strip of items
+                const int STRIP_OFFSET  = ITEM * BLOCK_THREADS;
+                const int STRIP_OOB     = STRIP_OFFSET + BLOCK_THREADS;
+
+                if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET))
+                {
+                    int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET;
+                    if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS))
+                    {
+                        if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                        temp_storage.buff[item_offset] = input_items[ITEM];
+                    }
+                }
+            }
+
+            CTA_SYNC();
+
+            if (warp_id == SLICE)
+            {
+                #pragma unroll
+                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+                {
+                    int item_offset = (lane_id * ITEMS_PER_THREAD) + ITEM;
+                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                    temp_items[ITEM] = temp_storage.buff[item_offset];
+                }
+            }
+        }
+
+        // Copy
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            output_items[ITEM] = temp_items[ITEM];
+        }
+    }
+
+
+    /**
+     * Transposes data items from <em>warp-striped</em> arrangement to <em>blocked</em> arrangement.  Specialized for no timeslicing
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void WarpStripedToBlocked(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        Int2Type<false> /*time_slicing*/)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = warp_offset + (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
+            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+            temp_storage.buff[item_offset] = input_items[ITEM];
+        }
+
+        WARP_SYNC(0xffffffff);
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = warp_offset + ITEM + (lane_id * ITEMS_PER_THREAD);
+            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+            output_items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+
+    /**
+     * Transposes data items from <em>warp-striped</em> arrangement to <em>blocked</em> arrangement.  Specialized for warp-timeslicing
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void WarpStripedToBlocked(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        Int2Type<true>  /*time_slicing*/)
+    {
+        #pragma unroll
+        for (unsigned int SLICE = 0; SLICE < TIME_SLICES; ++SLICE)
+        {
+            CTA_SYNC();
+
+            if (warp_id == SLICE)
+            {
+                #pragma unroll
+                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+                {
+                    int item_offset = (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
+                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                    temp_storage.buff[item_offset] = input_items[ITEM];
+                }
+
+                WARP_SYNC(0xffffffff);
+
+                #pragma unroll
+                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+                {
+                    int item_offset = ITEM + (lane_id * ITEMS_PER_THREAD);
+                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                    output_items[ITEM] = temp_storage.buff[item_offset];
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Exchanges data items annotated by rank into <em>blocked</em> arrangement.  Specialized for no timeslicing.
+     */
+    template <typename OutputT, typename OffsetT>
+    __device__ __forceinline__ void ScatterToBlocked(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OffsetT         ranks[ITEMS_PER_THREAD],    ///< [in] Corresponding scatter ranks
+        Int2Type<false> /*time_slicing*/)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = ranks[ITEM];
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            temp_storage.buff[item_offset] = input_items[ITEM];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM;
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            output_items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+    /**
+     * Exchanges data items annotated by rank into <em>blocked</em> arrangement.  Specialized for warp-timeslicing.
+     */
+    template <typename OutputT, typename OffsetT>
+    __device__ __forceinline__ void ScatterToBlocked(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OffsetT         ranks[ITEMS_PER_THREAD],    ///< [in] Corresponding scatter ranks
+        Int2Type<true>  /*time_slicing*/)
+    {
+        InputT temp_items[ITEMS_PER_THREAD];
+
+        #pragma unroll
+        for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++)
+        {
+            CTA_SYNC();
+
+            const int SLICE_OFFSET = TIME_SLICED_ITEMS * SLICE;
+
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                int item_offset = ranks[ITEM] - SLICE_OFFSET;
+                if ((item_offset >= 0) && (item_offset < WARP_TIME_SLICED_ITEMS))
+                {
+                    if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+                    temp_storage.buff[item_offset] = input_items[ITEM];
+                }
+            }
+
+            CTA_SYNC();
+
+            if (warp_id == SLICE)
+            {
+                #pragma unroll
+                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+                {
+                    int item_offset = (lane_id * ITEMS_PER_THREAD) + ITEM;
+                    if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+                    temp_items[ITEM] = temp_storage.buff[item_offset];
+                }
+            }
+        }
+
+        // Copy
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            output_items[ITEM] = temp_items[ITEM];
+        }
+    }
+
+
+    /**
+     * Exchanges data items annotated by rank into <em>striped</em> arrangement.  Specialized for no timeslicing.
+     */
+    template <typename OutputT, typename OffsetT>
+    __device__ __forceinline__ void ScatterToStriped(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OffsetT         ranks[ITEMS_PER_THREAD],    ///< [in] Corresponding scatter ranks
+        Int2Type<false> /*time_slicing*/)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = ranks[ITEM];
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            temp_storage.buff[item_offset] = input_items[ITEM];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            output_items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+
+    /**
+     * Exchanges data items annotated by rank into <em>striped</em> arrangement.  Specialized for warp-timeslicing.
+     */
+    template <typename OutputT, typename OffsetT>
+    __device__ __forceinline__ void ScatterToStriped(
+        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
+        OffsetT         ranks[ITEMS_PER_THREAD],    ///< [in] Corresponding scatter ranks
+        Int2Type<true> /*time_slicing*/)
+    {
+        InputT temp_items[ITEMS_PER_THREAD];
+
+        #pragma unroll
+        for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++)
+        {
+            const int SLICE_OFFSET  = SLICE * TIME_SLICED_ITEMS;
+            const int SLICE_OOB     = SLICE_OFFSET + TIME_SLICED_ITEMS;
+
+            CTA_SYNC();
+
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                int item_offset = ranks[ITEM] - SLICE_OFFSET;
+                if ((item_offset >= 0) && (item_offset < WARP_TIME_SLICED_ITEMS))
+                {
+                    if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+                    temp_storage.buff[item_offset] = input_items[ITEM];
+                }
+            }
+
+            CTA_SYNC();
+
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+            {
+                // Read a strip of items
+                const int STRIP_OFFSET  = ITEM * BLOCK_THREADS;
+                const int STRIP_OOB     = STRIP_OFFSET + BLOCK_THREADS;
+
+                if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET))
+                {
+                    int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET;
+                    if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS))
+                    {
+                        if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
+                        temp_items[ITEM] = temp_storage.buff[item_offset];
+                    }
+                }
+            }
+        }
+
+        // Copy
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            output_items[ITEM] = temp_items[ITEM];
+        }
+    }
+
+
+public:
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockExchange()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
+        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
+        lane_id(LaneId()),
+        warp_offset(warp_id * WARP_TIME_SLICED_ITEMS)
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockExchange(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
+        lane_id(LaneId()),
+        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
+        warp_offset(warp_id * WARP_TIME_SLICED_ITEMS)
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Structured exchanges
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Transposes data items from <em>striped</em> arrangement to <em>blocked</em> arrangement.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the conversion from a "striped" to a "blocked" arrangement
+     * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, ...)
+     * {
+     *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
+     *
+     *     // Allocate shared memory for BlockExchange
+     *     __shared__ typename BlockExchange::TempStorage temp_storage;
+     *
+     *     // Load a tile of ordered data into a striped arrangement across block threads
+     *     int thread_data[4];
+     *     cub::LoadDirectStriped<128>(threadIdx.x, d_data, thread_data);
+     *
+     *     // Collectively exchange data into a blocked arrangement across threads
+     *     BlockExchange(temp_storage).StripedToBlocked(thread_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of striped input \p thread_data across the block of threads is
+     * <tt>{ [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] }</tt> after loading from device-accessible memory.
+     * The corresponding output \p thread_data in those threads will be
+     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
+     *
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void StripedToBlocked(
+        InputT      input_items[ITEMS_PER_THREAD],    ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OutputT     output_items[ITEMS_PER_THREAD])   ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+    {
+        StripedToBlocked(input_items, output_items, Int2Type<WARP_TIME_SLICING>());
+    }
+
+
+    /**
+     * \brief Transposes data items from <em>blocked</em> arrangement to <em>striped</em> arrangement.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the conversion from a "blocked" to a "striped" arrangement
+     * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, ...)
+     * {
+     *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
+     *
+     *     // Allocate shared memory for BlockExchange
+     *     __shared__ typename BlockExchange::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively exchange data into a striped arrangement across threads
+     *     BlockExchange(temp_storage).BlockedToStriped(thread_data, thread_data);
+     *
+     *     // Store data striped across block threads into an ordered tile
+     *     cub::StoreDirectStriped<STORE_DEFAULT, 128>(threadIdx.x, d_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of blocked input \p thread_data across the block of threads is
+     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
+     * The corresponding output \p thread_data in those threads will be
+     * <tt>{ [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] }</tt> in
+     * preparation for storing to device-accessible memory.
+     *
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void BlockedToStriped(
+        InputT      input_items[ITEMS_PER_THREAD],    ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OutputT     output_items[ITEMS_PER_THREAD])   ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+    {
+        BlockedToStriped(input_items, output_items, Int2Type<WARP_TIME_SLICING>());
+    }
+
+
+
+    /**
+     * \brief Transposes data items from <em>warp-striped</em> arrangement to <em>blocked</em> arrangement.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the conversion from a "warp-striped" to a "blocked" arrangement
+     * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, ...)
+     * {
+     *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
+     *
+     *     // Allocate shared memory for BlockExchange
+     *     __shared__ typename BlockExchange::TempStorage temp_storage;
+     *
+     *     // Load a tile of ordered data into a warp-striped arrangement across warp threads
+     *     int thread_data[4];
+     *     cub::LoadSWarptriped<LOAD_DEFAULT>(threadIdx.x, d_data, thread_data);
+     *
+     *     // Collectively exchange data into a blocked arrangement across threads
+     *     BlockExchange(temp_storage).WarpStripedToBlocked(thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of warp-striped input \p thread_data across the block of threads is
+     * <tt>{ [0,32,64,96], [1,33,65,97], [2,34,66,98], ..., [415,447,479,511] }</tt>
+     * after loading from device-accessible memory.  (The first 128 items are striped across
+     * the first warp of 32 threads, the second 128 items are striped across the second warp, etc.)
+     * The corresponding output \p thread_data in those threads will be
+     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
+     *
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void WarpStripedToBlocked(
+        InputT      input_items[ITEMS_PER_THREAD],    ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OutputT     output_items[ITEMS_PER_THREAD])   ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+    {
+        WarpStripedToBlocked(input_items, output_items, Int2Type<WARP_TIME_SLICING>());
+    }
+
+
+
+    /**
+     * \brief Transposes data items from <em>blocked</em> arrangement to <em>warp-striped</em> arrangement.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the conversion from a "blocked" to a "warp-striped" arrangement
+     * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, ...)
+     * {
+     *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
+     *
+     *     // Allocate shared memory for BlockExchange
+     *     __shared__ typename BlockExchange::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively exchange data into a warp-striped arrangement across threads
+     *     BlockExchange(temp_storage).BlockedToWarpStriped(thread_data, thread_data);
+     *
+     *     // Store data striped across warp threads into an ordered tile
+     *     cub::StoreDirectStriped<STORE_DEFAULT, 128>(threadIdx.x, d_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of blocked input \p thread_data across the block of threads is
+     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
+     * The corresponding output \p thread_data in those threads will be
+     * <tt>{ [0,32,64,96], [1,33,65,97], [2,34,66,98], ..., [415,447,479,511] }</tt>
+     * in preparation for storing to device-accessible memory. (The first 128 items are striped across
+     * the first warp of 32 threads, the second 128 items are striped across the second warp, etc.)
+     *
+     */
+    template <typename OutputT>
+    __device__ __forceinline__ void BlockedToWarpStriped(
+        InputT      input_items[ITEMS_PER_THREAD],    ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OutputT     output_items[ITEMS_PER_THREAD])   ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+    {
+        BlockedToWarpStriped(input_items, output_items, Int2Type<WARP_TIME_SLICING>());
+    }
+
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Scatter exchanges
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Exchanges data items annotated by rank into <em>blocked</em> arrangement.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
+     */
+    template <typename OutputT, typename OffsetT>
+    __device__ __forceinline__ void ScatterToBlocked(
+        InputT      input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OutputT     output_items[ITEMS_PER_THREAD],     ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OffsetT     ranks[ITEMS_PER_THREAD])            ///< [in] Corresponding scatter ranks
+    {
+        ScatterToBlocked(input_items, output_items, ranks, Int2Type<WARP_TIME_SLICING>());
+    }
+
+
+
+    /**
+     * \brief Exchanges data items annotated by rank into <em>striped</em> arrangement.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
+     */
+    template <typename OutputT, typename OffsetT>
+    __device__ __forceinline__ void ScatterToStriped(
+        InputT      input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OutputT     output_items[ITEMS_PER_THREAD],     ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OffsetT     ranks[ITEMS_PER_THREAD])            ///< [in] Corresponding scatter ranks
+    {
+        ScatterToStriped(input_items, output_items, ranks, Int2Type<WARP_TIME_SLICING>());
+    }
+
+
+
+    /**
+     * \brief Exchanges data items annotated by rank into <em>striped</em> arrangement.  Items with rank -1 are not exchanged.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
+     */
+    template <typename OutputT, typename OffsetT>
+    __device__ __forceinline__ void ScatterToStripedGuarded(
+        InputT      input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OutputT     output_items[ITEMS_PER_THREAD],     ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OffsetT     ranks[ITEMS_PER_THREAD])            ///< [in] Corresponding scatter ranks
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = ranks[ITEM];
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            if (ranks[ITEM] >= 0)
+                temp_storage.buff[item_offset] = input_items[ITEM];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            output_items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+
+
+
+    /**
+     * \brief Exchanges valid data items annotated by rank into <em>striped</em> arrangement.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
+     * \tparam ValidFlag                            <b>[inferred]</b> FlagT type denoting which items are valid
+     */
+    template <typename OutputT, typename OffsetT, typename ValidFlag>
+    __device__ __forceinline__ void ScatterToStripedFlagged(
+        InputT      input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OutputT     output_items[ITEMS_PER_THREAD],     ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OffsetT     ranks[ITEMS_PER_THREAD],            ///< [in] Corresponding scatter ranks
+        ValidFlag   is_valid[ITEMS_PER_THREAD])         ///< [in] Corresponding flag denoting item validity
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = ranks[ITEM];
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            if (is_valid[ITEM])
+                temp_storage.buff[item_offset] = input_items[ITEM];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            output_items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+
+    //@}  end member group
+
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+
+    __device__ __forceinline__ void StripedToBlocked(
+        InputT      items[ITEMS_PER_THREAD])   ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+    {
+        StripedToBlocked(items, items);
+    }
+
+    __device__ __forceinline__ void BlockedToStriped(
+        InputT      items[ITEMS_PER_THREAD])   ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+    {
+        BlockedToStriped(items, items);
+    }
+
+    __device__ __forceinline__ void WarpStripedToBlocked(
+        InputT      items[ITEMS_PER_THREAD])    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+    {
+        WarpStripedToBlocked(items, items);
+    }
+
+    __device__ __forceinline__ void BlockedToWarpStriped(
+        InputT      items[ITEMS_PER_THREAD])    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+    {
+        BlockedToWarpStriped(items, items);
+    }
+
+    template <typename OffsetT>
+    __device__ __forceinline__ void ScatterToBlocked(
+        InputT      items[ITEMS_PER_THREAD],    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OffsetT     ranks[ITEMS_PER_THREAD])    ///< [in] Corresponding scatter ranks
+    {
+        ScatterToBlocked(items, items, ranks);
+    }
+
+    template <typename OffsetT>
+    __device__ __forceinline__ void ScatterToStriped(
+        InputT      items[ITEMS_PER_THREAD],    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OffsetT     ranks[ITEMS_PER_THREAD])    ///< [in] Corresponding scatter ranks
+    {
+        ScatterToStriped(items, items, ranks);
+    }
+
+    template <typename OffsetT>
+    __device__ __forceinline__ void ScatterToStripedGuarded(
+        InputT      items[ITEMS_PER_THREAD],    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OffsetT     ranks[ITEMS_PER_THREAD])    ///< [in] Corresponding scatter ranks
+    {
+        ScatterToStripedGuarded(items, items, ranks);
+    }
+
+    template <typename OffsetT, typename ValidFlag>
+    __device__ __forceinline__ void ScatterToStripedFlagged(
+        InputT      items[ITEMS_PER_THREAD],        ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
+        OffsetT     ranks[ITEMS_PER_THREAD],        ///< [in] Corresponding scatter ranks
+        ValidFlag   is_valid[ITEMS_PER_THREAD])     ///< [in] Corresponding flag denoting item validity
+    {
+        ScatterToStriped(items, items, ranks, is_valid);
+    }
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+};
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+
+template <
+    typename    T,
+    int         ITEMS_PER_THREAD,
+    int         LOGICAL_WARP_THREADS    = CUB_PTX_WARP_THREADS,
+    int         PTX_ARCH                = CUB_PTX_ARCH>
+class WarpExchange
+{
+private:
+
+    /******************************************************************************
+     * Constants
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        // Whether the logical warp size and the PTX warp size coincide
+        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
+
+        WARP_ITEMS                  = (ITEMS_PER_THREAD * LOGICAL_WARP_THREADS) + 1,
+
+        LOG_SMEM_BANKS              = CUB_LOG_SMEM_BANKS(PTX_ARCH),
+        SMEM_BANKS                  = 1 << LOG_SMEM_BANKS,
+
+        // Insert padding if the number of items per thread is a power of two and > 4 (otherwise we can typically use 128b loads)
+        INSERT_PADDING              = (ITEMS_PER_THREAD > 4) && (PowerOfTwo<ITEMS_PER_THREAD>::VALUE),
+        PADDING_ITEMS               = (INSERT_PADDING) ? (WARP_ITEMS >> LOG_SMEM_BANKS) : 0,
+    };
+
+    /******************************************************************************
+     * Type definitions
+     ******************************************************************************/
+
+    /// Shared memory storage layout type
+    struct _TempStorage
+    {
+        T buff[WARP_ITEMS + PADDING_ITEMS];
+    };
+
+public:
+
+    /// \smemstorage{WarpExchange}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+private:
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    _TempStorage    &temp_storage;
+    int             lane_id;
+
+public:
+
+    /******************************************************************************
+     * Construction
+     ******************************************************************************/
+
+    /// Constructor
+    __device__ __forceinline__ WarpExchange(
+        TempStorage &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        lane_id(IS_ARCH_WARP ?
+            LaneId() :
+            LaneId() % LOGICAL_WARP_THREADS)
+    {}
+
+
+    /******************************************************************************
+     * Interface
+     ******************************************************************************/
+
+    /**
+     * \brief Exchanges valid data items annotated by rank into <em>striped</em> arrangement.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
+     */
+    template <typename OffsetT>
+    __device__ __forceinline__ void ScatterToStriped(
+        T               items[ITEMS_PER_THREAD],        ///< [in-out] Items to exchange
+        OffsetT         ranks[ITEMS_PER_THREAD])        ///< [in] Corresponding scatter ranks
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            if (INSERT_PADDING) ranks[ITEM] = SHR_ADD(ranks[ITEM], LOG_SMEM_BANKS, ranks[ITEM]);
+            temp_storage.buff[ranks[ITEM]] = items[ITEM];
+        }
+
+        WARP_SYNC(0xffffffff);
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            int item_offset = (ITEM * LOGICAL_WARP_THREADS) + lane_id;
+            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
+            items[ITEM] = temp_storage.buff[item_offset];
+        }
+    }
+
+};
+
+
+
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_histogram.cuh b/third_party/cub/cub/block/block_histogram.cuh
new file mode 100644
index 0000000..b7cb970
--- /dev/null
+++ b/third_party/cub/cub/block/block_histogram.cuh
@@ -0,0 +1,415 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockHistogram class provides [<em>collective</em>](index.html#sec0) methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "specializations/block_histogram_sort.cuh"
+#include "specializations/block_histogram_atomic.cuh"
+#include "../util_ptx.cuh"
+#include "../util_arch.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Algorithmic variants
+ ******************************************************************************/
+
+/**
+ * \brief BlockHistogramAlgorithm enumerates alternative algorithms for the parallel construction of block-wide histograms.
+ */
+enum BlockHistogramAlgorithm
+{
+
+    /**
+     * \par Overview
+     * Sorting followed by differentiation.  Execution is comprised of two phases:
+     * -# Sort the data using efficient radix sort
+     * -# Look for "runs" of same-valued keys by detecting discontinuities; the run-lengths are histogram bin counts.
+     *
+     * \par Performance Considerations
+     * Delivers consistent throughput regardless of sample bin distribution.
+     */
+    BLOCK_HISTO_SORT,
+
+
+    /**
+     * \par Overview
+     * Use atomic addition to update byte counts directly
+     *
+     * \par Performance Considerations
+     * Performance is strongly tied to the hardware implementation of atomic
+     * addition, and may be significantly degraded for non uniformly-random
+     * input distributions where many concurrent updates are likely to be
+     * made to the same bin counter.
+     */
+    BLOCK_HISTO_ATOMIC,
+};
+
+
+
+/******************************************************************************
+ * Block histogram
+ ******************************************************************************/
+
+
+/**
+ * \brief The BlockHistogram class provides [<em>collective</em>](index.html#sec0) methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block. ![](histogram_logo.png)
+ * \ingroup BlockModule
+ *
+ * \tparam T                    The sample type being histogrammed (must be castable to an integer bin identifier)
+ * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
+ * \tparam ITEMS_PER_THREAD     The number of items per thread
+ * \tparam BINS                 The number bins within the histogram
+ * \tparam ALGORITHM            <b>[optional]</b> cub::BlockHistogramAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_HISTO_SORT)
+ * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - A <a href="http://en.wikipedia.org/wiki/Histogram"><em>histogram</em></a>
+ *   counts the number of observations that fall into each of the disjoint categories (known as <em>bins</em>).
+ * - BlockHistogram can be optionally specialized to use different algorithms:
+ *   -# <b>cub::BLOCK_HISTO_SORT</b>.  Sorting followed by differentiation. [More...](\ref cub::BlockHistogramAlgorithm)
+ *   -# <b>cub::BLOCK_HISTO_ATOMIC</b>.  Use atomic addition to update byte counts directly. [More...](\ref cub::BlockHistogramAlgorithm)
+ *
+ * \par Performance Considerations
+ * - \granularity
+ *
+ * \par A Simple Example
+ * \blockcollective{BlockHistogram}
+ * \par
+ * The code snippet below illustrates a 256-bin histogram of 512 integer samples that
+ * are partitioned across 128 threads where each thread owns 4 samples.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/block/block_histogram.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize a 256-bin BlockHistogram type for a 1D block of 128 threads having 4 character samples each
+ *     typedef cub::BlockHistogram<unsigned char, 128, 4, 256> BlockHistogram;
+ *
+ *     // Allocate shared memory for BlockHistogram
+ *     __shared__ typename BlockHistogram::TempStorage temp_storage;
+ *
+ *     // Allocate shared memory for block-wide histogram bin counts
+ *     __shared__ unsigned int smem_histogram[256];
+ *
+ *     // Obtain input samples per thread
+ *     unsigned char data[4];
+ *     ...
+ *
+ *     // Compute the block-wide histogram
+ *     BlockHistogram(temp_storage).Histogram(data, smem_histogram);
+ *
+ * \endcode
+ *
+ * \par Performance and Usage Considerations
+ * - The histogram output can be constructed in shared or device-accessible memory
+ * - See cub::BlockHistogramAlgorithm for performance details regarding algorithmic alternatives
+ *
+ */
+template <
+    typename                T,
+    int                     BLOCK_DIM_X,
+    int                     ITEMS_PER_THREAD,
+    int                     BINS,
+    BlockHistogramAlgorithm ALGORITHM           = BLOCK_HISTO_SORT,
+    int                     BLOCK_DIM_Y         = 1,
+    int                     BLOCK_DIM_Z         = 1,
+    int                     PTX_ARCH            = CUB_PTX_ARCH>
+class BlockHistogram
+{
+private:
+
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+    /**
+     * Ensure the template parameterization meets the requirements of the
+     * targeted device architecture.  BLOCK_HISTO_ATOMIC can only be used
+     * on version SM120 or later.  Otherwise BLOCK_HISTO_SORT is used
+     * regardless.
+     */
+    static const BlockHistogramAlgorithm SAFE_ALGORITHM =
+        ((ALGORITHM == BLOCK_HISTO_ATOMIC) && (PTX_ARCH < 120)) ?
+            BLOCK_HISTO_SORT :
+            ALGORITHM;
+
+    /// Internal specialization.
+    typedef typename If<(SAFE_ALGORITHM == BLOCK_HISTO_SORT),
+        BlockHistogramSort<T, BLOCK_DIM_X, ITEMS_PER_THREAD, BINS, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH>,
+        BlockHistogramAtomic<BINS> >::Type InternalBlockHistogram;
+
+    /// Shared memory storage layout type for BlockHistogram
+    typedef typename InternalBlockHistogram::TempStorage _TempStorage;
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+public:
+
+    /// \smemstorage{BlockHistogram}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockHistogram()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockHistogram(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Histogram operations
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Initialize the shared histogram counters to zero.
+     *
+     * \par Snippet
+     * The code snippet below illustrates a the initialization and update of a
+     * histogram of 512 integer samples that are partitioned across 128 threads
+     * where each thread owns 4 samples.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_histogram.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize a 256-bin BlockHistogram type for a 1D block of 128 threads having 4 character samples each
+     *     typedef cub::BlockHistogram<unsigned char, 128, 4, 256> BlockHistogram;
+     *
+     *     // Allocate shared memory for BlockHistogram
+     *     __shared__ typename BlockHistogram::TempStorage temp_storage;
+     *
+     *     // Allocate shared memory for block-wide histogram bin counts
+     *     __shared__ unsigned int smem_histogram[256];
+     *
+     *     // Obtain input samples per thread
+     *     unsigned char thread_samples[4];
+     *     ...
+     *
+     *     // Initialize the block-wide histogram
+     *     BlockHistogram(temp_storage).InitHistogram(smem_histogram);
+     *
+     *     // Update the block-wide histogram
+     *     BlockHistogram(temp_storage).Composite(thread_samples, smem_histogram);
+     *
+     * \endcode
+     *
+     * \tparam CounterT              <b>[inferred]</b> Histogram counter type
+     */
+    template <typename CounterT     >
+    __device__ __forceinline__ void InitHistogram(CounterT      histogram[BINS])
+    {
+        // Initialize histogram bin counts to zeros
+        int histo_offset = 0;
+
+        #pragma unroll
+        for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS)
+        {
+            histogram[histo_offset + linear_tid] = 0;
+        }
+        // Finish up with guarded initialization if necessary
+        if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS))
+        {
+            histogram[histo_offset + linear_tid] = 0;
+        }
+    }
+
+
+    /**
+     * \brief Constructs a block-wide histogram in shared/device-accessible memory.  Each thread contributes an array of input elements.
+     *
+     * \par
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a 256-bin histogram of 512 integer samples that
+     * are partitioned across 128 threads where each thread owns 4 samples.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_histogram.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize a 256-bin BlockHistogram type for a 1D block of 128 threads having 4 character samples each
+     *     typedef cub::BlockHistogram<unsigned char, 128, 4, 256> BlockHistogram;
+     *
+     *     // Allocate shared memory for BlockHistogram
+     *     __shared__ typename BlockHistogram::TempStorage temp_storage;
+     *
+     *     // Allocate shared memory for block-wide histogram bin counts
+     *     __shared__ unsigned int smem_histogram[256];
+     *
+     *     // Obtain input samples per thread
+     *     unsigned char thread_samples[4];
+     *     ...
+     *
+     *     // Compute the block-wide histogram
+     *     BlockHistogram(temp_storage).Histogram(thread_samples, smem_histogram);
+     *
+     * \endcode
+     *
+     * \tparam CounterT              <b>[inferred]</b> Histogram counter type
+     */
+    template <
+        typename            CounterT     >
+    __device__ __forceinline__ void Histogram(
+        T                   (&items)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input values to histogram
+        CounterT             histogram[BINS])                ///< [out] Reference to shared/device-accessible memory histogram
+    {
+        // Initialize histogram bin counts to zeros
+        InitHistogram(histogram);
+
+        CTA_SYNC();
+
+        // Composite the histogram
+        InternalBlockHistogram(temp_storage).Composite(items, histogram);
+    }
+
+
+
+    /**
+     * \brief Updates an existing block-wide histogram in shared/device-accessible memory.  Each thread composites an array of input elements.
+     *
+     * \par
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a the initialization and update of a
+     * histogram of 512 integer samples that are partitioned across 128 threads
+     * where each thread owns 4 samples.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_histogram.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize a 256-bin BlockHistogram type for a 1D block of 128 threads having 4 character samples each
+     *     typedef cub::BlockHistogram<unsigned char, 128, 4, 256> BlockHistogram;
+     *
+     *     // Allocate shared memory for BlockHistogram
+     *     __shared__ typename BlockHistogram::TempStorage temp_storage;
+     *
+     *     // Allocate shared memory for block-wide histogram bin counts
+     *     __shared__ unsigned int smem_histogram[256];
+     *
+     *     // Obtain input samples per thread
+     *     unsigned char thread_samples[4];
+     *     ...
+     *
+     *     // Initialize the block-wide histogram
+     *     BlockHistogram(temp_storage).InitHistogram(smem_histogram);
+     *
+     *     // Update the block-wide histogram
+     *     BlockHistogram(temp_storage).Composite(thread_samples, smem_histogram);
+     *
+     * \endcode
+     *
+     * \tparam CounterT              <b>[inferred]</b> Histogram counter type
+     */
+    template <
+        typename            CounterT     >
+    __device__ __forceinline__ void Composite(
+        T                   (&items)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input values to histogram
+        CounterT             histogram[BINS])                 ///< [out] Reference to shared/device-accessible memory histogram
+    {
+        InternalBlockHistogram(temp_storage).Composite(items, histogram);
+    }
+
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_load.cuh b/third_party/cub/cub/block/block_load.cuh
new file mode 100644
index 0000000..217f521
--- /dev/null
+++ b/third_party/cub/cub/block/block_load.cuh
@@ -0,0 +1,1241 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2016, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Operations for reading linear tiles of data into the CUDA thread block.
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "block_exchange.cuh"
+#include "../iterator/cache_modified_input_iterator.cuh"
+#include "../util_ptx.cuh"
+#include "../util_macro.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup UtilIo
+ * @{
+ */
+
+
+/******************************************************************//**
+ * \name Blocked arrangement I/O (direct)
+ *********************************************************************/
+//@{
+
+
+/**
+ * \brief Load a linear segment of items into a blocked arrangement across the thread block.
+ *
+ * \blocked
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    typename        InputT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectBlocked(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
+{
+    InputIteratorT thread_itr = block_itr + (linear_tid * ITEMS_PER_THREAD);
+
+    // Load directly in thread-blocked order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        items[ITEM] = thread_itr[ITEM];
+    }
+}
+
+
+/**
+ * \brief Load a linear segment of items into a blocked arrangement across the thread block, guarded by range.
+ *
+ * \blocked
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    typename        InputT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectBlocked(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
+    int             valid_items)                ///< [in] Number of valid items to load
+{
+    InputIteratorT thread_itr = block_itr + (linear_tid * ITEMS_PER_THREAD);
+
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        if ((linear_tid * ITEMS_PER_THREAD) + ITEM < valid_items)
+        {
+            items[ITEM] = thread_itr[ITEM];
+        }
+    }
+}
+
+
+/**
+ * \brief Load a linear segment of items into a blocked arrangement across the thread block, guarded by range, with a fall-back assignment of out-of-bound elements..
+ *
+ * \blocked
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    typename        InputT,
+    typename        DefaultT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectBlocked(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
+    int             valid_items,                ///< [in] Number of valid items to load
+    DefaultT        oob_default)                ///< [in] Default value to assign out-of-bound items
+{
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        items[ITEM] = oob_default;
+
+    LoadDirectBlocked(linear_tid, block_itr, items, valid_items);
+}
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+/**
+ * Internal implementation for load vectorization
+ */
+template <
+    CacheLoadModifier   MODIFIER,
+    typename            T,
+    int                 ITEMS_PER_THREAD>
+__device__ __forceinline__ void InternalLoadDirectBlockedVectorized(
+    int    linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    T      *block_ptr,                 ///< [in] Input pointer for loading from
+    T      (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
+{
+    // Biggest memory access word that T is a whole multiple of
+    typedef typename UnitWord<T>::DeviceWord DeviceWord;
+
+    enum
+    {
+        TOTAL_WORDS = sizeof(items) / sizeof(DeviceWord),
+
+        VECTOR_SIZE = (TOTAL_WORDS % 4 == 0) ?
+            4 :
+            (TOTAL_WORDS % 2 == 0) ?
+                2 :
+                1,
+
+        VECTORS_PER_THREAD = TOTAL_WORDS / VECTOR_SIZE,
+    };
+
+    // Vector type
+    typedef typename CubVector<DeviceWord, VECTOR_SIZE>::Type Vector;
+
+    // Vector items
+    Vector vec_items[VECTORS_PER_THREAD];
+
+    // Aliased input ptr
+    Vector* vec_ptr = reinterpret_cast<Vector*>(block_ptr) + (linear_tid * VECTORS_PER_THREAD);
+
+    // Load directly in thread-blocked order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < VECTORS_PER_THREAD; ITEM++)
+    {
+        vec_items[ITEM] = ThreadLoad<MODIFIER>(vec_ptr + ITEM);
+    }
+
+    // Copy
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        items[ITEM] = *(reinterpret_cast<T*>(vec_items) + ITEM);
+    }
+}
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/**
+ * \brief Load a linear segment of items into a blocked arrangement across the thread block.
+ *
+ * \blocked
+ *
+ * The input offset (\p block_ptr + \p block_offset) must be quad-item aligned
+ *
+ * The following conditions will prevent vectorization and loading will fall back to cub::BLOCK_LOAD_DIRECT:
+ *   - \p ITEMS_PER_THREAD is odd
+ *   - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.)
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ */
+template <
+    typename        T,
+    int             ITEMS_PER_THREAD>
+__device__ __forceinline__ void LoadDirectBlockedVectorized(
+    int linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    T   *block_ptr,                 ///< [in] Input pointer for loading from
+    T   (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
+{
+    InternalLoadDirectBlockedVectorized<LOAD_DEFAULT>(linear_tid, block_ptr, items);
+}
+
+
+//@}  end member group
+/******************************************************************//**
+ * \name Striped arrangement I/O (direct)
+ *********************************************************************/
+//@{
+
+
+/**
+ * \brief Load a linear segment of items into a striped arrangement across the thread block.
+ *
+ * \striped
+ *
+ * \tparam BLOCK_THREADS        The thread block size in threads
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    int             BLOCK_THREADS,
+    typename        InputT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectStriped(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
+{
+    InputIteratorT thread_itr = block_itr + linear_tid;
+
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        items[ITEM] = thread_itr[ITEM * BLOCK_THREADS];
+    }
+}
+
+
+/**
+ * \brief Load a linear segment of items into a striped arrangement across the thread block, guarded by range
+ *
+ * \striped
+ *
+ * \tparam BLOCK_THREADS        The thread block size in threads
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    int             BLOCK_THREADS,
+    typename        InputT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectStriped(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
+    int             valid_items)                ///< [in] Number of valid items to load
+{
+    InputIteratorT thread_itr = block_itr + linear_tid;
+
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        if (linear_tid + (ITEM * BLOCK_THREADS) < valid_items)
+        {
+            items[ITEM] = thread_itr[ITEM * BLOCK_THREADS];
+        }
+    }
+}
+
+
+/**
+ * \brief Load a linear segment of items into a striped arrangement across the thread block, guarded by range, with a fall-back assignment of out-of-bound elements.
+ *
+ * \striped
+ *
+ * \tparam BLOCK_THREADS        The thread block size in threads
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    int             BLOCK_THREADS,
+    typename        InputT,
+    typename        DefaultT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectStriped(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
+    int             valid_items,                ///< [in] Number of valid items to load
+    DefaultT        oob_default)                ///< [in] Default value to assign out-of-bound items
+{
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        items[ITEM] = oob_default;
+
+    LoadDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items, valid_items);
+}
+
+
+
+//@}  end member group
+/******************************************************************//**
+ * \name Warp-striped arrangement I/O (direct)
+ *********************************************************************/
+//@{
+
+
+/**
+ * \brief Load a linear segment of items into a warp-striped arrangement across the thread block.
+ *
+ * \warpstriped
+ *
+ * \par Usage Considerations
+ * The number of threads in the thread block must be a multiple of the architecture's warp size.
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    typename        InputT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectWarpStriped(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
+{
+    int tid                = linear_tid & (CUB_PTX_WARP_THREADS - 1);
+    int wid                = linear_tid >> CUB_PTX_LOG_WARP_THREADS;
+    int warp_offset        = wid * CUB_PTX_WARP_THREADS * ITEMS_PER_THREAD;
+
+    InputIteratorT thread_itr = block_itr + warp_offset + tid ;
+
+    // Load directly in warp-striped order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        items[ITEM] = thread_itr[(ITEM * CUB_PTX_WARP_THREADS)];
+    }
+}
+
+
+/**
+ * \brief Load a linear segment of items into a warp-striped arrangement across the thread block, guarded by range
+ *
+ * \warpstriped
+ *
+ * \par Usage Considerations
+ * The number of threads in the thread block must be a multiple of the architecture's warp size.
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT        <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    typename        InputT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectWarpStriped(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
+    int             valid_items)                ///< [in] Number of valid items to load
+{
+    int tid                = linear_tid & (CUB_PTX_WARP_THREADS - 1);
+    int wid                = linear_tid >> CUB_PTX_LOG_WARP_THREADS;
+    int warp_offset        = wid * CUB_PTX_WARP_THREADS * ITEMS_PER_THREAD;
+
+    InputIteratorT thread_itr = block_itr + warp_offset + tid ;
+
+    // Load directly in warp-striped order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        if (warp_offset + tid + (ITEM * CUB_PTX_WARP_THREADS) < valid_items)
+        {
+            items[ITEM] = thread_itr[(ITEM * CUB_PTX_WARP_THREADS)];
+        }
+    }
+}
+
+
+/**
+ * \brief Load a linear segment of items into a warp-striped arrangement across the thread block, guarded by range, with a fall-back assignment of out-of-bound elements.
+ *
+ * \warpstriped
+ *
+ * \par Usage Considerations
+ * The number of threads in the thread block must be a multiple of the architecture's warp size.
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to load.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam InputIteratorT        <b>[inferred]</b> The random-access iterator type for input \iterator.
+ */
+template <
+    typename        InputT,
+    typename        DefaultT,
+    int             ITEMS_PER_THREAD,
+    typename        InputIteratorT>
+__device__ __forceinline__ void LoadDirectWarpStriped(
+    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
+    int             valid_items,                ///< [in] Number of valid items to load
+    DefaultT        oob_default)                ///< [in] Default value to assign out-of-bound items
+{
+    // Load directly in warp-striped order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        items[ITEM] = oob_default;
+
+    LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items);
+}
+
+
+
+//@}  end member group
+
+/** @} */       // end group UtilIo
+
+
+
+//-----------------------------------------------------------------------------
+// Generic BlockLoad abstraction
+//-----------------------------------------------------------------------------
+
+/**
+ * \brief cub::BlockLoadAlgorithm enumerates alternative algorithms for cub::BlockLoad to read a linear segment of data from memory into a blocked arrangement across a CUDA thread block.
+ */
+
+/**
+ * \brief cub::BlockLoadAlgorithm enumerates alternative algorithms for cub::BlockLoad to read a linear segment of data from memory into a blocked arrangement across a CUDA thread block.
+ */
+enum BlockLoadAlgorithm
+{
+    /**
+     * \par Overview
+     *
+     * A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is read
+     * directly from memory.
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) decreases as the
+     *   access stride between threads increases (i.e., the number items per thread).
+     */
+    BLOCK_LOAD_DIRECT,
+
+    /**
+     * \par Overview
+     *
+     * A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is read
+     * from memory using CUDA's built-in vectorized loads as a coalescing optimization.
+     * For example, <tt>ld.global.v4.s32</tt> instructions will be generated
+     * when \p T = \p int and \p ITEMS_PER_THREAD % 4 == 0.
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) remains high until the the
+     *   access stride between threads (i.e., the number items per thread) exceeds the
+     *   maximum vector load width (typically 4 items or 64B, whichever is lower).
+     * - The following conditions will prevent vectorization and loading will fall back to cub::BLOCK_LOAD_DIRECT:
+     *   - \p ITEMS_PER_THREAD is odd
+     *   - The \p InputIteratorTis not a simple pointer type
+     *   - The block input offset is not quadword-aligned
+     *   - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.)
+     */
+    BLOCK_LOAD_VECTORIZE,
+
+    /**
+     * \par Overview
+     *
+     * A [<em>striped arrangement</em>](index.html#sec5sec3) of data is read
+     * efficiently from memory and then locally transposed into a
+     * [<em>blocked arrangement</em>](index.html#sec5sec3).
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) remains high regardless
+     *   of items loaded per thread.
+     * - The local reordering incurs slightly longer latencies and throughput than the
+     *   direct cub::BLOCK_LOAD_DIRECT and cub::BLOCK_LOAD_VECTORIZE alternatives.
+     */
+    BLOCK_LOAD_TRANSPOSE,
+
+
+    /**
+     * \par Overview
+     *
+     * A [<em>warp-striped arrangement</em>](index.html#sec5sec3) of data is
+     * read efficiently from memory and then locally transposed into a
+     * [<em>blocked arrangement</em>](index.html#sec5sec3).
+     *
+     * \par Usage Considerations
+     * - BLOCK_THREADS must be a multiple of WARP_THREADS
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) remains high regardless
+     *   of items loaded per thread.
+     * - The local reordering incurs slightly larger latencies than the
+     *   direct cub::BLOCK_LOAD_DIRECT and cub::BLOCK_LOAD_VECTORIZE alternatives.
+     * - Provisions more shared storage, but incurs smaller latencies than the
+     *   BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED alternative.
+     */
+    BLOCK_LOAD_WARP_TRANSPOSE,
+
+
+    /**
+     * \par Overview
+     *
+     * Like \p BLOCK_LOAD_WARP_TRANSPOSE, a [<em>warp-striped arrangement</em>](index.html#sec5sec3)
+     * of data is read directly from memory and then is locally transposed into a
+     * [<em>blocked arrangement</em>](index.html#sec5sec3). To reduce the shared memory
+     * requirement, only one warp's worth of shared memory is provisioned and is
+     * subsequently time-sliced among warps.
+     *
+     * \par Usage Considerations
+     * - BLOCK_THREADS must be a multiple of WARP_THREADS
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) remains high regardless
+     *   of items loaded per thread.
+     * - Provisions less shared memory temporary storage, but incurs larger
+     *   latencies than the BLOCK_LOAD_WARP_TRANSPOSE alternative.
+     */
+    BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED,
+};
+
+
+/**
+ * \brief The BlockLoad class provides [<em>collective</em>](index.html#sec0) data movement methods for loading a linear segment of items from memory into a [<em>blocked arrangement</em>](index.html#sec5sec3) across a CUDA thread block.  ![](block_load_logo.png)
+ * \ingroup BlockModule
+ * \ingroup UtilIo
+ *
+ * \tparam InputT               The data type to read into (which must be convertible from the input iterator's value type).
+ * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
+ * \tparam ITEMS_PER_THREAD     The number of consecutive items partitioned onto each thread.
+ * \tparam ALGORITHM            <b>[optional]</b> cub::BlockLoadAlgorithm tuning policy.  default: cub::BLOCK_LOAD_DIRECT.
+ * \tparam WARP_TIME_SLICING    <b>[optional]</b> Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any load-related data transpositions (versus each warp having its own storage). (default: false)
+ * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - The BlockLoad class provides a single data movement abstraction that can be specialized
+ *   to implement different cub::BlockLoadAlgorithm strategies.  This facilitates different
+ *   performance policies for different architectures, data types, granularity sizes, etc.
+ * - BlockLoad can be optionally specialized by different data movement strategies:
+ *   -# <b>cub::BLOCK_LOAD_DIRECT</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
+ *      of data is read directly from memory.  [More...](\ref cub::BlockLoadAlgorithm)
+ *   -# <b>cub::BLOCK_LOAD_VECTORIZE</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
+ *      of data is read directly from memory using CUDA's built-in vectorized loads as a
+ *      coalescing optimization.    [More...](\ref cub::BlockLoadAlgorithm)
+ *   -# <b>cub::BLOCK_LOAD_TRANSPOSE</b>.  A [<em>striped arrangement</em>](index.html#sec5sec3)
+ *      of data is read directly from memory and is then locally transposed into a
+ *      [<em>blocked arrangement</em>](index.html#sec5sec3).  [More...](\ref cub::BlockLoadAlgorithm)
+ *   -# <b>cub::BLOCK_LOAD_WARP_TRANSPOSE</b>.  A [<em>warp-striped arrangement</em>](index.html#sec5sec3)
+ *      of data is read directly from memory and is then locally transposed into a
+ *      [<em>blocked arrangement</em>](index.html#sec5sec3).  [More...](\ref cub::BlockLoadAlgorithm)
+ *   -# <b>cub::BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED,</b>.  A [<em>warp-striped arrangement</em>](index.html#sec5sec3)
+ *      of data is read directly from memory and is then locally transposed into a
+ *      [<em>blocked arrangement</em>](index.html#sec5sec3) one warp at a time.  [More...](\ref cub::BlockLoadAlgorithm)
+ * - \rowmajor
+ *
+ * \par A Simple Example
+ * \blockcollective{BlockLoad}
+ * \par
+ * The code snippet below illustrates the loading of a linear
+ * segment of 512 integers into a "blocked" arrangement across 128 threads where each
+ * thread owns 4 consecutive items.  The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE,
+ * meaning memory references are efficiently coalesced using a warp-striped access
+ * pattern (after which items are locally reordered among threads).
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/block/block_load.cuh>
+ *
+ * __global__ void ExampleKernel(int *d_data, ...)
+ * {
+ *     // Specialize BlockLoad for a 1D block of 128 threads owning 4 integer items each
+ *     typedef cub::BlockLoad<int, 128, 4, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoad;
+ *
+ *     // Allocate shared memory for BlockLoad
+ *     __shared__ typename BlockLoad::TempStorage temp_storage;
+ *
+ *     // Load a segment of consecutive items that are blocked across threads
+ *     int thread_data[4];
+ *     BlockLoad(temp_storage).Load(d_data, thread_data);
+ *
+ * \endcode
+ * \par
+ * Suppose the input \p d_data is <tt>0, 1, 2, 3, 4, 5, ...</tt>.
+ * The set of \p thread_data across the block of threads in those threads will be
+ * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
+ *
+ */
+template <
+    typename            InputT,
+    int                 BLOCK_DIM_X,
+    int                 ITEMS_PER_THREAD,
+    BlockLoadAlgorithm  ALGORITHM           = BLOCK_LOAD_DIRECT,
+    int                 BLOCK_DIM_Y         = 1,
+    int                 BLOCK_DIM_Z         = 1,
+    int                 PTX_ARCH            = CUB_PTX_ARCH>
+class BlockLoad
+{
+private:
+
+    /******************************************************************************
+     * Constants and typed definitions
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+
+    /******************************************************************************
+     * Algorithmic variants
+     ******************************************************************************/
+
+    /// Load helper
+    template <BlockLoadAlgorithm _POLICY, int DUMMY>
+    struct LoadInternal;
+
+
+    /**
+     * BLOCK_LOAD_DIRECT specialization of load helper
+     */
+    template <int DUMMY>
+    struct LoadInternal<BLOCK_LOAD_DIRECT, DUMMY>
+    {
+        /// Shared memory storage layout type
+        typedef NullType TempStorage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ LoadInternal(
+            TempStorage &/*temp_storage*/,
+            int linear_tid)
+        :
+            linear_tid(linear_tid)
+        {}
+
+        /// Load a linear segment of items from memory
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load
+        {
+            LoadDirectBlocked(linear_tid, block_itr, items);
+        }
+
+        /// Load a linear segment of items from memory, guarded by range
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items)                    ///< [in] Number of valid items to load
+        {
+            LoadDirectBlocked(linear_tid, block_itr, items, valid_items);
+        }
+
+        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
+        template <typename InputIteratorT, typename DefaultT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items,                    ///< [in] Number of valid items to load
+            DefaultT        oob_default)                    ///< [in] Default value to assign out-of-bound items
+        {
+            LoadDirectBlocked(linear_tid, block_itr, items, valid_items, oob_default);
+        }
+
+    };
+
+
+    /**
+     * BLOCK_LOAD_VECTORIZE specialization of load helper
+     */
+    template <int DUMMY>
+    struct LoadInternal<BLOCK_LOAD_VECTORIZE, DUMMY>
+    {
+        /// Shared memory storage layout type
+        typedef NullType TempStorage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ LoadInternal(
+            TempStorage &/*temp_storage*/,
+            int linear_tid)
+        :
+            linear_tid(linear_tid)
+        {}
+
+        /// Load a linear segment of items from memory, specialized for native pointer types (attempts vectorization)
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputT               *block_ptr,                     ///< [in] The thread block's base input iterator for loading from
+            InputT               (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load
+        {
+            InternalLoadDirectBlockedVectorized<LOAD_DEFAULT>(linear_tid, block_ptr, items);
+        }
+
+        /// Load a linear segment of items from memory, specialized for native pointer types (attempts vectorization)
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            const InputT         *block_ptr,                     ///< [in] The thread block's base input iterator for loading from
+            InputT               (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load
+        {
+            InternalLoadDirectBlockedVectorized<LOAD_DEFAULT>(linear_tid, block_ptr, items);
+        }
+
+        /// Load a linear segment of items from memory, specialized for native pointer types (attempts vectorization)
+        template <
+            CacheLoadModifier   MODIFIER,
+            typename            ValueType,
+            typename            OffsetT>
+        __device__ __forceinline__ void Load(
+            CacheModifiedInputIterator<MODIFIER, ValueType, OffsetT>    block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT                                                     (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load
+        {
+            InternalLoadDirectBlockedVectorized<MODIFIER>(linear_tid, block_itr.ptr, items);
+        }
+
+        /// Load a linear segment of items from memory, specialized for opaque input iterators (skips vectorization)
+        template <typename _InputIteratorT>
+        __device__ __forceinline__ void Load(
+            _InputIteratorT   block_itr,                    ///< [in] The thread block's base input iterator for loading from
+            InputT           (&items)[ITEMS_PER_THREAD])   ///< [out] Data to load
+        {
+            LoadDirectBlocked(linear_tid, block_itr, items);
+        }
+
+        /// Load a linear segment of items from memory, guarded by range (skips vectorization)
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items)                    ///< [in] Number of valid items to load
+        {
+            LoadDirectBlocked(linear_tid, block_itr, items, valid_items);
+        }
+
+        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements (skips vectorization)
+        template <typename InputIteratorT, typename DefaultT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items,                    ///< [in] Number of valid items to load
+            DefaultT          oob_default)                    ///< [in] Default value to assign out-of-bound items
+        {
+            LoadDirectBlocked(linear_tid, block_itr, items, valid_items, oob_default);
+        }
+
+    };
+
+
+    /**
+     * BLOCK_LOAD_TRANSPOSE specialization of load helper
+     */
+    template <int DUMMY>
+    struct LoadInternal<BLOCK_LOAD_TRANSPOSE, DUMMY>
+    {
+        // BlockExchange utility type for keys
+        typedef BlockExchange<InputT, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
+
+        /// Shared memory storage layout type
+        struct _TempStorage : BlockExchange::TempStorage
+        {};
+
+        /// Alias wrapper allowing storage to be unioned
+        struct TempStorage : Uninitialized<_TempStorage> {};
+
+        /// Thread reference to shared storage
+        _TempStorage &temp_storage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ LoadInternal(
+            TempStorage &temp_storage,
+            int linear_tid)
+        :
+            temp_storage(temp_storage.Alias()),
+            linear_tid(linear_tid)
+        {}
+
+        /// Load a linear segment of items from memory
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load{
+        {
+            LoadDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items);
+            BlockExchange(temp_storage).StripedToBlocked(items, items);
+        }
+
+        /// Load a linear segment of items from memory, guarded by range
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items)                    ///< [in] Number of valid items to load
+        {
+            LoadDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items, valid_items);
+            BlockExchange(temp_storage).StripedToBlocked(items, items);
+        }
+
+        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
+        template <typename InputIteratorT, typename DefaultT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items,                    ///< [in] Number of valid items to load
+            DefaultT        oob_default)                    ///< [in] Default value to assign out-of-bound items
+        {
+            LoadDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items, valid_items, oob_default);
+            BlockExchange(temp_storage).StripedToBlocked(items, items);
+        }
+
+    };
+
+
+    /**
+     * BLOCK_LOAD_WARP_TRANSPOSE specialization of load helper
+     */
+    template <int DUMMY>
+    struct LoadInternal<BLOCK_LOAD_WARP_TRANSPOSE, DUMMY>
+    {
+        enum
+        {
+            WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH)
+        };
+
+        // Assert BLOCK_THREADS must be a multiple of WARP_THREADS
+        CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS");
+
+        // BlockExchange utility type for keys
+        typedef BlockExchange<InputT, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
+
+        /// Shared memory storage layout type
+        struct _TempStorage : BlockExchange::TempStorage
+        {};
+
+        /// Alias wrapper allowing storage to be unioned
+        struct TempStorage : Uninitialized<_TempStorage> {};
+
+        /// Thread reference to shared storage
+        _TempStorage &temp_storage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ LoadInternal(
+            TempStorage &temp_storage,
+            int linear_tid)
+        :
+            temp_storage(temp_storage.Alias()),
+            linear_tid(linear_tid)
+        {}
+
+        /// Load a linear segment of items from memory
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load{
+        {
+            LoadDirectWarpStriped(linear_tid, block_itr, items);
+            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
+        }
+
+        /// Load a linear segment of items from memory, guarded by range
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items)                    ///< [in] Number of valid items to load
+        {
+            LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items);
+            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
+        }
+
+
+        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
+        template <typename InputIteratorT, typename DefaultT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items,                    ///< [in] Number of valid items to load
+            DefaultT        oob_default)                    ///< [in] Default value to assign out-of-bound items
+        {
+            LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items, oob_default);
+            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
+        }
+    };
+
+
+    /**
+     * BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED specialization of load helper
+     */
+    template <int DUMMY>
+    struct LoadInternal<BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED, DUMMY>
+    {
+        enum
+        {
+            WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH)
+        };
+
+        // Assert BLOCK_THREADS must be a multiple of WARP_THREADS
+        CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS");
+
+        // BlockExchange utility type for keys
+        typedef BlockExchange<InputT, BLOCK_DIM_X, ITEMS_PER_THREAD, true, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
+
+        /// Shared memory storage layout type
+        struct _TempStorage : BlockExchange::TempStorage
+        {};
+
+        /// Alias wrapper allowing storage to be unioned
+        struct TempStorage : Uninitialized<_TempStorage> {};
+
+        /// Thread reference to shared storage
+        _TempStorage &temp_storage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ LoadInternal(
+            TempStorage &temp_storage,
+            int linear_tid)
+        :
+            temp_storage(temp_storage.Alias()),
+            linear_tid(linear_tid)
+        {}
+
+        /// Load a linear segment of items from memory
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load{
+        {
+            LoadDirectWarpStriped(linear_tid, block_itr, items);
+            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
+        }
+
+        /// Load a linear segment of items from memory, guarded by range
+        template <typename InputIteratorT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items)                    ///< [in] Number of valid items to load
+        {
+            LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items);
+            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
+        }
+
+
+        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
+        template <typename InputIteratorT, typename DefaultT>
+        __device__ __forceinline__ void Load(
+            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
+            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
+            int             valid_items,                    ///< [in] Number of valid items to load
+            DefaultT        oob_default)                    ///< [in] Default value to assign out-of-bound items
+        {
+            LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items, oob_default);
+            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
+        }
+    };
+
+
+    /******************************************************************************
+     * Type definitions
+     ******************************************************************************/
+
+    /// Internal load implementation to use
+    typedef LoadInternal<ALGORITHM, 0> InternalLoad;
+
+
+    /// Shared memory storage layout type
+    typedef typename InternalLoad::TempStorage _TempStorage;
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Thread reference to shared storage
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    int linear_tid;
+
+public:
+
+    /// \smemstorage{BlockLoad}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockLoad()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockLoad(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Data movement
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Load a linear segment of items from memory.
+     *
+     * \par
+     * - \blocked
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the loading of a linear
+     * segment of 512 integers into a "blocked" arrangement across 128 threads where each
+     * thread owns 4 consecutive items.  The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE,
+     * meaning memory references are efficiently coalesced using a warp-striped access
+     * pattern (after which items are locally reordered among threads).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_load.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, ...)
+     * {
+     *     // Specialize BlockLoad for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockLoad<int, 128, 4, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoad;
+     *
+     *     // Allocate shared memory for BlockLoad
+     *     __shared__ typename BlockLoad::TempStorage temp_storage;
+     *
+     *     // Load a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     BlockLoad(temp_storage).Load(d_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>0, 1, 2, 3, 4, 5, ...</tt>.
+     * The set of \p thread_data across the block of threads in those threads will be
+     * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
+     *
+     */
+    template <typename InputIteratorT>
+    __device__ __forceinline__ void Load(
+        InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+        InputT          (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
+    {
+        InternalLoad(temp_storage, linear_tid).Load(block_itr, items);
+    }
+
+
+    /**
+     * \brief Load a linear segment of items from memory, guarded by range.
+     *
+     * \par
+     * - \blocked
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the guarded loading of a linear
+     * segment of 512 integers into a "blocked" arrangement across 128 threads where each
+     * thread owns 4 consecutive items.  The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE,
+     * meaning memory references are efficiently coalesced using a warp-striped access
+     * pattern (after which items are locally reordered among threads).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_load.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, int valid_items, ...)
+     * {
+     *     // Specialize BlockLoad for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockLoad<int, 128, 4, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoad;
+     *
+     *     // Allocate shared memory for BlockLoad
+     *     __shared__ typename BlockLoad::TempStorage temp_storage;
+     *
+     *     // Load a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     BlockLoad(temp_storage).Load(d_data, thread_data, valid_items);
+     *
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>0, 1, 2, 3, 4, 5, 6...</tt> and \p valid_items is \p 5.
+     * The set of \p thread_data across the block of threads in those threads will be
+     * <tt>{ [0,1,2,3], [4,?,?,?], ..., [?,?,?,?] }</tt>, with only the first two threads
+     * being unmasked to load portions of valid data (and other items remaining unassigned).
+     *
+     */
+    template <typename InputIteratorT>
+    __device__ __forceinline__ void Load(
+        InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+        InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
+        int             valid_items)                ///< [in] Number of valid items to load
+    {
+        InternalLoad(temp_storage, linear_tid).Load(block_itr, items, valid_items);
+    }
+
+
+    /**
+     * \brief Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
+     *
+     * \par
+     * - \blocked
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the guarded loading of a linear
+     * segment of 512 integers into a "blocked" arrangement across 128 threads where each
+     * thread owns 4 consecutive items.  The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE,
+     * meaning memory references are efficiently coalesced using a warp-striped access
+     * pattern (after which items are locally reordered among threads).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_load.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, int valid_items, ...)
+     * {
+     *     // Specialize BlockLoad for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockLoad<int, 128, 4, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoad;
+     *
+     *     // Allocate shared memory for BlockLoad
+     *     __shared__ typename BlockLoad::TempStorage temp_storage;
+     *
+     *     // Load a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     BlockLoad(temp_storage).Load(d_data, thread_data, valid_items, -1);
+     *
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>0, 1, 2, 3, 4, 5, 6...</tt>,
+     * \p valid_items is \p 5, and the out-of-bounds default is \p -1.
+     * The set of \p thread_data across the block of threads in those threads will be
+     * <tt>{ [0,1,2,3], [4,-1,-1,-1], ..., [-1,-1,-1,-1] }</tt>, with only the first two threads
+     * being unmasked to load portions of valid data (and other items are assigned \p -1)
+     *
+     */
+    template <typename InputIteratorT, typename DefaultT>
+    __device__ __forceinline__ void Load(
+        InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
+        InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
+        int             valid_items,                ///< [in] Number of valid items to load
+        DefaultT        oob_default)                ///< [in] Default value to assign out-of-bound items
+    {
+        InternalLoad(temp_storage, linear_tid).Load(block_itr, items, valid_items, oob_default);
+    }
+
+
+    //@}  end member group
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_radix_rank.cuh b/third_party/cub/cub/block/block_radix_rank.cuh
new file mode 100644
index 0000000..c26451c
--- /dev/null
+++ b/third_party/cub/cub/block/block_radix_rank.cuh
@@ -0,0 +1,696 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::BlockRadixRank provides operations for ranking unsigned integer types within a CUDA thread block
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include "../thread/thread_reduce.cuh"
+#include "../thread/thread_scan.cuh"
+#include "../block/block_scan.cuh"
+#include "../util_ptx.cuh"
+#include "../util_arch.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief BlockRadixRank provides operations for ranking unsigned integer types within a CUDA thread block.
+ * \ingroup BlockModule
+ *
+ * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
+ * \tparam RADIX_BITS           The number of radix bits per digit place
+ * \tparam IS_DESCENDING           Whether or not the sorted-order is high-to-low
+ * \tparam MEMOIZE_OUTER_SCAN   <b>[optional]</b> Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure (default: true for architectures SM35 and newer, false otherwise).  See BlockScanAlgorithm::BLOCK_SCAN_RAKING_MEMOIZE for more details.
+ * \tparam INNER_SCAN_ALGORITHM <b>[optional]</b> The cub::BlockScanAlgorithm algorithm to use (default: cub::BLOCK_SCAN_WARP_SCANS)
+ * \tparam SMEM_CONFIG          <b>[optional]</b> Shared memory bank mode (default: \p cudaSharedMemBankSizeFourByte)
+ * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * Blah...
+ * - Keys must be in a form suitable for radix ranking (i.e., unsigned bits).
+ * - \blocked
+ *
+ * \par Performance Considerations
+ * - \granularity
+ *
+ * \par Examples
+ * \par
+ * - <b>Example 1:</b> Simple radix rank of 32-bit integer keys
+ *      \code
+ *      #include <cub/cub.cuh>
+ *
+ *      template <int BLOCK_THREADS>
+ *      __global__ void ExampleKernel(...)
+ *      {
+ *
+ *      \endcode
+ */
+template <
+    int                     BLOCK_DIM_X,
+    int                     RADIX_BITS,
+    bool                    IS_DESCENDING,
+    bool                    MEMOIZE_OUTER_SCAN      = (CUB_PTX_ARCH >= 350) ? true : false,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM    = BLOCK_SCAN_WARP_SCANS,
+    cudaSharedMemConfig     SMEM_CONFIG             = cudaSharedMemBankSizeFourByte,
+    int                     BLOCK_DIM_Y             = 1,
+    int                     BLOCK_DIM_Z             = 1,
+    int                     PTX_ARCH                = CUB_PTX_ARCH>
+class BlockRadixRank
+{
+private:
+
+    /******************************************************************************
+     * Type definitions and constants
+     ******************************************************************************/
+
+    // Integer type for digit counters (to be packed into words of type PackedCounters)
+    typedef unsigned short DigitCounter;
+
+    // Integer type for packing DigitCounters into columns of shared memory banks
+    typedef typename If<(SMEM_CONFIG == cudaSharedMemBankSizeEightByte),
+        unsigned long long,
+        unsigned int>::Type PackedCounter;
+
+    enum
+    {
+        // The thread block size in threads
+        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        RADIX_DIGITS                = 1 << RADIX_BITS,
+
+        LOG_WARP_THREADS            = CUB_LOG_WARP_THREADS(PTX_ARCH),
+        WARP_THREADS                = 1 << LOG_WARP_THREADS,
+        WARPS                       = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+
+        BYTES_PER_COUNTER           = sizeof(DigitCounter),
+        LOG_BYTES_PER_COUNTER       = Log2<BYTES_PER_COUNTER>::VALUE,
+
+        PACKING_RATIO               = sizeof(PackedCounter) / sizeof(DigitCounter),
+        LOG_PACKING_RATIO           = Log2<PACKING_RATIO>::VALUE,
+
+        LOG_COUNTER_LANES           = CUB_MAX((RADIX_BITS - LOG_PACKING_RATIO), 0),                // Always at least one lane
+        COUNTER_LANES               = 1 << LOG_COUNTER_LANES,
+
+        // The number of packed counters per thread (plus one for padding)
+        PADDED_COUNTER_LANES        = COUNTER_LANES + 1,
+        RAKING_SEGMENT              = PADDED_COUNTER_LANES,
+    };
+
+public:
+
+    enum
+    {
+        /// Number of bin-starting offsets tracked per thread
+        BINS_TRACKED_PER_THREAD = CUB_MAX(1, (RADIX_DIGITS + BLOCK_THREADS - 1) / BLOCK_THREADS),
+    };
+
+private:
+
+
+    /// BlockScan type
+    typedef BlockScan<
+            PackedCounter,
+            BLOCK_DIM_X,
+            INNER_SCAN_ALGORITHM,
+            BLOCK_DIM_Y,
+            BLOCK_DIM_Z,
+            PTX_ARCH>
+        BlockScan;
+
+
+    /// Shared memory storage layout type for BlockRadixRank
+    struct __align__(16) _TempStorage
+    {
+        union Aliasable
+        {
+            DigitCounter            digit_counters[PADDED_COUNTER_LANES][BLOCK_THREADS][PACKING_RATIO];
+            PackedCounter           raking_grid[BLOCK_THREADS][RAKING_SEGMENT];
+
+        } aliasable;
+
+        // Storage for scanning local ranks
+        typename BlockScan::TempStorage block_scan;
+    };
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+    /// Copy of raking segment, promoted to registers
+    PackedCounter cached_segment[RAKING_SEGMENT];
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /**
+     * Internal storage allocator
+     */
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+    /**
+     * Performs upsweep raking reduction, returning the aggregate
+     */
+    __device__ __forceinline__ PackedCounter Upsweep()
+    {
+        PackedCounter *smem_raking_ptr = temp_storage.aliasable.raking_grid[linear_tid];
+        PackedCounter *raking_ptr;
+
+        if (MEMOIZE_OUTER_SCAN)
+        {
+            // Copy data into registers
+            #pragma unroll
+            for (int i = 0; i < RAKING_SEGMENT; i++)
+            {
+                cached_segment[i] = smem_raking_ptr[i];
+            }
+            raking_ptr = cached_segment;
+        }
+        else
+        {
+            raking_ptr = smem_raking_ptr;
+        }
+
+        return internal::ThreadReduce<RAKING_SEGMENT>(raking_ptr, Sum());
+    }
+
+
+    /// Performs exclusive downsweep raking scan
+    __device__ __forceinline__ void ExclusiveDownsweep(
+        PackedCounter raking_partial)
+    {
+        PackedCounter *smem_raking_ptr = temp_storage.aliasable.raking_grid[linear_tid];
+
+        PackedCounter *raking_ptr = (MEMOIZE_OUTER_SCAN) ?
+            cached_segment :
+            smem_raking_ptr;
+
+        // Exclusive raking downsweep scan
+        internal::ThreadScanExclusive<RAKING_SEGMENT>(raking_ptr, raking_ptr, Sum(), raking_partial);
+
+        if (MEMOIZE_OUTER_SCAN)
+        {
+            // Copy data back to smem
+            #pragma unroll
+            for (int i = 0; i < RAKING_SEGMENT; i++)
+            {
+                smem_raking_ptr[i] = cached_segment[i];
+            }
+        }
+    }
+
+
+    /**
+     * Reset shared memory digit counters
+     */
+    __device__ __forceinline__ void ResetCounters()
+    {
+        // Reset shared memory digit counters
+        #pragma unroll
+        for (int LANE = 0; LANE < PADDED_COUNTER_LANES; LANE++)
+        {
+            *((PackedCounter*) temp_storage.aliasable.digit_counters[LANE][linear_tid]) = 0;
+        }
+    }
+
+
+    /**
+     * Block-scan prefix callback
+     */
+    struct PrefixCallBack
+    {
+        __device__ __forceinline__ PackedCounter operator()(PackedCounter block_aggregate)
+        {
+            PackedCounter block_prefix = 0;
+
+            // Propagate totals in packed fields
+            #pragma unroll
+            for (int PACKED = 1; PACKED < PACKING_RATIO; PACKED++)
+            {
+                block_prefix += block_aggregate << (sizeof(DigitCounter) * 8 * PACKED);
+            }
+
+            return block_prefix;
+        }
+    };
+
+
+    /**
+     * Scan shared memory digit counters.
+     */
+    __device__ __forceinline__ void ScanCounters()
+    {
+        // Upsweep scan
+        PackedCounter raking_partial = Upsweep();
+
+        // Compute exclusive sum
+        PackedCounter exclusive_partial;
+        PrefixCallBack prefix_call_back;
+        BlockScan(temp_storage.block_scan).ExclusiveSum(raking_partial, exclusive_partial, prefix_call_back);
+
+        // Downsweep scan with exclusive partial
+        ExclusiveDownsweep(exclusive_partial);
+    }
+
+public:
+
+    /// \smemstorage{BlockScan}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockRadixRank()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockRadixRank(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Raking
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Rank keys.
+     */
+    template <
+        typename        UnsignedBits,
+        int             KEYS_PER_THREAD>
+    __device__ __forceinline__ void RankKeys(
+        UnsignedBits    (&keys)[KEYS_PER_THREAD],           ///< [in] Keys for this tile
+        int             (&ranks)[KEYS_PER_THREAD],          ///< [out] For each key, the local rank within the tile
+        int             current_bit,                        ///< [in] The least-significant bit position of the current digit to extract
+        int             num_bits)                           ///< [in] The number of bits in the current digit
+    {
+        DigitCounter    thread_prefixes[KEYS_PER_THREAD];   // For each key, the count of previous keys in this tile having the same digit
+        DigitCounter*   digit_counters[KEYS_PER_THREAD];    // For each key, the byte-offset of its corresponding digit counter in smem
+
+        // Reset shared memory digit counters
+        ResetCounters();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < KEYS_PER_THREAD; ++ITEM)
+        {
+            // Get digit
+            unsigned int digit = BFE(keys[ITEM], current_bit, num_bits);
+
+            // Get sub-counter
+            unsigned int sub_counter = digit >> LOG_COUNTER_LANES;
+
+            // Get counter lane
+            unsigned int counter_lane = digit & (COUNTER_LANES - 1);
+
+            if (IS_DESCENDING)
+            {
+                sub_counter = PACKING_RATIO - 1 - sub_counter;
+                counter_lane = COUNTER_LANES - 1 - counter_lane;
+            }
+
+            // Pointer to smem digit counter
+            digit_counters[ITEM] = &temp_storage.aliasable.digit_counters[counter_lane][linear_tid][sub_counter];
+
+            // Load thread-exclusive prefix
+            thread_prefixes[ITEM] = *digit_counters[ITEM];
+
+            // Store inclusive prefix
+            *digit_counters[ITEM] = thread_prefixes[ITEM] + 1;
+        }
+
+        CTA_SYNC();
+
+        // Scan shared memory counters
+        ScanCounters();
+
+        CTA_SYNC();
+
+        // Extract the local ranks of each key
+        for (int ITEM = 0; ITEM < KEYS_PER_THREAD; ++ITEM)
+        {
+            // Add in thread block exclusive prefix
+            ranks[ITEM] = thread_prefixes[ITEM] + *digit_counters[ITEM];
+        }
+    }
+
+
+    /**
+     * \brief Rank keys.  For the lower \p RADIX_DIGITS threads, digit counts for each digit are provided for the corresponding thread.
+     */
+    template <
+        typename        UnsignedBits,
+        int             KEYS_PER_THREAD>
+    __device__ __forceinline__ void RankKeys(
+        UnsignedBits    (&keys)[KEYS_PER_THREAD],           ///< [in] Keys for this tile
+        int             (&ranks)[KEYS_PER_THREAD],          ///< [out] For each key, the local rank within the tile (out parameter)
+        int             current_bit,                        ///< [in] The least-significant bit position of the current digit to extract
+        int             num_bits,                           ///< [in] The number of bits in the current digit
+        int             (&exclusive_digit_prefix)[BINS_TRACKED_PER_THREAD])            ///< [out] The exclusive prefix sum for the digits [(threadIdx.x * BINS_TRACKED_PER_THREAD) ... (threadIdx.x * BINS_TRACKED_PER_THREAD) + BINS_TRACKED_PER_THREAD - 1]
+    {
+        // Rank keys
+        RankKeys(keys, ranks, current_bit, num_bits);
+
+        // Get the inclusive and exclusive digit totals corresponding to the calling thread.
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (linear_tid * BINS_TRACKED_PER_THREAD) + track;
+
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+            {
+                if (IS_DESCENDING)
+                    bin_idx = RADIX_DIGITS - bin_idx - 1;
+
+                // Obtain ex/inclusive digit counts.  (Unfortunately these all reside in the
+                // first counter column, resulting in unavoidable bank conflicts.)
+                unsigned int counter_lane   = (bin_idx & (COUNTER_LANES - 1));
+                unsigned int sub_counter    = bin_idx >> (LOG_COUNTER_LANES);
+
+                exclusive_digit_prefix[track] = temp_storage.aliasable.digit_counters[counter_lane][0][sub_counter];
+            }
+        }
+    }
+};
+
+
+
+
+
+/**
+ * Radix-rank using match.any
+ */
+template <
+    int                     BLOCK_DIM_X,
+    int                     RADIX_BITS,
+    bool                    IS_DESCENDING,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM    = BLOCK_SCAN_WARP_SCANS,
+    int                     BLOCK_DIM_Y             = 1,
+    int                     BLOCK_DIM_Z             = 1,
+    int                     PTX_ARCH                = CUB_PTX_ARCH>
+class BlockRadixRankMatch
+{
+private:
+
+    /******************************************************************************
+     * Type definitions and constants
+     ******************************************************************************/
+
+    typedef int32_t    RankT;
+    typedef int32_t    DigitCounterT;
+
+    enum
+    {
+        // The thread block size in threads
+        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        RADIX_DIGITS                = 1 << RADIX_BITS,
+
+        LOG_WARP_THREADS            = CUB_LOG_WARP_THREADS(PTX_ARCH),
+        WARP_THREADS                = 1 << LOG_WARP_THREADS,
+        WARPS                       = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+
+        PADDED_WARPS            = ((WARPS & 0x1) == 0) ?
+                                    WARPS + 1 :
+                                    WARPS,
+
+        COUNTERS                = PADDED_WARPS * RADIX_DIGITS,
+        RAKING_SEGMENT          = (COUNTERS + BLOCK_THREADS - 1) / BLOCK_THREADS,
+        PADDED_RAKING_SEGMENT   = ((RAKING_SEGMENT & 0x1) == 0) ?
+                                    RAKING_SEGMENT + 1 :
+                                    RAKING_SEGMENT,
+    };
+
+public:
+
+    enum
+    {
+        /// Number of bin-starting offsets tracked per thread
+        BINS_TRACKED_PER_THREAD = CUB_MAX(1, (RADIX_DIGITS + BLOCK_THREADS - 1) / BLOCK_THREADS),
+    };
+
+private:
+
+    /// BlockScan type
+    typedef BlockScan<
+            DigitCounterT,
+            BLOCK_THREADS,
+            INNER_SCAN_ALGORITHM,
+            BLOCK_DIM_Y,
+            BLOCK_DIM_Z,
+            PTX_ARCH>
+        BlockScanT;
+
+
+    /// Shared memory storage layout type for BlockRadixRank
+    struct __align__(16) _TempStorage
+    {
+        typename BlockScanT::TempStorage            block_scan;
+
+        union __align__(16) Aliasable
+        {
+            volatile DigitCounterT                  warp_digit_counters[RADIX_DIGITS][PADDED_WARPS];
+            DigitCounterT                           raking_grid[BLOCK_THREADS][PADDED_RAKING_SEGMENT];
+
+        } aliasable;
+    };
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+
+
+public:
+
+    /// \smemstorage{BlockScan}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockRadixRankMatch(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Raking
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Rank keys.
+     */
+    template <
+        typename        UnsignedBits,
+        int             KEYS_PER_THREAD>
+    __device__ __forceinline__ void RankKeys(
+        UnsignedBits    (&keys)[KEYS_PER_THREAD],           ///< [in] Keys for this tile
+        int             (&ranks)[KEYS_PER_THREAD],          ///< [out] For each key, the local rank within the tile
+        int             current_bit,                        ///< [in] The least-significant bit position of the current digit to extract
+        int             num_bits)                           ///< [in] The number of bits in the current digit
+    {
+        // Initialize shared digit counters
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < PADDED_RAKING_SEGMENT; ++ITEM)
+            temp_storage.aliasable.raking_grid[linear_tid][ITEM] = 0;
+
+        CTA_SYNC();
+
+        // Each warp will strip-mine its section of input, one strip at a time
+
+        volatile DigitCounterT  *digit_counters[KEYS_PER_THREAD];
+        uint32_t                warp_id         = linear_tid >> LOG_WARP_THREADS;
+        uint32_t                lane_mask_lt    = LaneMaskLt();
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < KEYS_PER_THREAD; ++ITEM)
+        {
+            // My digit
+            uint32_t digit = BFE(keys[ITEM], current_bit, num_bits);
+
+            if (IS_DESCENDING)
+                digit = RADIX_DIGITS - digit - 1;
+
+            // Mask of peers who have same digit as me
+            uint32_t peer_mask = MatchAny<RADIX_BITS>(digit);
+
+            // Pointer to smem digit counter for this key
+            digit_counters[ITEM] = &temp_storage.aliasable.warp_digit_counters[digit][warp_id];
+
+            // Number of occurrences in previous strips
+            DigitCounterT warp_digit_prefix = *digit_counters[ITEM];
+
+            // Warp-sync
+            WARP_SYNC(0xFFFFFFFF);
+
+            // Number of peers having same digit as me
+            int32_t digit_count = __popc(peer_mask);
+
+            // Number of lower-ranked peers having same digit seen so far
+            int32_t peer_digit_prefix = __popc(peer_mask & lane_mask_lt);
+
+            if (peer_digit_prefix == 0)
+            {
+                // First thread for each digit updates the shared warp counter
+                *digit_counters[ITEM] = DigitCounterT(warp_digit_prefix + digit_count);
+            }
+
+            // Warp-sync
+            WARP_SYNC(0xFFFFFFFF);
+
+            // Number of prior keys having same digit
+            ranks[ITEM] = warp_digit_prefix + DigitCounterT(peer_digit_prefix);
+        }
+
+        CTA_SYNC();
+
+        // Scan warp counters
+
+        DigitCounterT scan_counters[PADDED_RAKING_SEGMENT];
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < PADDED_RAKING_SEGMENT; ++ITEM)
+            scan_counters[ITEM] = temp_storage.aliasable.raking_grid[linear_tid][ITEM];
+
+        BlockScanT(temp_storage.block_scan).ExclusiveSum(scan_counters, scan_counters);
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < PADDED_RAKING_SEGMENT; ++ITEM)
+            temp_storage.aliasable.raking_grid[linear_tid][ITEM] = scan_counters[ITEM];
+
+        CTA_SYNC();
+
+        // Seed ranks with counter values from previous warps
+        #pragma unroll
+        for (int ITEM = 0; ITEM < KEYS_PER_THREAD; ++ITEM)
+            ranks[ITEM] += *digit_counters[ITEM];
+    }
+
+
+    /**
+     * \brief Rank keys.  For the lower \p RADIX_DIGITS threads, digit counts for each digit are provided for the corresponding thread.
+     */
+    template <
+        typename        UnsignedBits,
+        int             KEYS_PER_THREAD>
+    __device__ __forceinline__ void RankKeys(
+        UnsignedBits    (&keys)[KEYS_PER_THREAD],           ///< [in] Keys for this tile
+        int             (&ranks)[KEYS_PER_THREAD],          ///< [out] For each key, the local rank within the tile (out parameter)
+        int             current_bit,                        ///< [in] The least-significant bit position of the current digit to extract
+        int             num_bits,                           ///< [in] The number of bits in the current digit
+        int             (&exclusive_digit_prefix)[BINS_TRACKED_PER_THREAD])            ///< [out] The exclusive prefix sum for the digits [(threadIdx.x * BINS_TRACKED_PER_THREAD) ... (threadIdx.x * BINS_TRACKED_PER_THREAD) + BINS_TRACKED_PER_THREAD - 1]
+    {
+        RankKeys(keys, ranks, current_bit, num_bits);
+
+        // Get exclusive count for each digit
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (linear_tid * BINS_TRACKED_PER_THREAD) + track;
+
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+            {
+                if (IS_DESCENDING)
+                    bin_idx = RADIX_DIGITS - bin_idx - 1;
+
+                exclusive_digit_prefix[track] = temp_storage.aliasable.warp_digit_counters[bin_idx][0];
+            }
+        }
+    }
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/block/block_radix_sort.cuh b/third_party/cub/cub/block/block_radix_sort.cuh
new file mode 100644
index 0000000..ac0c9f8
--- /dev/null
+++ b/third_party/cub/cub/block/block_radix_sort.cuh
@@ -0,0 +1,863 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockRadixSort class provides [<em>collective</em>](index.html#sec0) methods for radix sorting of items partitioned across a CUDA thread block.
+ */
+
+
+#pragma once
+
+#include "block_exchange.cuh"
+#include "block_radix_rank.cuh"
+#include "../util_ptx.cuh"
+#include "../util_arch.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief The BlockRadixSort class provides [<em>collective</em>](index.html#sec0) methods for sorting items partitioned across a CUDA thread block using a radix sorting method.  ![](sorting_logo.png)
+ * \ingroup BlockModule
+ *
+ * \tparam KeyT                 KeyT type
+ * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
+ * \tparam ITEMS_PER_THREAD     The number of items per thread
+ * \tparam ValueT               <b>[optional]</b> ValueT type (default: cub::NullType, which indicates a keys-only sort)
+ * \tparam RADIX_BITS           <b>[optional]</b> The number of radix bits per digit place (default: 4 bits)
+ * \tparam MEMOIZE_OUTER_SCAN   <b>[optional]</b> Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure (default: true for architectures SM35 and newer, false otherwise).
+ * \tparam INNER_SCAN_ALGORITHM <b>[optional]</b> The cub::BlockScanAlgorithm algorithm to use (default: cub::BLOCK_SCAN_WARP_SCANS)
+ * \tparam SMEM_CONFIG          <b>[optional]</b> Shared memory bank mode (default: \p cudaSharedMemBankSizeFourByte)
+ * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - The [<em>radix sorting method</em>](http://en.wikipedia.org/wiki/Radix_sort) arranges
+ *   items into ascending order.  It relies upon a positional representation for
+ *   keys, i.e., each key is comprised of an ordered sequence of symbols (e.g., digits,
+ *   characters, etc.) specified from least-significant to most-significant.  For a
+ *   given input sequence of keys and a set of rules specifying a total ordering
+ *   of the symbolic alphabet, the radix sorting method produces a lexicographic
+ *   ordering of those keys.
+ * - BlockRadixSort can sort all of the built-in C++ numeric primitive types
+ *   (<tt>unsigned char</tt>, \p int, \p double, etc.) as well as CUDA's \p __half
+ *   half-precision floating-point type. Within each key, the implementation treats fixed-length
+ *   bit-sequences of \p RADIX_BITS as radix digit places.  Although the direct radix sorting
+ *   method can only be applied to unsigned integral types, BlockRadixSort
+ *   is able to sort signed and floating-point types via simple bit-wise transformations
+ *   that ensure lexicographic key ordering.
+ * - \rowmajor
+ *
+ * \par Performance Considerations
+ * - \granularity
+ *
+ * \par A Simple Example
+ * \blockcollective{BlockRadixSort}
+ * \par
+ * The code snippet below illustrates a sort of 512 integer keys that
+ * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+ * where each thread owns 4 consecutive items.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer items each
+ *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
+ *
+ *     // Allocate shared memory for BlockRadixSort
+ *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+ *
+ *     // Obtain a segment of consecutive items that are blocked across threads
+ *     int thread_keys[4];
+ *     ...
+ *
+ *     // Collectively sort the keys
+ *     BlockRadixSort(temp_storage).Sort(thread_keys);
+ *
+ *     ...
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_keys across the block of threads is
+ * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
+ * corresponding output \p thread_keys in those threads will be
+ * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
+ *
+ */
+template <
+    typename                KeyT,
+    int                     BLOCK_DIM_X,
+    int                     ITEMS_PER_THREAD,
+    typename                ValueT                   = NullType,
+    int                     RADIX_BITS              = 4,
+    bool                    MEMOIZE_OUTER_SCAN      = (CUB_PTX_ARCH >= 350) ? true : false,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM    = BLOCK_SCAN_WARP_SCANS,
+    cudaSharedMemConfig     SMEM_CONFIG             = cudaSharedMemBankSizeFourByte,
+    int                     BLOCK_DIM_Y             = 1,
+    int                     BLOCK_DIM_Z             = 1,
+    int                     PTX_ARCH                = CUB_PTX_ARCH>
+class BlockRadixSort
+{
+private:
+
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    enum
+    {
+        // The thread block size in threads
+        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        // Whether or not there are values to be trucked along with keys
+        KEYS_ONLY                   = Equals<ValueT, NullType>::VALUE,
+    };
+
+    // KeyT traits and unsigned bits type
+    typedef Traits<KeyT>                        KeyTraits;
+    typedef typename KeyTraits::UnsignedBits    UnsignedBits;
+
+    /// Ascending BlockRadixRank utility type
+    typedef BlockRadixRank<
+            BLOCK_DIM_X,
+            RADIX_BITS,
+            false,
+            MEMOIZE_OUTER_SCAN,
+            INNER_SCAN_ALGORITHM,
+            SMEM_CONFIG,
+            BLOCK_DIM_Y,
+            BLOCK_DIM_Z,
+            PTX_ARCH>
+        AscendingBlockRadixRank;
+
+    /// Descending BlockRadixRank utility type
+    typedef BlockRadixRank<
+            BLOCK_DIM_X,
+            RADIX_BITS,
+            true,
+            MEMOIZE_OUTER_SCAN,
+            INNER_SCAN_ALGORITHM,
+            SMEM_CONFIG,
+            BLOCK_DIM_Y,
+            BLOCK_DIM_Z,
+            PTX_ARCH>
+        DescendingBlockRadixRank;
+
+    /// BlockExchange utility type for keys
+    typedef BlockExchange<KeyT, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchangeKeys;
+
+    /// BlockExchange utility type for values
+    typedef BlockExchange<ValueT, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchangeValues;
+
+    /// Shared memory storage layout type
+    union _TempStorage
+    {
+        typename AscendingBlockRadixRank::TempStorage  asending_ranking_storage;
+        typename DescendingBlockRadixRank::TempStorage descending_ranking_storage;
+        typename BlockExchangeKeys::TempStorage        exchange_keys;
+        typename BlockExchangeValues::TempStorage      exchange_values;
+    };
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+    /// Rank keys (specialized for ascending sort)
+    __device__ __forceinline__ void RankKeys(
+        UnsignedBits    (&unsigned_keys)[ITEMS_PER_THREAD],
+        int             (&ranks)[ITEMS_PER_THREAD],
+        int             begin_bit,
+        int             pass_bits,
+        Int2Type<false> /*is_descending*/)
+    {
+        AscendingBlockRadixRank(temp_storage.asending_ranking_storage).RankKeys(
+            unsigned_keys,
+            ranks,
+            begin_bit,
+            pass_bits);
+    }
+
+    /// Rank keys (specialized for descending sort)
+    __device__ __forceinline__ void RankKeys(
+        UnsignedBits    (&unsigned_keys)[ITEMS_PER_THREAD],
+        int             (&ranks)[ITEMS_PER_THREAD],
+        int             begin_bit,
+        int             pass_bits,
+        Int2Type<true>  /*is_descending*/)
+    {
+        DescendingBlockRadixRank(temp_storage.descending_ranking_storage).RankKeys(
+            unsigned_keys,
+            ranks,
+            begin_bit,
+            pass_bits);
+    }
+
+    /// ExchangeValues (specialized for key-value sort, to-blocked arrangement)
+    __device__ __forceinline__ void ExchangeValues(
+        ValueT          (&values)[ITEMS_PER_THREAD],
+        int             (&ranks)[ITEMS_PER_THREAD],
+        Int2Type<false> /*is_keys_only*/,
+        Int2Type<true>  /*is_blocked*/)
+    {
+        CTA_SYNC();
+
+        // Exchange values through shared memory in blocked arrangement
+        BlockExchangeValues(temp_storage.exchange_values).ScatterToBlocked(values, ranks);
+    }
+
+    /// ExchangeValues (specialized for key-value sort, to-striped arrangement)
+    __device__ __forceinline__ void ExchangeValues(
+        ValueT          (&values)[ITEMS_PER_THREAD],
+        int             (&ranks)[ITEMS_PER_THREAD],
+        Int2Type<false> /*is_keys_only*/,
+        Int2Type<false> /*is_blocked*/)
+    {
+        CTA_SYNC();
+
+        // Exchange values through shared memory in blocked arrangement
+        BlockExchangeValues(temp_storage.exchange_values).ScatterToStriped(values, ranks);
+    }
+
+    /// ExchangeValues (specialized for keys-only sort)
+    template <int IS_BLOCKED>
+    __device__ __forceinline__ void ExchangeValues(
+        ValueT                  (&/*values*/)[ITEMS_PER_THREAD],
+        int                     (&/*ranks*/)[ITEMS_PER_THREAD],
+        Int2Type<true>          /*is_keys_only*/,
+        Int2Type<IS_BLOCKED>    /*is_blocked*/)
+    {}
+
+    /// Sort blocked arrangement
+    template <int DESCENDING, int KEYS_ONLY>
+    __device__ __forceinline__ void SortBlocked(
+        KeyT                    (&keys)[ITEMS_PER_THREAD],          ///< Keys to sort
+        ValueT                  (&values)[ITEMS_PER_THREAD],        ///< Values to sort
+        int                     begin_bit,                          ///< The beginning (least-significant) bit index needed for key comparison
+        int                     end_bit,                            ///< The past-the-end (most-significant) bit index needed for key comparison
+        Int2Type<DESCENDING>    is_descending,                      ///< Tag whether is a descending-order sort
+        Int2Type<KEYS_ONLY>     is_keys_only)                       ///< Tag whether is keys-only sort
+    {
+        UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD] =
+            reinterpret_cast<UnsignedBits (&)[ITEMS_PER_THREAD]>(keys);
+
+        // Twiddle bits if necessary
+        #pragma unroll
+        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
+        {
+            unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]);
+        }
+
+        // Radix sorting passes
+        while (true)
+        {
+            int pass_bits = CUB_MIN(RADIX_BITS, end_bit - begin_bit);
+
+            // Rank the blocked keys
+            int ranks[ITEMS_PER_THREAD];
+            RankKeys(unsigned_keys, ranks, begin_bit, pass_bits, is_descending);
+            begin_bit += RADIX_BITS;
+
+            CTA_SYNC();
+
+            // Exchange keys through shared memory in blocked arrangement
+            BlockExchangeKeys(temp_storage.exchange_keys).ScatterToBlocked(keys, ranks);
+
+            // Exchange values through shared memory in blocked arrangement
+            ExchangeValues(values, ranks, is_keys_only, Int2Type<true>());
+
+            // Quit if done
+            if (begin_bit >= end_bit) break;
+
+            CTA_SYNC();
+        }
+
+        // Untwiddle bits if necessary
+        #pragma unroll
+        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
+        {
+            unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]);
+        }
+    }
+
+public:
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+    /// Sort blocked -> striped arrangement
+    template <int DESCENDING, int KEYS_ONLY>
+    __device__ __forceinline__ void SortBlockedToStriped(
+        KeyT                    (&keys)[ITEMS_PER_THREAD],          ///< Keys to sort
+        ValueT                  (&values)[ITEMS_PER_THREAD],        ///< Values to sort
+        int                     begin_bit,                          ///< The beginning (least-significant) bit index needed for key comparison
+        int                     end_bit,                            ///< The past-the-end (most-significant) bit index needed for key comparison
+        Int2Type<DESCENDING>    is_descending,                      ///< Tag whether is a descending-order sort
+        Int2Type<KEYS_ONLY>     is_keys_only)                       ///< Tag whether is keys-only sort
+    {
+        UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD] =
+            reinterpret_cast<UnsignedBits (&)[ITEMS_PER_THREAD]>(keys);
+
+        // Twiddle bits if necessary
+        #pragma unroll
+        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
+        {
+            unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]);
+        }
+
+        // Radix sorting passes
+        while (true)
+        {
+            int pass_bits = CUB_MIN(RADIX_BITS, end_bit - begin_bit);
+
+            // Rank the blocked keys
+            int ranks[ITEMS_PER_THREAD];
+            RankKeys(unsigned_keys, ranks, begin_bit, pass_bits, is_descending);
+            begin_bit += RADIX_BITS;
+
+            CTA_SYNC();
+
+            // Check if this is the last pass
+            if (begin_bit >= end_bit)
+            {
+                // Last pass exchanges keys through shared memory in striped arrangement
+                BlockExchangeKeys(temp_storage.exchange_keys).ScatterToStriped(keys, ranks);
+
+                // Last pass exchanges through shared memory in striped arrangement
+                ExchangeValues(values, ranks, is_keys_only, Int2Type<false>());
+
+                // Quit
+                break;
+            }
+
+            // Exchange keys through shared memory in blocked arrangement
+            BlockExchangeKeys(temp_storage.exchange_keys).ScatterToBlocked(keys, ranks);
+
+            // Exchange values through shared memory in blocked arrangement
+            ExchangeValues(values, ranks, is_keys_only, Int2Type<true>());
+
+            CTA_SYNC();
+        }
+
+        // Untwiddle bits if necessary
+        #pragma unroll
+        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
+        {
+            unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]);
+        }
+    }
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+    /// \smemstorage{BlockRadixSort}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockRadixSort()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockRadixSort(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Sorting (blocked arrangements)
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Performs an ascending block-wide radix sort over a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys.
+     *
+     * \par
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sort of 512 integer keys that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys each
+     *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
+     *
+     *     // Allocate shared memory for BlockRadixSort
+     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_keys[4];
+     *     ...
+     *
+     *     // Collectively sort the keys
+     *     BlockRadixSort(temp_storage).Sort(thread_keys);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_keys across the block of threads is
+     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.
+     * The corresponding output \p thread_keys in those threads will be
+     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
+     */
+    __device__ __forceinline__ void Sort(
+        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
+        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
+        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
+    {
+        NullType values[ITEMS_PER_THREAD];
+
+        SortBlocked(keys, values, begin_bit, end_bit, Int2Type<false>(), Int2Type<KEYS_ONLY>());
+    }
+
+
+    /**
+     * \brief Performs an ascending block-wide radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys and values.
+     *
+     * \par
+     * - BlockRadixSort can only accommodate one associated tile of values. To "truck along"
+     *   more than one tile of values, simply perform a key-value sort of the keys paired
+     *   with a temporary value array that enumerates the key indices.  The reordered indices
+     *   can then be used as a gather-vector for exchanging other associated tile data through
+     *   shared memory.
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sort of 512 integer keys and values that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive pairs.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys and values each
+     *     typedef cub::BlockRadixSort<int, 128, 4, int> BlockRadixSort;
+     *
+     *     // Allocate shared memory for BlockRadixSort
+     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_keys[4];
+     *     int thread_values[4];
+     *     ...
+     *
+     *     // Collectively sort the keys and values among block threads
+     *     BlockRadixSort(temp_storage).Sort(thread_keys, thread_values);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_keys across the block of threads is
+     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
+     * corresponding output \p thread_keys in those threads will be
+     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
+     *
+     */
+    __device__ __forceinline__ void Sort(
+        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
+        ValueT  (&values)[ITEMS_PER_THREAD],        ///< [in-out] Values to sort
+        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
+        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
+    {
+        SortBlocked(keys, values, begin_bit, end_bit, Int2Type<false>(), Int2Type<KEYS_ONLY>());
+    }
+
+    /**
+     * \brief Performs a descending block-wide radix sort over a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys.
+     *
+     * \par
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sort of 512 integer keys that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys each
+     *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
+     *
+     *     // Allocate shared memory for BlockRadixSort
+     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_keys[4];
+     *     ...
+     *
+     *     // Collectively sort the keys
+     *     BlockRadixSort(temp_storage).Sort(thread_keys);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_keys across the block of threads is
+     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.
+     * The corresponding output \p thread_keys in those threads will be
+     * <tt>{ [511,510,509,508], [11,10,9,8], [7,6,5,4], ..., [3,2,1,0] }</tt>.
+     */
+    __device__ __forceinline__ void SortDescending(
+        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
+        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
+        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
+    {
+        NullType values[ITEMS_PER_THREAD];
+
+        SortBlocked(keys, values, begin_bit, end_bit, Int2Type<true>(), Int2Type<KEYS_ONLY>());
+    }
+
+
+    /**
+     * \brief Performs a descending block-wide radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys and values.
+     *
+     * \par
+     * - BlockRadixSort can only accommodate one associated tile of values. To "truck along"
+     *   more than one tile of values, simply perform a key-value sort of the keys paired
+     *   with a temporary value array that enumerates the key indices.  The reordered indices
+     *   can then be used as a gather-vector for exchanging other associated tile data through
+     *   shared memory.
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sort of 512 integer keys and values that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive pairs.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys and values each
+     *     typedef cub::BlockRadixSort<int, 128, 4, int> BlockRadixSort;
+     *
+     *     // Allocate shared memory for BlockRadixSort
+     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_keys[4];
+     *     int thread_values[4];
+     *     ...
+     *
+     *     // Collectively sort the keys and values among block threads
+     *     BlockRadixSort(temp_storage).Sort(thread_keys, thread_values);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_keys across the block of threads is
+     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
+     * corresponding output \p thread_keys in those threads will be
+     * <tt>{ [511,510,509,508], [11,10,9,8], [7,6,5,4], ..., [3,2,1,0] }</tt>.
+     *
+     */
+    __device__ __forceinline__ void SortDescending(
+        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
+        ValueT  (&values)[ITEMS_PER_THREAD],        ///< [in-out] Values to sort
+        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
+        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
+    {
+        SortBlocked(keys, values, begin_bit, end_bit, Int2Type<true>(), Int2Type<KEYS_ONLY>());
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Sorting (blocked arrangement -> striped arrangement)
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Performs an ascending radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys, leaving them in a [<em>striped arrangement</em>](index.html#sec5sec3).
+     *
+     * \par
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sort of 512 integer keys that
+     * are initially partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive keys.  The final partitioning is striped.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys each
+     *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
+     *
+     *     // Allocate shared memory for BlockRadixSort
+     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_keys[4];
+     *     ...
+     *
+     *     // Collectively sort the keys
+     *     BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_keys across the block of threads is
+     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
+     * corresponding output \p thread_keys in those threads will be
+     * <tt>{ [0,128,256,384], [1,129,257,385], [2,130,258,386], ..., [127,255,383,511] }</tt>.
+     *
+     */
+    __device__ __forceinline__ void SortBlockedToStriped(
+        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
+        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
+        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
+    {
+        NullType values[ITEMS_PER_THREAD];
+
+        SortBlockedToStriped(keys, values, begin_bit, end_bit, Int2Type<false>(), Int2Type<KEYS_ONLY>());
+    }
+
+
+    /**
+     * \brief Performs an ascending radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys and values, leaving them in a [<em>striped arrangement</em>](index.html#sec5sec3).
+     *
+     * \par
+     * - BlockRadixSort can only accommodate one associated tile of values. To "truck along"
+     *   more than one tile of values, simply perform a key-value sort of the keys paired
+     *   with a temporary value array that enumerates the key indices.  The reordered indices
+     *   can then be used as a gather-vector for exchanging other associated tile data through
+     *   shared memory.
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sort of 512 integer keys and values that
+     * are initially partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive pairs.  The final partitioning is striped.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys and values each
+     *     typedef cub::BlockRadixSort<int, 128, 4, int> BlockRadixSort;
+     *
+     *     // Allocate shared memory for BlockRadixSort
+     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_keys[4];
+     *     int thread_values[4];
+     *     ...
+     *
+     *     // Collectively sort the keys and values among block threads
+     *     BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys, thread_values);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_keys across the block of threads is
+     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
+     * corresponding output \p thread_keys in those threads will be
+     * <tt>{ [0,128,256,384], [1,129,257,385], [2,130,258,386], ..., [127,255,383,511] }</tt>.
+     *
+     */
+    __device__ __forceinline__ void SortBlockedToStriped(
+        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
+        ValueT  (&values)[ITEMS_PER_THREAD],        ///< [in-out] Values to sort
+        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
+        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
+    {
+        SortBlockedToStriped(keys, values, begin_bit, end_bit, Int2Type<false>(), Int2Type<KEYS_ONLY>());
+    }
+
+
+    /**
+     * \brief Performs a descending radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys, leaving them in a [<em>striped arrangement</em>](index.html#sec5sec3).
+     *
+     * \par
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sort of 512 integer keys that
+     * are initially partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive keys.  The final partitioning is striped.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys each
+     *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
+     *
+     *     // Allocate shared memory for BlockRadixSort
+     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_keys[4];
+     *     ...
+     *
+     *     // Collectively sort the keys
+     *     BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_keys across the block of threads is
+     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
+     * corresponding output \p thread_keys in those threads will be
+     * <tt>{ [511,383,255,127], [386,258,130,2], [385,257,128,1], ..., [384,256,128,0] }</tt>.
+     *
+     */
+    __device__ __forceinline__ void SortDescendingBlockedToStriped(
+        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
+        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
+        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
+    {
+        NullType values[ITEMS_PER_THREAD];
+
+        SortBlockedToStriped(keys, values, begin_bit, end_bit, Int2Type<true>(), Int2Type<KEYS_ONLY>());
+    }
+
+
+    /**
+     * \brief Performs a descending radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys and values, leaving them in a [<em>striped arrangement</em>](index.html#sec5sec3).
+     *
+     * \par
+     * - BlockRadixSort can only accommodate one associated tile of values. To "truck along"
+     *   more than one tile of values, simply perform a key-value sort of the keys paired
+     *   with a temporary value array that enumerates the key indices.  The reordered indices
+     *   can then be used as a gather-vector for exchanging other associated tile data through
+     *   shared memory.
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sort of 512 integer keys and values that
+     * are initially partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive pairs.  The final partitioning is striped.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys and values each
+     *     typedef cub::BlockRadixSort<int, 128, 4, int> BlockRadixSort;
+     *
+     *     // Allocate shared memory for BlockRadixSort
+     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_keys[4];
+     *     int thread_values[4];
+     *     ...
+     *
+     *     // Collectively sort the keys and values among block threads
+     *     BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys, thread_values);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_keys across the block of threads is
+     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
+     * corresponding output \p thread_keys in those threads will be
+     * <tt>{ [511,383,255,127], [386,258,130,2], [385,257,128,1], ..., [384,256,128,0] }</tt>.
+     *
+     */
+    __device__ __forceinline__ void SortDescendingBlockedToStriped(
+        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
+        ValueT  (&values)[ITEMS_PER_THREAD],        ///< [in-out] Values to sort
+        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
+        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
+    {
+        SortBlockedToStriped(keys, values, begin_bit, end_bit, Int2Type<true>(), Int2Type<KEYS_ONLY>());
+    }
+
+
+    //@}  end member group
+
+};
+
+/**
+ * \example example_block_radix_sort.cu
+ */
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_raking_layout.cuh b/third_party/cub/cub/block/block_raking_layout.cuh
new file mode 100644
index 0000000..3500616
--- /dev/null
+++ b/third_party/cub/cub/block/block_raking_layout.cuh
@@ -0,0 +1,152 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::BlockRakingLayout provides a conflict-free shared memory layout abstraction for warp-raking across thread block data.
+ */
+
+
+#pragma once
+
+#include "../util_macro.cuh"
+#include "../util_arch.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief BlockRakingLayout provides a conflict-free shared memory layout abstraction for 1D raking across thread block data.    ![](raking.png)
+ * \ingroup BlockModule
+ *
+ * \par Overview
+ * This type facilitates a shared memory usage pattern where a block of CUDA
+ * threads places elements into shared memory and then reduces the active
+ * parallelism to one "raking" warp of threads for serially aggregating consecutive
+ * sequences of shared items.  Padding is inserted to eliminate bank conflicts
+ * (for most data types).
+ *
+ * \tparam T                        The data type to be exchanged.
+ * \tparam BLOCK_THREADS            The thread block size in threads.
+ * \tparam PTX_ARCH                 <b>[optional]</b> \ptxversion
+ */
+template <
+    typename    T,
+    int         BLOCK_THREADS,
+    int         PTX_ARCH = CUB_PTX_ARCH>
+struct BlockRakingLayout
+{
+    //---------------------------------------------------------------------
+    // Constants and type definitions
+    //---------------------------------------------------------------------
+
+    enum
+    {
+        /// The total number of elements that need to be cooperatively reduced
+        SHARED_ELEMENTS = BLOCK_THREADS,
+
+        /// Maximum number of warp-synchronous raking threads
+        MAX_RAKING_THREADS = CUB_MIN(BLOCK_THREADS, CUB_WARP_THREADS(PTX_ARCH)),
+
+        /// Number of raking elements per warp-synchronous raking thread (rounded up)
+        SEGMENT_LENGTH = (SHARED_ELEMENTS + MAX_RAKING_THREADS - 1) / MAX_RAKING_THREADS,
+
+        /// Never use a raking thread that will have no valid data (e.g., when BLOCK_THREADS is 62 and SEGMENT_LENGTH is 2, we should only use 31 raking threads)
+        RAKING_THREADS = (SHARED_ELEMENTS + SEGMENT_LENGTH - 1) / SEGMENT_LENGTH,
+
+        /// Whether we will have bank conflicts (technically we should find out if the GCD is > 1)
+        HAS_CONFLICTS = (CUB_SMEM_BANKS(PTX_ARCH) % SEGMENT_LENGTH == 0),
+
+        /// Degree of bank conflicts (e.g., 4-way)
+        CONFLICT_DEGREE = (HAS_CONFLICTS) ?
+            (MAX_RAKING_THREADS * SEGMENT_LENGTH) / CUB_SMEM_BANKS(PTX_ARCH) :
+            1,
+
+        /// Pad each segment length with one element if segment length is not relatively prime to warp size and can't be optimized as a vector load
+        USE_SEGMENT_PADDING = ((SEGMENT_LENGTH & 1) == 0) && (SEGMENT_LENGTH > 2),
+
+        /// Total number of elements in the raking grid
+        GRID_ELEMENTS = RAKING_THREADS * (SEGMENT_LENGTH + USE_SEGMENT_PADDING),
+
+        /// Whether or not we need bounds checking during raking (the number of reduction elements is not a multiple of the number of raking threads)
+        UNGUARDED = (SHARED_ELEMENTS % RAKING_THREADS == 0),
+    };
+
+
+    /**
+     * \brief Shared memory storage type
+     */
+    struct __align__(16) _TempStorage
+    {
+        T buff[BlockRakingLayout::GRID_ELEMENTS];
+    };
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /**
+     * \brief Returns the location for the calling thread to place data into the grid
+     */
+    static __device__ __forceinline__ T* PlacementPtr(
+        TempStorage &temp_storage,
+        unsigned int linear_tid)
+    {
+        // Offset for partial
+        unsigned int offset = linear_tid;
+
+        // Add in one padding element for every segment
+        if (USE_SEGMENT_PADDING > 0)
+        {
+            offset += offset / SEGMENT_LENGTH;
+        }
+
+        // Incorporating a block of padding partials every shared memory segment
+        return temp_storage.Alias().buff + offset;
+    }
+
+
+    /**
+     * \brief Returns the location for the calling thread to begin sequential raking
+     */
+    static __device__ __forceinline__ T* RakingPtr(
+        TempStorage &temp_storage,
+        unsigned int linear_tid)
+    {
+        return temp_storage.Alias().buff + (linear_tid * (SEGMENT_LENGTH + USE_SEGMENT_PADDING));
+    }
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_reduce.cuh b/third_party/cub/cub/block/block_reduce.cuh
new file mode 100644
index 0000000..261f2ea
--- /dev/null
+++ b/third_party/cub/cub/block/block_reduce.cuh
@@ -0,0 +1,607 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockReduce class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "specializations/block_reduce_raking.cuh"
+#include "specializations/block_reduce_raking_commutative_only.cuh"
+#include "specializations/block_reduce_warp_reductions.cuh"
+#include "../util_ptx.cuh"
+#include "../util_type.cuh"
+#include "../thread/thread_operators.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+
+/******************************************************************************
+ * Algorithmic variants
+ ******************************************************************************/
+
+/**
+ * BlockReduceAlgorithm enumerates alternative algorithms for parallel
+ * reduction across a CUDA thread block.
+ */
+enum BlockReduceAlgorithm
+{
+
+    /**
+     * \par Overview
+     * An efficient "raking" reduction algorithm that only supports commutative
+     * reduction operators (true for most operations, e.g., addition).
+     *
+     * \par
+     * Execution is comprised of three phases:
+     * -# Upsweep sequential reduction in registers (if threads contribute more
+     *    than one input each).  Threads in warps other than the first warp place
+     *    their partial reductions into shared memory.
+     * -# Upsweep sequential reduction in shared memory.  Threads within the first
+     *    warp continue to accumulate by raking across segments of shared partial reductions
+     * -# A warp-synchronous Kogge-Stone style reduction within the raking warp.
+     *
+     * \par
+     * \image html block_reduce.png
+     * <div class="centercaption">\p BLOCK_REDUCE_RAKING data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
+     *
+     * \par Performance Considerations
+     * - This variant performs less communication than BLOCK_REDUCE_RAKING_NON_COMMUTATIVE
+     *   and is preferable when the reduction operator is commutative.  This variant
+     *   applies fewer reduction operators  than BLOCK_REDUCE_WARP_REDUCTIONS, and can provide higher overall
+     *   throughput across the GPU when suitably occupied.  However, turn-around latency may be
+     *   higher than to BLOCK_REDUCE_WARP_REDUCTIONS and thus less-desirable
+     *   when the GPU is under-occupied.
+     */
+    BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY,
+
+
+    /**
+     * \par Overview
+     * An efficient "raking" reduction algorithm that supports commutative
+     * (e.g., addition) and non-commutative (e.g., string concatenation) reduction
+     * operators. \blocked.
+     *
+     * \par
+     * Execution is comprised of three phases:
+     * -# Upsweep sequential reduction in registers (if threads contribute more
+     *    than one input each).  Each thread then places the partial reduction
+     *    of its item(s) into shared memory.
+     * -# Upsweep sequential reduction in shared memory.  Threads within a
+     *    single warp rake across segments of shared partial reductions.
+     * -# A warp-synchronous Kogge-Stone style reduction within the raking warp.
+     *
+     * \par
+     * \image html block_reduce.png
+     * <div class="centercaption">\p BLOCK_REDUCE_RAKING data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
+     *
+     * \par Performance Considerations
+     * - This variant performs more communication than BLOCK_REDUCE_RAKING
+     *   and is only preferable when the reduction operator is non-commutative.  This variant
+     *   applies fewer reduction operators than BLOCK_REDUCE_WARP_REDUCTIONS, and can provide higher overall
+     *   throughput across the GPU when suitably occupied.  However, turn-around latency may be
+     *   higher than to BLOCK_REDUCE_WARP_REDUCTIONS and thus less-desirable
+     *   when the GPU is under-occupied.
+     */
+    BLOCK_REDUCE_RAKING,
+
+
+    /**
+     * \par Overview
+     * A quick "tiled warp-reductions" reduction algorithm that supports commutative
+     * (e.g., addition) and non-commutative (e.g., string concatenation) reduction
+     * operators.
+     *
+     * \par
+     * Execution is comprised of four phases:
+     * -# Upsweep sequential reduction in registers (if threads contribute more
+     *    than one input each).  Each thread then places the partial reduction
+     *    of its item(s) into shared memory.
+     * -# Compute a shallow, but inefficient warp-synchronous Kogge-Stone style
+     *    reduction within each warp.
+     * -# A propagation phase where the warp reduction outputs in each warp are
+     *    updated with the aggregate from each preceding warp.
+     *
+     * \par
+     * \image html block_scan_warpscans.png
+     * <div class="centercaption">\p BLOCK_REDUCE_WARP_REDUCTIONS data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
+     *
+     * \par Performance Considerations
+     * - This variant applies more reduction operators than BLOCK_REDUCE_RAKING
+     *   or BLOCK_REDUCE_RAKING_NON_COMMUTATIVE, which may result in lower overall
+     *   throughput across the GPU.  However turn-around latency may be lower and
+     *   thus useful when the GPU is under-occupied.
+     */
+    BLOCK_REDUCE_WARP_REDUCTIONS,
+};
+
+
+/******************************************************************************
+ * Block reduce
+ ******************************************************************************/
+
+/**
+ * \brief The BlockReduce class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread block. ![](reduce_logo.png)
+ * \ingroup BlockModule
+ *
+ * \tparam T                Data type being reduced
+ * \tparam BLOCK_DIM_X      The thread block length in threads along the X dimension
+ * \tparam ALGORITHM        <b>[optional]</b> cub::BlockReduceAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_REDUCE_WARP_REDUCTIONS)
+ * \tparam BLOCK_DIM_Y      <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z      <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH         <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
+ *   uses a binary combining operator to compute a single aggregate from a list of input elements.
+ * - \rowmajor
+ * - BlockReduce can be optionally specialized by algorithm to accommodate different latency/throughput workload profiles:
+ *   -# <b>cub::BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY</b>.  An efficient "raking" reduction algorithm that only supports commutative reduction operators. [More...](\ref cub::BlockReduceAlgorithm)
+ *   -# <b>cub::BLOCK_REDUCE_RAKING</b>.  An efficient "raking" reduction algorithm that supports commutative and non-commutative reduction operators. [More...](\ref cub::BlockReduceAlgorithm)
+ *   -# <b>cub::BLOCK_REDUCE_WARP_REDUCTIONS</b>.  A quick "tiled warp-reductions" reduction algorithm that supports commutative and non-commutative reduction operators. [More...](\ref cub::BlockReduceAlgorithm)
+ *
+ * \par Performance Considerations
+ * - \granularity
+ * - Very efficient (only one synchronization barrier).
+ * - Incurs zero bank conflicts for most types
+ * - Computation is slightly more efficient (i.e., having lower instruction overhead) for:
+ *   - Summation (<b><em>vs.</em></b> generic reduction)
+ *   - \p BLOCK_THREADS is a multiple of the architecture's warp size
+ *   - Every thread has a valid input (i.e., full <b><em>vs.</em></b> partial-tiles)
+ * - See cub::BlockReduceAlgorithm for performance details regarding algorithmic alternatives
+ *
+ * \par A Simple Example
+ * \blockcollective{BlockReduce}
+ * \par
+ * The code snippet below illustrates a sum reduction of 512 integer items that
+ * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+ * where each thread owns 4 consecutive items.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize BlockReduce for a 1D block of 128 threads on type int
+ *     typedef cub::BlockReduce<int, 128> BlockReduce;
+ *
+ *     // Allocate shared memory for BlockReduce
+ *     __shared__ typename BlockReduce::TempStorage temp_storage;
+ *
+ *     // Obtain a segment of consecutive items that are blocked across threads
+ *     int thread_data[4];
+ *     ...
+ *
+ *     // Compute the block-wide sum for thread0
+ *     int aggregate = BlockReduce(temp_storage).Sum(thread_data);
+ *
+ * \endcode
+ *
+ */
+template <
+    typename                T,
+    int                     BLOCK_DIM_X,
+    BlockReduceAlgorithm    ALGORITHM       = BLOCK_REDUCE_WARP_REDUCTIONS,
+    int                     BLOCK_DIM_Y     = 1,
+    int                     BLOCK_DIM_Z     = 1,
+    int                     PTX_ARCH        = CUB_PTX_ARCH>
+class BlockReduce
+{
+private:
+
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+    typedef BlockReduceWarpReductions<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH>           WarpReductions;
+    typedef BlockReduceRakingCommutativeOnly<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH>    RakingCommutativeOnly;
+    typedef BlockReduceRaking<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH>                   Raking;
+
+    /// Internal specialization type
+    typedef typename If<(ALGORITHM == BLOCK_REDUCE_WARP_REDUCTIONS),
+        WarpReductions,
+        typename If<(ALGORITHM == BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY),
+            RakingCommutativeOnly,
+            Raking>::Type>::Type InternalBlockReduce;     // BlockReduceRaking
+
+    /// Shared memory storage layout type for BlockReduce
+    typedef typename InternalBlockReduce::TempStorage _TempStorage;
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+
+public:
+
+    /// \smemstorage{BlockReduce}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockReduce()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockReduce(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Generic reductions
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes a block-wide reduction for thread<sub>0</sub> using the specified binary reduction functor.  Each thread contributes one input element.
+     *
+     * \par
+     * - The return value is undefined in threads other than thread<sub>0</sub>.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a max reduction of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
+     *     typedef cub::BlockReduce<int, 128> BlockReduce;
+     *
+     *     // Allocate shared memory for BlockReduce
+     *     __shared__ typename BlockReduce::TempStorage temp_storage;
+     *
+     *     // Each thread obtains an input item
+     *     int thread_data;
+     *     ...
+     *
+     *     // Compute the block-wide max for thread0
+     *     int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max());
+     *
+     * \endcode
+     *
+     * \tparam ReductionOp          <b>[inferred]</b> Binary reduction functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T               input,                      ///< [in] Calling thread's input
+        ReductionOp     reduction_op)               ///< [in] Binary reduction functor 
+    {
+        return InternalBlockReduce(temp_storage).template Reduce<true>(input, BLOCK_THREADS, reduction_op);
+    }
+
+
+    /**
+     * \brief Computes a block-wide reduction for thread<sub>0</sub> using the specified binary reduction functor.  Each thread contributes an array of consecutive input elements.
+     *
+     * \par
+     * - The return value is undefined in threads other than thread<sub>0</sub>.
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a max reduction of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
+     *     typedef cub::BlockReduce<int, 128> BlockReduce;
+     *
+     *     // Allocate shared memory for BlockReduce
+     *     __shared__ typename BlockReduce::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Compute the block-wide max for thread0
+     *     int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max());
+     *
+     * \endcode
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ReductionOp          <b>[inferred]</b> Binary reduction functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        int ITEMS_PER_THREAD,
+        typename ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T               (&inputs)[ITEMS_PER_THREAD],    ///< [in] Calling thread's input segment
+        ReductionOp     reduction_op)                   ///< [in] Binary reduction functor 
+    {
+        // Reduce partials
+        T partial = internal::ThreadReduce(inputs, reduction_op);
+        return Reduce(partial, reduction_op);
+    }
+
+
+    /**
+     * \brief Computes a block-wide reduction for thread<sub>0</sub> using the specified binary reduction functor.  The first \p num_valid threads each contribute one input element.
+     *
+     * \par
+     * - The return value is undefined in threads other than thread<sub>0</sub>.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a max reduction of a partially-full tile of integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
+     *
+     * __global__ void ExampleKernel(int num_valid, ...)
+     * {
+     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
+     *     typedef cub::BlockReduce<int, 128> BlockReduce;
+     *
+     *     // Allocate shared memory for BlockReduce
+     *     __shared__ typename BlockReduce::TempStorage temp_storage;
+     *
+     *     // Each thread obtains an input item
+     *     int thread_data;
+     *     if (threadIdx.x < num_valid) thread_data = ...
+     *
+     *     // Compute the block-wide max for thread0
+     *     int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max(), num_valid);
+     *
+     * \endcode
+     *
+     * \tparam ReductionOp          <b>[inferred]</b> Binary reduction functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T                   input,                  ///< [in] Calling thread's input
+        ReductionOp         reduction_op,           ///< [in] Binary reduction functor 
+        int                 num_valid)              ///< [in] Number of threads containing valid elements (may be less than BLOCK_THREADS)
+    {
+        // Determine if we scan skip bounds checking
+        if (num_valid >= BLOCK_THREADS)
+        {
+            return InternalBlockReduce(temp_storage).template Reduce<true>(input, num_valid, reduction_op);
+        }
+        else
+        {
+            return InternalBlockReduce(temp_storage).template Reduce<false>(input, num_valid, reduction_op);
+        }
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Summation reductions
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes a block-wide reduction for thread<sub>0</sub> using addition (+) as the reduction operator.  Each thread contributes one input element.
+     *
+     * \par
+     * - The return value is undefined in threads other than thread<sub>0</sub>.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sum reduction of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
+     *     typedef cub::BlockReduce<int, 128> BlockReduce;
+     *
+     *     // Allocate shared memory for BlockReduce
+     *     __shared__ typename BlockReduce::TempStorage temp_storage;
+     *
+     *     // Each thread obtains an input item
+     *     int thread_data;
+     *     ...
+     *
+     *     // Compute the block-wide sum for thread0
+     *     int aggregate = BlockReduce(temp_storage).Sum(thread_data);
+     *
+     * \endcode
+     *
+     */
+    __device__ __forceinline__ T Sum(
+        T   input)                      ///< [in] Calling thread's input
+    {
+        return InternalBlockReduce(temp_storage).template Sum<true>(input, BLOCK_THREADS);
+    }
+
+    /**
+     * \brief Computes a block-wide reduction for thread<sub>0</sub> using addition (+) as the reduction operator.  Each thread contributes an array of consecutive input elements.
+     *
+     * \par
+     * - The return value is undefined in threads other than thread<sub>0</sub>.
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sum reduction of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
+     *     typedef cub::BlockReduce<int, 128> BlockReduce;
+     *
+     *     // Allocate shared memory for BlockReduce
+     *     __shared__ typename BlockReduce::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Compute the block-wide sum for thread0
+     *     int aggregate = BlockReduce(temp_storage).Sum(thread_data);
+     *
+     * \endcode
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ T Sum(
+        T   (&inputs)[ITEMS_PER_THREAD])    ///< [in] Calling thread's input segment
+    {
+        // Reduce partials
+        T partial = internal::ThreadReduce(inputs, cub::Sum());
+        return Sum(partial);
+    }
+
+
+    /**
+     * \brief Computes a block-wide reduction for thread<sub>0</sub> using addition (+) as the reduction operator.  The first \p num_valid threads each contribute one input element.
+     *
+     * \par
+     * - The return value is undefined in threads other than thread<sub>0</sub>.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sum reduction of a partially-full tile of integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
+     *
+     * __global__ void ExampleKernel(int num_valid, ...)
+     * {
+     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
+     *     typedef cub::BlockReduce<int, 128> BlockReduce;
+     *
+     *     // Allocate shared memory for BlockReduce
+     *     __shared__ typename BlockReduce::TempStorage temp_storage;
+     *
+     *     // Each thread obtains an input item (up to num_items)
+     *     int thread_data;
+     *     if (threadIdx.x < num_valid)
+     *         thread_data = ...
+     *
+     *     // Compute the block-wide sum for thread0
+     *     int aggregate = BlockReduce(temp_storage).Sum(thread_data, num_valid);
+     *
+     * \endcode
+     *
+     */
+    __device__ __forceinline__ T Sum(
+        T   input,                  ///< [in] Calling thread's input
+        int num_valid)              ///< [in] Number of threads containing valid elements (may be less than BLOCK_THREADS)
+    {
+        // Determine if we scan skip bounds checking
+        if (num_valid >= BLOCK_THREADS)
+        {
+            return InternalBlockReduce(temp_storage).template Sum<true>(input, num_valid);
+        }
+        else
+        {
+            return InternalBlockReduce(temp_storage).template Sum<false>(input, num_valid);
+        }
+    }
+
+
+    //@}  end member group
+};
+
+/**
+ * \example example_block_reduce.cu
+ */
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_scan.cuh b/third_party/cub/cub/block/block_scan.cuh
new file mode 100644
index 0000000..27ea7ed
--- /dev/null
+++ b/third_party/cub/cub/block/block_scan.cuh
@@ -0,0 +1,2126 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockScan class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel prefix sum/scan of items partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "specializations/block_scan_raking.cuh"
+#include "specializations/block_scan_warp_scans.cuh"
+#include "../util_arch.cuh"
+#include "../util_type.cuh"
+#include "../util_ptx.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Algorithmic variants
+ ******************************************************************************/
+
+/**
+ * \brief BlockScanAlgorithm enumerates alternative algorithms for cub::BlockScan to compute a parallel prefix scan across a CUDA thread block.
+ */
+enum BlockScanAlgorithm
+{
+
+    /**
+     * \par Overview
+     * An efficient "raking reduce-then-scan" prefix scan algorithm.  Execution is comprised of five phases:
+     * -# Upsweep sequential reduction in registers (if threads contribute more than one input each).  Each thread then places the partial reduction of its item(s) into shared memory.
+     * -# Upsweep sequential reduction in shared memory.  Threads within a single warp rake across segments of shared partial reductions.
+     * -# A warp-synchronous Kogge-Stone style exclusive scan within the raking warp.
+     * -# Downsweep sequential exclusive scan in shared memory.  Threads within a single warp rake across segments of shared partial reductions, seeded with the warp-scan output.
+     * -# Downsweep sequential scan in registers (if threads contribute more than one input), seeded with the raking scan output.
+     *
+     * \par
+     * \image html block_scan_raking.png
+     * <div class="centercaption">\p BLOCK_SCAN_RAKING data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
+     *
+     * \par Performance Considerations
+     * - Although this variant may suffer longer turnaround latencies when the
+     *   GPU is under-occupied, it can often provide higher overall throughput
+     *   across the GPU when suitably occupied.
+     */
+    BLOCK_SCAN_RAKING,
+
+
+    /**
+     * \par Overview
+     * Similar to cub::BLOCK_SCAN_RAKING, but with fewer shared memory reads at
+     * the expense of higher register pressure.  Raking threads preserve their
+     * "upsweep" segment of values in registers while performing warp-synchronous
+     * scan, allowing the "downsweep" not to re-read them from shared memory.
+     */
+    BLOCK_SCAN_RAKING_MEMOIZE,
+
+
+    /**
+     * \par Overview
+     * A quick "tiled warpscans" prefix scan algorithm.  Execution is comprised of four phases:
+     * -# Upsweep sequential reduction in registers (if threads contribute more than one input each).  Each thread then places the partial reduction of its item(s) into shared memory.
+     * -# Compute a shallow, but inefficient warp-synchronous Kogge-Stone style scan within each warp.
+     * -# A propagation phase where the warp scan outputs in each warp are updated with the aggregate from each preceding warp.
+     * -# Downsweep sequential scan in registers (if threads contribute more than one input), seeded with the raking scan output.
+     *
+     * \par
+     * \image html block_scan_warpscans.png
+     * <div class="centercaption">\p BLOCK_SCAN_WARP_SCANS data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
+     *
+     * \par Performance Considerations
+     * - Although this variant may suffer lower overall throughput across the
+     *   GPU because due to a heavy reliance on inefficient warpscans, it can
+     *   often provide lower turnaround latencies when the GPU is under-occupied.
+     */
+    BLOCK_SCAN_WARP_SCANS,
+};
+
+
+/******************************************************************************
+ * Block scan
+ ******************************************************************************/
+
+/**
+ * \brief The BlockScan class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel prefix sum/scan of items partitioned across a CUDA thread block. ![](block_scan_logo.png)
+ * \ingroup BlockModule
+ *
+ * \tparam T                Data type being scanned
+ * \tparam BLOCK_DIM_X      The thread block length in threads along the X dimension
+ * \tparam ALGORITHM        <b>[optional]</b> cub::BlockScanAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_SCAN_RAKING)
+ * \tparam BLOCK_DIM_Y      <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z      <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH         <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - Given a list of input elements and a binary reduction operator, a [<em>prefix scan</em>](http://en.wikipedia.org/wiki/Prefix_sum)
+ *   produces an output list where each element is computed to be the reduction
+ *   of the elements occurring earlier in the input list.  <em>Prefix sum</em>
+ *   connotes a prefix scan with the addition operator. The term \em inclusive indicates
+ *   that the <em>i</em><sup>th</sup> output reduction incorporates the <em>i</em><sup>th</sup> input.
+ *   The term \em exclusive indicates the <em>i</em><sup>th</sup> input is not incorporated into
+ *   the <em>i</em><sup>th</sup> output reduction.
+ * - \rowmajor
+ * - BlockScan can be optionally specialized by algorithm to accommodate different workload profiles:
+ *   -# <b>cub::BLOCK_SCAN_RAKING</b>.  An efficient (high throughput) "raking reduce-then-scan" prefix scan algorithm. [More...](\ref cub::BlockScanAlgorithm)
+ *   -# <b>cub::BLOCK_SCAN_RAKING_MEMOIZE</b>.  Similar to cub::BLOCK_SCAN_RAKING, but having higher throughput at the expense of additional register pressure for intermediate storage. [More...](\ref cub::BlockScanAlgorithm)
+ *   -# <b>cub::BLOCK_SCAN_WARP_SCANS</b>.  A quick (low latency) "tiled warpscans" prefix scan algorithm. [More...](\ref cub::BlockScanAlgorithm)
+ *
+ * \par Performance Considerations
+ * - \granularity
+ * - Uses special instructions when applicable (e.g., warp \p SHFL)
+ * - Uses synchronization-free communication between warp lanes when applicable
+ * - Invokes a minimal number of minimal block-wide synchronization barriers (only
+ *   one or two depending on algorithm selection)
+ * - Incurs zero bank conflicts for most types
+ * - Computation is slightly more efficient (i.e., having lower instruction overhead) for:
+ *   - Prefix sum variants (<b><em>vs.</em></b> generic scan)
+ *   - \blocksize
+ * - See cub::BlockScanAlgorithm for performance details regarding algorithmic alternatives
+ *
+ * \par A Simple Example
+ * \blockcollective{BlockScan}
+ * \par
+ * The code snippet below illustrates an exclusive prefix sum of 512 integer items that
+ * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+ * where each thread owns 4 consecutive items.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize BlockScan for a 1D block of 128 threads on type int
+ *     typedef cub::BlockScan<int, 128> BlockScan;
+ *
+ *     // Allocate shared memory for BlockScan
+ *     __shared__ typename BlockScan::TempStorage temp_storage;
+ *
+ *     // Obtain a segment of consecutive items that are blocked across threads
+ *     int thread_data[4];
+ *     ...
+ *
+ *     // Collectively compute the block-wide exclusive prefix sum
+ *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the block of threads is
+ * <tt>{[1,1,1,1], [1,1,1,1], ..., [1,1,1,1]}</tt>.
+ * The corresponding output \p thread_data in those threads will be
+ * <tt>{[0,1,2,3], [4,5,6,7], ..., [508,509,510,511]}</tt>.
+ *
+ */
+template <
+    typename            T,
+    int                 BLOCK_DIM_X,
+    BlockScanAlgorithm  ALGORITHM       = BLOCK_SCAN_RAKING,
+    int                 BLOCK_DIM_Y     = 1,
+    int                 BLOCK_DIM_Z     = 1,
+    int                 PTX_ARCH        = CUB_PTX_ARCH>
+class BlockScan
+{
+private:
+
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+    /**
+     * Ensure the template parameterization meets the requirements of the
+     * specified algorithm. Currently, the BLOCK_SCAN_WARP_SCANS policy
+     * cannot be used with thread block sizes not a multiple of the
+     * architectural warp size.
+     */
+    static const BlockScanAlgorithm SAFE_ALGORITHM =
+        ((ALGORITHM == BLOCK_SCAN_WARP_SCANS) && (BLOCK_THREADS % CUB_WARP_THREADS(PTX_ARCH) != 0)) ?
+            BLOCK_SCAN_RAKING :
+            ALGORITHM;
+
+    typedef BlockScanWarpScans<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> WarpScans;
+    typedef BlockScanRaking<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, (SAFE_ALGORITHM == BLOCK_SCAN_RAKING_MEMOIZE), PTX_ARCH> Raking;
+
+    /// Define the delegate type for the desired algorithm
+    typedef typename If<(SAFE_ALGORITHM == BLOCK_SCAN_WARP_SCANS),
+        WarpScans,
+        Raking>::Type InternalBlockScan;
+
+    /// Shared memory storage layout type for BlockScan
+    typedef typename InternalBlockScan::TempStorage _TempStorage;
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+    /******************************************************************************
+     * Public types
+     ******************************************************************************/
+public:
+
+    /// \smemstorage{BlockScan}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockScan()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockScan(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Exclusive prefix sum operations
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  The value of 0 is applied as the initial value, and is assigned to \p output in <em>thread</em><sub>0</sub>.
+     *
+     * \par
+     * - \identityzero
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an exclusive prefix sum of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain input item for each thread
+     *     int thread_data;
+     *     ...
+     *
+     *     // Collectively compute the block-wide exclusive prefix sum
+     *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>1, 1, ..., 1</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>0, 1, ..., 127</tt>.
+     *
+     */
+    __device__ __forceinline__ void ExclusiveSum(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output)                        ///< [out] Calling thread's output item (may be aliased to \p input)
+    {
+        T initial_value = 0;
+        ExclusiveScan(input, output, initial_value, cub::Sum());
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  The value of 0 is applied as the initial value, and is assigned to \p output in <em>thread</em><sub>0</sub>.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - \identityzero
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an exclusive prefix sum of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain input item for each thread
+     *     int thread_data;
+     *     ...
+     *
+     *     // Collectively compute the block-wide exclusive prefix sum
+     *     int block_aggregate;
+     *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data, block_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>1, 1, ..., 1</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>0, 1, ..., 127</tt>.
+     * Furthermore the value \p 128 will be stored in \p block_aggregate for all threads.
+     *
+     */
+    __device__ __forceinline__ void ExclusiveSum(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
+    {
+        T initial_value = 0;
+        ExclusiveScan(input, output, initial_value, cub::Sum(), block_aggregate);
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - \identityzero
+     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
+     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
+     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
+     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a single thread block that progressively
+     * computes an exclusive prefix sum over multiple "tiles" of input using a
+     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
+     * of 128 integer items that are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * // A stateful callback functor that maintains a running prefix to be applied
+     * // during consecutive scan operations.
+     * struct BlockPrefixCallbackOp
+     * {
+     *     // Running prefix
+     *     int running_total;
+     *
+     *     // Constructor
+     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
+     *
+     *     // Callback operator to be entered by the first warp of threads in the block.
+     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
+     *     __device__ int operator()(int block_aggregate)
+     *     {
+     *         int old_prefix = running_total;
+     *         running_total += block_aggregate;
+     *         return old_prefix;
+     *     }
+     * };
+     *
+     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Initialize running total
+     *     BlockPrefixCallbackOp prefix_op(0);
+     *
+     *     // Have the block iterate over segments of items
+     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128)
+     *     {
+     *         // Load a segment of consecutive items that are blocked across threads
+     *         int thread_data = d_data[block_offset];
+     *
+     *         // Collectively compute the block-wide exclusive prefix sum
+     *         BlockScan(temp_storage).ExclusiveSum(
+     *             thread_data, thread_data, prefix_op);
+     *         CTA_SYNC();
+     *
+     *         // Store scanned items to output segment
+     *         d_data[block_offset] = thread_data;
+     *     }
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>1, 1, 1, 1, 1, 1, 1, 1, ...</tt>.
+     * The corresponding output for the first segment will be <tt>0, 1, ..., 127</tt>.
+     * The output for the second segment will be <tt>128, 129, ..., 255</tt>.
+     *
+     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
+     */
+    template <typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void ExclusiveSum(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
+    {
+        ExclusiveScan(input, output, cub::Sum(), block_prefix_callback_op);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Exclusive prefix sum operations (multiple data per thread)
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  The value of 0 is applied as the initial value, and is assigned to \p output[0] in <em>thread</em><sub>0</sub>.
+     *
+     * \par
+     * - \identityzero
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an exclusive prefix sum of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute the block-wide exclusive prefix sum
+     *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ void ExclusiveSum(
+        T                 (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
+        T                 (&output)[ITEMS_PER_THREAD])  ///< [out] Calling thread's output items (may be aliased to \p input)
+    {
+        T initial_value = 0;
+        ExclusiveScan(input, output, initial_value, cub::Sum());
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  The value of 0 is applied as the initial value, and is assigned to \p output[0] in <em>thread</em><sub>0</sub>.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - \identityzero
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an exclusive prefix sum of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute the block-wide exclusive prefix sum
+     *     int block_aggregate;
+     *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data, block_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
+     * Furthermore the value \p 512 will be stored in \p block_aggregate for all threads.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ void ExclusiveSum(
+        T                 (&input)[ITEMS_PER_THREAD],       ///< [in] Calling thread's input items
+        T                 (&output)[ITEMS_PER_THREAD],      ///< [out] Calling thread's output items (may be aliased to \p input)
+        T                 &block_aggregate)                 ///< [out] block-wide aggregate reduction of input items
+    {
+        // Reduce consecutive thread items in registers
+        T initial_value = 0;
+        ExclusiveScan(input, output, initial_value, cub::Sum(), block_aggregate);
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - \identityzero
+     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
+     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
+     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
+     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a single thread block that progressively
+     * computes an exclusive prefix sum over multiple "tiles" of input using a
+     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
+     * of 512 integer items that are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3)
+     * across 128 threads where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * // A stateful callback functor that maintains a running prefix to be applied
+     * // during consecutive scan operations.
+     * struct BlockPrefixCallbackOp
+     * {
+     *     // Running prefix
+     *     int running_total;
+     *
+     *     // Constructor
+     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
+     *
+     *     // Callback operator to be entered by the first warp of threads in the block.
+     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
+     *     __device__ int operator()(int block_aggregate)
+     *     {
+     *         int old_prefix = running_total;
+     *         running_total += block_aggregate;
+     *         return old_prefix;
+     *     }
+     * };
+     *
+     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
+     * {
+     *     // Specialize BlockLoad, BlockStore, and BlockScan for a 1D block of 128 threads, 4 ints per thread
+     *     typedef cub::BlockLoad<int*, 128, 4, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
+     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_TRANSPOSE>  BlockStore;
+     *     typedef cub::BlockScan<int, 128>                             BlockScan;
+     *
+     *     // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
+     *     __shared__ union {
+     *         typename BlockLoad::TempStorage     load;
+     *         typename BlockScan::TempStorage     scan;
+     *         typename BlockStore::TempStorage    store;
+     *     } temp_storage;
+     *
+     *     // Initialize running total
+     *     BlockPrefixCallbackOp prefix_op(0);
+     *
+     *     // Have the block iterate over segments of items
+     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4)
+     *     {
+     *         // Load a segment of consecutive items that are blocked across threads
+     *         int thread_data[4];
+     *         BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data);
+     *         CTA_SYNC();
+     *
+     *         // Collectively compute the block-wide exclusive prefix sum
+     *         int block_aggregate;
+     *         BlockScan(temp_storage.scan).ExclusiveSum(
+     *             thread_data, thread_data, prefix_op);
+     *         CTA_SYNC();
+     *
+     *         // Store scanned items to output segment
+     *         BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data);
+     *         CTA_SYNC();
+     *     }
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>1, 1, 1, 1, 1, 1, 1, 1, ...</tt>.
+     * The corresponding output for the first segment will be <tt>0, 1, 2, 3, ..., 510, 511</tt>.
+     * The output for the second segment will be <tt>512, 513, 514, 515, ..., 1022, 1023</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
+     */
+    template <
+        int ITEMS_PER_THREAD,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void ExclusiveSum(
+        T                       (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
+        T                       (&output)[ITEMS_PER_THREAD],  ///< [out] Calling thread's output items (may be aliased to \p input)
+        BlockPrefixCallbackOp   &block_prefix_callback_op)    ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
+    {
+        ExclusiveScan(input, output, cub::Sum(), block_prefix_callback_op);
+    }
+
+
+
+    //@}  end member group        // Exclusive prefix sums
+    /******************************************************************//**
+     * \name Exclusive prefix scan operations
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an exclusive prefix max scan of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain input item for each thread
+     *     int thread_data;
+     *     ...
+     *
+     *     // Collectively compute the block-wide exclusive prefix max scan
+     *     BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>0, -1, 2, -3, ..., 126, -127</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>INT_MIN, 0, 0, 2, ..., 124, 126</tt>.
+     *
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        T               initial_value,                  ///< [in] Initial value to seed the exclusive scan (and is assigned to \p output[0] in <em>thread</em><sub>0</sub>)
+        ScanOp          scan_op)                        ///< [in] Binary scan functor 
+    {
+        InternalBlockScan(temp_storage).ExclusiveScan(input, output, initial_value, scan_op);
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an exclusive prefix max scan of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain input item for each thread
+     *     int thread_data;
+     *     ...
+     *
+     *     // Collectively compute the block-wide exclusive prefix max scan
+     *     int block_aggregate;
+     *     BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max(), block_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>0, -1, 2, -3, ..., 126, -127</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>INT_MIN, 0, 0, 2, ..., 124, 126</tt>.
+     * Furthermore the value \p 126 will be stored in \p block_aggregate for all threads.
+     *
+     * \tparam ScanOp   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &output,            ///< [out] Calling thread's output items (may be aliased to \p input)
+        T               initial_value,      ///< [in] Initial value to seed the exclusive scan (and is assigned to \p output[0] in <em>thread</em><sub>0</sub>)
+        ScanOp          scan_op,            ///< [in] Binary scan functor 
+        T               &block_aggregate)   ///< [out] block-wide aggregate reduction of input items
+    {
+        InternalBlockScan(temp_storage).ExclusiveScan(input, output, initial_value, scan_op, block_aggregate);
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
+     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
+     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
+     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
+     * - Supports non-commutative scan operators.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a single thread block that progressively
+     * computes an exclusive prefix max scan over multiple "tiles" of input using a
+     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
+     * of 128 integer items that are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * // A stateful callback functor that maintains a running prefix to be applied
+     * // during consecutive scan operations.
+     * struct BlockPrefixCallbackOp
+     * {
+     *     // Running prefix
+     *     int running_total;
+     *
+     *     // Constructor
+     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
+     *
+     *     // Callback operator to be entered by the first warp of threads in the block.
+     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
+     *     __device__ int operator()(int block_aggregate)
+     *     {
+     *         int old_prefix = running_total;
+     *         running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix;
+     *         return old_prefix;
+     *     }
+     * };
+     *
+     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Initialize running total
+     *     BlockPrefixCallbackOp prefix_op(INT_MIN);
+     *
+     *     // Have the block iterate over segments of items
+     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128)
+     *     {
+     *         // Load a segment of consecutive items that are blocked across threads
+     *         int thread_data = d_data[block_offset];
+     *
+     *         // Collectively compute the block-wide exclusive prefix max scan
+     *         BlockScan(temp_storage).ExclusiveScan(
+     *             thread_data, thread_data, INT_MIN, cub::Max(), prefix_op);
+     *         CTA_SYNC();
+     *
+     *         // Store scanned items to output segment
+     *         d_data[block_offset] = thread_data;
+     *     }
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>0, -1, 2, -3, 4, -5, ...</tt>.
+     * The corresponding output for the first segment will be <tt>INT_MIN, 0, 0, 2, ..., 124, 126</tt>.
+     * The output for the second segment will be <tt>126, 128, 128, 130, ..., 252, 254</tt>.
+     *
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
+     */
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan functor 
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
+    {
+        InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op, block_prefix_callback_op);
+    }
+
+
+    //@}  end member group        // Inclusive prefix sums
+    /******************************************************************//**
+     * \name Exclusive prefix scan operations (multiple data per thread)
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an exclusive prefix max scan of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute the block-wide exclusive prefix max scan
+     *     BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }</tt>.
+     * The corresponding output \p thread_data in those threads will be
+     * <tt>{ [INT_MIN,0,0,2], [2,4,4,6], ..., [506,508,508,510] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                 (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
+        T                 (&output)[ITEMS_PER_THREAD],  ///< [out] Calling thread's output items (may be aliased to \p input)
+        T                 initial_value,                ///< [in] Initial value to seed the exclusive scan (and is assigned to \p output[0] in <em>thread</em><sub>0</sub>)
+        ScanOp            scan_op)                      ///< [in] Binary scan functor
+    {
+        // Reduce consecutive thread items in registers
+        T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+        // Exclusive thread block-scan
+        ExclusiveScan(thread_prefix, thread_prefix, initial_value, scan_op);
+
+        // Exclusive scan in registers with prefix as seed
+        internal::ThreadScanExclusive(input, output, scan_op, thread_prefix);
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an exclusive prefix max scan of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute the block-wide exclusive prefix max scan
+     *     int block_aggregate;
+     *     BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max(), block_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>{ [INT_MIN,0,0,2], [2,4,4,6], ..., [506,508,508,510] }</tt>.
+     * Furthermore the value \p 510 will be stored in \p block_aggregate for all threads.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                 (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
+        T                 (&output)[ITEMS_PER_THREAD],  ///< [out] Calling thread's output items (may be aliased to \p input)
+        T                 initial_value,                ///< [in] Initial value to seed the exclusive scan (and is assigned to \p output[0] in <em>thread</em><sub>0</sub>)
+        ScanOp            scan_op,                      ///< [in] Binary scan functor
+        T                 &block_aggregate)             ///< [out] block-wide aggregate reduction of input items
+    {
+        // Reduce consecutive thread items in registers
+        T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+        // Exclusive thread block-scan
+        ExclusiveScan(thread_prefix, thread_prefix, initial_value, scan_op, block_aggregate);
+
+        // Exclusive scan in registers with prefix as seed
+        internal::ThreadScanExclusive(input, output, scan_op, thread_prefix);
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
+     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
+     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
+     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
+     * - Supports non-commutative scan operators.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a single thread block that progressively
+     * computes an exclusive prefix max scan over multiple "tiles" of input using a
+     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
+     * of 128 integer items that are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * // A stateful callback functor that maintains a running prefix to be applied
+     * // during consecutive scan operations.
+     * struct BlockPrefixCallbackOp
+     * {
+     *     // Running prefix
+     *     int running_total;
+     *
+     *     // Constructor
+     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
+     *
+     *     // Callback operator to be entered by the first warp of threads in the block.
+     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
+     *     __device__ int operator()(int block_aggregate)
+     *     {
+     *         int old_prefix = running_total;
+     *         running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix;
+     *         return old_prefix;
+     *     }
+     * };
+     *
+     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
+     * {
+     *     // Specialize BlockLoad, BlockStore, and BlockScan for a 1D block of 128 threads, 4 ints per thread
+     *     typedef cub::BlockLoad<int*, 128, 4, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
+     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_TRANSPOSE>  BlockStore;
+     *     typedef cub::BlockScan<int, 128>                             BlockScan;
+     *
+     *     // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
+     *     __shared__ union {
+     *         typename BlockLoad::TempStorage     load;
+     *         typename BlockScan::TempStorage     scan;
+     *         typename BlockStore::TempStorage    store;
+     *     } temp_storage;
+     *
+     *     // Initialize running total
+     *     BlockPrefixCallbackOp prefix_op(0);
+     *
+     *     // Have the block iterate over segments of items
+     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4)
+     *     {
+     *         // Load a segment of consecutive items that are blocked across threads
+     *         int thread_data[4];
+     *         BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data);
+     *         CTA_SYNC();
+     *
+     *         // Collectively compute the block-wide exclusive prefix max scan
+     *         BlockScan(temp_storage.scan).ExclusiveScan(
+     *             thread_data, thread_data, INT_MIN, cub::Max(), prefix_op);
+     *         CTA_SYNC();
+     *
+     *         // Store scanned items to output segment
+     *         BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data);
+     *         CTA_SYNC();
+     *     }
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>0, -1, 2, -3, 4, -5, ...</tt>.
+     * The corresponding output for the first segment will be <tt>INT_MIN, 0, 0, 2, 2, 4, ..., 508, 510</tt>.
+     * The output for the second segment will be <tt>510, 512, 512, 514, 514, 516, ..., 1020, 1022</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD         <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp                   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     * \tparam BlockPrefixCallbackOp    <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        ScanOp,
+        typename        BlockPrefixCallbackOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                       (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
+        T                       (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan functor
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
+    {
+        // Reduce consecutive thread items in registers
+        T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+        // Exclusive thread block-scan
+        ExclusiveScan(thread_prefix, thread_prefix, scan_op, block_prefix_callback_op);
+
+        // Exclusive scan in registers with prefix as seed
+        internal::ThreadScanExclusive(input, output, scan_op, thread_prefix);
+    }
+
+
+    //@}  end member group
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document no-initial-value scans
+
+    /******************************************************************//**
+     * \name Exclusive prefix scan operations (no initial value, single datum per thread)
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan functor
+    {
+        InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op);
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \tparam ScanOp   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan functor
+        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
+    {
+        InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op, block_aggregate);
+    }
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Exclusive prefix scan operations (no initial value, multiple data per thread)
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                 (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
+        T                 (&output)[ITEMS_PER_THREAD],  ///< [out] Calling thread's output items (may be aliased to \p input)
+        ScanOp            scan_op)                      ///< [in] Binary scan functor
+    {
+        // Reduce consecutive thread items in registers
+        T thread_partial = internal::ThreadReduce(input, scan_op);
+
+        // Exclusive thread block-scan
+        ExclusiveScan(thread_partial, thread_partial, scan_op);
+
+        // Exclusive scan in registers with prefix
+        internal::ThreadScanExclusive(input, output, scan_op, thread_partial, (linear_tid != 0));
+    }
+
+
+    /**
+     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
+        T               (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan functor
+        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
+    {
+        // Reduce consecutive thread items in registers
+        T thread_partial = internal::ThreadReduce(input, scan_op);
+
+        // Exclusive thread block-scan
+        ExclusiveScan(thread_partial, thread_partial, scan_op, block_aggregate);
+
+        // Exclusive scan in registers with prefix
+        internal::ThreadScanExclusive(input, output, scan_op, thread_partial, (linear_tid != 0));
+    }
+
+
+    //@}  end member group
+#endif // DOXYGEN_SHOULD_SKIP_THIS  // Do not document no-initial-value scans
+
+    /******************************************************************//**
+     * \name Inclusive prefix sum operations
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.
+     *
+     * \par
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an inclusive prefix sum of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain input item for each thread
+     *     int thread_data;
+     *     ...
+     *
+     *     // Collectively compute the block-wide inclusive prefix sum
+     *     BlockScan(temp_storage).InclusiveSum(thread_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>1, 1, ..., 1</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>1, 2, ..., 128</tt>.
+     *
+     */
+    __device__ __forceinline__ void InclusiveSum(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output)                        ///< [out] Calling thread's output item (may be aliased to \p input)
+    {
+        InclusiveScan(input, output, cub::Sum());
+    }
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an inclusive prefix sum of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain input item for each thread
+     *     int thread_data;
+     *     ...
+     *
+     *     // Collectively compute the block-wide inclusive prefix sum
+     *     int block_aggregate;
+     *     BlockScan(temp_storage).InclusiveSum(thread_data, thread_data, block_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>1, 1, ..., 1</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>1, 2, ..., 128</tt>.
+     * Furthermore the value \p 128 will be stored in \p block_aggregate for all threads.
+     *
+     */
+    __device__ __forceinline__ void InclusiveSum(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
+    {
+        InclusiveScan(input, output, cub::Sum(), block_aggregate);
+    }
+
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
+     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
+     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
+     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a single thread block that progressively
+     * computes an inclusive prefix sum over multiple "tiles" of input using a
+     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
+     * of 128 integer items that are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * // A stateful callback functor that maintains a running prefix to be applied
+     * // during consecutive scan operations.
+     * struct BlockPrefixCallbackOp
+     * {
+     *     // Running prefix
+     *     int running_total;
+     *
+     *     // Constructor
+     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
+     *
+     *     // Callback operator to be entered by the first warp of threads in the block.
+     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
+     *     __device__ int operator()(int block_aggregate)
+     *     {
+     *         int old_prefix = running_total;
+     *         running_total += block_aggregate;
+     *         return old_prefix;
+     *     }
+     * };
+     *
+     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Initialize running total
+     *     BlockPrefixCallbackOp prefix_op(0);
+     *
+     *     // Have the block iterate over segments of items
+     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128)
+     *     {
+     *         // Load a segment of consecutive items that are blocked across threads
+     *         int thread_data = d_data[block_offset];
+     *
+     *         // Collectively compute the block-wide inclusive prefix sum
+     *         BlockScan(temp_storage).InclusiveSum(
+     *             thread_data, thread_data, prefix_op);
+     *         CTA_SYNC();
+     *
+     *         // Store scanned items to output segment
+     *         d_data[block_offset] = thread_data;
+     *     }
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>1, 1, 1, 1, 1, 1, 1, 1, ...</tt>.
+     * The corresponding output for the first segment will be <tt>1, 2, ..., 128</tt>.
+     * The output for the second segment will be <tt>129, 130, ..., 256</tt>.
+     *
+     * \tparam BlockPrefixCallbackOp          <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
+     */
+    template <typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void InclusiveSum(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
+    {
+        InclusiveScan(input, output, cub::Sum(), block_prefix_callback_op);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Inclusive prefix sum operations (multiple data per thread)
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.
+     *
+     * \par
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an inclusive prefix sum of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute the block-wide inclusive prefix sum
+     *     BlockScan(temp_storage).InclusiveSum(thread_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>{ [1,2,3,4], [5,6,7,8], ..., [509,510,511,512] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ void InclusiveSum(
+        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
+        T               (&output)[ITEMS_PER_THREAD])    ///< [out] Calling thread's output items (may be aliased to \p input)
+    {
+        if (ITEMS_PER_THREAD == 1)
+        {
+            InclusiveSum(input[0], output[0]);
+        }
+        else
+        {
+            // Reduce consecutive thread items in registers
+            Sum scan_op;
+            T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+            // Exclusive thread block-scan
+            ExclusiveSum(thread_prefix, thread_prefix);
+
+            // Inclusive scan in registers with prefix as seed
+            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix, (linear_tid != 0));
+        }
+    }
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an inclusive prefix sum of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute the block-wide inclusive prefix sum
+     *     int block_aggregate;
+     *     BlockScan(temp_storage).InclusiveSum(thread_data, thread_data, block_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }</tt>.  The
+     * corresponding output \p thread_data in those threads will be
+     * <tt>{ [1,2,3,4], [5,6,7,8], ..., [509,510,511,512] }</tt>.
+     * Furthermore the value \p 512 will be stored in \p block_aggregate for all threads.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ void InclusiveSum(
+        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
+        T               (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
+        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
+    {
+        if (ITEMS_PER_THREAD == 1)
+        {
+            InclusiveSum(input[0], output[0], block_aggregate);
+        }
+        else
+        {
+            // Reduce consecutive thread items in registers
+            Sum scan_op;
+            T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+            // Exclusive thread block-scan
+            ExclusiveSum(thread_prefix, thread_prefix, block_aggregate);
+
+            // Inclusive scan in registers with prefix as seed
+            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix, (linear_tid != 0));
+        }
+    }
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
+     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
+     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
+     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a single thread block that progressively
+     * computes an inclusive prefix sum over multiple "tiles" of input using a
+     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
+     * of 512 integer items that are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3)
+     * across 128 threads where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * // A stateful callback functor that maintains a running prefix to be applied
+     * // during consecutive scan operations.
+     * struct BlockPrefixCallbackOp
+     * {
+     *     // Running prefix
+     *     int running_total;
+     *
+     *     // Constructor
+     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
+     *
+     *     // Callback operator to be entered by the first warp of threads in the block.
+     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
+     *     __device__ int operator()(int block_aggregate)
+     *     {
+     *         int old_prefix = running_total;
+     *         running_total += block_aggregate;
+     *         return old_prefix;
+     *     }
+     * };
+     *
+     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
+     * {
+     *     // Specialize BlockLoad, BlockStore, and BlockScan for a 1D block of 128 threads, 4 ints per thread
+     *     typedef cub::BlockLoad<int*, 128, 4, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
+     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_TRANSPOSE>  BlockStore;
+     *     typedef cub::BlockScan<int, 128>                             BlockScan;
+     *
+     *     // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
+     *     __shared__ union {
+     *         typename BlockLoad::TempStorage     load;
+     *         typename BlockScan::TempStorage     scan;
+     *         typename BlockStore::TempStorage    store;
+     *     } temp_storage;
+     *
+     *     // Initialize running total
+     *     BlockPrefixCallbackOp prefix_op(0);
+     *
+     *     // Have the block iterate over segments of items
+     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4)
+     *     {
+     *         // Load a segment of consecutive items that are blocked across threads
+     *         int thread_data[4];
+     *         BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data);
+     *         CTA_SYNC();
+     *
+     *         // Collectively compute the block-wide inclusive prefix sum
+     *         BlockScan(temp_storage.scan).IncluisveSum(
+     *             thread_data, thread_data, prefix_op);
+     *         CTA_SYNC();
+     *
+     *         // Store scanned items to output segment
+     *         BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data);
+     *         CTA_SYNC();
+     *     }
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>1, 1, 1, 1, 1, 1, 1, 1, ...</tt>.
+     * The corresponding output for the first segment will be <tt>1, 2, 3, 4, ..., 511, 512</tt>.
+     * The output for the second segment will be <tt>513, 514, 515, 516, ..., 1023, 1024</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
+     */
+    template <
+        int ITEMS_PER_THREAD,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void InclusiveSum(
+        T                       (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
+        T                       (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
+    {
+        if (ITEMS_PER_THREAD == 1)
+        {
+            InclusiveSum(input[0], output[0], block_prefix_callback_op);
+        }
+        else
+        {
+            // Reduce consecutive thread items in registers
+            Sum scan_op;
+            T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+            // Exclusive thread block-scan
+            ExclusiveSum(thread_prefix, thread_prefix, block_prefix_callback_op);
+
+            // Inclusive scan in registers with prefix as seed
+            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix);
+        }
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Inclusive prefix scan operations
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an inclusive prefix max scan of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain input item for each thread
+     *     int thread_data;
+     *     ...
+     *
+     *     // Collectively compute the block-wide inclusive prefix max scan
+     *     BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>0, -1, 2, -3, ..., 126, -127</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>0, 0, 2, 2, ..., 126, 126</tt>.
+     *
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan functor 
+    {
+        InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op);
+    }
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an inclusive prefix max scan of 128 integer items that
+     * are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain input item for each thread
+     *     int thread_data;
+     *     ...
+     *
+     *     // Collectively compute the block-wide inclusive prefix max scan
+     *     int block_aggregate;
+     *     BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max(), block_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>0, -1, 2, -3, ..., 126, -127</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>0, 0, 2, 2, ..., 126, 126</tt>.
+     * Furthermore the value \p 126 will be stored in \p block_aggregate for all threads.
+     *
+     * \tparam ScanOp   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan functor 
+        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
+    {
+        InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op, block_aggregate);
+    }
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
+     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
+     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
+     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
+     * - Supports non-commutative scan operators.
+     * - \rowmajor
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a single thread block that progressively
+     * computes an inclusive prefix max scan over multiple "tiles" of input using a
+     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
+     * of 128 integer items that are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * // A stateful callback functor that maintains a running prefix to be applied
+     * // during consecutive scan operations.
+     * struct BlockPrefixCallbackOp
+     * {
+     *     // Running prefix
+     *     int running_total;
+     *
+     *     // Constructor
+     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
+     *
+     *     // Callback operator to be entered by the first warp of threads in the block.
+     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
+     *     __device__ int operator()(int block_aggregate)
+     *     {
+     *         int old_prefix = running_total;
+     *         running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix;
+     *         return old_prefix;
+     *     }
+     * };
+     *
+     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Initialize running total
+     *     BlockPrefixCallbackOp prefix_op(INT_MIN);
+     *
+     *     // Have the block iterate over segments of items
+     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128)
+     *     {
+     *         // Load a segment of consecutive items that are blocked across threads
+     *         int thread_data = d_data[block_offset];
+     *
+     *         // Collectively compute the block-wide inclusive prefix max scan
+     *         BlockScan(temp_storage).InclusiveScan(
+     *             thread_data, thread_data, cub::Max(), prefix_op);
+     *         CTA_SYNC();
+     *
+     *         // Store scanned items to output segment
+     *         d_data[block_offset] = thread_data;
+     *     }
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>0, -1, 2, -3, 4, -5, ...</tt>.
+     * The corresponding output for the first segment will be <tt>0, 0, 2, 2, ..., 126, 126</tt>.
+     * The output for the second segment will be <tt>128, 128, 130, 130, ..., 254, 254</tt>.
+     *
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
+     */
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan functor 
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
+    {
+        InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op, block_prefix_callback_op);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Inclusive prefix scan operations (multiple data per thread)
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an inclusive prefix max scan of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute the block-wide inclusive prefix max scan
+     *     BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }</tt>.  The
+     * corresponding output \p thread_data in those threads will be <tt>{ [0,0,2,2], [4,4,6,6], ..., [508,508,510,510] }</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
+        T               (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan functor 
+    {
+        if (ITEMS_PER_THREAD == 1)
+        {
+            InclusiveScan(input[0], output[0], scan_op);
+        }
+        else
+        {
+            // Reduce consecutive thread items in registers
+            T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+            // Exclusive thread block-scan
+            ExclusiveScan(thread_prefix, thread_prefix, scan_op);
+
+            // Inclusive scan in registers with prefix as seed (first thread does not seed)
+            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix, (linear_tid != 0));
+        }
+    }
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates an inclusive prefix max scan of 512 integer items that
+     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
+     * where each thread owns 4 consecutive items.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize BlockScan for a 1D block of 128 threads on type int
+     *     typedef cub::BlockScan<int, 128> BlockScan;
+     *
+     *     // Allocate shared memory for BlockScan
+     *     __shared__ typename BlockScan::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Collectively compute the block-wide inclusive prefix max scan
+     *     int block_aggregate;
+     *     BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max(), block_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is
+     * <tt>{ [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }</tt>.
+     * The corresponding output \p thread_data in those threads will be
+     * <tt>{ [0,0,2,2], [4,4,6,6], ..., [508,508,510,510] }</tt>.
+     * Furthermore the value \p 510 will be stored in \p block_aggregate for all threads.
+     *
+     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename         ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
+        T               (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan functor 
+        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
+    {
+        if (ITEMS_PER_THREAD == 1)
+        {
+            InclusiveScan(input[0], output[0], scan_op, block_aggregate);
+        }
+        else
+        {
+            // Reduce consecutive thread items in registers
+            T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+            // Exclusive thread block-scan (with no initial value)
+            ExclusiveScan(thread_prefix, thread_prefix, scan_op, block_aggregate);
+
+            // Inclusive scan in registers with prefix as seed (first thread does not seed)
+            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix, (linear_tid != 0));
+        }
+    }
+
+
+    /**
+     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+     *
+     * \par
+     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
+     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
+     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
+     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
+     * - Supports non-commutative scan operators.
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a single thread block that progressively
+     * computes an inclusive prefix max scan over multiple "tiles" of input using a
+     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
+     * of 128 integer items that are partitioned across 128 threads.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
+     *
+     * // A stateful callback functor that maintains a running prefix to be applied
+     * // during consecutive scan operations.
+     * struct BlockPrefixCallbackOp
+     * {
+     *     // Running prefix
+     *     int running_total;
+     *
+     *     // Constructor
+     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
+     *
+     *     // Callback operator to be entered by the first warp of threads in the block.
+     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
+     *     __device__ int operator()(int block_aggregate)
+     *     {
+     *         int old_prefix = running_total;
+     *         running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix;
+     *         return old_prefix;
+     *     }
+     * };
+     *
+     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
+     * {
+     *     // Specialize BlockLoad, BlockStore, and BlockScan for a 1D block of 128 threads, 4 ints per thread
+     *     typedef cub::BlockLoad<int*, 128, 4, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
+     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_TRANSPOSE>  BlockStore;
+     *     typedef cub::BlockScan<int, 128>                             BlockScan;
+     *
+     *     // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
+     *     __shared__ union {
+     *         typename BlockLoad::TempStorage     load;
+     *         typename BlockScan::TempStorage     scan;
+     *         typename BlockStore::TempStorage    store;
+     *     } temp_storage;
+     *
+     *     // Initialize running total
+     *     BlockPrefixCallbackOp prefix_op(0);
+     *
+     *     // Have the block iterate over segments of items
+     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4)
+     *     {
+     *         // Load a segment of consecutive items that are blocked across threads
+     *         int thread_data[4];
+     *         BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data);
+     *         CTA_SYNC();
+     *
+     *         // Collectively compute the block-wide inclusive prefix max scan
+     *         BlockScan(temp_storage.scan).InclusiveScan(
+     *             thread_data, thread_data, cub::Max(), prefix_op);
+     *         CTA_SYNC();
+     *
+     *         // Store scanned items to output segment
+     *         BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data);
+     *         CTA_SYNC();
+     *     }
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>0, -1, 2, -3, 4, -5, ...</tt>.
+     * The corresponding output for the first segment will be <tt>0, 0, 2, 2, 4, 4, ..., 510, 510</tt>.
+     * The output for the second segment will be <tt>512, 512, 514, 514, 516, 516, ..., 1022, 1022</tt>.
+     *
+     * \tparam ITEMS_PER_THREAD         <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+     * \tparam ScanOp                   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
+     * \tparam BlockPrefixCallbackOp    <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
+     */
+    template <
+        int             ITEMS_PER_THREAD,
+        typename        ScanOp,
+        typename        BlockPrefixCallbackOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T                       (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
+        T                       (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan functor 
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
+    {
+        if (ITEMS_PER_THREAD == 1)
+        {
+            InclusiveScan(input[0], output[0], scan_op, block_prefix_callback_op);
+        }
+        else
+        {
+            // Reduce consecutive thread items in registers
+            T thread_prefix = internal::ThreadReduce(input, scan_op);
+
+            // Exclusive thread block-scan
+            ExclusiveScan(thread_prefix, thread_prefix, scan_op, block_prefix_callback_op);
+
+            // Inclusive scan in registers with prefix as seed
+            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix);
+        }
+    }
+
+    //@}  end member group
+
+
+};
+
+/**
+ * \example example_block_scan.cu
+ */
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_shuffle.cuh b/third_party/cub/cub/block/block_shuffle.cuh
new file mode 100644
index 0000000..a0cc71d
--- /dev/null
+++ b/third_party/cub/cub/block/block_shuffle.cuh
@@ -0,0 +1,305 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockShuffle class provides [<em>collective</em>](index.html#sec0) methods for shuffling data partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../util_arch.cuh"
+#include "../util_ptx.cuh"
+#include "../util_macro.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief The BlockShuffle class provides [<em>collective</em>](index.html#sec0) methods for shuffling data partitioned across a CUDA thread block.
+ * \ingroup BlockModule
+ *
+ * \tparam T                    The data type to be exchanged.
+ * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
+ * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * It is commonplace for blocks of threads to rearrange data items between
+ * threads.  The BlockShuffle abstraction allows threads to efficiently shift items
+ * either (a) up to their successor or (b) down to their predecessor.
+ *
+ */
+template <
+    typename            T,
+    int                 BLOCK_DIM_X,
+    int                 BLOCK_DIM_Y         = 1,
+    int                 BLOCK_DIM_Z         = 1,
+    int                 PTX_ARCH            = CUB_PTX_ARCH>
+class BlockShuffle
+{
+private:
+
+    /******************************************************************************
+     * Constants
+     ******************************************************************************/
+
+    enum
+    {
+        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        LOG_WARP_THREADS            = CUB_LOG_WARP_THREADS(PTX_ARCH),
+        WARP_THREADS                = 1 << LOG_WARP_THREADS,
+        WARPS                       = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+    };
+
+    /******************************************************************************
+     * Type definitions
+     ******************************************************************************/
+
+    /// Shared memory storage layout type (last element from each thread's input)
+    struct _TempStorage
+    {
+        T prev[BLOCK_THREADS];
+        T next[BLOCK_THREADS];
+    };
+
+
+public:
+
+    /// \smemstorage{BlockShuffle}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+private:
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    unsigned int linear_tid;
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+public:
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockShuffle()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockShuffle(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Shuffle movement
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Each <em>thread<sub>i</sub></em> obtains the \p input provided by <em>thread</em><sub><em>i</em>+<tt>distance</tt></sub>. The offset \p distance may be negative.
+     *
+     * \par
+     * - \smemreuse
+     */
+    __device__ __forceinline__ void Offset(
+        T   input,                  ///< [in] The input item from the calling thread (<em>thread<sub>i</sub></em>)
+        T&  output,                 ///< [out] The \p input item from the successor (or predecessor) thread <em>thread</em><sub><em>i</em>+<tt>distance</tt></sub> (may be aliased to \p input).  This value is only updated for for <em>thread<sub>i</sub></em> when 0 <= (<em>i</em> + \p distance) < <tt>BLOCK_THREADS-1</tt>
+        int distance = 1)           ///< [in] Offset distance (may be negative)
+    {
+        temp_storage[linear_tid].prev = input;
+
+        CTA_SYNC();
+
+        if ((linear_tid + distance >= 0) && (linear_tid + distance < BLOCK_THREADS))
+            output = temp_storage[linear_tid + distance].prev;
+    }
+
+
+    /**
+     * \brief Each <em>thread<sub>i</sub></em> obtains the \p input provided by <em>thread</em><sub><em>i</em>+<tt>distance</tt></sub>.
+     *
+     * \par
+     * - \smemreuse
+     */
+    __device__ __forceinline__ void Rotate(
+        T   input,                  ///< [in] The calling thread's input item
+        T&  output,                 ///< [out] The \p input item from thread <em>thread</em><sub>(<em>i</em>+<tt>distance></tt>)%<tt><BLOCK_THREADS></tt></sub> (may be aliased to \p input).  This value is not updated for <em>thread</em><sub>BLOCK_THREADS-1</sub>
+        unsigned int distance = 1)  ///< [in] Offset distance (0 < \p distance < <tt>BLOCK_THREADS</tt>)
+    {
+        temp_storage[linear_tid].prev = input;
+
+        CTA_SYNC();
+
+        unsigned int offset = threadIdx.x + distance;
+        if (offset >= BLOCK_THREADS)
+            offset -= BLOCK_THREADS;
+
+        output = temp_storage[offset].prev;
+    }
+
+
+    /**
+     * \brief The thread block rotates its [<em>blocked arrangement</em>](index.html#sec5sec3) of \p input items, shifting it up by one item
+     *
+     * \par
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ void Up(
+        T (&input)[ITEMS_PER_THREAD],   ///< [in] The calling thread's input items
+        T (&prev)[ITEMS_PER_THREAD])    ///< [out] The corresponding predecessor items (may be aliased to \p input).  The item \p prev[0] is not updated for <em>thread</em><sub>0</sub>.
+    {
+        temp_storage[linear_tid].prev = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = ITEMS_PER_THREAD - 1; ITEM > 0; --ITEM)
+            prev[ITEM] = input[ITEM - 1];
+
+
+        if (linear_tid > 0)
+            prev[0] = temp_storage[linear_tid - 1].prev;
+    }
+
+
+    /**
+     * \brief The thread block rotates its [<em>blocked arrangement</em>](index.html#sec5sec3) of \p input items, shifting it up by one item.  All threads receive the \p input provided by <em>thread</em><sub><tt>BLOCK_THREADS-1</tt></sub>.
+     *
+     * \par
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ void Up(
+        T (&input)[ITEMS_PER_THREAD],   ///< [in] The calling thread's input items
+        T (&prev)[ITEMS_PER_THREAD],    ///< [out] The corresponding predecessor items (may be aliased to \p input).  The item \p prev[0] is not updated for <em>thread</em><sub>0</sub>.
+        T &block_suffix)                ///< [out] The item \p input[ITEMS_PER_THREAD-1] from <em>thread</em><sub><tt>BLOCK_THREADS-1</tt></sub>, provided to all threads
+    {
+        Up(input, prev);
+        block_suffix = temp_storage[BLOCK_THREADS - 1].prev;
+    }
+
+
+    /**
+     * \brief The thread block rotates its [<em>blocked arrangement</em>](index.html#sec5sec3) of \p input items, shifting it down by one item
+     *
+     * \par
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ void Down(
+        T (&input)[ITEMS_PER_THREAD],   ///< [in] The calling thread's input items
+        T (&prev)[ITEMS_PER_THREAD])    ///< [out] The corresponding predecessor items (may be aliased to \p input).  The value \p prev[0] is not updated for <em>thread</em><sub>BLOCK_THREADS-1</sub>.
+    {
+        temp_storage[linear_tid].prev = input[ITEMS_PER_THREAD - 1];
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int ITEM = ITEMS_PER_THREAD - 1; ITEM > 0; --ITEM)
+            prev[ITEM] = input[ITEM - 1];
+
+        if (linear_tid > 0)
+            prev[0] = temp_storage[linear_tid - 1].prev;
+    }
+
+
+    /**
+     * \brief The thread block rotates its [<em>blocked arrangement</em>](index.html#sec5sec3) of input items, shifting it down by one item.  All threads receive \p input[0] provided by <em>thread</em><sub><tt>0</tt></sub>.
+     *
+     * \par
+     * - \blocked
+     * - \granularity
+     * - \smemreuse
+     */
+    template <int ITEMS_PER_THREAD>
+    __device__ __forceinline__ void Down(
+        T (&input)[ITEMS_PER_THREAD],   ///< [in] The calling thread's input items
+        T (&prev)[ITEMS_PER_THREAD],    ///< [out] The corresponding predecessor items (may be aliased to \p input).  The value \p prev[0] is not updated for <em>thread</em><sub>BLOCK_THREADS-1</sub>.
+        T &block_prefix)                ///< [out] The item \p input[0] from <em>thread</em><sub><tt>0</tt></sub>, provided to all threads
+    {
+        Up(input, prev);
+        block_prefix = temp_storage[BLOCK_THREADS - 1].prev;
+    }
+
+    //@}  end member group
+
+
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/block_store.cuh b/third_party/cub/cub/block/block_store.cuh
new file mode 100644
index 0000000..648bf9f
--- /dev/null
+++ b/third_party/cub/cub/block/block_store.cuh
@@ -0,0 +1,1000 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Operations for writing linear segments of data from the CUDA thread block
+ */
+
+#pragma once
+
+#include <iterator>
+
+#include "block_exchange.cuh"
+#include "../util_ptx.cuh"
+#include "../util_macro.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup UtilIo
+ * @{
+ */
+
+
+/******************************************************************//**
+ * \name Blocked arrangement I/O (direct)
+ *********************************************************************/
+//@{
+
+/**
+ * \brief Store a blocked arrangement of items across a thread block into a linear segment of items.
+ *
+ * \blocked
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to store.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
+ */
+template <
+    typename            T,
+    int                 ITEMS_PER_THREAD,
+    typename            OutputIteratorT>
+__device__ __forceinline__ void StoreDirectBlocked(
+    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+    T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+{
+    OutputIteratorT thread_itr = block_itr + (linear_tid * ITEMS_PER_THREAD);
+
+    // Store directly in thread-blocked order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        thread_itr[ITEM] = items[ITEM];
+    }
+}
+
+
+/**
+ * \brief Store a blocked arrangement of items across a thread block into a linear segment of items, guarded by range
+ *
+ * \blocked
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to store.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
+ */
+template <
+    typename            T,
+    int                 ITEMS_PER_THREAD,
+    typename            OutputIteratorT>
+__device__ __forceinline__ void StoreDirectBlocked(
+    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+    T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
+    int                 valid_items)                ///< [in] Number of valid items to write
+{
+    OutputIteratorT thread_itr = block_itr + (linear_tid * ITEMS_PER_THREAD);
+
+    // Store directly in thread-blocked order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        if (ITEM + (linear_tid * ITEMS_PER_THREAD) < valid_items)
+        {
+            thread_itr[ITEM] = items[ITEM];
+        }
+    }
+}
+
+
+/**
+ * \brief Store a blocked arrangement of items across a thread block into a linear segment of items.
+ *
+ * \blocked
+ *
+ * The output offset (\p block_ptr + \p block_offset) must be quad-item aligned,
+ * which is the default starting offset returned by \p cudaMalloc()
+ *
+ * \par
+ * The following conditions will prevent vectorization and storing will fall back to cub::BLOCK_STORE_DIRECT:
+ *   - \p ITEMS_PER_THREAD is odd
+ *   - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.)
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to store.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ *
+ */
+template <
+    typename            T,
+    int                 ITEMS_PER_THREAD>
+__device__ __forceinline__ void StoreDirectBlockedVectorized(
+    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    T                   *block_ptr,                 ///< [in] Input pointer for storing from
+    T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+{
+    enum
+    {
+        // Maximum CUDA vector size is 4 elements
+        MAX_VEC_SIZE = CUB_MIN(4, ITEMS_PER_THREAD),
+
+        // Vector size must be a power of two and an even divisor of the items per thread
+        VEC_SIZE = ((((MAX_VEC_SIZE - 1) & MAX_VEC_SIZE) == 0) && ((ITEMS_PER_THREAD % MAX_VEC_SIZE) == 0)) ?
+            MAX_VEC_SIZE :
+            1,
+
+        VECTORS_PER_THREAD = ITEMS_PER_THREAD / VEC_SIZE,
+    };
+
+    // Vector type
+    typedef typename CubVector<T, VEC_SIZE>::Type Vector;
+
+    // Alias global pointer
+    Vector *block_ptr_vectors = reinterpret_cast<Vector*>(const_cast<T*>(block_ptr));
+
+    // Alias pointers (use "raw" array here which should get optimized away to prevent conservative PTXAS lmem spilling)
+    Vector raw_vector[VECTORS_PER_THREAD];
+    T *raw_items = reinterpret_cast<T*>(raw_vector);
+
+    // Copy
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        raw_items[ITEM] = items[ITEM];
+    }
+
+    // Direct-store using vector types
+    StoreDirectBlocked(linear_tid, block_ptr_vectors, raw_vector);
+}
+
+
+
+//@}  end member group
+/******************************************************************//**
+ * \name Striped arrangement I/O (direct)
+ *********************************************************************/
+//@{
+
+
+/**
+ * \brief Store a striped arrangement of data across the thread block into a linear segment of items.
+ *
+ * \striped
+ *
+ * \tparam BLOCK_THREADS        The thread block size in threads
+ * \tparam T                    <b>[inferred]</b> The data type to store.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
+ */
+template <
+    int                 BLOCK_THREADS,
+    typename            T,
+    int                 ITEMS_PER_THREAD,
+    typename            OutputIteratorT>
+__device__ __forceinline__ void StoreDirectStriped(
+    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+    T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+{
+    OutputIteratorT thread_itr = block_itr + linear_tid;
+
+    // Store directly in striped order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        thread_itr[(ITEM * BLOCK_THREADS)] = items[ITEM];
+    }
+}
+
+
+/**
+ * \brief Store a striped arrangement of data across the thread block into a linear segment of items, guarded by range
+ *
+ * \striped
+ *
+ * \tparam BLOCK_THREADS        The thread block size in threads
+ * \tparam T                    <b>[inferred]</b> The data type to store.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
+ */
+template <
+    int                 BLOCK_THREADS,
+    typename            T,
+    int                 ITEMS_PER_THREAD,
+    typename            OutputIteratorT>
+__device__ __forceinline__ void StoreDirectStriped(
+    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+    T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
+    int                 valid_items)                ///< [in] Number of valid items to write
+{
+    OutputIteratorT thread_itr = block_itr + linear_tid;
+
+    // Store directly in striped order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        if ((ITEM * BLOCK_THREADS) + linear_tid < valid_items)
+        {
+            thread_itr[(ITEM * BLOCK_THREADS)] = items[ITEM];
+        }
+    }
+}
+
+
+
+//@}  end member group
+/******************************************************************//**
+ * \name Warp-striped arrangement I/O (direct)
+ *********************************************************************/
+//@{
+
+
+/**
+ * \brief Store a warp-striped arrangement of data across the thread block into a linear segment of items.
+ *
+ * \warpstriped
+ *
+ * \par Usage Considerations
+ * The number of threads in the thread block must be a multiple of the architecture's warp size.
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to store.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
+ */
+template <
+    typename            T,
+    int                 ITEMS_PER_THREAD,
+    typename            OutputIteratorT>
+__device__ __forceinline__ void StoreDirectWarpStriped(
+    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+    T                   (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
+{
+    int tid         = linear_tid & (CUB_PTX_WARP_THREADS - 1);
+    int wid         = linear_tid >> CUB_PTX_LOG_WARP_THREADS;
+    int warp_offset = wid * CUB_PTX_WARP_THREADS * ITEMS_PER_THREAD;
+
+    OutputIteratorT thread_itr = block_itr + warp_offset + tid;
+
+    // Store directly in warp-striped order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        thread_itr[(ITEM * CUB_PTX_WARP_THREADS)] = items[ITEM];
+    }
+}
+
+
+/**
+ * \brief Store a warp-striped arrangement of data across the thread block into a linear segment of items, guarded by range
+ *
+ * \warpstriped
+ *
+ * \par Usage Considerations
+ * The number of threads in the thread block must be a multiple of the architecture's warp size.
+ *
+ * \tparam T                    <b>[inferred]</b> The data type to store.
+ * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
+ * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
+ */
+template <
+    typename            T,
+    int                 ITEMS_PER_THREAD,
+    typename            OutputIteratorT>
+__device__ __forceinline__ void StoreDirectWarpStriped(
+    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
+    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+    T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
+    int                 valid_items)                ///< [in] Number of valid items to write
+{
+    int tid         = linear_tid & (CUB_PTX_WARP_THREADS - 1);
+    int wid         = linear_tid >> CUB_PTX_LOG_WARP_THREADS;
+    int warp_offset = wid * CUB_PTX_WARP_THREADS * ITEMS_PER_THREAD;
+
+    OutputIteratorT thread_itr = block_itr + warp_offset + tid;
+
+    // Store directly in warp-striped order
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+    {
+        if (warp_offset + tid + (ITEM * CUB_PTX_WARP_THREADS) < valid_items)
+        {
+            thread_itr[(ITEM * CUB_PTX_WARP_THREADS)] = items[ITEM];
+        }
+    }
+}
+
+
+//@}  end member group
+
+
+/** @} */       // end group UtilIo
+
+
+//-----------------------------------------------------------------------------
+// Generic BlockStore abstraction
+//-----------------------------------------------------------------------------
+
+/**
+ * \brief cub::BlockStoreAlgorithm enumerates alternative algorithms for cub::BlockStore to write a blocked arrangement of items across a CUDA thread block to a linear segment of memory.
+ */
+enum BlockStoreAlgorithm
+{
+    /**
+     * \par Overview
+     *
+     * A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is written
+     * directly to memory.
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) decreases as the
+     *   access stride between threads increases (i.e., the number items per thread).
+     */
+    BLOCK_STORE_DIRECT,
+
+    /**
+     * \par Overview
+     *
+     * A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is written directly
+     * to memory using CUDA's built-in vectorized stores as a coalescing optimization.
+     * For example, <tt>st.global.v4.s32</tt> instructions will be generated
+     * when \p T = \p int and \p ITEMS_PER_THREAD % 4 == 0.
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) remains high until the the
+     *   access stride between threads (i.e., the number items per thread) exceeds the
+     *   maximum vector store width (typically 4 items or 64B, whichever is lower).
+     * - The following conditions will prevent vectorization and writing will fall back to cub::BLOCK_STORE_DIRECT:
+     *   - \p ITEMS_PER_THREAD is odd
+     *   - The \p OutputIteratorT is not a simple pointer type
+     *   - The block output offset is not quadword-aligned
+     *   - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.)
+     */
+    BLOCK_STORE_VECTORIZE,
+
+    /**
+     * \par Overview
+     * A [<em>blocked arrangement</em>](index.html#sec5sec3) is locally
+     * transposed and then efficiently written to memory as a [<em>striped arrangement</em>](index.html#sec5sec3).
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) remains high regardless
+     *   of items written per thread.
+     * - The local reordering incurs slightly longer latencies and throughput than the
+     *   direct cub::BLOCK_STORE_DIRECT and cub::BLOCK_STORE_VECTORIZE alternatives.
+     */
+    BLOCK_STORE_TRANSPOSE,
+
+    /**
+     * \par Overview
+     * A [<em>blocked arrangement</em>](index.html#sec5sec3) is locally
+     * transposed and then efficiently written to memory as a
+     * [<em>warp-striped arrangement</em>](index.html#sec5sec3)
+     *
+     * \par Usage Considerations
+     * - BLOCK_THREADS must be a multiple of WARP_THREADS
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) remains high regardless
+     *   of items written per thread.
+     * - The local reordering incurs slightly longer latencies and throughput than the
+     *   direct cub::BLOCK_STORE_DIRECT and cub::BLOCK_STORE_VECTORIZE alternatives.
+     */
+    BLOCK_STORE_WARP_TRANSPOSE,
+
+    /**
+     * \par Overview
+     * A [<em>blocked arrangement</em>](index.html#sec5sec3) is locally
+     * transposed and then efficiently written to memory as a
+     * [<em>warp-striped arrangement</em>](index.html#sec5sec3)
+     * To reduce the shared memory requirement, only one warp's worth of shared
+     * memory is provisioned and is subsequently time-sliced among warps.
+     *
+     * \par Usage Considerations
+     * - BLOCK_THREADS must be a multiple of WARP_THREADS
+     *
+     * \par Performance Considerations
+     * - The utilization of memory transactions (coalescing) remains high regardless
+     *   of items written per thread.
+     * - Provisions less shared memory temporary storage, but incurs larger
+     *   latencies than the BLOCK_STORE_WARP_TRANSPOSE alternative.
+     */
+    BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED,
+
+};
+
+
+/**
+ * \brief The BlockStore class provides [<em>collective</em>](index.html#sec0) data movement methods for writing a [<em>blocked arrangement</em>](index.html#sec5sec3) of items partitioned across a CUDA thread block to a linear segment of memory.  ![](block_store_logo.png)
+ * \ingroup BlockModule
+ * \ingroup UtilIo
+ *
+ * \tparam T                    The type of data to be written.
+ * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
+ * \tparam ITEMS_PER_THREAD     The number of consecutive items partitioned onto each thread.
+ * \tparam ALGORITHM            <b>[optional]</b> cub::BlockStoreAlgorithm tuning policy enumeration.  default: cub::BLOCK_STORE_DIRECT.
+ * \tparam WARP_TIME_SLICING    <b>[optional]</b> Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any load-related data transpositions (versus each warp having its own storage). (default: false)
+ * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
+ * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
+ * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - The BlockStore class provides a single data movement abstraction that can be specialized
+ *   to implement different cub::BlockStoreAlgorithm strategies.  This facilitates different
+ *   performance policies for different architectures, data types, granularity sizes, etc.
+ * - BlockStore can be optionally specialized by different data movement strategies:
+ *   -# <b>cub::BLOCK_STORE_DIRECT</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is written
+ *      directly to memory. [More...](\ref cub::BlockStoreAlgorithm)
+ *   -# <b>cub::BLOCK_STORE_VECTORIZE</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
+ *      of data is written directly to memory using CUDA's built-in vectorized stores as a
+ *      coalescing optimization.  [More...](\ref cub::BlockStoreAlgorithm)
+ *   -# <b>cub::BLOCK_STORE_TRANSPOSE</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
+ *      is locally transposed into a [<em>striped arrangement</em>](index.html#sec5sec3) which is
+ *      then written to memory.  [More...](\ref cub::BlockStoreAlgorithm)
+ *   -# <b>cub::BLOCK_STORE_WARP_TRANSPOSE</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
+ *      is locally transposed into a [<em>warp-striped arrangement</em>](index.html#sec5sec3) which is
+ *      then written to memory.  [More...](\ref cub::BlockStoreAlgorithm)
+ * - \rowmajor
+ *
+ * \par A Simple Example
+ * \blockcollective{BlockStore}
+ * \par
+ * The code snippet below illustrates the storing of a "blocked" arrangement
+ * of 512 integers across 128 threads (where each thread owns 4 consecutive items)
+ * into a linear segment of memory.  The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE,
+ * meaning items are locally reordered among threads so that memory references will be
+ * efficiently coalesced using a warp-striped access pattern.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/block/block_store.cuh>
+ *
+ * __global__ void ExampleKernel(int *d_data, ...)
+ * {
+ *     // Specialize BlockStore for a 1D block of 128 threads owning 4 integer items each
+ *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_WARP_TRANSPOSE> BlockStore;
+ *
+ *     // Allocate shared memory for BlockStore
+ *     __shared__ typename BlockStore::TempStorage temp_storage;
+ *
+ *     // Obtain a segment of consecutive items that are blocked across threads
+ *     int thread_data[4];
+ *     ...
+ *
+ *     // Store items to linear memory
+ *     int thread_data[4];
+ *     BlockStore(temp_storage).Store(d_data, thread_data);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of \p thread_data across the block of threads is
+ * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
+ * The output \p d_data will be <tt>0, 1, 2, 3, 4, 5, ...</tt>.
+ *
+ */
+template <
+    typename                T,
+    int                     BLOCK_DIM_X,
+    int                     ITEMS_PER_THREAD,
+    BlockStoreAlgorithm     ALGORITHM           = BLOCK_STORE_DIRECT,
+    int                     BLOCK_DIM_Y         = 1,
+    int                     BLOCK_DIM_Z         = 1,
+    int                     PTX_ARCH            = CUB_PTX_ARCH>
+class BlockStore
+{
+private:
+    /******************************************************************************
+     * Constants and typed definitions
+     ******************************************************************************/
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+
+    /******************************************************************************
+     * Algorithmic variants
+     ******************************************************************************/
+
+    /// Store helper
+    template <BlockStoreAlgorithm _POLICY, int DUMMY>
+    struct StoreInternal;
+
+
+    /**
+     * BLOCK_STORE_DIRECT specialization of store helper
+     */
+    template <int DUMMY>
+    struct StoreInternal<BLOCK_STORE_DIRECT, DUMMY>
+    {
+        /// Shared memory storage layout type
+        typedef NullType TempStorage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ StoreInternal(
+            TempStorage &/*temp_storage*/,
+            int linear_tid)
+        :
+            linear_tid(linear_tid)
+        {}
+
+        /// Store items into a linear segment of memory
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+        {
+            StoreDirectBlocked(linear_tid, block_itr, items);
+        }
+
+        /// Store items into a linear segment of memory, guarded by range
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
+            int                 valid_items)                ///< [in] Number of valid items to write
+        {
+            StoreDirectBlocked(linear_tid, block_itr, items, valid_items);
+        }
+    };
+
+
+    /**
+     * BLOCK_STORE_VECTORIZE specialization of store helper
+     */
+    template <int DUMMY>
+    struct StoreInternal<BLOCK_STORE_VECTORIZE, DUMMY>
+    {
+        /// Shared memory storage layout type
+        typedef NullType TempStorage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ StoreInternal(
+            TempStorage &/*temp_storage*/,
+            int linear_tid)
+        :
+            linear_tid(linear_tid)
+        {}
+
+        /// Store items into a linear segment of memory, specialized for native pointer types (attempts vectorization)
+        __device__ __forceinline__ void Store(
+            T                   *block_ptr,                 ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+        {
+            StoreDirectBlockedVectorized(linear_tid, block_ptr, items);
+        }
+
+        /// Store items into a linear segment of memory, specialized for opaque input iterators (skips vectorization)
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT    block_itr,                  ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+        {
+            StoreDirectBlocked(linear_tid, block_itr, items);
+        }
+
+        /// Store items into a linear segment of memory, guarded by range
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
+            int                 valid_items)                ///< [in] Number of valid items to write
+        {
+            StoreDirectBlocked(linear_tid, block_itr, items, valid_items);
+        }
+    };
+
+
+    /**
+     * BLOCK_STORE_TRANSPOSE specialization of store helper
+     */
+    template <int DUMMY>
+    struct StoreInternal<BLOCK_STORE_TRANSPOSE, DUMMY>
+    {
+        // BlockExchange utility type for keys
+        typedef BlockExchange<T, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
+
+        /// Shared memory storage layout type
+        struct _TempStorage : BlockExchange::TempStorage
+        {
+            /// Temporary storage for partially-full block guard
+            volatile int valid_items;
+        };
+
+        /// Alias wrapper allowing storage to be unioned
+        struct TempStorage : Uninitialized<_TempStorage> {};
+
+        /// Thread reference to shared storage
+        _TempStorage &temp_storage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ StoreInternal(
+            TempStorage &temp_storage,
+            int linear_tid)
+        :
+            temp_storage(temp_storage.Alias()),
+            linear_tid(linear_tid)
+        {}
+
+        /// Store items into a linear segment of memory
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+        {
+            BlockExchange(temp_storage).BlockedToStriped(items);
+            StoreDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items);
+        }
+
+        /// Store items into a linear segment of memory, guarded by range
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT   block_itr,                  ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
+            int                 valid_items)                ///< [in] Number of valid items to write
+        {
+            BlockExchange(temp_storage).BlockedToStriped(items);
+            if (linear_tid == 0)
+                temp_storage.valid_items = valid_items;     // Move through volatile smem as a workaround to prevent RF spilling on subsequent loads
+            CTA_SYNC();
+            StoreDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items, temp_storage.valid_items);
+        }
+    };
+
+
+    /**
+     * BLOCK_STORE_WARP_TRANSPOSE specialization of store helper
+     */
+    template <int DUMMY>
+    struct StoreInternal<BLOCK_STORE_WARP_TRANSPOSE, DUMMY>
+    {
+        enum
+        {
+            WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH)
+        };
+
+        // Assert BLOCK_THREADS must be a multiple of WARP_THREADS
+        CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS");
+
+        // BlockExchange utility type for keys
+        typedef BlockExchange<T, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
+
+        /// Shared memory storage layout type
+        struct _TempStorage : BlockExchange::TempStorage
+        {
+            /// Temporary storage for partially-full block guard
+            volatile int valid_items;
+        };
+
+        /// Alias wrapper allowing storage to be unioned
+        struct TempStorage : Uninitialized<_TempStorage> {};
+
+        /// Thread reference to shared storage
+        _TempStorage &temp_storage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ StoreInternal(
+            TempStorage &temp_storage,
+            int linear_tid)
+        :
+            temp_storage(temp_storage.Alias()),
+            linear_tid(linear_tid)
+        {}
+
+        /// Store items into a linear segment of memory
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT   block_itr,                    ///< [in] The thread block's base output iterator for storing to
+            T                 (&items)[ITEMS_PER_THREAD])   ///< [in] Data to store
+        {
+            BlockExchange(temp_storage).BlockedToWarpStriped(items);
+            StoreDirectWarpStriped(linear_tid, block_itr, items);
+        }
+
+        /// Store items into a linear segment of memory, guarded by range
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT   block_itr,                    ///< [in] The thread block's base output iterator for storing to
+            T                 (&items)[ITEMS_PER_THREAD],   ///< [in] Data to store
+            int               valid_items)                  ///< [in] Number of valid items to write
+        {
+            BlockExchange(temp_storage).BlockedToWarpStriped(items);
+            if (linear_tid == 0)
+                temp_storage.valid_items = valid_items;     // Move through volatile smem as a workaround to prevent RF spilling on subsequent loads
+            CTA_SYNC();
+            StoreDirectWarpStriped(linear_tid, block_itr, items, temp_storage.valid_items);
+        }
+    };
+
+
+    /**
+     * BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED specialization of store helper
+     */
+    template <int DUMMY>
+    struct StoreInternal<BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED, DUMMY>
+    {
+        enum
+        {
+            WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH)
+        };
+
+        // Assert BLOCK_THREADS must be a multiple of WARP_THREADS
+        CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS");
+
+        // BlockExchange utility type for keys
+        typedef BlockExchange<T, BLOCK_DIM_X, ITEMS_PER_THREAD, true, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
+
+        /// Shared memory storage layout type
+        struct _TempStorage : BlockExchange::TempStorage
+        {
+            /// Temporary storage for partially-full block guard
+            volatile int valid_items;
+        };
+
+        /// Alias wrapper allowing storage to be unioned
+        struct TempStorage : Uninitialized<_TempStorage> {};
+
+        /// Thread reference to shared storage
+        _TempStorage &temp_storage;
+
+        /// Linear thread-id
+        int linear_tid;
+
+        /// Constructor
+        __device__ __forceinline__ StoreInternal(
+            TempStorage &temp_storage,
+            int linear_tid)
+        :
+            temp_storage(temp_storage.Alias()),
+            linear_tid(linear_tid)
+        {}
+
+        /// Store items into a linear segment of memory
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+        {
+            BlockExchange(temp_storage).BlockedToWarpStriped(items);
+            StoreDirectWarpStriped(linear_tid, block_itr, items);
+        }
+
+        /// Store items into a linear segment of memory, guarded by range
+        template <typename OutputIteratorT>
+        __device__ __forceinline__ void Store(
+            OutputIteratorT   block_itr,                  ///< [in] The thread block's base output iterator for storing to
+            T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
+            int                 valid_items)                ///< [in] Number of valid items to write
+        {
+            BlockExchange(temp_storage).BlockedToWarpStriped(items);
+            if (linear_tid == 0)
+                temp_storage.valid_items = valid_items;     // Move through volatile smem as a workaround to prevent RF spilling on subsequent loads
+            CTA_SYNC();
+            StoreDirectWarpStriped(linear_tid, block_itr, items, temp_storage.valid_items);
+        }
+    };
+
+    /******************************************************************************
+     * Type definitions
+     ******************************************************************************/
+
+    /// Internal load implementation to use
+    typedef StoreInternal<ALGORITHM, 0> InternalStore;
+
+
+    /// Shared memory storage layout type
+    typedef typename InternalStore::TempStorage _TempStorage;
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Internal storage allocator
+    __device__ __forceinline__ _TempStorage& PrivateStorage()
+    {
+        __shared__ _TempStorage private_storage;
+        return private_storage;
+    }
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Thread reference to shared storage
+    _TempStorage &temp_storage;
+
+    /// Linear thread-id
+    int linear_tid;
+
+public:
+
+
+    /// \smemstorage{BlockStore}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
+     */
+    __device__ __forceinline__ BlockStore()
+    :
+        temp_storage(PrivateStorage()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.
+     */
+    __device__ __forceinline__ BlockStore(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Data movement
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Store items into a linear segment of memory.
+     *
+     * \par
+     * - \blocked
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the storing of a "blocked" arrangement
+     * of 512 integers across 128 threads (where each thread owns 4 consecutive items)
+     * into a linear segment of memory.  The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE,
+     * meaning items are locally reordered among threads so that memory references will be
+     * efficiently coalesced using a warp-striped access pattern.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_store.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, ...)
+     * {
+     *     // Specialize BlockStore for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_WARP_TRANSPOSE> BlockStore;
+     *
+     *     // Allocate shared memory for BlockStore
+     *     __shared__ typename BlockStore::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Store items to linear memory
+     *     int thread_data[4];
+     *     BlockStore(temp_storage).Store(d_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of \p thread_data across the block of threads is
+     * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
+     * The output \p d_data will be <tt>0, 1, 2, 3, 4, 5, ...</tt>.
+     *
+     */
+    template <typename OutputIteratorT>
+    __device__ __forceinline__ void Store(
+        OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+        T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
+    {
+        InternalStore(temp_storage, linear_tid).Store(block_itr, items);
+    }
+
+    /**
+     * \brief Store items into a linear segment of memory, guarded by range.
+     *
+     * \par
+     * - \blocked
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the guarded storing of a "blocked" arrangement
+     * of 512 integers across 128 threads (where each thread owns 4 consecutive items)
+     * into a linear segment of memory.  The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE,
+     * meaning items are locally reordered among threads so that memory references will be
+     * efficiently coalesced using a warp-striped access pattern.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_store.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, int valid_items, ...)
+     * {
+     *     // Specialize BlockStore for a 1D block of 128 threads owning 4 integer items each
+     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_WARP_TRANSPOSE> BlockStore;
+     *
+     *     // Allocate shared memory for BlockStore
+     *     __shared__ typename BlockStore::TempStorage temp_storage;
+     *
+     *     // Obtain a segment of consecutive items that are blocked across threads
+     *     int thread_data[4];
+     *     ...
+     *
+     *     // Store items to linear memory
+     *     int thread_data[4];
+     *     BlockStore(temp_storage).Store(d_data, thread_data, valid_items);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of \p thread_data across the block of threads is
+     * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt> and \p valid_items is \p 5.
+     * The output \p d_data will be <tt>0, 1, 2, 3, 4, ?, ?, ?, ...</tt>, with
+     * only the first two threads being unmasked to store portions of valid data.
+     *
+     */
+    template <typename OutputIteratorT>
+    __device__ __forceinline__ void Store(
+        OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
+        T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
+        int                 valid_items)                ///< [in] Number of valid items to write
+    {
+        InternalStore(temp_storage, linear_tid).Store(block_itr, items, valid_items);
+    }
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_histogram_atomic.cuh b/third_party/cub/cub/block/specializations/block_histogram_atomic.cuh
new file mode 100644
index 0000000..29db0df
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_histogram_atomic.cuh
@@ -0,0 +1,82 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockHistogramAtomic class provides atomic-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief The BlockHistogramAtomic class provides atomic-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
+ */
+template <int BINS>
+struct BlockHistogramAtomic
+{
+    /// Shared memory storage layout type
+    struct TempStorage {};
+
+
+    /// Constructor
+    __device__ __forceinline__ BlockHistogramAtomic(
+        TempStorage &temp_storage)
+    {}
+
+
+    /// Composite data onto an existing histogram
+    template <
+        typename            T,
+        typename            CounterT,     
+        int                 ITEMS_PER_THREAD>
+    __device__ __forceinline__ void Composite(
+        T                   (&items)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input values to histogram
+        CounterT             histogram[BINS])                 ///< [out] Reference to shared/device-accessible memory histogram
+    {
+        // Update histogram
+        #pragma unroll
+        for (int i = 0; i < ITEMS_PER_THREAD; ++i)
+        {
+              atomicAdd(histogram + items[i], 1);
+        }
+    }
+
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_histogram_sort.cuh b/third_party/cub/cub/block/specializations/block_histogram_sort.cuh
new file mode 100644
index 0000000..9ef417a
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_histogram_sort.cuh
@@ -0,0 +1,226 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::BlockHistogramSort class provides sorting-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../../block/block_radix_sort.cuh"
+#include "../../block/block_discontinuity.cuh"
+#include "../../util_ptx.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+
+/**
+ * \brief The BlockHistogramSort class provides sorting-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
+ */
+template <
+    typename    T,                  ///< Sample type
+    int         BLOCK_DIM_X,        ///< The thread block length in threads along the X dimension
+    int         ITEMS_PER_THREAD,   ///< The number of samples per thread
+    int         BINS,               ///< The number of bins into which histogram samples may fall
+    int         BLOCK_DIM_Y,        ///< The thread block length in threads along the Y dimension
+    int         BLOCK_DIM_Z,        ///< The thread block length in threads along the Z dimension
+    int         PTX_ARCH>           ///< The PTX compute capability for which to to specialize this collective
+struct BlockHistogramSort
+{
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+    // Parameterize BlockRadixSort type for our thread block
+    typedef BlockRadixSort<
+            T,
+            BLOCK_DIM_X,
+            ITEMS_PER_THREAD,
+            NullType,
+            4,
+            (PTX_ARCH >= 350) ? true : false,
+            BLOCK_SCAN_WARP_SCANS,
+            cudaSharedMemBankSizeFourByte,
+            BLOCK_DIM_Y,
+            BLOCK_DIM_Z,
+            PTX_ARCH>
+        BlockRadixSortT;
+
+    // Parameterize BlockDiscontinuity type for our thread block
+    typedef BlockDiscontinuity<
+            T,
+            BLOCK_DIM_X,
+            BLOCK_DIM_Y,
+            BLOCK_DIM_Z,
+            PTX_ARCH>
+        BlockDiscontinuityT;
+
+    /// Shared memory
+    union _TempStorage
+    {
+        // Storage for sorting bin values
+        typename BlockRadixSortT::TempStorage sort;
+
+        struct
+        {
+            // Storage for detecting discontinuities in the tile of sorted bin values
+            typename BlockDiscontinuityT::TempStorage flag;
+
+            // Storage for noting begin/end offsets of bin runs in the tile of sorted bin values
+            unsigned int run_begin[BINS];
+            unsigned int run_end[BINS];
+        };
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    // Thread fields
+    _TempStorage &temp_storage;
+    unsigned int linear_tid;
+
+
+    /// Constructor
+    __device__ __forceinline__ BlockHistogramSort(
+        TempStorage     &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    // Discontinuity functor
+    struct DiscontinuityOp
+    {
+        // Reference to temp_storage
+        _TempStorage &temp_storage;
+
+        // Constructor
+        __device__ __forceinline__ DiscontinuityOp(_TempStorage &temp_storage) :
+            temp_storage(temp_storage)
+        {}
+
+        // Discontinuity predicate
+        __device__ __forceinline__ bool operator()(const T &a, const T &b, int b_index)
+        {
+            if (a != b)
+            {
+                // Note the begin/end offsets in shared storage
+                temp_storage.run_begin[b] = b_index;
+                temp_storage.run_end[a] = b_index;
+
+                return true;
+            }
+            else
+            {
+                return false;
+            }
+        }
+    };
+
+
+    // Composite data onto an existing histogram
+    template <
+        typename            CounterT     >
+    __device__ __forceinline__ void Composite(
+        T                   (&items)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input values to histogram
+        CounterT            histogram[BINS])                 ///< [out] Reference to shared/device-accessible memory histogram
+    {
+        enum { TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD };
+
+        // Sort bytes in blocked arrangement
+        BlockRadixSortT(temp_storage.sort).Sort(items);
+
+        CTA_SYNC();
+
+        // Initialize the shared memory's run_begin and run_end for each bin
+        int histo_offset = 0;
+
+        #pragma unroll
+        for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS)
+        {
+            temp_storage.run_begin[histo_offset + linear_tid] = TILE_SIZE;
+            temp_storage.run_end[histo_offset + linear_tid] = TILE_SIZE;
+        }
+        // Finish up with guarded initialization if necessary
+        if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS))
+        {
+            temp_storage.run_begin[histo_offset + linear_tid] = TILE_SIZE;
+            temp_storage.run_end[histo_offset + linear_tid] = TILE_SIZE;
+        }
+
+        CTA_SYNC();
+
+        int flags[ITEMS_PER_THREAD];    // unused
+
+        // Compute head flags to demarcate contiguous runs of the same bin in the sorted tile
+        DiscontinuityOp flag_op(temp_storage);
+        BlockDiscontinuityT(temp_storage.flag).FlagHeads(flags, items, flag_op);
+
+        // Update begin for first item
+        if (linear_tid == 0) temp_storage.run_begin[items[0]] = 0;
+
+        CTA_SYNC();
+
+        // Composite into histogram
+        histo_offset = 0;
+
+        #pragma unroll
+        for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS)
+        {
+            int thread_offset = histo_offset + linear_tid;
+            CounterT      count = temp_storage.run_end[thread_offset] - temp_storage.run_begin[thread_offset];
+            histogram[thread_offset] += count;
+        }
+
+        // Finish up with guarded composition if necessary
+        if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS))
+        {
+            int thread_offset = histo_offset + linear_tid;
+            CounterT      count = temp_storage.run_end[thread_offset] - temp_storage.run_begin[thread_offset];
+            histogram[thread_offset] += count;
+        }
+    }
+
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_reduce_raking.cuh b/third_party/cub/cub/block/specializations/block_reduce_raking.cuh
new file mode 100644
index 0000000..aff97fc
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_reduce_raking.cuh
@@ -0,0 +1,226 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::BlockReduceRaking provides raking-based methods of parallel reduction across a CUDA thread block.  Supports non-commutative reduction operators.
+ */
+
+#pragma once
+
+#include "../../block/block_raking_layout.cuh"
+#include "../../warp/warp_reduce.cuh"
+#include "../../thread/thread_reduce.cuh"
+#include "../../util_ptx.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief BlockReduceRaking provides raking-based methods of parallel reduction across a CUDA thread block.  Supports non-commutative reduction operators.
+ *
+ * Supports non-commutative binary reduction operators.  Unlike commutative
+ * reduction operators (e.g., addition), the application of a non-commutative
+ * reduction operator (e.g, string concatenation) across a sequence of inputs must
+ * honor the relative ordering of items and partial reductions when applying the
+ * reduction operator.
+ *
+ * Compared to the implementation of BlockReduceRaking (which does not support
+ * non-commutative operators), this implementation requires a few extra
+ * rounds of inter-thread communication.
+ */
+template <
+    typename    T,              ///< Data type being reduced
+    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
+    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
+    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
+    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
+struct BlockReduceRaking
+{
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+    /// Layout type for padded thread block raking grid
+    typedef BlockRakingLayout<T, BLOCK_THREADS, PTX_ARCH> BlockRakingLayout;
+
+    ///  WarpReduce utility type
+    typedef typename WarpReduce<T, BlockRakingLayout::RAKING_THREADS, PTX_ARCH>::InternalWarpReduce WarpReduce;
+
+    /// Constants
+    enum
+    {
+        /// Number of raking threads
+        RAKING_THREADS = BlockRakingLayout::RAKING_THREADS,
+
+        /// Number of raking elements per warp synchronous raking thread
+        SEGMENT_LENGTH = BlockRakingLayout::SEGMENT_LENGTH,
+
+        /// Cooperative work can be entirely warp synchronous
+        WARP_SYNCHRONOUS = (RAKING_THREADS == BLOCK_THREADS),
+
+        /// Whether or not warp-synchronous reduction should be unguarded (i.e., the warp-reduction elements is a power of two
+        WARP_SYNCHRONOUS_UNGUARDED = PowerOfTwo<RAKING_THREADS>::VALUE,
+
+        /// Whether or not accesses into smem are unguarded
+        RAKING_UNGUARDED = BlockRakingLayout::UNGUARDED,
+
+    };
+
+
+    /// Shared memory storage layout type
+    union _TempStorage
+    {
+        typename WarpReduce::TempStorage            warp_storage;        ///< Storage for warp-synchronous reduction
+        typename BlockRakingLayout::TempStorage     raking_grid;         ///< Padded thread block raking grid
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    // Thread fields
+    _TempStorage &temp_storage;
+    unsigned int linear_tid;
+
+
+    /// Constructor
+    __device__ __forceinline__ BlockReduceRaking(
+        TempStorage &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    template <bool IS_FULL_TILE, typename ReductionOp, int ITERATION>
+    __device__ __forceinline__ T RakingReduction(
+        ReductionOp                 reduction_op,       ///< [in] Binary scan operator
+        T                           *raking_segment,
+        T                           partial,            ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
+        int                         num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+        Int2Type<ITERATION>         /*iteration*/)
+    {
+        // Update partial if addend is in range
+        if ((IS_FULL_TILE && RAKING_UNGUARDED) || ((linear_tid * SEGMENT_LENGTH) + ITERATION < num_valid))
+        {
+            T addend = raking_segment[ITERATION];
+            partial = reduction_op(partial, addend);
+        }
+        return RakingReduction<IS_FULL_TILE>(reduction_op, raking_segment, partial, num_valid, Int2Type<ITERATION + 1>());
+    }
+
+    template <bool IS_FULL_TILE, typename ReductionOp>
+    __device__ __forceinline__ T RakingReduction(
+        ReductionOp                 /*reduction_op*/,   ///< [in] Binary scan operator
+        T                           * /*raking_segment*/,
+        T                           partial,            ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
+        int                         /*num_valid*/,      ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+        Int2Type<SEGMENT_LENGTH>    /*iteration*/)
+    {
+        return partial;
+    }
+
+
+
+    /// Computes a thread block-wide reduction using the specified reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
+    template <
+        bool                IS_FULL_TILE,
+        typename            ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T                   partial,            ///< [in] Calling thread's input partial reductions
+        int                 num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+        ReductionOp         reduction_op)       ///< [in] Binary reduction operator
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp synchronous reduction (unguarded if active threads is a power-of-two)
+            partial = WarpReduce(temp_storage.warp_storage).template Reduce<IS_FULL_TILE>(
+                partial,
+                num_valid,
+                reduction_op);
+        }
+        else
+        {
+            // Place partial into shared memory grid.
+            *BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid) = partial;
+
+            CTA_SYNC();
+
+            // Reduce parallelism to one warp
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking reduction in grid
+                T *raking_segment = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
+                partial = raking_segment[0];
+
+                partial = RakingReduction<IS_FULL_TILE>(reduction_op, raking_segment, partial, num_valid, Int2Type<1>());
+
+                int valid_raking_threads = (IS_FULL_TILE) ?
+                    RAKING_THREADS :
+                    (num_valid + SEGMENT_LENGTH - 1) / SEGMENT_LENGTH;
+
+                partial = WarpReduce(temp_storage.warp_storage).template Reduce<IS_FULL_TILE && RAKING_UNGUARDED>(
+                    partial,
+                    valid_raking_threads,
+                    reduction_op);
+
+            }
+        }
+
+        return partial;
+    }
+
+
+    /// Computes a thread block-wide reduction using addition (+) as the reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
+    template <bool IS_FULL_TILE>
+    __device__ __forceinline__ T Sum(
+        T                   partial,            ///< [in] Calling thread's input partial reductions
+        int                 num_valid)          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+    {
+        cub::Sum reduction_op;
+
+        return Reduce<IS_FULL_TILE>(partial, num_valid, reduction_op);
+    }
+
+
+
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_reduce_raking_commutative_only.cuh b/third_party/cub/cub/block/specializations/block_reduce_raking_commutative_only.cuh
new file mode 100644
index 0000000..454fdaf
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_reduce_raking_commutative_only.cuh
@@ -0,0 +1,199 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::BlockReduceRakingCommutativeOnly provides raking-based methods of parallel reduction across a CUDA thread block.  Does not support non-commutative reduction operators.
+ */
+
+#pragma once
+
+#include "block_reduce_raking.cuh"
+#include "../../warp/warp_reduce.cuh"
+#include "../../thread/thread_reduce.cuh"
+#include "../../util_ptx.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief BlockReduceRakingCommutativeOnly provides raking-based methods of parallel reduction across a CUDA thread block.  Does not support non-commutative reduction operators.  Does not support block sizes that are not a multiple of the warp size.
+ */
+template <
+    typename    T,              ///< Data type being reduced
+    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
+    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
+    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
+    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
+struct BlockReduceRakingCommutativeOnly
+{
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+    // The fall-back implementation to use when BLOCK_THREADS is not a multiple of the warp size or not all threads have valid values
+    typedef BlockReduceRaking<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> FallBack;
+
+    /// Constants
+    enum
+    {
+        /// Number of warp threads
+        WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
+
+        /// Whether or not to use fall-back
+        USE_FALLBACK = ((BLOCK_THREADS % WARP_THREADS != 0) || (BLOCK_THREADS <= WARP_THREADS)),
+
+        /// Number of raking threads
+        RAKING_THREADS = WARP_THREADS,
+
+        /// Number of threads actually sharing items with the raking threads
+        SHARING_THREADS = CUB_MAX(1, BLOCK_THREADS - RAKING_THREADS),
+
+        /// Number of raking elements per warp synchronous raking thread
+        SEGMENT_LENGTH = SHARING_THREADS / WARP_THREADS,
+    };
+
+    ///  WarpReduce utility type
+    typedef WarpReduce<T, RAKING_THREADS, PTX_ARCH> WarpReduce;
+
+    /// Layout type for padded thread block raking grid
+    typedef BlockRakingLayout<T, SHARING_THREADS, PTX_ARCH> BlockRakingLayout;
+
+    /// Shared memory storage layout type
+    union _TempStorage
+    {
+        struct
+        {
+            typename WarpReduce::TempStorage        warp_storage;        ///< Storage for warp-synchronous reduction
+            typename BlockRakingLayout::TempStorage raking_grid;         ///< Padded thread block raking grid
+        };
+        typename FallBack::TempStorage              fallback_storage;    ///< Fall-back storage for non-commutative block scan
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    // Thread fields
+    _TempStorage &temp_storage;
+    unsigned int linear_tid;
+
+
+    /// Constructor
+    __device__ __forceinline__ BlockReduceRakingCommutativeOnly(
+        TempStorage &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    /// Computes a thread block-wide reduction using addition (+) as the reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
+    template <bool FULL_TILE>
+    __device__ __forceinline__ T Sum(
+        T                   partial,            ///< [in] Calling thread's input partial reductions
+        int                 num_valid)          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+    {
+        if (USE_FALLBACK || !FULL_TILE)
+        {
+            return FallBack(temp_storage.fallback_storage).template Sum<FULL_TILE>(partial, num_valid);
+        }
+        else
+        {
+            // Place partial into shared memory grid
+            if (linear_tid >= RAKING_THREADS)
+                *BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid - RAKING_THREADS) = partial;
+
+            CTA_SYNC();
+
+            // Reduce parallelism to one warp
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking reduction in grid
+                T *raking_segment = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
+                partial = internal::ThreadReduce<SEGMENT_LENGTH>(raking_segment, cub::Sum(), partial);
+
+                // Warpscan
+                partial = WarpReduce(temp_storage.warp_storage).Sum(partial);
+            }
+        }
+
+        return partial;
+    }
+
+
+    /// Computes a thread block-wide reduction using the specified reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
+    template <
+        bool                FULL_TILE,
+        typename            ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T                   partial,            ///< [in] Calling thread's input partial reductions
+        int                 num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+        ReductionOp         reduction_op)       ///< [in] Binary reduction operator
+    {
+        if (USE_FALLBACK || !FULL_TILE)
+        {
+            return FallBack(temp_storage.fallback_storage).template Reduce<FULL_TILE>(partial, num_valid, reduction_op);
+        }
+        else
+        {
+            // Place partial into shared memory grid
+            if (linear_tid >= RAKING_THREADS)
+                *BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid - RAKING_THREADS) = partial;
+
+            CTA_SYNC();
+
+            // Reduce parallelism to one warp
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking reduction in grid
+                T *raking_segment = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
+                partial = internal::ThreadReduce<SEGMENT_LENGTH>(raking_segment, reduction_op, partial);
+
+                // Warpscan
+                partial = WarpReduce(temp_storage.warp_storage).Reduce(partial, reduction_op);
+            }
+        }
+
+        return partial;
+    }
+
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_reduce_warp_reductions.cuh b/third_party/cub/cub/block/specializations/block_reduce_warp_reductions.cuh
new file mode 100644
index 0000000..10ba303
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_reduce_warp_reductions.cuh
@@ -0,0 +1,218 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::BlockReduceWarpReductions provides variants of warp-reduction-based parallel reduction across a CUDA thread block.  Supports non-commutative reduction operators.
+ */
+
+#pragma once
+
+#include "../../warp/warp_reduce.cuh"
+#include "../../util_ptx.cuh"
+#include "../../util_arch.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief BlockReduceWarpReductions provides variants of warp-reduction-based parallel reduction across a CUDA thread block.  Supports non-commutative reduction operators.
+ */
+template <
+    typename    T,              ///< Data type being reduced
+    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
+    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
+    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
+    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
+struct BlockReduceWarpReductions
+{
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        /// Number of warp threads
+        WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
+
+        /// Number of active warps
+        WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+
+        /// The logical warp size for warp reductions
+        LOGICAL_WARP_SIZE = CUB_MIN(BLOCK_THREADS, WARP_THREADS),
+
+        /// Whether or not the logical warp size evenly divides the thread block size
+        EVEN_WARP_MULTIPLE = (BLOCK_THREADS % LOGICAL_WARP_SIZE == 0)
+    };
+
+
+    ///  WarpReduce utility type
+    typedef typename WarpReduce<T, LOGICAL_WARP_SIZE, PTX_ARCH>::InternalWarpReduce WarpReduce;
+
+
+    /// Shared memory storage layout type
+    struct _TempStorage
+    {
+        typename WarpReduce::TempStorage    warp_reduce[WARPS];         ///< Buffer for warp-synchronous scan
+        T                                   warp_aggregates[WARPS];     ///< Shared totals from each warp-synchronous scan
+        T                                   block_prefix;               ///< Shared prefix for the entire thread block
+    };
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    // Thread fields
+    _TempStorage &temp_storage;
+    int linear_tid;
+    int warp_id;
+    int lane_id;
+
+
+    /// Constructor
+    __device__ __forceinline__ BlockReduceWarpReductions(
+        TempStorage &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
+        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
+        lane_id(LaneId())
+    {}
+
+
+    template <bool FULL_TILE, typename ReductionOp, int SUCCESSOR_WARP>
+    __device__ __forceinline__ T ApplyWarpAggregates(
+        ReductionOp                 reduction_op,       ///< [in] Binary scan operator
+        T                           warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
+        int                         num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+        Int2Type<SUCCESSOR_WARP>    /*successor_warp*/)
+    {
+        if (FULL_TILE || (SUCCESSOR_WARP * LOGICAL_WARP_SIZE < num_valid))
+        {
+            T addend = temp_storage.warp_aggregates[SUCCESSOR_WARP];
+            warp_aggregate = reduction_op(warp_aggregate, addend);
+        }
+        return ApplyWarpAggregates<FULL_TILE>(reduction_op, warp_aggregate, num_valid, Int2Type<SUCCESSOR_WARP + 1>());
+    }
+
+    template <bool FULL_TILE, typename ReductionOp>
+    __device__ __forceinline__ T ApplyWarpAggregates(
+        ReductionOp         /*reduction_op*/,   ///< [in] Binary scan operator
+        T                   warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
+        int                 /*num_valid*/,      ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+        Int2Type<WARPS>     /*successor_warp*/)
+    {
+        return warp_aggregate;
+    }
+
+
+    /// Returns block-wide aggregate in <em>thread</em><sub>0</sub>.
+    template <
+        bool                FULL_TILE,
+        typename            ReductionOp>
+    __device__ __forceinline__ T ApplyWarpAggregates(
+        ReductionOp         reduction_op,       ///< [in] Binary scan operator
+        T                   warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
+        int                 num_valid)          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+    {
+        // Share lane aggregates
+        if (lane_id == 0)
+        {
+            temp_storage.warp_aggregates[warp_id] = warp_aggregate;
+        }
+
+        CTA_SYNC();
+
+        // Update total aggregate in warp 0, lane 0
+        if (linear_tid == 0)
+        {
+            warp_aggregate = ApplyWarpAggregates<FULL_TILE>(reduction_op, warp_aggregate, num_valid, Int2Type<1>());
+        }
+
+        return warp_aggregate;
+    }
+
+
+    /// Computes a thread block-wide reduction using addition (+) as the reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
+    template <bool FULL_TILE>
+    __device__ __forceinline__ T Sum(
+        T                   input,          ///< [in] Calling thread's input partial reductions
+        int                 num_valid)      ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+    {
+        cub::Sum    reduction_op;
+        int         warp_offset = (warp_id * LOGICAL_WARP_SIZE);
+        int         warp_num_valid = ((FULL_TILE && EVEN_WARP_MULTIPLE) || (warp_offset + LOGICAL_WARP_SIZE <= num_valid)) ?
+                            LOGICAL_WARP_SIZE :
+                            num_valid - warp_offset;
+
+        // Warp reduction in every warp
+        T warp_aggregate = WarpReduce(temp_storage.warp_reduce[warp_id]).template Reduce<(FULL_TILE && EVEN_WARP_MULTIPLE)>(
+            input,
+            warp_num_valid,
+            cub::Sum());
+
+        // Update outputs and block_aggregate with warp-wide aggregates from lane-0s
+        return ApplyWarpAggregates<FULL_TILE>(reduction_op, warp_aggregate, num_valid);
+    }
+
+
+    /// Computes a thread block-wide reduction using the specified reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
+    template <
+        bool                FULL_TILE,
+        typename            ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T                   input,              ///< [in] Calling thread's input partial reductions
+        int                 num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
+        ReductionOp         reduction_op)       ///< [in] Binary reduction operator
+    {
+        int         warp_offset = warp_id * LOGICAL_WARP_SIZE;
+        int         warp_num_valid = ((FULL_TILE && EVEN_WARP_MULTIPLE) || (warp_offset + LOGICAL_WARP_SIZE <= num_valid)) ?
+                            LOGICAL_WARP_SIZE :
+                            num_valid - warp_offset;
+
+        // Warp reduction in every warp
+        T warp_aggregate = WarpReduce(temp_storage.warp_reduce[warp_id]).template Reduce<(FULL_TILE && EVEN_WARP_MULTIPLE)>(
+            input,
+            warp_num_valid,
+            reduction_op);
+
+        // Update outputs and block_aggregate with warp-wide aggregates from lane-0s
+        return ApplyWarpAggregates<FULL_TILE>(reduction_op, warp_aggregate, num_valid);
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_scan_raking.cuh b/third_party/cub/cub/block/specializations/block_scan_raking.cuh
new file mode 100644
index 0000000..a855cda
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_scan_raking.cuh
@@ -0,0 +1,666 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+
+/**
+ * \file
+ * cub::BlockScanRaking provides variants of raking-based parallel prefix scan across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../../util_ptx.cuh"
+#include "../../util_arch.cuh"
+#include "../../block/block_raking_layout.cuh"
+#include "../../thread/thread_reduce.cuh"
+#include "../../thread/thread_scan.cuh"
+#include "../../warp/warp_scan.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief BlockScanRaking provides variants of raking-based parallel prefix scan across a CUDA thread block.
+ */
+template <
+    typename    T,              ///< Data type being scanned
+    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
+    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
+    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
+    bool        MEMOIZE,        ///< Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure
+    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
+struct BlockScanRaking
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    };
+
+    /// Layout type for padded thread block raking grid
+    typedef BlockRakingLayout<T, BLOCK_THREADS, PTX_ARCH> BlockRakingLayout;
+
+    /// Constants
+    enum
+    {
+        /// Number of raking threads
+        RAKING_THREADS = BlockRakingLayout::RAKING_THREADS,
+
+        /// Number of raking elements per warp synchronous raking thread
+        SEGMENT_LENGTH = BlockRakingLayout::SEGMENT_LENGTH,
+
+        /// Cooperative work can be entirely warp synchronous
+        WARP_SYNCHRONOUS = (BLOCK_THREADS == RAKING_THREADS),
+    };
+
+    ///  WarpScan utility type
+    typedef WarpScan<T, RAKING_THREADS, PTX_ARCH> WarpScan;
+
+    /// Shared memory storage layout type
+    struct _TempStorage
+    {
+        typename WarpScan::TempStorage              warp_scan;          ///< Buffer for warp-synchronous scan
+        typename BlockRakingLayout::TempStorage     raking_grid;        ///< Padded thread block raking grid
+        T                                           block_aggregate;    ///< Block aggregate
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    // Thread fields
+    _TempStorage    &temp_storage;
+    unsigned int    linear_tid;
+    T               cached_segment[SEGMENT_LENGTH];
+
+
+    //---------------------------------------------------------------------
+    // Utility methods
+    //---------------------------------------------------------------------
+
+    /// Templated reduction
+    template <int ITERATION, typename ScanOp>
+    __device__ __forceinline__ T GuardedReduce(
+        T*                  raking_ptr,         ///< [in] Input array
+        ScanOp              scan_op,            ///< [in] Binary reduction operator
+        T                   raking_partial,     ///< [in] Prefix to seed reduction with
+        Int2Type<ITERATION> /*iteration*/)
+    {
+        if ((BlockRakingLayout::UNGUARDED) || (((linear_tid * SEGMENT_LENGTH) + ITERATION) < BLOCK_THREADS))
+        {
+            T addend = raking_ptr[ITERATION];
+            raking_partial = scan_op(raking_partial, addend);
+        }
+
+        return GuardedReduce(raking_ptr, scan_op, raking_partial, Int2Type<ITERATION + 1>());
+    }
+
+
+    /// Templated reduction (base case)
+    template <typename ScanOp>
+    __device__ __forceinline__ T GuardedReduce(
+        T*                          /*raking_ptr*/,    ///< [in] Input array
+        ScanOp                      /*scan_op*/,       ///< [in] Binary reduction operator
+        T                           raking_partial,    ///< [in] Prefix to seed reduction with
+        Int2Type<SEGMENT_LENGTH>    /*iteration*/)
+    {
+        return raking_partial;
+    }
+
+
+    /// Templated copy
+    template <int ITERATION>
+    __device__ __forceinline__ void CopySegment(
+        T*                  out,            ///< [out] Out array
+        T*                  in,             ///< [in] Input array
+        Int2Type<ITERATION> /*iteration*/)
+    {
+        out[ITERATION] = in[ITERATION];
+        CopySegment(out, in, Int2Type<ITERATION + 1>());
+    }
+
+ 
+    /// Templated copy (base case)
+    __device__ __forceinline__ void CopySegment(
+        T*                  /*out*/,            ///< [out] Out array
+        T*                  /*in*/,             ///< [in] Input array
+        Int2Type<SEGMENT_LENGTH> /*iteration*/)
+    {}
+
+
+    /// Performs upsweep raking reduction, returning the aggregate
+    template <typename ScanOp>
+    __device__ __forceinline__ T Upsweep(
+        ScanOp scan_op)
+    {
+        T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
+
+        // Read data into registers
+        CopySegment(cached_segment, smem_raking_ptr, Int2Type<0>());
+
+        T raking_partial = cached_segment[0];
+
+        return GuardedReduce(cached_segment, scan_op, raking_partial, Int2Type<1>());
+    }
+
+
+    /// Performs exclusive downsweep raking scan
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveDownsweep(
+        ScanOp          scan_op,
+        T               raking_partial,
+        bool            apply_prefix = true)
+    {
+        T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
+
+        // Read data back into registers
+        if (!MEMOIZE)
+        {
+            CopySegment(cached_segment, smem_raking_ptr, Int2Type<0>());
+        }
+
+        internal::ThreadScanExclusive(cached_segment, cached_segment, scan_op, raking_partial, apply_prefix);
+
+        // Write data back to smem
+        CopySegment(smem_raking_ptr, cached_segment, Int2Type<0>());
+    }
+
+
+    /// Performs inclusive downsweep raking scan
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveDownsweep(
+        ScanOp          scan_op,
+        T               raking_partial,
+        bool            apply_prefix = true)
+    {
+        T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
+
+        // Read data back into registers
+        if (!MEMOIZE)
+        {
+            CopySegment(cached_segment, smem_raking_ptr, Int2Type<0>());
+        }
+
+        internal::ThreadScanInclusive(cached_segment, cached_segment, scan_op, raking_partial, apply_prefix);
+
+        // Write data back to smem
+        CopySegment(smem_raking_ptr, cached_segment, Int2Type<0>());
+    }
+
+
+    //---------------------------------------------------------------------
+    // Constructors
+    //---------------------------------------------------------------------
+
+    /// Constructor
+    __device__ __forceinline__ BlockScanRaking(
+        TempStorage &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Exclusive scans
+    //---------------------------------------------------------------------
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &exclusive_output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan operator
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp-synchronous scan
+            WarpScan(temp_storage.warp_scan).ExclusiveScan(input, exclusive_output, scan_op);
+        }
+        else
+        {
+            // Place thread partial into shared memory raking grid
+            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
+            *placement_ptr = input;
+
+            CTA_SYNC();
+
+            // Reduce parallelism down to just raking threads
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking upsweep reduction across shared partials
+                T upsweep_partial = Upsweep(scan_op);
+
+                // Warp-synchronous scan
+                T exclusive_partial;
+                WarpScan(temp_storage.warp_scan).ExclusiveScan(upsweep_partial, exclusive_partial, scan_op);
+
+                // Exclusive raking downsweep scan
+                ExclusiveDownsweep(scan_op, exclusive_partial, (linear_tid != 0));
+            }
+
+            CTA_SYNC();
+
+            // Grab thread prefix from shared memory
+            exclusive_output = *placement_ptr;
+        }
+    }
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &output,            ///< [out] Calling thread's output items (may be aliased to \p input)
+        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp-synchronous scan
+            WarpScan(temp_storage.warp_scan).ExclusiveScan(input, output, initial_value, scan_op);
+        }
+        else
+        {
+            // Place thread partial into shared memory raking grid
+            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
+            *placement_ptr = input;
+
+            CTA_SYNC();
+
+            // Reduce parallelism down to just raking threads
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking upsweep reduction across shared partials
+                T upsweep_partial = Upsweep(scan_op);
+
+                // Exclusive Warp-synchronous scan
+                T exclusive_partial;
+                WarpScan(temp_storage.warp_scan).ExclusiveScan(upsweep_partial, exclusive_partial, initial_value, scan_op);
+
+                // Exclusive raking downsweep scan
+                ExclusiveDownsweep(scan_op, exclusive_partial);
+            }
+
+            CTA_SYNC();
+
+            // Grab exclusive partial from shared memory
+            output = *placement_ptr;
+        }
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan operator
+        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp-synchronous scan
+            WarpScan(temp_storage.warp_scan).ExclusiveScan(input, output, scan_op, block_aggregate);
+        }
+        else
+        {
+            // Place thread partial into shared memory raking grid
+            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
+            *placement_ptr = input;
+
+            CTA_SYNC();
+
+            // Reduce parallelism down to just raking threads
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking upsweep reduction across shared partials
+                T upsweep_partial= Upsweep(scan_op);
+
+                // Warp-synchronous scan
+                T inclusive_partial;
+                T exclusive_partial;
+                WarpScan(temp_storage.warp_scan).Scan(upsweep_partial, inclusive_partial, exclusive_partial, scan_op);
+
+                // Exclusive raking downsweep scan
+                ExclusiveDownsweep(scan_op, exclusive_partial, (linear_tid != 0));
+
+                // Broadcast aggregate to all threads
+                if (linear_tid == RAKING_THREADS - 1)
+                    temp_storage.block_aggregate = inclusive_partial;
+            }
+
+            CTA_SYNC();
+
+            // Grab thread prefix from shared memory
+            output = *placement_ptr;
+
+            // Retrieve block aggregate
+            block_aggregate = temp_storage.block_aggregate;
+        }
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &output,            ///< [out] Calling thread's output items (may be aliased to \p input)
+        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp-synchronous scan
+            WarpScan(temp_storage.warp_scan).ExclusiveScan(input, output, initial_value, scan_op, block_aggregate);
+        }
+        else
+        {
+            // Place thread partial into shared memory raking grid
+            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
+            *placement_ptr = input;
+
+            CTA_SYNC();
+
+            // Reduce parallelism down to just raking threads
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking upsweep reduction across shared partials
+                T upsweep_partial = Upsweep(scan_op);
+
+                // Warp-synchronous scan
+                T exclusive_partial;
+                WarpScan(temp_storage.warp_scan).ExclusiveScan(upsweep_partial, exclusive_partial, initial_value, scan_op, block_aggregate);
+
+                // Exclusive raking downsweep scan
+                ExclusiveDownsweep(scan_op, exclusive_partial);
+
+                // Broadcast aggregate to other threads
+                if (linear_tid == 0)
+                    temp_storage.block_aggregate = block_aggregate;
+            }
+
+            CTA_SYNC();
+
+            // Grab exclusive partial from shared memory
+            output = *placement_ptr;
+
+            // Retrieve block aggregate
+            block_aggregate = temp_storage.block_aggregate;
+        }
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan operator
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp-synchronous scan
+            T block_aggregate;
+            WarpScan warp_scan(temp_storage.warp_scan);
+            warp_scan.ExclusiveScan(input, output, scan_op, block_aggregate);
+
+            // Obtain warp-wide prefix in lane0, then broadcast to other lanes
+            T block_prefix = block_prefix_callback_op(block_aggregate);
+            block_prefix = warp_scan.Broadcast(block_prefix, 0);
+
+            output = scan_op(block_prefix, output);
+            if (linear_tid == 0)
+                output = block_prefix;
+        }
+        else
+        {
+            // Place thread partial into shared memory raking grid
+            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
+            *placement_ptr = input;
+
+            CTA_SYNC();
+
+            // Reduce parallelism down to just raking threads
+            if (linear_tid < RAKING_THREADS)
+            {
+                WarpScan warp_scan(temp_storage.warp_scan);
+
+                // Raking upsweep reduction across shared partials
+                T upsweep_partial = Upsweep(scan_op);
+
+                // Warp-synchronous scan
+                T exclusive_partial, block_aggregate;
+                warp_scan.ExclusiveScan(upsweep_partial, exclusive_partial, scan_op, block_aggregate);
+
+                // Obtain block-wide prefix in lane0, then broadcast to other lanes
+                T block_prefix = block_prefix_callback_op(block_aggregate);
+                block_prefix = warp_scan.Broadcast(block_prefix, 0);
+
+                // Update prefix with warpscan exclusive partial
+                T downsweep_prefix = scan_op(block_prefix, exclusive_partial);
+                if (linear_tid == 0)
+                    downsweep_prefix = block_prefix;
+
+                // Exclusive raking downsweep scan
+                ExclusiveDownsweep(scan_op, downsweep_prefix);
+            }
+
+            CTA_SYNC();
+
+            // Grab thread prefix from shared memory
+            output = *placement_ptr;
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Inclusive scans
+    //---------------------------------------------------------------------
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan operator
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp-synchronous scan
+            WarpScan(temp_storage.warp_scan).InclusiveScan(input, output, scan_op);
+        }
+        else
+        {
+            // Place thread partial into shared memory raking grid
+            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
+            *placement_ptr = input;
+
+            CTA_SYNC();
+
+            // Reduce parallelism down to just raking threads
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking upsweep reduction across shared partials
+                T upsweep_partial = Upsweep(scan_op);
+
+                // Exclusive Warp-synchronous scan
+                T exclusive_partial;
+                WarpScan(temp_storage.warp_scan).ExclusiveScan(upsweep_partial, exclusive_partial, scan_op);
+
+                // Inclusive raking downsweep scan
+                InclusiveDownsweep(scan_op, exclusive_partial, (linear_tid != 0));
+            }
+
+            CTA_SYNC();
+
+            // Grab thread prefix from shared memory
+            output = *placement_ptr;
+        }
+    }
+
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan operator
+        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp-synchronous scan
+            WarpScan(temp_storage.warp_scan).InclusiveScan(input, output, scan_op, block_aggregate);
+        }
+        else
+        {
+            // Place thread partial into shared memory raking grid
+            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
+            *placement_ptr = input;
+
+            CTA_SYNC();
+
+            // Reduce parallelism down to just raking threads
+            if (linear_tid < RAKING_THREADS)
+            {
+                // Raking upsweep reduction across shared partials
+                T upsweep_partial = Upsweep(scan_op);
+
+                // Warp-synchronous scan
+                T inclusive_partial;
+                T exclusive_partial;
+                WarpScan(temp_storage.warp_scan).Scan(upsweep_partial, inclusive_partial, exclusive_partial, scan_op);
+
+                // Inclusive raking downsweep scan
+                InclusiveDownsweep(scan_op, exclusive_partial, (linear_tid != 0));
+
+                // Broadcast aggregate to all threads
+                if (linear_tid == RAKING_THREADS - 1)
+                    temp_storage.block_aggregate = inclusive_partial;
+            }
+
+            CTA_SYNC();
+
+            // Grab thread prefix from shared memory
+            output = *placement_ptr;
+
+            // Retrieve block aggregate
+            block_aggregate = temp_storage.block_aggregate;
+        }
+    }
+
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan operator
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
+    {
+        if (WARP_SYNCHRONOUS)
+        {
+            // Short-circuit directly to warp-synchronous scan
+            T block_aggregate;
+            WarpScan warp_scan(temp_storage.warp_scan);
+            warp_scan.InclusiveScan(input, output, scan_op, block_aggregate);
+
+            // Obtain warp-wide prefix in lane0, then broadcast to other lanes
+            T block_prefix = block_prefix_callback_op(block_aggregate);
+            block_prefix = warp_scan.Broadcast(block_prefix, 0);
+
+            // Update prefix with exclusive warpscan partial
+            output = scan_op(block_prefix, output);
+        }
+        else
+        {
+            // Place thread partial into shared memory raking grid
+            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
+            *placement_ptr = input;
+
+            CTA_SYNC();
+
+            // Reduce parallelism down to just raking threads
+            if (linear_tid < RAKING_THREADS)
+            {
+                WarpScan warp_scan(temp_storage.warp_scan);
+
+                // Raking upsweep reduction across shared partials
+                T upsweep_partial = Upsweep(scan_op);
+
+                // Warp-synchronous scan
+                T exclusive_partial, block_aggregate;
+                warp_scan.ExclusiveScan(upsweep_partial, exclusive_partial, scan_op, block_aggregate);
+
+                // Obtain block-wide prefix in lane0, then broadcast to other lanes
+                T block_prefix = block_prefix_callback_op(block_aggregate);
+                block_prefix = warp_scan.Broadcast(block_prefix, 0);
+
+                // Update prefix with warpscan exclusive partial
+                T downsweep_prefix = scan_op(block_prefix, exclusive_partial);
+                if (linear_tid == 0)
+                    downsweep_prefix = block_prefix;
+
+                // Inclusive raking downsweep scan
+                InclusiveDownsweep(scan_op, downsweep_prefix);
+            }
+
+            CTA_SYNC();
+
+            // Grab thread prefix from shared memory
+            output = *placement_ptr;
+        }
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_scan_warp_scans.cuh b/third_party/cub/cub/block/specializations/block_scan_warp_scans.cuh
new file mode 100644
index 0000000..85e4d61
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_scan_warp_scans.cuh
@@ -0,0 +1,392 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../../util_arch.cuh"
+#include "../../util_ptx.cuh"
+#include "../../warp/warp_scan.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
+ */
+template <
+    typename    T,
+    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
+    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
+    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
+    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
+struct BlockScanWarpScans
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// Constants
+    enum
+    {
+        /// Number of warp threads
+        WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
+
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        /// Number of active warps
+        WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+    };
+
+    ///  WarpScan utility type
+    typedef WarpScan<T, WARP_THREADS, PTX_ARCH> WarpScanT;
+
+    ///  WarpScan utility type
+    typedef WarpScan<T, WARPS, PTX_ARCH> WarpAggregateScan;
+
+    /// Shared memory storage layout type
+
+    struct __align__(32) _TempStorage
+    {
+        T                               warp_aggregates[WARPS];
+        typename WarpScanT::TempStorage warp_scan[WARPS];           ///< Buffer for warp-synchronous scans
+        T                               block_prefix;               ///< Shared prefix for the entire thread block
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    // Thread fields
+    _TempStorage    &temp_storage;
+    unsigned int    linear_tid;
+    unsigned int    warp_id;
+    unsigned int    lane_id;
+
+
+    //---------------------------------------------------------------------
+    // Constructors
+    //---------------------------------------------------------------------
+
+    /// Constructor
+    __device__ __forceinline__ BlockScanWarpScans(
+        TempStorage &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
+        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
+        lane_id(LaneId())
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Utility methods
+    //---------------------------------------------------------------------
+
+    template <typename ScanOp, int WARP>
+    __device__ __forceinline__ void ApplyWarpAggregates(
+        T               &warp_prefix,           ///< [out] The calling thread's partial reduction
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
+        Int2Type<WARP>  /*addend_warp*/)
+    {
+        if (warp_id == WARP)
+            warp_prefix = block_aggregate;
+
+        T addend = temp_storage.warp_aggregates[WARP];
+        block_aggregate = scan_op(block_aggregate, addend);
+
+        ApplyWarpAggregates(warp_prefix, scan_op, block_aggregate, Int2Type<WARP + 1>());
+    }
+
+    template <typename ScanOp>
+    __device__ __forceinline__ void ApplyWarpAggregates(
+        T               &/*warp_prefix*/,       ///< [out] The calling thread's partial reduction
+        ScanOp          /*scan_op*/,            ///< [in] Binary scan operator
+        T               &/*block_aggregate*/,   ///< [out] Threadblock-wide aggregate reduction of input items
+        Int2Type<WARPS> /*addend_warp*/)
+    {}
+
+
+    /// Use the warp-wide aggregates to compute the calling warp's prefix.  Also returns block-wide aggregate in all threads.
+    template <typename ScanOp>
+    __device__ __forceinline__ T ComputeWarpPrefix(
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>WARP_THREADS - 1</sub> only]</b> Warp-wide aggregate reduction of input items
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        // Last lane in each warp shares its warp-aggregate
+        if (lane_id == WARP_THREADS - 1)
+            temp_storage.warp_aggregates[warp_id] = warp_aggregate;
+
+        CTA_SYNC();
+
+        // Accumulate block aggregates and save the one that is our warp's prefix
+        T warp_prefix;
+        block_aggregate = temp_storage.warp_aggregates[0];
+
+        // Use template unrolling (since the PTX backend can't handle unrolling it for SM1x)
+        ApplyWarpAggregates(warp_prefix, scan_op, block_aggregate, Int2Type<1>());
+/*
+        #pragma unroll
+        for (int WARP = 1; WARP < WARPS; ++WARP)
+        {
+            if (warp_id == WARP)
+                warp_prefix = block_aggregate;
+
+            T addend = temp_storage.warp_aggregates[WARP];
+            block_aggregate = scan_op(block_aggregate, addend);
+        }
+*/
+
+        return warp_prefix;
+    }
+
+
+    /// Use the warp-wide aggregates and initial-value to compute the calling warp's prefix.  Also returns block-wide aggregate in all threads.
+    template <typename ScanOp>
+    __device__ __forceinline__ T ComputeWarpPrefix(
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>WARP_THREADS - 1</sub> only]</b> Warp-wide aggregate reduction of input items
+        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
+        const T         &initial_value)     ///< [in] Initial value to seed the exclusive scan
+    {
+        T warp_prefix = ComputeWarpPrefix(scan_op, warp_aggregate, block_aggregate);
+
+        warp_prefix = scan_op(initial_value, warp_prefix);
+
+        if (warp_id == 0)
+            warp_prefix = initial_value;
+
+        return warp_prefix;
+    }
+
+    //---------------------------------------------------------------------
+    // Exclusive scans
+    //---------------------------------------------------------------------
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan operator
+    {
+        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
+        T block_aggregate;
+        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
+        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        T block_aggregate;
+        ExclusiveScan(input, exclusive_output, initial_value, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input item
+        T               &exclusive_output,  ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        T inclusive_output;
+        WarpScanT(temp_storage.warp_scan[warp_id]).Scan(input, inclusive_output, exclusive_output, scan_op);
+
+        // Compute the warp-wide prefix and block-wide aggregate for each warp.  Warp prefix for warp0 is invalid.
+        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate);
+
+        // Apply warp prefix to our lane's partial
+        if (warp_id != 0)
+        {
+            exclusive_output = scan_op(warp_prefix, exclusive_output);
+            if (lane_id == 0)
+                exclusive_output = warp_prefix;
+        }
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
+        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        T inclusive_output;
+        WarpScanT(temp_storage.warp_scan[warp_id]).Scan(input, inclusive_output, exclusive_output, scan_op);
+
+        // Compute the warp-wide prefix and block-wide aggregate for each warp
+        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate, initial_value);
+
+        // Apply warp prefix to our lane's partial
+        exclusive_output = scan_op(warp_prefix, exclusive_output);
+        if (lane_id == 0)
+            exclusive_output = warp_prefix;
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan operator
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
+    {
+        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
+        T block_aggregate;
+        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
+
+        // Use the first warp to determine the thread block prefix, returning the result in lane0
+        if (warp_id == 0)
+        {
+            T block_prefix = block_prefix_callback_op(block_aggregate);
+            if (lane_id == 0)
+            {
+                // Share the prefix with all threads
+                temp_storage.block_prefix = block_prefix;
+                exclusive_output = block_prefix;                // The block prefix is the exclusive output for tid0
+            }
+        }
+
+        CTA_SYNC();
+
+        // Incorporate thread block prefix into outputs
+        T block_prefix = temp_storage.block_prefix;
+        if (linear_tid > 0)
+        {
+            exclusive_output = scan_op(block_prefix, exclusive_output);
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Inclusive scans
+    //---------------------------------------------------------------------
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan operator
+    {
+        T block_aggregate;
+        InclusiveScan(input, inclusive_output, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan operator
+        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        WarpScanT(temp_storage.warp_scan[warp_id]).InclusiveScan(input, inclusive_output, scan_op);
+
+        // Compute the warp-wide prefix and block-wide aggregate for each warp.  Warp prefix for warp0 is invalid.
+        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate);
+
+        // Apply warp prefix to our lane's partial
+        if (warp_id != 0)
+        {
+            inclusive_output = scan_op(warp_prefix, inclusive_output);
+        }
+    }
+
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan operator
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
+    {
+        T block_aggregate;
+        InclusiveScan(input, exclusive_output, scan_op, block_aggregate);
+
+        // Use the first warp to determine the thread block prefix, returning the result in lane0
+        if (warp_id == 0)
+        {
+            T block_prefix = block_prefix_callback_op(block_aggregate);
+            if (lane_id == 0)
+            {
+                // Share the prefix with all threads
+                temp_storage.block_prefix = block_prefix;
+            }
+        }
+
+        CTA_SYNC();
+
+        // Incorporate thread block prefix into outputs
+        T block_prefix = temp_storage.block_prefix;
+        exclusive_output = scan_op(block_prefix, exclusive_output);
+    }
+
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_scan_warp_scans2.cuh b/third_party/cub/cub/block/specializations/block_scan_warp_scans2.cuh
new file mode 100644
index 0000000..4de7c69
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_scan_warp_scans2.cuh
@@ -0,0 +1,436 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../../util_arch.cuh"
+#include "../../util_ptx.cuh"
+#include "../../warp/warp_scan.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
+ */
+template <
+    typename    T,
+    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
+    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
+    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
+    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
+struct BlockScanWarpScans
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// Constants
+    enum
+    {
+        /// Number of warp threads
+        WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
+
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        /// Number of active warps
+        WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
+    };
+
+    ///  WarpScan utility type
+    typedef WarpScan<T, WARP_THREADS, PTX_ARCH> WarpScanT;
+
+    ///  WarpScan utility type
+    typedef WarpScan<T, WARPS, PTX_ARCH> WarpAggregateScanT;
+
+    /// Shared memory storage layout type
+    struct _TempStorage
+    {
+        typename WarpAggregateScanT::TempStorage    inner_scan[WARPS];          ///< Buffer for warp-synchronous scans
+        typename WarpScanT::TempStorage             warp_scan[WARPS];           ///< Buffer for warp-synchronous scans
+        T                                           warp_aggregates[WARPS];
+        T                                           block_prefix;               ///< Shared prefix for the entire thread block
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    // Thread fields
+    _TempStorage    &temp_storage;
+    unsigned int    linear_tid;
+    unsigned int    warp_id;
+    unsigned int    lane_id;
+
+
+    //---------------------------------------------------------------------
+    // Constructors
+    //---------------------------------------------------------------------
+
+    /// Constructor
+    __device__ __forceinline__ BlockScanWarpScans(
+        TempStorage &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
+        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
+        lane_id(LaneId())
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Utility methods
+    //---------------------------------------------------------------------
+
+    template <typename ScanOp, int WARP>
+    __device__ __forceinline__ void ApplyWarpAggregates(
+        T               &warp_prefix,           ///< [out] The calling thread's partial reduction
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
+        Int2Type<WARP>  addend_warp)
+    {
+        if (warp_id == WARP)
+            warp_prefix = block_aggregate;
+
+        T addend = temp_storage.warp_aggregates[WARP];
+        block_aggregate = scan_op(block_aggregate, addend);
+
+        ApplyWarpAggregates(warp_prefix, scan_op, block_aggregate, Int2Type<WARP + 1>());
+    }
+
+    template <typename ScanOp>
+    __device__ __forceinline__ void ApplyWarpAggregates(
+        T               &warp_prefix,           ///< [out] The calling thread's partial reduction
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
+        Int2Type<WARPS> addend_warp)
+    {}
+
+
+    /// Use the warp-wide aggregates to compute the calling warp's prefix.  Also returns block-wide aggregate in all threads.
+    template <typename ScanOp>
+    __device__ __forceinline__ T ComputeWarpPrefix(
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>WARP_THREADS - 1</sub> only]</b> Warp-wide aggregate reduction of input items
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        // Last lane in each warp shares its warp-aggregate
+        if (lane_id == WARP_THREADS - 1)
+            temp_storage.warp_aggregates[warp_id] = warp_aggregate;
+
+        CTA_SYNC();
+
+        // Accumulate block aggregates and save the one that is our warp's prefix
+        T warp_prefix;
+        block_aggregate = temp_storage.warp_aggregates[0];
+
+        // Use template unrolling (since the PTX backend can't handle unrolling it for SM1x)
+        ApplyWarpAggregates(warp_prefix, scan_op, block_aggregate, Int2Type<1>());
+/*
+        #pragma unroll
+        for (int WARP = 1; WARP < WARPS; ++WARP)
+        {
+            if (warp_id == WARP)
+                warp_prefix = block_aggregate;
+
+            T addend = temp_storage.warp_aggregates[WARP];
+            block_aggregate = scan_op(block_aggregate, addend);
+        }
+*/
+
+        return warp_prefix;
+    }
+
+
+    /// Use the warp-wide aggregates and initial-value to compute the calling warp's prefix.  Also returns block-wide aggregate in all threads.
+    template <typename ScanOp>
+    __device__ __forceinline__ T ComputeWarpPrefix(
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>WARP_THREADS - 1</sub> only]</b> Warp-wide aggregate reduction of input items
+        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
+        const T         &initial_value)     ///< [in] Initial value to seed the exclusive scan
+    {
+        T warp_prefix = ComputeWarpPrefix(scan_op, warp_aggregate, block_aggregate);
+
+        warp_prefix = scan_op(initial_value, warp_prefix);
+
+        if (warp_id == 0)
+            warp_prefix = initial_value;
+
+        return warp_prefix;
+    }
+
+    //---------------------------------------------------------------------
+    // Exclusive scans
+    //---------------------------------------------------------------------
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan operator
+    {
+        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
+        T block_aggregate;
+        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
+        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        T block_aggregate;
+        ExclusiveScan(input, exclusive_output, initial_value, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input item
+        T               &exclusive_output,  ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        WarpScanT my_warp_scan(temp_storage.warp_scan[warp_id]);
+
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        T inclusive_output;
+        my_warp_scan.Scan(input, inclusive_output, exclusive_output, scan_op);
+
+        // Compute the warp-wide prefix and block-wide aggregate for each warp.  Warp prefix for warp0 is invalid.
+//        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate);
+
+//--------------------------------------------------
+        // Last lane in each warp shares its warp-aggregate
+        if (lane_id == WARP_THREADS - 1)
+            temp_storage.warp_aggregates[warp_id] = inclusive_output;
+
+        CTA_SYNC();
+
+        // Get the warp scan partial
+        T warp_inclusive, warp_prefix;
+        if (lane_id < WARPS)
+        {
+            // Scan the warpscan partials
+            T warp_val = temp_storage.warp_aggregates[lane_id];
+            WarpAggregateScanT(temp_storage.inner_scan[warp_id]).Scan(warp_val, warp_inclusive, warp_prefix, scan_op);
+        }
+
+        warp_prefix         = my_warp_scan.Broadcast(warp_prefix, warp_id);
+        block_aggregate     = my_warp_scan.Broadcast(warp_inclusive, WARPS - 1);
+//--------------------------------------------------
+
+        // Apply warp prefix to our lane's partial
+        if (warp_id != 0)
+        {
+            exclusive_output = scan_op(warp_prefix, exclusive_output);
+            if (lane_id == 0)
+                exclusive_output = warp_prefix;
+        }
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
+        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        WarpScanT my_warp_scan(temp_storage.warp_scan[warp_id]);
+
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        T inclusive_output;
+        my_warp_scan.Scan(input, inclusive_output, exclusive_output, scan_op);
+
+        // Compute the warp-wide prefix and block-wide aggregate for each warp
+//        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate, initial_value);
+
+//--------------------------------------------------
+        // Last lane in each warp shares its warp-aggregate
+        if (lane_id == WARP_THREADS - 1)
+            temp_storage.warp_aggregates[warp_id] = inclusive_output;
+
+        CTA_SYNC();
+
+        // Get the warp scan partial
+        T warp_inclusive, warp_prefix;
+        if (lane_id < WARPS)
+        {
+            // Scan the warpscan partials
+            T warp_val = temp_storage.warp_aggregates[lane_id];
+            WarpAggregateScanT(temp_storage.inner_scan[warp_id]).Scan(warp_val, warp_inclusive, warp_prefix, initial_value, scan_op);
+        }
+
+        warp_prefix         = my_warp_scan.Broadcast(warp_prefix, warp_id);
+        block_aggregate     = my_warp_scan.Broadcast(warp_inclusive, WARPS - 1);
+//--------------------------------------------------
+
+        // Apply warp prefix to our lane's partial
+        exclusive_output = scan_op(warp_prefix, exclusive_output);
+        if (lane_id == 0)
+            exclusive_output = warp_prefix;
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan operator
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
+    {
+        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
+        T block_aggregate;
+        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
+
+        // Use the first warp to determine the thread block prefix, returning the result in lane0
+        if (warp_id == 0)
+        {
+            T block_prefix = block_prefix_callback_op(block_aggregate);
+            if (lane_id == 0)
+            {
+                // Share the prefix with all threads
+                temp_storage.block_prefix = block_prefix;
+                exclusive_output = block_prefix;                // The block prefix is the exclusive output for tid0
+            }
+        }
+
+        CTA_SYNC();
+
+        // Incorporate thread block prefix into outputs
+        T block_prefix = temp_storage.block_prefix;
+        if (linear_tid > 0)
+        {
+            exclusive_output = scan_op(block_prefix, exclusive_output);
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Inclusive scans
+    //---------------------------------------------------------------------
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan operator
+    {
+        T block_aggregate;
+        InclusiveScan(input, inclusive_output, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan operator
+        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        WarpScanT(temp_storage.warp_scan[warp_id]).InclusiveScan(input, inclusive_output, scan_op);
+
+        // Compute the warp-wide prefix and block-wide aggregate for each warp.  Warp prefix for warp0 is invalid.
+        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate);
+
+        // Apply warp prefix to our lane's partial
+        if (warp_id != 0)
+        {
+            inclusive_output = scan_op(warp_prefix, inclusive_output);
+        }
+    }
+
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan operator
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
+    {
+        T block_aggregate;
+        InclusiveScan(input, exclusive_output, scan_op, block_aggregate);
+
+        // Use the first warp to determine the thread block prefix, returning the result in lane0
+        if (warp_id == 0)
+        {
+            T block_prefix = block_prefix_callback_op(block_aggregate);
+            if (lane_id == 0)
+            {
+                // Share the prefix with all threads
+                temp_storage.block_prefix = block_prefix;
+            }
+        }
+
+        CTA_SYNC();
+
+        // Incorporate thread block prefix into outputs
+        T block_prefix = temp_storage.block_prefix;
+        exclusive_output = scan_op(block_prefix, exclusive_output);
+    }
+
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/block/specializations/block_scan_warp_scans3.cuh b/third_party/cub/cub/block/specializations/block_scan_warp_scans3.cuh
new file mode 100644
index 0000000..147ca4c
--- /dev/null
+++ b/third_party/cub/cub/block/specializations/block_scan_warp_scans3.cuh
@@ -0,0 +1,418 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
+ */
+
+#pragma once
+
+#include "../../util_arch.cuh"
+#include "../../util_ptx.cuh"
+#include "../../warp/warp_scan.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
+ */
+template <
+    typename    T,
+    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
+    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
+    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
+    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
+struct BlockScanWarpScans
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// Constants
+    enum
+    {
+        /// The thread block size in threads
+        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+
+        /// Number of warp threads
+        INNER_WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
+        OUTER_WARP_THREADS = BLOCK_THREADS / INNER_WARP_THREADS,
+
+        /// Number of outer scan warps
+        OUTER_WARPS = INNER_WARP_THREADS
+    };
+
+    ///  Outer WarpScan utility type
+    typedef WarpScan<T, OUTER_WARP_THREADS, PTX_ARCH> OuterWarpScanT;
+
+    ///  Inner WarpScan utility type
+    typedef WarpScan<T, INNER_WARP_THREADS, PTX_ARCH> InnerWarpScanT;
+
+    typedef typename OuterWarpScanT::TempStorage OuterScanArray[OUTER_WARPS];
+
+
+    /// Shared memory storage layout type
+    struct _TempStorage
+    {
+        union Aliasable
+        {
+            Uninitialized<OuterScanArray>           outer_warp_scan;  ///< Buffer for warp-synchronous outer scans
+            typename InnerWarpScanT::TempStorage    inner_warp_scan;  ///< Buffer for warp-synchronous inner scan
+
+        } aliasable;
+
+        T                               warp_aggregates[OUTER_WARPS];
+
+        T                               block_aggregate;                           ///< Shared prefix for the entire thread block
+    };
+
+
+    /// Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Per-thread fields
+    //---------------------------------------------------------------------
+
+    // Thread fields
+    _TempStorage    &temp_storage;
+    unsigned int    linear_tid;
+    unsigned int    warp_id;
+    unsigned int    lane_id;
+
+
+    //---------------------------------------------------------------------
+    // Constructors
+    //---------------------------------------------------------------------
+
+    /// Constructor
+    __device__ __forceinline__ BlockScanWarpScans(
+        TempStorage &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
+        warp_id((OUTER_WARPS == 1) ? 0 : linear_tid / OUTER_WARP_THREADS),
+        lane_id((OUTER_WARPS == 1) ? linear_tid : linear_tid % OUTER_WARP_THREADS)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Exclusive scans
+    //---------------------------------------------------------------------
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan operator
+    {
+        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
+        T block_aggregate;
+        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
+        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        T block_aggregate;
+        ExclusiveScan(input, exclusive_output, initial_value, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input item
+        T               &exclusive_output,  ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        T inclusive_output;
+        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).Scan(
+            input, inclusive_output, exclusive_output, scan_op);
+
+        // Share outer warp total
+        if (lane_id == OUTER_WARP_THREADS - 1)
+            temp_storage.warp_aggregates[warp_id] = inclusive_output;
+
+        CTA_SYNC();
+
+        if (linear_tid < INNER_WARP_THREADS)
+        {
+            T outer_warp_input = temp_storage.warp_aggregates[linear_tid];
+            T outer_warp_exclusive;
+
+            InnerWarpScanT(temp_storage.aliasable.inner_warp_scan).ExclusiveScan(
+                outer_warp_input, outer_warp_exclusive, scan_op, block_aggregate);
+
+            temp_storage.block_aggregate                = block_aggregate;
+            temp_storage.warp_aggregates[linear_tid]    = outer_warp_exclusive;
+        }
+
+        CTA_SYNC();
+
+        if (warp_id != 0)
+        {
+            // Retrieve block aggregate
+            block_aggregate = temp_storage.block_aggregate;
+
+            // Apply warp prefix to our lane's partial
+            T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
+            exclusive_output = scan_op(outer_warp_exclusive, exclusive_output);
+            if (lane_id == 0)
+                exclusive_output = outer_warp_exclusive;
+        }
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input items
+        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
+        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        T inclusive_output;
+        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).Scan(
+            input, inclusive_output, exclusive_output, scan_op);
+
+        // Share outer warp total
+        if (lane_id == OUTER_WARP_THREADS - 1)
+        {
+            temp_storage.warp_aggregates[warp_id] = inclusive_output;
+        }
+
+        CTA_SYNC();
+
+        if (linear_tid < INNER_WARP_THREADS)
+        {
+            T outer_warp_input = temp_storage.warp_aggregates[linear_tid];
+            T outer_warp_exclusive;
+
+            InnerWarpScanT(temp_storage.aliasable.inner_warp_scan).ExclusiveScan(
+                outer_warp_input, outer_warp_exclusive, initial_value, scan_op, block_aggregate);
+
+            temp_storage.block_aggregate                = block_aggregate;
+            temp_storage.warp_aggregates[linear_tid]    = outer_warp_exclusive;
+        }
+
+        CTA_SYNC();
+
+        // Retrieve block aggregate
+        block_aggregate = temp_storage.block_aggregate;
+
+        // Apply warp prefix to our lane's partial
+        T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
+        exclusive_output = scan_op(outer_warp_exclusive, exclusive_output);
+        if (lane_id == 0)
+            exclusive_output = outer_warp_exclusive;
+    }
+
+
+    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  The call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan operator
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
+    {
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        T inclusive_output;
+        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).Scan(
+            input, inclusive_output, exclusive_output, scan_op);
+
+        // Share outer warp total
+        if (lane_id == OUTER_WARP_THREADS - 1)
+            temp_storage.warp_aggregates[warp_id] = inclusive_output;
+
+        CTA_SYNC();
+
+        if (linear_tid < INNER_WARP_THREADS)
+        {
+            InnerWarpScanT inner_scan(temp_storage.aliasable.inner_warp_scan);
+
+            T upsweep = temp_storage.warp_aggregates[linear_tid];
+            T downsweep_prefix, block_aggregate;
+
+            inner_scan.ExclusiveScan(upsweep, downsweep_prefix, scan_op, block_aggregate);
+
+            // Use callback functor to get block prefix in lane0 and then broadcast to other lanes
+            T block_prefix = block_prefix_callback_op(block_aggregate);
+            block_prefix = inner_scan.Broadcast(block_prefix, 0);
+
+            downsweep_prefix = scan_op(block_prefix, downsweep_prefix);
+            if (linear_tid == 0)
+                downsweep_prefix = block_prefix;
+
+            temp_storage.warp_aggregates[linear_tid] = downsweep_prefix;
+        }
+
+        CTA_SYNC();
+
+        // Apply warp prefix to our lane's partial (or assign it if partial is invalid)
+        T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
+        exclusive_output = scan_op(outer_warp_exclusive, exclusive_output);
+        if (lane_id == 0)
+            exclusive_output = outer_warp_exclusive;
+    }
+
+
+    //---------------------------------------------------------------------
+    // Inclusive scans
+    //---------------------------------------------------------------------
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op)                        ///< [in] Binary scan operator
+    {
+        T block_aggregate;
+        InclusiveScan(input, inclusive_output, scan_op, block_aggregate);
+    }
+
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,                          ///< [in] Calling thread's input item
+        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp          scan_op,                        ///< [in] Binary scan operator
+        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
+    {
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).InclusiveScan(
+            input, inclusive_output, scan_op);
+
+        // Share outer warp total
+        if (lane_id == OUTER_WARP_THREADS - 1)
+            temp_storage.warp_aggregates[warp_id] = inclusive_output;
+
+        CTA_SYNC();
+
+        if (linear_tid < INNER_WARP_THREADS)
+        {
+            T outer_warp_input = temp_storage.warp_aggregates[linear_tid];
+            T outer_warp_exclusive;
+
+            InnerWarpScanT(temp_storage.aliasable.inner_warp_scan).ExclusiveScan(
+                outer_warp_input, outer_warp_exclusive, scan_op, block_aggregate);
+
+            temp_storage.block_aggregate                = block_aggregate;
+            temp_storage.warp_aggregates[linear_tid]    = outer_warp_exclusive;
+        }
+
+        CTA_SYNC();
+
+        if (warp_id != 0)
+        {
+            // Retrieve block aggregate
+            block_aggregate = temp_storage.block_aggregate;
+
+            // Apply warp prefix to our lane's partial
+            T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
+            inclusive_output = scan_op(outer_warp_exclusive, inclusive_output);
+        }
+    }
+
+
+    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.
+    template <
+        typename ScanOp,
+        typename BlockPrefixCallbackOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T                       input,                          ///< [in] Calling thread's input item
+        T                       &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
+        ScanOp                  scan_op,                        ///< [in] Binary scan operator
+        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
+    {
+        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
+        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).InclusiveScan(
+            input, inclusive_output, scan_op);
+
+        // Share outer warp total
+        if (lane_id == OUTER_WARP_THREADS - 1)
+            temp_storage.warp_aggregates[warp_id] = inclusive_output;
+
+        CTA_SYNC();
+
+        if (linear_tid < INNER_WARP_THREADS)
+        {
+            InnerWarpScanT inner_scan(temp_storage.aliasable.inner_warp_scan);
+
+            T upsweep = temp_storage.warp_aggregates[linear_tid];
+            T downsweep_prefix, block_aggregate;
+            inner_scan.ExclusiveScan(upsweep, downsweep_prefix, scan_op, block_aggregate);
+
+            // Use callback functor to get block prefix in lane0 and then broadcast to other lanes
+            T block_prefix = block_prefix_callback_op(block_aggregate);
+            block_prefix = inner_scan.Broadcast(block_prefix, 0);
+
+            downsweep_prefix = scan_op(block_prefix, downsweep_prefix);
+            if (linear_tid == 0)
+                downsweep_prefix = block_prefix;
+
+            temp_storage.warp_aggregates[linear_tid]    = downsweep_prefix;
+        }
+
+        CTA_SYNC();
+
+        // Apply warp prefix to our lane's partial
+        T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
+        inclusive_output = scan_op(outer_warp_exclusive, inclusive_output);
+    }
+
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/cub.cuh b/third_party/cub/cub/cub.cuh
new file mode 100644
index 0000000..3ece0f6
--- /dev/null
+++ b/third_party/cub/cub/cub.cuh
@@ -0,0 +1,95 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * CUB umbrella include file
+ */
+
+#pragma once
+
+
+// Block
+#include "block/block_histogram.cuh"
+#include "block/block_discontinuity.cuh"
+#include "block/block_exchange.cuh"
+#include "block/block_load.cuh"
+#include "block/block_radix_rank.cuh"
+#include "block/block_radix_sort.cuh"
+#include "block/block_reduce.cuh"
+#include "block/block_scan.cuh"
+#include "block/block_store.cuh"
+//#include "block/block_shift.cuh"
+
+// Device
+#include "device/device_histogram.cuh"
+#include "device/device_partition.cuh"
+#include "device/device_radix_sort.cuh"
+#include "device/device_reduce.cuh"
+#include "device/device_run_length_encode.cuh"
+#include "device/device_scan.cuh"
+#include "device/device_segmented_radix_sort.cuh"
+#include "device/device_segmented_reduce.cuh"
+#include "device/device_select.cuh"
+#include "device/device_spmv.cuh"
+
+// Grid
+//#include "grid/grid_barrier.cuh"
+#include "grid/grid_even_share.cuh"
+#include "grid/grid_mapping.cuh"
+#include "grid/grid_queue.cuh"
+
+// Thread
+#include "thread/thread_load.cuh"
+#include "thread/thread_operators.cuh"
+#include "thread/thread_reduce.cuh"
+#include "thread/thread_scan.cuh"
+#include "thread/thread_store.cuh"
+
+// Warp
+#include "warp/warp_reduce.cuh"
+#include "warp/warp_scan.cuh"
+
+// Iterator
+#include "iterator/arg_index_input_iterator.cuh"
+#include "iterator/cache_modified_input_iterator.cuh"
+#include "iterator/cache_modified_output_iterator.cuh"
+#include "iterator/constant_input_iterator.cuh"
+#include "iterator/counting_input_iterator.cuh"
+#include "iterator/tex_obj_input_iterator.cuh"
+#include "iterator/tex_ref_input_iterator.cuh"
+#include "iterator/transform_input_iterator.cuh"
+
+// Util
+#include "util_arch.cuh"
+#include "util_debug.cuh"
+#include "util_device.cuh"
+#include "util_macro.cuh"
+#include "util_ptx.cuh"
+#include "util_type.cuh"
+
diff --git a/third_party/cub/cub/device/device_histogram.cuh b/third_party/cub/cub/device/device_histogram.cuh
new file mode 100644
index 0000000..a2556a6
--- /dev/null
+++ b/third_party/cub/cub/device/device_histogram.cuh
@@ -0,0 +1,866 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceHistogram provides device-wide parallel operations for constructing histogram(s) from a sequence of samples data residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+#include <limits>
+
+#include "dispatch/dispatch_histogram.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceHistogram provides device-wide parallel operations for constructing histogram(s) from a sequence of samples data residing within device-accessible memory. ![](histogram_logo.png)
+ * \ingroup SingleModule
+ *
+ * \par Overview
+ * A <a href="http://en.wikipedia.org/wiki/Histogram"><em>histogram</em></a>
+ * counts the number of observations that fall into each of the disjoint categories (known as <em>bins</em>).
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceHistogram}
+ *
+ */
+struct DeviceHistogram
+{
+    /******************************************************************//**
+     * \name Evenly-segmented bin ranges
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Computes an intensity histogram from a sequence of data samples using equal-width bins.
+     *
+     * \par
+     * - The number of histogram bins is (\p num_levels - 1)
+     * - All bins comprise the same width of sample values: (\p upper_level - \p lower_level) / (\p num_levels - 1)
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the computation of a six-bin histogram
+     * from a sequence of float samples
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input samples and
+     * // output histogram
+     * int      num_samples;    // e.g., 10
+     * float*   d_samples;      // e.g., [2.2, 6.0, 7.1, 2.9, 3.5, 0.3, 2.9, 2.0, 6.1, 999.5]
+     * int*     d_histogram;    // e.g., [ -, -, -, -, -, -, -, -]
+     * int      num_levels;     // e.g., 7       (seven level boundaries for six bins)
+     * float    lower_level;    // e.g., 0.0     (lower sample value boundary of lowest bin)
+     * float    upper_level;    // e.g., 12.0    (upper sample value boundary of upper bin)
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceHistogram::HistogramEven(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, lower_level, upper_level, num_samples);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Compute histograms
+     * cub::DeviceHistogram::HistogramEven(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, lower_level, upper_level, num_samples);
+     *
+     * // d_histogram   <-- [1, 0, 5, 0, 3, 0, 0, 0];
+     *
+     * \endcode
+     *
+     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
+     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
+     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
+     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
+     */
+    template <
+        typename            SampleIteratorT,
+        typename            CounterT,
+        typename            LevelT,
+        typename            OffsetT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t HistogramEven(
+        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the input sequence of data samples.
+        CounterT*           d_histogram,                                ///< [out] The pointer to the histogram counter output array of length <tt>num_levels</tt> - 1.
+        int                 num_levels,                                 ///< [in] The number of boundaries (levels) for delineating histogram samples.  Implies that the number of bins is <tt>num_levels</tt> - 1.
+        LevelT              lower_level,                                ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin.
+        LevelT              upper_level,                                ///< [in] The upper sample value bound (exclusive) for the highest histogram bin.
+        OffsetT             num_samples,                                ///< [in] The number of input samples (i.e., the length of \p d_samples)
+        cudaStream_t        stream                  = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous       = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        /// The sample value type of the input iterator
+        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+        CounterT*           d_histogram1[1]     = {d_histogram};
+        int                 num_levels1[1]      = {num_levels};
+        LevelT              lower_level1[1]     = {lower_level};
+        LevelT              upper_level1[1]     = {upper_level};
+
+        return MultiHistogramEven<1, 1>(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram1,
+            num_levels1,
+            lower_level1,
+            upper_level1,
+            num_samples,
+            1,
+            sizeof(SampleT) * num_samples,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes an intensity histogram from a sequence of data samples using equal-width bins.
+     *
+     * \par
+     * - A two-dimensional <em>region of interest</em> within \p d_samples can be specified
+     *   using the \p num_row_samples, num_rows, and \p row_stride_bytes parameters.
+     * - The row stride must be a whole multiple of the sample data type
+     *   size, i.e., <tt>(row_stride_bytes % sizeof(SampleT)) == 0</tt>.
+     * - The number of histogram bins is (\p num_levels - 1)
+     * - All bins comprise the same width of sample values: (\p upper_level - \p lower_level) / (\p num_levels - 1)
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the computation of a six-bin histogram
+     * from a 2x5 region of interest within a flattened 2x7 array of float samples.
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input samples and
+     * // output histogram
+     * int      num_row_samples;    // e.g., 5
+     * int      num_rows;           // e.g., 2;
+     * size_t   row_stride_bytes;   // e.g., 7 * sizeof(float)
+     * float*   d_samples;          // e.g., [2.2, 6.0, 7.1, 2.9, 3.5,   -, -,
+     *                              //        0.3, 2.9, 2.0, 6.1, 999.5, -, -]
+     * int*     d_histogram;        // e.g., [ -, -, -, -, -, -, -, -]
+     * int      num_levels;         // e.g., 7       (seven level boundaries for six bins)
+     * float    lower_level;        // e.g., 0.0     (lower sample value boundary of lowest bin)
+     * float    upper_level;        // e.g., 12.0    (upper sample value boundary of upper bin)
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage  = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceHistogram::HistogramEven(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, lower_level, upper_level,
+     *     num_row_samples, num_rows, row_stride_bytes);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Compute histograms
+     * cub::DeviceHistogram::HistogramEven(d_temp_storage, temp_storage_bytes, d_samples, d_histogram,
+     *     d_samples, d_histogram, num_levels, lower_level, upper_level,
+     *     num_row_samples, num_rows, row_stride_bytes);
+     *
+     * // d_histogram   <-- [1, 0, 5, 0, 3, 0, 0, 0];
+     *
+     * \endcode
+     *
+     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
+     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
+     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
+     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
+     */
+    template <
+        typename            SampleIteratorT,
+        typename            CounterT,
+        typename            LevelT,
+        typename            OffsetT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t HistogramEven(
+        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the input sequence of data samples.
+        CounterT*           d_histogram,                                ///< [out] The pointer to the histogram counter output array of length <tt>num_levels</tt> - 1.
+        int                 num_levels,                                 ///< [in] The number of boundaries (levels) for delineating histogram samples.  Implies that the number of bins is <tt>num_levels</tt> - 1.
+        LevelT              lower_level,                                ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin.
+        LevelT              upper_level,                                ///< [in] The upper sample value bound (exclusive) for the highest histogram bin.
+        OffsetT             num_row_samples,                            ///< [in] The number of data samples per row in the region of interest
+        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+        size_t              row_stride_bytes,                           ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+        cudaStream_t        stream                  = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous       = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        CounterT*           d_histogram1[1]     = {d_histogram};
+        int                 num_levels1[1]      = {num_levels};
+        LevelT              lower_level1[1]     = {lower_level};
+        LevelT              upper_level1[1]     = {upper_level};
+
+        return MultiHistogramEven<1, 1>(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram1,
+            num_levels1,
+            lower_level1,
+            upper_level1,
+            num_row_samples,
+            num_rows,
+            row_stride_bytes,
+            stream,
+            debug_synchronous);
+    }
+
+    /**
+     * \brief Computes per-channel intensity histograms from a sequence of multi-channel "pixel" data samples using equal-width bins.
+     *
+     * \par
+     * - The input is a sequence of <em>pixel</em> structures, where each pixel comprises
+     *   a record of \p NUM_CHANNELS consecutive data samples (e.g., an <em>RGBA</em> pixel).
+     * - Of the \p NUM_CHANNELS specified, the function will only compute histograms
+     *   for the first \p NUM_ACTIVE_CHANNELS (e.g., only <em>RGB</em> histograms from <em>RGBA</em>
+     *   pixel samples).
+     * - The number of histogram bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+     * - For channel<sub><em>i</em></sub>, the range of values for all histogram bins
+     *   have the same width: (<tt>upper_level[i]</tt> - <tt>lower_level[i]</tt>) / (<tt> num_levels[i]</tt> - 1)
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the computation of three 256-bin <em>RGB</em> histograms
+     * from a quad-channel sequence of <em>RGBA</em> pixels (8 bits per channel per pixel)
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input samples
+     * // and output histograms
+     * int              num_pixels;         // e.g., 5
+     * unsigned char*   d_samples;          // e.g., [(2, 6, 7, 5), (3, 0, 2, 1), (7, 0, 6, 2),
+     *                                      //        (0, 6, 7, 5), (3, 0, 2, 6)]
+     * int*             d_histogram[3];     // e.g., three device pointers to three device buffers,
+     *                                      //       each allocated with 256 integer counters
+     * int              num_levels[3];      // e.g., {257, 257, 257};
+     * unsigned int     lower_level[3];     // e.g., {0, 0, 0};
+     * unsigned int     upper_level[3];     // e.g., {256, 256, 256};
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceHistogram::MultiHistogramEven<4, 3>(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, lower_level, upper_level, num_pixels);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Compute histograms
+     * cub::DeviceHistogram::MultiHistogramEven<4, 3>(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, lower_level, upper_level, num_pixels);
+     *
+     * // d_histogram   <-- [ [1, 0, 1, 2, 0, 0, 0, 1, 0, 0, 0, ..., 0],
+     * //                     [0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, ..., 0],
+     * //                     [0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, ..., 0] ]
+     *
+     * \endcode
+     *
+     * \tparam NUM_CHANNELS             Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
+     * \tparam NUM_ACTIVE_CHANNELS      <b>[inferred]</b> Number of channels actively being histogrammed
+     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
+     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
+     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
+     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
+     */
+    template <
+        int                 NUM_CHANNELS,
+        int                 NUM_ACTIVE_CHANNELS,
+        typename            SampleIteratorT,
+        typename            CounterT,
+        typename            LevelT,
+        typename            OffsetT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t MultiHistogramEven(
+        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four <em>RGBA</em> 8-bit samples).
+        CounterT*           d_histogram[NUM_ACTIVE_CHANNELS],           ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histogram[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+        int                 num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+        LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+        LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+        OffsetT             num_pixels,                                 ///< [in] The number of multi-channel pixels (i.e., the length of \p d_samples / NUM_CHANNELS)
+        cudaStream_t        stream                  = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous       = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        /// The sample value type of the input iterator
+        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+        return MultiHistogramEven<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram,
+            num_levels,
+            lower_level,
+            upper_level,
+            num_pixels,
+            1,
+            sizeof(SampleT) * NUM_CHANNELS * num_pixels,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes per-channel intensity histograms from a sequence of multi-channel "pixel" data samples using equal-width bins.
+     *
+     * \par
+     * - The input is a sequence of <em>pixel</em> structures, where each pixel comprises
+     *   a record of \p NUM_CHANNELS consecutive data samples (e.g., an <em>RGBA</em> pixel).
+     * - Of the \p NUM_CHANNELS specified, the function will only compute histograms
+     *   for the first \p NUM_ACTIVE_CHANNELS (e.g., only <em>RGB</em> histograms from <em>RGBA</em>
+     *   pixel samples).
+     * - A two-dimensional <em>region of interest</em> within \p d_samples can be specified
+     *   using the \p num_row_samples, num_rows, and \p row_stride_bytes parameters.
+     * - The row stride must be a whole multiple of the sample data type
+     *   size, i.e., <tt>(row_stride_bytes % sizeof(SampleT)) == 0</tt>.
+     * - The number of histogram bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+     * - For channel<sub><em>i</em></sub>, the range of values for all histogram bins
+     *   have the same width: (<tt>upper_level[i]</tt> - <tt>lower_level[i]</tt>) / (<tt> num_levels[i]</tt> - 1)
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the computation of three 256-bin <em>RGB</em> histograms from a 2x3 region of
+     * interest of within a flattened 2x4 array of quad-channel <em>RGBA</em> pixels (8 bits per channel per pixel).
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input samples
+     * // and output histograms
+     * int              num_row_pixels;     // e.g., 3
+     * int              num_rows;           // e.g., 2
+     * size_t           row_stride_bytes;   // e.g., 4 * sizeof(unsigned char) * NUM_CHANNELS
+     * unsigned char*   d_samples;          // e.g., [(2, 6, 7, 5), (3, 0, 2, 1), (7, 0, 6, 2), (-, -, -, -),
+     *                                      //        (0, 6, 7, 5), (3, 0, 2, 6), (1, 1, 1, 1), (-, -, -, -)]
+     * int*             d_histogram[3];     // e.g., three device pointers to three device buffers,
+     *                                      //       each allocated with 256 integer counters
+     * int              num_levels[3];      // e.g., {257, 257, 257};
+     * unsigned int     lower_level[3];     // e.g., {0, 0, 0};
+     * unsigned int     upper_level[3];     // e.g., {256, 256, 256};
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceHistogram::MultiHistogramEven<4, 3>(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, lower_level, upper_level,
+     *     num_row_pixels, num_rows, row_stride_bytes);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Compute histograms
+     * cub::DeviceHistogram::MultiHistogramEven<4, 3>(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, lower_level, upper_level,
+     *     num_row_pixels, num_rows, row_stride_bytes);
+     *
+     * // d_histogram   <-- [ [1, 1, 1, 2, 0, 0, 0, 1, 0, 0, 0, ..., 0],
+     * //                     [0, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, ..., 0],
+     * //                     [0, 1, 2, 0, 0, 0, 1, 2, 0, 0, 0, ..., 0] ]
+     *
+     * \endcode
+     *
+     * \tparam NUM_CHANNELS             Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
+     * \tparam NUM_ACTIVE_CHANNELS      <b>[inferred]</b> Number of channels actively being histogrammed
+     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
+     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
+     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
+     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
+     */
+    template <
+        int                 NUM_CHANNELS,
+        int                 NUM_ACTIVE_CHANNELS,
+        typename            SampleIteratorT,
+        typename            CounterT,
+        typename            LevelT,
+        typename            OffsetT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t MultiHistogramEven(
+        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four <em>RGBA</em> 8-bit samples).
+        CounterT*           d_histogram[NUM_ACTIVE_CHANNELS],           ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histogram[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+        int                 num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+        LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+        LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+        size_t              row_stride_bytes,                           ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+        cudaStream_t        stream                  = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous       = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        /// The sample value type of the input iterator
+        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+        Int2Type<sizeof(SampleT) == 1> is_byte_sample;
+
+        if ((sizeof(OffsetT) > sizeof(int)) &&
+            ((unsigned long long) (num_rows * row_stride_bytes) < (unsigned long long) std::numeric_limits<int>::max()))
+        {
+            // Down-convert OffsetT data type
+
+
+            return DipatchHistogram<NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, LevelT, int>::DispatchEven(
+                d_temp_storage, temp_storage_bytes, d_samples, d_histogram, num_levels, lower_level, upper_level,
+                (int) num_row_pixels, (int) num_rows, (int) (row_stride_bytes / sizeof(SampleT)),
+                stream, debug_synchronous, is_byte_sample);
+        }
+
+        return DipatchHistogram<NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, LevelT, OffsetT>::DispatchEven(
+            d_temp_storage, temp_storage_bytes, d_samples, d_histogram, num_levels, lower_level, upper_level,
+            num_row_pixels, num_rows, (OffsetT) (row_stride_bytes / sizeof(SampleT)),
+            stream, debug_synchronous, is_byte_sample);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Custom bin ranges
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Computes an intensity histogram from a sequence of data samples using the specified bin boundary levels.
+     *
+     * \par
+     * - The number of histogram bins is (\p num_levels - 1)
+     * - The value range for bin<sub><em>i</em></sub> is [<tt>level[i]</tt>, <tt>level[i+1]</tt>)
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the computation of an six-bin histogram
+     * from a sequence of float samples
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input samples and
+     * // output histogram
+     * int      num_samples;    // e.g., 10
+     * float*   d_samples;      // e.g., [2.2, 6.0, 7.1, 2.9, 3.5, 0.3, 2.9, 2.0, 6.1, 999.5]
+     * int*     d_histogram;    // e.g., [ -, -, -, -, -, -, -, -]
+     * int      num_levels      // e.g., 7 (seven level boundaries for six bins)
+     * float*   d_levels;       // e.g., [0.0, 2.0, 4.0, 6.0, 8.0, 12.0, 16.0]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceHistogram::HistogramRange(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, d_levels, num_samples);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Compute histograms
+     * cub::DeviceHistogram::HistogramRange(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, d_levels, num_samples);
+     *
+     * // d_histogram   <-- [1, 0, 5, 0, 3, 0, 0, 0];
+     *
+     * \endcode
+     *
+     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
+     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
+     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
+     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
+     */
+    template <
+        typename            SampleIteratorT,
+        typename            CounterT,
+        typename            LevelT,
+        typename            OffsetT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t HistogramRange(
+        void*               d_temp_storage,                         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                              ///< [in] The pointer to the input sequence of data samples.
+        CounterT*           d_histogram,                            ///< [out] The pointer to the histogram counter output array of length <tt>num_levels</tt> - 1.
+        int                 num_levels,                             ///< [in] The number of boundaries (levels) for delineating histogram samples.  Implies that the number of bins is <tt>num_levels</tt> - 1.
+        LevelT*             d_levels,                               ///< [in] The pointer to the array of boundaries (levels).  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
+        OffsetT             num_samples,                            ///< [in] The number of data samples per row in the region of interest
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        /// The sample value type of the input iterator
+        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+        CounterT*           d_histogram1[1] = {d_histogram};
+        int                 num_levels1[1]  = {num_levels};
+        LevelT*             d_levels1[1]    = {d_levels};
+
+        return MultiHistogramRange<1, 1>(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram1,
+            num_levels1,
+            d_levels1,
+            num_samples,
+            1,
+            sizeof(SampleT) * num_samples,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes an intensity histogram from a sequence of data samples using the specified bin boundary levels.
+     *
+     * \par
+     * - A two-dimensional <em>region of interest</em> within \p d_samples can be specified
+     *   using the \p num_row_samples, num_rows, and \p row_stride_bytes parameters.
+     * - The row stride must be a whole multiple of the sample data type
+     *   size, i.e., <tt>(row_stride_bytes % sizeof(SampleT)) == 0</tt>.
+     * - The number of histogram bins is (\p num_levels - 1)
+     * - The value range for bin<sub><em>i</em></sub> is [<tt>level[i]</tt>, <tt>level[i+1]</tt>)
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the computation of a six-bin histogram
+     * from a 2x5 region of interest within a flattened 2x7 array of float samples.
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input samples and
+     * // output histogram
+     * int      num_row_samples;    // e.g., 5
+     * int      num_rows;           // e.g., 2;
+     * int      row_stride_bytes;   // e.g., 7 * sizeof(float)
+     * float*   d_samples;          // e.g., [2.2, 6.0, 7.1, 2.9, 3.5,   -, -,
+     *                              //        0.3, 2.9, 2.0, 6.1, 999.5, -, -]
+     * int*     d_histogram;        // e.g., [ , , , , , , , ]
+     * int      num_levels          // e.g., 7 (seven level boundaries for six bins)
+     * float    *d_levels;          // e.g., [0.0, 2.0, 4.0, 6.0, 8.0, 12.0, 16.0]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceHistogram::HistogramRange(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, d_levels,
+     *     num_row_samples, num_rows, row_stride_bytes);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Compute histograms
+     * cub::DeviceHistogram::HistogramRange(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, d_levels,
+     *     num_row_samples, num_rows, row_stride_bytes);
+     *
+     * // d_histogram   <-- [1, 0, 5, 0, 3, 0, 0, 0];
+     *
+     * \endcode
+     *
+     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
+     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
+     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
+     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
+     */
+    template <
+        typename            SampleIteratorT,
+        typename            CounterT,
+        typename            LevelT,
+        typename            OffsetT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t HistogramRange(
+        void*               d_temp_storage,                         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                              ///< [in] The pointer to the input sequence of data samples.
+        CounterT*           d_histogram,                            ///< [out] The pointer to the histogram counter output array of length <tt>num_levels</tt> - 1.
+        int                 num_levels,                             ///< [in] The number of boundaries (levels) for delineating histogram samples.  Implies that the number of bins is <tt>num_levels</tt> - 1.
+        LevelT*             d_levels,                               ///< [in] The pointer to the array of boundaries (levels).  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
+        OffsetT             num_row_samples,                        ///< [in] The number of data samples per row in the region of interest
+        OffsetT             num_rows,                               ///< [in] The number of rows in the region of interest
+        size_t              row_stride_bytes,                       ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        CounterT*           d_histogram1[1]     = {d_histogram};
+        int                 num_levels1[1]      = {num_levels};
+        LevelT*             d_levels1[1]        = {d_levels};
+
+        return MultiHistogramRange<1, 1>(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram1,
+            num_levels1,
+            d_levels1,
+            num_row_samples,
+            num_rows,
+            row_stride_bytes,
+            stream,
+            debug_synchronous);
+    }
+
+    /**
+     * \brief Computes per-channel intensity histograms from a sequence of multi-channel "pixel" data samples using the specified bin boundary levels.
+     *
+     * \par
+     * - The input is a sequence of <em>pixel</em> structures, where each pixel comprises
+     *   a record of \p NUM_CHANNELS consecutive data samples (e.g., an <em>RGBA</em> pixel).
+     * - Of the \p NUM_CHANNELS specified, the function will only compute histograms
+     *   for the first \p NUM_ACTIVE_CHANNELS (e.g., <em>RGB</em> histograms from <em>RGBA</em>
+     *   pixel samples).
+     * - The number of histogram bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+     * - For channel<sub><em>i</em></sub>, the range of values for all histogram bins
+     *   have the same width: (<tt>upper_level[i]</tt> - <tt>lower_level[i]</tt>) / (<tt> num_levels[i]</tt> - 1)
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the computation of three 4-bin <em>RGB</em> histograms
+     * from a quad-channel sequence of <em>RGBA</em> pixels (8 bits per channel per pixel)
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input samples
+     * // and output histograms
+     * int            num_pixels;       // e.g., 5
+     * unsigned char  *d_samples;       // e.g., [(2, 6, 7, 5),(3, 0, 2, 1),(7, 0, 6, 2),
+     *                                  //        (0, 6, 7, 5),(3, 0, 2, 6)]
+     * unsigned int   *d_histogram[3];  // e.g., [[ -, -, -, -],[ -, -, -, -],[ -, -, -, -]];
+     * int            num_levels[3];    // e.g., {5, 5, 5};
+     * unsigned int   *d_levels[3];     // e.g., [ [0, 2, 4, 6, 8],
+     *                                  //         [0, 2, 4, 6, 8],
+     *                                  //         [0, 2, 4, 6, 8] ];
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceHistogram::MultiHistogramRange<4, 3>(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, d_levels, num_pixels);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Compute histograms
+     * cub::DeviceHistogram::MultiHistogramRange<4, 3>(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, d_levels, num_pixels);
+     *
+     * // d_histogram   <-- [ [1, 3, 0, 1],
+     * //                     [3, 0, 0, 2],
+     * //                     [0, 2, 0, 3] ]
+     *
+     * \endcode
+     *
+     * \tparam NUM_CHANNELS             Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
+     * \tparam NUM_ACTIVE_CHANNELS      <b>[inferred]</b> Number of channels actively being histogrammed
+     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
+     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
+     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
+     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
+     */
+    template <
+        int                 NUM_CHANNELS,
+        int                 NUM_ACTIVE_CHANNELS,
+        typename            SampleIteratorT,
+        typename            CounterT,
+        typename            LevelT,
+        typename            OffsetT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t MultiHistogramRange(
+        void*               d_temp_storage,                         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                              ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four <em>RGBA</em> 8-bit samples).
+        CounterT*           d_histogram[NUM_ACTIVE_CHANNELS],       ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histogram[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+        int                 num_levels[NUM_ACTIVE_CHANNELS],        ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+        LevelT*             d_levels[NUM_ACTIVE_CHANNELS],          ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
+        OffsetT             num_pixels,                             ///< [in] The number of multi-channel pixels (i.e., the length of \p d_samples / NUM_CHANNELS)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        /// The sample value type of the input iterator
+        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+        return MultiHistogramRange<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram,
+            num_levels,
+            d_levels,
+            num_pixels,
+            1,
+            sizeof(SampleT) * NUM_CHANNELS * num_pixels,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes per-channel intensity histograms from a sequence of multi-channel "pixel" data samples using the specified bin boundary levels.
+     *
+     * \par
+     * - The input is a sequence of <em>pixel</em> structures, where each pixel comprises
+     *   a record of \p NUM_CHANNELS consecutive data samples (e.g., an <em>RGBA</em> pixel).
+     * - Of the \p NUM_CHANNELS specified, the function will only compute histograms
+     *   for the first \p NUM_ACTIVE_CHANNELS (e.g., <em>RGB</em> histograms from <em>RGBA</em>
+     *   pixel samples).
+     * - A two-dimensional <em>region of interest</em> within \p d_samples can be specified
+     *   using the \p num_row_samples, num_rows, and \p row_stride_bytes parameters.
+     * - The row stride must be a whole multiple of the sample data type
+     *   size, i.e., <tt>(row_stride_bytes % sizeof(SampleT)) == 0</tt>.
+     * - The number of histogram bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+     * - For channel<sub><em>i</em></sub>, the range of values for all histogram bins
+     *   have the same width: (<tt>upper_level[i]</tt> - <tt>lower_level[i]</tt>) / (<tt> num_levels[i]</tt> - 1)
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the computation of three 4-bin <em>RGB</em> histograms from a 2x3 region of
+     * interest of within a flattened 2x4 array of quad-channel <em>RGBA</em> pixels (8 bits per channel per pixel).
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input samples
+     * // and output histograms
+     * int              num_row_pixels;     // e.g., 3
+     * int              num_rows;           // e.g., 2
+     * size_t           row_stride_bytes;   // e.g., 4 * sizeof(unsigned char) * NUM_CHANNELS
+     * unsigned char*   d_samples;          // e.g., [(2, 6, 7, 5),(3, 0, 2, 1),(1, 1, 1, 1),(-, -, -, -),
+     *                                      //        (7, 0, 6, 2),(0, 6, 7, 5),(3, 0, 2, 6),(-, -, -, -)]
+     * int*             d_histogram[3];     // e.g., [[ -, -, -, -],[ -, -, -, -],[ -, -, -, -]];
+     * int              num_levels[3];      // e.g., {5, 5, 5};
+     * unsigned int*    d_levels[3];        // e.g., [ [0, 2, 4, 6, 8],
+     *                                      //         [0, 2, 4, 6, 8],
+     *                                      //         [0, 2, 4, 6, 8] ];
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceHistogram::MultiHistogramRange<4, 3>(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, d_levels, num_row_pixels, num_rows, row_stride_bytes);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Compute histograms
+     * cub::DeviceHistogram::MultiHistogramRange<4, 3>(d_temp_storage, temp_storage_bytes,
+     *     d_samples, d_histogram, num_levels, d_levels, num_row_pixels, num_rows, row_stride_bytes);
+     *
+     * // d_histogram   <-- [ [2, 3, 0, 1],
+     * //                     [3, 0, 0, 2],
+     * //                     [1, 2, 0, 3] ]
+     *
+     * \endcode
+     *
+     * \tparam NUM_CHANNELS             Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
+     * \tparam NUM_ACTIVE_CHANNELS      <b>[inferred]</b> Number of channels actively being histogrammed
+     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
+     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
+     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
+     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
+     */
+    template <
+        int                 NUM_CHANNELS,
+        int                 NUM_ACTIVE_CHANNELS,
+        typename            SampleIteratorT,
+        typename            CounterT,
+        typename            LevelT,
+        typename            OffsetT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t MultiHistogramRange(
+        void*               d_temp_storage,                         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                              ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four <em>RGBA</em> 8-bit samples).
+        CounterT*           d_histogram[NUM_ACTIVE_CHANNELS],       ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histogram[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+        int                 num_levels[NUM_ACTIVE_CHANNELS],        ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+        LevelT*             d_levels[NUM_ACTIVE_CHANNELS],          ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
+        OffsetT             num_row_pixels,                         ///< [in] The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                               ///< [in] The number of rows in the region of interest
+        size_t              row_stride_bytes,                       ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        /// The sample value type of the input iterator
+        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+        Int2Type<sizeof(SampleT) == 1> is_byte_sample;
+
+        if ((sizeof(OffsetT) > sizeof(int)) &&
+            ((unsigned long long) (num_rows * row_stride_bytes) < (unsigned long long) std::numeric_limits<int>::max()))
+        {
+            // Down-convert OffsetT data type
+            return DipatchHistogram<NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, LevelT, int>::DispatchRange(
+                d_temp_storage, temp_storage_bytes, d_samples, d_histogram, num_levels, d_levels,
+                (int) num_row_pixels, (int) num_rows, (int) (row_stride_bytes / sizeof(SampleT)),
+                stream, debug_synchronous, is_byte_sample);
+        }
+
+        return DipatchHistogram<NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, LevelT, OffsetT>::DispatchRange(
+            d_temp_storage, temp_storage_bytes, d_samples, d_histogram, num_levels, d_levels,
+            num_row_pixels, num_rows, (OffsetT) (row_stride_bytes / sizeof(SampleT)),
+            stream, debug_synchronous, is_byte_sample);
+    }
+
+
+
+    //@}  end member group
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_partition.cuh b/third_party/cub/cub/device/device_partition.cuh
new file mode 100644
index 0000000..5053540
--- /dev/null
+++ b/third_party/cub/cub/device/device_partition.cuh
@@ -0,0 +1,273 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DevicePartition provides device-wide, parallel operations for partitioning sequences of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch/dispatch_select_if.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DevicePartition provides device-wide, parallel operations for partitioning sequences of data items residing within device-accessible memory. ![](partition_logo.png)
+ * \ingroup SingleModule
+ *
+ * \par Overview
+ * These operations apply a selection criterion to construct a partitioned output sequence from items selected/unselected from
+ * a specified input sequence.
+ *
+ * \par Usage Considerations
+ * \cdp_class{DevicePartition}
+ *
+ * \par Performance
+ * \linear_performance{partition}
+ *
+ * \par
+ * The following chart illustrates DevicePartition::If
+ * performance across different CUDA architectures for \p int32 items,
+ * where 50% of the items are randomly selected for the first partition.
+ * \plots_below
+ *
+ * \image html partition_if_int32_50_percent.png
+ *
+ */
+struct DevicePartition
+{
+    /**
+     * \brief Uses the \p d_flags sequence to split the corresponding items from \p d_in into a partitioned sequence \p d_out.  The total number of items copied into the first partition is written to \p d_num_selected_out. ![](partition_flags_logo.png)
+     *
+     * \par
+     * - The value type of \p d_flags must be castable to \p bool (e.g., \p bool, \p char, \p int, etc.).
+     * - Copies of the selected items are compacted into \p d_out and maintain their original
+     *   relative ordering, however copies of the unselected items are compacted into the
+     *   rear of \p d_out in reverse order.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>       // or equivalently <cub/device/device_partition.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input, flags, and output
+     * int  num_items;              // e.g., 8
+     * int  *d_in;                  // e.g., [1, 2, 3, 4, 5, 6, 7, 8]
+     * char *d_flags;               // e.g., [1, 0, 0, 1, 0, 1, 1, 0]
+     * int  *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int  *d_num_selected_out;    // e.g., [ ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DevicePartition::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run selection
+     * cub::DevicePartition::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items);
+     *
+     * // d_out                 <-- [1, 4, 6, 7, 8, 5, 3, 2]
+     * // d_num_selected_out    <-- [4]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam FlagIterator         <b>[inferred]</b> Random-access input iterator type for reading selection flags \iterator
+     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing output items \iterator
+     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    FlagIterator,
+        typename                    OutputIteratorT,
+        typename                    NumSelectedIteratorT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Flagged(
+        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
+        FlagIterator                d_flags,                        ///< [in] Pointer to the input sequence of selection flags
+        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of partitioned data items
+        NumSelectedIteratorT        d_num_selected_out,             ///< [out] Pointer to the output total number of items selected (i.e., the offset of the unselected partition)
+        int                         num_items,                      ///< [in] Total number of items to select from
+        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        typedef int                     OffsetT;         // Signed integer type for global offsets
+        typedef NullType                SelectOp;       // Selection op (not used)
+        typedef NullType                EqualityOp;     // Equality operator (not used)
+
+        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, true>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_flags,
+            d_out,
+            d_num_selected_out,
+            SelectOp(),
+            EqualityOp(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Uses the \p select_op functor to split the corresponding items from \p d_in into a partitioned sequence \p d_out.  The total number of items copied into the first partition is written to \p d_num_selected_out. ![](partition_logo.png)
+     *
+     * \par
+     * - Copies of the selected items are compacted into \p d_out and maintain their original
+     *   relative ordering, however copies of the unselected items are compacted into the
+     *   rear of \p d_out in reverse order.
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated partition-if performance across different
+     * CUDA architectures for \p int32 and \p int64 items, respectively.  Items are
+     * selected for the first partition with 50% probability.
+     *
+     * \image html partition_if_int32_50_percent.png
+     * \image html partition_if_int64_50_percent.png
+     *
+     * \par
+     * The following charts are similar, but 5% selection probability for the first partition:
+     *
+     * \image html partition_if_int32_5_percent.png
+     * \image html partition_if_int64_5_percent.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_partition.cuh>
+     *
+     * // Functor type for selecting values less than some criteria
+     * struct LessThan
+     * {
+     *     int compare;
+     *
+     *     CUB_RUNTIME_FUNCTION __forceinline__
+     *     LessThan(int compare) : compare(compare) {}
+     *
+     *     CUB_RUNTIME_FUNCTION __forceinline__
+     *     bool operator()(const int &a) const {
+     *         return (a < compare);
+     *     }
+     * };
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int      num_items;              // e.g., 8
+     * int      *d_in;                  // e.g., [0, 2, 3, 9, 5, 2, 81, 8]
+     * int      *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int      *d_num_selected_out;    // e.g., [ ]
+     * LessThan select_op(7);
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run selection
+     * cub::DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op);
+     *
+     * // d_out                 <-- [0, 2, 3, 5, 2, 8, 81, 9]
+     * // d_num_selected_out    <-- [5]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing output items \iterator
+     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
+     * \tparam SelectOp             <b>[inferred]</b> Selection functor type having member <tt>bool operator()(const T &a)</tt>
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT,
+        typename                    NumSelectedIteratorT,
+        typename                    SelectOp>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t If(
+        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of partitioned data items
+        NumSelectedIteratorT        d_num_selected_out,             ///< [out] Pointer to the output total number of items selected (i.e., the offset of the unselected partition)
+        int                         num_items,                      ///< [in] Total number of items to select from
+        SelectOp                    select_op,                      ///< [in] Unary selection operator
+        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        typedef int                     OffsetT;         // Signed integer type for global offsets
+        typedef NullType*               FlagIterator;   // FlagT iterator type (not used)
+        typedef NullType                EqualityOp;     // Equality operator (not used)
+
+        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, true>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            NULL,
+            d_out,
+            d_num_selected_out,
+            select_op,
+            EqualityOp(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+};
+
+/**
+ * \example example_device_partition_flagged.cu
+ * \example example_device_partition_if.cu
+ */
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_radix_sort.cuh b/third_party/cub/cub/device/device_radix_sort.cuh
new file mode 100644
index 0000000..1c0bdbe
--- /dev/null
+++ b/third_party/cub/cub/device/device_radix_sort.cuh
@@ -0,0 +1,797 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceRadixSort provides device-wide, parallel operations for computing a radix sort across a sequence of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch/dispatch_radix_sort.cuh"
+#include "../util_arch.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceRadixSort provides device-wide, parallel operations for computing a radix sort across a sequence of data items residing within device-accessible memory. ![](sorting_logo.png)
+ * \ingroup SingleModule
+ *
+ * \par Overview
+ * The [<em>radix sorting method</em>](http://en.wikipedia.org/wiki/Radix_sort) arranges
+ * items into ascending (or descending) order.  The algorithm relies upon a positional representation for
+ * keys, i.e., each key is comprised of an ordered sequence of symbols (e.g., digits,
+ * characters, etc.) specified from least-significant to most-significant.  For a
+ * given input sequence of keys and a set of rules specifying a total ordering
+ * of the symbolic alphabet, the radix sorting method produces a lexicographic
+ * ordering of those keys.
+ *
+ * \par
+ * DeviceRadixSort can sort all of the built-in C++ numeric primitive types
+ * (<tt>unsigned char</tt>, \p int, \p double, etc.) as well as CUDA's \p __half
+ * half-precision floating-point type.  Although the direct radix sorting
+ * method can only be applied to unsigned integral types, DeviceRadixSort
+ * is able to sort signed and floating-point types via simple bit-wise transformations
+ * that ensure lexicographic key ordering.
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceRadixSort}
+ *
+ * \par Performance
+ * \linear_performance{radix sort} The following chart illustrates DeviceRadixSort::SortKeys
+ * performance across different CUDA architectures for uniform-random \p uint32 keys.
+ * \plots_below
+ *
+ * \image html lsb_radix_sort_int32_keys.png
+ *
+ */
+struct DeviceRadixSort
+{
+
+    /******************************************************************//**
+     * \name KeyT-value pairs
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Sorts key-value pairs into ascending order. (~<em>2N </em>auxiliary storage required)
+     *
+     * \par
+     * - The contents of the input data are not altered by the sorting operation
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated sorting performance across different
+     * CUDA architectures for uniform-random <tt>uint32,uint32</tt> and
+     * <tt>uint64,uint64</tt> pairs, respectively.
+     *
+     * \image html lsb_radix_sort_int32_pairs.png
+     * \image html lsb_radix_sort_int64_pairs.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sorting of a device vector of \p int keys
+     * with associated vector of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_keys_out;        // e.g., [        ...        ]
+     * int  *d_values_in;       // e.g., [0, 1, 2, 3, 4, 5, 6]
+     * int  *d_values_out;      // e.g., [        ...        ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes,
+     *     d_keys_in, d_keys_out, d_values_in, d_values_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes,
+     *     d_keys_in, d_keys_out, d_values_in, d_values_out, num_items);
+     *
+     * // d_keys_out            <-- [0, 3, 5, 6, 7, 8, 9]
+     * // d_values_out          <-- [5, 4, 3, 1, 2, 0, 6]
+     *
+     * \endcode
+     *
+     * \tparam KeyT      <b>[inferred]</b> KeyT type
+     * \tparam ValueT    <b>[inferred]</b> ValueT type
+     */
+    template <
+        typename            KeyT,
+        typename            ValueT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortPairs(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        const KeyT          *d_keys_in,                             ///< [in] Pointer to the input data of key data to sort
+        KeyT                *d_keys_out,                            ///< [out] Pointer to the sorted output sequence of key data
+        const ValueT        *d_values_in,                           ///< [in] Pointer to the corresponding input sequence of associated value items
+        ValueT              *d_values_out,                          ///< [out] Pointer to the correspondingly-reordered output sequence of associated value items
+        int                 num_items,                              ///< [in] Number of items to sort
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        DoubleBuffer<KeyT>       d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
+        DoubleBuffer<ValueT>     d_values(const_cast<ValueT*>(d_values_in), d_values_out);
+
+        return DispatchRadixSort<false, KeyT, ValueT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            begin_bit,
+            end_bit,
+            false,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts key-value pairs into ascending order. (~<em>N </em>auxiliary storage required)
+     *
+     * \par
+     * - The sorting operation is given a pair of key buffers and a corresponding
+     *   pair of associated value buffers.  Each pair is managed by a DoubleBuffer
+     *   structure that indicates which of the two buffers is "current" (and thus
+     *   contains the input data to be sorted).
+     * - The contents of both buffers within each pair may be altered by the sorting
+     *   operation.
+     * - Upon completion, the sorting operation will update the "current" indicator
+     *   within each DoubleBuffer wrapper to reference which of the two buffers
+     *   now contains the sorted output sequence (a function of the number of key bits
+     *   specified and the targeted device architecture).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageP
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated sorting performance across different
+     * CUDA architectures for uniform-random <tt>uint32,uint32</tt> and
+     * <tt>uint64,uint64</tt> pairs, respectively.
+     *
+     * \image html lsb_radix_sort_int32_pairs.png
+     * \image html lsb_radix_sort_int64_pairs.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sorting of a device vector of \p int keys
+     * with associated vector of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_key_alt_buf;     // e.g., [        ...        ]
+     * int  *d_value_buf;       // e.g., [0, 1, 2, 3, 4, 5, 6]
+     * int  *d_value_alt_buf;   // e.g., [        ...        ]
+     * ...
+     *
+     * // Create a set of DoubleBuffers to wrap pairs of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     * cub::DoubleBuffer<int> d_values(d_value_buf, d_value_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items);
+     *
+     * // d_keys.Current()      <-- [0, 3, 5, 6, 7, 8, 9]
+     * // d_values.Current()    <-- [5, 4, 3, 1, 2, 0, 6]
+     *
+     * \endcode
+     *
+     * \tparam KeyT      <b>[inferred]</b> KeyT type
+     * \tparam ValueT    <b>[inferred]</b> ValueT type
+     */
+    template <
+        typename            KeyT,
+        typename            ValueT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortPairs(
+        void                    *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>      &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        DoubleBuffer<ValueT>    &d_values,                              ///< [in,out] Double-buffer of values whose "current" device-accessible buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
+        int                     num_items,                              ///< [in] Number of items to sort
+        int                     begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                     end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchRadixSort<false, KeyT, ValueT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            begin_bit,
+            end_bit,
+            true,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts key-value pairs into descending order. (~<em>2N</em> auxiliary storage required).
+     *
+     * \par
+     * - The contents of the input data are not altered by the sorting operation
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
+     * - \devicestorage
+     *
+     * \par Performance
+     * Performance is similar to DeviceRadixSort::SortPairs.
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sorting of a device vector of \p int keys
+     * with associated vector of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_keys_out;        // e.g., [        ...        ]
+     * int  *d_values_in;       // e.g., [0, 1, 2, 3, 4, 5, 6]
+     * int  *d_values_out;      // e.g., [        ...        ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes,
+     *     d_keys_in, d_keys_out, d_values_in, d_values_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes,
+     *     d_keys_in, d_keys_out, d_values_in, d_values_out, num_items);
+     *
+     * // d_keys_out            <-- [9, 8, 7, 6, 5, 3, 0]
+     * // d_values_out          <-- [6, 0, 2, 1, 3, 4, 5]
+     *
+     * \endcode
+     *
+     * \tparam KeyT      <b>[inferred]</b> KeyT type
+     * \tparam ValueT    <b>[inferred]</b> ValueT type
+     */
+    template <
+        typename            KeyT,
+        typename            ValueT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortPairsDescending(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        const KeyT          *d_keys_in,                             ///< [in] Pointer to the input data of key data to sort
+        KeyT                *d_keys_out,                            ///< [out] Pointer to the sorted output sequence of key data
+        const ValueT        *d_values_in,                           ///< [in] Pointer to the corresponding input sequence of associated value items
+        ValueT              *d_values_out,                          ///< [out] Pointer to the correspondingly-reordered output sequence of associated value items
+        int                 num_items,                              ///< [in] Number of items to sort
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        DoubleBuffer<KeyT>       d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
+        DoubleBuffer<ValueT>     d_values(const_cast<ValueT*>(d_values_in), d_values_out);
+
+        return DispatchRadixSort<true, KeyT, ValueT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            begin_bit,
+            end_bit,
+            false,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts key-value pairs into descending order. (~<em>N </em>auxiliary storage required).
+     *
+     * \par
+     * - The sorting operation is given a pair of key buffers and a corresponding
+     *   pair of associated value buffers.  Each pair is managed by a DoubleBuffer
+     *   structure that indicates which of the two buffers is "current" (and thus
+     *   contains the input data to be sorted).
+     * - The contents of both buffers within each pair may be altered by the sorting
+     *   operation.
+     * - Upon completion, the sorting operation will update the "current" indicator
+     *   within each DoubleBuffer wrapper to reference which of the two buffers
+     *   now contains the sorted output sequence (a function of the number of key bits
+     *   specified and the targeted device architecture).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageP
+     * - \devicestorage
+     *
+     * \par Performance
+     * Performance is similar to DeviceRadixSort::SortPairs.
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sorting of a device vector of \p int keys
+     * with associated vector of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_key_alt_buf;     // e.g., [        ...        ]
+     * int  *d_value_buf;       // e.g., [0, 1, 2, 3, 4, 5, 6]
+     * int  *d_value_alt_buf;   // e.g., [        ...        ]
+     * ...
+     *
+     * // Create a set of DoubleBuffers to wrap pairs of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     * cub::DoubleBuffer<int> d_values(d_value_buf, d_value_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items);
+     *
+     * // d_keys.Current()      <-- [9, 8, 7, 6, 5, 3, 0]
+     * // d_values.Current()    <-- [6, 0, 2, 1, 3, 4, 5]
+     *
+     * \endcode
+     *
+     * \tparam KeyT      <b>[inferred]</b> KeyT type
+     * \tparam ValueT    <b>[inferred]</b> ValueT type
+     */
+    template <
+        typename            KeyT,
+        typename            ValueT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortPairsDescending(
+        void                    *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>      &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        DoubleBuffer<ValueT>    &d_values,                              ///< [in,out] Double-buffer of values whose "current" device-accessible buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
+        int                     num_items,                              ///< [in] Number of items to sort
+        int                     begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                     end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchRadixSort<true, KeyT, ValueT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            begin_bit,
+            end_bit,
+            true,
+            stream,
+            debug_synchronous);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Keys-only
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Sorts keys into ascending order. (~<em>2N </em>auxiliary storage required)
+     *
+     * \par
+     * - The contents of the input data are not altered by the sorting operation
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated sorting performance across different
+     * CUDA architectures for uniform-random \p uint32 and \p uint64 keys, respectively.
+     *
+     * \image html lsb_radix_sort_int32_keys.png
+     * \image html lsb_radix_sort_int64_keys.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sorting of a device vector of \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_keys_out;        // e.g., [        ...        ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, num_items);
+     *
+     * // d_keys_out            <-- [0, 3, 5, 6, 7, 8, 9]
+     *
+     * \endcode
+     *
+     * \tparam KeyT      <b>[inferred]</b> KeyT type
+     */
+    template <typename KeyT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortKeys(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        const KeyT          *d_keys_in,                             ///< [in] Pointer to the input data of key data to sort
+        KeyT                *d_keys_out,                            ///< [out] Pointer to the sorted output sequence of key data
+        int                 num_items,                              ///< [in] Number of items to sort
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // Null value type
+        DoubleBuffer<KeyT>      d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
+        DoubleBuffer<NullType>  d_values;
+
+        return DispatchRadixSort<false, KeyT, NullType, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            begin_bit,
+            end_bit,
+            false,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts keys into ascending order. (~<em>N </em>auxiliary storage required).
+     *
+     * \par
+     * - The sorting operation is given a pair of key buffers managed by a
+     *   DoubleBuffer structure that indicates which of the two buffers is
+     *   "current" (and thus contains the input data to be sorted).
+     * - The contents of both buffers may be altered by the sorting operation.
+     * - Upon completion, the sorting operation will update the "current" indicator
+     *   within the DoubleBuffer wrapper to reference which of the two buffers
+     *   now contains the sorted output sequence (a function of the number of key bits
+     *   specified and the targeted device architecture).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageP
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated sorting performance across different
+     * CUDA architectures for uniform-random \p uint32 and \p uint64 keys, respectively.
+     *
+     * \image html lsb_radix_sort_int32_keys.png
+     * \image html lsb_radix_sort_int64_keys.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sorting of a device vector of \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_key_alt_buf;     // e.g., [        ...        ]
+     * ...
+     *
+     * // Create a DoubleBuffer to wrap the pair of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items);
+     *
+     * // d_keys.Current()      <-- [0, 3, 5, 6, 7, 8, 9]
+     *
+     * \endcode
+     *
+     * \tparam KeyT      <b>[inferred]</b> KeyT type
+     */
+    template <typename KeyT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortKeys(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>  &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        int                 num_items,                              ///< [in] Number of items to sort
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // Null value type
+        DoubleBuffer<NullType> d_values;
+
+        return DispatchRadixSort<false, KeyT, NullType, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            begin_bit,
+            end_bit,
+            true,
+            stream,
+            debug_synchronous);
+    }
+
+    /**
+     * \brief Sorts keys into descending order. (~<em>2N</em> auxiliary storage required).
+     *
+     * \par
+     * - The contents of the input data are not altered by the sorting operation
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
+     * - \devicestorage
+     *
+     * \par Performance
+     * Performance is similar to DeviceRadixSort::SortKeys.
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sorting of a device vector of \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_keys_out;        // e.g., [        ...        ]
+     * ...
+     *
+     * // Create a DoubleBuffer to wrap the pair of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, num_items);
+     *
+     * // d_keys_out            <-- [9, 8, 7, 6, 5, 3, 0]s
+     *
+     * \endcode
+     *
+     * \tparam KeyT      <b>[inferred]</b> KeyT type
+     */
+    template <typename KeyT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortKeysDescending(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        const KeyT          *d_keys_in,                             ///< [in] Pointer to the input data of key data to sort
+        KeyT                *d_keys_out,                            ///< [out] Pointer to the sorted output sequence of key data
+        int                 num_items,                              ///< [in] Number of items to sort
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        DoubleBuffer<KeyT>      d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
+        DoubleBuffer<NullType>  d_values;
+
+        return DispatchRadixSort<true, KeyT, NullType, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            begin_bit,
+            end_bit,
+            false,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts keys into descending order. (~<em>N </em>auxiliary storage required).
+     *
+     * \par
+     * - The sorting operation is given a pair of key buffers managed by a
+     *   DoubleBuffer structure that indicates which of the two buffers is
+     *   "current" (and thus contains the input data to be sorted).
+     * - The contents of both buffers may be altered by the sorting operation.
+     * - Upon completion, the sorting operation will update the "current" indicator
+     *   within the DoubleBuffer wrapper to reference which of the two buffers
+     *   now contains the sorted output sequence (a function of the number of key bits
+     *   specified and the targeted device architecture).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageP
+     * - \devicestorage
+     *
+     * \par Performance
+     * Performance is similar to DeviceRadixSort::SortKeys.
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sorting of a device vector of \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_key_alt_buf;     // e.g., [        ...        ]
+     * ...
+     *
+     * // Create a DoubleBuffer to wrap the pair of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys, num_items);
+     *
+     * // d_keys.Current()      <-- [9, 8, 7, 6, 5, 3, 0]
+     *
+     * \endcode
+     *
+     * \tparam KeyT      <b>[inferred]</b> KeyT type
+     */
+    template <typename KeyT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortKeysDescending(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>  &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        int                 num_items,                              ///< [in] Number of items to sort
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // Null value type
+        DoubleBuffer<NullType> d_values;
+
+        return DispatchRadixSort<true, KeyT, NullType, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            begin_bit,
+            end_bit,
+            true,
+            stream,
+            debug_synchronous);
+    }
+
+
+    //@}  end member group
+
+
+};
+
+/**
+ * \example example_device_radix_sort.cu
+ */
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_reduce.cuh b/third_party/cub/cub/device/device_reduce.cuh
new file mode 100644
index 0000000..13c7a72
--- /dev/null
+++ b/third_party/cub/cub/device/device_reduce.cuh
@@ -0,0 +1,734 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceReduce provides device-wide, parallel operations for computing a reduction across a sequence of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+#include <limits>
+
+#include "../iterator/arg_index_input_iterator.cuh"
+#include "dispatch/dispatch_reduce.cuh"
+#include "dispatch/dispatch_reduce_by_key.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceReduce provides device-wide, parallel operations for computing a reduction across a sequence of data items residing within device-accessible memory. ![](reduce_logo.png)
+ * \ingroup SingleModule
+ *
+ * \par Overview
+ * A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
+ * uses a binary combining operator to compute a single aggregate from a sequence of input elements.
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceReduce}
+ *
+ * \par Performance
+ * \linear_performance{reduction, reduce-by-key, and run-length encode}
+ *
+ * \par
+ * The following chart illustrates DeviceReduce::Sum
+ * performance across different CUDA architectures for \p int32 keys.
+ *
+ * \image html reduce_int32.png
+ *
+ * \par
+ * The following chart illustrates DeviceReduce::ReduceByKey (summation)
+ * performance across different CUDA architectures for \p fp32
+ * values.  Segments are identified by \p int32 keys, and have lengths uniformly sampled from [1,1000].
+ *
+ * \image html reduce_by_key_fp32_len_500.png
+ *
+ * \par
+ * \plots_below
+ *
+ */
+struct DeviceReduce
+{
+    /**
+     * \brief Computes a device-wide reduction using the specified binary \p reduction_op functor and initial value \p init.
+     *
+     * \par
+     * - Does not support binary reduction operators that are non-commutative.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates a user-defined min-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // CustomMin functor
+     * struct CustomMin
+     * {
+     *     template <typename T>
+     *     __device__ __forceinline__
+     *     T operator()(const T &a, const T &b) const {
+     *         return (b < a) ? b : a;
+     *     }
+     * };
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int          num_items;  // e.g., 7
+     * int          *d_in;      // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int          *d_out;     // e.g., [-]
+     * CustomMin    min_op;
+     * int          init;       // e.g., INT_MAX
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, min_op, init);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run reduction
+     * cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, min_op, init);
+     *
+     * // d_out <-- [0]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT      <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
+     * \tparam ReductionOpT         <b>[inferred]</b> Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
+     * \tparam T                    <b>[inferred]</b> Data element type that is convertible to the \p value type of \p InputIteratorT
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT,
+        typename                    ReductionOpT,
+        typename                    T>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t Reduce(
+        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
+        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
+        ReductionOpT                reduction_op,                       ///< [in] Binary reduction functor
+        T                           init,                               ///< [in] Initial value of the reduction
+        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchReduce<InputIteratorT, OutputIteratorT, OffsetT, ReductionOpT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            num_items,
+            reduction_op,
+            init,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide sum using the addition (\p +) operator.
+     *
+     * \par
+     * - Uses \p 0 as the initial value of the reduction.
+     * - Does not support \p + operators that are non-commutative..
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated sum-reduction performance across different
+     * CUDA architectures for \p int32 and \p int64 items, respectively.
+     *
+     * \image html reduce_int32.png
+     * \image html reduce_int64.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sum-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int  num_items;      // e.g., 7
+     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_out;         // e.g., [-]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sum-reduction
+     * cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
+     *
+     * // d_out <-- [38]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t Sum(
+        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
+        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The output value type
+        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+            typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+        return DispatchReduce<InputIteratorT, OutputIteratorT, OffsetT, cub::Sum>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            num_items,
+            cub::Sum(),
+            OutputT(),            // zero-initialize
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide minimum using the less-than ('<') operator.
+     *
+     * \par
+     * - Uses <tt>std::numeric_limits<T>::max()</tt> as the initial value of the reduction.
+     * - Does not support \p < operators that are non-commutative.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the min-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int  num_items;      // e.g., 7
+     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_out;         // e.g., [-]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run min-reduction
+     * cub::DeviceReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
+     *
+     * // d_out <-- [0]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t Min(
+        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
+        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The input value type
+        typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+        return DispatchReduce<InputIteratorT, OutputIteratorT, OffsetT, cub::Min>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            num_items,
+            cub::Min(),
+            Traits<InputT>::Max(), // replace with std::numeric_limits<T>::max() when C++11 support is more prevalent
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Finds the first device-wide minimum using the less-than ('<') operator, also returning the index of that item.
+     *
+     * \par
+     * - The output value type of \p d_out is cub::KeyValuePair <tt><int, T></tt> (assuming the value type of \p d_in is \p T)
+     *   - The minimum is written to <tt>d_out.value</tt> and its offset in the input array is written to <tt>d_out.key</tt>.
+     *   - The <tt>{1, std::numeric_limits<T>::max()}</tt> tuple is produced for zero-length inputs
+     * - Does not support \p < operators that are non-commutative.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the argmin-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int                      num_items;      // e.g., 7
+     * int                      *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * KeyValuePair<int, int>   *d_out;         // e.g., [{-,-}]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_argmin, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run argmin-reduction
+     * cub::DeviceReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_argmin, num_items);
+     *
+     * // d_out <-- [{5, 0}]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items (of some type \p T) \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate (having value type <tt>cub::KeyValuePair<int, T></tt>) \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t ArgMin(
+        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
+        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The input type
+        typedef typename std::iterator_traits<InputIteratorT>::value_type InputValueT;
+
+        // The output tuple type
+        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+            KeyValuePair<OffsetT, InputValueT>,                                                                 // ... then the key value pair OffsetT + InputValueT
+            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputTupleT;                     // ... else the output iterator's value type
+
+        // The output value type
+        typedef typename OutputTupleT::Value OutputValueT;
+
+        // Wrapped input iterator to produce index-value <OffsetT, InputT> tuples
+        typedef ArgIndexInputIterator<InputIteratorT, OffsetT, OutputValueT> ArgIndexInputIteratorT;
+        ArgIndexInputIteratorT d_indexed_in(d_in);
+
+        // Initial value
+        OutputTupleT initial_value(1, Traits<InputValueT>::Max());   // replace with std::numeric_limits<T>::max() when C++11 support is more prevalent
+
+        return DispatchReduce<ArgIndexInputIteratorT, OutputIteratorT, OffsetT, cub::ArgMin>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_indexed_in,
+            d_out,
+            num_items,
+            cub::ArgMin(),
+            initial_value,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide maximum using the greater-than ('>') operator.
+     *
+     * \par
+     * - Uses <tt>std::numeric_limits<T>::lowest()</tt> as the initial value of the reduction.
+     * - Does not support \p > operators that are non-commutative.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the max-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int  num_items;      // e.g., 7
+     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_out;         // e.g., [-]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_max, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run max-reduction
+     * cub::DeviceReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_max, num_items);
+     *
+     * // d_out <-- [9]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t Max(
+        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
+        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The input value type
+        typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+        return DispatchReduce<InputIteratorT, OutputIteratorT, OffsetT, cub::Max>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            num_items,
+            cub::Max(),
+            Traits<InputT>::Lowest(),    // replace with std::numeric_limits<T>::lowest() when C++11 support is more prevalent
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Finds the first device-wide maximum using the greater-than ('>') operator, also returning the index of that item
+     *
+     * \par
+     * - The output value type of \p d_out is cub::KeyValuePair <tt><int, T></tt> (assuming the value type of \p d_in is \p T)
+     *   - The maximum is written to <tt>d_out.value</tt> and its offset in the input array is written to <tt>d_out.key</tt>.
+     *   - The <tt>{1, std::numeric_limits<T>::lowest()}</tt> tuple is produced for zero-length inputs
+     * - Does not support \p > operators that are non-commutative.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the argmax-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_reduce.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int                      num_items;      // e.g., 7
+     * int                      *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * KeyValuePair<int, int>   *d_out;         // e.g., [{-,-}]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_argmax, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run argmax-reduction
+     * cub::DeviceReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_argmax, num_items);
+     *
+     * // d_out <-- [{6, 9}]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items (of some type \p T) \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate (having value type <tt>cub::KeyValuePair<int, T></tt>) \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t ArgMax(
+        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
+        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The input type
+        typedef typename std::iterator_traits<InputIteratorT>::value_type InputValueT;
+
+        // The output tuple type
+        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+            KeyValuePair<OffsetT, InputValueT>,                                                                 // ... then the key value pair OffsetT + InputValueT
+            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputTupleT;                     // ... else the output iterator's value type
+
+        // The output value type
+        typedef typename OutputTupleT::Value OutputValueT;
+
+        // Wrapped input iterator to produce index-value <OffsetT, InputT> tuples
+        typedef ArgIndexInputIterator<InputIteratorT, OffsetT, OutputValueT> ArgIndexInputIteratorT;
+        ArgIndexInputIteratorT d_indexed_in(d_in);
+
+        // Initial value
+        OutputTupleT initial_value(1, Traits<InputValueT>::Lowest());     // replace with std::numeric_limits<T>::lowest() when C++11 support is more prevalent
+
+        return DispatchReduce<ArgIndexInputIteratorT, OutputIteratorT, OffsetT, cub::ArgMax>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_indexed_in,
+            d_out,
+            num_items,
+            cub::ArgMax(),
+            initial_value,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Reduces segments of values, where segments are demarcated by corresponding runs of identical keys.
+     *
+     * \par
+     * This operation computes segmented reductions within \p d_values_in using
+     * the specified binary \p reduction_op functor.  The segments are identified by
+     * "runs" of corresponding keys in \p d_keys_in, where runs are maximal ranges of
+     * consecutive, identical keys.  For the <em>i</em><sup>th</sup> run encountered,
+     * the first key of the run and the corresponding value aggregate of that run are
+     * written to <tt>d_unique_out[<em>i</em>]</tt> and <tt>d_aggregates_out[<em>i</em>]</tt>,
+     * respectively. The total number of runs encountered is written to \p d_num_runs_out.
+     *
+     * \par
+     * - The <tt>==</tt> equality operator is used to determine whether keys are equivalent
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following chart illustrates reduction-by-key (sum) performance across
+     * different CUDA architectures for \p fp32 and \p fp64 values, respectively.  Segments
+     * are identified by \p int32 keys, and have lengths uniformly sampled from [1,1000].
+     *
+     * \image html reduce_by_key_fp32_len_500.png
+     * \image html reduce_by_key_fp64_len_500.png
+     *
+     * \par
+     * The following charts are similar, but with segment lengths uniformly sampled from [1,10]:
+     *
+     * \image html reduce_by_key_fp32_len_5.png
+     * \image html reduce_by_key_fp64_len_5.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the segmented reduction of \p int values grouped
+     * by runs of associated \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_reduce.cuh>
+     *
+     * // CustomMin functor
+     * struct CustomMin
+     * {
+     *     template <typename T>
+     *     CUB_RUNTIME_FUNCTION __forceinline__
+     *     T operator()(const T &a, const T &b) const {
+     *         return (b < a) ? b : a;
+     *     }
+     * };
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int          num_items;          // e.g., 8
+     * int          *d_keys_in;         // e.g., [0, 2, 2, 9, 5, 5, 5, 8]
+     * int          *d_values_in;       // e.g., [0, 7, 1, 6, 2, 5, 3, 4]
+     * int          *d_unique_out;      // e.g., [-, -, -, -, -, -, -, -]
+     * int          *d_aggregates_out;  // e.g., [-, -, -, -, -, -, -, -]
+     * int          *d_num_runs_out;    // e.g., [-]
+     * CustomMin    reduction_op;
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceReduce::ReduceByKey(d_temp_storage, temp_storage_bytes, d_keys_in, d_unique_out, d_values_in, d_aggregates_out, d_num_runs_out, reduction_op, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run reduce-by-key
+     * cub::DeviceReduce::ReduceByKey(d_temp_storage, temp_storage_bytes, d_keys_in, d_unique_out, d_values_in, d_aggregates_out, d_num_runs_out, reduction_op, num_items);
+     *
+     * // d_unique_out      <-- [0, 2, 9, 5, 8]
+     * // d_aggregates_out  <-- [0, 1, 6, 2, 4]
+     * // d_num_runs_out    <-- [5]
+     *
+     * \endcode
+     *
+     * \tparam KeysInputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input keys \iterator
+     * \tparam UniqueOutputIteratorT    <b>[inferred]</b> Random-access output iterator type for writing unique output keys \iterator
+     * \tparam ValuesInputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input values \iterator
+     * \tparam AggregatesOutputIterator <b>[inferred]</b> Random-access output iterator type for writing output value aggregates \iterator
+     * \tparam NumRunsOutputIteratorT   <b>[inferred]</b> Output iterator type for recording the number of runs encountered \iterator
+     * \tparam ReductionOpT              <b>[inferred]</b> Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
+     */
+    template <
+        typename                    KeysInputIteratorT,
+        typename                    UniqueOutputIteratorT,
+        typename                    ValuesInputIteratorT,
+        typename                    AggregatesOutputIteratorT,
+        typename                    NumRunsOutputIteratorT,
+        typename                    ReductionOpT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t ReduceByKey(
+        void                        *d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        KeysInputIteratorT          d_keys_in,                      ///< [in] Pointer to the input sequence of keys
+        UniqueOutputIteratorT       d_unique_out,                   ///< [out] Pointer to the output sequence of unique keys (one key per run)
+        ValuesInputIteratorT        d_values_in,                    ///< [in] Pointer to the input sequence of corresponding values
+        AggregatesOutputIteratorT   d_aggregates_out,               ///< [out] Pointer to the output sequence of value aggregates (one aggregate per run)
+        NumRunsOutputIteratorT      d_num_runs_out,                 ///< [out] Pointer to total number of runs encountered (i.e., the length of d_unique_out)
+        ReductionOpT                reduction_op,                   ///< [in] Binary reduction functor
+        int                         num_items,                      ///< [in] Total number of associated key+value pairs (i.e., the length of \p d_in_keys and \p d_in_values)
+        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // FlagT iterator type (not used)
+
+        // Selection op (not used)
+
+        // Default == operator
+        typedef Equality EqualityOp;
+
+        return DispatchReduceByKey<KeysInputIteratorT, UniqueOutputIteratorT, ValuesInputIteratorT, AggregatesOutputIteratorT, NumRunsOutputIteratorT, EqualityOp, ReductionOpT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys_in,
+            d_unique_out,
+            d_values_in,
+            d_aggregates_out,
+            d_num_runs_out,
+            EqualityOp(),
+            reduction_op,
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+};
+
+/**
+ * \example example_device_reduce.cu
+ */
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_run_length_encode.cuh b/third_party/cub/cub/device/device_run_length_encode.cuh
new file mode 100644
index 0000000..7a2e82d
--- /dev/null
+++ b/third_party/cub/cub/device/device_run_length_encode.cuh
@@ -0,0 +1,278 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceRunLengthEncode provides device-wide, parallel operations for computing a run-length encoding across a sequence of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch/dispatch_rle.cuh"
+#include "dispatch/dispatch_reduce_by_key.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceRunLengthEncode provides device-wide, parallel operations for demarcating "runs" of same-valued items within a sequence residing within device-accessible memory. ![](run_length_encode_logo.png)
+ * \ingroup SingleModule
+ *
+ * \par Overview
+ * A <a href="http://en.wikipedia.org/wiki/Run-length_encoding"><em>run-length encoding</em></a>
+ * computes a simple compressed representation of a sequence of input elements such that each
+ * maximal "run" of consecutive same-valued data items is encoded as a single data value along with a
+ * count of the elements in that run.
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceRunLengthEncode}
+ *
+ * \par Performance
+ * \linear_performance{run-length encode}
+ *
+ * \par
+ * The following chart illustrates DeviceRunLengthEncode::RunLengthEncode performance across
+ * different CUDA architectures for \p int32 items.
+ * Segments have lengths uniformly sampled from [1,1000].
+ *
+ * \image html rle_int32_len_500.png
+ *
+ * \par
+ * \plots_below
+ *
+ */
+struct DeviceRunLengthEncode
+{
+
+    /**
+     * \brief Computes a run-length encoding of the sequence \p d_in.
+     *
+     * \par
+     * - For the <em>i</em><sup>th</sup> run encountered, the first key of the run and its length are written to
+     *   <tt>d_unique_out[<em>i</em>]</tt> and <tt>d_counts_out[<em>i</em>]</tt>,
+     *   respectively.
+     * - The total number of runs encountered is written to \p d_num_runs_out.
+     * - The <tt>==</tt> equality operator is used to determine whether values are equivalent
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated encode performance across different
+     * CUDA architectures for \p int32 and \p int64 items, respectively.  Segments have
+     * lengths uniformly sampled from [1,1000].
+     *
+     * \image html rle_int32_len_500.png
+     * \image html rle_int64_len_500.png
+     *
+     * \par
+     * The following charts are similar, but with segment lengths uniformly sampled from [1,10]:
+     *
+     * \image html rle_int32_len_5.png
+     * \image html rle_int64_len_5.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the run-length encoding of a sequence of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_run_length_encode.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int          num_items;          // e.g., 8
+     * int          *d_in;              // e.g., [0, 2, 2, 9, 5, 5, 5, 8]
+     * int          *d_unique_out;      // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int          *d_counts_out;      // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int          *d_num_runs_out;    // e.g., [ ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRunLengthEncode::Encode(d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_counts_out, d_num_runs_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run encoding
+     * cub::DeviceRunLengthEncode::Encode(d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_counts_out, d_num_runs_out, num_items);
+     *
+     * // d_unique_out      <-- [0, 2, 9, 5, 8]
+     * // d_counts_out      <-- [1, 2, 1, 3, 1]
+     * // d_num_runs_out    <-- [5]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT           <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam UniqueOutputIteratorT    <b>[inferred]</b> Random-access output iterator type for writing unique output items \iterator
+     * \tparam LengthsOutputIteratorT   <b>[inferred]</b> Random-access output iterator type for writing output counts \iterator
+     * \tparam NumRunsOutputIteratorT   <b>[inferred]</b> Output iterator type for recording the number of runs encountered \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    UniqueOutputIteratorT,
+        typename                    LengthsOutputIteratorT,
+        typename                    NumRunsOutputIteratorT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Encode(
+        void*                       d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of keys
+        UniqueOutputIteratorT       d_unique_out,                   ///< [out] Pointer to the output sequence of unique keys (one key per run)
+        LengthsOutputIteratorT      d_counts_out,                   ///< [out] Pointer to the output sequence of run-lengths (one count per run)
+        NumRunsOutputIteratorT      d_num_runs_out,                     ///< [out] Pointer to total number of runs
+        int                         num_items,                      ///< [in] Total number of associated key+value pairs (i.e., the length of \p d_in_keys and \p d_in_values)
+        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        typedef int         OffsetT;                    // Signed integer type for global offsets
+        typedef NullType*   FlagIterator;               // FlagT iterator type (not used)
+        typedef NullType    SelectOp;                   // Selection op (not used)
+        typedef Equality    EqualityOp;                 // Default == operator
+        typedef cub::Sum    ReductionOp;                // Value reduction operator
+
+        // The lengths output value type
+        typedef typename If<(Equals<typename std::iterator_traits<LengthsOutputIteratorT>::value_type, void>::VALUE),   // LengthT =  (if output iterator's value type is void) ?
+            OffsetT,                                                                                                    // ... then the OffsetT type,
+            typename std::iterator_traits<LengthsOutputIteratorT>::value_type>::Type LengthT;                           // ... else the output iterator's value type
+
+        // Generator type for providing 1s values for run-length reduction
+        typedef ConstantInputIterator<LengthT, OffsetT> LengthsInputIteratorT;
+
+        return DispatchReduceByKey<InputIteratorT, UniqueOutputIteratorT, LengthsInputIteratorT, LengthsOutputIteratorT, NumRunsOutputIteratorT, EqualityOp, ReductionOp, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_unique_out,
+            LengthsInputIteratorT((LengthT) 1),
+            d_counts_out,
+            d_num_runs_out,
+            EqualityOp(),
+            ReductionOp(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Enumerates the starting offsets and lengths of all non-trivial runs (of length > 1) of same-valued keys in the sequence \p d_in.
+     *
+     * \par
+     * - For the <em>i</em><sup>th</sup> non-trivial run, the run's starting offset
+     *   and its length are written to <tt>d_offsets_out[<em>i</em>]</tt> and
+     *   <tt>d_lengths_out[<em>i</em>]</tt>, respectively.
+     * - The total number of runs encountered is written to \p d_num_runs_out.
+     * - The <tt>==</tt> equality operator is used to determine whether values are equivalent
+     * - \devicestorage
+     *
+     * \par Performance
+     *
+     * \par Snippet
+     * The code snippet below illustrates the identification of non-trivial runs within a sequence of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_run_length_encode.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int          num_items;          // e.g., 8
+     * int          *d_in;              // e.g., [0, 2, 2, 9, 5, 5, 5, 8]
+     * int          *d_offsets_out;     // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int          *d_lengths_out;     // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int          *d_num_runs_out;    // e.g., [ ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceRunLengthEncode::NonTrivialRuns(d_temp_storage, temp_storage_bytes, d_in, d_offsets_out, d_lengths_out, d_num_runs_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run encoding
+     * cub::DeviceRunLengthEncode::NonTrivialRuns(d_temp_storage, temp_storage_bytes, d_in, d_offsets_out, d_lengths_out, d_num_runs_out, num_items);
+     *
+     * // d_offsets_out         <-- [1, 4]
+     * // d_lengths_out         <-- [2, 3]
+     * // d_num_runs_out        <-- [2]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT           <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OffsetsOutputIteratorT   <b>[inferred]</b> Random-access output iterator type for writing run-offset values \iterator
+     * \tparam LengthsOutputIteratorT   <b>[inferred]</b> Random-access output iterator type for writing run-length values \iterator
+     * \tparam NumRunsOutputIteratorT   <b>[inferred]</b> Output iterator type for recording the number of runs encountered \iterator
+     */
+    template <
+        typename                InputIteratorT,
+        typename                OffsetsOutputIteratorT,
+        typename                LengthsOutputIteratorT,
+        typename                NumRunsOutputIteratorT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t NonTrivialRuns(
+        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT          d_in,                           ///< [in] Pointer to input sequence of data items
+        OffsetsOutputIteratorT  d_offsets_out,                  ///< [out] Pointer to output sequence of run-offsets (one offset per non-trivial run)
+        LengthsOutputIteratorT  d_lengths_out,                  ///< [out] Pointer to output sequence of run-lengths (one count per non-trivial run)
+        NumRunsOutputIteratorT  d_num_runs_out,                 ///< [out] Pointer to total number of runs (i.e., length of \p d_offsets_out)
+        int                     num_items,                      ///< [in] Total number of associated key+value pairs (i.e., the length of \p d_in_keys and \p d_in_values)
+        cudaStream_t            stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        typedef int         OffsetT;                    // Signed integer type for global offsets
+        typedef Equality    EqualityOp;                 // Default == operator
+
+        return DeviceRleDispatch<InputIteratorT, OffsetsOutputIteratorT, LengthsOutputIteratorT, NumRunsOutputIteratorT, EqualityOp, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_offsets_out,
+            d_lengths_out,
+            d_num_runs_out,
+            EqualityOp(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_scan.cuh b/third_party/cub/cub/device/device_scan.cuh
new file mode 100644
index 0000000..e86fefe
--- /dev/null
+++ b/third_party/cub/cub/device/device_scan.cuh
@@ -0,0 +1,443 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceScan provides device-wide, parallel operations for computing a prefix scan across a sequence of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch/dispatch_scan.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceScan provides device-wide, parallel operations for computing a prefix scan across a sequence of data items residing within device-accessible memory. ![](device_scan.png)
+ * \ingroup SingleModule
+ *
+ * \par Overview
+ * Given a sequence of input elements and a binary reduction operator, a [<em>prefix scan</em>](http://en.wikipedia.org/wiki/Prefix_sum)
+ * produces an output sequence where each element is computed to be the reduction
+ * of the elements occurring earlier in the input sequence.  <em>Prefix sum</em>
+ * connotes a prefix scan with the addition operator. The term \em inclusive indicates
+ * that the <em>i</em><sup>th</sup> output reduction incorporates the <em>i</em><sup>th</sup> input.
+ * The term \em exclusive indicates the <em>i</em><sup>th</sup> input is not incorporated into
+ * the <em>i</em><sup>th</sup> output reduction.
+ *
+ * \par
+ * As of CUB 1.0.1 (2013), CUB's device-wide scan APIs have implemented our <em>"decoupled look-back"</em> algorithm
+ * for performing global prefix scan with only a single pass through the
+ * input data, as described in our 2016 technical report [1].  The central
+ * idea is to leverage a small, constant factor of redundant work in order to overlap the latencies
+ * of global prefix propagation with local computation.  As such, our algorithm requires only
+ * ~2<em>n</em> data movement (<em>n</em> inputs are read, <em>n</em> outputs are written), and typically
+ * proceeds at "memcpy" speeds.
+ *
+ * \par
+ * [1] [Duane Merrill and Michael Garland.  "Single-pass Parallel Prefix Scan with Decoupled Look-back", <em>NVIDIA Technical Report NVR-2016-002</em>, 2016.](https://research.nvidia.com/publication/single-pass-parallel-prefix-scan-decoupled-look-back)
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceScan}
+ *
+ * \par Performance
+ * \linear_performance{prefix scan}
+ *
+ * \par
+ * The following chart illustrates DeviceScan::ExclusiveSum
+ * performance across different CUDA architectures for \p int32 keys.
+ * \plots_below
+ *
+ * \image html scan_int32.png
+ *
+ */
+struct DeviceScan
+{
+    /******************************************************************//**
+     * \name Exclusive scans
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Computes a device-wide exclusive prefix sum.  The value of 0 is applied as the initial value, and is assigned to *d_out.
+     *
+     * \par
+     * - Supports non-commutative sum operators.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated exclusive sum performance across different
+     * CUDA architectures for \p int32 and \p int64 items, respectively.
+     *
+     * \image html scan_int32.png
+     * \image html scan_int64.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the exclusive prefix sum of an \p int device vector.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_scan.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int  num_items;      // e.g., 7
+     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_out;         // e.g., [ ,  ,  ,  ,  ,  ,  ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run exclusive prefix sum
+     * cub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
+     *
+     * // d_out s<-- [0, 8, 14, 21, 26, 29, 29]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading scan inputs \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Random-access output iterator type for writing scan outputs \iterator
+     */
+    template <
+        typename        InputIteratorT,
+        typename        OutputIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t ExclusiveSum(
+        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT d_out,                              ///< [out] Pointer to the output sequence of data items
+        int             num_items,                          ///< [in] Total number of input items (i.e., the length of \p d_in)
+        cudaStream_t    stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool            debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The output value type
+        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+            typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+        // Initial value
+        OutputT init_value = 0;
+
+        return DispatchScan<InputIteratorT, OutputIteratorT, Sum, OutputT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            Sum(),
+            init_value,
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide exclusive prefix scan using the specified binary \p scan_op functor.  The \p init_value value is applied as the initial value, and is assigned to *d_out.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the exclusive prefix min-scan of an \p int device vector
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_scan.cuh>
+     *
+     * // CustomMin functor
+     * struct CustomMin
+     * {
+     *     template <typename T>
+     *     CUB_RUNTIME_FUNCTION __forceinline__
+     *     T operator()(const T &a, const T &b) const {
+     *         return (b < a) ? b : a;
+     *     }
+     * };
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int          num_items;      // e.g., 7
+     * int          *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int          *d_out;         // e.g., [ ,  ,  ,  ,  ,  ,  ]
+     * CustomMin    min_op
+     * ...
+     *
+     * // Determine temporary device storage requirements for exclusive prefix scan
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, min_op, (int) MAX_INT, num_items);
+     *
+     * // Allocate temporary storage for exclusive prefix scan
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run exclusive prefix min-scan
+     * cub::DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, min_op, (int) MAX_INT, num_items);
+     *
+     * // d_out <-- [2147483647, 8, 6, 6, 5, 3, 0]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT   <b>[inferred]</b> Random-access input iterator type for reading scan inputs \iterator
+     * \tparam OutputIteratorT  <b>[inferred]</b> Random-access output iterator type for writing scan outputs \iterator
+     * \tparam ScanOp           <b>[inferred]</b> Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+     * \tparam Identity         <b>[inferred]</b> Type of the \p identity value used Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        typename        InputIteratorT,
+        typename        OutputIteratorT,
+        typename        ScanOpT,
+        typename        InitValueT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t ExclusiveScan(
+        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT d_out,                              ///< [out] Pointer to the output sequence of data items
+        ScanOpT         scan_op,                            ///< [in] Binary scan functor
+        InitValueT      init_value,                         ///< [in] Initial value to seed the exclusive scan (and is assigned to *d_out)
+        int             num_items,                          ///< [in] Total number of input items (i.e., the length of \p d_in)
+        cudaStream_t    stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool            debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchScan<InputIteratorT, OutputIteratorT, ScanOpT, InitValueT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            scan_op,
+            init_value,
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Inclusive scans
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes a device-wide inclusive prefix sum.
+     *
+     * \par
+     * - Supports non-commutative sum operators.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the inclusive prefix sum of an \p int device vector.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_scan.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int  num_items;      // e.g., 7
+     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_out;         // e.g., [ ,  ,  ,  ,  ,  ,  ]
+     * ...
+     *
+     * // Determine temporary device storage requirements for inclusive prefix sum
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
+     *
+     * // Allocate temporary storage for inclusive prefix sum
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run inclusive prefix sum
+     * cub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
+     *
+     * // d_out <-- [8, 14, 21, 26, 29, 29, 38]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading scan inputs \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Random-access output iterator type for writing scan outputs \iterator
+     */
+    template <
+        typename            InputIteratorT,
+        typename            OutputIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t InclusiveSum(
+        void*               d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT      d_in,                           ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT     d_out,                          ///< [out] Pointer to the output sequence of data items
+        int                 num_items,                      ///< [in] Total number of input items (i.e., the length of \p d_in)
+        cudaStream_t        stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchScan<InputIteratorT, OutputIteratorT, Sum, NullType, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            Sum(),
+            NullType(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide inclusive prefix scan using the specified binary \p scan_op functor.
+     *
+     * \par
+     * - Supports non-commutative scan operators.
+     * - Provides "run-to-run" determinism for pseudo-associative reduction
+     *   (e.g., addition of floating point types) on the same GPU device.
+     *   However, results for pseudo-associative reduction may be inconsistent
+     *   from one device to a another device of a different compute-capability
+     *   because CUB can employ different tile-sizing for different architectures.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the inclusive prefix min-scan of an \p int device vector.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_scan.cuh>
+     *
+     * // CustomMin functor
+     * struct CustomMin
+     * {
+     *     template <typename T>
+     *     CUB_RUNTIME_FUNCTION __forceinline__
+     *     T operator()(const T &a, const T &b) const {
+     *         return (b < a) ? b : a;
+     *     }
+     * };
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int          num_items;      // e.g., 7
+     * int          *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int          *d_out;         // e.g., [ ,  ,  ,  ,  ,  ,  ]
+     * CustomMin    min_op;
+     * ...
+     *
+     * // Determine temporary device storage requirements for inclusive prefix scan
+     * void *d_temp_storage = NULL;
+     * size_t temp_storage_bytes = 0;
+     * cub::DeviceScan::InclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, min_op, num_items);
+     *
+     * // Allocate temporary storage for inclusive prefix scan
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run inclusive prefix min-scan
+     * cub::DeviceScan::InclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, min_op, num_items);
+     *
+     * // d_out <-- [8, 6, 6, 5, 3, 0, 0]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT   <b>[inferred]</b> Random-access input iterator type for reading scan inputs \iterator
+     * \tparam OutputIteratorT  <b>[inferred]</b> Random-access output iterator type for writing scan outputs \iterator
+     * \tparam ScanOp           <b>[inferred]</b> Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        typename        InputIteratorT,
+        typename        OutputIteratorT,
+        typename        ScanOpT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t InclusiveScan(
+        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT d_out,                              ///< [out] Pointer to the output sequence of data items
+        ScanOpT         scan_op,                            ///< [in] Binary scan functor
+        int             num_items,                          ///< [in] Total number of input items (i.e., the length of \p d_in)
+        cudaStream_t    stream             = 0,             ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool            debug_synchronous  = false)         ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchScan<InputIteratorT, OutputIteratorT, ScanOpT, NullType, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            scan_op,
+            NullType(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+    //@}  end member group
+
+};
+
+/**
+ * \example example_device_scan.cu
+ */
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_segmented_radix_sort.cuh b/third_party/cub/cub/device/device_segmented_radix_sort.cuh
new file mode 100644
index 0000000..0d36076
--- /dev/null
+++ b/third_party/cub/cub/device/device_segmented_radix_sort.cuh
@@ -0,0 +1,876 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceSegmentedRadixSort provides device-wide, parallel operations for computing a batched radix sort across multiple, non-overlapping sequences of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch/dispatch_radix_sort.cuh"
+#include "../util_arch.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceSegmentedRadixSort provides device-wide, parallel operations for computing a batched radix sort across multiple, non-overlapping sequences of data items residing within device-accessible memory. ![](segmented_sorting_logo.png)
+ * \ingroup SegmentedModule
+ *
+ * \par Overview
+ * The [<em>radix sorting method</em>](http://en.wikipedia.org/wiki/Radix_sort) arranges
+ * items into ascending (or descending) order.  The algorithm relies upon a positional representation for
+ * keys, i.e., each key is comprised of an ordered sequence of symbols (e.g., digits,
+ * characters, etc.) specified from least-significant to most-significant.  For a
+ * given input sequence of keys and a set of rules specifying a total ordering
+ * of the symbolic alphabet, the radix sorting method produces a lexicographic
+ * ordering of those keys.
+ *
+ * \par
+ * DeviceSegmentedRadixSort can sort all of the built-in C++ numeric primitive types
+ * (<tt>unsigned char</tt>, \p int, \p double, etc.) as well as CUDA's \p __half
+ * half-precision floating-point type.  Although the direct radix sorting
+ * method can only be applied to unsigned integral types, DeviceSegmentedRadixSort
+ * is able to sort signed and floating-point types via simple bit-wise transformations
+ * that ensure lexicographic key ordering.
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceSegmentedRadixSort}
+ *
+ */
+struct DeviceSegmentedRadixSort
+{
+
+    /******************************************************************//**
+     * \name Key-value pairs
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Sorts segments of key-value pairs into ascending order. (~<em>2N </em>auxiliary storage required)
+     *
+     * \par
+     * - The contents of the input data are not altered by the sorting operation
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys
+     * with associated vector of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  num_segments;       // e.g., 3
+     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
+     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_keys_out;        // e.g., [-, -, -, -, -, -, -]
+     * int  *d_values_in;       // e.g., [0, 1, 2, 3, 4, 5, 6]
+     * int  *d_values_out;      // e.g., [-, -, -, -, -, -, -]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes,
+     *     d_keys_in, d_keys_out, d_values_in, d_values_out,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes,
+     *     d_keys_in, d_keys_out, d_values_in, d_values_out,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_keys_out            <-- [6, 7, 8, 0, 3, 5, 9]
+     * // d_values_out          <-- [1, 2, 0, 5, 4, 3, 6]
+     *
+     * \endcode
+     *
+     * \tparam KeyT             <b>[inferred]</b> Key type
+     * \tparam ValueT           <b>[inferred]</b> Value type
+     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            KeyT,
+        typename            ValueT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortPairs(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        const KeyT          *d_keys_in,                             ///< [in] %Device-accessible pointer to the input data of key data to sort
+        KeyT                *d_keys_out,                            ///< [out] %Device-accessible pointer to the sorted output sequence of key data
+        const ValueT        *d_values_in,                           ///< [in] %Device-accessible pointer to the corresponding input sequence of associated value items
+        ValueT              *d_values_out,                          ///< [out] %Device-accessible pointer to the correspondingly-reordered output sequence of associated value items
+        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
+        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        DoubleBuffer<KeyT>       d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
+        DoubleBuffer<ValueT>     d_values(const_cast<ValueT*>(d_values_in), d_values_out);
+
+        return DispatchSegmentedRadixSort<false, KeyT, ValueT, OffsetIteratorT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            begin_bit,
+            end_bit,
+            false,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts segments of key-value pairs into ascending order. (~<em>N </em>auxiliary storage required)
+     *
+     * \par
+     * - The sorting operation is given a pair of key buffers and a corresponding
+     *   pair of associated value buffers.  Each pair is managed by a DoubleBuffer
+     *   structure that indicates which of the two buffers is "current" (and thus
+     *   contains the input data to be sorted).
+     * - The contents of both buffers within each pair may be altered by the sorting
+     *   operation.
+     * - Upon completion, the sorting operation will update the "current" indicator
+     *   within each DoubleBuffer wrapper to reference which of the two buffers
+     *   now contains the sorted output sequence (a function of the number of key bits
+     *   specified and the targeted device architecture).
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageP
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys
+     * with associated vector of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  num_segments;       // e.g., 3
+     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
+     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_key_alt_buf;     // e.g., [-, -, -, -, -, -, -]
+     * int  *d_value_buf;       // e.g., [0, 1, 2, 3, 4, 5, 6]
+     * int  *d_value_alt_buf;   // e.g., [-, -, -, -, -, -, -]
+     * ...
+     *
+     * // Create a set of DoubleBuffers to wrap pairs of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     * cub::DoubleBuffer<int> d_values(d_value_buf, d_value_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_keys.Current()      <-- [6, 7, 8, 0, 3, 5, 9]
+     * // d_values.Current()    <-- [5, 4, 3, 1, 2, 0, 6]
+     *
+     * \endcode
+     *
+     * \tparam KeyT             <b>[inferred]</b> Key type
+     * \tparam ValueT           <b>[inferred]</b> Value type
+     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename                KeyT,
+        typename                ValueT,
+        typename                OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortPairs(
+        void                    *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>      &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        DoubleBuffer<ValueT>    &d_values,                              ///< [in,out] Double-buffer of values whose "current" device-accessible buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
+        int                     num_items,                              ///< [in] The total number of items to sort (across all segments)
+        int                     num_segments,                           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT         d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT         d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                     begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                     end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchSegmentedRadixSort<false, KeyT, ValueT, OffsetIteratorT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            begin_bit,
+            end_bit,
+            true,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts segments of key-value pairs into descending order. (~<em>2N</em> auxiliary storage required).
+     *
+     * \par
+     * - The contents of the input data are not altered by the sorting operation
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys
+     * with associated vector of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  num_segments;       // e.g., 3
+     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
+     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_keys_out;        // e.g., [-, -, -, -, -, -, -]
+     * int  *d_values_in;       // e.g., [0, 1, 2, 3, 4, 5, 6]
+     * int  *d_values_out;      // e.g., [-, -, -, -, -, -, -]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes,
+     *     d_keys_in, d_keys_out, d_values_in, d_values_out,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceSegmentedRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes,
+     *     d_keys_in, d_keys_out, d_values_in, d_values_out,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_keys_out            <-- [8, 7, 6, 9, 5, 3, 0]
+     * // d_values_out          <-- [0, 2, 1, 6, 3, 4, 5]
+     *
+     * \endcode
+     *
+     * \tparam KeyT             <b>[inferred]</b> Key type
+     * \tparam ValueT           <b>[inferred]</b> Value type
+     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            KeyT,
+        typename            ValueT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortPairsDescending(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        const KeyT          *d_keys_in,                             ///< [in] %Device-accessible pointer to the input data of key data to sort
+        KeyT                *d_keys_out,                            ///< [out] %Device-accessible pointer to the sorted output sequence of key data
+        const ValueT        *d_values_in,                           ///< [in] %Device-accessible pointer to the corresponding input sequence of associated value items
+        ValueT              *d_values_out,                          ///< [out] %Device-accessible pointer to the correspondingly-reordered output sequence of associated value items
+        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
+        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        DoubleBuffer<KeyT>       d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
+        DoubleBuffer<ValueT>     d_values(const_cast<ValueT*>(d_values_in), d_values_out);
+
+        return DispatchSegmentedRadixSort<true, KeyT, ValueT, OffsetIteratorT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            begin_bit,
+            end_bit,
+            false,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts segments of key-value pairs into descending order. (~<em>N </em>auxiliary storage required).
+     *
+     * \par
+     * - The sorting operation is given a pair of key buffers and a corresponding
+     *   pair of associated value buffers.  Each pair is managed by a DoubleBuffer
+     *   structure that indicates which of the two buffers is "current" (and thus
+     *   contains the input data to be sorted).
+     * - The contents of both buffers within each pair may be altered by the sorting
+     *   operation.
+     * - Upon completion, the sorting operation will update the "current" indicator
+     *   within each DoubleBuffer wrapper to reference which of the two buffers
+     *   now contains the sorted output sequence (a function of the number of key bits
+     *   specified and the targeted device architecture).
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageP
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys
+     * with associated vector of \p int values.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  num_segments;       // e.g., 3
+     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
+     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_key_alt_buf;     // e.g., [-, -, -, -, -, -, -]
+     * int  *d_value_buf;       // e.g., [0, 1, 2, 3, 4, 5, 6]
+     * int  *d_value_alt_buf;   // e.g., [-, -, -, -, -, -, -]
+     * ...
+     *
+     * // Create a set of DoubleBuffers to wrap pairs of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     * cub::DoubleBuffer<int> d_values(d_value_buf, d_value_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, d_keys, d_values,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceSegmentedRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, d_keys, d_values,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_keys.Current()      <-- [8, 7, 6, 9, 5, 3, 0]
+     * // d_values.Current()    <-- [0, 2, 1, 6, 3, 4, 5]
+     *
+     * \endcode
+     *
+     * \tparam KeyT             <b>[inferred]</b> Key type
+     * \tparam ValueT           <b>[inferred]</b> Value type
+     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename                KeyT,
+        typename                ValueT,
+        typename                OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortPairsDescending(
+        void                    *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>      &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        DoubleBuffer<ValueT>    &d_values,                              ///< [in,out] Double-buffer of values whose "current" device-accessible buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
+        int                     num_items,                              ///< [in] The total number of items to sort (across all segments)
+        int                     num_segments,                           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT         d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT         d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                     begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                     end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchSegmentedRadixSort<true, KeyT, ValueT, OffsetIteratorT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            begin_bit,
+            end_bit,
+            true,
+            stream,
+            debug_synchronous);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Keys-only
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Sorts segments of keys into ascending order. (~<em>2N </em>auxiliary storage required)
+     *
+     * \par
+     * - The contents of the input data are not altered by the sorting operation
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  num_segments;       // e.g., 3
+     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
+     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_keys_out;        // e.g., [-, -, -, -, -, -, -]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceSegmentedRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_keys_out            <-- [6, 7, 8, 0, 3, 5, 9]
+     *
+     * \endcode
+     *
+     * \tparam KeyT             <b>[inferred]</b> Key type
+     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            KeyT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortKeys(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        const KeyT          *d_keys_in,                             ///< [in] %Device-accessible pointer to the input data of key data to sort
+        KeyT                *d_keys_out,                            ///< [out] %Device-accessible pointer to the sorted output sequence of key data
+        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
+        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // Null value type
+        DoubleBuffer<KeyT>      d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
+        DoubleBuffer<NullType>  d_values;
+
+        return DispatchSegmentedRadixSort<false, KeyT, NullType, OffsetIteratorT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            begin_bit,
+            end_bit,
+            false,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts segments of keys into ascending order. (~<em>N </em>auxiliary storage required).
+     *
+     * \par
+     * - The sorting operation is given a pair of key buffers managed by a
+     *   DoubleBuffer structure that indicates which of the two buffers is
+     *   "current" (and thus contains the input data to be sorted).
+     * - The contents of both buffers may be altered by the sorting operation.
+     * - Upon completion, the sorting operation will update the "current" indicator
+     *   within the DoubleBuffer wrapper to reference which of the two buffers
+     *   now contains the sorted output sequence (a function of the number of key bits
+     *   specified and the targeted device architecture).
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageP
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  num_segments;       // e.g., 3
+     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
+     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_key_alt_buf;     // e.g., [-, -, -, -, -, -, -]
+     * ...
+     *
+     * // Create a DoubleBuffer to wrap the pair of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceSegmentedRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_keys.Current()      <-- [6, 7, 8, 0, 3, 5, 9]
+     *
+     * \endcode
+     *
+     * \tparam KeyT             <b>[inferred]</b> Key type
+     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            KeyT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortKeys(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>  &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
+        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // Null value type
+        DoubleBuffer<NullType> d_values;
+
+        return DispatchSegmentedRadixSort<false, KeyT, NullType, OffsetIteratorT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            begin_bit,
+            end_bit,
+            true,
+            stream,
+            debug_synchronous);
+    }
+
+    /**
+     * \brief Sorts segments of keys into descending order. (~<em>2N</em> auxiliary storage required).
+     *
+     * \par
+     * - The contents of the input data are not altered by the sorting operation
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  num_segments;       // e.g., 3
+     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
+     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_keys_out;        // e.g., [-, -, -, -, -, -, -]
+     * ...
+     *
+     * // Create a DoubleBuffer to wrap the pair of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceSegmentedRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_keys_out            <-- [8, 7, 6, 9, 5, 3, 0]
+     *
+     * \endcode
+     *
+     * \tparam KeyT             <b>[inferred]</b> Key type
+     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            KeyT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortKeysDescending(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        const KeyT          *d_keys_in,                             ///< [in] %Device-accessible pointer to the input data of key data to sort
+        KeyT                *d_keys_out,                            ///< [out] %Device-accessible pointer to the sorted output sequence of key data
+        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
+        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        DoubleBuffer<KeyT>      d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
+        DoubleBuffer<NullType>  d_values;
+
+        return DispatchSegmentedRadixSort<true, KeyT, NullType, OffsetIteratorT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            begin_bit,
+            end_bit,
+            false,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Sorts segments of keys into descending order. (~<em>N </em>auxiliary storage required).
+     *
+     * \par
+     * - The sorting operation is given a pair of key buffers managed by a
+     *   DoubleBuffer structure that indicates which of the two buffers is
+     *   "current" (and thus contains the input data to be sorted).
+     * - The contents of both buffers may be altered by the sorting operation.
+     * - Upon completion, the sorting operation will update the "current" indicator
+     *   within the DoubleBuffer wrapper to reference which of the two buffers
+     *   now contains the sorted output sequence (a function of the number of key bits
+     *   specified and the targeted device architecture).
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
+     * - \devicestorageP
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for sorting data
+     * int  num_items;          // e.g., 7
+     * int  num_segments;       // e.g., 3
+     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
+     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int  *d_key_alt_buf;     // e.g., [-, -, -, -, -, -, -]
+     * ...
+     *
+     * // Create a DoubleBuffer to wrap the pair of device pointers
+     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sorting operation
+     * cub::DeviceSegmentedRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys,
+     *     num_items, num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_keys.Current()      <-- [8, 7, 6, 9, 5, 3, 0]
+     *
+     * \endcode
+     *
+     * \tparam KeyT             <b>[inferred]</b> Key type
+     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            KeyT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t SortKeysDescending(
+        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>  &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
+        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
+        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
+        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // Null value type
+        DoubleBuffer<NullType> d_values;
+
+        return DispatchSegmentedRadixSort<true, KeyT, NullType, OffsetIteratorT, OffsetT>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys,
+            d_values,
+            num_items,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            begin_bit,
+            end_bit,
+            true,
+            stream,
+            debug_synchronous);
+    }
+
+
+    //@}  end member group
+
+
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_segmented_reduce.cuh b/third_party/cub/cub/device/device_segmented_reduce.cuh
new file mode 100644
index 0000000..6c3b54a
--- /dev/null
+++ b/third_party/cub/cub/device/device_segmented_reduce.cuh
@@ -0,0 +1,619 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceSegmentedReduce provides device-wide, parallel operations for computing a batched reduction across multiple sequences of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "../iterator/arg_index_input_iterator.cuh"
+#include "dispatch/dispatch_reduce.cuh"
+#include "dispatch/dispatch_reduce_by_key.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceSegmentedReduce provides device-wide, parallel operations for computing a reduction across multiple sequences of data items residing within device-accessible memory. ![](reduce_logo.png)
+ * \ingroup SegmentedModule
+ *
+ * \par Overview
+ * A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
+ * uses a binary combining operator to compute a single aggregate from a sequence of input elements.
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceSegmentedReduce}
+ *
+ */
+struct DeviceSegmentedReduce
+{
+    /**
+     * \brief Computes a device-wide segmented reduction using the specified binary \p reduction_op functor.
+     *
+     * \par
+     * - Does not support binary reduction operators that are non-commutative.
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates a custom min-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // CustomMin functor
+     * struct CustomMin
+     * {
+     *     template <typename T>
+     *     CUB_RUNTIME_FUNCTION __forceinline__
+     *     T operator()(const T &a, const T &b) const {
+     *         return (b < a) ? b : a;
+     *     }
+     * };
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int          num_segments;   // e.g., 3
+     * int          *d_offsets;     // e.g., [0, 3, 3, 7]
+     * int          *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int          *d_out;         // e.g., [-, -, -]
+     * CustomMin    min_op;
+     * int          initial_value;           // e.g., INT_MAX
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedReduce::Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1, min_op, initial_value);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run reduction
+     * cub::DeviceSegmentedReduce::Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1, min_op, initial_value);
+     *
+     * // d_out <-- [6, INT_MAX, 0]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT      <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
+     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     * \tparam ReductionOp          <b>[inferred]</b> Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+     * \tparam T                    <b>[inferred]</b> Data element type that is convertible to the \p value type of \p InputIteratorT
+     */
+    template <
+        typename            InputIteratorT,
+        typename            OutputIteratorT,
+        typename            OffsetIteratorT,
+        typename            ReductionOp,
+        typename            T>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t Reduce(
+        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
+        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        ReductionOp         reduction_op,                       ///< [in] Binary reduction functor 
+        T                   initial_value,                      ///< [in] Initial value of the reduction for each segment
+        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        return DispatchSegmentedReduce<InputIteratorT, OutputIteratorT, OffsetIteratorT, OffsetT, ReductionOp>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            reduction_op,
+            initial_value,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide segmented sum using the addition ('+') operator.
+     *
+     * \par
+     * - Uses \p 0 as the initial value of the reduction for each segment.
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - Does not support \p + operators that are non-commutative..
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the sum reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int num_segments;   // e.g., 3
+     * int *d_offsets;     // e.g., [0, 3, 3, 7]
+     * int *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int *d_out;         // e.g., [-, -, -]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run sum-reduction
+     * cub::DeviceSegmentedReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_out <-- [21, 0, 17]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
+     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            InputIteratorT,
+        typename            OutputIteratorT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t Sum(
+        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
+        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The output value type
+        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+            typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+        return DispatchSegmentedReduce<InputIteratorT,  OutputIteratorT, OffsetIteratorT, OffsetT, cub::Sum>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            cub::Sum(),
+            OutputT(),            // zero-initialize
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide segmented minimum using the less-than ('<') operator.
+     *
+     * \par
+     * - Uses <tt>std::numeric_limits<T>::max()</tt> as the initial value of the reduction for each segment.
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - Does not support \p < operators that are non-commutative.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the min-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int num_segments;   // e.g., 3
+     * int *d_offsets;     // e.g., [0, 3, 3, 7]
+     * int *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int *d_out;         // e.g., [-, -, -]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run min-reduction
+     * cub::DeviceSegmentedReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_out <-- [6, INT_MAX, 0]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
+     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            InputIteratorT,
+        typename            OutputIteratorT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t Min(
+        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
+        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The input value type
+        typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+        return DispatchSegmentedReduce<InputIteratorT,  OutputIteratorT, OffsetIteratorT, OffsetT, cub::Min>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            cub::Min(),
+            Traits<InputT>::Max(),    // replace with std::numeric_limits<T>::max() when C++11 support is more prevalent
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Finds the first device-wide minimum in each segment using the less-than ('<') operator, also returning the in-segment index of that item.
+     *
+     * \par
+     * - The output value type of \p d_out is cub::KeyValuePair <tt><int, T></tt> (assuming the value type of \p d_in is \p T)
+     *   - The minimum of the <em>i</em><sup>th</sup> segment is written to <tt>d_out[i].value</tt> and its offset in that segment is written to <tt>d_out[i].key</tt>.
+     *   - The <tt>{1, std::numeric_limits<T>::max()}</tt> tuple is produced for zero-length inputs
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - Does not support \p < operators that are non-commutative.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the argmin-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int                      num_segments;   // e.g., 3
+     * int                      *d_offsets;     // e.g., [0, 3, 3, 7]
+     * int                      *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * KeyValuePair<int, int>   *d_out;         // e.g., [{-,-}, {-,-}, {-,-}]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run argmin-reduction
+     * cub::DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_out <-- [{1,6}, {1,INT_MAX}, {2,0}]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items (of some type \p T) \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate (having value type <tt>KeyValuePair<int, T></tt>) \iterator
+     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            InputIteratorT,
+        typename            OutputIteratorT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t ArgMin(
+        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
+        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The input type
+        typedef typename std::iterator_traits<InputIteratorT>::value_type InputValueT;
+
+        // The output tuple type
+        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+            KeyValuePair<OffsetT, InputValueT>,                                                                 // ... then the key value pair OffsetT + InputValueT
+            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputTupleT;                     // ... else the output iterator's value type
+
+        // The output value type
+        typedef typename OutputTupleT::Value OutputValueT;
+
+        // Wrapped input iterator to produce index-value <OffsetT, InputT> tuples
+        typedef ArgIndexInputIterator<InputIteratorT, OffsetT, OutputValueT> ArgIndexInputIteratorT;
+        ArgIndexInputIteratorT d_indexed_in(d_in);
+
+        // Initial value
+        OutputTupleT initial_value(1, Traits<InputValueT>::Max());   // replace with std::numeric_limits<T>::max() when C++11 support is more prevalent
+
+        return DispatchSegmentedReduce<ArgIndexInputIteratorT,  OutputIteratorT, OffsetIteratorT, OffsetT, cub::ArgMin>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_indexed_in,
+            d_out,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            cub::ArgMin(),
+            initial_value,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide segmented maximum using the greater-than ('>') operator.
+     *
+     * \par
+     * - Uses <tt>std::numeric_limits<T>::lowest()</tt> as the initial value of the reduction.
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - Does not support \p > operators that are non-commutative.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the max-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int num_segments;   // e.g., 3
+     * int *d_offsets;     // e.g., [0, 3, 3, 7]
+     * int *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * int *d_out;         // e.g., [-, -, -]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run max-reduction
+     * cub::DeviceSegmentedReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_out <-- [8, INT_MIN, 9]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
+     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            InputIteratorT,
+        typename            OutputIteratorT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t Max(
+        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
+        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The input value type
+        typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+        return DispatchSegmentedReduce<InputIteratorT,  OutputIteratorT, OffsetIteratorT, OffsetT, cub::Max>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_out,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            cub::Max(),
+            Traits<InputT>::Lowest(),    // replace with std::numeric_limits<T>::lowest() when C++11 support is more prevalent
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Finds the first device-wide maximum in each segment using the greater-than ('>') operator, also returning the in-segment index of that item
+     *
+     * \par
+     * - The output value type of \p d_out is cub::KeyValuePair <tt><int, T></tt> (assuming the value type of \p d_in is \p T)
+     *   - The maximum of the <em>i</em><sup>th</sup> segment is written to <tt>d_out[i].value</tt> and its offset in that segment is written to <tt>d_out[i].key</tt>.
+     *   - The <tt>{1, std::numeric_limits<T>::lowest()}</tt> tuple is produced for zero-length inputs
+     * - When input a contiguous sequence of segments, a single sequence
+     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
+     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
+     *   the latter is specified as <tt>segment_offsets+1</tt>).
+     * - Does not support \p > operators that are non-commutative.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the argmax-reduction of a device vector of \p int data elements.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_reduce.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int                      num_segments;   // e.g., 3
+     * int                      *d_offsets;     // e.g., [0, 3, 3, 7]
+     * int                      *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
+     * KeyValuePair<int, int>   *d_out;         // e.g., [{-,-}, {-,-}, {-,-}]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run argmax-reduction
+     * cub::DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_out,
+     *     num_segments, d_offsets, d_offsets + 1);
+     *
+     * // d_out <-- [{0,8}, {1,INT_MIN}, {3,9}]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items (of some type \p T) \iterator
+     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate (having value type <tt>KeyValuePair<int, T></tt>) \iterator
+     * \tparam OffsetIteratorT    <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
+     */
+    template <
+        typename            InputIteratorT,
+        typename            OutputIteratorT,
+        typename            OffsetIteratorT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t ArgMax(
+        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
+        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // The input type
+        typedef typename std::iterator_traits<InputIteratorT>::value_type InputValueT;
+
+        // The output tuple type
+        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+            KeyValuePair<OffsetT, InputValueT>,                                                                 // ... then the key value pair OffsetT + InputValueT
+            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputTupleT;                     // ... else the output iterator's value type
+
+        // The output value type
+        typedef typename OutputTupleT::Value OutputValueT;
+
+        // Wrapped input iterator to produce index-value <OffsetT, InputT> tuples
+        typedef ArgIndexInputIterator<InputIteratorT, OffsetT, OutputValueT> ArgIndexInputIteratorT;
+        ArgIndexInputIteratorT d_indexed_in(d_in);
+
+        // Initial value
+        OutputTupleT initial_value(1, Traits<InputValueT>::Lowest());     // replace with std::numeric_limits<T>::lowest() when C++11 support is more prevalent
+
+        return DispatchSegmentedReduce<ArgIndexInputIteratorT, OutputIteratorT, OffsetIteratorT, OffsetT, cub::ArgMax>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_indexed_in,
+            d_out,
+            num_segments,
+            d_begin_offsets,
+            d_end_offsets,
+            cub::ArgMax(),
+            initial_value,
+            stream,
+            debug_synchronous);
+    }
+
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_select.cuh b/third_party/cub/cub/device/device_select.cuh
new file mode 100644
index 0000000..52a3e12
--- /dev/null
+++ b/third_party/cub/cub/device/device_select.cuh
@@ -0,0 +1,369 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceSelect provides device-wide, parallel operations for compacting selected items from sequences of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch/dispatch_select_if.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceSelect provides device-wide, parallel operations for compacting selected items from sequences of data items residing within device-accessible memory. ![](select_logo.png)
+ * \ingroup SingleModule
+ *
+ * \par Overview
+ * These operations apply a selection criterion to selectively copy
+ * items from a specified input sequence to a compact output sequence.
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceSelect}
+ *
+ * \par Performance
+ * \linear_performance{select-flagged, select-if, and select-unique}
+ *
+ * \par
+ * The following chart illustrates DeviceSelect::If
+ * performance across different CUDA architectures for \p int32 items,
+ * where 50% of the items are randomly selected.
+ *
+ * \image html select_if_int32_50_percent.png
+ *
+ * \par
+ * The following chart illustrates DeviceSelect::Unique
+ * performance across different CUDA architectures for \p int32 items
+ * where segments have lengths uniformly sampled from [1,1000].
+ *
+ * \image html select_unique_int32_len_500.png
+ *
+ * \par
+ * \plots_below
+ *
+ */
+struct DeviceSelect
+{
+    /**
+     * \brief Uses the \p d_flags sequence to selectively copy the corresponding items from \p d_in into \p d_out.  The total number of items selected is written to \p d_num_selected_out. ![](select_flags_logo.png)
+     *
+     * \par
+     * - The value type of \p d_flags must be castable to \p bool (e.g., \p bool, \p char, \p int, etc.).
+     * - Copies of the selected items are compacted into \p d_out and maintain their original relative ordering.
+     * - \devicestorage
+     *
+     * \par Snippet
+     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>       // or equivalently <cub/device/device_select.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input, flags, and output
+     * int  num_items;              // e.g., 8
+     * int  *d_in;                  // e.g., [1, 2, 3, 4, 5, 6, 7, 8]
+     * char *d_flags;               // e.g., [1, 0, 0, 1, 0, 1, 1, 0]
+     * int  *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int  *d_num_selected_out;    // e.g., [ ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSelect::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run selection
+     * cub::DeviceSelect::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items);
+     *
+     * // d_out                 <-- [1, 4, 6, 7]
+     * // d_num_selected_out    <-- [4]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam FlagIterator         <b>[inferred]</b> Random-access input iterator type for reading selection flags \iterator
+     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing selected items \iterator
+     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    FlagIterator,
+        typename                    OutputIteratorT,
+        typename                    NumSelectedIteratorT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Flagged(
+        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
+        FlagIterator                d_flags,                        ///< [in] Pointer to the input sequence of selection flags
+        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of selected data items
+        NumSelectedIteratorT         d_num_selected_out,                 ///< [out] Pointer to the output total number of items selected (i.e., length of \p d_out)
+        int                         num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        typedef int                     OffsetT;         // Signed integer type for global offsets
+        typedef NullType                SelectOp;       // Selection op (not used)
+        typedef NullType                EqualityOp;     // Equality operator (not used)
+
+        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, false>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_flags,
+            d_out,
+            d_num_selected_out,
+            SelectOp(),
+            EqualityOp(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Uses the \p select_op functor to selectively copy items from \p d_in into \p d_out.  The total number of items selected is written to \p d_num_selected_out. ![](select_logo.png)
+     *
+     * \par
+     * - Copies of the selected items are compacted into \p d_out and maintain their original relative ordering.
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated select-if performance across different
+     * CUDA architectures for \p int32 and \p int64 items, respectively.  Items are
+     * selected with 50% probability.
+     *
+     * \image html select_if_int32_50_percent.png
+     * \image html select_if_int64_50_percent.png
+     *
+     * \par
+     * The following charts are similar, but 5% selection probability:
+     *
+     * \image html select_if_int32_5_percent.png
+     * \image html select_if_int64_5_percent.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_select.cuh>
+     *
+     * // Functor type for selecting values less than some criteria
+     * struct LessThan
+     * {
+     *     int compare;
+     *
+     *     CUB_RUNTIME_FUNCTION __forceinline__
+     *     LessThan(int compare) : compare(compare) {}
+     *
+     *     CUB_RUNTIME_FUNCTION __forceinline__
+     *     bool operator()(const int &a) const {
+     *         return (a < compare);
+     *     }
+     * };
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int      num_items;              // e.g., 8
+     * int      *d_in;                  // e.g., [0, 2, 3, 9, 5, 2, 81, 8]
+     * int      *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int      *d_num_selected_out;    // e.g., [ ]
+     * LessThan select_op(7);
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run selection
+     * cub::DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op);
+     *
+     * // d_out                 <-- [0, 2, 3, 5, 2]
+     * // d_num_selected_out    <-- [5]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing selected items \iterator
+     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
+     * \tparam SelectOp             <b>[inferred]</b> Selection operator type having member <tt>bool operator()(const T &a)</tt>
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT,
+        typename                    NumSelectedIteratorT,
+        typename                    SelectOp>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t If(
+        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of selected data items
+        NumSelectedIteratorT         d_num_selected_out,                 ///< [out] Pointer to the output total number of items selected (i.e., length of \p d_out)
+        int                         num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
+        SelectOp                    select_op,                      ///< [in] Unary selection operator
+        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        typedef int                     OffsetT;         // Signed integer type for global offsets
+        typedef NullType*               FlagIterator;   // FlagT iterator type (not used)
+        typedef NullType                EqualityOp;     // Equality operator (not used)
+
+        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, false>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            NULL,
+            d_out,
+            d_num_selected_out,
+            select_op,
+            EqualityOp(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Given an input sequence \p d_in having runs of consecutive equal-valued keys, only the first key from each run is selectively copied to \p d_out.  The total number of items selected is written to \p d_num_selected_out. ![](unique_logo.png)
+     *
+     * \par
+     * - The <tt>==</tt> equality operator is used to determine whether keys are equivalent
+     * - Copies of the selected items are compacted into \p d_out and maintain their original relative ordering.
+     * - \devicestorage
+     *
+     * \par Performance
+     * The following charts illustrate saturated select-unique performance across different
+     * CUDA architectures for \p int32 and \p int64 items, respectively.  Segments have
+     * lengths uniformly sampled from [1,1000].
+     *
+     * \image html select_unique_int32_len_500.png
+     * \image html select_unique_int64_len_500.png
+     *
+     * \par
+     * The following charts are similar, but with segment lengths uniformly sampled from [1,10]:
+     *
+     * \image html select_unique_int32_len_5.png
+     * \image html select_unique_int64_len_5.png
+     *
+     * \par Snippet
+     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>       // or equivalently <cub/device/device_select.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input and output
+     * int  num_items;              // e.g., 8
+     * int  *d_in;                  // e.g., [0, 2, 2, 9, 5, 5, 5, 8]
+     * int  *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
+     * int  *d_num_selected_out;    // e.g., [ ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void     *d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSelect::Unique(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run selection
+     * cub::DeviceSelect::Unique(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items);
+     *
+     * // d_out                 <-- [0, 2, 9, 5, 8]
+     * // d_num_selected_out    <-- [5]
+     *
+     * \endcode
+     *
+     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
+     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing selected items \iterator
+     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
+     */
+    template <
+        typename                    InputIteratorT,
+        typename                    OutputIteratorT,
+        typename                    NumSelectedIteratorT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Unique(
+        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of selected data items
+        NumSelectedIteratorT         d_num_selected_out,             ///< [out] Pointer to the output total number of items selected (i.e., length of \p d_out)
+        int                         num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        typedef int                     OffsetT;         // Signed integer type for global offsets
+        typedef NullType*               FlagIterator;   // FlagT iterator type (not used)
+        typedef NullType                SelectOp;       // Selection op (not used)
+        typedef Equality                EqualityOp;     // Default == operator
+
+        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, false>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            NULL,
+            d_out,
+            d_num_selected_out,
+            SelectOp(),
+            EqualityOp(),
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+
+};
+
+/**
+ * \example example_device_select_flagged.cu
+ * \example example_device_select_if.cu
+ * \example example_device_select_unique.cu
+ */
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/device_spmv.cuh b/third_party/cub/cub/device/device_spmv.cuh
new file mode 100644
index 0000000..63b6a7e
--- /dev/null
+++ b/third_party/cub/cub/device/device_spmv.cuh
@@ -0,0 +1,174 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceSpmv provides device-wide parallel operations for performing sparse-matrix * vector multiplication (SpMV).
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+#include <limits>
+
+#include "dispatch/dispatch_spmv_orig.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief DeviceSpmv provides device-wide parallel operations for performing sparse-matrix * dense-vector multiplication (SpMV).
+ * \ingroup SingleModule
+ *
+ * \par Overview
+ * The [<em>SpMV computation</em>](http://en.wikipedia.org/wiki/Sparse_matrix-vector_multiplication)
+ * performs the matrix-vector operation
+ * <em>y</em> = <em>alpha</em>*<b>A</b>*<em>x</em> + <em>beta</em>*<em>y</em>,
+ * where:
+ *  - <b>A</b> is an <em>m</em>x<em>n</em> sparse matrix whose non-zero structure is specified in
+ *    [<em>compressed-storage-row (CSR) format</em>](http://en.wikipedia.org/wiki/Sparse_matrix#Compressed_row_Storage_.28CRS_or_CSR.29)
+ *    (i.e., three arrays: <em>values</em>, <em>row_offsets</em>, and <em>column_indices</em>)
+ *  - <em>x</em> and <em>y</em> are dense vectors
+ *  - <em>alpha</em> and <em>beta</em> are scalar multiplicands
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceSpmv}
+ *
+ */
+struct DeviceSpmv
+{
+    /******************************************************************//**
+     * \name CSR matrix operations
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief This function performs the matrix-vector operation <em>y</em> = <b>A</b>*<em>x</em>.
+     *
+     * \par Snippet
+     * The code snippet below illustrates SpMV upon a 9x9 CSR matrix <b>A</b>
+     * representing a 3x3 lattice (24 non-zeros).
+     *
+     * \par
+     * \code
+     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_spmv.cuh>
+     *
+     * // Declare, allocate, and initialize device-accessible pointers for input matrix A, input vector x,
+     * // and output vector y
+     * int    num_rows = 9;
+     * int    num_cols = 9;
+     * int    num_nonzeros = 24;
+     *
+     * float* d_values;  // e.g., [1, 1, 1, 1, 1, 1, 1, 1,
+     *                   //        1, 1, 1, 1, 1, 1, 1, 1,
+     *                   //        1, 1, 1, 1, 1, 1, 1, 1]
+     *
+     * int*   d_column_indices; // e.g., [1, 3, 0, 2, 4, 1, 5, 0,
+     *                          //        4, 6, 1, 3, 5, 7, 2, 4,
+     *                          //        8, 3, 7, 4, 6, 8, 5, 7]
+     *
+     * int*   d_row_offsets;    // e.g., [0, 2, 5, 7, 10, 14, 17, 19, 22, 24]
+     *
+     * float* d_vector_x;       // e.g., [1, 1, 1, 1, 1, 1, 1, 1, 1]
+     * float* d_vector_y;       // e.g., [ ,  ,  ,  ,  ,  ,  ,  ,  ]
+     * ...
+     *
+     * // Determine temporary device storage requirements
+     * void*    d_temp_storage = NULL;
+     * size_t   temp_storage_bytes = 0;
+     * cub::DeviceSpmv::CsrMV(d_temp_storage, temp_storage_bytes, d_values,
+     *     d_row_offsets, d_column_indices, d_vector_x, d_vector_y,
+     *     num_rows, num_cols, num_nonzeros, alpha, beta);
+     *
+     * // Allocate temporary storage
+     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
+     *
+     * // Run SpMV
+     * cub::DeviceSpmv::CsrMV(d_temp_storage, temp_storage_bytes, d_values,
+     *     d_row_offsets, d_column_indices, d_vector_x, d_vector_y,
+     *     num_rows, num_cols, num_nonzeros, alpha, beta);
+     *
+     * // d_vector_y <-- [2, 3, 2, 3, 4, 3, 2, 3, 2]
+     *
+     * \endcode
+     *
+     * \tparam ValueT       <b>[inferred]</b> Matrix and vector value type (e.g., /p float, /p double, etc.)
+     */
+    template <
+        typename            ValueT>
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t CsrMV(
+        void*               d_temp_storage,                     ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                 ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        ValueT*             d_values,                           ///< [in] Pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix <b>A</b>.
+        int*                d_row_offsets,                      ///< [in] Pointer to the array of \p m + 1 offsets demarcating the start of every row in \p d_column_indices and \p d_values (with the final entry being equal to \p num_nonzeros)
+        int*                d_column_indices,                   ///< [in] Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix <b>A</b>.  (Indices are zero-valued.)
+        ValueT*             d_vector_x,                         ///< [in] Pointer to the array of \p num_cols values corresponding to the dense input vector <em>x</em>
+        ValueT*             d_vector_y,                         ///< [out] Pointer to the array of \p num_rows values corresponding to the dense output vector <em>y</em>
+        int                 num_rows,                           ///< [in] number of rows of matrix <b>A</b>.
+        int                 num_cols,                           ///< [in] number of columns of matrix <b>A</b>.
+        int                 num_nonzeros,                       ///< [in] number of nonzero elements of matrix <b>A</b>.
+        cudaStream_t        stream                  = 0,        ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous       = false)    ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        SpmvParams<ValueT, int> spmv_params;
+        spmv_params.d_values             = d_values;
+        spmv_params.d_row_end_offsets    = d_row_offsets + 1;
+        spmv_params.d_column_indices     = d_column_indices;
+        spmv_params.d_vector_x           = d_vector_x;
+        spmv_params.d_vector_y           = d_vector_y;
+        spmv_params.num_rows             = num_rows;
+        spmv_params.num_cols             = num_cols;
+        spmv_params.num_nonzeros         = num_nonzeros;
+        spmv_params.alpha                = 1.0;
+        spmv_params.beta                 = 0.0;
+
+        return DispatchSpmv<ValueT, int>::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            spmv_params,
+            stream,
+            debug_synchronous);
+    }
+
+    //@}  end member group
+};
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/dispatch/dispatch_histogram.cuh b/third_party/cub/cub/device/dispatch/dispatch_histogram.cuh
new file mode 100644
index 0000000..ab08e8e
--- /dev/null
+++ b/third_party/cub/cub/device/dispatch/dispatch_histogram.cuh
@@ -0,0 +1,1096 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceHistogram provides device-wide parallel operations for constructing histogram(s) from a sequence of samples data residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+#include <limits>
+
+#include "../../agent/agent_histogram.cuh"
+#include "../../util_debug.cuh"
+#include "../../util_device.cuh"
+#include "../../thread/thread_search.cuh"
+#include "../../grid/grid_queue.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+
+/******************************************************************************
+ * Histogram kernel entry points
+ *****************************************************************************/
+
+/**
+ * Histogram initialization kernel entry point
+ */
+template <
+    int                                             NUM_ACTIVE_CHANNELS,            ///< Number of channels actively being histogrammed
+    typename                                        CounterT,                       ///< Integer type for counting sample occurrences per histogram bin
+    typename                                        OffsetT>                        ///< Signed integer type for global offsets
+__global__ void DeviceHistogramInitKernel(
+    ArrayWrapper<int, NUM_ACTIVE_CHANNELS>          num_output_bins_wrapper,        ///< Number of output histogram bins per channel
+    ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS>    d_output_histograms_wrapper,    ///< Histogram counter data having logical dimensions <tt>CounterT[NUM_ACTIVE_CHANNELS][num_bins.array[CHANNEL]]</tt>
+    GridQueue<int>                                  tile_queue)                     ///< Drain queue descriptor for dynamically mapping tile data onto thread blocks
+{
+    if ((threadIdx.x == 0) && (blockIdx.x == 0))
+        tile_queue.ResetDrain();
+
+    int output_bin = (blockIdx.x * blockDim.x) + threadIdx.x;
+
+    #pragma unroll
+    for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+    {
+        if (output_bin < num_output_bins_wrapper.array[CHANNEL])
+            d_output_histograms_wrapper.array[CHANNEL][output_bin] = 0;
+    }
+}
+
+
+/**
+ * Histogram privatized sweep kernel entry point (multi-block).  Computes privatized histograms, one per thread block.
+ */
+template <
+    typename                                            AgentHistogramPolicyT,     ///< Parameterized AgentHistogramPolicy tuning policy type
+    int                                                 PRIVATIZED_SMEM_BINS,           ///< Maximum number of histogram bins per channel (e.g., up to 256)
+    int                                                 NUM_CHANNELS,                   ///< Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
+    int                                                 NUM_ACTIVE_CHANNELS,            ///< Number of channels actively being histogrammed
+    typename                                            SampleIteratorT,                ///< The input iterator type. \iterator.
+    typename                                            CounterT,                       ///< Integer type for counting sample occurrences per histogram bin
+    typename                                            PrivatizedDecodeOpT,            ///< The transform operator type for determining privatized counter indices from samples, one for each channel
+    typename                                            OutputDecodeOpT,                ///< The transform operator type for determining output bin-ids from privatized counter indices, one for each channel
+    typename                                            OffsetT>                        ///< Signed integer type for global offsets
+__launch_bounds__ (int(AgentHistogramPolicyT::BLOCK_THREADS))
+__global__ void DeviceHistogramSweepKernel(
+    SampleIteratorT                                         d_samples,                          ///< Input data to reduce
+    ArrayWrapper<int, NUM_ACTIVE_CHANNELS>                  num_output_bins_wrapper,            ///< The number bins per final output histogram
+    ArrayWrapper<int, NUM_ACTIVE_CHANNELS>                  num_privatized_bins_wrapper,        ///< The number bins per privatized histogram
+    ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS>            d_output_histograms_wrapper,        ///< Reference to final output histograms
+    ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS>            d_privatized_histograms_wrapper,    ///< Reference to privatized histograms
+    ArrayWrapper<OutputDecodeOpT, NUM_ACTIVE_CHANNELS>      output_decode_op_wrapper,           ///< The transform operator for determining output bin-ids from privatized counter indices, one for each channel
+    ArrayWrapper<PrivatizedDecodeOpT, NUM_ACTIVE_CHANNELS>  privatized_decode_op_wrapper,       ///< The transform operator for determining privatized counter indices from samples, one for each channel
+    OffsetT                                                 num_row_pixels,                     ///< The number of multi-channel pixels per row in the region of interest
+    OffsetT                                                 num_rows,                           ///< The number of rows in the region of interest
+    OffsetT                                                 row_stride_samples,                 ///< The number of samples between starts of consecutive rows in the region of interest
+    int                                                     tiles_per_row,                      ///< Number of image tiles per row
+    GridQueue<int>                                          tile_queue)                         ///< Drain queue descriptor for dynamically mapping tile data onto thread blocks
+{
+    // Thread block type for compositing input tiles
+    typedef AgentHistogram<
+            AgentHistogramPolicyT,
+            PRIVATIZED_SMEM_BINS,
+            NUM_CHANNELS,
+            NUM_ACTIVE_CHANNELS,
+            SampleIteratorT,
+            CounterT,
+            PrivatizedDecodeOpT,
+            OutputDecodeOpT,
+            OffsetT>
+        AgentHistogramT;
+
+    // Shared memory for AgentHistogram
+    __shared__ typename AgentHistogramT::TempStorage temp_storage;
+
+    AgentHistogramT agent(
+        temp_storage,
+        d_samples,
+        num_output_bins_wrapper.array,
+        num_privatized_bins_wrapper.array,
+        d_output_histograms_wrapper.array,
+        d_privatized_histograms_wrapper.array,
+        output_decode_op_wrapper.array,
+        privatized_decode_op_wrapper.array);
+
+    // Initialize counters
+    agent.InitBinCounters();
+
+    // Consume input tiles
+    agent.ConsumeTiles(
+        num_row_pixels,
+        num_rows,
+        row_stride_samples,
+        tiles_per_row,
+        tile_queue);
+
+    // Store output to global (if necessary)
+    agent.StoreOutput();
+
+}
+
+
+
+
+
+
+/******************************************************************************
+ * Dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for DeviceHistogram
+ */
+template <
+    int         NUM_CHANNELS,               ///< Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
+    int         NUM_ACTIVE_CHANNELS,        ///< Number of channels actively being histogrammed
+    typename    SampleIteratorT,            ///< Random-access input iterator type for reading input items \iterator
+    typename    CounterT,                   ///< Integer type for counting sample occurrences per histogram bin
+    typename    LevelT,                     ///< Type for specifying bin level boundaries
+    typename    OffsetT>                    ///< Signed integer type for global offsets
+struct DipatchHistogram
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    /// The sample value type of the input iterator
+    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+    enum
+    {
+        // Maximum number of bins per channel for which we will use a privatized smem strategy
+        MAX_PRIVATIZED_SMEM_BINS = 256
+    };
+
+
+    //---------------------------------------------------------------------
+    // Transform functors for converting samples to bin-ids
+    //---------------------------------------------------------------------
+
+    // Searches for bin given a list of bin-boundary levels
+    template <typename LevelIteratorT>
+    struct SearchTransform
+    {
+        LevelIteratorT  d_levels;                   // Pointer to levels array
+        int             num_output_levels;          // Number of levels in array
+
+        // Initializer
+        __host__ __device__ __forceinline__ void Init(
+            LevelIteratorT  d_levels,               // Pointer to levels array
+            int             num_output_levels)      // Number of levels in array
+        {
+            this->d_levels          = d_levels;
+            this->num_output_levels = num_output_levels;
+        }
+
+        // Method for converting samples to bin-ids
+        template <CacheLoadModifier LOAD_MODIFIER, typename _SampleT>
+        __host__ __device__ __forceinline__ void BinSelect(_SampleT sample, int &bin, bool valid)
+        {
+            /// Level iterator wrapper type
+            typedef typename If<IsPointer<LevelIteratorT>::VALUE,
+                    CacheModifiedInputIterator<LOAD_MODIFIER, LevelT, OffsetT>,     // Wrap the native input pointer with CacheModifiedInputIterator
+                    LevelIteratorT>::Type                                           // Directly use the supplied input iterator type
+                WrappedLevelIteratorT;
+
+            WrappedLevelIteratorT wrapped_levels(d_levels);
+
+            int num_bins = num_output_levels - 1;
+            if (valid)
+            {
+                bin = UpperBound(wrapped_levels, num_output_levels, (LevelT) sample) - 1;
+                if (bin >= num_bins)
+                    bin = -1;
+            }
+        }
+    };
+
+
+    // Scales samples to evenly-spaced bins
+    struct ScaleTransform
+    {
+        int    num_bins;    // Number of levels in array
+        LevelT max;         // Max sample level (exclusive)
+        LevelT min;         // Min sample level (inclusive)
+        LevelT scale;       // Bin scaling factor
+
+        // Initializer
+        template <typename _LevelT>
+        __host__ __device__ __forceinline__ void Init(
+            int     num_output_levels,  // Number of levels in array
+            _LevelT max,                // Max sample level (exclusive)
+            _LevelT min,                // Min sample level (inclusive)
+            _LevelT scale)              // Bin scaling factor
+        {
+            this->num_bins = num_output_levels - 1;
+            this->max = max;
+            this->min = min;
+            this->scale = scale;
+        }
+
+        // Initializer (float specialization)
+        __host__ __device__ __forceinline__ void Init(
+            int    num_output_levels,   // Number of levels in array
+            float   max,                // Max sample level (exclusive)
+            float   min,                // Min sample level (inclusive)
+            float   scale)              // Bin scaling factor
+        {
+            this->num_bins = num_output_levels - 1;
+            this->max = max;
+            this->min = min;
+            this->scale = float(1.0) / scale;
+        }
+
+        // Initializer (double specialization)
+        __host__ __device__ __forceinline__ void Init(
+            int    num_output_levels,   // Number of levels in array
+            double max,                 // Max sample level (exclusive)
+            double min,                 // Min sample level (inclusive)
+            double scale)               // Bin scaling factor
+        {
+            this->num_bins = num_output_levels - 1;
+            this->max = max;
+            this->min = min;
+            this->scale = double(1.0) / scale;
+        }
+
+        // Method for converting samples to bin-ids
+        template <CacheLoadModifier LOAD_MODIFIER, typename _SampleT>
+        __host__ __device__ __forceinline__ void BinSelect(_SampleT sample, int &bin, bool valid)
+        {
+            LevelT level_sample = (LevelT) sample;
+
+            if (valid && (level_sample >= min) && (level_sample < max))
+                bin = (int) ((level_sample - min) / scale);
+        }
+
+        // Method for converting samples to bin-ids (float specialization)
+        template <CacheLoadModifier LOAD_MODIFIER>
+        __host__ __device__ __forceinline__ void BinSelect(float sample, int &bin, bool valid)
+        {
+            LevelT level_sample = (LevelT) sample;
+
+            if (valid && (level_sample >= min) && (level_sample < max))
+                bin = (int) ((level_sample - min) * scale);
+        }
+
+        // Method for converting samples to bin-ids (double specialization)
+        template <CacheLoadModifier LOAD_MODIFIER>
+        __host__ __device__ __forceinline__ void BinSelect(double sample, int &bin, bool valid)
+        {
+            LevelT level_sample = (LevelT) sample;
+
+            if (valid && (level_sample >= min) && (level_sample < max))
+                bin = (int) ((level_sample - min) * scale);
+        }
+    };
+
+
+    // Pass-through bin transform operator
+    struct PassThruTransform
+    {
+        // Method for converting samples to bin-ids
+        template <CacheLoadModifier LOAD_MODIFIER, typename _SampleT>
+        __host__ __device__ __forceinline__ void BinSelect(_SampleT sample, int &bin, bool valid)
+        {
+            if (valid)
+                bin = (int) sample;
+        }
+    };
+
+
+
+    //---------------------------------------------------------------------
+    // Tuning policies
+    //---------------------------------------------------------------------
+
+    template <int NOMINAL_ITEMS_PER_THREAD>
+    struct TScale
+    {
+        enum
+        {
+            V_SCALE = (sizeof(SampleT) + sizeof(int) - 1) / sizeof(int),
+            VALUE   = CUB_MAX((NOMINAL_ITEMS_PER_THREAD / NUM_ACTIVE_CHANNELS / V_SCALE), 1)
+        };
+    };
+
+
+    /// SM11
+    struct Policy110
+    {
+        // HistogramSweepPolicy
+        typedef AgentHistogramPolicy<
+                512,
+                (NUM_CHANNELS == 1) ? 8 : 2,
+                BLOCK_LOAD_DIRECT,
+                LOAD_DEFAULT,
+                true,
+                GMEM,
+                false>
+            HistogramSweepPolicy;
+    };
+
+    /// SM20
+    struct Policy200
+    {
+        // HistogramSweepPolicy
+        typedef AgentHistogramPolicy<
+                (NUM_CHANNELS == 1) ? 256 : 128,
+                (NUM_CHANNELS == 1) ? 8 : 3,
+                (NUM_CHANNELS == 1) ? BLOCK_LOAD_DIRECT : BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                true,
+                SMEM,
+                false>
+            HistogramSweepPolicy;
+    };
+
+    /// SM30
+    struct Policy300
+    {
+        // HistogramSweepPolicy
+        typedef AgentHistogramPolicy<
+                512,
+                (NUM_CHANNELS == 1) ? 8 : 2,
+                BLOCK_LOAD_DIRECT,
+                LOAD_DEFAULT,
+                true,
+                GMEM,
+                false>
+            HistogramSweepPolicy;
+    };
+
+    /// SM35
+    struct Policy350
+    {
+        // HistogramSweepPolicy
+        typedef AgentHistogramPolicy<
+                128,
+                TScale<8>::VALUE,
+                BLOCK_LOAD_DIRECT,
+                LOAD_LDG,
+                true,
+                BLEND,
+                true>
+            HistogramSweepPolicy;
+    };
+
+    /// SM50
+    struct Policy500
+    {
+        // HistogramSweepPolicy
+        typedef AgentHistogramPolicy<
+                384,
+                TScale<16>::VALUE,
+                BLOCK_LOAD_DIRECT,
+                LOAD_LDG,
+                true,
+                SMEM,
+                false>
+            HistogramSweepPolicy;
+    };
+
+
+
+    //---------------------------------------------------------------------
+    // Tuning policies of current PTX compiler pass
+    //---------------------------------------------------------------------
+
+#if (CUB_PTX_ARCH >= 500)
+    typedef Policy500 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 350)
+    typedef Policy350 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 300)
+    typedef Policy300 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 200)
+    typedef Policy200 PtxPolicy;
+
+#else
+    typedef Policy110 PtxPolicy;
+
+#endif
+
+    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
+    struct PtxHistogramSweepPolicy : PtxPolicy::HistogramSweepPolicy {};
+
+
+    //---------------------------------------------------------------------
+    // Utilities
+    //---------------------------------------------------------------------
+
+    /**
+     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
+     */
+    template <typename KernelConfig>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t InitConfigs(
+        int             ptx_version,
+        KernelConfig    &histogram_sweep_config)
+    {
+    #if (CUB_PTX_ARCH > 0)
+
+        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
+        return histogram_sweep_config.template Init<PtxHistogramSweepPolicy>();
+
+    #else
+
+        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
+        if (ptx_version >= 500)
+        {
+            return histogram_sweep_config.template Init<typename Policy500::HistogramSweepPolicy>();
+        }
+        else if (ptx_version >= 350)
+        {
+            return histogram_sweep_config.template Init<typename Policy350::HistogramSweepPolicy>();
+        }
+        else if (ptx_version >= 300)
+        {
+            return histogram_sweep_config.template Init<typename Policy300::HistogramSweepPolicy>();
+        }
+        else if (ptx_version >= 200)
+        {
+            return histogram_sweep_config.template Init<typename Policy200::HistogramSweepPolicy>();
+        }
+        else if (ptx_version >= 110)
+        {
+            return histogram_sweep_config.template Init<typename Policy110::HistogramSweepPolicy>();
+        }
+        else
+        {
+            // No global atomic support
+            return cudaErrorNotSupported;
+        }
+
+    #endif
+    }
+
+
+    /**
+     * Kernel kernel dispatch configuration
+     */
+    struct KernelConfig
+    {
+        int                             block_threads;
+        int                             pixels_per_thread;
+
+        template <typename BlockPolicy>
+        CUB_RUNTIME_FUNCTION __forceinline__
+        cudaError_t Init()
+        {
+            block_threads               = BlockPolicy::BLOCK_THREADS;
+            pixels_per_thread           = BlockPolicy::PIXELS_PER_THREAD;
+
+            return cudaSuccess;
+        }
+    };
+
+
+    //---------------------------------------------------------------------
+    // Dispatch entrypoints
+    //---------------------------------------------------------------------
+
+    /**
+     * Privatization-based dispatch routine
+     */
+    template <
+        typename                            PrivatizedDecodeOpT,                            ///< The transform operator type for determining privatized counter indices from samples, one for each channel
+        typename                            OutputDecodeOpT,                                ///< The transform operator type for determining output bin-ids from privatized counter indices, one for each channel
+        typename                            DeviceHistogramInitKernelT,                     ///< Function type of cub::DeviceHistogramInitKernel
+        typename                            DeviceHistogramSweepKernelT>                    ///< Function type of cub::DeviceHistogramSweepKernel
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t PrivatizedDispatch(
+        void*                               d_temp_storage,                                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                             temp_storage_bytes,                             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT                     d_samples,                                      ///< [in] The pointer to the input sequence of sample items. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+        CounterT*                           d_output_histograms[NUM_ACTIVE_CHANNELS],       ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
+        int                                 num_privatized_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of bin level boundaries for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
+        PrivatizedDecodeOpT                 privatized_decode_op[NUM_ACTIVE_CHANNELS],      ///< [in] Transform operators for determining bin-ids from samples, one for each channel
+        int                                 num_output_levels[NUM_ACTIVE_CHANNELS],         ///< [in] The number of bin level boundaries for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
+        OutputDecodeOpT                     output_decode_op[NUM_ACTIVE_CHANNELS],          ///< [in] Transform operators for determining bin-ids from samples, one for each channel
+        int                                 max_num_output_bins,                            ///< [in] Maximum number of output bins in any channel
+        OffsetT                             num_row_pixels,                                 ///< [in] The number of multi-channel pixels per row in the region of interest
+        OffsetT                             num_rows,                                       ///< [in] The number of rows in the region of interest
+        OffsetT                             row_stride_samples,                             ///< [in] The number of samples between starts of consecutive rows in the region of interest
+        DeviceHistogramInitKernelT          histogram_init_kernel,                          ///< [in] Kernel function pointer to parameterization of cub::DeviceHistogramInitKernel
+        DeviceHistogramSweepKernelT         histogram_sweep_kernel,                         ///< [in] Kernel function pointer to parameterization of cub::DeviceHistogramSweepKernel
+        KernelConfig                        histogram_sweep_config,                         ///< [in] Dispatch parameters that match the policy that \p histogram_sweep_kernel was compiled for
+        cudaStream_t                        stream,                                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                                debug_synchronous)                              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+    #ifndef CUB_RUNTIME_ENABLED
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported);
+
+    #else
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Get SM occupancy for histogram_sweep_kernel
+            int histogram_sweep_sm_occupancy;
+            if (CubDebug(error = MaxSmOccupancy(
+                histogram_sweep_sm_occupancy,
+                histogram_sweep_kernel,
+                histogram_sweep_config.block_threads))) break;
+
+            // Get device occupancy for histogram_sweep_kernel
+            int histogram_sweep_occupancy = histogram_sweep_sm_occupancy * sm_count;
+
+            if (num_row_pixels * NUM_CHANNELS == row_stride_samples)
+            {
+                // Treat as a single linear array of samples
+                num_row_pixels      *= num_rows;
+                num_rows            = 1;
+                row_stride_samples  = num_row_pixels * NUM_CHANNELS;
+            }
+
+            // Get grid dimensions, trying to keep total blocks ~histogram_sweep_occupancy
+            int pixels_per_tile     = histogram_sweep_config.block_threads * histogram_sweep_config.pixels_per_thread;
+            int tiles_per_row       = int(num_row_pixels + pixels_per_tile - 1) / pixels_per_tile;
+            int blocks_per_row      = CUB_MIN(histogram_sweep_occupancy, tiles_per_row);
+            int blocks_per_col      = (blocks_per_row > 0) ?
+                                        int(CUB_MIN(histogram_sweep_occupancy / blocks_per_row, num_rows)) :
+                                        0;
+            int num_thread_blocks   = blocks_per_row * blocks_per_col;
+
+            dim3 sweep_grid_dims;
+            sweep_grid_dims.x = (unsigned int) blocks_per_row;
+            sweep_grid_dims.y = (unsigned int) blocks_per_col;
+            sweep_grid_dims.z = 1;
+
+            // Temporary storage allocation requirements
+            const int   NUM_ALLOCATIONS = NUM_ACTIVE_CHANNELS + 1;
+            void*       allocations[NUM_ALLOCATIONS];
+            size_t      allocation_sizes[NUM_ALLOCATIONS];
+
+            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+                allocation_sizes[CHANNEL] = size_t(num_thread_blocks) * (num_privatized_levels[CHANNEL] - 1) * sizeof(CounterT);
+
+            allocation_sizes[NUM_ALLOCATIONS - 1] = GridQueue<int>::AllocationSize();
+
+            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+            if (d_temp_storage == NULL)
+            {
+                // Return if the caller is simply requesting the size of the storage allocation
+                break;
+            }
+
+            // Construct the grid queue descriptor
+            GridQueue<int> tile_queue(allocations[NUM_ALLOCATIONS - 1]);
+
+            // Setup array wrapper for histogram channel output (because we can't pass static arrays as kernel parameters)
+            ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS> d_output_histograms_wrapper;
+            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+                d_output_histograms_wrapper.array[CHANNEL] = d_output_histograms[CHANNEL];
+
+            // Setup array wrapper for privatized per-block histogram channel output (because we can't pass static arrays as kernel parameters)
+            ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS> d_privatized_histograms_wrapper;
+            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+                d_privatized_histograms_wrapper.array[CHANNEL] = (CounterT*) allocations[CHANNEL];
+
+            // Setup array wrapper for sweep bin transforms (because we can't pass static arrays as kernel parameters)
+            ArrayWrapper<PrivatizedDecodeOpT, NUM_ACTIVE_CHANNELS> privatized_decode_op_wrapper;
+            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+                privatized_decode_op_wrapper.array[CHANNEL] = privatized_decode_op[CHANNEL];
+
+            // Setup array wrapper for aggregation bin transforms (because we can't pass static arrays as kernel parameters)
+            ArrayWrapper<OutputDecodeOpT, NUM_ACTIVE_CHANNELS> output_decode_op_wrapper;
+            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+                output_decode_op_wrapper.array[CHANNEL] = output_decode_op[CHANNEL];
+
+            // Setup array wrapper for num privatized bins (because we can't pass static arrays as kernel parameters)
+            ArrayWrapper<int, NUM_ACTIVE_CHANNELS> num_privatized_bins_wrapper;
+            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+                num_privatized_bins_wrapper.array[CHANNEL] = num_privatized_levels[CHANNEL] - 1;
+
+            // Setup array wrapper for num output bins (because we can't pass static arrays as kernel parameters)
+            ArrayWrapper<int, NUM_ACTIVE_CHANNELS> num_output_bins_wrapper;
+            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+                num_output_bins_wrapper.array[CHANNEL] = num_output_levels[CHANNEL] - 1;
+
+            int histogram_init_block_threads    = 256;
+            int histogram_init_grid_dims        = (max_num_output_bins + histogram_init_block_threads - 1) / histogram_init_block_threads;
+
+            // Log DeviceHistogramInitKernel configuration
+            if (debug_synchronous) _CubLog("Invoking DeviceHistogramInitKernel<<<%d, %d, 0, %lld>>>()\n",
+                histogram_init_grid_dims, histogram_init_block_threads, (long long) stream);
+
+            // Invoke histogram_init_kernel
+            histogram_init_kernel<<<histogram_init_grid_dims, histogram_init_block_threads, 0, stream>>>(
+                num_output_bins_wrapper,
+                d_output_histograms_wrapper,
+                tile_queue);
+
+            // Return if empty problem
+            if ((blocks_per_row == 0) || (blocks_per_col == 0))
+                break;
+
+            // Log histogram_sweep_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking histogram_sweep_kernel<<<{%d, %d, %d}, %d, 0, %lld>>>(), %d pixels per thread, %d SM occupancy\n",
+                sweep_grid_dims.x, sweep_grid_dims.y, sweep_grid_dims.z,
+                histogram_sweep_config.block_threads, (long long) stream, histogram_sweep_config.pixels_per_thread, histogram_sweep_sm_occupancy);
+
+            // Invoke histogram_sweep_kernel
+            histogram_sweep_kernel<<<sweep_grid_dims, histogram_sweep_config.block_threads, 0, stream>>>(
+                d_samples,
+                num_output_bins_wrapper,
+                num_privatized_bins_wrapper,
+                d_output_histograms_wrapper,
+                d_privatized_histograms_wrapper,
+                output_decode_op_wrapper,
+                privatized_decode_op_wrapper,
+                num_row_pixels,
+                num_rows,
+                row_stride_samples,
+                tiles_per_row,
+                tile_queue);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+        }
+        while (0);
+
+        return error;
+
+    #endif // CUB_RUNTIME_ENABLED
+    }
+
+
+
+    /**
+     * Dispatch routine for HistogramRange, specialized for sample types larger than 8bit
+     */
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t DispatchRange(
+        void*               d_temp_storage,                                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+        CounterT*           d_output_histograms[NUM_ACTIVE_CHANNELS],      ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
+        int                 num_output_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
+        LevelT              *d_levels[NUM_ACTIVE_CHANNELS],             ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
+        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+        OffsetT             row_stride_samples,                         ///< [in] The number of samples between starts of consecutive rows in the region of interest
+        cudaStream_t        stream,                                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous,                          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+        Int2Type<false>     is_byte_sample)                             ///< [in] Marker type indicating whether or not SampleT is a 8b type
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel dispatch configurations
+            KernelConfig histogram_sweep_config;
+            if (CubDebug(error = InitConfigs(ptx_version, histogram_sweep_config)))
+                break;
+
+            // Use the search transform op for converting samples to privatized bins
+            typedef SearchTransform<LevelT*> PrivatizedDecodeOpT;
+
+            // Use the pass-thru transform op for converting privatized bins to output bins
+            typedef PassThruTransform OutputDecodeOpT;
+
+            PrivatizedDecodeOpT     privatized_decode_op[NUM_ACTIVE_CHANNELS];
+            OutputDecodeOpT         output_decode_op[NUM_ACTIVE_CHANNELS];
+            int                     max_levels = num_output_levels[0];
+
+            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+            {
+                privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]);
+                if (num_output_levels[channel] > max_levels)
+                    max_levels = num_output_levels[channel];
+            }
+            int max_num_output_bins = max_levels - 1;
+
+            // Dispatch
+            if (max_num_output_bins > MAX_PRIVATIZED_SMEM_BINS)
+            {
+                // Too many bins to keep in shared memory.
+                const int PRIVATIZED_SMEM_BINS = 0;
+
+                if (CubDebug(error = PrivatizedDispatch(
+                    d_temp_storage,
+                    temp_storage_bytes,
+                    d_samples,
+                    d_output_histograms,
+                    num_output_levels,
+                    privatized_decode_op,
+                    num_output_levels,
+                    output_decode_op,
+                    max_num_output_bins,
+                    num_row_pixels,
+                    num_rows,
+                    row_stride_samples,
+                    DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
+                    DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
+                    histogram_sweep_config,
+                    stream,
+                    debug_synchronous))) break;
+            }
+            else
+            {
+                // Dispatch shared-privatized approach
+                const int PRIVATIZED_SMEM_BINS = MAX_PRIVATIZED_SMEM_BINS;
+
+                if (CubDebug(error = PrivatizedDispatch(
+                    d_temp_storage,
+                    temp_storage_bytes,
+                    d_samples,
+                    d_output_histograms,
+                    num_output_levels,
+                    privatized_decode_op,
+                    num_output_levels,
+                    output_decode_op,
+                    max_num_output_bins,
+                    num_row_pixels,
+                    num_rows,
+                    row_stride_samples,
+                    DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
+                    DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
+                    histogram_sweep_config,
+                    stream,
+                    debug_synchronous))) break;
+            }
+
+        } while (0);
+
+        return error;
+    }
+
+
+    /**
+     * Dispatch routine for HistogramRange, specialized for 8-bit sample types (computes 256-bin privatized histograms and then reduces to user-specified levels)
+     */
+    CUB_RUNTIME_FUNCTION
+    static cudaError_t DispatchRange(
+        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                         ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+        CounterT*           d_output_histograms[NUM_ACTIVE_CHANNELS],   ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
+        int                 num_output_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
+        LevelT              *d_levels[NUM_ACTIVE_CHANNELS],             ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
+        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+        OffsetT             row_stride_samples,                         ///< [in] The number of samples between starts of consecutive rows in the region of interest
+        cudaStream_t        stream,                                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous,                          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+        Int2Type<true>      is_byte_sample)                             ///< [in] Marker type indicating whether or not SampleT is a 8b type
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel dispatch configurations
+            KernelConfig histogram_sweep_config;
+            if (CubDebug(error = InitConfigs(ptx_version, histogram_sweep_config)))
+                break;
+
+            // Use the pass-thru transform op for converting samples to privatized bins
+            typedef PassThruTransform PrivatizedDecodeOpT;
+
+            // Use the search transform op for converting privatized bins to output bins
+            typedef SearchTransform<LevelT*> OutputDecodeOpT;
+
+            int                         num_privatized_levels[NUM_ACTIVE_CHANNELS];
+            PrivatizedDecodeOpT         privatized_decode_op[NUM_ACTIVE_CHANNELS];
+            OutputDecodeOpT             output_decode_op[NUM_ACTIVE_CHANNELS];
+            int                         max_levels = num_output_levels[0];              // Maximum number of levels in any channel
+
+            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+            {
+                num_privatized_levels[channel] = 257;
+                output_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]);
+
+                if (num_output_levels[channel] > max_levels)
+                    max_levels = num_output_levels[channel];
+            }
+            int max_num_output_bins = max_levels - 1;
+
+            const int PRIVATIZED_SMEM_BINS = 256;
+
+            if (CubDebug(error = PrivatizedDispatch(
+                d_temp_storage,
+                temp_storage_bytes,
+                d_samples,
+                d_output_histograms,
+                num_privatized_levels,
+                privatized_decode_op,
+                num_output_levels,
+                output_decode_op,
+                max_num_output_bins,
+                num_row_pixels,
+                num_rows,
+                row_stride_samples,
+                DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
+                DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
+                histogram_sweep_config,
+                stream,
+                debug_synchronous))) break;
+
+        } while (0);
+
+        return error;
+    }
+
+
+    /**
+     * Dispatch routine for HistogramEven, specialized for sample types larger than 8-bit
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t DispatchEven(
+        void*               d_temp_storage,                            ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the input sequence of sample items. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+        CounterT*           d_output_histograms[NUM_ACTIVE_CHANNELS],  ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
+        int                 num_output_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of bin level boundaries for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
+        LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+        LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+        OffsetT             row_stride_samples,                         ///< [in] The number of samples between starts of consecutive rows in the region of interest
+        cudaStream_t        stream,                                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous,                          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+        Int2Type<false>     is_byte_sample)                             ///< [in] Marker type indicating whether or not SampleT is a 8b type
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel dispatch configurations
+            KernelConfig histogram_sweep_config;
+            if (CubDebug(error = InitConfigs(ptx_version, histogram_sweep_config)))
+                break;
+
+            // Use the scale transform op for converting samples to privatized bins
+            typedef ScaleTransform PrivatizedDecodeOpT;
+
+            // Use the pass-thru transform op for converting privatized bins to output bins
+            typedef PassThruTransform OutputDecodeOpT;
+
+            PrivatizedDecodeOpT         privatized_decode_op[NUM_ACTIVE_CHANNELS];
+            OutputDecodeOpT             output_decode_op[NUM_ACTIVE_CHANNELS];
+            int                         max_levels = num_output_levels[0];
+
+            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+            {
+                int     bins    = num_output_levels[channel] - 1;
+                LevelT  scale   = (upper_level[channel] - lower_level[channel]) / bins;
+
+                privatized_decode_op[channel].Init(num_output_levels[channel], upper_level[channel], lower_level[channel], scale);
+
+                if (num_output_levels[channel] > max_levels)
+                    max_levels = num_output_levels[channel];
+            }
+            int max_num_output_bins = max_levels - 1;
+
+            if (max_num_output_bins > MAX_PRIVATIZED_SMEM_BINS)
+            {
+                // Dispatch shared-privatized approach
+                const int PRIVATIZED_SMEM_BINS = 0;
+
+                if (CubDebug(error = PrivatizedDispatch(
+                    d_temp_storage,
+                    temp_storage_bytes,
+                    d_samples,
+                    d_output_histograms,
+                    num_output_levels,
+                    privatized_decode_op,
+                    num_output_levels,
+                    output_decode_op,
+                    max_num_output_bins,
+                    num_row_pixels,
+                    num_rows,
+                    row_stride_samples,
+                    DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
+                    DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
+                    histogram_sweep_config,
+                    stream,
+                    debug_synchronous))) break;
+            }
+            else
+            {
+                // Dispatch shared-privatized approach
+                const int PRIVATIZED_SMEM_BINS = MAX_PRIVATIZED_SMEM_BINS;
+
+                if (CubDebug(error = PrivatizedDispatch(
+                    d_temp_storage,
+                    temp_storage_bytes,
+                    d_samples,
+                    d_output_histograms,
+                    num_output_levels,
+                    privatized_decode_op,
+                    num_output_levels,
+                    output_decode_op,
+                    max_num_output_bins,
+                    num_row_pixels,
+                    num_rows,
+                    row_stride_samples,
+                    DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
+                    DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
+                    histogram_sweep_config,
+                    stream,
+                    debug_synchronous))) break;
+            }
+        }
+        while (0);
+
+        return error;
+    }
+
+
+    /**
+     * Dispatch routine for HistogramEven, specialized for 8-bit sample types (computes 256-bin privatized histograms and then reduces to user-specified levels)
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t DispatchEven(
+        void*               d_temp_storage,                            ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the input sequence of sample items. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+        CounterT*           d_output_histograms[NUM_ACTIVE_CHANNELS],  ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
+        int                 num_output_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of bin level boundaries for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
+        LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+        LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+        OffsetT             row_stride_samples,                         ///< [in] The number of samples between starts of consecutive rows in the region of interest
+        cudaStream_t        stream,                                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous,                          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+        Int2Type<true>      is_byte_sample)                             ///< [in] Marker type indicating whether or not SampleT is a 8b type
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel dispatch configurations
+            KernelConfig histogram_sweep_config;
+            if (CubDebug(error = InitConfigs(ptx_version, histogram_sweep_config)))
+                break;
+
+            // Use the pass-thru transform op for converting samples to privatized bins
+            typedef PassThruTransform PrivatizedDecodeOpT;
+
+            // Use the scale transform op for converting privatized bins to output bins
+            typedef ScaleTransform OutputDecodeOpT;
+
+            int                     num_privatized_levels[NUM_ACTIVE_CHANNELS];
+            PrivatizedDecodeOpT     privatized_decode_op[NUM_ACTIVE_CHANNELS];
+            OutputDecodeOpT         output_decode_op[NUM_ACTIVE_CHANNELS];
+            int                     max_levels = num_output_levels[0];
+
+            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+            {
+                num_privatized_levels[channel] = 257;
+
+                int     bins    = num_output_levels[channel] - 1;
+                LevelT  scale   = (upper_level[channel] - lower_level[channel]) / bins;
+                output_decode_op[channel].Init(num_output_levels[channel], upper_level[channel], lower_level[channel], scale);
+
+                if (num_output_levels[channel] > max_levels)
+                    max_levels = num_output_levels[channel];
+            }
+            int max_num_output_bins = max_levels - 1;
+
+            const int PRIVATIZED_SMEM_BINS = 256;
+
+            if (CubDebug(error = PrivatizedDispatch(
+                d_temp_storage,
+                temp_storage_bytes,
+                d_samples,
+                d_output_histograms,
+                num_privatized_levels,
+                privatized_decode_op,
+                num_output_levels,
+                output_decode_op,
+                max_num_output_bins,
+                num_row_pixels,
+                num_rows,
+                row_stride_samples,
+                DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
+                DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
+                histogram_sweep_config,
+                stream,
+                debug_synchronous))) break;
+
+        }
+        while (0);
+
+        return error;
+    }
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/dispatch/dispatch_radix_sort.cuh b/third_party/cub/cub/device/dispatch/dispatch_radix_sort.cuh
new file mode 100644
index 0000000..d1a992d
--- /dev/null
+++ b/third_party/cub/cub/device/dispatch/dispatch_radix_sort.cuh
@@ -0,0 +1,1619 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceRadixSort provides device-wide, parallel operations for computing a radix sort across a sequence of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "../../agent/agent_radix_sort_upsweep.cuh"
+#include "../../agent/agent_radix_sort_downsweep.cuh"
+#include "../../agent/agent_scan.cuh"
+#include "../../block/block_radix_sort.cuh"
+#include "../../grid/grid_even_share.cuh"
+#include "../../util_type.cuh"
+#include "../../util_debug.cuh"
+#include "../../util_device.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/******************************************************************************
+ * Kernel entry points
+ *****************************************************************************/
+
+/**
+ * Upsweep digit-counting kernel entry point (multi-block).  Computes privatized digit histograms, one per block.
+ */
+template <
+    typename                ChainedPolicyT,                 ///< Chained tuning policy
+    bool                    ALT_DIGIT_BITS,                 ///< Whether or not to use the alternate (lower-bits) policy
+    bool                    IS_DESCENDING,                  ///< Whether or not the sorted-order is high-to-low
+    typename                KeyT,                           ///< Key type
+    typename                OffsetT>                        ///< Signed integer type for global offsets
+__launch_bounds__ (int((ALT_DIGIT_BITS) ?
+    ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::BLOCK_THREADS :
+    ChainedPolicyT::ActivePolicy::UpsweepPolicy::BLOCK_THREADS))
+__global__ void DeviceRadixSortUpsweepKernel(
+    const KeyT              *d_keys,                        ///< [in] Input keys buffer
+    OffsetT                 *d_spine,                       ///< [out] Privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.)
+    OffsetT                 /*num_items*/,                  ///< [in] Total number of input data items
+    int                     current_bit,                    ///< [in] Bit position of current radix digit
+    int                     num_bits,                       ///< [in] Number of bits of current radix digit
+    GridEvenShare<OffsetT>  even_share)                     ///< [in] Even-share descriptor for mapan equal number of tiles onto each thread block
+{
+    enum {
+        TILE_ITEMS = ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::BLOCK_THREADS *
+                        ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::ITEMS_PER_THREAD
+    };
+
+    // Parameterize AgentRadixSortUpsweep type for the current configuration
+    typedef AgentRadixSortUpsweep<
+            typename If<(ALT_DIGIT_BITS),
+                typename ChainedPolicyT::ActivePolicy::AltUpsweepPolicy,
+                typename ChainedPolicyT::ActivePolicy::UpsweepPolicy>::Type,
+            KeyT,
+            OffsetT>
+        AgentRadixSortUpsweepT;
+
+    // Shared memory storage
+    __shared__ typename AgentRadixSortUpsweepT::TempStorage temp_storage;
+
+    // Initialize GRID_MAPPING_RAKE even-share descriptor for this thread block
+    even_share.template BlockInit<TILE_ITEMS, GRID_MAPPING_RAKE>();
+
+    AgentRadixSortUpsweepT upsweep(temp_storage, d_keys, current_bit, num_bits);
+
+    upsweep.ProcessRegion(even_share.block_offset, even_share.block_end);
+
+    CTA_SYNC();
+
+    // Write out digit counts (striped)
+    upsweep.template ExtractCounts<IS_DESCENDING>(d_spine, gridDim.x, blockIdx.x);
+}
+
+
+/**
+ * Spine scan kernel entry point (single-block).  Computes an exclusive prefix sum over the privatized digit histograms
+ */
+template <
+    typename                ChainedPolicyT,                 ///< Chained tuning policy
+    typename                OffsetT>                        ///< Signed integer type for global offsets
+__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::ScanPolicy::BLOCK_THREADS), 1)
+__global__ void RadixSortScanBinsKernel(
+    OffsetT                 *d_spine,                       ///< [in,out] Privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.)
+    int                     num_counts)                     ///< [in] Total number of bin-counts
+{
+    // Parameterize the AgentScan type for the current configuration
+    typedef AgentScan<
+            typename ChainedPolicyT::ActivePolicy::ScanPolicy,
+            OffsetT*,
+            OffsetT*,
+            cub::Sum,
+            OffsetT,
+            OffsetT>
+        AgentScanT;
+
+    // Shared memory storage
+    __shared__ typename AgentScanT::TempStorage temp_storage;
+
+    // Block scan instance
+    AgentScanT block_scan(temp_storage, d_spine, d_spine, cub::Sum(), OffsetT(0)) ;
+
+    // Process full input tiles
+    int block_offset = 0;
+    BlockScanRunningPrefixOp<OffsetT, Sum> prefix_op(0, Sum());
+    while (block_offset + AgentScanT::TILE_ITEMS <= num_counts)
+    {
+        block_scan.template ConsumeTile<false, false>(block_offset, prefix_op);
+        block_offset += AgentScanT::TILE_ITEMS;
+    }
+}
+
+
+/**
+ * Downsweep pass kernel entry point (multi-block).  Scatters keys (and values) into corresponding bins for the current digit place.
+ */
+template <
+    typename                ChainedPolicyT,                 ///< Chained tuning policy
+    bool                    ALT_DIGIT_BITS,                 ///< Whether or not to use the alternate (lower-bits) policy
+    bool                    IS_DESCENDING,                  ///< Whether or not the sorted-order is high-to-low
+    typename                KeyT,                           ///< Key type
+    typename                ValueT,                         ///< Value type
+    typename                OffsetT>                        ///< Signed integer type for global offsets
+__launch_bounds__ (int((ALT_DIGIT_BITS) ?
+    ChainedPolicyT::ActivePolicy::AltDownsweepPolicy::BLOCK_THREADS :
+    ChainedPolicyT::ActivePolicy::DownsweepPolicy::BLOCK_THREADS))
+__global__ void DeviceRadixSortDownsweepKernel(
+    const KeyT              *d_keys_in,                     ///< [in] Input keys buffer
+    KeyT                    *d_keys_out,                    ///< [in] Output keys buffer
+    const ValueT            *d_values_in,                   ///< [in] Input values buffer
+    ValueT                  *d_values_out,                  ///< [in] Output values buffer
+    OffsetT                 *d_spine,                       ///< [in] Scan of privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.)
+    OffsetT                 num_items,                      ///< [in] Total number of input data items
+    int                     current_bit,                    ///< [in] Bit position of current radix digit
+    int                     num_bits,                       ///< [in] Number of bits of current radix digit
+    GridEvenShare<OffsetT>  even_share)                     ///< [in] Even-share descriptor for mapan equal number of tiles onto each thread block
+{
+    enum {
+        TILE_ITEMS = ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::BLOCK_THREADS *
+                        ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::ITEMS_PER_THREAD
+    };
+
+    // Parameterize AgentRadixSortDownsweep type for the current configuration
+    typedef AgentRadixSortDownsweep<
+            typename If<(ALT_DIGIT_BITS),
+                typename ChainedPolicyT::ActivePolicy::AltDownsweepPolicy,
+                typename ChainedPolicyT::ActivePolicy::DownsweepPolicy>::Type,
+            IS_DESCENDING,
+            KeyT,
+            ValueT,
+            OffsetT>
+        AgentRadixSortDownsweepT;
+
+    // Shared memory storage
+    __shared__  typename AgentRadixSortDownsweepT::TempStorage temp_storage;
+
+    // Initialize even-share descriptor for this thread block
+    even_share.template BlockInit<TILE_ITEMS, GRID_MAPPING_RAKE>();
+
+    // Process input tiles
+    AgentRadixSortDownsweepT(temp_storage, num_items, d_spine, d_keys_in, d_keys_out, d_values_in, d_values_out, current_bit, num_bits).ProcessRegion(
+        even_share.block_offset,
+        even_share.block_end);
+}
+
+
+/**
+ * Single pass kernel entry point (single-block).  Fully sorts a tile of input.
+ */
+template <
+    typename                ChainedPolicyT,                 ///< Chained tuning policy
+    bool                    IS_DESCENDING,                  ///< Whether or not the sorted-order is high-to-low
+    typename                KeyT,                           ///< Key type
+    typename                ValueT,                         ///< Value type
+    typename                OffsetT>                        ///< Signed integer type for global offsets
+__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::SingleTilePolicy::BLOCK_THREADS), 1)
+__global__ void DeviceRadixSortSingleTileKernel(
+    const KeyT              *d_keys_in,                     ///< [in] Input keys buffer
+    KeyT                    *d_keys_out,                    ///< [in] Output keys buffer
+    const ValueT            *d_values_in,                   ///< [in] Input values buffer
+    ValueT                  *d_values_out,                  ///< [in] Output values buffer
+    OffsetT                 num_items,                      ///< [in] Total number of input data items
+    int                     current_bit,                    ///< [in] Bit position of current radix digit
+    int                     end_bit)                        ///< [in] The past-the-end (most-significant) bit index needed for key comparison
+{
+    // Constants
+    enum
+    {
+        BLOCK_THREADS           = ChainedPolicyT::ActivePolicy::SingleTilePolicy::BLOCK_THREADS,
+        ITEMS_PER_THREAD        = ChainedPolicyT::ActivePolicy::SingleTilePolicy::ITEMS_PER_THREAD,
+        KEYS_ONLY               = Equals<ValueT, NullType>::VALUE,
+    };
+
+    // BlockRadixSort type
+    typedef BlockRadixSort<
+            KeyT,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            ValueT,
+            ChainedPolicyT::ActivePolicy::SingleTilePolicy::RADIX_BITS,
+            (ChainedPolicyT::ActivePolicy::SingleTilePolicy::RANK_ALGORITHM == RADIX_RANK_MEMOIZE),
+            ChainedPolicyT::ActivePolicy::SingleTilePolicy::SCAN_ALGORITHM>
+        BlockRadixSortT;
+
+    // BlockLoad type (keys)
+    typedef BlockLoad<
+        KeyT,
+        BLOCK_THREADS,
+        ITEMS_PER_THREAD,
+        ChainedPolicyT::ActivePolicy::SingleTilePolicy::LOAD_ALGORITHM> BlockLoadKeys;
+
+    // BlockLoad type (values)
+    typedef BlockLoad<
+        ValueT,
+        BLOCK_THREADS,
+        ITEMS_PER_THREAD,
+        ChainedPolicyT::ActivePolicy::SingleTilePolicy::LOAD_ALGORITHM> BlockLoadValues;
+
+    // Unsigned word for key bits
+    typedef typename Traits<KeyT>::UnsignedBits UnsignedBitsT;
+
+    // Shared memory storage
+    __shared__ union TempStorage
+    {
+        typename BlockRadixSortT::TempStorage       sort;
+        typename BlockLoadKeys::TempStorage         load_keys;
+        typename BlockLoadValues::TempStorage       load_values;
+
+    } temp_storage;
+
+    // Keys and values for the block
+    KeyT            keys[ITEMS_PER_THREAD];
+    ValueT          values[ITEMS_PER_THREAD];
+
+    // Get default (min/max) value for out-of-bounds keys
+    UnsignedBitsT   default_key_bits = (IS_DESCENDING) ? Traits<KeyT>::LOWEST_KEY : Traits<KeyT>::MAX_KEY;
+    KeyT            default_key = reinterpret_cast<KeyT&>(default_key_bits);
+
+    // Load keys
+    BlockLoadKeys(temp_storage.load_keys).Load(d_keys_in, keys, num_items, default_key);
+
+    CTA_SYNC();
+
+    // Load values
+    if (!KEYS_ONLY)
+    {
+        // Register pressure work-around: moving num_items through shfl prevents compiler
+        // from reusing guards/addressing from prior guarded loads
+        num_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(num_items, 0, 0xffffffff);
+
+        BlockLoadValues(temp_storage.load_values).Load(d_values_in, values, num_items);
+
+        CTA_SYNC();
+    }
+
+    // Sort tile
+    BlockRadixSortT(temp_storage.sort).SortBlockedToStriped(
+        keys,
+        values,
+        current_bit,
+        end_bit,
+        Int2Type<IS_DESCENDING>(),
+        Int2Type<KEYS_ONLY>());
+
+    // Store keys and values
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+    {
+        int item_offset = ITEM * BLOCK_THREADS + threadIdx.x;
+        if (item_offset < num_items)
+        {
+            d_keys_out[item_offset] = keys[ITEM];
+            if (!KEYS_ONLY)
+                d_values_out[item_offset] = values[ITEM];
+        }
+    }
+}
+
+
+/**
+ * Segmented radix sorting pass (one block per segment)
+ */
+template <
+    typename                ChainedPolicyT,                 ///< Chained tuning policy
+    bool                    ALT_DIGIT_BITS,                 ///< Whether or not to use the alternate (lower-bits) policy
+    bool                    IS_DESCENDING,                  ///< Whether or not the sorted-order is high-to-low
+    typename                KeyT,                           ///< Key type
+    typename                ValueT,                         ///< Value type
+    typename                OffsetIteratorT,                ///< Random-access input iterator type for reading segment offsets \iterator
+    typename                OffsetT>                        ///< Signed integer type for global offsets
+__launch_bounds__ (int((ALT_DIGIT_BITS) ?
+    ChainedPolicyT::ActivePolicy::AltSegmentedPolicy::BLOCK_THREADS :
+    ChainedPolicyT::ActivePolicy::SegmentedPolicy::BLOCK_THREADS))
+__global__ void DeviceSegmentedRadixSortKernel(
+    const KeyT              *d_keys_in,                     ///< [in] Input keys buffer
+    KeyT                    *d_keys_out,                    ///< [in] Output keys buffer
+    const ValueT            *d_values_in,                   ///< [in] Input values buffer
+    ValueT                  *d_values_out,                  ///< [in] Output values buffer
+    OffsetIteratorT         d_begin_offsets,                ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+    OffsetIteratorT         d_end_offsets,                  ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+    int                     /*num_segments*/,               ///< [in] The number of segments that comprise the sorting data
+    int                     current_bit,                    ///< [in] Bit position of current radix digit
+    int                     pass_bits)                      ///< [in] Number of bits of current radix digit
+{
+    //
+    // Constants
+    //
+
+    typedef typename If<(ALT_DIGIT_BITS),
+        typename ChainedPolicyT::ActivePolicy::AltSegmentedPolicy,
+        typename ChainedPolicyT::ActivePolicy::SegmentedPolicy>::Type SegmentedPolicyT;
+
+    enum
+    {
+        BLOCK_THREADS       = SegmentedPolicyT::BLOCK_THREADS,
+        ITEMS_PER_THREAD    = SegmentedPolicyT::ITEMS_PER_THREAD,
+        RADIX_BITS          = SegmentedPolicyT::RADIX_BITS,
+        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
+        RADIX_DIGITS        = 1 << RADIX_BITS,
+        KEYS_ONLY           = Equals<ValueT, NullType>::VALUE,
+    };
+
+    // Upsweep type
+    typedef AgentRadixSortUpsweep<
+            AgentRadixSortUpsweepPolicy<BLOCK_THREADS, ITEMS_PER_THREAD, SegmentedPolicyT::LOAD_MODIFIER, RADIX_BITS>,
+            KeyT,
+            OffsetT>
+        BlockUpsweepT;
+
+    // Digit-scan type
+    typedef BlockScan<OffsetT, BLOCK_THREADS> DigitScanT;
+
+    // Downsweep type
+    typedef AgentRadixSortDownsweep<SegmentedPolicyT, IS_DESCENDING, KeyT, ValueT, OffsetT> BlockDownsweepT;
+
+    enum
+    {
+        /// Number of bin-starting offsets tracked per thread
+        BINS_TRACKED_PER_THREAD = BlockDownsweepT::BINS_TRACKED_PER_THREAD
+    };
+
+    //
+    // Process input tiles
+    //
+
+    // Shared memory storage
+    __shared__ union
+    {
+        typename BlockUpsweepT::TempStorage     upsweep;
+        typename BlockDownsweepT::TempStorage   downsweep;
+        struct
+        {
+            volatile OffsetT                        reverse_counts_in[RADIX_DIGITS];
+            volatile OffsetT                        reverse_counts_out[RADIX_DIGITS];
+            typename DigitScanT::TempStorage        scan;
+        };
+
+    } temp_storage;
+
+    OffsetT segment_begin   = d_begin_offsets[blockIdx.x];
+    OffsetT segment_end     = d_end_offsets[blockIdx.x];
+    OffsetT num_items       = segment_end - segment_begin;
+
+    // Check if empty segment
+    if (num_items <= 0)
+        return;
+
+    // Upsweep
+    BlockUpsweepT upsweep(temp_storage.upsweep, d_keys_in, current_bit, pass_bits);
+    upsweep.ProcessRegion(segment_begin, segment_end);
+
+    CTA_SYNC();
+
+    // The count of each digit value in this pass (valid in the first RADIX_DIGITS threads)
+    OffsetT bin_count[BINS_TRACKED_PER_THREAD];
+    upsweep.ExtractCounts(bin_count);
+
+    CTA_SYNC();
+
+    if (IS_DESCENDING)
+    {
+        // Reverse bin counts
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+                temp_storage.reverse_counts_in[bin_idx] = bin_count[track];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+                bin_count[track] = temp_storage.reverse_counts_in[RADIX_DIGITS - bin_idx - 1];
+        }
+    }
+
+    // Scan
+    OffsetT bin_offset[BINS_TRACKED_PER_THREAD];     // The global scatter base offset for each digit value in this pass (valid in the first RADIX_DIGITS threads)
+    DigitScanT(temp_storage.scan).ExclusiveSum(bin_count, bin_offset);
+
+    #pragma unroll
+    for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+    {
+        bin_offset[track] += segment_begin;
+    }
+
+    if (IS_DESCENDING)
+    {
+        // Reverse bin offsets
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+                temp_storage.reverse_counts_out[threadIdx.x] = bin_offset[track];
+        }
+
+        CTA_SYNC();
+
+        #pragma unroll
+        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
+        {
+            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
+
+            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
+                bin_offset[track] = temp_storage.reverse_counts_out[RADIX_DIGITS - bin_idx - 1];
+        }
+    }
+
+    CTA_SYNC();
+
+    // Downsweep
+    BlockDownsweepT downsweep(temp_storage.downsweep, bin_offset, num_items, d_keys_in, d_keys_out, d_values_in, d_values_out, current_bit, pass_bits);
+    downsweep.ProcessRegion(segment_begin, segment_end);
+}
+
+
+
+/******************************************************************************
+ * Policy
+ ******************************************************************************/
+
+/**
+ * Tuning policy for kernel specialization
+ */
+template <
+    typename KeyT,          ///< Key type
+    typename ValueT,        ///< Value type
+    typename OffsetT>       ///< Signed integer type for global offsets
+struct DeviceRadixSortPolicy
+{
+    //------------------------------------------------------------------------------
+    // Constants
+    //------------------------------------------------------------------------------
+
+    enum
+    {
+        // Whether this is a keys-only (or key-value) sort
+        KEYS_ONLY = (Equals<ValueT, NullType>::VALUE),
+    };
+
+    // Dominant-sized key/value type
+    typedef typename If<(sizeof(ValueT) > 4) && (sizeof(KeyT) < sizeof(ValueT)), ValueT, KeyT>::Type DominantT;
+
+    //------------------------------------------------------------------------------
+    // Architecture-specific tuning policies
+    //------------------------------------------------------------------------------
+
+    /// SM20
+    struct Policy200 : ChainedPolicy<200, Policy200, Policy200>
+    {
+        enum {
+            PRIMARY_RADIX_BITS      = 5,
+            ALT_RADIX_BITS          = PRIMARY_RADIX_BITS - 1,
+
+            // Relative size of KeyT type to a 4-byte word
+            SCALE_FACTOR_4B = (CUB_MAX(sizeof(KeyT), sizeof(ValueT)) + 3) / 4,
+        };
+
+        // Keys-only upsweep policies
+        typedef AgentRadixSortUpsweepPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR_4B), LOAD_DEFAULT, PRIMARY_RADIX_BITS>    UpsweepPolicyKeys;
+        typedef AgentRadixSortUpsweepPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR_4B), LOAD_DEFAULT, ALT_RADIX_BITS>        AltUpsweepPolicyKeys;
+
+        // Key-value pairs upsweep policies
+        typedef AgentRadixSortUpsweepPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR_4B), LOAD_DEFAULT, PRIMARY_RADIX_BITS>   UpsweepPolicyPairs;
+        typedef AgentRadixSortUpsweepPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR_4B), LOAD_DEFAULT, ALT_RADIX_BITS>       AltUpsweepPolicyPairs;
+
+        // Upsweep policies
+        typedef typename If<KEYS_ONLY, UpsweepPolicyKeys, UpsweepPolicyPairs>::Type         UpsweepPolicy;
+        typedef typename If<KEYS_ONLY, AltUpsweepPolicyKeys, AltUpsweepPolicyPairs>::Type   AltUpsweepPolicy;
+
+        // Scan policy
+        typedef AgentScanPolicy <512, 4, BLOCK_LOAD_VECTORIZE, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
+
+        // Keys-only downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>    DownsweepPolicyKeys;
+        typedef AgentRadixSortDownsweepPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, ALT_RADIX_BITS>        AltDownsweepPolicyKeys;
+
+        // Key-value pairs downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>   DownsweepPolicyPairs;
+        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, ALT_RADIX_BITS>       AltDownsweepPolicyPairs;
+
+        // Downsweep policies
+        typedef typename If<KEYS_ONLY, DownsweepPolicyKeys, DownsweepPolicyPairs>::Type         DownsweepPolicy;
+        typedef typename If<KEYS_ONLY, AltDownsweepPolicyKeys, AltDownsweepPolicyPairs>::Type   AltDownsweepPolicy;
+
+        // Single-tile policy
+        typedef DownsweepPolicy SingleTilePolicy;
+
+        // Segmented policies
+        typedef DownsweepPolicy     SegmentedPolicy;
+        typedef AltDownsweepPolicy  AltSegmentedPolicy;
+    };
+
+    /// SM30
+    struct Policy300 : ChainedPolicy<300, Policy300, Policy200>
+    {
+        enum {
+            PRIMARY_RADIX_BITS      = 5,
+            ALT_RADIX_BITS          = PRIMARY_RADIX_BITS - 1,
+
+            // Relative size of KeyT type to a 4-byte word
+            SCALE_FACTOR_4B = (CUB_MAX(sizeof(KeyT), sizeof(ValueT)) + 3) / 4,
+        };
+
+        // Keys-only upsweep policies
+        typedef AgentRadixSortUpsweepPolicy <256, CUB_MAX(1, 7 / SCALE_FACTOR_4B), LOAD_DEFAULT, PRIMARY_RADIX_BITS>    UpsweepPolicyKeys;
+        typedef AgentRadixSortUpsweepPolicy <256, CUB_MAX(1, 7 / SCALE_FACTOR_4B), LOAD_DEFAULT, ALT_RADIX_BITS>        AltUpsweepPolicyKeys;
+
+        // Key-value pairs upsweep policies
+        typedef AgentRadixSortUpsweepPolicy <256, CUB_MAX(1, 5 / SCALE_FACTOR_4B), LOAD_DEFAULT, PRIMARY_RADIX_BITS>    UpsweepPolicyPairs;
+        typedef AgentRadixSortUpsweepPolicy <256, CUB_MAX(1, 5 / SCALE_FACTOR_4B), LOAD_DEFAULT, ALT_RADIX_BITS>        AltUpsweepPolicyPairs;
+
+        // Upsweep policies
+        typedef typename If<KEYS_ONLY, UpsweepPolicyKeys, UpsweepPolicyPairs>::Type         UpsweepPolicy;
+        typedef typename If<KEYS_ONLY, AltUpsweepPolicyKeys, AltUpsweepPolicyPairs>::Type   AltUpsweepPolicy;
+
+        // Scan policy
+        typedef AgentScanPolicy <1024, 4, BLOCK_LOAD_VECTORIZE, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, BLOCK_SCAN_WARP_SCANS> ScanPolicy;
+
+        // Keys-only downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 14 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>   DownsweepPolicyKeys;
+        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 14 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, ALT_RADIX_BITS>       AltDownsweepPolicyKeys;
+
+        // Key-value pairs downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 10 / SCALE_FACTOR_4B), BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>    DownsweepPolicyPairs;
+        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 10 / SCALE_FACTOR_4B), BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, ALT_RADIX_BITS>        AltDownsweepPolicyPairs;
+
+        // Downsweep policies
+        typedef typename If<KEYS_ONLY, DownsweepPolicyKeys, DownsweepPolicyPairs>::Type         DownsweepPolicy;
+        typedef typename If<KEYS_ONLY, AltDownsweepPolicyKeys, AltDownsweepPolicyPairs>::Type   AltDownsweepPolicy;
+
+        // Single-tile policy
+        typedef DownsweepPolicy SingleTilePolicy;
+
+        // Segmented policies
+        typedef DownsweepPolicy     SegmentedPolicy;
+        typedef AltDownsweepPolicy  AltSegmentedPolicy;
+    };
+
+
+    /// SM35
+    struct Policy350 : ChainedPolicy<350, Policy350, Policy300>
+    {
+        enum {
+            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 6 : 5,    // 1.72B 32b keys/s, 1.17B 32b pairs/s, 1.55B 32b segmented keys/s (K40m)
+        };
+
+        // Scan policy
+        typedef AgentScanPolicy <1024, 4, BLOCK_LOAD_VECTORIZE, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, BLOCK_SCAN_WARP_SCANS> ScanPolicy;
+
+        // Keys-only downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(128, 9, DominantT), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_LDG, RADIX_RANK_MATCH, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS> DownsweepPolicyKeys;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(64, 18, DominantT), BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS - 1> AltDownsweepPolicyKeys;
+
+        // Key-value pairs downsweep policies
+        typedef DownsweepPolicyKeys DownsweepPolicyPairs;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(128, 15, DominantT), BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS - 1> AltDownsweepPolicyPairs;
+
+        // Downsweep policies
+        typedef typename If<KEYS_ONLY, DownsweepPolicyKeys, DownsweepPolicyPairs>::Type DownsweepPolicy;
+        typedef typename If<KEYS_ONLY, AltDownsweepPolicyKeys, AltDownsweepPolicyPairs>::Type AltDownsweepPolicy;
+
+        // Upsweep policies
+        typedef DownsweepPolicy UpsweepPolicy;
+        typedef AltDownsweepPolicy AltUpsweepPolicy;
+
+        // Single-tile policy
+        typedef DownsweepPolicy SingleTilePolicy;
+
+        // Segmented policies
+        typedef DownsweepPolicy     SegmentedPolicy;
+        typedef AltDownsweepPolicy  AltSegmentedPolicy;
+
+
+    };
+
+
+    /// SM50
+    struct Policy500 : ChainedPolicy<500, Policy500, Policy350>
+    {
+        enum {
+            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 7 : 5,    // 3.5B 32b keys/s, 1.92B 32b pairs/s (TitanX)
+            SINGLE_TILE_RADIX_BITS  = (sizeof(KeyT) > 1) ? 6 : 5,
+            SEGMENTED_RADIX_BITS    = (sizeof(KeyT) > 1) ? 6 : 5,    // 3.1B 32b segmented keys/s (TitanX)
+        };
+
+        // ScanPolicy
+        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
+
+        // Downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(160, 39, DominantT),  BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>  DownsweepPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 16, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_RAKING_MEMOIZE, PRIMARY_RADIX_BITS - 1>   AltDownsweepPolicy;
+
+        // Upsweep policies
+        typedef DownsweepPolicy UpsweepPolicy;
+        typedef AltDownsweepPolicy AltUpsweepPolicy;
+
+        // Single-tile policy
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SINGLE_TILE_RADIX_BITS> SingleTilePolicy;
+
+        // Segmented policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 31, DominantT),  BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS>   SegmentedPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 11, DominantT),  BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS - 1>       AltSegmentedPolicy;
+    };
+
+
+    /// SM60 (GP100)
+    struct Policy600 : ChainedPolicy<600, Policy600, Policy500>
+    {
+        enum {
+            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 7 : 5,    // 6.9B 32b keys/s (Quadro P100)
+            SINGLE_TILE_RADIX_BITS  = (sizeof(KeyT) > 1) ? 6 : 5,
+            SEGMENTED_RADIX_BITS    = (sizeof(KeyT) > 1) ? 6 : 5,    // 5.9B 32b segmented keys/s (Quadro P100)
+        };
+
+        // ScanPolicy
+        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
+
+        // Downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 25, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MATCH, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>   DownsweepPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 39, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS - 1>   AltDownsweepPolicy;
+
+        // Upsweep policies
+        typedef DownsweepPolicy UpsweepPolicy;
+        typedef AltDownsweepPolicy AltUpsweepPolicy;
+
+        // Single-tile policy
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SINGLE_TILE_RADIX_BITS>          SingleTilePolicy;
+
+        // Segmented policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 39, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS>     SegmentedPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(384, 11, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS - 1> AltSegmentedPolicy;
+
+    };
+
+
+    /// SM61 (GP104)
+    struct Policy610 : ChainedPolicy<610, Policy610, Policy600>
+    {
+        enum {
+            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 7 : 5,    // 3.4B 32b keys/s, 1.83B 32b pairs/s (1080)
+            SINGLE_TILE_RADIX_BITS  = (sizeof(KeyT) > 1) ? 6 : 5,
+            SEGMENTED_RADIX_BITS    = (sizeof(KeyT) > 1) ? 6 : 5,    // 3.3B 32b segmented keys/s (1080)
+        };
+
+        // ScanPolicy
+        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
+
+        // Downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(384, 31, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MATCH, BLOCK_SCAN_RAKING_MEMOIZE, PRIMARY_RADIX_BITS>   DownsweepPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 35, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_RAKING_MEMOIZE, PRIMARY_RADIX_BITS - 1>   AltDownsweepPolicy;
+
+        // Upsweep policies
+        typedef AgentRadixSortUpsweepPolicy <CUB_SCALED_GRANULARITIES(128, 16, DominantT), LOAD_LDG, PRIMARY_RADIX_BITS>        UpsweepPolicy;
+        typedef AgentRadixSortUpsweepPolicy <CUB_SCALED_GRANULARITIES(128, 16, DominantT), LOAD_LDG, PRIMARY_RADIX_BITS - 1>    AltUpsweepPolicy;
+
+        // Single-tile policy
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SINGLE_TILE_RADIX_BITS>          SingleTilePolicy;
+
+        // Segmented policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 39, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS>     SegmentedPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(384, 11, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS - 1> AltSegmentedPolicy;
+    };
+
+
+    /// SM62 (Tegra, less RF)
+    struct Policy620 : ChainedPolicy<620, Policy620, Policy610>
+    {
+        enum {
+            PRIMARY_RADIX_BITS      = 5,
+            ALT_RADIX_BITS          = PRIMARY_RADIX_BITS - 1,
+        };
+
+        // ScanPolicy
+        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
+
+        // Downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 16, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_RAKING_MEMOIZE, PRIMARY_RADIX_BITS>   DownsweepPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 16, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_RAKING_MEMOIZE, ALT_RADIX_BITS>       AltDownsweepPolicy;
+
+        // Upsweep policies
+        typedef DownsweepPolicy UpsweepPolicy;
+        typedef AltDownsweepPolicy AltUpsweepPolicy;
+
+        // Single-tile policy
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS> SingleTilePolicy;
+
+        // Segmented policies
+        typedef DownsweepPolicy     SegmentedPolicy;
+        typedef AltDownsweepPolicy  AltSegmentedPolicy;
+    };
+
+
+    /// SM70 (GV100)
+    struct Policy700 : ChainedPolicy<700, Policy700, Policy620>
+    {
+        enum {
+            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 7 : 5,    // 7.62B 32b keys/s (GV100)
+            SINGLE_TILE_RADIX_BITS  = (sizeof(KeyT) > 1) ? 6 : 5,
+            SEGMENTED_RADIX_BITS    = (sizeof(KeyT) > 1) ? 6 : 5,    // 8.7B 32b segmented keys/s (GV100)
+        };
+
+        // ScanPolicy
+        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
+
+        // Downsweep policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 25, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MATCH, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>   DownsweepPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 25, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS - 1>   AltDownsweepPolicy;
+
+        // Upsweep policies
+        typedef DownsweepPolicy UpsweepPolicy;
+        typedef AltDownsweepPolicy AltUpsweepPolicy;
+
+        // Single-tile policy
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SINGLE_TILE_RADIX_BITS>          SingleTilePolicy;
+
+        // Segmented policies
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 39, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS>     SegmentedPolicy;
+        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(384, 11, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS - 1> AltSegmentedPolicy;
+    };
+
+
+    /// MaxPolicy
+    typedef Policy700 MaxPolicy;
+
+
+};
+
+
+
+/******************************************************************************
+ * Single-problem dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for device-wide radix sort
+ */
+template <
+    bool     IS_DESCENDING, ///< Whether or not the sorted-order is high-to-low
+    typename KeyT,          ///< Key type
+    typename ValueT,        ///< Value type
+    typename OffsetT>       ///< Signed integer type for global offsets
+struct DispatchRadixSort :
+    DeviceRadixSortPolicy<KeyT, ValueT, OffsetT>
+{
+    //------------------------------------------------------------------------------
+    // Constants
+    //------------------------------------------------------------------------------
+
+    enum
+    {
+        // Whether this is a keys-only (or key-value) sort
+        KEYS_ONLY = (Equals<ValueT, NullType>::VALUE),
+    };
+
+
+    //------------------------------------------------------------------------------
+    // Problem state
+    //------------------------------------------------------------------------------
+
+    void                    *d_temp_storage;        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+    size_t                  &temp_storage_bytes;    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+    DoubleBuffer<KeyT>      &d_keys;                ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+    DoubleBuffer<ValueT>    &d_values;              ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
+    OffsetT                 num_items;              ///< [in] Number of items to sort
+    int                     begin_bit;              ///< [in] The beginning (least-significant) bit index needed for key comparison
+    int                     end_bit;                ///< [in] The past-the-end (most-significant) bit index needed for key comparison
+    cudaStream_t            stream;                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+    bool                    debug_synchronous;      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    int                     ptx_version;            ///< [in] PTX version
+    bool                    is_overwrite_okay;      ///< [in] Whether is okay to overwrite source buffers
+
+
+    //------------------------------------------------------------------------------
+    // Constructor
+    //------------------------------------------------------------------------------
+
+    /// Constructor
+    CUB_RUNTIME_FUNCTION __forceinline__
+    DispatchRadixSort(
+        void*                   d_temp_storage,
+        size_t                  &temp_storage_bytes,
+        DoubleBuffer<KeyT>      &d_keys,
+        DoubleBuffer<ValueT>    &d_values,
+        OffsetT                 num_items,
+        int                     begin_bit,
+        int                     end_bit,
+        bool                    is_overwrite_okay,
+        cudaStream_t            stream,
+        bool                    debug_synchronous,
+        int                     ptx_version)
+    :
+        d_temp_storage(d_temp_storage),
+        temp_storage_bytes(temp_storage_bytes),
+        d_keys(d_keys),
+        d_values(d_values),
+        num_items(num_items),
+        begin_bit(begin_bit),
+        end_bit(end_bit),
+        stream(stream),
+        debug_synchronous(debug_synchronous),
+        ptx_version(ptx_version),
+        is_overwrite_okay(is_overwrite_okay)
+    {}
+
+
+    //------------------------------------------------------------------------------
+    // Small-problem (single tile) invocation
+    //------------------------------------------------------------------------------
+
+    /// Invoke a single block to sort in-core
+    template <
+        typename                ActivePolicyT,          ///< Umbrella policy active for the target device
+        typename                SingleTileKernelT>      ///< Function type of cub::DeviceRadixSortSingleTileKernel
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t InvokeSingleTile(
+        SingleTileKernelT       single_tile_kernel)     ///< [in] Kernel function pointer to parameterization of cub::DeviceRadixSortSingleTileKernel
+    {
+#ifndef CUB_RUNTIME_ENABLED
+        (void)single_tile_kernel;
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported );
+#else
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Return if the caller is simply requesting the size of the storage allocation
+            if (d_temp_storage == NULL)
+            {
+                temp_storage_bytes = 1;
+                break;
+            }
+
+            // Return if empty problem
+            if (num_items == 0)
+                break;
+
+            // Log single_tile_kernel configuration
+            if (debug_synchronous)
+                _CubLog("Invoking single_tile_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy, current bit %d, bit_grain %d\n",
+                    1, ActivePolicyT::SingleTilePolicy::BLOCK_THREADS, (long long) stream,
+                    ActivePolicyT::SingleTilePolicy::ITEMS_PER_THREAD, 1, begin_bit, ActivePolicyT::SingleTilePolicy::RADIX_BITS);
+
+            // Invoke upsweep_kernel with same grid size as downsweep_kernel
+            single_tile_kernel<<<1, ActivePolicyT::SingleTilePolicy::BLOCK_THREADS, 0, stream>>>(
+                d_keys.Current(),
+                d_keys.Alternate(),
+                d_values.Current(),
+                d_values.Alternate(),
+                num_items,
+                begin_bit,
+                end_bit);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Update selector
+            d_keys.selector ^= 1;
+            d_values.selector ^= 1;
+        }
+        while (0);
+
+        return error;
+
+#endif // CUB_RUNTIME_ENABLED
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Normal problem size invocation
+    //------------------------------------------------------------------------------
+
+    /**
+     * Invoke a three-kernel sorting pass at the current bit.
+     */
+    template <typename PassConfigT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t InvokePass(
+        const KeyT      *d_keys_in,
+        KeyT            *d_keys_out,
+        const ValueT    *d_values_in,
+        ValueT          *d_values_out,
+        OffsetT         *d_spine,
+        int             spine_length,
+        int             &current_bit,
+        PassConfigT     &pass_config)
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            int pass_bits = CUB_MIN(pass_config.radix_bits, (end_bit - current_bit));
+
+            // Log upsweep_kernel configuration
+            if (debug_synchronous)
+                _CubLog("Invoking upsweep_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy, current bit %d, bit_grain %d\n",
+                pass_config.even_share.grid_size, pass_config.upsweep_config.block_threads, (long long) stream,
+                pass_config.upsweep_config.items_per_thread, pass_config.upsweep_config.sm_occupancy, current_bit, pass_bits);
+
+            // Invoke upsweep_kernel with same grid size as downsweep_kernel
+            pass_config.upsweep_kernel<<<pass_config.even_share.grid_size, pass_config.upsweep_config.block_threads, 0, stream>>>(
+                d_keys_in,
+                d_spine,
+                num_items,
+                current_bit,
+                pass_bits,
+                pass_config.even_share);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Log scan_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking scan_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread\n",
+                1, pass_config.scan_config.block_threads, (long long) stream, pass_config.scan_config.items_per_thread);
+
+            // Invoke scan_kernel
+            pass_config.scan_kernel<<<1, pass_config.scan_config.block_threads, 0, stream>>>(
+                d_spine,
+                spine_length);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Log downsweep_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking downsweep_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                pass_config.even_share.grid_size, pass_config.downsweep_config.block_threads, (long long) stream,
+                pass_config.downsweep_config.items_per_thread, pass_config.downsweep_config.sm_occupancy);
+
+            // Invoke downsweep_kernel
+            pass_config.downsweep_kernel<<<pass_config.even_share.grid_size, pass_config.downsweep_config.block_threads, 0, stream>>>(
+                d_keys_in,
+                d_keys_out,
+                d_values_in,
+                d_values_out,
+                d_spine,
+                num_items,
+                current_bit,
+                pass_bits,
+                pass_config.even_share);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Update current bit
+            current_bit += pass_bits;
+        }
+        while (0);
+
+        return error;
+    }
+
+
+
+    /// Pass configuration structure
+    template <
+        typename UpsweepKernelT,
+        typename ScanKernelT,
+        typename DownsweepKernelT>
+    struct PassConfig
+    {
+        UpsweepKernelT          upsweep_kernel;
+        KernelConfig            upsweep_config;
+        ScanKernelT             scan_kernel;
+        KernelConfig            scan_config;
+        DownsweepKernelT        downsweep_kernel;
+        KernelConfig            downsweep_config;
+        int                     radix_bits;
+        int                     radix_digits;
+        int                     max_downsweep_grid_size;
+        GridEvenShare<OffsetT>  even_share;
+
+        /// Initialize pass configuration
+        template <
+            typename UpsweepPolicyT,
+            typename ScanPolicyT,
+            typename DownsweepPolicyT>
+        CUB_RUNTIME_FUNCTION __forceinline__
+        cudaError_t InitPassConfig(
+            UpsweepKernelT      upsweep_kernel,
+            ScanKernelT         scan_kernel,
+            DownsweepKernelT    downsweep_kernel,
+            int                 ptx_version,
+            int                 sm_count,
+            int                 num_items)
+        {
+            cudaError error = cudaSuccess;
+            do
+            {
+                this->upsweep_kernel    = upsweep_kernel;
+                this->scan_kernel       = scan_kernel;
+                this->downsweep_kernel  = downsweep_kernel;
+                radix_bits              = DownsweepPolicyT::RADIX_BITS;
+                radix_digits            = 1 << radix_bits;
+
+                if (CubDebug(error = upsweep_config.Init<UpsweepPolicyT>(upsweep_kernel))) break;
+                if (CubDebug(error = scan_config.Init<ScanPolicyT>(scan_kernel))) break;
+                if (CubDebug(error = downsweep_config.Init<DownsweepPolicyT>(downsweep_kernel))) break;
+
+                max_downsweep_grid_size = (downsweep_config.sm_occupancy * sm_count) * CUB_SUBSCRIPTION_FACTOR(ptx_version);
+
+                even_share.DispatchInit(
+                    num_items,
+                    max_downsweep_grid_size,
+                    CUB_MAX(downsweep_config.tile_size, upsweep_config.tile_size));
+
+            }
+            while (0);
+            return error;
+        }
+
+    };
+
+
+    /// Invocation (run multiple digit passes)
+    template <
+        typename            ActivePolicyT,          ///< Umbrella policy active for the target device
+        typename            UpsweepKernelT,         ///< Function type of cub::DeviceRadixSortUpsweepKernel
+        typename            ScanKernelT,            ///< Function type of cub::SpineScanKernel
+        typename            DownsweepKernelT>       ///< Function type of cub::DeviceRadixSortDownsweepKernel
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t InvokePasses(
+        UpsweepKernelT      upsweep_kernel,         ///< [in] Kernel function pointer to parameterization of cub::DeviceRadixSortUpsweepKernel
+        UpsweepKernelT      alt_upsweep_kernel,     ///< [in] Alternate kernel function pointer to parameterization of cub::DeviceRadixSortUpsweepKernel
+        ScanKernelT         scan_kernel,            ///< [in] Kernel function pointer to parameterization of cub::SpineScanKernel
+        DownsweepKernelT    downsweep_kernel,       ///< [in] Kernel function pointer to parameterization of cub::DeviceRadixSortDownsweepKernel
+        DownsweepKernelT    alt_downsweep_kernel)   ///< [in] Alternate kernel function pointer to parameterization of cub::DeviceRadixSortDownsweepKernel
+    {
+#ifndef CUB_RUNTIME_ENABLED
+        (void)upsweep_kernel;
+        (void)alt_upsweep_kernel;
+        (void)scan_kernel;
+        (void)downsweep_kernel;
+        (void)alt_downsweep_kernel;
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported );
+#else
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Init regular and alternate-digit kernel configurations
+            PassConfig<UpsweepKernelT, ScanKernelT, DownsweepKernelT> pass_config, alt_pass_config;
+            if ((error = pass_config.template InitPassConfig<
+                    typename ActivePolicyT::UpsweepPolicy, 
+                    typename ActivePolicyT::ScanPolicy, 
+                    typename ActivePolicyT::DownsweepPolicy>(
+                upsweep_kernel, scan_kernel, downsweep_kernel, ptx_version, sm_count, num_items))) break;
+
+            if ((error = alt_pass_config.template InitPassConfig<
+                    typename ActivePolicyT::AltUpsweepPolicy, 
+                    typename ActivePolicyT::ScanPolicy, 
+                    typename ActivePolicyT::AltDownsweepPolicy>(
+                alt_upsweep_kernel, scan_kernel, alt_downsweep_kernel, ptx_version, sm_count, num_items))) break;
+
+            // Get maximum spine length
+            int max_grid_size       = CUB_MAX(pass_config.max_downsweep_grid_size, alt_pass_config.max_downsweep_grid_size);
+            int spine_length        = (max_grid_size * pass_config.radix_digits) + pass_config.scan_config.tile_size;
+
+            // Temporary storage allocation requirements
+            void* allocations[3];
+            size_t allocation_sizes[3] =
+            {
+                spine_length * sizeof(OffsetT),                                         // bytes needed for privatized block digit histograms
+                (is_overwrite_okay) ? 0 : num_items * sizeof(KeyT),                     // bytes needed for 3rd keys buffer
+                (is_overwrite_okay || (KEYS_ONLY)) ? 0 : num_items * sizeof(ValueT),    // bytes needed for 3rd values buffer
+            };
+
+            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+
+            // Return if the caller is simply requesting the size of the storage allocation
+            if (d_temp_storage == NULL)
+                return cudaSuccess;
+
+            // Pass planning.  Run passes of the alternate digit-size configuration until we have an even multiple of our preferred digit size
+            int num_bits            = end_bit - begin_bit;
+            int num_passes          = (num_bits + pass_config.radix_bits - 1) / pass_config.radix_bits;
+            bool is_num_passes_odd  = num_passes & 1;
+            int max_alt_passes      = (num_passes * pass_config.radix_bits) - num_bits;
+            int alt_end_bit         = CUB_MIN(end_bit, begin_bit + (max_alt_passes * alt_pass_config.radix_bits));
+
+            // Alias the temporary storage allocations
+            OffsetT *d_spine = static_cast<OffsetT*>(allocations[0]);
+
+            DoubleBuffer<KeyT> d_keys_remaining_passes(
+                (is_overwrite_okay || is_num_passes_odd) ? d_keys.Alternate() : static_cast<KeyT*>(allocations[1]),
+                (is_overwrite_okay) ? d_keys.Current() : (is_num_passes_odd) ? static_cast<KeyT*>(allocations[1]) : d_keys.Alternate());
+
+            DoubleBuffer<ValueT> d_values_remaining_passes(
+                (is_overwrite_okay || is_num_passes_odd) ? d_values.Alternate() : static_cast<ValueT*>(allocations[2]),
+                (is_overwrite_okay) ? d_values.Current() : (is_num_passes_odd) ? static_cast<ValueT*>(allocations[2]) : d_values.Alternate());
+
+            // Run first pass, consuming from the input's current buffers
+            int current_bit = begin_bit;
+            if (CubDebug(error = InvokePass(
+                d_keys.Current(), d_keys_remaining_passes.Current(),
+                d_values.Current(), d_values_remaining_passes.Current(),
+                d_spine, spine_length, current_bit,
+                (current_bit < alt_end_bit) ? alt_pass_config : pass_config))) break;
+
+            // Run remaining passes
+            while (current_bit < end_bit)
+            {
+                if (CubDebug(error = InvokePass(
+                    d_keys_remaining_passes.d_buffers[d_keys_remaining_passes.selector],    d_keys_remaining_passes.d_buffers[d_keys_remaining_passes.selector ^ 1],
+                    d_values_remaining_passes.d_buffers[d_keys_remaining_passes.selector],  d_values_remaining_passes.d_buffers[d_keys_remaining_passes.selector ^ 1],
+                    d_spine, spine_length, current_bit,
+                    (current_bit < alt_end_bit) ? alt_pass_config : pass_config))) break;;
+
+                // Invert selectors
+                d_keys_remaining_passes.selector ^= 1;
+                d_values_remaining_passes.selector ^= 1;
+            }
+
+            // Update selector
+            if (!is_overwrite_okay) {
+                num_passes = 1; // Sorted data always ends up in the other vector
+            }
+
+            d_keys.selector = (d_keys.selector + num_passes) & 1;
+            d_values.selector = (d_values.selector + num_passes) & 1;
+        }
+        while (0);
+
+        return error;
+
+#endif // CUB_RUNTIME_ENABLED
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Chained policy invocation
+    //------------------------------------------------------------------------------
+
+    /// Invocation
+    template <typename ActivePolicyT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t Invoke()
+    {
+        typedef typename DispatchRadixSort::MaxPolicy       MaxPolicyT;
+        typedef typename ActivePolicyT::SingleTilePolicy    SingleTilePolicyT;
+
+        // Force kernel code-generation in all compiler passes
+        if (num_items <= (SingleTilePolicyT::BLOCK_THREADS * SingleTilePolicyT::ITEMS_PER_THREAD))
+        {
+            // Small, single tile size
+            return InvokeSingleTile<ActivePolicyT>(
+                DeviceRadixSortSingleTileKernel<MaxPolicyT, IS_DESCENDING, KeyT, ValueT, OffsetT>);
+        }
+        else
+        {
+            // Regular size
+            return InvokePasses<ActivePolicyT>(
+                DeviceRadixSortUpsweepKernel<   MaxPolicyT, false,   IS_DESCENDING, KeyT, OffsetT>,
+                DeviceRadixSortUpsweepKernel<   MaxPolicyT, true,    IS_DESCENDING, KeyT, OffsetT>,
+                RadixSortScanBinsKernel<        MaxPolicyT, OffsetT>,
+                DeviceRadixSortDownsweepKernel< MaxPolicyT, false,   IS_DESCENDING, KeyT, ValueT, OffsetT>,
+                DeviceRadixSortDownsweepKernel< MaxPolicyT, true,    IS_DESCENDING, KeyT, ValueT, OffsetT>);
+        }
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Dispatch entrypoints
+    //------------------------------------------------------------------------------
+
+    /**
+     * Internal dispatch routine
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                   d_temp_storage,         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>      &d_keys,                ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        DoubleBuffer<ValueT>    &d_values,              ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
+        OffsetT                 num_items,              ///< [in] Number of items to sort
+        int                     begin_bit,              ///< [in] The beginning (least-significant) bit index needed for key comparison
+        int                     end_bit,                ///< [in] The past-the-end (most-significant) bit index needed for key comparison
+        bool                    is_overwrite_okay,      ///< [in] Whether is okay to overwrite source buffers
+        cudaStream_t            stream,                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous)      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        typedef typename DispatchRadixSort::MaxPolicy MaxPolicyT;
+
+        cudaError_t error;
+        do {
+            // Get PTX version
+            int ptx_version;
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+
+            // Create dispatch functor
+            DispatchRadixSort dispatch(
+                d_temp_storage, temp_storage_bytes,
+                d_keys, d_values,
+                num_items, begin_bit, end_bit, is_overwrite_okay,
+                stream, debug_synchronous, ptx_version);
+
+            // Dispatch to chained policy
+            if (CubDebug(error = MaxPolicyT::Invoke(ptx_version, dispatch))) break;
+
+        } while (0);
+
+        return error;
+    }
+};
+
+
+
+
+/******************************************************************************
+ * Segmented dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for segmented device-wide radix sort
+ */
+template <
+    bool     IS_DESCENDING,     ///< Whether or not the sorted-order is high-to-low
+    typename KeyT,              ///< Key type
+    typename ValueT,            ///< Value type
+    typename OffsetIteratorT,   ///< Random-access input iterator type for reading segment offsets \iterator
+    typename OffsetT>           ///< Signed integer type for global offsets
+struct DispatchSegmentedRadixSort :
+    DeviceRadixSortPolicy<KeyT, ValueT, OffsetT>
+{
+    //------------------------------------------------------------------------------
+    // Constants
+    //------------------------------------------------------------------------------
+
+    enum
+    {
+        // Whether this is a keys-only (or key-value) sort
+        KEYS_ONLY = (Equals<ValueT, NullType>::VALUE),
+    };
+
+
+    //------------------------------------------------------------------------------
+    // Parameter members
+    //------------------------------------------------------------------------------
+
+    void                    *d_temp_storage;        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+    size_t                  &temp_storage_bytes;    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+    DoubleBuffer<KeyT>      &d_keys;                ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+    DoubleBuffer<ValueT>    &d_values;              ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
+    OffsetT                 num_items;              ///< [in] Number of items to sort
+    OffsetT                 num_segments;           ///< [in] The number of segments that comprise the sorting data
+    OffsetIteratorT         d_begin_offsets;        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+    OffsetIteratorT         d_end_offsets;          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+    int                     begin_bit;              ///< [in] The beginning (least-significant) bit index needed for key comparison
+    int                     end_bit;                ///< [in] The past-the-end (most-significant) bit index needed for key comparison
+    cudaStream_t            stream;                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+    bool                    debug_synchronous;      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    int                     ptx_version;            ///< [in] PTX version
+    bool                    is_overwrite_okay;      ///< [in] Whether is okay to overwrite source buffers
+
+
+    //------------------------------------------------------------------------------
+    // Constructors
+    //------------------------------------------------------------------------------
+
+    /// Constructor
+    CUB_RUNTIME_FUNCTION __forceinline__
+    DispatchSegmentedRadixSort(
+        void*                   d_temp_storage,
+        size_t                  &temp_storage_bytes,
+        DoubleBuffer<KeyT>      &d_keys,
+        DoubleBuffer<ValueT>    &d_values,
+        OffsetT                 num_items,
+        OffsetT                 num_segments,
+        OffsetIteratorT         d_begin_offsets,
+        OffsetIteratorT         d_end_offsets,
+        int                     begin_bit,
+        int                     end_bit,
+        bool                    is_overwrite_okay,
+        cudaStream_t            stream,
+        bool                    debug_synchronous,
+        int                     ptx_version)
+    :
+        d_temp_storage(d_temp_storage),
+        temp_storage_bytes(temp_storage_bytes),
+        d_keys(d_keys),
+        d_values(d_values),
+        num_items(num_items),
+        num_segments(num_segments),
+        d_begin_offsets(d_begin_offsets),
+        d_end_offsets(d_end_offsets),
+        begin_bit(begin_bit),
+        end_bit(end_bit),
+        is_overwrite_okay(is_overwrite_okay),
+        stream(stream),
+        debug_synchronous(debug_synchronous),
+        ptx_version(ptx_version)
+    {}
+
+
+    //------------------------------------------------------------------------------
+    // Multi-segment invocation
+    //------------------------------------------------------------------------------
+
+    /// Invoke a three-kernel sorting pass at the current bit.
+    template <typename PassConfigT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t InvokePass(
+        const KeyT      *d_keys_in,
+        KeyT            *d_keys_out,
+        const ValueT    *d_values_in,
+        ValueT          *d_values_out,
+        int             &current_bit,
+        PassConfigT     &pass_config)
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            int pass_bits = CUB_MIN(pass_config.radix_bits, (end_bit - current_bit));
+
+            // Log kernel configuration
+            if (debug_synchronous)
+                _CubLog("Invoking segmented_kernels<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy, current bit %d, bit_grain %d\n",
+                    num_segments, pass_config.segmented_config.block_threads, (long long) stream,
+                pass_config.segmented_config.items_per_thread, pass_config.segmented_config.sm_occupancy, current_bit, pass_bits);
+
+            pass_config.segmented_kernel<<<num_segments, pass_config.segmented_config.block_threads, 0, stream>>>(
+                d_keys_in, d_keys_out,
+                d_values_in,  d_values_out,
+                d_begin_offsets, d_end_offsets, num_segments,
+                current_bit, pass_bits);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Update current bit
+            current_bit += pass_bits;
+        }
+        while (0);
+
+        return error;
+    }
+
+
+    /// PassConfig data structure
+    template <typename SegmentedKernelT>
+    struct PassConfig
+    {
+        SegmentedKernelT    segmented_kernel;
+        KernelConfig        segmented_config;
+        int                 radix_bits;
+        int                 radix_digits;
+
+        /// Initialize pass configuration
+        template <typename SegmentedPolicyT>
+        CUB_RUNTIME_FUNCTION __forceinline__
+        cudaError_t InitPassConfig(SegmentedKernelT segmented_kernel)
+        {
+            this->segmented_kernel  = segmented_kernel;
+            this->radix_bits        = SegmentedPolicyT::RADIX_BITS;
+            this->radix_digits      = 1 << radix_bits;
+
+            return CubDebug(segmented_config.Init<SegmentedPolicyT>(segmented_kernel));
+        }
+    };
+
+
+    /// Invocation (run multiple digit passes)
+    template <
+        typename                ActivePolicyT,          ///< Umbrella policy active for the target device
+        typename                SegmentedKernelT>       ///< Function type of cub::DeviceSegmentedRadixSortKernel
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t InvokePasses(
+        SegmentedKernelT     segmented_kernel,          ///< [in] Kernel function pointer to parameterization of cub::DeviceSegmentedRadixSortKernel
+        SegmentedKernelT     alt_segmented_kernel)      ///< [in] Alternate kernel function pointer to parameterization of cub::DeviceSegmentedRadixSortKernel
+    {
+#ifndef CUB_RUNTIME_ENABLED
+      (void)segmented_kernel;
+      (void)alt_segmented_kernel;
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported );
+#else
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Init regular and alternate kernel configurations
+            PassConfig<SegmentedKernelT> pass_config, alt_pass_config;
+            if ((error = pass_config.template       InitPassConfig<typename ActivePolicyT::SegmentedPolicy>(segmented_kernel))) break;
+            if ((error = alt_pass_config.template   InitPassConfig<typename ActivePolicyT::AltSegmentedPolicy>(alt_segmented_kernel))) break;
+
+            // Temporary storage allocation requirements
+            void* allocations[2];
+            size_t allocation_sizes[2] =
+            {
+                (is_overwrite_okay) ? 0 : num_items * sizeof(KeyT),                      // bytes needed for 3rd keys buffer
+                (is_overwrite_okay || (KEYS_ONLY)) ? 0 : num_items * sizeof(ValueT),     // bytes needed for 3rd values buffer
+            };
+
+            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+
+            // Return if the caller is simply requesting the size of the storage allocation
+            if (d_temp_storage == NULL)
+            {
+                if (temp_storage_bytes == 0)
+                    temp_storage_bytes = 1;
+                return cudaSuccess;
+            }
+
+            // Pass planning.  Run passes of the alternate digit-size configuration until we have an even multiple of our preferred digit size
+            int radix_bits          = ActivePolicyT::SegmentedPolicy::RADIX_BITS;
+            int alt_radix_bits      = ActivePolicyT::AltSegmentedPolicy::RADIX_BITS;
+            int num_bits            = end_bit - begin_bit;
+            int num_passes          = (num_bits + radix_bits - 1) / radix_bits;
+            bool is_num_passes_odd  = num_passes & 1;
+            int max_alt_passes      = (num_passes * radix_bits) - num_bits;
+            int alt_end_bit         = CUB_MIN(end_bit, begin_bit + (max_alt_passes * alt_radix_bits));
+
+            DoubleBuffer<KeyT> d_keys_remaining_passes(
+                (is_overwrite_okay || is_num_passes_odd) ? d_keys.Alternate() : static_cast<KeyT*>(allocations[0]),
+                (is_overwrite_okay) ? d_keys.Current() : (is_num_passes_odd) ? static_cast<KeyT*>(allocations[0]) : d_keys.Alternate());
+
+            DoubleBuffer<ValueT> d_values_remaining_passes(
+                (is_overwrite_okay || is_num_passes_odd) ? d_values.Alternate() : static_cast<ValueT*>(allocations[1]),
+                (is_overwrite_okay) ? d_values.Current() : (is_num_passes_odd) ? static_cast<ValueT*>(allocations[1]) : d_values.Alternate());
+
+            // Run first pass, consuming from the input's current buffers
+            int current_bit = begin_bit;
+
+            if (CubDebug(error = InvokePass(
+                d_keys.Current(), d_keys_remaining_passes.Current(),
+                d_values.Current(), d_values_remaining_passes.Current(),
+                current_bit,
+                (current_bit < alt_end_bit) ? alt_pass_config : pass_config))) break;
+
+            // Run remaining passes
+            while (current_bit < end_bit)
+            {
+                if (CubDebug(error = InvokePass(
+                    d_keys_remaining_passes.d_buffers[d_keys_remaining_passes.selector],    d_keys_remaining_passes.d_buffers[d_keys_remaining_passes.selector ^ 1],
+                    d_values_remaining_passes.d_buffers[d_keys_remaining_passes.selector],  d_values_remaining_passes.d_buffers[d_keys_remaining_passes.selector ^ 1],
+                    current_bit,
+                    (current_bit < alt_end_bit) ? alt_pass_config : pass_config))) break;
+
+                // Invert selectors and update current bit
+                d_keys_remaining_passes.selector ^= 1;
+                d_values_remaining_passes.selector ^= 1;
+            }
+
+            // Update selector
+            if (!is_overwrite_okay) {
+                num_passes = 1; // Sorted data always ends up in the other vector
+            }
+
+            d_keys.selector = (d_keys.selector + num_passes) & 1;
+            d_values.selector = (d_values.selector + num_passes) & 1;
+        }
+        while (0);
+
+        return error;
+
+#endif // CUB_RUNTIME_ENABLED
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Chained policy invocation
+    //------------------------------------------------------------------------------
+
+    /// Invocation
+    template <typename ActivePolicyT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t Invoke()
+    {
+        typedef typename DispatchSegmentedRadixSort::MaxPolicy MaxPolicyT;
+
+        // Force kernel code-generation in all compiler passes
+        return InvokePasses<ActivePolicyT>(
+            DeviceSegmentedRadixSortKernel<MaxPolicyT, false,   IS_DESCENDING, KeyT, ValueT, OffsetIteratorT, OffsetT>,
+            DeviceSegmentedRadixSortKernel<MaxPolicyT, true,    IS_DESCENDING, KeyT, ValueT, OffsetIteratorT, OffsetT>);
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Dispatch entrypoints
+    //------------------------------------------------------------------------------
+
+
+    /// Internal dispatch routine
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                   d_temp_storage,         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        DoubleBuffer<KeyT>      &d_keys,                ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
+        DoubleBuffer<ValueT>    &d_values,              ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
+        int                     num_items,              ///< [in] Number of items to sort
+        int                     num_segments,           ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT         d_begin_offsets,        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT         d_end_offsets,          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        int                     begin_bit,              ///< [in] The beginning (least-significant) bit index needed for key comparison
+        int                     end_bit,                ///< [in] The past-the-end (most-significant) bit index needed for key comparison
+        bool                    is_overwrite_okay,      ///< [in] Whether is okay to overwrite source buffers
+        cudaStream_t            stream,                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous)      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        typedef typename DispatchSegmentedRadixSort::MaxPolicy MaxPolicyT;
+
+        cudaError_t error;
+        do {
+            // Get PTX version
+            int ptx_version;
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+
+            // Create dispatch functor
+            DispatchSegmentedRadixSort dispatch(
+                d_temp_storage, temp_storage_bytes,
+                d_keys, d_values,
+                num_items, num_segments, d_begin_offsets, d_end_offsets,
+                begin_bit, end_bit, is_overwrite_okay,
+                stream, debug_synchronous, ptx_version);
+
+            // Dispatch to chained policy
+            if (CubDebug(error = MaxPolicyT::Invoke(ptx_version, dispatch))) break;
+
+        } while (0);
+
+        return error;
+    }
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/dispatch/dispatch_reduce.cuh b/third_party/cub/cub/device/dispatch/dispatch_reduce.cuh
new file mode 100644
index 0000000..e9d1b7a
--- /dev/null
+++ b/third_party/cub/cub/device/dispatch/dispatch_reduce.cuh
@@ -0,0 +1,882 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceReduce provides device-wide, parallel operations for computing a reduction across a sequence of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "../../agent/agent_reduce.cuh"
+#include "../../iterator/arg_index_input_iterator.cuh"
+#include "../../thread/thread_operators.cuh"
+#include "../../grid/grid_even_share.cuh"
+#include "../../iterator/arg_index_input_iterator.cuh"
+#include "../../util_debug.cuh"
+#include "../../util_device.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/******************************************************************************
+ * Kernel entry points
+ *****************************************************************************/
+
+/**
+ * Reduce region kernel entry point (multi-block).  Computes privatized reductions, one per thread block.
+ */
+template <
+    typename                ChainedPolicyT,             ///< Chained tuning policy
+    typename                InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
+    typename                OutputIteratorT,            ///< Output iterator type for recording the reduced aggregate \iterator
+    typename                OffsetT,                    ///< Signed integer type for global offsets
+    typename                ReductionOpT>               ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::ReducePolicy::BLOCK_THREADS))
+__global__ void DeviceReduceKernel(
+    InputIteratorT          d_in,                       ///< [in] Pointer to the input sequence of data items
+    OutputIteratorT         d_out,                      ///< [out] Pointer to the output aggregate
+    OffsetT                 num_items,                  ///< [in] Total number of input data items
+    GridEvenShare<OffsetT>  even_share,                 ///< [in] Even-share descriptor for mapping an equal number of tiles onto each thread block
+    ReductionOpT            reduction_op)               ///< [in] Binary reduction functor
+{
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    // Thread block type for reducing input tiles
+    typedef AgentReduce<
+            typename ChainedPolicyT::ActivePolicy::ReducePolicy,
+            InputIteratorT,
+            OutputIteratorT,
+            OffsetT,
+            ReductionOpT>
+        AgentReduceT;
+
+    // Shared memory storage
+    __shared__ typename AgentReduceT::TempStorage temp_storage;
+
+    // Consume input tiles
+    OutputT block_aggregate = AgentReduceT(temp_storage, d_in, reduction_op).ConsumeTiles(even_share);
+
+    // Output result
+    if (threadIdx.x == 0)
+        d_out[blockIdx.x] = block_aggregate;
+}
+
+
+/**
+ * Reduce a single tile kernel entry point (single-block).  Can be used to aggregate privatized thread block reductions from a previous multi-block reduction pass.
+ */
+template <
+    typename                ChainedPolicyT,             ///< Chained tuning policy
+    typename                InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
+    typename                OutputIteratorT,            ///< Output iterator type for recording the reduced aggregate \iterator
+    typename                OffsetT,                    ///< Signed integer type for global offsets
+    typename                ReductionOpT,               ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+    typename                OuputT>                     ///< Data element type that is convertible to the \p value type of \p OutputIteratorT
+__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::SingleTilePolicy::BLOCK_THREADS), 1)
+__global__ void DeviceReduceSingleTileKernel(
+    InputIteratorT          d_in,                       ///< [in] Pointer to the input sequence of data items
+    OutputIteratorT         d_out,                      ///< [out] Pointer to the output aggregate
+    OffsetT                 num_items,                  ///< [in] Total number of input data items
+    ReductionOpT            reduction_op,               ///< [in] Binary reduction functor
+    OuputT                  init)                       ///< [in] The initial value of the reduction
+{
+    // Thread block type for reducing input tiles
+    typedef AgentReduce<
+            typename ChainedPolicyT::ActivePolicy::SingleTilePolicy,
+            InputIteratorT,
+            OutputIteratorT,
+            OffsetT,
+            ReductionOpT>
+        AgentReduceT;
+
+    // Shared memory storage
+    __shared__ typename AgentReduceT::TempStorage temp_storage;
+
+    // Check if empty problem
+    if (num_items == 0)
+    {
+        if (threadIdx.x == 0)
+            *d_out = init;
+        return;
+    }
+
+    // Consume input tiles
+    OuputT block_aggregate = AgentReduceT(temp_storage, d_in, reduction_op).ConsumeRange(
+        OffsetT(0),
+        num_items);
+
+    // Output result
+    if (threadIdx.x == 0)
+        *d_out = reduction_op(init, block_aggregate);
+}
+
+
+/// Normalize input iterator to segment offset
+template <typename T, typename OffsetT, typename IteratorT>
+__device__ __forceinline__
+void NormalizeReductionOutput(
+    T &/*val*/,
+    OffsetT /*base_offset*/,
+    IteratorT /*itr*/)
+{}
+
+
+/// Normalize input iterator to segment offset (specialized for arg-index)
+template <typename KeyValuePairT, typename OffsetT, typename WrappedIteratorT, typename OutputValueT>
+__device__ __forceinline__
+void NormalizeReductionOutput(
+    KeyValuePairT &val,
+    OffsetT base_offset,
+    ArgIndexInputIterator<WrappedIteratorT, OffsetT, OutputValueT> /*itr*/)
+{
+    val.key -= base_offset;
+}
+
+
+/**
+ * Segmented reduction (one block per segment)
+ */
+template <
+    typename                ChainedPolicyT,             ///< Chained tuning policy
+    typename                InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
+    typename                OutputIteratorT,            ///< Output iterator type for recording the reduced aggregate \iterator
+    typename                OffsetIteratorT,            ///< Random-access input iterator type for reading segment offsets \iterator
+    typename                OffsetT,                    ///< Signed integer type for global offsets
+    typename                ReductionOpT,               ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+    typename                OutputT>                    ///< Data element type that is convertible to the \p value type of \p OutputIteratorT
+__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::ReducePolicy::BLOCK_THREADS))
+__global__ void DeviceSegmentedReduceKernel(
+    InputIteratorT          d_in,                       ///< [in] Pointer to the input sequence of data items
+    OutputIteratorT         d_out,                      ///< [out] Pointer to the output aggregate
+    OffsetIteratorT         d_begin_offsets,            ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+    OffsetIteratorT         d_end_offsets,              ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+    int                     /*num_segments*/,           ///< [in] The number of segments that comprise the sorting data
+    ReductionOpT            reduction_op,               ///< [in] Binary reduction functor 
+    OutputT                 init)                       ///< [in] The initial value of the reduction
+{
+    // Thread block type for reducing input tiles
+    typedef AgentReduce<
+            typename ChainedPolicyT::ActivePolicy::ReducePolicy,
+            InputIteratorT,
+            OutputIteratorT,
+            OffsetT,
+            ReductionOpT>
+        AgentReduceT;
+
+    // Shared memory storage
+    __shared__ typename AgentReduceT::TempStorage temp_storage;
+
+    OffsetT segment_begin   = d_begin_offsets[blockIdx.x];
+    OffsetT segment_end     = d_end_offsets[blockIdx.x];
+
+    // Check if empty problem
+    if (segment_begin == segment_end)
+    {
+        if (threadIdx.x == 0)
+            d_out[blockIdx.x] = init;
+        return;
+    }
+
+    // Consume input tiles
+    OutputT block_aggregate = AgentReduceT(temp_storage, d_in, reduction_op).ConsumeRange(
+        segment_begin,
+        segment_end);
+
+    // Normalize as needed
+    NormalizeReductionOutput(block_aggregate, segment_begin, d_in);
+
+    if (threadIdx.x == 0)
+        d_out[blockIdx.x] = reduction_op(init, block_aggregate);;
+}
+
+
+
+
+/******************************************************************************
+ * Policy
+ ******************************************************************************/
+
+template <
+    typename OuputT,            ///< Data type
+    typename OffsetT,           ///< Signed integer type for global offsets
+    typename ReductionOpT>      ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
+struct DeviceReducePolicy
+{
+    //------------------------------------------------------------------------------
+    // Architecture-specific tuning policies
+    //------------------------------------------------------------------------------
+
+    /// SM13
+    struct Policy130 : ChainedPolicy<130, Policy130, Policy130>
+    {
+        // ReducePolicy
+        typedef AgentReducePolicy<
+                CUB_SCALED_GRANULARITIES(128, 8, OuputT), ///< Threads per block, items per thread
+                2,                                  ///< Number of items per vectorized load
+                BLOCK_REDUCE_RAKING,                ///< Cooperative block-wide reduction algorithm to use
+                LOAD_DEFAULT>                       ///< Cache load modifier
+            ReducePolicy;
+
+        // SingleTilePolicy
+        typedef ReducePolicy SingleTilePolicy;
+
+        // SegmentedReducePolicy
+        typedef ReducePolicy SegmentedReducePolicy;
+    };
+
+
+    /// SM20
+    struct Policy200 : ChainedPolicy<200, Policy200, Policy130>
+    {
+        // ReducePolicy (GTX 580: 178.9 GB/s @ 48M 4B items, 158.1 GB/s @ 192M 1B items)
+        typedef AgentReducePolicy<
+                CUB_SCALED_GRANULARITIES(128, 8, OuputT),     ///< Threads per block, items per thread
+                4,                                      ///< Number of items per vectorized load
+                BLOCK_REDUCE_RAKING,                    ///< Cooperative block-wide reduction algorithm to use
+                LOAD_DEFAULT>                           ///< Cache load modifier
+            ReducePolicy;
+
+        // SingleTilePolicy
+        typedef ReducePolicy SingleTilePolicy;
+
+        // SegmentedReducePolicy
+        typedef ReducePolicy SegmentedReducePolicy;
+    };
+
+
+    /// SM30
+    struct Policy300 : ChainedPolicy<300, Policy300, Policy200>
+    {
+        // ReducePolicy (GTX670: 154.0 @ 48M 4B items)
+        typedef AgentReducePolicy<
+                CUB_SCALED_GRANULARITIES(256, 20, OuputT),    ///< Threads per block, items per thread
+                2,                                      ///< Number of items per vectorized load
+                BLOCK_REDUCE_WARP_REDUCTIONS,           ///< Cooperative block-wide reduction algorithm to use
+                LOAD_DEFAULT>                           ///< Cache load modifier
+            ReducePolicy;
+
+        // SingleTilePolicy
+        typedef ReducePolicy SingleTilePolicy;
+
+        // SegmentedReducePolicy
+        typedef ReducePolicy SegmentedReducePolicy;
+    };
+
+
+    /// SM35
+    struct Policy350 : ChainedPolicy<350, Policy350, Policy300>
+    {
+        // ReducePolicy (GTX Titan: 255.1 GB/s @ 48M 4B items; 228.7 GB/s @ 192M 1B items)
+        typedef AgentReducePolicy<
+                CUB_SCALED_GRANULARITIES(256, 20, OuputT),    ///< Threads per block, items per thread
+                4,                                      ///< Number of items per vectorized load
+                BLOCK_REDUCE_WARP_REDUCTIONS,           ///< Cooperative block-wide reduction algorithm to use
+                LOAD_LDG>                               ///< Cache load modifier
+            ReducePolicy;
+
+        // SingleTilePolicy
+        typedef ReducePolicy SingleTilePolicy;
+
+        // SegmentedReducePolicy
+        typedef ReducePolicy SegmentedReducePolicy;
+    };
+
+    /// SM60
+    struct Policy600 : ChainedPolicy<600, Policy600, Policy350>
+    {
+        // ReducePolicy (P100: 591 GB/s @ 64M 4B items; 583 GB/s @ 256M 1B items)
+        typedef AgentReducePolicy<
+                CUB_SCALED_GRANULARITIES(256, 16, OuputT),    ///< Threads per block, items per thread
+                4,                                      ///< Number of items per vectorized load
+                BLOCK_REDUCE_WARP_REDUCTIONS,           ///< Cooperative block-wide reduction algorithm to use
+                LOAD_LDG>                               ///< Cache load modifier
+            ReducePolicy;
+
+        // SingleTilePolicy
+        typedef ReducePolicy SingleTilePolicy;
+
+        // SegmentedReducePolicy
+        typedef ReducePolicy SegmentedReducePolicy;
+    };
+
+
+    /// MaxPolicy
+    typedef Policy600 MaxPolicy;
+
+};
+
+
+
+/******************************************************************************
+ * Single-problem dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for device-wide reduction
+ */
+template <
+    typename InputIteratorT,    ///< Random-access input iterator type for reading input items \iterator
+    typename OutputIteratorT,   ///< Output iterator type for recording the reduced aggregate \iterator
+    typename OffsetT,           ///< Signed integer type for global offsets
+    typename ReductionOpT>      ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
+struct DispatchReduce :
+    DeviceReducePolicy<
+        typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+            typename std::iterator_traits<InputIteratorT>::value_type,                                  // ... then the input iterator's value type,
+            typename std::iterator_traits<OutputIteratorT>::value_type>::Type,                          // ... else the output iterator's value type
+        OffsetT,
+        ReductionOpT>
+{
+    //------------------------------------------------------------------------------
+    // Constants
+    //------------------------------------------------------------------------------
+
+    // Data type of output iterator
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+
+    //------------------------------------------------------------------------------
+    // Problem state
+    //------------------------------------------------------------------------------
+
+    void                *d_temp_storage;                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+    size_t              &temp_storage_bytes;            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+    InputIteratorT      d_in;                           ///< [in] Pointer to the input sequence of data items
+    OutputIteratorT     d_out;                          ///< [out] Pointer to the output aggregate
+    OffsetT             num_items;                      ///< [in] Total number of input items (i.e., length of \p d_in)
+    ReductionOpT        reduction_op;                   ///< [in] Binary reduction functor 
+    OutputT             init;                           ///< [in] The initial value of the reduction
+    cudaStream_t        stream;                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+    bool                debug_synchronous;              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    int                 ptx_version;                    ///< [in] PTX version
+
+    //------------------------------------------------------------------------------
+    // Constructor
+    //------------------------------------------------------------------------------
+
+    /// Constructor
+    CUB_RUNTIME_FUNCTION __forceinline__
+    DispatchReduce(
+        void*                   d_temp_storage,
+        size_t                  &temp_storage_bytes,
+        InputIteratorT          d_in,
+        OutputIteratorT         d_out,
+        OffsetT                 num_items,
+        ReductionOpT            reduction_op,
+        OutputT                 init,
+        cudaStream_t            stream,
+        bool                    debug_synchronous,
+        int                     ptx_version)
+    :
+        d_temp_storage(d_temp_storage),
+        temp_storage_bytes(temp_storage_bytes),
+        d_in(d_in),
+        d_out(d_out),
+        num_items(num_items),
+        reduction_op(reduction_op),
+        init(init),
+        stream(stream),
+        debug_synchronous(debug_synchronous),
+        ptx_version(ptx_version)
+    {}
+
+
+    //------------------------------------------------------------------------------
+    // Small-problem (single tile) invocation
+    //------------------------------------------------------------------------------
+
+    /// Invoke a single block block to reduce in-core
+    template <
+        typename                ActivePolicyT,          ///< Umbrella policy active for the target device
+        typename                SingleTileKernelT>      ///< Function type of cub::DeviceReduceSingleTileKernel
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t InvokeSingleTile(
+        SingleTileKernelT       single_tile_kernel)     ///< [in] Kernel function pointer to parameterization of cub::DeviceReduceSingleTileKernel
+    {
+#ifndef CUB_RUNTIME_ENABLED
+        (void)single_tile_kernel;
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported );
+#else
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Return if the caller is simply requesting the size of the storage allocation
+            if (d_temp_storage == NULL)
+            {
+                temp_storage_bytes = 1;
+                break;
+            }
+
+            // Log single_reduce_sweep_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking DeviceReduceSingleTileKernel<<<1, %d, 0, %lld>>>(), %d items per thread\n",
+                ActivePolicyT::SingleTilePolicy::BLOCK_THREADS,
+                (long long) stream,
+                ActivePolicyT::SingleTilePolicy::ITEMS_PER_THREAD);
+
+            // Invoke single_reduce_sweep_kernel
+            single_tile_kernel<<<1, ActivePolicyT::SingleTilePolicy::BLOCK_THREADS, 0, stream>>>(
+                d_in,
+                d_out,
+                num_items,
+                reduction_op,
+                init);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+        }
+        while (0);
+
+        return error;
+
+#endif // CUB_RUNTIME_ENABLED
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Normal problem size invocation (two-pass)
+    //------------------------------------------------------------------------------
+
+    /// Invoke two-passes to reduce
+    template <
+        typename                ActivePolicyT,              ///< Umbrella policy active for the target device
+        typename                ReduceKernelT,              ///< Function type of cub::DeviceReduceKernel
+        typename                SingleTileKernelT>          ///< Function type of cub::DeviceReduceSingleTileKernel
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t InvokePasses(
+        ReduceKernelT           reduce_kernel,          ///< [in] Kernel function pointer to parameterization of cub::DeviceReduceKernel
+        SingleTileKernelT       single_tile_kernel)     ///< [in] Kernel function pointer to parameterization of cub::DeviceReduceSingleTileKernel
+    {
+#ifndef CUB_RUNTIME_ENABLED
+        (void)                  reduce_kernel;
+        (void)                  single_tile_kernel;
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported );
+#else
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Init regular kernel configuration
+            KernelConfig reduce_config;
+            if (CubDebug(error = reduce_config.Init<typename ActivePolicyT::ReducePolicy>(reduce_kernel))) break;
+            int reduce_device_occupancy = reduce_config.sm_occupancy * sm_count;
+
+            // Even-share work distribution
+            int max_blocks = reduce_device_occupancy * CUB_SUBSCRIPTION_FACTOR(ptx_version);
+            GridEvenShare<OffsetT> even_share;
+            even_share.DispatchInit(num_items, max_blocks, reduce_config.tile_size);
+
+            // Temporary storage allocation requirements
+            void* allocations[1];
+            size_t allocation_sizes[1] =
+            {
+                max_blocks * sizeof(OutputT)    // bytes needed for privatized block reductions
+            };
+
+            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+            if (d_temp_storage == NULL)
+            {
+                // Return if the caller is simply requesting the size of the storage allocation
+                return cudaSuccess;
+            }
+
+            // Alias the allocation for the privatized per-block reductions
+            OutputT *d_block_reductions = (OutputT*) allocations[0];
+
+            // Get grid size for device_reduce_sweep_kernel
+            int reduce_grid_size = even_share.grid_size;
+
+            // Log device_reduce_sweep_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking DeviceReduceKernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                reduce_grid_size,
+                ActivePolicyT::ReducePolicy::BLOCK_THREADS,
+                (long long) stream,
+                ActivePolicyT::ReducePolicy::ITEMS_PER_THREAD,
+                reduce_config.sm_occupancy);
+
+            // Invoke DeviceReduceKernel
+            reduce_kernel<<<reduce_grid_size, ActivePolicyT::ReducePolicy::BLOCK_THREADS, 0, stream>>>(
+                d_in,
+                d_block_reductions,
+                num_items,
+                even_share,
+                reduction_op);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Log single_reduce_sweep_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking DeviceReduceSingleTileKernel<<<1, %d, 0, %lld>>>(), %d items per thread\n",
+                ActivePolicyT::SingleTilePolicy::BLOCK_THREADS,
+                (long long) stream,
+                ActivePolicyT::SingleTilePolicy::ITEMS_PER_THREAD);
+
+            // Invoke DeviceReduceSingleTileKernel
+            single_tile_kernel<<<1, ActivePolicyT::SingleTilePolicy::BLOCK_THREADS, 0, stream>>>(
+                d_block_reductions,
+                d_out,
+                reduce_grid_size,
+                reduction_op,
+                init);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+        }
+        while (0);
+
+        return error;
+
+#endif // CUB_RUNTIME_ENABLED
+
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Chained policy invocation
+    //------------------------------------------------------------------------------
+
+    /// Invocation
+    template <typename ActivePolicyT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t Invoke()
+    {
+        typedef typename ActivePolicyT::SingleTilePolicy    SingleTilePolicyT;
+        typedef typename DispatchReduce::MaxPolicy          MaxPolicyT;
+
+        // Force kernel code-generation in all compiler passes
+        if (num_items <= (SingleTilePolicyT::BLOCK_THREADS * SingleTilePolicyT::ITEMS_PER_THREAD))
+        {
+            // Small, single tile size
+            return InvokeSingleTile<ActivePolicyT>(
+                DeviceReduceSingleTileKernel<MaxPolicyT, InputIteratorT, OutputIteratorT, OffsetT, ReductionOpT, OutputT>);
+        }
+        else
+        {
+            // Regular size
+            return InvokePasses<ActivePolicyT>(
+                DeviceReduceKernel<typename DispatchReduce::MaxPolicy, InputIteratorT, OutputT*, OffsetT, ReductionOpT>,
+                DeviceReduceSingleTileKernel<MaxPolicyT, OutputT*, OutputIteratorT, OffsetT, ReductionOpT, OutputT>);
+        }
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Dispatch entrypoints
+    //------------------------------------------------------------------------------
+
+    /**
+     * Internal dispatch routine for computing a device-wide reduction
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT d_out,                              ///< [out] Pointer to the output aggregate
+        OffsetT         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
+        ReductionOpT    reduction_op,                       ///< [in] Binary reduction functor 
+        OutputT         init,                               ///< [in] The initial value of the reduction
+        cudaStream_t    stream,                             ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool            debug_synchronous)                  ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        typedef typename DispatchReduce::MaxPolicy MaxPolicyT;
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+
+            // Create dispatch functor
+            DispatchReduce dispatch(
+                d_temp_storage, temp_storage_bytes,
+                d_in, d_out, num_items, reduction_op, init,
+                stream, debug_synchronous, ptx_version);
+
+            // Dispatch to chained policy
+            if (CubDebug(error = MaxPolicyT::Invoke(ptx_version, dispatch))) break;
+        }
+        while (0);
+
+        return error;
+    }
+};
+
+
+
+/******************************************************************************
+ * Segmented dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for device-wide reduction
+ */
+template <
+    typename InputIteratorT,    ///< Random-access input iterator type for reading input items \iterator
+    typename OutputIteratorT,   ///< Output iterator type for recording the reduced aggregate \iterator
+    typename OffsetIteratorT,   ///< Random-access input iterator type for reading segment offsets \iterator
+    typename OffsetT,           ///< Signed integer type for global offsets
+    typename ReductionOpT>      ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
+struct DispatchSegmentedReduce :
+    DeviceReducePolicy<
+        typename std::iterator_traits<InputIteratorT>::value_type,
+        OffsetT,
+        ReductionOpT>
+{
+    //------------------------------------------------------------------------------
+    // Constants
+    //------------------------------------------------------------------------------
+
+    /// The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+
+    //------------------------------------------------------------------------------
+    // Problem state
+    //------------------------------------------------------------------------------
+
+    void                *d_temp_storage;        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+    size_t              &temp_storage_bytes;    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+    InputIteratorT      d_in;                   ///< [in] Pointer to the input sequence of data items
+    OutputIteratorT     d_out;                  ///< [out] Pointer to the output aggregate
+    OffsetT             num_segments;           ///< [in] The number of segments that comprise the sorting data
+    OffsetIteratorT     d_begin_offsets;        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+    OffsetIteratorT     d_end_offsets;          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+    ReductionOpT        reduction_op;           ///< [in] Binary reduction functor 
+    OutputT             init;                   ///< [in] The initial value of the reduction
+    cudaStream_t        stream;                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+    bool                debug_synchronous;      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    int                 ptx_version;            ///< [in] PTX version
+
+    //------------------------------------------------------------------------------
+    // Constructor
+    //------------------------------------------------------------------------------
+
+    /// Constructor
+    CUB_RUNTIME_FUNCTION __forceinline__
+    DispatchSegmentedReduce(
+        void*                   d_temp_storage,
+        size_t                  &temp_storage_bytes,
+        InputIteratorT          d_in,
+        OutputIteratorT         d_out,
+        OffsetT                 num_segments,
+        OffsetIteratorT         d_begin_offsets,
+        OffsetIteratorT         d_end_offsets,
+        ReductionOpT            reduction_op,
+        OutputT                 init,
+        cudaStream_t            stream,
+        bool                    debug_synchronous,
+        int                     ptx_version)
+    :
+        d_temp_storage(d_temp_storage),
+        temp_storage_bytes(temp_storage_bytes),
+        d_in(d_in),
+        d_out(d_out),
+        num_segments(num_segments),
+        d_begin_offsets(d_begin_offsets),
+        d_end_offsets(d_end_offsets),
+        reduction_op(reduction_op),
+        init(init),
+        stream(stream),
+        debug_synchronous(debug_synchronous),
+        ptx_version(ptx_version)
+    {}
+
+
+
+    //------------------------------------------------------------------------------
+    // Chained policy invocation
+    //------------------------------------------------------------------------------
+
+    /// Invocation
+    template <
+        typename                        ActivePolicyT,                  ///< Umbrella policy active for the target device
+        typename                        DeviceSegmentedReduceKernelT>   ///< Function type of cub::DeviceSegmentedReduceKernel
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t InvokePasses(
+        DeviceSegmentedReduceKernelT    segmented_reduce_kernel)        ///< [in] Kernel function pointer to parameterization of cub::DeviceSegmentedReduceKernel
+    {
+#ifndef CUB_RUNTIME_ENABLED
+        (void)segmented_reduce_kernel;
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported );
+#else
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Return if the caller is simply requesting the size of the storage allocation
+            if (d_temp_storage == NULL)
+            {
+                temp_storage_bytes = 1;
+                return cudaSuccess;
+            }
+
+            // Init kernel configuration
+            KernelConfig segmented_reduce_config;
+            if (CubDebug(error = segmented_reduce_config.Init<typename ActivePolicyT::SegmentedReducePolicy>(segmented_reduce_kernel))) break;
+
+            // Log device_reduce_sweep_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking SegmentedDeviceReduceKernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                num_segments,
+                ActivePolicyT::SegmentedReducePolicy::BLOCK_THREADS,
+                (long long) stream,
+                ActivePolicyT::SegmentedReducePolicy::ITEMS_PER_THREAD,
+                segmented_reduce_config.sm_occupancy);
+
+            // Invoke DeviceReduceKernel
+            segmented_reduce_kernel<<<num_segments, ActivePolicyT::SegmentedReducePolicy::BLOCK_THREADS, 0, stream>>>(
+                d_in,
+                d_out,
+                d_begin_offsets,
+                d_end_offsets,
+                num_segments,
+                reduction_op,
+                init);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+        }
+        while (0);
+
+        return error;
+
+#endif // CUB_RUNTIME_ENABLED
+
+    }
+
+
+    /// Invocation
+    template <typename ActivePolicyT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t Invoke()
+    {
+        typedef typename DispatchSegmentedReduce::MaxPolicy MaxPolicyT;
+
+        // Force kernel code-generation in all compiler passes
+        return InvokePasses<ActivePolicyT>(
+            DeviceSegmentedReduceKernel<MaxPolicyT, InputIteratorT, OutputIteratorT, OffsetIteratorT, OffsetT, ReductionOpT, OutputT>);
+    }
+
+
+    //------------------------------------------------------------------------------
+    // Dispatch entrypoints
+    //------------------------------------------------------------------------------
+
+    /**
+     * Internal dispatch routine for computing a device-wide reduction
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT d_out,                              ///< [out] Pointer to the output aggregate
+        int             num_segments,                       ///< [in] The number of segments that comprise the sorting data
+        OffsetIteratorT d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
+        OffsetIteratorT d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
+        ReductionOpT    reduction_op,                       ///< [in] Binary reduction functor 
+        OutputT         init,                               ///< [in] The initial value of the reduction
+        cudaStream_t    stream,                             ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool            debug_synchronous)                  ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        typedef typename DispatchSegmentedReduce::MaxPolicy MaxPolicyT;
+
+        if (num_segments <= 0)
+            return cudaSuccess;
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+
+            // Create dispatch functor
+            DispatchSegmentedReduce dispatch(
+                d_temp_storage, temp_storage_bytes,
+                d_in, d_out,
+                num_segments, d_begin_offsets, d_end_offsets,
+                reduction_op, init,
+                stream, debug_synchronous, ptx_version);
+
+            // Dispatch to chained policy
+            if (CubDebug(error = MaxPolicyT::Invoke(ptx_version, dispatch))) break;
+        }
+        while (0);
+
+        return error;
+    }
+};
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/dispatch/dispatch_reduce_by_key.cuh b/third_party/cub/cub/device/dispatch/dispatch_reduce_by_key.cuh
new file mode 100644
index 0000000..6f4837b
--- /dev/null
+++ b/third_party/cub/cub/device/dispatch/dispatch_reduce_by_key.cuh
@@ -0,0 +1,554 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceReduceByKey provides device-wide, parallel operations for reducing segments of values residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch_scan.cuh"
+#include "../../agent/agent_reduce_by_key.cuh"
+#include "../../thread/thread_operators.cuh"
+#include "../../grid/grid_queue.cuh"
+#include "../../util_device.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/******************************************************************************
+ * Kernel entry points
+ *****************************************************************************/
+
+/**
+ * Multi-block reduce-by-key sweep kernel entry point
+ */
+template <
+    typename            AgentReduceByKeyPolicyT,                 ///< Parameterized AgentReduceByKeyPolicyT tuning policy type
+    typename            KeysInputIteratorT,                     ///< Random-access input iterator type for keys
+    typename            UniqueOutputIteratorT,                  ///< Random-access output iterator type for keys
+    typename            ValuesInputIteratorT,                   ///< Random-access input iterator type for values
+    typename            AggregatesOutputIteratorT,              ///< Random-access output iterator type for values
+    typename            NumRunsOutputIteratorT,                 ///< Output iterator type for recording number of segments encountered
+    typename            ScanTileStateT,                         ///< Tile status interface type
+    typename            EqualityOpT,                            ///< KeyT equality operator type
+    typename            ReductionOpT,                           ///< ValueT reduction operator type
+    typename            OffsetT>                                ///< Signed integer type for global offsets
+__launch_bounds__ (int(AgentReduceByKeyPolicyT::BLOCK_THREADS))
+__global__ void DeviceReduceByKeyKernel(
+    KeysInputIteratorT          d_keys_in,                      ///< Pointer to the input sequence of keys
+    UniqueOutputIteratorT       d_unique_out,                   ///< Pointer to the output sequence of unique keys (one key per run)
+    ValuesInputIteratorT        d_values_in,                    ///< Pointer to the input sequence of corresponding values
+    AggregatesOutputIteratorT   d_aggregates_out,               ///< Pointer to the output sequence of value aggregates (one aggregate per run)
+    NumRunsOutputIteratorT      d_num_runs_out,                 ///< Pointer to total number of runs encountered (i.e., the length of d_unique_out)
+    ScanTileStateT              tile_state,                     ///< Tile status interface
+    int                         start_tile,                     ///< The starting tile for the current grid
+    EqualityOpT                 equality_op,                    ///< KeyT equality operator
+    ReductionOpT                reduction_op,                   ///< ValueT reduction operator
+    OffsetT                     num_items)                      ///< Total number of items to select from
+{
+    // Thread block type for reducing tiles of value segments
+    typedef AgentReduceByKey<
+            AgentReduceByKeyPolicyT,
+            KeysInputIteratorT,
+            UniqueOutputIteratorT,
+            ValuesInputIteratorT,
+            AggregatesOutputIteratorT,
+            NumRunsOutputIteratorT,
+            EqualityOpT,
+            ReductionOpT,
+            OffsetT>
+        AgentReduceByKeyT;
+
+    // Shared memory for AgentReduceByKey
+    __shared__ typename AgentReduceByKeyT::TempStorage temp_storage;
+
+    // Process tiles
+    AgentReduceByKeyT(temp_storage, d_keys_in, d_unique_out, d_values_in, d_aggregates_out, d_num_runs_out, equality_op, reduction_op).ConsumeRange(
+        num_items,
+        tile_state,
+        start_tile);
+}
+
+
+
+
+/******************************************************************************
+ * Dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for DeviceReduceByKey
+ */
+template <
+    typename    KeysInputIteratorT,         ///< Random-access input iterator type for keys
+    typename    UniqueOutputIteratorT,      ///< Random-access output iterator type for keys
+    typename    ValuesInputIteratorT,       ///< Random-access input iterator type for values
+    typename    AggregatesOutputIteratorT,  ///< Random-access output iterator type for values
+    typename    NumRunsOutputIteratorT,     ///< Output iterator type for recording number of segments encountered
+    typename    EqualityOpT,                ///< KeyT equality operator type
+    typename    ReductionOpT,               ///< ValueT reduction operator type
+    typename    OffsetT>                    ///< Signed integer type for global offsets
+struct DispatchReduceByKey
+{
+    //-------------------------------------------------------------------------
+    // Types and constants
+    //-------------------------------------------------------------------------
+
+    // The input keys type
+    typedef typename std::iterator_traits<KeysInputIteratorT>::value_type KeyInputT;
+
+    // The output keys type
+    typedef typename If<(Equals<typename std::iterator_traits<UniqueOutputIteratorT>::value_type, void>::VALUE),    // KeyOutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<KeysInputIteratorT>::value_type,                                              // ... then the input iterator's value type,
+        typename std::iterator_traits<UniqueOutputIteratorT>::value_type>::Type KeyOutputT;                         // ... else the output iterator's value type
+
+    // The input values type
+    typedef typename std::iterator_traits<ValuesInputIteratorT>::value_type ValueInputT;
+
+    // The output values type
+    typedef typename If<(Equals<typename std::iterator_traits<AggregatesOutputIteratorT>::value_type, void>::VALUE),    // ValueOutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<ValuesInputIteratorT>::value_type,                                                // ... then the input iterator's value type,
+        typename std::iterator_traits<AggregatesOutputIteratorT>::value_type>::Type ValueOutputT;                       // ... else the output iterator's value type
+
+    enum
+    {
+        INIT_KERNEL_THREADS     = 128,
+        MAX_INPUT_BYTES         = CUB_MAX(sizeof(KeyOutputT), sizeof(ValueOutputT)),
+        COMBINED_INPUT_BYTES    = sizeof(KeyOutputT) + sizeof(ValueOutputT),
+    };
+
+    // Tile status descriptor interface type
+    typedef ReduceByKeyScanTileState<ValueOutputT, OffsetT> ScanTileStateT;
+
+
+    //-------------------------------------------------------------------------
+    // Tuning policies
+    //-------------------------------------------------------------------------
+
+    /// SM35
+    struct Policy350
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 6,
+            ITEMS_PER_THREAD            = (MAX_INPUT_BYTES <= 8) ? 6 : CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, ((NOMINAL_4B_ITEMS_PER_THREAD * 8) + COMBINED_INPUT_BYTES - 1) / COMBINED_INPUT_BYTES)),
+        };
+
+        typedef AgentReduceByKeyPolicy<
+                128,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_DIRECT,
+                LOAD_LDG,
+                BLOCK_SCAN_WARP_SCANS>
+            ReduceByKeyPolicyT;
+    };
+
+    /// SM30
+    struct Policy300
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 6,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, ((NOMINAL_4B_ITEMS_PER_THREAD * 8) + COMBINED_INPUT_BYTES - 1) / COMBINED_INPUT_BYTES)),
+        };
+
+        typedef AgentReduceByKeyPolicy<
+                128,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_WARP_SCANS>
+            ReduceByKeyPolicyT;
+    };
+
+    /// SM20
+    struct Policy200
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 11,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, ((NOMINAL_4B_ITEMS_PER_THREAD * 8) + COMBINED_INPUT_BYTES - 1) / COMBINED_INPUT_BYTES)),
+        };
+
+        typedef AgentReduceByKeyPolicy<
+                128,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_WARP_SCANS>
+            ReduceByKeyPolicyT;
+    };
+
+    /// SM13
+    struct Policy130
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 7,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, ((NOMINAL_4B_ITEMS_PER_THREAD * 8) + COMBINED_INPUT_BYTES - 1) / COMBINED_INPUT_BYTES)),
+        };
+
+        typedef AgentReduceByKeyPolicy<
+                128,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_WARP_SCANS>
+            ReduceByKeyPolicyT;
+    };
+
+    /// SM11
+    struct Policy110
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 5,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 8) / COMBINED_INPUT_BYTES)),
+        };
+
+        typedef AgentReduceByKeyPolicy<
+                64,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_RAKING>
+            ReduceByKeyPolicyT;
+    };
+
+
+    /******************************************************************************
+     * Tuning policies of current PTX compiler pass
+     ******************************************************************************/
+
+#if (CUB_PTX_ARCH >= 350)
+    typedef Policy350 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 300)
+    typedef Policy300 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 200)
+    typedef Policy200 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 130)
+    typedef Policy130 PtxPolicy;
+
+#else
+    typedef Policy110 PtxPolicy;
+
+#endif
+
+    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
+    struct PtxReduceByKeyPolicy : PtxPolicy::ReduceByKeyPolicyT {};
+
+
+    /******************************************************************************
+     * Utilities
+     ******************************************************************************/
+
+    /**
+     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
+     */
+    template <typename KernelConfig>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static void InitConfigs(
+        int             ptx_version,
+        KernelConfig    &reduce_by_key_config)
+    {
+    #if (CUB_PTX_ARCH > 0)
+        (void)ptx_version;
+
+        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
+        reduce_by_key_config.template Init<PtxReduceByKeyPolicy>();
+
+    #else
+
+        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
+        if (ptx_version >= 350)
+        {
+            reduce_by_key_config.template Init<typename Policy350::ReduceByKeyPolicyT>();
+        }
+        else if (ptx_version >= 300)
+        {
+            reduce_by_key_config.template Init<typename Policy300::ReduceByKeyPolicyT>();
+        }
+        else if (ptx_version >= 200)
+        {
+            reduce_by_key_config.template Init<typename Policy200::ReduceByKeyPolicyT>();
+        }
+        else if (ptx_version >= 130)
+        {
+            reduce_by_key_config.template Init<typename Policy130::ReduceByKeyPolicyT>();
+        }
+        else
+        {
+            reduce_by_key_config.template Init<typename Policy110::ReduceByKeyPolicyT>();
+        }
+
+    #endif
+    }
+
+
+    /**
+     * Kernel kernel dispatch configuration.
+     */
+    struct KernelConfig
+    {
+        int block_threads;
+        int items_per_thread;
+        int tile_items;
+
+        template <typename PolicyT>
+        CUB_RUNTIME_FUNCTION __forceinline__
+        void Init()
+        {
+            block_threads       = PolicyT::BLOCK_THREADS;
+            items_per_thread    = PolicyT::ITEMS_PER_THREAD;
+            tile_items          = block_threads * items_per_thread;
+        }
+    };
+
+
+    //---------------------------------------------------------------------
+    // Dispatch entrypoints
+    //---------------------------------------------------------------------
+
+    /**
+     * Internal dispatch routine for computing a device-wide reduce-by-key using the
+     * specified kernel functions.
+     */
+    template <
+        typename                    ScanInitKernelT,         ///< Function type of cub::DeviceScanInitKernel
+        typename                    ReduceByKeyKernelT>      ///< Function type of cub::DeviceReduceByKeyKernelT
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                       d_temp_storage,             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                     temp_storage_bytes,         ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        KeysInputIteratorT          d_keys_in,                  ///< [in] Pointer to the input sequence of keys
+        UniqueOutputIteratorT       d_unique_out,               ///< [out] Pointer to the output sequence of unique keys (one key per run)
+        ValuesInputIteratorT        d_values_in,                ///< [in] Pointer to the input sequence of corresponding values
+        AggregatesOutputIteratorT   d_aggregates_out,           ///< [out] Pointer to the output sequence of value aggregates (one aggregate per run)
+        NumRunsOutputIteratorT      d_num_runs_out,             ///< [out] Pointer to total number of runs encountered (i.e., the length of d_unique_out)
+        EqualityOpT                 equality_op,                ///< [in] KeyT equality operator
+        ReductionOpT                reduction_op,               ///< [in] ValueT reduction operator
+        OffsetT                     num_items,                  ///< [in] Total number of items to select from
+        cudaStream_t                stream,                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous,          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+        int                         /*ptx_version*/,            ///< [in] PTX version of dispatch kernels
+        ScanInitKernelT                init_kernel,                ///< [in] Kernel function pointer to parameterization of cub::DeviceScanInitKernel
+        ReduceByKeyKernelT             reduce_by_key_kernel,       ///< [in] Kernel function pointer to parameterization of cub::DeviceReduceByKeyKernel
+        KernelConfig                reduce_by_key_config)       ///< [in] Dispatch parameters that match the policy that \p reduce_by_key_kernel was compiled for
+    {
+
+#ifndef CUB_RUNTIME_ENABLED
+      (void)d_temp_storage;
+      (void)temp_storage_bytes;
+      (void)d_keys_in;
+      (void)d_unique_out;
+      (void)d_values_in;
+      (void)d_aggregates_out;
+      (void)d_num_runs_out;
+      (void)equality_op;
+      (void)reduction_op;
+      (void)num_items;
+      (void)stream;
+      (void)debug_synchronous;
+      (void)init_kernel;
+      (void)reduce_by_key_kernel;
+      (void)reduce_by_key_config;
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported);
+
+#else
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Number of input tiles
+            int tile_size = reduce_by_key_config.block_threads * reduce_by_key_config.items_per_thread;
+            int num_tiles = (num_items + tile_size - 1) / tile_size;
+
+            // Specify temporary storage allocation requirements
+            size_t  allocation_sizes[1];
+            if (CubDebug(error = ScanTileStateT::AllocationSize(num_tiles, allocation_sizes[0]))) break;    // bytes needed for tile status descriptors
+
+            // Compute allocation pointers into the single storage blob (or compute the necessary size of the blob)
+            void* allocations[1];
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+            if (d_temp_storage == NULL)
+            {
+                // Return if the caller is simply requesting the size of the storage allocation
+                break;
+            }
+
+            // Construct the tile status interface
+            ScanTileStateT tile_state;
+            if (CubDebug(error = tile_state.Init(num_tiles, allocations[0], allocation_sizes[0]))) break;
+
+            // Log init_kernel configuration
+            int init_grid_size = CUB_MAX(1, (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS);
+            if (debug_synchronous) _CubLog("Invoking init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream);
+
+            // Invoke init_kernel to initialize tile descriptors
+            init_kernel<<<init_grid_size, INIT_KERNEL_THREADS, 0, stream>>>(
+                tile_state,
+                num_tiles,
+                d_num_runs_out);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Return if empty problem
+            if (num_items == 0)
+                break;
+
+            // Get SM occupancy for reduce_by_key_kernel
+            int reduce_by_key_sm_occupancy;
+            if (CubDebug(error = MaxSmOccupancy(
+                reduce_by_key_sm_occupancy,            // out
+                reduce_by_key_kernel,
+                reduce_by_key_config.block_threads))) break;
+
+            // Get max x-dimension of grid
+            int max_dim_x;
+            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
+
+            // Run grids in epochs (in case number of tiles exceeds max x-dimension
+            int scan_grid_size = CUB_MIN(num_tiles, max_dim_x);
+            for (int start_tile = 0; start_tile < num_tiles; start_tile += scan_grid_size)
+            {
+                // Log reduce_by_key_kernel configuration
+                if (debug_synchronous) _CubLog("Invoking %d reduce_by_key_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                    start_tile, scan_grid_size, reduce_by_key_config.block_threads, (long long) stream, reduce_by_key_config.items_per_thread, reduce_by_key_sm_occupancy);
+
+                // Invoke reduce_by_key_kernel
+                reduce_by_key_kernel<<<scan_grid_size, reduce_by_key_config.block_threads, 0, stream>>>(
+                    d_keys_in,
+                    d_unique_out,
+                    d_values_in,
+                    d_aggregates_out,
+                    d_num_runs_out,
+                    tile_state,
+                    start_tile,
+                    equality_op,
+                    reduction_op,
+                    num_items);
+
+                // Check for failure to launch
+                if (CubDebug(error = cudaPeekAtLastError())) break;
+
+                // Sync the stream if specified to flush runtime errors
+                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+            }
+        }
+        while (0);
+
+        return error;
+
+#endif  // CUB_RUNTIME_ENABLED
+    }
+
+
+    /**
+     * Internal dispatch routine
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        KeysInputIteratorT          d_keys_in,                      ///< [in] Pointer to the input sequence of keys
+        UniqueOutputIteratorT       d_unique_out,                   ///< [out] Pointer to the output sequence of unique keys (one key per run)
+        ValuesInputIteratorT        d_values_in,                    ///< [in] Pointer to the input sequence of corresponding values
+        AggregatesOutputIteratorT   d_aggregates_out,               ///< [out] Pointer to the output sequence of value aggregates (one aggregate per run)
+        NumRunsOutputIteratorT      d_num_runs_out,                 ///< [out] Pointer to total number of runs encountered (i.e., the length of d_unique_out)
+        EqualityOpT                 equality_op,                    ///< [in] KeyT equality operator
+        ReductionOpT                reduction_op,                   ///< [in] ValueT reduction operator
+        OffsetT                     num_items,                      ///< [in] Total number of items to select from
+        cudaStream_t                stream,                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous)              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel kernel dispatch configurations
+            KernelConfig reduce_by_key_config;
+            InitConfigs(ptx_version, reduce_by_key_config);
+
+            // Dispatch
+            if (CubDebug(error = Dispatch(
+                d_temp_storage,
+                temp_storage_bytes,
+                d_keys_in,
+                d_unique_out,
+                d_values_in,
+                d_aggregates_out,
+                d_num_runs_out,
+                equality_op,
+                reduction_op,
+                num_items,
+                stream,
+                debug_synchronous,
+                ptx_version,
+                DeviceCompactInitKernel<ScanTileStateT, NumRunsOutputIteratorT>,
+                DeviceReduceByKeyKernel<PtxReduceByKeyPolicy, KeysInputIteratorT, UniqueOutputIteratorT, ValuesInputIteratorT, AggregatesOutputIteratorT, NumRunsOutputIteratorT, ScanTileStateT, EqualityOpT, ReductionOpT, OffsetT>,
+                reduce_by_key_config))) break;
+        }
+        while (0);
+
+        return error;
+    }
+};
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/dispatch/dispatch_rle.cuh b/third_party/cub/cub/device/dispatch/dispatch_rle.cuh
new file mode 100644
index 0000000..98c3681
--- /dev/null
+++ b/third_party/cub/cub/device/dispatch/dispatch_rle.cuh
@@ -0,0 +1,538 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceRle provides device-wide, parallel operations for run-length-encoding sequences of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch_scan.cuh"
+#include "../../agent/agent_rle.cuh"
+#include "../../thread/thread_operators.cuh"
+#include "../../grid/grid_queue.cuh"
+#include "../../util_device.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Kernel entry points
+ *****************************************************************************/
+
+/**
+ * Select kernel entry point (multi-block)
+ *
+ * Performs functor-based selection if SelectOp functor type != NullType
+ * Otherwise performs flag-based selection if FlagIterator's value type != NullType
+ * Otherwise performs discontinuity selection (keep unique)
+ */
+template <
+    typename            AgentRlePolicyT,        ///< Parameterized AgentRlePolicyT tuning policy type
+    typename            InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
+    typename            OffsetsOutputIteratorT,     ///< Random-access output iterator type for writing run-offset values \iterator
+    typename            LengthsOutputIteratorT,     ///< Random-access output iterator type for writing run-length values \iterator
+    typename            NumRunsOutputIteratorT,     ///< Output iterator type for recording the number of runs encountered \iterator
+    typename            ScanTileStateT,              ///< Tile status interface type
+    typename            EqualityOpT,                 ///< T equality operator type
+    typename            OffsetT>                    ///< Signed integer type for global offsets
+__launch_bounds__ (int(AgentRlePolicyT::BLOCK_THREADS))
+__global__ void DeviceRleSweepKernel(
+    InputIteratorT              d_in,               ///< [in] Pointer to input sequence of data items
+    OffsetsOutputIteratorT      d_offsets_out,      ///< [out] Pointer to output sequence of run-offsets
+    LengthsOutputIteratorT      d_lengths_out,      ///< [out] Pointer to output sequence of run-lengths
+    NumRunsOutputIteratorT      d_num_runs_out,     ///< [out] Pointer to total number of runs (i.e., length of \p d_offsets_out)
+    ScanTileStateT              tile_status,        ///< [in] Tile status interface
+    EqualityOpT                 equality_op,        ///< [in] Equality operator for input items
+    OffsetT                     num_items,          ///< [in] Total number of input items (i.e., length of \p d_in)
+    int                         num_tiles)          ///< [in] Total number of tiles for the entire problem
+{
+    // Thread block type for selecting data from input tiles
+    typedef AgentRle<
+        AgentRlePolicyT,
+        InputIteratorT,
+        OffsetsOutputIteratorT,
+        LengthsOutputIteratorT,
+        EqualityOpT,
+        OffsetT> AgentRleT;
+
+    // Shared memory for AgentRle
+    __shared__ typename AgentRleT::TempStorage temp_storage;
+
+    // Process tiles
+    AgentRleT(temp_storage, d_in, d_offsets_out, d_lengths_out, equality_op, num_items).ConsumeRange(
+        num_tiles,
+        tile_status,
+        d_num_runs_out);
+}
+
+
+
+
+/******************************************************************************
+ * Dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for DeviceRle
+ */
+template <
+    typename            InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
+    typename            OffsetsOutputIteratorT,     ///< Random-access output iterator type for writing run-offset values \iterator
+    typename            LengthsOutputIteratorT,     ///< Random-access output iterator type for writing run-length values \iterator
+    typename            NumRunsOutputIteratorT,     ///< Output iterator type for recording the number of runs encountered \iterator
+    typename            EqualityOpT,                ///< T equality operator type
+    typename            OffsetT>                    ///< Signed integer type for global offsets
+struct DeviceRleDispatch
+{
+    /******************************************************************************
+     * Types and constants
+     ******************************************************************************/
+
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type T;
+
+    // The lengths output value type
+    typedef typename If<(Equals<typename std::iterator_traits<LengthsOutputIteratorT>::value_type, void>::VALUE),   // LengthT =  (if output iterator's value type is void) ?
+        OffsetT,                                                                                                    // ... then the OffsetT type,
+        typename std::iterator_traits<LengthsOutputIteratorT>::value_type>::Type LengthT;                           // ... else the output iterator's value type
+
+    enum
+    {
+        INIT_KERNEL_THREADS = 128,
+    };
+
+    // Tile status descriptor interface type
+    typedef ReduceByKeyScanTileState<LengthT, OffsetT> ScanTileStateT;
+
+
+    /******************************************************************************
+     * Tuning policies
+     ******************************************************************************/
+
+    /// SM35
+    struct Policy350
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 15,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
+        };
+
+        typedef AgentRlePolicy<
+                96,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_DIRECT,
+                LOAD_LDG,
+                true,
+                BLOCK_SCAN_WARP_SCANS>
+            RleSweepPolicy;
+    };
+
+    /// SM30
+    struct Policy300
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 5,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
+        };
+
+        typedef AgentRlePolicy<
+                256,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                true,
+                BLOCK_SCAN_RAKING_MEMOIZE>
+            RleSweepPolicy;
+    };
+
+    /// SM20
+    struct Policy200
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 15,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
+        };
+
+        typedef AgentRlePolicy<
+                128,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                false,
+                BLOCK_SCAN_WARP_SCANS>
+            RleSweepPolicy;
+    };
+
+    /// SM13
+    struct Policy130
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 9,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
+        };
+
+        typedef AgentRlePolicy<
+                64,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                true,
+                BLOCK_SCAN_RAKING_MEMOIZE>
+            RleSweepPolicy;
+    };
+
+    /// SM10
+    struct Policy100
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 9,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
+        };
+
+        typedef AgentRlePolicy<
+                256,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                true,
+                BLOCK_SCAN_RAKING_MEMOIZE>
+            RleSweepPolicy;
+    };
+
+
+    /******************************************************************************
+     * Tuning policies of current PTX compiler pass
+     ******************************************************************************/
+
+#if (CUB_PTX_ARCH >= 350)
+    typedef Policy350 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 300)
+    typedef Policy300 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 200)
+    typedef Policy200 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 130)
+    typedef Policy130 PtxPolicy;
+
+#else
+    typedef Policy100 PtxPolicy;
+
+#endif
+
+    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
+    struct PtxRleSweepPolicy : PtxPolicy::RleSweepPolicy {};
+
+
+    /******************************************************************************
+     * Utilities
+     ******************************************************************************/
+
+    /**
+     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
+     */
+    template <typename KernelConfig>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static void InitConfigs(
+        int             ptx_version,
+        KernelConfig&   device_rle_config)
+    {
+    #if (CUB_PTX_ARCH > 0)
+
+        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
+        device_rle_config.template Init<PtxRleSweepPolicy>();
+
+    #else
+
+        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
+        if (ptx_version >= 350)
+        {
+            device_rle_config.template Init<typename Policy350::RleSweepPolicy>();
+        }
+        else if (ptx_version >= 300)
+        {
+            device_rle_config.template Init<typename Policy300::RleSweepPolicy>();
+        }
+        else if (ptx_version >= 200)
+        {
+            device_rle_config.template Init<typename Policy200::RleSweepPolicy>();
+        }
+        else if (ptx_version >= 130)
+        {
+            device_rle_config.template Init<typename Policy130::RleSweepPolicy>();
+        }
+        else
+        {
+            device_rle_config.template Init<typename Policy100::RleSweepPolicy>();
+        }
+
+    #endif
+    }
+
+
+    /**
+     * Kernel kernel dispatch configuration.  Mirrors the constants within AgentRlePolicyT.
+     */
+    struct KernelConfig
+    {
+        int                     block_threads;
+        int                     items_per_thread;
+        BlockLoadAlgorithm      load_policy;
+        bool                    store_warp_time_slicing;
+        BlockScanAlgorithm      scan_algorithm;
+
+        template <typename AgentRlePolicyT>
+        CUB_RUNTIME_FUNCTION __forceinline__
+        void Init()
+        {
+            block_threads               = AgentRlePolicyT::BLOCK_THREADS;
+            items_per_thread            = AgentRlePolicyT::ITEMS_PER_THREAD;
+            load_policy                 = AgentRlePolicyT::LOAD_ALGORITHM;
+            store_warp_time_slicing     = AgentRlePolicyT::STORE_WARP_TIME_SLICING;
+            scan_algorithm              = AgentRlePolicyT::SCAN_ALGORITHM;
+        }
+
+        CUB_RUNTIME_FUNCTION __forceinline__
+        void Print()
+        {
+            printf("%d, %d, %d, %d, %d",
+                block_threads,
+                items_per_thread,
+                load_policy,
+                store_warp_time_slicing,
+                scan_algorithm);
+        }
+    };
+
+
+    /******************************************************************************
+     * Dispatch entrypoints
+     ******************************************************************************/
+
+    /**
+     * Internal dispatch routine for computing a device-wide run-length-encode using the
+     * specified kernel functions.
+     */
+    template <
+        typename                    DeviceScanInitKernelPtr,        ///< Function type of cub::DeviceScanInitKernel
+        typename                    DeviceRleSweepKernelPtr>        ///< Function type of cub::DeviceRleSweepKernelPtr
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
+        OffsetsOutputIteratorT      d_offsets_out,                  ///< [out] Pointer to the output sequence of run-offsets
+        LengthsOutputIteratorT      d_lengths_out,                  ///< [out] Pointer to the output sequence of run-lengths
+        NumRunsOutputIteratorT      d_num_runs_out,                 ///< [out] Pointer to the total number of runs encountered (i.e., length of \p d_offsets_out)
+        EqualityOpT                 equality_op,                    ///< [in] Equality operator for input items
+        OffsetT                     num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream,                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous,              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+        int                         ptx_version,                    ///< [in] PTX version of dispatch kernels
+        DeviceScanInitKernelPtr     device_scan_init_kernel,        ///< [in] Kernel function pointer to parameterization of cub::DeviceScanInitKernel
+        DeviceRleSweepKernelPtr     device_rle_sweep_kernel,        ///< [in] Kernel function pointer to parameterization of cub::DeviceRleSweepKernel
+        KernelConfig                device_rle_config)              ///< [in] Dispatch parameters that match the policy that \p device_rle_sweep_kernel was compiled for
+    {
+
+#ifndef CUB_RUNTIME_ENABLED
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported);
+
+#else
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Number of input tiles
+            int tile_size = device_rle_config.block_threads * device_rle_config.items_per_thread;
+            int num_tiles = (num_items + tile_size - 1) / tile_size;
+
+            // Specify temporary storage allocation requirements
+            size_t  allocation_sizes[1];
+            if (CubDebug(error = ScanTileStateT::AllocationSize(num_tiles, allocation_sizes[0]))) break;    // bytes needed for tile status descriptors
+
+            // Compute allocation pointers into the single storage blob (or compute the necessary size of the blob)
+            void* allocations[1];
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+            if (d_temp_storage == NULL)
+            {
+                // Return if the caller is simply requesting the size of the storage allocation
+                break;
+            }
+
+            // Construct the tile status interface
+            ScanTileStateT tile_status;
+            if (CubDebug(error = tile_status.Init(num_tiles, allocations[0], allocation_sizes[0]))) break;
+
+            // Log device_scan_init_kernel configuration
+            int init_grid_size = CUB_MAX(1, (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS);
+            if (debug_synchronous) _CubLog("Invoking device_scan_init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream);
+
+            // Invoke device_scan_init_kernel to initialize tile descriptors and queue descriptors
+            device_scan_init_kernel<<<init_grid_size, INIT_KERNEL_THREADS, 0, stream>>>(
+                tile_status,
+                num_tiles,
+                d_num_runs_out);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Return if empty problem
+            if (num_items == 0)
+                break;
+
+            // Get SM occupancy for device_rle_sweep_kernel
+            int device_rle_kernel_sm_occupancy;
+            if (CubDebug(error = MaxSmOccupancy(
+                device_rle_kernel_sm_occupancy,            // out
+                device_rle_sweep_kernel,
+                device_rle_config.block_threads))) break;
+
+            // Get max x-dimension of grid
+            int max_dim_x;
+            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
+
+            // Get grid size for scanning tiles
+            dim3 scan_grid_size;
+            scan_grid_size.z = 1;
+            scan_grid_size.y = ((unsigned int) num_tiles + max_dim_x - 1) / max_dim_x;
+            scan_grid_size.x = CUB_MIN(num_tiles, max_dim_x);
+
+            // Log device_rle_sweep_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking device_rle_sweep_kernel<<<{%d,%d,%d}, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                scan_grid_size.x, scan_grid_size.y, scan_grid_size.z, device_rle_config.block_threads, (long long) stream, device_rle_config.items_per_thread, device_rle_kernel_sm_occupancy);
+
+            // Invoke device_rle_sweep_kernel
+            device_rle_sweep_kernel<<<scan_grid_size, device_rle_config.block_threads, 0, stream>>>(
+                d_in,
+                d_offsets_out,
+                d_lengths_out,
+                d_num_runs_out,
+                tile_status,
+                equality_op,
+                num_items,
+                num_tiles);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+        }
+        while (0);
+
+        return error;
+
+#endif  // CUB_RUNTIME_ENABLED
+    }
+
+
+    /**
+     * Internal dispatch routine
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to input sequence of data items
+        OffsetsOutputIteratorT      d_offsets_out,                  ///< [out] Pointer to output sequence of run-offsets
+        LengthsOutputIteratorT      d_lengths_out,                  ///< [out] Pointer to output sequence of run-lengths
+        NumRunsOutputIteratorT      d_num_runs_out,                 ///< [out] Pointer to total number of runs (i.e., length of \p d_offsets_out)
+        EqualityOpT                 equality_op,                    ///< [in] Equality operator for input items
+        OffsetT                     num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream,                         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous)              ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel kernel dispatch configurations
+            KernelConfig device_rle_config;
+            InitConfigs(ptx_version, device_rle_config);
+
+            // Dispatch
+            if (CubDebug(error = Dispatch(
+                d_temp_storage,
+                temp_storage_bytes,
+                d_in,
+                d_offsets_out,
+                d_lengths_out,
+                d_num_runs_out,
+                equality_op,
+                num_items,
+                stream,
+                debug_synchronous,
+                ptx_version,
+                DeviceCompactInitKernel<ScanTileStateT, NumRunsOutputIteratorT>,
+                DeviceRleSweepKernel<PtxRleSweepPolicy, InputIteratorT, OffsetsOutputIteratorT, LengthsOutputIteratorT, NumRunsOutputIteratorT, ScanTileStateT, EqualityOpT, OffsetT>,
+                device_rle_config))) break;
+        }
+        while (0);
+
+        return error;
+    }
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/dispatch/dispatch_scan.cuh b/third_party/cub/cub/device/dispatch/dispatch_scan.cuh
new file mode 100644
index 0000000..3ef720a
--- /dev/null
+++ b/third_party/cub/cub/device/dispatch/dispatch_scan.cuh
@@ -0,0 +1,563 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceScan provides device-wide, parallel operations for computing a prefix scan across a sequence of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "../../agent/agent_scan.cuh"
+#include "../../thread/thread_operators.cuh"
+#include "../../grid/grid_queue.cuh"
+#include "../../util_arch.cuh"
+#include "../../util_debug.cuh"
+#include "../../util_device.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Kernel entry points
+ *****************************************************************************/
+
+/**
+ * Initialization kernel for tile status initialization (multi-block)
+ */
+template <
+    typename            ScanTileStateT>     ///< Tile status interface type
+__global__ void DeviceScanInitKernel(
+    ScanTileStateT      tile_state,         ///< [in] Tile status interface
+    int                 num_tiles)          ///< [in] Number of tiles
+{
+    // Initialize tile status
+    tile_state.InitializeStatus(num_tiles);
+}
+
+/**
+ * Initialization kernel for tile status initialization (multi-block)
+ */
+template <
+    typename                ScanTileStateT,         ///< Tile status interface type
+    typename                NumSelectedIteratorT>   ///< Output iterator type for recording the number of items selected
+__global__ void DeviceCompactInitKernel(
+    ScanTileStateT          tile_state,             ///< [in] Tile status interface
+    int                     num_tiles,              ///< [in] Number of tiles
+    NumSelectedIteratorT    d_num_selected_out)     ///< [out] Pointer to the total number of items selected (i.e., length of \p d_selected_out)
+{
+    // Initialize tile status
+    tile_state.InitializeStatus(num_tiles);
+
+    // Initialize d_num_selected_out
+    if ((blockIdx.x == 0) && (threadIdx.x == 0))
+        *d_num_selected_out = 0;
+}
+
+
+/**
+ * Scan kernel entry point (multi-block)
+ */
+template <
+    typename            ScanPolicyT,        ///< Parameterized ScanPolicyT tuning policy type
+    typename            InputIteratorT,     ///< Random-access input iterator type for reading scan inputs \iterator
+    typename            OutputIteratorT,    ///< Random-access output iterator type for writing scan outputs \iterator
+    typename            ScanTileStateT,     ///< Tile status interface type
+    typename            ScanOpT,            ///< Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+    typename            InitValueT,         ///< Initial value to seed the exclusive scan (cub::NullType for inclusive scans)
+    typename            OffsetT>            ///< Signed integer type for global offsets
+__launch_bounds__ (int(ScanPolicyT::BLOCK_THREADS))
+__global__ void DeviceScanKernel(
+    InputIteratorT      d_in,               ///< Input data
+    OutputIteratorT     d_out,              ///< Output data
+    ScanTileStateT      tile_state,         ///< Tile status interface
+    int                 start_tile,         ///< The starting tile for the current grid
+    ScanOpT             scan_op,            ///< Binary scan functor 
+    InitValueT          init_value,         ///< Initial value to seed the exclusive scan
+    OffsetT             num_items)          ///< Total number of scan items for the entire problem
+{
+    // Thread block type for scanning input tiles
+    typedef AgentScan<
+        ScanPolicyT,
+        InputIteratorT,
+        OutputIteratorT,
+        ScanOpT,
+        InitValueT,
+        OffsetT> AgentScanT;
+
+    // Shared memory for AgentScan
+    __shared__ typename AgentScanT::TempStorage temp_storage;
+
+    // Process tiles
+    AgentScanT(temp_storage, d_in, d_out, scan_op, init_value).ConsumeRange(
+        num_items,
+        tile_state,
+        start_tile);
+}
+
+
+
+
+/******************************************************************************
+ * Dispatch
+ ******************************************************************************/
+
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for DeviceScan
+ */
+template <
+    typename InputIteratorT,     ///< Random-access input iterator type for reading scan inputs \iterator
+    typename OutputIteratorT,    ///< Random-access output iterator type for writing scan outputs \iterator
+    typename ScanOpT,            ///< Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
+    typename InitValueT,          ///< The init_value element type for ScanOpT (cub::NullType for inclusive scans)
+    typename OffsetT>            ///< Signed integer type for global offsets
+struct DispatchScan
+{
+    //---------------------------------------------------------------------
+    // Constants and Types
+    //---------------------------------------------------------------------
+
+    enum
+    {
+        INIT_KERNEL_THREADS = 128
+    };
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    // Tile status descriptor interface type
+    typedef ScanTileState<OutputT> ScanTileStateT;
+
+
+    //---------------------------------------------------------------------
+    // Tuning policies
+    //---------------------------------------------------------------------
+
+    /// SM600
+    struct Policy600
+    {
+        typedef AgentScanPolicy<
+            CUB_SCALED_GRANULARITIES(128, 15, OutputT),      ///< Threads per block, items per thread
+                BLOCK_LOAD_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_STORE_TRANSPOSE,
+                BLOCK_SCAN_WARP_SCANS>
+            ScanPolicyT;
+    };
+
+
+    /// SM520
+    struct Policy520
+    {
+        // Titan X: 32.47B items/s @ 48M 32-bit T
+        typedef AgentScanPolicy<
+                CUB_SCALED_GRANULARITIES(128, 12, OutputT),      ///< Threads per block, items per thread
+                BLOCK_LOAD_DIRECT,
+                LOAD_LDG,
+                BLOCK_STORE_WARP_TRANSPOSE,
+                BLOCK_SCAN_WARP_SCANS>
+            ScanPolicyT;
+    };
+
+
+    /// SM35
+    struct Policy350
+    {
+        // GTX Titan: 29.5B items/s (232.4 GB/s) @ 48M 32-bit T
+        typedef AgentScanPolicy<
+                CUB_SCALED_GRANULARITIES(128, 12, OutputT),      ///< Threads per block, items per thread
+                BLOCK_LOAD_DIRECT,
+                LOAD_LDG,
+                BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED,
+                BLOCK_SCAN_RAKING>
+            ScanPolicyT;
+    };
+
+    /// SM30
+    struct Policy300
+    {
+        typedef AgentScanPolicy<
+                CUB_SCALED_GRANULARITIES(256, 9, OutputT),      ///< Threads per block, items per thread
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_STORE_WARP_TRANSPOSE,
+                BLOCK_SCAN_WARP_SCANS>
+            ScanPolicyT;
+    };
+
+    /// SM20
+    struct Policy200
+    {
+        // GTX 580: 20.3B items/s (162.3 GB/s) @ 48M 32-bit T
+        typedef AgentScanPolicy<
+                CUB_SCALED_GRANULARITIES(128, 12, OutputT),      ///< Threads per block, items per thread
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_STORE_WARP_TRANSPOSE,
+                BLOCK_SCAN_WARP_SCANS>
+            ScanPolicyT;
+    };
+
+    /// SM13
+    struct Policy130
+    {
+        typedef AgentScanPolicy<
+                CUB_SCALED_GRANULARITIES(96, 21, OutputT),      ///< Threads per block, items per thread
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_STORE_WARP_TRANSPOSE,
+                BLOCK_SCAN_RAKING_MEMOIZE>
+            ScanPolicyT;
+    };
+
+    /// SM10
+    struct Policy100
+    {
+        typedef AgentScanPolicy<
+                CUB_SCALED_GRANULARITIES(64, 9, OutputT),      ///< Threads per block, items per thread
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_STORE_WARP_TRANSPOSE,
+                BLOCK_SCAN_WARP_SCANS>
+            ScanPolicyT;
+    };
+
+
+    //---------------------------------------------------------------------
+    // Tuning policies of current PTX compiler pass
+    //---------------------------------------------------------------------
+
+#if (CUB_PTX_ARCH >= 600)
+    typedef Policy600 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 520)
+    typedef Policy520 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 350)
+    typedef Policy350 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 300)
+    typedef Policy300 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 200)
+    typedef Policy200 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 130)
+    typedef Policy130 PtxPolicy;
+
+#else
+    typedef Policy100 PtxPolicy;
+
+#endif
+
+    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
+    struct PtxAgentScanPolicy : PtxPolicy::ScanPolicyT {};
+
+
+    //---------------------------------------------------------------------
+    // Utilities
+    //---------------------------------------------------------------------
+
+    /**
+     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
+     */
+    template <typename KernelConfig>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static void InitConfigs(
+        int             ptx_version,
+        KernelConfig    &scan_kernel_config)
+    {
+    #if (CUB_PTX_ARCH > 0)
+        (void)ptx_version;
+
+        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
+        scan_kernel_config.template Init<PtxAgentScanPolicy>();
+
+    #else
+
+        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
+        if (ptx_version >= 600)
+        {
+            scan_kernel_config.template Init<typename Policy600::ScanPolicyT>();
+        }
+        else if (ptx_version >= 520)
+        {
+            scan_kernel_config.template Init<typename Policy520::ScanPolicyT>();
+        }
+        else if (ptx_version >= 350)
+        {
+            scan_kernel_config.template Init<typename Policy350::ScanPolicyT>();
+        }
+        else if (ptx_version >= 300)
+        {
+            scan_kernel_config.template Init<typename Policy300::ScanPolicyT>();
+        }
+        else if (ptx_version >= 200)
+        {
+            scan_kernel_config.template Init<typename Policy200::ScanPolicyT>();
+        }
+        else if (ptx_version >= 130)
+        {
+            scan_kernel_config.template Init<typename Policy130::ScanPolicyT>();
+        }
+        else
+        {
+            scan_kernel_config.template Init<typename Policy100::ScanPolicyT>();
+        }
+
+    #endif
+    }
+
+
+    /**
+     * Kernel kernel dispatch configuration.
+     */
+    struct KernelConfig
+    {
+        int block_threads;
+        int items_per_thread;
+        int tile_items;
+
+        template <typename PolicyT>
+        CUB_RUNTIME_FUNCTION __forceinline__
+        void Init()
+        {
+            block_threads       = PolicyT::BLOCK_THREADS;
+            items_per_thread    = PolicyT::ITEMS_PER_THREAD;
+            tile_items          = block_threads * items_per_thread;
+        }
+    };
+
+
+    //---------------------------------------------------------------------
+    // Dispatch entrypoints
+    //---------------------------------------------------------------------
+
+    /**
+     * Internal dispatch routine for computing a device-wide prefix scan using the
+     * specified kernel functions.
+     */
+    template <
+        typename            ScanInitKernelPtrT,     ///< Function type of cub::DeviceScanInitKernel
+        typename            ScanSweepKernelPtrT>    ///< Function type of cub::DeviceScanKernelPtrT
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*               d_temp_storage,         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&             temp_storage_bytes,     ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT      d_in,                   ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT     d_out,                  ///< [out] Pointer to the output sequence of data items
+        ScanOpT             scan_op,                ///< [in] Binary scan functor 
+        InitValueT          init_value,             ///< [in] Initial value to seed the exclusive scan
+        OffsetT             num_items,              ///< [in] Total number of input items (i.e., the length of \p d_in)
+        cudaStream_t        stream,                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                debug_synchronous,      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+        int                 /*ptx_version*/,        ///< [in] PTX version of dispatch kernels
+        ScanInitKernelPtrT  init_kernel,            ///< [in] Kernel function pointer to parameterization of cub::DeviceScanInitKernel
+        ScanSweepKernelPtrT scan_kernel,            ///< [in] Kernel function pointer to parameterization of cub::DeviceScanKernel
+        KernelConfig        scan_kernel_config)     ///< [in] Dispatch parameters that match the policy that \p scan_kernel was compiled for
+    {
+
+#ifndef CUB_RUNTIME_ENABLED
+        (void)d_temp_storage;
+        (void)temp_storage_bytes;
+        (void)d_in;
+        (void)d_out;
+        (void)scan_op;
+        (void)init_value;
+        (void)num_items;
+        (void)stream;
+        (void)debug_synchronous;
+        (void)init_kernel;
+        (void)scan_kernel;
+        (void)scan_kernel_config;
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported);
+
+#else
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Number of input tiles
+            int tile_size = scan_kernel_config.block_threads * scan_kernel_config.items_per_thread;
+            int num_tiles = (num_items + tile_size - 1) / tile_size;
+
+            // Specify temporary storage allocation requirements
+            size_t  allocation_sizes[1];
+            if (CubDebug(error = ScanTileStateT::AllocationSize(num_tiles, allocation_sizes[0]))) break;    // bytes needed for tile status descriptors
+
+            // Compute allocation pointers into the single storage blob (or compute the necessary size of the blob)
+            void* allocations[1];
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+            if (d_temp_storage == NULL)
+            {
+                // Return if the caller is simply requesting the size of the storage allocation
+                break;
+            }
+
+            // Return if empty problem
+            if (num_items == 0)
+                break;
+
+            // Construct the tile status interface
+            ScanTileStateT tile_state;
+            if (CubDebug(error = tile_state.Init(num_tiles, allocations[0], allocation_sizes[0]))) break;
+
+            // Log init_kernel configuration
+            int init_grid_size = (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS;
+            if (debug_synchronous) _CubLog("Invoking init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream);
+
+            // Invoke init_kernel to initialize tile descriptors
+            init_kernel<<<init_grid_size, INIT_KERNEL_THREADS, 0, stream>>>(
+                tile_state,
+                num_tiles);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Get SM occupancy for scan_kernel
+            int scan_sm_occupancy;
+            if (CubDebug(error = MaxSmOccupancy(
+                scan_sm_occupancy,            // out
+                scan_kernel,
+                scan_kernel_config.block_threads))) break;
+
+            // Get max x-dimension of grid
+            int max_dim_x;
+            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
+
+            // Run grids in epochs (in case number of tiles exceeds max x-dimension
+            int scan_grid_size = CUB_MIN(num_tiles, max_dim_x);
+            for (int start_tile = 0; start_tile < num_tiles; start_tile += scan_grid_size)
+            {
+                // Log scan_kernel configuration
+                if (debug_synchronous) _CubLog("Invoking %d scan_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                    start_tile, scan_grid_size, scan_kernel_config.block_threads, (long long) stream, scan_kernel_config.items_per_thread, scan_sm_occupancy);
+
+                // Invoke scan_kernel
+                scan_kernel<<<scan_grid_size, scan_kernel_config.block_threads, 0, stream>>>(
+                    d_in,
+                    d_out,
+                    tile_state,
+                    start_tile,
+                    scan_op,
+                    init_value,
+                    num_items);
+
+                // Check for failure to launch
+                if (CubDebug(error = cudaPeekAtLastError())) break;
+
+                // Sync the stream if specified to flush runtime errors
+                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+            }
+        }
+        while (0);
+
+        return error;
+
+#endif  // CUB_RUNTIME_ENABLED
+    }
+
+
+    /**
+     * Internal dispatch routine
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*           d_temp_storage,         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&         temp_storage_bytes,     ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT  d_in,                   ///< [in] Pointer to the input sequence of data items
+        OutputIteratorT d_out,                  ///< [out] Pointer to the output sequence of data items
+        ScanOpT         scan_op,                ///< [in] Binary scan functor 
+        InitValueT      init_value,             ///< [in] Initial value to seed the exclusive scan
+        OffsetT         num_items,              ///< [in] Total number of input items (i.e., the length of \p d_in)
+        cudaStream_t    stream,                 ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool            debug_synchronous)      ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+
+            // Get kernel kernel dispatch configurations
+            KernelConfig scan_kernel_config;
+            InitConfigs(ptx_version, scan_kernel_config);
+
+            // Dispatch
+            if (CubDebug(error = Dispatch(
+                d_temp_storage,
+                temp_storage_bytes,
+                d_in,
+                d_out,
+                scan_op,
+                init_value,
+                num_items,
+                stream,
+                debug_synchronous,
+                ptx_version,
+                DeviceScanInitKernel<ScanTileStateT>,
+                DeviceScanKernel<PtxAgentScanPolicy, InputIteratorT, OutputIteratorT, ScanTileStateT, ScanOpT, InitValueT, OffsetT>,
+                scan_kernel_config))) break;
+        }
+        while (0);
+
+        return error;
+    }
+};
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/dispatch/dispatch_select_if.cuh b/third_party/cub/cub/device/dispatch/dispatch_select_if.cuh
new file mode 100644
index 0000000..60b3313
--- /dev/null
+++ b/third_party/cub/cub/device/dispatch/dispatch_select_if.cuh
@@ -0,0 +1,542 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceSelect provides device-wide, parallel operations for selecting items from sequences of data items residing within device-accessible memory.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "dispatch_scan.cuh"
+#include "../../agent/agent_select_if.cuh"
+#include "../../thread/thread_operators.cuh"
+#include "../../grid/grid_queue.cuh"
+#include "../../util_device.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/******************************************************************************
+ * Kernel entry points
+ *****************************************************************************/
+
+/**
+ * Select kernel entry point (multi-block)
+ *
+ * Performs functor-based selection if SelectOpT functor type != NullType
+ * Otherwise performs flag-based selection if FlagsInputIterator's value type != NullType
+ * Otherwise performs discontinuity selection (keep unique)
+ */
+template <
+    typename            AgentSelectIfPolicyT,       ///< Parameterized AgentSelectIfPolicyT tuning policy type
+    typename            InputIteratorT,             ///< Random-access input iterator type for reading input items
+    typename            FlagsInputIteratorT,        ///< Random-access input iterator type for reading selection flags (NullType* if a selection functor or discontinuity flagging is to be used for selection)
+    typename            SelectedOutputIteratorT,    ///< Random-access output iterator type for writing selected items
+    typename            NumSelectedIteratorT,       ///< Output iterator type for recording the number of items selected
+    typename            ScanTileStateT,             ///< Tile status interface type
+    typename            SelectOpT,                  ///< Selection operator type (NullType if selection flags or discontinuity flagging is to be used for selection)
+    typename            EqualityOpT,                ///< Equality operator type (NullType if selection functor or selection flags is to be used for selection)
+    typename            OffsetT,                    ///< Signed integer type for global offsets
+    bool                KEEP_REJECTS>               ///< Whether or not we push rejected items to the back of the output
+__launch_bounds__ (int(AgentSelectIfPolicyT::BLOCK_THREADS))
+__global__ void DeviceSelectSweepKernel(
+    InputIteratorT          d_in,                   ///< [in] Pointer to the input sequence of data items
+    FlagsInputIteratorT     d_flags,                ///< [in] Pointer to the input sequence of selection flags (if applicable)
+    SelectedOutputIteratorT d_selected_out,         ///< [out] Pointer to the output sequence of selected data items
+    NumSelectedIteratorT    d_num_selected_out,     ///< [out] Pointer to the total number of items selected (i.e., length of \p d_selected_out)
+    ScanTileStateT          tile_status,            ///< [in] Tile status interface
+    SelectOpT               select_op,              ///< [in] Selection operator
+    EqualityOpT             equality_op,            ///< [in] Equality operator
+    OffsetT                 num_items,              ///< [in] Total number of input items (i.e., length of \p d_in)
+    int                     num_tiles)              ///< [in] Total number of tiles for the entire problem
+{
+    // Thread block type for selecting data from input tiles
+    typedef AgentSelectIf<
+        AgentSelectIfPolicyT,
+        InputIteratorT,
+        FlagsInputIteratorT,
+        SelectedOutputIteratorT,
+        SelectOpT,
+        EqualityOpT,
+        OffsetT,
+        KEEP_REJECTS> AgentSelectIfT;
+
+    // Shared memory for AgentSelectIf
+    __shared__ typename AgentSelectIfT::TempStorage temp_storage;
+
+    // Process tiles
+    AgentSelectIfT(temp_storage, d_in, d_flags, d_selected_out, select_op, equality_op, num_items).ConsumeRange(
+        num_tiles,
+        tile_status,
+        d_num_selected_out);
+}
+
+
+
+
+/******************************************************************************
+ * Dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for DeviceSelect
+ */
+template <
+    typename    InputIteratorT,                 ///< Random-access input iterator type for reading input items
+    typename    FlagsInputIteratorT,            ///< Random-access input iterator type for reading selection flags (NullType* if a selection functor or discontinuity flagging is to be used for selection)
+    typename    SelectedOutputIteratorT,        ///< Random-access output iterator type for writing selected items
+    typename    NumSelectedIteratorT,           ///< Output iterator type for recording the number of items selected
+    typename    SelectOpT,                      ///< Selection operator type (NullType if selection flags or discontinuity flagging is to be used for selection)
+    typename    EqualityOpT,                    ///< Equality operator type (NullType if selection functor or selection flags is to be used for selection)
+    typename    OffsetT,                        ///< Signed integer type for global offsets
+    bool        KEEP_REJECTS>                   ///< Whether or not we push rejected items to the back of the output
+struct DispatchSelectIf
+{
+    /******************************************************************************
+     * Types and constants
+     ******************************************************************************/
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<SelectedOutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                                  // ... then the input iterator's value type,
+        typename std::iterator_traits<SelectedOutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    // The flag value type
+    typedef typename std::iterator_traits<FlagsInputIteratorT>::value_type FlagT;
+
+    enum
+    {
+        INIT_KERNEL_THREADS = 128,
+    };
+
+    // Tile status descriptor interface type
+    typedef ScanTileState<OffsetT> ScanTileStateT;
+
+
+    /******************************************************************************
+     * Tuning policies
+     ******************************************************************************/
+
+    /// SM35
+    struct Policy350
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 10,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
+        };
+
+        typedef AgentSelectIfPolicy<
+                128,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_DIRECT,
+                LOAD_LDG,
+                BLOCK_SCAN_WARP_SCANS>
+            SelectIfPolicyT;
+    };
+
+    /// SM30
+    struct Policy300
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 7,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(3, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
+        };
+
+        typedef AgentSelectIfPolicy<
+                128,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_WARP_SCANS>
+            SelectIfPolicyT;
+    };
+
+    /// SM20
+    struct Policy200
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = (KEEP_REJECTS) ? 7 : 15,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
+        };
+
+        typedef AgentSelectIfPolicy<
+                128,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_WARP_SCANS>
+            SelectIfPolicyT;
+    };
+
+    /// SM13
+    struct Policy130
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 9,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
+        };
+
+        typedef AgentSelectIfPolicy<
+                64,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_RAKING_MEMOIZE>
+            SelectIfPolicyT;
+    };
+
+    /// SM10
+    struct Policy100
+    {
+        enum {
+            NOMINAL_4B_ITEMS_PER_THREAD = 9,
+            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
+        };
+
+        typedef AgentSelectIfPolicy<
+                64,
+                ITEMS_PER_THREAD,
+                BLOCK_LOAD_WARP_TRANSPOSE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_RAKING>
+            SelectIfPolicyT;
+    };
+
+
+    /******************************************************************************
+     * Tuning policies of current PTX compiler pass
+     ******************************************************************************/
+
+#if (CUB_PTX_ARCH >= 350)
+    typedef Policy350 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 300)
+    typedef Policy300 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 200)
+    typedef Policy200 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 130)
+    typedef Policy130 PtxPolicy;
+
+#else
+    typedef Policy100 PtxPolicy;
+
+#endif
+
+    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
+    struct PtxSelectIfPolicyT : PtxPolicy::SelectIfPolicyT {};
+
+
+    /******************************************************************************
+     * Utilities
+     ******************************************************************************/
+
+    /**
+     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
+     */
+    template <typename KernelConfig>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static void InitConfigs(
+        int             ptx_version,
+        KernelConfig    &select_if_config)
+    {
+    #if (CUB_PTX_ARCH > 0)
+        (void)ptx_version;
+
+        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
+        select_if_config.template Init<PtxSelectIfPolicyT>();
+
+    #else
+
+        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
+        if (ptx_version >= 350)
+        {
+            select_if_config.template Init<typename Policy350::SelectIfPolicyT>();
+        }
+        else if (ptx_version >= 300)
+        {
+            select_if_config.template Init<typename Policy300::SelectIfPolicyT>();
+        }
+        else if (ptx_version >= 200)
+        {
+            select_if_config.template Init<typename Policy200::SelectIfPolicyT>();
+        }
+        else if (ptx_version >= 130)
+        {
+            select_if_config.template Init<typename Policy130::SelectIfPolicyT>();
+        }
+        else
+        {
+            select_if_config.template Init<typename Policy100::SelectIfPolicyT>();
+        }
+
+    #endif
+    }
+
+
+    /**
+     * Kernel kernel dispatch configuration.
+     */
+    struct KernelConfig
+    {
+        int block_threads;
+        int items_per_thread;
+        int tile_items;
+
+        template <typename PolicyT>
+        CUB_RUNTIME_FUNCTION __forceinline__
+        void Init()
+        {
+            block_threads       = PolicyT::BLOCK_THREADS;
+            items_per_thread    = PolicyT::ITEMS_PER_THREAD;
+            tile_items          = block_threads * items_per_thread;
+        }
+    };
+
+
+    /******************************************************************************
+     * Dispatch entrypoints
+     ******************************************************************************/
+
+    /**
+     * Internal dispatch routine for computing a device-wide selection using the
+     * specified kernel functions.
+     */
+    template <
+        typename                    ScanInitKernelPtrT,             ///< Function type of cub::DeviceScanInitKernel
+        typename                    SelectIfKernelPtrT>             ///< Function type of cub::SelectIfKernelPtrT
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
+        FlagsInputIteratorT         d_flags,                        ///< [in] Pointer to the input sequence of selection flags (if applicable)
+        SelectedOutputIteratorT     d_selected_out,                 ///< [in] Pointer to the output sequence of selected data items
+        NumSelectedIteratorT        d_num_selected_out,             ///< [in] Pointer to the total number of items selected (i.e., length of \p d_selected_out)
+        SelectOpT                   select_op,                      ///< [in] Selection operator
+        EqualityOpT                 equality_op,                    ///< [in] Equality operator
+        OffsetT                     num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream,                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous,              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+        int                         /*ptx_version*/,                ///< [in] PTX version of dispatch kernels
+        ScanInitKernelPtrT          scan_init_kernel,               ///< [in] Kernel function pointer to parameterization of cub::DeviceScanInitKernel
+        SelectIfKernelPtrT          select_if_kernel,               ///< [in] Kernel function pointer to parameterization of cub::DeviceSelectSweepKernel
+        KernelConfig                select_if_config)               ///< [in] Dispatch parameters that match the policy that \p select_if_kernel was compiled for
+    {
+
+#ifndef CUB_RUNTIME_ENABLED
+        (void)d_temp_storage;
+        (void)temp_storage_bytes;
+        (void)d_in;
+        (void)d_flags;
+        (void)d_selected_out;
+        (void)d_num_selected_out;
+        (void)select_op;
+        (void)equality_op;
+        (void)num_items;
+        (void)stream;
+        (void)debug_synchronous;
+        (void)scan_init_kernel;
+        (void)select_if_kernel;
+        (void)select_if_config;
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported);
+
+#else
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Number of input tiles
+            int tile_size = select_if_config.block_threads * select_if_config.items_per_thread;
+            int num_tiles = (num_items + tile_size - 1) / tile_size;
+
+            // Specify temporary storage allocation requirements
+            size_t  allocation_sizes[1];
+            if (CubDebug(error = ScanTileStateT::AllocationSize(num_tiles, allocation_sizes[0]))) break;    // bytes needed for tile status descriptors
+
+            // Compute allocation pointers into the single storage blob (or compute the necessary size of the blob)
+            void* allocations[1];
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+            if (d_temp_storage == NULL)
+            {
+                // Return if the caller is simply requesting the size of the storage allocation
+                break;
+            }
+
+            // Construct the tile status interface
+            ScanTileStateT tile_status;
+            if (CubDebug(error = tile_status.Init(num_tiles, allocations[0], allocation_sizes[0]))) break;
+
+            // Log scan_init_kernel configuration
+            int init_grid_size = CUB_MAX(1, (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS);
+            if (debug_synchronous) _CubLog("Invoking scan_init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream);
+
+            // Invoke scan_init_kernel to initialize tile descriptors
+            scan_init_kernel<<<init_grid_size, INIT_KERNEL_THREADS, 0, stream>>>(
+                tile_status,
+                num_tiles,
+                d_num_selected_out);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Return if empty problem
+            if (num_items == 0)
+                break;
+
+            // Get SM occupancy for select_if_kernel
+            int range_select_sm_occupancy;
+            if (CubDebug(error = MaxSmOccupancy(
+                range_select_sm_occupancy,            // out
+                select_if_kernel,
+                select_if_config.block_threads))) break;
+
+            // Get max x-dimension of grid
+            int max_dim_x;
+            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
+
+            // Get grid size for scanning tiles
+            dim3 scan_grid_size;
+            scan_grid_size.z = 1;
+            scan_grid_size.y = ((unsigned int) num_tiles + max_dim_x - 1) / max_dim_x;
+            scan_grid_size.x = CUB_MIN(num_tiles, max_dim_x);
+
+            // Log select_if_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking select_if_kernel<<<{%d,%d,%d}, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                scan_grid_size.x, scan_grid_size.y, scan_grid_size.z, select_if_config.block_threads, (long long) stream, select_if_config.items_per_thread, range_select_sm_occupancy);
+
+            // Invoke select_if_kernel
+            select_if_kernel<<<scan_grid_size, select_if_config.block_threads, 0, stream>>>(
+                d_in,
+                d_flags,
+                d_selected_out,
+                d_num_selected_out,
+                tile_status,
+                select_op,
+                equality_op,
+                num_items,
+                num_tiles);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+        }
+        while (0);
+
+        return error;
+
+#endif  // CUB_RUNTIME_ENABLED
+    }
+
+
+    /**
+     * Internal dispatch routine
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
+        FlagsInputIteratorT         d_flags,                        ///< [in] Pointer to the input sequence of selection flags (if applicable)
+        SelectedOutputIteratorT     d_selected_out,                 ///< [in] Pointer to the output sequence of selected data items
+        NumSelectedIteratorT        d_num_selected_out,             ///< [in] Pointer to the total number of items selected (i.e., length of \p d_selected_out)
+        SelectOpT                   select_op,                      ///< [in] Selection operator
+        EqualityOpT                 equality_op,                    ///< [in] Equality operator
+        OffsetT                     num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
+        cudaStream_t                stream,                         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                        debug_synchronous)              ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel kernel dispatch configurations
+            KernelConfig select_if_config;
+            InitConfigs(ptx_version, select_if_config);
+
+            // Dispatch
+            if (CubDebug(error = Dispatch(
+                d_temp_storage,
+                temp_storage_bytes,
+                d_in,
+                d_flags,
+                d_selected_out,
+                d_num_selected_out,
+                select_op,
+                equality_op,
+                num_items,
+                stream,
+                debug_synchronous,
+                ptx_version,
+                DeviceCompactInitKernel<ScanTileStateT, NumSelectedIteratorT>,
+                DeviceSelectSweepKernel<PtxSelectIfPolicyT, InputIteratorT, FlagsInputIteratorT, SelectedOutputIteratorT, NumSelectedIteratorT, ScanTileStateT, SelectOpT, EqualityOpT, OffsetT, KEEP_REJECTS>,
+                select_if_config))) break;
+        }
+        while (0);
+
+        return error;
+    }
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/device/dispatch/dispatch_spmv_orig.cuh b/third_party/cub/cub/device/dispatch/dispatch_spmv_orig.cuh
new file mode 100644
index 0000000..ab9c534
--- /dev/null
+++ b/third_party/cub/cub/device/dispatch/dispatch_spmv_orig.cuh
@@ -0,0 +1,834 @@
+
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::DeviceSpmv provides device-wide parallel operations for performing sparse-matrix * vector multiplication (SpMV).
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include <iterator>
+
+#include "../../agent/single_pass_scan_operators.cuh"
+#include "../../agent/agent_segment_fixup.cuh"
+#include "../../agent/agent_spmv_orig.cuh"
+#include "../../util_type.cuh"
+#include "../../util_debug.cuh"
+#include "../../util_device.cuh"
+#include "../../thread/thread_search.cuh"
+#include "../../grid/grid_queue.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * SpMV kernel entry points
+ *****************************************************************************/
+
+/**
+ * Spmv search kernel. Identifies merge path starting coordinates for each tile.
+ */
+template <
+    typename    AgentSpmvPolicyT,           ///< Parameterized SpmvPolicy tuning policy type
+    typename    ValueT,                     ///< Matrix and vector value type
+    typename    OffsetT>                    ///< Signed integer type for sequence offsets
+__global__ void DeviceSpmv1ColKernel(
+    SpmvParams<ValueT, OffsetT> spmv_params)                ///< [in] SpMV input parameter bundle
+{
+    typedef CacheModifiedInputIterator<
+            AgentSpmvPolicyT::VECTOR_VALUES_LOAD_MODIFIER,
+            ValueT,
+            OffsetT>
+        VectorValueIteratorT;
+
+    VectorValueIteratorT wrapped_vector_x(spmv_params.d_vector_x);
+
+    int row_idx = (blockIdx.x * blockDim.x) + threadIdx.x;
+    if (row_idx < spmv_params.num_rows)
+    {
+        OffsetT     end_nonzero_idx = spmv_params.d_row_end_offsets[row_idx];
+        OffsetT     nonzero_idx = spmv_params.d_row_end_offsets[row_idx - 1];
+
+        ValueT value = 0.0;
+        if (end_nonzero_idx != nonzero_idx)
+        {
+            value = spmv_params.d_values[nonzero_idx] * wrapped_vector_x[spmv_params.d_column_indices[nonzero_idx]];
+        }
+
+        spmv_params.d_vector_y[row_idx] = value;
+    }
+}
+
+
+/**
+ * Spmv search kernel. Identifies merge path starting coordinates for each tile.
+ */
+template <
+    typename    SpmvPolicyT,                    ///< Parameterized SpmvPolicy tuning policy type
+    typename    OffsetT,                        ///< Signed integer type for sequence offsets
+    typename    CoordinateT,                    ///< Merge path coordinate type
+    typename    SpmvParamsT>                    ///< SpmvParams type
+__global__ void DeviceSpmvSearchKernel(
+    int             num_merge_tiles,            ///< [in] Number of SpMV merge tiles (spmv grid size)
+    CoordinateT*    d_tile_coordinates,         ///< [out] Pointer to the temporary array of tile starting coordinates
+    SpmvParamsT     spmv_params)                ///< [in] SpMV input parameter bundle
+{
+    /// Constants
+    enum
+    {
+        BLOCK_THREADS           = SpmvPolicyT::BLOCK_THREADS,
+        ITEMS_PER_THREAD        = SpmvPolicyT::ITEMS_PER_THREAD,
+        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
+    };
+
+    typedef CacheModifiedInputIterator<
+            SpmvPolicyT::ROW_OFFSETS_SEARCH_LOAD_MODIFIER,
+            OffsetT,
+            OffsetT>
+        RowOffsetsSearchIteratorT;
+
+    // Find the starting coordinate for all tiles (plus the end coordinate of the last one)
+    int tile_idx = (blockIdx.x * blockDim.x) + threadIdx.x;
+    if (tile_idx < num_merge_tiles + 1)
+    {
+        OffsetT                         diagonal = (tile_idx * TILE_ITEMS);
+        CoordinateT                     tile_coordinate;
+        CountingInputIterator<OffsetT>  nonzero_indices(0);
+
+        // Search the merge path
+        MergePathSearch(
+            diagonal,
+            RowOffsetsSearchIteratorT(spmv_params.d_row_end_offsets),
+            nonzero_indices,
+            spmv_params.num_rows,
+            spmv_params.num_nonzeros,
+            tile_coordinate);
+
+        // Output starting offset
+        d_tile_coordinates[tile_idx] = tile_coordinate;
+    }
+}
+
+
+/**
+ * Spmv agent entry point
+ */
+template <
+    typename        SpmvPolicyT,                ///< Parameterized SpmvPolicy tuning policy type
+    typename        ScanTileStateT,             ///< Tile status interface type
+    typename        ValueT,                     ///< Matrix and vector value type
+    typename        OffsetT,                    ///< Signed integer type for sequence offsets
+    typename        CoordinateT,                ///< Merge path coordinate type
+    bool            HAS_ALPHA,                  ///< Whether the input parameter Alpha is 1
+    bool            HAS_BETA>                   ///< Whether the input parameter Beta is 0
+__launch_bounds__ (int(SpmvPolicyT::BLOCK_THREADS))
+__global__ void DeviceSpmvKernel(
+    SpmvParams<ValueT, OffsetT>     spmv_params,                ///< [in] SpMV input parameter bundle
+    CoordinateT*                    d_tile_coordinates,         ///< [in] Pointer to the temporary array of tile starting coordinates
+    KeyValuePair<OffsetT,ValueT>*   d_tile_carry_pairs,         ///< [out] Pointer to the temporary array carry-out dot product row-ids, one per block
+    int                             num_tiles,                  ///< [in] Number of merge tiles
+    ScanTileStateT                  tile_state,                 ///< [in] Tile status interface for fixup reduce-by-key kernel
+    int                             num_segment_fixup_tiles)    ///< [in] Number of reduce-by-key tiles (fixup grid size)
+{
+    // Spmv agent type specialization
+    typedef AgentSpmv<
+            SpmvPolicyT,
+            ValueT,
+            OffsetT,
+            HAS_ALPHA,
+            HAS_BETA>
+        AgentSpmvT;
+
+    // Shared memory for AgentSpmv
+    __shared__ typename AgentSpmvT::TempStorage temp_storage;
+
+    AgentSpmvT(temp_storage, spmv_params).ConsumeTile(
+        d_tile_coordinates,
+        d_tile_carry_pairs,
+        num_tiles);
+
+    // Initialize fixup tile status
+    tile_state.InitializeStatus(num_segment_fixup_tiles);
+
+}
+
+
+/**
+ * Multi-block reduce-by-key sweep kernel entry point
+ */
+template <
+    typename    AgentSegmentFixupPolicyT,       ///< Parameterized AgentSegmentFixupPolicy tuning policy type
+    typename    PairsInputIteratorT,            ///< Random-access input iterator type for keys
+    typename    AggregatesOutputIteratorT,      ///< Random-access output iterator type for values
+    typename    OffsetT,                        ///< Signed integer type for global offsets
+    typename    ScanTileStateT>                 ///< Tile status interface type
+__launch_bounds__ (int(AgentSegmentFixupPolicyT::BLOCK_THREADS))
+__global__ void DeviceSegmentFixupKernel(
+    PairsInputIteratorT         d_pairs_in,         ///< [in] Pointer to the array carry-out dot product row-ids, one per spmv block
+    AggregatesOutputIteratorT   d_aggregates_out,   ///< [in,out] Output value aggregates
+    OffsetT                     num_items,          ///< [in] Total number of items to select from
+    int                         num_tiles,          ///< [in] Total number of tiles for the entire problem
+    ScanTileStateT              tile_state)         ///< [in] Tile status interface
+{
+    // Thread block type for reducing tiles of value segments
+    typedef AgentSegmentFixup<
+            AgentSegmentFixupPolicyT,
+            PairsInputIteratorT,
+            AggregatesOutputIteratorT,
+            cub::Equality,
+            cub::Sum,
+            OffsetT>
+        AgentSegmentFixupT;
+
+    // Shared memory for AgentSegmentFixup
+    __shared__ typename AgentSegmentFixupT::TempStorage temp_storage;
+
+    // Process tiles
+    AgentSegmentFixupT(temp_storage, d_pairs_in, d_aggregates_out, cub::Equality(), cub::Sum()).ConsumeRange(
+        num_items,
+        num_tiles,
+        tile_state);
+}
+
+
+/******************************************************************************
+ * Dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for DeviceSpmv
+ */
+template <
+    typename    ValueT,                     ///< Matrix and vector value type
+    typename    OffsetT>                    ///< Signed integer type for global offsets
+struct DispatchSpmv
+{
+    //---------------------------------------------------------------------
+    // Constants and Types
+    //---------------------------------------------------------------------
+
+    enum
+    {
+        INIT_KERNEL_THREADS = 128
+    };
+
+    // SpmvParams bundle type
+    typedef SpmvParams<ValueT, OffsetT> SpmvParamsT;
+
+    // 2D merge path coordinate type
+    typedef typename CubVector<OffsetT, 2>::Type CoordinateT;
+
+    // Tile status descriptor interface type
+    typedef ReduceByKeyScanTileState<ValueT, OffsetT> ScanTileStateT;
+
+    // Tuple type for scanning (pairs accumulated segment-value with segment-index)
+    typedef KeyValuePair<OffsetT, ValueT> KeyValuePairT;
+
+
+    //---------------------------------------------------------------------
+    // Tuning policies
+    //---------------------------------------------------------------------
+
+    /// SM11
+    struct Policy110
+    {
+        typedef AgentSpmvPolicy<
+                128,
+                1,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                false,
+                BLOCK_SCAN_WARP_SCANS>
+            SpmvPolicyT;
+
+        typedef AgentSegmentFixupPolicy<
+                128,
+                4,
+                BLOCK_LOAD_VECTORIZE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_WARP_SCANS>
+            SegmentFixupPolicyT;
+    };
+
+    /// SM20
+    struct Policy200 
+    {
+        typedef AgentSpmvPolicy<
+                96,
+                18,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                false,
+                BLOCK_SCAN_RAKING>
+            SpmvPolicyT;
+
+        typedef AgentSegmentFixupPolicy<
+                128,
+                4,
+                BLOCK_LOAD_VECTORIZE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_WARP_SCANS>
+            SegmentFixupPolicyT;
+
+    };
+
+
+
+    /// SM30
+    struct Policy300 
+    {
+        typedef AgentSpmvPolicy<
+                96,
+                6,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                false,
+                BLOCK_SCAN_WARP_SCANS>
+            SpmvPolicyT;
+
+        typedef AgentSegmentFixupPolicy<
+                128,
+                4,
+                BLOCK_LOAD_VECTORIZE,
+                LOAD_DEFAULT,
+                BLOCK_SCAN_WARP_SCANS>
+            SegmentFixupPolicyT;
+
+    };
+
+
+    /// SM35
+    struct Policy350
+    {
+        typedef AgentSpmvPolicy<
+                (sizeof(ValueT) > 4) ? 96 : 128,
+                (sizeof(ValueT) > 4) ? 4 : 7,
+                LOAD_LDG,
+                LOAD_CA,
+                LOAD_LDG,
+                LOAD_LDG,
+                LOAD_LDG,
+                (sizeof(ValueT) > 4) ? true : false,
+                BLOCK_SCAN_WARP_SCANS>
+            SpmvPolicyT;
+
+        typedef AgentSegmentFixupPolicy<
+                128,
+                3,
+                BLOCK_LOAD_VECTORIZE,
+                LOAD_LDG,
+                BLOCK_SCAN_WARP_SCANS>
+            SegmentFixupPolicyT;
+    };
+
+
+    /// SM37
+    struct Policy370
+    {
+
+        typedef AgentSpmvPolicy<
+                (sizeof(ValueT) > 4) ? 128 : 128,
+                (sizeof(ValueT) > 4) ? 9 : 14,
+                LOAD_LDG,
+                LOAD_CA,
+                LOAD_LDG,
+                LOAD_LDG,
+                LOAD_LDG,
+                false, 
+                BLOCK_SCAN_WARP_SCANS>
+            SpmvPolicyT;
+
+        typedef AgentSegmentFixupPolicy<
+                128,
+                3,
+                BLOCK_LOAD_VECTORIZE,
+                LOAD_LDG,
+                BLOCK_SCAN_WARP_SCANS>
+            SegmentFixupPolicyT;
+    };
+
+    /// SM50
+    struct Policy500
+    {
+        typedef AgentSpmvPolicy<
+                (sizeof(ValueT) > 4) ? 64 : 128,
+                (sizeof(ValueT) > 4) ? 6 : 7,
+                LOAD_LDG,
+                LOAD_DEFAULT,
+                (sizeof(ValueT) > 4) ? LOAD_LDG : LOAD_DEFAULT,
+                (sizeof(ValueT) > 4) ? LOAD_LDG : LOAD_DEFAULT,
+                LOAD_LDG,
+                (sizeof(ValueT) > 4) ? true : false,
+                (sizeof(ValueT) > 4) ? BLOCK_SCAN_WARP_SCANS : BLOCK_SCAN_RAKING_MEMOIZE>
+            SpmvPolicyT;
+
+
+        typedef AgentSegmentFixupPolicy<
+                128,
+                3,
+                BLOCK_LOAD_VECTORIZE,
+                LOAD_LDG,
+                BLOCK_SCAN_RAKING_MEMOIZE>
+            SegmentFixupPolicyT;
+    };
+
+
+    /// SM60
+    struct Policy600
+    {
+        typedef AgentSpmvPolicy<
+                (sizeof(ValueT) > 4) ? 64 : 128,
+                (sizeof(ValueT) > 4) ? 5 : 7,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                LOAD_DEFAULT,
+                false,
+                BLOCK_SCAN_WARP_SCANS>
+            SpmvPolicyT;
+
+
+        typedef AgentSegmentFixupPolicy<
+                128,
+                3,
+                BLOCK_LOAD_DIRECT,
+                LOAD_LDG,
+                BLOCK_SCAN_WARP_SCANS>
+            SegmentFixupPolicyT;
+    };
+
+
+
+    //---------------------------------------------------------------------
+    // Tuning policies of current PTX compiler pass
+    //---------------------------------------------------------------------
+
+#if (CUB_PTX_ARCH >= 600)
+    typedef Policy600 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 500)
+    typedef Policy500 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 370)
+    typedef Policy370 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 350)
+    typedef Policy350 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 300)
+    typedef Policy300 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 200)
+    typedef Policy200 PtxPolicy;
+
+#else
+    typedef Policy110 PtxPolicy;
+
+#endif
+
+    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
+    struct PtxSpmvPolicyT : PtxPolicy::SpmvPolicyT {};
+    struct PtxSegmentFixupPolicy : PtxPolicy::SegmentFixupPolicyT {};
+
+
+    //---------------------------------------------------------------------
+    // Utilities
+    //---------------------------------------------------------------------
+
+    /**
+     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
+     */
+    template <typename KernelConfig>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static void InitConfigs(
+        int             ptx_version,
+        KernelConfig    &spmv_config,
+        KernelConfig    &segment_fixup_config)
+    {
+    #if (CUB_PTX_ARCH > 0)
+
+        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
+        spmv_config.template Init<PtxSpmvPolicyT>();
+        segment_fixup_config.template Init<PtxSegmentFixupPolicy>();
+
+    #else
+
+        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
+        if (ptx_version >= 600)
+        {
+            spmv_config.template            Init<typename Policy600::SpmvPolicyT>();
+            segment_fixup_config.template   Init<typename Policy600::SegmentFixupPolicyT>();
+        }
+        else if (ptx_version >= 500)
+        {
+            spmv_config.template            Init<typename Policy500::SpmvPolicyT>();
+            segment_fixup_config.template   Init<typename Policy500::SegmentFixupPolicyT>();
+        }
+        else if (ptx_version >= 370)
+        {
+            spmv_config.template            Init<typename Policy370::SpmvPolicyT>();
+            segment_fixup_config.template   Init<typename Policy370::SegmentFixupPolicyT>();
+        }
+        else if (ptx_version >= 350)
+        {
+            spmv_config.template            Init<typename Policy350::SpmvPolicyT>();
+            segment_fixup_config.template   Init<typename Policy350::SegmentFixupPolicyT>();
+        }
+        else if (ptx_version >= 300)
+        {
+            spmv_config.template            Init<typename Policy300::SpmvPolicyT>();
+            segment_fixup_config.template   Init<typename Policy300::SegmentFixupPolicyT>();
+
+        }
+        else if (ptx_version >= 200)
+        {
+            spmv_config.template            Init<typename Policy200::SpmvPolicyT>();
+            segment_fixup_config.template   Init<typename Policy200::SegmentFixupPolicyT>();
+        }
+        else
+        {
+            spmv_config.template            Init<typename Policy110::SpmvPolicyT>();
+            segment_fixup_config.template   Init<typename Policy110::SegmentFixupPolicyT>();
+        }
+
+    #endif
+    }
+
+
+    /**
+     * Kernel kernel dispatch configuration.
+     */
+    struct KernelConfig
+    {
+        int block_threads;
+        int items_per_thread;
+        int tile_items;
+
+        template <typename PolicyT>
+        CUB_RUNTIME_FUNCTION __forceinline__
+        void Init()
+        {
+            block_threads       = PolicyT::BLOCK_THREADS;
+            items_per_thread    = PolicyT::ITEMS_PER_THREAD;
+            tile_items          = block_threads * items_per_thread;
+        }
+    };
+
+
+    //---------------------------------------------------------------------
+    // Dispatch entrypoints
+    //---------------------------------------------------------------------
+
+    /**
+     * Internal dispatch routine for computing a device-wide reduction using the
+     * specified kernel functions.
+     *
+     * If the input is larger than a single tile, this method uses two-passes of
+     * kernel invocations.
+     */
+    template <
+        typename                Spmv1ColKernelT,                    ///< Function type of cub::DeviceSpmv1ColKernel
+        typename                SpmvSearchKernelT,                  ///< Function type of cub::AgentSpmvSearchKernel
+        typename                SpmvKernelT,                        ///< Function type of cub::AgentSpmvKernel
+        typename                SegmentFixupKernelT>                 ///< Function type of cub::DeviceSegmentFixupKernelT
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                   d_temp_storage,                     ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                 temp_storage_bytes,                 ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SpmvParamsT&            spmv_params,                        ///< SpMV input parameter bundle
+        cudaStream_t            stream,                             ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous,                  ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+        Spmv1ColKernelT         spmv_1col_kernel,                   ///< [in] Kernel function pointer to parameterization of DeviceSpmv1ColKernel
+        SpmvSearchKernelT       spmv_search_kernel,                 ///< [in] Kernel function pointer to parameterization of AgentSpmvSearchKernel
+        SpmvKernelT             spmv_kernel,                        ///< [in] Kernel function pointer to parameterization of AgentSpmvKernel
+        SegmentFixupKernelT     segment_fixup_kernel,               ///< [in] Kernel function pointer to parameterization of cub::DeviceSegmentFixupKernel
+        KernelConfig            spmv_config,                        ///< [in] Dispatch parameters that match the policy that \p spmv_kernel was compiled for
+        KernelConfig            segment_fixup_config)               ///< [in] Dispatch parameters that match the policy that \p segment_fixup_kernel was compiled for
+    {
+#ifndef CUB_RUNTIME_ENABLED
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported );
+
+#else
+        cudaError error = cudaSuccess;
+        do
+        {
+            if (spmv_params.num_cols == 1)
+            {
+                if (d_temp_storage == NULL)
+                {
+                    // Return if the caller is simply requesting the size of the storage allocation
+                    temp_storage_bytes = 1;
+                    break;
+                }
+
+                // Get search/init grid dims
+                int degen_col_kernel_block_size     = INIT_KERNEL_THREADS;
+                int degen_col_kernel_grid_size      = (spmv_params.num_rows + degen_col_kernel_block_size - 1) / degen_col_kernel_block_size;
+
+                if (debug_synchronous) _CubLog("Invoking spmv_1col_kernel<<<%d, %d, 0, %lld>>>()\n",
+                    degen_col_kernel_grid_size, degen_col_kernel_block_size, (long long) stream);
+
+                // Invoke spmv_search_kernel
+                spmv_1col_kernel<<<degen_col_kernel_grid_size, degen_col_kernel_block_size, 0, stream>>>(
+                    spmv_params);
+
+                // Check for failure to launch
+                if (CubDebug(error = cudaPeekAtLastError())) break;
+
+                // Sync the stream if specified to flush runtime errors
+                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+                break;
+            }
+
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Get max x-dimension of grid
+            int max_dim_x;
+            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
+
+            // Total number of spmv work items
+            int num_merge_items = spmv_params.num_rows + spmv_params.num_nonzeros;
+
+            // Tile sizes of kernels
+            int merge_tile_size              = spmv_config.block_threads * spmv_config.items_per_thread;
+            int segment_fixup_tile_size     = segment_fixup_config.block_threads * segment_fixup_config.items_per_thread;
+
+            // Number of tiles for kernels
+            unsigned int num_merge_tiles            = (num_merge_items + merge_tile_size - 1) / merge_tile_size;
+            unsigned int num_segment_fixup_tiles    = (num_merge_tiles + segment_fixup_tile_size - 1) / segment_fixup_tile_size;
+
+            // Get SM occupancy for kernels
+            int spmv_sm_occupancy;
+            if (CubDebug(error = MaxSmOccupancy(
+                spmv_sm_occupancy,
+                spmv_kernel,
+                spmv_config.block_threads))) break;
+
+            int segment_fixup_sm_occupancy;
+            if (CubDebug(error = MaxSmOccupancy(
+                segment_fixup_sm_occupancy,
+                segment_fixup_kernel,
+                segment_fixup_config.block_threads))) break;
+
+            // Get grid dimensions
+            dim3 spmv_grid_size(
+                CUB_MIN(num_merge_tiles, max_dim_x),
+                (num_merge_tiles + max_dim_x - 1) / max_dim_x,
+                1);
+
+            dim3 segment_fixup_grid_size(
+                CUB_MIN(num_segment_fixup_tiles, max_dim_x),
+                (num_segment_fixup_tiles + max_dim_x - 1) / max_dim_x,
+                1);
+
+            // Get the temporary storage allocation requirements
+            size_t allocation_sizes[3];
+            if (CubDebug(error = ScanTileStateT::AllocationSize(num_segment_fixup_tiles, allocation_sizes[0]))) break;    // bytes needed for reduce-by-key tile status descriptors
+            allocation_sizes[1] = num_merge_tiles * sizeof(KeyValuePairT);       // bytes needed for block carry-out pairs
+            allocation_sizes[2] = (num_merge_tiles + 1) * sizeof(CoordinateT);   // bytes needed for tile starting coordinates
+
+            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
+            void* allocations[3];
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+            if (d_temp_storage == NULL)
+            {
+                // Return if the caller is simply requesting the size of the storage allocation
+                break;
+            }
+
+            // Construct the tile status interface
+            ScanTileStateT tile_state;
+            if (CubDebug(error = tile_state.Init(num_segment_fixup_tiles, allocations[0], allocation_sizes[0]))) break;
+
+            // Alias the other allocations
+            KeyValuePairT*  d_tile_carry_pairs      = (KeyValuePairT*) allocations[1];  // Agent carry-out pairs
+            CoordinateT*    d_tile_coordinates      = (CoordinateT*) allocations[2];    // Agent starting coordinates
+
+            // Get search/init grid dims
+            int search_block_size   = INIT_KERNEL_THREADS;
+            int search_grid_size    = (num_merge_tiles + 1 + search_block_size - 1) / search_block_size;
+
+#if (CUB_PTX_ARCH == 0)
+            // Init textures
+            if (CubDebug(error = spmv_params.t_vector_x.BindTexture(spmv_params.d_vector_x))) break;
+#endif
+
+            if (search_grid_size < sm_count)
+//            if (num_merge_tiles < spmv_sm_occupancy * sm_count)
+            {
+                // Not enough spmv tiles to saturate the device: have spmv blocks search their own staring coords
+                d_tile_coordinates = NULL;
+            }
+            else
+            {
+                // Use separate search kernel if we have enough spmv tiles to saturate the device
+
+                // Log spmv_search_kernel configuration
+                if (debug_synchronous) _CubLog("Invoking spmv_search_kernel<<<%d, %d, 0, %lld>>>()\n",
+                    search_grid_size, search_block_size, (long long) stream);
+
+                // Invoke spmv_search_kernel
+                spmv_search_kernel<<<search_grid_size, search_block_size, 0, stream>>>(
+                    num_merge_tiles,
+                    d_tile_coordinates,
+                    spmv_params);
+
+                // Check for failure to launch
+                if (CubDebug(error = cudaPeekAtLastError())) break;
+
+                // Sync the stream if specified to flush runtime errors
+                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+            }
+
+            // Log spmv_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking spmv_kernel<<<{%d,%d,%d}, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                spmv_grid_size.x, spmv_grid_size.y, spmv_grid_size.z, spmv_config.block_threads, (long long) stream, spmv_config.items_per_thread, spmv_sm_occupancy);
+
+            // Invoke spmv_kernel
+            spmv_kernel<<<spmv_grid_size, spmv_config.block_threads, 0, stream>>>(
+                spmv_params,
+                d_tile_coordinates,
+                d_tile_carry_pairs,
+                num_merge_tiles,
+                tile_state,
+                num_segment_fixup_tiles);
+
+            // Check for failure to launch
+            if (CubDebug(error = cudaPeekAtLastError())) break;
+
+            // Sync the stream if specified to flush runtime errors
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+
+            // Run reduce-by-key fixup if necessary
+            if (num_merge_tiles > 1)
+            {
+                // Log segment_fixup_kernel configuration
+                if (debug_synchronous) _CubLog("Invoking segment_fixup_kernel<<<{%d,%d,%d}, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                    segment_fixup_grid_size.x, segment_fixup_grid_size.y, segment_fixup_grid_size.z, segment_fixup_config.block_threads, (long long) stream, segment_fixup_config.items_per_thread, segment_fixup_sm_occupancy);
+
+                // Invoke segment_fixup_kernel
+                segment_fixup_kernel<<<segment_fixup_grid_size, segment_fixup_config.block_threads, 0, stream>>>(
+                    d_tile_carry_pairs,
+                    spmv_params.d_vector_y,
+                    num_merge_tiles,
+                    num_segment_fixup_tiles,
+                    tile_state);
+
+                // Check for failure to launch
+                if (CubDebug(error = cudaPeekAtLastError())) break;
+
+                // Sync the stream if specified to flush runtime errors
+                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+            }
+
+#if (CUB_PTX_ARCH == 0)
+            // Free textures
+            if (CubDebug(error = spmv_params.t_vector_x.UnbindTexture())) break;
+#endif
+        }
+        while (0);
+
+        return error;
+
+#endif // CUB_RUNTIME_ENABLED
+    }
+
+
+    /**
+     * Internal dispatch routine for computing a device-wide reduction
+     */
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Dispatch(
+        void*                   d_temp_storage,                     ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+        size_t&                 temp_storage_bytes,                 ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
+        SpmvParamsT&            spmv_params,                        ///< SpMV input parameter bundle
+        cudaStream_t            stream                  = 0,        ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous       = false)    ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel kernel dispatch configurations
+            KernelConfig spmv_config, segment_fixup_config;
+            InitConfigs(ptx_version, spmv_config, segment_fixup_config);
+
+            if (CubDebug(error = Dispatch(
+                d_temp_storage, temp_storage_bytes, spmv_params, stream, debug_synchronous,
+                DeviceSpmv1ColKernel<PtxSpmvPolicyT, ValueT, OffsetT>,
+                DeviceSpmvSearchKernel<PtxSpmvPolicyT, OffsetT, CoordinateT, SpmvParamsT>,
+                DeviceSpmvKernel<PtxSpmvPolicyT, ScanTileStateT, ValueT, OffsetT, CoordinateT, false, false>,
+                DeviceSegmentFixupKernel<PtxSegmentFixupPolicy, KeyValuePairT*, ValueT*, OffsetT, ScanTileStateT>,
+                spmv_config, segment_fixup_config))) break;
+
+        }
+        while (0);
+
+        return error;
+    }
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/grid/grid_barrier.cuh b/third_party/cub/cub/grid/grid_barrier.cuh
new file mode 100644
index 0000000..461fb44
--- /dev/null
+++ b/third_party/cub/cub/grid/grid_barrier.cuh
@@ -0,0 +1,211 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::GridBarrier implements a software global barrier among thread blocks within a CUDA grid
+ */
+
+#pragma once
+
+#include "../util_debug.cuh"
+#include "../util_namespace.cuh"
+#include "../thread/thread_load.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup GridModule
+ * @{
+ */
+
+
+/**
+ * \brief GridBarrier implements a software global barrier among thread blocks within a CUDA grid
+ */
+class GridBarrier
+{
+protected :
+
+    typedef unsigned int SyncFlag;
+
+    // Counters in global device memory
+    SyncFlag* d_sync;
+
+public:
+
+    /**
+     * Constructor
+     */
+    GridBarrier() : d_sync(NULL) {}
+
+
+    /**
+     * Synchronize
+     */
+    __device__ __forceinline__ void Sync() const
+    {
+        volatile SyncFlag *d_vol_sync = d_sync;
+
+        // Threadfence and syncthreads to make sure global writes are visible before
+        // thread-0 reports in with its sync counter
+        __threadfence();
+        CTA_SYNC();
+
+        if (blockIdx.x == 0)
+        {
+            // Report in ourselves
+            if (threadIdx.x == 0)
+            {
+                d_vol_sync[blockIdx.x] = 1;
+            }
+
+            CTA_SYNC();
+
+            // Wait for everyone else to report in
+            for (int peer_block = threadIdx.x; peer_block < gridDim.x; peer_block += blockDim.x)
+            {
+                while (ThreadLoad<LOAD_CG>(d_sync + peer_block) == 0)
+                {
+                    __threadfence_block();
+                }
+            }
+
+            CTA_SYNC();
+
+            // Let everyone know it's safe to proceed
+            for (int peer_block = threadIdx.x; peer_block < gridDim.x; peer_block += blockDim.x)
+            {
+                d_vol_sync[peer_block] = 0;
+            }
+        }
+        else
+        {
+            if (threadIdx.x == 0)
+            {
+                // Report in
+                d_vol_sync[blockIdx.x] = 1;
+
+                // Wait for acknowledgment
+                while (ThreadLoad<LOAD_CG>(d_sync + blockIdx.x) == 1)
+                {
+                    __threadfence_block();
+                }
+            }
+
+            CTA_SYNC();
+        }
+    }
+};
+
+
+/**
+ * \brief GridBarrierLifetime extends GridBarrier to provide lifetime management of the temporary device storage needed for cooperation.
+ *
+ * Uses RAII for lifetime, i.e., device resources are reclaimed when
+ * the destructor is called.
+ */
+class GridBarrierLifetime : public GridBarrier
+{
+protected:
+
+    // Number of bytes backed by d_sync
+    size_t sync_bytes;
+
+public:
+
+    /**
+     * Constructor
+     */
+    GridBarrierLifetime() : GridBarrier(), sync_bytes(0) {}
+
+
+    /**
+     * DeviceFrees and resets the progress counters
+     */
+    cudaError_t HostReset()
+    {
+        cudaError_t retval = cudaSuccess;
+        if (d_sync)
+        {
+            CubDebug(retval = cudaFree(d_sync));
+            d_sync = NULL;
+        }
+        sync_bytes = 0;
+        return retval;
+    }
+
+
+    /**
+     * Destructor
+     */
+    virtual ~GridBarrierLifetime()
+    {
+        HostReset();
+    }
+
+
+    /**
+     * Sets up the progress counters for the next kernel launch (lazily
+     * allocating and initializing them if necessary)
+     */
+    cudaError_t Setup(int sweep_grid_size)
+    {
+        cudaError_t retval = cudaSuccess;
+        do {
+            size_t new_sync_bytes = sweep_grid_size * sizeof(SyncFlag);
+            if (new_sync_bytes > sync_bytes)
+            {
+                if (d_sync)
+                {
+                    if (CubDebug(retval = cudaFree(d_sync))) break;
+                }
+
+                sync_bytes = new_sync_bytes;
+
+                // Allocate and initialize to zero
+                if (CubDebug(retval = cudaMalloc((void**) &d_sync, sync_bytes))) break;
+                if (CubDebug(retval = cudaMemset(d_sync, 0, new_sync_bytes))) break;
+            }
+        } while (0);
+
+        return retval;
+    }
+};
+
+
+/** @} */       // end group GridModule
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/grid/grid_even_share.cuh b/third_party/cub/cub/grid/grid_even_share.cuh
new file mode 100644
index 0000000..f0b3a69
--- /dev/null
+++ b/third_party/cub/cub/grid/grid_even_share.cuh
@@ -0,0 +1,222 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::GridEvenShare is a descriptor utility for distributing input among CUDA thread blocks in an "even-share" fashion.  Each thread block gets roughly the same number of fixed-size work units (grains).
+ */
+
+
+#pragma once
+
+#include "../util_namespace.cuh"
+#include "../util_macro.cuh"
+#include "grid_mapping.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup GridModule
+ * @{
+ */
+
+
+/**
+ * \brief GridEvenShare is a descriptor utility for distributing input among
+ * CUDA thread blocks in an "even-share" fashion.  Each thread block gets roughly
+ * the same number of input tiles.
+ *
+ * \par Overview
+ * Each thread block is assigned a consecutive sequence of input tiles.  To help
+ * preserve alignment and eliminate the overhead of guarded loads for all but the
+ * last thread block, to GridEvenShare assigns one of three different amounts of
+ * work to a given thread block: "big", "normal", or "last".  The "big" workloads
+ * are one scheduling grain larger than "normal".  The "last" work unit for the
+ * last thread block may be partially-full if the input is not an even multiple of
+ * the scheduling grain size.
+ *
+ * \par
+ * Before invoking a child grid, a parent thread will typically construct an
+ * instance of GridEvenShare.  The instance can be passed to child thread blocks
+ * which can initialize their per-thread block offsets using \p BlockInit().
+ */
+template <typename OffsetT>
+struct GridEvenShare
+{
+private:
+
+    OffsetT     total_tiles;
+    int         big_shares;
+    OffsetT     big_share_items;
+    OffsetT     normal_share_items;
+    OffsetT     normal_base_offset;
+
+public:
+
+    /// Total number of input items
+    OffsetT     num_items;
+
+    /// Grid size in thread blocks
+    int         grid_size;
+
+    /// OffsetT into input marking the beginning of the owning thread block's segment of input tiles
+    OffsetT     block_offset;
+
+    /// OffsetT into input of marking the end (one-past) of the owning thread block's segment of input tiles
+    OffsetT     block_end;
+
+    /// Stride between input tiles
+    OffsetT     block_stride;
+
+
+    /**
+     * \brief Constructor.
+     */
+    __host__ __device__ __forceinline__ GridEvenShare() :
+        total_tiles(0),
+        big_shares(0),
+        big_share_items(0),
+        normal_share_items(0),
+        normal_base_offset(0),
+        num_items(0),
+        grid_size(0),
+        block_offset(0),
+        block_end(0),
+        block_stride(0)
+    {}
+
+
+    /**
+     * \brief Dispatch initializer. To be called prior prior to kernel launch.
+     */
+    __host__ __device__ __forceinline__ void DispatchInit(
+        OffsetT num_items,          ///< Total number of input items
+        int     max_grid_size,      ///< Maximum grid size allowable (actual grid size may be less if not warranted by the the number of input items)
+        int     tile_items)         ///< Number of data items per input tile
+    {
+        this->block_offset          = num_items;    // Initialize past-the-end
+        this->block_end             = num_items;    // Initialize past-the-end
+        this->num_items             = num_items;
+        this->total_tiles           = (num_items + tile_items - 1) / tile_items;
+        this->grid_size             = CUB_MIN(total_tiles, max_grid_size);
+        OffsetT avg_tiles_per_block = total_tiles / grid_size;
+        this->big_shares            = total_tiles - (avg_tiles_per_block * grid_size);        // leftover grains go to big blocks
+        this->normal_share_items    = avg_tiles_per_block * tile_items;
+        this->normal_base_offset    = big_shares * tile_items;
+        this->big_share_items       = normal_share_items + tile_items;
+    }
+
+
+    /**
+     * \brief Initializes ranges for the specified thread block index.  Specialized
+     * for a "raking" access pattern in which each thread block is assigned a
+     * consecutive sequence of input tiles.
+     */
+    template <int TILE_ITEMS>
+    __device__ __forceinline__ void BlockInit(
+        int block_id,
+        Int2Type<GRID_MAPPING_RAKE> /*strategy_tag*/)
+    {
+        block_stride = TILE_ITEMS;
+        if (block_id < big_shares)
+        {
+            // This thread block gets a big share of grains (avg_tiles_per_block + 1)
+            block_offset = (block_id * big_share_items);
+            block_end = block_offset + big_share_items;
+        }
+        else if (block_id < total_tiles)
+        {
+            // This thread block gets a normal share of grains (avg_tiles_per_block)
+            block_offset = normal_base_offset + (block_id * normal_share_items);
+            block_end = CUB_MIN(num_items, block_offset + normal_share_items);
+        }
+        // Else default past-the-end
+    }
+
+
+    /**
+     * \brief Block-initialization, specialized for a "raking" access
+     * pattern in which each thread block is assigned a consecutive sequence
+     * of input tiles.
+     */
+    template <int TILE_ITEMS>
+    __device__ __forceinline__ void BlockInit(
+        int block_id,
+        Int2Type<GRID_MAPPING_STRIP_MINE> /*strategy_tag*/)
+    {
+        block_stride = grid_size * TILE_ITEMS;
+        block_offset = (block_id * TILE_ITEMS);
+        block_end = num_items;
+    }
+
+
+    /**
+     * \brief Block-initialization, specialized for "strip mining" access
+     * pattern in which the input tiles assigned to each thread block are
+     * separated by a stride equal to the the extent of the grid.
+     */
+    template <
+        int TILE_ITEMS,
+        GridMappingStrategy STRATEGY>
+    __device__ __forceinline__ void BlockInit()
+    {
+        BlockInit<TILE_ITEMS>(blockIdx.x, Int2Type<STRATEGY>());
+    }
+
+
+    /**
+     * \brief Block-initialization, specialized for a "raking" access
+     * pattern in which each thread block is assigned a consecutive sequence
+     * of input tiles.
+     */
+    template <int TILE_ITEMS>
+    __device__ __forceinline__ void BlockInit(
+        OffsetT block_offset,                       ///< [in] Threadblock begin offset (inclusive)
+        OffsetT block_end)                          ///< [in] Threadblock end offset (exclusive)
+    {
+        this->block_offset = block_offset;
+        this->block_end = block_end;
+        this->block_stride = TILE_ITEMS;
+    }
+
+
+};
+
+
+
+
+
+/** @} */       // end group GridModule
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/grid/grid_mapping.cuh b/third_party/cub/cub/grid/grid_mapping.cuh
new file mode 100644
index 0000000..f0e9fde
--- /dev/null
+++ b/third_party/cub/cub/grid/grid_mapping.cuh
@@ -0,0 +1,113 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::GridMappingStrategy enumerates alternative strategies for mapping constant-sized tiles of device-wide data onto a grid of CUDA thread blocks.
+ */
+
+#pragma once
+
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup GridModule
+ * @{
+ */
+
+
+/******************************************************************************
+ * Mapping policies
+ *****************************************************************************/
+
+
+/**
+ * \brief cub::GridMappingStrategy enumerates alternative strategies for mapping constant-sized tiles of device-wide data onto a grid of CUDA thread blocks.
+ */
+enum GridMappingStrategy
+{
+    /**
+     * \brief An a "raking" access pattern in which each thread block is
+     * assigned a consecutive sequence of input tiles
+     *
+     * \par Overview
+     * The input is evenly partitioned into \p p segments, where \p p is
+     * constant and corresponds loosely to the number of thread blocks that may
+     * actively reside on the target device. Each segment is comprised of
+     * consecutive tiles, where a tile is a small, constant-sized unit of input
+     * to be processed to completion before the thread block terminates or
+     * obtains more work.  The kernel invokes \p p thread blocks, each
+     * of which iteratively consumes a segment of <em>n</em>/<em>p</em> elements
+     * in tile-size increments.
+     */
+    GRID_MAPPING_RAKE,
+
+    /**
+     * \brief An a "strip mining" access pattern in which the input tiles assigned
+     * to each thread block are separated by a stride equal to the the extent of
+     * the grid.
+     *
+     * \par Overview
+     * The input is evenly partitioned into \p p sets, where \p p is
+     * constant and corresponds loosely to the number of thread blocks that may
+     * actively reside on the target device. Each set is comprised of
+     * data tiles separated by stride \p tiles, where a tile is a small,
+     * constant-sized unit of input to be processed to completion before the
+     * thread block terminates or obtains more work.  The kernel invokes \p p
+     * thread blocks, each of which iteratively consumes a segment of
+     * <em>n</em>/<em>p</em> elements in tile-size increments.
+     */
+    GRID_MAPPING_STRIP_MINE,
+
+    /**
+     * \brief A dynamic "queue-based" strategy for assigning input tiles to thread blocks.
+     *
+     * \par Overview
+     * The input is treated as a queue to be dynamically consumed by a grid of
+     * thread blocks.  Work is atomically dequeued in tiles, where a tile is a
+     * unit of input to be processed to completion before the thread block
+     * terminates or obtains more work.  The grid size \p p is constant,
+     * loosely corresponding to the number of thread blocks that may actively
+     * reside on the target device.
+     */
+    GRID_MAPPING_DYNAMIC,
+};
+
+
+/** @} */       // end group GridModule
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/grid/grid_queue.cuh b/third_party/cub/cub/grid/grid_queue.cuh
new file mode 100644
index 0000000..9615b14
--- /dev/null
+++ b/third_party/cub/cub/grid/grid_queue.cuh
@@ -0,0 +1,220 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::GridQueue is a descriptor utility for dynamic queue management.
+ */
+
+#pragma once
+
+#include "../util_namespace.cuh"
+#include "../util_debug.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup GridModule
+ * @{
+ */
+
+
+/**
+ * \brief GridQueue is a descriptor utility for dynamic queue management.
+ *
+ * \par Overview
+ * GridQueue descriptors provides abstractions for "filling" or
+ * "draining" globally-shared vectors.
+ *
+ * \par
+ * A "filling" GridQueue works by atomically-adding to a zero-initialized counter,
+ * returning a unique offset for the calling thread to write its items.
+ * The GridQueue maintains the total "fill-size".  The fill counter must be reset
+ * using GridQueue::ResetFill by the host or kernel instance prior to the kernel instance that
+ * will be filling.
+ *
+ * \par
+ * Similarly, a "draining" GridQueue works by works by atomically-incrementing a
+ * zero-initialized counter, returning a unique offset for the calling thread to
+ * read its items. Threads can safely drain until the array's logical fill-size is
+ * exceeded.  The drain counter must be reset using GridQueue::ResetDrain or
+ * GridQueue::FillAndResetDrain by the host or kernel instance prior to the kernel instance that
+ * will be filling.  (For dynamic work distribution of existing data, the corresponding fill-size
+ * is simply the number of elements in the array.)
+ *
+ * \par
+ * Iterative work management can be implemented simply with a pair of flip-flopping
+ * work buffers, each with an associated set of fill and drain GridQueue descriptors.
+ *
+ * \tparam OffsetT Signed integer type for global offsets
+ */
+template <typename OffsetT>
+class GridQueue
+{
+private:
+
+    /// Counter indices
+    enum
+    {
+        FILL    = 0,
+        DRAIN   = 1,
+    };
+
+    /// Pair of counters
+    OffsetT *d_counters;
+
+public:
+
+    /// Returns the device allocation size in bytes needed to construct a GridQueue instance
+    __host__ __device__ __forceinline__
+    static size_t AllocationSize()
+    {
+        return sizeof(OffsetT) * 2;
+    }
+
+
+    /// Constructs an invalid GridQueue descriptor
+    __host__ __device__ __forceinline__ GridQueue()
+    :
+        d_counters(NULL)
+    {}
+
+
+    /// Constructs a GridQueue descriptor around the device storage allocation
+    __host__ __device__ __forceinline__ GridQueue(
+        void *d_storage)                    ///< Device allocation to back the GridQueue.  Must be at least as big as <tt>AllocationSize()</tt>.
+    :
+        d_counters((OffsetT*) d_storage)
+    {}
+
+
+    /// This operation sets the fill-size and resets the drain counter, preparing the GridQueue for draining in the next kernel instance.  To be called by the host or by a kernel prior to that which will be draining.
+    __host__ __device__ __forceinline__ cudaError_t FillAndResetDrain(
+        OffsetT fill_size,
+        cudaStream_t stream = 0)
+    {
+#if (CUB_PTX_ARCH > 0)
+        (void)stream;
+        d_counters[FILL] = fill_size;
+        d_counters[DRAIN] = 0;
+        return cudaSuccess;
+#else
+        OffsetT counters[2];
+        counters[FILL] = fill_size;
+        counters[DRAIN] = 0;
+        return CubDebug(cudaMemcpyAsync(d_counters, counters, sizeof(OffsetT) * 2, cudaMemcpyHostToDevice, stream));
+#endif
+    }
+
+
+    /// This operation resets the drain so that it may advance to meet the existing fill-size.  To be called by the host or by a kernel prior to that which will be draining.
+    __host__ __device__ __forceinline__ cudaError_t ResetDrain(cudaStream_t stream = 0)
+    {
+#if (CUB_PTX_ARCH > 0)
+        (void)stream;
+        d_counters[DRAIN] = 0;
+        return cudaSuccess;
+#else
+        return CubDebug(cudaMemsetAsync(d_counters + DRAIN, 0, sizeof(OffsetT), stream));
+#endif
+    }
+
+
+    /// This operation resets the fill counter.  To be called by the host or by a kernel prior to that which will be filling.
+    __host__ __device__ __forceinline__ cudaError_t ResetFill(cudaStream_t stream = 0)
+    {
+#if (CUB_PTX_ARCH > 0)
+        (void)stream;
+        d_counters[FILL] = 0;
+        return cudaSuccess;
+#else
+        return CubDebug(cudaMemsetAsync(d_counters + FILL, 0, sizeof(OffsetT), stream));
+#endif
+    }
+
+
+    /// Returns the fill-size established by the parent or by the previous kernel.
+    __host__ __device__ __forceinline__ cudaError_t FillSize(
+        OffsetT &fill_size,
+        cudaStream_t stream = 0)
+    {
+#if (CUB_PTX_ARCH > 0)
+        (void)stream;
+        fill_size = d_counters[FILL];
+        return cudaSuccess;
+#else
+        return CubDebug(cudaMemcpyAsync(&fill_size, d_counters + FILL, sizeof(OffsetT), cudaMemcpyDeviceToHost, stream));
+#endif
+    }
+
+
+    /// Drain \p num_items from the queue.  Returns offset from which to read items.  To be called from CUDA kernel.
+    __device__ __forceinline__ OffsetT Drain(OffsetT num_items)
+    {
+        return atomicAdd(d_counters + DRAIN, num_items);
+    }
+
+
+    /// Fill \p num_items into the queue.  Returns offset from which to write items.    To be called from CUDA kernel.
+    __device__ __forceinline__ OffsetT Fill(OffsetT num_items)
+    {
+        return atomicAdd(d_counters + FILL, num_items);
+    }
+};
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+
+/**
+ * Reset grid queue (call with 1 block of 1 thread)
+ */
+template <typename OffsetT>
+__global__ void FillAndResetDrainKernel(
+    GridQueue<OffsetT>   grid_queue,
+    OffsetT              num_items)
+{
+    grid_queue.FillAndResetDrain(num_items);
+}
+
+
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/** @} */       // end group GridModule
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+
diff --git a/third_party/cub/cub/host/mutex.cuh b/third_party/cub/cub/host/mutex.cuh
new file mode 100644
index 0000000..ff7ec90
--- /dev/null
+++ b/third_party/cub/cub/host/mutex.cuh
@@ -0,0 +1,171 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Simple portable mutex
+ */
+
+
+#pragma once
+
+#if (__cplusplus > 199711L) || (defined(_MSC_VER) && _MSC_VER >= 1800)
+    #include <mutex>
+#else
+    #if defined(_WIN32) || defined(_WIN64)
+        #include <intrin.h>
+
+        #define WIN32_LEAN_AND_MEAN
+        #define NOMINMAX
+        #include <windows.h>
+        #undef WIN32_LEAN_AND_MEAN
+        #undef NOMINMAX
+
+        /**
+         * Compiler read/write barrier
+         */
+        #pragma intrinsic(_ReadWriteBarrier)
+
+    #endif
+#endif
+
+#include "../util_namespace.cuh"
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * Simple portable mutex
+ *   - Wraps std::mutex when compiled with C++11 or newer (supported on all platforms)
+ *   - Uses GNU/Windows spinlock mechanisms for pre C++11 (supported on x86/x64 when compiled with cl.exe or g++)
+ */
+struct Mutex
+{
+#if (__cplusplus > 199711L) || (defined(_MSC_VER) && _MSC_VER >= 1800)
+
+    std::mutex mtx;
+
+    void Lock()
+    {
+        mtx.lock();
+    }
+
+    void Unlock()
+    {
+        mtx.unlock();
+    }
+
+    void TryLock()
+    {
+        mtx.try_lock();
+    }
+
+#else       //__cplusplus > 199711L
+
+    #if defined(_MSC_VER)
+
+        // Microsoft VC++
+        typedef long Spinlock;
+
+    #else
+
+        // GNU g++
+        typedef int Spinlock;
+
+        /**
+         * Compiler read/write barrier
+         */
+        __forceinline__ void _ReadWriteBarrier()
+        {
+            __sync_synchronize();
+        }
+
+        /**
+         * Atomic exchange
+         */
+        __forceinline__ long _InterlockedExchange(volatile int * const Target, const int Value)
+        {
+            // NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier
+            _ReadWriteBarrier();
+            return __sync_lock_test_and_set(Target, Value);
+        }
+
+        /**
+         * Pause instruction to prevent excess processor bus usage
+         */
+        __forceinline__ void YieldProcessor()
+        {
+        }
+
+    #endif  // defined(_MSC_VER)
+
+        /// Lock member
+        volatile Spinlock lock;
+
+        /**
+         * Constructor
+         */
+        Mutex() : lock(0) {}
+
+        /**
+         * Return when the specified spinlock has been acquired
+         */
+        __forceinline__ void Lock()
+        {
+            while (1)
+            {
+                if (!_InterlockedExchange(&lock, 1)) return;
+                while (lock) YieldProcessor();
+            }
+        }
+
+
+        /**
+         * Release the specified spinlock
+         */
+        __forceinline__ void Unlock()
+        {
+            _ReadWriteBarrier();
+            lock = 0;
+        }
+
+#endif      // __cplusplus > 199711L
+
+};
+
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
diff --git a/third_party/cub/cub/iterator/arg_index_input_iterator.cuh b/third_party/cub/cub/iterator/arg_index_input_iterator.cuh
new file mode 100644
index 0000000..95a84a5
--- /dev/null
+++ b/third_party/cub/cub/iterator/arg_index_input_iterator.cuh
@@ -0,0 +1,259 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../util_device.cuh"
+#include "../util_namespace.cuh"
+
+#include <thrust/version.h>
+
+#if (THRUST_VERSION >= 100700)
+    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+
+/**
+ * \brief A random-access input wrapper for pairing dereferenced values with their corresponding indices (forming \p KeyValuePair tuples).
+ *
+ * \par Overview
+ * - ArgIndexInputIteratorTwraps a random access input iterator \p itr of type \p InputIteratorT.
+ *   Dereferencing an ArgIndexInputIteratorTat offset \p i produces a \p KeyValuePair value whose
+ *   \p key field is \p i and whose \p value field is <tt>itr[i]</tt>.
+ * - Can be used with any data type.
+ * - Can be constructed, manipulated, and exchanged within and between host and device
+ *   functions.  Wrapped host memory can only be dereferenced on the host, and wrapped
+ *   device memory can only be dereferenced on the device.
+ * - Compatible with Thrust API v1.7 or newer.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of \p ArgIndexInputIteratorTto
+ * dereference an array of doubles
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/iterator/arg_index_input_iterator.cuh>
+ *
+ * // Declare, allocate, and initialize a device array
+ * double *d_in;         // e.g., [8.0, 6.0, 7.0, 5.0, 3.0, 0.0, 9.0]
+ *
+ * // Create an iterator wrapper
+ * cub::ArgIndexInputIterator<double*> itr(d_in);
+ *
+ * // Within device code:
+ * typedef typename cub::ArgIndexInputIterator<double*>::value_type Tuple;
+ * Tuple item_offset_pair.key = *itr;
+ * printf("%f @ %d\n",
+ *   item_offset_pair.value,
+ *   item_offset_pair.key);   // 8.0 @ 0
+ *
+ * itr = itr + 6;
+ * item_offset_pair.key = *itr;
+ * printf("%f @ %d\n",
+ *   item_offset_pair.value,
+ *   item_offset_pair.key);   // 9.0 @ 6
+ *
+ * \endcode
+ *
+ * \tparam InputIteratorT       The value type of the wrapped input iterator
+ * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
+ * \tparam OutputValueT         The paired value type of the <offset,value> tuple (Default: value type of input iterator)
+ */
+template <
+    typename    InputIteratorT,
+    typename    OffsetT             = ptrdiff_t,
+    typename    OutputValueT        = typename std::iterator_traits<InputIteratorT>::value_type>
+class ArgIndexInputIterator
+{
+public:
+
+    // Required iterator traits
+    typedef ArgIndexInputIterator                       self_type;              ///< My own type
+    typedef OffsetT                                     difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef KeyValuePair<difference_type, OutputValueT> value_type;             ///< The type of the element the iterator can point to
+    typedef value_type*                                 pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef value_type                                  reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::any_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+private:
+
+    InputIteratorT  itr;
+    difference_type offset;
+
+public:
+
+    /// Constructor
+    __host__ __device__ __forceinline__ ArgIndexInputIterator(
+        InputIteratorT  itr,            ///< Input iterator to wrap
+        difference_type offset = 0)     ///< OffsetT (in items) from \p itr denoting the position of the iterator
+    :
+        itr(itr),
+        offset(offset)
+    {}
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        offset++;
+        return retval;
+    }
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        offset++;
+        return *this;
+    }
+
+    /// Indirection
+    __host__ __device__ __forceinline__ reference operator*() const
+    {
+        value_type retval;
+        retval.value = itr[offset];
+        retval.key = offset;
+        return retval;
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval(itr, offset + n);
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        offset += n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval(itr, offset - n);
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        offset -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return offset - other.offset;
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __host__ __device__ __forceinline__ reference operator[](Distance n) const
+    {
+        self_type offset = (*this) + n;
+        return *offset;
+    }
+
+    /// Structure dereference
+    __host__ __device__ __forceinline__ pointer operator->()
+    {
+        return &(*(*this));
+    }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return ((itr == rhs.itr) && (offset == rhs.offset));
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return ((itr != rhs.itr) || (offset != rhs.offset));
+    }
+
+    /// Normalize
+    __host__ __device__ __forceinline__ void normalize()
+    {
+        itr += offset;
+        offset = 0;
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& /*itr*/)
+    {
+        return os;
+    }
+};
+
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/iterator/cache_modified_input_iterator.cuh b/third_party/cub/cub/iterator/cache_modified_input_iterator.cuh
new file mode 100644
index 0000000..b4ad91e
--- /dev/null
+++ b/third_party/cub/cub/iterator/cache_modified_input_iterator.cuh
@@ -0,0 +1,240 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../util_device.cuh"
+#include "../util_namespace.cuh"
+
+#if (THRUST_VERSION >= 100700)
+    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+
+/**
+ * \brief A random-access input wrapper for dereferencing array values using a PTX cache load modifier.
+ *
+ * \par Overview
+ * - CacheModifiedInputIteratorTis a random-access input iterator that wraps a native
+ *   device pointer of type <tt>ValueType*</tt>. \p ValueType references are
+ *   made by reading \p ValueType values through loads modified by \p MODIFIER.
+ * - Can be used to load any data type from memory using PTX cache load modifiers (e.g., "LOAD_LDG",
+ *   "LOAD_CG", "LOAD_CA", "LOAD_CS", "LOAD_CV", etc.).
+ * - Can be constructed, manipulated, and exchanged within and between host and device
+ *   functions, but can only be dereferenced within device functions.
+ * - Compatible with Thrust API v1.7 or newer.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of \p CacheModifiedInputIteratorTto
+ * dereference a device array of double using the "ldg" PTX load modifier
+ * (i.e., load values through texture cache).
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/iterator/cache_modified_input_iterator.cuh>
+ *
+ * // Declare, allocate, and initialize a device array
+ * double *d_in;            // e.g., [8.0, 6.0, 7.0, 5.0, 3.0, 0.0, 9.0]
+ *
+ * // Create an iterator wrapper
+ * cub::CacheModifiedInputIterator<cub::LOAD_LDG, double> itr(d_in);
+ *
+ * // Within device code:
+ * printf("%f\n", itr[0]);  // 8.0
+ * printf("%f\n", itr[1]);  // 6.0
+ * printf("%f\n", itr[6]);  // 9.0
+ *
+ * \endcode
+ *
+ * \tparam CacheLoadModifier    The cub::CacheLoadModifier to use when accessing data
+ * \tparam ValueType            The value type of this iterator
+ * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
+ */
+template <
+    CacheLoadModifier   MODIFIER,
+    typename            ValueType,
+    typename            OffsetT = ptrdiff_t>
+class CacheModifiedInputIterator
+{
+public:
+
+    // Required iterator traits
+    typedef CacheModifiedInputIterator          self_type;              ///< My own type
+    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef ValueType                           value_type;             ///< The type of the element the iterator can point to
+    typedef ValueType*                          pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef ValueType                           reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::device_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+
+public:
+
+    /// Wrapped native pointer
+    ValueType* ptr;
+
+    /// Constructor
+    template <typename QualifiedValueType>
+    __host__ __device__ __forceinline__ CacheModifiedInputIterator(
+        QualifiedValueType* ptr)     ///< Native pointer to wrap
+    :
+        ptr(const_cast<typename RemoveQualifiers<QualifiedValueType>::Type *>(ptr))
+    {}
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        ptr++;
+        return retval;
+    }
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        ptr++;
+        return *this;
+    }
+
+    /// Indirection
+    __device__ __forceinline__ reference operator*() const
+    {
+        return ThreadLoad<MODIFIER>(ptr);
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval(ptr + n);
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        ptr += n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval(ptr - n);
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        ptr -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return ptr - other.ptr;
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __device__ __forceinline__ reference operator[](Distance n) const
+    {
+        return ThreadLoad<MODIFIER>(ptr + n);
+    }
+
+    /// Structure dereference
+    __device__ __forceinline__ pointer operator->()
+    {
+        return &ThreadLoad<MODIFIER>(ptr);
+    }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return (ptr == rhs.ptr);
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return (ptr != rhs.ptr);
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& /*itr*/)
+    {
+        return os;
+    }
+};
+
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/iterator/cache_modified_output_iterator.cuh b/third_party/cub/cub/iterator/cache_modified_output_iterator.cuh
new file mode 100644
index 0000000..c3e3321
--- /dev/null
+++ b/third_party/cub/cub/iterator/cache_modified_output_iterator.cuh
@@ -0,0 +1,254 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../util_device.cuh"
+#include "../util_namespace.cuh"
+
+#if (THRUST_VERSION >= 100700)
+    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+
+/**
+ * \brief A random-access output wrapper for storing array values using a PTX cache-modifier.
+ *
+ * \par Overview
+ * - CacheModifiedOutputIterator is a random-access output iterator that wraps a native
+ *   device pointer of type <tt>ValueType*</tt>. \p ValueType references are
+ *   made by writing \p ValueType values through stores modified by \p MODIFIER.
+ * - Can be used to store any data type to memory using PTX cache store modifiers (e.g., "STORE_WB",
+ *   "STORE_CG", "STORE_CS", "STORE_WT", etc.).
+ * - Can be constructed, manipulated, and exchanged within and between host and device
+ *   functions, but can only be dereferenced within device functions.
+ * - Compatible with Thrust API v1.7 or newer.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of \p CacheModifiedOutputIterator to
+ * dereference a device array of doubles using the "wt" PTX load modifier
+ * (i.e., write-through to system memory).
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/iterator/cache_modified_output_iterator.cuh>
+ *
+ * // Declare, allocate, and initialize a device array
+ * double *d_out;              // e.g., [, , , , , , ]
+ *
+ * // Create an iterator wrapper
+ * cub::CacheModifiedOutputIterator<cub::STORE_WT, double> itr(d_out);
+ *
+ * // Within device code:
+ * itr[0]  = 8.0;
+ * itr[1]  = 66.0;
+ * itr[55] = 24.0;
+ *
+ * \endcode
+ *
+ * \par Usage Considerations
+ * - Can only be dereferenced within device code
+ *
+ * \tparam CacheStoreModifier     The cub::CacheStoreModifier to use when accessing data
+ * \tparam ValueType            The value type of this iterator
+ * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
+ */
+template <
+    CacheStoreModifier  MODIFIER,
+    typename            ValueType,
+    typename            OffsetT = ptrdiff_t>
+class CacheModifiedOutputIterator
+{
+private:
+
+    // Proxy object
+    struct Reference
+    {
+        ValueType* ptr;
+
+        /// Constructor
+        __host__ __device__ __forceinline__ Reference(ValueType* ptr) : ptr(ptr) {}
+
+        /// Assignment
+        __device__ __forceinline__ ValueType operator =(ValueType val)
+        {
+            ThreadStore<MODIFIER>(ptr, val);
+            return val;
+        }
+    };
+
+public:
+
+    // Required iterator traits
+    typedef CacheModifiedOutputIterator         self_type;              ///< My own type
+    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef void                                value_type;             ///< The type of the element the iterator can point to
+    typedef void                                pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef Reference                           reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::device_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+private:
+
+    ValueType* ptr;
+
+public:
+
+    /// Constructor
+    template <typename QualifiedValueType>
+    __host__ __device__ __forceinline__ CacheModifiedOutputIterator(
+        QualifiedValueType* ptr)     ///< Native pointer to wrap
+    :
+        ptr(const_cast<typename RemoveQualifiers<QualifiedValueType>::Type *>(ptr))
+    {}
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        ptr++;
+        return retval;
+    }
+
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        ptr++;
+        return *this;
+    }
+
+    /// Indirection
+    __host__ __device__ __forceinline__ reference operator*() const
+    {
+        return Reference(ptr);
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval(ptr + n);
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        ptr += n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval(ptr - n);
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        ptr -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return ptr - other.ptr;
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __host__ __device__ __forceinline__ reference operator[](Distance n) const
+    {
+        return Reference(ptr + n);
+    }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return (ptr == rhs.ptr);
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return (ptr != rhs.ptr);
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
+    {
+        return os;
+    }
+};
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/iterator/constant_input_iterator.cuh b/third_party/cub/cub/iterator/constant_input_iterator.cuh
new file mode 100644
index 0000000..1e0a910
--- /dev/null
+++ b/third_party/cub/cub/iterator/constant_input_iterator.cuh
@@ -0,0 +1,235 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../util_namespace.cuh"
+
+#if (THRUST_VERSION >= 100700)
+    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+
+/**
+ * \brief A random-access input generator for dereferencing a sequence of homogeneous values
+ *
+ * \par Overview
+ * - Read references to a ConstantInputIteratorTiterator always return the supplied constant
+ *   of type \p ValueType.
+ * - Can be used with any data type.
+ * - Can be constructed, manipulated, dereferenced, and exchanged within and between host and device
+ *   functions.
+ * - Compatible with Thrust API v1.7 or newer.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of \p ConstantInputIteratorTto
+ * dereference a sequence of homogeneous doubles.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/iterator/constant_input_iterator.cuh>
+ *
+ * cub::ConstantInputIterator<double> itr(5.0);
+ *
+ * printf("%f\n", itr[0]);      // 5.0
+ * printf("%f\n", itr[1]);      // 5.0
+ * printf("%f\n", itr[2]);      // 5.0
+ * printf("%f\n", itr[50]);     // 5.0
+ *
+ * \endcode
+ *
+ * \tparam ValueType            The value type of this iterator
+ * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
+ */
+template <
+    typename ValueType,
+    typename OffsetT = ptrdiff_t>
+class ConstantInputIterator
+{
+public:
+
+    // Required iterator traits
+    typedef ConstantInputIterator               self_type;              ///< My own type
+    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef ValueType                           value_type;             ///< The type of the element the iterator can point to
+    typedef ValueType*                          pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef ValueType                           reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::any_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+private:
+
+    ValueType   val;
+    OffsetT     offset;
+#ifdef _WIN32
+    OffsetT     pad[CUB_MAX(1, (16 / sizeof(OffsetT) - 1))];        // Workaround for win32 parameter-passing bug (ulonglong2 argmin DeviceReduce)
+#endif
+
+public:
+
+    /// Constructor
+    __host__ __device__ __forceinline__ ConstantInputIterator(
+        ValueType   val,            ///< Starting value for the iterator instance to report
+        OffsetT     offset = 0)     ///< Base offset
+    :
+        val(val),
+        offset(offset)
+    {}
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        offset++;
+        return retval;
+    }
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        offset++;
+        return *this;
+    }
+
+    /// Indirection
+    __host__ __device__ __forceinline__ reference operator*() const
+    {
+        return val;
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval(val, offset + n);
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        offset += n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval(val, offset - n);
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        offset -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return offset - other.offset;
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __host__ __device__ __forceinline__ reference operator[](Distance /*n*/) const
+    {
+        return val;
+    }
+
+    /// Structure dereference
+    __host__ __device__ __forceinline__ pointer operator->()
+    {
+        return &val;
+    }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return (offset == rhs.offset) && ((val == rhs.val));
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return (offset != rhs.offset) || (val!= rhs.val);
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
+    {
+        os << "[" << itr.val << "," << itr.offset << "]";
+        return os;
+    }
+
+};
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/iterator/counting_input_iterator.cuh b/third_party/cub/cub/iterator/counting_input_iterator.cuh
new file mode 100644
index 0000000..7f49348
--- /dev/null
+++ b/third_party/cub/cub/iterator/counting_input_iterator.cuh
@@ -0,0 +1,228 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../util_device.cuh"
+#include "../util_namespace.cuh"
+
+#if (THRUST_VERSION >= 100700)
+    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+/**
+ * \brief A random-access input generator for dereferencing a sequence of incrementing integer values.
+ *
+ * \par Overview
+ * - After initializing a CountingInputIteratorTto a certain integer \p base, read references
+ *   at \p offset will return the value \p base + \p offset.
+ * - Can be constructed, manipulated, dereferenced, and exchanged within and between host and device
+ *   functions.
+ * - Compatible with Thrust API v1.7 or newer.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of \p CountingInputIteratorTto
+ * dereference a sequence of incrementing integers.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/iterator/counting_input_iterator.cuh>
+ *
+ * cub::CountingInputIterator<int> itr(5);
+ *
+ * printf("%d\n", itr[0]);      // 5
+ * printf("%d\n", itr[1]);      // 6
+ * printf("%d\n", itr[2]);      // 7
+ * printf("%d\n", itr[50]);     // 55
+ *
+ * \endcode
+ *
+ * \tparam ValueType            The value type of this iterator
+ * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
+ */
+template <
+    typename ValueType,
+    typename OffsetT = ptrdiff_t>
+class CountingInputIterator
+{
+public:
+
+    // Required iterator traits
+    typedef CountingInputIterator               self_type;              ///< My own type
+    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef ValueType                           value_type;             ///< The type of the element the iterator can point to
+    typedef ValueType*                          pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef ValueType                           reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::any_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+private:
+
+    ValueType val;
+
+public:
+
+    /// Constructor
+    __host__ __device__ __forceinline__ CountingInputIterator(
+        const ValueType &val)          ///< Starting value for the iterator instance to report
+    :
+        val(val)
+    {}
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        val++;
+        return retval;
+    }
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        val++;
+        return *this;
+    }
+
+    /// Indirection
+    __host__ __device__ __forceinline__ reference operator*() const
+    {
+        return val;
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval(val + (ValueType) n);
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        val += (ValueType) n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval(val - (ValueType) n);
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        val -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return (difference_type) (val - other.val);
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __host__ __device__ __forceinline__ reference operator[](Distance n) const
+    {
+        return val + (ValueType) n;
+    }
+
+    /// Structure dereference
+    __host__ __device__ __forceinline__ pointer operator->()
+    {
+        return &val;
+    }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return (val == rhs.val);
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return (val != rhs.val);
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
+    {
+        os << "[" << itr.val << "]";
+        return os;
+    }
+
+};
+
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/iterator/discard_output_iterator.cuh b/third_party/cub/cub/iterator/discard_output_iterator.cuh
new file mode 100644
index 0000000..28473e5
--- /dev/null
+++ b/third_party/cub/cub/iterator/discard_output_iterator.cuh
@@ -0,0 +1,220 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../util_namespace.cuh"
+#include "../util_macro.cuh"
+
+#if (THRUST_VERSION >= 100700)
+    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+
+/**
+ * \brief A discard iterator
+ */
+template <typename OffsetT = ptrdiff_t>
+class DiscardOutputIterator
+{
+public:
+
+    // Required iterator traits
+    typedef DiscardOutputIterator   self_type;              ///< My own type
+    typedef OffsetT                 difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef void                    value_type;             ///< The type of the element the iterator can point to
+    typedef void                    pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef void                    reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::any_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+private:
+
+    OffsetT offset;
+
+#if defined(_WIN32) || !defined(_WIN64)
+    // Workaround for win32 parameter-passing bug (ulonglong2 argmin DeviceReduce)
+    OffsetT pad[CUB_MAX(1, (16 / sizeof(OffsetT) - 1))];
+#endif
+
+public:
+
+    /// Constructor
+    __host__ __device__ __forceinline__ DiscardOutputIterator(
+        OffsetT offset = 0)     ///< Base offset
+    :
+        offset(offset)
+    {}
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        offset++;
+        return retval;
+    }
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        offset++;
+        return *this;
+    }
+
+    /// Indirection
+    __host__ __device__ __forceinline__ self_type& operator*()
+    {
+        // return self reference, which can be assigned to anything
+        return *this;
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval(offset + n);
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        offset += n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval(offset - n);
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        offset -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return offset - other.offset;
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator[](Distance n)
+    {
+        // return self reference, which can be assigned to anything
+        return *this;
+    }
+
+    /// Structure dereference
+    __host__ __device__ __forceinline__ pointer operator->()
+    {
+        return;
+    }
+
+    /// Assignment to self (no-op)
+    __host__ __device__ __forceinline__ void operator=(self_type const& other)
+    {
+        offset = other.offset;
+    }
+
+    /// Assignment to anything else (no-op)
+    template<typename T>
+    __host__ __device__ __forceinline__ void operator=(T const&)
+    {}
+
+    /// Cast to void* operator
+    __host__ __device__ __forceinline__ operator void*() const { return NULL; }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return (offset == rhs.offset);
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return (offset != rhs.offset);
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
+    {
+        os << "[" << itr.offset << "]";
+        return os;
+    }
+
+};
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/iterator/tex_obj_input_iterator.cuh b/third_party/cub/cub/iterator/tex_obj_input_iterator.cuh
new file mode 100644
index 0000000..b99103e
--- /dev/null
+++ b/third_party/cub/cub/iterator/tex_obj_input_iterator.cuh
@@ -0,0 +1,310 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../util_device.cuh"
+#include "../util_debug.cuh"
+#include "../util_namespace.cuh"
+
+#if (THRUST_VERSION >= 100700)
+    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+
+
+/**
+ * \brief A random-access input wrapper for dereferencing array values through texture cache.  Uses newer Kepler-style texture objects.
+ *
+ * \par Overview
+ * - TexObjInputIteratorTwraps a native device pointer of type <tt>ValueType*</tt>. References
+ *   to elements are to be loaded through texture cache.
+ * - Can be used to load any data type from memory through texture cache.
+ * - Can be manipulated and exchanged within and between host and device
+ *   functions, can only be constructed within host functions, and can only be
+ *   dereferenced within device functions.
+ * - With regard to nested/dynamic parallelism, TexObjInputIteratorTiterators may only be
+ *   created by the host thread, but can be used by any descendant kernel.
+ * - Compatible with Thrust API v1.7 or newer.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of \p TexRefInputIteratorTto
+ * dereference a device array of doubles through texture cache.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/iterator/tex_obj_input_iterator.cuh>
+ *
+ * // Declare, allocate, and initialize a device array
+ * int num_items;   // e.g., 7
+ * double *d_in;    // e.g., [8.0, 6.0, 7.0, 5.0, 3.0, 0.0, 9.0]
+ *
+ * // Create an iterator wrapper
+ * cub::TexObjInputIterator<double> itr;
+ * itr.BindTexture(d_in, sizeof(double) * num_items);
+ * ...
+ *
+ * // Within device code:
+ * printf("%f\n", itr[0]);      // 8.0
+ * printf("%f\n", itr[1]);      // 6.0
+ * printf("%f\n", itr[6]);      // 9.0
+ *
+ * ...
+ * itr.UnbindTexture();
+ *
+ * \endcode
+ *
+ * \tparam T                    The value type of this iterator
+ * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
+ */
+template <
+    typename    T,
+    typename    OffsetT = ptrdiff_t>
+class TexObjInputIterator
+{
+public:
+
+    // Required iterator traits
+    typedef TexObjInputIterator                 self_type;              ///< My own type
+    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef T                                   value_type;             ///< The type of the element the iterator can point to
+    typedef T*                                  pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef T                                   reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::device_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+private:
+
+    // Largest texture word we can use in device
+    typedef typename UnitWord<T>::TextureWord TextureWord;
+
+    // Number of texture words per T
+    enum {
+        TEXTURE_MULTIPLE = sizeof(T) / sizeof(TextureWord)
+    };
+
+private:
+
+    T*                  ptr;
+    difference_type     tex_offset;
+    cudaTextureObject_t tex_obj;
+
+public:
+
+    /// Constructor
+    __host__ __device__ __forceinline__ TexObjInputIterator()
+    :
+        ptr(NULL),
+        tex_offset(0),
+        tex_obj(0)
+    {}
+
+    /// Use this iterator to bind \p ptr with a texture reference
+    template <typename QualifiedT>
+    cudaError_t BindTexture(
+        QualifiedT      *ptr,               ///< Native pointer to wrap that is aligned to cudaDeviceProp::textureAlignment
+        size_t          bytes = size_t(-1),         ///< Number of bytes in the range
+        size_t          tex_offset = 0)     ///< OffsetT (in items) from \p ptr denoting the position of the iterator
+    {
+        this->ptr = const_cast<typename RemoveQualifiers<QualifiedT>::Type *>(ptr);
+        this->tex_offset = tex_offset;
+
+        cudaChannelFormatDesc   channel_desc = cudaCreateChannelDesc<TextureWord>();
+        cudaResourceDesc        res_desc;
+        cudaTextureDesc         tex_desc;
+        memset(&res_desc, 0, sizeof(cudaResourceDesc));
+        memset(&tex_desc, 0, sizeof(cudaTextureDesc));
+        res_desc.resType                = cudaResourceTypeLinear;
+        res_desc.res.linear.devPtr      = this->ptr;
+        res_desc.res.linear.desc        = channel_desc;
+        res_desc.res.linear.sizeInBytes = bytes;
+        tex_desc.readMode               = cudaReadModeElementType;
+        return cudaCreateTextureObject(&tex_obj, &res_desc, &tex_desc, NULL);
+    }
+
+    /// Unbind this iterator from its texture reference
+    cudaError_t UnbindTexture()
+    {
+        return cudaDestroyTextureObject(tex_obj);
+    }
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        tex_offset++;
+        return retval;
+    }
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        tex_offset++;
+        return *this;
+    }
+
+    /// Indirection
+    __host__ __device__ __forceinline__ reference operator*() const
+    {
+#if (CUB_PTX_ARCH == 0)
+        // Simply dereference the pointer on the host
+        return ptr[tex_offset];
+#else
+        // Move array of uninitialized words, then alias and assign to return value
+        TextureWord words[TEXTURE_MULTIPLE];
+
+        #pragma unroll
+        for (int i = 0; i < TEXTURE_MULTIPLE; ++i)
+        {
+            words[i] = tex1Dfetch<TextureWord>(
+                tex_obj,
+                (tex_offset * TEXTURE_MULTIPLE) + i);
+        }
+
+        // Load from words
+        return *reinterpret_cast<T*>(words);
+#endif
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval;
+        retval.ptr          = ptr;
+        retval.tex_obj      = tex_obj;
+        retval.tex_offset   = tex_offset + n;
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        tex_offset += n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval;
+        retval.ptr          = ptr;
+        retval.tex_obj      = tex_obj;
+        retval.tex_offset   = tex_offset - n;
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        tex_offset -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return tex_offset - other.tex_offset;
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __host__ __device__ __forceinline__ reference operator[](Distance n) const
+    {
+        self_type offset = (*this) + n;
+        return *offset;
+    }
+
+    /// Structure dereference
+    __host__ __device__ __forceinline__ pointer operator->()
+    {
+        return &(*(*this));
+    }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return ((ptr == rhs.ptr) && (tex_offset == rhs.tex_offset) && (tex_obj == rhs.tex_obj));
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return ((ptr != rhs.ptr) || (tex_offset != rhs.tex_offset) || (tex_obj != rhs.tex_obj));
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
+    {
+        return os;
+    }
+
+};
+
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/iterator/tex_ref_input_iterator.cuh b/third_party/cub/cub/iterator/tex_ref_input_iterator.cuh
new file mode 100644
index 0000000..95d0ffb
--- /dev/null
+++ b/third_party/cub/cub/iterator/tex_ref_input_iterator.cuh
@@ -0,0 +1,374 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../util_device.cuh"
+#include "../util_debug.cuh"
+#include "../util_namespace.cuh"
+
+#if (CUDA_VERSION >= 5050) || defined(DOXYGEN_ACTIVE)  // This iterator is compatible with CUDA 5.5 and newer
+
+#if (THRUST_VERSION >= 100700)    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/******************************************************************************
+ * Static file-scope Tesla/Fermi-style texture references
+ *****************************************************************************/
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+// Anonymous namespace
+namespace {
+
+/// Global texture reference specialized by type
+template <typename T>
+struct IteratorTexRef
+{
+    /// And by unique ID
+    template <int UNIQUE_ID>
+    struct TexId
+    {
+        // Largest texture word we can use in device
+        typedef typename UnitWord<T>::DeviceWord DeviceWord;
+        typedef typename UnitWord<T>::TextureWord TextureWord;
+
+        // Number of texture words per T
+        enum {
+            DEVICE_MULTIPLE = sizeof(T) / sizeof(DeviceWord),
+            TEXTURE_MULTIPLE = sizeof(T) / sizeof(TextureWord)
+        };
+
+        // Texture reference type
+        typedef texture<TextureWord> TexRef;
+
+        // Texture reference
+        static TexRef ref;
+
+        /// Bind texture
+        static cudaError_t BindTexture(void *d_in, size_t &offset)
+        {
+            if (d_in)
+            {
+                cudaChannelFormatDesc tex_desc = cudaCreateChannelDesc<TextureWord>();
+                ref.channelDesc = tex_desc;
+                return (CubDebug(cudaBindTexture(&offset, ref, d_in)));
+            }
+
+            return cudaSuccess;
+        }
+
+        /// Unbind texture
+        static cudaError_t UnbindTexture()
+        {
+            return CubDebug(cudaUnbindTexture(ref));
+        }
+
+        /// Fetch element
+        template <typename Distance>
+        static __device__ __forceinline__ T Fetch(Distance tex_offset)
+        {
+            DeviceWord temp[DEVICE_MULTIPLE];
+            TextureWord *words = reinterpret_cast<TextureWord*>(temp);
+
+            #pragma unroll
+            for (int i = 0; i < TEXTURE_MULTIPLE; ++i)
+            {
+                words[i] = tex1Dfetch(ref, (tex_offset * TEXTURE_MULTIPLE) + i);
+            }
+
+            return reinterpret_cast<T&>(temp);
+        }
+    };
+};
+
+// Texture reference definitions
+template <typename  T>
+template <int       UNIQUE_ID>
+typename IteratorTexRef<T>::template TexId<UNIQUE_ID>::TexRef IteratorTexRef<T>::template TexId<UNIQUE_ID>::ref = 0;
+
+
+} // Anonymous namespace
+
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+
+
+/**
+ * \brief A random-access input wrapper for dereferencing array values through texture cache.  Uses older Tesla/Fermi-style texture references.
+ *
+ * \par Overview
+ * - TexRefInputIteratorTwraps a native device pointer of type <tt>ValueType*</tt>. References
+ *   to elements are to be loaded through texture cache.
+ * - Can be used to load any data type from memory through texture cache.
+ * - Can be manipulated and exchanged within and between host and device
+ *   functions, can only be constructed within host functions, and can only be
+ *   dereferenced within device functions.
+ * - The \p UNIQUE_ID template parameter is used to statically name the underlying texture
+ *   reference.  Only one TexRefInputIteratorTinstance can be bound at any given time for a
+ *   specific combination of (1) data type \p T, (2) \p UNIQUE_ID, (3) host
+ *   thread, and (4) compilation .o unit.
+ * - With regard to nested/dynamic parallelism, TexRefInputIteratorTiterators may only be
+ *   created by the host thread and used by a top-level kernel (i.e. the one which is launched
+ *   from the host).
+ * - Compatible with Thrust API v1.7 or newer.
+ * - Compatible with CUDA toolkit v5.5 or newer.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of \p TexRefInputIteratorTto
+ * dereference a device array of doubles through texture cache.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/iterator/tex_ref_input_iterator.cuh>
+ *
+ * // Declare, allocate, and initialize a device array
+ * int num_items;   // e.g., 7
+ * double *d_in;    // e.g., [8.0, 6.0, 7.0, 5.0, 3.0, 0.0, 9.0]
+ *
+ * // Create an iterator wrapper
+ * cub::TexRefInputIterator<double, __LINE__> itr;
+ * itr.BindTexture(d_in, sizeof(double) * num_items);
+ * ...
+ *
+ * // Within device code:
+ * printf("%f\n", itr[0]);      // 8.0
+ * printf("%f\n", itr[1]);      // 6.0
+ * printf("%f\n", itr[6]);      // 9.0
+ *
+ * ...
+ * itr.UnbindTexture();
+ *
+ * \endcode
+ *
+ * \tparam T                    The value type of this iterator
+ * \tparam UNIQUE_ID            A globally-unique identifier (within the compilation unit) to name the underlying texture reference
+ * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
+ */
+template <
+    typename    T,
+    int         UNIQUE_ID,
+    typename    OffsetT = ptrdiff_t>
+class TexRefInputIterator
+{
+public:
+
+    // Required iterator traits
+    typedef TexRefInputIterator                 self_type;              ///< My own type
+    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef T                                   value_type;             ///< The type of the element the iterator can point to
+    typedef T*                                  pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef T                                   reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::device_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+private:
+
+    T*              ptr;
+    difference_type tex_offset;
+
+    // Texture reference wrapper (old Tesla/Fermi-style textures)
+    typedef typename IteratorTexRef<T>::template TexId<UNIQUE_ID> TexId;
+
+public:
+/*
+    /// Constructor
+    __host__ __device__ __forceinline__ TexRefInputIterator()
+    :
+        ptr(NULL),
+        tex_offset(0)
+    {}
+*/
+    /// Use this iterator to bind \p ptr with a texture reference
+    template <typename QualifiedT>
+    cudaError_t BindTexture(
+        QualifiedT      *ptr,                   ///< Native pointer to wrap that is aligned to cudaDeviceProp::textureAlignment
+        size_t          bytes = size_t(-1),     ///< Number of bytes in the range
+        size_t          tex_offset = 0)         ///< OffsetT (in items) from \p ptr denoting the position of the iterator
+    {
+        this->ptr = const_cast<typename RemoveQualifiers<QualifiedT>::Type *>(ptr);
+        size_t offset;
+        cudaError_t retval = TexId::BindTexture(this->ptr + tex_offset, offset);
+        this->tex_offset = (difference_type) (offset / sizeof(QualifiedT));
+        return retval;
+    }
+
+    /// Unbind this iterator from its texture reference
+    cudaError_t UnbindTexture()
+    {
+        return TexId::UnbindTexture();
+    }
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        tex_offset++;
+        return retval;
+    }
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        tex_offset++;
+        return *this;
+    }
+
+    /// Indirection
+    __host__ __device__ __forceinline__ reference operator*() const
+    {
+#if (CUB_PTX_ARCH == 0)
+        // Simply dereference the pointer on the host
+        return ptr[tex_offset];
+#else
+        // Use the texture reference
+        return TexId::Fetch(tex_offset);
+#endif
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval;
+        retval.ptr = ptr;
+        retval.tex_offset = tex_offset + n;
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        tex_offset += n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval;
+        retval.ptr = ptr;
+        retval.tex_offset = tex_offset - n;
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        tex_offset -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return tex_offset - other.tex_offset;
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __host__ __device__ __forceinline__ reference operator[](Distance n) const
+    {
+        self_type offset = (*this) + n;
+        return *offset;
+    }
+
+    /// Structure dereference
+    __host__ __device__ __forceinline__ pointer operator->()
+    {
+        return &(*(*this));
+    }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return ((ptr == rhs.ptr) && (tex_offset == rhs.tex_offset));
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return ((ptr != rhs.ptr) || (tex_offset != rhs.tex_offset));
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
+    {
+        return os;
+    }
+
+};
+
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
+
+#endif // CUDA_VERSION
diff --git a/third_party/cub/cub/iterator/transform_input_iterator.cuh b/third_party/cub/cub/iterator/transform_input_iterator.cuh
new file mode 100644
index 0000000..dad1f50
--- /dev/null
+++ b/third_party/cub/cub/iterator/transform_input_iterator.cuh
@@ -0,0 +1,252 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Random-access iterator types
+ */
+
+#pragma once
+
+#include <iterator>
+#include <iostream>
+
+#include "../thread/thread_load.cuh"
+#include "../thread/thread_store.cuh"
+#include "../util_device.cuh"
+#include "../util_namespace.cuh"
+
+#if (THRUST_VERSION >= 100700)
+    // This iterator is compatible with Thrust API 1.7 and newer
+    #include <thrust/iterator/iterator_facade.h>
+    #include <thrust/iterator/iterator_traits.h>
+#endif // THRUST_VERSION
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup UtilIterator
+ * @{
+ */
+
+
+/**
+ * \brief A random-access input wrapper for transforming dereferenced values.
+ *
+ * \par Overview
+ * - TransformInputIteratorTwraps a unary conversion functor of type \p
+ *   ConversionOp and a random-access input iterator of type <tt>InputIteratorT</tt>,
+ *   using the former to produce references of type \p ValueType from the latter.
+ * - Can be used with any data type.
+ * - Can be constructed, manipulated, and exchanged within and between host and device
+ *   functions.  Wrapped host memory can only be dereferenced on the host, and wrapped
+ *   device memory can only be dereferenced on the device.
+ * - Compatible with Thrust API v1.7 or newer.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of \p TransformInputIteratorTto
+ * dereference an array of integers, tripling the values and converting them to doubles.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/iterator/transform_input_iterator.cuh>
+ *
+ * // Functor for tripling integer values and converting to doubles
+ * struct TripleDoubler
+ * {
+ *     __host__ __device__ __forceinline__
+ *     double operator()(const int &a) const {
+ *         return double(a * 3);
+ *     }
+ * };
+ *
+ * // Declare, allocate, and initialize a device array
+ * int *d_in;                   // e.g., [8, 6, 7, 5, 3, 0, 9]
+ * TripleDoubler conversion_op;
+ *
+ * // Create an iterator wrapper
+ * cub::TransformInputIterator<double, TripleDoubler, int*> itr(d_in, conversion_op);
+ *
+ * // Within device code:
+ * printf("%f\n", itr[0]);  // 24.0
+ * printf("%f\n", itr[1]);  // 18.0
+ * printf("%f\n", itr[6]);  // 27.0
+ *
+ * \endcode
+ *
+ * \tparam ValueType            The value type of this iterator
+ * \tparam ConversionOp         Unary functor type for mapping objects of type \p InputType to type \p ValueType.  Must have member <tt>ValueType operator()(const InputType &datum)</tt>.
+ * \tparam InputIteratorT       The type of the wrapped input iterator
+ * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
+ *
+ */
+template <
+    typename ValueType,
+    typename ConversionOp,
+    typename InputIteratorT,
+    typename OffsetT = ptrdiff_t>
+class TransformInputIterator
+{
+public:
+
+    // Required iterator traits
+    typedef TransformInputIterator              self_type;              ///< My own type
+    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
+    typedef ValueType                           value_type;             ///< The type of the element the iterator can point to
+    typedef ValueType*                          pointer;                ///< The type of a pointer to an element the iterator can point to
+    typedef ValueType                           reference;              ///< The type of a reference to an element the iterator can point to
+
+#if (THRUST_VERSION >= 100700)
+    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
+    typedef typename thrust::detail::iterator_facade_category<
+        thrust::any_system_tag,
+        thrust::random_access_traversal_tag,
+        value_type,
+        reference
+      >::type iterator_category;                                        ///< The iterator category
+#else
+    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
+#endif  // THRUST_VERSION
+
+private:
+
+    ConversionOp    conversion_op;
+    InputIteratorT  input_itr;
+
+public:
+
+    /// Constructor
+    __host__ __device__ __forceinline__ TransformInputIterator(
+        InputIteratorT      input_itr,          ///< Input iterator to wrap
+        ConversionOp        conversion_op)      ///< Conversion functor to wrap
+    :
+        conversion_op(conversion_op),
+        input_itr(input_itr)
+    {}
+
+    /// Postfix increment
+    __host__ __device__ __forceinline__ self_type operator++(int)
+    {
+        self_type retval = *this;
+        input_itr++;
+        return retval;
+    }
+
+    /// Prefix increment
+    __host__ __device__ __forceinline__ self_type operator++()
+    {
+        input_itr++;
+        return *this;
+    }
+
+    /// Indirection
+    __host__ __device__ __forceinline__ reference operator*() const
+    {
+        return conversion_op(*input_itr);
+    }
+
+    /// Addition
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
+    {
+        self_type retval(input_itr + n, conversion_op);
+        return retval;
+    }
+
+    /// Addition assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
+    {
+        input_itr += n;
+        return *this;
+    }
+
+    /// Subtraction
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
+    {
+        self_type retval(input_itr - n, conversion_op);
+        return retval;
+    }
+
+    /// Subtraction assignment
+    template <typename Distance>
+    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
+    {
+        input_itr -= n;
+        return *this;
+    }
+
+    /// Distance
+    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
+    {
+        return input_itr - other.input_itr;
+    }
+
+    /// Array subscript
+    template <typename Distance>
+    __host__ __device__ __forceinline__ reference operator[](Distance n) const
+    {
+        return conversion_op(input_itr[n]);
+    }
+
+    /// Structure dereference
+    __host__ __device__ __forceinline__ pointer operator->()
+    {
+        return &conversion_op(*input_itr);
+    }
+
+    /// Equal to
+    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
+    {
+        return (input_itr == rhs.input_itr);
+    }
+
+    /// Not equal to
+    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
+    {
+        return (input_itr != rhs.input_itr);
+    }
+
+    /// ostream operator
+    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
+    {
+        return os;
+    }
+};
+
+
+
+/** @} */       // end group UtilIterator
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/thread/thread_load.cuh b/third_party/cub/cub/thread/thread_load.cuh
new file mode 100644
index 0000000..b1ca412
--- /dev/null
+++ b/third_party/cub/cub/thread/thread_load.cuh
@@ -0,0 +1,438 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Thread utilities for reading memory using PTX cache modifiers.
+ */
+
+#pragma once
+
+#include <cuda.h>
+
+#include <iterator>
+
+#include "../util_ptx.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup UtilIo
+ * @{
+ */
+
+//-----------------------------------------------------------------------------
+// Tags and constants
+//-----------------------------------------------------------------------------
+
+/**
+ * \brief Enumeration of cache modifiers for memory load operations.
+ */
+enum CacheLoadModifier
+{
+    LOAD_DEFAULT,       ///< Default (no modifier)
+    LOAD_CA,            ///< Cache at all levels
+    LOAD_CG,            ///< Cache at global level
+    LOAD_CS,            ///< Cache streaming (likely to be accessed once)
+    LOAD_CV,            ///< Cache as volatile (including cached system lines)
+    LOAD_LDG,           ///< Cache as texture
+    LOAD_VOLATILE,      ///< Volatile (any memory space)
+};
+
+
+/**
+ * \name Thread I/O (cache modified)
+ * @{
+ */
+
+/**
+ * \brief Thread utility for reading memory using cub::CacheLoadModifier cache modifiers.  Can be used to load any data type.
+ *
+ * \par Example
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/thread/thread_load.cuh>
+ *
+ * // 32-bit load using cache-global modifier:
+ * int *d_in;
+ * int val = cub::ThreadLoad<cub::LOAD_CA>(d_in + threadIdx.x);
+ *
+ * // 16-bit load using default modifier
+ * short *d_in;
+ * short val = cub::ThreadLoad<cub::LOAD_DEFAULT>(d_in + threadIdx.x);
+ *
+ * // 256-bit load using cache-volatile modifier
+ * double4 *d_in;
+ * double4 val = cub::ThreadLoad<cub::LOAD_CV>(d_in + threadIdx.x);
+ *
+ * // 96-bit load using cache-streaming modifier
+ * struct TestFoo { bool a; short b; };
+ * TestFoo *d_struct;
+ * TestFoo val = cub::ThreadLoad<cub::LOAD_CS>(d_in + threadIdx.x);
+ * \endcode
+ *
+ * \tparam MODIFIER             <b>[inferred]</b> CacheLoadModifier enumeration
+ * \tparam InputIteratorT       <b>[inferred]</b> Input iterator type \iterator
+ */
+template <
+    CacheLoadModifier MODIFIER,
+    typename InputIteratorT>
+__device__ __forceinline__ typename std::iterator_traits<InputIteratorT>::value_type ThreadLoad(InputIteratorT itr);
+
+
+//@}  end member group
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+
+/// Helper structure for templated load iteration (inductive case)
+template <int COUNT, int MAX>
+struct IterateThreadLoad
+{
+    template <CacheLoadModifier MODIFIER, typename T>
+    static __device__ __forceinline__ void Load(T const *ptr, T *vals)
+    {
+        vals[COUNT] = ThreadLoad<MODIFIER>(ptr + COUNT);
+        IterateThreadLoad<COUNT + 1, MAX>::template Load<MODIFIER>(ptr, vals);
+    }
+
+    template <typename InputIteratorT, typename T>
+    static __device__ __forceinline__ void Dereference(InputIteratorT itr, T *vals)
+    {
+        vals[COUNT] = itr[COUNT];
+        IterateThreadLoad<COUNT + 1, MAX>::Dereference(itr, vals);
+    }
+};
+
+
+/// Helper structure for templated load iteration (termination case)
+template <int MAX>
+struct IterateThreadLoad<MAX, MAX>
+{
+    template <CacheLoadModifier MODIFIER, typename T>
+    static __device__ __forceinline__ void Load(T const * /*ptr*/, T * /*vals*/) {}
+
+    template <typename InputIteratorT, typename T>
+    static __device__ __forceinline__ void Dereference(InputIteratorT /*itr*/, T * /*vals*/) {}
+};
+
+
+/**
+ * Define a uint4 (16B) ThreadLoad specialization for the given Cache load modifier
+ */
+#define _CUB_LOAD_16(cub_modifier, ptx_modifier)                                             \
+    template<>                                                                              \
+    __device__ __forceinline__ uint4 ThreadLoad<cub_modifier, uint4 const *>(uint4 const *ptr)                   \
+    {                                                                                       \
+        uint4 retval;                                                                       \
+        asm volatile ("ld."#ptx_modifier".v4.u32 {%0, %1, %2, %3}, [%4];" :                 \
+            "=r"(retval.x),                                                                 \
+            "=r"(retval.y),                                                                 \
+            "=r"(retval.z),                                                                 \
+            "=r"(retval.w) :                                                                \
+            _CUB_ASM_PTR_(ptr));                                                            \
+        return retval;                                                                      \
+    }                                                                                       \
+    template<>                                                                              \
+    __device__ __forceinline__ ulonglong2 ThreadLoad<cub_modifier, ulonglong2 const *>(ulonglong2 const *ptr)    \
+    {                                                                                       \
+        ulonglong2 retval;                                                                  \
+        asm volatile ("ld."#ptx_modifier".v2.u64 {%0, %1}, [%2];" :                         \
+            "=l"(retval.x),                                                                 \
+            "=l"(retval.y) :                                                                \
+            _CUB_ASM_PTR_(ptr));                                                            \
+        return retval;                                                                      \
+    }
+
+/**
+ * Define a uint2 (8B) ThreadLoad specialization for the given Cache load modifier
+ */
+#define _CUB_LOAD_8(cub_modifier, ptx_modifier)                                              \
+    template<>                                                                              \
+    __device__ __forceinline__ ushort4 ThreadLoad<cub_modifier, ushort4 const *>(ushort4 const *ptr)             \
+    {                                                                                       \
+        ushort4 retval;                                                                     \
+        asm volatile ("ld."#ptx_modifier".v4.u16 {%0, %1, %2, %3}, [%4];" :                 \
+            "=h"(retval.x),                                                                 \
+            "=h"(retval.y),                                                                 \
+            "=h"(retval.z),                                                                 \
+            "=h"(retval.w) :                                                                \
+            _CUB_ASM_PTR_(ptr));                                                            \
+        return retval;                                                                      \
+    }                                                                                       \
+    template<>                                                                              \
+    __device__ __forceinline__ uint2 ThreadLoad<cub_modifier, uint2 const *>(uint2 const *ptr)                   \
+    {                                                                                       \
+        uint2 retval;                                                                       \
+        asm volatile ("ld."#ptx_modifier".v2.u32 {%0, %1}, [%2];" :                         \
+            "=r"(retval.x),                                                                 \
+            "=r"(retval.y) :                                                                \
+            _CUB_ASM_PTR_(ptr));                                                            \
+        return retval;                                                                      \
+    }                                                                                       \
+    template<>                                                                              \
+    __device__ __forceinline__ unsigned long long ThreadLoad<cub_modifier, unsigned long long const *>(unsigned long long const *ptr)    \
+    {                                                                                       \
+        unsigned long long retval;                                                          \
+        asm volatile ("ld."#ptx_modifier".u64 %0, [%1];" :                                  \
+            "=l"(retval) :                                                                  \
+            _CUB_ASM_PTR_(ptr));                                                            \
+        return retval;                                                                      \
+    }
+
+/**
+ * Define a uint (4B) ThreadLoad specialization for the given Cache load modifier
+ */
+#define _CUB_LOAD_4(cub_modifier, ptx_modifier)                                              \
+    template<>                                                                              \
+    __device__ __forceinline__ unsigned int ThreadLoad<cub_modifier, unsigned int const *>(unsigned int const *ptr)                      \
+    {                                                                                       \
+        unsigned int retval;                                                                \
+        asm volatile ("ld."#ptx_modifier".u32 %0, [%1];" :                                  \
+            "=r"(retval) :                                                                  \
+            _CUB_ASM_PTR_(ptr));                                                            \
+        return retval;                                                                      \
+    }
+
+
+/**
+ * Define a unsigned short (2B) ThreadLoad specialization for the given Cache load modifier
+ */
+#define _CUB_LOAD_2(cub_modifier, ptx_modifier)                                              \
+    template<>                                                                              \
+    __device__ __forceinline__ unsigned short ThreadLoad<cub_modifier, unsigned short const *>(unsigned short const *ptr)                \
+    {                                                                                       \
+        unsigned short retval;                                                              \
+        asm volatile ("ld."#ptx_modifier".u16 %0, [%1];" :                                  \
+            "=h"(retval) :                                                                  \
+            _CUB_ASM_PTR_(ptr));                                                            \
+        return retval;                                                                      \
+    }
+
+
+/**
+ * Define an unsigned char (1B) ThreadLoad specialization for the given Cache load modifier
+ */
+#define _CUB_LOAD_1(cub_modifier, ptx_modifier)                                              \
+    template<>                                                                              \
+    __device__ __forceinline__ unsigned char ThreadLoad<cub_modifier, unsigned char const *>(unsigned char const *ptr)                   \
+    {                                                                                       \
+        unsigned short retval;                                                              \
+        asm volatile (                                                                      \
+        "{"                                                                                 \
+        "   .reg .u8 datum;"                                                                \
+        "    ld."#ptx_modifier".u8 datum, [%1];"                                            \
+        "    cvt.u16.u8 %0, datum;"                                                         \
+        "}" :                                                                               \
+            "=h"(retval) :                                                                  \
+            _CUB_ASM_PTR_(ptr));                                                            \
+        return (unsigned char) retval;                                                      \
+    }
+
+
+/**
+ * Define powers-of-two ThreadLoad specializations for the given Cache load modifier
+ */
+#define _CUB_LOAD_ALL(cub_modifier, ptx_modifier)                                            \
+    _CUB_LOAD_16(cub_modifier, ptx_modifier)                                                 \
+    _CUB_LOAD_8(cub_modifier, ptx_modifier)                                                  \
+    _CUB_LOAD_4(cub_modifier, ptx_modifier)                                                  \
+    _CUB_LOAD_2(cub_modifier, ptx_modifier)                                                  \
+    _CUB_LOAD_1(cub_modifier, ptx_modifier)                                                  \
+
+
+/**
+ * Define powers-of-two ThreadLoad specializations for the various Cache load modifiers
+ */
+#if CUB_PTX_ARCH >= 200
+    _CUB_LOAD_ALL(LOAD_CA, ca)
+    _CUB_LOAD_ALL(LOAD_CG, cg)
+    _CUB_LOAD_ALL(LOAD_CS, cs)
+    _CUB_LOAD_ALL(LOAD_CV, cv)
+#else
+    _CUB_LOAD_ALL(LOAD_CA, global)
+    // Use volatile to ensure coherent reads when this PTX is JIT'd to run on newer architectures with L1
+    _CUB_LOAD_ALL(LOAD_CG, volatile.global)
+    _CUB_LOAD_ALL(LOAD_CS, global)
+    _CUB_LOAD_ALL(LOAD_CV, volatile.global)
+#endif
+
+#if CUB_PTX_ARCH >= 350
+    _CUB_LOAD_ALL(LOAD_LDG, global.nc)
+#else
+    _CUB_LOAD_ALL(LOAD_LDG, global)
+#endif
+
+
+// Macro cleanup
+#undef _CUB_LOAD_ALL
+#undef _CUB_LOAD_1
+#undef _CUB_LOAD_2
+#undef _CUB_LOAD_4
+#undef _CUB_LOAD_8
+#undef _CUB_LOAD_16
+
+
+
+/**
+ * ThreadLoad definition for LOAD_DEFAULT modifier on iterator types
+ */
+template <typename InputIteratorT>
+__device__ __forceinline__ typename std::iterator_traits<InputIteratorT>::value_type ThreadLoad(
+    InputIteratorT          itr,
+    Int2Type<LOAD_DEFAULT>  /*modifier*/,
+    Int2Type<false>         /*is_pointer*/)
+{
+    return *itr;
+}
+
+
+/**
+ * ThreadLoad definition for LOAD_DEFAULT modifier on pointer types
+ */
+template <typename T>
+__device__ __forceinline__ T ThreadLoad(
+    T                       *ptr,
+    Int2Type<LOAD_DEFAULT>  /*modifier*/,
+    Int2Type<true>          /*is_pointer*/)
+{
+    return *ptr;
+}
+
+
+/**
+ * ThreadLoad definition for LOAD_VOLATILE modifier on primitive pointer types
+ */
+template <typename T>
+__device__ __forceinline__ T ThreadLoadVolatilePointer(
+    T                       *ptr,
+    Int2Type<true>          /*is_primitive*/)
+{
+    T retval = *reinterpret_cast<volatile T*>(ptr);
+    return retval;
+}
+
+
+/**
+ * ThreadLoad definition for LOAD_VOLATILE modifier on non-primitive pointer types
+ */
+template <typename T>
+__device__ __forceinline__ T ThreadLoadVolatilePointer(
+    T                       *ptr,
+    Int2Type<false>         /*is_primitive*/)
+{
+    typedef typename UnitWord<T>::VolatileWord VolatileWord;   // Word type for memcopying
+
+    const int VOLATILE_MULTIPLE = sizeof(T) / sizeof(VolatileWord);
+/*
+    VolatileWord words[VOLATILE_MULTIPLE];
+
+    IterateThreadLoad<0, VOLATILE_MULTIPLE>::Dereference(
+        reinterpret_cast<volatile VolatileWord*>(ptr),
+        words);
+
+    return *reinterpret_cast<T*>(words);
+*/
+
+    T retval;
+    VolatileWord *words = reinterpret_cast<VolatileWord*>(&retval);
+    IterateThreadLoad<0, VOLATILE_MULTIPLE>::Dereference(
+        reinterpret_cast<volatile VolatileWord*>(ptr),
+        words);
+    return retval;
+}
+
+
+/**
+ * ThreadLoad definition for LOAD_VOLATILE modifier on pointer types
+ */
+template <typename T>
+__device__ __forceinline__ T ThreadLoad(
+    T                       *ptr,
+    Int2Type<LOAD_VOLATILE> /*modifier*/,
+    Int2Type<true>          /*is_pointer*/)
+{
+    // Apply tags for partial-specialization
+    return ThreadLoadVolatilePointer(ptr, Int2Type<Traits<T>::PRIMITIVE>());
+}
+
+
+/**
+ * ThreadLoad definition for generic modifiers on pointer types
+ */
+template <typename T, int MODIFIER>
+__device__ __forceinline__ T ThreadLoad(
+    T const                 *ptr,
+    Int2Type<MODIFIER>      /*modifier*/,
+    Int2Type<true>          /*is_pointer*/)
+{
+    typedef typename UnitWord<T>::DeviceWord DeviceWord;
+
+    const int DEVICE_MULTIPLE = sizeof(T) / sizeof(DeviceWord);
+
+    DeviceWord words[DEVICE_MULTIPLE];
+
+    IterateThreadLoad<0, DEVICE_MULTIPLE>::template Load<CacheLoadModifier(MODIFIER)>(
+        reinterpret_cast<DeviceWord*>(const_cast<T*>(ptr)),
+        words);
+
+    return *reinterpret_cast<T*>(words);
+}
+
+
+/**
+ * ThreadLoad definition for generic modifiers
+ */
+template <
+    CacheLoadModifier MODIFIER,
+    typename InputIteratorT>
+__device__ __forceinline__ typename std::iterator_traits<InputIteratorT>::value_type ThreadLoad(InputIteratorT itr)
+{
+    // Apply tags for partial-specialization
+    return ThreadLoad(
+        itr,
+        Int2Type<MODIFIER>(),
+        Int2Type<IsPointer<InputIteratorT>::VALUE>());
+}
+
+
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/** @} */       // end group UtilIo
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/thread/thread_operators.cuh b/third_party/cub/cub/thread/thread_operators.cuh
new file mode 100644
index 0000000..76cd800
--- /dev/null
+++ b/third_party/cub/cub/thread/thread_operators.cuh
@@ -0,0 +1,317 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Simple binary operator functor types
+ */
+
+/******************************************************************************
+ * Simple functor operators
+ ******************************************************************************/
+
+#pragma once
+
+#include "../util_macro.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilModule
+ * @{
+ */
+
+/**
+ * \brief Default equality functor
+ */
+struct Equality
+{
+    /// Boolean equality operator, returns <tt>(a == b)</tt>
+    template <typename T>
+    __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b) const
+    {
+        return a == b;
+    }
+};
+
+
+/**
+ * \brief Default inequality functor
+ */
+struct Inequality
+{
+    /// Boolean inequality operator, returns <tt>(a != b)</tt>
+    template <typename T>
+    __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b) const
+    {
+        return a != b;
+    }
+};
+
+
+/**
+ * \brief Inequality functor (wraps equality functor)
+ */
+template <typename EqualityOp>
+struct InequalityWrapper
+{
+    /// Wrapped equality operator
+    EqualityOp op;
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    InequalityWrapper(EqualityOp op) : op(op) {}
+
+    /// Boolean inequality operator, returns <tt>(a != b)</tt>
+    template <typename T>
+    __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b)
+    {
+        return !op(a, b);
+    }
+};
+
+
+/**
+ * \brief Default sum functor
+ */
+struct Sum
+{
+    /// Boolean sum operator, returns <tt>a + b</tt>
+    template <typename T>
+    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
+    {
+        return a + b;
+    }
+};
+
+
+/**
+ * \brief Default max functor
+ */
+struct Max
+{
+    /// Boolean max operator, returns <tt>(a > b) ? a : b</tt>
+    template <typename T>
+    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
+    {
+        return CUB_MAX(a, b);
+    }
+};
+
+
+/**
+ * \brief Arg max functor (keeps the value and offset of the first occurrence of the larger item)
+ */
+struct ArgMax
+{
+    /// Boolean max operator, preferring the item having the smaller offset in case of ties
+    template <typename T, typename OffsetT>
+    __host__ __device__ __forceinline__ KeyValuePair<OffsetT, T> operator()(
+        const KeyValuePair<OffsetT, T> &a,
+        const KeyValuePair<OffsetT, T> &b) const
+    {
+// Mooch BUG (device reduce argmax gk110 3.2 million random fp32)
+//        return ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key))) ? b : a;
+
+        if ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key)))
+            return b;
+        return a;
+    }
+};
+
+
+/**
+ * \brief Default min functor
+ */
+struct Min
+{
+    /// Boolean min operator, returns <tt>(a < b) ? a : b</tt>
+    template <typename T>
+    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
+    {
+        return CUB_MIN(a, b);
+    }
+};
+
+
+/**
+ * \brief Arg min functor (keeps the value and offset of the first occurrence of the smallest item)
+ */
+struct ArgMin
+{
+    /// Boolean min operator, preferring the item having the smaller offset in case of ties
+    template <typename T, typename OffsetT>
+    __host__ __device__ __forceinline__ KeyValuePair<OffsetT, T> operator()(
+        const KeyValuePair<OffsetT, T> &a,
+        const KeyValuePair<OffsetT, T> &b) const
+    {
+// Mooch BUG (device reduce argmax gk110 3.2 million random fp32)
+//        return ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key))) ? b : a;
+
+        if ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key)))
+            return b;
+        return a;
+    }
+};
+
+
+/**
+ * \brief Default cast functor
+ */
+template <typename B>
+struct CastOp
+{
+    /// Cast operator, returns <tt>(B) a</tt>
+    template <typename A>
+    __host__ __device__ __forceinline__ B operator()(const A &a) const
+    {
+        return (B) a;
+    }
+};
+
+
+/**
+ * \brief Binary operator wrapper for switching non-commutative scan arguments
+ */
+template <typename ScanOp>
+class SwizzleScanOp
+{
+private:
+
+    /// Wrapped scan operator
+    ScanOp scan_op;
+
+public:
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    SwizzleScanOp(ScanOp scan_op) : scan_op(scan_op) {}
+
+    /// Switch the scan arguments
+    template <typename T>
+    __host__ __device__ __forceinline__
+    T operator()(const T &a, const T &b)
+    {
+      T _a(a);
+      T _b(b);
+
+      return scan_op(_b, _a);
+    }
+};
+
+
+/**
+ * \brief Reduce-by-segment functor.
+ *
+ * Given two cub::KeyValuePair inputs \p a and \p b and a
+ * binary associative combining operator \p <tt>f(const T &x, const T &y)</tt>,
+ * an instance of this functor returns a cub::KeyValuePair whose \p key
+ * field is <tt>a.key</tt> + <tt>b.key</tt>, and whose \p value field
+ * is either b.value if b.key is non-zero, or f(a.value, b.value) otherwise.
+ *
+ * ReduceBySegmentOp is an associative, non-commutative binary combining operator
+ * for input sequences of cub::KeyValuePair pairings.  Such
+ * sequences are typically used to represent a segmented set of values to be reduced
+ * and a corresponding set of {0,1}-valued integer "head flags" demarcating the
+ * first value of each segment.
+ *
+ */
+template <typename ReductionOpT>    ///< Binary reduction operator to apply to values
+struct ReduceBySegmentOp
+{
+    /// Wrapped reduction operator
+    ReductionOpT op;
+
+    /// Constructor
+    __host__ __device__ __forceinline__ ReduceBySegmentOp() {}
+
+    /// Constructor
+    __host__ __device__ __forceinline__ ReduceBySegmentOp(ReductionOpT op) : op(op) {}
+
+    /// Scan operator
+    template <typename KeyValuePairT>       ///< KeyValuePair pairing of T (value) and OffsetT (head flag)
+    __host__ __device__ __forceinline__ KeyValuePairT operator()(
+        const KeyValuePairT &first,         ///< First partial reduction
+        const KeyValuePairT &second)        ///< Second partial reduction
+    {
+        KeyValuePairT retval;
+        retval.key = first.key + second.key;
+        retval.value = (second.key) ?
+                second.value :                          // The second partial reduction spans a segment reset, so it's value aggregate becomes the running aggregate
+                op(first.value, second.value);          // The second partial reduction does not span a reset, so accumulate both into the running aggregate
+        return retval;
+    }
+};
+
+
+
+template <typename ReductionOpT>    ///< Binary reduction operator to apply to values
+struct ReduceByKeyOp
+{
+    /// Wrapped reduction operator
+    ReductionOpT op;
+
+    /// Constructor
+    __host__ __device__ __forceinline__ ReduceByKeyOp() {}
+
+    /// Constructor
+    __host__ __device__ __forceinline__ ReduceByKeyOp(ReductionOpT op) : op(op) {}
+
+    /// Scan operator
+    template <typename KeyValuePairT>
+    __host__ __device__ __forceinline__ KeyValuePairT operator()(
+        const KeyValuePairT &first,       ///< First partial reduction
+        const KeyValuePairT &second)      ///< Second partial reduction
+    {
+        KeyValuePairT retval = second;
+
+        if (first.key == second.key)
+            retval.value = op(first.value, retval.value);
+
+        return retval;
+    }
+};
+
+
+
+
+
+
+
+/** @} */       // end group UtilModule
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/thread/thread_reduce.cuh b/third_party/cub/cub/thread/thread_reduce.cuh
new file mode 100644
index 0000000..4c13688
--- /dev/null
+++ b/third_party/cub/cub/thread/thread_reduce.cuh
@@ -0,0 +1,152 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Thread utilities for sequential reduction over statically-sized array types
+ */
+
+#pragma once
+
+#include "../thread/thread_operators.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/// Internal namespace (to prevent ADL mishaps between static functions when mixing different CUB installations)
+namespace internal {
+
+/**
+ * Sequential reduction over statically-sized array types
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ReductionOp>
+__device__ __forceinline__ T ThreadReduce(
+    T*                  input,                  ///< [in] Input array
+    ReductionOp         reduction_op,           ///< [in] Binary reduction operator
+    T                   prefix,                 ///< [in] Prefix to seed reduction with
+    Int2Type<LENGTH>    /*length*/)
+{
+    T retval = prefix;
+
+    #pragma unroll
+    for (int i = 0; i < LENGTH; ++i)
+        retval = reduction_op(retval, input[i]);
+
+    return retval;
+}
+
+
+/**
+ * \brief Perform a sequential reduction over \p LENGTH elements of the \p input array, seeded with the specified \p prefix.  The aggregate is returned.
+ *
+ * \tparam LENGTH     LengthT of input array
+ * \tparam T          <b>[inferred]</b> The data type to be reduced.
+ * \tparam ScanOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ReductionOp>
+__device__ __forceinline__ T ThreadReduce(
+    T*          input,                  ///< [in] Input array
+    ReductionOp reduction_op,           ///< [in] Binary reduction operator
+    T           prefix)                 ///< [in] Prefix to seed reduction with
+{
+    return ThreadReduce(input, reduction_op, prefix, Int2Type<LENGTH>());
+}
+
+
+/**
+ * \brief Perform a sequential reduction over \p LENGTH elements of the \p input array.  The aggregate is returned.
+ *
+ * \tparam LENGTH     LengthT of input array
+ * \tparam T          <b>[inferred]</b> The data type to be reduced.
+ * \tparam ScanOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ReductionOp>
+__device__ __forceinline__ T ThreadReduce(
+    T*          input,                  ///< [in] Input array
+    ReductionOp reduction_op)           ///< [in] Binary reduction operator
+{
+    T prefix = input[0];
+    return ThreadReduce<LENGTH - 1>(input + 1, reduction_op, prefix);
+}
+
+
+/**
+ * \brief Perform a sequential reduction over the statically-sized \p input array, seeded with the specified \p prefix.  The aggregate is returned.
+ *
+ * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input array
+ * \tparam T          <b>[inferred]</b> The data type to be reduced.
+ * \tparam ScanOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ReductionOp>
+__device__ __forceinline__ T ThreadReduce(
+    T           (&input)[LENGTH],       ///< [in] Input array
+    ReductionOp reduction_op,           ///< [in] Binary reduction operator
+    T           prefix)                 ///< [in] Prefix to seed reduction with
+{
+    return ThreadReduce(input, reduction_op, prefix, Int2Type<LENGTH>());
+}
+
+
+/**
+ * \brief Serial reduction with the specified operator
+ *
+ * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input array
+ * \tparam T          <b>[inferred]</b> The data type to be reduced.
+ * \tparam ScanOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ReductionOp>
+__device__ __forceinline__ T ThreadReduce(
+    T           (&input)[LENGTH],       ///< [in] Input array
+    ReductionOp reduction_op)           ///< [in] Binary reduction operator
+{
+    return ThreadReduce<LENGTH>((T*) input, reduction_op);
+}
+
+
+}               // internal namespace
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/thread/thread_scan.cuh b/third_party/cub/cub/thread/thread_scan.cuh
new file mode 100644
index 0000000..8d67549
--- /dev/null
+++ b/third_party/cub/cub/thread/thread_scan.cuh
@@ -0,0 +1,268 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Thread utilities for sequential prefix scan over statically-sized array types
+ */
+
+#pragma once
+
+#include "../thread/thread_operators.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/// Internal namespace (to prevent ADL mishaps between static functions when mixing different CUB installations)
+namespace internal {
+
+
+/**
+ * \addtogroup UtilModule
+ * @{
+ */
+
+/**
+ * \name Sequential prefix scan over statically-sized array types
+ * @{
+ */
+
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ScanOp>
+__device__ __forceinline__ T ThreadScanExclusive(
+    T                   inclusive,
+    T                   exclusive,
+    T                   *input,                 ///< [in] Input array
+    T                   *output,                ///< [out] Output array (may be aliased to \p input)
+    ScanOp              scan_op,                ///< [in] Binary scan operator
+    Int2Type<LENGTH>    /*length*/)
+{
+    #pragma unroll
+    for (int i = 0; i < LENGTH; ++i)
+    {
+        inclusive = scan_op(exclusive, input[i]);
+        output[i] = exclusive;
+        exclusive = inclusive;
+    }
+
+    return inclusive;
+}
+
+
+
+/**
+ * \brief Perform a sequential exclusive prefix scan over \p LENGTH elements of the \p input array, seeded with the specified \p prefix.  The aggregate is returned.
+ *
+ * \tparam LENGTH     LengthT of \p input and \p output arrays
+ * \tparam T          <b>[inferred]</b> The data type to be scanned.
+ * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ScanOp>
+__device__ __forceinline__ T ThreadScanExclusive(
+    T           *input,                 ///< [in] Input array
+    T           *output,                ///< [out] Output array (may be aliased to \p input)
+    ScanOp      scan_op,                ///< [in] Binary scan operator
+    T           prefix,                 ///< [in] Prefix to seed scan with
+    bool        apply_prefix = true)    ///< [in] Whether or not the calling thread should apply its prefix.  If not, the first output element is undefined.  (Handy for preventing thread-0 from applying a prefix.)
+{
+    T inclusive = input[0];
+    if (apply_prefix)
+    {
+        inclusive = scan_op(prefix, inclusive);
+    }
+    output[0] = prefix;
+    T exclusive = inclusive;
+
+    return ThreadScanExclusive(inclusive, exclusive, input + 1, output + 1, scan_op, Int2Type<LENGTH - 1>());
+}
+
+
+/**
+ * \brief Perform a sequential exclusive prefix scan over the statically-sized \p input array, seeded with the specified \p prefix.  The aggregate is returned.
+ *
+ * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input and \p output arrays
+ * \tparam T          <b>[inferred]</b> The data type to be scanned.
+ * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ScanOp>
+__device__ __forceinline__ T ThreadScanExclusive(
+    T           (&input)[LENGTH],       ///< [in] Input array
+    T           (&output)[LENGTH],      ///< [out] Output array (may be aliased to \p input)
+    ScanOp      scan_op,                ///< [in] Binary scan operator
+    T           prefix,                 ///< [in] Prefix to seed scan with
+    bool        apply_prefix = true)    ///< [in] Whether or not the calling thread should apply its prefix.  (Handy for preventing thread-0 from applying a prefix.)
+{
+    return ThreadScanExclusive<LENGTH>((T*) input, (T*) output, scan_op, prefix, apply_prefix);
+}
+
+
+
+
+
+
+
+
+
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ScanOp>
+__device__ __forceinline__ T ThreadScanInclusive(
+    T                   inclusive,
+    T                   *input,                 ///< [in] Input array
+    T                   *output,                ///< [out] Output array (may be aliased to \p input)
+    ScanOp              scan_op,                ///< [in] Binary scan operator
+    Int2Type<LENGTH>    /*length*/)
+{
+    #pragma unroll
+    for (int i = 0; i < LENGTH; ++i)
+    {
+        inclusive = scan_op(inclusive, input[i]);
+        output[i] = inclusive;
+    }
+
+    return inclusive;
+}
+
+
+/**
+ * \brief Perform a sequential inclusive prefix scan over \p LENGTH elements of the \p input array.  The aggregate is returned.
+ *
+ * \tparam LENGTH     LengthT of \p input and \p output arrays
+ * \tparam T          <b>[inferred]</b> The data type to be scanned.
+ * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ScanOp>
+__device__ __forceinline__ T ThreadScanInclusive(
+    T           *input,                 ///< [in] Input array
+    T           *output,                ///< [out] Output array (may be aliased to \p input)
+    ScanOp      scan_op)                ///< [in] Binary scan operator
+{
+    T inclusive = input[0];
+    output[0] = inclusive;
+
+    // Continue scan
+    return ThreadScanInclusive(inclusive, input + 1, output + 1, scan_op, Int2Type<LENGTH - 1>());
+}
+
+
+/**
+ * \brief Perform a sequential inclusive prefix scan over the statically-sized \p input array.  The aggregate is returned.
+ *
+ * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input and \p output arrays
+ * \tparam T          <b>[inferred]</b> The data type to be scanned.
+ * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ScanOp>
+__device__ __forceinline__ T ThreadScanInclusive(
+    T           (&input)[LENGTH],       ///< [in] Input array
+    T           (&output)[LENGTH],      ///< [out] Output array (may be aliased to \p input)
+    ScanOp      scan_op)                ///< [in] Binary scan operator
+{
+    return ThreadScanInclusive<LENGTH>((T*) input, (T*) output, scan_op);
+}
+
+
+/**
+ * \brief Perform a sequential inclusive prefix scan over \p LENGTH elements of the \p input array, seeded with the specified \p prefix.  The aggregate is returned.
+ *
+ * \tparam LENGTH     LengthT of \p input and \p output arrays
+ * \tparam T          <b>[inferred]</b> The data type to be scanned.
+ * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ScanOp>
+__device__ __forceinline__ T ThreadScanInclusive(
+    T           *input,                 ///< [in] Input array
+    T           *output,                ///< [out] Output array (may be aliased to \p input)
+    ScanOp      scan_op,                ///< [in] Binary scan operator
+    T           prefix,                 ///< [in] Prefix to seed scan with
+    bool        apply_prefix = true)    ///< [in] Whether or not the calling thread should apply its prefix.  (Handy for preventing thread-0 from applying a prefix.)
+{
+    T inclusive = input[0];
+    if (apply_prefix)
+    {
+        inclusive = scan_op(prefix, inclusive);
+    }
+    output[0] = inclusive;
+
+    // Continue scan
+    return ThreadScanInclusive(inclusive, input + 1, output + 1, scan_op, Int2Type<LENGTH - 1>());
+}
+
+
+/**
+ * \brief Perform a sequential inclusive prefix scan over the statically-sized \p input array, seeded with the specified \p prefix.  The aggregate is returned.
+ *
+ * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input and \p output arrays
+ * \tparam T          <b>[inferred]</b> The data type to be scanned.
+ * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+ */
+template <
+    int         LENGTH,
+    typename    T,
+    typename    ScanOp>
+__device__ __forceinline__ T ThreadScanInclusive(
+    T           (&input)[LENGTH],       ///< [in] Input array
+    T           (&output)[LENGTH],      ///< [out] Output array (may be aliased to \p input)
+    ScanOp      scan_op,                ///< [in] Binary scan operator
+    T           prefix,                 ///< [in] Prefix to seed scan with
+    bool        apply_prefix = true)    ///< [in] Whether or not the calling thread should apply its prefix.  (Handy for preventing thread-0 from applying a prefix.)
+{
+    return ThreadScanInclusive<LENGTH>((T*) input, (T*) output, scan_op, prefix, apply_prefix);
+}
+
+
+//@}  end member group
+
+/** @} */       // end group UtilModule
+
+
+}               // internal namespace
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/thread/thread_search.cuh b/third_party/cub/cub/thread/thread_search.cuh
new file mode 100644
index 0000000..3099080
--- /dev/null
+++ b/third_party/cub/cub/thread/thread_search.cuh
@@ -0,0 +1,154 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Thread utilities for sequential search
+ */
+
+#pragma once
+
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * Computes the begin offsets into A and B for the specific diagonal
+ */
+template <
+    typename AIteratorT,
+    typename BIteratorT,
+    typename OffsetT,
+    typename CoordinateT>
+__host__ __device__ __forceinline__ void MergePathSearch(
+    OffsetT         diagonal,
+    AIteratorT      a,
+    BIteratorT      b,
+    OffsetT         a_len,
+    OffsetT         b_len,
+    CoordinateT&    path_coordinate)
+{
+    /// The value type of the input iterator
+    typedef typename std::iterator_traits<AIteratorT>::value_type T;
+
+    OffsetT split_min = CUB_MAX(diagonal - b_len, 0);
+    OffsetT split_max = CUB_MIN(diagonal, a_len);
+
+    while (split_min < split_max)
+    {
+        OffsetT split_pivot = (split_min + split_max) >> 1;
+        if (a[split_pivot] <= b[diagonal - split_pivot - 1])
+        {
+            // Move candidate split range up A, down B
+            split_min = split_pivot + 1;
+        }
+        else
+        {
+            // Move candidate split range up B, down A
+            split_max = split_pivot;
+        }
+    }
+
+    path_coordinate.x = CUB_MIN(split_min, a_len);
+    path_coordinate.y = diagonal - split_min;
+}
+
+
+
+/**
+ * \brief Returns the offset of the first value within \p input which does not compare less than \p val
+ */
+template <
+    typename InputIteratorT,
+    typename OffsetT,
+    typename T>
+__device__ __forceinline__ OffsetT LowerBound(
+    InputIteratorT      input,              ///< [in] Input sequence
+    OffsetT             num_items,          ///< [in] Input sequence length
+    T                   val)                ///< [in] Search key
+{
+    OffsetT retval = 0;
+    while (num_items > 0)
+    {
+        OffsetT half = num_items >> 1;
+        if (input[retval + half] < val)
+        {
+            retval = retval + (half + 1);
+            num_items = num_items - (half + 1);
+        }
+        else
+        {
+            num_items = half;
+        }
+    }
+
+    return retval;
+}
+
+
+/**
+ * \brief Returns the offset of the first value within \p input which compares greater than \p val
+ */
+template <
+    typename InputIteratorT,
+    typename OffsetT,
+    typename T>
+__device__ __forceinline__ OffsetT UpperBound(
+    InputIteratorT      input,              ///< [in] Input sequence
+    OffsetT             num_items,          ///< [in] Input sequence length
+    T                   val)                ///< [in] Search key
+{
+    OffsetT retval = 0;
+    while (num_items > 0)
+    {
+        OffsetT half = num_items >> 1;
+        if (val < input[retval + half])
+        {
+            num_items = half;
+        }
+        else
+        {
+            retval = retval + (half + 1);
+            num_items = num_items - (half + 1);
+        }
+    }
+
+    return retval;
+}
+
+
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/thread/thread_store.cuh b/third_party/cub/cub/thread/thread_store.cuh
new file mode 100644
index 0000000..ec20b36
--- /dev/null
+++ b/third_party/cub/cub/thread/thread_store.cuh
@@ -0,0 +1,422 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Thread utilities for writing memory using PTX cache modifiers.
+ */
+
+#pragma once
+
+#include <cuda.h>
+
+#include "../util_ptx.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup UtilIo
+ * @{
+ */
+
+
+//-----------------------------------------------------------------------------
+// Tags and constants
+//-----------------------------------------------------------------------------
+
+/**
+ * \brief Enumeration of cache modifiers for memory store operations.
+ */
+enum CacheStoreModifier
+{
+    STORE_DEFAULT,              ///< Default (no modifier)
+    STORE_WB,                   ///< Cache write-back all coherent levels
+    STORE_CG,                   ///< Cache at global level
+    STORE_CS,                   ///< Cache streaming (likely to be accessed once)
+    STORE_WT,                   ///< Cache write-through (to system memory)
+    STORE_VOLATILE,             ///< Volatile shared (any memory space)
+};
+
+
+/**
+ * \name Thread I/O (cache modified)
+ * @{
+ */
+
+/**
+ * \brief Thread utility for writing memory using cub::CacheStoreModifier cache modifiers.  Can be used to store any data type.
+ *
+ * \par Example
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/thread/thread_store.cuh>
+ *
+ * // 32-bit store using cache-global modifier:
+ * int *d_out;
+ * int val;
+ * cub::ThreadStore<cub::STORE_CG>(d_out + threadIdx.x, val);
+ *
+ * // 16-bit store using default modifier
+ * short *d_out;
+ * short val;
+ * cub::ThreadStore<cub::STORE_DEFAULT>(d_out + threadIdx.x, val);
+ *
+ * // 256-bit store using write-through modifier
+ * double4 *d_out;
+ * double4 val;
+ * cub::ThreadStore<cub::STORE_WT>(d_out + threadIdx.x, val);
+ *
+ * // 96-bit store using cache-streaming cache modifier
+ * struct TestFoo { bool a; short b; };
+ * TestFoo *d_struct;
+ * TestFoo val;
+ * cub::ThreadStore<cub::STORE_CS>(d_out + threadIdx.x, val);
+ * \endcode
+ *
+ * \tparam MODIFIER             <b>[inferred]</b> CacheStoreModifier enumeration
+ * \tparam InputIteratorT       <b>[inferred]</b> Output iterator type \iterator
+ * \tparam T                    <b>[inferred]</b> Data type of output value
+ */
+template <
+    CacheStoreModifier  MODIFIER,
+    typename            OutputIteratorT,
+    typename            T>
+__device__ __forceinline__ void ThreadStore(OutputIteratorT itr, T val);
+
+
+//@}  end member group
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+
+/// Helper structure for templated store iteration (inductive case)
+template <int COUNT, int MAX>
+struct IterateThreadStore
+{
+    template <CacheStoreModifier MODIFIER, typename T>
+    static __device__ __forceinline__ void Store(T *ptr, T *vals)
+    {
+        ThreadStore<MODIFIER>(ptr + COUNT, vals[COUNT]);
+        IterateThreadStore<COUNT + 1, MAX>::template Store<MODIFIER>(ptr, vals);
+    }
+
+    template <typename OutputIteratorT, typename T>
+    static __device__ __forceinline__ void Dereference(OutputIteratorT ptr, T *vals)
+    {
+        ptr[COUNT] = vals[COUNT];
+        IterateThreadStore<COUNT + 1, MAX>::Dereference(ptr, vals);
+    }
+
+};
+
+/// Helper structure for templated store iteration (termination case)
+template <int MAX>
+struct IterateThreadStore<MAX, MAX>
+{
+    template <CacheStoreModifier MODIFIER, typename T>
+    static __device__ __forceinline__ void Store(T * /*ptr*/, T * /*vals*/) {}
+
+    template <typename OutputIteratorT, typename T>
+    static __device__ __forceinline__ void Dereference(OutputIteratorT /*ptr*/, T * /*vals*/) {}
+};
+
+
+/**
+ * Define a uint4 (16B) ThreadStore specialization for the given Cache load modifier
+ */
+#define _CUB_STORE_16(cub_modifier, ptx_modifier)                                            \
+    template<>                                                                              \
+    __device__ __forceinline__ void ThreadStore<cub_modifier, uint4*, uint4>(uint4* ptr, uint4 val)                         \
+    {                                                                                       \
+        asm volatile ("st."#ptx_modifier".v4.u32 [%0], {%1, %2, %3, %4};" : :               \
+            _CUB_ASM_PTR_(ptr),                                                             \
+            "r"(val.x),                                                                     \
+            "r"(val.y),                                                                     \
+            "r"(val.z),                                                                     \
+            "r"(val.w));                                                                    \
+    }                                                                                       \
+    template<>                                                                              \
+    __device__ __forceinline__ void ThreadStore<cub_modifier, ulonglong2*, ulonglong2>(ulonglong2* ptr, ulonglong2 val)     \
+    {                                                                                       \
+        asm volatile ("st."#ptx_modifier".v2.u64 [%0], {%1, %2};" : :                       \
+            _CUB_ASM_PTR_(ptr),                                                             \
+            "l"(val.x),                                                                     \
+            "l"(val.y));                                                                    \
+    }
+
+
+/**
+ * Define a uint2 (8B) ThreadStore specialization for the given Cache load modifier
+ */
+#define _CUB_STORE_8(cub_modifier, ptx_modifier)                                             \
+    template<>                                                                              \
+    __device__ __forceinline__ void ThreadStore<cub_modifier, ushort4*, ushort4>(ushort4* ptr, ushort4 val)                 \
+    {                                                                                       \
+        asm volatile ("st."#ptx_modifier".v4.u16 [%0], {%1, %2, %3, %4};" : :               \
+            _CUB_ASM_PTR_(ptr),                                                             \
+            "h"(val.x),                                                                     \
+            "h"(val.y),                                                                     \
+            "h"(val.z),                                                                     \
+            "h"(val.w));                                                                    \
+    }                                                                                       \
+    template<>                                                                              \
+    __device__ __forceinline__ void ThreadStore<cub_modifier, uint2*, uint2>(uint2* ptr, uint2 val)                         \
+    {                                                                                       \
+        asm volatile ("st."#ptx_modifier".v2.u32 [%0], {%1, %2};" : :                       \
+            _CUB_ASM_PTR_(ptr),                                                             \
+            "r"(val.x),                                                                     \
+            "r"(val.y));                                                                    \
+    }                                                                                       \
+    template<>                                                                              \
+    __device__ __forceinline__ void ThreadStore<cub_modifier, unsigned long long*, unsigned long long>(unsigned long long* ptr, unsigned long long val)     \
+    {                                                                                       \
+        asm volatile ("st."#ptx_modifier".u64 [%0], %1;" : :                                \
+            _CUB_ASM_PTR_(ptr),                                                             \
+            "l"(val));                                                                      \
+    }
+
+/**
+ * Define a unsigned int (4B) ThreadStore specialization for the given Cache load modifier
+ */
+#define _CUB_STORE_4(cub_modifier, ptx_modifier)                                             \
+    template<>                                                                              \
+    __device__ __forceinline__ void ThreadStore<cub_modifier, unsigned int*, unsigned int>(unsigned int* ptr, unsigned int val)                             \
+    {                                                                                       \
+        asm volatile ("st."#ptx_modifier".u32 [%0], %1;" : :                                \
+            _CUB_ASM_PTR_(ptr),                                                             \
+            "r"(val));                                                                      \
+    }
+
+
+/**
+ * Define a unsigned short (2B) ThreadStore specialization for the given Cache load modifier
+ */
+#define _CUB_STORE_2(cub_modifier, ptx_modifier)                                             \
+    template<>                                                                              \
+    __device__ __forceinline__ void ThreadStore<cub_modifier, unsigned short*, unsigned short>(unsigned short* ptr, unsigned short val)                     \
+    {                                                                                       \
+        asm volatile ("st."#ptx_modifier".u16 [%0], %1;" : :                                \
+            _CUB_ASM_PTR_(ptr),                                                             \
+            "h"(val));                                                                      \
+    }
+
+
+/**
+ * Define a unsigned char (1B) ThreadStore specialization for the given Cache load modifier
+ */
+#define _CUB_STORE_1(cub_modifier, ptx_modifier)                                             \
+    template<>                                                                              \
+    __device__ __forceinline__ void ThreadStore<cub_modifier, unsigned char*, unsigned char>(unsigned char* ptr, unsigned char val)                         \
+    {                                                                                       \
+        asm volatile (                                                                      \
+        "{"                                                                                 \
+        "   .reg .u8 datum;"                                                                \
+        "   cvt.u8.u16 datum, %1;"                                                          \
+        "   st."#ptx_modifier".u8 [%0], datum;"                                             \
+        "}" : :                                                                             \
+            _CUB_ASM_PTR_(ptr),                                                             \
+            "h"((unsigned short) val));                                                               \
+    }
+
+/**
+ * Define powers-of-two ThreadStore specializations for the given Cache load modifier
+ */
+#define _CUB_STORE_ALL(cub_modifier, ptx_modifier)                                           \
+    _CUB_STORE_16(cub_modifier, ptx_modifier)                                                \
+    _CUB_STORE_8(cub_modifier, ptx_modifier)                                                 \
+    _CUB_STORE_4(cub_modifier, ptx_modifier)                                                 \
+    _CUB_STORE_2(cub_modifier, ptx_modifier)                                                 \
+    _CUB_STORE_1(cub_modifier, ptx_modifier)                                                 \
+
+
+/**
+ * Define ThreadStore specializations for the various Cache load modifiers
+ */
+#if CUB_PTX_ARCH >= 200
+    _CUB_STORE_ALL(STORE_WB, wb)
+    _CUB_STORE_ALL(STORE_CG, cg)
+    _CUB_STORE_ALL(STORE_CS, cs)
+    _CUB_STORE_ALL(STORE_WT, wt)
+#else
+    _CUB_STORE_ALL(STORE_WB, global)
+    _CUB_STORE_ALL(STORE_CG, global)
+    _CUB_STORE_ALL(STORE_CS, global)
+    _CUB_STORE_ALL(STORE_WT, volatile.global)
+#endif
+
+
+// Macro cleanup
+#undef _CUB_STORE_ALL
+#undef _CUB_STORE_1
+#undef _CUB_STORE_2
+#undef _CUB_STORE_4
+#undef _CUB_STORE_8
+#undef _CUB_STORE_16
+
+
+/**
+ * ThreadStore definition for STORE_DEFAULT modifier on iterator types
+ */
+template <typename OutputIteratorT, typename T>
+__device__ __forceinline__ void ThreadStore(
+    OutputIteratorT             itr,
+    T                           val,
+    Int2Type<STORE_DEFAULT>     /*modifier*/,
+    Int2Type<false>             /*is_pointer*/)
+{
+    *itr = val;
+}
+
+
+/**
+ * ThreadStore definition for STORE_DEFAULT modifier on pointer types
+ */
+template <typename T>
+__device__ __forceinline__ void ThreadStore(
+    T                           *ptr,
+    T                           val,
+    Int2Type<STORE_DEFAULT>     /*modifier*/,
+    Int2Type<true>              /*is_pointer*/)
+{
+    *ptr = val;
+}
+
+
+/**
+ * ThreadStore definition for STORE_VOLATILE modifier on primitive pointer types
+ */
+template <typename T>
+__device__ __forceinline__ void ThreadStoreVolatilePtr(
+    T                           *ptr,
+    T                           val,
+    Int2Type<true>              /*is_primitive*/)
+{
+    *reinterpret_cast<volatile T*>(ptr) = val;
+}
+
+
+/**
+ * ThreadStore definition for STORE_VOLATILE modifier on non-primitive pointer types
+ */
+template <typename T>
+__device__ __forceinline__ void ThreadStoreVolatilePtr(
+    T                           *ptr,
+    T                           val,
+    Int2Type<false>             /*is_primitive*/)
+{
+    // Create a temporary using shuffle-words, then store using volatile-words
+    typedef typename UnitWord<T>::VolatileWord  VolatileWord;  
+    typedef typename UnitWord<T>::ShuffleWord   ShuffleWord;
+
+    const int VOLATILE_MULTIPLE = sizeof(T) / sizeof(VolatileWord);
+    const int SHUFFLE_MULTIPLE  = sizeof(T) / sizeof(ShuffleWord);
+    
+    VolatileWord words[VOLATILE_MULTIPLE];
+
+    #pragma unroll
+    for (int i = 0; i < SHUFFLE_MULTIPLE; ++i)
+        reinterpret_cast<ShuffleWord*>(words)[i] = reinterpret_cast<ShuffleWord*>(&val)[i];
+
+    IterateThreadStore<0, VOLATILE_MULTIPLE>::template Dereference(
+        reinterpret_cast<volatile VolatileWord*>(ptr),
+        words);
+}
+
+
+/**
+ * ThreadStore definition for STORE_VOLATILE modifier on pointer types
+ */
+template <typename T>
+__device__ __forceinline__ void ThreadStore(
+    T                           *ptr,
+    T                           val,
+    Int2Type<STORE_VOLATILE>    /*modifier*/,
+    Int2Type<true>              /*is_pointer*/)
+{
+    ThreadStoreVolatilePtr(ptr, val, Int2Type<Traits<T>::PRIMITIVE>());
+}
+
+
+/**
+ * ThreadStore definition for generic modifiers on pointer types
+ */
+template <typename T, int MODIFIER>
+__device__ __forceinline__ void ThreadStore(
+    T                           *ptr,
+    T                           val,
+    Int2Type<MODIFIER>          /*modifier*/,
+    Int2Type<true>              /*is_pointer*/)
+{
+    // Create a temporary using shuffle-words, then store using device-words
+    typedef typename UnitWord<T>::DeviceWord    DeviceWord;  
+    typedef typename UnitWord<T>::ShuffleWord   ShuffleWord;
+
+    const int DEVICE_MULTIPLE   = sizeof(T) / sizeof(DeviceWord);
+    const int SHUFFLE_MULTIPLE  = sizeof(T) / sizeof(ShuffleWord);
+    
+    DeviceWord words[DEVICE_MULTIPLE];
+
+    #pragma unroll
+    for (int i = 0; i < SHUFFLE_MULTIPLE; ++i)
+        reinterpret_cast<ShuffleWord*>(words)[i] = reinterpret_cast<ShuffleWord*>(&val)[i];
+
+    IterateThreadStore<0, DEVICE_MULTIPLE>::template Store<CacheStoreModifier(MODIFIER)>(
+        reinterpret_cast<DeviceWord*>(ptr),
+        words);
+}
+
+
+/**
+ * ThreadStore definition for generic modifiers
+ */
+template <CacheStoreModifier MODIFIER, typename OutputIteratorT, typename T>
+__device__ __forceinline__ void ThreadStore(OutputIteratorT itr, T val)
+{
+    ThreadStore(
+        itr,
+        val,
+        Int2Type<MODIFIER>(),
+        Int2Type<IsPointer<OutputIteratorT>::VALUE>());
+}
+
+
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/** @} */       // end group UtilIo
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/util_allocator.cuh b/third_party/cub/cub/util_allocator.cuh
new file mode 100644
index 0000000..0e6dd04
--- /dev/null
+++ b/third_party/cub/cub/util_allocator.cuh
@@ -0,0 +1,708 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple caching allocator for device memory allocations. The allocator is
+ * thread-safe and capable of managing device allocations on multiple devices.
+ ******************************************************************************/
+
+#pragma once
+
+#include "util_namespace.cuh"
+#include "util_debug.cuh"
+
+#include <set>
+#include <map>
+
+#include "host/mutex.cuh"
+#include <math.h>
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilMgmt
+ * @{
+ */
+
+
+/******************************************************************************
+ * CachingDeviceAllocator (host use)
+ ******************************************************************************/
+
+/**
+ * \brief A simple caching allocator for device memory allocations.
+ *
+ * \par Overview
+ * The allocator is thread-safe and stream-safe and is capable of managing cached
+ * device allocations on multiple devices.  It behaves as follows:
+ *
+ * \par
+ * - Allocations from the allocator are associated with an \p active_stream.  Once freed,
+ *   the allocation becomes available immediately for reuse within the \p active_stream
+ *   with which it was associated with during allocation, and it becomes available for
+ *   reuse within other streams when all prior work submitted to \p active_stream has completed.
+ * - Allocations are categorized and cached by bin size.  A new allocation request of
+ *   a given size will only consider cached allocations within the corresponding bin.
+ * - Bin limits progress geometrically in accordance with the growth factor
+ *   \p bin_growth provided during construction.  Unused device allocations within
+ *   a larger bin cache are not reused for allocation requests that categorize to
+ *   smaller bin sizes.
+ * - Allocation requests below (\p bin_growth ^ \p min_bin) are rounded up to
+ *   (\p bin_growth ^ \p min_bin).
+ * - Allocations above (\p bin_growth ^ \p max_bin) are not rounded up to the nearest
+ *   bin and are simply freed when they are deallocated instead of being returned
+ *   to a bin-cache.
+ * - %If the total storage of cached allocations on a given device will exceed
+ *   \p max_cached_bytes, allocations for that device are simply freed when they are
+ *   deallocated instead of being returned to their bin-cache.
+ *
+ * \par
+ * For example, the default-constructed CachingDeviceAllocator is configured with:
+ * - \p bin_growth          = 8
+ * - \p min_bin             = 3
+ * - \p max_bin             = 7
+ * - \p max_cached_bytes    = 6MB - 1B
+ *
+ * \par
+ * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB
+ * and sets a maximum of 6,291,455 cached bytes per device
+ *
+ */
+struct CachingDeviceAllocator
+{
+
+    //---------------------------------------------------------------------
+    // Constants
+    //---------------------------------------------------------------------
+
+    /// Out-of-bounds bin
+    static const unsigned int INVALID_BIN = (unsigned int) -1;
+
+    /// Invalid size
+    static const size_t INVALID_SIZE = (size_t) -1;
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+    /// Invalid device ordinal
+    static const int INVALID_DEVICE_ORDINAL = -1;
+
+    //---------------------------------------------------------------------
+    // Type definitions and helper types
+    //---------------------------------------------------------------------
+
+    /**
+     * Descriptor for device memory allocations
+     */
+    struct BlockDescriptor
+    {
+        void*           d_ptr;              // Device pointer
+        size_t          bytes;              // Size of allocation in bytes
+        unsigned int    bin;                // Bin enumeration
+        int             device;             // device ordinal
+        cudaStream_t    associated_stream;  // Associated associated_stream
+        cudaEvent_t     ready_event;        // Signal when associated stream has run to the point at which this block was freed
+
+        // Constructor (suitable for searching maps for a specific block, given its pointer and device)
+        BlockDescriptor(void *d_ptr, int device) :
+            d_ptr(d_ptr),
+            bytes(0),
+            bin(INVALID_BIN),
+            device(device),
+            associated_stream(0),
+            ready_event(0)
+        {}
+
+        // Constructor (suitable for searching maps for a range of suitable blocks, given a device)
+        BlockDescriptor(int device) :
+            d_ptr(NULL),
+            bytes(0),
+            bin(INVALID_BIN),
+            device(device),
+            associated_stream(0),
+            ready_event(0)
+        {}
+
+        // Comparison functor for comparing device pointers
+        static bool PtrCompare(const BlockDescriptor &a, const BlockDescriptor &b)
+        {
+            if (a.device == b.device)
+                return (a.d_ptr < b.d_ptr);
+            else
+                return (a.device < b.device);
+        }
+
+        // Comparison functor for comparing allocation sizes
+        static bool SizeCompare(const BlockDescriptor &a, const BlockDescriptor &b)
+        {
+            if (a.device == b.device)
+                return (a.bytes < b.bytes);
+            else
+                return (a.device < b.device);
+        }
+    };
+
+    /// BlockDescriptor comparator function interface
+    typedef bool (*Compare)(const BlockDescriptor &, const BlockDescriptor &);
+
+    class TotalBytes {
+    public:
+        size_t free;
+        size_t live;
+        TotalBytes() { free = live = 0; }
+    };
+
+    /// Set type for cached blocks (ordered by size)
+    typedef std::multiset<BlockDescriptor, Compare> CachedBlocks;
+
+    /// Set type for live blocks (ordered by ptr)
+    typedef std::multiset<BlockDescriptor, Compare> BusyBlocks;
+
+    /// Map type of device ordinals to the number of cached bytes cached by each device
+    typedef std::map<int, TotalBytes> GpuCachedBytes;
+
+
+    //---------------------------------------------------------------------
+    // Utility functions
+    //---------------------------------------------------------------------
+
+    /**
+     * Integer pow function for unsigned base and exponent
+     */
+    static unsigned int IntPow(
+        unsigned int base,
+        unsigned int exp)
+    {
+        unsigned int retval = 1;
+        while (exp > 0)
+        {
+            if (exp & 1) {
+                retval = retval * base;        // multiply the result by the current base
+            }
+            base = base * base;                // square the base
+            exp = exp >> 1;                    // divide the exponent in half
+        }
+        return retval;
+    }
+
+
+    /**
+     * Round up to the nearest power-of
+     */
+    void NearestPowerOf(
+        unsigned int    &power,
+        size_t          &rounded_bytes,
+        unsigned int    base,
+        size_t          value)
+    {
+        power = 0;
+        rounded_bytes = 1;
+
+        if (value * base < value)
+        {
+            // Overflow
+            power = sizeof(size_t) * 8;
+            rounded_bytes = size_t(0) - 1;
+            return;
+        }
+
+        while (rounded_bytes < value)
+        {
+            rounded_bytes *= base;
+            power++;
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Fields
+    //---------------------------------------------------------------------
+
+    cub::Mutex      mutex;              /// Mutex for thread-safety
+
+    unsigned int    bin_growth;         /// Geometric growth factor for bin-sizes
+    unsigned int    min_bin;            /// Minimum bin enumeration
+    unsigned int    max_bin;            /// Maximum bin enumeration
+
+    size_t          min_bin_bytes;      /// Minimum bin size
+    size_t          max_bin_bytes;      /// Maximum bin size
+    size_t          max_cached_bytes;   /// Maximum aggregate cached bytes per device
+
+    const bool      skip_cleanup;       /// Whether or not to skip a call to FreeAllCached() when destructor is called.  (The CUDA runtime may have already shut down for statically declared allocators)
+    bool            debug;              /// Whether or not to print (de)allocation events to stdout
+
+    GpuCachedBytes  cached_bytes;       /// Map of device ordinal to aggregate cached bytes on that device
+    CachedBlocks    cached_blocks;      /// Set of cached device allocations available for reuse
+    BusyBlocks      live_blocks;        /// Set of live device allocations currently in use
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+    //---------------------------------------------------------------------
+    // Methods
+    //---------------------------------------------------------------------
+
+    /**
+     * \brief Constructor.
+     */
+    CachingDeviceAllocator(
+        unsigned int    bin_growth,                             ///< Geometric growth factor for bin-sizes
+        unsigned int    min_bin             = 1,                ///< Minimum bin (default is bin_growth ^ 1)
+        unsigned int    max_bin             = INVALID_BIN,      ///< Maximum bin (default is no max bin)
+        size_t          max_cached_bytes    = INVALID_SIZE,     ///< Maximum aggregate cached bytes per device (default is no limit)
+        bool            skip_cleanup        = false,            ///< Whether or not to skip a call to \p FreeAllCached() when the destructor is called (default is to deallocate)
+        bool            debug               = false)            ///< Whether or not to print (de)allocation events to stdout (default is no stderr output)
+    :
+        bin_growth(bin_growth),
+        min_bin(min_bin),
+        max_bin(max_bin),
+        min_bin_bytes(IntPow(bin_growth, min_bin)),
+        max_bin_bytes(IntPow(bin_growth, max_bin)),
+        max_cached_bytes(max_cached_bytes),
+        skip_cleanup(skip_cleanup),
+        debug(debug),
+        cached_blocks(BlockDescriptor::SizeCompare),
+        live_blocks(BlockDescriptor::PtrCompare)
+    {}
+
+
+    /**
+     * \brief Default constructor.
+     *
+     * Configured with:
+     * \par
+     * - \p bin_growth          = 8
+     * - \p min_bin             = 3
+     * - \p max_bin             = 7
+     * - \p max_cached_bytes    = (\p bin_growth ^ \p max_bin) * 3) - 1 = 6,291,455 bytes
+     *
+     * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB and
+     * sets a maximum of 6,291,455 cached bytes per device
+     */
+    CachingDeviceAllocator(
+        bool skip_cleanup = false,
+        bool debug = false)
+    :
+        bin_growth(8),
+        min_bin(3),
+        max_bin(7),
+        min_bin_bytes(IntPow(bin_growth, min_bin)),
+        max_bin_bytes(IntPow(bin_growth, max_bin)),
+        max_cached_bytes((max_bin_bytes * 3) - 1),
+        skip_cleanup(skip_cleanup),
+        debug(debug),
+        cached_blocks(BlockDescriptor::SizeCompare),
+        live_blocks(BlockDescriptor::PtrCompare)
+    {}
+
+
+    /**
+     * \brief Sets the limit on the number bytes this allocator is allowed to cache per device.
+     *
+     * Changing the ceiling of cached bytes does not cause any allocations (in-use or
+     * cached-in-reserve) to be freed.  See \p FreeAllCached().
+     */
+    cudaError_t SetMaxCachedBytes(
+        size_t max_cached_bytes)
+    {
+        // Lock
+        mutex.Lock();
+
+        if (debug) _CubLog("Changing max_cached_bytes (%lld -> %lld)\n", (long long) this->max_cached_bytes, (long long) max_cached_bytes);
+
+        this->max_cached_bytes = max_cached_bytes;
+
+        // Unlock
+        mutex.Unlock();
+
+        return cudaSuccess;
+    }
+
+
+    /**
+     * \brief Provides a suitable allocation of device memory for the given size on the specified device.
+     *
+     * Once freed, the allocation becomes available immediately for reuse within the \p active_stream
+     * with which it was associated with during allocation, and it becomes available for reuse within other
+     * streams when all prior work submitted to \p active_stream has completed.
+     */
+    cudaError_t DeviceAllocate(
+        int             device,             ///< [in] Device on which to place the allocation
+        void            **d_ptr,            ///< [out] Reference to pointer to the allocation
+        size_t          bytes,              ///< [in] Minimum number of bytes for the allocation
+        cudaStream_t    active_stream = 0)  ///< [in] The stream to be associated with this allocation
+    {
+        *d_ptr                          = NULL;
+        int entrypoint_device           = INVALID_DEVICE_ORDINAL;
+        cudaError_t error               = cudaSuccess;
+
+        if (device == INVALID_DEVICE_ORDINAL)
+        {
+            if (CubDebug(error = cudaGetDevice(&entrypoint_device))) return error;
+            device = entrypoint_device;
+        }
+
+        // Create a block descriptor for the requested allocation
+        bool found = false;
+        BlockDescriptor search_key(device);
+        search_key.associated_stream = active_stream;
+        NearestPowerOf(search_key.bin, search_key.bytes, bin_growth, bytes);
+
+        if (search_key.bin > max_bin)
+        {
+            // Bin is greater than our maximum bin: allocate the request
+            // exactly and give out-of-bounds bin.  It will not be cached
+            // for reuse when returned.
+            search_key.bin      = INVALID_BIN;
+            search_key.bytes    = bytes;
+        }
+        else
+        {
+            // Search for a suitable cached allocation: lock
+            mutex.Lock();
+
+            if (search_key.bin < min_bin)
+            {
+                // Bin is less than minimum bin: round up
+                search_key.bin      = min_bin;
+                search_key.bytes    = min_bin_bytes;
+            }
+
+            // Iterate through the range of cached blocks on the same device in the same bin
+            CachedBlocks::iterator block_itr = cached_blocks.lower_bound(search_key);
+            while ((block_itr != cached_blocks.end())
+                    && (block_itr->device == device)
+                    && (block_itr->bin == search_key.bin))
+            {
+                // To prevent races with reusing blocks returned by the host but still
+                // in use by the device, only consider cached blocks that are
+                // either (from the active stream) or (from an idle stream)
+                if ((active_stream == block_itr->associated_stream) ||
+                    (cudaEventQuery(block_itr->ready_event) != cudaErrorNotReady))
+                {
+                    // Reuse existing cache block.  Insert into live blocks.
+                    found = true;
+                    search_key = *block_itr;
+                    search_key.associated_stream = active_stream;
+                    live_blocks.insert(search_key);
+
+                    // Remove from free blocks
+                    cached_bytes[device].free -= search_key.bytes;
+                    cached_bytes[device].live += search_key.bytes;
+
+                    if (debug) _CubLog("\tDevice %d reused cached block at %p (%lld bytes) for stream %lld (previously associated with stream %lld).\n",
+                        device, search_key.d_ptr, (long long) search_key.bytes, (long long) search_key.associated_stream, (long long)  block_itr->associated_stream);
+
+                    cached_blocks.erase(block_itr);
+
+                    break;
+                }
+                block_itr++;
+            }
+
+            // Done searching: unlock
+            mutex.Unlock();
+        }
+
+        // Allocate the block if necessary
+        if (!found)
+        {
+            // Set runtime's current device to specified device (entrypoint may not be set)
+            if (device != entrypoint_device)
+            {
+                if (CubDebug(error = cudaGetDevice(&entrypoint_device))) return error;
+                if (CubDebug(error = cudaSetDevice(device))) return error;
+            }
+
+            // Attempt to allocate
+            if (CubDebug(error = cudaMalloc(&search_key.d_ptr, search_key.bytes)) == cudaErrorMemoryAllocation)
+            {
+                // The allocation attempt failed: free all cached blocks on device and retry
+                if (debug) _CubLog("\tDevice %d failed to allocate %lld bytes for stream %lld, retrying after freeing cached allocations",
+                      device, (long long) search_key.bytes, (long long) search_key.associated_stream);
+
+                error = cudaSuccess;    // Reset the error we will return
+                cudaGetLastError();     // Reset CUDART's error
+
+                // Lock
+                mutex.Lock();
+
+                // Iterate the range of free blocks on the same device
+                BlockDescriptor free_key(device);
+                CachedBlocks::iterator block_itr = cached_blocks.lower_bound(free_key);
+
+                while ((block_itr != cached_blocks.end()) && (block_itr->device == device))
+                {
+                    // No need to worry about synchronization with the device: cudaFree is
+                    // blocking and will synchronize across all kernels executing
+                    // on the current device
+
+                    // Free device memory and destroy stream event.
+                    if (CubDebug(error = cudaFree(block_itr->d_ptr))) break;
+                    if (CubDebug(error = cudaEventDestroy(block_itr->ready_event))) break;
+
+                    // Reduce balance and erase entry
+                    cached_bytes[device].free -= block_itr->bytes;
+
+                    if (debug) _CubLog("\tDevice %d freed %lld bytes.\n\t\t  %lld available blocks cached (%lld bytes), %lld live blocks (%lld bytes) outstanding.\n",
+                        device, (long long) block_itr->bytes, (long long) cached_blocks.size(), (long long) cached_bytes[device].free, (long long) live_blocks.size(), (long long) cached_bytes[device].live);
+
+                    cached_blocks.erase(block_itr);
+
+                    block_itr++;
+                }
+
+                // Unlock
+                mutex.Unlock();
+
+                // Return under error
+                if (error) return error;
+
+                // Try to allocate again
+                if (CubDebug(error = cudaMalloc(&search_key.d_ptr, search_key.bytes))) return error;
+            }
+
+            // Create ready event
+            if (CubDebug(error = cudaEventCreateWithFlags(&search_key.ready_event, cudaEventDisableTiming)))
+                return error;
+
+            // Insert into live blocks
+            mutex.Lock();
+            live_blocks.insert(search_key);
+            cached_bytes[device].live += search_key.bytes;
+            mutex.Unlock();
+
+            if (debug) _CubLog("\tDevice %d allocated new device block at %p (%lld bytes associated with stream %lld).\n",
+                      device, search_key.d_ptr, (long long) search_key.bytes, (long long) search_key.associated_stream);
+
+            // Attempt to revert back to previous device if necessary
+            if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && (entrypoint_device != device))
+            {
+                if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error;
+            }
+        }
+
+        // Copy device pointer to output parameter
+        *d_ptr = search_key.d_ptr;
+
+        if (debug) _CubLog("\t\t%lld available blocks cached (%lld bytes), %lld live blocks outstanding(%lld bytes).\n",
+            (long long) cached_blocks.size(), (long long) cached_bytes[device].free, (long long) live_blocks.size(), (long long) cached_bytes[device].live);
+
+        return error;
+    }
+
+
+    /**
+     * \brief Provides a suitable allocation of device memory for the given size on the current device.
+     *
+     * Once freed, the allocation becomes available immediately for reuse within the \p active_stream
+     * with which it was associated with during allocation, and it becomes available for reuse within other
+     * streams when all prior work submitted to \p active_stream has completed.
+     */
+    cudaError_t DeviceAllocate(
+        void            **d_ptr,            ///< [out] Reference to pointer to the allocation
+        size_t          bytes,              ///< [in] Minimum number of bytes for the allocation
+        cudaStream_t    active_stream = 0)  ///< [in] The stream to be associated with this allocation
+    {
+        return DeviceAllocate(INVALID_DEVICE_ORDINAL, d_ptr, bytes, active_stream);
+    }
+
+
+    /**
+     * \brief Frees a live allocation of device memory on the specified device, returning it to the allocator.
+     *
+     * Once freed, the allocation becomes available immediately for reuse within the \p active_stream
+     * with which it was associated with during allocation, and it becomes available for reuse within other
+     * streams when all prior work submitted to \p active_stream has completed.
+     */
+    cudaError_t DeviceFree(
+        int             device,
+        void*           d_ptr)
+    {
+        int entrypoint_device           = INVALID_DEVICE_ORDINAL;
+        cudaError_t error               = cudaSuccess;
+
+        if (device == INVALID_DEVICE_ORDINAL)
+        {
+            if (CubDebug(error = cudaGetDevice(&entrypoint_device)))
+                return error;
+            device = entrypoint_device;
+        }
+
+        // Lock
+        mutex.Lock();
+
+        // Find corresponding block descriptor
+        bool recached = false;
+        BlockDescriptor search_key(d_ptr, device);
+        BusyBlocks::iterator block_itr = live_blocks.find(search_key);
+        if (block_itr != live_blocks.end())
+        {
+            // Remove from live blocks
+            search_key = *block_itr;
+            live_blocks.erase(block_itr);
+            cached_bytes[device].live -= search_key.bytes;
+
+            // Keep the returned allocation if bin is valid and we won't exceed the max cached threshold
+            if ((search_key.bin != INVALID_BIN) && (cached_bytes[device].free + search_key.bytes <= max_cached_bytes))
+            {
+                // Insert returned allocation into free blocks
+                recached = true;
+                cached_blocks.insert(search_key);
+                cached_bytes[device].free += search_key.bytes;
+
+                if (debug) _CubLog("\tDevice %d returned %lld bytes from associated stream %lld.\n\t\t %lld available blocks cached (%lld bytes), %lld live blocks outstanding. (%lld bytes)\n",
+                    device, (long long) search_key.bytes, (long long) search_key.associated_stream, (long long) cached_blocks.size(),
+                    (long long) cached_bytes[device].free, (long long) live_blocks.size(), (long long) cached_bytes[device].live);
+            }
+        }
+
+        // Unlock
+        mutex.Unlock();
+
+        // First set to specified device (entrypoint may not be set)
+        if (device != entrypoint_device)
+        {
+            if (CubDebug(error = cudaGetDevice(&entrypoint_device))) return error;
+            if (CubDebug(error = cudaSetDevice(device))) return error;
+        }
+
+        if (recached)
+        {
+            // Insert the ready event in the associated stream (must have current device set properly)
+            if (CubDebug(error = cudaEventRecord(search_key.ready_event, search_key.associated_stream))) return error;
+        }
+        else
+        {
+            // Free the allocation from the runtime and cleanup the event.
+            if (CubDebug(error = cudaFree(d_ptr))) return error;
+            if (CubDebug(error = cudaEventDestroy(search_key.ready_event))) return error;
+
+            if (debug) _CubLog("\tDevice %d freed %lld bytes from associated stream %lld.\n\t\t  %lld available blocks cached (%lld bytes), %lld live blocks (%lld bytes) outstanding.\n",
+                device, (long long) search_key.bytes, (long long) search_key.associated_stream, (long long) cached_blocks.size(), (long long) cached_bytes[device].free, (long long) live_blocks.size(), (long long) cached_bytes[device].live);
+        }
+
+        // Reset device
+        if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && (entrypoint_device != device))
+        {
+            if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error;
+        }
+
+        return error;
+    }
+
+
+    /**
+     * \brief Frees a live allocation of device memory on the current device, returning it to the allocator.
+     *
+     * Once freed, the allocation becomes available immediately for reuse within the \p active_stream
+     * with which it was associated with during allocation, and it becomes available for reuse within other
+     * streams when all prior work submitted to \p active_stream has completed.
+     */
+    cudaError_t DeviceFree(
+        void*           d_ptr)
+    {
+        return DeviceFree(INVALID_DEVICE_ORDINAL, d_ptr);
+    }
+
+
+    /**
+     * \brief Frees all cached device allocations on all devices
+     */
+    cudaError_t FreeAllCached()
+    {
+        cudaError_t error         = cudaSuccess;
+        int entrypoint_device     = INVALID_DEVICE_ORDINAL;
+        int current_device        = INVALID_DEVICE_ORDINAL;
+
+        mutex.Lock();
+
+        while (!cached_blocks.empty())
+        {
+            // Get first block
+            CachedBlocks::iterator begin = cached_blocks.begin();
+
+            // Get entry-point device ordinal if necessary
+            if (entrypoint_device == INVALID_DEVICE_ORDINAL)
+            {
+                if (CubDebug(error = cudaGetDevice(&entrypoint_device))) break;
+            }
+
+            // Set current device ordinal if necessary
+            if (begin->device != current_device)
+            {
+                if (CubDebug(error = cudaSetDevice(begin->device))) break;
+                current_device = begin->device;
+            }
+
+            // Free device memory
+            if (CubDebug(error = cudaFree(begin->d_ptr))) break;
+            if (CubDebug(error = cudaEventDestroy(begin->ready_event))) break;
+
+            // Reduce balance and erase entry
+            cached_bytes[current_device].free -= begin->bytes;
+
+            if (debug) _CubLog("\tDevice %d freed %lld bytes.\n\t\t  %lld available blocks cached (%lld bytes), %lld live blocks (%lld bytes) outstanding.\n",
+                current_device, (long long) begin->bytes, (long long) cached_blocks.size(), (long long) cached_bytes[current_device].free, (long long) live_blocks.size(), (long long) cached_bytes[current_device].live);
+
+            cached_blocks.erase(begin);
+        }
+
+        mutex.Unlock();
+
+        // Attempt to revert back to entry-point device if necessary
+        if (entrypoint_device != INVALID_DEVICE_ORDINAL)
+        {
+            if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error;
+        }
+
+        return error;
+    }
+
+
+    /**
+     * \brief Destructor
+     */
+    virtual ~CachingDeviceAllocator()
+    {
+        if (!skip_cleanup)
+            FreeAllCached();
+    }
+
+};
+
+
+
+
+/** @} */       // end group UtilMgmt
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/util_arch.cuh b/third_party/cub/cub/util_arch.cuh
new file mode 100644
index 0000000..28d81e7
--- /dev/null
+++ b/third_party/cub/cub/util_arch.cuh
@@ -0,0 +1,151 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Static architectural properties by SM version.
+ */
+
+#pragma once
+
+#include "util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+#if (__CUDACC_VER_MAJOR__ >= 9) && !defined(CUB_USE_COOPERATIVE_GROUPS)
+    #define CUB_USE_COOPERATIVE_GROUPS
+#endif
+
+/// CUB_PTX_ARCH reflects the PTX version targeted by the active compiler pass (or zero during the host pass).
+#ifndef CUB_PTX_ARCH
+    #ifndef __CUDA_ARCH__
+        #define CUB_PTX_ARCH 0
+    #else
+        #define CUB_PTX_ARCH __CUDA_ARCH__
+    #endif
+#endif
+
+
+/// Whether or not the source targeted by the active compiler pass is allowed to  invoke device kernels or methods from the CUDA runtime API.
+#ifndef CUB_RUNTIME_FUNCTION
+    #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__>= 350 && defined(__CUDACC_RDC__))
+        #define CUB_RUNTIME_ENABLED
+        #define CUB_RUNTIME_FUNCTION __host__ __device__
+    #else
+        #define CUB_RUNTIME_FUNCTION __host__
+    #endif
+#endif
+
+
+/// Number of threads per warp
+#ifndef CUB_LOG_WARP_THREADS
+    #define CUB_LOG_WARP_THREADS(arch)                      \
+        (5)
+    #define CUB_WARP_THREADS(arch)                          \
+        (1 << CUB_LOG_WARP_THREADS(arch))
+
+    #define CUB_PTX_WARP_THREADS        CUB_WARP_THREADS(CUB_PTX_ARCH)
+    #define CUB_PTX_LOG_WARP_THREADS    CUB_LOG_WARP_THREADS(CUB_PTX_ARCH)
+#endif
+
+
+/// Number of smem banks
+#ifndef CUB_LOG_SMEM_BANKS
+    #define CUB_LOG_SMEM_BANKS(arch)                        \
+        ((arch >= 200) ?                                    \
+            (5) :                                           \
+            (4))
+    #define CUB_SMEM_BANKS(arch)                            \
+        (1 << CUB_LOG_SMEM_BANKS(arch))
+
+    #define CUB_PTX_LOG_SMEM_BANKS      CUB_LOG_SMEM_BANKS(CUB_PTX_ARCH)
+    #define CUB_PTX_SMEM_BANKS          CUB_SMEM_BANKS(CUB_PTX_ARCH)
+#endif
+
+
+/// Oversubscription factor
+#ifndef CUB_SUBSCRIPTION_FACTOR
+    #define CUB_SUBSCRIPTION_FACTOR(arch)                   \
+        ((arch >= 300) ?                                    \
+            (5) :                                           \
+            ((arch >= 200) ?                                \
+                (3) :                                       \
+                (10)))
+    #define CUB_PTX_SUBSCRIPTION_FACTOR             CUB_SUBSCRIPTION_FACTOR(CUB_PTX_ARCH)
+#endif
+
+
+/// Prefer padding overhead vs X-way conflicts greater than this threshold
+#ifndef CUB_PREFER_CONFLICT_OVER_PADDING
+    #define CUB_PREFER_CONFLICT_OVER_PADDING(arch)          \
+        ((arch >= 300) ?                                    \
+            (1) :                                           \
+            (4))
+    #define CUB_PTX_PREFER_CONFLICT_OVER_PADDING    CUB_PREFER_CONFLICT_OVER_PADDING(CUB_PTX_ARCH)
+#endif
+
+
+/// Scale down the number of threads to keep same amount of scratch storage as the nominal configuration for 4B data.  Minimum of two warps.
+#ifndef CUB_SCALED_BLOCK_THREADS
+    #define CUB_SCALED_BLOCK_THREADS(NOMINAL_4B_BLOCK_THREADS, T, PTX_ARCH)                   \
+        (CUB_MIN(                                                                           \
+            NOMINAL_4B_BLOCK_THREADS,                                                       \
+            CUB_WARP_THREADS(PTX_ARCH) * CUB_MAX(                                           \
+                2,                                                                          \
+                (NOMINAL_4B_BLOCK_THREADS / CUB_WARP_THREADS(PTX_ARCH)) * 4 / sizeof(T))))
+#endif
+
+/// Scale down number of items per thread to keep the same amount of register storage as the nominal configuration for 4B data.  Minimum 1 item per thread
+#ifndef CUB_SCALED_ITEMS_PER_THREAD
+    #define CUB_SCALED_ITEMS_PER_THREAD(NOMINAL_4B_ITEMS_PER_THREAD, NOMINAL_4B_BLOCK_THREADS, T, PTX_ARCH)     \
+        CUB_MAX(                                                                                                \
+            1,                                                                                                  \
+            (sizeof(T) < 4) ?                                                                                   \
+                ((NOMINAL_4B_ITEMS_PER_THREAD * NOMINAL_4B_BLOCK_THREADS * 4) / CUB_MAX(4, sizeof(T))) / CUB_SCALED_BLOCK_THREADS(NOMINAL_4B_BLOCK_THREADS, T, PTX_ARCH) / 2 :  \
+                ((NOMINAL_4B_ITEMS_PER_THREAD * NOMINAL_4B_BLOCK_THREADS * 4) / CUB_MAX(4, sizeof(T))) / CUB_SCALED_BLOCK_THREADS(NOMINAL_4B_BLOCK_THREADS, T, PTX_ARCH))
+#endif
+
+/// Define both nominal threads-per-block and items-per-thread
+#ifndef CUB_SCALED_GRANULARITIES
+    #define CUB_SCALED_GRANULARITIES(NOMINAL_4B_BLOCK_THREADS, NOMINAL_4B_ITEMS_PER_THREAD, T)      \
+        CUB_SCALED_BLOCK_THREADS(NOMINAL_4B_BLOCK_THREADS, T, 200),                                   \
+        CUB_SCALED_ITEMS_PER_THREAD(NOMINAL_4B_ITEMS_PER_THREAD, NOMINAL_4B_BLOCK_THREADS, T, 200)
+#endif
+
+
+
+#endif  // Do not document
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/util_debug.cuh b/third_party/cub/cub/util_debug.cuh
new file mode 100644
index 0000000..3ad832e
--- /dev/null
+++ b/third_party/cub/cub/util_debug.cuh
@@ -0,0 +1,145 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Error and event logging routines.
+ *
+ * The following macros definitions are supported:
+ * - \p CUB_LOG.  Simple event messages are printed to \p stdout.
+ */
+
+#pragma once
+
+#include <stdio.h>
+#include "util_namespace.cuh"
+#include "util_arch.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilMgmt
+ * @{
+ */
+
+
+/// CUB error reporting macro (prints error messages to stderr)
+#if (defined(DEBUG) || defined(_DEBUG)) && !defined(CUB_STDERR)
+    #define CUB_STDERR
+#endif
+
+
+
+/**
+ * \brief %If \p CUB_STDERR is defined and \p error is not \p cudaSuccess, the corresponding error message is printed to \p stderr (or \p stdout in device code) along with the supplied source context.
+ *
+ * \return The CUDA error.
+ */
+__host__ __device__ __forceinline__ cudaError_t Debug(
+    cudaError_t     error,
+    const char*     filename,
+    int             line)
+{
+    (void)filename;
+    (void)line;
+#ifdef CUB_STDERR
+    if (error)
+    {
+    #if (CUB_PTX_ARCH == 0)
+        fprintf(stderr, "CUDA error %d [%s, %d]: %s\n", error, filename, line, cudaGetErrorString(error));
+        fflush(stderr);
+    #elif (CUB_PTX_ARCH >= 200)
+        printf("CUDA error %d [block (%d,%d,%d) thread (%d,%d,%d), %s, %d]\n", error, blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, filename, line);
+    #endif
+    }
+#endif
+    return error;
+}
+
+
+/**
+ * \brief Debug macro
+ */
+#ifndef CubDebug
+    #define CubDebug(e) cub::Debug((cudaError_t) (e), __FILE__, __LINE__)
+#endif
+
+
+/**
+ * \brief Debug macro with exit
+ */
+#ifndef CubDebugExit
+    #define CubDebugExit(e) if (cub::Debug((cudaError_t) (e), __FILE__, __LINE__)) { exit(1); }
+#endif
+
+
+/**
+ * \brief Log macro for printf statements.
+ */
+#if !defined(_CubLog)
+    #if !(defined(__clang__) && defined(__CUDA__))
+        #if (CUB_PTX_ARCH == 0)
+            #define _CubLog(format, ...) printf(format,__VA_ARGS__);
+        #elif (CUB_PTX_ARCH >= 200)
+            #define _CubLog(format, ...) printf("[block (%d,%d,%d), thread (%d,%d,%d)]: " format, blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, __VA_ARGS__);
+        #endif
+    #else
+        // XXX shameless hack for clang around variadic printf...
+        //     Compilies w/o supplying -std=c++11 but shows warning,
+        //     so we sielence them :)
+        #pragma clang diagnostic ignored "-Wc++11-extensions"
+        #pragma clang diagnostic ignored "-Wunnamed-type-template-args"
+            template <class... Args>
+            inline __host__ __device__ void va_printf(char const* format, Args const&... args)
+            {
+        #ifdef __CUDA_ARCH__
+              printf(format, blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, args...);
+        #else
+              printf(format, args...);
+        #endif
+            }
+        #ifndef __CUDA_ARCH__
+            #define _CubLog(format, ...) va_printf(format,__VA_ARGS__);
+        #else
+            #define _CubLog(format, ...) va_printf("[block (%d,%d,%d), thread (%d,%d,%d)]: " format, __VA_ARGS__);
+        #endif
+    #endif
+#endif
+
+
+
+
+/** @} */       // end group UtilMgmt
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/util_device.cuh b/third_party/cub/cub/util_device.cuh
new file mode 100644
index 0000000..a5f3b61
--- /dev/null
+++ b/third_party/cub/cub/util_device.cuh
@@ -0,0 +1,347 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Properties of a given CUDA device and the corresponding PTX bundle
+ */
+
+#pragma once
+
+#include "util_type.cuh"
+#include "util_arch.cuh"
+#include "util_debug.cuh"
+#include "util_namespace.cuh"
+#include "util_macro.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilMgmt
+ * @{
+ */
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+
+/**
+ * Alias temporaries to externally-allocated device storage (or simply return the amount of storage needed).
+ */
+template <int ALLOCATIONS>
+__host__ __device__ __forceinline__
+cudaError_t AliasTemporaries(
+    void    *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
+    size_t  &temp_storage_bytes,                ///< [in,out] Size in bytes of \t d_temp_storage allocation
+    void*   (&allocations)[ALLOCATIONS],        ///< [in,out] Pointers to device allocations needed
+    size_t  (&allocation_sizes)[ALLOCATIONS])   ///< [in] Sizes in bytes of device allocations needed
+{
+    const int ALIGN_BYTES   = 256;
+    const int ALIGN_MASK    = ~(ALIGN_BYTES - 1);
+
+    // Compute exclusive prefix sum over allocation requests
+    size_t allocation_offsets[ALLOCATIONS];
+    size_t bytes_needed = 0;
+    for (int i = 0; i < ALLOCATIONS; ++i)
+    {
+        size_t allocation_bytes = (allocation_sizes[i] + ALIGN_BYTES - 1) & ALIGN_MASK;
+        allocation_offsets[i] = bytes_needed;
+        bytes_needed += allocation_bytes;
+    }
+    bytes_needed += ALIGN_BYTES - 1;
+
+    // Check if the caller is simply requesting the size of the storage allocation
+    if (!d_temp_storage)
+    {
+        temp_storage_bytes = bytes_needed;
+        return cudaSuccess;
+    }
+
+    // Check if enough storage provided
+    if (temp_storage_bytes < bytes_needed)
+    {
+        return CubDebug(cudaErrorInvalidValue);
+    }
+
+    // Alias
+    d_temp_storage = (void *) ((size_t(d_temp_storage) + ALIGN_BYTES - 1) & ALIGN_MASK);
+    for (int i = 0; i < ALLOCATIONS; ++i)
+    {
+        allocations[i] = static_cast<char*>(d_temp_storage) + allocation_offsets[i];
+    }
+
+    return cudaSuccess;
+}
+
+
+/**
+ * Empty kernel for querying PTX manifest metadata (e.g., version) for the current device
+ */
+template <typename T>
+__global__ void EmptyKernel(void) { }
+
+
+#endif  // DOXYGEN_SHOULD_SKIP_THIS
+
+/**
+ * \brief Retrieves the PTX version that will be used on the current device (major * 100 + minor * 10)
+ */
+CUB_RUNTIME_FUNCTION __forceinline__ cudaError_t PtxVersion(int &ptx_version)
+{
+    struct Dummy
+    {
+        /// Type definition of the EmptyKernel kernel entry point
+        typedef void (*EmptyKernelPtr)();
+
+        /// Force EmptyKernel<void> to be generated if this class is used
+        CUB_RUNTIME_FUNCTION __forceinline__
+        EmptyKernelPtr Empty()
+        {
+            return EmptyKernel<void>;
+        }
+    };
+
+
+#ifndef CUB_RUNTIME_ENABLED
+    (void)ptx_version;
+
+    // CUDA API calls not supported from this device
+    return cudaErrorInvalidConfiguration;
+
+#elif (CUB_PTX_ARCH > 0)
+
+    ptx_version = CUB_PTX_ARCH;
+    return cudaSuccess;
+
+#else
+
+    cudaError_t error = cudaSuccess;
+    do
+    {
+        cudaFuncAttributes empty_kernel_attrs;
+        if (CubDebug(error = cudaFuncGetAttributes(&empty_kernel_attrs, EmptyKernel<void>))) break;
+        ptx_version = empty_kernel_attrs.ptxVersion * 10;
+    }
+    while (0);
+
+    return error;
+
+#endif
+}
+
+
+/**
+ * \brief Retrieves the SM version (major * 100 + minor * 10)
+ */
+CUB_RUNTIME_FUNCTION __forceinline__ cudaError_t SmVersion(int &sm_version, int device_ordinal)
+{
+#ifndef CUB_RUNTIME_ENABLED
+    (void)sm_version;
+    (void)device_ordinal;
+
+    // CUDA API calls not supported from this device
+    return cudaErrorInvalidConfiguration;
+
+#else
+
+    cudaError_t error = cudaSuccess;
+    do
+    {
+        // Fill in SM version
+        int major, minor;
+        if (CubDebug(error = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, device_ordinal))) break;
+        if (CubDebug(error = cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, device_ordinal))) break;
+        sm_version = major * 100 + minor * 10;
+    }
+    while (0);
+
+    return error;
+
+#endif
+}
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+/**
+ * Synchronize the stream if specified
+ */
+CUB_RUNTIME_FUNCTION __forceinline__
+static cudaError_t SyncStream(cudaStream_t stream)
+{
+#if (CUB_PTX_ARCH == 0)
+    return cudaStreamSynchronize(stream);
+#else
+    (void)stream;
+    // Device can't yet sync on a specific stream
+    return cudaDeviceSynchronize();
+#endif
+}
+
+
+/**
+ * \brief Computes maximum SM occupancy in thread blocks for executing the given kernel function pointer \p kernel_ptr on the current device with \p block_threads per thread block.
+ *
+ * \par Snippet
+ * The code snippet below illustrates the use of the MaxSmOccupancy function.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/util_device.cuh>
+ *
+ * template <typename T>
+ * __global__ void ExampleKernel()
+ * {
+ *     // Allocate shared memory for BlockScan
+ *     __shared__ volatile T buffer[4096];
+ *
+ *        ...
+ * }
+ *
+ *     ...
+ *
+ * // Determine SM occupancy for ExampleKernel specialized for unsigned char
+ * int max_sm_occupancy;
+ * MaxSmOccupancy(max_sm_occupancy, ExampleKernel<unsigned char>, 64);
+ *
+ * // max_sm_occupancy  <-- 4 on SM10
+ * // max_sm_occupancy  <-- 8 on SM20
+ * // max_sm_occupancy  <-- 12 on SM35
+ *
+ * \endcode
+ *
+ */
+template <typename KernelPtr>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t MaxSmOccupancy(
+    int                 &max_sm_occupancy,          ///< [out] maximum number of thread blocks that can reside on a single SM
+    KernelPtr           kernel_ptr,                 ///< [in] Kernel pointer for which to compute SM occupancy
+    int                 block_threads,              ///< [in] Number of threads per thread block
+    int                 dynamic_smem_bytes = 0)
+{
+#ifndef CUB_RUNTIME_ENABLED
+    (void)dynamic_smem_bytes;
+    (void)block_threads;
+    (void)kernel_ptr;
+    (void)max_sm_occupancy;
+
+    // CUDA API calls not supported from this device
+    return CubDebug(cudaErrorInvalidConfiguration);
+
+#else
+
+    return cudaOccupancyMaxActiveBlocksPerMultiprocessor (
+        &max_sm_occupancy,
+        kernel_ptr,
+        block_threads,
+        dynamic_smem_bytes);
+
+#endif  // CUB_RUNTIME_ENABLED
+}
+
+
+/******************************************************************************
+ * Policy management
+ ******************************************************************************/
+
+/**
+ * Kernel dispatch configuration
+ */
+struct KernelConfig
+{
+    int block_threads;
+    int items_per_thread;
+    int tile_size;
+    int sm_occupancy;
+
+    CUB_RUNTIME_FUNCTION __forceinline__
+    KernelConfig() : block_threads(0), items_per_thread(0), tile_size(0), sm_occupancy(0) {}
+
+    template <typename AgentPolicyT, typename KernelPtrT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    cudaError_t Init(KernelPtrT kernel_ptr)
+    {
+        block_threads        = AgentPolicyT::BLOCK_THREADS;
+        items_per_thread     = AgentPolicyT::ITEMS_PER_THREAD;
+        tile_size            = block_threads * items_per_thread;
+        cudaError_t retval   = MaxSmOccupancy(sm_occupancy, kernel_ptr, block_threads);
+        return retval;
+    }
+};
+
+
+
+/// Helper for dispatching into a policy chain
+template <int PTX_VERSION, typename PolicyT, typename PrevPolicyT>
+struct ChainedPolicy
+{
+   /// The policy for the active compiler pass
+   typedef typename If<(CUB_PTX_ARCH < PTX_VERSION), typename PrevPolicyT::ActivePolicy, PolicyT>::Type ActivePolicy;
+
+   /// Specializes and dispatches op in accordance to the first policy in the chain of adequate PTX version
+   template <typename FunctorT>
+   CUB_RUNTIME_FUNCTION __forceinline__
+   static cudaError_t Invoke(int ptx_version, FunctorT &op)
+   {
+       if (ptx_version < PTX_VERSION) {
+           return PrevPolicyT::Invoke(ptx_version, op);
+       }
+       return op.template Invoke<PolicyT>();
+   }
+};
+
+/// Helper for dispatching into a policy chain (end-of-chain specialization)
+template <int PTX_VERSION, typename PolicyT>
+struct ChainedPolicy<PTX_VERSION, PolicyT, PolicyT>
+{
+    /// The policy for the active compiler pass
+    typedef PolicyT ActivePolicy;
+
+    /// Specializes and dispatches op in accordance to the first policy in the chain of adequate PTX version
+    template <typename FunctorT>
+    CUB_RUNTIME_FUNCTION __forceinline__
+    static cudaError_t Invoke(int /*ptx_version*/, FunctorT &op) {
+        return op.template Invoke<PolicyT>();
+    }
+};
+
+
+
+
+#endif  // Do not document
+
+
+
+
+/** @} */       // end group UtilMgmt
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/util_macro.cuh b/third_party/cub/cub/util_macro.cuh
new file mode 100644
index 0000000..ff86365
--- /dev/null
+++ b/third_party/cub/cub/util_macro.cuh
@@ -0,0 +1,103 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Common C/C++ macro utilities
+ ******************************************************************************/
+
+#pragma once
+
+#include "util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilModule
+ * @{
+ */
+
+#ifndef CUB_ALIGN
+    #if defined(_WIN32) || defined(_WIN64)
+        /// Align struct
+        #define CUB_ALIGN(bytes) __declspec(align(32))
+    #else
+        /// Align struct
+        #define CUB_ALIGN(bytes) __attribute__((aligned(bytes)))
+    #endif
+#endif
+
+#ifndef CUB_MAX
+    /// Select maximum(a, b)
+    #define CUB_MAX(a, b) (((b) > (a)) ? (b) : (a))
+#endif
+
+#ifndef CUB_MIN
+    /// Select minimum(a, b)
+    #define CUB_MIN(a, b) (((b) < (a)) ? (b) : (a))
+#endif
+
+#ifndef CUB_QUOTIENT_FLOOR
+    /// Quotient of x/y rounded down to nearest integer
+    #define CUB_QUOTIENT_FLOOR(x, y) ((x) / (y))
+#endif
+
+#ifndef CUB_QUOTIENT_CEILING
+    /// Quotient of x/y rounded up to nearest integer
+    #define CUB_QUOTIENT_CEILING(x, y) (((x) + (y) - 1) / (y))
+#endif
+
+#ifndef CUB_ROUND_UP_NEAREST
+    /// x rounded up to the nearest multiple of y
+    #define CUB_ROUND_UP_NEAREST(x, y) ((((x) + (y) - 1) / (y)) * y)
+#endif
+
+#ifndef CUB_ROUND_DOWN_NEAREST
+    /// x rounded down to the nearest multiple of y
+    #define CUB_ROUND_DOWN_NEAREST(x, y) (((x) / (y)) * y)
+#endif
+
+
+#ifndef CUB_STATIC_ASSERT
+    #ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+        #define CUB_CAT_(a, b) a ## b
+        #define CUB_CAT(a, b) CUB_CAT_(a, b)
+    #endif // DOXYGEN_SHOULD_SKIP_THIS
+
+    /// Static assert
+    #define CUB_STATIC_ASSERT(cond, msg) typedef int CUB_CAT(cub_static_assert, __LINE__)[(cond) ? 1 : -1]
+#endif
+
+/** @} */       // end group UtilModule
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/util_namespace.cuh b/third_party/cub/cub/util_namespace.cuh
new file mode 100644
index 0000000..c8991d0
--- /dev/null
+++ b/third_party/cub/cub/util_namespace.cuh
@@ -0,0 +1,46 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Place-holder for prefixing the cub namespace
+ */
+
+#pragma once
+
+// For example:
+//#define CUB_NS_PREFIX namespace thrust{ namespace detail {
+//#define CUB_NS_POSTFIX } }
+
+#ifndef CUB_NS_PREFIX
+#define CUB_NS_PREFIX
+#endif
+
+#ifndef CUB_NS_POSTFIX
+#define CUB_NS_POSTFIX
+#endif
diff --git a/third_party/cub/cub/util_ptx.cuh b/third_party/cub/cub/util_ptx.cuh
new file mode 100644
index 0000000..582ca0d
--- /dev/null
+++ b/third_party/cub/cub/util_ptx.cuh
@@ -0,0 +1,758 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * PTX intrinsics
+ */
+
+
+#pragma once
+
+#include "util_type.cuh"
+#include "util_arch.cuh"
+#include "util_namespace.cuh"
+#include "util_debug.cuh"
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilPtx
+ * @{
+ */
+
+
+/******************************************************************************
+ * PTX helper macros
+ ******************************************************************************/
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+/**
+ * Register modifier for pointer-types (for inlining PTX assembly)
+ */
+#if defined(_WIN64) || defined(__LP64__)
+    #define __CUB_LP64__ 1
+    // 64-bit register modifier for inlined asm
+    #define _CUB_ASM_PTR_ "l"
+    #define _CUB_ASM_PTR_SIZE_ "u64"
+#else
+    #define __CUB_LP64__ 0
+    // 32-bit register modifier for inlined asm
+    #define _CUB_ASM_PTR_ "r"
+    #define _CUB_ASM_PTR_SIZE_ "u32"
+#endif
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/******************************************************************************
+ * Inlined PTX intrinsics
+ ******************************************************************************/
+
+/**
+ * \brief Shift-right then add.  Returns (\p x >> \p shift) + \p addend.
+ */
+__device__ __forceinline__ unsigned int SHR_ADD(
+    unsigned int x,
+    unsigned int shift,
+    unsigned int addend)
+{
+    unsigned int ret;
+#if CUB_PTX_ARCH >= 200
+    asm ("vshr.u32.u32.u32.clamp.add %0, %1, %2, %3;" :
+        "=r"(ret) : "r"(x), "r"(shift), "r"(addend));
+#else
+    ret = (x >> shift) + addend;
+#endif
+    return ret;
+}
+
+
+/**
+ * \brief Shift-left then add.  Returns (\p x << \p shift) + \p addend.
+ */
+__device__ __forceinline__ unsigned int SHL_ADD(
+    unsigned int x,
+    unsigned int shift,
+    unsigned int addend)
+{
+    unsigned int ret;
+#if CUB_PTX_ARCH >= 200
+    asm ("vshl.u32.u32.u32.clamp.add %0, %1, %2, %3;" :
+        "=r"(ret) : "r"(x), "r"(shift), "r"(addend));
+#else
+    ret = (x << shift) + addend;
+#endif
+    return ret;
+}
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+/**
+ * Bitfield-extract.
+ */
+template <typename UnsignedBits, int BYTE_LEN>
+__device__ __forceinline__ unsigned int BFE(
+    UnsignedBits            source,
+    unsigned int            bit_start,
+    unsigned int            num_bits,
+    Int2Type<BYTE_LEN>      /*byte_len*/)
+{
+    unsigned int bits;
+#if CUB_PTX_ARCH >= 200
+    asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(bits) : "r"((unsigned int) source), "r"(bit_start), "r"(num_bits));
+#else
+    const unsigned int MASK = (1 << num_bits) - 1;
+    bits = (source >> bit_start) & MASK;
+#endif
+    return bits;
+}
+
+
+/**
+ * Bitfield-extract for 64-bit types.
+ */
+template <typename UnsignedBits>
+__device__ __forceinline__ unsigned int BFE(
+    UnsignedBits            source,
+    unsigned int            bit_start,
+    unsigned int            num_bits,
+    Int2Type<8>             /*byte_len*/)
+{
+    const unsigned long long MASK = (1ull << num_bits) - 1;
+    return (source >> bit_start) & MASK;
+}
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+/**
+ * \brief Bitfield-extract.  Extracts \p num_bits from \p source starting at bit-offset \p bit_start.  The input \p source may be an 8b, 16b, 32b, or 64b unsigned integer type.
+ */
+template <typename UnsignedBits>
+__device__ __forceinline__ unsigned int BFE(
+    UnsignedBits source,
+    unsigned int bit_start,
+    unsigned int num_bits)
+{
+    return BFE(source, bit_start, num_bits, Int2Type<sizeof(UnsignedBits)>());
+}
+
+
+/**
+ * \brief Bitfield insert.  Inserts the \p num_bits least significant bits of \p y into \p x at bit-offset \p bit_start.
+ */
+__device__ __forceinline__ void BFI(
+    unsigned int &ret,
+    unsigned int x,
+    unsigned int y,
+    unsigned int bit_start,
+    unsigned int num_bits)
+{
+#if CUB_PTX_ARCH >= 200
+    asm ("bfi.b32 %0, %1, %2, %3, %4;" :
+        "=r"(ret) : "r"(y), "r"(x), "r"(bit_start), "r"(num_bits));
+#else
+    x <<= bit_start;
+    unsigned int MASK_X = ((1 << num_bits) - 1) << bit_start;
+    unsigned int MASK_Y = ~MASK_X;
+    ret = (y & MASK_Y) | (x & MASK_X);
+#endif
+}
+
+
+/**
+ * \brief Three-operand add.  Returns \p x + \p y + \p z.
+ */
+__device__ __forceinline__ unsigned int IADD3(unsigned int x, unsigned int y, unsigned int z)
+{
+#if CUB_PTX_ARCH >= 200
+    asm ("vadd.u32.u32.u32.add %0, %1, %2, %3;" : "=r"(x) : "r"(x), "r"(y), "r"(z));
+#else
+    x = x + y + z;
+#endif
+    return x;
+}
+
+
+/**
+ * \brief Byte-permute. Pick four arbitrary bytes from two 32-bit registers, and reassemble them into a 32-bit destination register.  For SM2.0 or later.
+ *
+ * \par
+ * The bytes in the two source registers \p a and \p b are numbered from 0 to 7:
+ * {\p b, \p a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}. For each of the four bytes
+ * {b3, b2, b1, b0} selected in the return value, a 4-bit selector is defined within
+ * the four lower "nibbles" of \p index: {\p index } = {n7, n6, n5, n4, n3, n2, n1, n0}
+ *
+ * \par Snippet
+ * The code snippet below illustrates byte-permute.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     int a        = 0x03020100;
+ *     int b        = 0x07060504;
+ *     int index    = 0x00007531;
+ *
+ *     int selected = PRMT(a, b, index);    // 0x07050301
+ *
+ * \endcode
+ *
+ */
+__device__ __forceinline__ int PRMT(unsigned int a, unsigned int b, unsigned int index)
+{
+    int ret;
+    asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(ret) : "r"(a), "r"(b), "r"(index));
+    return ret;
+}
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+/**
+ * Sync-threads barrier.
+ */
+__device__ __forceinline__ void BAR(int count)
+{
+    asm volatile("bar.sync 1, %0;" : : "r"(count));
+}
+
+/**
+ * CTA barrier
+ */
+__device__  __forceinline__ void CTA_SYNC()
+{
+    __syncthreads();
+}
+
+
+/**
+ * CTA barrier with predicate
+ */
+__device__  __forceinline__ int CTA_SYNC_AND(int p)
+{
+    return __syncthreads_and(p);
+}
+
+
+/**
+ * Warp barrier
+ */
+__device__  __forceinline__ void WARP_SYNC(unsigned int member_mask)
+{
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+    __syncwarp(member_mask);
+#endif
+}
+
+
+/**
+ * Warp any
+ */
+__device__  __forceinline__ int WARP_ANY(int predicate, unsigned int member_mask)
+{
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+    return __any_sync(member_mask, predicate);
+#else
+    return ::__any(predicate);
+#endif
+}
+
+
+/**
+ * Warp any
+ */
+__device__  __forceinline__ int WARP_ALL(int predicate, unsigned int member_mask)
+{
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+    return __all_sync(member_mask, predicate);
+#else
+    return ::__all(predicate);
+#endif
+}
+
+
+/**
+ * Warp ballot
+ */
+__device__  __forceinline__ int WARP_BALLOT(int predicate, unsigned int member_mask)
+{
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+    return __ballot_sync(member_mask, predicate);
+#else
+    return __ballot(predicate);
+#endif
+}
+
+/**
+ * Warp synchronous shfl_up
+ */
+__device__ __forceinline__ 
+unsigned int SHFL_UP_SYNC(unsigned int word, int src_offset, int flags, unsigned int member_mask)
+{
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+    asm volatile("shfl.sync.up.b32 %0, %1, %2, %3, %4;"
+        : "=r"(word) : "r"(word), "r"(src_offset), "r"(flags), "r"(member_mask));
+#else
+    asm volatile("shfl.up.b32 %0, %1, %2, %3;"
+        : "=r"(word) : "r"(word), "r"(src_offset), "r"(flags));
+#endif
+    return word;
+}
+
+/**
+ * Warp synchronous shfl_down
+ */
+__device__ __forceinline__ 
+unsigned int SHFL_DOWN_SYNC(unsigned int word, int src_offset, int flags, unsigned int member_mask)
+{
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+    asm volatile("shfl.sync.down.b32 %0, %1, %2, %3, %4;"
+        : "=r"(word) : "r"(word), "r"(src_offset), "r"(flags), "r"(member_mask));
+#else
+    asm volatile("shfl.down.b32 %0, %1, %2, %3;"
+        : "=r"(word) : "r"(word), "r"(src_offset), "r"(flags));
+#endif
+    return word;
+}
+
+/**
+ * Warp synchronous shfl_idx
+ */
+__device__ __forceinline__ 
+unsigned int SHFL_IDX_SYNC(unsigned int word, int src_lane, int flags, unsigned int member_mask)
+{
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+    asm volatile("shfl.sync.idx.b32 %0, %1, %2, %3, %4;"
+        : "=r"(word) : "r"(word), "r"(src_lane), "r"(flags), "r"(member_mask));
+#else
+    asm volatile("shfl.idx.b32 %0, %1, %2, %3;"
+        : "=r"(word) : "r"(word), "r"(src_lane), "r"(flags));
+#endif
+    return word;
+}
+
+/**
+ * Floating point multiply. (Mantissa LSB rounds towards zero.)
+ */
+__device__ __forceinline__ float FMUL_RZ(float a, float b)
+{
+    float d;
+    asm ("mul.rz.f32 %0, %1, %2;" : "=f"(d) : "f"(a), "f"(b));
+    return d;
+}
+
+
+/**
+ * Floating point multiply-add. (Mantissa LSB rounds towards zero.)
+ */
+__device__ __forceinline__ float FFMA_RZ(float a, float b, float c)
+{
+    float d;
+    asm ("fma.rz.f32 %0, %1, %2, %3;" : "=f"(d) : "f"(a), "f"(b), "f"(c));
+    return d;
+}
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+/**
+ * \brief Terminates the calling thread
+ */
+__device__ __forceinline__ void ThreadExit() {
+    asm volatile("exit;");
+}    
+
+
+/**
+ * \brief  Abort execution and generate an interrupt to the host CPU
+ */
+__device__ __forceinline__ void ThreadTrap() {
+    asm volatile("trap;");
+}
+
+
+/**
+ * \brief Returns the row-major linear thread identifier for a multidimensional thread block
+ */
+__device__ __forceinline__ int RowMajorTid(int block_dim_x, int block_dim_y, int block_dim_z)
+{
+    return ((block_dim_z == 1) ? 0 : (threadIdx.z * block_dim_x * block_dim_y)) +
+            ((block_dim_y == 1) ? 0 : (threadIdx.y * block_dim_x)) +
+            threadIdx.x;
+}
+
+
+/**
+ * \brief Returns the warp lane ID of the calling thread
+ */
+__device__ __forceinline__ unsigned int LaneId()
+{
+    unsigned int ret;
+    asm ("mov.u32 %0, %%laneid;" : "=r"(ret) );
+    return ret;
+}
+
+
+/**
+ * \brief Returns the warp ID of the calling thread.  Warp ID is guaranteed to be unique among warps, but may not correspond to a zero-based ranking within the thread block.
+ */
+__device__ __forceinline__ unsigned int WarpId()
+{
+    unsigned int ret;
+    asm ("mov.u32 %0, %%warpid;" : "=r"(ret) );
+    return ret;
+}
+
+/**
+ * \brief Returns the warp lane mask of all lanes less than the calling thread
+ */
+__device__ __forceinline__ unsigned int LaneMaskLt()
+{
+    unsigned int ret;
+    asm ("mov.u32 %0, %%lanemask_lt;" : "=r"(ret) );
+    return ret;
+}
+
+/**
+ * \brief Returns the warp lane mask of all lanes less than or equal to the calling thread
+ */
+__device__ __forceinline__ unsigned int LaneMaskLe()
+{
+    unsigned int ret;
+    asm ("mov.u32 %0, %%lanemask_le;" : "=r"(ret) );
+    return ret;
+}
+
+/**
+ * \brief Returns the warp lane mask of all lanes greater than the calling thread
+ */
+__device__ __forceinline__ unsigned int LaneMaskGt()
+{
+    unsigned int ret;
+    asm ("mov.u32 %0, %%lanemask_gt;" : "=r"(ret) );
+    return ret;
+}
+
+/**
+ * \brief Returns the warp lane mask of all lanes greater than or equal to the calling thread
+ */
+__device__ __forceinline__ unsigned int LaneMaskGe()
+{
+    unsigned int ret;
+    asm ("mov.u32 %0, %%lanemask_ge;" : "=r"(ret) );
+    return ret;
+}
+
+/** @} */       // end group UtilPtx
+
+
+
+
+/**
+ * \brief Shuffle-up for any data type.  Each <em>warp-lane<sub>i</sub></em> obtains the value \p input contributed by <em>warp-lane</em><sub><em>i</em>-<tt>src_offset</tt></sub>.  For thread lanes \e i < src_offset, the thread's own \p input is returned to the thread. ![](shfl_up_logo.png)
+ * \ingroup WarpModule
+ *
+ * \tparam LOGICAL_WARP_THREADS     The number of threads per "logical" warp.  Must be a power-of-two <= 32.
+ * \tparam T                        <b>[inferred]</b> The input/output element type
+ *
+ * \par
+ * - Available only for SM3.0 or newer
+ *
+ * \par Snippet
+ * The code snippet below illustrates each thread obtaining a \p double value from the
+ * predecessor of its predecessor.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/util_ptx.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Obtain one input item per thread
+ *     double thread_data = ...
+ *
+ *     // Obtain item from two ranks below
+ *     double peer_data = ShuffleUp<32>(thread_data, 2, 0, 0xffffffff);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the first warp of threads is <tt>{1.0, 2.0, 3.0, 4.0, 5.0, ..., 32.0}</tt>.
+ * The corresponding output \p peer_data will be <tt>{1.0, 2.0, 1.0, 2.0, 3.0, ..., 30.0}</tt>.
+ *
+ */
+template <
+    int LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
+    typename T>
+__device__ __forceinline__ T ShuffleUp(
+    T               input,              ///< [in] The value to broadcast
+    int             src_offset,         ///< [in] The relative down-offset of the peer to read from
+    int             first_thread,       ///< [in] Index of first lane in logical warp (typically 0)
+    unsigned int    member_mask)        ///< [in] 32-bit mask of participating warp lanes
+{
+    /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
+    enum {
+        SHFL_C = (32 - LOGICAL_WARP_THREADS) << 8
+    };
+
+    typedef typename UnitWord<T>::ShuffleWord ShuffleWord;
+
+    const int       WORDS           = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord);
+ 
+    T               output;
+    ShuffleWord     *output_alias   = reinterpret_cast<ShuffleWord *>(&output);
+    ShuffleWord     *input_alias    = reinterpret_cast<ShuffleWord *>(&input);
+
+    unsigned int shuffle_word;
+    shuffle_word = SHFL_UP_SYNC((unsigned int)input_alias[0], src_offset, first_thread | SHFL_C, member_mask);
+    output_alias[0] = shuffle_word;
+
+    #pragma unroll
+    for (int WORD = 1; WORD < WORDS; ++WORD)
+    {
+        shuffle_word       = SHFL_UP_SYNC((unsigned int)input_alias[WORD], src_offset, first_thread | SHFL_C, member_mask);
+        output_alias[WORD] = shuffle_word;
+    }
+
+    return output;
+}
+
+
+/**
+ * \brief Shuffle-down for any data type.  Each <em>warp-lane<sub>i</sub></em> obtains the value \p input contributed by <em>warp-lane</em><sub><em>i</em>+<tt>src_offset</tt></sub>.  For thread lanes \e i >= WARP_THREADS, the thread's own \p input is returned to the thread.  ![](shfl_down_logo.png)
+ * \ingroup WarpModule
+ *
+ * \tparam LOGICAL_WARP_THREADS     The number of threads per "logical" warp.  Must be a power-of-two <= 32.
+ * \tparam T                        <b>[inferred]</b> The input/output element type
+ *
+ * \par
+ * - Available only for SM3.0 or newer
+ *
+ * \par Snippet
+ * The code snippet below illustrates each thread obtaining a \p double value from the
+ * successor of its successor.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/util_ptx.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Obtain one input item per thread
+ *     double thread_data = ...
+ *
+ *     // Obtain item from two ranks below
+ *     double peer_data = ShuffleDown<32>(thread_data, 2, 31, 0xffffffff);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the first warp of threads is <tt>{1.0, 2.0, 3.0, 4.0, 5.0, ..., 32.0}</tt>.
+ * The corresponding output \p peer_data will be <tt>{3.0, 4.0, 5.0, 6.0, 7.0, ..., 32.0}</tt>.
+ *
+ */
+template <
+    int LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
+    typename T>
+__device__ __forceinline__ T ShuffleDown(
+    T               input,              ///< [in] The value to broadcast
+    int             src_offset,         ///< [in] The relative up-offset of the peer to read from
+    int             last_thread,        ///< [in] Index of last thread in logical warp (typically 31 for a 32-thread warp)
+    unsigned int    member_mask)        ///< [in] 32-bit mask of participating warp lanes
+{
+    /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
+    enum {
+        SHFL_C = (32 - LOGICAL_WARP_THREADS) << 8
+    };
+
+    typedef typename UnitWord<T>::ShuffleWord ShuffleWord;
+
+    const int       WORDS           = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord);
+
+    T               output;
+    ShuffleWord     *output_alias   = reinterpret_cast<ShuffleWord *>(&output);
+    ShuffleWord     *input_alias    = reinterpret_cast<ShuffleWord *>(&input);
+
+    unsigned int shuffle_word;
+    shuffle_word    = SHFL_DOWN_SYNC((unsigned int)input_alias[0], src_offset, last_thread | SHFL_C, member_mask);
+    output_alias[0] = shuffle_word;
+
+    #pragma unroll
+    for (int WORD = 1; WORD < WORDS; ++WORD)
+    {
+        shuffle_word       = SHFL_DOWN_SYNC((unsigned int)input_alias[WORD], src_offset, last_thread | SHFL_C, member_mask);
+        output_alias[WORD] = shuffle_word;
+    }
+
+    return output;
+}
+
+
+/**
+ * \brief Shuffle-broadcast for any data type.  Each <em>warp-lane<sub>i</sub></em> obtains the value \p input
+ * contributed by <em>warp-lane</em><sub><tt>src_lane</tt></sub>.  For \p src_lane < 0 or \p src_lane >= WARP_THREADS,
+ * then the thread's own \p input is returned to the thread. ![](shfl_broadcast_logo.png)
+ *
+ * \tparam LOGICAL_WARP_THREADS     The number of threads per "logical" warp.  Must be a power-of-two <= 32.
+ * \tparam T                        <b>[inferred]</b> The input/output element type
+ *
+ * \ingroup WarpModule
+ *
+ * \par
+ * - Available only for SM3.0 or newer
+ *
+ * \par Snippet
+ * The code snippet below illustrates each thread obtaining a \p double value from <em>warp-lane</em><sub>0</sub>.
+ *
+ * \par
+ * \code
+ * #include <cub/cub.cuh>   // or equivalently <cub/util_ptx.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Obtain one input item per thread
+ *     double thread_data = ...
+ *
+ *     // Obtain item from thread 0
+ *     double peer_data = ShuffleIndex<32>(thread_data, 0, 0xffffffff);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the first warp of threads is <tt>{1.0, 2.0, 3.0, 4.0, 5.0, ..., 32.0}</tt>.
+ * The corresponding output \p peer_data will be <tt>{1.0, 1.0, 1.0, 1.0, 1.0, ..., 1.0}</tt>.
+ *
+ */
+template <
+    int LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
+    typename T>
+__device__ __forceinline__ T ShuffleIndex(
+    T               input,                  ///< [in] The value to broadcast
+    int             src_lane,               ///< [in] Which warp lane is to do the broadcasting
+    unsigned int    member_mask)            ///< [in] 32-bit mask of participating warp lanes
+{
+    /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
+    enum {
+        SHFL_C = ((32 - LOGICAL_WARP_THREADS) << 8) | (LOGICAL_WARP_THREADS - 1)
+    };
+
+    typedef typename UnitWord<T>::ShuffleWord ShuffleWord;
+
+    const int       WORDS           = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord);
+
+    T               output;
+    ShuffleWord     *output_alias   = reinterpret_cast<ShuffleWord *>(&output);
+    ShuffleWord     *input_alias    = reinterpret_cast<ShuffleWord *>(&input);
+
+    unsigned int shuffle_word;
+    shuffle_word = SHFL_IDX_SYNC((unsigned int)input_alias[0],
+                                 src_lane,
+                                 SHFL_C,
+                                 member_mask);
+
+    output_alias[0] = shuffle_word;
+
+    #pragma unroll
+    for (int WORD = 1; WORD < WORDS; ++WORD)
+    {
+        shuffle_word = SHFL_IDX_SYNC((unsigned int)input_alias[WORD],
+                                     src_lane,
+                                     SHFL_C,
+                                     member_mask);
+
+        output_alias[WORD] = shuffle_word;
+    }
+
+    return output;
+}
+
+
+
+/**
+ * Compute a 32b mask of threads having the same least-significant
+ * LABEL_BITS of \p label as the calling thread.
+ */
+template <int LABEL_BITS>
+inline __device__ unsigned int MatchAny(unsigned int label)
+{
+    unsigned int retval;
+
+    // Extract masks of common threads for each bit
+    #pragma unroll
+    for (int BIT = 0; BIT < LABEL_BITS; ++BIT)
+    {
+        unsigned int mask;
+        unsigned int current_bit = 1 << BIT;
+        asm ("{\n"
+            "    .reg .pred p;\n"
+            "    and.b32 %0, %1, %2;"
+            "    setp.eq.u32 p, %0, %2;\n"
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+            "    vote.ballot.sync.b32 %0, p, 0xffffffff;\n"
+#else
+            "    vote.ballot.b32 %0, p;\n"
+#endif
+            "    @!p not.b32 %0, %0;\n"
+            "}\n" : "=r"(mask) : "r"(label), "r"(current_bit));
+
+        // Remove peers who differ
+        retval = (BIT == 0) ? mask : retval & mask;
+    }
+
+    return retval;
+
+//  // VOLTA match
+//    unsigned int retval;
+//    asm ("{\n"
+//         "    match.any.sync.b32 %0, %1, 0xffffffff;\n"
+//         "}\n" : "=r"(retval) : "r"(label));
+//    return retval;
+
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/util_type.cuh b/third_party/cub/cub/util_type.cuh
new file mode 100644
index 0000000..0ba41e1
--- /dev/null
+++ b/third_party/cub/cub/util_type.cuh
@@ -0,0 +1,1167 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * Common type manipulation (metaprogramming) utilities
+ */
+
+#pragma once
+
+#include <iostream>
+#include <limits>
+#include <cfloat>
+
+#if (__CUDACC_VER_MAJOR__ >= 9)
+    #include <cuda_fp16.h>
+#endif
+
+#include "util_macro.cuh"
+#include "util_arch.cuh"
+#include "util_namespace.cuh"
+
+
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup UtilModule
+ * @{
+ */
+
+
+
+/******************************************************************************
+ * Type equality
+ ******************************************************************************/
+
+/**
+ * \brief Type selection (<tt>IF ? ThenType : ElseType</tt>)
+ */
+template <bool IF, typename ThenType, typename ElseType>
+struct If
+{
+    /// Conditional type result
+    typedef ThenType Type;      // true
+};
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+template <typename ThenType, typename ElseType>
+struct If<false, ThenType, ElseType>
+{
+    typedef ElseType Type;      // false
+};
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+
+/******************************************************************************
+ * Conditional types
+ ******************************************************************************/
+
+/**
+ * \brief Type equality test
+ */
+template <typename A, typename B>
+struct Equals
+{
+    enum {
+        VALUE = 0,
+        NEGATE = 1
+    };
+};
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+template <typename A>
+struct Equals <A, A>
+{
+    enum {
+        VALUE = 1,
+        NEGATE = 0
+    };
+};
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/******************************************************************************
+ * Static math
+ ******************************************************************************/
+
+/**
+ * \brief Statically determine log2(N), rounded up.
+ *
+ * For example:
+ *     Log2<8>::VALUE   // 3
+ *     Log2<3>::VALUE   // 2
+ */
+template <int N, int CURRENT_VAL = N, int COUNT = 0>
+struct Log2
+{
+    /// Static logarithm value
+    enum { VALUE = Log2<N, (CURRENT_VAL >> 1), COUNT + 1>::VALUE };         // Inductive case
+};
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+template <int N, int COUNT>
+struct Log2<N, 0, COUNT>
+{
+    enum {VALUE = (1 << (COUNT - 1) < N) ?                                  // Base case
+        COUNT :
+        COUNT - 1 };
+};
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/**
+ * \brief Statically determine if N is a power-of-two
+ */
+template <int N>
+struct PowerOfTwo
+{
+    enum { VALUE = ((N & (N - 1)) == 0) };
+};
+
+
+
+/******************************************************************************
+ * Pointer vs. iterator detection
+ ******************************************************************************/
+
+/**
+ * \brief Pointer vs. iterator
+ */
+template <typename Tp>
+struct IsPointer
+{
+    enum { VALUE = 0 };
+};
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+template <typename Tp>
+struct IsPointer<Tp*>
+{
+    enum { VALUE = 1 };
+};
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+
+/******************************************************************************
+ * Qualifier detection
+ ******************************************************************************/
+
+/**
+ * \brief Volatile modifier test
+ */
+template <typename Tp>
+struct IsVolatile
+{
+    enum { VALUE = 0 };
+};
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+template <typename Tp>
+struct IsVolatile<Tp volatile>
+{
+    enum { VALUE = 1 };
+};
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/******************************************************************************
+ * Qualifier removal
+ ******************************************************************************/
+
+/**
+ * \brief Removes \p const and \p volatile qualifiers from type \p Tp.
+ *
+ * For example:
+ *     <tt>typename RemoveQualifiers<volatile int>::Type         // int;</tt>
+ */
+template <typename Tp, typename Up = Tp>
+struct RemoveQualifiers
+{
+    /// Type without \p const and \p volatile qualifiers
+    typedef Up Type;
+};
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+template <typename Tp, typename Up>
+struct RemoveQualifiers<Tp, volatile Up>
+{
+    typedef Up Type;
+};
+
+template <typename Tp, typename Up>
+struct RemoveQualifiers<Tp, const Up>
+{
+    typedef Up Type;
+};
+
+template <typename Tp, typename Up>
+struct RemoveQualifiers<Tp, const volatile Up>
+{
+    typedef Up Type;
+};
+
+
+/******************************************************************************
+ * Marker types
+ ******************************************************************************/
+
+/**
+ * \brief A simple "NULL" marker type
+ */
+struct NullType
+{
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+    template <typename T>
+    __host__ __device__ __forceinline__ NullType& operator =(const T&) { return *this; }
+
+    __host__ __device__ __forceinline__ bool operator ==(const NullType&) { return true; }
+
+    __host__ __device__ __forceinline__ bool operator !=(const NullType&) { return false; }
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+};
+
+
+/**
+ * \brief Allows for the treatment of an integral constant as a type at compile-time (e.g., to achieve static call dispatch based on constant integral values)
+ */
+template <int A>
+struct Int2Type
+{
+   enum {VALUE = A};
+};
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+
+/******************************************************************************
+ * Size and alignment
+ ******************************************************************************/
+
+/// Structure alignment
+template <typename T>
+struct AlignBytes
+{
+    struct Pad
+    {
+        T       val;
+        char    byte;
+    };
+
+    enum
+    {
+        /// The "true CUDA" alignment of T in bytes
+        ALIGN_BYTES = sizeof(Pad) - sizeof(T)
+    };
+
+    /// The "truly aligned" type
+    typedef T Type;
+};
+
+// Specializations where host C++ compilers (e.g., 32-bit Windows) may disagree
+// with device C++ compilers (EDG) on types passed as template parameters through
+// kernel functions
+
+#define __CUB_ALIGN_BYTES(t, b)         \
+    template <> struct AlignBytes<t>    \
+    { enum { ALIGN_BYTES = b }; typedef __align__(b) t Type; };
+
+__CUB_ALIGN_BYTES(short4, 8)
+__CUB_ALIGN_BYTES(ushort4, 8)
+__CUB_ALIGN_BYTES(int2, 8)
+__CUB_ALIGN_BYTES(uint2, 8)
+__CUB_ALIGN_BYTES(long long, 8)
+__CUB_ALIGN_BYTES(unsigned long long, 8)
+__CUB_ALIGN_BYTES(float2, 8)
+__CUB_ALIGN_BYTES(double, 8)
+#ifdef _WIN32
+    __CUB_ALIGN_BYTES(long2, 8)
+    __CUB_ALIGN_BYTES(ulong2, 8)
+#else
+    __CUB_ALIGN_BYTES(long2, 16)
+    __CUB_ALIGN_BYTES(ulong2, 16)
+#endif
+__CUB_ALIGN_BYTES(int4, 16)
+__CUB_ALIGN_BYTES(uint4, 16)
+__CUB_ALIGN_BYTES(float4, 16)
+__CUB_ALIGN_BYTES(long4, 16)
+__CUB_ALIGN_BYTES(ulong4, 16)
+__CUB_ALIGN_BYTES(longlong2, 16)
+__CUB_ALIGN_BYTES(ulonglong2, 16)
+__CUB_ALIGN_BYTES(double2, 16)
+__CUB_ALIGN_BYTES(longlong4, 16)
+__CUB_ALIGN_BYTES(ulonglong4, 16)
+__CUB_ALIGN_BYTES(double4, 16)
+
+template <typename T> struct AlignBytes<volatile T> : AlignBytes<T> {};
+template <typename T> struct AlignBytes<const T> : AlignBytes<T> {};
+template <typename T> struct AlignBytes<const volatile T> : AlignBytes<T> {};
+
+
+/// Unit-words of data movement
+template <typename T>
+struct UnitWord
+{
+    enum {
+        ALIGN_BYTES = AlignBytes<T>::ALIGN_BYTES
+    };
+
+    template <typename Unit>
+    struct IsMultiple
+    {
+        enum {
+            UNIT_ALIGN_BYTES    = AlignBytes<Unit>::ALIGN_BYTES,
+            IS_MULTIPLE         = (sizeof(T) % sizeof(Unit) == 0) && (ALIGN_BYTES % UNIT_ALIGN_BYTES == 0)
+        };
+    };
+
+    /// Biggest shuffle word that T is a whole multiple of and is not larger than the alignment of T
+    typedef typename If<IsMultiple<int>::IS_MULTIPLE,
+        unsigned int,
+        typename If<IsMultiple<short>::IS_MULTIPLE,
+            unsigned short,
+            unsigned char>::Type>::Type         ShuffleWord;
+
+    /// Biggest volatile word that T is a whole multiple of and is not larger than the alignment of T
+    typedef typename If<IsMultiple<long long>::IS_MULTIPLE,
+        unsigned long long,
+        ShuffleWord>::Type                      VolatileWord;
+
+    /// Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T
+    typedef typename If<IsMultiple<longlong2>::IS_MULTIPLE,
+        ulonglong2,
+        VolatileWord>::Type                     DeviceWord;
+
+    /// Biggest texture reference word that T is a whole multiple of and is not larger than the alignment of T
+    typedef typename If<IsMultiple<int4>::IS_MULTIPLE,
+        uint4,
+        typename If<IsMultiple<int2>::IS_MULTIPLE,
+            uint2,
+            ShuffleWord>::Type>::Type           TextureWord;
+};
+
+
+// float2 specialization workaround (for SM10-SM13)
+template <>
+struct UnitWord <float2>
+{
+    typedef int         ShuffleWord;
+#if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
+    typedef float       VolatileWord;
+    typedef uint2       DeviceWord;
+#else
+    typedef unsigned long long   VolatileWord;
+    typedef unsigned long long   DeviceWord;
+#endif
+    typedef float2      TextureWord;
+};
+
+// float4 specialization workaround (for SM10-SM13)
+template <>
+struct UnitWord <float4>
+{
+    typedef int         ShuffleWord;
+#if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
+    typedef float               VolatileWord;
+    typedef uint4               DeviceWord;
+#else
+    typedef unsigned long long  VolatileWord;
+    typedef ulonglong2          DeviceWord;
+#endif
+    typedef float4              TextureWord;
+};
+
+
+// char2 specialization workaround (for SM10-SM13)
+template <>
+struct UnitWord <char2>
+{
+    typedef unsigned short      ShuffleWord;
+#if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
+    typedef unsigned short      VolatileWord;
+    typedef short               DeviceWord;
+#else
+    typedef unsigned short      VolatileWord;
+    typedef unsigned short      DeviceWord;
+#endif
+    typedef unsigned short      TextureWord;
+};
+
+
+template <typename T> struct UnitWord<volatile T> : UnitWord<T> {};
+template <typename T> struct UnitWord<const T> : UnitWord<T> {};
+template <typename T> struct UnitWord<const volatile T> : UnitWord<T> {};
+
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+
+/******************************************************************************
+ * Vector type inference utilities.
+ ******************************************************************************/
+
+/**
+ * \brief Exposes a member typedef \p Type that names the corresponding CUDA vector type if one exists.  Otherwise \p Type refers to the CubVector structure itself, which will wrap the corresponding \p x, \p y, etc. vector fields.
+ */
+template <typename T, int vec_elements> struct CubVector;
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+enum
+{
+    /// The maximum number of elements in CUDA vector types
+    MAX_VEC_ELEMENTS = 4,
+};
+
+
+/**
+ * Generic vector-1 type
+ */
+template <typename T>
+struct CubVector<T, 1>
+{
+    T x;
+
+    typedef T BaseType;
+    typedef CubVector<T, 1> Type;
+};
+
+/**
+ * Generic vector-2 type
+ */
+template <typename T>
+struct CubVector<T, 2>
+{
+    T x;
+    T y;
+
+    typedef T BaseType;
+    typedef CubVector<T, 2> Type;
+};
+
+/**
+ * Generic vector-3 type
+ */
+template <typename T>
+struct CubVector<T, 3>
+{
+    T x;
+    T y;
+    T z;
+
+    typedef T BaseType;
+    typedef CubVector<T, 3> Type;
+};
+
+/**
+ * Generic vector-4 type
+ */
+template <typename T>
+struct CubVector<T, 4>
+{
+    T x;
+    T y;
+    T z;
+    T w;
+
+    typedef T BaseType;
+    typedef CubVector<T, 4> Type;
+};
+
+
+/**
+ * Macro for expanding partially-specialized built-in vector types
+ */
+#define CUB_DEFINE_VECTOR_TYPE(base_type,short_type)                                                    \
+                                                                                                        \
+    template<> struct CubVector<base_type, 1> : short_type##1                                           \
+    {                                                                                                   \
+      typedef base_type       BaseType;                                                                 \
+      typedef short_type##1   Type;                                                                     \
+      __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const {           \
+          CubVector retval;                                                                             \
+          retval.x = x + other.x;                                                                       \
+          return retval;                                                                                \
+      }                                                                                                 \
+      __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const {           \
+          CubVector retval;                                                                             \
+          retval.x = x - other.x;                                                                       \
+          return retval;                                                                                \
+      }                                                                                                 \
+    };                                                                                                  \
+                                                                                                        \
+    template<> struct CubVector<base_type, 2> : short_type##2                                           \
+    {                                                                                                   \
+        typedef base_type       BaseType;                                                               \
+        typedef short_type##2   Type;                                                                   \
+        __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const {         \
+            CubVector retval;                                                                           \
+            retval.x = x + other.x;                                                                     \
+            retval.y = y + other.y;                                                                     \
+            return retval;                                                                              \
+        }                                                                                               \
+        __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const {         \
+            CubVector retval;                                                                           \
+            retval.x = x - other.x;                                                                     \
+            retval.y = y - other.y;                                                                     \
+            return retval;                                                                              \
+        }                                                                                               \
+    };                                                                                                  \
+                                                                                                        \
+    template<> struct CubVector<base_type, 3> : short_type##3                                           \
+    {                                                                                                   \
+        typedef base_type       BaseType;                                                               \
+        typedef short_type##3   Type;                                                                   \
+        __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const {         \
+            CubVector retval;                                                                           \
+            retval.x = x + other.x;                                                                     \
+            retval.y = y + other.y;                                                                     \
+            retval.z = z + other.z;                                                                     \
+            return retval;                                                                              \
+        }                                                                                               \
+        __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const {         \
+            CubVector retval;                                                                           \
+            retval.x = x - other.x;                                                                     \
+            retval.y = y - other.y;                                                                     \
+            retval.z = z - other.z;                                                                     \
+            return retval;                                                                              \
+        }                                                                                               \
+    };                                                                                                  \
+                                                                                                        \
+    template<> struct CubVector<base_type, 4> : short_type##4                                           \
+    {                                                                                                   \
+        typedef base_type       BaseType;                                                               \
+        typedef short_type##4   Type;                                                                   \
+        __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const {         \
+            CubVector retval;                                                                           \
+            retval.x = x + other.x;                                                                     \
+            retval.y = y + other.y;                                                                     \
+            retval.z = z + other.z;                                                                     \
+            retval.w = w + other.w;                                                                     \
+            return retval;                                                                              \
+        }                                                                                               \
+        __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const {         \
+            CubVector retval;                                                                           \
+            retval.x = x - other.x;                                                                     \
+            retval.y = y - other.y;                                                                     \
+            retval.z = z - other.z;                                                                     \
+            retval.w = w - other.w;                                                                     \
+            return retval;                                                                              \
+        }                                                                                               \
+    };
+
+
+
+// Expand CUDA vector types for built-in primitives
+CUB_DEFINE_VECTOR_TYPE(char,               char)
+CUB_DEFINE_VECTOR_TYPE(signed char,        char)
+CUB_DEFINE_VECTOR_TYPE(short,              short)
+CUB_DEFINE_VECTOR_TYPE(int,                int)
+CUB_DEFINE_VECTOR_TYPE(long,               long)
+CUB_DEFINE_VECTOR_TYPE(long long,          longlong)
+CUB_DEFINE_VECTOR_TYPE(unsigned char,      uchar)
+CUB_DEFINE_VECTOR_TYPE(unsigned short,     ushort)
+CUB_DEFINE_VECTOR_TYPE(unsigned int,       uint)
+CUB_DEFINE_VECTOR_TYPE(unsigned long,      ulong)
+CUB_DEFINE_VECTOR_TYPE(unsigned long long, ulonglong)
+CUB_DEFINE_VECTOR_TYPE(float,              float)
+CUB_DEFINE_VECTOR_TYPE(double,             double)
+CUB_DEFINE_VECTOR_TYPE(bool,               uchar)
+
+// Undefine macros
+#undef CUB_DEFINE_VECTOR_TYPE
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+
+/******************************************************************************
+ * Wrapper types
+ ******************************************************************************/
+
+/**
+ * \brief A storage-backing wrapper that allows types with non-trivial constructors to be aliased in unions
+ */
+template <typename T>
+struct Uninitialized
+{
+    /// Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T
+    typedef typename UnitWord<T>::DeviceWord DeviceWord;
+
+    enum
+    {
+        WORDS = sizeof(T) / sizeof(DeviceWord)
+    };
+
+    /// Backing storage
+    DeviceWord storage[WORDS];
+
+    /// Alias
+    __host__ __device__ __forceinline__ T& Alias()
+    {
+        return reinterpret_cast<T&>(*this);
+    }
+};
+
+
+/**
+ * \brief A key identifier paired with a corresponding value
+ */
+template <
+    typename    _Key,
+    typename    _Value
+#if defined(_WIN32) && !defined(_WIN64)
+    , bool KeyIsLT = (AlignBytes<_Key>::ALIGN_BYTES < AlignBytes<_Value>::ALIGN_BYTES)
+    , bool ValIsLT = (AlignBytes<_Value>::ALIGN_BYTES < AlignBytes<_Key>::ALIGN_BYTES)
+#endif // #if defined(_WIN32) && !defined(_WIN64)
+    >
+struct KeyValuePair
+{
+    typedef _Key    Key;                ///< Key data type
+    typedef _Value  Value;              ///< Value data type
+
+    Key     key;                        ///< Item key
+    Value   value;                      ///< Item value
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    KeyValuePair() {}
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    KeyValuePair(Key const& key, Value const& value) : key(key), value(value) {}
+
+    /// Inequality operator
+    __host__ __device__ __forceinline__ bool operator !=(const KeyValuePair &b)
+    {
+        return (value != b.value) || (key != b.key);
+    }
+};
+
+#if defined(_WIN32) && !defined(_WIN64)
+
+/**
+ * Win32 won't do 16B alignment.  This can present two problems for
+ * should-be-16B-aligned (but actually 8B aligned) built-in and intrinsics members:
+ * 1) If a smaller-aligned item were to be listed first, the host compiler places the
+ *    should-be-16B item at too early an offset (and disagrees with device compiler)
+ * 2) Or, if a smaller-aligned item lists second, the host compiler gets the size
+ *    of the struct wrong (and disagrees with device compiler)
+ *
+ * So we put the larger-should-be-aligned item first, and explicitly pad the
+ * end of the struct
+ */
+
+/// Smaller key specialization
+template <typename K, typename V>
+struct KeyValuePair<K, V, true, false>
+{
+    typedef K Key;
+    typedef V Value;
+
+    typedef char Pad[AlignBytes<V>::ALIGN_BYTES - AlignBytes<K>::ALIGN_BYTES];
+
+    Value   value;  // Value has larger would-be alignment and goes first
+    Key     key;
+    Pad     pad;
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    KeyValuePair() {}
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    KeyValuePair(Key const& key, Value const& value) : key(key), value(value) {}
+
+    /// Inequality operator
+    __host__ __device__ __forceinline__ bool operator !=(const KeyValuePair &b)
+    {
+        return (value != b.value) || (key != b.key);
+    }
+};
+
+
+/// Smaller value specialization
+template <typename K, typename V>
+struct KeyValuePair<K, V, false, true>
+{
+    typedef K Key;
+    typedef V Value;
+
+    typedef char Pad[AlignBytes<K>::ALIGN_BYTES - AlignBytes<V>::ALIGN_BYTES];
+
+    Key     key;    // Key has larger would-be alignment and goes first
+    Value   value;
+    Pad     pad;
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    KeyValuePair() {}
+
+    /// Constructor
+    __host__ __device__ __forceinline__
+    KeyValuePair(Key const& key, Value const& value) : key(key), value(value) {}
+
+    /// Inequality operator
+    __host__ __device__ __forceinline__ bool operator !=(const KeyValuePair &b)
+    {
+        return (value != b.value) || (key != b.key);
+    }
+};
+
+#endif // #if defined(_WIN32) && !defined(_WIN64)
+
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+
+/**
+ * \brief A wrapper for passing simple static arrays as kernel parameters
+ */
+template <typename T, int COUNT>
+struct ArrayWrapper
+{
+
+    /// Statically-sized array of type \p T
+    T array[COUNT];
+
+    /// Constructor
+    __host__ __device__ __forceinline__ ArrayWrapper() {}
+};
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+/**
+ * \brief Double-buffer storage wrapper for multi-pass stream transformations that require more than one storage array for streaming intermediate results back and forth.
+ *
+ * Many multi-pass computations require a pair of "ping-pong" storage
+ * buffers (e.g., one for reading from and the other for writing to, and then
+ * vice-versa for the subsequent pass).  This structure wraps a set of device
+ * buffers and a "selector" member to track which is "current".
+ */
+template <typename T>
+struct DoubleBuffer
+{
+    /// Pair of device buffer pointers
+    T *d_buffers[2];
+
+    ///  Selector into \p d_buffers (i.e., the active/valid buffer)
+    int selector;
+
+    /// \brief Constructor
+    __host__ __device__ __forceinline__ DoubleBuffer()
+    {
+        selector = 0;
+        d_buffers[0] = NULL;
+        d_buffers[1] = NULL;
+    }
+
+    /// \brief Constructor
+    __host__ __device__ __forceinline__ DoubleBuffer(
+        T *d_current,         ///< The currently valid buffer
+        T *d_alternate)       ///< Alternate storage buffer of the same size as \p d_current
+    {
+        selector = 0;
+        d_buffers[0] = d_current;
+        d_buffers[1] = d_alternate;
+    }
+
+    /// \brief Return pointer to the currently valid buffer
+    __host__ __device__ __forceinline__ T* Current() { return d_buffers[selector]; }
+
+    /// \brief Return pointer to the currently invalid buffer
+    __host__ __device__ __forceinline__ T* Alternate() { return d_buffers[selector ^ 1]; }
+
+};
+
+
+
+/******************************************************************************
+ * Typedef-detection
+ ******************************************************************************/
+
+
+/**
+ * \brief Defines a structure \p detector_name that is templated on type \p T.  The \p detector_name struct exposes a constant member \p VALUE indicating whether or not parameter \p T exposes a nested type \p nested_type_name
+ */
+#define CUB_DEFINE_DETECT_NESTED_TYPE(detector_name, nested_type_name)  \
+    template <typename T>                                               \
+    struct detector_name                                                \
+    {                                                                   \
+        template <typename C>                                           \
+        static char& test(typename C::nested_type_name*);               \
+        template <typename>                                             \
+        static int& test(...);                                          \
+        enum                                                            \
+        {                                                               \
+            VALUE = sizeof(test<T>(0)) < sizeof(int)                    \
+        };                                                              \
+    };
+
+
+
+/******************************************************************************
+ * Simple enable-if (similar to Boost)
+ ******************************************************************************/
+
+/**
+ * \brief Simple enable-if (similar to Boost)
+ */
+template <bool Condition, class T = void>
+struct EnableIf
+{
+    /// Enable-if type for SFINAE dummy variables
+    typedef T Type;
+};
+
+
+template <class T>
+struct EnableIf<false, T> {};
+
+
+
+/******************************************************************************
+ * Typedef-detection
+ ******************************************************************************/
+
+/**
+ * \brief Determine whether or not BinaryOp's functor is of the form <tt>bool operator()(const T& a, const T&b)</tt> or <tt>bool operator()(const T& a, const T&b, unsigned int idx)</tt>
+ */
+template <typename T, typename BinaryOp>
+struct BinaryOpHasIdxParam
+{
+private:
+/*
+    template <typename BinaryOpT, bool (BinaryOpT::*)(const T &a, const T &b, unsigned int idx) const>  struct SFINAE1 {};
+    template <typename BinaryOpT, bool (BinaryOpT::*)(const T &a, const T &b, unsigned int idx)>        struct SFINAE2 {};
+    template <typename BinaryOpT, bool (BinaryOpT::*)(T a, T b, unsigned int idx) const>                struct SFINAE3 {};
+    template <typename BinaryOpT, bool (BinaryOpT::*)(T a, T b, unsigned int idx)>                      struct SFINAE4 {};
+*/
+    template <typename BinaryOpT, bool (BinaryOpT::*)(const T &a, const T &b, int idx) const>           struct SFINAE5 {};
+    template <typename BinaryOpT, bool (BinaryOpT::*)(const T &a, const T &b, int idx)>                 struct SFINAE6 {};
+    template <typename BinaryOpT, bool (BinaryOpT::*)(T a, T b, int idx) const>                         struct SFINAE7 {};
+    template <typename BinaryOpT, bool (BinaryOpT::*)(T a, T b, int idx)>                               struct SFINAE8 {};
+/*
+    template <typename BinaryOpT> static char Test(SFINAE1<BinaryOpT, &BinaryOpT::operator()> *);
+    template <typename BinaryOpT> static char Test(SFINAE2<BinaryOpT, &BinaryOpT::operator()> *);
+    template <typename BinaryOpT> static char Test(SFINAE3<BinaryOpT, &BinaryOpT::operator()> *);
+    template <typename BinaryOpT> static char Test(SFINAE4<BinaryOpT, &BinaryOpT::operator()> *);
+*/
+    template <typename BinaryOpT> __host__ __device__ static char Test(SFINAE5<BinaryOpT, &BinaryOpT::operator()> *);
+    template <typename BinaryOpT> __host__ __device__ static char Test(SFINAE6<BinaryOpT, &BinaryOpT::operator()> *);
+    template <typename BinaryOpT> __host__ __device__ static char Test(SFINAE7<BinaryOpT, &BinaryOpT::operator()> *);
+    template <typename BinaryOpT> __host__ __device__ static char Test(SFINAE8<BinaryOpT, &BinaryOpT::operator()> *);
+
+    template <typename BinaryOpT> static int Test(...);
+
+public:
+
+    /// Whether the functor BinaryOp has a third <tt>unsigned int</tt> index param
+    static const bool HAS_PARAM = sizeof(Test<BinaryOp>(NULL)) == sizeof(char);
+};
+
+
+
+
+/******************************************************************************
+ * Simple type traits utilities.
+ *
+ * For example:
+ *     Traits<int>::CATEGORY             // SIGNED_INTEGER
+ *     Traits<NullType>::NULL_TYPE       // true
+ *     Traits<uint4>::CATEGORY           // NOT_A_NUMBER
+ *     Traits<uint4>::PRIMITIVE;         // false
+ *
+ ******************************************************************************/
+
+/**
+ * \brief Basic type traits categories
+ */
+enum Category
+{
+    NOT_A_NUMBER,
+    SIGNED_INTEGER,
+    UNSIGNED_INTEGER,
+    FLOATING_POINT
+};
+
+
+/**
+ * \brief Basic type traits
+ */
+template <Category _CATEGORY, bool _PRIMITIVE, bool _NULL_TYPE, typename _UnsignedBits, typename T>
+struct BaseTraits
+{
+    /// Category
+    static const Category CATEGORY      = _CATEGORY;
+    enum
+    {
+        PRIMITIVE       = _PRIMITIVE,
+        NULL_TYPE       = _NULL_TYPE,
+    };
+};
+
+
+/**
+ * Basic type traits (unsigned primitive specialization)
+ */
+template <typename _UnsignedBits, typename T>
+struct BaseTraits<UNSIGNED_INTEGER, true, false, _UnsignedBits, T>
+{
+    typedef _UnsignedBits       UnsignedBits;
+
+    static const Category       CATEGORY    = UNSIGNED_INTEGER;
+    static const UnsignedBits   LOWEST_KEY  = UnsignedBits(0);
+    static const UnsignedBits   MAX_KEY     = UnsignedBits(-1);
+
+    enum
+    {
+        PRIMITIVE       = true,
+        NULL_TYPE       = false,
+    };
+
+
+    static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
+    {
+        return key;
+    }
+
+    static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
+    {
+        return key;
+    }
+
+    static __host__ __device__ __forceinline__ T Max()
+    {
+        UnsignedBits retval = MAX_KEY;
+        return reinterpret_cast<T&>(retval);
+    }
+
+    static __host__ __device__ __forceinline__ T Lowest()
+    {
+        UnsignedBits retval = LOWEST_KEY;
+        return reinterpret_cast<T&>(retval);
+    }
+};
+
+
+/**
+ * Basic type traits (signed primitive specialization)
+ */
+template <typename _UnsignedBits, typename T>
+struct BaseTraits<SIGNED_INTEGER, true, false, _UnsignedBits, T>
+{
+    typedef _UnsignedBits       UnsignedBits;
+
+    static const Category       CATEGORY    = SIGNED_INTEGER;
+    static const UnsignedBits   HIGH_BIT    = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1);
+    static const UnsignedBits   LOWEST_KEY  = HIGH_BIT;
+    static const UnsignedBits   MAX_KEY     = UnsignedBits(-1) ^ HIGH_BIT;
+
+    enum
+    {
+        PRIMITIVE       = true,
+        NULL_TYPE       = false,
+    };
+
+    static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
+    {
+        return key ^ HIGH_BIT;
+    };
+
+    static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
+    {
+        return key ^ HIGH_BIT;
+    };
+
+    static __host__ __device__ __forceinline__ T Max()
+    {
+        UnsignedBits retval = MAX_KEY;
+        return reinterpret_cast<T&>(retval);
+    }
+
+    static __host__ __device__ __forceinline__ T Lowest()
+    {
+        UnsignedBits retval = LOWEST_KEY;
+        return reinterpret_cast<T&>(retval);
+    }
+};
+
+template <typename _T>
+struct FpLimits;
+
+template <>
+struct FpLimits<float>
+{
+    static __host__ __device__ __forceinline__ float Max() {
+        return FLT_MAX;
+    }
+
+    static __host__ __device__ __forceinline__ float Lowest() {
+        return FLT_MAX * float(-1);
+    }
+};
+
+template <>
+struct FpLimits<double>
+{
+    static __host__ __device__ __forceinline__ double Max() {
+        return DBL_MAX;
+    }
+
+    static __host__ __device__ __forceinline__ double Lowest() {
+        return DBL_MAX  * double(-1);
+    }
+};
+
+
+#if (__CUDACC_VER_MAJOR__ >= 9)
+template <>
+struct FpLimits<__half>
+{
+    static __host__ __device__ __forceinline__ __half Max() {
+        unsigned short max_word = 0x7BFF;
+        return reinterpret_cast<__half&>(max_word);
+    }
+
+    static __host__ __device__ __forceinline__ __half Lowest() {
+        unsigned short lowest_word = 0xFBFF;
+        return reinterpret_cast<__half&>(lowest_word);
+    }
+};
+#endif
+
+
+/**
+ * Basic type traits (fp primitive specialization)
+ */
+template <typename _UnsignedBits, typename T>
+struct BaseTraits<FLOATING_POINT, true, false, _UnsignedBits, T>
+{
+    typedef _UnsignedBits       UnsignedBits;
+
+    static const Category       CATEGORY    = FLOATING_POINT;
+    static const UnsignedBits   HIGH_BIT    = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1);
+    static const UnsignedBits   LOWEST_KEY  = UnsignedBits(-1);
+    static const UnsignedBits   MAX_KEY     = UnsignedBits(-1) ^ HIGH_BIT;
+
+    enum
+    {
+        PRIMITIVE       = true,
+        NULL_TYPE       = false,
+    };
+
+    static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
+    {
+        UnsignedBits mask = (key & HIGH_BIT) ? UnsignedBits(-1) : HIGH_BIT;
+        return key ^ mask;
+    };
+
+    static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
+    {
+        UnsignedBits mask = (key & HIGH_BIT) ? HIGH_BIT : UnsignedBits(-1);
+        return key ^ mask;
+    };
+
+    static __host__ __device__ __forceinline__ T Max() {
+        return FpLimits<T>::Max();
+    }
+
+    static __host__ __device__ __forceinline__ T Lowest() {
+        return FpLimits<T>::Lowest();
+    }
+};
+
+
+/**
+ * \brief Numeric type traits
+ */
+template <typename T> struct NumericTraits :            BaseTraits<NOT_A_NUMBER, false, false, T, T> {};
+
+template <> struct NumericTraits<NullType> :            BaseTraits<NOT_A_NUMBER, false, true, NullType, NullType> {};
+
+template <> struct NumericTraits<char> :                BaseTraits<(std::numeric_limits<char>::is_signed) ? SIGNED_INTEGER : UNSIGNED_INTEGER, true, false, unsigned char, char> {};
+template <> struct NumericTraits<signed char> :         BaseTraits<SIGNED_INTEGER, true, false, unsigned char, signed char> {};
+template <> struct NumericTraits<short> :               BaseTraits<SIGNED_INTEGER, true, false, unsigned short, short> {};
+template <> struct NumericTraits<int> :                 BaseTraits<SIGNED_INTEGER, true, false, unsigned int, int> {};
+template <> struct NumericTraits<long> :                BaseTraits<SIGNED_INTEGER, true, false, unsigned long, long> {};
+template <> struct NumericTraits<long long> :           BaseTraits<SIGNED_INTEGER, true, false, unsigned long long, long long> {};
+
+template <> struct NumericTraits<unsigned char> :       BaseTraits<UNSIGNED_INTEGER, true, false, unsigned char, unsigned char> {};
+template <> struct NumericTraits<unsigned short> :      BaseTraits<UNSIGNED_INTEGER, true, false, unsigned short, unsigned short> {};
+template <> struct NumericTraits<unsigned int> :        BaseTraits<UNSIGNED_INTEGER, true, false, unsigned int, unsigned int> {};
+template <> struct NumericTraits<unsigned long> :       BaseTraits<UNSIGNED_INTEGER, true, false, unsigned long, unsigned long> {};
+template <> struct NumericTraits<unsigned long long> :  BaseTraits<UNSIGNED_INTEGER, true, false, unsigned long long, unsigned long long> {};
+
+template <> struct NumericTraits<float> :               BaseTraits<FLOATING_POINT, true, false, unsigned int, float> {};
+template <> struct NumericTraits<double> :              BaseTraits<FLOATING_POINT, true, false, unsigned long long, double> {};
+#if (__CUDACC_VER_MAJOR__ >= 9)
+    template <> struct NumericTraits<__half> :          BaseTraits<FLOATING_POINT, true, false, unsigned short, __half> {};
+#endif
+
+template <> struct NumericTraits<bool> :                BaseTraits<UNSIGNED_INTEGER, true, false, typename UnitWord<bool>::VolatileWord, bool> {};
+
+
+
+/**
+ * \brief Type traits
+ */
+template <typename T>
+struct Traits : NumericTraits<typename RemoveQualifiers<T>::Type> {};
+
+
+#endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+/** @} */       // end group UtilModule
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/warp/specializations/warp_reduce_shfl.cuh b/third_party/cub/cub/warp/specializations/warp_reduce_shfl.cuh
new file mode 100644
index 0000000..bbbf37e
--- /dev/null
+++ b/third_party/cub/cub/warp/specializations/warp_reduce_shfl.cuh
@@ -0,0 +1,541 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::WarpReduceShfl provides SHFL-based variants of parallel reduction of items partitioned across a CUDA thread warp.
+ */
+
+#pragma once
+
+#include "../../thread/thread_operators.cuh"
+#include "../../util_ptx.cuh"
+#include "../../util_type.cuh"
+#include "../../util_macro.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \brief WarpReduceShfl provides SHFL-based variants of parallel reduction of items partitioned across a CUDA thread warp.
+ *
+ * LOGICAL_WARP_THREADS must be a power-of-two
+ */
+template <
+    typename    T,                      ///< Data type being reduced
+    int         LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
+    int         PTX_ARCH>               ///< The PTX compute capability for which to to specialize this collective
+struct WarpReduceShfl
+{
+    //---------------------------------------------------------------------
+    // Constants and type definitions
+    //---------------------------------------------------------------------
+
+    enum
+    {
+        /// Whether the logical warp size and the PTX warp size coincide
+        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
+
+        /// The number of warp reduction steps
+        STEPS = Log2<LOGICAL_WARP_THREADS>::VALUE,
+
+        /// Number of logical warps in a PTX warp
+        LOGICAL_WARPS = CUB_WARP_THREADS(PTX_ARCH) / LOGICAL_WARP_THREADS,
+
+        /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
+        SHFL_C = (CUB_WARP_THREADS(PTX_ARCH) - LOGICAL_WARP_THREADS) << 8
+
+    };
+
+    template <typename S>
+    struct IsInteger
+    {
+        enum {
+            ///Whether the data type is a small (32b or less) integer for which we can use a single SFHL instruction per exchange
+            IS_SMALL_UNSIGNED = (Traits<S>::CATEGORY == UNSIGNED_INTEGER) && (sizeof(S) <= sizeof(unsigned int))
+        };
+    };
+
+
+    /// Shared memory storage layout type
+    typedef NullType TempStorage;
+
+
+    //---------------------------------------------------------------------
+    // Thread fields
+    //---------------------------------------------------------------------
+
+    /// Lane index in logical warp
+    unsigned int lane_id;
+
+    /// Logical warp index in 32-thread physical warp
+    unsigned int warp_id;
+
+    /// 32-thread physical warp member mask of logical warp
+    unsigned int member_mask;
+
+
+    //---------------------------------------------------------------------
+    // Construction
+    //---------------------------------------------------------------------
+
+    /// Constructor
+    __device__ __forceinline__ WarpReduceShfl(
+        TempStorage &/*temp_storage*/)
+    {
+        lane_id = LaneId();
+        warp_id = 0;
+        member_mask = 0xffffffffu >> (CUB_WARP_THREADS(PTX_ARCH) - LOGICAL_WARP_THREADS);
+
+        if (!IS_ARCH_WARP)
+        {
+            warp_id = lane_id / LOGICAL_WARP_THREADS;
+            lane_id = lane_id % LOGICAL_WARP_THREADS;
+            member_mask = member_mask << (warp_id * LOGICAL_WARP_THREADS);
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Reduction steps
+    //---------------------------------------------------------------------
+
+    /// Reduction (specialized for summation across uint32 types)
+    __device__ __forceinline__ unsigned int ReduceStep(
+        unsigned int    input,              ///< [in] Calling thread's input item.
+        cub::Sum        /*reduction_op*/,   ///< [in] Binary reduction operator
+        int             last_lane,          ///< [in] Index of last lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        unsigned int output;
+        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .u32 r0;"
+            "  .reg .pred p;"
+            "  shfl.sync.down.b32 r0|p, %1, %2, %3, %5;"
+            "  @p add.u32 r0, r0, %4;"
+            "  mov.u32 %0, r0;"
+            "}"
+            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .u32 r0;"
+            "  .reg .pred p;"
+            "  shfl.down.b32 r0|p, %1, %2, %3;"
+            "  @p add.u32 r0, r0, %4;"
+            "  mov.u32 %0, r0;"
+            "}"
+            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input));
+#endif
+
+        return output;
+    }
+
+
+    /// Reduction (specialized for summation across fp32 types)
+    __device__ __forceinline__ float ReduceStep(
+        float           input,              ///< [in] Calling thread's input item.
+        cub::Sum        /*reduction_op*/,   ///< [in] Binary reduction operator
+        int             last_lane,          ///< [in] Index of last lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        float output;
+        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .f32 r0;"
+            "  .reg .pred p;"
+            "  shfl.sync.down.b32 r0|p, %1, %2, %3, %5;"
+            "  @p add.f32 r0, r0, %4;"
+            "  mov.f32 %0, r0;"
+            "}"
+            : "=f"(output) : "f"(input), "r"(offset), "r"(shfl_c), "f"(input), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .f32 r0;"
+            "  .reg .pred p;"
+            "  shfl.down.b32 r0|p, %1, %2, %3;"
+            "  @p add.f32 r0, r0, %4;"
+            "  mov.f32 %0, r0;"
+            "}"
+            : "=f"(output) : "f"(input), "r"(offset), "r"(shfl_c), "f"(input));
+#endif
+
+        return output;
+    }
+
+
+    /// Reduction (specialized for summation across unsigned long long types)
+    __device__ __forceinline__ unsigned long long ReduceStep(
+        unsigned long long  input,              ///< [in] Calling thread's input item.
+        cub::Sum            /*reduction_op*/,   ///< [in] Binary reduction operator
+        int                 last_lane,          ///< [in] Index of last lane in segment
+        int                 offset)             ///< [in] Up-offset to pull from
+    {
+        unsigned long long output;
+        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
+
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.sync.down.b32 lo|p, lo, %2, %3, %4;"
+            "  shfl.sync.down.b32 hi|p, hi, %2, %3, %4;"
+            "  mov.b64 %0, {lo, hi};"
+            "  @p add.u64 %0, %0, %1;"
+            "}"
+            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.down.b32 lo|p, lo, %2, %3;"
+            "  shfl.down.b32 hi|p, hi, %2, %3;"
+            "  mov.b64 %0, {lo, hi};"
+            "  @p add.u64 %0, %0, %1;"
+            "}"
+            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c));
+#endif
+
+        return output;
+    }
+
+
+    /// Reduction (specialized for summation across long long types)
+    __device__ __forceinline__ long long ReduceStep(
+        long long           input,              ///< [in] Calling thread's input item.
+        cub::Sum            /*reduction_op*/,   ///< [in] Binary reduction operator
+        int                 last_lane,          ///< [in] Index of last lane in segment
+        int                 offset)             ///< [in] Up-offset to pull from
+    {
+        long long output;
+        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.sync.down.b32 lo|p, lo, %2, %3, %4;"
+            "  shfl.sync.down.b32 hi|p, hi, %2, %3, %4;"
+            "  mov.b64 %0, {lo, hi};"
+            "  @p add.s64 %0, %0, %1;"
+            "}"
+            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.down.b32 lo|p, lo, %2, %3;"
+            "  shfl.down.b32 hi|p, hi, %2, %3;"
+            "  mov.b64 %0, {lo, hi};"
+            "  @p add.s64 %0, %0, %1;"
+            "}"
+            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c));
+#endif
+
+        return output;
+    }
+
+
+    /// Reduction (specialized for summation across double types)
+    __device__ __forceinline__ double ReduceStep(
+        double              input,              ///< [in] Calling thread's input item.
+        cub::Sum            /*reduction_op*/,   ///< [in] Binary reduction operator
+        int                 last_lane,          ///< [in] Index of last lane in segment
+        int                 offset)             ///< [in] Up-offset to pull from
+    {
+        double output;
+        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  .reg .f64 r0;"
+            "  mov.b64 %0, %1;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.sync.down.b32 lo|p, lo, %2, %3, %4;"
+            "  shfl.sync.down.b32 hi|p, hi, %2, %3, %4;"
+            "  mov.b64 r0, {lo, hi};"
+            "  @p add.f64 %0, %0, r0;"
+            "}"
+            : "=d"(output) : "d"(input), "r"(offset), "r"(shfl_c), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  .reg .f64 r0;"
+            "  mov.b64 %0, %1;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.down.b32 lo|p, lo, %2, %3;"
+            "  shfl.down.b32 hi|p, hi, %2, %3;"
+            "  mov.b64 r0, {lo, hi};"
+            "  @p add.f64 %0, %0, r0;"
+            "}"
+            : "=d"(output) : "d"(input), "r"(offset), "r"(shfl_c));
+#endif
+
+        return output;
+    }
+
+
+    /// Reduction (specialized for swizzled ReduceByKeyOp<cub::Sum> across KeyValuePair<KeyT, ValueT> types)
+    template <typename ValueT, typename KeyT>
+    __device__ __forceinline__ KeyValuePair<KeyT, ValueT> ReduceStep(
+        KeyValuePair<KeyT, ValueT>                  input,              ///< [in] Calling thread's input item.
+        SwizzleScanOp<ReduceByKeyOp<cub::Sum> >     /*reduction_op*/,   ///< [in] Binary reduction operator
+        int                                         last_lane,          ///< [in] Index of last lane in segment
+        int                                         offset)             ///< [in] Up-offset to pull from
+    {
+        KeyValuePair<KeyT, ValueT> output;
+
+        KeyT other_key = ShuffleDown<LOGICAL_WARP_THREADS>(input.key, offset, last_lane, member_mask);
+        
+        output.key = input.key;
+        output.value = ReduceStep(
+            input.value, 
+            cub::Sum(), 
+            last_lane, 
+            offset, 
+            Int2Type<IsInteger<ValueT>::IS_SMALL_UNSIGNED>());
+
+        if (input.key != other_key)
+            output.value = input.value;
+
+        return output;
+    }
+
+
+
+    /// Reduction (specialized for swizzled ReduceBySegmentOp<cub::Sum> across KeyValuePair<OffsetT, ValueT> types)
+    template <typename ValueT, typename OffsetT>
+    __device__ __forceinline__ KeyValuePair<OffsetT, ValueT> ReduceStep(
+        KeyValuePair<OffsetT, ValueT>                 input,              ///< [in] Calling thread's input item.
+        SwizzleScanOp<ReduceBySegmentOp<cub::Sum> >   /*reduction_op*/,   ///< [in] Binary reduction operator
+        int                                           last_lane,          ///< [in] Index of last lane in segment
+        int                                           offset)             ///< [in] Up-offset to pull from
+    {
+        KeyValuePair<OffsetT, ValueT> output;
+
+        output.value = ReduceStep(input.value, cub::Sum(), last_lane, offset, Int2Type<IsInteger<ValueT>::IS_SMALL_UNSIGNED>());
+        output.key = ReduceStep(input.key, cub::Sum(), last_lane, offset, Int2Type<IsInteger<OffsetT>::IS_SMALL_UNSIGNED>());
+
+        if (input.key > 0)
+            output.value = input.value;
+
+        return output;
+    }
+
+
+    /// Reduction step (generic)
+    template <typename _T, typename ReductionOp>
+    __device__ __forceinline__ _T ReduceStep(
+        _T                  input,              ///< [in] Calling thread's input item.
+        ReductionOp         reduction_op,       ///< [in] Binary reduction operator
+        int                 last_lane,          ///< [in] Index of last lane in segment
+        int                 offset)             ///< [in] Up-offset to pull from
+    {
+        _T output = input;
+
+        _T temp = ShuffleDown<LOGICAL_WARP_THREADS>(output, offset, last_lane, member_mask);
+
+        // Perform reduction op if valid
+        if (offset + lane_id <= last_lane)
+            output = reduction_op(input, temp);
+
+        return output;
+    }
+
+
+    /// Reduction step (specialized for small unsigned integers size 32b or less)
+    template <typename _T, typename ReductionOp>
+    __device__ __forceinline__ _T ReduceStep(
+        _T              input,                  ///< [in] Calling thread's input item.
+        ReductionOp     reduction_op,           ///< [in] Binary reduction operator
+        int             last_lane,              ///< [in] Index of last lane in segment
+        int             offset,                 ///< [in] Up-offset to pull from
+        Int2Type<true>  /*is_small_unsigned*/)  ///< [in] Marker type indicating whether T is a small unsigned integer
+    {
+        return ReduceStep(input, reduction_op, last_lane, offset);
+    }
+
+
+    /// Reduction step (specialized for types other than small unsigned integers size 32b or less)
+    template <typename _T, typename ReductionOp>
+    __device__ __forceinline__ _T ReduceStep(
+        _T              input,                  ///< [in] Calling thread's input item.
+        ReductionOp     reduction_op,           ///< [in] Binary reduction operator
+        int             last_lane,              ///< [in] Index of last lane in segment
+        int             offset,                 ///< [in] Up-offset to pull from
+        Int2Type<false> /*is_small_unsigned*/)  ///< [in] Marker type indicating whether T is a small unsigned integer
+    {
+        return ReduceStep(input, reduction_op, last_lane, offset);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Templated inclusive scan iteration
+    //---------------------------------------------------------------------
+
+    template <typename ReductionOp, int STEP>
+    __device__ __forceinline__ void ReduceStep(
+        T&              input,              ///< [in] Calling thread's input item.
+        ReductionOp     reduction_op,       ///< [in] Binary reduction operator
+        int             last_lane,          ///< [in] Index of last lane in segment
+        Int2Type<STEP>  /*step*/)
+    {
+        input = ReduceStep(input, reduction_op, last_lane, 1 << STEP, Int2Type<IsInteger<T>::IS_SMALL_UNSIGNED>());
+
+        ReduceStep(input, reduction_op, last_lane, Int2Type<STEP + 1>());
+    }
+
+    template <typename ReductionOp>
+    __device__ __forceinline__ void ReduceStep(
+        T&              /*input*/,              ///< [in] Calling thread's input item.
+        ReductionOp     /*reduction_op*/,       ///< [in] Binary reduction operator
+        int             /*last_lane*/,          ///< [in] Index of last lane in segment
+        Int2Type<STEPS> /*step*/)
+    {}
+
+
+    //---------------------------------------------------------------------
+    // Reduction operations
+    //---------------------------------------------------------------------
+
+    /// Reduction
+    template <
+        bool            ALL_LANES_VALID,        ///< Whether all lanes in each warp are contributing a valid fold of items
+        typename        ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T               input,                  ///< [in] Calling thread's input
+        int             valid_items,            ///< [in] Total number of valid items across the logical warp
+        ReductionOp     reduction_op)           ///< [in] Binary reduction operator
+    {
+        int last_lane = (ALL_LANES_VALID) ?
+                            LOGICAL_WARP_THREADS - 1 :
+                            valid_items - 1;
+
+        T output = input;
+
+//        // Iterate reduction steps
+//        #pragma unroll
+//        for (int STEP = 0; STEP < STEPS; STEP++)
+//        {
+//            output = ReduceStep(output, reduction_op, last_lane, 1 << STEP, Int2Type<IsInteger<T>::IS_SMALL_UNSIGNED>());
+//        }
+
+        // Template-iterate reduction steps
+        ReduceStep(output, reduction_op, last_lane, Int2Type<0>());
+
+        return output;
+    }
+
+
+    /// Segmented reduction
+    template <
+        bool            HEAD_SEGMENTED,     ///< Whether flags indicate a segment-head or a segment-tail
+        typename        FlagT,
+        typename        ReductionOp>
+    __device__ __forceinline__ T SegmentedReduce(
+        T               input,              ///< [in] Calling thread's input
+        FlagT           flag,               ///< [in] Whether or not the current lane is a segment head/tail
+        ReductionOp     reduction_op)       ///< [in] Binary reduction operator
+    {
+        // Get the start flags for each thread in the warp.
+        int warp_flags = WARP_BALLOT(flag, member_mask);
+
+        // Convert to tail-segmented
+        if (HEAD_SEGMENTED)
+            warp_flags >>= 1;
+
+        // Mask out the bits below the current thread
+        warp_flags &= LaneMaskGe();
+
+        // Mask of physical lanes outside the logical warp and convert to logical lanemask
+        if (!IS_ARCH_WARP)
+        {
+            warp_flags = (warp_flags & member_mask) >> (warp_id * LOGICAL_WARP_THREADS);
+        }
+
+        // Mask in the last lane of logical warp
+        warp_flags |= 1u << (LOGICAL_WARP_THREADS - 1);
+
+        // Find the next set flag
+        int last_lane = __clz(__brev(warp_flags));
+
+        T output = input;
+
+//        // Iterate reduction steps
+//        #pragma unroll
+//        for (int STEP = 0; STEP < STEPS; STEP++)
+//        {
+//            output = ReduceStep(output, reduction_op, last_lane, 1 << STEP, Int2Type<IsInteger<T>::IS_SMALL_UNSIGNED>());
+//        }
+
+        // Template-iterate reduction steps
+        ReduceStep(output, reduction_op, last_lane, Int2Type<0>());
+
+        return output;
+    }
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/warp/specializations/warp_reduce_smem.cuh b/third_party/cub/cub/warp/specializations/warp_reduce_smem.cuh
new file mode 100644
index 0000000..7baa573
--- /dev/null
+++ b/third_party/cub/cub/warp/specializations/warp_reduce_smem.cuh
@@ -0,0 +1,372 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::WarpReduceSmem provides smem-based variants of parallel reduction of items partitioned across a CUDA thread warp.
+ */
+
+#pragma once
+
+#include "../../thread/thread_operators.cuh"
+#include "../../thread/thread_load.cuh"
+#include "../../thread/thread_store.cuh"
+#include "../../util_type.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief WarpReduceSmem provides smem-based variants of parallel reduction of items partitioned across a CUDA thread warp.
+ */
+template <
+    typename    T,                      ///< Data type being reduced
+    int         LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
+    int         PTX_ARCH>               ///< The PTX compute capability for which to to specialize this collective
+struct WarpReduceSmem
+{
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    enum
+    {
+        /// Whether the logical warp size and the PTX warp size coincide
+        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
+
+        /// Whether the logical warp size is a power-of-two
+        IS_POW_OF_TWO = PowerOfTwo<LOGICAL_WARP_THREADS>::VALUE,
+
+        /// The number of warp scan steps
+        STEPS = Log2<LOGICAL_WARP_THREADS>::VALUE,
+
+        /// The number of threads in half a warp
+        HALF_WARP_THREADS = 1 << (STEPS - 1),
+
+        /// The number of shared memory elements per warp
+        WARP_SMEM_ELEMENTS =  LOGICAL_WARP_THREADS + HALF_WARP_THREADS,
+
+        /// FlagT status (when not using ballot)
+        UNSET   = 0x0,  // Is initially unset
+        SET     = 0x1,  // Is initially set
+        SEEN    = 0x2,  // Has seen another head flag from a successor peer
+    };
+
+    /// Shared memory flag type
+    typedef unsigned char SmemFlag;
+
+    /// Shared memory storage layout type (1.5 warps-worth of elements for each warp)
+    struct _TempStorage
+    {
+        T           reduce[WARP_SMEM_ELEMENTS];
+        SmemFlag    flags[WARP_SMEM_ELEMENTS];
+    };
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    _TempStorage    &temp_storage;
+    unsigned int    lane_id;
+    unsigned int    member_mask;
+
+
+    /******************************************************************************
+     * Construction
+     ******************************************************************************/
+
+    /// Constructor
+    __device__ __forceinline__ WarpReduceSmem(
+        TempStorage     &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+
+        lane_id(IS_ARCH_WARP ?
+            LaneId() :
+            LaneId() % LOGICAL_WARP_THREADS),
+
+        member_mask((0xffffffff >> (32 - LOGICAL_WARP_THREADS)) << ((IS_ARCH_WARP || !IS_POW_OF_TWO ) ?
+            0 : // arch-width and non-power-of-two subwarps cannot be tiled with the arch-warp
+            ((LaneId() / LOGICAL_WARP_THREADS) * LOGICAL_WARP_THREADS)))
+    {}
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    //---------------------------------------------------------------------
+    // Regular reduction
+    //---------------------------------------------------------------------
+
+    /**
+     * Reduction step
+     */
+    template <
+        bool                ALL_LANES_VALID,        ///< Whether all lanes in each warp are contributing a valid fold of items
+        typename            ReductionOp,
+        int                 STEP>
+    __device__ __forceinline__ T ReduceStep(
+        T                   input,                  ///< [in] Calling thread's input
+        int                 valid_items,            ///< [in] Total number of valid items across the logical warp
+        ReductionOp         reduction_op,           ///< [in] Reduction operator
+        Int2Type<STEP>      /*step*/)
+    {
+        const int OFFSET = 1 << STEP;
+
+        // Share input through buffer
+        ThreadStore<STORE_VOLATILE>(&temp_storage.reduce[lane_id], input);
+
+        WARP_SYNC(member_mask);
+
+        // Update input if peer_addend is in range
+        if ((ALL_LANES_VALID && IS_POW_OF_TWO) || ((lane_id + OFFSET) < valid_items))
+        {
+            T peer_addend = ThreadLoad<LOAD_VOLATILE>(&temp_storage.reduce[lane_id + OFFSET]);
+            input = reduction_op(input, peer_addend);
+        }
+
+        WARP_SYNC(member_mask);
+
+        return ReduceStep<ALL_LANES_VALID>(input, valid_items, reduction_op, Int2Type<STEP + 1>());
+    }
+
+
+    /**
+     * Reduction step (terminate)
+     */
+    template <
+        bool                ALL_LANES_VALID,            ///< Whether all lanes in each warp are contributing a valid fold of items
+        typename            ReductionOp>
+    __device__ __forceinline__ T ReduceStep(
+        T                   input,                      ///< [in] Calling thread's input
+        int                 valid_items,                ///< [in] Total number of valid items across the logical warp
+        ReductionOp         /*reduction_op*/,           ///< [in] Reduction operator
+        Int2Type<STEPS>     /*step*/)
+    {
+        return input;
+    }
+
+
+    //---------------------------------------------------------------------
+    // Segmented reduction
+    //---------------------------------------------------------------------
+
+
+    /**
+     * Ballot-based segmented reduce
+     */
+    template <
+        bool            HEAD_SEGMENTED,     ///< Whether flags indicate a segment-head or a segment-tail
+        typename        FlagT,
+        typename        ReductionOp>
+    __device__ __forceinline__ T SegmentedReduce(
+        T               input,                  ///< [in] Calling thread's input
+        FlagT           flag,                   ///< [in] Whether or not the current lane is a segment head/tail
+        ReductionOp     reduction_op,           ///< [in] Reduction operator
+        Int2Type<true>  /*has_ballot*/)         ///< [in] Marker type for whether the target arch has ballot functionality
+    {
+        // Get the start flags for each thread in the warp.
+        int warp_flags = WARP_BALLOT(flag, member_mask);
+
+        if (!HEAD_SEGMENTED)
+            warp_flags <<= 1;
+
+        // Keep bits above the current thread.
+        warp_flags &= LaneMaskGt();
+
+        // Accommodate packing of multiple logical warps in a single physical warp
+        if (!IS_ARCH_WARP)
+        {
+            warp_flags >>= (LaneId() / LOGICAL_WARP_THREADS) * LOGICAL_WARP_THREADS;
+        }
+
+        // Find next flag
+        int next_flag = __clz(__brev(warp_flags));
+
+        // Clip the next segment at the warp boundary if necessary
+        if (LOGICAL_WARP_THREADS != 32)
+            next_flag = CUB_MIN(next_flag, LOGICAL_WARP_THREADS);
+
+        #pragma unroll
+        for (int STEP = 0; STEP < STEPS; STEP++)
+        {
+            const int OFFSET = 1 << STEP;
+
+            // Share input into buffer
+            ThreadStore<STORE_VOLATILE>(&temp_storage.reduce[lane_id], input);
+
+            WARP_SYNC(member_mask);
+
+            // Update input if peer_addend is in range
+            if (OFFSET + lane_id < next_flag)
+            {
+                T peer_addend = ThreadLoad<LOAD_VOLATILE>(&temp_storage.reduce[lane_id + OFFSET]);
+                input = reduction_op(input, peer_addend);
+            }
+
+            WARP_SYNC(member_mask);
+        }
+
+        return input;
+    }
+
+
+    /**
+     * Smem-based segmented reduce
+     */
+    template <
+        bool            HEAD_SEGMENTED,     ///< Whether flags indicate a segment-head or a segment-tail
+        typename        FlagT,
+        typename        ReductionOp>
+    __device__ __forceinline__ T SegmentedReduce(
+        T               input,                  ///< [in] Calling thread's input
+        FlagT           flag,                   ///< [in] Whether or not the current lane is a segment head/tail
+        ReductionOp     reduction_op,           ///< [in] Reduction operator
+        Int2Type<false> /*has_ballot*/)         ///< [in] Marker type for whether the target arch has ballot functionality
+    {
+        enum
+        {
+            UNSET   = 0x0,  // Is initially unset
+            SET     = 0x1,  // Is initially set
+            SEEN    = 0x2,  // Has seen another head flag from a successor peer
+        };
+
+        // Alias flags onto shared data storage
+        volatile SmemFlag *flag_storage = temp_storage.flags;
+
+        SmemFlag flag_status = (flag) ? SET : UNSET;
+
+        for (int STEP = 0; STEP < STEPS; STEP++)
+        {
+            const int OFFSET = 1 << STEP;
+
+            // Share input through buffer
+            ThreadStore<STORE_VOLATILE>(&temp_storage.reduce[lane_id], input);
+
+            WARP_SYNC(member_mask);
+
+            // Get peer from buffer
+            T peer_addend = ThreadLoad<LOAD_VOLATILE>(&temp_storage.reduce[lane_id + OFFSET]);
+
+            WARP_SYNC(member_mask);
+
+            // Share flag through buffer
+            flag_storage[lane_id] = flag_status;
+
+            // Get peer flag from buffer
+            SmemFlag peer_flag_status = flag_storage[lane_id + OFFSET];
+
+            // Update input if peer was in range
+            if (lane_id < LOGICAL_WARP_THREADS - OFFSET)
+            {
+                if (HEAD_SEGMENTED)
+                {
+                    // Head-segmented
+                    if ((flag_status & SEEN) == 0)
+                    {
+                        // Has not seen a more distant head flag
+                        if (peer_flag_status & SET)
+                        {
+                            // Has now seen a head flag
+                            flag_status |= SEEN;
+                        }
+                        else
+                        {
+                            // Peer is not a head flag: grab its count
+                            input = reduction_op(input, peer_addend);
+                        }
+
+                        // Update seen status to include that of peer
+                        flag_status |= (peer_flag_status & SEEN);
+                    }
+                }
+                else
+                {
+                    // Tail-segmented.  Simply propagate flag status
+                    if (!flag_status)
+                    {
+                        input = reduction_op(input, peer_addend);
+                        flag_status |= peer_flag_status;
+                    }
+
+                }
+            }
+        }
+
+        return input;
+    }
+
+
+    /******************************************************************************
+     * Interface
+     ******************************************************************************/
+
+    /**
+     * Reduction
+     */
+    template <
+        bool                ALL_LANES_VALID,        ///< Whether all lanes in each warp are contributing a valid fold of items
+        typename            ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T                   input,                  ///< [in] Calling thread's input
+        int                 valid_items,            ///< [in] Total number of valid items across the logical warp
+        ReductionOp         reduction_op)           ///< [in] Reduction operator
+    {
+        return ReduceStep<ALL_LANES_VALID>(input, valid_items, reduction_op, Int2Type<0>());
+    }
+
+
+    /**
+     * Segmented reduction
+     */
+    template <
+        bool            HEAD_SEGMENTED,     ///< Whether flags indicate a segment-head or a segment-tail
+        typename        FlagT,
+        typename        ReductionOp>
+    __device__ __forceinline__ T SegmentedReduce(
+        T               input,              ///< [in] Calling thread's input
+        FlagT            flag,               ///< [in] Whether or not the current lane is a segment head/tail
+        ReductionOp     reduction_op)       ///< [in] Reduction operator
+    {
+        return SegmentedReduce<HEAD_SEGMENTED>(input, flag, reduction_op, Int2Type<(PTX_ARCH >= 200)>());
+    }
+
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/warp/specializations/warp_scan_shfl.cuh b/third_party/cub/cub/warp/specializations/warp_scan_shfl.cuh
new file mode 100644
index 0000000..7f4e1c9
--- /dev/null
+++ b/third_party/cub/cub/warp/specializations/warp_scan_shfl.cuh
@@ -0,0 +1,632 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::WarpScanShfl provides SHFL-based variants of parallel prefix scan of items partitioned across a CUDA thread warp.
+ */
+
+#pragma once
+
+#include "../../thread/thread_operators.cuh"
+#include "../../util_type.cuh"
+#include "../../util_ptx.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief WarpScanShfl provides SHFL-based variants of parallel prefix scan of items partitioned across a CUDA thread warp.
+ *
+ * LOGICAL_WARP_THREADS must be a power-of-two
+ */
+template <
+    typename    T,                      ///< Data type being scanned
+    int         LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
+    int         PTX_ARCH>               ///< The PTX compute capability for which to to specialize this collective
+struct WarpScanShfl
+{
+    //---------------------------------------------------------------------
+    // Constants and type definitions
+    //---------------------------------------------------------------------
+
+    enum
+    {
+        /// Whether the logical warp size and the PTX warp size coincide
+        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
+
+        /// The number of warp scan steps
+        STEPS = Log2<LOGICAL_WARP_THREADS>::VALUE,
+
+        /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
+        SHFL_C = (CUB_WARP_THREADS(PTX_ARCH) - LOGICAL_WARP_THREADS) << 8
+    };
+
+    template <typename S>
+    struct IntegerTraits
+    {
+        enum {
+            ///Whether the data type is a small (32b or less) integer for which we can use a single SFHL instruction per exchange
+            IS_SMALL_UNSIGNED = (Traits<S>::CATEGORY == UNSIGNED_INTEGER) && (sizeof(S) <= sizeof(unsigned int))
+        };
+    };
+
+    /// Shared memory storage layout type
+    struct TempStorage {};
+
+
+    //---------------------------------------------------------------------
+    // Thread fields
+    //---------------------------------------------------------------------
+
+    /// Lane index in logical warp
+    unsigned int lane_id;
+
+    /// Logical warp index in 32-thread physical warp
+    unsigned int warp_id;
+
+    /// 32-thread physical warp member mask of logical warp
+    unsigned int member_mask;
+
+    //---------------------------------------------------------------------
+    // Construction
+    //---------------------------------------------------------------------
+
+    /// Constructor
+    __device__ __forceinline__ WarpScanShfl(
+        TempStorage &/*temp_storage*/)
+    {
+        lane_id = LaneId();
+        warp_id = 0;
+        member_mask = 0xffffffffu >> (CUB_WARP_THREADS(PTX_ARCH) - LOGICAL_WARP_THREADS);
+
+        if (!IS_ARCH_WARP)
+        {
+            warp_id = lane_id / LOGICAL_WARP_THREADS;
+            lane_id = lane_id % LOGICAL_WARP_THREADS;
+            member_mask = member_mask << (warp_id * LOGICAL_WARP_THREADS);
+        }
+    }
+
+
+    //---------------------------------------------------------------------
+    // Inclusive scan steps
+    //---------------------------------------------------------------------
+
+    /// Inclusive prefix scan step (specialized for summation across int32 types)
+    __device__ __forceinline__ int InclusiveScanStep(
+        int             input,              ///< [in] Calling thread's input item.
+        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        int output;
+        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .s32 r0;"
+            "  .reg .pred p;"
+            "  shfl.sync.up.b32 r0|p, %1, %2, %3, %5;"
+            "  @p add.s32 r0, r0, %4;"
+            "  mov.s32 %0, r0;"
+            "}"
+            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .s32 r0;"
+            "  .reg .pred p;"
+            "  shfl.up.b32 r0|p, %1, %2, %3;"
+            "  @p add.s32 r0, r0, %4;"
+            "  mov.s32 %0, r0;"
+            "}"
+            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input));
+#endif
+
+        return output;
+    }
+
+    /// Inclusive prefix scan step (specialized for summation across uint32 types)
+    __device__ __forceinline__ unsigned int InclusiveScanStep(
+        unsigned int    input,              ///< [in] Calling thread's input item.
+        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        unsigned int output;
+        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .u32 r0;"
+            "  .reg .pred p;"
+            "  shfl.sync.up.b32 r0|p, %1, %2, %3, %5;"
+            "  @p add.u32 r0, r0, %4;"
+            "  mov.u32 %0, r0;"
+            "}"
+            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .u32 r0;"
+            "  .reg .pred p;"
+            "  shfl.up.b32 r0|p, %1, %2, %3;"
+            "  @p add.u32 r0, r0, %4;"
+            "  mov.u32 %0, r0;"
+            "}"
+            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input));
+#endif
+
+        return output;
+    }
+
+
+    /// Inclusive prefix scan step (specialized for summation across fp32 types)
+    __device__ __forceinline__ float InclusiveScanStep(
+        float           input,              ///< [in] Calling thread's input item.
+        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        float output;
+        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .f32 r0;"
+            "  .reg .pred p;"
+            "  shfl.sync.up.b32 r0|p, %1, %2, %3, %5;"
+            "  @p add.f32 r0, r0, %4;"
+            "  mov.f32 %0, r0;"
+            "}"
+            : "=f"(output) : "f"(input), "r"(offset), "r"(shfl_c), "f"(input), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .f32 r0;"
+            "  .reg .pred p;"
+            "  shfl.up.b32 r0|p, %1, %2, %3;"
+            "  @p add.f32 r0, r0, %4;"
+            "  mov.f32 %0, r0;"
+            "}"
+            : "=f"(output) : "f"(input), "r"(offset), "r"(shfl_c), "f"(input));
+#endif
+
+        return output;
+    }
+
+
+    /// Inclusive prefix scan step (specialized for summation across unsigned long long types)
+    __device__ __forceinline__ unsigned long long InclusiveScanStep(
+        unsigned long long  input,              ///< [in] Calling thread's input item.
+        cub::Sum            /*scan_op*/,        ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        unsigned long long output;
+        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .u64 r0;"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.sync.up.b32 lo|p, lo, %2, %3, %5;"
+            "  shfl.sync.up.b32 hi|p, hi, %2, %3, %5;"
+            "  mov.b64 r0, {lo, hi};"
+            "  @p add.u64 r0, r0, %4;"
+            "  mov.u64 %0, r0;"
+            "}"
+            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "l"(input), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .u64 r0;"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.up.b32 lo|p, lo, %2, %3;"
+            "  shfl.up.b32 hi|p, hi, %2, %3;"
+            "  mov.b64 r0, {lo, hi};"
+            "  @p add.u64 r0, r0, %4;"
+            "  mov.u64 %0, r0;"
+            "}"
+            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "l"(input));
+#endif
+
+        return output;
+    }
+
+
+    /// Inclusive prefix scan step (specialized for summation across long long types)
+    __device__ __forceinline__ long long InclusiveScanStep(
+        long long       input,              ///< [in] Calling thread's input item.
+        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        long long output;
+        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .s64 r0;"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.sync.up.b32 lo|p, lo, %2, %3, %5;"
+            "  shfl.sync.up.b32 hi|p, hi, %2, %3, %5;"
+            "  mov.b64 r0, {lo, hi};"
+            "  @p add.s64 r0, r0, %4;"
+            "  mov.s64 %0, r0;"
+            "}"
+            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "l"(input), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .s64 r0;"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.up.b32 lo|p, lo, %2, %3;"
+            "  shfl.up.b32 hi|p, hi, %2, %3;"
+            "  mov.b64 r0, {lo, hi};"
+            "  @p add.s64 r0, r0, %4;"
+            "  mov.s64 %0, r0;"
+            "}"
+            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "l"(input));
+#endif
+
+        return output;
+    }
+
+
+    /// Inclusive prefix scan step (specialized for summation across fp64 types)
+    __device__ __forceinline__ double InclusiveScanStep(
+        double          input,              ///< [in] Calling thread's input item.
+        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        double output;
+        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
+
+        // Use predicate set from SHFL to guard against invalid peers
+#ifdef CUB_USE_COOPERATIVE_GROUPS
+        asm volatile(
+            "{"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  .reg .f64 r0;"
+            "  mov.b64 %0, %1;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.sync.up.b32 lo|p, lo, %2, %3, %4;"
+            "  shfl.sync.up.b32 hi|p, hi, %2, %3, %4;"
+            "  mov.b64 r0, {lo, hi};"
+            "  @p add.f64 %0, %0, r0;"
+            "}"
+            : "=d"(output) : "d"(input), "r"(offset), "r"(shfl_c), "r"(member_mask));
+#else
+        asm volatile(
+            "{"
+            "  .reg .u32 lo;"
+            "  .reg .u32 hi;"
+            "  .reg .pred p;"
+            "  .reg .f64 r0;"
+            "  mov.b64 %0, %1;"
+            "  mov.b64 {lo, hi}, %1;"
+            "  shfl.up.b32 lo|p, lo, %2, %3;"
+            "  shfl.up.b32 hi|p, hi, %2, %3;"
+            "  mov.b64 r0, {lo, hi};"
+            "  @p add.f64 %0, %0, r0;"
+            "}"
+            : "=d"(output) : "d"(input), "r"(offset), "r"(shfl_c));
+#endif
+
+        return output;
+    }
+
+
+/*
+    /// Inclusive prefix scan (specialized for ReduceBySegmentOp<cub::Sum> across KeyValuePair<OffsetT, Value> types)
+    template <typename Value, typename OffsetT>
+    __device__ __forceinline__ KeyValuePair<OffsetT, Value>InclusiveScanStep(
+        KeyValuePair<OffsetT, Value>    input,              ///< [in] Calling thread's input item.
+        ReduceBySegmentOp<cub::Sum>     scan_op,            ///< [in] Binary scan operator
+        int                             first_lane,         ///< [in] Index of first lane in segment
+        int                             offset)             ///< [in] Up-offset to pull from
+    {
+        KeyValuePair<OffsetT, Value> output;
+
+        output.value = InclusiveScanStep(input.value, cub::Sum(), first_lane, offset, Int2Type<IntegerTraits<Value>::IS_SMALL_UNSIGNED>());
+        output.key = InclusiveScanStep(input.key, cub::Sum(), first_lane, offset, Int2Type<IntegerTraits<OffsetT>::IS_SMALL_UNSIGNED>());
+
+        if (input.key > 0)
+            output.value = input.value;
+
+        return output;
+    }
+*/
+
+    /// Inclusive prefix scan step (generic)
+    template <typename _T, typename ScanOpT>
+    __device__ __forceinline__ _T InclusiveScanStep(
+        _T              input,              ///< [in] Calling thread's input item.
+        ScanOpT         scan_op,            ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset)             ///< [in] Up-offset to pull from
+    {
+        _T temp = ShuffleUp<LOGICAL_WARP_THREADS>(input, offset, first_lane, member_mask);
+
+        // Perform scan op if from a valid peer
+        _T output = scan_op(temp, input);
+        if (static_cast<int>(lane_id) < first_lane + offset)
+            output = input;
+
+        return output;
+    }
+
+
+    /// Inclusive prefix scan step (specialized for small integers size 32b or less)
+    template <typename _T, typename ScanOpT>
+    __device__ __forceinline__ _T InclusiveScanStep(
+        _T              input,              ///< [in] Calling thread's input item.
+        ScanOpT         scan_op,            ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset,             ///< [in] Up-offset to pull from
+        Int2Type<true>  /*is_small_unsigned*/)  ///< [in] Marker type indicating whether T is a small integer
+    {
+        return InclusiveScanStep(input, scan_op, first_lane, offset);
+    }
+
+
+    /// Inclusive prefix scan step (specialized for types other than small integers size 32b or less)
+    template <typename _T, typename ScanOpT>
+    __device__ __forceinline__ _T InclusiveScanStep(
+        _T              input,              ///< [in] Calling thread's input item.
+        ScanOpT          scan_op,            ///< [in] Binary scan operator
+        int             first_lane,         ///< [in] Index of first lane in segment
+        int             offset,             ///< [in] Up-offset to pull from
+        Int2Type<false> /*is_small_unsigned*/)  ///< [in] Marker type indicating whether T is a small integer
+    {
+        return InclusiveScanStep(input, scan_op, first_lane, offset);
+    }
+
+
+    /******************************************************************************
+     * Interface
+     ******************************************************************************/
+
+    //---------------------------------------------------------------------
+    // Broadcast
+    //---------------------------------------------------------------------
+
+    /// Broadcast
+    __device__ __forceinline__ T Broadcast(
+        T               input,              ///< [in] The value to broadcast
+        int             src_lane)           ///< [in] Which warp lane is to do the broadcasting
+    {
+        return ShuffleIndex<LOGICAL_WARP_THREADS>(input, src_lane, member_mask);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Inclusive operations
+    //---------------------------------------------------------------------
+
+    /// Inclusive scan
+    template <typename _T, typename ScanOpT>
+    __device__ __forceinline__ void InclusiveScan(
+        _T              input,              ///< [in] Calling thread's input item.
+        _T              &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOpT         scan_op)            ///< [in] Binary scan operator
+    {
+        inclusive_output = input;
+
+        // Iterate scan steps
+        int segment_first_lane = 0;
+
+        // Iterate scan steps
+        #pragma unroll
+        for (int STEP = 0; STEP < STEPS; STEP++)
+        {
+            inclusive_output = InclusiveScanStep(
+                inclusive_output,
+                scan_op,
+                segment_first_lane,
+                (1 << STEP),
+                Int2Type<IntegerTraits<T>::IS_SMALL_UNSIGNED>());
+        }
+
+    }
+
+    /// Inclusive scan, specialized for reduce-value-by-key
+    template <typename KeyT, typename ValueT, typename ReductionOpT>
+    __device__ __forceinline__ void InclusiveScan(
+        KeyValuePair<KeyT, ValueT>      input,              ///< [in] Calling thread's input item.
+        KeyValuePair<KeyT, ValueT>      &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ReduceByKeyOp<ReductionOpT >    scan_op)            ///< [in] Binary scan operator
+    {
+        inclusive_output = input;
+
+        KeyT pred_key = ShuffleUp<LOGICAL_WARP_THREADS>(inclusive_output.key, 1, 0, member_mask);
+
+        unsigned int ballot = WARP_BALLOT((pred_key != inclusive_output.key), member_mask);
+
+        // Mask away all lanes greater than ours
+        ballot = ballot & LaneMaskLe();
+
+        // Find index of first set bit
+        int segment_first_lane = CUB_MAX(0, 31 - __clz(ballot));
+
+        // Iterate scan steps
+        #pragma unroll
+        for (int STEP = 0; STEP < STEPS; STEP++)
+        {
+            inclusive_output.value = InclusiveScanStep(
+                inclusive_output.value,
+                scan_op.op,
+                segment_first_lane,
+                (1 << STEP),
+                Int2Type<IntegerTraits<T>::IS_SMALL_UNSIGNED>());
+        }
+    }
+
+
+    /// Inclusive scan with aggregate
+    template <typename ScanOpT>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOpT         scan_op,            ///< [in] Binary scan operator
+        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
+    {
+        InclusiveScan(input, inclusive_output, scan_op);
+
+        // Grab aggregate from last warp lane
+        warp_aggregate = ShuffleIndex<LOGICAL_WARP_THREADS>(inclusive_output, LOGICAL_WARP_THREADS - 1, member_mask);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Get exclusive from inclusive
+    //---------------------------------------------------------------------
+
+    /// Update inclusive and exclusive using input and inclusive
+    template <typename ScanOpT, typename IsIntegerT>
+    __device__ __forceinline__ void Update(
+        T                       /*input*/,          ///< [in]
+        T                       &inclusive,         ///< [in, out]
+        T                       &exclusive,         ///< [out]
+        ScanOpT                 /*scan_op*/,        ///< [in]
+        IsIntegerT              /*is_integer*/)     ///< [in]
+    {
+        // initial value unknown
+        exclusive = ShuffleUp<LOGICAL_WARP_THREADS>(inclusive, 1, 0, member_mask);
+    }
+
+    /// Update inclusive and exclusive using input and inclusive (specialized for summation of integer types)
+    __device__ __forceinline__ void Update(
+        T                       input,
+        T                       &inclusive,
+        T                       &exclusive,
+        cub::Sum                /*scan_op*/,
+        Int2Type<true>          /*is_integer*/)
+    {
+        // initial value presumed 0
+        exclusive = inclusive - input;
+    }
+
+    /// Update inclusive and exclusive using initial value using input, inclusive, and initial value
+    template <typename ScanOpT, typename IsIntegerT>
+    __device__ __forceinline__ void Update (
+        T                       /*input*/,
+        T                       &inclusive,
+        T                       &exclusive,
+        ScanOpT                 scan_op,
+        T                       initial_value,
+        IsIntegerT              /*is_integer*/)
+    {
+        inclusive = scan_op(initial_value, inclusive);
+        exclusive = ShuffleUp<LOGICAL_WARP_THREADS>(inclusive, 1, 0, member_mask);
+
+        if (lane_id == 0)
+            exclusive = initial_value;
+    }
+
+    /// Update inclusive and exclusive using initial value using input and inclusive (specialized for summation of integer types)
+    __device__ __forceinline__ void Update (
+        T                       input,
+        T                       &inclusive,
+        T                       &exclusive,
+        cub::Sum                scan_op,
+        T                       initial_value,
+        Int2Type<true>          /*is_integer*/)
+    {
+        inclusive = scan_op(initial_value, inclusive);
+        exclusive = inclusive - input;
+    }
+
+
+    /// Update inclusive, exclusive, and warp aggregate using input and inclusive
+    template <typename ScanOpT, typename IsIntegerT>
+    __device__ __forceinline__ void Update (
+        T                       input,
+        T                       &inclusive,
+        T                       &exclusive,
+        T                       &warp_aggregate,
+        ScanOpT                 scan_op,
+        IsIntegerT              is_integer)
+    {
+        warp_aggregate = ShuffleIndex<LOGICAL_WARP_THREADS>(inclusive, LOGICAL_WARP_THREADS - 1, member_mask);
+        Update(input, inclusive, exclusive, scan_op, is_integer);
+    }
+
+    /// Update inclusive, exclusive, and warp aggregate using input, inclusive, and initial value
+    template <typename ScanOpT, typename IsIntegerT>
+    __device__ __forceinline__ void Update (
+        T                       input,
+        T                       &inclusive,
+        T                       &exclusive,
+        T                       &warp_aggregate,
+        ScanOpT                 scan_op,
+        T                       initial_value,
+        IsIntegerT              is_integer)
+    {
+        warp_aggregate = ShuffleIndex<LOGICAL_WARP_THREADS>(inclusive, LOGICAL_WARP_THREADS - 1, member_mask);
+        Update(input, inclusive, exclusive, scan_op, initial_value, is_integer);
+    }
+
+
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/warp/specializations/warp_scan_smem.cuh b/third_party/cub/cub/warp/specializations/warp_scan_smem.cuh
new file mode 100644
index 0000000..3237fcb
--- /dev/null
+++ b/third_party/cub/cub/warp/specializations/warp_scan_smem.cuh
@@ -0,0 +1,397 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * cub::WarpScanSmem provides smem-based variants of parallel prefix scan of items partitioned across a CUDA thread warp.
+ */
+
+#pragma once
+
+#include "../../thread/thread_operators.cuh"
+#include "../../thread/thread_load.cuh"
+#include "../../thread/thread_store.cuh"
+#include "../../util_type.cuh"
+#include "../../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \brief WarpScanSmem provides smem-based variants of parallel prefix scan of items partitioned across a CUDA thread warp.
+ */
+template <
+    typename    T,                      ///< Data type being scanned
+    int         LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
+    int         PTX_ARCH>               ///< The PTX compute capability for which to to specialize this collective
+struct WarpScanSmem
+{
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    enum
+    {
+        /// Whether the logical warp size and the PTX warp size coincide
+        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
+
+        /// Whether the logical warp size is a power-of-two
+        IS_POW_OF_TWO = PowerOfTwo<LOGICAL_WARP_THREADS>::VALUE,
+
+        /// The number of warp scan steps
+        STEPS = Log2<LOGICAL_WARP_THREADS>::VALUE,
+
+        /// The number of threads in half a warp
+        HALF_WARP_THREADS = 1 << (STEPS - 1),
+
+        /// The number of shared memory elements per warp
+        WARP_SMEM_ELEMENTS =  LOGICAL_WARP_THREADS + HALF_WARP_THREADS,
+    };
+
+    /// Storage cell type (workaround for SM1x compiler bugs with custom-ops like Max() on signed chars)
+    typedef typename If<((Equals<T, char>::VALUE || Equals<T, signed char>::VALUE) && (PTX_ARCH < 200)), int, T>::Type CellT;
+
+    /// Shared memory storage layout type (1.5 warps-worth of elements for each warp)
+    typedef CellT _TempStorage[WARP_SMEM_ELEMENTS];
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    _TempStorage    &temp_storage;
+    unsigned int    lane_id;
+    unsigned int    member_mask;
+
+
+    /******************************************************************************
+     * Construction
+     ******************************************************************************/
+
+    /// Constructor
+    __device__ __forceinline__ WarpScanSmem(
+        TempStorage     &temp_storage)
+    :
+        temp_storage(temp_storage.Alias()),
+
+        lane_id(IS_ARCH_WARP ?
+            LaneId() :
+            LaneId() % LOGICAL_WARP_THREADS),
+
+        member_mask((0xffffffff >> (32 - LOGICAL_WARP_THREADS)) << ((IS_ARCH_WARP || !IS_POW_OF_TWO ) ?
+            0 : // arch-width and non-power-of-two subwarps cannot be tiled with the arch-warp
+            ((LaneId() / LOGICAL_WARP_THREADS) * LOGICAL_WARP_THREADS)))
+    {}
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+    /// Basic inclusive scan iteration (template unrolled, inductive-case specialization)
+    template <
+        bool        HAS_IDENTITY,
+        int         STEP,
+        typename    ScanOp>
+    __device__ __forceinline__ void ScanStep(
+        T                       &partial,
+        ScanOp                  scan_op,
+        Int2Type<STEP>          /*step*/)
+    {
+        const int OFFSET = 1 << STEP;
+
+        // Share partial into buffer
+        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) partial);
+
+        WARP_SYNC(member_mask);
+
+        // Update partial if addend is in range
+        if (HAS_IDENTITY || (lane_id >= OFFSET))
+        {
+            T addend = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - OFFSET]);
+            partial = scan_op(addend, partial);
+        }
+        WARP_SYNC(member_mask);
+
+        ScanStep<HAS_IDENTITY>(partial, scan_op, Int2Type<STEP + 1>());
+    }
+
+
+    /// Basic inclusive scan iteration(template unrolled, base-case specialization)
+    template <
+        bool        HAS_IDENTITY,
+        typename    ScanOp>
+    __device__ __forceinline__ void ScanStep(
+        T                       &/*partial*/,
+        ScanOp                  /*scan_op*/,
+        Int2Type<STEPS>         /*step*/)
+    {}
+
+
+    /// Inclusive prefix scan (specialized for summation across primitive types)
+    __device__ __forceinline__ void InclusiveScan(
+        T                       input,              ///< [in] Calling thread's input item.
+        T                       &output,            ///< [out] Calling thread's output item.  May be aliased with \p input.
+        Sum                     scan_op,            ///< [in] Binary scan operator
+        Int2Type<true>          /*is_primitive*/)   ///< [in] Marker type indicating whether T is primitive type
+    {
+        T identity = 0;
+        ThreadStore<STORE_VOLATILE>(&temp_storage[lane_id], (CellT) identity);
+
+        WARP_SYNC(member_mask);
+
+        // Iterate scan steps
+        output = input;
+        ScanStep<true>(output, scan_op, Int2Type<0>());
+    }
+
+
+    /// Inclusive prefix scan
+    template <typename ScanOp, int IS_PRIMITIVE>
+    __device__ __forceinline__ void InclusiveScan(
+        T                       input,              ///< [in] Calling thread's input item.
+        T                       &output,            ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOp                  scan_op,            ///< [in] Binary scan operator
+        Int2Type<IS_PRIMITIVE>  /*is_primitive*/)   ///< [in] Marker type indicating whether T is primitive type
+    {
+        // Iterate scan steps
+        output = input;
+        ScanStep<false>(output, scan_op, Int2Type<0>());
+    }
+
+
+    /******************************************************************************
+     * Interface
+     ******************************************************************************/
+
+    //---------------------------------------------------------------------
+    // Broadcast
+    //---------------------------------------------------------------------
+
+    /// Broadcast
+    __device__ __forceinline__ T Broadcast(
+        T               input,              ///< [in] The value to broadcast
+        unsigned int    src_lane)           ///< [in] Which warp lane is to do the broadcasting
+    {
+        if (lane_id == src_lane)
+        {
+            ThreadStore<STORE_VOLATILE>(temp_storage, (CellT) input);
+        }
+
+        WARP_SYNC(member_mask);
+
+        return (T)ThreadLoad<LOAD_VOLATILE>(temp_storage);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Inclusive operations
+    //---------------------------------------------------------------------
+
+    /// Inclusive scan
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        InclusiveScan(input, inclusive_output, scan_op, Int2Type<Traits<T>::PRIMITIVE>());
+    }
+
+
+    /// Inclusive scan with aggregate
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
+    {
+        InclusiveScan(input, inclusive_output, scan_op);
+
+        // Retrieve aggregate
+        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive_output);
+
+        WARP_SYNC(member_mask);
+
+        warp_aggregate = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[WARP_SMEM_ELEMENTS - 1]);
+
+        WARP_SYNC(member_mask);
+    }
+
+
+    //---------------------------------------------------------------------
+    // Get exclusive from inclusive
+    //---------------------------------------------------------------------
+
+    /// Update inclusive and exclusive using input and inclusive
+    template <typename ScanOpT, typename IsIntegerT>
+    __device__ __forceinline__ void Update(
+        T                       /*input*/,      ///< [in]
+        T                       &inclusive,     ///< [in, out]
+        T                       &exclusive,     ///< [out]
+        ScanOpT                 /*scan_op*/,    ///< [in]
+        IsIntegerT              /*is_integer*/) ///< [in]
+    {
+        // initial value unknown
+        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
+
+        WARP_SYNC(member_mask);
+
+        exclusive = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 1]);
+    }
+
+    /// Update inclusive and exclusive using input and inclusive (specialized for summation of integer types)
+    __device__ __forceinline__ void Update(
+        T                       input,
+        T                       &inclusive,
+        T                       &exclusive,
+        cub::Sum                /*scan_op*/,
+        Int2Type<true>          /*is_integer*/)
+    {
+        // initial value presumed 0
+        exclusive = inclusive - input;
+    }
+
+    /// Update inclusive and exclusive using initial value using input, inclusive, and initial value
+    template <typename ScanOpT, typename IsIntegerT>
+    __device__ __forceinline__ void Update (
+        T                       /*input*/,
+        T                       &inclusive,
+        T                       &exclusive,
+        ScanOpT                 scan_op,
+        T                       initial_value,
+        IsIntegerT              /*is_integer*/)
+    {
+        inclusive = scan_op(initial_value, inclusive);
+        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
+
+        WARP_SYNC(member_mask);
+
+        exclusive = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 1]);
+        if (lane_id == 0)
+            exclusive = initial_value;
+    }
+
+    /// Update inclusive and exclusive using initial value using input and inclusive (specialized for summation of integer types)
+    __device__ __forceinline__ void Update (
+        T                       input,
+        T                       &inclusive,
+        T                       &exclusive,
+        cub::Sum                scan_op,
+        T                       initial_value,
+        Int2Type<true>          /*is_integer*/)
+    {
+        inclusive = scan_op(initial_value, inclusive);
+        exclusive = inclusive - input;
+    }
+
+
+    /// Update inclusive, exclusive, and warp aggregate using input and inclusive
+    template <typename ScanOpT, typename IsIntegerT>
+    __device__ __forceinline__ void Update (
+        T                       /*input*/,
+        T                       &inclusive,
+        T                       &exclusive,
+        T                       &warp_aggregate,
+        ScanOpT                 /*scan_op*/,
+        IsIntegerT              /*is_integer*/)
+    {
+        // Initial value presumed to be unknown or identity (either way our padding is correct)
+        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
+
+        WARP_SYNC(member_mask);
+
+        exclusive = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 1]);
+        warp_aggregate = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[WARP_SMEM_ELEMENTS - 1]);
+    }
+
+    /// Update inclusive, exclusive, and warp aggregate using input and inclusive (specialized for summation of integer types)
+    __device__ __forceinline__ void Update (
+        T                       input,
+        T                       &inclusive,
+        T                       &exclusive,
+        T                       &warp_aggregate,
+        cub::Sum                /*scan_o*/,
+        Int2Type<true>          /*is_integer*/)
+    {
+        // Initial value presumed to be unknown or identity (either way our padding is correct)
+        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
+
+        WARP_SYNC(member_mask);
+
+        warp_aggregate = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[WARP_SMEM_ELEMENTS - 1]);
+        exclusive = inclusive - input;
+    }
+
+    /// Update inclusive, exclusive, and warp aggregate using input, inclusive, and initial value
+    template <typename ScanOpT, typename IsIntegerT>
+    __device__ __forceinline__ void Update (
+        T                       /*input*/,
+        T                       &inclusive,
+        T                       &exclusive,
+        T                       &warp_aggregate,
+        ScanOpT                 scan_op,
+        T                       initial_value,
+        IsIntegerT              /*is_integer*/)
+    {
+        // Broadcast warp aggregate
+        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
+
+        WARP_SYNC(member_mask);
+
+        warp_aggregate = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[WARP_SMEM_ELEMENTS - 1]);
+
+        WARP_SYNC(member_mask);
+
+        // Update inclusive with initial value
+        inclusive = scan_op(initial_value, inclusive);
+
+        // Get exclusive from exclusive
+        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 1], (CellT) inclusive);
+
+        WARP_SYNC(member_mask);
+
+        exclusive = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 2]);
+
+        if (lane_id == 0)
+            exclusive = initial_value;
+    }
+
+
+};
+
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/warp/warp_reduce.cuh b/third_party/cub/cub/warp/warp_reduce.cuh
new file mode 100644
index 0000000..189896b
--- /dev/null
+++ b/third_party/cub/cub/warp/warp_reduce.cuh
@@ -0,0 +1,612 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::WarpReduce class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread warp.
+ */
+
+#pragma once
+
+#include "specializations/warp_reduce_shfl.cuh"
+#include "specializations/warp_reduce_smem.cuh"
+#include "../thread/thread_operators.cuh"
+#include "../util_arch.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+
+/**
+ * \addtogroup WarpModule
+ * @{
+ */
+
+/**
+ * \brief The WarpReduce class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread warp. ![](warp_reduce_logo.png)
+ *
+ * \tparam T                        The reduction input/output element type
+ * \tparam LOGICAL_WARP_THREADS     <b>[optional]</b> The number of threads per "logical" warp (may be less than the number of hardware warp threads).  Default is the warp size of the targeted CUDA compute-capability (e.g., 32 threads for SM20).
+ * \tparam PTX_ARCH                 <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
+ *   uses a binary combining operator to compute a single aggregate from a list of input elements.
+ * - Supports "logical" warps smaller than the physical warp size (e.g., logical warps of 8 threads)
+ * - The number of entrant threads must be an multiple of \p LOGICAL_WARP_THREADS
+ *
+ * \par Performance Considerations
+ * - Uses special instructions when applicable (e.g., warp \p SHFL instructions)
+ * - Uses synchronization-free communication between warp lanes when applicable
+ * - Incurs zero bank conflicts for most types
+ * - Computation is slightly more efficient (i.e., having lower instruction overhead) for:
+ *     - Summation (<b><em>vs.</em></b> generic reduction)
+ *     - The architecture's warp size is a whole multiple of \p LOGICAL_WARP_THREADS
+ *
+ * \par Simple Examples
+ * \warpcollective{WarpReduce}
+ * \par
+ * The code snippet below illustrates four concurrent warp sum reductions within a block of
+ * 128 threads (one per each of the 32-thread warps).
+ * \par
+ * \code
+ * #include <cub/cub.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize WarpReduce for type int
+ *     typedef cub::WarpReduce<int> WarpReduce;
+ *
+ *     // Allocate WarpReduce shared memory for 4 warps
+ *     __shared__ typename WarpReduce::TempStorage temp_storage[4];
+ *
+ *     // Obtain one input item per thread
+ *     int thread_data = ...
+ *
+ *     // Return the warp-wide sums to each lane0 (threads 0, 32, 64, and 96)
+ *     int warp_id = threadIdx.x / 32;
+ *     int aggregate = WarpReduce(temp_storage[warp_id]).Sum(thread_data);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the block of threads is <tt>{0, 1, 2, 3, ..., 127}</tt>.
+ * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 496, \p 1520,
+ * \p 2544, and \p 3568, respectively (and is undefined in other threads).
+ *
+ * \par
+ * The code snippet below illustrates a single warp sum reduction within a block of
+ * 128 threads.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize WarpReduce for type int
+ *     typedef cub::WarpReduce<int> WarpReduce;
+ *
+ *     // Allocate WarpReduce shared memory for one warp
+ *     __shared__ typename WarpReduce::TempStorage temp_storage;
+ *     ...
+ *
+ *     // Only the first warp performs a reduction
+ *     if (threadIdx.x < 32)
+ *     {
+ *         // Obtain one input item per thread
+ *         int thread_data = ...
+ *
+ *         // Return the warp-wide sum to lane0
+ *         int aggregate = WarpReduce(temp_storage).Sum(thread_data);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the warp of threads is <tt>{0, 1, 2, 3, ..., 31}</tt>.
+ * The corresponding output \p aggregate in thread0 will be \p 496 (and is undefined in other threads).
+ *
+ */
+template <
+    typename    T,
+    int         LOGICAL_WARP_THREADS    = CUB_PTX_WARP_THREADS,
+    int         PTX_ARCH                = CUB_PTX_ARCH>
+class WarpReduce
+{
+private:
+
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    enum
+    {
+        /// Whether the logical warp size and the PTX warp size coincide
+        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
+
+        /// Whether the logical warp size is a power-of-two
+        IS_POW_OF_TWO = PowerOfTwo<LOGICAL_WARP_THREADS>::VALUE,
+    };
+
+public:
+
+    #ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
+
+    /// Internal specialization.  Use SHFL-based reduction if (architecture is >= SM30) and (LOGICAL_WARP_THREADS is a power-of-two)
+    typedef typename If<(PTX_ARCH >= 300) && (IS_POW_OF_TWO),
+        WarpReduceShfl<T, LOGICAL_WARP_THREADS, PTX_ARCH>,
+        WarpReduceSmem<T, LOGICAL_WARP_THREADS, PTX_ARCH> >::Type InternalWarpReduce;
+
+    #endif // DOXYGEN_SHOULD_SKIP_THIS
+
+
+private:
+
+    /// Shared memory storage layout type for WarpReduce
+    typedef typename InternalWarpReduce::TempStorage _TempStorage;
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage &temp_storage;
+
+
+    /******************************************************************************
+     * Utility methods
+     ******************************************************************************/
+
+public:
+
+    /// \smemstorage{WarpReduce}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.  Logical warp and lane identifiers are constructed from <tt>threadIdx.x</tt>.
+     */
+    __device__ __forceinline__ WarpReduce(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias())
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Summation reductions
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes a warp-wide sum in the calling warp.  The output is valid in warp <em>lane</em><sub>0</sub>.
+     *
+     * \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp sum reductions within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpReduce for type int
+     *     typedef cub::WarpReduce<int> WarpReduce;
+     *
+     *     // Allocate WarpReduce shared memory for 4 warps
+     *     __shared__ typename WarpReduce::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Return the warp-wide sums to each lane0
+     *     int warp_id = threadIdx.x / 32;
+     *     int aggregate = WarpReduce(temp_storage[warp_id]).Sum(thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, 1, 2, 3, ..., 127}</tt>.
+     * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 496, \p 1520,
+     * \p 2544, and \p 3568, respectively (and is undefined in other threads).
+     *
+     */
+    __device__ __forceinline__ T Sum(
+        T                   input)              ///< [in] Calling thread's input
+    {
+        return InternalWarpReduce(temp_storage).template Reduce<true>(input, LOGICAL_WARP_THREADS, cub::Sum());
+    }
+
+    /**
+     * \brief Computes a partially-full warp-wide sum in the calling warp.  The output is valid in warp <em>lane</em><sub>0</sub>.
+     *
+     * All threads across the calling warp must agree on the same value for \p valid_items.  Otherwise the result is undefined.
+     *
+     * \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a sum reduction within a single, partially-full
+     * block of 32 threads (one warp).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, int valid_items)
+     * {
+     *     // Specialize WarpReduce for type int
+     *     typedef cub::WarpReduce<int> WarpReduce;
+     *
+     *     // Allocate WarpReduce shared memory for one warp
+     *     __shared__ typename WarpReduce::TempStorage temp_storage;
+     *
+     *     // Obtain one input item per thread if in range
+     *     int thread_data;
+     *     if (threadIdx.x < valid_items)
+     *         thread_data = d_data[threadIdx.x];
+     *
+     *     // Return the warp-wide sums to each lane0
+     *     int aggregate = WarpReduce(temp_storage).Sum(
+     *         thread_data, valid_items);
+     *
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>{0, 1, 2, 3, 4, ...</tt> and \p valid_items
+     * is \p 4.  The corresponding output \p aggregate in thread0 is \p 6 (and is
+     * undefined in other threads).
+     *
+     */
+    __device__ __forceinline__ T Sum(
+        T                   input,              ///< [in] Calling thread's input
+        int                 valid_items)        ///< [in] Total number of valid items in the calling thread's logical warp (may be less than \p LOGICAL_WARP_THREADS)
+    {
+        // Determine if we don't need bounds checking
+        return InternalWarpReduce(temp_storage).template Reduce<false>(input, valid_items, cub::Sum());
+    }
+
+
+    /**
+     * \brief Computes a segmented sum in the calling warp where segments are defined by head-flags.  The sum of each segment is returned to the first lane in that segment (which always includes <em>lane</em><sub>0</sub>).
+     *
+     * \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a head-segmented warp sum
+     * reduction within a block of 32 threads (one warp).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpReduce for type int
+     *     typedef cub::WarpReduce<int> WarpReduce;
+     *
+     *     // Allocate WarpReduce shared memory for one warp
+     *     __shared__ typename WarpReduce::TempStorage temp_storage;
+     *
+     *     // Obtain one input item and flag per thread
+     *     int thread_data = ...
+     *     int head_flag = ...
+     *
+     *     // Return the warp-wide sums to each lane0
+     *     int aggregate = WarpReduce(temp_storage).HeadSegmentedSum(
+     *         thread_data, head_flag);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data and \p head_flag across the block of threads
+     * is <tt>{0, 1, 2, 3, ..., 31</tt> and is <tt>{1, 0, 0, 0, 1, 0, 0, 0, ..., 1, 0, 0, 0</tt>,
+     * respectively.  The corresponding output \p aggregate in threads 0, 4, 8, etc. will be
+     * \p 6, \p 22, \p 38, etc. (and is undefined in other threads).
+     *
+     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     *
+     */
+    template <
+        typename            FlagT>
+    __device__ __forceinline__ T HeadSegmentedSum(
+        T                   input,              ///< [in] Calling thread's input
+        FlagT                head_flag)          ///< [in] Head flag denoting whether or not \p input is the start of a new segment
+    {
+        return HeadSegmentedReduce(input, head_flag, cub::Sum());
+    }
+
+
+    /**
+     * \brief Computes a segmented sum in the calling warp where segments are defined by tail-flags.  The sum of each segment is returned to the first lane in that segment (which always includes <em>lane</em><sub>0</sub>).
+     *
+     * \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a tail-segmented warp sum
+     * reduction within a block of 32 threads (one warp).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpReduce for type int
+     *     typedef cub::WarpReduce<int> WarpReduce;
+     *
+     *     // Allocate WarpReduce shared memory for one warp
+     *     __shared__ typename WarpReduce::TempStorage temp_storage;
+     *
+     *     // Obtain one input item and flag per thread
+     *     int thread_data = ...
+     *     int tail_flag = ...
+     *
+     *     // Return the warp-wide sums to each lane0
+     *     int aggregate = WarpReduce(temp_storage).TailSegmentedSum(
+     *         thread_data, tail_flag);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data and \p tail_flag across the block of threads
+     * is <tt>{0, 1, 2, 3, ..., 31</tt> and is <tt>{0, 0, 0, 1, 0, 0, 0, 1, ..., 0, 0, 0, 1</tt>,
+     * respectively.  The corresponding output \p aggregate in threads 0, 4, 8, etc. will be
+     * \p 6, \p 22, \p 38, etc. (and is undefined in other threads).
+     *
+     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        typename            FlagT>
+    __device__ __forceinline__ T TailSegmentedSum(
+        T                   input,              ///< [in] Calling thread's input
+        FlagT                tail_flag)          ///< [in] Head flag denoting whether or not \p input is the start of a new segment
+    {
+        return TailSegmentedReduce(input, tail_flag, cub::Sum());
+    }
+
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Generic reductions
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Computes a warp-wide reduction in the calling warp using the specified binary reduction functor.  The output is valid in warp <em>lane</em><sub>0</sub>.
+     *
+     * Supports non-commutative reduction operators
+     *
+     * \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp max reductions within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpReduce for type int
+     *     typedef cub::WarpReduce<int> WarpReduce;
+     *
+     *     // Allocate WarpReduce shared memory for 4 warps
+     *     __shared__ typename WarpReduce::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Return the warp-wide reductions to each lane0
+     *     int warp_id = threadIdx.x / 32;
+     *     int aggregate = WarpReduce(temp_storage[warp_id]).Reduce(
+     *         thread_data, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, 1, 2, 3, ..., 127}</tt>.
+     * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 31, \p 63,
+     * \p 95, and \p 127, respectively  (and is undefined in other threads).
+     *
+     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T                   input,              ///< [in] Calling thread's input
+        ReductionOp         reduction_op)       ///< [in] Binary reduction operator
+    {
+        return InternalWarpReduce(temp_storage).template Reduce<true>(input, LOGICAL_WARP_THREADS, reduction_op);
+    }
+
+    /**
+     * \brief Computes a partially-full warp-wide reduction in the calling warp using the specified binary reduction functor.  The output is valid in warp <em>lane</em><sub>0</sub>.
+     *
+     * All threads across the calling warp must agree on the same value for \p valid_items.  Otherwise the result is undefined.
+     *
+     * Supports non-commutative reduction operators
+     *
+     * \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a max reduction within a single, partially-full
+     * block of 32 threads (one warp).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(int *d_data, int valid_items)
+     * {
+     *     // Specialize WarpReduce for type int
+     *     typedef cub::WarpReduce<int> WarpReduce;
+     *
+     *     // Allocate WarpReduce shared memory for one warp
+     *     __shared__ typename WarpReduce::TempStorage temp_storage;
+     *
+     *     // Obtain one input item per thread if in range
+     *     int thread_data;
+     *     if (threadIdx.x < valid_items)
+     *         thread_data = d_data[threadIdx.x];
+     *
+     *     // Return the warp-wide reductions to each lane0
+     *     int aggregate = WarpReduce(temp_storage).Reduce(
+     *         thread_data, cub::Max(), valid_items);
+     *
+     * \endcode
+     * \par
+     * Suppose the input \p d_data is <tt>{0, 1, 2, 3, 4, ...</tt> and \p valid_items
+     * is \p 4.  The corresponding output \p aggregate in thread0 is \p 3 (and is
+     * undefined in other threads).
+     *
+     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ReductionOp>
+    __device__ __forceinline__ T Reduce(
+        T                   input,              ///< [in] Calling thread's input
+        ReductionOp         reduction_op,       ///< [in] Binary reduction operator
+        int                 valid_items)        ///< [in] Total number of valid items in the calling thread's logical warp (may be less than \p LOGICAL_WARP_THREADS)
+    {
+        return InternalWarpReduce(temp_storage).template Reduce<false>(input, valid_items, reduction_op);
+    }
+
+
+    /**
+     * \brief Computes a segmented reduction in the calling warp where segments are defined by head-flags.  The reduction of each segment is returned to the first lane in that segment (which always includes <em>lane</em><sub>0</sub>).
+     *
+     * Supports non-commutative reduction operators
+     *
+     * \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a head-segmented warp max
+     * reduction within a block of 32 threads (one warp).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpReduce for type int
+     *     typedef cub::WarpReduce<int> WarpReduce;
+     *
+     *     // Allocate WarpReduce shared memory for one warp
+     *     __shared__ typename WarpReduce::TempStorage temp_storage;
+     *
+     *     // Obtain one input item and flag per thread
+     *     int thread_data = ...
+     *     int head_flag = ...
+     *
+     *     // Return the warp-wide reductions to each lane0
+     *     int aggregate = WarpReduce(temp_storage).HeadSegmentedReduce(
+     *         thread_data, head_flag, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data and \p head_flag across the block of threads
+     * is <tt>{0, 1, 2, 3, ..., 31</tt> and is <tt>{1, 0, 0, 0, 1, 0, 0, 0, ..., 1, 0, 0, 0</tt>,
+     * respectively.  The corresponding output \p aggregate in threads 0, 4, 8, etc. will be
+     * \p 3, \p 7, \p 11, etc. (and is undefined in other threads).
+     *
+     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        typename            ReductionOp,
+        typename            FlagT>
+    __device__ __forceinline__ T HeadSegmentedReduce(
+        T                   input,              ///< [in] Calling thread's input
+        FlagT               head_flag,          ///< [in] Head flag denoting whether or not \p input is the start of a new segment
+        ReductionOp         reduction_op)       ///< [in] Reduction operator
+    {
+        return InternalWarpReduce(temp_storage).template SegmentedReduce<true>(input, head_flag, reduction_op);
+    }
+
+
+    /**
+     * \brief Computes a segmented reduction in the calling warp where segments are defined by tail-flags.  The reduction of each segment is returned to the first lane in that segment (which always includes <em>lane</em><sub>0</sub>).
+     *
+     * Supports non-commutative reduction operators
+     *
+     * \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates a tail-segmented warp max
+     * reduction within a block of 32 threads (one warp).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpReduce for type int
+     *     typedef cub::WarpReduce<int> WarpReduce;
+     *
+     *     // Allocate WarpReduce shared memory for one warp
+     *     __shared__ typename WarpReduce::TempStorage temp_storage;
+     *
+     *     // Obtain one input item and flag per thread
+     *     int thread_data = ...
+     *     int tail_flag = ...
+     *
+     *     // Return the warp-wide reductions to each lane0
+     *     int aggregate = WarpReduce(temp_storage).TailSegmentedReduce(
+     *         thread_data, tail_flag, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data and \p tail_flag across the block of threads
+     * is <tt>{0, 1, 2, 3, ..., 31</tt> and is <tt>{0, 0, 0, 1, 0, 0, 0, 1, ..., 0, 0, 0, 1</tt>,
+     * respectively.  The corresponding output \p aggregate in threads 0, 4, 8, etc. will be
+     * \p 3, \p 7, \p 11, etc. (and is undefined in other threads).
+     *
+     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        typename            ReductionOp,
+        typename            FlagT>
+    __device__ __forceinline__ T TailSegmentedReduce(
+        T                   input,              ///< [in] Calling thread's input
+        FlagT               tail_flag,          ///< [in] Tail flag denoting whether or not \p input is the end of the current segment
+        ReductionOp         reduction_op)       ///< [in] Reduction operator
+    {
+        return InternalWarpReduce(temp_storage).template SegmentedReduce<false>(input, tail_flag, reduction_op);
+    }
+
+
+
+    //@}  end member group
+};
+
+/** @} */       // end group WarpModule
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/cub/warp/warp_scan.cuh b/third_party/cub/cub/warp/warp_scan.cuh
new file mode 100644
index 0000000..c7af0d3
--- /dev/null
+++ b/third_party/cub/cub/warp/warp_scan.cuh
@@ -0,0 +1,936 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/**
+ * \file
+ * The cub::WarpScan class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel prefix scan of items partitioned across a CUDA thread warp.
+ */
+
+#pragma once
+
+#include "specializations/warp_scan_shfl.cuh"
+#include "specializations/warp_scan_smem.cuh"
+#include "../thread/thread_operators.cuh"
+#include "../util_arch.cuh"
+#include "../util_type.cuh"
+#include "../util_namespace.cuh"
+
+/// Optional outer namespace(s)
+CUB_NS_PREFIX
+
+/// CUB namespace
+namespace cub {
+
+/**
+ * \addtogroup WarpModule
+ * @{
+ */
+
+/**
+ * \brief The WarpScan class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel prefix scan of items partitioned across a CUDA thread warp.  ![](warp_scan_logo.png)
+ *
+ * \tparam T                        The scan input/output element type
+ * \tparam LOGICAL_WARP_THREADS     <b>[optional]</b> The number of threads per "logical" warp (may be less than the number of hardware warp threads).  Default is the warp size associated with the CUDA Compute Capability targeted by the compiler (e.g., 32 threads for SM20).
+ * \tparam PTX_ARCH                 <b>[optional]</b> \ptxversion
+ *
+ * \par Overview
+ * - Given a list of input elements and a binary reduction operator, a [<em>prefix scan</em>](http://en.wikipedia.org/wiki/Prefix_sum)
+ *   produces an output list where each element is computed to be the reduction
+ *   of the elements occurring earlier in the input list.  <em>Prefix sum</em>
+ *   connotes a prefix scan with the addition operator. The term \em inclusive indicates
+ *   that the <em>i</em><sup>th</sup> output reduction incorporates the <em>i</em><sup>th</sup> input.
+ *   The term \em exclusive indicates the <em>i</em><sup>th</sup> input is not incorporated into
+ *   the <em>i</em><sup>th</sup> output reduction.
+ * - Supports non-commutative scan operators
+ * - Supports "logical" warps smaller than the physical warp size (e.g., a logical warp of 8 threads)
+ * - The number of entrant threads must be an multiple of \p LOGICAL_WARP_THREADS
+ *
+ * \par Performance Considerations
+ * - Uses special instructions when applicable (e.g., warp \p SHFL)
+ * - Uses synchronization-free communication between warp lanes when applicable
+ * - Incurs zero bank conflicts for most types
+ * - Computation is slightly more efficient (i.e., having lower instruction overhead) for:
+ *     - Summation (<b><em>vs.</em></b> generic scan)
+ *     - The architecture's warp size is a whole multiple of \p LOGICAL_WARP_THREADS
+ *
+ * \par Simple Examples
+ * \warpcollective{WarpScan}
+ * \par
+ * The code snippet below illustrates four concurrent warp prefix sums within a block of
+ * 128 threads (one per each of the 32-thread warps).
+ * \par
+ * \code
+ * #include <cub/cub.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize WarpScan for type int
+ *     typedef cub::WarpScan<int> WarpScan;
+ *
+ *     // Allocate WarpScan shared memory for 4 warps
+ *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+ *
+ *     // Obtain one input item per thread
+ *     int thread_data = ...
+ *
+ *     // Compute warp-wide prefix sums
+ *     int warp_id = threadIdx.x / 32;
+ *     WarpScan(temp_storage[warp_id]).ExclusiveSum(thread_data, thread_data);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
+ * The corresponding output \p thread_data in each of the four warps of threads will be
+ * <tt>0, 1, 2, 3, ..., 31}</tt>.
+ *
+ * \par
+ * The code snippet below illustrates a single warp prefix sum within a block of
+ * 128 threads.
+ * \par
+ * \code
+ * #include <cub/cub.cuh>
+ *
+ * __global__ void ExampleKernel(...)
+ * {
+ *     // Specialize WarpScan for type int
+ *     typedef cub::WarpScan<int> WarpScan;
+ *
+ *     // Allocate WarpScan shared memory for one warp
+ *     __shared__ typename WarpScan::TempStorage temp_storage;
+ *     ...
+ *
+ *     // Only the first warp performs a prefix sum
+ *     if (threadIdx.x < 32)
+ *     {
+ *         // Obtain one input item per thread
+ *         int thread_data = ...
+ *
+ *         // Compute warp-wide prefix sums
+ *         WarpScan(temp_storage).ExclusiveSum(thread_data, thread_data);
+ *
+ * \endcode
+ * \par
+ * Suppose the set of input \p thread_data across the warp of threads is <tt>{1, 1, 1, 1, ...}</tt>.
+ * The corresponding output \p thread_data will be <tt>{0, 1, 2, 3, ..., 31}</tt>.
+ *
+ */
+template <
+    typename    T,
+    int         LOGICAL_WARP_THREADS    = CUB_PTX_WARP_THREADS,
+    int         PTX_ARCH                = CUB_PTX_ARCH>
+class WarpScan
+{
+private:
+
+    /******************************************************************************
+     * Constants and type definitions
+     ******************************************************************************/
+
+    enum
+    {
+        /// Whether the logical warp size and the PTX warp size coincide
+        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
+
+        /// Whether the logical warp size is a power-of-two
+        IS_POW_OF_TWO = ((LOGICAL_WARP_THREADS & (LOGICAL_WARP_THREADS - 1)) == 0),
+
+        /// Whether the data type is an integer (which has fully-associative addition)
+        IS_INTEGER = ((Traits<T>::CATEGORY == SIGNED_INTEGER) || (Traits<T>::CATEGORY == UNSIGNED_INTEGER))
+    };
+
+    /// Internal specialization.  Use SHFL-based scan if (architecture is >= SM30) and (LOGICAL_WARP_THREADS is a power-of-two)
+    typedef typename If<(PTX_ARCH >= 300) && (IS_POW_OF_TWO),
+        WarpScanShfl<T, LOGICAL_WARP_THREADS, PTX_ARCH>,
+        WarpScanSmem<T, LOGICAL_WARP_THREADS, PTX_ARCH> >::Type InternalWarpScan;
+
+    /// Shared memory storage layout type for WarpScan
+    typedef typename InternalWarpScan::TempStorage _TempStorage;
+
+
+    /******************************************************************************
+     * Thread fields
+     ******************************************************************************/
+
+    /// Shared storage reference
+    _TempStorage    &temp_storage;
+    unsigned int    lane_id;
+
+
+
+    /******************************************************************************
+     * Public types
+     ******************************************************************************/
+
+public:
+
+    /// \smemstorage{WarpScan}
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    /******************************************************************//**
+     * \name Collective constructors
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Collective constructor using the specified memory allocation as temporary storage.  Logical warp and lane identifiers are constructed from <tt>threadIdx.x</tt>.
+     */
+    __device__ __forceinline__ WarpScan(
+        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
+    :
+        temp_storage(temp_storage.Alias()),
+        lane_id(IS_ARCH_WARP ?
+            LaneId() :
+            LaneId() % LOGICAL_WARP_THREADS)
+    {}
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Inclusive prefix sums
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an inclusive prefix sum across the calling warp.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide inclusive prefix sums within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute inclusive warp-wide prefix sums
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).InclusiveSum(thread_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
+     * The corresponding output \p thread_data in each of the four warps of threads will be
+     * <tt>1, 2, 3, ..., 32}</tt>.
+     */
+    __device__ __forceinline__ void InclusiveSum(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output)  ///< [out] Calling thread's output item.  May be aliased with \p input.
+    {
+        InclusiveScan(input, inclusive_output, cub::Sum());
+    }
+
+
+    /**
+     * \brief Computes an inclusive prefix sum across the calling warp.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide inclusive prefix sums within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute inclusive warp-wide prefix sums
+     *     int warp_aggregate;
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).InclusiveSum(thread_data, thread_data, warp_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
+     * The corresponding output \p thread_data in each of the four warps of threads will be
+     * <tt>1, 2, 3, ..., 32}</tt>.  Furthermore, \p warp_aggregate for all threads in all warps will be \p 32.
+     */
+    __device__ __forceinline__ void InclusiveSum(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
+    {
+        InclusiveScan(input, inclusive_output, cub::Sum(), warp_aggregate);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Exclusive prefix sums
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes an exclusive prefix sum across the calling warp.  The value of 0 is applied as the initial value, and is assigned to \p exclusive_output in <em>thread</em><sub>0</sub>.
+     *
+     * \par
+     *  - \identityzero
+     *  - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide exclusive prefix sums within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute exclusive warp-wide prefix sums
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).ExclusiveSum(thread_data, thread_data);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
+     * The corresponding output \p thread_data in each of the four warps of threads will be
+     * <tt>0, 1, 2, ..., 31}</tt>.
+     *
+     */
+    __device__ __forceinline__ void ExclusiveSum(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &exclusive_output)  ///< [out] Calling thread's output item.  May be aliased with \p input.
+    {
+        T initial_value = 0;
+        ExclusiveScan(input, exclusive_output, initial_value, cub::Sum());
+    }
+
+
+    /**
+     * \brief Computes an exclusive prefix sum across the calling warp.  The value of 0 is applied as the initial value, and is assigned to \p exclusive_output in <em>thread</em><sub>0</sub>.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
+     *
+     * \par
+     *  - \identityzero
+     *  - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide exclusive prefix sums within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute exclusive warp-wide prefix sums
+     *     int warp_aggregate;
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).ExclusiveSum(thread_data, thread_data, warp_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
+     * The corresponding output \p thread_data in each of the four warps of threads will be
+     * <tt>0, 1, 2, ..., 31}</tt>.  Furthermore, \p warp_aggregate for all threads in all warps will be \p 32.
+     */
+    __device__ __forceinline__ void ExclusiveSum(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &exclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
+    {
+        T initial_value = 0;
+        ExclusiveScan(input, exclusive_output, initial_value, cub::Sum(), warp_aggregate);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Inclusive prefix scans
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Computes an inclusive prefix scan using the specified binary scan functor across the calling warp.
+     *
+     * \par
+     *  - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide inclusive prefix max scans within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute inclusive warp-wide prefix max scans
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).InclusiveScan(thread_data, thread_data, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
+     * The corresponding output \p thread_data in the first warp would be
+     * <tt>0, 0, 2, 2, ..., 30, 30</tt>, the output for the second warp would be <tt>32, 32, 34, 34, ..., 62, 62</tt>, etc.
+     *
+     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        InternalWarpScan(temp_storage).InclusiveScan(input, inclusive_output, scan_op);
+    }
+
+
+    /**
+     * \brief Computes an inclusive prefix scan using the specified binary scan functor across the calling warp.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide inclusive prefix max scans within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute inclusive warp-wide prefix max scans
+     *     int warp_aggregate;
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).InclusiveScan(
+     *         thread_data, thread_data, cub::Max(), warp_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
+     * The corresponding output \p thread_data in the first warp would be
+     * <tt>0, 0, 2, 2, ..., 30, 30</tt>, the output for the second warp would be <tt>32, 32, 34, 34, ..., 62, 62</tt>, etc.
+     * Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads
+     * in the second warp, etc.
+     *
+     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void InclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
+    {
+        InternalWarpScan(temp_storage).InclusiveScan(input, inclusive_output, scan_op, warp_aggregate);
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Exclusive prefix scans
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Computes an exclusive prefix scan using the specified binary scan functor across the calling warp.  Because no initial value is supplied, the \p output computed for <em>warp-lane</em><sub>0</sub> is undefined.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute exclusive warp-wide prefix max scans
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).ExclusiveScan(thread_data, thread_data, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
+     * The corresponding output \p thread_data in the first warp would be
+     * <tt>?, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>?, 32, 32, 34, ..., 60, 62</tt>, etc.
+     * (The output \p thread_data in warp lane<sub>0</sub> is undefined.)
+     *
+     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &exclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        InternalWarpScan internal(temp_storage);
+
+        T inclusive_output;
+        internal.InclusiveScan(input, inclusive_output, scan_op);
+
+        internal.Update(
+            input,
+            inclusive_output,
+            exclusive_output,
+            scan_op,
+            Int2Type<IS_INTEGER>());
+    }
+
+
+    /**
+     * \brief Computes an exclusive prefix scan using the specified binary scan functor across the calling warp.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute exclusive warp-wide prefix max scans
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
+     * The corresponding output \p thread_data in the first warp would be
+     * <tt>INT_MIN, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>30, 32, 32, 34, ..., 60, 62</tt>, etc.
+     *
+     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &exclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        T               initial_value,      ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        InternalWarpScan internal(temp_storage);
+
+        T inclusive_output;
+        internal.InclusiveScan(input, inclusive_output, scan_op);
+
+        internal.Update(
+            input,
+            inclusive_output,
+            exclusive_output,
+            scan_op,
+            initial_value,
+            Int2Type<IS_INTEGER>());
+    }
+
+
+    /**
+     * \brief Computes an exclusive prefix scan using the specified binary scan functor across the calling warp.  Because no initial value is supplied, the \p output computed for <em>warp-lane</em><sub>0</sub> is undefined.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute exclusive warp-wide prefix max scans
+     *     int warp_aggregate;
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).ExclusiveScan(thread_data, thread_data, cub::Max(), warp_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
+     * The corresponding output \p thread_data in the first warp would be
+     * <tt>?, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>?, 32, 32, 34, ..., 60, 62</tt>, etc.
+     * (The output \p thread_data in warp lane<sub>0</sub> is undefined.)  Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads
+     * in the second warp, etc.
+     *
+     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &exclusive_output,   ///< [out] Calling thread's output item.  May be aliased with \p input.
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
+    {
+        InternalWarpScan internal(temp_storage);
+
+        T inclusive_output;
+        internal.InclusiveScan(input, inclusive_output, scan_op);
+
+        internal.Update(
+            input,
+            inclusive_output,
+            exclusive_output,
+            warp_aggregate,
+            scan_op,
+            Int2Type<IS_INTEGER>());
+    }
+
+
+    /**
+     * \brief Computes an exclusive prefix scan using the specified binary scan functor across the calling warp.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute exclusive warp-wide prefix max scans
+     *     int warp_aggregate;
+     *     int warp_id = threadIdx.x / 32;
+     *     WarpScan(temp_storage[warp_id]).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max(), warp_aggregate);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
+     * The corresponding output \p thread_data in the first warp would be
+     * <tt>INT_MIN, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>30, 32, 32, 34, ..., 60, 62</tt>, etc.
+     * Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads
+     * in the second warp, etc.
+     *
+     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void ExclusiveScan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &exclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
+        T               initial_value,      ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op,            ///< [in] Binary scan operator
+        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
+    {
+        InternalWarpScan internal(temp_storage);
+
+        T inclusive_output;
+        internal.InclusiveScan(input, inclusive_output, scan_op);
+
+        internal.Update(
+            input,
+            inclusive_output,
+            exclusive_output,
+            warp_aggregate,
+            scan_op,
+            initial_value,
+            Int2Type<IS_INTEGER>());
+    }
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Combination (inclusive & exclusive) prefix scans
+     *********************************************************************/
+    //@{
+
+
+    /**
+     * \brief Computes both inclusive and exclusive prefix scans using the specified binary scan functor across the calling warp.  Because no initial value is supplied, the \p exclusive_output computed for <em>warp-lane</em><sub>0</sub> is undefined.
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute exclusive warp-wide prefix max scans
+     *     int inclusive_partial, exclusive_partial;
+     *     WarpScan(temp_storage[warp_id]).Scan(thread_data, inclusive_partial, exclusive_partial, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
+     * The corresponding output \p inclusive_partial in the first warp would be
+     * <tt>0, 0, 2, 2, ..., 30, 30</tt>, the output for the second warp would be <tt>32, 32, 34, 34, ..., 62, 62</tt>, etc.
+     * The corresponding output \p exclusive_partial in the first warp would be
+     * <tt>?, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>?, 32, 32, 34, ..., 60, 62</tt>, etc.
+     * (The output \p thread_data in warp lane<sub>0</sub> is undefined.)
+     *
+     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void Scan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output,  ///< [out] Calling thread's inclusive-scan output item.
+        T               &exclusive_output,  ///< [out] Calling thread's exclusive-scan output item.
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        InternalWarpScan internal(temp_storage);
+
+        internal.InclusiveScan(input, inclusive_output, scan_op);
+
+        internal.Update(
+            input,
+            inclusive_output,
+            exclusive_output,
+            scan_op,
+            Int2Type<IS_INTEGER>());
+    }
+
+
+    /**
+     * \brief Computes both inclusive and exclusive prefix scans using the specified binary scan functor across the calling warp.
+     *
+     * \par
+     *  - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates four concurrent warp-wide prefix max scans within a block of
+     * 128 threads (one per each of the 32-thread warps).
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Compute inclusive warp-wide prefix max scans
+     *     int warp_id = threadIdx.x / 32;
+     *     int inclusive_partial, exclusive_partial;
+     *     WarpScan(temp_storage[warp_id]).Scan(thread_data, inclusive_partial, exclusive_partial, INT_MIN, cub::Max());
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
+     * The corresponding output \p inclusive_partial in the first warp would be
+     * <tt>0, 0, 2, 2, ..., 30, 30</tt>, the output for the second warp would be <tt>32, 32, 34, 34, ..., 62, 62</tt>, etc.
+     * The corresponding output \p exclusive_partial in the first warp would be
+     * <tt>INT_MIN, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>30, 32, 32, 34, ..., 60, 62</tt>, etc.
+     *
+     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <typename ScanOp>
+    __device__ __forceinline__ void Scan(
+        T               input,              ///< [in] Calling thread's input item.
+        T               &inclusive_output,  ///< [out] Calling thread's inclusive-scan output item.
+        T               &exclusive_output,  ///< [out] Calling thread's exclusive-scan output item.
+        T               initial_value,      ///< [in] Initial value to seed the exclusive scan
+        ScanOp          scan_op)            ///< [in] Binary scan operator
+    {
+        InternalWarpScan internal(temp_storage);
+
+        internal.InclusiveScan(input, inclusive_output, scan_op);
+
+        internal.Update(
+            input,
+            inclusive_output,
+            exclusive_output,
+            scan_op,
+            initial_value,
+            Int2Type<IS_INTEGER>());
+    }
+
+
+
+    //@}  end member group
+    /******************************************************************//**
+     * \name Data exchange
+     *********************************************************************/
+    //@{
+
+    /**
+     * \brief Broadcast the value \p input from <em>warp-lane</em><sub><tt>src_lane</tt></sub> to all lanes in the warp
+     *
+     * \par
+     * - \smemreuse
+     *
+     * \par Snippet
+     * The code snippet below illustrates the warp-wide broadcasts of values from
+     * lanes<sub>0</sub> in each of four warps to all other threads in those warps.
+     * \par
+     * \code
+     * #include <cub/cub.cuh>
+     *
+     * __global__ void ExampleKernel(...)
+     * {
+     *     // Specialize WarpScan for type int
+     *     typedef cub::WarpScan<int> WarpScan;
+     *
+     *     // Allocate WarpScan shared memory for 4 warps
+     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
+     *
+     *     // Obtain one input item per thread
+     *     int thread_data = ...
+     *
+     *     // Broadcast from lane0 in each warp to all other threads in the warp
+     *     int warp_id = threadIdx.x / 32;
+     *     thread_data = WarpScan(temp_storage[warp_id]).Broadcast(thread_data, 0);
+     *
+     * \endcode
+     * \par
+     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, 1, 2, 3, ..., 127}</tt>.
+     * The corresponding output \p thread_data will be
+     * <tt>{0, 0, ..., 0}</tt> in warp<sub>0</sub>,
+     * <tt>{32, 32, ..., 32}</tt> in warp<sub>1</sub>,
+     * <tt>{64, 64, ..., 64}</tt> in warp<sub>2</sub>, etc.
+     */
+    __device__ __forceinline__ T Broadcast(
+        T               input,              ///< [in] The value to broadcast
+        unsigned int    src_lane)           ///< [in] Which warp lane is to do the broadcasting
+    {
+        return InternalWarpScan(temp_storage).Broadcast(input, src_lane);
+    }
+
+    //@}  end member group
+
+};
+
+/** @} */       // end group WarpModule
+
+}               // CUB namespace
+CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/third_party/cub/eclipse code style profile.xml b/third_party/cub/eclipse code style profile.xml
new file mode 100644
index 0000000..3ca7f77
--- /dev/null
+++ b/third_party/cub/eclipse code style profile.xml	
@@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<profiles version="1">
+<profile kind="CodeFormatterProfile" name="B40C" version="1">
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.lineSplit" value="80"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_base_types" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_switchstatements_compare_to_switch" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_exception_specification" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_base_types" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_body_declarations_compare_to_access_specifier" value="true"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_exception_specification" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_template_arguments" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_declarator_list" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_bracket" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.tabulation.size" value="4"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_before_else_in_if_statement" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_enumerator_list" value="48"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_declarator_list" value="16"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_empty_lines" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
+<setting id="org.eclipse.cdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.brace_position_for_method_declaration" value="next_line"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_closing_angle_bracket_in_template_arguments" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_colon_in_base_clause" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_declarator_list" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_between_empty_brackets" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_bracket" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_parameters_in_method_declaration" value="48"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.brace_position_for_block" value="next_line"/>
+<setting id="org.eclipse.cdt.core.formatter.brace_position_for_type_declaration" value="next_line"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_angle_bracket_in_template_arguments" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_expression_list" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_angle_bracket_in_template_parameters" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.continuation_indentation" value="1"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_expression_list" value="0"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_template_parameters" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_conditional_expression" value="48"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_access_specifier_compare_to_type_header" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.continuation_indentation_for_array_initializer" value="1"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_body_declarations_compare_to_namespace_header" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_compact_if" value="0"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_template_parameters" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_expression_list" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_exception_specification" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_before_identifier_in_function_declaration" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_base_clause_in_type_declaration" value="48"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_exception_specification" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_declaration_compare_to_template_header" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_statements_compare_to_body" value="true"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indent_statements_compare_to_block" value="true"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_template_arguments" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="48"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_angle_bracket_in_template_parameters" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.tabulation.char" value="space"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_angle_bracket_in_template_parameters" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
+<setting id="org.eclipse.cdt.core.formatter.compact_else_if" value="true"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_colon_in_base_clause" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_after_template_declaration" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
+<setting id="org.eclipse.cdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.keep_imple_if_on_one_line" value="true"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.indentation.size" value="4"/>
+<setting id="org.eclipse.cdt.core.formatter.brace_position_for_namespace_declaration" value="end_of_line"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_angle_bracket_in_template_arguments" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.brace_position_for_array_initializer" value="next_line"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_namespace_declaration" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_closing_bracket" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_closing_angle_bracket_in_template_parameters" value="insert"/>
+<setting id="org.eclipse.cdt.core.formatter.insert_space_after_opening_angle_bracket_in_template_arguments" value="do not insert"/>
+</profile>
+</profiles>
diff --git a/third_party/cub/examples/block/.gitignore b/third_party/cub/examples/block/.gitignore
new file mode 100644
index 0000000..9dad963
--- /dev/null
+++ b/third_party/cub/examples/block/.gitignore
@@ -0,0 +1,7 @@
+/bin
+/Debug
+/Release
+/cuda55.sdf
+/cuda55.suo
+/cuda60.sdf
+/cuda60.suo
diff --git a/third_party/cub/examples/block/Makefile b/third_party/cub/examples/block/Makefile
new file mode 100644
index 0000000..b173c2a
--- /dev/null
+++ b/third_party/cub/examples/block/Makefile
@@ -0,0 +1,128 @@
+#/******************************************************************************
+# * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+# * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+# * 
+# * Redistribution and use in source and binary forms, with or without
+# * modification, are permitted provided that the following conditions are met:
+# *	 * Redistributions of source code must retain the above copyright
+# *	   notice, this list of conditions and the following disclaimer.
+# *	 * Redistributions in binary form must reproduce the above copyright
+# *	   notice, this list of conditions and the following disclaimer in the
+# *	   documentation and/or other materials provided with the distribution.
+# *	 * Neither the name of the NVIDIA CORPORATION nor the
+# *	   names of its contributors may be used to endorse or promote products
+# *	   derived from this software without specific prior written permission.
+# * 
+# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+# * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *
+#******************************************************************************/
+
+#-------------------------------------------------------------------------------
+#
+# Makefile usage
+#
+# make <target> [sm=<XXX,...>] [cdp=<0|1>] [force32=<0|1>] [abi=<0|1>] [open64=<0|1>] [verbose=<0|1>] [keep=<0|1>]
+#
+#-------------------------------------------------------------------------------
+ 
+include ../../common.mk 
+ 
+ 
+#-------------------------------------------------------------------------------
+# Includes
+#-------------------------------------------------------------------------------
+
+INC += -I$(CUB_DIR) -I$(CUB_DIR)test 
+
+
+
+#-------------------------------------------------------------------------------
+# Dependency Lists
+#-------------------------------------------------------------------------------
+
+rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
+
+DEPS =				$(CUB_DEPS) \
+					$(CUB_DIR)test/Makefile \
+					$(CUB_DIR)test/test_util.h \
+					$(CUB_DIR)test/mersenne.h \
+		
+ALL = 	example_block_radix_sort \
+	 	example_block_reduce \
+	 	example_block_scan
+		
+
+
+#-------------------------------------------------------------------------------
+# make default
+#-------------------------------------------------------------------------------
+
+default:
+
+
+#-------------------------------------------------------------------------------
+# make clean
+#-------------------------------------------------------------------------------
+
+clean :
+	rm -f bin/*$(CPU_ARCH_SUFFIX)* 
+	rm -f *.i* *.cubin *.cu.c *.cudafe* *.fatbin.c *.ptx *.hash *.cu.cpp *.o
+
+
+#-------------------------------------------------------------------------------
+# make all
+#-------------------------------------------------------------------------------
+
+all : $(ALL)
+
+#-------------------------------------------------------------------------------
+# make run
+#-------------------------------------------------------------------------------
+
+run : 
+	for i in $(ALL); do ./bin/$${i}_$(BIN_SUFFIX) --device=$(device) || exit 1; done
+
+
+
+
+#-------------------------------------------------------------------------------
+# make example_block_reduce
+#-------------------------------------------------------------------------------
+
+example_block_reduce: bin/example_block_reduce_$(BIN_SUFFIX)
+
+bin/example_block_reduce_$(BIN_SUFFIX) : example_block_reduce.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_block_reduce_$(BIN_SUFFIX) example_block_reduce.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make example_block_scan
+#-------------------------------------------------------------------------------
+
+example_block_scan: bin/example_block_scan_$(BIN_SUFFIX)
+
+bin/example_block_scan_$(BIN_SUFFIX) : example_block_scan.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_block_scan_$(BIN_SUFFIX) example_block_scan.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make example_block_radix_sort
+#-------------------------------------------------------------------------------
+
+example_block_radix_sort: bin/example_block_radix_sort_$(BIN_SUFFIX)
+
+bin/example_block_radix_sort_$(BIN_SUFFIX) : example_block_radix_sort.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_block_radix_sort_$(BIN_SUFFIX) example_block_radix_sort.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+	
diff --git a/third_party/cub/examples/block/example_block_radix_sort.cu b/third_party/cub/examples/block/example_block_radix_sort.cu
new file mode 100644
index 0000000..2fbeda9
--- /dev/null
+++ b/third_party/cub/examples/block/example_block_radix_sort.cu
@@ -0,0 +1,323 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple demonstration of cub::BlockRadixSort
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_block_radix_sort.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console (define before including cub.h)
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <iostream>
+#include <algorithm>
+
+#include <cub/block/block_load.cuh>
+#include <cub/block/block_store.cuh>
+#include <cub/block/block_radix_sort.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+/// Verbose output
+bool g_verbose = false;
+
+/// Timing iterations
+int g_timing_iterations = 100;
+
+/// Default grid size
+int g_grid_size = 1;
+
+/// Uniform key samples
+bool g_uniform_keys;
+
+
+//---------------------------------------------------------------------
+// Kernels
+//---------------------------------------------------------------------
+
+/**
+ * Simple kernel for performing a block-wide sorting over integers
+ */
+template <
+    typename    Key,
+    int         BLOCK_THREADS,
+    int         ITEMS_PER_THREAD>
+__launch_bounds__ (BLOCK_THREADS)
+__global__ void BlockSortKernel(
+    Key         *d_in,          // Tile of input
+    Key         *d_out,         // Tile of output
+    clock_t     *d_elapsed)     // Elapsed cycle count of block scan
+{
+    enum { TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD };
+
+    // Specialize BlockLoad type for our thread block (uses warp-striped loads for coalescing, then transposes in shared memory to a blocked arrangement)
+    typedef BlockLoad<Key, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoadT;
+
+    // Specialize BlockRadixSort type for our thread block
+    typedef BlockRadixSort<Key, BLOCK_THREADS, ITEMS_PER_THREAD> BlockRadixSortT;
+
+    // Shared memory
+    __shared__ union TempStorage
+    {
+        typename BlockLoadT::TempStorage        load;
+        typename BlockRadixSortT::TempStorage   sort;
+    } temp_storage;
+
+    // Per-thread tile items
+    Key items[ITEMS_PER_THREAD];
+
+    // Our current block's offset
+    int block_offset = blockIdx.x * TILE_SIZE;
+
+    // Load items into a blocked arrangement
+    BlockLoadT(temp_storage.load).Load(d_in + block_offset, items);
+
+    // Barrier for smem reuse
+    __syncthreads();
+
+    // Start cycle timer
+    clock_t start = clock();
+
+    // Sort keys
+    BlockRadixSortT(temp_storage.sort).SortBlockedToStriped(items);
+
+    // Stop cycle timer
+    clock_t stop = clock();
+
+    // Store output in striped fashion
+    StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_out + block_offset, items);
+
+    // Store elapsed clocks
+    if (threadIdx.x == 0)
+    {
+        d_elapsed[blockIdx.x] = (start > stop) ? start - stop : stop - start;
+    }
+}
+
+
+
+//---------------------------------------------------------------------
+// Host utilities
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize sorting problem (and solution).
+ */
+template <typename Key>
+void Initialize(
+    Key *h_in,
+    Key *h_reference,
+    int num_items,
+    int tile_size)
+{
+    for (int i = 0; i < num_items; ++i)
+    {
+        if (g_uniform_keys)
+        {
+            h_in[i] = 0;
+        }
+        else
+        {
+            RandomBits(h_in[i]);
+        }
+        h_reference[i] = h_in[i];
+    }
+
+    // Only sort the first tile
+    std::sort(h_reference, h_reference + tile_size);
+}
+
+
+/**
+ * Test BlockScan
+ */
+template <
+    typename    Key,
+    int         BLOCK_THREADS,
+    int         ITEMS_PER_THREAD>
+void Test()
+{
+    const int TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    // Allocate host arrays
+    Key *h_in               = new Key[TILE_SIZE * g_grid_size];
+    Key *h_reference        = new Key[TILE_SIZE * g_grid_size];
+    clock_t *h_elapsed      = new clock_t[g_grid_size];
+
+    // Initialize problem and reference output on host
+    Initialize(h_in, h_reference, TILE_SIZE * g_grid_size, TILE_SIZE);
+
+    // Initialize device arrays
+    Key *d_in       = NULL;
+    Key *d_out      = NULL;
+    clock_t *d_elapsed  = NULL;
+    CubDebugExit(cudaMalloc((void**)&d_in,          sizeof(Key) * TILE_SIZE * g_grid_size));
+    CubDebugExit(cudaMalloc((void**)&d_out,         sizeof(Key) * TILE_SIZE * g_grid_size));
+    CubDebugExit(cudaMalloc((void**)&d_elapsed,     sizeof(clock_t) * g_grid_size));
+
+    // Display input problem data
+    if (g_verbose)
+    {
+        printf("Input data: ");
+        for (int i = 0; i < TILE_SIZE; i++)
+            std::cout << h_in[i] << ", ";
+        printf("\n\n");
+    }
+
+    // Kernel props
+    int max_sm_occupancy;
+    CubDebugExit(MaxSmOccupancy(max_sm_occupancy, BlockSortKernel<Key, BLOCK_THREADS, ITEMS_PER_THREAD>, BLOCK_THREADS));
+
+    // Copy problem to device
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(Key) * TILE_SIZE * g_grid_size, cudaMemcpyHostToDevice));
+
+    printf("BlockRadixSort %d items (%d timing iterations, %d blocks, %d threads, %d items per thread, %d SM occupancy):\n",
+        TILE_SIZE * g_grid_size, g_timing_iterations, g_grid_size, BLOCK_THREADS, ITEMS_PER_THREAD, max_sm_occupancy);
+    fflush(stdout);
+
+    // Run kernel once to prime caches and check result
+    BlockSortKernel<Key, BLOCK_THREADS, ITEMS_PER_THREAD><<<g_grid_size, BLOCK_THREADS>>>(
+        d_in,
+        d_out,
+        d_elapsed);
+
+    // Check for kernel errors and STDIO from the kernel, if any
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Check results
+    printf("\tOutput items: ");
+    int compare = CompareDeviceResults(h_reference, d_out, TILE_SIZE, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+    fflush(stdout);
+
+    // Run this several times and average the performance results
+    GpuTimer            timer;
+    float               elapsed_millis          = 0.0;
+    unsigned long long  elapsed_clocks          = 0;
+
+    for (int i = 0; i < g_timing_iterations; ++i)
+    {
+        timer.Start();
+
+        // Run kernel
+        BlockSortKernel<Key, BLOCK_THREADS, ITEMS_PER_THREAD><<<g_grid_size, BLOCK_THREADS>>>(
+            d_in,
+            d_out,
+            d_elapsed);
+
+        timer.Stop();
+        elapsed_millis += timer.ElapsedMillis();
+
+        // Copy clocks from device
+        CubDebugExit(cudaMemcpy(h_elapsed, d_elapsed, sizeof(clock_t) * g_grid_size, cudaMemcpyDeviceToHost));
+        for (int i = 0; i < g_grid_size; i++)
+            elapsed_clocks += h_elapsed[i];
+    }
+
+    // Check for kernel errors and STDIO from the kernel, if any
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Display timing results
+    float avg_millis            = elapsed_millis / g_timing_iterations;
+    float avg_items_per_sec     = float(TILE_SIZE * g_grid_size) / avg_millis / 1000.0f;
+    double avg_clocks           = double(elapsed_clocks) / g_timing_iterations / g_grid_size;
+    double avg_clocks_per_item  = avg_clocks / TILE_SIZE;
+
+    printf("\tAverage BlockRadixSort::SortBlocked clocks: %.3f\n", avg_clocks);
+    printf("\tAverage BlockRadixSort::SortBlocked clocks per item: %.3f\n", avg_clocks_per_item);
+    printf("\tAverage kernel millis: %.4f\n", avg_millis);
+    printf("\tAverage million items / sec: %.4f\n", avg_items_per_sec);
+    fflush(stdout);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (h_elapsed) delete[] h_elapsed;
+    if (d_in) CubDebugExit(cudaFree(d_in));
+    if (d_out) CubDebugExit(cudaFree(d_out));
+    if (d_elapsed) CubDebugExit(cudaFree(d_elapsed));
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    g_uniform_keys = args.CheckCmdLineFlag("uniform");
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("grid-size", g_grid_size);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--i=<timing iterations (default:%d)>]"
+            "[--grid-size=<grid size (default:%d)>]"
+            "[--v] "
+            "\n", argv[0], g_timing_iterations, g_grid_size);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+    fflush(stdout);
+
+    // Run tests
+    printf("\nuint32:\n"); fflush(stdout);
+    Test<unsigned int, 128, 13>();
+    printf("\n"); fflush(stdout);
+
+    printf("\nfp32:\n"); fflush(stdout);
+    Test<float, 128, 13>();
+    printf("\n"); fflush(stdout);
+
+    printf("\nuint8:\n"); fflush(stdout);
+    Test<unsigned char, 128, 13>();
+    printf("\n"); fflush(stdout);
+
+    return 0;
+}
+
diff --git a/third_party/cub/examples/block/example_block_reduce.cu b/third_party/cub/examples/block/example_block_reduce.cu
new file mode 100644
index 0000000..bad8001
--- /dev/null
+++ b/third_party/cub/examples/block/example_block_reduce.cu
@@ -0,0 +1,290 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple demonstration of cub::BlockReduce
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_block_reduce.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console (define before including cub.h)
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <iostream>
+
+#include <cub/block/block_load.cuh>
+#include <cub/block/block_store.cuh>
+#include <cub/block/block_reduce.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+/// Verbose output
+bool g_verbose = false;
+
+/// Timing iterations
+int g_timing_iterations = 100;
+
+/// Default grid size
+int g_grid_size = 1;
+
+
+
+//---------------------------------------------------------------------
+// Kernels
+//---------------------------------------------------------------------
+
+/**
+ * Simple kernel for performing a block-wide exclusive prefix sum over integers
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    BlockReduceAlgorithm    ALGORITHM>
+__global__ void BlockSumKernel(
+    int         *d_in,          // Tile of input
+    int         *d_out,         // Tile aggregate
+    clock_t     *d_elapsed)     // Elapsed cycle count of block reduction
+{
+    // Specialize BlockReduce type for our thread block
+    typedef BlockReduce<int, BLOCK_THREADS, ALGORITHM> BlockReduceT;
+
+    // Shared memory
+    __shared__ typename BlockReduceT::TempStorage temp_storage;
+
+    // Per-thread tile data
+    int data[ITEMS_PER_THREAD];
+    LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_in, data);
+
+    // Start cycle timer
+    clock_t start = clock();
+
+    // Compute sum
+    int aggregate = BlockReduceT(temp_storage).Sum(data);
+
+    // Stop cycle timer
+    clock_t stop = clock();
+
+    // Store aggregate and elapsed clocks
+    if (threadIdx.x == 0)
+    {
+        *d_elapsed = (start > stop) ? start - stop : stop - start;
+        *d_out = aggregate;
+    }
+}
+
+
+
+//---------------------------------------------------------------------
+// Host utilities
+//---------------------------------------------------------------------
+
+/**
+ * Initialize reduction problem (and solution).
+ * Returns the aggregate
+ */
+int Initialize(int *h_in, int num_items)
+{
+    int inclusive = 0;
+
+    for (int i = 0; i < num_items; ++i)
+    {
+        h_in[i] = i % 17;
+        inclusive += h_in[i];
+    }
+
+    return inclusive;
+}
+
+
+/**
+ * Test thread block reduction
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    BlockReduceAlgorithm    ALGORITHM>
+void Test()
+{
+    const int TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    // Allocate host arrays
+    int *h_in           = new int[TILE_SIZE];
+    int *h_gpu          = new int[TILE_SIZE + 1];
+
+    // Initialize problem and reference output on host
+    int h_aggregate = Initialize(h_in, TILE_SIZE);
+
+    // Initialize device arrays
+    int *d_in           = NULL;
+    int *d_out          = NULL;
+    clock_t *d_elapsed  = NULL;
+    cudaMalloc((void**)&d_in,          sizeof(int) * TILE_SIZE);
+    cudaMalloc((void**)&d_out,         sizeof(int) * 1);
+    cudaMalloc((void**)&d_elapsed,     sizeof(clock_t));
+
+    // Display input problem data
+    if (g_verbose)
+    {
+        printf("Input data: ");
+        for (int i = 0; i < TILE_SIZE; i++)
+            printf("%d, ", h_in[i]);
+        printf("\n\n");
+    }
+
+    // Kernel props
+    int max_sm_occupancy;
+    CubDebugExit(MaxSmOccupancy(max_sm_occupancy, BlockSumKernel<BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM>, BLOCK_THREADS));
+
+    // Copy problem to device
+    cudaMemcpy(d_in, h_in, sizeof(int) * TILE_SIZE, cudaMemcpyHostToDevice);
+
+    printf("BlockReduce algorithm %s on %d items (%d timing iterations, %d blocks, %d threads, %d items per thread, %d SM occupancy):\n",
+        (ALGORITHM == BLOCK_REDUCE_RAKING) ? "BLOCK_REDUCE_RAKING" : "BLOCK_REDUCE_WARP_REDUCTIONS",
+        TILE_SIZE, g_timing_iterations, g_grid_size, BLOCK_THREADS, ITEMS_PER_THREAD, max_sm_occupancy);
+
+    // Run aggregate/prefix kernel
+    BlockSumKernel<BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM><<<g_grid_size, BLOCK_THREADS>>>(
+        d_in,
+        d_out,
+        d_elapsed);
+
+    // Check total aggregate
+    printf("\tAggregate: ");
+    int compare = CompareDeviceResults(&h_aggregate, d_out, 1, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Run this several times and average the performance results
+    GpuTimer    timer;
+    float       elapsed_millis          = 0.0;
+    clock_t     elapsed_clocks          = 0;
+
+    for (int i = 0; i < g_timing_iterations; ++i)
+    {
+        // Copy problem to device
+        cudaMemcpy(d_in, h_in, sizeof(int) * TILE_SIZE, cudaMemcpyHostToDevice);
+
+        timer.Start();
+
+        // Run aggregate/prefix kernel
+        BlockSumKernel<BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM><<<g_grid_size, BLOCK_THREADS>>>(
+            d_in,
+            d_out,
+            d_elapsed);
+
+        timer.Stop();
+        elapsed_millis += timer.ElapsedMillis();
+
+        // Copy clocks from device
+        clock_t clocks;
+        CubDebugExit(cudaMemcpy(&clocks, d_elapsed, sizeof(clock_t), cudaMemcpyDeviceToHost));
+        elapsed_clocks += clocks;
+
+    }
+
+    // Check for kernel errors and STDIO from the kernel, if any
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Display timing results
+    float avg_millis            = elapsed_millis / g_timing_iterations;
+    float avg_items_per_sec     = float(TILE_SIZE * g_grid_size) / avg_millis / 1000.0f;
+    float avg_clocks            = float(elapsed_clocks) / g_timing_iterations;
+    float avg_clocks_per_item   = avg_clocks / TILE_SIZE;
+
+    printf("\tAverage BlockReduce::Sum clocks: %.3f\n", avg_clocks);
+    printf("\tAverage BlockReduce::Sum clocks per item: %.3f\n", avg_clocks_per_item);
+    printf("\tAverage kernel millis: %.4f\n", avg_millis);
+    printf("\tAverage million items / sec: %.4f\n", avg_items_per_sec);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_gpu) delete[] h_gpu;
+    if (d_in) cudaFree(d_in);
+    if (d_out) cudaFree(d_out);
+    if (d_elapsed) cudaFree(d_elapsed);
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("grid-size", g_grid_size);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--i=<timing iterations>] "
+            "[--grid-size=<grid size>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Run tests
+    Test<1024, 1, BLOCK_REDUCE_RAKING>();
+    Test<512, 2, BLOCK_REDUCE_RAKING>();
+    Test<256, 4, BLOCK_REDUCE_RAKING>();
+    Test<128, 8, BLOCK_REDUCE_RAKING>();
+    Test<64, 16, BLOCK_REDUCE_RAKING>();
+    Test<32, 32, BLOCK_REDUCE_RAKING>();
+    Test<16, 64, BLOCK_REDUCE_RAKING>();
+
+    printf("-------------\n");
+
+    Test<1024, 1, BLOCK_REDUCE_WARP_REDUCTIONS>();
+    Test<512, 2, BLOCK_REDUCE_WARP_REDUCTIONS>();
+    Test<256, 4, BLOCK_REDUCE_WARP_REDUCTIONS>();
+    Test<128, 8, BLOCK_REDUCE_WARP_REDUCTIONS>();
+    Test<64, 16, BLOCK_REDUCE_WARP_REDUCTIONS>();
+    Test<32, 32, BLOCK_REDUCE_WARP_REDUCTIONS>();
+    Test<16, 64, BLOCK_REDUCE_WARP_REDUCTIONS>();
+
+    return 0;
+}
+
diff --git a/third_party/cub/examples/block/example_block_scan.cu b/third_party/cub/examples/block/example_block_scan.cu
new file mode 100644
index 0000000..fa709a5
--- /dev/null
+++ b/third_party/cub/examples/block/example_block_scan.cu
@@ -0,0 +1,334 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple demonstration of cub::BlockScan
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_block_scan.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console (define before including cub.h)
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <iostream>
+
+#include <cub/block/block_load.cuh>
+#include <cub/block/block_store.cuh>
+#include <cub/block/block_scan.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+/// Verbose output
+bool g_verbose = false;
+
+/// Timing iterations
+int g_timing_iterations = 100;
+
+/// Default grid size
+int g_grid_size = 1;
+
+
+
+//---------------------------------------------------------------------
+// Kernels
+//---------------------------------------------------------------------
+
+/**
+ * Simple kernel for performing a block-wide exclusive prefix sum over integers
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    BlockScanAlgorithm      ALGORITHM>
+__global__ void BlockPrefixSumKernel(
+    int         *d_in,          // Tile of input
+    int         *d_out,         // Tile of output
+    clock_t     *d_elapsed)     // Elapsed cycle count of block scan
+{
+    // Specialize BlockLoad type for our thread block (uses warp-striped loads for coalescing, then transposes in shared memory to a blocked arrangement)
+    typedef BlockLoad<int, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoadT;
+
+    // Specialize BlockStore type for our thread block (uses warp-striped loads for coalescing, then transposes in shared memory to a blocked arrangement)
+    typedef BlockStore<int, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_STORE_WARP_TRANSPOSE> BlockStoreT;
+
+    // Specialize BlockScan type for our thread block
+    typedef BlockScan<int, BLOCK_THREADS, ALGORITHM> BlockScanT;
+
+    // Shared memory
+    __shared__ union TempStorage
+    {
+        typename BlockLoadT::TempStorage    load;
+        typename BlockStoreT::TempStorage   store;
+        typename BlockScanT::TempStorage    scan;
+    } temp_storage;
+
+    // Per-thread tile data
+    int data[ITEMS_PER_THREAD];
+
+    // Load items into a blocked arrangement
+    BlockLoadT(temp_storage.load).Load(d_in, data);
+
+    // Barrier for smem reuse
+    __syncthreads();
+
+    // Start cycle timer
+    clock_t start = clock();
+
+    // Compute exclusive prefix sum
+    int aggregate;
+    BlockScanT(temp_storage.scan).ExclusiveSum(data, data, aggregate);
+
+    // Stop cycle timer
+    clock_t stop = clock();
+
+    // Barrier for smem reuse
+    __syncthreads();
+
+    // Store items from a blocked arrangement
+    BlockStoreT(temp_storage.store).Store(d_out, data);
+
+    // Store aggregate and elapsed clocks
+    if (threadIdx.x == 0)
+    {
+        *d_elapsed = (start > stop) ? start - stop : stop - start;
+        d_out[BLOCK_THREADS * ITEMS_PER_THREAD] = aggregate;
+    }
+}
+
+
+
+//---------------------------------------------------------------------
+// Host utilities
+//---------------------------------------------------------------------
+
+/**
+ * Initialize exclusive prefix sum problem (and solution).
+ * Returns the aggregate
+ */
+int Initialize(
+    int *h_in,
+    int *h_reference,
+    int num_items)
+{
+    int inclusive = 0;
+
+    for (int i = 0; i < num_items; ++i)
+    {
+        h_in[i] = i % 17;
+
+        h_reference[i] = inclusive;
+        inclusive += h_in[i];
+    }
+
+    return inclusive;
+}
+
+
+/**
+ * Test thread block scan
+ */
+template <
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    BlockScanAlgorithm  ALGORITHM>
+void Test()
+{
+    const int TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    // Allocate host arrays
+    int *h_in           = new int[TILE_SIZE];
+    int *h_reference    = new int[TILE_SIZE];
+    int *h_gpu          = new int[TILE_SIZE + 1];
+
+    // Initialize problem and reference output on host
+    int h_aggregate = Initialize(h_in, h_reference, TILE_SIZE);
+
+    // Initialize device arrays
+    int *d_in           = NULL;
+    int *d_out          = NULL;
+    clock_t *d_elapsed  = NULL;
+    cudaMalloc((void**)&d_in,          sizeof(int) * TILE_SIZE);
+    cudaMalloc((void**)&d_out,         sizeof(int) * (TILE_SIZE + 1));
+    cudaMalloc((void**)&d_elapsed,     sizeof(clock_t));
+
+    // Display input problem data
+    if (g_verbose)
+    {
+        printf("Input data: ");
+        for (int i = 0; i < TILE_SIZE; i++)
+            printf("%d, ", h_in[i]);
+        printf("\n\n");
+    }
+
+    // Kernel props
+    int max_sm_occupancy;
+    CubDebugExit(MaxSmOccupancy(max_sm_occupancy, BlockPrefixSumKernel<BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM>, BLOCK_THREADS));
+
+    // Copy problem to device
+    cudaMemcpy(d_in, h_in, sizeof(int) * TILE_SIZE, cudaMemcpyHostToDevice);
+
+    printf("BlockScan algorithm %s on %d items (%d timing iterations, %d blocks, %d threads, %d items per thread, %d SM occupancy):\n",
+        (ALGORITHM == BLOCK_SCAN_RAKING) ? "BLOCK_SCAN_RAKING" : (ALGORITHM == BLOCK_SCAN_RAKING_MEMOIZE) ? "BLOCK_SCAN_RAKING_MEMOIZE" : "BLOCK_SCAN_WARP_SCANS",
+        TILE_SIZE, g_timing_iterations, g_grid_size, BLOCK_THREADS, ITEMS_PER_THREAD, max_sm_occupancy);
+
+    // Run aggregate/prefix kernel
+    BlockPrefixSumKernel<BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM><<<g_grid_size, BLOCK_THREADS>>>(
+        d_in,
+        d_out,
+        d_elapsed);
+
+    // Check results
+    printf("\tOutput items: ");
+    int compare = CompareDeviceResults(h_reference, d_out, TILE_SIZE, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Check total aggregate
+    printf("\tAggregate: ");
+    compare = CompareDeviceResults(&h_aggregate, d_out + TILE_SIZE, 1, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Run this several times and average the performance results
+    GpuTimer    timer;
+    float       elapsed_millis          = 0.0;
+    clock_t     elapsed_clocks          = 0;
+
+    for (int i = 0; i < g_timing_iterations; ++i)
+    {
+        // Copy problem to device
+        cudaMemcpy(d_in, h_in, sizeof(int) * TILE_SIZE, cudaMemcpyHostToDevice);
+
+        timer.Start();
+
+        // Run aggregate/prefix kernel
+        BlockPrefixSumKernel<BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM><<<g_grid_size, BLOCK_THREADS>>>(
+            d_in,
+            d_out,
+            d_elapsed);
+
+        timer.Stop();
+        elapsed_millis += timer.ElapsedMillis();
+
+        // Copy clocks from device
+        clock_t clocks;
+        CubDebugExit(cudaMemcpy(&clocks, d_elapsed, sizeof(clock_t), cudaMemcpyDeviceToHost));
+        elapsed_clocks += clocks;
+
+    }
+
+    // Check for kernel errors and STDIO from the kernel, if any
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Display timing results
+    float avg_millis            = elapsed_millis / g_timing_iterations;
+    float avg_items_per_sec     = float(TILE_SIZE * g_grid_size) / avg_millis / 1000.0f;
+    float avg_clocks            = float(elapsed_clocks) / g_timing_iterations;
+    float avg_clocks_per_item   = avg_clocks / TILE_SIZE;
+
+    printf("\tAverage BlockScan::Sum clocks: %.3f\n", avg_clocks);
+    printf("\tAverage BlockScan::Sum clocks per item: %.3f\n", avg_clocks_per_item);
+    printf("\tAverage kernel millis: %.4f\n", avg_millis);
+    printf("\tAverage million items / sec: %.4f\n", avg_items_per_sec);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (h_gpu) delete[] h_gpu;
+    if (d_in) cudaFree(d_in);
+    if (d_out) cudaFree(d_out);
+    if (d_elapsed) cudaFree(d_elapsed);
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("grid-size", g_grid_size);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--i=<timing iterations (default:%d)>]"
+            "[--grid-size=<grid size (default:%d)>]"
+            "[--v] "
+            "\n", argv[0], g_timing_iterations, g_grid_size);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Run tests
+    Test<1024, 1, BLOCK_SCAN_RAKING>();
+    Test<512, 2, BLOCK_SCAN_RAKING>();
+    Test<256, 4, BLOCK_SCAN_RAKING>();
+    Test<128, 8, BLOCK_SCAN_RAKING>();
+    Test<64, 16, BLOCK_SCAN_RAKING>();
+    Test<32, 32, BLOCK_SCAN_RAKING>();
+
+    printf("-------------\n");
+
+    Test<1024, 1, BLOCK_SCAN_RAKING_MEMOIZE>();
+    Test<512, 2, BLOCK_SCAN_RAKING_MEMOIZE>();
+    Test<256, 4, BLOCK_SCAN_RAKING_MEMOIZE>();
+    Test<128, 8, BLOCK_SCAN_RAKING_MEMOIZE>();
+    Test<64, 16, BLOCK_SCAN_RAKING_MEMOIZE>();
+    Test<32, 32, BLOCK_SCAN_RAKING_MEMOIZE>();
+
+    printf("-------------\n");
+
+    Test<1024, 1, BLOCK_SCAN_WARP_SCANS>();
+    Test<512, 2, BLOCK_SCAN_WARP_SCANS>();
+    Test<256, 4, BLOCK_SCAN_WARP_SCANS>();
+    Test<128, 8, BLOCK_SCAN_WARP_SCANS>();
+    Test<64, 16, BLOCK_SCAN_WARP_SCANS>();
+    Test<32, 32, BLOCK_SCAN_WARP_SCANS>();
+
+
+    return 0;
+}
+
diff --git a/third_party/cub/examples/block/reduce_by_key.cu b/third_party/cub/examples/block/reduce_by_key.cu
new file mode 100644
index 0000000..d74e162
--- /dev/null
+++ b/third_party/cub/examples/block/reduce_by_key.cu
@@ -0,0 +1,57 @@
+
+
+#include <cub/cub.cuh>
+
+
+template <
+    int         BLOCK_THREADS,          ///< Number of CTA threads
+    typename    KeyT,                   ///< Key type
+    typename    ValueT>                 ///< Value type
+__global__ void Kernel()
+{
+    // Tuple type for scanning (pairs accumulated segment-value with segment-index)
+    typedef cub::KeyValuePair<int, ValueT> OffsetValuePairT;
+
+    // Reduce-value-by-segment scan operator
+    typedef cub::ReduceBySegmentOp<cub::Sum> ReduceBySegmentOpT;
+
+    // Parameterized BlockDiscontinuity type for setting head flags
+    typedef cub::BlockDiscontinuity<
+            KeyT,
+            BLOCK_THREADS>
+        BlockDiscontinuityKeysT;
+
+    // Parameterized BlockScan type
+    typedef cub::BlockScan<
+            OffsetValuePairT,
+            BLOCK_THREADS,
+            cub::BLOCK_SCAN_WARP_SCANS>
+        BlockScanT;
+
+    // Shared memory
+    __shared__ union TempStorage
+    {
+        typename BlockScanT::TempStorage                scan;           // Scan storage
+        typename BlockDiscontinuityKeysT::TempStorage   discontinuity;  // Discontinuity storage
+    } temp_storage;
+
+
+    // Read data (each thread gets 3 items each, every 9 items is a segment)
+    KeyT    my_keys[3]      = {threadIdx.x / 3, threadIdx.x / 3, threadIdx.x / 3};
+    ValueT  my_values[3]    = {1, 1, 1};
+
+    // Set head segment head flags
+    int     my_flags[3];
+    BlockDiscontinuityKeysT(temp_storage.discontinuity).FlagHeads(
+        my_flags,
+        my_keys,
+        cub::Inequality());
+
+    __syncthreads();
+
+
+
+
+
+
+}
diff --git a/third_party/cub/examples/device/.gitignore b/third_party/cub/examples/device/.gitignore
new file mode 100644
index 0000000..7032b5a
--- /dev/null
+++ b/third_party/cub/examples/device/.gitignore
@@ -0,0 +1,8 @@
+/bin
+/Debug
+/ipch
+/Release
+/cuda55.sdf
+/cuda55.suo
+/cuda60.sdf
+/cuda60.suo
diff --git a/third_party/cub/examples/device/Makefile b/third_party/cub/examples/device/Makefile
new file mode 100644
index 0000000..fea1494
--- /dev/null
+++ b/third_party/cub/examples/device/Makefile
@@ -0,0 +1,197 @@
+#/******************************************************************************
+# * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+# * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+# * 
+# * Redistribution and use in source and binary forms, with or without
+# * modification, are permitted provided that the following conditions are met:
+# *	 * Redistributions of source code must retain the above copyright
+# *	   notice, this list of conditions and the following disclaimer.
+# *	 * Redistributions in binary form must reproduce the above copyright
+# *	   notice, this list of conditions and the following disclaimer in the
+# *	   documentation and/or other materials provided with the distribution.
+# *	 * Neither the name of the NVIDIA CORPORATION nor the
+# *	   names of its contributors may be used to endorse or promote products
+# *	   derived from this software without specific prior written permission.
+# * 
+# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+# * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *
+#******************************************************************************/
+
+#-------------------------------------------------------------------------------
+#
+# Makefile usage
+#
+# make <target> [sm=<XXX,...>] [cdp=<0|1>] [force32=<0|1>] [abi=<0|1>] [open64=<0|1>] [verbose=<0|1>] [keep=<0|1>]
+#
+#-------------------------------------------------------------------------------
+ 
+include ../../common.mk 
+ 
+ 
+#-------------------------------------------------------------------------------
+# Includes
+#-------------------------------------------------------------------------------
+
+INC += -I$(CUB_DIR) -I$(CUB_DIR)test 
+
+
+
+#-------------------------------------------------------------------------------
+# Dependency Lists
+#-------------------------------------------------------------------------------
+
+rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
+
+DEPS =				$(CUB_DEPS) \
+					$(CUB_DIR)test/Makefile \
+					$(CUB_DIR)test/test_util.h \
+					$(CUB_DIR)test/mersenne.h \
+		
+ALL = 	example_device_partition_flagged \
+		example_device_partition_if \
+	 	example_device_radix_sort \
+		example_device_reduce \
+	 	example_device_scan \
+	 	example_device_select_unique \
+		example_device_select_flagged \
+		example_device_select_if \
+		example_device_sort_find_non_trivial_runs
+		
+
+
+#-------------------------------------------------------------------------------
+# make default
+#-------------------------------------------------------------------------------
+
+default:
+
+
+#-------------------------------------------------------------------------------
+# make clean
+#-------------------------------------------------------------------------------
+
+clean :
+	rm -f bin/*$(CPU_ARCH_SUFFIX)* 
+	rm -f *.i* *.cubin *.cu.c *.cudafe* *.fatbin.c *.ptx *.hash *.cu.cpp *.o
+
+
+#-------------------------------------------------------------------------------
+# make all
+#-------------------------------------------------------------------------------
+
+all : $(ALL)
+
+#-------------------------------------------------------------------------------
+# make run
+#-------------------------------------------------------------------------------
+
+run : 
+	for i in $(ALL); do ./bin/$${i}_$(BIN_SUFFIX) --device=$(device) || exit 1; done
+
+
+#-------------------------------------------------------------------------------
+# make example_device_reduce
+#-------------------------------------------------------------------------------
+
+example_device_reduce: bin/example_device_reduce_$(BIN_SUFFIX)
+
+bin/example_device_reduce_$(BIN_SUFFIX) : example_device_reduce.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_reduce_$(BIN_SUFFIX) example_device_reduce.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make example_device_partition_flagged
+#-------------------------------------------------------------------------------
+
+example_device_partition_flagged: bin/example_device_partition_flagged_$(BIN_SUFFIX)
+
+bin/example_device_partition_flagged_$(BIN_SUFFIX) : example_device_partition_flagged.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_partition_flagged_$(BIN_SUFFIX) example_device_partition_flagged.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+#-------------------------------------------------------------------------------
+# make example_device_partition_if
+#-------------------------------------------------------------------------------
+
+example_device_partition_if: bin/example_device_partition_if_$(BIN_SUFFIX)
+
+bin/example_device_partition_if_$(BIN_SUFFIX) : example_device_partition_if.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_partition_if_$(BIN_SUFFIX) example_device_partition_if.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+#-------------------------------------------------------------------------------
+# make example_device_scan
+#-------------------------------------------------------------------------------
+
+example_device_scan: bin/example_device_scan_$(BIN_SUFFIX)
+
+bin/example_device_scan_$(BIN_SUFFIX) : example_device_scan.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_scan_$(BIN_SUFFIX) example_device_scan.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make example_device_radix_sort
+#-------------------------------------------------------------------------------
+
+example_device_radix_sort: bin/example_device_radix_sort_$(BIN_SUFFIX)
+
+bin/example_device_radix_sort_$(BIN_SUFFIX) : example_device_radix_sort.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_radix_sort_$(BIN_SUFFIX) example_device_radix_sort.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make example_device_select_unique
+#-------------------------------------------------------------------------------
+
+example_device_select_unique: bin/example_device_select_unique_$(BIN_SUFFIX)
+
+bin/example_device_select_unique_$(BIN_SUFFIX) : example_device_select_unique.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_select_unique_$(BIN_SUFFIX) example_device_select_unique.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make example_device_select_flagged
+#-------------------------------------------------------------------------------
+
+example_device_select_flagged: bin/example_device_select_flagged_$(BIN_SUFFIX)
+
+bin/example_device_select_flagged_$(BIN_SUFFIX) : example_device_select_flagged.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_select_flagged_$(BIN_SUFFIX) example_device_select_flagged.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+#-------------------------------------------------------------------------------
+# make example_device_select_if
+#-------------------------------------------------------------------------------
+
+example_device_select_if: bin/example_device_select_if_$(BIN_SUFFIX)
+
+bin/example_device_select_if_$(BIN_SUFFIX) : example_device_select_if.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_select_if_$(BIN_SUFFIX) example_device_select_if.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make example_device_sort_find_non_trivial_runs
+#-------------------------------------------------------------------------------
+
+example_device_sort_find_non_trivial_runs: bin/example_device_sort_find_non_trivial_runs_$(BIN_SUFFIX)
+
+bin/example_device_sort_find_non_trivial_runs_$(BIN_SUFFIX) : example_device_sort_find_non_trivial_runs.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/example_device_sort_find_non_trivial_runs_$(BIN_SUFFIX) example_device_sort_find_non_trivial_runs.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+
diff --git a/third_party/cub/examples/device/example_device_partition_flagged.cu b/third_party/cub/examples/device/example_device_partition_flagged.cu
new file mode 100644
index 0000000..ae02b3c
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_partition_flagged.cu
@@ -0,0 +1,233 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of DevicePartition::Flagged().
+ *
+ * Partition flagged items from from a sequence of int keys using a
+ * corresponding sequence of unsigned char flags.
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_partition_flagged.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_partition.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem, setting flags at distances of random length
+ * chosen from [1..max_segment]
+ */
+void Initialize(
+    int             *h_in,
+    unsigned char   *h_flags,
+    int             num_items,
+    int             max_segment)
+{
+    unsigned short max_short = (unsigned short) -1;
+
+    int key = 0;
+    int i = 0;
+    while (i < num_items)
+    {
+        // Select number of repeating occurrences
+        unsigned short repeat;
+        RandomBits(repeat);
+        repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short))));
+        repeat = CUB_MAX(1, repeat);
+
+        int j = i;
+        while (j < CUB_MIN(i + repeat, num_items))
+        {
+            h_flags[j] = 0;
+            h_in[j] = key;
+            j++;
+        }
+
+        h_flags[i] = 1;
+        i = j;
+        key++;
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("Flags:\n");
+        DisplayResults(h_flags, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve unique problem
+ */
+int Solve(
+    int             *h_in,
+    unsigned char   *h_flags,
+    int             *h_reference,
+    int             num_items)
+{
+    int num_selected = 0;
+    for (int i = 0; i < num_items; ++i)
+    {
+        if (h_flags[i])
+        {
+            h_reference[num_selected] = h_in[i];
+            num_selected++;
+        }
+        else
+        {
+            h_reference[num_items - (i - num_selected) - 1] = h_in[i];
+        }
+    }
+
+    return num_selected;
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = 150;
+    int max_segment         = 40;       // Maximum segment length
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("maxseg", max_segment);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--device=<device-id>] "
+            "[--maxseg=<max segment length>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Allocate host arrays
+    int             *h_in        = new int[num_items];
+    int             *h_reference = new int[num_items];
+    unsigned char   *h_flags     = new unsigned char[num_items];
+
+    // Initialize problem and solution
+    Initialize(h_in, h_flags, num_items, max_segment);
+    int num_selected = Solve(h_in, h_flags, h_reference, num_items);
+
+    printf("cub::DevicePartition::Flagged %d items, %d selected (avg distance %d), %d-byte elements\n",
+        num_items, num_selected, (num_selected > 0) ? num_items / num_selected : 0, (int) sizeof(int));
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    int             *d_in = NULL;
+    unsigned char   *d_flags = NULL;
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(int) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_flags, sizeof(unsigned char) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_flags, h_flags, sizeof(unsigned char) * num_items, cudaMemcpyHostToDevice));
+
+    // Allocate device output array and num selected
+    int     *d_out            = NULL;
+    int     *d_num_selected_out   = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(int) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_selected_out, sizeof(int)));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(DevicePartition::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Run
+    CubDebugExit(DevicePartition::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_out, num_items, true, g_verbose);
+    printf("\t Data %s ", compare ? "FAIL" : "PASS");
+    compare |= CompareDeviceResults(&num_selected, d_num_selected_out, 1, true, g_verbose);
+    printf("\t Count %s ", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_num_selected_out) CubDebugExit(g_allocator.DeviceFree(d_num_selected_out));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_flags) CubDebugExit(g_allocator.DeviceFree(d_flags));
+
+    printf("\n\n");
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/examples/device/example_device_partition_if.cu b/third_party/cub/examples/device/example_device_partition_if.cu
new file mode 100644
index 0000000..7bf1c16
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_partition_if.cu
@@ -0,0 +1,244 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of DevicePartition::If().
+ *
+ * Partitions items from from a sequence of int keys using a
+ * section functor (greater-than)
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_select_if.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_partition.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+/// Selection functor type
+struct GreaterThan
+{
+    int compare;
+
+    __host__ __device__ __forceinline__
+    GreaterThan(int compare) : compare(compare) {}
+
+    __host__ __device__ __forceinline__
+    bool operator()(const int &a) const {
+        return (a > compare);
+    }
+};
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+/**
+ * Initialize problem, setting runs of random length chosen from [1..max_segment]
+ */
+void Initialize(
+    int     *h_in,
+    int     num_items,
+    int     max_segment)
+{
+    int key = 0;
+    int i = 0;
+    while (i < num_items)
+    {
+        // Randomly select number of repeating occurrences uniformly from [1..max_segment]
+        unsigned short max_short = (unsigned short) -1;
+        unsigned short repeat;
+        RandomBits(repeat);
+        repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short))));
+        repeat = CUB_MAX(1, repeat);
+
+        int j = i;
+        while (j < CUB_MIN(i + repeat, num_items))
+        {
+            h_in[j] = key;
+            j++;
+        }
+
+        i = j;
+        key++;
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve unique problem
+ */
+template <typename SelectOp>
+int Solve(
+    int             *h_in,
+    SelectOp        select_op,
+    int             *h_reference,
+    int             num_items)
+{
+    int num_selected = 0;
+    for (int i = 0; i < num_items; ++i)
+    {
+        if (select_op(h_in[i]))
+        {
+            h_reference[num_selected] = h_in[i];
+            num_selected++;
+        }
+        else
+        {
+            h_reference[num_items - (i - num_selected) - 1] = h_in[i];
+        }
+    }
+
+    return num_selected;
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = 150;
+    int max_segment         = 40;       // Maximum segment length
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("maxseg", max_segment);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--device=<device-id>] "
+            "[--maxseg=<max segment length>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Allocate host arrays
+    int *h_in        = new int[num_items];
+    int *h_reference = new int[num_items];
+
+    // DevicePartition a pivot index
+    unsigned int pivot_index;
+    unsigned int max_int = (unsigned int) -1;
+    RandomBits(pivot_index);
+    pivot_index = (unsigned int) ((float(pivot_index) * (float(num_items - 1) / float(max_int))));
+    printf("Pivot idx: %d\n", pivot_index); fflush(stdout);
+
+    // Initialize problem and solution
+    Initialize(h_in, num_items, max_segment);
+    GreaterThan select_op(h_in[pivot_index]);
+
+    int num_selected = Solve(h_in, select_op, h_reference, num_items);
+
+    printf("cub::DevicePartition::If %d items, %d selected (avg run length %d), %d-byte elements\n",
+        num_items, num_selected, (num_selected > 0) ? num_items / num_selected : 0, (int) sizeof(int));
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    int *d_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(int) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+
+    // Allocate device output array and num selected
+    int     *d_out            = NULL;
+    int     *d_num_selected_out   = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(int) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_selected_out, sizeof(int)));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(DevicePartition::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Run
+    CubDebugExit(DevicePartition::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_out, num_items, true, g_verbose);
+    printf("\t Data %s ", compare ? "FAIL" : "PASS");
+    compare = compare | CompareDeviceResults(&num_selected, d_num_selected_out, 1, true, g_verbose);
+    printf("\t Count %s ", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_num_selected_out) CubDebugExit(g_allocator.DeviceFree(d_num_selected_out));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    printf("\n\n");
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/examples/device/example_device_radix_sort.cu b/third_party/cub/examples/device/example_device_radix_sort.cu
new file mode 100644
index 0000000..1494ccb
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_radix_sort.cu
@@ -0,0 +1,226 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of DeviceRadixSort::SortPairs().
+ *
+ * Sorts an array of float keys paired with a corresponding array of int values.
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_radix_sort.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <algorithm>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_radix_sort.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+/**
+ * Simple key-value pairing for floating point types.  Distinguishes
+ * between positive and negative zero.
+ */
+struct Pair
+{
+    float   key;
+    int     value;
+
+    bool operator<(const Pair &b) const
+    {
+        if (key < b.key)
+            return true;
+
+        if (key > b.key)
+            return false;
+
+        // Return true if key is negative zero and b.key is positive zero
+        unsigned int key_bits   = *reinterpret_cast<unsigned*>(const_cast<float*>(&key));
+        unsigned int b_key_bits = *reinterpret_cast<unsigned*>(const_cast<float*>(&b.key));
+        unsigned int HIGH_BIT   = 1u << 31;
+
+        return ((key_bits & HIGH_BIT) != 0) && ((b_key_bits & HIGH_BIT) == 0);
+    }
+};
+
+
+/**
+ * Initialize key-value sorting problem.
+ */
+void Initialize(
+    float           *h_keys,
+    int             *h_values,
+    float           *h_reference_keys,
+    int             *h_reference_values,
+    int             num_items)
+{
+    Pair *h_pairs = new Pair[num_items];
+
+    for (int i = 0; i < num_items; ++i)
+    {
+        RandomBits(h_keys[i]);
+        RandomBits(h_values[i]);
+        h_pairs[i].key    = h_keys[i];
+        h_pairs[i].value  = h_values[i];
+    }
+
+    if (g_verbose)
+    {
+        printf("Input keys:\n");
+        DisplayResults(h_keys, num_items);
+        printf("\n\n");
+
+        printf("Input values:\n");
+        DisplayResults(h_values, num_items);
+        printf("\n\n");
+    }
+
+    std::stable_sort(h_pairs, h_pairs + num_items);
+
+    for (int i = 0; i < num_items; ++i)
+    {
+        h_reference_keys[i]     = h_pairs[i].key;
+        h_reference_values[i]   = h_pairs[i].value;
+    }
+
+    delete[] h_pairs;
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items = 150;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--device=<device-id>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    printf("cub::DeviceRadixSort::SortPairs() %d items (%d-byte keys %d-byte values)\n",
+        num_items, int(sizeof(float)), int(sizeof(int)));
+    fflush(stdout);
+
+    // Allocate host arrays
+    float   *h_keys             = new float[num_items];
+    float   *h_reference_keys   = new float[num_items];
+    int     *h_values           = new int[num_items];
+    int     *h_reference_values = new int[num_items];
+
+    // Initialize problem and solution on host
+    Initialize(h_keys, h_values, h_reference_keys, h_reference_values, num_items);
+
+    // Allocate device arrays
+    DoubleBuffer<float> d_keys;
+    DoubleBuffer<int>   d_values;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys.d_buffers[0], sizeof(float) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys.d_buffers[1], sizeof(float) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values.d_buffers[0], sizeof(int) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values.d_buffers[1], sizeof(int) * num_items));
+
+    // Allocate temporary storage
+    size_t  temp_storage_bytes  = 0;
+    void    *d_temp_storage     = NULL;
+
+    CubDebugExit(DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Initialize device arrays
+    CubDebugExit(cudaMemcpy(d_keys.d_buffers[d_keys.selector], h_keys, sizeof(float) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_values.d_buffers[d_values.selector], h_values, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+
+    // Run
+    CubDebugExit(DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference_keys, d_keys.Current(), num_items, true, g_verbose);
+    printf("\t Compare keys (selector %d): %s\n", d_keys.selector, compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+    compare = CompareDeviceResults(h_reference_values, d_values.Current(), num_items, true, g_verbose);
+    printf("\t Compare values (selector %d): %s\n", d_values.selector, compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_keys) delete[] h_keys;
+    if (h_reference_keys) delete[] h_reference_keys;
+    if (h_values) delete[] h_values;
+    if (h_reference_values) delete[] h_reference_values;
+
+    if (d_keys.d_buffers[0]) CubDebugExit(g_allocator.DeviceFree(d_keys.d_buffers[0]));
+    if (d_keys.d_buffers[1]) CubDebugExit(g_allocator.DeviceFree(d_keys.d_buffers[1]));
+    if (d_values.d_buffers[0]) CubDebugExit(g_allocator.DeviceFree(d_values.d_buffers[0]));
+    if (d_values.d_buffers[1]) CubDebugExit(g_allocator.DeviceFree(d_values.d_buffers[1]));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    printf("\n\n");
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/examples/device/example_device_reduce.cu b/third_party/cub/examples/device/example_device_reduce.cu
new file mode 100644
index 0000000..fc8fddb
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_reduce.cu
@@ -0,0 +1,180 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of DeviceReduce::Sum().
+ *
+ * Sums an array of int keys.
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_reduce.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_reduce.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+/**
+ * Initialize problem
+ */
+void Initialize(
+    int   *h_in,
+    int     num_items)
+{
+    for (int i = 0; i < num_items; ++i)
+        h_in[i] = i;
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Compute solution
+ */
+void Solve(
+    int           *h_in,
+    int           &h_reference,
+    int             num_items)
+{
+    for (int i = 0; i < num_items; ++i)
+    {
+        if (i == 0)
+            h_reference = h_in[0];
+        else
+            h_reference += h_in[i];
+    }
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items = 150;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--device=<device-id>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    printf("cub::DeviceReduce::Sum() %d items (%d-byte elements)\n",
+        num_items, (int) sizeof(int));
+    fflush(stdout);
+
+    // Allocate host arrays
+    int* h_in = new int[num_items];
+    int  h_reference;
+
+    // Initialize problem and solution
+    Initialize(h_in, num_items);
+    Solve(h_in, h_reference, num_items);
+
+    // Allocate problem device arrays
+    int *d_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(int) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+
+    // Allocate device output array
+    int *d_out = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(int) * 1));
+
+    // Request and allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Run
+    CubDebugExit(DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(&h_reference, d_out, 1, g_verbose, g_verbose);
+    printf("\t%s", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    printf("\n\n");
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/examples/device/example_device_scan.cu b/third_party/cub/examples/device/example_device_scan.cu
new file mode 100644
index 0000000..3c85526
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_scan.cu
@@ -0,0 +1,186 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of DeviceScan::ExclusiveSum().
+ *
+ * Computes an exclusive sum of int keys.
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_scan.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem
+ */
+void Initialize(
+    int        *h_in,
+    int          num_items)
+{
+    for (int i = 0; i < num_items; ++i)
+        h_in[i] = i;
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+/**
+ * Solve exclusive-scan problem
+ */
+int Solve(
+    int           *h_in,
+    int           *h_reference,
+    int             num_items)
+{
+    int inclusive = 0;
+    int aggregate = 0;
+
+    for (int i = 0; i < num_items; ++i)
+    {
+        h_reference[i] = inclusive;
+        inclusive += h_in[i];
+        aggregate += h_in[i];
+    }
+
+    return aggregate;
+}
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items = 150;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--device=<device-id>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    printf("cub::DeviceScan::ExclusiveSum %d items (%d-byte elements)\n",
+        num_items, (int) sizeof(int));
+    fflush(stdout);
+
+    // Allocate host arrays
+    int*  h_in = new int[num_items];
+    int*  h_reference = new int[num_items];
+
+    // Initialize problem and solution
+    Initialize(h_in, num_items);
+    Solve(h_in, h_reference, num_items);
+
+    // Allocate problem device arrays
+    int *d_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(int) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+
+    // Allocate device output array
+    int *d_out = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(int) * num_items));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Run
+    CubDebugExit(DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_out, num_items, true, g_verbose);
+    printf("\t%s", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    printf("\n\n");
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/examples/device/example_device_select_flagged.cu b/third_party/cub/examples/device/example_device_select_flagged.cu
new file mode 100644
index 0000000..12581f8
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_select_flagged.cu
@@ -0,0 +1,233 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of DeviceSelect::Flagged().
+ *
+ * Selects flagged items from from a sequence of int keys using a
+ * corresponding sequence of unsigned char flags.
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_select_flagged.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_select.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem, setting flags at distances of random length
+ * chosen from [1..max_segment]
+ */
+void Initialize(
+    int             *h_in,
+    unsigned char   *h_flags,
+    int             num_items,
+    int             max_segment)
+{
+    unsigned short max_short = (unsigned short) -1;
+
+    int key = 0;
+    int i = 0;
+    while (i < num_items)
+    {
+        // Select number of repeating occurrences
+        unsigned short repeat;
+        RandomBits(repeat);
+        repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short))));
+        repeat = CUB_MAX(1, repeat);
+
+        int j = i;
+        while (j < CUB_MIN(i + repeat, num_items))
+        {
+            h_flags[j] = 0;
+            h_in[j] = key;
+            j++;
+        }
+
+        h_flags[i] = 1;
+        i = j;
+        key++;
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("Flags:\n");
+        DisplayResults(h_flags, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve unique problem
+ */
+int Solve(
+    int             *h_in,
+    unsigned char   *h_flags,
+    int             *h_reference,
+    int             num_items)
+{
+    int num_selected = 0;
+    for (int i = 0; i < num_items; ++i)
+    {
+        if (h_flags[i])
+        {
+            h_reference[num_selected] = h_in[i];
+            num_selected++;
+        }
+        else
+        {
+            h_reference[num_items - (i - num_selected) - 1] = h_in[i];
+        }
+    }
+
+    return num_selected;
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = 150;
+    int max_segment         = 40;       // Maximum segment length
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("maxseg", max_segment);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--device=<device-id>] "
+            "[--maxseg=<max segment length>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Allocate host arrays
+    int             *h_in        = new int[num_items];
+    int             *h_reference = new int[num_items];
+    unsigned char   *h_flags     = new unsigned char[num_items];
+
+    // Initialize problem and solution
+    Initialize(h_in, h_flags, num_items, max_segment);
+    int num_selected = Solve(h_in, h_flags, h_reference, num_items);
+
+    printf("cub::DeviceSelect::Flagged %d items, %d selected (avg distance %d), %d-byte elements\n",
+        num_items, num_selected, (num_selected > 0) ? num_items / num_selected : 0, (int) sizeof(int));
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    int             *d_in = NULL;
+    unsigned char   *d_flags = NULL;
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(int) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_flags, sizeof(unsigned char) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_flags, h_flags, sizeof(unsigned char) * num_items, cudaMemcpyHostToDevice));
+
+    // Allocate device output array and num selected
+    int     *d_out            = NULL;
+    int     *d_num_selected_out   = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(int) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_selected_out, sizeof(int)));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(DeviceSelect::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Run
+    CubDebugExit(DeviceSelect::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_out, num_selected, true, g_verbose);
+    printf("\t Data %s ", compare ? "FAIL" : "PASS");
+    compare |= CompareDeviceResults(&num_selected, d_num_selected_out, 1, true, g_verbose);
+    printf("\t Count %s ", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_num_selected_out) CubDebugExit(g_allocator.DeviceFree(d_num_selected_out));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_flags) CubDebugExit(g_allocator.DeviceFree(d_flags));
+
+    printf("\n\n");
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/examples/device/example_device_select_if.cu b/third_party/cub/examples/device/example_device_select_if.cu
new file mode 100644
index 0000000..689c99b
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_select_if.cu
@@ -0,0 +1,242 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of DeviceSelect::If().
+ *
+ * Selects items from from a sequence of int keys using a
+ * section functor (greater-than)
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_select_if.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_select.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+/// Selection functor type
+struct GreaterThan
+{
+    int compare;
+
+    __host__ __device__ __forceinline__
+    GreaterThan(int compare) : compare(compare) {}
+
+    __host__ __device__ __forceinline__
+    bool operator()(const int &a) const {
+        return (a > compare);
+    }
+};
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+/**
+ * Initialize problem, setting runs of random length chosen from [1..max_segment]
+ */
+void Initialize(
+    int     *h_in,
+    int     num_items,
+    int     max_segment)
+{
+    int key = 0;
+    int i = 0;
+    while (i < num_items)
+    {
+        // Randomly select number of repeating occurrences uniformly from [1..max_segment]
+        unsigned short max_short = (unsigned short) -1;
+        unsigned short repeat;
+        RandomBits(repeat);
+        repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short))));
+        repeat = CUB_MAX(1, repeat);
+
+        int j = i;
+        while (j < CUB_MIN(i + repeat, num_items))
+        {
+            h_in[j] = key;
+            j++;
+        }
+
+        i = j;
+        key++;
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve unique problem
+ */
+template <typename SelectOp>
+int Solve(
+    int             *h_in,
+    SelectOp        select_op,
+    int             *h_reference,
+    int             num_items)
+{
+    int num_selected = 0;
+    for (int i = 0; i < num_items; ++i)
+    {
+        if (select_op(h_in[i]))
+        {
+            h_reference[num_selected] = h_in[i];
+            num_selected++;
+        }
+        else
+        {
+            h_reference[num_items - (i - num_selected) - 1] = h_in[i];
+        }
+    }
+
+    return num_selected;
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = 150;
+    int max_segment         = 40;       // Maximum segment length
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("maxseg", max_segment);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--device=<device-id>] "
+            "[--maxseg=<max segment length>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Allocate host arrays
+    int *h_in        = new int[num_items];
+    int *h_reference = new int[num_items];
+
+    // Select a pivot index
+    unsigned int pivot_index;
+    unsigned int max_int = (unsigned int) -1;
+    RandomBits(pivot_index);
+    pivot_index = (unsigned int) ((float(pivot_index) * (float(num_items - 1) / float(max_int))));
+    printf("Pivot idx: %d\n", pivot_index); fflush(stdout);
+
+    // Initialize problem and solution
+    Initialize(h_in, num_items, max_segment);
+    GreaterThan select_op(h_in[pivot_index]);
+
+    int num_selected = Solve(h_in, select_op, h_reference, num_items);
+
+    printf("cub::DeviceSelect::If %d items, %d selected (avg run length %d), %d-byte elements\n",
+        num_items, num_selected, (num_selected > 0) ? num_items / num_selected : 0, (int) sizeof(int));
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    int *d_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(int) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+
+    // Allocate device output array and num selected
+    int     *d_out            = NULL;
+    int     *d_num_selected_out   = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(int) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_selected_out, sizeof(int)));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Run
+    CubDebugExit(DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_out, num_selected, true, g_verbose);
+    printf("\t Data %s ", compare ? "FAIL" : "PASS");
+    compare = compare | CompareDeviceResults(&num_selected, d_num_selected_out, 1, true, g_verbose);
+    printf("\t Count %s ", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_num_selected_out) CubDebugExit(g_allocator.DeviceFree(d_num_selected_out));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    printf("\n\n");
+
+    return 0;
+}
+
diff --git a/third_party/cub/examples/device/example_device_select_unique.cu b/third_party/cub/examples/device/example_device_select_unique.cu
new file mode 100644
index 0000000..e9cefd5
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_select_unique.cu
@@ -0,0 +1,221 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of DeviceSelect::Unique().
+ *
+ * Selects the first element from each run of identical values from a sequence
+ * of int keys.
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_select_unique.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_select.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem, setting runs of random length chosen from [1..max_segment]
+ */
+void Initialize(
+    int     *h_in,
+    int     num_items,
+    int     max_segment)
+{
+    int key = 0;
+    int i = 0;
+    while (i < num_items)
+    {
+        // Randomly select number of repeating occurrences uniformly from [1..max_segment]
+        unsigned short max_short = (unsigned short) -1;
+        unsigned short repeat;
+        RandomBits(repeat);
+        repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short))));
+        repeat = CUB_MAX(1, repeat);
+
+        int j = i;
+        while (j < CUB_MIN(i + repeat, num_items))
+        {
+            h_in[j] = key;
+            j++;
+        }
+
+        i = j;
+        key++;
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve unique problem
+ */
+int Solve(
+    int         *h_in,
+    int         *h_reference,
+    int         num_items)
+{
+    int num_selected = 0;
+    if (num_items > 0)
+    {
+        h_reference[num_selected] = h_in[0];
+        num_selected++;
+    }
+
+    for (int i = 1; i < num_items; ++i)
+    {
+        if (h_in[i] != h_in[i - 1])
+        {
+            h_reference[num_selected] = h_in[i];
+            num_selected++;
+        }
+    }
+
+    return num_selected;
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = 150;
+    int max_segment         = 40;       // Maximum segment length
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("maxseg", max_segment);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--device=<device-id>] "
+            "[--maxseg=<max segment length>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Allocate host arrays
+    int*  h_in        = new int[num_items];
+    int*  h_reference = new int[num_items];
+
+    // Initialize problem and solution
+    Initialize(h_in, num_items, max_segment);
+    int num_selected = Solve(h_in, h_reference, num_items);
+
+    printf("cub::DeviceSelect::Unique %d items (%d-byte elements), %d selected (avg run length %d)\n",
+        num_items, (int) sizeof(int), num_selected, num_items / num_selected);
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    int *d_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(int) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+
+    // Allocate device output array and num selected
+    int     *d_out            = NULL;
+    int     *d_num_selected_out   = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(int) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_selected_out, sizeof(int)));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(DeviceSelect::Unique(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Run
+    CubDebugExit(DeviceSelect::Unique(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_out, num_selected, true, g_verbose);
+    printf("\t Data %s ", compare ? "FAIL" : "PASS");
+    compare = compare | CompareDeviceResults(&num_selected, d_num_selected_out, 1, true, g_verbose);
+    printf("\t Count %s ", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_num_selected_out) CubDebugExit(g_allocator.DeviceFree(d_num_selected_out));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    printf("\n\n");
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/examples/device/example_device_sort_find_non_trivial_runs.cu b/third_party/cub/examples/device/example_device_sort_find_non_trivial_runs.cu
new file mode 100644
index 0000000..ed70248
--- /dev/null
+++ b/third_party/cub/examples/device/example_device_sort_find_non_trivial_runs.cu
@@ -0,0 +1,384 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Simple example of sorting a sequence of keys and values (each pair is a
+ * randomly-selected int32 paired with its original offset in the unsorted sequence), and then
+ * isolating all maximal, non-trivial (having length > 1) "runs" of duplicates.
+ *
+ * To compile using the command line:
+ *   nvcc -arch=sm_XX example_device_sort_find_non_trivial_runs.cu -I../.. -lcudart -O3
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <algorithm>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_radix_sort.cuh>
+#include <cub/device/device_run_length_encode.cuh>
+
+#include "../../test/test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+/**
+ * Simple key-value pairing for using std::sort on key-value pairs.
+ */
+template <typename Key, typename Value>
+struct Pair
+{
+    Key     key;
+    Value   value;
+
+    bool operator<(const Pair &b) const
+    {
+        return (key < b.key);
+    }
+};
+
+
+/**
+ * Pair ostream operator
+ */
+template <typename Key, typename Value>
+std::ostream& operator<<(std::ostream& os, const Pair<Key, Value>& val)
+{
+    os << '<' << val.key << ',' << val.value << '>';
+    return os;
+}
+
+
+/**
+ * Initialize problem
+ */
+template <typename Key, typename Value>
+void Initialize(
+    Key    *h_keys,
+    Value  *h_values,
+    int    num_items,
+    int    max_key)
+{
+    float scale = float(max_key) / float(UINT_MAX);
+    for (int i = 0; i < num_items; ++i)
+    {
+        Key sample;
+        RandomBits(sample);
+        h_keys[i] = (max_key == -1) ? i : (Key) (scale * sample);
+        h_values[i] = i;
+    }
+
+    if (g_verbose)
+    {
+        printf("Keys:\n");
+        DisplayResults(h_keys, num_items);
+        printf("\n\n");
+
+        printf("Values:\n");
+        DisplayResults(h_values, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve sorted non-trivial subrange problem.  Returns the number
+ * of non-trivial runs found.
+ */
+template <typename Key, typename Value>
+int Solve(
+    Key     *h_keys,
+    Value   *h_values,
+    int     num_items,
+    int     *h_offsets_reference,
+    int     *h_lengths_reference)
+{
+    // Sort
+
+    Pair<Key, Value> *h_pairs = new Pair<Key, Value>[num_items];
+    for (int i = 0; i < num_items; ++i)
+    {
+        h_pairs[i].key    = h_keys[i];
+        h_pairs[i].value  = h_values[i];
+    }
+
+    std::stable_sort(h_pairs, h_pairs + num_items);
+
+    if (g_verbose)
+    {
+        printf("Sorted pairs:\n");
+        DisplayResults(h_pairs, num_items);
+        printf("\n\n");
+    }
+
+    // Find non-trivial runs
+
+    Key     previous        = h_pairs[0].key;
+    int     length          = 1;
+    int     num_runs        = 0;
+    int     run_begin       = 0;
+
+    for (int i = 1; i < num_items; ++i)
+    {
+        if (previous != h_pairs[i].key)
+        {
+            if (length > 1)
+            {
+                h_offsets_reference[num_runs]     = run_begin;
+                h_lengths_reference[num_runs]     = length;
+                num_runs++;
+            }
+            length = 1;
+            run_begin = i;
+        }
+        else
+        {
+            length++;
+        }
+        previous = h_pairs[i].key;
+    }
+
+    if (length > 1)
+    {
+        h_offsets_reference[num_runs]   = run_begin;
+        h_lengths_reference[num_runs]   = length;
+        num_runs++;
+    }
+
+    delete[] h_pairs;
+
+    return num_runs;
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    typedef unsigned int    Key;
+    typedef int             Value;
+
+    int timing_iterations   = 0;
+    int num_items           = 40;
+    Key max_key             = 20;       // Max item
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("maxkey", max_key);
+    args.GetCmdLineArgument("i", timing_iterations);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--i=<timing iterations> "
+            "[--n=<input items, default 40> "
+            "[--maxkey=<max key, default 20 (use -1 to test only unique keys)>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Allocate host arrays (problem and reference solution)
+
+    Key     *h_keys                 = new Key[num_items];
+    Value   *h_values               = new Value[num_items];
+    int     *h_offsets_reference    = new int[num_items];
+    int     *h_lengths_reference    = new int[num_items];
+
+    // Initialize key-value pairs and compute reference solution (sort them, and identify non-trivial runs)
+    printf("Computing reference solution on CPU for %d items (max key %d)\n", num_items, max_key);
+    fflush(stdout);
+
+    Initialize(h_keys, h_values, num_items, max_key);
+    int num_runs = Solve(h_keys, h_values, num_items, h_offsets_reference, h_lengths_reference);
+
+    printf("%d non-trivial runs\n", num_runs);
+    fflush(stdout);
+
+    // Repeat for performance timing
+    GpuTimer gpu_timer;
+    GpuTimer gpu_rle_timer;
+    float elapsed_millis = 0.0;
+    float elapsed_rle_millis = 0.0;
+    for (int i = 0; i <= timing_iterations; ++i)
+    {
+
+        // Allocate and initialize device arrays for sorting
+        DoubleBuffer<Key>       d_keys;
+        DoubleBuffer<Value>     d_values;
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys.d_buffers[0], sizeof(Key) * num_items));
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys.d_buffers[1], sizeof(Key) * num_items));
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values.d_buffers[0], sizeof(Value) * num_items));
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values.d_buffers[1], sizeof(Value) * num_items));
+
+        CubDebugExit(cudaMemcpy(d_keys.d_buffers[d_keys.selector], h_keys, sizeof(float) * num_items, cudaMemcpyHostToDevice));
+        CubDebugExit(cudaMemcpy(d_values.d_buffers[d_values.selector], h_values, sizeof(int) * num_items, cudaMemcpyHostToDevice));
+
+        // Start timer
+        gpu_timer.Start();
+
+        // Allocate temporary storage for sorting
+        size_t  temp_storage_bytes  = 0;
+        void    *d_temp_storage     = NULL;
+        CubDebugExit(DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items));
+        CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+        // Do the sort
+        CubDebugExit(DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items));
+
+        // Free unused buffers and sorting temporary storage
+        if (d_keys.d_buffers[d_keys.selector ^ 1]) CubDebugExit(g_allocator.DeviceFree(d_keys.d_buffers[d_keys.selector ^ 1]));
+        if (d_values.d_buffers[d_values.selector ^ 1]) CubDebugExit(g_allocator.DeviceFree(d_values.d_buffers[d_values.selector ^ 1]));
+        if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+        // Start timer
+        gpu_rle_timer.Start();
+
+        // Allocate device arrays for enumerating non-trivial runs
+        int     *d_offests_out   = NULL;
+        int     *d_lengths_out   = NULL;
+        int     *d_num_runs      = NULL;
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_offests_out, sizeof(int) * num_items));
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_lengths_out, sizeof(int) * num_items));
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_runs, sizeof(int) * 1));
+
+        // Allocate temporary storage for isolating non-trivial runs
+        d_temp_storage = NULL;
+        CubDebugExit(DeviceRunLengthEncode::NonTrivialRuns(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys.d_buffers[d_keys.selector],
+            d_offests_out,
+            d_lengths_out,
+            d_num_runs,
+            num_items));
+        CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+        // Do the isolation
+        CubDebugExit(DeviceRunLengthEncode::NonTrivialRuns(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys.d_buffers[d_keys.selector],
+            d_offests_out,
+            d_lengths_out,
+            d_num_runs,
+            num_items));
+
+        // Free keys buffer
+        if (d_keys.d_buffers[d_keys.selector]) CubDebugExit(g_allocator.DeviceFree(d_keys.d_buffers[d_keys.selector]));
+
+        //
+        // Hypothetically do stuff with the original key-indices corresponding to non-trivial runs of identical keys
+        //
+
+        // Stop sort timer
+        gpu_timer.Stop();
+        gpu_rle_timer.Stop();
+
+        if (i == 0)
+        {
+            // First iteration is a warmup: // Check for correctness (and display results, if specified)
+
+            printf("\nRUN OFFSETS: \n");
+            int compare = CompareDeviceResults(h_offsets_reference, d_offests_out, num_runs, true, g_verbose);
+            printf("\t\t %s ", compare ? "FAIL" : "PASS");
+
+            printf("\nRUN LENGTHS: \n");
+            compare |= CompareDeviceResults(h_lengths_reference, d_lengths_out, num_runs, true, g_verbose);
+            printf("\t\t %s ", compare ? "FAIL" : "PASS");
+
+            printf("\nNUM RUNS: \n");
+            compare |= CompareDeviceResults(&num_runs, d_num_runs, 1, true, g_verbose);
+            printf("\t\t %s ", compare ? "FAIL" : "PASS");
+
+            AssertEquals(0, compare);
+        }
+        else
+        {
+            elapsed_millis += gpu_timer.ElapsedMillis();
+            elapsed_rle_millis += gpu_rle_timer.ElapsedMillis();
+        }
+
+        // GPU cleanup
+
+        if (d_values.d_buffers[d_values.selector]) CubDebugExit(g_allocator.DeviceFree(d_values.d_buffers[d_values.selector]));
+        if (d_offests_out) CubDebugExit(g_allocator.DeviceFree(d_offests_out));
+        if (d_lengths_out) CubDebugExit(g_allocator.DeviceFree(d_lengths_out));
+        if (d_num_runs) CubDebugExit(g_allocator.DeviceFree(d_num_runs));
+        if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+    }
+
+    // Host cleanup
+    if (h_keys) delete[] h_keys;
+    if (h_values) delete[] h_values;
+    if (h_offsets_reference) delete[] h_offsets_reference;
+    if (h_lengths_reference) delete[] h_lengths_reference;
+
+    printf("\n\n");
+
+    if (timing_iterations > 0)
+    {
+        printf("%d timing iterations, average time to sort and isolate non-trivial duplicates: %.3f ms (%.3f ms spent in RLE isolation)\n",
+            timing_iterations,
+            elapsed_millis / timing_iterations,
+            elapsed_rle_millis / timing_iterations);
+    }
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/experimental/.gitignore b/third_party/cub/experimental/.gitignore
new file mode 100644
index 0000000..5e56e04
--- /dev/null
+++ b/third_party/cub/experimental/.gitignore
@@ -0,0 +1 @@
+/bin
diff --git a/third_party/cub/experimental/Makefile b/third_party/cub/experimental/Makefile
new file mode 100644
index 0000000..7165d93
--- /dev/null
+++ b/third_party/cub/experimental/Makefile
@@ -0,0 +1,125 @@
+#/******************************************************************************
+# * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+# * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+# * 
+# * Redistribution and use in source and binary forms, with or without
+# * modification, are permitted provided that the following conditions are met:
+# *	 * Redistributions of source code must retain the above copyright
+# *	   notice, this list of conditions and the following disclaimer.
+# *	 * Redistributions in binary form must reproduce the above copyright
+# *	   notice, this list of conditions and the following disclaimer in the
+# *	   documentation and/or other materials provided with the distribution.
+# *	 * Neither the name of the NVIDIA CORPORATION nor the
+# *	   names of its contributors may be used to endorse or promote products
+# *	   derived from this software without specific prior written permission.
+# * 
+# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+# * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *
+#******************************************************************************/
+
+#-------------------------------------------------------------------------------
+#
+# Makefile usage
+#
+# make <target> [sm=<XXX,...>] [cdp=<0|1>] [force32=<0|1>] [abi=<0|1>] [open64=<0|1>] [verbose=<0|1>] [keep=<0|1>] [quicktest=<0|1>]
+#
+#-------------------------------------------------------------------------------
+ 
+include ../common.mk 
+
+#-------------------------------------------------------------------------------
+# Commandline Options
+#-------------------------------------------------------------------------------
+
+# [mkl=<0|1>] compile against Intel MKL
+ifeq ($(mkl), 1)
+	DEFINES 	+= -DCUB_MKL
+
+ifeq (WIN_NT, $(findstring WIN_NT, $(OSUPPER)))
+	LIBS 		+=	mkl_intel_lp64.lib mkl_intel_thread.lib  mkl_core.lib libiomp5md.lib
+	NVCCFLAGS 	+= -Xcompiler /openmp
+else
+	LIBS		+= -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -liomp5 -lpthread -lm
+	NVCCFLAGS 	+= -Xcompiler -fopenmp
+	
+endif	
+
+endif
+
+
+#-------------------------------------------------------------------------------
+# Compiler and compilation platform
+#-------------------------------------------------------------------------------
+
+# Includes
+INC += -I$(CUB_DIR) -I$(CUB_DIR)test 
+
+# detect OS
+OSUPPER = $(shell uname -s 2>/dev/null | tr [:lower:] [:upper:])
+
+#-------------------------------------------------------------------------------
+# Dependency Lists
+#-------------------------------------------------------------------------------
+
+exp_rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
+
+EXP_DEPS = 	$(call rwildcard, ./,*.cuh) \
+			$(call rwildcard, ./,*.h)
+
+DEPS =				$(CUB_DEPS) \
+					$(EXP_DEPS) \
+					$(CUB_DIR)test/Makefile \
+					$(CUB_DIR)test/test_util.h \
+					$(CUB_DIR)test/mersenne.h \
+
+		
+
+#-------------------------------------------------------------------------------
+# make default
+#-------------------------------------------------------------------------------
+
+default:
+
+
+#-------------------------------------------------------------------------------
+# make clean
+#-------------------------------------------------------------------------------
+
+clean :
+	rm -f bin/*$(CPU_ARCH_SUFFIX)* 
+	rm -f *.i* *.cubin *.cu.c *.cudafe* *.fatbin.c *.ptx *.hash *.cu.cpp *.o
+
+
+
+#-------------------------------------------------------------------------------
+# make histogram_compare
+#-------------------------------------------------------------------------------
+
+histogram_compare: bin/histogram_compare_$(BIN_SUFFIX)
+
+bin/histogram_compare_$(BIN_SUFFIX) : histogram_compare.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/histogram_compare_$(BIN_SUFFIX) histogram_compare.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+	
+
+
+#-------------------------------------------------------------------------------
+# make spmv_compare
+#-------------------------------------------------------------------------------
+
+spmv_compare: bin/spmv_compare_$(BIN_SUFFIX)
+
+bin/spmv_compare_$(BIN_SUFFIX) : spmv_compare.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/spmv_compare_$(BIN_SUFFIX) spmv_compare.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -lcusparse $(MKL_LIBS) -O3
+	
+
diff --git a/third_party/cub/experimental/defunct/example_coo_spmv.cu b/third_party/cub/experimental/defunct/example_coo_spmv.cu
new file mode 100644
index 0000000..6b33e1f
--- /dev/null
+++ b/third_party/cub/experimental/defunct/example_coo_spmv.cu
@@ -0,0 +1,1070 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * An implementation of COO SpMV using prefix scan to implement a
+ * reduce-value-by-row strategy
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <iterator>
+#include <vector>
+#include <string>
+#include <algorithm>
+#include <stdio.h>
+
+#include <cub/cub.cuh>
+
+#include "coo_graph.cuh"
+#include "../test/test_util.h"
+
+using namespace cub;
+using namespace std;
+
+
+/******************************************************************************
+ * Globals, constants, and typedefs
+ ******************************************************************************/
+
+typedef int         VertexId;   // uint32s as vertex ids
+typedef double      Value;      // double-precision floating point values
+
+bool                    g_verbose       = false;
+int                     g_timing_iterations    = 1;
+CachingDeviceAllocator  g_allocator;
+
+
+/******************************************************************************
+ * Texture referencing
+ ******************************************************************************/
+
+/**
+ * Templated texture reference type for multiplicand vector
+ */
+template <typename Value>
+struct TexVector
+{
+    // Texture type to actually use (e.g., because CUDA doesn't load doubles as texture items)
+    typedef typename If<(Equals<Value, double>::VALUE), uint2, Value>::Type CastType;
+
+    // Texture reference type
+    typedef texture<CastType, cudaTextureType1D, cudaReadModeElementType> TexRef;
+
+    static TexRef ref;
+
+    /**
+     * Bind textures
+     */
+    static void BindTexture(void *d_in, int elements)
+    {
+        cudaChannelFormatDesc tex_desc = cudaCreateChannelDesc<CastType>();
+        if (d_in)
+        {
+            size_t offset;
+            size_t bytes = sizeof(CastType) * elements;
+            CubDebugExit(cudaBindTexture(&offset, ref, d_in, tex_desc, bytes));
+        }
+    }
+
+    /**
+     * Unbind textures
+     */
+    static void UnbindTexture()
+    {
+        CubDebugExit(cudaUnbindTexture(ref));
+    }
+
+    /**
+     * Load
+     */
+    static __device__ __forceinline__ Value Load(int offset)
+    {
+        Value output;
+        reinterpret_cast<typename TexVector<Value>::CastType &>(output) = tex1Dfetch(TexVector<Value>::ref, offset);
+        return output;
+    }
+};
+
+// Texture reference definitions
+template <typename Value>
+typename TexVector<Value>::TexRef TexVector<Value>::ref = 0;
+
+
+/******************************************************************************
+ * Utility types
+ ******************************************************************************/
+
+
+/**
+ * A partial dot-product sum paired with a corresponding row-id
+ */
+template <typename VertexId, typename Value>
+struct PartialProduct
+{
+    VertexId    row;            /// Row-id
+    Value       partial;        /// PartialProduct sum
+};
+
+
+/**
+ * A partial dot-product sum paired with a corresponding row-id (specialized for double-int pairings)
+ */
+template <>
+struct PartialProduct<int, double>
+{
+    long long   row;            /// Row-id
+    double      partial;        /// PartialProduct sum
+};
+
+
+/**
+ * Reduce-value-by-row scan operator
+ */
+struct ReduceByKeyOp
+{
+    template <typename PartialProduct>
+    __device__ __forceinline__ PartialProduct operator()(
+        const PartialProduct &first,
+        const PartialProduct &second)
+    {
+        PartialProduct retval;
+
+        retval.partial = (second.row != first.row) ?
+                second.partial :
+                first.partial + second.partial;
+
+        retval.row = second.row;
+        return retval;
+    }
+};
+
+
+/**
+ * Stateful block-wide prefix operator for BlockScan
+ */
+template <typename PartialProduct>
+struct BlockPrefixCallbackOp
+{
+    // Running block-wide prefix
+    PartialProduct running_prefix;
+
+    /**
+     * Returns the block-wide running_prefix in thread-0
+     */
+    __device__ __forceinline__ PartialProduct operator()(
+        const PartialProduct &block_aggregate)              ///< The aggregate sum of the BlockScan inputs
+    {
+        ReduceByKeyOp scan_op;
+
+        PartialProduct retval = running_prefix;
+        running_prefix = scan_op(running_prefix, block_aggregate);
+        return retval;
+    }
+};
+
+
+/**
+ * Operator for detecting discontinuities in a list of row identifiers.
+ */
+struct NewRowOp
+{
+    /// Returns true if row_b is the start of a new row
+    template <typename VertexId>
+    __device__ __forceinline__ bool operator()(
+        const VertexId& row_a,
+        const VertexId& row_b)
+    {
+        return (row_a != row_b);
+    }
+};
+
+
+
+/******************************************************************************
+ * Persistent thread block types
+ ******************************************************************************/
+
+/**
+ * SpMV thread block abstraction for processing a contiguous segment of
+ * sparse COO tiles.
+ */
+template <
+    int             BLOCK_THREADS,
+    int             ITEMS_PER_THREAD,
+    typename        VertexId,
+    typename        Value>
+struct PersistentBlockSpmv
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    // Constants
+    enum
+    {
+        TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD,
+    };
+
+    // Head flag type
+    typedef int HeadFlag;
+
+    // Partial dot product type
+    typedef PartialProduct<VertexId, Value> PartialProduct;
+
+    // Parameterized BlockScan type for reduce-value-by-row scan
+    typedef BlockScan<PartialProduct, BLOCK_THREADS, BLOCK_SCAN_RAKING_MEMOIZE> BlockScan;
+
+    // Parameterized BlockExchange type for exchanging rows between warp-striped -> blocked arrangements
+    typedef BlockExchange<VertexId, BLOCK_THREADS, ITEMS_PER_THREAD, true> BlockExchangeRows;
+
+    // Parameterized BlockExchange type for exchanging values between warp-striped -> blocked arrangements
+    typedef BlockExchange<Value, BLOCK_THREADS, ITEMS_PER_THREAD, true> BlockExchangeValues;
+
+    // Parameterized BlockDiscontinuity type for setting head-flags for each new row segment
+    typedef BlockDiscontinuity<HeadFlag, BLOCK_THREADS> BlockDiscontinuity;
+
+    // Shared memory type for this thread block
+    struct TempStorage
+    {
+        union
+        {
+            typename BlockExchangeRows::TempStorage         exchange_rows;      // Smem needed for BlockExchangeRows
+            typename BlockExchangeValues::TempStorage       exchange_values;    // Smem needed for BlockExchangeValues
+            struct
+            {
+                typename BlockScan::TempStorage             scan;               // Smem needed for BlockScan
+                typename BlockDiscontinuity::TempStorage    discontinuity;      // Smem needed for BlockDiscontinuity
+            };
+        };
+
+        VertexId        first_block_row;    ///< The first row-ID seen by this thread block
+        VertexId        last_block_row;     ///< The last row-ID seen by this thread block
+        Value           first_product;      ///< The first dot-product written by this thread block
+    };
+
+    //---------------------------------------------------------------------
+    // Thread fields
+    //---------------------------------------------------------------------
+
+    TempStorage                     &temp_storage;
+    BlockPrefixCallbackOp<PartialProduct>   prefix_op;
+    VertexId                        *d_rows;
+    VertexId                        *d_columns;
+    Value                           *d_values;
+    Value                           *d_vector;
+    Value                           *d_result;
+    PartialProduct                  *d_block_partials;
+    int                             block_offset;
+    int                             block_end;
+
+
+    //---------------------------------------------------------------------
+    // Operations
+    //---------------------------------------------------------------------
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__
+    PersistentBlockSpmv(
+        TempStorage                 &temp_storage,
+        VertexId                    *d_rows,
+        VertexId                    *d_columns,
+        Value                       *d_values,
+        Value                       *d_vector,
+        Value                       *d_result,
+        PartialProduct              *d_block_partials,
+        int                         block_offset,
+        int                         block_end)
+    :
+        temp_storage(temp_storage),
+        d_rows(d_rows),
+        d_columns(d_columns),
+        d_values(d_values),
+        d_vector(d_vector),
+        d_result(d_result),
+        d_block_partials(d_block_partials),
+        block_offset(block_offset),
+        block_end(block_end)
+    {
+        // Initialize scalar shared memory values
+        if (threadIdx.x == 0)
+        {
+            VertexId first_block_row            = d_rows[block_offset];
+            VertexId last_block_row             = d_rows[block_end - 1];
+
+            temp_storage.first_block_row        = first_block_row;
+            temp_storage.last_block_row         = last_block_row;
+            temp_storage.first_product          = Value(0);
+
+            // Initialize prefix_op to identity
+            prefix_op.running_prefix.row        = first_block_row;
+            prefix_op.running_prefix.partial    = Value(0);
+        }
+
+        __syncthreads();
+    }
+
+
+    /**
+     * Processes a COO input tile of edges, outputting dot products for each row
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__ void ProcessTile(
+        int block_offset,
+        int guarded_items = 0)
+    {
+        VertexId        columns[ITEMS_PER_THREAD];
+        VertexId        rows[ITEMS_PER_THREAD];
+        Value           values[ITEMS_PER_THREAD];
+        PartialProduct  partial_sums[ITEMS_PER_THREAD];
+        HeadFlag        head_flags[ITEMS_PER_THREAD];
+
+        // Load a thread block-striped tile of A (sparse row-ids, column-ids, and values)
+        if (FULL_TILE)
+        {
+            // Unguarded loads
+            LoadDirectWarpStriped<LOAD_DEFAULT>(threadIdx.x, d_columns + block_offset, columns);
+            LoadDirectWarpStriped<LOAD_DEFAULT>(threadIdx.x, d_values + block_offset, values);
+            LoadDirectWarpStriped<LOAD_DEFAULT>(threadIdx.x, d_rows + block_offset, rows);
+        }
+        else
+        {
+            // This is a partial-tile (e.g., the last tile of input).  Extend the coordinates of the last
+            // vertex for out-of-bound items, but zero-valued
+            LoadDirectWarpStriped<LOAD_DEFAULT>(threadIdx.x, d_columns + block_offset, columns, guarded_items, VertexId(0));
+            LoadDirectWarpStriped<LOAD_DEFAULT>(threadIdx.x, d_values + block_offset, values, guarded_items, Value(0));
+            LoadDirectWarpStriped<LOAD_DEFAULT>(threadIdx.x, d_rows + block_offset, rows, guarded_items, temp_storage.last_block_row);
+        }
+
+        // Load the referenced values from x and compute the dot product partials sums
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+#if CUB_PTX_ARCH >= 350
+            values[ITEM] *= ThreadLoad<LOAD_LDG>(d_vector + columns[ITEM]);
+#else
+            values[ITEM] *= TexVector<Value>::Load(columns[ITEM]);
+#endif
+        }
+
+        // Transpose from warp-striped to blocked arrangement
+        BlockExchangeValues(temp_storage.exchange_values).WarpStripedToBlocked(values);
+
+        __syncthreads();
+
+        // Transpose from warp-striped to blocked arrangement
+        BlockExchangeRows(temp_storage.exchange_rows).WarpStripedToBlocked(rows);
+
+        // Barrier for smem reuse and coherence
+        __syncthreads();
+
+        // FlagT row heads by looking for discontinuities
+        BlockDiscontinuity(temp_storage.discontinuity).FlagHeads(
+            head_flags,                     // (Out) Head flags
+            rows,                           // Original row ids
+            NewRowOp(),                     // Functor for detecting start of new rows
+            prefix_op.running_prefix.row);  // Last row ID from previous tile to compare with first row ID in this tile
+
+        // Assemble partial product structures
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            partial_sums[ITEM].partial = values[ITEM];
+            partial_sums[ITEM].row = rows[ITEM];
+        }
+
+        // Reduce reduce-value-by-row across partial_sums using exclusive prefix scan
+        PartialProduct block_aggregate;
+        BlockScan(temp_storage.scan).ExclusiveScan(
+            partial_sums,                   // Scan input
+            partial_sums,                   // Scan output
+            ReduceByKeyOp(),                // Scan operator
+            block_aggregate,                // Block-wide total (unused)
+            prefix_op);                     // Prefix operator for seeding the block-wide scan with the running total
+
+        // Barrier for smem reuse and coherence
+        __syncthreads();
+
+        // Scatter an accumulated dot product if it is the head of a valid row
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            if (head_flags[ITEM])
+            {
+                d_result[partial_sums[ITEM].row] = partial_sums[ITEM].partial;
+
+                // Save off the first partial product that this thread block will scatter
+                if (partial_sums[ITEM].row == temp_storage.first_block_row)
+                {
+                    temp_storage.first_product = partial_sums[ITEM].partial;
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Iterate over input tiles belonging to this thread block
+     */
+    __device__ __forceinline__
+    void ProcessTiles()
+    {
+        // Process full tiles
+        while (block_offset <= block_end - TILE_ITEMS)
+        {
+            ProcessTile<true>(block_offset);
+            block_offset += TILE_ITEMS;
+        }
+
+        // Process the last, partially-full tile (if present)
+        int guarded_items = block_end - block_offset;
+        if (guarded_items)
+        {
+            ProcessTile<false>(block_offset, guarded_items);
+        }
+
+        if (threadIdx.x == 0)
+        {
+            if (gridDim.x == 1)
+            {
+                // Scatter the final aggregate (this kernel contains only 1 thread block)
+                d_result[prefix_op.running_prefix.row] = prefix_op.running_prefix.partial;
+            }
+            else
+            {
+                // Write the first and last partial products from this thread block so
+                // that they can be subsequently "fixed up" in the next kernel.
+
+                PartialProduct first_product;
+                first_product.row       = temp_storage.first_block_row;
+                first_product.partial   = temp_storage.first_product;
+
+                d_block_partials[blockIdx.x * 2]          = first_product;
+                d_block_partials[(blockIdx.x * 2) + 1]    = prefix_op.running_prefix;
+            }
+        }
+    }
+};
+
+
+/**
+ * Threadblock abstraction for "fixing up" an array of interblock SpMV partial products.
+ */
+template <
+    int             BLOCK_THREADS,
+    int             ITEMS_PER_THREAD,
+    typename        VertexId,
+    typename        Value>
+struct FinalizeSpmvBlock
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    // Constants
+    enum
+    {
+        TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD,
+    };
+
+    // Head flag type
+    typedef int HeadFlag;
+
+    // Partial dot product type
+    typedef PartialProduct<VertexId, Value> PartialProduct;
+
+    // Parameterized BlockScan type for reduce-value-by-row scan
+    typedef BlockScan<PartialProduct, BLOCK_THREADS, BLOCK_SCAN_RAKING_MEMOIZE> BlockScan;
+
+    // Parameterized BlockDiscontinuity type for setting head-flags for each new row segment
+    typedef BlockDiscontinuity<HeadFlag, BLOCK_THREADS> BlockDiscontinuity;
+
+    // Shared memory type for this thread block
+    struct TempStorage
+    {
+        typename BlockScan::TempStorage           scan;               // Smem needed for reduce-value-by-row scan
+        typename BlockDiscontinuity::TempStorage  discontinuity;      // Smem needed for head-flagging
+
+        VertexId last_block_row;
+    };
+
+
+    //---------------------------------------------------------------------
+    // Thread fields
+    //---------------------------------------------------------------------
+
+    TempStorage                     &temp_storage;
+    BlockPrefixCallbackOp<PartialProduct>   prefix_op;
+    Value                           *d_result;
+    PartialProduct                  *d_block_partials;
+    int                             num_partials;
+
+
+    //---------------------------------------------------------------------
+    // Operations
+    //---------------------------------------------------------------------
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__
+    FinalizeSpmvBlock(
+        TempStorage                 &temp_storage,
+        Value                       *d_result,
+        PartialProduct              *d_block_partials,
+        int                         num_partials)
+    :
+        temp_storage(temp_storage),
+        d_result(d_result),
+        d_block_partials(d_block_partials),
+        num_partials(num_partials)
+    {
+        // Initialize scalar shared memory values
+        if (threadIdx.x == 0)
+        {
+            VertexId first_block_row            = d_block_partials[0].row;
+            VertexId last_block_row             = d_block_partials[num_partials - 1].row;
+            temp_storage.last_block_row         = last_block_row;
+
+            // Initialize prefix_op to identity
+            prefix_op.running_prefix.row        = first_block_row;
+            prefix_op.running_prefix.partial    = Value(0);
+        }
+
+        __syncthreads();
+    }
+
+
+    /**
+     * Processes a COO input tile of edges, outputting dot products for each row
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__
+    void ProcessTile(
+        int block_offset,
+        int guarded_items = 0)
+    {
+        VertexId        rows[ITEMS_PER_THREAD];
+        PartialProduct  partial_sums[ITEMS_PER_THREAD];
+        HeadFlag        head_flags[ITEMS_PER_THREAD];
+
+        // Load a tile of block partials from previous kernel
+        if (FULL_TILE)
+        {
+            // Full tile
+#if CUB_PTX_ARCH >= 350
+            LoadDirectBlocked<LOAD_LDG>(threadIdx.x, d_block_partials + block_offset, partial_sums);
+#else
+            LoadDirectBlocked(threadIdx.x, d_block_partials + block_offset, partial_sums);
+#endif
+        }
+        else
+        {
+            // Partial tile (extend zero-valued coordinates of the last partial-product for out-of-bounds items)
+            PartialProduct default_sum;
+            default_sum.row = temp_storage.last_block_row;
+            default_sum.partial = Value(0);
+
+#if CUB_PTX_ARCH >= 350
+            LoadDirectBlocked<LOAD_LDG>(threadIdx.x, d_block_partials + block_offset, partial_sums, guarded_items, default_sum);
+#else
+            LoadDirectBlocked(threadIdx.x, d_block_partials + block_offset, partial_sums, guarded_items, default_sum);
+#endif
+        }
+
+        // Copy out row IDs for row-head flagging
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            rows[ITEM] = partial_sums[ITEM].row;
+        }
+
+        // FlagT row heads by looking for discontinuities
+        BlockDiscontinuity(temp_storage.discontinuity).FlagHeads(
+            rows,                           // Original row ids
+            head_flags,                     // (Out) Head flags
+            NewRowOp(),                     // Functor for detecting start of new rows
+            prefix_op.running_prefix.row);   // Last row ID from previous tile to compare with first row ID in this tile
+
+        // Reduce reduce-value-by-row across partial_sums using exclusive prefix scan
+        PartialProduct block_aggregate;
+        BlockScan(temp_storage.scan).ExclusiveScan(
+            partial_sums,                   // Scan input
+            partial_sums,                   // Scan output
+            ReduceByKeyOp(),                // Scan operator
+            block_aggregate,                // Block-wide total (unused)
+            prefix_op);                     // Prefix operator for seeding the block-wide scan with the running total
+
+        // Scatter an accumulated dot product if it is the head of a valid row
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            if (head_flags[ITEM])
+            {
+                d_result[partial_sums[ITEM].row] = partial_sums[ITEM].partial;
+            }
+        }
+    }
+
+
+    /**
+     * Iterate over input tiles belonging to this thread block
+     */
+    __device__ __forceinline__
+    void ProcessTiles()
+    {
+        // Process full tiles
+        int block_offset = 0;
+        while (block_offset <= num_partials - TILE_ITEMS)
+        {
+            ProcessTile<true>(block_offset);
+            block_offset += TILE_ITEMS;
+        }
+
+        // Process final partial tile (if present)
+        int guarded_items = num_partials - block_offset;
+        if (guarded_items)
+        {
+            ProcessTile<false>(block_offset, guarded_items);
+        }
+
+        // Scatter the final aggregate (this kernel contains only 1 thread block)
+        if (threadIdx.x == 0)
+        {
+            d_result[prefix_op.running_prefix.row] = prefix_op.running_prefix.partial;
+        }
+    }
+};
+
+
+/******************************************************************************
+ * Kernel entrypoints
+ ******************************************************************************/
+
+
+
+/**
+ * SpMV kernel whose thread blocks each process a contiguous segment of sparse COO tiles.
+ */
+template <
+    int                             BLOCK_THREADS,
+    int                             ITEMS_PER_THREAD,
+    typename                        VertexId,
+    typename                        Value>
+__launch_bounds__ (BLOCK_THREADS)
+__global__ void CooKernel(
+    GridEvenShare<int>              even_share,
+    PartialProduct<VertexId, Value> *d_block_partials,
+    VertexId                        *d_rows,
+    VertexId                        *d_columns,
+    Value                           *d_values,
+    Value                           *d_vector,
+    Value                           *d_result)
+{
+    // Specialize SpMV thread block abstraction type
+    typedef PersistentBlockSpmv<BLOCK_THREADS, ITEMS_PER_THREAD, VertexId, Value> PersistentBlockSpmv;
+
+    // Shared memory allocation
+    __shared__ typename PersistentBlockSpmv::TempStorage temp_storage;
+
+    // Initialize thread block even-share to tell us where to start and stop our tile-processing
+    even_share.BlockInit();
+
+    // Construct persistent thread block
+    PersistentBlockSpmv persistent_block(
+        temp_storage,
+        d_rows,
+        d_columns,
+        d_values,
+        d_vector,
+        d_result,
+        d_block_partials,
+        even_share.block_offset,
+        even_share.block_end);
+
+    // Process input tiles
+    persistent_block.ProcessTiles();
+}
+
+
+/**
+ * Kernel for "fixing up" an array of interblock SpMV partial products.
+ */
+template <
+    int                             BLOCK_THREADS,
+    int                             ITEMS_PER_THREAD,
+    typename                        VertexId,
+    typename                        Value>
+__launch_bounds__ (BLOCK_THREADS,  1)
+__global__ void CooFinalizeKernel(
+    PartialProduct<VertexId, Value> *d_block_partials,
+    int                             num_partials,
+    Value                           *d_result)
+{
+    // Specialize "fix-up" thread block abstraction type
+    typedef FinalizeSpmvBlock<BLOCK_THREADS, ITEMS_PER_THREAD, VertexId, Value> FinalizeSpmvBlock;
+
+    // Shared memory allocation
+    __shared__ typename FinalizeSpmvBlock::TempStorage temp_storage;
+
+    // Construct persistent thread block
+    FinalizeSpmvBlock persistent_block(temp_storage, d_result, d_block_partials, num_partials);
+
+    // Process input tiles
+    persistent_block.ProcessTiles();
+}
+
+
+
+//---------------------------------------------------------------------
+// Host subroutines
+//---------------------------------------------------------------------
+
+
+/**
+ * Simple test of device
+ */
+template <
+    int                         COO_BLOCK_THREADS,
+    int                         COO_ITEMS_PER_THREAD,
+    int                         COO_SUBSCRIPTION_FACTOR,
+    int                         FINALIZE_BLOCK_THREADS,
+    int                         FINALIZE_ITEMS_PER_THREAD,
+    typename                    VertexId,
+    typename                    Value>
+void TestDevice(
+    CooGraph<VertexId, Value>&  coo_graph,
+    Value*                      h_vector,
+    Value*                      h_reference)
+{
+    typedef PartialProduct<VertexId, Value> PartialProduct;
+
+    const int COO_TILE_SIZE = COO_BLOCK_THREADS * COO_ITEMS_PER_THREAD;
+
+    // SOA device storage
+    VertexId        *d_rows;             // SOA graph row coordinates
+    VertexId        *d_columns;          // SOA graph col coordinates
+    Value           *d_values;           // SOA graph values
+    Value           *d_vector;           // Vector multiplicand
+    Value           *d_result;           // Output row
+    PartialProduct  *d_block_partials;   // Temporary storage for communicating dot product partials between thread blocks
+
+    // Create SOA version of coo_graph on host
+    int             num_edges   = coo_graph.coo_tuples.size();
+    VertexId        *h_rows     = new VertexId[num_edges];
+    VertexId        *h_columns  = new VertexId[num_edges];
+    Value           *h_values   = new Value[num_edges];
+    for (int i = 0; i < num_edges; i++)
+    {
+        h_rows[i]       = coo_graph.coo_tuples[i].row;
+        h_columns[i]    = coo_graph.coo_tuples[i].col;
+        h_values[i]     = coo_graph.coo_tuples[i].val;
+    }
+
+    // Get CUDA properties
+    Device device_props;
+    CubDebugExit(device_props.Init());
+
+    // Determine launch configuration from kernel properties
+    int coo_sm_occupancy;
+    CubDebugExit(device_props.MaxSmOccupancy(
+        coo_sm_occupancy,
+        CooKernel<COO_BLOCK_THREADS, COO_ITEMS_PER_THREAD, VertexId, Value>,
+        COO_BLOCK_THREADS));
+    int max_coo_grid_size   = device_props.sm_count * coo_sm_occupancy * COO_SUBSCRIPTION_FACTOR;
+
+    // Construct an even-share work distribution
+    GridEvenShare<int> even_share(num_edges, max_coo_grid_size, COO_TILE_SIZE);
+    int coo_grid_size  = even_share.grid_size;
+    int num_partials   = coo_grid_size * 2;
+
+    // Allocate COO device arrays
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_rows,            sizeof(VertexId) * num_edges));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_columns,         sizeof(VertexId) * num_edges));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values,          sizeof(Value) * num_edges));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_vector,          sizeof(Value) * coo_graph.col_dim));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_result,          sizeof(Value) * coo_graph.row_dim));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_block_partials,  sizeof(PartialProduct) * num_partials));
+
+    // Copy host arrays to device
+    CubDebugExit(cudaMemcpy(d_rows,     h_rows,     sizeof(VertexId) * num_edges,       cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_columns,  h_columns,  sizeof(VertexId) * num_edges,       cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_values,   h_values,   sizeof(Value) * num_edges,          cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_vector,   h_vector,   sizeof(Value) * coo_graph.col_dim,  cudaMemcpyHostToDevice));
+
+    // Bind textures
+    TexVector<Value>::BindTexture(d_vector, coo_graph.col_dim);
+
+    // Print debug info
+    printf("CooKernel<%d, %d><<<%d, %d>>>(...), Max SM occupancy: %d\n",
+        COO_BLOCK_THREADS, COO_ITEMS_PER_THREAD, coo_grid_size, COO_BLOCK_THREADS, coo_sm_occupancy);
+    if (coo_grid_size > 1)
+    {
+        printf("CooFinalizeKernel<<<1, %d>>>(...)\n", FINALIZE_BLOCK_THREADS);
+    }
+    fflush(stdout);
+
+    CubDebugExit(cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte));
+
+    // Run kernel (always run one iteration without timing)
+    GpuTimer gpu_timer;
+    float elapsed_millis = 0.0;
+    for (int i = 0; i <= g_timing_iterations; i++)
+    {
+        gpu_timer.Start();
+
+        // Initialize output
+        CubDebugExit(cudaMemset(d_result, 0, coo_graph.row_dim * sizeof(Value)));
+
+        // Run the COO kernel
+        CooKernel<COO_BLOCK_THREADS, COO_ITEMS_PER_THREAD><<<coo_grid_size, COO_BLOCK_THREADS>>>(
+            even_share,
+            d_block_partials,
+            d_rows,
+            d_columns,
+            d_values,
+            d_vector,
+            d_result);
+
+        if (coo_grid_size > 1)
+        {
+            // Run the COO finalize kernel
+            CooFinalizeKernel<FINALIZE_BLOCK_THREADS, FINALIZE_ITEMS_PER_THREAD><<<1, FINALIZE_BLOCK_THREADS>>>(
+                d_block_partials,
+                num_partials,
+                d_result);
+        }
+
+        gpu_timer.Stop();
+
+        if (i > 0)
+            elapsed_millis += gpu_timer.ElapsedMillis();
+    }
+
+    // Force any kernel stdio to screen
+    CubDebugExit(cudaThreadSynchronize());
+    fflush(stdout);
+
+    // Display timing
+    if (g_timing_iterations > 0)
+    {
+        float avg_elapsed = elapsed_millis / g_timing_iterations;
+        int total_bytes = ((sizeof(VertexId) + sizeof(VertexId)) * 2 * num_edges) + (sizeof(Value) * coo_graph.row_dim);
+        printf("%d iterations, average elapsed (%.3f ms), utilized bandwidth (%.3f GB/s), GFLOPS(%.3f)\n",
+            g_timing_iterations,
+            avg_elapsed,
+            total_bytes / avg_elapsed / 1000.0 / 1000.0,
+            num_edges * 2 / avg_elapsed / 1000.0 / 1000.0);
+    }
+
+    // Check results
+    int compare = CompareDeviceResults(h_reference, d_result, coo_graph.row_dim, true, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    TexVector<Value>::UnbindTexture();
+    CubDebugExit(g_allocator.DeviceFree(d_block_partials));
+    CubDebugExit(g_allocator.DeviceFree(d_rows));
+    CubDebugExit(g_allocator.DeviceFree(d_columns));
+    CubDebugExit(g_allocator.DeviceFree(d_values));
+    CubDebugExit(g_allocator.DeviceFree(d_vector));
+    CubDebugExit(g_allocator.DeviceFree(d_result));
+    delete[] h_rows;
+    delete[] h_columns;
+    delete[] h_values;
+}
+
+
+/**
+ * Compute reference answer on CPU
+ */
+template <typename VertexId, typename Value>
+void ComputeReference(
+    CooGraph<VertexId, Value>&  coo_graph,
+    Value*                      h_vector,
+    Value*                      h_reference)
+{
+    for (VertexId i = 0; i < coo_graph.row_dim; i++)
+    {
+        h_reference[i] = 0.0;
+    }
+
+    for (VertexId i = 0; i < coo_graph.coo_tuples.size(); i++)
+    {
+        h_reference[coo_graph.coo_tuples[i].row] +=
+            coo_graph.coo_tuples[i].val *
+            h_vector[coo_graph.coo_tuples[i].col];
+    }
+}
+
+
+/**
+ * Assign arbitrary values to vector items
+ */
+template <typename Value>
+void AssignVectorValues(Value *vector, int col_dim)
+{
+    for (int i = 0; i < col_dim; i++)
+    {
+        vector[i] = 1.0;
+    }
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("i", g_timing_iterations);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s\n [--device=<device-id>] [--v] [--iterations=<test iterations>] [--grid-size=<grid-size>]\n"
+            "\t--type=wheel --spokes=<spokes>\n"
+            "\t--type=grid2d --width=<width> [--no-self-loops]\n"
+            "\t--type=grid3d --width=<width> [--no-self-loops]\n"
+            "\t--type=market --file=<file>\n"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get graph type
+    string type;
+    args.GetCmdLineArgument("type", type);
+
+    // Generate graph structure
+
+    CpuTimer timer;
+    timer.Start();
+    CooGraph<VertexId, Value> coo_graph;
+    if (type == string("grid2d"))
+    {
+        VertexId width;
+        args.GetCmdLineArgument("width", width);
+        bool self_loops = !args.CheckCmdLineFlag("no-self-loops");
+        printf("Generating %s grid2d width(%d)... ", (self_loops) ? "5-pt" : "4-pt", width); fflush(stdout);
+        if (coo_graph.InitGrid2d(width, self_loops)) exit(1);
+    } else if (type == string("grid3d"))
+    {
+        VertexId width;
+        args.GetCmdLineArgument("width", width);
+        bool self_loops = !args.CheckCmdLineFlag("no-self-loops");
+        printf("Generating %s grid3d width(%d)... ", (self_loops) ? "7-pt" : "6-pt", width); fflush(stdout);
+        if (coo_graph.InitGrid3d(width, self_loops)) exit(1);
+    }
+    else if (type == string("wheel"))
+    {
+        VertexId spokes;
+        args.GetCmdLineArgument("spokes", spokes);
+        printf("Generating wheel spokes(%d)... ", spokes); fflush(stdout);
+        if (coo_graph.InitWheel(spokes)) exit(1);
+    }
+    else if (type == string("market"))
+    {
+        string filename;
+        args.GetCmdLineArgument("file", filename);
+        printf("Generating MARKET for %s... ", filename.c_str()); fflush(stdout);
+        if (coo_graph.InitMarket(filename)) exit(1);
+    }
+    else
+    {
+        printf("Unsupported graph type\n");
+        exit(1);
+    }
+    timer.Stop();
+    printf("Done (%.3fs). %d non-zeros, %d rows, %d columns\n",
+        timer.ElapsedMillis() / 1000.0,
+        coo_graph.coo_tuples.size(),
+        coo_graph.row_dim,
+        coo_graph.col_dim);
+    fflush(stdout);
+
+    if (g_verbose)
+    {
+        cout << coo_graph << "\n";
+    }
+
+    // Create vector
+    Value *h_vector = new Value[coo_graph.col_dim];
+    AssignVectorValues(h_vector, coo_graph.col_dim);
+    if (g_verbose)
+    {
+        printf("Vector[%d]: ", coo_graph.col_dim);
+        DisplayResults(h_vector, coo_graph.col_dim);
+        printf("\n\n");
+    }
+
+    // Compute reference answer
+    Value *h_reference = new Value[coo_graph.row_dim];
+    ComputeReference(coo_graph, h_vector, h_reference);
+    if (g_verbose)
+    {
+        printf("Results[%d]: ", coo_graph.row_dim);
+        DisplayResults(h_reference, coo_graph.row_dim);
+        printf("\n\n");
+    }
+
+    // Parameterization for SM35
+    enum
+    {
+        COO_BLOCK_THREADS           = 64,
+        COO_ITEMS_PER_THREAD        = 10,
+        COO_SUBSCRIPTION_FACTOR     = 4,
+        FINALIZE_BLOCK_THREADS      = 256,
+        FINALIZE_ITEMS_PER_THREAD   = 4,
+    };
+
+    // Run GPU version
+    TestDevice<
+        COO_BLOCK_THREADS,
+        COO_ITEMS_PER_THREAD,
+        COO_SUBSCRIPTION_FACTOR,
+        FINALIZE_BLOCK_THREADS,
+        FINALIZE_ITEMS_PER_THREAD>(coo_graph, h_vector, h_reference);
+
+    // Cleanup
+    delete[] h_vector;
+    delete[] h_reference;
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/experimental/defunct/test_device_seg_reduce.cu b/third_party/cub/experimental/defunct/test_device_seg_reduce.cu
new file mode 100644
index 0000000..d2e55b9
--- /dev/null
+++ b/third_party/cub/experimental/defunct/test_device_seg_reduce.cu
@@ -0,0 +1,2142 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * An implementation of segmented reduction using a load-balanced parallelization
+ * strategy based on the MergePath decision path.
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <iterator>
+#include <vector>
+#include <string>
+#include <algorithm>
+#include <stdio.h>
+
+#include <cub/cub.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+using namespace std;
+
+
+/******************************************************************************
+ * Globals, constants, and typedefs
+ ******************************************************************************/
+
+bool                    g_verbose           = false;
+int                     g_timing_iterations = 1;
+CachingDeviceAllocator  g_allocator(true);
+
+
+/******************************************************************************
+ * Utility routines
+ ******************************************************************************/
+
+
+/**
+ * An pair of index offsets
+ */
+template <typename OffsetT>
+struct IndexPair
+{
+    OffsetT a_idx;
+    OffsetT b_idx;
+};
+
+
+/**
+ * Computes the begin offsets into A and B for the specified
+ * location (diagonal) along the merge decision path
+ */
+template <
+    int                 BLOCK_THREADS,
+    typename            IteratorA,
+    typename            IteratorB,
+    typename            OffsetT>
+__device__ __forceinline__ void ParallelMergePathSearch(
+    OffsetT             diagonal,
+    IteratorA           a,
+    IteratorB           b,
+    IndexPair<OffsetT>  begin,          // Begin offsets into a and b
+    IndexPair<OffsetT>  end,            // End offsets into a and b
+    IndexPair<OffsetT>  &intersection)  // [out] Intersection offsets into a and b
+{
+    OffsetT a_split_min = CUB_MAX(diagonal - end.b_idx, begin.a_idx);
+    OffsetT a_split_max = CUB_MIN(diagonal, end.a_idx);
+
+    while (a_split_min < a_split_max)
+    {
+        OffsetT a_distance       = a_split_max - a_split_min;
+        OffsetT a_slice          = (a_distance + BLOCK_THREADS - 1) >> Log2<BLOCK_THREADS>::VALUE;
+        OffsetT a_split_pivot    = CUB_MIN(a_split_min + (threadIdx.x * a_slice), end.a_idx - 1);
+
+        int move_up = (a[a_split_pivot] <= b[diagonal - a_split_pivot - 1]);
+        int num_up = __syncthreads_count(move_up);
+/*
+        _CubLog("a_split_min(%d), a_split_max(%d) a_distance(%d), a_slice(%d), a_split_pivot(%d), move_up(%d), num_up(%d), a_begin(%d), a_end(%d)\n",
+            a_split_min, a_split_max, a_distance, a_slice, a_split_pivot, move_up, num_up, a_begin, a_end);
+*/
+        a_split_max = CUB_MIN(num_up * a_slice, end.a_idx);
+        a_split_min = CUB_MAX(a_split_max - a_slice, begin.a_idx) + 1;
+    }
+
+    intersection.a_idx = CUB_MIN(a_split_min, end.a_idx);
+    intersection.b_idx = CUB_MIN(diagonal - a_split_min, end.b_idx);
+}
+
+/**
+ * Computes the begin offsets into A and B for the specified
+ * location (diagonal) along the merge decision path
+ */
+template <
+    typename            IteratorA,
+    typename            IteratorB,
+    typename            OffsetT>
+__device__ __forceinline__ void MergePathSearch(
+    OffsetT             diagonal,
+    IteratorA           a,
+    IteratorB           b,
+    IndexPair<OffsetT>  begin,          // Begin offsets into a and b
+    IndexPair<OffsetT>  end,            // End offsets into a and b
+    IndexPair<OffsetT>  &intersection)  // [out] Intersection offsets into a and b
+{
+    OffsetT split_min = CUB_MAX(diagonal - end.b_idx, begin.a_idx);
+    OffsetT split_max = CUB_MIN(diagonal, end.a_idx);
+
+    while (split_min < split_max)
+    {
+        OffsetT split_pivot = (split_min + split_max) >> 1;
+        if (a[split_pivot] <= b[diagonal - split_pivot - 1])
+        {
+            // Move candidate split range up A, down B
+            split_min = split_pivot + 1;
+        }
+        else
+        {
+            // Move candidate split range up B, down A
+            split_max = split_pivot;
+        }
+    }
+
+    intersection.a_idx = CUB_MIN(split_min, end.a_idx);
+    intersection.b_idx = CUB_MIN(diagonal - split_min, end.b_idx);
+}
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for BlockSegReduceRegion
+ */
+template <
+    int                     _BLOCK_THREADS,             ///< Threads per thread block
+    int                     _ITEMS_PER_THREAD,          ///< Items per thread (per tile of input)
+    bool                    _USE_SMEM_SEGMENT_CACHE,    ///< Whether or not to cache incoming segment offsets in shared memory before reducing each tile
+    bool                    _USE_SMEM_VALUE_CACHE,      ///< Whether or not to cache incoming values in shared memory before reducing each tile
+    CacheLoadModifier       _LOAD_MODIFIER_SEGMENTS,    ///< Cache load modifier for reading segment offsets
+    CacheLoadModifier       _LOAD_MODIFIER_VALUES,      ///< Cache load modifier for reading values
+    BlockReduceAlgorithm    _REDUCE_ALGORITHM,          ///< The BlockReduce algorithm to use
+    BlockScanAlgorithm      _SCAN_ALGORITHM>            ///< The BlockScan algorithm to use
+struct BlockSegReduceRegionPolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
+        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
+        USE_SMEM_SEGMENT_CACHE  = _USE_SMEM_SEGMENT_CACHE,      ///< Whether or not to cache incoming segment offsets in shared memory before reducing each tile
+        USE_SMEM_VALUE_CACHE    = _USE_SMEM_VALUE_CACHE,        ///< Whether or not to cache incoming upcoming values in shared memory before reducing each tile
+    };
+
+    static const CacheLoadModifier      LOAD_MODIFIER_SEGMENTS  = _LOAD_MODIFIER_SEGMENTS;  ///< Cache load modifier for reading segment offsets
+    static const CacheLoadModifier      LOAD_MODIFIER_VALUES    = _LOAD_MODIFIER_VALUES;    ///< Cache load modifier for reading values
+    static const BlockReduceAlgorithm   REDUCE_ALGORITHM        = _REDUCE_ALGORITHM;        ///< The BlockReduce algorithm to use
+    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;          ///< The BlockScan algorithm to use
+};
+
+
+/******************************************************************************
+ * Persistent thread block types
+ ******************************************************************************/
+
+/**
+ * \brief BlockSegReduceTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide segmented reduction.
+ */
+template <
+    typename BlockSegReduceRegionPolicy,    ///< Parameterized BlockSegReduceRegionPolicy tuning policy
+    typename SegmentOffsetIterator,         ///< Random-access input iterator type for reading segment end-offsets
+    typename ValueIterator,                 ///< Random-access input iterator type for reading values
+    typename OutputIteratorT,               ///< Random-access output iterator type for writing segment reductions
+    typename ReductionOp,                   ///< Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+    typename OffsetT>                       ///< Signed integer type for global offsets
+struct BlockSegReduceRegion
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    // Constants
+    enum
+    {
+        BLOCK_THREADS       = BlockSegReduceRegionPolicy::BLOCK_THREADS,
+        ITEMS_PER_THREAD    = BlockSegReduceRegionPolicy::ITEMS_PER_THREAD,
+        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,                     /// Number of work items to be processed per tile
+
+        USE_SMEM_SEGMENT_CACHE  = BlockSegReduceRegionPolicy::USE_SMEM_SEGMENT_CACHE,      ///< Whether or not to cache incoming segment offsets in shared memory before reducing each tile
+        USE_SMEM_VALUE_CACHE    = BlockSegReduceRegionPolicy::USE_SMEM_VALUE_CACHE,        ///< Whether or not to cache incoming upcoming values in shared memory before reducing each tile
+
+        SMEM_SEGMENT_CACHE_ITEMS    = USE_SMEM_SEGMENT_CACHE ? TILE_ITEMS : 1,
+        SMEM_VALUE_CACHE_ITEMS      = USE_SMEM_VALUE_CACHE ? TILE_ITEMS : 1,
+    };
+
+    // Segment offset type
+    typedef typename std::iterator_traits<SegmentOffsetIterator>::value_type SegmentOffset;
+
+    // Value type
+    typedef typename std::iterator_traits<ValueIterator>::value_type Value;
+
+    // Counting iterator type
+    typedef CountingInputIterator<SegmentOffsetT, OffsetT> CountingIterator;
+
+    // Segment offsets iterator wrapper type
+    typedef typename If<(IsPointer<SegmentOffsetIterator>::VALUE),
+            CacheModifiedInputIterator<BlockSegReduceRegionPolicy::LOAD_MODIFIER_SEGMENTS, SegmentOffsetT, OffsetT>,  // Wrap the native input pointer with CacheModifiedInputIterator
+            SegmentOffsetIterator>::Type                                                                            // Directly use the supplied input iterator type
+        WrappedSegmentOffsetIterator;
+
+    // Values iterator wrapper type
+    typedef typename If<(IsPointer<ValueIterator>::VALUE),
+            CacheModifiedInputIterator<BlockSegReduceRegionPolicy::LOAD_MODIFIER_VALUES, Value, OffsetT>,        // Wrap the native input pointer with CacheModifiedInputIterator
+            ValueIterator>::Type                                                                                // Directly use the supplied input iterator type
+        WrappedValueIterator;
+
+    // Tail flag type for marking segment discontinuities
+    typedef int TailFlag;
+
+    // Reduce-by-key data type tuple (segment-ID, value)
+    typedef KeyValuePair<OffsetT, Value> KeyValuePair;
+
+    // Index pair data type
+    typedef IndexPair<OffsetT> IndexPair;
+
+    // BlockScan scan operator for reduction-by-segment
+    typedef ReduceByKeyOp<ReductionOp> ReduceByKeyOp;
+
+    // Stateful BlockScan prefix callback type for managing a running total while scanning consecutive tiles
+    typedef RunningBlockPrefixCallbackOp<
+            KeyValuePair,
+            ReduceByKeyOp>
+        RunningPrefixCallbackOp;
+
+    // Parameterized BlockShift type for exchanging index pairs
+    typedef BlockShift<
+            IndexPair,
+            BLOCK_THREADS>
+        BlockShift;
+
+    // Parameterized BlockReduce type for block-wide reduction
+    typedef BlockReduce<
+            Value,
+            BLOCK_THREADS,
+            BlockSegReduceRegionPolicy::REDUCE_ALGORITHM>
+        BlockReduce;
+
+    // Parameterized BlockScan type for block-wide reduce-value-by-key
+    typedef BlockScan<
+            KeyValuePair,
+            BLOCK_THREADS,
+            BlockSegReduceRegionPolicy::SCAN_ALGORITHM>
+        BlockScan;
+
+    // Shared memory type for this thread block
+    struct _TempStorage
+    {
+        union
+        {
+            // Smem needed for BlockScan
+            typename BlockScan::TempStorage scan;
+
+            // Smem needed for BlockReduce
+            typename BlockReduce::TempStorage reduce;
+
+            struct
+            {
+                // Smem needed for communicating start/end indices between threads for a given work tile
+                typename BlockShift::TempStorage shift;
+
+                // Smem needed for caching segment end-offsets
+                SegmentOffset cached_segment_end_offsets[SMEM_SEGMENT_CACHE_ITEMS + 1];
+            };
+
+            // Smem needed for caching values
+            Value cached_values[SMEM_VALUE_CACHE_ITEMS];
+        };
+
+        IndexPair block_region_idx[2];      // The starting [0] and ending [1] pairs of segment and value indices for the thread block's region
+
+        // The first partial reduction tuple scattered by this thread block
+        KeyValuePair first_tuple;
+    };
+
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Thread fields
+    //---------------------------------------------------------------------
+
+    _TempStorage                    &temp_storage;          ///< Reference to shared storage
+    WrappedSegmentOffsetIterator    d_segment_end_offsets;  ///< A sequence of \p num_segments segment end-offsets
+    WrappedValueIterator            d_values;               ///< A sequence of \p num_values data to reduce
+    OutputIteratorT                  d_output;               ///< A sequence of \p num_segments segment totals
+    CountingIterator                d_value_offsets;        ///< A sequence of \p num_values value-offsets
+    IndexPair                       *d_block_idx;
+    OffsetT                         num_values;             ///< Total number of values to reduce
+    OffsetT                         num_segments;           ///< Number of segments being reduced
+    Value                           identity;               ///< Identity value (for zero-length segments)
+    ReductionOp                     reduction_op;           ///< Reduction operator
+    ReduceByKeyOp                   scan_op;                ///< Reduce-by-key scan operator
+    RunningPrefixCallbackOp         prefix_op;              ///< Stateful running total for block-wide prefix scan of partial reduction tuples
+
+
+    //---------------------------------------------------------------------
+    // Operations
+    //---------------------------------------------------------------------
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__
+    BlockSegReduceRegion(
+        TempStorage             &temp_storage,          ///< Reference to shared storage
+        SegmentOffsetIterator   d_segment_end_offsets,  ///< A sequence of \p num_segments segment end-offsets
+        ValueIterator           d_values,               ///< A sequence of \p num_values values
+        OutputIteratorT          d_output,               ///< A sequence of \p num_segments segment totals
+        IndexPair               *d_block_idx,
+        OffsetT                 num_values,             ///< Number of values to reduce
+        OffsetT                 num_segments,           ///< Number of segments being reduced
+        Value                   identity,               ///< Identity value (for zero-length segments)
+        ReductionOp             reduction_op)           ///< Reduction operator
+    :
+        temp_storage(temp_storage.Alias()),
+        d_segment_end_offsets(d_segment_end_offsets),
+        d_values(d_values),
+        d_value_offsets(0),
+        d_output(d_output),
+        d_block_idx(d_block_idx),
+        num_values(num_values),
+        num_segments(num_segments),
+        identity(identity),
+        reduction_op(reduction_op),
+        scan_op(reduction_op),
+        prefix_op(scan_op)
+    {}
+
+
+    /**
+     * Fast-path single-segment tile reduction.  Perform a
+     * simple block-wide reduction and accumulate the result into
+     * the running total.
+     */
+    __device__ __forceinline__ void SingleSegmentTile(
+        IndexPair next_tile_idx,
+        IndexPair block_idx)
+    {
+        OffsetT tile_values = next_tile_idx.b_idx - block_idx.b_idx;
+
+        // Load a tile's worth of values (using identity for out-of-bounds items)
+        Value values[ITEMS_PER_THREAD];
+        LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_values + block_idx.b_idx, values, tile_values, identity);
+
+        // Barrier for smem reuse
+        __syncthreads();
+
+        // Reduce the tile of values and update the running total in thread-0
+        KeyValuePair tile_aggregate;
+        tile_aggregate.key      = block_idx.a_idx;
+        tile_aggregate.value    = BlockReduce(temp_storage.reduce).Reduce(values, reduction_op);
+
+        if (threadIdx.x == 0)
+        {
+            prefix_op.running_total = scan_op(prefix_op.running_total, tile_aggregate);
+        }
+    }
+
+    /**
+     * Fast-path empty-segment tile reduction.  Write out a tile of identity
+     * values to output.
+     */
+    __device__ __forceinline__ void EmptySegmentsTile(
+        IndexPair next_tile_idx,
+        IndexPair block_idx)
+    {
+        Value segment_reductions[ITEMS_PER_THREAD];
+
+        if (threadIdx.x == 0)
+        {
+            // The first segment gets the running segment total
+            segment_reductions[0] = prefix_op.running_total.value;
+
+            // Update the running prefix
+            prefix_op.running_total.value = identity;
+            prefix_op.running_total.key = next_tile_idx.a_idx;
+        }
+        else
+        {
+            // Remainder of segments in this tile get identity
+            segment_reductions[0] = identity;
+        }
+
+        // Remainder of segments in this tile get identity
+        #pragma unroll
+        for (int ITEM = 1; ITEM < ITEMS_PER_THREAD; ++ITEM)
+            segment_reductions[ITEM] = identity;
+
+        // Store reductions
+        OffsetT tile_segments = next_tile_idx.a_idx - block_idx.a_idx;
+        StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_output + block_idx.a_idx, segment_reductions, tile_segments);
+    }
+
+
+    /**
+     * Multi-segment tile reduction.
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__ void MultiSegmentTile(
+        IndexPair block_idx,
+        IndexPair thread_idx,
+        IndexPair next_thread_idx,
+        IndexPair next_tile_idx)
+    {
+        IndexPair local_thread_idx;
+        local_thread_idx.a_idx = thread_idx.a_idx - block_idx.a_idx;
+        local_thread_idx.b_idx = thread_idx.b_idx - block_idx.b_idx;
+
+        // Check if first segment end-offset is in range
+        bool valid_segment = FULL_TILE || (thread_idx.a_idx < next_thread_idx.a_idx);
+
+        // Check if first value offset is in range
+        bool valid_value = FULL_TILE || (thread_idx.b_idx < next_thread_idx.b_idx);
+
+        // Load first segment end-offset
+        OffsetT segment_end_offset = (valid_segment) ?
+            (USE_SMEM_SEGMENT_CACHE)?
+                temp_storage.cached_segment_end_offsets[local_thread_idx.a_idx] :
+                d_segment_end_offsets[thread_idx.a_idx] :
+            -1;
+
+        OffsetT segment_ids[ITEMS_PER_THREAD];
+        OffsetT value_offsets[ITEMS_PER_THREAD];
+
+        KeyValuePair first_partial;
+        first_partial.key    = thread_idx.a_idx;
+        first_partial.value  = identity;
+
+        // Get segment IDs and gather-offsets for values
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            segment_ids[ITEM]   = -1;
+            value_offsets[ITEM] = -1;
+
+            // Whether or not we slide (a) right along the segment path or (b) down the value path
+            if (valid_segment && (!valid_value || (segment_end_offset <= thread_idx.b_idx)))
+            {
+                // Consume this segment index
+                segment_ids[ITEM] = thread_idx.a_idx;
+                thread_idx.a_idx++;
+                local_thread_idx.a_idx++;
+
+                valid_segment = FULL_TILE || (thread_idx.a_idx < next_thread_idx.a_idx);
+
+                // Read next segment end-offset (if valid)
+                if (valid_segment)
+                {
+                    if (USE_SMEM_SEGMENT_CACHE)
+                        segment_end_offset = temp_storage.cached_segment_end_offsets[local_thread_idx.a_idx];
+                    else
+                        segment_end_offset = d_segment_end_offsets[thread_idx.a_idx];
+                }
+            }
+            else if (valid_value)
+            {
+                // Consume this value index
+                value_offsets[ITEM] = thread_idx.b_idx;
+                thread_idx.b_idx++;
+                local_thread_idx.b_idx++;
+
+                valid_value = FULL_TILE || (thread_idx.b_idx < next_thread_idx.b_idx);
+            }
+        }
+
+        // Load values
+        Value values[ITEMS_PER_THREAD];
+
+        if (USE_SMEM_VALUE_CACHE)
+        {
+            // Barrier for smem reuse
+            __syncthreads();
+
+            OffsetT tile_values = next_tile_idx.b_idx - block_idx.b_idx;
+
+            // Load a tile's worth of values (using identity for out-of-bounds items)
+            LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_values + block_idx.b_idx, values, tile_values, identity);
+
+            // Store to shared
+            StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, temp_storage.cached_values, values, tile_values);
+
+            // Barrier for smem reuse
+            __syncthreads();
+
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+            {
+                values[ITEM] = (value_offsets[ITEM] == -1) ?
+                    identity :
+                    temp_storage.cached_values[value_offsets[ITEM] - block_idx.b_idx];
+            }
+        }
+        else
+        {
+            #pragma unroll
+            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+            {
+                values[ITEM] = (value_offsets[ITEM] == -1) ?
+                    identity :
+                    d_values[value_offsets[ITEM]];
+            }
+        }
+
+        // Reduce within thread segments
+        KeyValuePair running_total = first_partial;
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            if (segment_ids[ITEM] != -1)
+            {
+                // Consume this segment index
+                d_output[segment_ids[ITEM]] = running_total.value;
+
+//                _CubLog("Updating segment %d with value %lld\n", segment_ids[ITEM], running_total.value)
+
+                if (first_partial.key == segment_ids[ITEM])
+                    first_partial.value = running_total.value;
+
+                running_total.key    = segment_ids[ITEM];
+                running_total.value  = identity;
+            }
+
+            running_total.value = reduction_op(running_total.value, values[ITEM]);
+        }
+/*
+
+        // Barrier for smem reuse
+        __syncthreads();
+
+        // Use prefix scan to reduce values by segment-id.  The segment-reductions end up in items flagged as segment-tails.
+        KeyValuePair block_aggregate;
+        BlockScan(temp_storage.scan).InclusiveScan(
+            pairs,                          // Scan input
+            pairs,                          // Scan output
+            scan_op,                        // Scan operator
+            block_aggregate,                // Block-wide total (unused)
+            prefix_op);                     // Prefix operator for seeding the block-wide scan with the running total
+*/
+
+/*
+        // Check if first segment end-offset is in range
+        bool valid_segment = (thread_idx.a_idx < next_thread_idx.a_idx);
+
+        // Check if first value offset is in range
+        bool valid_value = (thread_idx.b_idx < next_thread_idx.b_idx);
+
+        // Load first segment end-offset
+        OffsetT segment_end_offset = (valid_segment) ?
+            d_segment_end_offsets[thread_idx.a_idx] :
+            num_values;                                                     // Out of range (the last segment end-offset is one-past the last value offset)
+
+        // Load first value offset
+        OffsetT value_offset = (valid_value) ?
+            d_value_offsets[thread_idx.b_idx] :
+            num_values;                                                     // Out of range (one-past the last value offset)
+
+        // Assemble segment-demarcating tail flags and partial reduction tuples
+        TailFlag        tail_flags[ITEMS_PER_THREAD];
+        KeyValuePair    partial_reductions[ITEMS_PER_THREAD];
+
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            // Default tuple and flag values
+            partial_reductions[ITEM].key    = thread_idx.a_idx;
+            partial_reductions[ITEM].value  = identity;
+            tail_flags[ITEM]                = 0;
+
+            // Whether or not we slide (a) right along the segment path or (b) down the value path
+            if (valid_segment && (!valid_value || (segment_end_offset <= value_offset)))
+            {
+                // Consume this segment index
+
+                // Set tail flag noting the end of the segment
+                tail_flags[ITEM] = 1;
+
+                // Increment segment index
+                thread_idx.a_idx++;
+
+                // Read next segment end-offset (if valid)
+                if ((valid_segment = (thread_idx.a_idx < next_thread_idx.a_idx)))
+                    segment_end_offset = d_segment_end_offsets[thread_idx.a_idx];
+            }
+            else if (valid_value)
+            {
+                // Consume this value index
+
+                // Update the tuple's value with the value at this index.
+                partial_reductions[ITEM].value = d_values[value_offset];
+
+                // Increment value index
+                thread_idx.b_idx++;
+
+                // Read next value offset (if valid)
+                if ((valid_value = (thread_idx.b_idx < next_thread_idx.b_idx)))
+                    value_offset = d_value_offsets[thread_idx.b_idx];
+            }
+        }
+
+        // Use prefix scan to reduce values by segment-id.  The segment-reductions end up in items flagged as segment-tails.
+        KeyValuePair block_aggregate;
+        BlockScan(temp_storage.scan).InclusiveScan(
+            partial_reductions,             // Scan input
+            partial_reductions,             // Scan output
+            scan_op,                        // Scan operator
+            block_aggregate,                // Block-wide total (unused)
+            prefix_op);                     // Prefix operator for seeding the block-wide scan with the running total
+
+        // The first segment index for this region (hoist?)
+        OffsetT first_segment_idx = temp_storage.block_idx.a_idx[0];
+
+        // Scatter an accumulated reduction if it is the head of a valid segment
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            if (tail_flags[ITEM])
+            {
+                OffsetT segment_idx = partial_reductions[ITEM].key;
+                Value   value       = partial_reductions[ITEM].value;
+
+                // Write value reduction to corresponding segment id
+                d_output[segment_idx] = value;
+
+                // Save off the first value product that this thread block will scatter
+                if (segment_idx == first_segment_idx)
+                {
+                    temp_storage.first_tuple.value = value;
+                }
+            }
+        }
+*/
+    }
+
+
+
+    /**
+     * Have the thread block process the specified region of the MergePath decision path
+     */
+    __device__ __forceinline__ void ProcessRegion(
+        OffsetT         block_diagonal,
+        OffsetT         next_block_diagonal,
+        KeyValuePair    &first_tuple,       // [Out] Valid in thread-0
+        KeyValuePair    &last_tuple)        // [Out] Valid in thread-0
+    {
+        // Thread block initialization
+        if (threadIdx.x < 2)
+        {
+            // Retrieve block starting and ending indices
+            IndexPair block_idx = {0, 0};
+            if (gridDim.x > 1)
+            {
+                block_idx = d_block_idx[blockIdx.x + threadIdx.x];
+            }
+            else if (threadIdx.x > 0)
+            {
+                block_idx.a_idx = num_segments;
+                block_idx.b_idx = num_values;
+            }
+
+            // Share block starting and ending indices
+            temp_storage.block_region_idx[threadIdx.x] = block_idx;
+
+            // Initialize the block's running prefix
+            if (threadIdx.x == 0)
+            {
+                prefix_op.running_total.key    = block_idx.a_idx;
+                prefix_op.running_total.value  = identity;
+
+                // Initialize the "first scattered partial reduction tuple" to the prefix tuple (in case we don't actually scatter one)
+                temp_storage.first_tuple = prefix_op.running_total;
+            }
+        }
+
+        // Ensure coherence of region indices
+        __syncthreads();
+
+        // Read block's starting indices
+        IndexPair block_idx = temp_storage.block_region_idx[0];
+
+        // Have the thread block iterate over the region
+        #pragma unroll 1
+        while (block_diagonal < next_block_diagonal)
+        {
+            // Read block's ending indices (hoist?)
+            IndexPair next_block_idx = temp_storage.block_region_idx[1];
+
+            // Clamp the per-thread search range to within one work-tile of block's current indices
+            IndexPair next_tile_idx;
+            next_tile_idx.a_idx = CUB_MIN(next_block_idx.a_idx, block_idx.a_idx + TILE_ITEMS);
+            next_tile_idx.b_idx = CUB_MIN(next_block_idx.b_idx, block_idx.b_idx + TILE_ITEMS);
+
+            // Have each thread search for the end-indices of its subranges within the segment and value inputs
+            IndexPair next_thread_idx;
+            if (USE_SMEM_SEGMENT_CACHE)
+            {
+                // Search in smem cache
+                OffsetT num_segments = next_tile_idx.a_idx - block_idx.a_idx;
+
+                // Load global
+                SegmentOffset segment_offsets[ITEMS_PER_THREAD];
+                LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_segment_end_offsets + block_idx.a_idx, segment_offsets, num_segments, num_values);
+
+                // Store to shared
+                StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, temp_storage.cached_segment_end_offsets, segment_offsets);
+
+                __syncthreads();
+
+                OffsetT next_thread_diagonal = block_diagonal + ((threadIdx.x + 1) * ITEMS_PER_THREAD);
+
+                MergePathSearch(
+                    next_thread_diagonal,                       // Next thread diagonal
+                    temp_storage.cached_segment_end_offsets - block_idx.a_idx,                      // A (segment end-offsets)
+                    d_value_offsets,                            // B (value offsets)
+                    block_idx,                                  // Start indices into A and B
+                    next_tile_idx,                              // End indices into A and B
+                    next_thread_idx);                           // [out] diagonal intersection indices into A and B
+            }
+            else
+            {
+                // Search in global
+
+                OffsetT next_thread_diagonal = block_diagonal + ((threadIdx.x + 1) * ITEMS_PER_THREAD);
+
+                MergePathSearch(
+                    next_thread_diagonal,                       // Next thread diagonal
+                    d_segment_end_offsets,                      // A (segment end-offsets)
+                    d_value_offsets,                            // B (value offsets)
+                    block_idx,                                  // Start indices into A and B
+                    next_tile_idx,                              // End indices into A and B
+                    next_thread_idx);                           // [out] diagonal intersection indices into A and B
+            }
+
+            // Share thread end-indices to get thread begin-indices and tile end-indices
+            IndexPair thread_idx;
+
+            BlockShift(temp_storage.shift).Up(
+                next_thread_idx,    // Input item
+                thread_idx,         // [out] Output item
+                block_idx,          // Prefix item to be provided to <em>thread</em><sub>0</sub>
+                next_tile_idx);     // [out] Suffix item shifted out by the <em>thread</em><sub><tt>BLOCK_THREADS-1</tt></sub> to be provided to all threads
+
+//            if (block_idx.a_idx == next_tile_idx.a_idx)
+//            {
+//                // There are no segment end-offsets in this tile.  Perform a
+//                // simple block-wide reduction and accumulate the result into
+//                // the running total.
+//                SingleSegmentTile(next_tile_idx, block_idx);
+//            }
+//          else if (block_idx.b_idx == next_tile_idx.b_idx)
+//            {
+//                // There are no values in this tile (only empty segments).
+//                EmptySegmentsTile(next_tile_idx.a_idx, block_idx.a_idx);
+//            }
+//            else
+            if ((next_tile_idx.a_idx < num_segments) && (next_tile_idx.b_idx < num_values))
+            {
+                // Merge the tile's segment and value indices (full tile)
+                MultiSegmentTile<true>(block_idx, thread_idx, next_thread_idx, next_tile_idx);
+            }
+            else
+            {
+                // Merge the tile's segment and value indices (partially full tile)
+                MultiSegmentTile<false>(block_idx, thread_idx, next_thread_idx, next_tile_idx);
+            }
+
+            // Advance the block's indices in preparation for the next tile
+            block_idx = next_tile_idx;
+
+            // Advance to the next region in the decision path
+            block_diagonal += TILE_ITEMS;
+
+            // Barrier for smem reuse
+            __syncthreads();
+        }
+
+        // Get first and last tuples for the region
+        if (threadIdx.x == 0)
+        {
+            first_tuple = temp_storage.first_tuple;
+            last_tuple = prefix_op.running_total;
+        }
+
+    }
+
+
+};
+
+
+
+
+
+
+
+
+/******************************************************************************
+ * Tuning policy types
+ ******************************************************************************/
+
+/**
+ * Parameterizable tuning policy type for BlockSegReduceRegionByKey
+ */
+template <
+    int                     _BLOCK_THREADS,             ///< Threads per thread block
+    int                     _ITEMS_PER_THREAD,          ///< Items per thread (per tile of input)
+    BlockLoadAlgorithm      _LOAD_ALGORITHM,            ///< The BlockLoad algorithm to use
+    bool                    _LOAD_WARP_TIME_SLICING,    ///< Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any load-related data transpositions (versus each warp having its own storage)
+    CacheLoadModifier       _LOAD_MODIFIER,             ///< Cache load modifier for reading input elements
+    BlockScanAlgorithm      _SCAN_ALGORITHM>            ///< The BlockScan algorithm to use
+struct BlockSegReduceRegionByKeyPolicy
+{
+    enum
+    {
+        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
+        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
+        LOAD_WARP_TIME_SLICING  = _LOAD_WARP_TIME_SLICING,      ///< Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any load-related data transpositions (versus each warp having its own storage)    };
+    };
+
+    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
+    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
+    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
+};
+
+
+/******************************************************************************
+ * Persistent thread block types
+ ******************************************************************************/
+
+/**
+ * \brief BlockSegReduceRegionByKey implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key.
+ */
+template <
+    typename    BlockSegReduceRegionByKeyPolicy,        ///< Parameterized BlockSegReduceRegionByKeyPolicy tuning policy
+    typename    InputIteratorT,                         ///< Random-access iterator referencing key-value input tuples
+    typename    OutputIteratorT,                        ///< Random-access iterator referencing segment output totals
+    typename    ReductionOp>                            ///< Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+struct BlockSegReduceRegionByKey
+{
+    //---------------------------------------------------------------------
+    // Types and constants
+    //---------------------------------------------------------------------
+
+    // Constants
+    enum
+    {
+        BLOCK_THREADS       = BlockSegReduceRegionByKeyPolicy::BLOCK_THREADS,
+        ITEMS_PER_THREAD    = BlockSegReduceRegionByKeyPolicy::ITEMS_PER_THREAD,
+        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
+    };
+
+    // KeyValuePair input type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type KeyValuePair;
+
+    // Signed integer type for global offsets
+    typedef typename KeyValuePair::Key OffsetT;
+
+    // Value type
+    typedef typename KeyValuePair::Value Value;
+
+    // Head flag type
+    typedef int HeadFlag;
+
+    // Input iterator wrapper type for loading KeyValuePair elements through cache
+    typedef CacheModifiedInputIterator<
+            BlockSegReduceRegionByKeyPolicy::LOAD_MODIFIER,
+            KeyValuePair,
+            OffsetT>
+        WrappedInputIteratorT;
+
+    // Parameterized BlockLoad type
+    typedef BlockLoad<
+            WrappedInputIteratorT,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            BlockSegReduceRegionByKeyPolicy::LOAD_ALGORITHM,
+            BlockSegReduceRegionByKeyPolicy::LOAD_WARP_TIME_SLICING>
+        BlockLoad;
+
+    // BlockScan scan operator for reduction-by-segment
+    typedef ReduceByKeyOp<ReductionOp> ReduceByKeyOp;
+
+    // Stateful BlockScan prefix callback type for managing a running total while scanning consecutive tiles
+    typedef RunningBlockPrefixCallbackOp<
+            KeyValuePair,
+            ReduceByKeyOp>
+        RunningPrefixCallbackOp;
+
+    // Parameterized BlockScan type for block-wide reduce-value-by-key
+    typedef BlockScan<
+            KeyValuePair,
+            BLOCK_THREADS,
+            BlockSegReduceRegionByKeyPolicy::SCAN_ALGORITHM>
+        BlockScan;
+
+    // Parameterized BlockDiscontinuity type for identifying key discontinuities
+    typedef BlockDiscontinuity<
+            OffsetT,
+            BLOCK_THREADS>
+        BlockDiscontinuity;
+
+    // Operator for detecting discontinuities in a list of segment identifiers.
+    struct NewSegmentOp
+    {
+        /// Returns true if row_b is the start of a new row
+        __device__ __forceinline__ bool operator()(const OffsetT& b, const OffsetT& a)
+        {
+            return (a != b);
+        }
+    };
+
+    // Shared memory type for this thread block
+    struct _TempStorage
+    {
+        union
+        {
+            typename BlockLoad::TempStorage                 load;           // Smem needed for tile loading
+            struct {
+                typename BlockScan::TempStorage             scan;           // Smem needed for reduce-value-by-segment scan
+                typename BlockDiscontinuity::TempStorage    discontinuity;  // Smem needed for head-flagging
+            };
+        };
+    };
+
+    // Alias wrapper allowing storage to be unioned
+    struct TempStorage : Uninitialized<_TempStorage> {};
+
+
+    //---------------------------------------------------------------------
+    // Thread fields
+    //---------------------------------------------------------------------
+
+    _TempStorage                &temp_storage;          ///< Reference to shared storage
+    WrappedInputIteratorT       d_tuple_partials;       ///< A sequence of partial reduction tuples to scan
+    OutputIteratorT              d_output;               ///< A sequence of segment totals
+    Value                       identity;               ///< Identity value (for zero-length segments)
+    ReduceByKeyOp               scan_op;                ///< Reduce-by-key scan operator
+    RunningPrefixCallbackOp     prefix_op;              ///< Stateful running total for block-wide prefix scan of partial reduction tuples
+
+
+    //---------------------------------------------------------------------
+    // Operations
+    //---------------------------------------------------------------------
+
+    /**
+     * Constructor
+     */
+    __device__ __forceinline__
+    BlockSegReduceRegionByKey(
+        TempStorage             &temp_storage,          ///< Reference to shared storage
+        InputIteratorT          d_tuple_partials,       ///< A sequence of partial reduction tuples to scan
+        OutputIteratorT          d_output,               ///< A sequence of segment totals
+        Value                   identity,               ///< Identity value (for zero-length segments)
+        ReductionOp             reduction_op)           ///< Reduction operator
+    :
+        temp_storage(temp_storage.Alias()),
+        d_tuple_partials(d_tuple_partials),
+        d_output(d_output),
+        identity(identity),
+        scan_op(reduction_op),
+        prefix_op(scan_op)
+    {}
+
+
+
+    /**
+     * Processes a reduce-value-by-key input tile, outputting reductions for each segment
+     */
+    template <bool FULL_TILE>
+    __device__ __forceinline__
+    void ProcessTile(
+        OffsetT block_offset,
+        OffsetT first_segment_idx,
+        OffsetT last_segment_idx,
+        int guarded_items = TILE_ITEMS)
+    {
+        KeyValuePair    partial_reductions[ITEMS_PER_THREAD];
+        OffsetT         segment_ids[ITEMS_PER_THREAD];
+        HeadFlag        head_flags[ITEMS_PER_THREAD];
+
+        // Load a tile of block partials from previous kernel
+        if (FULL_TILE)
+        {
+            // Full tile
+            BlockLoad(temp_storage.load).Load(d_tuple_partials + block_offset, partial_reductions);
+        }
+        else
+        {
+            KeyValuePair oob_default;
+            oob_default.key    = last_segment_idx;       // The last segment ID to be reduced
+            oob_default.value  = identity;
+
+            // Partially-full tile
+            BlockLoad(temp_storage.load).Load(d_tuple_partials + block_offset, partial_reductions, guarded_items, oob_default);
+        }
+
+        // Barrier for shared memory reuse
+        __syncthreads();
+
+        // Copy the segment IDs for head-flagging
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            segment_ids[ITEM] = partial_reductions[ITEM].key;
+        }
+
+        // FlagT segment heads by looking for discontinuities
+        BlockDiscontinuity(temp_storage.discontinuity).FlagHeads(
+            head_flags,                         // [out] Head flags
+            segment_ids,                        // Segment ids
+            NewSegmentOp(),                     // Functor for detecting start of new rows
+            prefix_op.running_total.key);       // Last segment ID from previous tile to compare with first segment ID in this tile
+
+        // Reduce-value-by-segment across partial_reductions using exclusive prefix scan
+        KeyValuePair block_aggregate;
+        BlockScan(temp_storage.scan).ExclusiveScan(
+            partial_reductions,                   // Scan input
+            partial_reductions,                   // Scan output
+            scan_op,                        // Scan operator
+            block_aggregate,                // Block-wide total (unused)
+            prefix_op);                     // Prefix operator for seeding the block-wide scan with the running total
+
+        // Scatter an accumulated reduction if it is the head of a valid segment
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
+        {
+            if (head_flags[ITEM])
+            {
+                d_output[partial_reductions[ITEM].key] = partial_reductions[ITEM].value;
+            }
+        }
+    }
+
+
+    /**
+     * Iterate over input tiles belonging to this thread block
+     */
+    __device__ __forceinline__
+    void ProcessRegion(
+        OffsetT block_offset,
+        OffsetT block_end,
+        OffsetT first_segment_idx,
+        OffsetT last_segment_idx)
+    {
+        if (threadIdx.x == 0)
+        {
+            // Initialize running prefix to the first segment index paired with identity
+            prefix_op.running_total.key    = first_segment_idx;
+            prefix_op.running_total.value  = identity;
+        }
+
+        // Process full tiles
+        while (block_offset + TILE_ITEMS <= block_end)
+        {
+            ProcessTile<true>(block_offset, first_segment_idx, last_segment_idx);
+            __syncthreads();
+
+            block_offset += TILE_ITEMS;
+        }
+
+        // Process final value tile (if present)
+        int guarded_items = block_end - block_offset;
+        if (guarded_items)
+        {
+            ProcessTile<false>(block_offset, first_segment_idx, last_segment_idx, guarded_items);
+        }
+    }
+};
+
+
+
+/******************************************************************************
+ * Kernel entrypoints
+ ******************************************************************************/
+
+/**
+ * Segmented reduce region kernel entry point (multi-block).
+ */
+
+template <
+    typename SegmentOffsetIterator,             ///< Random-access input iterator type for reading segment end-offsets
+    typename OffsetT>                           ///< Signed integer type for global offsets
+__global__ void SegReducePartitionKernel(
+    SegmentOffsetIterator       d_segment_end_offsets,  ///< [in] A sequence of \p num_segments segment end-offsets
+    IndexPair<OffsetT>          *d_block_idx,
+    int                         num_partition_samples,
+    OffsetT                     num_values,             ///< [in] Number of values to reduce
+    OffsetT                     num_segments,           ///< [in] Number of segments being reduced
+    GridEvenShare<OffsetT>      even_share)             ///< [in] Even-share descriptor for mapping an equal number of tiles onto each thread block
+{
+    // Segment offset type
+    typedef typename std::iterator_traits<SegmentOffsetIterator>::value_type SegmentOffset;
+
+    // Counting iterator type
+    typedef CountingInputIterator<SegmentOffsetT, OffsetT> CountingIterator;
+
+    // Cache-modified iterator for segment end-offsets
+    CacheModifiedInputIterator<LOAD_LDG, SegmentOffsetT, OffsetT> d_wrapped_segment_end_offsets(d_segment_end_offsets);
+
+    // Counting iterator for value offsets
+    CountingIterator d_value_offsets(0);
+
+    // Initialize even-share to tell us where to start and stop our tile-processing
+    int partition_id = (blockDim.x * blockIdx.x) + threadIdx.x;
+    even_share.Init(partition_id);
+
+    // Search for block starting and ending indices
+    IndexPair<OffsetT> start_idx = {0, 0};
+    IndexPair<OffsetT> end_idx   = {num_segments, num_values};
+    IndexPair<OffsetT> block_idx;
+
+    MergePathSearch(
+        even_share.block_offset,            // Next thread diagonal
+        d_wrapped_segment_end_offsets,      // A (segment end-offsets)
+        d_value_offsets,                    // B (value offsets)
+        start_idx,                          // Start indices into A and B
+        end_idx,                            // End indices into A and B
+        block_idx);                         // [out] diagonal intersection indices into A and B
+
+    // Write output
+    if (partition_id < num_partition_samples)
+    {
+        d_block_idx[partition_id] = block_idx;
+    }
+}
+
+
+/**
+ * Segmented reduce region kernel entry point (multi-block).
+ */
+template <
+    typename BlockSegReduceRegionPolicy,        ///< Parameterized BlockSegReduceRegionPolicy tuning policy
+    typename SegmentOffsetIterator,             ///< Random-access input iterator type for reading segment end-offsets
+    typename ValueIterator,                     ///< Random-access input iterator type for reading values
+    typename OutputIteratorT,                   ///< Random-access output iterator type for writing segment reductions
+    typename ReductionOp,                       ///< Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+    typename OffsetT,                           ///< Signed integer type for global offsets
+    typename Value>                             ///< Value type
+__launch_bounds__ (BlockSegReduceRegionPolicy::BLOCK_THREADS)
+__global__ void SegReduceRegionKernel(
+    SegmentOffsetIterator       d_segment_end_offsets,  ///< [in] A sequence of \p num_segments segment end-offsets
+    ValueIterator               d_values,               ///< [in] A sequence of \p num_values values
+    OutputIteratorT              d_output,               ///< [out] A sequence of \p num_segments segment totals
+    KeyValuePair<OffsetT, Value> *d_tuple_partials,      ///< [out] A sequence of (gridDim.x * 2) partial reduction tuples
+    IndexPair<OffsetT>          *d_block_idx,
+    OffsetT                     num_values,             ///< [in] Number of values to reduce
+    OffsetT                     num_segments,           ///< [in] Number of segments being reduced
+    Value                       identity,               ///< [in] Identity value (for zero-length segments)
+    ReductionOp                 reduction_op,           ///< [in] Reduction operator
+    GridEvenShare<OffsetT>      even_share)             ///< [in] Even-share descriptor for mapping an equal number of tiles onto each thread block
+{
+    typedef KeyValuePair<OffsetT, Value> KeyValuePair;
+
+    // Specialize thread block abstraction type for reducing a range of segmented values
+    typedef BlockSegReduceRegion<
+            BlockSegReduceRegionPolicy,
+            SegmentOffsetIterator,
+            ValueIterator,
+            OutputIteratorT,
+            ReductionOp,
+            OffsetT>
+        BlockSegReduceRegion;
+
+    // Shared memory allocation
+    __shared__ typename BlockSegReduceRegion::TempStorage temp_storage;
+
+    // Initialize thread block even-share to tell us where to start and stop our tile-processing
+    even_share.BlockInit();
+
+    // Construct persistent thread block
+    BlockSegReduceRegion thread_block(
+        temp_storage,
+        d_segment_end_offsets,
+        d_values,
+        d_output,
+        d_block_idx,
+        num_values,
+        num_segments,
+        identity,
+        reduction_op);
+
+    // First and last partial reduction tuples within the range (valid in thread-0)
+    KeyValuePair first_tuple, last_tuple;
+
+    // Consume block's region of work
+    thread_block.ProcessRegion(
+        even_share.block_offset,
+        even_share.block_end,
+        first_tuple,
+        last_tuple);
+
+    if (threadIdx.x == 0)
+    {
+        if (gridDim.x > 1)
+        {
+            // Special case where the first segment written and the carry-out are for the same segment
+            if (first_tuple.key == last_tuple.key)
+            {
+                first_tuple.value = identity;
+            }
+
+            // Write the first and last partial products from this thread block so
+            // that they can be subsequently "fixed up" in the next kernel.
+            d_tuple_partials[blockIdx.x * 2]          = first_tuple;
+            d_tuple_partials[(blockIdx.x * 2) + 1]    = last_tuple;
+        }
+    }
+
+}
+
+
+/**
+ * Segmented reduce region kernel entry point (single-block).
+ */
+template <
+    typename    BlockSegReduceRegionByKeyPolicy,        ///< Parameterized BlockSegReduceRegionByKeyPolicy tuning policy
+    typename    InputIteratorT,                         ///< Random-access iterator referencing key-value input tuples
+    typename    OutputIteratorT,                        ///< Random-access iterator referencing segment output totals
+    typename    ReductionOp,                            ///< Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+    typename    OffsetT,                                ///< Signed integer type for global offsets
+    typename    Value>                                  ///< Value type
+__launch_bounds__ (BlockSegReduceRegionByKeyPolicy::BLOCK_THREADS, 1)
+__global__ void SegReduceRegionByKeyKernel(
+    InputIteratorT          d_tuple_partials,           ///< [in] A sequence of partial reduction tuples
+    OutputIteratorT          d_output,                   ///< [out] A sequence of \p num_segments segment totals
+    OffsetT                 num_segments,               ///< [in] Number of segments in the \p d_output sequence
+    int                     num_tuple_partials,         ///< [in] Number of partial reduction tuples being reduced
+    Value                   identity,                   ///< [in] Identity value (for zero-length segments)
+    ReductionOp             reduction_op)               ///< [in] Reduction operator
+{
+    // Specialize thread block abstraction type for reducing a range of values by key
+    typedef BlockSegReduceRegionByKey<
+            BlockSegReduceRegionByKeyPolicy,
+            InputIteratorT,
+            OutputIteratorT,
+            ReductionOp>
+        BlockSegReduceRegionByKey;
+
+    // Shared memory allocation
+    __shared__ typename BlockSegReduceRegionByKey::TempStorage temp_storage;
+
+    // Construct persistent thread block
+    BlockSegReduceRegionByKey thread_block(
+        temp_storage,
+        d_tuple_partials,
+        d_output,
+        identity,
+        reduction_op);
+
+    // Process input tiles
+    thread_block.ProcessRegion(
+        0,                          // Region start
+        num_tuple_partials,         // Region end
+        0,                          // First segment ID
+        num_segments);              // Last segment ID (one-past)
+}
+
+
+
+
+/******************************************************************************
+ * Dispatch
+ ******************************************************************************/
+
+/**
+ * Utility class for dispatching the appropriately-tuned kernels for DeviceReduce
+ */
+template <
+    typename ValueIterator,                     ///< Random-access input iterator type for reading values
+    typename SegmentOffsetIterator,             ///< Random-access input iterator type for reading segment end-offsets
+    typename OutputIteratorT,                   ///< Random-access output iterator type for writing segment reductions
+    typename ReductionOp,                       ///< Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+    typename OffsetT>                           ///< Signed integer type for global offsets
+struct DeviceSegReduceDispatch
+{
+    // Value type
+    typedef typename std::iterator_traits<ValueIterator>::value_type Value;
+
+    // Reduce-by-key data type tuple (segment-ID, value)
+    typedef KeyValuePair<OffsetT, Value> KeyValuePair;
+
+    // Index pair data type
+    typedef IndexPair<OffsetT>IndexPair;
+
+
+    /******************************************************************************
+     * Tuning policies
+     ******************************************************************************/
+
+    /// SM35
+    struct Policy350
+    {
+        // ReduceRegionPolicy
+        typedef BlockSegReduceRegionPolicy<
+                128,                            ///< Threads per thread block
+                6,                              ///< Items per thread (per tile of input)
+                true,                           ///< Whether or not to cache incoming segment offsets in shared memory before reducing each tile
+                false,                          ///< Whether or not to cache incoming values in shared memory before reducing each tile
+                LOAD_DEFAULT,                   ///< Cache load modifier for reading segment offsets
+                LOAD_LDG,                       ///< Cache load modifier for reading values
+                BLOCK_REDUCE_RAKING,            ///< The BlockReduce algorithm to use
+                BLOCK_SCAN_WARP_SCANS>          ///< The BlockScan algorithm to use
+            SegReduceRegionPolicy;
+
+        // ReduceRegionByKeyPolicy
+        typedef BlockSegReduceRegionByKeyPolicy<
+                256,                            ///< Threads per thread block
+                9,                             ///< Items per thread (per tile of input)
+                BLOCK_LOAD_DIRECT,              ///< The BlockLoad algorithm to use
+                false,                          ///< Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any load-related data transpositions (versus each warp having its own storage)
+                LOAD_LDG,                       ///< Cache load modifier for reading input elements
+                BLOCK_SCAN_WARP_SCANS>          ///< The BlockScan algorithm to use
+            SegReduceRegionByKeyPolicy;
+    };
+
+
+    /// SM10
+    struct Policy100
+    {
+        // ReduceRegionPolicy
+        typedef BlockSegReduceRegionPolicy<
+                128,                            ///< Threads per thread block
+                3,                              ///< Items per thread (per tile of input)
+                false,                          ///< Whether or not to cache incoming segment offsets in shared memory before reducing each tile
+                false,                          ///< Whether or not to cache incoming values in shared memory before reducing each tile
+                LOAD_DEFAULT,                   ///< Cache load modifier for reading segment offsets
+                LOAD_DEFAULT,                   ///< Cache load modifier for reading values
+                BLOCK_REDUCE_RAKING,            ///< The BlockReduce algorithm to use
+                BLOCK_SCAN_RAKING>              ///< The BlockScan algorithm to use
+            SegReduceRegionPolicy;
+
+        // ReduceRegionByKeyPolicy
+        typedef BlockSegReduceRegionByKeyPolicy<
+                128,                            ///< Threads per thread block
+                3,                              ///< Items per thread (per tile of input)
+                BLOCK_LOAD_WARP_TRANSPOSE,      ///< The BlockLoad algorithm to use
+                false,                          ///< Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any load-related data transpositions (versus each warp having its own storage)
+                LOAD_DEFAULT,                   ///< Cache load modifier for reading input elements
+                BLOCK_SCAN_WARP_SCANS>          ///< The BlockScan algorithm to use
+            SegReduceRegionByKeyPolicy;
+    };
+
+
+    /******************************************************************************
+     * Tuning policies of current PTX compiler pass
+     ******************************************************************************/
+
+#if (CUB_PTX_ARCH >= 350)
+    typedef Policy350 PtxPolicy;
+/*
+#elif (CUB_PTX_ARCH >= 300)
+    typedef Policy300 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 200)
+    typedef Policy200 PtxPolicy;
+
+#elif (CUB_PTX_ARCH >= 130)
+    typedef Policy130 PtxPolicy;
+*/
+#else
+    typedef Policy100 PtxPolicy;
+
+#endif
+
+    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
+    struct PtxSegReduceRegionPolicy           : PtxPolicy::SegReduceRegionPolicy {};
+    struct PtxSegReduceRegionByKeyPolicy      : PtxPolicy::SegReduceRegionByKeyPolicy {};
+
+
+    /******************************************************************************
+     * Utilities
+     ******************************************************************************/
+
+    /**
+     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
+     */
+    template <
+        typename SegReduceKernelConfig,
+        typename SegReduceByKeyKernelConfig>
+    __host__ __device__ __forceinline__
+    static void InitConfigs(
+        int                         ptx_version,
+        SegReduceKernelConfig       &seg_reduce_region_config,
+        SegReduceByKeyKernelConfig  &seg_reduce_region_by_key_config)
+    {
+    #if (CUB_PTX_ARCH > 0)
+
+        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
+        seg_reduce_region_config.Init<PtxSegReduceRegionPolicy>();
+        seg_reduce_region_by_key_config.Init<PtxSegReduceRegionByKeyPolicy>();
+
+    #else
+
+        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
+        if (ptx_version >= 350)
+        {
+            seg_reduce_region_config.template          Init<typename Policy350::SegReduceRegionPolicy>();
+            seg_reduce_region_by_key_config.template   Init<typename Policy350::SegReduceRegionByKeyPolicy>();
+        }
+/*
+        else if (ptx_version >= 300)
+        {
+            seg_reduce_region_config.template          Init<typename Policy300::SegReduceRegionPolicy>();
+            seg_reduce_region_by_key_config.template   Init<typename Policy300::SegReduceRegionByKeyPolicy>();
+        }
+        else if (ptx_version >= 200)
+        {
+            seg_reduce_region_config.template          Init<typename Policy200::SegReduceRegionPolicy>();
+            seg_reduce_region_by_key_config.template   Init<typename Policy200::SegReduceRegionByKeyPolicy>();
+        }
+        else if (ptx_version >= 130)
+        {
+            seg_reduce_region_config.template          Init<typename Policy130::SegReduceRegionPolicy>();
+            seg_reduce_region_by_key_config.template   Init<typename Policy130::SegReduceRegionByKeyPolicy>();
+        }
+*/
+        else
+        {
+            seg_reduce_region_config.template          Init<typename Policy100::SegReduceRegionPolicy>();
+            seg_reduce_region_by_key_config.template   Init<typename Policy100::SegReduceRegionByKeyPolicy>();
+        }
+
+    #endif
+    }
+
+
+    /**
+     * SegReduceRegionKernel kernel dispatch configuration
+     */
+    struct SegReduceKernelConfig
+    {
+        int                     block_threads;
+        int                     items_per_thread;
+        bool                    use_smem_segment_cache;
+        bool                    use_smem_value_cache;
+        CacheLoadModifier       load_modifier_segments;
+        CacheLoadModifier       load_modifier_values;
+        BlockReduceAlgorithm    reduce_algorithm;
+        BlockScanAlgorithm      scan_algorithm;
+
+        template <typename SegReduceRegionPolicy>
+        __host__ __device__ __forceinline__
+        void Init()
+        {
+            block_threads               = SegReduceRegionPolicy::BLOCK_THREADS;
+            items_per_thread            = SegReduceRegionPolicy::ITEMS_PER_THREAD;
+            use_smem_segment_cache      = SegReduceRegionPolicy::USE_SMEM_SEGMENT_CACHE;
+            use_smem_value_cache        = SegReduceRegionPolicy::USE_SMEM_VALUE_CACHE;
+            load_modifier_segments      = SegReduceRegionPolicy::LOAD_MODIFIER_SEGMENTS;
+            load_modifier_values        = SegReduceRegionPolicy::LOAD_MODIFIER_VALUES;
+            reduce_algorithm            = SegReduceRegionPolicy::REDUCE_ALGORITHM;
+            scan_algorithm              = SegReduceRegionPolicy::SCAN_ALGORITHM;
+        }
+    };
+
+    /**
+     * SegReduceRegionByKeyKernel kernel dispatch configuration
+     */
+    struct SegReduceByKeyKernelConfig
+    {
+        int                     block_threads;
+        int                     items_per_thread;
+        BlockLoadAlgorithm      load_algorithm;
+        bool                    load_warp_time_slicing;
+        CacheLoadModifier       load_modifier;
+        BlockScanAlgorithm      scan_algorithm;
+
+        template <typename SegReduceRegionByKeyPolicy>
+        __host__ __device__ __forceinline__
+        void Init()
+        {
+            block_threads               = SegReduceRegionByKeyPolicy::BLOCK_THREADS;
+            items_per_thread            = SegReduceRegionByKeyPolicy::ITEMS_PER_THREAD;
+            load_algorithm              = SegReduceRegionByKeyPolicy::LOAD_ALGORITHM;
+            load_warp_time_slicing      = SegReduceRegionByKeyPolicy::LOAD_WARP_TIME_SLICING;
+            load_modifier               = SegReduceRegionByKeyPolicy::LOAD_MODIFIER;
+            scan_algorithm              = SegReduceRegionByKeyPolicy::SCAN_ALGORITHM;
+        }
+    };
+
+
+    /******************************************************************************
+     * Dispatch entrypoints
+     ******************************************************************************/
+
+    /**
+     * Internal dispatch routine for computing a device-wide segmented reduction.
+     */
+    template <
+        typename                        SegReducePartitionKernelPtr,
+        typename                        SegReduceRegionKernelPtr,               ///< Function type of cub::SegReduceRegionKernel
+        typename                        SegReduceRegionByKeyKernelPtr>          ///< Function type of cub::SegReduceRegionByKeyKernel
+    __host__ __device__ __forceinline__
+    static cudaError_t Dispatch(
+        void*               d_temp_storage,                        ///< [in] %Device allocation of temporary storage.  When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done.
+        size_t                          &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation.
+        ValueIterator                   d_values,                               ///< [in] A sequence of \p num_values data to reduce
+        SegmentOffsetIterator           d_segment_offsets,                      ///< [in] A sequence of (\p num_segments + 1) segment offsets
+        OutputIteratorT                  d_output,                               ///< [out] A sequence of \p num_segments segment totals
+        OffsetT                         num_values,                             ///< [in] Total number of values to reduce
+        OffsetT                         num_segments,                           ///< [in] Number of segments being reduced
+        Value                           identity,                               ///< [in] Identity value (for zero-length segments)
+        ReductionOp                     reduction_op,                           ///< [in] Reduction operator
+        cudaStream_t                    stream,                                 ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                            debug_synchronous,                      ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+        int                             sm_version,                             ///< [in] SM version of target device to use when computing SM occupancy
+        SegReducePartitionKernelPtr     seg_reduce_partition_kernel,            ///< [in] Kernel function pointer to parameterization of cub::SegReduceRegionKernel
+        SegReduceRegionKernelPtr        seg_reduce_region_kernel,               ///< [in] Kernel function pointer to parameterization of cub::SegReduceRegionKernel
+        SegReduceRegionByKeyKernelPtr   seg_reduce_region_by_key_kernel,        ///< [in] Kernel function pointer to parameterization of cub::SegReduceRegionByKeyKernel
+        SegReduceKernelConfig           &seg_reduce_region_config,              ///< [in] Dispatch parameters that match the policy that \p seg_reduce_region_kernel was compiled for
+        SegReduceByKeyKernelConfig      &seg_reduce_region_by_key_config)       ///< [in] Dispatch parameters that match the policy that \p seg_reduce_region_by_key_kernel was compiled for
+    {
+#ifndef CUB_RUNTIME_ENABLED
+
+        // Kernel launch not supported from this device
+        return CubDebug(cudaErrorNotSupported );
+
+#else
+
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Dispatch two kernels: (1) a multi-block segmented reduction
+            // to reduce regions by block, and (2) a single-block reduce-by-key kernel
+            // to "fix up" segments spanning more than one region.
+
+            // Tile size of seg_reduce_region_kernel
+            int tile_size = seg_reduce_region_config.block_threads * seg_reduce_region_config.items_per_thread;
+
+            // Get device ordinal
+            int device_ordinal;
+            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
+
+            // Get SM count
+            int sm_count;
+            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
+
+            // Get SM occupancy for histogram_region_kernel
+            int seg_reduce_region_sm_occupancy;
+            if (CubDebug(error = MaxSmOccupancy(
+                seg_reduce_region_sm_occupancy,
+                sm_version,
+                seg_reduce_region_kernel,
+                seg_reduce_region_config.block_threads))) break;
+
+            // Get device occupancy for histogram_region_kernel
+            int seg_reduce_region_occupancy = seg_reduce_region_sm_occupancy * sm_count;
+
+            // Even-share work distribution
+            int num_diagonals = num_values + num_segments;                  // Total number of work items
+            int subscription_factor = seg_reduce_region_sm_occupancy;       // Amount of CTAs to oversubscribe the device beyond actively-resident (heuristic)
+            int max_grid_size = seg_reduce_region_occupancy * subscription_factor;
+            GridEvenShare<OffsetT>even_share(
+                num_diagonals,
+                max_grid_size,
+                tile_size);
+
+            // Get grid size for seg_reduce_region_kernel
+            int seg_reduce_region_grid_size = even_share.grid_size;
+
+            // Number of "fix-up" reduce-by-key tuples (2 per thread block)
+            int num_tuple_partials = seg_reduce_region_grid_size * 2;
+            int num_partition_samples = seg_reduce_region_grid_size + 1;
+
+            // Temporary storage allocation requirements
+            void* allocations[2];
+            size_t allocation_sizes[2] =
+            {
+                num_tuple_partials * sizeof(KeyValuePair),  // bytes needed for "fix-up" reduce-by-key tuples
+                num_partition_samples * sizeof(IndexPair),  // bytes needed block indices
+            };
+
+            // Alias the temporary allocations from the single storage blob (or set the necessary size of the blob)
+            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
+            if (d_temp_storage == NULL)
+            {
+                // Return if the caller is simply requesting the size of the storage allocation
+                return cudaSuccess;
+            }
+
+            // Alias the allocations
+            KeyValuePair    *d_tuple_partials   = (KeyValuePair*) allocations[0];           // "fix-up" tuples
+            IndexPair       *d_block_idx        = (IndexPair *) allocations[1];             // block starting/ending indices
+
+            // Array of segment end-offsets
+            SegmentOffsetIterator d_segment_end_offsets = d_segment_offsets + 1;
+
+            // Grid launch params for seg_reduce_partition_kernel
+            int partition_block_size = 32;
+            int partition_grid_size = (num_partition_samples + partition_block_size - 1) / partition_block_size;
+
+            // Partition work among multiple thread blocks if necessary
+            if (seg_reduce_region_grid_size > 1)
+            {
+                // Log seg_reduce_partition_kernel configuration
+                if (debug_synchronous) _CubLog("Invoking seg_reduce_partition_kernel<<<%d, %d, 0, %lld>>>()\n",
+                    partition_grid_size, partition_block_size, (long long) stream);
+
+                // Invoke seg_reduce_partition_kernel
+                seg_reduce_partition_kernel<<<partition_grid_size, partition_block_size, 0, stream>>>(
+                    d_segment_end_offsets,  ///< [in] A sequence of \p num_segments segment end-offsets
+                    d_block_idx,
+                    num_partition_samples,
+                    num_values,             ///< [in] Number of values to reduce
+                    num_segments,           ///< [in] Number of segments being reduced
+                    even_share);            ///< [in] Even-share descriptor for mapping an equal number of tiles onto each thread block
+
+                // Sync the stream if specified
+                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+            }
+
+            // Log seg_reduce_region_kernel configuration
+            if (debug_synchronous) _CubLog("Invoking seg_reduce_region_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
+                seg_reduce_region_grid_size, seg_reduce_region_config.block_threads, (long long) stream, seg_reduce_region_config.items_per_thread, seg_reduce_region_sm_occupancy);
+
+            // Mooch
+            if (CubDebug(error = cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte))) break;
+
+            // Invoke seg_reduce_region_kernel
+            seg_reduce_region_kernel<<<seg_reduce_region_grid_size, seg_reduce_region_config.block_threads, 0, stream>>>(
+                d_segment_end_offsets,
+                d_values,
+                d_output,
+                d_tuple_partials,
+                d_block_idx,
+                num_values,
+                num_segments,
+                identity,
+                reduction_op,
+                even_share);
+
+            // Sync the stream if specified
+            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+/*
+            // Perform "fix-up" of region partial reductions if grid size is greater than one thread block
+            if (seg_reduce_region_grid_size > 1)
+            {
+                // Log seg_reduce_region_by_key_kernel configuration
+                if (debug_synchronous) _CubLog("Invoking seg_reduce_region_by_key_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread\n",
+                    1, seg_reduce_region_by_key_config.block_threads, (long long) stream, seg_reduce_region_by_key_config.items_per_thread);
+
+                // Invoke seg_reduce_region_by_key_kernel
+                seg_reduce_region_by_key_kernel<<<1, seg_reduce_region_by_key_config.block_threads, 0, stream>>>(
+                    d_tuple_partials,
+                    d_output,
+                    num_segments,
+                    num_tuple_partials,
+                    identity,
+                    reduction_op);
+
+                // Sync the stream if specified
+                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
+            }
+*/
+        }
+
+        while (0);
+
+        return error;
+
+#endif // CUB_RUNTIME_ENABLED
+    }
+
+
+    /**
+     * Internal dispatch routine for computing a device-wide segmented reduction.
+     */
+    __host__ __device__ __forceinline__
+    static cudaError_t Dispatch(
+        void*               d_temp_storage,                        ///< [in] %Device allocation of temporary storage.  When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done.
+        size_t                          &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation.
+        ValueIterator                   d_values,                               ///< [in] A sequence of \p num_values data to reduce
+        SegmentOffsetIterator           d_segment_offsets,                      ///< [in] A sequence of (\p num_segments + 1) segment offsets
+        OutputIteratorT                  d_output,                               ///< [out] A sequence of \p num_segments segment totals
+        OffsetT                         num_values,                             ///< [in] Total number of values to reduce
+        OffsetT                         num_segments,                           ///< [in] Number of segments being reduced
+        Value                           identity,                               ///< [in] Identity value (for zero-length segments)
+        ReductionOp                     reduction_op,                           ///< [in] Reduction operator
+        cudaStream_t                    stream,                                 ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                            debug_synchronous)                      ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        cudaError error = cudaSuccess;
+        do
+        {
+            // Get PTX version
+            int ptx_version;
+    #if (CUB_PTX_ARCH == 0)
+            if (CubDebug(error = PtxVersion(ptx_version))) break;
+    #else
+            ptx_version = CUB_PTX_ARCH;
+    #endif
+
+            // Get kernel kernel dispatch configurations
+            SegReduceKernelConfig seg_reduce_region_config;
+            SegReduceByKeyKernelConfig seg_reduce_region_by_key_config;
+
+            InitConfigs(ptx_version, seg_reduce_region_config, seg_reduce_region_by_key_config);
+
+            // Dispatch
+            if (CubDebug(error = Dispatch(
+                d_temp_storage,
+                temp_storage_bytes,
+                d_values,
+                d_segment_offsets,
+                d_output,
+                num_values,
+                num_segments,
+                identity,
+                reduction_op,
+                stream,
+                debug_synchronous,
+                ptx_version,            // Use PTX version instead of SM version because, as a statically known quantity, this improves device-side launch dramatically but at the risk of imprecise occupancy calculation for mismatches
+                SegReducePartitionKernel<SegmentOffsetIterator, OffsetT>,
+                SegReduceRegionKernel<PtxSegReduceRegionPolicy, SegmentOffsetIterator, ValueIterator, OutputIteratorT, ReductionOp, OffsetT, Value>,
+                SegReduceRegionByKeyKernel<PtxSegReduceRegionByKeyPolicy, KeyValuePair*, OutputIteratorT, ReductionOp, OffsetT, Value>,
+                seg_reduce_region_config,
+                seg_reduce_region_by_key_config))) break;
+        }
+        while (0);
+
+        return error;
+
+    }
+};
+
+
+
+
+/******************************************************************************
+ * DeviceSegReduce
+ *****************************************************************************/
+
+/**
+ * \brief DeviceSegReduce provides operations for computing a device-wide, parallel segmented reduction across a sequence of data items residing within global memory.
+ * \ingroup DeviceModule
+ *
+ * \par Overview
+ * A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
+ * uses a binary combining operator to compute a single aggregate from a list of input elements.
+ *
+ * \par Usage Considerations
+ * \cdp_class{DeviceReduce}
+ *
+ */
+struct DeviceSegReduce
+{
+    /**
+     * \brief Computes a device-wide segmented reduction using the specified binary \p reduction_op functor.
+     *
+     * \par
+     * Does not support non-commutative reduction operators.
+     *
+     * \devicestorage
+     *
+     * \cdp
+     *
+     * \iterator
+     *
+     * \tparam ValueIterator            <b>[inferred]</b> Random-access input iterator type for reading values
+     * \tparam SegmentOffsetIterator    <b>[inferred]</b> Random-access input iterator type for reading segment end-offsets
+     * \tparam OutputIteratorT           <b>[inferred]</b> Random-access output iterator type for writing segment reductions
+     * \tparam Value                    <b>[inferred]</b> Value type
+     * \tparam ReductionOp              <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
+     */
+    template <
+        typename                ValueIterator,
+        typename                SegmentOffsetIterator,
+        typename                OutputIteratorT,
+        typename                Value,
+        typename                ReductionOp>
+    __host__ __device__ __forceinline__
+    static cudaError_t Reduce(
+        void*               d_temp_storage,                        ///< [in] %Device allocation of temporary storage.  When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation.
+        ValueIterator           d_values,                               ///< [in] A sequence of \p num_values data to reduce
+        SegmentOffsetIterator   d_segment_offsets,                      ///< [in] A sequence of (\p num_segments + 1) segment offsets
+        OutputIteratorT          d_output,                               ///< [out] A sequence of \p num_segments segment totals
+        int                     num_values,                             ///< [in] Total number of values to reduce
+        int                     num_segments,                           ///< [in] Number of segments being reduced
+        Value                   identity,                               ///< [in] Identity value (for zero-length segments)
+        ReductionOp             reduction_op,                           ///< [in] Reduction operator
+        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        typedef DeviceSegReduceDispatch<
+                ValueIterator,
+                SegmentOffsetIterator,
+                OutputIteratorT,
+                ReductionOp,
+                OffsetT>
+            DeviceSegReduceDispatch;
+
+        return DeviceSegReduceDispatch::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_values,
+            d_segment_offsets,
+            d_output,
+            num_values,
+            num_segments,
+            identity,
+            reduction_op,
+            stream,
+            debug_synchronous);
+    }
+
+
+    /**
+     * \brief Computes a device-wide segmented sum using the addition ('+') operator.
+     *
+     * \par
+     * Does not support non-commutative summation.
+     *
+     * \devicestorage
+     *
+     * \cdp
+     *
+     * \iterator
+     *
+     * \tparam ValueIterator            <b>[inferred]</b> Random-access input iterator type for reading values
+     * \tparam SegmentOffsetIterator    <b>[inferred]</b> Random-access input iterator type for reading segment end-offsets
+     * \tparam OutputIteratorT           <b>[inferred]</b> Random-access output iterator type for writing segment reductions
+     */
+    template <
+        typename                ValueIterator,
+        typename                SegmentOffsetIterator,
+        typename                OutputIteratorT>
+    __host__ __device__ __forceinline__
+    static cudaError_t Sum(
+        void*               d_temp_storage,                        ///< [in] %Device allocation of temporary storage.  When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done.
+        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation.
+        ValueIterator           d_values,                               ///< [in] A sequence of \p num_values data to reduce
+        SegmentOffsetIterator   d_segment_offsets,                      ///< [in] A sequence of (\p num_segments + 1) segment offsets
+        OutputIteratorT          d_output,                               ///< [out] A sequence of \p num_segments segment totals
+        int                     num_values,                             ///< [in] Total number of values to reduce
+        int                     num_segments,                           ///< [in] Number of segments being reduced
+        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
+        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
+    {
+        // Signed integer type for global offsets
+        typedef int OffsetT;
+
+        // Value type
+        typedef typename std::iterator_traits<ValueIterator>::value_type Value;
+
+        Value identity = Value();
+        cub::Sum reduction_op;
+
+        typedef DeviceSegReduceDispatch<
+                ValueIterator,
+                SegmentOffsetIterator,
+                OutputIteratorT,
+                cub::Sum,
+                OffsetT>
+            DeviceSegReduceDispatch;
+
+        return DeviceSegReduceDispatch::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_values,
+            d_segment_offsets,
+            d_output,
+            num_values,
+            num_segments,
+            identity,
+            reduction_op,
+            stream,
+            debug_synchronous);
+    }
+};
+
+
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+/**
+ * Initialize problem
+ */
+template <typename OffsetT, typename Value>
+void Initialize(
+    GenMode         gen_mode,
+    Value           *h_values,
+    vector<OffsetT> &segment_offsets,
+    int             num_values,
+    int             avg_segment_size)
+{
+    // Initialize values
+//    if (g_verbose) printf("Values: ");
+    for (int i = 0; i < num_values; ++i)
+    {
+        InitValue(gen_mode, h_values[i], i);
+//        if (g_verbose) std::cout << h_values[i] << ", ";
+    }
+//    if (g_verbose) printf("\n\n");
+
+    // Initialize segment lengths
+    const unsigned int  MAX_INTEGER         = -1u;
+    const unsigned int  MAX_SEGMENT_LENGTH  = avg_segment_size * 2;
+    const double        SCALE_FACTOR        = double(MAX_SEGMENT_LENGTH) / double(MAX_INTEGER);
+
+    segment_offsets.push_back(0);
+
+    OffsetT consumed = 0;
+    OffsetT remaining = num_values;
+    while (remaining > 0)
+    {
+        // Randomly sample a 32-bit unsigned int
+        unsigned int segment_length;
+        RandomBits(segment_length);
+
+        // Scale to maximum segment length
+        segment_length = (unsigned int) (double(segment_length) * SCALE_FACTOR);
+        segment_length = CUB_MIN(segment_length, remaining);
+
+        consumed += segment_length;
+        remaining -= segment_length;
+
+        segment_offsets.push_back(consumed);
+    }
+}
+
+
+/**
+ * Compute reference answer
+ */
+template <typename OffsetT, typename Value>
+void ComputeReference(
+    Value       *h_values,
+    OffsetT     *h_segment_offsets,
+    Value       *h_reference,
+    int         num_segments,
+    Value       identity)
+{
+    if (g_verbose) printf("%d segment reductions: ", num_segments);
+    for (int segment = 0; segment < num_segments; ++segment)
+    {
+        h_reference[segment] = identity;
+
+        for (int i = h_segment_offsets[segment]; i < h_segment_offsets[segment + 1]; ++i)
+        {
+            h_reference[segment] += h_values[i];
+        }
+        if (g_verbose) std::cout << h_reference[segment] << ", ";
+    }
+    if (g_verbose) printf("\n\n");
+}
+
+
+/**
+ * Simple test of device
+ */
+template <
+    bool            CDP,
+    typename        OffsetT,
+    typename        Value,
+    typename        ReductionOp>
+void Test(
+    OffsetT         num_values,
+    int             avg_segment_size,
+    ReductionOp     reduction_op,
+    Value           identity,
+    char*           type_string)
+{
+    Value   *h_values = NULL;
+    Value   *h_reference = NULL;
+    OffsetT *h_segment_offsets = NULL;
+
+    printf("%d\n", num_values);
+
+    // Initialize problem on host
+    h_values = new Value[num_values];
+    vector<OffsetT> segment_offsets;
+    Initialize(UNIFORM, h_values, segment_offsets, num_values, avg_segment_size);
+
+    // Allocate simple offsets array and copy STL vector into it
+    h_segment_offsets = new OffsetT[segment_offsets.size()];
+    for (int i = 0; i < segment_offsets.size(); ++i)
+        h_segment_offsets[i] = segment_offsets[i];
+
+    OffsetT num_segments = segment_offsets.size() - 1;
+    if (g_verbose)
+    {
+        printf("%d segment offsets: ", num_segments);
+        for (int i = 0; i < num_segments; ++i)
+            std::cout << h_segment_offsets[i] << "(" << h_segment_offsets[i + 1] - h_segment_offsets[i] << "), ";
+        if (g_verbose) std::cout << std::endl << std::endl;
+    }
+
+    // Solve problem on host
+    h_reference = new Value[num_segments];
+    ComputeReference(h_values, h_segment_offsets, h_reference, num_segments, identity);
+
+    printf("\n\n%s cub::DeviceSegReduce::%s %d items (%d-byte %s), %d segments (%d-byte offset indices)\n",
+        (CDP) ? "CDP device invoked" : "Host-invoked",
+        (Equals<ReductionOp, Sum>::VALUE) ? "Sum" : "Reduce",
+        num_values, (int) sizeof(Value), type_string,
+        num_segments, (int) sizeof(OffsetT));
+    fflush(stdout);
+
+    // Allocate and initialize problem on device
+    Value   *d_values = NULL;
+    OffsetT *d_segment_offsets = NULL;
+    Value   *d_output = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values, sizeof(Value) * num_values));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_segment_offsets, sizeof(OffsetT) * (num_segments + 1)));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_output, sizeof(Value) * num_segments));
+    CubDebugExit(cudaMemcpy(d_values, h_values, sizeof(Value) * num_values, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_segment_offsets, h_segment_offsets, sizeof(OffsetT) * (num_segments + 1), cudaMemcpyHostToDevice));
+
+    // Request and allocate temporary storage
+    void    *d_temp_storage = NULL;
+    size_t  temp_storage_bytes = 0;
+    CubDebugExit(DeviceSegReduce::Sum(d_temp_storage, temp_storage_bytes, d_values, d_segment_offsets, d_output, num_values, num_segments, 0, false));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Clear device output
+    CubDebugExit(cudaMemset(d_output, 0, sizeof(Value) * num_segments));
+
+    // Run warmup/correctness iteration
+    CubDebugExit(DeviceSegReduce::Sum(d_temp_storage, temp_storage_bytes, d_values, d_segment_offsets, d_output, num_values, num_segments, 0, true));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_output, num_segments, true, g_verbose);
+    printf("\t%s", compare ? "FAIL" : "PASS");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Performance
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+    for (int i = 0; i < g_timing_iterations; ++i)
+    {
+        CubDebugExit(DeviceSegReduce::Sum(d_temp_storage, temp_storage_bytes, d_values, d_segment_offsets, d_output, num_values, num_segments, 0, false));
+    }
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float avg_millis = elapsed_millis / g_timing_iterations;
+        float giga_rate = float(num_values) / avg_millis / 1000.0 / 1000.0;
+        float giga_bandwidth = giga_rate *
+        printf(", %.3f avg ms, %.3f billion items/s, %.3f logical GB/s", avg_millis, giga_rate, giga_bandwidth);
+    }
+
+    // Device cleanup
+    if (d_values) CubDebugExit(g_allocator.DeviceFree(d_values));
+    if (d_segment_offsets) CubDebugExit(g_allocator.DeviceFree(d_segment_offsets));
+    if (d_output) CubDebugExit(g_allocator.DeviceFree(d_output));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Host cleanup
+    if (h_values)           delete[] h_values;
+    if (h_segment_offsets)  delete[] h_segment_offsets;
+    if (h_reference)        delete[] h_reference;
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_values          = 32 * 1024 * 1024;
+    int avg_segment_size    = 500;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_values);
+    args.GetCmdLineArgument("ss", avg_segment_size);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--v] "
+            "[--i=<timing iterations>] "
+            "[--n=<input samples>]\n"
+            "[--ss=<average segment size>]\n"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    Test<false>((int) num_values, avg_segment_size, Sum(), (long long) 0, CUB_TYPE_STRING(long long));
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/experimental/histogram/histogram_cub.h b/third_party/cub/experimental/histogram/histogram_cub.h
new file mode 100644
index 0000000..07c2e4a
--- /dev/null
+++ b/third_party/cub/experimental/histogram/histogram_cub.h
@@ -0,0 +1,109 @@
+/******************************************************************************
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+#include <cub/device/device_histogram.cuh>
+
+using namespace cub;
+
+template <
+    int         NUM_CHANNELS,
+    int         ACTIVE_CHANNELS,
+    int         NUM_BINS,
+    typename    PixelType>
+double run_cub_histogram(
+    PixelType *d_image,
+    int width,
+    int height,
+    unsigned int *d_hist, 
+    bool is_warmup)
+{
+    enum {
+        is_float = Equals<PixelType, float4>::VALUE,
+    };
+
+    typedef typename If<is_float, float, unsigned char>::Type    SampleT;    // Sample type
+    typedef typename If<is_float, float, unsigned int>::Type     LevelT;     // Level type (uint32 for uchar)
+
+    // Setup data structures
+    unsigned int*       d_histogram[ACTIVE_CHANNELS];
+    int                 num_levels[ACTIVE_CHANNELS];            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT              lower_level[ACTIVE_CHANNELS];           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    LevelT              upper_level[ACTIVE_CHANNELS];           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+
+    for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+    {
+        d_histogram[CHANNEL] = d_hist + (CHANNEL * NUM_BINS);
+        num_levels[CHANNEL] = NUM_BINS + 1;
+        lower_level[CHANNEL] = 0;
+        upper_level[CHANNEL] = (is_float) ? 1 : 256;
+    }
+
+    // Allocate temporary storage
+    size_t temp_storage_bytes = 0;
+    void *d_temp_storage = NULL;
+
+    SampleT* d_image_samples = (SampleT*) d_image;
+
+    // Get amount of temporary storage needed
+    DeviceHistogram::MultiHistogramEven<NUM_CHANNELS, ACTIVE_CHANNELS>(
+        d_temp_storage,
+        temp_storage_bytes,
+        d_image_samples,
+        d_histogram,
+        num_levels,
+        lower_level,
+        upper_level,
+        width * height, 
+        (cudaStream_t) 0,
+        is_warmup);
+
+    cudaMalloc(&d_temp_storage, temp_storage_bytes);
+
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+
+    // Compute histogram
+    DeviceHistogram::MultiHistogramEven<NUM_CHANNELS, ACTIVE_CHANNELS>(
+        d_temp_storage,
+        temp_storage_bytes,
+        d_image_samples,
+        d_histogram,
+        num_levels,
+        lower_level,
+        upper_level,
+        width * height, 
+        (cudaStream_t) 0,
+        is_warmup);
+
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    cudaFree(d_temp_storage);
+
+    return elapsed_millis;
+}
+
diff --git a/third_party/cub/experimental/histogram/histogram_gmem_atomics.h b/third_party/cub/experimental/histogram/histogram_gmem_atomics.h
new file mode 100644
index 0000000..3308a28
--- /dev/null
+++ b/third_party/cub/experimental/histogram/histogram_gmem_atomics.h
@@ -0,0 +1,185 @@
+/******************************************************************************
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+#include <test/test_util.h>
+
+namespace histogram_gmem_atomics
+{
+    // Decode float4 pixel into bins
+    template <int NUM_BINS, int ACTIVE_CHANNELS>
+    __device__ __forceinline__ void DecodePixel(float4 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+    {
+        float* samples = reinterpret_cast<float*>(&pixel);
+
+        #pragma unroll
+        for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+            bins[CHANNEL] = (unsigned int) (samples[CHANNEL] * float(NUM_BINS));
+    }
+
+    // Decode uchar4 pixel into bins
+    template <int NUM_BINS, int ACTIVE_CHANNELS>
+    __device__ __forceinline__ void DecodePixel(uchar4 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+    {
+        unsigned char* samples = reinterpret_cast<unsigned char*>(&pixel);
+
+        #pragma unroll
+        for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+            bins[CHANNEL] = (unsigned int) (samples[CHANNEL]);
+    }
+
+    // Decode uchar1 pixel into bins
+    template <int NUM_BINS, int ACTIVE_CHANNELS>
+    __device__ __forceinline__ void DecodePixel(uchar1 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+    {
+        bins[0] = (unsigned int) pixel.x;
+    }
+
+    // First-pass histogram kernel (binning into privatized counters)
+    template <
+        int         NUM_PARTS,
+        int         ACTIVE_CHANNELS,
+        int         NUM_BINS,
+        typename    PixelType>
+    __global__ void histogram_gmem_atomics(
+        const PixelType *in,
+        int width,
+        int height,
+        unsigned int *out)
+    {
+        // global position and size
+        int x = blockIdx.x * blockDim.x + threadIdx.x;
+        int y = blockIdx.y * blockDim.y + threadIdx.y;
+        int nx = blockDim.x * gridDim.x;
+        int ny = blockDim.y * gridDim.y;
+
+        // threads in workgroup
+        int t = threadIdx.x + threadIdx.y * blockDim.x; // thread index in workgroup, linear in 0..nt-1
+        int nt = blockDim.x * blockDim.y; // total threads in workgroup
+
+        // group index in 0..ngroups-1
+        int g = blockIdx.x + blockIdx.y * gridDim.x;
+
+        // initialize smem
+        unsigned int *gmem = out + g * NUM_PARTS;
+        for (int i = t; i < ACTIVE_CHANNELS * NUM_BINS; i += nt)
+            gmem[i] = 0;
+        __syncthreads();
+
+        // process pixels (updates our group's partial histogram in gmem)
+        for (int col = x; col < width; col += nx)
+        {
+            for (int row = y; row < height; row += ny)
+            {
+                PixelType pixel = in[row * width + col];
+
+                unsigned int bins[ACTIVE_CHANNELS];
+                DecodePixel<NUM_BINS>(pixel, bins);
+
+                #pragma unroll
+                for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+                    atomicAdd(&gmem[(NUM_BINS * CHANNEL) + bins[CHANNEL]], 1);
+            }
+        }
+    }
+
+    // Second pass histogram kernel (accumulation)
+    template <
+        int         NUM_PARTS,
+        int         ACTIVE_CHANNELS,
+        int         NUM_BINS>
+    __global__ void histogram_gmem_accum(
+        const unsigned int *in,
+        int n,
+        unsigned int *out)
+    {
+        int i = blockIdx.x * blockDim.x + threadIdx.x;
+        if (i > ACTIVE_CHANNELS * NUM_BINS)
+            return; // out of range
+
+        unsigned int total = 0;
+        for (int j = 0; j < n; j++)
+            total += in[i + NUM_PARTS * j];
+
+        out[i] = total;
+    }
+
+
+}   // namespace histogram_gmem_atomics
+
+
+template <
+    int         ACTIVE_CHANNELS,
+    int         NUM_BINS,
+    typename    PixelType>
+double run_gmem_atomics(
+    PixelType *d_image,
+    int width,
+    int height,
+    unsigned int *d_hist,
+    bool warmup)
+{
+    enum
+    {
+        NUM_PARTS = 1024
+    };
+
+    cudaDeviceProp props;
+    cudaGetDeviceProperties(&props, 0);
+
+    dim3 block(32, 4);
+    dim3 grid(16, 16);
+    int total_blocks = grid.x * grid.y;
+
+    // allocate partial histogram
+    unsigned int *d_part_hist;
+    cudaMalloc(&d_part_hist, total_blocks * NUM_PARTS * sizeof(unsigned int));
+
+    dim3 block2(128);
+    dim3 grid2((3 * NUM_BINS + block.x - 1) / block.x);
+
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+
+    histogram_gmem_atomics::histogram_gmem_atomics<NUM_PARTS, ACTIVE_CHANNELS, NUM_BINS><<<grid, block>>>(
+        d_image,
+        width,
+        height,
+        d_part_hist);
+
+    histogram_gmem_atomics::histogram_gmem_accum<NUM_PARTS, ACTIVE_CHANNELS, NUM_BINS><<<grid2, block2>>>(
+        d_part_hist,
+        total_blocks,
+        d_hist);
+
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    cudaFree(d_part_hist);
+
+    return elapsed_millis;
+}
+
diff --git a/third_party/cub/experimental/histogram/histogram_smem_atomics.h b/third_party/cub/experimental/histogram/histogram_smem_atomics.h
new file mode 100644
index 0000000..2c70702
--- /dev/null
+++ b/third_party/cub/experimental/histogram/histogram_smem_atomics.h
@@ -0,0 +1,195 @@
+/******************************************************************************
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+#include <test/test_util.h>
+
+namespace histogram_smem_atomics
+{
+    // Decode float4 pixel into bins
+    template <int NUM_BINS, int ACTIVE_CHANNELS>
+    __device__ __forceinline__ void DecodePixel(float4 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+    {
+        float* samples = reinterpret_cast<float*>(&pixel);
+
+        #pragma unroll
+        for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+            bins[CHANNEL] = (unsigned int) (samples[CHANNEL] * float(NUM_BINS));
+    }
+
+    // Decode uchar4 pixel into bins
+    template <int NUM_BINS, int ACTIVE_CHANNELS>
+    __device__ __forceinline__ void DecodePixel(uchar4 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+    {
+        unsigned char* samples = reinterpret_cast<unsigned char*>(&pixel);
+
+        #pragma unroll
+        for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+            bins[CHANNEL] = (unsigned int) (samples[CHANNEL]);
+    }
+
+    // Decode uchar1 pixel into bins
+    template <int NUM_BINS, int ACTIVE_CHANNELS>
+    __device__ __forceinline__ void DecodePixel(uchar1 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+    {
+        bins[0] = (unsigned int) pixel.x;
+    }
+
+    // First-pass histogram kernel (binning into privatized counters)
+    template <
+        int         NUM_PARTS,
+        int         ACTIVE_CHANNELS,
+        int         NUM_BINS,
+        typename    PixelType>
+    __global__ void histogram_smem_atomics(
+        const PixelType *in,
+        int width,
+        int height,
+        unsigned int *out)
+    {
+        // global position and size
+        int x = blockIdx.x * blockDim.x + threadIdx.x;
+        int y = blockIdx.y * blockDim.y + threadIdx.y;
+        int nx = blockDim.x * gridDim.x;
+        int ny = blockDim.y * gridDim.y;
+
+        // threads in workgroup
+        int t = threadIdx.x + threadIdx.y * blockDim.x; // thread index in workgroup, linear in 0..nt-1
+        int nt = blockDim.x * blockDim.y; // total threads in workgroup
+
+        // group index in 0..ngroups-1
+        int g = blockIdx.x + blockIdx.y * gridDim.x;
+
+        // initialize smem
+        __shared__ unsigned int smem[ACTIVE_CHANNELS * NUM_BINS + 3];
+        for (int i = t; i < ACTIVE_CHANNELS * NUM_BINS + 3; i += nt)
+            smem[i] = 0;
+        __syncthreads();
+
+        // process pixels
+        // updates our group's partial histogram in smem
+        for (int col = x; col < width; col += nx)
+        {
+            for (int row = y; row < height; row += ny)
+            {
+                PixelType pixel = in[row * width + col];
+
+                unsigned int bins[ACTIVE_CHANNELS];
+                DecodePixel<NUM_BINS>(pixel, bins);
+
+                #pragma unroll
+                for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+                    atomicAdd(&smem[(NUM_BINS * CHANNEL) + bins[CHANNEL] + CHANNEL], 1);
+            }
+        }
+
+        __syncthreads();
+
+        // move to our workgroup's slice of output
+        out += g * NUM_PARTS;
+
+        // store local output to global
+        for (int i = t; i < NUM_BINS; i += nt)
+        {
+            #pragma unroll
+            for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+                out[i + NUM_BINS * CHANNEL] = smem[i + NUM_BINS * CHANNEL + CHANNEL];
+        }
+    }
+
+    // Second pass histogram kernel (accumulation)
+    template <
+        int         NUM_PARTS,
+        int         ACTIVE_CHANNELS,
+        int         NUM_BINS>
+    __global__ void histogram_smem_accum(
+        const unsigned int *in,
+        int n,
+        unsigned int *out)
+    {
+        int i = blockIdx.x * blockDim.x + threadIdx.x;
+        if (i > ACTIVE_CHANNELS * NUM_BINS) return; // out of range
+        unsigned int total = 0;
+        for (int j = 0; j < n; j++)
+            total += in[i + NUM_PARTS * j];
+        out[i] = total;
+    }
+
+}   // namespace histogram_smem_atomics
+
+
+template <
+    int         ACTIVE_CHANNELS,
+    int         NUM_BINS,
+    typename    PixelType>
+double run_smem_atomics(
+    PixelType *d_image,
+    int width,
+    int height,
+    unsigned int *d_hist, 
+    bool warmup)
+{
+    enum
+    {
+        NUM_PARTS = 1024
+    };
+
+    cudaDeviceProp props;
+    cudaGetDeviceProperties(&props, 0);
+
+    dim3 block(32, 4);
+    dim3 grid(16, 16);
+    int total_blocks = grid.x * grid.y;
+
+    // allocate partial histogram
+    unsigned int *d_part_hist;
+    cudaMalloc(&d_part_hist, total_blocks * NUM_PARTS * sizeof(unsigned int));
+
+    dim3 block2(128);
+    dim3 grid2((ACTIVE_CHANNELS * NUM_BINS + block.x - 1) / block.x);
+
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+
+    histogram_smem_atomics::histogram_smem_atomics<NUM_PARTS, ACTIVE_CHANNELS, NUM_BINS><<<grid, block>>>(
+        d_image,
+        width,
+        height,
+        d_part_hist);
+
+    histogram_smem_atomics::histogram_smem_accum<NUM_PARTS, ACTIVE_CHANNELS, NUM_BINS><<<grid2, block2>>>(
+        d_part_hist,
+        total_blocks,
+        d_hist);
+
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    cudaFree(d_part_hist);
+
+    return elapsed_millis;
+}
+
diff --git a/third_party/cub/experimental/histogram_compare.cu b/third_party/cub/experimental/histogram_compare.cu
new file mode 100644
index 0000000..7ab66a1
--- /dev/null
+++ b/third_party/cub/experimental/histogram_compare.cu
@@ -0,0 +1,635 @@
+/******************************************************************************
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+#include <stdio.h>
+#include <map>
+#include <vector>
+#include <algorithm>
+#include <cstdio>
+#include <fstream>
+
+#include "histogram/histogram_gmem_atomics.h"
+#include "histogram/histogram_smem_atomics.h"
+#include "histogram/histogram_cub.h"
+
+#include <cub/util_allocator.cuh>
+#include <test/test_util.h>
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants, and type declarations
+//---------------------------------------------------------------------
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+bool                    g_verbose = false;  // Whether to display input/output to console
+bool                    g_report = false;   // Whether to display a full report in CSV format
+CachingDeviceAllocator  g_allocator(true);  // Caching allocator for device memory
+
+struct less_than_value
+{
+    inline bool operator()(
+        const std::pair<std::string, double> &a,
+        const std::pair<std::string, double> &b)
+    {
+        return a.second < b.second;
+    }
+};
+
+
+//---------------------------------------------------------------------
+// Targa (.tga) image file parsing
+//---------------------------------------------------------------------
+
+/**
+ * TGA image header info
+ */
+struct TgaHeader
+{
+    char idlength;
+    char colormaptype;
+    char datatypecode;
+    short colormaporigin;
+    short colormaplength;
+    char colormapdepth;
+    short x_origin;
+    short y_origin;
+    short width;
+    short height;
+    char bitsperpixel;
+    char imagedescriptor;
+
+    void Parse (FILE *fptr)
+    {
+        idlength = fgetc(fptr);
+        colormaptype = fgetc(fptr);
+        datatypecode = fgetc(fptr);
+        fread(&colormaporigin, 2, 1, fptr);
+        fread(&colormaplength, 2, 1, fptr);
+        colormapdepth = fgetc(fptr);
+        fread(&x_origin, 2, 1, fptr);
+        fread(&y_origin, 2, 1, fptr);
+        fread(&width, 2, 1, fptr);
+        fread(&height, 2, 1, fptr);
+        bitsperpixel = fgetc(fptr);
+        imagedescriptor = fgetc(fptr);
+    }
+
+    void Display (FILE *fptr)
+    {
+        fprintf(fptr, "ID length:           %d\n", idlength);
+        fprintf(fptr, "Color map type:      %d\n", colormaptype);
+        fprintf(fptr, "Image type:          %d\n", datatypecode);
+        fprintf(fptr, "Color map offset:    %d\n", colormaporigin);
+        fprintf(fptr, "Color map length:    %d\n", colormaplength);
+        fprintf(fptr, "Color map depth:     %d\n", colormapdepth);
+        fprintf(fptr, "X origin:            %d\n", x_origin);
+        fprintf(fptr, "Y origin:            %d\n", y_origin);
+        fprintf(fptr, "Width:               %d\n", width);
+        fprintf(fptr, "Height:              %d\n", height);
+        fprintf(fptr, "Bits per pixel:      %d\n", bitsperpixel);
+        fprintf(fptr, "Descriptor:          %d\n", imagedescriptor);
+    }
+};
+
+
+/**
+ * Decode image byte data into pixel
+ */
+void ParseTgaPixel(uchar4 &pixel, unsigned char *tga_pixel, int bytes)
+{
+    if (bytes == 4)
+    {
+        pixel.x = tga_pixel[2];
+        pixel.y = tga_pixel[1];
+        pixel.z = tga_pixel[0];
+        pixel.w = tga_pixel[3];
+    }
+    else if (bytes == 3)
+    {
+        pixel.x = tga_pixel[2];
+        pixel.y = tga_pixel[1];
+        pixel.z = tga_pixel[0];
+        pixel.w = 0;
+    }
+    else if (bytes == 2)
+    {
+        pixel.x = (tga_pixel[1] & 0x7c) << 1;
+        pixel.y = ((tga_pixel[1] & 0x03) << 6) | ((tga_pixel[0] & 0xe0) >> 2);
+        pixel.z = (tga_pixel[0] & 0x1f) << 3;
+        pixel.w = (tga_pixel[1] & 0x80);
+    }
+}
+
+
+/**
+ * Reads a .tga image file
+ */
+void ReadTga(uchar4* &pixels, int &width, int &height, const char *filename)
+{
+    // Open the file
+    FILE *fptr;
+    if ((fptr = fopen(filename, "rb")) == NULL)
+    {
+        fprintf(stderr, "File open failed\n");
+        exit(-1);
+    }
+
+    // Parse header
+    TgaHeader header;
+    header.Parse(fptr);
+//    header.Display(stdout);
+    width = header.width;
+    height = header.height;
+
+    // Verify compatibility
+    if (header.datatypecode != 2 && header.datatypecode != 10)
+    {
+        fprintf(stderr, "Can only handle image type 2 and 10\n");
+        exit(-1);
+    }
+    if (header.bitsperpixel != 16 && header.bitsperpixel != 24 && header.bitsperpixel != 32)
+    {
+        fprintf(stderr, "Can only handle pixel depths of 16, 24, and 32\n");
+        exit(-1);
+    }
+    if (header.colormaptype != 0 && header.colormaptype != 1)
+    {
+        fprintf(stderr, "Can only handle color map types of 0 and 1\n");
+        exit(-1);
+    }
+
+    // Skip unnecessary header info
+    int skip_bytes = header.idlength + (header.colormaptype * header.colormaplength);
+    fseek(fptr, skip_bytes, SEEK_CUR);
+
+    // Read the image
+    int pixel_bytes = header.bitsperpixel / 8;
+
+    // Allocate and initialize pixel data
+    size_t image_bytes = width * height * sizeof(uchar4);
+    if ((pixels == NULL) && ((pixels = (uchar4*) malloc(image_bytes)) == NULL))
+    {
+        fprintf(stderr, "malloc of image failed\n");
+        exit(-1);
+    }
+    memset(pixels, 0, image_bytes);
+
+    // Parse pixels
+    unsigned char   tga_pixel[5];
+    int             current_pixel = 0;
+    while (current_pixel < header.width * header.height)
+    {
+        if (header.datatypecode == 2)
+        {
+            // Uncompressed
+            if (fread(tga_pixel, 1, pixel_bytes, fptr) != pixel_bytes)
+            {
+                fprintf(stderr, "Unexpected end of file at pixel %d  (uncompressed)\n", current_pixel);
+                exit(-1);
+            }
+            ParseTgaPixel(pixels[current_pixel], tga_pixel, pixel_bytes);
+            current_pixel++;
+        }
+        else if (header.datatypecode == 10)
+        {
+            // Compressed
+            if (fread(tga_pixel, 1, pixel_bytes + 1, fptr) != pixel_bytes + 1)
+            {
+                fprintf(stderr, "Unexpected end of file at pixel %d (compressed)\n", current_pixel);
+                exit(-1);
+            }
+            int run_length = tga_pixel[0] & 0x7f;
+            ParseTgaPixel(pixels[current_pixel], &(tga_pixel[1]), pixel_bytes);
+            current_pixel++;
+
+            if (tga_pixel[0] & 0x80)
+            {
+                // RLE chunk
+                for (int i = 0; i < run_length; i++)
+                {
+                    ParseTgaPixel(pixels[current_pixel], &(tga_pixel[1]), pixel_bytes);
+                    current_pixel++;
+                }
+            }
+            else
+            {
+                // Normal chunk
+                for (int i = 0; i < run_length; i++)
+                {
+                    if (fread(tga_pixel, 1, pixel_bytes, fptr) != pixel_bytes)
+                    {
+                        fprintf(stderr, "Unexpected end of file at pixel %d (normal)\n", current_pixel);
+                        exit(-1);
+                    }
+                    ParseTgaPixel(pixels[current_pixel], tga_pixel, pixel_bytes);
+                    current_pixel++;
+                }
+            }
+        }
+    }
+
+    // Close file
+    fclose(fptr);
+}
+
+
+
+//---------------------------------------------------------------------
+// Random image generation
+//---------------------------------------------------------------------
+
+/**
+ * Generate a random image with specified entropy
+ */
+void GenerateRandomImage(uchar4* &pixels, int width, int height, int entropy_reduction)
+{
+    int num_pixels = width * height;
+    size_t image_bytes = num_pixels * sizeof(uchar4);
+    if ((pixels == NULL) && ((pixels = (uchar4*) malloc(image_bytes)) == NULL))
+    {
+        fprintf(stderr, "malloc of image failed\n");
+        exit(-1);
+    }
+
+    for (int i = 0; i < num_pixels; ++i)
+    {
+        RandomBits(pixels[i].x, entropy_reduction);
+        RandomBits(pixels[i].y, entropy_reduction);
+        RandomBits(pixels[i].z, entropy_reduction);
+        RandomBits(pixels[i].w, entropy_reduction);
+    }
+}
+
+
+
+//---------------------------------------------------------------------
+// Histogram verification
+//---------------------------------------------------------------------
+
+// Decode float4 pixel into bins
+template <int NUM_BINS, int ACTIVE_CHANNELS>
+void DecodePixelGold(float4 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+{
+    float* samples = reinterpret_cast<float*>(&pixel);
+
+    for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+        bins[CHANNEL] = (unsigned int) (samples[CHANNEL] * float(NUM_BINS));
+}
+
+// Decode uchar4 pixel into bins
+template <int NUM_BINS, int ACTIVE_CHANNELS>
+void DecodePixelGold(uchar4 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+{
+    unsigned char* samples = reinterpret_cast<unsigned char*>(&pixel);
+
+    for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+        bins[CHANNEL] = (unsigned int) (samples[CHANNEL]);
+}
+
+// Decode uchar1 pixel into bins
+template <int NUM_BINS, int ACTIVE_CHANNELS>
+void DecodePixelGold(uchar1 pixel, unsigned int (&bins)[ACTIVE_CHANNELS])
+{
+    bins[0] = (unsigned int) pixel.x;
+}
+
+
+// Compute reference histogram.  Specialized for uchar4
+template <
+    int         ACTIVE_CHANNELS,
+    int         NUM_BINS,
+    typename    PixelType>
+void HistogramGold(PixelType *image, int width, int height, unsigned int* hist)
+{
+    memset(hist, 0, ACTIVE_CHANNELS * NUM_BINS * sizeof(unsigned int));
+
+    for (int i = 0; i < width; i++)
+    {
+        for (int j = 0; j < height; j++)
+        {
+            PixelType pixel = image[i + j * width];
+
+            unsigned int bins[ACTIVE_CHANNELS];
+            DecodePixelGold<NUM_BINS>(pixel, bins);
+
+            for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL)
+            {
+                hist[(NUM_BINS * CHANNEL) + bins[CHANNEL]]++;
+            }
+        }
+    }
+}
+
+
+//---------------------------------------------------------------------
+// Test execution
+//---------------------------------------------------------------------
+
+/**
+ * Run a specific histogram implementation
+ */
+template <
+    int         ACTIVE_CHANNELS,
+    int         NUM_BINS,
+    typename    PixelType>
+void RunTest(
+    std::vector<std::pair<std::string, double> >&   timings,
+    PixelType*                                      d_pixels,
+    const int                                       width,
+    const int                                       height,
+    unsigned int *                                  d_hist,
+    unsigned int *                                  h_hist,
+    int                                             timing_iterations,
+    const char *                                    long_name,
+    const char *                                    short_name,
+    double (*f)(PixelType*, int, int, unsigned int*, bool))
+{
+    if (!g_report) printf("%s ", long_name); fflush(stdout);
+
+    // Run single test to verify (and code cache)
+    (*f)(d_pixels, width, height, d_hist, !g_report);
+
+    int compare = CompareDeviceResults(h_hist, d_hist, ACTIVE_CHANNELS * NUM_BINS, true, g_verbose);
+    if (!g_report) printf("\t%s\n", compare ? "FAIL" : "PASS"); fflush(stdout);
+
+    double elapsed_ms = 0;
+    for (int i = 0; i < timing_iterations; i++)
+    {
+        elapsed_ms += (*f)(d_pixels, width, height, d_hist, false);
+    }
+    double avg_us = (elapsed_ms / timing_iterations) * 1000;    // average in us
+    timings.push_back(std::pair<std::string, double>(short_name, avg_us));
+
+    if (!g_report)
+    {
+        printf("Avg time %.3f us (%d iterations)\n", avg_us, timing_iterations); fflush(stdout);
+    }
+    else
+    {
+        printf("%.3f, ", avg_us); fflush(stdout);
+    }
+
+    AssertEquals(0, compare);
+}
+
+
+/**
+ * Evaluate corpus of histogram implementations
+ */
+template <
+    int         NUM_CHANNELS,
+    int         ACTIVE_CHANNELS,
+    int         NUM_BINS,
+    typename    PixelType>
+void TestMethods(
+    PixelType*  h_pixels,
+    int         height,
+    int         width,
+    int         timing_iterations,
+    double      bandwidth_GBs)
+{
+    // Copy data to gpu
+    PixelType* d_pixels;
+    size_t pixel_bytes = width * height * sizeof(PixelType);
+    CubDebugExit(g_allocator.DeviceAllocate((void**) &d_pixels, pixel_bytes));
+    CubDebugExit(cudaMemcpy(d_pixels, h_pixels, pixel_bytes, cudaMemcpyHostToDevice));
+
+    if (g_report) printf("%.3f, ", double(pixel_bytes) / bandwidth_GBs / 1000);
+
+    // Allocate results arrays on cpu/gpu
+    unsigned int *h_hist;
+    unsigned int *d_hist;
+    size_t histogram_bytes = NUM_BINS * ACTIVE_CHANNELS * sizeof(unsigned int);
+    h_hist = (unsigned int *) malloc(histogram_bytes);
+    g_allocator.DeviceAllocate((void **) &d_hist, histogram_bytes);
+
+    // Compute reference cpu histogram
+    HistogramGold<ACTIVE_CHANNELS, NUM_BINS>(h_pixels, width, height, h_hist);
+
+    // Store timings
+    std::vector<std::pair<std::string, double> > timings;
+
+    // Run experiments
+    RunTest<ACTIVE_CHANNELS, NUM_BINS>(timings, d_pixels, width, height, d_hist, h_hist, timing_iterations,
+        "CUB", "CUB", run_cub_histogram<NUM_CHANNELS, ACTIVE_CHANNELS, NUM_BINS, PixelType>);
+    RunTest<ACTIVE_CHANNELS, NUM_BINS>(timings, d_pixels, width, height, d_hist, h_hist, timing_iterations,
+        "Shared memory atomics", "smem atomics", run_smem_atomics<ACTIVE_CHANNELS, NUM_BINS, PixelType>);
+    RunTest<ACTIVE_CHANNELS, NUM_BINS>(timings, d_pixels, width, height, d_hist, h_hist, timing_iterations,
+        "Global memory atomics", "gmem atomics", run_gmem_atomics<ACTIVE_CHANNELS, NUM_BINS, PixelType>);
+
+    // Report timings
+    if (!g_report)
+    {
+        std::sort(timings.begin(), timings.end(), less_than_value());
+        printf("Timings (us):\n");
+        for (int i = 0; i < timings.size(); i++)
+        {
+            double bandwidth = height * width * sizeof(PixelType) / timings[i].second / 1000;
+            printf("\t %.3f %s (%.3f GB/s, %.3f%% peak)\n", timings[i].second, timings[i].first.c_str(), bandwidth, bandwidth / bandwidth_GBs * 100);
+        }
+        printf("\n");
+    }
+
+    // Free data
+    CubDebugExit(g_allocator.DeviceFree(d_pixels));
+    CubDebugExit(g_allocator.DeviceFree(d_hist));
+    free(h_hist);
+}
+
+
+/**
+ * Test different problem genres
+ */
+void TestGenres(
+    uchar4*     uchar4_pixels,
+    int         height,
+    int         width,
+    int         timing_iterations,
+    double      bandwidth_GBs)
+{
+    int num_pixels = width * height;
+
+    {
+        if (!g_report) printf("1 channel uchar1 tests (256-bin):\n\n"); fflush(stdout);
+
+        size_t      image_bytes     = num_pixels * sizeof(uchar1);
+        uchar1*     uchar1_pixels   = (uchar1*) malloc(image_bytes);
+
+        // Convert to 1-channel (averaging first 3 channels)
+        for (int i = 0; i < num_pixels; ++i)
+        {
+            uchar1_pixels[i].x = (unsigned char)
+                (((unsigned int) uchar4_pixels[i].x +
+                  (unsigned int) uchar4_pixels[i].y +
+                  (unsigned int) uchar4_pixels[i].z) / 3);
+        }
+
+        TestMethods<1, 1, 256>(uchar1_pixels, width, height, timing_iterations, bandwidth_GBs);
+        free(uchar1_pixels);
+        if (g_report) printf(", ");
+    }
+
+    {
+        if (!g_report) printf("3/4 channel uchar4 tests (256-bin):\n\n"); fflush(stdout);
+        TestMethods<4, 3, 256>(uchar4_pixels, width, height, timing_iterations, bandwidth_GBs);
+        if (g_report) printf(", ");
+    }
+
+    {
+        if (!g_report) printf("3/4 channel float4 tests (256-bin):\n\n"); fflush(stdout);
+        size_t      image_bytes     = num_pixels * sizeof(float4);
+        float4*     float4_pixels   = (float4*) malloc(image_bytes);
+
+        // Convert to float4 with range [0.0, 1.0)
+        for (int i = 0; i < num_pixels; ++i)
+        {
+            float4_pixels[i].x = float(uchar4_pixels[i].x) / 256;
+            float4_pixels[i].y = float(uchar4_pixels[i].y) / 256;
+            float4_pixels[i].z = float(uchar4_pixels[i].z) / 256;
+            float4_pixels[i].w = float(uchar4_pixels[i].w) / 256;
+        }
+        TestMethods<4, 3, 256>(float4_pixels, width, height, timing_iterations, bandwidth_GBs);
+        free(float4_pixels);
+        if (g_report) printf("\n");
+    }
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char **argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf(
+            "%s "
+            "[--device=<device-id>] "
+            "[--v] "
+            "[--i=<timing iterations>] "
+            "\n\t"
+                "--file=<.tga filename> "
+            "\n\t"
+                "--entropy=<-1 (0%), 0 (100%), 1 (81%), 2 (54%), 3 (34%), 4 (20%), ..."
+                "[--height=<default: 1080>] "
+                "[--width=<default: 1920>] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    std::string         filename;
+    int                 timing_iterations   = 100;
+    int                 entropy_reduction   = 0;
+    int                 height              = 1080;
+    int                 width               = 1920;
+
+    g_verbose = args.CheckCmdLineFlag("v");
+    g_report = args.CheckCmdLineFlag("report");
+    args.GetCmdLineArgument("i", timing_iterations);
+    args.GetCmdLineArgument("file", filename);
+    args.GetCmdLineArgument("height", height);
+    args.GetCmdLineArgument("width", width);
+    args.GetCmdLineArgument("entropy", entropy_reduction);
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get GPU device bandwidth (GB/s)
+    int device_ordinal, bus_width, mem_clock_khz;
+    CubDebugExit(cudaGetDevice(&device_ordinal));
+    CubDebugExit(cudaDeviceGetAttribute(&bus_width, cudaDevAttrGlobalMemoryBusWidth, device_ordinal));
+    CubDebugExit(cudaDeviceGetAttribute(&mem_clock_khz, cudaDevAttrMemoryClockRate, device_ordinal));
+    double bandwidth_GBs = double(bus_width) * mem_clock_khz * 2 / 8 / 1000 / 1000;
+
+    // Run test(s)
+    uchar4* uchar4_pixels = NULL;
+    if (!g_report)
+    {
+        if (!filename.empty())
+        {
+            // Parse targa file
+            ReadTga(uchar4_pixels, width, height, filename.c_str());
+            printf("File %s: width(%d) height(%d)\n\n", filename.c_str(), width, height); fflush(stdout);
+        }
+        else
+        {
+            // Generate image
+            GenerateRandomImage(uchar4_pixels, width, height, entropy_reduction);
+            printf("Random image: entropy-reduction(%d) width(%d) height(%d)\n\n", entropy_reduction, width, height); fflush(stdout);
+        }
+
+        TestGenres(uchar4_pixels, height, width, timing_iterations, bandwidth_GBs);
+    }
+    else
+    {
+        // Run test suite
+        printf("Test, MIN, RLE CUB, SMEM, GMEM, , MIN, RLE_CUB, SMEM, GMEM, , MIN, RLE_CUB, SMEM, GMEM\n");
+
+        // Entropy reduction tests
+        for (entropy_reduction = 0; entropy_reduction < 5; ++entropy_reduction)
+        {
+            printf("entropy reduction %d, ", entropy_reduction);
+            GenerateRandomImage(uchar4_pixels, width, height, entropy_reduction);
+            TestGenres(uchar4_pixels, height, width, timing_iterations, bandwidth_GBs);
+        }
+        printf("entropy reduction -1, ");
+        GenerateRandomImage(uchar4_pixels, width, height, -1);
+        TestGenres(uchar4_pixels, height, width, timing_iterations, bandwidth_GBs);
+        printf("\n");
+
+        // File image tests
+        std::vector<std::string> file_tests;
+        file_tests.push_back("animals");
+        file_tests.push_back("apples");
+        file_tests.push_back("sunset");
+        file_tests.push_back("cheetah");
+        file_tests.push_back("nature");
+        file_tests.push_back("operahouse");
+        file_tests.push_back("austin");
+        file_tests.push_back("cityscape");
+
+        for (int i = 0; i < file_tests.size(); ++i)
+        {
+            printf("%s, ", file_tests[i].c_str());
+            std::string filename = std::string("histogram/benchmark/") + file_tests[i] + ".tga";
+            ReadTga(uchar4_pixels, width, height, filename.c_str());
+            TestGenres(uchar4_pixels, height, width, timing_iterations, bandwidth_GBs);
+        }
+    }
+
+    free(uchar4_pixels);
+
+    CubDebugExit(cudaDeviceSynchronize());
+    printf("\n\n");
+
+    return 0;
+}
diff --git a/third_party/cub/experimental/sparse_matrix.h b/third_party/cub/experimental/sparse_matrix.h
new file mode 100644
index 0000000..1fb5233
--- /dev/null
+++ b/third_party/cub/experimental/sparse_matrix.h
@@ -0,0 +1,1244 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Matrix data structures and parsing logic
+ ******************************************************************************/
+
+#pragma once
+
+#include <cmath>
+#include <cstring>
+
+#include <iterator>
+#include <string>
+#include <algorithm>
+#include <iostream>
+#include <queue>
+#include <set>
+#include <fstream>
+#include <stdio.h>
+
+#ifdef CUB_MKL
+    #include <numa.h>
+    #include <mkl.h>
+#endif
+
+using namespace std;
+
+/******************************************************************************
+ * COO matrix type
+ ******************************************************************************/
+
+struct GraphStats
+{
+    int         num_rows;
+    int         num_cols;
+    int         num_nonzeros;
+
+    double      diag_dist_mean;         // mean
+    double      diag_dist_std_dev;      // sample std dev
+    double      pearson_r;    // coefficient of variation
+
+    double      row_length_mean;        // mean
+    double      row_length_std_dev;     // sample std_dev
+    double      row_length_variation;   // coefficient of variation
+    double      row_length_skewness;    // skewness
+
+    void Display(bool show_labels = true)
+    {
+        if (show_labels)
+            printf("\n"
+                "\t num_rows: %d\n"
+                "\t num_cols: %d\n"
+                "\t num_nonzeros: %d\n"
+                "\t diag_dist_mean: %.2f\n"
+                "\t diag_dist_std_dev: %.2f\n"
+                "\t pearson_r: %f\n"
+                "\t row_length_mean: %.5f\n"
+                "\t row_length_std_dev: %.5f\n"
+                "\t row_length_variation: %.5f\n"
+                "\t row_length_skewness: %.5f\n",
+                    num_rows,
+                    num_cols,
+                    num_nonzeros,
+                    diag_dist_mean,
+                    diag_dist_std_dev,
+                    pearson_r,
+                    row_length_mean,
+                    row_length_std_dev,
+                    row_length_variation,
+                    row_length_skewness);
+        else
+            printf(
+                "%d, "
+                "%d, "
+                "%d, "
+                "%.2f, "
+                "%.2f, "
+                "%f, "
+                "%.5f, "
+                "%.5f, "
+                "%.5f, "
+                "%.5f, ",
+                    num_rows,
+                    num_cols,
+                    num_nonzeros,
+                    diag_dist_mean,
+                    diag_dist_std_dev,
+                    pearson_r,
+                    row_length_mean,
+                    row_length_std_dev,
+                    row_length_variation,
+                    row_length_skewness);
+    }
+};
+
+
+
+/******************************************************************************
+ * COO matrix type
+ ******************************************************************************/
+
+
+/**
+ * COO matrix type.  A COO matrix is just a vector of edge tuples.  Tuples are sorted
+ * first by row, then by column.
+ */
+template<typename ValueT, typename OffsetT>
+struct CooMatrix
+{
+    //---------------------------------------------------------------------
+    // Type definitions and constants
+    //---------------------------------------------------------------------
+
+    // COO edge tuple
+    struct CooTuple
+    {
+        OffsetT            row;
+        OffsetT            col;
+        ValueT             val;
+
+        CooTuple() {}
+        CooTuple(OffsetT row, OffsetT col) : row(row), col(col) {}
+        CooTuple(OffsetT row, OffsetT col, ValueT val) : row(row), col(col), val(val) {}
+
+        /**
+         * Comparator for sorting COO sparse format num_nonzeros
+         */
+        bool operator<(const CooTuple &other) const
+        {
+            if ((row < other.row) || ((row == other.row) && (col < other.col)))
+            {
+                return true;
+            }
+
+            return false;
+        }
+    };
+
+
+    //---------------------------------------------------------------------
+    // Data members
+    //---------------------------------------------------------------------
+
+    // Fields
+    int                 num_rows;
+    int                 num_cols;
+    int                 num_nonzeros;
+    CooTuple*           coo_tuples;
+
+    //---------------------------------------------------------------------
+    // Methods
+    //---------------------------------------------------------------------
+
+    // Constructor
+    CooMatrix() : num_rows(0), num_cols(0), num_nonzeros(0), coo_tuples(NULL) {}
+
+
+    /**
+     * Clear
+     */
+    void Clear()
+    {
+        if (coo_tuples) delete[] coo_tuples;
+        coo_tuples = NULL;
+    }
+
+
+    // Destructor
+    ~CooMatrix()
+    {
+        Clear();
+    }
+
+
+    // Display matrix to stdout
+    void Display()
+    {
+        cout << "COO Matrix (" << num_rows << " rows, " << num_cols << " columns, " << num_nonzeros << " non-zeros):\n";
+        cout << "Ordinal, Row, Column, Value\n";
+        for (int i = 0; i < num_nonzeros; i++)
+        {
+            cout << '\t' << i << ',' << coo_tuples[i].row << ',' << coo_tuples[i].col << ',' << coo_tuples[i].val << "\n";
+        }
+    }
+
+
+    /**
+     * Builds a symmetric COO sparse from an asymmetric CSR matrix.
+     */
+    template <typename CsrMatrixT>
+    void InitCsrSymmetric(CsrMatrixT &csr_matrix)
+    {
+        if (coo_tuples)
+        {
+            fprintf(stderr, "Matrix already constructed\n");
+            exit(1);
+        }
+
+        num_rows        = csr_matrix.num_cols;
+        num_cols        = csr_matrix.num_rows;
+        num_nonzeros    = csr_matrix.num_nonzeros * 2;
+        coo_tuples      = new CooTuple[num_nonzeros];
+
+        for (OffsetT row = 0; row < csr_matrix.num_rows; ++row)
+        {
+            for (OffsetT nonzero = csr_matrix.row_offsets[row]; nonzero < csr_matrix.row_offsets[row + 1]; ++nonzero)
+            {
+                coo_tuples[nonzero].row = row;
+                coo_tuples[nonzero].col = csr_matrix.column_indices[nonzero];
+                coo_tuples[nonzero].val = csr_matrix.values[nonzero];
+
+                coo_tuples[csr_matrix.num_nonzeros + nonzero].row = coo_tuples[nonzero].col;
+                coo_tuples[csr_matrix.num_nonzeros + nonzero].col = coo_tuples[nonzero].row;
+                coo_tuples[csr_matrix.num_nonzeros + nonzero].val = csr_matrix.values[nonzero];
+
+            }
+        }
+
+        // Sort by rows, then columns
+        std::stable_sort(coo_tuples, coo_tuples + num_nonzeros);
+    }
+
+    /**
+     * Builds a COO sparse from a relabeled CSR matrix.
+     */
+    template <typename CsrMatrixT>
+    void InitCsrRelabel(CsrMatrixT &csr_matrix, OffsetT* relabel_indices)
+    {
+        if (coo_tuples)
+        {
+            fprintf(stderr, "Matrix already constructed\n");
+            exit(1);
+        }
+
+        num_rows        = csr_matrix.num_rows;
+        num_cols        = csr_matrix.num_cols;
+        num_nonzeros    = csr_matrix.num_nonzeros;
+        coo_tuples      = new CooTuple[num_nonzeros];
+
+        for (OffsetT row = 0; row < num_rows; ++row)
+        {
+            for (OffsetT nonzero = csr_matrix.row_offsets[row]; nonzero < csr_matrix.row_offsets[row + 1]; ++nonzero)
+            {
+                coo_tuples[nonzero].row = relabel_indices[row];
+                coo_tuples[nonzero].col = relabel_indices[csr_matrix.column_indices[nonzero]];
+                coo_tuples[nonzero].val = csr_matrix.values[nonzero];
+            }
+        }
+
+        // Sort by rows, then columns
+        std::stable_sort(coo_tuples, coo_tuples + num_nonzeros);
+    }
+
+
+
+    /**
+     * Builds a METIS COO sparse from the given file.
+     */
+    void InitMetis(const string &metis_filename)
+    {
+        if (coo_tuples)
+        {
+            fprintf(stderr, "Matrix already constructed\n");
+            exit(1);
+        }
+
+        // TODO
+    }
+
+
+    /**
+     * Builds a MARKET COO sparse from the given file.
+     */
+    void InitMarket(
+        const string&   market_filename,
+        ValueT          default_value       = 1.0,
+        bool            verbose             = false)
+    {
+        if (verbose) {
+            printf("Reading... "); fflush(stdout);
+        }
+
+        if (coo_tuples)
+        {
+            fprintf(stderr, "Matrix already constructed\n");
+            exit(1);
+        }
+
+        std::ifstream ifs;
+        ifs.open(market_filename.c_str(), std::ifstream::in);
+        if (!ifs.good())
+        {
+            fprintf(stderr, "Error opening file\n");
+            exit(1);
+        }
+
+        bool    array = false;
+        bool    symmetric = false;
+        bool    skew = false;
+        int     current_edge = -1;
+        char    line[1024];
+
+        if (verbose) {
+            printf("Parsing... "); fflush(stdout);
+        }
+
+        while (true)
+        {
+            ifs.getline(line, 1024);
+            if (!ifs.good())
+            {
+                // Done
+                break;
+            }
+
+            if (line[0] == '%')
+            {
+                // Comment
+                if (line[1] == '%')
+                {
+                    // Banner
+                    symmetric   = (strstr(line, "symmetric") != NULL);
+                    skew        = (strstr(line, "skew") != NULL);
+                    array       = (strstr(line, "array") != NULL);
+
+                    if (verbose) {
+                        printf("(symmetric: %d, skew: %d, array: %d) ", symmetric, skew, array); fflush(stdout);
+                    }
+                }
+            }
+            else if (current_edge == -1)
+            {
+                // Problem description
+                int nparsed = sscanf(line, "%d %d %d", &num_rows, &num_cols, &num_nonzeros);
+                if ((!array) && (nparsed == 3))
+                {
+                    if (symmetric)
+                        num_nonzeros *= 2;
+
+                    // Allocate coo matrix
+                    coo_tuples = new CooTuple[num_nonzeros];
+                    current_edge = 0;
+
+                }
+                else if (array && (nparsed == 2))
+                {
+                    // Allocate coo matrix
+                    num_nonzeros = num_rows * num_cols;
+                    coo_tuples = new CooTuple[num_nonzeros];
+                    current_edge = 0;
+                }
+                else
+                {
+                    fprintf(stderr, "Error parsing MARKET matrix: invalid problem description: %s\n", line);
+                    exit(1);
+                }
+
+            }
+            else
+            {
+                // Edge
+                if (current_edge >= num_nonzeros)
+                {
+                    fprintf(stderr, "Error parsing MARKET matrix: encountered more than %d num_nonzeros\n", num_nonzeros);
+                    exit(1);
+                }
+
+                int row, col;
+                double val;
+
+                if (array)
+                {
+                    if (sscanf(line, "%lf", &val) != 1)
+                    {
+                        fprintf(stderr, "Error parsing MARKET matrix: badly formed current_edge: '%s' at edge %d\n", line, current_edge);
+                        exit(1);
+                    }
+                    col = (current_edge / num_rows);
+                    row = (current_edge - (num_rows * col));
+
+                    coo_tuples[current_edge] = CooTuple(row, col, val);    // Convert indices to zero-based
+                }
+                else
+                {
+                    // Parse nonzero (note: using strtol and strtod is 2x faster than sscanf or istream parsing)
+                    char *l = line;
+                    char *t = NULL;
+
+                    // parse row
+                    row = strtol(l, &t, 0);
+                    if (t == l)
+                    {
+                        fprintf(stderr, "Error parsing MARKET matrix: badly formed row at edge %d\n", current_edge);
+                        exit(1);
+                    }
+                    l = t;
+
+                    // parse col
+                    col = strtol(l, &t, 0);
+                    if (t == l)
+                    {
+                        fprintf(stderr, "Error parsing MARKET matrix: badly formed col at edge %d\n", current_edge);
+                        exit(1);
+                    }
+                    l = t;
+
+                    // parse val
+                    val = strtod(l, &t);
+                    if (t == l)
+                    {
+                        val = default_value;
+                    }
+/*
+                    int nparsed = sscanf(line, "%d %d %lf", &row, &col, &val);
+                    if (nparsed == 2)
+                    {
+                        // No value specified
+                        val = default_value;
+                        
+                    }
+                    else if (nparsed != 3)
+                    {
+                        fprintf(stderr, "Error parsing MARKET matrix 1: badly formed current_edge: %d parsed at edge %d\n", nparsed, current_edge);
+                        exit(1);
+                    }
+*/
+
+                    coo_tuples[current_edge] = CooTuple(row - 1, col - 1, val);    // Convert indices to zero-based
+
+                }
+
+                current_edge++;
+
+                if (symmetric && (row != col))
+                {
+                    coo_tuples[current_edge].row = coo_tuples[current_edge - 1].col;
+                    coo_tuples[current_edge].col = coo_tuples[current_edge - 1].row;
+                    coo_tuples[current_edge].val = coo_tuples[current_edge - 1].val * (skew ? -1 : 1);
+                    current_edge++;
+                }
+            }
+        }
+
+        // Adjust nonzero count (nonzeros along the diagonal aren't reversed)
+        num_nonzeros = current_edge;
+
+        if (verbose) {
+            printf("done. Ordering..."); fflush(stdout);
+        }
+
+        // Sort by rows, then columns
+        std::stable_sort(coo_tuples, coo_tuples + num_nonzeros);
+
+        if (verbose) {
+            printf("done. "); fflush(stdout);
+        }
+
+        ifs.close();
+    }
+
+
+    /**
+     * Builds a dense matrix
+     */
+    int InitDense(
+        OffsetT     num_rows,
+        OffsetT     num_cols,
+        ValueT      default_value   = 1.0,
+        bool        verbose         = false)
+    {
+        if (coo_tuples)
+        {
+            fprintf(stderr, "Matrix already constructed\n");
+            exit(1);
+        }
+
+        this->num_rows  = num_rows;
+        this->num_cols  = num_cols;
+
+        num_nonzeros    = num_rows * num_cols;
+        coo_tuples      = new CooTuple[num_nonzeros];
+
+        for (OffsetT row = 0; row < num_rows; ++row)
+        {
+            for (OffsetT col = 0; col < num_cols; ++col)
+            {
+                coo_tuples[(row * num_cols) + col] = CooTuple(row, col, default_value);
+            }
+        }
+
+        // Sort by rows, then columns
+        std::stable_sort(coo_tuples, coo_tuples + num_nonzeros);
+
+        return 0;
+    }
+
+    /**
+     * Builds a wheel COO sparse matrix having spokes spokes.
+     */
+    int InitWheel(
+        OffsetT     spokes,
+        ValueT      default_value   = 1.0,
+        bool        verbose         = false)
+    {
+        if (coo_tuples)
+        {
+            fprintf(stderr, "Matrix already constructed\n");
+            exit(1);
+        }
+
+        num_rows        = spokes + 1;
+        num_cols        = num_rows;
+        num_nonzeros    = spokes * 2;
+        coo_tuples      = new CooTuple[num_nonzeros];
+
+        // Add spoke num_nonzeros
+        int current_edge = 0;
+        for (OffsetT i = 0; i < spokes; i++)
+        {
+            coo_tuples[current_edge] = CooTuple(0, i + 1, default_value);
+            current_edge++;
+        }
+
+        // Add rim
+        for (OffsetT i = 0; i < spokes; i++)
+        {
+            OffsetT dest = (i + 1) % spokes;
+            coo_tuples[current_edge] = CooTuple(i + 1, dest + 1, default_value);
+            current_edge++;
+        }
+
+        // Sort by rows, then columns
+        std::stable_sort(coo_tuples, coo_tuples + num_nonzeros);
+
+        return 0;
+    }
+
+
+    /**
+     * Builds a square 2D grid CSR matrix.  Interior num_vertices have degree 5 when including
+     * a self-loop.
+     *
+     * Returns 0 on success, 1 on failure.
+     */
+    int InitGrid2d(OffsetT width, bool self_loop, ValueT default_value = 1.0)
+    {
+        if (coo_tuples)
+        {
+            fprintf(stderr, "Matrix already constructed\n");
+            exit(1);
+        }
+
+        int     interior_nodes  = (width - 2) * (width - 2);
+        int     edge_nodes      = (width - 2) * 4;
+        int     corner_nodes    = 4;
+        num_rows                       = width * width;
+        num_cols                       = num_rows;
+        num_nonzeros                   = (interior_nodes * 4) + (edge_nodes * 3) + (corner_nodes * 2);
+
+        if (self_loop)
+            num_nonzeros += num_rows;
+
+        coo_tuples          = new CooTuple[num_nonzeros];
+        int current_edge    = 0;
+
+        for (OffsetT j = 0; j < width; j++)
+        {
+            for (OffsetT k = 0; k < width; k++)
+            {
+                OffsetT me = (j * width) + k;
+
+                // West
+                OffsetT neighbor = (j * width) + (k - 1);
+                if (k - 1 >= 0) {
+                    coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                    current_edge++;
+                }
+
+                // East
+                neighbor = (j * width) + (k + 1);
+                if (k + 1 < width) {
+                    coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                    current_edge++;
+                }
+
+                // North
+                neighbor = ((j - 1) * width) + k;
+                if (j - 1 >= 0) {
+                    coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                    current_edge++;
+                }
+
+                // South
+                neighbor = ((j + 1) * width) + k;
+                if (j + 1 < width) {
+                    coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                    current_edge++;
+                }
+
+                if (self_loop)
+                {
+                    neighbor = me;
+                    coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                    current_edge++;
+                }
+            }
+        }
+
+        // Sort by rows, then columns, update dims
+        std::stable_sort(coo_tuples, coo_tuples + num_nonzeros);
+
+        return 0;
+    }
+
+
+    /**
+     * Builds a square 3D grid COO sparse matrix.  Interior num_vertices have degree 7 when including
+     * a self-loop.  Values are unintialized, coo_tuples are sorted.
+     */
+    int InitGrid3d(OffsetT width, bool self_loop, ValueT default_value = 1.0)
+    {
+        if (coo_tuples)
+        {
+            fprintf(stderr, "Matrix already constructed\n");
+            return -1;
+        }
+
+        OffsetT interior_nodes  = (width - 2) * (width - 2) * (width - 2);
+        OffsetT face_nodes      = (width - 2) * (width - 2) * 6;
+        OffsetT edge_nodes      = (width - 2) * 12;
+        OffsetT corner_nodes    = 8;
+        num_cols                       = width * width * width;
+        num_rows                       = num_cols;
+        num_nonzeros                     = (interior_nodes * 6) + (face_nodes * 5) + (edge_nodes * 4) + (corner_nodes * 3);
+
+        if (self_loop)
+            num_nonzeros += num_rows;
+
+        coo_tuples          = new CooTuple[num_nonzeros];
+        int current_edge    = 0;
+
+        for (OffsetT i = 0; i < width; i++)
+        {
+            for (OffsetT j = 0; j < width; j++)
+            {
+                for (OffsetT k = 0; k < width; k++)
+                {
+
+                    OffsetT me = (i * width * width) + (j * width) + k;
+
+                    // Up
+                    OffsetT neighbor = (i * width * width) + (j * width) + (k - 1);
+                    if (k - 1 >= 0) {
+                        coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                        current_edge++;
+                    }
+
+                    // Down
+                    neighbor = (i * width * width) + (j * width) + (k + 1);
+                    if (k + 1 < width) {
+                        coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                        current_edge++;
+                    }
+
+                    // West
+                    neighbor = (i * width * width) + ((j - 1) * width) + k;
+                    if (j - 1 >= 0) {
+                        coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                        current_edge++;
+                    }
+
+                    // East
+                    neighbor = (i * width * width) + ((j + 1) * width) + k;
+                    if (j + 1 < width) {
+                        coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                        current_edge++;
+                    }
+
+                    // North
+                    neighbor = ((i - 1) * width * width) + (j * width) + k;
+                    if (i - 1 >= 0) {
+                        coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                        current_edge++;
+                    }
+
+                    // South
+                    neighbor = ((i + 1) * width * width) + (j * width) + k;
+                    if (i + 1 < width) {
+                        coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                        current_edge++;
+                    }
+
+                    if (self_loop)
+                    {
+                        neighbor = me;
+                        coo_tuples[current_edge] = CooTuple(me, neighbor, default_value);
+                        current_edge++;
+                    }
+                }
+            }
+        }
+
+        // Sort by rows, then columns, update dims
+        std::stable_sort(coo_tuples, coo_tuples + num_nonzeros);
+
+        return 0;
+    }
+};
+
+
+
+/******************************************************************************
+ * COO matrix type
+ ******************************************************************************/
+
+
+/**
+ * CSR sparse format matrix
+ */
+template<
+    typename ValueT,
+    typename OffsetT>
+struct CsrMatrix
+{
+    int         num_rows;
+    int         num_cols;
+    int         num_nonzeros;
+    OffsetT*    row_offsets;
+    OffsetT*    column_indices;
+    ValueT*     values;
+    bool        numa_malloc;
+
+    /**
+     * Constructor
+     */
+    CsrMatrix() : num_rows(0), num_cols(0), num_nonzeros(0), row_offsets(NULL), column_indices(NULL), values(NULL) 
+    {
+#ifdef CUB_MKL
+        numa_malloc = ((numa_available() >= 0) && (numa_num_task_nodes() > 1));
+#else
+        numa_malloc = false;
+#endif
+    }
+
+
+    /**
+     * Clear
+     */
+    void Clear()
+    {
+#ifdef CUB_MKL
+        if (numa_malloc) 
+        {
+            numa_free(row_offsets, sizeof(OffsetT) * (num_rows + 1));
+            numa_free(values, sizeof(ValueT) * num_nonzeros);
+            numa_free(column_indices, sizeof(OffsetT) * num_nonzeros);
+        }
+        else
+        {
+            if (row_offsets)    mkl_free(row_offsets);
+            if (column_indices) mkl_free(column_indices);
+            if (values)         mkl_free(values);
+        }
+
+#else
+        if (row_offsets)    delete[] row_offsets;
+        if (column_indices) delete[] column_indices;
+        if (values)         delete[] values;
+#endif
+
+        row_offsets = NULL;
+        column_indices = NULL;
+        values = NULL;
+    }
+
+    /**
+     * Destructor
+     */
+    ~CsrMatrix()
+    {
+        Clear();
+    }
+
+    GraphStats Stats()
+    {
+        GraphStats stats;
+        stats.num_rows = num_rows;
+        stats.num_cols = num_cols;
+        stats.num_nonzeros = num_nonzeros;
+
+        //
+        // Compute diag-distance statistics
+        //
+
+        OffsetT samples     = 0;
+        double  mean        = 0.0;
+        double  ss_tot      = 0.0;
+
+        for (OffsetT row = 0; row < num_rows; ++row)
+        {
+            OffsetT nz_idx_start    = row_offsets[row];
+            OffsetT nz_idx_end      = row_offsets[row + 1];
+
+            for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx)
+            {
+                OffsetT col             = column_indices[nz_idx];
+                double x                = (col > row) ? col - row : row - col;
+
+                samples++;
+                double delta            = x - mean;
+                mean                    = mean + (delta / samples);
+                ss_tot                  += delta * (x - mean);
+            }
+        }
+        stats.diag_dist_mean            = mean;
+        double variance                 = ss_tot / samples;
+        stats.diag_dist_std_dev         = sqrt(variance);
+
+
+        //
+        // Compute deming statistics
+        //
+
+        samples         = 0;
+        double mean_x   = 0.0;
+        double mean_y   = 0.0;
+        double ss_x     = 0.0;
+        double ss_y     = 0.0;
+
+        for (OffsetT row = 0; row < num_rows; ++row)
+        {
+            OffsetT nz_idx_start    = row_offsets[row];
+            OffsetT nz_idx_end      = row_offsets[row + 1];
+
+            for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx)
+            {
+                OffsetT col             = column_indices[nz_idx];
+
+                samples++;
+                double x                = col;
+                double y                = row;
+                double delta;
+
+                delta                   = x - mean_x;
+                mean_x                  = mean_x + (delta / samples);
+                ss_x                    += delta * (x - mean_x);
+
+                delta                   = y - mean_y;
+                mean_y                  = mean_y + (delta / samples);
+                ss_y                    += delta * (y - mean_y);
+            }
+        }
+
+        samples         = 0;
+        double s_xy     = 0.0;
+        double s_xxy    = 0.0;
+        double s_xyy    = 0.0;
+        for (OffsetT row = 0; row < num_rows; ++row)
+        {
+            OffsetT nz_idx_start    = row_offsets[row];
+            OffsetT nz_idx_end      = row_offsets[row + 1];
+
+            for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx)
+            {
+                OffsetT col             = column_indices[nz_idx];
+
+                samples++;
+                double x                = col;
+                double y                = row;
+
+                double xy =             (x - mean_x) * (y - mean_y);
+                double xxy =            (x - mean_x) * (x - mean_x) * (y - mean_y);
+                double xyy =            (x - mean_x) * (y - mean_y) * (y - mean_y);
+                double delta;
+
+                delta                   = xy - s_xy;
+                s_xy                    = s_xy + (delta / samples);
+
+                delta                   = xxy - s_xxy;
+                s_xxy                   = s_xxy + (delta / samples);
+
+                delta                   = xyy - s_xyy;
+                s_xyy                   = s_xyy + (delta / samples);
+            }
+        }
+
+        double s_xx     = ss_x / num_nonzeros;
+        double s_yy     = ss_y / num_nonzeros;
+
+        double deming_slope = (s_yy - s_xx + sqrt(((s_yy - s_xx) * (s_yy - s_xx)) + (4 * s_xy * s_xy))) / (2 * s_xy);
+
+        stats.pearson_r = (num_nonzeros * s_xy) / (sqrt(ss_x) * sqrt(ss_y));
+
+
+        //
+        // Compute row-length statistics
+        //
+
+        // Sample mean
+        stats.row_length_mean       = double(num_nonzeros) / num_rows;
+        variance                    = 0.0;
+        stats.row_length_skewness   = 0.0;
+        for (OffsetT row = 0; row < num_rows; ++row)
+        {
+            OffsetT length              = row_offsets[row + 1] - row_offsets[row];
+            double delta                = double(length) - stats.row_length_mean;
+            variance   += (delta * delta);
+            stats.row_length_skewness   += (delta * delta * delta);
+        }
+        variance                    /= num_rows;
+        stats.row_length_std_dev    = sqrt(variance);
+        stats.row_length_skewness   = (stats.row_length_skewness / num_rows) / pow(stats.row_length_std_dev, 3.0);
+        stats.row_length_variation  = stats.row_length_std_dev / stats.row_length_mean;
+
+        return stats;
+    }
+
+    /**
+     * Build CSR matrix from sorted COO matrix
+     */
+    void FromCoo(const CooMatrix<ValueT, OffsetT> &coo_matrix)
+    {
+        num_rows        = coo_matrix.num_rows;
+        num_cols        = coo_matrix.num_cols;
+        num_nonzeros    = coo_matrix.num_nonzeros;
+
+#ifdef CUB_MKL
+
+        if (numa_malloc)
+        {
+            numa_set_strict(1);
+//            numa_set_bind_policy(1);
+
+//        values          = (ValueT*) numa_alloc_interleaved(sizeof(ValueT) * num_nonzeros);
+//        row_offsets     = (OffsetT*) numa_alloc_interleaved(sizeof(OffsetT) * (num_rows + 1));
+//        column_indices  = (OffsetT*) numa_alloc_interleaved(sizeof(OffsetT) * num_nonzeros);
+
+            row_offsets     = (OffsetT*) numa_alloc_onnode(sizeof(OffsetT) * (num_rows + 1), 0);
+            column_indices  = (OffsetT*) numa_alloc_onnode(sizeof(OffsetT) * num_nonzeros, 0);
+            values          = (ValueT*) numa_alloc_onnode(sizeof(ValueT) * num_nonzeros, 1);
+        }
+        else
+        {
+            values          = (ValueT*) mkl_malloc(sizeof(ValueT) * num_nonzeros, 4096);
+            row_offsets     = (OffsetT*) mkl_malloc(sizeof(OffsetT) * (num_rows + 1), 4096);
+            column_indices  = (OffsetT*) mkl_malloc(sizeof(OffsetT) * num_nonzeros, 4096);
+
+        }
+
+#else
+        row_offsets     = new OffsetT[num_rows + 1];
+        column_indices  = new OffsetT[num_nonzeros];
+        values          = new ValueT[num_nonzeros];
+#endif
+
+        OffsetT prev_row = -1;
+        for (OffsetT current_edge = 0; current_edge < num_nonzeros; current_edge++)
+        {
+            OffsetT current_row = coo_matrix.coo_tuples[current_edge].row;
+
+            // Fill in rows up to and including the current row
+            for (OffsetT row = prev_row + 1; row <= current_row; row++)
+            {
+                row_offsets[row] = current_edge;
+            }
+            prev_row = current_row;
+
+            column_indices[current_edge]    = coo_matrix.coo_tuples[current_edge].col;
+            values[current_edge]            = coo_matrix.coo_tuples[current_edge].val;
+        }
+
+        // Fill out any trailing edgeless vertices (and the end-of-list element)
+        for (OffsetT row = prev_row + 1; row <= num_rows; row++)
+        {
+            row_offsets[row] = num_nonzeros;
+        }
+    }
+
+
+    /**
+     * Display log-histogram to stdout
+     */
+    void DisplayHistogram()
+    {
+        // Initialize
+        int log_counts[9];
+        for (int i = 0; i < 9; i++)
+        {
+            log_counts[i] = 0;
+        }
+
+        // Scan
+        int max_log_length = -1;
+        for (OffsetT row = 0; row < num_rows; row++)
+        {
+            OffsetT length = row_offsets[row + 1] - row_offsets[row];
+
+            int log_length = -1;
+            while (length > 0)
+            {
+                length /= 10;
+                log_length++;
+            }
+            if (log_length > max_log_length)
+            {
+                max_log_length = log_length;
+            }
+
+            log_counts[log_length + 1]++;
+        }
+        printf("CSR matrix (%d rows, %d columns, %d non-zeros):\n", (int) num_rows, (int) num_cols, (int) num_nonzeros);
+        for (int i = -1; i < max_log_length + 1; i++)
+        {
+            printf("\tDegree 1e%d: \t%d (%.2f%%)\n", i, log_counts[i + 1], (float) log_counts[i + 1] * 100.0 / num_cols);
+        }
+        fflush(stdout);
+    }
+
+
+    /**
+     * Display matrix to stdout
+     */
+    void Display()
+    {
+        printf("Input Matrix:\n");
+        for (OffsetT row = 0; row < num_rows; row++)
+        {
+            printf("%d [@%d, #%d]: ", row, row_offsets[row], row_offsets[row + 1] - row_offsets[row]);
+            for (OffsetT current_edge = row_offsets[row]; current_edge < row_offsets[row + 1]; current_edge++)
+            {
+                printf("%d (%f), ", column_indices[current_edge], values[current_edge]);
+            }
+            printf("\n");
+        }
+        fflush(stdout);
+    }
+
+
+};
+
+
+
+/******************************************************************************
+ * Matrix transformations
+ ******************************************************************************/
+
+// Comparator for ordering rows by degree (lowest first), then by row-id (lowest first)
+template <typename OffsetT>
+struct OrderByLow
+{
+    OffsetT* row_degrees;
+    OrderByLow(OffsetT* row_degrees) : row_degrees(row_degrees) {}
+
+    bool operator()(const OffsetT &a, const OffsetT &b)
+    {
+        if (row_degrees[a] < row_degrees[b])
+            return true;
+        else if (row_degrees[a] > row_degrees[b])
+            return false;
+        else
+            return (a < b);
+    }
+};
+
+// Comparator for ordering rows by degree (highest first), then by row-id (lowest first)
+template <typename OffsetT>
+struct OrderByHigh
+{
+    OffsetT* row_degrees;
+    OrderByHigh(OffsetT* row_degrees) : row_degrees(row_degrees) {}
+
+    bool operator()(const OffsetT &a, const OffsetT &b)
+    {
+        if (row_degrees[a] > row_degrees[b])
+            return true;
+        else if (row_degrees[a] < row_degrees[b])
+            return false;
+        else
+            return (a < b);
+    }
+};
+
+
+
+/**
+ * Reverse Cuthill-McKee
+ */
+template <typename ValueT, typename OffsetT>
+void RcmRelabel(
+    CsrMatrix<ValueT, OffsetT>&     matrix,
+    OffsetT*                        relabel_indices)
+{
+    // Initialize row degrees
+    OffsetT* row_degrees_in     = new OffsetT[matrix.num_rows];
+    OffsetT* row_degrees_out    = new OffsetT[matrix.num_rows];
+    for (OffsetT row = 0; row < matrix.num_rows; ++row)
+    {
+        row_degrees_in[row]         = 0;
+        row_degrees_out[row]        = matrix.row_offsets[row + 1] - matrix.row_offsets[row];
+    }
+    for (OffsetT nonzero = 0; nonzero < matrix.num_nonzeros; ++nonzero)
+    {
+        row_degrees_in[matrix.column_indices[nonzero]]++;
+    }
+
+    // Initialize unlabeled set 
+    typedef std::set<OffsetT, OrderByLow<OffsetT> > UnlabeledSet;
+    typename UnlabeledSet::key_compare  unlabeled_comp(row_degrees_in);
+    UnlabeledSet                        unlabeled(unlabeled_comp);
+    for (OffsetT row = 0; row < matrix.num_rows; ++row)
+    {
+        relabel_indices[row]    = -1;
+        unlabeled.insert(row);
+    }
+
+    // Initialize queue set
+    std::deque<OffsetT> q;
+
+    // Process unlabeled vertices (traverse connected components)
+    OffsetT relabel_idx = 0;
+    while (!unlabeled.empty())
+    {
+        // Seed the unvisited frontier queue with the unlabeled vertex of lowest-degree
+        OffsetT vertex = *unlabeled.begin();
+        q.push_back(vertex);
+
+        while (!q.empty())
+        {
+            vertex = q.front();
+            q.pop_front();
+
+            if (relabel_indices[vertex] == -1)
+            {
+                // Update this vertex
+                unlabeled.erase(vertex);
+                relabel_indices[vertex] = relabel_idx;
+                relabel_idx++;
+
+                // Sort neighbors by degree
+                OrderByLow<OffsetT> neighbor_comp(row_degrees_in);
+                std::sort(
+                    matrix.column_indices + matrix.row_offsets[vertex],
+                    matrix.column_indices + matrix.row_offsets[vertex + 1],
+                    neighbor_comp);
+
+                // Inspect neighbors, adding to the out frontier if unlabeled
+                for (OffsetT neighbor_idx = matrix.row_offsets[vertex];
+                    neighbor_idx < matrix.row_offsets[vertex + 1];
+                    ++neighbor_idx)
+                {
+                    OffsetT neighbor = matrix.column_indices[neighbor_idx];
+                    q.push_back(neighbor);
+                }
+            }
+        }
+    }
+
+/*
+    // Reverse labels
+    for (int row = 0; row < matrix.num_rows; ++row)
+    {
+        relabel_indices[row] = matrix.num_rows - relabel_indices[row] - 1;
+    }
+*/
+
+    // Cleanup
+    if (row_degrees_in) delete[] row_degrees_in;
+    if (row_degrees_out) delete[] row_degrees_out;
+}
+
+
+/**
+ * Reverse Cuthill-McKee
+ */
+template <typename ValueT, typename OffsetT>
+void RcmRelabel(
+    CsrMatrix<ValueT, OffsetT>&     matrix,
+    bool                            verbose = false)
+{
+    // Do not process if not square
+    if (matrix.num_cols != matrix.num_rows)
+    {
+        if (verbose) {
+            printf("RCM transformation ignored (not square)\n"); fflush(stdout);
+        }
+        return;
+    }
+
+    // Initialize relabel indices
+    OffsetT* relabel_indices = new OffsetT[matrix.num_rows];
+
+    if (verbose) {
+        printf("RCM relabeling... "); fflush(stdout);
+    }
+
+    RcmRelabel(matrix, relabel_indices);
+
+    if (verbose) {
+        printf("done. Reconstituting... "); fflush(stdout);
+    }
+
+    // Create a COO matrix from the relabel indices
+    CooMatrix<ValueT, OffsetT> coo_matrix;
+    coo_matrix.InitCsrRelabel(matrix, relabel_indices);
+
+    // Reconstitute the CSR matrix from the sorted COO tuples
+    if (relabel_indices) delete[] relabel_indices;
+    matrix.Clear();
+    matrix.FromCoo(coo_matrix);
+
+    if (verbose) {
+        printf("done. "); fflush(stdout);
+    }
+}
+
+
+
+
diff --git a/third_party/cub/experimental/spmv_compare.cu b/third_party/cub/experimental/spmv_compare.cu
new file mode 100644
index 0000000..b64297d
--- /dev/null
+++ b/third_party/cub/experimental/spmv_compare.cu
@@ -0,0 +1,917 @@
+/******************************************************************************
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIAeBILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+//---------------------------------------------------------------------
+// SpMV comparison tool
+//---------------------------------------------------------------------
+
+#include <stdio.h>
+#include <map>
+#include <vector>
+#include <algorithm>
+#include <cstdio>
+#include <fstream>
+
+#include <cusparse.h>
+
+#include "sparse_matrix.h"
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <cub/device/device_spmv.cuh>
+#include <cub/util_allocator.cuh>
+#include <cub/iterator/tex_ref_input_iterator.cuh>
+#include <test/test_util.h>
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants, and type declarations
+//---------------------------------------------------------------------
+
+bool                    g_quiet     = false;        // Whether to display stats in CSV format
+bool                    g_verbose   = false;        // Whether to display output to console
+bool                    g_verbose2  = false;        // Whether to display input to console
+CachingDeviceAllocator  g_allocator(true);          // Caching allocator for device memory
+
+
+//---------------------------------------------------------------------
+// SpMV verification
+//---------------------------------------------------------------------
+
+// Compute reference SpMV y = Ax
+template <
+    typename ValueT,
+    typename OffsetT>
+void SpmvGold(
+    CsrMatrix<ValueT, OffsetT>&     a,
+    ValueT*                         vector_x,
+    ValueT*                         vector_y_in,
+    ValueT*                         vector_y_out,
+    ValueT                          alpha,
+    ValueT                          beta)
+{
+    for (OffsetT row = 0; row < a.num_rows; ++row)
+    {
+        ValueT partial = beta * vector_y_in[row];
+        for (
+            OffsetT offset = a.row_offsets[row];
+            offset < a.row_offsets[row + 1];
+            ++offset)
+        {
+            partial += alpha * a.values[offset] * vector_x[a.column_indices[offset]];
+        }
+        vector_y_out[row] = partial;
+    }
+}
+
+
+//---------------------------------------------------------------------
+// GPU I/O proxy
+//---------------------------------------------------------------------
+
+/**
+ * Read every matrix nonzero value, read every corresponding vector value
+ */
+template <
+    int         BLOCK_THREADS,
+    int         ITEMS_PER_THREAD,
+    typename    ValueT,
+    typename    OffsetT,
+    typename    VectorItr>
+__launch_bounds__ (int(BLOCK_THREADS))
+__global__ void NonZeroIoKernel(
+    SpmvParams<ValueT, OffsetT> params,
+    VectorItr                   d_vector_x)
+{
+    enum
+    {
+        TILE_ITEMS      = BLOCK_THREADS * ITEMS_PER_THREAD,
+    };
+
+
+    ValueT nonzero = 0.0;
+
+    int tile_idx = blockIdx.x;
+
+    OffsetT block_offset = tile_idx * TILE_ITEMS;
+
+    OffsetT column_indices[ITEMS_PER_THREAD];
+    ValueT values[ITEMS_PER_THREAD];
+
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+    {
+        OffsetT nonzero_idx = block_offset + (ITEM * BLOCK_THREADS) + threadIdx.x;
+
+        OffsetT* ci = params.d_column_indices + nonzero_idx;
+        ValueT*a = params.d_values + nonzero_idx;
+
+        column_indices[ITEM]    = (nonzero_idx < params.num_nonzeros) ? *ci : 0;
+        values[ITEM]            = (nonzero_idx < params.num_nonzeros) ? *a : 0.0;
+    }
+
+    __syncthreads();
+
+    // Read vector
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+    {
+        ValueT vector_value    = ThreadLoad<LOAD_LDG>(params.d_vector_x + column_indices[ITEM]);
+        nonzero                += vector_value * values[ITEM];
+    }
+
+    __syncthreads();
+
+    if (block_offset < params.num_rows)
+    {
+        #pragma unroll
+        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        {
+            OffsetT row_idx = block_offset + (ITEM * BLOCK_THREADS) + threadIdx.x;
+            if (row_idx < params.num_rows)
+            {
+                OffsetT row_end_offset = ThreadLoad<LOAD_DEFAULT>(params.d_row_end_offsets + row_idx);
+
+                if ((row_end_offset >= 0) && (nonzero == nonzero))
+                    params.d_vector_y[row_idx] = nonzero;
+            }
+        }
+    }
+
+}
+
+
+/**
+ * Run GPU I/O proxy
+ */
+template <
+    typename ValueT,
+    typename OffsetT>
+float TestGpuCsrIoProxy(
+    SpmvParams<ValueT, OffsetT>&    params,
+    int                             timing_iterations)
+{
+    enum {
+        BLOCK_THREADS       = 128,
+        ITEMS_PER_THREAD    = 7,
+        TILE_SIZE           = BLOCK_THREADS * ITEMS_PER_THREAD,
+    };
+
+//    size_t smem = 1024 * 16;
+    size_t smem = 1024 * 0;
+
+    unsigned int nonzero_blocks = (params.num_nonzeros + TILE_SIZE - 1) / TILE_SIZE;
+    unsigned int row_blocks = (params.num_rows + TILE_SIZE - 1) / TILE_SIZE;
+    unsigned int blocks = std::max(nonzero_blocks, row_blocks);
+
+    typedef TexRefInputIterator<ValueT, 1234, int> TexItr;
+    TexItr x_itr;
+    CubDebugExit(x_itr.BindTexture(params.d_vector_x));
+
+    // Get device ordinal
+    int device_ordinal;
+    CubDebugExit(cudaGetDevice(&device_ordinal));
+
+    // Get device SM version
+    int sm_version;
+    CubDebugExit(SmVersion(sm_version, device_ordinal));
+
+    void (*kernel)(SpmvParams<ValueT, OffsetT>, TexItr) = NonZeroIoKernel<BLOCK_THREADS, ITEMS_PER_THREAD>;
+
+
+    int spmv_sm_occupancy;
+    CubDebugExit(MaxSmOccupancy(spmv_sm_occupancy, kernel, BLOCK_THREADS, smem));
+
+    if (!g_quiet)
+        printf("NonZeroIoKernel<%d,%d><<<%d, %d>>>, sm occupancy %d\n", BLOCK_THREADS, ITEMS_PER_THREAD, blocks, BLOCK_THREADS, spmv_sm_occupancy);
+
+    // Warmup
+    NonZeroIoKernel<BLOCK_THREADS, ITEMS_PER_THREAD><<<blocks, BLOCK_THREADS, smem>>>(params, x_itr);
+
+    // Check for failures
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(SyncStream(0));
+
+    // Timing
+    GpuTimer timer;
+    float elapsed_millis = 0.0;
+    timer.Start();
+    for (int it = 0; it < timing_iterations; ++it)
+    {
+        NonZeroIoKernel<BLOCK_THREADS, ITEMS_PER_THREAD><<<blocks, BLOCK_THREADS, smem>>>(params, x_itr);
+    }
+    timer.Stop();
+    elapsed_millis += timer.ElapsedMillis();
+
+    CubDebugExit(x_itr.UnbindTexture());
+
+    return elapsed_millis / timing_iterations;
+}
+
+
+
+//---------------------------------------------------------------------
+// cuSparse HybMV
+//---------------------------------------------------------------------
+
+/**
+ * Run cuSparse HYB SpMV (specialized for fp32)
+ */
+template <
+    typename OffsetT>
+float TestCusparseHybmv(
+    float*                          vector_y_in,
+    float*                          reference_vector_y_out,
+    SpmvParams<float, OffsetT>&     params,
+    int                             timing_iterations,
+    cusparseHandle_t                cusparse)
+{
+    CpuTimer cpu_timer;
+    cpu_timer.Start();
+
+    // Construct Hyb matrix
+    cusparseMatDescr_t mat_desc;
+    cusparseHybMat_t hyb_desc;
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseCreateMatDescr(&mat_desc));
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseCreateHybMat(&hyb_desc));
+    cusparseStatus_t status = cusparseScsr2hyb(
+        cusparse,
+        params.num_rows, params.num_cols,
+        mat_desc,
+        params.d_values, params.d_row_end_offsets, params.d_column_indices,
+        hyb_desc,
+        0,
+        CUSPARSE_HYB_PARTITION_AUTO);
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, status);
+
+    cudaDeviceSynchronize();
+    cpu_timer.Stop();
+    float elapsed_millis = cpu_timer.ElapsedMillis();
+    printf("HYB setup ms, %.5f, ", elapsed_millis);
+
+    // Reset input/output vector y
+    CubDebugExit(cudaMemcpy(params.d_vector_y, vector_y_in, sizeof(float) * params.num_rows, cudaMemcpyHostToDevice));
+
+    // Warmup
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseShybmv(
+        cusparse,
+        CUSPARSE_OPERATION_NON_TRANSPOSE,
+        &params.alpha, mat_desc,
+        hyb_desc,
+        params.d_vector_x, &params.beta, params.d_vector_y));
+
+    if (!g_quiet)
+    {
+        int compare = CompareDeviceResults(reference_vector_y_out, params.d_vector_y, params.num_rows, true, g_verbose);
+        printf("\t%s\n", compare ? "FAIL" : "PASS"); fflush(stdout);
+    }
+
+    // Timing
+    elapsed_millis    = 0.0;
+    GpuTimer timer;
+
+    timer.Start();
+    for(int it = 0; it < timing_iterations; ++it)
+    {
+        AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseShybmv(
+            cusparse,
+            CUSPARSE_OPERATION_NON_TRANSPOSE,
+            &params.alpha, mat_desc,
+            hyb_desc,
+            params.d_vector_x, &params.beta, params.d_vector_y));
+    }
+    timer.Stop();
+    elapsed_millis += timer.ElapsedMillis();
+
+    // Cleanup
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDestroyHybMat(hyb_desc));
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDestroyMatDescr(mat_desc));
+
+    return elapsed_millis / timing_iterations;
+}
+
+
+/**
+ * Run cuSparse HYB SpMV (specialized for fp64)
+ */
+template <
+    typename OffsetT>
+float TestCusparseHybmv(
+    double*                         vector_y_in,
+    double*                         reference_vector_y_out,
+    SpmvParams<double, OffsetT>&    params,
+    int                             timing_iterations,
+    cusparseHandle_t                cusparse)
+{
+    CpuTimer cpu_timer;
+    cpu_timer.Start();
+
+    // Construct Hyb matrix
+    cusparseMatDescr_t mat_desc;
+    cusparseHybMat_t hyb_desc;
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseCreateMatDescr(&mat_desc));
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseCreateHybMat(&hyb_desc));
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDcsr2hyb(
+        cusparse,
+        params.num_rows, params.num_cols,
+        mat_desc,
+        params.d_values, params.d_row_end_offsets, params.d_column_indices,
+        hyb_desc,
+        0,
+        CUSPARSE_HYB_PARTITION_AUTO));
+
+    cudaDeviceSynchronize();
+    cpu_timer.Stop();
+    float elapsed_millis = cpu_timer.ElapsedMillis();
+    printf("HYB setup ms, %.5f, ", elapsed_millis);
+
+    // Reset input/output vector y
+    CubDebugExit(cudaMemcpy(params.d_vector_y, vector_y_in, sizeof(float) * params.num_rows, cudaMemcpyHostToDevice));
+
+    // Warmup
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDhybmv(
+        cusparse,
+        CUSPARSE_OPERATION_NON_TRANSPOSE,
+        &params.alpha, mat_desc,
+        hyb_desc,
+        params.d_vector_x, &params.beta, params.d_vector_y));
+
+    if (!g_quiet)
+    {
+        int compare = CompareDeviceResults(reference_vector_y_out, params.d_vector_y, params.num_rows, true, g_verbose);
+        printf("\t%s\n", compare ? "FAIL" : "PASS"); fflush(stdout);
+    }
+
+    // Timing
+    elapsed_millis    = 0.0;
+    GpuTimer timer;
+
+    timer.Start();
+    for(int it = 0; it < timing_iterations; ++it)
+    {
+        AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDhybmv(
+            cusparse,
+            CUSPARSE_OPERATION_NON_TRANSPOSE,
+            &params.alpha, mat_desc,
+            hyb_desc,
+            params.d_vector_x, &params.beta, params.d_vector_y));
+    }
+    timer.Stop();
+    elapsed_millis += timer.ElapsedMillis();
+
+    // Cleanup
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDestroyHybMat(hyb_desc));
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDestroyMatDescr(mat_desc));
+
+    return elapsed_millis / timing_iterations;
+}
+
+
+
+//---------------------------------------------------------------------
+// cuSparse CsrMV
+//---------------------------------------------------------------------
+
+/**
+ * Run cuSparse SpMV (specialized for fp32)
+ */
+template <
+    typename OffsetT>
+float TestCusparseCsrmv(
+    float*                          vector_y_in,
+    float*                          reference_vector_y_out,
+    SpmvParams<float, OffsetT>&     params,
+    int                             timing_iterations,
+    cusparseHandle_t                cusparse)
+{
+    cusparseMatDescr_t desc;
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseCreateMatDescr(&desc));
+
+    // Reset input/output vector y
+    CubDebugExit(cudaMemcpy(params.d_vector_y, vector_y_in, sizeof(float) * params.num_rows, cudaMemcpyHostToDevice));
+
+    // Warmup
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseScsrmv(
+        cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+        params.num_rows, params.num_cols, params.num_nonzeros, &params.alpha, desc,
+        params.d_values, params.d_row_end_offsets, params.d_column_indices,
+        params.d_vector_x, &params.beta, params.d_vector_y));
+
+    if (!g_quiet)
+    {
+        int compare = CompareDeviceResults(reference_vector_y_out, params.d_vector_y, params.num_rows, true, g_verbose);
+        printf("\t%s\n", compare ? "FAIL" : "PASS"); fflush(stdout);
+    }
+
+    // Timing
+    float elapsed_millis    = 0.0;
+    GpuTimer timer;
+
+    timer.Start();
+    for(int it = 0; it < timing_iterations; ++it)
+    {
+        AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseScsrmv(
+            cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+            params.num_rows, params.num_cols, params.num_nonzeros, &params.alpha, desc,
+            params.d_values, params.d_row_end_offsets, params.d_column_indices,
+            params.d_vector_x, &params.beta, params.d_vector_y));
+    }
+    timer.Stop();
+    elapsed_millis += timer.ElapsedMillis();
+
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDestroyMatDescr(desc));
+    return elapsed_millis / timing_iterations;
+}
+
+
+/**
+ * Run cuSparse SpMV (specialized for fp64)
+ */
+template <
+    typename OffsetT>
+float TestCusparseCsrmv(
+    double*                         vector_y_in,
+    double*                         reference_vector_y_out,
+    SpmvParams<double, OffsetT>&    params,
+    int                             timing_iterations,
+    cusparseHandle_t                cusparse)
+{
+    cusparseMatDescr_t desc;
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseCreateMatDescr(&desc));
+
+    // Reset input/output vector y
+    CubDebugExit(cudaMemcpy(params.d_vector_y, vector_y_in, sizeof(float) * params.num_rows, cudaMemcpyHostToDevice));
+
+    // Warmup
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDcsrmv(
+        cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+        params.num_rows, params.num_cols, params.num_nonzeros, &params.alpha, desc,
+        params.d_values, params.d_row_end_offsets, params.d_column_indices,
+        params.d_vector_x, &params.beta, params.d_vector_y));
+
+    if (!g_quiet)
+    {
+        int compare = CompareDeviceResults(reference_vector_y_out, params.d_vector_y, params.num_rows, true, g_verbose);
+        printf("\t%s\n", compare ? "FAIL" : "PASS"); fflush(stdout);
+    }
+
+    // Timing
+    float elapsed_millis = 0.0;
+    GpuTimer timer;
+    timer.Start();
+    for(int it = 0; it < timing_iterations; ++it)
+    {
+        AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDcsrmv(
+            cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+            params.num_rows, params.num_cols, params.num_nonzeros, &params.alpha, desc,
+            params.d_values, params.d_row_end_offsets, params.d_column_indices,
+            params.d_vector_x, &params.beta, params.d_vector_y));
+
+    }
+    timer.Stop();
+    elapsed_millis += timer.ElapsedMillis();
+
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseDestroyMatDescr(desc));
+    return elapsed_millis / timing_iterations;
+}
+
+//---------------------------------------------------------------------
+// GPU Merge-based SpMV
+//---------------------------------------------------------------------
+
+/**
+ * Run CUB SpMV
+ */
+template <
+    typename ValueT,
+    typename OffsetT>
+float TestGpuMergeCsrmv(
+    ValueT*                         vector_y_in,
+    ValueT*                         reference_vector_y_out,
+    SpmvParams<ValueT, OffsetT>&    params,
+    int                             timing_iterations)
+{
+    // Allocate temporary storage
+    size_t temp_storage_bytes = 0;
+    void *d_temp_storage = NULL;
+
+    // Get amount of temporary storage needed
+    CubDebugExit(DeviceSpmv::CsrMV(
+        d_temp_storage, temp_storage_bytes,
+        params.d_values, params.d_row_end_offsets, params.d_column_indices,
+        params.d_vector_x, params.d_vector_y,
+        params.num_rows, params.num_cols, params.num_nonzeros,
+// params.alpha, params.beta,
+        (cudaStream_t) 0, false));
+
+    // Allocate
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Reset input/output vector y
+    CubDebugExit(cudaMemcpy(params.d_vector_y, vector_y_in, sizeof(ValueT) * params.num_rows, cudaMemcpyHostToDevice));
+
+    // Warmup
+    CubDebugExit(DeviceSpmv::CsrMV(
+        d_temp_storage, temp_storage_bytes,
+        params.d_values, params.d_row_end_offsets, params.d_column_indices,
+        params.d_vector_x, params.d_vector_y,
+        params.num_rows, params.num_cols, params.num_nonzeros, 
+// params.alpha, params.beta,
+        (cudaStream_t) 0, !g_quiet));
+
+    if (!g_quiet)
+    {
+        int compare = CompareDeviceResults(reference_vector_y_out, params.d_vector_y, params.num_rows, true, g_verbose);
+        printf("\t%s\n", compare ? "FAIL" : "PASS"); fflush(stdout);
+    }
+
+    // Timing
+    GpuTimer timer;
+    float elapsed_millis = 0.0;
+
+    timer.Start();
+    for(int it = 0; it < timing_iterations; ++it)
+    {
+        CubDebugExit(DeviceSpmv::CsrMV(
+            d_temp_storage, temp_storage_bytes,
+            params.d_values, params.d_row_end_offsets, params.d_column_indices,
+            params.d_vector_x, params.d_vector_y,
+            params.num_rows, params.num_cols, params.num_nonzeros, 
+// params.alpha, params.beta,
+            (cudaStream_t) 0, false));
+    }
+    timer.Stop();
+    elapsed_millis += timer.ElapsedMillis();
+
+    return elapsed_millis / timing_iterations;
+}
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+/**
+ * Display perf
+ */
+template <typename ValueT, typename OffsetT>
+void DisplayPerf(
+    float                           device_giga_bandwidth,
+    double                          avg_millis,
+    CsrMatrix<ValueT, OffsetT>&     csr_matrix)
+{
+    double nz_throughput, effective_bandwidth;
+    size_t total_bytes = (csr_matrix.num_nonzeros * (sizeof(ValueT) * 2 + sizeof(OffsetT))) +
+        (csr_matrix.num_rows) * (sizeof(OffsetT) + sizeof(ValueT));
+
+    nz_throughput       = double(csr_matrix.num_nonzeros) / avg_millis / 1.0e6;
+    effective_bandwidth = double(total_bytes) / avg_millis / 1.0e6;
+
+    if (!g_quiet)
+        printf("fp%d: %.4f avg ms, %.5f gflops, %.3lf effective GB/s (%.2f%% peak)\n",
+            sizeof(ValueT) * 8,
+            avg_millis,
+            2 * nz_throughput,
+            effective_bandwidth,
+            effective_bandwidth / device_giga_bandwidth * 100);
+    else
+        printf("%.5f, %.6f, %.3lf, %.2f%%, ",
+            avg_millis,
+            2 * nz_throughput,
+            effective_bandwidth,
+            effective_bandwidth / device_giga_bandwidth * 100);
+
+    fflush(stdout);
+}
+
+
+
+/**
+ * Run tests
+ */
+template <
+    typename ValueT,
+    typename OffsetT>
+void RunTest(
+    bool                        rcm_relabel,
+    ValueT                      alpha,
+    ValueT                      beta,
+    CooMatrix<ValueT, OffsetT>& coo_matrix,
+    int                         timing_iterations,
+    CommandLineArgs&            args)
+{
+    // Adaptive timing iterations: run 16 billion nonzeros through
+    if (timing_iterations == -1)
+        timing_iterations = std::min(50000ull, std::max(100ull, ((16ull << 30) / coo_matrix.num_nonzeros)));
+
+    if (!g_quiet)
+        printf("\t%d timing iterations\n", timing_iterations);
+
+    // Convert to CSR
+    CsrMatrix<ValueT, OffsetT> csr_matrix;
+    csr_matrix.FromCoo(coo_matrix);
+    if (!args.CheckCmdLineFlag("csrmv"))
+        coo_matrix.Clear();
+
+    // Relabel
+    if (rcm_relabel)
+    {
+        if (!g_quiet)
+        {
+            csr_matrix.Stats().Display();
+            printf("\n");
+            csr_matrix.DisplayHistogram();
+            printf("\n");
+            if (g_verbose2)
+                csr_matrix.Display();
+            printf("\n");
+        }
+
+        RcmRelabel(csr_matrix, !g_quiet);
+
+        if (!g_quiet) printf("\n");
+    }
+
+    // Display matrix info
+    csr_matrix.Stats().Display(!g_quiet);
+    if (!g_quiet)
+    {
+        printf("\n");
+        csr_matrix.DisplayHistogram();
+        printf("\n");
+        if (g_verbose2)
+            csr_matrix.Display();
+        printf("\n");
+    }
+    fflush(stdout);
+
+    // Allocate input and output vectors
+    ValueT* vector_x        = new ValueT[csr_matrix.num_cols];
+    ValueT* vector_y_in     = new ValueT[csr_matrix.num_rows];
+    ValueT* vector_y_out    = new ValueT[csr_matrix.num_rows];
+
+    for (int col = 0; col < csr_matrix.num_cols; ++col)
+        vector_x[col] = 1.0;
+
+    for (int row = 0; row < csr_matrix.num_rows; ++row)
+        vector_y_in[row] = 1.0;
+
+    // Compute reference answer
+    SpmvGold(csr_matrix, vector_x, vector_y_in, vector_y_out, alpha, beta);
+
+    float avg_millis;
+
+    if (g_quiet) {
+        printf("%s, %s, ", args.deviceProp.name, (sizeof(ValueT) > 4) ? "fp64" : "fp32"); fflush(stdout);
+    }
+
+    // Get GPU device bandwidth (GB/s)
+    float device_giga_bandwidth = args.device_giga_bandwidth;
+
+    // Allocate and initialize GPU problem
+    SpmvParams<ValueT, OffsetT> params;
+
+    CubDebugExit(g_allocator.DeviceAllocate((void **) &params.d_values,          sizeof(ValueT) * csr_matrix.num_nonzeros));
+    CubDebugExit(g_allocator.DeviceAllocate((void **) &params.d_row_end_offsets, sizeof(OffsetT) * (csr_matrix.num_rows + 1)));
+    CubDebugExit(g_allocator.DeviceAllocate((void **) &params.d_column_indices,  sizeof(OffsetT) * csr_matrix.num_nonzeros));
+    CubDebugExit(g_allocator.DeviceAllocate((void **) &params.d_vector_x,        sizeof(ValueT) * csr_matrix.num_cols));
+    CubDebugExit(g_allocator.DeviceAllocate((void **) &params.d_vector_y,        sizeof(ValueT) * csr_matrix.num_rows));
+    params.num_rows         = csr_matrix.num_rows;
+    params.num_cols         = csr_matrix.num_cols;
+    params.num_nonzeros     = csr_matrix.num_nonzeros;
+    params.alpha            = alpha;
+    params.beta             = beta;
+
+    CubDebugExit(cudaMemcpy(params.d_values,            csr_matrix.values,          sizeof(ValueT) * csr_matrix.num_nonzeros, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(params.d_row_end_offsets,   csr_matrix.row_offsets,     sizeof(OffsetT) * (csr_matrix.num_rows + 1), cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(params.d_column_indices,    csr_matrix.column_indices,  sizeof(OffsetT) * csr_matrix.num_nonzeros, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(params.d_vector_x,          vector_x,                   sizeof(ValueT) * csr_matrix.num_cols, cudaMemcpyHostToDevice));
+
+    if (!g_quiet) printf("\n\n");
+    printf("GPU CSR I/O Prox, "); fflush(stdout);
+    avg_millis = TestGpuCsrIoProxy(params, timing_iterations);
+    DisplayPerf(device_giga_bandwidth, avg_millis, csr_matrix);
+
+    if (args.CheckCmdLineFlag("csrmv"))
+    {
+        if (!g_quiet) printf("\n\n");
+        printf("CUB, "); fflush(stdout);
+        avg_millis = TestGpuMergeCsrmv(vector_y_in, vector_y_out, params, timing_iterations);
+        DisplayPerf(device_giga_bandwidth, avg_millis, csr_matrix);
+    }
+
+    // Initialize cuSparse
+    cusparseHandle_t cusparse;
+    AssertEquals(CUSPARSE_STATUS_SUCCESS, cusparseCreate(&cusparse));
+
+    if (args.CheckCmdLineFlag("csrmv"))
+    {
+        if (!g_quiet) printf("\n\n");
+        printf("Cusparse CsrMV, "); fflush(stdout);
+        avg_millis = TestCusparseCsrmv(vector_y_in, vector_y_out, params, timing_iterations, cusparse);
+        DisplayPerf(device_giga_bandwidth, avg_millis, csr_matrix);
+    }
+
+    if (args.CheckCmdLineFlag("hybmv"))
+    {
+        if (!g_quiet) printf("\n\n");
+        printf("Cusparse HybMV, "); fflush(stdout);
+
+        avg_millis = TestCusparseHybmv(vector_y_in, vector_y_out, params, timing_iterations, cusparse);
+        DisplayPerf(device_giga_bandwidth, avg_millis, csr_matrix);
+    }
+
+
+    // Cleanup
+    if (params.d_values)            CubDebugExit(g_allocator.DeviceFree(params.d_values));
+    if (params.d_row_end_offsets)   CubDebugExit(g_allocator.DeviceFree(params.d_row_end_offsets));
+    if (params.d_column_indices)    CubDebugExit(g_allocator.DeviceFree(params.d_column_indices));
+    if (params.d_vector_x)          CubDebugExit(g_allocator.DeviceFree(params.d_vector_x));
+    if (params.d_vector_y)          CubDebugExit(g_allocator.DeviceFree(params.d_vector_y));
+
+    if (vector_x)                   delete[] vector_x;
+    if (vector_y_in)                delete[] vector_y_in;
+    if (vector_y_out)               delete[] vector_y_out;
+}
+
+/**
+ * Run tests
+ */
+template <
+    typename ValueT,
+    typename OffsetT>
+void RunTests(
+    bool                rcm_relabel,
+    ValueT              alpha,
+    ValueT              beta,
+    const std::string&  mtx_filename,
+    int                 grid2d,
+    int                 grid3d,
+    int                 wheel,
+    int                 dense,
+    int                 timing_iterations,
+    CommandLineArgs&    args)
+{
+    // Initialize matrix in COO form
+    CooMatrix<ValueT, OffsetT> coo_matrix;
+
+    if (!mtx_filename.empty())
+    {
+        // Parse matrix market file
+        printf("%s, ", mtx_filename.c_str()); fflush(stdout);
+        coo_matrix.InitMarket(mtx_filename, 1.0, !g_quiet);
+
+        if ((coo_matrix.num_rows == 1) || (coo_matrix.num_cols == 1) || (coo_matrix.num_nonzeros == 1))
+        {
+            if (!g_quiet) printf("Trivial dataset\n");
+            exit(0);
+        }
+    }
+    else if (grid2d > 0)
+    {
+        // Generate 2D lattice
+        printf("grid2d_%d, ", grid2d); fflush(stdout);
+        coo_matrix.InitGrid2d(grid2d, false);
+    }
+    else if (grid3d > 0)
+    {
+        // Generate 3D lattice
+        printf("grid3d_%d, ", grid3d); fflush(stdout);
+        coo_matrix.InitGrid3d(grid3d, false);
+    }
+    else if (wheel > 0)
+    {
+        // Generate wheel graph
+        printf("wheel_%d, ", grid2d); fflush(stdout);
+        coo_matrix.InitWheel(wheel);
+    }
+    else if (dense > 0)
+    {
+        // Generate dense graph
+        OffsetT size = 1 << 24; // 16M nnz
+        args.GetCmdLineArgument("size", size);
+
+        OffsetT rows = size / dense;
+        printf("dense_%d_x_%d, ", rows, dense); fflush(stdout);
+        coo_matrix.InitDense(rows, dense);
+    }
+    else
+    {
+        fprintf(stderr, "No graph type specified.\n");
+        exit(1);
+    }
+
+    RunTest(
+        rcm_relabel,
+        alpha,
+        beta,
+        coo_matrix,
+        timing_iterations,
+        args);
+}
+
+
+
+/**
+ * Main
+ */
+int main(int argc, char **argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf(
+            "%s "
+            "[--csrmv | --hybmv | --bsrmv ] "
+            "[--device=<device-id>] "
+            "[--quiet] "
+            "[--v] "
+            "[--i=<timing iterations>] "
+            "[--fp64] "
+            "[--rcm] "
+            "[--alpha=<alpha scalar (default: 1.0)>] "
+            "[--beta=<beta scalar (default: 0.0)>] "
+            "\n\t"
+                "--mtx=<matrix market file> "
+            "\n\t"
+                "--dense=<cols>"
+            "\n\t"
+                "--grid2d=<width>"
+            "\n\t"
+                "--grid3d=<width>"
+            "\n\t"
+                "--wheel=<spokes>"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    bool                fp64;
+    bool                rcm_relabel;
+    std::string         mtx_filename;
+    int                 grid2d              = -1;
+    int                 grid3d              = -1;
+    int                 wheel               = -1;
+    int                 dense               = -1;
+    int                 timing_iterations   = -1;
+    float               alpha               = 1.0;
+    float               beta                = 0.0;
+
+    g_verbose = args.CheckCmdLineFlag("v");
+    g_verbose2 = args.CheckCmdLineFlag("v2");
+    g_quiet = args.CheckCmdLineFlag("quiet");
+    fp64 = args.CheckCmdLineFlag("fp64");
+    rcm_relabel = args.CheckCmdLineFlag("rcm");
+    args.GetCmdLineArgument("i", timing_iterations);
+    args.GetCmdLineArgument("mtx", mtx_filename);
+    args.GetCmdLineArgument("grid2d", grid2d);
+    args.GetCmdLineArgument("grid3d", grid3d);
+    args.GetCmdLineArgument("wheel", wheel);
+    args.GetCmdLineArgument("dense", dense);
+    args.GetCmdLineArgument("alpha", alpha);
+    args.GetCmdLineArgument("beta", beta);
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Run test(s)
+    if (fp64)
+    {
+        RunTests<double, int>(rcm_relabel, alpha, beta, mtx_filename, grid2d, grid3d, wheel, dense, timing_iterations, args);
+    }
+    else
+    {
+        RunTests<float, int>(rcm_relabel, alpha, beta, mtx_filename, grid2d, grid3d, wheel, dense, timing_iterations, args);
+    }
+
+    CubDebugExit(cudaDeviceSynchronize());
+    printf("\n");
+
+    return 0;
+}
diff --git a/third_party/cub/experimental/spmv_script.sh b/third_party/cub/experimental/spmv_script.sh
new file mode 100755
index 0000000..f432043
--- /dev/null
+++ b/third_party/cub/experimental/spmv_script.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+for i in 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152 4194304 8388608 16777216
+do
+	echo `date`, `$1 --dense=$i $2 $3 $4 $5 $6 $7`
+done
+
+echo
+echo
+
+for i in `ls /home/dumerrill/graphs/spmv/*.mtx`
+do
+    if [[ ( "`head -n 50 $i | grep complex`" = "" ) && ( "`head -n 50 $i | grep array`" = "" ) ]] 
+    then
+    	echo `date`, `$1 --mtx=$i $2 $3 $4 $5 $6 $7 2>/dev/null`
+    fi
+done
+
+echo
+echo
+
+for i in `ls /scratch/dumerrill/graphs/mtx/*.mtx`
+#for i in `ls /cygdrive/w/Dev/UFget/mtx/*.mtx`
+do 
+    if [[ ( "`head -n 50 $i | grep complex`" = "" ) && ( "`head -n 50 $i | grep array`" = "" ) ]] 
+    then
+    	echo `date`, `$1 --mtx=$i $2 $3 $4 $5 $6 $7 2>/dev/null`
+    fi
+done 
+
diff --git a/third_party/cub/test/.gitignore b/third_party/cub/test/.gitignore
new file mode 100644
index 0000000..978ba97
--- /dev/null
+++ b/third_party/cub/test/.gitignore
@@ -0,0 +1,3 @@
+/bin
+/link_main.obj
+/dummy/
diff --git a/third_party/cub/test/Makefile b/third_party/cub/test/Makefile
new file mode 100644
index 0000000..958760a
--- /dev/null
+++ b/third_party/cub/test/Makefile
@@ -0,0 +1,468 @@
+#/******************************************************************************
+# * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+# * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+# * 
+# * Redistribution and use in source and binary forms, with or without
+# * modification, are permitted provided that the following conditions are met:
+# *	 * Redistributions of source code must retain the above copyright
+# *	   notice, this list of conditions and the following disclaimer.
+# *	 * Redistributions in binary form must reproduce the above copyright
+# *	   notice, this list of conditions and the following disclaimer in the
+# *	   documentation and/or other materials provided with the distribution.
+# *	 * Neither the name of the NVIDIA CORPORATION nor the
+# *	   names of its contributors may be used to endorse or promote products
+# *	   derived from this software without specific prior written permission.
+# * 
+# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+# * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *
+#******************************************************************************/
+
+
+#-------------------------------------------------------------------------------
+#
+# Makefile usage
+#
+# make <target> [sm=<XXX,...>] [cdp=<0|1>] [force32=<0|1>] [abi=<0|1>] [open64=<0|1>] [verbose=<0|1>] [keep=<0|1>] [quicktest=<0|1>] [quickertest=<0|1>]
+#
+#-------------------------------------------------------------------------------
+
+include ../common.mk 
+ 
+#-------------------------------------------------------------------------------
+# Commandline Options
+#-------------------------------------------------------------------------------
+
+# Testing mode option (quick/thorough)
+ifeq ($(quickertest), 1)
+	NVCCFLAGS += -DQUICKER_TEST
+	TEST_SUFFIX = quicker
+else ifeq ($(quicktest), 1)
+	NVCCFLAGS += -DQUICK_TEST
+	TEST_SUFFIX = quick
+else 
+	TEST_SUFFIX = thorough
+	NPPI = 
+endif
+
+
+# CUDA memcheck (enabled by default) 
+ifeq ($(memcheck), 0)
+	MEMCHECK = 
+else 
+	MEMCHECK = cuda-memcheck
+endif
+
+
+#-------------------------------------------------------------------------------
+# Compiler and compilation platform
+#-------------------------------------------------------------------------------
+
+# Includes
+INC += -I$(CUB_DIR) -I$(CUB_DIR)test 
+
+# Suffix to append to each binary
+SUFFIX = $(BIN_SUFFIX)_$(TEST_SUFFIX)
+
+# Define test arch
+DEFINES += -DTEST_ARCH=$(TEST_ARCH)
+
+
+#-------------------------------------------------------------------------------
+# Dependency Lists
+#-------------------------------------------------------------------------------
+
+rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
+
+DEPS =				$(CUB_DEPS) \
+					$(CUB_DIR)test/Makefile \
+					$(CUB_DIR)test/test_util.h \
+					$(CUB_DIR)test/mersenne.h \
+
+BLOCK_REDUCE = 		test_block_reduce_raking \
+	 				test_block_reduce_warp_reductions		
+
+
+BLOCK_SCAN = 		test_block_scan_raking \
+	 				test_block_scan_raking_memoize \
+	 				test_block_scan_warp_scans		
+
+
+BLOCK_RADIX_SORT = 	test_block_radix_sort_keys \
+	 				test_block_radix_sort_pairs	
+
+DEVICE_RADIX_SORT = 	test_device_radix_sort \
+	 					test_device_radix_sort_segmented	
+		
+ALL = 				link \
+	 				test_iterator \
+	 				test_allocator \
+	 				test_warp_scan \
+	 				test_warp_reduce \
+	 				$(BLOCK_REDUCE) \
+	 				$(BLOCK_SCAN) \
+	 				$(BLOCK_RADIX_SORT) \
+	 				test_block_load_store \
+	 				test_block_histogram \
+				 	test_device_reduce \
+			 		test_device_histogram \
+			 		test_device_scan \
+	 				$(DEVICE_RADIX_SORT) \
+					test_device_reduce_by_key\
+					test_device_run_length_encode\
+		 			test_device_select_unique \
+					test_device_select_if 
+		
+#	 	test_grid_barrier \		fails on sm110
+#	 	test_device_seg_reduce
+		
+
+
+#-------------------------------------------------------------------------------
+# make default
+#-------------------------------------------------------------------------------
+
+default:
+
+
+#-------------------------------------------------------------------------------
+# make clean
+#-------------------------------------------------------------------------------
+
+clean :
+	rm -f bin/*$(CPU_ARCH_SUFFIX)* 
+	rm -f *.i* *.cubin *.cu.c *.cudafe* *.fatbin.c *.ptx *.hash *.cu.cpp *.o
+
+
+#-------------------------------------------------------------------------------
+# make all
+#-------------------------------------------------------------------------------
+
+all : $(ALL)
+
+
+#-------------------------------------------------------------------------------
+# make run
+#-------------------------------------------------------------------------------
+
+run : 
+	for i in $(ALL); do $(MEMCHECK) ./bin/$${i}_$(SUFFIX) --device=$(device) || exit 1; done
+
+run_block_reduce : 
+	for i in $(BLOCK_REDUCE); do $(MEMCHECK) ./bin/$${i}_$(SUFFIX) --device=$(device) || exit 1; done
+
+run_block_scan : 
+	for i in $(BLOCK_SCAN); do $(MEMCHECK) ./bin/$${i}_$(SUFFIX) --device=$(device) || exit 1; done
+
+run_block_radix_sort : 
+	for i in $(BLOCK_RADIX_SORT); do $(MEMCHECK) ./bin/$${i}_$(SUFFIX) --device=$(device) || exit 1; done
+
+run_device_radix_sort : 
+	for i in $(DEVICE_RADIX_SORT); do $(MEMCHECK) ./bin/$${i}_$(SUFFIX) --device=$(device) || exit 1; done
+
+
+#-------------------------------------------------------------------------------
+# make link
+#-------------------------------------------------------------------------------
+
+link : bin/link_$(SUFFIX)
+
+bin/link_$(SUFFIX) : link_a.cu link_b.cu link_main.cpp $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(DEFINES) $(SM_TARGETS) link_a.cu -c -o bin/link_a.obj
+	$(NVCC) $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(DEFINES) $(SM_TARGETS) link_b.cu -c -o bin/link_b.obj
+	$(NVCC) $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(DEFINES) $(SM_TARGETS) link_main.cpp bin/link_a.obj bin/link_b.obj -o bin/link_$(SUFFIX)
+
+
+#-------------------------------------------------------------------------------
+# make test_iterator 
+#-------------------------------------------------------------------------------
+
+test_iterator: bin/test_iterator_$(SUFFIX)
+
+bin/test_iterator_$(SUFFIX) : test_iterator.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_iterator_$(SUFFIX) test_iterator.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_allocator 
+#-------------------------------------------------------------------------------
+
+test_allocator: bin/test_allocator_$(SUFFIX)
+
+bin/test_allocator_$(SUFFIX) : test_allocator.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_allocator_$(SUFFIX) test_allocator.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+	
+	
+#-------------------------------------------------------------------------------
+# make test_grid_barrier 
+#-------------------------------------------------------------------------------
+
+test_grid_barrier: bin/test_grid_barrier_$(SUFFIX)
+
+bin/test_grid_barrier_$(SUFFIX) : test_grid_barrier.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_grid_barrier_$(SUFFIX) test_grid_barrier.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3  
+	
+
+#-------------------------------------------------------------------------------
+# make test_warp_scan 
+#-------------------------------------------------------------------------------
+
+test_warp_scan: bin/test_warp_scan_$(SUFFIX)
+
+bin/test_warp_scan_$(SUFFIX) : test_warp_scan.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_warp_scan_$(SUFFIX) test_warp_scan.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3  
+
+
+#-------------------------------------------------------------------------------
+# make test_warp_reduce 
+#-------------------------------------------------------------------------------
+
+test_warp_reduce: bin/test_warp_reduce_$(SUFFIX)
+
+bin/test_warp_reduce_$(SUFFIX) : test_warp_reduce.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_warp_reduce_$(SUFFIX) test_warp_reduce.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3  
+
+
+#-------------------------------------------------------------------------------
+# make test_block_reduce_raking
+#-------------------------------------------------------------------------------
+
+test_block_reduce_raking: bin/test_block_reduce_raking_$(SUFFIX)
+
+bin/test_block_reduce_raking_$(SUFFIX) : test_block_reduce.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) -DTEST_RAKING $(SM_TARGETS) -o bin/test_block_reduce_raking_$(SUFFIX) test_block_reduce.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3  
+
+
+#-------------------------------------------------------------------------------
+# make test_block_reduce_warp_reductions 
+#-------------------------------------------------------------------------------
+
+test_block_reduce_warp_reductions: bin/test_block_reduce_warp_reductions_$(SUFFIX)
+
+bin/test_block_reduce_warp_reductions_$(SUFFIX) : test_block_reduce.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) -DTEST_WARP_REDUCTIONS $(SM_TARGETS) -o bin/test_block_reduce_warp_reductions_$(SUFFIX) test_block_reduce.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3  
+
+
+#-------------------------------------------------------------------------------
+# make test_block_reduce 
+#-------------------------------------------------------------------------------
+
+test_block_reduce: $(BLOCK_REDUCE)
+
+
+#-------------------------------------------------------------------------------
+# make test_block_scan_raking
+#-------------------------------------------------------------------------------
+
+test_block_scan_raking: bin/test_block_scan_raking_$(SUFFIX)
+
+bin/test_block_scan_raking_$(SUFFIX) : test_block_scan.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) -DTEST_RAKING $(SM_TARGETS) -o bin/test_block_scan_raking_$(SUFFIX) test_block_scan.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3  
+
+
+#-------------------------------------------------------------------------------
+# make test_block_scan_raking_memoize
+#-------------------------------------------------------------------------------
+
+test_block_scan_raking_memoize: bin/test_block_scan_raking_memoize_$(SUFFIX)
+
+bin/test_block_scan_raking_memoize_$(SUFFIX) : test_block_scan.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) -DTEST_RAKING_MEMOIZE $(SM_TARGETS) -o bin/test_block_scan_raking_memoize_$(SUFFIX) test_block_scan.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3  
+
+
+#-------------------------------------------------------------------------------
+# make test_block_scan_warp_scans
+#-------------------------------------------------------------------------------
+
+test_block_scan_warp_scans: bin/test_block_scan_warp_scans_$(SUFFIX)
+
+bin/test_block_scan_warp_scans_$(SUFFIX) : test_block_scan.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) -DTEST_WARP_SCANS $(SM_TARGETS) -o bin/test_block_scan_warp_scans_$(SUFFIX) test_block_scan.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3  
+
+
+#-------------------------------------------------------------------------------
+# make test_block_scan 
+#-------------------------------------------------------------------------------
+
+test_block_scan: $(BLOCK_SCAN)
+
+
+#-------------------------------------------------------------------------------
+# make test_block_load_store 
+#-------------------------------------------------------------------------------
+
+test_block_load_store: bin/test_block_load_store_$(SUFFIX)
+
+bin/test_block_load_store_$(SUFFIX) : test_block_load_store.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_block_load_store_$(SUFFIX) test_block_load_store.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+	
+	
+#-------------------------------------------------------------------------------
+# make test_block_radix_sort_keys 
+#-------------------------------------------------------------------------------
+
+test_block_radix_sort_keys: bin/test_block_radix_sort_keys_$(SUFFIX)
+
+bin/test_block_radix_sort_keys_$(SUFFIX) : test_block_radix_sort.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) -DTEST_KEYS_ONLY $(SM_TARGETS) -o bin/test_block_radix_sort_keys_$(SUFFIX) test_block_radix_sort.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+#-------------------------------------------------------------------------------
+# make test_block_radix_sort_pairs 
+#-------------------------------------------------------------------------------
+
+test_block_radix_sort_pairs: bin/test_block_radix_sort_pairs_$(SUFFIX)
+
+bin/test_block_radix_sort_pairs_$(SUFFIX) : test_block_radix_sort.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_block_radix_sort_pairs_$(SUFFIX) test_block_radix_sort.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_block_radix_sort
+#-------------------------------------------------------------------------------
+
+test_block_radix_sort : $(BLOCK_RADIX_SORT)
+
+
+#-------------------------------------------------------------------------------
+# make test_block_histogram 
+#-------------------------------------------------------------------------------
+
+test_block_histogram: bin/test_block_histogram_$(SUFFIX)
+
+bin/test_block_histogram_$(SUFFIX) : test_block_histogram.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_block_histogram_$(SUFFIX) test_block_histogram.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_device_reduce
+#-------------------------------------------------------------------------------
+
+test_device_reduce: bin/test_device_reduce_$(SUFFIX)
+
+bin/test_device_reduce_$(SUFFIX) : test_device_reduce.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_reduce_$(SUFFIX) test_device_reduce.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_device_histogram
+#-------------------------------------------------------------------------------
+
+test_device_histogram: bin/test_device_histogram_$(SUFFIX)
+
+bin/test_device_histogram_$(SUFFIX) : test_device_histogram.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_histogram_$(SUFFIX) test_device_histogram.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) $(NPPI) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_device_scan
+#-------------------------------------------------------------------------------
+
+test_device_scan: bin/test_device_scan_$(SUFFIX)
+
+bin/test_device_scan_$(SUFFIX) : test_device_scan.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_scan_$(SUFFIX) test_device_scan.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_device_radix_sort
+#-------------------------------------------------------------------------------
+
+test_device_radix_sort: bin/test_device_radix_sort_$(SUFFIX)
+
+bin/test_device_radix_sort_$(SUFFIX) : test_device_radix_sort.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_radix_sort_$(SUFFIX) test_device_radix_sort.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_device_radix_sort_segmented
+#-------------------------------------------------------------------------------
+
+test_device_radix_sort_segmented: bin/test_device_radix_sort_segmented_$(SUFFIX)
+
+bin/test_device_radix_sort_segmented_$(SUFFIX) : test_device_radix_sort.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) -DSEGMENTED_SORT $(SM_TARGETS) -o bin/test_device_radix_sort_segmented_$(SUFFIX) test_device_radix_sort.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_device_select_unique
+#-------------------------------------------------------------------------------
+
+test_device_select_unique: bin/test_device_select_unique_$(SUFFIX)
+
+bin/test_device_select_unique_$(SUFFIX) : test_device_select_unique.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_select_unique_$(SUFFIX) test_device_select_unique.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+#-------------------------------------------------------------------------------
+# make test_device_select_if
+#-------------------------------------------------------------------------------
+
+test_device_select_if: bin/test_device_select_if_$(SUFFIX)
+
+bin/test_device_select_if_$(SUFFIX) : test_device_select_if.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_select_if_$(SUFFIX) test_device_select_if.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+#-------------------------------------------------------------------------------
+# make test_device_reduce_by_key
+#-------------------------------------------------------------------------------
+
+test_device_reduce_by_key: bin/test_device_reduce_by_key_$(SUFFIX)
+
+bin/test_device_reduce_by_key_$(SUFFIX) : test_device_reduce_by_key.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_reduce_by_key_$(SUFFIX) test_device_reduce_by_key.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+#-------------------------------------------------------------------------------
+# make test_device_run_length_encode
+#-------------------------------------------------------------------------------
+
+test_device_run_length_encode: bin/test_device_run_length_encode_$(SUFFIX)
+
+bin/test_device_run_length_encode_$(SUFFIX) : test_device_run_length_encode.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_run_length_encode_$(SUFFIX) test_device_run_length_encode.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
+
+
+#-------------------------------------------------------------------------------
+# make test_device_seg_reduce
+#-------------------------------------------------------------------------------
+#
+#test_device_seg_reduce: bin/test_device_seg_reduce_$(SUFFIX)
+#
+#bin/test_device_seg_reduce_$(SUFFIX) : test_device_seg_reduce.cu $(DEPS)
+#	mkdir -p bin
+#	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/test_device_seg_reduce_$(SUFFIX) test_device_seg_reduce.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3
+
+
diff --git a/third_party/cub/test/half.h b/third_party/cub/test/half.h
new file mode 100644
index 0000000..e4fd65c
--- /dev/null
+++ b/third_party/cub/test/half.h
@@ -0,0 +1,298 @@
+/******************************************************************************
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are not permitted.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+#pragma once
+
+/**
+ * \file
+ * Utilities for interacting with the opaque CUDA __half type
+ */
+
+#include <stdint.h>
+#include <cuda_fp16.h>
+#include <iosfwd>
+
+#include <cub/util_type.cuh>
+
+
+/******************************************************************************
+ * half_t
+ ******************************************************************************/
+
+/**
+ * Host-based fp16 data type compatible and convertible with __half
+ */
+struct half_t
+{
+    uint16_t __x;
+
+    /// Constructor from __half
+    __host__ __device__ __forceinline__
+    half_t(const __half &other)
+    {
+        __x = reinterpret_cast<const uint16_t&>(other);
+    }
+
+    /// Constructor from integer
+    __host__ __device__ __forceinline__
+    half_t(int a)
+    {
+        *this = half_t(float(a));
+    }
+
+    /// Default constructor
+    __host__ __device__ __forceinline__
+    half_t() : __x(0)
+    {}
+
+    /// Constructor from float
+    __host__ __device__ __forceinline__
+    half_t(float a)
+    {
+        // Stolen from Norbert Juffa
+        uint32_t ia = *reinterpret_cast<uint32_t*>(&a);
+        uint16_t ir;
+
+        ir = (ia >> 16) & 0x8000;
+
+        if ((ia & 0x7f800000) == 0x7f800000)
+        {
+            if ((ia & 0x7fffffff) == 0x7f800000)
+            {
+                ir |= 0x7c00; /* infinity */
+            }
+            else
+            {
+                ir = 0x7fff; /* canonical NaN */
+            }
+        }
+        else if ((ia & 0x7f800000) >= 0x33000000)
+        {
+            int32_t shift = (int32_t) ((ia >> 23) & 0xff) - 127;
+            if (shift > 15)
+            {
+                ir |= 0x7c00; /* infinity */
+            }
+            else
+            {
+                ia = (ia & 0x007fffff) | 0x00800000; /* extract mantissa */
+                if (shift < -14)
+                { /* denormal */
+                    ir |= ia >> (-1 - shift);
+                    ia = ia << (32 - (-1 - shift));
+                }
+                else
+                { /* normal */
+                    ir |= ia >> (24 - 11);
+                    ia = ia << (32 - (24 - 11));
+                    ir = ir + ((14 + shift) << 10);
+                }
+                /* IEEE-754 round to nearest of even */
+                if ((ia > 0x80000000) || ((ia == 0x80000000) && (ir & 1)))
+                {
+                    ir++;
+                }
+            }
+        }
+
+        this->__x = ir;
+    }
+
+    /// Cast to __half
+    __host__ __device__ __forceinline__
+    operator __half() const
+    {
+        return reinterpret_cast<const __half&>(__x);
+    }
+
+    /// Cast to float
+    __host__ __device__ __forceinline__
+    operator float() const
+    {
+        // Stolen from Andrew Kerr
+
+        int sign        = ((this->__x >> 15) & 1);
+        int exp         = ((this->__x >> 10) & 0x1f);
+        int mantissa    = (this->__x & 0x3ff);
+        uint32_t f      = 0;
+
+        if (exp > 0 && exp < 31)
+        {
+            // normal
+            exp += 112;
+            f = (sign << 31) | (exp << 23) | (mantissa << 13);
+        }
+        else if (exp == 0)
+        {
+            if (mantissa)
+            {
+                // subnormal
+                exp += 113;
+                while ((mantissa & (1 << 10)) == 0)
+                {
+                    mantissa <<= 1;
+                    exp--;
+                }
+                mantissa &= 0x3ff;
+                f = (sign << 31) | (exp << 23) | (mantissa << 13);
+            }
+            else if (sign)
+            {
+                f = 0x80000000; // negative zero
+            }
+            else
+            {
+                f = 0x0;        // zero
+            }
+        }
+        else if (exp == 31)
+        {
+            if (mantissa)
+            {
+                f = 0x7fffffff;     // not a number
+            }
+            else
+            {
+                f = (0xff << 23) | (sign << 31);    //  inf
+            }
+        }
+        return *reinterpret_cast<float const *>(&f);
+    }
+
+
+    /// Get raw storage
+    __host__ __device__ __forceinline__
+    uint16_t raw()
+    {
+        return this->__x;
+    }
+
+    /// Equality
+    __host__ __device__ __forceinline__
+    bool operator ==(const half_t &other)
+    {
+        return (this->__x == other.__x);
+    }
+
+    /// Inequality
+    __host__ __device__ __forceinline__
+    bool operator !=(const half_t &other)
+    {
+        return (this->__x != other.__x);
+    }
+
+    /// Assignment by sum
+    __host__ __device__ __forceinline__
+    half_t& operator +=(const half_t &rhs)
+    {
+        *this = half_t(float(*this) + float(rhs));
+        return *this;
+    }
+
+    /// Multiply
+    __host__ __device__ __forceinline__
+    half_t operator*(const half_t &other)
+    {
+        return half_t(float(*this) * float(other));
+    }
+
+    /// Add
+    __host__ __device__ __forceinline__
+    half_t operator+(const half_t &other)
+    {
+        return half_t(float(*this) + float(other));
+    }
+
+    /// Less-than
+    __host__ __device__ __forceinline__
+    bool operator<(const half_t &other) const
+    {
+        return float(*this) < float(other);
+    }
+
+    /// Less-than-equal
+    __host__ __device__ __forceinline__
+    bool operator<=(const half_t &other) const
+    {
+        return float(*this) <= float(other);
+    }
+
+    /// Greater-than
+    __host__ __device__ __forceinline__
+    bool operator>(const half_t &other) const
+    {
+        return float(*this) > float(other);
+    }
+
+    /// Greater-than-equal
+    __host__ __device__ __forceinline__
+    bool operator>=(const half_t &other) const
+    {
+        return float(*this) >= float(other);
+    }
+
+    /// numeric_traits<half_t>::max
+    __host__ __device__ __forceinline__
+    static half_t max() {
+        uint16_t max_word = 0x7BFF;
+        return reinterpret_cast<half_t&>(max_word);
+    }
+
+    /// numeric_traits<half_t>::lowest
+    __host__ __device__ __forceinline__
+    static half_t lowest() {
+        uint16_t lowest_word = 0xFBFF;
+        return reinterpret_cast<half_t&>(lowest_word);
+    }
+};
+
+
+/******************************************************************************
+ * I/O stream overloads
+ ******************************************************************************/
+
+/// Insert formatted \p half_t into the output stream
+std::ostream& operator<<(std::ostream &out, const half_t &x)
+{
+    out << (float)x;
+    return out;
+}
+
+
+/// Insert formatted \p __half into the output stream
+std::ostream& operator<<(std::ostream &out, const __half &x)
+{
+    return out << half_t(x);
+}
+
+
+/******************************************************************************
+ * Traits overloads
+ ******************************************************************************/
+
+template <>
+struct cub::FpLimits<half_t>
+{
+    static __host__ __device__ __forceinline__ half_t Max() { return half_t::max(); }
+
+    static __host__ __device__ __forceinline__ half_t Lowest() { return half_t::lowest(); }
+};
+
+template <> struct cub::NumericTraits<half_t> : cub::BaseTraits<FLOATING_POINT, true, false, unsigned short, half_t> {};
+
diff --git a/third_party/cub/test/link_a.cu b/third_party/cub/test/link_a.cu
new file mode 100644
index 0000000..8a9b19f
--- /dev/null
+++ b/third_party/cub/test/link_a.cu
@@ -0,0 +1,11 @@
+#include <cub/cub.cuh>
+
+void a()
+{
+    printf("a() called\n");
+
+    cub::DoubleBuffer<unsigned int>     d_keys;
+    cub::DoubleBuffer<cub::NullType>    d_values;
+    size_t                              temp_storage_bytes = 0;
+    cub::DeviceRadixSort::SortPairs(NULL, temp_storage_bytes, d_keys, d_values, 1024);
+}
diff --git a/third_party/cub/test/link_b.cu b/third_party/cub/test/link_b.cu
new file mode 100644
index 0000000..a19ec40
--- /dev/null
+++ b/third_party/cub/test/link_b.cu
@@ -0,0 +1,11 @@
+#include <cub/cub.cuh>
+
+void b()
+{
+    printf("b() called\n");
+
+    cub::DoubleBuffer<unsigned int>     d_keys;
+    cub::DoubleBuffer<cub::NullType>    d_values;
+    size_t                              temp_storage_bytes = 0;
+    cub::DeviceRadixSort::SortPairs(NULL, temp_storage_bytes, d_keys, d_values, 1024);
+}
diff --git a/third_party/cub/test/link_main.cpp b/third_party/cub/test/link_main.cpp
new file mode 100644
index 0000000..ef677ee
--- /dev/null
+++ b/third_party/cub/test/link_main.cpp
@@ -0,0 +1,10 @@
+#include <stdio.h>
+
+extern void a();
+extern void b();
+
+int main()
+{
+    printf("hello world\n");
+    return 0;
+}
diff --git a/third_party/cub/test/mersenne.h b/third_party/cub/test/mersenne.h
new file mode 100644
index 0000000..76aae80
--- /dev/null
+++ b/third_party/cub/test/mersenne.h
@@ -0,0 +1,160 @@
+/*
+ A C-program for MT19937, with initialization improved 2002/1/26.
+ Coded by Takuji Nishimura and Makoto Matsumoto.
+
+ Before using, initialize the state by using init_genrand(seed)
+ or init_by_array(init_key, key_length).
+
+ Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ 3. The names of its contributors may not be used to endorse or promote
+ products derived from this software without specific prior written
+ permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+ Any feedback is very welcome.
+ http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html
+ email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space)
+ */
+
+#include <stdio.h>
+
+namespace mersenne {
+
+/* Period parameters */
+const unsigned int N          = 624;
+const unsigned int M          = 397;
+const unsigned int MATRIX_A   = 0x9908b0df; /* constant vector a */
+const unsigned int UPPER_MASK = 0x80000000; /* most significant w-r bits */
+const unsigned int LOWER_MASK = 0x7fffffff; /* least significant r bits */
+
+static unsigned int mt[N];  /* the array for the state vector  */
+static int mti = N + 1;     /* mti==N+1 means mt[N] is not initialized */
+
+/* initializes mt[N] with a seed */
+void init_genrand(unsigned int s)
+{
+    mt[0] = s & 0xffffffff;
+    for (mti = 1; mti < N; mti++)
+    {
+        mt[mti] = (1812433253 * (mt[mti - 1] ^ (mt[mti - 1] >> 30)) + mti);
+
+        /* See Knuth TAOCP Vol2. 3rd Ed. P.106 for mtiplier. */
+        /* In the previous versions, MSBs of the seed affect   */
+        /* only MSBs of the array mt[].                        */
+        /* 2002/01/09 modified by Makoto Matsumoto             */
+
+        mt[mti] &= 0xffffffff;
+        /* for >32 bit machines */
+    }
+}
+
+/* initialize by an array with array-length */
+/* init_key is the array for initializing keys */
+/* key_length is its length */
+/* slight change for C++, 2004/2/26 */
+void init_by_array(unsigned int init_key[], int key_length)
+{
+    int i, j, k;
+    init_genrand(19650218);
+    i = 1;
+    j = 0;
+    k = (N > key_length ? N : key_length);
+    for (; k; k--)
+    {
+        mt[i] = (mt[i] ^ ((mt[i - 1] ^ (mt[i - 1] >> 30)) * 1664525))
+            + init_key[j] + j;  /* non linear */
+        mt[i] &= 0xffffffff;    /* for WORDSIZE > 32 machines */
+        i++;
+        j++;
+        if (i >= N)
+        {
+            mt[0] = mt[N - 1];
+            i = 1;
+        }
+        if (j >= key_length) j = 0;
+    }
+    for (k = N - 1; k; k--)
+    {
+        mt[i] = (mt[i] ^ ((mt[i - 1] ^ (mt[i - 1] >> 30)) * 1566083941)) - i; /* non linear */
+        mt[i] &= 0xffffffff; /* for WORDSIZE > 32 machines */
+        i++;
+        if (i >= N)
+        {
+            mt[0] = mt[N - 1];
+            i = 1;
+        }
+    }
+
+    mt[0] = 0x80000000; /* MSB is 1; assuring non-zero initial array */
+}
+
+/* generates a random number on [0,0xffffffff]-interval */
+unsigned int genrand_int32(void)
+{
+    unsigned int y;
+    static unsigned int mag01[2] = { 0x0, MATRIX_A };
+
+    /* mag01[x] = x * MATRIX_A  for x=0,1 */
+
+    if (mti >= N)
+    { /* generate N words at one time */
+        int kk;
+
+        if (mti == N + 1) /* if init_genrand() has not been called, */
+        init_genrand(5489); /* a defat initial seed is used */
+
+        for (kk = 0; kk < N - M; kk++)
+        {
+            y = (mt[kk] & UPPER_MASK) | (mt[kk + 1] & LOWER_MASK);
+            mt[kk] = mt[kk + M] ^ (y >> 1) ^ mag01[y & 0x1];
+        }
+        for (; kk < N - 1; kk++)
+        {
+            y = (mt[kk] & UPPER_MASK) | (mt[kk + 1] & LOWER_MASK);
+            mt[kk] = mt[kk + (M - N)] ^ (y >> 1) ^ mag01[y & 0x1];
+        }
+        y = (mt[N - 1] & UPPER_MASK) | (mt[0] & LOWER_MASK);
+        mt[N - 1] = mt[M - 1] ^ (y >> 1) ^ mag01[y & 0x1];
+
+        mti = 0;
+    }
+
+    y = mt[mti++];
+
+    /* Tempering */
+    y ^= (y >> 11);
+    y ^= (y << 7) & 0x9d2c5680;
+    y ^= (y << 15) & 0xefc60000;
+    y ^= (y >> 18);
+
+    return y;
+}
+
+
+
+} // namespace mersenne
diff --git a/third_party/cub/test/test_allocator.cu b/third_party/cub/test/test_allocator.cu
new file mode 100644
index 0000000..f771435
--- /dev/null
+++ b/third_party/cub/test/test_allocator.cu
@@ -0,0 +1,459 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test evaluation for caching allocator of device memory
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/util_allocator.cuh>
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>]"
+            "[--bytes=<timing bytes>]"
+            "[--i=<timing iterations>]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+#if (CUB_PTX_ARCH == 0)
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get number of GPUs and current GPU
+    int num_gpus;
+    int initial_gpu;
+    int timing_iterations           = 10000;
+    int timing_bytes                = 1024 * 1024;
+
+    if (CubDebug(cudaGetDeviceCount(&num_gpus))) exit(1);
+    if (CubDebug(cudaGetDevice(&initial_gpu))) exit(1);
+    args.GetCmdLineArgument("i", timing_iterations);
+    args.GetCmdLineArgument("bytes", timing_bytes);
+
+    // Create default allocator (caches up to 6MB in device allocations per GPU)
+    CachingDeviceAllocator allocator;
+    allocator.debug = true;
+
+    printf("Running single-gpu tests...\n"); fflush(stdout);
+
+    //
+    // Test0
+    //
+
+    // Create a new stream
+    cudaStream_t other_stream;
+    CubDebugExit(cudaStreamCreate(&other_stream));
+
+    // Allocate 999 bytes on the current gpu in stream0
+    char *d_999B_stream0_a;
+    char *d_999B_stream0_b;
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_999B_stream0_a, 999, 0));
+
+    // Run some big kernel in stream 0
+    EmptyKernel<void><<<32000, 512, 1024 * 8, 0>>>();
+
+    // Free d_999B_stream0_a
+    CubDebugExit(allocator.DeviceFree(d_999B_stream0_a));
+
+    // Allocate another 999 bytes in stream 0
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_999B_stream0_b, 999, 0));
+
+    // Check that that we have 1 live block on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 1);
+
+    // Check that that we have no cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 0);
+
+    // Run some big kernel in stream 0
+    EmptyKernel<void><<<32000, 512, 1024 * 8, 0>>>();
+
+    // Free d_999B_stream0_b
+    CubDebugExit(allocator.DeviceFree(d_999B_stream0_b));
+
+    // Allocate 999 bytes on the current gpu in other_stream
+    char *d_999B_stream_other_a;
+    char *d_999B_stream_other_b;
+    allocator.DeviceAllocate((void **) &d_999B_stream_other_a, 999, other_stream);
+
+    // Check that that we have 1 live blocks on the initial GPU (that we allocated a new one because d_999B_stream0_b is only available for stream 0 until it becomes idle)
+    AssertEquals(allocator.live_blocks.size(), 1);
+
+    // Check that that we have one cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 1);
+
+    // Run some big kernel in other_stream
+    EmptyKernel<void><<<32000, 512, 1024 * 8, other_stream>>>();
+
+    // Free d_999B_stream_other
+    CubDebugExit(allocator.DeviceFree(d_999B_stream_other_a));
+
+    // Check that we can now use both allocations in stream 0 after synchronizing the device
+    CubDebugExit(cudaDeviceSynchronize());
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_999B_stream0_a, 999, 0));
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_999B_stream0_b, 999, 0));
+
+    // Check that that we have 2 live blocks on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 2);
+
+    // Check that that we have no cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 0);
+
+    // Free d_999B_stream0_a and d_999B_stream0_b
+    CubDebugExit(allocator.DeviceFree(d_999B_stream0_a));
+    CubDebugExit(allocator.DeviceFree(d_999B_stream0_b));
+
+    // Check that we can now use both allocations in other_stream
+    CubDebugExit(cudaDeviceSynchronize());
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_999B_stream_other_a, 999, other_stream));
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_999B_stream_other_b, 999, other_stream));
+
+    // Check that that we have 2 live blocks on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 2);
+
+    // Check that that we have no cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 0);
+
+    // Run some big kernel in other_stream
+    EmptyKernel<void><<<32000, 512, 1024 * 8, other_stream>>>();
+
+    // Free d_999B_stream_other_a and d_999B_stream_other_b
+    CubDebugExit(allocator.DeviceFree(d_999B_stream_other_a));
+    CubDebugExit(allocator.DeviceFree(d_999B_stream_other_b));
+
+    // Check that we can now use both allocations in stream 0 after synchronizing the device and destroying the other stream
+    CubDebugExit(cudaDeviceSynchronize());
+    CubDebugExit(cudaStreamDestroy(other_stream));
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_999B_stream0_a, 999, 0));
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_999B_stream0_b, 999, 0));
+
+    // Check that that we have 2 live blocks on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 2);
+
+    // Check that that we have no cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 0);
+
+    // Free d_999B_stream0_a and d_999B_stream0_b
+    CubDebugExit(allocator.DeviceFree(d_999B_stream0_a));
+    CubDebugExit(allocator.DeviceFree(d_999B_stream0_b));
+
+    // Free all cached
+    CubDebugExit(allocator.FreeAllCached());
+
+    //
+    // Test1
+    //
+
+    // Allocate 5 bytes on the current gpu
+    char *d_5B;
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_5B, 5));
+
+    // Check that that we have zero free bytes cached on the initial GPU
+    AssertEquals(allocator.cached_bytes[initial_gpu].free, 0);
+
+    // Check that that we have 1 live block on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 1);
+
+    //
+    // Test2
+    //
+
+    // Allocate 4096 bytes on the current gpu
+    char *d_4096B;
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_4096B, 4096));
+
+    // Check that that we have 2 live blocks on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 2);
+
+    //
+    // Test3
+    //
+
+    // DeviceFree d_5B
+    CubDebugExit(allocator.DeviceFree(d_5B));
+
+    // Check that that we have min_bin_bytes free bytes cached on the initial gpu
+    AssertEquals(allocator.cached_bytes[initial_gpu].free, allocator.min_bin_bytes);
+
+    // Check that that we have 1 live block on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 1);
+
+    // Check that that we have 1 cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 1);
+
+    //
+    // Test4
+    //
+
+    // DeviceFree d_4096B
+    CubDebugExit(allocator.DeviceFree(d_4096B));
+
+    // Check that that we have the 4096 + min_bin free bytes cached on the initial gpu
+    AssertEquals(allocator.cached_bytes[initial_gpu].free, allocator.min_bin_bytes + 4096);
+
+    // Check that that we have 0 live block on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 0);
+
+    // Check that that we have 2 cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 2);
+
+    //
+    // Test5
+    //
+
+    // Allocate 768 bytes on the current gpu
+    char *d_768B;
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_768B, 768));
+
+    // Check that that we have the min_bin free bytes cached on the initial gpu (4096 was reused)
+    AssertEquals(allocator.cached_bytes[initial_gpu].free, allocator.min_bin_bytes);
+
+    // Check that that we have 1 live block on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 1);
+
+    // Check that that we have 1 cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 1);
+
+    //
+    // Test6
+    //
+
+    // Allocate max_cached_bytes on the current gpu
+    char *d_max_cached;
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_max_cached, allocator.max_cached_bytes));
+
+    // DeviceFree d_max_cached
+    CubDebugExit(allocator.DeviceFree(d_max_cached));
+
+    // Check that that we have the min_bin free bytes cached on the initial gpu (max cached was not returned because we went over)
+    AssertEquals(allocator.cached_bytes[initial_gpu].free, allocator.min_bin_bytes);
+
+    // Check that that we have 1 live block on the initial GPU
+    AssertEquals(allocator.live_blocks.size(), 1);
+
+    // Check that that we still have 1 cached block on the initial GPU
+    AssertEquals(allocator.cached_blocks.size(), 1);
+
+    //
+    // Test7
+    //
+
+    // Free all cached blocks on all GPUs
+    CubDebugExit(allocator.FreeAllCached());
+
+    // Check that that we have 0 bytes cached on the initial GPU
+    AssertEquals(allocator.cached_bytes[initial_gpu].free, 0);
+
+    // Check that that we have 0 cached blocks across all GPUs
+    AssertEquals(allocator.cached_blocks.size(), 0);
+
+    // Check that that still we have 1 live block across all GPUs
+    AssertEquals(allocator.live_blocks.size(), 1);
+
+    //
+    // Test8
+    //
+
+    // Allocate max cached bytes + 1 on the current gpu
+    char *d_max_cached_plus;
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_max_cached_plus, allocator.max_cached_bytes + 1));
+
+    // DeviceFree max cached bytes
+    CubDebugExit(allocator.DeviceFree(d_max_cached_plus));
+
+    // DeviceFree d_768B
+    CubDebugExit(allocator.DeviceFree(d_768B));
+
+    unsigned int power;
+    size_t rounded_bytes;
+    allocator.NearestPowerOf(power, rounded_bytes, allocator.bin_growth, 768);
+
+    // Check that that we have 4096 free bytes cached on the initial gpu
+    AssertEquals(allocator.cached_bytes[initial_gpu].free, rounded_bytes);
+
+    // Check that that we have 1 cached blocks across all GPUs
+    AssertEquals(allocator.cached_blocks.size(), 1);
+
+    // Check that that still we have 0 live block across all GPUs
+    AssertEquals(allocator.live_blocks.size(), 0);
+
+#ifndef CUB_CDP
+    // BUG: find out why these tests fail when one GPU is CDP compliant and the other is not
+
+    if (num_gpus > 1)
+    {
+        printf("\nRunning multi-gpu tests...\n"); fflush(stdout);
+
+        //
+        // Test9
+        //
+
+        // Allocate 768 bytes on the next gpu
+        int next_gpu = (initial_gpu + 1) % num_gpus;
+        char *d_768B_2;
+        CubDebugExit(allocator.DeviceAllocate(next_gpu, (void **) &d_768B_2, 768));
+
+        // DeviceFree d_768B on the next gpu
+        CubDebugExit(allocator.DeviceFree(next_gpu, d_768B_2));
+
+        // Re-allocate 768 bytes on the next gpu
+        CubDebugExit(allocator.DeviceAllocate(next_gpu, (void **) &d_768B_2, 768));
+
+        // Re-free d_768B on the next gpu
+        CubDebugExit(allocator.DeviceFree(next_gpu, d_768B_2));
+
+        // Check that that we have 4096 free bytes cached on the initial gpu
+        AssertEquals(allocator.cached_bytes[initial_gpu].free, rounded_bytes);
+
+        // Check that that we have 4096 free bytes cached on the second gpu
+        AssertEquals(allocator.cached_bytes[next_gpu].free, rounded_bytes);
+
+        // Check that that we have 2 cached blocks across all GPUs
+        AssertEquals(allocator.cached_blocks.size(), 2);
+
+        // Check that that still we have 0 live block across all GPUs
+        AssertEquals(allocator.live_blocks.size(), 0);
+    }
+#endif  // CUB_CDP
+
+    //
+    // Performance
+    //
+
+    printf("\nCPU Performance (%d timing iterations, %d bytes):\n", timing_iterations, timing_bytes);
+    fflush(stdout); fflush(stderr);
+
+    // CPU performance comparisons vs cached.  Allocate and free a 1MB block 2000 times
+    CpuTimer    cpu_timer;
+    char        *d_1024MB                       = NULL;
+    allocator.debug                             = false;
+
+    // Prime the caching allocator and the kernel
+    CubDebugExit(allocator.DeviceAllocate((void **) &d_1024MB, timing_bytes));
+    CubDebugExit(allocator.DeviceFree(d_1024MB));
+    cub::EmptyKernel<void><<<1, 32>>>();
+
+    // CUDA
+    cpu_timer.Start();
+    for (int i = 0; i < timing_iterations; ++i)
+    {
+        CubDebugExit(cudaMalloc((void **) &d_1024MB, timing_bytes));
+        CubDebugExit(cudaFree(d_1024MB));
+    }
+    cpu_timer.Stop();
+    float cuda_malloc_elapsed_millis = cpu_timer.ElapsedMillis();
+
+    // CUB
+    cpu_timer.Start();
+    for (int i = 0; i < timing_iterations; ++i)
+    {
+        CubDebugExit(allocator.DeviceAllocate((void **) &d_1024MB, timing_bytes));
+        CubDebugExit(allocator.DeviceFree(d_1024MB));
+    }
+    cpu_timer.Stop();
+    float cub_calloc_elapsed_millis = cpu_timer.ElapsedMillis();
+
+    printf("\t CUB CachingDeviceAllocator allocation CPU speedup: %.2f (avg cudaMalloc %.4f ms vs. avg DeviceAllocate %.4f ms)\n",
+        cuda_malloc_elapsed_millis / cub_calloc_elapsed_millis,
+        cuda_malloc_elapsed_millis / timing_iterations,
+        cub_calloc_elapsed_millis / timing_iterations);
+
+    // GPU performance comparisons.  Allocate and free a 1MB block 2000 times
+    GpuTimer gpu_timer;
+
+    printf("\nGPU Performance (%d timing iterations, %d bytes):\n", timing_iterations, timing_bytes);
+    fflush(stdout); fflush(stderr);
+
+    // Kernel-only
+    gpu_timer.Start();
+    for (int i = 0; i < timing_iterations; ++i)
+    {
+        cub::EmptyKernel<void><<<1, 32>>>();
+    }
+    gpu_timer.Stop();
+    float cuda_empty_elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // CUDA
+    gpu_timer.Start();
+    for (int i = 0; i < timing_iterations; ++i)
+    {
+        CubDebugExit(cudaMalloc((void **) &d_1024MB, timing_bytes));
+        cub::EmptyKernel<void><<<1, 32>>>();
+        CubDebugExit(cudaFree(d_1024MB));
+    }
+    gpu_timer.Stop();
+    cuda_malloc_elapsed_millis = gpu_timer.ElapsedMillis() - cuda_empty_elapsed_millis;
+
+    // CUB
+    gpu_timer.Start();
+    for (int i = 0; i < timing_iterations; ++i)
+    {
+        CubDebugExit(allocator.DeviceAllocate((void **) &d_1024MB, timing_bytes));
+        cub::EmptyKernel<void><<<1, 32>>>();
+        CubDebugExit(allocator.DeviceFree(d_1024MB));
+    }
+    gpu_timer.Stop();
+    cub_calloc_elapsed_millis = gpu_timer.ElapsedMillis() - cuda_empty_elapsed_millis;
+
+    printf("\t CUB CachingDeviceAllocator allocation GPU speedup: %.2f (avg cudaMalloc %.4f ms vs. avg DeviceAllocate %.4f ms)\n",
+        cuda_malloc_elapsed_millis / cub_calloc_elapsed_millis,
+        cuda_malloc_elapsed_millis / timing_iterations,
+        cub_calloc_elapsed_millis / timing_iterations);
+
+
+#endif
+
+    printf("Success\n");
+
+    return 0;
+}
+
diff --git a/third_party/cub/test/test_block_histogram.cu b/third_party/cub/test/test_block_histogram.cu
new file mode 100644
index 0000000..b76466f
--- /dev/null
+++ b/third_party/cub/test/test_block_histogram.cu
@@ -0,0 +1,310 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of BlockHistogram utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <limits>
+#include <string>
+#include <typeinfo>
+
+#include <cub/block/block_histogram.cuh>
+#include <cub/block/block_load.cuh>
+#include <cub/block/block_store.cuh>
+#include <cub/util_allocator.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose           = false;
+int                     g_timing_iterations = 0;
+int                     g_repeat            = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+/**
+ * BlockHistogram test kernel.
+ */
+template <
+    int                     BINS,
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    BlockHistogramAlgorithm ALGORITHM,
+    typename                T,
+    typename                HistoCounter>
+__global__ void BlockHistogramKernel(
+    T                       *d_samples,
+    HistoCounter            *d_histogram)
+{
+    // Parameterize BlockHistogram type for our thread block
+    typedef BlockHistogram<T, BLOCK_THREADS, ITEMS_PER_THREAD, BINS, ALGORITHM> BlockHistogram;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename BlockHistogram::TempStorage temp_storage;
+
+    // Per-thread tile data
+    T data[ITEMS_PER_THREAD];
+    LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_samples, data);
+
+    // Test histo (writing directly to histogram buffer in global)
+    BlockHistogram(temp_storage).Histogram(data, d_histogram);
+}
+
+
+/**
+ * Initialize problem (and solution)
+ */
+template <
+    int             BINS,
+    typename        SampleT>
+void Initialize(
+    GenMode         gen_mode,
+    SampleT         *h_samples,
+    int             *h_histograms_linear,
+    int             num_samples)
+{
+    // Init bins
+    for (int bin = 0; bin < BINS; ++bin)
+    {
+        h_histograms_linear[bin] = 0;
+    }
+
+    if (g_verbose) printf("Samples: \n");
+
+    // Initialize interleaved channel samples and histogram them correspondingly
+    for (int i = 0; i < num_samples; ++i)
+    {
+        InitValue(gen_mode, h_samples[i], i);
+        h_samples[i] %= BINS;
+
+        if (g_verbose) std::cout << CoutCast(h_samples[i]) << ", ";
+
+        h_histograms_linear[h_samples[i]]++;
+    }
+
+    if (g_verbose) printf("\n\n");
+}
+
+
+/**
+ * Test BlockHistogram
+ */
+template <
+    typename                    SampleT,
+    int                         BINS,
+    int                         BLOCK_THREADS,
+    int                         ITEMS_PER_THREAD,
+    BlockHistogramAlgorithm     ALGORITHM>
+void Test(
+    GenMode                     gen_mode)
+{
+    int num_samples = BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    printf("cub::BlockHistogram %s %d %s samples (%dB), %d bins, %d threads, gen-mode %s\n",
+        (ALGORITHM == BLOCK_HISTO_SORT) ? "BLOCK_HISTO_SORT" : "BLOCK_HISTO_ATOMIC",
+        num_samples,
+        typeid(SampleT).name(),
+        (int) sizeof(SampleT),
+        BINS,
+        BLOCK_THREADS,
+        (gen_mode == RANDOM) ? "RANDOM" : (gen_mode == INTEGER_SEED) ? "SEQUENTIAL" : "HOMOGENOUS");
+    fflush(stdout);
+
+    // Allocate host arrays
+    SampleT         *h_samples          = new SampleT[num_samples];
+    int   *h_reference = new int[BINS];
+
+    // Initialize problem
+    Initialize<BINS>(gen_mode, h_samples, h_reference, num_samples);
+
+    // Allocate problem device arrays
+    SampleT         *d_samples = NULL;
+    int             *d_histogram = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_samples,             sizeof(SampleT) * num_samples));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_histogram,   sizeof(int) * BINS));
+
+    // Initialize/clear device arrays
+    CubDebugExit(cudaMemcpy(d_samples, h_samples, sizeof(SampleT) * num_samples, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_histogram, 0, sizeof(int) * BINS));
+
+    // Run kernel
+    BlockHistogramKernel<BINS, BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM><<<1, BLOCK_THREADS>>>(
+        d_samples,
+        d_histogram);
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults((int*) h_reference, d_histogram, BINS, g_verbose, g_verbose);
+    printf("\t%s\n\n", compare ? "FAIL" : "PASS");
+
+    // Flush any stdout/stderr
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+    fflush(stdout);
+    fflush(stderr);
+
+    // Cleanup
+    if (h_samples) delete[] h_samples;
+    if (h_reference) delete[] h_reference;
+    if (d_samples) CubDebugExit(g_allocator.DeviceFree(d_samples));
+    if (d_histogram) CubDebugExit(g_allocator.DeviceFree(d_histogram));
+
+    // Correctness asserts
+    AssertEquals(0, compare);
+}
+
+
+/**
+ * Test different sample distributions
+ */
+template <
+    typename                    SampleT,
+    int                         BINS,
+    int                         BLOCK_THREADS,
+    int                         ITEMS_PER_THREAD,
+    BlockHistogramAlgorithm     ALGORITHM>
+void Test()
+{
+    Test<SampleT, BINS, BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM>(UNIFORM);
+    Test<SampleT, BINS, BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM>(INTEGER_SEED);
+    Test<SampleT, BINS, BLOCK_THREADS, ITEMS_PER_THREAD, ALGORITHM>(RANDOM);
+}
+
+
+/**
+ * Test different ALGORITHM
+ */
+template <
+    typename                    SampleT,
+    int                         BINS,
+    int                         BLOCK_THREADS,
+    int                         ITEMS_PER_THREAD>
+void Test()
+{
+    Test<SampleT, BINS, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_HISTO_SORT>();
+    Test<SampleT, BINS, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_HISTO_ATOMIC>();
+}
+
+
+/**
+ * Test different ITEMS_PER_THREAD
+ */
+template <
+    typename                    SampleT,
+    int                         BINS,
+    int                         BLOCK_THREADS>
+void Test()
+{
+    Test<SampleT, BINS, BLOCK_THREADS, 1>();
+    Test<SampleT, BINS, BLOCK_THREADS, 5>();
+}
+
+
+/**
+ * Test different BLOCK_THREADS
+ */
+template <
+    typename                    SampleT,
+    int                         BINS>
+void Test()
+{
+    Test<SampleT, BINS, 32>();
+    Test<SampleT, BINS, 96>();
+    Test<SampleT, BINS, 128>();
+}
+
+
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("repeat", g_repeat);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<total input samples across all channels> "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+#ifdef QUICK_TEST
+
+    // Compile/run quick tests
+    Test<unsigned char, 256, 128, 4, BLOCK_HISTO_SORT>(RANDOM);
+    Test<unsigned char, 256, 128, 4, BLOCK_HISTO_ATOMIC>(RANDOM);
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        Test<unsigned char, 32>();
+        Test<unsigned char, 256>();
+        Test<unsigned short, 1024>();
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_block_load_store.cu b/third_party/cub/test/test_block_load_store.cu
new file mode 100644
index 0000000..f1a0bf3
--- /dev/null
+++ b/third_party/cub/test/test_block_load_store.cu
@@ -0,0 +1,549 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of BlockLoad and BlockStore utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <iterator>
+#include <stdio.h>
+
+#include <cub/block/block_load.cuh>
+#include <cub/block/block_store.cuh>
+#include <cub/iterator/cache_modified_input_iterator.cuh>
+#include <cub/iterator/cache_modified_output_iterator.cuh>
+#include <cub/iterator/discard_output_iterator.cuh>
+#include <cub/util_allocator.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;
+CachingDeviceAllocator  g_allocator(true);
+
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+
+/**
+ * Test load/store kernel.
+ */
+template <
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    BlockLoadAlgorithm  LOAD_ALGORITHM,
+    BlockStoreAlgorithm STORE_ALGORITHM,
+    typename            InputIteratorT,
+    typename            OutputIteratorT>
+__launch_bounds__ (BLOCK_THREADS, 1)
+__global__ void Kernel(
+    InputIteratorT    d_in,
+    OutputIteratorT    d_out_unguarded,
+    OutputIteratorT    d_out_guarded,
+    int               num_items)
+{
+    enum
+    {
+        TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD
+    };
+
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    // Threadblock load/store abstraction types
+    typedef BlockLoad<InputT, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM> BlockLoad;
+    typedef BlockStore<OutputT, BLOCK_THREADS, ITEMS_PER_THREAD, STORE_ALGORITHM> BlockStore;
+
+    // Shared memory type for this thread block
+    union TempStorage
+    {
+        typename BlockLoad::TempStorage     load;
+        typename BlockStore::TempStorage    store;
+    };
+
+    // Allocate temp storage in shared memory
+    __shared__ TempStorage temp_storage;
+
+    // Threadblock work bounds
+    int block_offset = blockIdx.x * TILE_SIZE;
+    int guarded_elements = num_items - block_offset;
+
+    // Tile of items
+    OutputT data[ITEMS_PER_THREAD];
+
+    // Load data
+    BlockLoad(temp_storage.load).Load(d_in + block_offset, data);
+
+    __syncthreads();
+
+    // Store data
+    BlockStore(temp_storage.store).Store(d_out_unguarded + block_offset, data);
+
+    __syncthreads();
+
+    // reset data
+    #pragma unroll
+    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
+        data[ITEM] = OutputT();
+
+    __syncthreads();
+
+    // Load data
+    BlockLoad(temp_storage.load).Load(d_in + block_offset, data, guarded_elements);
+
+    __syncthreads();
+
+    // Store data
+    BlockStore(temp_storage.store).Store(d_out_guarded + block_offset, data, guarded_elements);
+}
+
+
+//---------------------------------------------------------------------
+// Host testing subroutines
+//---------------------------------------------------------------------
+
+
+/**
+ * Test load/store variants
+ */
+template <
+    typename            T,
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    BlockLoadAlgorithm  LOAD_ALGORITHM,
+    BlockStoreAlgorithm STORE_ALGORITHM,
+    typename            InputIteratorT,
+    typename            OutputIteratorT>
+void TestKernel(
+    T                   *h_in,
+    InputIteratorT      d_in,
+    OutputIteratorT      d_out_unguarded_itr,
+    OutputIteratorT      d_out_guarded_itr,
+    T                   *d_out_unguarded_ptr,
+    T                   *d_out_guarded_ptr,
+    int                 grid_size,
+    int                 guarded_elements)
+{
+    int compare;
+
+    int unguarded_elements = grid_size * BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    // Test with discard output iterator
+    typedef typename std::iterator_traits<InputIteratorT>::difference_type OffsetT;
+    DiscardOutputIterator<OffsetT> discard_itr;
+
+    Kernel<BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM>
+        <<<grid_size, BLOCK_THREADS>>>(
+            d_in,
+            discard_itr,
+            discard_itr,
+            guarded_elements);
+
+    // Test with regular output iterator
+    Kernel<BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM>
+        <<<grid_size, BLOCK_THREADS>>>(
+            d_in,
+            d_out_unguarded_itr,
+            d_out_guarded_itr,
+            guarded_elements);
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Check results
+    compare = CompareDeviceResults(h_in, d_out_guarded_ptr, guarded_elements, g_verbose, g_verbose);
+    printf("\tGuarded: %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Check results
+    compare = CompareDeviceResults(h_in, d_out_unguarded_ptr, unguarded_elements, g_verbose, g_verbose);
+    printf("\tUnguarded: %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+}
+
+
+/**
+ * Test native pointer.  Specialized for sufficient resources
+ */
+template <
+    typename            T,
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    BlockLoadAlgorithm  LOAD_ALGORITHM,
+    BlockStoreAlgorithm STORE_ALGORITHM>
+void TestNative(
+    int                 grid_size,
+    float               fraction_valid,
+    Int2Type<true>      sufficient_resources)
+{
+    int unguarded_elements = grid_size * BLOCK_THREADS * ITEMS_PER_THREAD;
+    int guarded_elements = int(fraction_valid * float(unguarded_elements));
+
+    // Allocate host arrays
+    T *h_in = (T*) malloc(unguarded_elements * sizeof(T));
+
+    // Allocate device arrays
+    T *d_in = NULL;
+    T *d_out_unguarded = NULL;
+    T *d_out_guarded = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * unguarded_elements));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out_unguarded, sizeof(T) * unguarded_elements));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out_guarded, sizeof(T) * guarded_elements));
+    CubDebugExit(cudaMemset(d_out_unguarded, 0, sizeof(T) * unguarded_elements));
+    CubDebugExit(cudaMemset(d_out_guarded, 0, sizeof(T) * guarded_elements));
+
+    // Initialize problem on host and device
+    for (int i = 0; i < unguarded_elements; ++i)
+    {
+        InitValue(INTEGER_SEED, h_in[i], i);
+    }
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * unguarded_elements, cudaMemcpyHostToDevice));
+
+    printf("TestNative "
+        "grid_size(%d) "
+        "guarded_elements(%d) "
+        "unguarded_elements(%d) "
+        "BLOCK_THREADS(%d) "
+        "ITEMS_PER_THREAD(%d) "
+        "LOAD_ALGORITHM(%d) "
+        "STORE_ALGORITHM(%d) "
+        "sizeof(T)(%d)\n",
+            grid_size, guarded_elements, unguarded_elements, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM, (int) sizeof(T));
+
+    TestKernel<T, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM>(
+        h_in,
+        (T const *) d_in,   // Test const
+        d_out_unguarded,
+        d_out_guarded,
+        d_out_unguarded,
+        d_out_guarded,
+        grid_size,
+        guarded_elements);
+
+    // Cleanup
+    if (h_in) free(h_in);
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out_unguarded) CubDebugExit(g_allocator.DeviceFree(d_out_unguarded));
+    if (d_out_guarded) CubDebugExit(g_allocator.DeviceFree(d_out_guarded));
+}
+
+
+/**
+ * Test native pointer.  Specialized for insufficient resources
+ */
+template <
+    typename            T,
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    BlockLoadAlgorithm  LOAD_ALGORITHM,
+    BlockStoreAlgorithm STORE_ALGORITHM>
+void TestNative(
+    int                 grid_size,
+    float               fraction_valid,
+    Int2Type<false>      sufficient_resources)
+{}
+
+
+/**
+ * Test iterator.  Specialized for sufficient resources.
+ */
+template <
+    typename            T,
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    BlockLoadAlgorithm  LOAD_ALGORITHM,
+    BlockStoreAlgorithm STORE_ALGORITHM,
+    CacheLoadModifier   LOAD_MODIFIER,
+    CacheStoreModifier  STORE_MODIFIER>
+void TestIterator(
+    int                 grid_size,
+    float               fraction_valid,
+    Int2Type<true>      sufficient_resources)
+{
+    int unguarded_elements = grid_size * BLOCK_THREADS * ITEMS_PER_THREAD;
+    int guarded_elements = int(fraction_valid * float(unguarded_elements));
+
+    // Allocate host arrays
+    T *h_in = (T*) malloc(unguarded_elements * sizeof(T));
+
+    // Allocate device arrays
+    T *d_in = NULL;
+    T *d_out_unguarded = NULL;
+    T *d_out_guarded = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * unguarded_elements));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out_unguarded, sizeof(T) * unguarded_elements));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out_guarded, sizeof(T) * guarded_elements));
+    CubDebugExit(cudaMemset(d_out_unguarded, 0, sizeof(T) * unguarded_elements));
+    CubDebugExit(cudaMemset(d_out_guarded, 0, sizeof(T) * guarded_elements));
+
+    // Initialize problem on host and device
+    for (int i = 0; i < unguarded_elements; ++i)
+    {
+        InitValue(INTEGER_SEED, h_in[i], i);
+    }
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * unguarded_elements, cudaMemcpyHostToDevice));
+
+    printf("TestIterator "
+        "grid_size(%d) "
+        "guarded_elements(%d) "
+        "unguarded_elements(%d) "
+        "BLOCK_THREADS(%d) "
+        "ITEMS_PER_THREAD(%d) "
+        "LOAD_ALGORITHM(%d) "
+        "STORE_ALGORITHM(%d) "
+        "LOAD_MODIFIER(%d) "
+        "STORE_MODIFIER(%d) "
+        "sizeof(T)(%d)\n",
+            grid_size, guarded_elements, unguarded_elements, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM, LOAD_MODIFIER, STORE_MODIFIER, (int) sizeof(T));
+
+    TestKernel<T, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM>(
+        h_in,
+        CacheModifiedInputIterator<LOAD_MODIFIER, T>(d_in),
+        CacheModifiedOutputIterator<STORE_MODIFIER, T>(d_out_unguarded),
+        CacheModifiedOutputIterator<STORE_MODIFIER, T>(d_out_guarded),
+        d_out_unguarded,
+        d_out_guarded,
+        grid_size,
+        guarded_elements);
+
+    // Cleanup
+    if (h_in) free(h_in);
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out_unguarded) CubDebugExit(g_allocator.DeviceFree(d_out_unguarded));
+    if (d_out_guarded) CubDebugExit(g_allocator.DeviceFree(d_out_guarded));
+}
+
+/**
+ * Test iterator.  Specialized for insufficient resources.
+ */
+template <
+    typename            T,
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    BlockLoadAlgorithm  LOAD_ALGORITHM,
+    BlockStoreAlgorithm STORE_ALGORITHM,
+    CacheLoadModifier   LOAD_MODIFIER,
+    CacheStoreModifier  STORE_MODIFIER>
+void TestIterator(
+    int                 grid_size,
+    float               fraction_valid,
+    Int2Type<false>     sufficient_resources)
+{}
+
+
+/**
+ * Evaluate different pointer access types
+ */
+template <
+    typename                T,
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    BlockLoadAlgorithm      LOAD_ALGORITHM,
+    BlockStoreAlgorithm     STORE_ALGORITHM>
+void TestPointerType(
+    int             grid_size,
+    float           fraction_valid)
+{
+    // Threadblock load/store abstraction types
+    typedef BlockLoad<T, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM> BlockLoad;
+    typedef BlockStore<T, BLOCK_THREADS, ITEMS_PER_THREAD, STORE_ALGORITHM> BlockStore;
+
+#if defined(SM100) || defined(SM110) || defined(SM130)
+    static const bool sufficient_load_smem  = sizeof(typename BlockLoad::TempStorage)   <= 1024 * 16;
+    static const bool sufficient_store_smem = sizeof(typename BlockStore::TempStorage)  <= 1024 * 16;
+    static const bool sufficient_threads    = BLOCK_THREADS <= 512;
+#else
+    static const bool sufficient_load_smem  = sizeof(typename BlockLoad::TempStorage)   <= 1024 * 48;
+    static const bool sufficient_store_smem = sizeof(typename BlockStore::TempStorage)  <= 1024 * 48;
+    static const bool sufficient_threads    = BLOCK_THREADS <= 1024;
+#endif
+
+    static const bool sufficient_resources  = sufficient_load_smem && sufficient_store_smem && sufficient_threads;
+
+    TestNative<T, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM>(grid_size, fraction_valid, Int2Type<sufficient_resources>());
+    TestIterator<T, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM, LOAD_DEFAULT, STORE_DEFAULT>(grid_size, fraction_valid, Int2Type<sufficient_resources>());
+}
+
+
+/**
+ * Evaluate different time-slicing strategies
+ */
+template <
+    typename                T,
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    BlockLoadAlgorithm      LOAD_ALGORITHM,
+    BlockStoreAlgorithm     STORE_ALGORITHM>
+void TestSlicedStrategy(
+    int             grid_size,
+    float           fraction_valid)
+{
+    TestPointerType<T, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM, true>(grid_size, fraction_valid);
+    TestPointerType<T, BLOCK_THREADS, ITEMS_PER_THREAD, LOAD_ALGORITHM, STORE_ALGORITHM, false>(grid_size, fraction_valid);
+}
+
+
+
+/**
+ * Evaluate different load/store strategies (specialized for block sizes that are not a multiple of 32)
+ */
+template <
+    typename        T,
+    int             BLOCK_THREADS,
+    int             ITEMS_PER_THREAD>
+void TestStrategy(
+    int             grid_size,
+    float           fraction_valid,
+    Int2Type<false> is_warp_multiple)
+{
+    TestPointerType<T, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_LOAD_DIRECT, BLOCK_STORE_DIRECT>(grid_size, fraction_valid);
+    TestPointerType<T, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_LOAD_TRANSPOSE, BLOCK_STORE_TRANSPOSE>(grid_size, fraction_valid);
+    TestPointerType<T, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_LOAD_VECTORIZE, BLOCK_STORE_VECTORIZE>(grid_size, fraction_valid);
+}
+
+
+/**
+ * Evaluate different load/store strategies (specialized for block sizes that are a multiple of 32)
+ */
+template <
+    typename        T,
+    int             BLOCK_THREADS,
+    int             ITEMS_PER_THREAD>
+void TestStrategy(
+    int             grid_size,
+    float           fraction_valid,
+    Int2Type<true>  is_warp_multiple)
+{
+    TestStrategy<T, BLOCK_THREADS, ITEMS_PER_THREAD>(grid_size, fraction_valid, Int2Type<false>());
+    TestPointerType<T, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_LOAD_WARP_TRANSPOSE, BLOCK_STORE_WARP_TRANSPOSE>(grid_size, fraction_valid);
+    TestPointerType<T, BLOCK_THREADS, ITEMS_PER_THREAD, BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED, BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED>(grid_size, fraction_valid);
+}
+
+
+/**
+ * Evaluate different register blocking
+ */
+template <
+    typename T,
+    int BLOCK_THREADS>
+void TestItemsPerThread(
+    int grid_size,
+    float fraction_valid)
+{
+    Int2Type<BLOCK_THREADS % 32 == 0> is_warp_multiple;
+
+    TestStrategy<T, BLOCK_THREADS, 1>(grid_size, fraction_valid, is_warp_multiple);
+    TestStrategy<T, BLOCK_THREADS, 3>(grid_size, fraction_valid, is_warp_multiple);
+    TestStrategy<T, BLOCK_THREADS, 4>(grid_size, fraction_valid, is_warp_multiple);
+    TestStrategy<T, BLOCK_THREADS, 11>(grid_size, fraction_valid, is_warp_multiple);
+}
+
+
+/**
+ * Evaluate different thread block sizes
+ */
+template <typename T>
+void TestThreads(
+    int grid_size,
+    float fraction_valid)
+{
+    TestItemsPerThread<T, 15>(grid_size, fraction_valid);
+    TestItemsPerThread<T, 32>(grid_size, fraction_valid);
+    TestItemsPerThread<T, 72>(grid_size, fraction_valid);
+    TestItemsPerThread<T, 96>(grid_size, fraction_valid);
+    TestItemsPerThread<T, 128>(grid_size, fraction_valid);
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+#ifdef QUICK_TEST
+
+    // Compile/run quick tests
+    TestNative<     int, 64, 2, BLOCK_LOAD_WARP_TRANSPOSE, BLOCK_STORE_WARP_TRANSPOSE>(1, 0.8f, Int2Type<true>());
+    TestIterator<   int, 64, 2, BLOCK_LOAD_WARP_TRANSPOSE, BLOCK_STORE_WARP_TRANSPOSE, LOAD_DEFAULT, STORE_DEFAULT>(1, 0.8f, Int2Type<true>());
+
+#else
+
+    // Compile/run thorough tests
+    TestThreads<char>(2, 0.8f);
+    TestThreads<int>(2, 0.8f);
+    TestThreads<long>(2, 0.8f);
+    TestThreads<long2>(2, 0.8f);
+
+    if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+        TestThreads<double2>(2, 0.8f);
+    TestThreads<TestFoo>(2, 0.8f);
+    TestThreads<TestBar>(2, 0.8f);
+
+#endif
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_block_radix_sort.cu b/third_party/cub/test/test_block_radix_sort.cu
new file mode 100644
index 0000000..959018b
--- /dev/null
+++ b/third_party/cub/test/test_block_radix_sort.cu
@@ -0,0 +1,717 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of BlockRadixSort utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <algorithm>
+#include <iostream>
+
+#include <cub/block/block_radix_sort.cuh>
+#include <cub/block/block_load.cuh>
+#include <cub/block/block_store.cuh>
+#include <cub/util_allocator.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;
+CachingDeviceAllocator  g_allocator(true);
+
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+
+/// Specialized descending, blocked -> blocked
+template <int BLOCK_THREADS, typename BlockRadixSort, int ITEMS_PER_THREAD, typename Key, typename Value>
+__device__ __forceinline__ void TestBlockSort(
+    typename BlockRadixSort::TempStorage &temp_storage,
+    Key                         (&keys)[ITEMS_PER_THREAD],
+    Value                       (&values)[ITEMS_PER_THREAD],
+    Key                         *d_keys,
+    Value                       *d_values,
+    int                         begin_bit,
+    int                         end_bit,
+    clock_t                     &stop,
+    Int2Type<true>              is_descending,
+    Int2Type<true>              is_blocked_output)
+{
+    BlockRadixSort(temp_storage).SortDescending(keys, values, begin_bit, end_bit);
+    stop = clock();
+    StoreDirectBlocked(threadIdx.x, d_keys, keys);
+    StoreDirectBlocked(threadIdx.x, d_values, values);
+}
+
+/// Specialized descending, blocked -> striped
+template <int BLOCK_THREADS, typename BlockRadixSort, int ITEMS_PER_THREAD, typename Key, typename Value>
+__device__ __forceinline__ void TestBlockSort(
+    typename BlockRadixSort::TempStorage &temp_storage,
+    Key                         (&keys)[ITEMS_PER_THREAD],
+    Value                       (&values)[ITEMS_PER_THREAD],
+    Key                         *d_keys,
+    Value                       *d_values,
+    int                         begin_bit,
+    int                         end_bit,
+    clock_t                     &stop,
+    Int2Type<true>              is_descending,
+    Int2Type<false>             is_blocked_output)
+{
+    BlockRadixSort(temp_storage).SortDescendingBlockedToStriped(keys, values, begin_bit, end_bit);
+    stop = clock();
+    StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_keys, keys);
+    StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_values, values);
+}
+
+/// Specialized ascending, blocked -> blocked
+template <int BLOCK_THREADS, typename BlockRadixSort, int ITEMS_PER_THREAD, typename Key, typename Value>
+__device__ __forceinline__ void TestBlockSort(
+    typename BlockRadixSort::TempStorage &temp_storage,
+    Key                         (&keys)[ITEMS_PER_THREAD],
+    Value                       (&values)[ITEMS_PER_THREAD],
+    Key                         *d_keys,
+    Value                       *d_values,
+    int                         begin_bit,
+    int                         end_bit,
+    clock_t                     &stop,
+    Int2Type<false>             is_descending,
+    Int2Type<true>              is_blocked_output)
+{
+    BlockRadixSort(temp_storage).Sort(keys, values, begin_bit, end_bit);
+    stop = clock();
+    StoreDirectBlocked(threadIdx.x, d_keys, keys);
+    StoreDirectBlocked(threadIdx.x, d_values, values);
+}
+
+/// Specialized ascending, blocked -> striped
+template <int BLOCK_THREADS, typename BlockRadixSort, int ITEMS_PER_THREAD, typename Key, typename Value>
+__device__ __forceinline__ void TestBlockSort(
+    typename BlockRadixSort::TempStorage &temp_storage,
+    Key                         (&keys)[ITEMS_PER_THREAD],
+    Value                       (&values)[ITEMS_PER_THREAD],
+    Key                         *d_keys,
+    Value                       *d_values,
+    int                         begin_bit,
+    int                         end_bit,
+    clock_t                     &stop,
+    Int2Type<false>             is_descending,
+    Int2Type<false>             is_blocked_output)
+{
+    BlockRadixSort(temp_storage).SortBlockedToStriped(keys, values, begin_bit, end_bit);
+    stop = clock();
+    StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_keys, keys);
+    StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_values, values);
+}
+
+
+
+/**
+ * BlockRadixSort kernel
+ */
+template <
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    int                 RADIX_BITS,
+    bool                MEMOIZE_OUTER_SCAN,
+    BlockScanAlgorithm  INNER_SCAN_ALGORITHM,
+    cudaSharedMemConfig SMEM_CONFIG,
+    int                 DESCENDING,
+    int                 BLOCKED_OUTPUT,
+    typename            Key,
+    typename            Value>
+__launch_bounds__ (BLOCK_THREADS, 1)
+__global__ void Kernel(
+    Key                         *d_keys,
+    Value                       *d_values,
+    int                         begin_bit,
+    int                         end_bit,
+    clock_t                     *d_elapsed)
+{
+    // Threadblock load/store abstraction types
+    typedef BlockRadixSort<
+            Key,
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            Value,
+            RADIX_BITS,
+            MEMOIZE_OUTER_SCAN,
+            INNER_SCAN_ALGORITHM,
+            SMEM_CONFIG>
+        BlockRadixSortT;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename BlockRadixSortT::TempStorage temp_storage;
+
+    // Items per thread
+    Key     keys[ITEMS_PER_THREAD];
+    Value   values[ITEMS_PER_THREAD];
+
+    LoadDirectBlocked(threadIdx.x, d_keys, keys);
+    LoadDirectBlocked(threadIdx.x, d_values, values);
+
+    // Start cycle timer
+    clock_t stop;
+    clock_t start = clock();
+
+    TestBlockSort<BLOCK_THREADS, BlockRadixSortT>(
+        temp_storage, keys, values, d_keys, d_values, begin_bit, end_bit, stop, Int2Type<DESCENDING>(), Int2Type<BLOCKED_OUTPUT>());
+
+    // Store time
+    if (threadIdx.x == 0)
+        *d_elapsed = (start > stop) ? start - stop : stop - start;
+}
+
+
+
+//---------------------------------------------------------------------
+// Host testing subroutines
+//---------------------------------------------------------------------
+
+
+/**
+ * Simple key-value pairing
+ */
+template <
+    typename Key,
+    typename Value,
+    bool IS_FLOAT = (Traits<Key>::CATEGORY == FLOATING_POINT)>
+struct Pair
+{
+    Key     key;
+    Value   value;
+
+    bool operator<(const Pair &b) const
+    {
+        return (key < b.key);
+    }
+};
+
+/**
+ * Simple key-value pairing (specialized for floating point types)
+ */
+template <typename Key, typename Value>
+struct Pair<Key, Value, true>
+{
+    Key     key;
+    Value   value;
+
+    bool operator<(const Pair &b) const
+    {
+        if (key < b.key)
+            return true;
+
+        if (key > b.key)
+            return false;
+
+        // Key in unsigned bits
+        typedef typename Traits<Key>::UnsignedBits UnsignedBits;
+
+        // Return true if key is negative zero and b.key is positive zero
+        UnsignedBits key_bits   = *reinterpret_cast<UnsignedBits*>(const_cast<Key*>(&key));
+        UnsignedBits b_key_bits = *reinterpret_cast<UnsignedBits*>(const_cast<Key*>(&b.key));
+        UnsignedBits HIGH_BIT   = Traits<Key>::HIGH_BIT;
+
+        return ((key_bits & HIGH_BIT) != 0) && ((b_key_bits & HIGH_BIT) == 0);
+    }
+};
+
+
+/**
+ * Initialize key-value sorting problem.
+ */
+template <bool DESCENDING, typename Key, typename Value>
+void Initialize(
+    GenMode         gen_mode,
+    Key             *h_keys,
+    Value           *h_values,
+    Key             *h_reference_keys,
+    Value           *h_reference_values,
+    int             num_items,
+    int             entropy_reduction,
+    int             begin_bit,
+    int             end_bit)
+{
+    Pair<Key, Value> *h_pairs = new Pair<Key, Value>[num_items];
+
+    for (int i = 0; i < num_items; ++i)
+    {
+        InitValue(gen_mode, h_keys[i], i);
+
+        RandomBits(h_values[i]);
+
+        // Mask off unwanted portions
+        int num_bits = end_bit - begin_bit;
+        if ((begin_bit > 0) || (end_bit < sizeof(Key) * 8))
+        {
+            unsigned long long base = 0;
+            memcpy(&base, &h_keys[i], sizeof(Key));
+            base &= ((1ull << num_bits) - 1) << begin_bit;
+            memcpy(&h_keys[i], &base, sizeof(Key));
+        }
+
+        h_pairs[i].key    = h_keys[i];
+        h_pairs[i].value  = h_values[i];
+    }
+
+    if (DESCENDING) std::reverse(h_pairs, h_pairs + num_items);
+    std::stable_sort(h_pairs, h_pairs + num_items);
+    if (DESCENDING) std::reverse(h_pairs, h_pairs + num_items);
+
+    for (int i = 0; i < num_items; ++i)
+    {
+        h_reference_keys[i]     = h_pairs[i].key;
+        h_reference_values[i]   = h_pairs[i].value;
+    }
+
+    delete[] h_pairs;
+}
+
+
+
+
+/**
+ * Test BlockRadixSort kernel
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS,
+    bool                    MEMOIZE_OUTER_SCAN,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM,
+    cudaSharedMemConfig     SMEM_CONFIG,
+    bool                    DESCENDING,
+    bool                    BLOCKED_OUTPUT,
+    typename                Key,
+    typename                Value>
+void TestDriver(
+    GenMode                 gen_mode,
+    int                     entropy_reduction,
+    int                     begin_bit,
+    int                     end_bit)
+{
+    enum
+    {
+        TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD,
+        KEYS_ONLY = Equals<Value, NullType>::VALUE,
+    };
+
+    // Allocate host arrays
+    Key     *h_keys             = new Key[TILE_SIZE];
+    Key     *h_reference_keys   = new Key[TILE_SIZE];
+    Value   *h_values           = new Value[TILE_SIZE];
+    Value   *h_reference_values = new Value[TILE_SIZE];
+
+    // Allocate device arrays
+    Key     *d_keys     = NULL;
+    Value   *d_values   = NULL;
+    clock_t *d_elapsed  = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys, sizeof(Key) * TILE_SIZE));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values, sizeof(Value) * TILE_SIZE));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_elapsed, sizeof(clock_t)));
+
+    // Initialize problem and solution on host
+    Initialize<DESCENDING>(gen_mode, h_keys, h_values, h_reference_keys, h_reference_values,
+        TILE_SIZE, entropy_reduction, begin_bit, end_bit);
+
+    // Copy problem to device
+    CubDebugExit(cudaMemcpy(d_keys, h_keys, sizeof(Key) * TILE_SIZE, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_values, h_values, sizeof(Value) * TILE_SIZE, cudaMemcpyHostToDevice));
+
+    printf("%s "
+        "BLOCK_THREADS(%d) "
+        "ITEMS_PER_THREAD(%d) "
+        "RADIX_BITS(%d) "
+        "MEMOIZE_OUTER_SCAN(%d) "
+        "INNER_SCAN_ALGORITHM(%d) "
+        "SMEM_CONFIG(%d) "
+        "DESCENDING(%d) "
+        "BLOCKED_OUTPUT(%d) "
+        "sizeof(Key)(%d) "
+        "sizeof(Value)(%d) "
+        "gen_mode(%d), "
+        "entropy_reduction(%d) "
+        "begin_bit(%d) "
+        "end_bit(%d), "
+        "samples(%d)\n",
+            ((KEYS_ONLY) ? "Keys-only" : "Key-value"),
+            BLOCK_THREADS,
+            ITEMS_PER_THREAD,
+            RADIX_BITS,
+            MEMOIZE_OUTER_SCAN,
+            INNER_SCAN_ALGORITHM,
+            SMEM_CONFIG,
+            DESCENDING,
+            BLOCKED_OUTPUT,
+            (int) sizeof(Key),
+            (int) sizeof(Value),
+            gen_mode,
+            entropy_reduction,
+            begin_bit,
+            end_bit,
+            g_num_rand_samples);
+
+    // Set shared memory config
+    cudaDeviceSetSharedMemConfig(SMEM_CONFIG);
+
+    // Run kernel
+    Kernel<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG, DESCENDING, BLOCKED_OUTPUT><<<1, BLOCK_THREADS>>>(
+        d_keys, d_values, begin_bit, end_bit, d_elapsed);
+
+    // Flush kernel output / errors
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Check keys results
+    printf("\tKeys: ");
+    int compare = CompareDeviceResults(h_reference_keys, d_keys, TILE_SIZE, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Check value results
+    if (!KEYS_ONLY)
+    {
+        printf("\tValues: ");
+        int compare = CompareDeviceResults(h_reference_values, d_values, TILE_SIZE, g_verbose, g_verbose);
+        printf("%s\n", compare ? "FAIL" : "PASS");
+        AssertEquals(0, compare);
+    }
+    printf("\n");
+
+    printf("\tElapsed clocks: ");
+    DisplayDeviceResults(d_elapsed, 1);
+    printf("\n");
+
+    // Cleanup
+    if (h_keys)             delete[] h_keys;
+    if (h_reference_keys)   delete[] h_reference_keys;
+    if (h_values)           delete[] h_values;
+    if (h_reference_values) delete[] h_reference_values;
+    if (d_keys)             CubDebugExit(g_allocator.DeviceFree(d_keys));
+    if (d_values)           CubDebugExit(g_allocator.DeviceFree(d_values));
+    if (d_elapsed)          CubDebugExit(g_allocator.DeviceFree(d_elapsed));
+}
+
+
+/**
+ * Test driver (valid tile size <= MAX_SMEM_BYTES)
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS,
+    bool                    MEMOIZE_OUTER_SCAN,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM,
+    cudaSharedMemConfig     SMEM_CONFIG,
+    bool                    DESCENDING,
+    bool                    BLOCKED_OUTPUT,
+    typename                Key,
+    typename                Value>
+void TestValid(Int2Type<true> fits_smem_capacity)
+{
+    // Iterate begin_bit
+    for (int begin_bit = 0; begin_bit <= 1; begin_bit++)
+    {
+        // Iterate end bit
+        for (int end_bit = begin_bit + 1; end_bit <= sizeof(Key) * 8; end_bit = end_bit * 2 + begin_bit)
+        {
+            // Uniform key distribution
+            TestDriver<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG, DESCENDING, BLOCKED_OUTPUT, Key, Value>(
+                UNIFORM, 0, begin_bit, end_bit);
+
+            // Sequential key distribution
+            TestDriver<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG, DESCENDING, BLOCKED_OUTPUT, Key, Value>(
+                INTEGER_SEED, 0, begin_bit, end_bit);
+
+            // Iterate random with entropy_reduction
+            for (int entropy_reduction = 0; entropy_reduction <= 9; entropy_reduction += 3)
+            {
+                TestDriver<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG, DESCENDING, BLOCKED_OUTPUT, Key, Value>(
+                    RANDOM, entropy_reduction, begin_bit, end_bit);
+            }
+        }
+    }
+}
+
+
+/**
+ * Test driver (invalid tile size)
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS,
+    bool                    MEMOIZE_OUTER_SCAN,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM,
+    cudaSharedMemConfig     SMEM_CONFIG,
+    bool                    DESCENDING,
+    bool                    BLOCKED_OUTPUT,
+    typename                Key,
+    typename                Value>
+void TestValid(Int2Type<false> fits_smem_capacity)
+{}
+
+
+/**
+ * Test ascending/descending and to-blocked/to-striped
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS,
+    bool                    MEMOIZE_OUTER_SCAN,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM,
+    cudaSharedMemConfig     SMEM_CONFIG,
+    typename                Key,
+    typename                Value>
+void Test()
+{
+    // Check size of smem storage for the target arch to make sure it will fit
+    typedef BlockRadixSort<Key, BLOCK_THREADS, ITEMS_PER_THREAD, Value, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG> BlockRadixSortT;
+
+#if defined(SM100) || defined(SM110) || defined(SM130)
+    Int2Type<sizeof(typename BlockRadixSortT::TempStorage) <= 16 * 1024> fits_smem_capacity;
+#else
+    Int2Type<(sizeof(typename BlockRadixSortT::TempStorage) <= 48 * 1024)> fits_smem_capacity;
+#endif
+
+    // Sort-ascending, to-striped
+    TestValid<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG, true, false, Key, Value>(fits_smem_capacity);
+
+    // Sort-descending, to-blocked
+    TestValid<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG, false, true, Key, Value>(fits_smem_capacity);
+
+    // Not necessary
+//    TestValid<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG, false, false, Key, Value>(fits_smem_capacity);
+//    TestValid<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, SMEM_CONFIG, true, true, Key, Value>(fits_smem_capacity);
+}
+
+
+/**
+ * Test value type and smem config
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS,
+    bool                    MEMOIZE_OUTER_SCAN,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM,
+    typename                Key>
+void TestKeys()
+{
+    // Test keys-only sorting with both smem configs
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, cudaSharedMemBankSizeFourByte, Key, NullType>();    // Keys-only (4-byte smem bank config)
+#if !defined(SM100) && !defined(SM110) && !defined(SM130) && !defined(SM200)
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, cudaSharedMemBankSizeEightByte, Key, NullType>();   // Keys-only (8-byte smem bank config)
+#endif
+}
+
+
+/**
+ * Test value type and smem config
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS,
+    bool                    MEMOIZE_OUTER_SCAN,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM,
+    typename                Key>
+void TestKeysAndPairs()
+{
+    // Test pairs sorting with only 4-byte configs
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, cudaSharedMemBankSizeFourByte, Key, char>();        // With small-values
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, cudaSharedMemBankSizeFourByte, Key, Key>();         // With same-values
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, cudaSharedMemBankSizeFourByte, Key, TestFoo>();     // With large values
+}
+
+
+/**
+ * Test key type
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS,
+    bool                    MEMOIZE_OUTER_SCAN,
+    BlockScanAlgorithm      INNER_SCAN_ALGORITHM>
+void Test()
+{
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+#ifdef TEST_KEYS_ONLY
+
+    // Test unsigned types with keys-only
+    TestKeys<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, unsigned char>();
+    TestKeys<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, unsigned short>();
+    TestKeys<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, unsigned int>();
+    TestKeys<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, unsigned long>();
+    TestKeys<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, unsigned long long>();
+
+#else
+
+    // Test signed and fp types with paired values
+    TestKeysAndPairs<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, char>();
+    TestKeysAndPairs<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, short>();
+    TestKeysAndPairs<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, int>();
+    TestKeysAndPairs<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, long>();
+    TestKeysAndPairs<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, long long>();
+    TestKeysAndPairs<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, float>();
+    if (ptx_version > 120)
+    {
+        // Don't check doubles on PTX120 or below because they're down-converted
+        TestKeysAndPairs<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, INNER_SCAN_ALGORITHM, double>();
+    }
+
+#endif
+}
+
+
+/**
+ * Test inner scan algorithm
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS,
+    bool                    MEMOIZE_OUTER_SCAN>
+void Test()
+{
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, BLOCK_SCAN_RAKING>();
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, MEMOIZE_OUTER_SCAN, BLOCK_SCAN_WARP_SCANS>();
+}
+
+
+/**
+ * Test outer scan algorithm
+ */
+template <
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    int                     RADIX_BITS>
+void Test()
+{
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, true>();
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, RADIX_BITS, false>();
+}
+
+
+/**
+ * Test radix bits
+ */
+template <
+    int BLOCK_THREADS,
+    int ITEMS_PER_THREAD>
+void Test()
+{
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, 1>();
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, 2>();
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, 5>();
+}
+
+
+/**
+ * Test items per thread
+ */
+template <int BLOCK_THREADS>
+void Test()
+{
+    Test<BLOCK_THREADS, 1>();
+#if defined(SM100) || defined(SM110) || defined(SM130)
+    // Open64 compiler can't handle the number of test cases
+#else
+    Test<BLOCK_THREADS, 4>();
+#endif
+    Test<BLOCK_THREADS, 11>();
+}
+
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+#ifdef QUICK_TEST
+
+    {
+        typedef float T;
+        TestDriver<32, 4, 4, true, BLOCK_SCAN_WARP_SCANS, cudaSharedMemBankSizeFourByte, false, false, T, NullType>(INTEGER_SEED, 0, 0, sizeof(T) * 8);
+    }
+/*
+    // Compile/run quick tests
+    typedef unsigned int T;
+    TestDriver<64, 17, 4, true, BLOCK_SCAN_WARP_SCANS, cudaSharedMemBankSizeFourByte, false, false, T, NullType>(RANDOM, 0, 0, sizeof(T) * 8);
+    TestDriver<96, 8, 4, true, BLOCK_SCAN_WARP_SCANS, cudaSharedMemBankSizeFourByte, false, false, T, NullType>(RANDOM, 0, 0, sizeof(T) * 8);
+    TestDriver<128, 2, 4, true, BLOCK_SCAN_WARP_SCANS, cudaSharedMemBankSizeFourByte, false, false, T, NullType>(RANDOM, 0, 0, sizeof(T) * 8);
+*/
+
+#else
+
+    // Compile/run thorough tests
+    Test<32>();
+    Test<64>();
+    Test<160>();
+
+
+#endif  // QUICK_TEST
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_block_reduce.cu b/third_party/cub/test/test_block_reduce.cu
new file mode 100644
index 0000000..c8df4bc
--- /dev/null
+++ b/third_party/cub/test/test_block_reduce.cu
@@ -0,0 +1,822 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of BlockReduce utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <device_functions.h>
+#include <typeinfo>
+
+#include <cub/block/block_reduce.cuh>
+#include <cub/block/block_load.cuh>
+#include <cub/util_ptx.cuh>
+#include <cub/util_allocator.cuh>
+#include <cub/util_debug.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose       = false;
+int                     g_repeat        = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+
+/// Generic reduction (full, 1)
+template <typename BlockReduceT, typename T, typename ReductionOp>
+__device__ __forceinline__ T DeviceTest(
+    BlockReduceT &block_reduce, T (&data)[1], ReductionOp &reduction_op)
+{
+    return block_reduce.Reduce(data[0], reduction_op);
+}
+
+/// Generic reduction (full, ITEMS_PER_THREAD)
+template <typename BlockReduceT, typename T, int ITEMS_PER_THREAD, typename ReductionOp>
+__device__ __forceinline__ T DeviceTest(
+    BlockReduceT &block_reduce, T (&data)[ITEMS_PER_THREAD], ReductionOp &reduction_op)
+{
+    return block_reduce.Reduce(data, reduction_op);
+}
+
+/// Generic reduction (partial, 1)
+template <typename BlockReduceT, typename T, typename ReductionOp>
+__device__ __forceinline__ T DeviceTest(
+    BlockReduceT &block_reduce, T &data, ReductionOp &reduction_op, int valid_threads)
+{
+    return block_reduce.Reduce(data, reduction_op, valid_threads);
+}
+
+/// Sum reduction (full, 1)
+template <typename BlockReduceT, typename T>
+__device__ __forceinline__ T DeviceTest(
+    BlockReduceT &block_reduce, T (&data)[1], Sum &reduction_op)
+{
+    return block_reduce.Sum(data[0]);
+}
+
+/// Sum reduction (full, ITEMS_PER_THREAD)
+template <typename BlockReduceT, typename T, int ITEMS_PER_THREAD>
+__device__ __forceinline__ T DeviceTest(
+    BlockReduceT &block_reduce, T (&data)[ITEMS_PER_THREAD], Sum &reduction_op)
+{
+    return block_reduce.Sum(data);
+}
+
+/// Sum reduction (partial, 1)
+template <typename BlockReduceT, typename T>
+__device__ __forceinline__ T DeviceTest(
+    BlockReduceT &block_reduce, T &data, Sum &reduction_op, int valid_threads)
+{
+    return block_reduce.Sum(data, valid_threads);
+}
+
+
+/**
+ * Test full-tile reduction kernel (where num_items is an even
+ * multiple of BLOCK_THREADS)
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_DIM_X,
+    int                     BLOCK_DIM_Y,
+    int                     BLOCK_DIM_Z,
+    int                     ITEMS_PER_THREAD,
+    typename                T,
+    typename                ReductionOp>
+__launch_bounds__ (BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z)
+__global__ void FullTileReduceKernel(
+    T                       *d_in,
+    T                       *d_out,
+    ReductionOp             reduction_op,
+    int                     tiles,
+    clock_t                 *d_elapsed)
+{
+    const int BLOCK_THREADS     = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z;
+    const int TILE_SIZE         = BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    // Cooperative thread block reduction utility type (returns aggregate in thread 0)
+    typedef BlockReduce<T, BLOCK_DIM_X, ALGORITHM, BLOCK_DIM_Y, BLOCK_DIM_Z> BlockReduceT;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename BlockReduceT::TempStorage temp_storage;
+
+    int linear_tid = RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z);
+
+    // Per-thread tile data
+    T data[ITEMS_PER_THREAD];
+
+    // Load first tile of data
+    int block_offset = 0;
+
+    if (block_offset < TILE_SIZE * tiles)
+    {
+        LoadDirectBlocked(linear_tid, d_in + block_offset, data);
+        block_offset += TILE_SIZE;
+
+        // Start cycle timer
+        clock_t start = clock();
+
+        // Cooperative reduce first tile
+        BlockReduceT block_reduce(temp_storage) ;
+        T block_aggregate = DeviceTest(block_reduce, data, reduction_op);
+
+        // Stop cycle timer
+ #if CUB_PTX_ARCH == 100
+        // Bug: recording stop clock causes mis-write of running prefix value
+        clock_t stop = 0;
+#else
+        clock_t stop = clock();
+#endif // CUB_PTX_ARCH == 100
+        clock_t elapsed = (start > stop) ? start - stop : stop - start;
+
+        // Loop over input tiles
+        while (block_offset < TILE_SIZE * tiles)
+        {
+            // TestBarrier between thread block reductions
+            __syncthreads();
+    
+            // Load tile of data
+            LoadDirectBlocked(linear_tid, d_in + block_offset, data);
+            block_offset += TILE_SIZE;
+
+            // Start cycle timer
+            clock_t start = clock();
+
+            // Cooperatively reduce the tile's aggregate
+            BlockReduceT block_reduce(temp_storage) ;
+            T tile_aggregate = DeviceTest(block_reduce, data, reduction_op);
+
+            // Stop cycle timer
+#if CUB_PTX_ARCH == 100
+            // Bug: recording stop clock causes mis-write of running prefix value
+            clock_t stop = 0;
+#else
+            clock_t stop = clock();
+#endif // CUB_PTX_ARCH == 100
+            elapsed += (start > stop) ? start - stop : stop - start;
+
+            // Reduce thread block aggregate
+            block_aggregate = reduction_op(block_aggregate, tile_aggregate);
+        }
+
+        // Store data
+        if (linear_tid == 0)
+        {
+            d_out[0] = block_aggregate;
+            *d_elapsed = elapsed;
+        }
+    }
+}
+
+
+
+/**
+ * Test partial-tile reduction kernel (where num_items < BLOCK_THREADS)
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_DIM_X,
+    int                     BLOCK_DIM_Y,
+    int                     BLOCK_DIM_Z,
+    typename                T,
+    typename                ReductionOp>
+__launch_bounds__ (BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z)
+__global__ void PartialTileReduceKernel(
+    T                       *d_in,
+    T                       *d_out,
+    int                     num_items,
+    ReductionOp             reduction_op,
+    clock_t                 *d_elapsed)
+{
+    // Cooperative thread block reduction utility type (returns aggregate only in thread-0)
+    typedef BlockReduce<T, BLOCK_DIM_X, ALGORITHM, BLOCK_DIM_Y, BLOCK_DIM_Z> BlockReduceT;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename BlockReduceT::TempStorage temp_storage;
+
+    int linear_tid = RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z);
+
+    // Per-thread tile data
+    T partial;
+
+    // Load partial tile data
+    if (linear_tid < num_items)
+    {
+        partial = d_in[linear_tid];
+    }
+
+    // Start cycle timer
+    clock_t start = clock();
+
+    // Cooperatively reduce the tile's aggregate
+    BlockReduceT block_reduce(temp_storage) ;
+    T tile_aggregate = DeviceTest(block_reduce, partial, reduction_op, num_items);
+
+    // Stop cycle timer
+#if CUB_PTX_ARCH == 100
+    // Bug: recording stop clock causes mis-write of running prefix value
+    clock_t stop = 0;
+#else
+    clock_t stop = clock();
+#endif // CUB_PTX_ARCH == 100
+
+    clock_t elapsed = (start > stop) ? start - stop : stop - start;
+
+    // Store data
+    if (linear_tid == 0)
+    {
+        d_out[0] = tile_aggregate;
+        *d_elapsed = elapsed;
+    }
+}
+
+
+//---------------------------------------------------------------------
+// Host utility subroutines
+//---------------------------------------------------------------------
+
+/**
+ * Initialize problem (and solution)
+ */
+template <
+    typename    T,
+    typename    ReductionOp>
+void Initialize(
+    GenMode     gen_mode,
+    T           *h_in,
+    T           h_reference[1],
+    ReductionOp reduction_op,
+    int         num_items)
+{
+    for (int i = 0; i < num_items; ++i)
+    {
+        InitValue(gen_mode, h_in[i], i);
+        if (i == 0)
+            h_reference[0] = h_in[0];
+        else
+            h_reference[0] = reduction_op(h_reference[0], h_in[i]);
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n");
+    }
+}
+
+
+//---------------------------------------------------------------------
+// Full tile test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Test full-tile reduction.  (Specialized for sufficient resources)
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_DIM_X,
+    int                     BLOCK_DIM_Y,
+    int                     BLOCK_DIM_Z,
+    int                     ITEMS_PER_THREAD,
+    typename                T,
+    typename                ReductionOp>
+void TestFullTile(
+    GenMode                 gen_mode,
+    int                     tiles,
+    ReductionOp             reduction_op,
+    Int2Type<true>          sufficient_resources)
+{
+    const int BLOCK_THREADS     = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z;
+    const int TILE_SIZE         = BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    int num_items = TILE_SIZE * tiles;
+
+    // Allocate host arrays
+    T *h_in = new T[num_items];
+    T h_reference[1];
+
+    // Initialize problem
+    Initialize(gen_mode, h_in, h_reference, reduction_op, num_items);
+
+    // Initialize/clear device arrays
+    T       *d_in = NULL;
+    T       *d_out = NULL;
+    clock_t *d_elapsed = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_elapsed, sizeof(unsigned long long)));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(T) * 1));
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(T) * 1));
+
+    // Test multi-tile (unguarded)
+    printf("TestFullTile %s, %s, gen-mode %d, num_items(%d), BLOCK_THREADS(%d) (%d,%d,%d), ITEMS_PER_THREAD(%d), tiles(%d), %s (%d bytes) elements:\n",
+        Equals<ReductionOp, Sum>::VALUE ? "Sum" : "Max",
+        (ALGORITHM == BLOCK_REDUCE_RAKING) ? "BLOCK_REDUCE_RAKING" : (ALGORITHM == BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY) ? "BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY" : "BLOCK_REDUCE_WARP_REDUCTIONS",
+        gen_mode,
+        num_items,
+        BLOCK_THREADS, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z,
+        ITEMS_PER_THREAD,
+        tiles,
+        typeid(T).name(),
+        (int) sizeof(T));
+    fflush(stdout);
+
+    dim3 block_dims(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z);
+    FullTileReduceKernel<ALGORITHM, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, ITEMS_PER_THREAD><<<1, block_dims>>>(
+        d_in,
+        d_out,
+        reduction_op,
+        tiles,
+        d_elapsed);
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Copy out and display results
+    printf("\tReduction results: ");
+    int compare = CompareDeviceResults(h_reference, d_out, 1, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    printf("\tElapsed clocks: ");
+    DisplayDeviceResults(d_elapsed, 1);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_elapsed) CubDebugExit(g_allocator.DeviceFree(d_elapsed));
+}
+
+
+/**
+ * Test full-tile reduction.  (Specialized for insufficient resources)
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_DIM_X,
+    int                     BLOCK_DIM_Y,
+    int                     BLOCK_DIM_Z,
+    int                     ITEMS_PER_THREAD,
+    typename                T,
+    typename                ReductionOp>
+void TestFullTile(
+    GenMode                 gen_mode,
+    int                     tiles,
+    ReductionOp             reduction_op,
+    Int2Type<false>         sufficient_resources)
+{}
+
+
+/**
+ * Test full-tile reduction.
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_DIM_X,
+    int                     BLOCK_DIM_Y,
+    int                     BLOCK_DIM_Z,
+    int                     ITEMS_PER_THREAD,
+    typename                T,
+    typename                ReductionOp>
+void TestFullTile(
+    GenMode                 gen_mode,
+    int                     tiles,
+    ReductionOp             reduction_op)
+{
+    // Check size of smem storage for the target arch to make sure it will fit
+    typedef BlockReduce<T, BLOCK_DIM_X, ALGORITHM, BLOCK_DIM_Y, BLOCK_DIM_Z, TEST_ARCH> BlockReduceT;
+
+    enum 
+    {
+#if defined(SM100) || defined(SM110) || defined(SM130)
+        sufficient_smem       = (sizeof(typename BlockReduceT::TempStorage) <= 16 * 1024),
+        sufficient_threads    = ((BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z) <= 512),
+#else
+        sufficient_smem       = (sizeof(typename BlockReduceT::TempStorage) <= 48 * 1024),
+        sufficient_threads    = ((BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z) <= 1024),
+#endif
+    };
+
+    TestFullTile<ALGORITHM, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, ITEMS_PER_THREAD, T>(gen_mode, tiles, reduction_op, Int2Type<sufficient_smem && sufficient_threads>());
+}
+
+
+/**
+ * Run battery of tests for different thread block dimensions
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_THREADS,
+    int                     ITEMS_PER_THREAD,
+    typename                T,
+    typename                ReductionOp>
+void TestFullTile(
+    GenMode                 gen_mode,
+    int                     tiles,
+    ReductionOp             reduction_op)
+{
+    TestFullTile<ALGORITHM, BLOCK_THREADS, 1, 1, ITEMS_PER_THREAD, T>(gen_mode, tiles, reduction_op);
+    TestFullTile<ALGORITHM, BLOCK_THREADS, 2, 2, ITEMS_PER_THREAD, T>(gen_mode, tiles, reduction_op);
+}
+
+/**
+ * Run battery of tests for different thread items
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_THREADS,
+    typename                T,
+    typename                ReductionOp>
+void TestFullTile(
+    GenMode                 gen_mode,
+    int                     tiles,
+    ReductionOp             reduction_op)
+{
+    TestFullTile<ALGORITHM, BLOCK_THREADS, 1, T>(gen_mode, tiles, reduction_op);
+    TestFullTile<ALGORITHM, BLOCK_THREADS, 4, T>(gen_mode, tiles, reduction_op);
+}
+
+
+/**
+ * Run battery of full-tile tests for different numbers of tiles
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_THREADS,
+    typename                T,
+    typename                ReductionOp>
+void TestFullTile(
+    GenMode                 gen_mode,
+    ReductionOp             reduction_op)
+{
+    for (int tiles = 1; tiles < 3; tiles++)
+    {
+        TestFullTile<ALGORITHM, BLOCK_THREADS, T>(gen_mode, tiles, reduction_op);
+    }
+}
+
+
+//---------------------------------------------------------------------
+// Partial-tile test generation
+//---------------------------------------------------------------------
+
+/**
+ * Test partial-tile reduction.  (Specialized for sufficient resources)
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_DIM_X,
+    int                     BLOCK_DIM_Y,
+    int                     BLOCK_DIM_Z,
+    typename                T,
+    typename                ReductionOp>
+void TestPartialTile(
+    GenMode                 gen_mode,
+    int                     num_items,
+    ReductionOp             reduction_op,
+    Int2Type<true>          sufficient_resources)
+{
+    const int BLOCK_THREADS     = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z;
+    const int TILE_SIZE         = BLOCK_THREADS;
+
+    // Allocate host arrays
+    T *h_in = new T[num_items];
+    T h_reference[1];
+
+    // Initialize problem
+    Initialize(gen_mode, h_in, h_reference, reduction_op, num_items);
+
+    // Initialize/clear device arrays
+    T       *d_in = NULL;
+    T       *d_out = NULL;
+    clock_t *d_elapsed = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_elapsed, sizeof(unsigned long long)));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * TILE_SIZE));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(T) * 1));
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(T) * 1));
+
+    printf("TestPartialTile %s, gen-mode %d, num_items(%d), BLOCK_THREADS(%d) (%d,%d,%d), %s (%d bytes) elements:\n",
+        (ALGORITHM == BLOCK_REDUCE_RAKING) ? "BLOCK_REDUCE_RAKING" : (ALGORITHM == BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY) ? "BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY" : "BLOCK_REDUCE_WARP_REDUCTIONS",
+        gen_mode,
+        num_items,
+        BLOCK_THREADS, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z,
+        typeid(T).name(),
+        (int) sizeof(T));
+    fflush(stdout);
+
+    dim3 block_dims(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z);
+    PartialTileReduceKernel<ALGORITHM, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z><<<1, block_dims>>>(
+        d_in,
+        d_out,
+        num_items,
+        reduction_op,
+        d_elapsed);
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Copy out and display results
+    printf("\tReduction results: ");
+    int compare = CompareDeviceResults(h_reference, d_out, 1, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    printf("\tElapsed clocks: ");
+    DisplayDeviceResults(d_elapsed, 1);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_elapsed) CubDebugExit(g_allocator.DeviceFree(d_elapsed));
+}
+
+
+
+/**
+ * Test partial-tile reduction (specialized for insufficient resources)
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_DIM_X,
+    int                     BLOCK_DIM_Y,
+    int                     BLOCK_DIM_Z,
+    typename                T,
+    typename                ReductionOp>
+void TestPartialTile(
+    GenMode                 gen_mode,
+    int                     num_items,
+    ReductionOp             reduction_op,
+    Int2Type<false>         sufficient_resources)
+{}
+
+
+/**
+ *  Run battery of partial-tile tests for different numbers of effective threads and thread dimensions
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_DIM_X,
+    int                     BLOCK_DIM_Y,
+    int                     BLOCK_DIM_Z,
+    typename                T,
+    typename                ReductionOp>
+void TestPartialTile(
+    GenMode                 gen_mode,
+    int                     num_items,
+    ReductionOp             reduction_op)
+{
+    // Check size of smem storage for the target arch to make sure it will fit
+    typedef BlockReduce<T, BLOCK_DIM_X, ALGORITHM, BLOCK_DIM_Y, BLOCK_DIM_Z, TEST_ARCH> BlockReduceT;
+
+    enum 
+    {
+#if defined(SM100) || defined(SM110) || defined(SM130)
+        sufficient_smem       = sizeof(typename BlockReduceT::TempStorage)  <= 16 * 1024,
+        sufficient_threads    = (BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z)   <= 512,
+#else
+        sufficient_smem       = sizeof(typename BlockReduceT::TempStorage)  <= 48 * 1024,
+        sufficient_threads    = (BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z)   <= 1024,
+#endif
+    };
+
+    TestPartialTile<ALGORITHM, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, T>(gen_mode, num_items, reduction_op, Int2Type<sufficient_smem && sufficient_threads>());
+}
+
+
+
+/**
+ *  Run battery of partial-tile tests for different numbers of effective threads and thread dimensions
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_THREADS,
+    typename                T,
+    typename                ReductionOp>
+void TestPartialTile(
+    GenMode                 gen_mode,
+    ReductionOp             reduction_op)
+{
+    for (
+        int num_items = 1;
+        num_items < BLOCK_THREADS;
+        num_items += CUB_MAX(1, BLOCK_THREADS / 5))
+    {
+        TestPartialTile<ALGORITHM, BLOCK_THREADS, 1, 1, T>(gen_mode, num_items, reduction_op);
+        TestPartialTile<ALGORITHM, BLOCK_THREADS, 2, 2, T>(gen_mode, num_items, reduction_op);
+    }
+}
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Run battery of full-tile tests for different gen modes
+ */
+template <
+    BlockReduceAlgorithm    ALGORITHM,
+    int                     BLOCK_THREADS,
+    typename                T,
+    typename                ReductionOp>
+void Test(
+    ReductionOp             reduction_op)
+{
+    TestFullTile<ALGORITHM, BLOCK_THREADS, T>(UNIFORM, reduction_op);
+    TestPartialTile<ALGORITHM, BLOCK_THREADS, T>(UNIFORM, reduction_op);
+
+    TestFullTile<ALGORITHM, BLOCK_THREADS, T>(INTEGER_SEED, reduction_op);
+    TestPartialTile<ALGORITHM, BLOCK_THREADS, T>(INTEGER_SEED, reduction_op);
+
+    if (Traits<T>::CATEGORY != FLOATING_POINT)
+    {
+        // Don't test randomly-generated floats b/c of stability
+        TestFullTile<ALGORITHM, BLOCK_THREADS, T>(RANDOM, reduction_op);
+        TestPartialTile<ALGORITHM, BLOCK_THREADS, T>(RANDOM, reduction_op);
+    }
+}
+
+
+/**
+ * Run battery of tests for different block-reduction algorithmic variants
+ */
+template <
+    int             BLOCK_THREADS,
+    typename        T,
+    typename        ReductionOp>
+void Test(
+    ReductionOp     reduction_op)
+{
+#ifdef TEST_RAKING
+    Test<BLOCK_REDUCE_RAKING, BLOCK_THREADS, T>(reduction_op);
+    Test<BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY, BLOCK_THREADS, T>(reduction_op);
+#endif
+#ifdef TEST_WARP_REDUCTIONS
+    Test<BLOCK_REDUCE_WARP_REDUCTIONS, BLOCK_THREADS, T>(reduction_op);
+#endif
+}
+
+
+/**
+ * Run battery of tests for different block sizes
+ */
+template <
+    typename        T,
+    typename        ReductionOp>
+void Test(
+    ReductionOp     reduction_op)
+{
+    Test<7,   T>(reduction_op);
+    Test<32,  T>(reduction_op);
+    Test<63,  T>(reduction_op);
+    Test<97,  T>(reduction_op);
+    Test<128, T>(reduction_op);
+    Test<238, T>(reduction_op);
+}
+
+
+/**
+ * Run battery of tests for different block sizes
+ */
+template <typename T>
+void Test()
+{
+    Test<T>(Sum());
+    Test<T>(Max());
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("repeat", g_repeat);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+#ifdef QUICK_TEST
+
+    // Compile/run quick tests
+
+
+    printf("\n full tile ------------------------\n\n");
+
+    TestFullTile<BLOCK_REDUCE_RAKING,                   128, 1, 1, 4, int>(RANDOM, 1, Sum());
+    TestFullTile<BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY,  128, 1, 1, 4, int>(RANDOM, 1, Sum());
+    TestFullTile<BLOCK_REDUCE_WARP_REDUCTIONS,          128, 1, 1, 4, int>(RANDOM, 1, Sum());
+
+    TestFullTile<BLOCK_REDUCE_RAKING,                   128, 1, 1, 1, int>(RANDOM, 1, Sum());
+    TestFullTile<BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY,  128, 1, 1, 1, int>(RANDOM, 1, Sum());
+    TestFullTile<BLOCK_REDUCE_WARP_REDUCTIONS,          128, 1, 1, 1, int>(RANDOM, 1, Sum());
+
+    printf("\n partial tile ------------------------\n\n");
+
+    TestPartialTile<BLOCK_REDUCE_RAKING,                   128, 1, 1, int>(RANDOM, 7, Sum());
+    TestPartialTile<BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY,  128, 1, 1, int>(RANDOM, 7, Sum());
+    TestPartialTile<BLOCK_REDUCE_WARP_REDUCTIONS,          128, 1, 1, int>(RANDOM, 7, Sum());
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // primitives
+        Test<char>();
+        Test<short>();
+        Test<int>();
+        Test<long long>();
+        if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+            Test<double>();
+
+        Test<float>();
+
+        // vector types
+        Test<char2>();
+        Test<short2>();
+        Test<int2>();
+        Test<longlong2>();
+
+        Test<char4>();
+        Test<short4>();
+        Test<int4>();
+        Test<longlong4>();
+
+        // Complex types
+        Test<TestFoo>();
+        Test<TestBar>();
+    }
+
+#endif
+
+    return 0;
+}
+
+
diff --git a/third_party/cub/test/test_block_scan.cu b/third_party/cub/test/test_block_scan.cu
new file mode 100644
index 0000000..192fb51
--- /dev/null
+++ b/third_party/cub/test/test_block_scan.cu
@@ -0,0 +1,929 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of BlockScan utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <iostream>
+#include <limits>
+#include <typeinfo>
+
+#include <cub/block/block_scan.cuh>
+#include <cub/block/block_load.cuh>
+#include <cub/block/block_store.cuh>
+#include <cub/util_ptx.cuh>
+#include <cub/util_allocator.cuh>
+
+#include "test_util.h"
+
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose       = false;
+int                     g_repeat        = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+
+/**
+ * Primitive variant to test
+ */
+enum TestMode
+{
+    BASIC,
+    AGGREGATE,
+    PREFIX,
+};
+
+
+/**
+ * Scan mode to test
+ */
+enum ScanMode
+{
+    EXCLUSIVE,
+    INCLUSIVE
+};
+
+
+/**
+ * \brief WrapperFunctor (for precluding test-specialized dispatch to *Sum variants)
+ */
+template<typename OpT>
+struct WrapperFunctor
+{
+    OpT op;
+
+    WrapperFunctor(OpT op) : op(op) {}
+
+    template <typename T>
+    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
+    {
+        return op(a, b);
+    }
+};
+
+
+/**
+ * Stateful prefix functor
+ */
+template <
+    typename T,
+    typename ScanOpT>
+struct BlockPrefixCallbackOp
+{
+    int     linear_tid;
+    T       prefix;
+    ScanOpT  scan_op;
+
+    __device__ __forceinline__
+    BlockPrefixCallbackOp(int linear_tid, T prefix, ScanOpT scan_op) :
+        linear_tid(linear_tid),
+        prefix(prefix),
+        scan_op(scan_op)
+    {}
+
+    __device__ __forceinline__
+    T operator()(T block_aggregate)
+    {
+        // For testing purposes
+        T retval = (linear_tid == 0) ? prefix  : T();
+        prefix = scan_op(prefix, block_aggregate);
+        return retval;
+    }
+};
+
+
+//---------------------------------------------------------------------
+// Exclusive scan
+//---------------------------------------------------------------------
+
+/// Exclusive scan (BASIC, 1)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<BASIC> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.ExclusiveScan(data[0], data[0], initial_value, scan_op);
+}
+
+/// Exclusive scan (BASIC, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, int ITEMS_PER_THREAD, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<BASIC> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.ExclusiveScan(data, data, initial_value, scan_op);
+}
+
+/// Exclusive scan (AGGREGATE, 1)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<AGGREGATE> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.ExclusiveScan(data[0], data[0], initial_value, scan_op, block_aggregate);
+}
+
+/// Exclusive scan (AGGREGATE, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, int ITEMS_PER_THREAD, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<AGGREGATE> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.ExclusiveScan(data, data, initial_value, scan_op, block_aggregate);
+}
+
+/// Exclusive scan (PREFIX, 1)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<PREFIX> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.ExclusiveScan(data[0], data[0], scan_op, prefix_op);
+}
+
+/// Exclusive scan (PREFIX, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, int ITEMS_PER_THREAD, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<PREFIX> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.ExclusiveScan(data, data, scan_op, prefix_op);
+}
+
+
+//---------------------------------------------------------------------
+// Exclusive sum
+//---------------------------------------------------------------------
+
+/// Exclusive sum (BASIC, 1)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<BASIC> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.ExclusiveSum(data[0], data[0]);
+}
+
+/// Exclusive sum (BASIC, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp, int ITEMS_PER_THREAD>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<BASIC> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.ExclusiveSum(data, data);
+}
+
+/// Exclusive sum (AGGREGATE, 1)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<AGGREGATE> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.ExclusiveSum(data[0], data[0], block_aggregate);
+}
+
+/// Exclusive sum (AGGREGATE, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp, int ITEMS_PER_THREAD>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<AGGREGATE> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.ExclusiveSum(data, data, block_aggregate);
+}
+
+/// Exclusive sum (PREFIX, 1)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<PREFIX> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.ExclusiveSum(data[0], data[0], prefix_op);
+}
+
+/// Exclusive sum (PREFIX, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp, int ITEMS_PER_THREAD>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<EXCLUSIVE> scan_mode, Int2Type<PREFIX> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.ExclusiveSum(data, data, prefix_op);
+}
+
+
+//---------------------------------------------------------------------
+// Inclusive scan
+//---------------------------------------------------------------------
+
+/// Inclusive scan (BASIC, 1)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<BASIC> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.InclusiveScan(data[0], data[0], scan_op);
+}
+
+/// Inclusive scan (BASIC, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, int ITEMS_PER_THREAD, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<BASIC> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.InclusiveScan(data, data, scan_op);
+}
+
+/// Inclusive scan (AGGREGATE, 1)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<AGGREGATE> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.InclusiveScan(data[0], data[0], scan_op, block_aggregate);
+}
+
+/// Inclusive scan (AGGREGATE, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, int ITEMS_PER_THREAD, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<AGGREGATE> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.InclusiveScan(data, data, scan_op, block_aggregate);
+}
+
+/// Inclusive scan (PREFIX, 1)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<PREFIX> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.InclusiveScan(data[0], data[0], scan_op, prefix_op);
+}
+
+/// Inclusive scan (PREFIX, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename ScanOpT, typename PrefixCallbackOp, int ITEMS_PER_THREAD, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, ScanOpT &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<PREFIX> test_mode, IsPrimitiveT is_primitive)
+{
+    block_scan.InclusiveScan(data, data, scan_op, prefix_op);
+}
+
+
+//---------------------------------------------------------------------
+// Inclusive sum
+//---------------------------------------------------------------------
+
+/// Inclusive sum (BASIC, 1)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<BASIC> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.InclusiveSum(data[0], data[0]);
+}
+
+/// Inclusive sum (BASIC, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp, int ITEMS_PER_THREAD>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<BASIC> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.InclusiveSum(data, data);
+}
+
+/// Inclusive sum (AGGREGATE, 1)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<AGGREGATE> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.InclusiveSum(data[0], data[0], block_aggregate);
+}
+
+/// Inclusive sum (AGGREGATE, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp, int ITEMS_PER_THREAD>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<AGGREGATE> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.InclusiveSum(data, data, block_aggregate);
+}
+
+/// Inclusive sum (PREFIX, 1)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[1], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<PREFIX> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.InclusiveSum(data[0], data[0], prefix_op);
+}
+
+/// Inclusive sum (PREFIX, ITEMS_PER_THREAD)
+template <typename BlockScanT, typename T, typename PrefixCallbackOp, int ITEMS_PER_THREAD>
+__device__ __forceinline__ void DeviceTest(
+    BlockScanT &block_scan, T (&data)[ITEMS_PER_THREAD], T &initial_value, Sum &scan_op, T &block_aggregate, PrefixCallbackOp &prefix_op,
+    Int2Type<INCLUSIVE> scan_mode, Int2Type<PREFIX> test_mode, Int2Type<true> is_primitive)
+{
+    block_scan.InclusiveSum(data, data, prefix_op);
+}
+
+
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+/**
+ * BlockScan test kernel.
+ */
+template <
+    int                 BLOCK_DIM_X,
+    int                 BLOCK_DIM_Y,
+    int                 BLOCK_DIM_Z,
+    int                 ITEMS_PER_THREAD,
+    ScanMode            SCAN_MODE,
+    TestMode            TEST_MODE,
+    BlockScanAlgorithm  ALGORITHM,
+    typename            T,
+    typename            ScanOpT>
+__launch_bounds__ (BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z)
+__global__ void BlockScanKernel(
+    T                   *d_in,
+    T                   *d_out,
+    T                   *d_aggregate,
+    ScanOpT              scan_op,
+    T                   initial_value,
+    clock_t             *d_elapsed)
+{
+    const int BLOCK_THREADS     = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z;
+    const int TILE_SIZE         = BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    // Parameterize BlockScan type for our thread block
+    typedef BlockScan<T, BLOCK_DIM_X, ALGORITHM, BLOCK_DIM_Y, BLOCK_DIM_Z> BlockScanT;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename BlockScanT::TempStorage temp_storage;
+
+    int linear_tid = RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z);
+
+    // Per-thread tile data
+    T data[ITEMS_PER_THREAD];
+    LoadDirectBlocked(linear_tid, d_in, data);
+
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t start = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    // Test scan
+    T                                   block_aggregate;
+    BlockScanT                          block_scan(temp_storage);
+    BlockPrefixCallbackOp<T, ScanOpT>   prefix_op(linear_tid, initial_value, scan_op);
+
+    DeviceTest(block_scan, data, initial_value, scan_op, block_aggregate, prefix_op,
+        Int2Type<SCAN_MODE>(), Int2Type<TEST_MODE>(), Int2Type<Traits<T>::PRIMITIVE>());
+
+    // Stop cycle timer
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t stop = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    // Store output
+    StoreDirectBlocked(linear_tid, d_out, data);
+
+    // Store block_aggregate
+    if (TEST_MODE != BASIC)
+        d_aggregate[linear_tid] = block_aggregate;
+
+    // Store prefix
+    if (TEST_MODE == PREFIX)
+    {
+        if (linear_tid == 0)
+            d_out[TILE_SIZE] = prefix_op.prefix;
+    }
+
+    // Store time
+    if (linear_tid == 0)
+        *d_elapsed = (start > stop) ? start - stop : stop - start;
+}
+
+
+
+//---------------------------------------------------------------------
+// Host utility subroutines
+//---------------------------------------------------------------------
+
+/**
+ * Initialize exclusive-scan problem (and solution)
+ */
+template <typename T, typename ScanOpT>
+T Initialize(
+    GenMode     gen_mode,
+    T           *h_in,
+    T           *h_reference,
+    int         num_items,
+    ScanOpT     scan_op,
+    T           initial_value,
+    Int2Type<EXCLUSIVE>)
+{
+    InitValue(gen_mode, h_in[0], 0);
+
+    T block_aggregate   = h_in[0];
+    h_reference[0]      = initial_value;
+    T inclusive         = scan_op(initial_value, h_in[0]);
+
+    for (int i = 1; i < num_items; ++i)
+    {
+        InitValue(gen_mode, h_in[i], i);
+        h_reference[i] = inclusive;
+        inclusive = scan_op(inclusive, h_in[i]);
+        block_aggregate = scan_op(block_aggregate, h_in[i]);
+    }
+
+    return block_aggregate;
+}
+
+
+/**
+ * Initialize inclusive-scan problem (and solution)
+ */
+template <typename T, typename ScanOpT>
+T Initialize(
+    GenMode     gen_mode,
+    T           *h_in,
+    T           *h_reference,
+    int         num_items,
+    ScanOpT      scan_op,
+    T           initial_value,
+    Int2Type<INCLUSIVE>)
+{
+    InitValue(gen_mode, h_in[0], 0);
+
+    T block_aggregate   = h_in[0];
+    T inclusive         = scan_op(initial_value, h_in[0]);
+    h_reference[0]      = inclusive;
+
+    for (int i = 1; i < num_items; ++i)
+    {
+        InitValue(gen_mode, h_in[i], i);
+        inclusive = scan_op(inclusive, h_in[i]);
+        block_aggregate = scan_op(block_aggregate, h_in[i]);
+        h_reference[i] = inclusive;
+    }
+
+    return block_aggregate;
+}
+
+
+/**
+ * Test thread block scan.  (Specialized for sufficient resources)
+ */
+template <
+    int                 BLOCK_DIM_X,
+    int                 BLOCK_DIM_Y,
+    int                 BLOCK_DIM_Z,
+    int                 ITEMS_PER_THREAD,
+    ScanMode            SCAN_MODE,
+    TestMode            TEST_MODE,
+    BlockScanAlgorithm  ALGORITHM,
+    typename            ScanOpT,
+    typename            T>
+void Test(
+    GenMode             gen_mode,
+    ScanOpT             scan_op,
+    T                   initial_value,
+    Int2Type<true>      sufficient_resources)
+{
+    const int BLOCK_THREADS     = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z;
+    const int TILE_SIZE         = BLOCK_THREADS * ITEMS_PER_THREAD;
+
+    // Allocate host arrays
+    T *h_in = new T[TILE_SIZE];
+    T *h_reference = new T[TILE_SIZE];
+    T *h_aggregate = new T[BLOCK_THREADS];
+
+    // Initialize problem
+    T block_aggregate = Initialize(
+        gen_mode,
+        h_in,
+        h_reference,
+        TILE_SIZE,
+        scan_op,
+        initial_value,
+        Int2Type<SCAN_MODE>());
+
+    // Test reference block_aggregate is returned in all threads
+    for (int i = 0; i < BLOCK_THREADS; ++i)
+    {
+        h_aggregate[i] = block_aggregate;
+    }
+
+    // Run kernel
+    printf("Test-mode %d, gen-mode %d, policy %d, %s %s BlockScan, %d (%d,%d,%d) thread block threads, %d items per thread, %d tile size, %s (%d bytes) elements:\n",
+        TEST_MODE, gen_mode, ALGORITHM,
+        (SCAN_MODE == INCLUSIVE) ? "Inclusive" : "Exclusive", typeid(ScanOpT).name(),
+        BLOCK_THREADS, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z,
+        ITEMS_PER_THREAD,  TILE_SIZE,
+        typeid(T).name(), (int) sizeof(T));
+    fflush(stdout);
+
+    // Initialize/clear device arrays
+    T       *d_in = NULL;
+    T       *d_out = NULL;
+    T       *d_aggregate = NULL;
+    clock_t *d_elapsed = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_elapsed, sizeof(unsigned long long)));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * TILE_SIZE));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(T) * (TILE_SIZE + 2)));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_aggregate, sizeof(T) * BLOCK_THREADS));
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * TILE_SIZE, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(T) * (TILE_SIZE + 1)));
+    CubDebugExit(cudaMemset(d_aggregate, 0, sizeof(T) * BLOCK_THREADS));
+
+    // Display input problem data
+    if (g_verbose)
+    {
+        printf("Input data: ");
+        for (int i = 0; i < TILE_SIZE; i++)
+        {
+            std::cout << CoutCast(h_in[i]) << ", ";
+        }
+        printf("\n\n");
+    }
+
+    // Run block_aggregate/prefix kernel
+    dim3 block_dims(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z);
+    BlockScanKernel<BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, ITEMS_PER_THREAD, SCAN_MODE, TEST_MODE, ALGORITHM><<<1, block_dims>>>(
+        d_in,
+        d_out,
+        d_aggregate,
+        scan_op,
+        initial_value,
+        d_elapsed);
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Copy out and display results
+    printf("\tScan results: ");
+    int compare = CompareDeviceResults(h_reference, d_out, TILE_SIZE, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    if (TEST_MODE == AGGREGATE)
+    {
+        // Copy out and display block_aggregate
+        printf("\tScan block aggregate: ");
+        compare = CompareDeviceResults(h_aggregate, d_aggregate, BLOCK_THREADS, g_verbose, g_verbose);
+        printf("%s\n", compare ? "FAIL" : "PASS");
+        AssertEquals(0, compare);
+    }
+
+    if (TEST_MODE == PREFIX)
+    {
+        // Copy out and display updated prefix
+        printf("\tScan running total: ");
+        T running_total = scan_op(initial_value, block_aggregate);
+        compare = CompareDeviceResults(&running_total, d_out + TILE_SIZE, 1, g_verbose, g_verbose);
+        printf("%s\n", compare ? "FAIL" : "PASS");
+        AssertEquals(0, compare);
+    }
+
+    printf("\tElapsed clocks: ");
+    DisplayDeviceResults(d_elapsed, 1);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (h_aggregate) delete[] h_aggregate;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_aggregate) CubDebugExit(g_allocator.DeviceFree(d_aggregate));
+    if (d_elapsed) CubDebugExit(g_allocator.DeviceFree(d_elapsed));
+}
+
+
+/**
+ * Test thread block scan.  (Specialized for insufficient resources)
+ */
+template <
+    int                 BLOCK_DIM_X,
+    int                 BLOCK_DIM_Y,
+    int                 BLOCK_DIM_Z,
+    int                 ITEMS_PER_THREAD,
+    ScanMode            SCAN_MODE,
+    TestMode            TEST_MODE,
+    BlockScanAlgorithm  ALGORITHM,
+    typename            ScanOpT,
+    typename            T>
+void Test(
+    GenMode             gen_mode,
+    ScanOpT             scan_op,
+    T                   initial_value,
+    Int2Type<false>     sufficient_resources)
+{}
+
+
+/**
+ * Test thread block scan.
+ */
+template <
+    int                 BLOCK_DIM_X,
+    int                 BLOCK_DIM_Y,
+    int                 BLOCK_DIM_Z,
+    int                 ITEMS_PER_THREAD,
+    ScanMode            SCAN_MODE,
+    TestMode            TEST_MODE,
+    BlockScanAlgorithm  ALGORITHM,
+    typename            ScanOpT,
+    typename            T>
+void Test(
+    GenMode             gen_mode,
+    ScanOpT             scan_op,
+    T                   initial_value)
+{
+    // Check size of smem storage for the target arch to make sure it will fit
+    typedef BlockScan<T, BLOCK_DIM_X, ALGORITHM, BLOCK_DIM_Y, BLOCK_DIM_Z> BlockScanT;
+
+    enum
+    {
+#if defined(SM100) || defined(SM110) || defined(SM130)
+        sufficient_smem         = (sizeof(typename BlockScanT::TempStorage)     <= 16 * 1024),
+        sufficient_threads      = ((BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z)    <= 512),
+#else
+        sufficient_smem         = (sizeof(typename BlockScanT::TempStorage)     <= 16 * 1024),
+        sufficient_threads      = ((BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z)    <= 1024),
+#endif
+
+#if defined(_WIN32) || defined(_WIN64)
+        // Accommodate ptxas crash bug (access violation) on Windows
+        special_skip            = ((TEST_ARCH <= 130) && (Equals<T, TestBar>::VALUE) && (BLOCK_DIM_Z > 1)),
+#else
+        special_skip            = false,
+#endif
+        sufficient_resources    = (sufficient_smem && sufficient_threads && !special_skip),
+    };
+
+    Test<BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, ITEMS_PER_THREAD, SCAN_MODE, TEST_MODE, ALGORITHM>(
+        gen_mode, scan_op, initial_value, Int2Type<sufficient_resources>());
+}
+
+
+
+/**
+ * Run test for different thread block dimensions
+ */
+template <
+    int                 BLOCK_THREADS,
+    int                 ITEMS_PER_THREAD,
+    ScanMode            SCAN_MODE,
+    TestMode            TEST_MODE,
+    BlockScanAlgorithm  ALGORITHM,
+    typename            ScanOpT,
+    typename            T>
+void Test(
+    GenMode     gen_mode,
+    ScanOpT     scan_op,
+    T           initial_value)
+{
+    Test<BLOCK_THREADS, 1, 1, ITEMS_PER_THREAD, SCAN_MODE, TEST_MODE, ALGORITHM>(gen_mode, scan_op, initial_value);
+    Test<BLOCK_THREADS, 2, 2, ITEMS_PER_THREAD, SCAN_MODE, TEST_MODE, ALGORITHM>(gen_mode, scan_op, initial_value);
+}
+
+
+/**
+ * Run test for different policy types
+ */
+template <
+    int         BLOCK_THREADS,
+    int         ITEMS_PER_THREAD,
+    ScanMode    SCAN_MODE,
+    TestMode    TEST_MODE,
+    typename    ScanOpT,
+    typename    T>
+void Test(
+    GenMode     gen_mode,
+    ScanOpT     scan_op,
+    T           initial_value)
+{
+#ifdef TEST_RAKING
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, SCAN_MODE, TEST_MODE, BLOCK_SCAN_RAKING>(gen_mode, scan_op, initial_value);
+#endif
+#ifdef TEST_RAKING_MEMOIZE
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, SCAN_MODE, TEST_MODE, BLOCK_SCAN_RAKING_MEMOIZE>(gen_mode, scan_op, initial_value);
+#endif
+#ifdef TEST_WARP_SCANS
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, SCAN_MODE, TEST_MODE, BLOCK_SCAN_WARP_SCANS>(gen_mode, scan_op, initial_value);
+#endif
+}
+
+
+/**
+ * Run tests for different primitive variants
+ */
+template <
+    int         BLOCK_THREADS,
+    int         ITEMS_PER_THREAD,
+    typename    ScanOpT,
+    typename    T>
+void Test(
+    GenMode     gen_mode,
+    ScanOpT     scan_op,
+    T           identity,
+    T           initial_value)
+{
+    // Exclusive (use identity as initial value because it will dispatch to *Sum variants that don't take initial values)
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, EXCLUSIVE, BASIC>(gen_mode, scan_op, identity);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, EXCLUSIVE, AGGREGATE>(gen_mode, scan_op, identity);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, EXCLUSIVE, PREFIX>(gen_mode, scan_op, identity);
+
+    // Exclusive (non-specialized, so we can use initial-value)
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, EXCLUSIVE, BASIC>(gen_mode, WrapperFunctor<ScanOpT>(scan_op), initial_value);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, EXCLUSIVE, AGGREGATE>(gen_mode, WrapperFunctor<ScanOpT>(scan_op), initial_value);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, EXCLUSIVE, PREFIX>(gen_mode, WrapperFunctor<ScanOpT>(scan_op), initial_value);
+
+    // Inclusive
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, INCLUSIVE, BASIC>(gen_mode, scan_op, identity);      // This scan doesn't take an initial value
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, INCLUSIVE, AGGREGATE>(gen_mode, scan_op, identity);  // This scan doesn't take an initial value
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD, INCLUSIVE, PREFIX>(gen_mode, scan_op, initial_value);
+}
+
+
+/**
+ * Run tests for different problem-generation options
+ */
+template <
+    int         BLOCK_THREADS,
+    int         ITEMS_PER_THREAD,
+    typename    ScanOpT,
+    typename    T>
+void Test(
+    ScanOpT     scan_op,
+    T           identity,
+    T           initial_value)
+{
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(UNIFORM, scan_op, identity, initial_value);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(INTEGER_SEED, scan_op, identity, initial_value);
+
+    // Don't test randomly-generated floats b/c of stability
+    if (Traits<T>::CATEGORY != FLOATING_POINT)
+        Test<BLOCK_THREADS, ITEMS_PER_THREAD>(RANDOM, scan_op, identity, initial_value);
+}
+
+
+/**
+ * Run tests for different data types and scan ops
+ */
+template <
+    int BLOCK_THREADS,
+    int ITEMS_PER_THREAD>
+void Test()
+{
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+    // primitive
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), (unsigned char) 0, (unsigned char) 99);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), (unsigned short) 0, (unsigned short) 99);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), (unsigned int) 0, (unsigned int) 99);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), (unsigned long long) 0, (unsigned long long) 99);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), (float) 0, (float) 99);
+
+    // primitive (alternative scan op)
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Max(), std::numeric_limits<char>::min(), (char) 99);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Max(), std::numeric_limits<short>::min(), (short) 99);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Max(), std::numeric_limits<int>::min(), (int) 99);
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Max(), std::numeric_limits<long long>::min(), (long long) 99);
+
+    if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+        Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Max(), std::numeric_limits<double>::max() * -1, (double) 99);
+
+    // vec-1
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_uchar1(0), make_uchar1(17));
+
+    // vec-2
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_uchar2(0, 0), make_uchar2(17, 21));
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_ushort2(0, 0), make_ushort2(17, 21));
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_uint2(0, 0), make_uint2(17, 21));
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_ulonglong2(0, 0), make_ulonglong2(17, 21));
+
+    // vec-4
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_char4(0, 0, 0, 0), make_char4(17, 21, 32, 85));
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_short4(0, 0, 0, 0), make_short4(17, 21, 32, 85));
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_int4(0, 0, 0, 0), make_int4(17, 21, 32, 85));
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), make_longlong4(0, 0, 0, 0), make_longlong4(17, 21, 32, 85));
+
+    // complex
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), TestFoo::MakeTestFoo(0, 0, 0, 0), TestFoo::MakeTestFoo(17, 21, 32, 85));
+    Test<BLOCK_THREADS, ITEMS_PER_THREAD>(Sum(), TestBar(0, 0), TestBar(17, 21));
+
+}
+
+
+/**
+ * Run tests for different items per thread
+ */
+template <int BLOCK_THREADS>
+void Test()
+{
+    Test<BLOCK_THREADS, 1>();
+    Test<BLOCK_THREADS, 2>();
+    Test<BLOCK_THREADS, 9>();
+}
+
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("repeat", g_repeat);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+#ifdef QUICK_TEST
+
+    Test<128, 1, 1, 1, EXCLUSIVE, AGGREGATE, BLOCK_SCAN_WARP_SCANS>(UNIFORM, Sum(), int(0));
+
+    // Compile/run quick tests
+    Test<128, 1, 1, 4, EXCLUSIVE, AGGREGATE, BLOCK_SCAN_WARP_SCANS>(UNIFORM, Sum(), int(0));
+    Test<128, 1, 1, 4, EXCLUSIVE, AGGREGATE, BLOCK_SCAN_RAKING>(UNIFORM, Sum(), int(0));
+    Test<128, 1, 1, 4, EXCLUSIVE, AGGREGATE, BLOCK_SCAN_RAKING_MEMOIZE>(UNIFORM, Sum(), int(0));
+
+    Test<128, 1, 1, 2, INCLUSIVE, PREFIX, BLOCK_SCAN_RAKING>(INTEGER_SEED, Sum(), TestFoo::MakeTestFoo(17, 21, 32, 85));
+    Test<128, 1, 1, 1, EXCLUSIVE, AGGREGATE, BLOCK_SCAN_WARP_SCANS>(UNIFORM, Sum(), make_longlong4(17, 21, 32, 85));
+
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // Run tests for different thread block sizes
+        Test<17>();
+        Test<32>();
+        Test<62>();
+        Test<65>();
+//            Test<96>();             // TODO: file bug for UNREACHABLE error for Test<96, 9, BASIC, BLOCK_SCAN_RAKING>(UNIFORM, Sum(), NullType(), make_ulonglong2(17, 21));
+        Test<128>();
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
+
diff --git a/third_party/cub/test/test_device_histogram.cu b/third_party/cub/test/test_device_histogram.cu
new file mode 100644
index 0000000..da4e7db
--- /dev/null
+++ b/third_party/cub/test/test_device_histogram.cu
@@ -0,0 +1,1669 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of DeviceHistogram utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <limits>
+#include <algorithm>
+#include <typeinfo>
+
+#if defined(QUICK_TEST) || defined(QUICKER_TEST)
+    #include <npp.h>
+#endif
+
+#include <cub/util_allocator.cuh>
+#include <cub/iterator/constant_input_iterator.cuh>
+#include <cub/device/device_histogram.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+
+// Dispatch types
+enum Backend
+{
+    CUB,        // CUB method
+    NPP,        // NPP method
+    CDP,        // GPU-based (dynamic parallelism) dispatch to CUB method
+};
+
+
+bool                    g_verbose_input     = false;
+bool                    g_verbose           = false;
+int                     g_timing_iterations = 0;
+int                     g_repeat            = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+
+
+
+//---------------------------------------------------------------------
+// Dispatch to NPP histogram
+//---------------------------------------------------------------------
+
+#if defined(QUICK_TEST) || defined(QUICKER_TEST)
+
+/**
+ * Dispatch to single-channel 8b NPP histo-even
+ */
+template <typename CounterT, typename LevelT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t DispatchEven(
+    Int2Type<1>             num_channels,
+    Int2Type<1>             num_active_channels,
+    Int2Type<NPP>           dispatch_to,
+    int                     timing_timing_iterations,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    unsigned char       *d_samples,               ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+    CounterT            *d_histogram[1],          ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+    int                 num_levels[1],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT              lower_level[1],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    LevelT              upper_level[1],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+    OffsetT             num_row_pixels,           ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT             num_rows,                 ///< [in] The number of rows in the region of interest
+    OffsetT             row_stride_bytes,         ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    typedef unsigned char SampleT;
+
+    cudaError_t error = cudaSuccess;
+    NppiSize oSizeROI = {
+        num_row_pixels,
+        num_rows
+    };
+
+    if (d_temp_storage_bytes == NULL)
+    {
+        int nDeviceBufferSize;
+        nppiHistogramEvenGetBufferSize_8u_C1R(oSizeROI, num_levels[0] ,&nDeviceBufferSize);
+        temp_storage_bytes = nDeviceBufferSize;
+    }
+    else
+    {
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            // compute the histogram
+            nppiHistogramEven_8u_C1R(
+                d_samples,
+                row_stride_bytes,
+                oSizeROI,
+                d_histogram[0],
+                num_levels[0],
+                lower_level[0],
+                upper_level[0],
+                (Npp8u*) d_temp_storage);
+        }
+    }
+
+    return error;
+}
+
+
+/**
+ * Dispatch to 3/4 8b NPP histo-even
+ */
+template <typename CounterT, typename LevelT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t DispatchEven(
+    Int2Type<4>          num_channels,
+    Int2Type<3>   num_active_channels,
+    Int2Type<NPP>           dispatch_to,
+    int                     timing_timing_iterations,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    unsigned char       *d_samples,               ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+    CounterT            *d_histogram[3],          ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+    int                 num_levels[3],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT              lower_level[3],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    LevelT              upper_level[3],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+    OffsetT             num_row_pixels,           ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT             num_rows,                 ///< [in] The number of rows in the region of interest
+    OffsetT             row_stride_bytes,         ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    typedef unsigned char SampleT;
+
+    cudaError_t error = cudaSuccess;
+    NppiSize oSizeROI = {
+        num_row_pixels,
+        num_rows
+    };
+
+    if (d_temp_storage_bytes == NULL)
+    {
+        int nDeviceBufferSize;
+        nppiHistogramEvenGetBufferSize_8u_AC4R(oSizeROI, num_levels ,&nDeviceBufferSize);
+        temp_storage_bytes = nDeviceBufferSize;
+    }
+    else
+    {
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            // compute the histogram
+            nppiHistogramEven_8u_AC4R(
+                d_samples,
+                row_stride_bytes,
+                oSizeROI,
+                d_histogram,
+                num_levels,
+                lower_level,
+                upper_level,
+                (Npp8u*) d_temp_storage);
+        }
+    }
+
+    return error;
+}
+
+
+#endif // #if defined(QUICK_TEST) || defined(QUICKER_TEST)
+
+
+//---------------------------------------------------------------------
+// Dispatch to different DeviceHistogram entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to CUB single histogram-even entrypoint
+ */
+template <typename SampleIteratorT, typename CounterT, typename LevelT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t DispatchEven(
+    Int2Type<1>             num_channels,
+    Int2Type<1>             num_active_channels,
+    Int2Type<CUB>           dispatch_to,
+    int                     timing_timing_iterations,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+    CounterT            *d_histogram[1],                            ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+    int                 num_levels[1],                              ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT              lower_level[1],                             ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    LevelT              upper_level[1],                             ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+    OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+    OffsetT             row_stride_bytes,                                 ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceHistogram::HistogramEven(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram[0],
+            num_levels[0],
+            lower_level[0],
+            upper_level[0],
+            num_row_pixels,
+            num_rows,
+            row_stride_bytes,
+            stream,
+            debug_synchronous);
+    }
+    return error;
+}
+
+/**
+ * Dispatch to CUB multi histogram-even entrypoint
+ */
+template <int NUM_ACTIVE_CHANNELS, int NUM_CHANNELS, typename SampleIteratorT, typename CounterT, typename LevelT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t DispatchEven(
+    Int2Type<NUM_CHANNELS>          num_channels,
+    Int2Type<NUM_ACTIVE_CHANNELS>   num_active_channels,
+    Int2Type<CUB>           dispatch_to,
+    int                     timing_timing_iterations,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+    CounterT            *d_histogram[NUM_ACTIVE_CHANNELS],          ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+    int                 num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+    OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+    OffsetT             row_stride_bytes,                                 ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceHistogram::MultiHistogramEven<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram,
+            num_levels,
+            lower_level,
+            upper_level,
+            num_row_pixels,
+            num_rows,
+            row_stride_bytes,
+            stream,
+            debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to CUB single histogram-range entrypoint
+ */
+template <typename SampleIteratorT, typename CounterT, typename LevelT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t DispatchRange(
+    Int2Type<1>             num_channels,
+    Int2Type<1>             num_active_channels,
+    Int2Type<CUB>           dispatch_to,
+    int                     timing_timing_iterations,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+    CounterT            *d_histogram[1],                            ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+    int                 num_levels[1],                              ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT              *d_levels[1],                               ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
+    OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+    OffsetT             row_stride_bytes,                                 ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceHistogram::HistogramRange(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram[0],
+            num_levels[0],
+            d_levels[0],
+            num_row_pixels,
+            num_rows,
+            row_stride_bytes,
+            stream,
+            debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to CUB multi histogram-range entrypoint
+ */
+template <int NUM_ACTIVE_CHANNELS, int NUM_CHANNELS, typename SampleIteratorT, typename CounterT, typename LevelT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t DispatchRange(
+    Int2Type<NUM_CHANNELS>          num_channels,
+    Int2Type<NUM_ACTIVE_CHANNELS>   num_active_channels,
+    Int2Type<CUB>           dispatch_to,
+    int                     timing_timing_iterations,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
+    CounterT            *d_histogram[NUM_ACTIVE_CHANNELS],          ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+    int                 num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT              *d_levels[NUM_ACTIVE_CHANNELS],             ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
+    OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
+    OffsetT             row_stride_bytes,                                 ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceHistogram::MultiHistogramRange<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_samples,
+            d_histogram,
+            num_levels,
+            d_levels,
+            num_row_pixels,
+            num_rows,
+            row_stride_bytes,
+            stream,
+            debug_synchronous);
+    }
+    return error;
+}
+
+
+
+//---------------------------------------------------------------------
+// CUDA nested-parallelism test kernel
+//---------------------------------------------------------------------
+
+/**
+ * Simple wrapper kernel to invoke DeviceHistogram
+ * /
+template <int BINS, int NUM_CHANNELS, int NUM_ACTIVE_CHANNELS, typename SampleT, typename SampleIteratorT, typename CounterT, int ALGORITHM>
+__global__ void CnpDispatchKernel(
+    Int2Type<ALGORITHM> algorithm,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t              temp_storage_bytes,
+    SampleT             *d_samples,
+    SampleIteratorT      d_sample_itr,
+    ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS> d_out_histograms,
+    int                 num_samples,
+    bool                debug_synchronous)
+{
+#ifndef CUB_CDP
+    *d_cdp_error = cudaErrorNotSupported;
+#else
+    *d_cdp_error = Dispatch<BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(algorithm, Int2Type<false>(), timing_timing_iterations, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_samples, d_sample_itr, d_out_histograms.array, num_samples, 0, debug_synchronous);
+    *d_temp_storage_bytes = temp_storage_bytes;
+#endif
+}
+
+
+/ **
+ * Dispatch to CDP kernel
+ * /
+template <int BINS, int NUM_CHANNELS, int NUM_ACTIVE_CHANNELS, typename SampleT, typename SampleIteratorT, typename CounterT, int ALGORITHM>
+cudaError_t Dispatch(
+    Int2Type<ALGORITHM> algorithm,
+    Int2Type<true>      use_cdp,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    SampleT             *d_samples,
+    SampleIteratorT      d_sample_itr,
+    CounterT        *d_histograms[NUM_ACTIVE_CHANNELS],
+    int                 num_samples,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Setup array wrapper for histogram channel output (because we can't pass static arrays as kernel parameters)
+    ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS> d_histo_wrapper;
+    for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+        d_histo_wrapper.array[CHANNEL] = d_histograms[CHANNEL];
+
+    // Invoke kernel to invoke device-side dispatch
+    CnpDispatchKernel<BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, ALGORITHM><<<1,1>>>(algorithm, timing_timing_iterations, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_samples, d_sample_itr, d_histo_wrapper, num_samples, debug_synchronous);
+
+    // Copy out temp_storage_bytes
+    CubDebugExit(cudaMemcpy(&temp_storage_bytes, d_temp_storage_bytes, sizeof(size_t) * 1, cudaMemcpyDeviceToHost));
+
+    // Copy out error
+    cudaError_t retval;
+    CubDebugExit(cudaMemcpy(&retval, d_cdp_error, sizeof(cudaError_t) * 1, cudaMemcpyDeviceToHost));
+    return retval;
+}
+*/
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+// Searches for bin given a list of bin-boundary levels
+template <typename LevelT>
+struct SearchTransform
+{
+    LevelT          *levels;      // Pointer to levels array
+    int             num_levels;   // Number of levels in array
+
+    // Functor for converting samples to bin-ids (num_levels is returned if sample is out of range)
+    template <typename SampleT>
+    int operator()(SampleT sample)
+    {
+        int bin = int(std::upper_bound(levels, levels + num_levels, (LevelT) sample) - levels - 1);
+        if (bin < 0)
+        {
+            // Sample out of range
+            return num_levels;
+        }
+        return bin;
+    }
+};
+
+
+// Scales samples to evenly-spaced bins
+template <typename LevelT>
+struct ScaleTransform
+{
+    int    num_levels;  // Number of levels in array
+    LevelT max;         // Max sample level (exclusive)
+    LevelT min;         // Min sample level (inclusive)
+    LevelT scale;       // Bin scaling factor
+
+    void Init(
+        int    num_levels,  // Number of levels in array
+        LevelT max,         // Max sample level (exclusive)
+        LevelT min,         // Min sample level (inclusive)
+        LevelT scale)       // Bin scaling factor
+    {
+        this->num_levels = num_levels;
+        this->max = max;
+        this->min = min;
+        this->scale = scale;
+    }
+
+    // Functor for converting samples to bin-ids  (num_levels is returned if sample is out of range)
+    template <typename SampleT>
+    int operator()(SampleT sample)
+    {
+        if ((sample < min) || (sample >= max))
+        {
+            // Sample out of range
+            return num_levels;
+        }
+
+        return (int) ((((LevelT) sample) - min) / scale);
+    }
+};
+
+// Scales samples to evenly-spaced bins
+template <>
+struct ScaleTransform<float>
+{
+    int   num_levels;  // Number of levels in array
+    float max;         // Max sample level (exclusive)
+    float min;         // Min sample level (inclusive)
+    float scale;       // Bin scaling factor
+
+    void Init(
+        int    num_levels,  // Number of levels in array
+        float max,         // Max sample level (exclusive)
+        float min,         // Min sample level (inclusive)
+        float scale)       // Bin scaling factor
+    {
+        this->num_levels = num_levels;
+        this->max = max;
+        this->min = min;
+        this->scale = 1.0f / scale;
+    }
+
+    // Functor for converting samples to bin-ids  (num_levels is returned if sample is out of range)
+    template <typename SampleT>
+    int operator()(SampleT sample)
+    {
+        if ((sample < min) || (sample >= max))
+        {
+            // Sample out of range
+            return num_levels;
+        }
+
+        return (int) ((((float) sample) - min) * scale);
+    }
+};
+
+
+/**
+ * Generate sample
+ */
+template <typename T, typename LevelT>
+void Sample(T &datum, LevelT max_level, int entropy_reduction)
+{
+    unsigned int max = (unsigned int) -1;
+    unsigned int bits;
+    RandomBits(bits, entropy_reduction);
+    float fraction = (float(bits) / max);
+
+    datum = (T) (fraction * max_level);
+}
+
+
+/**
+ * Initialize histogram samples
+ */
+template <
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        LevelT,
+    typename        SampleT,
+    typename        OffsetT>
+void InitializeSamples(
+    LevelT          max_level,
+    int             entropy_reduction,
+    SampleT         *h_samples,
+    OffsetT         num_row_pixels,         ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT         num_rows,               ///< [in] The number of rows in the region of interest
+    OffsetT         row_stride_bytes)       ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+{
+    // Initialize samples
+    for (OffsetT row = 0; row < num_rows; ++row)
+    {
+        for (OffsetT pixel = 0; pixel < num_row_pixels; ++pixel)
+        {
+            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+            {
+                // Sample offset
+                OffsetT offset = (row * (row_stride_bytes / sizeof(SampleT))) + (pixel * NUM_CHANNELS) + channel;
+
+                // Init sample value
+                Sample(h_samples[offset], max_level, entropy_reduction);
+                if (g_verbose_input)
+                {
+                    if (channel > 0) printf(", ");
+                    std::cout << CoutCast(h_samples[offset]);
+                }
+            }
+        }
+    }
+}
+
+
+/**
+ * Initialize histogram solutions
+ */
+template <
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        CounterT,
+    typename        SampleIteratorT,
+    typename        TransformOp,
+    typename        OffsetT>
+void InitializeBins(
+    SampleIteratorT h_samples,
+    int             num_levels[NUM_ACTIVE_CHANNELS],        ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    TransformOp     transform_op[NUM_ACTIVE_CHANNELS],      ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    CounterT        *h_histogram[NUM_ACTIVE_CHANNELS],      ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_levels[i]</tt> - 1.
+    OffsetT         num_row_pixels,                         ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT         num_rows,                               ///< [in] The number of rows in the region of interest
+    OffsetT         row_stride_bytes)                       ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+{
+    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
+
+    // Init bins
+    for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+    {
+        for (int bin = 0; bin < num_levels[CHANNEL] - 1; ++bin)
+        {
+            h_histogram[CHANNEL][bin] = 0;
+        }
+    }
+
+    // Initialize samples
+    if (g_verbose_input) printf("Samples: \n");
+    for (OffsetT row = 0; row < num_rows; ++row)
+    {
+        for (OffsetT pixel = 0; pixel < num_row_pixels; ++pixel)
+        {
+            if (g_verbose_input) printf("[");
+            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+            {
+                // Sample offset
+                OffsetT offset = (row * (row_stride_bytes / sizeof(SampleT))) + (pixel * NUM_CHANNELS) + channel;
+
+                // Update sample bin
+                int bin = transform_op[channel](h_samples[offset]);
+                if (g_verbose_input) printf(" (%d)", bin); fflush(stdout);
+                if ((bin >= 0) && (bin < num_levels[channel] - 1))
+                {
+                    // valid bin
+                    h_histogram[channel][bin]++;
+                }
+            }
+            if (g_verbose_input) printf("]");
+        }
+        if (g_verbose_input) printf("\n\n");
+    }
+}
+
+
+
+/**
+ * Test histogram-even
+ */
+template <
+    Backend         BACKEND,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        SampleT,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT,
+    typename        SampleIteratorT>
+void TestEven(
+    LevelT          max_level,
+    int             entropy_reduction,
+    int             num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT          lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    LevelT          upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+    OffsetT         num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT         num_rows,                                   ///< [in] The number of rows in the region of interest
+    OffsetT         row_stride_bytes,                           ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+    SampleIteratorT h_samples,
+    SampleIteratorT d_samples)
+{
+    OffsetT total_samples = num_rows * (row_stride_bytes / sizeof(SampleT));
+
+    printf("\n----------------------------\n");
+    printf("%s cub::DeviceHistogramEven (%s) %d pixels (%d height, %d width, %d-byte row stride), %d %d-byte %s samples (entropy reduction %d), %s counters, %d/%d channels, max sample ",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == NPP) ? "NPP" : "CUB",
+        (IsPointer<SampleIteratorT>::VALUE) ? "pointer" : "iterator",
+        (int) (num_row_pixels * num_rows),
+        (int) num_rows,
+        (int) num_row_pixels,
+        (int) row_stride_bytes,
+        (int) total_samples,
+        (int) sizeof(SampleT),
+        typeid(SampleT).name(),
+        entropy_reduction,
+        typeid(CounterT).name(),
+        NUM_ACTIVE_CHANNELS,
+        NUM_CHANNELS);
+    std::cout << CoutCast(max_level) << "\n";
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+        std::cout << "\n\tChannel " << channel << ": " << num_levels[channel] - 1 << " bins [" << lower_level[channel] << ", " << upper_level[channel] << ")\n";
+    fflush(stdout);
+
+    // Allocate and initialize host and device data
+
+    typedef SampleT Foo;        // rename type to quelch gcc warnings (bug?)
+    CounterT*                   h_histogram[NUM_ACTIVE_CHANNELS];
+    ScaleTransform<LevelT>      transform_op[NUM_ACTIVE_CHANNELS];
+
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        int bins = num_levels[channel] - 1;
+        h_histogram[channel] = new CounterT[bins];
+
+        transform_op[channel].Init(
+            num_levels[channel],
+            upper_level[channel],
+            lower_level[channel],
+            ((upper_level[channel] - lower_level[channel]) / bins));
+    }
+
+    InitializeBins<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
+        h_samples, num_levels, transform_op, h_histogram, num_row_pixels, num_rows, row_stride_bytes);
+
+    // Allocate and initialize device data
+
+    CounterT* d_histogram[NUM_ACTIVE_CHANNELS];
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_histogram[channel], sizeof(CounterT) * (num_levels[channel] - 1)));
+        CubDebugExit(cudaMemset(d_histogram[channel], 0, sizeof(CounterT) * (num_levels[channel] - 1)));
+    }
+
+    // Allocate CDP device arrays
+    size_t          *d_temp_storage_bytes = NULL;
+    cudaError_t     *d_cdp_error = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes,  sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error,           sizeof(cudaError_t) * 1));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+
+    DispatchEven(
+        Int2Type<NUM_CHANNELS>(), Int2Type<NUM_ACTIVE_CHANNELS>(), Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes,
+        d_samples, d_histogram, num_levels, lower_level, upper_level,
+        num_row_pixels, num_rows, row_stride_bytes,
+        0, true);
+
+    // Allocate temporary storage with "canary" zones
+    int     canary_bytes    = 256;
+    char    canary_token    = 8;
+    char*   canary_zone     = new char[canary_bytes];
+
+    memset(canary_zone, canary_token, canary_bytes);
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes + (canary_bytes * 2)));
+    CubDebugExit(cudaMemset(d_temp_storage, canary_token, temp_storage_bytes + (canary_bytes * 2)));
+
+    // Run warmup/correctness iteration
+    DispatchEven(
+        Int2Type<NUM_CHANNELS>(), Int2Type<NUM_ACTIVE_CHANNELS>(), Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error,
+        ((char *) d_temp_storage) + canary_bytes, temp_storage_bytes,
+        d_samples, d_histogram, num_levels, lower_level, upper_level,
+        num_row_pixels, num_rows, row_stride_bytes,
+        0, true);
+
+    // Check canary zones
+    int error = CompareDeviceResults(canary_zone, (char *) d_temp_storage, canary_bytes, true, g_verbose);
+    AssertEquals(0, error);
+    error = CompareDeviceResults(canary_zone, ((char *) d_temp_storage) + canary_bytes + temp_storage_bytes, canary_bytes, true, g_verbose);
+    AssertEquals(0, error);
+
+    // Flush any stdout/stderr
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+    fflush(stdout);
+    fflush(stderr);
+
+    // Check for correctness (and display results, if specified)
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        int channel_error = CompareDeviceResults(h_histogram[channel], d_histogram[channel], num_levels[channel] - 1, true, g_verbose);
+        printf("\tChannel %d %s", channel, channel_error ? "FAIL" : "PASS\n");
+        error |= channel_error;
+    }
+
+    // Performance
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+
+    DispatchEven(
+        Int2Type<NUM_CHANNELS>(), Int2Type<NUM_ACTIVE_CHANNELS>(), Int2Type<BACKEND>(), g_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes,
+        d_samples, d_histogram, num_levels, lower_level, upper_level,
+        num_row_pixels, num_rows, row_stride_bytes,
+        0, false);
+
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float avg_millis = elapsed_millis / g_timing_iterations;
+        float giga_rate = float(total_samples) / avg_millis / 1000.0f / 1000.0f;
+        float giga_bandwidth = giga_rate * sizeof(SampleT);
+        printf("\t%.3f avg ms, %.3f billion samples/s, %.3f billion bins/s, %.3f billion pixels/s, %.3f logical GB/s",
+            avg_millis,
+            giga_rate,
+            giga_rate * NUM_ACTIVE_CHANNELS / NUM_CHANNELS,
+            giga_rate / NUM_CHANNELS,
+            giga_bandwidth);
+    }
+
+    printf("\n\n");
+
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        if (h_histogram[channel])
+            delete[] h_histogram[channel];
+
+        if (d_histogram[channel])
+            CubDebugExit(g_allocator.DeviceFree(d_histogram[channel]));
+    }
+
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Correctness asserts
+    AssertEquals(0, error);
+}
+
+
+/**
+ * Test histogram-even (native pointer input)
+ */
+template <
+    Backend         BACKEND,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        SampleT,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void TestEvenNative(
+    LevelT          max_level,
+    int             entropy_reduction,
+    int             num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT          lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    LevelT          upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+    OffsetT         num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT         num_rows,                                   ///< [in] The number of rows in the region of interest
+    OffsetT         row_stride_bytes)                                 ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+{
+    OffsetT total_samples = num_rows * (row_stride_bytes / sizeof(SampleT));
+
+    // Allocate and initialize host sample data
+    typedef SampleT Foo;        // rename type to quelch gcc warnings (bug?)
+    SampleT*                    h_samples = new Foo[total_samples];
+
+    InitializeSamples<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
+        max_level, entropy_reduction, h_samples, num_row_pixels, num_rows, row_stride_bytes);
+
+    // Allocate and initialize device data
+    SampleT* d_samples = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_samples, sizeof(SampleT) * total_samples));
+    CubDebugExit(cudaMemcpy(d_samples, h_samples, sizeof(SampleT) * total_samples, cudaMemcpyHostToDevice));
+
+    TestEven<BACKEND, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleT, CounterT, LevelT, OffsetT>(
+        max_level, entropy_reduction, num_levels, lower_level, upper_level,
+        num_row_pixels, num_rows, row_stride_bytes,
+        h_samples, d_samples);
+
+    // Cleanup
+    if (h_samples) delete[] h_samples;
+    if (d_samples) CubDebugExit(g_allocator.DeviceFree(d_samples));
+}
+
+
+/**
+ * Test histogram-even (native pointer input)
+ */
+template <
+    Backend         BACKEND,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        SampleT,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void TestEvenIterator(
+    LevelT          max_level,
+    int             entropy_reduction,
+    int             num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT          lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    LevelT          upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
+    OffsetT         num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT         num_rows,                                   ///< [in] The number of rows in the region of interest
+    OffsetT         row_stride_bytes)                                 ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+{
+    SampleT sample = (SampleT) lower_level[0];
+    ConstantInputIterator<SampleT> sample_itr(sample);
+
+    TestEven<BACKEND, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleT, CounterT, LevelT, OffsetT>(
+        max_level, entropy_reduction, num_levels, lower_level, upper_level,
+        num_row_pixels, num_rows, row_stride_bytes,
+        sample_itr, sample_itr);
+
+}
+
+
+/**
+ * Test histogram-range
+ */
+template <
+    Backend         BACKEND,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        SampleT,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void TestRange(
+    LevelT          max_level,
+    int             entropy_reduction,
+    int             num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
+    LevelT*         levels[NUM_ACTIVE_CHANNELS],                ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
+    OffsetT         num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
+    OffsetT         num_rows,                                   ///< [in] The number of rows in the region of interest
+    OffsetT         row_stride_bytes)                                 ///< [in] The number of bytes between starts of consecutive rows in the region of interest
+{
+    OffsetT total_samples = num_rows * (row_stride_bytes / sizeof(SampleT));
+
+    printf("\n----------------------------\n");
+    printf("%s cub::DeviceHistogramRange %d pixels (%d height, %d width, %d-byte row stride), %d %d-byte %s samples (entropy reduction %d), %s counters, %d/%d channels, max sample ",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == NPP) ? "NPP" : "CUB",
+        (int) (num_row_pixels * num_rows),
+        (int) num_rows,
+        (int) num_row_pixels,
+        (int) row_stride_bytes,
+        (int) total_samples,
+        (int) sizeof(SampleT),
+        typeid(SampleT).name(),
+        entropy_reduction,
+        typeid(CounterT).name(),
+        NUM_ACTIVE_CHANNELS,
+        NUM_CHANNELS);
+    std::cout << CoutCast(max_level) << "\n";
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        printf("Channel %d: %d bins [", channel, num_levels[channel] - 1);
+        std::cout << levels[channel][0];
+        for (int level = 1; level < num_levels[channel]; ++level)
+            std::cout << ", " << levels[channel][level];
+        printf("]\n");
+    }
+    fflush(stdout);
+
+    // Allocate and initialize host and device data
+    typedef SampleT Foo;        // rename type to quelch gcc warnings (bug?)
+    SampleT*                    h_samples = new Foo[total_samples];
+    CounterT*                   h_histogram[NUM_ACTIVE_CHANNELS];
+    SearchTransform<LevelT>     transform_op[NUM_ACTIVE_CHANNELS];
+
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        transform_op[channel].levels = levels[channel];
+        transform_op[channel].num_levels = num_levels[channel];
+
+        int bins = num_levels[channel] - 1;
+        h_histogram[channel] = new CounterT[bins];
+    }
+
+    InitializeSamples<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
+        max_level, entropy_reduction, h_samples, num_row_pixels, num_rows, row_stride_bytes);
+
+    InitializeBins<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
+        h_samples, num_levels, transform_op, h_histogram, num_row_pixels, num_rows, row_stride_bytes);
+
+    // Allocate and initialize device data
+    SampleT*        d_samples = NULL;
+    LevelT*         d_levels[NUM_ACTIVE_CHANNELS];
+    CounterT*       d_histogram[NUM_ACTIVE_CHANNELS];
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_samples, sizeof(SampleT) * total_samples));
+    CubDebugExit(cudaMemcpy(d_samples, h_samples, sizeof(SampleT) * total_samples, cudaMemcpyHostToDevice));
+
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_levels[channel], sizeof(LevelT) * num_levels[channel]));
+        CubDebugExit(cudaMemcpy(d_levels[channel], levels[channel],         sizeof(LevelT) * num_levels[channel], cudaMemcpyHostToDevice));
+
+        int bins = num_levels[channel] - 1;
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_histogram[channel],  sizeof(CounterT) * bins));
+        CubDebugExit(cudaMemset(d_histogram[channel], 0,                        sizeof(CounterT) * bins));
+    }
+
+    // Allocate CDP device arrays
+    size_t          *d_temp_storage_bytes = NULL;
+    cudaError_t     *d_cdp_error = NULL;
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes,  sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error,           sizeof(cudaError_t) * 1));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+
+    DispatchRange(
+        Int2Type<NUM_CHANNELS>(), Int2Type<NUM_ACTIVE_CHANNELS>(), Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes,
+        d_samples, d_histogram, num_levels, d_levels,
+        num_row_pixels, num_rows, row_stride_bytes,
+        0, true);
+
+    // Allocate temporary storage with "canary" zones
+    int     canary_bytes    = 256;
+    char    canary_token    = 9;
+    char*   canary_zone     = new char[canary_bytes];
+
+    memset(canary_zone, canary_token, canary_bytes);
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes + (canary_bytes * 2)));
+    CubDebugExit(cudaMemset(d_temp_storage, canary_token, temp_storage_bytes + (canary_bytes * 2)));
+
+    // Run warmup/correctness iteration
+    DispatchRange(
+        Int2Type<NUM_CHANNELS>(), Int2Type<NUM_ACTIVE_CHANNELS>(), Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error,
+        ((char *) d_temp_storage) + canary_bytes, temp_storage_bytes,
+        d_samples, d_histogram, num_levels, d_levels,
+        num_row_pixels, num_rows, row_stride_bytes,
+        0, true);
+
+    // Check canary zones
+    int error = CompareDeviceResults(canary_zone, (char *) d_temp_storage, canary_bytes, true, g_verbose);
+    AssertEquals(0, error);
+    error = CompareDeviceResults(canary_zone, ((char *) d_temp_storage) + canary_bytes + temp_storage_bytes, canary_bytes, true, g_verbose);
+    AssertEquals(0, error);
+
+    // Flush any stdout/stderr
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+    fflush(stdout);
+    fflush(stderr);
+
+    // Check for correctness (and display results, if specified)
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        int channel_error = CompareDeviceResults(h_histogram[channel], d_histogram[channel], num_levels[channel] - 1, true, g_verbose);
+        printf("\tChannel %d %s", channel, channel_error ? "FAIL" : "PASS\n");
+        error |= channel_error;
+    }
+
+    // Performance
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+
+    DispatchRange(
+        Int2Type<NUM_CHANNELS>(), Int2Type<NUM_ACTIVE_CHANNELS>(), Int2Type<BACKEND>(), g_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes,
+        d_samples, d_histogram, num_levels, d_levels,
+        num_row_pixels, num_rows, row_stride_bytes,
+        0, false);
+
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float avg_millis = elapsed_millis / g_timing_iterations;
+        float giga_rate = float(total_samples) / avg_millis / 1000.0f / 1000.0f;
+        float giga_bandwidth = giga_rate * sizeof(SampleT);
+        printf("\t%.3f avg ms, %.3f billion samples/s, %.3f billion bins/s, %.3f billion pixels/s, %.3f logical GB/s",
+            avg_millis,
+            giga_rate,
+            giga_rate * NUM_ACTIVE_CHANNELS / NUM_CHANNELS,
+            giga_rate / NUM_CHANNELS,
+            giga_bandwidth);
+    }
+
+    printf("\n\n");
+
+    // Cleanup
+    if (h_samples) delete[] h_samples;
+
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        if (h_histogram[channel])
+            delete[] h_histogram[channel];
+
+        if (d_histogram[channel])
+            CubDebugExit(g_allocator.DeviceFree(d_histogram[channel]));
+
+        if (d_levels[channel])
+            CubDebugExit(g_allocator.DeviceFree(d_levels[channel]));
+    }
+
+    if (d_samples) CubDebugExit(g_allocator.DeviceFree(d_samples));
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Correctness asserts
+    AssertEquals(0, error);
+}
+
+
+/**
+ * Test histogram-even
+ */
+template <
+    Backend         BACKEND,
+    typename        SampleT,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void TestEven(
+    OffsetT         num_row_pixels,
+    OffsetT         num_rows,
+    OffsetT         row_stride_bytes,
+    int             entropy_reduction,
+    int             num_levels[NUM_ACTIVE_CHANNELS],
+    LevelT          max_level,
+    int             max_num_levels)
+{
+    LevelT lower_level[NUM_ACTIVE_CHANNELS];
+    LevelT upper_level[NUM_ACTIVE_CHANNELS];
+
+    // Find smallest level increment
+    int max_bins = max_num_levels - 1;
+    LevelT min_level_increment = max_level / max_bins;
+
+    // Set upper and lower levels for each channel
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        int num_bins = num_levels[channel] - 1;
+        lower_level[channel] = (max_level - (num_bins * min_level_increment)) / 2;
+        upper_level[channel] = (max_level + (num_bins * min_level_increment)) / 2;
+    }
+
+    // Test pointer-based samples
+    TestEvenNative<BACKEND, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleT, CounterT, LevelT, OffsetT>(
+        max_level, entropy_reduction, num_levels, lower_level, upper_level, num_row_pixels, num_rows, row_stride_bytes);
+
+    // Test iterator-based samples (CUB-only)
+    TestEvenIterator<CUB, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleT, CounterT, LevelT, OffsetT>(
+        max_level, entropy_reduction, num_levels, lower_level, upper_level, num_row_pixels, num_rows, row_stride_bytes);
+}
+
+
+
+/**
+ * Test histogram-range
+ */
+template <
+    Backend         BACKEND,
+    typename        SampleT,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void TestRange(
+    OffsetT         num_row_pixels,
+    OffsetT         num_rows,
+    OffsetT         row_stride_bytes,
+    int             entropy_reduction,
+    int             num_levels[NUM_ACTIVE_CHANNELS],
+    LevelT          max_level,
+    int             max_num_levels)
+{
+    // Find smallest level increment
+    int max_bins = max_num_levels - 1;
+    LevelT min_level_increment = max_level / max_bins;
+
+    LevelT* levels[NUM_ACTIVE_CHANNELS];
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        levels[channel] = new LevelT[num_levels[channel]];
+
+        int num_bins = num_levels[channel] - 1;
+        LevelT lower_level = (max_level - (num_bins * min_level_increment)) / 2;
+
+        for (int level = 0; level < num_levels[channel]; ++level)
+            levels[channel][level] = lower_level + (level * min_level_increment);
+    }
+
+    TestRange<BACKEND, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleT, CounterT, LevelT, OffsetT>(
+        max_level, entropy_reduction, num_levels, levels, num_row_pixels, num_rows, row_stride_bytes);
+
+    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+        delete[] levels[channel];
+
+}
+
+
+
+/**
+ * Test different entrypoints
+ */
+template <
+    typename        SampleT,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void Test(
+    OffsetT         num_row_pixels,
+    OffsetT         num_rows,
+    OffsetT         row_stride_bytes,
+    int             entropy_reduction,
+    int             num_levels[NUM_ACTIVE_CHANNELS],
+    LevelT          max_level,
+    int             max_num_levels)
+{
+    TestEven<CUB, SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, max_num_levels);
+
+    TestRange<CUB, SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, max_num_levels);
+}
+
+
+/**
+ * Test different number of levels
+ */
+template <
+    typename        SampleT,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void Test(
+    OffsetT         num_row_pixels,
+    OffsetT         num_rows,
+    OffsetT         row_stride_bytes,
+    int             entropy_reduction,
+    LevelT          max_level,
+    int             max_num_levels)
+{
+    int num_levels[NUM_ACTIVE_CHANNELS];
+
+// Unnecessary testing
+//    // All the same level
+//    for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
+//    {
+//        num_levels[channel] = max_num_levels;
+//    }
+//    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+//        num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, max_num_levels);
+
+    // All different levels
+    num_levels[0] = max_num_levels;
+    for (int channel = 1; channel < NUM_ACTIVE_CHANNELS; ++channel)
+    {
+        num_levels[channel] = (num_levels[channel - 1] / 2) + 1;
+    }
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, max_num_levels);
+}
+
+
+
+/**
+ * Test different entropy-levels
+ */
+template <
+    typename        SampleT,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void Test(
+    OffsetT         num_row_pixels,
+    OffsetT         num_rows,
+    OffsetT         row_stride_bytes,
+    LevelT          max_level,
+    int             max_num_levels)
+{
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        num_row_pixels, num_rows, row_stride_bytes, 0,   max_level, max_num_levels);
+
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        num_row_pixels, num_rows, row_stride_bytes, -1,  max_level, max_num_levels);
+
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        num_row_pixels, num_rows, row_stride_bytes, 5,   max_level, max_num_levels);
+}
+
+
+/**
+ * Test different row strides
+ */
+template <
+    typename        SampleT,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void Test(
+    OffsetT         num_row_pixels,
+    OffsetT         num_rows,
+    LevelT          max_level,
+    int             max_num_levels)
+{
+    OffsetT row_stride_bytes = num_row_pixels * NUM_CHANNELS * sizeof(SampleT);
+
+    // No padding
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        num_row_pixels, num_rows, row_stride_bytes, max_level, max_num_levels);
+
+    // 13 samples padding
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        num_row_pixels, num_rows, row_stride_bytes + (13 * sizeof(SampleT)), max_level, max_num_levels);
+}
+
+
+/**
+ * Test different problem sizes
+ */
+template <
+    typename        SampleT,
+    int             NUM_CHANNELS,
+    int             NUM_ACTIVE_CHANNELS,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void Test(
+    LevelT          max_level,
+    int             max_num_levels)
+{
+    // 0 row/col images
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        OffsetT(1920), OffsetT(0), max_level, max_num_levels);
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        OffsetT(0), OffsetT(0), max_level, max_num_levels);
+
+    // 1080 image
+    Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+        OffsetT(1920), OffsetT(1080), max_level, max_num_levels);
+
+    // Sample different aspect ratios sizes
+    for (OffsetT rows = 1; rows < 1000000; rows *= 1000)
+    {
+        for (OffsetT cols = 1; cols < (1000000 / rows); cols *= 1000)
+        {
+            Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+                cols, rows, max_level, max_num_levels);
+        }
+    }
+
+    // Randomly select linear problem size between 1:10,000,000
+    unsigned int max_int = (unsigned int) -1;
+    for (int i = 0; i < 4; ++i)
+    {
+        unsigned int num_items;
+        RandomBits(num_items);
+        num_items = (unsigned int) ((double(num_items) * double(10000000)) / double(max_int));
+        num_items = CUB_MAX(1, num_items);
+
+        Test<SampleT, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, CounterT, LevelT, OffsetT>(
+            OffsetT(num_items), 1, max_level, max_num_levels);
+    }
+}
+
+
+
+/**
+ * Test different channel interleavings (valid specialiation)
+ */
+template <
+    typename        SampleT,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void TestChannels(
+    LevelT          max_level,
+    int             max_num_levels,
+    Int2Type<true>  is_valid_tag)
+{
+    Test<SampleT, 1, 1, CounterT, LevelT, OffsetT>(max_level, max_num_levels);
+    Test<SampleT, 4, 3, CounterT, LevelT, OffsetT>(max_level, max_num_levels);
+    Test<SampleT, 3, 3, CounterT, LevelT, OffsetT>(max_level, max_num_levels);
+    Test<SampleT, 4, 4, CounterT, LevelT, OffsetT>(max_level, max_num_levels);
+}
+
+
+/**
+ * Test different channel interleavings (invalid specialiation)
+ */
+template <
+    typename        SampleT,
+    typename        CounterT,
+    typename        LevelT,
+    typename        OffsetT>
+void TestChannels(
+    LevelT          max_level,
+    int             max_num_levels,
+    Int2Type<false> is_valid_tag)
+{}
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_row_pixels = -1;
+    int entropy_reduction = 0;
+    int num_rows = 1;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    g_verbose_input = args.CheckCmdLineFlag("v2");
+    args.GetCmdLineArgument("n", num_row_pixels);
+
+    int row_stride_pixels = num_row_pixels;
+
+    args.GetCmdLineArgument("rows", num_rows);
+    args.GetCmdLineArgument("stride", row_stride_pixels);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("repeat", g_repeat);
+    args.GetCmdLineArgument("entropy", entropy_reduction);
+#if defined(QUICK_TEST) || defined(QUICKER_TEST)
+    bool compare_npp = args.CheckCmdLineFlag("npp");
+#endif
+
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<pixels per row> "
+            "[--rows=<number of rows> "
+            "[--stride=<row stride in pixels> "
+            "[--i=<timing iterations> "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--entropy=<entropy-reduction factor (default 0)>]"
+            "[--v] "
+            "[--cdp]"
+            "[--npp]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+    if (num_row_pixels < 0)
+    {
+        num_row_pixels      = 1920 * 1080;
+        row_stride_pixels   = num_row_pixels;
+    }
+
+#if defined(QUICKER_TEST)
+
+    // Compile/run quick tests
+    {
+        // HistogramEven: unsigned char 256 bins
+        typedef unsigned char       SampleT;
+        typedef int                 LevelT;
+
+        LevelT  max_level           = 256;
+        int     num_levels[1]       = {257};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 1;
+
+        TestEven<CUB, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+        if (compare_npp)
+            TestEven<NPP, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+
+#elif defined(QUICK_TEST)
+
+    // Compile/run quick tests
+    {
+        // HistogramEven: unsigned char 256 bins
+        typedef unsigned char       SampleT;
+        typedef int                 LevelT;
+
+        LevelT  max_level           = 256;
+        int     num_levels[1]       = {257};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 1;
+
+        TestEven<CUB, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+        if (compare_npp)
+            TestEven<NPP, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    {
+        // HistogramEven: 4/4 multichannel Unsigned char 256 bins
+        typedef unsigned char       SampleT;
+        typedef int                 LevelT;
+
+        LevelT  max_level           = 256;
+        int     num_levels[4]       = {257, 257, 257, 257};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 4;
+
+        TestEven<CUB, SampleT, 4, 4, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    {
+        // HistogramEven: 3/4 multichannel Unsigned char 256 bins
+        typedef unsigned char       SampleT;
+        typedef int                 LevelT;
+
+        LevelT  max_level           = 256;
+        int     num_levels[3]       = {257, 257, 257};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 4;
+
+        TestEven<CUB, SampleT, 4, 3, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+        if (compare_npp)
+            TestEven<NPP, SampleT, 4, 3, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    {
+        // HistogramEven: short [0,1024] 256 bins
+        typedef unsigned short      SampleT;
+        typedef unsigned short      LevelT;
+
+        LevelT  max_level           = 1024;
+        int     num_levels[1]       = {257};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 1;
+
+        TestEven<CUB, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    {
+        // HistogramEven: float [0,1.0] 256 bins
+        typedef float               SampleT;
+        typedef float               LevelT;
+
+        LevelT  max_level           = 1.0;
+        int     num_levels[1]       = {257};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 1;
+
+        TestEven<CUB, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    {
+        // HistogramEven: 3/4 multichannel float [0,1.0] 256 bins
+        typedef float               SampleT;
+        typedef float               LevelT;
+
+         LevelT  max_level           = 1.0;
+         int     num_levels[3]       = {257, 257, 257};
+         int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 4;
+
+         TestEven<CUB, SampleT, 4, 3, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    {
+        // HistogramRange: signed char 256 bins
+        typedef signed char         SampleT;
+        typedef int                 LevelT;
+
+        LevelT  max_level           = 256;
+        int     num_levels[1]       = {257};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 1;
+
+        TestRange<CUB, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    {
+        // HistogramRange: 3/4 channel, unsigned char, varied bins (256, 128, 64)
+        typedef unsigned char       SampleT;
+        typedef int                 LevelT;
+
+        LevelT  max_level           = 256;
+        int     num_levels[3]       = {257, 129, 65};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 4;
+
+        TestRange<CUB, SampleT, 4, 3, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+    {
+        // HistogramEven: double [0,1.0] 64 bins
+        typedef double              SampleT;
+        typedef double              LevelT;
+
+        LevelT  max_level           = 1.0;
+        int     num_levels[1]       = {65};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 1;
+
+        TestEven<CUB, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+    {
+        // HistogramEven: short [0,1024] 512 bins
+        typedef unsigned short      SampleT;
+        typedef unsigned short      LevelT;
+
+        LevelT  max_level           = 1024;
+        int     num_levels[1]       = {513};
+        int     row_stride_bytes    = sizeof(SampleT) * row_stride_pixels * 1;
+
+        TestEven<CUB, SampleT, 1, 1, int, LevelT, int>(num_row_pixels, num_rows, row_stride_bytes, entropy_reduction, num_levels, max_level, num_levels[0]);
+    }
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        TestChannels <unsigned char,    int, int,   int>(256,   256 + 1, Int2Type<true>());
+        TestChannels <signed char,      int, int,   int>(256,   256 + 1, Int2Type<true>());
+        TestChannels <unsigned short,   int, int,   int>(128,   128 + 1, Int2Type<true>());
+        TestChannels <unsigned short,   int, int,   int>(8192,  8192 + 1, Int2Type<true>());
+        TestChannels <float,            int, float, int>(1.0,   256 + 1, Int2Type<true>());
+
+		// Test down-conversion of size_t offsets to int
+        TestChannels <unsigned char,    int, int,   long long>(256, 256 + 1, Int2Type<(sizeof(size_t) != sizeof(int))>());
+    }
+
+#endif
+
+    return 0;
+}
+
diff --git a/third_party/cub/test/test_device_radix_sort.cu b/third_party/cub/test/test_device_radix_sort.cu
new file mode 100644
index 0000000..b2e387f
--- /dev/null
+++ b/third_party/cub/test/test_device_radix_sort.cu
@@ -0,0 +1,1298 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of DeviceRadixSort utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <algorithm>
+#include <typeinfo>
+
+#if (__CUDACC_VER_MAJOR__ >= 9)
+    #include <cuda_fp16.h>
+#endif
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_radix_sort.cuh>
+#include <cub/device/device_segmented_radix_sort.cuh>
+
+#include "test_util.h"
+
+#include <thrust/device_ptr.h>
+#include <thrust/sort.h>
+#include <thrust/reverse.h>
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose           = false;
+int                     g_timing_iterations = 0;
+int                     g_repeat            = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+// Dispatch types
+enum Backend
+{
+    CUB,                        // CUB method (allows overwriting of input)
+    CUB_NO_OVERWRITE,           // CUB method (disallows overwriting of input)
+
+    CUB_SEGMENTED,              // CUB method (allows overwriting of input)
+    CUB_SEGMENTED_NO_OVERWRITE, // CUB method (disallows overwriting of input)
+
+    THRUST,                     // Thrust method
+    CDP,                        // GPU-based (dynamic parallelism) dispatch to CUB method
+};
+
+
+//---------------------------------------------------------------------
+// Dispatch to different DeviceRadixSort entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to CUB sorting entrypoint (specialized for ascending)
+ */
+template <typename KeyT, typename ValueT>
+CUB_RUNTIME_FUNCTION
+__forceinline__
+cudaError_t Dispatch(
+    Int2Type<false>         is_descending,
+    Int2Type<CUB>           dispatch_to,
+    int                     *d_selector,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*                   d_temp_storage,
+    size_t&                 temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    return DeviceRadixSort::SortPairs(
+        d_temp_storage, temp_storage_bytes,
+        d_keys, d_values,
+        num_items, begin_bit, end_bit, stream, debug_synchronous);
+}
+
+/**
+ * Dispatch to CUB_NO_OVERWRITE sorting entrypoint (specialized for ascending)
+ */
+template <typename KeyT, typename ValueT>
+CUB_RUNTIME_FUNCTION
+__forceinline__
+cudaError_t Dispatch(
+    Int2Type<false>             is_descending,
+    Int2Type<CUB_NO_OVERWRITE>  dispatch_to,
+    int                         *d_selector,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*                   d_temp_storage,
+    size_t&                 temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    KeyT      const *const_keys_itr     = d_keys.Current();
+    ValueT    const *const_values_itr   = d_values.Current();
+
+    cudaError_t retval = DeviceRadixSort::SortPairs(
+        d_temp_storage, temp_storage_bytes,
+        const_keys_itr, d_keys.Alternate(), const_values_itr, d_values.Alternate(),
+        num_items, begin_bit, end_bit, stream, debug_synchronous);
+
+    d_keys.selector ^= 1;
+    d_values.selector ^= 1;
+    return retval;
+}
+
+/**
+ * Dispatch to CUB sorting entrypoint (specialized for descending)
+ */
+template <typename KeyT, typename ValueT>
+CUB_RUNTIME_FUNCTION
+__forceinline__
+cudaError_t Dispatch(
+    Int2Type<true>          is_descending,
+    Int2Type<CUB>           dispatch_to,
+    int                     *d_selector,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*                   d_temp_storage,
+    size_t&                 temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    return DeviceRadixSort::SortPairsDescending(
+        d_temp_storage, temp_storage_bytes,
+        d_keys, d_values,
+        num_items, begin_bit, end_bit, stream, debug_synchronous);
+}
+
+
+/**
+ * Dispatch to CUB_NO_OVERWRITE sorting entrypoint (specialized for descending)
+ */
+template <typename KeyT, typename ValueT>
+CUB_RUNTIME_FUNCTION
+__forceinline__
+cudaError_t Dispatch(
+    Int2Type<true>              is_descending,
+    Int2Type<CUB_NO_OVERWRITE>  dispatch_to,
+    int                         *d_selector,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*                   d_temp_storage,
+    size_t&                 temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    KeyT      const *const_keys_itr     = d_keys.Current();
+    ValueT    const *const_values_itr   = d_values.Current();
+
+    cudaError_t retval = DeviceRadixSort::SortPairsDescending(
+        d_temp_storage, temp_storage_bytes,
+        const_keys_itr, d_keys.Alternate(), const_values_itr, d_values.Alternate(),
+        num_items, begin_bit, end_bit, stream, debug_synchronous);
+
+    d_keys.selector ^= 1;
+    d_values.selector ^= 1;
+    return retval;
+}
+
+//---------------------------------------------------------------------
+// Dispatch to different DeviceRadixSort entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to CUB_SEGMENTED sorting entrypoint (specialized for ascending)
+ */
+template <typename KeyT, typename ValueT>
+CUB_RUNTIME_FUNCTION
+__forceinline__
+cudaError_t Dispatch(
+    Int2Type<false>         is_descending,
+    Int2Type<CUB_SEGMENTED> dispatch_to,
+    int                     *d_selector,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*                   d_temp_storage,
+    size_t&                 temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    return DeviceSegmentedRadixSort::SortPairs(
+        d_temp_storage, temp_storage_bytes,
+        d_keys, d_values,
+        num_items, num_segments, d_segment_offsets, d_segment_offsets + 1,
+        begin_bit, end_bit, stream, debug_synchronous);
+}
+
+/**
+ * Dispatch to CUB_SEGMENTED_NO_OVERWRITE sorting entrypoint (specialized for ascending)
+ */
+template <typename KeyT, typename ValueT>
+CUB_RUNTIME_FUNCTION
+__forceinline__
+cudaError_t Dispatch(
+    Int2Type<false>                         is_descending,
+    Int2Type<CUB_SEGMENTED_NO_OVERWRITE>    dispatch_to,
+    int                                     *d_selector,
+    size_t                                  *d_temp_storage_bytes,
+    cudaError_t                             *d_cdp_error,
+
+    void*                   d_temp_storage,
+    size_t&                 temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    KeyT      const *const_keys_itr     = d_keys.Current();
+    ValueT    const *const_values_itr   = d_values.Current();
+
+    cudaError_t retval = DeviceSegmentedRadixSort::SortPairs(
+        d_temp_storage, temp_storage_bytes,
+        const_keys_itr, d_keys.Alternate(), const_values_itr, d_values.Alternate(),
+        num_items, num_segments, d_segment_offsets, d_segment_offsets + 1,
+        begin_bit, end_bit, stream, debug_synchronous);
+
+    d_keys.selector ^= 1;
+    d_values.selector ^= 1;
+    return retval;
+}
+
+
+/**
+ * Dispatch to CUB_SEGMENTED sorting entrypoint (specialized for descending)
+ */
+template <typename KeyT, typename ValueT>
+CUB_RUNTIME_FUNCTION
+__forceinline__
+cudaError_t Dispatch(
+    Int2Type<true>          is_descending,
+    Int2Type<CUB_SEGMENTED> dispatch_to,
+    int                     *d_selector,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void*                   d_temp_storage,
+    size_t&                 temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    return DeviceSegmentedRadixSort::SortPairsDescending(
+        d_temp_storage, temp_storage_bytes,
+        d_keys, d_values,
+        num_items, num_segments, d_segment_offsets, d_segment_offsets + 1,
+        begin_bit, end_bit, stream, debug_synchronous);
+}
+
+/**
+ * Dispatch to CUB_SEGMENTED_NO_OVERWRITE sorting entrypoint (specialized for descending)
+ */
+template <typename KeyT, typename ValueT>
+CUB_RUNTIME_FUNCTION
+__forceinline__
+cudaError_t Dispatch(
+    Int2Type<true>                          is_descending,
+    Int2Type<CUB_SEGMENTED_NO_OVERWRITE>    dispatch_to,
+    int                                     *d_selector,
+    size_t                                  *d_temp_storage_bytes,
+    cudaError_t                             *d_cdp_error,
+
+    void*                   d_temp_storage,
+    size_t&                 temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    KeyT      const *const_keys_itr     = d_keys.Current();
+    ValueT    const *const_values_itr   = d_values.Current();
+
+    cudaError_t retval = DeviceSegmentedRadixSort::SortPairsDescending(
+        d_temp_storage, temp_storage_bytes,
+        const_keys_itr, d_keys.Alternate(), const_values_itr, d_values.Alternate(),
+        num_items, num_segments, d_segment_offsets, d_segment_offsets + 1,
+        begin_bit, end_bit, stream, debug_synchronous);
+
+    d_keys.selector ^= 1;
+    d_values.selector ^= 1;
+    return retval;
+}
+
+
+//---------------------------------------------------------------------
+// Dispatch to different Thrust entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch keys-only to Thrust sorting entrypoint
+ */
+template <int IS_DESCENDING, typename KeyT>
+cudaError_t Dispatch(
+    Int2Type<IS_DESCENDING> is_descending,
+    Int2Type<THRUST>        dispatch_to,
+    int                     *d_selector,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void                    *d_temp_storage,
+    size_t                  &temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<NullType>  &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<KeyT> d_keys_wrapper(d_keys.Current());
+
+        if (IS_DESCENDING) thrust::reverse(d_keys_wrapper, d_keys_wrapper + num_items);
+        thrust::sort(d_keys_wrapper, d_keys_wrapper + num_items);
+        if (IS_DESCENDING) thrust::reverse(d_keys_wrapper, d_keys_wrapper + num_items);
+    }
+
+    return cudaSuccess;
+}
+
+
+/**
+ * Dispatch key-value pairs to Thrust sorting entrypoint
+ */
+template <int IS_DESCENDING, typename KeyT, typename ValueT>
+cudaError_t Dispatch(
+    Int2Type<IS_DESCENDING> is_descending,
+    Int2Type<THRUST>        dispatch_to,
+    int                     *d_selector,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void                    *d_temp_storage,
+    size_t                  &temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<KeyT>     d_keys_wrapper(d_keys.Current());
+        thrust::device_ptr<ValueT>   d_values_wrapper(d_values.Current());
+
+        if (IS_DESCENDING) {
+            thrust::reverse(d_keys_wrapper, d_keys_wrapper + num_items);
+            thrust::reverse(d_values_wrapper, d_values_wrapper + num_items);
+        }
+
+        thrust::sort_by_key(d_keys_wrapper, d_keys_wrapper + num_items, d_values_wrapper);
+
+        if (IS_DESCENDING) {
+            thrust::reverse(d_keys_wrapper, d_keys_wrapper + num_items);
+            thrust::reverse(d_values_wrapper, d_values_wrapper + num_items);
+        }
+    }
+
+    return cudaSuccess;
+}
+
+
+//---------------------------------------------------------------------
+// CUDA Nested Parallelism Test Kernel
+//---------------------------------------------------------------------
+
+/**
+ * Simple wrapper kernel to invoke DeviceRadixSort
+ */
+template <int IS_DESCENDING, typename KeyT, typename ValueT>
+__global__ void CnpDispatchKernel(
+    Int2Type<IS_DESCENDING> is_descending,
+    int                     *d_selector,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void                    *d_temp_storage,
+    size_t                  temp_storage_bytes,
+    DoubleBuffer<KeyT>      d_keys,
+    DoubleBuffer<ValueT>    d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    bool                    debug_synchronous)
+{
+#ifndef CUB_CDP
+    *d_cdp_error            = cudaErrorNotSupported;
+#else
+    *d_cdp_error            = Dispatch(
+                                is_descending, Int2Type<CUB>(), d_selector, d_temp_storage_bytes, d_cdp_error,
+                                d_temp_storage, temp_storage_bytes, d_keys, d_values,
+                                num_items, num_segments, d_segment_offsets,
+                                begin_bit, end_bit, 0, debug_synchronous);
+    *d_temp_storage_bytes   = temp_storage_bytes;
+    *d_selector             = d_keys.selector;
+#endif
+}
+
+
+/**
+ * Dispatch to CDP kernel
+ */
+template <int IS_DESCENDING, typename KeyT, typename ValueT>
+cudaError_t Dispatch(
+    Int2Type<IS_DESCENDING> is_descending,
+    Int2Type<CDP>           dispatch_to,
+    int                     *d_selector,
+    size_t                  *d_temp_storage_bytes,
+    cudaError_t             *d_cdp_error,
+
+    void                    *d_temp_storage,
+    size_t                  &temp_storage_bytes,
+    DoubleBuffer<KeyT>      &d_keys,
+    DoubleBuffer<ValueT>    &d_values,
+    int                     num_items,
+    int                     num_segments,
+    const int               *d_segment_offsets,
+    int                     begin_bit,
+    int                     end_bit,
+    cudaStream_t            stream,
+    bool                    debug_synchronous)
+{
+    // Invoke kernel to invoke device-side dispatch
+    CnpDispatchKernel<<<1,1>>>(
+        is_descending, d_selector, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_keys, d_values,
+        num_items, num_segments, d_segment_offsets,
+        begin_bit, end_bit, debug_synchronous);
+
+    // Copy out selector
+    CubDebugExit(cudaMemcpy(&d_keys.selector, d_selector, sizeof(int) * 1, cudaMemcpyDeviceToHost));
+    d_values.selector = d_keys.selector;
+
+    // Copy out temp_storage_bytes
+    CubDebugExit(cudaMemcpy(&temp_storage_bytes, d_temp_storage_bytes, sizeof(size_t) * 1, cudaMemcpyDeviceToHost));
+
+    // Copy out error
+    cudaError_t retval;
+    CubDebugExit(cudaMemcpy(&retval, d_cdp_error, sizeof(cudaError_t) * 1, cudaMemcpyDeviceToHost));
+    return retval;
+}
+
+
+
+//---------------------------------------------------------------------
+// Problem generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Simple key-value pairing
+ */
+template <
+    typename KeyT,
+    typename ValueT,
+    bool IS_FLOAT = (Traits<KeyT>::CATEGORY == FLOATING_POINT)>
+struct Pair
+{
+    KeyT     key;
+    ValueT   value;
+
+    bool operator<(const Pair &b) const
+    {
+        return (key < b.key);
+    }
+};
+
+
+/**
+ * Simple key-value pairing (specialized for bool types)
+ */
+template <typename ValueT>
+struct Pair<bool, ValueT, false>
+{
+    bool     key;
+    ValueT   value;
+
+    bool operator<(const Pair &b) const
+    {
+        return (!key && b.key);
+    }
+};
+
+
+/**
+ * Simple key-value pairing (specialized for floating point types)
+ */
+template <typename KeyT, typename ValueT>
+struct Pair<KeyT, ValueT, true>
+{
+    KeyT     key;
+    ValueT   value;
+
+    bool operator<(const Pair &b) const
+    {
+        if (key < b.key)
+            return true;
+
+        if (key > b.key)
+            return false;
+
+        // KeyT in unsigned bits
+        typedef typename Traits<KeyT>::UnsignedBits UnsignedBits;
+
+        // Return true if key is negative zero and b.key is positive zero
+        UnsignedBits key_bits   = *reinterpret_cast<UnsignedBits*>(const_cast<KeyT*>(&key));
+        UnsignedBits b_key_bits = *reinterpret_cast<UnsignedBits*>(const_cast<KeyT*>(&b.key));
+        UnsignedBits HIGH_BIT   = Traits<KeyT>::HIGH_BIT;
+
+        return ((key_bits & HIGH_BIT) != 0) && ((b_key_bits & HIGH_BIT) == 0);
+    }
+};
+
+
+/**
+ * Initialize key data
+ */
+template <typename KeyT>
+void InitializeKeyBits(
+    GenMode         gen_mode,
+    KeyT            *h_keys,
+    int             num_items,
+    int             entropy_reduction)
+{
+    for (int i = 0; i < num_items; ++i)
+        InitValue(gen_mode, h_keys[i], i);
+}
+
+
+/**
+ * Initialize solution
+ */
+template <bool IS_DESCENDING, typename KeyT>
+void InitializeSolution(
+    KeyT    *h_keys,
+    int     num_items,
+    int     num_segments,
+    int     *h_segment_offsets,
+    int     begin_bit,
+    int     end_bit,
+    int     *&h_reference_ranks,
+    KeyT    *&h_reference_keys)
+{
+    typedef Pair<KeyT, int> PairT;
+
+    PairT *h_pairs = new PairT[num_items];
+
+    int num_bits = end_bit - begin_bit;
+    for (int i = 0; i < num_items; ++i)
+    {
+
+        // Mask off unwanted portions
+        if (num_bits < sizeof(KeyT) * 8)
+        {
+            unsigned long long base = 0;
+            memcpy(&base, &h_keys[i], sizeof(KeyT));
+            base &= ((1ull << num_bits) - 1) << begin_bit;
+            memcpy(&h_pairs[i].key, &base, sizeof(KeyT));
+        }
+        else
+        {
+            h_pairs[i].key = h_keys[i];
+        }
+
+        h_pairs[i].value = i;
+    }
+
+    printf("\nSorting reference solution on CPU (%d segments)...", num_segments); fflush(stdout);
+
+    for (int i = 0; i < num_segments; ++i)
+    {
+        if (IS_DESCENDING) std::reverse(h_pairs + h_segment_offsets[i], h_pairs + h_segment_offsets[i + 1]);
+        std::stable_sort(               h_pairs + h_segment_offsets[i], h_pairs + h_segment_offsets[i + 1]);
+        if (IS_DESCENDING) std::reverse(h_pairs + h_segment_offsets[i], h_pairs + h_segment_offsets[i + 1]);
+    }
+
+    printf(" Done.\n"); fflush(stdout);
+
+    h_reference_ranks  = new int[num_items];
+    h_reference_keys   = new KeyT[num_items];
+
+    for (int i = 0; i < num_items; ++i)
+    {
+        h_reference_ranks[i]    = h_pairs[i].value;
+        h_reference_keys[i]     = h_keys[h_pairs[i].value];
+    }
+
+    if (h_pairs) delete[] h_pairs;
+}
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Test DeviceRadixSort
+ */
+template <
+    Backend     BACKEND,
+    bool        IS_DESCENDING,
+    typename    KeyT,
+    typename    ValueT>
+void Test(
+    KeyT        *h_keys,
+    ValueT      *h_values,
+    int         num_items,
+    int         num_segments,
+    int         *h_segment_offsets,
+    int         begin_bit,
+    int         end_bit,
+    KeyT        *h_reference_keys,
+    ValueT      *h_reference_values)
+{
+    // Key alias type
+#if (__CUDACC_VER_MAJOR__ >= 9)
+    typedef typename If<Equals<KeyT, half_t>::VALUE, __half, KeyT>::Type KeyAliasT;
+#else
+    typedef KeyT KeyAliasT;
+#endif
+
+    const bool KEYS_ONLY = Equals<ValueT, NullType>::VALUE;
+
+    printf("%s %s cub::DeviceRadixSort %d items, %d segments, %d-byte keys (%s) %d-byte values (%s), descending %d, begin_bit %d, end_bit %d\n",
+        (BACKEND == CUB_NO_OVERWRITE) ? "CUB_NO_OVERWRITE" : (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        (KEYS_ONLY) ? "keys-only" : "key-value",
+        num_items, num_segments,
+        (int) sizeof(KeyT), typeid(KeyT).name(), (KEYS_ONLY) ? 0 : (int) sizeof(ValueT), typeid(ValueT).name(),
+        IS_DESCENDING, begin_bit, end_bit);
+    fflush(stdout);
+
+    if (g_verbose)
+    {
+        printf("Input keys:\n");
+        DisplayResults(h_keys, num_items);
+        printf("\n\n");
+    }
+
+    // Allocate device arrays
+    DoubleBuffer<KeyAliasT> d_keys;
+    DoubleBuffer<ValueT>    d_values;
+    int                     *d_selector;
+    int                     *d_segment_offsets;
+    size_t                  *d_temp_storage_bytes;
+    cudaError_t             *d_cdp_error;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys.d_buffers[0], sizeof(KeyT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys.d_buffers[1], sizeof(KeyT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_selector, sizeof(int) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_segment_offsets, sizeof(int) * (num_segments + 1)));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes, sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error, sizeof(cudaError_t) * 1));
+    if (!KEYS_ONLY)
+    {
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values.d_buffers[0], sizeof(ValueT) * num_items));
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values.d_buffers[1], sizeof(ValueT) * num_items));
+    }
+
+    // Allocate temporary storage (and make it un-aligned)
+    size_t  temp_storage_bytes  = 0;
+    void    *d_temp_storage     = NULL;
+    CubDebugExit(Dispatch(
+        Int2Type<IS_DESCENDING>(), Int2Type<BACKEND>(), d_selector, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_keys, d_values,
+        num_items, num_segments, d_segment_offsets,
+        begin_bit, end_bit, 0, true));
+
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes + 1));
+    void* mis_aligned_temp = static_cast<char*>(d_temp_storage) + 1;
+
+    // Initialize/clear device arrays
+    d_keys.selector = 0;
+    CubDebugExit(cudaMemcpy(d_keys.d_buffers[0], h_keys, sizeof(KeyT) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_keys.d_buffers[1], 0, sizeof(KeyT) * num_items));
+    if (!KEYS_ONLY)
+    {
+        d_values.selector = 0;
+        CubDebugExit(cudaMemcpy(d_values.d_buffers[0], h_values, sizeof(ValueT) * num_items, cudaMemcpyHostToDevice));
+        CubDebugExit(cudaMemset(d_values.d_buffers[1], 0, sizeof(ValueT) * num_items));
+    }
+    CubDebugExit(cudaMemcpy(d_segment_offsets, h_segment_offsets, sizeof(int) * (num_segments + 1), cudaMemcpyHostToDevice));
+
+    // Run warmup/correctness iteration
+    CubDebugExit(Dispatch(
+        Int2Type<IS_DESCENDING>(), Int2Type<BACKEND>(), d_selector, d_temp_storage_bytes, d_cdp_error,
+        mis_aligned_temp, temp_storage_bytes, d_keys, d_values,
+        num_items, num_segments, d_segment_offsets,
+        begin_bit, end_bit, 0, true));
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Check for correctness (and display results, if specified)
+    printf("Warmup done.  Checking results:\n"); fflush(stdout);
+    int compare = CompareDeviceResults(h_reference_keys, reinterpret_cast<KeyT*>(d_keys.Current()), num_items, true, g_verbose);
+    printf("\t Compare keys (selector %d): %s ", d_keys.selector, compare ? "FAIL" : "PASS"); fflush(stdout);
+    if (!KEYS_ONLY)
+    {
+        int values_compare = CompareDeviceResults(h_reference_values, d_values.Current(), num_items, true, g_verbose);
+        compare |= values_compare;
+        printf("\t Compare values (selector %d): %s ", d_values.selector, values_compare ? "FAIL" : "PASS"); fflush(stdout);
+    }
+    if (BACKEND == CUB_NO_OVERWRITE)
+    {
+        // Check that input isn't overwritten
+        int input_compare = CompareDeviceResults(h_keys, reinterpret_cast<KeyT*>(d_keys.d_buffers[0]), num_items, true, g_verbose);
+        compare |= input_compare;
+        printf("\t Compare input keys: %s ", input_compare ? "FAIL" : "PASS"); fflush(stdout);
+    }
+
+    // Performance
+    if (g_timing_iterations)
+        printf("\nPerforming timing iterations:\n"); fflush(stdout);
+
+    GpuTimer gpu_timer;
+    float elapsed_millis = 0.0f;
+    for (int i = 0; i < g_timing_iterations; ++i)
+    {
+        // Initialize/clear device arrays
+        CubDebugExit(cudaMemcpy(d_keys.d_buffers[d_keys.selector], h_keys, sizeof(KeyT) * num_items, cudaMemcpyHostToDevice));
+        CubDebugExit(cudaMemset(d_keys.d_buffers[d_keys.selector ^ 1], 0, sizeof(KeyT) * num_items));
+        if (!KEYS_ONLY)
+        {
+            CubDebugExit(cudaMemcpy(d_values.d_buffers[d_values.selector], h_values, sizeof(ValueT) * num_items, cudaMemcpyHostToDevice));
+            CubDebugExit(cudaMemset(d_values.d_buffers[d_values.selector ^ 1], 0, sizeof(ValueT) * num_items));
+        }
+
+        gpu_timer.Start();
+        CubDebugExit(Dispatch(
+            Int2Type<IS_DESCENDING>(), Int2Type<BACKEND>(), d_selector, d_temp_storage_bytes, d_cdp_error,
+            mis_aligned_temp, temp_storage_bytes, d_keys, d_values,
+            num_items, num_segments, d_segment_offsets,
+            begin_bit, end_bit, 0, false));
+        gpu_timer.Stop();
+        elapsed_millis += gpu_timer.ElapsedMillis();
+    }
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float avg_millis = elapsed_millis / g_timing_iterations;
+        float giga_rate = float(num_items) / avg_millis / 1000.0f / 1000.0f;
+        float giga_bandwidth = (KEYS_ONLY) ?
+            giga_rate * sizeof(KeyT) * 2 :
+            giga_rate * (sizeof(KeyT) + sizeof(ValueT)) * 2;
+        printf("\n%.3f elapsed ms, %.3f avg ms, %.3f billion items/s, %.3f logical GB/s", elapsed_millis, avg_millis, giga_rate, giga_bandwidth);
+    }
+
+    printf("\n\n");
+
+    // Cleanup
+    if (d_keys.d_buffers[0]) CubDebugExit(g_allocator.DeviceFree(d_keys.d_buffers[0]));
+    if (d_keys.d_buffers[1]) CubDebugExit(g_allocator.DeviceFree(d_keys.d_buffers[1]));
+    if (d_values.d_buffers[0]) CubDebugExit(g_allocator.DeviceFree(d_values.d_buffers[0]));
+    if (d_values.d_buffers[1]) CubDebugExit(g_allocator.DeviceFree(d_values.d_buffers[1]));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_selector) CubDebugExit(g_allocator.DeviceFree(d_selector));
+    if (d_segment_offsets) CubDebugExit(g_allocator.DeviceFree(d_segment_offsets));
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+
+    // Correctness asserts
+    AssertEquals(0, compare);
+}
+
+
+/**
+ * Test backend
+ */
+template <bool IS_DESCENDING, typename KeyT, typename ValueT>
+void TestBackend(
+    KeyT    *h_keys,
+    int     num_items,
+    int     num_segments,
+    int     *h_segment_offsets,
+    int     begin_bit,
+    int     end_bit,
+    KeyT    *h_reference_keys,
+    int     *h_reference_ranks)
+{
+    const bool KEYS_ONLY = Equals<ValueT, NullType>::VALUE;
+
+    ValueT *h_values             = NULL;
+    ValueT *h_reference_values   = NULL;
+
+    if (!KEYS_ONLY)
+    {
+        h_values            = new ValueT[num_items];
+        h_reference_values  = new ValueT[num_items];
+
+        for (int i = 0; i < num_items; ++i)
+        {
+            InitValue(INTEGER_SEED, h_values[i], i);
+            InitValue(INTEGER_SEED, h_reference_values[i], h_reference_ranks[i]);
+        }
+    }
+
+#ifdef SEGMENTED_SORT
+    // Test multi-segment implementations
+    Test<CUB_SEGMENTED, IS_DESCENDING>(               h_keys, h_values, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_values);
+    Test<CUB_SEGMENTED_NO_OVERWRITE, IS_DESCENDING>(  h_keys, h_values, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_values);
+#else   // SEGMENTED_SORT
+    if (num_segments == 1)
+    {
+        // Test single-segment implementations
+        Test<CUB, IS_DESCENDING>(               h_keys, h_values, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_values);
+        Test<CUB_NO_OVERWRITE, IS_DESCENDING>(  h_keys, h_values, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_values);
+    #ifdef CUB_CDP
+        Test<CDP, IS_DESCENDING>(               h_keys, h_values, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_values);
+    #endif
+    }
+#endif  // SEGMENTED_SORT
+
+    if (h_values) delete[] h_values;
+    if (h_reference_values) delete[] h_reference_values;
+}
+
+
+
+
+/**
+ * Test value type
+ */
+template <bool IS_DESCENDING, typename KeyT>
+void TestValueTypes(
+    KeyT    *h_keys,
+    int     num_items,
+    int     num_segments,
+    int     *h_segment_offsets,
+    int     begin_bit,
+    int     end_bit)
+{
+    // Initialize the solution
+
+    int *h_reference_ranks = NULL;
+    KeyT *h_reference_keys = NULL;
+    InitializeSolution<IS_DESCENDING>(h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_ranks, h_reference_keys);
+
+    // Test keys-only
+    TestBackend<IS_DESCENDING, KeyT, NullType>          (h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_ranks);
+
+    // Test with 8b value
+    TestBackend<IS_DESCENDING, KeyT, unsigned char>     (h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_ranks);
+
+    // Test with 32b value
+    TestBackend<IS_DESCENDING, KeyT, unsigned int>      (h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_ranks);
+
+    // Test with 64b value
+    TestBackend<IS_DESCENDING, KeyT, unsigned long long>(h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_ranks);
+
+    // Test with non-trivially-constructable value
+    TestBackend<IS_DESCENDING, KeyT, TestBar>           (h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit, h_reference_keys, h_reference_ranks);
+
+    // Cleanup
+    if (h_reference_ranks) delete[] h_reference_ranks;
+    if (h_reference_keys) delete[] h_reference_keys;
+}
+
+
+
+/**
+ * Test ascending/descending
+ */
+template <typename KeyT>
+void TestDirection(
+    KeyT    *h_keys,
+    int     num_items,
+    int     num_segments,
+    int     *h_segment_offsets,
+    int     begin_bit,
+    int     end_bit)
+{
+    TestValueTypes<true>(h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit);
+    TestValueTypes<false>(h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit);
+}
+
+
+/**
+ * Test different bit ranges
+ */
+template <typename KeyT>
+void TestBits(
+    KeyT    *h_keys,
+    int     num_items,
+    int     num_segments,
+    int     *h_segment_offsets)
+{
+    // Don't test partial-word sorting for boolean, fp, or signed types (the bit-flipping techniques get in the way)
+    if ((Traits<KeyT>::CATEGORY == UNSIGNED_INTEGER) && (!Equals<KeyT, bool>::VALUE))
+    {
+        // Partial bits
+        int begin_bit = 1;
+        int end_bit = (sizeof(KeyT) * 8) - 1;
+        printf("Testing key bits [%d,%d)\n", begin_bit, end_bit); fflush(stdout);
+        TestDirection(h_keys, num_items, num_segments, h_segment_offsets, begin_bit, end_bit);
+
+        // Across subword boundaries
+        int mid_bit = sizeof(KeyT) * 4;
+        printf("Testing key bits [%d,%d)\n", mid_bit - 1, mid_bit + 1); fflush(stdout);
+        TestDirection(h_keys, num_items, num_segments, h_segment_offsets, mid_bit - 1, mid_bit + 1);
+    }
+
+    printf("Testing key bits [%d,%d)\n", 0, int(sizeof(KeyT)) * 8); fflush(stdout);
+    TestDirection(h_keys, num_items, num_segments, h_segment_offsets, 0, sizeof(KeyT) * 8);
+}
+
+
+/**
+ * Test different segment compositions
+ */
+template <typename KeyT>
+void TestSegments(
+    KeyT    *h_keys,
+    int     num_items,
+    int     max_segments)
+{
+    int *h_segment_offsets = new int[max_segments + 1];
+
+#ifdef SEGMENTED_SORT
+    for (int num_segments = max_segments; num_segments > 1; num_segments = (num_segments + 32 - 1) / 32)
+    {
+        if (num_items / num_segments < 128 * 1000) {
+            // Right now we assign a single thread block to each segment, so lets keep it to under 128K items per segment
+            InitializeSegments(num_items, num_segments, h_segment_offsets);
+            TestBits(h_keys, num_items, num_segments, h_segment_offsets);
+        }
+    }
+#else
+    // Test single segment
+    if (num_items < 128 * 1000) {
+        // Right now we assign a single thread block to each segment, so lets keep it to under 128K items per segment
+        InitializeSegments(num_items, 1, h_segment_offsets);
+        TestBits(h_keys, num_items, 1, h_segment_offsets);
+    }
+#endif
+    if (h_segment_offsets) delete[] h_segment_offsets;
+}
+
+
+/**
+ * Test different (sub)lengths and number of segments
+ */
+template <typename KeyT>
+void TestSizes(
+    KeyT    *h_keys,
+    int     max_items,
+    int     max_segments)
+{
+    for (int num_items = max_items; num_items > 1; num_items = (num_items + 32 - 1) / 32)
+    {
+        TestSegments(h_keys, num_items, max_segments);
+    }
+    TestSegments(h_keys, 1, max_segments);
+    TestSegments(h_keys, 0, max_segments);
+}
+
+
+/**
+ * Test key sampling distributions
+ */
+template <typename KeyT>
+void TestGen(
+    int             max_items,
+    int             max_segments)
+{
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+    if (max_items < 0)
+        max_items = (ptx_version > 100) ? 9000003 : max_items = 5000003;
+
+    if (max_segments < 0)
+        max_segments = 5003;
+
+    KeyT *h_keys = new KeyT[max_items];
+
+    for (int entropy_reduction = 0; entropy_reduction <= 6; entropy_reduction += 3)
+    {
+        printf("\nTesting random %s keys with entropy reduction factor %d\n", typeid(KeyT).name(), entropy_reduction); fflush(stdout);
+        InitializeKeyBits(RANDOM, h_keys, max_items, entropy_reduction);
+        TestSizes(h_keys, max_items, max_segments);
+    }
+
+    printf("\nTesting uniform %s keys\n", typeid(KeyT).name()); fflush(stdout);
+    InitializeKeyBits(UNIFORM, h_keys, max_items, 0);
+    TestSizes(h_keys, max_items, max_segments);
+
+    printf("\nTesting natural number %s keys\n", typeid(KeyT).name()); fflush(stdout);
+    InitializeKeyBits(INTEGER_SEED, h_keys, max_items, 0);
+    TestSizes(h_keys, max_items, max_segments);
+
+    if (h_keys) delete[] h_keys;
+}
+
+
+//---------------------------------------------------------------------
+// Simple test
+//---------------------------------------------------------------------
+
+template <
+    Backend     BACKEND,
+    typename    KeyT,
+    typename    ValueT,
+    bool        IS_DESCENDING>
+void Test(
+    int         num_items,
+    int         num_segments,
+    GenMode     gen_mode,
+    int         entropy_reduction,
+    int         begin_bit,
+    int         end_bit)
+{
+    const bool KEYS_ONLY = Equals<ValueT, NullType>::VALUE;
+
+    KeyT    *h_keys             = new KeyT[num_items];
+    int     *h_reference_ranks  = NULL;
+    KeyT    *h_reference_keys   = NULL;
+    ValueT  *h_values           = NULL;
+    ValueT  *h_reference_values = NULL;
+    int     *h_segment_offsets  = new int[num_segments + 1];
+
+    if (end_bit < 0)
+        end_bit = sizeof(KeyT) * 8;
+
+    InitializeKeyBits(gen_mode, h_keys, num_items, entropy_reduction);
+    InitializeSegments(num_items, num_segments, h_segment_offsets);
+    InitializeSolution<IS_DESCENDING>(
+        h_keys, num_items, num_segments, h_segment_offsets,
+        begin_bit, end_bit, h_reference_ranks, h_reference_keys);
+
+    if (!KEYS_ONLY)
+    {
+        h_values            = new ValueT[num_items];
+        h_reference_values  = new ValueT[num_items];
+
+        for (int i = 0; i < num_items; ++i)
+        {
+            InitValue(INTEGER_SEED, h_values[i], i);
+            InitValue(INTEGER_SEED, h_reference_values[i], h_reference_ranks[i]);
+        }
+    }
+    if (h_reference_ranks) delete[] h_reference_ranks;
+
+    printf("\nTesting bits [%d,%d) of %s keys with gen-mode %d\n", begin_bit, end_bit, typeid(KeyT).name(), gen_mode); fflush(stdout);
+    Test<BACKEND, IS_DESCENDING>(
+        h_keys, h_values,
+        num_items, num_segments, h_segment_offsets,
+        begin_bit, end_bit, h_reference_keys, h_reference_values);
+
+    if (h_keys)             delete[] h_keys;
+    if (h_reference_keys)   delete[] h_reference_keys;
+    if (h_values)           delete[] h_values;
+    if (h_reference_values) delete[] h_reference_values;
+    if (h_segment_offsets)  delete[] h_segment_offsets;
+}
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int bits = -1;
+    int num_items = -1;
+    int num_segments = -1;
+    int entropy_reduction = 0;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("s", num_segments);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("repeat", g_repeat);
+    args.GetCmdLineArgument("bits", bits);
+    args.GetCmdLineArgument("entropy", entropy_reduction);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--bits=<valid key bits>]"
+            "[--n=<input items> "
+            "[--s=<num segments> "
+            "[--i=<timing iterations> "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "[--entropy=<entropy-reduction factor (default 0)>]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+#ifdef QUICKER_TEST
+
+    enum {
+        IS_DESCENDING   = false
+    };
+
+    // Compile/run basic CUB test
+    if (num_items < 0)      num_items       = 48000000;
+    if (num_segments < 0)   num_segments    = 5000;
+
+    Test<CUB,           unsigned char, NullType, IS_DESCENDING>(num_items, 1, RANDOM, entropy_reduction, 0, bits);
+    Test<CUB,           unsigned char, unsigned int, IS_DESCENDING>(num_items, 1, RANDOM, entropy_reduction, 0, bits);
+
+#if (__CUDACC_VER_MAJOR__ >= 9)
+    Test<CUB,           half_t,       NullType, IS_DESCENDING>(       num_items, 1,               RANDOM, entropy_reduction, 0, bits);
+#endif
+
+    Test<CUB_SEGMENTED, unsigned int,       NullType, IS_DESCENDING>(       num_items, num_segments,    RANDOM, entropy_reduction, 0, bits);
+
+    Test<CUB,           unsigned int,       NullType, IS_DESCENDING>(       num_items, 1,               RANDOM, entropy_reduction, 0, bits);
+    Test<CUB,           unsigned long long, NullType, IS_DESCENDING>(       num_items, 1,               RANDOM, entropy_reduction, 0, bits);
+
+    Test<CUB,           unsigned int,       unsigned int, IS_DESCENDING>(   num_items, 1,               RANDOM, entropy_reduction, 0, bits);
+    Test<CUB,           unsigned long long, unsigned int, IS_DESCENDING>(   num_items, 1,               RANDOM, entropy_reduction, 0, bits);
+
+#elif defined(QUICK_TEST)
+
+    // Compile/run quick tests
+    if (num_items < 0)      num_items       = 48000000;
+    if (num_segments < 0)   num_segments    = 5000;
+
+    // Compare CUB and thrust on 32b keys-only
+    Test<CUB, unsigned int, NullType, false> (                      num_items, 1, RANDOM, entropy_reduction, 0, bits);
+    Test<THRUST, unsigned int, NullType, false> (                   num_items, 1, RANDOM, entropy_reduction, 0, bits);
+
+    // Compare CUB and thrust on 64b keys-only
+    Test<CUB, unsigned long long, NullType, false> (                num_items, 1, RANDOM, entropy_reduction, 0, bits);
+    Test<THRUST, unsigned long long, NullType, false> (             num_items, 1, RANDOM, entropy_reduction, 0, bits);
+
+
+    // Compare CUB and thrust on 32b key-value pairs
+    Test<CUB, unsigned int, unsigned int, false> (                  num_items, 1, RANDOM, entropy_reduction, 0, bits);
+    Test<THRUST, unsigned int, unsigned int, false> (               num_items, 1, RANDOM, entropy_reduction, 0, bits);
+
+    // Compare CUB and thrust on 64b key-value pairs
+    Test<CUB, unsigned long long, unsigned long long, false> (      num_items, 1, RANDOM, entropy_reduction, 0, bits);
+    Test<THRUST, unsigned long long, unsigned long long, false> (   num_items, 1, RANDOM, entropy_reduction, 0, bits);
+
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        TestGen<bool>                 (num_items, num_segments);
+
+        TestGen<char>                 (num_items, num_segments);
+        TestGen<signed char>          (num_items, num_segments);
+        TestGen<unsigned char>        (num_items, num_segments);
+
+        TestGen<short>                (num_items, num_segments);
+        TestGen<unsigned short>       (num_items, num_segments);
+
+        TestGen<int>                  (num_items, num_segments);
+        TestGen<unsigned int>         (num_items, num_segments);
+
+        TestGen<long>                 (num_items, num_segments);
+        TestGen<unsigned long>        (num_items, num_segments);
+
+        TestGen<long long>            (num_items, num_segments);
+        TestGen<unsigned long long>   (num_items, num_segments);
+
+#if (__CUDACC_VER_MAJOR__ >= 9)
+        TestGen<half_t>                (num_items, num_segments);
+#endif
+        TestGen<float>                (num_items, num_segments);
+
+        if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+            TestGen<double>           (num_items, num_segments);
+
+    }
+
+#endif
+
+    return 0;
+}
+
diff --git a/third_party/cub/test/test_device_reduce.cu b/third_party/cub/test/test_device_reduce.cu
new file mode 100644
index 0000000..275d8e1
--- /dev/null
+++ b/third_party/cub/test/test_device_reduce.cu
@@ -0,0 +1,1359 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of DeviceReduce utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <limits>
+#include <typeinfo>
+
+#include <thrust/device_ptr.h>
+#include <thrust/reduce.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_reduce.cuh>
+#include <cub/device/device_segmented_reduce.cuh>
+#include <cub/iterator/constant_input_iterator.cuh>
+#include <cub/iterator/discard_output_iterator.cuh>
+#include <cub/iterator/transform_input_iterator.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+int                     g_ptx_version;
+int                     g_sm_count;
+bool                    g_verbose           = false;
+bool                    g_verbose_input     = false;
+int                     g_timing_iterations = 0;
+int                     g_repeat            = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+
+// Dispatch types
+enum Backend
+{
+    CUB,            // CUB method
+    CUB_SEGMENTED,  // CUB segmented method
+    CUB_CDP,        // GPU-based (dynamic parallelism) dispatch to CUB method
+    THRUST,         // Thrust method
+};
+
+
+// Custom max functor
+struct CustomMax
+{
+    /// Boolean max operator, returns <tt>(a > b) ? a : b</tt>
+    template <typename OutputT>
+    __host__ __device__ __forceinline__ OutputT operator()(const OutputT &a, const OutputT &b)
+    {
+        return CUB_MAX(a, b);
+    }
+};
+
+
+//---------------------------------------------------------------------
+// Dispatch to different CUB DeviceReduce entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to reduce entrypoint (custom-max)
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT, typename ReductionOpT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    ReductionOpT        reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    // Max-identity
+    OutputT identity = Traits<InputT>::Lowest(); // replace with std::numeric_limits<OutputT>::lowest() when C++ support is more prevalent
+
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes,
+            d_in, d_out, num_items, reduction_op, identity,
+            stream, debug_synchronous);
+    }
+
+    printf("\t timing_timing_iterations: %d, temp_storage_bytes: %lld\n",
+        timing_timing_iterations, temp_storage_bytes);
+
+    return error;
+}
+
+/**
+ * Dispatch to sum entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::Sum            reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream, debug_synchronous);
+    }
+
+    printf("\t timing_timing_iterations: %d, temp_storage_bytes: %lld\n",
+        timing_timing_iterations, temp_storage_bytes);
+
+    return error;
+}
+
+/**
+ * Dispatch to min entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::Min            reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream, debug_synchronous);
+    }
+
+    printf("\t timing_timing_iterations: %d, temp_storage_bytes: %lld\n",
+        timing_timing_iterations, temp_storage_bytes);
+
+    return error;
+}
+
+/**
+ * Dispatch to max entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::Max            reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream, debug_synchronous);
+    }
+
+    printf("\t timing_timing_iterations: %d, temp_storage_bytes: %lld\n",
+        timing_timing_iterations, temp_storage_bytes);
+
+    return error;
+}
+
+/**
+ * Dispatch to argmin entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::ArgMin         reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream, debug_synchronous);
+    }
+
+    printf("\t timing_timing_iterations: %d, temp_storage_bytes: %lld\n",
+        timing_timing_iterations, temp_storage_bytes);
+
+    return error;
+}
+
+/**
+ * Dispatch to argmax entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::ArgMax         reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream, debug_synchronous);
+    }
+
+    printf("\t timing_timing_iterations: %d, temp_storage_bytes: %lld\n",
+        timing_timing_iterations, temp_storage_bytes);
+
+    return error;
+}
+
+
+//---------------------------------------------------------------------
+// Dispatch to different CUB DeviceSegmentedReduce entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to reduce entrypoint (custom-max)
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT, typename ReductionOpT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB_SEGMENTED>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    ReductionOpT        reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    // Max-identity
+    OutputT identity = Traits<InputT>::Lowest(); // replace with std::numeric_limits<OutputT>::lowest() when C++ support is more prevalent
+
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSegmentedReduce::Reduce(d_temp_storage, temp_storage_bytes,
+            d_in, d_out, max_segments, d_segment_offsets, d_segment_offsets + 1, reduction_op, identity,
+            stream, debug_synchronous);
+    }
+    return error;
+}
+
+/**
+ * Dispatch to sum entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB_SEGMENTED>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::Sum            reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSegmentedReduce::Sum(d_temp_storage, temp_storage_bytes,
+            d_in, d_out, max_segments, d_segment_offsets, d_segment_offsets + 1,
+            stream, debug_synchronous);
+    }
+    return error;
+}
+
+/**
+ * Dispatch to min entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB_SEGMENTED>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::Min            reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSegmentedReduce::Min(d_temp_storage, temp_storage_bytes,
+            d_in, d_out, max_segments, d_segment_offsets, d_segment_offsets + 1,
+            stream, debug_synchronous);
+    }
+    return error;
+}
+
+/**
+ * Dispatch to max entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB_SEGMENTED>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::Max            reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSegmentedReduce::Max(d_temp_storage, temp_storage_bytes,
+            d_in, d_out, max_segments, d_segment_offsets, d_segment_offsets + 1,
+            stream, debug_synchronous);
+    }
+    return error;
+}
+
+/**
+ * Dispatch to argmin entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB_SEGMENTED>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::ArgMin         reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes,
+            d_in, d_out, max_segments, d_segment_offsets, d_segment_offsets + 1,
+            stream, debug_synchronous);
+    }
+    return error;
+}
+
+/**
+ * Dispatch to argmax entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB_SEGMENTED>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    cub::ArgMax         reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to device reduction directly
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes,
+            d_in, d_out, max_segments, d_segment_offsets, d_segment_offsets + 1,
+            stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+//---------------------------------------------------------------------
+// Dispatch to different Thrust entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to reduction entrypoint (min or max specialization)
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT, typename ReductionOpT>
+cudaError_t Dispatch(
+    Int2Type<THRUST>    dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    ReductionOpT         reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        OutputT init;
+        CubDebugExit(cudaMemcpy(&init, d_in + 0, sizeof(OutputT), cudaMemcpyDeviceToHost));
+
+        thrust::device_ptr<OutputT> d_in_wrapper(d_in);
+        OutputT retval;
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            retval = thrust::reduce(d_in_wrapper, d_in_wrapper + num_items, init, reduction_op);
+        }
+
+        if (!Equals<OutputIteratorT, DiscardOutputIterator<int> >::VALUE)
+            CubDebugExit(cudaMemcpy(d_out, &retval, sizeof(OutputT), cudaMemcpyHostToDevice));
+    }
+
+    return cudaSuccess;
+}
+
+/**
+ * Dispatch to reduction entrypoint (sum specialization)
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT>
+cudaError_t Dispatch(
+    Int2Type<THRUST>    dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    Sum                 reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<OutputT> d_in_wrapper(d_in);
+        OutputT retval;
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            retval = thrust::reduce(d_in_wrapper, d_in_wrapper + num_items);
+        }
+
+        if (!Equals<OutputIteratorT, DiscardOutputIterator<int> >::VALUE)
+            CubDebugExit(cudaMemcpy(d_out, &retval, sizeof(OutputT), cudaMemcpyHostToDevice));
+    }
+
+    return cudaSuccess;
+}
+
+
+//---------------------------------------------------------------------
+// CUDA nested-parallelism test kernel
+//---------------------------------------------------------------------
+
+/**
+ * Simple wrapper kernel to invoke DeviceReduce
+ */
+template <
+    typename            InputIteratorT,
+    typename            OutputIteratorT,
+    typename            OffsetIteratorT,
+    typename            ReductionOpT>
+__global__ void CnpDispatchKernel(
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t              temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    ReductionOpT        reduction_op,
+    bool                debug_synchronous)
+{
+#ifndef CUB_CDP
+    *d_cdp_error = cudaErrorNotSupported;
+#else
+    *d_cdp_error = Dispatch(Int2Type<CUB>(), timing_timing_iterations, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes,
+        d_in, d_out, num_items, max_segments, d_segment_offsets, reduction_op, 0, debug_synchronous);
+    *d_temp_storage_bytes = temp_storage_bytes;
+#endif
+}
+
+
+/**
+ * Dispatch to CUB_CDP kernel
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetIteratorT, typename ReductionOpT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB_CDP>       dispatch_to,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    int                 num_items,
+    int                 max_segments,
+    OffsetIteratorT     d_segment_offsets,
+    ReductionOpT        reduction_op,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to invoke device-side dispatch
+    CnpDispatchKernel<<<1,1>>>(timing_timing_iterations, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes,
+        d_in, d_out, num_items, max_segments, d_segment_offsets, reduction_op, debug_synchronous);
+
+    // Copy out temp_storage_bytes
+    CubDebugExit(cudaMemcpy(&temp_storage_bytes, d_temp_storage_bytes, sizeof(size_t) * 1, cudaMemcpyDeviceToHost));
+
+    // Copy out error
+    cudaError_t retval;
+    CubDebugExit(cudaMemcpy(&retval, d_cdp_error, sizeof(cudaError_t) * 1, cudaMemcpyDeviceToHost));
+    return retval;
+}
+
+
+
+//---------------------------------------------------------------------
+// Problem generation
+//---------------------------------------------------------------------
+
+/// Initialize problem
+template <typename InputT>
+void Initialize(
+    GenMode         gen_mode,
+    InputT          *h_in,
+    int             num_items)
+{
+    for (int i = 0; i < num_items; ++i)
+    {
+        InitValue(gen_mode, h_in[i], i);
+    }
+
+    if (g_verbose_input)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/// Solve problem (max/custom-max functor)
+template <typename ReductionOpT, typename InputT, typename _OutputT>
+struct Solution
+{
+    typedef _OutputT OutputT;
+
+    template <typename HostInputIteratorT, typename OffsetT, typename OffsetIteratorT>
+    static void Solve(HostInputIteratorT h_in, OutputT *h_reference, OffsetT num_segments, OffsetIteratorT h_segment_offsets,
+        ReductionOpT reduction_op)
+    {
+        for (int i = 0; i < num_segments; ++i)
+        {
+            OutputT aggregate = Traits<InputT>::Lowest(); // replace with std::numeric_limits<OutputT>::lowest() when C++ support is more prevalent
+            for (int j = h_segment_offsets[i]; j < h_segment_offsets[i + 1]; ++j)
+                aggregate = reduction_op(aggregate, OutputT(h_in[j]));
+            h_reference[i] = aggregate;
+        }
+    }
+};
+
+/// Solve problem (min functor)
+template <typename InputT, typename _OutputT>
+struct Solution<cub::Min, InputT, _OutputT>
+{
+    typedef _OutputT OutputT;
+
+    template <typename HostInputIteratorT, typename OffsetT, typename OffsetIteratorT>
+    static void Solve(HostInputIteratorT h_in, OutputT *h_reference, OffsetT num_segments, OffsetIteratorT h_segment_offsets,
+        cub::Min reduction_op)
+    {
+        for (int i = 0; i < num_segments; ++i)
+        {
+            OutputT aggregate = Traits<InputT>::Max();    // replace with std::numeric_limits<OutputT>::max() when C++ support is more prevalent
+            for (int j = h_segment_offsets[i]; j < h_segment_offsets[i + 1]; ++j)
+                aggregate = reduction_op(aggregate, OutputT(h_in[j]));
+            h_reference[i] = aggregate;
+        }
+    }
+};
+
+
+/// Solve problem (sum functor)
+template <typename InputT, typename _OutputT>
+struct Solution<cub::Sum, InputT, _OutputT>
+{
+    typedef _OutputT OutputT;
+
+    template <typename HostInputIteratorT, typename OffsetT, typename OffsetIteratorT>
+    static void Solve(HostInputIteratorT h_in, OutputT *h_reference, OffsetT num_segments, OffsetIteratorT h_segment_offsets,
+        cub::Sum reduction_op)
+    {
+        for (int i = 0; i < num_segments; ++i)
+        {
+            OutputT aggregate;
+            InitValue(INTEGER_SEED, aggregate, 0);
+            for (int j = h_segment_offsets[i]; j < h_segment_offsets[i + 1]; ++j)
+                aggregate = reduction_op(aggregate, OutputT(h_in[j]));
+            h_reference[i] = aggregate;
+        }
+    }
+};
+
+/// Solve problem (argmin functor)
+template <typename InputValueT, typename OutputValueT>
+struct Solution<cub::ArgMin, InputValueT, OutputValueT>
+{
+    typedef KeyValuePair<int, OutputValueT> OutputT;
+
+    template <typename HostInputIteratorT, typename OffsetT, typename OffsetIteratorT>
+    static void Solve(HostInputIteratorT h_in, OutputT *h_reference, OffsetT num_segments, OffsetIteratorT h_segment_offsets,
+        cub::ArgMin reduction_op)
+    {
+        for (int i = 0; i < num_segments; ++i)
+        {
+            OutputT aggregate(1, Traits<InputValueT>::Max()); // replace with std::numeric_limits<OutputT>::max() when C++ support is more prevalent
+            for (int j = h_segment_offsets[i]; j < h_segment_offsets[i + 1]; ++j)
+            {
+                OutputT item(j - h_segment_offsets[i], OutputValueT(h_in[j]));
+                aggregate = reduction_op(aggregate, item);
+            }
+            h_reference[i] = aggregate;
+        }
+    }
+};
+
+
+/// Solve problem (argmax functor)
+template <typename InputValueT, typename OutputValueT>
+struct Solution<cub::ArgMax, InputValueT, OutputValueT>
+{
+    typedef KeyValuePair<int, OutputValueT> OutputT;
+
+    template <typename HostInputIteratorT, typename OffsetT, typename OffsetIteratorT>
+    static void Solve(HostInputIteratorT h_in, OutputT *h_reference, OffsetT num_segments, OffsetIteratorT h_segment_offsets,
+        cub::ArgMax reduction_op)
+    {
+        for (int i = 0; i < num_segments; ++i)
+        {
+            OutputT aggregate(1, Traits<InputValueT>::Lowest()); // replace with std::numeric_limits<OutputT>::lowest() when C++ support is more prevalent
+            for (int j = h_segment_offsets[i]; j < h_segment_offsets[i + 1]; ++j)
+            {
+                OutputT item(j - h_segment_offsets[i], OutputValueT(h_in[j]));
+                aggregate = reduction_op(aggregate, item);
+            }
+            h_reference[i] = aggregate;
+        }
+    }
+};
+
+
+//---------------------------------------------------------------------
+// Problem generation
+//---------------------------------------------------------------------
+
+/// Test DeviceReduce for a given problem input
+template <
+    typename                BackendT,
+    typename                DeviceInputIteratorT,
+    typename                DeviceOutputIteratorT,
+    typename                HostReferenceIteratorT,
+    typename                OffsetT,
+    typename                OffsetIteratorT,
+    typename                ReductionOpT>
+void Test(
+    BackendT                backend,
+    DeviceInputIteratorT    d_in,
+    DeviceOutputIteratorT   d_out,
+    OffsetT                 num_items,
+    OffsetT                 num_segments,
+    OffsetIteratorT         d_segment_offsets,
+    ReductionOpT            reduction_op,
+    HostReferenceIteratorT  h_reference)
+{
+    // Input data types
+    typedef typename std::iterator_traits<DeviceInputIteratorT>::value_type InputT;
+
+    // Allocate CUB_CDP device arrays for temp storage size and error
+    size_t          *d_temp_storage_bytes = NULL;
+    cudaError_t     *d_cdp_error = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes,  sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error,           sizeof(cudaError_t) * 1));
+
+    // Inquire temp device storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(Dispatch(backend, 1,
+        d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes,
+        d_in, d_out, num_items, num_segments, d_segment_offsets,
+        reduction_op, 0, true));
+
+    // Allocate temp device storage
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Run warmup/correctness iteration
+    CubDebugExit(Dispatch(backend, 1,
+        d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes,
+        d_in, d_out, num_items, num_segments, d_segment_offsets,
+        reduction_op, 0, true));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_out, num_segments, g_verbose, g_verbose);
+    printf("\t%s", compare ? "FAIL" : "PASS");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Performance
+    if (g_timing_iterations > 0)
+    {
+        GpuTimer gpu_timer;
+        gpu_timer.Start();
+
+        CubDebugExit(Dispatch(backend, g_timing_iterations,
+            d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes,
+            d_in, d_out, num_items, num_segments, d_segment_offsets,
+            reduction_op, 0, false));
+
+        gpu_timer.Stop();
+        float elapsed_millis = gpu_timer.ElapsedMillis();
+
+        // Display performance
+        float avg_millis = elapsed_millis / g_timing_iterations;
+        float giga_rate = float(num_items) / avg_millis / 1000.0f / 1000.0f;
+        float giga_bandwidth = giga_rate * sizeof(InputT);
+        printf(", %.3f avg ms, %.3f billion items/s, %.3f logical GB/s", avg_millis, giga_rate, giga_bandwidth);
+    }
+
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Correctness asserts
+    AssertEquals(0, compare);
+}
+
+
+/// Test DeviceReduce
+template <
+    Backend                 BACKEND,
+    typename                OutputValueT,
+    typename                HostInputIteratorT,
+    typename                DeviceInputIteratorT,
+    typename                OffsetT,
+    typename                OffsetIteratorT,
+    typename                ReductionOpT>
+void SolveAndTest(
+    HostInputIteratorT      h_in,
+    DeviceInputIteratorT    d_in,
+    OffsetT                 num_items,
+    OffsetT                 num_segments,
+    OffsetIteratorT         h_segment_offsets,
+    OffsetIteratorT         d_segment_offsets,
+    ReductionOpT            reduction_op)
+{
+    typedef typename std::iterator_traits<DeviceInputIteratorT>::value_type     InputValueT;
+    typedef Solution<ReductionOpT, InputValueT, OutputValueT>                   SolutionT;
+    typedef typename SolutionT::OutputT                                         OutputT;
+
+    printf("\n\n%s cub::DeviceReduce<%s> %d items (%s), %d segments\n",
+        (BACKEND == CUB_CDP) ? "CUB_CDP" : (BACKEND == THRUST) ? "Thrust" : (BACKEND == CUB_SEGMENTED) ? "CUB_SEGMENTED" : "CUB",
+        typeid(ReductionOpT).name(), num_items, typeid(HostInputIteratorT).name(), num_segments);
+    fflush(stdout);
+
+    // Allocate and solve solution
+    OutputT *h_reference = new OutputT[num_segments];
+    SolutionT::Solve(h_in, h_reference, num_segments, h_segment_offsets, reduction_op);
+
+    // Run with discard iterator
+    DiscardOutputIterator<OffsetT> discard_itr;
+    Test(Int2Type<BACKEND>(), d_in, discard_itr, num_items, num_segments, d_segment_offsets, reduction_op, h_reference);
+
+    // Run with output data (cleared for sanity-check)
+    OutputT *d_out = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(OutputT) * num_segments));
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(OutputT) * num_segments));
+    Test(Int2Type<BACKEND>(), d_in, d_out, num_items, num_segments, d_segment_offsets, reduction_op, h_reference);
+
+    // Cleanup
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (h_reference) delete[] h_reference;
+}
+
+
+/// Test specific problem type
+template <
+    Backend         BACKEND,
+    typename        InputT,
+    typename        OutputT,
+    typename        OffsetT,
+    typename        ReductionOpT>
+void TestProblem(
+    OffsetT         num_items,
+    OffsetT         num_segments,
+    GenMode         gen_mode,
+    ReductionOpT    reduction_op)
+{
+    printf("\n\nInitializing %d %s->%s (gen mode %d)... ", num_items, typeid(InputT).name(), typeid(OutputT).name(), gen_mode); fflush(stdout);
+    fflush(stdout);
+
+    // Initialize value data
+    InputT* h_in = new InputT[num_items];
+    Initialize(gen_mode, h_in, num_items);
+
+    // Initialize segment data
+    OffsetT *h_segment_offsets = new OffsetT[num_segments + 1];
+    InitializeSegments(num_items, num_segments, h_segment_offsets, g_verbose_input);
+
+    // Initialize device data
+    OffsetT *d_segment_offsets      = NULL;
+    InputT  *d_in                   = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in,              sizeof(InputT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_segment_offsets, sizeof(OffsetT) * (num_segments + 1)));
+    CubDebugExit(cudaMemcpy(d_in,               h_in,                   sizeof(InputT) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_segment_offsets,  h_segment_offsets,      sizeof(OffsetT) * (num_segments + 1), cudaMemcpyHostToDevice));
+
+    SolveAndTest<BACKEND, OutputT>(h_in, d_in, num_items, num_segments, h_segment_offsets, d_segment_offsets, reduction_op);
+
+    if (h_segment_offsets)  delete[] h_segment_offsets;
+    if (d_segment_offsets)  CubDebugExit(g_allocator.DeviceFree(d_segment_offsets));
+    if (h_in)               delete[] h_in;
+    if (d_in)               CubDebugExit(g_allocator.DeviceFree(d_in));
+}
+
+
+/// Test different operators
+template <
+    Backend             BACKEND,
+    typename            OutputT,
+    typename            HostInputIteratorT,
+    typename            DeviceInputIteratorT,
+    typename            OffsetT,
+    typename            OffsetIteratorT>
+void TestByOp(
+    HostInputIteratorT      h_in,
+    DeviceInputIteratorT    d_in,
+    OffsetT                 num_items,
+    OffsetT                 num_segments,
+    OffsetIteratorT         h_segment_offsets,
+    OffsetIteratorT         d_segment_offsets)
+{
+    SolveAndTest<BACKEND, OutputT>(h_in, d_in, num_items, num_segments, h_segment_offsets, d_segment_offsets, CustomMax());
+    SolveAndTest<BACKEND, OutputT>(h_in, d_in, num_items, num_segments, h_segment_offsets, d_segment_offsets, Sum());
+    SolveAndTest<BACKEND, OutputT>(h_in, d_in, num_items, num_segments, h_segment_offsets, d_segment_offsets, Min());
+    SolveAndTest<BACKEND, OutputT>(h_in, d_in, num_items, num_segments, h_segment_offsets, d_segment_offsets, ArgMin());
+    SolveAndTest<BACKEND, OutputT>(h_in, d_in, num_items, num_segments, h_segment_offsets, d_segment_offsets, Max());
+    SolveAndTest<BACKEND, OutputT>(h_in, d_in, num_items, num_segments, h_segment_offsets, d_segment_offsets, ArgMax());
+}
+
+
+/// Test different backends
+template <
+    typename    InputT,
+    typename    OutputT,
+    typename    OffsetT>
+void TestByBackend(
+    OffsetT     num_items,
+    OffsetT     max_segments,
+    GenMode     gen_mode)
+{
+    // Initialize host data
+    printf("\n\nInitializing %d %s -> %s (gen mode %d)... ",
+        num_items, typeid(InputT).name(), typeid(OutputT).name(), gen_mode); fflush(stdout);
+
+    InputT  *h_in               = new InputT[num_items];
+    OffsetT *h_segment_offsets  = new OffsetT[max_segments + 1];
+    Initialize(gen_mode, h_in, num_items);
+
+    // Initialize device data
+    InputT  *d_in               = NULL;
+    OffsetT *d_segment_offsets  = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(InputT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_segment_offsets, sizeof(OffsetT) * (max_segments + 1)));
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(InputT) * num_items, cudaMemcpyHostToDevice));
+
+    //
+    // Test single-segment implementations
+    //
+
+    InitializeSegments(num_items, 1, h_segment_offsets, g_verbose_input);
+
+    // Page-aligned-input tests
+    TestByOp<CUB, OutputT>(h_in, d_in, num_items, 1, h_segment_offsets, (OffsetT*) NULL);                 // Host-dispatch
+#ifdef CUB_CDP
+    TestByOp<CUB_CDP, OutputT>(h_in, d_in, num_items, 1, h_segment_offsets, (OffsetT*) NULL);             // Device-dispatch
+#endif
+
+    // Non-page-aligned-input tests
+    if (num_items > 1)
+    {
+        InitializeSegments(num_items - 1, 1, h_segment_offsets, g_verbose_input);
+        TestByOp<CUB, OutputT>(h_in + 1, d_in + 1, num_items - 1, 1, h_segment_offsets, (OffsetT*) NULL);
+    }
+
+    //
+    // Test segmented implementation
+    //
+
+    // Right now we assign a single thread block to each segment, so lets keep it to under 128K items per segment
+    int max_items_per_segment = 128000;
+
+    for (int num_segments = (num_items + max_items_per_segment - 1) / max_items_per_segment;
+        num_segments < max_segments;
+        num_segments = (num_segments * 32) + 1)
+    {
+        // Test with segment pointer
+        InitializeSegments(num_items, num_segments, h_segment_offsets, g_verbose_input);
+        CubDebugExit(cudaMemcpy(d_segment_offsets, h_segment_offsets, sizeof(OffsetT) * (num_segments + 1), cudaMemcpyHostToDevice));
+        TestByOp<CUB_SEGMENTED, OutputT>(
+            h_in, d_in, num_items, num_segments, h_segment_offsets, d_segment_offsets);
+
+        // Test with segment iterator
+        typedef CastOp<OffsetT> IdentityOpT;
+        IdentityOpT identity_op;
+        TransformInputIterator<OffsetT, IdentityOpT, OffsetT*, OffsetT> h_segment_offsets_itr(
+            h_segment_offsets,
+            identity_op);
+       TransformInputIterator<OffsetT, IdentityOpT, OffsetT*, OffsetT> d_segment_offsets_itr(
+            d_segment_offsets,
+            identity_op);
+
+        TestByOp<CUB_SEGMENTED, OutputT>(
+            h_in, d_in, num_items, num_segments, h_segment_offsets_itr, d_segment_offsets_itr);
+    }
+
+    if (h_in)               delete[] h_in;
+    if (h_segment_offsets)  delete[] h_segment_offsets;
+    if (d_in)               CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_segment_offsets)  CubDebugExit(g_allocator.DeviceFree(d_segment_offsets));
+}
+
+
+/// Test different input-generation modes
+template <
+    typename InputT,
+    typename OutputT,
+    typename OffsetT>
+void TestByGenMode(
+    OffsetT num_items,
+    OffsetT max_segments)
+{
+    //
+    // Test pointer support using different input-generation modes
+    //
+
+    TestByBackend<InputT, OutputT>(num_items, max_segments, UNIFORM);
+    TestByBackend<InputT, OutputT>(num_items, max_segments, INTEGER_SEED);
+    TestByBackend<InputT, OutputT>(num_items, max_segments, RANDOM);
+
+    //
+    // Test iterator support using a constant-iterator and SUM
+    //
+
+    InputT val;
+    InitValue(UNIFORM, val, 0);
+    ConstantInputIterator<InputT, OffsetT> h_in(val);
+
+    OffsetT *h_segment_offsets = new OffsetT[1 + 1];
+    InitializeSegments(num_items, 1, h_segment_offsets, g_verbose_input);
+
+    SolveAndTest<CUB, OutputT>(h_in, h_in, num_items, 1, h_segment_offsets, (OffsetT*) NULL, Sum());
+#ifdef CUB_CDP
+    SolveAndTest<CUB_CDP, OutputT>(h_in, h_in, num_items, 1, h_segment_offsets, (OffsetT*) NULL, Sum());
+#endif
+
+    if (h_segment_offsets) delete[] h_segment_offsets;
+}
+
+
+/// Test different problem sizes
+template <
+    typename InputT,
+    typename OutputT,
+    typename OffsetT>
+struct TestBySize
+{
+    OffsetT max_items;
+    OffsetT max_segments;
+
+    TestBySize(OffsetT max_items, OffsetT max_segments) :
+        max_items(max_items),
+        max_segments(max_segments)
+    {}
+
+    template <typename ActivePolicyT>
+    cudaError_t Invoke()
+    {
+        //
+        // Black-box testing on all backends
+        //
+
+        // Test 0, 1, many
+        TestByGenMode<InputT, OutputT>(0,           max_segments);
+        TestByGenMode<InputT, OutputT>(1,           max_segments);
+        TestByGenMode<InputT, OutputT>(max_items,   max_segments);
+
+        // Test random problem sizes from a log-distribution [8, max_items-ish)
+        int     num_iterations = 8;
+        double  max_exp = log(double(max_items)) / log(double(2.0));
+        for (int i = 0; i < num_iterations; ++i)
+        {
+            OffsetT num_items = (OffsetT) pow(2.0, RandomValue(max_exp - 3.0) + 3.0);
+            TestByGenMode<InputT, OutputT>(num_items, max_segments);
+        }
+
+        //
+        // White-box testing of single-segment problems around specific sizes
+        //
+
+        // Tile-boundaries: multiple blocks, one tile per block
+        OffsetT tile_size = ActivePolicyT::ReducePolicy::BLOCK_THREADS * ActivePolicyT::ReducePolicy::ITEMS_PER_THREAD;
+        TestProblem<CUB, InputT, OutputT>(tile_size * 4,  1,      RANDOM, Sum());
+        TestProblem<CUB, InputT, OutputT>(tile_size * 4 + 1, 1,   RANDOM, Sum());
+        TestProblem<CUB, InputT, OutputT>(tile_size * 4 - 1, 1,   RANDOM, Sum());
+
+        // Tile-boundaries: multiple blocks, multiple tiles per block
+        OffsetT sm_occupancy = 32;
+        OffsetT occupancy = tile_size * sm_occupancy * g_sm_count;
+        TestProblem<CUB, InputT, OutputT>(occupancy,  1,      RANDOM, Sum());
+        TestProblem<CUB, InputT, OutputT>(occupancy + 1, 1,   RANDOM, Sum());
+        TestProblem<CUB, InputT, OutputT>(occupancy - 1, 1,   RANDOM, Sum());
+
+        return cudaSuccess;
+    }
+};
+
+
+/// Test problem type
+template <
+    typename    InputT,
+    typename    OutputT,
+    typename    OffsetT>
+void TestType(
+    OffsetT     max_items,
+    OffsetT     max_segments)
+{
+    typedef typename DeviceReducePolicy<OutputT, OffsetT, cub::Sum>::MaxPolicy MaxPolicyT;
+
+    TestBySize<InputT, OutputT, OffsetT> dispatch(max_items, max_segments);
+
+    MaxPolicyT::Invoke(g_ptx_version, dispatch);
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    typedef int OffsetT;
+
+    OffsetT max_items       = 27000000;
+    OffsetT max_segments    = 34000;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    g_verbose_input = args.CheckCmdLineFlag("v2");
+    args.GetCmdLineArgument("n", max_items);
+    args.GetCmdLineArgument("s", max_segments);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("repeat", g_repeat);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--s=<num segments> "
+            "[--i=<timing iterations> "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "[--cdp]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get ptx version
+    CubDebugExit(PtxVersion(g_ptx_version));
+
+    // Get SM count
+    g_sm_count = args.deviceProp.multiProcessorCount;
+
+    std::numeric_limits<float>::max();
+
+#ifdef QUICKER_TEST
+
+    // Compile/run basic test
+
+
+
+    TestProblem<CUB, int, int>(     max_items, 1, RANDOM, Sum());
+
+    TestProblem<CUB, char, int>(    max_items, 1, RANDOM, Sum());
+
+    TestProblem<CUB, int, int>(     max_items, 1, RANDOM, ArgMax());
+
+    TestProblem<CUB, float, float>( max_items, 1, RANDOM, Sum());
+
+    TestProblem<CUB_SEGMENTED, int, int>(max_items, max_segments, RANDOM, Sum());
+
+
+#elif defined(QUICK_TEST)
+
+    // Compile/run quick comparison tests
+
+    TestProblem<CUB, char, char>(         max_items * 4, 1, UNIFORM, Sum());
+    TestProblem<THRUST, char, char>(      max_items * 4, 1, UNIFORM, Sum());
+
+    printf("\n----------------------------\n");
+    TestProblem<CUB, short, short>(        max_items * 2, 1, UNIFORM, Sum());
+    TestProblem<THRUST, short, short>(     max_items * 2, 1, UNIFORM, Sum());
+
+    printf("\n----------------------------\n");
+    TestProblem<CUB, int, int>(          max_items,     1, UNIFORM, Sum());
+    TestProblem<THRUST, int, int>(       max_items,     1, UNIFORM, Sum());
+
+    printf("\n----------------------------\n");
+    TestProblem<CUB, long long, long long>(    max_items / 2, 1, UNIFORM, Sum());
+    TestProblem<THRUST, long long, long long>( max_items / 2, 1, UNIFORM, Sum());
+
+    printf("\n----------------------------\n");
+    TestProblem<CUB, TestFoo, TestFoo>(      max_items / 4, 1, UNIFORM, Max());
+    TestProblem<THRUST, TestFoo, TestFoo>(   max_items / 4, 1, UNIFORM, Max());
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // Test different input types
+        TestType<char, char>(max_items, max_segments);
+
+        TestType<unsigned char, unsigned char>(max_items, max_segments);
+
+        TestType<char, int>(max_items, max_segments);
+
+//        TestType<short, short>(max_items, max_segments);
+//        TestType<int, int>(max_items, max_segments);
+//        TestType<long, long>(max_items, max_segments);
+//        TestType<long long, long long>(max_items, max_segments);
+//
+//        TestType<uchar2, uchar2>(max_items, max_segments);
+//        TestType<uint2, uint2>(max_items, max_segments);
+//        TestType<ulonglong2, ulonglong2>(max_items, max_segments);
+//        TestType<ulonglong4, ulonglong4>(max_items, max_segments);
+//
+//        TestType<TestFoo, TestFoo>(max_items, max_segments);
+//        TestType<TestBar, TestBar>(max_items, max_segments);
+
+    }
+
+#endif
+
+
+    printf("\n");
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_device_reduce_by_key.cu b/third_party/cub/test/test_device_reduce_by_key.cu
new file mode 100644
index 0000000..7d35eef
--- /dev/null
+++ b/third_party/cub/test/test_device_reduce_by_key.cu
@@ -0,0 +1,853 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of DeviceReduce::ReduceByKey utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <typeinfo>
+
+#include <thrust/device_ptr.h>
+#include <thrust/reduce.h>
+#include <thrust/iterator/constant_iterator.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/iterator/constant_input_iterator.cuh>
+#include <cub/device/device_reduce.cuh>
+#include <cub/device/device_run_length_encode.cuh>
+#include <cub/thread/thread_operators.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose           = false;
+int                     g_timing_iterations = 0;
+int                     g_repeat            = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+// Dispatch types
+enum Backend
+{
+    CUB,        // CUB method
+    THRUST,     // Thrust method
+    CDP,        // GPU-based (dynamic parallelism) dispatch to CUB method
+};
+
+
+//---------------------------------------------------------------------
+// Dispatch to different CUB entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to reduce-by-key entrypoint
+ */
+template <
+    typename                    KeyInputIteratorT,
+    typename                    KeyOutputIteratorT,
+    typename                    ValueInputIteratorT,
+    typename                    ValueOutputIteratorT,
+    typename                    NumRunsIteratorT,
+    typename                    EqualityOpT,
+    typename                    ReductionOpT,
+    typename                    OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>               dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void                        *d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    KeyInputIteratorT           d_keys_in,
+    KeyOutputIteratorT          d_keys_out,
+    ValueInputIteratorT         d_values_in,
+    ValueOutputIteratorT        d_values_out,
+    NumRunsIteratorT            d_num_runs,
+    EqualityOpT                  equality_op,
+    ReductionOpT                 reduction_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceReduce::ReduceByKey(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_keys_in,
+            d_keys_out,
+            d_values_in,
+            d_values_out,
+            d_num_runs,
+            reduction_op,
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+    return error;
+}
+
+
+//---------------------------------------------------------------------
+// Dispatch to different Thrust entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to reduce-by-key entrypoint
+ */
+template <
+    typename                    KeyInputIteratorT,
+    typename                    KeyOutputIteratorT,
+    typename                    ValueInputIteratorT,
+    typename                    ValueOutputIteratorT,
+    typename                    NumRunsIteratorT,
+    typename                    EqualityOpT,
+    typename                    ReductionOpT,
+    typename                    OffsetT>
+cudaError_t Dispatch(
+    Int2Type<THRUST>            dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void                        *d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    KeyInputIteratorT           d_keys_in,
+    KeyOutputIteratorT          d_keys_out,
+    ValueInputIteratorT         d_values_in,
+    ValueOutputIteratorT        d_values_out,
+    NumRunsIteratorT            d_num_runs,
+    EqualityOpT                 equality_op,
+    ReductionOpT                reduction_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // The input keys type
+    typedef typename std::iterator_traits<KeyInputIteratorT>::value_type KeyInputT;
+
+    // The output keys type
+    typedef typename If<(Equals<typename std::iterator_traits<KeyOutputIteratorT>::value_type, void>::VALUE),   // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<KeyInputIteratorT>::value_type,                                           // ... then the input iterator's value type,
+        typename std::iterator_traits<KeyOutputIteratorT>::value_type>::Type KeyOutputT;                        // ... else the output iterator's value type
+
+    // The input values type
+    typedef typename std::iterator_traits<ValueInputIteratorT>::value_type ValueInputT;
+
+    // The output values type
+    typedef typename If<(Equals<typename std::iterator_traits<ValueOutputIteratorT>::value_type, void>::VALUE), // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<ValueInputIteratorT>::value_type,                                         // ... then the input iterator's value type,
+        typename std::iterator_traits<ValueOutputIteratorT>::value_type>::Type ValueOuputT;                     // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<KeyInputT> d_keys_in_wrapper(d_keys_in);
+        thrust::device_ptr<KeyOutputT> d_keys_out_wrapper(d_keys_out);
+
+        thrust::device_ptr<ValueInputT> d_values_in_wrapper(d_values_in);
+        thrust::device_ptr<ValueOuputT> d_values_out_wrapper(d_values_out);
+
+        thrust::pair<thrust::device_ptr<KeyOutputT>, thrust::device_ptr<ValueOuputT> > d_out_ends;
+
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            d_out_ends = thrust::reduce_by_key(
+                d_keys_in_wrapper,
+                d_keys_in_wrapper + num_items,
+                d_values_in_wrapper,
+                d_keys_out_wrapper,
+                d_values_out_wrapper);
+        }
+
+        OffsetT num_segments = OffsetT(d_out_ends.first - d_keys_out_wrapper);
+        CubDebugExit(cudaMemcpy(d_num_runs, &num_segments, sizeof(OffsetT), cudaMemcpyHostToDevice));
+
+    }
+
+    return cudaSuccess;
+}
+
+
+
+//---------------------------------------------------------------------
+// CUDA Nested Parallelism Test Kernel
+//---------------------------------------------------------------------
+
+/**
+ * Simple wrapper kernel to invoke DeviceSelect
+ */
+template <
+    typename                    KeyInputIteratorT,
+    typename                    KeyOutputIteratorT,
+    typename                    ValueInputIteratorT,
+    typename                    ValueOutputIteratorT,
+    typename                    NumRunsIteratorT,
+    typename                    EqualityOpT,
+    typename                    ReductionOpT,
+    typename                    OffsetT>
+__global__ void CnpDispatchKernel(
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void                        *d_temp_storage,
+    size_t                      temp_storage_bytes,
+    KeyInputIteratorT           d_keys_in,
+    KeyOutputIteratorT          d_keys_out,
+    ValueInputIteratorT         d_values_in,
+    ValueOutputIteratorT        d_values_out,
+    NumRunsIteratorT            d_num_runs,
+    EqualityOpT                 equality_op,
+    ReductionOpT                reduction_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+
+#ifndef CUB_CDP
+    *d_cdp_error = cudaErrorNotSupported;
+#else
+    *d_cdp_error = Dispatch(Int2Type<CUB>(), timing_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, d_num_runs, equality_op, reduction_op, num_items, 0, debug_synchronous);
+
+    *d_temp_storage_bytes = temp_storage_bytes;
+#endif
+}
+
+
+/**
+ * Dispatch to CDP kernel
+ */
+template <
+    typename                    KeyInputIteratorT,
+    typename                    KeyOutputIteratorT,
+    typename                    ValueInputIteratorT,
+    typename                    ValueOutputIteratorT,
+    typename                    NumRunsIteratorT,
+    typename                    EqualityOpT,
+    typename                    ReductionOpT,
+    typename                    OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CDP>               dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void                        *d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    KeyInputIteratorT           d_keys_in,
+    KeyOutputIteratorT          d_keys_out,
+    ValueInputIteratorT         d_values_in,
+    ValueOutputIteratorT        d_values_out,
+    NumRunsIteratorT            d_num_runs,
+    EqualityOpT                 equality_op,
+    ReductionOpT                reduction_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // Invoke kernel to invoke device-side dispatch
+    CnpDispatchKernel<<<1,1>>>(timing_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, d_num_runs, equality_op, reduction_op, num_items, 0, debug_synchronous);
+
+    // Copy out temp_storage_bytes
+    CubDebugExit(cudaMemcpy(&temp_storage_bytes, d_temp_storage_bytes, sizeof(size_t) * 1, cudaMemcpyDeviceToHost));
+
+    // Copy out error
+    cudaError_t retval;
+    CubDebugExit(cudaMemcpy(&retval, d_cdp_error, sizeof(cudaError_t) * 1, cudaMemcpyDeviceToHost));
+    return retval;
+}
+
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem
+ */
+template <typename T>
+void Initialize(
+    int         entropy_reduction,
+    T           *h_in,
+    int         num_items,
+    int         max_segment)
+{
+    unsigned int max_int = (unsigned int) -1;
+
+    int key = 0;
+    int i = 0;
+    while (i < num_items)
+    {
+        // Select number of repeating occurrences
+
+        int repeat;
+
+        if (max_segment < 0)
+        {
+            repeat = num_items;
+        }
+        else if (max_segment < 2)
+        {
+            repeat = 1;
+        }
+        else
+        {
+            RandomBits(repeat, entropy_reduction);
+            repeat = (int) ((double(repeat) * double(max_segment)) / double(max_int));
+            repeat = CUB_MAX(1, repeat);
+        }
+
+        int j = i;
+        while (j < CUB_MIN(i + repeat, num_items))
+        {
+            InitValue(INTEGER_SEED, h_in[j], key);
+            j++;
+        }
+
+        i = j;
+        key++;
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve problem.  Returns total number of segments identified
+ */
+template <
+    typename        KeyInputIteratorT,
+    typename        ValueInputIteratorT,
+    typename        KeyT,
+    typename        ValueT,
+    typename        EqualityOpT,
+    typename        ReductionOpT>
+int Solve(
+    KeyInputIteratorT       h_keys_in,
+    KeyT                    *h_keys_reference,
+    ValueInputIteratorT     h_values_in,
+    ValueT                  *h_values_reference,
+    EqualityOpT             equality_op,
+    ReductionOpT            reduction_op,
+    int                     num_items)
+{
+    // First item
+    KeyT previous        = h_keys_in[0];
+    ValueT aggregate     = h_values_in[0];
+    int num_segments    = 0;
+
+    // Subsequent items
+    for (int i = 1; i < num_items; ++i)
+    {
+        if (!equality_op(previous, h_keys_in[i]))
+        {
+            h_keys_reference[num_segments] = previous;
+            h_values_reference[num_segments] = aggregate;
+            num_segments++;
+            aggregate = h_values_in[i];
+        }
+        else
+        {
+            aggregate = reduction_op(aggregate, h_values_in[i]);
+        }
+        previous = h_keys_in[i];
+    }
+
+    h_keys_reference[num_segments] = previous;
+    h_values_reference[num_segments] = aggregate;
+    num_segments++;
+
+    return num_segments;
+}
+
+
+
+/**
+ * Test DeviceSelect for a given problem input
+ */
+template <
+    Backend             BACKEND,
+    typename            DeviceKeyInputIteratorT,
+    typename            DeviceValueInputIteratorT,
+    typename            KeyT,
+    typename            ValueT,
+    typename            EqualityOpT,
+    typename            ReductionOpT>
+void Test(
+    DeviceKeyInputIteratorT     d_keys_in,
+    DeviceValueInputIteratorT   d_values_in,
+    KeyT*                       h_keys_reference,
+    ValueT*                     h_values_reference,
+    EqualityOpT                 equality_op,
+    ReductionOpT                reduction_op,
+    int                         num_segments,
+    int                         num_items)
+{
+    // Allocate device output arrays and number of segments
+    KeyT*   d_keys_out             = NULL;
+    ValueT* d_values_out           = NULL;
+    int*    d_num_runs         = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys_out, sizeof(KeyT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values_out, sizeof(ValueT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_runs, sizeof(int)));
+
+    // Allocate CDP device arrays
+    size_t          *d_temp_storage_bytes = NULL;
+    cudaError_t     *d_cdp_error = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes,  sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error,           sizeof(cudaError_t) * 1));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, d_num_runs, equality_op, reduction_op, num_items, 0, true));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Clear device output arrays
+    CubDebugExit(cudaMemset(d_keys_out, 0, sizeof(KeyT) * num_items));
+    CubDebugExit(cudaMemset(d_values_out, 0, sizeof(ValueT) * num_items));
+    CubDebugExit(cudaMemset(d_num_runs, 0, sizeof(int)));
+
+    // Run warmup/correctness iteration
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, d_num_runs, equality_op, reduction_op, num_items, 0, true));
+
+    // Check for correctness (and display results, if specified)
+    int compare1 = CompareDeviceResults(h_keys_reference, d_keys_out, num_segments, true, g_verbose);
+    printf("\t Keys %s ", compare1 ? "FAIL" : "PASS");
+
+    int compare2 = CompareDeviceResults(h_values_reference, d_values_out, num_segments, true, g_verbose);
+    printf("\t Values %s ", compare2 ? "FAIL" : "PASS");
+
+    int compare3 = CompareDeviceResults(&num_segments, d_num_runs, 1, true, g_verbose);
+    printf("\t Count %s ", compare3 ? "FAIL" : "PASS");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Performance
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), g_timing_iterations, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, d_num_runs, equality_op, reduction_op, num_items, 0, false));
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float   avg_millis  = elapsed_millis / g_timing_iterations;
+        float   giga_rate   = float(num_items) / avg_millis / 1000.0f / 1000.0f;
+        int     bytes_moved = ((num_items + num_segments) * sizeof(KeyT)) + ((num_items + num_segments) * sizeof(ValueT));
+        float   giga_bandwidth  = float(bytes_moved) / avg_millis / 1000.0f / 1000.0f;
+        printf(", %.3f avg ms, %.3f billion items/s, %.3f logical GB/s", avg_millis, giga_rate, giga_bandwidth);
+    }
+    printf("\n\n");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Cleanup
+    if (d_keys_out) CubDebugExit(g_allocator.DeviceFree(d_keys_out));
+    if (d_values_out) CubDebugExit(g_allocator.DeviceFree(d_values_out));
+    if (d_num_runs) CubDebugExit(g_allocator.DeviceFree(d_num_runs));
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Correctness asserts
+    AssertEquals(0, compare1 | compare2 | compare3);
+}
+
+
+/**
+ * Test DeviceSelect on pointer type
+ */
+template <
+    Backend         BACKEND,
+    typename        KeyT,
+    typename        ValueT,
+    typename        ReductionOpT>
+void TestPointer(
+    int             num_items,
+    int             entropy_reduction,
+    int             max_segment,
+    ReductionOpT    reduction_op)
+{
+    // Allocate host arrays
+    KeyT* h_keys_in        = new KeyT[num_items];
+    KeyT* h_keys_reference = new KeyT[num_items];
+
+    ValueT* h_values_in        = new ValueT[num_items];
+    ValueT* h_values_reference = new ValueT[num_items];
+
+    for (int i = 0; i < num_items; ++i)
+        InitValue(INTEGER_SEED, h_values_in[i], 1);
+
+    // Initialize problem and solution
+    Equality equality_op;
+    Initialize(entropy_reduction, h_keys_in, num_items, max_segment);
+    int num_segments = Solve(h_keys_in, h_keys_reference, h_values_in, h_values_reference, equality_op, reduction_op, num_items);
+
+    printf("\nPointer %s cub::DeviceReduce::ReduceByKey %s reduction of %d items, %d segments (avg run length %.3f), {%s,%s} key value pairs, max_segment %d, entropy_reduction %d\n",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        (Equals<ReductionOpT, Sum>::VALUE) ? "Sum" : "Max",
+        num_items, num_segments, float(num_items) / num_segments,
+        typeid(KeyT).name(), typeid(ValueT).name(),
+        max_segment, entropy_reduction);
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    KeyT     *d_keys_in = NULL;
+    ValueT   *d_values_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys_in, sizeof(KeyT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_values_in, sizeof(ValueT) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_keys_in, h_keys_in, sizeof(KeyT) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_values_in, h_values_in, sizeof(ValueT) * num_items, cudaMemcpyHostToDevice));
+
+    // Run Test
+    Test<BACKEND>(d_keys_in, d_values_in, h_keys_reference, h_values_reference, equality_op, reduction_op, num_segments, num_items);
+
+    // Cleanup
+    if (h_keys_in) delete[] h_keys_in;
+    if (h_values_in) delete[] h_values_in;
+    if (h_keys_reference) delete[] h_keys_reference;
+    if (h_values_reference) delete[] h_values_reference;
+    if (d_keys_in) CubDebugExit(g_allocator.DeviceFree(d_keys_in));
+    if (d_values_in) CubDebugExit(g_allocator.DeviceFree(d_values_in));
+}
+
+
+/**
+ * Test on iterator type
+ */
+template <
+    Backend         BACKEND,
+    typename        KeyT,
+    typename        ValueT,
+    typename        ReductionOpT>
+void TestIterator(
+    int             num_items,
+    int             entropy_reduction,
+    int             max_segment,
+    ReductionOpT    reduction_op)
+{
+    // Allocate host arrays
+    KeyT* h_keys_in        = new KeyT[num_items];
+    KeyT* h_keys_reference = new KeyT[num_items];
+
+    ValueT one_val;
+    InitValue(INTEGER_SEED, one_val, 1);
+    ConstantInputIterator<ValueT, int> h_values_in(one_val);
+    ValueT* h_values_reference = new ValueT[num_items];
+
+    // Initialize problem and solution
+    Equality equality_op;
+    Initialize(entropy_reduction, h_keys_in, num_items, max_segment);
+    int num_segments = Solve(h_keys_in, h_keys_reference, h_values_in, h_values_reference, equality_op, reduction_op, num_items);
+
+    printf("\nIterator %s cub::DeviceReduce::ReduceByKey %s reduction of %d items, %d segments (avg run length %.3f), {%s,%s} key value pairs, max_segment %d, entropy_reduction %d\n",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        (Equals<ReductionOpT, Sum>::VALUE) ? "Sum" : "Max",
+        num_items, num_segments, float(num_items) / num_segments,
+        typeid(KeyT).name(), typeid(ValueT).name(),
+        max_segment, entropy_reduction);
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    KeyT     *d_keys_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_keys_in, sizeof(KeyT) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_keys_in, h_keys_in, sizeof(KeyT) * num_items, cudaMemcpyHostToDevice));
+
+    // Run Test
+    Test<BACKEND>(d_keys_in, h_values_in, h_keys_reference, h_values_reference, equality_op, reduction_op, num_segments, num_items);
+
+    // Cleanup
+    if (h_keys_in) delete[] h_keys_in;
+    if (h_keys_reference) delete[] h_keys_reference;
+    if (h_values_reference) delete[] h_values_reference;
+    if (d_keys_in) CubDebugExit(g_allocator.DeviceFree(d_keys_in));
+}
+
+
+/**
+ * Test different gen modes
+ */
+template <
+    Backend         BACKEND,
+    typename        KeyT,
+    typename        ValueT,
+    typename        ReductionOpT>
+void Test(
+    int             num_items,
+    ReductionOpT    reduction_op,
+    int             max_segment)
+{
+    // 0 key-bit entropy reduction rounds
+    TestPointer<BACKEND, KeyT, ValueT>(num_items, 0, max_segment, reduction_op);
+
+    if (max_segment > 1)
+    {
+        // 2 key-bit entropy reduction rounds
+        TestPointer<BACKEND, KeyT, ValueT>(num_items, 2, max_segment, reduction_op);
+
+        // 7 key-bit entropy reduction rounds
+        TestPointer<BACKEND, KeyT, ValueT>(num_items, 7, max_segment, reduction_op);
+    }
+}
+
+
+/**
+ * Test different avg segment lengths modes
+ */
+template <
+    Backend         BACKEND,
+    typename        KeyT,
+    typename        ValueT,
+    typename        ReductionOpT>
+void Test(
+    int             num_items,
+    ReductionOpT    reduction_op)
+{
+    Test<BACKEND, KeyT, ValueT>(num_items, reduction_op, -1);
+    Test<BACKEND, KeyT, ValueT>(num_items, reduction_op, 1);
+
+    // Evaluate different max-segment lengths
+    for (int max_segment = 3; max_segment < CUB_MIN(num_items, (unsigned short) -1); max_segment *= 11)
+    {
+        Test<BACKEND, KeyT, ValueT>(num_items, reduction_op, max_segment);
+    }
+}
+
+
+
+/**
+ * Test different dispatch
+ */
+template <
+    typename        KeyT,
+    typename        ValueT,
+    typename        ReductionOpT>
+void TestDispatch(
+    int             num_items,
+    ReductionOpT    reduction_op)
+{
+    Test<CUB, KeyT, ValueT>(num_items, reduction_op);
+#ifdef CUB_CDP
+    Test<CDP, KeyT, ValueT>(num_items, reduction_op);
+#endif
+}
+
+
+/**
+ * Test different input sizes
+ */
+template <
+    typename        KeyT,
+    typename        ValueT,
+    typename        ReductionOpT>
+void TestSize(
+    int             num_items,
+    ReductionOpT    reduction_op)
+{
+    if (num_items < 0)
+    {
+        TestDispatch<KeyT, ValueT>(1,        reduction_op);
+        TestDispatch<KeyT, ValueT>(100,      reduction_op);
+        TestDispatch<KeyT, ValueT>(10000,    reduction_op);
+        TestDispatch<KeyT, ValueT>(1000000,  reduction_op);
+    }
+    else
+    {
+        TestDispatch<KeyT, ValueT>(num_items, reduction_op);
+    }
+
+}
+
+
+template <
+    typename        KeyT,
+    typename        ValueT>
+void TestOp(
+    int             num_items)
+{
+    TestSize<KeyT, ValueT>(num_items, cub::Sum());
+    TestSize<KeyT, ValueT>(num_items, cub::Max());
+}
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = -1;
+    int entropy_reduction   = 0;
+    int maxseg              = 1000;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("repeat", g_repeat);
+    args.GetCmdLineArgument("maxseg", maxseg);
+    args.GetCmdLineArgument("entropy", entropy_reduction);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--i=<timing iterations> "
+            "[--device=<device-id>] "
+            "[--maxseg=<max segment length>]"
+            "[--entropy=<segment length bit entropy reduction rounds>]"
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "[--cdp]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+    printf("\n");
+
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+#ifdef QUICKER_TEST
+
+    // Compile/run basic CUB test
+    if (num_items < 0) num_items = 32000000;
+
+    TestPointer<CUB, int, double>(num_items, entropy_reduction, maxseg, cub::Sum());
+    TestPointer<CUB, int, int>(num_items, entropy_reduction, maxseg, cub::Sum());
+    TestIterator<CUB, int, int>(num_items, entropy_reduction, maxseg, cub::Sum());
+
+#elif defined(QUICK_TEST)
+
+    // Compile/run quick tests
+    if (num_items < 0) num_items = 32000000;
+
+    printf("---- RLE int ---- \n");
+    TestIterator<CUB, int, int>(num_items, entropy_reduction, maxseg, cub::Sum());
+
+    printf("---- RLE long long ---- \n");
+    TestIterator<CUB, long long, int>(num_items, entropy_reduction, maxseg, cub::Sum());
+
+    printf("---- int ---- \n");
+    TestPointer<CUB, int, int>(num_items, entropy_reduction, maxseg, cub::Sum());
+    TestPointer<THRUST, int, int>(num_items, entropy_reduction, maxseg, cub::Sum());
+
+    printf("---- float ---- \n");
+    TestPointer<CUB, int, float>(num_items, entropy_reduction, maxseg, cub::Sum());
+    TestPointer<THRUST, int, float>(num_items, entropy_reduction, maxseg, cub::Sum());
+
+    if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+    {
+        printf("---- double ---- \n");
+        TestPointer<CUB, int, double>(num_items, entropy_reduction, maxseg, cub::Sum());
+        TestPointer<THRUST, int, double>(num_items, entropy_reduction, maxseg, cub::Sum());
+    }
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+
+        // Test different input types
+        TestOp<int, char>(num_items);
+        TestOp<int, short>(num_items);
+        TestOp<int, int>(num_items);
+        TestOp<int, long>(num_items);
+        TestOp<int, long long>(num_items);
+        TestOp<int, float>(num_items);
+        if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+            TestOp<int, double>(num_items);
+
+        TestOp<int, uchar2>(num_items);
+        TestOp<int, uint2>(num_items);
+        TestOp<int, uint3>(num_items);
+        TestOp<int, uint4>(num_items);
+        TestOp<int, ulonglong4>(num_items);
+        TestOp<int, TestFoo>(num_items);
+        TestOp<int, TestBar>(num_items);
+
+        TestOp<char, int>(num_items);
+        TestOp<long long, int>(num_items);
+        TestOp<TestFoo, int>(num_items);
+        TestOp<TestBar, int>(num_items);
+
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_device_run_length_encode.cu b/third_party/cub/test/test_device_run_length_encode.cu
new file mode 100644
index 0000000..7309db9
--- /dev/null
+++ b/third_party/cub/test/test_device_run_length_encode.cu
@@ -0,0 +1,890 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of DeviceReduce::RunLengthEncode utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <typeinfo>
+
+#include <thrust/device_ptr.h>
+#include <thrust/reduce.h>
+#include <thrust/iterator/constant_iterator.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/iterator/constant_input_iterator.cuh>
+#include <cub/device/device_reduce.cuh>
+#include <cub/device/device_run_length_encode.cuh>
+#include <cub/thread/thread_operators.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose           = false;
+int                     g_timing_iterations = 0;
+int                     g_repeat            = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+// Dispatch types
+enum Backend
+{
+    CUB,        // CUB method
+    THRUST,     // Thrust method
+    CDP,        // GPU-based (dynamic parallelism) dispatch to CUB method
+};
+
+// Operation types
+enum RleMethod
+{
+    RLE,                // Run length encode
+    NON_TRIVIAL,
+    CSR,
+};
+
+
+//---------------------------------------------------------------------
+// Dispatch to different CUB entrypoints
+//---------------------------------------------------------------------
+
+
+/**
+ * Dispatch to run-length encode entrypoint
+ */
+template <
+    typename                    InputIteratorT,
+    typename                    UniqueOutputIteratorT,
+    typename                    OffsetsOutputIteratorT,
+    typename                    LengthsOutputIteratorT,
+    typename                    NumRunsIterator,
+    typename                    OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<RLE>               method,
+    Int2Type<CUB>               dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    InputIteratorT              d_in,
+    UniqueOutputIteratorT       d_unique_out,
+    OffsetsOutputIteratorT      d_offsets_out,
+    LengthsOutputIteratorT      d_lengths_out,
+    NumRunsIterator             d_num_runs,
+    cub::Equality               equality_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceRunLengthEncode::Encode(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_unique_out,
+            d_lengths_out,
+            d_num_runs,
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to non-trivial runs entrypoint
+ */
+template <
+    typename                    InputIteratorT,
+    typename                    UniqueOutputIteratorT,
+    typename                    OffsetsOutputIteratorT,
+    typename                    LengthsOutputIteratorT,
+    typename                    NumRunsIterator,
+    typename                    OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<NON_TRIVIAL>       method,
+    Int2Type<CUB>               dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    InputIteratorT              d_in,
+    UniqueOutputIteratorT       d_unique_out,
+    OffsetsOutputIteratorT      d_offsets_out,
+    LengthsOutputIteratorT      d_lengths_out,
+    NumRunsIterator             d_num_runs,
+    cub::Equality               equality_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceRunLengthEncode::NonTrivialRuns(
+            d_temp_storage,
+            temp_storage_bytes,
+            d_in,
+            d_offsets_out,
+            d_lengths_out,
+            d_num_runs,
+            num_items,
+            stream,
+            debug_synchronous);
+    }
+    return error;
+}
+
+
+
+//---------------------------------------------------------------------
+// Dispatch to different Thrust entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to run-length encode entrypoint
+ */
+template <
+    typename                    InputIteratorT,
+    typename                    UniqueOutputIteratorT,
+    typename                    OffsetsOutputIteratorT,
+    typename                    LengthsOutputIteratorT,
+    typename                    NumRunsIterator,
+    typename                    OffsetT>
+cudaError_t Dispatch(
+    Int2Type<RLE>               method,
+    Int2Type<THRUST>            dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void                        *d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    InputIteratorT              d_in,
+    UniqueOutputIteratorT       d_unique_out,
+    OffsetsOutputIteratorT      d_offsets_out,
+    LengthsOutputIteratorT      d_lengths_out,
+    NumRunsIterator             d_num_runs,
+    cub::Equality               equality_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<UniqueOutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                                // ... then the input iterator's value type,
+        typename std::iterator_traits<UniqueOutputIteratorT>::value_type>::Type UniqueT;                          // ... else the output iterator's value type
+
+    // The lengths output value type
+    typedef typename If<(Equals<typename std::iterator_traits<LengthsOutputIteratorT>::value_type, void>::VALUE),   // LengthT =  (if output iterator's value type is void) ?
+        OffsetT,                                                                                                    // ... then the OffsetT type,
+        typename std::iterator_traits<LengthsOutputIteratorT>::value_type>::Type LengthT;                           // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<InputT>      d_in_wrapper(d_in);
+        thrust::device_ptr<UniqueT>     d_unique_out_wrapper(d_unique_out);
+        thrust::device_ptr<LengthT>     d_lengths_out_wrapper(d_lengths_out);
+
+        thrust::pair<thrust::device_ptr<UniqueT>, thrust::device_ptr<LengthT> > d_out_ends;
+
+        LengthT one_val;
+        InitValue(INTEGER_SEED, one_val, 1);
+        thrust::constant_iterator<LengthT> constant_one(one_val);
+
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            d_out_ends = thrust::reduce_by_key(
+                d_in_wrapper,
+                d_in_wrapper + num_items,
+                constant_one,
+                d_unique_out_wrapper,
+                d_lengths_out_wrapper);
+        }
+
+        OffsetT num_runs = OffsetT(d_out_ends.first - d_unique_out_wrapper);
+        CubDebugExit(cudaMemcpy(d_num_runs, &num_runs, sizeof(OffsetT), cudaMemcpyHostToDevice));
+    }
+
+    return cudaSuccess;
+}
+
+
+
+//---------------------------------------------------------------------
+// CUDA Nested Parallelism Test Kernel
+//---------------------------------------------------------------------
+
+/**
+ * Simple wrapper kernel to invoke DeviceRunLengthEncode
+ */
+template <
+    int                         RLE_METHOD,
+    typename                    InputIteratorT,
+    typename                    UniqueOutputIteratorT,
+    typename                    OffsetsOutputIteratorT,
+    typename                    LengthsOutputIteratorT,
+    typename                    NumRunsIterator,
+    typename                    EqualityOp,
+    typename                    OffsetT>
+__global__ void CnpDispatchKernel(
+    Int2Type<RLE_METHOD>            method,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t                      temp_storage_bytes,
+    InputIteratorT              d_in,
+    UniqueOutputIteratorT       d_unique_out,
+    OffsetsOutputIteratorT      d_offsets_out,
+    LengthsOutputIteratorT      d_lengths_out,
+    NumRunsIterator             d_num_runs,
+    cub::Equality               equality_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+
+#ifndef CUB_CDP
+    *d_cdp_error = cudaErrorNotSupported;
+#else
+    *d_cdp_error = Dispatch(method, Int2Type<CUB>(), timing_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_offsets_out, d_lengths_out, d_num_runs, equality_op, num_items, 0, debug_synchronous);
+
+    *d_temp_storage_bytes = temp_storage_bytes;
+#endif
+}
+
+
+/**
+ * Dispatch to CDP kernel
+ */
+template <
+    int                         RLE_METHOD,
+    typename                    InputIteratorT,
+    typename                    UniqueOutputIteratorT,
+    typename                    OffsetsOutputIteratorT,
+    typename                    LengthsOutputIteratorT,
+    typename                    NumRunsIterator,
+    typename                    EqualityOp,
+    typename                    OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<RLE_METHOD>        method,
+    Int2Type<CDP>               dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    InputIteratorT              d_in,
+    UniqueOutputIteratorT       d_unique_out,
+    OffsetsOutputIteratorT      d_offsets_out,
+    LengthsOutputIteratorT      d_lengths_out,
+    NumRunsIterator             d_num_runs,
+    EqualityOp                  equality_op,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // Invoke kernel to invoke device-side dispatch
+    CnpDispatchKernel<<<1,1>>>(method, timing_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_offsets_out, d_lengths_out, d_num_runs, equality_op, num_items, 0, debug_synchronous);
+
+    // Copy out temp_storage_bytes
+    CubDebugExit(cudaMemcpy(&temp_storage_bytes, d_temp_storage_bytes, sizeof(size_t) * 1, cudaMemcpyDeviceToHost));
+
+    // Copy out error
+    cudaError_t retval;
+    CubDebugExit(cudaMemcpy(&retval, d_cdp_error, sizeof(cudaError_t) * 1, cudaMemcpyDeviceToHost));
+    return retval;
+}
+
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem
+ */
+template <typename T>
+void Initialize(
+    int         entropy_reduction,
+    T           *h_in,
+    int         num_items,
+    int         max_segment)
+{
+    unsigned int max_int = (unsigned int) -1;
+
+    int key = 0;
+    int i = 0;
+    while (i < num_items)
+    {
+        // Select number of repeating occurrences for the current run
+        int repeat;
+        if (max_segment < 0)
+        {
+            repeat = num_items;
+        }
+        else if (max_segment < 2)
+        {
+            repeat = 1;
+        }
+        else
+        {
+            RandomBits(repeat, entropy_reduction);
+            repeat = (int) ((double(repeat) * double(max_segment)) / double(max_int));
+            repeat = CUB_MAX(1, repeat);
+        }
+
+        int j = i;
+        while (j < CUB_MIN(i + repeat, num_items))
+        {
+            InitValue(INTEGER_SEED, h_in[j], key);
+            j++;
+        }
+
+        i = j;
+        key++;
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve problem.  Returns total number of segments identified
+ */
+template <
+    RleMethod       RLE_METHOD,
+    typename        InputIteratorT,
+    typename        T,
+    typename        OffsetT,
+    typename        LengthT,
+    typename        EqualityOp>
+int Solve(
+    InputIteratorT  h_in,
+    T               *h_unique_reference,
+    OffsetT         *h_offsets_reference,
+    LengthT         *h_lengths_reference,
+    EqualityOp      equality_op,
+    int             num_items)
+{
+    if (num_items == 0) 
+        return 0;
+
+    // First item
+    T       previous        = h_in[0];
+    LengthT  length          = 1;
+    int     num_runs        = 0;
+    int     run_begin       = 0;
+
+    // Subsequent items
+    for (int i = 1; i < num_items; ++i)
+    {
+        if (!equality_op(previous, h_in[i]))
+        {
+            if ((RLE_METHOD != NON_TRIVIAL) || (length > 1))
+            {
+                h_unique_reference[num_runs]      = previous;
+                h_offsets_reference[num_runs]     = run_begin;
+                h_lengths_reference[num_runs]     = length;
+                num_runs++;
+            }
+            length = 1;
+            run_begin = i;
+        }
+        else
+        {
+            length++;
+        }
+        previous = h_in[i];
+    }
+
+    if ((RLE_METHOD != NON_TRIVIAL) || (length > 1))
+    {
+        h_unique_reference[num_runs]    = previous;
+        h_offsets_reference[num_runs]   = run_begin;
+        h_lengths_reference[num_runs]   = length;
+        num_runs++;
+    }
+
+    return num_runs;
+}
+
+
+
+/**
+ * Test DeviceRunLengthEncode for a given problem input
+ */
+template <
+    RleMethod           RLE_METHOD,
+    Backend             BACKEND,
+    typename            DeviceInputIteratorT,
+    typename            T,
+    typename            OffsetT,
+    typename            LengthT,
+    typename            EqualityOp>
+void Test(
+    DeviceInputIteratorT d_in,
+    T                   *h_unique_reference,
+    OffsetT             *h_offsets_reference,
+    LengthT             *h_lengths_reference,
+    EqualityOp          equality_op,
+    int                 num_runs,
+    int                 num_items)
+{
+    // Allocate device output arrays and number of segments
+    T*          d_unique_out       = NULL;
+    LengthT*    d_offsets_out      = NULL;
+    OffsetT*    d_lengths_out      = NULL;
+    int*        d_num_runs         = NULL;
+
+    if (RLE_METHOD == RLE)
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_unique_out, sizeof(T) * num_items));
+    if (RLE_METHOD == NON_TRIVIAL)
+        CubDebugExit(g_allocator.DeviceAllocate((void**)&d_offsets_out, sizeof(OffsetT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_lengths_out, sizeof(LengthT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_runs, sizeof(int)));
+
+    // Allocate CDP device arrays
+    size_t*          d_temp_storage_bytes = NULL;
+    cudaError_t*     d_cdp_error = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes,  sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error,           sizeof(cudaError_t) * 1));
+
+    // Allocate temporary storage
+    void*           d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(Dispatch(Int2Type<RLE_METHOD>(), Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_offsets_out, d_lengths_out, d_num_runs, equality_op, num_items, 0, true));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Clear device output arrays
+    if (RLE_METHOD == RLE)
+        CubDebugExit(cudaMemset(d_unique_out,   0, sizeof(T) * num_items));
+    if (RLE_METHOD == NON_TRIVIAL)
+        CubDebugExit(cudaMemset(d_offsets_out,  0, sizeof(OffsetT) * num_items));
+    CubDebugExit(cudaMemset(d_lengths_out,  0, sizeof(LengthT) * num_items));
+    CubDebugExit(cudaMemset(d_num_runs,     0, sizeof(int)));
+
+    // Run warmup/correctness iteration
+    CubDebugExit(Dispatch(Int2Type<RLE_METHOD>(), Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_offsets_out, d_lengths_out, d_num_runs, equality_op, num_items, 0, true));
+
+    // Check for correctness (and display results, if specified)
+    int compare0 = 0;
+    int compare1 = 0;
+    int compare2 = 0;
+    int compare3 = 0;
+
+    if (RLE_METHOD == RLE)
+    {
+        compare0 = CompareDeviceResults(h_unique_reference, d_unique_out, num_runs, true, g_verbose);
+        printf("\t Keys %s\n", compare0 ? "FAIL" : "PASS");
+    }
+
+    if (RLE_METHOD != RLE)
+    {
+        compare1 = CompareDeviceResults(h_offsets_reference, d_offsets_out, num_runs, true, g_verbose);
+        printf("\t Offsets %s\n", compare1 ? "FAIL" : "PASS");
+    }
+
+    if (RLE_METHOD != CSR)
+    {
+        compare2 = CompareDeviceResults(h_lengths_reference, d_lengths_out, num_runs, true, g_verbose);
+        printf("\t Lengths %s\n", compare2 ? "FAIL" : "PASS");
+    }
+
+    compare3 = CompareDeviceResults(&num_runs, d_num_runs, 1, true, g_verbose);
+    printf("\t Count %s\n", compare3 ? "FAIL" : "PASS");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Performance
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+    CubDebugExit(Dispatch(Int2Type<RLE_METHOD>(), Int2Type<BACKEND>(), g_timing_iterations, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_offsets_out, d_lengths_out, d_num_runs, equality_op, num_items, 0, false));
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float avg_millis = elapsed_millis / g_timing_iterations;
+        float giga_rate = float(num_items) / avg_millis / 1000.0f / 1000.0f;
+        int bytes_moved = (num_items * sizeof(T)) + (num_runs * (sizeof(OffsetT) + sizeof(LengthT)));
+        float giga_bandwidth = float(bytes_moved) / avg_millis / 1000.0f / 1000.0f;
+        printf(", %.3f avg ms, %.3f billion items/s, %.3f logical GB/s", avg_millis, giga_rate, giga_bandwidth);
+    }
+    printf("\n\n");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Cleanup
+    if (d_unique_out) CubDebugExit(g_allocator.DeviceFree(d_unique_out));
+    if (d_offsets_out) CubDebugExit(g_allocator.DeviceFree(d_offsets_out));
+    if (d_lengths_out) CubDebugExit(g_allocator.DeviceFree(d_lengths_out));
+    if (d_num_runs) CubDebugExit(g_allocator.DeviceFree(d_num_runs));
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Correctness asserts
+    AssertEquals(0, compare0 | compare1 | compare2 | compare3);
+}
+
+
+/**
+ * Test DeviceRunLengthEncode on pointer type
+ */
+template <
+    RleMethod       RLE_METHOD,
+    Backend         BACKEND,
+    typename        T,
+    typename        OffsetT,
+    typename        LengthT>
+void TestPointer(
+    int             num_items,
+    int             entropy_reduction,
+    int             max_segment)
+{
+    // Allocate host arrays
+    T*      h_in                    = new T[num_items];
+    T*      h_unique_reference      = new T[num_items];
+    OffsetT* h_offsets_reference     = new OffsetT[num_items];
+    LengthT* h_lengths_reference     = new LengthT[num_items];
+
+    for (int i = 0; i < num_items; ++i)
+        InitValue(INTEGER_SEED, h_offsets_reference[i], 1);
+
+    // Initialize problem and solution
+    Equality equality_op;
+    Initialize(entropy_reduction, h_in, num_items, max_segment);
+
+    int num_runs = Solve<RLE_METHOD>(h_in, h_unique_reference, h_offsets_reference, h_lengths_reference, equality_op, num_items);
+
+    printf("\nPointer %s cub::%s on %d items, %d segments (avg run length %.3f), {%s key, %s offset, %s length}, max_segment %d, entropy_reduction %d\n",
+        (RLE_METHOD == RLE) ? "DeviceReduce::RunLengthEncode" : (RLE_METHOD == NON_TRIVIAL) ? "DeviceRunLengthEncode::NonTrivialRuns" : "Other",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        num_items, num_runs, float(num_items) / num_runs,
+        typeid(T).name(), typeid(OffsetT).name(), typeid(LengthT).name(),
+        max_segment, entropy_reduction);
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    T* d_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * num_items, cudaMemcpyHostToDevice));
+
+    // Run Test
+    Test<RLE_METHOD, BACKEND>(d_in, h_unique_reference, h_offsets_reference, h_lengths_reference, equality_op, num_runs, num_items);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_unique_reference) delete[] h_unique_reference;
+    if (h_offsets_reference) delete[] h_offsets_reference;
+    if (h_lengths_reference) delete[] h_lengths_reference;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+}
+
+
+/**
+ * Test on iterator type
+ */
+template <
+    RleMethod       RLE_METHOD,
+    Backend         BACKEND,
+    typename        T,
+    typename        OffsetT,
+    typename        LengthT>
+void TestIterator(
+    int             num_items,
+    Int2Type<true>  is_primitive)
+{
+    // Allocate host arrays
+    T* h_unique_reference       = new T[num_items];
+    OffsetT* h_offsets_reference = new OffsetT[num_items];
+    LengthT* h_lengths_reference = new LengthT[num_items];
+
+    T one_val;
+    InitValue(INTEGER_SEED, one_val, 1);
+    ConstantInputIterator<T, int> h_in(one_val);
+
+    // Initialize problem and solution
+    Equality equality_op;
+    int num_runs = Solve<RLE_METHOD>(h_in, h_unique_reference, h_offsets_reference, h_lengths_reference, equality_op, num_items);
+
+    printf("\nIterator %s cub::%s on %d items, %d segments (avg run length %.3f), {%s key, %s offset, %s length}\n",
+        (RLE_METHOD == RLE) ? "DeviceReduce::RunLengthEncode" : (RLE_METHOD == NON_TRIVIAL) ? "DeviceRunLengthEncode::NonTrivialRuns" : "Other",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        num_items, num_runs, float(num_items) / num_runs,
+        typeid(T).name(), typeid(OffsetT).name(), typeid(LengthT).name());
+    fflush(stdout);
+
+    // Run Test
+    Test<RLE_METHOD, BACKEND>(h_in, h_unique_reference, h_offsets_reference, h_lengths_reference, equality_op, num_runs, num_items);
+
+    // Cleanup
+    if (h_unique_reference) delete[] h_unique_reference;
+    if (h_offsets_reference) delete[] h_offsets_reference;
+    if (h_lengths_reference) delete[] h_lengths_reference;
+}
+
+
+template <
+    RleMethod       RLE_METHOD,
+    Backend         BACKEND,
+    typename        T,
+    typename        OffsetT,
+    typename        LengthT>
+void TestIterator(
+    int             num_items,
+    Int2Type<false> is_primitive)
+{}
+
+
+/**
+ * Test different gen modes
+ */
+template <
+    RleMethod       RLE_METHOD,
+    Backend         BACKEND,
+    typename        T,
+    typename        OffsetT,
+    typename        LengthT>
+void Test(
+    int             num_items)
+{
+    // Test iterator (one run)
+    TestIterator<RLE_METHOD, BACKEND, T, OffsetT, LengthT>(num_items, Int2Type<Traits<T>::PRIMITIVE>());
+
+    // num_items runs
+    TestPointer<RLE_METHOD, BACKEND, T, OffsetT, LengthT>(num_items, 0, 1);
+
+    // Evaluate different run lengths
+    for (int max_segment = 3; max_segment < CUB_MIN(num_items, (unsigned short) -1); max_segment *= 3)
+    {
+        // Uniform selection run length
+        TestPointer<RLE_METHOD, BACKEND, T, OffsetT, LengthT>(num_items, 0, max_segment);
+
+        // Reduced-entropy run length
+        TestPointer<RLE_METHOD, BACKEND, T, OffsetT, LengthT>(num_items, 4, max_segment);
+    }
+}
+
+
+/**
+ * Test different dispatch
+ */
+template <
+    typename        T,
+    typename        OffsetT,
+    typename        LengthT>
+void TestDispatch(
+    int             num_items)
+{
+    Test<RLE,           CUB, T, OffsetT, LengthT>(num_items);
+    Test<NON_TRIVIAL,   CUB, T, OffsetT, LengthT>(num_items);
+
+#ifdef CUB_CDP
+    Test<RLE,           CDP, T, OffsetT, LengthT>(num_items);
+    Test<NON_TRIVIAL,   CDP, T, OffsetT, LengthT>(num_items);
+#endif
+}
+
+
+/**
+ * Test different input sizes
+ */
+template <
+    typename        T,
+    typename        OffsetT,
+    typename        LengthT>
+void TestSize(
+    int             num_items)
+{
+    if (num_items < 0)
+    {
+        TestDispatch<T, OffsetT, LengthT>(0);
+        TestDispatch<T, OffsetT, LengthT>(1);
+        TestDispatch<T, OffsetT, LengthT>(100);
+        TestDispatch<T, OffsetT, LengthT>(10000);
+        TestDispatch<T, OffsetT, LengthT>(1000000);
+
+        // Randomly select problem size between 1:10,000,000
+        unsigned int max_int = (unsigned int) -1;
+        for (int i = 0; i < 10; ++i)
+        {
+            unsigned int num_items;
+            RandomBits(num_items);
+            num_items = (unsigned int) ((double(num_items) * double(10000000)) / double(max_int));
+            num_items = CUB_MAX(1, num_items);
+            TestDispatch<T, OffsetT, LengthT>(num_items);
+        }
+    }
+    else
+    {
+        TestDispatch<T, OffsetT, LengthT>(num_items);
+    }
+
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = -1;
+    int entropy_reduction   = 0;
+    int max_segment              = 1000;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("repeat", g_repeat);
+    args.GetCmdLineArgument("maxseg", max_segment);
+    args.GetCmdLineArgument("entropy", entropy_reduction);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--i=<timing iterations> "
+            "[--device=<device-id>] "
+            "[--maxseg=<max segment length>]"
+            "[--entropy=<segment length bit entropy reduction rounds>]"
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "[--cdp]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+    printf("\n");
+
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+#ifdef QUICKER_TEST
+
+    // Compile/run basic CUB test
+    if (num_items < 0) num_items = 32000000;
+
+    TestPointer<RLE,            CUB, int, int, int>(    num_items, entropy_reduction, max_segment);
+    TestPointer<NON_TRIVIAL,    CUB, int, int, int>(    num_items, entropy_reduction, max_segment);
+    TestIterator<RLE,           CUB, float, int, int>(  num_items, Int2Type<Traits<float>::PRIMITIVE>());
+
+
+#elif defined(QUICK_TEST)
+
+    // Compile/run quick tests
+    if (num_items < 0) num_items = 32000000;
+
+    TestPointer<RLE,            CUB, int, int, int>(    num_items, entropy_reduction, max_segment);
+    TestPointer<RLE,            THRUST, int, int, int>(    num_items, entropy_reduction, max_segment);
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // Test different input types
+        TestSize<char,          int, int>(num_items);
+        TestSize<short,         int, int>(num_items);
+        TestSize<int,           int, int>(num_items);
+        TestSize<long,          int, int>(num_items);
+        TestSize<long long,     int, int>(num_items);
+        TestSize<float,         int, int>(num_items);
+        TestSize<double,        int, int>(num_items);
+
+        TestSize<uchar2,        int, int>(num_items);
+        TestSize<uint2,         int, int>(num_items);
+        TestSize<uint3,         int, int>(num_items);
+        TestSize<uint4,         int, int>(num_items);
+        TestSize<ulonglong4,    int, int>(num_items);
+        TestSize<TestFoo,       int, int>(num_items);
+        TestSize<TestBar,       int, int>(num_items);
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_device_scan.cu b/third_party/cub/test/test_device_scan.cu
new file mode 100644
index 0000000..1f97d05
--- /dev/null
+++ b/third_party/cub/test/test_device_scan.cu
@@ -0,0 +1,1015 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of DeviceScan utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <typeinfo>
+
+#include <thrust/device_ptr.h>
+#include <thrust/scan.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/iterator/constant_input_iterator.cuh>
+#include <cub/iterator/discard_output_iterator.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose           = false;
+int                     g_timing_iterations = 0;
+int                     g_repeat            = 0;
+double                  g_device_giga_bandwidth;
+CachingDeviceAllocator  g_allocator(true);
+
+// Dispatch types
+enum Backend
+{
+    CUB,        // CUB method
+    THRUST,     // Thrust method
+    CDP,        // GPU-based (dynamic parallelism) dispatch to CUB method
+};
+
+
+/**
+ * \brief WrapperFunctor (for precluding test-specialized dispatch to *Sum variants)
+ */
+template<typename OpT>
+struct WrapperFunctor
+{
+    OpT op;
+
+    WrapperFunctor(OpT op) : op(op) {}
+
+    template <typename T>
+    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
+    {
+        return op(a, b);
+    }
+};
+
+
+//---------------------------------------------------------------------
+// Dispatch to different CUB DeviceScan entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to exclusive scan entrypoint
+ */
+template <typename IsPrimitiveT, typename InputIteratorT, typename OutputIteratorT, typename ScanOpT, typename InitialValueT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    IsPrimitiveT        is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    ScanOpT             scan_op,
+    InitialValueT       initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, scan_op, initial_value, num_items, stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to exclusive sum entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename InitialValueT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    Int2Type<true>      is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    Sum                 scan_op,
+    InitialValueT       initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to inclusive scan entrypoint
+ */
+template <typename IsPrimitiveT, typename InputIteratorT, typename OutputIteratorT, typename ScanOpT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    IsPrimitiveT        is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    ScanOpT             scan_op,
+    NullType            initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceScan::InclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, scan_op, num_items, stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to inclusive sum entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>       dispatch_to,
+    Int2Type<true>      is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    Sum                 scan_op,
+    NullType            initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream, debug_synchronous);
+    }
+    return error;
+}
+
+//---------------------------------------------------------------------
+// Dispatch to different Thrust entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to exclusive scan entrypoint
+ */
+template <typename IsPrimitiveT, typename InputIteratorT, typename OutputIteratorT, typename ScanOpT, typename InitialValueT, typename OffsetT>
+cudaError_t Dispatch(
+    Int2Type<THRUST>    dispatch_to,
+    IsPrimitiveT        is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    ScanOpT             scan_op,
+    InitialValueT       initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<InputT> d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT> d_out_wrapper(d_out);
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            thrust::exclusive_scan(d_in_wrapper, d_in_wrapper + num_items, d_out_wrapper, initial_value, scan_op);
+        }
+    }
+
+    return cudaSuccess;
+}
+
+
+/**
+ * Dispatch to exclusive sum entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename InitialValueT, typename OffsetT>
+cudaError_t Dispatch(
+    Int2Type<THRUST>    dispatch_to,
+    Int2Type<true>      is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    Sum                 scan_op,
+    InitialValueT       initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<InputT> d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT> d_out_wrapper(d_out);
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            thrust::exclusive_scan(d_in_wrapper, d_in_wrapper + num_items, d_out_wrapper);
+        }
+    }
+
+    return cudaSuccess;
+}
+
+
+/**
+ * Dispatch to inclusive scan entrypoint
+ */
+template <typename IsPrimitiveT, typename InputIteratorT, typename OutputIteratorT, typename ScanOpT, typename OffsetT>
+cudaError_t Dispatch(
+    Int2Type<THRUST>    dispatch_to,
+    IsPrimitiveT        is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    ScanOpT             scan_op,
+    NullType            initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<InputT> d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT> d_out_wrapper(d_out);
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            thrust::inclusive_scan(d_in_wrapper, d_in_wrapper + num_items, d_out_wrapper, scan_op);
+        }
+    }
+
+    return cudaSuccess;
+}
+
+
+/**
+ * Dispatch to inclusive sum entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename OffsetT>
+cudaError_t Dispatch(
+    Int2Type<THRUST>    dispatch_to,
+    Int2Type<true>      is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    Sum                 scan_op,
+    NullType            initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<InputT> d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT> d_out_wrapper(d_out);
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            thrust::inclusive_scan(d_in_wrapper, d_in_wrapper + num_items, d_out_wrapper);
+        }
+    }
+
+    return cudaSuccess;
+}
+
+
+
+//---------------------------------------------------------------------
+// CUDA Nested Parallelism Test Kernel
+//---------------------------------------------------------------------
+
+/**
+ * Simple wrapper kernel to invoke DeviceScan
+ */
+template <typename IsPrimitiveT, typename InputIteratorT, typename OutputIteratorT, typename ScanOpT, typename InitialValueT, typename OffsetT>
+__global__ void CnpDispatchKernel(
+    IsPrimitiveT        is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t              temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    ScanOpT             scan_op,
+    InitialValueT       initial_value,
+    OffsetT             num_items,
+    bool                debug_synchronous)
+{
+#ifndef CUB_CDP
+    *d_cdp_error = cudaErrorNotSupported;
+#else
+    *d_cdp_error = Dispatch(
+        Int2Type<CUB>(),
+        is_primitive,
+        timing_timing_iterations,
+        d_temp_storage_bytes,
+        d_cdp_error,
+        d_temp_storage,
+        temp_storage_bytes,
+        d_in,
+        d_out,
+        scan_op,
+        initial_value,
+        num_items,
+        0,
+        debug_synchronous);
+
+    *d_temp_storage_bytes = temp_storage_bytes;
+#endif
+}
+
+
+/**
+ * Dispatch to CDP kernel
+ */
+template <typename IsPrimitiveT, typename InputIteratorT, typename OutputIteratorT, typename ScanOpT, typename InitialValueT, typename OffsetT>
+cudaError_t Dispatch(
+    Int2Type<CDP>       dispatch_to,
+    IsPrimitiveT        is_primitive,
+    int                 timing_timing_iterations,
+    size_t              *d_temp_storage_bytes,
+    cudaError_t         *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t&             temp_storage_bytes,
+    InputIteratorT      d_in,
+    OutputIteratorT     d_out,
+    ScanOpT             scan_op,
+    InitialValueT       initial_value,
+    OffsetT             num_items,
+    cudaStream_t        stream,
+    bool                debug_synchronous)
+{
+    // Invoke kernel to invoke device-side dispatch
+    CnpDispatchKernel<<<1,1>>>(
+        is_primitive,
+        timing_timing_iterations,
+        d_temp_storage_bytes,
+        d_cdp_error,
+        d_temp_storage,
+        temp_storage_bytes,
+        d_in,
+        d_out,
+        scan_op,
+        initial_value,
+        num_items,
+        debug_synchronous);
+
+    // Copy out temp_storage_bytes
+    CubDebugExit(cudaMemcpy(&temp_storage_bytes, d_temp_storage_bytes, sizeof(size_t) * 1, cudaMemcpyDeviceToHost));
+
+    // Copy out error
+    cudaError_t retval;
+    CubDebugExit(cudaMemcpy(&retval, d_cdp_error, sizeof(cudaError_t) * 1, cudaMemcpyDeviceToHost));
+    return retval;
+}
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem
+ */
+template <typename T>
+void Initialize(
+    GenMode      gen_mode,
+    T            *h_in,
+    int          num_items)
+{
+    for (int i = 0; i < num_items; ++i)
+    {
+        InitValue(gen_mode, h_in[i], i);
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+/**
+ * Solve exclusive-scan problem
+ */
+template <
+    typename        InputIteratorT,
+    typename        OutputT,
+    typename        ScanOpT>
+void Solve(
+    InputIteratorT  h_in,
+    OutputT         *h_reference,
+    int             num_items,
+    ScanOpT         scan_op,
+    OutputT         initial_value)
+{
+    if (num_items > 0)
+    {
+        OutputT val         = h_in[0];
+        h_reference[0]      = initial_value;
+        OutputT inclusive   = scan_op(initial_value, val);
+
+        for (int i = 1; i < num_items; ++i)
+        {
+            val = h_in[i];
+            h_reference[i] = inclusive;
+            inclusive = scan_op(inclusive, val);
+        }
+    }
+}
+
+
+/**
+ * Solve inclusive-scan problem
+ */
+template <
+    typename        InputIteratorT,
+    typename        OutputT,
+    typename        ScanOpT>
+void Solve(
+    InputIteratorT  h_in,
+    OutputT         *h_reference,
+    int             num_items,
+    ScanOpT         scan_op,
+    NullType)
+{
+    if (num_items > 0)
+    {
+        OutputT inclusive   = h_in[0];
+        h_reference[0]      = inclusive;
+
+        for (int i = 1; i < num_items; ++i)
+        {
+            OutputT val = h_in[i];
+            inclusive = scan_op(inclusive, val);
+            h_reference[i] = inclusive;
+        }
+    }
+}
+
+
+/**
+ * Test DeviceScan for a given problem input
+ */
+template <
+    Backend             BACKEND,
+    typename            DeviceInputIteratorT,
+    typename            OutputT,
+    typename            ScanOpT,
+    typename            InitialValueT>
+void Test(
+    DeviceInputIteratorT    d_in,
+    OutputT                 *h_reference,
+    int                     num_items,
+    ScanOpT                 scan_op,
+    InitialValueT           initial_value)
+{
+    typedef typename std::iterator_traits<DeviceInputIteratorT>::value_type InputT;
+
+    // Allocate device output array
+    OutputT *d_out = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(OutputT) * num_items));
+
+    // Allocate CDP device arrays
+    size_t          *d_temp_storage_bytes = NULL;
+    cudaError_t     *d_cdp_error = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes,  sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error,   sizeof(cudaError_t) * 1));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(Dispatch(
+        Int2Type<BACKEND>(),
+        Int2Type<Traits<OutputT>::PRIMITIVE>(),
+        1,
+        d_temp_storage_bytes,
+        d_cdp_error,
+        d_temp_storage,
+        temp_storage_bytes,
+        d_in,
+        d_out,
+        scan_op,
+        initial_value,
+        num_items,
+        0,
+        true));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Clear device output array
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(OutputT) * num_items));
+
+    // Run warmup/correctness iteration
+    CubDebugExit(Dispatch(
+        Int2Type<BACKEND>(),
+        Int2Type<Traits<OutputT>::PRIMITIVE>(),
+        1,
+        d_temp_storage_bytes,
+        d_cdp_error,
+        d_temp_storage,
+        temp_storage_bytes,
+        d_in,
+        d_out,
+        scan_op,
+        initial_value,
+        num_items,
+        0,
+        true));
+
+    // Check for correctness (and display results, if specified)
+    int compare = CompareDeviceResults(h_reference, d_out, num_items, true, g_verbose);
+    printf("\t%s", compare ? "FAIL" : "PASS");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Performance
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(),
+        Int2Type<Traits<OutputT>::PRIMITIVE>(),
+        g_timing_iterations,
+        d_temp_storage_bytes,
+        d_cdp_error,
+        d_temp_storage,
+        temp_storage_bytes,
+        d_in,
+        d_out,
+        scan_op,
+        initial_value,
+        num_items,
+        0,
+        false));
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float avg_millis = elapsed_millis / g_timing_iterations;
+        float giga_rate = float(num_items) / avg_millis / 1000.0f / 1000.0f;
+        float giga_bandwidth = giga_rate * (sizeof(InputT) + sizeof(OutputT));
+        printf(", %.3f avg ms, %.3f billion items/s, %.3f logical GB/s, %.1f%% peak", avg_millis, giga_rate, giga_bandwidth, giga_bandwidth / g_device_giga_bandwidth * 100.0);
+    }
+
+    printf("\n\n");
+
+    // Cleanup
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Correctness asserts
+    AssertEquals(0, compare);
+}
+
+
+/**
+ * Test DeviceScan on pointer type
+ */
+template <
+    Backend         BACKEND,
+    typename        InputT,
+    typename        OutputT,
+    typename        ScanOpT,
+    typename        InitialValueT>
+void TestPointer(
+    int             num_items,
+    GenMode         gen_mode,
+    ScanOpT         scan_op,
+    InitialValueT   initial_value)
+{
+    printf("\nPointer %s %s cub::DeviceScan::%s %d items, %s->%s (%d->%d bytes) , gen-mode %s\n",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        (Equals<InitialValueT, NullType>::VALUE) ? "Inclusive" : "Exclusive",
+        (Equals<ScanOpT, Sum>::VALUE) ? "Sum" : "Scan",
+        num_items,
+        typeid(InputT).name(), typeid(OutputT).name(), (int) sizeof(InputT), (int) sizeof(OutputT),
+        (gen_mode == RANDOM) ? "RANDOM" : (gen_mode == INTEGER_SEED) ? "SEQUENTIAL" : "HOMOGENOUS");
+    fflush(stdout);
+
+    // Allocate host arrays
+    InputT*     h_in        = new InputT[num_items];
+    OutputT*    h_reference = new OutputT[num_items];
+
+    // Initialize problem and solution
+    Initialize(gen_mode, h_in, num_items);
+    Solve(h_in, h_reference, num_items, scan_op, initial_value);
+
+    // Allocate problem device arrays
+    InputT *d_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(InputT) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(InputT) * num_items, cudaMemcpyHostToDevice));
+
+    // Run Test
+    Test<BACKEND>(d_in, h_reference, num_items, scan_op, initial_value);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+}
+
+
+/**
+ * Test DeviceScan on iterator type
+ */
+template <
+    Backend         BACKEND,
+    typename        InputT,
+    typename        OutputT,
+    typename        ScanOpT,
+    typename        InitialValueT>
+void TestIterator(
+    int             num_items,
+    ScanOpT         scan_op,
+    InitialValueT   initial_value)
+{
+    printf("\nIterator %s %s cub::DeviceScan::%s %d items, %s->%s (%d->%d bytes)\n",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        (Equals<InitialValueT, NullType>::VALUE) ? "Inclusive" : "Exclusive",
+        (Equals<ScanOpT, Sum>::VALUE) ? "Sum" : "Scan",
+        num_items,
+        typeid(InputT).name(), typeid(OutputT).name(), (int) sizeof(InputT), (int) sizeof(OutputT));
+    fflush(stdout);
+
+    // Use a constant iterator as the input
+    InputT val = InputT();
+    ConstantInputIterator<InputT, int> h_in(val);
+
+    // Allocate host arrays
+    OutputT*  h_reference = new OutputT[num_items];
+
+    // Initialize problem and solution
+    Solve(h_in, h_reference, num_items, scan_op, initial_value);
+
+    // Run Test
+    Test<BACKEND>(h_in, h_reference, num_items, scan_op, initial_value);
+
+    // Cleanup
+    if (h_reference) delete[] h_reference;
+}
+
+
+/**
+ * Test different gen modes
+ */
+template <
+    Backend         BACKEND,
+    typename        InputT,
+    typename        OutputT,
+    typename        ScanOpT,
+    typename        InitialValueT>
+void Test(
+    int             num_items,
+    ScanOpT         scan_op,
+    InitialValueT   initial_value)
+{
+    TestPointer<BACKEND, InputT, OutputT>(  num_items, UNIFORM, scan_op, initial_value);
+    TestPointer<BACKEND, InputT, OutputT>(  num_items, RANDOM,  scan_op, initial_value);
+    TestIterator<BACKEND, InputT, OutputT>( num_items, scan_op, initial_value);
+}
+
+
+/**
+ * Test different dispatch
+ */
+template <
+    typename        InputT,
+    typename        OutputT,
+    typename        ScanOpT,
+    typename        InitialValueT>
+void Test(
+    int             num_items,
+    ScanOpT         scan_op,
+    InitialValueT   initial_value)
+{
+    Test<CUB, InputT, OutputT>(num_items, scan_op, initial_value);
+#ifdef CUB_CDP
+    Test<CDP, InputT, OutputT>(num_items, scan_op, initial_value);
+#endif
+}
+
+
+/**
+ * Test different operators
+ */
+template <typename InputT, typename OutputT>
+void TestOp(
+    int             num_items,
+    OutputT         identity,
+    OutputT         initial_value)
+{
+    // Exclusive (use identity as initial value because it will dispatch to *Sum variants that don't take initial values)
+    Test<InputT, OutputT>(num_items, cub::Sum(), identity);
+    Test<InputT, OutputT>(num_items, cub::Max(), identity);
+
+    // Exclusive (non-specialized, so we can test initial-value)
+    Test<InputT, OutputT>(num_items, WrapperFunctor<cub::Sum>(cub::Sum()), initial_value);
+    Test<InputT, OutputT>(num_items, WrapperFunctor<cub::Max>(cub::Max()), initial_value);
+
+    // Inclusive (no initial value)
+    Test<InputT, OutputT>(num_items, cub::Sum(), NullType());
+    Test<InputT, OutputT>(num_items, cub::Max(), NullType());
+}
+
+
+/**
+ * Test different input sizes
+ */
+template <
+    typename InputT,
+    typename OutputT>
+void TestSize(
+    int     num_items,
+    OutputT identity,
+    OutputT initial_value)
+{
+    if (num_items < 0)
+    {
+        TestOp<InputT>(0,        identity, initial_value);
+        TestOp<InputT>(1,        identity, initial_value);
+        TestOp<InputT>(100,      identity, initial_value);
+        TestOp<InputT>(10000,    identity, initial_value);
+        TestOp<InputT>(1000000,  identity, initial_value);
+
+        // Randomly select problem size between 1:10,000,000
+        unsigned int max_int = (unsigned int) -1;
+        for (int i = 0; i < 10; ++i)
+        {
+            unsigned int num_items;
+            RandomBits(num_items);
+            num_items = (unsigned int) ((double(num_items) * double(10000000)) / double(max_int));
+            num_items = CUB_MAX(1, num_items);
+            TestOp<InputT>(num_items,  identity, initial_value);
+        }
+    }
+    else
+    {
+        TestOp<InputT>(num_items, identity, initial_value);
+    }
+}
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items = -1;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("repeat", g_repeat);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--i=<timing iterations> "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "[--cdp]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+    g_device_giga_bandwidth = args.device_giga_bandwidth;
+    printf("\n");
+
+#ifdef QUICKER_TEST
+
+    // Compile/run basic CUB test
+    if (num_items < 0) num_items = 32000000;
+
+    TestPointer<CUB, char, int>(         num_items    , UNIFORM, Sum(), (int) (0));
+    TestPointer<CUB, int, int>(         num_items    , UNIFORM, Sum(), (int) (0));
+
+#elif defined(QUICK_TEST)
+
+    // Get device ordinal
+    int device_ordinal;
+    CubDebugExit(cudaGetDevice(&device_ordinal));
+
+    // Get device SM version
+    int sm_version;
+    CubDebugExit(SmVersion(sm_version, device_ordinal));
+
+    // Compile/run quick tests
+    if (num_items < 0) num_items = 32000000;
+
+    TestPointer<CUB, char, char>(        num_items * ((sm_version <= 130) ? 1 : 4), UNIFORM, Sum(), char(0));
+    TestPointer<THRUST, char, char>(     num_items * ((sm_version <= 130) ? 1 : 4), UNIFORM, Sum(), char(0));
+
+    printf("----------------------------\n");
+    TestPointer<CUB, short, short>(       num_items * ((sm_version <= 130) ? 1 : 2), UNIFORM, Sum(), short(0));
+    TestPointer<THRUST, short, short>(    num_items * ((sm_version <= 130) ? 1 : 2), UNIFORM, Sum(), short(0));
+
+    printf("----------------------------\n");
+    TestPointer<CUB, int, int>(         num_items    , UNIFORM, Sum(), (int) (0));
+    TestPointer<THRUST, int, int>(      num_items    , UNIFORM, Sum(), (int) (0));
+
+    printf("----------------------------\n");
+    TestPointer<CUB, long long, long long>(   num_items / 2, UNIFORM, Sum(), (long long) (0));
+    TestPointer<THRUST, long long, long long>(num_items / 2, UNIFORM, Sum(), (long long) (0));
+
+    printf("----------------------------\n");
+    TestPointer<CUB, TestBar, TestBar>(     num_items / 4, UNIFORM, Sum(), TestBar());
+    TestPointer<THRUST, TestBar, TestBar>(  num_items / 4, UNIFORM, Sum(), TestBar());
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // Test different input+output data types
+        TestSize<unsigned char>(num_items,      (int) 0, (int) 99);
+
+        // Test same intput+output data types
+        TestSize<unsigned char>(num_items,      (unsigned char) 0,      (unsigned char) 99);
+        TestSize<char>(num_items,               (char) 0,               (char) 99);
+        TestSize<unsigned short>(num_items,     (unsigned short) 0,     (unsigned short)99);
+        TestSize<unsigned int>(num_items,       (unsigned int) 0,       (unsigned int) 99);
+        TestSize<unsigned long long>(num_items, (unsigned long long) 0, (unsigned long long) 99);
+
+        TestSize<uchar2>(num_items,     make_uchar2(0, 0),              make_uchar2(17, 21));
+        TestSize<char2>(num_items,      make_char2(0, 0),               make_char2(17, 21));
+        TestSize<ushort2>(num_items,    make_ushort2(0, 0),             make_ushort2(17, 21));
+        TestSize<uint2>(num_items,      make_uint2(0, 0),               make_uint2(17, 21));
+        TestSize<ulonglong2>(num_items, make_ulonglong2(0, 0),          make_ulonglong2(17, 21));
+        TestSize<uchar4>(num_items,     make_uchar4(0, 0, 0, 0),        make_uchar4(17, 21, 32, 85));
+        TestSize<char4>(num_items,      make_char4(0, 0, 0, 0),         make_char4(17, 21, 32, 85));
+
+        TestSize<ushort4>(num_items,    make_ushort4(0, 0, 0, 0),       make_ushort4(17, 21, 32, 85));
+        TestSize<uint4>(num_items,      make_uint4(0, 0, 0, 0),         make_uint4(17, 21, 32, 85));
+        TestSize<ulonglong4>(num_items, make_ulonglong4(0, 0, 0, 0),    make_ulonglong4(17, 21, 32, 85));
+
+        TestSize<TestFoo>(num_items,
+            TestFoo::MakeTestFoo(0, 0, 0, 0),
+            TestFoo::MakeTestFoo(1ll << 63, 1 << 31, short(1 << 15), char(1 << 7)));
+
+        TestSize<TestBar>(num_items,
+            TestBar(0, 0),
+            TestBar(1ll << 63, 1 << 31));
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_device_select_if.cu b/third_party/cub/test/test_device_select_if.cu
new file mode 100644
index 0000000..9bdca34
--- /dev/null
+++ b/third_party/cub/test/test_device_select_if.cu
@@ -0,0 +1,1039 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of DeviceSelect::If and DevicePartition::If utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <typeinfo>
+
+#include <thrust/device_ptr.h>
+#include <thrust/copy.h>
+#include <thrust/partition.h>
+#include <thrust/iterator/reverse_iterator.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/device/device_select.cuh>
+#include <cub/device/device_partition.cuh>
+#include <cub/iterator/counting_input_iterator.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose               = false;
+int                     g_timing_iterations     = 0;
+int                     g_repeat                = 0;
+float                   g_device_giga_bandwidth;
+CachingDeviceAllocator  g_allocator(true);
+
+// Dispatch types
+enum Backend
+{
+    CUB,        // CUB method
+    THRUST,     // Thrust method
+    CDP,        // GPU-based (dynamic parallelism) dispatch to CUB method
+};
+
+
+// Selection functor type
+template <typename T>
+struct LessThan
+{
+    T compare;
+
+    __host__ __device__ __forceinline__
+    LessThan(T compare) : compare(compare) {}
+
+    __host__ __device__ __forceinline__
+    bool operator()(const T &a) const {
+        return (a < compare);
+    }
+};
+
+//---------------------------------------------------------------------
+// Dispatch to different CUB DeviceSelect entrypoints
+//---------------------------------------------------------------------
+
+
+/**
+ * Dispatch to select if entrypoint
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>               dispatch_to,
+    Int2Type<false>             is_flagged,
+    Int2Type<false>             is_partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op, stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to partition if entrypoint
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>               dispatch_to,
+    Int2Type<false>             is_flagged,
+    Int2Type<true>              is_partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DevicePartition::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op, stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to select flagged entrypoint
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>               dispatch_to,
+    Int2Type<true>              is_flagged,
+    Int2Type<false>             partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSelect::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items, stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+/**
+ * Dispatch to partition flagged entrypoint
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>               dispatch_to,
+    Int2Type<true>              is_flagged,
+    Int2Type<true>              partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DevicePartition::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items, stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+//---------------------------------------------------------------------
+// Dispatch to different Thrust entrypoints
+//---------------------------------------------------------------------
+
+/**
+ * Dispatch to select if entrypoint
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+__host__ __forceinline__
+cudaError_t Dispatch(
+    Int2Type<THRUST>            dispatch_to,
+    Int2Type<false>             is_flagged,
+    Int2Type<false>             is_partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<OutputT>         d_out_wrapper_end;
+        thrust::device_ptr<InputT>          d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT>         d_out_wrapper(d_out);
+
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            d_out_wrapper_end = thrust::copy_if(d_in_wrapper, d_in_wrapper + num_items, d_out_wrapper, select_op);
+        }
+
+        OffsetT num_selected = OffsetT(d_out_wrapper_end - d_out_wrapper);
+        CubDebugExit(cudaMemcpy(d_num_selected_out, &num_selected, sizeof(OffsetT), cudaMemcpyHostToDevice));
+    }
+
+    return cudaSuccess;
+}
+
+
+/**
+ * Dispatch to partition if entrypoint
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+__host__ __forceinline__
+cudaError_t Dispatch(
+    Int2Type<THRUST>            dispatch_to,
+    Int2Type<false>             is_flagged,
+    Int2Type<true>              is_partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    typedef thrust::reverse_iterator<thrust::device_ptr<OutputT> > ReverseOutputIteratorT;
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::pair<thrust::device_ptr<OutputT>, ReverseOutputIteratorT> d_out_wrapper_end;
+
+        thrust::device_ptr<InputT>       d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT>       d_out_wrapper(d_out);
+
+        ReverseOutputIteratorT d_out_unselected(d_out_wrapper + num_items);
+
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            d_out_wrapper_end = thrust::partition_copy(
+                d_in_wrapper,
+                d_in_wrapper + num_items,
+                d_out_wrapper,
+                d_out_unselected,
+                select_op);
+        }
+
+        OffsetT num_selected = OffsetT(d_out_wrapper_end.first - d_out_wrapper);
+        CubDebugExit(cudaMemcpy(d_num_selected_out, &num_selected, sizeof(OffsetT), cudaMemcpyHostToDevice));
+    }
+
+    return cudaSuccess;
+}
+
+
+/**
+ * Dispatch to select flagged entrypoint
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+__host__ __forceinline__
+cudaError_t Dispatch(
+    Int2Type<THRUST>            dispatch_to,
+    Int2Type<true>              is_flagged,
+    Int2Type<false>             is_partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // The flag type
+    typedef typename std::iterator_traits<FlagIteratorT>::value_type FlagT;
+
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<OutputT>     d_out_wrapper_end;
+        thrust::device_ptr<InputT>      d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT>     d_out_wrapper(d_out);
+        thrust::device_ptr<FlagT>       d_flags_wrapper(d_flags);
+
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            d_out_wrapper_end = thrust::copy_if(d_in_wrapper, d_in_wrapper + num_items, d_flags_wrapper, d_out_wrapper, CastOp<bool>());
+        }
+
+        OffsetT num_selected = OffsetT(d_out_wrapper_end - d_out_wrapper);
+        CubDebugExit(cudaMemcpy(d_num_selected_out, &num_selected, sizeof(OffsetT), cudaMemcpyHostToDevice));
+    }
+
+    return cudaSuccess;
+}
+
+
+/**
+ * Dispatch to partition flagged entrypoint
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+__host__ __forceinline__
+cudaError_t Dispatch(
+    Int2Type<THRUST>            dispatch_to,
+    Int2Type<true>              is_flagged,
+    Int2Type<true>              is_partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // The flag type
+    typedef typename std::iterator_traits<FlagIteratorT>::value_type FlagT;
+
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    typedef thrust::reverse_iterator<thrust::device_ptr<OutputT> > ReverseOutputIteratorT;
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::pair<thrust::device_ptr<OutputT>, ReverseOutputIteratorT> d_out_wrapper_end;
+
+        thrust::device_ptr<InputT>  d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT> d_out_wrapper(d_out);
+        thrust::device_ptr<FlagT>   d_flags_wrapper(d_flags);
+        ReverseOutputIteratorT      d_out_unselected(d_out_wrapper + num_items);
+
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            d_out_wrapper_end = thrust::partition_copy(
+                d_in_wrapper,
+                d_in_wrapper + num_items,
+                d_flags_wrapper,
+                d_out_wrapper,
+                d_out_unselected,
+                CastOp<bool>());
+        }
+
+        OffsetT num_selected = OffsetT(d_out_wrapper_end.first - d_out_wrapper);
+        CubDebugExit(cudaMemcpy(d_num_selected_out, &num_selected, sizeof(OffsetT), cudaMemcpyHostToDevice));
+    }
+
+    return cudaSuccess;
+}
+
+
+//---------------------------------------------------------------------
+// CUDA Nested Parallelism Test Kernel
+//---------------------------------------------------------------------
+
+/**
+ * Simple wrapper kernel to invoke DeviceSelect
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT, typename IsFlaggedTag, typename IsPartitionTag>
+__global__ void CnpDispatchKernel(
+    IsFlaggedTag                is_flagged,
+    IsPartitionTag              is_partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t                      temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    bool                        debug_synchronous)
+{
+
+#ifndef CUB_CDP
+    *d_cdp_error = cudaErrorNotSupported;
+#else
+    *d_cdp_error = Dispatch(Int2Type<CUB>(), is_flagged, is_partition, timing_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items, select_op, 0, debug_synchronous);
+    *d_temp_storage_bytes = temp_storage_bytes;
+#endif
+}
+
+
+/**
+ * Dispatch to CDP kernel
+ */
+template <typename InputIteratorT, typename FlagIteratorT, typename SelectOpT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT, typename IsFlaggedTag, typename IsPartitionTag>
+cudaError_t Dispatch(
+    Int2Type<CDP>               dispatch_to,
+    IsFlaggedTag                is_flagged,
+    IsPartitionTag              is_partition,
+    int                         timing_timing_iterations,
+    size_t*                     d_temp_storage_bytes,
+    cudaError_t*                d_cdp_error,
+
+    void*                       d_temp_storage,
+    size_t&                     temp_storage_bytes,
+    InputIteratorT              d_in,
+    FlagIteratorT               d_flags,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    SelectOpT                   select_op,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // Invoke kernel to invoke device-side dispatch
+    CnpDispatchKernel<<<1,1>>>(is_flagged, is_partition, timing_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items, select_op, debug_synchronous);
+
+    // Copy out temp_storage_bytes
+    CubDebugExit(cudaMemcpy(&temp_storage_bytes, d_temp_storage_bytes, sizeof(size_t) * 1, cudaMemcpyDeviceToHost));
+
+    // Copy out error
+    cudaError_t retval;
+    CubDebugExit(cudaMemcpy(&retval, d_cdp_error, sizeof(cudaError_t) * 1, cudaMemcpyDeviceToHost));
+    return retval;
+}
+
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem
+ */
+template <typename T>
+void Initialize(
+    T*  h_in,
+    int num_items)
+{
+    for (int i = 0; i < num_items; ++i)
+    {
+        // Initialize each item to a randomly selected value from [0..126]
+        unsigned int value;
+        RandomBits(value, 0, 0, 7);
+        if (value == 127)
+            value = 126;
+        InitValue(INTEGER_SEED, h_in[i], value);
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve selection problem (and set corresponding flags)
+ */
+template <
+    typename        InputIteratorT,
+    typename        FlagIteratorT,
+    typename        SelectOpT,
+    typename        T>
+int Solve(
+    InputIteratorT  h_in,
+    SelectOpT       select_op,
+    T*              h_reference,
+    FlagIteratorT   h_flags,
+    int             num_items)
+{
+    int num_selected = 0;
+    for (int i = 0; i < num_items; ++i)
+    {
+        if ((h_flags[i] = select_op(h_in[i])))
+        {
+            h_reference[num_selected] = h_in[i];
+            num_selected++;
+        }
+        else
+        {
+            h_reference[num_items - (i - num_selected) - 1] = h_in[i];
+        }
+    }
+
+    return num_selected;
+}
+
+
+
+/**
+ * Test DeviceSelect for a given problem input
+ */
+template <
+    Backend             BACKEND,
+    bool                IS_FLAGGED,
+    bool                IS_PARTITION,
+    typename            DeviceInputIteratorT,
+    typename            FlagT,
+    typename            SelectOpT,
+    typename            T>
+void Test(
+    DeviceInputIteratorT    d_in,
+    FlagT*                  h_flags,
+    SelectOpT               select_op,
+    T*                      h_reference,
+    int                     num_selected,
+    int                     num_items)
+{
+    // Allocate device flags, output, and num-selected
+    FlagT*      d_flags = NULL;
+    T*          d_out = NULL;
+    int*        d_num_selected_out = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_flags, sizeof(FlagT) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(T) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_selected_out, sizeof(int)));
+
+    // Allocate CDP device arrays
+    size_t*         d_temp_storage_bytes = NULL;
+    cudaError_t*    d_cdp_error = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes,  sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error,           sizeof(cudaError_t) * 1));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), Int2Type<IS_FLAGGED>(), Int2Type<IS_PARTITION>(), 1, d_temp_storage_bytes, d_cdp_error,
+    d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items, select_op, 0, true));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Copy flags and clear device output array
+    CubDebugExit(cudaMemcpy(d_flags, h_flags, sizeof(FlagT) * num_items, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(T) * num_items));
+    CubDebugExit(cudaMemset(d_num_selected_out, 0, sizeof(int)));
+
+    // Run warmup/correctness iteration
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), Int2Type<IS_FLAGGED>(), Int2Type<IS_PARTITION>(), 1, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items, select_op, 0, true));
+
+    // Check for correctness (and display results, if specified)
+    int compare1 = (IS_PARTITION) ?
+        CompareDeviceResults(h_reference, d_out, num_items, true, g_verbose) :
+        CompareDeviceResults(h_reference, d_out, num_selected, true, g_verbose);
+    printf("\t Data %s\n", compare1 ? "FAIL" : "PASS");
+
+    int compare2 = CompareDeviceResults(&num_selected, d_num_selected_out, 1, true, g_verbose);
+    printf("\t Count %s\n", compare2 ? "FAIL" : "PASS");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Performance
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), Int2Type<IS_FLAGGED>(), Int2Type<IS_PARTITION>(), g_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items, select_op, 0, false));
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float   avg_millis          = elapsed_millis / g_timing_iterations;
+        float   giga_rate           = float(num_items) / avg_millis / 1000.0f / 1000.0f;
+        int     num_output_items    = (IS_PARTITION) ? num_items : num_selected;
+        int     num_flag_items      = (IS_FLAGGED) ? num_items : 0;
+        size_t  num_bytes           = sizeof(T) * (num_items + num_output_items) + sizeof(FlagT) * num_flag_items;
+        float   giga_bandwidth      = float(num_bytes) / avg_millis / 1000.0f / 1000.0f;
+
+        printf(", %.3f avg ms, %.3f billion items/s, %.3f logical GB/s, %.1f%% peak", avg_millis, giga_rate, giga_bandwidth, giga_bandwidth / g_device_giga_bandwidth * 100.0);
+    }
+    printf("\n\n");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Cleanup
+    if (d_flags) CubDebugExit(g_allocator.DeviceFree(d_flags));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_num_selected_out) CubDebugExit(g_allocator.DeviceFree(d_num_selected_out));
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Correctness asserts
+    AssertEquals(0, compare1 | compare2);
+}
+
+
+/**
+ * Test on pointer type
+ */
+template <
+    Backend         BACKEND,
+    bool            IS_FLAGGED,
+    bool            IS_PARTITION,
+    typename        T>
+void TestPointer(
+    int             num_items,
+    float           select_ratio)
+{
+    typedef char FlagT;
+
+    // Allocate host arrays
+    T*      h_in        = new T[num_items];
+    FlagT*  h_flags     = new FlagT[num_items];
+    T*      h_reference = new T[num_items];
+
+    // Initialize input
+    Initialize(h_in, num_items);
+
+    // Select a comparison value that is select_ratio through the space of [0,127]
+    T compare;
+    if (select_ratio <= 0.0)
+        InitValue(INTEGER_SEED, compare, 0);        // select none
+    else if (select_ratio >= 1.0)
+        InitValue(INTEGER_SEED, compare, 127);      // select all
+    else
+        InitValue(INTEGER_SEED, compare, int(double(double(127) * select_ratio)));
+
+    LessThan<T> select_op(compare);
+    int num_selected = Solve(h_in, select_op, h_reference, h_flags, num_items);
+
+    if (g_verbose) std::cout << "\nComparison item: " << compare << "\n";
+    printf("\nPointer %s cub::%s::%s %d items, %d selected (select ratio %.3f), %s %d-byte elements\n",
+        (IS_PARTITION) ? "DevicePartition" : "DeviceSelect",
+        (IS_FLAGGED) ? "Flagged" : "If",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        num_items, num_selected, float(num_selected) / num_items, typeid(T).name(), (int) sizeof(T));
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    T *d_in = NULL;
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * num_items, cudaMemcpyHostToDevice));
+
+    // Run Test
+    Test<BACKEND, IS_FLAGGED, IS_PARTITION>(d_in, h_flags, select_op, h_reference, num_selected, num_items);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (h_flags) delete[] h_flags;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+}
+
+
+/**
+ * Test on iterator type
+ */
+template <
+    Backend         BACKEND,
+    bool            IS_FLAGGED,
+    bool            IS_PARTITION,
+    typename        T>
+void TestIterator(
+    int             num_items,
+    float           select_ratio)
+{
+    typedef char FlagT;
+
+    // Allocate host arrays
+    T*      h_reference = new T[num_items];
+    FlagT*  h_flags = new FlagT[num_items];
+
+    // Use counting iterator as the input
+    CountingInputIterator<T, int> h_in(0);
+
+    // Select a comparison value that is select_ratio through the space of [0,127]
+    T compare;
+    if (select_ratio <= 0.0)
+        InitValue(INTEGER_SEED, compare, 0);        // select none
+    else if (select_ratio >= 1.0)
+        InitValue(INTEGER_SEED, compare, 127);      // select all
+    else
+        InitValue(INTEGER_SEED, compare, int(double(double(127) * select_ratio)));
+
+    LessThan<T> select_op(compare);
+    int num_selected = Solve(h_in, select_op, h_reference, h_flags, num_items);
+
+    if (g_verbose) std::cout << "\nComparison item: " << compare << "\n";
+    printf("\nIterator %s cub::%s::%s %d items, %d selected (select ratio %.3f), %s %d-byte elements\n",
+        (IS_PARTITION) ? "DevicePartition" : "DeviceSelect",
+        (IS_FLAGGED) ? "Flagged" : "If",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        num_items, num_selected, float(num_selected) / num_items, typeid(T).name(), (int) sizeof(T));
+    fflush(stdout);
+
+    // Run Test
+    Test<BACKEND, IS_FLAGGED, IS_PARTITION>(h_in, h_flags, select_op, h_reference, num_selected, num_items);
+
+    // Cleanup
+    if (h_reference) delete[] h_reference;
+    if (h_flags) delete[] h_flags;
+}
+
+
+/**
+ * Test different selection ratios
+ */
+template <
+    Backend         BACKEND,
+    bool            IS_FLAGGED,
+    bool            IS_PARTITION,
+    typename        T>
+void Test(
+    int             num_items)
+{
+    for (float select_ratio = 0.0f; select_ratio <= 1.0f; select_ratio += 0.2f)
+    {
+        TestPointer<BACKEND, IS_FLAGGED, IS_PARTITION, T>(num_items, select_ratio);
+    }
+}
+
+
+/**
+ * Test (select vs. partition) and (flagged vs. functor)
+ */
+template <
+    Backend         BACKEND,
+    typename        T>
+void TestMethod(
+    int             num_items)
+{
+    // Functor
+    Test<BACKEND, false, false, T>(num_items);
+    Test<BACKEND, false, true, T>(num_items);
+
+    // Flagged
+    Test<BACKEND, true, false, T>(num_items);
+    Test<BACKEND, true, true, T>(num_items);
+}
+
+
+/**
+ * Test different dispatch
+ */
+template <
+    typename        T>
+void TestOp(
+    int             num_items)
+{
+    TestMethod<CUB, T>(num_items);
+#ifdef CUB_CDP
+    TestMethod<CDP, T>(num_items);
+#endif
+}
+
+
+/**
+ * Test different input sizes
+ */
+template <typename T>
+void Test(
+    int             num_items)
+{
+    if (num_items < 0)
+    {
+        TestOp<T>(0);
+        TestOp<T>(1);
+        TestOp<T>(100);
+        TestOp<T>(10000);
+        TestOp<T>(1000000);
+    }
+    else
+    {
+        TestOp<T>(num_items);
+    }
+}
+
+/**
+ * Test select/partition on pointer types
+ */
+template <typename T>
+void ComparePointer(
+    int             num_items,
+    float           select_ratio)
+{
+    printf("-- Select-if ----------------------------\n");
+    TestPointer<CUB, false, false, T>(num_items, select_ratio);
+    TestPointer<THRUST, false, false, T>(num_items, select_ratio);
+
+    printf("-- Partition-if ----------------------------\n");
+    TestPointer<CUB, false, true, T>(num_items, select_ratio);
+    TestPointer<THRUST, false, true, T>(num_items, select_ratio);
+
+    printf("-- Select-flagged ----------------------------\n");
+    TestPointer<CUB, true, false, T>(num_items, select_ratio);
+    TestPointer<THRUST, true, false, T>(num_items, select_ratio);
+
+    printf("-- Partition-flagged ----------------------------\n");
+    TestPointer<CUB, true, true, T>(num_items, select_ratio);
+    TestPointer<THRUST, true, true, T>(num_items, select_ratio);
+
+}
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = -1;
+    float select_ratio      = 0.5;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("repeat", g_repeat);
+    args.GetCmdLineArgument("ratio", select_ratio);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--i=<timing iterations> "
+            "[--device=<device-id>] "
+            "[--ratio=<selection ratio, default 0.5>] "
+            "[--repeat=<repetitions of entire test suite>] "
+            "[--v] "
+            "[--cdp] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+    g_device_giga_bandwidth = args.device_giga_bandwidth;
+    printf("\n");
+
+#ifdef QUICKER_TEST
+
+    // Compile/run basic CUB test
+    if (num_items < 0) num_items = 32000000;
+
+    printf("-- Select-if ----------------------------\n");
+    TestPointer<CUB, false, false, int>(num_items, select_ratio);
+
+    printf("-- Partition-if ----------------------------\n");
+    TestPointer<CUB, false, true, int>(num_items, select_ratio);
+
+    printf("-- Select-flagged ----------------------------\n");
+    TestPointer<CUB, true, false, int>(num_items, select_ratio);
+
+    printf("-- Partition-flagged ----------------------------\n");
+    TestPointer<CUB, true, true, int>(num_items, select_ratio);
+
+
+#elif defined(QUICK_TEST)
+
+    // Get device ordinal
+    int device_ordinal;
+    CubDebugExit(cudaGetDevice(&device_ordinal));
+
+    // Get device SM version
+    int sm_version;
+    CubDebugExit(SmVersion(sm_version, device_ordinal));
+
+    // Compile/run quick tests
+    if (num_items < 0) num_items = 32000000;
+
+    printf("-- Iterator ----------------------------\n");
+    TestIterator<CUB, false, false, int>(num_items, select_ratio);
+
+    ComparePointer<char>(       num_items * ((sm_version <= 130) ? 1 : 4),  select_ratio);
+    ComparePointer<short>(      num_items * ((sm_version <= 130) ? 1 : 2),  select_ratio);
+    ComparePointer<int>(        num_items,                                  select_ratio);
+    ComparePointer<long long>(  num_items / 2,                              select_ratio);
+    ComparePointer<TestFoo>(    num_items / 4,                              select_ratio);
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // Test different input types
+        Test<unsigned char>(num_items);
+        Test<unsigned short>(num_items);
+        Test<unsigned int>(num_items);
+        Test<unsigned long long>(num_items);
+
+        Test<uchar2>(num_items);
+        Test<ushort2>(num_items);
+        Test<uint2>(num_items);
+        Test<ulonglong2>(num_items);
+
+        Test<uchar4>(num_items);
+        Test<ushort4>(num_items);
+        Test<uint4>(num_items);
+        Test<ulonglong4>(num_items);
+
+        Test<TestFoo>(num_items);
+        Test<TestBar>(num_items);
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_device_select_unique.cu b/third_party/cub/test/test_device_select_unique.cu
new file mode 100644
index 0000000..fff2958
--- /dev/null
+++ b/third_party/cub/test/test_device_select_unique.cu
@@ -0,0 +1,651 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of DeviceSelect::Unique utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <typeinfo>
+
+#include <thrust/device_ptr.h>
+#include <thrust/unique.h>
+
+#include <cub/util_allocator.cuh>
+#include <cub/iterator/counting_input_iterator.cuh>
+#include <cub/device/device_select.cuh>
+
+#include <thrust/device_ptr.h>
+#include <thrust/unique.h>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose               = false;
+int                     g_timing_iterations     = 0;
+int                     g_repeat                = 0;
+float                   g_device_giga_bandwidth;
+CachingDeviceAllocator  g_allocator(true);
+
+// Dispatch types
+enum Backend
+{
+    CUB,        // CUB method
+    THRUST,     // Thrust method
+    CDP,        // GPU-based (dynamic parallelism) dispatch to CUB method
+};
+
+
+//---------------------------------------------------------------------
+// Dispatch to different CUB DeviceSelect entrypoints
+//---------------------------------------------------------------------
+
+
+/**
+ * Dispatch to unique entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+CUB_RUNTIME_FUNCTION __forceinline__
+cudaError_t Dispatch(
+    Int2Type<CUB>               dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    InputIteratorT              d_in,
+    OutputIteratorT              d_out,
+    NumSelectedIteratorT         d_num_selected_out,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    cudaError_t error = cudaSuccess;
+    for (int i = 0; i < timing_timing_iterations; ++i)
+    {
+        error = DeviceSelect::Unique(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, stream, debug_synchronous);
+    }
+    return error;
+}
+
+
+//---------------------------------------------------------------------
+// Dispatch to different Thrust entrypoints
+//---------------------------------------------------------------------
+
+
+/**
+ * Dispatch to unique entrypoint
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+__host__ __forceinline__
+cudaError_t Dispatch(
+    Int2Type<THRUST>            dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void                        *d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    InputIteratorT              d_in,
+    OutputIteratorT             d_out,
+    NumSelectedIteratorT        d_num_selected_out,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // The input value type
+    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
+
+    // The output value type
+    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
+        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
+        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
+
+    if (d_temp_storage == 0)
+    {
+        temp_storage_bytes = 1;
+    }
+    else
+    {
+        thrust::device_ptr<OutputT> d_out_wrapper_end;
+        thrust::device_ptr<InputT> d_in_wrapper(d_in);
+        thrust::device_ptr<OutputT> d_out_wrapper(d_out);
+        for (int i = 0; i < timing_timing_iterations; ++i)
+        {
+            d_out_wrapper_end = thrust::unique_copy(d_in_wrapper, d_in_wrapper + num_items, d_out_wrapper);
+        }
+
+        OffsetT num_selected = OffsetT(d_out_wrapper_end - d_out_wrapper);
+        CubDebugExit(cudaMemcpy(d_num_selected_out, &num_selected, sizeof(OffsetT), cudaMemcpyHostToDevice));
+
+    }
+
+    return cudaSuccess;
+}
+
+
+
+//---------------------------------------------------------------------
+// CUDA Nested Parallelism Test Kernel
+//---------------------------------------------------------------------
+
+/**
+ * Simple wrapper kernel to invoke DeviceSelect
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+__global__ void CnpDispatchKernel(
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t                      temp_storage_bytes,
+    InputIteratorT              d_in,
+    OutputIteratorT              d_out,
+    NumSelectedIteratorT         d_num_selected_out,
+    OffsetT                     num_items,
+    bool                        debug_synchronous)
+{
+
+#ifndef CUB_CDP
+    *d_cdp_error = cudaErrorNotSupported;
+#else
+    *d_cdp_error = Dispatch(Int2Type<CUB>(), timing_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, 0, debug_synchronous);
+    *d_temp_storage_bytes = temp_storage_bytes;
+#endif
+}
+
+
+/**
+ * Dispatch to CDP kernel
+ */
+template <typename InputIteratorT, typename OutputIteratorT, typename NumSelectedIteratorT, typename OffsetT>
+cudaError_t Dispatch(
+    Int2Type<CDP>               dispatch_to,
+    int                         timing_timing_iterations,
+    size_t                      *d_temp_storage_bytes,
+    cudaError_t                 *d_cdp_error,
+
+    void*               d_temp_storage,
+    size_t                      &temp_storage_bytes,
+    InputIteratorT              d_in,
+    OutputIteratorT              d_out,
+    NumSelectedIteratorT         d_num_selected_out,
+    OffsetT                     num_items,
+    cudaStream_t                stream,
+    bool                        debug_synchronous)
+{
+    // Invoke kernel to invoke device-side dispatch
+    CnpDispatchKernel<<<1,1>>>(timing_timing_iterations, d_temp_storage_bytes, d_cdp_error,
+        d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, debug_synchronous);
+
+    // Copy out temp_storage_bytes
+    CubDebugExit(cudaMemcpy(&temp_storage_bytes, d_temp_storage_bytes, sizeof(size_t) * 1, cudaMemcpyDeviceToHost));
+
+    // Copy out error
+    cudaError_t retval;
+    CubDebugExit(cudaMemcpy(&retval, d_cdp_error, sizeof(cudaError_t) * 1, cudaMemcpyDeviceToHost));
+    return retval;
+}
+
+
+
+//---------------------------------------------------------------------
+// Test generation
+//---------------------------------------------------------------------
+
+
+/**
+ * Initialize problem
+ */
+template <typename T>
+void Initialize(
+    int         entropy_reduction,
+    T           *h_in,
+    int         num_items,
+    int         max_segment)
+{
+    unsigned int max_int = (unsigned int) -1;
+
+    int key = 0;
+    int i = 0;
+    while (i < num_items)
+    {
+        // Select number of repeating occurrences for the current run
+        int repeat;
+        if (max_segment < 0)
+        {
+            repeat = num_items;
+        }
+        else if (max_segment < 2)
+        {
+            repeat = 1;
+        }
+        else
+        {
+            RandomBits(repeat, entropy_reduction);
+            repeat = (int) ((double(repeat) * double(max_segment)) / double(max_int));
+            repeat = CUB_MAX(1, repeat);
+        }
+
+        int j = i;
+        while (j < CUB_MIN(i + repeat, num_items))
+        {
+            InitValue(INTEGER_SEED, h_in[j], key);
+            j++;
+        }
+
+        i = j;
+        key++;
+    }
+
+    if (g_verbose)
+    {
+        printf("Input:\n");
+        DisplayResults(h_in, num_items);
+        printf("\n\n");
+    }
+}
+
+
+/**
+ * Solve unique problem
+ */
+template <
+    typename        InputIteratorT,
+    typename        T>
+int Solve(
+    InputIteratorT  h_in,
+    T               *h_reference,
+    int             num_items)
+{
+    int num_selected = 0;
+    if (num_items > 0)
+    {
+        h_reference[num_selected] = h_in[0];
+        num_selected++;
+    }
+
+    for (int i = 1; i < num_items; ++i)
+    {
+        if (h_in[i] != h_in[i - 1])
+        {
+            h_reference[num_selected] = h_in[i];
+            num_selected++;
+        }
+    }
+
+    return num_selected;
+}
+
+
+
+/**
+ * Test DeviceSelect for a given problem input
+ */
+template <
+    Backend             BACKEND,
+    typename            DeviceInputIteratorT,
+    typename            T>
+void Test(
+    DeviceInputIteratorT d_in,
+    T                   *h_reference,
+    int                 num_selected,
+    int                 num_items)
+{
+    // Allocate device output array and num selected
+    T       *d_out            = NULL;
+    int     *d_num_selected_out   = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(T) * num_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_num_selected_out, sizeof(int)));
+
+    // Allocate CDP device arrays
+    size_t          *d_temp_storage_bytes = NULL;
+    cudaError_t     *d_cdp_error = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_temp_storage_bytes,  sizeof(size_t) * 1));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_cdp_error,           sizeof(cudaError_t) * 1));
+
+    // Allocate temporary storage
+    void            *d_temp_storage = NULL;
+    size_t          temp_storage_bytes = 0;
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, 0, true));
+    CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+    // Clear device output array
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(T) * num_items));
+    CubDebugExit(cudaMemset(d_num_selected_out, 0, sizeof(int)));
+
+    // Run warmup/correctness iteration
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), 1, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, 0, true));
+
+    // Check for correctness (and display results, if specified)
+    int compare1 = CompareDeviceResults(h_reference, d_out, num_selected, true, g_verbose);
+    printf("\t Data %s ", compare1 ? "FAIL" : "PASS");
+
+    int compare2 = CompareDeviceResults(&num_selected, d_num_selected_out, 1, true, g_verbose);
+    printf("\t Count %s ", compare2 ? "FAIL" : "PASS");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Performance
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+    CubDebugExit(Dispatch(Int2Type<BACKEND>(), g_timing_iterations, d_temp_storage_bytes, d_cdp_error, d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, 0, false));
+    gpu_timer.Stop();
+    float elapsed_millis = gpu_timer.ElapsedMillis();
+
+    // Display performance
+    if (g_timing_iterations > 0)
+    {
+        float avg_millis        = elapsed_millis / g_timing_iterations;
+        float giga_rate         = float(num_items) / avg_millis / 1000.0f / 1000.0f;
+        float giga_bandwidth    = float((num_items + num_selected) * sizeof(T)) / avg_millis / 1000.0f / 1000.0f;
+        printf(", %.3f avg ms, %.3f billion items/s, %.3f logical GB/s, %.1f%% peak", avg_millis, giga_rate, giga_bandwidth, giga_bandwidth / g_device_giga_bandwidth * 100.0);
+    }
+    printf("\n\n");
+
+    // Flush any stdout/stderr
+    fflush(stdout);
+    fflush(stderr);
+
+    // Cleanup
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_num_selected_out) CubDebugExit(g_allocator.DeviceFree(d_num_selected_out));
+    if (d_temp_storage_bytes) CubDebugExit(g_allocator.DeviceFree(d_temp_storage_bytes));
+    if (d_cdp_error) CubDebugExit(g_allocator.DeviceFree(d_cdp_error));
+    if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+
+    // Correctness asserts
+    AssertEquals(0, compare1 | compare2);
+}
+
+
+/**
+ * Test DeviceSelect on pointer type
+ */
+template <
+    Backend         BACKEND,
+    typename        T>
+void TestPointer(
+    int             num_items,
+    int             entropy_reduction,
+    int             max_segment)
+{
+    // Allocate host arrays
+    T*  h_in        = new T[num_items];
+    T*  h_reference = new T[num_items];
+
+    // Initialize problem and solution
+    Initialize(entropy_reduction, h_in, num_items, max_segment);
+    int num_selected = Solve(h_in, h_reference, num_items);
+
+    printf("\nPointer %s cub::DeviceSelect::Unique %d items, %d selected (avg run length %.3f), %s %d-byte elements, entropy_reduction %d\n",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        num_items, num_selected, float(num_items) / num_selected,
+        typeid(T).name(),
+        (int) sizeof(T),
+        entropy_reduction);
+    fflush(stdout);
+
+    // Allocate problem device arrays
+    T *d_in = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * num_items));
+
+    // Initialize device input
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * num_items, cudaMemcpyHostToDevice));
+
+    // Run Test
+    Test<BACKEND>(d_in, h_reference, num_selected, num_items);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+}
+
+
+/**
+ * Test DeviceSelect on iterator type
+ */
+template <
+    Backend         BACKEND,
+    typename        T>
+void TestIterator(
+    int             num_items)
+{
+    // Use a counting iterator as the input
+    CountingInputIterator<T, int> h_in(0);
+
+    // Allocate host arrays
+    T*  h_reference = new T[num_items];
+
+    // Initialize problem and solution
+    int num_selected = Solve(h_in, h_reference, num_items);
+
+    printf("\nIterator %s cub::DeviceSelect::Unique %d items, %d selected (avg run length %.3f), %s %d-byte elements\n",
+        (BACKEND == CDP) ? "CDP CUB" : (BACKEND == THRUST) ? "Thrust" : "CUB",
+        num_items, num_selected, float(num_items) / num_selected,
+        typeid(T).name(),
+        (int) sizeof(T));
+    fflush(stdout);
+
+    // Run Test
+    Test<BACKEND>(h_in, h_reference, num_selected, num_items);
+
+    // Cleanup
+    if (h_reference) delete[] h_reference;
+}
+
+
+/**
+ * Test different gen modes
+ */
+template <
+    Backend         BACKEND,
+    typename        T>
+void Test(
+    int             num_items)
+{
+    for (int max_segment = 1; ((max_segment > 0) && (max_segment < num_items)); max_segment *= 11)
+    {
+        TestPointer<BACKEND, T>(num_items, 0, max_segment);
+        TestPointer<BACKEND, T>(num_items, 2, max_segment);
+        TestPointer<BACKEND, T>(num_items, 7, max_segment);
+    }
+}
+
+
+/**
+ * Test different dispatch
+ */
+template <
+    typename        T>
+void TestOp(
+    int             num_items)
+{
+    Test<CUB, T>(num_items);
+#ifdef CUB_CDP
+    Test<CDP, T>(num_items);
+#endif
+}
+
+
+/**
+ * Test different input sizes
+ */
+template <typename T>
+void Test(
+    int             num_items)
+{
+    if (num_items < 0)
+    {
+        TestOp<T>(0);
+        TestOp<T>(1);
+        TestOp<T>(100);
+        TestOp<T>(10000);
+        TestOp<T>(1000000);
+    }
+    else
+    {
+        TestOp<T>(num_items);
+    }
+}
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    int num_items           = -1;
+    int entropy_reduction   = 0;
+    int maxseg              = 1000;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("n", num_items);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    args.GetCmdLineArgument("repeat", g_repeat);
+    args.GetCmdLineArgument("maxseg", maxseg);
+    args.GetCmdLineArgument("entropy", entropy_reduction);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--n=<input items> "
+            "[--i=<timing iterations> "
+            "[--device=<device-id>] "
+            "[--maxseg=<max segment length>]"
+            "[--entropy=<segment length bit entropy reduction rounds>]"
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "[--cdp]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+    g_device_giga_bandwidth = args.device_giga_bandwidth;
+    printf("\n");
+
+#ifdef QUICKER_TEST
+
+    // Compile/run basic CUB test
+    if (num_items < 0) num_items = 32000000;
+    TestPointer<CUB, int>(         num_items,                                 entropy_reduction, maxseg);
+
+#elif defined(QUICK_TEST)
+
+    // Get device ordinal
+    int device_ordinal;
+    CubDebugExit(cudaGetDevice(&device_ordinal));
+
+    // Get device SM version
+    int sm_version;
+    CubDebugExit(SmVersion(sm_version, device_ordinal));
+
+    // Compile/run quick tests
+    if (num_items < 0) num_items = 32000000;
+
+    printf("-- Iterator ----------------------------\n");
+    TestIterator<CUB, int>(        num_items);
+
+    printf("----------------------------\n");
+    TestPointer<CUB, char>(        num_items * ((sm_version <= 130) ? 1 : 4), entropy_reduction, maxseg);
+    TestPointer<THRUST, char>(     num_items * ((sm_version <= 130) ? 1 : 4), entropy_reduction, maxseg);
+
+    printf("----------------------------\n");
+    TestPointer<CUB, short>(       num_items * ((sm_version <= 130) ? 1 : 2), entropy_reduction, maxseg);
+    TestPointer<THRUST, short>(    num_items * ((sm_version <= 130) ? 1 : 2), entropy_reduction, maxseg);
+
+    printf("----------------------------\n");
+    TestPointer<CUB, int>(         num_items,                                 entropy_reduction, maxseg);
+    TestPointer<THRUST, int>(      num_items,                                 entropy_reduction, maxseg);
+
+    printf("----------------------------\n");
+    TestPointer<CUB, long long>(   num_items / 2,                             entropy_reduction, maxseg);
+    TestPointer<THRUST, long long>(num_items / 2,                             entropy_reduction, maxseg);
+
+    printf("----------------------------\n");
+    TestPointer<CUB, TestFoo>(     num_items / 4,                             entropy_reduction, maxseg);
+    TestPointer<THRUST, TestFoo>(  num_items / 4,                             entropy_reduction, maxseg);
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // Test different input types
+        Test<unsigned char>(num_items);
+        Test<unsigned short>(num_items);
+        Test<unsigned int>(num_items);
+        Test<unsigned long long>(num_items);
+
+        Test<uchar2>(num_items);
+        Test<ushort2>(num_items);
+        Test<uint2>(num_items);
+        Test<ulonglong2>(num_items);
+
+        Test<uchar4>(num_items);
+        Test<ushort4>(num_items);
+        Test<uint4>(num_items);
+        Test<ulonglong4>(num_items);
+
+        Test<TestFoo>(num_items);
+        Test<TestBar>(num_items);
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_grid_barrier.cu b/third_party/cub/test/test_grid_barrier.cu
new file mode 100644
index 0000000..e6e3b81
--- /dev/null
+++ b/third_party/cub/test/test_grid_barrier.cu
@@ -0,0 +1,152 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test evaluation for software global barrier throughput
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+
+#include <cub/grid/grid_barrier.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+/**
+ * Kernel that iterates through the specified number of software global barriers
+ */
+__global__ void Kernel(
+    GridBarrier global_barrier,
+    int iterations)
+{
+    for (int i = 0; i < iterations; i++)
+    {
+        global_barrier.Sync();
+    }
+}
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    cudaError_t retval = cudaSuccess;
+
+    // Defaults
+    int iterations = 10000;
+    int block_size = 128;
+    int grid_size = -1;
+
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+
+    // Get args
+    args.GetCmdLineArgument("i", iterations);
+    args.GetCmdLineArgument("grid-size", grid_size);
+    args.GetCmdLineArgument("block-size", block_size);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>]"
+            "[--i=<iterations>]"
+            "[--grid-size<grid-size>]"
+            "[--block-size<block-size>]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get device ordinal
+    int device_ordinal;
+    CubDebugExit(cudaGetDevice(&device_ordinal));
+
+    // Get device SM version
+    int sm_version;
+    CubDebugExit(SmVersion(sm_version, device_ordinal));
+
+    // Get SM properties
+    int sm_count, max_block_threads, max_sm_occupancy;
+    CubDebugExit(cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal));
+    CubDebugExit(cudaDeviceGetAttribute(&max_block_threads, cudaDevAttrMaxThreadsPerBlock, device_ordinal));
+    CubDebugExit(MaxSmOccupancy(max_sm_occupancy, EmptyKernel<void>, 32));
+
+    // Compute grid size and occupancy
+    int occupancy = CUB_MIN((max_block_threads / block_size), max_sm_occupancy);
+
+    if (grid_size == -1)
+    {
+        grid_size = occupancy * sm_count;
+    }
+    else
+    {
+        occupancy = grid_size / sm_count;
+    }
+
+    printf("Initializing software global barrier for Kernel<<<%d,%d>>> with %d occupancy\n",
+        grid_size, block_size, occupancy);
+    fflush(stdout);
+
+    // Init global barrier
+    GridBarrierLifetime global_barrier;
+    global_barrier.Setup(grid_size);
+
+    // Time kernel
+    GpuTimer gpu_timer;
+    gpu_timer.Start();
+    Kernel<<<grid_size, block_size>>>(global_barrier, iterations);
+    gpu_timer.Stop();
+
+    retval = CubDebug(cudaThreadSynchronize());
+
+    // Output timing results
+    float avg_elapsed = gpu_timer.ElapsedMillis() / float(iterations);
+    printf("%d iterations, %f total elapsed millis, %f avg elapsed millis\n",
+        iterations,
+        gpu_timer.ElapsedMillis(),
+        avg_elapsed);
+
+    return retval;
+}
diff --git a/third_party/cub/test/test_iterator.cu b/third_party/cub/test/test_iterator.cu
new file mode 100644
index 0000000..cb9b47e
--- /dev/null
+++ b/third_party/cub/test/test_iterator.cu
@@ -0,0 +1,805 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of iterator utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <iterator>
+#include <stdio.h>
+#include <typeinfo>
+
+#include <cub/iterator/arg_index_input_iterator.cuh>
+#include <cub/iterator/cache_modified_input_iterator.cuh>
+#include <cub/iterator/cache_modified_output_iterator.cuh>
+#include <cub/iterator/constant_input_iterator.cuh>
+#include <cub/iterator/counting_input_iterator.cuh>
+#include <cub/iterator/tex_obj_input_iterator.cuh>
+#include <cub/iterator/tex_ref_input_iterator.cuh>
+#include <cub/iterator/transform_input_iterator.cuh>
+
+#include <cub/util_type.cuh>
+#include <cub/util_allocator.cuh>
+
+#include "test_util.h"
+
+#include <thrust/device_ptr.h>
+#include <thrust/copy.h>
+
+using namespace cub;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose = false;
+CachingDeviceAllocator  g_allocator(true);
+
+// Dispatch types
+enum Backend
+{
+    CUB,        // CUB method
+    THRUST,     // Thrust method
+    CDP,        // GPU-based (dynamic parallelism) dispatch to CUB method
+};
+
+
+template <typename T>
+struct TransformOp
+{
+    // Increment transform
+    __host__ __device__ __forceinline__ T operator()(T input) const
+    {
+        T addend;
+        InitValue(INTEGER_SEED, addend, 1);
+        return input + addend;
+    }
+};
+
+struct SelectOp
+{
+    template <typename T>
+    __host__ __device__ __forceinline__ bool operator()(T input)
+    {
+        return true;
+    }
+};
+
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+/**
+ * Test random access input iterator
+ */
+template <
+    typename InputIteratorT,
+    typename T>
+__global__ void Kernel(
+    InputIteratorT    d_in,
+    T                 *d_out,
+    InputIteratorT    *d_itrs)
+{
+    d_out[0] = *d_in;               // Value at offset 0
+    d_out[1] = d_in[100];           // Value at offset 100
+    d_out[2] = *(d_in + 1000);      // Value at offset 1000
+    d_out[3] = *(d_in + 10000);     // Value at offset 10000
+
+    d_in++;
+    d_out[4] = d_in[0];             // Value at offset 1
+
+    d_in += 20;
+    d_out[5] = d_in[0];             // Value at offset 21
+    d_itrs[0] = d_in;               // Iterator at offset 21
+
+    d_in -= 10;
+    d_out[6] = d_in[0];             // Value at offset 11;
+
+    d_in -= 11;
+    d_out[7] = d_in[0];             // Value at offset 0
+    d_itrs[1] = d_in;               // Iterator at offset 0
+}
+
+
+
+//---------------------------------------------------------------------
+// Host testing subroutines
+//---------------------------------------------------------------------
+
+
+/**
+ * Run iterator test on device
+ */
+template <
+    typename        InputIteratorT,
+    typename        T,
+    int             TEST_VALUES>
+void Test(
+    InputIteratorT  d_in,
+    T               (&h_reference)[TEST_VALUES])
+{
+    // Allocate device arrays
+    T                 *d_out    = NULL;
+    InputIteratorT    *d_itrs   = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out,     sizeof(T) * TEST_VALUES));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_itrs,    sizeof(InputIteratorT) * 2));
+
+    int compare;
+
+    // Run unguarded kernel
+    Kernel<<<1, 1>>>(d_in, d_out, d_itrs);
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Check results
+    compare = CompareDeviceResults(h_reference, d_out, TEST_VALUES, g_verbose, g_verbose);
+    printf("\tValues: %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Check iterator at offset 21
+    InputIteratorT h_itr = d_in + 21;
+    compare = CompareDeviceResults(&h_itr, d_itrs, 1, g_verbose, g_verbose);
+    printf("\tIterators: %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Check iterator at offset 0
+    compare = CompareDeviceResults(&d_in, d_itrs + 1, 1, g_verbose, g_verbose);
+    printf("\tIterators: %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_itrs) CubDebugExit(g_allocator.DeviceFree(d_itrs));
+}
+
+
+/**
+ * Test constant iterator
+ */
+template <typename T>
+void TestConstant(T base)
+{
+    printf("\nTesting constant iterator on type %s (base: %lld)\n", typeid(T).name(), (unsigned long long) (base)); fflush(stdout);
+
+    //
+    // Test iterator manipulation in kernel
+    //
+
+    T h_reference[8] = {base, base, base, base, base, base, base, base};
+    ConstantInputIterator<T> d_itr(base);
+    Test(d_itr, h_reference);
+
+#if (THRUST_VERSION >= 100700)  // Thrust 1.7 or newer
+
+    //
+    // Test with thrust::copy_if()
+    //
+
+    int copy_items  = 100;
+    T   *h_copy     = new T[copy_items];
+    T   *d_copy     = NULL;
+
+    for (int i = 0; i < copy_items; ++i)
+        h_copy[i] = d_itr[i];
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_copy, sizeof(T) * copy_items));
+    thrust::device_ptr<T> d_copy_wrapper(d_copy);
+
+    thrust::copy_if(d_itr, d_itr + copy_items, d_copy_wrapper, SelectOp());
+
+    int compare = CompareDeviceResults(h_copy, d_copy, copy_items, g_verbose, g_verbose);
+    printf("\tthrust::copy_if(): %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    if (h_copy) delete[] h_copy;
+    if (d_copy) CubDebugExit(g_allocator.DeviceFree(d_copy));
+
+#endif // THRUST_VERSION
+}
+
+
+/**
+ * Test counting iterator
+ */
+template <typename T>
+void TestCounting(T base)
+{
+    printf("\nTesting counting iterator on type %s (base: %d) \n", typeid(T).name(), int(base)); fflush(stdout);
+
+    //
+    // Test iterator manipulation in kernel
+    //
+
+    // Initialize reference data
+    T h_reference[8];
+    h_reference[0] = base + 0;          // Value at offset 0
+    h_reference[1] = base + 100;        // Value at offset 100
+    h_reference[2] = base + 1000;       // Value at offset 1000
+    h_reference[3] = base + 10000;      // Value at offset 10000
+    h_reference[4] = base + 1;          // Value at offset 1
+    h_reference[5] = base + 21;         // Value at offset 21
+    h_reference[6] = base + 11;         // Value at offset 11
+    h_reference[7] = base + 0;          // Value at offset 0;
+
+    CountingInputIterator<T> d_itr(base);
+    Test(d_itr, h_reference);
+
+#if (THRUST_VERSION >= 100700)  // Thrust 1.7 or newer
+
+    //
+    // Test with thrust::copy_if()
+    //
+
+    unsigned long long  max_items   = ((1ull << ((sizeof(T) * 8) - 1)) - 1);
+    size_t  copy_items              = (size_t) CUB_MIN(max_items - base, 100);     // potential issue with differencing overflows when T is a smaller type than can handle the offset
+    T                   *h_copy     = new T[copy_items];
+    T                   *d_copy     = NULL;
+
+    for (unsigned long long i = 0; i < copy_items; ++i)
+        h_copy[i] = d_itr[i];
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_copy, sizeof(T) * copy_items));
+    thrust::device_ptr<T> d_copy_wrapper(d_copy);
+    thrust::copy_if(d_itr, d_itr + copy_items, d_copy_wrapper, SelectOp());
+
+    int compare = CompareDeviceResults(h_copy, d_copy, copy_items, g_verbose, g_verbose);
+    printf("\tthrust::copy_if(): %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    if (h_copy) delete[] h_copy;
+    if (d_copy) CubDebugExit(g_allocator.DeviceFree(d_copy));
+
+#endif // THRUST_VERSION
+}
+
+
+/**
+ * Test modified iterator
+ */
+template <typename T, typename CastT>
+void TestModified()
+{
+    printf("\nTesting cache-modified iterator on type %s\n", typeid(T).name()); fflush(stdout);
+
+    //
+    // Test iterator manipulation in kernel
+    //
+
+    const unsigned int TEST_VALUES = 11000;
+
+    T *h_data = new T[TEST_VALUES];
+    for (int i = 0; i < TEST_VALUES; ++i)
+    {
+        RandomBits(h_data[i]);
+    }
+
+    // Allocate device arrays
+    T *d_data = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_data, sizeof(T) * TEST_VALUES));
+    CubDebugExit(cudaMemcpy(d_data, h_data, sizeof(T) * TEST_VALUES, cudaMemcpyHostToDevice));
+
+    // Initialize reference data
+    T h_reference[8];
+    h_reference[0] = h_data[0];          // Value at offset 0
+    h_reference[1] = h_data[100];        // Value at offset 100
+    h_reference[2] = h_data[1000];       // Value at offset 1000
+    h_reference[3] = h_data[10000];      // Value at offset 10000
+    h_reference[4] = h_data[1];          // Value at offset 1
+    h_reference[5] = h_data[21];         // Value at offset 21
+    h_reference[6] = h_data[11];         // Value at offset 11
+    h_reference[7] = h_data[0];          // Value at offset 0;
+
+    Test(CacheModifiedInputIterator<LOAD_DEFAULT, T>((CastT*) d_data), h_reference);
+    Test(CacheModifiedInputIterator<LOAD_CA, T>((CastT*) d_data), h_reference);
+    Test(CacheModifiedInputIterator<LOAD_CG, T>((CastT*) d_data), h_reference);
+    Test(CacheModifiedInputIterator<LOAD_CS, T>((CastT*) d_data), h_reference);
+    Test(CacheModifiedInputIterator<LOAD_CV, T>((CastT*) d_data), h_reference);
+    Test(CacheModifiedInputIterator<LOAD_LDG, T>((CastT*) d_data), h_reference);
+    Test(CacheModifiedInputIterator<LOAD_VOLATILE, T>((CastT*) d_data), h_reference);
+
+#if (THRUST_VERSION >= 100700)  // Thrust 1.7 or newer
+
+    //
+    // Test with thrust::copy_if()
+    //
+
+    T *d_copy = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_copy, sizeof(T) * TEST_VALUES));
+
+    CacheModifiedInputIterator<LOAD_CG, T> d_in_itr((CastT*) d_data);
+    CacheModifiedOutputIterator<STORE_CG, T> d_out_itr((CastT*) d_copy);
+
+    thrust::copy_if(d_in_itr, d_in_itr + TEST_VALUES, d_out_itr, SelectOp());
+
+    int compare = CompareDeviceResults(h_data, d_copy, TEST_VALUES, g_verbose, g_verbose);
+    printf("\tthrust::copy_if(): %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (d_copy) CubDebugExit(g_allocator.DeviceFree(d_copy));
+
+#endif // THRUST_VERSION
+
+    if (h_data) delete[] h_data;
+    if (d_data) CubDebugExit(g_allocator.DeviceFree(d_data));
+}
+
+
+/**
+ * Test transform iterator
+ */
+template <typename T, typename CastT>
+void TestTransform()
+{
+    printf("\nTesting transform iterator on type %s\n", typeid(T).name()); fflush(stdout);
+
+    //
+    // Test iterator manipulation in kernel
+    //
+
+    const unsigned int TEST_VALUES = 11000;
+
+    T *h_data = new T[TEST_VALUES];
+    for (int i = 0; i < TEST_VALUES; ++i)
+    {
+        InitValue(INTEGER_SEED, h_data[i], i);
+    }
+
+    // Allocate device arrays
+    T *d_data = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_data, sizeof(T) * TEST_VALUES));
+    CubDebugExit(cudaMemcpy(d_data, h_data, sizeof(T) * TEST_VALUES, cudaMemcpyHostToDevice));
+
+    TransformOp<T> op;
+
+    // Initialize reference data
+    T h_reference[8];
+    h_reference[0] = op(h_data[0]);          // Value at offset 0
+    h_reference[1] = op(h_data[100]);        // Value at offset 100
+    h_reference[2] = op(h_data[1000]);       // Value at offset 1000
+    h_reference[3] = op(h_data[10000]);      // Value at offset 10000
+    h_reference[4] = op(h_data[1]);          // Value at offset 1
+    h_reference[5] = op(h_data[21]);         // Value at offset 21
+    h_reference[6] = op(h_data[11]);         // Value at offset 11
+    h_reference[7] = op(h_data[0]);          // Value at offset 0;
+
+    TransformInputIterator<T, TransformOp<T>, CastT*> d_itr((CastT*) d_data, op);
+    Test(d_itr, h_reference);
+
+#if (THRUST_VERSION >= 100700)  // Thrust 1.7 or newer
+
+    //
+    // Test with thrust::copy_if()
+    //
+
+    T *h_copy = new T[TEST_VALUES];
+    for (int i = 0; i < TEST_VALUES; ++i)
+        h_copy[i] = op(h_data[i]);
+
+    T *d_copy = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_copy, sizeof(T) * TEST_VALUES));
+    thrust::device_ptr<T> d_copy_wrapper(d_copy);
+
+    thrust::copy_if(d_itr, d_itr + TEST_VALUES, d_copy_wrapper, SelectOp());
+
+    int compare = CompareDeviceResults(h_copy, d_copy, TEST_VALUES, g_verbose, g_verbose);
+    printf("\tthrust::copy_if(): %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_copy) delete[] h_copy;
+    if (d_copy) CubDebugExit(g_allocator.DeviceFree(d_copy));
+
+#endif // THRUST_VERSION
+
+    if (h_data) delete[] h_data;
+    if (d_data) CubDebugExit(g_allocator.DeviceFree(d_data));
+}
+
+
+/**
+ * Test tex-obj texture iterator
+ */
+template <typename T, typename CastT>
+void TestTexObj()
+{
+    printf("\nTesting tex-obj iterator on type %s\n", typeid(T).name()); fflush(stdout);
+
+    //
+    // Test iterator manipulation in kernel
+    //
+
+    const unsigned int TEST_VALUES          = 11000;
+    const unsigned int DUMMY_OFFSET         = 500;
+    const unsigned int DUMMY_TEST_VALUES    = TEST_VALUES - DUMMY_OFFSET;
+
+    T *h_data = new T[TEST_VALUES];
+    for (int i = 0; i < TEST_VALUES; ++i)
+    {
+        RandomBits(h_data[i]);
+    }
+
+    // Allocate device arrays
+    T *d_data   = NULL;
+    T *d_dummy  = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_data, sizeof(T) * TEST_VALUES));
+    CubDebugExit(cudaMemcpy(d_data, h_data, sizeof(T) * TEST_VALUES, cudaMemcpyHostToDevice));
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_dummy, sizeof(T) * DUMMY_TEST_VALUES));
+    CubDebugExit(cudaMemcpy(d_dummy, h_data + DUMMY_OFFSET, sizeof(T) * DUMMY_TEST_VALUES, cudaMemcpyHostToDevice));
+
+    // Initialize reference data
+    T h_reference[8];
+    h_reference[0] = h_data[0];          // Value at offset 0
+    h_reference[1] = h_data[100];        // Value at offset 100
+    h_reference[2] = h_data[1000];       // Value at offset 1000
+    h_reference[3] = h_data[10000];      // Value at offset 10000
+    h_reference[4] = h_data[1];          // Value at offset 1
+    h_reference[5] = h_data[21];         // Value at offset 21
+    h_reference[6] = h_data[11];         // Value at offset 11
+    h_reference[7] = h_data[0];          // Value at offset 0;
+
+    // Create and bind obj-based test iterator
+    TexObjInputIterator<T> d_obj_itr;
+    CubDebugExit(d_obj_itr.BindTexture((CastT*) d_data, sizeof(T) * TEST_VALUES));
+
+    Test(d_obj_itr, h_reference);
+
+#if (THRUST_VERSION >= 100700)  // Thrust 1.7 or newer
+
+    //
+    // Test with thrust::copy_if()
+    //
+
+    T *d_copy = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_copy, sizeof(T) * TEST_VALUES));
+    thrust::device_ptr<T> d_copy_wrapper(d_copy);
+
+    CubDebugExit(cudaMemset(d_copy, 0, sizeof(T) * TEST_VALUES));
+    thrust::copy_if(d_obj_itr, d_obj_itr + TEST_VALUES, d_copy_wrapper, SelectOp());
+
+    int compare = CompareDeviceResults(h_data, d_copy, TEST_VALUES, g_verbose, g_verbose);
+    printf("\tthrust::copy_if(): %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    CubDebugExit(d_obj_itr.UnbindTexture());
+
+    if (d_copy) CubDebugExit(g_allocator.DeviceFree(d_copy));
+
+#endif  // THRUST_VERSION
+
+    if (h_data) delete[] h_data;
+    if (d_data) CubDebugExit(g_allocator.DeviceFree(d_data));
+    if (d_dummy) CubDebugExit(g_allocator.DeviceFree(d_dummy));
+}
+
+
+#if CUDA_VERSION >= 5050
+
+/**
+ * Test tex-ref texture iterator
+ */
+template <typename T, typename CastT>
+void TestTexRef()
+{
+    printf("\nTesting tex-ref iterator on type %s\n", typeid(T).name()); fflush(stdout);
+
+    //
+    // Test iterator manipulation in kernel
+    //
+
+    const unsigned int TEST_VALUES          = 11000;
+    const unsigned int DUMMY_OFFSET         = 500;
+    const unsigned int DUMMY_TEST_VALUES    = TEST_VALUES - DUMMY_OFFSET;
+
+    T *h_data = new T[TEST_VALUES];
+    for (int i = 0; i < TEST_VALUES; ++i)
+    {
+        RandomBits(h_data[i]);
+    }
+
+    // Allocate device arrays
+    T *d_data   = NULL;
+    T *d_dummy  = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_data, sizeof(T) * TEST_VALUES));
+    CubDebugExit(cudaMemcpy(d_data, h_data, sizeof(T) * TEST_VALUES, cudaMemcpyHostToDevice));
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_dummy, sizeof(T) * DUMMY_TEST_VALUES));
+    CubDebugExit(cudaMemcpy(d_dummy, h_data + DUMMY_OFFSET, sizeof(T) * DUMMY_TEST_VALUES, cudaMemcpyHostToDevice));
+
+    // Initialize reference data
+    T h_reference[8];
+    h_reference[0] = h_data[0];          // Value at offset 0
+    h_reference[1] = h_data[100];        // Value at offset 100
+    h_reference[2] = h_data[1000];       // Value at offset 1000
+    h_reference[3] = h_data[10000];      // Value at offset 10000
+    h_reference[4] = h_data[1];          // Value at offset 1
+    h_reference[5] = h_data[21];         // Value at offset 21
+    h_reference[6] = h_data[11];         // Value at offset 11
+    h_reference[7] = h_data[0];          // Value at offset 0;
+
+    // Create and bind ref-based test iterator
+    TexRefInputIterator<T, __LINE__> d_ref_itr;
+    CubDebugExit(d_ref_itr.BindTexture((CastT*) d_data, sizeof(T) * TEST_VALUES));
+
+    // Create and bind dummy iterator of same type to check with interferance
+    TexRefInputIterator<T, __LINE__> d_ref_itr2;
+    CubDebugExit(d_ref_itr2.BindTexture((CastT*) d_dummy, sizeof(T) * DUMMY_TEST_VALUES));
+
+    Test(d_ref_itr, h_reference);
+
+#if (THRUST_VERSION >= 100700)  // Thrust 1.7 or newer
+
+    //
+    // Test with thrust::copy_if()
+    //
+
+    T *d_copy = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_copy, sizeof(T) * TEST_VALUES));
+    thrust::device_ptr<T> d_copy_wrapper(d_copy);
+
+    CubDebugExit(cudaMemset(d_copy, 0, sizeof(T) * TEST_VALUES));
+    thrust::copy_if(d_ref_itr, d_ref_itr + TEST_VALUES, d_copy_wrapper, SelectOp());
+
+    int compare = CompareDeviceResults(h_data, d_copy, TEST_VALUES, g_verbose, g_verbose);
+    printf("\tthrust::copy_if(): %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    if (d_copy) CubDebugExit(g_allocator.DeviceFree(d_copy));
+
+#endif  // THRUST_VERSION
+
+    CubDebugExit(d_ref_itr.UnbindTexture());
+    CubDebugExit(d_ref_itr2.UnbindTexture());
+
+    if (h_data) delete[] h_data;
+    if (d_data) CubDebugExit(g_allocator.DeviceFree(d_data));
+    if (d_dummy) CubDebugExit(g_allocator.DeviceFree(d_dummy));
+}
+
+
+/**
+ * Test texture transform iterator
+ */
+template <typename T, typename CastT>
+void TestTexTransform()
+{
+    printf("\nTesting tex-transform iterator on type %s\n", typeid(T).name()); fflush(stdout);
+
+    //
+    // Test iterator manipulation in kernel
+    //
+
+    const unsigned int TEST_VALUES = 11000;
+
+    T *h_data = new T[TEST_VALUES];
+    for (int i = 0; i < TEST_VALUES; ++i)
+    {
+        InitValue(INTEGER_SEED, h_data[i], i);
+    }
+
+    // Allocate device arrays
+    T *d_data = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_data, sizeof(T) * TEST_VALUES));
+    CubDebugExit(cudaMemcpy(d_data, h_data, sizeof(T) * TEST_VALUES, cudaMemcpyHostToDevice));
+
+    TransformOp<T> op;
+
+    // Initialize reference data
+    T h_reference[8];
+    h_reference[0] = op(h_data[0]);          // Value at offset 0
+    h_reference[1] = op(h_data[100]);        // Value at offset 100
+    h_reference[2] = op(h_data[1000]);       // Value at offset 1000
+    h_reference[3] = op(h_data[10000]);      // Value at offset 10000
+    h_reference[4] = op(h_data[1]);          // Value at offset 1
+    h_reference[5] = op(h_data[21]);         // Value at offset 21
+    h_reference[6] = op(h_data[11]);         // Value at offset 11
+    h_reference[7] = op(h_data[0]);          // Value at offset 0;
+
+    // Create and bind texture iterator
+    typedef TexRefInputIterator<T, __LINE__> TextureIterator;
+
+    TextureIterator d_tex_itr;
+    CubDebugExit(d_tex_itr.BindTexture((CastT*) d_data, sizeof(T) * TEST_VALUES));
+
+    // Create transform iterator
+    TransformInputIterator<T, TransformOp<T>, TextureIterator> xform_itr(d_tex_itr, op);
+
+    Test(xform_itr, h_reference);
+
+#if (THRUST_VERSION >= 100700)  // Thrust 1.7 or newer
+
+    //
+    // Test with thrust::copy_if()
+    //
+
+    T *h_copy = new T[TEST_VALUES];
+    for (int i = 0; i < TEST_VALUES; ++i)
+        h_copy[i] = op(h_data[i]);
+
+    T *d_copy = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_copy, sizeof(T) * TEST_VALUES));
+    thrust::device_ptr<T> d_copy_wrapper(d_copy);
+
+    thrust::copy_if(xform_itr, xform_itr + TEST_VALUES, d_copy_wrapper, SelectOp());
+
+    int compare = CompareDeviceResults(h_copy, d_copy, TEST_VALUES, g_verbose, g_verbose);
+    printf("\tthrust::copy_if(): %s\n", (compare) ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Cleanup
+    if (h_copy) delete[] h_copy;
+    if (d_copy) CubDebugExit(g_allocator.DeviceFree(d_copy));
+
+#endif  // THRUST_VERSION
+
+    CubDebugExit(d_tex_itr.UnbindTexture());
+    if (h_data) delete[] h_data;
+    if (d_data) CubDebugExit(g_allocator.DeviceFree(d_data));
+}
+
+#endif  // CUDA_VERSION
+
+
+
+
+/**
+ * Run non-integer tests
+ */
+template <typename T, typename CastT>
+void Test(Int2Type<false> is_integer)
+{
+    TestModified<T, CastT>();
+    TestTransform<T, CastT>();
+
+#if CUB_CDP
+    // Test tex-obj iterators if CUDA dynamic parallelism enabled
+    TestTexObj<T, CastT>(type_string);
+#endif  // CUB_CDP
+
+#if CUDA_VERSION >= 5050
+    // Test tex-ref iterators for CUDA 5.5
+    TestTexRef<T, CastT>();
+    TestTexTransform<T, CastT>();
+#endif  // CUDA_VERSION
+}
+
+/**
+ * Run integer tests
+ */
+template <typename T, typename CastT>
+void Test(Int2Type<true> is_integer)
+{
+    TestConstant<T>(0);
+    TestConstant<T>(99);
+
+    TestCounting<T>(0);
+    TestCounting<T>(99);
+
+    // Run non-integer tests
+    Test<T, CastT>(Int2Type<false>());
+}
+
+/**
+ * Run tests
+ */
+template <typename T>
+void Test()
+{
+    enum {
+        IS_INTEGER = (Traits<T>::CATEGORY == SIGNED_INTEGER) || (Traits<T>::CATEGORY == UNSIGNED_INTEGER)
+    };
+
+    // Test non-const type
+    Test<T, T>(Int2Type<IS_INTEGER>());
+
+    // Test non-const type
+    Test<T, const T>(Int2Type<IS_INTEGER>());
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+    // Evaluate different data types
+    Test<char>();
+    Test<short>();
+    Test<int>();
+    Test<long>();
+    Test<long long>();
+    Test<float>();
+    if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+        Test<double>();
+
+    Test<char2>();
+    Test<short2>();
+    Test<int2>();
+    Test<long2>();
+    Test<longlong2>();
+    Test<float2>();
+    if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+        Test<double2>();
+
+    Test<char3>();
+    Test<short3>();
+    Test<int3>();
+    Test<long3>();
+    Test<longlong3>();
+    Test<float3>();
+    if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+        Test<double3>();
+
+    Test<char4>();
+    Test<short4>();
+    Test<int4>();
+    Test<long4>();
+    Test<longlong4>();
+    Test<float4>();
+    if (ptx_version > 120)                          // Don't check doubles on PTX120 or below because they're down-converted
+        Test<double4>();
+
+    Test<TestFoo>();
+    Test<TestBar>();
+
+    printf("\nTest complete\n"); fflush(stdout);
+
+    return 0;
+}
+
+
+
diff --git a/third_party/cub/test/test_util.h b/third_party/cub/test/test_util.h
new file mode 100644
index 0000000..491a5be
--- /dev/null
+++ b/third_party/cub/test/test_util.h
@@ -0,0 +1,1628 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+
+#pragma once
+
+#if defined(_WIN32) || defined(_WIN64)
+    #include <windows.h>
+    #undef small            // Windows is terrible for polluting macro namespace
+#else
+    #include <sys/resource.h>
+#endif
+
+#include <cuda_runtime.h>
+
+#include <stdio.h>
+#include <math.h>
+#include <float.h>
+
+#include <string>
+#include <vector>
+#include <sstream>
+#include <iostream>
+#include <limits>
+
+#include "mersenne.h"
+#include "half.h"
+
+#include "cub/util_debug.cuh"
+#include "cub/util_device.cuh"
+#include "cub/util_type.cuh"
+#include "cub/util_macro.cuh"
+#include "cub/iterator/discard_output_iterator.cuh"
+
+
+/******************************************************************************
+ * Assertion macros
+ ******************************************************************************/
+
+/**
+ * Assert equals
+ */
+#define AssertEquals(a, b) if ((a) != (b)) { std::cerr << "\n(" << __FILE__ << ": " << __LINE__ << ")\n"; exit(1);}
+
+
+/******************************************************************************
+ * Command-line parsing functionality
+ ******************************************************************************/
+
+/**
+ * Utility for parsing command line arguments
+ */
+struct CommandLineArgs
+{
+
+    std::vector<std::string>    keys;
+    std::vector<std::string>    values;
+    std::vector<std::string>    args;
+    cudaDeviceProp              deviceProp;
+    float                       device_giga_bandwidth;
+    size_t                      device_free_physmem;
+    size_t                      device_total_physmem;
+
+    /**
+     * Constructor
+     */
+    CommandLineArgs(int argc, char **argv) :
+        keys(10),
+        values(10)
+    {
+        using namespace std;
+
+        // Initialize mersenne generator
+        unsigned int mersenne_init[4]=  {0x123, 0x234, 0x345, 0x456};
+        mersenne::init_by_array(mersenne_init, 4);
+
+        for (int i = 1; i < argc; i++)
+        {
+            string arg = argv[i];
+
+            if ((arg[0] != '-') || (arg[1] != '-'))
+            {
+                args.push_back(arg);
+                continue;
+            }
+
+            string::size_type pos;
+            string key, val;
+            if ((pos = arg.find('=')) == string::npos) {
+                key = string(arg, 2, arg.length() - 2);
+                val = "";
+            } else {
+                key = string(arg, 2, pos - 2);
+                val = string(arg, pos + 1, arg.length() - 1);
+            }
+
+            keys.push_back(key);
+            values.push_back(val);
+        }
+    }
+
+
+    /**
+     * Checks whether a flag "--<flag>" is present in the commandline
+     */
+    bool CheckCmdLineFlag(const char* arg_name)
+    {
+        using namespace std;
+
+        for (int i = 0; i < int(keys.size()); ++i)
+        {
+            if (keys[i] == string(arg_name))
+                return true;
+        }
+        return false;
+    }
+
+
+    /**
+     * Returns number of naked (non-flag and non-key-value) commandline parameters
+     */
+    template <typename T>
+    int NumNakedArgs()
+    {
+        return args.size();
+    }
+
+
+    /**
+     * Returns the commandline parameter for a given index (not including flags)
+     */
+    template <typename T>
+    void GetCmdLineArgument(int index, T &val)
+    {
+        using namespace std;
+        if (index < args.size()) {
+            istringstream str_stream(args[index]);
+            str_stream >> val;
+        }
+    }
+
+    /**
+     * Returns the value specified for a given commandline parameter --<flag>=<value>
+     */
+    template <typename T>
+    void GetCmdLineArgument(const char *arg_name, T &val)
+    {
+        using namespace std;
+
+        for (int i = 0; i < int(keys.size()); ++i)
+        {
+            if (keys[i] == string(arg_name))
+            {
+                istringstream str_stream(values[i]);
+                str_stream >> val;
+            }
+        }
+    }
+
+
+    /**
+     * Returns the values specified for a given commandline parameter --<flag>=<value>,<value>*
+     */
+    template <typename T>
+    void GetCmdLineArguments(const char *arg_name, std::vector<T> &vals)
+    {
+        using namespace std;
+
+        if (CheckCmdLineFlag(arg_name))
+        {
+            // Clear any default values
+            vals.clear();
+
+            // Recover from multi-value string
+            for (int i = 0; i < keys.size(); ++i)
+            {
+                if (keys[i] == string(arg_name))
+                {
+                    string val_string(values[i]);
+                    istringstream str_stream(val_string);
+                    string::size_type old_pos = 0;
+                    string::size_type new_pos = 0;
+
+                    // Iterate comma-separated values
+                    T val;
+                    while ((new_pos = val_string.find(',', old_pos)) != string::npos)
+                    {
+                        if (new_pos != old_pos)
+                        {
+                            str_stream.width(new_pos - old_pos);
+                            str_stream >> val;
+                            vals.push_back(val);
+                        }
+
+                        // skip over comma
+                        str_stream.ignore(1);
+                        old_pos = new_pos + 1;
+                    }
+
+                    // Read last value
+                    str_stream >> val;
+                    vals.push_back(val);
+                }
+            }
+        }
+    }
+
+
+    /**
+     * The number of pairs parsed
+     */
+    int ParsedArgc()
+    {
+        return (int) keys.size();
+    }
+
+    /**
+     * Initialize device
+     */
+    cudaError_t DeviceInit(int dev = -1)
+    {
+        cudaError_t error = cudaSuccess;
+
+        do
+        {
+            int deviceCount;
+            error = CubDebug(cudaGetDeviceCount(&deviceCount));
+            if (error) break;
+
+            if (deviceCount == 0) {
+                fprintf(stderr, "No devices supporting CUDA.\n");
+                exit(1);
+            }
+            if (dev < 0)
+            {
+                GetCmdLineArgument("device", dev);
+            }
+            if ((dev > deviceCount - 1) || (dev < 0))
+            {
+                dev = 0;
+            }
+
+            error = CubDebug(cudaSetDevice(dev));
+            if (error) break;
+
+            CubDebugExit(cudaMemGetInfo(&device_free_physmem, &device_total_physmem));
+
+            int ptx_version;
+            error = CubDebug(cub::PtxVersion(ptx_version));
+            if (error) break;
+
+            error = CubDebug(cudaGetDeviceProperties(&deviceProp, dev));
+            if (error) break;
+
+            if (deviceProp.major < 1) {
+                fprintf(stderr, "Device does not support CUDA.\n");
+                exit(1);
+            }
+
+            device_giga_bandwidth = float(deviceProp.memoryBusWidth) * deviceProp.memoryClockRate * 2 / 8 / 1000 / 1000;
+
+            if (!CheckCmdLineFlag("quiet"))
+            {
+                printf(
+                        "Using device %d: %s (PTX version %d, SM%d, %d SMs, "
+                        "%lld free / %lld total MB physmem, "
+                        "%.3f GB/s @ %d kHz mem clock, ECC %s)\n",
+                    dev,
+                    deviceProp.name,
+                    ptx_version,
+                    deviceProp.major * 100 + deviceProp.minor * 10,
+                    deviceProp.multiProcessorCount,
+                    (unsigned long long) device_free_physmem / 1024 / 1024,
+                    (unsigned long long) device_total_physmem / 1024 / 1024,
+                    device_giga_bandwidth,
+                    deviceProp.memoryClockRate,
+                    (deviceProp.ECCEnabled) ? "on" : "off");
+                fflush(stdout);
+            }
+
+        } while (0);
+
+        return error;
+    }
+};
+
+/******************************************************************************
+ * Random bits generator
+ ******************************************************************************/
+
+int g_num_rand_samples = 0;
+
+
+template <typename T>
+bool IsNaN(T val) { return false; }
+
+template<>
+__noinline__ bool IsNaN<float>(float val)
+{
+    volatile unsigned int bits = reinterpret_cast<unsigned int &>(val);
+
+    return (((bits >= 0x7F800001) && (bits <= 0x7FFFFFFF)) || 
+        ((bits >= 0xFF800001) && (bits <= 0xFFFFFFFF)));
+}
+
+template<>
+__noinline__ bool IsNaN<float1>(float1 val)
+{
+    return (IsNaN(val.x));
+}
+
+template<>
+__noinline__ bool IsNaN<float2>(float2 val)
+{
+    return (IsNaN(val.y) || IsNaN(val.x));
+}
+
+template<>
+__noinline__ bool IsNaN<float3>(float3 val)
+{
+    return (IsNaN(val.z) || IsNaN(val.y) || IsNaN(val.x));
+}
+
+template<>
+__noinline__ bool IsNaN<float4>(float4 val)
+{
+    return (IsNaN(val.y) || IsNaN(val.x) || IsNaN(val.w) || IsNaN(val.z));
+}
+
+template<>
+__noinline__ bool IsNaN<double>(double val)
+{
+    volatile unsigned long long bits = *reinterpret_cast<unsigned long long *>(&val);
+
+    return (((bits >= 0x7FF0000000000001) && (bits <= 0x7FFFFFFFFFFFFFFF)) || 
+        ((bits >= 0xFFF0000000000001) && (bits <= 0xFFFFFFFFFFFFFFFF)));
+}
+
+template<>
+__noinline__ bool IsNaN<double1>(double1 val)
+{
+    return (IsNaN(val.x));
+}
+
+template<>
+__noinline__ bool IsNaN<double2>(double2 val)
+{
+    return (IsNaN(val.y) || IsNaN(val.x));
+}
+
+template<>
+__noinline__ bool IsNaN<double3>(double3 val)
+{
+    return (IsNaN(val.z) || IsNaN(val.y) || IsNaN(val.x));
+}
+
+template<>
+__noinline__ bool IsNaN<double4>(double4 val)
+{
+    return (IsNaN(val.y) || IsNaN(val.x) || IsNaN(val.w) || IsNaN(val.z));
+}
+
+
+template<>
+__noinline__ bool IsNaN<half_t>(half_t val)
+{
+    volatile unsigned short bits = reinterpret_cast<unsigned short &>(val);
+
+    return (((bits >= 0x7C01) && (bits <= 0x7FFF)) ||
+        ((bits >= 0xFC01) && (bits <= 0xFFFFFFFF)));
+}
+
+
+
+/**
+ * Generates random keys.
+ *
+ * We always take the second-order byte from rand() because the higher-order
+ * bits returned by rand() are commonly considered more uniformly distributed
+ * than the lower-order bits.
+ *
+ * We can decrease the entropy level of keys by adopting the technique
+ * of Thearling and Smith in which keys are computed from the bitwise AND of
+ * multiple random samples:
+ *
+ * entropy_reduction    | Effectively-unique bits per key
+ * -----------------------------------------------------
+ * -1                   | 0
+ * 0                    | 32
+ * 1                    | 25.95 (81%)
+ * 2                    | 17.41 (54%)
+ * 3                    | 10.78 (34%)
+ * 4                    | 6.42 (20%)
+ * ...                  | ...
+ *
+ */
+template <typename K>
+void RandomBits(
+    K &key,
+    int entropy_reduction = 0,
+    int begin_bit = 0,
+    int end_bit = sizeof(K) * 8)
+{
+    const int NUM_BYTES = sizeof(K);
+    const int WORD_BYTES = sizeof(unsigned int);
+    const int NUM_WORDS = (NUM_BYTES + WORD_BYTES - 1) / WORD_BYTES;
+
+    unsigned int word_buff[NUM_WORDS];
+
+    if (entropy_reduction == -1)
+    {
+        memset((void *) &key, 0, sizeof(key));
+        return;
+    }
+
+    if (end_bit < 0)
+        end_bit = sizeof(K) * 8;
+
+    while (true) 
+    {
+        // Generate random word_buff
+        for (int j = 0; j < NUM_WORDS; j++)
+        {
+            int current_bit = j * WORD_BYTES * 8;
+
+            unsigned int word = 0xffffffff;
+            word &= 0xffffffff << CUB_MAX(0, begin_bit - current_bit);
+            word &= 0xffffffff >> CUB_MAX(0, (current_bit + (WORD_BYTES * 8)) - end_bit);
+
+            for (int i = 0; i <= entropy_reduction; i++)
+            {
+                // Grab some of the higher bits from rand (better entropy, supposedly)
+                word &= mersenne::genrand_int32();
+                g_num_rand_samples++;                
+            }
+
+            word_buff[j] = word;
+        }
+
+        memcpy(&key, word_buff, sizeof(K));
+
+        K copy = key;
+        if (!IsNaN(copy))
+            break;          // avoids NaNs when generating random floating point numbers
+    }
+}
+
+/// Randomly select number between [0:max)
+template <typename T>
+T RandomValue(T max)
+{
+    unsigned int bits;
+    unsigned int max_int = (unsigned int) -1;
+    do {
+        RandomBits(bits);
+    } while (bits == max_int);
+
+    return (T) ((double(bits) / double(max_int)) * double(max));
+}
+
+
+/******************************************************************************
+ * Console printing utilities
+ ******************************************************************************/
+
+/**
+ * Helper for casting character types to integers for cout printing
+ */
+template <typename T>
+T CoutCast(T val) { return val; }
+
+int CoutCast(char val) { return val; }
+
+int CoutCast(unsigned char val) { return val; }
+
+int CoutCast(signed char val) { return val; }
+
+
+
+/******************************************************************************
+ * Test value initialization utilities
+ ******************************************************************************/
+
+/**
+ * Test problem generation options
+ */
+enum GenMode
+{
+    UNIFORM,            // Assign to '2', regardless of integer seed
+    INTEGER_SEED,       // Assign to integer seed
+    RANDOM,             // Assign to random, regardless of integer seed
+};
+
+/**
+ * Initialize value
+ */
+template <typename T>
+__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T &value, int index = 0)
+{
+    switch (gen_mode)
+    {
+#if (CUB_PTX_ARCH == 0)
+    case RANDOM:
+         RandomBits(value);
+         break;
+#endif
+     case UNIFORM:
+        value = 2;
+        break;
+    case INTEGER_SEED:
+    default:
+         value = (T) index;
+        break;
+    }
+}
+
+
+/**
+ * Initialize value (bool)
+ */
+__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, bool &value, int index = 0)
+{
+    switch (gen_mode)
+    {
+#if (CUB_PTX_ARCH == 0)
+    case RANDOM:
+        char c;
+        RandomBits(c, 0, 0, 1);
+        value = (c > 0);
+        break;
+#endif
+     case UNIFORM:
+        value = true;
+        break;
+    case INTEGER_SEED:
+    default:
+        value = (index > 0);
+        break;
+    }
+}
+
+
+/**
+ * cub::NullType test initialization
+ */
+__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, cub::NullType &value, int index = 0)
+{}
+
+
+/**
+ * cub::KeyValuePair<OffsetT, ValueT>test initialization
+ */
+template <typename KeyT, typename ValueT>
+__host__ __device__ __forceinline__ void InitValue(
+    GenMode                             gen_mode,
+    cub::KeyValuePair<KeyT, ValueT>&    value,
+    int                                 index = 0)
+{
+    InitValue(gen_mode, value.value, index);
+
+    // Assign corresponding flag with a likelihood of the last bit being set with entropy-reduction level 3
+    RandomBits(value.key, 3);
+    value.key = (value.key & 0x1);
+}
+
+
+
+/******************************************************************************
+ * Comparison and ostream operators
+ ******************************************************************************/
+
+/**
+ * KeyValuePair ostream operator
+ */
+template <typename Key, typename Value>
+std::ostream& operator<<(std::ostream& os, const cub::KeyValuePair<Key, Value> &val)
+{
+    os << '(' << CoutCast(val.key) << ',' << CoutCast(val.value) << ')';
+    return os;
+}
+
+
+/******************************************************************************
+ * Comparison and ostream operators for CUDA vector types
+ ******************************************************************************/
+
+/**
+ * Vector1 overloads
+ */
+#define CUB_VEC_OVERLOAD_1(T, BaseT)                        \
+    /* Ostream output */                                    \
+    std::ostream& operator<<(                               \
+        std::ostream& os,                                   \
+        const T& val)                                       \
+    {                                                       \
+        os << '(' << CoutCast(val.x) << ')';                \
+        return os;                                          \
+    }                                                       \
+    /* Inequality */                                        \
+    __host__ __device__ __forceinline__ bool operator!=(    \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x != b.x);                                \
+    }                                                       \
+    /* Equality */                                          \
+    __host__ __device__ __forceinline__ bool operator==(    \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x == b.x);                                \
+    }                                                       \
+    /* Test initialization */                               \
+    __host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T &value, int index = 0)   \
+    {                                                       \
+        InitValue(gen_mode, value.x, index);                \
+    }                                                       \
+    /* Max */                                               \
+    __host__ __device__ __forceinline__ bool operator>(     \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x > b.x);                                 \
+    }                                                       \
+    /* Min */                                               \
+    __host__ __device__ __forceinline__ bool operator<(     \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x < b.x);                                 \
+    }                                                       \
+    /* Summation (non-reference addends for VS2003 -O3 warpscan workaround */                       \
+    __host__ __device__ __forceinline__ T operator+(        \
+        T a,                                                \
+        T b)                                                \
+    {                                                       \
+        T retval = make_##T(a.x + b.x);                     \
+        return retval;                                      \
+    }                                                       \
+    namespace cub {                                         \
+    template<>                                              \
+    struct NumericTraits<T>                                 \
+    {                                                       \
+        static const Category CATEGORY = NOT_A_NUMBER;      \
+        enum {                                              \
+            PRIMITIVE       = false,                        \
+            NULL_TYPE       = false,                        \
+        };                                                  \
+        static T Max()                                      \
+        {                                                   \
+            T retval = {                                    \
+                NumericTraits<BaseT>::Max()};               \
+            return retval;                                  \
+        }                                                   \
+        static T Lowest()                                   \
+        {                                                   \
+            T retval = {                                    \
+                NumericTraits<BaseT>::Lowest()};            \
+            return retval;                                  \
+        }                                                   \
+    };                                                      \
+    } /* namespace std */
+
+
+
+/**
+ * Vector2 overloads
+ */
+#define CUB_VEC_OVERLOAD_2(T, BaseT)                        \
+    /* Ostream output */                                    \
+    std::ostream& operator<<(                               \
+        std::ostream& os,                                   \
+        const T& val)                                       \
+    {                                                       \
+        os << '('                                           \
+            << CoutCast(val.x) << ','                       \
+            << CoutCast(val.y) << ')';                      \
+        return os;                                          \
+    }                                                       \
+    /* Inequality */                                        \
+    __host__ __device__ __forceinline__ bool operator!=(    \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x != b.x) ||                              \
+            (a.y != b.y);                                   \
+    }                                                       \
+    /* Equality */                                          \
+    __host__ __device__ __forceinline__ bool operator==(    \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x == b.x) &&                              \
+            (a.y == b.y);                                   \
+    }                                                       \
+    /* Test initialization */                               \
+    __host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T &value, int index = 0)   \
+    {                                                       \
+        InitValue(gen_mode, value.x, index);                \
+        InitValue(gen_mode, value.y, index);                \
+    }                                                       \
+    /* Max */                                               \
+    __host__ __device__ __forceinline__ bool operator>(     \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        if (a.x > b.x) return true; else if (b.x > a.x) return false;   \
+        return a.y > b.y;                                               \
+    }                                                       \
+    /* Min */                                               \
+    __host__ __device__ __forceinline__ bool operator<(     \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        if (a.x < b.x) return true; else if (b.x < a.x) return false;   \
+        return a.y < b.y;                                               \
+    }                                                       \
+    /* Summation (non-reference addends for VS2003 -O3 warpscan workaround */                                         \
+    __host__ __device__ __forceinline__ T operator+(        \
+        T a,                                         \
+        T b)                                         \
+    {                                                       \
+        T retval = make_##T(                                        \
+            a.x + b.x,                                      \
+            a.y + b.y);                                     \
+        return retval;                                      \
+    }                                                       \
+    namespace cub {                                         \
+    template<>                                              \
+    struct NumericTraits<T>                                 \
+    {                                                       \
+        static const Category CATEGORY = NOT_A_NUMBER;      \
+        enum {                                              \
+            PRIMITIVE       = false,                        \
+            NULL_TYPE       = false,                        \
+        };                                                  \
+        static T Max()                                      \
+        {                                                   \
+            T retval = {                                    \
+                NumericTraits<BaseT>::Max(),                \
+                NumericTraits<BaseT>::Max()};               \
+            return retval;                                  \
+        }                                                   \
+        static T Lowest()                                   \
+        {                                                   \
+            T retval = {                                    \
+                NumericTraits<BaseT>::Lowest(),             \
+                NumericTraits<BaseT>::Lowest()};            \
+            return retval;                                  \
+        }                                                   \
+    };                                                      \
+    } /* namespace cub */
+
+
+
+/**
+ * Vector3 overloads
+ */
+#define CUB_VEC_OVERLOAD_3(T, BaseT)                        \
+    /* Ostream output */                                    \
+    std::ostream& operator<<(                               \
+        std::ostream& os,                                   \
+        const T& val)                                       \
+    {                                                       \
+        os << '('                                           \
+            << CoutCast(val.x) << ','                       \
+            << CoutCast(val.y) << ','                       \
+            << CoutCast(val.z) << ')';                      \
+        return os;                                          \
+    }                                                       \
+    /* Inequality */                                        \
+    __host__ __device__ __forceinline__ bool operator!=(    \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x != b.x) ||                              \
+            (a.y != b.y) ||                                 \
+            (a.z != b.z);                                   \
+    }                                                       \
+    /* Equality */                                          \
+    __host__ __device__ __forceinline__ bool operator==(    \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x == b.x) &&                              \
+            (a.y == b.y) &&                                 \
+            (a.z == b.z);                                   \
+    }                                                       \
+    /* Test initialization */                               \
+    __host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T &value, int index = 0)   \
+    {                                                       \
+        InitValue(gen_mode, value.x, index);                \
+        InitValue(gen_mode, value.y, index);                \
+        InitValue(gen_mode, value.z, index);                \
+    }                                                       \
+    /* Max */                                               \
+    __host__ __device__ __forceinline__ bool operator>(     \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        if (a.x > b.x) return true; else if (b.x > a.x) return false;   \
+        if (a.y > b.y) return true; else if (b.y > a.y) return false;   \
+        return a.z > b.z;                                               \
+    }                                                       \
+    /* Min */                                               \
+    __host__ __device__ __forceinline__ bool operator<(     \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        if (a.x < b.x) return true; else if (b.x < a.x) return false;   \
+        if (a.y < b.y) return true; else if (b.y < a.y) return false;   \
+        return a.z < b.z;                                               \
+    }                                                       \
+    /* Summation (non-reference addends for VS2003 -O3 warpscan workaround */                                         \
+    __host__ __device__ __forceinline__ T operator+(        \
+        T a,                                                \
+        T b)                                                \
+    {                                                       \
+        T retval = make_##T(                                        \
+            a.x + b.x,                                      \
+            a.y + b.y,                                      \
+            a.z + b.z);                                     \
+        return retval;                                      \
+    }                                                       \
+    namespace cub {                                         \
+    template<>                                              \
+    struct NumericTraits<T>                                 \
+    {                                                       \
+        static const Category CATEGORY = NOT_A_NUMBER;      \
+        enum {                                              \
+            PRIMITIVE       = false,                        \
+            NULL_TYPE       = false,                        \
+        };                                                  \
+        static T Max()                                      \
+        {                                                   \
+            T retval = {                                    \
+                NumericTraits<BaseT>::Max(),                \
+                NumericTraits<BaseT>::Max(),                \
+                NumericTraits<BaseT>::Max()};               \
+            return retval;                                  \
+        }                                                   \
+        static T Lowest()                                   \
+        {                                                   \
+            T retval = {                                    \
+                NumericTraits<BaseT>::Lowest(),             \
+                NumericTraits<BaseT>::Lowest(),             \
+                NumericTraits<BaseT>::Lowest()};            \
+            return retval;                                  \
+        }                                                   \
+    };                                                      \
+    } /* namespace cub */
+
+
+/**
+ * Vector4 overloads
+ */
+#define CUB_VEC_OVERLOAD_4(T, BaseT)                        \
+    /* Ostream output */                                    \
+    std::ostream& operator<<(                               \
+        std::ostream& os,                                   \
+        const T& val)                                       \
+    {                                                       \
+        os << '('                                           \
+            << CoutCast(val.x) << ','                       \
+            << CoutCast(val.y) << ','                       \
+            << CoutCast(val.z) << ','                       \
+            << CoutCast(val.w) << ')';                      \
+        return os;                                          \
+    }                                                       \
+    /* Inequality */                                        \
+    __host__ __device__ __forceinline__ bool operator!=(    \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x != b.x) ||                              \
+            (a.y != b.y) ||                                 \
+            (a.z != b.z) ||                                 \
+            (a.w != b.w);                                   \
+    }                                                       \
+    /* Equality */                                          \
+    __host__ __device__ __forceinline__ bool operator==(    \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        return (a.x == b.x) &&                              \
+            (a.y == b.y) &&                                 \
+            (a.z == b.z) &&                                 \
+            (a.w == b.w);                                   \
+    }                                                       \
+    /* Test initialization */                               \
+    __host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T &value, int index = 0)   \
+    {                                                       \
+        InitValue(gen_mode, value.x, index);                \
+        InitValue(gen_mode, value.y, index);                \
+        InitValue(gen_mode, value.z, index);                \
+        InitValue(gen_mode, value.w, index);                \
+    }                                                       \
+    /* Max */                                               \
+    __host__ __device__ __forceinline__ bool operator>(     \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        if (a.x > b.x) return true; else if (b.x > a.x) return false;   \
+        if (a.y > b.y) return true; else if (b.y > a.y) return false;   \
+        if (a.z > b.z) return true; else if (b.z > a.z) return false;   \
+        return a.w > b.w;                                               \
+    }                                                       \
+    /* Min */                                               \
+    __host__ __device__ __forceinline__ bool operator<(     \
+        const T &a,                                         \
+        const T &b)                                         \
+    {                                                       \
+        if (a.x < b.x) return true; else if (b.x < a.x) return false;   \
+        if (a.y < b.y) return true; else if (b.y < a.y) return false;   \
+        if (a.z < b.z) return true; else if (b.z < a.z) return false;   \
+        return a.w < b.w;                                               \
+    }                                                       \
+    /* Summation (non-reference addends for VS2003 -O3 warpscan workaround */                                         \
+    __host__ __device__ __forceinline__ T operator+(        \
+        T a,                                                \
+        T b)                                                \
+    {                                                       \
+        T retval = make_##T(                                        \
+            a.x + b.x,                                      \
+            a.y + b.y,                                      \
+            a.z + b.z,                                      \
+            a.w + b.w);                                     \
+        return retval;                                      \
+    }                                                       \
+    namespace cub {                                         \
+    template<>                                              \
+    struct NumericTraits<T>                                 \
+    {                                                       \
+        static const Category CATEGORY = NOT_A_NUMBER;      \
+        enum {                                              \
+            PRIMITIVE       = false,                        \
+            NULL_TYPE       = false,                        \
+        };                                                  \
+        static T Max()                                      \
+        {                                                   \
+            T retval = {                                    \
+                NumericTraits<BaseT>::Max(),                \
+                NumericTraits<BaseT>::Max(),                \
+                NumericTraits<BaseT>::Max(),                \
+                NumericTraits<BaseT>::Max()};               \
+            return retval;                                  \
+        }                                                   \
+        static T Lowest()                                   \
+        {                                                   \
+            T retval = {                                    \
+                NumericTraits<BaseT>::Lowest(),             \
+                NumericTraits<BaseT>::Lowest(),             \
+                NumericTraits<BaseT>::Lowest(),             \
+                NumericTraits<BaseT>::Lowest()};            \
+            return retval;                                  \
+        }                                                   \
+    };                                                      \
+    } /* namespace cub */
+
+/**
+ * All vector overloads
+ */
+#define CUB_VEC_OVERLOAD(COMPONENT_T, BaseT)                    \
+    CUB_VEC_OVERLOAD_1(COMPONENT_T##1, BaseT)                   \
+    CUB_VEC_OVERLOAD_2(COMPONENT_T##2, BaseT)                   \
+    CUB_VEC_OVERLOAD_3(COMPONENT_T##3, BaseT)                   \
+    CUB_VEC_OVERLOAD_4(COMPONENT_T##4, BaseT)
+
+/**
+ * Define for types
+ */
+CUB_VEC_OVERLOAD(char, char)
+CUB_VEC_OVERLOAD(short, short)
+CUB_VEC_OVERLOAD(int, int)
+CUB_VEC_OVERLOAD(long, long)
+CUB_VEC_OVERLOAD(longlong, long long)
+CUB_VEC_OVERLOAD(uchar, unsigned char)
+CUB_VEC_OVERLOAD(ushort, unsigned short)
+CUB_VEC_OVERLOAD(uint, unsigned int)
+CUB_VEC_OVERLOAD(ulong, unsigned long)
+CUB_VEC_OVERLOAD(ulonglong, unsigned long long)
+CUB_VEC_OVERLOAD(float, float)
+CUB_VEC_OVERLOAD(double, double)
+
+
+//---------------------------------------------------------------------
+// Complex data type TestFoo
+//---------------------------------------------------------------------
+
+/**
+ * TestFoo complex data type
+ */
+struct TestFoo
+{
+    long long   x;
+    int         y;
+    short       z;
+    char        w;
+
+    // Factory
+    static __host__ __device__ __forceinline__ TestFoo MakeTestFoo(long long x, int y, short z, char w)
+    {
+        TestFoo retval = {x, y, z, w};
+        return retval;
+    }
+
+    // Assignment from int operator
+    __host__ __device__ __forceinline__ TestFoo& operator =(int b)
+    {
+        x = b;
+        y = b;
+        z = b;
+        w = b;
+        return *this;
+    }
+
+    // Summation operator
+    __host__ __device__ __forceinline__ TestFoo operator+(const TestFoo &b) const
+    {
+        return MakeTestFoo(x + b.x, y + b.y, z + b.z, w + b.w);
+    }
+
+    // Inequality operator
+    __host__ __device__ __forceinline__ bool operator !=(const TestFoo &b) const
+    {
+        return (x != b.x) || (y != b.y) || (z != b.z) || (w != b.w);
+    }
+
+    // Equality operator
+    __host__ __device__ __forceinline__ bool operator ==(const TestFoo &b) const
+    {
+        return (x == b.x) && (y == b.y) && (z == b.z) && (w == b.w);
+    }
+
+    // Less than operator
+    __host__ __device__ __forceinline__ bool operator <(const TestFoo &b) const
+    {
+        if (x < b.x) return true; else if (b.x < x) return false;
+        if (y < b.y) return true; else if (b.y < y) return false;
+        if (z < b.z) return true; else if (b.z < z) return false;
+        return w < b.w;
+    }
+
+    // Greater than operator
+    __host__ __device__ __forceinline__ bool operator >(const TestFoo &b) const
+    {
+        if (x > b.x) return true; else if (b.x > x) return false;
+        if (y > b.y) return true; else if (b.y > y) return false;
+        if (z > b.z) return true; else if (b.z > z) return false;
+        return w > b.w;
+    }
+
+};
+
+/**
+ * TestFoo ostream operator
+ */
+std::ostream& operator<<(std::ostream& os, const TestFoo& val)
+{
+    os << '(' << val.x << ',' << val.y << ',' << val.z << ',' << CoutCast(val.w) << ')';
+    return os;
+}
+
+/**
+ * TestFoo test initialization
+ */
+__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, TestFoo &value, int index = 0)
+{
+    InitValue(gen_mode, value.x, index);
+    InitValue(gen_mode, value.y, index);
+    InitValue(gen_mode, value.z, index);
+    InitValue(gen_mode, value.w, index);
+}
+
+
+/// numeric_limits<TestFoo> specialization
+namespace cub {
+template<>
+struct NumericTraits<TestFoo>
+{
+    static const Category CATEGORY = NOT_A_NUMBER;
+    enum {
+        PRIMITIVE       = false,
+        NULL_TYPE       = false,
+    };
+    static TestFoo Max()
+    {
+        return TestFoo::MakeTestFoo(
+            NumericTraits<long long>::Max(),
+            NumericTraits<int>::Max(),
+            NumericTraits<short>::Max(),
+            NumericTraits<char>::Max());
+    }
+
+    static TestFoo Lowest()
+    {
+        return TestFoo::MakeTestFoo(
+            NumericTraits<long long>::Lowest(),
+            NumericTraits<int>::Lowest(),
+            NumericTraits<short>::Lowest(),
+            NumericTraits<char>::Lowest());
+    }
+};
+} // namespace cub
+
+
+//---------------------------------------------------------------------
+// Complex data type TestBar (with optimizations for fence-free warp-synchrony)
+//---------------------------------------------------------------------
+
+/**
+ * TestBar complex data type
+ */
+struct TestBar
+{
+    long long       x;
+    int             y;
+
+    // Constructor
+    __host__ __device__ __forceinline__ TestBar() : x(0), y(0)
+    {}
+
+    // Constructor
+    __host__ __device__ __forceinline__ TestBar(int b) : x(b), y(b)
+    {}
+
+    // Constructor
+    __host__ __device__ __forceinline__ TestBar(long long x, int y) : x(x), y(y)
+    {}
+
+    // Assignment from int operator
+    __host__ __device__ __forceinline__ TestBar& operator =(int b)
+    {
+        x = b;
+        y = b;
+        return *this;
+    }
+
+    // Summation operator
+    __host__ __device__ __forceinline__ TestBar operator+(const TestBar &b) const
+    {
+        return TestBar(x + b.x, y + b.y);
+    }
+
+    // Inequality operator
+    __host__ __device__ __forceinline__ bool operator !=(const TestBar &b) const
+    {
+        return (x != b.x) || (y != b.y);
+    }
+
+    // Equality operator
+    __host__ __device__ __forceinline__ bool operator ==(const TestBar &b) const
+    {
+        return (x == b.x) && (y == b.y);
+    }
+
+    // Less than operator
+    __host__ __device__ __forceinline__ bool operator <(const TestBar &b) const
+    {
+        if (x < b.x) return true; else if (b.x < x) return false;
+        return y < b.y;
+    }
+
+    // Greater than operator
+    __host__ __device__ __forceinline__ bool operator >(const TestBar &b) const
+    {
+        if (x > b.x) return true; else if (b.x > x) return false;
+        return y > b.y;
+    }
+
+};
+
+
+/**
+ * TestBar ostream operator
+ */
+std::ostream& operator<<(std::ostream& os, const TestBar& val)
+{
+    os << '(' << val.x << ',' << val.y << ')';
+    return os;
+}
+
+/**
+ * TestBar test initialization
+ */
+__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, TestBar &value, int index = 0)
+{
+    InitValue(gen_mode, value.x, index);
+    InitValue(gen_mode, value.y, index);
+}
+
+/// numeric_limits<TestBar> specialization
+namespace cub {
+template<>
+struct NumericTraits<TestBar>
+{
+    static const Category CATEGORY = NOT_A_NUMBER;
+    enum {
+        PRIMITIVE       = false,
+        NULL_TYPE       = false,
+    };
+    static TestBar Max()
+    {
+        return TestBar(
+            NumericTraits<long long>::Max(),
+            NumericTraits<int>::Max());
+    }
+
+    static TestBar Lowest()
+    {
+        return TestBar(
+            NumericTraits<long long>::Lowest(),
+            NumericTraits<int>::Lowest());
+    }
+};
+} // namespace cub
+
+
+/******************************************************************************
+ * Helper routines for list comparison and display
+ ******************************************************************************/
+
+
+/**
+ * Compares the equivalence of two arrays
+ */
+template <typename S, typename T, typename OffsetT>
+int CompareResults(T* computed, S* reference, OffsetT len, bool verbose = true)
+{
+    for (OffsetT i = 0; i < len; i++)
+    {
+        if (computed[i] != reference[i])
+        {
+            if (verbose) std::cout << "INCORRECT: [" << i << "]: "
+                << CoutCast(computed[i]) << " != "
+                << CoutCast(reference[i]);
+            return 1;
+        }
+    }
+    return 0;
+}
+
+
+/**
+ * Compares the equivalence of two arrays
+ */
+template <typename OffsetT>
+int CompareResults(float* computed, float* reference, OffsetT len, bool verbose = true)
+{
+    for (OffsetT i = 0; i < len; i++)
+    {
+        if (computed[i] != reference[i])
+        {
+            float difference = std::abs(computed[i]-reference[i]);
+            float fraction = difference / std::abs(reference[i]);
+
+            if (fraction > 0.0001)
+            {
+                if (verbose) std::cout << "INCORRECT: [" << i << "]: "
+                    << "(computed) " << CoutCast(computed[i]) << " != "
+                    << CoutCast(reference[i]) << " (difference:" << difference << ", fraction: " << fraction << ")";
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+
+/**
+ * Compares the equivalence of two arrays
+ */
+template <typename OffsetT>
+int CompareResults(cub::NullType* computed, cub::NullType* reference, OffsetT len, bool verbose = true)
+{
+    return 0;
+}
+
+/**
+ * Compares the equivalence of two arrays
+ */
+template <typename OffsetT>
+int CompareResults(double* computed, double* reference, OffsetT len, bool verbose = true)
+{
+    for (OffsetT i = 0; i < len; i++)
+    {
+        if (computed[i] != reference[i])
+        {
+            double difference = std::abs(computed[i]-reference[i]);
+            double fraction = difference / std::abs(reference[i]);
+
+            if (fraction > 0.0001)
+            {
+                if (verbose) std::cout << "INCORRECT: [" << i << "]: "
+                    << CoutCast(computed[i]) << " != "
+                    << CoutCast(reference[i]) << " (difference:" << difference << ", fraction: " << fraction << ")";
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+
+/**
+ * Verify the contents of a device array match those
+ * of a host array
+ */
+int CompareDeviceResults(
+    cub::NullType *h_reference,
+    cub::NullType *d_data,
+    size_t num_items,
+    bool verbose = true,
+    bool display_data = false)
+{
+    return 0;
+}
+
+/**
+ * Verify the contents of a device array match those
+ * of a host array
+ */
+template <typename S, typename OffsetT>
+int CompareDeviceResults(
+    S *h_reference,
+    cub::DiscardOutputIterator<OffsetT> d_data,
+    size_t num_items,
+    bool verbose = true,
+    bool display_data = false)
+{
+    return 0;
+}
+
+/**
+ * Verify the contents of a device array match those
+ * of a host array
+ */
+template <typename S, typename T>
+int CompareDeviceResults(
+    S *h_reference,
+    T *d_data,
+    size_t num_items,
+    bool verbose = true,
+    bool display_data = false)
+{
+    // Allocate array on host
+    T *h_data = (T*) malloc(num_items * sizeof(T));
+
+    // Copy data back
+    cudaMemcpy(h_data, d_data, sizeof(T) * num_items, cudaMemcpyDeviceToHost);
+
+    // Display data
+    if (display_data)
+    {
+        printf("Reference:\n");
+        for (int i = 0; i < int(num_items); i++)
+        {
+            std::cout << CoutCast(h_reference[i]) << ", ";
+        }
+        printf("\n\nComputed:\n");
+        for (int i = 0; i < int(num_items); i++)
+        {
+            std::cout << CoutCast(h_data[i]) << ", ";
+        }
+        printf("\n\n");
+    }
+
+    // Check
+    int retval = CompareResults(h_data, h_reference, num_items, verbose);
+
+    // Cleanup
+    if (h_data) free(h_data);
+
+    return retval;
+}
+
+
+/**
+ * Verify the contents of a device array match those
+ * of a device array
+ */
+template <typename T>
+int CompareDeviceDeviceResults(
+    T *d_reference,
+    T *d_data,
+    size_t num_items,
+    bool verbose = true,
+    bool display_data = false)
+{
+    // Allocate array on host
+    T *h_reference = (T*) malloc(num_items * sizeof(T));
+    T *h_data = (T*) malloc(num_items * sizeof(T));
+
+    // Copy data back
+    cudaMemcpy(h_reference, d_reference, sizeof(T) * num_items, cudaMemcpyDeviceToHost);
+    cudaMemcpy(h_data, d_data, sizeof(T) * num_items, cudaMemcpyDeviceToHost);
+
+    // Display data
+    if (display_data) {
+        printf("Reference:\n");
+        for (int i = 0; i < num_items; i++)
+        {
+            std::cout << CoutCast(h_reference[i]) << ", ";
+        }
+        printf("\n\nComputed:\n");
+        for (int i = 0; i < num_items; i++)
+        {
+            std::cout << CoutCast(h_data[i]) << ", ";
+        }
+        printf("\n\n");
+    }
+
+    // Check
+    int retval = CompareResults(h_data, h_reference, num_items, verbose);
+
+    // Cleanup
+    if (h_reference) free(h_reference);
+    if (h_data) free(h_data);
+
+    return retval;
+}
+
+
+/**
+ * Print the contents of a host array
+ */
+void DisplayResults(
+    cub::NullType   *h_data,
+    size_t          num_items)
+{}
+
+
+/**
+ * Print the contents of a host array
+ */
+template <typename InputIteratorT>
+void DisplayResults(
+    InputIteratorT h_data,
+    size_t num_items)
+{
+    // Display data
+    for (int i = 0; i < int(num_items); i++)
+    {
+        std::cout << CoutCast(h_data[i]) << ", ";
+    }
+    printf("\n");
+}
+
+
+/**
+ * Print the contents of a device array
+ */
+template <typename T>
+void DisplayDeviceResults(
+    T *d_data,
+    size_t num_items)
+{
+    // Allocate array on host
+    T *h_data = (T*) malloc(num_items * sizeof(T));
+
+    // Copy data back
+    cudaMemcpy(h_data, d_data, sizeof(T) * num_items, cudaMemcpyDeviceToHost);
+
+    DisplayResults(h_data, num_items);
+
+    // Cleanup
+    if (h_data) free(h_data);
+}
+
+
+/******************************************************************************
+ * Segment descriptor generation
+ ******************************************************************************/
+
+/**
+ * Initialize segments
+ */
+void InitializeSegments(
+    int     num_items,
+    int     num_segments,
+    int     *h_segment_offsets,
+    bool    verbose = false)
+{
+    if (num_segments <= 0)
+        return;
+
+    unsigned int expected_segment_length = (num_items + num_segments - 1) / num_segments;
+    int offset = 0;
+    for (int i = 0; i < num_segments; ++i)
+    {
+        h_segment_offsets[i] = offset;
+
+        unsigned int segment_length = RandomValue((expected_segment_length * 2) + 1);
+        offset += segment_length;
+        offset = CUB_MIN(offset, num_items);
+    }
+    h_segment_offsets[num_segments] = num_items;
+
+    if (verbose)
+    {
+        printf("Segment offsets: ");
+        DisplayResults(h_segment_offsets, num_segments + 1);
+    }
+}
+
+
+/******************************************************************************
+ * Timing
+ ******************************************************************************/
+
+
+struct CpuTimer
+{
+#if defined(_WIN32) || defined(_WIN64)
+
+    LARGE_INTEGER ll_freq;
+    LARGE_INTEGER ll_start;
+    LARGE_INTEGER ll_stop;
+
+    CpuTimer()
+    {
+        QueryPerformanceFrequency(&ll_freq);
+    }
+
+    void Start()
+    {
+        QueryPerformanceCounter(&ll_start);
+    }
+
+    void Stop()
+    {
+        QueryPerformanceCounter(&ll_stop);
+    }
+
+    float ElapsedMillis()
+    {
+        double start = double(ll_start.QuadPart) / double(ll_freq.QuadPart);
+        double stop  = double(ll_stop.QuadPart) / double(ll_freq.QuadPart);
+
+        return float((stop - start) * 1000);
+    }
+
+#else
+
+    rusage start;
+    rusage stop;
+
+    void Start()
+    {
+        getrusage(RUSAGE_SELF, &start);
+    }
+
+    void Stop()
+    {
+        getrusage(RUSAGE_SELF, &stop);
+    }
+
+    float ElapsedMillis()
+    {
+        float sec = stop.ru_utime.tv_sec - start.ru_utime.tv_sec;
+        float usec = stop.ru_utime.tv_usec - start.ru_utime.tv_usec;
+
+        return (sec * 1000) + (usec / 1000);
+    }
+
+#endif
+};
+
+struct GpuTimer
+{
+    cudaEvent_t start;
+    cudaEvent_t stop;
+
+    GpuTimer()
+    {
+        cudaEventCreate(&start);
+        cudaEventCreate(&stop);
+    }
+
+    ~GpuTimer()
+    {
+        cudaEventDestroy(start);
+        cudaEventDestroy(stop);
+    }
+
+    void Start()
+    {
+        cudaEventRecord(start, 0);
+    }
+
+    void Stop()
+    {
+        cudaEventRecord(stop, 0);
+    }
+
+    float ElapsedMillis()
+    {
+        float elapsed;
+        cudaEventSynchronize(stop);
+        cudaEventElapsedTime(&elapsed, start, stop);
+        return elapsed;
+    }
+};
diff --git a/third_party/cub/test/test_warp_reduce.cu b/third_party/cub/test/test_warp_reduce.cu
new file mode 100644
index 0000000..673219a
--- /dev/null
+++ b/third_party/cub/test/test_warp_reduce.cu
@@ -0,0 +1,840 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of WarpReduce utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <typeinfo>
+
+#include <cub/warp/warp_reduce.cuh>
+#include <cub/util_allocator.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+bool                    g_verbose       = false;
+int                     g_repeat        = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+
+/**
+ * \brief WrapperFunctor (for precluding test-specialized dispatch to *Sum variants)
+ */
+template<
+    typename    OpT,
+    int         LOGICAL_WARP_THREADS>
+struct WrapperFunctor
+{
+    OpT op;
+    int num_valid;
+
+    inline __host__ __device__ WrapperFunctor(OpT op, int num_valid) : op(op), num_valid(num_valid) {}
+
+    template <typename T>
+    inline __host__ __device__ T operator()(const T &a, const T &b) const
+    {
+#if CUB_PTX_ARCH != 0
+        if ((cub::LaneId() % LOGICAL_WARP_THREADS) >= num_valid)
+            cub::ThreadTrap();
+#endif
+
+        return op(a, b);
+    }
+
+};
+
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+/**
+ * Generic reduction
+ */
+template <
+    typename    T,
+    typename    ReductionOp,
+    typename    WarpReduce,
+    bool        PRIMITIVE = Traits<T>::PRIMITIVE>
+struct DeviceTest
+{
+    static __device__ __forceinline__ T Reduce(
+        typename WarpReduce::TempStorage    &temp_storage,
+        T                                   &data,
+        ReductionOp                         &reduction_op)
+    {
+        return WarpReduce(temp_storage).Reduce(data, reduction_op);
+    }
+
+    static __device__ __forceinline__ T Reduce(
+        typename WarpReduce::TempStorage    &temp_storage,
+        T                                   &data,
+        ReductionOp                         &reduction_op,
+        const int                           &valid_warp_threads)
+    {
+        return WarpReduce(temp_storage).Reduce(data, reduction_op, valid_warp_threads);
+    }
+
+    template <typename FlagT>
+    static __device__ __forceinline__ T HeadSegmentedReduce(
+        typename WarpReduce::TempStorage    &temp_storage,
+        T                                   &data,
+        FlagT                                &flag,
+        ReductionOp                         &reduction_op)
+    {
+        return WarpReduce(temp_storage).HeadSegmentedReduce(data, flag, reduction_op);
+    }
+
+    template <typename FlagT>
+    static __device__ __forceinline__ T TailSegmentedReduce(
+        typename WarpReduce::TempStorage    &temp_storage,
+        T                                   &data,
+        FlagT                                &flag,
+        ReductionOp                         &reduction_op)
+    {
+        return WarpReduce(temp_storage).TailSegmentedReduce(data, flag, reduction_op);
+    }
+
+};
+
+
+/**
+ * Summation
+ */
+template <
+    typename    T,
+    typename    WarpReduce>
+struct DeviceTest<T, Sum, WarpReduce, true>
+{
+    static __device__ __forceinline__ T Reduce(
+        typename WarpReduce::TempStorage    &temp_storage,
+        T                                   &data,
+        Sum                              &reduction_op)
+    {
+        return WarpReduce(temp_storage).Sum(data);
+    }
+
+    static __device__ __forceinline__ T Reduce(
+        typename WarpReduce::TempStorage    &temp_storage,
+        T                                   &data,
+        Sum                              &reduction_op,
+        const int                           &valid_warp_threads)
+    {
+        return WarpReduce(temp_storage).Sum(data, valid_warp_threads);
+    }
+
+    template <typename FlagT>
+    static __device__ __forceinline__ T HeadSegmentedReduce(
+        typename WarpReduce::TempStorage    &temp_storage,
+        T                                   &data,
+        FlagT                                &flag,
+        Sum                              &reduction_op)
+    {
+        return WarpReduce(temp_storage).HeadSegmentedSum(data, flag);
+    }
+
+    template <typename FlagT>
+    static __device__ __forceinline__ T TailSegmentedReduce(
+        typename WarpReduce::TempStorage    &temp_storage,
+        T                                   &data,
+        FlagT                                &flag,
+        Sum                              &reduction_op)
+    {
+        return WarpReduce(temp_storage).TailSegmentedSum(data, flag);
+    }
+
+};
+
+
+/**
+ * Full-tile warp reduction kernel
+ */
+template <
+    int         WARPS,
+    int         LOGICAL_WARP_THREADS,
+    typename    T,
+    typename    ReductionOp>
+__global__ void FullWarpReduceKernel(
+    T               *d_in,
+    T               *d_out,
+    ReductionOp     reduction_op,
+    clock_t         *d_elapsed)
+{
+    // Cooperative warp-reduce utility type (1 warp)
+    typedef WarpReduce<T, LOGICAL_WARP_THREADS> WarpReduce;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename WarpReduce::TempStorage temp_storage[WARPS];
+
+    // Per-thread tile data
+    T input = d_in[threadIdx.x];
+
+    // Record elapsed clocks
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t start = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    // Test warp reduce
+    int warp_id = threadIdx.x / LOGICAL_WARP_THREADS;
+
+    T output = DeviceTest<T, ReductionOp, WarpReduce>::Reduce(
+        temp_storage[warp_id], input, reduction_op);
+
+    // Record elapsed clocks
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t stop = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    *d_elapsed = stop - start;
+
+    // Store aggregate
+    d_out[threadIdx.x] = (threadIdx.x % LOGICAL_WARP_THREADS == 0) ?
+        output :
+        input;
+}
+
+/**
+ * Partially-full warp reduction kernel
+ */
+template <
+    int         WARPS,
+    int         LOGICAL_WARP_THREADS,
+    typename    T,
+    typename    ReductionOp>
+__global__ void PartialWarpReduceKernel(
+    T           *d_in,
+    T           *d_out,
+    ReductionOp reduction_op,
+    clock_t     *d_elapsed,
+    int         valid_warp_threads)
+{
+    // Cooperative warp-reduce utility type
+    typedef WarpReduce<T, LOGICAL_WARP_THREADS> WarpReduce;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename WarpReduce::TempStorage temp_storage[WARPS];
+
+    // Per-thread tile data
+    T input = d_in[threadIdx.x];
+
+    // Record elapsed clocks
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t start = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    // Test partial-warp reduce
+    int warp_id = threadIdx.x / LOGICAL_WARP_THREADS;
+    T output = DeviceTest<T, ReductionOp, WarpReduce>::Reduce(
+        temp_storage[warp_id], input, reduction_op, valid_warp_threads);
+
+    // Record elapsed clocks
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t stop = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    *d_elapsed = stop - start;
+
+    // Store aggregate
+    d_out[threadIdx.x] = (threadIdx.x % LOGICAL_WARP_THREADS == 0) ?
+        output :
+        input;
+}
+
+
+/**
+ * Head-based segmented warp reduction test kernel
+ */
+template <
+    int         WARPS,
+    int         LOGICAL_WARP_THREADS,
+    typename    T,
+    typename    FlagT,
+    typename    ReductionOp>
+__global__ void WarpHeadSegmentedReduceKernel(
+    T           *d_in,
+    FlagT        *d_head_flags,
+    T           *d_out,
+    ReductionOp reduction_op,
+    clock_t     *d_elapsed)
+{
+    // Cooperative warp-reduce utility type
+    typedef WarpReduce<T, LOGICAL_WARP_THREADS> WarpReduce;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename WarpReduce::TempStorage temp_storage[WARPS];
+
+    // Per-thread tile data
+    T       input       = d_in[threadIdx.x];
+    FlagT   head_flag   = d_head_flags[threadIdx.x];
+
+    // Record elapsed clocks
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t start = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    // Test segmented warp reduce
+    int warp_id = threadIdx.x / LOGICAL_WARP_THREADS;
+    T output = DeviceTest<T, ReductionOp, WarpReduce>::HeadSegmentedReduce(
+        temp_storage[warp_id], input, head_flag, reduction_op);
+
+    // Record elapsed clocks
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t stop = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    *d_elapsed = stop - start;
+
+    // Store aggregate
+    d_out[threadIdx.x] = ((threadIdx.x % LOGICAL_WARP_THREADS == 0) || head_flag) ?
+        output :
+        input;
+}
+
+
+/**
+ * Tail-based segmented warp reduction test kernel
+ */
+template <
+    int         WARPS,
+    int         LOGICAL_WARP_THREADS,
+    typename    T,
+    typename    FlagT,
+    typename    ReductionOp>
+__global__ void WarpTailSegmentedReduceKernel(
+    T           *d_in,
+    FlagT       *d_tail_flags,
+    T           *d_out,
+    ReductionOp reduction_op,
+    clock_t     *d_elapsed)
+{
+    // Cooperative warp-reduce utility type
+    typedef WarpReduce<T, LOGICAL_WARP_THREADS> WarpReduce;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename WarpReduce::TempStorage temp_storage[WARPS];
+
+    // Per-thread tile data
+    T       input       = d_in[threadIdx.x];
+    FlagT    tail_flag   = d_tail_flags[threadIdx.x];
+    FlagT    head_flag   = (threadIdx.x == 0) ?
+                            0 :
+                            d_tail_flags[threadIdx.x - 1];
+
+    // Record elapsed clocks
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t start = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    // Test segmented warp reduce
+    int warp_id = threadIdx.x / LOGICAL_WARP_THREADS;
+    T output = DeviceTest<T, ReductionOp, WarpReduce>::TailSegmentedReduce(
+        temp_storage[warp_id], input, tail_flag, reduction_op);
+
+    // Record elapsed clocks
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t stop = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    *d_elapsed = stop - start;
+
+    // Store aggregate
+    d_out[threadIdx.x] = ((threadIdx.x % LOGICAL_WARP_THREADS == 0) || head_flag) ?
+        output :
+        input;
+}
+
+
+//---------------------------------------------------------------------
+// Host utility subroutines
+//---------------------------------------------------------------------
+
+/**
+ * Initialize reduction problem (and solution)
+ */
+template <
+    typename    T,
+    typename    ReductionOp>
+void Initialize(
+    GenMode     gen_mode,
+    int         flag_entropy,
+    T           *h_in,
+    int         *h_flags,
+    int         warps,
+    int         warp_threads,
+    int         valid_warp_threads,
+    ReductionOp reduction_op,
+    T           *h_head_out,
+    T           *h_tail_out)
+{
+    for (int i = 0; i < warps * warp_threads; ++i)
+    {
+        // Sample a value for this item
+        InitValue(gen_mode, h_in[i], i);
+        h_head_out[i] = h_in[i];
+        h_tail_out[i] = h_in[i];
+
+        // Sample whether or not this item will be a segment head
+        char bits;
+        RandomBits(bits, flag_entropy);
+        h_flags[i] = bits & 0x1;
+    }
+
+    // Accumulate segments (lane 0 of each warp is implicitly a segment head)
+    for (int warp = 0; warp < warps; ++warp)
+    {
+        int warp_offset  = warp * warp_threads;
+        int item_offset = warp_offset + valid_warp_threads - 1;
+
+        // Last item in warp
+        T head_aggregate = h_in[item_offset];
+        T tail_aggregate = h_in[item_offset];
+
+        if (h_flags[item_offset])
+            h_head_out[item_offset] = head_aggregate;
+        item_offset--;
+
+        // Work backwards
+        while (item_offset >= warp_offset)
+        {
+            if (h_flags[item_offset + 1])
+            {
+                head_aggregate = h_in[item_offset];
+            }
+            else
+            {
+                head_aggregate = reduction_op(head_aggregate, h_in[item_offset]);
+            }
+
+            if (h_flags[item_offset])
+            {
+                h_head_out[item_offset] = head_aggregate;
+                h_tail_out[item_offset + 1] = tail_aggregate;
+                tail_aggregate = h_in[item_offset];
+            }
+            else
+            {
+                tail_aggregate = reduction_op(tail_aggregate, h_in[item_offset]);
+            }
+
+            item_offset--;
+        }
+
+        // Record last segment head_aggregate to head offset
+        h_head_out[warp_offset] = head_aggregate;
+        h_tail_out[warp_offset] = tail_aggregate;
+    }
+}
+
+
+/**
+ * Test warp reduction
+ */
+template <
+    int         WARPS,
+    int         LOGICAL_WARP_THREADS,
+    typename    T,
+    typename    ReductionOp>
+void TestReduce(
+    GenMode     gen_mode,
+    ReductionOp reduction_op,
+    int         valid_warp_threads = LOGICAL_WARP_THREADS)
+{
+    const int BLOCK_THREADS = LOGICAL_WARP_THREADS * WARPS;
+
+    // Allocate host arrays
+    T   *h_in           = new T[BLOCK_THREADS];
+    int *h_flags        = new int[BLOCK_THREADS];
+    T   *h_out          = new T[BLOCK_THREADS];
+    T   *h_tail_out     = new T[BLOCK_THREADS];
+
+    // Initialize problem
+    Initialize(gen_mode, -1, h_in, h_flags, WARPS, LOGICAL_WARP_THREADS, valid_warp_threads, reduction_op, h_out, h_tail_out);
+
+    // Initialize/clear device arrays
+    T *d_in = NULL;
+    T *d_out = NULL;
+    clock_t *d_elapsed = NULL;
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * BLOCK_THREADS));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(T) * BLOCK_THREADS));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_elapsed, sizeof(clock_t)));
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * BLOCK_THREADS, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(T) * BLOCK_THREADS));
+
+    if (g_verbose)
+    {
+        printf("Data:\n");
+        for (int i = 0; i < WARPS; ++i)
+            DisplayResults(h_in + (i * LOGICAL_WARP_THREADS), valid_warp_threads);
+    }
+
+    // Run kernel
+    printf("\nGen-mode %d, %d warps, %d warp threads, %d valid lanes, %s (%d bytes) elements:\n",
+        gen_mode,
+        WARPS,
+        LOGICAL_WARP_THREADS,
+        valid_warp_threads,
+        typeid(T).name(),
+        (int) sizeof(T));
+    fflush(stdout);
+
+    if (valid_warp_threads == LOGICAL_WARP_THREADS)
+    {
+        // Run full-warp kernel
+        FullWarpReduceKernel<WARPS, LOGICAL_WARP_THREADS><<<1, BLOCK_THREADS>>>(
+            d_in,
+            d_out,
+            reduction_op,
+            d_elapsed);
+    }
+    else
+    {
+        // Run partial-warp kernel
+        PartialWarpReduceKernel<WARPS, LOGICAL_WARP_THREADS><<<1, BLOCK_THREADS>>>(
+            d_in,
+            d_out,
+            reduction_op,
+            d_elapsed,
+            valid_warp_threads);
+    }
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Copy out and display results
+    printf("\tReduction results: ");
+    int compare = CompareDeviceResults(h_out, d_out, BLOCK_THREADS, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+    printf("\tElapsed clocks: ");
+    DisplayDeviceResults(d_elapsed, 1);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_flags) delete[] h_flags;
+    if (h_out) delete[] h_out;
+    if (h_tail_out) delete[] h_tail_out;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_elapsed) CubDebugExit(g_allocator.DeviceFree(d_elapsed));
+}
+
+
+/**
+ * Test warp segmented reduction
+ */
+template <
+    int         WARPS,
+    int         LOGICAL_WARP_THREADS,
+    typename    T,
+    typename    ReductionOp>
+void TestSegmentedReduce(
+    GenMode     gen_mode,
+    int         flag_entropy,
+    ReductionOp reduction_op)
+{
+    const int BLOCK_THREADS = LOGICAL_WARP_THREADS * WARPS;
+
+    // Allocate host arrays
+    int compare;
+    T   *h_in           = new T[BLOCK_THREADS];
+    int *h_flags        = new int[BLOCK_THREADS];
+    T   *h_head_out     = new T[BLOCK_THREADS];
+    T   *h_tail_out     = new T[BLOCK_THREADS];
+
+    // Initialize problem
+    Initialize(gen_mode, flag_entropy, h_in, h_flags, WARPS, LOGICAL_WARP_THREADS, LOGICAL_WARP_THREADS, reduction_op, h_head_out, h_tail_out);
+
+    // Initialize/clear device arrays
+    T           *d_in = NULL;
+    int         *d_flags = NULL;
+    T           *d_head_out = NULL;
+    T           *d_tail_out = NULL;
+    clock_t     *d_elapsed = NULL;
+
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * BLOCK_THREADS));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_flags, sizeof(int) * BLOCK_THREADS));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_head_out, sizeof(T) * BLOCK_THREADS));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_tail_out, sizeof(T) * BLOCK_THREADS));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_elapsed, sizeof(clock_t)));
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * BLOCK_THREADS, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemcpy(d_flags, h_flags, sizeof(int) * BLOCK_THREADS, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_head_out, 0, sizeof(T) * BLOCK_THREADS));
+    CubDebugExit(cudaMemset(d_tail_out, 0, sizeof(T) * BLOCK_THREADS));
+
+    if (g_verbose)
+    {
+        printf("Data:\n");
+        for (int i = 0; i < WARPS; ++i)
+            DisplayResults(h_in + (i * LOGICAL_WARP_THREADS), LOGICAL_WARP_THREADS);
+
+        printf("\nFlags:\n");
+        for (int i = 0; i < WARPS; ++i)
+            DisplayResults(h_flags + (i * LOGICAL_WARP_THREADS), LOGICAL_WARP_THREADS);
+    }
+
+    printf("\nGen-mode %d, head flag entropy reduction %d, %d warps, %d warp threads, %s (%d bytes) elements:\n",
+        gen_mode,
+        flag_entropy,
+        WARPS,
+        LOGICAL_WARP_THREADS,
+        typeid(T).name(),
+        (int) sizeof(T));
+    fflush(stdout);
+
+    // Run head-based kernel
+    WarpHeadSegmentedReduceKernel<WARPS, LOGICAL_WARP_THREADS><<<1, BLOCK_THREADS>>>(
+        d_in,
+        d_flags,
+        d_head_out,
+        reduction_op,
+        d_elapsed);
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Copy out and display results
+    printf("\tHead-based segmented reduction results: ");
+    compare = CompareDeviceResults(h_head_out, d_head_out, BLOCK_THREADS, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+    printf("\tElapsed clocks: ");
+    DisplayDeviceResults(d_elapsed, 1);
+
+    // Run tail-based kernel
+    WarpTailSegmentedReduceKernel<WARPS, LOGICAL_WARP_THREADS><<<1, BLOCK_THREADS>>>(
+        d_in,
+        d_flags,
+        d_tail_out,
+        reduction_op,
+        d_elapsed);
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Copy out and display results
+    printf("\tTail-based segmented reduction results: ");
+    compare = CompareDeviceResults(h_tail_out, d_tail_out, BLOCK_THREADS, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+    printf("\tElapsed clocks: ");
+    DisplayDeviceResults(d_elapsed, 1);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_flags) delete[] h_flags;
+    if (h_head_out) delete[] h_head_out;
+    if (h_tail_out) delete[] h_tail_out;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_flags) CubDebugExit(g_allocator.DeviceFree(d_flags));
+    if (d_head_out) CubDebugExit(g_allocator.DeviceFree(d_head_out));
+    if (d_tail_out) CubDebugExit(g_allocator.DeviceFree(d_tail_out));
+    if (d_elapsed) CubDebugExit(g_allocator.DeviceFree(d_elapsed));
+}
+
+
+/**
+ * Run battery of tests for different full and partial tile sizes
+ */
+template <
+    int         WARPS,
+    int         LOGICAL_WARP_THREADS,
+    typename    T,
+    typename    ReductionOp>
+void Test(
+    GenMode     gen_mode,
+    ReductionOp reduction_op)
+{
+    // Partial tiles
+    for (
+        int valid_warp_threads = 1;
+        valid_warp_threads < LOGICAL_WARP_THREADS;
+        valid_warp_threads += CUB_MAX(1, LOGICAL_WARP_THREADS / 5))
+    {
+        // Without wrapper (to test non-excepting PTX POD-op specializations)
+        TestReduce<WARPS, LOGICAL_WARP_THREADS, T>(gen_mode, reduction_op, valid_warp_threads);
+
+        // With wrapper to ensure no ops called on OOB lanes
+        WrapperFunctor<ReductionOp, LOGICAL_WARP_THREADS> wrapped_op(reduction_op, valid_warp_threads);
+        TestReduce<WARPS, LOGICAL_WARP_THREADS, T>(gen_mode, wrapped_op, valid_warp_threads);
+    }
+
+    // Full tile
+    TestReduce<WARPS, LOGICAL_WARP_THREADS, T>(gen_mode, reduction_op, LOGICAL_WARP_THREADS);
+
+    // Segmented reduction with different head flags
+    for (int flag_entropy = 0; flag_entropy < 10; ++flag_entropy)
+    {
+        TestSegmentedReduce<WARPS, LOGICAL_WARP_THREADS, T>(gen_mode, flag_entropy, reduction_op);
+    }
+}
+
+
+/**
+ * Run battery of tests for different data types and reduce ops
+ */
+template <
+    int WARPS,
+    int LOGICAL_WARP_THREADS>
+void Test(GenMode gen_mode)
+{
+    // primitive
+    Test<WARPS, LOGICAL_WARP_THREADS, char>(                gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, short>(               gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, int>(                 gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, long long>(           gen_mode, Sum());
+
+    Test<WARPS, LOGICAL_WARP_THREADS, unsigned char>(       gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, unsigned short>(      gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, unsigned int>(        gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, unsigned long long>(  gen_mode, Sum());
+
+    if (gen_mode != RANDOM)
+    {
+        Test<WARPS, LOGICAL_WARP_THREADS, float>(           gen_mode, Sum());
+        Test<WARPS, LOGICAL_WARP_THREADS, double>(          gen_mode, Sum());
+    }
+
+    // primitive (alternative reduce op)
+    Test<WARPS, LOGICAL_WARP_THREADS, unsigned char>(       gen_mode, Max());
+    Test<WARPS, LOGICAL_WARP_THREADS, unsigned short>(      gen_mode, Max());
+    Test<WARPS, LOGICAL_WARP_THREADS, unsigned int>(        gen_mode, Max());
+    Test<WARPS, LOGICAL_WARP_THREADS, unsigned long long>(  gen_mode, Max());
+
+    // vec-1
+    Test<WARPS, LOGICAL_WARP_THREADS, uchar1>(              gen_mode, Sum());
+
+    // vec-2
+    Test<WARPS, LOGICAL_WARP_THREADS, uchar2>(              gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, ushort2>(             gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, uint2>(               gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, ulonglong2>(          gen_mode, Sum());
+
+    // vec-4
+    Test<WARPS, LOGICAL_WARP_THREADS, uchar4>(              gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, ushort4>(             gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, uint4>(               gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, ulonglong4>(          gen_mode, Sum());
+
+    // complex
+    Test<WARPS, LOGICAL_WARP_THREADS, TestFoo>(             gen_mode, Sum());
+    Test<WARPS, LOGICAL_WARP_THREADS, TestBar>(             gen_mode, Sum());
+}
+
+
+/**
+ * Run battery of tests for different problem generation options
+ */
+template <
+    int WARPS,
+    int LOGICAL_WARP_THREADS>
+void Test()
+{
+    Test<WARPS, LOGICAL_WARP_THREADS>(UNIFORM);
+    Test<WARPS, LOGICAL_WARP_THREADS>(INTEGER_SEED);
+    Test<WARPS, LOGICAL_WARP_THREADS>(RANDOM);
+}
+
+
+/**
+ * Run battery of tests for different number of active warps
+ */
+template <int LOGICAL_WARP_THREADS>
+void Test()
+{
+    Test<1, LOGICAL_WARP_THREADS>();
+
+    // Only power-of-two subwarps can be tiled
+    if ((LOGICAL_WARP_THREADS == 32) || PowerOfTwo<LOGICAL_WARP_THREADS>::VALUE)
+        Test<2, LOGICAL_WARP_THREADS>();
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("repeat", g_repeat);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+#ifdef QUICK_TEST
+
+    // Compile/run quick tests
+    TestReduce<1, 32, int>(UNIFORM, Sum());
+
+    TestReduce<1, 32, double>(UNIFORM, Sum());
+    TestReduce<2, 16, TestBar>(UNIFORM, Sum());
+    TestSegmentedReduce<1, 32, int>(UNIFORM, 1, Sum());
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // Test logical warp sizes
+        Test<32>();
+        Test<16>();
+        Test<9>();
+        Test<7>();
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
+
diff --git a/third_party/cub/test/test_warp_scan.cu b/third_party/cub/test/test_warp_scan.cu
new file mode 100644
index 0000000..ba8e5cf
--- /dev/null
+++ b/third_party/cub/test/test_warp_scan.cu
@@ -0,0 +1,661 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Test of WarpScan utilities
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <stdio.h>
+#include <typeinfo>
+
+#include <cub/warp/warp_scan.cuh>
+#include <cub/util_allocator.cuh>
+
+#include "test_util.h"
+
+using namespace cub;
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+static const int        NUM_WARPS       = 2;
+
+
+bool                    g_verbose       = false;
+int                     g_repeat        = 0;
+CachingDeviceAllocator  g_allocator(true);
+
+
+/**
+ * Primitive variant to test
+ */
+enum TestMode
+{
+    BASIC,
+    AGGREGATE,
+};
+
+
+
+/**
+ * \brief WrapperFunctor (for precluding test-specialized dispatch to *Sum variants)
+ */
+template<typename OpT>
+struct WrapperFunctor
+{
+    OpT op;
+
+    WrapperFunctor(OpT op) : op(op) {}
+
+    template <typename T>
+    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
+    {
+        return op(a, b);
+    }
+};
+
+//---------------------------------------------------------------------
+// Test kernels
+//---------------------------------------------------------------------
+
+/// Exclusive scan basic
+template <typename WarpScanT, typename T, typename ScanOpT, typename IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    WarpScanT                       &warp_scan,
+    T                               &data,
+    T                               &initial_value,
+    ScanOpT                         &scan_op,
+    T                               &aggregate,
+    Int2Type<BASIC>                 test_mode,
+    IsPrimitiveT                    is_primitive)
+{
+    // Test basic warp scan
+    warp_scan.ExclusiveScan(data, data, initial_value, scan_op);
+}
+
+/// Exclusive scan aggregate
+template <
+    typename    WarpScanT,
+    typename    T,
+    typename    ScanOpT,
+    typename    IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    WarpScanT                       &warp_scan,
+    T                               &data,
+    T                               &initial_value,
+    ScanOpT                         &scan_op,
+    T                               &aggregate,
+    Int2Type<AGGREGATE>             test_mode,
+    IsPrimitiveT                    is_primitive)
+{
+    // Test with cumulative aggregate
+    warp_scan.ExclusiveScan(data, data, initial_value, scan_op, aggregate);
+}
+
+
+/// Exclusive sum basic
+template <
+    typename    WarpScanT,
+    typename    T>
+__device__ __forceinline__ void DeviceTest(
+    WarpScanT                       &warp_scan,
+    T                               &data,
+    T                               &initial_value,
+    Sum                             &scan_op,
+    T                               &aggregate,
+    Int2Type<BASIC>                 test_mode,
+    Int2Type<true>                  is_primitive)
+{
+    // Test basic warp scan
+    warp_scan.ExclusiveSum(data, data);
+}
+
+
+/// Exclusive sum aggregate
+template <
+    typename    WarpScanT,
+    typename    T>
+__device__ __forceinline__ void DeviceTest(
+    WarpScanT                       &warp_scan,
+    T                               &data,
+    T                               &initial_value,
+    Sum                             &scan_op,
+    T                               &aggregate,
+    Int2Type<AGGREGATE>             test_mode,
+    Int2Type<true>                  is_primitive)
+{
+    // Test with cumulative aggregate
+    warp_scan.ExclusiveSum(data, data, aggregate);
+}
+
+
+/// Inclusive scan basic
+template <
+    typename    WarpScanT,
+    typename    T,
+    typename    ScanOpT,
+    typename    IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    WarpScanT                       &warp_scan,
+    T                               &data,
+    NullType                        &initial_value,
+    ScanOpT                         &scan_op,
+    T                               &aggregate,
+    Int2Type<BASIC>                 test_mode,
+    IsPrimitiveT                    is_primitive)
+{
+    // Test basic warp scan
+    warp_scan.InclusiveScan(data, data, scan_op);
+}
+
+/// Inclusive scan aggregate
+template <
+    typename    WarpScanT,
+    typename    T,
+    typename    ScanOpT,
+    typename    IsPrimitiveT>
+__device__ __forceinline__ void DeviceTest(
+    WarpScanT                       &warp_scan,
+    T                               &data,
+    NullType                        &initial_value,
+    ScanOpT                         &scan_op,
+    T                               &aggregate,
+    Int2Type<AGGREGATE>             test_mode,
+    IsPrimitiveT                    is_primitive)
+{
+    // Test with cumulative aggregate
+    warp_scan.InclusiveScan(data, data, scan_op, aggregate);
+}
+
+/// Inclusive sum basic
+template <
+    typename    WarpScanT,
+    typename    T,
+    typename    InitialValueT>
+__device__ __forceinline__ void DeviceTest(
+    WarpScanT                       &warp_scan,
+    T                               &data,
+    NullType                        &initial_value,
+    Sum                             &scan_op,
+    T                               &aggregate,
+    Int2Type<BASIC>                 test_mode,
+    Int2Type<true>                  is_primitive)
+{
+    // Test basic warp scan
+    warp_scan.InclusiveSum(data, data);
+}
+
+/// Inclusive sum aggregate
+template <
+    typename    WarpScanT,
+    typename    T,
+    typename    InitialValueT>
+__device__ __forceinline__ void DeviceTest(
+    WarpScanT                       &warp_scan,
+    T                               &data,
+    NullType                        &initial_value,
+    Sum                             &scan_op,
+    T                               &aggregate,
+    Int2Type<AGGREGATE>             test_mode,
+    Int2Type<true>                  is_primitive)
+{
+    // Test with cumulative aggregate
+    warp_scan.InclusiveSum(data, data, aggregate);
+}
+
+
+/**
+ * WarpScan test kernel
+ */
+template <
+    int         LOGICAL_WARP_THREADS,
+    TestMode    TEST_MODE,
+    typename    T,
+    typename    ScanOpT,
+    typename    InitialValueT>
+__global__ void WarpScanKernel(
+    T               *d_in,
+    T               *d_out,
+    T               *d_aggregate,
+    ScanOpT         scan_op,
+    InitialValueT   initial_value,
+    clock_t         *d_elapsed)
+{
+    // Cooperative warp-scan utility type (1 warp)
+    typedef WarpScan<T, LOGICAL_WARP_THREADS> WarpScanT;
+
+    // Allocate temp storage in shared memory
+    __shared__ typename WarpScanT::TempStorage temp_storage[NUM_WARPS];
+
+    // Get warp index
+    int warp_id = threadIdx.x / LOGICAL_WARP_THREADS;
+
+    // Per-thread tile data
+    T data = d_in[threadIdx.x];
+
+    // Start cycle timer
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t start = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    T aggregate;
+
+    // Test scan
+    WarpScanT warp_scan(temp_storage[warp_id]);
+    DeviceTest(
+        warp_scan,
+        data,
+        initial_value,
+        scan_op,
+        aggregate,
+        Int2Type<TEST_MODE>(),
+        Int2Type<Traits<T>::PRIMITIVE>());
+
+    // Stop cycle timer
+    __threadfence_block();      // workaround to prevent clock hoisting
+    clock_t stop = clock();
+    __threadfence_block();      // workaround to prevent clock hoisting
+
+    // Store data
+    d_out[threadIdx.x] = data;
+
+    if (TEST_MODE != BASIC)
+    {
+        // Store aggregate
+        d_aggregate[threadIdx.x] = aggregate;
+    }
+
+    // Store time
+    if (threadIdx.x == 0)
+    {
+        *d_elapsed = (start > stop) ? start - stop : stop - start;
+    }
+}
+
+
+//---------------------------------------------------------------------
+// Host utility subroutines
+//---------------------------------------------------------------------
+
+/**
+ * Initialize exclusive-scan problem (and solution)
+ */
+template <
+    typename        T,
+    typename        ScanOpT>
+void Initialize(
+    GenMode         gen_mode,
+    T               *h_in,
+    T               *h_reference,
+    int             logical_warp_items,
+    ScanOpT         scan_op,
+    T               initial_value,
+    T               warp_aggregates[NUM_WARPS])
+{
+    for (int w = 0; w < NUM_WARPS; ++w)
+    {
+        int base_idx = (w * logical_warp_items);
+        int i = base_idx;
+
+        InitValue(gen_mode, h_in[i], i);
+
+        T warp_aggregate   = h_in[i];
+        h_reference[i]      = initial_value;
+        T inclusive         = scan_op(initial_value, h_in[i]);
+
+        for (i = i + 1; i < base_idx + logical_warp_items; ++i)
+        {
+            InitValue(gen_mode, h_in[i], i);
+            h_reference[i] = inclusive;
+            inclusive = scan_op(inclusive, h_in[i]);
+            warp_aggregate = scan_op(warp_aggregate, h_in[i]);
+        }
+
+        warp_aggregates[w] = warp_aggregate;
+    }
+
+}
+
+
+/**
+ * Initialize inclusive-scan problem (and solution)
+ */
+template <
+    typename    T,
+    typename    ScanOpT>
+void Initialize(
+    GenMode     gen_mode,
+    T           *h_in,
+    T           *h_reference,
+    int         logical_warp_items,
+    ScanOpT     scan_op,
+    NullType,
+    T           warp_aggregates[NUM_WARPS])
+{
+    for (int w = 0; w < NUM_WARPS; ++w)
+    {
+        int base_idx = (w * logical_warp_items);
+        int i = base_idx;
+
+        InitValue(gen_mode, h_in[i], i);
+
+        T warp_aggregate    = h_in[i];
+        T inclusive         = h_in[i];
+        h_reference[i]      = inclusive;
+
+        for (i = i + 1; i < base_idx + logical_warp_items; ++i)
+        {
+            InitValue(gen_mode, h_in[i], i);
+            inclusive = scan_op(inclusive, h_in[i]);
+            warp_aggregate = scan_op(warp_aggregate, h_in[i]);
+            h_reference[i] = inclusive;
+        }
+
+        warp_aggregates[w] = warp_aggregate;
+    }
+}
+
+
+/**
+ * Test warp scan
+ */
+template <
+    int             LOGICAL_WARP_THREADS,
+    TestMode        TEST_MODE,
+    typename        T,
+    typename        ScanOpT,
+    typename        InitialValueT>        // NullType implies inclusive-scan, otherwise inclusive scan
+void Test(
+    GenMode         gen_mode,
+    ScanOpT         scan_op,
+    InitialValueT   initial_value)
+{
+    enum {
+        TOTAL_ITEMS = LOGICAL_WARP_THREADS * NUM_WARPS,
+    };
+
+    // Allocate host arrays
+    T *h_in = new T[TOTAL_ITEMS];
+    T *h_reference = new T[TOTAL_ITEMS];
+    T *h_aggregate = new T[TOTAL_ITEMS];
+
+    // Initialize problem
+    T aggregates[NUM_WARPS];
+
+    Initialize(
+        gen_mode,
+        h_in,
+        h_reference,
+        LOGICAL_WARP_THREADS,
+        scan_op,
+        initial_value,
+        aggregates);
+
+    if (g_verbose)
+    {
+        printf("Input: \n");
+        DisplayResults(h_in, TOTAL_ITEMS);
+        printf("\n");
+    }
+
+    for (int w = 0; w < NUM_WARPS; ++w)
+    {
+        for (int i = 0; i < LOGICAL_WARP_THREADS; ++i)
+        {
+            h_aggregate[(w * LOGICAL_WARP_THREADS) + i] = aggregates[w];
+        }
+    }
+
+    // Initialize/clear device arrays
+    T *d_in = NULL;
+    T *d_out = NULL;
+    T *d_aggregate = NULL;
+    clock_t *d_elapsed = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * TOTAL_ITEMS));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(T) * (TOTAL_ITEMS + 1)));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_aggregate, sizeof(T) * TOTAL_ITEMS));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_elapsed, sizeof(clock_t)));
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * TOTAL_ITEMS, cudaMemcpyHostToDevice));
+    CubDebugExit(cudaMemset(d_out, 0, sizeof(T) * (TOTAL_ITEMS + 1)));
+    CubDebugExit(cudaMemset(d_aggregate, 0, sizeof(T) * TOTAL_ITEMS));
+
+    // Run kernel
+    printf("Test-mode %d (%s), gen-mode %d (%s), %s warpscan, %d warp threads, %s (%d bytes) elements:\n",
+        TEST_MODE, typeid(TEST_MODE).name(),
+        gen_mode, typeid(gen_mode).name(),
+        (Equals<InitialValueT, NullType>::VALUE) ? "Inclusive" : "Exclusive",
+        LOGICAL_WARP_THREADS,
+        typeid(T).name(),
+        (int) sizeof(T));
+    fflush(stdout);
+
+    // Run aggregate/prefix kernel
+    WarpScanKernel<LOGICAL_WARP_THREADS, TEST_MODE><<<1, TOTAL_ITEMS>>>(
+        d_in,
+        d_out,
+        d_aggregate,
+        scan_op,
+        initial_value,
+        d_elapsed);
+
+    printf("\tElapsed clocks: ");
+    DisplayDeviceResults(d_elapsed, 1);
+
+    CubDebugExit(cudaPeekAtLastError());
+    CubDebugExit(cudaDeviceSynchronize());
+
+    // Copy out and display results
+    printf("\tScan results: ");
+    int compare = CompareDeviceResults(h_reference, d_out, TOTAL_ITEMS, g_verbose, g_verbose);
+    printf("%s\n", compare ? "FAIL" : "PASS");
+    AssertEquals(0, compare);
+
+    // Copy out and display aggregate
+    if (TEST_MODE == AGGREGATE)
+    {
+        printf("\tScan aggregate: ");
+        compare = CompareDeviceResults(h_aggregate, d_aggregate, TOTAL_ITEMS, g_verbose, g_verbose);
+        printf("%s\n", compare ? "FAIL" : "PASS");
+        AssertEquals(0, compare);
+    }
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (h_reference) delete[] h_reference;
+    if (h_aggregate) delete[] h_aggregate;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+    if (d_aggregate) CubDebugExit(g_allocator.DeviceFree(d_aggregate));
+    if (d_elapsed) CubDebugExit(g_allocator.DeviceFree(d_elapsed));
+}
+
+
+/**
+ * Run battery of tests for different primitive variants
+ */
+template <
+    int         LOGICAL_WARP_THREADS,
+    typename    ScanOpT,
+    typename    T>
+void Test(
+    GenMode     gen_mode,
+    ScanOpT     scan_op,
+    T           initial_value)
+{
+    // Exclusive
+    Test<LOGICAL_WARP_THREADS, BASIC, T>(gen_mode, scan_op, T());
+    Test<LOGICAL_WARP_THREADS, AGGREGATE, T>(gen_mode, scan_op, T());
+
+    // Exclusive (non-specialized, so we can use initial-value)
+    Test<LOGICAL_WARP_THREADS, BASIC, T>(gen_mode, WrapperFunctor<ScanOpT>(scan_op), initial_value);
+    Test<LOGICAL_WARP_THREADS, AGGREGATE, T>(gen_mode, WrapperFunctor<ScanOpT>(scan_op), initial_value);
+
+    // Inclusive
+    Test<LOGICAL_WARP_THREADS, BASIC, T>(gen_mode, scan_op, NullType());
+    Test<LOGICAL_WARP_THREADS, AGGREGATE, T>(gen_mode, scan_op, NullType());
+}
+
+
+/**
+ * Run battery of tests for different data types and scan ops
+ */
+template <int LOGICAL_WARP_THREADS>
+void Test(GenMode gen_mode)
+{
+    // Get device ordinal
+    int device_ordinal;
+    CubDebugExit(cudaGetDevice(&device_ordinal));
+
+    // Get ptx version
+    int ptx_version;
+    CubDebugExit(PtxVersion(ptx_version));
+
+    // primitive
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), (char) 99);
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), (short) 99);
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), (int) 99);
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), (long) 99);
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), (long long) 99);
+    if (gen_mode != RANDOM) {
+        // Only test numerically stable inputs
+        Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), (float) 99);
+        if (ptx_version > 100)
+            Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), (double) 99);
+    }
+
+    // primitive (alternative scan op)
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Max(), (unsigned char) 99);
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Max(), (unsigned short) 99);
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Max(), (unsigned int) 99);
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Max(), (unsigned long long) 99);
+
+    // vec-2
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_uchar2(17, 21));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_ushort2(17, 21));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_uint2(17, 21));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_ulong2(17, 21));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_ulonglong2(17, 21));
+    if (gen_mode != RANDOM) {
+        // Only test numerically stable inputs
+        Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_float2(17, 21));
+        if (ptx_version > 100)
+            Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_double2(17, 21));
+    }
+
+    // vec-4
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_char4(17, 21, 32, 85));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_short4(17, 21, 32, 85));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_int4(17, 21, 32, 85));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_long4(17, 21, 32, 85));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_longlong4(17, 21, 32, 85));
+    if (gen_mode != RANDOM) {
+        // Only test numerically stable inputs
+        Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_float4(17, 21, 32, 85));
+        if (ptx_version > 100)
+            Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), make_double4(17, 21, 32, 85));
+    }
+
+    // complex
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), TestFoo::MakeTestFoo(17, 21, 32, 85));
+    Test<LOGICAL_WARP_THREADS>(gen_mode, Sum(), TestBar(17, 21));
+
+}
+
+
+/**
+ * Run battery of tests for different problem generation options
+ */
+template <int LOGICAL_WARP_THREADS>
+void Test()
+{
+    Test<LOGICAL_WARP_THREADS>(UNIFORM);
+    Test<LOGICAL_WARP_THREADS>(INTEGER_SEED);
+    Test<LOGICAL_WARP_THREADS>(RANDOM);
+}
+
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    g_verbose = args.CheckCmdLineFlag("v");
+    args.GetCmdLineArgument("repeat", g_repeat);
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--repeat=<repetitions of entire test suite>]"
+            "[--v] "
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+#ifdef QUICK_TEST
+
+    // Compile/run quick tests
+    Test<32, AGGREGATE, int>(UNIFORM, Sum(), (int) 0);
+    Test<32, AGGREGATE, float>(UNIFORM, Sum(), (float) 0);
+    Test<32, AGGREGATE, long long>(UNIFORM, Sum(), (long long) 0);
+    Test<32, AGGREGATE, double>(UNIFORM, Sum(), (double) 0);
+
+    typedef KeyValuePair<int, float> T;
+    cub::Sum sum_op;
+    Test<32, AGGREGATE, T>(UNIFORM, ReduceBySegmentOp<cub::Sum>(sum_op), T());
+
+#else
+
+    // Compile/run thorough tests
+    for (int i = 0; i <= g_repeat; ++i)
+    {
+        // Test logical warp sizes
+        Test<32>();
+        Test<16>();
+        Test<9>();
+        Test<2>();
+    }
+
+#endif
+
+    return 0;
+}
+
+
+
+
diff --git a/third_party/cub/tune/.gitignore b/third_party/cub/tune/.gitignore
new file mode 100644
index 0000000..5e56e04
--- /dev/null
+++ b/third_party/cub/tune/.gitignore
@@ -0,0 +1 @@
+/bin
diff --git a/third_party/cub/tune/Makefile b/third_party/cub/tune/Makefile
new file mode 100644
index 0000000..926b340
--- /dev/null
+++ b/third_party/cub/tune/Makefile
@@ -0,0 +1,192 @@
+#/******************************************************************************
+# * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+# * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+# * 
+# * Redistribution and use in source and binary forms, with or without
+# * modification, are permitted provided that the following conditions are met:
+# *	 * Redistributions of source code must retain the above copyright
+# *	   notice, this list of conditions and the following disclaimer.
+# *	 * Redistributions in binary form must reproduce the above copyright
+# *	   notice, this list of conditions and the following disclaimer in the
+# *	   documentation and/or other materials provided with the distribution.
+# *	 * Neither the name of the NVIDIA CORPORATION nor the
+# *	   names of its contributors may be used to endorse or promote products
+# *	   derived from this software without specific prior written permission.
+# * 
+# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+# * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *
+#******************************************************************************/
+ 
+#-------------------------------------------------------------------------------
+# Build script for project
+#-------------------------------------------------------------------------------
+
+NVCC = "$(shell which nvcc)"
+NVCC_VERSION = $(strip $(shell nvcc --version | grep release | sed 's/.*release //' |  sed 's/,.*//'))
+
+# detect OS
+OSUPPER = $(shell uname -s 2>/dev/null | tr [:lower:] [:upper:])
+
+#-------------------------------------------------------------------------------
+# Libs
+#-------------------------------------------------------------------------------
+
+
+#-------------------------------------------------------------------------------
+# Includes
+#-------------------------------------------------------------------------------
+
+INC = -I. -I.. -I../test
+
+#-------------------------------------------------------------------------------
+# Libs
+#-------------------------------------------------------------------------------
+
+LIBS += -lcudart 
+
+#-------------------------------------------------------------------------------
+# Defines
+#-------------------------------------------------------------------------------
+
+DEFINES = 
+
+#-------------------------------------------------------------------------------
+# SM Arch
+#-------------------------------------------------------------------------------
+
+ifdef sm
+	SM_ARCH = $(sm)
+else 
+    SM_ARCH = 200
+endif
+
+# Only one arch per tuning binary
+ifeq (350, $(findstring 350, $(SM_ARCH)))
+    SM_TARGETS = -arch=sm_35
+    SM_ARCH = 350
+endif
+ifeq (300, $(findstring 300, $(SM_ARCH)))
+    SM_TARGETS = -arch=sm_30
+    SM_ARCH = 300
+endif
+ifeq (200, $(findstring 200, $(SM_ARCH)))
+    SM_TARGETS = -arch=sm_20
+    SM_ARCH = 200
+endif
+ifeq (130, $(findstring 130, $(SM_ARCH)))
+    SM_TARGETS = -arch=sm_13
+    SM_ARCH = 130
+endif
+ifeq (110, $(findstring 110, $(SM_ARCH)))
+    SM_TARGETS = -arch=sm_11 
+    SM_ARCH = 110
+endif
+ifeq (100, $(findstring 100, $(SM_ARCH)))
+    SM_TARGETS = -arch=sm_10 
+    SM_ARCH = 100
+endif
+
+
+#-------------------------------------------------------------------------------
+# Compiler Flags
+#-------------------------------------------------------------------------------
+
+NVCCFLAGS = -Xptxas -v -Xcudafe -\#
+
+# Help the compiler/linker work with huge numbers of kernels on Windows
+ifeq (WIN_NT, $(findstring WIN_NT, $(OSUPPER)))
+	NVCCFLAGS += -Xcompiler /bigobj -Xcompiler /Zm500
+endif
+
+# 32/64-bit (32-bit device pointers by default) 
+ifeq ($(force32), 1)
+	CPU_ARCH = -m32
+	CPU_ARCH_SUFFIX = i386
+else
+	CPU_ARCH = -m64
+	CPU_ARCH_SUFFIX = x86_64
+endif
+
+# CUDA ABI enable/disable (enabled by default) 
+ifneq ($(abi), 0)
+	ABI_SUFFIX = abi
+else 
+	NVCCFLAGS += -Xptxas -abi=no
+	ABI_SUFFIX = noabi
+endif
+
+# NVVM/Open64 middle-end compiler (nvvm by default)
+ifeq ($(open64), 1)
+	NVCCFLAGS += -open64
+	PTX_SUFFIX = open64
+else 
+	PTX_SUFFIX = nvvm
+endif
+
+# Verbose toolchain output from nvcc
+ifeq ($(verbose), 1)
+	NVCCFLAGS += -v
+endif
+
+# Keep intermediate compilation artifacts
+ifeq ($(keep), 1)
+	NVCCFLAGS += -keep
+endif
+
+# Data type size to compile a schmoo binary for
+ifdef tunesize
+    TUNE_SIZE = $(tunesize)
+else 
+	TUNE_SIZE = 4
+endif
+
+
+SUFFIX = $(TUNE_SIZE)B_sm$(SM_ARCH)_$(PTX_SUFFIX)_$(NVCC_VERSION)_$(ABI_SUFFIX)_$(CPU_ARCH_SUFFIX)
+
+#-------------------------------------------------------------------------------
+# Dependency Lists
+#-------------------------------------------------------------------------------
+
+rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
+
+DEPS =	 ./Makefile \
+		../test/test_util.h \
+		$(call rwildcard,../cub/,*.cuh)
+
+
+#-------------------------------------------------------------------------------
+# make default
+#-------------------------------------------------------------------------------
+
+default:
+
+
+#-------------------------------------------------------------------------------
+# make clean
+#-------------------------------------------------------------------------------
+
+clean :
+	rm -f bin/*$(CPU_ARCH_SUFFIX)* 
+	rm -f *.i* *.cubin *.cu.c *.cudafe* *.fatbin.c *.ptx *.hash *.cu.cpp *.o
+
+
+
+#-------------------------------------------------------------------------------
+# make tune_device_reduce
+#-------------------------------------------------------------------------------
+
+tune_device_reduce: bin/tune_device_reduce_$(SUFFIX)
+
+bin/tune_device_reduce_$(SUFFIX) : tune_device_reduce.cu $(DEPS)
+	mkdir -p bin
+	$(NVCC) $(DEFINES) $(SM_TARGETS) -o bin/tune_device_reduce_$(SUFFIX) tune_device_reduce.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBS) -O3 -DTUNE_ARCH=$(SM_ARCH) -DTUNE_SIZE=$(TUNE_SIZE)
+
diff --git a/third_party/cub/tune/tune_device_reduce.cu b/third_party/cub/tune/tune_device_reduce.cu
new file mode 100644
index 0000000..ec0cf57
--- /dev/null
+++ b/third_party/cub/tune/tune_device_reduce.cu
@@ -0,0 +1,763 @@
+/******************************************************************************
+ * Copyright (c) 2011, Duane Merrill.  All rights reserved.
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ ******************************************************************************/
+
+/******************************************************************************
+ * Evaluates different tuning configurations of DeviceReduce.
+ *
+ * The best way to use this program:
+ * (1) Find the best all-around single-block tune for a given arch.
+ *     For example, 100 samples [1 ..512], 100 timing iterations per config per sample:
+ *         ./bin/tune_device_reduce_sm200_nvvm_5.0_abi_i386 --i=100 --s=100 --n=512 --single --device=0
+ * (2) Update the single tune in device_reduce.cuh
+ * (3) Find the best all-around multi-block tune for a given arch.
+ *     For example, 100 samples [single-block tile-size ..  50,331,648], 100 timing iterations per config per sample:
+ *         ./bin/tune_device_reduce_sm200_nvvm_5.0_abi_i386 --i=100 --s=100 --device=0
+ * (4) Update the multi-block tune in device_reduce.cuh
+ *
+ ******************************************************************************/
+
+// Ensure printing of CUDA runtime errors to console
+#define CUB_STDERR
+
+#include <vector>
+#include <algorithm>
+#include <stdio.h>
+#include <cub/cub.cuh>
+#include "../test/test_util.h"
+
+using namespace cub;
+using namespace std;
+
+
+//---------------------------------------------------------------------
+// Globals, constants and typedefs
+//---------------------------------------------------------------------
+
+#ifndef TUNE_ARCH
+#define TUNE_ARCH 100
+#endif
+
+int     g_max_items         = 48 * 1024 * 1024;
+int     g_samples           = 100;
+int     g_timing_iterations        = 2;
+bool    g_verbose           = false;
+bool    g_single            = false;
+bool    g_verify            = true;
+CachingDeviceAllocator  g_allocator;
+
+
+//---------------------------------------------------------------------
+// Host utility subroutines
+//---------------------------------------------------------------------
+
+/**
+ * Initialize problem
+ */
+template <typename T>
+void Initialize(
+    GenMode         gen_mode,
+    T               *h_in,
+    int             num_items)
+{
+    for (int i = 0; i < num_items; ++i)
+    {
+        InitValue(gen_mode, h_in[i], i);
+    }
+}
+
+/**
+ * Sequential reduction
+ */
+template <typename T, typename ReductionOp>
+T Reduce(
+    T               *h_in,
+    ReductionOp     reduction_op,
+    int             num_items)
+{
+    T retval = h_in[0];
+    for (int i = 1; i < num_items; ++i)
+        retval = reduction_op(retval, h_in[i]);
+
+    return retval;
+}
+
+
+
+//---------------------------------------------------------------------
+// Full tile test generation
+//---------------------------------------------------------------------
+
+
+
+/**
+ * Wrapper structure for generating and running different tuning configurations
+ */
+template <
+    typename T,
+    typename OffsetT,
+    typename ReductionOp>
+struct Schmoo
+{
+    //---------------------------------------------------------------------
+    // Types
+    //---------------------------------------------------------------------
+
+    /// Pairing of kernel function pointer and corresponding dispatch params
+    template <typename KernelPtr>
+    struct DispatchTuple
+    {
+        KernelPtr                           kernel_ptr;
+        DeviceReduce::KernelDispachParams   params;
+
+        float                               avg_throughput;
+        float                               best_avg_throughput;
+        OffsetT                              best_size;
+        float                               hmean_speedup;
+
+
+        DispatchTuple() :
+            kernel_ptr(0),
+            params(DeviceReduce::KernelDispachParams()),
+            avg_throughput(0.0),
+            best_avg_throughput(0.0),
+            hmean_speedup(0.0),
+            best_size(0)
+        {}
+    };
+
+    /**
+     * Comparison operator for DispatchTuple.avg_throughput
+     */
+    template <typename Tuple>
+    static bool MinSpeedup(const Tuple &a, const Tuple &b)
+    {
+        float delta = a.hmean_speedup - b.hmean_speedup;
+
+        return ((delta < 0.02) && (delta > -0.02)) ?
+            (a.best_avg_throughput < b.best_avg_throughput) :       // Negligible average performance differences: defer to best performance
+            (a.hmean_speedup < b.hmean_speedup);
+    }
+
+
+
+    /// Multi-block reduction kernel type and dispatch tuple type
+    typedef void (*MultiBlockDeviceReduceKernelPtr)(T*, T*, OffsetT, GridEvenShare<OffsetT>, GridQueue<OffsetT>, ReductionOp);
+    typedef DispatchTuple<MultiBlockDeviceReduceKernelPtr> MultiDispatchTuple;
+
+    /// Single-block reduction kernel type and dispatch tuple type
+    typedef void (*SingleBlockDeviceReduceKernelPtr)(T*, T*, OffsetT, ReductionOp);
+    typedef DispatchTuple<SingleBlockDeviceReduceKernelPtr> SingleDispatchTuple;
+
+
+    //---------------------------------------------------------------------
+    // Fields
+    //---------------------------------------------------------------------
+
+    vector<MultiDispatchTuple> multi_kernels;       // List of generated multi-block kernels
+    vector<SingleDispatchTuple> single_kernels;     // List of generated single-block kernels
+
+
+    //---------------------------------------------------------------------
+    // Kernel enumeration methods
+    //---------------------------------------------------------------------
+
+    /**
+     * Must have smem that fits in the SM
+     * Must have vector load length that divides items per thread
+     */
+    template <typename TilesReducePolicy, typename ReductionOp>
+    struct SmemSize
+    {
+        enum
+        {
+            BYTES = sizeof(typename BlockReduceTiles<TilesReducePolicy, T*, OffsetT, ReductionOp>::TempStorage),
+            IS_OK = ((BYTES < ArchProps<TUNE_ARCH>::SMEM_BYTES) &&
+                     (TilesReducePolicy::ITEMS_PER_THREAD % TilesReducePolicy::VECTOR_LOAD_LENGTH == 0))
+        };
+    };
+
+
+    /**
+     * Specialization that allows kernel generation with the specified TilesReducePolicy
+     */
+    template <
+        typename    TilesReducePolicy,
+        bool        IsOk = SmemSize<TilesReducePolicy, ReductionOp>::IS_OK>
+    struct Ok
+    {
+        /// Enumerate multi-block kernel and add to the list
+        template <typename KernelsVector>
+        static void GenerateMulti(
+            KernelsVector &multi_kernels,
+            int subscription_factor)
+        {
+            MultiDispatchTuple tuple;
+            tuple.params.template Init<TilesReducePolicy>(subscription_factor);
+            tuple.kernel_ptr = ReducePrivatizedKernel<TilesReducePolicy, T*, T*, OffsetT, ReductionOp>;
+            multi_kernels.push_back(tuple);
+        }
+
+
+        /// Enumerate single-block kernel and add to the list
+        template <typename KernelsVector>
+        static void GenerateSingle(KernelsVector &single_kernels)
+        {
+            SingleDispatchTuple tuple;
+            tuple.params.template Init<TilesReducePolicy>();
+            tuple.kernel_ptr = ReduceSingleKernel<TilesReducePolicy, T*, T*, OffsetT, ReductionOp>;
+            single_kernels.push_back(tuple);
+        }
+    };
+
+    /**
+     * Specialization that rejects kernel generation with the specified TilesReducePolicy
+     */
+    template <typename TilesReducePolicy>
+    struct Ok<TilesReducePolicy, false>
+    {
+        template <typename KernelsVector>
+        static void GenerateMulti(KernelsVector &multi_kernels, int subscription_factor) {}
+
+        template <typename KernelsVector>
+        static void GenerateSingle(KernelsVector &single_kernels) {}
+    };
+
+
+    /// Enumerate block-scheduling variations
+    template <
+        int                     BLOCK_THREADS,
+        int                     ITEMS_PER_THREAD,
+        int                     VECTOR_LOAD_LENGTH,
+        BlockReduceAlgorithm    BLOCK_ALGORITHM,
+        CacheLoadModifier      LOAD_MODIFIER>
+    void Enumerate()
+    {
+        // Multi-block kernels
+        Ok<BlockReduceTilesPolicy<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_ALGORITHM, LOAD_MODIFIER, GRID_MAPPING_RAKE> >::GenerateMulti(multi_kernels, 1);
+        Ok<BlockReduceTilesPolicy<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_ALGORITHM, LOAD_MODIFIER, GRID_MAPPING_RAKE> >::GenerateMulti(multi_kernels, 2);
+        Ok<BlockReduceTilesPolicy<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_ALGORITHM, LOAD_MODIFIER, GRID_MAPPING_RAKE> >::GenerateMulti(multi_kernels, 4);
+        Ok<BlockReduceTilesPolicy<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_ALGORITHM, LOAD_MODIFIER, GRID_MAPPING_RAKE> >::GenerateMulti(multi_kernels, 8);
+#if TUNE_ARCH >= 200
+        Ok<BlockReduceTilesPolicy<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_ALGORITHM, LOAD_MODIFIER, GRID_MAPPING_DYNAMIC> >::GenerateMulti(multi_kernels, 1);
+#endif
+
+        // Single-block kernels
+        Ok<BlockReduceTilesPolicy<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_ALGORITHM, LOAD_MODIFIER, GRID_MAPPING_RAKE> >::GenerateSingle(single_kernels);
+    }
+
+
+    /// Enumerate load modifier variations
+    template <
+        int                     BLOCK_THREADS,
+        int                     ITEMS_PER_THREAD,
+        int                     VECTOR_LOAD_LENGTH,
+        BlockReduceAlgorithm    BLOCK_ALGORITHM>
+    void Enumerate()
+    {
+        Enumerate<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_ALGORITHM, LOAD_DEFAULT>();
+#if TUNE_ARCH >= 350
+        Enumerate<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_ALGORITHM, LOAD_LDG>();
+#endif
+    }
+
+
+    /// Enumerate block algorithms
+    template <
+        int BLOCK_THREADS,
+        int ITEMS_PER_THREAD,
+        int VECTOR_LOAD_LENGTH>
+    void Enumerate()
+    {
+        Enumerate<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_REDUCE_RAKING>();
+        Enumerate<BLOCK_THREADS, ITEMS_PER_THREAD, VECTOR_LOAD_LENGTH, BLOCK_REDUCE_WARP_REDUCTIONS>();
+    }
+
+
+    /// Enumerate vectorization variations
+    template <
+        int BLOCK_THREADS,
+        int ITEMS_PER_THREAD>
+    void Enumerate()
+    {
+        Enumerate<BLOCK_THREADS, ITEMS_PER_THREAD, 1>();
+        Enumerate<BLOCK_THREADS, ITEMS_PER_THREAD, 2>();
+        Enumerate<BLOCK_THREADS, ITEMS_PER_THREAD, 4>();
+    }
+
+
+    /// Enumerate thread-granularity variations
+    template <int BLOCK_THREADS>
+    void Enumerate()
+    {
+        Enumerate<BLOCK_THREADS, 7>();
+        Enumerate<BLOCK_THREADS, 8>();
+        Enumerate<BLOCK_THREADS, 9>();
+
+        Enumerate<BLOCK_THREADS, 11>();
+        Enumerate<BLOCK_THREADS, 12>();
+        Enumerate<BLOCK_THREADS, 13>();
+
+        Enumerate<BLOCK_THREADS, 15>();
+        Enumerate<BLOCK_THREADS, 16>();
+        Enumerate<BLOCK_THREADS, 17>();
+
+        Enumerate<BLOCK_THREADS, 19>();
+        Enumerate<BLOCK_THREADS, 20>();
+        Enumerate<BLOCK_THREADS, 21>();
+
+        Enumerate<BLOCK_THREADS, 23>();
+        Enumerate<BLOCK_THREADS, 24>();
+        Enumerate<BLOCK_THREADS, 25>();
+    }
+
+
+    /// Enumerate block size variations
+    void Enumerate()
+    {
+        printf("\nEnumerating kernels\n"); fflush(stdout);
+
+        Enumerate<32>();
+        Enumerate<64>();
+        Enumerate<96>();
+        Enumerate<128>();
+        Enumerate<160>();
+        Enumerate<192>();
+        Enumerate<256>();
+        Enumerate<512>();
+    }
+
+
+    //---------------------------------------------------------------------
+    // Test methods
+    //---------------------------------------------------------------------
+
+    /**
+     * Test a configuration
+     */
+    void TestConfiguration(
+        MultiDispatchTuple      &multi_dispatch,
+        SingleDispatchTuple     &single_dispatch,
+        T*                      d_in,
+        T*                      d_out,
+        T*                      h_reference,
+        OffsetT                  num_items,
+        ReductionOp             reduction_op)
+    {
+        // Clear output
+        if (g_verify) CubDebugExit(cudaMemset(d_out, 0, sizeof(T)));
+
+        // Allocate temporary storage
+        void            *d_temp_storage = NULL;
+        size_t          temp_storage_bytes = 0;
+        CubDebugExit(DeviceReduce::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            multi_dispatch.kernel_ptr,
+            single_dispatch.kernel_ptr,
+            FillAndResetDrainKernel<OffsetT>,
+            multi_dispatch.params,
+            single_dispatch.params,
+            d_in,
+            d_out,
+            num_items,
+            reduction_op));
+        CubDebugExit(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes));
+
+        // Warmup/correctness iteration
+        CubDebugExit(DeviceReduce::Dispatch(
+            d_temp_storage,
+            temp_storage_bytes,
+            multi_dispatch.kernel_ptr,
+            single_dispatch.kernel_ptr,
+            FillAndResetDrainKernel<OffsetT>,
+            multi_dispatch.params,
+            single_dispatch.params,
+            d_in,
+            d_out,
+            num_items,
+            reduction_op));
+
+        if (g_verify) CubDebugExit(cudaDeviceSynchronize());
+
+        // Copy out and display results
+        int compare = (g_verify) ?
+            CompareDeviceResults(h_reference, d_out, 1, true, false) :
+            0;
+
+        // Performance
+        GpuTimer gpu_timer;
+        float elapsed_millis = 0.0;
+        for (int i = 0; i < g_timing_iterations; i++)
+        {
+            gpu_timer.Start();
+
+            CubDebugExit(DeviceReduce::Dispatch(
+                d_temp_storage,
+                temp_storage_bytes,
+                multi_dispatch.kernel_ptr,
+                single_dispatch.kernel_ptr,
+                FillAndResetDrainKernel<OffsetT>,
+                multi_dispatch.params,
+                single_dispatch.params,
+                d_in,
+                d_out,
+                num_items,
+                reduction_op));
+
+            gpu_timer.Stop();
+            elapsed_millis += gpu_timer.ElapsedMillis();
+        }
+
+        // Mooch
+        CubDebugExit(cudaDeviceSynchronize());
+
+        float avg_elapsed = elapsed_millis / g_timing_iterations;
+        float avg_throughput = float(num_items) / avg_elapsed / 1000.0 / 1000.0;
+        float avg_bandwidth = avg_throughput * sizeof(T);
+
+        multi_dispatch.avg_throughput = CUB_MAX(avg_throughput, multi_dispatch.avg_throughput);
+        if (avg_throughput > multi_dispatch.best_avg_throughput)
+        {
+            multi_dispatch.best_avg_throughput = avg_throughput;
+            multi_dispatch.best_size = num_items;
+        }
+
+        single_dispatch.avg_throughput = CUB_MAX(avg_throughput, single_dispatch.avg_throughput);
+        if (avg_throughput > single_dispatch.best_avg_throughput)
+        {
+            single_dispatch.best_avg_throughput = avg_throughput;
+            single_dispatch.best_size = num_items;
+        }
+
+        if (g_verbose)
+        {
+            printf("\t%.2f GB/s, multi_dispatch( ", avg_bandwidth);
+            multi_dispatch.params.Print();
+            printf(" ), single_dispatch( ");
+            single_dispatch.params.Print();
+            printf(" )\n");
+            fflush(stdout);
+        }
+
+        AssertEquals(0, compare);
+
+        // Cleanup temporaries
+        if (d_temp_storage) CubDebugExit(g_allocator.DeviceFree(d_temp_storage));
+    }
+
+
+    /**
+     * Evaluate multi-block configurations
+     */
+    void TestMulti(
+        T*                      h_in,
+        T*                      d_in,
+        T*                      d_out,
+        ReductionOp             reduction_op)
+    {
+        // Simple single kernel tuple for use with multi kernel sweep
+        typedef typename DeviceReduce::TunedPolicies<T, OffsetT, TUNE_ARCH>::SinglePolicy SimpleSinglePolicy;
+        SingleDispatchTuple simple_single_tuple;
+        simple_single_tuple.params.template Init<SimpleSinglePolicy>();
+        simple_single_tuple.kernel_ptr = ReduceSingleKernel<SimpleSinglePolicy, T*, T*, OffsetT, ReductionOp>;
+
+        double max_exponent      = log2(double(g_max_items));
+        double min_exponent      = log2(double(simple_single_tuple.params.tile_size));
+        unsigned int max_int     = (unsigned int) -1;
+
+        for (int sample = 0; sample < g_samples; ++sample)
+        {
+            printf("\nMulti-block sample %d, ", sample);
+
+            int num_items;
+            if (sample == 0)
+            {
+                // First sample: use max items
+                num_items = g_max_items;
+                printf("num_items: %d", num_items); fflush(stdout);
+            }
+            else
+            {
+                // Sample a problem size from [2^g_min_exponent, g_max_items].  First 2/3 of the samples are log-distributed, the other 1/3 are uniformly-distributed.
+                unsigned int bits;
+                RandomBits(bits);
+                double scale = double(bits) / max_int;
+
+                if (sample < g_samples / 2)
+                {
+                    // log bias
+                    double exponent = ((max_exponent - min_exponent) * scale) + min_exponent;
+                    num_items = pow(2.0, exponent);
+                    num_items = CUB_MIN(num_items, g_max_items);
+                    printf("num_items: %d (2^%.2f)", num_items, exponent); fflush(stdout);
+                }
+                else
+                {
+                    // uniform bias
+                    num_items = CUB_MAX(pow(2.0, min_exponent), scale * g_max_items);
+                    num_items = CUB_MIN(num_items, g_max_items);
+                    printf("num_items: %d (%.2f * %d)", num_items, scale, g_max_items); fflush(stdout);
+                }
+            }
+            if (g_verbose)
+                printf("\n");
+            else
+                printf(", ");
+
+            // Compute reference
+            T h_reference = Reduce(h_in, reduction_op, num_items);
+
+            // Run test on each multi-kernel configuration
+            float best_avg_throughput = 0.0;
+            for (int j = 0; j < multi_kernels.size(); ++j)
+            {
+                multi_kernels[j].avg_throughput = 0.0;
+
+                TestConfiguration(multi_kernels[j], simple_single_tuple, d_in, d_out, &h_reference, num_items, reduction_op);
+
+                best_avg_throughput = CUB_MAX(best_avg_throughput, multi_kernels[j].avg_throughput);
+            }
+
+            // Print best throughput for this problem size
+            printf("Best: %.2fe9 items/s (%.2f GB/s)\n", best_avg_throughput, best_avg_throughput * sizeof(T));
+
+            // Accumulate speedup (inverse for harmonic mean)
+            for (int j = 0; j < multi_kernels.size(); ++j)
+                multi_kernels[j].hmean_speedup += best_avg_throughput / multi_kernels[j].avg_throughput;
+        }
+
+        // Find max overall throughput and compute hmean speedups
+        float overall_max_throughput = 0.0;
+        for (int j = 0; j < multi_kernels.size(); ++j)
+        {
+            overall_max_throughput = CUB_MAX(overall_max_throughput, multi_kernels[j].best_avg_throughput);
+            multi_kernels[j].hmean_speedup = float(g_samples) / multi_kernels[j].hmean_speedup;
+        }
+
+        // Sort by cumulative speedup
+        sort(multi_kernels.begin(), multi_kernels.end(), MinSpeedup<MultiDispatchTuple>);
+
+        // Print ranked multi configurations
+        printf("\nRanked multi_kernels:\n");
+        for (int j = 0; j < multi_kernels.size(); ++j)
+        {
+            printf("\t (%d) params( ", multi_kernels.size() - j);
+            multi_kernels[j].params.Print();
+            printf(" ) hmean speedup: %.3f, best throughput %.2f @ %d elements (%.2f GB/s, %.2f%%)\n",
+                multi_kernels[j].hmean_speedup,
+                multi_kernels[j].best_avg_throughput,
+                (int) multi_kernels[j].best_size,
+                multi_kernels[j].best_avg_throughput * sizeof(T),
+                multi_kernels[j].best_avg_throughput / overall_max_throughput);
+        }
+
+        printf("\nMax multi-block throughput %.2f (%.2f GB/s)\n", overall_max_throughput, overall_max_throughput * sizeof(T));
+    }
+
+
+    /**
+     * Evaluate single-block configurations
+     */
+    void TestSingle(
+        T*                      h_in,
+        T*                      d_in,
+        T*                      d_out,
+        ReductionOp             reduction_op)
+     {
+        // Construct a NULL-ptr multi-kernel tuple that forces a single-kernel pass
+        MultiDispatchTuple multi_tuple;
+
+        double max_exponent     = log2(double(g_max_items));
+        unsigned int max_int    = (unsigned int) -1;
+
+        for (int sample = 0; sample < g_samples; ++sample)
+        {
+            printf("\nSingle-block sample %d, ", sample);
+
+            int num_items;
+            if (sample == 0)
+            {
+                // First sample: use max items
+                num_items = g_max_items;
+                printf("num_items: %d", num_items); fflush(stdout);
+            }
+            else
+            {
+                // Sample a problem size from [2, g_max_items], log-distributed
+                unsigned int bits;
+                RandomBits(bits);
+                double scale = double(bits) / max_int;
+                double exponent = ((max_exponent - 1) * scale) + 1;
+                num_items = pow(2.0, exponent);
+                printf("num_items: %d (2^%.2f)", num_items, exponent); fflush(stdout);
+            }
+
+            if (g_verbose)
+                printf("\n");
+            else
+                printf(", ");
+
+            // Compute reference
+            T h_reference = Reduce(h_in, reduction_op, num_items);
+
+            // Run test on each single-kernel configuration (pick first multi-config to use, which shouldn't be
+            float best_avg_throughput = 0.0;
+            for (int j = 0; j < single_kernels.size(); ++j)
+            {
+                single_kernels[j].avg_throughput = 0.0;
+
+                TestConfiguration(multi_tuple, single_kernels[j], d_in, d_out, &h_reference, num_items, reduction_op);
+
+                best_avg_throughput = CUB_MAX(best_avg_throughput, single_kernels[j].avg_throughput);
+            }
+
+            // Print best throughput for this problem size
+            printf("Best: %.2fe9 items/s (%.2f GB/s)\n", best_avg_throughput, best_avg_throughput * sizeof(T));
+
+            // Accumulate speedup (inverse for harmonic mean)
+            for (int j = 0; j < single_kernels.size(); ++j)
+                single_kernels[j].hmean_speedup += best_avg_throughput / single_kernels[j].avg_throughput;
+        }
+
+        // Find max overall throughput and compute hmean speedups
+        float overall_max_throughput = 0.0;
+        for (int j = 0; j < single_kernels.size(); ++j)
+        {
+            overall_max_throughput = CUB_MAX(overall_max_throughput, single_kernels[j].best_avg_throughput);
+            single_kernels[j].hmean_speedup = float(g_samples) / single_kernels[j].hmean_speedup;
+        }
+
+        // Sort by cumulative speedup
+        sort(single_kernels.begin(), single_kernels.end(), MinSpeedup<SingleDispatchTuple>);
+
+        // Print ranked single configurations
+        printf("\nRanked single_kernels:\n");
+        for (int j = 0; j < single_kernels.size(); ++j)
+        {
+            printf("\t (%d) params( ", single_kernels.size() - j);
+            single_kernels[j].params.Print();
+            printf(" ) hmean speedup: %.3f, best throughput %.2f @ %d elements (%.2f GB/s, %.2f%%)\n",
+                single_kernels[j].hmean_speedup,
+                single_kernels[j].best_avg_throughput,
+                (int) single_kernels[j].best_size,
+                single_kernels[j].best_avg_throughput * sizeof(T),
+                single_kernels[j].best_avg_throughput / overall_max_throughput);
+        }
+
+        printf("\nMax single-block throughput %.2f (%.2f GB/s)\n", overall_max_throughput, overall_max_throughput * sizeof(T));
+    }
+
+};
+
+
+
+//---------------------------------------------------------------------
+// Main
+//---------------------------------------------------------------------
+
+/**
+ * Main
+ */
+int main(int argc, char** argv)
+{
+    // Initialize command line
+    CommandLineArgs args(argc, argv);
+    args.GetCmdLineArgument("n", g_max_items);
+    args.GetCmdLineArgument("s", g_samples);
+    args.GetCmdLineArgument("i", g_timing_iterations);
+    g_verbose = args.CheckCmdLineFlag("v");
+    g_single = args.CheckCmdLineFlag("single");
+    g_verify = !args.CheckCmdLineFlag("noverify");
+
+    // Print usage
+    if (args.CheckCmdLineFlag("help"))
+    {
+        printf("%s "
+            "[--device=<device-id>] "
+            "[--n=<max items>]"
+            "[--s=<samples>]"
+            "[--i=<timing iterations>]"
+            "[--single]"
+            "[--v]"
+            "[--noverify]"
+            "\n", argv[0]);
+        exit(0);
+    }
+
+    // Initialize device
+    CubDebugExit(args.DeviceInit());
+
+#if (TUNE_SIZE == 1)
+    typedef unsigned char T;
+#elif (TUNE_SIZE == 2)
+    typedef unsigned short T;
+#elif (TUNE_SIZE == 4)
+    typedef unsigned int T;
+#elif (TUNE_SIZE == 8)
+    typedef unsigned long long T;
+#else
+    // Default
+    typedef unsigned int T;
+#endif
+
+    typedef unsigned int OffsetT;
+    Sum reduction_op;
+
+    // Enumerate kernels
+    Schmoo<T, OffsetT, Sum > schmoo;
+    schmoo.Enumerate();
+
+    // Allocate host arrays
+    T *h_in = new T[g_max_items];
+
+    // Initialize problem
+    Initialize(UNIFORM, h_in, g_max_items);
+
+    // Initialize device arrays
+    T *d_in = NULL;
+    T *d_out = NULL;
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_in, sizeof(T) * g_max_items));
+    CubDebugExit(g_allocator.DeviceAllocate((void**)&d_out, sizeof(T) * 1));
+    CubDebugExit(cudaMemcpy(d_in, h_in, sizeof(T) * g_max_items, cudaMemcpyHostToDevice));
+
+    // Test kernels
+    if (g_single)
+        schmoo.TestSingle(h_in, d_in, d_out, reduction_op);
+    else
+        schmoo.TestMulti(h_in, d_in, d_out, reduction_op);
+
+    // Cleanup
+    if (h_in) delete[] h_in;
+    if (d_in) CubDebugExit(g_allocator.DeviceFree(d_in));
+    if (d_out) CubDebugExit(g_allocator.DeviceFree(d_out));
+
+    return 0;
+}
+
+
+
-- 
GitLab